Commit e3b33142 authored by Delaigue Olivier's avatar Delaigue Olivier
Browse files

Merge branch '59-add-comparative-tests-with-master-branch-on-examples' into 'dev'

Resolve "Add comparative tests with master branch on examples"

Closes #59

See merge request !13
parents 5f4d38f1 b05d5530
Pipeline #16693 passed with stages
in 42 minutes and 38 seconds
......@@ -2,3 +2,5 @@
^\.Rproj\.user$
^\.Rprofile$
^packrat/
^tests/tmp/
^\.regressionignore$
......@@ -3,12 +3,13 @@ packrat/lib*/
# Compiled files
/src/*.o
/src/*.so
/src/*.dll
/src-*
# Test temporary files
/tests/tmp/
/tests/testthat/*.pdf
*.pdf
######################################################################################################
### Generic .gitignore for R (source: https://github.com/github/gitignore/blob/master/R.gitignore) ###
......
stages:
- update_packages
- build
- regression
- tests
default:
......@@ -30,6 +31,20 @@ default:
untracked: true
expire_in: 1 week
.regression:
stage: regression
only:
refs:
- schedules
script:
- Rscript -e 'source("tests/testthat/store_examples.R"); StoreRefExampleResults("airGR");'
- R CMD INSTALL .
- Rscript -e 'source("tests/testthat/store_examples.R"); StoreTestExampleResults("airGR");'
artifacts:
paths:
- tests/tmp/
expire_in: 1 week
.check_not_cran:
stage: tests
variables:
......@@ -52,6 +67,21 @@ build_patched:
R_VERSION: "patched"
extends: .build
regression_patched:
variables:
R_VERSION: "patched"
extends: .regression
regression_devel:
variables:
R_VERSION: "devel"
extends: .regression
regression_oldrel:
variables:
R_VERSION: "oldrel"
extends: .regression
check_not_cran_patched:
variables:
R_VERSION: "patched"
......@@ -125,3 +155,4 @@ check_as_cran_oldrel:
variables:
R_VERSION: "oldrel"
extends: .check_as_cran
# .test-regression.ignore contains the list of topic/variables produces by
# documentation examples that should be ignore in the regression test
# The format of this file is: 5 lines of comments followed by one line by
# ignored variable : [Topic]<SPACE>[Variable].
# Example for ignoring OutputsModel variable produced by example("RunModel_GR2M"): RunModel_GR2M OutputsModel
RunModel_GR2M OutputsModel
RunModel_GR2M RunOptions
StoreRefExampleResults <- function(package, ...) {
install.packages(package, repos = "http://cran.r-project.org")
StoreExampleResults(package = package, path = "tests/tmp/ref", ...)
}
StoreTestExampleResults <- function(package, ...) {
StoreExampleResults(
package = package,
path = file.path("tests/tmp", Sys.getenv("R_VERSION", "test")),
...
)
}
#' Run examples of a package and store the output variables in RDS files for further testing.
#'
#' @param package Name of the package from which examples are tested.
#' @param path Path where to record the files.
#' @param run.dontrun See \code{\link{example}}.
#' @param run.donttest See \code{\link{example}}.
#'
#' @return
#' @export
#'
#' @examples
StoreExampleResults <- function(package, path, run.dontrun = FALSE, run.donttest = TRUE) {
# Install and load stable version of the package
library(package, character.only = TRUE)
# Get the list of documentation pages
rd <- unique(readRDS(system.file("help", "aliases.rds", package = package)))
dir.create(path, showWarnings = FALSE)
lapply(
rd,
StoreTopicResults,
package, path, run.dontrun = run.dontrun, run.donttest = run.donttest
)
}
StoreTopicResults <- function(topic, package, path, run.dontrun = TRUE, run.donttest = TRUE) {
cat("*******************************\n")
cat("*", topic, "\n")
cat("*******************************\n")
par(ask = FALSE) #https://stackoverflow.com/questions/34756905/how-to-turn-off-the-hit-return-to-see-next-plot-prompt-plot3d
varBefore <- c()
varBefore <- ls(envir = globalenv())
example(
topic, package = package, character.only = TRUE, echo = FALSE, ask = FALSE, local = FALSE, setRNG = TRUE,
run.dontrun = run.dontrun, run.donttest = run.donttest
)
dev.off()
varAfter <- ls(envir = globalenv())
varToSave <- setdiff(varAfter, varBefore)
if (length(varToSave) > 0) {
path <- file.path(path, topic)
dir.create(path, showWarnings = FALSE, recursive = TRUE)
lapply(varToSave, function(x) {
saveRDS(get(x), file = file.path(path, paste0(x, ".rds")))
})
}
rm(list = varToSave, envir = globalenv())
}
context("Compare example outputs with CRAN")
CompareWithRef <- function(refVarFile, testDir, regIgnore) {
v <- data.frame(topic = basename(dirname(refVarFile)),
var = gsub("\\.rds$", "", basename(refVarFile)))
if (is.null(regIgnore) || all(apply(regIgnore, 1, function(x) !all(x == v)))) {
test_that(paste("Compare", v$topic, v$var), {
skip_on_cran()
testVarFile <- paste0(
file.path("../tmp", Sys.getenv("R_VERSION", "test"), v$topic, v$var),
".rds"
)
expect_true(file.exists(testVarFile))
if (file.exists(testVarFile)) {
testVar <- readRDS(testVarFile)
refVar <- readRDS(refVarFile)
expect_equivalent(testVar, refVar)
}
})
}
}
if (dir.exists("../tmp/ref") & dir.exists("../tmp/test")) {
refVarFiles <- list.files("../tmp/ref", recursive = TRUE, full.names = TRUE)
regIgnoreFile <- "../../.regressionignore2"
if (file.exists(regIgnoreFile)) {
regIgnore <- read.table(file = regIgnoreFile,
sep = " ", header = FALSE, skip = 5,
col.names = c("topic", "var"),
stringsAsFactors = FALSE)
} else {
regIgnore <- NULL
}
lapply(X = refVarFiles, CompareWithRef, testDir = "../tmp/test", regIgnore = regIgnore)
} else {
warning("Regression tests compared to released version needs that you run the following instructions first:\n",
"Rscript -e 'source(\"tests/testthat/store_examples.R\"); StoreRefExampleResults(\"airGR\");'\n",
"R CMD INSTALL .\n",
"Rscript -e 'source(\"tests/testthat/store_examples.R\"); StoreTestExampleResults(\"airGR\");'\n")
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment