diff --git a/.Rbuildignore b/.Rbuildignore index 02bca46bffb309865243a7c0ee8e0bd876f74ed7..7cc5eb755f01fb77fc981ab3e94f2416f465bbe9 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -2,3 +2,5 @@ ^\.Rproj\.user$ ^\.Rprofile$ ^packrat/ +^tests/tmp/ +^\.regressionignore$ diff --git a/.gitignore b/.gitignore index 188eac083f4542383a37a30f2515c869bc20cc09..bbbbef6ecb0ca44d33804cdcc6837939839ea88b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,12 +3,13 @@ packrat/lib*/ # Compiled files /src/*.o +/src/*.so /src/*.dll /src-* # Test temporary files /tests/tmp/ -/tests/testthat/*.pdf +*.pdf ###################################################################################################### ### Generic .gitignore for R (source: https://github.com/github/gitignore/blob/master/R.gitignore) ### diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 082c3a975dd7e0a3ba29f8158c60756313ad9eca..4b1828ba0745a86928138167e60cd1cbb30f4681 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,7 @@ stages: - update_packages - build + - regression - tests default: @@ -30,6 +31,20 @@ default: untracked: true expire_in: 1 week +.regression: + stage: regression + only: + refs: + - schedules + script: + - Rscript -e 'source("tests/testthat/store_examples.R"); StoreRefExampleResults("airGR");' + - R CMD INSTALL . + - Rscript -e 'source("tests/testthat/store_examples.R"); StoreTestExampleResults("airGR");' + artifacts: + paths: + - tests/tmp/ + expire_in: 1 week + .check_not_cran: stage: tests variables: @@ -52,6 +67,21 @@ build_patched: R_VERSION: "patched" extends: .build +regression_patched: + variables: + R_VERSION: "patched" + extends: .regression + +regression_devel: + variables: + R_VERSION: "devel" + extends: .regression + +regression_oldrel: + variables: + R_VERSION: "oldrel" + extends: .regression + check_not_cran_patched: variables: R_VERSION: "patched" @@ -125,3 +155,4 @@ check_as_cran_oldrel: variables: R_VERSION: "oldrel" extends: .check_as_cran + diff --git a/.regressionignore b/.regressionignore new file mode 100644 index 0000000000000000000000000000000000000000..f045b6921506c22997c1b574c55b606d36a1c05f --- /dev/null +++ b/.regressionignore @@ -0,0 +1,7 @@ +# .test-regression.ignore contains the list of topic/variables produces by +# documentation examples that should be ignore in the regression test +# The format of this file is: 5 lines of comments followed by one line by +# ignored variable : [Topic]<SPACE>[Variable]. +# Example for ignoring OutputsModel variable produced by example("RunModel_GR2M"): RunModel_GR2M OutputsModel +RunModel_GR2M OutputsModel +RunModel_GR2M RunOptions diff --git a/tests/testthat/store_examples.R b/tests/testthat/store_examples.R new file mode 100644 index 0000000000000000000000000000000000000000..24160040b04747114a25f5fc6006c727c5df1839 --- /dev/null +++ b/tests/testthat/store_examples.R @@ -0,0 +1,74 @@ +StoreRefExampleResults <- function(package, ...) { + install.packages(package, repos = "http://cran.r-project.org") + StoreExampleResults(package = package, path = "tests/tmp/ref", ...) +} + +StoreTestExampleResults <- function(package, ...) { + StoreExampleResults( + package = package, + path = file.path("tests/tmp", Sys.getenv("R_VERSION", "test")), + ... + ) +} + +#' Run examples of a package and store the output variables in RDS files for further testing. +#' +#' @param package Name of the package from which examples are tested. +#' @param path Path where to record the files. +#' @param run.dontrun See \code{\link{example}}. +#' @param run.donttest See \code{\link{example}}. +#' +#' @return +#' @export +#' +#' @examples +StoreExampleResults <- function(package, path, run.dontrun = FALSE, run.donttest = TRUE) { + + # Install and load stable version of the package + library(package, character.only = TRUE) + + # Get the list of documentation pages + rd <- unique(readRDS(system.file("help", "aliases.rds", package = package))) + + dir.create(path, showWarnings = FALSE) + + lapply( + rd, + StoreTopicResults, + package, path, run.dontrun = run.dontrun, run.donttest = run.donttest + ) +} + +StoreTopicResults <- function(topic, package, path, run.dontrun = TRUE, run.donttest = TRUE) { + + cat("*******************************\n") + cat("*", topic, "\n") + cat("*******************************\n") + + par(ask = FALSE) #https://stackoverflow.com/questions/34756905/how-to-turn-off-the-hit-return-to-see-next-plot-prompt-plot3d + + varBefore <- c() + varBefore <- ls(envir = globalenv()) + + example( + topic, package = package, character.only = TRUE, echo = FALSE, ask = FALSE, local = FALSE, setRNG = TRUE, + run.dontrun = run.dontrun, run.donttest = run.donttest + ) + dev.off() + + varAfter <- ls(envir = globalenv()) + + varToSave <- setdiff(varAfter, varBefore) + + if (length(varToSave) > 0) { + path <- file.path(path, topic) + dir.create(path, showWarnings = FALSE, recursive = TRUE) + lapply(varToSave, function(x) { + saveRDS(get(x), file = file.path(path, paste0(x, ".rds"))) + }) + } + + rm(list = varToSave, envir = globalenv()) + +} + diff --git a/tests/testthat/test-regression.R b/tests/testthat/test-regression.R new file mode 100644 index 0000000000000000000000000000000000000000..432c561d644017d382aaacbd66b3753a54a33ec8 --- /dev/null +++ b/tests/testthat/test-regression.R @@ -0,0 +1,42 @@ +context("Compare example outputs with CRAN") + +CompareWithRef <- function(refVarFile, testDir, regIgnore) { + v <- data.frame(topic = basename(dirname(refVarFile)), + var = gsub("\\.rds$", "", basename(refVarFile))) + if (is.null(regIgnore) || all(apply(regIgnore, 1, function(x) !all(x == v)))) { + test_that(paste("Compare", v$topic, v$var), { + skip_on_cran() + testVarFile <- paste0( + file.path("../tmp", Sys.getenv("R_VERSION", "test"), v$topic, v$var), + ".rds" + ) + expect_true(file.exists(testVarFile)) + if (file.exists(testVarFile)) { + testVar <- readRDS(testVarFile) + refVar <- readRDS(refVarFile) + expect_equivalent(testVar, refVar) + } + }) + } +} + +if (dir.exists("../tmp/ref") & dir.exists("../tmp/test")) { + refVarFiles <- list.files("../tmp/ref", recursive = TRUE, full.names = TRUE) + regIgnoreFile <- "../../.regressionignore2" + if (file.exists(regIgnoreFile)) { + regIgnore <- read.table(file = regIgnoreFile, + sep = " ", header = FALSE, skip = 5, + col.names = c("topic", "var"), + stringsAsFactors = FALSE) + } else { + regIgnore <- NULL + } + lapply(X = refVarFiles, CompareWithRef, testDir = "../tmp/test", regIgnore = regIgnore) +} else { + warning("Regression tests compared to released version needs that you run the following instructions first:\n", + "Rscript -e 'source(\"tests/testthat/store_examples.R\"); StoreRefExampleResults(\"airGR\");'\n", + "R CMD INSTALL .\n", + "Rscript -e 'source(\"tests/testthat/store_examples.R\"); StoreTestExampleResults(\"airGR\");'\n") +} + +