Merge branch '69-regression-tests-are-not-functionning' into 'dev'

Resolve "Regression tests are not functionning"

Closes #69

See merge request !18

.gitlab-ci.yml

@@ -33,17 +33,11 @@ default:
 
 .regression:
   stage: regression
-  only:
-    refs:
-      - schedules
   script:
-    - Rscript -e 'source("tests/testthat/store_examples.R"); StoreRefExampleResults("airGR");'
+    - Rscript tests/testthat/regression_tests.R stable
     - R CMD INSTALL .
-    - Rscript -e 'source("tests/testthat/store_examples.R"); StoreTestExampleResults("airGR");'
-  artifacts:
-    paths:
-      - tests/tmp/
-    expire_in: 1 week
+    - Rscript tests/testthat/regression_tests.R dev
+    - Rscript tests/testthat/regression_tests.R compare
 
 .check_not_cran:
   stage: tests
@@ -73,11 +67,17 @@ regression_patched:
   extends: .regression
 
 regression_devel:
+  only:
+    refs:
+      - schedules
   variables:
     R_VERSION: "devel"
   extends: .regression
 
 regression_oldrel:
+  only:
+    refs:
+      - schedules
   variables:
     R_VERSION: "oldrel"
   extends: .regression

.regressionignore

@@ -5,3 +5,4 @@
 # Example for ignoring OutputsModel variable produced by example("RunModel_GR2M"): RunModel_GR2M OutputsModel
 RunModel_GR2M OutputsModel
 RunModel_GR2M RunOptions
+RunModel_GR1A OutputsModel

tests/testthat/store_examples.R → tests/testthat/helper_regression.R

-StoreRefExampleResults <- function(package, ...) {
+StoreStableExampleResults <- function(
+  package = "airGR", 
+  path = file.path("tests/tmp", Sys.getenv("R_VERSION"), "stable"),
+  ...) {
   install.packages(package, repos = "http://cran.r-project.org")
-  StoreExampleResults(package = package, path = "tests/tmp/ref", ...)
+  StoreExampleResults(package = package, path = path, ...)
 }
 
-StoreTestExampleResults <- function(package, ...) {
-  StoreExampleResults(
-    package = package,
-    path = file.path("tests/tmp", Sys.getenv("R_VERSION", "test")),
-    ...
-  )
+StoreDevExampleResults <- function(
+  package = "airGR", 
+  path = file.path("tests/tmp", Sys.getenv("R_VERSION"), "dev"), 
+  ...) {
+  StoreExampleResults(package = package, path = path, ...)
 }
 
 #' Run examples of a package and store the output variables in RDS files for further testing.
@@ -72,3 +74,8 @@ StoreTopicResults <- function(topic, package, path, run.dontrun = TRUE, run.dont
 
 }
 
+CompareStableDev <- function() {
+  res = testthat::test_file("tests/testthat/regression.R")
+  dRes = as.data.frame(res)
+  if(any(dRes[,"failed"]>0) | any(dRes[,"error"])) quit(status = 1)
+}

tests/testthat/test-regression.R → tests/testthat/regression.R

 context("Compare example outputs with CRAN")
 
-CompareWithRef <- function(refVarFile, testDir, regIgnore) {
+CompareWithStable <- function(refVarFile, testDir, regIgnore) {
   v <- data.frame(topic = basename(dirname(refVarFile)),
                   var = gsub("\\.rds$", "", basename(refVarFile)))
   if (is.null(regIgnore) || all(apply(regIgnore, 1, function(x) !all(x == v)))) {
     test_that(paste("Compare", v$topic, v$var), {
-      skip_on_cran()
       testVarFile <- paste0(
-        file.path("../tmp", Sys.getenv("R_VERSION", "test"), v$topic, v$var),
+        file.path(testDir, v$topic, v$var),
         ".rds"
       )
       expect_true(file.exists(testVarFile))
@@ -20,23 +19,26 @@ CompareWithRef <- function(refVarFile, testDir, regIgnore) {
   }
 }
 
-if (dir.exists("../tmp/ref") & dir.exists("../tmp/test")) {
-  refVarFiles <- list.files("../tmp/ref", recursive = TRUE, full.names = TRUE)
-  regIgnoreFile <- "../../.regressionignore2"
+tmp_path <- file.path("../tmp", Sys.getenv("R_VERSION"));
+
+if (dir.exists(file.path(tmp_path, "stable")) & dir.exists(file.path(tmp_path, "dev"))) {
+  refVarFiles <- list.files(file.path(tmp_path, "stable"), recursive = TRUE, full.names = TRUE)
+  regIgnoreFile <- "../../.regressionignore"
   if (file.exists(regIgnoreFile)) {
+    message("Using .regressionignore file. The following variables are going to be skipped:")
     regIgnore <- read.table(file = regIgnoreFile,
                             sep = " ", header = FALSE, skip = 5,
                             col.names = c("topic", "var"),
                             stringsAsFactors = FALSE)
+    apply(regIgnore, 1, function(x) message(x[1], ": ", x[2]))
   } else {
+    message("File ", file.path(getwd(), regIgnoreFile), " not found")
     regIgnore <- NULL
   }
-  lapply(X = refVarFiles, CompareWithRef, testDir = "../tmp/test", regIgnore = regIgnore)
+  lapply(X = refVarFiles, CompareWithStable, testDir = file.path(tmp_path, "dev"), regIgnore = regIgnore)
 } else {
-  warning("Regression tests compared to released version needs that you run the following instructions first:\n",
-          "Rscript -e 'source(\"tests/testthat/store_examples.R\"); StoreRefExampleResults(\"airGR\");'\n",
-          "R CMD INSTALL .\n",
-          "Rscript -e 'source(\"tests/testthat/store_examples.R\"); StoreTestExampleResults(\"airGR\");'\n")
+  stop("Regression tests compared to released version needs that you run the following instructions first:\n",
+       "Rscript tests/testthat/regression_tests.R stable\n",
+       "R CMD INSTALL .\n",
+       "Rscript tests/testthat/regression_tests.R dev")
 }
-
-

tests/testthat/regression_tests.R

+# Execute Regression test by comparing RD files stored in folders /tests/tmp/ref and /tests/tmp/test
+Args = commandArgs(trailingOnly=TRUE)
+
+source("tests/testthat/helper_regression.R")
+
+lActions = list(
+  stable = StoreStableExampleResults,
+  dev = StoreDevExampleResults,
+  compare = CompareStableDev
+)
+
+if(Args %in% names(lActions)) {
+  lActions[[Args]]()
+} else {
+  stop("This script should be run with one argument in the command line:\n",
+       "`Rscript tests/regression_tests.R [stable|dev|compare]`.\n",
+       "Available arguments are:\n",
+       "- stable: install stable version from CRAN, run and store examples\n", 
+       "- dev: install dev version from current directory, run and store examples\n",
+       "- compare: stored results of both versions")
+}