ci: Extract regression tests from standard thestthat procedure

- Regression test are now totally run with `Rscript tests\regression_tests.R`

Refs #69

.gitlab-ci.yml

@@ -33,14 +33,10 @@ default:
 
 .regression:
   stage: regression
-  variables:
-    NOT_CRAN: "true"
   script:
-    - Rscript -e 'source("tests/testthat/store_examples.R"); StoreRefExampleResults("airGR");'
-    - R CMD INSTALL .
-    - Rscript -e 'source("tests/testthat/store_examples.R"); StoreTestExampleResults("airGR");'
-    - ls -l tests/tmp
-    - Rscript -e 'Sys.setenv(NOT_CRAN="true"); setwd("tests/testthat"); res <- testthat::test_file("test-regression.R"); dRes = as.data.frame(res); if(any(dRes[,"failed"]>0) | any(dRes[,"error"])) {quit(status = 1)};'
+    - Rscript tests/regression_tests.R stable
+    - Rscript tests/regression_tests.R dev
+    - Rscript tests/regression_tests.R compare
 
 .check_not_cran:
   stage: tests

tests/regression_tests.R

+# Execute Regression test by comparing RD files stored in folders /tests/tmp/ref and /tests/tmp/test
+Args = commandArgs(trailingOnly=TRUE)
+
+source("tests/testthat/helper_regression.R")
+
+lActions = list(
+  stable = StoreStableExampleResults,
+  dev = StoreDevExampleResults,
+  compare = CompareStableDev
+)
+
+if(Args %in% names(lActions)) {
+  lActions[[Args]]()
+} else {
+  stop("This script should be run with one argument in the command line:\n",
+       "`Rscript tests/regression_tests.R [stable|dev|compare]`.\n",
+       "Available arguments are:\n",
+       "- stable: install stable version from CRAN, run and store examples\n", 
+       "- dev: install dev version from current directory, run and store examples\n",
+       "- compare: stored results of both versions")
+}

tests/testthat/store_examples.R → tests/testthat/helper_regression.R

-StoreRefExampleResults <- function(package, ...) {
+StoreStableExampleResults <- function(
+  package = "airGR", 
+  path = file.path("tests/tmp", Sys.getenv("R_VERSION"), "stable"),
+  ...) {
   install.packages(package, repos = "http://cran.r-project.org")
-  StoreExampleResults(package = package, path = "tests/tmp/ref", ...)
+  StoreExampleResults(package = package, path = path, ...)
 }
 
-StoreTestExampleResults <- function(package, ...) {
-  StoreExampleResults(
-    package = package,
-    path = file.path("tests/tmp", Sys.getenv("R_VERSION", "test")),
-    ...
-  )
+StoreDevExampleResults <- function(
+  package = "airGR", 
+  path = file.path("tests/tmp", Sys.getenv("R_VERSION"), "dev"), 
+  ...) {
+  devtools::install()
+  StoreExampleResults(package = package, path = path, ...)
 }
 
 #' Run examples of a package and store the output variables in RDS files for further testing.
@@ -72,3 +75,8 @@ StoreTopicResults <- function(topic, package, path, run.dontrun = TRUE, run.dont
 
 }
 
+CompareStableDev <- function() {
+  res = testthat::test_file("tests/testthat/regression.R")
+  dRes = as.data.frame(res)
+  if(any(dRes[,"failed"]>0) | any(dRes[,"error"])) quit(status = 1)
+}

tests/testthat/test-regression.R → tests/testthat/regression.R

 context("Compare example outputs with CRAN")
 
-CompareWithRef <- function(refVarFile, testDir, regIgnore) {
+CompareWithStable <- function(refVarFile, testDir, regIgnore) {
   v <- data.frame(topic = basename(dirname(refVarFile)),
                   var = gsub("\\.rds$", "", basename(refVarFile)))
   if (is.null(regIgnore) || all(apply(regIgnore, 1, function(x) !all(x == v)))) {
     test_that(paste("Compare", v$topic, v$var), {
-      skip_on_cran()
       testVarFile <- paste0(
-        file.path("../tmp", Sys.getenv("R_VERSION", "test"), v$topic, v$var),
+        file.path(testDir, v$topic, v$var),
         ".rds"
       )
       expect_true(file.exists(testVarFile))
@@ -19,24 +18,27 @@ CompareWithRef <- function(refVarFile, testDir, regIgnore) {
     })
   }
 }
-print(getwd())
-if (dir.exists("../tmp/ref") & dir.exists("../tmp/test")) {
-  refVarFiles <- list.files("../tmp/ref", recursive = TRUE, full.names = TRUE)
+
+tmp_path <- file.path("../tmp", Sys.getenv("R_VERSION"));
+
+if (dir.exists(file.path(tmp_path, "stable")) & dir.exists(file.path(tmp_path, "dev"))) {
+  refVarFiles <- list.files(file.path(tmp_path, "stable"), recursive = TRUE, full.names = TRUE)
   regIgnoreFile <- "../../.regressionignore"
   if (file.exists(regIgnoreFile)) {
+    message("Using .regressionignore file. The following variables are going to be skipped:")
     regIgnore <- read.table(file = regIgnoreFile,
                             sep = " ", header = FALSE, skip = 5,
                             col.names = c("topic", "var"),
                             stringsAsFactors = FALSE)
+    apply(regIgnore, 1, function(x) message(x[1], ": ", x[2]))
   } else {
+    message("File ", file.path(getwd(), regIgnoreFile), " not found")
     regIgnore <- NULL
   }
-  lapply(X = refVarFiles, CompareWithRef, testDir = "../tmp/test", regIgnore = regIgnore)
+  lapply(X = refVarFiles, CompareWithStable, testDir = file.path(tmp_path, "dev"), regIgnore = regIgnore)
 } else {
-  warning("Regression tests compared to released version needs that you run the following instructions first:\n",
-          "Rscript -e 'source(\"tests/testthat/store_examples.R\"); StoreRefExampleResults(\"airGR\");'\n",
+  stop("Regression tests compared to released version needs that you run the following instructions first:\n",
+          "Rscript -e 'source(\"tests/testthat/store_examples.R\"); StoreStableExampleResults(\"airGR\");'\n",
           "R CMD INSTALL .\n",
-          "Rscript -e 'source(\"tests/testthat/store_examples.R\"); StoreTestExampleResults(\"airGR\");'\n")
+          "Rscript -e 'source(\"tests/testthat/store_examples.R\"); StoreDevExampleResults(\"airGR\");'\n")
 }
-
-