Merge branch '59-add-comparative-tests-with-master-branch-on-examples' into 'dev'

Resolve "Add comparative tests with master branch on examples"

Closes #59

See merge request HYCAR-Hydro/airgr!13

.Rbuildignore

@@ -2,3 +2,5 @@
 ^\.Rproj\.user$
 ^\.Rprofile$
 ^packrat/
+^tests/tmp/
+^\.regressionignore$

.gitignore

@@ -3,12 +3,13 @@ packrat/lib*/
 
 # Compiled files
 /src/*.o
+/src/*.so
 /src/*.dll
 /src-*
 
 # Test temporary files
 /tests/tmp/
-/tests/testthat/*.pdf
+*.pdf
 
 ######################################################################################################
 ### Generic .gitignore for R (source: https://github.com/github/gitignore/blob/master/R.gitignore) ###

.gitlab-ci.yml

 stages:
   - update_packages
   - build
+  - regression
   - tests
 
 default:
@@ -30,6 +31,20 @@ default:
     untracked: true
     expire_in: 1 week
 
+.regression:
+  stage: regression
+  only:
+    refs:
+      - schedules
+  script:
+    - Rscript -e 'source("tests/testthat/store_examples.R"); StoreRefExampleResults("airGR");'
+    - R CMD INSTALL .
+    - Rscript -e 'source("tests/testthat/store_examples.R"); StoreTestExampleResults("airGR");'
+  artifacts:
+    paths:
+      - tests/tmp/
+    expire_in: 1 week
+
 .check_not_cran:
   stage: tests
   variables:
@@ -52,6 +67,21 @@ build_patched:
     R_VERSION: "patched"
   extends: .build
 
+regression_patched:
+  variables:
+    R_VERSION: "patched"
+  extends: .regression
+
+regression_devel:
+  variables:
+    R_VERSION: "devel"
+  extends: .regression
+
+regression_oldrel:
+  variables:
+    R_VERSION: "oldrel"
+  extends: .regression
+
 check_not_cran_patched:
   variables:
     R_VERSION: "patched"
@@ -125,3 +155,4 @@ check_as_cran_oldrel:
   variables:
     R_VERSION: "oldrel"
   extends: .check_as_cran
+

.regressionignore

+# .test-regression.ignore contains the list of topic/variables produces by
+# documentation examples that should be ignore in the regression test
+# The format of this file is: 5 lines of comments followed by one line by
+# ignored variable : [Topic]<SPACE>[Variable].
+# Example for ignoring OutputsModel variable produced by example("RunModel_GR2M"): RunModel_GR2M OutputsModel
+RunModel_GR2M OutputsModel
+RunModel_GR2M RunOptions

tests/testthat/store_examples.R

+StoreRefExampleResults <- function(package, ...) {
+  install.packages(package, repos = "http://cran.r-project.org")
+  StoreExampleResults(package = package, path = "tests/tmp/ref", ...)
+}
+
+StoreTestExampleResults <- function(package, ...) {
+  StoreExampleResults(
+    package = package,
+    path = file.path("tests/tmp", Sys.getenv("R_VERSION", "test")),
+    ...
+  )
+}
+
+#' Run examples of a package and store the output variables in RDS files for further testing.
+#'
+#' @param package Name of the package from which examples are tested.
+#' @param path Path where to record the files.
+#' @param run.dontrun See \code{\link{example}}.
+#' @param run.donttest See \code{\link{example}}.
+#'
+#' @return
+#' @export
+#'
+#' @examples
+StoreExampleResults <- function(package, path, run.dontrun = FALSE, run.donttest = TRUE) {
+
+  # Install and load stable version of the package
+  library(package, character.only = TRUE)
+
+  # Get the list of documentation pages
+  rd <- unique(readRDS(system.file("help", "aliases.rds", package = package)))
+
+  dir.create(path, showWarnings = FALSE)
+
+  lapply(
+    rd,
+    StoreTopicResults,
+    package, path, run.dontrun = run.dontrun, run.donttest = run.donttest
+  )
+}
+
+StoreTopicResults <- function(topic, package, path, run.dontrun = TRUE, run.donttest = TRUE) {
+
+  cat("*******************************\n")
+  cat("*", topic, "\n")
+  cat("*******************************\n")
+
+  par(ask = FALSE) #https://stackoverflow.com/questions/34756905/how-to-turn-off-the-hit-return-to-see-next-plot-prompt-plot3d
+
+  varBefore <- c()
+  varBefore <- ls(envir = globalenv())
+
+  example(
+    topic, package = package, character.only = TRUE, echo = FALSE, ask = FALSE, local = FALSE, setRNG = TRUE,
+    run.dontrun = run.dontrun, run.donttest = run.donttest
+  )
+  dev.off()
+
+  varAfter <- ls(envir = globalenv())
+
+  varToSave <- setdiff(varAfter, varBefore)
+
+  if (length(varToSave) > 0) {
+    path <- file.path(path, topic)
+    dir.create(path, showWarnings = FALSE, recursive = TRUE)
+    lapply(varToSave, function(x) {
+      saveRDS(get(x), file = file.path(path, paste0(x, ".rds")))
+    })
+  }
+
+  rm(list = varToSave, envir = globalenv())
+
+}
+

tests/testthat/test-regression.R

+context("Compare example outputs with CRAN")
+
+CompareWithRef <- function(refVarFile, testDir, regIgnore) {
+  v <- data.frame(topic = basename(dirname(refVarFile)),
+                  var = gsub("\\.rds$", "", basename(refVarFile)))
+  if (is.null(regIgnore) || all(apply(regIgnore, 1, function(x) !all(x == v)))) {
+    test_that(paste("Compare", v$topic, v$var), {
+      skip_on_cran()
+      testVarFile <- paste0(
+        file.path("../tmp", Sys.getenv("R_VERSION", "test"), v$topic, v$var),
+        ".rds"
+      )
+      expect_true(file.exists(testVarFile))
+      if (file.exists(testVarFile)) {
+        testVar <- readRDS(testVarFile)
+        refVar <- readRDS(refVarFile)
+        expect_equivalent(testVar, refVar)
+      }
+    })
+  }
+}
+
+if (dir.exists("../tmp/ref") & dir.exists("../tmp/test")) {
+  refVarFiles <- list.files("../tmp/ref", recursive = TRUE, full.names = TRUE)
+  regIgnoreFile <- "../../.regressionignore2"
+  if (file.exists(regIgnoreFile)) {
+    regIgnore <- read.table(file = regIgnoreFile,
+                            sep = " ", header = FALSE, skip = 5,
+                            col.names = c("topic", "var"),
+                            stringsAsFactors = FALSE)
+  } else {
+    regIgnore <- NULL
+  }
+  lapply(X = refVarFiles, CompareWithRef, testDir = "../tmp/test", regIgnore = regIgnore)
+} else {
+  warning("Regression tests compared to released version needs that you run the following instructions first:\n",
+          "Rscript -e 'source(\"tests/testthat/store_examples.R\"); StoreRefExampleResults(\"airGR\");'\n",
+          "R CMD INSTALL .\n",
+          "Rscript -e 'source(\"tests/testthat/store_examples.R\"); StoreTestExampleResults(\"airGR\");'\n")
+}
+
+