Commit 30dbee89 authored by Dorchies David's avatar Dorchies David
Browse files

ci: Extract regression tests from standard thestthat procedure

- Regression test are now totally run with `Rscript tests\regression_tests.R`

Refs #69
parent 04e68e4b
Pipeline #17770 failed with stages
in 1 minute and 32 seconds
...@@ -33,14 +33,10 @@ default: ...@@ -33,14 +33,10 @@ default:
.regression: .regression:
stage: regression stage: regression
variables:
NOT_CRAN: "true"
script: script:
- Rscript -e 'source("tests/testthat/store_examples.R"); StoreRefExampleResults("airGR");' - Rscript tests/regression_tests.R stable
- R CMD INSTALL . - Rscript tests/regression_tests.R dev
- Rscript -e 'source("tests/testthat/store_examples.R"); StoreTestExampleResults("airGR");' - Rscript tests/regression_tests.R compare
- ls -l tests/tmp
- Rscript -e 'Sys.setenv(NOT_CRAN="true"); setwd("tests/testthat"); res <- testthat::test_file("test-regression.R"); dRes = as.data.frame(res); if(any(dRes[,"failed"]>0) | any(dRes[,"error"])) {quit(status = 1)};'
.check_not_cran: .check_not_cran:
stage: tests stage: tests
......
# Execute Regression test by comparing RD files stored in folders /tests/tmp/ref and /tests/tmp/test
Args = commandArgs(trailingOnly=TRUE)
source("tests/testthat/helper_regression.R")
lActions = list(
stable = StoreStableExampleResults,
dev = StoreDevExampleResults,
compare = CompareStableDev
)
if(Args %in% names(lActions)) {
lActions[[Args]]()
} else {
stop("This script should be run with one argument in the command line:\n",
"`Rscript tests/regression_tests.R [stable|dev|compare]`.\n",
"Available arguments are:\n",
"- stable: install stable version from CRAN, run and store examples\n",
"- dev: install dev version from current directory, run and store examples\n",
"- compare: stored results of both versions")
}
StoreRefExampleResults <- function(package, ...) { StoreStableExampleResults <- function(
package = "airGR",
path = file.path("tests/tmp", Sys.getenv("R_VERSION"), "stable"),
...) {
install.packages(package, repos = "http://cran.r-project.org") install.packages(package, repos = "http://cran.r-project.org")
StoreExampleResults(package = package, path = "tests/tmp/ref", ...) StoreExampleResults(package = package, path = path, ...)
} }
StoreTestExampleResults <- function(package, ...) { StoreDevExampleResults <- function(
StoreExampleResults( package = "airGR",
package = package, path = file.path("tests/tmp", Sys.getenv("R_VERSION"), "dev"),
path = file.path("tests/tmp", Sys.getenv("R_VERSION", "test")), ...) {
... devtools::install()
) StoreExampleResults(package = package, path = path, ...)
} }
#' Run examples of a package and store the output variables in RDS files for further testing. #' Run examples of a package and store the output variables in RDS files for further testing.
...@@ -72,3 +75,8 @@ StoreTopicResults <- function(topic, package, path, run.dontrun = TRUE, run.dont ...@@ -72,3 +75,8 @@ StoreTopicResults <- function(topic, package, path, run.dontrun = TRUE, run.dont
} }
CompareStableDev <- function() {
res = testthat::test_file("tests/testthat/regression.R")
dRes = as.data.frame(res)
if(any(dRes[,"failed"]>0) | any(dRes[,"error"])) quit(status = 1)
}
context("Compare example outputs with CRAN") context("Compare example outputs with CRAN")
CompareWithRef <- function(refVarFile, testDir, regIgnore) { CompareWithStable <- function(refVarFile, testDir, regIgnore) {
v <- data.frame(topic = basename(dirname(refVarFile)), v <- data.frame(topic = basename(dirname(refVarFile)),
var = gsub("\\.rds$", "", basename(refVarFile))) var = gsub("\\.rds$", "", basename(refVarFile)))
if (is.null(regIgnore) || all(apply(regIgnore, 1, function(x) !all(x == v)))) { if (is.null(regIgnore) || all(apply(regIgnore, 1, function(x) !all(x == v)))) {
test_that(paste("Compare", v$topic, v$var), { test_that(paste("Compare", v$topic, v$var), {
skip_on_cran()
testVarFile <- paste0( testVarFile <- paste0(
file.path("../tmp", Sys.getenv("R_VERSION", "test"), v$topic, v$var), file.path(testDir, v$topic, v$var),
".rds" ".rds"
) )
expect_true(file.exists(testVarFile)) expect_true(file.exists(testVarFile))
...@@ -19,24 +18,27 @@ CompareWithRef <- function(refVarFile, testDir, regIgnore) { ...@@ -19,24 +18,27 @@ CompareWithRef <- function(refVarFile, testDir, regIgnore) {
}) })
} }
} }
print(getwd())
if (dir.exists("../tmp/ref") & dir.exists("../tmp/test")) { tmp_path <- file.path("../tmp", Sys.getenv("R_VERSION"));
refVarFiles <- list.files("../tmp/ref", recursive = TRUE, full.names = TRUE)
if (dir.exists(file.path(tmp_path, "stable")) & dir.exists(file.path(tmp_path, "dev"))) {
refVarFiles <- list.files(file.path(tmp_path, "stable"), recursive = TRUE, full.names = TRUE)
regIgnoreFile <- "../../.regressionignore" regIgnoreFile <- "../../.regressionignore"
if (file.exists(regIgnoreFile)) { if (file.exists(regIgnoreFile)) {
message("Using .regressionignore file. The following variables are going to be skipped:")
regIgnore <- read.table(file = regIgnoreFile, regIgnore <- read.table(file = regIgnoreFile,
sep = " ", header = FALSE, skip = 5, sep = " ", header = FALSE, skip = 5,
col.names = c("topic", "var"), col.names = c("topic", "var"),
stringsAsFactors = FALSE) stringsAsFactors = FALSE)
apply(regIgnore, 1, function(x) message(x[1], ": ", x[2]))
} else { } else {
message("File ", file.path(getwd(), regIgnoreFile), " not found")
regIgnore <- NULL regIgnore <- NULL
} }
lapply(X = refVarFiles, CompareWithRef, testDir = "../tmp/test", regIgnore = regIgnore) lapply(X = refVarFiles, CompareWithStable, testDir = file.path(tmp_path, "dev"), regIgnore = regIgnore)
} else { } else {
warning("Regression tests compared to released version needs that you run the following instructions first:\n", stop("Regression tests compared to released version needs that you run the following instructions first:\n",
"Rscript -e 'source(\"tests/testthat/store_examples.R\"); StoreRefExampleResults(\"airGR\");'\n", "Rscript -e 'source(\"tests/testthat/store_examples.R\"); StoreStableExampleResults(\"airGR\");'\n",
"R CMD INSTALL .\n", "R CMD INSTALL .\n",
"Rscript -e 'source(\"tests/testthat/store_examples.R\"); StoreTestExampleResults(\"airGR\");'\n") "Rscript -e 'source(\"tests/testthat/store_examples.R\"); StoreDevExampleResults(\"airGR\");'\n")
} }
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment