From 9e5a0008c5ec4bccab875b1a9c309b6303352f11 Mon Sep 17 00:00:00 2001
From: Dorchies David <david.dorchies@inrae.fr>
Date: Wed, 27 Oct 2021 17:47:15 +0200
Subject: [PATCH] tests: try to solve false negative on execution time
benchmark
- new tunning of the limit threshold (Fail on 4ms RunModel with a gap of 0.25
- move benchmark test from check to a specific job and only for patched version
Refs #136
---
.gitlab-ci.yml | 20 +++++++++++++-------
tests/testthat/benchmarkRunModel.R | 2 +-
2 files changed, 14 insertions(+), 8 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a95b86c2..c3cc5f65 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -28,13 +28,6 @@ default:
script:
- R -e 'rcmdcheck::rcmdcheck(args = ifelse(as.logical(Sys.getenv("NOT_CRAN")), "", "--as-cran"), error_on = "warning")'
- R -e 'install.packages(c("microbenchmark", "airGR"), repos = "http://cran.r-project.org")'
- - Rscript tests/scheduled_tests/benchmarkRunModel.R
- - R CMD INSTALL .
- - Rscript tests/scheduled_tests/benchmarkRunModel.R
- artifacts:
- paths:
- - tests/tmp/benchmark.tsv
- - tests/tmp/mean_execution_time.tsv
.check_not_cran:
variables:
@@ -46,6 +39,19 @@ default:
NOT_CRAN: "false"
extends: .check
+benchmark_patched:
+ stage: check
+ variables:
+ R_VERSION: "patched"
+ script:
+ - Rscript tests/scheduled_tests/benchmarkRunModel.R
+ - R CMD INSTALL .
+ - Rscript tests/scheduled_tests/benchmarkRunModel.R
+ artifacts:
+ paths:
+ - tests/tmp/benchmark.tsv
+ - tests/tmp/mean_execution_time.tsv
+
scheduled_tests_patched:
# only:
# refs:
diff --git a/tests/testthat/benchmarkRunModel.R b/tests/testthat/benchmarkRunModel.R
index b38016f3..bc539a32 100644
--- a/tests/testthat/benchmarkRunModel.R
+++ b/tests/testthat/benchmarkRunModel.R
@@ -6,7 +6,7 @@ lapply(df$model, function(model) {
# Limit threshold for evolution of execution time (in %) between the 2 versions
# Negative values of evolution are expected but we apply an error margin depending on execution time
# decrease from 1.5 at 0.0ms to 0.5 at 10ms with a minimum at 0.5
- threshold <- max(0.5, -0.1 * df[sel, 2] + 1.5)
+ threshold <- max(0.5, 2 - 0.1 * df[sel, 2])
expect_lt(df$evolution[sel], threshold)
})
})
--
GitLab