From 076c6975631d69b1c9070ad3c08c50cb36be13c1 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@irstea.fr>
Date: Wed, 1 Jun 2022 22:09:04 +0200
Subject: [PATCH] CI: deploy CPU and GPU images
---
.gitlab-ci.yml | 69 +++++++++++++++++++++++++++++++++++++-------------
1 file changed, 52 insertions(+), 17 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6fba59c..98adeb9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,3 +1,11 @@
+variables:
+ TEST_IMAGE_NAME: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
+ CPU_IMAGE_NAME: $CI_REGISTRY_IMAGE:cpu
+ GPU_IMAGE_NAME: $CI_REGISTRY_IMAGE:gpu
+ DOCKER_BUILDKIT: 1
+ DOCKER_DRIVER: overlay2
+ CPU_BASE_IMAGE: gitlab-registry.irstea.fr/remi.cresson/otbtf/3.2.1:cpu-basic-dev
+ GPU_BASE_IMAGE: gitlab-registry.irstea.fr/remi.cresson/otbtf/3.2.1:gpu-basic-dev
workflow:
rules:
@@ -35,19 +43,16 @@ Build the docker image:
- >
docker build
--pull
- --cache-from $CI_REGISTRY_IMAGE:latest
- --label "org.opencontainers.image.title=$CI_PROJECT_TITLE"
- --label "org.opencontainers.image.url=$CI_PROJECT_URL"
- --label "org.opencontainers.image.created=$CI_JOB_STARTED_AT"
- --label "org.opencontainers.image.revision=$CI_COMMIT_SHA"
- --label "org.opencontainers.image.version=$CI_COMMIT_REF_NAME"
- --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
- --build-arg "BASE_IMAGE=gitlab-registry.irstea.fr/remi.cresson/otbtf/3.2.1:cpu-basic-dev"
+ --cache-from $CI_REGISTRY_IMAGE:cpu
+ --cache-from $TEST_IMAGE_NAME
+ --tag $TEST_IMAGE_NAME
+ --build-arg "BASE_IMAGE=$CPU_BASE_IMAGE"
+ --build-arg BUILDKIT_INLINE_CACHE=1
.
- - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
+ - docker push $TEST_IMAGE_NAME
.static_analysis_base:
- image: $CI_REGISTRY_IMAGE:latest
+ image: $TEST_IMAGE_NAME
stage: Static Analysis
allow_failure: true
@@ -67,7 +72,7 @@ codespell:
- sudo pip install codespell && codespell --skip="*.png,*.template,*.pbs,*.jpg,*git/lfs*"
.applications_test_base:
- image: $CI_REGISTRY_IMAGE:latest
+ image: $TEST_IMAGE_NAME
stage: Tests
before_script:
- export PYTHONPATH=$PYTHONPATH:$PWD
@@ -103,7 +108,6 @@ inference:
script:
- pytest -o log_cli=true --log-cli-level=INFO --junitxml=report_inference.xml tests/inference_unittest.py
-
s1_prepare:
extends: .applications_test_base
script:
@@ -129,13 +133,44 @@ train_from_tfrecords:
script:
- pytest -o log_cli=true --log-cli-level=INFO --junitxml=report_train_from_tfrecords.xml tests/train_from_tfrecords_unittest.py
-deploy:
+.ship_base:
stage: Ship
only:
- master
extends: .dind_base
+
+cpu_deploy:
+ extends: .ship_base
script:
- - echo "Shipping!"
- - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
- - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME $CI_REGISTRY_IMAGE:latest
- - docker push $CI_REGISTRY_IMAGE:latest
+ - >
+ docker build
+ --pull
+ --cache-from $CPU_IMAGE_NAME
+ --cache-from $TEST_IMAGE_NAME
+ --label "org.opencontainers.image.title=$CI_PROJECT_TITLE"
+ --label "org.opencontainers.image.url=$CI_PROJECT_URL"
+ --label "org.opencontainers.image.created=$CI_JOB_STARTED_AT"
+ --label "org.opencontainers.image.revision=$CI_COMMIT_SHA"
+ --label "org.opencontainers.image.version=$CI_COMMIT_REF_NAME"
+ --tag $CPU_IMAGE_NAME
+ --build-arg "BASE_IMAGE=$CPU_BASE_IMAGE"
+ --build-arg BUILDKIT_INLINE_CACHE=1
+ .
+ - docker push $CPU_IMAGE_NAME
+
+gpu_deploy:
+ extends: .ship_base
+ - >
+ docker build
+ --pull
+ --cache-from $GPU_IMAGE_NAME
+ --label "org.opencontainers.image.title=$CI_PROJECT_TITLE"
+ --label "org.opencontainers.image.url=$CI_PROJECT_URL"
+ --label "org.opencontainers.image.created=$CI_JOB_STARTED_AT"
+ --label "org.opencontainers.image.revision=$CI_COMMIT_SHA"
+ --label "org.opencontainers.image.version=$CI_COMMIT_REF_NAME"
+ --tag $GPU_IMAGE_NAME
+ --build-arg "BASE_IMAGE=$GPU_BASE_IMAGE"
+ --build-arg BUILDKIT_INLINE_CACHE=1
+ .
+ - docker push $GPU_IMAGE_NAME
--
GitLab