Skip to content
GitLab
Select Git revision
  • master default
  • v2024_rc
  • 144-create-testing-framework-for-tests-automation
  • v2025_alpha
  • dev/dummy
  • code_refactor
  • board_v4
  • MB.2024.ter_restored
  • Refactor_mb_2023
  • MUX_board_v2024
  • EGU23
  • update_V2023
  • battery_charger
  • mqtt
  • Ohmpi_1.0x
  • v2024.0.35
  • v2024.0.34
  • v2024.0.33
  • v2024.0.32
  • v2024.0.31
  • v2024.0.30
  • v2024.0.29
  • v2024.0.28
  • v2024.0.27
  • v2024.0.26
  • v2024.0.25
  • v2024.0.24
  • v2024.0.23
  • v2024.0.22
  • v2024.0.21
  • v2024.0.20
  • v2024.0.19
  • v2024.0.18
  • v2024.0.17
  • v2024.0.16
Find file BlameHistoryPermalink
This GitLab CI configuration is valid. Learn more
.gitlab-ci.yml 5.83 KiB
1 
variables:
2 
  TEST_IMAGE_NAME: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
3 
  CPU_IMAGE_NAME: $CI_REGISTRY_IMAGE:cpu
4 
  GPU_IMAGE_NAME: $CI_REGISTRY_IMAGE:gpu
5 
  DOCKER_BUILDKIT: 1
6 
  DOCKER_DRIVER: overlay2
7 
  CPU_BASE_IMAGE: gitlab-registry.irstea.fr/remi.cresson/otbtf:3.3.2-cpu-dev
8 
  GPU_BASE_IMAGE: gitlab-registry.irstea.fr/remi.cresson/otbtf:3.3.2-gpu-dev
9
10 
workflow:
11 
  rules:
12 
    - if: $CI_MERGE_REQUEST_ID || $CI_COMMIT_REF_NAME =~ /master/ # Execute jobs in merge request context, or commit in master branch
13
14 
stages:
15 
  - Docker build
16 
  - Static Analysis
17 
  - Tests
18 
  - Ship
19
20 
.dind_base:
21 
  tags: [dind]
22 
  image: docker/compose:1.29.2
23 
  variables:
24 
    DOCKER_TLS_CERTDIR: ""
25 
    DOCKER_HOST: tcp://docker:2375
26 
  services:
27 
    - name: docker:dind
28 
      command: [dockerd, '-H', 'tcp://0.0.0.0:2375']
29 
  before_script:
30 
    # docker login asks for the password to be passed through stdin for security
31 
    # we use $CI_REGISTRY_PASSWORD here which is a special variable provided by GitLab
32 
    # https://docs.gitlab.com/ce/ci/variables/predefined_variables.html
33 
    - echo -n $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
34
35 
Build the docker image:
36 
  stage: Docker build
37 
  allow_failure: false
38 
  extends: .dind_base
39 
  except:
40 
    - master
41 
  script:
42 
    - docker info
43 
    - >
44 
      docker build
45 
      --pull
46 
      --cache-from $CI_REGISTRY_IMAGE:cpu
47 
      --cache-from $TEST_IMAGE_NAME
48 
      --tag $TEST_IMAGE_NAME
49 
      --build-arg "BASE_IMAGE=$CPU_BASE_IMAGE"
50 
      --build-arg BUILDKIT_INLINE_CACHE=1
51 
      .
52 
    - docker push $TEST_IMAGE_NAME
53
54 
.static_analysis_base:
55 
  image: $TEST_IMAGE_NAME
56 
  stage: Static Analysis
57 
  allow_failure: true
58
59 
flake8:
60 
  extends: .static_analysis_base
61 
  script:
62 
   - sudo apt update && sudo apt install -y flake8 && python -m flake8 --ignore=E402 --max-line-length=120 $PWD/decloud
63
64 
pylint:
65 
  extends: .static_analysis_base
66 
  script:
67 
  - sudo apt update && sudo apt install -y pylint && pylint --disable=too-many-nested-blocks,too-many-locals,too-many-statements,too-few-public-methods,too-many-instance-attributes,too-many-arguments,invalid-name,cell-var-from-loop,too-many-branches,too-many-ancestors --ignored-modules=tensorflow,git,rtree,scipy,tensorboard,libamalthee,pandas --max-line-length=120 $PWD/decloud
68
69 
codespell:
70 
  extends: .static_analysis_base
7172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140
script: - sudo pip install codespell && codespell --skip="*.png,*.template,*.pbs,*.jpg,*git/lfs*" .applications_test_base: image: $TEST_IMAGE_NAME stage: Tests before_script: - export PYTHONPATH=$PYTHONPATH:$PWD - wget -P decloud_data --no-verbose --recursive --level=inf --no-parent -R "index.html*" --cut-dirs=3 --no-host-directories http://indexof.montpellier.irstea.priv/projets/geocicd/decloud/ - mkdir tests_artifacts - export DECLOUD_DATA_DIR="$PWD/decloud_data" artifacts: when: on_failure paths: - "*.xml" - "tests_artifacts/*.*" expire_in: 1 week after_script: - cp -r /tmp/* tests_artifacts/ display_apps_helps: extends: .applications_test_base script: - python decloud/models/train_from_tfrecords.py --help - python decloud/models/create_tfrecords.py --help - python decloud/preprocessing/sentinel1_prepare.py --help - python decloud/preprocessing/sentinel2_prepare.py --help - python decloud/preprocessing/dem_prepare.py --help - python decloud/analysis/tile_coverage.py --help - python decloud/analysis/images_stats.py --help - python decloud/analysis/patches_coverage.py --help - python decloud/production/crga_processor.py --help inference: extends: .applications_test_base script: - pytest -o log_cli=true --log-cli-level=INFO --junitxml=report_inference.xml tests/inference_unittest.py s1_prepare: extends: .applications_test_base script: - pytest -o log_cli=true --log-cli-level=INFO --junitxml=report_sentinel1_prepare.xml tests/sentinel1_prepare_unittest.py s2_prepare: extends: .applications_test_base script: - pytest -o log_cli=true --log-cli-level=INFO --junitxml=report_sentinel2_prepare.xml tests/sentinel2_prepare_unittest.py DEM_prepare: extends: .applications_test_base script: - pytest -o log_cli=true --log-cli-level=INFO --junitxml=report_dem_prepare.xml tests/dem_unittest.py create_tfrecords: extends: .applications_test_base script: - pytest -o log_cli=true --log-cli-level=INFO --junitxml=report_create_tfrecords.xml tests/create_tfrecords_unittest.py train_from_tfrecords: extends: .applications_test_base script: - pytest -o log_cli=true --log-cli-level=INFO --junitxml=report_train_from_tfrecords.xml tests/train_from_tfrecords_unittest.py .ship_base: stage: Ship only: - master extends: .dind_base
141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178
cpu_deploy: extends: .ship_base script: - > docker build --pull --cache-from $CPU_IMAGE_NAME --cache-from $TEST_IMAGE_NAME --label "org.opencontainers.image.title=$CI_PROJECT_TITLE" --label "org.opencontainers.image.url=$CI_PROJECT_URL" --label "org.opencontainers.image.created=$CI_JOB_STARTED_AT" --label "org.opencontainers.image.revision=$CI_COMMIT_SHA" --label "org.opencontainers.image.version=$CI_COMMIT_REF_NAME" --tag $CPU_IMAGE_NAME --build-arg "BASE_IMAGE=$CPU_BASE_IMAGE" --build-arg BUILDKIT_INLINE_CACHE=1 . - docker push $CPU_IMAGE_NAME gpu_deploy: extends: .ship_base script: - > docker build --pull --cache-from $GPU_IMAGE_NAME --label "org.opencontainers.image.title=$CI_PROJECT_TITLE" --label "org.opencontainers.image.url=$CI_PROJECT_URL" --label "org.opencontainers.image.created=$CI_JOB_STARTED_AT" --label "org.opencontainers.image.revision=$CI_COMMIT_SHA" --label "org.opencontainers.image.version=$CI_COMMIT_REF_NAME" --tag $GPU_IMAGE_NAME --build-arg "BASE_IMAGE=$GPU_BASE_IMAGE" --build-arg BUILDKIT_INLINE_CACHE=1 . - docker push $GPU_IMAGE_NAME