Commit 09952702 authored by Cresson Remi's avatar Cresson Remi
Browse files

Merge branch 'develop' into 'master'

Develop

See merge request !47
2 merge requests!50OTB release 8x,!47Develop
Pipeline #36524 canceled with stages
in 168 minutes and 28 seconds
Showing with 90 additions and 43 deletions
+90 -43
......@@ -13,7 +13,9 @@ variables:
CACHE_IMAGE_BUILDER: $CI_REGISTRY_IMAGE:builder
BRANCH_IMAGE: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
DEV_IMAGE: $CI_REGISTRY_IMAGE:cpu-basic-dev-testing
CI_REGISTRY_PUBIMG: $CI_REGISTRY_IMAGE/$OTBTF_VERSION
DOCKERHUB_IMAGE_BASE: mdl4eo/ptbtf${OTBTF_VERSION}
workflow:
rules:
- if: $CI_MERGE_REQUEST_ID || $CI_COMMIT_REF_NAME =~ /master/ # Execute jobs in merge request context, or commit in master branch
......@@ -23,20 +25,26 @@ stages:
- Static Analysis
- Test
- Applications Test
- Update dev image
- Ship
docker image:
stage: Build
.docker_build_base:
allow_failure: false
tags: [godzilla]
image: docker/compose:latest
except:
- develop
services:
- name: docker:dind
before_script:
- echo -n $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
timeout: 10 hours
docker image:
extends: .docker_build_base
stage: Build
except:
- develop
- master
script:
- >
docker build
......@@ -149,34 +157,70 @@ sr4rs:
- export PYTHONPATH=$PYTHONPATH:$PWD/sr4rs
- python -m pytest --junitxml=$ARTIFACT_TEST_DIR/report_sr4rs.xml $OTBTF_SRC/test/sr4rs_unittest.py
deploy:
stage: Ship
only:
deploy_cpu-dev-testing:
stage: Update dev image
extends: .docker_build_base
except:
- master
tags: [godzilla]
image: docker/compose:latest
services:
- name: docker:dind
variables:
CI_REGISTRY_PUBIMG: $CI_REGISTRY_IMAGE/$OTBTF_VERSION
before_script:
- echo -n $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
timeout: 10 hours
script:
- echo "Shippping!"
- docker pull $BRANCH_IMAGE
- docker tag $BRANCH_IMAGE $DEV_IMAGE
- docker push $DEV_IMAGE
- docker build --network='host' --tag $CI_REGISTRY_PUBIMG:cpu-basic --build-arg BASE_IMG=ubuntu:20.04 --build-arg BZL_CONFIGS="" . # cpu-basic
- docker push $CI_REGISTRY_PUBIMG:cpu-basic
- docker build --network='host' --tag $CI_REGISTRY_PUBIMG:cpu-basic-dev --build-arg BASE_IMG=ubuntu:20.04 --build-arg BZL_CONFIGS="" --build-arg KEEP_SRC_OTB=true . # cpu-basic-dev
- docker push $CI_REGISTRY_PUBIMG:cpu-basic-dev
- docker build --network='host' --tag $CI_REGISTRY_PUBIMG:gpu --build-arg BASE_IMG=nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04 . # gpu
- docker push $CI_REGISTRY_PUBIMG:gpu
- docker build --network='host' --tag $CI_REGISTRY_PUBIMG:gpu-dev --build-arg BASE_IMG=nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04 --build-arg KEEP_SRC_OTB=true . # gpu-dev
- docker push $CI_REGISTRY_PUBIMG:gpu-dev
- docker build --network='host' --tag $CI_REGISTRY_PUBIMG:gpu-basic --build-arg BASE_IMG=nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04 --build-arg BZL_CONFIGS="" . # gpu-basic
- docker push $CI_REGISTRY_PUBIMG:gpu-basic
- docker build --network='host' --tag $CI_REGISTRY_PUBIMG:gpu-basic-dev --build-arg BZL_CONFIGS="" --build-arg BASE_IMG=nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04 --build-arg KEEP_SRC_OTB=true . # gpu-basic-dev
- docker push $CI_REGISTRY_PUBIMG:gpu-basic-dev
.ship base:
extends: .docker_build_base
stage: Ship
only:
- master
deploy_cpu:
extends: .ship base
variables:
IMAGE_CPU: $CI_REGISTRY_PUBIMG:cpu
IMAGE_CPUDEV: $CI_REGISTRY_PUBIMG:cpu-dev
DOCKERHUB_CPU: $DOCKERHUB_IMAGE_BASE:cpu
DOCKERHUB_CPUDEV: $DOCKERHUB_IMAGE_BASE:cpu-dev
script:
# cpu
- docker build --network='host' --tag $IMAGE_CPU --build-arg BASE_IMG=ubuntu:20.04 --build-arg BZL_CONFIGS="" .
- docker push $IMAGE_CPU
# cpu-dev
- docker build --network='host' --tag $IMAGE_CPUDEV --build-arg BASE_IMG=ubuntu:20.04 --build-arg BZL_CONFIGS="" --build-arg KEEP_SRC_OTB=true .
- docker push $IMAGE_CPUDEV
# push images on dockerhub
- echo -n $DOCKERHUB_TOKEN | docker login -u mdl4eo --password-stdin
- docker tag $IMAGE_CPU $DOCKERHUB_CPU
- docker push $DOCKERHUB_CPU
- docker tag $IMAGE_CPUDEV $DOCKERHUB_CPUDEV
- docker push $DOCKERHUB_CPUDEV
deploy_gpu:
extends: .ship base
variables:
IMAGE_GPU: $CI_REGISTRY_PUBIMG:gpu
IMAGE_GPUDEV: $CI_REGISTRY_PUBIMG:gpu-dev
IMAGE_GPUOPT: $CI_REGISTRY_PUBIMG:gpu-opt
IMAGE_GPUOPTDEV: $CI_REGISTRY_PUBIMG:gpu-opt-dev
DOCKERHUB_GPU: $DOCKERHUB_IMAGE_BASE:gpu
DOCKERHUB_GPUDEV: $DOCKERHUB_IMAGE_BASE:gpu-dev
script:
# gpu-opt
- docker build --network='host' --tag $IMAGE_GPUOPT --build-arg BASE_IMG=nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04 .
- docker push $IMAGE_GPUOPT
# gpu-opt-dev
- docker build --network='host' --tag $IMAGE_GPUOPTDEV --build-arg BASE_IMG=nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04 --build-arg KEEP_SRC_OTB=true .
- docker push $IMAGE_GPUOPTDEV
# gpu-basic
- docker build --network='host' --tag $IMAGE_GPU --build-arg BASE_IMG=nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04 --build-arg BZL_CONFIGS="" .
- docker push $IMAGE_GPU
# gpu-basic-dev
- docker build --network='host' --tag $IMAGE_GPUDEV --build-arg BZL_CONFIGS="" --build-arg BASE_IMG=nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04 --build-arg KEEP_SRC_OTB=true .
- docker push $IMAGE_GPUDEV
# push gpu-basic* images on dockerhub
- echo -n $DOCKERHUB_TOKEN | docker login -u mdl4eo --password-stdin
- docker tag $IMAGE_GPU $DOCKERHUB_GPU
- docker push $DOCKERHUB_GPU
- docker tag $IMAGE_GPUDEV $DOCKERHUB_GPUDEV
- docker push $DOCKERHUB_GPUDEV
Version 3.2.1 (1 jun 2022)
----------------------------------------------------------------
* Changing docker images naming convention (cpu/gpu-basic* --> cpu/gpu*, cpu/gpu* --> cpu/gpu-opt*) + only images without optimizations are pushed on dockerhub
* Enhance CI with automatic builds of docker images
* Fix a bug in dataset.py (output type was always Float32)
* Fix a bug in `otbTensorflowSampler.hxx` (see https://github.com/remicres/otbtf/issues/81)
......
......@@ -3,6 +3,7 @@
### Available images
Here is the list of OTBTF docker images hosted on [dockerhub](https://hub.docker.com/u/mdl4eo).
Since OTBTF >= 3.2.1 you can find latest docker images on [gitlab.irstea.fr](https://gitlab.irstea.fr/remi.cresson/otbtf/container_registry).
| Name | Os | TF | OTB | Description | Dev files | Compute capability |
| --------------------------------- | ------------- | ------ | ----- | ---------------------- | --------- | ------------------ |
......@@ -30,11 +31,12 @@ Here is the list of OTBTF docker images hosted on [dockerhub](https://hub.docker
| **mdl4eo/otbtf3.1:gpu-basic-dev** | Ubuntu Focal | r2.8 | 7.4.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf3.1:gpu** | Ubuntu Focal | r2.8 | 7.4.0 | GPU | no | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf3.1:gpu-dev** | Ubuntu Focal | r2.8 | 7.4.0 | GPU (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
- `cpu` tagged docker images are compiled for CPU usage only.
- `xxx-basic` tagged docker images are compiled without CPU optimization flags. Other images are compiled with AVX, SSE, FMA.
- `gpu` tagged docker images are suited for **NVIDIA GPUs**. They use CUDA/CUDNN support.
- `cpu-mkl` tagged docker image is experimental, it is optimized for Intel CPUs with AVX512 flags.
| **mdl4eo/otbtf3.2.1:cpu** | Ubuntu Focal | r2.8 | 7.4.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf3.2.1:cpu-dev** | Ubuntu Focal | r2.8 | 7.4.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf3.2.1:gpu** | Ubuntu Focal | r2.8 | 7.4.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf3.2.1:gpu-dev** | Ubuntu Focal | r2.8 | 7.4.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf3.2.1:gpu-opt** | Ubuntu Focal | r2.8 | 7.4.0 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6|
| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf3.2.1:gpu-opt-dev** | Ubuntu Focal | r2.8 | 7.4.0 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
You can also find more interesting OTBTF flavored images at [LaTelescop gitlab registry](https://gitlab.com/latelescop/docker/otbtf/container_registry/).
......@@ -56,7 +58,7 @@ For instance, suppose you have some data in `/mnt/my_device/` that you want to u
The following command shows you how to access the folder from the docker image.
```bash
docker run -v /mnt/my_device/:/data/ -ti mdl4eo/otbtf3.1:cpu-basic bash -c "ls /data"
docker run -v /mnt/my_device/:/data/ -ti mdl4eo/otbtf3.2.1:cpu bash -c "ls /data"
```
Beware of ownership issues! see the last section of this doc.
......@@ -69,13 +71,13 @@ You can then use the OTBTF `gpu` tagged docker images with the **NVIDIA runtime*
With Docker version earlier than 19.03 :
```bash
docker run --runtime=nvidia -ti mdl4eo/otbtf3.1:gpu bash
docker run --runtime=nvidia -ti mdl4eo/otbtf3.2.1:gpu bash
```
With Docker version including and after 19.03 :
```bash
docker run --gpus all -ti mdl4eo/otbtf3.1:gpu bash
docker run --gpus all -ti mdl4eo/otbtf3.2.1:gpu bash
```
You can find some details on the **GPU docker image** and some **docker tips and tricks** on [this blog](https://mdl4eo.irstea.fr/2019/10/15/otbtf-docker-image-with-gpu/).
......@@ -88,7 +90,7 @@ Be careful though, these infos might be a bit outdated...
1. Install [WSL2](https://docs.microsoft.com/en-us/windows/wsl/install-win10#manual-installation-steps) (Windows Subsystem for Linux)
2. Install [docker desktop](https://www.docker.com/products/docker-desktop)
3. Start **docker desktop** and **enable WSL2** from *Settings* > *General* then tick the box *Use the WSL2 based engine*
3. Open a **cmd.exe** or **PowerShell** terminal, and type `docker create --name otbtf-cpu --interactive --tty mdl4eo/otbtf3.1:cpu`
3. Open a **cmd.exe** or **PowerShell** terminal, and type `docker create --name otbtf-cpu --interactive --tty mdl4eo/otbtf3.2.1:cpu`
4. Open **docker desktop**, and check that the docker is running in the **Container/Apps** menu
![Docker desktop, after the docker image is downloaded and ready to use](images/docker_desktop_1.jpeg)
5. From **docker desktop**, click on the icon highlighted as shown below, and use the bash terminal that should pop up!
......@@ -137,12 +139,12 @@ sudo systemctl {status,enable,disable,start,stop} docker
Run a simple command in a one-shot container:
```bash
docker run mdl4eo/otbtf3.1:cpu otbcli_PatchesExtraction
docker run mdl4eo/otbtf3.2.1:cpu otbcli_PatchesExtraction
```
You can also use the image in interactive mode with bash:
```bash
docker run -ti mdl4eo/otbtf3.1:cpu bash
docker run -ti mdl4eo/otbtf3.2.1:cpu bash
```
### Persistent container
......@@ -152,7 +154,7 @@ Beware of ownership issues, see the last section of this doc.
```bash
docker create --interactive --tty --volume /home/$USER:/home/otbuser/ \
--name otbtf mdl4eo/otbtf3.1:cpu /bin/bash
--name otbtf mdl4eo/otbtf3.2.1:cpu /bin/bash
```
### Interactive session
......@@ -216,7 +218,7 @@ Create a named container (here with your HOME as volume), Docker will automatica
```bash
docker create --interactive --tty --volume /home/$USER:/home/otbuser \
--name otbtf mdl4eo/otbtf3.1:cpu /bin/bash
--name otbtf mdl4eo/otbtf3.2.1:cpu /bin/bash
```
Start a background container process:
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment