diff --git a/Dockerfile b/Dockerfile index 383e51a8b1baa69fde1a8e06ad28edfb7b14f273..7343c01360afbdf9c77d7ad01f018255bb922e1c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,7 +40,7 @@ WORKDIR /src/tf RUN git config --global advice.detachedHead false ### TF -ARG TF=v2.4.1 +ARG TF=v2.5.0 # Install bazelisk (will read .bazelversion and download the right bazel binary - latest by default) RUN wget -qO /opt/otbtf/bin/bazelisk https://github.com/bazelbuild/bazelisk/releases/latest/download/bazelisk-linux-amd64 \ && chmod +x /opt/otbtf/bin/bazelisk \ @@ -83,7 +83,7 @@ RUN git clone --single-branch -b $TF https://github.com/tensorflow/tensorflow.gi ### OTB ARG GUI=false -ARG OTB=7.2.0 +ARG OTB=7.3.0 RUN mkdir /src/otb WORKDIR /src/otb @@ -97,7 +97,7 @@ RUN git clone --single-branch -b $OTB https://gitlab.orfeo-toolbox.org/orfeotool && if $GUI; then \ sed -i -r "s/-DOTB_USE_(QT|OPENGL|GL[UFE][WT])=OFF/-DOTB_USE_\1=ON/" ../build-flags-otb.txt; fi \ # Possible ENH: superbuild-all-dependencies switch, with separated build-deps-minimal.txt and build-deps-otbcli.txt) - #&& if $OTB_SUPERBUILD_ALL; then sed -i -r "s/-DUSE_SYSTEM_([A-Z0-9]*)=ON/-DUSE_SYSTEM_\1=OFF/"" ../build-flags-otb.txt; fi \ + #&& if $OTB_SUPERBUILD_ALL; then sed -i -r "s/-DUSE_SYSTEM_([A-Z0-9]*)=ON/-DUSE_SYSTEM_\1=OFF/ " ../build-flags-otb.txt; fi \ && OTB_FLAGS=$(cat "../build-flags-otb.txt") \ && cmake ../otb/SuperBuild -DCMAKE_INSTALL_PREFIX=/opt/otbtf $OTB_FLAGS \ && make -j $(python -c "import os; print(round( os.cpu_count() * $CPU_RATIO ))") diff --git a/tools/docker/README.md b/tools/docker/README.md index 8d6522dfc0e09a894bad97d1bc3841c41147bece..0b51a08239adf89d040a7555965b2341e5de37ae 100644 --- a/tools/docker/README.md +++ b/tools/docker/README.md @@ -8,7 +8,7 @@ You can build a custom image using `--build-arg` and several config files : ### Base images ```bash UBUNTU=20.04 # or 16.04, 18.04 -CUDA=11.0.3 # or 10.1, 10.2 +CUDA=11.2.2 # or 10.1, 10.2, 11.0.3 CUDNN=8 # or 7 IMG=ubuntu:$UBUNTU GPU_IMG=nvidia/cuda:$CUDA-cudnn$CUDNN-devel-ubuntu$UBUNTU @@ -17,11 +17,11 @@ GPU_IMG=nvidia/cuda:$CUDA-cudnn$CUDNN-devel-ubuntu$UBUNTU ### Default arguments ```bash BASE_IMG # mandatory -CPU_RATIO=0.95 +CPU_RATIO=1 GUI=false -NUMPY_SPEC="~=1.19" -TF=r2.4.1 -OTB=7.2.0 +NUMPY_SPEC="==1.19.*" +TF=v2.5.0 +OTB=7.3.0 BZL_TARGETS="//tensorflow:libtensorflow_cc.so //tensorflow/tools/pip_package:build_pip_package" BZL_CONFIGS="--config=nogcp --config=noaws --config=nohdfs --config=opt" BZL_OPTIONS="--verbose_failures --remote_cache=http://localhost:9090" @@ -31,7 +31,7 @@ SUDO=true # NumPy version requirement : # TF < 2.4 : "numpy<1.19.0,>=1.16.0" -# TF >= 2.4 : "numpy~=1.19" +# TF >= 2.4 : "numpy==1.19.*" ``` ### Bazel remote cache daemon @@ -61,11 +61,11 @@ docker build --network='host' -t otbtf:cpu-mkl --build-arg BZL_CONFIGS="$MKL_CON # Build for GPU (if you're building for your system only you should edit CUDA_COMPUTE_CAPABILITIES in build-env-tf.sh) docker build --network='host' -t otbtf:gpu --build-arg BASE_IMG=nvidia/cuda:11.0.3-cudnn8-devel-ubuntu20.04 . -# Build dev with TF and OTB sources (huge image) + set git branches/tags to clone -docker build --network='host' -t otbtf:gpu-dev-full --build-arg BASE_IMG=nvidia/cuda:11.0.3-cudnn8-devel-ubuntu20.04 \ - --build-arg KEEP_SRC_OTB=true --build-arg KEEP_SRC_TF=true --build-arg TF=nightly --build-arg OTB=develop . +# Build latest TF and OTB, set git branches/tags to clone +docker build --network='host' -t otbtf:gpu-dev --build-arg BASE_IMG=nvidia/cuda:11.2.2-cudnn8-devel-ubuntu20.04 \ + --build-arg KEEP_SRC_OTB=true --build-arg TF=nightly --build-arg OTB=develop . -# Build old release +# Build old release (TF-2.1) docker build --network='host' -t otbtf:oldstable-gpu --build-arg BASE_IMG=nvidia/cuda:10.1-cudnn7-devel-ubuntu18.04 \ --build-arg TF=r2.1 --build-arg NUMPY_SPEC="<1.19" \ --build-arg BAZEL_OPTIONS="--noincompatible_do_not_split_linking_cmdline --verbose_failures --remote_cache=http://localhost:9090" . @@ -80,7 +80,7 @@ If you see OOM errors during SuperBuild you should decrease CPU_RATIO (e.g. 0.75 ## Container examples ```bash # Pull GPU image and create a new container with your home directory as volume (requires apt package nvidia-docker2 and CUDA>=11.0) -docker create --gpus=all --volume $HOME:/home/otbuser/volume -it --name otbtf-gpu mdl4eo/otbtf2.1:gpu +docker create --gpus=all --volume $HOME:/home/otbuser/volume -it --name otbtf-gpu mdl4eo/otbtf2.4:gpu # Run interactive docker start -i otbtf-gpu @@ -92,7 +92,7 @@ docker exec otbtf-gpu python -c 'import tensorflow as tf; print(tf.test.is_gpu_a ### Rebuild OTB with more modules ```bash -docker create --gpus=all -it --name otbtf-gpu-dev mdl4eo/otbtf2.1:gpu-dev +docker create --gpus=all -it --name otbtf-gpu-dev mdl4eo/otbtf2.4:gpu-dev docker start -i otbtf-gpu-dev ``` ```bash @@ -114,7 +114,7 @@ docker start -i otbtf-gui $ mapla ``` -### Common errors +## Common errors Buid : `Error response from daemon: manifest for nvidia/cuda:11.0-cudnn8-devel-ubuntu20.04 not found: manifest unknown: manifest unknown` => Image is missing from dockerhub diff --git a/tools/docker/build-env-tf.sh b/tools/docker/build-env-tf.sh index 1169190dfe7a93cd15403057a0e1239ccb514ec6..b29f6c1ad94a6ca97c8d98b3da56661453073a1a 100644 --- a/tools/docker/build-env-tf.sh +++ b/tools/docker/build-env-tf.sh @@ -34,7 +34,7 @@ export CUDA_TOOLKIT_PATH=$(find /usr/local -maxdepth 1 -type d -name 'cuda-*') if [ ! -z $CUDA_TOOLKIT_PATH ] ; then export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$CUDA_TOOLKIT_PATH/lib64:$CUDA_TOOLKIT_PATH/lib64/stubs" export TF_CUDA_VERSION=$(echo $CUDA_TOOLKIT_PATH | sed -r 's/.*\/cuda-(.*)/\1/') - export TF_CUDA_COMPUTE_CAPABILITIES="5.2,6.1,7.0,7.5" + export TF_CUDA_COMPUTE_CAPABILITIES="5.2,6.1,7.0,7.5,8.6" export TF_NEED_CUDA=1 export TF_CUDA_CLANG=0 export TF_NEED_TENSORRT=0 diff --git a/tools/docker/multibuild.sh b/tools/docker/multibuild.sh index ad96461ad5905d16caec661adb3479dc4f23dfd2..21b68c291100c9c4ae54920ccb71377de5ffcd29 100644 --- a/tools/docker/multibuild.sh +++ b/tools/docker/multibuild.sh @@ -1,8 +1,8 @@ #!/bin/bash ### Docker multibuild and push, see default args and more examples in tools/docker/README.md -RELEASE=2.1 +RELEASE=2.5 UBUNTU=20.04 -CUDA=11.0.3 +CUDA=11.2.2 CUDNN=8 IMG=ubuntu:$UBUNTU GPU_IMG=nvidia/cuda:$CUDA-cudnn$CUDNN-devel-ubuntu$UBUNTU