From c2731bc24ead7f6122ccdfda7000def15e5edeaa Mon Sep 17 00:00:00 2001 From: Vincent Delbar <vincent.delbar@latelescop.fr> Date: Fri, 22 Jan 2021 17:52:03 +0100 Subject: [PATCH] ENH: Docker - last comments + version bumps --- Dockerfile | 45 +++++++++------- tools/docker/README.md | 102 ++++++++++++++++++++--------------- tools/docker/build-env-tf.sh | 30 +++++------ tools/docker/multibuild.sh | 47 ++++++++-------- 4 files changed, 125 insertions(+), 99 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1f011a7..f846f41 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ RUN if $GUI; then \ ### Python3 links and pip packages RUN ln -s /usr/bin/python3 /usr/local/bin/python && ln -s /usr/bin/pip3 /usr/local/bin/pip -# NumPy version is conflicting with system's gdal dep, may require venv +# NumPy version is conflicting with system's gdal dep and may require venv ARG NUMPY_SPEC="~=1.19" RUN pip install --no-cache-dir -U pip wheel mock six future "numpy$NUMPY_SPEC" \ && pip install --no-cache-dir --no-deps keras_applications keras_preprocessing @@ -31,23 +31,25 @@ RUN pip install --no-cache-dir -U pip wheel mock six future "numpy$NUMPY_SPEC" \ # ---------------------------------------------------------------------------- # Tmp builder stage - dangling cache should persist until "docker builder prune" FROM otbtf-base AS builder -# 0.75 may be required to avoid OOM errors (especially for OTB GUI) +# A smaller value may be required to avoid OOM errors when building OTB GUI ARG CPU_RATIO=1 RUN mkdir -p /src/tf /opt/otbtf/bin /opt/otbtf/include /opt/otbtf/lib WORKDIR /src/tf +RUN git config --global advice.detachedHead false + ### TF -ARG TF=v2.4.0 -# Install bazelisk (will read .bazelrc and download the right bazel version - latest by default) -RUN wget -O /opt/otbtf/bin/bazelisk https://github.com/bazelbuild/bazelisk/releases/latest/download/bazelisk-linux-amd64 \ - && chmod +x /opt/otbtf/bin/bazelisk-linux-amd64 \ - && ln -s /opt/otbtf/bin/bazelisk-linux-amd64 /opt/otbtf/bin/bazel +ARG TF=v2.4.1 +# Install bazelisk (will read .bazelversion and download the right bazel binary - latest by default) +RUN wget -qO /opt/otbtf/bin/bazelisk https://github.com/bazelbuild/bazelisk/releases/latest/download/bazelisk-linux-amd64 \ + && chmod +x /opt/otbtf/bin/bazelisk \ + && ln -s /opt/otbtf/bin/bazelisk /opt/otbtf/bin/bazel ARG BZL_TARGETS="//tensorflow:libtensorflow_cc.so //tensorflow/tools/pip_package:build_pip_package" -# --config=opt with bazel's default optimizations (otherwise edit CC_OPT_FLAGS in build-env-tf.sh) +# "--config=opt" will enable 'march=native' (otherwise edit CC_OPT_FLAGS in build-env-tf.sh) ARG BZL_CONFIGS="--config=nogcp --config=noaws --config=nohdfs --config=opt" -# --compilation_mode opt is already enabled by default (see tf repo /.bazelrc and /configure.py) +# "--compilation_mode opt" is already enabled by default (see tf repo .bazelrc and configure.py) ARG BZL_OPTIONS="--verbose_failures --remote_cache=http://localhost:9090" # Build @@ -63,7 +65,7 @@ RUN git clone --single-branch -b $TF https://github.com/tensorflow/tensorflow.gi && export TMP=/tmp/bazel \ && BZL_CMD="build $BZL_TARGETS $BZL_CONFIGS $BZL_OPTIONS" \ && bazel $BZL_CMD --jobs="HOST_CPUS*$CPU_RATIO" ' \ -# Installation - split here if you need to debug ^ +# Installation - split here if you want to check files ^ #RUN cd tensorflow \ && ./bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_pkg \ && pip3 install --no-cache-dir --prefix=/opt/otbtf /tmp/tensorflow_pkg/tensorflow*.whl \ @@ -75,7 +77,7 @@ RUN git clone --single-branch -b $TF https://github.com/tensorflow/tensorflow.gi # Symlink external libs (required for MKL - libiomp5) && for f in $(find -L /opt/otbtf/include/tf -wholename "*/external/*/*.so"); do ln -s $f /opt/otbtf/lib/; done \ # Cleaning - && mv /root/.cache/bazel* /src/tf/ \ + && rm -rf bazel-* \ && ( $KEEP_SRC_TF || rm -rf /src/tf ) \ && rm -rf /root/.cache/ /tmp/* @@ -94,11 +96,13 @@ RUN git clone --single-branch -b $OTB https://gitlab.orfeo-toolbox.org/orfeotool # Set GL/Qt build flags && if $GUI; then \ sed -i -r "s/-DOTB_USE_(QT|OPENGL|GL[UFE][WT])=OFF/-DOTB_USE_\1=ON/" ../build-flags-otb.txt; fi \ + # Possible ENH: superbuild-all-dependencies switch, with separated build-deps-minimal.txt and build-deps-otbcli.txt) + #&& if $OTB_SUPERBUILD_ALL; then sed -i -r "s/-DOTB_USE_SYSTEM_([A-Z0-9]*)=ON/-DOTB_USE_SYSTEM_\1=OFF/"" ../build-flags-otb.txt; fi \ && OTB_FLAGS=$(cat "../build-flags-otb.txt") \ && cmake ../otb/SuperBuild -DCMAKE_INSTALL_PREFIX=/opt/otbtf $OTB_FLAGS \ && make -j $(python -c "import os; print(round( os.cpu_count() * $CPU_RATIO ))") -### OTBTF - copy (without .git) or clone repo +### OTBTF - copy (without .git/) or clone repository COPY . /src/otbtf #RUN git clone https://github.com/remicres/otbtf.git /src/otbtf RUN ln -s /src/otbtf /src/otb/otb/Modules/Remote/otbtf @@ -113,6 +117,7 @@ RUN cd /src/otb/build/OTB/build \ -DOTB_WRAP_PYTHON=ON -DPYTHON_EXECUTABLE=/usr/bin/python3 \ -DOTB_USE_TENSORFLOW=ON -DModule_OTBTensorflow=ON \ -Dtensorflow_include_dir=/opt/otbtf/include/tf \ + # Forcing TF>=2, this Dockerfile hasn't been tested with v1 + missing link for libtensorflow_framework.so in the wheel -DTENSORFLOW_CC_LIB=/opt/otbtf/lib/libtensorflow_cc.so.2 \ -DTENSORFLOW_FRAMEWORK_LIB=/opt/otbtf/lib/python3/site-packages/tensorflow/libtensorflow_framework.so.2 \ && make install -j $(python -c "import os; print(round( os.cpu_count() * $CPU_RATIO ))") \ @@ -129,11 +134,9 @@ RUN for f in /src/otbtf/python/*.py; do if [ -x $f ]; then ln -s $f /opt/otbtf/b FROM otbtf-base MAINTAINER Remi Cresson <remi.cresson[at]inrae[dot]fr> -COPY --from=builder /opt/otbtf /opt/ -COPY --from=builder /src / -# Relocate ~/.cache/bazel and ~/.cache/bazelisk -RUN if [ -d /src/tf/bazel ]; then \ - mkdir -p /root/.cache && mv /src/tf/bazel* /root/.cache/ +# Copy files from intermediate stage +COPY --from=builder /opt/otbtf /opt/otbtf +COPY --from=builder /src /src # System-wide ENV ENV PATH="/opt/otbtf/bin:$PATH" @@ -141,7 +144,7 @@ ENV LD_LIBRARY_PATH="/opt/otbtf/lib:$LD_LIBRARY_PATH" ENV PYTHONPATH="/opt/otbtf/lib/python3/site-packages:/opt/otbtf/lib/otb/python:/src/otbtf/python" ENV OTB_APPLICATION_PATH="/opt/otbtf/lib/otb/applications" -# Default user, directory and command (bash = 'docker create' entrypoint) +# Default user, directory and command (bash is the entrypoint when using 'docker create') RUN useradd -s /bin/bash -m otbuser WORKDIR /home/otbuser @@ -151,7 +154,7 @@ RUN if $SUDO; then \ usermod -a -G sudo otbuser \ && echo "otbuser ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers; fi -# Set /src/otbtf ownership to otbuser (but you will need 'sudo -i' in order to rebuild TF or OTB) +# Set /src/otbtf ownership to otbuser (but you still need 'sudo -i' in order to rebuild TF or OTB) RUN chown -R otbuser:otbuser /src/otbtf # This won't prevent ownership problems with volumes if you're not UID 1000 @@ -159,4 +162,6 @@ USER otbuser # User-only ENV # Test python imports -RUN python -c "import numpy, tensorflow, otbtf, tricks, otbApplication" +RUN python -c "import tensorflow" +RUN python -c "import otbtf, tricks" +RUN python -c "import otbApplication as otb; otb.Registry.CreateApplication('ImageClassifierFromDeepFeatures')" diff --git a/tools/docker/README.md b/tools/docker/README.md index f12476a..b343787 100644 --- a/tools/docker/README.md +++ b/tools/docker/README.md @@ -1,80 +1,85 @@ -# Docker multi-stage build with external bazel cache +# Build with Docker Docker build has to be called from the root of the repository (i.e. `docker build .` or `bash tools/docker/multibuild.sh`). -You may build a custom docker image using `--build-arg` and the config files in this directory. -For TensorFlow, see the `TF` arg for the git branch/tag, [build-env-tf.sh](build-env-tf.sh) and BZL_* arguments for the build configuration. -Regarding OTB, you can edit cmake flags in [build-flags-otb.txt](build-flags-otb.txt) and the `OTB` argument for the git branch/tag to clone. -If you need additional Ubuntu packages see [build-deps-cli.txt](build-deps-cli.txt) and [build-deps-gui.txt](build-deps-gui.txt) for GUI related packages - it is disabled by default in order to save space, and because docker xvfb isn't working properly with opengl. +You can build a custom image using `--build-arg` and several config files : +- Ubuntu : `BASE_IMG` should accept any version, for additional packages see [build-deps-cli.txt](build-deps-cli.txt) and [build-deps-gui.txt](build-deps-gui.txt) +- TensorFlow : `TF` arg for the git branch or tag + [build-env-tf.sh](build-env-tf.sh) and BZL_* arguments for the build configuration +- OrfeoToolBox : `OTB` arg for the git branch or tag + [build-flags-otb.txt](build-flags-otb.txt) to edit cmake flags -## Default arguments +### Base images +```bash +UBUNTU=20.04 # or 16.04, 18.04 +CUDA=11.0.3 # or 10.1, 10.2 +CUDNN=8 # or 7 +IMG=ubuntu:$UBUNTU +GPU_IMG=nvidia/cuda:$CUDA-cudnn$CUDNN-devel-ubuntu$UBUNTU ``` -BASE_IMG (mandatory) + +### Default arguments +```bash +BASE_IMG # mandatory CPU_RATIO=0.95 GUI=false NUMPY_SPEC="~=1.19" -TF=r2.4.0 +TF=r2.4.1 OTB=7.2.0 -BZL_TARGETS="//tensorflow:libtensorflow_cc.so //tensorflow:libtensorflow_framework.so //tensorflow/tools/pip_package:build_pip_package" +BZL_TARGETS="//tensorflow:libtensorflow_cc.so //tensorflow/tools/pip_package:build_pip_package" BZL_CONFIGS="--config=nogcp --config=noaws --config=nohdfs --config=opt" BZL_OPTIONS="--verbose_failures --remote_cache=http://localhost:9090" KEEP_SRC_TF=false KEEP_SRC_OTB=false SUDO=true + +# NumPy version requirement : +# TF < 2.4 : "numpy<1.19.0,>=1.16.0" +# TF >= 2.4 : "numpy~=1.19" ``` -## Bazel remote cache daemon -There is no way make a common build of OTB shared between docker builds, since we're using different BASE_IMG and because of the multi-stage Dockerfile. -But if you just need to rebuild with different GUI or KEEP_SRC arguments, or may be a different branch of OTB, bazel cache may help you to rebuild everything except TF, even if docker cache was purged (after `docker system prune`). -In order to recycle the cache, bazel config and TF git tag has to be exactly the same, any change in [build-env-tf.sh](build-env-tf.sh) and `--build-arg` (if related to bazel env, cuda, mkl, xla...) may result in a complete new build. +### Bazel remote cache daemon +If you just need to rebuild with different GUI or KEEP_SRC arguments, or may be a different branch of OTB, bazel cache will help you to rebuild everything except TF, even if the docker cache was purged (after `docker [system|builder] prune`). +In order to recycle the cache, bazel config and TF git tag should be exactly the same, any change in [build-env-tf.sh](build-env-tf.sh) and `--build-arg` (if related to bazel env, cuda, mkl, xla...) may result in a fresh new build. Start a cache daemon - here with max 20GB but 12GB should be enough to save 2 TF builds (GPU and CPU): ```bash mkdir -p $HOME/.cache/bazel-remote docker run --detach -u 1000:1000 -v $HOME/.cache/bazel-remote:/data -p 9090:8080 buchgr/bazel-remote-cache --max_size=20 ``` -Then just add ` --network='host'` to the docker build command, or connect bazel to another adress (see the 'BZL_OPTIONS' build argument - the other way of docker is a virtual bridge, you'll need to edit the IP address). - +Then just add ` --network='host'` to the docker build command, or connect bazel to a remote server - see 'BZL_OPTIONS'. +The other way of docker is a virtual bridge, but you'll need to edit the IP address. ## Build examples ```bash -# Build for CPU using default Dockerfiles args (without AWS, HDFS and GCP support) -docker build --network='host' -t otbtf:cpu --build-arg BASE_IMG=ubuntu:18.04 . +# Build for CPU using default Dockerfiles args (without AWS, HDFS or GCP support) +docker build --network='host' -t otbtf:cpu --build-arg BASE_IMG=ubuntu:20.04 . -# Clear bazel config var (deactivate compilation optimizations and unset noaws/nogcp/nohdfs) -docker build --network='host' -t otbtf:cpu --build-arg BASE_IMG=ubuntu:18.04 --build-arg BZL_CONFIGS= . +# Clear bazel config var (deactivate default optimizations and unset noaws/nogcp/nohdfs) +docker build --network='host' -t otbtf:cpu --build-arg BASE_IMG=ubuntu:20.04 --build-arg BZL_CONFIGS= . # Enable MKL -MKL_CONFIG="--config=nogcp --config=noaws --config=nohdfs --config=opt --config=mkl --copt='-mfpmath=both'" -docker build --network='host' -t otbtf:cpu-mkl --build-arg BZL_CONFIGS="$MKL_CONFIG" --build-arg BASE_IMG=ubuntu:18.04 . +MKL_CONFIG="--config=nogcp --config=noaws --config=nohdfs --config=opt --config=mkl" +docker build --network='host' -t otbtf:cpu-mkl --build-arg BZL_CONFIGS="$MKL_CONFIG" --build-arg BASE_IMG=ubuntu:20.04 . -# Build for GPU (if you're building for your system only you should change the CUDA_COMPUTE_CAPABILTIES in build-env-tf.sh) -docker build --network='host' -t otbtf:gpu --build-arg BASE_IMG=nvidia/cuda:11.0-cudnn8-devel-ubuntu18.04 . +# Build for GPU (if you're building for your system only you should edit CUDA_COMPUTE_CAPABILITIES in build-env-tf.sh) +docker build --network='host' -t otbtf:gpu --build-arg BASE_IMG=nvidia/cuda:11.0.3-cudnn8-devel-ubuntu20.04 . -# Build dev with TF and OTB sources (huge image) + set git branch/tags to clone -docker build --network='host' -t otbtf:gpu-dev-full --build-arg BASE_IMG=nvidia/cuda:11.0-cudnn8-devel-ubuntu18.04 \ - --build-arg KEEP_SRC_OTB=true --buid-arg KEEP_SRC_TF=true --build-arg TF=r2.4 --build-arg OTB=develop . +# Build dev with TF and OTB sources (huge image) + set git branches/tags to clone +docker build --network='host' -t otbtf:gpu-dev-full --build-arg BASE_IMG=nvidia/cuda:11.0.3-cudnn8-devel-ubuntu20.04 \ + --build-arg KEEP_SRC_OTB=true --buid-arg KEEP_SRC_TF=true --build-arg TF=nightly --build-arg OTB=develop . # Build old release docker build --network='host' -t otbtf:oldstable-gpu --build-arg BASE_IMG=nvidia/cuda:10.1-cudnn7-devel-ubuntu18.04 \ - --build-arg TF=r2.1 --build-arg NUMPY_SPEC="<1.19" --build-arg OTB=release-7.2 \ + --build-arg TF=r2.1 --build-arg NUMPY_SPEC="<1.19" \ --build-arg BAZEL_OPTIONS="--noincompatible_do_not_split_linking_cmdline --verbose_failures --remote_cache=http://localhost:9090" . -# You may edit the Dockerfile to clone an old branch of the repo instead of copying files from the build context - -# Numpy version requirement : -# TF < 2.4 ==> numpy<1.19.0,>=1.16.0 -# TF >= 2.4 ==> numpy~=1.19 +# You could edit the Dockerfile in order to clone an old branch of the repo instead of copying files from the build context ``` ### Debug build -If you fail to build, you can log into the last layer and check CMake logs. -Run `docker images`, find the latest layer ID and run a tmp container (`docker run -it d60496d9612e bash`). +If you fail to build, you can log into the last layer and check CMake logs. Run `docker images`, find the latest layer ID and run a tmp container (`docker run -it d60496d9612e bash`). You may also need to split some multi-command layers in the Dockerfile. If you see OOM errors during SuperBuild you should decrease CPU_RATIO (e.g. 0.75). - ## Container examples ```bash -# Pull GPU image and create container with your home directory as volume -# (requires apt package nvidia-docker2 and CUDA>=11.0) +# Pull GPU image and create a new container with your home directory as volume (requires apt package nvidia-docker2 and CUDA>=11.0) docker create --gpus=all --volume $HOME:/home/otbuser/volume -it --name otbtf-gpu mdl4eo/otbtf2.1:gpu # Run interactive @@ -83,13 +88,15 @@ docker start -i otbtf-gpu # Run in background docker start otbtf-gpu docker exec otbtf-gpu python -c 'import tensorflow as tf; print(tf.test.is_gpu_available())' +``` -# Rebuild OTB with more modules (e.g. otbSelectiveHaralickTextures) +### Rebuild OTB with more modules +```bash docker create --gpus=all -it --name otbtf-gpu-dev mdl4eo/otbtf2.1:gpu-dev docker start -i otbtf-gpu-dev ``` -In the container shell: ```bash +# From the container shell: sudo -i cd /src/otb/otb/Modules/Remote git clone https://gitlab.irstea.fr/raffaele.gaetano/otbSelectiveHaralickTextures.git @@ -97,12 +104,21 @@ cd /src/otb/build/OTB/build cmake -DModule_OTBAppSelectiveHaralickTextures=ON /src/otb/otb && make install -j ``` - -## GUI +### Container with GUI ```bash -# With GUI (disabled by default): otbgui seems ok but monteverdi (OpenGL) isn't working -docker build --network='host' -t otbtf:cpu-gui --build-arg BASE_IMG=ubuntu:18.04 --build-arg GUI=true . +# GUI is disabled by default in order to save space, and because docker xvfb isn't working properly with OpenGL. +# => otbgui seems OK but monteverdi isn't working +docker build --network='host' -t otbtf:cpu-gui --build-arg BASE_IMG=ubuntu:20.04 --build-arg GUI=true . docker create -v /tmp/.X11-unix:/tmp/.X11-unix -e DISPLAY=$DISPLAY -it --name otbtf-gui otbtf:cpu-gui docker start -i otbtf-gui $ mapla ``` + +### Common errors +Buid : +`Error response from daemon: manifest for nvidia/cuda:11.0-cudnn8-devel-ubuntu20.04 not found: manifest unknown: manifest unknown` +=> Image is missing from dockerhub + +Run : +`failed call to cuInit: UNKNOWN ERROR (303) / no NVIDIA GPU device is present: /dev/nvidia0 does not exist` +=> Nvidia driver is missing or disabled, make sure to add ` --gpus=all` to your docker run or create command diff --git a/tools/docker/build-env-tf.sh b/tools/docker/build-env-tf.sh index 76adc8c..406b5dd 100644 --- a/tools/docker/build-env-tf.sh +++ b/tools/docker/build-env-tf.sh @@ -1,40 +1,40 @@ -# TF - bazel build env variables +### TF - bazel build env variables -# Optimization is controlled with bazel --config=opt, will set AVX and SSE flags on linux, without '-march=native' (disabled in TF2.4) -# Just uncomment CC_OPT_FLAGS or append --copt='-march=native' to the BZL_CONFIGS arg in case you need it -#export CC_OPT_FLAGS="-march=native -Wno-sign-compare" +# As in official TF wheels, you'll need to remove "-march=native" for old CPUs compatibity (no AVX2) +export CC_OPT_FLAGS="-march=native -Wno-sign-compare" export GCC_HOST_COMPILER_PATH=$(which gcc) export PYTHON_BIN_PATH=$(which python) export PYTHON_LIB_PATH="$($PYTHON_BIN_PATH -c 'import site; print(site.getsitepackages()[0])')" +export TF_DOWNLOAD_CLANG=0 export TF_ENABLE_XLA=1 -export TF_NEED_MPI=0 +export TF_NEED_COMPUTECPP=0 export TF_NEED_GDR=0 +export TF_NEED_JEMALLOC=1 export TF_NEED_KAFKA=0 +export TF_NEED_MPI=0 export TF_NEED_OPENCL=0 -export TF_NEED_JEMALLOC=1 -export TF_NEED_VERBS=0 export TF_NEED_OPENCL_SYCL=0 -export TF_NEED_COMPUTECPP=0 -export TF_NEED_ROCM=0 +export TF_NEED_VERBS=0 export TF_SET_ANDROID_WORKSPACE=0 -# We need to set BZL_CONFIGS=" --config=nogcp --config=noaws --config=nohdfs" +# For MKL support BZL_CONFIGS+=" --config=mkl" +#export TF_DOWNLOAD_MKL=1 +#export TF_NEED_MKL=0 +# Needed BZL_CONFIGS=" --config=nogcp --config=noaws --config=nohdfs" #export TF_NEED_S3=0 #export TF_NEED_AWS=0 #export TF_NEED_GCP=0 #export TF_NEED_HDFS=0 -# For MKL support BZL_CONFIGS+=" --config=mkl --config=opt --copt='-mfpmath=both'" -#export TF_DOWNLOAD_MKL=1 -#export TF_NEED_MKL=0 -# CUDA +## GPU +export TF_NEED_ROCM=0 export TF_NEED_CUDA=0 export CUDA_TOOLKIT_PATH=$(find /usr/local -maxdepth 1 -type d -name 'cuda-*') if [ ! -z $CUDA_TOOLKIT_PATH ] ; then export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$CUDA_TOOLKIT_PATH/lib64:$CUDA_TOOLKIT_PATH/lib64/stubs" export TF_CUDA_VERSION=$(echo $CUDA_TOOLKIT_PATH | sed -r 's/.*\/cuda-(.*)/\1/') - export TF_CUDA_CLANG=0 export TF_CUDA_COMPUTE_CAPABILITIES="5.2,6.1,7.0,7.5" export TF_NEED_CUDA=1 + export TF_CUDA_CLANG=0 export TF_NEED_TENSORRT=0 export CUDNN_INSTALL_PATH="/usr/" export TF_CUDNN_VERSION=$(sed -n 's/^#define CUDNN_MAJOR\s*\(.*\).*/\1/p' $CUDNN_INSTALL_PATH/include/cudnn.h) diff --git a/tools/docker/multibuild.sh b/tools/docker/multibuild.sh index 72cd284..ad96461 100644 --- a/tools/docker/multibuild.sh +++ b/tools/docker/multibuild.sh @@ -1,36 +1,41 @@ #!/bin/bash -# Batch several docker -# See default args and more examples in tools/docker/README.md +### Docker multibuild and push, see default args and more examples in tools/docker/README.md RELEASE=2.1 -UBUNTU=18.04 -CUDA=11.0 +UBUNTU=20.04 +CUDA=11.0.3 CUDNN=8 IMG=ubuntu:$UBUNTU GPU_IMG=nvidia/cuda:$CUDA-cudnn$CUDNN-devel-ubuntu$UBUNTU -# ubuntu20.04 (python3.8) should work but 11.0-cudnn8-devel-ubuntu20.04 is missing from nvidia's docker repo -# Bazel remote cache daemon +## Bazel remote cache daemon mkdir -p $HOME/.cache/bazel-remote docker run -d -u 1000:1000 -v $HOME/.cache/bazel-remote:/data -p 9090:8080 buchgr/bazel-remote-cache --max_size=20 -# CPU (no MKL) -docker build --network='host' -t mdl4eo/otbtf$RELEASE:cpu --build-arg BASE_IMG=$IMG . +### CPU (no MKL) +#docker build --network='host' -t mdl4eo/otbtf$RELEASE:-cpu-dev-all --build-arg BASE_IMG=$IMG --build-arg KEEP_SRC_OTB=true --build-arg KEEP_SRC_TF=true . docker build --network='host' -t mdl4eo/otbtf$RELEASE:cpu-dev --build-arg BASE_IMG=$IMG --build-arg KEEP_SRC_OTB=true . -# Enable MKL with bazel config flag (tested on CNN : actually slower than a normal CPU build) -#MKL_CONF="--config=nogcp --config=noaws --config=nohdfs --config=mkl --config=opt --copt='-mfpmath=both' --copt='-march=native'" -#docker build --network='host' -t mdl4eo/otbtf$RELEASE:cpu --build-arg BASE_IMG=$IMG --build-arg BZL_CONFIG="$MKL_CONF" . -# Keep OTB src and build files in order to rebuild with other modules -#docker build --network='host' -t mdl4eo/otbtf$RELEASE:cpu-dev --build-arg BASE_IMG=$IMG --build-arg BZL_CONFIG="$MKL_CONF" --build-arg KEEP_SRC_OTB=true . +docker build --network='host' -t mdl4eo/otbtf$RELEASE:cpu --build-arg BASE_IMG=$IMG . +#docker build --network='host' -t mdl4eo/otbtf$RELEASE:-cpu-gui --build-arg BASE_IMG=$IMG --build-arg GUI=true . -# GPU support is enabled if CUDA is found in /usr/local -docker build --network='host' -t mdl4eo/otbtf$RELEASE:gpu --build-arg BASE_IMG=$GPU_IMG . +### MKL is enabled with bazel config flag +#MKL_CONF="--config=nogcp --config=noaws --config=nohdfs --config=mkl --config=opt" +#docker build --network='host' -t mdl4eo/otbtf$RELEASE:-cpu-mkl --build-arg BASE_IMG=$IMG --build-arg BZL_CONFIGS="$MKL_CONF" . +#docker build --network='host' -t mdl4eo/otbtf$RELEASE:-cpu-mkl-dev --build-arg BASE_IMG=$IMG --build-arg BZL_CONFIGS="$MKL_CONF" --build-arg KEEP_SRC_OTB=true . + +### GPU support is enabled if CUDA is found in /usr/local +#docker build --network='host' -t mdl4eo/otbtf$RELEASE:-gpu-dev-all --build-arg BASE_IMG=$GPU_IMG --build-arg KEEP_SRC_OTB=true --build-arg KEEP_SRC_TF=true . docker build --network='host' -t mdl4eo/otbtf$RELEASE:gpu-dev --build-arg BASE_IMG=$GPU_IMG --build-arg KEEP_SRC_OTB=true . +docker build --network='host' -t mdl4eo/otbtf$RELEASE:gpu --build-arg BASE_IMG=$GPU_IMG . +#docker build --network='host' -t mdl4eo/otbtf$RELEASE:-gpu-gui --build-arg BASE_IMG=$GPU_IMG --build-arg GUI=true . #docker login -docker push mdl4eo/otbtf$RELEASE:cpu -docker push mdl4eo/otbtf$RELEASE:cpu-dev -#docker push mdl4eo/otbtf$RELEASE:cpu-gui +#docker push mdl4eo/otbtf$RELEASE:-cpu-dev-all +docker push mdl4eo/otbtf$RELEASE:-cpu-dev +docker push mdl4eo/otbtf$RELEASE:-cpu +#docker push mdl4eo/otbtf$RELEASE:-cpu-gui +#docker push mdl4eo/otbtf$RELEASE:-cpu-mkl -docker push mdl4eo/otbtf$RELEASE:gpu -docker push mdl4eo/otbtf$RELEASE:gpu-dev -#docker push mdl4eo/otbtf$RELEASE:gpu-gui +#docker push mdl4eo/otbtf$RELEASE:-gpu-dev-all +docker push mdl4eo/otbtf$RELEASE:-gpu-dev +docker push mdl4eo/otbtf$RELEASE:-gpu +#docker push mdl4eo/otbtf$RELEASE:-gpu-gui -- GitLab