Commit b3c7f92e authored by Gaetano Raffaele's avatar Gaetano Raffaele
Browse files

Merge branch 'develop' into s1process_light

parents a210702d 3bc1e253
**/Dockerfile.* gitlab-language=docker
FROM ubuntu:20.04
ARG BASE_IMG=ubuntu:20.04
# Base image can be either focal (20.04) or bionic (18.04) and descendants
FROM $BASE_IMG
LABEL description="A base image with OTB and remote modules for the MORINGA processing chain, CIRAD/UMR TETIS"
# Build env
WORKDIR /tmp
ARG OTB_VERSION=7.2
# System dependencies
# Controls USE_SYSTEM_* variables
ARG USE_SYSTEM_DEPS=false
# DEPS_INSTALL_PREFIX=/usr is required when USE_SYSTEM_DEPS=true
ARG DEPS_INSTALL_PREFIX=/opt/otb
# Install build dependencies
RUN apt-get update \
&& DEBIAN_FRONTEND=noninteractive \
apt-get install -y --no-install-recommends \
&& export DEBIAN_FRONTEND=noninteractive \
&& apt-get install -y --no-install-recommends \
bzip2 \
cmake \
curl \
......@@ -31,32 +36,66 @@ RUN apt-get update \
vim \
wget \
zip \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
&& if $USE_SYSTEM_DEPS; then \
apt-get install -y --no-install-recommends \
bison \
gdal-bin \
python3-gdal \
libboost-date-time-dev \
libboost-filesystem-dev \
libboost-graph-dev \
libboost-program-options-dev \
libboost-system-dev \
libboost-thread-dev \
libcurl4-gnutls-dev \
libexpat1-dev \
libfftw3-dev \
libgdal-dev \
libgeotiff-dev \
libgsl-dev \
libinsighttoolkit4-dev \
libkml-dev \
libmuparser-dev \
libmuparserx-dev \
libopencv-core-dev \
libopencv-ml-dev \
libopenthreads-dev \
libossim-dev \
libpng-dev \
libsvm-dev \
libtinyxml-dev \
zlib1g-dev; fi \
&& apt-get clean && rm -rf /var/lib/apt/lists/*
# OTB branch to clone
ARG OTB_BRANCH=release-7.2
# Set USE_SYSTEM_* flags
COPY docker/build-flags-otb.txt .
RUN if ! $USE_SYSTEM_DEPS; then \
sed -i -r "s/-DUSE_SYSTEM_([A-Z0-9]*)=ON/-DUSE_SYSTEM_\1=OFF/" build-flags-otb.txt ; fi
# SuperBuild OTB with remote modules, no GUI, no system libs
RUN git clone -b release-$OTB_VERSION "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb.git" \
&& mkdir /tmp/SuperBuild-archives /tmp/build /opt/otb \
# SuperBuild OTB
RUN mkdir /tmp/SuperBuild-archives /tmp/build /opt/otb \
&& git clone --single-branch -b $OTB_BRANCH "https://gitlab.orfeo-toolbox.org/orfeotoolbox/otb.git" \
&& if ! $USE_SYSTEM_DEPS; then \
cd SuperBuild-archives \
&& OTB_VERSION=$(head -n1 /tmp/otb/RELEASE_NOTES.txt | sed -r 's/OTB-v ([0-9].[0-9]).[0-9] -.*/\1/') \
&& curl -s "https://www.orfeo-toolbox.org/packages/archives/OTB/SuperBuild-archives-$OTB_VERSION.tar.bz2" \
| tar -xjv --exclude 'q*' --exclude '*gl*'; fi \
&& cd /tmp/build \
# Build dependencies
&& OTB_CMAKE_FLAGS=$(cat "../build-flags-otb.txt") \
&& cmake /tmp/otb/SuperBuild \
-DCMAKE_BUILD_TYPE='Release' \
-DOTB_USE_QT=OFF \
-DOTB_USE_OPENGL=OFF \
-DOTB_USE_GLUT=OFF \
-DOTB_USE_GLEW=OFF \
-DOTB_USE_GLFW=OFF \
$OTB_CMAKE_FLAGS \
-DCMAKE_BUILD_TYPE="Release" \
-DDOWNLOAD_LOCATION=/tmp/SuperBuild-archives \
-DCMAKE_INSTALL_PREFIX=/opt/otb \
&& make OTB_DEPENDS -j4 \
# Clone modules
&& make OTB_DEPENDS -j12 \
&& cd /tmp/otb/Modules/Remote \
&& rm -f otbGRM.remote.cmake \
&& git clone "https://gitlab.irstea.fr/remi.cresson/GRM.git" \
&& git clone "https://gitlab.irstea.fr/remi.cresson/LSGRM.git" \
&& git clone "https://gitlab.irstea.fr/raffaele.gaetano/otbVectorClassification.git" \
&& git clone "https://gitlab.irstea.fr/raffaele.gaetano/otbSelectiveHaralickTextures.git" \
# Build OTB and modules
&& cd /tmp/build/OTB/build \
&& cmake /tmp/otb \
-DModule_otbGRM=ON \
......@@ -66,18 +105,16 @@ RUN git clone -b release-$OTB_VERSION "https://gitlab.orfeo-toolbox.org/orfeotoo
-DModule_OTBTemporalGapFilling=ON \
-DModule_S1TilingSupportApplications=ON \
-DOTB_WRAP_PYTHON=ON \
-DPYTHON_EXECUTABLE=/usr/bin/python3.8 \
-DPYTHON_EXECUTABLE=/usr/bin/python3 \
-DCMAKE_INSTALL_PREFIX=/opt/otb \
&& cd /tmp/build \
&& make -j8 \
# For a lightweight layer
&& rm -rf /tmp/SuperBuild-archives /tmp/otb* /tmp/build /root/.cache /opt/otb/bin/otbgui_*
&& rm -rf /tmp/* /root/.cache /opt/otb/bin/otbgui_*
# Persistent environment variables
ENV PATH=/opt/otb/bin:$PATH
ENV PYTHONPATH=/opt/otb/lib/python3/dist-packages:/opt/otb/lib/otb/python:$PYTHONPATH
ENV LD_LIBRARY_PATH=/opt/otb/lib:$LD_LIBRARY_PATH
ENV OTB_APPLICATION_PATH=/opt/otb/lib/otb/applications
ENV PROJ_LIB=/opt/otb/share/proj
ENV GDAL_DATA=/opt/otb/share/gdal
ENV GEOTIFF_CSV=/opt/otb/share/epsg_csv
ENV PATH="/opt/otb/bin:$PATH"
ENV PYTHONPATH="/opt/otb/lib/python3/dist-packages:/opt/otb/lib/otb/python:$PYTHONPATH"
ENV LD_LIBRARY_PATH="/opt/otb/lib:$LD_LIBRARY_PATH"
ENV OTB_APPLICATION_PATH="/opt/otb/lib/otb/applications"
ENV PROJ_LIB="$DEPS_INSTALL_PREFIX/share/proj"
ENV GDAL_DATA="$DEPS_INSTALL_PREFIX/share/gdal"
ARG BASE_IMG=gitlab-registry.irstea.fr/raffaele.gaetano/moringa:base-otb
ARG BASE_IMG=gitlab-registry.irstea.fr/raffaele.gaetano/moringa/base:latest
FROM $BASE_IMG
LABEL description="A container for the MORINGA processing chain, CIRAD/UMR TETIS"
LABEL maintainer="Raffaele GAETANO"
# Compile obiatools library
RUN git clone "https://gitlab.irstea.fr/raffaele.gaetano/obiatools.git" && cd obiatools/Release \
&& sed -i 's/$HOME\/moringa/\/usr\/local\/lib\/python3.8\/dist-packages/g' configure.sh \
&& sed -i 's/python3.6/python3.8/g' configure.sh \
&& mkdir /usr/local/lib/python3.8/dist-packages/sitsproc_c_modules \
&& touch /usr/local/lib/python3.8/dist-packages/sitsproc_c_modules/__init__.py \
&& bash -c "source configure.sh && make" && cd ../.. && rm -rf obiatools
# Other python deps: rios, fmask
# PIP dependencies : rios, fmask
RUN pip3 install rios.core \
&& wget -q "https://github.com/ubarsc/python-fmask/releases/download/pythonfmask-0.5.5/python-fmask-0.5.5.tar.gz" \
&& tar -xzf python-fmask-0.5.5.tar.gz && cd python-fmask-0.5.5 \
&& python3 setup.py install && cd .. && rm -rf python-fmask-*
ENV RIOS_DFLT_DRIVER="GTiff"
# Make python3 the default python command
RUN ln -s /usr/bin/python3 /usr/bin/python
# Default user and directory when running a container, is active now
# Compile obiatools library
RUN export PY_MINOR=$(python3 -c "import sys; print(sys.version_info.minor)") \
&& git clone "https://gitlab.irstea.fr/raffaele.gaetano/obiatools.git" && cd obiatools/Release \
&& sed -i "s/\$HOME\/moringa/\/usr\/local\/lib\/python3.$PY_MINOR\/dist-packages/g" configure.sh \
&& sed -i "s/python3.6/python3.$PY_MINOR/g" configure.sh \
&& if [ $PY_MINOR = "6" ]; then sed -i "s/=python3\.6/=python3\.6m/g" configure.sh ; fi \
&& mkdir "/usr/local/lib/python3.$PY_MINOR/dist-packages/sitsproc_c_modules" \
&& touch "/usr/local/lib/python3.$PY_MINOR/dist-packages/sitsproc_c_modules/__init__.py" \
&& bash -c "source configure.sh && make -j4" && cd ../.. && rm -rf obiatools
# Default user and directory, active from now on (for later RUN commands)
RUN useradd -s /bin/bash -m ubuntu
USER ubuntu
WORKDIR /home/ubuntu
......@@ -32,6 +33,9 @@ RUN git clone -b python3 "https://github.com/olivierhagolle/theia_download.git"
RUN wget -q "http://step.esa.int/thirdparties/sen2cor/2.8.0/Sen2Cor-02.08.00-Linux64.run" \
&& bash Sen2Cor-02.08.00-Linux64.run && rm Sen2Cor-02.08.00-Linux64.run -f
# Copy full repository
# Install MORINGA
RUN mkdir moringa
# Copy files without .git
COPY --chown=ubuntu . /home/ubuntu/moringa
# OR
#RUN git clone --single-branch -b develop https://gitlab.irstea.fr/raffaele.gaetano/moringa.git
## Docker overview
### Installation
# Docker overview
## Installation
*Windows 10*
```
Install Docker Desktop then use cmd.exe to execute docker pull and docker create.
```
*Debian and Ubuntu*
*Debian and Ubuntu*
See full documentation here : https://docs.docker.com/engine/install/ubuntu/
```bash
# https://docs.docker.com/install/linux/docker-ce/ubuntu/
# Uninstall old versions (an old docker version is available via apt default repositories)
sudo apt-get remove docker docker-engine docker.io containerd runc
# System dependencies
sudo apt-get update
sudo apt-get install -y apt-transport-https ca-certificates curl \
gnupg-agent software-properties-common
# Read system id and codename
source /etc/os-release
# Get gpg key
curl -fsSL https://download.docker.com/linux/$ID/gpg | sudo apt-key add -
# Verify 9DC8 5822 9FC7 DD38 854A E2D8 8D81 803C 0EBF CD88
sudo apt-key fingerprint 0EBFCD88
# Add repo
sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/$ID $VERSION_CODENAME stable"
# Install docker
sudo apt-get install \
apt-transport-https \
ca-certificates \
curl \
gnupg \
lsb-release
# Get Docker GPG key
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
# Add Docker apt repository
echo \
"deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \
$(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
# Install latest Docker Community Edition
sudo apt-get update
sudo apt-get install -y docker-ce docker-ce-cli containerd.io
sudo apt-get install docker-ce docker-ce-cli containerd.io
# https://docs.docker.com/install/linux/linux-postinstall/
# Allow user to run docker commands, see https://docs.docker.com/engine/install/linux-postinstall/
# Only if you want to avoid using sudo for each docker command ; docker group grants privileges equivalent to the root user.
sudo groupadd docker
sudo usermod -aG docker $USER
# Test docker run
docker run hello-world
# Useful commands
docker info # System info
docker images # List local images
docker container ls # List containers
docker ps # Show running containers
# Control state with systemd
#sudo systemctl {status,enable,disable,start,stop} docker
```
### Usage
## Usage
### Pull, create, exec...
N.B. : in previous docker images, default user was **moringa** and every python files were located in `/home/moringa`
In newer images (tag develop, now based on ubuntu:20.04), default user is **ubuntu** and you'll find evey python files in `/home/ubuntu/moringa`
```bash
# Pull image
# Pull image (default tag is "latest")
docker pull gitlab-registry.irstea.fr/raffaele.gaetano/moringa
# Simple command in a one-shot container
# Simple command in a one-shot container (anonymous container will not persist)
docker run gitlab-registry.irstea.fr/raffaele.gaetano/moringa otbcli_BandMathX
# Persistent container with volume
docker create --interactive --tty --volume /home/$USER:/home/moringa/data \
# Persistent (named) container with volume, here with home dir, but it can be any directory
# Beware of ownership issues, see the last section of this doc
docker create --interactive --tty --volume /home/$USER:/home/ubuntu/data \
--name moringa gitlab-registry.irstea.fr/raffaele.gaetano/moringa /bin/bash
# Start a background container
# Interactive
docker start -i moringa
# Background container
docker start moringa
docker exec moringa ls -alh
docker stop moringa
# Interactive
docker start -i moringa
# Running commands with root user (background container is the easiest way)
docker start moringa
# Example with apt update (you can't use &&, one docker exec is required for each command)
docker exec --user root moringa apt-get update
docker exec --user root moringa apt-get upgrade -y
# Useful container-specific commands, especially for background containers
docker inspect moringa # See full container info dump
docker logs moringa # See command logs and outputs
docker stats moringa # Real time container statistics
docker {pause,unpause} moringa # Freeze container
# Don't forget to kill a background container when you're done
docker stop moringa
# Remove a persistent container
docker rm moringa
```
### How to build
### Build images
```bash
git clone -b develop https://gitlab.irstea.fr/raffaele.gaetano/moringa.git
cd moringa
docker login gitlab-registry.irstea.fr
# OTB base image, example with arg OTB_VERSION=7.2 (default value)
docker build --build-arg OTB_VERSION=7.2 -f docker/base-otb/Dockerfile \
-t gitlab-registry.irstea.fr/raffaele.gaetano/moringa:base-otb-7.2 .
docker push gitlab-registry.irstea.fr/raffaele.gaetano/moringa:base-otb-7.2
# OTB base image, with default USE_SYSTEM_DEPS=false (build all dependencies from source)
docker build -f docker/Dockerfile.base \
-t gitlab-registry.irstea.fr/raffaele.gaetano/moringa/base:focal-otb72-superbuild .
docker push gitlab-registry.irstea.fr/raffaele.gaetano/moringa/base:focal-otb72-superbuild
# Tag 'moringa/base:latest' == default tag and default base image for the MORINGA build
docker tag gitlab-registry.irstea.fr/raffaele.gaetano/moringa/base:focal-otb72-superbuild \
gitlab-registry.irstea.fr/raffaele.gaetano/moringa/base:latest
# Default pulled or pushed tag is "latest" (if not specified)
docker push gitlab-registry.irstea.fr/raffaele.gaetano/moringa/base
# MORINGA build from the default 'moringa/base' image
# You don't need to rebuild OTB if you just want to update moringa files using current branch / local commits
#git checkout custom_branch
docker build -f docker/Dockerfile.moringa -t gitlab-registry.irstea.fr/raffaele.gaetano/moringa .
docker push gitlab-registry.irstea.fr/raffaele.gaetano/moringa:latest
# Keep tracks using tags
docker tag gitlab-registry.irstea.fr/raffaele.gaetano/moringa:latest gitlab-registry.irstea.fr/raffaele.gaetano/moringa:ubuntu20
docker push gitlab-registry.irstea.fr/raffaele.gaetano/moringa:ubuntu20
# Build base image using system deps (libs installed from apt repository)
docker build -f docker/Dockerfile.base \
--build-arg USE_SYSTEM_DEPS=true --build-arg DEPS_INSTALL_PREFIX=/usr \
-t gitlab-registry.irstea.fr/raffaele.gaetano/moringa/base:focal-otb72-sysdeps .
docker push gitlab-registry.irstea.fr/raffaele.gaetano/moringa/base:focal-otb72-sysdeps
# Build with custom ubuntu base image
docker build --build-arg BASE_IMG=ubuntu:18.04 -f docker/Dockerfile.base \
-t gitlab-registry.irstea.fr/raffaele.gaetano/moringa/base:bionic-otb72-superbuild .
docker build -f docker/Dockerfile.moringa --build-arg BASE_IMG=gitlab-registry.irstea.fr/raffaele.gaetano/moringa/base:bionic-otb72-superbuild \
-t gitlab-registry.irstea.fr/raffaele.gaetano/moringa:ubuntu18 .
```
# Tag 'base-otb' == default image for next build
docker tag gitlab-registry.irstea.fr/raffaele.gaetano/moringa:base-otb-7.2 \
gitlab-registry.irstea.fr/raffaele.gaetano/moringa:base-otb
docker push gitlab-registry.irstea.fr/raffaele.gaetano/moringa:base-otb
### Fix volume ownership issue (required if host's UID > 1000)
When mounting a volume, you may experience errors while trying to write files from within the container.
Since the default user (ubuntu) is UID 1000, you won't be able to write files into your volume
which is mounted with the same UID than your linux host user (may be UID 1001 or more).
In order to address this, you need to edit the container's user UID and GID to match the right numerical value.
This will only persist in a named container, it is required every time you're creating a new one.
# MORINGA build from the 'base-otb' image
docker build -f docker/moringa/Dockerfile -t gitlab-registry.irstea.fr/raffaele.gaetano/moringa .
docker push gitlab-registry.irstea.fr/raffaele.gaetano/moringa:latest
# MORINGA build from custom image
docker build -f docker/moringa/Dockerfile \
--build-arg BASE_IMG=gitlab-registry.irstea.fr/raffaele.gaetano/moringa:base-otb-dev \
-t gitlab-registry.irstea.fr/raffaele.gaetano/moringa .
```bash
# Create a named container (here with your HOME as volume), Docker will automatically pull image
docker create --interactive --tty --volume /home/$USER:/home/ubuntu/data \
--name moringa gitlab-registry.irstea.fr/raffaele.gaetano/moringa /bin/bash
# Start a background container process (in order to exec root commands, because default user isn't sudoer)
docker start moringa
# Exec required commands with user root (here with host's ID, replace $UID and $GID with desired values)
docker exec --user root moringa usermod ubuntu -u $UID
docker exec --user root moringa groupmod ubuntu -g $GID
# Force reset ownership with updated UID and GID. Make sure to double check that
docker exec moringa id
# Because recursive chown will apply to your volume in /home/ubuntu/data
docker exec --user root moringa chown -R ubuntu:ubuntu /home/ubuntu
# Stop the background container and start a new interactive shell
docker stop moringa
docker start -i moringa
```
```bash
# Check if ownership is right
id
ls -Alh /home/ubuntu
ls -Alh /home/ubuntu/data
# Test writing a file
touch /home/ubuntu/data/test.txt
```
-DUSE_SYSTEM_BOOST=ON
-DUSE_SYSTEM_CURL=ON
-DUSE_SYSTEM_EXPAT=ON
-DUSE_SYSTEM_FFTW=ON
-DUSE_SYSTEM_FREETYPE=ON
-DUSE_SYSTEM_GDAL=ON
-DUSE_SYSTEM_GEOS=ON
-DUSE_SYSTEM_GEOTIFF=ON
-DUSE_SYSTEM_GSL=ON
-DUSE_SYSTEM_ITK=ON
-DUSE_SYSTEM_LIBKML=ON
-DUSE_SYSTEM_LIBSVM=ON
-DUSE_SYSTEM_MUPARSER=ON
-DUSE_SYSTEM_MUPARSERX=ON
-DUSE_SYSTEM_OPENCV=ON
-DUSE_SYSTEM_OPENTHREADS=ON
-DUSE_SYSTEM_OSSIM=ON
-DUSE_SYSTEM_PNG=ON
-DUSE_SYSTEM_SHARK=OFF
-DUSE_SYSTEM_TINYXML=ON
-DUSE_SYSTEM_ZLIB=ON
-DOTB_USE_QT=OFF
-DOTB_USE_OPENGL=OFF
-DOTB_USE_GLUT=OFF
-DOTB_USE_GLEW=OFF
-DOTB_USE_GLFW=OFF
......@@ -5,8 +5,8 @@ import getopt
import shutil
from mtdUtils import queuedProcess
def runSen2Cor(fld, s2c_fld, nproc = 1):
basecmd = [s2c_fld + os.sep + 'bin' + os.sep + 'L2A_Process']
def runSen2Cor(fld, s2c_fld, nproc = 1, options = []):
basecmd = [s2c_fld + os.sep + 'bin' + os.sep + 'L2A_Process'] + options
lst = glob.glob(fld + os.sep + '*MSIL1C*.SAFE')
cmdlist = [' '.join(basecmd + [l] + ['>', l.replace('.SAFE','.log')]) for l in lst]
......@@ -25,17 +25,20 @@ def runSen2Cor(fld, s2c_fld, nproc = 1):
if __name__ == '__main__':
if len(sys.argv) < 3:
sys.exit(
'Usage: python runSen2Cor.py [-n <number of parallel processes> def. 1] <Sen2Cor install folder> <folder containing S2 MSIL1C>\n'
'Usage: python runSen2Cor.py [-n <number of parallel processes> def. 1] [-o <Sen2Cor options in a quote>] <Sen2Cor install folder> <folder containing S2 MSIL1C>\n'
'Ex. python runSen2Cor.py -n 12 /home/ocsol_data/_tools/Sen2Cor-02.05.05-Linux64 /home/ocsol_data/Montcuq/S2')
else:
try:
opts, args = getopt.getopt(sys.argv[1:], 'n:', [])
opts, args = getopt.getopt(sys.argv[1:], 'n:o:', [])
except getopt.GetoptError as err:
print(str(err))
options = []
nproc = 1
for opt, val in opts:
if opt == '-n':
nproc = int(val)
if opt == '-o':
options = val.split(' ')
runSen2Cor(args[1], args[0], nproc)
runSen2Cor(args[1], args[0], nproc, options)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment