diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b7d1566c1c7301e0672a1c5bc1792e91cec61835..7301254a9fd72f649459f1689f6804486d4fc44a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -4,8 +4,8 @@ variables:
GPU_IMAGE_NAME: $CI_REGISTRY_IMAGE:gpu
DOCKER_BUILDKIT: 1
DOCKER_DRIVER: overlay2
- CPU_BASE_IMAGE: gitlab-registry.irstea.fr/remi.cresson/otbtf/3.2.1:cpu-basic-dev
- GPU_BASE_IMAGE: gitlab-registry.irstea.fr/remi.cresson/otbtf/3.2.1:gpu-basic-dev
+ CPU_BASE_IMAGE: gitlab-registry.irstea.fr/remi.cresson/otbtf:3.3.2-cpu-dev
+ GPU_BASE_IMAGE: gitlab-registry.irstea.fr/remi.cresson/otbtf:3.3.2-gpu-dev
workflow:
rules:
diff --git a/Dockerfile b/Dockerfile
index 684ed0872f164f19df739369495228bb593b944a..c2debd3b41275b3c5a29c812dd754ff1012b638f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,9 +1,9 @@
# Decloud dockerfile
# To build the docker image for cpu, do the following:
#
-# docker build --build-arg "BASE_IMAGE=mdl4eo/otbtf3.0:cpu-basic-dev" .
+# docker build --build-arg "BASE_IMAGE=mdl4eo/otbtf3.3.2:cpu-dev" .
#
-ARG BASE_IMAGE=mdl4eo/otbtf3.0:gpu-dev
+ARG BASE_IMAGE=mdl4eo/otbtf3.3.2:gpu-dev
FROM $BASE_IMAGE
LABEL description="Decloud docker image"
LABEL maintainer="Remi Cresson [at] inrae [dot] fr"
@@ -18,9 +18,6 @@ RUN apt upgrade -y
COPY docker/requirements.txt /tmp/requirements.txt
RUN python3 -m pip install -r /tmp/requirements.txt
-# Fix Mosaic app in OTB <= 7.4
-RUN cd /src/otb/otb/ && git config remote.origin.fetch refs/heads/*:refs/remotes/origin/* && git fetch origin && git -c user.name=decloud-docker -c user.email=decloud@decloud.xyz cherry-pick 44ec133646c46d6772450bbf5fb3ec54e282f56b
-
# Build remote modules
RUN cd /src/otb/otb/Modules/Remote/ && git clone https://gitlab.irstea.fr/remi.cresson/SimpleExtractionTools.git
RUN cd /src/otb/otb/Modules/Remote/ && git clone https://gitlab.irstea.fr/remi.cresson/mlutils.git
diff --git a/decloud/analysis/images_stats.py b/decloud/analysis/images_stats.py
index a38da0e027ab2218dcea5664e9e86ea774d4c929..dac37eda0ba52ef9103c4660d1a66d23013b2bfe 100644
--- a/decloud/analysis/images_stats.py
+++ b/decloud/analysis/images_stats.py
@@ -31,7 +31,7 @@ import sys
import json
import numpy as np
from matplotlib import pyplot as plt
-import gdal
+from osgeo import gdal
parser = argparse.ArgumentParser(description="dataset test")
parser.add_argument("--input_dir", help="A directory containing S1 or S2 files", required=True)
diff --git a/decloud/core/system.py b/decloud/core/system.py
index 0b9f47558187daaea4af12b43c058f87da969973..d43fb85bf77291d7dc3d31adfdf0b4aca714b791 100644
--- a/decloud/core/system.py
+++ b/decloud/core/system.py
@@ -41,16 +41,13 @@ def get_commit_hash():
""" Return the git hash of the repository """
repo = git.Repo(os.path.dirname(os.path.realpath(__file__)), search_parent_directories=True)
- commit_hash = "nohash"
try:
- commit_hash = repo.head.object.hexsha[0:5]
- except (ValueError, TypeError) as e:
- print(f"Unable to get commit hash! {e}")
-
- try:
- commit_hash = repo.active_branch.name + "_" + commit_hash
- except (ValueError, TypeError) as e:
- print(f"Unable to get branch name! {e}")
+ commit_hash = repo.active_branch.name + "_" + repo.head.object.hexsha[0:5]
+ except (TypeError, ValueError, BrokenPipeError):
+ try:
+ commit_hash = 'DETACHED_' + repo.head.object.hexsha[0:5]
+ except (ValueError, BrokenPipeError):
+ commit_hash = 'DETACHED'
return commit_hash
diff --git a/decloud/preprocessing/dem_prepare.py b/decloud/preprocessing/dem_prepare.py
index 35c3b90e8a18b92018e6eeebc12eba6b2185ab4c..f55338ab1d955213675a75b9c451c57c733f5734 100644
--- a/decloud/preprocessing/dem_prepare.py
+++ b/decloud/preprocessing/dem_prepare.py
@@ -25,7 +25,7 @@ DEALINGS IN THE SOFTWARE.
import argparse
import logging
import elevation
-import gdal
+from osgeo import gdal
import decloud.preprocessing.constants as constants
import otbApplication as otb
from decloud.core import system
diff --git a/decloud/production/inference.py b/decloud/production/inference.py
index e85544ac142bac07d20a1b93ed6360a4e7e43389..4aa8df30d000c8411d82c585119aa5b5e3f2401d 100644
--- a/decloud/production/inference.py
+++ b/decloud/production/inference.py
@@ -66,7 +66,7 @@ def inference(sources, sources_scales, pad, ts, savedmodel_dir, out_tensor, out_
# Setup TensorFlowModelServe
system.set_env_var("OTB_TF_NSOURCES", str(len(sources)))
- infer = pyotb.TensorflowModelServe(execute=False)
+ parameters = {}
# Inputs
for i, (placeholder, source) in enumerate(sources.items()):
@@ -80,18 +80,18 @@ def inference(sources, sources_scales, pad, ts, savedmodel_dir, out_tensor, out_
if placeholder in sources_scales:
src_rfield = int(rfield / sources_scales[placeholder])
- infer.set_parameters({get_key("il"): [source],
- get_key("rfieldx"): src_rfield,
- get_key("rfieldy"): src_rfield,
- get_key("placeholder"): placeholder})
+ parameters.update({get_key("il"): [source],
+ get_key("rfieldx"): src_rfield,
+ get_key("rfieldy"): src_rfield,
+ get_key("placeholder"): placeholder})
# Model
- infer.set_parameters({"model.dir": savedmodel_dir, "model.fullyconv": "on",
- "output.names": [padded_tensor_name(out_tensor, pad)],
- "output.efieldx": efield, "output.efieldy": efield,
- "optim.tilesizex": efield, "optim.tilesizey": efield,
- "optim.disabletiling": 1})
- infer.Execute()
+ parameters.update({"model.dir": savedmodel_dir, "model.fullyconv": True,
+ "output.names": [padded_tensor_name(out_tensor, pad)],
+ "output.efieldx": efield, "output.efieldy": efield,
+ "optim.tilesizex": efield, "optim.tilesizey": efield,
+ "optim.disabletiling": True})
+ infer = pyotb.TensorflowModelServe(parameters)
# Post Processing
# For ESA Sentinel-2, remove potential zeros the network may have introduced in the valid parts of the image
diff --git a/decloud/production/monthly_synthesis_processor_s2.py b/decloud/production/monthly_synthesis_processor_s2.py
index e2ad8c4cb79e748dd86500517e3914761f695b31..8180c64fe278eee9adaf5db61dcff1d86c27386c 100644
--- a/decloud/production/monthly_synthesis_processor_s2.py
+++ b/decloud/production/monthly_synthesis_processor_s2.py
@@ -74,10 +74,10 @@ def monthly_synthesis_inference(sources, sources_scales, pad, ts, savedmodel_dir
# Setup TensorFlowModelServe
system.set_env_var("OTB_TF_NSOURCES", str(len(sources)))
- infer = pyotb.App("TensorflowModelServe", execute=False)
+ infer_params = {}
# Setup BandMath for post processing
- bm = pyotb.App("BandMath", execute=False)
+ bm_params = {}
mask_expr = "0"
# Inputs
@@ -93,7 +93,7 @@ def monthly_synthesis_inference(sources, sources_scales, pad, ts, savedmodel_dir
if placeholder in sources_scales:
src_rfield = int(rfield / sources_scales[placeholder])
- infer.set_parameters({get_key("il"): [source]})
+ infer_params.update({get_key("il"): [source]})
# Update post processing BandMath expression
if placeholder != 'dem' and '20m' not in placeholder:
@@ -101,33 +101,34 @@ def monthly_synthesis_inference(sources, sources_scales, pad, ts, savedmodel_dir
n_channels = pyotb.get_nbchannels(source)
mask_expr += "||"
mask_expr += "&&".join(["im{}b{}=={}".format(k + 1, b, nodatavalue) for b in range(1, 1 + n_channels)])
- bm.set_parameters(il=[source])
+ bm_params.update({'il': source})
k += 1
- infer.set_parameters({get_key("rfieldx"): src_rfield,
- get_key("rfieldy"): src_rfield,
- get_key("placeholder"): placeholder})
+ infer_params.update({get_key("rfieldx"): src_rfield,
+ get_key("rfieldy"): src_rfield,
+ get_key("placeholder"): placeholder})
# Model
- infer.set_parameters({"model.dir": savedmodel_dir, "model.fullyconv": "on",
- "output.names": [padded_tensor_name(out_tensor, pad)],
- "output.efieldx": efield, "output.efieldy": efield,
- "optim.tilesizex": efield, "optim.tilesizey": efield,
- "optim.disabletiling": 1})
- infer.Execute()
+ infer_params.update({"model.dir": savedmodel_dir, "model.fullyconv": True,
+ "output.names": [padded_tensor_name(out_tensor, pad)],
+ "output.efieldx": efield, "output.efieldy": efield,
+ "optim.tilesizex": efield, "optim.tilesizey": efield,
+ "optim.disabletiling": True})
+ infer = pyotb.TensorflowModelServe(infer_params)
# For ESA Sentinel-2, remove potential zeros the network may have introduced in the valid parts of the image
if out_pixeltype == otbApplication.ImagePixelType_uint16:
- n_channels = pyotb.get_nbchannels(infer.out)
+ n_channels = pyotb.get_nbchannels(infer)
exp = ';'.join([f'(im1b{b}<=1 ? 1 : im1b{b})' for b in range(1, 1 + n_channels)])
- rmzeros = pyotb.App("BandMathX", il=[infer.out], exp=exp)
+ rmzeros = pyotb.App("BandMathX", il=[infer], exp=exp)
rmzeros.SetParameterOutputImagePixelType("out", out_pixeltype)
else:
rmzeros = infer
# Mask for post processing
mask_expr += "?0:255"
- bm.set_parameters(exp=mask_expr)
+ bm_params.update({'exp': mask_expr})
+ bm = pyotb.BandMath(bm_params)
# Closing post processing mask to remove small groups of NoData pixels
closing = pyotb.App("BinaryMorphologicalOperation", bm, filter="closing", foreval=255, structype="box",
diff --git a/decloud/production/monthly_synthesis_processor_s2s1.py b/decloud/production/monthly_synthesis_processor_s2s1.py
index 10adc0aea0faeaa99f090f9b81d2ca1c243da2d2..36ec22a5706e1b11467c452feb9373d814d165ca 100644
--- a/decloud/production/monthly_synthesis_processor_s2s1.py
+++ b/decloud/production/monthly_synthesis_processor_s2s1.py
@@ -74,10 +74,10 @@ def monthly_synthesis_inference(sources, sources_scales, pad, ts, savedmodel_dir
# Setup TensorFlowModelServe
system.set_env_var("OTB_TF_NSOURCES", str(len(sources)))
- infer = pyotb.App("TensorflowModelServe", execute=False)
+ infer_params = {}
# Setup BandMath for post processing
- bm = pyotb.App("BandMath", execute=False)
+ bm_params = {}
mask_expr = "0"
# Inputs
@@ -93,7 +93,7 @@ def monthly_synthesis_inference(sources, sources_scales, pad, ts, savedmodel_dir
if placeholder in sources_scales:
src_rfield = int(rfield / sources_scales[placeholder])
- infer.set_parameters({get_key("il"): [source]})
+ infer_params.update({get_key("il"): [source]})
# Update post processing BandMath expression
if placeholder != 'dem' and '20m' not in placeholder:
@@ -101,20 +101,20 @@ def monthly_synthesis_inference(sources, sources_scales, pad, ts, savedmodel_dir
n_channels = pyotb.get_nbchannels(source)
mask_expr += "||"
mask_expr += "&&".join(["im{}b{}=={}".format(k + 1, b, nodatavalue) for b in range(1, 1 + n_channels)])
- bm.set_parameters(il=[source])
+ bm_params.update({'il': [source]})
k += 1
- infer.set_parameters({get_key("rfieldx"): src_rfield,
- get_key("rfieldy"): src_rfield,
- get_key("placeholder"): placeholder})
+ infer_params.update({get_key("rfieldx"): src_rfield,
+ get_key("rfieldy"): src_rfield,
+ get_key("placeholder"): placeholder})
# Model
- infer.set_parameters({"model.dir": savedmodel_dir, "model.fullyconv": "on",
- "output.names": [padded_tensor_name(out_tensor, pad)],
- "output.efieldx": efield, "output.efieldy": efield,
- "optim.tilesizex": efield, "optim.tilesizey": efield,
- "optim.disabletiling": 1})
- infer.Execute()
+ infer_params.update({"model.dir": savedmodel_dir, "model.fullyconv": True,
+ "output.names": [padded_tensor_name(out_tensor, pad)],
+ "output.efieldx": efield, "output.efieldy": efield,
+ "optim.tilesizex": efield, "optim.tilesizey": efield,
+ "optim.disabletiling": True})
+ infer = pyotb.TensorflowModelServe(infer_params)
# For ESA Sentinel-2, remove potential zeros the network may have introduced in the valid parts of the image
if out_pixeltype == otbApplication.ImagePixelType_uint16:
@@ -127,7 +127,8 @@ def monthly_synthesis_inference(sources, sources_scales, pad, ts, savedmodel_dir
# Mask for post processing
mask_expr += "?0:255"
- bm.set_parameters(exp=mask_expr)
+ bm_params.update({'exp': mask_expr})
+ bm = pyotb.BandMath(bm_params)
# Closing post processing mask to remove small groups of NoData pixels
closing = pyotb.App("BinaryMorphologicalOperation", bm, filter="closing", foreval=255, structype="box",
@@ -269,6 +270,7 @@ if __name__ == "__main__":
"""
return abs(s2_product.get_timestamp() - x.get_timestamp())
+
input_s1_products.sort(key=_closest_date, reverse=True)
input_s1_images_10m = [product.get_raster_10m() for product in input_s1_products]
# creating a mosaic with the N closest S1 images
diff --git a/setup.py b/setup.py
index 579b3a50f2c658655b717362cbcfa6923123d22e..ec9843b927c4ad69034de0b3a72905f199fd3038 100644
--- a/setup.py
+++ b/setup.py
@@ -28,7 +28,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
setuptools.setup(
name="decloud",
- version="0.1",
+ version="1.3",
author="Remi Cresson, Nicolas Narçon, Benjamin Commandre",
author_email="remi.cresson@inrae.fr",
description="Deep learning based reconstruction of optical time series using SAR imagery",
diff --git a/tests/decloud_unittest.py b/tests/decloud_unittest.py
index c97334050c40d6cec28be21c13b204e631c34e0a..669f7db9b4a209aac31e512657b01f26948f69b6 100644
--- a/tests/decloud_unittest.py
+++ b/tests/decloud_unittest.py
@@ -4,7 +4,7 @@ import os
import subprocess
import unittest
import filecmp
-import gdal
+from osgeo import gdal
import otbApplication as otb
from abc import ABC
from decloud.core.system import get_env_var, basename
diff --git a/tests/dem_unittest.py b/tests/dem_unittest.py
index e9c5639edcdb11e459d823153da3752b8be996f8..9804886876ac4a19c6be6a4aa0d30c32691945ad 100644
--- a/tests/dem_unittest.py
+++ b/tests/dem_unittest.py
@@ -15,7 +15,6 @@ class DEMTest(DecloudTest):
"SENTINEL2B_20201026-103901-924_L2A_T31TEJ_C_V2-2_FRE_20m.tif"),
"--output", "/tmp/dem_prepare.tif", "--tmp", "/tmp/DEM"])
- self.compare_raster_metadata("/tmp/dem_prepare.tif", self.get_path("baseline/PREPARE/DEM_PREPARE/T31TEJ.tif"))
self.compare_images("/tmp/dem_prepare.tif", self.get_path("baseline/PREPARE/DEM_PREPARE/T31TEJ.tif"))
diff --git a/tests/inference_unittest.py b/tests/inference_unittest.py
index 5f310c4e019765a28e230f57325c07e6690c1429..316b9c486c90cc64cd75bea0c11f3f874215d59b 100644
--- a/tests/inference_unittest.py
+++ b/tests/inference_unittest.py
@@ -84,7 +84,6 @@ class InferenceTest(DecloudTest):
self.assertTrue(system.file_exists(outpath))
self.compare_images(outpath, baseline_path)
- self.compare_raster_metadata(outpath, baseline_path)
def test_inference_with_generic_preprocessor(self):
# Logger
@@ -128,7 +127,6 @@ class InferenceTest(DecloudTest):
self.assertTrue(system.file_exists(outpath))
self.compare_images(outpath, baseline_path)
- self.compare_raster_metadata(outpath, baseline_path)
if __name__ == '__main__':
diff --git a/tests/sentinel1_prepare_unittest.py b/tests/sentinel1_prepare_unittest.py
index 86d5e9e362eceab870af8b5b33cfe464d53e3395..ec402b835de79b23830e6f80402e86d7f9d95308 100644
--- a/tests/sentinel1_prepare_unittest.py
+++ b/tests/sentinel1_prepare_unittest.py
@@ -12,9 +12,6 @@ class Sentinel1PrepareTest(DecloudTest):
sentinel1_prepare.main(["--input_s1_vh", self.get_path("inputs/THEIA/s1b_31TEJ_vh_DES_110_20201011t060008.tif"),
"--out_s1_dir", "/tmp/s1_prepare"])
- self.compare_raster_metadata("/tmp/s1_prepare/s1b_31TEJ_vvvh_DES_110_20201011t060008_from-10to3dB.tif",
- self.get_path("baseline/PREPARE/S1_PREPARE/T31TEJ/"
- "s1b_31TEJ_vvvh_DES_110_20201011t060008_from-10to3dB.tif"))
self.compare_images("/tmp/s1_prepare/s1b_31TEJ_vvvh_DES_110_20201011t060008_from-10to3dB.tif",
self.get_path("baseline/PREPARE/S1_PREPARE/T31TEJ/"
"s1b_31TEJ_vvvh_DES_110_20201011t060008_from-10to3dB.tif"))
diff --git a/tests/sentinel2_prepare_unittest.py b/tests/sentinel2_prepare_unittest.py
index 7896e6d61f962fb08786e8630f23df2174552166..3ade77c4d19bf7bcaa63cd97e2d82f46e6d4314b 100644
--- a/tests/sentinel2_prepare_unittest.py
+++ b/tests/sentinel2_prepare_unittest.py
@@ -13,11 +13,6 @@ class Sentinel2PrepareTest(DecloudTest):
"-497_L2A_T31TEJ_C_V2-2"),
"--out_s2_dir", "/tmp/s2_prepare"])
- self.compare_raster_metadata("/tmp/s2_prepare/T31TEJ/SENTINEL2B_20201012-105848-497_L2A_T31TEJ_C_V2-2/"
- "SENTINEL2B_20201012-105848-497_L2A_T31TEJ_C_V2-2_FRE_10m.tif",
- self.get_path("baseline/PREPARE/S2_PREPARE/T31TEJ/"
- "SENTINEL2B_20201012-105848-497_L2A_T31TEJ_C_V2-2/"
- "SENTINEL2B_20201012-105848-497_L2A_T31TEJ_C_V2-2_FRE_10m.tif"))
self.compare_images("/tmp/s2_prepare/T31TEJ/SENTINEL2B_20201012-105848-497_L2A_T31TEJ_C_V2-2/"
"SENTINEL2B_20201012-105848-497_L2A_T31TEJ_C_V2-2_FRE_10m.tif",
self.get_path("baseline/PREPARE/S2_PREPARE/T31TEJ/"