Commit d625ea6d authored by Cresson Remi's avatar Cresson Remi
Browse files

Merge branch '5-pyotb_update' into 'master'

FIX: follow API changes in pyotb

Closes #5

See merge request !7
1 merge request!7FIX: follow API changes in pyotb
Pipeline #39794 failed with stages
in 37 seconds
Showing with 58 additions and 72 deletions
+58 -72
......@@ -4,8 +4,8 @@ variables:
GPU_IMAGE_NAME: $CI_REGISTRY_IMAGE:gpu
DOCKER_BUILDKIT: 1
DOCKER_DRIVER: overlay2
CPU_BASE_IMAGE: gitlab-registry.irstea.fr/remi.cresson/otbtf/3.2.1:cpu-basic-dev
GPU_BASE_IMAGE: gitlab-registry.irstea.fr/remi.cresson/otbtf/3.2.1:gpu-basic-dev
CPU_BASE_IMAGE: gitlab-registry.irstea.fr/remi.cresson/otbtf:3.3.2-cpu-dev
GPU_BASE_IMAGE: gitlab-registry.irstea.fr/remi.cresson/otbtf:3.3.2-gpu-dev
workflow:
rules:
......
# Decloud dockerfile
# To build the docker image for cpu, do the following:
#
# docker build --build-arg "BASE_IMAGE=mdl4eo/otbtf3.0:cpu-basic-dev" .
# docker build --build-arg "BASE_IMAGE=mdl4eo/otbtf3.3.2:cpu-dev" .
#
ARG BASE_IMAGE=mdl4eo/otbtf3.0:gpu-dev
ARG BASE_IMAGE=mdl4eo/otbtf3.3.2:gpu-dev
FROM $BASE_IMAGE
LABEL description="Decloud docker image"
LABEL maintainer="Remi Cresson [at] inrae [dot] fr"
......@@ -18,9 +18,6 @@ RUN apt upgrade -y
COPY docker/requirements.txt /tmp/requirements.txt
RUN python3 -m pip install -r /tmp/requirements.txt
# Fix Mosaic app in OTB <= 7.4
RUN cd /src/otb/otb/ && git config remote.origin.fetch refs/heads/*:refs/remotes/origin/* && git fetch origin && git -c user.name=decloud-docker -c user.email=decloud@decloud.xyz cherry-pick 44ec133646c46d6772450bbf5fb3ec54e282f56b
# Build remote modules
RUN cd /src/otb/otb/Modules/Remote/ && git clone https://gitlab.irstea.fr/remi.cresson/SimpleExtractionTools.git
RUN cd /src/otb/otb/Modules/Remote/ && git clone https://gitlab.irstea.fr/remi.cresson/mlutils.git
......
......@@ -31,7 +31,7 @@ import sys
import json
import numpy as np
from matplotlib import pyplot as plt
import gdal
from osgeo import gdal
parser = argparse.ArgumentParser(description="dataset test")
parser.add_argument("--input_dir", help="A directory containing S1 or S2 files", required=True)
......
......@@ -41,16 +41,13 @@ def get_commit_hash():
""" Return the git hash of the repository """
repo = git.Repo(os.path.dirname(os.path.realpath(__file__)), search_parent_directories=True)
commit_hash = "nohash"
try:
commit_hash = repo.head.object.hexsha[0:5]
except (ValueError, TypeError) as e:
print(f"Unable to get commit hash! {e}")
try:
commit_hash = repo.active_branch.name + "_" + commit_hash
except (ValueError, TypeError) as e:
print(f"Unable to get branch name! {e}")
commit_hash = repo.active_branch.name + "_" + repo.head.object.hexsha[0:5]
except (TypeError, ValueError, BrokenPipeError):
try:
commit_hash = 'DETACHED_' + repo.head.object.hexsha[0:5]
except (ValueError, BrokenPipeError):
commit_hash = 'DETACHED'
return commit_hash
......
......@@ -25,7 +25,7 @@ DEALINGS IN THE SOFTWARE.
import argparse
import logging
import elevation
import gdal
from osgeo import gdal
import decloud.preprocessing.constants as constants
import otbApplication as otb
from decloud.core import system
......
......@@ -66,7 +66,7 @@ def inference(sources, sources_scales, pad, ts, savedmodel_dir, out_tensor, out_
# Setup TensorFlowModelServe
system.set_env_var("OTB_TF_NSOURCES", str(len(sources)))
infer = pyotb.TensorflowModelServe(execute=False)
parameters = {}
# Inputs
for i, (placeholder, source) in enumerate(sources.items()):
......@@ -80,18 +80,18 @@ def inference(sources, sources_scales, pad, ts, savedmodel_dir, out_tensor, out_
if placeholder in sources_scales:
src_rfield = int(rfield / sources_scales[placeholder])
infer.set_parameters({get_key("il"): [source],
get_key("rfieldx"): src_rfield,
get_key("rfieldy"): src_rfield,
get_key("placeholder"): placeholder})
parameters.update({get_key("il"): [source],
get_key("rfieldx"): src_rfield,
get_key("rfieldy"): src_rfield,
get_key("placeholder"): placeholder})
# Model
infer.set_parameters({"model.dir": savedmodel_dir, "model.fullyconv": "on",
"output.names": [padded_tensor_name(out_tensor, pad)],
"output.efieldx": efield, "output.efieldy": efield,
"optim.tilesizex": efield, "optim.tilesizey": efield,
"optim.disabletiling": 1})
infer.Execute()
parameters.update({"model.dir": savedmodel_dir, "model.fullyconv": True,
"output.names": [padded_tensor_name(out_tensor, pad)],
"output.efieldx": efield, "output.efieldy": efield,
"optim.tilesizex": efield, "optim.tilesizey": efield,
"optim.disabletiling": True})
infer = pyotb.TensorflowModelServe(parameters)
# Post Processing
# For ESA Sentinel-2, remove potential zeros the network may have introduced in the valid parts of the image
......
......@@ -74,10 +74,10 @@ def monthly_synthesis_inference(sources, sources_scales, pad, ts, savedmodel_dir
# Setup TensorFlowModelServe
system.set_env_var("OTB_TF_NSOURCES", str(len(sources)))
infer = pyotb.App("TensorflowModelServe", execute=False)
infer_params = {}
# Setup BandMath for post processing
bm = pyotb.App("BandMath", execute=False)
bm_params = {}
mask_expr = "0"
# Inputs
......@@ -93,7 +93,7 @@ def monthly_synthesis_inference(sources, sources_scales, pad, ts, savedmodel_dir
if placeholder in sources_scales:
src_rfield = int(rfield / sources_scales[placeholder])
infer.set_parameters({get_key("il"): [source]})
infer_params.update({get_key("il"): [source]})
# Update post processing BandMath expression
if placeholder != 'dem' and '20m' not in placeholder:
......@@ -101,33 +101,34 @@ def monthly_synthesis_inference(sources, sources_scales, pad, ts, savedmodel_dir
n_channels = pyotb.get_nbchannels(source)
mask_expr += "||"
mask_expr += "&&".join(["im{}b{}=={}".format(k + 1, b, nodatavalue) for b in range(1, 1 + n_channels)])
bm.set_parameters(il=[source])
bm_params.update({'il': source})
k += 1
infer.set_parameters({get_key("rfieldx"): src_rfield,
get_key("rfieldy"): src_rfield,
get_key("placeholder"): placeholder})
infer_params.update({get_key("rfieldx"): src_rfield,
get_key("rfieldy"): src_rfield,
get_key("placeholder"): placeholder})
# Model
infer.set_parameters({"model.dir": savedmodel_dir, "model.fullyconv": "on",
"output.names": [padded_tensor_name(out_tensor, pad)],
"output.efieldx": efield, "output.efieldy": efield,
"optim.tilesizex": efield, "optim.tilesizey": efield,
"optim.disabletiling": 1})
infer.Execute()
infer_params.update({"model.dir": savedmodel_dir, "model.fullyconv": True,
"output.names": [padded_tensor_name(out_tensor, pad)],
"output.efieldx": efield, "output.efieldy": efield,
"optim.tilesizex": efield, "optim.tilesizey": efield,
"optim.disabletiling": True})
infer = pyotb.TensorflowModelServe(infer_params)
# For ESA Sentinel-2, remove potential zeros the network may have introduced in the valid parts of the image
if out_pixeltype == otbApplication.ImagePixelType_uint16:
n_channels = pyotb.get_nbchannels(infer.out)
n_channels = pyotb.get_nbchannels(infer)
exp = ';'.join([f'(im1b{b}<=1 ? 1 : im1b{b})' for b in range(1, 1 + n_channels)])
rmzeros = pyotb.App("BandMathX", il=[infer.out], exp=exp)
rmzeros = pyotb.App("BandMathX", il=[infer], exp=exp)
rmzeros.SetParameterOutputImagePixelType("out", out_pixeltype)
else:
rmzeros = infer
# Mask for post processing
mask_expr += "?0:255"
bm.set_parameters(exp=mask_expr)
bm_params.update({'exp': mask_expr})
bm = pyotb.BandMath(bm_params)
# Closing post processing mask to remove small groups of NoData pixels
closing = pyotb.App("BinaryMorphologicalOperation", bm, filter="closing", foreval=255, structype="box",
......
......@@ -74,10 +74,10 @@ def monthly_synthesis_inference(sources, sources_scales, pad, ts, savedmodel_dir
# Setup TensorFlowModelServe
system.set_env_var("OTB_TF_NSOURCES", str(len(sources)))
infer = pyotb.App("TensorflowModelServe", execute=False)
infer_params = {}
# Setup BandMath for post processing
bm = pyotb.App("BandMath", execute=False)
bm_params = {}
mask_expr = "0"
# Inputs
......@@ -93,7 +93,7 @@ def monthly_synthesis_inference(sources, sources_scales, pad, ts, savedmodel_dir
if placeholder in sources_scales:
src_rfield = int(rfield / sources_scales[placeholder])
infer.set_parameters({get_key("il"): [source]})
infer_params.update({get_key("il"): [source]})
# Update post processing BandMath expression
if placeholder != 'dem' and '20m' not in placeholder:
......@@ -101,20 +101,20 @@ def monthly_synthesis_inference(sources, sources_scales, pad, ts, savedmodel_dir
n_channels = pyotb.get_nbchannels(source)
mask_expr += "||"
mask_expr += "&&".join(["im{}b{}=={}".format(k + 1, b, nodatavalue) for b in range(1, 1 + n_channels)])
bm.set_parameters(il=[source])
bm_params.update({'il': [source]})
k += 1
infer.set_parameters({get_key("rfieldx"): src_rfield,
get_key("rfieldy"): src_rfield,
get_key("placeholder"): placeholder})
infer_params.update({get_key("rfieldx"): src_rfield,
get_key("rfieldy"): src_rfield,
get_key("placeholder"): placeholder})
# Model
infer.set_parameters({"model.dir": savedmodel_dir, "model.fullyconv": "on",
"output.names": [padded_tensor_name(out_tensor, pad)],
"output.efieldx": efield, "output.efieldy": efield,
"optim.tilesizex": efield, "optim.tilesizey": efield,
"optim.disabletiling": 1})
infer.Execute()
infer_params.update({"model.dir": savedmodel_dir, "model.fullyconv": True,
"output.names": [padded_tensor_name(out_tensor, pad)],
"output.efieldx": efield, "output.efieldy": efield,
"optim.tilesizex": efield, "optim.tilesizey": efield,
"optim.disabletiling": True})
infer = pyotb.TensorflowModelServe(infer_params)
# For ESA Sentinel-2, remove potential zeros the network may have introduced in the valid parts of the image
if out_pixeltype == otbApplication.ImagePixelType_uint16:
......@@ -127,7 +127,8 @@ def monthly_synthesis_inference(sources, sources_scales, pad, ts, savedmodel_dir
# Mask for post processing
mask_expr += "?0:255"
bm.set_parameters(exp=mask_expr)
bm_params.update({'exp': mask_expr})
bm = pyotb.BandMath(bm_params)
# Closing post processing mask to remove small groups of NoData pixels
closing = pyotb.App("BinaryMorphologicalOperation", bm, filter="closing", foreval=255, structype="box",
......@@ -269,6 +270,7 @@ if __name__ == "__main__":
"""
return abs(s2_product.get_timestamp() - x.get_timestamp())
input_s1_products.sort(key=_closest_date, reverse=True)
input_s1_images_10m = [product.get_raster_10m() for product in input_s1_products]
# creating a mosaic with the N closest S1 images
......
......@@ -28,7 +28,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
setuptools.setup(
name="decloud",
version="0.1",
version="1.3",
author="Remi Cresson, Nicolas Narçon, Benjamin Commandre",
author_email="remi.cresson@inrae.fr",
description="Deep learning based reconstruction of optical time series using SAR imagery",
......
......@@ -4,7 +4,7 @@ import os
import subprocess
import unittest
import filecmp
import gdal
from osgeo import gdal
import otbApplication as otb
from abc import ABC
from decloud.core.system import get_env_var, basename
......
......@@ -15,7 +15,6 @@ class DEMTest(DecloudTest):
"SENTINEL2B_20201026-103901-924_L2A_T31TEJ_C_V2-2_FRE_20m.tif"),
"--output", "/tmp/dem_prepare.tif", "--tmp", "/tmp/DEM"])
self.compare_raster_metadata("/tmp/dem_prepare.tif", self.get_path("baseline/PREPARE/DEM_PREPARE/T31TEJ.tif"))
self.compare_images("/tmp/dem_prepare.tif", self.get_path("baseline/PREPARE/DEM_PREPARE/T31TEJ.tif"))
......
......@@ -84,7 +84,6 @@ class InferenceTest(DecloudTest):
self.assertTrue(system.file_exists(outpath))
self.compare_images(outpath, baseline_path)
self.compare_raster_metadata(outpath, baseline_path)
def test_inference_with_generic_preprocessor(self):
# Logger
......@@ -128,7 +127,6 @@ class InferenceTest(DecloudTest):
self.assertTrue(system.file_exists(outpath))
self.compare_images(outpath, baseline_path)
self.compare_raster_metadata(outpath, baseline_path)
if __name__ == '__main__':
......
......@@ -12,9 +12,6 @@ class Sentinel1PrepareTest(DecloudTest):
sentinel1_prepare.main(["--input_s1_vh", self.get_path("inputs/THEIA/s1b_31TEJ_vh_DES_110_20201011t060008.tif"),
"--out_s1_dir", "/tmp/s1_prepare"])
self.compare_raster_metadata("/tmp/s1_prepare/s1b_31TEJ_vvvh_DES_110_20201011t060008_from-10to3dB.tif",
self.get_path("baseline/PREPARE/S1_PREPARE/T31TEJ/"
"s1b_31TEJ_vvvh_DES_110_20201011t060008_from-10to3dB.tif"))
self.compare_images("/tmp/s1_prepare/s1b_31TEJ_vvvh_DES_110_20201011t060008_from-10to3dB.tif",
self.get_path("baseline/PREPARE/S1_PREPARE/T31TEJ/"
"s1b_31TEJ_vvvh_DES_110_20201011t060008_from-10to3dB.tif"))
......
......@@ -13,11 +13,6 @@ class Sentinel2PrepareTest(DecloudTest):
"-497_L2A_T31TEJ_C_V2-2"),
"--out_s2_dir", "/tmp/s2_prepare"])
self.compare_raster_metadata("/tmp/s2_prepare/T31TEJ/SENTINEL2B_20201012-105848-497_L2A_T31TEJ_C_V2-2/"
"SENTINEL2B_20201012-105848-497_L2A_T31TEJ_C_V2-2_FRE_10m.tif",
self.get_path("baseline/PREPARE/S2_PREPARE/T31TEJ/"
"SENTINEL2B_20201012-105848-497_L2A_T31TEJ_C_V2-2/"
"SENTINEL2B_20201012-105848-497_L2A_T31TEJ_C_V2-2_FRE_10m.tif"))
self.compare_images("/tmp/s2_prepare/T31TEJ/SENTINEL2B_20201012-105848-497_L2A_T31TEJ_C_V2-2/"
"SENTINEL2B_20201012-105848-497_L2A_T31TEJ_C_V2-2_FRE_10m.tif",
self.get_path("baseline/PREPARE/S2_PREPARE/T31TEJ/"
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment