Commit 21e7d58d authored by Florian de Boissieu's avatar Florian de Boissieu
Browse files

fix merge develop

parents c883d151 957389e4
.git
install/*.docx
deprecated
sitsproc_*
\ No newline at end of file
sitsproc_*
......@@ -91,28 +91,28 @@ def main(argv):
if skip == 'n':
shutil.rmtree(download_folder)
if download_platform == 'S2PEPS':
cmd = ['python', cmd_path + 'getS2PEPSFromShp.py', '-w', download_folder, '-d', date_beg, '-f', date_end, download_shp_file, platform_path]
cmd = ['python3', cmd_path + 'getS2PEPSFromShp.py', '-w', download_folder, '-d', date_beg, '-f', date_end, download_shp_file, platform_path]
elif download_platform == 'S2THEIA':
cmd = ['python', cmd_path + 'getS2THEIAFromShp.py', '-w', download_folder, '-d', date_beg, '-f', date_end, download_shp_file, platform_path]
cmd = ['python3', cmd_path + 'getS2THEIAFromShp.py', '-w', download_folder, '-d', date_beg, '-f', date_end, download_shp_file, platform_path]
elif download_platform == 'VenusTHEIA':
cmd = ['python', cmd_path + 'getVenusFromShp.py', '-w', download_folder, '-d', date_beg, '-f', date_end, download_shp_file, platform_path]
cmd = ['python3', cmd_path + 'getVenusFromShp.py', '-w', download_folder, '-d', date_beg, '-f', date_end, download_shp_file, platform_path]
subprocess.call(cmd, shell=sh)
else :
if download_platform == 'S2PEPS':
cmd = ['python', cmd_path + 'getS2PEPSFromShp.py', '-w', download_folder, '-d', date_beg, '-f', date_end, download_shp_file, platform_path]
cmd = ['python3', cmd_path + 'getS2PEPSFromShp.py', '-w', download_folder, '-d', date_beg, '-f', date_end, download_shp_file, platform_path]
elif download_platform == 'S2THEIA':
cmd = ['python', cmd_path + 'getS2THEIAFromShp.py', '-w', download_folder, '-d', date_beg, '-f', date_end, download_shp_file, platform_path]
cmd = ['python3', cmd_path + 'getS2THEIAFromShp.py', '-w', download_folder, '-d', date_beg, '-f', date_end, download_shp_file, platform_path]
elif download_platform == 'VenusTHEIA':
cmd = ['python', cmd_path + 'getVenusFromShp.py', '-w', download_folder, '-d', date_beg, '-f', date_end, download_shp_file, platform_path]
cmd = ['python3', cmd_path + 'getVenusFromShp.py', '-w', download_folder, '-d', date_beg, '-f', date_end, download_shp_file, platform_path]
subprocess.call(cmd, shell=sh)
cmd = ['python', cmd_path + 'batchUnzip.py', '-r', download_folder]
cmd = ['python3', cmd_path + 'batchUnzip.py', '-r', download_folder]
subprocess.call(cmd, shell=sh)
preprocess_folder = os.path.join(output_folder, pattern)
if download_platform == 'S2PEPS':
print("Prepare S2PEPS files")
cmd = ['python', cmd_path + 'prepareS2PEPS.py', '-o', preprocess_folder, '-c', shp_file, download_folder]
cmd = ['python3', cmd_path + 'prepareS2PEPS.py', '-o', preprocess_folder, '-c', shp_file, download_folder]
subprocess.call(cmd, shell=sh)
cmd_list = []
with open(os.path.join(cmd_path,'S2PEPSPreparation' + shext)) as f :
......@@ -123,7 +123,7 @@ def main(argv):
# subprocess.call(cmd, shell=sh)
elif download_platform == 'S2THEIA':
print("Prepare S2THEIA files")
cmd = ['python', cmd_path + 'prepareS2THEIA.py', '-o', preprocess_folder, '-c', shp_file, download_folder]
cmd = ['python3', cmd_path + 'prepareS2THEIA.py', '-o', preprocess_folder, '-c', shp_file, download_folder]
subprocess.call(cmd, shell=sh)
cmd_list = []
with open(os.path.join(cmd_path,'S2THEIAPreparation' + shext)) as f :
......@@ -134,7 +134,7 @@ def main(argv):
# subprocess.call(cmd, shell=sh)
elif download_platform == 'VenusTHEIA':
print("Prepare VenusTHEIA files")
cmd = ['python', cmd_path + 'prepareVENUS.py', '-o', preprocess_folder, '-c', shp_file, download_folder]
cmd = ['python3', cmd_path + 'prepareVENUS.py', '-o', preprocess_folder, '-c', shp_file, download_folder]
subprocess.call(cmd, shell=sh)
cmd_list = []
with open(os.path.join(cmd_path,'VENUSPreparation' + shext)) as f :
......@@ -147,7 +147,7 @@ def main(argv):
if remove_src == True :
shutil.rmtree(download_folder)
print("Preprocess %s files" % download_platform)
cmd = ['python', cmd_path + 'genProcessScript.py', '--cloudmask', preprocess_folder]
cmd = ['python3', cmd_path + 'genProcessScript.py', '--cloudmask', preprocess_folder]
subprocess.call(cmd, shell=sh)
cmd_list = []
with open(os.path.join(cmd_path,'ProcessScript' + shext)) as f :
......@@ -159,7 +159,7 @@ def main(argv):
print("Gapfilling %s files" % download_platform)
gapf_folder = os.path.join(output_folder, pattern + '_GAPF')
cmd = ['python', 'cloudfreeComposites.py', '-o', gapf_folder]
cmd = ['python3', 'cloudfreeComposites.py', '-o', gapf_folder]
if dates_file != None :
cmd += ['-d', dates_file]
cmd += [preprocess_folder]
......
......@@ -287,6 +287,8 @@ def genComposite(fld, od_fn, ofld = '.', onExtent = None):
gapmask_calc.SetParameterString("out", gap_fn)
gapmask_calc.SetParameterOutputImagePixelType("out",commonPixTypeToOTB('uint8'))
gapmask_calc.ExecuteAndWriteOutput()
setNoDataValue(gap_fn, None)
# gap_cmd = ['otbcli_BandMathX', '-il', env_fn, msk_fn, '-exp', '{im1b1 != 0 && im2b1 == 0}', '-out', gap_fn, 'uint8']
gaplist.append(gap_fn)
# gapcmdlist.append(gap_cmd)
......
......@@ -433,7 +433,7 @@ def S2THEIA_getExpressions(feat, cfg):
# Red-edge NDVI
if 'NDRE' in feat:
if cfg['b8'].isdigit() and cfg['b5'].isdigit():
expr[feat.index('RNDVI')] = '(im1b' + cfg['b8'] + ' - im1b' + cfg['b5'] + ') / (im1b' + cfg[
expr[feat.index('NDRE')] = '(im1b' + cfg['b8'] + ' - im1b' + cfg['b5'] + ') / (im1b' + cfg[
'b8'] + ' + im1b' + cfg['b5'] + ')'
else:
warnings.warn("No NDRE produced on Sentinel-2, invalid or non present B8 or B5.")
......
......@@ -67,13 +67,13 @@ cd moringa
docker login gitlab-registry.irstea.fr
# OTB base image, example with arg OTB_VERSION=7.1 (default value)
docker build --build-arg OTB_VERSION=7.1 -f docker/base-otb/Dockerfile \
-t gitlab-registry.irstea.fr/raffaele.gaetano/moringa:base-otb-7.1 .
docker push gitlab-registry.irstea.fr/raffaele.gaetano/moringa:base-otb-7.1
# OTB base image, example with arg OTB_VERSION=7.2 (default value)
docker build --build-arg OTB_VERSION=7.2 -f docker/base-otb/Dockerfile \
-t gitlab-registry.irstea.fr/raffaele.gaetano/moringa:base-otb-7.2 .
docker push gitlab-registry.irstea.fr/raffaele.gaetano/moringa:base-otb-7.2
# Tag 'base-otb' == default image for next build
docker tag gitlab-registry.irstea.fr/raffaele.gaetano/moringa:base-otb-7.1 \
docker tag gitlab-registry.irstea.fr/raffaele.gaetano/moringa:base-otb-7.2 \
gitlab-registry.irstea.fr/raffaele.gaetano/moringa:base-otb
docker push gitlab-registry.irstea.fr/raffaele.gaetano/moringa:base-otb
......
FROM ubuntu:18.04
FROM ubuntu:20.04
LABEL description="A base image with OTB and remote modules for the MORINGA processing chain, CIRAD/UMR TETIS"
# Build env
WORKDIR /tmp
ARG OTB_VERSION=7.1
ARG OTB_VERSION=7.2
# System dependencies
RUN apt-get update \
......@@ -39,7 +39,7 @@ RUN git clone -b release-$OTB_VERSION "https://gitlab.orfeo-toolbox.org/orfeotoo
&& mkdir /tmp/SuperBuild-archives /tmp/build /opt/otb \
&& cd /tmp/build \
# Build dependencies
&& cmake ../otb/SuperBuild \
&& cmake /tmp/otb/SuperBuild \
-DCMAKE_BUILD_TYPE='Release' \
-DOTB_USE_QT=OFF \
-DOTB_USE_OPENGL=OFF \
......@@ -50,9 +50,8 @@ RUN git clone -b release-$OTB_VERSION "https://gitlab.orfeo-toolbox.org/orfeotoo
-DCMAKE_INSTALL_PREFIX=/opt/otb \
&& make OTB_DEPENDS -j4 \
# Clone modules
&& cd /tmp && git clone -b memChain "https://framagit.org/ArthurV/otb-for-biomass.git" \
&& ln -s /tmp/otb-for-biomass/MultitempFiltering /tmp/otb/Modules/Remote/MultitempFiltering \
&& cd otb/Modules/Remote && rm -f otbGRM.remote.cmake \
&& cd /tmp/otb/Modules/Remote \
&& rm -f otbGRM.remote.cmake \
&& git clone "https://gitlab.irstea.fr/remi.cresson/GRM.git" \
&& git clone "https://gitlab.irstea.fr/remi.cresson/LSGRM.git" \
&& git clone "https://gitlab.irstea.fr/raffaele.gaetano/otbVectorClassification.git" \
......@@ -60,16 +59,14 @@ RUN git clone -b release-$OTB_VERSION "https://gitlab.orfeo-toolbox.org/orfeotoo
# Build OTB and modules
&& cd /tmp/build/OTB/build \
&& cmake /tmp/otb \
-DOPENCV_core_LIBRARY=/opt/otb/lib/libopencv_core.so \
-DOPENCV_ml_LIBRARY=/opt/otb/lib/libopencv_ml.so \
-DModule_otbGRM=ON \
-DModule_LSGRM=ON \
-DModule_OTBAppSelectiveHaralickTextures=ON \
-DModule_OTBAppVectorFeaturesStatistics=ON \
-DModule_MultitempFiltering=ON \
-DModule_OTBTemporalGapFilling=ON \
-DModule_S1TilingSupportApplications=ON \
-DOTB_WRAP_PYTHON=ON \
-DPYTHON_EXECUTABLE=/usr/bin/python3.6 \
-DPYTHON_EXECUTABLE=/usr/bin/python3.8 \
-DCMAKE_INSTALL_PREFIX=/opt/otb \
&& cd /tmp/build \
&& make -j8 \
......
......@@ -4,25 +4,25 @@ LABEL description="A container for the MORINGA processing chain, CIRAD/UMR TETIS
# Compile obiatools library
RUN git clone "https://gitlab.irstea.fr/raffaele.gaetano/obiatools.git" && cd obiatools/Release \
&& sed -i 's/$HOME\/moringa/\/usr\/local\/lib\/python3.6\/dist-packages/g' configure.sh \
&& sed -i 's/LIB=python3.6/LIB=python3.6m/g' configure.sh \
&& mkdir /usr/local/lib/python3.6/dist-packages/sitsproc_c_modules \
&& touch /usr/local/lib/python3.6/dist-packages/sitsproc_c_modules/__init__.py \
&& sed -i 's/$HOME\/moringa/\/usr\/local\/lib\/python3.8\/dist-packages/g' configure.sh \
&& sed -i 's/python3.6/python3.8/g' configure.sh \
&& mkdir /usr/local/lib/python3.8/dist-packages/sitsproc_c_modules \
&& touch /usr/local/lib/python3.8/dist-packages/sitsproc_c_modules/__init__.py \
&& bash -c "source configure.sh && make" && cd ../.. && rm -rf obiatools
# Other python deps: rios, fmask
RUN pip3 install rios.core \
&& wget -q "https://github.com/ubarsc/python-fmask/releases/download/pythonfmask-0.5.4/python-fmask-0.5.4.tar.gz" \
&& tar -xzf python-fmask-0.5.4.tar.gz && cd python-fmask-0.5.4 \
&& wget -q "https://github.com/ubarsc/python-fmask/releases/download/pythonfmask-0.5.5/python-fmask-0.5.5.tar.gz" \
&& tar -xzf python-fmask-0.5.5.tar.gz && cd python-fmask-0.5.5 \
&& python3 setup.py install && cd .. && rm -rf python-fmask-*
ENV RIOS_DFLT_DRIVER="GTiff"
# Make python3 the default python command
RUN ln -s /usr/bin/python3 /usr/bin/python
# Default user and directory when running a container, is active now
RUN useradd -s /bin/bash -m moringa
USER moringa
WORKDIR /home/moringa
RUN useradd -s /bin/bash -m ubuntu
USER ubuntu
WORKDIR /home/ubuntu
# Clone peps and theia utils
RUN git clone -b python3 "https://github.com/olivierhagolle/theia_download.git" \
......@@ -32,6 +32,6 @@ RUN git clone -b python3 "https://github.com/olivierhagolle/theia_download.git"
RUN wget -q "http://step.esa.int/thirdparties/sen2cor/2.8.0/Sen2Cor-02.08.00-Linux64.run" \
&& bash Sen2Cor-02.08.00-Linux64.run && rm Sen2Cor-02.08.00-Linux64.run -f
# Copy all required files from repo
COPY --chown=moringa aux_data /home/moringa/aux_data
COPY --chown=moringa *.py *.cfg /home/moringa/
# Copy full repository
RUN mkdir moringa
COPY --chown=ubuntu . /home/ubuntu/moringa
......@@ -63,15 +63,15 @@ def main(argv):
f = open('ProcessScript.sh', 'w')
for d in valid_dirs_L8:
f.write('python preprocess.py ' + opt_str + ' ' + d + '\n')
f.write('python3 preprocess.py ' + opt_str + ' ' + d + '\n')
for d in valid_dirs_S2:
f.write('python preprocess.py ' + opt_str + ' ' + opt_str_s2 + ' ' + d + '\n')
f.write('python3 preprocess.py ' + opt_str + ' ' + opt_str_s2 + ' ' + d + '\n')
for d in valid_dirs_S2L2A:
f.write('python preprocess.py ' + opt_str + ' ' + opt_str_s2 + ' ' + d + '\n')
f.write('python3 preprocess.py ' + opt_str + ' ' + opt_str_s2 + ' ' + d + '\n')
for d in valid_dirs_S2_THEIA:
f.write('python preprocess.py ' + opt_str + ' ' + opt_str_s2 + ' ' + d + '\n')
f.write('python3 preprocess.py ' + opt_str + ' ' + opt_str_s2 + ' ' + d + '\n')
for d in valid_dirs_Venus:
f.write('python preprocess.py ' + opt_str + ' ' + opt_str_s2 + ' ' + d + '\n')
f.write('python3 preprocess.py ' + opt_str + ' ' + opt_str_s2 + ' ' + d + '\n')
f.close()
if platform.system() == 'Linux':
......@@ -86,19 +86,19 @@ def main(argv):
f.write('all : ' + joball + '\n\n')
for d in valid_dirs_L8:
f.write(joblist.pop(0) + ':\n')
f.write('\tpython preprocess.py ' + opt_str + ' ' + d + '\n\n')
f.write('\tpython3 preprocess.py ' + opt_str + ' ' + d + '\n\n')
for d in valid_dirs_S2:
f.write(joblist.pop(0) + ':\n')
f.write('\tpython preprocess.py ' + opt_str + ' ' + opt_str_s2 + ' ' + d + '\n\n')
f.write('\tpython3 preprocess.py ' + opt_str + ' ' + opt_str_s2 + ' ' + d + '\n\n')
for d in valid_dirs_S2L2A:
f.write(joblist.pop(0) + ':\n')
f.write('\tpython preprocess.py ' + opt_str + ' ' + opt_str_s2 + ' ' + d + '\n\n')
f.write('\tpython3 preprocess.py ' + opt_str + ' ' + opt_str_s2 + ' ' + d + '\n\n')
for d in valid_dirs_S2_THEIA:
f.write(joblist.pop(0) + ':\n')
f.write('\tpython preprocess.py ' + opt_str + ' ' + opt_str_s2 + ' ' + d + '\n\n')
f.write('\tpython3 preprocess.py ' + opt_str + ' ' + opt_str_s2 + ' ' + d + '\n\n')
for d in valid_dirs_Venus:
f.write(joblist.pop(0) + ':\n')
f.write('\tpython preprocess.py ' + opt_str + ' ' + opt_str_s2 + ' ' + d + '\n\n')
f.write('\tpython3 preprocess.py ' + opt_str + ' ' + opt_str_s2 + ' ' + d + '\n\n')
f.close()
......
......@@ -195,7 +195,7 @@ def main(argv):
if len(valid_dates) > 0:
print("Products have been found!")
for dt in valid_dates:
f.write('python getSentinel2.py ' + opt_str + ' ' + '--sequence ' + str(dt[1]) + ' ' + dt[0] + ' ' + tile + '\n')
f.write('python3 getSentinel2.py ' + opt_str + ' ' + '--sequence ' + str(dt[1]) + ' ' + dt[0] + ' ' + tile + '\n')
f.close()
g.close()
DEVN.close()
......
......@@ -71,7 +71,7 @@ def main(argv):
extent = getShapefileExtent_WGS84(shp_file)
cmd = ['python', peps_folder+'/peps_download.py', '-a', id_file, '-w', output_folder, '-c', collection, '-p', product_type, '--clouds', cloud_cover,
cmd = ['python3', peps_folder+'/peps_download.py', '-a', id_file, '-w', output_folder, '-c', collection, '-p', product_type, '--clouds', cloud_cover,
'--lonmin', str(extent[0]), '--lonmax', str(extent[2]), '--latmin', str(extent[3]), '--latmax', str(extent[1])]
if date_beg != None :
cmd += ['-d', date_beg]
......
......@@ -59,7 +59,7 @@ def main(argv):
extent = getShapefileExtent_WGS84(shp_file)
cmd = ['python2', theia_folder+'/theia_download.py', '-a', id_file, '-w', output_folder, '-c', collection,
cmd = ['python3', theia_folder+'/theia_download.py', '-a', id_file, '-w', output_folder, '-c', collection,
'--lonmin', str(extent[0]), '--lonmax', str(extent[2]), '--latmin', str(extent[3]), '--latmax', str(extent[1]), '--level', 'LEVEL2A']
if date_beg != None :
cmd += ['-d', date_beg]
......
......@@ -59,7 +59,7 @@ def main(argv):
extent = getShapefileExtent_WGS84(shp_file)
cmd = ['python', theia_folder+'/theia_download.py', '-a', id_file, '-w', output_folder, '-c', collection,
cmd = ['python3', theia_folder+'/theia_download.py', '-a', id_file, '-w', output_folder, '-c', collection,
'--lonmin', str(extent[0]), '--lonmax', str(extent[2]), '--latmin', str(extent[3]), '--latmax', str(extent[1])]
if date_beg != None :
cmd += ['-d', date_beg]
......
......@@ -319,7 +319,7 @@ def main(argv):
refl = glob.glob(f + '/' + ptrn)[0]
if ref_S2L2A is None:
# Build reference for S2L2A time series
ref_S2 = f + '/../coreg_ref_S2L2A.tif'
ref_S2L2A = f + '/../coreg_ref_S2L2A.tif'
if not os.path.exists(ref_S2L2A) or force_coreg:
cmd = ['otbcli_Superimpose', '-inr', refl, '-inm', ms_file, '-out', ref_S2L2A,
'uint16']
......
......@@ -1020,4 +1020,43 @@ def per_class_shapefiles(fname,class_fld):
for xds in ds_out:
xds = None
return
\ No newline at end of file
return
def create_unique_db(vec_list,vec_out,class_names=None,class_ids=None):
if class_names is None:
class_names = [os.path.splitext(os.path.basename(x))[0] for x in vec_list]
if class_ids is None:
class_ids = range(1, len(class_names) + 1)
ds_out = None
ly_out = None
fd_out = None
i = 0
for v in vec_list:
ds = ogr.Open(v)
if ds_out is None:
drv = ogr.GetDriverByName('ESRI Shapefile')
ds_out = drv.CreateDataSource(vec_out)
ly_out = ds_out.CreateLayer(os.path.splitext(os.path.basename(vec_out))[0],
ds.GetLayer(0).GetSpatialRef(), ogr.wkbPolygon)
ly_out.CreateField(ogr.FieldDefn('ocs_type', ogr.OFTString))
ly_out.CreateField(ogr.FieldDefn('ocs_code', ogr.OFTInteger))
fd_out = ly_out.GetLayerDefn()
ly = ds.GetLayer()
ly.ResetReading()
for f in ly:
of = ogr.Feature(fd_out)
of.SetGeometry(f.GetGeometryRef())
of.SetField('ocs_type', class_names[i])
of.SetField('ocs_code', class_ids[i])
ly_out.CreateFeature(of)
of = None
ds = None
i += 1
ds_out = None
return
......@@ -59,7 +59,7 @@ def genScript(argv):
else:
mtd = glob.glob(dir + '/*MTD_MSIL2A.xml')
if len(mtd) > 0:
cmd = ['python', 'prepareS2L2A.py', '-o', od]
cmd = ['python3', 'prepareS2L2A.py', '-o', od]
if clip_shp is not None:
cmd += ['-c', clip_shp]
cmd += ['--srs', srs]
......
......@@ -55,7 +55,7 @@ def genScript(argv):
else:
mtd = glob.glob(dir + '/*MTD_MSIL1C.xml')
if len(mtd) > 0:
cmd = ['python', 'prepareS2PEPS.py', '-o', od]
cmd = ['python3', 'prepareS2PEPS.py', '-o', od]
if clip_shp is not None:
cmd += ['-c', clip_shp]
cmd += ['--srs', srs]
......
......@@ -59,7 +59,7 @@ def genScript(argv):
mtd = glob.glob(dir + '/*_MTD_ALL.xml')
if len(mtd) > 0:
#cmd = ['python', 'prepareS2THEIA.py'] + argv[:-1] + [dir]
cmd = ['python', 'prepareS2THEIA.py', '-o', od]
cmd = ['python3', 'prepareS2THEIA.py', '-o', od]
if clip_shp is not None:
cmd += ['-c', clip_shp]
cmd += ['--srs', srs]
......
......@@ -56,7 +56,7 @@ def genScript(argv):
mtd = glob.glob(dir + '/*_MTD_ALL.xml')
if len(mtd) > 0:
#cmd = ['python', 'prepareS2THEIA.py'] + argv[:-1] + [dir]
cmd = ['python', 'prepareS2THEIA_L3A.py', '-o', '\"' + od + '\"']
cmd = ['python3', 'prepareS2THEIA_L3A.py', '-o', '\"' + od + '\"']
if clip_shp is not None:
cmd += ['-c', '\"' + clip_shp + '\"']
cmd += ['--srs', srs]
......
......@@ -55,7 +55,7 @@ def genScript(argv):
mtd = glob.glob(os.path.join(dir, '*_L2VALD_*.HDR'))
if len(mtd) > 0:
#cmd = ['python', 'prepareVENUS.py'] + argv[:-1] + [dir]
cmd = ['python', 'prepareVENUS.py', '-o', od ]
cmd = ['python3', 'prepareVENUS.py', '-o', od ]
if clip_shp is not None:
cmd += ['-c', clip_shp ]
cmd += ['--srs', srs]
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment