ADD Eodag download

Dockerfile

@@ -144,7 +144,7 @@ RUN mkdir -p /work/otb/build \
 
 
 RUN cd /work/otb/otb/Modules/Remote \
- && git clone https://10.134.193.38/loic.lozach/agrifrozenareas.git \
+ && git clone https://gitlab.irstea.fr/loic.lozach/agrifrozenareas.git \
  && cd /work/otb/build/OTB/build \
  && cmake /work/otb/otb \
 	-DOTB_WRAP_PYTHON=ON \
@@ -165,6 +165,7 @@ WORKDIR /work/python
 RUN pip3 install --upgrade pip
 RUN pip3 install -r requirements.txt
 
+COPY eodag_sentinelsat.py.patch /usr/local/lib/python3.6/dist-packages/eodag_sentinelsat/eodag_sentinelsat.py
 # ----------------------------------------------------------------------------
 # Clean
 # ----------------------------------------------------------------------------

FD_LUT.txt

+0	0	255	0
+1	255	255	0
+2	255	0	0
+255	255	255	255

Preprocess4Thisme.py

+#!/usr/bin/python
+__author__ = "Loic Lozach"
+__date__ = "$Dec 14, 2018 12:40:21 PM$"
+
+
+import json, os, sys, argparse, zipfile
+from subprocess import Popen, PIPE
+try:
+    from osgeo import ogr, osr, gdal
+except:
+    sys.exit('ERROR: cannot find GDAL/OGR modules')
+import otbApplication
+
+
+def process_command(cmd,chgdir):
+    print("Starting : "+" ".join(cmd))
+    print("working dir : "+chgdir)
+    wd = os.getcwd()
+    os.chdir(chgdir)
+    p = Popen(cmd, stdout=PIPE)
+#    p.wait()
+    output = p.communicate()[0]
+    if p.returncode != 0: 
+        print("process failed %d : %s" % (p.returncode, output))
+    print("#################################################")
+    
+    os.chdir(wd)
+    return p.returncode
+
+
+def create_dict_assets(smhref, qklhref):
+    mvassets = {
+            "shapefile":{
+                "href":smhref,
+                "type":"application/zip"
+            }, "thumbnail":{
+                "href":qklhref,
+                "type":"image/png"
+            }
+    }
+    return mvassets
+
+def create_dict_properties(datetime,platform,gsd,epsg,title,description):
+    #gsd => resolution en metre
+    mvproperties = {
+        "datetime":datetime,
+        "title":title,
+        "description":description,
+        "platform":platform,
+        "eo:gsd":gsd,
+        "eo:epsg":epsg
+    }
+    return mvproperties
+
+def create_dict_geometry(coordinates,geomtype="Polygon",epsg=2154):
+    mvgeom = {
+        "type":geomtype,
+        "coordinates":[coordinates],
+        "crs":{
+            "type":"name",
+             "properties":{
+                  "name":"epsg:"+str(epsg)
+             }
+        }
+    }
+    return mvgeom
+
+
+def create_dict_image(imgid, geometry, properties, assets, imgtype="Feature"):
+    mvdata = {
+        "id":imgid,
+        "type":imgtype,
+        "geometry":geometry,
+        "properties":properties,
+        "assets":assets
+    }
+    
+    return mvdata
+
+def search_files(directory='.', resolution='S1', extension='SAFE', fictype='d'):
+    images=[]
+    extension = extension.lower()
+    resolution = resolution.lower()
+    for dirpath, dirnames, files in os.walk(directory):
+        if fictype == 'f':
+            for name in files:
+    #            print(os.path.join(dirpath, name) + " test")
+                if extension and name.lower().endswith(extension) and name.lower().find(resolution) >= 0 :
+                    
+    #                print(os.path.join(dirpath, name) + " OK")
+                    abspath = os.path.abspath(os.path.join(dirpath, name))
+                    images.append(abspath)
+        elif fictype == 'd':
+            for dirname in dirnames:
+    #            print(os.path.join(dirpath, name) + " test")
+                if extension and dirname.lower().endswith(extension) and dirname.lower().find(resolution) >= 0 :
+                    
+    #                print(os.path.join(dirpath, name) + " OK")
+                    abspath = os.path.abspath(os.path.join(dirpath, dirname))
+                    images.append(abspath)
+        else:
+            print("search_files type error")
+            exit()
+            
+    return images
+
+
+def generate_quicklook(shp, ref, lut, quick):
+    
+    app0 = otbApplication.Registry.CreateApplication("Rasterization")
+    app0.SetParameterString("in",shp)
+    app0.SetParameterString("out", "temp0.tif")
+    app0.SetParameterString("im", ref)
+    app0.SetParameterString("mode", "attribute")
+    app0.SetParameterString("mode.attribute.field", "FROZ_TYPE")
+    app0.SetParameterInt("background", 255)
+    app0.SetParameterOutputImagePixelType("out", otbApplication.ImagePixelType_uint8)
+    app0.Execute()
+    # The following line creates an instance of the Superimpose application
+    app1 = otbApplication.Registry.CreateApplication("ColorMapping")
+    # The following lines set all the application parameters:
+    app1.SetParameterInputImage("in", app0.GetParameterOutputImage("out"))
+    app1.SetParameterString("out", "temp1.tif")
+    app1.SetParameterString("op", "labeltocolor")
+    app1.SetParameterString("method", "custom")
+    app1.SetParameterString("method.custom.lut", lut)
+    
+    print("Launching... Resampling")
+    # The following line execute the application
+    app1.Execute() #ExecuteAndWriteOutput() 
+    print("End of Resampling \n")
+    
+    app2 = otbApplication.Registry.CreateApplication("Quicklook")
+    
+    # The following lines set all the application parameters:
+    app2.SetParameterInputImage("in", app1.GetParameterOutputImage("out"))
+    app2.SetParameterString("out", quick)
+    app2.SetParameterInt("sx", 1024)
+    
+    print("Launching... Resampling")
+    # The following line execute the application
+    app2.ExecuteAndWriteOutput() #ExecuteAndWriteOutput() 
+    print("End of Resampling \n")
+    
+def reprojectRaster(absfile):
+    temp_file = absfile + ".old"
+    os.rename(absfile, temp_file)
+    
+    
+    p = Popen(['gdalwarp', '-t_srs', 'EPSG:4326', temp_file, absfile], stdout=PIPE)
+#    p.wait()
+    output = p.communicate()[0]
+    if p.returncode != 0: 
+        print("gdalwarp failed %d : %s" % (p.returncode, output))
+        with open("gdalwrap_log.err",'a') as err:
+            err.write("######################################################################################################")
+            err.write(absfile)
+            err.write(output)
+            err.write("######################################################################################################")
+            os.remove(absfile)
+            os.rename(temp_file, absfile)
+        return 1
+        
+    print("gdalwarp succeeded on : "+absfile)
+    os.remove(temp_file)
+    return 0
+
+def get_coordinates(img):
+    raster = gdal.Open(img)
+    proj = osr.SpatialReference(wkt=raster.GetProjection())
+    
+    mv_epsg = int(proj.GetAttrValue('AUTHORITY',1))
+    
+    upx, xres, xskew, upy, yskew, yres = raster.GetGeoTransform()
+    cols = raster.RasterXSize
+    rows = raster.RasterYSize
+     
+    ulx = upx + 0*xres + 0*xskew
+    uly = upy + 0*yskew + 0*yres
+     
+    llx = upx + 0*xres + rows*xskew
+    lly = upy + 0*yskew + rows*yres
+     
+    lrx = upx + cols*xres + rows*xskew
+    lry = upy + cols*yskew + rows*yres
+     
+    urx = upx + cols*xres + 0*xskew
+    ury = upy + cols*yskew + 0*yres
+    
+    pointLL, pointUL, pointUR, pointLR = ogr.Geometry(ogr.wkbPoint), ogr.Geometry(ogr.wkbPoint), ogr.Geometry(ogr.wkbPoint), ogr.Geometry(ogr.wkbPoint)
+    pointLL.AddPoint(llx, lly)
+    pointUR.AddPoint(urx, ury)
+    pointUL.AddPoint(ulx, uly)
+    pointLR.AddPoint(lrx, lry)
+    
+#     if not proj.IsGeographic() :
+#         
+#         outSpatialRef = osr.SpatialReference()
+#         outSpatialRef.ImportFromEPSG(4326)
+#         coordTransform = osr.CoordinateTransformation(proj, outSpatialRef)
+#         
+#         pointLL.Transform(coordTransform)
+#         pointUR.Transform(coordTransform)
+#         pointUL.Transform(coordTransform)
+#         pointLR.Transform(coordTransform)
+    
+    return mv_epsg, xres, [[pointLL.GetX(),pointLL.GetY()],[pointUL.GetX(),pointUL.GetY()],[pointUR.GetX(),pointUR.GetY()],[pointLR.GetX(),pointLR.GetY()],[pointLL.GetX(),pointLL.GetY()]]
+    
+if __name__ == "__main__":
+    # Make parser object
+    parser = argparse.ArgumentParser(description=
+        """
+        Create zipfile, geojson metadata and png quicklook from freeze detection shapefile for THISME ingestion 
+        Output directory must be in /mnt/GEOSUD_WA/SoilMoisture/FreezeDetection
+        """)
+    
+
+    #####################################################
+    driver = gdal.GetDriverByName('GTiff')
+    driver.Register()
+    #####################################################    
+    parser.add_argument('-indir', action='store', required=True, help="Directory to find recursively shapefiles")
+    parser.add_argument('-outdir', action='store', required=True, help="Output directory for zipfiles")
+    parser.add_argument('-refimg', action='store', required=True, help="Reference raster for rasterization, RPG tif file")
+    parser.add_argument('-lut', action='store', required=False, help='LUT file for quicklook generation')
+    parser.add_argument('-overwrite', choices=['no', 'all',"zip","json","png"],  default='no', required=False, help='[Optional] Overwrite already existing files, default no')
+
+    
+    args=parser.parse_args()
+    if not os.path.exists(args.indir):
+        print("Error: Directory does not exist "+args.indir)
+    if not os.path.exists(args.outdir):
+        os.mkdir(args.outdir)
+        
+        
+    shpfiles = search_files(args.indir,resolution='FREEZEDETECT_', extension='shp', fictype='f')
+    shpexts = ["shp","shx","dbf","prj"]
+    
+    mv_epsg, pixelWidth, mv_coordinates = get_coordinates(args.refimg)
+    
+    for shp in shpfiles :
+        dirshp = os.path.dirname(shp)
+        outzip = os.path.basename(shp)[:-3]+"zip"
+        outabszip = os.path.join(args.outdir,outzip)
+        ashp = outzip[:-3]+"*"
+        geojsonfile = outabszip[:-3]+"JSON"
+        quicklookfile = outabszip[:-3]+"PNG"
+        
+        print("################################################################")
+        print("Processing file " + outabszip)
+        
+        if not os.path.exists(outabszip) or args.overwrite == "all" or args.overwrite == "zip":
+#             cmd=["zip", os.path.join(args.outdir,outzip) , ashp ]
+#             process_command(cmd, dirshp) 
+            with zipfile.ZipFile(os.path.join(args.outdir,outzip), "w", compression=zipfile.ZIP_DEFLATED) as zf:
+                for shpex in shpexts :
+                    tozip = shp[:-3]+shpex
+                    zf.write(tozip, os.path.basename(tozip))
+        else:
+            print("Already exists. Passing. ")
+
+
+        print("Processing file " + geojsonfile)
+        if not os.path.exists(geojsonfile) or args.overwrite == "all" or args.overwrite == "json" :
+            spltpath = outabszip.split("/")
+            spltfilename = outzip[:-4].split("_")
+
+            ind = spltpath.index("SoilMoisture")
+            mv_url = "https://products.thisme.cines.teledetection.fr/" + '/'.join(spltpath[ind+1:])
+            qkl_url = mv_url[:-3]+"PNG"
+            
+            date = spltfilename[2]
+            date_in_format = date[0:4]+"-"+date[4:6]+"-"+date[6:8]+date[8:11]+":"+date[11:13]+":"+date[13:15]+"Z"
+
+            mv_title=spltfilename[1]
+            mv_description="Detection of frozen soils in agricultural areas over MRGS tile #"+mv_title
+            if "DES" in shp:
+                orb = "DES"
+            else:
+                orb = "ASC"
+            mv_plateform = "S1X-"+orb
+            
+            
+            mv_properties = create_dict_properties(date_in_format,mv_plateform,pixelWidth,mv_epsg,mv_title,mv_description)
+            
+            mv_assets = create_dict_assets(mv_url, qkl_url)
+            
+            mv_geometry = create_dict_geometry(mv_coordinates)
+            
+            mv_geojson = create_dict_image(outzip[:-4], mv_geometry, mv_properties, mv_assets)
+            
+            print("Writing json file.")
+            with open(geojsonfile,'w') as jsonf:
+                json.dump(mv_geojson, jsonf, indent=4)
+        else:
+            print("Already exists. Passing. ")
+        
+        print("Processing file " + quicklookfile)
+        if not os.path.exists(quicklookfile) or args.overwrite == "all" or args.overwrite == "png" :
+            print("Generating quicklook")
+            generate_quicklook(shp, args.refimg, args.lut, quicklookfile)
+        else:
+            print("Already exists. Passing. ")
+        
+    
+    print("################################################################")
+    print("Process done.")

SentinelDM/misc.py

-import time
-import enum
-from logging import getLogger
-import logging
-
-class Timer():
-    def __init__(self):
-        self._max=125
-        self._tick=0
-        self._running = False
-    def tick(self):
-        """ Call this to start timer """
-        # Store the current time in _tick
-        self._tick=time.time()
-        self._running = True
-        
-    def tock(self):
-        """ Call this to stop timer """
-        # Add the delta in the times-list
-#        if time.time()-self._tick > self._max :
-#            self.tick()
-        return (time.time()-self._tick)
-        
-    def half_tick(self):
-        if not self._running:
-            self.tick()
-        self._tick = self._max/2
-    
-    def running(self):
-        return self._running
-    
-    def stop(self):
-        self._running = False
-        
-    def has_elapsed(self,seconds):
-        if time.time() - self._tick >= seconds:
-            return True
-        else:
-            return False
-
-
-class Status(enum.Enum):
-    
-    Offline=0
-    Pending = 1
-    Requested=2
-    Online=3
-    Downloading=4
-    Downloaded=5
-    Hash=6
-    Succeeded=7
-    Failed = 8
-    Corrupted=9
-     
-    # TODO: replace enums with aenums later
-    def __str__(self):
-        
-        if self.value == 0:
-            return "Offline"
-        elif self.value == 1:
-            return "Pending"
-        elif self.value == 2:
-            return "Requested"
-        elif self.value == 3:
-            return "Online"
-        elif self.value == 4:
-            return "Downloading"
-        elif self.value == 5:
-            return "Downloaded"
-        elif self.value == 6:
-            return "Hashing"
-        elif self.value == 7:
-            return "Succeeded"
-        elif self.value == 8:
-            return "Failed"
-        elif self.value == 9:
-            return "Corrupted"
-
-        
-    def __lt__(self, other):
-        if isinstance(other, self.__class__):
-            return self.value < other.value
-        return False
-    
-    def __le__(self, other):
-        if isinstance(other, self.__class__):
-            return self.value <= other.value
-        return False
-    
-    def __gt__(self, other):
-        if isinstance(other, self.__class__):
-            return self.value > other.value
-        return False
-    
-    def __ge__(self, other):
-        if isinstance(other, self.__class__):
-            return self.value >= other.value
-        return False
-    
-    def __ne__(self, other):
-        if isinstance(other, self.__class__):
-            return self.value != other.value
-        return False
-    
-    def __eq__(self, other):
-        if isinstance(other, self.__class__):
-            return self.value == other.value
-        return False
-    
-def setup_logger(key,sio):
-        
-        fmt = '%(asctime)s [%(levelname)s] %(message)s'
-        datefmt='%d/%d %H:%M'
-        
-        log = getLogger(key)
-        formatter = logging.Formatter(fmt,datefmt)
-        console = logging.StreamHandler(sio)
-        console.setFormatter(formatter)
-        log.addHandler(console)
-        log.setLevel("INFO")
-        return log

SentinelDM/payloadmanager.py

-from misc import Status,setup_logger
-from pySmartDL import SmartDL as DM
-from pySmartDL import utils
-from io import StringIO
-import requests
-import threading
-import time
-from multiprocessing import Process,Manager
-import os
-import hashlib
-import uuid
-import json 
-import logging
-
-_checksum_process = None
-_checksum_process_manager = None
-_checksum_process_queue = None
-_checksum_process_results = None
-_checksum_process_signaler = None
-
-class Payload:
-    
-    # Recheck period in seconds
-    ESA_CREDENTIALS = ()
-    __thread_rlock = threading.RLock()
-    
-    
-    def __init__(self, 
-                 key, 
-                 download_link, 
-                 output_dir, 
-                 filename, 
-                 md5_checksum,
-                 payload_size,
-                 status=Status.Offline,
-                 ext=".zip",
-                 checksum_download=True,
-                 abandon_download=False):
-        
-        self.__key = key
-        self.__filename = filename
-        self.__download_link = download_link
-        self.__output_dir = output_dir
-        self.__credentials =  Payload.ESA_CREDENTIALS
-        self.__status = status
-        self.__last_status = status
-        self.__changed_status = False
-        self.__logger_stream = StringIO()
-        self.__logger = setup_logger(key,self.__logger_stream)
-        self.__downloader = None
-        self.__downloader_ready = False
-        self.__queued_for_download = True
-        self.__md5_checksum = md5_checksum.lower()
-        self.__payload_size = payload_size
-        self.__ext = ext
-        self.__hash_check_thread = None
-        self.__request_thread = None
-        self.__recheck_thread = None
-        self.__fail_retries=3
-        self.__do_checksum = checksum_download
-        self.__abandon = abandon_download
-        # seconds to wait before re-downloading
-        self.__fail_retry_in = 60
-        self.__fail_retry_at = None
-        self.__retry_in_multiplier = 2
-        self.__uid = uuid.uuid4().hex
-        
-    @property
-    def key(self):
-        return self.__key
-    
-    @property
-    def filename(self):
-        return self.__filename
-    
-    @property
-    def product_short_name(self):
-        prodName = self.__filename
-        return prodName[0:4]+prodName[17:32]
-    
-    @property
-    def download_link(self):
-        return self.__download_link
-        
-    @property
-    def output_dir(self):
-        return self.__output_dir
-    
-    @output_dir.setter
-    def output_dir(self, value):
-        self.__output_dir = value
-        
-    @property
-    def full_path(self):
-        return self.__output_dir+"/"+self.__filename+self.__ext
-    
-    @property
-    def size(self):
-        return self.__payload_size
-    
-    @property
-    def esa_credentials(self):
-        return self.__credentials
-    
-    @esa_credentials.setter
-    def esa_credentials(self,value):
-        if type(value) is not tuple and len(value) != 2:
-            raise ValueError('Credentials should be a tuple of length 2')
-        else:
-            self.__credentials = value
-    
-    @property
-    def downloader(self):
-        return self.__downloader
-    
-    @property
-    def status(self):
-        return self.__status
-
-    @status.setter
-    def status(self,value):
-        self.__last_status = self.__status
-        self.__status = value
-    
-    @property
-    def status_change(self):
-        self.__changed_status = self.__status != self.__last_status
-        return self.__changed_status
-    
-    @property
-    def last_status(self):
-        return self.__last_status
-    
-    @property
-    def log(self):
-        return self.__logger
-    
-    @property
-    def log_text(self):
-        return self.__logger_stream
-    
-    @property
-    def queued_for_download(self):
-        return self.__queued_for_download
-    
-    @queued_for_download.setter
-    def queued_for_fownload(self,value):
-        self.__queued_for_download = value
-    
-    def toggle_download(self):
-        self.__queued_for_download = not self.__queued_for_download
-