Commit 94d91afb authored by Lozac'h Loic's avatar Lozac'h Loic
Browse files

ADD Eodag download

parents 1f983c18 dbeb90dc
......@@ -144,7 +144,7 @@ RUN mkdir -p /work/otb/build \
RUN cd /work/otb/otb/Modules/Remote \
&& git clone https://10.134.193.38/loic.lozach/agrifrozenareas.git \
&& git clone https://gitlab.irstea.fr/loic.lozach/agrifrozenareas.git \
&& cd /work/otb/build/OTB/build \
&& cmake /work/otb/otb \
-DOTB_WRAP_PYTHON=ON \
......@@ -165,6 +165,7 @@ WORKDIR /work/python
RUN pip3 install --upgrade pip
RUN pip3 install -r requirements.txt
COPY eodag_sentinelsat.py.patch /usr/local/lib/python3.6/dist-packages/eodag_sentinelsat/eodag_sentinelsat.py
# ----------------------------------------------------------------------------
# Clean
# ----------------------------------------------------------------------------
......
0 0 255 0
1 255 255 0
2 255 0 0
255 255 255 255
#!/usr/bin/python
__author__ = "Loic Lozach"
__date__ = "$Dec 14, 2018 12:40:21 PM$"
import json, os, sys, argparse, zipfile
from subprocess import Popen, PIPE
try:
from osgeo import ogr, osr, gdal
except:
sys.exit('ERROR: cannot find GDAL/OGR modules')
import otbApplication
def process_command(cmd,chgdir):
print("Starting : "+" ".join(cmd))
print("working dir : "+chgdir)
wd = os.getcwd()
os.chdir(chgdir)
p = Popen(cmd, stdout=PIPE)
# p.wait()
output = p.communicate()[0]
if p.returncode != 0:
print("process failed %d : %s" % (p.returncode, output))
print("#################################################")
os.chdir(wd)
return p.returncode
def create_dict_assets(smhref, qklhref):
mvassets = {
"shapefile":{
"href":smhref,
"type":"application/zip"
}, "thumbnail":{
"href":qklhref,
"type":"image/png"
}
}
return mvassets
def create_dict_properties(datetime,platform,gsd,epsg,title,description):
#gsd => resolution en metre
mvproperties = {
"datetime":datetime,
"title":title,
"description":description,
"platform":platform,
"eo:gsd":gsd,
"eo:epsg":epsg
}
return mvproperties
def create_dict_geometry(coordinates,geomtype="Polygon",epsg=2154):
mvgeom = {
"type":geomtype,
"coordinates":[coordinates],
"crs":{
"type":"name",
"properties":{
"name":"epsg:"+str(epsg)
}
}
}
return mvgeom
def create_dict_image(imgid, geometry, properties, assets, imgtype="Feature"):
mvdata = {
"id":imgid,
"type":imgtype,
"geometry":geometry,
"properties":properties,
"assets":assets
}
return mvdata
def search_files(directory='.', resolution='S1', extension='SAFE', fictype='d'):
images=[]
extension = extension.lower()
resolution = resolution.lower()
for dirpath, dirnames, files in os.walk(directory):
if fictype == 'f':
for name in files:
# print(os.path.join(dirpath, name) + " test")
if extension and name.lower().endswith(extension) and name.lower().find(resolution) >= 0 :
# print(os.path.join(dirpath, name) + " OK")
abspath = os.path.abspath(os.path.join(dirpath, name))
images.append(abspath)
elif fictype == 'd':
for dirname in dirnames:
# print(os.path.join(dirpath, name) + " test")
if extension and dirname.lower().endswith(extension) and dirname.lower().find(resolution) >= 0 :
# print(os.path.join(dirpath, name) + " OK")
abspath = os.path.abspath(os.path.join(dirpath, dirname))
images.append(abspath)
else:
print("search_files type error")
exit()
return images
def generate_quicklook(shp, ref, lut, quick):
app0 = otbApplication.Registry.CreateApplication("Rasterization")
app0.SetParameterString("in",shp)
app0.SetParameterString("out", "temp0.tif")
app0.SetParameterString("im", ref)
app0.SetParameterString("mode", "attribute")
app0.SetParameterString("mode.attribute.field", "FROZ_TYPE")
app0.SetParameterInt("background", 255)
app0.SetParameterOutputImagePixelType("out", otbApplication.ImagePixelType_uint8)
app0.Execute()
# The following line creates an instance of the Superimpose application
app1 = otbApplication.Registry.CreateApplication("ColorMapping")
# The following lines set all the application parameters:
app1.SetParameterInputImage("in", app0.GetParameterOutputImage("out"))
app1.SetParameterString("out", "temp1.tif")
app1.SetParameterString("op", "labeltocolor")
app1.SetParameterString("method", "custom")
app1.SetParameterString("method.custom.lut", lut)
print("Launching... Resampling")
# The following line execute the application
app1.Execute() #ExecuteAndWriteOutput()
print("End of Resampling \n")
app2 = otbApplication.Registry.CreateApplication("Quicklook")
# The following lines set all the application parameters:
app2.SetParameterInputImage("in", app1.GetParameterOutputImage("out"))
app2.SetParameterString("out", quick)
app2.SetParameterInt("sx", 1024)
print("Launching... Resampling")
# The following line execute the application
app2.ExecuteAndWriteOutput() #ExecuteAndWriteOutput()
print("End of Resampling \n")
def reprojectRaster(absfile):
temp_file = absfile + ".old"
os.rename(absfile, temp_file)
p = Popen(['gdalwarp', '-t_srs', 'EPSG:4326', temp_file, absfile], stdout=PIPE)
# p.wait()
output = p.communicate()[0]
if p.returncode != 0:
print("gdalwarp failed %d : %s" % (p.returncode, output))
with open("gdalwrap_log.err",'a') as err:
err.write("######################################################################################################")
err.write(absfile)
err.write(output)
err.write("######################################################################################################")
os.remove(absfile)
os.rename(temp_file, absfile)
return 1
print("gdalwarp succeeded on : "+absfile)
os.remove(temp_file)
return 0
def get_coordinates(img):
raster = gdal.Open(img)
proj = osr.SpatialReference(wkt=raster.GetProjection())
mv_epsg = int(proj.GetAttrValue('AUTHORITY',1))
upx, xres, xskew, upy, yskew, yres = raster.GetGeoTransform()
cols = raster.RasterXSize
rows = raster.RasterYSize
ulx = upx + 0*xres + 0*xskew
uly = upy + 0*yskew + 0*yres
llx = upx + 0*xres + rows*xskew
lly = upy + 0*yskew + rows*yres
lrx = upx + cols*xres + rows*xskew
lry = upy + cols*yskew + rows*yres
urx = upx + cols*xres + 0*xskew
ury = upy + cols*yskew + 0*yres
pointLL, pointUL, pointUR, pointLR = ogr.Geometry(ogr.wkbPoint), ogr.Geometry(ogr.wkbPoint), ogr.Geometry(ogr.wkbPoint), ogr.Geometry(ogr.wkbPoint)
pointLL.AddPoint(llx, lly)
pointUR.AddPoint(urx, ury)
pointUL.AddPoint(ulx, uly)
pointLR.AddPoint(lrx, lry)
# if not proj.IsGeographic() :
#
# outSpatialRef = osr.SpatialReference()
# outSpatialRef.ImportFromEPSG(4326)
# coordTransform = osr.CoordinateTransformation(proj, outSpatialRef)
#
# pointLL.Transform(coordTransform)
# pointUR.Transform(coordTransform)
# pointUL.Transform(coordTransform)
# pointLR.Transform(coordTransform)
return mv_epsg, xres, [[pointLL.GetX(),pointLL.GetY()],[pointUL.GetX(),pointUL.GetY()],[pointUR.GetX(),pointUR.GetY()],[pointLR.GetX(),pointLR.GetY()],[pointLL.GetX(),pointLL.GetY()]]
if __name__ == "__main__":
# Make parser object
parser = argparse.ArgumentParser(description=
"""
Create zipfile, geojson metadata and png quicklook from freeze detection shapefile for THISME ingestion
Output directory must be in /mnt/GEOSUD_WA/SoilMoisture/FreezeDetection
""")
#####################################################
driver = gdal.GetDriverByName('GTiff')
driver.Register()
#####################################################
parser.add_argument('-indir', action='store', required=True, help="Directory to find recursively shapefiles")
parser.add_argument('-outdir', action='store', required=True, help="Output directory for zipfiles")
parser.add_argument('-refimg', action='store', required=True, help="Reference raster for rasterization, RPG tif file")
parser.add_argument('-lut', action='store', required=False, help='LUT file for quicklook generation')
parser.add_argument('-overwrite', choices=['no', 'all',"zip","json","png"], default='no', required=False, help='[Optional] Overwrite already existing files, default no')
args=parser.parse_args()
if not os.path.exists(args.indir):
print("Error: Directory does not exist "+args.indir)
if not os.path.exists(args.outdir):
os.mkdir(args.outdir)
shpfiles = search_files(args.indir,resolution='FREEZEDETECT_', extension='shp', fictype='f')
shpexts = ["shp","shx","dbf","prj"]
mv_epsg, pixelWidth, mv_coordinates = get_coordinates(args.refimg)
for shp in shpfiles :
dirshp = os.path.dirname(shp)
outzip = os.path.basename(shp)[:-3]+"zip"
outabszip = os.path.join(args.outdir,outzip)
ashp = outzip[:-3]+"*"
geojsonfile = outabszip[:-3]+"JSON"
quicklookfile = outabszip[:-3]+"PNG"
print("################################################################")
print("Processing file " + outabszip)
if not os.path.exists(outabszip) or args.overwrite == "all" or args.overwrite == "zip":
# cmd=["zip", os.path.join(args.outdir,outzip) , ashp ]
# process_command(cmd, dirshp)
with zipfile.ZipFile(os.path.join(args.outdir,outzip), "w", compression=zipfile.ZIP_DEFLATED) as zf:
for shpex in shpexts :
tozip = shp[:-3]+shpex
zf.write(tozip, os.path.basename(tozip))
else:
print("Already exists. Passing. ")
print("Processing file " + geojsonfile)
if not os.path.exists(geojsonfile) or args.overwrite == "all" or args.overwrite == "json" :
spltpath = outabszip.split("/")
spltfilename = outzip[:-4].split("_")
ind = spltpath.index("SoilMoisture")
mv_url = "https://products.thisme.cines.teledetection.fr/" + '/'.join(spltpath[ind+1:])
qkl_url = mv_url[:-3]+"PNG"
date = spltfilename[2]
date_in_format = date[0:4]+"-"+date[4:6]+"-"+date[6:8]+date[8:11]+":"+date[11:13]+":"+date[13:15]+"Z"
mv_title=spltfilename[1]
mv_description="Detection of frozen soils in agricultural areas over MRGS tile #"+mv_title
if "DES" in shp:
orb = "DES"
else:
orb = "ASC"
mv_plateform = "S1X-"+orb
mv_properties = create_dict_properties(date_in_format,mv_plateform,pixelWidth,mv_epsg,mv_title,mv_description)
mv_assets = create_dict_assets(mv_url, qkl_url)
mv_geometry = create_dict_geometry(mv_coordinates)
mv_geojson = create_dict_image(outzip[:-4], mv_geometry, mv_properties, mv_assets)
print("Writing json file.")
with open(geojsonfile,'w') as jsonf:
json.dump(mv_geojson, jsonf, indent=4)
else:
print("Already exists. Passing. ")
print("Processing file " + quicklookfile)
if not os.path.exists(quicklookfile) or args.overwrite == "all" or args.overwrite == "png" :
print("Generating quicklook")
generate_quicklook(shp, args.refimg, args.lut, quicklookfile)
else:
print("Already exists. Passing. ")
print("################################################################")
print("Process done.")
This diff is collapsed.
import time
import enum
from logging import getLogger
import logging
class Timer():
def __init__(self):
self._max=125
self._tick=0
self._running = False
def tick(self):
""" Call this to start timer """
# Store the current time in _tick
self._tick=time.time()
self._running = True
def tock(self):
""" Call this to stop timer """
# Add the delta in the times-list
# if time.time()-self._tick > self._max :
# self.tick()
return (time.time()-self._tick)
def half_tick(self):
if not self._running:
self.tick()
self._tick = self._max/2
def running(self):
return self._running
def stop(self):
self._running = False
def has_elapsed(self,seconds):
if time.time() - self._tick >= seconds:
return True
else:
return False
class Status(enum.Enum):
Offline=0
Pending = 1
Requested=2
Online=3
Downloading=4
Downloaded=5
Hash=6
Succeeded=7
Failed = 8
Corrupted=9
# TODO: replace enums with aenums later
def __str__(self):
if self.value == 0:
return "Offline"
elif self.value == 1:
return "Pending"
elif self.value == 2:
return "Requested"
elif self.value == 3:
return "Online"
elif self.value == 4:
return "Downloading"
elif self.value == 5:
return "Downloaded"
elif self.value == 6:
return "Hashing"
elif self.value == 7:
return "Succeeded"
elif self.value == 8:
return "Failed"
elif self.value == 9:
return "Corrupted"
def __lt__(self, other):
if isinstance(other, self.__class__):
return self.value < other.value
return False
def __le__(self, other):
if isinstance(other, self.__class__):
return self.value <= other.value
return False
def __gt__(self, other):
if isinstance(other, self.__class__):
return self.value > other.value
return False
def __ge__(self, other):
if isinstance(other, self.__class__):
return self.value >= other.value
return False
def __ne__(self, other):
if isinstance(other, self.__class__):
return self.value != other.value
return False
def __eq__(self, other):
if isinstance(other, self.__class__):
return self.value == other.value
return False
def setup_logger(key,sio):
fmt = '%(asctime)s [%(levelname)s] %(message)s'
datefmt='%d/%d %H:%M'
log = getLogger(key)
formatter = logging.Formatter(fmt,datefmt)
console = logging.StreamHandler(sio)
console.setFormatter(formatter)
log.addHandler(console)
log.setLevel("INFO")
return log
This diff is collapsed.
from misc import Status,setup_logger
from pySmartDL import SmartDL as DM
from pySmartDL import utils
from io import StringIO
import requests
import threading
import time
from multiprocessing import Process,Manager
import os
import hashlib
import uuid
import json
import logging
_checksum_process = None
_checksum_process_manager = None
_checksum_process_queue = None
_checksum_process_results = None
_checksum_process_signaler = None
class Payload:
# Recheck period in seconds
ESA_CREDENTIALS = ()
__thread_rlock = threading.RLock()
def __init__(self,
key,
download_link,
output_dir,
filename,
md5_checksum,
payload_size,
status=Status.Offline,
ext=".zip",
checksum_download=True,
abandon_download=False):
self.__key = key
self.__filename = filename
self.__download_link = download_link
self.__output_dir = output_dir
self.__credentials = Payload.ESA_CREDENTIALS
self.__status = status
self.__last_status = status
self.__changed_status = False
self.__logger_stream = StringIO()
self.__logger = setup_logger(key,self.__logger_stream)
self.__downloader = None
self.__downloader_ready = False
self.__queued_for_download = True
self.__md5_checksum = md5_checksum.lower()
self.__payload_size = payload_size
self.__ext = ext
self.__hash_check_thread = None
self.__request_thread = None
self.__recheck_thread = None
self.__fail_retries=3
self.__do_checksum = checksum_download
self.__abandon = abandon_download
# seconds to wait before re-downloading
self.__fail_retry_in = 60
self.__fail_retry_at = None
self.__retry_in_multiplier = 2
self.__uid = uuid.uuid4().hex
@property
def key(self):
return self.__key
@property
def filename(self):
return self.__filename
@property
def product_short_name(self):
prodName = self.__filename
return prodName[0:4]+prodName[17:32]
@property
def download_link(self):
return self.__download_link
@property
def output_dir(self):
return self.__output_dir
@output_dir.setter
def output_dir(self, value):
self.__output_dir = value
@property
def full_path(self):
return self.__output_dir+"/"+self.__filename+self.__ext
@property
def size(self):
return self.__payload_size
@property
def esa_credentials(self):
return self.__credentials
@esa_credentials.setter
def esa_credentials(self,value):
if type(value) is not tuple and len(value) != 2:
raise ValueError('Credentials should be a tuple of length 2')
else:
self.__credentials = value
@property
def downloader(self):
return self.__downloader
@property
def status(self):
return self.__status
@status.setter
def status(self,value):
self.__last_status = self.__status
self.__status = value
@property
def status_change(self):
self.__changed_status = self.__status != self.__last_status
return self.__changed_status
@property
def last_status(self):
return self.__last_status
@property
def log(self):
return self.__logger
@property
def log_text(self):
return self.__logger_stream
@property
def queued_for_download(self):
return self.__queued_for_download
@queued_for_download.setter
def queued_for_fownload(self,value):
self.__queued_for_download = value
def toggle_download(self):
self.__queued_for_download = not self.__queued_for_download