Commit f012c1b6 authored by Lozac'h Loic's avatar Lozac'h Loic
Browse files

eodag1

parent e1b2fd1c
......@@ -11,6 +11,8 @@ from django.contrib.gis.db.models.functions import Transform
from eodag.utils import ProgressCallback
from contextlib import closing
from io import StringIO
from datetime import datetime, timedelta
import time
# os.environ["EODAG__PEPS__AUTH__CREDENTIALS__USERNAME"] = "loic.lozach"
# os.environ["EODAG__PEPS__AUTH__CREDENTIALS__PASSWORD"] = "PMUQc22$"
......@@ -27,12 +29,18 @@ class EodagS1Downloader():
user_conf_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),"eodag/user_conf.yml")
locations_conf_path = None
# providers_conf_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),"eodag/providers.yml")
productOnline = None
productOffline = None
def __init__(self, mrgstile, startdate, enddate):
def __init__(self, mrgstile, startdate, enddate, downdir):
'''
Constructor
'''
if not os.path.exists(downdir):
os.mkdir(downdir)
setup_logging(verbose=3)
qs_tile = TileEnvelope.objects.annotate(wgs84=Transform('geom',4326)).filter(tile__exact=mrgstile
......@@ -48,13 +56,27 @@ class EodagS1Downloader():
print(mpolywkt[1])
self.mEODataAccessGateway = EODataAccessGateway(self.user_conf_file_path)
print("debug")
print(self.mEODataAccessGateway.get_preferred_provider())
print("Search on Scihub...")
self.mEODataAccessGateway.set_preferred_provider("schihub")
product_type = 'S1_SAR_GRD'
sensor_mode = 'IW'
products, estimated_total_nbr_of_results = self.mEODataAccessGateway.search(
products_scihub, estimated_scihub_of_results = self.mEODataAccessGateway.search(
productType=product_type,
sensoroperationalmode=sensor_mode,
start=startdate,
end=enddate,
geom=mpolywkt[1]
)
print(
f"The Scihub search has found an estimated number of {estimated_scihub_of_results} products matching your criteria "
)
print("Search on Peps...")
self.mEODataAccessGateway.set_preferred_provider("peps")
products_peps, estimated_peps_of_results = self.mEODataAccessGateway.search(
productType=product_type,
sensoroperationalmode=sensor_mode,
start=startdate,
......@@ -63,14 +85,95 @@ class EodagS1Downloader():
)
print(
f"The search has found an estimated number of {estimated_total_nbr_of_results} products matching "
f"our criteria. It has returned the first {len(products)} ones available."
f"The PEPS search has found an estimated number of {estimated_peps_of_results} products matching your criteria "
)
nbsci_online=0
products_availability={}
for ps in products_scihub :
if ps.properties["storageStatus"] == "ONLINE":
nbsci_online += 1
found=False
for pp in products_peps :
if pp.properties["id"] == ps.properties["id"]:
products_availability[ps.properties["id"]] = [ps,pp]
found=True
break
if found == False:
products_availability[ps.properties["id"]] = [ps]
nbpeps_online=0
for pp in products_peps :
found=False
if pp.properties["storageStatus"] == "ONLINE":
nbpeps_online += 1
if not pp.properties["id"] in products_availability.keys():
products_availability[pp.properties["id"]] = [pp]
print(
f"Scihub has got {nbsci_online} ONLINE products over {estimated_scihub_of_results}"
f"PEPS has got {nbpeps_online} ONLINE products over {estimated_peps_of_results}"
)
for prod in products:
print(prod.as_dict()['properties']["identifier"])
nb_dblav=0
for p in sorted(products_availability, key=len(products_availability.get), reverse=True):
if len(products_availability[p]) == 1 :
print(
f"Product {p} is only available on {products_availability[p][0].provider}"
)
else:
nb_dblav+=1
products_downloaded={}
passwait=[10,30,60,120,300]
still2download = True
nbProducts2d = len(products_availability)
nb_allretry = 0
nowstr = datetime.now().strftime("%Y%m%dT%H%M%S")
allserialfile = os.path.join(downdir,nowstr+"_alldownloads.json")
remainserialfile = os.path.join(downdir,nowstr+"_remaindownloads.json")
while still2download:
for p in sorted(products_availability, key=len(products_availability.get), reverse=True):
print(f"Starting Download {p}...")
with closing(StringIO()) as tqdm_out:
with ProgressCallback(file=tqdm_out) as bar:
products_availability[p][0].download( bar, extract=False, outputs_prefix=downdir)
if products_availability[p][0].location[:5] == "file:":
products_downloaded[p] = products_availability[p][0]
for p in products_downloaded.keys():
if p in products_availability.keys():
products_availability.pop(p)
nb_allretry += 1
print("Starting Download...")
with closing(StringIO()) as tqdm_out:
with ProgressCallback(file=tqdm_out) as bar:
self.mEODataAccessGateway.download_all(products, bar, 30, 2880, extract=False, outputs_prefix="/data/S1/download")
if len(products_downloaded) == nbProducts2d :
still2download = False
print(
f"All downloads successful."
f"Serializing search products to {allserialfile}"
f"Exiting."
)
elif nb_allretry == 5 :
still2download = False
print(
f"Max number of retries has been completed."
f"Serializing all search products to {allserialfile}"
f"Serializing remain search products to {remainserialfile}"
f"Exiting."
)
else:
print(f"Pass n#{nb_allretry} has reached {len(products_downloaded)} over {nbProducts2d} expected")
print(f"Waiting {passwait[nb_allretry-1]} minutes before retrying...")
restartd = datetime.now()+timedelta(minutes=passwait[nb_allretry-1])
restartstr = restartd.strftime("%Y/%m/%d-%H:%M:%S")
print(f"Restart at {restartstr}")
time.sleep(passwait[nb_allretry-1]*60)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment