Commit bf5e9caa authored by Lozac'h Loic's avatar Lozac'h Loic
Browse files

debug

parent 46116ebb
...@@ -25,7 +25,7 @@ class EodagS1(): ...@@ -25,7 +25,7 @@ class EodagS1():
''' '''
classdocs classdocs
''' '''
logger=None
mEODataAccessGateway = None mEODataAccessGateway = None
user_conf_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),"eodag/user_conf.yml") user_conf_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),"eodag/user_conf.yml")
locations_conf_path = None locations_conf_path = None
...@@ -39,19 +39,19 @@ class EodagS1(): ...@@ -39,19 +39,19 @@ class EodagS1():
''' '''
self.mEODataAccessGateway = EODataAccessGateway(user_conf_file_path=self.user_conf_file_path) self.mEODataAccessGateway = EODataAccessGateway(user_conf_file_path=self.user_conf_file_path)
setup_logging(verbose=0,no_progress_bar=True) setup_logging(verbose=0,no_progress_bar=True)
logger = logging.getLogger(__name__) self.logger = logging.getLogger(__name__)
logger.info("EODataAccessGateway initialized.") self.logger.info("EODataAccessGateway initialized.")
def search_and_download(self, prodtypenum, mrgstile, startdate, enddate, downdir): def search_and_download(self, prodtypenum, mrgstile, startdate, enddate, downdir):
qs_tile = TileEnvelope.objects.annotate(wgs84=Transform('geom',4326)).filter(tile__exact=mrgstile qs_tile = TileEnvelope.objects.annotate(wgs84=Transform('geom',4326)).filter(tile__exact=mrgstile
) )
if len(qs_tile) != 1 : if len(qs_tile) != 1 :
logger.info("Can't find tile "+mrgstile) self.logger.info("Can't find tile "+mrgstile)
exit() exit()
# geomwgs84 = Transform(qs_tile[0].geom, 4326) # geomwgs84 = Transform(qs_tile[0].geom, 4326)
logger.info(qs_tile[0].wgs84) self.logger.info(qs_tile[0].wgs84)
mpolywkt= str(qs_tile[0].wgs84.ewkt).split(";") mpolywkt= str(qs_tile[0].wgs84.ewkt).split(";")
print(mpolywkt[1]) print(mpolywkt[1])
...@@ -60,7 +60,7 @@ class EodagS1(): ...@@ -60,7 +60,7 @@ class EodagS1():
products_availability={} products_availability={}
if prodtypenum in [0,1]: if prodtypenum in [0,1]:
logger.info("Search on Scihub...") self.logger.info("Search on Scihub...")
self.mEODataAccessGateway.set_preferred_provider("scihub") self.mEODataAccessGateway.set_preferred_provider("scihub")
...@@ -71,10 +71,10 @@ class EodagS1(): ...@@ -71,10 +71,10 @@ class EodagS1():
geom=mpolywkt[1] geom=mpolywkt[1]
) )
logger.info( self.logger.info(
f"The Scihub search has found an estimated number of {estimated_scihub_of_results} products matching your criteria " f"The Scihub search has found an estimated number of {estimated_scihub_of_results} products matching your criteria "
) )
logger.info("Search on Peps...") self.logger.info("Search on Peps...")
self.mEODataAccessGateway.set_preferred_provider("peps") self.mEODataAccessGateway.set_preferred_provider("peps")
products_peps, estimated_peps_of_results = self.mEODataAccessGateway.search( products_peps, estimated_peps_of_results = self.mEODataAccessGateway.search(
...@@ -84,7 +84,7 @@ class EodagS1(): ...@@ -84,7 +84,7 @@ class EodagS1():
geom=mpolywkt[1] geom=mpolywkt[1]
) )
logger.info( self.logger.info(
f"The PEPS search has found an estimated number of {estimated_peps_of_results} products matching your criteria " f"The PEPS search has found an estimated number of {estimated_peps_of_results} products matching your criteria "
) )
...@@ -123,13 +123,13 @@ class EodagS1(): ...@@ -123,13 +123,13 @@ class EodagS1():
if pp.properties["storageStatus"] == "ONLINE": if pp.properties["storageStatus"] == "ONLINE":
nbpeps_online += 1 nbpeps_online += 1
logger.info( self.logger.info(
f"\nScihub has got {nbsci_online} ONLINE products over {estimated_scihub_of_results}" f"\nScihub has got {nbsci_online} ONLINE products over {estimated_scihub_of_results}"
f"\nPEPS has got {nbpeps_online} ONLINE products over {estimated_peps_of_results}" f"\nPEPS has got {nbpeps_online} ONLINE products over {estimated_peps_of_results}"
) )
elif prodtypenum == 2: elif prodtypenum == 2:
logger.info("Search on Theia...") self.logger.info("Search on Theia...")
self.mEODataAccessGateway.set_preferred_provider("theia") self.mEODataAccessGateway.set_preferred_provider("theia")
...@@ -139,7 +139,7 @@ class EodagS1(): ...@@ -139,7 +139,7 @@ class EodagS1():
end=enddate, end=enddate,
geom=mpolywkt[1] geom=mpolywkt[1]
) )
logger.info( self.logger.info(
f"The THEIA search has found an estimated number of {estimated_theia_of_results} products matching your criteria " f"The THEIA search has found an estimated number of {estimated_theia_of_results} products matching your criteria "
) )
nbtheia_online=0 nbtheia_online=0
...@@ -148,12 +148,12 @@ class EodagS1(): ...@@ -148,12 +148,12 @@ class EodagS1():
if pt.properties["storageStatus"] == "ONLINE": if pt.properties["storageStatus"] == "ONLINE":
nbtheia_online += 1 nbtheia_online += 1
logger.info( self.logger.info(
f"\nTHIEA has got {nbtheia_online} ONLINE products over {estimated_theia_of_results}" f"\nTHIEA has got {nbtheia_online} ONLINE products over {estimated_theia_of_results}"
) )
else: else:
logger.error("error") self.logger.error("error")
return return
self.products_availability = products_availability self.products_availability = products_availability
...@@ -177,12 +177,12 @@ class EodagS1(): ...@@ -177,12 +177,12 @@ class EodagS1():
if len(self.products_downloaded[d]) == 2 : if len(self.products_downloaded[d]) == 2 :
altdownloadedprods.append(self.products_downloaded[d][1]) altdownloadedprods.append(self.products_downloaded[d][1])
logger.info(f"\nSerializing downloaded products to {serialfile}") self.logger.info(f"\nSerializing downloaded products to {serialfile}")
dps = SearchResult(downloadedprods) dps = SearchResult(downloadedprods)
self.mEODataAccessGateway.serialize(dps, serialfile) self.mEODataAccessGateway.serialize(dps, serialfile)
if len(altdownloadedprods) > 0 : if len(altdownloadedprods) > 0 :
logger.info(f"\nSerializing alternate products to {altserialfile}") self.logger.info(f"\nSerializing alternate products to {altserialfile}")
adps = SearchResult(altdownloadedprods) adps = SearchResult(altdownloadedprods)
self.mEODataAccessGateway.serialize(adps, altserialfile) self.mEODataAccessGateway.serialize(adps, altserialfile)
...@@ -195,12 +195,12 @@ class EodagS1(): ...@@ -195,12 +195,12 @@ class EodagS1():
if len(self.products_availability[d]) == 2 : if len(self.products_availability[d]) == 2 :
altremainingprods.append(self.products_availability[d][1]) altremainingprods.append(self.products_availability[d][1])
logger.info(f"\nSerializing downloaded products to {remainserialfile}") self.logger.info(f"\nSerializing downloaded products to {remainserialfile}")
rps = SearchResult(remainingprods) rps = SearchResult(remainingprods)
self.mEODataAccessGateway.serialize(rps, remainserialfile) self.mEODataAccessGateway.serialize(rps, remainserialfile)
if len(altremainingprods) > 0 : if len(altremainingprods) > 0 :
logger.info(f"\nSerializing alternate products to {altremainserialfile}") self.logger.info(f"\nSerializing alternate products to {altremainserialfile}")
arps = SearchResult(altremainingprods) arps = SearchResult(altremainingprods)
self.mEODataAccessGateway.serialize(arps, altremainserialfile) self.mEODataAccessGateway.serialize(arps, altremainserialfile)
...@@ -224,8 +224,8 @@ class EodagS1(): ...@@ -224,8 +224,8 @@ class EodagS1():
else: else:
self.products_availability[pp.properties["id"]] = [pp] self.products_availability[pp.properties["id"]] = [pp]
logger.info("Geojson files successfully imported") self.logger.info("Geojson files successfully imported")
logger.info("Use start_downloads() to launch downloads") self.logger.info("Use start_downloads() to launch downloads")
def start_downloads(self,downdir): def start_downloads(self,downdir):
...@@ -242,7 +242,7 @@ class EodagS1(): ...@@ -242,7 +242,7 @@ class EodagS1():
while still2download: while still2download:
for p in sorted(self.products_availability, key=lambda k: len(self.products_availability[k]), reverse=True): for p in sorted(self.products_availability, key=lambda k: len(self.products_availability[k]), reverse=True):
firstprovider = self.products_availability[p][0].provider firstprovider = self.products_availability[p][0].provider
logger.info(f"\nStarting Download {p} from {firstprovider}...") self.logger.info(f"\nStarting Download {p} from {firstprovider}...")
self.mEODataAccessGateway.set_preferred_provider(firstprovider) self.mEODataAccessGateway.set_preferred_provider(firstprovider)
failed=False failed=False
try: try:
...@@ -259,7 +259,7 @@ class EodagS1(): ...@@ -259,7 +259,7 @@ class EodagS1():
if failed and len(self.products_availability[p]) == 2: if failed and len(self.products_availability[p]) == 2:
secndprovider = self.products_availability[p][1].provider secndprovider = self.products_availability[p][1].provider
failed=False failed=False
logger.info(f"\nDownload failed. Trying with {secndprovider}") self.logger.info(f"\nDownload failed. Trying with {secndprovider}")
self.mEODataAccessGateway.set_preferred_provider(secndprovider) self.mEODataAccessGateway.set_preferred_provider(secndprovider)
try: try:
with closing(StringIO()) as tqdm_out: with closing(StringIO()) as tqdm_out:
...@@ -283,7 +283,7 @@ class EodagS1(): ...@@ -283,7 +283,7 @@ class EodagS1():
if len(self.products_downloaded) == nbProducts2d : if len(self.products_downloaded) == nbProducts2d :
still2download = False still2download = False
logger.info( self.logger.info(
f"\nAll downloads successful." f"\nAll downloads successful."
) )
self.__export_geojson(downdir, 0) self.__export_geojson(downdir, 0)
...@@ -292,7 +292,7 @@ class EodagS1(): ...@@ -292,7 +292,7 @@ class EodagS1():
elif nb_allretry == 5 : elif nb_allretry == 5 :
still2download = False still2download = False
logger.info( self.logger.info(
""" """
Max number of retries has been completed. Max number of retries has been completed.
There is still products to download. There is still products to download.
...@@ -301,13 +301,13 @@ class EodagS1(): ...@@ -301,13 +301,13 @@ class EodagS1():
) )
self.__export_geojson(downdir, 1) self.__export_geojson(downdir, 1)
logger.info("Done.") self.logger.info("Done.")
else: else:
logger.info(f"\nPass n#{nb_allretry} has reached {len(self.products_downloaded)} over {nbProducts2d} expected") self.logger.info(f"\nPass n#{nb_allretry} has reached {len(self.products_downloaded)} over {nbProducts2d} expected")
logger.info(f"\nWaiting {passwait[nb_allretry-1]} minutes before retrying...") self.logger.info(f"\nWaiting {passwait[nb_allretry-1]} minutes before retrying...")
restartd = datetime.now()+timedelta(minutes=passwait[nb_allretry-1]) restartd = datetime.now()+timedelta(minutes=passwait[nb_allretry-1])
restartstr = restartd.strftime("%Y/%m/%d-%H:%M:%S") restartstr = restartd.strftime("%Y/%m/%d-%H:%M:%S")
logger.info(f"\nRestart at {restartstr}") self.logger.info(f"\nRestart at {restartstr}")
time.sleep(passwait[nb_allretry-1]*60) time.sleep(passwait[nb_allretry-1]*60)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment