Skip to content

GitLab

Commit 46116ebb authored by Lozac'h Loic's avatar Lozac'h Loic
Browse files

debug

parent ed9af134
Hide whitespace changes
Inline Side-by-side
frozenapp/eodags1.py
View file @ 46116ebb
......@@ -3,7 +3,7 @@ Created on 18 juin 2021
@author: loic
'''
import os
import os, time, logging
from eodag.utils.logging import setup_logging
from eodag.api.core import EODataAccessGateway
from eodag.api.search_result import SearchResult
......@@ -13,7 +13,7 @@ from eodag.utils import ProgressCallback
from contextlib import closing
from io import StringIO
from datetime import datetime, timedelta
import time
# os.environ["EODAG__PEPS__AUTH__CREDENTIALS__USERNAME"] = "loic.lozach"
# os.environ["EODAG__PEPS__AUTH__CREDENTIALS__PASSWORD"] = "PMUQc22$"
......@@ -39,18 +39,19 @@ class EodagS1():
'''
self.mEODataAccessGateway = EODataAccessGateway(user_conf_file_path=self.user_conf_file_path)
setup_logging(verbose=0,no_progress_bar=True)
print("EODataAccessGateway initialized.")
logger = logging.getLogger(__name__)
logger.info("EODataAccessGateway initialized.")
def search_and_download(self, prodtypenum, mrgstile, startdate, enddate, downdir):
qs_tile = TileEnvelope.objects.annotate(wgs84=Transform('geom',4326)).filter(tile__exact=mrgstile
)
if len(qs_tile) != 1 :
print("Can't find tile "+mrgstile)
logger.info("Can't find tile "+mrgstile)
exit()
# geomwgs84 = Transform(qs_tile[0].geom, 4326)
print(qs_tile[0].wgs84)
logger.info(qs_tile[0].wgs84)
mpolywkt= str(qs_tile[0].wgs84.ewkt).split(";")
print(mpolywkt[1])
......@@ -59,7 +60,7 @@ class EodagS1():
products_availability={}
if prodtypenum in [0,1]:
print("Search on Scihub...")
logger.info("Search on Scihub...")
self.mEODataAccessGateway.set_preferred_provider("scihub")
......@@ -70,10 +71,10 @@ class EodagS1():
geom=mpolywkt[1]
)
print(
logger.info(
f"The Scihub search has found an estimated number of {estimated_scihub_of_results} products matching your criteria "
)
print("Search on Peps...")
logger.info("Search on Peps...")
self.mEODataAccessGateway.set_preferred_provider("peps")
products_peps, estimated_peps_of_results = self.mEODataAccessGateway.search(
......@@ -83,7 +84,7 @@ class EodagS1():
geom=mpolywkt[1]
)
print(
logger.info(
f"The PEPS search has found an estimated number of {estimated_peps_of_results} products matching your criteria "
)
......@@ -122,13 +123,13 @@ class EodagS1():
if pp.properties["storageStatus"] == "ONLINE":
nbpeps_online += 1
print(
logger.info(
f"\nScihub has got {nbsci_online} ONLINE products over {estimated_scihub_of_results}"
f"\nPEPS has got {nbpeps_online} ONLINE products over {estimated_peps_of_results}"
)
elif prodtypenum == 2:
print("Search on Theia...")
logger.info("Search on Theia...")
self.mEODataAccessGateway.set_preferred_provider("theia")
......@@ -138,7 +139,7 @@ class EodagS1():
end=enddate,
geom=mpolywkt[1]
)
print(
logger.info(
f"The THEIA search has found an estimated number of {estimated_theia_of_results} products matching your criteria "
)
nbtheia_online=0
......@@ -147,12 +148,12 @@ class EodagS1():
if pt.properties["storageStatus"] == "ONLINE":
nbtheia_online += 1
print(
logger.info(
f"\nTHIEA has got {nbtheia_online} ONLINE products over {estimated_theia_of_results}"
)
else:
print("error")
logger.error("error")
return
self.products_availability = products_availability
......@@ -176,12 +177,12 @@ class EodagS1():
if len(self.products_downloaded[d]) == 2 :
altdownloadedprods.append(self.products_downloaded[d][1])
print(f"\nSerializing downloaded products to {serialfile}")
logger.info(f"\nSerializing downloaded products to {serialfile}")
dps = SearchResult(downloadedprods)
self.mEODataAccessGateway.serialize(dps, serialfile)
if len(altdownloadedprods) > 0 :
print(f"\nSerializing alternate products to {altserialfile}")
logger.info(f"\nSerializing alternate products to {altserialfile}")
adps = SearchResult(altdownloadedprods)
self.mEODataAccessGateway.serialize(adps, altserialfile)
......@@ -194,12 +195,12 @@ class EodagS1():
if len(self.products_availability[d]) == 2 :
altremainingprods.append(self.products_availability[d][1])
print(f"\nSerializing downloaded products to {remainserialfile}")
logger.info(f"\nSerializing downloaded products to {remainserialfile}")
rps = SearchResult(remainingprods)
self.mEODataAccessGateway.serialize(rps, remainserialfile)
if len(altremainingprods) > 0 :
print(f"\nSerializing alternate products to {altremainserialfile}")
logger.info(f"\nSerializing alternate products to {altremainserialfile}")
arps = SearchResult(altremainingprods)
self.mEODataAccessGateway.serialize(arps, altremainserialfile)
......@@ -223,8 +224,8 @@ class EodagS1():
else:
self.products_availability[pp.properties["id"]] = [pp]
print("Geojson files successfully imported")
print("Use start_downloads() to launch downloads")
logger.info("Geojson files successfully imported")
logger.info("Use start_downloads() to launch downloads")
def start_downloads(self,downdir):
......@@ -241,13 +242,13 @@ class EodagS1():
while still2download:
for p in sorted(self.products_availability, key=lambda k: len(self.products_availability[k]), reverse=True):
firstprovider = self.products_availability[p][0].provider
print(f"\nStarting Download {p} from {firstprovider}...")
logger.info(f"\nStarting Download {p} from {firstprovider}...")
self.mEODataAccessGateway.set_preferred_provider(firstprovider)
failed=False
try:
with closing(StringIO()) as tqdm_out:
with ProgressCallback(file=tqdm_out) as bar:
self.mEODataAccessGateway.download(self.products_availability[p][0], bar, extract=False, outputs_prefix=downdir)
self.mEODataAccessGateway.download(self.products_availability[p][0], bar, extract=False, outputs_prefix=downdir, timeout=1)
except Exception as err:
failed=True
print(err)
......@@ -258,12 +259,12 @@ class EodagS1():
if failed and len(self.products_availability[p]) == 2:
secndprovider = self.products_availability[p][1].provider
failed=False
print(f"\nDownload failed. Trying with {secndprovider}")
logger.info(f"\nDownload failed. Trying with {secndprovider}")
self.mEODataAccessGateway.set_preferred_provider(secndprovider)
try:
with closing(StringIO()) as tqdm_out:
with ProgressCallback(file=tqdm_out) as bar:
self.mEODataAccessGateway.download(self.products_availability[p][1], bar, extract=False, outputs_prefix=downdir)
self.mEODataAccessGateway.download(self.products_availability[p][1], bar, extract=False, outputs_prefix=downdir, timeout=1)
except Exception as err:
failed=True
print(err)
......@@ -282,7 +283,7 @@ class EodagS1():
if len(self.products_downloaded) == nbProducts2d :
still2download = False
print(
logger.info(
f"\nAll downloads successful."
)
self.__export_geojson(downdir, 0)
......@@ -291,7 +292,7 @@ class EodagS1():
elif nb_allretry == 5 :
still2download = False
print(
logger.info(
"""
Max number of retries has been completed.
There is still products to download.
......@@ -300,13 +301,13 @@ class EodagS1():
)
self.__export_geojson(downdir, 1)
print("Done.")
logger.info("Done.")
else:
print(f"\nPass n#{nb_allretry} has reached {len(self.products_downloaded)} over {nbProducts2d} expected")
print(f"\nWaiting {passwait[nb_allretry-1]} minutes before retrying...")
logger.info(f"\nPass n#{nb_allretry} has reached {len(self.products_downloaded)} over {nbProducts2d} expected")
logger.info(f"\nWaiting {passwait[nb_allretry-1]} minutes before retrying...")
restartd = datetime.now()+timedelta(minutes=passwait[nb_allretry-1])
restartstr = restartd.strftime("%Y/%m/%d-%H:%M:%S")
print(f"\nRestart at {restartstr}")
logger.info(f"\nRestart at {restartstr}")
time.sleep(passwait[nb_allretry-1]*60)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment