Skip to content

GitLab

Commit dee1fda3 authored by Lozac'h Loic's avatar Lozac'h Loic
Browse files

test

parent 57df37b7
Hide whitespace changes
Inline Side-by-side
frozenapp/eodags1.py
View file @ dee1fda3
......@@ -40,7 +40,7 @@ class EodagS1():
Constructor
'''
self.mEODataAccessGateway = EODataAccessGateway(user_conf_file_path=self.user_conf_file_path)
setup_logging(verbose=3,no_progress_bar=True)
setup_logging(verbose=2,no_progress_bar=False)
logger.info("EODataAccessGateway initialized.")
def search_and_download(self, prodtypenum, mrgstile, startdate, enddate, downdir):
......@@ -124,10 +124,8 @@ class EodagS1():
if pp.properties["storageStatus"] == "ONLINE":
nbpeps_online += 1
logger.info(
f"\nScihub has got {nbsci_online} ONLINE products over {estimated_scihub_of_results}"
f"\nPEPS has got {nbpeps_online} ONLINE products over {estimated_peps_of_results}"
)
logger.info(f"Scihub has got {nbsci_online} ONLINE products over {estimated_scihub_of_results}")
logger.info(f"PEPS has got {nbpeps_online} ONLINE products over {estimated_peps_of_results}")
elif prodtypenum == 2:
logger.info("Search on Theia...")
......@@ -150,7 +148,7 @@ class EodagS1():
nbtheia_online += 1
logger.info(
f"\nTHIEA has got {nbtheia_online} ONLINE products over {estimated_theia_of_results}"
f"THIEA has got {nbtheia_online} ONLINE products over {estimated_theia_of_results}"
)
else:
......@@ -178,12 +176,12 @@ class EodagS1():
if len(self.products_downloaded[d]) == 2 :
altdownloadedprods.append(self.products_downloaded[d][1])
logger.info(f"\nSerializing downloaded products to {serialfile}")
logger.info(f"Serializing downloaded products to {serialfile}")
dps = SearchResult(downloadedprods)
self.mEODataAccessGateway.serialize(dps, serialfile)
if len(altdownloadedprods) > 0 :
logger.info(f"\nSerializing alternate products to {altserialfile}")
logger.info(f"Serializing alternate products to {altserialfile}")
adps = SearchResult(altdownloadedprods)
self.mEODataAccessGateway.serialize(adps, altserialfile)
......@@ -196,12 +194,12 @@ class EodagS1():
if len(self.products_availability[d]) == 2 :
altremainingprods.append(self.products_availability[d][1])
logger.info(f"\nSerializing downloaded products to {remainserialfile}")
logger.info(f"Serializing downloaded products to {remainserialfile}")
rps = SearchResult(remainingprods)
self.mEODataAccessGateway.serialize(rps, remainserialfile)
if len(altremainingprods) > 0 :
logger.info(f"\nSerializing alternate products to {altremainserialfile}")
logger.info(f"Serializing alternate products to {altremainserialfile}")
arps = SearchResult(altremainingprods)
self.mEODataAccessGateway.serialize(arps, altremainserialfile)
......@@ -243,13 +241,13 @@ class EodagS1():
while still2download:
for p in sorted(self.products_availability, key=lambda k: len(self.products_availability[k]), reverse=True):
firstprovider = self.products_availability[p][0].provider
logger.info(f"\nStarting Download {p} from {firstprovider}...")
logger.info(f"Starting Download {p} from {firstprovider}...")
self.mEODataAccessGateway.set_preferred_provider(firstprovider)
failed=False
try:
with closing(StringIO()) as tqdm_out:
with ProgressCallback(file=tqdm_out) as bar:
self.mEODataAccessGateway.download(self.products_availability[p][0], bar, extract=False, outputs_prefix=downdir, timeout=1)
self.mEODataAccessGateway.download(self.products_availability[p][0], bar, extract=False, outputs_prefix=downdir,wait=0, timeout=0)
except Exception as err:
failed=True
print(err)
......@@ -260,12 +258,12 @@ class EodagS1():
if failed and len(self.products_availability[p]) == 2:
secndprovider = self.products_availability[p][1].provider
failed=False
logger.info(f"\nDownload failed. Trying with {secndprovider}")
logger.info(f"Download failed. Trying with {secndprovider}")
self.mEODataAccessGateway.set_preferred_provider(secndprovider)
try:
with closing(StringIO()) as tqdm_out:
with ProgressCallback(file=tqdm_out) as bar:
self.mEODataAccessGateway.download(self.products_availability[p][1], bar, extract=False, outputs_prefix=downdir, timeout=1)
self.mEODataAccessGateway.download(self.products_availability[p][1], bar, extract=False, outputs_prefix=downdir,wait=0, timeout=0)
except Exception as err:
failed=True
print(err)
......@@ -285,11 +283,11 @@ class EodagS1():
if len(self.products_downloaded) == nbProducts2d :
still2download = False
logger.info(
f"\nAll downloads successful."
f"All downloads successful."
)
self.__export_geojson(downdir, 0)
print("Done.")
logger.info("Done.")
elif nb_allretry == 5 :
still2download = False
......@@ -304,11 +302,11 @@ class EodagS1():
logger.info("Done.")
else:
logger.info(f"\nPass n#{nb_allretry} has reached {len(self.products_downloaded)} over {nbProducts2d} expected")
logger.info(f"\nWaiting {passwait[nb_allretry-1]} minutes before retrying...")
logger.info(f"Pass n#{nb_allretry} has reached {len(self.products_downloaded)} over {nbProducts2d} expected")
logger.info(f"Waiting {passwait[nb_allretry-1]} minutes before retrying...")
restartd = datetime.now()+timedelta(minutes=passwait[nb_allretry-1])
restartstr = restartd.strftime("%Y/%m/%d-%H:%M:%S")
logger.info(f"\nRestart at {restartstr}")
logger.info(f"Restart at {restartstr}")
time.sleep(passwait[nb_allretry-1]*60)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment