Commit c2074427 authored by Lozac'h Loic's avatar Lozac'h Loic
Browse files

debug

parent f7aa5661
......@@ -147,61 +147,77 @@ class EodagS1Downloader():
nowstr = datetime.now().strftime("%Y%m%dT%H%M%S")
allserialfile = os.path.join(downdir,nowstr+"_alldownloads.json")
remainserialfile = os.path.join(downdir,nowstr+"_remaindownloads.json")
while still2download:
for p in sorted(products_availability, key=lambda k: len(products_availability[k]), reverse=True):
print(f"\nStarting Download {p}...")
failed=False
try:
with closing(StringIO()) as tqdm_out:
with ProgressCallback(file=tqdm_out) as bar:
products_availability[p][0].download( bar, extract=False, outputs_prefix=downdir)
except Exception as err:
failed=True
print(err)
if failed and len(products_availability[p]) == 2:
failed=False
print("Download failed. Trying with PEPS")
try:
with closing(StringIO()) as tqdm_out:
with ProgressCallback(file=tqdm_out) as bar:
products_availability[p][1].download( bar, extract=False, outputs_prefix=downdir)
except Exception as err:
failed=True
print(err)
if products_availability[p][0].location[:5] == "file:":
products_downloaded[p] = products_availability[p][0]
for p in products_downloaded.keys():
if p in products_availability.keys():
products_availability.pop(p)
nb_allretry += 1
if len(products_downloaded) == nbProducts2d :
still2download = False
print(
f"\nAll downloads successful."
f"\nSerializing search products to {allserialfile}"
f"\nExiting."
)
elif nb_allretry == 5 :
still2download = False
print(
f"\nMax number of retries has been completed."
f"\nSerializing all search products to {allserialfile}"
f"\nSerializing remain search products to {remainserialfile}"
f"\nExiting."
)
else:
print(f"\nPass n#{nb_allretry} has reached {len(products_downloaded)} over {nbProducts2d} expected")
print(f"\nWaiting {passwait[nb_allretry-1]} minutes before retrying...")
restartd = datetime.now()+timedelta(minutes=passwait[nb_allretry-1])
restartstr = restartd.strftime("%Y/%m/%d-%H:%M:%S")
print(f"\nRestart at {restartstr}")
time.sleep(passwait[nb_allretry-1]*60)
#DEBUG
for p in sorted(products_availability, key=lambda k: len(products_availability[k]), reverse=True):
prod0 = products_availability[p][0]
prod0dict = prod0.as_dict()
for item in prod0dict.items():
print(item)
prod1 = products_availability[p][1]
prod1dict = prod1.as_dict()
for item in prod1dict.items():
print(item)
break
# while still2download:
# for p in sorted(products_availability, key=lambda k: len(products_availability[k]), reverse=True):
# firstprovider = products_availability[p][0].provider
# print(f"\nStarting Download {p} from {firstprovider}...")
# failed=False
# try:
# with closing(StringIO()) as tqdm_out:
# with ProgressCallback(file=tqdm_out) as bar:
# products_availability[p][0].download( bar, extract=False, outputs_prefix=downdir)
# except Exception as err:
# failed=True
# print(err)
#
# if failed and len(products_availability[p]) == 2:
# secndprovider = products_availability[p][1].provider
# failed=False
# print("Download failed. Trying with {secndprovider}")
# try:
# with closing(StringIO()) as tqdm_out:
# with ProgressCallback(file=tqdm_out) as bar:
# products_availability[p][1].download( bar, extract=False, outputs_prefix=downdir)
# except Exception as err:
# failed=True
# print(err)
#
#
# if products_availability[p][0].location[:5] == "file:":
# products_downloaded[p] = products_availability[p][0]
#
# for p in products_downloaded.keys():
# if p in products_availability.keys():
# products_availability.pop(p)
# nb_allretry += 1
#
# if len(products_downloaded) == nbProducts2d :
# still2download = False
# print(
# f"\nAll downloads successful."
# f"\nSerializing search products to {allserialfile}"
# f"\nExiting."
# )
# elif nb_allretry == 5 :
# still2download = False
# print(
# f"\nMax number of retries has been completed."
# f"\nSerializing all search products to {allserialfile}"
# f"\nSerializing remain search products to {remainserialfile}"
# f"\nExiting."
# )
# else:
# print(f"\nPass n#{nb_allretry} has reached {len(products_downloaded)} over {nbProducts2d} expected")
# print(f"\nWaiting {passwait[nb_allretry-1]} minutes before retrying...")
# restartd = datetime.now()+timedelta(minutes=passwait[nb_allretry-1])
# restartstr = restartd.strftime("%Y/%m/%d-%H:%M:%S")
# print(f"\nRestart at {restartstr}")
# time.sleep(passwait[nb_allretry-1]*60)
#
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment