Skip to content

GitLab

Commit 000ffac0 authored by Lozac'h Loic's avatar Lozac'h Loic
Browse files

debug

parent e3491bd2
Hide whitespace changes
Inline Side-by-side
frozenapp/eodag/user_conf.yml
View file @ 000ffac0
......@@ -16,7 +16,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
scihub:
priority: 2
priority: 1
api:
credentials:
username: loic.lozach
......@@ -25,7 +25,8 @@ scihub:
peps:
priority: 1
api:
credentials:
username: loic.lozach@irstea.fr
password: PMUQc22$
auth:
credentials:
username: loic.lozach@irstea.fr
password: PMUQc22$
frozenapp/eodags1.py
View file @ 000ffac0
......@@ -35,17 +35,17 @@ class EodagS1Downloader():
def __init__(self, mrgstile, startdate, enddate, downdir):
def print_ndict(data):
if type(data) == dict:
for k, v in data.items():
print("key: "+str(k))
print_ndict(v)
else:
try:
print("value: "+str(data))
except Exception as err:
print(err)
print(type(data))
# def print_ndict(data):
# if type(data) == dict:
# for k, v in data.items():
# print("key: "+str(k))
# print_ndict(v)
# else:
# try:
# print("value: "+str(data))
# except Exception as err:
# print(err)
# print(type(data))
'''
Constructor
'''
......@@ -139,16 +139,30 @@ class EodagS1Downloader():
f"\nPEPS has got {nbpeps_online} ONLINE products over {estimated_peps_of_results}"
)
nb_dblav=0
for p in sorted(products_availability, key=lambda k: len(products_availability[k]), reverse=True):
if len(products_availability[p]) == 1 :
print(
f"\nProduct {p} is only available on {products_availability[p][0].provider}"
)
else:
nb_dblav+=1
# products_priority1_peps=[]
# products_priority2_peps=[]
# products_priority1_scihub=[]
# products_priority2_scihub=[]
# for p in sorted(products_availability, key=lambda k: len(products_availability[k]), reverse=True):
#
# if len(products_availability[p]) == 1 :
# print(
# f"\nProduct {p} is only available on {products_availability[p][0].provider}"
# )
# if products_availability[p][0].provider == "peps":
# products_priority1_peps.append(products_availability[p][0])
# elif products_availability[p][0].provider == "scihub":
# products_priority1_scihub.append(products_availability[p][0])
# else:
# if products_availability[p][0].provider == "peps":
# products_priority1_peps.append(products_availability[p][0])
# elif products_availability[p][0].provider == "scihub":
# products_priority1_scihub.append(products_availability[p][0])
#
# if products_availability[p][1].provider == "peps":
# products_priority2_peps.append(products_availability[p][1])
# elif products_availability[p][1].provider == "scihub":
# products_priority2_scihub.append(products_availability[p][1])
products_downloaded={}
passwait=[10,30,60,120,300]
......@@ -159,74 +173,76 @@ class EodagS1Downloader():
allserialfile = os.path.join(downdir,nowstr+"_alldownloads.json")
remainserialfile = os.path.join(downdir,nowstr+"_remaindownloads.json")
#DEBUG
for p in sorted(products_availability, key=lambda k: len(products_availability[k]), reverse=True):
prod0 = products_availability[p][0]
prod0dict = prod0.as_dict()
print_ndict(prod0dict)
prod1 = products_availability[p][1]
prod1dict = prod1.as_dict()
print_ndict(prod1dict)
break
# for p in sorted(products_availability, key=lambda k: len(products_availability[k]), reverse=True):
# prod0 = products_availability[p][0]
# prod0dict = prod0.as_dict()
# print_ndict(prod0dict)
# prod1 = products_availability[p][1]
# prod1dict = prod1.as_dict()
# print_ndict(prod1dict)
#
# break
# while still2download:
# for p in sorted(products_availability, key=lambda k: len(products_availability[k]), reverse=True):
# firstprovider = products_availability[p][0].provider
# print(f"\nStarting Download {p} from {firstprovider}...")
# failed=False
# try:
# with closing(StringIO()) as tqdm_out:
# with ProgressCallback(file=tqdm_out) as bar:
# products_availability[p][0].download( bar, extract=False, outputs_prefix=downdir)
# except Exception as err:
# failed=True
# print(err)
#
# if failed and len(products_availability[p]) == 2:
# secndprovider = products_availability[p][1].provider
# failed=False
# print("Download failed. Trying with {secndprovider}")
# try:
# with closing(StringIO()) as tqdm_out:
# with ProgressCallback(file=tqdm_out) as bar:
# products_availability[p][1].download( bar, extract=False, outputs_prefix=downdir)
# except Exception as err:
# failed=True
# print(err)
#
#
# if products_availability[p][0].location[:5] == "file:":
# products_downloaded[p] = products_availability[p][0]
#
# for p in products_downloaded.keys():
# if p in products_availability.keys():
# products_availability.pop(p)
# nb_allretry += 1
#
# if len(products_downloaded) == nbProducts2d :
# still2download = False
# print(
# f"\nAll downloads successful."
# f"\nSerializing search products to {allserialfile}"
# f"\nExiting."
# )
# elif nb_allretry == 5 :
# still2download = False
# print(
# f"\nMax number of retries has been completed."
# f"\nSerializing all search products to {allserialfile}"
# f"\nSerializing remain search products to {remainserialfile}"
# f"\nExiting."
# )
# else:
# print(f"\nPass n#{nb_allretry} has reached {len(products_downloaded)} over {nbProducts2d} expected")
# print(f"\nWaiting {passwait[nb_allretry-1]} minutes before retrying...")
# restartd = datetime.now()+timedelta(minutes=passwait[nb_allretry-1])
# restartstr = restartd.strftime("%Y/%m/%d-%H:%M:%S")
# print(f"\nRestart at {restartstr}")
# time.sleep(passwait[nb_allretry-1]*60)
#
while still2download:
for p in sorted(products_availability, key=lambda k: len(products_availability[k]), reverse=True):
firstprovider = products_availability[p][0].provider
print(f"\nStarting Download {p} from {firstprovider}...")
self.mEODataAccessGateway.set_preferred_provider(firstprovider)
failed=False
try:
with closing(StringIO()) as tqdm_out:
with ProgressCallback(file=tqdm_out) as bar:
self.mEODataAccessGateway.download(products_availability[p][0], bar, extract=False, outputs_prefix=downdir)
except Exception as err:
failed=True
print(err)
if failed and len(products_availability[p]) == 2:
secndprovider = products_availability[p][1].provider
failed=False
print("Download failed. Trying with {secndprovider}")
self.mEODataAccessGateway.set_preferred_provider(secndprovider)
try:
with closing(StringIO()) as tqdm_out:
with ProgressCallback(file=tqdm_out) as bar:
self.mEODataAccessGateway.download(products_availability[p][1], bar, extract=False, outputs_prefix=downdir)
except Exception as err:
failed=True
print(err)
if products_availability[p][0].location[:5] == "file:":
products_downloaded[p] = products_availability[p][0]
for p in products_downloaded.keys():
if p in products_availability.keys():
products_availability.pop(p)
nb_allretry += 1
if len(products_downloaded) == nbProducts2d :
still2download = False
print(
f"\nAll downloads successful."
f"\nSerializing search products to {allserialfile}"
f"\nExiting."
)
elif nb_allretry == 5 :
still2download = False
print(
f"\nMax number of retries has been completed."
f"\nSerializing all search products to {allserialfile}"
f"\nSerializing remain search products to {remainserialfile}"
f"\nExiting."
)
else:
print(f"\nPass n#{nb_allretry} has reached {len(products_downloaded)} over {nbProducts2d} expected")
print(f"\nWaiting {passwait[nb_allretry-1]} minutes before retrying...")
restartd = datetime.now()+timedelta(minutes=passwait[nb_allretry-1])
restartstr = restartd.strftime("%Y/%m/%d-%H:%M:%S")
print(f"\nRestart at {restartstr}")
time.sleep(passwait[nb_allretry-1]*60)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment