Skip to content

GitLab

Commit 6250a895 authored by Lozac'h Loic's avatar Lozac'h Loic
Browse files

debug

parent 2d58c904
Hide whitespace changes
Inline Side-by-side
frozenapp/eodags1.py
View file @ 6250a895
......@@ -3,7 +3,8 @@ Created on 18 juin 2021
@author: loic
'''
import os, time, logging
import os, time
import logging as py_logging
from eodag.utils.logging import setup_logging
from eodag.api.core import EODataAccessGateway
from eodag.api.search_result import SearchResult
......@@ -21,11 +22,12 @@ from datetime import datetime, timedelta
# os.environ["EODAG__SCIHUB__AUTH__CREDENTIALS__PASSWORD"] = "PMUQc22$"
CONFIG_SCIHUB_ENDPOINT="https://apihub.copernicus.eu/apihub/"
logger = py_logging.getLogger("frozenapp.eodags1")
class EodagS1():
'''
classdocs
'''
logger=None
mEODataAccessGateway = None
user_conf_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),"eodag/user_conf.yml")
locations_conf_path = None
......@@ -39,19 +41,18 @@ class EodagS1():
'''
self.mEODataAccessGateway = EODataAccessGateway(user_conf_file_path=self.user_conf_file_path)
setup_logging(verbose=3,no_progress_bar=True)
self.logger = logging.getLogger(__name__)
self.logger.info("EODataAccessGateway initialized.")
logger.info("EODataAccessGateway initialized.")
def search_and_download(self, prodtypenum, mrgstile, startdate, enddate, downdir):
qs_tile = TileEnvelope.objects.annotate(wgs84=Transform('geom',4326)).filter(tile__exact=mrgstile
)
if len(qs_tile) != 1 :
self.logger.info("Can't find tile "+mrgstile)
logger.info("Can't find tile "+mrgstile)
exit()
# geomwgs84 = Transform(qs_tile[0].geom, 4326)
self.logger.info(qs_tile[0].wgs84)
logger.info(qs_tile[0].wgs84)
mpolywkt= str(qs_tile[0].wgs84.ewkt).split(";")
print(mpolywkt[1])
......@@ -60,7 +61,7 @@ class EodagS1():
products_availability={}
if prodtypenum in [0,1]:
self.logger.info("Search on Scihub...")
logger.info("Search on Scihub...")
self.mEODataAccessGateway.set_preferred_provider("scihub")
......@@ -71,10 +72,10 @@ class EodagS1():
geom=mpolywkt[1]
)
self.logger.info(
logger.info(
f"The Scihub search has found an estimated number of {estimated_scihub_of_results} products matching your criteria "
)
self.logger.info("Search on Peps...")
logger.info("Search on Peps...")
self.mEODataAccessGateway.set_preferred_provider("peps")
products_peps, estimated_peps_of_results = self.mEODataAccessGateway.search(
......@@ -84,7 +85,7 @@ class EodagS1():
geom=mpolywkt[1]
)
self.logger.info(
logger.info(
f"The PEPS search has found an estimated number of {estimated_peps_of_results} products matching your criteria "
)
......@@ -123,13 +124,13 @@ class EodagS1():
if pp.properties["storageStatus"] == "ONLINE":
nbpeps_online += 1
self.logger.info(
logger.info(
f"\nScihub has got {nbsci_online} ONLINE products over {estimated_scihub_of_results}"
f"\nPEPS has got {nbpeps_online} ONLINE products over {estimated_peps_of_results}"
)
elif prodtypenum == 2:
self.logger.info("Search on Theia...")
logger.info("Search on Theia...")
self.mEODataAccessGateway.set_preferred_provider("theia")
......@@ -139,7 +140,7 @@ class EodagS1():
end=enddate,
geom=mpolywkt[1]
)
self.logger.info(
logger.info(
f"The THEIA search has found an estimated number of {estimated_theia_of_results} products matching your criteria "
)
nbtheia_online=0
......@@ -148,12 +149,12 @@ class EodagS1():
if pt.properties["storageStatus"] == "ONLINE":
nbtheia_online += 1
self.logger.info(
logger.info(
f"\nTHIEA has got {nbtheia_online} ONLINE products over {estimated_theia_of_results}"
)
else:
self.logger.error("error")
logger.error("error")
return
self.products_availability = products_availability
......@@ -177,12 +178,12 @@ class EodagS1():
if len(self.products_downloaded[d]) == 2 :
altdownloadedprods.append(self.products_downloaded[d][1])
self.logger.info(f"\nSerializing downloaded products to {serialfile}")
logger.info(f"\nSerializing downloaded products to {serialfile}")
dps = SearchResult(downloadedprods)
self.mEODataAccessGateway.serialize(dps, serialfile)
if len(altdownloadedprods) > 0 :
self.logger.info(f"\nSerializing alternate products to {altserialfile}")
logger.info(f"\nSerializing alternate products to {altserialfile}")
adps = SearchResult(altdownloadedprods)
self.mEODataAccessGateway.serialize(adps, altserialfile)
......@@ -195,12 +196,12 @@ class EodagS1():
if len(self.products_availability[d]) == 2 :
altremainingprods.append(self.products_availability[d][1])
self.logger.info(f"\nSerializing downloaded products to {remainserialfile}")
logger.info(f"\nSerializing downloaded products to {remainserialfile}")
rps = SearchResult(remainingprods)
self.mEODataAccessGateway.serialize(rps, remainserialfile)
if len(altremainingprods) > 0 :
self.logger.info(f"\nSerializing alternate products to {altremainserialfile}")
logger.info(f"\nSerializing alternate products to {altremainserialfile}")
arps = SearchResult(altremainingprods)
self.mEODataAccessGateway.serialize(arps, altremainserialfile)
......@@ -224,8 +225,8 @@ class EodagS1():
else:
self.products_availability[pp.properties["id"]] = [pp]
self.logger.info("Geojson files successfully imported")
self.logger.info("Use start_downloads() to launch downloads")
logger.info("Geojson files successfully imported")
logger.info("Use start_downloads() to launch downloads")
def start_downloads(self,downdir):
......@@ -242,7 +243,7 @@ class EodagS1():
while still2download:
for p in sorted(self.products_availability, key=lambda k: len(self.products_availability[k]), reverse=True):
firstprovider = self.products_availability[p][0].provider
self.logger.info(f"\nStarting Download {p} from {firstprovider}...")
logger.info(f"\nStarting Download {p} from {firstprovider}...")
self.mEODataAccessGateway.set_preferred_provider(firstprovider)
failed=False
try:
......@@ -259,7 +260,7 @@ class EodagS1():
if failed and len(self.products_availability[p]) == 2:
secndprovider = self.products_availability[p][1].provider
failed=False
self.logger.info(f"\nDownload failed. Trying with {secndprovider}")
logger.info(f"\nDownload failed. Trying with {secndprovider}")
self.mEODataAccessGateway.set_preferred_provider(secndprovider)
try:
with closing(StringIO()) as tqdm_out:
......@@ -283,7 +284,7 @@ class EodagS1():
if len(self.products_downloaded) == nbProducts2d :
still2download = False
self.logger.info(
logger.info(
f"\nAll downloads successful."
)
self.__export_geojson(downdir, 0)
......@@ -292,7 +293,7 @@ class EodagS1():
elif nb_allretry == 5 :
still2download = False
self.logger.info(
logger.info(
"""
Max number of retries has been completed.
There is still products to download.
......@@ -301,13 +302,13 @@ class EodagS1():
)
self.__export_geojson(downdir, 1)
self.logger.info("Done.")
logger.info("Done.")
else:
self.logger.info(f"\nPass n#{nb_allretry} has reached {len(self.products_downloaded)} over {nbProducts2d} expected")
self.logger.info(f"\nWaiting {passwait[nb_allretry-1]} minutes before retrying...")
logger.info(f"\nPass n#{nb_allretry} has reached {len(self.products_downloaded)} over {nbProducts2d} expected")
logger.info(f"\nWaiting {passwait[nb_allretry-1]} minutes before retrying...")
restartd = datetime.now()+timedelta(minutes=passwait[nb_allretry-1])
restartstr = restartd.strftime("%Y/%m/%d-%H:%M:%S")
self.logger.info(f"\nRestart at {restartstr}")
logger.info(f"\nRestart at {restartstr}")
time.sleep(passwait[nb_allretry-1]*60)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment