''' Created on 18 juin 2021 @author: loic ''' import os from eodag.utils.logging import setup_logging from eodag.api.core import EODataAccessGateway from eodag.api.search_result import SearchResult from frozenapp.models import TileEnvelope from django.contrib.gis.db.models.functions import Transform from eodag.utils import ProgressCallback from contextlib import closing from io import StringIO from datetime import datetime, timedelta import time # os.environ["EODAG__PEPS__AUTH__CREDENTIALS__USERNAME"] = "loic.lozach" # os.environ["EODAG__PEPS__AUTH__CREDENTIALS__PASSWORD"] = "PMUQc22$" # os.environ["EODAG__SCIHUB__AUTH__CREDENTIALS__USERNAME"] = "loic.lozach" # os.environ["EODAG__SCIHUB__AUTH__CREDENTIALS__PASSWORD"] = "PMUQc22$" CONFIG_SCIHUB_ENDPOINT="https://apihub.copernicus.eu/apihub/" class EodagS1(): ''' classdocs ''' mEODataAccessGateway = None user_conf_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),"eodag/user_conf.yml") locations_conf_path = None # providers_conf_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),"eodag/providers.yml") products_downloaded = {} products_availability = {} def __init__(self): ''' Constructor ''' self.mEODataAccessGateway = EODataAccessGateway(user_conf_file_path=self.user_conf_file_path) setup_logging(verbose=0) print("") def search_and_download(self, prodtypenum, mrgstile, startdate, enddate, downdir): qs_tile = TileEnvelope.objects.annotate(wgs84=Transform('geom',4326)).filter(tile__exact=mrgstile ) if len(qs_tile) != 1 : print("Can't find tile "+mrgstile) exit() # geomwgs84 = Transform(qs_tile[0].geom, 4326) print(qs_tile[0].wgs84) mpolywkt= str(qs_tile[0].wgs84.ewkt).split(";") print(mpolywkt[1]) product_types = ['S1_SAR_GRD','S2_MSI_L2A','S2_MSI_L3A_WASP'] products_availability={} if prodtypenum in [0,1]: print("Search on Scihub...") self.mEODataAccessGateway.set_preferred_provider("scihub") products_scihub, estimated_scihub_of_results = self.mEODataAccessGateway.search( productType=product_types[prodtypenum], start=startdate, end=enddate, geom=mpolywkt[1] ) print( f"The Scihub search has found an estimated number of {estimated_scihub_of_results} products matching your criteria " ) print("Search on Peps...") self.mEODataAccessGateway.set_preferred_provider("peps") products_peps, estimated_peps_of_results = self.mEODataAccessGateway.search( productType=product_types[prodtypenum], start=startdate, end=enddate, geom=mpolywkt[1] ) print( f"The PEPS search has found an estimated number of {estimated_peps_of_results} products matching your criteria " ) nbsci_online=0 nbpeps_online=0 for ps in products_scihub : found=False for pp in products_peps : if pp.properties["id"] == ps.properties["id"]: if ps.properties["storageStatus"] == "ONLINE" and pp.properties["storageStatus"] == "ONLINE": products_availability[ps.properties["id"]] = [ps,pp] nbsci_online += 1 nbpeps_online += 1 elif ps.properties["storageStatus"] == "ONLINE" and pp.properties["storageStatus"] == "OFFLINE": products_availability[ps.properties["id"]] = [ps,pp] nbsci_online += 1 elif ps.properties["storageStatus"] == "OFFLINE" and pp.properties["storageStatus"] == "ONLINE": products_availability[ps.properties["id"]] = [pp,ps] nbpeps_online += 1 elif ps.properties["storageStatus"] == "OFFLINE" and pp.properties["storageStatus"] == "OFFLINE": products_availability[ps.properties["id"]] = [ps,pp] found=True break if found == False: products_availability[ps.properties["id"]] = [ps] if ps.properties["storageStatus"] == "ONLINE": nbsci_online += 1 for pp in products_peps : found=False if not pp.properties["id"] in products_availability.keys(): products_availability[pp.properties["id"]] = [pp] if pp.properties["storageStatus"] == "ONLINE": nbpeps_online += 1 print( f"\nScihub has got {nbsci_online} ONLINE products over {estimated_scihub_of_results}" f"\nPEPS has got {nbpeps_online} ONLINE products over {estimated_peps_of_results}" ) elif prodtypenum == 2: print("Search on Theia...") self.mEODataAccessGateway.set_preferred_provider("theia") products_theia, estimated_theia_of_results = self.mEODataAccessGateway.search( productType=product_types[prodtypenum], start=startdate, end=enddate, geom=mpolywkt[1] ) print( f"The THEIA search has found an estimated number of {estimated_theia_of_results} products matching your criteria " ) nbtheia_online=0 for pt in products_theia : products_availability[pt.properties["id"]] = [pt] if pt.properties["storageStatus"] == "ONLINE": nbtheia_online += 1 print( f"\nTHIEA has got {nbtheia_online} ONLINE products over {estimated_theia_of_results}" ) else: print("error") return self.products_availability = products_availability self.start_downloads(downdir) def __export_geojson(self,downdir, remain): nowstr = datetime.now().strftime("%Y%m%dT%H%M%S") serialfile = os.path.join(downdir,nowstr+"_DownloadedProducts.geojson") altserialfile = os.path.join(downdir,nowstr+"_AlternateProducts.geojson") remainserialfile = os.path.join(downdir,nowstr+"_RemainingProducts.geojson") altremainserialfile = os.path.join(downdir,nowstr+"_AlternateRemainingProducts.geojson") if len(self.products_downloaded) > 0 : downloadedprods = [] altdownloadedprods = [] for d in self.products_downloaded.keys() : downloadedprods.append(self.products_downloaded[d][0]) if len(self.products_downloaded[d]) == 2 : altdownloadedprods.append(self.products_downloaded[d][1]) print(f"\nSerializing downloaded products to {serialfile}") dps = SearchResult(downloadedprods) self.mEODataAccessGateway.serialize(dps, serialfile) if len(altdownloadedprods) > 0 : print(f"\nSerializing alternate products to {altserialfile}") adps = SearchResult(altdownloadedprods) self.mEODataAccessGateway.serialize(adps, altserialfile) if remain == 1 : remainingprods = [] altremainingprods = [] for d in self.products_availability.keys() : remainingprods.append(self.products_availability[d][0]) if len(self.products_availability[d]) == 2 : altremainingprods.append(self.products_availability[d][1]) print(f"\nSerializing downloaded products to {remainserialfile}") rps = SearchResult(remainingprods) self.mEODataAccessGateway.serialize(rps, remainserialfile) if len(altremainingprods) > 0 : print(f"\nSerializing alternate products to {altremainserialfile}") arps = SearchResult(altremainingprods) self.mEODataAccessGateway.serialize(arps, altremainserialfile) def import_geojson(self,products_geojson,alternate_products_geojson=None): self.products_availability = {} deserialized_products = self.mEODataAccessGateway.deserialize(products_geojson) for pt in deserialized_products : self.products_availability[pt.properties["id"]] = [pt] if not alternate_products_geojson == None : deserialized_alt_products = self.mEODataAccessGateway.deserialize(alternate_products_geojson) for pp in deserialized_alt_products : if pp.properties["id"] in self.products_availability.keys(): self.products_availability[pp.properties["id"]].append(pp) else: self.products_availability[pp.properties["id"]] = [pp] print("Geojson files successfully imported") print("Use start_downloads() to launch downloads") def start_downloads(self,downdir): if not os.path.exists(downdir): os.mkdir(downdir) self.products_downloaded={} passwait=[10,30,60,120,300] still2download = True nbProducts2d = len(self.products_availability) nb_allretry = 0 while still2download: for p in sorted(self.products_availability, key=lambda k: len(self.products_availability[k]), reverse=True): firstprovider = self.products_availability[p][0].provider print(f"\nStarting Download {p} from {firstprovider}...") self.mEODataAccessGateway.set_preferred_provider(firstprovider) failed=False try: with closing(StringIO()) as tqdm_out: with ProgressCallback(file=tqdm_out) as bar: self.mEODataAccessGateway.download(self.products_availability[p][0], bar, extract=False, outputs_prefix=downdir) except Exception as err: failed=True print(err) if not failed and self.products_availability[p][0].location[-3:] != "zip": failed=True if failed and len(self.products_availability[p]) == 2: secndprovider = self.products_availability[p][1].provider failed=False print(f"\nDownload failed. Trying with {secndprovider}") self.mEODataAccessGateway.set_preferred_provider(secndprovider) try: with closing(StringIO()) as tqdm_out: with ProgressCallback(file=tqdm_out) as bar: self.mEODataAccessGateway.download(self.products_availability[p][1], bar, extract=False, outputs_prefix=downdir) except Exception as err: failed=True print(err) self.products_availability[p].pop(0) if self.products_availability[p][0].location[:5] == "file:": self.products_downloaded[p] = self.products_availability[p] for p in self.products_downloaded.keys(): if p in self.products_availability.keys(): self.products_availability.pop(p) nb_allretry += 1 if len(self.products_downloaded) == nbProducts2d : still2download = False print( f"\nAll downloads successful." ) self.__export_geojson(downdir, 0) print("Done.") elif nb_allretry == 5 : still2download = False print( """ Max number of retries has been completed. There is still products to download. Use import_geoson() with _RemainingProducts.geojson file to restart downloads. """ ) self.__export_geojson(downdir, 1) print("Done.") else: print(f"\nPass n#{nb_allretry} has reached {len(self.products_downloaded)} over {nbProducts2d} expected") print(f"\nWaiting {passwait[nb_allretry-1]} minutes before retrying...") restartd = datetime.now()+timedelta(minutes=passwait[nb_allretry-1]) restartstr = restartd.strftime("%Y/%m/%d-%H:%M:%S") print(f"\nRestart at {restartstr}") time.sleep(passwait[nb_allretry-1]*60)