eodags1.py

'''
Created on 18 juin 2021

@author: loic
'''
import os, time, logging
from eodag.utils.logging import setup_logging
from eodag.api.core import EODataAccessGateway
from eodag.api.search_result import SearchResult
from frozenapp.models import TileEnvelope
from django.contrib.gis.db.models.functions import Transform
from eodag.utils import ProgressCallback
from contextlib import closing
from io import StringIO
from datetime import datetime, timedelta


# os.environ["EODAG__PEPS__AUTH__CREDENTIALS__USERNAME"] = "loic.lozach"
# os.environ["EODAG__PEPS__AUTH__CREDENTIALS__PASSWORD"] = "PMUQc22$"
# os.environ["EODAG__SCIHUB__AUTH__CREDENTIALS__USERNAME"] = "loic.lozach"
# os.environ["EODAG__SCIHUB__AUTH__CREDENTIALS__PASSWORD"] = "PMUQc22$"
CONFIG_SCIHUB_ENDPOINT="https://apihub.copernicus.eu/apihub/"

class EodagS1():
    '''
    classdocs
    '''
    logger=None
    mEODataAccessGateway = None
    user_conf_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),"eodag/user_conf.yml")
    locations_conf_path = None
#     providers_conf_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),"eodag/providers.yml")
    products_downloaded = {}
    products_availability = {}
    
    def __init__(self):
        '''
        Constructor
        '''
        self.mEODataAccessGateway = EODataAccessGateway(user_conf_file_path=self.user_conf_file_path)
        setup_logging(verbose=0,no_progress_bar=True)
        self.logger = logging.getLogger(__name__)
        self.logger.info("EODataAccessGateway initialized.")

    def search_and_download(self, prodtypenum, mrgstile, startdate, enddate, downdir):

        qs_tile =  TileEnvelope.objects.annotate(wgs84=Transform('geom',4326)).filter(tile__exact=mrgstile
                                                  )
        if len(qs_tile) != 1 :
            self.logger.info("Can't find tile "+mrgstile)
            exit()
            
#         geomwgs84 = Transform(qs_tile[0].geom, 4326)
        self.logger.info(qs_tile[0].wgs84)
        
        mpolywkt= str(qs_tile[0].wgs84.ewkt).split(";")
        print(mpolywkt[1])
        
        product_types = ['S1_SAR_GRD','S2_MSI_L2A','S2_MSI_L3A_WASP']
        
        products_availability={}
        if prodtypenum in [0,1]:
            self.logger.info("Search on Scihub...")
            self.mEODataAccessGateway.set_preferred_provider("scihub")
            
            
            products_scihub, estimated_scihub_of_results = self.mEODataAccessGateway.search(
                    productType=product_types[prodtypenum],
                    start=startdate,
                    end=enddate,
                    geom=mpolywkt[1]
                )
            
            self.logger.info(
                    f"The Scihub search has found an estimated number of {estimated_scihub_of_results} products matching your criteria "
                )
            self.logger.info("Search on Peps...")
            self.mEODataAccessGateway.set_preferred_provider("peps")
            
            products_peps, estimated_peps_of_results = self.mEODataAccessGateway.search(
                    productType=product_types[prodtypenum],
                    start=startdate,
                    end=enddate,
                    geom=mpolywkt[1]
                )
            
            self.logger.info(
                    f"The PEPS search has found an estimated number of {estimated_peps_of_results} products matching your criteria "
                )
            
            nbsci_online=0
            nbpeps_online=0
            
            for ps in products_scihub :
                found=False
                for pp in products_peps :
                    if pp.properties["id"] == ps.properties["id"]:
                        if ps.properties["storageStatus"] == "ONLINE" and pp.properties["storageStatus"] == "ONLINE":
                            products_availability[ps.properties["id"]] = [ps,pp]
                            nbsci_online += 1
                            nbpeps_online += 1
                        elif ps.properties["storageStatus"] == "ONLINE" and pp.properties["storageStatus"] == "OFFLINE":
                            products_availability[ps.properties["id"]] = [ps,pp]
                            nbsci_online += 1
                        elif ps.properties["storageStatus"] == "OFFLINE" and pp.properties["storageStatus"] == "ONLINE":
                            products_availability[ps.properties["id"]] = [pp,ps]
                            nbpeps_online += 1
                        elif ps.properties["storageStatus"] == "OFFLINE" and pp.properties["storageStatus"] == "OFFLINE":
                            products_availability[ps.properties["id"]] = [ps,pp]
                            
                        found=True
                        break
                if found == False:
                    products_availability[ps.properties["id"]] = [ps]
                    if ps.properties["storageStatus"] == "ONLINE":
                        nbsci_online += 1
            
            for pp in products_peps :
                found=False
                
                if not pp.properties["id"] in products_availability.keys():
                    products_availability[pp.properties["id"]] = [pp]
                    if pp.properties["storageStatus"] == "ONLINE":
                        nbpeps_online += 1
            
            self.logger.info(
                    f"\nScihub has got {nbsci_online} ONLINE products over {estimated_scihub_of_results}"
                    f"\nPEPS has got {nbpeps_online} ONLINE products over {estimated_peps_of_results}"
                )
            
        elif prodtypenum == 2:
            self.logger.info("Search on Theia...")
            self.mEODataAccessGateway.set_preferred_provider("theia")
            
            
            products_theia, estimated_theia_of_results = self.mEODataAccessGateway.search(
                    productType=product_types[prodtypenum],
                    start=startdate,
                    end=enddate,
                    geom=mpolywkt[1]
                )
            self.logger.info(
                    f"The THEIA search has found an estimated number of {estimated_theia_of_results} products matching your criteria "
                )
            nbtheia_online=0
            for pt in products_theia :
                products_availability[pt.properties["id"]] = [pt]
                if pt.properties["storageStatus"] == "ONLINE":
                        nbtheia_online += 1
            
            self.logger.info(
                    f"\nTHIEA has got {nbtheia_online} ONLINE products over {estimated_theia_of_results}"
                )
            
        else:
            self.logger.error("error")
            return
        
        self.products_availability = products_availability
        
        self.start_downloads(downdir)
        

    def __export_geojson(self,downdir, remain):
        nowstr = datetime.now().strftime("%Y%m%dT%H%M%S")
        serialfile = os.path.join(downdir,nowstr+"_DownloadedProducts.geojson")
        altserialfile = os.path.join(downdir,nowstr+"_AlternateProducts.geojson")
        remainserialfile = os.path.join(downdir,nowstr+"_RemainingProducts.geojson")
        altremainserialfile = os.path.join(downdir,nowstr+"_AlternateRemainingProducts.geojson")
        
        if len(self.products_downloaded) > 0 :

            downloadedprods = []
            altdownloadedprods = []
            for d in self.products_downloaded.keys() :
                downloadedprods.append(self.products_downloaded[d][0])
                if len(self.products_downloaded[d]) == 2 :
                    altdownloadedprods.append(self.products_downloaded[d][1])
            
            self.logger.info(f"\nSerializing downloaded products to {serialfile}")
            dps = SearchResult(downloadedprods)
            self.mEODataAccessGateway.serialize(dps, serialfile)
            
            if len(altdownloadedprods) > 0 :
                self.logger.info(f"\nSerializing alternate products to {altserialfile}")
                adps = SearchResult(altdownloadedprods)
                self.mEODataAccessGateway.serialize(adps, altserialfile)
        
        if remain == 1 :
            
            remainingprods = []
            altremainingprods = []
            for d in self.products_availability.keys() :
                remainingprods.append(self.products_availability[d][0])
                if len(self.products_availability[d]) == 2 :
                    altremainingprods.append(self.products_availability[d][1])
            
            self.logger.info(f"\nSerializing downloaded products to {remainserialfile}")
            rps = SearchResult(remainingprods)
            self.mEODataAccessGateway.serialize(rps, remainserialfile)
            
            if len(altremainingprods) > 0 :
                self.logger.info(f"\nSerializing alternate products to {altremainserialfile}")
                arps = SearchResult(altremainingprods)
                self.mEODataAccessGateway.serialize(arps, altremainserialfile)

            
        
        

    def import_geojson(self,products_geojson,alternate_products_geojson=None):
        
        self.products_availability = {}
        deserialized_products = self.mEODataAccessGateway.deserialize(products_geojson)
        
        for pt in deserialized_products :
            self.products_availability[pt.properties["id"]] = [pt]
        
        if not alternate_products_geojson == None :
            deserialized_alt_products = self.mEODataAccessGateway.deserialize(alternate_products_geojson)
            for pp in deserialized_alt_products :
                if pp.properties["id"] in self.products_availability.keys():
                    self.products_availability[pp.properties["id"]].append(pp)
                else:
                    self.products_availability[pp.properties["id"]] = [pp]
                    
        self.logger.info("Geojson files successfully imported")
        self.logger.info("Use start_downloads() to launch downloads")
    
    
    def start_downloads(self,downdir):
        
        if not os.path.exists(downdir):
            os.mkdir(downdir)
        
        self.products_downloaded={}
        passwait=[10,30,60,120,300]
        still2download = True
        nbProducts2d = len(self.products_availability)
        nb_allretry = 0
        
        while still2download:
            for p in sorted(self.products_availability, key=lambda k: len(self.products_availability[k]), reverse=True):
                firstprovider = self.products_availability[p][0].provider
                self.logger.info(f"\nStarting Download {p} from {firstprovider}...")
                self.mEODataAccessGateway.set_preferred_provider(firstprovider)
                failed=False
                try:
                    with closing(StringIO()) as tqdm_out:
                        with ProgressCallback(file=tqdm_out) as bar:
                            self.mEODataAccessGateway.download(self.products_availability[p][0], bar, extract=False, outputs_prefix=downdir, timeout=1)
                except Exception as err:
                    failed=True
                    print(err)
                    
                if not failed and self.products_availability[p][0].location[-3:] != "zip":
                    failed=True
             
                if failed and len(self.products_availability[p]) == 2:
                    secndprovider = self.products_availability[p][1].provider
                    failed=False
                    self.logger.info(f"\nDownload failed. Trying with {secndprovider}")
                    self.mEODataAccessGateway.set_preferred_provider(secndprovider)
                    try:
                        with closing(StringIO()) as tqdm_out:
                            with ProgressCallback(file=tqdm_out) as bar:
                                self.mEODataAccessGateway.download(self.products_availability[p][1], bar, extract=False, outputs_prefix=downdir, timeout=1)
                    except Exception as err:
                        failed=True
                        print(err)
                        
                    self.products_availability[p].pop(0)
   
         
                if self.products_availability[p][0].location[:5] == "file:":
                    self.products_downloaded[p] = self.products_availability[p]
         
            for p in self.products_downloaded.keys():
                if p in self.products_availability.keys():
                    self.products_availability.pop(p)
                    
            nb_allretry += 1
             
            if len(self.products_downloaded) == nbProducts2d :
                still2download = False
                self.logger.info(
                    f"\nAll downloads successful."
                    )
                self.__export_geojson(downdir, 0)
                
                print("Done.")
                
            elif nb_allretry == 5 :
                still2download = False
                self.logger.info(
                    """
                    Max number of retries has been completed.
                    There is still products to download.
                    Use import_geoson() with _RemainingProducts.geojson file to restart downloads.
                    """
                    )
                self.__export_geojson(downdir, 1)
                
                self.logger.info("Done.")
            else:
                self.logger.info(f"\nPass n#{nb_allretry} has reached {len(self.products_downloaded)} over {nbProducts2d} expected")
                self.logger.info(f"\nWaiting {passwait[nb_allretry-1]} minutes before retrying...")
                restartd = datetime.now()+timedelta(minutes=passwait[nb_allretry-1])
                restartstr = restartd.strftime("%Y/%m/%d-%H:%M:%S")
                self.logger.info(f"\nRestart at {restartstr}")
                time.sleep(passwait[nb_allretry-1]*60)