eodags1.py 9.51 KB
Newer Older
Lozac'h Loic's avatar
Lozac'h Loic committed
1
2
3
4
5
6
7
8
9
'''
Created on 18 juin 2021

@author: loic
'''
import os
from eodag.utils.logging import setup_logging
from eodag.api.core import EODataAccessGateway
from frozenapp.models import TileEnvelope
Lozac'h Loic's avatar
Lozac'h Loic committed
10
11
12
13
from django.contrib.gis.db.models.functions import Transform
from eodag.utils import ProgressCallback
from contextlib import closing
from io import StringIO
Lozac'h Loic's avatar
eodag1    
Lozac'h Loic committed
14
15
from datetime import datetime, timedelta
import time
Lozac'h Loic's avatar
Lozac'h Loic committed
16

Lozac'h Loic's avatar
Lozac'h Loic committed
17
18
19
20
21
# os.environ["EODAG__PEPS__AUTH__CREDENTIALS__USERNAME"] = "loic.lozach"
# os.environ["EODAG__PEPS__AUTH__CREDENTIALS__PASSWORD"] = "PMUQc22$"
# os.environ["EODAG__SCIHUB__AUTH__CREDENTIALS__USERNAME"] = "loic.lozach"
# os.environ["EODAG__SCIHUB__AUTH__CREDENTIALS__PASSWORD"] = "PMUQc22$"
CONFIG_SCIHUB_ENDPOINT="https://apihub.copernicus.eu/apihub/"
Lozac'h Loic's avatar
Lozac'h Loic committed
22
23
24
25
26
27
28
29
30

class EodagS1Downloader():
    '''
    classdocs
    '''
    
    mEODataAccessGateway = None
    user_conf_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),"eodag/user_conf.yml")
    locations_conf_path = None
Lozac'h Loic's avatar
Lozac'h Loic committed
31
#     providers_conf_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),"eodag/providers.yml")
Lozac'h Loic's avatar
eodag1    
Lozac'h Loic committed
32
33
34
    productOnline = None
    productOffline = None
    
Lozac'h Loic's avatar
debug    
Lozac'h Loic committed
35
    
Lozac'h Loic's avatar
Lozac'h Loic committed
36

Lozac'h Loic's avatar
eodag1    
Lozac'h Loic committed
37
    def __init__(self, mrgstile, startdate, enddate, downdir):
Lozac'h Loic's avatar
debug    
Lozac'h Loic committed
38
39
        def print_ndict(data):
            if type(data) == dict:
Lozac'h Loic's avatar
debug    
Lozac'h Loic committed
40
41
42
                for k, v in data.items():
                        print("key: "+str(k))
                        print_ndict(v)
Lozac'h Loic's avatar
debug    
Lozac'h Loic committed
43
            else:
Lozac'h Loic's avatar
debug    
Lozac'h Loic committed
44
                try:
Lozac'h Loic's avatar
debug    
Lozac'h Loic committed
45
                    print("value: "+str(data))
Lozac'h Loic's avatar
debug    
Lozac'h Loic committed
46
47
48
                except Exception as err:
                    print(err)
                    print(type(data))
Lozac'h Loic's avatar
Lozac'h Loic committed
49
50
51
        '''
        Constructor
        '''
Lozac'h Loic's avatar
eodag1    
Lozac'h Loic committed
52
53
54
        if not os.path.exists(downdir):
            os.mkdir(downdir)
            
Lozac'h Loic's avatar
Lozac'h Loic committed
55
        setup_logging(verbose=3)
Lozac'h Loic's avatar
Lozac'h Loic committed
56
        
Lozac'h Loic's avatar
Lozac'h Loic committed
57
        qs_tile =  TileEnvelope.objects.annotate(wgs84=Transform('geom',4326)).filter(tile__exact=mrgstile
Lozac'h Loic's avatar
Lozac'h Loic committed
58
59
60
61
                                                  )
        if len(qs_tile) != 1 :
            print("Can't find tile "+mrgstile)
            exit()
Lozac'h Loic's avatar
Lozac'h Loic committed
62
63
64
            
#         geomwgs84 = Transform(qs_tile[0].geom, 4326)
        print(qs_tile[0].wgs84)
Lozac'h Loic's avatar
Lozac'h Loic committed
65
        
Lozac'h Loic's avatar
Lozac'h Loic committed
66
        mpolywkt= str(qs_tile[0].wgs84.ewkt).split(";")
Lozac'h Loic's avatar
Lozac'h Loic committed
67
68
69
        print(mpolywkt[1])
        
        self.mEODataAccessGateway = EODataAccessGateway(self.user_conf_file_path)
Lozac'h Loic's avatar
eodag1    
Lozac'h Loic committed
70
        print("Search on Scihub...")
Lozac'h Loic's avatar
debug    
Lozac'h Loic committed
71
        self.mEODataAccessGateway.set_preferred_provider("scihub")
Lozac'h Loic's avatar
Lozac'h Loic committed
72
73
        
        product_type = 'S1_SAR_GRD'
Lozac'h Loic's avatar
Lozac'h Loic committed
74
        sensor_mode = 'IW'
Lozac'h Loic's avatar
Lozac'h Loic committed
75
        
Lozac'h Loic's avatar
eodag1    
Lozac'h Loic committed
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
        products_scihub, estimated_scihub_of_results = self.mEODataAccessGateway.search(
                productType=product_type,
                sensoroperationalmode=sensor_mode,
                start=startdate,
                end=enddate,
                geom=mpolywkt[1]
            )
        
        print(
                f"The Scihub search has found an estimated number of {estimated_scihub_of_results} products matching your criteria "
            )
        print("Search on Peps...")
        self.mEODataAccessGateway.set_preferred_provider("peps")
        
        products_peps, estimated_peps_of_results = self.mEODataAccessGateway.search(
Lozac'h Loic's avatar
Lozac'h Loic committed
91
                productType=product_type,
Lozac'h Loic's avatar
Lozac'h Loic committed
92
                sensoroperationalmode=sensor_mode,
Lozac'h Loic's avatar
Lozac'h Loic committed
93
94
95
96
97
98
                start=startdate,
                end=enddate,
                geom=mpolywkt[1]
            )
        
        print(
Lozac'h Loic's avatar
eodag1    
Lozac'h Loic committed
99
100
101
102
                f"The PEPS search has found an estimated number of {estimated_peps_of_results} products matching your criteria "
            )
        
        nbsci_online=0
Lozac'h Loic's avatar
debug    
Lozac'h Loic committed
103
        nbpeps_online=0
Lozac'h Loic's avatar
eodag1    
Lozac'h Loic committed
104
105
106
107
108
        products_availability={}
        for ps in products_scihub :
            found=False
            for pp in products_peps :
                if pp.properties["id"] == ps.properties["id"]:
Lozac'h Loic's avatar
debug    
Lozac'h Loic committed
109
110
111
112
113
114
115
116
117
118
119
120
121
                    if ps.properties["storageStatus"] == "ONLINE" and pp.properties["storageStatus"] == "ONLINE":
                        products_availability[ps.properties["id"]] = [ps,pp]
                        nbsci_online += 1
                        nbpeps_online += 1
                    elif ps.properties["storageStatus"] == "ONLINE" and pp.properties["storageStatus"] == "OFFLINE":
                        products_availability[ps.properties["id"]] = [ps,pp]
                        nbsci_online += 1
                    elif ps.properties["storageStatus"] == "OFFLINE" and pp.properties["storageStatus"] == "ONLINE":
                        products_availability[ps.properties["id"]] = [pp,ps]
                        nbpeps_online += 1
                    elif ps.properties["storageStatus"] == "OFFLINE" and pp.properties["storageStatus"] == "OFFLINE":
                        products_availability[ps.properties["id"]] = [ps,pp]
                        
Lozac'h Loic's avatar
eodag1    
Lozac'h Loic committed
122
123
124
125
                    found=True
                    break
            if found == False:
                products_availability[ps.properties["id"]] = [ps]
Lozac'h Loic's avatar
debug    
Lozac'h Loic committed
126
127
                if ps.properties["storageStatus"] == "ONLINE":
                    nbsci_online += 1
Lozac'h Loic's avatar
eodag1    
Lozac'h Loic committed
128
129
130
131
132
133
        
        for pp in products_peps :
            found=False
            
            if not pp.properties["id"] in products_availability.keys():
                products_availability[pp.properties["id"]] = [pp]
Lozac'h Loic's avatar
debug    
Lozac'h Loic committed
134
135
                if pp.properties["storageStatus"] == "ONLINE":
                    nbpeps_online += 1
Lozac'h Loic's avatar
eodag1    
Lozac'h Loic committed
136
137
        
        print(
Lozac'h Loic's avatar
debug    
Lozac'h Loic committed
138
139
                f"\nScihub has got {nbsci_online} ONLINE products over {estimated_scihub_of_results}"
                f"\nPEPS has got {nbpeps_online} ONLINE products over {estimated_peps_of_results}"
Lozac'h Loic's avatar
Lozac'h Loic committed
140
            )
Lozac'h Loic's avatar
eodag1    
Lozac'h Loic committed
141
142
        
        nb_dblav=0
Lozac'h Loic's avatar
debug    
Lozac'h Loic committed
143
        for p in sorted(products_availability, key=lambda k: len(products_availability[k]), reverse=True):
Lozac'h Loic's avatar
eodag1    
Lozac'h Loic committed
144
145
146
            
            if len(products_availability[p]) == 1 :
                print(
Lozac'h Loic's avatar
debug    
Lozac'h Loic committed
147
                    f"\nProduct {p} is only available on {products_availability[p][0].provider}"
Lozac'h Loic's avatar
eodag1    
Lozac'h Loic committed
148
149
150
151
152
153
154
155
156
157
158
159
160
                )
            else:
                nb_dblav+=1
        

        products_downloaded={}
        passwait=[10,30,60,120,300]
        still2download = True
        nbProducts2d = len(products_availability)
        nb_allretry = 0
        nowstr = datetime.now().strftime("%Y%m%dT%H%M%S")
        allserialfile = os.path.join(downdir,nowstr+"_alldownloads.json")
        remainserialfile = os.path.join(downdir,nowstr+"_remaindownloads.json")
Lozac'h Loic's avatar
debug    
Lozac'h Loic committed
161
162
163
164
        #DEBUG
        for p in sorted(products_availability, key=lambda k: len(products_availability[k]), reverse=True):
            prod0 = products_availability[p][0]
            prod0dict = prod0.as_dict()
Lozac'h Loic's avatar
debug    
Lozac'h Loic committed
165
            print_ndict(prod0dict)
Lozac'h Loic's avatar
debug    
Lozac'h Loic committed
166
167
            prod1 = products_availability[p][1]
            prod1dict = prod1.as_dict()
Lozac'h Loic's avatar
debug    
Lozac'h Loic committed
168
            print_ndict(prod1dict)
Lozac'h Loic's avatar
Lozac'h Loic committed
169
            
Lozac'h Loic's avatar
debug    
Lozac'h Loic committed
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
            break
        
        
#         while still2download:
#             for p in sorted(products_availability, key=lambda k: len(products_availability[k]), reverse=True):
#                 firstprovider = products_availability[p][0].provider
#                 print(f"\nStarting Download {p} from {firstprovider}...")
#                 failed=False
#                 try:
#                     with closing(StringIO()) as tqdm_out:
#                         with ProgressCallback(file=tqdm_out) as bar:
#                             products_availability[p][0].download( bar, extract=False, outputs_prefix=downdir)
#                 except Exception as err:
#                     failed=True
#                     print(err)
#             
#                 if failed and len(products_availability[p]) == 2:
#                     secndprovider = products_availability[p][1].provider
#                     failed=False
#                     print("Download failed. Trying with {secndprovider}")
#                     try:
#                         with closing(StringIO()) as tqdm_out:
#                             with ProgressCallback(file=tqdm_out) as bar:
#                                 products_availability[p][1].download( bar, extract=False, outputs_prefix=downdir)
#                     except Exception as err:
#                         failed=True
#                         print(err)
#   
#         
#                 if products_availability[p][0].location[:5] == "file:":
#                     products_downloaded[p] = products_availability[p][0]
#         
#             for p in products_downloaded.keys():
#                 if p in products_availability.keys():
#                     products_availability.pop(p)
#             nb_allretry += 1
#             
#             if len(products_downloaded) == nbProducts2d :
#                 still2download = False
#                 print(
#                     f"\nAll downloads successful."
#                     f"\nSerializing search products to {allserialfile}"
#                     f"\nExiting."
#                     )
#             elif nb_allretry == 5 :
#                 still2download = False
#                 print(
#                     f"\nMax number of retries has been completed."
#                     f"\nSerializing all search products to {allserialfile}"
#                     f"\nSerializing remain search products to {remainserialfile}"
#                     f"\nExiting."
#                     )
#             else:
#                 print(f"\nPass n#{nb_allretry} has reached {len(products_downloaded)} over {nbProducts2d} expected")
#                 print(f"\nWaiting {passwait[nb_allretry-1]} minutes before retrying...")
#                 restartd = datetime.now()+timedelta(minutes=passwait[nb_allretry-1])
#                 restartstr = restartd.strftime("%Y/%m/%d-%H:%M:%S")
#                 print(f"\nRestart at {restartstr}")
#                 time.sleep(passwait[nb_allretry-1]*60)
#             
Lozac'h Loic's avatar
eodag1    
Lozac'h Loic committed
230
231
232