WIP: S2 download

scenes/download.py

@@ -86,7 +86,7 @@ class TheiaDownloader:
             print("Empty token. Please check your credentials in config file.")
         return token
 
-    def search(self, dict_query):
+    def query(self, dict_query):
         """
         Search products
         Return a dict with the following structure
@@ -99,7 +99,7 @@ class TheiaDownloader:
 
         """
         # 1. Get the JSON
-        url = "{}/{}/api/collections/SENTINEL2/search.json?{}".format(self.config["serveur"],
+        url = "{}/{}/api/collections/SENTINEL2/query.json?{}".format(self.config["serveur"],
                                                                       self.config["resto"], urlencode(dict_query))
         print("Ask Theia catalog...")
         search = json.loads(curl_url(url, None))
@@ -109,11 +109,12 @@ class TheiaDownloader:
         features = search["features"]
         tile_dict = dict()
         for record in features:
+            print(record)
             rid = record["id"]
             rdate = datetime.datetime.strptime(record["properties"]["completionDate"][0:10], "%Y-%m-%d")
             rloc = record["properties"]["location"]
-            rchksum = record["properties"]["services"]["download"]["checksum"]
-            rurl = record["properties"]["services"]["download"]["url"]
+            rchksum = record["properties"]["services"]["download_closest"]["checksum"]
+            rurl = record["properties"]["services"]["download_closest"]["url"]
             if rloc not in tile_dict.keys():
                 tile_dict[rloc] = dict()
             if rdate not in tile_dict[rloc].keys():
@@ -134,7 +135,6 @@ class TheiaDownloader:
                 hash_md5.update(chunk)
         return hash_md5.hexdigest()
 
-
     def _file_complete(self, filename, md5sum):
         """
         Tell if a file is complete
@@ -146,7 +146,6 @@ class TheiaDownloader:
         # Does the file completed?
         return md5sum == self._md5(filename)
 
-
     def download(self, tiles_dict, outfn_func):
         """
         Download a product
@@ -170,9 +169,9 @@ class TheiaDownloader:
                 else:
                     print("\t{} already in cache. Skipping.".format(_rdate))
 
-def download(config_file, acq_envelope, acq_date):
+def download_closest(config_file, acq_envelope, acq_date, level="LEVEL3A"):
     """
-    search theia catalog, download the files
+    query theia catalog, download_closest the files
     """
 
     # Important parameters
@@ -190,45 +189,45 @@ def download(config_file, acq_envelope, acq_date):
     dict_query['startDate'] = start_date.strftime("%Y-%m-%d")
     dict_query['completionDate'] = end_date.strftime("%Y-%m-%d")
     dict_query['maxRecords'] = 500
-    dict_query['processingLevel'] = "LEVEL3A"
+    dict_query['processingLevel'] = level
 
     # Theia downloader
     downloader = TheiaDownloader(config_file)
 
     # Search products
-    tile = downloader.search(dict_query)
+    search_results = downloader.query(dict_query)
 
     # DELTAS RANKING
     # Add the "Delta" key/value
-    for _rloc in tile:
-        print(_rloc)
-        for _rdate in tile[_rloc]:
+    for tile_name in search_results:
+        print(tile_name)
+        for _rdate in search_results[tile_name]:
             delta = acq_date - _rdate
             delta = delta.days
-            tile[_rloc][_rdate]["delta"] = delta
+            search_results[tile_name][_rdate]["delta"] = delta
 
     # Rank delta
     selected_tile = dict()
-    for _rloc in tile:
+    for tile_name in search_results:
         n_dates = 0
-        x = tile[_rloc]
+        x = search_results[tile_name]
         sorted_x = sorted(x.items(), key=lambda kv: abs(kv[1]["delta"]))
-        selected_tile[_rloc] = dict()
+        selected_tile[tile_name] = dict()
         for i in sorted_x:
             _rdate = i[0]
             entry = i[1]
-            selected_tile[_rloc][_rdate] = entry
+            selected_tile[tile_name][_rdate] = entry
             n_dates += 1
             if n_dates == 1:
                 break
 
     # Print summary
     print("Best tiles/dates:")
-    for _rloc in selected_tile:
-        print("Tile {}".format(_rloc))
+    for tile_name in selected_tile:
+        print("Tile {}".format(tile_name))
         print("\tDate (delta)")
-        for _rdate in selected_tile[_rloc]:
-            print("\t{} ({})".format(_rdate, selected_tile[_rloc][_rdate]["delta"]))
+        for _rdate in selected_tile[tile_name]:
+            print("\t{} ({})".format(_rdate, selected_tile[tile_name][_rdate]["delta"]))
 
     # Download products
     #downloader.download(selected_tile, get_local_file)

test/download_test.py

@@ -10,4 +10,5 @@ params = parser.parse_args()
 
 # Get all scenes in the root_dir
 _, _, bbox = utils.get_epsg_extent_bbox(params.refimage)
-download.download(config_file=params.theia_cfg, acq_envelope=bbox, acq_date=datetime.datetime(year=2020, month=1, day=1))
+acq_date = datetime.datetime(year=2020, month=1, day=1)
\ No newline at end of file
+download.download_closest(config_file=params.theia_cfg, acq_envelope=bbox, acq_date=acq_date)
\ No newline at end of file

test/drs_search.py

@@ -16,8 +16,8 @@ scenes = drs.get_spot67_scenes(params.root_dir)
 print("Indexation...")
 idx = indexation.Index(scenes)
 
-# search
-print("search roi")
+# query
+print("query roi")
 if params.roi.lower().endswith(".tif"):
     gdal_ds = gdal.Open(params.roi)
     bbox = utils.get_bbox_wgs84_from_gdal_ds(gdal_ds=gdal_ds)

test/drs_stack.py

@@ -17,8 +17,8 @@ scenes = drs.get_spot67_scenes(params.root_dir)
 print("Indexation...")
 idx = indexation.Index(scenes)
 
-# search
-print("search roi")
+# query
+print("query roi")
 gdal_ds = gdal.Open(params.roi)
 bbox = utils.get_extent_wgs84(gdal_ds)
 matches = idx.find(bbox_wgs84=bbox)