diff --git a/apps/__init__.py b/apps/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e63a98c126e5a231a657b295a7c84db87de8c790 --- /dev/null +++ b/apps/__init__.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- +""" +This module contains some applications to work with scenes +""" diff --git a/apps/drs_spot67_import.py b/apps/drs_spot67_import.py index 88c3617b3c9abad3d0c03436c9c077fcaeeb565e..8d91005f2266d989cc4724a53d400672bfc11b59 100755 --- a/apps/drs_spot67_import.py +++ b/apps/drs_spot67_import.py @@ -4,7 +4,9 @@ This application enables to search existing Spot-6/7 products from different loc containing all the `scene.spot.Spot67Scene` instances ``` command-line -drs_spot67_import --root_dirs /ortho_drs/2018/ /ortho_drs/2019 --out_pickle collection +drs_spot67_import + --root_dirs /ortho_drs/2018/ /ortho_drs/2019 + --out_pickle collection ``` """ diff --git a/apps/s2_download.py b/apps/s2_download.py index b75c48e0e4bba6f662180f32ed1483bef34beb11..d766b7374fa055f83b6a90eb8ac0a1359dc75818 100755 --- a/apps/s2_download.py +++ b/apps/s2_download.py @@ -5,9 +5,24 @@ This application enables to download all Sentinel-2 images overlapping a referen The Theia credential have to be provided. ``` command-line -s2_download --refimage raster.tif --theia_cfg ~/cfg.txt --download_dir /tmp --year 2022 --month 12 --day 1 +s2_download + --refimage raster.tif + --theia_cfg ~/cfg.txt + --download_dir /tmp + --year 2022 --month 12 --day 1 ``` +With `~/cfg.txt`: + +``` +serveur = https://theia.cnes.fr/atdistrib +resto = resto2 +token_type = text +login_theia = remi.cresson@irstea.fr +password_theia = thisisnotmyrealpassword +``` + + """ import sys import argparse diff --git a/apps/s2_import.py b/apps/s2_import.py index fbe975e66da4c299a849c4950bf7a06a751379b4..92ae46beda0b385791f6ab328e7dd1482a8645ae 100755 --- a/apps/s2_import.py +++ b/apps/s2_import.py @@ -1,6 +1,13 @@ #!/usr/bin/env python3 """ This application enables to save a collection of Sentinel-2 image into a pickle object. + +``` +s2_import + --root_dirs /path/to/S2_3A/T31TEJ /path/to/S2_3A/T31TEK + --out_pickle s2_collection +``` + """ import sys import argparse diff --git a/apps/search.py b/apps/search.py index fa5ca8fd397f3292909f21ee9a0d4b6fa86a626b..9c3336be123308636ff5793e9377c5598de9ae23 100755 --- a/apps/search.py +++ b/apps/search.py @@ -1,6 +1,22 @@ #!/usr/bin/env python3 """ This application enables to perform a spatial query over a collection of image, from the specific ROI (vector or raster) + +With a vector: +``` +search.py + --pickle_file s2_collection + --roi toto.gpkg +``` + +With a raster: + +``` +search.py + --pickle_file s2_collection + --roi toto.tif +``` + """ import sys import argparse diff --git a/doc/gen_ref_pages.py b/doc/gen_ref_pages.py index 0b61a47ce34b45ab6a692188095c60d111dce4eb..38d9a25f44cac715e78313caf4423591f9db217a 100755 --- a/doc/gen_ref_pages.py +++ b/doc/gen_ref_pages.py @@ -24,9 +24,7 @@ for path in paths: parts = tuple(module_path.parts) print(f"parts: {parts}") - if parts[-1] == "__init__": - if len(parts) <= 1: - continue + if parts[-1] == "__init__": parts = parts[:-1] doc_path = doc_path.with_name("index.md") full_doc_path = full_doc_path.with_name("index.md") diff --git a/doc/arch.md b/doc/index.md similarity index 66% rename from doc/arch.md rename to doc/index.md index 1a109eb27d3eca5dfd19a5190cae44e224b4f42b..5d486829045064781255c6144f4ef5d882d26f15 100644 --- a/doc/arch.md +++ b/doc/index.md @@ -1,4 +1,4 @@ -# Features +# Scenes `Scenes` is a small python library aiming to provide a unified access to common remote sensing products. Currently, supported sensors are: @@ -8,70 +8,72 @@ Currently, supported sensors are: - Sentinel-2 (Theia, Level 2) - Sentinel-2 (Theia, Level 3) -# Spot-6 and Spot-7 +## Generic features -## Instantiation +`Scene` instances have generic methods and attributes. + +- **Metadata**: metadata access with the `get_metadata()` method. +``` py +for key, value in sc.get_metadata(): + print(f"md[{key}]: {value}") +``` +- **Acquisition date**: the `acquisition_date` member returns the acquisition date in the form of a `datetime.datetime` instance. +- **EPSG code**: the `epsg` member is an int for the EPSG of the scene. +- **Bounding box**: the `bbox_wgs84` member is a bounding box in WGS84 (EPSG 4326) coordinate reference system. + +## Spot-6 and Spot-7 + +### Instantiation Spot 6/7 scenes are instantiated from XS and PAN XML files of DIMAPS products. + ``` py from scenes.spot import Spot67Scene sc = Spot67Scene(dimap_file_xs="/path/to/DIM_SPOT7_MS_..._1.XML", dimap_file_pan="/path/to/DIM_SPOT7_P_..._1.XML") ``` -## Metadata +### Image sources -Like any `Scene` objects, the `Spot67Scene` has a `get_metadata()` method to access the dictionary containing the metadata. -``` py -for key, value in sc.get_metadata(): - print(f"md[{key}]: {value}") -``` +Spot-6/7 scenes have 3 images sources. -## Imagery +| Source | `Scene` method | Description | +|--------|----------------|--------------------------------------------------------------------| +| XS | `get_xs()` | 4 channels (Red, Green, Blue, Near Infrared), ~6m physical spacing | +| Pan | `get_pan()` | 1 channel (visible domain), ~1.5m physical spacing | +| PXS | `get_pxs(method=...)` | 4 channels (same as XS), ~1.5m physical spacing | -The `Imagery` is the root for the `Sources`. -In the case of Spot-6/7, the `Spot67Imagery` enables to select the radiometry of the different sources (DN, TOA or TOC). +Each source can be computed in DN/TOA/TOC reflectance. ``` py -dn = sc.get_imagery() -toa = sc.get_imagery("toa") -toc = sc.get_imagery("toc") +xs_raw = sc.get_xs() # DN +xs_raw = sc.get_xs(reflectance="dn") # DN +xs_toa = sc.get_xs(reflectance="toa") # TOA +xs_toc = sc.get_xs(reflectance="toc") # TOC ``` -## Sources - -Spot-6/7 image have 3 sources: - -- XS -- PAN -- PXS - Each source can be masked with the cloud masks of the original product. The no-data value can be chosen. ``` py -for img in [dn, toa, toc]: - for src in [img.get_xs(), img.get_pan(), img.get(pxs)]: - masked_src = src.cld_msk_drilled() # (1) +xs_toa_cld = xs_toa.cld_msk_drilled() # (1) ``` 1. To change the no-data inside the clouds mask: `masked_src = src.cld_msk_drilled(nodata=-999)` -## Examples +### Examples Computing NDVI from XS image in TOA reflectance: ``` py -toa = sc.get_imagery(reflectance="toa") # (1) -xs = toa.get_xs() # (2) +xs = toa.get_xs(reflectance="toa") # (1) exp = "(im1b4-im1b1)/(im1b4+im1b1)" -ndvi = pyotb.bandmath(exp=exp, il=[xs]) # (3) +ndvi = pyotb.bandmath(exp=exp, il=[xs]) # (2) ndvi.write("ndvi.tif") ``` -1. `toa` is a `scenes.spot.Spot67Imagery` instance -2. `xs` is a `scenes.spot.Spot67Source` instance -3. `ndvi` is a `pyotb.app` that inputs `xs` +1. `xs` is a `scenes.spot.Spot67Source` instance +2. `ndvi` is a `pyotb.app` that inputs `xs` The next example is a set of preprocessing operations on a Spot-6/7 XS image: @@ -81,18 +83,16 @@ The next example is a set of preprocessing operations on a Spot-6/7 XS image: 4. clip the result over a reference raster ``` py -toa = scene.get_imagery(reflectance="toa") # (1) -pxs = toa.get_pxs() # (2) -drilled = pxs.cld_msk_drilled() # (3) +pxs = sc.get_pxs(reflectance="toa") # (1) +drilled = pxs.cld_msk_drilled() # (2) ref_img = "/tmp/S2A_2020...._FRE_10m.tif" -subset = drilled.clip_over_img(ref_img) # (4) +subset = drilled.clip_over_img(ref_img) # (3) subset.write("subset.tif") ``` -1. `toa` is a `scenes.spot.Spot67Imagery` instance -2. `pxs` is a `scenes.spot.Spot67Imagery` instance -3. `drilled` is a `scenes.spot.Spot67Imagery` instance -4. `subset` is a `scenes.spot.Spot67Imagery` instance +1. `pxs` is a `scenes.spot.Spot67Source` instance +2. `drilled` is a `scenes.spot.Spot67Source` instance +3. `subset` is a `scenes.spot.Spot67Source` instance Note that we could have changed the no-data value in the cloud masking: @@ -104,16 +104,16 @@ Superimpose an image over a reference image. In the example below, `ref_img` is another `scenes.core.Source` instance. ``` py -toa = scene.get_imagery(reflectance="toa") +toa = sc.get_pxs(reflectance="toa") superimposed = toa.resample_over(ref_img) superimposed.write("superimposed.tif") ``` -# Sentinel-2 +## Sentinel-2 Currently, Level-2 and Level-3 products from the [Theia land data center](https://www.theia-land.fr/en/product/sentinel-2-surface-reflectance/) are supported. -## Instantiation +### Instantiation Sentinel-2 scenes are instantiated from the product archive or folder. @@ -123,19 +123,21 @@ sc_level2 = Sentinel22AScene("/path/to/SENTINEL2A_..._V1-8/.zip") sc_level3 = Sentinel23AScene("/path/to/SENTINEL2X_...L3A_T31TEJ_D/.zip") ``` -## Imagery and sources +### Sources -Sentinel-2 images have a single imagery instance, containing 2 sources: +Sentinel-2 images include 2 sources. -- 10m bands -- 20m bands +| Source | `Scene` method | Description | +|-----------|-------------------|-----------------------------------------------------------| +| 10m bands | `get_10m_bands()` | 10m physical spacing spectral bands (4, 3, 2, 8) | +| 20m bands | `get_20m_bands()` | 20m physical spacing spectral bands (5, 6, 7, 8a, 11, 12) | Each source can be drilled with no-data values. For Level 2 products, using the cloud mask: ``` py -src = sc_level2.get_imagery().get_10m_bands() +src = sc_level2.get_10m_bands() masked_src = src.cld_msk_drilled() # (1) ``` @@ -144,14 +146,18 @@ masked_src = src.cld_msk_drilled() # (1) For Level 3 products, using the quality mask: ``` py -src = sc_level3.get_imagery().get_10m_bands() +src = sc_level3.get_10m_bands() masked_src = src.flg_msk_drilled() # (1) ``` 1. To change the no-data inside the clouds mask: `masked_src = src.flg_msk_drilled(nodata=-999)`. Also, the `keep_flags_values` can be used to select the pixels to keep from a list of values in the quality mask (land, water, snow, ...). -# Spatial and temporal indexation +## Spatial and temporal indexation + +Scenes includes a module to perform spatial and temporal indexation of `Scene` instances. + +### Query in space Perform a query in space (WGS84 bounding box) and time (optional) with an indexation structure. ``` py @@ -167,6 +173,8 @@ for sc in matches: 2. `idx` is a `scenes.indexation.Index` instance, namely a spatio-temporal index 3. `matches` is a list of `scenes.core.Scene` instances +### Query in space and time + To perform searches in time: ``` py matches1 = idx.find(bbox_wgs84=bbox, "01/02/2019", "01-12-2020") # (1) @@ -179,11 +187,11 @@ matches3 = idx.find(bbox_wgs84=bbox, date_max="01/02/2019") # (3) 3. You can also specify only an upper bound, without lower bound (use the keywords `date_min` and `date_max` -# Classes +## Architecture -## Scene class +### Scene class -The scene class handles all the metadata and the imagery. +The scene class handles all the metadata and the image sources. ``` mermaid @@ -194,7 +202,7 @@ classDiagram Sentinel2SceneBase <|-- Sentinel22AScene Sentinel2SceneBase <|-- Sentinel23AScene - Scene --* Imagery: root_scene + Scene --* Source: root_scene class Scene{ +datetime acquisition_date @@ -211,7 +219,6 @@ classDiagram +float incidence_angle +float sun_azimuth_angle +float sun_elev_angle - +Spot67Imagery get_imagery() +get_metadata() +Spot67Source get_xs() +Spot67Source get_pan() @@ -235,7 +242,6 @@ classDiagram +str clm_r2_msk_file +str edg_r1_msk_file +str edg_r2_msk_file - +Sentinel2AImagery get_imagery() +get_metadata() } @@ -248,7 +254,7 @@ classDiagram ``` -## Source class +### Source class The source stores the image pipeline that delivers the pixels. @@ -261,8 +267,8 @@ classDiagram Sentinel2Source <|-- Sentinel23ASource class Source{ - +__init__(root_imagery, out, parent=None) - +Imagery root_imagery + +__init__(root_scene, out, parent=None) + +Scene root_scene +Source parent +Source drilled(msk_vec_file, nodata=0) +Source cld_msk_drilled(nodata=0) @@ -292,10 +298,18 @@ classDiagram ``` -## Implement a new sensor from scratch +### Implement a new sensor from scratch You can implement quickly a new sensor in `scenes`. +- One or multiple `MySensorSource1`, ..., `MySensorSourceM` classes, inheriting from `scenes.core.Source`, and implementing the image access for the new sensor. - A new `MySensorScene` class, inheriting from `scenes.core.Scene`. This class must provide one or multiple methods to its sources. -- One or multiple `MySensorSource1`, ..., `MySensorSourceM` classes, inheriting from `scenes.core.Source` +``` mermaid +classDiagram + + Source <|-- NewSensorSource + Scene <|-- NewSensorScene + NewSensorScene --* NewSensorSource: root_scene + +``` diff --git a/mkdocs.yml b/mkdocs.yml index c124d5f1445f93ad1fa19b26d3a78e0ee80cf986..8d63f459eb41023d1af26e8a58f7863d6852850e 100755 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -5,7 +5,8 @@ theme: repo: material/github features: - content.code.annotate - + - navigation.tabs + - toc.follow plugins: - search @@ -22,7 +23,7 @@ plugins: nav: - API: reference/ -- Overview: arch.md +- Home: index.md # Customization extra: @@ -33,6 +34,9 @@ extra: link: https://gitlab.irstea.fr/umr-tetis/scenes markdown_extensions: + - toc: + permalink: true + title: On this page - pymdownx.highlight: anchor_linenums: true - pymdownx.inlinehilite @@ -42,9 +46,9 @@ markdown_extensions: custom_fences: - name: mermaid class: mermaid - format: !!python/name:mermaid2.fence_mermaid + format: !!python/name:mermaid2.fence_mermaid + # rest of the navigation.. -::: site_name: Scenes docs_dir: doc/ diff --git a/scenes/core.py b/scenes/core.py index 500d19fd0fa7a59d5e1c7f740b1b524c3909bd38..b62771de8a832ea4a01fb8858efc7398a2085f41 100644 --- a/scenes/core.py +++ b/scenes/core.py @@ -205,7 +205,6 @@ class Scene(ABC): Scene class. The class carries all the metadata from the scene, and can be used to retrieve its imagery. - The get_imagery() function is abstract and must be implemented in child classes. """ @@ -248,7 +247,7 @@ class Scene(ABC): """ return {k: str(v) for k, v in self.get_metadata().items()} - def __repr__(self): + def __repr__(self) -> str: """ Enable one instance to be used with print() diff --git a/scenes/deepnets.py b/scenes/deepnets.py index e353693608791e78e8126f38f14e40cafce8ec3e..eabfc8a359bd14370f209c88359e6b6ea3323d96 100644 --- a/scenes/deepnets.py +++ b/scenes/deepnets.py @@ -15,7 +15,7 @@ For example, here is how we perform the super-resolution of a Theia S2 product. import scenes archive = "SENTINEL2B_..._T31TEJ_D_V1-8.zip" s2_scene = scenes.sentinel.Sentinel22AScene(archive) -s2_image = s2_scene.get_imagery().get_10m_bands() +s2_image = s2_scene.get_10m_bands() sr = scenes.deepnets.sr4rs(s2_image) # pyotb.core.otbObject sr.write("sr.tif") ``` diff --git a/scenes/download.py b/scenes/download.py index 346724568da0a6fb9755e7e9019cf360f6954a85..89f74e4d0ffb36547c9d6af605990421ff599b25 100644 --- a/scenes/download.py +++ b/scenes/download.py @@ -180,6 +180,7 @@ class TheiaDownloader: Args: config_file: Theia configuration file The file contents should look like this: + ``` serveur = https://theia.cnes.fr/atdistrib resto = resto2 @@ -187,6 +188,7 @@ class TheiaDownloader: login_theia = remi.cresson@irstea.fr password_theia = thisisnotmyrealpassword ``` + max_records: Maximum number of records """