Commit e1b2fd1c authored by Lozac'h Loic's avatar Lozac'h Loic
Browse files

test eodag

parent 1731cc67
......@@ -144,7 +144,7 @@ RUN mkdir -p /work/otb/build \
RUN cd /work/otb/otb/Modules/Remote \
&& GIT_SSL_NO_VERIFY=true git clone https://gitlab.irstea.fr/loic.lozach/agrifrozenareas.git \
&& git clone https://gitlab.irstea.fr/loic.lozach/agrifrozenareas.git \
&& cd /work/otb/build/OTB/build \
&& cmake /work/otb/otb \
-DOTB_WRAP_PYTHON=ON \
......@@ -165,6 +165,7 @@ WORKDIR /work/python
RUN pip3 install --upgrade pip
RUN pip3 install -r requirements.txt
COPY eodag_sentinelsat_dev.py.patch /usr/local/lib/python3.6/dist-packages/eodag_sentinelsat/eodag_sentinelsat.py
# ----------------------------------------------------------------------------
# Clean
# ----------------------------------------------------------------------------
......@@ -197,7 +198,6 @@ ENV TERM=xterm-256color
ENV GDAL_DATA=/usr/share/gdal/2.2/
ENV GDAL_DRIVER_PATH="disable"
ENV LC_NUMERIC=C
ENV PYTHONUNBUFFERED=1
#RUN unset LD_LIBRARY_PATH
RUN alias ll='ls -al'
RUN sudo ln -s /usr/bin/python3 /usr/bin/python
......
......@@ -16,8 +16,8 @@ services:
image: kartoza/postgis:12.0
ports:
- 25432:5432
volumes:
- /work/database/freezedb:/var/lib/postgresql
# volumes:
# - /work/database/freezedb:/var/lib/postgresql
environment:
- POSTGRES_DBNAME=postgres
- POSTGRES_USER=postgres
......
......@@ -15,10 +15,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
---
---
!provider
provider:
name: scihub
priority: 2
api: !plugin
......@@ -199,172 +198,3 @@
#'title': 'S1A_IW_SLC__1SDV_20170425T155612_20170425T155639_016302_01AF91_46FF',
#'uuid': '04548172-c64a-418f-8e83-7a4d148adf1e'
---
!provider
name: peps
priority: 1
description: |
The PEPS platform, the French "mirror site", redistributes the products of Sentinel satellites,
S1A, S1B, S2A and S2B, S3A and S3B from COPERNICUS, the European system for the Earth monitoring.
roles:
- host
url: https://peps.cnes.fr
search: !plugin
type: QueryStringSearch
api_endpoint: 'https://peps.cnes.fr/resto/api/collections/{collection}/search.json'
pagination:
next_page_url_tpl: '{url}?{search}&maxRecords={items_per_page}&page={page}'
total_items_nb_key_path: '$.properties.totalResults'
# 2021/03/19: 500 is the max, no error if greater
max_items_per_page: 500
discover_metadata:
auto_discovery: true
metadata_pattern: '^(?!collection)[a-zA-Z0-9_]+$'
search_param: '{metadata}={{{metadata}}}'
metadata_path: '$.properties.*'
metadata_mapping:
# Opensearch resource identifier within the search engine context (in our case
# within the context of the data provider)
uid: '$.id'
# OpenSearch Parameters for Collection Search (Table 3)
productType:
- productType
- '$.properties.productType'
platform: '$.properties.collection'
platformSerialIdentifier:
- platform
- '$.properties.platform'
instrument:
- instrument
- '$.properties.instrument'
processingLevel:
- processingLevel
- '$.properties.processingLevel'
# INSPIRE obligated OpenSearch Parameters for Collection Search (Table 4)
title: '$.properties.title'
topicCategory: '$.properties.topicCategory'
keyword: '$.properties.keywords'
abstract: '$.properties.description'
resolution:
- 'resolution'
- '$.properties.resolution'
organisationName:
- 'organisationName'
- '$.properties.organisationName'
publicationDate: '$.properties.published'
accessConstraint: '$.properties.license'
# OpenSearch Parameters for Product Search (Table 5)
parentIdentifier:
- 'parentIdentifier'
- '$.properties.parentIdentifier'
acquisitionType: '$.properties.acquisitionType'
orbitNumber:
- 'orbitNumber'
- '$.properties.orbitNumber'
orbitDirection:
- 'orbitDirection'
- '$.properties.orbitDirection'
swathIdentifier:
- 'swath'
- '$.properties.swath'
cloudCover:
- 'cloudCover=[0,{cloudCover}]'
- '$.properties.cloudCover'
snowCover:
- 'snowCover=[0,{snowCover}]'
- '$.properties.snowCover'
productVersion: '$.properties.version'
productQualityStatus: '$.properties.onlineQualityCheck'
processorName: '$.properties.processingName'
processingCenter: '$.properties.processingCenter'
creationDate: '$.properties.dhusIngestDate'
modificationDate: '$.properties.updated'
sensorMode:
- 'sensorMode'
- '$.properties.sensorMode'
# OpenSearch Parameters for Acquistion Parameters Search (Table 6)
startTimeFromAscendingNode:
- startDate
- '$.properties.startDate'
completionTimeFromAscendingNode:
- completionDate
- '$.properties.completionDate'
polarizationMode:
- 'polarisation'
- '$.properties.polarisation'
# Custom parameters (not defined in the base document referenced above)
id:
- 'productIdentifier={id#remove_extension}'
- '$.properties.productIdentifier'
# The geographic extent of the product
geometry:
- 'geometry={geometry#to_rounded_wkt}'
- '$.geometry'
# The url of the quicklook
quicklook: '$.properties.quicklook'
# The url to download the product "as is" (literal or as a template to be completed either after the search result
# is obtained from the provider or during the eodag download phase)
downloadLink: '$.properties.services.download.url'
# storageStatus: must be one of ONLINE, STAGING, OFFLINE
storageStatus: '{$.properties.storage.mode#get_group_name((?P<ONLINE>disk)|(?P<STAGING>staging)|(?P<OFFLINE>tape))}'
# Additional metadata provided by the providers but that don't appear in the reference spec
thumbnail: '$.properties.thumbnail'
products:
S1_SAR_OCN:
productType: OCN
collection: S1
S1_SAR_GRD:
productType: GRD
collection: S1
S1_SAR_SLC:
productType: SLC
collection: S1
S2_MSI_L1C:
collection: S2ST
productType: S2MSI1C
S2_MSI_L2A:
collection: S2ST
productType: S2MSI2A
S3_EFR:
productType: OL_1_EFR___
collection: S3
S3_ERR:
productType: OL_1_ERR___
collection: S3
S3_OLCI_L2LFR:
productType: OL_2_LFR___
collection: S3
S3_OLCI_L2LRR:
productType: OL_2_LRR___
collection: S3
S3_SLSTR_L1RBT:
productType: SL_1_RBT___
collection: S3
S3_SLSTR_L2LST:
productType: SL_2_LST___
collection: S3
S3_LAN:
productType: SR_2_LAN___
collection: S3
GENERIC_PRODUCT_TYPE:
productType: '{productType}'
collection: '{collection}'
download: !plugin
type: HTTPDownload
base_uri: 'https://peps.cnes.fr/resto'
extract: true
archive_depth: 2
order_enabled: true
auth_error_code: 401
dl_url_params:
issuerId: peps
auth: !plugin
type: GenericAuth
auth_uri: 'https://peps.cnes.fr/resto/api/users/connect'
......@@ -16,21 +16,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
scihub:
priority: 1 # Lower value means lower priority (Default: 1)
download:
extract: false # whether to extract the downloaded products, only applies to archived products (true or false, Default: true).
outputs_prefix: "/data/S1-store"# where to store downloaded products, as an absolute file path (Default: local temporary directory)
auth:
priority: 2
api:
credentials:
username:"loic.lozach"
password:"PMUQc22$"
username: loic.lozach
password: PMUQc22$
endpoint: https://apihub.copernicus.eu/apihub/
peps:
priority: 0 # Lower value means lower priority (Default: 1)
download:
extract: # whether to extract the downloaded products, only applies to archived products (true or false, Default: true).
outputs_prefix: "/data/S1-store" # where to store downloaded products, as an absolute file path (Default: local temporary directory)
auth:
credentials:
username:"loic.lozach"
password:"PMUQc22$"
priority: 1
api:
auth:
credentials:
username: loic.lozach
password: PMUQc22$
......@@ -7,7 +7,16 @@ import os
from eodag.utils.logging import setup_logging
from eodag.api.core import EODataAccessGateway
from frozenapp.models import TileEnvelope
from django.contrib.gis.db.models.functions import Transform
from eodag.utils import ProgressCallback
from contextlib import closing
from io import StringIO
# os.environ["EODAG__PEPS__AUTH__CREDENTIALS__USERNAME"] = "loic.lozach"
# os.environ["EODAG__PEPS__AUTH__CREDENTIALS__PASSWORD"] = "PMUQc22$"
# os.environ["EODAG__SCIHUB__AUTH__CREDENTIALS__USERNAME"] = "loic.lozach"
# os.environ["EODAG__SCIHUB__AUTH__CREDENTIALS__PASSWORD"] = "PMUQc22$"
CONFIG_SCIHUB_ENDPOINT="https://apihub.copernicus.eu/apihub/"
class EodagS1Downloader():
'''
......@@ -17,32 +26,37 @@ class EodagS1Downloader():
mEODataAccessGateway = None
user_conf_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),"eodag/user_conf.yml")
locations_conf_path = None
providers_conf_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),"eodag/providers.yml")
# providers_conf_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),"eodag/providers.yml")
def __init__(self, mrgstile, startdate, enddate):
'''
Constructor
'''
setup_logging(verbose=3)
qs_tile = TileEnvelope.objects.filter(tile__exact=mrgstile
qs_tile = TileEnvelope.objects.annotate(wgs84=Transform('geom',4326)).filter(tile__exact=mrgstile
)
if len(qs_tile) != 1 :
print("Can't find tile "+mrgstile)
exit()
# geomwgs84 = Transform(qs_tile[0].geom, 4326)
print(qs_tile[0].wgs84)
mpolywkt= str(qs_tile[0].geom.ewkt).split(";")
mpolywkt= str(qs_tile[0].wgs84.ewkt).split(";")
print(mpolywkt[1])
self.mEODataAccessGateway = EODataAccessGateway(self.user_conf_file_path)
with open(self.providers_conf_file_path, 'r') as stream:
self.mEODataAccessGateway.update_providers_config(stream)
print("debug")
print(self.mEODataAccessGateway.get_preferred_provider())
product_type = 'S1_SAR_GRD'
sensor_mode = 'IW'
products, estimated_total_nbr_of_results = self.mEODataAccessGateway.search(
productType=product_type,
sensoroperationalmode=sensor_mode,
start=startdate,
end=enddate,
geom=mpolywkt[1]
......@@ -52,4 +66,11 @@ class EodagS1Downloader():
f"The search has found an estimated number of {estimated_total_nbr_of_results} products matching "
f"our criteria. It has returned the first {len(products)} ones available."
)
for prod in products:
print(prod.as_dict()['properties']["identifier"])
print("Starting Download...")
with closing(StringIO()) as tqdm_out:
with ProgressCallback(file=tqdm_out) as bar:
self.mEODataAccessGateway.download_all(products, bar, 30, 2880, extract=False, outputs_prefix="/data/S1/download")
# Generated by Django 2.2.14 on 2020-07-17 17:53
# Generated by Django 3.2.4 on 2021-06-23 17:00
import django.contrib.gis.db.models.fields
from django.db import migrations, models
......@@ -46,19 +46,20 @@ class Migration(migrations.Migration):
('sentinel1_pltf', models.CharField(blank=True, max_length=3, null=True)),
('sentinel1_orbit', models.CharField(blank=True, max_length=3, null=True)),
('processed', models.BooleanField(default=False)),
('tile', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='frozenapp.TileEnvelope')),
('tile', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='frozenapp.tileenvelope')),
],
),
migrations.CreateModel(
name='ParcelEvent',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('meansigma40', models.FloatField()),
('meantemp', models.FloatField()),
('maxrefsigma40', models.FloatField(blank=True, null=True)),
('meansigma40', models.FloatField(blank=True, null=True)),
('meantemp', models.FloatField(blank=True, null=True)),
('frozen_type', models.IntegerField(blank=True, null=True)),
('parcelle_type', models.IntegerField()),
('parcelle', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='frozenapp.ParcelleGraphique')),
('sentinel1', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='frozenapp.Sentinel1Image')),
('parcelle', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='frozenapp.parcellegraphique')),
('sentinel1', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='frozenapp.sentinel1image')),
],
),
migrations.AddConstraint(
......
BEGIN;
--
-- Create model ParcelleGraphique
--
CREATE TABLE "frozenapp_parcellegraphique" ("id" serial NOT NULL PRIMARY KEY, "id_parcel" varchar(10) NOT NULL, "surf_parc" double precision NOT NULL, "code_cultu" varchar(3) NOT NULL, "code_group" varchar(2) NOT NULL, "culture_d1" varchar(3) NULL, "culture_d2" varchar(3) NULL, "mpoly" geometry(MULTIPOLYGON,2154) NOT NULL);
--
-- Create model TileEnvelope
--
CREATE TABLE "frozenapp_tileenvelope" ("id" serial NOT NULL PRIMARY KEY, "fid" bigint NOT NULL, "area" double precision NOT NULL, "perimeter" double precision NOT NULL, "tile" varchar(254) NOT NULL, "geom" geometry(MULTIPOLYGON,2154) NOT NULL);
CREATE TABLE "frozenapp_tileenvelope_parcelles" ("id" serial NOT NULL PRIMARY KEY, "tileenvelope_id" integer NOT NULL, "parcellegraphique_id" integer NOT NULL);
--
-- Create model Sentinel1Image
--
CREATE TABLE "frozenapp_sentinel1image" ("id" serial NOT NULL PRIMARY KEY, "sentinel1_date" timestamp with time zone NOT NULL, "sentinel1_pltf" varchar(3) NULL, "sentinel1_orbit" varchar(3) NULL, "processed" boolean NOT NULL, "tile_id" integer NOT NULL);
--
-- Create model ParcelEvent
--
CREATE TABLE "frozenapp_parcelevent" ("id" serial NOT NULL PRIMARY KEY, "meansigma40" double precision NOT NULL, "meantemp" double precision NOT NULL, "frozen_type" integer NULL, "parcelle_type" integer NOT NULL, "parcelle_id" integer NOT NULL, "sentinel1_id" integer NOT NULL);
--
-- Create constraint datepartile_unique on model sentinel1image
--
ALTER TABLE "frozenapp_sentinel1image" ADD CONSTRAINT "datepartile_unique" UNIQUE ("sentinel1_date", "tile_id");
--
-- Create constraint parcelpardate_unique on model parcelevent
--
ALTER TABLE "frozenapp_parcelevent" ADD CONSTRAINT "parcelpardate_unique" UNIQUE ("parcelle_id", "sentinel1_id");
CREATE INDEX "frozenapp_parcellegraphique_mpoly_id" ON "frozenapp_parcellegraphique"USING GIST ("mpoly");
CREATE INDEX "frozenapp_tileenvelope_geom_id" ON "frozenapp_tileenvelope"USING GIST ("geom");
ALTER TABLE "frozenapp_tileenvelope_parcelles" ADD CONSTRAINT "frozenapp_tileenvelo_tileenvelope_id_30b5ec89_fk_frozenapp" FOREIGN KEY ("tileenvelope_id") REFERENCES "frozenapp_tileenvelope" ("id") DEFERRABLE INITIALLY DEFERRED;
ALTER TABLE "frozenapp_tileenvelope_parcelles" ADD CONSTRAINT "frozenapp_tileenvelo_parcellegraphique_id_1f78866f_fk_frozenapp" FOREIGN KEY ("parcellegraphique_id") REFERENCES "frozenapp_parcellegraphique" ("id") DEFERRABLE INITIALLY DEFERRED;
ALTER TABLE "frozenapp_tileenvelope_parcelles" ADD CONSTRAINT "frozenapp_tileenvelope_p_tileenvelope_id_parcelle_add74ab4_uniq" UNIQUE ("tileenvelope_id", "parcellegraphique_id");
CREATE INDEX "frozenapp_tileenvelope_parcelles_tileenvelope_id_30b5ec89" ON "frozenapp_tileenvelope_parcelles" ("tileenvelope_id");
CREATE INDEX "frozenapp_tileenvelope_parcelles_parcellegraphique_id_1f78866f" ON "frozenapp_tileenvelope_parcelles" ("parcellegraphique_id");
ALTER TABLE "frozenapp_sentinel1image" ADD CONSTRAINT "frozenapp_sentinel1i_tile_id_8db585ab_fk_frozenapp" FOREIGN KEY ("tile_id") REFERENCES "frozenapp_tileenvelope" ("id") DEFERRABLE INITIALLY DEFERRED;
CREATE INDEX "frozenapp_sentinel1image_tile_id_8db585ab" ON "frozenapp_sentinel1image" ("tile_id");
ALTER TABLE "frozenapp_parcelevent" ADD CONSTRAINT "frozenapp_parceleven_parcelle_id_22cc6b01_fk_frozenapp" FOREIGN KEY ("parcelle_id") REFERENCES "frozenapp_parcellegraphique" ("id") DEFERRABLE INITIALLY DEFERRED;
ALTER TABLE "frozenapp_parcelevent" ADD CONSTRAINT "frozenapp_parceleven_sentinel1_id_b4a22989_fk_frozenapp" FOREIGN KEY ("sentinel1_id") REFERENCES "frozenapp_sentinel1image" ("id") DEFERRABLE INITIALLY DEFERRED;
CREATE INDEX "frozenapp_parcelevent_parcelle_id_22cc6b01" ON "frozenapp_parcelevent" ("parcelle_id");
CREATE INDEX "frozenapp_parcelevent_sentinel1_id_b4a22989" ON "frozenapp_parcelevent" ("sentinel1_id");
COMMIT;
......@@ -16,3 +16,5 @@ psycopg2-binary
django-leaflet
django-extensions
eodag
#https://github.com/CS-SI/eodag-sentinelsat/archive/refs/heads/develop.zip
eodag-sentinelsat
\ No newline at end of file
USERUID=1001
USERUID=1000
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment