Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
Lozac'h Loic
AgriFrozenAreaProject
Commits
6250a895
Commit
6250a895
authored
Aug 25, 2021
by
Lozac'h Loic
Browse files
debug
parent
2d58c904
Changes
1
Hide whitespace changes
Inline
Side-by-side
frozenapp/eodags1.py
View file @
6250a895
...
...
@@ -3,7 +3,8 @@ Created on 18 juin 2021
@author: loic
'''
import
os
,
time
,
logging
import
os
,
time
import
logging
as
py_logging
from
eodag.utils.logging
import
setup_logging
from
eodag.api.core
import
EODataAccessGateway
from
eodag.api.search_result
import
SearchResult
...
...
@@ -21,11 +22,12 @@ from datetime import datetime, timedelta
# os.environ["EODAG__SCIHUB__AUTH__CREDENTIALS__PASSWORD"] = "PMUQc22$"
CONFIG_SCIHUB_ENDPOINT
=
"https://apihub.copernicus.eu/apihub/"
logger
=
py_logging
.
getLogger
(
"frozenapp.eodags1"
)
class
EodagS1
():
'''
classdocs
'''
logger
=
None
mEODataAccessGateway
=
None
user_conf_file_path
=
os
.
path
.
join
(
os
.
path
.
dirname
(
os
.
path
.
abspath
(
__file__
)),
"eodag/user_conf.yml"
)
locations_conf_path
=
None
...
...
@@ -39,19 +41,18 @@ class EodagS1():
'''
self
.
mEODataAccessGateway
=
EODataAccessGateway
(
user_conf_file_path
=
self
.
user_conf_file_path
)
setup_logging
(
verbose
=
3
,
no_progress_bar
=
True
)
self
.
logger
=
logging
.
getLogger
(
__name__
)
self
.
logger
.
info
(
"EODataAccessGateway initialized."
)
logger
.
info
(
"EODataAccessGateway initialized."
)
def
search_and_download
(
self
,
prodtypenum
,
mrgstile
,
startdate
,
enddate
,
downdir
):
qs_tile
=
TileEnvelope
.
objects
.
annotate
(
wgs84
=
Transform
(
'geom'
,
4326
)).
filter
(
tile__exact
=
mrgstile
)
if
len
(
qs_tile
)
!=
1
:
self
.
logger
.
info
(
"Can't find tile "
+
mrgstile
)
logger
.
info
(
"Can't find tile "
+
mrgstile
)
exit
()
# geomwgs84 = Transform(qs_tile[0].geom, 4326)
self
.
logger
.
info
(
qs_tile
[
0
].
wgs84
)
logger
.
info
(
qs_tile
[
0
].
wgs84
)
mpolywkt
=
str
(
qs_tile
[
0
].
wgs84
.
ewkt
).
split
(
";"
)
print
(
mpolywkt
[
1
])
...
...
@@ -60,7 +61,7 @@ class EodagS1():
products_availability
=
{}
if
prodtypenum
in
[
0
,
1
]:
self
.
logger
.
info
(
"Search on Scihub..."
)
logger
.
info
(
"Search on Scihub..."
)
self
.
mEODataAccessGateway
.
set_preferred_provider
(
"scihub"
)
...
...
@@ -71,10 +72,10 @@ class EodagS1():
geom
=
mpolywkt
[
1
]
)
self
.
logger
.
info
(
logger
.
info
(
f
"The Scihub search has found an estimated number of
{
estimated_scihub_of_results
}
products matching your criteria "
)
self
.
logger
.
info
(
"Search on Peps..."
)
logger
.
info
(
"Search on Peps..."
)
self
.
mEODataAccessGateway
.
set_preferred_provider
(
"peps"
)
products_peps
,
estimated_peps_of_results
=
self
.
mEODataAccessGateway
.
search
(
...
...
@@ -84,7 +85,7 @@ class EodagS1():
geom
=
mpolywkt
[
1
]
)
self
.
logger
.
info
(
logger
.
info
(
f
"The PEPS search has found an estimated number of
{
estimated_peps_of_results
}
products matching your criteria "
)
...
...
@@ -123,13 +124,13 @@ class EodagS1():
if
pp
.
properties
[
"storageStatus"
]
==
"ONLINE"
:
nbpeps_online
+=
1
self
.
logger
.
info
(
logger
.
info
(
f
"
\n
Scihub has got
{
nbsci_online
}
ONLINE products over
{
estimated_scihub_of_results
}
"
f
"
\n
PEPS has got
{
nbpeps_online
}
ONLINE products over
{
estimated_peps_of_results
}
"
)
elif
prodtypenum
==
2
:
self
.
logger
.
info
(
"Search on Theia..."
)
logger
.
info
(
"Search on Theia..."
)
self
.
mEODataAccessGateway
.
set_preferred_provider
(
"theia"
)
...
...
@@ -139,7 +140,7 @@ class EodagS1():
end
=
enddate
,
geom
=
mpolywkt
[
1
]
)
self
.
logger
.
info
(
logger
.
info
(
f
"The THEIA search has found an estimated number of
{
estimated_theia_of_results
}
products matching your criteria "
)
nbtheia_online
=
0
...
...
@@ -148,12 +149,12 @@ class EodagS1():
if
pt
.
properties
[
"storageStatus"
]
==
"ONLINE"
:
nbtheia_online
+=
1
self
.
logger
.
info
(
logger
.
info
(
f
"
\n
THIEA has got
{
nbtheia_online
}
ONLINE products over
{
estimated_theia_of_results
}
"
)
else
:
self
.
logger
.
error
(
"error"
)
logger
.
error
(
"error"
)
return
self
.
products_availability
=
products_availability
...
...
@@ -177,12 +178,12 @@ class EodagS1():
if
len
(
self
.
products_downloaded
[
d
])
==
2
:
altdownloadedprods
.
append
(
self
.
products_downloaded
[
d
][
1
])
self
.
logger
.
info
(
f
"
\n
Serializing downloaded products to
{
serialfile
}
"
)
logger
.
info
(
f
"
\n
Serializing downloaded products to
{
serialfile
}
"
)
dps
=
SearchResult
(
downloadedprods
)
self
.
mEODataAccessGateway
.
serialize
(
dps
,
serialfile
)
if
len
(
altdownloadedprods
)
>
0
:
self
.
logger
.
info
(
f
"
\n
Serializing alternate products to
{
altserialfile
}
"
)
logger
.
info
(
f
"
\n
Serializing alternate products to
{
altserialfile
}
"
)
adps
=
SearchResult
(
altdownloadedprods
)
self
.
mEODataAccessGateway
.
serialize
(
adps
,
altserialfile
)
...
...
@@ -195,12 +196,12 @@ class EodagS1():
if
len
(
self
.
products_availability
[
d
])
==
2
:
altremainingprods
.
append
(
self
.
products_availability
[
d
][
1
])
self
.
logger
.
info
(
f
"
\n
Serializing downloaded products to
{
remainserialfile
}
"
)
logger
.
info
(
f
"
\n
Serializing downloaded products to
{
remainserialfile
}
"
)
rps
=
SearchResult
(
remainingprods
)
self
.
mEODataAccessGateway
.
serialize
(
rps
,
remainserialfile
)
if
len
(
altremainingprods
)
>
0
:
self
.
logger
.
info
(
f
"
\n
Serializing alternate products to
{
altremainserialfile
}
"
)
logger
.
info
(
f
"
\n
Serializing alternate products to
{
altremainserialfile
}
"
)
arps
=
SearchResult
(
altremainingprods
)
self
.
mEODataAccessGateway
.
serialize
(
arps
,
altremainserialfile
)
...
...
@@ -224,8 +225,8 @@ class EodagS1():
else
:
self
.
products_availability
[
pp
.
properties
[
"id"
]]
=
[
pp
]
self
.
logger
.
info
(
"Geojson files successfully imported"
)
self
.
logger
.
info
(
"Use start_downloads() to launch downloads"
)
logger
.
info
(
"Geojson files successfully imported"
)
logger
.
info
(
"Use start_downloads() to launch downloads"
)
def
start_downloads
(
self
,
downdir
):
...
...
@@ -242,7 +243,7 @@ class EodagS1():
while
still2download
:
for
p
in
sorted
(
self
.
products_availability
,
key
=
lambda
k
:
len
(
self
.
products_availability
[
k
]),
reverse
=
True
):
firstprovider
=
self
.
products_availability
[
p
][
0
].
provider
self
.
logger
.
info
(
f
"
\n
Starting Download
{
p
}
from
{
firstprovider
}
..."
)
logger
.
info
(
f
"
\n
Starting Download
{
p
}
from
{
firstprovider
}
..."
)
self
.
mEODataAccessGateway
.
set_preferred_provider
(
firstprovider
)
failed
=
False
try
:
...
...
@@ -259,7 +260,7 @@ class EodagS1():
if
failed
and
len
(
self
.
products_availability
[
p
])
==
2
:
secndprovider
=
self
.
products_availability
[
p
][
1
].
provider
failed
=
False
self
.
logger
.
info
(
f
"
\n
Download failed. Trying with
{
secndprovider
}
"
)
logger
.
info
(
f
"
\n
Download failed. Trying with
{
secndprovider
}
"
)
self
.
mEODataAccessGateway
.
set_preferred_provider
(
secndprovider
)
try
:
with
closing
(
StringIO
())
as
tqdm_out
:
...
...
@@ -283,7 +284,7 @@ class EodagS1():
if
len
(
self
.
products_downloaded
)
==
nbProducts2d
:
still2download
=
False
self
.
logger
.
info
(
logger
.
info
(
f
"
\n
All downloads successful."
)
self
.
__export_geojson
(
downdir
,
0
)
...
...
@@ -292,7 +293,7 @@ class EodagS1():
elif
nb_allretry
==
5
:
still2download
=
False
self
.
logger
.
info
(
logger
.
info
(
"""
Max number of retries has been completed.
There is still products to download.
...
...
@@ -301,13 +302,13 @@ class EodagS1():
)
self
.
__export_geojson
(
downdir
,
1
)
self
.
logger
.
info
(
"Done."
)
logger
.
info
(
"Done."
)
else
:
self
.
logger
.
info
(
f
"
\n
Pass n#
{
nb_allretry
}
has reached
{
len
(
self
.
products_downloaded
)
}
over
{
nbProducts2d
}
expected"
)
self
.
logger
.
info
(
f
"
\n
Waiting
{
passwait
[
nb_allretry
-
1
]
}
minutes before retrying..."
)
logger
.
info
(
f
"
\n
Pass n#
{
nb_allretry
}
has reached
{
len
(
self
.
products_downloaded
)
}
over
{
nbProducts2d
}
expected"
)
logger
.
info
(
f
"
\n
Waiting
{
passwait
[
nb_allretry
-
1
]
}
minutes before retrying..."
)
restartd
=
datetime
.
now
()
+
timedelta
(
minutes
=
passwait
[
nb_allretry
-
1
])
restartstr
=
restartd
.
strftime
(
"%Y/%m/%d-%H:%M:%S"
)
self
.
logger
.
info
(
f
"
\n
Restart at
{
restartstr
}
"
)
logger
.
info
(
f
"
\n
Restart at
{
restartstr
}
"
)
time
.
sleep
(
passwait
[
nb_allretry
-
1
]
*
60
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment