Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
Lozac'h Loic
AgriFrozenAreaProject
Commits
bf5e9caa
Commit
bf5e9caa
authored
Aug 25, 2021
by
Lozac'h Loic
Browse files
debug
parent
46116ebb
Changes
1
Hide whitespace changes
Inline
Side-by-side
frozenapp/eodags1.py
View file @
bf5e9caa
...
...
@@ -25,7 +25,7 @@ class EodagS1():
'''
classdocs
'''
logger
=
None
mEODataAccessGateway
=
None
user_conf_file_path
=
os
.
path
.
join
(
os
.
path
.
dirname
(
os
.
path
.
abspath
(
__file__
)),
"eodag/user_conf.yml"
)
locations_conf_path
=
None
...
...
@@ -39,19 +39,19 @@ class EodagS1():
'''
self
.
mEODataAccessGateway
=
EODataAccessGateway
(
user_conf_file_path
=
self
.
user_conf_file_path
)
setup_logging
(
verbose
=
0
,
no_progress_bar
=
True
)
logger
=
logging
.
getLogger
(
__name__
)
logger
.
info
(
"EODataAccessGateway initialized."
)
self
.
logger
=
logging
.
getLogger
(
__name__
)
self
.
logger
.
info
(
"EODataAccessGateway initialized."
)
def
search_and_download
(
self
,
prodtypenum
,
mrgstile
,
startdate
,
enddate
,
downdir
):
qs_tile
=
TileEnvelope
.
objects
.
annotate
(
wgs84
=
Transform
(
'geom'
,
4326
)).
filter
(
tile__exact
=
mrgstile
)
if
len
(
qs_tile
)
!=
1
:
logger
.
info
(
"Can't find tile "
+
mrgstile
)
self
.
logger
.
info
(
"Can't find tile "
+
mrgstile
)
exit
()
# geomwgs84 = Transform(qs_tile[0].geom, 4326)
logger
.
info
(
qs_tile
[
0
].
wgs84
)
self
.
logger
.
info
(
qs_tile
[
0
].
wgs84
)
mpolywkt
=
str
(
qs_tile
[
0
].
wgs84
.
ewkt
).
split
(
";"
)
print
(
mpolywkt
[
1
])
...
...
@@ -60,7 +60,7 @@ class EodagS1():
products_availability
=
{}
if
prodtypenum
in
[
0
,
1
]:
logger
.
info
(
"Search on Scihub..."
)
self
.
logger
.
info
(
"Search on Scihub..."
)
self
.
mEODataAccessGateway
.
set_preferred_provider
(
"scihub"
)
...
...
@@ -71,10 +71,10 @@ class EodagS1():
geom
=
mpolywkt
[
1
]
)
logger
.
info
(
self
.
logger
.
info
(
f
"The Scihub search has found an estimated number of
{
estimated_scihub_of_results
}
products matching your criteria "
)
logger
.
info
(
"Search on Peps..."
)
self
.
logger
.
info
(
"Search on Peps..."
)
self
.
mEODataAccessGateway
.
set_preferred_provider
(
"peps"
)
products_peps
,
estimated_peps_of_results
=
self
.
mEODataAccessGateway
.
search
(
...
...
@@ -84,7 +84,7 @@ class EodagS1():
geom
=
mpolywkt
[
1
]
)
logger
.
info
(
self
.
logger
.
info
(
f
"The PEPS search has found an estimated number of
{
estimated_peps_of_results
}
products matching your criteria "
)
...
...
@@ -123,13 +123,13 @@ class EodagS1():
if
pp
.
properties
[
"storageStatus"
]
==
"ONLINE"
:
nbpeps_online
+=
1
logger
.
info
(
self
.
logger
.
info
(
f
"
\n
Scihub has got
{
nbsci_online
}
ONLINE products over
{
estimated_scihub_of_results
}
"
f
"
\n
PEPS has got
{
nbpeps_online
}
ONLINE products over
{
estimated_peps_of_results
}
"
)
elif
prodtypenum
==
2
:
logger
.
info
(
"Search on Theia..."
)
self
.
logger
.
info
(
"Search on Theia..."
)
self
.
mEODataAccessGateway
.
set_preferred_provider
(
"theia"
)
...
...
@@ -139,7 +139,7 @@ class EodagS1():
end
=
enddate
,
geom
=
mpolywkt
[
1
]
)
logger
.
info
(
self
.
logger
.
info
(
f
"The THEIA search has found an estimated number of
{
estimated_theia_of_results
}
products matching your criteria "
)
nbtheia_online
=
0
...
...
@@ -148,12 +148,12 @@ class EodagS1():
if
pt
.
properties
[
"storageStatus"
]
==
"ONLINE"
:
nbtheia_online
+=
1
logger
.
info
(
self
.
logger
.
info
(
f
"
\n
THIEA has got
{
nbtheia_online
}
ONLINE products over
{
estimated_theia_of_results
}
"
)
else
:
logger
.
error
(
"error"
)
self
.
logger
.
error
(
"error"
)
return
self
.
products_availability
=
products_availability
...
...
@@ -177,12 +177,12 @@ class EodagS1():
if
len
(
self
.
products_downloaded
[
d
])
==
2
:
altdownloadedprods
.
append
(
self
.
products_downloaded
[
d
][
1
])
logger
.
info
(
f
"
\n
Serializing downloaded products to
{
serialfile
}
"
)
self
.
logger
.
info
(
f
"
\n
Serializing downloaded products to
{
serialfile
}
"
)
dps
=
SearchResult
(
downloadedprods
)
self
.
mEODataAccessGateway
.
serialize
(
dps
,
serialfile
)
if
len
(
altdownloadedprods
)
>
0
:
logger
.
info
(
f
"
\n
Serializing alternate products to
{
altserialfile
}
"
)
self
.
logger
.
info
(
f
"
\n
Serializing alternate products to
{
altserialfile
}
"
)
adps
=
SearchResult
(
altdownloadedprods
)
self
.
mEODataAccessGateway
.
serialize
(
adps
,
altserialfile
)
...
...
@@ -195,12 +195,12 @@ class EodagS1():
if
len
(
self
.
products_availability
[
d
])
==
2
:
altremainingprods
.
append
(
self
.
products_availability
[
d
][
1
])
logger
.
info
(
f
"
\n
Serializing downloaded products to
{
remainserialfile
}
"
)
self
.
logger
.
info
(
f
"
\n
Serializing downloaded products to
{
remainserialfile
}
"
)
rps
=
SearchResult
(
remainingprods
)
self
.
mEODataAccessGateway
.
serialize
(
rps
,
remainserialfile
)
if
len
(
altremainingprods
)
>
0
:
logger
.
info
(
f
"
\n
Serializing alternate products to
{
altremainserialfile
}
"
)
self
.
logger
.
info
(
f
"
\n
Serializing alternate products to
{
altremainserialfile
}
"
)
arps
=
SearchResult
(
altremainingprods
)
self
.
mEODataAccessGateway
.
serialize
(
arps
,
altremainserialfile
)
...
...
@@ -224,8 +224,8 @@ class EodagS1():
else
:
self
.
products_availability
[
pp
.
properties
[
"id"
]]
=
[
pp
]
logger
.
info
(
"Geojson files successfully imported"
)
logger
.
info
(
"Use start_downloads() to launch downloads"
)
self
.
logger
.
info
(
"Geojson files successfully imported"
)
self
.
logger
.
info
(
"Use start_downloads() to launch downloads"
)
def
start_downloads
(
self
,
downdir
):
...
...
@@ -242,7 +242,7 @@ class EodagS1():
while
still2download
:
for
p
in
sorted
(
self
.
products_availability
,
key
=
lambda
k
:
len
(
self
.
products_availability
[
k
]),
reverse
=
True
):
firstprovider
=
self
.
products_availability
[
p
][
0
].
provider
logger
.
info
(
f
"
\n
Starting Download
{
p
}
from
{
firstprovider
}
..."
)
self
.
logger
.
info
(
f
"
\n
Starting Download
{
p
}
from
{
firstprovider
}
..."
)
self
.
mEODataAccessGateway
.
set_preferred_provider
(
firstprovider
)
failed
=
False
try
:
...
...
@@ -259,7 +259,7 @@ class EodagS1():
if
failed
and
len
(
self
.
products_availability
[
p
])
==
2
:
secndprovider
=
self
.
products_availability
[
p
][
1
].
provider
failed
=
False
logger
.
info
(
f
"
\n
Download failed. Trying with
{
secndprovider
}
"
)
self
.
logger
.
info
(
f
"
\n
Download failed. Trying with
{
secndprovider
}
"
)
self
.
mEODataAccessGateway
.
set_preferred_provider
(
secndprovider
)
try
:
with
closing
(
StringIO
())
as
tqdm_out
:
...
...
@@ -283,7 +283,7 @@ class EodagS1():
if
len
(
self
.
products_downloaded
)
==
nbProducts2d
:
still2download
=
False
logger
.
info
(
self
.
logger
.
info
(
f
"
\n
All downloads successful."
)
self
.
__export_geojson
(
downdir
,
0
)
...
...
@@ -292,7 +292,7 @@ class EodagS1():
elif
nb_allretry
==
5
:
still2download
=
False
logger
.
info
(
self
.
logger
.
info
(
"""
Max number of retries has been completed.
There is still products to download.
...
...
@@ -301,13 +301,13 @@ class EodagS1():
)
self
.
__export_geojson
(
downdir
,
1
)
logger
.
info
(
"Done."
)
self
.
logger
.
info
(
"Done."
)
else
:
logger
.
info
(
f
"
\n
Pass n#
{
nb_allretry
}
has reached
{
len
(
self
.
products_downloaded
)
}
over
{
nbProducts2d
}
expected"
)
logger
.
info
(
f
"
\n
Waiting
{
passwait
[
nb_allretry
-
1
]
}
minutes before retrying..."
)
self
.
logger
.
info
(
f
"
\n
Pass n#
{
nb_allretry
}
has reached
{
len
(
self
.
products_downloaded
)
}
over
{
nbProducts2d
}
expected"
)
self
.
logger
.
info
(
f
"
\n
Waiting
{
passwait
[
nb_allretry
-
1
]
}
minutes before retrying..."
)
restartd
=
datetime
.
now
()
+
timedelta
(
minutes
=
passwait
[
nb_allretry
-
1
])
restartstr
=
restartd
.
strftime
(
"%Y/%m/%d-%H:%M:%S"
)
logger
.
info
(
f
"
\n
Restart at
{
restartstr
}
"
)
self
.
logger
.
info
(
f
"
\n
Restart at
{
restartstr
}
"
)
time
.
sleep
(
passwait
[
nb_allretry
-
1
]
*
60
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment