Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
Lozac'h Loic
AgriFrozenAreaProject
Commits
46116ebb
Commit
46116ebb
authored
Aug 25, 2021
by
Lozac'h Loic
Browse files
debug
parent
ed9af134
Changes
1
Hide whitespace changes
Inline
Side-by-side
frozenapp/eodags1.py
View file @
46116ebb
...
...
@@ -3,7 +3,7 @@ Created on 18 juin 2021
@author: loic
'''
import
os
import
os
,
time
,
logging
from
eodag.utils.logging
import
setup_logging
from
eodag.api.core
import
EODataAccessGateway
from
eodag.api.search_result
import
SearchResult
...
...
@@ -13,7 +13,7 @@ from eodag.utils import ProgressCallback
from
contextlib
import
closing
from
io
import
StringIO
from
datetime
import
datetime
,
timedelta
import
time
# os.environ["EODAG__PEPS__AUTH__CREDENTIALS__USERNAME"] = "loic.lozach"
# os.environ["EODAG__PEPS__AUTH__CREDENTIALS__PASSWORD"] = "PMUQc22$"
...
...
@@ -39,18 +39,19 @@ class EodagS1():
'''
self
.
mEODataAccessGateway
=
EODataAccessGateway
(
user_conf_file_path
=
self
.
user_conf_file_path
)
setup_logging
(
verbose
=
0
,
no_progress_bar
=
True
)
print
(
"EODataAccessGateway initialized."
)
logger
=
logging
.
getLogger
(
__name__
)
logger
.
info
(
"EODataAccessGateway initialized."
)
def
search_and_download
(
self
,
prodtypenum
,
mrgstile
,
startdate
,
enddate
,
downdir
):
qs_tile
=
TileEnvelope
.
objects
.
annotate
(
wgs84
=
Transform
(
'geom'
,
4326
)).
filter
(
tile__exact
=
mrgstile
)
if
len
(
qs_tile
)
!=
1
:
print
(
"Can't find tile "
+
mrgstile
)
logger
.
info
(
"Can't find tile "
+
mrgstile
)
exit
()
# geomwgs84 = Transform(qs_tile[0].geom, 4326)
print
(
qs_tile
[
0
].
wgs84
)
logger
.
info
(
qs_tile
[
0
].
wgs84
)
mpolywkt
=
str
(
qs_tile
[
0
].
wgs84
.
ewkt
).
split
(
";"
)
print
(
mpolywkt
[
1
])
...
...
@@ -59,7 +60,7 @@ class EodagS1():
products_availability
=
{}
if
prodtypenum
in
[
0
,
1
]:
print
(
"Search on Scihub..."
)
logger
.
info
(
"Search on Scihub..."
)
self
.
mEODataAccessGateway
.
set_preferred_provider
(
"scihub"
)
...
...
@@ -70,10 +71,10 @@ class EodagS1():
geom
=
mpolywkt
[
1
]
)
print
(
logger
.
info
(
f
"The Scihub search has found an estimated number of
{
estimated_scihub_of_results
}
products matching your criteria "
)
print
(
"Search on Peps..."
)
logger
.
info
(
"Search on Peps..."
)
self
.
mEODataAccessGateway
.
set_preferred_provider
(
"peps"
)
products_peps
,
estimated_peps_of_results
=
self
.
mEODataAccessGateway
.
search
(
...
...
@@ -83,7 +84,7 @@ class EodagS1():
geom
=
mpolywkt
[
1
]
)
print
(
logger
.
info
(
f
"The PEPS search has found an estimated number of
{
estimated_peps_of_results
}
products matching your criteria "
)
...
...
@@ -122,13 +123,13 @@ class EodagS1():
if
pp
.
properties
[
"storageStatus"
]
==
"ONLINE"
:
nbpeps_online
+=
1
print
(
logger
.
info
(
f
"
\n
Scihub has got
{
nbsci_online
}
ONLINE products over
{
estimated_scihub_of_results
}
"
f
"
\n
PEPS has got
{
nbpeps_online
}
ONLINE products over
{
estimated_peps_of_results
}
"
)
elif
prodtypenum
==
2
:
print
(
"Search on Theia..."
)
logger
.
info
(
"Search on Theia..."
)
self
.
mEODataAccessGateway
.
set_preferred_provider
(
"theia"
)
...
...
@@ -138,7 +139,7 @@ class EodagS1():
end
=
enddate
,
geom
=
mpolywkt
[
1
]
)
print
(
logger
.
info
(
f
"The THEIA search has found an estimated number of
{
estimated_theia_of_results
}
products matching your criteria "
)
nbtheia_online
=
0
...
...
@@ -147,12 +148,12 @@ class EodagS1():
if
pt
.
properties
[
"storageStatus"
]
==
"ONLINE"
:
nbtheia_online
+=
1
print
(
logger
.
info
(
f
"
\n
THIEA has got
{
nbtheia_online
}
ONLINE products over
{
estimated_theia_of_results
}
"
)
else
:
print
(
"error"
)
logger
.
error
(
"error"
)
return
self
.
products_availability
=
products_availability
...
...
@@ -176,12 +177,12 @@ class EodagS1():
if
len
(
self
.
products_downloaded
[
d
])
==
2
:
altdownloadedprods
.
append
(
self
.
products_downloaded
[
d
][
1
])
print
(
f
"
\n
Serializing downloaded products to
{
serialfile
}
"
)
logger
.
info
(
f
"
\n
Serializing downloaded products to
{
serialfile
}
"
)
dps
=
SearchResult
(
downloadedprods
)
self
.
mEODataAccessGateway
.
serialize
(
dps
,
serialfile
)
if
len
(
altdownloadedprods
)
>
0
:
print
(
f
"
\n
Serializing alternate products to
{
altserialfile
}
"
)
logger
.
info
(
f
"
\n
Serializing alternate products to
{
altserialfile
}
"
)
adps
=
SearchResult
(
altdownloadedprods
)
self
.
mEODataAccessGateway
.
serialize
(
adps
,
altserialfile
)
...
...
@@ -194,12 +195,12 @@ class EodagS1():
if
len
(
self
.
products_availability
[
d
])
==
2
:
altremainingprods
.
append
(
self
.
products_availability
[
d
][
1
])
print
(
f
"
\n
Serializing downloaded products to
{
remainserialfile
}
"
)
logger
.
info
(
f
"
\n
Serializing downloaded products to
{
remainserialfile
}
"
)
rps
=
SearchResult
(
remainingprods
)
self
.
mEODataAccessGateway
.
serialize
(
rps
,
remainserialfile
)
if
len
(
altremainingprods
)
>
0
:
print
(
f
"
\n
Serializing alternate products to
{
altremainserialfile
}
"
)
logger
.
info
(
f
"
\n
Serializing alternate products to
{
altremainserialfile
}
"
)
arps
=
SearchResult
(
altremainingprods
)
self
.
mEODataAccessGateway
.
serialize
(
arps
,
altremainserialfile
)
...
...
@@ -223,8 +224,8 @@ class EodagS1():
else
:
self
.
products_availability
[
pp
.
properties
[
"id"
]]
=
[
pp
]
print
(
"Geojson files successfully imported"
)
print
(
"Use start_downloads() to launch downloads"
)
logger
.
info
(
"Geojson files successfully imported"
)
logger
.
info
(
"Use start_downloads() to launch downloads"
)
def
start_downloads
(
self
,
downdir
):
...
...
@@ -241,13 +242,13 @@ class EodagS1():
while
still2download
:
for
p
in
sorted
(
self
.
products_availability
,
key
=
lambda
k
:
len
(
self
.
products_availability
[
k
]),
reverse
=
True
):
firstprovider
=
self
.
products_availability
[
p
][
0
].
provider
print
(
f
"
\n
Starting Download
{
p
}
from
{
firstprovider
}
..."
)
logger
.
info
(
f
"
\n
Starting Download
{
p
}
from
{
firstprovider
}
..."
)
self
.
mEODataAccessGateway
.
set_preferred_provider
(
firstprovider
)
failed
=
False
try
:
with
closing
(
StringIO
())
as
tqdm_out
:
with
ProgressCallback
(
file
=
tqdm_out
)
as
bar
:
self
.
mEODataAccessGateway
.
download
(
self
.
products_availability
[
p
][
0
],
bar
,
extract
=
False
,
outputs_prefix
=
downdir
)
self
.
mEODataAccessGateway
.
download
(
self
.
products_availability
[
p
][
0
],
bar
,
extract
=
False
,
outputs_prefix
=
downdir
,
timeout
=
1
)
except
Exception
as
err
:
failed
=
True
print
(
err
)
...
...
@@ -258,12 +259,12 @@ class EodagS1():
if
failed
and
len
(
self
.
products_availability
[
p
])
==
2
:
secndprovider
=
self
.
products_availability
[
p
][
1
].
provider
failed
=
False
print
(
f
"
\n
Download failed. Trying with
{
secndprovider
}
"
)
logger
.
info
(
f
"
\n
Download failed. Trying with
{
secndprovider
}
"
)
self
.
mEODataAccessGateway
.
set_preferred_provider
(
secndprovider
)
try
:
with
closing
(
StringIO
())
as
tqdm_out
:
with
ProgressCallback
(
file
=
tqdm_out
)
as
bar
:
self
.
mEODataAccessGateway
.
download
(
self
.
products_availability
[
p
][
1
],
bar
,
extract
=
False
,
outputs_prefix
=
downdir
)
self
.
mEODataAccessGateway
.
download
(
self
.
products_availability
[
p
][
1
],
bar
,
extract
=
False
,
outputs_prefix
=
downdir
,
timeout
=
1
)
except
Exception
as
err
:
failed
=
True
print
(
err
)
...
...
@@ -282,7 +283,7 @@ class EodagS1():
if
len
(
self
.
products_downloaded
)
==
nbProducts2d
:
still2download
=
False
print
(
logger
.
info
(
f
"
\n
All downloads successful."
)
self
.
__export_geojson
(
downdir
,
0
)
...
...
@@ -291,7 +292,7 @@ class EodagS1():
elif
nb_allretry
==
5
:
still2download
=
False
print
(
logger
.
info
(
"""
Max number of retries has been completed.
There is still products to download.
...
...
@@ -300,13 +301,13 @@ class EodagS1():
)
self
.
__export_geojson
(
downdir
,
1
)
print
(
"Done."
)
logger
.
info
(
"Done."
)
else
:
print
(
f
"
\n
Pass n#
{
nb_allretry
}
has reached
{
len
(
self
.
products_downloaded
)
}
over
{
nbProducts2d
}
expected"
)
print
(
f
"
\n
Waiting
{
passwait
[
nb_allretry
-
1
]
}
minutes before retrying..."
)
logger
.
info
(
f
"
\n
Pass n#
{
nb_allretry
}
has reached
{
len
(
self
.
products_downloaded
)
}
over
{
nbProducts2d
}
expected"
)
logger
.
info
(
f
"
\n
Waiting
{
passwait
[
nb_allretry
-
1
]
}
minutes before retrying..."
)
restartd
=
datetime
.
now
()
+
timedelta
(
minutes
=
passwait
[
nb_allretry
-
1
])
restartstr
=
restartd
.
strftime
(
"%Y/%m/%d-%H:%M:%S"
)
print
(
f
"
\n
Restart at
{
restartstr
}
"
)
logger
.
info
(
f
"
\n
Restart at
{
restartstr
}
"
)
time
.
sleep
(
passwait
[
nb_allretry
-
1
]
*
60
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment