Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
Lozac'h Loic
AgriFrozenAreaProject
Commits
dee1fda3
Commit
dee1fda3
authored
Aug 25, 2021
by
Lozac'h Loic
Browse files
test
parent
57df37b7
Changes
1
Hide whitespace changes
Inline
Side-by-side
frozenapp/eodags1.py
View file @
dee1fda3
...
...
@@ -40,7 +40,7 @@ class EodagS1():
Constructor
'''
self
.
mEODataAccessGateway
=
EODataAccessGateway
(
user_conf_file_path
=
self
.
user_conf_file_path
)
setup_logging
(
verbose
=
3
,
no_progress_bar
=
Tru
e
)
setup_logging
(
verbose
=
2
,
no_progress_bar
=
Fals
e
)
logger
.
info
(
"EODataAccessGateway initialized."
)
def
search_and_download
(
self
,
prodtypenum
,
mrgstile
,
startdate
,
enddate
,
downdir
):
...
...
@@ -124,10 +124,8 @@ class EodagS1():
if
pp
.
properties
[
"storageStatus"
]
==
"ONLINE"
:
nbpeps_online
+=
1
logger
.
info
(
f
"
\n
Scihub has got
{
nbsci_online
}
ONLINE products over
{
estimated_scihub_of_results
}
"
f
"
\n
PEPS has got
{
nbpeps_online
}
ONLINE products over
{
estimated_peps_of_results
}
"
)
logger
.
info
(
f
"Scihub has got
{
nbsci_online
}
ONLINE products over
{
estimated_scihub_of_results
}
"
)
logger
.
info
(
f
"PEPS has got
{
nbpeps_online
}
ONLINE products over
{
estimated_peps_of_results
}
"
)
elif
prodtypenum
==
2
:
logger
.
info
(
"Search on Theia..."
)
...
...
@@ -150,7 +148,7 @@ class EodagS1():
nbtheia_online
+=
1
logger
.
info
(
f
"
\n
THIEA has got
{
nbtheia_online
}
ONLINE products over
{
estimated_theia_of_results
}
"
f
"THIEA has got
{
nbtheia_online
}
ONLINE products over
{
estimated_theia_of_results
}
"
)
else
:
...
...
@@ -178,12 +176,12 @@ class EodagS1():
if
len
(
self
.
products_downloaded
[
d
])
==
2
:
altdownloadedprods
.
append
(
self
.
products_downloaded
[
d
][
1
])
logger
.
info
(
f
"
\n
Serializing downloaded products to
{
serialfile
}
"
)
logger
.
info
(
f
"Serializing downloaded products to
{
serialfile
}
"
)
dps
=
SearchResult
(
downloadedprods
)
self
.
mEODataAccessGateway
.
serialize
(
dps
,
serialfile
)
if
len
(
altdownloadedprods
)
>
0
:
logger
.
info
(
f
"
\n
Serializing alternate products to
{
altserialfile
}
"
)
logger
.
info
(
f
"Serializing alternate products to
{
altserialfile
}
"
)
adps
=
SearchResult
(
altdownloadedprods
)
self
.
mEODataAccessGateway
.
serialize
(
adps
,
altserialfile
)
...
...
@@ -196,12 +194,12 @@ class EodagS1():
if
len
(
self
.
products_availability
[
d
])
==
2
:
altremainingprods
.
append
(
self
.
products_availability
[
d
][
1
])
logger
.
info
(
f
"
\n
Serializing downloaded products to
{
remainserialfile
}
"
)
logger
.
info
(
f
"Serializing downloaded products to
{
remainserialfile
}
"
)
rps
=
SearchResult
(
remainingprods
)
self
.
mEODataAccessGateway
.
serialize
(
rps
,
remainserialfile
)
if
len
(
altremainingprods
)
>
0
:
logger
.
info
(
f
"
\n
Serializing alternate products to
{
altremainserialfile
}
"
)
logger
.
info
(
f
"Serializing alternate products to
{
altremainserialfile
}
"
)
arps
=
SearchResult
(
altremainingprods
)
self
.
mEODataAccessGateway
.
serialize
(
arps
,
altremainserialfile
)
...
...
@@ -243,13 +241,13 @@ class EodagS1():
while
still2download
:
for
p
in
sorted
(
self
.
products_availability
,
key
=
lambda
k
:
len
(
self
.
products_availability
[
k
]),
reverse
=
True
):
firstprovider
=
self
.
products_availability
[
p
][
0
].
provider
logger
.
info
(
f
"
\n
Starting Download
{
p
}
from
{
firstprovider
}
..."
)
logger
.
info
(
f
"Starting Download
{
p
}
from
{
firstprovider
}
..."
)
self
.
mEODataAccessGateway
.
set_preferred_provider
(
firstprovider
)
failed
=
False
try
:
with
closing
(
StringIO
())
as
tqdm_out
:
with
ProgressCallback
(
file
=
tqdm_out
)
as
bar
:
self
.
mEODataAccessGateway
.
download
(
self
.
products_availability
[
p
][
0
],
bar
,
extract
=
False
,
outputs_prefix
=
downdir
,
timeout
=
1
)
self
.
mEODataAccessGateway
.
download
(
self
.
products_availability
[
p
][
0
],
bar
,
extract
=
False
,
outputs_prefix
=
downdir
,
wait
=
0
,
timeout
=
0
)
except
Exception
as
err
:
failed
=
True
print
(
err
)
...
...
@@ -260,12 +258,12 @@ class EodagS1():
if
failed
and
len
(
self
.
products_availability
[
p
])
==
2
:
secndprovider
=
self
.
products_availability
[
p
][
1
].
provider
failed
=
False
logger
.
info
(
f
"
\n
Download failed. Trying with
{
secndprovider
}
"
)
logger
.
info
(
f
"Download failed. Trying with
{
secndprovider
}
"
)
self
.
mEODataAccessGateway
.
set_preferred_provider
(
secndprovider
)
try
:
with
closing
(
StringIO
())
as
tqdm_out
:
with
ProgressCallback
(
file
=
tqdm_out
)
as
bar
:
self
.
mEODataAccessGateway
.
download
(
self
.
products_availability
[
p
][
1
],
bar
,
extract
=
False
,
outputs_prefix
=
downdir
,
timeout
=
1
)
self
.
mEODataAccessGateway
.
download
(
self
.
products_availability
[
p
][
1
],
bar
,
extract
=
False
,
outputs_prefix
=
downdir
,
wait
=
0
,
timeout
=
0
)
except
Exception
as
err
:
failed
=
True
print
(
err
)
...
...
@@ -285,11 +283,11 @@ class EodagS1():
if
len
(
self
.
products_downloaded
)
==
nbProducts2d
:
still2download
=
False
logger
.
info
(
f
"
\n
All downloads successful."
f
"All downloads successful."
)
self
.
__export_geojson
(
downdir
,
0
)
print
(
"Done."
)
logger
.
info
(
"Done."
)
elif
nb_allretry
==
5
:
still2download
=
False
...
...
@@ -304,11 +302,11 @@ class EodagS1():
logger
.
info
(
"Done."
)
else
:
logger
.
info
(
f
"
\n
Pass n#
{
nb_allretry
}
has reached
{
len
(
self
.
products_downloaded
)
}
over
{
nbProducts2d
}
expected"
)
logger
.
info
(
f
"
\n
Waiting
{
passwait
[
nb_allretry
-
1
]
}
minutes before retrying..."
)
logger
.
info
(
f
"Pass n#
{
nb_allretry
}
has reached
{
len
(
self
.
products_downloaded
)
}
over
{
nbProducts2d
}
expected"
)
logger
.
info
(
f
"Waiting
{
passwait
[
nb_allretry
-
1
]
}
minutes before retrying..."
)
restartd
=
datetime
.
now
()
+
timedelta
(
minutes
=
passwait
[
nb_allretry
-
1
])
restartstr
=
restartd
.
strftime
(
"%Y/%m/%d-%H:%M:%S"
)
logger
.
info
(
f
"
\n
Restart at
{
restartstr
}
"
)
logger
.
info
(
f
"Restart at
{
restartstr
}
"
)
time
.
sleep
(
passwait
[
nb_allretry
-
1
]
*
60
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment