Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
Lozac'h Loic
AgriFrozenAreaProject
Commits
000ffac0
Commit
000ffac0
authored
Aug 19, 2021
by
Lozac'h Loic
Browse files
debug
parent
e3491bd2
Changes
2
Hide whitespace changes
Inline
Side-by-side
frozenapp/eodag/user_conf.yml
View file @
000ffac0
...
...
@@ -16,7 +16,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
scihub
:
priority
:
2
priority
:
1
api
:
credentials
:
username
:
loic.lozach
...
...
@@ -25,7 +25,8 @@ scihub:
peps
:
priority
:
1
api
:
credentials
:
username
:
loic.lozach@irstea.fr
password
:
PMUQc22$
auth
:
credentials
:
username
:
loic.lozach@irstea.fr
password
:
PMUQc22$
frozenapp/eodags1.py
View file @
000ffac0
...
...
@@ -35,17 +35,17 @@ class EodagS1Downloader():
def
__init__
(
self
,
mrgstile
,
startdate
,
enddate
,
downdir
):
def
print_ndict
(
data
):
if
type
(
data
)
==
dict
:
for
k
,
v
in
data
.
items
():
print
(
"key: "
+
str
(
k
))
print_ndict
(
v
)
else
:
try
:
print
(
"value: "
+
str
(
data
))
except
Exception
as
err
:
print
(
err
)
print
(
type
(
data
))
#
def print_ndict(data):
#
if type(data) == dict:
#
for k, v in data.items():
#
print("key: "+str(k))
#
print_ndict(v)
#
else:
#
try:
#
print("value: "+str(data))
#
except Exception as err:
#
print(err)
#
print(type(data))
'''
Constructor
'''
...
...
@@ -139,16 +139,30 @@ class EodagS1Downloader():
f
"
\n
PEPS has got
{
nbpeps_online
}
ONLINE products over
{
estimated_peps_of_results
}
"
)
nb_dblav
=
0
for
p
in
sorted
(
products_availability
,
key
=
lambda
k
:
len
(
products_availability
[
k
]),
reverse
=
True
):
if
len
(
products_availability
[
p
])
==
1
:
print
(
f
"
\n
Product
{
p
}
is only available on
{
products_availability
[
p
][
0
].
provider
}
"
)
else
:
nb_dblav
+=
1
# products_priority1_peps=[]
# products_priority2_peps=[]
# products_priority1_scihub=[]
# products_priority2_scihub=[]
# for p in sorted(products_availability, key=lambda k: len(products_availability[k]), reverse=True):
#
# if len(products_availability[p]) == 1 :
# print(
# f"\nProduct {p} is only available on {products_availability[p][0].provider}"
# )
# if products_availability[p][0].provider == "peps":
# products_priority1_peps.append(products_availability[p][0])
# elif products_availability[p][0].provider == "scihub":
# products_priority1_scihub.append(products_availability[p][0])
# else:
# if products_availability[p][0].provider == "peps":
# products_priority1_peps.append(products_availability[p][0])
# elif products_availability[p][0].provider == "scihub":
# products_priority1_scihub.append(products_availability[p][0])
#
# if products_availability[p][1].provider == "peps":
# products_priority2_peps.append(products_availability[p][1])
# elif products_availability[p][1].provider == "scihub":
# products_priority2_scihub.append(products_availability[p][1])
products_downloaded
=
{}
passwait
=
[
10
,
30
,
60
,
120
,
300
]
...
...
@@ -159,74 +173,76 @@ class EodagS1Downloader():
allserialfile
=
os
.
path
.
join
(
downdir
,
nowstr
+
"_alldownloads.json"
)
remainserialfile
=
os
.
path
.
join
(
downdir
,
nowstr
+
"_remaindownloads.json"
)
#DEBUG
for
p
in
sorted
(
products_availability
,
key
=
lambda
k
:
len
(
products_availability
[
k
]),
reverse
=
True
):
prod0
=
products_availability
[
p
][
0
]
prod0dict
=
prod0
.
as_dict
()
print_ndict
(
prod0dict
)
prod1
=
products_availability
[
p
][
1
]
prod1dict
=
prod1
.
as_dict
()
print_ndict
(
prod1dict
)
break
#
for p in sorted(products_availability, key=lambda k: len(products_availability[k]), reverse=True):
#
prod0 = products_availability[p][0]
#
prod0dict = prod0.as_dict()
#
print_ndict(prod0dict)
#
prod1 = products_availability[p][1]
#
prod1dict = prod1.as_dict()
#
print_ndict(prod1dict)
#
#
break
# while still2download:
# for p in sorted(products_availability, key=lambda k: len(products_availability[k]), reverse=True):
# firstprovider = products_availability[p][0].provider
# print(f"\nStarting Download {p} from {firstprovider}...")
# failed=False
# try:
# with closing(StringIO()) as tqdm_out:
# with ProgressCallback(file=tqdm_out) as bar:
# products_availability[p][0].download( bar, extract=False, outputs_prefix=downdir)
# except Exception as err:
# failed=True
# print(err)
#
# if failed and len(products_availability[p]) == 2:
# secndprovider = products_availability[p][1].provider
# failed=False
# print("Download failed. Trying with {secndprovider}")
# try:
# with closing(StringIO()) as tqdm_out:
# with ProgressCallback(file=tqdm_out) as bar:
# products_availability[p][1].download( bar, extract=False, outputs_prefix=downdir)
# except Exception as err:
# failed=True
# print(err)
#
#
# if products_availability[p][0].location[:5] == "file:":
# products_downloaded[p] = products_availability[p][0]
#
# for p in products_downloaded.keys():
# if p in products_availability.keys():
# products_availability.pop(p)
# nb_allretry += 1
#
# if len(products_downloaded) == nbProducts2d :
# still2download = False
# print(
# f"\nAll downloads successful."
# f"\nSerializing search products to {allserialfile}"
# f"\nExiting."
# )
# elif nb_allretry == 5 :
# still2download = False
# print(
# f"\nMax number of retries has been completed."
# f"\nSerializing all search products to {allserialfile}"
# f"\nSerializing remain search products to {remainserialfile}"
# f"\nExiting."
# )
# else:
# print(f"\nPass n#{nb_allretry} has reached {len(products_downloaded)} over {nbProducts2d} expected")
# print(f"\nWaiting {passwait[nb_allretry-1]} minutes before retrying...")
# restartd = datetime.now()+timedelta(minutes=passwait[nb_allretry-1])
# restartstr = restartd.strftime("%Y/%m/%d-%H:%M:%S")
# print(f"\nRestart at {restartstr}")
# time.sleep(passwait[nb_allretry-1]*60)
#
while
still2download
:
for
p
in
sorted
(
products_availability
,
key
=
lambda
k
:
len
(
products_availability
[
k
]),
reverse
=
True
):
firstprovider
=
products_availability
[
p
][
0
].
provider
print
(
f
"
\n
Starting Download
{
p
}
from
{
firstprovider
}
..."
)
self
.
mEODataAccessGateway
.
set_preferred_provider
(
firstprovider
)
failed
=
False
try
:
with
closing
(
StringIO
())
as
tqdm_out
:
with
ProgressCallback
(
file
=
tqdm_out
)
as
bar
:
self
.
mEODataAccessGateway
.
download
(
products_availability
[
p
][
0
],
bar
,
extract
=
False
,
outputs_prefix
=
downdir
)
except
Exception
as
err
:
failed
=
True
print
(
err
)
if
failed
and
len
(
products_availability
[
p
])
==
2
:
secndprovider
=
products_availability
[
p
][
1
].
provider
failed
=
False
print
(
"Download failed. Trying with {secndprovider}"
)
self
.
mEODataAccessGateway
.
set_preferred_provider
(
secndprovider
)
try
:
with
closing
(
StringIO
())
as
tqdm_out
:
with
ProgressCallback
(
file
=
tqdm_out
)
as
bar
:
self
.
mEODataAccessGateway
.
download
(
products_availability
[
p
][
1
],
bar
,
extract
=
False
,
outputs_prefix
=
downdir
)
except
Exception
as
err
:
failed
=
True
print
(
err
)
if
products_availability
[
p
][
0
].
location
[:
5
]
==
"file:"
:
products_downloaded
[
p
]
=
products_availability
[
p
][
0
]
for
p
in
products_downloaded
.
keys
():
if
p
in
products_availability
.
keys
():
products_availability
.
pop
(
p
)
nb_allretry
+=
1
if
len
(
products_downloaded
)
==
nbProducts2d
:
still2download
=
False
print
(
f
"
\n
All downloads successful."
f
"
\n
Serializing search products to
{
allserialfile
}
"
f
"
\n
Exiting."
)
elif
nb_allretry
==
5
:
still2download
=
False
print
(
f
"
\n
Max number of retries has been completed."
f
"
\n
Serializing all search products to
{
allserialfile
}
"
f
"
\n
Serializing remain search products to
{
remainserialfile
}
"
f
"
\n
Exiting."
)
else
:
print
(
f
"
\n
Pass n#
{
nb_allretry
}
has reached
{
len
(
products_downloaded
)
}
over
{
nbProducts2d
}
expected"
)
print
(
f
"
\n
Waiting
{
passwait
[
nb_allretry
-
1
]
}
minutes before retrying..."
)
restartd
=
datetime
.
now
()
+
timedelta
(
minutes
=
passwait
[
nb_allretry
-
1
])
restartstr
=
restartd
.
strftime
(
"%Y/%m/%d-%H:%M:%S"
)
print
(
f
"
\n
Restart at
{
restartstr
}
"
)
time
.
sleep
(
passwait
[
nb_allretry
-
1
]
*
60
)
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment