Commit 392687a4 authored by Lozac'h Loic's avatar Lozac'h Loic
Browse files

first commit eodag

parent f9e65000
......@@ -2,7 +2,6 @@
<?eclipse-pydev version="1.0"?><pydev_project>
<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python interpreter</pydev_property>
<pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
<path>/${PROJECT_DIR_NAME}/SentinelDM</path>
</pydev_pathproperty>
</pydev_project>
......@@ -144,7 +144,7 @@ RUN mkdir -p /work/otb/build \
RUN cd /work/otb/otb/Modules/Remote \
&& GIT_SSL_NO_VERIFY=true git clone https://10.134.193.38/loic.lozach/agrifrozenareas.git \
&& GIT_SSL_NO_VERIFY=true git clone https://gitlab.irstea.fr/loic.lozach/agrifrozenareas.git \
&& cd /work/otb/build/OTB/build \
&& cmake /work/otb/otb \
-DOTB_WRAP_PYTHON=ON \
......
This diff is collapsed.
import time
import enum
from logging import getLogger
import logging
class Timer():
def __init__(self):
self._max=125
self._tick=0
self._running = False
def tick(self):
""" Call this to start timer """
# Store the current time in _tick
self._tick=time.time()
self._running = True
def tock(self):
""" Call this to stop timer """
# Add the delta in the times-list
# if time.time()-self._tick > self._max :
# self.tick()
return (time.time()-self._tick)
def half_tick(self):
if not self._running:
self.tick()
self._tick = self._max/2
def running(self):
return self._running
def stop(self):
self._running = False
def has_elapsed(self,seconds):
if time.time() - self._tick >= seconds:
return True
else:
return False
class Status(enum.Enum):
Offline=0
Pending = 1
Requested=2
Online=3
Downloading=4
Downloaded=5
Hash=6
Succeeded=7
Failed = 8
Corrupted=9
# TODO: replace enums with aenums later
def __str__(self):
if self.value == 0:
return "Offline"
elif self.value == 1:
return "Pending"
elif self.value == 2:
return "Requested"
elif self.value == 3:
return "Online"
elif self.value == 4:
return "Downloading"
elif self.value == 5:
return "Downloaded"
elif self.value == 6:
return "Hashing"
elif self.value == 7:
return "Succeeded"
elif self.value == 8:
return "Failed"
elif self.value == 9:
return "Corrupted"
def __lt__(self, other):
if isinstance(other, self.__class__):
return self.value < other.value
return False
def __le__(self, other):
if isinstance(other, self.__class__):
return self.value <= other.value
return False
def __gt__(self, other):
if isinstance(other, self.__class__):
return self.value > other.value
return False
def __ge__(self, other):
if isinstance(other, self.__class__):
return self.value >= other.value
return False
def __ne__(self, other):
if isinstance(other, self.__class__):
return self.value != other.value
return False
def __eq__(self, other):
if isinstance(other, self.__class__):
return self.value == other.value
return False
def setup_logger(key,sio):
fmt = '%(asctime)s [%(levelname)s] %(message)s'
datefmt='%d/%d %H:%M'
log = getLogger(key)
formatter = logging.Formatter(fmt,datefmt)
console = logging.StreamHandler(sio)
console.setFormatter(formatter)
log.addHandler(console)
log.setLevel("INFO")
return log
This diff is collapsed.
from misc import Status,setup_logger
from pySmartDL import SmartDL as DM
from pySmartDL import utils
from io import StringIO
import requests
import threading
import time
from multiprocessing import Process,Manager
import os
import hashlib
import uuid
import json
import logging
_checksum_process = None
_checksum_process_manager = None
_checksum_process_queue = None
_checksum_process_results = None
_checksum_process_signaler = None
class Payload:
# Recheck period in seconds
ESA_CREDENTIALS = ()
__thread_rlock = threading.RLock()
def __init__(self,
key,
download_link,
output_dir,
filename,
md5_checksum,
payload_size,
status=Status.Offline,
ext=".zip",
checksum_download=True,
abandon_download=False):
self.__key = key
self.__filename = filename
self.__download_link = download_link
self.__output_dir = output_dir
self.__credentials = Payload.ESA_CREDENTIALS
self.__status = status
self.__last_status = status
self.__changed_status = False
self.__logger_stream = StringIO()
self.__logger = setup_logger(key,self.__logger_stream)
self.__downloader = None
self.__downloader_ready = False
self.__queued_for_download = True
self.__md5_checksum = md5_checksum.lower()
self.__payload_size = payload_size
self.__ext = ext
self.__hash_check_thread = None
self.__request_thread = None
self.__recheck_thread = None
self.__fail_retries=3
self.__do_checksum = checksum_download
self.__abandon = abandon_download
# seconds to wait before re-downloading
self.__fail_retry_in = 60
self.__fail_retry_at = None
self.__retry_in_multiplier = 2
self.__uid = uuid.uuid4().hex
@property
def key(self):
return self.__key
@property
def filename(self):
return self.__filename
@property
def product_short_name(self):
prodName = self.__filename
return prodName[0:4]+prodName[17:32]
@property
def download_link(self):
return self.__download_link
@property
def output_dir(self):
return self.__output_dir
@output_dir.setter
def output_dir(self, value):
self.__output_dir = value
@property
def full_path(self):
return self.__output_dir+"/"+self.__filename+self.__ext
@property
def size(self):
return self.__payload_size
@property
def esa_credentials(self):
return self.__credentials
@esa_credentials.setter
def esa_credentials(self,value):
if type(value) is not tuple and len(value) != 2:
raise ValueError('Credentials should be a tuple of length 2')
else:
self.__credentials = value
@property
def downloader(self):
return self.__downloader
@property
def status(self):
return self.__status
@status.setter
def status(self,value):
self.__last_status = self.__status
self.__status = value
@property
def status_change(self):
self.__changed_status = self.__status != self.__last_status
return self.__changed_status
@property
def last_status(self):
return self.__last_status
@property
def log(self):
return self.__logger
@property
def log_text(self):
return self.__logger_stream
@property
def queued_for_download(self):
return self.__queued_for_download
@queued_for_download.setter
def queued_for_fownload(self,value):
self.__queued_for_download = value
def toggle_download(self):
self.__queued_for_download = not self.__queued_for_download
def configure_downloader(self):
if type(self.__credentials) is not tuple and len(self.__credentials) != 2:
raise ValueError('Credentials Not setup correctly')
self.__downloader = DM(self.__download_link,dest=self.__output_dir+"/"+self.__filename+self.__ext,
progress_bar=False,fix_urls=False,logger=self.__logger,hash_checksum=self.__md5_checksum,hash_algo='md5',username=self.__credentials[0],password=self.__credentials[1],threads=1)
self.__downloader.add_basic_authentication(self.__credentials[0],self.__credentials[1])
self.__downloader_ready = True
def download(self):
if self.__downloader_ready and self.__status is Status.Online:
try:
self.__downloader.start(blocking=False)
self.__last_status = self.__status
self.__status = Status.Downloading
except:
self.__last_status = self.__status
self.__status = Status.Failed
if self.__fail_retries > 1:
self.__fail_retries = self.__fail_retries - 1
self.__fail_retry_at = time.time() + self.__fail_retry_in
self.__logger.warning("Download failed - Retries left [{}] - Next retry in: {}".format(self.__fail_retries,utils.time_human(self.__fail_retry_in,fmt_short=True)))
self.__fail_retry_in *= self.__retry_in_multiplier
elif self.__faile_retries == 1:
self.__status = Status.Offline
self.__logger.warning("One attempt left. Set to offline")
elif self.__abandon:
self.__status = Status.Corrupted
self.__logger.error("Download failed - No retries left\nEnd Log\n{}".format("#"*20))
else:
self.__fail_retries = 3
self.__status = Status.Offline
else:
if not self.__downloader_ready:
self.__logger.warning("Could not start downlad because the download manager is not Ready")
elif self.__status is Status.Downloading:
self.__logger.warning("could not start download because download is already in progress")
def attempt_request(self):
self.__thread_rlock.acquire()
try:
self.__logger.info("Requesting")
r = requests.get(self.__download_link,auth=self.__credentials, verify=True,stream=True, timeout=10)
except Exception as e:
self.__logger.error("Could not attempt request:\n{}".format(str(e)))
self.__thread_rlock.release()
return
if r.status_code == 202:
self.__last_status = self.__status
self.__status = Status.Requested
self.__logger.info("Image has been requested")
elif r.status_code == 200:
self.__last_status = self.__status
self.__status = Status.Online
self.__logger.info("Image is Online - Queued for download")
elif r.status_code == 403:
self.__last_status = self.__status
self.__status = Status.Pending
self.__thread_rlock.release()
def check_online(self):
self.__thread_rlock.acquire()
try:
r = requests.get(self.__download_link.replace('$value','?$format=json'),auth=self.__credentials, verify=True,stream=True, timeout=10)
if r.status_code == 200 :
self.update_checksum_from_meta(r.json()['d'])
self.update_status_from_meta(r.json()['d'])
self.__thread_rlock.release()
except Exception as e:
self.__logger.info("Could not check Online availability:\n{}".format(str(e)))
self.__thread_rlock.release()
return
def auto_update_status(self):
if (self.__status < Status.Requested
and ((self.__request_thread and self.__request_thread.is_alive() is not True)
or self.__request_thread is None)):
self.__request_thread = threading.Thread(target=self.attempt_request)
self.__request_thread.daemon = True
self.__request_thread.start()
elif (self.__status == Status.Requested
and ((self.__recheck_thread and self.__recheck_thread.is_alive() is not True)
or self.__recheck_thread is None)):
self.__recheck_thread = threading.Thread(target=self.check_online)
self.__recheck_thread.daemon = True
self.__recheck_thread.start()
# elif self.__status == Status.Online:
# self.download()
# self.__status = Status.Downloading
elif self.__status == Status.Downloading:
if self.__downloader.isFinished():
if self.__downloader.isSuccessful():
self.__last_status = self.__status
if self.__do_checksum:
self.__status = Status.Hash
global _checksum_process_queue
init_checksum_process()
_checksum_process_queue.put_nowait(
(os.path.join(self.__output_dir,self.__filename+self.__ext),
self.__md5_checksum,
self.__uid)
)
else:
self.__status = Status.Succeeded
self.__logger.info("Download Finished\nEnd Log\n{}".format("#"*20))
else:
self.__last_status = self.__status
self.__status = Status.Failed
if self.__fail_retries > 0:
self.__fail_retries = self.__fail_retries - 1
self.__fail_retry_at = time.time() + self.__fail_retry_in
self.__logger.warning("Download failed - Retries left [{}] - Next retry in: {}".format(self.__fail_retries,utils.time_human(self.__fail_retry_in,fmt_short=True)))
self.__fail_retry_in *= self.__retry_in_multiplier
elif self.__abandon:
self.__status = Status.Corrupted
self.__logger.error("Download failed - No retries left\nEnd Log\n{}".format("#"*20))
else:
self.__fail_retries = 3
self.__status = Status.Failed
# elif self.__status == Status.Downloaded:
# self.__last_status = self.__status
# self.__status = Status.Hash
# self.__is_hash_correct.append(-2)
# self.verify_hash_threaded()
# elif self.__status == Status.Failed:
# if time.time() >= self.__fail_retry_at:
# self.__logger.info("Resetting Downloader and requeuing for download")
# self.configure_downloader()
# self.__last_status = self.__status
# self.__status = Status.Online
elif self.__status == Status.Hash:
global _checksum_process_results
if self.__uid in _checksum_process_results:
if _checksum_process_results[self.__uid] == 1:
self.__status = Status.Succeeded
self.__logger.info("Hash matches\nEnd Log\n{}".format("#"*20))
elif _checksum_process_results[self.__uid] == 0:
self.__status = Status.Corrupted
self.__logger.info("Hash doesn't match.\nDownloaded file is corrupted")
def update_checksum_from_meta(self,responseJson):
self.__md5_checksum = responseJson['Checksum']['Value'].lower()
def update_status_from_meta(self,responseJson):
if responseJson['Online']:
self.__last_status = self.__status
self.__status = Status.Online
def init_checksum_process():
try:
global _checksum_process,_checksum_process_manager,_checksum_process_queue,_checksum_process_results,_checksum_process_signaler
if not _checksum_process or not _checksum_process.is_alive():
_checksum_process_manager = Manager()
_checksum_process_queue = _checksum_process_manager.Queue()
_checksum_process_results = _checksum_process_manager.dict()
_checksum_process_signaler = _checksum_process_manager.Value('i',1)
_checksum_process = Process(target=checksum_verifier,
args=(
_checksum_process_queue,
_checksum_process_results,
_checksum_process_signaler,
),
name="P-checksum"
)
_checksum_process.start()
except Exception as e:
print(str(e))
def checksum_process_terminate():
global _checksum_process,_checksum_process,_checksum_process_signaler
if _checksum_process and _checksum_process.is_alive():
_checksum_process_signaler.value = 0
def is_queue_empty():
global _checksum_process_queue
return _checksum_process_queue.empty()
def checksum_verifier(queue,results,sig):
while True:
if sig.value == 0:
return
if not queue.empty():
try:
filename,checksum,uid = queue.get_nowait()
if(os.path.exists(filename)):
md5_hash = hashlib.md5()
with open(filename,"rb") as f:
# Read and update hash in chunks of 4K
for byte_block in iter(lambda: f.read(4096),b""):
md5_hash.update(byte_block)
if sig.value == 0:
return
results[uid] = int(md5_hash.hexdigest() == checksum.lower())
except Exception:
results[uid] = -1
time.sleep(5)
\ No newline at end of file
from .pySmartDL import SmartDL, HashFailedException, CanceledException
from . import utils
__version__ = pySmartDL.__version__
\ No newline at end of file
import threading
import time
from . import utils
class ControlThread(threading.Thread):
"A class that shows information about a running SmartDL object."
def __init__(self, obj):
threading.Thread.__init__(self)
self.obj = obj
self.progress_bar = obj.progress_bar
self.logger = obj.logger
self.shared_var = obj.shared_var
self.dl_speed = 0
self.eta = 0
self.lastBytesSamples = [] # list with last 50 Bytes Samples.
self.last_calculated_totalBytes = 0
self.calcETA_queue = []
self.calcETA_i = 0
self.calcETA_val = 0
self.dl_time = -1.0
self.daemon = True
self.start()
def run(self):
t1 = time.time()
self.logger.info("Control thread has been started.")
while not self.obj.pool.done():
self.dl_speed = self.calcDownloadSpeed(self.shared_var.value)
if self.dl_speed > 0:
self.eta = self.calcETA((self.obj.filesize-self.shared_var.value)/self.dl_speed)
if self.progress_bar:
if self.obj.filesize:
status = r"[*] %s / %s @ %s/s %s [%3.1f%%, %s left] " % (utils.sizeof_human(self.shared_var.value), utils.sizeof_human(self.obj.filesize), utils.sizeof_human(self.dl_speed), utils.progress_bar(1.0*self.shared_var.value/self.obj.filesize), self.shared_var.value * 100.0 / self.obj.filesize, utils.time_human(self.eta, fmt_short=True))
else:
status = r"[*] %s / ??? MB @ %s/s " % (utils.sizeof_human(self.shared_var.value), utils.sizeof_human(self.dl_speed))
status = status + chr(8)*(len(status)+1)
print(status, end=' ', flush=True)
time.sleep(0.1)
if self.obj._killed:
self.logger.info("File download process has been stopped.")
return
if self.progress_bar:
if self.obj.filesize:
print(r"[*] %s / %s @ %s/s %s [100%%, 0s left] " % (utils.sizeof_human(self.obj.filesize), utils.sizeof_human(self.obj.filesize), utils.sizeof_human(self.dl_speed), utils.progress_bar(1.0)))
else:
print(r"[*] %s / %s @ %s/s " % (utils.sizeof_human(self.shared_var.value), utils.sizeof_human(self.shared_var.value), utils.sizeof_human(self.dl_speed)))
t2 = time.time()
self.dl_time = float(t2-t1)
while self.obj.post_threadpool_thread.is_alive():
time.sleep(0.1)
self.obj.pool.shutdown()
self.obj.status = "finished"
if not self.obj.errors:
self.logger.info("File downloaded within %.2f seconds." % self.dl_time)
def get_eta(self):
if self.eta <= 0 or self.obj.status == 'paused':
return 0
return self.eta
def get_speed(self):
if self.obj.status == 'paused':
return 0
return self.dl_speed
def get_dl_size(self):
if self.shared_var.value > self.obj.filesize: