#! /usr/bin/env python import sys import os import getopt import platform import subprocess import shutil from mtdUtils import queuedProcess import mtdUtils def main(argv): if platform.system() == 'Linux': sh = False shext = '.sh' elif platform.system() == 'Windows': sh = True shext = '.bat' else: sys.exit("Platform not supported!") cmd_path = os.path.dirname(__file__) try: opts, args = getopt.getopt(argv, 'w:o:d:f:t:p:s:r', ['srs=']) except getopt.GetoptError as err: sys.exit(err) shp_file = args[-1] if not os.path.exists(shp_file): sys.exit('%s does not exist'%shp_file) output_folder = None dates_file = None date_beg = None date_end = None download_platform = None pattern = None platform_path = None download_shp_file = None srs = None remove_src = False for opt,val in opts: if opt == '-w': output_folder = str(val) if not os.path.exists(output_folder): os.mkdir(output_folder) elif opt == '-o': dates_file = str(val) elif opt == '-d': date_beg = str(val) elif opt == '-f': date_end = str(val) elif opt == '-t' : if val in ['S2PEPS','S2THEIA','VenusTHEIA']: download_platform = str(val) if val == 'S2PEPS': pattern = 'S2' elif val == 'S2THEIA': pattern = 'S2_THEIA' elif val == 'VenusTHEIA': pattern = 'Venus_THEIA' else : sys.exit("%s is not a valide platform, please use one of the following values : \ ['S2PEPS','S2THEIA','VenusTHEIA']"%val) elif opt == '--srs': srs = str(val) elif opt == '-p': platform_path = str(val) elif opt == '-s': download_shp_file = str(val) elif opt == '-r': remove_src = True if dates_file != None : if dates_file.isdigit() is True: date_step = int(dates_file) dates_file = os.path.join(output_folder,'output_dates.txt') mtdUtils.periodic_date_generator(dates_file,date_beg,date_end,date_step) else : if os.path.exists(dates_file) is not True: sys.exit("\nWARNING : -o argument must be an existant file of dates or an integer to generate the dates file\n") if download_shp_file == None: download_shp_file = shp_file download_folder = os.path.join(output_folder, 'src') if os.path.exists(download_folder) == True: skip = None while skip not in ['y','n'] : print "%s already exists, do you want to skip download step ? (y/n)" % download_folder skip = raw_input().lower() if skip == 'n': shutil.rmtree(download_folder) if download_platform == 'S2PEPS': cmd = ['python', cmd_path + 'getS2PEPSFromShp.py', '-w', download_folder, '-d', date_beg, '-f', date_end, download_shp_file, platform_path] elif download_platform == 'S2THEIA': cmd = ['python', cmd_path + 'getS2THEIAFromShp.py', '-w', download_folder, '-d', date_beg, '-f', date_end, download_shp_file, platform_path] elif download_platform == 'VenusTHEIA': cmd = ['python', cmd_path + 'getVenusFromShp.py', '-w', download_folder, '-d', date_beg, '-f', date_end, download_shp_file, platform_path] subprocess.call(cmd, shell=sh) else : if download_platform == 'S2PEPS': cmd = ['python', cmd_path + 'getS2PEPSFromShp.py', '-w', download_folder, '-d', date_beg, '-f', date_end, download_shp_file, platform_path] elif download_platform == 'S2THEIA': cmd = ['python', cmd_path + 'getS2THEIAFromShp.py', '-w', download_folder, '-d', date_beg, '-f', date_end, download_shp_file, platform_path] elif download_platform == 'VenusTHEIA': cmd = ['python', cmd_path + 'getVenusFromShp.py', '-w', download_folder, '-d', date_beg, '-f', date_end, download_shp_file, platform_path] subprocess.call(cmd, shell=sh) cmd = ['python', cmd_path + 'batchUnzip.py', '-r', download_folder] subprocess.call(cmd, shell=sh) preprocess_folder = os.path.join(output_folder, pattern) if download_platform == 'S2PEPS': print "Prepare S2PEPS files" cmd = ['python', cmd_path + 'prepareS2PEPS.py', '-o', preprocess_folder, '-c', shp_file, download_folder] subprocess.call(cmd, shell=sh) cmd_list = [] with open(os.path.join(cmd_path,'S2PEPSPreparation' + shext)) as f : for l in f: cmd_list.append(l.replace('\n','').replace(' ',' ').replace('"','').split(' ')) queuedProcess(cmd_list, N_processes=6, shell=sh) # cmd = ['bash', cmd_path + 'S2PEPSPreparation.sh'] # subprocess.call(cmd, shell=sh) elif download_platform == 'S2THEIA': print "Prepare S2THEIA files" cmd = ['python', cmd_path + 'prepareS2THEIA.py', '-o', preprocess_folder, '-c', shp_file, download_folder] subprocess.call(cmd, shell=sh) cmd_list = [] with open(os.path.join(cmd_path,'S2THEIAPreparation' + shext)) as f : for l in f: cmd_list.append(l.replace('\n','').replace(' ',' ').split(' ')) queuedProcess(cmd_list, N_processes=6, shell=sh) # cmd = ['bash', cmd_path + 'S2THEIAPreparation.sh'] # subprocess.call(cmd, shell=sh) elif download_platform == 'VenusTHEIA': print "Prepare VenusTHEIA files" cmd = ['python', cmd_path + 'prepareVENUS.py', '-o', preprocess_folder, '-c', shp_file, download_folder] subprocess.call(cmd, shell=sh) cmd_list = [] with open(os.path.join(cmd_path,'VENUSPreparation' + shext)) as f : for l in f: cmd_list.append(l.replace('\n','').replace(' ',' ').split(' ')) queuedProcess(cmd_list, N_processes=6, shell=sh) # cmd = ['bash', cmd_path + 'VENUSPreparation.sh'] # subprocess.call(cmd, shell=sh) if remove_src == True : shutil.rmtree(download_folder) print "Preprocess %s files" % download_platform cmd = ['python', cmd_path + 'genProcessScript.py', '--cloudmask', preprocess_folder] subprocess.call(cmd, shell=sh) cmd_list = [] with open(os.path.join(cmd_path,'ProcessScript' + shext)) as f : for l in f: cmd_list.append(l.replace('\n','').replace(' ',' ').split(' ')) queuedProcess(cmd_list, N_processes=6, shell=sh) # cmd = ['bash', cmd_path + 'ProcessScript.sh'] # subprocess.call(cmd, shell=sh) print "Gapfilling %s files" % download_platform gapf_folder = os.path.join(output_folder, pattern + '_GAPF') cmd = ['python', 'cloudfreeComposites.py', '-o', gapf_folder] if dates_file != None : cmd += ['-d', dates_file] cmd += [preprocess_folder] subprocess.call(cmd, shell=sh) if __name__ == '__main__': if len(sys.argv) < 3: sys.exit( '''Usage: python TimeSeries_allpreprocess.py [-w <output folder>] [-o <gapfilled dates file>] [-d <begin-date YYYY-MM-DD>] [-f <end-date YYYY-MM-DD>] [--srs <EPSG:4326>] [-t <data type : S2PEPS / S2THEIA / VenusTHEIA>] [-p <platform installation path>] [-s <shapefile reference for download>] [-r <remove src folder>] <shp-file> ''') else: main(sys.argv[1:])