An error occurred while loading the file. Please try again.
-
Gaetano Raffaele authored44163041
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
#! /usr/bin/env python
import sys
import os
import getopt
import platform
import subprocess
import shutil
from mtdUtils import queuedProcess
import mtdUtils
def main(argv):
if platform.system() == 'Linux':
sh = False
shext = '.sh'
elif platform.system() == 'Windows':
sh = True
shext = '.bat'
else:
sys.exit("Platform not supported!")
cmd_path = os.path.dirname(__file__)
try:
opts, args = getopt.getopt(argv, 'w:o:d:f:t:p:s:r', ['srs='])
except getopt.GetoptError as err:
sys.exit(err)
shp_file = args[-1]
if not os.path.exists(shp_file):
sys.exit('%s does not exist'%shp_file)
output_folder = None
dates_file = None
date_beg = None
date_end = None
download_platform = None
pattern = None
platform_path = None
download_shp_file = None
srs = None
remove_src = False
for opt,val in opts:
if opt == '-w':
output_folder = str(val)
if not os.path.exists(output_folder):
os.mkdir(output_folder)
elif opt == '-o':
dates_file = str(val)
elif opt == '-d':
date_beg = str(val)
elif opt == '-f':
date_end = str(val)
elif opt == '-t' :
if val in ['S2PEPS','S2THEIA','VenusTHEIA']:
download_platform = str(val)
if val == 'S2PEPS':
pattern = 'S2'
elif val == 'S2THEIA':
pattern = 'S2_THEIA'
elif val == 'VenusTHEIA':
pattern = 'Venus_THEIA'
else :
sys.exit("%s is not a valide platform, please use one of the following values : \
['S2PEPS','S2THEIA','VenusTHEIA']"%val)
elif opt == '--srs':
srs = str(val)
elif opt == '-p':
platform_path = str(val)
elif opt == '-s':
download_shp_file = str(val)
elif opt == '-r':
remove_src = True
if dates_file != None :
if dates_file.isdigit() is True:
date_step = int(dates_file)
dates_file = os.path.join(output_folder,'output_dates.txt')
mtdUtils.periodic_date_generator(dates_file,date_beg,date_end,date_step)
else :
if os.path.exists(dates_file) is not True:
sys.exit("\nWARNING : -o argument must be an existant file of dates or an integer to generate the dates file\n")
if download_shp_file == None:
download_shp_file = shp_file
download_folder = os.path.join(output_folder, 'src')
if os.path.exists(download_folder) == True:
skip = None
while skip not in ['y','n'] :
print "%s already exists, do you want to skip download step ? (y/n)" % download_folder
skip = raw_input().lower()
if skip == 'n':
shutil.rmtree(download_folder)
if download_platform == 'S2PEPS':
cmd = ['python', cmd_path + 'getS2PEPSFromShp.py', '-w', download_folder, '-d', date_beg, '-f', date_end, download_shp_file, platform_path]
elif download_platform == 'S2THEIA':
cmd = ['python', cmd_path + 'getS2THEIAFromShp.py', '-w', download_folder, '-d', date_beg, '-f', date_end, download_shp_file, platform_path]
elif download_platform == 'VenusTHEIA':
cmd = ['python', cmd_path + 'getVenusFromShp.py', '-w', download_folder, '-d', date_beg, '-f', date_end, download_shp_file, platform_path]
subprocess.call(cmd, shell=sh)
else :
if download_platform == 'S2PEPS':
cmd = ['python', cmd_path + 'getS2PEPSFromShp.py', '-w', download_folder, '-d', date_beg, '-f', date_end, download_shp_file, platform_path]
elif download_platform == 'S2THEIA':
cmd = ['python', cmd_path + 'getS2THEIAFromShp.py', '-w', download_folder, '-d', date_beg, '-f', date_end, download_shp_file, platform_path]
elif download_platform == 'VenusTHEIA':
cmd = ['python', cmd_path + 'getVenusFromShp.py', '-w', download_folder, '-d', date_beg, '-f', date_end, download_shp_file, platform_path]
subprocess.call(cmd, shell=sh)
cmd = ['python', cmd_path + 'batchUnzip.py', '-r', download_folder]
subprocess.call(cmd, shell=sh)
preprocess_folder = os.path.join(output_folder, pattern)
if download_platform == 'S2PEPS':
print "Prepare S2PEPS files"
cmd = ['python', cmd_path + 'prepareS2PEPS.py', '-o', preprocess_folder, '-c', shp_file, download_folder]
subprocess.call(cmd, shell=sh)
cmd_list = []
with open(os.path.join(cmd_path,'S2PEPSPreparation' + shext)) as f :
for l in f:
cmd_list.append(l.replace('\n','').replace(' ',' ').replace('"','').split(' '))
queuedProcess(cmd_list, N_processes=6, shell=sh)
# cmd = ['bash', cmd_path + 'S2PEPSPreparation.sh']
# subprocess.call(cmd, shell=sh)
elif download_platform == 'S2THEIA':
print "Prepare S2THEIA files"
cmd = ['python', cmd_path + 'prepareS2THEIA.py', '-o', preprocess_folder, '-c', shp_file, download_folder]
subprocess.call(cmd, shell=sh)
cmd_list = []
with open(os.path.join(cmd_path,'S2THEIAPreparation' + shext)) as f :
for l in f:
cmd_list.append(l.replace('\n','').replace(' ',' ').split(' '))
queuedProcess(cmd_list, N_processes=6, shell=sh)
# cmd = ['bash', cmd_path + 'S2THEIAPreparation.sh']
# subprocess.call(cmd, shell=sh)
elif download_platform == 'VenusTHEIA':
print "Prepare VenusTHEIA files"
cmd = ['python', cmd_path + 'prepareVENUS.py', '-o', preprocess_folder, '-c', shp_file, download_folder]
subprocess.call(cmd, shell=sh)
cmd_list = []
with open(os.path.join(cmd_path,'VENUSPreparation' + shext)) as f :
for l in f:
cmd_list.append(l.replace('\n','').replace(' ',' ').split(' '))
queuedProcess(cmd_list, N_processes=6, shell=sh)
# cmd = ['bash', cmd_path + 'VENUSPreparation.sh']
# subprocess.call(cmd, shell=sh)
if remove_src == True :
shutil.rmtree(download_folder)
print "Preprocess %s files" % download_platform
cmd = ['python', cmd_path + 'genProcessScript.py', '--cloudmask', preprocess_folder]
subprocess.call(cmd, shell=sh)
cmd_list = []
with open(os.path.join(cmd_path,'ProcessScript' + shext)) as f :
for l in f:
cmd_list.append(l.replace('\n','').replace(' ',' ').split(' '))
queuedProcess(cmd_list, N_processes=6, shell=sh)
# cmd = ['bash', cmd_path + 'ProcessScript.sh']
# subprocess.call(cmd, shell=sh)
print "Gapfilling %s files" % download_platform
gapf_folder = os.path.join(output_folder, pattern + '_GAPF')
cmd = ['python', 'cloudfreeComposites.py', '-o', gapf_folder]
if dates_file != None :
cmd += ['-d', dates_file]
cmd += [preprocess_folder]
subprocess.call(cmd, shell=sh)
if __name__ == '__main__':
if len(sys.argv) < 3:
sys.exit(
'''Usage: python TimeSeries_allpreprocess.py [-w <output folder>] [-o <gapfilled dates file>]
[-d <begin-date YYYY-MM-DD>] [-f <end-date YYYY-MM-DD>] [--srs <EPSG:4326>]
[-t <data type : S2PEPS / S2THEIA / VenusTHEIA>] [-p <platform installation path>]
[-s <shapefile reference for download>] [-r <remove src folder>] <shp-file> ''')
else:
main(sys.argv[1:])