TimeSeries_allpreprocess.py 7.41 KiB
#! /usr/bin/env python
import sys
import os
import getopt
import platform
import subprocess
import shutil
from mtdUtils import queuedProcess
import mtdUtils

def main(argv):
    if platform.system() == 'Linux':
        sh = False
        shext = '.sh'
    elif platform.system() == 'Windows':
        sh = True
        shext = '.bat'
    else:
        sys.exit("Platform not supported!")

    cmd_path = os.path.dirname(__file__)

    try:
        opts, args = getopt.getopt(argv, 'w:o:d:f:t:p:s:r', ['srs='])
    except getopt.GetoptError as err:
        sys.exit(err)

    shp_file = args[-1]
    if not os.path.exists(shp_file):
        sys.exit('%s does not exist'%shp_file)

    output_folder = None
    dates_file = None
    date_beg = None
    date_end = None
    download_platform = None
    pattern = None
    platform_path = None
    download_shp_file = None
    srs = None
    remove_src = False
    for opt,val in opts:
        if opt == '-w':
            output_folder = str(val)
            if not os.path.exists(output_folder):
                os.mkdir(output_folder)
        elif opt == '-o':
        	dates_file = str(val)
        elif opt == '-d':
            date_beg = str(val)
        elif opt == '-f':
            date_end = str(val)
        elif opt == '-t' :
            if val in ['S2PEPS','S2THEIA','VenusTHEIA']:
                download_platform = str(val)
                if val == 'S2PEPS':
                	pattern = 'S2'
                elif val == 'S2THEIA':
                	pattern = 'S2_THEIA'
                elif val == 'VenusTHEIA':
                    pattern = 'Venus_THEIA'
            else :
                sys.exit("%s is not a valide platform, please use one of the following values : \
                        ['S2PEPS','S2THEIA','VenusTHEIA']"%val)
        elif opt == '--srs':
            srs = str(val)
        elif opt == '-p':
            platform_path = str(val)
        elif opt == '-s':
            download_shp_file = str(val)
        elif opt == '-r':
            remove_src = True

    if dates_file != None :
        if dates_file.isdigit() is True:
            date_step = int(dates_file)
            dates_file = os.path.join(output_folder,'output_dates.txt')
            mtdUtils.periodic_date_generator(dates_file,date_beg,date_end,date_step)
        else :
            if os.path.exists(dates_file) is not True:
                sys.exit("\nWARNING : -o argument must be an existant file of dates or an integer to generate the dates file\n")

    if download_shp_file  == None:
        download_shp_file = shp_file
    download_folder = os.path.join(output_folder, 'src')
    if os.path.exists(download_folder) == True:
        skip = None
        while skip not in ['y','n'] :
            print "%s already exists, do you want to skip download step ? (y/n)" % download_folder
            skip = raw_input().lower()
        if skip == 'n':
            shutil.rmtree(download_folder)
            if download_platform == 'S2PEPS':
        	    cmd = ['python', cmd_path + 'getS2PEPSFromShp.py', '-w', download_folder, '-d', date_beg, '-f', date_end, download_shp_file, platform_path]
            elif download_platform == 'S2THEIA':
                cmd = ['python', cmd_path + 'getS2THEIAFromShp.py', '-w', download_folder, '-d', date_beg, '-f', date_end, download_shp_file, platform_path]
            elif download_platform == 'VenusTHEIA':
        	    cmd = ['python', cmd_path + 'getVenusFromShp.py', '-w', download_folder, '-d', date_beg, '-f', date_end, download_shp_file, platform_path]
            subprocess.call(cmd, shell=sh)
    else :
        if download_platform == 'S2PEPS':
            cmd = ['python', cmd_path + 'getS2PEPSFromShp.py', '-w', download_folder, '-d', date_beg, '-f', date_end, download_shp_file, platform_path]
        elif download_platform == 'S2THEIA':
            cmd = ['python', cmd_path + 'getS2THEIAFromShp.py', '-w', download_folder, '-d', date_beg, '-f', date_end, download_shp_file, platform_path]
        elif download_platform == 'VenusTHEIA':
            cmd = ['python', cmd_path + 'getVenusFromShp.py', '-w', download_folder, '-d', date_beg, '-f', date_end, download_shp_file, platform_path]
        subprocess.call(cmd, shell=sh)
    
    cmd = ['python', cmd_path + 'batchUnzip.py', '-r', download_folder]
    subprocess.call(cmd, shell=sh)

    preprocess_folder = os.path.join(output_folder, pattern)
    if download_platform == 'S2PEPS':
        print "Prepare S2PEPS files"
        cmd = ['python', cmd_path + 'prepareS2PEPS.py', '-o', preprocess_folder, '-c', shp_file, download_folder]
        subprocess.call(cmd, shell=sh)
        cmd_list = []
        with open(os.path.join(cmd_path,'S2PEPSPreparation' + shext)) as f :
            for l in f:
                cmd_list.append(l.replace('\n','').replace('  ',' ').replace('"','').split(' '))
        queuedProcess(cmd_list, N_processes=6, shell=sh)
        # cmd = ['bash', cmd_path + 'S2PEPSPreparation.sh']
        # subprocess.call(cmd, shell=sh)
    elif download_platform == 'S2THEIA':
        print "Prepare S2THEIA files"
        cmd = ['python', cmd_path + 'prepareS2THEIA.py', '-o', preprocess_folder, '-c', shp_file, download_folder]
        subprocess.call(cmd, shell=sh)
        cmd_list = []
        with open(os.path.join(cmd_path,'S2THEIAPreparation' + shext)) as f :
            for l in f:
                cmd_list.append(l.replace('\n','').replace('  ',' ').split(' '))
        queuedProcess(cmd_list, N_processes=6, shell=sh)
        # cmd = ['bash', cmd_path + 'S2THEIAPreparation.sh']
        # subprocess.call(cmd, shell=sh)
    elif download_platform == 'VenusTHEIA':
        print "Prepare VenusTHEIA files"
        cmd = ['python', cmd_path + 'prepareVENUS.py', '-o', preprocess_folder, '-c', shp_file, download_folder]
        subprocess.call(cmd, shell=sh)
        cmd_list = []
        with open(os.path.join(cmd_path,'VENUSPreparation' + shext)) as f :
            for l in f:
                cmd_list.append(l.replace('\n','').replace('  ',' ').split(' '))
        queuedProcess(cmd_list, N_processes=6, shell=sh)
        # cmd = ['bash', cmd_path + 'VENUSPreparation.sh']
        # subprocess.call(cmd, shell=sh)

    if remove_src == True :
        shutil.rmtree(download_folder)
    print "Preprocess %s files" % download_platform
    cmd = ['python', cmd_path + 'genProcessScript.py', '--cloudmask', preprocess_folder]
    subprocess.call(cmd, shell=sh)
    cmd_list = []
    with open(os.path.join(cmd_path,'ProcessScript' + shext)) as f :
        for l in f:
            cmd_list.append(l.replace('\n','').replace('  ',' ').split(' '))
    queuedProcess(cmd_list, N_processes=6, shell=sh)
    # cmd = ['bash', cmd_path + 'ProcessScript.sh']
    # subprocess.call(cmd, shell=sh)

    print "Gapfilling %s files" % download_platform
    gapf_folder = os.path.join(output_folder, pattern + '_GAPF')
    cmd = ['python', 'cloudfreeComposites.py', '-o', gapf_folder]
    if dates_file != None :
    	cmd += ['-d', dates_file]
    cmd += [preprocess_folder]
    subprocess.call(cmd, shell=sh)

if __name__ == '__main__':
    if len(sys.argv) < 3:
        sys.exit(
            '''Usage: python TimeSeries_allpreprocess.py [-w <output folder>] [-o <gapfilled dates file>]
            [-d <begin-date YYYY-MM-DD>] [-f <end-date YYYY-MM-DD>] [--srs <EPSG:4326>]
            [-t <data type : S2PEPS / S2THEIA / VenusTHEIA>] [-p <platform installation path>] 
            [-s <shapefile reference for download>] [-r <remove src folder>] <shp-file> ''')
    else:
        main(sys.argv[1:])