Commit f0842134 authored by eudesyawog's avatar eudesyawog

new splits and minor changes

parent 5de0c392
...@@ -2,20 +2,22 @@ import pandas as pd ...@@ -2,20 +2,22 @@ import pandas as pd
import numpy as np import numpy as np
import os import os
def normalize_time_series (radar_file, opt_file, norm="meanstd", radar_bands=["VH","VV"], def normalize_time_series (radar_file, opt_file, norm="minmax", radar_bands=["VH","VV"],
opt_bands=["B2","B3","B4","B8","B5","B6","B7","B8A","B11","B12"], opt_bands=["B2","B3","B4","B8","B5","B6","B7","B8A","B11","B12"],
opt_indices = ["NDVI","NDWI","EVI","MSAVI2","GDVI","CIGreen","CIRedEdge"]): opt_indices = ["NDVI","NDWI","EVI","MSAVI2","GDVI","CIGreen","CIRedEdge"]):
radar_df = pd.read_csv(radar_file) radar_df = pd.read_csv(radar_file)
radar_array = None radar_array = None
for band in radar_bands : for band in radar_bands :
columns = [col for col in radar_df.columns if col.startswith(band) and col.endswith("Mean")] columns = [col for col in radar_df.columns if col.startswith(band)]
columns.sort() columns.sort()
ts_values = radar_df[columns].values ts_values = radar_df[columns].values
if norm == "meanstd": if norm == "meanstd":
ts_values = (ts_values - ts_values.mean()) / ts_values.std() ts_values = (ts_values - ts_values.mean()) / ts_values.std()
elif norm == "minmax": elif norm == "minmax":
ts_values = (ts_values - ts_values.min()) / (ts_values.max() - ts_values.min()) ts_values = (ts_values - ts_values.min()) / (ts_values.max() - ts_values.min())
if radar_array is None : if radar_array is None :
radar_array = ts_values radar_array = ts_values
else : else :
...@@ -38,7 +40,7 @@ def normalize_time_series (radar_file, opt_file, norm="meanstd", radar_bands=["V ...@@ -38,7 +40,7 @@ def normalize_time_series (radar_file, opt_file, norm="meanstd", radar_bands=["V
opt_df = pd.read_csv(opt_file) opt_df = pd.read_csv(opt_file)
opt_array = None opt_array = None
for band in opt_bands : for band in opt_bands :
columns = [col for col in opt_df.columns if col.split("_")[0]==band and col.endswith("Mean")] columns = [col for col in opt_df.columns if col.split("_")[0]==band]
columns.sort() columns.sort()
ts_values = opt_df[columns].values ts_values = opt_df[columns].values
if norm == "meanstd": if norm == "meanstd":
...@@ -67,7 +69,7 @@ def normalize_time_series (radar_file, opt_file, norm="meanstd", radar_bands=["V ...@@ -67,7 +69,7 @@ def normalize_time_series (radar_file, opt_file, norm="meanstd", radar_bands=["V
indices_df = pd.read_csv(opt_file) indices_df = pd.read_csv(opt_file)
indices_array = None indices_array = None
for band in opt_indices : for band in opt_indices :
columns = [col for col in indices_df.columns if col.startswith(band) and col.endswith("Mean")] columns = [col for col in indices_df.columns if col.startswith(band)]
columns.sort() columns.sort()
ts_values = indices_df[columns].values ts_values = indices_df[columns].values
if norm == "meanstd": if norm == "meanstd":
...@@ -93,6 +95,8 @@ def normalize_time_series (radar_file, opt_file, norm="meanstd", radar_bands=["V ...@@ -93,6 +95,8 @@ def normalize_time_series (radar_file, opt_file, norm="meanstd", radar_bands=["V
else : else :
indices_seq = np.hstack((indices_seq,np.stack(lst, axis=1))) indices_seq = np.hstack((indices_seq,np.stack(lst, axis=1)))
if not os.path.exists("./data"):
os.makedirs("./data")
ptrn = os.path.basename(radar_file).split('_')[0]+"_"+os.path.basename(radar_file).split('_')[1] ptrn = os.path.basename(radar_file).split('_')[0]+"_"+os.path.basename(radar_file).split('_')[1]
np.save("./data/{}_rad_seq.npy".format(ptrn),radar_seq) np.save("./data/{}_rad_seq.npy".format(ptrn),radar_seq)
np.save("./data/{}_opt_seq.npy".format(ptrn),opt_seq) np.save("./data/{}_opt_seq.npy".format(ptrn),opt_seq)
...@@ -100,7 +104,7 @@ def normalize_time_series (radar_file, opt_file, norm="meanstd", radar_bands=["V ...@@ -100,7 +104,7 @@ def normalize_time_series (radar_file, opt_file, norm="meanstd", radar_bands=["V
try: try:
rdt_df = radar_df.merge(opt_df[["ID"]],on="ID") rdt_df = radar_df.merge(opt_df[["ID"]],on="ID")
rdt = rdt_df[["Rdt_s"]].values rdt = rdt_df[["Biom_f","Biom_s","Rdt_f","Rdt_s"]].values
np.save("./data/{}_yields.npy".format(ptrn),rdt) np.save("./data/{}_yields.npy".format(ptrn),rdt)
except Exception as error : except Exception as error :
print (error) print (error)
...@@ -110,16 +114,16 @@ def normalize_time_series (radar_file, opt_file, norm="meanstd", radar_bands=["V ...@@ -110,16 +114,16 @@ def normalize_time_series (radar_file, opt_file, norm="meanstd", radar_bands=["V
if __name__ == '__main__' : if __name__ == '__main__' :
# Niakhar 2017 # Niakhar 2017
radar_file = "./data/niakhar_2017_radar_notree.csv" radar_file = "./interpolate/niakhar_2017_radar_notree_interpolate.csv"
opt_file = "./data/niakhar_2017_opt_gapf_notree.csv" opt_file = "./interpolate/niakhar_2017_opt_gapf_notree_interpolate.csv"
normalize_time_series(radar_file,opt_file,norm="meanstd") normalize_time_series(radar_file,opt_file,norm="minmax")
# Niakhar 2018 # Niakhar 2018
radar_file = "./data/niakhar_2018_radar_notree.csv" radar_file = "./interpolate/niakhar_2018_radar_notree_interpolate.csv"
opt_file = "./data/niakhar_2018_opt_gapf_notree.csv" opt_file = "./interpolate/niakhar_2018_opt_gapf_notree_interpolate.csv"
normalize_time_series(radar_file,opt_file,norm="meanstd") normalize_time_series(radar_file,opt_file,norm="minmax")
# Nioro 2018 # Nioro 2018
radar_file = "./data/nioro_2018_radar_notree.csv" radar_file = "./interpolate/nioro_2018_radar_notree_interpolate.csv"
opt_file = "./data/nioro_2018_opt_gapf_notree.csv" opt_file = "./interpolate/nioro_2018_opt_gapf_notree_interpolate.csv"
normalize_time_series(radar_file,opt_file,norm="meanstd") normalize_time_series(radar_file,opt_file,norm="minmax")
\ No newline at end of file \ No newline at end of file
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.