new splits and minor changes

norm_data.py

@@ -2,20 +2,22 @@ import pandas as pd
 import numpy as np
 import os
 
-def normalize_time_series (radar_file, opt_file, norm="meanstd", radar_bands=["VH","VV"], 
+def normalize_time_series (radar_file, opt_file, norm="minmax", radar_bands=["VH","VV"], 
                             opt_bands=["B2","B3","B4","B8","B5","B6","B7","B8A","B11","B12"], 
                             opt_indices = ["NDVI","NDWI","EVI","MSAVI2","GDVI","CIGreen","CIRedEdge"]):
     
     radar_df = pd.read_csv(radar_file)
     radar_array = None
     for band in radar_bands :
-        columns = [col for col in radar_df.columns if col.startswith(band) and col.endswith("Mean")]
+        columns = [col for col in radar_df.columns if col.startswith(band)]
         columns.sort()
+        
         ts_values = radar_df[columns].values
         if norm == "meanstd":
             ts_values = (ts_values - ts_values.mean()) / ts_values.std()
         elif norm == "minmax":
             ts_values = (ts_values - ts_values.min()) / (ts_values.max() - ts_values.min())
+            
         if radar_array is None :
             radar_array = ts_values
         else :
@@ -38,7 +40,7 @@ def normalize_time_series (radar_file, opt_file, norm="meanstd", radar_bands=["V
     opt_df = pd.read_csv(opt_file)
     opt_array = None
     for band in opt_bands :
-        columns = [col for col in opt_df.columns if col.split("_")[0]==band and col.endswith("Mean")]
+        columns = [col for col in opt_df.columns if col.split("_")[0]==band]
         columns.sort()
         ts_values = opt_df[columns].values
         if norm == "meanstd":
@@ -67,7 +69,7 @@ def normalize_time_series (radar_file, opt_file, norm="meanstd", radar_bands=["V
     indices_df = pd.read_csv(opt_file)
     indices_array = None
     for band in opt_indices :
-        columns = [col for col in indices_df.columns if col.startswith(band) and col.endswith("Mean")]
+        columns = [col for col in indices_df.columns if col.startswith(band)]
         columns.sort()
         ts_values = indices_df[columns].values
         if norm == "meanstd":
@@ -93,6 +95,8 @@ def normalize_time_series (radar_file, opt_file, norm="meanstd", radar_bands=["V
         else :
             indices_seq = np.hstack((indices_seq,np.stack(lst, axis=1)))
     
+    if not os.path.exists("./data"):
+        os.makedirs("./data")
     ptrn = os.path.basename(radar_file).split('_')[0]+"_"+os.path.basename(radar_file).split('_')[1]
     np.save("./data/{}_rad_seq.npy".format(ptrn),radar_seq)
     np.save("./data/{}_opt_seq.npy".format(ptrn),opt_seq)
@@ -100,7 +104,7 @@ def normalize_time_series (radar_file, opt_file, norm="meanstd", radar_bands=["V
 
     try:
         rdt_df = radar_df.merge(opt_df[["ID"]],on="ID")
-        rdt = rdt_df[["Rdt_s"]].values
+        rdt = rdt_df[["Biom_f","Biom_s","Rdt_f","Rdt_s"]].values
         np.save("./data/{}_yields.npy".format(ptrn),rdt)
     except Exception as error : 
         print (error)
@@ -110,16 +114,16 @@ def normalize_time_series (radar_file, opt_file, norm="meanstd", radar_bands=["V
 if __name__ == '__main__' :
 
     # Niakhar 2017
-    radar_file = "./data/niakhar_2017_radar_notree.csv"
-    opt_file = "./data/niakhar_2017_opt_gapf_notree.csv"
-    normalize_time_series(radar_file,opt_file,norm="meanstd")
+    radar_file = "./interpolate/niakhar_2017_radar_notree_interpolate.csv"
+    opt_file = "./interpolate/niakhar_2017_opt_gapf_notree_interpolate.csv"
+    normalize_time_series(radar_file,opt_file,norm="minmax")
 
     # Niakhar 2018
-    radar_file = "./data/niakhar_2018_radar_notree.csv"
-    opt_file = "./data/niakhar_2018_opt_gapf_notree.csv"
-    normalize_time_series(radar_file,opt_file,norm="meanstd")
+    radar_file = "./interpolate/niakhar_2018_radar_notree_interpolate.csv"
+    opt_file = "./interpolate/niakhar_2018_opt_gapf_notree_interpolate.csv"
+    normalize_time_series(radar_file,opt_file,norm="minmax")
 
     # Nioro 2018
-    radar_file = "./data/nioro_2018_radar_notree.csv"
-    opt_file = "./data/nioro_2018_opt_gapf_notree.csv"
-    normalize_time_series(radar_file,opt_file,norm="meanstd")
\ No newline at end of file
+    radar_file = "./interpolate/nioro_2018_radar_notree_interpolate.csv"
+    opt_file = "./interpolate/nioro_2018_opt_gapf_notree_interpolate.csv"
+    normalize_time_series(radar_file,opt_file,norm="minmax")
\ No newline at end of file