...
 
Commits (2)
import os
import numpy as np
import pandas as pd
import scipy as sp
from scipy import sparse, linalg
import seaborn as sns
import matplotlib.pyplot as plt
sns.set()
def smooth_plot (csv,lstBands):
ptrn = os.path.basename(csv)
year = ptrn.split('_')[1]
outFolder = "./interpolate"
if not os.path.exists(outFolder):
os.makedirs(outFolder)
df = pd.read_csv(csv)
outdf = None
ggplot_dic = {}
for band in lstBands:
cols = [col for col in df.columns if col.startswith(band) and col.endswith("Mean")]
cols.sort()
times = pd.date_range(start="{}0501".format(year), end = "{}1031".format(year), freq='D')
dic = {}
for index, row in df.iterrows():
lstValues = []
for j in range(len(times)): # dates
date = str(times.strftime('{}_%Y%m%d_Mean'.format(band))[j])
if date in cols:
lstValues.append(row[date])
else :
lstValues.append(np.nan)
upsampled = pd.Series(lstValues,index=times)
interpolated = upsampled.interpolate(method='linear',limit_direction='both')
resampled = interpolated.resample('5D').mean()
dic.setdefault('feat_index',[]).append(index)
if band == lstBands[0] :
dic.setdefault('ID',[]).append(row['ID'])
dic.setdefault('Biom_f',[]).append(row['Biom_f'])
dic.setdefault('Biom_s',[]).append(row['Biom_s'])
dic.setdefault('Rdt_f',[]).append(row['Rdt_f'])
dic.setdefault('Rdt_s',[]).append(row['Rdt_s'])
for idx , value in resampled.items():
dic.setdefault('{}_{}'.format(band,idx.strftime("%Y%m%d")),[]).append(value)
ggplot_dic.setdefault('Date',[]).append(idx.strftime("%Y-%m-%d"))
ggplot_dic.setdefault('Band',[]).append(band)
ggplot_dic.setdefault('Plot',[]).append(row['ID'])
# ggplot_dic.setdefault('Crop',[]).append(row['Crop'])
ggplot_dic.setdefault('Sowing',[]).append(row['Sowing'])
ggplot_dic.setdefault('Harvest',[]).append(row['Harvest'])
ggplot_dic.setdefault('Value',[]).append(value)
if outdf is None :
outdf = pd.DataFrame.from_dict(dic)
else:
outdf = outdf.merge(pd.DataFrame.from_dict(dic),on='feat_index',how='inner')
outCSV = os.path.join(outFolder,'{}'.format(ptrn.replace(".csv","_interpolate.csv")))
outdf = outdf.drop(columns='feat_index')
outdf.to_csv(outCSV,index=False)
# To ggplot
plotCSV = os.path.join(outFolder,'{}'.format(ptrn.replace(".csv","_interpolate_toggplot.csv")))
ggplot_df = pd.DataFrame.from_dict(ggplot_dic)
ggplot_df = ggplot_df.loc[ggplot_df.Plot.isin(outdf['ID'])]
ggplot_df.to_csv(plotCSV,index=False)
if __name__=='__main__':
csv = "./stats/niakhar_2017_opt_gapf_notree.csv"
smooth_plot(csv,["B2","B3","B4","B8","B5","B6","B7","B8A","B11","B12","NDVI","NDWI","EVI","MSAVI2","GDVI","CIGreen","CIRedEdge"])
csv = "./stats/niakhar_2017_radar_notree.csv"
smooth_plot(csv,["VH","VV"])
csv = "./stats/niakhar_2018_opt_gapf_notree.csv"
smooth_plot(csv,["B2","B3","B4","B8","B5","B6","B7","B8A","B11","B12","NDVI","NDWI","EVI","MSAVI2","GDVI","CIGreen","CIRedEdge"])
csv = "./stats/niakhar_2018_radar_notree.csv"
smooth_plot(csv,["VH","VV"])
csv = "./stats/nioro_2018_opt_gapf_notree.csv"
smooth_plot(csv,["B2","B3","B4","B8","B5","B6","B7","B8A","B11","B12","NDVI","NDWI","EVI","MSAVI2","GDVI","CIGreen","CIRedEdge"])
csv = "./stats/nioro_2018_radar_notree.csv"
smooth_plot(csv,["VH","VV"])
\ No newline at end of file
This source diff could not be displayed because it is too large. You can view the blob instead.
This diff is collapsed.
This source diff could not be displayed because it is too large. You can view the blob instead.
This diff is collapsed.
This diff is collapsed.
This source diff could not be displayed because it is too large. You can view the blob instead.
This diff is collapsed.
This diff is collapsed.
This source diff could not be displayed because it is too large. You can view the blob instead.
......@@ -2,20 +2,22 @@ import pandas as pd
import numpy as np
import os
def normalize_time_series (radar_file, opt_file, norm="meanstd", radar_bands=["VH","VV"],
def normalize_time_series (radar_file, opt_file, norm="minmax", radar_bands=["VH","VV"],
opt_bands=["B2","B3","B4","B8","B5","B6","B7","B8A","B11","B12"],
opt_indices = ["NDVI","NDWI","EVI","MSAVI2","GDVI","CIGreen","CIRedEdge"]):
radar_df = pd.read_csv(radar_file)
radar_array = None
for band in radar_bands :
columns = [col for col in radar_df.columns if col.startswith(band) and col.endswith("Mean")]
columns = [col for col in radar_df.columns if col.startswith(band)]
columns.sort()
ts_values = radar_df[columns].values
if norm == "meanstd":
ts_values = (ts_values - ts_values.mean()) / ts_values.std()
elif norm == "minmax":
ts_values = (ts_values - ts_values.min()) / (ts_values.max() - ts_values.min())
if radar_array is None :
radar_array = ts_values
else :
......@@ -38,7 +40,7 @@ def normalize_time_series (radar_file, opt_file, norm="meanstd", radar_bands=["V
opt_df = pd.read_csv(opt_file)
opt_array = None
for band in opt_bands :
columns = [col for col in opt_df.columns if col.split("_")[0]==band and col.endswith("Mean")]
columns = [col for col in opt_df.columns if col.split("_")[0]==band]
columns.sort()
ts_values = opt_df[columns].values
if norm == "meanstd":
......@@ -67,7 +69,7 @@ def normalize_time_series (radar_file, opt_file, norm="meanstd", radar_bands=["V
indices_df = pd.read_csv(opt_file)
indices_array = None
for band in opt_indices :
columns = [col for col in indices_df.columns if col.startswith(band) and col.endswith("Mean")]
columns = [col for col in indices_df.columns if col.startswith(band)]
columns.sort()
ts_values = indices_df[columns].values
if norm == "meanstd":
......@@ -93,6 +95,8 @@ def normalize_time_series (radar_file, opt_file, norm="meanstd", radar_bands=["V
else :
indices_seq = np.hstack((indices_seq,np.stack(lst, axis=1)))
if not os.path.exists("./data"):
os.makedirs("./data")
ptrn = os.path.basename(radar_file).split('_')[0]+"_"+os.path.basename(radar_file).split('_')[1]
np.save("./data/{}_rad_seq.npy".format(ptrn),radar_seq)
np.save("./data/{}_opt_seq.npy".format(ptrn),opt_seq)
......@@ -100,7 +104,7 @@ def normalize_time_series (radar_file, opt_file, norm="meanstd", radar_bands=["V
try:
rdt_df = radar_df.merge(opt_df[["ID"]],on="ID")
rdt = rdt_df[["Rdt_s"]].values
rdt = rdt_df[["Biom_f","Biom_s","Rdt_f","Rdt_s"]].values
np.save("./data/{}_yields.npy".format(ptrn),rdt)
except Exception as error :
print (error)
......@@ -110,16 +114,16 @@ def normalize_time_series (radar_file, opt_file, norm="meanstd", radar_bands=["V
if __name__ == '__main__' :
# Niakhar 2017
radar_file = "./data/niakhar_2017_radar_notree.csv"
opt_file = "./data/niakhar_2017_opt_gapf_notree.csv"
normalize_time_series(radar_file,opt_file,norm="meanstd")
radar_file = "./interpolate/niakhar_2017_radar_notree_interpolate.csv"
opt_file = "./interpolate/niakhar_2017_opt_gapf_notree_interpolate.csv"
normalize_time_series(radar_file,opt_file,norm="minmax")
# Niakhar 2018
radar_file = "./data/niakhar_2018_radar_notree.csv"
opt_file = "./data/niakhar_2018_opt_gapf_notree.csv"
normalize_time_series(radar_file,opt_file,norm="meanstd")
radar_file = "./interpolate/niakhar_2018_radar_notree_interpolate.csv"
opt_file = "./interpolate/niakhar_2018_opt_gapf_notree_interpolate.csv"
normalize_time_series(radar_file,opt_file,norm="minmax")
# Nioro 2018
radar_file = "./data/nioro_2018_radar_notree.csv"
opt_file = "./data/nioro_2018_opt_gapf_notree.csv"
normalize_time_series(radar_file,opt_file,norm="meanstd")
\ No newline at end of file
radar_file = "./interpolate/nioro_2018_radar_notree_interpolate.csv"
opt_file = "./interpolate/nioro_2018_opt_gapf_notree_interpolate.csv"
normalize_time_series(radar_file,opt_file,norm="minmax")
\ No newline at end of file
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.