Commit c9a02a96 authored by Harmel Tristan's avatar Harmel Tristan
Browse files

reorganize package trios and reproject all data on common wavelengths (set in config.py)

parent 6b4880fc
...@@ -58,7 +58,7 @@ def odbc2lst(conn, query): ...@@ -58,7 +58,7 @@ def odbc2lst(conn, query):
""" Processing """ """ Processing """
''' '''
STEP 1: process TRIOS data STEP 1: trios TRIOS data
''' '''
# connect to bd TRIOS # connect to bd TRIOS
...@@ -132,7 +132,7 @@ for key in dicmeth.keys(): ...@@ -132,7 +132,7 @@ for key in dicmeth.keys():
pass pass
''' '''
STEP 2: process HOBO data STEP 2: trios HOBO data
''' '''
if hobo_process: if hobo_process:
......
''' modules dedicated to structure and exploit ancillary data (normally stored in aux folder)''' ''' modules dedicated to structure and exploit ancillary data (normally stored in aux folder)'''
import pandas as pd import pandas as pd
import numpy as np
from scipy.interpolate import interp1d from scipy.interpolate import interp1d
from config import * from trios.config import *
class iopw: class iopw:
......
import pandas as pd import pandas as pd
import numpy as np
from scipy.interpolate import interp1d from scipy.interpolate import interp1d
from utils.sunposition import sunpos from utils.sunposition import sunpos
from trios.config import *
class awr_data: class awr_data:
''' '''
...@@ -46,18 +45,23 @@ class awr_data: ...@@ -46,18 +45,23 @@ class awr_data:
Lsky, wl_Lsky = d.load_csv(self.Lskyf) Lsky, wl_Lsky = d.load_csv(self.Lskyf)
Lt, wl_Lt = d.load_csv(self.Ltf) Lt, wl_Lt = d.load_csv(self.Ltf)
# ''' interpolate Ed and Lsky data upon Lt wavelength''' # ''' interpolate Ed, Lt and Lsky data upon common wavelength'''
wl = wl_Lt wl = wl_common
Lt.columns = pd.MultiIndex.from_tuples(zip(['Lt'] * len(wl), wl), names=['param', 'wl'])
intEd = interp1d(wl_Ed, Ed.values, fill_value='extrapolate')(wl) intEd = interp1d(wl_Ed, Ed.values, fill_value='extrapolate')(wl)
newEd = pd.DataFrame(index=Ed.index, newEd = pd.DataFrame(index=Ed.index,columns=pd.MultiIndex.from_tuples(zip(['Ed'] * len(wl), wl),
columns=pd.MultiIndex.from_tuples(zip(['Ed'] * len(wl), wl), names=['param', 'wl']), names=['param', 'wl']),data=intEd)
data=intEd)
intLt = interp1d(wl_Lt, Lt.values, fill_value='extrapolate')(wl)
newLt = pd.DataFrame(index=Lt.index,columns=pd.MultiIndex.from_tuples(zip(['Lt'] * len(wl), wl),
names=['param', 'wl']),data=intLt)
intLsky = interp1d(wl_Lsky, Lsky.values, fill_value='extrapolate')(wl) intLsky = interp1d(wl_Lsky, Lsky.values, fill_value='extrapolate')(wl)
newLsky = pd.DataFrame(index=Lsky.index, columns=pd.MultiIndex.from_tuples(zip(['Lsky'] * len(wl), wl), newLsky = pd.DataFrame(index=Lsky.index, columns=pd.MultiIndex.from_tuples(zip(['Lsky'] * len(wl), wl),
names=['param', 'wl']), data=intLsky) names=['param', 'wl']), data=intLsky)
# merge sensor data on time # merge sensor data on time
df = pd.merge_asof(Lt, newEd, left_index=True, right_index=True, tolerance=pd.Timedelta("2 seconds"), df = pd.merge_asof(newLt, newEd, left_index=True, right_index=True, tolerance=pd.Timedelta("2 seconds"),
direction="nearest") direction="nearest")
df = pd.merge_asof(df, newLsky, left_index=True, right_index=True, tolerance=pd.Timedelta("2 seconds"), df = pd.merge_asof(df, newLsky, left_index=True, right_index=True, tolerance=pd.Timedelta("2 seconds"),
direction="nearest") direction="nearest")
...@@ -126,32 +130,35 @@ class iwr_data: ...@@ -126,32 +130,35 @@ class iwr_data:
# Edz.mask(Edz<0,inplace=True) # Edz.mask(Edz<0,inplace=True)
# Luz.mask(Luz<0,inplace=True) # Luz.mask(Luz<0,inplace=True)
# copy depth data to Ed frame on date index # copy depth data to Ed frame on date index
# Ed.index = Ed.index.droplevel(level=1) # Ed.index = Ed.index.droplevel(level=1)
#''' interpolate Ed and Lsky data upon Lt wavelength''' #''' interpolate Ed, Edz and Luz data upon common wavelength'''
wl = wl_Luz wl = wl_common
Luz.columns = pd.MultiIndex.from_tuples(list(zip(['Luz'] * len(wl), wl)), names=['param', 'wl'])
intEd = interp1d(wl_Ed, Ed.values, fill_value='extrapolate')(wl) intEd = interp1d(wl_Ed, Ed.values, fill_value='extrapolate')(wl)
newEd = pd.DataFrame(index=Ed.index.get_level_values(0), newEd = pd.DataFrame(index=Ed.index.get_level_values(0),
columns=pd.MultiIndex.from_tuples(list(zip(['Ed'] * len(wl), wl)), names=['param', 'wl']), columns=pd.MultiIndex.from_tuples(list(zip(['Ed'] * len(wl), wl)), names=['param', 'wl']),
data=intEd) data=intEd)
intEdz = interp1d(wl_Edz, Edz.values, fill_value='extrapolate')(wl) intEdz = interp1d(wl_Edz, Edz.values, fill_value='extrapolate')(wl)
newEdz = pd.DataFrame(index=Edz.index, columns=pd.MultiIndex.from_tuples(list(zip(['Edz'] * len(wl), wl)), newEdz = pd.DataFrame(index=Edz.index, columns=pd.MultiIndex.from_tuples(list(zip(['Edz'] * len(wl), wl)),
names=['param', 'wl']), data=intEdz) names=['param', 'wl']), data=intEdz)
intLuz = interp1d(wl_Luz, Luz.values, fill_value='extrapolate')(wl)
newLuz = pd.DataFrame(index=Luz.index, columns=pd.MultiIndex.from_tuples(list(zip(['Luz'] * len(wl), wl)),
names=['param', 'wl']), data=intLuz)
# correct depth data for sensor to sensor distance # correct depth data for sensor to sensor distance
Luz.reset_index(level=1, inplace=True) newLuz.reset_index(level=1, inplace=True)
Luz.iloc[:, 0] = Luz.iloc[:, 0] + delta_Lu_depth newLuz.iloc[:, 0] = Luz.iloc[:, 0] + delta_Lu_depth
# newEd.reset_index(level=1,inplace=True) # newEd.reset_index(level=1,inplace=True)
newEdz.reset_index(level=1, inplace=True) newEdz.reset_index(level=1, inplace=True)
newEdz.iloc[:, 0] = newEdz.iloc[:, 0] + delta_Edz_depth newEdz.iloc[:, 0] = newEdz.iloc[:, 0] + delta_Edz_depth
# merge sensor data on time # merge sensor data on time
df = pd.merge_asof(Luz, newEd, left_index=True, right_index=True, tolerance=pd.Timedelta("2 seconds"), df = pd.merge_asof(newLuz, newEd, left_index=True, right_index=True, tolerance=pd.Timedelta("2 seconds"),
direction="nearest") direction="nearest")
df = pd.merge_asof(df, newEdz, left_index=True, right_index=True, suffixes=('_Luz', '_Edz'), df = pd.merge_asof(df, newEdz, left_index=True, right_index=True, suffixes=('_Luz', '_Edz'),
tolerance=pd.Timedelta("2 seconds"), tolerance=pd.Timedelta("2 seconds"),
...@@ -174,7 +181,7 @@ class iwr_data: ...@@ -174,7 +181,7 @@ class iwr_data:
# #
# dateparse = lambda x: pd.datetime.strptime(x, '%Y-%m-%d %H:%M:%S') # dateparse = lambda x: pd.datetime.strptime(x, '%Y-%m-%d %H:%M:%S')
# if len(file) > 1: # if len(file) > 1:
# print('Warning! Multiple files found but only one expected, process first file of the list:') # print('Warning! Multiple files found but only one expected, trios first file of the list:')
# print(file) # print(file)
# file = file[0] # file = file[0]
# df = pd.read_csv(file, sep=';', index_col=[1, 0], na_values=['-NAN']) # df = pd.read_csv(file, sep=';', index_col=[1, 0], na_values=['-NAN'])
...@@ -225,16 +232,18 @@ class swr_data: ...@@ -225,16 +232,18 @@ class swr_data:
Ed, wl_Ed = data().load_csv(self.Edf) Ed, wl_Ed = data().load_csv(self.Edf)
Lu0, wl_Lu0 = data().load_csv(self.Lu0f) Lu0, wl_Lu0 = data().load_csv(self.Lu0f)
# ''' interpolate Ed and Lsky data upon Lt wavelength''' # ''' interpolate Ed and Lsky data upon common wavelengths'''
wl = wl_Lu0 wl = wl_common
Lu0.columns = pd.MultiIndex.from_tuples(zip(['Lu0+'] * len(wl), wl), names=['param', 'wl'])
intEd = interp1d(wl_Ed, Ed.values, fill_value='extrapolate')(wl) intEd = interp1d(wl_Ed, Ed.values, fill_value='extrapolate')(wl)
newEd = pd.DataFrame(index=Ed.index, newEd = pd.DataFrame(index=Ed.index,
columns=pd.MultiIndex.from_tuples(zip(['Ed'] * len(wl), wl), names=['param', 'wl']), columns=pd.MultiIndex.from_tuples(zip(['Ed'] * len(wl), wl), names=['param', 'wl']),
data=intEd) data=intEd)
intLu0 = interp1d(wl_Lu0, Lu0.values, fill_value='extrapolate')(wl)
newLu0 = pd.DataFrame(index=Lu0.index, columns=pd.MultiIndex.from_tuples(zip(['Lu0+'] * len(wl), wl),
names=['param', 'wl']), data=intLu0)
# merge sensor data on time # merge sensor data on time
df = pd.merge_asof(Lu0, newEd, left_index=True, right_index=True, tolerance=pd.Timedelta("2 seconds"), df = pd.merge_asof(newLu0, newEd, left_index=True, right_index=True, tolerance=pd.Timedelta("2 seconds"),
direction="nearest") direction="nearest")
# add solar angle data and idpr # add solar angle data and idpr
...@@ -261,7 +270,7 @@ class data: ...@@ -261,7 +270,7 @@ class data:
print(file) print(file)
dateparse = lambda x: pd.datetime.strptime(x, '%Y-%m-%d %H:%M:%S') dateparse = lambda x: pd.datetime.strptime(x, '%Y-%m-%d %H:%M:%S')
if len(file) > 1: if len(file) > 1:
print('Warning! Multiple files found but only one expected, process first file of the list:') print('Warning! Multiple files found but only one expected, trios first file of the list:')
print(file) print(file)
file_ = file[0] file_ = file[0]
# df = pd.read_csv(file, date_parser=dateparse, sep=';', index_col=0, na_values=['-NAN']) # df = pd.read_csv(file, date_parser=dateparse, sep=';', index_col=0, na_values=['-NAN'])
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment