Commit dfa9ec33 authored by Harmel Tristan's avatar Harmel Tristan
Browse files

debug iwr processing and call (input files, quantile computation)

parent 4282aabf
...@@ -51,48 +51,27 @@ def generate_database_info(idirs, infofile): ...@@ -51,48 +51,27 @@ def generate_database_info(idirs, infofile):
['Lt' + s for s in meta_attrs]])) ['Lt' + s for s in meta_attrs]]))
# loop on directories (for each date) # loop on directories (for each date)
for date in metas.DATE.unique():
date_dir =pd.to_datetime(str(date)).strftime('%Y%m%d')
meta = metas[metas.DATE==date]
idir_ = os.path.join(idir, 'L1', date_dir)
files = glob.glob(idir_ + '/*.csv')
print(idir_)
for file in files:
name = opb(file).replace('.csv', '')
name_ =name.split('_')
print(name_)
if name_[2]=="eau":
print(name_)
for id in meta.SAMPLE.unique():
print(idir_,id)
files = np.unique(np.append(glob.glob(idir_ + '/' + id.lower() + '_*.csv'),
glob.glob(idir_ + '/' + id + '_*.csv')))
for i, meta in metas.iterrows(): for i, meta in metas.iterrows():
# print(meta) #print(i)
date_dir = meta[0].strftime('%Y%m%d') date_dir = meta[0].strftime('%Y%m%d')
full_date = dt.datetime.combine(meta[0], meta[1]) full_date = dt.datetime.combine(meta[0], meta[1])
ID, site, comment, lat, lon = meta[2], meta[3], meta[4], meta[5], meta[6] ID, site, comment, lat, lon, wind = meta[2:8]
idir_ = os.path.join(idir, 'L1', date_dir)
date = opb(idir_) date = opb(idir_)
print(idir_ + '/' + ID.lower()) print(idir_ + '/' + ID.lower() + ID.replace('id',''))
files = np.unique(np.append(glob.glob(idir_ + '/' + ID.lower() + '_*.csv'), files = pd.Series(
glob.glob(idir_ + '/' + ID + '_*.csv'))) np.unique(
np.append(glob.glob(idir_ + '/awr/' + ID.lower() + '_*.mlb'),
glob.glob(idir_ + '/awr/ID' + ID.replace('id','').replace('ID','') + '_*.mlb'))))
print('nb files ',len(files))
for file in files: for file in files:
name = opb(file).replace('.csv','') name = opb(file).replace('.mlb','')
print(name.split('_')) print(name.split('_'))
if i> 4:
break
print(len(files))
# loop on data files (for each acquisition sequence) # loop on data files (for each acquisition sequence)
for Edf in files: for Edf in files:
ID_ = ID.lower() #Edf.split('_')[-1].replace('.mlb', '') ID_ = ID.lower() #Edf.split('_')[-1].replace('.mlb', '')
......
...@@ -19,7 +19,7 @@ setup( ...@@ -19,7 +19,7 @@ setup(
author_email='tristan.harmel@gmail.com', author_email='tristan.harmel@gmail.com',
description='Package to help trios TriOS radiometer data for various above-water or in-water setups', description='Package to help trios TriOS radiometer data for various above-water or in-water setups',
# TODO update Dependent packages (distributions) # TODO update Dependent packages (distributions)
install_requires=['cmocean','dash','dash_core_components','dash_html_components','pandas', 'scipy', 'numpy', install_requires=['pandas','cmocean','dash','dash_core_components','dash_html_components','pandas', 'scipy', 'numpy',
'pyodbc', 'netCDF4', 'matplotlib', 'docopt', 'GDAL', 'python-dateutil','plotly'], 'pyodbc', 'netCDF4', 'matplotlib', 'docopt', 'GDAL', 'python-dateutil','plotly'],
entry_points={ entry_points={
......
...@@ -460,8 +460,9 @@ class iwr_process: ...@@ -460,8 +460,9 @@ class iwr_process:
mean = df.groupby('rounded_depth').mean() mean = df.groupby('rounded_depth').mean()
median = df.groupby('rounded_depth').median() median = df.groupby('rounded_depth').median()
std = df.groupby('rounded_depth').std() std = df.groupby('rounded_depth').std()
q25 = df.groupby('rounded_depth').quantile(0.25) df_=df.drop(df.columns[df.dtypes=='object'],axis=1)
q75 = df.groupby('rounded_depth').quantile(0.75) q25 = df_.groupby('rounded_depth').quantile(0.25)
q75 = df_.groupby('rounded_depth').quantile(0.75)
# --------------------------- # ---------------------------
# Data processing # Data processing
......
...@@ -296,12 +296,12 @@ class data: ...@@ -296,12 +296,12 @@ class data:
def load_csv(self, file, utc_conv=0): def load_csv(self, file, utc_conv=0):
print(file) print(file)
# dateparse = lambda x: pd.datetime.strptime(x, '%Y-%m-%d %H:%M:%S') + pd.to_timedelta(utc_conv, 'h') # dateparse = lambda x: pd.datetime.strptime(x, '%Y-%m-%d %H:%M:%S') + pd.to_timedelta(utc_conv, 'h')
if len(file) > 1 and not isinstance(file, str): if len(file) > 1 or not isinstance(file, str):
print('Warning! Multiple files found but only one expected, trios first file of the list:') print('Warning! Multiple files found but only one expected, trios first file of the list:')
print(file) print(file)
file_ = file[0] file_ = file[0]
else: else:
file_ = file[0] file_ = file
# df = pd.read_csv(file, date_parser=dateparse, sep=';', index_col=0, na_values=['-NAN']) # df = pd.read_csv(file, date_parser=dateparse, sep=';', index_col=0, na_values=['-NAN'])
df = pd.read_csv(file_, sep=';|,', na_values=['-NAN'], engine='python') df = pd.read_csv(file_, sep=';|,', na_values=['-NAN'], engine='python')
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment