En raison du déménagement des baies serveurs, les services gitlab.irstea.fr et mattermost.irstea.fr seront interrompus le samedi 2 octobre 2021 au matin. Ils devraient revenir à la normale dans la journée.

Commit dfa9ec33 authored by Harmel Tristan's avatar Harmel Tristan
Browse files

debug iwr processing and call (input files, quantile computation)

parent 4282aabf
......@@ -51,48 +51,27 @@ def generate_database_info(idirs, infofile):
['Lt' + s for s in meta_attrs]]))
# loop on directories (for each date)
for date in metas.DATE.unique():
date_dir =pd.to_datetime(str(date)).strftime('%Y%m%d')
meta = metas[metas.DATE==date]
idir_ = os.path.join(idir, 'L1', date_dir)
files = glob.glob(idir_ + '/*.csv')
print(idir_)
for file in files:
name = opb(file).replace('.csv', '')
name_ =name.split('_')
print(name_)
if name_[2]=="eau":
print(name_)
for id in meta.SAMPLE.unique():
print(idir_,id)
files = np.unique(np.append(glob.glob(idir_ + '/' + id.lower() + '_*.csv'),
glob.glob(idir_ + '/' + id + '_*.csv')))
for i, meta in metas.iterrows():
# print(meta)
#print(i)
date_dir = meta[0].strftime('%Y%m%d')
full_date = dt.datetime.combine(meta[0], meta[1])
ID, site, comment, lat, lon = meta[2], meta[3], meta[4], meta[5], meta[6]
ID, site, comment, lat, lon, wind = meta[2:8]
idir_ = os.path.join(idir, 'L1', date_dir)
date = opb(idir_)
print(idir_ + '/' + ID.lower())
print(idir_ + '/' + ID.lower() + ID.replace('id',''))
files = np.unique(np.append(glob.glob(idir_ + '/' + ID.lower() + '_*.csv'),
glob.glob(idir_ + '/' + ID + '_*.csv')))
files = pd.Series(
np.unique(
np.append(glob.glob(idir_ + '/awr/' + ID.lower() + '_*.mlb'),
glob.glob(idir_ + '/awr/ID' + ID.replace('id','').replace('ID','') + '_*.mlb'))))
print('nb files ',len(files))
for file in files:
name = opb(file).replace('.csv','')
name = opb(file).replace('.mlb','')
print(name.split('_'))
if i> 4:
break
print(len(files))
# loop on data files (for each acquisition sequence)
for Edf in files:
ID_ = ID.lower() #Edf.split('_')[-1].replace('.mlb', '')
......
......@@ -19,7 +19,7 @@ setup(
author_email='tristan.harmel@gmail.com',
description='Package to help trios TriOS radiometer data for various above-water or in-water setups',
# TODO update Dependent packages (distributions)
install_requires=['cmocean','dash','dash_core_components','dash_html_components','pandas', 'scipy', 'numpy',
install_requires=['pandas','cmocean','dash','dash_core_components','dash_html_components','pandas', 'scipy', 'numpy',
'pyodbc', 'netCDF4', 'matplotlib', 'docopt', 'GDAL', 'python-dateutil','plotly'],
entry_points={
......
......@@ -460,8 +460,9 @@ class iwr_process:
mean = df.groupby('rounded_depth').mean()
median = df.groupby('rounded_depth').median()
std = df.groupby('rounded_depth').std()
q25 = df.groupby('rounded_depth').quantile(0.25)
q75 = df.groupby('rounded_depth').quantile(0.75)
df_=df.drop(df.columns[df.dtypes=='object'],axis=1)
q25 = df_.groupby('rounded_depth').quantile(0.25)
q75 = df_.groupby('rounded_depth').quantile(0.75)
# ---------------------------
# Data processing
......
......@@ -296,12 +296,12 @@ class data:
def load_csv(self, file, utc_conv=0):
print(file)
# dateparse = lambda x: pd.datetime.strptime(x, '%Y-%m-%d %H:%M:%S') + pd.to_timedelta(utc_conv, 'h')
if len(file) > 1 and not isinstance(file, str):
if len(file) > 1 or not isinstance(file, str):
print('Warning! Multiple files found but only one expected, trios first file of the list:')
print(file)
file_ = file[0]
else:
file_ = file[0]
file_ = file
# df = pd.read_csv(file, date_parser=dateparse, sep=';', index_col=0, na_values=['-NAN'])
df = pd.read_csv(file_, sep=';|,', na_values=['-NAN'], engine='python')
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment