debug iwr processing and call (input files, quantile computation)

exe/process_hydrosim.py

@@ -51,48 +51,27 @@ def generate_database_info(idirs, infofile):
                                               ['Lt' + s for s in meta_attrs]]))
 
     # loop on directories (for each date)
-    for date in metas.DATE.unique():
-        date_dir =pd.to_datetime(str(date)).strftime('%Y%m%d')
-        meta = metas[metas.DATE==date]
-        idir_ = os.path.join(idir, 'L1', date_dir)
-        files = glob.glob(idir_ + '/*.csv')
-        print(idir_)
-        for file in files:
-            name = opb(file).replace('.csv', '')
-            name_ =name.split('_')
-            print(name_)
-            if name_[2]=="eau":
-                print(name_)
-
-        for id in meta.SAMPLE.unique():
-            print(idir_,id)
-            files = np.unique(np.append(glob.glob(idir_ + '/' + id.lower() + '_*.csv'),
-                                        glob.glob(idir_ + '/' + id + '_*.csv')))
-
-
-
     for i, meta in metas.iterrows():
-        # print(meta)
+        #print(i)
         date_dir = meta[0].strftime('%Y%m%d')
         full_date = dt.datetime.combine(meta[0], meta[1])
-        ID, site, comment, lat, lon = meta[2], meta[3], meta[4], meta[5], meta[6]
-
+        ID, site, comment, lat, lon, wind = meta[2:8]
+        idir_ = os.path.join(idir, 'L1', date_dir)
 
 
         date = opb(idir_)
-        print(idir_ + '/' + ID.lower())
+        print(idir_ + '/' + ID.lower() + ID.replace('id',''))
 
-        files = np.unique(np.append(glob.glob(idir_ + '/' + ID.lower() + '_*.csv'),
-                          glob.glob(idir_ + '/' + ID + '_*.csv')))
+        files = pd.Series(
+                    np.unique(
+                        np.append(glob.glob(idir_ + '/awr/' + ID.lower() + '_*.mlb'),
+                                    glob.glob(idir_ + '/awr/ID' + ID.replace('id','').replace('ID','') + '_*.mlb'))))
+        print('nb files ',len(files))
 
         for file in files:
-            name = opb(file).replace('.csv','')
+            name = opb(file).replace('.mlb','')
             print(name.split('_'))
 
-
-        if i> 4:
-            break
-        print(len(files))
         # loop on data files (for each acquisition sequence)
         for Edf in files:
             ID_ = ID.lower() #Edf.split('_')[-1].replace('.mlb', '')

setup.py

@@ -19,7 +19,7 @@ setup(
     author_email='tristan.harmel@gmail.com',
     description='Package to help trios TriOS radiometer data for various above-water or in-water setups',
     # TODO update Dependent packages (distributions)
-    install_requires=['cmocean','dash','dash_core_components','dash_html_components','pandas', 'scipy', 'numpy',
+    install_requires=['pandas','cmocean','dash','dash_core_components','dash_html_components','pandas', 'scipy', 'numpy',
                       'pyodbc', 'netCDF4', 'matplotlib', 'docopt', 'GDAL', 'python-dateutil','plotly'],
 
     entry_points={

trios/process.py

@@ -460,8 +460,9 @@ class iwr_process:
         mean = df.groupby('rounded_depth').mean()
         median = df.groupby('rounded_depth').median()
         std = df.groupby('rounded_depth').std()
-        q25 = df.groupby('rounded_depth').quantile(0.25)
-        q75 = df.groupby('rounded_depth').quantile(0.75)
+        df_=df.drop(df.columns[df.dtypes=='object'],axis=1)
+        q25 = df_.groupby('rounded_depth').quantile(0.25)
+        q75 = df_.groupby('rounded_depth').quantile(0.75)
 
         # ---------------------------
         # Data processing

trios/utils/utils.py

@@ -296,12 +296,12 @@ class data:
     def load_csv(self, file, utc_conv=0):
         print(file)
         # dateparse = lambda x: pd.datetime.strptime(x, '%Y-%m-%d %H:%M:%S') + pd.to_timedelta(utc_conv, 'h')
-        if len(file) > 1 and not isinstance(file, str):
+        if len(file) > 1 or not isinstance(file, str):
             print('Warning! Multiple files found but only one expected, trios first file of the list:')
             print(file)
             file_ = file[0]
         else:
-            file_ = file[0]
+            file_ = file
         # df = pd.read_csv(file, date_parser=dateparse, sep=';', index_col=0, na_values=['-NAN'])
         df = pd.read_csv(file_, sep=';|,', na_values=['-NAN'], engine='python')