Extract NV and BH

processing/extract.R → processing/extractBH.R

@@ -52,7 +52,7 @@ iQHE = c('0'='qualité hautes eaux inconnue',
 # Get the selection of data from the 'Liste-station_RRSE' file and the BanqueHydro directory
 get_selection = function (computer_data_path, listdir, listname,
                           cnames=c('code','station', 'BV_km2', 'axe_principal_concerne', 'longueur_serie', 'commentaires', 'choix'), 
-                          cisnum=c('BV_km2', 'longueur_serie')) {
+                          c_num=c('BV_km2', 'longueur_serie')) {
     
     # Get the file path to the data
     list_path = file.path(computer_data_path, listdir, listname)
@@ -66,28 +66,31 @@ get_selection = function (computer_data_path, listdir, listname,
     splits <- lapply(splits, function(x) x$text)
     
     # Combine columns back together in wide format
-    df_list <- bind_cols(splits)
+    df_selec <- bind_cols(splits)
     
-    df_list = df_list[-1,]
+    df_selec = df_selec[-1,]
     
-    names(df_list) = cnames
+    names(df_selec) = cnames
 
-    for (c in cisnum) {
-        df_list$c = as.numeric(sub(",", ".",
-                                   pull(df_list, c)))
+    for (c in c_num) {
+        df_selec$c = as.numeric(sub(",", ".",
+                                    pull(df_selec, c)))
     }
     
-    df_selec = df_list[df_list$choix == 'A garder' | df_list$choix == 'Ajout',]
+    selec = (df_selec$choix == 'A garder' | df_selec$choix == 'Ajout')
     
     df_selec = bind_cols(df_selec, 
-                         filename=paste(df_selec$code, '_HYDRO_QJM.txt', sep=''))
+                         filename=paste(df_selec$code, '_HYDRO_QJM.txt', sep=''),
+                         ok=selec
+                         )
+    
     return (df_selec)
 }
 
 # Example
 # df_selec = get_selection(
 #     "/home/louis/Documents/bouleau/INRAE/CDD_stationnarite/data",
-#     "liste_station",
+#     "",
 #     "Liste-station_RRSE.docx",
 #     cnames=c('code','station', 
 #              'BV_km2',
@@ -95,12 +98,12 @@ get_selection = function (computer_data_path, listdir, listname,
 #              'longueur_serie',
 #              'commentaires',
 #              'choix'), 
-#     cisnum=c('BV_km2',
+#     c_num=c('BV_km2',
 #              'longueur_serie'))
 
 
 # Extraction of information
-extract_info = function (computer_data_path, filedir, filename, verbose=TRUE) {
+extractBH_info = function (computer_data_path, filedir, filename, verbose=TRUE) {
     
     # Convert the filename in vector
     filename = c(filename)
@@ -139,9 +142,9 @@ extract_info = function (computer_data_path, filedir, filename, verbose=TRUE) {
             
             # Concatenate by raw data frames created by this function when filename correspond to only one filename
             df_info = rbind(df_info,
-                            extract_info(computer_data_path, 
-                                         filedir, 
-                                         f))
+                            extractBH_info(computer_data_path, 
+                                           filedir, 
+                                           f))
         }
         
         # Set the rownames by default (to avoid strange numbering)
@@ -173,7 +176,7 @@ extract_info = function (computer_data_path, filedir, filename, verbose=TRUE) {
                    territoire=trimws(substr(infotxt[13], 39, nchar(infotxt[13]))),
                    L93X=as.numeric(substr(infotxt[16], 38, 50)),
                    L93Y=as.numeric(substr(infotxt[16], 52, 63)),
-                   surface=as.numeric(substr(infotxt[19], 38, 50)),
+                   surface_km2=as.numeric(substr(infotxt[19], 38, 50)),
                    statut=iStatut[trimws(substr(infotxt[26], 38, 50))],
                    finalite=iFinalite[trimws(substr(infotxt[26], 52, 56))],
                    type=iType[trimws(substr(infotxt[26], 58, 58))],
@@ -194,14 +197,14 @@ extract_info = function (computer_data_path, filedir, filename, verbose=TRUE) {
 }
 
 # Example
-# df_info = extract_info(
+# df_info = extractBH_info(
     # "/home/louis/Documents/bouleau/INRAE/CDD_stationnarite/data",
     # '',
     # c('H5920011_HYDRO_QJM.txt', 'K4470010_HYDRO_QJM.txt'))
 
 
 # Extraction of data
-extract_data = function (computer_data_path, filedir, filename, verbose=TRUE) {
+extractBH_data = function (computer_data_path, filedir, filename, verbose=TRUE) {
     
     # Convert the filename in vector
     filename = c(filename)
@@ -239,9 +242,9 @@ extract_data = function (computer_data_path, filedir, filename, verbose=TRUE) {
 
             # Concatenate by raw data frames created by this function when filename correspond to only one filename
             df_data = rbind(df_data,
-                            extract_data(computer_data_path, 
-                                         filedir, 
-                                         f))
+                            extractBH_data(computer_data_path, 
+                                           filedir, 
+                                           f))
         }
 
         # Set the rownames by default (to avoid strange numbering)
@@ -270,7 +273,7 @@ extract_data = function (computer_data_path, filedir, filename, verbose=TRUE) {
                              skip=41)   
 
         # Extract all the information for the station
-        df_info = extract_info(computer_data_path, filedir, filename, verbose=FALSE)
+        df_info = extractBH_info(computer_data_path, filedir, filename, verbose=FALSE)
         # Get the code of the station
         code = df_info$code
         # Create a tibble with the date as Date class and the code of the station
@@ -289,7 +292,7 @@ extract_data = function (computer_data_path, filedir, filename, verbose=TRUE) {
 }
 
 # Example
-# df_data = extract_data(
+# df_data = extractBH_data(
 #     "/home/louis/Documents/bouleau/INRAE/CDD_stationnarite/data",
 #     '',
 #     c('H5920011_HYDRO_QJM.txt', 'K4470010_HYDRO_QJM.txt'))

processing/extractNV.R

+# Usefull library
+library(tools)
+library(dplyr)
+
+
+# Extraction of information
+extractNVlist_info = function (computer_data_path, filedir, listdir, listname, verbose=TRUE) {
+    
+    # Print information if asked
+    if (verbose) {
+        print(paste("extraction of info for file :", listname))
+    }
+
+    # Get the file path to the data
+    list_path = file.path(computer_data_path, listdir, listname)        
+
+    # Extract the data as a data frame
+    df_info = read.table(list_path,
+                         header=TRUE) 
+
+    # Create a filelist to store all the filename
+    codelist = c()
+    # Get all the filename in the data directory selected
+    filelist_tmp = list.files(file.path(computer_data_path,
+                                        filedir))
+
+    # For all the filename in the directory selected
+    for (f in filelist_tmp) {
+        # If the filename extention is 'txt'
+        if (file_ext(f) == 'txt') {
+            # Store the filename in the filelist
+            codelist = c(codelist, gsub('.txt', '', f)) 
+        }
+    }  
+
+    exist = df_info$CODE %in% codelist
+    missing = codelist[!(codelist %in% df_info$CODE)]
+    print(paste('station missing :', missing))
+    
+    df_info = df_info[exist,]
+
+    # Create a tibble with all the information needed
+    df_info =
+        tibble(code=as.character(df_info$CODE),
+               nom=as.character(df_info$NOM),
+               L93X=df_info$X_L2E,
+               L93Y=df_info$Y_L2E,
+               surface_km2=df_info$S_km2,
+               altitude_m=df_info$Alt,
+               file_path=paste(df_info$CODE, '.txt', sep='')
+               )
+
+    return (df_info)
+}
+
+# Example
+# df_info = extractNVlist_info(
+#     "/home/louis/Documents/bouleau/INRAE/CDD_stationnarite/data",
+#     'France207',
+#     '',
+#     'liste_bv_principaux_global.txt')
+
+
+# Extraction of data
+extractNV_data = function (computer_data_path, filedir, filename, verbose=TRUE) {
+    
+    # Convert the filename in vector
+    filename = c(filename)
+
+    # If the filename is 'all' or regroup more than one filename
+    if (all(filename == 'all') | length(filename) > 1) {
+
+        # If the filename is 'all'
+        if (all(filename == 'all')) {
+            # Create a filelist to store all the filename
+            filelist = c()
+             # Get all the filename in the data directory selected
+            filelist_tmp = list.files(file.path(computer_data_path,
+                                                filedir))
+
+            # For all the filename in the directory selected
+            for (f in filelist_tmp) {
+                # If the filename extention is 'txt'
+                if (file_ext(f) == 'txt') {
+                    # Store the filename in the filelist
+                    filelist = c(filelist, f) 
+                }
+            }
+            # If the filename regroup more than one filename
+        } else if (length(filename > 1)) {
+             # The filelist correspond to the filename
+            filelist = filename
+        } 
+
+        # Create a blank data frame
+        df_data = data.frame()
+
+        # For all the file in the filelist
+        for (f in filelist) {
+
+            # Concatenate by raw data frames created by this function when filename correspond to only one filename
+            df_data = rbind(df_data,
+                            extractNV_data(computer_data_path, 
+                                         filedir, 
+                                         f))
+        }
+
+        # Set the rownames by default (to avoid strange numbering)
+        rownames(df_data) = NULL
+        return (df_data)
+    }
+
+    # Get the filename from the vector
+    filename = filename[1]
+    
+    # Print information if asked
+    if (verbose) {
+        print(paste("extraction of data for file :", filename))
+    }
+
+    # Get the file path to the data
+    file_path = file.path(computer_data_path, filedir, filename)
+    
+    if (file.exists(file_path)) {
+
+        # Extract the data as a data frame
+        df_data = read.table(file_path,
+                             header=FALSE,
+                             skip=1,
+                             na.strings=c('-1', '-99.000'))   
+
+        # Create a tibble with the date as Date class and the code of the station
+        date = paste(df_data[,1],
+                     df_data[,2],
+                     df_data[,3],
+                     sep='-')
+
+        df_data = tibble(Date=as.Date(as.character(date),
+                                      format="%Y-%m-%d"),
+                         Qm3s=df_data[,4],
+                         QCode=df_data[,5],
+                         code=gsub('.txt', '', filename))
+
+        return (df_data)
+
+    } else {
+        print(paste('filename', file_path, 'do not exist'))
+        return (NULL)
+    }
+}
+
+# Example
+# df_data = extractNV_data(
+    # "/home/louis/Documents/bouleau/INRAE/CDD_stationnarite/data",
+    # 'France207',
+    # c('O0015310.txt', 'Q0214010.txt', 'P0115020.txt'))

script.R

@@ -46,7 +46,8 @@ BHdatadir =
 setwd(computer_work_path)
 
 # Sourcing R file
-source('processing/extract.R')
+source('processing/extractBH.R')
+source('processing/extractNV.R')
 source('processing/analyse.R')
 source('plotting/panel.R')
 
@@ -80,21 +81,21 @@ if (is.character(BHlistdir) & is.character(BHlistname) & is.character(BHdatadir)
                                       'longueur_serie',
                                       'commentaires',
                                       'choix'), 
-                             cisnum=c('BV_km2',
+                             c_num=c('BV_km2',
                                       'longueur_serie'))
     BHfiledir = BHdatadir
-    BHfilename = df_selec$filename
+    BHfilename = df_selec[df_selec$ok,]$filename
 }
 
 # Extract information about selected stations
-df_info = extract_info(computer_data_path, BHfiledir, BHfilename)
+df_info_BH = extractBH_info(computer_data_path, BHfiledir, BHfilename)
 
 # Extract data about selected stations
-df_data = extract_data(computer_data_path, BHfiledir, BHfilename)
+df_data_BH = extractBH_data(computer_data_path, BHfiledir, BHfilename)
 
 # Plot time panel of debit by stations
-panel(df_data, df_info, figdir, BHfiledir)
-# panel(df_data, df_info, figdir, BHfiledir, is_sqrt=TRUE)
+panel(df_data_BH, df_info_BH, figdir, BHfiledir)
+# panel(df_data_BH, df_info_BH, figdir, BHfiledir, is_sqrt=TRUE)
 
 # Compute gap parameters for stations
-df_lac = get_lacune(df_data, df_info)
+df_lac = get_lacune(df_data_BH, df_info_BH)