Skip to content

GitLab

Commit 85145070 authored by Dorchies David's avatar Dorchies David
Browse files

refactor: change BNPE to Hub'eau 

Closes #3
parent 9d2507fa
Pipeline #25967 passed with stages
in 3 minutes and 34 seconds
Hide whitespace changes
Inline Side-by-side
.Rbuildignore
View file @ 85145070
......@@ -4,3 +4,6 @@
^LICENSE\.md$
^man-roxygen$
^docs$
^\.gitlab-ci\.yml$
^_pkgdown.yml$
^pkgdown$
.gitlab-ci.yml
View file @ 85145070
......@@ -37,4 +37,4 @@ website:
- R -e 'devtools::update_packages(packages = "pkgdown")'
- R -e 'pkgdown::build_site()'
- sudo apt-get update && sudo apt-get install -y sshpass rsync
- sshpass -p "${OVH_PASS}" rsync -a -e "ssh -o StrictHostKeyChecking=no" docs/ ${OVH_LOGIN}@${OVH_SFTP}:/home/${OVH_LOGIN}/in-wop/bnpe/
- sshpass -p "${OVH_PASS}" rsync -a -e "ssh -o StrictHostKeyChecking=no" docs/ ${OVH_LOGIN}@${OVH_SFTP}:/home/${OVH_LOGIN}/in-wop/hubeau/
DESCRIPTION
View file @ 85145070
Package: bnpe
Package: hubeau
Type: Package
Title: Retrieval Functions for the French national data bank of quantitative withdrawals (BNPE)
Title: Functions for the French API on water data "Hub'eau"
Version: 0.2.0.9000
Authors@R: c(
person("David", "Dorchies", role = c("aut", "cre"), comment = c(ORCID = "0000-0002-6595-7984"), email = "david.dorchies@inrae.fr")
)
Description: Collection of functions to help retrieve the French national data bank of
quantitative withdrawals (Banque Nationale des Prélèvements quantitatifs en Eau - BNPE)
available on its website <https://bnpe.eaufrance.fr/>.
Description: Collection of functions to help retrieve the French national data bank on water <https://hubeau.eaufrance.fr/>.
License: MIT + file LICENSE
Encoding: UTF-8
LazyData: true
Imports:
Imports:
config,
httr,
purrr,
......
NAMESPACE
View file @ 85145070
# Generated by roxygen2: do not edit by hand
export(doQuery)
export(getComSeriesDep)
export(getCookie)
export(getOuvrageSeries)
export(getTimeSeriesCom)
export(getTimeSeriesDep)
export(get_chronique)
export(get_points_prelevement)
export(get_prelevements_chronique)
export(get_prelevements_prelevement)
NEWS.md
View file @ 85145070
# bnpe
# hubeau
## 0.2.0 - 2021-06-24
......
R/doApiQuery.R
View file @ 85145070
doApiQuery <- function(url_path,
params,
cfg = config::get(file = system.file("config.yml", package = "bnpe"))) {
cfg = config::get(file = system.file("config.yml", package = "hubeau"))) {
query <- file.path(cfg$api$url, url_path)
for (paramName in names(params)) {
if (!is.null(params[[paramName]])) {
......
R/doQuery.R deleted 100644 → 0
View file @ 9d2507fa
#' Internal function for doing the queries for the bnpe package
#'
#' @param url_path the end of the URL path
#' @param params A list containing the parameters sent in the query
#' @inheritParams getCookie
#' @param cookie a named [character] [vector] with the values of the cookies (See [getCookie])
#'
#' @return a [httr::response] object from a [httr::GET]
#' @export
#'
doQuery <- function(url_path,
params = list(code_usage = NULL,
code_type_eau = NULL,
commune_adj = NULL,
ecrasant = "false"),
cfg = config::get(file = system.file("config.yml", package = "bnpe")),
cookie = getCookie(cfg)) {
paramsDef <- list(code_usage = NULL,
code_type_eau = NULL,
commune_adj = NULL,
ecrasant = "false")
params <- utils::modifyList(paramsDef, as.list(params))
# Building URL query https://website/?param1=value1&param2=value2...
query <- file.path(cfg$url, url_path)
for (paramName in names(params)) {
if (!is.null(params[[paramName]])) {
query <- urltools::param_set(query,
key = paramName,
value = params[[paramName]])
}
}
# Request
resp <- httr::GET(query,
httr::user_agent(cfg$user_agent),
httr::set_cookies(.cookies = cookie))
if(length(grep("<title>Request Rejected</title>", httr::content(resp, "text"))) > 0) {
stop("The following URL request has been rejected: ", query)
}
return(resp)
}
R/getComSeries.R deleted 100644 → 0
View file @ 9d2507fa
#' Retrieve data by commune for one departement and one year
#'
#' @param code_dep [character] of length 2
#' @param year [numeric] Year of the data to retrieve
#' @template param_get_common
#'
#' @return [data.frame] with one row by commune and two columns: "insee_com" and "volume"
#' @export
#'
#' @examples
#' getComSeriesDep("08", 2016, code_usage = "AEP", code_type_eau = "CONT")
getComSeriesDep <- function(code_dep,
year,
code_usage = NULL,
code_type_eau = NULL,
cfg = config::get(file = system.file("config.yml", package = "bnpe")),
cookie = getCookie(cfg)) {
params <- list(
code_dep = code_dep,
annee = year,
code_usage = code_usage,
code_type_eau = code_type_eau
)
resp <- doQuery(url_path = cfg$url_com_series,
params = params,
cfg = cfg,
cookie = cookie)
l <- httr::content(resp, "parsed")
codesInsee <- sapply(l$features, function(x) { x$properties$codeInsee })
hasVolume <- sapply(l$features, function(x) { !is.null(x$properties$volume) })
volumes <- unlist(sapply(l$features, function(x) { x$properties$volume }))
data.frame(insee_com = codesInsee[hasVolume], volume = volumes)
}
R/getCookie.R deleted 100644 → 0
View file @ 9d2507fa
#' Get cookie from the BNPE website in order to be able to do requests
#'
#' By default it is automatically called at each request but this function
#' can be used once to store a unique cookie and avoiding asking a cookie for each request (See example).
#'
#' @param cfg a [config] object Configuration of the communication. Use by default the internal package
#' configuration stored at location `system.file("config.yml", package = "bnpe")`
#'
#' @return a named [character] [vector] with the values of the cookies
#' @export
#'
#' @examples
#' cookie <- getCookie()
#' getTimeSeriesDep("08", cookie = cookie)
#' getTimeSeriesCom("08105", cookie = cookie)
getCookie <- function(cfg = config::get(file = system.file("config.yml", package = "bnpe"))) {
URL1 <- paste0(urltools::scheme(cfg$url), "://", urltools::domain(cfg$url))
userAgent <- httr::user_agent(cfg$user_agent)
resp <- httr::GET(URL1, userAgent)
ck <- httr::cookies(resp)$value
names(ck) <- httr::cookies(resp)$name
return(ck)
}
R/getOuvrageSeries.R deleted 100644 → 0
View file @ 9d2507fa
#' Retrieve data by device ("ouvrage") (metadata and annual withdrawal volumes)
#'
#' @param code_sandre [character] Sandre identifier of the device
#' @param cookie a named [character] [vector] with the values of the cookies (See [getCookie])
#' @inheritParams getCookie
#'
#' @return [list] compiling data of the device with the following items:
#' - `id`: BNPE identifier
#' - `code`: code Sandre
#' - `codeCommune`: insee code of the commune
#' - `codePrecision`: ???
#' - `codeStatut`: ???
#' - `codeTypeEau`: the withdrawal source which can take the following values:
#' - "CONT" for continental surface
#' - "SOUT" for subsurface
#' - "LIT" for littoral
#' - `codeUsage`: a numeric code related to water destination: energy, canals, turbined, AEP, industry or irrigation
#' - `commentaire`: comment
#' - `exploitationDebut`: start date of the
#'
#' @export
#'
#' @examples
#' getOuvrageSeries("OPR0000200109")
getOuvrageSeries <- function(code_sandre,
cfg = config::get(file = system.file("config.yml",
package = "bnpe")),
cookie = getCookie(cfg)) {
params <- list(
code = code_sandre
)
resp <- doQuery(url_path = cfg$url_ouvrage,
params = params,
cfg = cfg,
cookie = cookie)
l <- httr::content(resp, "parsed")[[1]]
l$exploitationDebut <- as.Date(
format(
as.POSIXct(l$exploitationDebut / 1000, origin = "1970-01-01", tz = "CET")
)
)
params <- list(
code_ouvrage = code_sandre,
ecrasant = "false"
)
resp <- doQuery(url_path = cfg$url_ouv_series,
params = params,
cfg = cfg,
cookie = cookie)
lTS <- httr::content(resp, "parsed")
hasVolume <- sapply(lTS, function(x) { !is.null(x$volume) })
annees <- sapply(lTS, "[[", "annee")
volumes <- unlist(sapply(lTS, "[[", "volume"))
peuplements <- unlist(sapply(lTS, "[[", "peuplement"))
l$TS <- data.frame(annee = annees[hasVolume],
volume = volumes,
peuplement = peuplements[hasVolume])
return(l)
}
R/getTimeSeries.R deleted 100644 → 0
View file @ 9d2507fa
#' Get withdrawal time series for a geographical entity
#' @param insee_com a [character] of length 5 representing the INSEE code of the commune
#' @template param_get_common
#'
#' @return a [data.frame] with one year by row and the following columns:
#' - "volume" the withdrawal annual volume in cubic meters
#' - "annee" the year
#' - "peuplement" a unknown code
#' @export
#' @rdname getTimeSeries
#'
#' @examples
#' # Withdrawal time series for Ardennes department
#' getTimeSeriesDep("08")
#'
#' # Withdrawal time series for the commune of Charleville-Mézières
#' getTimeSeriesCom("08105")
#'
getTimeSeriesCom <- function(insee_com,
code_usage = NULL,
code_type_eau = NULL,
cfg = config::get(file = system.file("config.yml", package = "bnpe")),
cookie = getCookie(cfg)) {
getTimeSeries(level = "commune",
id_name = "insee_com",
id_value = insee_com,
commune_adj = "false",
cfg = cfg,
cookie = cookie)
}
#' @param insee_dep a [character] of length 2 representing the INSEE code of the commune
#' @rdname getTimeSeries
#' @export
getTimeSeriesDep <- function(insee_dep,
code_usage = NULL,
code_type_eau = NULL,
cfg = config::get(file = system.file("config.yml", package = "bnpe")),
cookie = getCookie(cfg)) {
getTimeSeries(level ="departement",
id_name = "code_dep",
id_value = insee_dep,
code_usage = code_usage,
code_type_eau = code_type_eau,
cfg = cfg,
cookie = cookie)
}
#' @param level [character] geographical level ("commune", "departement", ...)
#' @param id_name geographical identifier name ("insee_com", "code_dep", ...)
#' @param id_value geographical identifier value ("08105", "08", ...)
#' @inherit getTimeSeries
#' @noRd
getTimeSeries <-
function(level,
id_name,
id_value,
code_usage = NULL,
code_type_eau = NULL,
commune_adj = NULL,
cfg = config::get(file = system.file("config.yml", package = "bnpe")),
cookie = getCookie(cfg)) {
params <- list(
id = id_value,
code_usage = code_usage,
code_type_eau = code_type_eau,
commune_adj = commune_adj
)
names(params)[names(params) == "id"] <- id_name
resp <- doQuery(url_path = paste(cfg$url_time_series, level, sep = "_"),
params = params, cfg = cfg, cookie = cookie)
# return a data.frame
do.call(rbind, httr::content(resp, "parsed"))
}
R/get_chronique.R → R/get_prelevements_chronique.R
View file @ 85145070
......@@ -3,17 +3,18 @@
#' See the API documentation for available filter parameters: \url{https://hubeau.eaufrance.fr/page/api-prelevements-eau#/prelevements/chronique}
#'
#' @param params [list] where the keys are the names of the filtered parameters and the values are the values of the filters. See the API documentation for the complete list of available filter parameters
#' @inheritParams getCookie
#' @param cfg a [config] object Configuration of the communication. Use by default the internal package
#' configuration
#'
#' @return a [tibble::tibble] with all available parameters in columns and one row by device, year and usage.
#' @export
#'
#' @examples
#' # For retrieving the withdrawal time series of the devices located in Romilly-sur-Seine
#' get_chronique(list(code_commune_insee = "10323"))
#' get_prelevements_chronique(list(code_commune_insee = "10323"))
#'
get_chronique <- function(params, cfg = config::get(file = system.file("config.yml",
package = "bnpe"))) {
get_prelevements_chronique <- function(params, cfg = config::get(file = system.file("config.yml",
package = "hubeau"))) {
l <- doApiQuery(cfg$api_prelevements$chronique, params)
l <- lapply(l, function(x) {
x$geometry <- NULL
......
R/get_points_prelevement.R → R/get_prelevements_prelevement.R
View file @ 85145070
......@@ -3,17 +3,18 @@
#' See the API documentation for available filter parameters: \url{https://hubeau.eaufrance.fr/page/api-prelevements-eau#/prelevements/prelevement}
#'
#' @param params [list] where the keys are the names of the filtered parameters and the values are the values of the filters. See the API documentation for the complete list of available filter parameters
#' @inheritParams getCookie
#' @param cfg a [config] object Configuration of the communication. Use by default the internal package
#' configuration
#'
#' @return a [tibble::tibble] with all available parameters in columns and one row by device.
#' @export
#'
#' @examples
#' # For retrieving the withdrawal points located in Romilly-sur-Seine
#' get_points_prelevement(list(code_commune_insee = "10323"))
#' get_prelevements_prelevement(list(code_commune_insee = "10323"))
#'
get_points_prelevement <- function(params, cfg = config::get(file = system.file("config.yml",
package = "bnpe"))) {
get_prelevements_prelevement <- function(params, cfg = config::get(file = system.file("config.yml",
package = "hubeau"))) {
l <- doApiQuery(cfg$api_prelevements$points_prelevement, params)
l <- lapply(l, function(row) {lapply(row, function(cell) { if(is.null(unlist(cell))) NA else unlist(cell) })})
return(purrr::map_df(l, tibble::as_tibble))
......
README.Rmd
View file @ 85145070
......@@ -13,38 +13,36 @@ knitr::opts_chunk$set(
)
```
# bnpe
# hubeau
<!-- badges: start -->
`r badger::badge_license(color = "orange")`
`r badger::badge_lifecycle("experimental", color = "blue")`
<!-- badges: end -->
'bnpe' is an R-package proposing a collection of function to help retrieve the French national data bank of quantitative withdrawals (Banque Nationale des Prélèvements quantitatifs en Eau - BNPE) available on its website https://bnpe.eaufrance.fr and on https://hubeau.eaufrance.fr/page/api-prelevements-eau
'hubeau' is an R-package proposing a collection of function to help retrieve the French national data bank of quantitative withdrawals (Banque Nationale des Prélèvements quantitatifs en Eau - BNPE) available on its website https://hubeau.eaufrance.fr and on https://hubeau.eaufrance.fr/page/api-prelevements-eau
## Installation
You can install the bnpe R-package from it's development repository with:
You can install the hubeau R-package from it's development repository with:
``` r
install.packages("remotes")
remotes::install_gitlab("in-wop/bnpe", host = "gitlab.irstea.fr")
remotes::install_gitlab("in-wop/hubeau", host = "gitlab.irstea.fr")
```
## Get started
### Loading library
## Loading library
```{r library}
library(bnpe)
library(hubeau)
```
### Loading data from the Hub'Eau API
## Loading data from the Hub'Eau API
Two functions are available:
- `get_points_prelevement`: retrieve data about abstraction points
- `get_chronique`: retrieve annual volume time series and characteristics
- `get_prelevements_prelevement`: retrieve data about abstraction points
- `get_prelevements_chronique`: retrieve annual volume time series and characteristics
The available filters for these functions are detailed in the API documentation respectively: https://hubeau.eaufrance.fr/page/api-prelevements-eau#/prelevements/prelevement and https://hubeau.eaufrance.fr/page/api-prelevements-eau#/prelevements/chronique
......@@ -52,56 +50,14 @@ Examples:
```{r}
# Characteristics of surface abstraction points in the Ardennes departement
pp08 <- get_points_prelevement(list(code_departement = "08", code_type_milieu = "CONT"))
pp08 <- get_prelevements_prelevement(list(code_departement = "08", code_type_milieu = "CONT"))
str(pp08)
# Time series of annual abstracted volumes for drinking water supply from surface water in the Ardennes departement
# As the parameter "source of the water" (ground, surface...) is not available here, we can use the list of abstraction points previously downloaded as filter:
dfAEP <- get_chronique(list(code_ouvrage = paste(pp08$code_ouvrage, collapse = ","),
dfAEP <- get_prelevements_chronique(list(code_ouvrage = paste(pp08$code_ouvrage, collapse = ","),
code_usage = "AEP"))
str(dfAEP)
```
### Scrapping aggregated data from the BNPE website
Functions for getting time series are:
- `getTimeSeriesDep`: for one department
- `getTimeSeriesCom`: for one commune
These functions uses the French official geographical codification (https://fr.wikipedia.org/wiki/Code_officiel_g%C3%A9ographique).
Type `?getTimeSeriesCom` for getting help.
Examples:
```{r timeseries}
# Time series for all type of withdrawal sources and destinations in the Ardennes departement
getTimeSeriesDep("08")
# Time series for drinking water withdrawals in surface water bodies in the Ardennes department
getTimeSeriesDep("08", code_usage = "AEP", code_type_eau = "CONT")
# Withdrawal time series for the commune of Charleville-Mézières
getTimeSeriesCom("08105")
```
### Communal data in one departement
For getting list of communes of departement with corresponding withdrawal volumes for one year, use the function `getComSeriesDep` as follow:
```{r}
# All source and destination withdrawals for the year 2016 in the Ardennes departement
getComSeriesDep("08", 2016, code_usage = "AEP", code_type_eau = "CONT")
```
### Properties and time series from one device ("ouvrage")
For getting metadata and withdrawal yearly time series from on device identified by its "code Sandre" (See https://www.sandre.eaufrance.fr/atlas/srv/fre/catalog.search#/metadata/e315633f-0930-41e8-a1c4-61fb2303039c):
```{r}
# metadata and timeseries from withdrawal "COPRIMANCHE-forage LD LE PACO (40m)"
ouvrage <- getOuvrageSeries("OPR0000200109")
str(ouvrage)
```
README.md
View file @ 85145070
<!-- README.md is generated from README.Rmd. Please edit that file -->
# bnpe
# hubeau
<!-- badges: start -->
......@@ -10,36 +10,35 @@ MIT](https://img.shields.io/badge/license-MIT-orange.svg)](https://cran.r-projec
[![](https://img.shields.io/badge/lifecycle-experimental-blue.svg)](https://lifecycle.r-lib.org/articles/stages.html#experimental)
<!-- badges: end -->
‘bnpe’ is an R-package proposing a collection of function to help
‘hubeau’ is an R-package proposing a collection of function to help
retrieve the French national data bank of quantitative withdrawals
(Banque Nationale des Prélèvements quantitatifs en Eau - BNPE) available
on its website <https://bnpe.eaufrance.fr> and on
on its website <https://hubeau.eaufrance.fr> and on
<https://hubeau.eaufrance.fr/page/api-prelevements-eau>
## Installation
You can install the bnpe R-package from it’s development repository
You can install the hubeau R-package from it’s development repository
with:
``` r
install.packages("remotes")
remotes::install_gitlab("in-wop/bnpe", host = "gitlab.irstea.fr")
remotes::install_gitlab("in-wop/hubeau", host = "gitlab.irstea.fr")
```
## Get started
### Loading library
## Loading library
``` r
library(bnpe)
library(hubeau)
```
### Loading data from the Hub’Eau API
## Loading data from the Hub’Eau API
Two functions are available:
- `get_points_prelevement`: retrieve data about abstraction points
- `get_chronique`: retrieve annual volume time series and
- `get_prelevements_prelevement`: retrieve data about abstraction
points
- `get_prelevements_chronique`: retrieve annual volume time series and
characteristics
The available filters for these functions are detailed in the API
......@@ -52,7 +51,7 @@ Examples:
``` r
# Characteristics of surface abstraction points in the Ardennes departement
pp08 <- get_points_prelevement(list(code_departement = "08", code_type_milieu = "CONT"))
pp08 <- get_prelevements_prelevement(list(code_departement = "08", code_type_milieu = "CONT"))
str(pp08)
#> tibble [85 x 28] (S3: tbl_df/tbl/data.frame)
#> $ code_point_prelevement : chr [1:85] "PTP000000000005792" "PTP000000000005793" "PTP000000000005794" "PTP000000000005795" ...
......@@ -86,7 +85,7 @@ str(pp08)
# Time series of annual abstracted volumes for drinking water supply from surface water in the Ardennes departement
# As the parameter "source of the water" (ground, surface...) is not available here, we can use the list of abstraction points previously downloaded as filter:
dfAEP <- get_chronique(list(code_ouvrage = paste(pp08$code_ouvrage, collapse = ","),
dfAEP <- get_prelevements_chronique(list(code_ouvrage = paste(pp08$code_ouvrage, collapse = ","),
code_usage = "AEP"))
str(dfAEP)
#> tibble [40 x 23] (S3: tbl_df/tbl/data.frame)
......@@ -114,115 +113,3 @@ str(dfAEP)
#> $ nom_ouvrage : chr [1:40] "COMMUNE DE GIVET" "COMMUNE DE GIVET" "COMMUNE DE GIVET" "COMMUNE DE GIVET" ...
#> $ uri_ouvrage : chr [1:40] "https://id.eaufrance.fr/OuvragePrel/OPR0000000767" "https://id.eaufrance.fr/OuvragePrel/OPR0000000767" "https://id.eaufrance.fr/OuvragePrel/OPR0000000767" "https://id.eaufrance.fr/OuvragePrel/OPR0000000767" ...
```
### Scrapping aggregated data from the BNPE website
Functions for getting time series are:
- `getTimeSeriesDep`: for one department
- `getTimeSeriesCom`: for one commune
These functions uses the French official geographical codification
(<https://fr.wikipedia.org/wiki/Code_officiel_g%C3%A9ographique>).
Type `?getTimeSeriesCom` for getting help.
Examples:
``` r
# Time series for all type of withdrawal sources and destinations in the Ardennes departement
getTimeSeriesDep("08")
#> volume annee peuplement
#> [1,] NULL "2008" "2"
#> [2,] NULL "2009" "2"
#> [3,] NULL "2010" "2"
#> [4,] NULL "2011" "2"
#> [5,] 750878007 "2012" "1"
#> [6,] 1070936432 "2013" "1"
#> [7,] 930232502 "2014" "1"
#> [8,] 918264038 "2015" "1"
#> [9,] 899230351 "2016" "1"
#> [10,] 790975515 "2017" "1"
#> [11,] 324131543 "2018" "1"
#> [12,] 163293169 "2019" "3"
# Time series for drinking water withdrawals in surface water bodies in the Ardennes department
getTimeSeriesDep("08", code_usage = "AEP", code_type_eau = "CONT")
#> volume annee peuplement
#> [1,] NULL "2008" "2"
#> [2,] NULL "2009" "2"
#> [3,] NULL "2010" "2"
#> [4,] NULL "2011" "2"
#> [5,] 506356 "2012" "1"
#> [6,] 479622 "2013" "1"
#> [7,] 599626 "2014" "1"
#> [8,] 631363 "2015" "1"
#> [9,] 581305 "2016" "1"
#> [10,] 1134831 "2017" "1"
#> [11,] 504790 "2018" "1"
#> [12,] 380556 "2019" "3"
# Withdrawal time series for the commune of Charleville-Mézières
getTimeSeriesCom("08105")
#> volume annee peuplement
#> [1,] NULL "2008" "2"
#> [2,] NULL "2009" "2"
#> [3,] NULL "2010" "2"
#> [4,] NULL "2011" "2"
#> [5,] 55898 "2012" "1"
#> [6,] 15359 "2013" "1"
#> [7,] 79086 "2014" "1"
#> [8,] 100624 "2015" "1"
#> [9,] 16359 "2016" "1"
#> [10,] 17162 "2017" "1"
#> [11,] 67910 "2018" "1"
#> [12,] 512130 "2019" "1"
```
### Communal data in one departement
For getting list of communes of departement with corresponding
withdrawal volumes for one year, use the function `getComSeriesDep` as
follow:
``` r
# All source and destination withdrawals for the year 2016 in the Ardennes departement
getComSeriesDep("08", 2016, code_usage = "AEP", code_type_eau = "CONT")
#> insee_com volume
#> 1 08096 33024
#> 2 08135 13583
#> 3 08247 439424
#> 4 08420 95274
```
### Properties and time series from one device (“ouvrage”)