Commit 66035b12 authored by Thibault Hallouin's avatar Thibault Hallouin
Browse files

update to latest evalhyd version

which adds the temporal masking functionality, and
the sites/leadtimes dimensions to the inputs
No related merge requests found
Showing with 48 additions and 38 deletions
+48 -38
Subproject commit e4b61c01421a021c9d7c89139c0c0ef23d6524ed Subproject commit 187c45ade7e8405296f1137e068d68b76158f67d
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
import numpy import numpy
import pandas import pandas
import evalhyd import evalhyd
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Calculate Brier score and Brier skill score at once ## Calculate Brier score and Brier skill score at once
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
# streamflow observations # streamflow observations (1 site)
obs = numpy.array( obs = numpy.array(
[[4.7, 4.3, 5.5, 2.7, 4.1]] [[4.7, 4.3, 5.5, 2.7, 4.1]]
) )
# streamflow predictions (3 forecast members) # streamflow predictions (1 site, 1 lead time, 3 forecast members)
prd = numpy.array( prd = numpy.array(
[[5.3, 4.2, 5.7, 2.3, 3.1], [[[[5.3, 4.2, 5.7, 2.3, 3.1],
[4.3, 4.2, 4.7, 4.3, 3.3], [4.3, 4.2, 4.7, 4.3, 3.3],
[5.3, 5.2, 5.7, 2.3, 3.9]] [5.3, 5.2, 5.7, 2.3, 3.9]]]]
) )
# streamflow thresholds (2) # streamflow thresholds (2)
thr = [4., 5.] thr = [4., 5.]
print(obs.shape, prd.shape) print(obs.shape, prd.shape)
``` ```
%% Output %% Output
(1, 5) (3, 5) (1, 5) (1, 1, 3, 5)
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
bs, bss = evalhyd.evalp(obs, prd, ['BS', 'BSS'], thr) bs, bss = evalhyd.evalp(obs, prd, ['BS', 'BSS'], thr)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
print( print(
pandas.DataFrame( pandas.DataFrame(
bs, [f"q > {thr[0]}", f"q > {thr[1]}"], bs[0, 0, 0], [f"q > {thr[0]}", f"q > {thr[1]}"],
["Brier score"] ["Brier score"]
) )
) )
``` ```
%% Output %% Output
Brier score Brier score
q > 4.0 0.222222 q > 4.0 0.222222
q > 5.0 0.133333 q > 5.0 0.133333
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
print( print(
pandas.DataFrame( pandas.DataFrame(
bss, [f"q > {thr[0]}", f"q > {thr[1]}"], bss[0, 0, 0], [f"q > {thr[0]}", f"q > {thr[1]}"],
["Brier skill score"] ["Brier skill score"]
) )
) )
``` ```
%% Output %% Output
Brier skill score Brier skill score
q > 4.0 -0.388889 q > 4.0 -0.388889
q > 5.0 0.166667 q > 5.0 0.166667
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Calculate decompositions of Brier score ## Calculate decompositions of Brier score
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
bs_crd, bs_lbd = evalhyd.evalp(obs, prd, ['BS_CRD', 'BS_LBD'], [4., 5.]) bs_crd, bs_lbd = evalhyd.evalp(obs, prd, ['BS_CRD', 'BS_LBD'], [4., 5.])
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
### calibration-refinement decomposition ### calibration-refinement decomposition
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
print( print(
pandas.DataFrame( pandas.DataFrame(
bs_crd, [f"q > {thr[0]}", f"q > {thr[1]}"], bs_crd[0, 0, 0], [f"q > {thr[0]}", f"q > {thr[1]}"],
["reliability", "resolution", "uncertainty"] ["reliability", "resolution", "uncertainty"]
) )
) )
``` ```
%% Output %% Output
reliability resolution uncertainty reliability resolution uncertainty
q > 4.0 0.222222 0.16 0.16 q > 4.0 0.222222 0.16 0.16
q > 5.0 0.033333 0.06 0.16 q > 5.0 0.033333 0.06 0.16
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
#### bs = rel - res + unc? #### bs = rel - res + unc?
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
numpy.allclose( numpy.allclose(
numpy.squeeze(bs), bs, bs_crd[..., 0] - bs_crd[..., 1] + bs_crd[..., 2]
bs_crd[:, 0] - bs_crd[:, 1] + bs_crd[:, 2]
) )
``` ```
%% Output %% Output
True True
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
### likelihood-base rate decomposition ### likelihood-base rate decomposition
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
print( print(
pandas.DataFrame( pandas.DataFrame(
bs_lbd, [f"q > {thr[0]}", f"q > {thr[1]}"], bs_lbd[0, 0, 0], [f"q > {thr[0]}", f"q > {thr[1]}"],
["type 2 bias", "discrimination", "sharpness"] ["type 2 bias", "discrimination", "sharpness"]
) )
) )
``` ```
%% Output %% Output
type 2 bias discrimination sharpness type 2 bias discrimination sharpness
q > 4.0 0.072222 0.027778 0.177778 q > 4.0 0.072222 0.027778 0.177778
q > 5.0 0.072222 0.027778 0.088889 q > 5.0 0.072222 0.027778 0.088889
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
#### bs = t2b - dis + shr? #### bs = t2b - dis + shr?
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
numpy.allclose( numpy.allclose(
numpy.squeeze(bs), bs, bs_lbd[..., 0] - bs_lbd[..., 1] + bs_lbd[..., 2]
bs_lbd[:, 0] - bs_lbd[:, 1] + bs_lbd[:, 2]
) )
``` ```
%% Output %% Output
True True
......
...@@ -41,10 +41,9 @@ ext_modules = [ ...@@ -41,10 +41,9 @@ ext_modules = [
Extension( Extension(
'evalhyd', 'evalhyd',
['src/evalhyd-python.cpp', ['src/evalhyd-python.cpp',
'deps/evalhyd/include/evalhyd/probabilist/evaluator_brier.cpp', 'deps/evalhyd/src/probabilist/evaluator_brier.cpp',
'deps/evalhyd/include/evalhyd/probabilist/evaluator_elements.cpp', 'deps/evalhyd/src/probabilist/evaluator_elements.cpp',
'deps/evalhyd/include/evalhyd/probabilist/evaluator_quantiles.cpp', 'deps/evalhyd/src/probabilist/evaluator_quantiles.cpp'],
'deps/evalhyd/include/evalhyd/probabilist/evaluator_utils.cpp'],
include_dirs=[ include_dirs=[
# Path to pybind11 headers # Path to pybind11 headers
get_pybind_include(), get_pybind_include(),
...@@ -54,6 +53,7 @@ ext_modules = [ ...@@ -54,6 +53,7 @@ ext_modules = [
os.path.join(os.getcwd(), 'deps', 'evalhyd', 'deps', 'xtensor', 'include'), os.path.join(os.getcwd(), 'deps', 'evalhyd', 'deps', 'xtensor', 'include'),
os.path.join(os.getcwd(), 'deps', 'xtensor-python', 'include'), os.path.join(os.getcwd(), 'deps', 'xtensor-python', 'include'),
os.path.join(os.getcwd(), 'deps', 'evalhyd', 'include'), os.path.join(os.getcwd(), 'deps', 'evalhyd', 'include'),
os.path.join(os.getcwd(), 'deps', 'evalhyd', 'src'),
os.path.join(sys.prefix, 'include'), os.path.join(sys.prefix, 'include'),
os.path.join(sys.prefix, 'Library', 'include') os.path.join(sys.prefix, 'Library', 'include')
], ],
......
...@@ -4,8 +4,8 @@ ...@@ -4,8 +4,8 @@
#define FORCE_IMPORT_ARRAY #define FORCE_IMPORT_ARRAY
#include <xtensor-python/pytensor.hpp> #include <xtensor-python/pytensor.hpp>
#include "evalhyd/determinist.hpp" #include "evalhyd/evald.hpp"
#include "evalhyd/probabilist.hpp" #include "evalhyd/evalp.hpp"
namespace py = pybind11; namespace py = pybind11;
...@@ -16,13 +16,6 @@ PYBIND11_MODULE(evalhyd, m) ...@@ -16,13 +16,6 @@ PYBIND11_MODULE(evalhyd, m)
m.doc() = R"pbdoc( m.doc() = R"pbdoc(
Utility for evaluation of streamflow predictions. Utility for evaluation of streamflow predictions.
.. currentmodule:: evalhyd
.. autosummary::
evald
evalp
)pbdoc"; )pbdoc";
// deterministic evaluation // deterministic evaluation
...@@ -35,9 +28,11 @@ PYBIND11_MODULE(evalhyd, m) ...@@ -35,9 +28,11 @@ PYBIND11_MODULE(evalhyd, m)
q_obs: `numpy.ndarray` q_obs: `numpy.ndarray`
1D array of streamflow observations. 1D array of streamflow observations.
shape: (time,)
q_prd: `numpy.ndarray` q_prd: `numpy.ndarray`
1D array of streamflow predictions. 1D array of streamflow predictions.
shape: (time,)
metrics: `List[str]` metrics: `List[str]`
The sequence of evaluation metrics to be computed. The sequence of evaluation metrics to be computed.
...@@ -47,6 +42,7 @@ PYBIND11_MODULE(evalhyd, m) ...@@ -47,6 +42,7 @@ PYBIND11_MODULE(evalhyd, m)
`List[numpy.ndarray]` `List[numpy.ndarray]`
The sequence of evaluation metrics computed The sequence of evaluation metrics computed
in the same order as given in *metrics*. in the same order as given in *metrics*.
shape: [(components,)+]
:Examples: :Examples:
...@@ -74,12 +70,12 @@ PYBIND11_MODULE(evalhyd, m) ...@@ -74,12 +70,12 @@ PYBIND11_MODULE(evalhyd, m)
:Parameters: :Parameters:
q_obs: `numpy.ndarray` q_obs: `numpy.ndarray`
2D array of streamflow observations (with its temporal 2D array of streamflow observations.
dimension on axis 1). shape: (1, time)
q_prd: `numpy.ndarray` q_prd: `numpy.ndarray`
2D array of streamflow predictions (with its temporal 2D array of streamflow predictions.
dimension on axis 1). shape: (1+, time)
metrics: `List[str]` metrics: `List[str]`
The sequence of evaluation metrics to be computed. The sequence of evaluation metrics to be computed.
...@@ -89,6 +85,7 @@ PYBIND11_MODULE(evalhyd, m) ...@@ -89,6 +85,7 @@ PYBIND11_MODULE(evalhyd, m)
`List[numpy.ndarray]` `List[numpy.ndarray]`
The sequence of evaluation metrics computed The sequence of evaluation metrics computed
in the same order as given in *metrics*. in the same order as given in *metrics*.
shape: [(1+, components), ...]
:Examples: :Examples:
...@@ -114,6 +111,10 @@ PYBIND11_MODULE(evalhyd, m) ...@@ -114,6 +111,10 @@ PYBIND11_MODULE(evalhyd, m)
); );
// probabilistic evaluation // probabilistic evaluation
py::list empty_1d;
py::list empty_2d;
empty_2d.append(py::list());
m.def( m.def(
"evalp", evalhyd::evalp, "evalp", evalhyd::evalp,
R"pbdoc( R"pbdoc(
...@@ -122,13 +123,12 @@ PYBIND11_MODULE(evalhyd, m) ...@@ -122,13 +123,12 @@ PYBIND11_MODULE(evalhyd, m)
:Parameters: :Parameters:
q_obs: `numpy.ndarray` q_obs: `numpy.ndarray`
2D array of streamflow observations (with size 1 for 2D array of streamflow observations.
axis 0, and with the temporal dimension on axis 1). shape: (sites, time)
q_prd: `numpy.ndarray` q_prd: `numpy.ndarray`
2D array of streamflow predictions (with the ensemble 4D array of streamflow predictions.
members on axis 0, and with the temporal dimension on shape: (sites, lead times, members, time)
axis 1).
metrics: `List[str]` metrics: `List[str]`
The sequence of evaluation metrics to be computed. The sequence of evaluation metrics to be computed.
...@@ -137,12 +137,24 @@ PYBIND11_MODULE(evalhyd, m) ...@@ -137,12 +137,24 @@ PYBIND11_MODULE(evalhyd, m)
The streamflow threshold(s) to consider for the *metrics* The streamflow threshold(s) to consider for the *metrics*
assessing the prediction of exceedance events. If not assessing the prediction of exceedance events. If not
provided, set to default value as an empty `list`. provided, set to default value as an empty `list`.
shape: (thresholds,)
t_msk: `numpy.ndarray`, optional
2D array of masks to generate temporal subsets of the whole
streamflow time series (where True/False is used for the
time steps to include/discard in a given subset). If not
provided, no subset is performed and only one set of metrics
is returned corresponding to the whole time series. If
provided, as many sets of metrics are returned as they are
masks provided.
shape: (subsets, time)
:Returns: :Returns:
`List[numpy.ndarray]` `List[numpy.ndarray]`
The sequence of evaluation metrics computed The sequence of evaluation metrics computed
in the same order as given in *metrics*. in the same order as given in *metrics*.
shape: [(sites, lead times, subsets, {quantiles,} {thresholds,} {components}), ...]
:Examples: :Examples:
...@@ -171,6 +183,6 @@ PYBIND11_MODULE(evalhyd, m) ...@@ -171,6 +183,6 @@ PYBIND11_MODULE(evalhyd, m)
)pbdoc", )pbdoc",
py::arg("q_obs"), py::arg("q_prd"), py::arg("metrics"), py::arg("q_obs"), py::arg("q_prd"), py::arg("metrics"),
py::arg("q_thr") = py::list() py::arg("q_thr") = empty_1d, py::arg("t_msk") = empty_2d
); );
} }
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment