From 7a23f80e00c596140dbe14de8372c1ef0d9a2fab Mon Sep 17 00:00:00 2001 From: Thibault Hallouin <thibault.hallouin@inrae.fr> Date: Wed, 22 Feb 2023 10:55:45 +0100 Subject: [PATCH] add new metric REL_DIAG --- deps/evalhyd | 2 +- tests/expected/evalp/REL_DIAG.csv | 208 ++++++++++++++++++++++++++++++ tests/test_probabilist.py | 10 +- 3 files changed, 217 insertions(+), 3 deletions(-) create mode 100644 tests/expected/evalp/REL_DIAG.csv diff --git a/deps/evalhyd b/deps/evalhyd index 017726c..b203210 160000 --- a/deps/evalhyd +++ b/deps/evalhyd @@ -1 +1 @@ -Subproject commit 017726c604595dbb87a0458d8190a52b2bf443af +Subproject commit b20321058a877c029040ef5bad0d1298b2cd05b3 diff --git a/tests/expected/evalp/REL_DIAG.csv b/tests/expected/evalp/REL_DIAG.csv new file mode 100644 index 0000000..3568332 --- /dev/null +++ b/tests/expected/evalp/REL_DIAG.csv @@ -0,0 +1,208 @@ +0.0000000000000,0.1060606060606,132.0000000000000 +0.0196078431373,0.0000000000000,0.0000000000000 +0.0392156862745,0.0000000000000,0.0000000000000 +0.0588235294118,0.0000000000000,2.0000000000000 +0.0784313725490,0.0000000000000,0.0000000000000 +0.0980392156863,0.0000000000000,0.0000000000000 +0.1176470588235,0.0000000000000,0.0000000000000 +0.1372549019608,0.0000000000000,0.0000000000000 +0.1568627450980,0.0000000000000,0.0000000000000 +0.1764705882353,0.0000000000000,0.0000000000000 +0.1960784313725,0.0000000000000,0.0000000000000 +0.2156862745098,0.0000000000000,0.0000000000000 +0.2352941176471,0.0000000000000,0.0000000000000 +0.2549019607843,0.0000000000000,0.0000000000000 +0.2745098039216,0.0000000000000,0.0000000000000 +0.2941176470588,0.0000000000000,0.0000000000000 +0.3137254901961,0.0000000000000,0.0000000000000 +0.3333333333333,0.0000000000000,0.0000000000000 +0.3529411764706,0.0000000000000,0.0000000000000 +0.3725490196078,0.0000000000000,0.0000000000000 +0.3921568627451,0.0000000000000,0.0000000000000 +0.4117647058824,0.0000000000000,0.0000000000000 +0.4313725490196,0.0000000000000,0.0000000000000 +0.4509803921569,0.0000000000000,0.0000000000000 +0.4705882352941,0.0000000000000,0.0000000000000 +0.4901960784314,0.0000000000000,0.0000000000000 +0.5098039215686,0.0000000000000,0.0000000000000 +0.5294117647059,0.0000000000000,0.0000000000000 +0.5490196078431,0.0000000000000,0.0000000000000 +0.5686274509804,0.0000000000000,0.0000000000000 +0.5882352941176,0.0000000000000,0.0000000000000 +0.6078431372549,0.0000000000000,0.0000000000000 +0.6274509803922,0.0000000000000,0.0000000000000 +0.6470588235294,0.0000000000000,0.0000000000000 +0.6666666666667,0.0000000000000,0.0000000000000 +0.6862745098039,0.0000000000000,0.0000000000000 +0.7058823529412,0.0000000000000,0.0000000000000 +0.7254901960784,0.0000000000000,0.0000000000000 +0.7450980392157,0.0000000000000,0.0000000000000 +0.7647058823529,0.0000000000000,0.0000000000000 +0.7843137254902,0.0000000000000,0.0000000000000 +0.8039215686275,0.0000000000000,0.0000000000000 +0.8235294117647,0.0000000000000,0.0000000000000 +0.8431372549020,0.0000000000000,0.0000000000000 +0.8627450980392,0.0000000000000,0.0000000000000 +0.8823529411765,0.0000000000000,0.0000000000000 +0.9019607843137,0.0000000000000,0.0000000000000 +0.9215686274510,1.0000000000000,1.0000000000000 +0.9411764705882,0.0000000000000,0.0000000000000 +0.9607843137255,0.0000000000000,0.0000000000000 +0.9803921568627,0.0000000000000,0.0000000000000 +1.0000000000000,0.8920454545455,176.0000000000000 +0.0000000000000,0.0842105263158,95.0000000000000 +0.0196078431373,0.0000000000000,1.0000000000000 +0.0392156862745,0.0000000000000,0.0000000000000 +0.0588235294118,0.0000000000000,0.0000000000000 +0.0784313725490,0.0000000000000,0.0000000000000 +0.0980392156863,0.0000000000000,0.0000000000000 +0.1176470588235,0.0000000000000,0.0000000000000 +0.1372549019608,0.0000000000000,0.0000000000000 +0.1568627450980,0.0000000000000,0.0000000000000 +0.1764705882353,0.0000000000000,0.0000000000000 +0.1960784313725,0.0000000000000,0.0000000000000 +0.2156862745098,0.0000000000000,0.0000000000000 +0.2352941176471,0.0000000000000,0.0000000000000 +0.2549019607843,0.0000000000000,0.0000000000000 +0.2745098039216,0.0000000000000,0.0000000000000 +0.2941176470588,0.0000000000000,0.0000000000000 +0.3137254901961,0.0000000000000,0.0000000000000 +0.3333333333333,0.0000000000000,0.0000000000000 +0.3529411764706,0.0000000000000,0.0000000000000 +0.3725490196078,0.0000000000000,0.0000000000000 +0.3921568627451,0.0000000000000,0.0000000000000 +0.4117647058824,0.0000000000000,0.0000000000000 +0.4313725490196,0.0000000000000,0.0000000000000 +0.4509803921569,0.0000000000000,0.0000000000000 +0.4705882352941,0.0000000000000,0.0000000000000 +0.4901960784314,0.0000000000000,0.0000000000000 +0.5098039215686,0.0000000000000,0.0000000000000 +0.5294117647059,0.0000000000000,0.0000000000000 +0.5490196078431,0.0000000000000,0.0000000000000 +0.5686274509804,0.0000000000000,0.0000000000000 +0.5882352941176,0.0000000000000,0.0000000000000 +0.6078431372549,0.0000000000000,0.0000000000000 +0.6274509803922,0.0000000000000,0.0000000000000 +0.6470588235294,0.0000000000000,0.0000000000000 +0.6666666666667,0.0000000000000,0.0000000000000 +0.6862745098039,0.0000000000000,0.0000000000000 +0.7058823529412,0.0000000000000,0.0000000000000 +0.7254901960784,0.0000000000000,0.0000000000000 +0.7450980392157,0.0000000000000,0.0000000000000 +0.7647058823529,0.0000000000000,0.0000000000000 +0.7843137254902,0.0000000000000,0.0000000000000 +0.8039215686275,0.0000000000000,0.0000000000000 +0.8235294117647,0.0000000000000,0.0000000000000 +0.8431372549020,0.0000000000000,0.0000000000000 +0.8627450980392,0.0000000000000,0.0000000000000 +0.8823529411765,0.0000000000000,0.0000000000000 +0.9019607843137,0.0000000000000,0.0000000000000 +0.9215686274510,0.0000000000000,0.0000000000000 +0.9411764705882,0.0000000000000,0.0000000000000 +0.9607843137255,0.0000000000000,0.0000000000000 +0.9803921568627,0.0000000000000,0.0000000000000 +1.0000000000000,0.9302325581395,215.0000000000000 +0.0000000000000,0.0724637681159,69.0000000000000 +0.0196078431373,1.0000000000000,1.0000000000000 +0.0392156862745,0.0000000000000,0.0000000000000 +0.0588235294118,0.0000000000000,0.0000000000000 +0.0784313725490,0.0000000000000,0.0000000000000 +0.0980392156863,0.0000000000000,0.0000000000000 +0.1176470588235,0.0000000000000,0.0000000000000 +0.1372549019608,0.0000000000000,0.0000000000000 +0.1568627450980,0.0000000000000,0.0000000000000 +0.1764705882353,0.0000000000000,0.0000000000000 +0.1960784313725,0.0000000000000,0.0000000000000 +0.2156862745098,0.0000000000000,0.0000000000000 +0.2352941176471,0.0000000000000,0.0000000000000 +0.2549019607843,0.0000000000000,0.0000000000000 +0.2745098039216,0.0000000000000,0.0000000000000 +0.2941176470588,0.0000000000000,0.0000000000000 +0.3137254901961,0.0000000000000,0.0000000000000 +0.3333333333333,0.0000000000000,0.0000000000000 +0.3529411764706,0.0000000000000,0.0000000000000 +0.3725490196078,0.0000000000000,0.0000000000000 +0.3921568627451,0.0000000000000,0.0000000000000 +0.4117647058824,0.0000000000000,0.0000000000000 +0.4313725490196,0.0000000000000,0.0000000000000 +0.4509803921569,0.0000000000000,0.0000000000000 +0.4705882352941,0.0000000000000,0.0000000000000 +0.4901960784314,0.0000000000000,0.0000000000000 +0.5098039215686,0.0000000000000,0.0000000000000 +0.5294117647059,0.0000000000000,0.0000000000000 +0.5490196078431,0.0000000000000,0.0000000000000 +0.5686274509804,0.0000000000000,0.0000000000000 +0.5882352941176,0.0000000000000,0.0000000000000 +0.6078431372549,0.0000000000000,0.0000000000000 +0.6274509803922,0.0000000000000,0.0000000000000 +0.6470588235294,0.0000000000000,0.0000000000000 +0.6666666666667,0.0000000000000,0.0000000000000 +0.6862745098039,0.0000000000000,0.0000000000000 +0.7058823529412,0.0000000000000,0.0000000000000 +0.7254901960784,0.0000000000000,0.0000000000000 +0.7450980392157,0.0000000000000,0.0000000000000 +0.7647058823529,0.0000000000000,0.0000000000000 +0.7843137254902,0.0000000000000,0.0000000000000 +0.8039215686275,0.0000000000000,0.0000000000000 +0.8235294117647,0.0000000000000,0.0000000000000 +0.8431372549020,0.0000000000000,0.0000000000000 +0.8627450980392,0.0000000000000,0.0000000000000 +0.8823529411765,0.0000000000000,0.0000000000000 +0.9019607843137,0.0000000000000,0.0000000000000 +0.9215686274510,0.0000000000000,0.0000000000000 +0.9411764705882,0.0000000000000,0.0000000000000 +0.9607843137255,0.0000000000000,0.0000000000000 +0.9803921568627,0.0000000000000,0.0000000000000 +1.0000000000000,0.9128630705394,241.0000000000000 +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan +nan,nan,nan diff --git a/tests/test_probabilist.py b/tests/test_probabilist.py index ac28a8c..c65da24 100644 --- a/tests/test_probabilist.py +++ b/tests/test_probabilist.py @@ -14,7 +14,7 @@ _obs = numpy.genfromtxt("./data/q_obs.csv", delimiter=',')[numpy.newaxis, :] # list all available probabilistic metrics _all_metrics = ( # threshold-based - 'BS', 'BSS', 'BS_CRD', 'BS_LBD', + 'BS', 'BSS', 'BS_CRD', 'BS_LBD', 'REL_DIAG', # quantile-based 'QS', 'CRPS', # contingency table-based @@ -32,8 +32,14 @@ class TestMetrics(unittest.TestCase): metric: ( numpy.genfromtxt(f"./expected/evalp/{metric}.csv", delimiter=',') [numpy.newaxis, numpy.newaxis, numpy.newaxis, numpy.newaxis, ...] - ) for metric in ('BS', 'BSS', 'BS_CRD', 'BS_LBD') + ) for metric in ('BS', 'BSS', 'BS_CRD', 'BS_LBD', 'REL_DIAG') } + # /!\ stacked-up thresholds in CSV file for REL_DIAG + # because 7D metric so need to reshape array + expected_thr['REL_DIAG'] = ( + expected_thr['REL_DIAG'].reshape(expected_thr['BS'].shape + + (_prd.shape[2] + 1, 3)) + ) expected_qtl = { metric: ( -- GitLab