Commit 03ce110a authored by François Bourgin's avatar François Bourgin
Browse files

Merge branch '1-evaluation-of-quantiles-prediction' into 'dev'

Resolve "evaluation of quantiles prediction"

See merge request !4

Closes #1
1 merge request!4Resolve "evaluation of quantiles prediction"
Pipeline #63783 passed with stage
in 4 minutes and 8 seconds
Showing with 54 additions and 0 deletions
+54 -0
...@@ -168,6 +168,9 @@ struct EvalPInterface ...@@ -168,6 +168,9 @@ struct EvalPInterface
// optionals related to intervals-based metrics // optionals related to intervals-based metrics
std::vector<double> lvl = {}; std::vector<double> lvl = {};
// optionals related to quantile levels
std::vector<double> qlvl = {};
// optionals related to temporal subsetting // optionals related to temporal subsetting
std::string msk; std::string msk;
std::string cdt; std::string cdt;
...@@ -441,6 +444,7 @@ void run_evalp(EvalPInterface const &opt) ...@@ -441,6 +444,7 @@ void run_evalp(EvalPInterface const &opt)
? opt.events.value() ? opt.events.value()
: xtl::missing<std::string>(), : xtl::missing<std::string>(),
opt.lvl, opt.lvl,
opt.qlvl,
masks, masks,
conditions, conditions,
(opt.bootstrap.has_value()) (opt.bootstrap.has_value())
...@@ -605,6 +609,9 @@ void setup_evalp(CLI::App &app) ...@@ -605,6 +609,9 @@ void setup_evalp(CLI::App &app)
sub->add_option("--c_lvl", opt->lvl, sub->add_option("--c_lvl", opt->lvl,
"Confidence levels to use for intervals-based metrics"); "Confidence levels to use for intervals-based metrics");
sub->add_option("--q_lvl", opt->qlvl,
"Quantile levels to use for quantiles prediction");
sub->add_option("--t_msk", opt->msk, sub->add_option("--t_msk", opt->msk,
"Path to temporal masks folder") "Path to temporal masks folder")
->check(CLI::ExistingDirectory); ->check(CLI::ExistingDirectory);
......
2520,2270,2130,1950,1820,1680,1590,1560,1570,1450,1410,1370,1350,1280,1230,1290,1320,1210,1160,1120,1130,1120,1100,1070,1050,1060,1590,2280,2300,2480,2250,2800,3490,9270,7180,4210,3160,2600,2260,2630,5160,5430,4420,8780,5270,3890,3840,4120,3410,2840,2530,2200,1960,1810,1820,2470,2980,3710,3160,4330,3760,4300,3200,9720,10200,5410,3980,4560,4290,3560,3140,2860,2530,2300,2120,1890,1770,1690,1620,1550,1490,1380,1320,1290,1270,1230,1200,1210,1430,1220,1200,1250,1200,1100,1080,1060,1030,1000,977,965,944,924,910,873,829,805,776,753,735,728,706,689,661,638,635,629,639,725,712,870,856,1120,886,722,738,997,723,642,610,593,1930,1740,1470,3100,1740,1190,949,798,725,653,618,590,3210,2120,1080,864,749,660,608,585,563,538,504,494,1070,717,660,803,654,666,796,899,843,698,1040,681,804,2810,1630,1360,1000,768,631,595,548,498,523,604,568,691,509,459,564,468,421,414,406,398,385,350,329,322,332,322,302,315,378,420,374,309,262,236,234,230,215,218,206,202,200,199,196,193,207,651,529,466,347,279,252,226,205,179,192,200,215,236,281,217,340,314,282,228,195,192,199,186,177,161,157,395,509,1020,704,394,354,299,268,245,258,277,255,241,225,223,229,227,224,220,218,228,241,242,257,249,261,309,495,675,416,398,360,729,491,402,467,1200,622,424,420,552,447,374,657,588,593,480,419,409,723,731,744,565,503,468,455,1270,1730,967,2290,1790,1090,946,791,734,719,681,758,934,814,737,707
2247,2044,2037,1825,1699,1600,1501,1432,1440,1388,1299,1259,1211,1177,1124,1121,1399,1179,1102,1055,1017,1001,979,953,925,927,1899,4782,2601,3050,1998,3478,5762,8745,7777,4086,2968,2456,2138,3199,7187,7165,3942,11588,4643,3618,3184,6052,3723,3356,2645,2330,2103,1928,1890,3624,3647,5821,2949,4161,3855,5200,3162,3896,19818,5228,3711,4874,4868,4267,2978,2748,2500,2276,2107,1969,1815,1714,1649,1561,1491,1428,1347,1288,1245,1207,1163,1189,1558,1313,1186,1147,1143,1099,1036,1008,982,953,924,899,880,858,836,818,791,762,742,720,701,685,674,658,643,625,609,602,594,594,625,616,707,759,929,755,684,737,804,680,639,620,706,2474,1406,2686,2037,1410,1130,992,901,847,798,766,750,2793,1574,1087,960,883,823,782,755,730,705,677,744,932,770,794,804,728,721,765,826,795,784,875,711,795,2295,1265,1115,936,817,742,713,680,647,647,671,651,692,605,576,612,564,536,526,516,506,494,473,459,451,450,441,428,430,453,467,444,413,390,376,373,368,360,359,351,348,345,343,340,337,341,529,476,448,397,367,354,342,332,320,324,327,332,340,359,331,382,371,356,333,318,316,318,312,308,300,298,399,448,669,534,402,384,361,347,337,342,349,339,333,325,324,326,324,322,320,318,322,327,327,333,329,333,353,436,516,412,402,398,551,449,416,484,801,537,457,483,534,477,483,582,575,557,504,477,515,643,627,627,546,515,497,577,1439,1198,1591,1815,1448,1117,960,847,809,788,749,896,870,801,752
2247,2063,2041,1825,1699,1600,1501,1446,1440,1388,1299,1259,1211,1177,1124,1192,1399,1183,1102,1055,1017,1001,979,953,929,981,1999,4851,2699,3050,2052,3680,5984,8953,7777,4086,2968,2456,2138,3765,7372,7173,4306,11588,4643,3618,3342,6076,3786,3357,2645,2330,2103,1928,1969,3710,3771,5821,3041,4161,4093,5200,3162,4227,19932,5228,3851,4907,5010,4267,2978,2748,2500,2276,2107,1969,1815,1714,1699,1561,1491,1428,1347,1288,1245,1207,1163,1240,1558,1313,1186,1147,1143,1099,1036,1008,982,953,924,899,880,858,836,818,791,762,742,720,701,685,674,658,643,625,609,602,594,597,625,617,720,768,929,755,699,737,804,680,639,620,749,2474,1479,2686,2037,1410,1130,992,901,847,798,766,777,2793,1574,1087,960,883,823,782,755,730,705,678,744,932,779,794,804,734,724,765,826,863,784,875,711,809,2295,1265,1115,936,817,742,713,680,647,647,673,651,692,605,576,612,564,536,526,516,506,494,473,459,451,450,441,428,430,453,467,444,413,390,376,373,368,360,359,351,348,345,343,340,337,341,530,476,448,397,367,354,342,332,320,324,327,332,340,359,331,382,371,356,333,318,316,318,312,308,300,298,399,448,669,534,402,384,361,347,337,342,349,339,333,325,324,326,324,322,320,318,322,327,327,333,329,333,353,437,517,412,404,398,551,449,420,488,801,539,462,483,534,479,485,584,575,557,504,481,518,643,627,627,546,515,502,593,1439,1255,1613,1815,1463,1117,960,847,809,788,758,896,870,801,752
2247,2074,2045,1825,1699,1600,1501,1451,1441,1388,1299,1259,1211,1177,1124,1238,1399,1187,1102,1055,1017,1001,979,953,932,1023,2144,4885,2743,3050,2084,3746,6108,9050,7777,4086,2968,2456,2138,3944,7442,7190,4453,11588,4643,3618,3478,6109,3806,3371,2645,2330,2103,1928,2019,3776,3811,5821,3109,4219,4230,5209,3179,4436,20023,5228,3913,4928,5053,4267,2978,2748,2500,2276,2107,1969,1815,1714,1737,1561,1491,1428,1347,1288,1245,1207,1163,1281,1558,1313,1186,1147,1143,1099,1036,1008,982,953,924,899,880,858,836,818,791,762,742,720,701,685,674,658,643,625,609,602,594,601,625,618,733,774,929,755,725,737,804,680,639,620,789,2474,1529,2686,2037,1410,1130,992,901,847,798,766,789,2793,1574,1087,960,883,823,782,755,730,705,680,744,932,784,794,804,743,726,765,826,947,784,875,713,816,2295,1265,1115,936,817,742,713,680,647,647,675,651,692,605,576,612,564,536,526,516,506,494,474,459,451,450,441,428,430,453,467,444,413,390,376,373,368,360,359,351,348,345,343,340,337,342,530,476,448,397,367,354,342,332,320,324,327,332,340,359,331,382,371,356,333,318,316,318,312,308,300,298,400,449,669,534,402,384,361,347,337,342,349,339,333,325,324,326,324,322,320,318,322,327,327,333,329,333,353,437,517,413,405,398,551,450,424,491,801,540,466,484,534,482,486,585,575,557,504,482,519,643,627,627,546,515,506,603,1439,1293,1626,1815,1472,1121,960,847,811,788,763,896,870,801,752
2247,2177,2056,1825,1705,1600,1501,1477,1448,1388,1299,1259,1211,1177,1124,1420,1411,1206,1102,1055,1017,1001,984,954,963,1426,2614,4959,2923,3050,2212,4284,6685,9417,7782,4086,2968,2456,2284,4451,7676,7273,5053,11664,4643,3618,4118,6230,3886,3396,2645,2330,2130,1928,2211,4371,3940,5900,3487,4420,4600,5290,3263,5401,20348,5240,4273,5090,5561,4275,2990,2748,2500,2276,2107,1969,1815,1714,1890,1571,1491,1428,1347,1288,1245,1207,1163,1376,1558,1313,1186,1147,1143,1099,1036,1008,982,953,924,899,880,858,836,818,791,762,742,720,701,685,674,658,643,625,609,602,601,621,626,630,821,830,929,767,891,737,804,680,646,624,1025,2481,1666,2693,2037,1410,1130,992,901,847,798,766,847,2793,1574,1087,960,883,823,782,755,730,705,704,754,937,819,802,815,792,746,766,826,1983,784,887,737,886,2337,1290,1117,936,817,742,713,680,664,647,697,651,692,612,578,612,564,536,526,516,506,494,476,459,451,450,441,429,430,453,467,444,413,390,376,373,368,360,359,351,348,345,343,340,337,342,535,476,448,397,367,354,342,332,320,324,327,333,340,359,331,383,371,356,333,318,316,318,312,308,300,299,400,450,670,534,402,384,361,347,337,342,349,339,333,325,324,326,324,322,320,318,322,327,327,333,329,333,354,438,520,413,410,399,551,451,454,503,806,549,504,488,534,487,494,591,578,558,504,489,524,644,627,627,546,515,521,677,1439,1390,1718,1848,1504,1155,960,852,836,896,792,910,870,801,752
0.00643087,0.0514469
...@@ -6,10 +6,14 @@ import tempfile ...@@ -6,10 +6,14 @@ import tempfile
_obs = "./data/evalp/obs" _obs = "./data/evalp/obs"
_obs_qlvl = "./data/evalp/obs_qlvl"
_prd = "./data/evalp/prd" _prd = "./data/evalp/prd"
_prd_qlvl = "./data/evalp/prd_qlvl"
_thr = "./data/evalp/thr" _thr = "./data/evalp/thr"
_events = "high" _events = "high"
_lvl = ("30.", "80.") _lvl = ("30.", "80.")
_lvl_qlvl = ("50.", "80.")
_qlvl = ("10.", "25.", "75.", "90.")
_msk = "./data/evalp/msk" _msk = "./data/evalp/msk"
_cdt = "./data/evalp/cdt" _cdt = "./data/evalp/cdt"
...@@ -84,6 +88,13 @@ class TestMetrics(unittest.TestCase): ...@@ -84,6 +88,13 @@ class TestMetrics(unittest.TestCase):
) for metric in ('CR', 'AW', 'AWN', 'WS') ) for metric in ('CR', 'AW', 'AWN', 'WS')
} }
expected_itv_qlvl = {
metric: (
numpy.genfromtxt(f"./expected/evalp/{metric}_QLVL.csv", delimiter=',')
[numpy.newaxis, numpy.newaxis, numpy.newaxis, numpy.newaxis, ...]
) for metric in ('CR',)
}
expected_mvr = { expected_mvr = {
metric: ( metric: (
numpy.genfromtxt(f"./expected/evalp/{metric}.csv", delimiter=',') numpy.genfromtxt(f"./expected/evalp/{metric}.csv", delimiter=',')
...@@ -259,6 +270,36 @@ class TestMetrics(unittest.TestCase): ...@@ -259,6 +270,36 @@ class TestMetrics(unittest.TestCase):
6) 6)
) )
def test_intervals_metrics_qlvl_to_file(self):
# clean up outputs directory to make sure
# it is checking newly created files
subprocess.run(['rm -rf ./outputs/*/*.csv'], shell=True)
# compute all metrics with evalhyd evald
subprocess.run(
['evalhyd', 'evalp', _obs_qlvl, _prd_qlvl,
*tuple(self.expected_itv_qlvl.keys()),
'--c_lvl', *_lvl_qlvl,
'--q_lvl', *_qlvl,
'--to_file', '--out_dir', './outputs']
)
# read in and check data in CSV files
for metric in self.expected_itv_qlvl.keys():
for lt in ['leadtime_1']:
for st in ['site_a']:
res = numpy.genfromtxt(
f"./outputs/{lt}/{st}_{metric}.csv", delimiter=','
)
with self.subTest(metric=metric, leadtime=lt, site=st):
numpy.testing.assert_almost_equal(
res, self.expected_itv_qlvl[metric][0, 0, 0, 0],
decimal=(2 if metric == 'WS' else
4 if metric == 'AW' else
6)
)
def test_multivariate_metrics_to_file(self): def test_multivariate_metrics_to_file(self):
# clean up outputs directory to make sure # clean up outputs directory to make sure
# it is checking newly created files # it is checking newly created files
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment