From a4b56e9ea9bb89d0e167693486b234e69c2ec584 Mon Sep 17 00:00:00 2001
From: Le Roux Erwan <erwan.le-roux@irstea.fr>
Date: Thu, 28 Feb 2019 10:57:45 +0100
Subject: [PATCH] [SCM] ensure that each daily_series are indeed daily series.
add test
---
.../meteo_france_SCM_study/abstract_study.py | 2 +-
.../abstract_variable.py | 3 +-
.../meteo_france_SCM_study/crocus/crocus.py | 6 +++-
.../crocus/crocus_variables.py | 26 +++++++++++------
.../meteo_france_SCM_study/main_visualize.py | 4 +--
.../meteo_france_SCM_study/safran/safran.py | 8 ++++++
.../safran/safran_snowfall_variable.py | 6 ++--
.../safran/safran_visualizer.py | 8 ++++--
.../__init__.py | 0
.../test_meteo_france_SCM_study/__init__.py | 0
.../test_SCM_study.py | 12 ++++++--
.../test_safran_gev.py | 0
test/test_utils.py | 28 +++++++++++++++----
13 files changed, 78 insertions(+), 25 deletions(-)
rename test/{test_meteo_france_SCM_study => test_experiment}/__init__.py (100%)
create mode 100644 test/test_experiment/test_meteo_france_SCM_study/__init__.py
rename test/{ => test_experiment}/test_meteo_france_SCM_study/test_SCM_study.py (69%)
rename test/{ => test_experiment}/test_meteo_france_SCM_study/test_safran_gev.py (100%)
diff --git a/experiment/meteo_france_SCM_study/abstract_study.py b/experiment/meteo_france_SCM_study/abstract_study.py
index 7b60a4c4..c47ee107 100644
--- a/experiment/meteo_france_SCM_study/abstract_study.py
+++ b/experiment/meteo_france_SCM_study/abstract_study.py
@@ -67,7 +67,7 @@ class AbstractStudy(object):
return year_to_annual_maxima
def instantiate_variable_object(self, dataset) -> AbstractVariable:
- return self.variable_class(dataset)
+ return self.variable_class(dataset, self.altitude)
""" Private methods to be overwritten """
diff --git a/experiment/meteo_france_SCM_study/abstract_variable.py b/experiment/meteo_france_SCM_study/abstract_variable.py
index 777d7f21..d60cdfc3 100644
--- a/experiment/meteo_france_SCM_study/abstract_variable.py
+++ b/experiment/meteo_france_SCM_study/abstract_variable.py
@@ -4,8 +4,9 @@ class AbstractVariable(object):
NAME = ''
- def __init__(self, dataset):
+ def __init__(self, dataset, altitude):
self.dataset = dataset
+ self.altitude = altitude
@property
def daily_time_serie(self):
diff --git a/experiment/meteo_france_SCM_study/crocus/crocus.py b/experiment/meteo_france_SCM_study/crocus/crocus.py
index b14c2fdb..43caec2a 100644
--- a/experiment/meteo_france_SCM_study/crocus/crocus.py
+++ b/experiment/meteo_france_SCM_study/crocus/crocus.py
@@ -1,3 +1,5 @@
+import numpy as np
+
from experiment.meteo_france_SCM_study.abstract_extended_study import AbstractExtendedStudy
from experiment.meteo_france_SCM_study.abstract_study import AbstractStudy
from experiment.meteo_france_SCM_study.crocus.crocus_variables import CrocusSweVariable, CrocusDepthVariable
@@ -36,8 +38,10 @@ class ExtendedCrocusDepth(AbstractExtendedStudy, CrocusDepth):
if __name__ == '__main__':
for variable_class in [CrocusSweVariable, CrocusDepthVariable]:
- study = Crocus(variable_class=variable_class)
+ study = Crocus(variable_class=variable_class, altitude=2400)
d = study.year_to_dataset_ordered_dict[1960]
+ time_arr = np.array(d.variables['time'])
+ print(time_arr)
# print(d)
a = study.year_to_daily_time_serie[1960]
print(a.shape)
diff --git a/experiment/meteo_france_SCM_study/crocus/crocus_variables.py b/experiment/meteo_france_SCM_study/crocus/crocus_variables.py
index 516221c9..c8b1c120 100644
--- a/experiment/meteo_france_SCM_study/crocus/crocus_variables.py
+++ b/experiment/meteo_france_SCM_study/crocus/crocus_variables.py
@@ -5,27 +5,37 @@ from experiment.meteo_france_SCM_study.abstract_variable import AbstractVariable
class CrocusVariable(AbstractVariable):
- def __init__(self, dataset, variable_name):
- super().__init__(dataset)
+ def __init__(self, dataset, altitude, variable_name):
+ super().__init__(dataset, altitude)
self.variable_name = variable_name
@property
def daily_time_serie(self):
- # So far the dimension of the time serie is 1460 x 23
- return np.array(self.dataset.variables[self.variable_name])[:, 0, :]
+ time_serie_every_6_hours = np.array(self.dataset.variables[self.variable_name])[:, 0, :]
+ if self.altitude == 2400:
+ time_serie_daily = time_serie_every_6_hours
+ else:
+ # Take the mean over a full day (WARNING: by doing that I am potentially destroying some maxima)
+ # TODO: I could create a special mode where I take the maximum instead of the mean here
+ nb_days = len(time_serie_every_6_hours) // 4
+ time_serie_daily = np.array([np.mean(time_serie_every_6_hours[4 * i:4 * (i + 1)], axis=0)
+ for i in range(nb_days)])
+ return time_serie_daily
class CrocusSweVariable(CrocusVariable):
NAME = 'Snow Water Equivalent'
- def __init__(self, dataset):
+ def __init__(self, dataset, altitude):
# Units are kg m-2
- super().__init__(dataset, 'SNOWSWE')
+ super().__init__(dataset, altitude, 'SNOWSWE')
class CrocusDepthVariable(CrocusVariable):
+ """Crocus Depth data is every 6 hours
+ To obtain daily data, we take the average over the 4 slots of 6 hours that compose a full day """
NAME = 'Snow Depth'
- def __init__(self, dataset):
+ def __init__(self, dataset, altitude):
# Units are m
- super().__init__(dataset, "SNOWDEPTH")
+ super().__init__(dataset, altitude, "SNOWDEPTH")
diff --git a/experiment/meteo_france_SCM_study/main_visualize.py b/experiment/meteo_france_SCM_study/main_visualize.py
index 84bde0b8..8322c9e6 100644
--- a/experiment/meteo_france_SCM_study/main_visualize.py
+++ b/experiment/meteo_france_SCM_study/main_visualize.py
@@ -15,7 +15,7 @@ SCM_STUDY_TO_EXTENDED_STUDY = OrderedDict(zip(SCM_STUDIES, SCM_EXTENDED_STUDIES)
def study_iterator(study_class, only_first_one=False, both_altitude=False, verbose=True):
all_studies = []
is_safran_study = study_class in [Safran, ExtendedSafran]
- nb_days = [3, 1] if is_safran_study else [1]
+ nb_days = [1] if is_safran_study else [1]
if verbose:
print('Loading studies....')
for nb_day in nb_days:
@@ -35,7 +35,7 @@ def study_iterator(study_class, only_first_one=False, both_altitude=False, verbo
def extended_visualization():
for study_class in SCM_EXTENDED_STUDIES[:]:
for study in study_iterator(study_class, only_first_one=False):
- study_visualizer = StudyVisualizer(study, save_to_file=True, only_one_graph=True, plot_bm_quantiles=True)
+ study_visualizer = StudyVisualizer(study, save_to_file=True, only_one_graph=True, plot_block_maxima_quantiles=True)
# study_visualizer.visualize_all_mean_and_max_graphs()
study_visualizer.visualize_all_experimental_law()
# for study_class in SCM_EXTENDED_STUDIES[:]:
diff --git a/experiment/meteo_france_SCM_study/safran/safran.py b/experiment/meteo_france_SCM_study/safran/safran.py
index f5877b9e..6588b1e5 100644
--- a/experiment/meteo_france_SCM_study/safran/safran.py
+++ b/experiment/meteo_france_SCM_study/safran/safran.py
@@ -21,3 +21,11 @@ class Safran(AbstractStudy):
class ExtendedSafran(AbstractExtendedStudy, Safran):
pass
+
+
+if __name__ == '__main__':
+ study = Safran()
+ d = study.year_to_dataset_ordered_dict[1958]
+ print(d.variables['time'])
+ print(study.year_to_daily_time_serie[1958].shape)
+ print(len(d.variables['time']))
\ No newline at end of file
diff --git a/experiment/meteo_france_SCM_study/safran/safran_snowfall_variable.py b/experiment/meteo_france_SCM_study/safran/safran_snowfall_variable.py
index 491dd3d2..1e919aa8 100644
--- a/experiment/meteo_france_SCM_study/safran/safran_snowfall_variable.py
+++ b/experiment/meteo_france_SCM_study/safran/safran_snowfall_variable.py
@@ -5,6 +5,8 @@ from experiment.meteo_france_SCM_study.abstract_variable import AbstractVariable
class SafranSnowfallVariable(AbstractVariable):
""""
+ Safran data is hourly
+
Hypothesis:
-How to count how much snowfall in one hour ?
@@ -21,8 +23,8 @@ class SafranSnowfallVariable(AbstractVariable):
NAME = 'Snowfall'
- def __init__(self, dataset, nb_consecutive_days_of_snowfall=1):
- super().__init__(dataset)
+ def __init__(self, dataset, altitude, nb_consecutive_days_of_snowfall=1):
+ super().__init__(dataset, altitude)
self.nb_consecutive_days_of_snowfall = nb_consecutive_days_of_snowfall
# Compute the daily snowfall in kg/m2
snowfall_rates = np.array(dataset.variables['Snowf'])
diff --git a/experiment/meteo_france_SCM_study/safran/safran_visualizer.py b/experiment/meteo_france_SCM_study/safran/safran_visualizer.py
index d72300d4..430f9641 100644
--- a/experiment/meteo_france_SCM_study/safran/safran_visualizer.py
+++ b/experiment/meteo_france_SCM_study/safran/safran_visualizer.py
@@ -31,7 +31,7 @@ BLOCK_MAXIMA_DISPLAY_NAME = 'block maxima '
class StudyVisualizer(object):
def __init__(self, study: AbstractStudy, show=True, save_to_file=False, only_one_graph=False, only_first_row=False,
- vertical_kde_plot=False, year_for_kde_plot=None, plot_bm_quantiles=False):
+ vertical_kde_plot=False, year_for_kde_plot=None, plot_block_maxima_quantiles=False):
self.only_first_row = only_first_row
self.only_one_graph = only_one_graph
self.save_to_file = save_to_file
@@ -40,7 +40,7 @@ class StudyVisualizer(object):
# KDE PLOT ARGUMENTS
self.vertical_kde_plot = vertical_kde_plot
self.year_for_kde_plot = year_for_kde_plot
- self.plot_bm_quantiles = plot_bm_quantiles
+ self.plot_block_maxima_quantiles = plot_block_maxima_quantiles
self.show = False if self.save_to_file else show
self.window_size_for_smoothing = 21
@@ -115,7 +115,9 @@ class StudyVisualizer(object):
x_level = all_massif_data[int(p * len(all_massif_data))]
x_level_to_color[x_level] = (color, name)
# Plot some additional quantiles from the correspond Annual Maxima law
- if self.plot_bm_quantiles:
+ if self.plot_block_maxima_quantiles:
+ # This formula can only be applied if we have a daily time serie
+ assert len(self.study.year_to_daily_time_serie[1958]) in [365, 366]
p = p ** (1 / 365)
x_level = all_massif_data[int(p * len(all_massif_data))]
x_level_to_color[x_level] = (color, BLOCK_MAXIMA_DISPLAY_NAME + name)
diff --git a/test/test_meteo_france_SCM_study/__init__.py b/test/test_experiment/__init__.py
similarity index 100%
rename from test/test_meteo_france_SCM_study/__init__.py
rename to test/test_experiment/__init__.py
diff --git a/test/test_experiment/test_meteo_france_SCM_study/__init__.py b/test/test_experiment/test_meteo_france_SCM_study/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/test/test_meteo_france_SCM_study/test_SCM_study.py b/test/test_experiment/test_meteo_france_SCM_study/test_SCM_study.py
similarity index 69%
rename from test/test_meteo_france_SCM_study/test_SCM_study.py
rename to test/test_experiment/test_meteo_france_SCM_study/test_SCM_study.py
index 2f670d3d..01c0f512 100644
--- a/test/test_meteo_france_SCM_study/test_SCM_study.py
+++ b/test/test_experiment/test_meteo_france_SCM_study/test_SCM_study.py
@@ -3,10 +3,11 @@ import unittest
import pandas as pd
-from experiment.meteo_france_SCM_study.crocus.crocus import ExtendedCrocusSwe
+from experiment.meteo_france_SCM_study.crocus.crocus import ExtendedCrocusSwe, Crocus
from experiment.meteo_france_SCM_study.main_visualize import study_iterator
from experiment.meteo_france_SCM_study.safran.safran import Safran, ExtendedSafran
from experiment.meteo_france_SCM_study.safran.safran_visualizer import StudyVisualizer
+from test.test_utils import load_scm_studies
class TestSCMStudy(unittest.TestCase):
@@ -22,11 +23,18 @@ class TestSCMStudy(unittest.TestCase):
def test_extended_run(self):
for study_class in [ExtendedSafran, ExtendedCrocusSwe]:
- for study in study_iterator(study_class, only_first_one=True, both_altitude=True, verbose=False):
+ for study in study_iterator(study_class, only_first_one=True, both_altitude=False, verbose=False):
study_visualizer = StudyVisualizer(study, show=False, save_to_file=False)
study_visualizer.visualize_all_mean_and_max_graphs()
self.assertTrue(True)
+ def test_scm_daily_data(self):
+ for study in load_scm_studies():
+ time_serie = study.year_to_daily_time_serie[1958]
+ self.assertTrue(time_serie.ndim == 2, msg='for {} ndim={}'.format(study.__repr__(), time_serie.ndim))
+ self.assertTrue(len(time_serie) in [365, 366],
+ msg="current time serie length for {} is {}".format(study.__repr__(), len(time_serie)))
+
if __name__ == '__main__':
unittest.main()
diff --git a/test/test_meteo_france_SCM_study/test_safran_gev.py b/test/test_experiment/test_meteo_france_SCM_study/test_safran_gev.py
similarity index 100%
rename from test/test_meteo_france_SCM_study/test_safran_gev.py
rename to test/test_experiment/test_meteo_france_SCM_study/test_safran_gev.py
diff --git a/test/test_utils.py b/test/test_utils.py
index 36e1ea3a..8cd297f0 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -1,3 +1,9 @@
+from itertools import product
+from typing import List
+
+from experiment.meteo_france_SCM_study.abstract_study import AbstractStudy
+from experiment.meteo_france_SCM_study.crocus.crocus import Crocus, CrocusSwe, CrocusDepth
+from experiment.meteo_france_SCM_study.crocus.crocus_variables import CrocusSweVariable, CrocusDepthVariable
from extreme_estimator.estimator.full_estimator.abstract_full_estimator import SmoothMarginalsThenUnitaryMsp, \
FullEstimatorInASingleStepWithSmoothMargin
from extreme_estimator.estimator.max_stable_estimator.abstract_max_stable_estimator import MaxStableEstimator
@@ -15,7 +21,8 @@ from spatio_temporal_dataset.coordinates.spatial_coordinates.generated_spatial_c
from spatio_temporal_dataset.coordinates.spatio_temporal_coordinates.generated_spatio_temporal_coordinates import \
UniformSpatioTemporalCoordinates
from spatio_temporal_dataset.coordinates.spatial_coordinates.coordinates_1D import UniformSpatialCoordinates
-from spatio_temporal_dataset.coordinates.temporal_coordinates.generated_temporal_coordinates import ConsecutiveTemporalCoordinates
+from spatio_temporal_dataset.coordinates.temporal_coordinates.generated_temporal_coordinates import \
+ ConsecutiveTemporalCoordinates
"""
Common objects to load for the test.
@@ -87,7 +94,18 @@ def load_test_spatiotemporal_coordinates(nb_points, nb_steps, train_split_ratio=
for coordinate_class in TEST_SPATIO_TEMPORAL_COORDINATES]
-def load_safran_objects():
- nb_days_list = [1, 3, 5][:1]
- safran_altitude_list = [1800, 2400][:1]
- return [Safran(safran_altitude, nb_days) for safran_altitude in safran_altitude_list for nb_days in nb_days_list]
\ No newline at end of file
+def load_safran_studies(altitudes) -> List[Safran]:
+ nb_days_list = [1]
+ return [Safran(safran_altitude, nb_days) for safran_altitude in altitudes for nb_days in nb_days_list]
+
+
+def load_crocus_studies(altitudes) -> List[Crocus]:
+ crocus_classes = [CrocusSwe, CrocusDepth][:]
+ return [crocus_class(altitude) for crocus_class, altitude in product(crocus_classes, altitudes)]
+
+
+def load_scm_studies() -> List[AbstractStudy]:
+ altitudes = [1800, 2400][:]
+ scm_studies = load_safran_studies(altitudes)
+ scm_studies += load_crocus_studies(altitudes)
+ return scm_studies
--
GitLab