[SCM] ensure that each daily_series are indeed daily series. add test

experiment/meteo_france_SCM_study/abstract_study.py

@@ -67,7 +67,7 @@ class AbstractStudy(object):
         return year_to_annual_maxima
 
     def instantiate_variable_object(self, dataset) -> AbstractVariable:
-        return self.variable_class(dataset)
+        return self.variable_class(dataset, self.altitude)
 
     """ Private methods to be overwritten """
 

experiment/meteo_france_SCM_study/abstract_variable.py

@@ -4,8 +4,9 @@ class AbstractVariable(object):
 
     NAME = ''
 
-    def __init__(self, dataset):
+    def __init__(self, dataset, altitude):
         self.dataset = dataset
+        self.altitude = altitude
 
     @property
     def daily_time_serie(self):

experiment/meteo_france_SCM_study/crocus/crocus.py

+import numpy as np
+
 from experiment.meteo_france_SCM_study.abstract_extended_study import AbstractExtendedStudy
 from experiment.meteo_france_SCM_study.abstract_study import AbstractStudy
 from experiment.meteo_france_SCM_study.crocus.crocus_variables import CrocusSweVariable, CrocusDepthVariable
@@ -36,8 +38,10 @@ class ExtendedCrocusDepth(AbstractExtendedStudy, CrocusDepth):
 
 if __name__ == '__main__':
     for variable_class in [CrocusSweVariable, CrocusDepthVariable]:
-        study = Crocus(variable_class=variable_class)
+        study = Crocus(variable_class=variable_class, altitude=2400)
         d = study.year_to_dataset_ordered_dict[1960]
+        time_arr = np.array(d.variables['time'])
+        print(time_arr)
         # print(d)
         a = study.year_to_daily_time_serie[1960]
         print(a.shape)

experiment/meteo_france_SCM_study/crocus/crocus_variables.py

@@ -5,27 +5,37 @@ from experiment.meteo_france_SCM_study.abstract_variable import AbstractVariable
 
 class CrocusVariable(AbstractVariable):
 
-    def __init__(self, dataset, variable_name):
-        super().__init__(dataset)
+    def __init__(self, dataset, altitude, variable_name):
+        super().__init__(dataset, altitude)
         self.variable_name = variable_name
 
     @property
     def daily_time_serie(self):
-        # So far the dimension of the time serie is 1460 x 23
-        return np.array(self.dataset.variables[self.variable_name])[:, 0, :]
+        time_serie_every_6_hours = np.array(self.dataset.variables[self.variable_name])[:, 0, :]
+        if self.altitude == 2400:
+            time_serie_daily = time_serie_every_6_hours
+        else:
+            # Take the mean over a full day (WARNING: by doing that I am potentially destroying some maxima)
+            # TODO: I could create a special mode where I take the maximum instead of the mean here
+            nb_days = len(time_serie_every_6_hours) // 4
+            time_serie_daily = np.array([np.mean(time_serie_every_6_hours[4 * i:4 * (i + 1)], axis=0)
+                                         for i in range(nb_days)])
+        return time_serie_daily
 
 
 class CrocusSweVariable(CrocusVariable):
     NAME = 'Snow Water Equivalent'
 
-    def __init__(self, dataset):
+    def __init__(self, dataset, altitude):
         # Units are kg m-2
-        super().__init__(dataset, 'SNOWSWE')
+        super().__init__(dataset, altitude, 'SNOWSWE')
 
 
 class CrocusDepthVariable(CrocusVariable):
+    """Crocus Depth  data is every 6 hours
+    To obtain daily data, we take the average over the 4 slots of 6 hours that compose a full day """
     NAME = 'Snow Depth'
 
-    def __init__(self, dataset):
+    def __init__(self, dataset, altitude):
         # Units are m
-        super().__init__(dataset, "SNOWDEPTH")
+        super().__init__(dataset, altitude, "SNOWDEPTH")

experiment/meteo_france_SCM_study/main_visualize.py

@@ -15,7 +15,7 @@ SCM_STUDY_TO_EXTENDED_STUDY = OrderedDict(zip(SCM_STUDIES, SCM_EXTENDED_STUDIES)
 def study_iterator(study_class, only_first_one=False, both_altitude=False, verbose=True):
     all_studies = []
     is_safran_study = study_class in [Safran, ExtendedSafran]
-    nb_days = [3, 1] if is_safran_study else [1]
+    nb_days = [1] if is_safran_study else [1]
     if verbose:
         print('Loading studies....')
     for nb_day in nb_days:
@@ -35,7 +35,7 @@ def study_iterator(study_class, only_first_one=False, both_altitude=False, verbo
 def extended_visualization():
     for study_class in SCM_EXTENDED_STUDIES[:]:
         for study in study_iterator(study_class, only_first_one=False):
-            study_visualizer = StudyVisualizer(study, save_to_file=True, only_one_graph=True, plot_bm_quantiles=True)
+            study_visualizer = StudyVisualizer(study, save_to_file=True, only_one_graph=True, plot_block_maxima_quantiles=True)
             # study_visualizer.visualize_all_mean_and_max_graphs()
             study_visualizer.visualize_all_experimental_law()
     # for study_class in SCM_EXTENDED_STUDIES[:]:

experiment/meteo_france_SCM_study/safran/safran.py

@@ -21,3 +21,11 @@ class Safran(AbstractStudy):
 
 class ExtendedSafran(AbstractExtendedStudy, Safran):
     pass
+
+
+if __name__ == '__main__':
+    study = Safran()
+    d = study.year_to_dataset_ordered_dict[1958]
+    print(d.variables['time'])
+    print(study.year_to_daily_time_serie[1958].shape)
+    print(len(d.variables['time']))
\ No newline at end of file

experiment/meteo_france_SCM_study/safran/safran_snowfall_variable.py

@@ -5,6 +5,8 @@ from experiment.meteo_france_SCM_study.abstract_variable import AbstractVariable
 
 class SafranSnowfallVariable(AbstractVariable):
     """"
+    Safran data is hourly
+
     Hypothesis:
 
     -How to count how much snowfall in one hour ?
@@ -21,8 +23,8 @@ class SafranSnowfallVariable(AbstractVariable):
 
     NAME = 'Snowfall'
 
-    def __init__(self, dataset, nb_consecutive_days_of_snowfall=1):
-        super().__init__(dataset)
+    def __init__(self, dataset, altitude, nb_consecutive_days_of_snowfall=1):
+        super().__init__(dataset, altitude)
         self.nb_consecutive_days_of_snowfall = nb_consecutive_days_of_snowfall
         # Compute the daily snowfall in kg/m2
         snowfall_rates = np.array(dataset.variables['Snowf'])

experiment/meteo_france_SCM_study/safran/safran_visualizer.py

@@ -31,7 +31,7 @@ BLOCK_MAXIMA_DISPLAY_NAME = 'block maxima '
 class StudyVisualizer(object):
 
     def __init__(self, study: AbstractStudy, show=True, save_to_file=False, only_one_graph=False, only_first_row=False,
-                 vertical_kde_plot=False, year_for_kde_plot=None, plot_bm_quantiles=False):
+                 vertical_kde_plot=False, year_for_kde_plot=None, plot_block_maxima_quantiles=False):
         self.only_first_row = only_first_row
         self.only_one_graph = only_one_graph
         self.save_to_file = save_to_file
@@ -40,7 +40,7 @@ class StudyVisualizer(object):
         # KDE PLOT ARGUMENTS
         self.vertical_kde_plot = vertical_kde_plot
         self.year_for_kde_plot = year_for_kde_plot
-        self.plot_bm_quantiles = plot_bm_quantiles
+        self.plot_block_maxima_quantiles = plot_block_maxima_quantiles
 
         self.show = False if self.save_to_file else show
         self.window_size_for_smoothing = 21
@@ -115,7 +115,9 @@ class StudyVisualizer(object):
             x_level = all_massif_data[int(p * len(all_massif_data))]
             x_level_to_color[x_level] = (color, name)
             # Plot some additional quantiles from the correspond Annual Maxima law
-            if self.plot_bm_quantiles:
+            if self.plot_block_maxima_quantiles:
+                # This formula can only be applied if we have a daily time serie
+                assert len(self.study.year_to_daily_time_serie[1958]) in [365, 366]
                 p = p ** (1 / 365)
                 x_level = all_massif_data[int(p * len(all_massif_data))]
                 x_level_to_color[x_level] = (color, BLOCK_MAXIMA_DISPLAY_NAME + name)

test/test_meteo_france_SCM_study/test_SCM_study.py → test/test_experiment/test_meteo_france_SCM_study/test_SCM_study.py

@@ -3,10 +3,11 @@ import unittest
 
 import pandas as pd
 
-from experiment.meteo_france_SCM_study.crocus.crocus import ExtendedCrocusSwe
+from experiment.meteo_france_SCM_study.crocus.crocus import ExtendedCrocusSwe, Crocus
 from experiment.meteo_france_SCM_study.main_visualize import study_iterator
 from experiment.meteo_france_SCM_study.safran.safran import Safran, ExtendedSafran
 from experiment.meteo_france_SCM_study.safran.safran_visualizer import StudyVisualizer
+from test.test_utils import load_scm_studies
 
 
 class TestSCMStudy(unittest.TestCase):
@@ -22,11 +23,18 @@ class TestSCMStudy(unittest.TestCase):
 
     def test_extended_run(self):
         for study_class in [ExtendedSafran, ExtendedCrocusSwe]:
-            for study in study_iterator(study_class, only_first_one=True, both_altitude=True, verbose=False):
+            for study in study_iterator(study_class, only_first_one=True, both_altitude=False, verbose=False):
                 study_visualizer = StudyVisualizer(study, show=False, save_to_file=False)
                 study_visualizer.visualize_all_mean_and_max_graphs()
         self.assertTrue(True)
 
+    def test_scm_daily_data(self):
+        for study in load_scm_studies():
+            time_serie = study.year_to_daily_time_serie[1958]
+            self.assertTrue(time_serie.ndim == 2, msg='for {} ndim={}'.format(study.__repr__(), time_serie.ndim))
+            self.assertTrue(len(time_serie) in [365, 366],
+                            msg="current time serie length for {} is {}".format(study.__repr__(), len(time_serie)))
+
 
 if __name__ == '__main__':
     unittest.main()

test/test_utils.py

+from itertools import product
+from typing import List
+
+from experiment.meteo_france_SCM_study.abstract_study import AbstractStudy
+from experiment.meteo_france_SCM_study.crocus.crocus import Crocus, CrocusSwe, CrocusDepth
+from experiment.meteo_france_SCM_study.crocus.crocus_variables import CrocusSweVariable, CrocusDepthVariable
 from extreme_estimator.estimator.full_estimator.abstract_full_estimator import SmoothMarginalsThenUnitaryMsp, \
     FullEstimatorInASingleStepWithSmoothMargin
 from extreme_estimator.estimator.max_stable_estimator.abstract_max_stable_estimator import MaxStableEstimator
@@ -15,7 +21,8 @@ from spatio_temporal_dataset.coordinates.spatial_coordinates.generated_spatial_c
 from spatio_temporal_dataset.coordinates.spatio_temporal_coordinates.generated_spatio_temporal_coordinates import \
     UniformSpatioTemporalCoordinates
 from spatio_temporal_dataset.coordinates.spatial_coordinates.coordinates_1D import UniformSpatialCoordinates
-from spatio_temporal_dataset.coordinates.temporal_coordinates.generated_temporal_coordinates import ConsecutiveTemporalCoordinates
+from spatio_temporal_dataset.coordinates.temporal_coordinates.generated_temporal_coordinates import \
+    ConsecutiveTemporalCoordinates
 
 """
 Common objects to load for the test.
@@ -87,7 +94,18 @@ def load_test_spatiotemporal_coordinates(nb_points, nb_steps, train_split_ratio=
             for coordinate_class in TEST_SPATIO_TEMPORAL_COORDINATES]
 
 
-def load_safran_objects():
-    nb_days_list = [1, 3, 5][:1]
-    safran_altitude_list = [1800, 2400][:1]
-    return [Safran(safran_altitude, nb_days) for safran_altitude in safran_altitude_list for nb_days in nb_days_list]
\ No newline at end of file
+def load_safran_studies(altitudes) -> List[Safran]:
+    nb_days_list = [1]
+    return [Safran(safran_altitude, nb_days) for safran_altitude in altitudes for nb_days in nb_days_list]
+
+
+def load_crocus_studies(altitudes) -> List[Crocus]:
+    crocus_classes = [CrocusSwe, CrocusDepth][:]
+    return [crocus_class(altitude) for crocus_class, altitude in product(crocus_classes, altitudes)]
+
+
+def load_scm_studies() -> List[AbstractStudy]:
+    altitudes = [1800, 2400][:]
+    scm_studies = load_safran_studies(altitudes)
+    scm_studies += load_crocus_studies(altitudes)
+    return scm_studies