[HYPERCUBE] refactor hypercube. add the fact that for the fast mode, massif...

[HYPERCUBE] refactor hypercube. add the fact that for the fast mode, massif names are sampled randomly for the fast mode. this enables to explore the change point test results under different viewpoint.

experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/abstract_hypercube_visualizer.py

@@ -25,7 +25,6 @@ class AbstractHypercubeVisualizer(object):
                  last_starting_year=None):
         self.last_starting_year = last_starting_year
         self.nb_data_for_fast_mode = 7 if nb_data_reduced_for_speed else None
-        self.nb_top_likelihood_values = nb_top_likelihood_values
         self.save_to_file = save_to_file
         self.trend_test_class = trend_test_class
         self.tuple_to_study_visualizer = tuple_to_study_visualizer  # type: Dict[Tuple, StudyVisualizer]
@@ -55,8 +54,7 @@ class AbstractHypercubeVisualizer(object):
     @cached_property
     def df_trends_spatio_temporal(self):
         return [study_visualizer.df_trend_spatio_temporal(self.trend_test_class, self.starting_years,
-                                                          self.nb_data_for_fast_mode,
-                                                          self.nb_top_likelihood_values)
+                                                          self.nb_data_for_fast_mode)
                 for study_visualizer in self.tuple_to_study_visualizer.values()]
 
     @cached_property

experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/main_hypercube_visualization.py

@@ -2,15 +2,16 @@ import time
 from collections import OrderedDict
 from itertools import product
 
-from experiment.meteo_france_data.scm_models_data.visualization import \
+from experiment.meteo_france_data.scm_models_data.visualization.hypercube_visualization.altitude_hypercube_visualizer import \
     AltitudeHypercubeVisualizer
-from experiment.meteo_france_data.scm_models_data.visualization import \
+from experiment.meteo_france_data.scm_models_data.visualization.hypercube_visualization.altitude_year_hypercube_visualizer import \
     Altitude_Hypercube_Year_Visualizer
-from experiment.meteo_france_data.scm_models_data.visualization import \
+from experiment.meteo_france_data.scm_models_data.visualization.hypercube_visualization.quantity_altitude_visualizer import \
     QuantityAltitudeHypercubeVisualizer
-from experiment.meteo_france_data.scm_models_data.visualization import ALL_ALTITUDES, \
-    SCM_STUDIES, study_iterator, study_iterator_global
-from experiment.meteo_france_data.scm_models_data.visualization import StudyVisualizer
+from experiment.meteo_france_data.scm_models_data.visualization.study_visualization.main_study_visualizer import \
+    ALL_ALTITUDES, SCM_STUDIES, study_iterator, study_iterator_global
+from experiment.meteo_france_data.scm_models_data.visualization.study_visualization.study_visualizer import \
+    StudyVisualizer
 from experiment.trend_analysis.univariate_test.abstract_gev_change_point_test import GevLocationChangePointTest, \
     GevScaleChangePointTest, GevShapeChangePointTest
 from utils import get_display_name_from_object_type
@@ -141,8 +142,8 @@ def fast_quantity_altitude_hypercube():
 
 def main_run():
     # fast_altitude_hypercube()
-    # fast_altitude_year_hypercube()
-    full_altitude_year_hypercube()
+    fast_altitude_year_hypercube()
+    # full_altitude_year_hypercube()
     # fast_quantity_altitude_hypercube()
     # full_quantity_altitude_hypercube()
 

experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/quantity_altitude_visualizer.py

 import pandas as pd
 
-from experiment.meteo_france_data.scm_models_data.visualization import \
+from experiment.meteo_france_data.scm_models_data.visualization.hypercube_visualization.altitude_hypercube_visualizer import \
     AltitudeHypercubeVisualizer
 
 

experiment/meteo_france_data/scm_models_data/visualization/study_visualization/study_visualizer.py

@@ -2,6 +2,7 @@ import os
 import os.path as op
 from collections import OrderedDict
 from multiprocessing.pool import Pool
+from random import sample
 
 import math
 import matplotlib.pyplot as plt
@@ -112,6 +113,11 @@ class StudyVisualizer(VisualizationParameters):
         # INCREASE THE TEMPORAL STEPS FOR VISUALIZATION
         AbstractMarginFunction.VISUALIZATION_TEMPORAL_STEPS = 5
 
+        # Change point parameters
+        self.trend_test_class_for_change_point_test = None
+        self.starting_years_for_change_point_test = None
+        self.nb_massif_for_change_point_test = None
+
     @property
     def dataset(self):
         if self._dataset is None:
@@ -390,8 +396,37 @@ class StudyVisualizer(VisualizationParameters):
             massif_name_to_df_trend_type[massif_name] = df
         return massif_name_to_df_trend_type
 
-    def df_trend_spatio_temporal(self, trend_test_class, starting_years, nb_massif_for_fast_mode=None,
-                                 nb_top_likelihood_values=1):
+    def massif_name_to_gev_change_point_test_results(self, trend_test_class_for_change_point_test,
+                                                     starting_years_for_change_point_test,
+                                                     nb_massif_for_change_point_test=None):
+        if self.trend_test_class_for_change_point_test is None:
+            # Set the attribute is not already done
+            self.trend_test_class_for_change_point_test = trend_test_class_for_change_point_test
+            self.starting_years_for_change_point_test = starting_years_for_change_point_test
+            self.nb_massif_for_change_point_test = nb_massif_for_change_point_test
+        else:
+             # Check that the argument are the same
+            assert self.trend_test_class_for_change_point_test == trend_test_class_for_change_point_test
+            assert self.starting_years == starting_years_for_change_point_test
+            assert self.nb_massif_for_change_point_test == nb_massif_for_change_point_test
+        return self._massif_name_to_gev_change_point_test_results
+
+    @cached_property
+    def _massif_name_to_gev_change_point_test_results(self):
+        massif_name_to_gev_change_point_test_results = {}
+        massif_names = self.study.study_massif_names
+        if self.nb_massif_for_change_point_test is not None:
+            massif_names = sample(massif_names, self.nb_massif_for_change_point_test)
+        for massif_id, massif_name in enumerate(massif_names):
+            years, smooth_maxima = self.smooth_maxima_x_y(massif_id)
+            gev_change_point_test_results = compute_gev_change_point_test_results(self.multiprocessing, smooth_maxima,
+                                                                                  self.starting_years_for_change_point_test,
+                                                                                  self.trend_test_class_for_change_point_test, years)
+            massif_name_to_gev_change_point_test_results[massif_name] = gev_change_point_test_results
+        return massif_name_to_gev_change_point_test_results
+
+    def df_trend_spatio_temporal(self, trend_test_class_for_change_point_test, starting_years_for_change_point_test,
+                                 nb_massif_for_change_point_test=None):
         """
         Index are the massif
         Columns are the starting year
@@ -400,14 +435,14 @@ class StudyVisualizer(VisualizationParameters):
         :param starting_year_to_weight:
         :return:
         """
+        # Set the attributes
+
         massif_name_to_trend_res = {}
-        massif_names = self.study.study_massif_names
-        if nb_massif_for_fast_mode is not None:
-            massif_names = massif_names[:nb_massif_for_fast_mode]
-        for massif_id, massif_name in enumerate(massif_names):
-            years, smooth_maxima = self.smooth_maxima_x_y(massif_id)
-            trend_test_res, best_idxs = compute_gev_change_point_test_results(self.multiprocessing, smooth_maxima, starting_years,
-                                                                   trend_test_class, years)
+        massif_name_to_gev_change_point_test_results = self.massif_name_to_gev_change_point_test_results(trend_test_class_for_change_point_test,
+                                                                                                         starting_years_for_change_point_test,
+                                                                                                         nb_massif_for_change_point_test)
+        for massif_name, gev_change_point_test_results in massif_name_to_gev_change_point_test_results.items():
+            trend_test_res, best_idxs = gev_change_point_test_results
             trend_test_res = [(a, b) if i in best_idxs else (np.nan, np.nan)
                               for i, (a, b, *_) in enumerate(trend_test_res)]
             massif_name_to_trend_res[massif_name] = list(zip(*trend_test_res))
@@ -415,7 +450,7 @@ class StudyVisualizer(VisualizationParameters):
         assert nb_res == 2
         all_massif_name_to_res = [{k: v[idx_res] for k, v in massif_name_to_trend_res.items()}
                                   for idx_res in range(nb_res)]
-        return [pd.DataFrame(massif_name_to_res, index=starting_years).transpose()
+        return [pd.DataFrame(massif_name_to_res, index=self.starting_years_for_change_point_test).transpose()
                 for massif_name_to_res in all_massif_name_to_res]
 
     @staticmethod

test/test_experiment/test_SCM_study.py

@@ -22,8 +22,7 @@ class TestSCMAllStudy(unittest.TestCase):
         for study_class in [ExtendedSafranSnowfall]:
             for study in study_iterator(study_class, only_first_one=True, verbose=False):
                 study_visualizer = StudyVisualizer(study, show=False, save_to_file=False, multiprocessing=True)
-                study_visualizer.df_trend_spatio_temporal(GevLocationChangePointTest, [1958, 1959, 1960],
-                                                          nb_massif_for_fast_mode=1)
+                study_visualizer.df_trend_spatio_temporal(GevLocationChangePointTest, [1958, 1959, 1960], nb_massif_for_change_point_test=3)
         self.assertTrue(True)
 
     def test_instantiate_studies(self):