[SCM][HYPERCUBE] Add possibility to create altitude hypercube effortlessly...

[SCM][HYPERCUBE] Add possibility to create altitude hypercube effortlessly from quantity altitude hypercube (with an additional argument called add_detailed_plot)

experiment/meteo_france_SCM_study/visualization/hypercube_visualization/hypercube_visualizer.py

@@ -59,9 +59,13 @@ class HypercubeVisualizer(object):
 
     # Some properties
 
+    @property
+    def study_title(self):
+        return self.study.title
+
     def show_or_save_to_file(self, specific_title=''):
         if self.save_to_file:
-            main_title, _ = '_'.join(self.study.title.split()).split('/')
+            main_title, *_ = '_'.join(self.study_title.split()).split('/')
             filename = "{}/{}/".format(VERSION_TIME, main_title)
             filename += specific_title
             filepath = op.join(self.study.result_full_path, filename + '.png')
@@ -119,11 +123,21 @@ class AltitudeHypercubeVisualizer(HypercubeVisualizer):
             AbstractTrendTest.SIGNIFICATIVE_NEGATIVE_TREND]
         return trend_type_to_s_percentages
 
-    def visualize_trend_test_evolution(self, reduction_function, xlabel, xlabel_values, ax=None, marker='o'):
+    def subtitle_to_reduction_function(self, reduction_function, level=None, add_detailed_plot=False):
+        def reduction_function_with_level(df_bool):
+            if level is None:
+                return reduction_function(df_bool)
+            else:
+                return reduction_function(df_bool, level)
+        return {'global': reduction_function_with_level}
+
+    def visualize_trend_test_evolution(self, reduction_function, xlabel, xlabel_values, ax=None, marker='o',
+                                       subtitle=''):
         if ax is None:
             fig, ax = plt.subplots(1, 1, figsize=self.study_visualizer.figsize)
 
-        trend_type_to_percentages_values = {k: s.values for k, s in self.trend_type_to_s_percentages(reduction_function).items()}
+        trend_type_to_percentages_values = {k: s.values for k, s in
+                                            self.trend_type_to_s_percentages(reduction_function).items()}
         for trend_type in self.trend_types:
             style = self.trend_type_to_style[trend_type]
             percentages_values = trend_type_to_percentages_values[trend_type]
@@ -149,10 +163,11 @@ class AltitudeHypercubeVisualizer(HypercubeVisualizer):
         name = get_display_name_from_object_type(self.trend_test_class)
         title = 'Evolution of {} trends (significative or not) wrt to the {} with {}'.format(variable_name, xlabel,
                                                                                              name)
+        title += 'with {} data'.format(subtitle)
         ax.set_title(title)
         self.show_or_save_to_file(specific_title=title)
 
-    def visualize_trend_test_repartition(self, reduction_function, axes=None):
+    def visualize_trend_test_repartition(self, reduction_function, axes=None, subtitle=''):
         if axes is None:
             nb_trend_type = len(self.trend_test_class.trend_type_to_style())
             fig, axes = plt.subplots(1, nb_trend_type, figsize=self.study_visualizer.figsize)
@@ -170,48 +185,84 @@ class AltitudeHypercubeVisualizer(HypercubeVisualizer):
         name = get_display_name_from_object_type(self.trend_test_class)
         title = 'Repartition of trends (significative or not) with {}'.format(name)
         title += '\n(in % averaged on altitudes & averaged on starting years)'
+        title += 'with {} data'.format(subtitle)
         StudyVisualizer.clean_axes_write_title_on_the_left(axes, title, left_border=None)
         plt.suptitle(title)
         self.show_or_save_to_file(specific_title=title)
 
-    def visualize_year_trend_test(self, ax=None, marker='o'):
+    @property
+    def altitude_index_level(self):
+        return 0
+
+    @property
+    def massif_index_level(self):
+        return 1
+
+    def visualize_year_trend_test(self, ax=None, marker='o', add_detailed_plots=False):
         def year_reduction(df_bool):
             # Take the mean with respect to all the first axis indices
             return df_bool.mean(axis=0)
 
-        self.visualize_trend_test_evolution(reduction_function=year_reduction, xlabel='starting years',
-                                            xlabel_values=self.starting_years, ax=ax, marker=marker)
+        for subtitle, reduction_function in self.subtitle_to_reduction_function(year_reduction,
+                                                                                add_detailed_plot=add_detailed_plots).items():
+            self.visualize_trend_test_evolution(reduction_function=reduction_function, xlabel='starting years',
+                                                xlabel_values=self.starting_years, ax=ax, marker=marker,
+                                                subtitle=subtitle)
 
-    @property
-    def altitude_index_level(self):
-        return 0
-
-    def visualize_altitude_trend_test(self, ax=None, marker='o'):
-        def altitude_reduction(df_bool):
+    def visualize_altitude_trend_test(self, ax=None, marker='o', add_detailed_plots=False):
+        def altitude_reduction(df_bool, level):
             # Take the mean with respect to the years
             df_bool = df_bool.mean(axis=1)
             # Take the mean with respect the massifs
-            return df_bool.mean(level=self.altitude_index_level)
-
-        self.visualize_trend_test_evolution(reduction_function=altitude_reduction, xlabel='altitude',
-                                            xlabel_values=self.altitudes, ax=ax, marker=marker)
+            print(df_bool.head())
+            return df_bool.mean(level=level)
 
-    @property
-    def massif_index_level(self):
-        return 1
+        for subtitle, reduction_function in self.subtitle_to_reduction_function(altitude_reduction, level=self.altitude_index_level,
+                                                                                add_detailed_plot=add_detailed_plots).items():
+            self.visualize_trend_test_evolution(reduction_function=reduction_function, xlabel='altitude',
+                                                xlabel_values=self.altitudes, ax=ax, marker=marker,
+                                                subtitle=subtitle)
 
-    def visualize_massif_trend_test(self, axes=None):
-        def massif_reduction(df_bool):
+    def visualize_massif_trend_test(self, axes=None, add_detailed_plots=False):
+        def massif_reduction(df_bool, level):
             # Take the mean with respect to the years
             df_bool = df_bool.mean(axis=1)
             # Take the mean with respect the altitude
-            return df_bool.mean(level=self.massif_index_level)
+            return df_bool.mean(level=level)
 
-        self.visualize_trend_test_repartition(massif_reduction, axes)
+        for subtitle, reduction_function in self.subtitle_to_reduction_function(massif_reduction,level=self.massif_index_level,
+                                                                                add_detailed_plot=add_detailed_plots).items():
+            self.visualize_trend_test_repartition(reduction_function, axes, subtitle=subtitle)
 
 
 class QuantityAltitudeHypercubeVisualizer(AltitudeHypercubeVisualizer):
 
+    @property
+    def study_title(self):
+        return 'Quantity Altitude Study'
+
+    def subtitle_to_reduction_function(self, reduction_function, level=None, add_detailed_plot=False):
+        subtitle_to_reduction_function = super().subtitle_to_reduction_function(reduction_function, level, add_detailed_plot)
+
+        def get_function_from_tuple(tuple_for_axis_0):
+            def f(df_bool: pd.DataFrame):
+                # Loc with a tuple with respect the axis 0
+                df_bool = df_bool.loc[tuple_for_axis_0, :].copy()
+                # Apply the reduction function
+                if level is None:
+                    return reduction_function(df_bool)
+                else:
+                    return reduction_function(df_bool, level-1)
+
+            return f
+
+        # Add the detailed plot, taken by loc with respect to the first index
+        if add_detailed_plot:
+            tuples_axis_0 = self.tuple_values(idx=0)
+            for tuple_axis_0 in tuples_axis_0:
+                subtitle_to_reduction_function[tuple_axis_0] = get_function_from_tuple(tuple_axis_0)
+        return subtitle_to_reduction_function
+
     @property
     def quantities(self):
         return self.tuple_values(idx=0)

experiment/meteo_france_SCM_study/visualization/hypercube_visualization/main_hypercube_visualization.py

@@ -13,22 +13,42 @@ from experiment.trend_analysis.univariate_trend_test.abstract_trend_test import
 from utils import get_display_name_from_object_type
 
 
-def full_trends_with_altitude_hypercube():
+# def full_trends_with_altitude_hypercube():
+#     save_to_file = True
+#     only_first_one = False
+#     fast = False
+#     altitudes = ALL_ALTITUDES[3:-6]
+#     for study_class in SCM_STUDIES[:]:
+#         for trend_test_class in [MannKendallTrendTest, GevLocationTrendTest, GevScaleTrendTest, GevShapeTrendTest][:]:
+#             visualizers = [StudyVisualizer(study, temporal_non_stationarity=True, verbose=False, multiprocessing=True)
+#                            for study in study_iterator(study_class=study_class, only_first_one=only_first_one,
+#                                                        altitudes=altitudes)]
+#             altitude_to_visualizer = OrderedDict(zip(altitudes, visualizers))
+#             visualizer = AltitudeHypercubeVisualizer(altitude_to_visualizer, save_to_file=save_to_file,
+#                                                      trend_test_class=trend_test_class, fast=fast)
+#             visualizer.visualize_massif_trend_test()
+#             visualizer.visualize_year_trend_test()
+#             visualizer.visualize_altitude_trend_test()
+
+
+def full_trends_with_quantity_altitude_hypercube():
     save_to_file = True
     only_first_one = False
     fast = False
     altitudes = ALL_ALTITUDES[3:-6]
-    for study_class in SCM_STUDIES[:]:
-        for trend_test_class in [MannKendallTrendTest, GevLocationTrendTest, GevScaleTrendTest, GevShapeTrendTest][:]:
-            visualizers = [StudyVisualizer(study, temporal_non_stationarity=True, verbose=False, multiprocessing=True)
-                           for study in study_iterator(study_class=study_class, only_first_one=only_first_one,
-                                                       altitudes=altitudes)]
-            altitude_to_visualizer = OrderedDict(zip(altitudes, visualizers))
-            visualizer = AltitudeHypercubeVisualizer(altitude_to_visualizer, save_to_file=save_to_file,
-                                                     trend_test_class=trend_test_class, fast=fast)
-            visualizer.visualize_massif_trend_test()
-            visualizer.visualize_year_trend_test()
-            visualizer.visualize_altitude_trend_test()
+    study_classes = SCM_STUDIES
+    for trend_test_class in [MannKendallTrendTest, GevLocationTrendTest, GevScaleTrendTest, GevShapeTrendTest][:]:
+        visualizers = [StudyVisualizer(study, temporal_non_stationarity=True, verbose=False, multiprocessing=True)
+                       for study in study_iterator_global(study_classes=study_classes, only_first_one=only_first_one,
+                                                          altitudes=altitudes)]
+        study_classes_str = [get_display_name_from_object_type(c) for c in study_classes]
+        quantity_altitude_tuples = list(product(study_classes_str, altitudes))
+        quantity_altitude_to_visualizer = OrderedDict(zip(quantity_altitude_tuples, visualizers))
+        visualizer = QuantityAltitudeHypercubeVisualizer(quantity_altitude_to_visualizer, save_to_file=save_to_file,
+                                                         trend_test_class=trend_test_class, fast=fast)
+        visualizer.visualize_year_trend_test(add_detailed_plots=True)
+        visualizer.visualize_massif_trend_test(add_detailed_plots=True)
+        visualizer.visualize_altitude_trend_test(add_detailed_plots=True)
 
 
 def fast_trends_with_altitude_hypercube():
@@ -49,26 +69,6 @@ def fast_trends_with_altitude_hypercube():
             # visualizer.visualize_altitude_trend_test()
 
 
-def full_trends_with_quantity_altitude_hypercube():
-    save_to_file = True
-    only_first_one = False
-    fast = False
-    altitudes = ALL_ALTITUDES[3:-6]
-    study_classes = SCM_STUDIES
-    for trend_test_class in [MannKendallTrendTest, GevLocationTrendTest, GevScaleTrendTest, GevShapeTrendTest][:1]:
-        visualizers = [StudyVisualizer(study, temporal_non_stationarity=True, verbose=False, multiprocessing=True)
-                       for study in study_iterator_global(study_classes=study_classes, only_first_one=only_first_one,
-                                                          altitudes=altitudes)]
-        study_classes_str = [get_display_name_from_object_type(c) for c in study_classes]
-        quantity_altitude_tuples = list(product(study_classes_str, altitudes))
-        quantity_altitude_to_visualizer = OrderedDict(zip(quantity_altitude_tuples, visualizers))
-        visualizer = QuantityAltitudeHypercubeVisualizer(quantity_altitude_to_visualizer, save_to_file=save_to_file,
-                                                         trend_test_class=trend_test_class, fast=fast)
-        visualizer.visualize_year_trend_test()
-        # visualizer.visualize_massif_trend_test()
-        # visualizer.visualize_altitude_trend_test()
-
-
 def fast_trends_with_quantity_altitude_hypercube():
     save_to_file = False
     only_first_one = False
@@ -84,13 +84,13 @@ def fast_trends_with_quantity_altitude_hypercube():
         quantity_altitude_to_visualizer = OrderedDict(zip(quantity_altitude_tuples, visualizers))
         visualizer = QuantityAltitudeHypercubeVisualizer(quantity_altitude_to_visualizer, save_to_file=save_to_file,
                                                          trend_test_class=trend_test_class, fast=fast)
-        # visualizer.visualize_year_trend_test()
+        visualizer.visualize_year_trend_test()
         # visualizer.visualize_massif_trend_test()
-        visualizer.visualize_altitude_trend_test()
+        # visualizer.visualize_altitude_trend_test()
+
 
 def main_run():
     # fast_trends_with_altitude_hypercube()
-    # full_trends_with_altitude_hypercube()
     # fast_trends_with_quantity_altitude_hypercube()
     full_trends_with_quantity_altitude_hypercube()
 

experiment/trend_analysis/univariate_trend_test/abstract_gev_trend_test.py

@@ -21,8 +21,9 @@ from spatio_temporal_dataset.spatio_temporal_observations.abstract_spatio_tempor
 class AbstractGevTrendTest(AbstractTrendTest):
     RRunTimeError_TREND = 'R RunTimeError trend'
 
-    def __init__(self, years_after_change_point, maxima_after_change_point, non_stationary_model_class):
+    def __init__(self, years_after_change_point, maxima_after_change_point, non_stationary_model_class, gev_param_name):
         super().__init__(years_after_change_point, maxima_after_change_point)
+        self.gev_param_name = gev_param_name
         df = pd.DataFrame({AbstractCoordinates.COORDINATE_T: years_after_change_point})
         df_maxima_gev = pd.DataFrame(maxima_after_change_point, index=df.index)
         observations = AbstractSpatioTemporalObservations(df_maxima_gev=df_maxima_gev)
@@ -66,35 +67,42 @@ class AbstractGevTrendTest(AbstractTrendTest):
         else:
             return super().test_trend_type
 
+    def get_coef(self, estimator):
+        return estimator.margin_function_fitted.get_coef(self.gev_param_name, AbstractCoordinates.COORDINATE_T)
 
-class GevLocationTrendTest(AbstractGevTrendTest):
+    @property
+    def stationary_coef(self):
+        return self.get_coef(self.stationary_estimator)
 
-    def __init__(self, years_after_change_point, maxima_after_change_point):
-        super().__init__(years_after_change_point, maxima_after_change_point, NonStationaryLocationStationModel)
+    @property
+    def non_stationary_coef(self):
+        return self.get_coef(self.non_stationary_estimator)
+
+    @property
+    def ratio_non_stationary_coef(self):
+        pass
 
     @property
     def test_sign(self) -> int:
-        return np.sign(self.non_stationary_estimator.margin_function_fitted.get_coef(GevParams.LOC,
-                                                                                     AbstractCoordinates.COORDINATE_T))
+        return np.sign(self.non_stationary_coef)
 
 
-class GevScaleTrendTest(AbstractGevTrendTest):
+class GevLocationTrendTest(AbstractGevTrendTest):
 
     def __init__(self, years_after_change_point, maxima_after_change_point):
-        super().__init__(years_after_change_point, maxima_after_change_point, NonStationaryScaleStationModel)
+        super().__init__(years_after_change_point, maxima_after_change_point,
+                         NonStationaryLocationStationModel, GevParams.LOC)
 
-    @property
-    def test_sign(self) -> int:
-        return np.sign(self.non_stationary_estimator.margin_function_fitted.get_coef(GevParams.SCALE,
-                                                                                     AbstractCoordinates.COORDINATE_T))
+
+class GevScaleTrendTest(AbstractGevTrendTest):
+
+    def __init__(self, years_after_change_point, maxima_after_change_point):
+        super().__init__(years_after_change_point, maxima_after_change_point,
+                         NonStationaryScaleStationModel, GevParams.SCALE)
 
 
 class GevShapeTrendTest(AbstractGevTrendTest):
 
     def __init__(self, years_after_change_point, maxima_after_change_point):
-        super().__init__(years_after_change_point, maxima_after_change_point, NonStationaryShapeStationModel)
-
-    @property
-    def test_sign(self) -> int:
-        return np.sign(self.non_stationary_estimator.margin_function_fitted.get_coef(GevParams.SHAPE,
-                                                                                     AbstractCoordinates.COORDINATE_T))
+        super().__init__(years_after_change_point, maxima_after_change_point,
+                         NonStationaryShapeStationModel, GevParams.SHAPE)

experiment/trend_analysis/univariate_trend_test/abstract_trend_test.py

@@ -85,6 +85,7 @@ class ExampleRandomTrendTest(AbstractTrendTest):
 class WarningScoreValue(Warning):
     pass
 
+
 class MannKendallTrendTest(AbstractTrendTest):
 
     def __init__(self, years_after_change_point, maxima_after_change_point):