[HYPERCUBE] add AltitudeHypercubeVisualizerWithoutTrendType (and its...

[HYPERCUBE] add AltitudeHypercubeVisualizerWithoutTrendType (and its extension) to visualize log likelihood evolution. add new altitudes bands for the extended plots. add constrained plot argument when creating multiple plots.

experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/altitude_hypercube_visualizer.py

-import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
+import matplotlib.pyplot as plt
 
 from experiment.meteo_france_data.scm_models_data.visualization.hypercube_visualization.abstract_hypercube_visualizer import \
     AbstractHypercubeVisualizer
@@ -31,13 +31,19 @@ class AltitudeHypercubeVisualizer(AbstractHypercubeVisualizer):
     def nb_rows(self):
         return 1
 
+    def ylabel_to_series(self, reduction_function, isin_parameters=None):
+        return {}
+
     def trend_type_to_series(self, reduction_function, isin_parameters=None):
         # Map each trend type to its serie with percentages
         # Define here all the trend type we might need in the results/displays
-        trend_types_to_process = list(self.display_trend_types) + [AbstractUnivariateTest.SIGNIFICATIVE_ALL_TREND]
         return {trend_type: self.trend_type_reduction_series(reduction_function=reduction_function,
                                                              df_bool=self.df_bool(trend_type, isin_parameters).copy())
-                for trend_type in trend_types_to_process}
+                for trend_type in self.trend_types_to_process}
+
+    @property
+    def trend_types_to_process(self):
+        return list(self.display_trend_types) + [AbstractUnivariateTest.SIGNIFICATIVE_ALL_TREND]
 
     def df_bool(self, display_trend_type, isin_parameters=None):
         return self.df_hypercube_trend_type.isin(AbstractUnivariateTest.get_real_trend_types(display_trend_type))
@@ -90,14 +96,17 @@ class AltitudeHypercubeVisualizer(AbstractHypercubeVisualizer):
         else:
             assert len(axes) == self.nb_rows
 
+        # Plot in one graph several graph that correspond to the same trend_type
         trend_type_to_series = self.trend_type_to_series(reduction_function, isin_parameters)
-        for ax_idx, ax in enumerate(axes):
+        end_idx = len(list(trend_type_to_series.values())[0])
+        axes_for_trend_type = axes[:end_idx]
+        for ax_idx, ax in enumerate(axes_for_trend_type):
             for display_trend_type in self.display_trend_types:
                 style = self.display_trend_type_to_style[display_trend_type]
-                percentages_values = trend_type_to_series[display_trend_type][ax_idx]
-                xlabel_values = list(percentages_values.index)
-                percentages_values = list(percentages_values.values)
-                ax.plot(xlabel_values, percentages_values, style + marker, label=display_trend_type)
+                values = trend_type_to_series[display_trend_type][ax_idx]
+                xlabel_values = list(values.index)
+                values = list(values.values)
+                ax.plot(xlabel_values, values, style + marker, label=display_trend_type)
 
             if ax_idx == 0:
                 # Global information
@@ -118,6 +127,17 @@ class AltitudeHypercubeVisualizer(AbstractHypercubeVisualizer):
             if plot_title is not None:
                 ax.set_title(plot_title)
 
+        # Plot other graphs where there is a single line that do not correspond to trend types
+        axes_remaining = axes[end_idx:]
+        ylabel_to_series = self.ylabel_to_series(reduction_function, isin_parameters)
+        assert len(axes_remaining) == len(ylabel_to_series), '{}, {}'.format(len(axes_remaining), len(ylabel_to_series))
+        for ax_idx, (ax, (ylabel, serie)) in enumerate(zip(axes_remaining, ylabel_to_series.items())):
+            xlabel_values = list(serie.index)
+            values = list(serie.values)
+            ax.plot(xlabel_values, values)
+            ax.set_ylabel(ylabel)
+            ax.set_title(plot_title)
+
         specific_title = 'Evolution of {} trends (significative or not) wrt to the {} with {}'.format(subtitle, xlabel,
                                                                                                       self.trend_test_name)
         specific_title += '\n ' + self.get_title_plot(xlabel)
@@ -142,20 +162,19 @@ class AltitudeHypercubeVisualizer(AbstractHypercubeVisualizer):
         return specific_title
 
     def load_trend_test_evolution_axes(self, nb_rows):
-        fig, axes = plt.subplots(nb_rows, 1, figsize=self.study_visualizer.figsize)
+        fig, axes = plt.subplots(nb_rows, 1, figsize=self.study_visualizer.figsize, constrained_layout=True)
         if not isinstance(axes, np.ndarray):
             axes = [axes]
         return axes
 
     def load_trend_test_evolution_axes_with_columns(self, nb_rows, nb_columns):
-        fig, axes = plt.subplots(nb_rows, nb_columns, figsize=self.study_visualizer.figsize)
+        fig, axes = plt.subplots(nb_rows, nb_columns, figsize=self.study_visualizer.figsize, constrained_layout=True)
         if not isinstance(axes, np.ndarray):
             axes = [axes]
         else:
             axes = axes.reshape((nb_rows * nb_columns))
         return axes
 
-
     def visualize_trend_test_repartition(self, reduction_function, axes=None, subtitle='', isin_parameters=None,
                                          show_or_save_to_file=True, plot_title=None):
         if axes is None:
@@ -198,7 +217,8 @@ class AltitudeHypercubeVisualizer(AbstractHypercubeVisualizer):
 
     def load_axes_for_trend_test_repartition(self, nb_rows):
         nb_trend_type = len(self.display_trend_type_to_style)
-        fig, axes = plt.subplots(nb_rows, nb_trend_type, figsize=self.study_visualizer.figsize)
+        fig, axes = plt.subplots(nb_rows, nb_trend_type, figsize=self.study_visualizer.figsize,
+                                 constrained_layout=True)
         return axes
 
     @property

experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/altitude_hypercube_visualizer_extended.py

@@ -4,7 +4,7 @@ from experiment.meteo_france_data.scm_models_data.visualization.hypercube_visual
 from experiment.meteo_france_data.scm_models_data.visualization.hypercube_visualization.altitude_hypercube_visualizer import \
     AltitudeHypercubeVisualizer
 from experiment.meteo_france_data.scm_models_data.visualization.hypercube_visualization.altitude_year_hypercube_visualizer import \
-    Altitude_Hypercube_Year_Visualizer, AltitudeHypercubeVisualizerBis
+    Altitude_Hypercube_Year_Visualizer, AltitudeHypercubeVisualizerBis, AltitudeHypercubeVisualizerWithoutTrendType
 
 
 class AltitudeHypercubeVisualizerExtended(AltitudeHypercubeVisualizer):
@@ -12,16 +12,20 @@ class AltitudeHypercubeVisualizerExtended(AltitudeHypercubeVisualizer):
     def df_bool(self, display_trend_type, isin_parameters=None):
         df_bool = super().df_bool(display_trend_type)
         # Slice a part of the array
+        df_bool = self.isin_slicing(df_bool, isin_parameters)
+        return df_bool
+
+    def isin_slicing(self, df, isin_parameters):
         if isin_parameters is not None:
             assert isinstance(isin_parameters, list)
             for isin_parameter in isin_parameters:
                 transpose, values, level = isin_parameter
                 if transpose:
-                    df_bool = df_bool.transpose()
-                ind = df_bool.index.isin(values=values, level=level)
-                res = df_bool.loc[ind].copy()
-                df_bool = res.transpose() if transpose else res
-        return df_bool
+                    df = df.transpose()
+                ind = df.index.isin(values=values, level=level)
+                res = df.loc[ind].copy()
+                df = res.transpose() if transpose else res
+        return df
 
     def _visualize_meta(self, visualization_function, loading_function, name_to_isin_parameters=None,
                         multiplication_factor_column=None):
@@ -70,15 +74,21 @@ class AltitudeHypercubeVisualizerExtended(AltitudeHypercubeVisualizer):
 
     @property
     def altitude_band_name_to_values(self):
-        altitude_band = 1000
-        group_idxs = [a // altitude_band for a in self.altitudes]
-        altitude_band_name_to_values = {'All altitudes': self.altitudes}
-        for group_idx in set(group_idxs):
-            values = [a for a, i in zip(self.altitudes, group_idxs) if i == group_idx]
-            altitude_band_name = '{}m <= altitude <={}m'.format(group_idx * altitude_band,
-                                                                (group_idx + 1) * altitude_band)
-            altitude_band_name_to_values[altitude_band_name] = values
-        return altitude_band_name_to_values
+        return {
+            '900m <= alti <= 3000m': self.altitudes,
+            '900m <= alti <= 1800m': [900, 1200, 1500, 1800],
+            '2100m <= alti <= 3000m': [2100, 2400, 2700, 3000],
+        }
+
+        # altitude_band = 1000
+        # group_idxs = [a // altitude_band for a in self.altitudes]
+        # altitude_band_name_to_values = {'All altitudes': self.altitudes}
+        # for group_idx in set(group_idxs):
+        #     values = [a for a, i in zip(self.altitudes, group_idxs) if i == group_idx]
+        #     altitude_band_name = '{}m <= altitude <={}m'.format(group_idx * altitude_band,
+        #                                                         (group_idx + 1) * altitude_band)
+        #     altitude_band_name_to_values[altitude_band_name] = values
+        # return altitude_band_name_to_values
 
     @property
     def altitude_band_name_to_isin_parameters(self):
@@ -104,11 +114,16 @@ class AltitudeHypercubeVisualizerExtended(AltitudeHypercubeVisualizer):
                                     multiplication_factor_column=len(self.altitude_band_name_to_isin_parameters))
 
 
-
-
 class AltitudeHypercubeVisualizerBisExtended(AltitudeHypercubeVisualizerExtended, AltitudeHypercubeVisualizerBis):
     pass
 
 
+class AltitudeHypercubeVisualizerWithoutTrendExtended(AltitudeHypercubeVisualizerExtended,
+                                                      AltitudeHypercubeVisualizerWithoutTrendType):
+
+    def df_loglikelihood(self, isin_parameters=None):
+        return self.isin_slicing(df=super().df_loglikelihood(), isin_parameters=isin_parameters)
+
+
 class AltitudeYearHypercubeVisualizerExtended(AltitudeHypercubeVisualizerExtended, Altitude_Hypercube_Year_Visualizer):
     pass

experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/altitude_year_hypercube_visualizer.py

@@ -4,6 +4,18 @@ from experiment.meteo_france_data.scm_models_data.visualization.hypercube_visual
     AltitudeHypercubeVisualizer
 
 
+class AltitudeHypercubeVisualizerWithoutTrendType(AltitudeHypercubeVisualizer):
+
+    def trend_type_to_series(self, reduction_function, isin_parameters=None):
+        return {trend_type: [] for trend_type in self.trend_types_to_process}
+
+    def df_loglikelihood(self, isin_parameters=None):
+        return -self.df_hypercube_trend_nllh
+
+    def ylabel_to_series(self, reduction_function, isin_parameters=None):
+        return {'mean LogLik': reduction_function(self.df_loglikelihood(isin_parameters=isin_parameters))}
+
+
 class AltitudeHypercubeVisualizerBis(AltitudeHypercubeVisualizer):
 
     @staticmethod

experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/main_hypercube_visualization.py

@@ -5,9 +5,10 @@ from itertools import product
 from experiment.meteo_france_data.scm_models_data.visualization.hypercube_visualization.altitude_hypercube_visualizer import \
     AltitudeHypercubeVisualizer
 from experiment.meteo_france_data.scm_models_data.visualization.hypercube_visualization.altitude_hypercube_visualizer_extended import \
-    AltitudeYearHypercubeVisualizerExtended, AltitudeHypercubeVisualizerExtended, AltitudeHypercubeVisualizerBisExtended
+    AltitudeYearHypercubeVisualizerExtended, AltitudeHypercubeVisualizerExtended, \
+    AltitudeHypercubeVisualizerBisExtended, AltitudeHypercubeVisualizerWithoutTrendExtended
 from experiment.meteo_france_data.scm_models_data.visualization.hypercube_visualization.altitude_year_hypercube_visualizer import \
-    Altitude_Hypercube_Year_Visualizer
+    Altitude_Hypercube_Year_Visualizer, AltitudeHypercubeVisualizerWithoutTrendType
 from experiment.meteo_france_data.scm_models_data.visualization.hypercube_visualization.quantity_altitude_visualizer import \
     QuantityAltitudeHypercubeVisualizer
 from experiment.meteo_france_data.scm_models_data.visualization.study_visualization.main_study_visualizer import \
@@ -98,8 +99,30 @@ def fast_altitude_year_hypercube():
                 # visualizer.visualize_massif_trend_test()
 
 
+def fast_altitude_without_trend_type_hypercube_extended():
+    save_to_file = False
+    only_first_one = False
+    nb_data_reduced_for_speed = True
+    altitudes = [ALL_ALTITUDES[3], ALL_ALTITUDES[-7]]
+    for study_class in SCM_STUDIES[:1]:
+        for last_starting_year in [None, 1989, 1999][:1]:
+            for trend_test_class in [GevLocationChangePointTest, GevScaleChangePointTest, GevShapeChangePointTest][:1]:
+                visualizers = [
+                    StudyVisualizer(study, temporal_non_stationarity=True, verbose=False, multiprocessing=True)
+                    for study in study_iterator(study_class=study_class, only_first_one=only_first_one,
+                                                altitudes=altitudes)]
+                altitude_to_visualizer = OrderedDict(zip(altitudes, visualizers))
+                visualizer = AltitudeHypercubeVisualizerWithoutTrendExtended(altitude_to_visualizer,
+                                                                             save_to_file=save_to_file,
+                                                                             trend_test_class=trend_test_class,
+                                                                             nb_data_reduced_for_speed=nb_data_reduced_for_speed,
+                                                                             last_starting_year=last_starting_year)
+                # visualizer.visualize_year_trend_test()
+                visualizer.vsualize_year_trend_by_regions_and_altitudes()
+
+
 def fast_altitude_year_hypercube_extended():
-    save_to_file = True
+    save_to_file = False
     only_first_one = False
     nb_data_reduced_for_speed = True
     altitudes = [ALL_ALTITUDES[3], ALL_ALTITUDES[-7]]
@@ -116,9 +139,9 @@ def fast_altitude_year_hypercube_extended():
                                                                  nb_data_reduced_for_speed=nb_data_reduced_for_speed,
                                                                  last_starting_year=last_starting_year)
                 # visualizer.visualize_year_trend_test()
-                visualizer.vsualize_year_trend_by_regions_and_altitudes()
                 # visualizer.visualize_altitude_trend_test()
                 # visualizer.visualize_massif_trend_test_by_altitudes()
+                visualizer.vsualize_year_trend_by_regions_and_altitudes()
                 visualizer.visualize_massif_trend_test_by_altitudes()
                 visualizer.visualize_altitute_trend_test_by_regions()
                 # visualizer.visualize_massif_trend_test()
@@ -154,10 +177,10 @@ def full_altitude_year_hypercube_extended():
     only_first_one = False
     nb_data_reduced_for_speed = False
     altitudes = ALL_ALTITUDES[3:-6]
-    for study_class in SCM_STUDIES[:]:
+    for study_class in SCM_STUDIES[1:]:
         for trend_test_class in [GevLocationChangePointTest, GevScaleChangePointTest,
                                  GevShapeChangePointTest][:1]:
-            years = [1967, 1977, 1987, 1997, 2007, None][-4:][::-1]
+            years = [1967, 1977, 1987, 1997, 2007, None][-4:]
             for last_starting_year in years:
                 for days in [1, 3][1:]:
                     visualizers = [
@@ -171,14 +194,39 @@ def full_altitude_year_hypercube_extended():
                                                                         nb_data_reduced_for_speed=nb_data_reduced_for_speed,
                                                                         last_starting_year=last_starting_year,
                                                                         )
+                    visualizer.vsualize_year_trend_by_regions_and_altitudes()
                     visualizer.visualize_altitute_trend_test_by_regions()
                     visualizer.visualize_massif_trend_test_by_altitudes()
-                    visualizer.vsualize_year_trend_by_regions_and_altitudes()
                     # visualizer.visualize_year_trend_test()
                     # visualizer.visualize_massif_trend_test()
                     # visualizer.visualize_altitude_trend_test()
 
 
+def full_altitude_year_hypercube__without_trend_extended():
+    save_to_file = True
+    only_first_one = False
+    nb_data_reduced_for_speed = False
+    altitudes = ALL_ALTITUDES[3:-6]
+    for study_class in SCM_STUDIES[1:]:
+        for trend_test_class in [GevLocationChangePointTest, GevScaleChangePointTest,
+                                 GevShapeChangePointTest][:1]:
+            years = [None][:]
+            for last_starting_year in years:
+                for days in [1, 3][1:]:
+                    visualizers = [
+                        StudyVisualizer(study, temporal_non_stationarity=True, verbose=False, multiprocessing=True)
+                        for study in study_iterator(study_class=study_class, only_first_one=only_first_one,
+                                                    altitudes=altitudes, nb_consecutive_days=days)]
+                    altitude_to_visualizer = OrderedDict(zip(altitudes, visualizers))
+                    visualizer = AltitudeHypercubeVisualizerWithoutTrendExtended(altitude_to_visualizer,
+                                                                                 save_to_file=save_to_file,
+                                                                                 trend_test_class=trend_test_class,
+                                                                                 nb_data_reduced_for_speed=nb_data_reduced_for_speed,
+                                                                                 last_starting_year=last_starting_year,
+                                                                                 )
+                    visualizer.vsualize_year_trend_by_regions_and_altitudes()
+
+
 def fast_quantity_altitude_hypercube():
     save_to_file = False
     only_first_one = False
@@ -203,8 +251,11 @@ def main_run():
     # fast_altitude_hypercube()
     # fast_altitude_year_hypercube()
 
+    # fast_altitude_without_trend_type_hypercube_extended()
+    full_altitude_year_hypercube__without_trend_extended()
+
     # fast_altitude_year_hypercube_extended()
-    full_altitude_year_hypercube_extended()
+    # full_altitude_year_hypercube_extended()
 
     # full_altitude_year_hypercube()
     # fast_quantity_altitude_hypercube()