[HYPERCUBE VISUALIZER] add last_starting_year as argument. add many improvement for the displays

experiment/meteo_france_data/visualization/hypercube_visualization/abstract_hypercube_visualizer.py

@@ -20,12 +20,18 @@ class AbstractHypercubeVisualizer(object):
                  trend_test_class,
                  nb_data_reduced_for_speed=False,
                  save_to_file=False,
-                 nb_top_likelihood_values=1):
+                 nb_top_likelihood_values=1,
+                 last_starting_year=None):
+        self.last_starting_year = last_starting_year
         self.nb_data_for_fast_mode = 7 if nb_data_reduced_for_speed else None
         self.nb_top_likelihood_values = nb_top_likelihood_values
         self.save_to_file = save_to_file
         self.trend_test_class = trend_test_class
         self.tuple_to_study_visualizer = tuple_to_study_visualizer  # type: Dict[Tuple, StudyVisualizer]
+        print('Hypercube with parameters:')
+        print(self.last_starting_year)
+        print(self.trend_test_class)
+        # print(self.nb_data_for_fast_mode)
 
     # Main attributes defining the hypercube
 
@@ -35,7 +41,9 @@ class AbstractHypercubeVisualizer(object):
 
     @cached_property
     def starting_years(self):
-        starting_years = self.study_visualizer.starting_years[:]
+        starting_years = self.study_visualizer.starting_years
+        if self.last_starting_year is not None:
+            starting_years = [year for year in starting_years if year <= self.last_starting_year]
         if self.nb_data_for_fast_mode is not None:
             starting_years = starting_years[:self.nb_data_for_fast_mode]
         return starting_years
@@ -86,9 +94,3 @@ class AbstractHypercubeVisualizer(object):
     @property
     def study(self):
         return self.study_visualizer.study
-
-    @property
-    def starting_year_to_weights(self):
-        # Load uniform weights by default
-        uniform_weight = 1 / len(self.starting_years)
-        return {year: uniform_weight for year in self.starting_years}

experiment/meteo_france_data/visualization/hypercube_visualization/altitude_hypercube_visualizer.py

+from typing import List
+
 import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
@@ -33,8 +35,10 @@ class AltitudeHypercubeVisualizer(AbstractHypercubeVisualizer):
 
     def trend_type_to_series(self, reduction_function):
         # Map each trend type to its serie with percentages
-        return {display_trend_type: self.trend_type_reduction(reduction_function, display_trend_type)[0]
-                for display_trend_type in self.display_trend_types}
+        # Define here all the trend type we might need in the results/displays
+        trend_types_to_process = list(self.display_trend_types) + [AbstractUnivariateTest.SIGNIFICATIVE_ALL_TREND]
+        return {trend_type: self.trend_type_reduction(reduction_function, trend_type)[0]
+                for trend_type in trend_types_to_process}
 
     def trend_type_reduction(self, reduction_function, display_trend_type):
         # Reduce df_bool df to a serie s_trend_type_percentage
@@ -53,10 +57,12 @@ class AltitudeHypercubeVisualizer(AbstractHypercubeVisualizer):
 
     def subtitle_to_reduction_function(self, reduction_function, level=None, add_detailed_plot=False, subtitle=None):
         def reduction_function_with_level(df_bool, **kwargs):
-            return reduction_function(df_bool, **kwargs) if level is None else reduction_function(df_bool, level, **kwargs)
+            return reduction_function(df_bool, **kwargs) if level is None else reduction_function(df_bool, level,
+                                                                                                  **kwargs)
 
         if subtitle is None:
-            subtitle = self.study.variable_name
+            subtitle = self.study.variable_name[:5]
+
         return {subtitle: reduction_function_with_level}
 
     def get_title_plot(self, xlabel, ax_idx=None):
@@ -71,6 +77,7 @@ class AltitudeHypercubeVisualizer(AbstractHypercubeVisualizer):
             if xlabel != 'starting years':
                 labels.remove('starting years')
             common_txt = 'averaged on {}'.format(' & '.join(labels))
+            common_txt += 'with any starting year <= {}'.format(str(self.last_starting_year))
             return common_txt
 
     def visualize_trend_test_evolution(self, reduction_function, xlabel, xlabel_values, axes=None, marker='o',
@@ -82,10 +89,10 @@ class AltitudeHypercubeVisualizer(AbstractHypercubeVisualizer):
 
         trend_type_to_series = self.trend_type_to_series(reduction_function)
         for ax_idx, ax in enumerate(axes):
-            for trend_type in self.display_trend_types:
-                style = self.display_trend_type_to_style[trend_type]
-                percentages_values = trend_type_to_series[trend_type][ax_idx]
-                ax.plot(xlabel_values, percentages_values, style + marker, label=trend_type)
+            for display_trend_type in self.display_trend_types:
+                style = self.display_trend_type_to_style[display_trend_type]
+                percentages_values = trend_type_to_series[display_trend_type][ax_idx]
+                ax.plot(xlabel_values, percentages_values, style + marker, label=display_trend_type)
 
             if ax_idx == 0:
                 # Global information
@@ -104,11 +111,25 @@ class AltitudeHypercubeVisualizer(AbstractHypercubeVisualizer):
             ax.grid()
             ax.legend()
 
-        title = 'Evolution of {} trends (significative or not) wrt to the {} with {}'.format(subtitle, xlabel,
-                                                                                             self.trend_test_name)
-        title += '\n ' + self.get_title_plot(xlabel)
-        plt.suptitle(title)
-        self.show_or_save_to_file(specific_title=title)
+        specific_title = 'Evolution of {} trends (significative or not) wrt to the {} with {}'.format(subtitle, xlabel,
+                                                                                                      self.trend_test_name)
+        specific_title += '\n ' + self.get_title_plot(xlabel)
+
+        # Figure title
+        specific_title += '\n'
+
+        trend_types = [AbstractUnivariateTest.SIGNIFICATIVE_ALL_TREND,
+                       AbstractUnivariateTest.SIGNIFICATIVE_POSITIVE_TREND,
+                       AbstractUnivariateTest.SIGNIFICATIVE_NEGATIVE_TREND]
+        series = [trend_type_to_series[trend_type][0] for trend_type in trend_types]
+        percents = [serie.sum() if xlabel == STARTING_YEARS_XLABEL else serie.mean() for serie in series]
+        percents = [round(p) for p in percents]
+
+        specific_title += 'Total ' if xlabel == STARTING_YEARS_XLABEL else 'Mean '
+        specific_title += 'of significative trends: {} (+:{}  -{})'.format(*percents)
+        plt.suptitle(specific_title)
+
+        self.show_or_save_to_file(specific_title=specific_title)
 
     def visualize_trend_test_repartition(self, reduction_function, axes=None, subtitle=''):
         if axes is None:
@@ -116,16 +137,21 @@ class AltitudeHypercubeVisualizer(AbstractHypercubeVisualizer):
             fig, axes = plt.subplots(self.nb_axes, nb_trend_type, figsize=self.study_visualizer.figsize)
 
         for i, axes_row in enumerate(axes):
-            trend_type_to_serie = {k: v[i] for k, v in self.trend_type_to_series(reduction_function).items()}
+            trend_type_to_serie = {k: v[i].replace(0.0, np.nan) for k, v in
+                                   self.trend_type_to_series(reduction_function).items()}
             vmax = max([s.max() for s in trend_type_to_serie.values()])
             vmin = min([s.min() for s in trend_type_to_serie.values()])
             vmax = max(vmax, 0.01)
-            for ax, trend_type in zip(axes_row, self.display_trend_types):
-                serie = trend_type_to_serie[trend_type]
+            if vmin == vmax:
+                epislon = 0.001 * vmax
+                vmin -= epislon
+                vmax += epislon
+            for ax, display_trend_type in zip(axes_row, self.display_trend_types):
+                serie = trend_type_to_serie[display_trend_type]
                 massif_to_value = dict(serie)
-                cmap = self.trend_test_class.get_cmap_from_trend_type(trend_type)
+                cmap = self.trend_test_class.get_cmap_from_trend_type(display_trend_type)
                 self.study.visualize_study(ax, massif_to_value, show=False, cmap=cmap, label=None, vmax=vmax, vmin=vmin)
-                ax.set_title(trend_type)
+                ax.set_title(display_trend_type)
             row_title = self.get_title_plot(xlabel='massifs', ax_idx=i)
             StudyVisualizer.clean_axes_write_title_on_the_left(axes_row, row_title, left_border=None)
 

experiment/meteo_france_data/visualization/hypercube_visualization/main_hypercube_visualization.py

@@ -16,22 +16,22 @@ from experiment.trend_analysis.univariate_test.abstract_gev_change_point_test im
 from utils import get_display_name_from_object_type
 
 
-# def full_trends_with_altitude_hypercube():
-#     save_to_file = True
-#     only_first_one = False
-#     fast = False
-#     altitudes = ALL_ALTITUDES[3:-6]
-#     for study_class in SCM_STUDIES[:]:
-#         for trend_test_class in [MannKendallTrendTest, GevLocationTrendTest, GevScaleTrendTest, GevShapeTrendTest][:]:
-#             visualizers = [StudyVisualizer(study, temporal_non_stationarity=True, verbose=False, multiprocessing=True)
-#                            for study in study_iterator(study_class=study_class, only_first_one=only_first_one,
-#                                                        altitudes=altitudes)]
-#             altitude_to_visualizer = OrderedDict(zip(altitudes, visualizers))
-#             visualizer = AltitudeHypercubeVisualizer(altitude_to_visualizer, save_to_file=save_to_file,
-#                                                      trend_test_class=trend_test_class, fast=fast)
-#             visualizer.visualize_massif_trend_test()
-#             visualizer.visualize_year_trend_test()
-#             visualizer.visualize_altitude_trend_test()
+def full_trends_with_altitude_hypercube():
+    save_to_file = True
+    only_first_one = False
+    fast = False
+    altitudes = ALL_ALTITUDES[3:-6]
+    for study_class in SCM_STUDIES[:1]:
+        for trend_test_class in [GevLocationChangePointTest, GevScaleChangePointTest, GevShapeChangePointTest][:1]:
+            visualizers = [StudyVisualizer(study, temporal_non_stationarity=True, verbose=False, multiprocessing=True)
+                           for study in study_iterator(study_class=study_class, only_first_one=only_first_one,
+                                                       altitudes=altitudes)]
+            altitude_to_visualizer = OrderedDict(zip(altitudes, visualizers))
+            visualizer = AltitudeHypercubeVisualizer(altitude_to_visualizer, save_to_file=save_to_file,
+                                                     trend_test_class=trend_test_class, nb_data_reduced_for_speed=False)
+            visualizer.visualize_massif_trend_test()
+            visualizer.visualize_year_trend_test()
+            visualizer.visualize_altitude_trend_test()
 
 
 def full_quantity_altitude_hypercube():
@@ -59,37 +59,39 @@ def fast_altitude_hypercube():
     save_to_file = False
     only_first_one = False
     fast = True
-    altitudes = ALL_ALTITUDES[2:4]
+    altitudes = [ALL_ALTITUDES[3], ALL_ALTITUDES[-7]]
     for study_class in SCM_STUDIES[:1]:
-        for trend_test_class in [GevLocationChangePointTest, GevScaleChangePointTest, GevShapeChangePointTest][1:2]:
+        for trend_test_class in [GevLocationChangePointTest, GevScaleChangePointTest, GevShapeChangePointTest][:1]:
             visualizers = [StudyVisualizer(study, temporal_non_stationarity=True, verbose=False, multiprocessing=True)
                            for study in study_iterator(study_class=study_class, only_first_one=only_first_one,
                                                        altitudes=altitudes)]
             altitude_to_visualizer = OrderedDict(zip(altitudes, visualizers))
             visualizer = AltitudeHypercubeVisualizer(altitude_to_visualizer, save_to_file=save_to_file,
-                                                     trend_test_class=trend_test_class, fast=fast)
+                                                     trend_test_class=trend_test_class, nb_data_reduced_for_speed=fast)
             # visualizer.visualize_year_trend_test()
             # visualizer.visualize_massif_trend_test()
             visualizer.visualize_altitude_trend_test()
 
 
 def fast_altitude_year_hypercube():
-    save_to_file = False
+    save_to_file = True
     only_first_one = False
     nb_data_reduced_for_speed = True
     altitudes = [ALL_ALTITUDES[3], ALL_ALTITUDES[-7]]
     for study_class in SCM_STUDIES[:1]:
-        for trend_test_class in [GevLocationChangePointTest, GevScaleChangePointTest, GevShapeChangePointTest][:1]:
-            visualizers = [StudyVisualizer(study, temporal_non_stationarity=True, verbose=False, multiprocessing=True)
-                           for study in study_iterator(study_class=study_class, only_first_one=only_first_one,
-                                                       altitudes=altitudes)]
-            altitude_to_visualizer = OrderedDict(zip(altitudes, visualizers))
-            visualizer = Altitude_Hypercube_Year_Visualizer(altitude_to_visualizer, save_to_file=save_to_file,
-                                                            trend_test_class=trend_test_class,
-                                                            nb_data_reduced_for_speed=nb_data_reduced_for_speed)
-            visualizer.visualize_year_trend_test()
-            # visualizer.visualize_altitude_trend_test()
-            # visualizer.visualize_massif_trend_test()
+        for last_starting_year in [1989, 1999]:
+            for trend_test_class in [GevLocationChangePointTest, GevScaleChangePointTest, GevShapeChangePointTest][:1]:
+                visualizers = [StudyVisualizer(study, temporal_non_stationarity=True, verbose=False, multiprocessing=True)
+                               for study in study_iterator(study_class=study_class, only_first_one=only_first_one,
+                                                           altitudes=altitudes)]
+                altitude_to_visualizer = OrderedDict(zip(altitudes, visualizers))
+                visualizer = Altitude_Hypercube_Year_Visualizer(altitude_to_visualizer, save_to_file=save_to_file,
+                                                                trend_test_class=trend_test_class,
+                                                                nb_data_reduced_for_speed=nb_data_reduced_for_speed,
+                                                                last_starting_year=last_starting_year)
+                visualizer.visualize_year_trend_test()
+                visualizer.visualize_altitude_trend_test()
+                visualizer.visualize_massif_trend_test()
 
 
 def full_altitude_year_hypercube():
@@ -100,17 +102,19 @@ def full_altitude_year_hypercube():
     for study_class in SCM_STUDIES[:1]:
         for trend_test_class in [GevLocationChangePointTest, GevScaleChangePointTest,
                                  GevShapeChangePointTest][:1]:
-            visualizers = [
-                StudyVisualizer(study, temporal_non_stationarity=True, verbose=False, multiprocessing=True)
-                for study in study_iterator(study_class=study_class, only_first_one=only_first_one,
-                                            altitudes=altitudes)]
-            altitude_to_visualizer = OrderedDict(zip(altitudes, visualizers))
-            visualizer = Altitude_Hypercube_Year_Visualizer(altitude_to_visualizer,
-                                                            save_to_file=save_to_file,
-                                                            trend_test_class=trend_test_class, nb_data_reduced_for_speed=nb_data_reduced_for_speed)
-            visualizer.visualize_year_trend_test()
-            visualizer.visualize_massif_trend_test()
-            visualizer.visualize_altitude_trend_test()
+            for last_starting_year in [1967, 1977, 1987, 1997, 2007, None]:
+                visualizers = [
+                    StudyVisualizer(study, temporal_non_stationarity=True, verbose=False, multiprocessing=True)
+                    for study in study_iterator(study_class=study_class, only_first_one=only_first_one,
+                                                altitudes=altitudes)]
+                altitude_to_visualizer = OrderedDict(zip(altitudes, visualizers))
+                visualizer = Altitude_Hypercube_Year_Visualizer(altitude_to_visualizer,
+                                                                save_to_file=save_to_file,
+                                                                trend_test_class=trend_test_class, nb_data_reduced_for_speed=nb_data_reduced_for_speed,
+                                                                last_starting_year=last_starting_year)
+                visualizer.visualize_year_trend_test()
+                visualizer.visualize_massif_trend_test()
+                visualizer.visualize_altitude_trend_test()
 
 
 def fast_quantity_altitude_hypercube():