[SCM][HYPERCUBE] Add quantity altitude hypercube visualizer

experiment/meteo_france_SCM_study/visualization/hypercube_visualization/hypercube_visualizer.py

@@ -47,6 +47,9 @@ class HypercubeVisualizer(object):
             for study_visualizer in self.tuple_to_study_visualizer.values()]
         return dict(zip(self.tuple_to_study_visualizer.keys(), df_spatio_temporal_trend_types))
 
+    def tuple_values(self, idx):
+        return sorted(set([t[idx] if isinstance(t, tuple) else t for t in self.tuple_to_study_visualizer.keys()]))
+
     @cached_property
     def df_hypercube(self) -> pd.DataFrame:
         keys = list(self.tuple_to_df_trend_type.keys())
@@ -88,8 +91,8 @@ class AltitudeHypercubeVisualizer(HypercubeVisualizer):
 
     @property
     def altitudes(self):
-        return list(self.tuple_to_study_visualizer.keys())
-    
+        return self.tuple_values(idx=0)
+
     @property
     def trend_type_to_style(self):
         return self.trend_test_class.trend_type_to_style()
@@ -97,7 +100,7 @@ class AltitudeHypercubeVisualizer(HypercubeVisualizer):
     @property
     def trend_types(self):
         return self.trend_type_to_style.keys()
-    
+
     def trend_type_to_s_percentages(self, reduction_function):
         # Map each trend type to its serie with percentages
         trend_type_to_s_percentages = {}
@@ -106,22 +109,30 @@ class AltitudeHypercubeVisualizer(HypercubeVisualizer):
             # Reduce the entire dataframe to a serie
             s_percentages = reduction_function(df_bool)
             assert isinstance(s_percentages, pd.Series)
+            assert not isinstance(s_percentages.index, pd.MultiIndex)
             s_percentages *= 100
             trend_type_to_s_percentages[trend_type] = s_percentages
         # Post processing - Add the significant trend into the count of normal trend
-        trend_type_to_s_percentages[AbstractTrendTest.POSITIVE_TREND] += trend_type_to_s_percentages[AbstractTrendTest.SIGNIFICATIVE_POSITIVE_TREND]
-        trend_type_to_s_percentages[AbstractTrendTest.NEGATIVE_TREND] += trend_type_to_s_percentages[AbstractTrendTest.SIGNIFICATIVE_NEGATIVE_TREND]
+        trend_type_to_s_percentages[AbstractTrendTest.POSITIVE_TREND] += trend_type_to_s_percentages[
+            AbstractTrendTest.SIGNIFICATIVE_POSITIVE_TREND]
+        trend_type_to_s_percentages[AbstractTrendTest.NEGATIVE_TREND] += trend_type_to_s_percentages[
+            AbstractTrendTest.SIGNIFICATIVE_NEGATIVE_TREND]
         return trend_type_to_s_percentages
-    
+
     def visualize_trend_test_evolution(self, reduction_function, xlabel, xlabel_values, ax=None, marker='o'):
         if ax is None:
             fig, ax = plt.subplots(1, 1, figsize=self.study_visualizer.figsize)
 
-        trend_type_to_s_percentages = self.trend_type_to_s_percentages(reduction_function)
+        trend_type_to_percentages_values = {k: s.values for k, s in self.trend_type_to_s_percentages(reduction_function).items()}
         for trend_type in self.trend_types:
             style = self.trend_type_to_style[trend_type]
-            s_percentages = trend_type_to_s_percentages[trend_type]
-            ax.plot(xlabel_values, s_percentages.values, style + marker, label=trend_type)
+            percentages_values = trend_type_to_percentages_values[trend_type]
+            ax.plot(xlabel_values, percentages_values, style + marker, label=trend_type)
+
+        # Plot the total value of significative values
+        significative_values = trend_type_to_percentages_values[AbstractTrendTest.SIGNIFICATIVE_NEGATIVE_TREND] \
+                               + trend_type_to_percentages_values[AbstractTrendTest.SIGNIFICATIVE_POSITIVE_TREND]
+        ax.plot(xlabel_values, significative_values, 'y-' + marker, label=AbstractTrendTest.SIGNIFICATIVE + ' trends')
 
         # Global information
         added_str = 'weighted '
@@ -165,33 +176,54 @@ class AltitudeHypercubeVisualizer(HypercubeVisualizer):
 
     def visualize_year_trend_test(self, ax=None, marker='o'):
         def year_reduction(df_bool):
-            # Take the mean with respect the massifs
-            df_bool = df_bool.mean(axis=0, level=0)
-            # Take the mean with respect to the altitude
+            # Take the mean with respect to all the first axis indices
             return df_bool.mean(axis=0)
 
         self.visualize_trend_test_evolution(reduction_function=year_reduction, xlabel='starting years',
                                             xlabel_values=self.starting_years, ax=ax, marker=marker)
 
+    @property
+    def altitude_index_level(self):
+        return 0
+
     def visualize_altitude_trend_test(self, ax=None, marker='o'):
         def altitude_reduction(df_bool):
             # Take the mean with respect to the years
             df_bool = df_bool.mean(axis=1)
             # Take the mean with respect the massifs
-            return df_bool.mean(axis=0, level=0)
+            return df_bool.mean(level=self.altitude_index_level)
 
         self.visualize_trend_test_evolution(reduction_function=altitude_reduction, xlabel='altitude',
                                             xlabel_values=self.altitudes, ax=ax, marker=marker)
 
+    @property
+    def massif_index_level(self):
+        return 1
+
     def visualize_massif_trend_test(self, axes=None):
         def massif_reduction(df_bool):
             # Take the mean with respect to the years
             df_bool = df_bool.mean(axis=1)
             # Take the mean with respect the altitude
-            return df_bool.mean(axis=0, level=1)
+            return df_bool.mean(level=self.massif_index_level)
 
         self.visualize_trend_test_repartition(massif_reduction, axes)
 
 
-class QuantitityAltitudeHypercubeVisualizer(HypercubeVisualizer):
-    pass
+class QuantityAltitudeHypercubeVisualizer(AltitudeHypercubeVisualizer):
+
+    @property
+    def quantities(self):
+        return self.tuple_values(idx=0)
+
+    @property
+    def altitudes(self):
+        return self.tuple_values(idx=1)
+
+    @property
+    def altitude_index_level(self):
+        return 1
+
+    @property
+    def massif_index_level(self):
+        return 2

experiment/meteo_france_SCM_study/visualization/hypercube_visualization/main_hypercube_visualization.py

 import time
+from itertools import product
 from collections import OrderedDict
 
 from experiment.meteo_france_SCM_study.visualization.hypercube_visualization.hypercube_visualizer import \
-    AltitudeHypercubeVisualizer
+    AltitudeHypercubeVisualizer, QuantityAltitudeHypercubeVisualizer
 from experiment.meteo_france_SCM_study.visualization.study_visualization.main_study_visualizer import ALL_ALTITUDES, \
-    SCM_STUDIES, study_iterator
+    SCM_STUDIES, study_iterator, study_iterator_global
 from experiment.meteo_france_SCM_study.visualization.study_visualization.study_visualizer import StudyVisualizer
 from experiment.trend_analysis.univariate_trend_test.abstract_gev_trend_test import GevLocationTrendTest, \
     GevScaleTrendTest, GevShapeTrendTest
 from experiment.trend_analysis.univariate_trend_test.abstract_trend_test import MannKendallTrendTest
+from utils import get_display_name_from_object_type
 
 
-def altitude_trend_with_hypercube():
+def full_trends_with_altitude_hypercube():
     save_to_file = True
     only_first_one = False
     fast = False
     altitudes = ALL_ALTITUDES[3:-6]
-    # altitudes = ALL_ALTITUDES[2:4]
     for study_class in SCM_STUDIES[:]:
         for trend_test_class in [MannKendallTrendTest, GevLocationTrendTest, GevScaleTrendTest, GevShapeTrendTest][:]:
             visualizers = [StudyVisualizer(study, temporal_non_stationarity=True, verbose=False, multiprocessing=True)
@@ -25,14 +26,15 @@ def altitude_trend_with_hypercube():
             altitude_to_visualizer = OrderedDict(zip(altitudes, visualizers))
             visualizer = AltitudeHypercubeVisualizer(altitude_to_visualizer, save_to_file=save_to_file,
                                                      trend_test_class=trend_test_class, fast=fast)
+            visualizer.visualize_massif_trend_test()
+            visualizer.visualize_year_trend_test()
             visualizer.visualize_altitude_trend_test()
 
 
-def spatial_trend_with_hypercube():
+def fast_trends_with_altitude_hypercube():
     save_to_file = False
     only_first_one = False
     fast = True
-    # altitudes = ALL_ALTITUDES[3:-6]
     altitudes = ALL_ALTITUDES[2:4]
     for study_class in SCM_STUDIES[:1]:
         for trend_test_class in [MannKendallTrendTest, GevLocationTrendTest, GevScaleTrendTest, GevShapeTrendTest][:1]:
@@ -42,16 +44,55 @@ def spatial_trend_with_hypercube():
             altitude_to_visualizer = OrderedDict(zip(altitudes, visualizers))
             visualizer = AltitudeHypercubeVisualizer(altitude_to_visualizer, save_to_file=save_to_file,
                                                      trend_test_class=trend_test_class, fast=fast)
-            visualizer.visualize_massif_trend_test()
             visualizer.visualize_year_trend_test()
-            visualizer.visualize_altitude_trend_test()
+            # visualizer.visualize_massif_trend_test()
+            # visualizer.visualize_altitude_trend_test()
+
+
+def full_trends_with_quantity_altitude_hypercube():
+    save_to_file = True
+    only_first_one = False
+    fast = False
+    altitudes = ALL_ALTITUDES[3:-6]
+    study_classes = SCM_STUDIES
+    for trend_test_class in [MannKendallTrendTest, GevLocationTrendTest, GevScaleTrendTest, GevShapeTrendTest][:1]:
+        visualizers = [StudyVisualizer(study, temporal_non_stationarity=True, verbose=False, multiprocessing=True)
+                       for study in study_iterator_global(study_classes=study_classes, only_first_one=only_first_one,
+                                                          altitudes=altitudes)]
+        study_classes_str = [get_display_name_from_object_type(c) for c in study_classes]
+        quantity_altitude_tuples = list(product(study_classes_str, altitudes))
+        quantity_altitude_to_visualizer = OrderedDict(zip(quantity_altitude_tuples, visualizers))
+        visualizer = QuantityAltitudeHypercubeVisualizer(quantity_altitude_to_visualizer, save_to_file=save_to_file,
+                                                         trend_test_class=trend_test_class, fast=fast)
+        visualizer.visualize_year_trend_test()
+        # visualizer.visualize_massif_trend_test()
+        # visualizer.visualize_altitude_trend_test()
 
 
+def fast_trends_with_quantity_altitude_hypercube():
+    save_to_file = False
+    only_first_one = False
+    fast = True
+    altitudes = ALL_ALTITUDES[2:4]
+    study_classes = SCM_STUDIES[:2]
+    for trend_test_class in [MannKendallTrendTest, GevLocationTrendTest, GevScaleTrendTest, GevShapeTrendTest][:1]:
+        visualizers = [StudyVisualizer(study, temporal_non_stationarity=True, verbose=False, multiprocessing=True)
+                       for study in study_iterator_global(study_classes=study_classes, only_first_one=only_first_one,
+                                                          altitudes=altitudes)]
+        study_classes_str = [get_display_name_from_object_type(c) for c in study_classes]
+        quantity_altitude_tuples = list(product(study_classes_str, altitudes))
+        quantity_altitude_to_visualizer = OrderedDict(zip(quantity_altitude_tuples, visualizers))
+        visualizer = QuantityAltitudeHypercubeVisualizer(quantity_altitude_to_visualizer, save_to_file=save_to_file,
+                                                         trend_test_class=trend_test_class, fast=fast)
+        # visualizer.visualize_year_trend_test()
+        # visualizer.visualize_massif_trend_test()
+        visualizer.visualize_altitude_trend_test()
 
 def main_run():
-    # altitude_trends()
-    # altitude_trends_significant()
-    spatial_trend_with_hypercube()
+    # fast_trends_with_altitude_hypercube()
+    # full_trends_with_altitude_hypercube()
+    # fast_trends_with_quantity_altitude_hypercube()
+    full_trends_with_quantity_altitude_hypercube()
 
 
 if __name__ == '__main__':

experiment/trend_analysis/univariate_trend_test/abstract_gev_trend_test.py

+import numpy as np
 import pandas as pd
 from scipy.stats import chi2
-import numpy as np
-from sklearn.preprocessing import normalize
+
 from experiment.trend_analysis.univariate_trend_test.abstract_trend_test import AbstractTrendTest
 from extreme_estimator.estimator.margin_estimator.abstract_margin_estimator import LinearMarginEstimator
 from extreme_estimator.extreme_models.margin_model.temporal_linear_margin_model import StationaryStationModel, \
@@ -13,8 +13,6 @@ from spatio_temporal_dataset.coordinates.temporal_coordinates.abstract_temporal_
     AbstractTemporalCoordinates
 from spatio_temporal_dataset.coordinates.transformed_coordinates.transformation.abstract_transformation import \
     CenteredScaledNormalization
-from spatio_temporal_dataset.coordinates.transformed_coordinates.transformation.uniform_normalization import \
-    BetweenZeroAndOneNormalization
 from spatio_temporal_dataset.dataset.abstract_dataset import AbstractDataset
 from spatio_temporal_dataset.spatio_temporal_observations.abstract_spatio_temporal_observations import \
     AbstractSpatioTemporalObservations

experiment/trend_analysis/univariate_trend_test/abstract_trend_test.py

 import random
+import warnings
+
 import matplotlib.pyplot as plt
 from collections import OrderedDict
 
@@ -80,6 +82,9 @@ class ExampleRandomTrendTest(AbstractTrendTest):
         return random.randint(1, 10) == 10
 
 
+class WarningScoreValue(Warning):
+    pass
+
 class MannKendallTrendTest(AbstractTrendTest):
 
     def __init__(self, years_after_change_point, maxima_after_change_point):
@@ -94,7 +99,9 @@ class MannKendallTrendTest(AbstractTrendTest):
                                   eps=1e-5,
                                   alpha=self.SIGNIFICANCE_LEVEL,
                                   Ha='upordown')
-        assert S == self.score_value
+        # Raise warning if scores are differents
+        if S != self.score_value:
+            warnings.warn('S={} is different that score_value={}'.format(S, self.score_value), WarningScoreValue)
         self.MK = MK
 
     @property