[SCM][TREND TEST] refactor

experiment/meteo_france_SCM_study/visualization/studies_visualization/main_studies_visualizer.py

@@ -46,13 +46,13 @@ def altitude_trends_significant():
     # altitudes = ALL_ALTITUDES[3:5]
     altitudes = ALL_ALTITUDES[2:4]
     for study_class in SCM_STUDIES[:1]:
-        trend_test_classes = [MannKendallTrendTest, GevLocationTrendTest, GevScaleTrendTest, GevShapeTrendTest][3:]
+        trend_test_classes = [MannKendallTrendTest, GevLocationTrendTest, GevScaleTrendTest, GevShapeTrendTest][:1]
         visualizers = [StudyVisualizer(study, temporal_non_stationarity=True, verbose=False)
                        for study in study_iterator_global(study_classes=[study_class], only_first_one=only_first_one,
                                                           altitudes=altitudes)]
         altitude_to_visualizer = OrderedDict(zip(altitudes, visualizers))
         visualizer = AltitudeVisualizer(altitude_to_visualizer, multiprocessing=False, save_to_file=save_to_file)
-        visualizer.trend_tests_percentage_evolution(trend_test_classes, starting_year_to_weights=None)
+        visualizer.trend_tests_percentage_evolution_with_altitude(trend_test_classes, starting_year_to_weights=None)
 
 
 if __name__ == '__main__':

experiment/meteo_france_SCM_study/visualization/studies_visualization/studies_visualizer.py

@@ -164,12 +164,15 @@ class AltitudeVisualizer(object):
 
     # Trend tests evolution
 
-    def trend_tests_percentage_evolution(self, trend_test_classes, starting_year_to_weights: None):
+    def trend_tests_percentage_evolution_with_altitude(self, trend_test_classes, starting_year_to_weights: None):
         # Load uniform weights by default
         if starting_year_to_weights is None:
             startings_years = self.any_study_visualizer.starting_years
             uniform_weight = 1 / len(startings_years)
             starting_year_to_weights = {year: uniform_weight for year in startings_years}
+
+            # To get a single year, I could do:
+            # starting_year_to_weights = {1980: 1.0}
         else:
             uniform_weight = 0.0
 
@@ -181,7 +184,7 @@ class AltitudeVisualizer(object):
         # Add a second legend for the color and to explain the line
 
         for marker, trend_test_class in zip(markers, trend_test_classes):
-            self.trend_test_percentages_evolution(ax, marker, trend_test_class, starting_year_to_weights)
+            self.trend_test_weighted_percentages(ax, marker, trend_test_class, starting_year_to_weights)
 
         # Add the color legend
         handles, labels = ax.get_legend_handles_labels()
@@ -195,6 +198,8 @@ class AltitudeVisualizer(object):
         # Add the marker legend
         names = [get_display_name_from_object_type(c) for c in trend_test_classes]
         handles_ax2, labels_ax2 = handles[::nb_trend_types], names
+        for handle in handles_ax2:
+            handle.set_color('k')
         ax2 = ax.twinx()
         ax2.legend(handles_ax2, labels_ax2, loc=2)
         ax2.set_yticks([])
@@ -210,25 +215,18 @@ class AltitudeVisualizer(object):
         ax.set_title(title)
         self.show_or_save_to_file(specific_title=title)
 
-    def trend_test_percentages_evolution(self, ax, marker, trend_test_class, starting_year_to_weights):
-        """
-        Positive trend in green
-        Negative trend in red
-        Non significative trend with dotted line
-        Significative trend with real line
-
-        :return:
-        """
+    def trend_test_weighted_percentages(self, ax, marker, trend_test_class, starting_year_to_weights):
         # Build OrderedDict mapping altitude to a mean serie
         altitude_to_serie_with_mean_percentages = OrderedDict()
         for altitude, study_visualizer in self.altitude_to_study_visualizer.items():
-            s = study_visualizer.serie_mean_trend_test_count(trend_test_class, starting_year_to_weights)
+            s = study_visualizer.df_trend_test_count(trend_test_class, starting_year_to_weights).mean(axis=1)
             altitude_to_serie_with_mean_percentages[altitude] = s
-        # Plot lines
-        for trend_type, style in AbstractTrendTest.trend_type_to_style().items():
-            percentages = [v.loc[trend_type] if trend_type in v.index else 0.0
+        # Plot weighted percentages over the years
+        for trend_type, style in trend_test_class.trend_type_to_style().items():
+
+            weighted_percentages = [v.loc[trend_type] if trend_type in v.index else 0.0
                            for v in altitude_to_serie_with_mean_percentages.values()]
-            if set(percentages) == {0.0}:
-                continue
+            if set(weighted_percentages) == {0.0}:
+                ax.plot([], [], style + marker, label=trend_type)
             else:
-                ax.plot(self.altitudes, percentages, style + marker, label=trend_type)
+                ax.plot(self.altitudes, weighted_percentages, style + marker, label=trend_type)

experiment/meteo_france_SCM_study/visualization/study_visualization/study_visualizer.py

@@ -362,34 +362,46 @@ class StudyVisualizer(object):
             massif_name_to_scores[massif_name] = np.array(detailed_scores)
         return massif_name_to_scores
 
-    def massif_name_to_trend_test_count(self, trend_test_class, starting_year_to_weight):
-        massif_name_to_serie_percentages = {}
+    def massif_name_to_df_trend_type(self, trend_test_class, starting_year_to_weight):
+        """
+        Create a DataFrame with massif as index
+        :param trend_test_class:
+        :param starting_year_to_weight:
+        :return:
+        """
+        massif_name_to_df_trend_type = {}
         for massif_id, massif_name in enumerate(self.study.study_massif_names):
             trend_type_and_weight = []
             years, smooth_maxima = self.smooth_maxima_x_y(massif_id)
             for starting_year, weight in starting_year_to_weight.items():
                 idx = years.index(starting_year)
-                years, smooth_maxima = years[idx:], smooth_maxima[idx:]
-                assert years[0] == starting_year, "{} {}".format(years[0], starting_year)
-                trend_test = trend_test_class(years, smooth_maxima) # type: AbstractTrendTest
+                # assert years[0] == starting_year, "{} {}".format(years[0], starting_year)
+                trend_test = trend_test_class(years[:][idx:], smooth_maxima[:][idx:])  # type: AbstractTrendTest
                 trend_type_and_weight.append((trend_test.test_trend_type, weight))
             df = pd.DataFrame(trend_type_and_weight, columns=['trend type', 'weight'])
-            serie = df.groupby(['trend type']).sum()
-            massif_name_to_serie_percentages[massif_name] = serie * 100
-        return massif_name_to_serie_percentages
+            massif_name_to_df_trend_type[massif_name] = df
+        return massif_name_to_df_trend_type
 
-    def serie_mean_trend_test_count(self, trend_test_class, starting_year_to_weight):
-        massif_name_to_trend_test_count = self.massif_name_to_trend_test_count(trend_test_class, starting_year_to_weight)
-        df = pd.concat(list(massif_name_to_trend_test_count.values()), axis=1, sort=False)
+    def df_trend_test_count(self, trend_test_class, starting_year_to_weight):
+        """
+        Index are the trend type
+        Columns are the massif
+
+        :param starting_year_to_weight:
+        :param trend_test_class:
+        :return:
+        """
+        massif_name_to_df_trend_type = self.massif_name_to_df_trend_type(trend_test_class, starting_year_to_weight)
+        df = pd.concat([100 * v.groupby(['trend type']).sum()
+                        for v in massif_name_to_df_trend_type.values()], axis=1, sort=False)
         df.fillna(0.0, inplace=True)
-        s = df.mean(axis=1)
-        assert np.allclose(df.sum(), 100)
+        assert np.allclose(df.sum(axis=0), 100)
         # Add the significant trend into the count of normal trend
-        if AbstractTrendTest.SIGNIFICATIVE_POSITIVE_TREND in s.index:
-            s[AbstractTrendTest.POSITIVE_TREND] += s[AbstractTrendTest.SIGNIFICATIVE_POSITIVE_TREND]
-        if AbstractTrendTest.SIGNIFICATIVE_NEGATIVE_TREND in s.index:
-            s[AbstractTrendTest.NEGATIVE_TREND] += s[AbstractTrendTest.SIGNIFICATIVE_NEGATIVE_TREND]
-        return s
+        if AbstractTrendTest.SIGNIFICATIVE_POSITIVE_TREND in df.index:
+            df.loc[AbstractTrendTest.POSITIVE_TREND] += df.loc[AbstractTrendTest.SIGNIFICATIVE_POSITIVE_TREND]
+        if AbstractTrendTest.SIGNIFICATIVE_NEGATIVE_TREND in df.index:
+            df.loc[AbstractTrendTest.NEGATIVE_TREND] += df.loc[AbstractTrendTest.SIGNIFICATIVE_NEGATIVE_TREND]
+        return df
 
     @cached_property
     def massif_name_to_scores(self):