From 979c622d35be074483392a5d3c5294b565bc853c Mon Sep 17 00:00:00 2001 From: Le Roux Erwan <erwan.le-roux@irstea.fr> Date: Wed, 22 May 2019 12:00:08 +0200 Subject: [PATCH] [SCM][HYPERCUBE] Add quantity altitude hypercube visualizer --- .../hypercube_visualizer.py | 64 ++++++++++++++----- .../main_hypercube_visualization.py | 63 ++++++++++++++---- .../abstract_gev_trend_test.py | 6 +- .../abstract_trend_test.py | 9 ++- 4 files changed, 110 insertions(+), 32 deletions(-) diff --git a/experiment/meteo_france_SCM_study/visualization/hypercube_visualization/hypercube_visualizer.py b/experiment/meteo_france_SCM_study/visualization/hypercube_visualization/hypercube_visualizer.py index e5f6a654..9ba5a126 100644 --- a/experiment/meteo_france_SCM_study/visualization/hypercube_visualization/hypercube_visualizer.py +++ b/experiment/meteo_france_SCM_study/visualization/hypercube_visualization/hypercube_visualizer.py @@ -47,6 +47,9 @@ class HypercubeVisualizer(object): for study_visualizer in self.tuple_to_study_visualizer.values()] return dict(zip(self.tuple_to_study_visualizer.keys(), df_spatio_temporal_trend_types)) + def tuple_values(self, idx): + return sorted(set([t[idx] if isinstance(t, tuple) else t for t in self.tuple_to_study_visualizer.keys()])) + @cached_property def df_hypercube(self) -> pd.DataFrame: keys = list(self.tuple_to_df_trend_type.keys()) @@ -88,8 +91,8 @@ class AltitudeHypercubeVisualizer(HypercubeVisualizer): @property def altitudes(self): - return list(self.tuple_to_study_visualizer.keys()) - + return self.tuple_values(idx=0) + @property def trend_type_to_style(self): return self.trend_test_class.trend_type_to_style() @@ -97,7 +100,7 @@ class AltitudeHypercubeVisualizer(HypercubeVisualizer): @property def trend_types(self): return self.trend_type_to_style.keys() - + def trend_type_to_s_percentages(self, reduction_function): # Map each trend type to its serie with percentages trend_type_to_s_percentages = {} @@ -106,22 +109,30 @@ class AltitudeHypercubeVisualizer(HypercubeVisualizer): # Reduce the entire dataframe to a serie s_percentages = reduction_function(df_bool) assert isinstance(s_percentages, pd.Series) + assert not isinstance(s_percentages.index, pd.MultiIndex) s_percentages *= 100 trend_type_to_s_percentages[trend_type] = s_percentages # Post processing - Add the significant trend into the count of normal trend - trend_type_to_s_percentages[AbstractTrendTest.POSITIVE_TREND] += trend_type_to_s_percentages[AbstractTrendTest.SIGNIFICATIVE_POSITIVE_TREND] - trend_type_to_s_percentages[AbstractTrendTest.NEGATIVE_TREND] += trend_type_to_s_percentages[AbstractTrendTest.SIGNIFICATIVE_NEGATIVE_TREND] + trend_type_to_s_percentages[AbstractTrendTest.POSITIVE_TREND] += trend_type_to_s_percentages[ + AbstractTrendTest.SIGNIFICATIVE_POSITIVE_TREND] + trend_type_to_s_percentages[AbstractTrendTest.NEGATIVE_TREND] += trend_type_to_s_percentages[ + AbstractTrendTest.SIGNIFICATIVE_NEGATIVE_TREND] return trend_type_to_s_percentages - + def visualize_trend_test_evolution(self, reduction_function, xlabel, xlabel_values, ax=None, marker='o'): if ax is None: fig, ax = plt.subplots(1, 1, figsize=self.study_visualizer.figsize) - trend_type_to_s_percentages = self.trend_type_to_s_percentages(reduction_function) + trend_type_to_percentages_values = {k: s.values for k, s in self.trend_type_to_s_percentages(reduction_function).items()} for trend_type in self.trend_types: style = self.trend_type_to_style[trend_type] - s_percentages = trend_type_to_s_percentages[trend_type] - ax.plot(xlabel_values, s_percentages.values, style + marker, label=trend_type) + percentages_values = trend_type_to_percentages_values[trend_type] + ax.plot(xlabel_values, percentages_values, style + marker, label=trend_type) + + # Plot the total value of significative values + significative_values = trend_type_to_percentages_values[AbstractTrendTest.SIGNIFICATIVE_NEGATIVE_TREND] \ + + trend_type_to_percentages_values[AbstractTrendTest.SIGNIFICATIVE_POSITIVE_TREND] + ax.plot(xlabel_values, significative_values, 'y-' + marker, label=AbstractTrendTest.SIGNIFICATIVE + ' trends') # Global information added_str = 'weighted ' @@ -165,33 +176,54 @@ class AltitudeHypercubeVisualizer(HypercubeVisualizer): def visualize_year_trend_test(self, ax=None, marker='o'): def year_reduction(df_bool): - # Take the mean with respect the massifs - df_bool = df_bool.mean(axis=0, level=0) - # Take the mean with respect to the altitude + # Take the mean with respect to all the first axis indices return df_bool.mean(axis=0) self.visualize_trend_test_evolution(reduction_function=year_reduction, xlabel='starting years', xlabel_values=self.starting_years, ax=ax, marker=marker) + @property + def altitude_index_level(self): + return 0 + def visualize_altitude_trend_test(self, ax=None, marker='o'): def altitude_reduction(df_bool): # Take the mean with respect to the years df_bool = df_bool.mean(axis=1) # Take the mean with respect the massifs - return df_bool.mean(axis=0, level=0) + return df_bool.mean(level=self.altitude_index_level) self.visualize_trend_test_evolution(reduction_function=altitude_reduction, xlabel='altitude', xlabel_values=self.altitudes, ax=ax, marker=marker) + @property + def massif_index_level(self): + return 1 + def visualize_massif_trend_test(self, axes=None): def massif_reduction(df_bool): # Take the mean with respect to the years df_bool = df_bool.mean(axis=1) # Take the mean with respect the altitude - return df_bool.mean(axis=0, level=1) + return df_bool.mean(level=self.massif_index_level) self.visualize_trend_test_repartition(massif_reduction, axes) -class QuantitityAltitudeHypercubeVisualizer(HypercubeVisualizer): - pass +class QuantityAltitudeHypercubeVisualizer(AltitudeHypercubeVisualizer): + + @property + def quantities(self): + return self.tuple_values(idx=0) + + @property + def altitudes(self): + return self.tuple_values(idx=1) + + @property + def altitude_index_level(self): + return 1 + + @property + def massif_index_level(self): + return 2 diff --git a/experiment/meteo_france_SCM_study/visualization/hypercube_visualization/main_hypercube_visualization.py b/experiment/meteo_france_SCM_study/visualization/hypercube_visualization/main_hypercube_visualization.py index 800229b1..c999d634 100644 --- a/experiment/meteo_france_SCM_study/visualization/hypercube_visualization/main_hypercube_visualization.py +++ b/experiment/meteo_france_SCM_study/visualization/hypercube_visualization/main_hypercube_visualization.py @@ -1,22 +1,23 @@ import time +from itertools import product from collections import OrderedDict from experiment.meteo_france_SCM_study.visualization.hypercube_visualization.hypercube_visualizer import \ - AltitudeHypercubeVisualizer + AltitudeHypercubeVisualizer, QuantityAltitudeHypercubeVisualizer from experiment.meteo_france_SCM_study.visualization.study_visualization.main_study_visualizer import ALL_ALTITUDES, \ - SCM_STUDIES, study_iterator + SCM_STUDIES, study_iterator, study_iterator_global from experiment.meteo_france_SCM_study.visualization.study_visualization.study_visualizer import StudyVisualizer from experiment.trend_analysis.univariate_trend_test.abstract_gev_trend_test import GevLocationTrendTest, \ GevScaleTrendTest, GevShapeTrendTest from experiment.trend_analysis.univariate_trend_test.abstract_trend_test import MannKendallTrendTest +from utils import get_display_name_from_object_type -def altitude_trend_with_hypercube(): +def full_trends_with_altitude_hypercube(): save_to_file = True only_first_one = False fast = False altitudes = ALL_ALTITUDES[3:-6] - # altitudes = ALL_ALTITUDES[2:4] for study_class in SCM_STUDIES[:]: for trend_test_class in [MannKendallTrendTest, GevLocationTrendTest, GevScaleTrendTest, GevShapeTrendTest][:]: visualizers = [StudyVisualizer(study, temporal_non_stationarity=True, verbose=False, multiprocessing=True) @@ -25,14 +26,15 @@ def altitude_trend_with_hypercube(): altitude_to_visualizer = OrderedDict(zip(altitudes, visualizers)) visualizer = AltitudeHypercubeVisualizer(altitude_to_visualizer, save_to_file=save_to_file, trend_test_class=trend_test_class, fast=fast) + visualizer.visualize_massif_trend_test() + visualizer.visualize_year_trend_test() visualizer.visualize_altitude_trend_test() -def spatial_trend_with_hypercube(): +def fast_trends_with_altitude_hypercube(): save_to_file = False only_first_one = False fast = True - # altitudes = ALL_ALTITUDES[3:-6] altitudes = ALL_ALTITUDES[2:4] for study_class in SCM_STUDIES[:1]: for trend_test_class in [MannKendallTrendTest, GevLocationTrendTest, GevScaleTrendTest, GevShapeTrendTest][:1]: @@ -42,16 +44,55 @@ def spatial_trend_with_hypercube(): altitude_to_visualizer = OrderedDict(zip(altitudes, visualizers)) visualizer = AltitudeHypercubeVisualizer(altitude_to_visualizer, save_to_file=save_to_file, trend_test_class=trend_test_class, fast=fast) - visualizer.visualize_massif_trend_test() visualizer.visualize_year_trend_test() - visualizer.visualize_altitude_trend_test() + # visualizer.visualize_massif_trend_test() + # visualizer.visualize_altitude_trend_test() + + +def full_trends_with_quantity_altitude_hypercube(): + save_to_file = True + only_first_one = False + fast = False + altitudes = ALL_ALTITUDES[3:-6] + study_classes = SCM_STUDIES + for trend_test_class in [MannKendallTrendTest, GevLocationTrendTest, GevScaleTrendTest, GevShapeTrendTest][:1]: + visualizers = [StudyVisualizer(study, temporal_non_stationarity=True, verbose=False, multiprocessing=True) + for study in study_iterator_global(study_classes=study_classes, only_first_one=only_first_one, + altitudes=altitudes)] + study_classes_str = [get_display_name_from_object_type(c) for c in study_classes] + quantity_altitude_tuples = list(product(study_classes_str, altitudes)) + quantity_altitude_to_visualizer = OrderedDict(zip(quantity_altitude_tuples, visualizers)) + visualizer = QuantityAltitudeHypercubeVisualizer(quantity_altitude_to_visualizer, save_to_file=save_to_file, + trend_test_class=trend_test_class, fast=fast) + visualizer.visualize_year_trend_test() + # visualizer.visualize_massif_trend_test() + # visualizer.visualize_altitude_trend_test() +def fast_trends_with_quantity_altitude_hypercube(): + save_to_file = False + only_first_one = False + fast = True + altitudes = ALL_ALTITUDES[2:4] + study_classes = SCM_STUDIES[:2] + for trend_test_class in [MannKendallTrendTest, GevLocationTrendTest, GevScaleTrendTest, GevShapeTrendTest][:1]: + visualizers = [StudyVisualizer(study, temporal_non_stationarity=True, verbose=False, multiprocessing=True) + for study in study_iterator_global(study_classes=study_classes, only_first_one=only_first_one, + altitudes=altitudes)] + study_classes_str = [get_display_name_from_object_type(c) for c in study_classes] + quantity_altitude_tuples = list(product(study_classes_str, altitudes)) + quantity_altitude_to_visualizer = OrderedDict(zip(quantity_altitude_tuples, visualizers)) + visualizer = QuantityAltitudeHypercubeVisualizer(quantity_altitude_to_visualizer, save_to_file=save_to_file, + trend_test_class=trend_test_class, fast=fast) + # visualizer.visualize_year_trend_test() + # visualizer.visualize_massif_trend_test() + visualizer.visualize_altitude_trend_test() def main_run(): - # altitude_trends() - # altitude_trends_significant() - spatial_trend_with_hypercube() + # fast_trends_with_altitude_hypercube() + # full_trends_with_altitude_hypercube() + # fast_trends_with_quantity_altitude_hypercube() + full_trends_with_quantity_altitude_hypercube() if __name__ == '__main__': diff --git a/experiment/trend_analysis/univariate_trend_test/abstract_gev_trend_test.py b/experiment/trend_analysis/univariate_trend_test/abstract_gev_trend_test.py index 97caaa58..3c35595b 100644 --- a/experiment/trend_analysis/univariate_trend_test/abstract_gev_trend_test.py +++ b/experiment/trend_analysis/univariate_trend_test/abstract_gev_trend_test.py @@ -1,7 +1,7 @@ +import numpy as np import pandas as pd from scipy.stats import chi2 -import numpy as np -from sklearn.preprocessing import normalize + from experiment.trend_analysis.univariate_trend_test.abstract_trend_test import AbstractTrendTest from extreme_estimator.estimator.margin_estimator.abstract_margin_estimator import LinearMarginEstimator from extreme_estimator.extreme_models.margin_model.temporal_linear_margin_model import StationaryStationModel, \ @@ -13,8 +13,6 @@ from spatio_temporal_dataset.coordinates.temporal_coordinates.abstract_temporal_ AbstractTemporalCoordinates from spatio_temporal_dataset.coordinates.transformed_coordinates.transformation.abstract_transformation import \ CenteredScaledNormalization -from spatio_temporal_dataset.coordinates.transformed_coordinates.transformation.uniform_normalization import \ - BetweenZeroAndOneNormalization from spatio_temporal_dataset.dataset.abstract_dataset import AbstractDataset from spatio_temporal_dataset.spatio_temporal_observations.abstract_spatio_temporal_observations import \ AbstractSpatioTemporalObservations diff --git a/experiment/trend_analysis/univariate_trend_test/abstract_trend_test.py b/experiment/trend_analysis/univariate_trend_test/abstract_trend_test.py index eec14042..a20465bb 100644 --- a/experiment/trend_analysis/univariate_trend_test/abstract_trend_test.py +++ b/experiment/trend_analysis/univariate_trend_test/abstract_trend_test.py @@ -1,4 +1,6 @@ import random +import warnings + import matplotlib.pyplot as plt from collections import OrderedDict @@ -80,6 +82,9 @@ class ExampleRandomTrendTest(AbstractTrendTest): return random.randint(1, 10) == 10 +class WarningScoreValue(Warning): + pass + class MannKendallTrendTest(AbstractTrendTest): def __init__(self, years_after_change_point, maxima_after_change_point): @@ -94,7 +99,9 @@ class MannKendallTrendTest(AbstractTrendTest): eps=1e-5, alpha=self.SIGNIFICANCE_LEVEL, Ha='upordown') - assert S == self.score_value + # Raise warning if scores are differents + if S != self.score_value: + warnings.warn('S={} is different that score_value={}'.format(S, self.score_value), WarningScoreValue) self.MK = MK @property -- GitLab