From 979c622d35be074483392a5d3c5294b565bc853c Mon Sep 17 00:00:00 2001
From: Le Roux Erwan <erwan.le-roux@irstea.fr>
Date: Wed, 22 May 2019 12:00:08 +0200
Subject: [PATCH] [SCM][HYPERCUBE] Add quantity altitude hypercube visualizer
---
.../hypercube_visualizer.py | 64 ++++++++++++++-----
.../main_hypercube_visualization.py | 63 ++++++++++++++----
.../abstract_gev_trend_test.py | 6 +-
.../abstract_trend_test.py | 9 ++-
4 files changed, 110 insertions(+), 32 deletions(-)
diff --git a/experiment/meteo_france_SCM_study/visualization/hypercube_visualization/hypercube_visualizer.py b/experiment/meteo_france_SCM_study/visualization/hypercube_visualization/hypercube_visualizer.py
index e5f6a654..9ba5a126 100644
--- a/experiment/meteo_france_SCM_study/visualization/hypercube_visualization/hypercube_visualizer.py
+++ b/experiment/meteo_france_SCM_study/visualization/hypercube_visualization/hypercube_visualizer.py
@@ -47,6 +47,9 @@ class HypercubeVisualizer(object):
for study_visualizer in self.tuple_to_study_visualizer.values()]
return dict(zip(self.tuple_to_study_visualizer.keys(), df_spatio_temporal_trend_types))
+ def tuple_values(self, idx):
+ return sorted(set([t[idx] if isinstance(t, tuple) else t for t in self.tuple_to_study_visualizer.keys()]))
+
@cached_property
def df_hypercube(self) -> pd.DataFrame:
keys = list(self.tuple_to_df_trend_type.keys())
@@ -88,8 +91,8 @@ class AltitudeHypercubeVisualizer(HypercubeVisualizer):
@property
def altitudes(self):
- return list(self.tuple_to_study_visualizer.keys())
-
+ return self.tuple_values(idx=0)
+
@property
def trend_type_to_style(self):
return self.trend_test_class.trend_type_to_style()
@@ -97,7 +100,7 @@ class AltitudeHypercubeVisualizer(HypercubeVisualizer):
@property
def trend_types(self):
return self.trend_type_to_style.keys()
-
+
def trend_type_to_s_percentages(self, reduction_function):
# Map each trend type to its serie with percentages
trend_type_to_s_percentages = {}
@@ -106,22 +109,30 @@ class AltitudeHypercubeVisualizer(HypercubeVisualizer):
# Reduce the entire dataframe to a serie
s_percentages = reduction_function(df_bool)
assert isinstance(s_percentages, pd.Series)
+ assert not isinstance(s_percentages.index, pd.MultiIndex)
s_percentages *= 100
trend_type_to_s_percentages[trend_type] = s_percentages
# Post processing - Add the significant trend into the count of normal trend
- trend_type_to_s_percentages[AbstractTrendTest.POSITIVE_TREND] += trend_type_to_s_percentages[AbstractTrendTest.SIGNIFICATIVE_POSITIVE_TREND]
- trend_type_to_s_percentages[AbstractTrendTest.NEGATIVE_TREND] += trend_type_to_s_percentages[AbstractTrendTest.SIGNIFICATIVE_NEGATIVE_TREND]
+ trend_type_to_s_percentages[AbstractTrendTest.POSITIVE_TREND] += trend_type_to_s_percentages[
+ AbstractTrendTest.SIGNIFICATIVE_POSITIVE_TREND]
+ trend_type_to_s_percentages[AbstractTrendTest.NEGATIVE_TREND] += trend_type_to_s_percentages[
+ AbstractTrendTest.SIGNIFICATIVE_NEGATIVE_TREND]
return trend_type_to_s_percentages
-
+
def visualize_trend_test_evolution(self, reduction_function, xlabel, xlabel_values, ax=None, marker='o'):
if ax is None:
fig, ax = plt.subplots(1, 1, figsize=self.study_visualizer.figsize)
- trend_type_to_s_percentages = self.trend_type_to_s_percentages(reduction_function)
+ trend_type_to_percentages_values = {k: s.values for k, s in self.trend_type_to_s_percentages(reduction_function).items()}
for trend_type in self.trend_types:
style = self.trend_type_to_style[trend_type]
- s_percentages = trend_type_to_s_percentages[trend_type]
- ax.plot(xlabel_values, s_percentages.values, style + marker, label=trend_type)
+ percentages_values = trend_type_to_percentages_values[trend_type]
+ ax.plot(xlabel_values, percentages_values, style + marker, label=trend_type)
+
+ # Plot the total value of significative values
+ significative_values = trend_type_to_percentages_values[AbstractTrendTest.SIGNIFICATIVE_NEGATIVE_TREND] \
+ + trend_type_to_percentages_values[AbstractTrendTest.SIGNIFICATIVE_POSITIVE_TREND]
+ ax.plot(xlabel_values, significative_values, 'y-' + marker, label=AbstractTrendTest.SIGNIFICATIVE + ' trends')
# Global information
added_str = 'weighted '
@@ -165,33 +176,54 @@ class AltitudeHypercubeVisualizer(HypercubeVisualizer):
def visualize_year_trend_test(self, ax=None, marker='o'):
def year_reduction(df_bool):
- # Take the mean with respect the massifs
- df_bool = df_bool.mean(axis=0, level=0)
- # Take the mean with respect to the altitude
+ # Take the mean with respect to all the first axis indices
return df_bool.mean(axis=0)
self.visualize_trend_test_evolution(reduction_function=year_reduction, xlabel='starting years',
xlabel_values=self.starting_years, ax=ax, marker=marker)
+ @property
+ def altitude_index_level(self):
+ return 0
+
def visualize_altitude_trend_test(self, ax=None, marker='o'):
def altitude_reduction(df_bool):
# Take the mean with respect to the years
df_bool = df_bool.mean(axis=1)
# Take the mean with respect the massifs
- return df_bool.mean(axis=0, level=0)
+ return df_bool.mean(level=self.altitude_index_level)
self.visualize_trend_test_evolution(reduction_function=altitude_reduction, xlabel='altitude',
xlabel_values=self.altitudes, ax=ax, marker=marker)
+ @property
+ def massif_index_level(self):
+ return 1
+
def visualize_massif_trend_test(self, axes=None):
def massif_reduction(df_bool):
# Take the mean with respect to the years
df_bool = df_bool.mean(axis=1)
# Take the mean with respect the altitude
- return df_bool.mean(axis=0, level=1)
+ return df_bool.mean(level=self.massif_index_level)
self.visualize_trend_test_repartition(massif_reduction, axes)
-class QuantitityAltitudeHypercubeVisualizer(HypercubeVisualizer):
- pass
+class QuantityAltitudeHypercubeVisualizer(AltitudeHypercubeVisualizer):
+
+ @property
+ def quantities(self):
+ return self.tuple_values(idx=0)
+
+ @property
+ def altitudes(self):
+ return self.tuple_values(idx=1)
+
+ @property
+ def altitude_index_level(self):
+ return 1
+
+ @property
+ def massif_index_level(self):
+ return 2
diff --git a/experiment/meteo_france_SCM_study/visualization/hypercube_visualization/main_hypercube_visualization.py b/experiment/meteo_france_SCM_study/visualization/hypercube_visualization/main_hypercube_visualization.py
index 800229b1..c999d634 100644
--- a/experiment/meteo_france_SCM_study/visualization/hypercube_visualization/main_hypercube_visualization.py
+++ b/experiment/meteo_france_SCM_study/visualization/hypercube_visualization/main_hypercube_visualization.py
@@ -1,22 +1,23 @@
import time
+from itertools import product
from collections import OrderedDict
from experiment.meteo_france_SCM_study.visualization.hypercube_visualization.hypercube_visualizer import \
- AltitudeHypercubeVisualizer
+ AltitudeHypercubeVisualizer, QuantityAltitudeHypercubeVisualizer
from experiment.meteo_france_SCM_study.visualization.study_visualization.main_study_visualizer import ALL_ALTITUDES, \
- SCM_STUDIES, study_iterator
+ SCM_STUDIES, study_iterator, study_iterator_global
from experiment.meteo_france_SCM_study.visualization.study_visualization.study_visualizer import StudyVisualizer
from experiment.trend_analysis.univariate_trend_test.abstract_gev_trend_test import GevLocationTrendTest, \
GevScaleTrendTest, GevShapeTrendTest
from experiment.trend_analysis.univariate_trend_test.abstract_trend_test import MannKendallTrendTest
+from utils import get_display_name_from_object_type
-def altitude_trend_with_hypercube():
+def full_trends_with_altitude_hypercube():
save_to_file = True
only_first_one = False
fast = False
altitudes = ALL_ALTITUDES[3:-6]
- # altitudes = ALL_ALTITUDES[2:4]
for study_class in SCM_STUDIES[:]:
for trend_test_class in [MannKendallTrendTest, GevLocationTrendTest, GevScaleTrendTest, GevShapeTrendTest][:]:
visualizers = [StudyVisualizer(study, temporal_non_stationarity=True, verbose=False, multiprocessing=True)
@@ -25,14 +26,15 @@ def altitude_trend_with_hypercube():
altitude_to_visualizer = OrderedDict(zip(altitudes, visualizers))
visualizer = AltitudeHypercubeVisualizer(altitude_to_visualizer, save_to_file=save_to_file,
trend_test_class=trend_test_class, fast=fast)
+ visualizer.visualize_massif_trend_test()
+ visualizer.visualize_year_trend_test()
visualizer.visualize_altitude_trend_test()
-def spatial_trend_with_hypercube():
+def fast_trends_with_altitude_hypercube():
save_to_file = False
only_first_one = False
fast = True
- # altitudes = ALL_ALTITUDES[3:-6]
altitudes = ALL_ALTITUDES[2:4]
for study_class in SCM_STUDIES[:1]:
for trend_test_class in [MannKendallTrendTest, GevLocationTrendTest, GevScaleTrendTest, GevShapeTrendTest][:1]:
@@ -42,16 +44,55 @@ def spatial_trend_with_hypercube():
altitude_to_visualizer = OrderedDict(zip(altitudes, visualizers))
visualizer = AltitudeHypercubeVisualizer(altitude_to_visualizer, save_to_file=save_to_file,
trend_test_class=trend_test_class, fast=fast)
- visualizer.visualize_massif_trend_test()
visualizer.visualize_year_trend_test()
- visualizer.visualize_altitude_trend_test()
+ # visualizer.visualize_massif_trend_test()
+ # visualizer.visualize_altitude_trend_test()
+
+
+def full_trends_with_quantity_altitude_hypercube():
+ save_to_file = True
+ only_first_one = False
+ fast = False
+ altitudes = ALL_ALTITUDES[3:-6]
+ study_classes = SCM_STUDIES
+ for trend_test_class in [MannKendallTrendTest, GevLocationTrendTest, GevScaleTrendTest, GevShapeTrendTest][:1]:
+ visualizers = [StudyVisualizer(study, temporal_non_stationarity=True, verbose=False, multiprocessing=True)
+ for study in study_iterator_global(study_classes=study_classes, only_first_one=only_first_one,
+ altitudes=altitudes)]
+ study_classes_str = [get_display_name_from_object_type(c) for c in study_classes]
+ quantity_altitude_tuples = list(product(study_classes_str, altitudes))
+ quantity_altitude_to_visualizer = OrderedDict(zip(quantity_altitude_tuples, visualizers))
+ visualizer = QuantityAltitudeHypercubeVisualizer(quantity_altitude_to_visualizer, save_to_file=save_to_file,
+ trend_test_class=trend_test_class, fast=fast)
+ visualizer.visualize_year_trend_test()
+ # visualizer.visualize_massif_trend_test()
+ # visualizer.visualize_altitude_trend_test()
+def fast_trends_with_quantity_altitude_hypercube():
+ save_to_file = False
+ only_first_one = False
+ fast = True
+ altitudes = ALL_ALTITUDES[2:4]
+ study_classes = SCM_STUDIES[:2]
+ for trend_test_class in [MannKendallTrendTest, GevLocationTrendTest, GevScaleTrendTest, GevShapeTrendTest][:1]:
+ visualizers = [StudyVisualizer(study, temporal_non_stationarity=True, verbose=False, multiprocessing=True)
+ for study in study_iterator_global(study_classes=study_classes, only_first_one=only_first_one,
+ altitudes=altitudes)]
+ study_classes_str = [get_display_name_from_object_type(c) for c in study_classes]
+ quantity_altitude_tuples = list(product(study_classes_str, altitudes))
+ quantity_altitude_to_visualizer = OrderedDict(zip(quantity_altitude_tuples, visualizers))
+ visualizer = QuantityAltitudeHypercubeVisualizer(quantity_altitude_to_visualizer, save_to_file=save_to_file,
+ trend_test_class=trend_test_class, fast=fast)
+ # visualizer.visualize_year_trend_test()
+ # visualizer.visualize_massif_trend_test()
+ visualizer.visualize_altitude_trend_test()
def main_run():
- # altitude_trends()
- # altitude_trends_significant()
- spatial_trend_with_hypercube()
+ # fast_trends_with_altitude_hypercube()
+ # full_trends_with_altitude_hypercube()
+ # fast_trends_with_quantity_altitude_hypercube()
+ full_trends_with_quantity_altitude_hypercube()
if __name__ == '__main__':
diff --git a/experiment/trend_analysis/univariate_trend_test/abstract_gev_trend_test.py b/experiment/trend_analysis/univariate_trend_test/abstract_gev_trend_test.py
index 97caaa58..3c35595b 100644
--- a/experiment/trend_analysis/univariate_trend_test/abstract_gev_trend_test.py
+++ b/experiment/trend_analysis/univariate_trend_test/abstract_gev_trend_test.py
@@ -1,7 +1,7 @@
+import numpy as np
import pandas as pd
from scipy.stats import chi2
-import numpy as np
-from sklearn.preprocessing import normalize
+
from experiment.trend_analysis.univariate_trend_test.abstract_trend_test import AbstractTrendTest
from extreme_estimator.estimator.margin_estimator.abstract_margin_estimator import LinearMarginEstimator
from extreme_estimator.extreme_models.margin_model.temporal_linear_margin_model import StationaryStationModel, \
@@ -13,8 +13,6 @@ from spatio_temporal_dataset.coordinates.temporal_coordinates.abstract_temporal_
AbstractTemporalCoordinates
from spatio_temporal_dataset.coordinates.transformed_coordinates.transformation.abstract_transformation import \
CenteredScaledNormalization
-from spatio_temporal_dataset.coordinates.transformed_coordinates.transformation.uniform_normalization import \
- BetweenZeroAndOneNormalization
from spatio_temporal_dataset.dataset.abstract_dataset import AbstractDataset
from spatio_temporal_dataset.spatio_temporal_observations.abstract_spatio_temporal_observations import \
AbstractSpatioTemporalObservations
diff --git a/experiment/trend_analysis/univariate_trend_test/abstract_trend_test.py b/experiment/trend_analysis/univariate_trend_test/abstract_trend_test.py
index eec14042..a20465bb 100644
--- a/experiment/trend_analysis/univariate_trend_test/abstract_trend_test.py
+++ b/experiment/trend_analysis/univariate_trend_test/abstract_trend_test.py
@@ -1,4 +1,6 @@
import random
+import warnings
+
import matplotlib.pyplot as plt
from collections import OrderedDict
@@ -80,6 +82,9 @@ class ExampleRandomTrendTest(AbstractTrendTest):
return random.randint(1, 10) == 10
+class WarningScoreValue(Warning):
+ pass
+
class MannKendallTrendTest(AbstractTrendTest):
def __init__(self, years_after_change_point, maxima_after_change_point):
@@ -94,7 +99,9 @@ class MannKendallTrendTest(AbstractTrendTest):
eps=1e-5,
alpha=self.SIGNIFICANCE_LEVEL,
Ha='upordown')
- assert S == self.score_value
+ # Raise warning if scores are differents
+ if S != self.score_value:
+ warnings.warn('S={} is different that score_value={}'.format(S, self.score_value), WarningScoreValue)
self.MK = MK
@property
--
GitLab