From 05bc5a5eadda1a3ca0864b2b641572756265aad9 Mon Sep 17 00:00:00 2001
From: Le Roux Erwan <erwan.le-roux@irstea.fr>
Date: Sun, 9 Jun 2019 11:10:37 +0200
Subject: [PATCH] [HYPERCUBE] refactor hypercube. add the fact that for the
fast mode, massif names are sampled randomly for the fast mode. this enables
to explore the change point test results under different viewpoint.
---
.../abstract_hypercube_visualizer.py | 4 +-
.../main_hypercube_visualization.py | 17 +++---
.../quantity_altitude_visualizer.py | 2 +-
.../study_visualization/study_visualizer.py | 55 +++++++++++++++----
test/test_experiment/test_SCM_study.py | 3 +-
5 files changed, 57 insertions(+), 24 deletions(-)
diff --git a/experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/abstract_hypercube_visualizer.py b/experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/abstract_hypercube_visualizer.py
index 0f7454c1..73e13c6a 100644
--- a/experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/abstract_hypercube_visualizer.py
+++ b/experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/abstract_hypercube_visualizer.py
@@ -25,7 +25,6 @@ class AbstractHypercubeVisualizer(object):
last_starting_year=None):
self.last_starting_year = last_starting_year
self.nb_data_for_fast_mode = 7 if nb_data_reduced_for_speed else None
- self.nb_top_likelihood_values = nb_top_likelihood_values
self.save_to_file = save_to_file
self.trend_test_class = trend_test_class
self.tuple_to_study_visualizer = tuple_to_study_visualizer # type: Dict[Tuple, StudyVisualizer]
@@ -55,8 +54,7 @@ class AbstractHypercubeVisualizer(object):
@cached_property
def df_trends_spatio_temporal(self):
return [study_visualizer.df_trend_spatio_temporal(self.trend_test_class, self.starting_years,
- self.nb_data_for_fast_mode,
- self.nb_top_likelihood_values)
+ self.nb_data_for_fast_mode)
for study_visualizer in self.tuple_to_study_visualizer.values()]
@cached_property
diff --git a/experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/main_hypercube_visualization.py b/experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/main_hypercube_visualization.py
index 4021a101..c577d90e 100644
--- a/experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/main_hypercube_visualization.py
+++ b/experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/main_hypercube_visualization.py
@@ -2,15 +2,16 @@ import time
from collections import OrderedDict
from itertools import product
-from experiment.meteo_france_data.scm_models_data.visualization import \
+from experiment.meteo_france_data.scm_models_data.visualization.hypercube_visualization.altitude_hypercube_visualizer import \
AltitudeHypercubeVisualizer
-from experiment.meteo_france_data.scm_models_data.visualization import \
+from experiment.meteo_france_data.scm_models_data.visualization.hypercube_visualization.altitude_year_hypercube_visualizer import \
Altitude_Hypercube_Year_Visualizer
-from experiment.meteo_france_data.scm_models_data.visualization import \
+from experiment.meteo_france_data.scm_models_data.visualization.hypercube_visualization.quantity_altitude_visualizer import \
QuantityAltitudeHypercubeVisualizer
-from experiment.meteo_france_data.scm_models_data.visualization import ALL_ALTITUDES, \
- SCM_STUDIES, study_iterator, study_iterator_global
-from experiment.meteo_france_data.scm_models_data.visualization import StudyVisualizer
+from experiment.meteo_france_data.scm_models_data.visualization.study_visualization.main_study_visualizer import \
+ ALL_ALTITUDES, SCM_STUDIES, study_iterator, study_iterator_global
+from experiment.meteo_france_data.scm_models_data.visualization.study_visualization.study_visualizer import \
+ StudyVisualizer
from experiment.trend_analysis.univariate_test.abstract_gev_change_point_test import GevLocationChangePointTest, \
GevScaleChangePointTest, GevShapeChangePointTest
from utils import get_display_name_from_object_type
@@ -141,8 +142,8 @@ def fast_quantity_altitude_hypercube():
def main_run():
# fast_altitude_hypercube()
- # fast_altitude_year_hypercube()
- full_altitude_year_hypercube()
+ fast_altitude_year_hypercube()
+ # full_altitude_year_hypercube()
# fast_quantity_altitude_hypercube()
# full_quantity_altitude_hypercube()
diff --git a/experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/quantity_altitude_visualizer.py b/experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/quantity_altitude_visualizer.py
index 399a075c..7cb8e55d 100644
--- a/experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/quantity_altitude_visualizer.py
+++ b/experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/quantity_altitude_visualizer.py
@@ -1,6 +1,6 @@
import pandas as pd
-from experiment.meteo_france_data.scm_models_data.visualization import \
+from experiment.meteo_france_data.scm_models_data.visualization.hypercube_visualization.altitude_hypercube_visualizer import \
AltitudeHypercubeVisualizer
diff --git a/experiment/meteo_france_data/scm_models_data/visualization/study_visualization/study_visualizer.py b/experiment/meteo_france_data/scm_models_data/visualization/study_visualization/study_visualizer.py
index b8d0740b..cc03044e 100644
--- a/experiment/meteo_france_data/scm_models_data/visualization/study_visualization/study_visualizer.py
+++ b/experiment/meteo_france_data/scm_models_data/visualization/study_visualization/study_visualizer.py
@@ -2,6 +2,7 @@ import os
import os.path as op
from collections import OrderedDict
from multiprocessing.pool import Pool
+from random import sample
import math
import matplotlib.pyplot as plt
@@ -112,6 +113,11 @@ class StudyVisualizer(VisualizationParameters):
# INCREASE THE TEMPORAL STEPS FOR VISUALIZATION
AbstractMarginFunction.VISUALIZATION_TEMPORAL_STEPS = 5
+ # Change point parameters
+ self.trend_test_class_for_change_point_test = None
+ self.starting_years_for_change_point_test = None
+ self.nb_massif_for_change_point_test = None
+
@property
def dataset(self):
if self._dataset is None:
@@ -390,8 +396,37 @@ class StudyVisualizer(VisualizationParameters):
massif_name_to_df_trend_type[massif_name] = df
return massif_name_to_df_trend_type
- def df_trend_spatio_temporal(self, trend_test_class, starting_years, nb_massif_for_fast_mode=None,
- nb_top_likelihood_values=1):
+ def massif_name_to_gev_change_point_test_results(self, trend_test_class_for_change_point_test,
+ starting_years_for_change_point_test,
+ nb_massif_for_change_point_test=None):
+ if self.trend_test_class_for_change_point_test is None:
+ # Set the attribute is not already done
+ self.trend_test_class_for_change_point_test = trend_test_class_for_change_point_test
+ self.starting_years_for_change_point_test = starting_years_for_change_point_test
+ self.nb_massif_for_change_point_test = nb_massif_for_change_point_test
+ else:
+ # Check that the argument are the same
+ assert self.trend_test_class_for_change_point_test == trend_test_class_for_change_point_test
+ assert self.starting_years == starting_years_for_change_point_test
+ assert self.nb_massif_for_change_point_test == nb_massif_for_change_point_test
+ return self._massif_name_to_gev_change_point_test_results
+
+ @cached_property
+ def _massif_name_to_gev_change_point_test_results(self):
+ massif_name_to_gev_change_point_test_results = {}
+ massif_names = self.study.study_massif_names
+ if self.nb_massif_for_change_point_test is not None:
+ massif_names = sample(massif_names, self.nb_massif_for_change_point_test)
+ for massif_id, massif_name in enumerate(massif_names):
+ years, smooth_maxima = self.smooth_maxima_x_y(massif_id)
+ gev_change_point_test_results = compute_gev_change_point_test_results(self.multiprocessing, smooth_maxima,
+ self.starting_years_for_change_point_test,
+ self.trend_test_class_for_change_point_test, years)
+ massif_name_to_gev_change_point_test_results[massif_name] = gev_change_point_test_results
+ return massif_name_to_gev_change_point_test_results
+
+ def df_trend_spatio_temporal(self, trend_test_class_for_change_point_test, starting_years_for_change_point_test,
+ nb_massif_for_change_point_test=None):
"""
Index are the massif
Columns are the starting year
@@ -400,14 +435,14 @@ class StudyVisualizer(VisualizationParameters):
:param starting_year_to_weight:
:return:
"""
+ # Set the attributes
+
massif_name_to_trend_res = {}
- massif_names = self.study.study_massif_names
- if nb_massif_for_fast_mode is not None:
- massif_names = massif_names[:nb_massif_for_fast_mode]
- for massif_id, massif_name in enumerate(massif_names):
- years, smooth_maxima = self.smooth_maxima_x_y(massif_id)
- trend_test_res, best_idxs = compute_gev_change_point_test_results(self.multiprocessing, smooth_maxima, starting_years,
- trend_test_class, years)
+ massif_name_to_gev_change_point_test_results = self.massif_name_to_gev_change_point_test_results(trend_test_class_for_change_point_test,
+ starting_years_for_change_point_test,
+ nb_massif_for_change_point_test)
+ for massif_name, gev_change_point_test_results in massif_name_to_gev_change_point_test_results.items():
+ trend_test_res, best_idxs = gev_change_point_test_results
trend_test_res = [(a, b) if i in best_idxs else (np.nan, np.nan)
for i, (a, b, *_) in enumerate(trend_test_res)]
massif_name_to_trend_res[massif_name] = list(zip(*trend_test_res))
@@ -415,7 +450,7 @@ class StudyVisualizer(VisualizationParameters):
assert nb_res == 2
all_massif_name_to_res = [{k: v[idx_res] for k, v in massif_name_to_trend_res.items()}
for idx_res in range(nb_res)]
- return [pd.DataFrame(massif_name_to_res, index=starting_years).transpose()
+ return [pd.DataFrame(massif_name_to_res, index=self.starting_years_for_change_point_test).transpose()
for massif_name_to_res in all_massif_name_to_res]
@staticmethod
diff --git a/test/test_experiment/test_SCM_study.py b/test/test_experiment/test_SCM_study.py
index 0379f390..c039d98c 100644
--- a/test/test_experiment/test_SCM_study.py
+++ b/test/test_experiment/test_SCM_study.py
@@ -22,8 +22,7 @@ class TestSCMAllStudy(unittest.TestCase):
for study_class in [ExtendedSafranSnowfall]:
for study in study_iterator(study_class, only_first_one=True, verbose=False):
study_visualizer = StudyVisualizer(study, show=False, save_to_file=False, multiprocessing=True)
- study_visualizer.df_trend_spatio_temporal(GevLocationChangePointTest, [1958, 1959, 1960],
- nb_massif_for_fast_mode=1)
+ study_visualizer.df_trend_spatio_temporal(GevLocationChangePointTest, [1958, 1959, 1960], nb_massif_for_change_point_test=3)
self.assertTrue(True)
def test_instantiate_studies(self):
--
GitLab