From 8ba3037d95029e0b33fad0a93abae863508626bb Mon Sep 17 00:00:00 2001
From: Le Roux Erwan <erwan.le-roux@irstea.fr>
Date: Tue, 28 May 2019 10:30:48 +0200
Subject: [PATCH] [SCM] add 3 consecutive nb_days as default argument. add this
argument for SWE and SD. Also add a test to SCM study
---
.../study/abstract_study.py | 5 ++-
.../study/abstract_variable.py | 6 ++--
.../study/crocus/crocus.py | 5 +--
.../study/crocus/crocus_variables.py | 8 ++---
.../study/cumulated_study.py | 13 ++++----
.../study/safran/safran_variable.py | 24 +++++++-------
.../main_study_visualizer.py | 26 +++++++--------
test/test_experiment/test_SCM_study.py | 32 +++++++++++++++----
8 files changed, 71 insertions(+), 48 deletions(-)
diff --git a/experiment/meteo_france_SCM_models/study/abstract_study.py b/experiment/meteo_france_SCM_models/study/abstract_study.py
index 2be3827e..13aca7a3 100644
--- a/experiment/meteo_france_SCM_models/study/abstract_study.py
+++ b/experiment/meteo_france_SCM_models/study/abstract_study.py
@@ -108,12 +108,15 @@ class AbstractStudy(object):
def load_variables(self, path_file):
dataset = Dataset(path_file)
- keyword = self.variable_class.keyword()
+ keyword = self.load_keyword()
if isinstance(keyword, str):
return np.array(dataset.variables[keyword])
else:
return [np.array(dataset.variables[k]) for k in keyword]
+ def load_keyword(self):
+ return self.variable_class.keyword()
+
@property
def start_year_and_stop_year(self) -> Tuple[int, int]:
ordered_years = self.ordered_years
diff --git a/experiment/meteo_france_SCM_models/study/abstract_variable.py b/experiment/meteo_france_SCM_models/study/abstract_variable.py
index 9e5a07c4..62e9457c 100644
--- a/experiment/meteo_france_SCM_models/study/abstract_variable.py
+++ b/experiment/meteo_france_SCM_models/study/abstract_variable.py
@@ -9,12 +9,12 @@ class AbstractVariable(object):
NAME = ''
UNIT = ''
- def __init__(self, variable_array, nb_consecutive_days_of_snowfall=1):
+ def __init__(self, variable_array, nb_consecutive_days=3):
self.variable_array = variable_array
- self.nb_consecutive_days_of_snowfall = nb_consecutive_days_of_snowfall
+ self.nb_consecutive_days = nb_consecutive_days
@classmethod
- def keyword(cls):
+ def keyword(cls, nb_consecutive_days=3):
raise NotImplementedError
@property
diff --git a/experiment/meteo_france_SCM_models/study/crocus/crocus.py b/experiment/meteo_france_SCM_models/study/crocus/crocus.py
index fc3b39f4..e5671176 100644
--- a/experiment/meteo_france_SCM_models/study/crocus/crocus.py
+++ b/experiment/meteo_france_SCM_models/study/crocus/crocus.py
@@ -40,10 +40,11 @@ class ExtendedCrocusSwe(AbstractExtendedStudy, CrocusSwe):
pass
-class CrocusDepth(Crocus):
+class CrocusDepth(Crocus, CumulatedStudy):
def __init__(self, *args, **kwargs):
- super().__init__(CrocusDepthVariable, *args, **kwargs)
+ CumulatedStudy.__init__(self, CrocusDepthVariable, *args, **kwargs)
+ Crocus.__init__(self, CrocusDepthVariable, *args, **kwargs)
def apply_annual_aggregation(self, time_serie):
return self.winter_annual_aggregation(time_serie)
diff --git a/experiment/meteo_france_SCM_models/study/crocus/crocus_variables.py b/experiment/meteo_france_SCM_models/study/crocus/crocus_variables.py
index 874517a3..4663bf4f 100644
--- a/experiment/meteo_france_SCM_models/study/crocus/crocus_variables.py
+++ b/experiment/meteo_france_SCM_models/study/crocus/crocus_variables.py
@@ -15,8 +15,8 @@ class CrocusSweVariable(CrocusVariable):
UNIT = 'kg per m2 or mm'
@classmethod
- def keyword(cls):
- return 'SWE_1DY_ISBA'
+ def keyword(cls, nb_consecutive_days=3):
+ return 'SWE_{}DY_ISBA'.format(nb_consecutive_days)
class CrocusDepthVariable(CrocusVariable):
@@ -24,5 +24,5 @@ class CrocusDepthVariable(CrocusVariable):
UNIT = 'm'
@classmethod
- def keyword(cls):
- return "SD_1DY_ISBA"
+ def keyword(cls, nb_consecutive_days=3):
+ return "SD_{}DY_ISBA".format(nb_consecutive_days)
diff --git a/experiment/meteo_france_SCM_models/study/cumulated_study.py b/experiment/meteo_france_SCM_models/study/cumulated_study.py
index e5300ff4..8dd806ca 100644
--- a/experiment/meteo_france_SCM_models/study/cumulated_study.py
+++ b/experiment/meteo_france_SCM_models/study/cumulated_study.py
@@ -1,20 +1,21 @@
-import numpy as np
-
from experiment.meteo_france_SCM_models.study.abstract_study import AbstractStudy
from experiment.meteo_france_SCM_models.study.abstract_variable import AbstractVariable
+NB_DAYS = [1, 3, 5, 7]
+
class CumulatedStudy(AbstractStudy):
- def __init__(self, variable_class: type, nb_consecutive_days: int = 1, *args, **kwargs):
- assert nb_consecutive_days in [1, 3, 5, 7]
+ def __init__(self, variable_class: type, nb_consecutive_days: int = 3, *args, **kwargs):
+ assert nb_consecutive_days in NB_DAYS
super().__init__(variable_class, *args, **kwargs)
self.nb_consecutive_days = nb_consecutive_days
def instantiate_variable_object(self, variable_array) -> AbstractVariable:
return self.variable_class(variable_array, self.nb_consecutive_days)
+ def load_keyword(self):
+ return self.variable_class.keyword(self.nb_consecutive_days)
+
@property
def variable_name(self):
return super().variable_name + ' cumulated over {} day(s)'.format(self.nb_consecutive_days)
-
-
diff --git a/experiment/meteo_france_SCM_models/study/safran/safran_variable.py b/experiment/meteo_france_SCM_models/study/safran/safran_variable.py
index d2c8faa1..b40b6cd5 100644
--- a/experiment/meteo_france_SCM_models/study/safran/safran_variable.py
+++ b/experiment/meteo_france_SCM_models/study/safran/safran_variable.py
@@ -25,12 +25,12 @@ class SafranSnowfallVariable(AbstractVariable):
UNIT = 'kg per m2 or mm'
@classmethod
- def keyword(cls):
+ def keyword(cls, nb_consecutive_days=3):
return 'Snowf'
- def __init__(self, variable_array, nb_consecutive_days_of_snowfall=1):
+ def __init__(self, variable_array, nb_consecutive_days):
super().__init__(variable_array)
- self.nb_consecutive_days_of_snowfall = nb_consecutive_days_of_snowfall
+ self.nb_consecutive_days_of_snowfall = nb_consecutive_days
# Compute the daily snowfall in kg/m2
snowfall_rates = variable_array
@@ -56,10 +56,12 @@ class SafranSnowfallVariable(AbstractVariable):
# The zip is done with respect to the shortest list
snowfall_in_consecutive_days = np.array([sum(e) for e in zip(*shifted_list)])
# The returned array is of size n-nb_days+1 x nb_massif
-
# so that the length of the vector match a year, we can add zeros (since it corresponds to the July month,
# we are sure that there is no snowfall at this period) However such trick does not work for other variable such as Temperature
-
+ nb_days_in_a_year = len(self.daily_snowfall)
+ nb_days_in_vector, nb_altitudes = snowfall_in_consecutive_days.shape
+ zeros_to_add = np.zeros([nb_days_in_a_year - nb_days_in_vector, nb_altitudes])
+ snowfall_in_consecutive_days = np.concatenate([snowfall_in_consecutive_days, zeros_to_add])
return snowfall_in_consecutive_days
@@ -68,19 +70,19 @@ class SafranRainfallVariable(SafranSnowfallVariable):
UNIT = 'kg per m2 or mm'
@classmethod
- def keyword(cls):
+ def keyword(cls, nb_consecutive_days=3):
return 'Rainf'
class SafranTotalPrecipVariable(AbstractVariable):
- def __init__(self, snow_variable_array, rain_variable_array, nb_consecutive_days_of_snowfall=1):
+ def __init__(self, snow_variable_array, rain_variable_array, nb_consecutive_days):
super().__init__(None)
- self.snow_precipitation = SafranSnowfallVariable(snow_variable_array, nb_consecutive_days_of_snowfall)
- self.rain_precipitation = SafranRainfallVariable(rain_variable_array, nb_consecutive_days_of_snowfall)
+ self.snow_precipitation = SafranSnowfallVariable(snow_variable_array, nb_consecutive_days)
+ self.rain_precipitation = SafranRainfallVariable(rain_variable_array, nb_consecutive_days)
@classmethod
- def keyword(cls):
+ def keyword(cls, nb_consecutive_days=3):
return [SafranSnowfallVariable.keyword(), SafranRainfallVariable.keyword()]
@property
@@ -93,7 +95,7 @@ class SafranTemperatureVariable(AbstractVariable):
UNIT = 'Celsius Degrees'
@classmethod
- def keyword(cls):
+ def keyword(cls, nb_consecutive_days=3):
return 'Tair'
def __init__(self, variable_array):
diff --git a/experiment/meteo_france_SCM_models/visualization/study_visualization/main_study_visualizer.py b/experiment/meteo_france_SCM_models/visualization/study_visualization/main_study_visualizer.py
index b09cd17a..893fb39c 100644
--- a/experiment/meteo_france_SCM_models/visualization/study_visualization/main_study_visualizer.py
+++ b/experiment/meteo_france_SCM_models/visualization/study_visualization/main_study_visualizer.py
@@ -36,28 +36,26 @@ List[AbstractStudy]:
break
-def study_iterator(study_class, only_first_one=False, verbose=True, altitudes=None, nb_days=None) -> List[AbstractStudy]:
+def study_iterator(study_class, only_first_one=False, verbose=True, altitudes=None, nb_consecutive_days=None) -> List[AbstractStudy]:
# Default argument
- nb_days = [1] if nb_days is None else nb_days
+ nb_consecutive_days = 1 if nb_consecutive_days is None else nb_consecutive_days
altis = [1800] if altitudes is None else altitudes
if verbose:
print('\n\n\n\n\nLoading studies....')
- for nb_day in nb_days:
- for alti in altis:
- if verbose:
- print('alti: {}, nb_day: {} '.format(alti, nb_day), end='')
+ for alti in altis:
+ if verbose:
+ print('alti: {}, nb_day: {} '.format(alti, nb_consecutive_days), end='')
- study = study_class(altitude=alti)
+ study = study_class(altitude=alti,
+ nb_consecutive_days=nb_consecutive_days)
- if verbose:
- massifs = study.altitude_to_massif_names[alti]
- print('{} massifs: {} \n'.format(len(massifs), massifs))
- yield study
+ if verbose:
+ massifs = study.altitude_to_massif_names[alti]
+ print('{} massifs: {} \n'.format(len(massifs), massifs))
+ yield study
- # Stop iterations on purpose
- if only_first_one:
- break
+ # Stop iterations on purpose
if only_first_one:
break
diff --git a/test/test_experiment/test_SCM_study.py b/test/test_experiment/test_SCM_study.py
index a36b3c35..de6dd512 100644
--- a/test/test_experiment/test_SCM_study.py
+++ b/test/test_experiment/test_SCM_study.py
@@ -1,25 +1,43 @@
import os.path as op
import unittest
+from random import sample
import pandas as pd
-from experiment.meteo_france_SCM_models.visualization.study_visualization.main_study_visualizer import study_iterator
-from experiment.meteo_france_SCM_models.study.safran.safran import SafranSnowfall, ExtendedSafranSnowfall, SafranTemperature, \
+from experiment.meteo_france_SCM_models.study.cumulated_study import NB_DAYS
+from experiment.meteo_france_SCM_models.study.safran.safran import SafranSnowfall, ExtendedSafranSnowfall, \
+ SafranTemperature, \
SafranTotalPrecip
+from experiment.meteo_france_SCM_models.visualization.study_visualization.main_study_visualizer import study_iterator, \
+ study_iterator_global, SCM_STUDIES, ALL_ALTITUDES
from experiment.meteo_france_SCM_models.visualization.study_visualization.study_visualizer import StudyVisualizer
+from experiment.trend_analysis.univariate_test.abstract_gev_change_point_test import GevLocationChangePointTest
+from utils import get_display_name_from_object_type
-# TESTS TO REACTIVATE SOMETIMES
-
class TestSCMAllStudy(unittest.TestCase):
def test_extended_run(self):
for study_class in [ExtendedSafranSnowfall]:
for study in study_iterator(study_class, only_first_one=True, verbose=False):
- study_visualizer = StudyVisualizer(study, show=False, save_to_file=False)
- study_visualizer.visualize_all_mean_and_max_graphs()
+ study_visualizer = StudyVisualizer(study, show=False, save_to_file=False, multiprocessing=True)
+ study_visualizer.df_trend_spatio_temporal(GevLocationChangePointTest, [1958, 1959, 1960],
+ nb_massif_for_fast_mode=1)
self.assertTrue(True)
+ def test_instantiate_studies(self):
+ nb_sample = 2
+ for nb_days in sample(set(NB_DAYS), k=nb_sample):
+ for study in study_iterator_global(study_classes=SCM_STUDIES,
+ only_first_one=False, verbose=False,
+ altitudes=sample(set(ALL_ALTITUDES), k=nb_sample), nb_days=nb_days):
+ self.assertTrue('day' in study.variable_name)
+ first_path_file = study.ordered_years_and_path_files()[0][0]
+ variable_array = study.load_variables(path_file=first_path_file)
+ variable_object = study.instantiate_variable_object(variable_array)
+ self.assertEqual((365, 263), variable_object.daily_time_serie_array.shape,
+ msg='{} days for type {}'.format(nb_days, get_display_name_from_object_type(type(variable_object))))
+
class TestSCMStudy(unittest.TestCase):
@@ -53,7 +71,7 @@ class TestSCMPrecipitation(TestSCMStudy):
def setUp(self) -> None:
super().setUp()
- self.study = SafranTotalPrecip(altitude=1800, year_min=1958, year_max=2002)
+ self.study = SafranTotalPrecip(altitude=1800, year_min=1958, year_max=2002, nb_consecutive_days=1)
def test_durand(self):
# Test based on Durand paper
--
GitLab