From f28d5eb27e476c971495e35f53b952a006037d53 Mon Sep 17 00:00:00 2001
From: Le Roux Erwan <erwan.le-roux@irstea.fr>
Date: Mon, 27 May 2019 17:59:23 +0200
Subject: [PATCH] [SCM] remove map issue that was due to the fact of caching
year_to_dataset_ordered_dict attribute
---
.../study/abstract_study.py | 8 +-
.../study/crocus/crocus.py | 9 +-
.../main_hypercube_visualization.py | 4 +-
.../main_study_visualizer.py | 29 +++--
.../meteo_france_SCM_study/crocus/__init__.py | 0
.../meteo_france_SCM_study/crocus/crocus.py | 71 ------------
.../crocus/crocus_variables.py | 31 -----
.../meteo_france_SCM_study/safran/__init__.py | 0
.../meteo_france_SCM_study/safran/safran.py | 88 ---------------
.../safran/safran_variable.py | 106 ------------------
.../visualization/__init__.py | 0
test/test_experiment/test_SCM_study.py | 2 +-
12 files changed, 26 insertions(+), 322 deletions(-)
delete mode 100644 experiment/meteo_france_SCM_study/crocus/__init__.py
delete mode 100644 experiment/meteo_france_SCM_study/crocus/crocus.py
delete mode 100644 experiment/meteo_france_SCM_study/crocus/crocus_variables.py
delete mode 100644 experiment/meteo_france_SCM_study/safran/__init__.py
delete mode 100644 experiment/meteo_france_SCM_study/safran/safran.py
delete mode 100644 experiment/meteo_france_SCM_study/safran/safran_variable.py
delete mode 100644 experiment/meteo_france_SCM_study/visualization/__init__.py
diff --git a/experiment/meteo_france_SCM_models/study/abstract_study.py b/experiment/meteo_france_SCM_models/study/abstract_study.py
index 7b37cc84..2be3827e 100644
--- a/experiment/meteo_france_SCM_models/study/abstract_study.py
+++ b/experiment/meteo_france_SCM_models/study/abstract_study.py
@@ -75,7 +75,7 @@ class AbstractStudy(object):
""" Load some attributes only once """
- @cached_property
+ @property
def year_to_dataset_ordered_dict(self) -> OrderedDict:
print('This code is quite long... '
'You should consider year_to_variable which is way faster when multiprocessing=True')
@@ -101,12 +101,12 @@ class AbstractStudy(object):
path_files, ordered_years = self.ordered_years_and_path_files()
if self.multiprocessing:
with Pool(NB_CORES) as p:
- variables = p.map(self.load_variable, path_files)
+ variables = p.map(self.load_variables, path_files)
else:
- variables = [self.load_variable(path_file) for path_file in path_files]
+ variables = [self.load_variables(path_file) for path_file in path_files]
return OrderedDict(zip(ordered_years, variables))
- def load_variable(self, path_file):
+ def load_variables(self, path_file):
dataset = Dataset(path_file)
keyword = self.variable_class.keyword()
if isinstance(keyword, str):
diff --git a/experiment/meteo_france_SCM_models/study/crocus/crocus.py b/experiment/meteo_france_SCM_models/study/crocus/crocus.py
index e9740802..fc3b39f4 100644
--- a/experiment/meteo_france_SCM_models/study/crocus/crocus.py
+++ b/experiment/meteo_france_SCM_models/study/crocus/crocus.py
@@ -3,6 +3,7 @@ import numpy as np
from experiment.meteo_france_SCM_models.study.abstract_extended_study import AbstractExtendedStudy
from experiment.meteo_france_SCM_models.study.abstract_study import AbstractStudy
from experiment.meteo_france_SCM_models.study.crocus.crocus_variables import CrocusSweVariable, CrocusDepthVariable
+from experiment.meteo_france_SCM_models.study.cumulated_study import CumulatedStudy
class Crocus(AbstractStudy):
@@ -25,10 +26,11 @@ class Crocus(AbstractStudy):
return super().apply_annual_aggregation(time_serie[91:-92, ...])
-class CrocusSwe(Crocus):
+class CrocusSwe(Crocus, CumulatedStudy):
def __init__(self, *args, **kwargs):
- super().__init__(CrocusSweVariable, *args, **kwargs)
+ CumulatedStudy.__init__(self, CrocusSweVariable, *args, **kwargs)
+ Crocus.__init__(self, CrocusSweVariable, *args, **kwargs)
def apply_annual_aggregation(self, time_serie):
return self.winter_annual_aggregation(time_serie)
@@ -62,8 +64,7 @@ class CrocusDaysWithSnowOnGround(Crocus):
if __name__ == '__main__':
- for variable_class in [CrocusSweVariable, CrocusDepthVariable][:1]:
- study = Crocus(variable_class=variable_class, altitude=2400)
+ for study in [CrocusSwe(altitude=900)]:
d = study.year_to_dataset_ordered_dict[1960]
print(study.df_massifs_longitude_and_latitude)
time_arr = np.array(d.variables['time'])
diff --git a/experiment/meteo_france_SCM_models/visualization/hypercube_visualization/main_hypercube_visualization.py b/experiment/meteo_france_SCM_models/visualization/hypercube_visualization/main_hypercube_visualization.py
index b46406c7..4ff80b08 100644
--- a/experiment/meteo_france_SCM_models/visualization/hypercube_visualization/main_hypercube_visualization.py
+++ b/experiment/meteo_france_SCM_models/visualization/hypercube_visualization/main_hypercube_visualization.py
@@ -131,8 +131,8 @@ def fast_quantity_altitude_hypercube():
def main_run():
- # fast_altitude_hypercube()
- fast_altitude_year_hypercube()
+ fast_altitude_hypercube()
+ # fast_altitude_year_hypercube()
# full_altitude_year_hypercube()
# fast_quantity_altitude_hypercube()
# full_quantity_altitude_hypercube()
diff --git a/experiment/meteo_france_SCM_models/visualization/study_visualization/main_study_visualizer.py b/experiment/meteo_france_SCM_models/visualization/study_visualization/main_study_visualizer.py
index 57323d90..b09cd17a 100644
--- a/experiment/meteo_france_SCM_models/visualization/study_visualization/main_study_visualizer.py
+++ b/experiment/meteo_france_SCM_models/visualization/study_visualization/main_study_visualizer.py
@@ -27,41 +27,40 @@ full_altitude_with_at_least_2_stations = [0, 300, 600, 900, 1200, 1500, 1800, 21
ALL_STUDIES = SCM_STUDIES + [SafranTemperature, SafranRainfall]
-def study_iterator_global(study_classes, only_first_one=False, both_altitude=False, verbose=True, altitudes=None) -> \
+def study_iterator_global(study_classes, only_first_one=False, verbose=True, altitudes=None, nb_days=None) -> \
List[AbstractStudy]:
for study_class in study_classes:
- for study in study_iterator(study_class, only_first_one, both_altitude, verbose, altitudes):
+ for study in study_iterator(study_class, only_first_one, verbose, altitudes, nb_days):
yield study
if only_first_one:
break
-def study_iterator(study_class, only_first_one=False, both_altitude=False, verbose=True, altitudes=None) -> List[
- AbstractStudy]:
- all_studies = []
- is_safran_study = study_class in [SafranSnowfall, ExtendedSafranSnowfall]
- nb_days = [1] if is_safran_study else [1]
+def study_iterator(study_class, only_first_one=False, verbose=True, altitudes=None, nb_days=None) -> List[AbstractStudy]:
+ # Default argument
+ nb_days = [1] if nb_days is None else nb_days
+ altis = [1800] if altitudes is None else altitudes
+
if verbose:
- print('\n\n\n\n\nLoading studies....', end='')
+ print('\n\n\n\n\nLoading studies....')
for nb_day in nb_days:
- altis = [1800] if altitudes is None else altitudes
-
for alti in altis:
-
if verbose:
print('alti: {}, nb_day: {} '.format(alti, nb_day), end='')
+
study = study_class(altitude=alti)
- massifs = study.altitude_to_massif_names[alti]
+
if verbose:
+ massifs = study.altitude_to_massif_names[alti]
print('{} massifs: {} \n'.format(len(massifs), massifs))
yield study
- if only_first_one and not both_altitude:
+
+ # Stop iterations on purpose
+ if only_first_one:
break
if only_first_one:
break
- return all_studies
-
def extended_visualization():
save_to_file = False
diff --git a/experiment/meteo_france_SCM_study/crocus/__init__.py b/experiment/meteo_france_SCM_study/crocus/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/experiment/meteo_france_SCM_study/crocus/crocus.py b/experiment/meteo_france_SCM_study/crocus/crocus.py
deleted file mode 100644
index 56d75595..00000000
--- a/experiment/meteo_france_SCM_study/crocus/crocus.py
+++ /dev/null
@@ -1,71 +0,0 @@
-import numpy as np
-
-from experiment.meteo_france_SCM_study.abstract_extended_study import AbstractExtendedStudy
-from experiment.meteo_france_SCM_study.abstract_study import AbstractStudy
-from experiment.meteo_france_SCM_study.crocus.crocus_variables import CrocusSweVariable, CrocusDepthVariable
-
-
-class Crocus(AbstractStudy):
- """
- In the Crocus data, there is no 'massifsList' variable, thus we assume massifs are ordered just like Safran data
- """
-
- def __init__(self, variable_class, *args, **kwargs):
- assert variable_class in [CrocusSweVariable, CrocusDepthVariable]
- super().__init__(variable_class, *args, **kwargs)
- self.model_name = 'Crocus'
-
- def annual_aggregation_function(self, *args, **kwargs):
- return np.mean(*args, **kwargs)
-
- def winter_annual_aggregation(self, time_serie):
- # In the Durand paper, we only want the data from November to April
- # 91 = 30 + 31 + 30 first days of the time serie correspond to the month of August + September + October
- # 92 = 31 + 30 + 31 last days correspond to the month of May + June + JUly
- return super().apply_annual_aggregation(time_serie[91:-92, ...])
-
-
-class CrocusSwe(Crocus):
-
- def __init__(self, *args, **kwargs):
- super().__init__(CrocusSweVariable, *args, **kwargs)
-
- def apply_annual_aggregation(self, time_serie):
- return self.winter_annual_aggregation(time_serie)
-
-
-class ExtendedCrocusSwe(AbstractExtendedStudy, CrocusSwe):
- pass
-
-
-class CrocusDepth(Crocus):
-
- def __init__(self, *args, **kwargs):
- super().__init__(CrocusDepthVariable, *args, **kwargs)
-
- def apply_annual_aggregation(self, time_serie):
- return self.winter_annual_aggregation(time_serie)
-
-
-class ExtendedCrocusDepth(AbstractExtendedStudy, CrocusDepth):
- pass
-
-
-class CrocusDaysWithSnowOnGround(Crocus):
- """Having snow on the ground is equivalent to snow depth > 0"""
-
- def __init__(self, *args, **kwargs):
- super().__init__(CrocusDepthVariable, *args, **kwargs)
-
- def annual_aggregation_function(self, *args, **kwargs):
- return np.count_nonzero(*args, **kwargs)
-
-
-if __name__ == '__main__':
- for variable_class in [CrocusSweVariable, CrocusDepthVariable][:1]:
- study = Crocus(variable_class=variable_class, altitude=2400)
- d = study.year_to_dataset_ordered_dict[1960]
- print(study.df_massifs_longitude_and_latitude)
- time_arr = np.array(d.variables['time'])
- a = study.year_to_daily_time_serie_array[1960]
- print(a.shape)
diff --git a/experiment/meteo_france_SCM_study/crocus/crocus_variables.py b/experiment/meteo_france_SCM_study/crocus/crocus_variables.py
deleted file mode 100644
index 6f1bf6fc..00000000
--- a/experiment/meteo_france_SCM_study/crocus/crocus_variables.py
+++ /dev/null
@@ -1,31 +0,0 @@
-import numpy as np
-
-from experiment.meteo_france_SCM_study.abstract_variable import AbstractVariable
-
-
-class CrocusVariable(AbstractVariable):
-
- def __init__(self, variable_array):
- super().__init__(variable_array)
-
- @property
- def daily_time_serie_array(self) -> np.ndarray:
- return self.variable_array
-
-
-class CrocusSweVariable(CrocusVariable):
- NAME = 'Snow Water Equivalent'
- UNIT = 'kg per m2 or mm'
-
- @classmethod
- def keyword(cls):
- return 'SWE_1DY_ISBA'
-
-
-class CrocusDepthVariable(CrocusVariable):
- NAME = 'Snow Depth'
- UNIT = 'm'
-
- @classmethod
- def keyword(cls):
- return "SD_1DY_ISBA"
diff --git a/experiment/meteo_france_SCM_study/safran/__init__.py b/experiment/meteo_france_SCM_study/safran/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/experiment/meteo_france_SCM_study/safran/safran.py b/experiment/meteo_france_SCM_study/safran/safran.py
deleted file mode 100644
index 0258c0ff..00000000
--- a/experiment/meteo_france_SCM_study/safran/safran.py
+++ /dev/null
@@ -1,88 +0,0 @@
-import numpy as np
-
-from experiment.meteo_france_SCM_study.abstract_extended_study import AbstractExtendedStudy
-from experiment.meteo_france_SCM_study.abstract_study import AbstractStudy
-from experiment.meteo_france_SCM_study.abstract_variable import AbstractVariable
-from experiment.meteo_france_SCM_study.safran.safran_variable import SafranSnowfallVariable, \
- SafranRainfallVariable, SafranTemperatureVariable, SafranTotalPrecipVariable
-
-
-class Safran(AbstractStudy):
-
- def __init__(self, variable_class: type, *args, **kwargs):
- assert variable_class in [SafranSnowfallVariable, SafranRainfallVariable, SafranTemperatureVariable,
- SafranTotalPrecipVariable]
- super().__init__(variable_class, *args, **kwargs)
- self.model_name = 'Safran'
-
-
-class SafranFrequency(Safran):
-
- def __init__(self, variable_class: type, nb_consecutive_days: int = 1, *args, **kwargs):
- assert 1 <= nb_consecutive_days <= 7
- super().__init__(variable_class, *args, **kwargs)
- self.nb_consecutive_days = nb_consecutive_days
-
- def instantiate_variable_object(self, variable_array) -> AbstractVariable:
- return self.variable_class(variable_array, self.nb_consecutive_days)
-
- @property
- def variable_name(self):
- return super().variable_name + ' cumulated over {} day(s)'.format(self.nb_consecutive_days)
-
- def annual_aggregation_function(self, *args, **kwargs):
- return np.sum(*args, **kwargs)
-
-
-class SafranSnowfall(SafranFrequency):
-
- def __init__(self, *args, **kwargs):
- super().__init__(SafranSnowfallVariable, *args, **kwargs)
-
-
-class ExtendedSafranSnowfall(AbstractExtendedStudy, SafranSnowfall):
- pass
-
-
-class SafranRainfall(SafranFrequency):
-
- def __init__(self, *args, **kwargs):
- super().__init__(SafranRainfallVariable, *args, **kwargs)
-
-
-class SafranTotalPrecip(SafranFrequency):
-
- def __init__(self, *args, **kwargs):
- super().__init__(SafranTotalPrecipVariable, *args, **kwargs)
-
- def instantiate_variable_object(self, variable_array) -> AbstractVariable:
- variable_array_snowfall, variable_array_rainfall = variable_array
- return self.variable_class(variable_array_snowfall, variable_array_rainfall, self.nb_consecutive_days)
-
-
-class ExtendedSafranTotalPrecip(AbstractExtendedStudy, SafranTotalPrecip):
- pass
-
-
-class SafranTemperature(Safran):
-
- def __init__(self, *args, **kwargs):
- super().__init__(SafranTemperatureVariable, *args, **kwargs)
-
- def annual_aggregation_function(self, *args, **kwargs):
- return np.mean(*args, **kwargs)
-
-
-if __name__ == '__main__':
- study = SafranSnowfall(altitude=1800)
- d = study.year_to_dataset_ordered_dict[1958]
- # print(d.variables['time'])
- # print(study.all_massif_names)
- # print(study.massif_name_to_altitudes)
- # print(study.year_to_daily_time_serie_array[1958].shape)
- print(study.missing_massif_name)
-
-
- # print(len(d.variables['time']))
- # print(study.year_to_annual_total)
- # print(study.df_annual_total.columns)
diff --git a/experiment/meteo_france_SCM_study/safran/safran_variable.py b/experiment/meteo_france_SCM_study/safran/safran_variable.py
deleted file mode 100644
index 6ed6bd83..00000000
--- a/experiment/meteo_france_SCM_study/safran/safran_variable.py
+++ /dev/null
@@ -1,106 +0,0 @@
-import numpy as np
-
-from experiment.meteo_france_SCM_study.abstract_variable import AbstractVariable
-
-
-class SafranSnowfallVariable(AbstractVariable):
- """"
- Safran data is hourly
-
- Hypothesis:
-
- -How to count how much snowfall in one hour ?
- I take the average between the rhythm of snowfall per second between the start and the end
- and multiply that by 60 x 60 which corresponds to the number of seconds in one hour
-
- -How do how I define the limit of a day ?
- From the start, i.e. in August at 4am something like that,then if I add a 24H duration, I arrive to the next day
-
- -How do you aggregate several days ?
- We aggregate all the N consecutive days into a value x_i, then we take the max
- (but here the problem might be that the x_i are not idnependent, they are highly dependent one from another)
- """
-
- NAME = 'Snowfall'
- UNIT = 'kg per m2 or mm'
-
- @classmethod
- def keyword(cls):
- return 'Snowf'
-
- def __init__(self, variable_array, nb_consecutive_days_of_snowfall=1):
- super().__init__(variable_array)
- self.nb_consecutive_days_of_snowfall = nb_consecutive_days_of_snowfall
- # Compute the daily snowfall in kg/m2
- snowfall_rates = variable_array
-
- # Compute the mean snowrate, then multiply it by 60 * 60 * 24
- # day_duration_in_seconds = 24 * 60 * 60
- # nb_days = len(snowfall_rates) // 24
- # print(nb_days)
- # daily_snowrate = [np.mean(snowfall_rates[24 * i:24 * (i + 1) + 1], axis=0) for i in range(nb_days)]
- # self.daily_snowfall = day_duration_in_seconds * np.array(daily_snowrate)
-
- # Compute the hourly snowfall first, then aggregate
- mean_snowfall_rates = 0.5 * (snowfall_rates[:-1] + snowfall_rates[1:])
- hourly_snowfall = 60 * 60 * mean_snowfall_rates
- # Transform the snowfall amount into a dataframe
- nb_days = len(hourly_snowfall) // 24
- self.daily_snowfall = [sum(hourly_snowfall[24 * i:24 * (i + 1)]) for i in range(nb_days)]
-
- @property
- def daily_time_serie_array(self) -> np.ndarray:
- # Aggregate the daily snowfall by the number of consecutive days
- shifted_list = [self.daily_snowfall[i:] for i in range(self.nb_consecutive_days_of_snowfall)]
- # First element of shifted_list is of length n, Second element of length n-1, Third element n-2....
- # The zip is done with respect to the shortest list
- snowfall_in_consecutive_days = np.array([sum(e) for e in zip(*shifted_list)])
- # The returned array is of size n-nb_days+1 x nb_massif
- return snowfall_in_consecutive_days
-
-
-class SafranRainfallVariable(SafranSnowfallVariable):
-
- NAME = 'Rainfall'
- UNIT = 'kg per m2 or mm'
-
- @classmethod
- def keyword(cls):
- return 'Rainf'
-
-
-class SafranTotalPrecipVariable(AbstractVariable):
-
- def __init__(self, snow_variable_array, rain_variable_array, nb_consecutive_days_of_snowfall=1):
- super().__init__(None)
- self.snow_precipitation = SafranSnowfallVariable(snow_variable_array, nb_consecutive_days_of_snowfall)
- self.rain_precipitation = SafranRainfallVariable(rain_variable_array, nb_consecutive_days_of_snowfall)
-
- @classmethod
- def keyword(cls):
- return [SafranSnowfallVariable.keyword(), SafranRainfallVariable.keyword()]
-
- @property
- def daily_time_serie_array(self) -> np.ndarray:
- return self.snow_precipitation.daily_time_serie_array + self.rain_precipitation.daily_time_serie_array
-
-
-class SafranTemperatureVariable(AbstractVariable):
-
- NAME = 'Temperature'
- UNIT = 'Celsius Degrees'
-
- @classmethod
- def keyword(cls):
- return 'Tair'
-
- def __init__(self, variable_array):
- super().__init__(variable_array)
- # Temperature are in K, I transform them as celsius
- self.hourly_temperature = self.variable_array - 273.15
- nb_days = len(self.hourly_temperature) // 24
- self.daily_temperature = [np.mean(self.hourly_temperature[24 * i:24 * (i + 1)], axis=0) for i in range(nb_days)]
-
- @property
- def daily_time_serie_array(self):
- return np.array(self.daily_temperature)
diff --git a/experiment/meteo_france_SCM_study/visualization/__init__.py b/experiment/meteo_france_SCM_study/visualization/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/test/test_experiment/test_SCM_study.py b/test/test_experiment/test_SCM_study.py
index cde5ebf2..a36b3c35 100644
--- a/test/test_experiment/test_SCM_study.py
+++ b/test/test_experiment/test_SCM_study.py
@@ -15,7 +15,7 @@ class TestSCMAllStudy(unittest.TestCase):
def test_extended_run(self):
for study_class in [ExtendedSafranSnowfall]:
- for study in study_iterator(study_class, only_first_one=True, both_altitude=False, verbose=False):
+ for study in study_iterator(study_class, only_first_one=True, verbose=False):
study_visualizer = StudyVisualizer(study, show=False, save_to_file=False)
study_visualizer.visualize_all_mean_and_max_graphs()
self.assertTrue(True)
--
GitLab