[SCM] refactor study folder using multiple inherentence for Safran studies

experiment/meteo_france_SCM_study/abstract_extended_study.py → experiment/meteo_france_SCM_models/study/abstract_extended_study.py

 import numpy as np
 from collections import OrderedDict
 
-from experiment.meteo_france_SCM_study.abstract_study import AbstractStudy
+from experiment.meteo_france_SCM_models.study.abstract_study import AbstractStudy
 from spatio_temporal_dataset.coordinates.spatial_coordinates.abstract_spatial_coordinates import \
     AbstractSpatialCoordinates
 

experiment/meteo_france_SCM_study/abstract_study.py → experiment/meteo_france_SCM_models/study/abstract_study.py

@@ -15,9 +15,9 @@ from matplotlib import cm
 from matplotlib.colors import Normalize
 from netCDF4 import Dataset
 
-from experiment.meteo_france_SCM_study.abstract_variable import AbstractVariable
-from experiment.meteo_france_SCM_study.scm_constants import ALTITUDES, ZS_INT_23, ZS_INT_MASK, LONGITUDES, LATITUDES
-from experiment.meteo_france_SCM_study.visualization.utils import get_km_formatter
+from experiment.meteo_france_SCM_models.study.abstract_variable import AbstractVariable
+from experiment.meteo_france_SCM_models.study.scm_constants import ALTITUDES, ZS_INT_23, ZS_INT_MASK, LONGITUDES, LATITUDES
+from experiment.meteo_france_SCM_models.visualization.utils import get_km_formatter
 from extreme_estimator.extreme_models.margin_model.margin_function.abstract_margin_function import \
     AbstractMarginFunction
 from extreme_estimator.margin_fits.plot.create_shifted_cmap import get_color_rbga_shifted, create_colorbase_axis

experiment/meteo_france_SCM_study/abstract_variable.py → experiment/meteo_france_SCM_models/study/abstract_variable.py

@@ -9,8 +9,9 @@ class AbstractVariable(object):
     NAME = ''
     UNIT = ''
 
-    def __init__(self, variable_array):
+    def __init__(self, variable_array, nb_consecutive_days_of_snowfall=1):
         self.variable_array = variable_array
+        self.nb_consecutive_days_of_snowfall = nb_consecutive_days_of_snowfall
 
     @classmethod
     def keyword(cls):

experiment/meteo_france_SCM_models/study/crocus/crocus.py

+import numpy as np
+
+from experiment.meteo_france_SCM_models.study.abstract_extended_study import AbstractExtendedStudy
+from experiment.meteo_france_SCM_models.study.abstract_study import AbstractStudy
+from experiment.meteo_france_SCM_models.study.crocus.crocus_variables import CrocusSweVariable, CrocusDepthVariable
+
+
+class Crocus(AbstractStudy):
+    """
+    In the Crocus data, there is no 'massifsList' variable, thus we assume massifs are ordered just like Safran data
+    """
+
+    def __init__(self, variable_class, *args, **kwargs):
+        assert variable_class in [CrocusSweVariable, CrocusDepthVariable]
+        super().__init__(variable_class, *args, **kwargs)
+        self.model_name = 'Crocus'
+
+    def annual_aggregation_function(self, *args, **kwargs):
+        return np.mean(*args, **kwargs)
+
+    def winter_annual_aggregation(self, time_serie):
+        # In the Durand paper, we only want the data from November to April
+        # 91 = 30 + 31 + 30 first days of the time serie correspond to the month of August + September + October
+        # 92 = 31 + 30 + 31 last days correspond to the month of May + June + JUly
+        return super().apply_annual_aggregation(time_serie[91:-92, ...])
+
+
+class CrocusSwe(Crocus):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(CrocusSweVariable, *args, **kwargs)
+
+    def apply_annual_aggregation(self, time_serie):
+        return self.winter_annual_aggregation(time_serie)
+
+
+class ExtendedCrocusSwe(AbstractExtendedStudy, CrocusSwe):
+    pass
+
+
+class CrocusDepth(Crocus):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(CrocusDepthVariable, *args, **kwargs)
+
+    def apply_annual_aggregation(self, time_serie):
+        return self.winter_annual_aggregation(time_serie)
+
+
+class ExtendedCrocusDepth(AbstractExtendedStudy, CrocusDepth):
+    pass
+
+
+class CrocusDaysWithSnowOnGround(Crocus):
+    """Having snow on the ground is equivalent to snow depth > 0"""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(CrocusDepthVariable, *args, **kwargs)
+
+    def annual_aggregation_function(self, *args, **kwargs):
+        return np.count_nonzero(*args, **kwargs)
+
+
+if __name__ == '__main__':
+    for variable_class in [CrocusSweVariable, CrocusDepthVariable][:1]:
+        study = Crocus(variable_class=variable_class, altitude=2400)
+        d = study.year_to_dataset_ordered_dict[1960]
+        print(study.df_massifs_longitude_and_latitude)
+        time_arr = np.array(d.variables['time'])
+        a = study.year_to_daily_time_serie_array[1960]
+        print(a.shape)

experiment/meteo_france_SCM_models/study/crocus/crocus_variables.py

+import numpy as np
+
+from experiment.meteo_france_SCM_models.study.abstract_variable import AbstractVariable
+
+
+class CrocusVariable(AbstractVariable):
+
+    @property
+    def daily_time_serie_array(self) -> np.ndarray:
+        return self.variable_array
+
+
+class CrocusSweVariable(CrocusVariable):
+    NAME = 'Snow Water Equivalent'
+    UNIT = 'kg per m2 or mm'
+
+    @classmethod
+    def keyword(cls):
+        return 'SWE_1DY_ISBA'
+
+
+class CrocusDepthVariable(CrocusVariable):
+    NAME = 'Snow Depth'
+    UNIT = 'm'
+
+    @classmethod
+    def keyword(cls):
+        return "SD_1DY_ISBA"

experiment/meteo_france_SCM_models/study/cumulated_study.py

+import numpy as np
+
+from experiment.meteo_france_SCM_models.study.abstract_study import AbstractStudy
+from experiment.meteo_france_SCM_models.study.abstract_variable import AbstractVariable
+
+
+class CumulatedStudy(AbstractStudy):
+    def __init__(self, variable_class: type, nb_consecutive_days: int = 1, *args, **kwargs):
+        assert nb_consecutive_days in [1, 3, 5, 7]
+        super().__init__(variable_class, *args, **kwargs)
+        self.nb_consecutive_days = nb_consecutive_days
+
+    def instantiate_variable_object(self, variable_array) -> AbstractVariable:
+        return self.variable_class(variable_array, self.nb_consecutive_days)
+
+    @property
+    def variable_name(self):
+        return super().variable_name + ' cumulated over {} day(s)'.format(self.nb_consecutive_days)
+
+

experiment/meteo_france_SCM_models/study/safran/safran.py

+import numpy as np
+
+from experiment.meteo_france_SCM_models.study.abstract_extended_study import AbstractExtendedStudy
+from experiment.meteo_france_SCM_models.study.abstract_study import AbstractStudy
+from experiment.meteo_france_SCM_models.study.abstract_variable import AbstractVariable
+from experiment.meteo_france_SCM_models.study.cumulated_study import CumulatedStudy
+from experiment.meteo_france_SCM_models.study.safran.safran_variable import SafranSnowfallVariable, \
+    SafranRainfallVariable, SafranTemperatureVariable, SafranTotalPrecipVariable
+
+
+class Safran(AbstractStudy):
+
+    def __init__(self, variable_class: type, *args, **kwargs):
+        assert variable_class in [SafranSnowfallVariable, SafranRainfallVariable, SafranTemperatureVariable,
+                                  SafranTotalPrecipVariable]
+        super().__init__(variable_class, *args, **kwargs)
+        self.model_name = 'Safran'
+
+    def annual_aggregation_function(self, *args, **kwargs):
+        return np.sum(*args, **kwargs)
+
+
+class SafranSnowfall(Safran, CumulatedStudy):
+
+    def __init__(self, *args, **kwargs):
+        CumulatedStudy.__init__(self, SafranSnowfallVariable, *args, **kwargs)
+        Safran.__init__(self, SafranSnowfallVariable, *args, **kwargs)
+
+
+class ExtendedSafranSnowfall(AbstractExtendedStudy, SafranSnowfall):
+    pass
+
+
+class SafranRainfall(CumulatedStudy, Safran):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(SafranRainfallVariable, *args, **kwargs)
+
+
+class SafranTotalPrecip(CumulatedStudy, Safran):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(SafranTotalPrecipVariable, *args, **kwargs)
+
+    def instantiate_variable_object(self, variable_array) -> AbstractVariable:
+        variable_array_snowfall, variable_array_rainfall = variable_array
+        return self.variable_class(variable_array_snowfall, variable_array_rainfall, self.nb_consecutive_days)
+
+
+class ExtendedSafranTotalPrecip(AbstractExtendedStudy, SafranTotalPrecip):
+    pass
+
+
+class SafranTemperature(Safran):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(SafranTemperatureVariable, *args, **kwargs)
+
+    def annual_aggregation_function(self, *args, **kwargs):
+        return np.mean(*args, **kwargs)
+
+
+if __name__ == '__main__':
+    study = SafranSnowfall()
+    d = study.year_to_dataset_ordered_dict[1958]
+    # print(d.variables['time'])
+    # print(study.all_massif_names)
+    # print(study.massif_name_to_altitudes)
+    # print(study.year_to_daily_time_serie_array[1958].shape)
+    print(study.missing_massif_name)
+
+    # print(len(d.variables['time']))
+    # print(study.year_to_annual_total)
+    # print(study.df_annual_total.columns)

experiment/meteo_france_SCM_models/study/safran/safran_variable.py

+import numpy as np
+
+from experiment.meteo_france_SCM_models.study.abstract_variable import AbstractVariable
+
+
+class SafranSnowfallVariable(AbstractVariable):
+    """"
+    Safran data is hourly
+
+    Hypothesis:
+
+    -How to count how much snowfall in one hour ?
+        I take the average between the rhythm of snowfall per second between the start and the end
+        and multiply that by 60 x 60 which corresponds to the number of seconds in one hour
+
+    -How do how I define the limit of a day ?
+        From the start, i.e. in August at 4am something like that,then if I add a 24H duration, I arrive to the next day
+
+    -How do you aggregate several days ?
+        We aggregate all the N consecutive days into a value x_i, then we take the max
+        (but here the problem might be that the x_i are not idnependent, they are highly dependent one from another)
+    """
+
+    NAME = 'Snowfall'
+    UNIT = 'kg per m2 or mm'
+
+    @classmethod
+    def keyword(cls):
+        return 'Snowf'
+
+    def __init__(self, variable_array, nb_consecutive_days_of_snowfall=1):
+        super().__init__(variable_array)
+        self.nb_consecutive_days_of_snowfall = nb_consecutive_days_of_snowfall
+        # Compute the daily snowfall in kg/m2
+        snowfall_rates = variable_array
+
+        # Compute the mean snowrate, then multiply it by 60 * 60 * 24
+        # day_duration_in_seconds = 24 * 60 * 60
+        # nb_days = len(snowfall_rates) // 24
+        # print(nb_days)
+        # daily_snowrate = [np.mean(snowfall_rates[24 * i:24 * (i + 1) + 1], axis=0) for i in range(nb_days)]
+        # self.daily_snowfall = day_duration_in_seconds * np.array(daily_snowrate)
+
+        # Compute the hourly snowfall first, then aggregate
+        mean_snowfall_rates = 0.5 * (snowfall_rates[:-1] + snowfall_rates[1:])
+        hourly_snowfall = 60 * 60 * mean_snowfall_rates
+        # Transform the snowfall amount into a dataframe
+        nb_days = len(hourly_snowfall) // 24
+        self.daily_snowfall = [sum(hourly_snowfall[24 * i:24 * (i + 1)]) for i in range(nb_days)]
+
+    @property
+    def daily_time_serie_array(self) -> np.ndarray:
+        # Aggregate the daily snowfall by the number of consecutive days
+        shifted_list = [self.daily_snowfall[i:] for i in range(self.nb_consecutive_days_of_snowfall)]
+        # First element of shifted_list is of length n, Second element of length n-1, Third element n-2....
+        # The zip is done with respect to the shortest list
+        snowfall_in_consecutive_days = np.array([sum(e) for e in zip(*shifted_list)])
+        # The returned array is of size n-nb_days+1 x nb_massif
+
+        # so that the length of the vector match a year, we can add zeros (since it corresponds to the July month,
+        # we are sure that there is no snowfall at this period) However such trick does not work for other variable such as Temperature
+
+        return snowfall_in_consecutive_days
+
+
+class SafranRainfallVariable(SafranSnowfallVariable):
+    NAME = 'Rainfall'
+    UNIT = 'kg per m2 or mm'
+
+    @classmethod
+    def keyword(cls):
+        return 'Rainf'
+
+
+class SafranTotalPrecipVariable(AbstractVariable):
+
+    def __init__(self, snow_variable_array, rain_variable_array, nb_consecutive_days_of_snowfall=1):
+        super().__init__(None)
+        self.snow_precipitation = SafranSnowfallVariable(snow_variable_array, nb_consecutive_days_of_snowfall)
+        self.rain_precipitation = SafranRainfallVariable(rain_variable_array, nb_consecutive_days_of_snowfall)
+
+    @classmethod
+    def keyword(cls):
+        return [SafranSnowfallVariable.keyword(), SafranRainfallVariable.keyword()]
+
+    @property
+    def daily_time_serie_array(self) -> np.ndarray:
+        return self.snow_precipitation.daily_time_serie_array + self.rain_precipitation.daily_time_serie_array
+
+
+class SafranTemperatureVariable(AbstractVariable):
+    NAME = 'Temperature'
+    UNIT = 'Celsius Degrees'
+
+    @classmethod
+    def keyword(cls):
+        return 'Tair'
+
+    def __init__(self, variable_array):
+        super().__init__(variable_array)
+        # Temperature are in K, I transform them as celsius
+        self.hourly_temperature = self.variable_array - 273.15
+        nb_days = len(self.hourly_temperature) // 24
+        # Compute the mean temperature
+        self.daily_temperature = [np.mean(self.hourly_temperature[24 * i:24 * (i + 1)], axis=0) for i in range(nb_days)]
+
+    @property
+    def daily_time_serie_array(self):
+        return np.array(self.daily_temperature)

experiment/meteo_france_SCM_study/visualization/hypercube_visualization/abstract_hypercube_visualizer.py → experiment/meteo_france_SCM_models/visualization/hypercube_visualization/abstract_hypercube_visualizer.py

@@ -5,7 +5,7 @@ from typing import Dict, Tuple
 import matplotlib.pyplot as plt
 import pandas as pd
 
-from experiment.meteo_france_SCM_study.visualization.study_visualization.study_visualizer import StudyVisualizer
+from experiment.meteo_france_SCM_models.visualization.study_visualization.study_visualizer import StudyVisualizer
 from utils import cached_property, VERSION_TIME, get_display_name_from_object_type
 
 

experiment/meteo_france_SCM_study/visualization/hypercube_visualization/altitude_hypercube_visualizer.py → experiment/meteo_france_SCM_models/visualization/hypercube_visualization/altitude_hypercube_visualizer.py

@@ -2,9 +2,9 @@ import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
 
-from experiment.meteo_france_SCM_study.visualization.hypercube_visualization.abstract_hypercube_visualizer import \
+from experiment.meteo_france_SCM_models.visualization.hypercube_visualization.abstract_hypercube_visualizer import \
     AbstractHypercubeVisualizer
-from experiment.meteo_france_SCM_study.visualization.study_visualization.study_visualizer import StudyVisualizer
+from experiment.meteo_france_SCM_models.visualization.study_visualization.study_visualizer import StudyVisualizer
 from experiment.trend_analysis.univariate_test.abstract_univariate_test import AbstractUnivariateTest
 from utils import get_display_name_from_object_type
 

experiment/meteo_france_SCM_study/visualization/hypercube_visualization/main_hypercube_visualization.py → experiment/meteo_france_SCM_models/visualization/hypercube_visualization/main_hypercube_visualization.py

@@ -2,13 +2,13 @@ import time
 from itertools import product
 from collections import OrderedDict
 
-from experiment.meteo_france_SCM_study.visualization.hypercube_visualization.altitude_hypercube_visualizer import \
+from experiment.meteo_france_SCM_models.visualization.hypercube_visualization.altitude_hypercube_visualizer import \
     AltitudeHypercubeVisualizer, Altitude_Hypercube_Year_Visualizer
-from experiment.meteo_france_SCM_study.visualization.hypercube_visualization.quantity_altitude_visualizer import \
+from experiment.meteo_france_SCM_models.visualization.hypercube_visualization.quantity_altitude_visualizer import \
     QuantityAltitudeHypercubeVisualizer
-from experiment.meteo_france_SCM_study.visualization.study_visualization.main_study_visualizer import ALL_ALTITUDES, \
+from experiment.meteo_france_SCM_models.visualization.study_visualization.main_study_visualizer import ALL_ALTITUDES, \
     SCM_STUDIES, study_iterator, study_iterator_global
-from experiment.meteo_france_SCM_study.visualization.study_visualization.study_visualizer import StudyVisualizer
+from experiment.meteo_france_SCM_models.visualization.study_visualization.study_visualizer import StudyVisualizer
 from experiment.trend_analysis.univariate_test.abstract_gev_change_point_test import GevLocationChangePointTest, \
     GevScaleChangePointTest, GevShapeChangePointTest
 from utils import get_display_name_from_object_type

experiment/meteo_france_SCM_study/visualization/hypercube_visualization/quantity_altitude_visualizer.py → experiment/meteo_france_SCM_models/visualization/hypercube_visualization/quantity_altitude_visualizer.py

 import pandas as pd
 
-from experiment.meteo_france_SCM_study.visualization.hypercube_visualization.altitude_hypercube_visualizer import \
+from experiment.meteo_france_SCM_models.visualization.hypercube_visualization.altitude_hypercube_visualizer import \
     AltitudeHypercubeVisualizer