[projections] add TestSpatioTemporalDatasetForClimateModels. add TemperatureTemporalCovariate.

integrate DatasetForClimateModels into altitude_studies.py

extreme_data/meteo_france_data/adamont_data/cmip5/climate_explorer_cimp5.py

@@ -49,7 +49,8 @@ def years_and_global_mean_temps(gcm, scenario, year_min=None, year_max=None, rol
         download_dat(dat_filepath, txt_filepath)
     # Transform nc file into csv file
     if not op.exists(csv_filepath):
-        dat_to_csv(csv_filepath, txt_filepath, mean_annual_column_name, rolling_mean_annual_column_name, rolling=rolling)
+        dat_to_csv(csv_filepath, txt_filepath, mean_annual_column_name, rolling_mean_annual_column_name,
+                   rolling=rolling)
 
     # Load csv file
     df = pd.read_csv(csv_filepath, index_col=0)
@@ -119,12 +120,14 @@ def main_test_cmip5_loader():
             print(years)
             print(temps)
 
+
 def test_rolling():
     df = pd.DataFrame([1, 2, 3, 4, 5])
     print(df)
     df2 = df.rolling(window=3).mean()
     print(df2)
 
+
 if __name__ == '__main__':
     # main_example()
     # test_rolling()

extreme_data/meteo_france_data/adamont_data/utils/download_cmip5.py

-import cdsapi
-
-c = cdsapi.Client()
-
-models = ['mpi_esm_lr', 'cnrm_cm5']
-
-
-def get_year_min_and_max(model, experiment):
-    pass
-
-
-def retrieve(model, experiment):
-    year_min, year_max = get_year_min_and_max(model, experiment)
-    c.retrieve(
-        'projections-cmip5-monthly-single-levels',
-        {
-            'experiment': 'historical',
-            'variable': '2m_temperature',
-            'model': 'cnrm_cm5',
-            'ensemble_member': 'r1i1p1',
-            'period': '195001-200512',
-            'format': 'zip',
-        },
-        'download.zip')

projects/altitude_spatial_model/altitudes_fit/altitudes_studies.py

@@ -4,6 +4,8 @@ from collections import OrderedDict
 
 from cached_property import cached_property
 
+from extreme_data.meteo_france_data.adamont_data.abstract_adamont_study import AbstractAdamontStudy
+from extreme_data.meteo_france_data.adamont_data.adamont_scenario import scenario_to_str
 from extreme_data.meteo_france_data.scm_models_data.abstract_study import AbstractStudy
 from extreme_data.meteo_france_data.scm_models_data.visualization.main_study_visualizer import \
     SCM_STUDY_CLASS_TO_ABBREVIATION, STUDY_CLASS_TO_ABBREVIATION
@@ -14,6 +16,8 @@ from spatio_temporal_dataset.coordinates.spatial_coordinates.abstract_spatial_co
     AbstractSpatialCoordinates
 from spatio_temporal_dataset.coordinates.spatio_temporal_coordinates.abstract_spatio_temporal_coordinates import \
     AbstractSpatioTemporalCoordinates
+from spatio_temporal_dataset.coordinates.spatio_temporal_coordinates.spatio_temporal_coordinates_for_climate_models import \
+    SpatioTemporalCoordinatesForClimateModels
 from spatio_temporal_dataset.coordinates.temporal_coordinates.generated_temporal_coordinates import \
     ConsecutiveTemporalCoordinates
 from spatio_temporal_dataset.dataset.abstract_dataset import AbstractDataset
@@ -78,12 +82,23 @@ class AltitudesStudies(object):
             assert len(massif_altitudes) > 0
             spatial_coordinates = self.spatial_coordinates_for_altitudes(massif_altitudes)
         slicer_class = get_slicer_class_from_s_splits(s_split_spatial, s_split_temporal)
-        return AbstractSpatioTemporalCoordinates(slicer_class=slicer_class,
-                                                 s_split_spatial=s_split_spatial,
-                                                 s_split_temporal=s_split_temporal,
-                                                 transformation_class=self.spatial_transformation_class,
-                                                 spatial_coordinates=spatial_coordinates,
-                                                 temporal_coordinates=self.temporal_coordinates)
+        if isinstance(self.study, AbstractAdamontStudy):
+            return SpatioTemporalCoordinatesForClimateModels(slicer_class=slicer_class,
+                                                             s_split_spatial=s_split_spatial,
+                                                             s_split_temporal=s_split_temporal,
+                                                             transformation_class=self.spatial_transformation_class,
+                                                             spatial_coordinates=spatial_coordinates,
+                                                             temporal_coordinates=self.temporal_coordinates,
+                                                             gcm_rcm_couple=self.study.gcm_rcm_couple,
+                                                             scenario_str=scenario_to_str(self.study.scenario),
+                                                             )
+        else:
+            return AbstractSpatioTemporalCoordinates(slicer_class=slicer_class,
+                                                     s_split_spatial=s_split_spatial,
+                                                     s_split_temporal=s_split_temporal,
+                                                     transformation_class=self.spatial_transformation_class,
+                                                     spatial_coordinates=spatial_coordinates,
+                                                     temporal_coordinates=self.temporal_coordinates)
 
     @cached_property
     def temporal_coordinates(self):
@@ -115,7 +130,8 @@ class AltitudesStudies(object):
     def show_or_save_to_file(self, plot_name, show=False, no_title=False, tight_layout=None):
         study_visualizer = StudyVisualizer(study=self.study, show=show, save_to_file=not show)
         study_visualizer.plot_name = plot_name
-        study_visualizer.show_or_save_to_file(add_classic_title=False, dpi=500, no_title=no_title, tight_layout=tight_layout)
+        study_visualizer.show_or_save_to_file(add_classic_title=False, dpi=500, no_title=no_title,
+                                              tight_layout=tight_layout)
 
     def run_for_each_massif(self, function, massif_names, **kwargs):
         massif_names = massif_names if massif_names is not None else self.study.all_massif_names()
@@ -165,7 +181,7 @@ class AltitudesStudies(object):
             moment += ' change (between two block of 30 years) for'
         moment += 'mean' if not std else 'std'
         # if change is False:
-            # moment += ' (for the 60 years of data)'
+        # moment += ' (for the 60 years of data)'
         plot_name = '{} of {} maxima of {}'.format(moment, self.study.season_name,
                                                    SCM_STUDY_CLASS_TO_ABBREVIATION[self.study_class])
         ax.set_ylabel('{} ({})'.format(plot_name, self.study.variable_unit), fontsize=15)
@@ -196,4 +212,4 @@ class AltitudesStudies(object):
                     moment = function(annual_maxima)
                 mean_moment.append(moment)
                 altitudes.append(altitude)
-        plot_against_altitude(altitudes, ax, massif_id, massif_name, mean_moment)
+        plot_against_altitude(altitudes, ax, massif_id, massif_name, mean_moment)
\ No newline at end of file

spatio_temporal_dataset/coordinates/abstract_coordinates.py

@@ -6,8 +6,6 @@ import numpy as np
 import pandas as pd
 from mpl_toolkits.mplot3d import Axes3D
 
-from spatio_temporal_dataset.coordinates.temporal_coordinates.abstract_temporal_covariate_for_fit import \
-    AbstractTemporalCovariateForFit
 from spatio_temporal_dataset.coordinates.transformed_coordinates.transformation.abstract_transformation import \
     AbstractTransformation, IdentityTransformation
 from spatio_temporal_dataset.coordinates.utils import get_index_without_spatio_temporal_index_suffix
@@ -35,8 +33,13 @@ class AbstractCoordinates(object):
     # Temporal columns
     COORDINATE_T = 'coord_t'
     TEMPORAL_SPLIT = 'temporal_split'
+    # Climate model columns
+    COORDINATE_RCP = 'coord_rcp'
+    COORDINATE_GCM = 'coord_gcm'
+    COORDINATE_RCM = 'coord_rcm'
+    COORDINATE_CLIMATE_MODEL_NAMES = [COORDINATE_RCP, COORDINATE_GCM, COORDINATE_RCM]
     # Coordinates columns
-    COORDINATES_NAMES = COORDINATE_SPATIAL_NAMES + [COORDINATE_T]
+    COORDINATES_NAMES = COORDINATE_SPATIAL_NAMES + [COORDINATE_T] + COORDINATE_CLIMATE_MODEL_NAMES
     # Coordinate type
     ALL_COORDINATES_ACCEPTED_TYPES = ['int64', 'float64']
     COORDINATE_TYPE = 'float64'
@@ -50,7 +53,10 @@ class AbstractCoordinates(object):
         sorted_coordinates_columns = [c for c in self.COORDINATES_NAMES if c in coordinate_columns]
         self.df_all_coordinates = df.loc[:, sorted_coordinates_columns].copy()  # type: pd.DataFrame
         # Cast coordinates
-        self.df_all_coordinates = self.df_all_coordinates.astype(self.COORDINATE_TYPE)  # type: pd.DataFrame
+        ind = self.df_all_coordinates.columns.isin(self.COORDINATE_CLIMATE_MODEL_NAMES)
+        self.df_coordinate_climate_model = self.df_all_coordinates.loc[:, ind].copy()
+        self.df_all_coordinates = self.df_all_coordinates.loc[:, ~ind] # type: pd.DataFrame
+        self.df_all_coordinates = self.df_all_coordinates.astype(self.COORDINATE_TYPE)
 
         # Slicing attributes
         self.s_split_spatial = s_split_spatial  # type: pd.Series
@@ -64,7 +70,7 @@ class AbstractCoordinates(object):
         # Transformation only works for float coordinates
         accepted_dtypes = [self.COORDINATE_TYPE]
         assert len(self.df_all_coordinates.select_dtypes(include=accepted_dtypes).columns) \
-               == len(coordinate_columns), 'coordinates columns dtypes should belong to {}'.format(accepted_dtypes)
+               == len(coordinate_columns) - sum(ind), 'coordinates columns dtypes should belong to {}'.format(accepted_dtypes)
         # Transformation class is instantiated with all coordinates
         self.transformation = transformation_class(self.df_all_coordinates)  # type: AbstractTransformation
         assert isinstance(self.transformation, AbstractTransformation)
@@ -278,8 +284,7 @@ class AbstractCoordinates(object):
             df = temporal_transformation.transform_df(df_temporal_coordinates)
         # Potentially transform the time covariate into another covariate
         if temporal_covariate_for_fit is not None:
-            assert issubclass(temporal_covariate_for_fit, AbstractTemporalCovariateForFit)
-            df.iloc[:, 0] = df.iloc[:, 0].apply(temporal_covariate_for_fit.get_temporal_covariate)
+            df.loc[:, self.COORDINATE_T] = df.apply(temporal_covariate_for_fit.get_temporal_covariate, axis=1)
         return df
 
     @property
@@ -382,4 +387,4 @@ class AbstractCoordinates(object):
         return self.df_merged.equals(other.df_merged)
 
     def __str__(self):
-        return self.df_coordinates().__str__()
+        return pd.concat([self.df_coordinates(), self.df_coordinate_climate_model], axis=1).__str__()

spatio_temporal_dataset/coordinates/spatio_temporal_coordinates/abstract_spatio_temporal_coordinates.py

@@ -19,9 +19,7 @@ class AbstractSpatioTemporalCoordinates(AbstractCoordinates):
                  transformation_class: type = None,
                  spatial_coordinates: AbstractSpatialCoordinates = None,
                  temporal_coordinates: AbstractTemporalCoordinates = None):
-        if df is None:
-            assert spatial_coordinates is not None and temporal_coordinates is not None
-            df = self.get_df_from_spatial_and_temporal_coordinates(spatial_coordinates, temporal_coordinates)
+        df = self.load_df_is_needed(df, spatial_coordinates, temporal_coordinates)
         super().__init__(df, slicer_class, s_split_spatial, s_split_temporal, None)
         # Spatial coordinates'
         if spatial_coordinates is None:
@@ -42,6 +40,12 @@ class AbstractSpatioTemporalCoordinates(AbstractCoordinates):
         self.transformation = MultipleTransformation(transformation_1=self.spatial_coordinates.transformation,
                                                      transformation_2=self.temporal_coordinates.transformation)
 
+    def load_df_is_needed(self, df, spatial_coordinates, temporal_coordinates):
+        if df is None:
+            assert spatial_coordinates is not None and temporal_coordinates is not None
+            df = self.get_df_from_spatial_and_temporal_coordinates(spatial_coordinates, temporal_coordinates)
+        return df
+
     @property
     def spatial_coordinates(self):
         return self._spatial_coordinates

spatio_temporal_dataset/coordinates/spatio_temporal_coordinates/spatio_temporal_coordinates_for_climate_models.py

+import pandas as pd
+
+from spatio_temporal_dataset.coordinates.spatial_coordinates.abstract_spatial_coordinates import \
+    AbstractSpatialCoordinates
+from spatio_temporal_dataset.coordinates.spatio_temporal_coordinates.abstract_spatio_temporal_coordinates import \
+    AbstractSpatioTemporalCoordinates
+from spatio_temporal_dataset.coordinates.temporal_coordinates.abstract_temporal_coordinates import \
+    AbstractTemporalCoordinates
+from spatio_temporal_dataset.slicer.spatio_temporal_slicer import SpatioTemporalSlicer
+
+
+class SpatioTemporalCoordinatesForClimateModels(AbstractSpatioTemporalCoordinates):
+
+    def __init__(self, df: pd.DataFrame = None, slicer_class: type = SpatioTemporalSlicer,
+                 s_split_spatial: pd.Series = None, s_split_temporal: pd.Series = None,
+                 transformation_class: type = None, spatial_coordinates: AbstractSpatialCoordinates = None,
+                 temporal_coordinates: AbstractTemporalCoordinates = None,
+                 gcm_rcm_couple=None,
+                 scenario_str=None):
+        df = self.load_df_is_needed(df, spatial_coordinates, temporal_coordinates)
+        # Assign the climate model coordinates
+        gcm, rcm = gcm_rcm_couple
+        df[self.COORDINATE_RCP] = scenario_str
+        df[self.COORDINATE_GCM] = gcm
+        df[self.COORDINATE_RCM] = rcm
+        super().__init__(df, slicer_class, s_split_spatial, s_split_temporal, transformation_class, spatial_coordinates,
+                         temporal_coordinates)

spatio_temporal_dataset/coordinates/temporal_coordinates/abstract_temporal_covariate_for_fit.py

+import pandas as pd
+
+from extreme_data.meteo_france_data.adamont_data.cmip5.climate_explorer_cimp5 import year_to_global_mean_temp
+from spatio_temporal_dataset.coordinates.abstract_coordinates import AbstractCoordinates
+
+
 class AbstractTemporalCovariateForFit(object):
 
     @classmethod
-    def get_temporal_covariate(cls, t):
+    def get_temporal_covariate(cls, row: pd.Series):
         raise NotImplementedError
 
 
 class TimeTemporalCovariate(AbstractTemporalCovariateForFit):
 
     @classmethod
-    def get_temporal_covariate(cls, t):
-        return t
+    def get_temporal_covariate(cls, row: pd.Series):
+        return row[AbstractCoordinates.COORDINATE_T]
 
 
+class TemperatureTemporalCovariate(AbstractTemporalCovariateForFit):
+    gcm_and_scenario_to_d = {}
+
+    @classmethod
+    def get_temporal_covariate(cls, row: pd.Series):
+        year = row[AbstractCoordinates.COORDINATE_T]
+        gcm = None
+        scenario = None
+        if (gcm, scenario) not in cls.gcm_and_scenario_to_d:
+            d = year_to_global_mean_temp(gcm, scenario)
+            cls.gcm_and_scenario_to_d[(gcm, scenario)] = d
+        d = cls.gcm_and_scenario_to_d[(gcm, scenario)]
+        global_mean_temp = d[year]
+        print(type(global_mean_temp))
+        return global_mean_temp

test/test_projects/test_altitude_spatial/test_one_fold_fit.py

@@ -9,7 +9,7 @@ from extreme_fit.model.margin_model.polynomial_margin_model.models_based_on_pari
 from projects.altitude_spatial_model.altitudes_fit.altitudes_studies import AltitudesStudies
 from projects.altitude_spatial_model.altitudes_fit.one_fold_analysis.one_fold_fit import OneFoldFit
 from spatio_temporal_dataset.coordinates.temporal_coordinates.abstract_temporal_covariate_for_fit import \
-    TimeTemporalCovariate
+    TimeTemporalCovariate, TemperatureTemporalCovariate
 
 
 class TestOneFoldFit(unittest.TestCase):
@@ -35,7 +35,7 @@ class TestOneFoldFit(unittest.TestCase):
             dataset = self.load_dataset(study_class)
             one_fold_fit = OneFoldFit(self.massif_name, dataset,
                                       models_classes=self.model_classes, temporal_covariate_for_fit=None)
-            print(type(one_fold_fit.best_estimator.margin_model))
+            _ = one_fold_fit.best_estimator.margin_model
         self.assertTrue(True)
 
     def test_with_temporal_covariate_for_time(self):
@@ -44,9 +44,18 @@ class TestOneFoldFit(unittest.TestCase):
             one_fold_fit = OneFoldFit(self.massif_name, dataset,
                                       models_classes=self.model_classes,
                                       temporal_covariate_for_fit=TimeTemporalCovariate)
-            print(type(one_fold_fit.best_estimator.margin_model))
+            _ = one_fold_fit.best_estimator.margin_model
         self.assertTrue(True)
 
+    # def test_with_temporal_covariate_for_temperature(self):
+    #     for study_class in [AdamontSnowfall][:]:
+    #         dataset = self.load_dataset(study_class)
+    #         one_fold_fit = OneFoldFit(self.massif_name, dataset,
+    #                                   models_classes=self.model_classes,
+    #                                   temporal_covariate_for_fit=TemperatureTemporalCovariate)
+    #         print(type(one_fold_fit.best_estimator.margin_model))
+    #     self.assertTrue(True)
+
 
 if __name__ == '__main__':
     unittest.main()

test/test_projects/test_contrasting/test_altitudes_studies.py

 import unittest
 
+from extreme_data.meteo_france_data.adamont_data.adamont.adamont_snowfall import AdamontSnowfall
+from extreme_data.meteo_france_data.adamont_data.adamont_scenario import AdamontScenario
 from extreme_data.meteo_france_data.scm_models_data.safran.safran import SafranSnowfall1Day
 from projects.altitude_spatial_model.altitudes_fit.altitudes_studies import AltitudesStudies
 from spatio_temporal_dataset.slicer.split import Split
@@ -77,5 +79,20 @@ class TestSpatioTemporalDataset(TestAltitudesStudies):
         self.assertEqual(len(dataset.maxima_gev(split=Split.test_spatiotemporal_spatial)), 3)
 
 
+class TestSpatioTemporalDatasetForClimateModels(unittest.TestCase):
+
+    def setUp(self) -> None:
+        super().setUp()
+        altitudes = [900, 1200]
+        study_class = AdamontSnowfall
+        self.studies = AltitudesStudies(study_class, altitudes,
+                                        year_min=2009, year_max=2012,
+                                        scenario=AdamontScenario.rcp85)
+        self.massif_name = "Vercors"
+
+    def test_dataset(self):
+        dataset = self.studies.spatio_temporal_dataset(self.massif_name)
+        self.assertEqual(len(dataset.coordinates.df_coordinate_climate_model.columns), 3)
+
 if __name__ == '__main__':
     unittest.main()