[contrasting project] fix memory issue that was happening while loading many...

[contrasting project] fix memory issue that was happening while loading many precipitation studies. issue #300

spatio_temporal_dataset/coordinates/abstract_coordinates.py

@@ -6,6 +6,8 @@ import numpy as np
 import pandas as pd
 from mpl_toolkits.mplot3d import Axes3D
 
+from spatio_temporal_dataset.coordinates.temporal_coordinates.abstract_temporal_covariate_for_fit import \
+    AbstractTemporalCovariateForFit
 from spatio_temporal_dataset.coordinates.transformed_coordinates.transformation.abstract_transformation import \
     AbstractTransformation, IdentityTransformation
 from spatio_temporal_dataset.coordinates.utils import get_index_without_spatio_temporal_index_suffix
@@ -240,9 +242,11 @@ class AbstractCoordinates(object):
             return self.df_coordinates(split, transformed=transformed).loc[:, self.temporal_coordinates_names] \
                 .drop_duplicates()
 
-    def df_temporal_coordinates_for_fit(self, split=Split.all, starting_point=None) -> pd.DataFrame:
+    def df_temporal_coordinates_for_fit(self, split=Split.all, starting_point=None,
+                                        temporal_covariate_for_fit: Union[None, type] = None) -> pd.DataFrame:
+        # Load time covariate
         if starting_point is None:
-            return self.df_temporal_coordinates(split=split, transformed=True)
+            df = self.df_temporal_coordinates(split=split, transformed=True)
         else:
             # Load the un transformed coordinates
             df_temporal_coordinates = self.df_temporal_coordinates(split=split, transformed=False)
@@ -252,7 +256,7 @@ class AbstractCoordinates(object):
             ind_to_modify = df_temporal_coordinates.iloc[:, 0] <= starting_point  # type: pd.Series
             # Assert that some coordinates are selected but not all
             msg = '{} First year={} Last_year={}'.format(sum(ind_to_modify), df_temporal_coordinates.iloc[0, 0],
-                                  df_temporal_coordinates.iloc[-1, 0])
+                                                         df_temporal_coordinates.iloc[-1, 0])
             assert 0 < sum(ind_to_modify) < len(ind_to_modify), msg
             # Modify the temporal coordinates to enforce the stationarity
             df_temporal_coordinates.loc[ind_to_modify] = starting_point
@@ -260,7 +264,12 @@ class AbstractCoordinates(object):
             temporal_transformation = self.temporal_coordinates.transformation_class(
                 df_temporal_coordinates)  # type: AbstractTransformation
             # Return the result of the temporal transformation
-            return temporal_transformation.transform_df(df_temporal_coordinates)
+            df = temporal_transformation.transform_df(df_temporal_coordinates)
+        # Potentially transform the time covariate into another covariate
+        if temporal_covariate_for_fit is not None:
+            assert issubclass(temporal_covariate_for_fit, AbstractTemporalCovariateForFit)
+            df = df.apply(temporal_covariate_for_fit.get_temporal_covariate)
+        return df
 
     @property
     def temporal_coordinates(self):

spatio_temporal_dataset/coordinates/temporal_coordinates/abstract_temporal_covariate_for_fit.py

+class AbstractTemporalCovariateForFit(object):
+
+    @classmethod
+    def get_temporal_covariate(cls, t):
+        raise NotImplementedError
+
+
+class TimeTemporalCovariate(AbstractTemporalCovariateForFit):
+
+    @classmethod
+    def get_temporal_covariate(cls, t):
+        return t
+
+
+class MeanGlobalTemperatureCovariate(AbstractTemporalCovariateForFit):
+
+    @classmethod
+    def get_temporal_covariate(cls, t):
+        pass

test/test_spatio_temporal_dataset/test_coordinates.py

@@ -17,6 +17,8 @@ from spatio_temporal_dataset.coordinates.spatial_coordinates.alps_station_3D_coo
     AlpsStation3DCoordinatesWithAnisotropy
 from spatio_temporal_dataset.coordinates.spatial_coordinates.generated_spatial_coordinates import \
     CircleSpatialCoordinates
+from spatio_temporal_dataset.coordinates.temporal_coordinates.abstract_temporal_covariate_for_fit import \
+    AbstractTemporalCovariateForFit, TimeTemporalCovariate
 from spatio_temporal_dataset.coordinates.transformed_coordinates.transformation.abstract_transformation import \
     CenteredScaledNormalization
 from spatio_temporal_dataset.coordinates.transformed_coordinates.transformation.uniform_normalization import \
@@ -137,18 +139,30 @@ class TestCoordinatesWithTransformedStartingPoint(unittest.TestCase):
                                                                      transformation_class=BetweenZeroAndOneNormalization)[
             0]
         temporal_coordinates = \
-        load_test_temporal_coordinates(nb_steps=self.nb_steps, transformation_class=CenteredScaledNormalization)[0]
+            load_test_temporal_coordinates(nb_steps=self.nb_steps, transformation_class=CenteredScaledNormalization)[0]
         coordinates = AbstractSpatioTemporalCoordinates.from_spatial_coordinates_and_temporal_coordinates(
             spatial_coordinates=spatial_coordinate,
             temporal_coordinates=temporal_coordinates)
         # Check that df_all_coordinates have not yet been normalized
         self.assertEqual(coordinates.df_temporal_coordinates(transformed=False).iloc[-1, 0], 49.0)
         # Check that the normalization is working
-        self.assertAlmostEqual(coordinates.df_temporal_coordinates_for_fit(starting_point=None).iloc[0, 0], -1.697749375254331)
-        self.assertAlmostEqual(coordinates.df_temporal_coordinates_for_fit(starting_point=2).iloc[2, 0], -1.5739459974625107)
+        self.assertAlmostEqual(coordinates.df_temporal_coordinates_for_fit(starting_point=None).iloc[0, 0],
+                               -1.697749375254331)
+        self.assertAlmostEqual(coordinates.df_temporal_coordinates_for_fit(starting_point=2).iloc[2, 0],
+                               -1.5739459974625107)
         self.assertNotEqual(coordinates.df_temporal_coordinates_for_fit(starting_point=2).iloc[2, 0],
                             coordinates.df_temporal_coordinates_for_fit(starting_point=2).iloc[3, 0])
 
 
+class TestCoordinatesWithModifiedCovariate(unittest.TestCase):
+
+    def test_time_covariate(self):
+        coordinates = load_test_temporal_coordinates(nb_steps=10)[0]
+        old_df = coordinates.df_temporal_coordinates_for_fit().copy()
+        new_df = coordinates.df_temporal_coordinates_for_fit(temporal_covariate_for_fit=TimeTemporalCovariate)
+        pd.testing.assert_frame_equal(old_df, new_df)
+        # pd.as.assertEqual(old_df, new_df)
+
+
 if __name__ == '__main__':
     unittest.main()

test/test_utils.py

@@ -28,6 +28,8 @@ from spatio_temporal_dataset.coordinates.spatial_coordinates.generated_spatial_c
 from spatio_temporal_dataset.coordinates.spatio_temporal_coordinates.generated_spatio_temporal_coordinates import \
     UniformSpatioTemporalCoordinates, LinSpaceSpatial2DSpatioTemporalCoordinates
 from spatio_temporal_dataset.coordinates.spatial_coordinates.coordinates_1D import UniformSpatialCoordinates
+from spatio_temporal_dataset.coordinates.temporal_coordinates.abstract_temporal_coordinates import \
+    AbstractTemporalCoordinates
 from spatio_temporal_dataset.coordinates.temporal_coordinates.generated_temporal_coordinates import \
     ConsecutiveTemporalCoordinates
 
@@ -105,7 +107,7 @@ def load_test_3D_spatial_coordinates(nb_points, transformation_class=None) -> Li
                                          transformation_class=transformation_class)
 
 
-def load_test_temporal_coordinates(nb_steps, train_split_ratio=None, transformation_class=None):
+def load_test_temporal_coordinates(nb_steps, train_split_ratio=None, transformation_class=None) -> List[AbstractTemporalCoordinates]:
     return [coordinate_class.from_nb_temporal_steps(nb_temporal_steps=nb_steps,
                                                     train_split_ratio=train_split_ratio,
                                                     transformation_class=transformation_class)