From ec20ef1ae3e062993789877cde18a14baa0216b8 Mon Sep 17 00:00:00 2001
From: Le Roux Erwan <erwan.le-roux@irstea.fr>
Date: Fri, 20 Mar 2020 18:00:07 +0100
Subject: [PATCH] [quantile regression project] refactor imports for quantile
estimators. two fix: nb_obs=1 in GevSimulation.py and df_merged correspond to
df_coordinates from now on (such as it takes into account the transformation
if there is one)
---
.../abstract_quantile_estimator.py | 14 +-------------
.../quantile_estimator_from_margin.py | 14 ++------------
.../quantile_estimator_from_regression.py | 12 +-----------
.../quantile_model/quantile_regression_model.py | 1 -
.../AbstractSimulation.py | 10 +++++++---
.../quantile_regression_vs_evt/GevSimulation.py | 6 +++---
.../main_quantile_regression.py | 17 +++++++++++++++++
.../coordinates/abstract_coordinates.py | 2 +-
.../test_function/test_param_function.py | 2 +-
9 files changed, 33 insertions(+), 45 deletions(-)
create mode 100644 projects/quantile_regression_vs_evt/main_quantile_regression.py
diff --git a/extreme_fit/estimator/quantile_estimator/abstract_quantile_estimator.py b/extreme_fit/estimator/quantile_estimator/abstract_quantile_estimator.py
index 0c1a1a1d..c16b79c3 100644
--- a/extreme_fit/estimator/quantile_estimator/abstract_quantile_estimator.py
+++ b/extreme_fit/estimator/quantile_estimator/abstract_quantile_estimator.py
@@ -1,21 +1,9 @@
from abc import ABC
-import numpy as np
from cached_property import cached_property
from extreme_fit.estimator.abstract_estimator import AbstractEstimator
-from extreme_fit.estimator.margin_estimator.abstract_margin_estimator import LinearMarginEstimator
-from extreme_fit.function.abstract_quantile_function import AbstractQuantileFunction, \
- QuantileFunctionFromMarginFunction, QuantileFunctionFromParamFunction
-from extreme_fit.function.margin_function.abstract_margin_function import AbstractMarginFunction
-from extreme_fit.function.param_function.linear_coef import LinearCoef
-from extreme_fit.function.param_function.param_function import LinearParamFunction
-from extreme_fit.model.margin_model.linear_margin_model.abstract_temporal_linear_margin_model import \
- AbstractTemporalLinearMarginModel
-from extreme_fit.model.margin_model.linear_margin_model.linear_margin_model import LinearMarginModel
-from extreme_fit.model.quantile_model.quantile_regression_model import AbstractQuantileRegressionModel
-from extreme_fit.model.result_from_model_fit.abstract_result_from_model_fit import AbstractResultFromModelFit
-from extreme_fit.model.result_from_model_fit.result_from_quantilreg import ResultFromQuantreg
+from extreme_fit.function.abstract_quantile_function import AbstractQuantileFunction
from spatio_temporal_dataset.dataset.abstract_dataset import AbstractDataset
diff --git a/extreme_fit/estimator/quantile_estimator/quantile_estimator_from_margin.py b/extreme_fit/estimator/quantile_estimator/quantile_estimator_from_margin.py
index 6a65903a..7a672a42 100644
--- a/extreme_fit/estimator/quantile_estimator/quantile_estimator_from_margin.py
+++ b/extreme_fit/estimator/quantile_estimator/quantile_estimator_from_margin.py
@@ -1,22 +1,12 @@
-from abc import ABC
-
-import numpy as np
from cached_property import cached_property
-from extreme_fit.estimator.abstract_estimator import AbstractEstimator
from extreme_fit.estimator.margin_estimator.abstract_margin_estimator import LinearMarginEstimator
from extreme_fit.estimator.quantile_estimator.abstract_quantile_estimator import AbstractQuantileEstimator
from extreme_fit.function.abstract_quantile_function import AbstractQuantileFunction, \
- QuantileFunctionFromMarginFunction, QuantileFunctionFromParamFunction
+ QuantileFunctionFromMarginFunction
from extreme_fit.function.margin_function.abstract_margin_function import AbstractMarginFunction
-from extreme_fit.function.param_function.linear_coef import LinearCoef
-from extreme_fit.function.param_function.param_function import LinearParamFunction
from extreme_fit.model.margin_model.linear_margin_model.abstract_temporal_linear_margin_model import \
- AbstractTemporalLinearMarginModel, TemporalMarginFitMethod
-from extreme_fit.model.margin_model.linear_margin_model.linear_margin_model import LinearMarginModel
-from extreme_fit.model.quantile_model.quantile_regression_model import AbstractQuantileRegressionModel
-from extreme_fit.model.result_from_model_fit.abstract_result_from_model_fit import AbstractResultFromModelFit
-from extreme_fit.model.result_from_model_fit.result_from_quantilreg import ResultFromQuantreg
+ TemporalMarginFitMethod
from spatio_temporal_dataset.dataset.abstract_dataset import AbstractDataset
diff --git a/extreme_fit/estimator/quantile_estimator/quantile_estimator_from_regression.py b/extreme_fit/estimator/quantile_estimator/quantile_estimator_from_regression.py
index 8b7d2b0c..acd1e56a 100644
--- a/extreme_fit/estimator/quantile_estimator/quantile_estimator_from_regression.py
+++ b/extreme_fit/estimator/quantile_estimator/quantile_estimator_from_regression.py
@@ -1,20 +1,10 @@
-from abc import ABC
-
-import numpy as np
from cached_property import cached_property
-from extreme_fit.estimator.abstract_estimator import AbstractEstimator
-from extreme_fit.estimator.margin_estimator.abstract_margin_estimator import LinearMarginEstimator
from extreme_fit.estimator.quantile_estimator.abstract_quantile_estimator import AbstractQuantileEstimator
from extreme_fit.function.abstract_quantile_function import AbstractQuantileFunction, \
- QuantileFunctionFromMarginFunction, QuantileFunctionFromParamFunction
-from extreme_fit.function.margin_function.abstract_margin_function import AbstractMarginFunction
+ QuantileFunctionFromParamFunction
from extreme_fit.function.param_function.linear_coef import LinearCoef
from extreme_fit.function.param_function.param_function import LinearParamFunction
-from extreme_fit.model.margin_model.linear_margin_model.abstract_temporal_linear_margin_model import \
- AbstractTemporalLinearMarginModel
-from extreme_fit.model.margin_model.linear_margin_model.linear_margin_model import LinearMarginModel
-from extreme_fit.model.quantile_model.quantile_regression_model import AbstractQuantileRegressionModel
from extreme_fit.model.result_from_model_fit.abstract_result_from_model_fit import AbstractResultFromModelFit
from extreme_fit.model.result_from_model_fit.result_from_quantilreg import ResultFromQuantreg
from spatio_temporal_dataset.dataset.abstract_dataset import AbstractDataset
diff --git a/extreme_fit/model/quantile_model/quantile_regression_model.py b/extreme_fit/model/quantile_model/quantile_regression_model.py
index 88459f77..a9e377c6 100644
--- a/extreme_fit/model/quantile_model/quantile_regression_model.py
+++ b/extreme_fit/model/quantile_model/quantile_regression_model.py
@@ -27,7 +27,6 @@ class AbstractQuantileRegressionModel(AbstractModel):
'tau': self.quantile,
'data': self.data,
'formula': self.formula
-
}
res = safe_run_r_estimator(r.rq, **parameters)
return ResultFromQuantreg(res)
diff --git a/projects/quantile_regression_vs_evt/AbstractSimulation.py b/projects/quantile_regression_vs_evt/AbstractSimulation.py
index 14dbb926..a757e77e 100644
--- a/projects/quantile_regression_vs_evt/AbstractSimulation.py
+++ b/projects/quantile_regression_vs_evt/AbstractSimulation.py
@@ -15,6 +15,8 @@ from root_utils import get_display_name_from_object_type
from spatio_temporal_dataset.coordinates.abstract_coordinates import AbstractCoordinates
from spatio_temporal_dataset.coordinates.temporal_coordinates.generated_temporal_coordinates import \
ConsecutiveTemporalCoordinates
+from spatio_temporal_dataset.coordinates.transformed_coordinates.transformation.abstract_transformation import \
+ CenteredScaledNormalization, IdentityTransformation
from spatio_temporal_dataset.dataset.abstract_dataset import AbstractDataset
from spatio_temporal_dataset.spatio_temporal_observations.abstract_spatio_temporal_observations import \
AbstractSpatioTemporalObservations
@@ -27,7 +29,8 @@ class Coordinates(object):
class AbstractSimulation(object):
def __init__(self, nb_time_series, quantile, time_series_lengths=None, multiprocessing=False,
- model_classes=None):
+ model_classes=None, transformation_class=CenteredScaledNormalization):
+ self.transformation_class = transformation_class
self.models_classes = model_classes
self.multiprocessing = multiprocessing
self.quantile = quantile
@@ -48,7 +51,8 @@ class AbstractSimulation(object):
def time_serie_length_to_coordinates(self) -> Dict[int, AbstractCoordinates]:
d = OrderedDict()
for length in self.time_series_lengths:
- d[length] = ConsecutiveTemporalCoordinates.from_nb_temporal_steps(length)
+ d[length] = ConsecutiveTemporalCoordinates.from_nb_temporal_steps(length,
+ transformation_class=self.transformation_class)
return d
@cached_property
@@ -100,7 +104,7 @@ class AbstractSimulation(object):
label = get_display_name_from_object_type(model_class)
ax.plot(lengths, mean_error, label=label)
ax.set_xlabel('# Data')
- ax.set_ylabel('Absolute error for the {} quantile at the last coordinate'.format(self.quantile))
+ ax.set_ylabel('Relative error for the {} quantile at the last coordinate'.format(self.quantile))
ax.legend()
if show:
plt.show()
diff --git a/projects/quantile_regression_vs_evt/GevSimulation.py b/projects/quantile_regression_vs_evt/GevSimulation.py
index e6d66047..62109e0e 100644
--- a/projects/quantile_regression_vs_evt/GevSimulation.py
+++ b/projects/quantile_regression_vs_evt/GevSimulation.py
@@ -33,7 +33,7 @@ class GevSimulation(AbstractSimulation):
def generate_all_observation(self, nb_time_series, length) -> List[AbstractSpatioTemporalObservations]:
coordinates = self.time_serie_length_to_coordinates[length]
margin_model = self.time_series_lengths_to_margin_model[length]
- return [MarginAnnualMaxima.from_sampling(nb_obs=length, coordinates=coordinates, margin_model=margin_model)
+ return [MarginAnnualMaxima.from_sampling(nb_obs=1, coordinates=coordinates, margin_model=margin_model)
for _ in range(nb_time_series)]
def compute_errors(self, length: int, estimators: List[AbstractQuantileEstimator]):
@@ -44,7 +44,7 @@ class GevSimulation(AbstractSimulation):
true_quantile = margin_model.margin_function_sample.get_gev_params(last_coordinate).quantile(self.quantile)
# Compute estimated values
estimated_quantiles = [estimator.function_from_fit.get_quantile(last_coordinate) for estimator in estimators]
- return np.abs(np.array(estimated_quantiles) - true_quantile)
+ return 100 * np.abs(np.array(estimated_quantiles) - true_quantile) / true_quantile
class StationarySimulation(GevSimulation):
@@ -63,7 +63,7 @@ class NonStationaryLocationSimulation(GevSimulation):
def create_model(self, coordinates):
gev_param_name_to_coef_list = {
- GevParams.LOC: [0, 1],
+ GevParams.LOC: [0, 10],
GevParams.SHAPE: [0],
GevParams.SCALE: [1],
}
diff --git a/projects/quantile_regression_vs_evt/main_quantile_regression.py b/projects/quantile_regression_vs_evt/main_quantile_regression.py
new file mode 100644
index 00000000..c13d6da7
--- /dev/null
+++ b/projects/quantile_regression_vs_evt/main_quantile_regression.py
@@ -0,0 +1,17 @@
+from extreme_fit.model.margin_model.linear_margin_model.temporal_linear_margin_models import StationaryTemporalModel, \
+ NonStationaryLocationTemporalModel
+from extreme_fit.model.quantile_model.quantile_regression_model import ConstantQuantileRegressionModel, \
+ TemporalCoordinatesQuantileRegressionModel
+from projects.quantile_regression_vs_evt.GevSimulation import StationarySimulation, NonStationaryLocationSimulation
+
+nb_time_series = 10
+quantile = 0.9
+time_series_lengths = [50, 100, 200]
+
+# simulation = StationarySimulation(nb_time_series=nb_time_series, quantile=quantile, time_series_lengths=time_series_lengths,
+# model_classes=[StationaryTemporalModel, ConstantQuantileRegressionModel])
+# simulation.plot_error_for_last_year_quantile()
+
+simulation = NonStationaryLocationSimulation(nb_time_series=nb_time_series, quantile=quantile, time_series_lengths=time_series_lengths,
+ model_classes=[NonStationaryLocationTemporalModel, TemporalCoordinatesQuantileRegressionModel][:])
+simulation.plot_error_for_last_year_quantile()
diff --git a/spatio_temporal_dataset/coordinates/abstract_coordinates.py b/spatio_temporal_dataset/coordinates/abstract_coordinates.py
index 16cc04ad..af07c111 100644
--- a/spatio_temporal_dataset/coordinates/abstract_coordinates.py
+++ b/spatio_temporal_dataset/coordinates/abstract_coordinates.py
@@ -129,7 +129,7 @@ class AbstractCoordinates(object):
@property
def df_merged(self) -> pd.DataFrame:
# Merged DataFrame of df_coord with s_split
- return self.df_all_coordinates.join(self.df_split)
+ return self.df_coordinates().join(self.df_split)
# Split
diff --git a/test/test_extreme_fit/test_function/test_param_function.py b/test/test_extreme_fit/test_function/test_param_function.py
index fba861ce..2604da08 100644
--- a/test/test_extreme_fit/test_function/test_param_function.py
+++ b/test/test_extreme_fit/test_function/test_param_function.py
@@ -6,7 +6,7 @@ from extreme_fit.function.param_function.linear_coef import LinearCoef
from extreme_fit.function.param_function.param_function import LinearParamFunction
-class ParamFunction(unittest.TestCase):
+class TestParamFunction(unittest.TestCase):
def test_out_of_bounds(self):
param_function = LinearParamFunction(dims=[0], coordinates=np.array([[0]]), linear_coef=LinearCoef())
--
GitLab