From 02df2455fd366d11e5693f339162d4bab27ff7a3 Mon Sep 17 00:00:00 2001
From: Le Roux Erwan <erwan.le-roux@irstea.fr>
Date: Tue, 4 Jun 2019 10:51:59 +0200
Subject: [PATCH] [METEO FRANCE DATA] refactor folder organization. add
comparisons_visualization
---
.../scm_models_data/abstract_study.py | 2 +-
.../visualization/__init__.py | 0
.../hypercube_visualization/__init__.py | 0
.../abstract_hypercube_visualizer.py | 2 +-
.../altitude_hypercube_visualizer.py | 7 +-
.../altitude_year_hypercube_visualizer.py | 2 +-
.../main_hypercube_visualization.py | 18 ++---
.../quantity_altitude_visualizer.py | 2 +-
.../studies_visualization/__init__.py | 0
.../main_studies_visualizer.py | 8 +--
.../studies_visualization/studies.py | 0
.../studies_visualizer.py | 6 +-
.../study_visualization/__init__.py | 0
.../main_study_visualizer.py | 3 +-
.../study_visualization/study_visualizer.py | 2 +-
.../visualization/utils.py | 0
.../stations_data/__init__.py | 0
.../stations_data/comparison_analysis.py | 67 +----------------
.../stations_data/main_spatial_comparison.py | 71 +++++++++++++++++++
.../stations_data/main_station_comparison.py | 7 ++
.../stations_data/visualization/__init__.py | 0
.../comparisons_visualization/__init__.py | 0
.../comparisons_visualization.py | 0
.../main_comparison_visualization.py | 0
.../abstract_margin_function.py | 3 +-
test/test_experiment/test_SCM_study.py | 7 +-
.../test_coordinate_sensitivity.py | 5 +-
27 files changed, 113 insertions(+), 99 deletions(-)
rename experiment/meteo_france_data/{ => scm_models_data}/visualization/__init__.py (100%)
rename experiment/meteo_france_data/{ => scm_models_data}/visualization/hypercube_visualization/__init__.py (100%)
rename experiment/meteo_france_data/{ => scm_models_data}/visualization/hypercube_visualization/abstract_hypercube_visualizer.py (97%)
rename experiment/meteo_france_data/{ => scm_models_data}/visualization/hypercube_visualization/altitude_hypercube_visualizer.py (97%)
rename experiment/meteo_france_data/{ => scm_models_data}/visualization/hypercube_visualization/altitude_year_hypercube_visualizer.py (91%)
rename experiment/meteo_france_data/{ => scm_models_data}/visualization/hypercube_visualization/main_hypercube_visualization.py (91%)
rename experiment/meteo_france_data/{ => scm_models_data}/visualization/hypercube_visualization/quantity_altitude_visualizer.py (93%)
rename experiment/meteo_france_data/{ => scm_models_data}/visualization/studies_visualization/__init__.py (100%)
rename experiment/meteo_france_data/{ => scm_models_data}/visualization/studies_visualization/main_studies_visualizer.py (87%)
rename experiment/meteo_france_data/{ => scm_models_data}/visualization/studies_visualization/studies.py (100%)
rename experiment/meteo_france_data/{ => scm_models_data}/visualization/studies_visualization/studies_visualizer.py (97%)
rename experiment/meteo_france_data/{ => scm_models_data}/visualization/study_visualization/__init__.py (100%)
rename experiment/meteo_france_data/{ => scm_models_data}/visualization/study_visualization/main_study_visualizer.py (98%)
rename experiment/meteo_france_data/{ => scm_models_data}/visualization/study_visualization/study_visualizer.py (99%)
rename experiment/meteo_france_data/{ => scm_models_data}/visualization/utils.py (100%)
create mode 100644 experiment/meteo_france_data/stations_data/__init__.py
create mode 100644 experiment/meteo_france_data/stations_data/main_spatial_comparison.py
create mode 100644 experiment/meteo_france_data/stations_data/main_station_comparison.py
create mode 100644 experiment/meteo_france_data/stations_data/visualization/__init__.py
create mode 100644 experiment/meteo_france_data/stations_data/visualization/comparisons_visualization/__init__.py
create mode 100644 experiment/meteo_france_data/stations_data/visualization/comparisons_visualization/comparisons_visualization.py
create mode 100644 experiment/meteo_france_data/stations_data/visualization/comparisons_visualization/main_comparison_visualization.py
diff --git a/experiment/meteo_france_data/scm_models_data/abstract_study.py b/experiment/meteo_france_data/scm_models_data/abstract_study.py
index 7badd0f5..96c348f2 100644
--- a/experiment/meteo_france_data/scm_models_data/abstract_study.py
+++ b/experiment/meteo_france_data/scm_models_data/abstract_study.py
@@ -18,7 +18,7 @@ from netCDF4 import Dataset
from experiment.meteo_france_data.scm_models_data.abstract_variable import AbstractVariable
from experiment.meteo_france_data.scm_models_data.scm_constants import ALTITUDES, ZS_INT_23, ZS_INT_MASK, LONGITUDES, \
LATITUDES
-from experiment.meteo_france_data.visualization.utils import get_km_formatter
+from experiment.meteo_france_data.scm_models_data.visualization.utils import get_km_formatter
from extreme_estimator.extreme_models.margin_model.margin_function.abstract_margin_function import \
AbstractMarginFunction
from extreme_estimator.margin_fits.plot.create_shifted_cmap import get_color_rbga_shifted, create_colorbase_axis
diff --git a/experiment/meteo_france_data/visualization/__init__.py b/experiment/meteo_france_data/scm_models_data/visualization/__init__.py
similarity index 100%
rename from experiment/meteo_france_data/visualization/__init__.py
rename to experiment/meteo_france_data/scm_models_data/visualization/__init__.py
diff --git a/experiment/meteo_france_data/visualization/hypercube_visualization/__init__.py b/experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/__init__.py
similarity index 100%
rename from experiment/meteo_france_data/visualization/hypercube_visualization/__init__.py
rename to experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/__init__.py
diff --git a/experiment/meteo_france_data/visualization/hypercube_visualization/abstract_hypercube_visualizer.py b/experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/abstract_hypercube_visualizer.py
similarity index 97%
rename from experiment/meteo_france_data/visualization/hypercube_visualization/abstract_hypercube_visualizer.py
rename to experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/abstract_hypercube_visualizer.py
index d8eadaae..72d51cc7 100644
--- a/experiment/meteo_france_data/visualization/hypercube_visualization/abstract_hypercube_visualizer.py
+++ b/experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/abstract_hypercube_visualizer.py
@@ -5,7 +5,7 @@ from typing import Dict, Tuple
import matplotlib.pyplot as plt
import pandas as pd
-from experiment.meteo_france_data.visualization.study_visualization.study_visualizer import StudyVisualizer
+from experiment.meteo_france_data.scm_models_data.visualization import StudyVisualizer
from utils import cached_property, VERSION_TIME, get_display_name_from_object_type
diff --git a/experiment/meteo_france_data/visualization/hypercube_visualization/altitude_hypercube_visualizer.py b/experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/altitude_hypercube_visualizer.py
similarity index 97%
rename from experiment/meteo_france_data/visualization/hypercube_visualization/altitude_hypercube_visualizer.py
rename to experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/altitude_hypercube_visualizer.py
index 2e9beb56..c45e4912 100644
--- a/experiment/meteo_france_data/visualization/hypercube_visualization/altitude_hypercube_visualizer.py
+++ b/experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/altitude_hypercube_visualizer.py
@@ -1,14 +1,11 @@
-from typing import List
-
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
-from experiment.meteo_france_data.visualization.hypercube_visualization.abstract_hypercube_visualizer import \
+from experiment.meteo_france_data.scm_models_data.visualization.hypercube_visualization.abstract_hypercube_visualizer import \
AbstractHypercubeVisualizer
-from experiment.meteo_france_data.visualization.study_visualization.study_visualizer import StudyVisualizer
+from experiment.meteo_france_data.scm_models_data.visualization import StudyVisualizer
from experiment.trend_analysis.univariate_test.abstract_univariate_test import AbstractUnivariateTest
-from utils import get_display_name_from_object_type
ALTITUDES_XLABEL = 'altitudes'
diff --git a/experiment/meteo_france_data/visualization/hypercube_visualization/altitude_year_hypercube_visualizer.py b/experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/altitude_year_hypercube_visualizer.py
similarity index 91%
rename from experiment/meteo_france_data/visualization/hypercube_visualization/altitude_year_hypercube_visualizer.py
rename to experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/altitude_year_hypercube_visualizer.py
index 3009fd26..e7c000b8 100644
--- a/experiment/meteo_france_data/visualization/hypercube_visualization/altitude_year_hypercube_visualizer.py
+++ b/experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/altitude_year_hypercube_visualizer.py
@@ -1,6 +1,6 @@
import numpy as np
-from experiment.meteo_france_data.visualization.hypercube_visualization.altitude_hypercube_visualizer import \
+from experiment.meteo_france_data.scm_models_data.visualization import \
AltitudeHypercubeVisualizer
diff --git a/experiment/meteo_france_data/visualization/hypercube_visualization/main_hypercube_visualization.py b/experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/main_hypercube_visualization.py
similarity index 91%
rename from experiment/meteo_france_data/visualization/hypercube_visualization/main_hypercube_visualization.py
rename to experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/main_hypercube_visualization.py
index cc84a572..4021a101 100644
--- a/experiment/meteo_france_data/visualization/hypercube_visualization/main_hypercube_visualization.py
+++ b/experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/main_hypercube_visualization.py
@@ -1,16 +1,16 @@
import time
-from itertools import product
from collections import OrderedDict
+from itertools import product
-from experiment.meteo_france_data.visualization.hypercube_visualization.altitude_hypercube_visualizer import \
+from experiment.meteo_france_data.scm_models_data.visualization import \
AltitudeHypercubeVisualizer
-from experiment.meteo_france_data.visualization.hypercube_visualization.altitude_year_hypercube_visualizer import \
+from experiment.meteo_france_data.scm_models_data.visualization import \
Altitude_Hypercube_Year_Visualizer
-from experiment.meteo_france_data.visualization.hypercube_visualization.quantity_altitude_visualizer import \
+from experiment.meteo_france_data.scm_models_data.visualization import \
QuantityAltitudeHypercubeVisualizer
-from experiment.meteo_france_data.visualization.study_visualization.main_study_visualizer import ALL_ALTITUDES, \
+from experiment.meteo_france_data.scm_models_data.visualization import ALL_ALTITUDES, \
SCM_STUDIES, study_iterator, study_iterator_global
-from experiment.meteo_france_data.visualization.study_visualization.study_visualizer import StudyVisualizer
+from experiment.meteo_france_data.scm_models_data.visualization import StudyVisualizer
from experiment.trend_analysis.univariate_test.abstract_gev_change_point_test import GevLocationChangePointTest, \
GevScaleChangePointTest, GevShapeChangePointTest
from utils import get_display_name_from_object_type
@@ -102,7 +102,8 @@ def full_altitude_year_hypercube():
for study_class in SCM_STUDIES[:1]:
for trend_test_class in [GevLocationChangePointTest, GevScaleChangePointTest,
GevShapeChangePointTest][:1]:
- for last_starting_year in [1967, 1977, 1987, 1997, 2007, None][-1:]:
+ years = [1967, 1977, 1987, 1997, 2007, None][:-1][::-1]
+ for last_starting_year in years:
visualizers = [
StudyVisualizer(study, temporal_non_stationarity=True, verbose=False, multiprocessing=True)
for study in study_iterator(study_class=study_class, only_first_one=only_first_one,
@@ -110,7 +111,8 @@ def full_altitude_year_hypercube():
altitude_to_visualizer = OrderedDict(zip(altitudes, visualizers))
visualizer = Altitude_Hypercube_Year_Visualizer(altitude_to_visualizer,
save_to_file=save_to_file,
- trend_test_class=trend_test_class, nb_data_reduced_for_speed=nb_data_reduced_for_speed,
+ trend_test_class=trend_test_class,
+ nb_data_reduced_for_speed=nb_data_reduced_for_speed,
last_starting_year=last_starting_year)
visualizer.visualize_year_trend_test()
visualizer.visualize_massif_trend_test()
diff --git a/experiment/meteo_france_data/visualization/hypercube_visualization/quantity_altitude_visualizer.py b/experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/quantity_altitude_visualizer.py
similarity index 93%
rename from experiment/meteo_france_data/visualization/hypercube_visualization/quantity_altitude_visualizer.py
rename to experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/quantity_altitude_visualizer.py
index 12f31e37..399a075c 100644
--- a/experiment/meteo_france_data/visualization/hypercube_visualization/quantity_altitude_visualizer.py
+++ b/experiment/meteo_france_data/scm_models_data/visualization/hypercube_visualization/quantity_altitude_visualizer.py
@@ -1,6 +1,6 @@
import pandas as pd
-from experiment.meteo_france_data.visualization.hypercube_visualization.altitude_hypercube_visualizer import \
+from experiment.meteo_france_data.scm_models_data.visualization import \
AltitudeHypercubeVisualizer
diff --git a/experiment/meteo_france_data/visualization/studies_visualization/__init__.py b/experiment/meteo_france_data/scm_models_data/visualization/studies_visualization/__init__.py
similarity index 100%
rename from experiment/meteo_france_data/visualization/studies_visualization/__init__.py
rename to experiment/meteo_france_data/scm_models_data/visualization/studies_visualization/__init__.py
diff --git a/experiment/meteo_france_data/visualization/studies_visualization/main_studies_visualizer.py b/experiment/meteo_france_data/scm_models_data/visualization/studies_visualization/main_studies_visualizer.py
similarity index 87%
rename from experiment/meteo_france_data/visualization/studies_visualization/main_studies_visualizer.py
rename to experiment/meteo_france_data/scm_models_data/visualization/studies_visualization/main_studies_visualizer.py
index 9cd31321..4b47fcbe 100644
--- a/experiment/meteo_france_data/visualization/studies_visualization/main_studies_visualizer.py
+++ b/experiment/meteo_france_data/scm_models_data/visualization/studies_visualization/main_studies_visualizer.py
@@ -4,13 +4,13 @@ from experiment.trend_analysis.univariate_test.abstract_gev_change_point_test im
GevScaleChangePointTest, GevShapeChangePointTest
from experiment.trend_analysis.univariate_test.abstract_univariate_test import MannKendallTrendTest
from experiment.meteo_france_data.scm_models_data.safran.safran import ExtendedSafranTotalPrecip
-from experiment.meteo_france_data.visualization.studies_visualization.studies import Studies
-from experiment.meteo_france_data.visualization.studies_visualization.studies_visualizer import StudiesVisualizer, \
+from experiment.meteo_france_data.scm_models_data.visualization import Studies
+from experiment.meteo_france_data.scm_models_data.visualization import StudiesVisualizer, \
AltitudeVisualizer
-from experiment.meteo_france_data.visualization.study_visualization.main_study_visualizer import ALL_ALTITUDES, \
+from experiment.meteo_france_data.scm_models_data.visualization import ALL_ALTITUDES, \
study_iterator_global, SCM_STUDIES
-from experiment.meteo_france_data.visualization.study_visualization.study_visualizer import StudyVisualizer
+from experiment.meteo_france_data.scm_models_data.visualization import StudyVisualizer
from collections import OrderedDict
diff --git a/experiment/meteo_france_data/visualization/studies_visualization/studies.py b/experiment/meteo_france_data/scm_models_data/visualization/studies_visualization/studies.py
similarity index 100%
rename from experiment/meteo_france_data/visualization/studies_visualization/studies.py
rename to experiment/meteo_france_data/scm_models_data/visualization/studies_visualization/studies.py
diff --git a/experiment/meteo_france_data/visualization/studies_visualization/studies_visualizer.py b/experiment/meteo_france_data/scm_models_data/visualization/studies_visualization/studies_visualizer.py
similarity index 97%
rename from experiment/meteo_france_data/visualization/studies_visualization/studies_visualizer.py
rename to experiment/meteo_france_data/scm_models_data/visualization/studies_visualization/studies_visualizer.py
index bdb536e2..c9c979c4 100644
--- a/experiment/meteo_france_data/visualization/studies_visualization/studies_visualizer.py
+++ b/experiment/meteo_france_data/scm_models_data/visualization/studies_visualization/studies_visualizer.py
@@ -11,10 +11,10 @@ from matplotlib.lines import Line2D
from experiment.meteo_france_data.scm_models_data.abstract_extended_study import AbstractExtendedStudy
from experiment.trend_analysis.univariate_test.abstract_univariate_test import AbstractUnivariateTest
-from experiment.meteo_france_data.visualization.studies_visualization.studies import \
+from experiment.meteo_france_data.scm_models_data.visualization import \
Studies
-from experiment.meteo_france_data.visualization.study_visualization.study_visualizer import StudyVisualizer
-from experiment.meteo_france_data.visualization.utils import plot_df
+from experiment.meteo_france_data.scm_models_data.visualization import StudyVisualizer
+from experiment.meteo_france_data.scm_models_data.visualization.utils import plot_df
from utils import cached_property, get_display_name_from_object_type, VERSION_TIME
diff --git a/experiment/meteo_france_data/visualization/study_visualization/__init__.py b/experiment/meteo_france_data/scm_models_data/visualization/study_visualization/__init__.py
similarity index 100%
rename from experiment/meteo_france_data/visualization/study_visualization/__init__.py
rename to experiment/meteo_france_data/scm_models_data/visualization/study_visualization/__init__.py
diff --git a/experiment/meteo_france_data/visualization/study_visualization/main_study_visualizer.py b/experiment/meteo_france_data/scm_models_data/visualization/study_visualization/main_study_visualizer.py
similarity index 98%
rename from experiment/meteo_france_data/visualization/study_visualization/main_study_visualizer.py
rename to experiment/meteo_france_data/scm_models_data/visualization/study_visualization/main_study_visualizer.py
index 6bed93e8..e626ad6f 100644
--- a/experiment/meteo_france_data/visualization/study_visualization/main_study_visualizer.py
+++ b/experiment/meteo_france_data/scm_models_data/visualization/study_visualization/main_study_visualizer.py
@@ -1,6 +1,8 @@
import time
from typing import List
+from experiment.meteo_france_data.scm_models_data.visualization.study_visualization.study_visualizer import \
+ StudyVisualizer
from experiment.trend_analysis.abstract_score import MannKendall
from experiment.meteo_france_data.scm_models_data.abstract_study import AbstractStudy
from experiment.meteo_france_data.scm_models_data.crocus.crocus import CrocusDepth, CrocusSwe, ExtendedCrocusDepth, \
@@ -8,7 +10,6 @@ from experiment.meteo_france_data.scm_models_data.crocus.crocus import CrocusDep
from experiment.meteo_france_data.scm_models_data.safran.safran import SafranSnowfall, ExtendedSafranSnowfall, SafranRainfall, \
SafranTemperature, SafranTotalPrecip
-from experiment.meteo_france_data.visualization.study_visualization.study_visualizer import StudyVisualizer
from collections import OrderedDict
from experiment.trend_analysis.univariate_test.abstract_gev_change_point_test import GevLocationChangePointTest
diff --git a/experiment/meteo_france_data/visualization/study_visualization/study_visualizer.py b/experiment/meteo_france_data/scm_models_data/visualization/study_visualization/study_visualizer.py
similarity index 99%
rename from experiment/meteo_france_data/visualization/study_visualization/study_visualizer.py
rename to experiment/meteo_france_data/scm_models_data/visualization/study_visualization/study_visualizer.py
index aa6d260a..ad99f029 100644
--- a/experiment/meteo_france_data/visualization/study_visualization/study_visualizer.py
+++ b/experiment/meteo_france_data/scm_models_data/visualization/study_visualization/study_visualizer.py
@@ -15,7 +15,7 @@ from experiment.trend_analysis.univariate_test.abstract_gev_change_point_test im
from experiment.trend_analysis.univariate_test.abstract_univariate_test import AbstractUnivariateTest
from experiment.trend_analysis.non_stationary_trends import \
ConditionalIndedendenceLocationTrendTest, MaxStableLocationTrendTest, IndependenceLocationTrendTest
-from experiment.meteo_france_data.visualization.utils import create_adjusted_axes
+from experiment.meteo_france_data.scm_models_data.visualization.utils import create_adjusted_axes
from experiment.utils import average_smoothing_with_sliding_window
from extreme_estimator.estimator.full_estimator.abstract_full_estimator import \
FullEstimatorInASingleStepWithSmoothMargin
diff --git a/experiment/meteo_france_data/visualization/utils.py b/experiment/meteo_france_data/scm_models_data/visualization/utils.py
similarity index 100%
rename from experiment/meteo_france_data/visualization/utils.py
rename to experiment/meteo_france_data/scm_models_data/visualization/utils.py
diff --git a/experiment/meteo_france_data/stations_data/__init__.py b/experiment/meteo_france_data/stations_data/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/experiment/meteo_france_data/stations_data/comparison_analysis.py b/experiment/meteo_france_data/stations_data/comparison_analysis.py
index 76c7d3da..daf5e819 100644
--- a/experiment/meteo_france_data/stations_data/comparison_analysis.py
+++ b/experiment/meteo_france_data/stations_data/comparison_analysis.py
@@ -5,11 +5,10 @@ from typing import List
from cached_property import cached_property
from experiment.meteo_france_data.scm_models_data.safran.safran import SafranSnowfall
-from experiment.meteo_france_data.visualization.study_visualization.main_study_visualizer import \
+from experiment.meteo_france_data.scm_models_data.visualization import \
ALL_ALTITUDES
from extreme_estimator.estimator.full_estimator.abstract_full_estimator import \
FullEstimatorInASingleStepWithSmoothMargin
-from extreme_estimator.extreme_models.margin_model.linear_margin_model import LinearAllParametersAllDimsMarginModel
from extreme_estimator.extreme_models.max_stable_model.abstract_max_stable_model import CovarianceFunction
from extreme_estimator.extreme_models.max_stable_model.max_stable_models import ExtremalT, BrownResnick
from spatio_temporal_dataset.coordinates.abstract_coordinates import AbstractCoordinates
@@ -237,67 +236,3 @@ class ComparisonAnalysis(object):
different_sign = [k for k, v in coefs[0].items() if np.sign(coefs[1][k]) != np.sign(v) ]
print('All linear coefficient have the same sign: {}, different_signs for: {}'.format(len(different_sign) == 0, different_sign))
-def choice_of_altitude_and_nb_border_data_to_remove_to_get_data_without_nan():
- for margin in [50, 100, 150, 200, 250, 300][2:3]:
- for altitude in [900, 1200, 1800][:1]:
- for nb in range(1, 4):
- s = ComparisonAnalysis(altitude=altitude, nb_border_data_to_remove=nb, margin=margin)
- print(margin, altitude, nb, 'nb massifs', len(s.intersection_massif_names), 'nb stations',
- len(s.stations_observations), 'nb observations', s.stations_observations.nb_obs,
- s.study_observations.nb_obs,
- s.stations_coordinates.index)
-
-
-def run_comparison_for_optimal_parameters_for_altitude_900():
- for nb in [0, 1, 2][:]:
- for transformation_class in [None, BetweenZeroAndOneNormalization][1:]:
- comparison = ComparisonAnalysis(altitude=900, nb_border_data_to_remove=nb, margin=150,
- exclude_some_massifs_from_the_intersection=nb == 2,
- transformation_class=transformation_class,
- normalize_observations=True)
- print('\n-----------\nnb:', nb, comparison.intersection_massif_names)
- # margin_model_classes = [LinearShapeAllDimsMarginModel, LinearLocationAllDimsMarginModel,
- # LinearAllParametersAllDimsMarginModel]
- for margin_model_class in [LinearAllParametersAllDimsMarginModel]:
- print(get_display_name_from_object_type(margin_model_class))
- comparison.spatial_comparison(margin_model_class)
-
-
-"""
-Comparaison données de re-analysis et données de stations
-
-J'ai utilisé le fichier "PrecipitationsAvalanches_MaxPrecipit_ParPoste_ParHiver_traites.xls"
-
-Après des analyses avec la fonction 'choice_of_altitude_and_nb_border_data_to_remove_to_get_data_without_nan'
-j'ai choisis de lancer mes analyses avec:
- -une altitude de 900m
- -une margin de 150m (donc je selectionne toutes les stations entre 750m et 1050m).
-Je ne choisis que des stations qui ont des observations complètes sur toute la periode d'observation.
-et je m'asssure de n'avoir une seule station par massif (qui appartient à l intersection des massifs entre les study et les stations)
-
-Souvent les observations manquantes se situaient dans les premières ou dans les dernières années
-j'ai donc ajouté un parametre nb_to_remove_border qui enlever ces observations (à la fois pour les study et les stations).
-Ce parametre entrainent donc des datasets avec moins d observations, mais avec plus de masssifs/stations
-
-Par contre, dans le cas nb_to_remove=2, il y avait de grosses différences si j'incluais ou non le massif Mercantour
-donc en tout attendant de mieux comprendre, j'ai prefere exclure ce massif dans ce cas
-
-Dans tous les cas, nb_to_remove de 0 Ã 2
-pour n'importe quel modele de marges
-et pour un max stable BrownResnick ou ExtremalT
-alors le signe des coefficient de marges selon les coordonées Lambert sont toujours les mêmes que l'on utilise les données
-de reanalysis ou les données de stations
-"""
-
-
-"""
-A way to improve the analysis would be to have another altitude of reference with a lot of data
-But for the other altitude, we have data issues because there is a Nan in the middle of the data
-Instead of removing on the side, I should remove the years that concerns as much station from the same altitude level
-I should find the "optimal" years to remove
-Then I should find a way to remove the same years in the study
-"""
-
-if __name__ == '__main__':
- run_comparison_for_optimal_parameters_for_altitude_900()
- # choice_of_altitude_and_nb_border_data_to_remove_to_get_data_without_nan()
diff --git a/experiment/meteo_france_data/stations_data/main_spatial_comparison.py b/experiment/meteo_france_data/stations_data/main_spatial_comparison.py
new file mode 100644
index 00000000..c3b50c4c
--- /dev/null
+++ b/experiment/meteo_france_data/stations_data/main_spatial_comparison.py
@@ -0,0 +1,71 @@
+from experiment.meteo_france_data.stations_data.comparison_analysis import ComparisonAnalysis
+from extreme_estimator.extreme_models.margin_model.linear_margin_model import LinearAllParametersAllDimsMarginModel
+from spatio_temporal_dataset.coordinates.transformed_coordinates.transformation.uniform_normalization import \
+ BetweenZeroAndOneNormalization
+from utils import get_display_name_from_object_type
+
+
+def choice_of_altitude_and_nb_border_data_to_remove_to_get_data_without_nan():
+ for margin in [50, 100, 150, 200, 250, 300][2:3]:
+ for altitude in [900, 1200, 1800][:1]:
+ for nb in range(1, 4):
+ s = ComparisonAnalysis(altitude=altitude, nb_border_data_to_remove=nb, margin=margin)
+ print(margin, altitude, nb, 'nb massifs', len(s.intersection_massif_names), 'nb stations',
+ len(s.stations_observations), 'nb observations', s.stations_observations.nb_obs,
+ s.study_observations.nb_obs,
+ s.stations_coordinates.index)
+
+
+def run_comparison_for_optimal_parameters_for_altitude_900():
+ for nb in [0, 1, 2][:]:
+ for transformation_class in [None, BetweenZeroAndOneNormalization][1:]:
+ comparison = ComparisonAnalysis(altitude=900, nb_border_data_to_remove=nb, margin=150,
+ exclude_some_massifs_from_the_intersection=nb == 2,
+ transformation_class=transformation_class,
+ normalize_observations=True)
+ print('\n-----------\nnb:', nb, comparison.intersection_massif_names)
+ # margin_model_classes = [LinearShapeAllDimsMarginModel, LinearLocationAllDimsMarginModel,
+ # LinearAllParametersAllDimsMarginModel]
+ for margin_model_class in [LinearAllParametersAllDimsMarginModel]:
+ print(get_display_name_from_object_type(margin_model_class))
+ comparison.spatial_comparison(margin_model_class)
+
+
+"""
+Comparaison données de re-analysis et données de stations
+
+J'ai utilisé le fichier "PrecipitationsAvalanches_MaxPrecipit_ParPoste_ParHiver_traites.xls"
+
+Après des analyses avec la fonction 'choice_of_altitude_and_nb_border_data_to_remove_to_get_data_without_nan'
+j'ai choisis de lancer mes analyses avec:
+ -une altitude de 900m
+ -une margin de 150m (donc je selectionne toutes les stations entre 750m et 1050m).
+Je ne choisis que des stations qui ont des observations complètes sur toute la periode d'observation.
+et je m'asssure de n'avoir une seule station par massif (qui appartient à l intersection des massifs entre les study et les stations)
+
+Souvent les observations manquantes se situaient dans les premières ou dans les dernières années
+j'ai donc ajouté un parametre nb_to_remove_border qui enlever ces observations (à la fois pour les study et les stations).
+Ce parametre entrainent donc des datasets avec moins d observations, mais avec plus de masssifs/stations
+
+Par contre, dans le cas nb_to_remove=2, il y avait de grosses différences si j'incluais ou non le massif Mercantour
+donc en tout attendant de mieux comprendre, j'ai prefere exclure ce massif dans ce cas
+
+Dans tous les cas, nb_to_remove de 0 Ã 2
+pour n'importe quel modele de marges
+et pour un max stable BrownResnick ou ExtremalT
+alors le signe des coefficient de marges selon les coordonées Lambert sont toujours les mêmes que l'on utilise les données
+de reanalysis ou les données de stations
+"""
+
+
+"""
+A way to improve the analysis would be to have another altitude of reference with a lot of data
+But for the other altitude, we have data issues because there is a Nan in the middle of the data
+Instead of removing on the side, I should remove the years that concerns as much station from the same altitude level
+I should find the "optimal" years to remove
+Then I should find a way to remove the same years in the study
+"""
+
+if __name__ == '__main__':
+ run_comparison_for_optimal_parameters_for_altitude_900()
+ # choice_of_altitude_and_nb_border_data_to_remove_to_get_data_without_nan()
diff --git a/experiment/meteo_france_data/stations_data/main_station_comparison.py b/experiment/meteo_france_data/stations_data/main_station_comparison.py
new file mode 100644
index 00000000..bbfcfd17
--- /dev/null
+++ b/experiment/meteo_france_data/stations_data/main_station_comparison.py
@@ -0,0 +1,7 @@
+from experiment.meteo_france_data.stations_data.comparison_analysis import ComparisonAnalysis
+
+if __name__ == '__main__':
+ comparison = ComparisonAnalysis(altitude=900, nb_border_data_to_remove=nb, margin=150,
+ exclude_some_massifs_from_the_intersection=nb == 2,
+ transformation_class=transformation_class,
+ normalize_observations=True)
\ No newline at end of file
diff --git a/experiment/meteo_france_data/stations_data/visualization/__init__.py b/experiment/meteo_france_data/stations_data/visualization/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/experiment/meteo_france_data/stations_data/visualization/comparisons_visualization/__init__.py b/experiment/meteo_france_data/stations_data/visualization/comparisons_visualization/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/experiment/meteo_france_data/stations_data/visualization/comparisons_visualization/comparisons_visualization.py b/experiment/meteo_france_data/stations_data/visualization/comparisons_visualization/comparisons_visualization.py
new file mode 100644
index 00000000..e69de29b
diff --git a/experiment/meteo_france_data/stations_data/visualization/comparisons_visualization/main_comparison_visualization.py b/experiment/meteo_france_data/stations_data/visualization/comparisons_visualization/main_comparison_visualization.py
new file mode 100644
index 00000000..e69de29b
diff --git a/extreme_estimator/extreme_models/margin_model/margin_function/abstract_margin_function.py b/extreme_estimator/extreme_models/margin_model/margin_function/abstract_margin_function.py
index 7d9c2a7a..57150769 100644
--- a/extreme_estimator/extreme_models/margin_model/margin_function/abstract_margin_function.py
+++ b/extreme_estimator/extreme_models/margin_model/margin_function/abstract_margin_function.py
@@ -1,11 +1,10 @@
from typing import Dict, List
-import matplotlib.cm as cm
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
-from experiment.meteo_france_data.visualization.utils import create_adjusted_axes
+from experiment.meteo_france_data.scm_models_data.visualization.utils import create_adjusted_axes
from extreme_estimator.margin_fits.gev.gev_params import GevParams
from extreme_estimator.margin_fits.plot.create_shifted_cmap import imshow_shifted
from spatio_temporal_dataset.coordinates.abstract_coordinates import AbstractCoordinates
diff --git a/test/test_experiment/test_SCM_study.py b/test/test_experiment/test_SCM_study.py
index eeed25f9..05c0e83d 100644
--- a/test/test_experiment/test_SCM_study.py
+++ b/test/test_experiment/test_SCM_study.py
@@ -8,9 +8,10 @@ from experiment.meteo_france_data.scm_models_data.cumulated_study import NB_DAYS
from experiment.meteo_france_data.scm_models_data.safran.safran import SafranSnowfall, ExtendedSafranSnowfall, \
SafranTemperature, \
SafranTotalPrecip
-from experiment.meteo_france_data.visualization.study_visualization.main_study_visualizer import study_iterator, \
- study_iterator_global, SCM_STUDIES, ALL_ALTITUDES
-from experiment.meteo_france_data.visualization.study_visualization.study_visualizer import StudyVisualizer
+from experiment.meteo_france_data.scm_models_data.visualization.study_visualization.main_study_visualizer import \
+ study_iterator, study_iterator_global, SCM_STUDIES, ALL_ALTITUDES
+from experiment.meteo_france_data.scm_models_data.visualization.study_visualization.study_visualizer import \
+ StudyVisualizer
from experiment.trend_analysis.univariate_test.abstract_gev_change_point_test import GevLocationChangePointTest
from utils import get_display_name_from_object_type
diff --git a/test/test_experiment/test_coordinate_sensitivity.py b/test/test_experiment/test_coordinate_sensitivity.py
index da06bb49..75f4525b 100644
--- a/test/test_experiment/test_coordinate_sensitivity.py
+++ b/test/test_experiment/test_coordinate_sensitivity.py
@@ -1,11 +1,12 @@
import unittest
from experiment.meteo_france_data.scm_models_data.crocus.crocus import CrocusSwe
-from experiment.meteo_france_data.visualization.study_visualization.main_study_visualizer import \
+from experiment.meteo_france_data.scm_models_data.visualization.study_visualization.main_study_visualizer import \
study_iterator_global
+from experiment.meteo_france_data.scm_models_data.visualization.study_visualization.study_visualizer import \
+ StudyVisualizer
from experiment.trend_analysis.non_stationary_trends import \
ConditionalIndedendenceLocationTrendTest
-from experiment.meteo_france_data.visualization.study_visualization.study_visualizer import StudyVisualizer
from spatio_temporal_dataset.coordinates.transformed_coordinates.transformation.uniform_normalization import \
BetweenZeroAndOneNormalization, BetweenZeroAndOneNormalizationMinEpsilon, BetweenZeroAndOneNormalizationMaxEpsilon
from utils import get_display_name_from_object_type
--
GitLab