Write (optionnaly) shapes of modified patches at the last iteration.

ID for each scenario added.

scenariosAleatoires/MCMC.py

@@ -17,10 +17,13 @@ from patutils import md5sum, load_pat_patches
 import seaborn as sns
 import multiprocessing as mp
 from functools import partial
+import itertools
 
 class Scenario:
+    _counter = itertools.count()
     def __init__(self, patches):
         self.patches = patches
+        self.id = next(Scenario._counter)
 
     def load_shp(shpfilename):
         '''
@@ -95,6 +98,7 @@ class MCMC:
         	sys.exit(1)
         self.mcmc_config = yaml.load(open(mcmc_config_filename,'r'))
         self.multiprocessing = self.mcmc_config.get('multiprocessing', False)
+        self.write_shp = self.mcmc_config.get('write_shp', False)
         self.patches_md5sum = md5sum(self.mcmc_config['patches'])
         if 'rng_seed' in self.mcmc_config:
             self.rng = np.random.RandomState(self.mcmc_config['rng_seed'])
@@ -165,7 +169,7 @@ class MCMC:
     def scenario_scores(patches, indicators):
         return indicators.compute_indicators(patches)
 
-    def step(self, iter_nb, nb_particles, scenarios, scores=None):
+    def step(self, iter_nb, nb_particles, scenarios, scores=None, write_shp=False):
         '''
         Sample new scenarios, evaluate their scores and retain only pareto front.
 
@@ -176,30 +180,32 @@ class MCMC:
         if given scenario are only used for sampling and will not be considered for pareto front.
         '''
         new_scenarios = []
+        new_scenarios_id = []
         new_scores = []
         # Loop of sampling and scoring
         for index, patches in tqdm(scenarios.sample(nb_particles, replace=True, random_state=self.rng).iterrows(), total=nb_particles):
             scenario = Scenario(patches[0].copy())
             scenario.reallocate(self.rng, self.targetPAT, self.mcmc_config['ratio_patches_to_modify']) # 3.8 ms
             new_scenarios.append(scenario.patches)
+            new_scenarios_id.append(scenario.id)
             if not self.multiprocessing:
                 new_scores.append(self.indicators.compute_indicators(scenario.patches))
         if self.multiprocessing:
-            new_scores = self.pool.map(partial(MCMC.scenario_scores, indicators=self.indicators), new_scenarios)
+            new_scores = list(tqdm(self.pool.imap(partial(MCMC.scenario_scores, indicators=self.indicators), new_scenarios, chunksize=50), total=len(new_scenarios)))
         # merging with precedent data
-        new_scores = pd.DataFrame(new_scores, columns=self.indicators.indicators_names)
+        new_scores = pd.DataFrame(new_scores, index=new_scenarios_id, columns=self.indicators.indicators_names)
         new_scores['iteration'] = iter_nb
         if scores is None:
             scores = new_scores
-            scenarios = pd.DataFrame(new_scenarios)
+            scenarios = pd.DataFrame(new_scenarios, index=new_scenarios_id)
         else:
-            scores = scores.append(new_scores, ignore_index=True, sort=False)
-            scenarios = scenarios.append(pd.DataFrame(new_scenarios), ignore_index=True, sort=False)
+            scores = scores.append(new_scores, sort=False)
+            scenarios = scenarios.append(pd.DataFrame(new_scenarios, index=new_scenarios_id), sort=False)
         # Computing pareto front
         pareto_mask = MCMC.is_pareto_efficient(scores[['Resilience','Proximity','Productivity','Biodiversity','Social']].values)
         scores['pareto'] = pareto_mask
         # Writing output data
-        scores.to_csv(self.outputdir+'/mcmc_iter_{}.csv'.format(iter_nb), index=False)
+        scores.to_csv(self.outputdir+'/mcmc_iter_{}.csv'.format(iter_nb), index=True)
         try:
             sns.pairplot(scores, vars=['Resilience','Proximity','Productivity','Biodiversity','Social'],
                     diag_kind="kde",hue="pareto"
@@ -211,11 +217,12 @@ class MCMC:
         scenarios = scenarios[pareto_mask]
         scores = scores[pareto_mask]
         # Writing shapefiles
-        shp_dir = self.outputdir + '/patches_iter_{}'.format(iter_nb)
-        os.makedirs(shp_dir)
-        for index,scenario in scenarios.iterrows():
-            scenario = scenario[0]
-            scenario[scenario['cultmodified']].drop('cultmodified', axis=1).to_file(shp_dir+'/{}_{}'.format(iter_nb,index), encoding='utf-8')
+        if write_shp:
+            shp_dir = self.outputdir + '/patches_iter_{}'.format(iter_nb)
+            os.makedirs(shp_dir)
+            for index,scenario in scenarios.iterrows():
+                scenario = scenario[0]
+                scenario[scenario['cultmodified']].drop('cultmodified', axis=1).to_file(shp_dir+'/{}_{}'.format(iter_nb,index), encoding='utf-8')
         return [scenarios, scores]
 
     def run(self, nb_processes=mp.cpu_count()):
@@ -226,9 +233,10 @@ class MCMC:
         scenarios_init = pd.DataFrame([self.patches])
         scenarios, scores = self.step(0, nb_particles, scenarios_init)
         # Iteration
-        for i in range(10):
+        for i in range(self.mcmc_config['max_iterations']):
             print('Iteration #{}'.format(i))
-            scenarios, scores = self.step(i+1, nb_particles, scenarios, scores)
+            # Write SHP only for the last iteration
+            scenarios, scores = self.step(i+1, nb_particles, scenarios, scores, i == self.mcmc_config['max_iterations']-1)
         # TODO
         # Storing variation of indicators
         init_var = scores.std()

scenariosAleatoires/MCMC_config.yml

 patches: ../output/PAT_patches/PAT_patches.shp
 target: ../resources/targetPAT.csv
 output_dir: ../output/MCMC
-initial_nb_particles: 10000
+write_shp: True
+initial_nb_particles: 1000
+max_iterations: 10
 multiprocessing: False
 ratio_patches_to_modify: 0.01
 indicators_config: