fixes for full reallocated gathering

scenariosAleatoires/MCMC.py

@@ -27,7 +27,7 @@ class MCMC:
             self.rng = np.random.RandomState(42)
             print('MCMC initialized with default seed') # self.rng.get_state()
         # Copying input data in output dir
-        self.outputdir = self.mcmc_config['output_dir'] + '/' + time.strftime('%Y%m%d-%H%M%S')
+        self.outputdir = self.mcmc_config['output_dir'] + '/' + time.strftime('%Y%m%d-%H%M%S') + '_{initial_nb_particles}_{step_nb_particles}_{ratio_patches_to_modify}'.format(**self.mcmc_config)
         if output:
             if not os.path.exists(self.outputdir):
                 os.makedirs(self.outputdir)
@@ -100,16 +100,14 @@ class MCMC:
         '''
         # Loop of sampling and scoring
         start_time = time.time()
-        data={'scenarios_patches':[], 'scenarios_cult':[], 'previous_indexes':[]}
+        data={'scenarios_patches':[], 'scenarios_cult':[], 'previous_indexes':[], 'fullreallocated':[]}
         for index, scenario in tqdm(scenarios.sample(nb_particles, self.rng).iterrows(), total=nb_particles, disable=disable_progress):
-            scenario = scenarios.reallocate(index)
-            if scenario is None:
-                scenarios.setFullReallocated(index, True)
-            else:
-                data['scenarios_patches'].append(scenario)
-                data['scenarios_cult'].append(scenario['cultgeopat'])
-                data['previous_indexes'].append(index)
-        scenarios.reconstitute(iter_nb, **data)
+            [fullreallocated,scenario] = scenarios.reallocate(index)
+            data['fullreallocated'].append(fullreallocated)
+            data['scenarios_patches'].append(scenario)
+            data['scenarios_cult'].append(scenario['cultgeopat'])
+            data['previous_indexes'].append(index)
+        scenarios.reconstitute(iter_nb, **data, disable_progress=disable_progress)
         elapsed_time = time.time() - start_time
         print('Iteration duration: {} - {:.2f} scen/s '.format(time.strftime("%M:%S", time.gmtime(elapsed_time)), nb_particles/elapsed_time), end="", flush=True)
         # Computing pareto front
@@ -117,7 +115,7 @@ class MCMC:
         pareto_mask = MCMC.is_pareto_efficient(scenarios.scores[['Resilience','Proximity','Productivity','Biodiversity','Social','TargetDelta']].values)
         scenarios.scores['pareto'] = pareto_mask
         elapsed_time = time.time() - start_time
-        print(' Fully reallocated scenarios {}/{} - '.format(scenarios.nbFullReallocated(), len(scenarios.scores)), end="", flush=True)
+        print(' Fully reallocated scenarios {}/{} - '.format(scenarios.nbFullReallocated(), nb_particles), end="", flush=True)
         if (len(scenarios.scores)>0):
             print('{:.2f}% on pareto front (computed in {}) '.format(100*np.sum(pareto_mask)/len(pareto_mask), time.strftime("%M:%S", time.gmtime(elapsed_time))), end="", flush=True)
         # Writing output data
@@ -136,7 +134,7 @@ class MCMC:
                     )
                 pairplot.savefig(self.outputdir+'/mcmc_iter_{0:03d}.png'.format(iter_nb))
                 plt.close(pairplot.fig)
-            except (np.linalg.linalg.LinAlgError, ValueError, AttributeError, UnboundLocalError) as e:
+            except:
                 # when data doesn't vary enough on a dimension, it is impossible to generate the pareto density
                 # and an error is triggered. Here, we ignore this error.
                 pass

scenariosAleatoires/scenarios.py

@@ -15,6 +15,13 @@ class Reallocator:
         self.PAT_cult_to_decrease = surfDelta[surfDelta<0].sort_values(ascending=True).keys().tolist()
         self.PAT_cult_to_increase = surfDelta[surfDelta>0].sort_values(ascending=False).keys().tolist()
 
+    def is_fullreallocated(self, patches):
+        nbPatches = int(len(patches)*self.ratioNbPatches)
+        surfDelta = self.targetPAT - patches.groupby('cultgeopat')['SURF_PARC'].sum()
+        cult_to_decrease = surfDelta.loc[self.PAT_cult_to_decrease][surfDelta<0].sort_values(ascending=True).keys().tolist()
+        cult_to_increase = surfDelta.loc[self.PAT_cult_to_increase][surfDelta>0].sort_values(ascending=True).keys().tolist()
+        return len(cult_to_increase)==0 or len(cult_to_decrease)==0
+
     def reallocate(self, patches):
         nbPatches = int(len(patches)*self.ratioNbPatches)
         surfDelta = self.targetPAT - patches.groupby('cultgeopat')['SURF_PARC'].sum()
@@ -35,7 +42,10 @@ class Reallocator:
         newCult = newCult.sample(frac=1, random_state=self.rng)[:len(samples)].reset_index(drop=True) # shuffle and cut extra elements
         # Doing the reallocation
         patches.loc[samples.index.values,'cultgeopat'] = newCult.values
-        return patches
+        surfDelta = self.targetPAT - patches.groupby('cultgeopat')['SURF_PARC'].sum()
+        cult_to_decrease = surfDelta.loc[self.PAT_cult_to_decrease][surfDelta<0].sort_values(ascending=True).keys().tolist()
+        cult_to_increase = surfDelta.loc[self.PAT_cult_to_increase][surfDelta>0].sort_values(ascending=True).keys().tolist()
+        return [len(cult_to_increase)==0 or len(cult_to_decrease)==0, patches]
 
 class ScenariosStack:
 
@@ -48,13 +58,17 @@ class ScenariosStack:
         self.cultgeopat = initial_patches['cultgeopat'].to_frame().T
         # Now we have the id_parcel as columns, and each row will represent a scenario
         self.cultgeopat.rename(index={'cultgeopat':scenario_id}, inplace=True)
-        self.initscores([self.indicators.compute_indicators(initial_patches) + [0, 0, -1, False, False]], [scenario_id])
+        self.initscores([self.indicators.compute_indicators(initial_patches) + [0, 0, -1, False, False]], [scenario_id], constructor=True)
 
-    def initscores(self, scores, indexes):
-        self.scores = pd.DataFrame(scores,
+    def initscores(self, scores, indexes, constructor=False):
+        new_scores = pd.DataFrame(scores,
             columns=self.indicators.indicators_names+['iteration','cumulated_it', 'previous_index', 'pareto','full_reallocated'],
             index=indexes,
             dtype='float64')
+        if constructor:
+            self.scores = new_scores
+        else:
+            self.scores = pd.concat([self.scores[self.scores['full_reallocated']==True], new_scores], sort=False)
         for c in ['pareto','full_reallocated']:
             self.scores[c] = self.scores[c].astype('bool')
         for c in ['iteration','cumulated_it', 'previous_index']:
@@ -70,13 +84,15 @@ class ScenariosStack:
             self.scores[c] = self.scores[c].astype('int')
         return id
 
-    def reconstitute(self, iter_nb, scenarios_patches, scenarios_cult, previous_indexes, disable_progress=False):
+    def reconstitute(self, iter_nb, scenarios_patches, scenarios_cult, previous_indexes, fullreallocated, disable_progress=False):
         indexes = [next(self._counter) for id in range(len(scenarios_cult))]
-        self.cultgeopat = pd.DataFrame(scenarios_cult, index=indexes, columns=self.cultgeopat.columns, dtype='int8')
+        self.cultgeopat =  pd.concat([self.cultgeopat[self.scores['full_reallocated']==True],
+            pd.DataFrame(scenarios_cult, index=indexes, columns=self.cultgeopat.columns, dtype='int8')
+            ], sort=False)
         list_scores = []
         for i, previous_index in tqdm(enumerate(previous_indexes), total=len(previous_indexes), disable=disable_progress):
             scenario_patch = scenarios_patches[i]
-            list_scores.append(self.indicators.compute_indicators(scenario_patch) + [iter_nb, self.scores.loc[previous_index]['cumulated_it']+1, previous_index, False, False])
+            list_scores.append(self.indicators.compute_indicators(scenario_patch) + [iter_nb, self.scores.loc[previous_index]['cumulated_it']+1, previous_index, False, fullreallocated[i]])
         self.initscores(list_scores, indexes)
 
     def retain(self, mask):