Eval.ipynb

  In [1]: 
 import pandas as pd
import matplotlib.pyplot as plt
import networkx as nx
import json,glob,re
from scipy.spatial.distance import jaccard
import numpy as np
from math import*

import matplotlib.pyplot as plt
import seaborn as sns
#%matplotlib inline
#from IPython.display import set_matplotlib_formats
#set_matplotlib_formats('pdf', 'svg')
#sns.set_palette(sns.color_palette("hls", 8))


import plotly.graph_objs as go
import plotly.offline as py
from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot
init_notebook_mode(connected=True)


def jaccard_similarity(x,y): 
    intersection_cardinality = len(set.intersection(*[set(x), set(y)]))
    union_cardinality = len(set.union(*[set(x), set(y)]))
    return intersection_cardinality/float(union_cardinality)
%cd ..
  Out [1]: 
 /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/requests/__init__.py:80: RequestsDependencyWarning: urllib3 (1.22) or chardet (2.3.0) doesn't match a supported version!
  RequestsDependencyWarning)
 
 Out [1]: 
 /Users/jacquesfize/nas_cloud/Code/str-python
 In [2]: 
 df=pd.read_csv("resources/results_graph_exp18fev.tsv",delimiter="\t",index_col=0)
new_df=pd.DataFrame(columns=df.columns)

selected_graph=json.load(open("data/graph_exp_fev_18/selected.json"))
types=df.type.unique()
graph_size={}
graphs_={}

files_glob= glob.glob("data/graph_exp_fev_18/normal/*.gexf")
for fn in files_glob:
    id_ = int(re.findall("\d+", fn)[-1])
    graphs_[id_]=nx.read_gexf(fn)
    graph_size[id_]=len(graphs_[id_])
graph_size[999]=0
nb_of_g_w_es_com={}
for g in graphs_:
    if not g in nb_of_g_w_es_com:
        nb_of_g_w_es_com[g]=0
    for g2 in graphs_:
        if not g2 == g:
            if set(graphs_[g].nodes()).intersection(set(graphs_[g2].nodes())):
                nb_of_g_w_es_com[g]+=1    

 
 In [3]: 
 df_mesure=pd.read_csv("resources/mesures.tsv",delimiter="\t")
rank_data=json.load(open("data/graph_exp_fev_18/rank.json"))
 
 In [4]: 
 n=3
df_copy=pd.DataFrame(columns=df.columns)
for t in types:
    mesures=df[df.type == t].mesure.unique()
    for m in mesures:
        data=df[(df.mesure == m) & (df.type == t)]
        for g in selected_graph:
            subset=data[data.id_g1 == g].iloc[:n]
            if len(subset)<1:#No graph found
                df_2=pd.DataFrame([[g,999,m,t,3,0,0,0,0]],columns=df.columns)
                for i in range(n):df_copy=df_copy.append(df_2)
            elif len(subset)
 
 In [5]: 
 df.head(20)
  Out [5]: 
  
 (Click to sort ascending) id_g1
 (Click to sort ascending) id_g2
 (Click to sort ascending) mesure
 (Click to sort ascending) type
 (Click to sort ascending) id_user
 (Click to sort ascending) c1_val
 (Click to sort ascending) c2_val
 (Click to sort ascending) c3_val
 (Click to sort ascending) c4_val
 (Click to sort ascending)
0 2.0 230.0 1.0 normal 3.0 1.0 1.0 1.0 1.0
1 2.0 275.0 1.0 normal 3.0 1.0 1.0 1.0 1.0
2 2.0 269.0 1.0 normal 3.0 1.0 1.0 1.0 1.0
3 7.0 437.0 1.0 normal 3.0 1.0 1.0 0.0 0.0
4 7.0 4.0 1.0 normal 3.0 1.0 1.0 0.0 0.0
5 7.0 371.0 1.0 normal 3.0 1.0 1.0 0.0 0.0
6 14.0 443.0 1.0 normal 3.0 1.0 1.0 1.0 1.0
7 14.0 437.0 1.0 normal 3.0 1.0 1.0 1.0 1.0
8 14.0 4.0 1.0 normal 3.0 1.0 1.0 1.0 1.0
9 27.0 26.0 1.0 normal 3.0 1.0 0.0 0.0 0.0
10 27.0 33.0 1.0 normal 3.0 1.0 1.0 0.0 0.0
11 27.0 40.0 1.0 normal 3.0 1.0 0.0 0.0 0.0
12 45.0 21.0 1.0 normal 3.0 1.0 1.0 0.0 1.0
13 45.0 23.0 1.0 normal 3.0 1.0 1.0 0.0 0.0
14 45.0 44.0 1.0 normal 3.0 1.0 1.0 0.0 0.0
15 47.0 81.0 1.0 normal 3.0 1.0 1.0 0.0 0.0
16 47.0 170.0 1.0 normal 3.0 1.0 1.0 0.0 0.0
17 47.0 488.0 1.0 normal 3.0 1.0 1.0 0.0 0.0
18 53.0 154.0 1.0 normal 3.0 1.0 1.0 0.0 0.0
19 53.0 512.0 1.0 normal 3.0 1.0 1.0 0.0 0.0
 In [6]: 
 df["g1_size"]=df["id_g1"].apply(lambda x:graph_size[int(x)])
df["g2_size"]=df["id_g2"].apply(lambda x:graph_size[int(x)])
df["mesureL"]=df["mesure"].apply(lambda x:df_mesure[df_mesure.id==x].values[0][-1])
df['c1*c2']=df.c1_val*df.c2_val
df['c1+c2']=(df.c1_val+df.c2_val).apply(lambda x:x if x<2 else 1)
df['c1*c2*c3']=df.c1_val*df.c2_val*df.c3_val
df['c1*c3']=df.c1_val*df.c3_val
df['c1+c3']=(df.c1_val+df.c3_val).apply(lambda x:x if x<2 else 1)
df['c2*c3']=df.c2_val*df.c3_val
df["for_c"]=df["id_g2"].apply(lambda x:1)
df["es_in_common"]=df["id_g1"].apply(lambda x:nb_of_g_w_es_com[x])
normal=df[df.type == "normal"]
gen_country=df[df.type == "gen_country"]
gen_region=df[df.type == "gen_region"]
extension_1=df[df.type == "extension_1"]
 
 In [9]: 
 new_df=pd.DataFrame(data=None,columns=df.columns)
for id,row in df.iterrows():
    ranks=set(rank_data[row.type][row.mesureL][str(int(row.id_g1))][:5])
    if int(row.id_g2) in ranks:
        new_df=new_df.append(row)
 
 In [10]: 
 df=new_df
 
 In [11]: 
 colorized_subset=['c1_val', 'c2_val', 'c3_val',
       'c4_val', 'c1*c2', 'c1+c2', 'c1*c2*c3', 'c1*c3', 'c1+c3', 'c2*c3']
 
 Raw Performance
 Normal
 In [12]: 
 score_per_mesure=normal.groupby(["id_g1","mesureL"], as_index=False).mean()
score_normal=score_per_mesure.groupby(["mesureL"],as_index=False).mean()
def highlight_max(s):
    '''
    highlight the maximum in a Series yellow.
    '''
    is_max = s == s.max()
    return ['background-color: yellow' if v else '' for v in is_max]
def highlight_min(s):
    '''
    highlight the maximum in a Series yellow.
    '''
    is_max = s == s.min()
    return ['background-color: red;color:white;' if v else '' for v in is_max]
score_normal.style.apply(highlight_max,subset=colorized_subset).apply(highlight_min,subset=colorized_subset)
  Out [12]: 
 
 In [14]: 
 score_normal[["c1_val","c2_val","c3_val","c4_val"]].plot.bar(x=score_normal["mesure"].unique())
plt.show()
score_normal[["c1*c2","c1+c2","c1*c2*c3","c1+c3"]].plot.bar(x=score_normal["mesure"].unique())
plt.show()
 
 Generalisation Pays
 In [16]: 
 score_per_mesure=gen_country.groupby(["id_g1","mesureL"], as_index=False).mean()
score_gen_country=score_per_mesure.groupby(["mesureL"],as_index=False).mean()
score_gen_country.style.apply(highlight_max,subset=colorized_subset).apply(highlight_min,subset=colorized_subset)
  Out [16]: 
 
 In [37]: 
 score_gen_country[["c1_val","c2_val","c3_val","c4_val"]].plot.bar(x=score_gen_country["mesure"].unique())
plt.show()
score_gen_country[["c1*c2","c1+c2","c1*c2*c3","c1+c3"]].plot.bar(x=score_gen_country["mesure"].unique())
plt.show()
 
 Generalisation region
 In [21]: 
 score_per_mesure=gen_region.groupby(["id_g1","mesureL"], as_index=False).mean()
score_gen_region=score_per_mesure.groupby(["mesureL"],as_index=False).mean()
score_gen_region.style.apply(highlight_max,subset=colorized_subset).apply(highlight_min,subset=colorized_subset)
  Out [21]: 
 
 In [39]: 
 score_gen_region[["c1_val","c2_val","c3_val","c4_val"]].plot.bar(x=score_gen_region["mesure"].unique())
plt.show()
score_gen_region[["c1*c2","c1+c2","c1*c2*c3","c1+c3"]].plot.bar(x=score_gen_region["mesure"].unique())
plt.show()
 
 Extension 1
 In [20]: 
 score_per_mesure=extension_1.groupby(["id_g1","mesureL"], as_index=False).mean()
score_ext_1=score_per_mesure.groupby(["mesureL"],as_index=False).mean()
score_ext_1.style.apply(highlight_max,subset=colorized_subset).apply(highlight_min,subset=colorized_subset)
  Out [20]: 
 
 In [41]: 
 score_ext_1[["c1_val","c2_val","c3_val","c4_val"]].plot.bar(x=score_ext_1["mesure"].unique())
plt.show()
score_ext_1[["c1*c2","c1+c2","c1*c2*c3","c1+c3"]].plot.bar(x=score_ext_1["mesure"].unique())
plt.show()
 
 Impact of the graph size over criteria
 In [42]: 
 df=df[df["id_g2"] != 999]
 
 In [43]: 
 x_label="g1_size"
 
 Score Gradient over size
 Selon la taille sans prendre en compte mesure
Normal
 In [44]: 
 score_normal_size=normal.groupby([x_label],as_index=False).mean()
score_normal_size.style.background_gradient(subset=colorized_subset,low=0,high=0)
  Out [44]: 
 
 Extension 1
 In [45]: 
 score_extension_1_size=extension_1.groupby([x_label],as_index=False).mean()
score_extension_1_size.style.background_gradient(subset=colorized_subset,low=0,high=0)
  Out [45]: 
 
 gen country
 In [46]: 
 score_gen_country_size=gen_country.groupby([x_label],as_index=False).mean()
score_gen_country_size.style.background_gradient(subset=colorized_subset,low=0,high=0)
  Out [46]: 
 
 Gen region
 In [47]: 
 score_gen_region_size=gen_region.groupby([x_label],as_index=False).mean()
score_gen_region_size.style.background_gradient(subset=colorized_subset,low=0,high=0)
  Out [47]: 
 
 Par taille et par mesure
 In [48]: 
 
score_normal_size=normal.groupby([x_label,"mesure"],as_index=False).mean()
score_normal_size.style.background_gradient(subset=colorized_subset,low=0,high=1)
  Out [48]: 
 
 In [49]: 
 score_gen_country_size=gen_country.groupby([x_label,"mesure"],as_index=False).mean()
score_gen_country_size.style.background_gradient(subset=colorized_subset,low=0,high=1)
  Out [49]: 
 
 In [50]: 
 score_gen_region_size=gen_region.groupby([x_label,"mesure"],as_index=False).mean()
score_gen_region_size.style.background_gradient(subset=colorized_subset,low=0,high=1)
  Out [50]: 
 
 In [51]: 
 score_extension_1_size=extension_1.groupby([x_label,"mesure"],as_index=False).mean()
score_extension_1_size.style.background_gradient(subset=colorized_subset,low=0,high=1)
  Out [51]: 
 
 In [52]: 
 key=["g1_size","mesure","c1_val","c2_val","c3_val","c4_val","c1*c2","c1+c2","c1*c2*c3","c1*c3","c1+c3","c2*c3"]
writer = pd.ExcelWriter('PerSizeAndPerMesurePerType.xlsx')
score_normal_size[key].to_excel(writer,'Normal')
score_gen_country_size[key].to_excel(writer,'GenCountry')
score_gen_region_size[key].to_excel(writer,'GenRegion')
score_extension_1_size[key].to_excel(writer,'Extension_1')
writer.save()
 
 Pareto Frontier with x = G1_size
 In [53]: 
 # Thanks to http://hinnefe2.github.io/python/tools/2015/09/21/mario-kart.html
def is_pareto_front(dataf,row, xlabel, ylabel):
    
    x = row[xlabel]
    y = row[ylabel]
    
    # look for points with the same y value but larger x value
    is_max_x = dataf.loc[dataf[ylabel]==y].max()[xlabel] <= x
    # look for points with the same x value but larger y value
    is_max_y = dataf.loc[dataf[xlabel]==x].max()[ylabel] <= y
    # look for points that are larger in both x and y
    is_double = len(dataf.loc[(dataf[xlabel]>x) & (dataf[ylabel]>y)])==0
    
    return is_max_x and is_max_y and is_double
 
 In [244]: 
 
import colorlover as cl
cm=sns.color_palette("hls", 8)
def draw_pareto_static(df,x_label,criteria,x_ax_label="X",y_ax_label="Y",title="Titre"):
    
    fig, ax = plt.subplots(figsize=(10,5),ncols=1)
    #axes=(ax1,ax2,ax3,ax4)
    for i in range(len(criteria)):
        y_label=criteria[i]
        #df.assign(normalized=df.bought.div(df.groupby('user').bought.transform('sum')))
        df_is_pareto = df.apply(lambda row: is_pareto_front(df,row, [x_label, y_label]), axis=1)
        df_pareto = df.ix[df_is_pareto].sort_values(by=x_label)

        
        sns.swarmplot(x=x_label, y=y_label, data=df, ax=ax,color=cm[i])
        # plot the pareto frontier
        ax.plot(df_pareto[x_label].index,df_pareto[y_label].values, '--',color=cm[i], label='P. Frontier for {0}'.format(criteria[i]))

        #ax.legend(loc='best')
        plt.xlabel(x_ax_label)
        plt.ylabel(y_ax_label)
        plt.xticks(rotation=90)
        plt.title(title)
        

def data_pareto(df,x_label,criteria):
    
    data=[]
    for i in range(len(criteria)):
        y_label=criteria[i]
        #df.assign(normalized=df.bought.div(df.groupby('user').bought.transform('sum')))
        df_is_pareto = df.apply(lambda row: is_pareto_front(df,row, [x_label, y_label]), axis=1)
        df_pareto = df.ix[df_is_pareto].sort_values(by=x_label)
        data.append(go.Scatter(
                x=df[x_label], # assign x as the dataframe column 'x'
                y=df[y_label],
                mode="markers",
                marker = dict(
                    color = ("rgb"+str(cm[i])),
                    ),
                name="{0} ".format(criteria[i]),
            ))

        data.append(
            go.Scatter(
                x=df_pareto[x_label], # assign x as the dataframe column 'x'
                y=df_pareto[y_label],
                name="{0} Pareto Frontier".format(criteria[i]),
                line = dict(
                    color = ("rgb"+str(cm[i])),
                    width = 4,)
                    )

        )
    return data

def draw_pareto2(df,x_label,criteria,typeSTR="normal",layout=None):
    if not layout:
        fig = go.Figure(data=data_pareto(df,x_label,criteria))
    else:
        fig = go.Figure(data=data_pareto(df,x_label,criteria), layout=layout)
    return iplot(fig,filename='{0}_{1}_{2}.png'.format(x_label,",".join(criteria),typeSTR))
 
 In [108]: 
 def get_pareto_graph_ids(df,criteria):
    set_=set([])
    for c in criteria:
        df_is_pareto = df.apply(lambda row: is_pareto_front(df,row, x_label, c), axis=1)
        df_pareto = df.ix[df_is_pareto]
        for i,row in df_pareto.iterrows():
            set_.add(int(row.id_g1))
    return list(set_)
 
 In [58]: 
 # Données nécessaire
x_label="id_g1"
extension_1_g1_mean=extension_1.groupby([x_label],as_index=False).mean()
normal_g1_mean=normal.groupby([x_label],as_index=False).mean()
gen_region_g1_mean=gen_region.groupby([x_label],as_index=False).mean()
gen_country_g1_mean=gen_country.groupby([x_label],as_index=False).mean()
keys_alone=['c1_val', 'c2_val', 'c3_val', 'c4_val']
keys_combined=['c1*c2', 'c1*c2*c3', 'c1*c3', 'c2*c3']

pareto_graphs={
        "normal":sorted(get_pareto_graph_ids(normal_g1_mean,keys_alone)),
        "ext_1":sorted(get_pareto_graph_ids(extension_1_g1_mean,keys_alone)),
        "gen_country":sorted(get_pareto_graph_ids(gen_country_g1_mean,keys_alone)),
        "gen_region":sorted(get_pareto_graph_ids(gen_region_g1_mean,keys_alone)),
}
        
df_data={}
for d in pareto_graphs:
    if not d in df_data:df_data[d]={}
    for d2 in pareto_graphs:
        df_data[d][d2]=jaccard_similarity(pareto_graphs[d],pareto_graphs[d2])
 
 Quelles graphes maximisent un critère ?
Normal
 In [69]: 
 draw_pareto2(normal_g1_mean,x_label,keys_alone,"normal")
  Out [69]: 
 
 Gen_region
 In [70]: 
 
draw_pareto2(gen_region_g1_mean,x_label,keys_alone,"gen_region")
  Out [70]: 
 
 Gen_country
 In [71]: 
 draw_pareto2(gen_country_g1_mean,x_label,keys_alone,"gen_country")
  Out [71]: 
 
 Extension_1
 In [72]: 
 draw_pareto2(extension_1_g1_mean,x_label,keys_alone,"extension")
  Out [72]: 
 
 Est que ces graphes partagent une ou plusieurs entités avec beaucoup de graphe ?
 In [337]: 
 normal_size_mean=normal.groupby(["id_g1"],as_index=False).mean()
keys_=['es_in_common']
draw_pareto2(normal_size_mean,[x_label,'es_in_common'],"normal")
  Out [337]: 
  ---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pandas/indexes/base.py in get_value(self, series, key)
   2174             try:
-> 2175                 return tslib.get_value_box(s, key)
   2176             except IndexError:

pandas/tslib.pyx in pandas.tslib.get_value_box (pandas/tslib.c:19053)()

pandas/tslib.pyx in pandas.tslib.get_value_box (pandas/tslib.c:18687)()

TypeError: 'str' object cannot be interpreted as an integer

During handling of the above exception, another exception occurred:

KeyError                                  Traceback (most recent call last)
<ipython-input-337-7cdd5e29f18a> in <module>()
      1 normal_size_mean=normal.groupby(["id_g1"],as_index=False).mean()
      2 keys_=['es_in_common']
----> 3 draw_pareto2(normal_size_mean,[x_label,'es_in_common'],"normal")

<ipython-input-244-85c76a567b54> in draw_pareto2(df, x_label, criteria, typeSTR, layout)
     57 def draw_pareto2(df,x_label,criteria,typeSTR="normal",layout=None):
     58     if not layout:
---> 59         fig = go.Figure(data=data_pareto(df,x_label,criteria))
     60     else:
     61         fig = go.Figure(data=data_pareto(df,x_label,criteria), layout=layout)

<ipython-input-244-85c76a567b54> in data_pareto(df, x_label, criteria)
     30         y_label=criteria[i]
     31         #df.assign(normalized=df.bought.div(df.groupby('user').bought.transform('sum')))
---> 32         df_is_pareto = df.apply(lambda row: is_pareto_front(df,row, [x_label, y_label]), axis=1)
     33         df_pareto = df.ix[df_is_pareto].sort_values(by=x_label)
     34         data.append(go.Scatter(

/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pandas/core/frame.py in apply(self, func, axis, broadcast, raw, reduce, args, **kwds)
   4150                     if reduce is None:
   4151                         reduce = True
-> 4152                     return self._apply_standard(f, axis, reduce=reduce)
   4153             else:
   4154                 return self._apply_broadcast(f, axis)

/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pandas/core/frame.py in _apply_standard(self, func, axis, ignore_failures, reduce)
   4246             try:
   4247                 for i, v in enumerate(series_gen):
-> 4248                     results[i] = func(v)
   4249                     keys.append(v.name)
   4250             except Exception as e:

<ipython-input-244-85c76a567b54> in <lambda>(row)
     30         y_label=criteria[i]
     31         #df.assign(normalized=df.bought.div(df.groupby('user').bought.transform('sum')))
---> 32         df_is_pareto = df.apply(lambda row: is_pareto_front(df,row, [x_label, y_label]), axis=1)
     33         df_pareto = df.ix[df_is_pareto].sort_values(by=x_label)
     34         data.append(go.Scatter(

<ipython-input-287-187d0a772f0a> in is_pareto_front(dataf, row, columns)
      2 def is_pareto_front(dataf,row,columns):
      3 
----> 4     values=[row[col] for col in columns]
      5     boolean_is_max=[]
      6     #if max(value)

<ipython-input-287-187d0a772f0a> in <listcomp>(.0)
      2 def is_pareto_front(dataf,row,columns):
      3 
----> 4     values=[row[col] for col in columns]
      5     boolean_is_max=[]
      6     #if max(value)

/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pandas/core/series.py in __getitem__(self, key)
    601         key = com._apply_if_callable(key, self)
    602         try:
--> 603             result = self.index.get_value(self, key)
    604 
    605             if not is_scalar(result):

/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pandas/indexes/base.py in get_value(self, series, key)
   2181                     raise InvalidIndexError(key)
   2182                 else:
-> 2183                     raise e1
   2184             except Exception:  # pragma: no cover
   2185                 raise e1

/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pandas/indexes/base.py in get_value(self, series, key)
   2167         try:
   2168             return self._engine.get_value(s, k,
-> 2169                                           tz=getattr(series.dtype, 'tz', None))
   2170         except KeyError as e1:
   2171             if len(self) > 0 and self.inferred_type in ['integer', 'boolean']:

pandas/index.pyx in pandas.index.IndexEngine.get_value (pandas/index.c:3557)()

pandas/index.pyx in pandas.index.IndexEngine.get_value (pandas/index.c:3240)()

pandas/index.pyx in pandas.index.IndexEngine.get_loc (pandas/index.c:4279)()

pandas/src/hashtable_class_helper.pxi in pandas.hashtable.PyObjectHashTable.get_item (pandas/hashtable.c:13742)()

pandas/src/hashtable_class_helper.pxi in pandas.hashtable.PyObjectHashTable.get_item (pandas/hashtable.c:13696)()

KeyError: ('n', 'occurred at index 0')

 In [76]: 
 pareto_graphs
  Out [76]: 
 {'ext_1': [14, 236, 304, 346, 359, 429, 476, 499, 503, 527],
 'gen_country': [92, 236, 300, 375, 426, 439, 453, 499, 503, 527],
 'gen_region': [14, 92, 211, 236, 304, 346, 359, 429, 439, 499, 527],
 'normal': [14, 92, 236, 346, 359, 424, 426, 429, 476, 499, 527]}
 In [77]: 
 pd.DataFrame(df_data,columns=df_data.keys())
  Out [77]: 
  
 (Click to sort ascending) normal
 (Click to sort ascending) ext_1
 (Click to sort ascending) gen_country
 (Click to sort ascending) gen_region
 (Click to sort ascending)
0 0.615385 1.000000 0.2500 0.615385
1 0.312500 0.250000 1.0000 0.312500
2 0.571429 0.615385 0.3125 1.000000
3 1.000000 0.615385 0.3125 0.571429
 GenCountry apporte le plus de changement dans les graphes "optimum", i.e., graphes validant un critère sur les différentes mesures.
 In [140]: 
 # Données nécessaire
x_label="mesure"
dataf=df.groupby(['type',x_label],as_index=False).mean()
dataf['fusion'] = dataf[['type',x_label]].apply(lambda x: ''.join(x),axis=1)
#normal_g1_mean=normal.groupby(['type',x_label],as_index=False).mean()
#gen_region_g1_mean=gen_region.groupby([x_label],as_index=False).mean()
#gen_country_g1_mean=gen_country.groupby([x_label],as_index=False).mean()
keys_alone=['c1_val', 'c2_val', 'c3_val', 'c4_val']
keys_combined=['c1*c2', 'c1*c2*c3', 'c1*c3', 'c2*c3']

 
 In [144]: 
 dataf
  Out [144]: 
  
 (Click to sort ascending) type
 (Click to sort ascending) mesure
 (Click to sort ascending) id_g1
 (Click to sort ascending) id_g2
 (Click to sort ascending) id_user
 (Click to sort ascending) c1_val
 (Click to sort ascending) c2_val
 (Click to sort ascending) c3_val
 (Click to sort ascending) c4_val
 (Click to sort ascending) g1_size
 (Click to sort ascending) g2_size
 (Click to sort ascending) c1*c2
 (Click to sort ascending) c1+c2
 (Click to sort ascending) c1*c2*c3
 (Click to sort ascending) c1*c3
 (Click to sort ascending) c1+c3
 (Click to sort ascending) c2*c3
 (Click to sort ascending) for_c
 (Click to sort ascending) es_in_common
 (Click to sort ascending) fusion
 (Click to sort ascending)
0 extension_1 BOC 241.380000 252.433333 3.0 0.920000 0.720000 0.440000 0.400000 4.920000 3.840000 0.693333 0.946667 0.413333 0.433333 0.926667 0.420000 1.0 70.320000 extension_1BOC
1 extension_1 BOWSE 241.380000 252.866667 3.0 0.926667 0.780000 0.460000 0.440000 4.920000 4.066667 0.753333 0.953333 0.420000 0.453333 0.933333 0.426667 1.0 70.320000 extension_1BOWSE
2 extension_1 JACCARD 241.380000 288.740000 3.0 0.473333 0.480000 0.260000 0.246667 4.920000 4.986667 0.466667 0.486667 0.260000 0.260000 0.473333 0.260000 1.0 70.320000 extension_1JACCARD
3 extension_1 MCS 241.380000 270.100000 3.0 0.926667 0.833333 0.493333 0.440000 4.920000 4.760000 0.813333 0.946667 0.453333 0.486667 0.933333 0.460000 1.0 70.320000 extension_1MCS
4 extension_1 VEO 241.380000 255.640000 3.0 0.920000 0.820000 0.486667 0.453333 4.920000 4.533333 0.800000 0.940000 0.446667 0.480000 0.926667 0.453333 1.0 70.320000 extension_1VEO
5 extension_1 WLSUBTREE 241.380000 270.600000 3.0 0.786667 0.706667 0.413333 0.280000 4.920000 4.840000 0.673333 0.820000 0.386667 0.400000 0.800000 0.400000 1.0 70.320000 extension_1WLSUBTREE
6 gen_country BOC 237.653061 253.938776 3.0 0.727891 0.884354 0.489796 0.367347 5.000000 4.530612 0.680272 0.931973 0.414966 0.428571 0.789116 0.476190 1.0 70.000000 gen_countryBOC
7 gen_country BOWSE 241.489796 249.619048 3.0 0.768707 0.911565 0.530612 0.394558 4.979592 4.591837 0.714286 0.965986 0.442177 0.469388 0.829932 0.503401 1.0 70.224490 gen_countryBOWSE
8 gen_country GREEDY 236.604167 204.062500 3.0 0.548611 0.652778 0.354167 0.256944 5.062500 4.111111 0.451389 0.750000 0.305556 0.319444 0.583333 0.340278 1.0 69.854167 gen_countryGREEDY
9 gen_country HED 242.325581 264.108527 3.0 0.829457 0.875969 0.519380 0.403101 5.325581 3.914729 0.744186 0.961240 0.449612 0.465116 0.883721 0.503876 1.0 78.488372 gen_countryHED
10 gen_country JACCARD 241.380000 308.413333 3.0 0.440000 0.480000 0.220000 0.193333 4.920000 5.373333 0.440000 0.480000 0.213333 0.213333 0.446667 0.220000 1.0 70.320000 gen_countryJACCARD
11 gen_country MCS 241.380000 256.973333 3.0 0.793333 0.926667 0.506667 0.420000 4.920000 5.280000 0.746667 0.973333 0.453333 0.473333 0.826667 0.486667 1.0 70.320000 gen_countryMCS
12 gen_country VEO 241.380000 258.473333 3.0 0.820000 0.926667 0.533333 0.440000 4.920000 5.100000 0.773333 0.973333 0.466667 0.486667 0.866667 0.513333 1.0 70.320000 gen_countryVEO
13 gen_region BOC 241.380000 255.513333 3.0 0.920000 0.773333 0.466667 0.406667 4.920000 4.140000 0.733333 0.960000 0.440000 0.460000 0.926667 0.446667 1.0 70.320000 gen_regionBOC
14 gen_region BOWSE 241.380000 259.640000 3.0 0.926667 0.833333 0.513333 0.433333 4.920000 4.386667 0.800000 0.960000 0.473333 0.506667 0.933333 0.480000 1.0 70.320000 gen_regionBOWSE
15 gen_region GREEDY 241.380000 184.693333 3.0 0.520000 0.500000 0.293333 0.240000 4.920000 3.573333 0.446667 0.573333 0.280000 0.280000 0.533333 0.293333 1.0 70.320000 gen_regionGREEDY
16 gen_region HED 221.592593 254.888889 3.0 1.000000 0.753086 0.370370 0.283951 7.407407 3.123457 0.753086 1.000000 0.370370 0.370370 1.000000 0.370370 1.0 111.259259 gen_regionHED
17 gen_region JACCARD 241.380000 288.466667 3.0 0.533333 0.526667 0.280000 0.273333 4.920000 5.180000 0.513333 0.546667 0.280000 0.280000 0.533333 0.280000 1.0 70.320000 gen_regionJACCARD
18 gen_region MCS 241.380000 266.933333 3.0 0.920000 0.873333 0.520000 0.446667 4.920000 4.780000 0.846667 0.946667 0.486667 0.513333 0.926667 0.493333 1.0 70.320000 gen_regionMCS
19 gen_region VEO 241.380000 259.226667 3.0 0.926667 0.840000 0.506667 0.440000 4.920000 4.660000 0.820000 0.946667 0.466667 0.500000 0.933333 0.473333 1.0 70.320000 gen_regionVEO
20 normal BOC 241.380000 267.100000 3.0 0.926667 0.746667 0.460000 0.406667 4.920000 4.100000 0.720000 0.953333 0.433333 0.453333 0.933333 0.440000 1.0 70.320000 normalBOC
21 normal BOWSE 244.387755 265.387755 3.0 0.925170 0.809524 0.476190 0.435374 4.877551 4.136054 0.782313 0.952381 0.435374 0.469388 0.931973 0.442177 1.0 71.306122 normalBOWSE
22 normal GED 221.088235 273.127451 3.0 0.950980 0.784314 0.372549 0.343137 6.294118 3.372549 0.774510 0.960784 0.343137 0.372549 0.950980 0.343137 1.0 94.823529 normalGED
23 normal GREEDY 237.346939 180.870748 3.0 0.455782 0.380952 0.265306 0.224490 4.938776 3.897959 0.367347 0.469388 0.265306 0.265306 0.455782 0.265306 1.0 71.326531 normalGREEDY
24 normal HED 219.700000 246.266667 3.0 0.988889 0.744444 0.355556 0.344444 6.933333 3.033333 0.744444 0.988889 0.355556 0.355556 0.988889 0.355556 1.0 103.133333 normalHED
25 normal JACCARD 241.380000 290.133333 3.0 0.466667 0.473333 0.266667 0.260000 4.920000 5.193333 0.460000 0.480000 0.266667 0.266667 0.466667 0.266667 1.0 70.320000 normalJACCARD
26 normal MCS 240.102041 265.972789 3.0 0.925170 0.850340 0.489796 0.428571 4.979592 4.993197 0.836735 0.938776 0.469388 0.482993 0.931973 0.476190 1.0 71.448980 normalMCS
 Quelles couples mesures/types maximisent un critère ?
 In [197]: 
 layout = go.Layout(
    xaxis=dict(
        tickangle=45
    ),
    margin = go.Margin(b = 160)
)
draw_pareto2(dataf,"fusion",keys_alone,"normal",layout)
  Out [197]: 
 
 Quelles couples mesures/types maximisent des critères combinées?
 In [81]: 
 layout = go.Layout(
    xaxis=dict(
        tickangle=45
    ),
    margin = go.Margin(b = 160)
)
draw_pareto2(dataf,"fusion",keys_combined,"normal",layout)
  Out [81]: 
 
 In [287]: 
 # Thanks to http://hinnefe2.github.io/python/tools/2015/09/21/mario-kart.html
def is_pareto_front(dataf,row,columns):
    
    values=[row[col] for col in columns]
    boolean_is_max=[]
    #if max(value)
    for c in range(len(columns)):
        val=values[c]
        col=columns[c]
        bool_temp=True
        for c2 in range(len(columns)):
            if c != c2:break
            val2=values[c]
            col2=columns[c]
            bool_temp=bool_temp and (dataf.loc[dataf[col2]==val2].max()[col] <= val)
        boolean_is_max.append(bool_temp)
    #if no criteria superior
    daf=dataf.copy()
    for c in range(len(columns)):
        val=values[c]
        col=columns[c]
        daf=daf.loc[(dataf[col]>val)]
    return sum(map(int,boolean_is_max))==len(columns) and len(daf)==0
 
 
 In [294]: 
 df_is_pareto = d_pc.apply(lambda row: is_pareto_front(d_pc,row, ['id_g1','c1_val','c2_val' ]), axis=1)
df_pareto = d_pc.ix[df_is_pareto].sort_values(by="id_g1")
 
 def get_pareto_couples(df,criteria):
    data={}
    for c in criteria:
        set_=[]
        df_is_pareto = df.apply(lambda row: is_pareto_front(df,row, "fusion", c), axis=1)
        df_pareto = df.ix[df_is_pareto].sort_values(by=x_label)
        for i,row in df_pareto.iterrows():
            set_.append(row.fusion)
        data[c]=set_
    return data


df_data={}
par=get_pareto_couples(dataf,keys_alone)
for d in par:
    if not d in df_data:df_data[d]={}
    for d2 in par:
        df_data[d][d2]=jaccard_similarity(par[d],par[d2])
pd.DataFrame(df_data,columns=df_data.keys())
 
 In [202]: 
 par
  Out [202]: 
 {'c1_val': ['gen_regionHED', 'normalHED', 'normalMCS'],
 'c2_val': ['gen_regionMCS', 'normalMCS', 'gen_countryVEO'],
 'c3_val': ['gen_regionMCS', 'normalMCS', 'gen_countryVEO', 'gen_regionVEO'],
 'c4_val': ['normalBOWSE',
  'gen_regionMCS',
  'normalMCS',
  'extension_1VEO',
  'gen_regionVEO']}
 Pour l'instant, c'est le couple VEO et gen_region qui obtient de meilleur score
 In [204]: 
 df_data={}
par=get_pareto_couples(dataf,keys_combined)
for d in par:
    if not d in df_data:df_data[d]={}
    for d2 in par:
        df_data[d][d2]=jaccard_similarity(par[d],par[d2])
pd.DataFrame(df_data,columns=df_data.keys())
  Out [204]: 
  
 (Click to sort ascending) c1*c2
 (Click to sort ascending) c1*c2*c3
 (Click to sort ascending) c1*c3
 (Click to sort ascending) c2*c3
 (Click to sort ascending)
0 1.000000 1.000000 0.666667 0.666667
1 1.000000 1.000000 0.666667 0.666667
2 0.666667 0.666667 1.000000 0.500000
3 0.666667 0.666667 0.500000 1.000000
 In [205]: 
 par
  Out [205]: 
 {'c1*c2': ['gen_regionMCS', 'normalMCS'],
 'c1*c2*c3': ['gen_regionMCS', 'normalMCS'],
 'c1*c3': ['gen_regionMCS', 'normalMCS', 'gen_regionVEO'],
 'c2*c3': ['gen_regionMCS', 'normalMCS', 'gen_countryVEO']}
 In [293]: 
 d_pc=df.groupby(['id_g1'],as_index=False).mean()[['id_g1','c1_val','c2_val','c3_val','c4_val']]
d_pc['id_g1']=d_pc['id_g1'].apply(lambda x:int(x))
 
 # Thanks to http://hinnefe2.github.io/python/tools/2015/09/21/mario-kart.html
def is_pareto_front(dataf,row, xlabel, ylabel):
    
    x = row[xlabel]
    y = row[ylabel]
    
    # look for points with the same y value but larger x value
    is_max_x = dataf.loc[dataf[ylabel]==y].max()[xlabel] <= x
    # look for points with the same x value but larger y value
    is_max_y = dataf.loc[dataf[xlabel]==x].max()[ylabel] <= y
    # look for points that are larger in both x and y
    is_double = len(dataf.loc[(dataf[xlabel]>x) & (dataf[ylabel]>y)])==0
    
    return is_max_x and is_max_y and is_double