diff --git a/elasticsearch/src/eda_templates/filter_by_state_and_date.j2 b/elasticsearch/src/eda_templates/filter_by_state_and_date.j2 index e5abf50c1dd2bdd05e62e3b94dbef75880d58221..7cbf771e50481f5740f8819cb92b4575996e66f3 100644 --- a/elasticsearch/src/eda_templates/filter_by_state_and_date.j2 +++ b/elasticsearch/src/eda_templates/filter_by_state_and_date.j2 @@ -6,8 +6,14 @@ }, "text", "id", + "in_reply_to_status_id", "extended_tweet.full_text", "retweeted_status.id", + "user.location", + "entities.media.expanded_url", + "entities.urls.expanded_url", + "user.name", + "retweeted_status.user.name", "rest.*" ], "_source": false, diff --git a/elasticsearch/src/tf-idf-es.py b/elasticsearch/src/tf-idf-es.py index bdf1dc2541ccae1f69beac893def41aa5b6e8700..4b077bd0628aa322db6a141de2bf4bbab940d1e4 100644 --- a/elasticsearch/src/tf-idf-es.py +++ b/elasticsearch/src/tf-idf-es.py @@ -137,3 +137,26 @@ if __name__ == '__main__': df_tfidf["tf_idf_terms"] = df_tfidf.index df_tfidf = df_tfidf.merge(df_results, on="index_of_tweet") df_tfidf.to_csv("/home/rdecoupe/Téléchargements/acquitaine3.csv") + # prepare to Gephi for graph vizu + # gephi = df_results + # gephi["Source"] = gephi["id"] + # gephi["Target"] = gephi["retweeted_status.id"] + # gephi["ID"] = gephi.index + # gephi["Timestamp"] = gephi["@timestamp"] + # gephi["label"] = gephi["retweeted_status.user.name"] + # gephi = gephi[gephi["Target"].str.len() !=0] # filter out tweet that are not retweeted + # gephi[["ID", "label", "Source", "Target", "Timestamp"]].to_csv( + # "/home/rdecoupe/Téléchargements/acquitaine_script_gephi.csv", + # index = False + # ) + gephi = df_results + gephi["Source"] = gephi["user.name"] + gephi["Target"] = gephi["retweeted_status.user.name"] + gephi["ID"] = gephi.index + gephi["Timestamp"] = gephi["@timestamp"] + gephi["label"] = gephi["retweeted_status.user.name"] + gephi = gephi[gephi["Target"].str.len() !=0] # filter out tweet that are not retweeted + gephi[["ID", "label", "Source", "Target", "Timestamp"]].to_csv( + "/home/rdecoupe/Téléchargements/acquitaine_script_gephi.csv", + index=False + )