From 8815dccb2d945d4d7b6eb408e7ff8837ee22d6ea Mon Sep 17 00:00:00 2001 From: Fize Jacques <jacques.fize@cirad.fr> Date: Fri, 28 Jun 2019 15:02:07 +0200 Subject: [PATCH] debug --- strpython/pipeline.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/strpython/pipeline.py b/strpython/pipeline.py index a25a66a..640ca80 100644 --- a/strpython/pipeline.py +++ b/strpython/pipeline.py @@ -41,7 +41,7 @@ class Pipeline(object): """ self.lang=lang[:2] self.ner = kwargs["ner"] if "ner" in kwargs else Spacy(lang=lang[:2]) - self.disambiguator=kwargs["disambiguator"] if "disambiguator" in kwargs else MostCommonDisambiguator() + self.disambiguator=kwargs["disambiguator"] if "disambiguator" in kwargs else WikipediaDisambiguator() self.corpus_name = kwargs["corpus_name"] if "corpus_name" in kwargs else "no_name" self.no_name = False @@ -52,7 +52,7 @@ class Pipeline(object): self.verbose = kwargs.get("verbose",False) - def parse(self,text,debug=False): + def parse(self,text,debug=False,stop_words=[]): """ :param text: @@ -65,7 +65,7 @@ class Pipeline(object): # Disambiguation se_identified = self.disambiguator.disambiguate(self.lang,ner_output=output) for top_, id in list(se_identified.items()): - if not id.startswith("GD"): + if not id.startswith("GD") or top_.lower() in stop_words: del se_identified[top_] if debug: print(se_identified) @@ -124,6 +124,8 @@ class Pipeline(object): def pipe_build(self,texts, cpu_count=cpu_count(), **kwargs): # Extract Spatial entities + + stop_words = kwargs.get("stop_words",[]) text_and_spatial_entities = [self.parse(text) for text in tqdm(texts,desc="Extract spatial entities from the texts", disable=(not self.verbose))] # Filter Output -- GitLab