Commit 00131927 authored by Decoupes Remy's avatar Decoupes Remy
Browse files

try/catch bad formated tweets and logged it in on error level

parent 721b23c8
......@@ -55,10 +55,13 @@ for root, dirs, files in os.walk(path_dir_in):
fr.seek(0) # go to the start of the file
fw = open(path_dir_out + "/" + name, "w")
for line in fr:
json_dat = json.dumps(ast.literal_eval(line))
dict_dat = json.loads(json_dat)
json.dump(dict_dat, fw)
fw.write("\n")
try:
json_dat = json.dumps(ast.literal_eval(line))
dict_dat = json.loads(json_dat)
json.dump(dict_dat, fw)
fw.write("\n")
except:
logger.error("Error on tweet in file: " + name + " with tweet: " + line)
nb_lines_out = nb_lines_out + 1
logger.info(name + ": number of tweets: " + str(nb_lines_out))
else:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment