Commit c9838f7b authored by Rémy Decoupes's avatar Rémy Decoupes
Browse files

fix when fix_bad_quote is stopped and restarted

parent 2132c4ec
...@@ -43,14 +43,16 @@ logger.info("Transform jsonl single quotes into double quotes") ...@@ -43,14 +43,16 @@ logger.info("Transform jsonl single quotes into double quotes")
for root, dirs, files in os.walk(path_dir_in): for root, dirs, files in os.walk(path_dir_in):
for name in files: for name in files:
fr = open(path_dir_in + "/" + name) fr = open(path_dir_in + "/" + name)
fw = open(path_dir_out + "/" + name, "w") fw = open(path_dir_out + "/" + name)
nb_lines_in = sum(1 for line in fr) nb_lines_in = sum(1 for line in fr)
try: try:
nb_lines_out = sum(1 for line in fw) nb_lines_out = sum(1 for line in fw)
except: #file is empty except: #file is empty
nb_lines_out = 0 nb_lines_out = 0
logger.info("file: " + name + " in: "+ str(nb_lines_in) + " and out:" + str(nb_lines_out))
if nb_lines_in != nb_lines_out: if nb_lines_in != nb_lines_out:
fr.seek(0) # go to the start of the file fr.seek(0) # go to the start of the file
fw = open(path_dir_out + "/" + name, "w")
for line in fr: for line in fr:
json_dat = json.dumps(ast.literal_eval(line)) json_dat = json.dumps(ast.literal_eval(line))
dict_dat = json.loads(json_dat) dict_dat = json.loads(json_dat)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment