update script

script.py

@@ -32,7 +32,11 @@ def process_dataset(dataset_dir: str):
 
     # check if metadata.jsonl exists
     if os.path.exists(os.path.join(dataset_dir, "metadata.jsonl")):
-        metadata = json.load(open(os.path.join(dataset_dir, "metadata.jsonl")))
+        metadata = []
+        with open(os.path.join(dataset_dir, "metadata.jsonl"), "r") as f:
+            for line in f:
+                if len(line.strip()) > 0:
+                    metadata.append(json.loads(line))
         for item in metadata:
             txt_path = os.path.join(dataset_dir, item["file_name"])
             txt_path = txt_path.rsplit(".", 1)[0] + ".txt"