Add ignore errors

run_batches.py

@@ -23,6 +23,7 @@ def main():
     parser.add_argument("output_dir", type=str, help="directory to write processed files to")
     parser.add_argument("--batch_size", type=int, default=8)
     parser.add_argument("--max_workers", type=int, default=os.cpu_count() - 4)
+    parser.add_argument("--ignore_errors", action="store_true", help="ignore errors")
     args = parser.parse_args()
 
     input_dir = Path(args.input_dir)
@@ -49,12 +50,13 @@ def main():
         [input_dir] * len(input_files_batches),
         [output_dir] * len(input_files_batches),
         [args.mode] * len(input_files_batches),
+        [args.ignore_errors] * len(input_files_batches),
         max_workers=args.max_workers,
         chunksize=1,
     )
 
 
-def run_batch(input_files_batch, input_dir, output_dir, mode):
+def run_batch(input_files_batch, input_dir, output_dir, mode, ignore_errors):
     with TemporaryDirectory() as temp_dir_SR, TemporaryDirectory() as temp_dir_GNR, TemporaryDirectory() as temp_dir_SA, TemporaryDirectory() as input_temp_dir, TemporaryDirectory() as output_temp_dir:
         input_temp_dir = Path(input_temp_dir)
         output_temp_dir = Path(output_temp_dir)
@@ -87,13 +89,15 @@ def run_batch(input_files_batch, input_dir, output_dir, mode):
                 if "returned non-zero exit status 137" in str(e):
                     logging.error("Process killed due to memory limit")
                     return
+                elif ignore_errors:
+                    logging.error("Ignoring error")
+                    return
                 raise e
 
         output_paths = list(output_temp_dir.rglob("*"))
         for output_path in output_paths:
             logging.info(f"cp {output_path} {output_dir}")
             shutil.copy(output_path, output_dir)
-            output_file = output_path.name
 
 
 if __name__ == "__main__":