Forgot to double check.

gguf-imat-llama-3.py

@@ -70,6 +70,7 @@ def download_cudart_if_necessary(latest_release_tag):
 def download_model_repo():
     base_dir = os.path.dirname(os.path.abspath(__file__))
     models_dir = os.path.join(base_dir, "models")
+
     if not os.path.exists(models_dir):
         os.makedirs(models_dir)
 
@@ -77,27 +78,27 @@ def download_model_repo():
     model_name = model_id.split("/")[-1]
     model_dir = os.path.join(models_dir, model_name)
 
-    if os.path.exists(model_dir):
-        print("Model repository already exists. Using existing repository.")
-
-        delete_model_dir = input("Remove HF model folder after converting original model to GGUF? (yes/no) (default: no): ").strip().lower()
-
-        imatrix_file_name = input("Enter the name of the imatrix.txt file (default: imatrix.txt): ").strip() or "imatrix.txt"
-
-        convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name)
+    gguf_dir = os.path.join(base_dir, "models", f"{model_name}-GGUF")
+    gguf_model_path = os.path.join(gguf_dir, f"{model_name}-F16.gguf")
+    imatrix_file_name = input("Enter the name of the imatrix.txt file (default: imatrix.txt): ").strip() or "imatrix.txt"
+    delete_model_dir = input("Remove HF model folder after converting original model to GGUF? (yes/no) (default: no): ").strip().lower()
 
-    else:
-        revision = input("Enter the revision (branch, tag, or commit) to download (default: main): ") or "main"
+    if os.path.exists(gguf_model_path):
+        create_imatrix(base_dir, gguf_dir, gguf_model_path, model_name, imatrix_file_name)
+    else: 
+        if os.path.exists(model_dir):
+            print("Model repository already exists. Using existing repository.")  
 
-        delete_model_dir = input("Remove HF model folder after converting original model to GGUF? (yes/no) (default: no): ").strip().lower()
+            convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name)
 
-        print("Downloading model repository...")
-        snapshot_download(repo_id=model_id, local_dir=model_dir, revision=revision)
-        print("Model repository downloaded successfully.")
+        else:
+            revision = input("Enter the revision (branch, tag, or commit) to download (default: main): ") or "main"
 
-        imatrix_file_name = input("Enter the name of the imatrix.txt file (default: imatrix.txt): ").strip() or "imatrix.txt"
+            print("Downloading model repository...")
+            snapshot_download(repo_id=model_id, local_dir=model_dir, revision=revision)
+            print("Model repository downloaded successfully.")
 
-        convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name)
+            convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name)
 
 def convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name):
     convert_script = os.path.join(base_dir, "llama.cpp", "convert.py")
@@ -115,7 +116,11 @@ def convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir,
             print(f"Original model directory '{model_dir}' deleted.")
         else:
             print(f"Original model directory '{model_dir}' was not deleted. You can remove it manually.")
+        
+    
+    create_imatrix(base_dir, gguf_dir, gguf_model_path, model_name, imatrix_file_name)
 
+def create_imatrix(base_dir, gguf_dir, gguf_model_path, model_name, imatrix_file_name):
     imatrix_exe = os.path.join(base_dir, "bin", "imatrix.exe")
     imatrix_output_src = os.path.join(gguf_dir, "imatrix.dat")
     imatrix_output_dst = os.path.join(gguf_dir, "imatrix.dat")
@@ -161,4 +166,4 @@ def main():
     print("Finished preparing resources.")
 
 if __name__ == "__main__":
-    main()
+    main()

gguf-imat.py

@@ -70,6 +70,7 @@ def download_cudart_if_necessary(latest_release_tag):
 def download_model_repo():
     base_dir = os.path.dirname(os.path.abspath(__file__))
     models_dir = os.path.join(base_dir, "models")
+
     if not os.path.exists(models_dir):
         os.makedirs(models_dir)
 
@@ -77,27 +78,27 @@ def download_model_repo():
     model_name = model_id.split("/")[-1]
     model_dir = os.path.join(models_dir, model_name)
 
-    if os.path.exists(model_dir):
-        print("Model repository already exists. Using existing repository.")
-
-        delete_model_dir = input("Remove HF model folder after converting original model to GGUF? (yes/no) (default: no): ").strip().lower()
-
-        imatrix_file_name = input("Enter the name of the imatrix.txt file (default: imatrix.txt): ").strip() or "imatrix.txt"
-
-        convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name)
+    gguf_dir = os.path.join(base_dir, "models", f"{model_name}-GGUF")
+    gguf_model_path = os.path.join(gguf_dir, f"{model_name}-F16.gguf")
+    imatrix_file_name = input("Enter the name of the imatrix.txt file (default: imatrix.txt): ").strip() or "imatrix.txt"
+    delete_model_dir = input("Remove HF model folder after converting original model to GGUF? (yes/no) (default: no): ").strip().lower()
 
-    else:
-        revision = input("Enter the revision (branch, tag, or commit) to download (default: main): ") or "main"
+    if os.path.exists(gguf_model_path):
+        create_imatrix(base_dir, gguf_dir, gguf_model_path, model_name, imatrix_file_name)
+    else: 
+        if os.path.exists(model_dir):
+            print("Model repository already exists. Using existing repository.")  
 
-        delete_model_dir = input("Remove HF model folder after converting original model to GGUF? (yes/no) (default: no): ").strip().lower()
+            convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name)
 
-        print("Downloading model repository...")
-        snapshot_download(repo_id=model_id, local_dir=model_dir, revision=revision)
-        print("Model repository downloaded successfully.")
+        else:
+            revision = input("Enter the revision (branch, tag, or commit) to download (default: main): ") or "main"
 
-        imatrix_file_name = input("Enter the name of the imatrix.txt file (default: imatrix.txt): ").strip() or "imatrix.txt"
+            print("Downloading model repository...")
+            snapshot_download(repo_id=model_id, local_dir=model_dir, revision=revision)
+            print("Model repository downloaded successfully.")
 
-        convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name)
+            convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name)
 
 def convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir, imatrix_file_name):
     convert_script = os.path.join(base_dir, "llama.cpp", "convert.py")
@@ -115,13 +116,17 @@ def convert_model_to_gguf_f16(base_dir, model_dir, model_name, delete_model_dir,
             print(f"Original model directory '{model_dir}' deleted.")
         else:
             print(f"Original model directory '{model_dir}' was not deleted. You can remove it manually.")
+        
+    
+    create_imatrix(base_dir, gguf_dir, gguf_model_path, model_name, imatrix_file_name)
 
+def create_imatrix(base_dir, gguf_dir, gguf_model_path, model_name, imatrix_file_name):
     imatrix_exe = os.path.join(base_dir, "bin", "imatrix.exe")
     imatrix_output_src = os.path.join(gguf_dir, "imatrix.dat")
     imatrix_output_dst = os.path.join(gguf_dir, "imatrix.dat")
     if not os.path.exists(imatrix_output_dst):
         try:
-            subprocess.run([imatrix_exe, "-m", gguf_model_path, "-f", os.path.join(base_dir, "imatrix", imatrix_file_name), "-ngl", "8"], cwd=gguf_dir)
+            subprocess.run([imatrix_exe, "-m", gguf_model_path, "-f", os.path.join(base_dir, "imatrix", imatrix_file_name), "-ngl", "7"], cwd=gguf_dir)
             shutil.move(imatrix_output_src, imatrix_output_dst)
             print("imatrix.dat moved successfully.")
         except Exception as e: