Update EXL2_Private_Quant_V3.ipynb

ipynb/EXL2_Private_Quant_V3.ipynb

@@ -163,31 +163,35 @@
         "#@markdown ![Example Image](https://huggingface.co/Anthonyg5005/hf-scripts/resolve/main/ipynb/dataset-example.jpg \"Copy from download button\")\\\n",
         "#@markdown If dataset is jsonl then convert to parquet. (Not always reliable, must be formatted correctly)\n",
         "convert_parquet = False # @param {type:\"boolean\"}\n",
+        "\n",
+        "                                                            #GITHUB COPILOT GENERATED START\n",
         "if convert_parquet == True:\n",
-        "    import pandas as pd #GitHub Copilot generated\n",
-        "    import pyarrow as pa #GitHub Copilot generated\n",
-        "    import pyarrow.parquet as pq #GitHub Copilot generated\n",
-        "    import json #GitHub Copilot generated\n",
+        "    import pandas as pd\n",
+        "    import pyarrow as pa\n",
+        "    import pyarrow.parquet as pq\n",
+        "    import json\n",
         "\n",
-        "    def jsonl_to_parquet(jsonl_file, parquet_file): #GitHub Copilot generated\n",
+        "    def jsonl_to_parquet(jsonl_file, parquet_file):\n",
         "        # Read JSONL file line by line with explicit encoding\n",
-        "        with open(jsonl_file, 'r', encoding='utf-8') as f: #GitHub Copilot generated\n",
-        "            lines = f.readlines() #GitHub Copilot generated\n",
+        "        with open(jsonl_file, 'r', encoding='utf-8') as f:\n",
+        "            lines = f.readlines()\n",
         "\n",
         "        # Parse JSON lines and store as list of dictionaries\n",
-        "        data = [json.loads(line.strip()) for line in lines] #GitHub Copilot generated\n",
+        "        data = [json.loads(line.strip()) for line in lines]\n",
         "\n",
         "        # Convert to Pandas DataFrame\n",
-        "        df = pd.DataFrame(data) #GitHub Copilot generated\n",
+        "        df = pd.DataFrame(data)\n",
         "\n",
         "        # Convert DataFrame to PyArrow Table\n",
-        "        table = pa.Table.from_pandas(df) #GitHub Copilot generated\n",
+        "        table = pa.Table.from_pandas(df)\n",
         "\n",
         "        # Write PyArrow Table to Parquet file\n",
-        "        pq.write_table(table, parquet_file) #GitHub Copilot generated\n",
+        "        pq.write_table(table, parquet_file)\n",
         "\n",
-        "    jsonl_to_parquet(dataset, f\"{dataset_jtp}.parquet\") #GitHub Copilot generated\n",
+        "    jsonl_to_parquet(dataset, f\"{dataset_jtp}.parquet\")\n",
         "    dataset = f\"{dataset_jtp}.parquet\"\n",
+        "                                                            #GITHUB COPILOT GENERATED END\n",
+        "\n",
         "#@markdown Quantizing only allows for parquet datasets to be used. Enable convert_parquet if your dataset ends in a .jsonl extention.\\\n",
         "#@markdown ![Example Image](https://huggingface.co/Anthonyg5005/hf-scripts/resolve/main/ipynb/jsonl-example.jpg \"File extension is .jsonl\")\\\n",
         "#@markdown pippa is used as an example in this image. This dataset contains content that is not suitable for users under 18. This dataset also may or may not be against colab TOS. It won't be allowed under free colab usage although you're able to use it with paid compute units.\n",