wow

Signed-off-by: Balazs Horvath <acsipont@gmail.com>

dataset_tools/Create Empty txt Files for Images.ipynb

@@ -1,144 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\00a58ce07a5186d4edbe9eb29bd3fe53_1.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\044ce673a55771bd2b5700318547bb89.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\06b6dc04ce7e684216090f6a333a0f24.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\0b5371361af9d5706e010e6b91af8269.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\0b5ead570498882ea2c4f83970007faf.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\127ab13e2bb2c049e409eb7fdcbc9b7a.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\1308b781e85b8c64d7ee3ba1b64bf444.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\1ad66dc78c035d3241601072bb41a455.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\1caf765f1a0301e47680520dab472744.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\20aad729e70607539a47d0a2ccdf382b.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\2241b31138f5ada56d5e14ba371f82e7.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\22e6a9e62f29ac02ea1fbc78f6bd304f.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\24ea0ace58127a97cff268843615744f.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\2877886ddfdd0177f831dccb8e68aa4f.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\2e15a82de44733126b9982987f5badf1.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\2f9f20922622a03f1c94669a6f8c00f5.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\319aa3ba392b11796db84e266b58cd38.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\330d109c6b45475765563acb5780c44c.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\331101a6a21b450274b3b3e90d7593c6.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\33e345444a93860e274794d2dd3702a1.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\3439faa91c896be8787813572547b2a9.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\351af7711b3a57c84c0c28dfce9641b3.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\3cead9d7fccf871396db6f1986ad1188.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\42681e66733bdfd38800eb01c9b21522.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\4402451c6dc515356d25e10c620f2003.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\584a8f97df450c7742ecc1df77ed37fe.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\5ade980fb60341029c46d927948cffcc.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\5c6d81b6743953086bb1f6c7d2bbbd70_1.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\6c6f0b0fd01b8cf509e44bac38032a15.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\7051f37f3a100ecd09b8b704f7756fe3.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\744500bacde986871a6d4f464266a915.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\76fc8b8c80b70271f09c383c42da1d69.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\77aeebd0874339a181566c9d683ee0c0.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\79009dc71f54764093fc00fe30bfe1ec.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\82faee78f1ad3c32ca9045619c04d50e.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\856a37de0b35e336bc17bad772badc79.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\8b3e3bc11f014e3a9ad5ab2be648fc2c.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\90e06213e04da5adedeafbdb9cd365eb.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\94f4e009078007e3ed1051bf89de4034.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\9a104277b8f586d9090864bba6ffb4c3.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\9a1e5a6ebd91cc2263503c44f85b2335.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\9e995b2e5a51203aadade15076c3e43b.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\a1bb3892776dbaa41ebb9860f5173956.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\a3ddc2fbe220dc85b585294e1b7b355f.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\a493be0b29b9390f046795cbae93b0d5.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\a4fffd4a03a0bef737b7cfec5a8cc6f4.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\adbda3e546986a40f2476ef8f3e07118.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\afd1c3f2f8ea7693fb7a6cb692b50403.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\b5ca2054eb91296ba19809ae1d306b4d.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\b8f87e4deca6cbf37396e66447e31921.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\ba77c681986211c9b7da20b5050859a3.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\babaf50487ea84064971eb0d1757df23.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\bafc1dc4937f7ecf7cc707d32500b60d.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\c2b56828f5b306a9d1ef2f4970cb2a46.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\c8f28965c1bde921f89e7f7e49c0b078.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\d125439867ab61520cf8a5f9391a0cd9.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\d28945f8c9556d071028356457f16c34.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\d309a6abca98c989f24f3050c2a1bdd1.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\d709a20dff4ec44d84a446caac98d276.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\d756957fe6549e07e9b1236d36561bde.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\e1515ab206479a8673f14e1ae6fa66b0.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\eccdc7a822051408cab1f0e791e4e308.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\ecd6c90a0ea837644ca91faab0af71e0.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\f3062506cd9a84097f937f46181bfc4f.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\f82a37b693c9c6b626f1fe9b16840d3e.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\f873d55150075322fd60a47bdf940420.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\faaea9eccdc0e63abe7379fdf21d6fee.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\fdffb5c652d76e5ff33ab65d365b6802.txt\n",
-      "Created C:\\Users\\kade\\Desktop\\1_by_spaceengine\\fe5f1e6bd6a302b4830e34823a54bc34.txt\n"
-     ]
-    }
-   ],
-   "source": [
-    "import os\n",
-    "\n",
-    "def create_empty_txt_files(directory):\n",
-    "    # Check if the directory exists\n",
-    "    if not os.path.exists(directory):\n",
-    "        print(\"Directory does not exist.\")\n",
-    "        return\n",
-    "    \n",
-    "    # Get a list of all files in the directory\n",
-    "    files = os.listdir(directory)\n",
-    "    \n",
-    "    # Iterate over each file\n",
-    "    for file in files:\n",
-    "        # Check if the file is an image (you can add more image extensions if needed)\n",
-    "        if file.endswith(\".jpg\") or file.endswith(\".png\") or file.endswith(\".jpeg\"):\n",
-    "            # Extract the filename without extension\n",
-    "            filename = os.path.splitext(file)[0]\n",
-    "            \n",
-    "            # Create a corresponding txt file with the same name\n",
-    "            txt_filename = os.path.join(directory, filename + \".txt\")\n",
-    "            \n",
-    "            # Check if the txt file already exists\n",
-    "            if not os.path.exists(txt_filename):\n",
-    "                # Create an empty txt file\n",
-    "                with open(txt_filename, 'w') as f:\n",
-    "                    pass\n",
-    "                print(f\"Created {txt_filename}\")\n",
-    "            else:\n",
-    "                print(f\"{txt_filename} already exists\")\n",
-    "\n",
-    "# Path to the directory containing the images\n",
-    "image_directory = r'C:\\Users\\kade\\Desktop\\1_by_spaceengine'\n",
-    "\n",
-    "# Call the function to create empty txt files\n",
-    "create_empty_txt_files(image_directory)"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.12.3"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}

dataset_tools/done/Convert RGBA to RGB in PNGs.ipynb

@@ -0,0 +1,96 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Convert RGBA to RGB in PNGs\n",
+    "\n",
+    "---\n",
+    "\n",
+    "This Python script automates the process of converting `.png` images from RGBA to RGB format in a specified directory, utilizing multiprocessing to enhance efficiency."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "from PIL import Image\n",
+    "import glob\n",
+    "import multiprocessing\n",
+    "\n",
+    "# Set the maximum number of pixels allowed in an image to prevent DecompressionBombWarning.\n",
+    "Image.MAX_IMAGE_PIXELS = 139211472\n",
+    "\n",
+    "def convert_rgba_to_rgb(image_path):\n",
+    "    \"\"\"\n",
+    "    Convert an RGBA image to RGB format.\n",
+    "\n",
+    "    This function opens an image from a given path and checks if it's in RGBA mode.\n",
+    "    If it is, the image is converted to RGB mode and saved back to the same path.\n",
+    "    Any errors encountered during processing are caught and printed.\n",
+    "\n",
+    "    Parameters:\n",
+    "    - image_path (str): The file path of the image to be converted.\n",
+    "\n",
+    "    Returns:\n",
+    "    None\n",
+    "    \"\"\"\n",
+    "    try:\n",
+    "        with Image.open(image_path) as image:\n",
+    "            if image.mode == 'RGBA':\n",
+    "                rgb_image = image.convert('RGB')\n",
+    "                rgb_image.save(image_path)\n",
+    "                print(f\"Converted {image_path} to RGB.\")\n",
+    "            else:\n",
+    "                print(f\"{image_path} is not an RGBA image.\")\n",
+    "    except Exception as e:\n",
+    "        print(f\"Error processing {image_path}: {e}\")\n",
+    "\n",
+    "def main():\n",
+    "    \"\"\"\n",
+    "    Main function to convert all RGBA images to RGB in a directory.\n",
+    "\n",
+    "    This function searches for all .png files in a specified directory and its subdirectories.\n",
+    "    It then creates a pool of processes equal to the number of available CPUs and uses them\n",
+    "    to convert each RGBA image to RGB format concurrently.\n",
+    "\n",
+    "    Returns:\n",
+    "    None\n",
+    "    \"\"\"\n",
+    "    directory = r'E:\\training_dir'\n",
+    "    # Get all .png files in the directory recursively\n",
+    "    files = glob.glob(os.path.join(directory, '**', '*.png'), recursive=True)\n",
+    "    \n",
+    "    # Determine the number of processes based on the available CPUs\n",
+    "    num_processes = multiprocessing.cpu_count()\n",
+    "\n",
+    "    # Create a pool of processes\n",
+    "    with multiprocessing.Pool(num_processes) as pool:\n",
+    "        # Map the convert_rgba_to_rgb function to the files\n",
+    "        pool.map(convert_rgba_to_rgb, files)\n",
+    "\n",
+    "    print(\"Conversion complete.\")\n",
+    "\n",
+    "if __name__ == \"__main__\":\n",
+    "    main()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "base",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "name": "python",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

dataset_tools/done/Count Images in Folder.ipynb

@@ -0,0 +1,92 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Count Images in Folder\n",
+    "\n",
+    "---\n",
+    "\n",
+    "This script counts the total number of JPEG and PNG images in a specified directory."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Total number of images: 21891\n"
+     ]
+    }
+   ],
+   "source": [
+    "from pathlib import Path\n",
+    "import glob\n",
+    "\n",
+    "\"\"\"\n",
+    "This script counts the total number of JPEG and PNG images in a specified directory.\n",
+    "\n",
+    "The script uses the `glob` module to find all files with .jpg, .jpeg, and .png extensions\n",
+    "within the given directory and its subdirectories. It then calculates the total count of these\n",
+    "image files and prints the result.\n",
+    "\n",
+    "Functions:\n",
+    "    count_images(directory): Counts and returns the number of image files in the directory.\n",
+    "\n",
+    "Usage:\n",
+    "    To use this script, simply set the `directory_path` variable to the path of the directory\n",
+    "    you want to process. Then run the script, and it will output the total number of images found.\n",
+    "\n",
+    "Note:\n",
+    "    Ensure that the directory path uses double backslashes (\\\\) or raw string literals to avoid escape sequence errors.\n",
+    "\"\"\"\n",
+    "\n",
+    "def count_images(directory):\n",
+    "    # Create a Path object for the directory\n",
+    "    path = Path(directory)\n",
+    "    \n",
+    "    # Use glob to find all jpg, jpeg, and png files\n",
+    "    jpg_files = glob.glob(str(path / '**/*.jpg'), recursive=True)\n",
+    "    jpeg_files = glob.glob(str(path / '**/*.jpeg'), recursive=True)\n",
+    "    png_files = glob.glob(str(path / '**/*.png'), recursive=True)\n",
+    "    \n",
+    "    # Count the total number of image files\n",
+    "    total_images = len(jpg_files) + len(jpeg_files) + len(png_files)\n",
+    "    \n",
+    "    return total_images\n",
+    "\n",
+    "# Specify the directory path\n",
+    "directory_path = 'E:\\\\training_dir'\n",
+    "\n",
+    "# Call the function and print the result\n",
+    "print(f'Total number of images: {count_images(directory_path)}')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "base",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

dataset_tools/done/Create Empty Captions for Images.ipynb

@@ -0,0 +1,95 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Create Empty Captions for Images\n",
+    "\n",
+    "---\n",
+    "\n",
+    "This Python script creates an empty text file with the same name as each image file (.jpg, .png, or .jpeg) present in a specified directory. The script checks if the directory exists, and then iterates through all the image files in the directory. For each image file, it creates a corresponding text file with the same name but with a .txt extension in the same directory, unless a text file with that name already exists."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import glob\n",
+    "\n",
+    "def create_empty_txt_files(directory):\n",
+    "    \"\"\"\n",
+    "    Create empty .txt files for each image in a specified directory.\n",
+    "\n",
+    "    This function checks for the existence of a directory and then iterates over all .jpg, .png, and .jpeg files within it.\n",
+    "    For each image file, it creates an empty .txt file with the same name if it doesn't already exist.\n",
+    "\n",
+    "    Parameters:\n",
+    "    - directory (str): The path to the directory where the image files are located and where the .txt files will be created.\n",
+    "\n",
+    "    Returns:\n",
+    "    None: This function does not return any value.\n",
+    "\n",
+    "    Prints:\n",
+    "    - A message indicating that a .txt file has been created or already exists for each image file.\n",
+    "    \"\"\"\n",
+    "    # Check if the directory exists\n",
+    "    if not os.path.exists(directory):\n",
+    "        print(\"Directory does not exist.\")\n",
+    "        return\n",
+    "\n",
+    "    # Get a list of all image files in the directory\n",
+    "    image_files = glob.glob(os.path.join(directory, \"*.jpg\")) + \\\n",
+    "                  glob.glob(os.path.join(directory, \"*.png\")) + \\\n",
+    "                  glob.glob(os.path.join(directory, \"*.jpeg\"))\n",
+    "\n",
+    "    # Iterate over each image file\n",
+    "    for image_file in image_files:\n",
+    "        # Extract the filename without extension\n",
+    "        filename = os.path.splitext(os.path.basename(image_file))[0]\n",
+    "\n",
+    "        # Create a corresponding txt file with the same name\n",
+    "        txt_filename = os.path.join(directory, filename + \".txt\")\n",
+    "\n",
+    "        # Check if the txt file already exists\n",
+    "        if not os.path.exists(txt_filename):\n",
+    "            # Create an empty txt file\n",
+    "            with open(txt_filename, 'w') as f:\n",
+    "                pass\n",
+    "            print(f\"Created {txt_filename}\")\n",
+    "        else:\n",
+    "            print(f\"{txt_filename} already exists\")\n",
+    "\n",
+    "# Path to the directory containing the images\n",
+    "image_directory = r'C:\\Users\\kade\\Desktop\\training_dir_staging\\1_by_spaceengine'\n",
+    "\n",
+    "# Call the function to create empty txt files\n",
+    "create_empty_txt_files(image_directory)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

dataset_tools/{Newlines to Commas.ipynb → done/Newlines to Commas.ipynb}

@@ -16,24 +16,45 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import os\n",
+    "\"\"\"\n",
+    "This script is designed to process all text files within a specified directory\n",
+    "and its subdirectories. It modifies the content of each text file by replacing\n",
+    "newlines with a comma followed by a space.\n",
+    "\n",
+    "Functions:\n",
+    "    process_directory(directory): Recursively processes all '.txt' files in the given directory.\n",
+    "\n",
+    "Args:\n",
+    "    directory (str): The path to the directory containing the text files to be processed.\n",
+    "\n",
+    "Usage:\n",
+    "    Set the 'directory_path' variable to the path of the target directory and run the script.\n",
+    "    The script will modify all '.txt' files within this directory and its subdirectories.\n",
+    "\"\"\"\n",
+    "from pathlib import Path\n",
     "\n",
     "def process_directory(directory):\n",
-    "    for root, dirs, files in os.walk(directory):\n",
-    "        for filename in files:\n",
-    "            if filename.endswith(\".txt\"):\n",
-    "                file_path = os.path.join(root, filename)\n",
-    "                \n",
-    "                # Read the content of the file\n",
-    "                with open(file_path, 'r') as file:\n",
-    "                    content = file.read()\n",
-    "                \n",
-    "                # Replace newline with a comma and space\n",
-    "                modified_content = content.replace('\\n', ', ')\n",
-    "                \n",
-    "                # Write the modified content back to the file\n",
-    "                with open(file_path, 'w') as file:\n",
-    "                    file.write(modified_content)\n",
+    "    \"\"\"\n",
+    "    Process all '.txt' files in the given directory, replacing newlines with a comma and space.\n",
+    "\n",
+    "    Parameters:\n",
+    "    - directory (str): The path to the directory to process.\n",
+    "    \"\"\"\n",
+    "    # Create a Path object for the directory\n",
+    "    path = Path(directory)\n",
+    "    \n",
+    "    # Use glob pattern to match all .txt files recursively\n",
+    "    for file_path in path.rglob('*.txt'):\n",
+    "        # Read the content of the file\n",
+    "        with open(file_path, 'r') as file:\n",
+    "            content = file.read()\n",
+    "        \n",
+    "        # Replace newline with a comma and space\n",
+    "        modified_content = content.replace('\\n', ', ')\n",
+    "        \n",
+    "        # Write the modified content back to the file\n",
+    "        with open(file_path, 'w') as file:\n",
+    "            file.write(modified_content)\n",
     "\n",
     "# Directory path\n",
     "directory_path = r'C:\\Users\\kade\\Desktop\\training_dir_staging'\n",

dataset_tools/done/Replace Transparency with Black.ipynb

@@ -4,10 +4,11 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Replace Transparency with Black\n",
+    "# Replace Transparency with Black\n",
+    "\n",
     "----\n",
     "\n",
-    "This Python script utilizes the PIL (Python Imaging Library) to recursively traverse a specified directory, identifying image files with extension `.png`, and adds a black layer behind the main layer of each image, effectively removing any existing transparency, before overwriting the original files with the modified versions."
+    "This Python script processes all `.png` images in a specified directory by adding a black layer to each, utilizing multiprocessing to handle the images in parallel for efficiency."
    ]
   },
   {
@@ -19,33 +20,65 @@
     "import os\n",
     "from PIL import Image\n",
     "import glob\n",
+    "from multiprocessing import Pool\n",
     "\n",
     "def add_black_layer(image_path):\n",
     "    \"\"\"\n",
-    "    Adds a black layer to the image at the given path and overwrites it.\n",
+    "    Adds a black layer to the image at the given path and saves the modified image.\n",
+    "\n",
+    "    This function opens an image, converts it to 'RGBA' mode, creates a new black layer,\n",
+    "    pastes the original image onto the black layer, and saves the result back to the disk.\n",
     "\n",
     "    Parameters:\n",
-    "    image_path (str): The file path to the image.\n",
+    "    image_path (str): The file path to the image to be processed.\n",
+    "\n",
+    "    Raises:\n",
+    "    Exception: If there is an error opening or processing the image.\n",
     "    \"\"\"\n",
     "    try:\n",
     "        with Image.open(image_path) as img:\n",
-    "            black_layer = Image.new('RGB', img.size, (0, 0, 0))\n",
+    "            # Ensure the image is in 'RGBA' mode to handle transparency\n",
+    "            img = img.convert('RGBA')\n",
+    "            black_layer = Image.new('RGBA', img.size, (0, 0, 0, 255))  # The fourth value is the alpha channel\n",
     "            black_layer.paste(img, (0, 0), img)\n",
     "            black_layer.save(image_path)\n",
     "            print(f\"Black layer added to and overwritten {image_path}\")\n",
     "    except Exception as e:\n",
     "        print(f\"Error processing {image_path}: {e}\")\n",
     "\n",
-    "def process_directory(directory):\n",
+    "def process_image(image_path):\n",
     "    \"\"\"\n",
-    "    Processes all .png images in the given directory and adds a black layer to them.\n",
+    "    Processes a single image by adding a black layer.\n",
+    "\n",
+    "    This function is designed to be used with multiprocessing. It calls the 'add_black_layer'\n",
+    "    function and handles any exceptions that occur.\n",
     "\n",
     "    Parameters:\n",
-    "    directory (str): The directory path where the images are located.\n",
+    "    image_path (str): The file path to the image to be processed.\n",
     "    \"\"\"\n",
-    "    # Using glob to find all .png files in the directory recursively\n",
-    "    for image_path in glob.glob(os.path.join(directory, '**', '*.png'), recursive=True):\n",
+    "    try:\n",
     "        add_black_layer(image_path)\n",
+    "        print(f\"Black layer added to and overwritten {image_path}\")\n",
+    "    except Exception as e:\n",
+    "        print(f\"Error processing {image_path}: {e}\")\n",
+    "\n",
+    "def process_directory(directory):\n",
+    "    \"\"\"\n",
+    "    Processes all .png images in a directory by adding a black layer to each.\n",
+    "\n",
+    "    This function finds all .png images within the specified directory (including subdirectories),\n",
+    "    then creates a pool of worker processes to process each image in parallel.\n",
+    "\n",
+    "    Parameters:\n",
+    "    directory (str): The directory path where the .png images are located.\n",
+    "    \"\"\"\n",
+    "    # Get a list of all .png images in the directory recursively\n",
+    "    image_paths = glob.glob(os.path.join(directory, '**', '*.png'), recursive=True)\n",
+    "    \n",
+    "    # Create a pool of workers equal to the number of CPU cores\n",
+    "    with Pool() as pool:\n",
+    "        # Map the process_image function to the list of image paths\n",
+    "        pool.map(process_image, image_paths)\n",
     "\n",
     "if __name__ == \"__main__\":\n",
     "    directory = r'E:\\training_dir'\n",