Updates scripts to prepare data

prepare_data.py

@@ -1,21 +0,0 @@
-import os
-import pandas as pd
-from sklearn.model_selection import train_test_split
-
-DATA_DIR = "data/train"
-CSV_PATH = "data/trainLabels.csv"
-TEST_SIZE = 0.2
-RANDOM_STATE = 42
-
-# Load the CSV file into a pandas DataFrame and add the image path
-df = pd.read_csv(CSV_PATH, names=['image_path', 'label'], converters={'image_path': lambda x: f"{DATA_DIR}/{x}.jpeg"})
-
-#  drop row where image does not exist
-df = df[df['image_path'].apply(lambda x: os.path.exists(x))]
-
-# split the data into train and validation sets such that the class distribution is the same in both sets
-df_train, df_val = train_test_split(df, test_size=TEST_SIZE, stratify=df['label'], random_state=RANDOM_STATE)
-
-# Save the train and validation sets to CSV files
-df_train.to_csv("data/train.csv", index=False)
-df_val.to_csv("data/val.csv", index=False)

scripts/{merge-zip-parts.sh → merge_and_extract.sh}

@@ -8,26 +8,31 @@ log() {
     echo "$(date +"%Y-%m-%d %H:%M:%S") $1"
 }
 
-# Function to merge zip parts
-merge_zip() {
+# Function to merge and extract zip files
+merge_and_extract_zip() {
     local zip_name="$1"
     log "Merging $zip_name parts into a single zip file..."
     cat "$DATASET_DIR/$zip_name".zip.* > "$DATASET_DIR/$zip_name.zip"
     log "Merged $zip_name.zip created at $DATASET_DIR"
 
-    # remove partition files
+    # Remove partition files
     rm "$DATASET_DIR/$zip_name".zip.*
     log "Removing $zip_name parts"
+
+    # Extract the merged file
+    log "Extracting $zip_name.zip..."
+    unzip -o "$DATASET_DIR/$zip_name.zip" -d "$DATASET_DIR"
+    log "Extracted $zip_name.zip at $DATASET_DIR"
 }
 
-# Merge train.zip parts;
-merge_zip "train" &
+# Merge and extract train.zip parts;
+merge_and_extract_zip "train" &
 
-# Merge test.zip parts
-merge_zip "test" &
+# Merge and extract test.zip parts
+merge_and_extract_zip "test" &
 
 # End of script
 log "Script execution completed."
 
 # Wait for all background processes to finish
-wait
+wait

scripts/split_dataset.py

@@ -0,0 +1,33 @@
+import os
+import argparse
+import pandas as pd
+from sklearn.model_selection import train_test_split
+
+def load_data(data_dir, csv_path):
+    df = pd.read_csv(csv_path, names=['image_path', 'label'], converters={'image_path': lambda x: f"{data_dir}/{x}.jpeg"})
+    df = df[df['image_path'].apply(lambda x: os.path.exists(x))]
+    return df
+
+def main(data_dir, csv_path, train_csv_path, val_csv_path, test_size=0.2, random_state=42):
+    # Load data from CSV
+    df = load_data(data_dir, csv_path)
+
+    # Split the data into train and validation sets
+    df_train, df_val = train_test_split(df, test_size=test_size, stratify=df['label'], random_state=random_state)
+
+    # Save the train and validation sets to CSV files
+    df_train.to_csv(train_csv_path, index=False)
+    df_val.to_csv(val_csv_path, index=False)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Split dataset into train and validation sets.")
+    parser.add_argument("--data_dir", type=str, default="data/diabetic-retinopathy-dataset/train", help="Directory containing images.")
+    parser.add_argument("--csv_path", type=str, default="data/diabetic-retinopathy-dataset/trainLabels.csv", help="Path to CSV file containing image labels.")
+    parser.add_argument("--train_csv_path", type=str, default="data/diabetic-retinopathy-dataset/train.csv", help="Path to save train CSV file.")
+    parser.add_argument("--val_csv_path", type=str, default="data/diabetic-retinopathy-dataset/val.csv", help="Path to save validation CSV file.")
+    parser.add_argument("--test_size", type=float, default=0.2, help="Proportion of the dataset to include in the validation split.")
+    parser.add_argument("--random_state", type=int, default=42, help="Seed for random number generator.")
+
+    args = parser.parse_args()
+
+    main(args.data_dir, args.csv_path, args.train_csv_path, args.val_csv_path, args.test_size, args.random_state)