diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..8a97f11067104c30ca2d2fb764f4cb206f5934e6 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,74 @@ +/research/ai_detector/checkpoint-281/model.safetensors filter=lfs diff=lfs merge=lfs -text +/research/ai_detector/checkpoint-281/optimizer.pt filter=lfs diff=lfs merge=lfs -text +/research/ai_detector/checkpoint-562/model.safetensors filter=lfs diff=lfs merge=lfs -text +/research/ai_detector/checkpoint-562/optimizer.pt filter=lfs diff=lfs merge=lfs -text +/research/ai_detector/checkpoint-843/model.safetensors filter=lfs diff=lfs merge=lfs -text +/research/ai_detector/checkpoint-843/optimizer.pt filter=lfs diff=lfs merge=lfs -text +/research/ai_detector_v2/checkpoint-1210/model.safetensors filter=lfs diff=lfs merge=lfs -text +/research/ai_detector_v2/checkpoint-1210/optimizer.pt filter=lfs diff=lfs merge=lfs -text +/research/ai_detector_v2/checkpoint-1815/model.safetensors filter=lfs diff=lfs merge=lfs -text +/research/ai_detector_v2/checkpoint-1815/optimizer.pt filter=lfs diff=lfs merge=lfs -text +/research/ai_detector_v2/checkpoint-605/model.safetensors filter=lfs diff=lfs merge=lfs -text +/research/ai_detector_v2/checkpoint-605/optimizer.pt filter=lfs diff=lfs merge=lfs -text +/research/dataset/test/data-00000-of-00013.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/test/data-00001-of-00013.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/test/data-00002-of-00013.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/test/data-00003-of-00013.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/test/data-00004-of-00013.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/test/data-00005-of-00013.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/test/data-00006-of-00013.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/test/data-00007-of-00013.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/test/data-00008-of-00013.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/test/data-00009-of-00013.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/test/data-00010-of-00013.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/test/data-00011-of-00013.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/test/data-00012-of-00013.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/train/data-00000-of-00024.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/train/data-00001-of-00024.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/train/data-00002-of-00024.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/train/data-00003-of-00024.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/train/data-00004-of-00024.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/train/data-00005-of-00024.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/train/data-00006-of-00024.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/train/data-00007-of-00024.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/train/data-00008-of-00024.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/train/data-00009-of-00024.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/train/data-00010-of-00024.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/train/data-00011-of-00024.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/train/data-00012-of-00024.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/train/data-00013-of-00024.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/train/data-00014-of-00024.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/train/data-00015-of-00024.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/train/data-00016-of-00024.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/train/data-00017-of-00024.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/train/data-00018-of-00024.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/train/data-00019-of-00024.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/train/data-00020-of-00024.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/train/data-00021-of-00024.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/train/data-00022-of-00024.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/train/data-00023-of-00024.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/validation/data-00000-of-00025.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/validation/data-00001-of-00025.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/validation/data-00002-of-00025.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/validation/data-00003-of-00025.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/validation/data-00004-of-00025.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/validation/data-00005-of-00025.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/validation/data-00006-of-00025.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/validation/data-00007-of-00025.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/validation/data-00008-of-00025.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/validation/data-00009-of-00025.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/validation/data-00010-of-00025.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/validation/data-00011-of-00025.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/validation/data-00012-of-00025.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/validation/data-00013-of-00025.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/validation/data-00014-of-00025.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/validation/data-00015-of-00025.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/validation/data-00016-of-00025.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/validation/data-00017-of-00025.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/validation/data-00018-of-00025.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/validation/data-00019-of-00025.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/validation/data-00020-of-00025.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/validation/data-00021-of-00025.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/validation/data-00022-of-00025.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/validation/data-00023-of-00025.arrow filter=lfs diff=lfs merge=lfs -text +/research/dataset/validation/data-00024-of-00025.arrow filter=lfs diff=lfs merge=lfs -text diff --git a/app.py b/app.py new file mode 100644 index 0000000000000000000000000000000000000000..abc6658971b4e97adba52d25ea468d22d5d286c5 --- /dev/null +++ b/app.py @@ -0,0 +1,15 @@ +from transformers import pipeline +from PIL import Image +import streamlit as st +classifier = pipeline("image-classification", model="research/ai_detector_v2/checkpoint-1815") + + +with st.sidebar: + image= st.sidebar.file_uploader("Input image:", type=["png", "jpg"]) + btn= st.button("Submit") + +if btn: + img= original_image= Image.open(image) + st.json( + classifier(img) + ) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..70c5b8ed428a18b88c7c4cf23b8f492efb757f2e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +transformers +evaluate +datasets +Pillow +torch +torchvision +scikit-learn +accelerate +streamlit \ No newline at end of file diff --git a/research/ai_detector/checkpoint-281/config.json b/research/ai_detector/checkpoint-281/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c22e54fb4d45712ec595a0701007dd36bcd4c7ee --- /dev/null +++ b/research/ai_detector/checkpoint-281/config.json @@ -0,0 +1,63 @@ +{ + "_name_or_path": "umm-maybe/AI-image-detector", + "architectures": [ + "SwinForImageClassification" + ], + "attention_probs_dropout_prob": 0.0, + "depths": [ + 2, + 2, + 18, + 2 + ], + "drop_path_rate": 0.1, + "embed_dim": 128, + "encoder_stride": 32, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.0, + "hidden_size": 1024, + "id2label": { + "0": "ai_gen", + "1": "human" + }, + "image_size": 224, + "initializer_range": 0.02, + "label2id": { + "ai_gen": "0", + "human": "1" + }, + "layer_norm_eps": 1e-05, + "max_length": 128, + "mlp_ratio": 4.0, + "model_type": "swin", + "num_channels": 3, + "num_heads": [ + 4, + 8, + 16, + 32 + ], + "num_layers": 4, + "out_features": [ + "stage4" + ], + "out_indices": [ + 4 + ], + "padding": "max_length", + "patch_size": 4, + "path_norm": true, + "problem_type": "single_label_classification", + "qkv_bias": true, + "stage_names": [ + "stem", + "stage1", + "stage2", + "stage3", + "stage4" + ], + "torch_dtype": "float32", + "transformers_version": "4.37.2", + "use_absolute_embeddings": false, + "window_size": 7 +} diff --git a/research/ai_detector/checkpoint-281/model.safetensors b/research/ai_detector/checkpoint-281/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..046d8f07bef40d80125f554448e09c67a46058c6 --- /dev/null +++ b/research/ai_detector/checkpoint-281/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15a1521fef8e655b49176248ecc2bd0a275b3b7c2e4d3ff372f55a6afbe5f1ec +size 347498816 diff --git a/research/ai_detector/checkpoint-281/optimizer.pt b/research/ai_detector/checkpoint-281/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..6f1a66be4ecf1cee4f6f8d180402e36c66f5dc51 --- /dev/null +++ b/research/ai_detector/checkpoint-281/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ca28a5528de829c1aa0c706f308e3a9084c481b474a7bb2983dfc59db461795 +size 694317645 diff --git a/research/ai_detector/checkpoint-281/preprocessor_config.json b/research/ai_detector/checkpoint-281/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..273d24924a8bd10227b532d603484f165dcb0b5d --- /dev/null +++ b/research/ai_detector/checkpoint-281/preprocessor_config.json @@ -0,0 +1,22 @@ +{ + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "ViTImageProcessor", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "resample": 2, + "rescale_factor": 0.00392156862745098, + "size": { + "height": 224, + "width": 224 + } +} diff --git a/research/ai_detector/checkpoint-281/rng_state.pth b/research/ai_detector/checkpoint-281/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..54f1fe58271f70c93d9ceeb032544a1fff8145f6 Binary files /dev/null and b/research/ai_detector/checkpoint-281/rng_state.pth differ diff --git a/research/ai_detector/checkpoint-281/scheduler.pt b/research/ai_detector/checkpoint-281/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..33233299e81b04a3134c6e814761b06265bd724b Binary files /dev/null and b/research/ai_detector/checkpoint-281/scheduler.pt differ diff --git a/research/ai_detector/checkpoint-281/trainer_state.json b/research/ai_detector/checkpoint-281/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..ed3eaeeb7993cc91e86970dffb7e33b0888bc1ce --- /dev/null +++ b/research/ai_detector/checkpoint-281/trainer_state.json @@ -0,0 +1,198 @@ +{ + "best_metric": 0.9964804552942939, + "best_model_checkpoint": "ai_detector/checkpoint-281", + "epoch": 0.9991111111111111, + "eval_steps": 500, + "global_step": 281, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.04, + "learning_rate": 5.882352941176471e-06, + "loss": 0.6876, + "step": 10 + }, + { + "epoch": 0.07, + "learning_rate": 1.1764705882352942e-05, + "loss": 0.4009, + "step": 20 + }, + { + "epoch": 0.11, + "learning_rate": 1.7647058823529414e-05, + "loss": 0.2111, + "step": 30 + }, + { + "epoch": 0.14, + "learning_rate": 2.3529411764705884e-05, + "loss": 0.0845, + "step": 40 + }, + { + "epoch": 0.18, + "learning_rate": 2.9411764705882354e-05, + "loss": 0.052, + "step": 50 + }, + { + "epoch": 0.21, + "learning_rate": 3.529411764705883e-05, + "loss": 0.0831, + "step": 60 + }, + { + "epoch": 0.25, + "learning_rate": 4.11764705882353e-05, + "loss": 0.0719, + "step": 70 + }, + { + "epoch": 0.28, + "learning_rate": 4.705882352941177e-05, + "loss": 0.0493, + "step": 80 + }, + { + "epoch": 0.32, + "learning_rate": 4.967018469656992e-05, + "loss": 0.0278, + "step": 90 + }, + { + "epoch": 0.36, + "learning_rate": 4.901055408970976e-05, + "loss": 0.0447, + "step": 100 + }, + { + "epoch": 0.39, + "learning_rate": 4.835092348284961e-05, + "loss": 0.0597, + "step": 110 + }, + { + "epoch": 0.43, + "learning_rate": 4.7691292875989446e-05, + "loss": 0.0194, + "step": 120 + }, + { + "epoch": 0.46, + "learning_rate": 4.703166226912929e-05, + "loss": 0.0308, + "step": 130 + }, + { + "epoch": 0.5, + "learning_rate": 4.6372031662269136e-05, + "loss": 0.0328, + "step": 140 + }, + { + "epoch": 0.53, + "learning_rate": 4.5712401055408974e-05, + "loss": 0.0415, + "step": 150 + }, + { + "epoch": 0.57, + "learning_rate": 4.505277044854881e-05, + "loss": 0.0075, + "step": 160 + }, + { + "epoch": 0.6, + "learning_rate": 4.439313984168866e-05, + "loss": 0.0358, + "step": 170 + }, + { + "epoch": 0.64, + "learning_rate": 4.3733509234828496e-05, + "loss": 0.0684, + "step": 180 + }, + { + "epoch": 0.68, + "learning_rate": 4.307387862796834e-05, + "loss": 0.0382, + "step": 190 + }, + { + "epoch": 0.71, + "learning_rate": 4.2414248021108186e-05, + "loss": 0.0252, + "step": 200 + }, + { + "epoch": 0.75, + "learning_rate": 4.1754617414248024e-05, + "loss": 0.0356, + "step": 210 + }, + { + "epoch": 0.78, + "learning_rate": 4.109498680738786e-05, + "loss": 0.0214, + "step": 220 + }, + { + "epoch": 0.82, + "learning_rate": 4.043535620052771e-05, + "loss": 0.0137, + "step": 230 + }, + { + "epoch": 0.85, + "learning_rate": 3.9775725593667545e-05, + "loss": 0.091, + "step": 240 + }, + { + "epoch": 0.89, + "learning_rate": 3.911609498680739e-05, + "loss": 0.0103, + "step": 250 + }, + { + "epoch": 0.92, + "learning_rate": 3.8456464379947235e-05, + "loss": 0.0418, + "step": 260 + }, + { + "epoch": 0.96, + "learning_rate": 3.779683377308707e-05, + "loss": 0.0089, + "step": 270 + }, + { + "epoch": 1.0, + "learning_rate": 3.713720316622691e-05, + "loss": 0.0347, + "step": 280 + }, + { + "epoch": 1.0, + "eval_accuracy": 0.9964804552942939, + "eval_loss": 0.013468941673636436, + "eval_runtime": 252.3562, + "eval_samples_per_second": 52.917, + "eval_steps_per_second": 3.309, + "step": 281 + } + ], + "logging_steps": 10, + "max_steps": 843, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 500, + "total_flos": 1.410220001304576e+18, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/research/ai_detector/checkpoint-281/training_args.bin b/research/ai_detector/checkpoint-281/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..4ebf8bb1c6ee308237e15e5e8ff75b961d2c0426 Binary files /dev/null and b/research/ai_detector/checkpoint-281/training_args.bin differ diff --git a/research/ai_detector/checkpoint-562/config.json b/research/ai_detector/checkpoint-562/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c22e54fb4d45712ec595a0701007dd36bcd4c7ee --- /dev/null +++ b/research/ai_detector/checkpoint-562/config.json @@ -0,0 +1,63 @@ +{ + "_name_or_path": "umm-maybe/AI-image-detector", + "architectures": [ + "SwinForImageClassification" + ], + "attention_probs_dropout_prob": 0.0, + "depths": [ + 2, + 2, + 18, + 2 + ], + "drop_path_rate": 0.1, + "embed_dim": 128, + "encoder_stride": 32, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.0, + "hidden_size": 1024, + "id2label": { + "0": "ai_gen", + "1": "human" + }, + "image_size": 224, + "initializer_range": 0.02, + "label2id": { + "ai_gen": "0", + "human": "1" + }, + "layer_norm_eps": 1e-05, + "max_length": 128, + "mlp_ratio": 4.0, + "model_type": "swin", + "num_channels": 3, + "num_heads": [ + 4, + 8, + 16, + 32 + ], + "num_layers": 4, + "out_features": [ + "stage4" + ], + "out_indices": [ + 4 + ], + "padding": "max_length", + "patch_size": 4, + "path_norm": true, + "problem_type": "single_label_classification", + "qkv_bias": true, + "stage_names": [ + "stem", + "stage1", + "stage2", + "stage3", + "stage4" + ], + "torch_dtype": "float32", + "transformers_version": "4.37.2", + "use_absolute_embeddings": false, + "window_size": 7 +} diff --git a/research/ai_detector/checkpoint-562/model.safetensors b/research/ai_detector/checkpoint-562/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..94156de0d53bcc8ff931cea159874bef9263701d --- /dev/null +++ b/research/ai_detector/checkpoint-562/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9c54f1d503fc157004c496f277f4f6f5612c61527096bfb4b8afb7d5a18df36 +size 347498816 diff --git a/research/ai_detector/checkpoint-562/optimizer.pt b/research/ai_detector/checkpoint-562/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..3320dd66a0c59e608cba591f908f7a0a2328d893 --- /dev/null +++ b/research/ai_detector/checkpoint-562/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bb5c0e90ba3073b92823ea93c8ef1fe01f6e396117253d69b9cd60966177348 +size 694317645 diff --git a/research/ai_detector/checkpoint-562/preprocessor_config.json b/research/ai_detector/checkpoint-562/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..273d24924a8bd10227b532d603484f165dcb0b5d --- /dev/null +++ b/research/ai_detector/checkpoint-562/preprocessor_config.json @@ -0,0 +1,22 @@ +{ + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "ViTImageProcessor", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "resample": 2, + "rescale_factor": 0.00392156862745098, + "size": { + "height": 224, + "width": 224 + } +} diff --git a/research/ai_detector/checkpoint-562/rng_state.pth b/research/ai_detector/checkpoint-562/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..dfd28c4a1e107aa4f6f4b9b33c617153ced560c1 Binary files /dev/null and b/research/ai_detector/checkpoint-562/rng_state.pth differ diff --git a/research/ai_detector/checkpoint-562/scheduler.pt b/research/ai_detector/checkpoint-562/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..ee2e5cca9329b10967708543b8cb2ccc6d9e6b5b Binary files /dev/null and b/research/ai_detector/checkpoint-562/scheduler.pt differ diff --git a/research/ai_detector/checkpoint-562/trainer_state.json b/research/ai_detector/checkpoint-562/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..bfc00c9425819a7149e32acbc26956e90bf4906c --- /dev/null +++ b/research/ai_detector/checkpoint-562/trainer_state.json @@ -0,0 +1,375 @@ +{ + "best_metric": 0.9964804552942939, + "best_model_checkpoint": "ai_detector/checkpoint-281", + "epoch": 1.9982222222222221, + "eval_steps": 500, + "global_step": 562, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.04, + "learning_rate": 5.882352941176471e-06, + "loss": 0.6876, + "step": 10 + }, + { + "epoch": 0.07, + "learning_rate": 1.1764705882352942e-05, + "loss": 0.4009, + "step": 20 + }, + { + "epoch": 0.11, + "learning_rate": 1.7647058823529414e-05, + "loss": 0.2111, + "step": 30 + }, + { + "epoch": 0.14, + "learning_rate": 2.3529411764705884e-05, + "loss": 0.0845, + "step": 40 + }, + { + "epoch": 0.18, + "learning_rate": 2.9411764705882354e-05, + "loss": 0.052, + "step": 50 + }, + { + "epoch": 0.21, + "learning_rate": 3.529411764705883e-05, + "loss": 0.0831, + "step": 60 + }, + { + "epoch": 0.25, + "learning_rate": 4.11764705882353e-05, + "loss": 0.0719, + "step": 70 + }, + { + "epoch": 0.28, + "learning_rate": 4.705882352941177e-05, + "loss": 0.0493, + "step": 80 + }, + { + "epoch": 0.32, + "learning_rate": 4.967018469656992e-05, + "loss": 0.0278, + "step": 90 + }, + { + "epoch": 0.36, + "learning_rate": 4.901055408970976e-05, + "loss": 0.0447, + "step": 100 + }, + { + "epoch": 0.39, + "learning_rate": 4.835092348284961e-05, + "loss": 0.0597, + "step": 110 + }, + { + "epoch": 0.43, + "learning_rate": 4.7691292875989446e-05, + "loss": 0.0194, + "step": 120 + }, + { + "epoch": 0.46, + "learning_rate": 4.703166226912929e-05, + "loss": 0.0308, + "step": 130 + }, + { + "epoch": 0.5, + "learning_rate": 4.6372031662269136e-05, + "loss": 0.0328, + "step": 140 + }, + { + "epoch": 0.53, + "learning_rate": 4.5712401055408974e-05, + "loss": 0.0415, + "step": 150 + }, + { + "epoch": 0.57, + "learning_rate": 4.505277044854881e-05, + "loss": 0.0075, + "step": 160 + }, + { + "epoch": 0.6, + "learning_rate": 4.439313984168866e-05, + "loss": 0.0358, + "step": 170 + }, + { + "epoch": 0.64, + "learning_rate": 4.3733509234828496e-05, + "loss": 0.0684, + "step": 180 + }, + { + "epoch": 0.68, + "learning_rate": 4.307387862796834e-05, + "loss": 0.0382, + "step": 190 + }, + { + "epoch": 0.71, + "learning_rate": 4.2414248021108186e-05, + "loss": 0.0252, + "step": 200 + }, + { + "epoch": 0.75, + "learning_rate": 4.1754617414248024e-05, + "loss": 0.0356, + "step": 210 + }, + { + "epoch": 0.78, + "learning_rate": 4.109498680738786e-05, + "loss": 0.0214, + "step": 220 + }, + { + "epoch": 0.82, + "learning_rate": 4.043535620052771e-05, + "loss": 0.0137, + "step": 230 + }, + { + "epoch": 0.85, + "learning_rate": 3.9775725593667545e-05, + "loss": 0.091, + "step": 240 + }, + { + "epoch": 0.89, + "learning_rate": 3.911609498680739e-05, + "loss": 0.0103, + "step": 250 + }, + { + "epoch": 0.92, + "learning_rate": 3.8456464379947235e-05, + "loss": 0.0418, + "step": 260 + }, + { + "epoch": 0.96, + "learning_rate": 3.779683377308707e-05, + "loss": 0.0089, + "step": 270 + }, + { + "epoch": 1.0, + "learning_rate": 3.713720316622691e-05, + "loss": 0.0347, + "step": 280 + }, + { + "epoch": 1.0, + "eval_accuracy": 0.9964804552942939, + "eval_loss": 0.013468941673636436, + "eval_runtime": 252.3562, + "eval_samples_per_second": 52.917, + "eval_steps_per_second": 3.309, + "step": 281 + }, + { + "epoch": 1.03, + "learning_rate": 3.6477572559366756e-05, + "loss": 0.0258, + "step": 290 + }, + { + "epoch": 1.07, + "learning_rate": 3.58179419525066e-05, + "loss": 0.0297, + "step": 300 + }, + { + "epoch": 1.1, + "learning_rate": 3.515831134564644e-05, + "loss": 0.029, + "step": 310 + }, + { + "epoch": 1.14, + "learning_rate": 3.4498680738786285e-05, + "loss": 0.0023, + "step": 320 + }, + { + "epoch": 1.17, + "learning_rate": 3.383905013192612e-05, + "loss": 0.013, + "step": 330 + }, + { + "epoch": 1.21, + "learning_rate": 3.317941952506596e-05, + "loss": 0.0203, + "step": 340 + }, + { + "epoch": 1.24, + "learning_rate": 3.2519788918205806e-05, + "loss": 0.0286, + "step": 350 + }, + { + "epoch": 1.28, + "learning_rate": 3.186015831134565e-05, + "loss": 0.0236, + "step": 360 + }, + { + "epoch": 1.32, + "learning_rate": 3.120052770448549e-05, + "loss": 0.0126, + "step": 370 + }, + { + "epoch": 1.35, + "learning_rate": 3.0540897097625334e-05, + "loss": 0.02, + "step": 380 + }, + { + "epoch": 1.39, + "learning_rate": 2.9881266490765176e-05, + "loss": 0.0215, + "step": 390 + }, + { + "epoch": 1.42, + "learning_rate": 2.9221635883905014e-05, + "loss": 0.0143, + "step": 400 + }, + { + "epoch": 1.46, + "learning_rate": 2.8562005277044855e-05, + "loss": 0.0054, + "step": 410 + }, + { + "epoch": 1.49, + "learning_rate": 2.79023746701847e-05, + "loss": 0.0072, + "step": 420 + }, + { + "epoch": 1.53, + "learning_rate": 2.724274406332454e-05, + "loss": 0.005, + "step": 430 + }, + { + "epoch": 1.56, + "learning_rate": 2.658311345646438e-05, + "loss": 0.0142, + "step": 440 + }, + { + "epoch": 1.6, + "learning_rate": 2.5923482849604225e-05, + "loss": 0.0155, + "step": 450 + }, + { + "epoch": 1.64, + "learning_rate": 2.5263852242744063e-05, + "loss": 0.0109, + "step": 460 + }, + { + "epoch": 1.67, + "learning_rate": 2.4604221635883905e-05, + "loss": 0.0073, + "step": 470 + }, + { + "epoch": 1.71, + "learning_rate": 2.3944591029023746e-05, + "loss": 0.0126, + "step": 480 + }, + { + "epoch": 1.74, + "learning_rate": 2.328496042216359e-05, + "loss": 0.0154, + "step": 490 + }, + { + "epoch": 1.78, + "learning_rate": 2.262532981530343e-05, + "loss": 0.0065, + "step": 500 + }, + { + "epoch": 1.81, + "learning_rate": 2.196569920844327e-05, + "loss": 0.0275, + "step": 510 + }, + { + "epoch": 1.85, + "learning_rate": 2.1306068601583116e-05, + "loss": 0.0296, + "step": 520 + }, + { + "epoch": 1.88, + "learning_rate": 2.0646437994722954e-05, + "loss": 0.0043, + "step": 530 + }, + { + "epoch": 1.92, + "learning_rate": 1.9986807387862796e-05, + "loss": 0.0012, + "step": 540 + }, + { + "epoch": 1.96, + "learning_rate": 1.932717678100264e-05, + "loss": 0.0044, + "step": 550 + }, + { + "epoch": 1.99, + "learning_rate": 1.866754617414248e-05, + "loss": 0.0009, + "step": 560 + }, + { + "epoch": 2.0, + "eval_accuracy": 0.9942339373970346, + "eval_loss": 0.02113017998635769, + "eval_runtime": 251.3615, + "eval_samples_per_second": 53.127, + "eval_steps_per_second": 3.322, + "step": 562 + } + ], + "logging_steps": 10, + "max_steps": 843, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 500, + "total_flos": 2.820440002609152e+18, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/research/ai_detector/checkpoint-562/training_args.bin b/research/ai_detector/checkpoint-562/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..4ebf8bb1c6ee308237e15e5e8ff75b961d2c0426 Binary files /dev/null and b/research/ai_detector/checkpoint-562/training_args.bin differ diff --git a/research/ai_detector/checkpoint-843/config.json b/research/ai_detector/checkpoint-843/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c22e54fb4d45712ec595a0701007dd36bcd4c7ee --- /dev/null +++ b/research/ai_detector/checkpoint-843/config.json @@ -0,0 +1,63 @@ +{ + "_name_or_path": "umm-maybe/AI-image-detector", + "architectures": [ + "SwinForImageClassification" + ], + "attention_probs_dropout_prob": 0.0, + "depths": [ + 2, + 2, + 18, + 2 + ], + "drop_path_rate": 0.1, + "embed_dim": 128, + "encoder_stride": 32, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.0, + "hidden_size": 1024, + "id2label": { + "0": "ai_gen", + "1": "human" + }, + "image_size": 224, + "initializer_range": 0.02, + "label2id": { + "ai_gen": "0", + "human": "1" + }, + "layer_norm_eps": 1e-05, + "max_length": 128, + "mlp_ratio": 4.0, + "model_type": "swin", + "num_channels": 3, + "num_heads": [ + 4, + 8, + 16, + 32 + ], + "num_layers": 4, + "out_features": [ + "stage4" + ], + "out_indices": [ + 4 + ], + "padding": "max_length", + "patch_size": 4, + "path_norm": true, + "problem_type": "single_label_classification", + "qkv_bias": true, + "stage_names": [ + "stem", + "stage1", + "stage2", + "stage3", + "stage4" + ], + "torch_dtype": "float32", + "transformers_version": "4.37.2", + "use_absolute_embeddings": false, + "window_size": 7 +} diff --git a/research/ai_detector/checkpoint-843/model.safetensors b/research/ai_detector/checkpoint-843/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..ef94a879a6d71f6e843daf1a90df494629ca9d10 --- /dev/null +++ b/research/ai_detector/checkpoint-843/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4fd37a2fe0e66c4eb2afe850d5514b90ab5ab8ad6dd35cfc36b2088766aae32 +size 347498816 diff --git a/research/ai_detector/checkpoint-843/optimizer.pt b/research/ai_detector/checkpoint-843/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..ef6f14729304eaa5dfcfb7b51aca5ad77fcbf79c --- /dev/null +++ b/research/ai_detector/checkpoint-843/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c022c912a00d486187e84b7e495af6dafbd3d78eaca3dfcd6a78f6ace13eb50 +size 694317645 diff --git a/research/ai_detector/checkpoint-843/preprocessor_config.json b/research/ai_detector/checkpoint-843/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..273d24924a8bd10227b532d603484f165dcb0b5d --- /dev/null +++ b/research/ai_detector/checkpoint-843/preprocessor_config.json @@ -0,0 +1,22 @@ +{ + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "ViTImageProcessor", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "resample": 2, + "rescale_factor": 0.00392156862745098, + "size": { + "height": 224, + "width": 224 + } +} diff --git a/research/ai_detector/checkpoint-843/rng_state.pth b/research/ai_detector/checkpoint-843/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..9e7da01b3fe595e325245a468681aaa800ec9141 Binary files /dev/null and b/research/ai_detector/checkpoint-843/rng_state.pth differ diff --git a/research/ai_detector/checkpoint-843/scheduler.pt b/research/ai_detector/checkpoint-843/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..34f21f3336ae6ce1b03a3691064cab6cfe56563e Binary files /dev/null and b/research/ai_detector/checkpoint-843/scheduler.pt differ diff --git a/research/ai_detector/checkpoint-843/trainer_state.json b/research/ai_detector/checkpoint-843/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..22093bfd476c6d514e7d943b379fb594254d5b43 --- /dev/null +++ b/research/ai_detector/checkpoint-843/trainer_state.json @@ -0,0 +1,552 @@ +{ + "best_metric": 0.9975288303130149, + "best_model_checkpoint": "ai_detector/checkpoint-843", + "epoch": 2.997333333333333, + "eval_steps": 500, + "global_step": 843, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.04, + "learning_rate": 5.882352941176471e-06, + "loss": 0.6876, + "step": 10 + }, + { + "epoch": 0.07, + "learning_rate": 1.1764705882352942e-05, + "loss": 0.4009, + "step": 20 + }, + { + "epoch": 0.11, + "learning_rate": 1.7647058823529414e-05, + "loss": 0.2111, + "step": 30 + }, + { + "epoch": 0.14, + "learning_rate": 2.3529411764705884e-05, + "loss": 0.0845, + "step": 40 + }, + { + "epoch": 0.18, + "learning_rate": 2.9411764705882354e-05, + "loss": 0.052, + "step": 50 + }, + { + "epoch": 0.21, + "learning_rate": 3.529411764705883e-05, + "loss": 0.0831, + "step": 60 + }, + { + "epoch": 0.25, + "learning_rate": 4.11764705882353e-05, + "loss": 0.0719, + "step": 70 + }, + { + "epoch": 0.28, + "learning_rate": 4.705882352941177e-05, + "loss": 0.0493, + "step": 80 + }, + { + "epoch": 0.32, + "learning_rate": 4.967018469656992e-05, + "loss": 0.0278, + "step": 90 + }, + { + "epoch": 0.36, + "learning_rate": 4.901055408970976e-05, + "loss": 0.0447, + "step": 100 + }, + { + "epoch": 0.39, + "learning_rate": 4.835092348284961e-05, + "loss": 0.0597, + "step": 110 + }, + { + "epoch": 0.43, + "learning_rate": 4.7691292875989446e-05, + "loss": 0.0194, + "step": 120 + }, + { + "epoch": 0.46, + "learning_rate": 4.703166226912929e-05, + "loss": 0.0308, + "step": 130 + }, + { + "epoch": 0.5, + "learning_rate": 4.6372031662269136e-05, + "loss": 0.0328, + "step": 140 + }, + { + "epoch": 0.53, + "learning_rate": 4.5712401055408974e-05, + "loss": 0.0415, + "step": 150 + }, + { + "epoch": 0.57, + "learning_rate": 4.505277044854881e-05, + "loss": 0.0075, + "step": 160 + }, + { + "epoch": 0.6, + "learning_rate": 4.439313984168866e-05, + "loss": 0.0358, + "step": 170 + }, + { + "epoch": 0.64, + "learning_rate": 4.3733509234828496e-05, + "loss": 0.0684, + "step": 180 + }, + { + "epoch": 0.68, + "learning_rate": 4.307387862796834e-05, + "loss": 0.0382, + "step": 190 + }, + { + "epoch": 0.71, + "learning_rate": 4.2414248021108186e-05, + "loss": 0.0252, + "step": 200 + }, + { + "epoch": 0.75, + "learning_rate": 4.1754617414248024e-05, + "loss": 0.0356, + "step": 210 + }, + { + "epoch": 0.78, + "learning_rate": 4.109498680738786e-05, + "loss": 0.0214, + "step": 220 + }, + { + "epoch": 0.82, + "learning_rate": 4.043535620052771e-05, + "loss": 0.0137, + "step": 230 + }, + { + "epoch": 0.85, + "learning_rate": 3.9775725593667545e-05, + "loss": 0.091, + "step": 240 + }, + { + "epoch": 0.89, + "learning_rate": 3.911609498680739e-05, + "loss": 0.0103, + "step": 250 + }, + { + "epoch": 0.92, + "learning_rate": 3.8456464379947235e-05, + "loss": 0.0418, + "step": 260 + }, + { + "epoch": 0.96, + "learning_rate": 3.779683377308707e-05, + "loss": 0.0089, + "step": 270 + }, + { + "epoch": 1.0, + "learning_rate": 3.713720316622691e-05, + "loss": 0.0347, + "step": 280 + }, + { + "epoch": 1.0, + "eval_accuracy": 0.9964804552942939, + "eval_loss": 0.013468941673636436, + "eval_runtime": 252.3562, + "eval_samples_per_second": 52.917, + "eval_steps_per_second": 3.309, + "step": 281 + }, + { + "epoch": 1.03, + "learning_rate": 3.6477572559366756e-05, + "loss": 0.0258, + "step": 290 + }, + { + "epoch": 1.07, + "learning_rate": 3.58179419525066e-05, + "loss": 0.0297, + "step": 300 + }, + { + "epoch": 1.1, + "learning_rate": 3.515831134564644e-05, + "loss": 0.029, + "step": 310 + }, + { + "epoch": 1.14, + "learning_rate": 3.4498680738786285e-05, + "loss": 0.0023, + "step": 320 + }, + { + "epoch": 1.17, + "learning_rate": 3.383905013192612e-05, + "loss": 0.013, + "step": 330 + }, + { + "epoch": 1.21, + "learning_rate": 3.317941952506596e-05, + "loss": 0.0203, + "step": 340 + }, + { + "epoch": 1.24, + "learning_rate": 3.2519788918205806e-05, + "loss": 0.0286, + "step": 350 + }, + { + "epoch": 1.28, + "learning_rate": 3.186015831134565e-05, + "loss": 0.0236, + "step": 360 + }, + { + "epoch": 1.32, + "learning_rate": 3.120052770448549e-05, + "loss": 0.0126, + "step": 370 + }, + { + "epoch": 1.35, + "learning_rate": 3.0540897097625334e-05, + "loss": 0.02, + "step": 380 + }, + { + "epoch": 1.39, + "learning_rate": 2.9881266490765176e-05, + "loss": 0.0215, + "step": 390 + }, + { + "epoch": 1.42, + "learning_rate": 2.9221635883905014e-05, + "loss": 0.0143, + "step": 400 + }, + { + "epoch": 1.46, + "learning_rate": 2.8562005277044855e-05, + "loss": 0.0054, + "step": 410 + }, + { + "epoch": 1.49, + "learning_rate": 2.79023746701847e-05, + "loss": 0.0072, + "step": 420 + }, + { + "epoch": 1.53, + "learning_rate": 2.724274406332454e-05, + "loss": 0.005, + "step": 430 + }, + { + "epoch": 1.56, + "learning_rate": 2.658311345646438e-05, + "loss": 0.0142, + "step": 440 + }, + { + "epoch": 1.6, + "learning_rate": 2.5923482849604225e-05, + "loss": 0.0155, + "step": 450 + }, + { + "epoch": 1.64, + "learning_rate": 2.5263852242744063e-05, + "loss": 0.0109, + "step": 460 + }, + { + "epoch": 1.67, + "learning_rate": 2.4604221635883905e-05, + "loss": 0.0073, + "step": 470 + }, + { + "epoch": 1.71, + "learning_rate": 2.3944591029023746e-05, + "loss": 0.0126, + "step": 480 + }, + { + "epoch": 1.74, + "learning_rate": 2.328496042216359e-05, + "loss": 0.0154, + "step": 490 + }, + { + "epoch": 1.78, + "learning_rate": 2.262532981530343e-05, + "loss": 0.0065, + "step": 500 + }, + { + "epoch": 1.81, + "learning_rate": 2.196569920844327e-05, + "loss": 0.0275, + "step": 510 + }, + { + "epoch": 1.85, + "learning_rate": 2.1306068601583116e-05, + "loss": 0.0296, + "step": 520 + }, + { + "epoch": 1.88, + "learning_rate": 2.0646437994722954e-05, + "loss": 0.0043, + "step": 530 + }, + { + "epoch": 1.92, + "learning_rate": 1.9986807387862796e-05, + "loss": 0.0012, + "step": 540 + }, + { + "epoch": 1.96, + "learning_rate": 1.932717678100264e-05, + "loss": 0.0044, + "step": 550 + }, + { + "epoch": 1.99, + "learning_rate": 1.866754617414248e-05, + "loss": 0.0009, + "step": 560 + }, + { + "epoch": 2.0, + "eval_accuracy": 0.9942339373970346, + "eval_loss": 0.02113017998635769, + "eval_runtime": 251.3615, + "eval_samples_per_second": 53.127, + "eval_steps_per_second": 3.322, + "step": 562 + }, + { + "epoch": 2.03, + "learning_rate": 1.8007915567282324e-05, + "loss": 0.002, + "step": 570 + }, + { + "epoch": 2.06, + "learning_rate": 1.7348284960422166e-05, + "loss": 0.0154, + "step": 580 + }, + { + "epoch": 2.1, + "learning_rate": 1.6688654353562007e-05, + "loss": 0.0058, + "step": 590 + }, + { + "epoch": 2.13, + "learning_rate": 1.602902374670185e-05, + "loss": 0.0066, + "step": 600 + }, + { + "epoch": 2.17, + "learning_rate": 1.536939313984169e-05, + "loss": 0.0007, + "step": 610 + }, + { + "epoch": 2.2, + "learning_rate": 1.470976253298153e-05, + "loss": 0.0008, + "step": 620 + }, + { + "epoch": 2.24, + "learning_rate": 1.4050131926121373e-05, + "loss": 0.0037, + "step": 630 + }, + { + "epoch": 2.28, + "learning_rate": 1.3390501319261215e-05, + "loss": 0.0057, + "step": 640 + }, + { + "epoch": 2.31, + "learning_rate": 1.2730870712401055e-05, + "loss": 0.0002, + "step": 650 + }, + { + "epoch": 2.35, + "learning_rate": 1.2071240105540896e-05, + "loss": 0.0241, + "step": 660 + }, + { + "epoch": 2.38, + "learning_rate": 1.141160949868074e-05, + "loss": 0.0007, + "step": 670 + }, + { + "epoch": 2.42, + "learning_rate": 1.0751978891820581e-05, + "loss": 0.0058, + "step": 680 + }, + { + "epoch": 2.45, + "learning_rate": 1.0092348284960421e-05, + "loss": 0.0003, + "step": 690 + }, + { + "epoch": 2.49, + "learning_rate": 9.432717678100264e-06, + "loss": 0.0075, + "step": 700 + }, + { + "epoch": 2.52, + "learning_rate": 8.773087071240106e-06, + "loss": 0.001, + "step": 710 + }, + { + "epoch": 2.56, + "learning_rate": 8.113456464379948e-06, + "loss": 0.002, + "step": 720 + }, + { + "epoch": 2.6, + "learning_rate": 7.453825857519789e-06, + "loss": 0.0125, + "step": 730 + }, + { + "epoch": 2.63, + "learning_rate": 6.794195250659631e-06, + "loss": 0.0001, + "step": 740 + }, + { + "epoch": 2.67, + "learning_rate": 6.134564643799472e-06, + "loss": 0.0003, + "step": 750 + }, + { + "epoch": 2.7, + "learning_rate": 5.474934036939315e-06, + "loss": 0.001, + "step": 760 + }, + { + "epoch": 2.74, + "learning_rate": 4.8153034300791555e-06, + "loss": 0.0001, + "step": 770 + }, + { + "epoch": 2.77, + "learning_rate": 4.155672823218998e-06, + "loss": 0.0078, + "step": 780 + }, + { + "epoch": 2.81, + "learning_rate": 3.496042216358839e-06, + "loss": 0.0031, + "step": 790 + }, + { + "epoch": 2.84, + "learning_rate": 2.836411609498681e-06, + "loss": 0.0264, + "step": 800 + }, + { + "epoch": 2.88, + "learning_rate": 2.1767810026385226e-06, + "loss": 0.0124, + "step": 810 + }, + { + "epoch": 2.92, + "learning_rate": 1.5171503957783642e-06, + "loss": 0.0195, + "step": 820 + }, + { + "epoch": 2.95, + "learning_rate": 8.575197889182058e-07, + "loss": 0.005, + "step": 830 + }, + { + "epoch": 2.99, + "learning_rate": 1.9788918205804752e-07, + "loss": 0.0241, + "step": 840 + }, + { + "epoch": 3.0, + "eval_accuracy": 0.9975288303130149, + "eval_loss": 0.010735484771430492, + "eval_runtime": 284.9371, + "eval_samples_per_second": 46.866, + "eval_steps_per_second": 2.93, + "step": 843 + } + ], + "logging_steps": 10, + "max_steps": 843, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 500, + "total_flos": 4.2268994172435825e+18, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/research/ai_detector/checkpoint-843/training_args.bin b/research/ai_detector/checkpoint-843/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..4ebf8bb1c6ee308237e15e5e8ff75b961d2c0426 Binary files /dev/null and b/research/ai_detector/checkpoint-843/training_args.bin differ diff --git a/research/ai_detector_v2/checkpoint-1210/config.json b/research/ai_detector_v2/checkpoint-1210/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c22e54fb4d45712ec595a0701007dd36bcd4c7ee --- /dev/null +++ b/research/ai_detector_v2/checkpoint-1210/config.json @@ -0,0 +1,63 @@ +{ + "_name_or_path": "umm-maybe/AI-image-detector", + "architectures": [ + "SwinForImageClassification" + ], + "attention_probs_dropout_prob": 0.0, + "depths": [ + 2, + 2, + 18, + 2 + ], + "drop_path_rate": 0.1, + "embed_dim": 128, + "encoder_stride": 32, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.0, + "hidden_size": 1024, + "id2label": { + "0": "ai_gen", + "1": "human" + }, + "image_size": 224, + "initializer_range": 0.02, + "label2id": { + "ai_gen": "0", + "human": "1" + }, + "layer_norm_eps": 1e-05, + "max_length": 128, + "mlp_ratio": 4.0, + "model_type": "swin", + "num_channels": 3, + "num_heads": [ + 4, + 8, + 16, + 32 + ], + "num_layers": 4, + "out_features": [ + "stage4" + ], + "out_indices": [ + 4 + ], + "padding": "max_length", + "patch_size": 4, + "path_norm": true, + "problem_type": "single_label_classification", + "qkv_bias": true, + "stage_names": [ + "stem", + "stage1", + "stage2", + "stage3", + "stage4" + ], + "torch_dtype": "float32", + "transformers_version": "4.37.2", + "use_absolute_embeddings": false, + "window_size": 7 +} diff --git a/research/ai_detector_v2/checkpoint-1210/model.safetensors b/research/ai_detector_v2/checkpoint-1210/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..11926ed48a7962ec76d2c0d73ff3446164ff296b --- /dev/null +++ b/research/ai_detector_v2/checkpoint-1210/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56c806ccd482578edc7d4bdddd66e768243a958ccef4dacca5a40f90a996175c +size 347498816 diff --git a/research/ai_detector_v2/checkpoint-1210/optimizer.pt b/research/ai_detector_v2/checkpoint-1210/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2e3eda248222519e82c02b1cdda68eab05d15ba --- /dev/null +++ b/research/ai_detector_v2/checkpoint-1210/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a354c6fadbd0a0cd8312faca439578b35e360eb92978f86758071123c0a0cb9 +size 694317645 diff --git a/research/ai_detector_v2/checkpoint-1210/preprocessor_config.json b/research/ai_detector_v2/checkpoint-1210/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..273d24924a8bd10227b532d603484f165dcb0b5d --- /dev/null +++ b/research/ai_detector_v2/checkpoint-1210/preprocessor_config.json @@ -0,0 +1,22 @@ +{ + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "ViTImageProcessor", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "resample": 2, + "rescale_factor": 0.00392156862745098, + "size": { + "height": 224, + "width": 224 + } +} diff --git a/research/ai_detector_v2/checkpoint-1210/rng_state.pth b/research/ai_detector_v2/checkpoint-1210/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..b56618d4834933a78834cc5f616edb50985d7265 Binary files /dev/null and b/research/ai_detector_v2/checkpoint-1210/rng_state.pth differ diff --git a/research/ai_detector_v2/checkpoint-1210/scheduler.pt b/research/ai_detector_v2/checkpoint-1210/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..090d414b56dba51fbbd6343716d49c119665bbe9 Binary files /dev/null and b/research/ai_detector_v2/checkpoint-1210/scheduler.pt differ diff --git a/research/ai_detector_v2/checkpoint-1210/trainer_state.json b/research/ai_detector_v2/checkpoint-1210/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..05ebd907a50698dcbe9abc021383fd855a81bf79 --- /dev/null +++ b/research/ai_detector_v2/checkpoint-1210/trainer_state.json @@ -0,0 +1,765 @@ +{ + "best_metric": 0.9980530178223753, + "best_model_checkpoint": "ai_detector_v2/checkpoint-1210", + "epoch": 2.0, + "eval_steps": 500, + "global_step": 1210, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.02, + "learning_rate": 2.747252747252747e-06, + "loss": 0.7391, + "step": 10 + }, + { + "epoch": 0.03, + "learning_rate": 5.494505494505494e-06, + "loss": 0.4175, + "step": 20 + }, + { + "epoch": 0.05, + "learning_rate": 8.241758241758243e-06, + "loss": 0.3476, + "step": 30 + }, + { + "epoch": 0.07, + "learning_rate": 1.0989010989010989e-05, + "loss": 0.1961, + "step": 40 + }, + { + "epoch": 0.08, + "learning_rate": 1.3736263736263738e-05, + "loss": 0.1436, + "step": 50 + }, + { + "epoch": 0.1, + "learning_rate": 1.6483516483516486e-05, + "loss": 0.0595, + "step": 60 + }, + { + "epoch": 0.12, + "learning_rate": 1.923076923076923e-05, + "loss": 0.0585, + "step": 70 + }, + { + "epoch": 0.13, + "learning_rate": 2.1978021978021977e-05, + "loss": 0.0607, + "step": 80 + }, + { + "epoch": 0.15, + "learning_rate": 2.4725274725274727e-05, + "loss": 0.041, + "step": 90 + }, + { + "epoch": 0.17, + "learning_rate": 2.7472527472527476e-05, + "loss": 0.0561, + "step": 100 + }, + { + "epoch": 0.18, + "learning_rate": 3.021978021978022e-05, + "loss": 0.0517, + "step": 110 + }, + { + "epoch": 0.2, + "learning_rate": 3.296703296703297e-05, + "loss": 0.0581, + "step": 120 + }, + { + "epoch": 0.21, + "learning_rate": 3.571428571428572e-05, + "loss": 0.0281, + "step": 130 + }, + { + "epoch": 0.23, + "learning_rate": 3.846153846153846e-05, + "loss": 0.0612, + "step": 140 + }, + { + "epoch": 0.25, + "learning_rate": 4.120879120879121e-05, + "loss": 0.0297, + "step": 150 + }, + { + "epoch": 0.26, + "learning_rate": 4.3956043956043955e-05, + "loss": 0.021, + "step": 160 + }, + { + "epoch": 0.28, + "learning_rate": 4.670329670329671e-05, + "loss": 0.019, + "step": 170 + }, + { + "epoch": 0.3, + "learning_rate": 4.945054945054945e-05, + "loss": 0.0233, + "step": 180 + }, + { + "epoch": 0.31, + "learning_rate": 4.9755052051439074e-05, + "loss": 0.0298, + "step": 190 + }, + { + "epoch": 0.33, + "learning_rate": 4.944886711573791e-05, + "loss": 0.0421, + "step": 200 + }, + { + "epoch": 0.35, + "learning_rate": 4.914268218003674e-05, + "loss": 0.0277, + "step": 210 + }, + { + "epoch": 0.36, + "learning_rate": 4.883649724433558e-05, + "loss": 0.0361, + "step": 220 + }, + { + "epoch": 0.38, + "learning_rate": 4.8530312308634416e-05, + "loss": 0.0579, + "step": 230 + }, + { + "epoch": 0.4, + "learning_rate": 4.822412737293325e-05, + "loss": 0.0338, + "step": 240 + }, + { + "epoch": 0.41, + "learning_rate": 4.791794243723209e-05, + "loss": 0.0366, + "step": 250 + }, + { + "epoch": 0.43, + "learning_rate": 4.761175750153093e-05, + "loss": 0.0092, + "step": 260 + }, + { + "epoch": 0.45, + "learning_rate": 4.7305572565829766e-05, + "loss": 0.063, + "step": 270 + }, + { + "epoch": 0.46, + "learning_rate": 4.69993876301286e-05, + "loss": 0.0527, + "step": 280 + }, + { + "epoch": 0.48, + "learning_rate": 4.6693202694427433e-05, + "loss": 0.023, + "step": 290 + }, + { + "epoch": 0.5, + "learning_rate": 4.6387017758726274e-05, + "loss": 0.0437, + "step": 300 + }, + { + "epoch": 0.51, + "learning_rate": 4.608083282302511e-05, + "loss": 0.0381, + "step": 310 + }, + { + "epoch": 0.53, + "learning_rate": 4.577464788732395e-05, + "loss": 0.0288, + "step": 320 + }, + { + "epoch": 0.55, + "learning_rate": 4.546846295162278e-05, + "loss": 0.0385, + "step": 330 + }, + { + "epoch": 0.56, + "learning_rate": 4.516227801592162e-05, + "loss": 0.0442, + "step": 340 + }, + { + "epoch": 0.58, + "learning_rate": 4.485609308022046e-05, + "loss": 0.0128, + "step": 350 + }, + { + "epoch": 0.6, + "learning_rate": 4.454990814451929e-05, + "loss": 0.0189, + "step": 360 + }, + { + "epoch": 0.61, + "learning_rate": 4.4243723208818125e-05, + "loss": 0.0219, + "step": 370 + }, + { + "epoch": 0.63, + "learning_rate": 4.3937538273116966e-05, + "loss": 0.0296, + "step": 380 + }, + { + "epoch": 0.64, + "learning_rate": 4.363135333741581e-05, + "loss": 0.058, + "step": 390 + }, + { + "epoch": 0.66, + "learning_rate": 4.332516840171464e-05, + "loss": 0.0317, + "step": 400 + }, + { + "epoch": 0.68, + "learning_rate": 4.3018983466013475e-05, + "loss": 0.0364, + "step": 410 + }, + { + "epoch": 0.69, + "learning_rate": 4.271279853031231e-05, + "loss": 0.0239, + "step": 420 + }, + { + "epoch": 0.71, + "learning_rate": 4.240661359461115e-05, + "loss": 0.0265, + "step": 430 + }, + { + "epoch": 0.73, + "learning_rate": 4.210042865890998e-05, + "loss": 0.0046, + "step": 440 + }, + { + "epoch": 0.74, + "learning_rate": 4.179424372320882e-05, + "loss": 0.0182, + "step": 450 + }, + { + "epoch": 0.76, + "learning_rate": 4.148805878750766e-05, + "loss": 0.0255, + "step": 460 + }, + { + "epoch": 0.78, + "learning_rate": 4.118187385180649e-05, + "loss": 0.0143, + "step": 470 + }, + { + "epoch": 0.79, + "learning_rate": 4.087568891610533e-05, + "loss": 0.0348, + "step": 480 + }, + { + "epoch": 0.81, + "learning_rate": 4.0569503980404167e-05, + "loss": 0.035, + "step": 490 + }, + { + "epoch": 0.83, + "learning_rate": 4.0263319044703e-05, + "loss": 0.0397, + "step": 500 + }, + { + "epoch": 0.84, + "learning_rate": 3.9957134109001834e-05, + "loss": 0.0194, + "step": 510 + }, + { + "epoch": 0.86, + "learning_rate": 3.9650949173300675e-05, + "loss": 0.0075, + "step": 520 + }, + { + "epoch": 0.88, + "learning_rate": 3.9344764237599516e-05, + "loss": 0.0025, + "step": 530 + }, + { + "epoch": 0.89, + "learning_rate": 3.903857930189835e-05, + "loss": 0.002, + "step": 540 + }, + { + "epoch": 0.91, + "learning_rate": 3.8732394366197184e-05, + "loss": 0.0093, + "step": 550 + }, + { + "epoch": 0.93, + "learning_rate": 3.8426209430496025e-05, + "loss": 0.0027, + "step": 560 + }, + { + "epoch": 0.94, + "learning_rate": 3.812002449479486e-05, + "loss": 0.011, + "step": 570 + }, + { + "epoch": 0.96, + "learning_rate": 3.781383955909369e-05, + "loss": 0.0133, + "step": 580 + }, + { + "epoch": 0.98, + "learning_rate": 3.7507654623392526e-05, + "loss": 0.0314, + "step": 590 + }, + { + "epoch": 0.99, + "learning_rate": 3.720146968769137e-05, + "loss": 0.0187, + "step": 600 + }, + { + "epoch": 1.0, + "eval_accuracy": 0.9973790624531975, + "eval_loss": 0.010242484509944916, + "eval_runtime": 251.9013, + "eval_samples_per_second": 53.013, + "eval_steps_per_second": 3.315, + "step": 605 + }, + { + "epoch": 1.01, + "learning_rate": 3.689528475199021e-05, + "loss": 0.0335, + "step": 610 + }, + { + "epoch": 1.02, + "learning_rate": 3.658909981628904e-05, + "loss": 0.0224, + "step": 620 + }, + { + "epoch": 1.04, + "learning_rate": 3.6282914880587876e-05, + "loss": 0.0192, + "step": 630 + }, + { + "epoch": 1.06, + "learning_rate": 3.597672994488671e-05, + "loss": 0.0164, + "step": 640 + }, + { + "epoch": 1.07, + "learning_rate": 3.567054500918555e-05, + "loss": 0.0109, + "step": 650 + }, + { + "epoch": 1.09, + "learning_rate": 3.5364360073484384e-05, + "loss": 0.013, + "step": 660 + }, + { + "epoch": 1.11, + "learning_rate": 3.5058175137783225e-05, + "loss": 0.0055, + "step": 670 + }, + { + "epoch": 1.12, + "learning_rate": 3.475199020208206e-05, + "loss": 0.0191, + "step": 680 + }, + { + "epoch": 1.14, + "learning_rate": 3.44458052663809e-05, + "loss": 0.0013, + "step": 690 + }, + { + "epoch": 1.16, + "learning_rate": 3.4139620330679734e-05, + "loss": 0.0186, + "step": 700 + }, + { + "epoch": 1.17, + "learning_rate": 3.383343539497857e-05, + "loss": 0.0001, + "step": 710 + }, + { + "epoch": 1.19, + "learning_rate": 3.35272504592774e-05, + "loss": 0.0127, + "step": 720 + }, + { + "epoch": 1.21, + "learning_rate": 3.322106552357624e-05, + "loss": 0.0052, + "step": 730 + }, + { + "epoch": 1.22, + "learning_rate": 3.291488058787508e-05, + "loss": 0.0278, + "step": 740 + }, + { + "epoch": 1.24, + "learning_rate": 3.260869565217392e-05, + "loss": 0.0015, + "step": 750 + }, + { + "epoch": 1.26, + "learning_rate": 3.230251071647275e-05, + "loss": 0.0026, + "step": 760 + }, + { + "epoch": 1.27, + "learning_rate": 3.1996325780771585e-05, + "loss": 0.0243, + "step": 770 + }, + { + "epoch": 1.29, + "learning_rate": 3.1690140845070426e-05, + "loss": 0.0142, + "step": 780 + }, + { + "epoch": 1.31, + "learning_rate": 3.138395590936926e-05, + "loss": 0.007, + "step": 790 + }, + { + "epoch": 1.32, + "learning_rate": 3.1077770973668093e-05, + "loss": 0.0175, + "step": 800 + }, + { + "epoch": 1.34, + "learning_rate": 3.0771586037966934e-05, + "loss": 0.0064, + "step": 810 + }, + { + "epoch": 1.36, + "learning_rate": 3.046540110226577e-05, + "loss": 0.0168, + "step": 820 + }, + { + "epoch": 1.37, + "learning_rate": 3.015921616656461e-05, + "loss": 0.002, + "step": 830 + }, + { + "epoch": 1.39, + "learning_rate": 2.9853031230863443e-05, + "loss": 0.0129, + "step": 840 + }, + { + "epoch": 1.4, + "learning_rate": 2.9546846295162277e-05, + "loss": 0.0134, + "step": 850 + }, + { + "epoch": 1.42, + "learning_rate": 2.9240661359461114e-05, + "loss": 0.0194, + "step": 860 + }, + { + "epoch": 1.44, + "learning_rate": 2.8934476423759955e-05, + "loss": 0.0096, + "step": 870 + }, + { + "epoch": 1.45, + "learning_rate": 2.862829148805879e-05, + "loss": 0.0048, + "step": 880 + }, + { + "epoch": 1.47, + "learning_rate": 2.8322106552357626e-05, + "loss": 0.0003, + "step": 890 + }, + { + "epoch": 1.49, + "learning_rate": 2.801592161665646e-05, + "loss": 0.0115, + "step": 900 + }, + { + "epoch": 1.5, + "learning_rate": 2.77097366809553e-05, + "loss": 0.0264, + "step": 910 + }, + { + "epoch": 1.52, + "learning_rate": 2.7403551745254135e-05, + "loss": 0.0233, + "step": 920 + }, + { + "epoch": 1.54, + "learning_rate": 2.7097366809552972e-05, + "loss": 0.0008, + "step": 930 + }, + { + "epoch": 1.55, + "learning_rate": 2.6791181873851806e-05, + "loss": 0.0103, + "step": 940 + }, + { + "epoch": 1.57, + "learning_rate": 2.6484996938150647e-05, + "loss": 0.0356, + "step": 950 + }, + { + "epoch": 1.59, + "learning_rate": 2.617881200244948e-05, + "loss": 0.0115, + "step": 960 + }, + { + "epoch": 1.6, + "learning_rate": 2.5872627066748318e-05, + "loss": 0.0229, + "step": 970 + }, + { + "epoch": 1.62, + "learning_rate": 2.5566442131047152e-05, + "loss": 0.0163, + "step": 980 + }, + { + "epoch": 1.64, + "learning_rate": 2.526025719534599e-05, + "loss": 0.0052, + "step": 990 + }, + { + "epoch": 1.65, + "learning_rate": 2.4954072259644827e-05, + "loss": 0.0237, + "step": 1000 + }, + { + "epoch": 1.67, + "learning_rate": 2.4647887323943664e-05, + "loss": 0.0121, + "step": 1010 + }, + { + "epoch": 1.69, + "learning_rate": 2.4341702388242498e-05, + "loss": 0.0022, + "step": 1020 + }, + { + "epoch": 1.7, + "learning_rate": 2.4035517452541335e-05, + "loss": 0.0105, + "step": 1030 + }, + { + "epoch": 1.72, + "learning_rate": 2.3729332516840173e-05, + "loss": 0.0138, + "step": 1040 + }, + { + "epoch": 1.74, + "learning_rate": 2.342314758113901e-05, + "loss": 0.0019, + "step": 1050 + }, + { + "epoch": 1.75, + "learning_rate": 2.3116962645437844e-05, + "loss": 0.0075, + "step": 1060 + }, + { + "epoch": 1.77, + "learning_rate": 2.2810777709736685e-05, + "loss": 0.0103, + "step": 1070 + }, + { + "epoch": 1.79, + "learning_rate": 2.250459277403552e-05, + "loss": 0.0107, + "step": 1080 + }, + { + "epoch": 1.8, + "learning_rate": 2.2198407838334356e-05, + "loss": 0.0018, + "step": 1090 + }, + { + "epoch": 1.82, + "learning_rate": 2.1892222902633193e-05, + "loss": 0.0038, + "step": 1100 + }, + { + "epoch": 1.83, + "learning_rate": 2.1586037966932027e-05, + "loss": 0.0028, + "step": 1110 + }, + { + "epoch": 1.85, + "learning_rate": 2.1279853031230864e-05, + "loss": 0.0095, + "step": 1120 + }, + { + "epoch": 1.87, + "learning_rate": 2.09736680955297e-05, + "loss": 0.0106, + "step": 1130 + }, + { + "epoch": 1.88, + "learning_rate": 2.066748315982854e-05, + "loss": 0.0285, + "step": 1140 + }, + { + "epoch": 1.9, + "learning_rate": 2.0361298224127373e-05, + "loss": 0.017, + "step": 1150 + }, + { + "epoch": 1.92, + "learning_rate": 2.005511328842621e-05, + "loss": 0.0025, + "step": 1160 + }, + { + "epoch": 1.93, + "learning_rate": 1.9748928352725048e-05, + "loss": 0.0079, + "step": 1170 + }, + { + "epoch": 1.95, + "learning_rate": 1.9442743417023885e-05, + "loss": 0.0167, + "step": 1180 + }, + { + "epoch": 1.97, + "learning_rate": 1.913655848132272e-05, + "loss": 0.0202, + "step": 1190 + }, + { + "epoch": 1.98, + "learning_rate": 1.8830373545621556e-05, + "loss": 0.0053, + "step": 1200 + }, + { + "epoch": 2.0, + "learning_rate": 1.8524188609920394e-05, + "loss": 0.0079, + "step": 1210 + }, + { + "epoch": 2.0, + "eval_accuracy": 0.9980530178223753, + "eval_loss": 0.00832486990839243, + "eval_runtime": 254.4173, + "eval_samples_per_second": 52.489, + "eval_steps_per_second": 3.282, + "step": 1210 + } + ], + "logging_steps": 10, + "max_steps": 1815, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 500, + "total_flos": 6.066296372278518e+18, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/research/ai_detector_v2/checkpoint-1210/training_args.bin b/research/ai_detector_v2/checkpoint-1210/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..657dfe02f52791427d6caa7cfb437c50d6956924 Binary files /dev/null and b/research/ai_detector_v2/checkpoint-1210/training_args.bin differ diff --git a/research/ai_detector_v2/checkpoint-1815/config.json b/research/ai_detector_v2/checkpoint-1815/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c22e54fb4d45712ec595a0701007dd36bcd4c7ee --- /dev/null +++ b/research/ai_detector_v2/checkpoint-1815/config.json @@ -0,0 +1,63 @@ +{ + "_name_or_path": "umm-maybe/AI-image-detector", + "architectures": [ + "SwinForImageClassification" + ], + "attention_probs_dropout_prob": 0.0, + "depths": [ + 2, + 2, + 18, + 2 + ], + "drop_path_rate": 0.1, + "embed_dim": 128, + "encoder_stride": 32, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.0, + "hidden_size": 1024, + "id2label": { + "0": "ai_gen", + "1": "human" + }, + "image_size": 224, + "initializer_range": 0.02, + "label2id": { + "ai_gen": "0", + "human": "1" + }, + "layer_norm_eps": 1e-05, + "max_length": 128, + "mlp_ratio": 4.0, + "model_type": "swin", + "num_channels": 3, + "num_heads": [ + 4, + 8, + 16, + 32 + ], + "num_layers": 4, + "out_features": [ + "stage4" + ], + "out_indices": [ + 4 + ], + "padding": "max_length", + "patch_size": 4, + "path_norm": true, + "problem_type": "single_label_classification", + "qkv_bias": true, + "stage_names": [ + "stem", + "stage1", + "stage2", + "stage3", + "stage4" + ], + "torch_dtype": "float32", + "transformers_version": "4.37.2", + "use_absolute_embeddings": false, + "window_size": 7 +} diff --git a/research/ai_detector_v2/checkpoint-1815/model.safetensors b/research/ai_detector_v2/checkpoint-1815/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..555baef60317372ddaf5b991ffb5b32f86bc3ffc --- /dev/null +++ b/research/ai_detector_v2/checkpoint-1815/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9cfc3eefb0ba85fbd001d0b0e6dad70c75cae6afb2f34440f8d0916c58ba262 +size 347498816 diff --git a/research/ai_detector_v2/checkpoint-1815/optimizer.pt b/research/ai_detector_v2/checkpoint-1815/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..9c6c683c43878ef3d406559c45b8e9ee74ede9e6 --- /dev/null +++ b/research/ai_detector_v2/checkpoint-1815/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe1835eb1c6220260837f7b7b350aabef912d71c9ca332eb1b4ae0481d592f5f +size 694317645 diff --git a/research/ai_detector_v2/checkpoint-1815/preprocessor_config.json b/research/ai_detector_v2/checkpoint-1815/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..273d24924a8bd10227b532d603484f165dcb0b5d --- /dev/null +++ b/research/ai_detector_v2/checkpoint-1815/preprocessor_config.json @@ -0,0 +1,22 @@ +{ + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "ViTImageProcessor", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "resample": 2, + "rescale_factor": 0.00392156862745098, + "size": { + "height": 224, + "width": 224 + } +} diff --git a/research/ai_detector_v2/checkpoint-1815/rng_state.pth b/research/ai_detector_v2/checkpoint-1815/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..cffe589d194e5b08f1ea0a2c3ed625e162ff6858 Binary files /dev/null and b/research/ai_detector_v2/checkpoint-1815/rng_state.pth differ diff --git a/research/ai_detector_v2/checkpoint-1815/scheduler.pt b/research/ai_detector_v2/checkpoint-1815/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..117bd1216938dce31e7271469a1a60bf1532ba0d Binary files /dev/null and b/research/ai_detector_v2/checkpoint-1815/scheduler.pt differ diff --git a/research/ai_detector_v2/checkpoint-1815/trainer_state.json b/research/ai_detector_v2/checkpoint-1815/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..29eed32c6bba68411a3c6ab620bca4f961b55f0b --- /dev/null +++ b/research/ai_detector_v2/checkpoint-1815/trainer_state.json @@ -0,0 +1,1134 @@ +{ + "best_metric": 0.9982776696121013, + "best_model_checkpoint": "ai_detector_v2/checkpoint-1815", + "epoch": 3.0, + "eval_steps": 500, + "global_step": 1815, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.02, + "learning_rate": 2.747252747252747e-06, + "loss": 0.7391, + "step": 10 + }, + { + "epoch": 0.03, + "learning_rate": 5.494505494505494e-06, + "loss": 0.4175, + "step": 20 + }, + { + "epoch": 0.05, + "learning_rate": 8.241758241758243e-06, + "loss": 0.3476, + "step": 30 + }, + { + "epoch": 0.07, + "learning_rate": 1.0989010989010989e-05, + "loss": 0.1961, + "step": 40 + }, + { + "epoch": 0.08, + "learning_rate": 1.3736263736263738e-05, + "loss": 0.1436, + "step": 50 + }, + { + "epoch": 0.1, + "learning_rate": 1.6483516483516486e-05, + "loss": 0.0595, + "step": 60 + }, + { + "epoch": 0.12, + "learning_rate": 1.923076923076923e-05, + "loss": 0.0585, + "step": 70 + }, + { + "epoch": 0.13, + "learning_rate": 2.1978021978021977e-05, + "loss": 0.0607, + "step": 80 + }, + { + "epoch": 0.15, + "learning_rate": 2.4725274725274727e-05, + "loss": 0.041, + "step": 90 + }, + { + "epoch": 0.17, + "learning_rate": 2.7472527472527476e-05, + "loss": 0.0561, + "step": 100 + }, + { + "epoch": 0.18, + "learning_rate": 3.021978021978022e-05, + "loss": 0.0517, + "step": 110 + }, + { + "epoch": 0.2, + "learning_rate": 3.296703296703297e-05, + "loss": 0.0581, + "step": 120 + }, + { + "epoch": 0.21, + "learning_rate": 3.571428571428572e-05, + "loss": 0.0281, + "step": 130 + }, + { + "epoch": 0.23, + "learning_rate": 3.846153846153846e-05, + "loss": 0.0612, + "step": 140 + }, + { + "epoch": 0.25, + "learning_rate": 4.120879120879121e-05, + "loss": 0.0297, + "step": 150 + }, + { + "epoch": 0.26, + "learning_rate": 4.3956043956043955e-05, + "loss": 0.021, + "step": 160 + }, + { + "epoch": 0.28, + "learning_rate": 4.670329670329671e-05, + "loss": 0.019, + "step": 170 + }, + { + "epoch": 0.3, + "learning_rate": 4.945054945054945e-05, + "loss": 0.0233, + "step": 180 + }, + { + "epoch": 0.31, + "learning_rate": 4.9755052051439074e-05, + "loss": 0.0298, + "step": 190 + }, + { + "epoch": 0.33, + "learning_rate": 4.944886711573791e-05, + "loss": 0.0421, + "step": 200 + }, + { + "epoch": 0.35, + "learning_rate": 4.914268218003674e-05, + "loss": 0.0277, + "step": 210 + }, + { + "epoch": 0.36, + "learning_rate": 4.883649724433558e-05, + "loss": 0.0361, + "step": 220 + }, + { + "epoch": 0.38, + "learning_rate": 4.8530312308634416e-05, + "loss": 0.0579, + "step": 230 + }, + { + "epoch": 0.4, + "learning_rate": 4.822412737293325e-05, + "loss": 0.0338, + "step": 240 + }, + { + "epoch": 0.41, + "learning_rate": 4.791794243723209e-05, + "loss": 0.0366, + "step": 250 + }, + { + "epoch": 0.43, + "learning_rate": 4.761175750153093e-05, + "loss": 0.0092, + "step": 260 + }, + { + "epoch": 0.45, + "learning_rate": 4.7305572565829766e-05, + "loss": 0.063, + "step": 270 + }, + { + "epoch": 0.46, + "learning_rate": 4.69993876301286e-05, + "loss": 0.0527, + "step": 280 + }, + { + "epoch": 0.48, + "learning_rate": 4.6693202694427433e-05, + "loss": 0.023, + "step": 290 + }, + { + "epoch": 0.5, + "learning_rate": 4.6387017758726274e-05, + "loss": 0.0437, + "step": 300 + }, + { + "epoch": 0.51, + "learning_rate": 4.608083282302511e-05, + "loss": 0.0381, + "step": 310 + }, + { + "epoch": 0.53, + "learning_rate": 4.577464788732395e-05, + "loss": 0.0288, + "step": 320 + }, + { + "epoch": 0.55, + "learning_rate": 4.546846295162278e-05, + "loss": 0.0385, + "step": 330 + }, + { + "epoch": 0.56, + "learning_rate": 4.516227801592162e-05, + "loss": 0.0442, + "step": 340 + }, + { + "epoch": 0.58, + "learning_rate": 4.485609308022046e-05, + "loss": 0.0128, + "step": 350 + }, + { + "epoch": 0.6, + "learning_rate": 4.454990814451929e-05, + "loss": 0.0189, + "step": 360 + }, + { + "epoch": 0.61, + "learning_rate": 4.4243723208818125e-05, + "loss": 0.0219, + "step": 370 + }, + { + "epoch": 0.63, + "learning_rate": 4.3937538273116966e-05, + "loss": 0.0296, + "step": 380 + }, + { + "epoch": 0.64, + "learning_rate": 4.363135333741581e-05, + "loss": 0.058, + "step": 390 + }, + { + "epoch": 0.66, + "learning_rate": 4.332516840171464e-05, + "loss": 0.0317, + "step": 400 + }, + { + "epoch": 0.68, + "learning_rate": 4.3018983466013475e-05, + "loss": 0.0364, + "step": 410 + }, + { + "epoch": 0.69, + "learning_rate": 4.271279853031231e-05, + "loss": 0.0239, + "step": 420 + }, + { + "epoch": 0.71, + "learning_rate": 4.240661359461115e-05, + "loss": 0.0265, + "step": 430 + }, + { + "epoch": 0.73, + "learning_rate": 4.210042865890998e-05, + "loss": 0.0046, + "step": 440 + }, + { + "epoch": 0.74, + "learning_rate": 4.179424372320882e-05, + "loss": 0.0182, + "step": 450 + }, + { + "epoch": 0.76, + "learning_rate": 4.148805878750766e-05, + "loss": 0.0255, + "step": 460 + }, + { + "epoch": 0.78, + "learning_rate": 4.118187385180649e-05, + "loss": 0.0143, + "step": 470 + }, + { + "epoch": 0.79, + "learning_rate": 4.087568891610533e-05, + "loss": 0.0348, + "step": 480 + }, + { + "epoch": 0.81, + "learning_rate": 4.0569503980404167e-05, + "loss": 0.035, + "step": 490 + }, + { + "epoch": 0.83, + "learning_rate": 4.0263319044703e-05, + "loss": 0.0397, + "step": 500 + }, + { + "epoch": 0.84, + "learning_rate": 3.9957134109001834e-05, + "loss": 0.0194, + "step": 510 + }, + { + "epoch": 0.86, + "learning_rate": 3.9650949173300675e-05, + "loss": 0.0075, + "step": 520 + }, + { + "epoch": 0.88, + "learning_rate": 3.9344764237599516e-05, + "loss": 0.0025, + "step": 530 + }, + { + "epoch": 0.89, + "learning_rate": 3.903857930189835e-05, + "loss": 0.002, + "step": 540 + }, + { + "epoch": 0.91, + "learning_rate": 3.8732394366197184e-05, + "loss": 0.0093, + "step": 550 + }, + { + "epoch": 0.93, + "learning_rate": 3.8426209430496025e-05, + "loss": 0.0027, + "step": 560 + }, + { + "epoch": 0.94, + "learning_rate": 3.812002449479486e-05, + "loss": 0.011, + "step": 570 + }, + { + "epoch": 0.96, + "learning_rate": 3.781383955909369e-05, + "loss": 0.0133, + "step": 580 + }, + { + "epoch": 0.98, + "learning_rate": 3.7507654623392526e-05, + "loss": 0.0314, + "step": 590 + }, + { + "epoch": 0.99, + "learning_rate": 3.720146968769137e-05, + "loss": 0.0187, + "step": 600 + }, + { + "epoch": 1.0, + "eval_accuracy": 0.9973790624531975, + "eval_loss": 0.010242484509944916, + "eval_runtime": 251.9013, + "eval_samples_per_second": 53.013, + "eval_steps_per_second": 3.315, + "step": 605 + }, + { + "epoch": 1.01, + "learning_rate": 3.689528475199021e-05, + "loss": 0.0335, + "step": 610 + }, + { + "epoch": 1.02, + "learning_rate": 3.658909981628904e-05, + "loss": 0.0224, + "step": 620 + }, + { + "epoch": 1.04, + "learning_rate": 3.6282914880587876e-05, + "loss": 0.0192, + "step": 630 + }, + { + "epoch": 1.06, + "learning_rate": 3.597672994488671e-05, + "loss": 0.0164, + "step": 640 + }, + { + "epoch": 1.07, + "learning_rate": 3.567054500918555e-05, + "loss": 0.0109, + "step": 650 + }, + { + "epoch": 1.09, + "learning_rate": 3.5364360073484384e-05, + "loss": 0.013, + "step": 660 + }, + { + "epoch": 1.11, + "learning_rate": 3.5058175137783225e-05, + "loss": 0.0055, + "step": 670 + }, + { + "epoch": 1.12, + "learning_rate": 3.475199020208206e-05, + "loss": 0.0191, + "step": 680 + }, + { + "epoch": 1.14, + "learning_rate": 3.44458052663809e-05, + "loss": 0.0013, + "step": 690 + }, + { + "epoch": 1.16, + "learning_rate": 3.4139620330679734e-05, + "loss": 0.0186, + "step": 700 + }, + { + "epoch": 1.17, + "learning_rate": 3.383343539497857e-05, + "loss": 0.0001, + "step": 710 + }, + { + "epoch": 1.19, + "learning_rate": 3.35272504592774e-05, + "loss": 0.0127, + "step": 720 + }, + { + "epoch": 1.21, + "learning_rate": 3.322106552357624e-05, + "loss": 0.0052, + "step": 730 + }, + { + "epoch": 1.22, + "learning_rate": 3.291488058787508e-05, + "loss": 0.0278, + "step": 740 + }, + { + "epoch": 1.24, + "learning_rate": 3.260869565217392e-05, + "loss": 0.0015, + "step": 750 + }, + { + "epoch": 1.26, + "learning_rate": 3.230251071647275e-05, + "loss": 0.0026, + "step": 760 + }, + { + "epoch": 1.27, + "learning_rate": 3.1996325780771585e-05, + "loss": 0.0243, + "step": 770 + }, + { + "epoch": 1.29, + "learning_rate": 3.1690140845070426e-05, + "loss": 0.0142, + "step": 780 + }, + { + "epoch": 1.31, + "learning_rate": 3.138395590936926e-05, + "loss": 0.007, + "step": 790 + }, + { + "epoch": 1.32, + "learning_rate": 3.1077770973668093e-05, + "loss": 0.0175, + "step": 800 + }, + { + "epoch": 1.34, + "learning_rate": 3.0771586037966934e-05, + "loss": 0.0064, + "step": 810 + }, + { + "epoch": 1.36, + "learning_rate": 3.046540110226577e-05, + "loss": 0.0168, + "step": 820 + }, + { + "epoch": 1.37, + "learning_rate": 3.015921616656461e-05, + "loss": 0.002, + "step": 830 + }, + { + "epoch": 1.39, + "learning_rate": 2.9853031230863443e-05, + "loss": 0.0129, + "step": 840 + }, + { + "epoch": 1.4, + "learning_rate": 2.9546846295162277e-05, + "loss": 0.0134, + "step": 850 + }, + { + "epoch": 1.42, + "learning_rate": 2.9240661359461114e-05, + "loss": 0.0194, + "step": 860 + }, + { + "epoch": 1.44, + "learning_rate": 2.8934476423759955e-05, + "loss": 0.0096, + "step": 870 + }, + { + "epoch": 1.45, + "learning_rate": 2.862829148805879e-05, + "loss": 0.0048, + "step": 880 + }, + { + "epoch": 1.47, + "learning_rate": 2.8322106552357626e-05, + "loss": 0.0003, + "step": 890 + }, + { + "epoch": 1.49, + "learning_rate": 2.801592161665646e-05, + "loss": 0.0115, + "step": 900 + }, + { + "epoch": 1.5, + "learning_rate": 2.77097366809553e-05, + "loss": 0.0264, + "step": 910 + }, + { + "epoch": 1.52, + "learning_rate": 2.7403551745254135e-05, + "loss": 0.0233, + "step": 920 + }, + { + "epoch": 1.54, + "learning_rate": 2.7097366809552972e-05, + "loss": 0.0008, + "step": 930 + }, + { + "epoch": 1.55, + "learning_rate": 2.6791181873851806e-05, + "loss": 0.0103, + "step": 940 + }, + { + "epoch": 1.57, + "learning_rate": 2.6484996938150647e-05, + "loss": 0.0356, + "step": 950 + }, + { + "epoch": 1.59, + "learning_rate": 2.617881200244948e-05, + "loss": 0.0115, + "step": 960 + }, + { + "epoch": 1.6, + "learning_rate": 2.5872627066748318e-05, + "loss": 0.0229, + "step": 970 + }, + { + "epoch": 1.62, + "learning_rate": 2.5566442131047152e-05, + "loss": 0.0163, + "step": 980 + }, + { + "epoch": 1.64, + "learning_rate": 2.526025719534599e-05, + "loss": 0.0052, + "step": 990 + }, + { + "epoch": 1.65, + "learning_rate": 2.4954072259644827e-05, + "loss": 0.0237, + "step": 1000 + }, + { + "epoch": 1.67, + "learning_rate": 2.4647887323943664e-05, + "loss": 0.0121, + "step": 1010 + }, + { + "epoch": 1.69, + "learning_rate": 2.4341702388242498e-05, + "loss": 0.0022, + "step": 1020 + }, + { + "epoch": 1.7, + "learning_rate": 2.4035517452541335e-05, + "loss": 0.0105, + "step": 1030 + }, + { + "epoch": 1.72, + "learning_rate": 2.3729332516840173e-05, + "loss": 0.0138, + "step": 1040 + }, + { + "epoch": 1.74, + "learning_rate": 2.342314758113901e-05, + "loss": 0.0019, + "step": 1050 + }, + { + "epoch": 1.75, + "learning_rate": 2.3116962645437844e-05, + "loss": 0.0075, + "step": 1060 + }, + { + "epoch": 1.77, + "learning_rate": 2.2810777709736685e-05, + "loss": 0.0103, + "step": 1070 + }, + { + "epoch": 1.79, + "learning_rate": 2.250459277403552e-05, + "loss": 0.0107, + "step": 1080 + }, + { + "epoch": 1.8, + "learning_rate": 2.2198407838334356e-05, + "loss": 0.0018, + "step": 1090 + }, + { + "epoch": 1.82, + "learning_rate": 2.1892222902633193e-05, + "loss": 0.0038, + "step": 1100 + }, + { + "epoch": 1.83, + "learning_rate": 2.1586037966932027e-05, + "loss": 0.0028, + "step": 1110 + }, + { + "epoch": 1.85, + "learning_rate": 2.1279853031230864e-05, + "loss": 0.0095, + "step": 1120 + }, + { + "epoch": 1.87, + "learning_rate": 2.09736680955297e-05, + "loss": 0.0106, + "step": 1130 + }, + { + "epoch": 1.88, + "learning_rate": 2.066748315982854e-05, + "loss": 0.0285, + "step": 1140 + }, + { + "epoch": 1.9, + "learning_rate": 2.0361298224127373e-05, + "loss": 0.017, + "step": 1150 + }, + { + "epoch": 1.92, + "learning_rate": 2.005511328842621e-05, + "loss": 0.0025, + "step": 1160 + }, + { + "epoch": 1.93, + "learning_rate": 1.9748928352725048e-05, + "loss": 0.0079, + "step": 1170 + }, + { + "epoch": 1.95, + "learning_rate": 1.9442743417023885e-05, + "loss": 0.0167, + "step": 1180 + }, + { + "epoch": 1.97, + "learning_rate": 1.913655848132272e-05, + "loss": 0.0202, + "step": 1190 + }, + { + "epoch": 1.98, + "learning_rate": 1.8830373545621556e-05, + "loss": 0.0053, + "step": 1200 + }, + { + "epoch": 2.0, + "learning_rate": 1.8524188609920394e-05, + "loss": 0.0079, + "step": 1210 + }, + { + "epoch": 2.0, + "eval_accuracy": 0.9980530178223753, + "eval_loss": 0.00832486990839243, + "eval_runtime": 254.4173, + "eval_samples_per_second": 52.489, + "eval_steps_per_second": 3.282, + "step": 1210 + }, + { + "epoch": 2.02, + "learning_rate": 1.821800367421923e-05, + "loss": 0.0127, + "step": 1220 + }, + { + "epoch": 2.03, + "learning_rate": 1.7911818738518065e-05, + "loss": 0.0273, + "step": 1230 + }, + { + "epoch": 2.05, + "learning_rate": 1.7605633802816902e-05, + "loss": 0.0085, + "step": 1240 + }, + { + "epoch": 2.07, + "learning_rate": 1.729944886711574e-05, + "loss": 0.0045, + "step": 1250 + }, + { + "epoch": 2.08, + "learning_rate": 1.6993263931414574e-05, + "loss": 0.0048, + "step": 1260 + }, + { + "epoch": 2.1, + "learning_rate": 1.668707899571341e-05, + "loss": 0.0016, + "step": 1270 + }, + { + "epoch": 2.12, + "learning_rate": 1.6380894060012248e-05, + "loss": 0.0012, + "step": 1280 + }, + { + "epoch": 2.13, + "learning_rate": 1.6074709124311086e-05, + "loss": 0.0065, + "step": 1290 + }, + { + "epoch": 2.15, + "learning_rate": 1.576852418860992e-05, + "loss": 0.005, + "step": 1300 + }, + { + "epoch": 2.17, + "learning_rate": 1.5462339252908757e-05, + "loss": 0.0421, + "step": 1310 + }, + { + "epoch": 2.18, + "learning_rate": 1.5156154317207594e-05, + "loss": 0.0077, + "step": 1320 + }, + { + "epoch": 2.2, + "learning_rate": 1.4849969381506432e-05, + "loss": 0.0012, + "step": 1330 + }, + { + "epoch": 2.21, + "learning_rate": 1.4543784445805267e-05, + "loss": 0.0091, + "step": 1340 + }, + { + "epoch": 2.23, + "learning_rate": 1.4237599510104105e-05, + "loss": 0.0091, + "step": 1350 + }, + { + "epoch": 2.25, + "learning_rate": 1.393141457440294e-05, + "loss": 0.0256, + "step": 1360 + }, + { + "epoch": 2.26, + "learning_rate": 1.3625229638701776e-05, + "loss": 0.0114, + "step": 1370 + }, + { + "epoch": 2.28, + "learning_rate": 1.3319044703000613e-05, + "loss": 0.0059, + "step": 1380 + }, + { + "epoch": 2.3, + "learning_rate": 1.3012859767299449e-05, + "loss": 0.0012, + "step": 1390 + }, + { + "epoch": 2.31, + "learning_rate": 1.2706674831598286e-05, + "loss": 0.0031, + "step": 1400 + }, + { + "epoch": 2.33, + "learning_rate": 1.2400489895897123e-05, + "loss": 0.0005, + "step": 1410 + }, + { + "epoch": 2.35, + "learning_rate": 1.2094304960195959e-05, + "loss": 0.0013, + "step": 1420 + }, + { + "epoch": 2.36, + "learning_rate": 1.1788120024494795e-05, + "loss": 0.0006, + "step": 1430 + }, + { + "epoch": 2.38, + "learning_rate": 1.1481935088793632e-05, + "loss": 0.011, + "step": 1440 + }, + { + "epoch": 2.4, + "learning_rate": 1.1175750153092468e-05, + "loss": 0.0173, + "step": 1450 + }, + { + "epoch": 2.41, + "learning_rate": 1.0869565217391305e-05, + "loss": 0.0077, + "step": 1460 + }, + { + "epoch": 2.43, + "learning_rate": 1.056338028169014e-05, + "loss": 0.0007, + "step": 1470 + }, + { + "epoch": 2.45, + "learning_rate": 1.0257195345988978e-05, + "loss": 0.0077, + "step": 1480 + }, + { + "epoch": 2.46, + "learning_rate": 9.951010410287814e-06, + "loss": 0.0093, + "step": 1490 + }, + { + "epoch": 2.48, + "learning_rate": 9.644825474586651e-06, + "loss": 0.01, + "step": 1500 + }, + { + "epoch": 2.5, + "learning_rate": 9.338640538885487e-06, + "loss": 0.0016, + "step": 1510 + }, + { + "epoch": 2.51, + "learning_rate": 9.032455603184324e-06, + "loss": 0.0001, + "step": 1520 + }, + { + "epoch": 2.53, + "learning_rate": 8.726270667483161e-06, + "loss": 0.0068, + "step": 1530 + }, + { + "epoch": 2.55, + "learning_rate": 8.420085731781997e-06, + "loss": 0.0007, + "step": 1540 + }, + { + "epoch": 2.56, + "learning_rate": 8.113900796080834e-06, + "loss": 0.0093, + "step": 1550 + }, + { + "epoch": 2.58, + "learning_rate": 7.807715860379668e-06, + "loss": 0.0004, + "step": 1560 + }, + { + "epoch": 2.6, + "learning_rate": 7.501530924678506e-06, + "loss": 0.0051, + "step": 1570 + }, + { + "epoch": 2.61, + "learning_rate": 7.195345988977342e-06, + "loss": 0.0046, + "step": 1580 + }, + { + "epoch": 2.63, + "learning_rate": 6.889161053276179e-06, + "loss": 0.0002, + "step": 1590 + }, + { + "epoch": 2.64, + "learning_rate": 6.582976117575015e-06, + "loss": 0.0059, + "step": 1600 + }, + { + "epoch": 2.66, + "learning_rate": 6.276791181873852e-06, + "loss": 0.0038, + "step": 1610 + }, + { + "epoch": 2.68, + "learning_rate": 5.970606246172689e-06, + "loss": 0.0002, + "step": 1620 + }, + { + "epoch": 2.69, + "learning_rate": 5.664421310471525e-06, + "loss": 0.0003, + "step": 1630 + }, + { + "epoch": 2.71, + "learning_rate": 5.358236374770362e-06, + "loss": 0.0048, + "step": 1640 + }, + { + "epoch": 2.73, + "learning_rate": 5.052051439069198e-06, + "loss": 0.001, + "step": 1650 + }, + { + "epoch": 2.74, + "learning_rate": 4.745866503368034e-06, + "loss": 0.0006, + "step": 1660 + }, + { + "epoch": 2.76, + "learning_rate": 4.4396815676668705e-06, + "loss": 0.012, + "step": 1670 + }, + { + "epoch": 2.78, + "learning_rate": 4.133496631965708e-06, + "loss": 0.0002, + "step": 1680 + }, + { + "epoch": 2.79, + "learning_rate": 3.827311696264544e-06, + "loss": 0.0001, + "step": 1690 + }, + { + "epoch": 2.81, + "learning_rate": 3.521126760563381e-06, + "loss": 0.0026, + "step": 1700 + }, + { + "epoch": 2.83, + "learning_rate": 3.2149418248622173e-06, + "loss": 0.0085, + "step": 1710 + }, + { + "epoch": 2.84, + "learning_rate": 2.9087568891610534e-06, + "loss": 0.0096, + "step": 1720 + }, + { + "epoch": 2.86, + "learning_rate": 2.60257195345989e-06, + "loss": 0.0001, + "step": 1730 + }, + { + "epoch": 2.88, + "learning_rate": 2.2963870177587263e-06, + "loss": 0.0076, + "step": 1740 + }, + { + "epoch": 2.89, + "learning_rate": 1.990202082057563e-06, + "loss": 0.0002, + "step": 1750 + }, + { + "epoch": 2.91, + "learning_rate": 1.6840171463563995e-06, + "loss": 0.0044, + "step": 1760 + }, + { + "epoch": 2.93, + "learning_rate": 1.3778322106552358e-06, + "loss": 0.0005, + "step": 1770 + }, + { + "epoch": 2.94, + "learning_rate": 1.0716472749540723e-06, + "loss": 0.0001, + "step": 1780 + }, + { + "epoch": 2.96, + "learning_rate": 7.654623392529088e-07, + "loss": 0.0024, + "step": 1790 + }, + { + "epoch": 2.98, + "learning_rate": 4.5927740355174527e-07, + "loss": 0.0001, + "step": 1800 + }, + { + "epoch": 2.99, + "learning_rate": 1.5309246785058176e-07, + "loss": 0.0002, + "step": 1810 + }, + { + "epoch": 3.0, + "eval_accuracy": 0.9982776696121013, + "eval_loss": 0.007270948030054569, + "eval_runtime": 259.3731, + "eval_samples_per_second": 51.486, + "eval_steps_per_second": 3.219, + "step": 1815 + } + ], + "logging_steps": 10, + "max_steps": 1815, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 500, + "total_flos": 9.099444558417777e+18, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/research/ai_detector_v2/checkpoint-1815/training_args.bin b/research/ai_detector_v2/checkpoint-1815/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..657dfe02f52791427d6caa7cfb437c50d6956924 Binary files /dev/null and b/research/ai_detector_v2/checkpoint-1815/training_args.bin differ diff --git a/research/ai_detector_v2/checkpoint-605/config.json b/research/ai_detector_v2/checkpoint-605/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c22e54fb4d45712ec595a0701007dd36bcd4c7ee --- /dev/null +++ b/research/ai_detector_v2/checkpoint-605/config.json @@ -0,0 +1,63 @@ +{ + "_name_or_path": "umm-maybe/AI-image-detector", + "architectures": [ + "SwinForImageClassification" + ], + "attention_probs_dropout_prob": 0.0, + "depths": [ + 2, + 2, + 18, + 2 + ], + "drop_path_rate": 0.1, + "embed_dim": 128, + "encoder_stride": 32, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.0, + "hidden_size": 1024, + "id2label": { + "0": "ai_gen", + "1": "human" + }, + "image_size": 224, + "initializer_range": 0.02, + "label2id": { + "ai_gen": "0", + "human": "1" + }, + "layer_norm_eps": 1e-05, + "max_length": 128, + "mlp_ratio": 4.0, + "model_type": "swin", + "num_channels": 3, + "num_heads": [ + 4, + 8, + 16, + 32 + ], + "num_layers": 4, + "out_features": [ + "stage4" + ], + "out_indices": [ + 4 + ], + "padding": "max_length", + "patch_size": 4, + "path_norm": true, + "problem_type": "single_label_classification", + "qkv_bias": true, + "stage_names": [ + "stem", + "stage1", + "stage2", + "stage3", + "stage4" + ], + "torch_dtype": "float32", + "transformers_version": "4.37.2", + "use_absolute_embeddings": false, + "window_size": 7 +} diff --git a/research/ai_detector_v2/checkpoint-605/model.safetensors b/research/ai_detector_v2/checkpoint-605/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8d6416de6f83bd2b76e11874c2fbc26829897782 --- /dev/null +++ b/research/ai_detector_v2/checkpoint-605/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a29b82064c6e5bf3f5657a83c36e71b62d7a33a333123f90919c243b2285ed89 +size 347498816 diff --git a/research/ai_detector_v2/checkpoint-605/optimizer.pt b/research/ai_detector_v2/checkpoint-605/optimizer.pt new file mode 100644 index 0000000000000000000000000000000000000000..d74831bec1e18add203b1e66968abe387d8ae9fb --- /dev/null +++ b/research/ai_detector_v2/checkpoint-605/optimizer.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7926d08708895df2e9db701575d1770ec5a9941783292a0ff61156f4297a9dd8 +size 694317645 diff --git a/research/ai_detector_v2/checkpoint-605/preprocessor_config.json b/research/ai_detector_v2/checkpoint-605/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..273d24924a8bd10227b532d603484f165dcb0b5d --- /dev/null +++ b/research/ai_detector_v2/checkpoint-605/preprocessor_config.json @@ -0,0 +1,22 @@ +{ + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "image_mean": [ + 0.5, + 0.5, + 0.5 + ], + "image_processor_type": "ViTImageProcessor", + "image_std": [ + 0.5, + 0.5, + 0.5 + ], + "resample": 2, + "rescale_factor": 0.00392156862745098, + "size": { + "height": 224, + "width": 224 + } +} diff --git a/research/ai_detector_v2/checkpoint-605/rng_state.pth b/research/ai_detector_v2/checkpoint-605/rng_state.pth new file mode 100644 index 0000000000000000000000000000000000000000..eafa8bf238fb4dfdb5181157fa235bae2d7263ae Binary files /dev/null and b/research/ai_detector_v2/checkpoint-605/rng_state.pth differ diff --git a/research/ai_detector_v2/checkpoint-605/scheduler.pt b/research/ai_detector_v2/checkpoint-605/scheduler.pt new file mode 100644 index 0000000000000000000000000000000000000000..fb7fa0363a69b0eda3f9668ef95c9a92dfa601db Binary files /dev/null and b/research/ai_detector_v2/checkpoint-605/scheduler.pt differ diff --git a/research/ai_detector_v2/checkpoint-605/trainer_state.json b/research/ai_detector_v2/checkpoint-605/trainer_state.json new file mode 100644 index 0000000000000000000000000000000000000000..8e913831cbc56e1b5f00424b2668dec86935c979 --- /dev/null +++ b/research/ai_detector_v2/checkpoint-605/trainer_state.json @@ -0,0 +1,390 @@ +{ + "best_metric": 0.9973790624531975, + "best_model_checkpoint": "ai_detector_v2/checkpoint-605", + "epoch": 1.0, + "eval_steps": 500, + "global_step": 605, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.02, + "learning_rate": 2.747252747252747e-06, + "loss": 0.7391, + "step": 10 + }, + { + "epoch": 0.03, + "learning_rate": 5.494505494505494e-06, + "loss": 0.4175, + "step": 20 + }, + { + "epoch": 0.05, + "learning_rate": 8.241758241758243e-06, + "loss": 0.3476, + "step": 30 + }, + { + "epoch": 0.07, + "learning_rate": 1.0989010989010989e-05, + "loss": 0.1961, + "step": 40 + }, + { + "epoch": 0.08, + "learning_rate": 1.3736263736263738e-05, + "loss": 0.1436, + "step": 50 + }, + { + "epoch": 0.1, + "learning_rate": 1.6483516483516486e-05, + "loss": 0.0595, + "step": 60 + }, + { + "epoch": 0.12, + "learning_rate": 1.923076923076923e-05, + "loss": 0.0585, + "step": 70 + }, + { + "epoch": 0.13, + "learning_rate": 2.1978021978021977e-05, + "loss": 0.0607, + "step": 80 + }, + { + "epoch": 0.15, + "learning_rate": 2.4725274725274727e-05, + "loss": 0.041, + "step": 90 + }, + { + "epoch": 0.17, + "learning_rate": 2.7472527472527476e-05, + "loss": 0.0561, + "step": 100 + }, + { + "epoch": 0.18, + "learning_rate": 3.021978021978022e-05, + "loss": 0.0517, + "step": 110 + }, + { + "epoch": 0.2, + "learning_rate": 3.296703296703297e-05, + "loss": 0.0581, + "step": 120 + }, + { + "epoch": 0.21, + "learning_rate": 3.571428571428572e-05, + "loss": 0.0281, + "step": 130 + }, + { + "epoch": 0.23, + "learning_rate": 3.846153846153846e-05, + "loss": 0.0612, + "step": 140 + }, + { + "epoch": 0.25, + "learning_rate": 4.120879120879121e-05, + "loss": 0.0297, + "step": 150 + }, + { + "epoch": 0.26, + "learning_rate": 4.3956043956043955e-05, + "loss": 0.021, + "step": 160 + }, + { + "epoch": 0.28, + "learning_rate": 4.670329670329671e-05, + "loss": 0.019, + "step": 170 + }, + { + "epoch": 0.3, + "learning_rate": 4.945054945054945e-05, + "loss": 0.0233, + "step": 180 + }, + { + "epoch": 0.31, + "learning_rate": 4.9755052051439074e-05, + "loss": 0.0298, + "step": 190 + }, + { + "epoch": 0.33, + "learning_rate": 4.944886711573791e-05, + "loss": 0.0421, + "step": 200 + }, + { + "epoch": 0.35, + "learning_rate": 4.914268218003674e-05, + "loss": 0.0277, + "step": 210 + }, + { + "epoch": 0.36, + "learning_rate": 4.883649724433558e-05, + "loss": 0.0361, + "step": 220 + }, + { + "epoch": 0.38, + "learning_rate": 4.8530312308634416e-05, + "loss": 0.0579, + "step": 230 + }, + { + "epoch": 0.4, + "learning_rate": 4.822412737293325e-05, + "loss": 0.0338, + "step": 240 + }, + { + "epoch": 0.41, + "learning_rate": 4.791794243723209e-05, + "loss": 0.0366, + "step": 250 + }, + { + "epoch": 0.43, + "learning_rate": 4.761175750153093e-05, + "loss": 0.0092, + "step": 260 + }, + { + "epoch": 0.45, + "learning_rate": 4.7305572565829766e-05, + "loss": 0.063, + "step": 270 + }, + { + "epoch": 0.46, + "learning_rate": 4.69993876301286e-05, + "loss": 0.0527, + "step": 280 + }, + { + "epoch": 0.48, + "learning_rate": 4.6693202694427433e-05, + "loss": 0.023, + "step": 290 + }, + { + "epoch": 0.5, + "learning_rate": 4.6387017758726274e-05, + "loss": 0.0437, + "step": 300 + }, + { + "epoch": 0.51, + "learning_rate": 4.608083282302511e-05, + "loss": 0.0381, + "step": 310 + }, + { + "epoch": 0.53, + "learning_rate": 4.577464788732395e-05, + "loss": 0.0288, + "step": 320 + }, + { + "epoch": 0.55, + "learning_rate": 4.546846295162278e-05, + "loss": 0.0385, + "step": 330 + }, + { + "epoch": 0.56, + "learning_rate": 4.516227801592162e-05, + "loss": 0.0442, + "step": 340 + }, + { + "epoch": 0.58, + "learning_rate": 4.485609308022046e-05, + "loss": 0.0128, + "step": 350 + }, + { + "epoch": 0.6, + "learning_rate": 4.454990814451929e-05, + "loss": 0.0189, + "step": 360 + }, + { + "epoch": 0.61, + "learning_rate": 4.4243723208818125e-05, + "loss": 0.0219, + "step": 370 + }, + { + "epoch": 0.63, + "learning_rate": 4.3937538273116966e-05, + "loss": 0.0296, + "step": 380 + }, + { + "epoch": 0.64, + "learning_rate": 4.363135333741581e-05, + "loss": 0.058, + "step": 390 + }, + { + "epoch": 0.66, + "learning_rate": 4.332516840171464e-05, + "loss": 0.0317, + "step": 400 + }, + { + "epoch": 0.68, + "learning_rate": 4.3018983466013475e-05, + "loss": 0.0364, + "step": 410 + }, + { + "epoch": 0.69, + "learning_rate": 4.271279853031231e-05, + "loss": 0.0239, + "step": 420 + }, + { + "epoch": 0.71, + "learning_rate": 4.240661359461115e-05, + "loss": 0.0265, + "step": 430 + }, + { + "epoch": 0.73, + "learning_rate": 4.210042865890998e-05, + "loss": 0.0046, + "step": 440 + }, + { + "epoch": 0.74, + "learning_rate": 4.179424372320882e-05, + "loss": 0.0182, + "step": 450 + }, + { + "epoch": 0.76, + "learning_rate": 4.148805878750766e-05, + "loss": 0.0255, + "step": 460 + }, + { + "epoch": 0.78, + "learning_rate": 4.118187385180649e-05, + "loss": 0.0143, + "step": 470 + }, + { + "epoch": 0.79, + "learning_rate": 4.087568891610533e-05, + "loss": 0.0348, + "step": 480 + }, + { + "epoch": 0.81, + "learning_rate": 4.0569503980404167e-05, + "loss": 0.035, + "step": 490 + }, + { + "epoch": 0.83, + "learning_rate": 4.0263319044703e-05, + "loss": 0.0397, + "step": 500 + }, + { + "epoch": 0.84, + "learning_rate": 3.9957134109001834e-05, + "loss": 0.0194, + "step": 510 + }, + { + "epoch": 0.86, + "learning_rate": 3.9650949173300675e-05, + "loss": 0.0075, + "step": 520 + }, + { + "epoch": 0.88, + "learning_rate": 3.9344764237599516e-05, + "loss": 0.0025, + "step": 530 + }, + { + "epoch": 0.89, + "learning_rate": 3.903857930189835e-05, + "loss": 0.002, + "step": 540 + }, + { + "epoch": 0.91, + "learning_rate": 3.8732394366197184e-05, + "loss": 0.0093, + "step": 550 + }, + { + "epoch": 0.93, + "learning_rate": 3.8426209430496025e-05, + "loss": 0.0027, + "step": 560 + }, + { + "epoch": 0.94, + "learning_rate": 3.812002449479486e-05, + "loss": 0.011, + "step": 570 + }, + { + "epoch": 0.96, + "learning_rate": 3.781383955909369e-05, + "loss": 0.0133, + "step": 580 + }, + { + "epoch": 0.98, + "learning_rate": 3.7507654623392526e-05, + "loss": 0.0314, + "step": 590 + }, + { + "epoch": 0.99, + "learning_rate": 3.720146968769137e-05, + "loss": 0.0187, + "step": 600 + }, + { + "epoch": 1.0, + "eval_accuracy": 0.9973790624531975, + "eval_loss": 0.010242484509944916, + "eval_runtime": 251.9013, + "eval_samples_per_second": 53.013, + "eval_steps_per_second": 3.315, + "step": 605 + } + ], + "logging_steps": 10, + "max_steps": 1815, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 500, + "total_flos": 3.033148186139259e+18, + "train_batch_size": 16, + "trial_name": null, + "trial_params": null +} diff --git a/research/ai_detector_v2/checkpoint-605/training_args.bin b/research/ai_detector_v2/checkpoint-605/training_args.bin new file mode 100644 index 0000000000000000000000000000000000000000..657dfe02f52791427d6caa7cfb437c50d6956924 Binary files /dev/null and b/research/ai_detector_v2/checkpoint-605/training_args.bin differ diff --git a/research/dataset/dataset_dict.json b/research/dataset/dataset_dict.json new file mode 100644 index 0000000000000000000000000000000000000000..9195703312e22d2b9b9fe14951aa733949480a2d --- /dev/null +++ b/research/dataset/dataset_dict.json @@ -0,0 +1 @@ +{"splits": ["train", "validation", "test"]} \ No newline at end of file diff --git a/research/dataset/test/data-00000-of-00013.arrow b/research/dataset/test/data-00000-of-00013.arrow new file mode 100644 index 0000000000000000000000000000000000000000..9ad013a25cff8c2a0262a7b8ad21dd6d2601ac0f --- /dev/null +++ b/research/dataset/test/data-00000-of-00013.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6e9a2ceeb207f54bbdd077b22a2c3febd412ee19b1b52f7216abf4cfbcf2e74 +size 1135142488 diff --git a/research/dataset/test/data-00001-of-00013.arrow b/research/dataset/test/data-00001-of-00013.arrow new file mode 100644 index 0000000000000000000000000000000000000000..63b52b81fdb59eb29596b592728195a994e4a78d --- /dev/null +++ b/research/dataset/test/data-00001-of-00013.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ab8969d3d1d6c18876aa6f489d5211ccf87340375df9a40e95a3793b6325642 +size 1197424344 diff --git a/research/dataset/test/data-00002-of-00013.arrow b/research/dataset/test/data-00002-of-00013.arrow new file mode 100644 index 0000000000000000000000000000000000000000..9b47048dcb83129977c3a77443e668b277a17d13 --- /dev/null +++ b/research/dataset/test/data-00002-of-00013.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cc4d2eaff85bb3403407c92aeb3951c865ae6e1fb65fcc5d956b029d9711109 +size 1192652856 diff --git a/research/dataset/test/data-00003-of-00013.arrow b/research/dataset/test/data-00003-of-00013.arrow new file mode 100644 index 0000000000000000000000000000000000000000..545048412e8a57b5d9d6ce700e4da1765f78e4d7 --- /dev/null +++ b/research/dataset/test/data-00003-of-00013.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3680d196e1439121dda2b1905ae11a18bf00e07bb303bdadbe82fa730df379f1 +size 1267263520 diff --git a/research/dataset/test/data-00004-of-00013.arrow b/research/dataset/test/data-00004-of-00013.arrow new file mode 100644 index 0000000000000000000000000000000000000000..8a19226150ba5d5fd552e59845429317a23c619b --- /dev/null +++ b/research/dataset/test/data-00004-of-00013.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21c65f14afc01c16a0922b30f41b799f73096677f365574f28e1b780a08498e4 +size 437015360 diff --git a/research/dataset/test/data-00005-of-00013.arrow b/research/dataset/test/data-00005-of-00013.arrow new file mode 100644 index 0000000000000000000000000000000000000000..b7ab4cd9315386f393938cefe089fcee87b4804d --- /dev/null +++ b/research/dataset/test/data-00005-of-00013.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51028e0463f78049d8a1108988588c99bec78184cd0498644cacca02d99b28bf +size 126114272 diff --git a/research/dataset/test/data-00006-of-00013.arrow b/research/dataset/test/data-00006-of-00013.arrow new file mode 100644 index 0000000000000000000000000000000000000000..976584b1019f1b5f8177a9535a6e496b5b6f4787 --- /dev/null +++ b/research/dataset/test/data-00006-of-00013.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90d3899591c0c4f540198527b1449adee32c11e62d6b59ce1d0eaa45f1d3dc2a +size 103618744 diff --git a/research/dataset/test/data-00007-of-00013.arrow b/research/dataset/test/data-00007-of-00013.arrow new file mode 100644 index 0000000000000000000000000000000000000000..6e5bbe82ef068f0a2b3ecd7d08470e7e2e0d6009 --- /dev/null +++ b/research/dataset/test/data-00007-of-00013.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7672685be497b99975b9c3132b29beb0ad98ebefb37d78544685ab1a4d2c6b4e +size 116825272 diff --git a/research/dataset/test/data-00008-of-00013.arrow b/research/dataset/test/data-00008-of-00013.arrow new file mode 100644 index 0000000000000000000000000000000000000000..e9c1a05d68e4e5f8a6b06e28ffd841fc5ae771f2 --- /dev/null +++ b/research/dataset/test/data-00008-of-00013.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d58e9f577d26e1c65ee30cbc1700812f927083c94fdeca604fa59bb2d0061df5 +size 106812376 diff --git a/research/dataset/test/data-00009-of-00013.arrow b/research/dataset/test/data-00009-of-00013.arrow new file mode 100644 index 0000000000000000000000000000000000000000..7a8a4450fc42bc587ae2b7c08e482cfa6be3c284 --- /dev/null +++ b/research/dataset/test/data-00009-of-00013.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:663f63f11ec40434e1c24e9432b4d17f97de15ec1bc35923fdb72213abc06a9b +size 101042408 diff --git a/research/dataset/test/data-00010-of-00013.arrow b/research/dataset/test/data-00010-of-00013.arrow new file mode 100644 index 0000000000000000000000000000000000000000..a4279080b267215d06b08a10eea9aa6a4bfbaa1e --- /dev/null +++ b/research/dataset/test/data-00010-of-00013.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fabceb9408d3fd0058b177d105b3babe7c37bd4c09a1c45365f9dc54c6a1dfd9 +size 104948984 diff --git a/research/dataset/test/data-00011-of-00013.arrow b/research/dataset/test/data-00011-of-00013.arrow new file mode 100644 index 0000000000000000000000000000000000000000..8578da7f5d420e550186c98db547d2646165fe19 --- /dev/null +++ b/research/dataset/test/data-00011-of-00013.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f258bbd1db9399eebcc405d708e62e8b44f1dcd13e9f9a9b3ec95c072f22c669 +size 107268248 diff --git a/research/dataset/test/data-00012-of-00013.arrow b/research/dataset/test/data-00012-of-00013.arrow new file mode 100644 index 0000000000000000000000000000000000000000..7bf7fb93efec17ea72a953a3b1267eb11e1e6fa1 --- /dev/null +++ b/research/dataset/test/data-00012-of-00013.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99f5115e3d851d6479a26e685d8062e9a649bf39a6174aac865d80cab3a3cb45 +size 117276200 diff --git a/research/dataset/test/dataset_info.json b/research/dataset/test/dataset_info.json new file mode 100644 index 0000000000000000000000000000000000000000..1f7bfeddcb56cccbadfa192d8f4c025994e51555 --- /dev/null +++ b/research/dataset/test/dataset_info.json @@ -0,0 +1,566 @@ +{ + "builder_name": "parquet", + "citation": "", + "config_name": "default", + "dataset_name": "ai_generated_images_midjourney", + "dataset_size": 30136745063, + "description": "", + "download_checksums": { + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00000-of-00041.parquet": { + "num_bytes": 538696797, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00001-of-00041.parquet": { + "num_bytes": 461123819, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00002-of-00041.parquet": { + "num_bytes": 470632938, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00003-of-00041.parquet": { + "num_bytes": 531704324, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00004-of-00041.parquet": { + "num_bytes": 536933200, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00005-of-00041.parquet": { + "num_bytes": 501545607, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00006-of-00041.parquet": { + "num_bytes": 475535302, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00007-of-00041.parquet": { + "num_bytes": 539136860, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00008-of-00041.parquet": { + "num_bytes": 530528421, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00009-of-00041.parquet": { + "num_bytes": 457521582, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00010-of-00041.parquet": { + "num_bytes": 480092663, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00011-of-00041.parquet": { + "num_bytes": 566820126, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00012-of-00041.parquet": { + "num_bytes": 590606849, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00013-of-00041.parquet": { + "num_bytes": 565073338, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00014-of-00041.parquet": { + "num_bytes": 500476843, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00015-of-00041.parquet": { + "num_bytes": 482574110, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00016-of-00041.parquet": { + "num_bytes": 693849912, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00017-of-00041.parquet": { + "num_bytes": 513087321, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00018-of-00041.parquet": { + "num_bytes": 620252104, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00019-of-00041.parquet": { + "num_bytes": 528086346, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00020-of-00041.parquet": { + "num_bytes": 365200077, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00021-of-00041.parquet": { + "num_bytes": 54093315, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00022-of-00041.parquet": { + "num_bytes": 55640622, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00023-of-00041.parquet": { + "num_bytes": 53305953, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00024-of-00041.parquet": { + "num_bytes": 54647593, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00025-of-00041.parquet": { + "num_bytes": 37422870, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00026-of-00041.parquet": { + "num_bytes": 41390853, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00027-of-00041.parquet": { + "num_bytes": 50695714, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00028-of-00041.parquet": { + "num_bytes": 51990185, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00029-of-00041.parquet": { + "num_bytes": 45319038, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00030-of-00041.parquet": { + "num_bytes": 46656899, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00031-of-00041.parquet": { + "num_bytes": 45404530, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00032-of-00041.parquet": { + "num_bytes": 62950847, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00033-of-00041.parquet": { + "num_bytes": 44203406, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00034-of-00041.parquet": { + "num_bytes": 43126562, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00035-of-00041.parquet": { + "num_bytes": 46022991, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00036-of-00041.parquet": { + "num_bytes": 42937642, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00037-of-00041.parquet": { + "num_bytes": 44411462, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00038-of-00041.parquet": { + "num_bytes": 53668351, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00039-of-00041.parquet": { + "num_bytes": 42803446, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00040-of-00041.parquet": { + "num_bytes": 55959938, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00000-of-00044.parquet": { + "num_bytes": 519960937, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00001-of-00044.parquet": { + "num_bytes": 470189498, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00002-of-00044.parquet": { + "num_bytes": 510429724, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00003-of-00044.parquet": { + "num_bytes": 528953635, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00004-of-00044.parquet": { + "num_bytes": 551914249, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00005-of-00044.parquet": { + "num_bytes": 497136131, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00006-of-00044.parquet": { + "num_bytes": 524143143, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00007-of-00044.parquet": { + "num_bytes": 558934796, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00008-of-00044.parquet": { + "num_bytes": 534389065, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00009-of-00044.parquet": { + "num_bytes": 440695730, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00010-of-00044.parquet": { + "num_bytes": 624890101, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00011-of-00044.parquet": { + "num_bytes": 665636628, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00012-of-00044.parquet": { + "num_bytes": 578849954, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00013-of-00044.parquet": { + "num_bytes": 551809155, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00014-of-00044.parquet": { + "num_bytes": 606563690, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00015-of-00044.parquet": { + "num_bytes": 716391233, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00016-of-00044.parquet": { + "num_bytes": 605827423, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00017-of-00044.parquet": { + "num_bytes": 571776176, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00018-of-00044.parquet": { + "num_bytes": 653923999, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00019-of-00044.parquet": { + "num_bytes": 104682207, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00020-of-00044.parquet": { + "num_bytes": 57458766, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00021-of-00044.parquet": { + "num_bytes": 58489375, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00022-of-00044.parquet": { + "num_bytes": 56127202, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00023-of-00044.parquet": { + "num_bytes": 45088252, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00024-of-00044.parquet": { + "num_bytes": 44580259, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00025-of-00044.parquet": { + "num_bytes": 51690525, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00026-of-00044.parquet": { + "num_bytes": 61310052, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00027-of-00044.parquet": { + "num_bytes": 52937857, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00028-of-00044.parquet": { + "num_bytes": 51738042, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00029-of-00044.parquet": { + "num_bytes": 44217831, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00030-of-00044.parquet": { + "num_bytes": 52242916, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00031-of-00044.parquet": { + "num_bytes": 49028669, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00032-of-00044.parquet": { + "num_bytes": 50493104, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00033-of-00044.parquet": { + "num_bytes": 47701963, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00034-of-00044.parquet": { + "num_bytes": 46804807, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00035-of-00044.parquet": { + "num_bytes": 47035708, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00036-of-00044.parquet": { + "num_bytes": 46467984, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00037-of-00044.parquet": { + "num_bytes": 59408865, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00038-of-00044.parquet": { + "num_bytes": 58599403, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00039-of-00044.parquet": { + "num_bytes": 60635882, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00040-of-00044.parquet": { + "num_bytes": 60540727, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00041-of-00044.parquet": { + "num_bytes": 60384160, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00042-of-00044.parquet": { + "num_bytes": 54147340, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00043-of-00044.parquet": { + "num_bytes": 59571437, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00000-of-00030.parquet": { + "num_bytes": 475655190, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00001-of-00030.parquet": { + "num_bytes": 490593955, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00002-of-00030.parquet": { + "num_bytes": 535054474, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00003-of-00030.parquet": { + "num_bytes": 506117205, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00004-of-00030.parquet": { + "num_bytes": 486988749, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00005-of-00030.parquet": { + "num_bytes": 531641230, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00006-of-00030.parquet": { + "num_bytes": 561486423, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00007-of-00030.parquet": { + "num_bytes": 559940739, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00008-of-00030.parquet": { + "num_bytes": 550009283, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00009-of-00030.parquet": { + "num_bytes": 445251176, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00010-of-00030.parquet": { + "num_bytes": 57812917, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00011-of-00030.parquet": { + "num_bytes": 55598886, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00012-of-00030.parquet": { + "num_bytes": 51362692, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00013-of-00030.parquet": { + "num_bytes": 56520014, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00014-of-00030.parquet": { + "num_bytes": 40715076, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00015-of-00030.parquet": { + "num_bytes": 44925020, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00016-of-00030.parquet": { + "num_bytes": 50081227, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00017-of-00030.parquet": { + "num_bytes": 52794961, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00018-of-00030.parquet": { + "num_bytes": 48030675, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00019-of-00030.parquet": { + "num_bytes": 45308603, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00020-of-00030.parquet": { + "num_bytes": 46384072, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00021-of-00030.parquet": { + "num_bytes": 43727079, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00022-of-00030.parquet": { + "num_bytes": 43721221, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00023-of-00030.parquet": { + "num_bytes": 45856286, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00024-of-00030.parquet": { + "num_bytes": 45429417, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00025-of-00030.parquet": { + "num_bytes": 46105068, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00026-of-00030.parquet": { + "num_bytes": 45733724, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00027-of-00030.parquet": { + "num_bytes": 44971267, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00028-of-00030.parquet": { + "num_bytes": 47918831, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00029-of-00030.parquet": { + "num_bytes": 54836889, + "checksum": null + } + }, + "download_size": 30126501705, + "features": { + "image": { + "_type": "Image" + }, + "label": { + "names": [ + "ai_gen", + "human" + ], + "_type": "ClassLabel" + } + }, + "homepage": "", + "license": "", + "size_in_bytes": 60263246768, + "splits": { + "train": { + "name": "train", + "num_bytes": 11925226072, + "num_examples": 18000, + "shard_lengths": [ + 440, + 878, + 439, + 439, + 439, + 878, + 439, + 878, + 439, + 439, + 439, + 439, + 878, + 439, + 439, + 439, + 1756, + 4829, + 2634 + ], + "dataset_name": "ai_generated_images_midjourney" + }, + "validation": { + "name": "validation", + "num_bytes": 12098126605, + "num_examples": 20715, + "shard_lengths": [ + 471, + 942, + 471, + 471, + 942, + 471, + 471, + 942, + 471, + 471, + 471, + 471, + 471, + 471, + 471, + 471, + 4239, + 5177, + 2350 + ], + "dataset_name": "ai_generated_images_midjourney" + }, + "test": { + "name": "test", + "num_bytes": 6113392386, + "num_examples": 13354, + "shard_lengths": [ + 892, + 446, + 446, + 890, + 445, + 445, + 445, + 890, + 4895, + 3560 + ], + "dataset_name": "ai_generated_images_midjourney" + } + }, + "version": { + "version_str": "0.0.0", + "major": 0, + "minor": 0, + "patch": 0 + } +} \ No newline at end of file diff --git a/research/dataset/test/state.json b/research/dataset/test/state.json new file mode 100644 index 0000000000000000000000000000000000000000..355cb783aeacf6f3812c45d1be30f20f5ed60f06 --- /dev/null +++ b/research/dataset/test/state.json @@ -0,0 +1,49 @@ +{ + "_data_files": [ + { + "filename": "data-00000-of-00013.arrow" + }, + { + "filename": "data-00001-of-00013.arrow" + }, + { + "filename": "data-00002-of-00013.arrow" + }, + { + "filename": "data-00003-of-00013.arrow" + }, + { + "filename": "data-00004-of-00013.arrow" + }, + { + "filename": "data-00005-of-00013.arrow" + }, + { + "filename": "data-00006-of-00013.arrow" + }, + { + "filename": "data-00007-of-00013.arrow" + }, + { + "filename": "data-00008-of-00013.arrow" + }, + { + "filename": "data-00009-of-00013.arrow" + }, + { + "filename": "data-00010-of-00013.arrow" + }, + { + "filename": "data-00011-of-00013.arrow" + }, + { + "filename": "data-00012-of-00013.arrow" + } + ], + "_fingerprint": "a2ce704b062039ff", + "_format_columns": null, + "_format_kwargs": {}, + "_format_type": null, + "_output_all_columns": false, + "_split": "test" +} \ No newline at end of file diff --git a/research/dataset/train/data-00000-of-00024.arrow b/research/dataset/train/data-00000-of-00024.arrow new file mode 100644 index 0000000000000000000000000000000000000000..ef0ce443c64c978934c79984f9303d2f810e5f6f --- /dev/null +++ b/research/dataset/train/data-00000-of-00024.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0033b6c353d18280be7b71a1b778dc72463551ec2db855537359d011017f8637 +size 873562928 diff --git a/research/dataset/train/data-00001-of-00024.arrow b/research/dataset/train/data-00001-of-00024.arrow new file mode 100644 index 0000000000000000000000000000000000000000..aeca7ab1b99dde1c53097f8160b75584b381a1bf --- /dev/null +++ b/research/dataset/train/data-00001-of-00024.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94139383ad58113ccdc42c1de93826142cf98f38e20759bf7195baf0a9309c50 +size 830818768 diff --git a/research/dataset/train/data-00002-of-00024.arrow b/research/dataset/train/data-00002-of-00024.arrow new file mode 100644 index 0000000000000000000000000000000000000000..51d3ca5868c122d2a3d9c6cda13f84582914e65d --- /dev/null +++ b/research/dataset/train/data-00002-of-00024.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aeb95f3849de13fa819d45134d409c33e861e62ea4b0e7e8c4c94d46cab2bc92 +size 901115248 diff --git a/research/dataset/train/data-00003-of-00024.arrow b/research/dataset/train/data-00003-of-00024.arrow new file mode 100644 index 0000000000000000000000000000000000000000..200a929288c4c8344acbfc3a4de887a1291cca98 --- /dev/null +++ b/research/dataset/train/data-00003-of-00024.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f79b7ecbfedd1795a02b1fbb1ca07783e8566f42213feccf5fa51c22913267b +size 848593472 diff --git a/research/dataset/train/data-00004-of-00024.arrow b/research/dataset/train/data-00004-of-00024.arrow new file mode 100644 index 0000000000000000000000000000000000000000..4ccb37319f9cf58cf5be87976dbd058b53a66316 --- /dev/null +++ b/research/dataset/train/data-00004-of-00024.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f88182e9b0dc933e1edd0de7be07a5040be8a557a05bdb99c4df33b09fef1ce +size 863751256 diff --git a/research/dataset/train/data-00005-of-00024.arrow b/research/dataset/train/data-00005-of-00024.arrow new file mode 100644 index 0000000000000000000000000000000000000000..fe5a1fb34a3e7144f52211ffb428caa87f913c48 --- /dev/null +++ b/research/dataset/train/data-00005-of-00024.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4d8fa08414a07a28b11b26ce11728daf0546f90b8250299b43805171685b099 +size 851480032 diff --git a/research/dataset/train/data-00006-of-00024.arrow b/research/dataset/train/data-00006-of-00024.arrow new file mode 100644 index 0000000000000000000000000000000000000000..b3116aabaded1ee5fb6f40215e941584de45ab33 --- /dev/null +++ b/research/dataset/train/data-00006-of-00024.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9aff3f2ada35a2fa8d17d8c5857c46fd1918d3d96ab3b0f7d85c56906ae3041 +size 900824088 diff --git a/research/dataset/train/data-00007-of-00024.arrow b/research/dataset/train/data-00007-of-00024.arrow new file mode 100644 index 0000000000000000000000000000000000000000..5601b7faa587bba7a4e700a601001b8fc32bc935 --- /dev/null +++ b/research/dataset/train/data-00007-of-00024.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:44e7391ea1c930364bbd48ace5b7090c84b23f025c5ed384082dbefbcfddcf0b +size 973347304 diff --git a/research/dataset/train/data-00008-of-00024.arrow b/research/dataset/train/data-00008-of-00024.arrow new file mode 100644 index 0000000000000000000000000000000000000000..e2b19d683138719ec72efd96edad206bad686944 --- /dev/null +++ b/research/dataset/train/data-00008-of-00024.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf0e8e39482aae6a2fa1de6e804778e527872f221369808fc7a007d6045f8403 +size 882646408 diff --git a/research/dataset/train/data-00009-of-00024.arrow b/research/dataset/train/data-00009-of-00024.arrow new file mode 100644 index 0000000000000000000000000000000000000000..bcf7a48b7d126386878d89c234df1c56f265f4ec --- /dev/null +++ b/research/dataset/train/data-00009-of-00024.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60733514f967967c6844dce3dea06cf3242e24291701a9e5b80b499537638239 +size 1036134784 diff --git a/research/dataset/train/data-00010-of-00024.arrow b/research/dataset/train/data-00010-of-00024.arrow new file mode 100644 index 0000000000000000000000000000000000000000..55a30cbd18eecf12df39a1e20a04129606318f0e --- /dev/null +++ b/research/dataset/train/data-00010-of-00024.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:caaec15b88c7a289413a27514d116a51de1807905eba29efe7cc6c6a24501d7b +size 974204112 diff --git a/research/dataset/train/data-00011-of-00024.arrow b/research/dataset/train/data-00011-of-00024.arrow new file mode 100644 index 0000000000000000000000000000000000000000..f623958a42b7e8644d00d9aec89a3193f02a0023 --- /dev/null +++ b/research/dataset/train/data-00011-of-00024.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b949a7f3bd142adca7184d914e4aec28c28d6bec22d1c5de7bb4cb4511c574dc +size 983976016 diff --git a/research/dataset/train/data-00012-of-00024.arrow b/research/dataset/train/data-00012-of-00024.arrow new file mode 100644 index 0000000000000000000000000000000000000000..8430baa2d241469771d1966403f135c5ca8016e3 --- /dev/null +++ b/research/dataset/train/data-00012-of-00024.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec438c3d9ddcfee8f9f21dcf80ecef7f386a920ecd83b82e59201440fbfe6da2 +size 97167384 diff --git a/research/dataset/train/data-00013-of-00024.arrow b/research/dataset/train/data-00013-of-00024.arrow new file mode 100644 index 0000000000000000000000000000000000000000..c9c5a048351a84849d33320fbd4b86fe2c7bcb9d --- /dev/null +++ b/research/dataset/train/data-00013-of-00024.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f896d3e5737dff270f5284609715090e2328acec0f9f2765f30c21b5f365fc1c +size 91327576 diff --git a/research/dataset/train/data-00014-of-00024.arrow b/research/dataset/train/data-00014-of-00024.arrow new file mode 100644 index 0000000000000000000000000000000000000000..73faed80007d3b9979c878e9795cdf9fc1274fb5 --- /dev/null +++ b/research/dataset/train/data-00014-of-00024.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9916e2930654dc6f983233ffb4205bbcce05b7836e189bfd3630a73c4587e357 +size 81903192 diff --git a/research/dataset/train/data-00015-of-00024.arrow b/research/dataset/train/data-00015-of-00024.arrow new file mode 100644 index 0000000000000000000000000000000000000000..3641811cfb025d44878f885d5e2ce9cd75085842 --- /dev/null +++ b/research/dataset/train/data-00015-of-00024.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a2ab5e843f2af768772a29414b443313fb6774fb7c115d9074f636d7a7d0b33 +size 70866464 diff --git a/research/dataset/train/data-00016-of-00024.arrow b/research/dataset/train/data-00016-of-00024.arrow new file mode 100644 index 0000000000000000000000000000000000000000..e6ac077254314fb467592ea17f7c6c1ca71676e7 --- /dev/null +++ b/research/dataset/train/data-00016-of-00024.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ad4b9cc2993c5b8be67e9e8ecab96e2396877de79d4277876d639fe741b4595 +size 89663928 diff --git a/research/dataset/train/data-00017-of-00024.arrow b/research/dataset/train/data-00017-of-00024.arrow new file mode 100644 index 0000000000000000000000000000000000000000..bd455269c8541317959128c38e37aeddc91eeecc --- /dev/null +++ b/research/dataset/train/data-00017-of-00024.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f20e26b7863187c9543dc15e2b1c8017eb9dcf3dca523b5d1ecc39e7a742e8ad +size 79693328 diff --git a/research/dataset/train/data-00018-of-00024.arrow b/research/dataset/train/data-00018-of-00024.arrow new file mode 100644 index 0000000000000000000000000000000000000000..f9f0b60b8e9fa7a9c2fb690303f6a913bda4b94a --- /dev/null +++ b/research/dataset/train/data-00018-of-00024.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:070762b57ef545bf3de16a26e5f7ea20d6d56eab8498c65c3d2967a56ec0d551 +size 79008736 diff --git a/research/dataset/train/data-00019-of-00024.arrow b/research/dataset/train/data-00019-of-00024.arrow new file mode 100644 index 0000000000000000000000000000000000000000..85be540a9472ff8a7bf546195901b28fd64189c4 --- /dev/null +++ b/research/dataset/train/data-00019-of-00024.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:756d329547b4a0a4a29f4d68dc2412f28ff0bad440fd214ffd7f4813fbbe98ee +size 91979312 diff --git a/research/dataset/train/data-00020-of-00024.arrow b/research/dataset/train/data-00020-of-00024.arrow new file mode 100644 index 0000000000000000000000000000000000000000..ca19ec181a8d0c15fd370684442414323f27fb91 --- /dev/null +++ b/research/dataset/train/data-00020-of-00024.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ee35e8a6a2169024bea62fdd29d1bba9b495283c43b914d83f47b0cab668a92 +size 75674720 diff --git a/research/dataset/train/data-00021-of-00024.arrow b/research/dataset/train/data-00021-of-00024.arrow new file mode 100644 index 0000000000000000000000000000000000000000..c9c07430fb178698e957c7d8c6a23973e4526abe --- /dev/null +++ b/research/dataset/train/data-00021-of-00024.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2db1591fd6a74323bf3edd90bc19f82546c43b22af7e097ff1bb15d60ce1024c +size 77859112 diff --git a/research/dataset/train/data-00022-of-00024.arrow b/research/dataset/train/data-00022-of-00024.arrow new file mode 100644 index 0000000000000000000000000000000000000000..f75c8253fb1625c838e5a0f7e78cdebea65f546e --- /dev/null +++ b/research/dataset/train/data-00022-of-00024.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a625145d7e5fc4575983ba77e74afe4048c782f5e1061e00bcfbf3a9a53ecf1 +size 82657560 diff --git a/research/dataset/train/data-00023-of-00024.arrow b/research/dataset/train/data-00023-of-00024.arrow new file mode 100644 index 0000000000000000000000000000000000000000..6c06a1d838d470e5a5927b15d63717f15855ba25 --- /dev/null +++ b/research/dataset/train/data-00023-of-00024.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bfc82237b41bb01287cd1c8d29ae3085e29d0f8261a5e14a86d05bb9e8ce0108 +size 86986888 diff --git a/research/dataset/train/dataset_info.json b/research/dataset/train/dataset_info.json new file mode 100644 index 0000000000000000000000000000000000000000..1f7bfeddcb56cccbadfa192d8f4c025994e51555 --- /dev/null +++ b/research/dataset/train/dataset_info.json @@ -0,0 +1,566 @@ +{ + "builder_name": "parquet", + "citation": "", + "config_name": "default", + "dataset_name": "ai_generated_images_midjourney", + "dataset_size": 30136745063, + "description": "", + "download_checksums": { + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00000-of-00041.parquet": { + "num_bytes": 538696797, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00001-of-00041.parquet": { + "num_bytes": 461123819, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00002-of-00041.parquet": { + "num_bytes": 470632938, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00003-of-00041.parquet": { + "num_bytes": 531704324, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00004-of-00041.parquet": { + "num_bytes": 536933200, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00005-of-00041.parquet": { + "num_bytes": 501545607, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00006-of-00041.parquet": { + "num_bytes": 475535302, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00007-of-00041.parquet": { + "num_bytes": 539136860, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00008-of-00041.parquet": { + "num_bytes": 530528421, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00009-of-00041.parquet": { + "num_bytes": 457521582, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00010-of-00041.parquet": { + "num_bytes": 480092663, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00011-of-00041.parquet": { + "num_bytes": 566820126, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00012-of-00041.parquet": { + "num_bytes": 590606849, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00013-of-00041.parquet": { + "num_bytes": 565073338, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00014-of-00041.parquet": { + "num_bytes": 500476843, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00015-of-00041.parquet": { + "num_bytes": 482574110, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00016-of-00041.parquet": { + "num_bytes": 693849912, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00017-of-00041.parquet": { + "num_bytes": 513087321, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00018-of-00041.parquet": { + "num_bytes": 620252104, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00019-of-00041.parquet": { + "num_bytes": 528086346, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00020-of-00041.parquet": { + "num_bytes": 365200077, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00021-of-00041.parquet": { + "num_bytes": 54093315, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00022-of-00041.parquet": { + "num_bytes": 55640622, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00023-of-00041.parquet": { + "num_bytes": 53305953, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00024-of-00041.parquet": { + "num_bytes": 54647593, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00025-of-00041.parquet": { + "num_bytes": 37422870, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00026-of-00041.parquet": { + "num_bytes": 41390853, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00027-of-00041.parquet": { + "num_bytes": 50695714, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00028-of-00041.parquet": { + "num_bytes": 51990185, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00029-of-00041.parquet": { + "num_bytes": 45319038, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00030-of-00041.parquet": { + "num_bytes": 46656899, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00031-of-00041.parquet": { + "num_bytes": 45404530, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00032-of-00041.parquet": { + "num_bytes": 62950847, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00033-of-00041.parquet": { + "num_bytes": 44203406, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00034-of-00041.parquet": { + "num_bytes": 43126562, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00035-of-00041.parquet": { + "num_bytes": 46022991, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00036-of-00041.parquet": { + "num_bytes": 42937642, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00037-of-00041.parquet": { + "num_bytes": 44411462, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00038-of-00041.parquet": { + "num_bytes": 53668351, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00039-of-00041.parquet": { + "num_bytes": 42803446, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00040-of-00041.parquet": { + "num_bytes": 55959938, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00000-of-00044.parquet": { + "num_bytes": 519960937, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00001-of-00044.parquet": { + "num_bytes": 470189498, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00002-of-00044.parquet": { + "num_bytes": 510429724, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00003-of-00044.parquet": { + "num_bytes": 528953635, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00004-of-00044.parquet": { + "num_bytes": 551914249, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00005-of-00044.parquet": { + "num_bytes": 497136131, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00006-of-00044.parquet": { + "num_bytes": 524143143, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00007-of-00044.parquet": { + "num_bytes": 558934796, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00008-of-00044.parquet": { + "num_bytes": 534389065, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00009-of-00044.parquet": { + "num_bytes": 440695730, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00010-of-00044.parquet": { + "num_bytes": 624890101, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00011-of-00044.parquet": { + "num_bytes": 665636628, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00012-of-00044.parquet": { + "num_bytes": 578849954, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00013-of-00044.parquet": { + "num_bytes": 551809155, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00014-of-00044.parquet": { + "num_bytes": 606563690, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00015-of-00044.parquet": { + "num_bytes": 716391233, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00016-of-00044.parquet": { + "num_bytes": 605827423, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00017-of-00044.parquet": { + "num_bytes": 571776176, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00018-of-00044.parquet": { + "num_bytes": 653923999, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00019-of-00044.parquet": { + "num_bytes": 104682207, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00020-of-00044.parquet": { + "num_bytes": 57458766, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00021-of-00044.parquet": { + "num_bytes": 58489375, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00022-of-00044.parquet": { + "num_bytes": 56127202, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00023-of-00044.parquet": { + "num_bytes": 45088252, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00024-of-00044.parquet": { + "num_bytes": 44580259, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00025-of-00044.parquet": { + "num_bytes": 51690525, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00026-of-00044.parquet": { + "num_bytes": 61310052, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00027-of-00044.parquet": { + "num_bytes": 52937857, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00028-of-00044.parquet": { + "num_bytes": 51738042, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00029-of-00044.parquet": { + "num_bytes": 44217831, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00030-of-00044.parquet": { + "num_bytes": 52242916, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00031-of-00044.parquet": { + "num_bytes": 49028669, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00032-of-00044.parquet": { + "num_bytes": 50493104, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00033-of-00044.parquet": { + "num_bytes": 47701963, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00034-of-00044.parquet": { + "num_bytes": 46804807, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00035-of-00044.parquet": { + "num_bytes": 47035708, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00036-of-00044.parquet": { + "num_bytes": 46467984, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00037-of-00044.parquet": { + "num_bytes": 59408865, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00038-of-00044.parquet": { + "num_bytes": 58599403, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00039-of-00044.parquet": { + "num_bytes": 60635882, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00040-of-00044.parquet": { + "num_bytes": 60540727, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00041-of-00044.parquet": { + "num_bytes": 60384160, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00042-of-00044.parquet": { + "num_bytes": 54147340, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00043-of-00044.parquet": { + "num_bytes": 59571437, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00000-of-00030.parquet": { + "num_bytes": 475655190, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00001-of-00030.parquet": { + "num_bytes": 490593955, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00002-of-00030.parquet": { + "num_bytes": 535054474, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00003-of-00030.parquet": { + "num_bytes": 506117205, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00004-of-00030.parquet": { + "num_bytes": 486988749, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00005-of-00030.parquet": { + "num_bytes": 531641230, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00006-of-00030.parquet": { + "num_bytes": 561486423, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00007-of-00030.parquet": { + "num_bytes": 559940739, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00008-of-00030.parquet": { + "num_bytes": 550009283, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00009-of-00030.parquet": { + "num_bytes": 445251176, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00010-of-00030.parquet": { + "num_bytes": 57812917, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00011-of-00030.parquet": { + "num_bytes": 55598886, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00012-of-00030.parquet": { + "num_bytes": 51362692, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00013-of-00030.parquet": { + "num_bytes": 56520014, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00014-of-00030.parquet": { + "num_bytes": 40715076, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00015-of-00030.parquet": { + "num_bytes": 44925020, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00016-of-00030.parquet": { + "num_bytes": 50081227, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00017-of-00030.parquet": { + "num_bytes": 52794961, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00018-of-00030.parquet": { + "num_bytes": 48030675, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00019-of-00030.parquet": { + "num_bytes": 45308603, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00020-of-00030.parquet": { + "num_bytes": 46384072, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00021-of-00030.parquet": { + "num_bytes": 43727079, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00022-of-00030.parquet": { + "num_bytes": 43721221, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00023-of-00030.parquet": { + "num_bytes": 45856286, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00024-of-00030.parquet": { + "num_bytes": 45429417, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00025-of-00030.parquet": { + "num_bytes": 46105068, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00026-of-00030.parquet": { + "num_bytes": 45733724, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00027-of-00030.parquet": { + "num_bytes": 44971267, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00028-of-00030.parquet": { + "num_bytes": 47918831, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00029-of-00030.parquet": { + "num_bytes": 54836889, + "checksum": null + } + }, + "download_size": 30126501705, + "features": { + "image": { + "_type": "Image" + }, + "label": { + "names": [ + "ai_gen", + "human" + ], + "_type": "ClassLabel" + } + }, + "homepage": "", + "license": "", + "size_in_bytes": 60263246768, + "splits": { + "train": { + "name": "train", + "num_bytes": 11925226072, + "num_examples": 18000, + "shard_lengths": [ + 440, + 878, + 439, + 439, + 439, + 878, + 439, + 878, + 439, + 439, + 439, + 439, + 878, + 439, + 439, + 439, + 1756, + 4829, + 2634 + ], + "dataset_name": "ai_generated_images_midjourney" + }, + "validation": { + "name": "validation", + "num_bytes": 12098126605, + "num_examples": 20715, + "shard_lengths": [ + 471, + 942, + 471, + 471, + 942, + 471, + 471, + 942, + 471, + 471, + 471, + 471, + 471, + 471, + 471, + 471, + 4239, + 5177, + 2350 + ], + "dataset_name": "ai_generated_images_midjourney" + }, + "test": { + "name": "test", + "num_bytes": 6113392386, + "num_examples": 13354, + "shard_lengths": [ + 892, + 446, + 446, + 890, + 445, + 445, + 445, + 890, + 4895, + 3560 + ], + "dataset_name": "ai_generated_images_midjourney" + } + }, + "version": { + "version_str": "0.0.0", + "major": 0, + "minor": 0, + "patch": 0 + } +} \ No newline at end of file diff --git a/research/dataset/train/state.json b/research/dataset/train/state.json new file mode 100644 index 0000000000000000000000000000000000000000..17d2e7a585fa3d009cf11e22b11ba220018a5779 --- /dev/null +++ b/research/dataset/train/state.json @@ -0,0 +1,82 @@ +{ + "_data_files": [ + { + "filename": "data-00000-of-00024.arrow" + }, + { + "filename": "data-00001-of-00024.arrow" + }, + { + "filename": "data-00002-of-00024.arrow" + }, + { + "filename": "data-00003-of-00024.arrow" + }, + { + "filename": "data-00004-of-00024.arrow" + }, + { + "filename": "data-00005-of-00024.arrow" + }, + { + "filename": "data-00006-of-00024.arrow" + }, + { + "filename": "data-00007-of-00024.arrow" + }, + { + "filename": "data-00008-of-00024.arrow" + }, + { + "filename": "data-00009-of-00024.arrow" + }, + { + "filename": "data-00010-of-00024.arrow" + }, + { + "filename": "data-00011-of-00024.arrow" + }, + { + "filename": "data-00012-of-00024.arrow" + }, + { + "filename": "data-00013-of-00024.arrow" + }, + { + "filename": "data-00014-of-00024.arrow" + }, + { + "filename": "data-00015-of-00024.arrow" + }, + { + "filename": "data-00016-of-00024.arrow" + }, + { + "filename": "data-00017-of-00024.arrow" + }, + { + "filename": "data-00018-of-00024.arrow" + }, + { + "filename": "data-00019-of-00024.arrow" + }, + { + "filename": "data-00020-of-00024.arrow" + }, + { + "filename": "data-00021-of-00024.arrow" + }, + { + "filename": "data-00022-of-00024.arrow" + }, + { + "filename": "data-00023-of-00024.arrow" + } + ], + "_fingerprint": "d493ae9fd6d24933", + "_format_columns": null, + "_format_kwargs": {}, + "_format_type": null, + "_output_all_columns": false, + "_split": "train" +} \ No newline at end of file diff --git a/research/dataset/validation/data-00000-of-00025.arrow b/research/dataset/validation/data-00000-of-00025.arrow new file mode 100644 index 0000000000000000000000000000000000000000..f498ba753876d60f9d7dd49807d9cdb7cdcf484e --- /dev/null +++ b/research/dataset/validation/data-00000-of-00025.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b7657cd6ed4e2734899fb388bbb24534f209a743f7fe9418c33e8c2decf0952 +size 889710848 diff --git a/research/dataset/validation/data-00001-of-00025.arrow b/research/dataset/validation/data-00001-of-00025.arrow new file mode 100644 index 0000000000000000000000000000000000000000..1334f7d8e3ca644eb306917b376f5b4096891cce --- /dev/null +++ b/research/dataset/validation/data-00001-of-00025.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17306ea878c2e384bc93dad0b9041bbd207ea6087e8cd09336c67272b2964e07 +size 906745536 diff --git a/research/dataset/validation/data-00002-of-00025.arrow b/research/dataset/validation/data-00002-of-00025.arrow new file mode 100644 index 0000000000000000000000000000000000000000..a03b13bc743d9fed3dd2fb0c024109d07267fa47 --- /dev/null +++ b/research/dataset/validation/data-00002-of-00025.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:044c5a33d756b6a328a4ef6e3da6caaa7cebfee1b464e736d31fa4a3271a3182 +size 933881176 diff --git a/research/dataset/validation/data-00003-of-00025.arrow b/research/dataset/validation/data-00003-of-00025.arrow new file mode 100644 index 0000000000000000000000000000000000000000..0b20cec9ea1c25a4e9d227823e0bfc15da68b97a --- /dev/null +++ b/research/dataset/validation/data-00003-of-00025.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b336fca677f3083451195b68487f05f2c8f0dadd1ff6d9377e6ced85c6fcaeb1 +size 912667560 diff --git a/research/dataset/validation/data-00004-of-00025.arrow b/research/dataset/validation/data-00004-of-00025.arrow new file mode 100644 index 0000000000000000000000000000000000000000..9093cf58d1db212eb2d608be51a151c8d5386335 --- /dev/null +++ b/research/dataset/validation/data-00004-of-00025.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cae3edb6f0ce613f4070d7c3392e5a5f994c17c856b59f4b0fe66759cefea110 +size 968992352 diff --git a/research/dataset/validation/data-00005-of-00025.arrow b/research/dataset/validation/data-00005-of-00025.arrow new file mode 100644 index 0000000000000000000000000000000000000000..b0aa5affbbb613242a0071b46ce34e8050a41520 --- /dev/null +++ b/research/dataset/validation/data-00005-of-00025.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:068c3dffaea272baf02ae6182505e26f336139893f9c84d78f77c2494e875ffe +size 850917808 diff --git a/research/dataset/validation/data-00006-of-00025.arrow b/research/dataset/validation/data-00006-of-00025.arrow new file mode 100644 index 0000000000000000000000000000000000000000..59b63fec81d1602593507cfb9b77433b00962b99 --- /dev/null +++ b/research/dataset/validation/data-00006-of-00025.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2122138d5fecc3c610898d7a1bf588288a74862a66cc881d9c8a7d8d1b94ca7 +size 1147240120 diff --git a/research/dataset/validation/data-00007-of-00025.arrow b/research/dataset/validation/data-00007-of-00025.arrow new file mode 100644 index 0000000000000000000000000000000000000000..f729d2161f36c42dafed3ed728f83714a66d761a --- /dev/null +++ b/research/dataset/validation/data-00007-of-00025.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c2314902000f4875a5a2a34c250d624e25b1fa7cde13bf0d0fa6494c7e8ad0e +size 1006984616 diff --git a/research/dataset/validation/data-00008-of-00025.arrow b/research/dataset/validation/data-00008-of-00025.arrow new file mode 100644 index 0000000000000000000000000000000000000000..809303f946f0f5b658731afcb7135da39fd57c59 --- /dev/null +++ b/research/dataset/validation/data-00008-of-00025.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b46a37f440bce9146becf23951803ea3f88d0286aabce94cedeaa7d13c70e5af +size 1181867112 diff --git a/research/dataset/validation/data-00009-of-00025.arrow b/research/dataset/validation/data-00009-of-00025.arrow new file mode 100644 index 0000000000000000000000000000000000000000..f4bdb903e92de8776fb1e250db3bb689bca2ead7 --- /dev/null +++ b/research/dataset/validation/data-00009-of-00025.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22fe6397038067a8ca161718e6ee90b26a75ff44c2eaadd0ba4064aeb2e3e227 +size 1051237008 diff --git a/research/dataset/validation/data-00010-of-00025.arrow b/research/dataset/validation/data-00010-of-00025.arrow new file mode 100644 index 0000000000000000000000000000000000000000..7e4b7273b9e5c85b8d8c633c3fcea1b5807f992b --- /dev/null +++ b/research/dataset/validation/data-00010-of-00025.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:923c33b152bf514463cf847d66282c6631831c371262ca32d9900a9e7fa4cf4d +size 932198904 diff --git a/research/dataset/validation/data-00011-of-00025.arrow b/research/dataset/validation/data-00011-of-00025.arrow new file mode 100644 index 0000000000000000000000000000000000000000..888d4216ed437f7cbc90015a5927a764da89c6db --- /dev/null +++ b/research/dataset/validation/data-00011-of-00025.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f8ad3d71328b56fc3b4334ed02eb8c2bc81b32d0b1246f95df786f9e57b9d886 +size 101391248 diff --git a/research/dataset/validation/data-00012-of-00025.arrow b/research/dataset/validation/data-00012-of-00025.arrow new file mode 100644 index 0000000000000000000000000000000000000000..6e19d7755426d1dd81ab2f7bd57cc16170bbcc59 --- /dev/null +++ b/research/dataset/validation/data-00012-of-00025.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffe4dbddf47023653455cca9c5a6ae234c7b4c5d1a5ea7820baf408a4e401762 +size 100204936 diff --git a/research/dataset/validation/data-00013-of-00025.arrow b/research/dataset/validation/data-00013-of-00025.arrow new file mode 100644 index 0000000000000000000000000000000000000000..bf14fb88f6e2bda26e00f2261f10201283acc446 --- /dev/null +++ b/research/dataset/validation/data-00013-of-00025.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3e50b8180f017010360ca05b07cd3626c521796c39c1044e5be6bc2bdeaa54c +size 80014512 diff --git a/research/dataset/validation/data-00014-of-00025.arrow b/research/dataset/validation/data-00014-of-00025.arrow new file mode 100644 index 0000000000000000000000000000000000000000..47c7842215482d17185fdc110bf7037c0e28098e --- /dev/null +++ b/research/dataset/validation/data-00014-of-00025.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cc50991efe4129c5c13bf615ba53ba630fa160fe3b99e5ccb66479f783962ac +size 93211488 diff --git a/research/dataset/validation/data-00015-of-00025.arrow b/research/dataset/validation/data-00015-of-00025.arrow new file mode 100644 index 0000000000000000000000000000000000000000..6f07407dacea0b1b91f8a616843a49ed177b38b3 --- /dev/null +++ b/research/dataset/validation/data-00015-of-00025.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86bccc00bff9ab8c750b7340aa0add6d2a0120b0954f910261634ebc44bc536b +size 97607592 diff --git a/research/dataset/validation/data-00016-of-00025.arrow b/research/dataset/validation/data-00016-of-00025.arrow new file mode 100644 index 0000000000000000000000000000000000000000..da7584ec4b375387246b26234f0f413eaf110dba --- /dev/null +++ b/research/dataset/validation/data-00016-of-00025.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebc3000f9fa20f8b33e6bbc6992cbc39a33e3068144c49c993b554a2ba8a1a40 +size 85224200 diff --git a/research/dataset/validation/data-00017-of-00025.arrow b/research/dataset/validation/data-00017-of-00025.arrow new file mode 100644 index 0000000000000000000000000000000000000000..0d6def7882e6b7c8c0157125cf5f430c38446f1c --- /dev/null +++ b/research/dataset/validation/data-00017-of-00025.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:017cf2d24ea121b2158c1273bd7f0e98b1362bb79ed0d96ed72149f8edae7ab4 +size 90482472 diff --git a/research/dataset/validation/data-00018-of-00025.arrow b/research/dataset/validation/data-00018-of-00025.arrow new file mode 100644 index 0000000000000000000000000000000000000000..8e726cec8cd00ec45444cac994426b9711089ba6 --- /dev/null +++ b/research/dataset/validation/data-00018-of-00025.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7685a5ade153642d3a72ab1682eb52ae35375a40389c563499366c64caa7234f +size 84908480 diff --git a/research/dataset/validation/data-00019-of-00025.arrow b/research/dataset/validation/data-00019-of-00025.arrow new file mode 100644 index 0000000000000000000000000000000000000000..e982c705238d540b66c54bfa9abeb4b864acc8c0 --- /dev/null +++ b/research/dataset/validation/data-00019-of-00025.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c509dc5f8fd0a964a52cc8a111c09efe599aece9bf5c067cc5912b0880c3f288 +size 82601616 diff --git a/research/dataset/validation/data-00020-of-00025.arrow b/research/dataset/validation/data-00020-of-00025.arrow new file mode 100644 index 0000000000000000000000000000000000000000..ace8808408d33033f0711c6145e50412690c94d0 --- /dev/null +++ b/research/dataset/validation/data-00020-of-00025.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:178c13fddbfc4706be4498b790f7088a5b883eedbbda4cd11a6dff51c739f7f6 +size 82941360 diff --git a/research/dataset/validation/data-00021-of-00025.arrow b/research/dataset/validation/data-00021-of-00025.arrow new file mode 100644 index 0000000000000000000000000000000000000000..d3afab4e2a820f7909bcd138bc47db518ba30a00 --- /dev/null +++ b/research/dataset/validation/data-00021-of-00025.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f59cd1ef76686e756d1602ed7473a0cd790d17a6ab95341a6942356f2ec3b117 +size 101996936 diff --git a/research/dataset/validation/data-00022-of-00025.arrow b/research/dataset/validation/data-00022-of-00025.arrow new file mode 100644 index 0000000000000000000000000000000000000000..ab3aafb634e15083be241e3cc0efbdf509937453 --- /dev/null +++ b/research/dataset/validation/data-00022-of-00025.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2dfab89b7ab4ad79278f3718c799182657fad502ebe628d8386d32b75fa6a73 +size 110015896 diff --git a/research/dataset/validation/data-00023-of-00025.arrow b/research/dataset/validation/data-00023-of-00025.arrow new file mode 100644 index 0000000000000000000000000000000000000000..88e08bff094c7d68e68b415f332b0f9fbe62f9f7 --- /dev/null +++ b/research/dataset/validation/data-00023-of-00025.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e024cf9bfe355a80a5f463757d22f3c4263e47687ff4c2f2bd9933027e1f4925 +size 103858944 diff --git a/research/dataset/validation/data-00024-of-00025.arrow b/research/dataset/validation/data-00024-of-00025.arrow new file mode 100644 index 0000000000000000000000000000000000000000..c1d3c4938a5629ff8406cb4d4ad1c8c6c99bca5c --- /dev/null +++ b/research/dataset/validation/data-00024-of-00025.arrow @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5a8a42ff7b23b14ca4d3cb602c66957d875a58707c99d7462cf6d845e44b4a5 +size 101240736 diff --git a/research/dataset/validation/dataset_info.json b/research/dataset/validation/dataset_info.json new file mode 100644 index 0000000000000000000000000000000000000000..1f7bfeddcb56cccbadfa192d8f4c025994e51555 --- /dev/null +++ b/research/dataset/validation/dataset_info.json @@ -0,0 +1,566 @@ +{ + "builder_name": "parquet", + "citation": "", + "config_name": "default", + "dataset_name": "ai_generated_images_midjourney", + "dataset_size": 30136745063, + "description": "", + "download_checksums": { + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00000-of-00041.parquet": { + "num_bytes": 538696797, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00001-of-00041.parquet": { + "num_bytes": 461123819, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00002-of-00041.parquet": { + "num_bytes": 470632938, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00003-of-00041.parquet": { + "num_bytes": 531704324, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00004-of-00041.parquet": { + "num_bytes": 536933200, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00005-of-00041.parquet": { + "num_bytes": 501545607, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00006-of-00041.parquet": { + "num_bytes": 475535302, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00007-of-00041.parquet": { + "num_bytes": 539136860, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00008-of-00041.parquet": { + "num_bytes": 530528421, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00009-of-00041.parquet": { + "num_bytes": 457521582, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00010-of-00041.parquet": { + "num_bytes": 480092663, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00011-of-00041.parquet": { + "num_bytes": 566820126, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00012-of-00041.parquet": { + "num_bytes": 590606849, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00013-of-00041.parquet": { + "num_bytes": 565073338, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00014-of-00041.parquet": { + "num_bytes": 500476843, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00015-of-00041.parquet": { + "num_bytes": 482574110, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00016-of-00041.parquet": { + "num_bytes": 693849912, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00017-of-00041.parquet": { + "num_bytes": 513087321, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00018-of-00041.parquet": { + "num_bytes": 620252104, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00019-of-00041.parquet": { + "num_bytes": 528086346, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00020-of-00041.parquet": { + "num_bytes": 365200077, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00021-of-00041.parquet": { + "num_bytes": 54093315, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00022-of-00041.parquet": { + "num_bytes": 55640622, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00023-of-00041.parquet": { + "num_bytes": 53305953, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00024-of-00041.parquet": { + "num_bytes": 54647593, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00025-of-00041.parquet": { + "num_bytes": 37422870, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00026-of-00041.parquet": { + "num_bytes": 41390853, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00027-of-00041.parquet": { + "num_bytes": 50695714, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00028-of-00041.parquet": { + "num_bytes": 51990185, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00029-of-00041.parquet": { + "num_bytes": 45319038, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00030-of-00041.parquet": { + "num_bytes": 46656899, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00031-of-00041.parquet": { + "num_bytes": 45404530, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00032-of-00041.parquet": { + "num_bytes": 62950847, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00033-of-00041.parquet": { + "num_bytes": 44203406, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00034-of-00041.parquet": { + "num_bytes": 43126562, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00035-of-00041.parquet": { + "num_bytes": 46022991, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00036-of-00041.parquet": { + "num_bytes": 42937642, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00037-of-00041.parquet": { + "num_bytes": 44411462, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00038-of-00041.parquet": { + "num_bytes": 53668351, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00039-of-00041.parquet": { + "num_bytes": 42803446, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/train-00040-of-00041.parquet": { + "num_bytes": 55959938, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00000-of-00044.parquet": { + "num_bytes": 519960937, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00001-of-00044.parquet": { + "num_bytes": 470189498, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00002-of-00044.parquet": { + "num_bytes": 510429724, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00003-of-00044.parquet": { + "num_bytes": 528953635, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00004-of-00044.parquet": { + "num_bytes": 551914249, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00005-of-00044.parquet": { + "num_bytes": 497136131, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00006-of-00044.parquet": { + "num_bytes": 524143143, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00007-of-00044.parquet": { + "num_bytes": 558934796, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00008-of-00044.parquet": { + "num_bytes": 534389065, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00009-of-00044.parquet": { + "num_bytes": 440695730, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00010-of-00044.parquet": { + "num_bytes": 624890101, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00011-of-00044.parquet": { + "num_bytes": 665636628, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00012-of-00044.parquet": { + "num_bytes": 578849954, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00013-of-00044.parquet": { + "num_bytes": 551809155, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00014-of-00044.parquet": { + "num_bytes": 606563690, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00015-of-00044.parquet": { + "num_bytes": 716391233, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00016-of-00044.parquet": { + "num_bytes": 605827423, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00017-of-00044.parquet": { + "num_bytes": 571776176, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00018-of-00044.parquet": { + "num_bytes": 653923999, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00019-of-00044.parquet": { + "num_bytes": 104682207, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00020-of-00044.parquet": { + "num_bytes": 57458766, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00021-of-00044.parquet": { + "num_bytes": 58489375, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00022-of-00044.parquet": { + "num_bytes": 56127202, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00023-of-00044.parquet": { + "num_bytes": 45088252, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00024-of-00044.parquet": { + "num_bytes": 44580259, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00025-of-00044.parquet": { + "num_bytes": 51690525, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00026-of-00044.parquet": { + "num_bytes": 61310052, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00027-of-00044.parquet": { + "num_bytes": 52937857, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00028-of-00044.parquet": { + "num_bytes": 51738042, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00029-of-00044.parquet": { + "num_bytes": 44217831, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00030-of-00044.parquet": { + "num_bytes": 52242916, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00031-of-00044.parquet": { + "num_bytes": 49028669, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00032-of-00044.parquet": { + "num_bytes": 50493104, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00033-of-00044.parquet": { + "num_bytes": 47701963, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00034-of-00044.parquet": { + "num_bytes": 46804807, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00035-of-00044.parquet": { + "num_bytes": 47035708, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00036-of-00044.parquet": { + "num_bytes": 46467984, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00037-of-00044.parquet": { + "num_bytes": 59408865, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00038-of-00044.parquet": { + "num_bytes": 58599403, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00039-of-00044.parquet": { + "num_bytes": 60635882, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00040-of-00044.parquet": { + "num_bytes": 60540727, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00041-of-00044.parquet": { + "num_bytes": 60384160, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00042-of-00044.parquet": { + "num_bytes": 54147340, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/validation-00043-of-00044.parquet": { + "num_bytes": 59571437, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00000-of-00030.parquet": { + "num_bytes": 475655190, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00001-of-00030.parquet": { + "num_bytes": 490593955, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00002-of-00030.parquet": { + "num_bytes": 535054474, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00003-of-00030.parquet": { + "num_bytes": 506117205, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00004-of-00030.parquet": { + "num_bytes": 486988749, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00005-of-00030.parquet": { + "num_bytes": 531641230, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00006-of-00030.parquet": { + "num_bytes": 561486423, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00007-of-00030.parquet": { + "num_bytes": 559940739, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00008-of-00030.parquet": { + "num_bytes": 550009283, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00009-of-00030.parquet": { + "num_bytes": 445251176, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00010-of-00030.parquet": { + "num_bytes": 57812917, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00011-of-00030.parquet": { + "num_bytes": 55598886, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00012-of-00030.parquet": { + "num_bytes": 51362692, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00013-of-00030.parquet": { + "num_bytes": 56520014, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00014-of-00030.parquet": { + "num_bytes": 40715076, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00015-of-00030.parquet": { + "num_bytes": 44925020, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00016-of-00030.parquet": { + "num_bytes": 50081227, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00017-of-00030.parquet": { + "num_bytes": 52794961, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00018-of-00030.parquet": { + "num_bytes": 48030675, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00019-of-00030.parquet": { + "num_bytes": 45308603, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00020-of-00030.parquet": { + "num_bytes": 46384072, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00021-of-00030.parquet": { + "num_bytes": 43727079, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00022-of-00030.parquet": { + "num_bytes": 43721221, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00023-of-00030.parquet": { + "num_bytes": 45856286, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00024-of-00030.parquet": { + "num_bytes": 45429417, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00025-of-00030.parquet": { + "num_bytes": 46105068, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00026-of-00030.parquet": { + "num_bytes": 45733724, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00027-of-00030.parquet": { + "num_bytes": 44971267, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00028-of-00030.parquet": { + "num_bytes": 47918831, + "checksum": null + }, + "hf://datasets/ideepankarsharma2003/AIGeneratedImages_Midjourney@9851ccc4fea851b0f43c4480b8a2795a1fcc0034/data/test-00029-of-00030.parquet": { + "num_bytes": 54836889, + "checksum": null + } + }, + "download_size": 30126501705, + "features": { + "image": { + "_type": "Image" + }, + "label": { + "names": [ + "ai_gen", + "human" + ], + "_type": "ClassLabel" + } + }, + "homepage": "", + "license": "", + "size_in_bytes": 60263246768, + "splits": { + "train": { + "name": "train", + "num_bytes": 11925226072, + "num_examples": 18000, + "shard_lengths": [ + 440, + 878, + 439, + 439, + 439, + 878, + 439, + 878, + 439, + 439, + 439, + 439, + 878, + 439, + 439, + 439, + 1756, + 4829, + 2634 + ], + "dataset_name": "ai_generated_images_midjourney" + }, + "validation": { + "name": "validation", + "num_bytes": 12098126605, + "num_examples": 20715, + "shard_lengths": [ + 471, + 942, + 471, + 471, + 942, + 471, + 471, + 942, + 471, + 471, + 471, + 471, + 471, + 471, + 471, + 471, + 4239, + 5177, + 2350 + ], + "dataset_name": "ai_generated_images_midjourney" + }, + "test": { + "name": "test", + "num_bytes": 6113392386, + "num_examples": 13354, + "shard_lengths": [ + 892, + 446, + 446, + 890, + 445, + 445, + 445, + 890, + 4895, + 3560 + ], + "dataset_name": "ai_generated_images_midjourney" + } + }, + "version": { + "version_str": "0.0.0", + "major": 0, + "minor": 0, + "patch": 0 + } +} \ No newline at end of file diff --git a/research/dataset/validation/state.json b/research/dataset/validation/state.json new file mode 100644 index 0000000000000000000000000000000000000000..baf22d6a43de2265dc14905d433007c2679f131d --- /dev/null +++ b/research/dataset/validation/state.json @@ -0,0 +1,85 @@ +{ + "_data_files": [ + { + "filename": "data-00000-of-00025.arrow" + }, + { + "filename": "data-00001-of-00025.arrow" + }, + { + "filename": "data-00002-of-00025.arrow" + }, + { + "filename": "data-00003-of-00025.arrow" + }, + { + "filename": "data-00004-of-00025.arrow" + }, + { + "filename": "data-00005-of-00025.arrow" + }, + { + "filename": "data-00006-of-00025.arrow" + }, + { + "filename": "data-00007-of-00025.arrow" + }, + { + "filename": "data-00008-of-00025.arrow" + }, + { + "filename": "data-00009-of-00025.arrow" + }, + { + "filename": "data-00010-of-00025.arrow" + }, + { + "filename": "data-00011-of-00025.arrow" + }, + { + "filename": "data-00012-of-00025.arrow" + }, + { + "filename": "data-00013-of-00025.arrow" + }, + { + "filename": "data-00014-of-00025.arrow" + }, + { + "filename": "data-00015-of-00025.arrow" + }, + { + "filename": "data-00016-of-00025.arrow" + }, + { + "filename": "data-00017-of-00025.arrow" + }, + { + "filename": "data-00018-of-00025.arrow" + }, + { + "filename": "data-00019-of-00025.arrow" + }, + { + "filename": "data-00020-of-00025.arrow" + }, + { + "filename": "data-00021-of-00025.arrow" + }, + { + "filename": "data-00022-of-00025.arrow" + }, + { + "filename": "data-00023-of-00025.arrow" + }, + { + "filename": "data-00024-of-00025.arrow" + } + ], + "_fingerprint": "085ca55fbd13db74", + "_format_columns": null, + "_format_kwargs": {}, + "_format_type": null, + "_output_all_columns": false, + "_split": "validation" +} \ No newline at end of file diff --git a/research/trials_01.ipynb b/research/trials_01.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..93bfdc2d60a2be5c0f07dc70868b7120b649acf5 --- /dev/null +++ b/research/trials_01.ipynb @@ -0,0 +1,250 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Resolving data files: 100%|██████████| 41/41 [00:00<00:00, 217679.07it/s]\n", + "Resolving data files: 100%|██████████| 44/44 [00:00<00:00, 150.82it/s]\n", + "Resolving data files: 100%|██████████| 30/30 [00:00<00:00, 207638.81it/s]\n", + "Resolving data files: 100%|██████████| 41/41 [00:00<00:00, 170771.07it/s]\n", + "Resolving data files: 100%|██████████| 44/44 [00:00<00:00, 173.94it/s]\n", + "Resolving data files: 100%|██████████| 30/30 [00:00<00:00, 213995.10it/s]\n", + "Downloading data: 100%|██████████| 539M/539M [00:27<00:00, 19.7MB/s]\n", + "Downloading data: 100%|██████████| 531M/531M [00:36<00:00, 14.7MB/s]\n", + "Downloading data: 100%|██████████| 458M/458M [00:33<00:00, 13.6MB/s]\n", + "Downloading data: 100%|██████████| 480M/480M [00:35<00:00, 13.7MB/s]\n", + "Downloading data: 100%|██████████| 567M/567M [00:40<00:00, 14.0MB/s]\n", + "Downloading data: 100%|██████████| 591M/591M [00:41<00:00, 14.2MB/s]\n", + "Downloading data: 100%|██████████| 565M/565M [00:41<00:00, 13.6MB/s]\n", + "Downloading data: 100%|██████████| 500M/500M [00:33<00:00, 15.0MB/s]\n", + "Downloading data: 100%|██████████| 483M/483M [00:35<00:00, 13.7MB/s]\n", + "Downloading data: 100%|██████████| 694M/694M [00:47<00:00, 14.5MB/s]\n", + "Downloading data: 100%|██████████| 513M/513M [00:36<00:00, 14.2MB/s]\n", + "Downloading data: 100%|██████████| 620M/620M [00:45<00:00, 13.7MB/s]\n", + "Downloading data: 100%|██████████| 528M/528M [00:35<00:00, 14.8MB/s]\n", + "Downloading data: 100%|██████████| 365M/365M [00:25<00:00, 14.2MB/s]\n", + "Downloading data: 100%|██████████| 54.1M/54.1M [00:03<00:00, 15.3MB/s]\n", + "Downloading data: 100%|██████████| 55.6M/55.6M [00:04<00:00, 12.7MB/s]\n", + "Downloading data: 100%|██████████| 53.3M/53.3M [00:03<00:00, 17.4MB/s]\n", + "Downloading data: 100%|██████████| 54.6M/54.6M [00:03<00:00, 15.8MB/s]\n", + "Downloading data: 100%|██████████| 37.4M/37.4M [00:02<00:00, 14.7MB/s]\n", + "Downloading data: 100%|██████████| 41.4M/41.4M [00:02<00:00, 15.0MB/s]\n", + "Downloading data: 100%|██████████| 50.7M/50.7M [00:04<00:00, 12.0MB/s]\n", + "Downloading data: 100%|██████████| 52.0M/52.0M [00:04<00:00, 12.7MB/s]\n", + "Downloading data: 100%|██████████| 45.3M/45.3M [00:03<00:00, 14.3MB/s]\n", + "Downloading data: 100%|██████████| 46.7M/46.7M [00:03<00:00, 12.8MB/s]\n", + "Downloading data: 100%|██████████| 45.4M/45.4M [00:03<00:00, 12.3MB/s]\n", + "Downloading data: 100%|██████████| 63.0M/63.0M [00:04<00:00, 15.7MB/s]\n", + "Downloading data: 100%|██████████| 44.2M/44.2M [00:03<00:00, 14.0MB/s]\n", + "Downloading data: 100%|██████████| 43.1M/43.1M [00:03<00:00, 11.3MB/s]\n", + "Downloading data: 100%|██████████| 46.0M/46.0M [00:02<00:00, 16.8MB/s]\n", + "Downloading data: 100%|██████████| 42.9M/42.9M [00:03<00:00, 13.0MB/s]\n", + "Downloading data: 100%|██████████| 44.4M/44.4M [00:02<00:00, 19.7MB/s]\n", + "Downloading data: 100%|██████████| 53.7M/53.7M [00:03<00:00, 16.0MB/s]\n", + "Downloading data: 100%|██████████| 42.8M/42.8M [00:01<00:00, 22.9MB/s]\n", + "Downloading data: 100%|██████████| 56.0M/56.0M [00:03<00:00, 14.3MB/s]\n", + "Downloading data: 100%|██████████| 520M/520M [00:36<00:00, 14.4MB/s]\n", + "Downloading data: 100%|██████████| 470M/470M [00:31<00:00, 14.8MB/s]\n", + "Downloading data: 100%|██████████| 510M/510M [00:35<00:00, 14.4MB/s]\n", + "Downloading data: 100%|██████████| 529M/529M [00:34<00:00, 15.2MB/s]\n", + "Downloading data: 100%|██████████| 552M/552M [00:37<00:00, 14.9MB/s]\n", + "Downloading data: 100%|██████████| 497M/497M [00:32<00:00, 15.1MB/s]\n", + "Downloading data: 100%|██████████| 524M/524M [00:35<00:00, 14.8MB/s]\n", + "Downloading data: 100%|██████████| 559M/559M [00:39<00:00, 14.0MB/s]\n", + "Downloading data: 100%|██████████| 534M/534M [00:36<00:00, 14.7MB/s]\n", + "Downloading data: 100%|██████████| 441M/441M [00:27<00:00, 16.2MB/s]\n", + "Downloading data: 100%|██████████| 625M/625M [00:41<00:00, 15.1MB/s]\n", + "Downloading data: 100%|██████████| 666M/666M [00:45<00:00, 14.6MB/s]\n", + "Downloading data: 100%|██████████| 579M/579M [00:40<00:00, 14.5MB/s]\n", + "Downloading data: 100%|██████████| 552M/552M [00:34<00:00, 16.2MB/s]\n", + "Downloading data: 100%|██████████| 607M/607M [00:39<00:00, 15.4MB/s]\n", + "Downloading data: 100%|██████████| 716M/716M [00:50<00:00, 14.3MB/s]\n", + "Downloading data: 100%|██████████| 606M/606M [00:39<00:00, 15.3MB/s]\n", + "Downloading data: 100%|██████████| 572M/572M [00:36<00:00, 15.5MB/s]\n", + "Downloading data: 100%|██████████| 654M/654M [00:44<00:00, 14.7MB/s]\n", + "Downloading data: 100%|██████████| 105M/105M [00:07<00:00, 14.5MB/s]\n", + "Downloading data: 100%|██████████| 57.5M/57.5M [00:04<00:00, 13.9MB/s]\n", + "Downloading data: 100%|██████████| 58.5M/58.5M [00:03<00:00, 17.0MB/s]\n", + "Downloading data: 100%|██████████| 56.1M/56.1M [00:03<00:00, 15.3MB/s]\n", + "Downloading data: 100%|██████████| 45.1M/45.1M [00:02<00:00, 17.3MB/s]\n", + "Downloading data: 100%|██████████| 44.6M/44.6M [00:02<00:00, 18.4MB/s]\n", + "Downloading data: 100%|██████████| 51.7M/51.7M [00:03<00:00, 13.7MB/s]\n", + "Downloading data: 100%|██████████| 61.3M/61.3M [00:03<00:00, 15.5MB/s]\n", + "Downloading data: 100%|██████████| 52.9M/52.9M [00:02<00:00, 17.8MB/s]\n", + "Downloading data: 100%|██████████| 51.7M/51.7M [00:03<00:00, 15.6MB/s]\n", + "Downloading data: 100%|██████████| 44.2M/44.2M [00:03<00:00, 14.2MB/s]\n", + "Downloading data: 100%|██████████| 52.2M/52.2M [00:03<00:00, 16.7MB/s]\n", + "Downloading data: 100%|██████████| 49.0M/49.0M [00:02<00:00, 17.3MB/s]\n", + "Downloading data: 100%|██████████| 50.5M/50.5M [00:03<00:00, 13.1MB/s]\n", + "Downloading data: 100%|██████████| 47.7M/47.7M [00:02<00:00, 20.3MB/s]\n", + "Downloading data: 100%|██████████| 46.8M/46.8M [00:02<00:00, 17.3MB/s]\n", + "Downloading data: 100%|██████████| 47.0M/47.0M [00:03<00:00, 14.1MB/s]\n", + "Downloading data: 100%|██████████| 46.5M/46.5M [00:03<00:00, 13.9MB/s]\n", + "Downloading data: 100%|██████████| 59.4M/59.4M [00:03<00:00, 16.6MB/s]\n", + "Downloading data: 100%|██████████| 58.6M/58.6M [00:03<00:00, 16.4MB/s]\n", + "Downloading data: 100%|██████████| 60.6M/60.6M [00:04<00:00, 14.9MB/s]\n", + "Downloading data: 100%|██████████| 60.5M/60.5M [00:03<00:00, 15.6MB/s]\n", + "Downloading data: 100%|██████████| 60.4M/60.4M [00:03<00:00, 18.4MB/s]\n", + "Downloading data: 100%|██████████| 54.1M/54.1M [00:03<00:00, 15.4MB/s]\n", + "Downloading data: 100%|██████████| 59.6M/59.6M [00:04<00:00, 14.7MB/s]\n", + "Downloading data: 100%|██████████| 476M/476M [00:32<00:00, 14.5MB/s]\n", + "Downloading data: 100%|██████████| 491M/491M [00:33<00:00, 14.7MB/s]\n", + "Downloading data: 100%|██████████| 535M/535M [00:35<00:00, 14.9MB/s]\n", + "Downloading data: 100%|██████████| 506M/506M [00:35<00:00, 14.2MB/s]\n", + "Downloading data: 100%|██████████| 487M/487M [00:31<00:00, 15.4MB/s]\n", + "Downloading data: 100%|██████████| 532M/532M [00:36<00:00, 14.6MB/s]\n", + "Downloading data: 100%|██████████| 561M/561M [00:34<00:00, 16.2MB/s]\n", + "Downloading data: 100%|██████████| 560M/560M [00:35<00:00, 15.6MB/s]\n", + "Downloading data: 100%|██████████| 550M/550M [00:36<00:00, 15.2MB/s]\n", + "Downloading data: 100%|██████████| 445M/445M [00:28<00:00, 15.5MB/s]\n", + "Downloading data: 100%|██████████| 57.8M/57.8M [00:04<00:00, 14.3MB/s]\n", + "Downloading data: 100%|██████████| 55.6M/55.6M [00:03<00:00, 16.7MB/s]\n", + "Downloading data: 100%|██████████| 51.4M/51.4M [00:03<00:00, 16.7MB/s]\n", + "Downloading data: 100%|██████████| 56.5M/56.5M [00:05<00:00, 9.58MB/s]\n", + "Downloading data: 100%|██████████| 40.7M/40.7M [00:01<00:00, 25.5MB/s]\n", + "Downloading data: 100%|██████████| 44.9M/44.9M [00:02<00:00, 19.3MB/s]\n", + "Downloading data: 100%|██████████| 50.1M/50.1M [00:03<00:00, 13.7MB/s]\n", + "Downloading data: 100%|██████████| 52.8M/52.8M [00:03<00:00, 16.1MB/s]\n", + "Downloading data: 100%|██████████| 48.0M/48.0M [00:02<00:00, 17.8MB/s]\n", + "Downloading data: 100%|██████████| 45.3M/45.3M [00:03<00:00, 12.1MB/s]\n", + "Downloading data: 100%|██████████| 46.4M/46.4M [00:02<00:00, 16.4MB/s]\n", + "Downloading data: 100%|██████████| 43.7M/43.7M [00:03<00:00, 13.1MB/s]\n", + "Downloading data: 100%|██████████| 43.7M/43.7M [00:02<00:00, 15.9MB/s]\n", + "Downloading data: 100%|██████████| 45.9M/45.9M [00:02<00:00, 16.6MB/s]\n", + "Downloading data: 100%|██████████| 45.4M/45.4M [00:03<00:00, 13.8MB/s]\n", + "Downloading data: 100%|██████████| 46.1M/46.1M [00:03<00:00, 15.2MB/s]\n", + "Downloading data: 100%|██████████| 45.7M/45.7M [00:03<00:00, 15.0MB/s]\n", + "Downloading data: 100%|██████████| 45.0M/45.0M [00:03<00:00, 14.7MB/s]\n", + "Downloading data: 100%|██████████| 47.9M/47.9M [00:02<00:00, 16.2MB/s]\n", + "Downloading data: 100%|██████████| 54.8M/54.8M [00:04<00:00, 13.4MB/s]\n", + "Generating train split: 100%|██████████| 18000/18000 [01:37<00:00, 184.78 examples/s]\n", + "Generating validation split: 100%|██████████| 20715/20715 [01:38<00:00, 210.47 examples/s]\n", + "Generating test split: 100%|██████████| 13354/13354 [00:50<00:00, 264.34 examples/s]\n" + ] + } + ], + "source": [ + "from datasets import load_dataset\n", + "\n", + "data = load_dataset(\"ideepankarsharma2003/AIGeneratedImages_Midjourney\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Saving the dataset (24/24 shards): 100%|██████████| 18000/18000 [01:38<00:00, 183.49 examples/s] \n", + "Saving the dataset (25/25 shards): 100%|██████████| 20715/20715 [01:42<00:00, 202.85 examples/s]\n", + "Saving the dataset (13/13 shards): 100%|██████████| 13354/13354 [00:44<00:00, 302.42 examples/s]\n" + ] + } + ], + "source": [ + "data.save_to_disk(\"dataset\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['image', 'label'],\n", + " num_rows: 18000\n", + " })\n", + " validation: Dataset({\n", + " features: ['image', 'label'],\n", + " num_rows: 20715\n", + " })\n", + " test: Dataset({\n", + " features: ['image', 'label'],\n", + " num_rows: 13354\n", + " })\n", + "})" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "ename": "ImportError", + "evalue": "To support decoding images, please install 'Pillow'.", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mImportError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[7], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mdata\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mtrain\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\n", + "File \u001b[0;32m~/AI_Image_Classification/venv/lib/python3.10/site-packages/datasets/arrow_dataset.py:2800\u001b[0m, in \u001b[0;36mDataset.__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m 2798\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__getitem__\u001b[39m(\u001b[38;5;28mself\u001b[39m, key): \u001b[38;5;66;03m# noqa: F811\u001b[39;00m\n\u001b[1;32m 2799\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Can be used to index columns (by string names) or rows (by integer index or iterable of indices or bools).\"\"\"\u001b[39;00m\n\u001b[0;32m-> 2800\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_getitem\u001b[49m\u001b[43m(\u001b[49m\u001b[43mkey\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/AI_Image_Classification/venv/lib/python3.10/site-packages/datasets/arrow_dataset.py:2785\u001b[0m, in \u001b[0;36mDataset._getitem\u001b[0;34m(self, key, **kwargs)\u001b[0m\n\u001b[1;32m 2783\u001b[0m formatter \u001b[38;5;241m=\u001b[39m get_formatter(format_type, features\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_info\u001b[38;5;241m.\u001b[39mfeatures, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mformat_kwargs)\n\u001b[1;32m 2784\u001b[0m pa_subtable \u001b[38;5;241m=\u001b[39m query_table(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_data, key, indices\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_indices \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_indices \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m)\n\u001b[0;32m-> 2785\u001b[0m formatted_output \u001b[38;5;241m=\u001b[39m \u001b[43mformat_table\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 2786\u001b[0m \u001b[43m \u001b[49m\u001b[43mpa_subtable\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mkey\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mformatter\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mformatter\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mformat_columns\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mformat_columns\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moutput_all_columns\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_all_columns\u001b[49m\n\u001b[1;32m 2787\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 2788\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m formatted_output\n", + "File \u001b[0;32m~/AI_Image_Classification/venv/lib/python3.10/site-packages/datasets/formatting/formatting.py:629\u001b[0m, in \u001b[0;36mformat_table\u001b[0;34m(table, key, formatter, format_columns, output_all_columns)\u001b[0m\n\u001b[1;32m 627\u001b[0m python_formatter \u001b[38;5;241m=\u001b[39m PythonFormatter(features\u001b[38;5;241m=\u001b[39mformatter\u001b[38;5;241m.\u001b[39mfeatures)\n\u001b[1;32m 628\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m format_columns \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 629\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mformatter\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpa_table\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mquery_type\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mquery_type\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 630\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m query_type \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcolumn\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 631\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m key \u001b[38;5;129;01min\u001b[39;00m format_columns:\n", + "File \u001b[0;32m~/AI_Image_Classification/venv/lib/python3.10/site-packages/datasets/formatting/formatting.py:396\u001b[0m, in \u001b[0;36mFormatter.__call__\u001b[0;34m(self, pa_table, query_type)\u001b[0m\n\u001b[1;32m 394\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, pa_table: pa\u001b[38;5;241m.\u001b[39mTable, query_type: \u001b[38;5;28mstr\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Union[RowFormat, ColumnFormat, BatchFormat]:\n\u001b[1;32m 395\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m query_type \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrow\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m--> 396\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mformat_row\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpa_table\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 397\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m query_type \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcolumn\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m 398\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mformat_column(pa_table)\n", + "File \u001b[0;32m~/AI_Image_Classification/venv/lib/python3.10/site-packages/datasets/formatting/formatting.py:437\u001b[0m, in \u001b[0;36mPythonFormatter.format_row\u001b[0;34m(self, pa_table)\u001b[0m\n\u001b[1;32m 435\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m LazyRow(pa_table, \u001b[38;5;28mself\u001b[39m)\n\u001b[1;32m 436\u001b[0m row \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpython_arrow_extractor()\u001b[38;5;241m.\u001b[39mextract_row(pa_table)\n\u001b[0;32m--> 437\u001b[0m row \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpython_features_decoder\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode_row\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrow\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 438\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m row\n", + "File \u001b[0;32m~/AI_Image_Classification/venv/lib/python3.10/site-packages/datasets/formatting/formatting.py:215\u001b[0m, in \u001b[0;36mPythonFeaturesDecoder.decode_row\u001b[0;34m(self, row)\u001b[0m\n\u001b[1;32m 214\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecode_row\u001b[39m(\u001b[38;5;28mself\u001b[39m, row: \u001b[38;5;28mdict\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mdict\u001b[39m:\n\u001b[0;32m--> 215\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfeatures\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode_example\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrow\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfeatures \u001b[38;5;28;01melse\u001b[39;00m row\n", + "File \u001b[0;32m~/AI_Image_Classification/venv/lib/python3.10/site-packages/datasets/features/features.py:1929\u001b[0m, in \u001b[0;36mFeatures.decode_example\u001b[0;34m(self, example, token_per_repo_id)\u001b[0m\n\u001b[1;32m 1915\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecode_example\u001b[39m(\u001b[38;5;28mself\u001b[39m, example: \u001b[38;5;28mdict\u001b[39m, token_per_repo_id: Optional[Dict[\u001b[38;5;28mstr\u001b[39m, Union[\u001b[38;5;28mstr\u001b[39m, \u001b[38;5;28mbool\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m]]] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[1;32m 1916\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Decode example with custom feature decoding.\u001b[39;00m\n\u001b[1;32m 1917\u001b[0m \n\u001b[1;32m 1918\u001b[0m \u001b[38;5;124;03m Args:\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1926\u001b[0m \u001b[38;5;124;03m `dict[str, Any]`\u001b[39;00m\n\u001b[1;32m 1927\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[0;32m-> 1929\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m {\n\u001b[1;32m 1930\u001b[0m column_name: decode_nested_example(feature, value, token_per_repo_id\u001b[38;5;241m=\u001b[39mtoken_per_repo_id)\n\u001b[1;32m 1931\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_column_requires_decoding[column_name]\n\u001b[1;32m 1932\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m value\n\u001b[1;32m 1933\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m column_name, (feature, value) \u001b[38;5;129;01min\u001b[39;00m zip_dict(\n\u001b[1;32m 1934\u001b[0m {key: value \u001b[38;5;28;01mfor\u001b[39;00m key, value \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mitems() \u001b[38;5;28;01mif\u001b[39;00m key \u001b[38;5;129;01min\u001b[39;00m example}, example\n\u001b[1;32m 1935\u001b[0m )\n\u001b[1;32m 1936\u001b[0m }\n", + "File \u001b[0;32m~/AI_Image_Classification/venv/lib/python3.10/site-packages/datasets/features/features.py:1930\u001b[0m, in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m 1915\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecode_example\u001b[39m(\u001b[38;5;28mself\u001b[39m, example: \u001b[38;5;28mdict\u001b[39m, token_per_repo_id: Optional[Dict[\u001b[38;5;28mstr\u001b[39m, Union[\u001b[38;5;28mstr\u001b[39m, \u001b[38;5;28mbool\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m]]] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[1;32m 1916\u001b[0m \u001b[38;5;250m \u001b[39m\u001b[38;5;124;03m\"\"\"Decode example with custom feature decoding.\u001b[39;00m\n\u001b[1;32m 1917\u001b[0m \n\u001b[1;32m 1918\u001b[0m \u001b[38;5;124;03m Args:\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 1926\u001b[0m \u001b[38;5;124;03m `dict[str, Any]`\u001b[39;00m\n\u001b[1;32m 1927\u001b[0m \u001b[38;5;124;03m \"\"\"\u001b[39;00m\n\u001b[1;32m 1929\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m {\n\u001b[0;32m-> 1930\u001b[0m column_name: \u001b[43mdecode_nested_example\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfeature\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvalue\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtoken_per_repo_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtoken_per_repo_id\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1931\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_column_requires_decoding[column_name]\n\u001b[1;32m 1932\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m value\n\u001b[1;32m 1933\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m column_name, (feature, value) \u001b[38;5;129;01min\u001b[39;00m zip_dict(\n\u001b[1;32m 1934\u001b[0m {key: value \u001b[38;5;28;01mfor\u001b[39;00m key, value \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mitems() \u001b[38;5;28;01mif\u001b[39;00m key \u001b[38;5;129;01min\u001b[39;00m example}, example\n\u001b[1;32m 1935\u001b[0m )\n\u001b[1;32m 1936\u001b[0m }\n", + "File \u001b[0;32m~/AI_Image_Classification/venv/lib/python3.10/site-packages/datasets/features/features.py:1339\u001b[0m, in \u001b[0;36mdecode_nested_example\u001b[0;34m(schema, obj, token_per_repo_id)\u001b[0m\n\u001b[1;32m 1336\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(schema, (Audio, Image)):\n\u001b[1;32m 1337\u001b[0m \u001b[38;5;66;03m# we pass the token to read and decode files from private repositories in streaming mode\u001b[39;00m\n\u001b[1;32m 1338\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m obj \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m schema\u001b[38;5;241m.\u001b[39mdecode:\n\u001b[0;32m-> 1339\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mschema\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode_example\u001b[49m\u001b[43m(\u001b[49m\u001b[43mobj\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mtoken_per_repo_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtoken_per_repo_id\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1340\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m obj\n", + "File \u001b[0;32m~/AI_Image_Classification/venv/lib/python3.10/site-packages/datasets/features/image.py:155\u001b[0m, in \u001b[0;36mImage.decode_example\u001b[0;34m(self, value, token_per_repo_id)\u001b[0m\n\u001b[1;32m 153\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mPIL\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mImage\u001b[39;00m\n\u001b[1;32m 154\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 155\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mTo support decoding images, please install \u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mPillow\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 157\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m token_per_repo_id \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 158\u001b[0m token_per_repo_id \u001b[38;5;241m=\u001b[39m {}\n", + "\u001b[0;31mImportError\u001b[0m: To support decoding images, please install 'Pillow'." + ] + } + ], + "source": [ + "data['train'][0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/research/trials_02.ipynb b/research/trials_02.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..b0cf461184ce9f14ce84341f2ded15a9a13c4bb9 --- /dev/null +++ b/research/trials_02.ipynb @@ -0,0 +1,374 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/ubuntu/AI_Image_Classification/venv/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n", + "Resolving data files: 100%|██████████| 25/25 [00:00<00:00, 203606.99it/s]\n", + "Resolving data files: 100%|██████████| 26/26 [00:00<00:00, 203076.17it/s]\n" + ] + } + ], + "source": [ + "from datasets import load_dataset\n", + "\n", + "data = load_dataset(\"dataset\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# !pip install Pillow" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'image': ,\n", + " 'label': 0}" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data['train'][0]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['ai_gen', 'human']" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "labels = data[\"train\"].features[\"label\"].names\n", + "labels" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "label2id, id2label = dict(), dict()\n", + "for i, label in enumerate(labels):\n", + " label2id[label] = str(i)\n", + " id2label[str(i)] = label" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'ai_gen': '0', 'human': '1'}" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "label2id" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import AutoImageProcessor\n", + "\n", + "checkpoint = \"google/vit-base-patch16-224-in21k\"\n", + "image_processor = AutoImageProcessor.from_pretrained(checkpoint)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "from torchvision.transforms import RandomResizedCrop, Compose, Normalize, ToTensor\n", + "\n", + "normalize = Normalize(mean=image_processor.image_mean, std=image_processor.image_std)\n", + "size = (\n", + " image_processor.size[\"shortest_edge\"]\n", + " if \"shortest_edge\" in image_processor.size\n", + " else (image_processor.size[\"height\"], image_processor.size[\"width\"])\n", + ")\n", + "_transforms = Compose([RandomResizedCrop(size), ToTensor(), normalize])" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "def transforms(examples):\n", + " examples[\"pixel_values\"] = [_transforms(img.convert(\"RGB\")) for img in examples[\"image\"]]\n", + " del examples[\"image\"]\n", + " return examples" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "data = data.with_transform(transforms)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['image', 'label'],\n", + " num_rows: 18000\n", + " })\n", + " validation: Dataset({\n", + " features: ['image', 'label'],\n", + " num_rows: 20715\n", + " })\n", + " test: Dataset({\n", + " features: ['image', 'label'],\n", + " num_rows: 13354\n", + " })\n", + "})" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import DefaultDataCollator\n", + "\n", + "data_collator = DefaultDataCollator()" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/ubuntu/AI_Image_Classification/venv/lib/python3.10/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly. To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n", + " return self.fget.__get__(instance, owner)()\n" + ] + } + ], + "source": [ + "from transformers import AutoModelForImageClassification, TrainingArguments, Trainer\n", + "\n", + "model = AutoModelForImageClassification.from_pretrained(\n", + " \"umm-maybe/AI-image-detector\",\n", + " num_labels=len(labels),\n", + " id2label=id2label,\n", + " label2id=label2id,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "import evaluate\n", + "\n", + "accuracy = evaluate.load(\"accuracy\")\n", + "\n", + "import numpy as np\n", + "\n", + "\n", + "def compute_metrics(eval_pred):\n", + " predictions, labels = eval_pred\n", + " predictions = np.argmax(predictions, axis=1)\n", + " return accuracy.compute(predictions=predictions, references=labels)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "
\n", + " \n", + " \n", + " [843/843 43:11, Epoch 2/3]\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
EpochTraining LossValidation LossAccuracy
00.0347000.0134690.996480
10.0009000.0211300.994234
20.0241000.0107350.997529

" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Some non-default generation parameters are set in the model config. These should go into a GenerationConfig file (https://huggingface.co/docs/transformers/generation_strategies#save-a-custom-decoding-strategy-with-your-model) instead. This warning will be raised to an exception in v4.41.\n", + "Non-default generation parameters: {'max_length': 128}\n", + "Some non-default generation parameters are set in the model config. These should go into a GenerationConfig file (https://huggingface.co/docs/transformers/generation_strategies#save-a-custom-decoding-strategy-with-your-model) instead. This warning will be raised to an exception in v4.41.\n", + "Non-default generation parameters: {'max_length': 128}\n", + "Some non-default generation parameters are set in the model config. These should go into a GenerationConfig file (https://huggingface.co/docs/transformers/generation_strategies#save-a-custom-decoding-strategy-with-your-model) instead. This warning will be raised to an exception in v4.41.\n", + "Non-default generation parameters: {'max_length': 128}\n" + ] + }, + { + "data": { + "text/plain": [ + "TrainOutput(global_step=843, training_loss=0.034790725539037115, metrics={'train_runtime': 2594.7053, 'train_samples_per_second': 20.812, 'train_steps_per_second': 0.325, 'total_flos': 4.2268994172435825e+18, 'train_loss': 0.034790725539037115, 'epoch': 3.0})" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "training_args = TrainingArguments(\n", + " output_dir=\"ai_detector\",\n", + " remove_unused_columns=False,\n", + " evaluation_strategy=\"epoch\",\n", + " save_strategy=\"epoch\",\n", + " learning_rate=5e-5,\n", + " per_device_train_batch_size=16,\n", + " gradient_accumulation_steps=4,\n", + " per_device_eval_batch_size=16,\n", + " num_train_epochs=3,\n", + " warmup_ratio=0.1,\n", + " logging_steps=10,\n", + " load_best_model_at_end=True,\n", + " metric_for_best_model=\"accuracy\",\n", + " # push_to_hub=True,\n", + ")\n", + "\n", + "trainer = Trainer(\n", + " model=model,\n", + " args=training_args,\n", + " data_collator=data_collator,\n", + " train_dataset=data[\"train\"],\n", + " eval_dataset=data[\"test\"],\n", + " tokenizer=image_processor,\n", + " compute_metrics=compute_metrics,\n", + ")\n", + "\n", + "trainer.train()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/research/trials_03.ipynb b/research/trials_03.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..94be94abacebb2a397641254636b297f8cc83f28 --- /dev/null +++ b/research/trials_03.ipynb @@ -0,0 +1,459 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/ubuntu/AI_Image_Classification/venv/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n", + "Resolving data files: 100%|██████████| 25/25 [00:00<00:00, 150874.24it/s]\n", + "Resolving data files: 100%|██████████| 26/26 [00:00<00:00, 154683.55it/s]\n" + ] + } + ], + "source": [ + "from datasets import load_dataset\n", + "\n", + "data = load_dataset(\"dataset\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# !pip install Pillow" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'image': ,\n", + " 'label': 0}" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data['train'][0]" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['ai_gen', 'human']" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "labels = data[\"train\"].features[\"label\"].names\n", + "labels" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "label2id, id2label = dict(), dict()\n", + "for i, label in enumerate(labels):\n", + " label2id[label] = str(i)\n", + " id2label[str(i)] = label" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'ai_gen': '0', 'human': '1'}" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "label2id" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import AutoImageProcessor\n", + "\n", + "checkpoint = \"google/vit-base-patch16-224-in21k\"\n", + "image_processor = AutoImageProcessor.from_pretrained(checkpoint)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "from torchvision.transforms import RandomResizedCrop, Compose, Normalize, ToTensor\n", + "\n", + "normalize = Normalize(mean=image_processor.image_mean, std=image_processor.image_std)\n", + "size = (\n", + " image_processor.size[\"shortest_edge\"]\n", + " if \"shortest_edge\" in image_processor.size\n", + " else (image_processor.size[\"height\"], image_processor.size[\"width\"])\n", + ")\n", + "_transforms = Compose([RandomResizedCrop(size), ToTensor(), normalize])" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "def transforms(examples):\n", + " examples[\"pixel_values\"] = [_transforms(img.convert(\"RGB\")) for img in examples[\"image\"]]\n", + " del examples[\"image\"]\n", + " return examples" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "data = data.with_transform(transforms)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['image', 'label'],\n", + " num_rows: 18000\n", + " })\n", + " validation: Dataset({\n", + " features: ['image', 'label'],\n", + " num_rows: 20715\n", + " })\n", + " test: Dataset({\n", + " features: ['image', 'label'],\n", + " num_rows: 13354\n", + " })\n", + "})" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['image', 'label'],\n", + " num_rows: 38715\n", + " })\n", + " test: Dataset({\n", + " features: ['image', 'label'],\n", + " num_rows: 13354\n", + " })\n", + "})" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from datasets import DatasetDict, concatenate_datasets\n", + "\n", + "\n", + "# Concatenate train and validation datasets\n", + "combined_train_validation = concatenate_datasets([data['train'], data['validation']])\n", + "\n", + "# Now your datasets are\n", + "datasets = DatasetDict({\n", + " 'train': combined_train_validation,\n", + " 'test': data['test']\n", + "})\n", + "datasets" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "data= datasets" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "DatasetDict({\n", + " train: Dataset({\n", + " features: ['image', 'label'],\n", + " num_rows: 38715\n", + " })\n", + " test: Dataset({\n", + " features: ['image', 'label'],\n", + " num_rows: 13354\n", + " })\n", + "})" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "from transformers import DefaultDataCollator\n", + "\n", + "data_collator = DefaultDataCollator()" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/ubuntu/AI_Image_Classification/venv/lib/python3.10/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly. To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n", + " return self.fget.__get__(instance, owner)()\n" + ] + } + ], + "source": [ + "from transformers import AutoModelForImageClassification, TrainingArguments, Trainer\n", + "\n", + "model = AutoModelForImageClassification.from_pretrained(\n", + " \"umm-maybe/AI-image-detector\",\n", + " num_labels=len(labels),\n", + " id2label=id2label,\n", + " label2id=label2id,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "import evaluate\n", + "\n", + "accuracy = evaluate.load(\"accuracy\")\n", + "\n", + "import numpy as np\n", + "\n", + "\n", + "def compute_metrics(eval_pred):\n", + " predictions, labels = eval_pred\n", + " predictions = np.argmax(predictions, axis=1)\n", + " return accuracy.compute(predictions=predictions, references=labels)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "

\n", + " \n", + " \n", + " [1815/1815 1:16:27, Epoch 3/3]\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
EpochTraining LossValidation LossAccuracy
10.0187000.0102420.997379
20.0079000.0083250.998053
30.0002000.0072710.998278

" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Some non-default generation parameters are set in the model config. These should go into a GenerationConfig file (https://huggingface.co/docs/transformers/generation_strategies#save-a-custom-decoding-strategy-with-your-model) instead. This warning will be raised to an exception in v4.41.\n", + "Non-default generation parameters: {'max_length': 128}\n", + "Some non-default generation parameters are set in the model config. These should go into a GenerationConfig file (https://huggingface.co/docs/transformers/generation_strategies#save-a-custom-decoding-strategy-with-your-model) instead. This warning will be raised to an exception in v4.41.\n", + "Non-default generation parameters: {'max_length': 128}\n", + "Some non-default generation parameters are set in the model config. These should go into a GenerationConfig file (https://huggingface.co/docs/transformers/generation_strategies#save-a-custom-decoding-strategy-with-your-model) instead. This warning will be raised to an exception in v4.41.\n", + "Non-default generation parameters: {'max_length': 128}\n" + ] + }, + { + "data": { + "text/plain": [ + "TrainOutput(global_step=1815, training_loss=0.025701069246692784, metrics={'train_runtime': 4592.2162, 'train_samples_per_second': 25.292, 'train_steps_per_second': 0.395, 'total_flos': 9.099444558417777e+18, 'train_loss': 0.025701069246692784, 'epoch': 3.0})" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "training_args = TrainingArguments(\n", + " output_dir=\"ai_detector_v2\",\n", + " remove_unused_columns=False,\n", + " evaluation_strategy=\"epoch\",\n", + " save_strategy=\"epoch\",\n", + " learning_rate=5e-5,\n", + " per_device_train_batch_size=16,\n", + " gradient_accumulation_steps=4,\n", + " per_device_eval_batch_size=16,\n", + " num_train_epochs=3,\n", + " warmup_ratio=0.1,\n", + " logging_steps=10,\n", + " load_best_model_at_end=True,\n", + " metric_for_best_model=\"accuracy\",\n", + " # push_to_hub=True,\n", + ")\n", + "\n", + "trainer = Trainer(\n", + " model=model,\n", + " args=training_args,\n", + " data_collator=data_collator,\n", + " train_dataset=data[\"train\"],\n", + " eval_dataset=data[\"test\"],\n", + " tokenizer=image_processor,\n", + " compute_metrics=compute_metrics,\n", + ")\n", + "\n", + "trainer.train()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}