Spaces:

soutrik
/

gradio_demo_CatDogClassifier

Runtime error

App Files Files Community

soutrik commited on Dec 2, 2024

Commit

4828471

1 Parent(s): f057c2a

added new changes as per ResnetClassifier and tested with local and docker

Browse files

Files changed (10) hide show

configs/experiment/catdog_experiment_resnet.yaml +4 -5
configs/infer.yaml +1 -1
configs/model/catdog_classifier_resnet.yaml +3 -6
docker-compose.yaml +5 -38
poetry.lock +75 -75
pyproject.toml +1 -1
src/infer.py +10 -3
src/models/catdog_model_resnet.py +4 -2
src/train_optuna_callbacks.py +11 -3
src/utils/aws_s3_services.py +78 -0

configs/experiment/catdog_experiment_resnet.yaml CHANGED Viewed

@@ -6,7 +6,7 @@
 defaults:
   - override /paths: catdog
   - override /data: catdog
-  - override /model: catdog_classifier_convnext
   - override /callbacks: default
   - override /logger: default
   - override /trainer: default
@@ -15,7 +15,7 @@ defaults:
 # this allows you to overwrite only specified parameters
 seed: 42
-name: "catdog_experiment_convnext"
 # Logger-specific configurations
 logger:
@@ -33,7 +33,7 @@ data:
   image_size: 160
 model:
-  base_model: convnext_tiny.fb_in22k_ft_in1k
   pretrained: True
   lr: 1e-3
   weight_decay: 1e-5
@@ -41,11 +41,10 @@ model:
   patience: 5
   min_lr: 1e-6
   num_classes: 2
-  kernel_sizes: 7
 trainer:
   min_epochs: 1
-  max_epochs: 3
 callbacks:
   model_checkpoint:

 defaults:
   - override /paths: catdog
   - override /data: catdog
+  - override /model: catdog_classifier_resnet
   - override /callbacks: default
   - override /logger: default
   - override /trainer: default
 # this allows you to overwrite only specified parameters
 seed: 42
+name: "catdog_experiment_resnet"
 # Logger-specific configurations
 logger:
   image_size: 160
 model:
+  base_model: efficientnet_b0
   pretrained: True
   lr: 1e-3
   weight_decay: 1e-5
   patience: 5
   min_lr: 1e-6
   num_classes: 2
 trainer:
   min_epochs: 1
+  max_epochs: 5
 callbacks:
   model_checkpoint:

configs/infer.yaml CHANGED Viewed

@@ -5,7 +5,7 @@
 defaults:
   - _self_
   - data: catdog
-  - model: catdog_model
   - callbacks: default
   - logger: null # set logger here or use command line (e.g. `python train.py logger=tensorboard`)
   - trainer: default

 defaults:
   - _self_
   - data: catdog
+  - model: catdog_classifier
   - callbacks: default
   - logger: null # set logger here or use command line (e.g. `python train.py logger=tensorboard`)
   - trainer: default

configs/model/catdog_classifier_resnet.yaml CHANGED Viewed

@@ -1,16 +1,13 @@
-_target_: src.models.catdog_classifier_convnext.ConvNextClassifier
 # model params
-base_model: convnext_tiny.in12k_ft_in1k
 pretrained: True
 num_classes: 2
-kernel_sizes: 7
 # optimizer params
 lr: 1e-3
 weight_decay: 1e-5
 # scheduler params
 factor: 0.1
 patience: 10
-min_lr: 1e-6

+_target_: src.models.catdog_model_resnet.ResnetClassifier
 # model params
+base_model: efficientnet_b0
 pretrained: True
 num_classes: 2
 # optimizer params
 lr: 1e-3
 weight_decay: 1e-5
 # scheduler params
 factor: 0.1
 patience: 10
+min_lr: 1e-6

docker-compose.yaml CHANGED Viewed

@@ -3,7 +3,7 @@ services:
     build:
       context: .
     command: |
-      python -m src.train_optuna_callbacks experiment=catdog_experiment ++task_name=train ++train=True ++test=False && \
       python -m src.create_artifacts && \
       touch ./checkpoints/train_done.flag
     volumes:
@@ -31,7 +31,7 @@ services:
     build:
       context: .
     command: |
-      sh -c 'while [ ! -f /app/checkpoints/train_done.flag ]; do sleep 10; done && python -m src.train_optuna_callbacks experiment=catdog_experiment ++task_name=test ++train=False ++test=True'
     volumes:
       - ./data:/app/data
       - ./checkpoints:/app/checkpoints
@@ -52,13 +52,12 @@ services:
             - driver: nvidia
               count: 1
               capabilities: [gpu]
-  server:
     build:
       context: .
     command: |
-      sh -c 'while [ ! -f /app/checkpoints/train_done.flag ]; do sleep 10; done && python -m src.server'
     volumes:
       - ./data:/app/data
       - ./checkpoints:/app/checkpoints
@@ -67,14 +66,11 @@ services:
     environment:
       - PYTHONUNBUFFERED=1
       - PYTHONPATH=/app
-      - SERVER_URL=http://localhost:8080
     shm_size: '4g'
     networks:
       - default
     env_file:
       - .env
-    ports:
-      - "8080:8080"
     deploy:
       resources:
         reservations:
@@ -82,37 +78,8 @@ services:
             - driver: nvidia
               count: 1
               capabilities: [gpu]
-  client:
-    build:
-      context: .
-    command: |
-      sh -c 'until curl -s http://server:8080/health; do echo "Waiting for server to be ready..."; sleep 5; done && \
-      ./run_client.sh'
-    volumes:
-      - ./data:/app/data
-      - ./checkpoints:/app/checkpoints
-      - ./artifacts:/app/artifacts
-      - ./logs:/app/logs
-    environment:
-      - PYTHONUNBUFFERED=1
-      - PYTHONPATH=/app
-      - SERVER_URL=http://server:8080
-    shm_size: '4g'
-    networks:
-      - default
-    env_file:
-      - .env
-    deploy:
-      resources:
-        reservations:
-          devices:
-            - driver: nvidia
-              count: 1
-              capabilities: [gpu]
 volumes:
   data:
   checkpoints:

     build:
       context: .
     command: |
+      python -m src.train_optuna_callbacks experiment=catdog_experiment_resnet ++task_name=train ++train=True ++test=False && \
       python -m src.create_artifacts && \
       touch ./checkpoints/train_done.flag
     volumes:
     build:
       context: .
     command: |
+      sh -c 'while [ ! -f /app/checkpoints/train_done.flag ]; do sleep 10; done && python -m src.train_optuna_callbacks experiment=catdog_experiment_resnet ++task_name=test ++train=False ++test=True'
     volumes:
       - ./data:/app/data
       - ./checkpoints:/app/checkpoints
             - driver: nvidia
               count: 1
               capabilities: [gpu]
+  inference:
     build:
       context: .
     command: |
+      sh -c 'while [ ! -f /app/checkpoints/train_done.flag ]; do sleep 10; done && python -m src.infer experiment=catdog_experiment_resnet'
     volumes:
       - ./data:/app/data
       - ./checkpoints:/app/checkpoints
     environment:
       - PYTHONUNBUFFERED=1
       - PYTHONPATH=/app
     shm_size: '4g'
     networks:
       - default
     env_file:
       - .env
     deploy:
       resources:
         reservations:
             - driver: nvidia
               count: 1
               capabilities: [gpu]
 volumes:
   data:
   checkpoints:

poetry.lock CHANGED Viewed

@@ -3014,70 +3014,70 @@ test = ["objgraph", "psutil"]
 [[package]]
 name = "grpcio"
-version = "1.68.0"
 description = "HTTP/2-based RPC framework"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "grpcio-1.68.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:619b5d0f29f4f5351440e9343224c3e19912c21aeda44e0c49d0d147a8d01544"},
-    {file = "grpcio-1.68.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:a59f5822f9459bed098ffbceb2713abbf7c6fd13f2b9243461da5c338d0cd6c3"},
-    {file = "grpcio-1.68.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:c03d89df516128febc5a7e760d675b478ba25802447624edf7aa13b1e7b11e2a"},
-    {file = "grpcio-1.68.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:44bcbebb24363d587472089b89e2ea0ab2e2b4df0e4856ba4c0b087c82412121"},
-    {file = "grpcio-1.68.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79f81b7fbfb136247b70465bd836fa1733043fdee539cd6031cb499e9608a110"},
-    {file = "grpcio-1.68.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:88fb2925789cfe6daa20900260ef0a1d0a61283dfb2d2fffe6194396a354c618"},
-    {file = "grpcio-1.68.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:99f06232b5c9138593ae6f2e355054318717d32a9c09cdc5a2885540835067a1"},
-    {file = "grpcio-1.68.0-cp310-cp310-win32.whl", hash = "sha256:a6213d2f7a22c3c30a479fb5e249b6b7e648e17f364598ff64d08a5136fe488b"},
-    {file = "grpcio-1.68.0-cp310-cp310-win_amd64.whl", hash = "sha256:15327ab81131ef9b94cb9f45b5bd98803a179c7c61205c8c0ac9aff9d6c4e82a"},
-    {file = "grpcio-1.68.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:3b2b559beb2d433129441783e5f42e3be40a9e1a89ec906efabf26591c5cd415"},
-    {file = "grpcio-1.68.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e46541de8425a4d6829ac6c5d9b16c03c292105fe9ebf78cb1c31e8d242f9155"},
-    {file = "grpcio-1.68.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:c1245651f3c9ea92a2db4f95d37b7597db6b246d5892bca6ee8c0e90d76fb73c"},
-    {file = "grpcio-1.68.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f1931c7aa85be0fa6cea6af388e576f3bf6baee9e5d481c586980c774debcb4"},
-    {file = "grpcio-1.68.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b0ff09c81e3aded7a183bc6473639b46b6caa9c1901d6f5e2cba24b95e59e30"},
-    {file = "grpcio-1.68.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8c73f9fbbaee1a132487e31585aa83987ddf626426d703ebcb9a528cf231c9b1"},
-    {file = "grpcio-1.68.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6b2f98165ea2790ea159393a2246b56f580d24d7da0d0342c18a085299c40a75"},
-    {file = "grpcio-1.68.0-cp311-cp311-win32.whl", hash = "sha256:e1e7ed311afb351ff0d0e583a66fcb39675be112d61e7cfd6c8269884a98afbc"},
-    {file = "grpcio-1.68.0-cp311-cp311-win_amd64.whl", hash = "sha256:e0d2f68eaa0a755edd9a47d40e50dba6df2bceda66960dee1218da81a2834d27"},
-    {file = "grpcio-1.68.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:8af6137cc4ae8e421690d276e7627cfc726d4293f6607acf9ea7260bd8fc3d7d"},
-    {file = "grpcio-1.68.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:4028b8e9a3bff6f377698587d642e24bd221810c06579a18420a17688e421af7"},
-    {file = "grpcio-1.68.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:f60fa2adf281fd73ae3a50677572521edca34ba373a45b457b5ebe87c2d01e1d"},
-    {file = "grpcio-1.68.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e18589e747c1e70b60fab6767ff99b2d0c359ea1db8a2cb524477f93cdbedf5b"},
-    {file = "grpcio-1.68.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e0d30f3fee9372796f54d3100b31ee70972eaadcc87314be369360248a3dcffe"},
-    {file = "grpcio-1.68.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:7e0a3e72c0e9a1acab77bef14a73a416630b7fd2cbd893c0a873edc47c42c8cd"},
-    {file = "grpcio-1.68.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a831dcc343440969aaa812004685ed322cdb526cd197112d0db303b0da1e8659"},
-    {file = "grpcio-1.68.0-cp312-cp312-win32.whl", hash = "sha256:5a180328e92b9a0050958ced34dddcb86fec5a8b332f5a229e353dafc16cd332"},
-    {file = "grpcio-1.68.0-cp312-cp312-win_amd64.whl", hash = "sha256:2bddd04a790b69f7a7385f6a112f46ea0b34c4746f361ebafe9ca0be567c78e9"},
-    {file = "grpcio-1.68.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:fc05759ffbd7875e0ff2bd877be1438dfe97c9312bbc558c8284a9afa1d0f40e"},
-    {file = "grpcio-1.68.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:15fa1fe25d365a13bc6d52fcac0e3ee1f9baebdde2c9b3b2425f8a4979fccea1"},
-    {file = "grpcio-1.68.0-cp313-cp313-manylinux_2_17_aarch64.whl", hash = "sha256:32a9cb4686eb2e89d97022ecb9e1606d132f85c444354c17a7dbde4a455e4a3b"},
-    {file = "grpcio-1.68.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dba037ff8d284c8e7ea9a510c8ae0f5b016004f13c3648f72411c464b67ff2fb"},
-    {file = "grpcio-1.68.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0efbbd849867e0e569af09e165363ade75cf84f5229b2698d53cf22c7a4f9e21"},
-    {file = "grpcio-1.68.0-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:4e300e6978df0b65cc2d100c54e097c10dfc7018b9bd890bbbf08022d47f766d"},
-    {file = "grpcio-1.68.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:6f9c7ad1a23e1047f827385f4713b5b8c6c7d325705be1dd3e31fb00dcb2f665"},
-    {file = "grpcio-1.68.0-cp313-cp313-win32.whl", hash = "sha256:3ac7f10850fd0487fcce169c3c55509101c3bde2a3b454869639df2176b60a03"},
-    {file = "grpcio-1.68.0-cp313-cp313-win_amd64.whl", hash = "sha256:afbf45a62ba85a720491bfe9b2642f8761ff348006f5ef67e4622621f116b04a"},
-    {file = "grpcio-1.68.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:f8f695d9576ce836eab27ba7401c60acaf9ef6cf2f70dfe5462055ba3df02cc3"},
-    {file = "grpcio-1.68.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:9fe1b141cda52f2ca73e17d2d3c6a9f3f3a0c255c216b50ce616e9dca7e3441d"},
-    {file = "grpcio-1.68.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:4df81d78fd1646bf94ced4fb4cd0a7fe2e91608089c522ef17bc7db26e64effd"},
-    {file = "grpcio-1.68.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:46a2d74d4dd8993151c6cd585594c082abe74112c8e4175ddda4106f2ceb022f"},
-    {file = "grpcio-1.68.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a17278d977746472698460c63abf333e1d806bd41f2224f90dbe9460101c9796"},
-    {file = "grpcio-1.68.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:15377bce516b1c861c35e18eaa1c280692bf563264836cece693c0f169b48829"},
-    {file = "grpcio-1.68.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:cc5f0a4f5904b8c25729a0498886b797feb817d1fd3812554ffa39551112c161"},
-    {file = "grpcio-1.68.0-cp38-cp38-win32.whl", hash = "sha256:def1a60a111d24376e4b753db39705adbe9483ef4ca4761f825639d884d5da78"},
-    {file = "grpcio-1.68.0-cp38-cp38-win_amd64.whl", hash = "sha256:55d3b52fd41ec5772a953612db4e70ae741a6d6ed640c4c89a64f017a1ac02b5"},
-    {file = "grpcio-1.68.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:0d230852ba97654453d290e98d6aa61cb48fa5fafb474fb4c4298d8721809354"},
-    {file = "grpcio-1.68.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:50992f214264e207e07222703c17d9cfdcc2c46ed5a1ea86843d440148ebbe10"},
-    {file = "grpcio-1.68.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:14331e5c27ed3545360464a139ed279aa09db088f6e9502e95ad4bfa852bb116"},
-    {file = "grpcio-1.68.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f84890b205692ea813653ece4ac9afa2139eae136e419231b0eec7c39fdbe4c2"},
-    {file = "grpcio-1.68.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b0cf343c6f4f6aa44863e13ec9ddfe299e0be68f87d68e777328bff785897b05"},
-    {file = "grpcio-1.68.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:fd2c2d47969daa0e27eadaf15c13b5e92605c5e5953d23c06d0b5239a2f176d3"},
-    {file = "grpcio-1.68.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:18668e36e7f4045820f069997834e94e8275910b1f03e078a6020bd464cb2363"},
-    {file = "grpcio-1.68.0-cp39-cp39-win32.whl", hash = "sha256:2af76ab7c427aaa26aa9187c3e3c42f38d3771f91a20f99657d992afada2294a"},
-    {file = "grpcio-1.68.0-cp39-cp39-win_amd64.whl", hash = "sha256:e694b5928b7b33ca2d3b4d5f9bf8b5888906f181daff6b406f4938f3a997a490"},
-    {file = "grpcio-1.68.0.tar.gz", hash = "sha256:7e7483d39b4a4fddb9906671e9ea21aaad4f031cdfc349fec76bdfa1e404543a"},
-]
-[package.extras]
-protobuf = ["grpcio-tools (>=1.68.0)"]
 [[package]]
 name = "gto"
@@ -3600,13 +3600,13 @@ files = [
 [[package]]
 name = "jsonargparse"
-version = "4.34.0"
 description = "Implement minimal boilerplate CLIs derived from type hints and parse from command line, config files and environment variables."
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "jsonargparse-4.34.0-py3-none-any.whl", hash = "sha256:a3eb8a9a289332066b1b33463efa49d5d2a8d729b6cb60e9e30231d0c19dfb13"},
-    {file = "jsonargparse-4.34.0.tar.gz", hash = "sha256:88b3ff0beaff40909dc69244f0527b054f8be0132086c72aa7e1d99414024b43"},
 ]
 [package.dependencies]
@@ -5744,13 +5744,13 @@ tests = ["chardet", "parameterized", "pytest", "pytest-cov", "pytest-xdist[psuti
 [[package]]
 name = "pydrive2"
-version = "1.21.2"
 description = "Google Drive API made easy. Maintained fork of PyDrive."
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "PyDrive2-1.21.2-py3-none-any.whl", hash = "sha256:a3b72e522b8f5ba4e93ab165bbf120544567583c61a6c7904ef1ff47afc005d6"},
-    {file = "pydrive2-1.21.2.tar.gz", hash = "sha256:2a21c8319a225943c70e7566eb13a1524d1d7193621de1eb8e5f95e037641508"},
 ]
 [package.dependencies]
@@ -5759,7 +5759,7 @@ fsspec = {version = ">=2021.07.0", optional = true, markers = "extra == \"fsspec
 funcy = {version = ">=1.14", optional = true, markers = "extra == \"fsspec\""}
 google-api-python-client = ">=1.12.5"
 oauth2client = ">=4.0.0"
-pyOpenSSL = ">=19.1.0,<=24.2.1"
 PyYAML = ">=3.0"
 tqdm = {version = ">=4.0.0", optional = true, markers = "extra == \"fsspec\""}
@@ -5877,21 +5877,21 @@ tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"]
 [[package]]
 name = "pyopenssl"
-version = "22.0.0"
 description = "Python wrapper module around the OpenSSL library"
 optional = false
-python-versions = ">=3.6"
 files = [
-    {file = "pyOpenSSL-22.0.0-py2.py3-none-any.whl", hash = "sha256:ea252b38c87425b64116f808355e8da644ef9b07e429398bfece610f893ee2e0"},
-    {file = "pyOpenSSL-22.0.0.tar.gz", hash = "sha256:660b1b1425aac4a1bea1d94168a85d99f0b3144c869dd4390d27629d0087f1bf"},
 ]
 [package.dependencies]
-cryptography = ">=35.0"
 [package.extras]
-docs = ["sphinx", "sphinx-rtd-theme"]
-test = ["flaky", "pretend", "pytest (>=3.0.1)"]
 [[package]]
 name = "pyparsing"
@@ -8397,4 +8397,4 @@ type = ["pytest-mypy"]
 [metadata]
 lock-version = "2.0"
 python-versions = "3.10.15"
-content-hash = "13c48bb5304783670c78f25e07ef651b8b5ff25ca5f04bced20f9d76327958d3"

 [[package]]
 name = "grpcio"
+version = "1.68.1"
 description = "HTTP/2-based RPC framework"
 optional = false
 python-versions = ">=3.8"
 files = [
+    {file = "grpcio-1.68.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:d35740e3f45f60f3c37b1e6f2f4702c23867b9ce21c6410254c9c682237da68d"},
+    {file = "grpcio-1.68.1-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:d99abcd61760ebb34bdff37e5a3ba333c5cc09feda8c1ad42547bea0416ada78"},
+    {file = "grpcio-1.68.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:f8261fa2a5f679abeb2a0a93ad056d765cdca1c47745eda3f2d87f874ff4b8c9"},
+    {file = "grpcio-1.68.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0feb02205a27caca128627bd1df4ee7212db051019a9afa76f4bb6a1a80ca95e"},
+    {file = "grpcio-1.68.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:919d7f18f63bcad3a0f81146188e90274fde800a94e35d42ffe9eadf6a9a6330"},
+    {file = "grpcio-1.68.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:963cc8d7d79b12c56008aabd8b457f400952dbea8997dd185f155e2f228db079"},
+    {file = "grpcio-1.68.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ccf2ebd2de2d6661e2520dae293298a3803a98ebfc099275f113ce1f6c2a80f1"},
+    {file = "grpcio-1.68.1-cp310-cp310-win32.whl", hash = "sha256:2cc1fd04af8399971bcd4f43bd98c22d01029ea2e56e69c34daf2bf8470e47f5"},
+    {file = "grpcio-1.68.1-cp310-cp310-win_amd64.whl", hash = "sha256:ee2e743e51cb964b4975de572aa8fb95b633f496f9fcb5e257893df3be854746"},
+    {file = "grpcio-1.68.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:55857c71641064f01ff0541a1776bfe04a59db5558e82897d35a7793e525774c"},
+    {file = "grpcio-1.68.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4b177f5547f1b995826ef529d2eef89cca2f830dd8b2c99ffd5fde4da734ba73"},
+    {file = "grpcio-1.68.1-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:3522c77d7e6606d6665ec8d50e867f13f946a4e00c7df46768f1c85089eae515"},
+    {file = "grpcio-1.68.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9d1fae6bbf0816415b81db1e82fb3bf56f7857273c84dcbe68cbe046e58e1ccd"},
+    {file = "grpcio-1.68.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:298ee7f80e26f9483f0b6f94cc0a046caf54400a11b644713bb5b3d8eb387600"},
+    {file = "grpcio-1.68.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cbb5780e2e740b6b4f2d208e90453591036ff80c02cc605fea1af8e6fc6b1bbe"},
+    {file = "grpcio-1.68.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ddda1aa22495d8acd9dfbafff2866438d12faec4d024ebc2e656784d96328ad0"},
+    {file = "grpcio-1.68.1-cp311-cp311-win32.whl", hash = "sha256:b33bd114fa5a83f03ec6b7b262ef9f5cac549d4126f1dc702078767b10c46ed9"},
+    {file = "grpcio-1.68.1-cp311-cp311-win_amd64.whl", hash = "sha256:7f20ebec257af55694d8f993e162ddf0d36bd82d4e57f74b31c67b3c6d63d8b2"},
+    {file = "grpcio-1.68.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:8829924fffb25386995a31998ccbbeaa7367223e647e0122043dfc485a87c666"},
+    {file = "grpcio-1.68.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:3aed6544e4d523cd6b3119b0916cef3d15ef2da51e088211e4d1eb91a6c7f4f1"},
+    {file = "grpcio-1.68.1-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:4efac5481c696d5cb124ff1c119a78bddbfdd13fc499e3bc0ca81e95fc573684"},
+    {file = "grpcio-1.68.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ab2d912ca39c51f46baf2a0d92aa265aa96b2443266fc50d234fa88bf877d8e"},
+    {file = "grpcio-1.68.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95c87ce2a97434dffe7327a4071839ab8e8bffd0054cc74cbe971fba98aedd60"},
+    {file = "grpcio-1.68.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:e4842e4872ae4ae0f5497bf60a0498fa778c192cc7a9e87877abd2814aca9475"},
+    {file = "grpcio-1.68.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:255b1635b0ed81e9f91da4fcc8d43b7ea5520090b9a9ad9340d147066d1d3613"},
+    {file = "grpcio-1.68.1-cp312-cp312-win32.whl", hash = "sha256:7dfc914cc31c906297b30463dde0b9be48e36939575eaf2a0a22a8096e69afe5"},
+    {file = "grpcio-1.68.1-cp312-cp312-win_amd64.whl", hash = "sha256:a0c8ddabef9c8f41617f213e527254c41e8b96ea9d387c632af878d05db9229c"},
+    {file = "grpcio-1.68.1-cp313-cp313-linux_armv7l.whl", hash = "sha256:a47faedc9ea2e7a3b6569795c040aae5895a19dde0c728a48d3c5d7995fda385"},
+    {file = "grpcio-1.68.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:390eee4225a661c5cd133c09f5da1ee3c84498dc265fd292a6912b65c421c78c"},
+    {file = "grpcio-1.68.1-cp313-cp313-manylinux_2_17_aarch64.whl", hash = "sha256:66a24f3d45c33550703f0abb8b656515b0ab777970fa275693a2f6dc8e35f1c1"},
+    {file = "grpcio-1.68.1-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c08079b4934b0bf0a8847f42c197b1d12cba6495a3d43febd7e99ecd1cdc8d54"},
+    {file = "grpcio-1.68.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8720c25cd9ac25dd04ee02b69256d0ce35bf8a0f29e20577427355272230965a"},
+    {file = "grpcio-1.68.1-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:04cfd68bf4f38f5bb959ee2361a7546916bd9a50f78617a346b3aeb2b42e2161"},
+    {file = "grpcio-1.68.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:c28848761a6520c5c6071d2904a18d339a796ebe6b800adc8b3f474c5ce3c3ad"},
+    {file = "grpcio-1.68.1-cp313-cp313-win32.whl", hash = "sha256:77d65165fc35cff6e954e7fd4229e05ec76102d4406d4576528d3a3635fc6172"},
+    {file = "grpcio-1.68.1-cp313-cp313-win_amd64.whl", hash = "sha256:a8040f85dcb9830d8bbb033ae66d272614cec6faceee88d37a88a9bd1a7a704e"},
+    {file = "grpcio-1.68.1-cp38-cp38-linux_armv7l.whl", hash = "sha256:eeb38ff04ab6e5756a2aef6ad8d94e89bb4a51ef96e20f45c44ba190fa0bcaad"},
+    {file = "grpcio-1.68.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:8a3869a6661ec8f81d93f4597da50336718bde9eb13267a699ac7e0a1d6d0bea"},
+    {file = "grpcio-1.68.1-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:2c4cec6177bf325eb6faa6bd834d2ff6aa8bb3b29012cceb4937b86f8b74323c"},
+    {file = "grpcio-1.68.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12941d533f3cd45d46f202e3667be8ebf6bcb3573629c7ec12c3e211d99cfccf"},
+    {file = "grpcio-1.68.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80af6f1e69c5e68a2be529990684abdd31ed6622e988bf18850075c81bb1ad6e"},
+    {file = "grpcio-1.68.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e8dbe3e00771bfe3d04feed8210fc6617006d06d9a2679b74605b9fed3e8362c"},
+    {file = "grpcio-1.68.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:83bbf5807dc3ee94ce1de2dfe8a356e1d74101e4b9d7aa8c720cc4818a34aded"},
+    {file = "grpcio-1.68.1-cp38-cp38-win32.whl", hash = "sha256:8cb620037a2fd9eeee97b4531880e439ebfcd6d7d78f2e7dcc3726428ab5ef63"},
+    {file = "grpcio-1.68.1-cp38-cp38-win_amd64.whl", hash = "sha256:52fbf85aa71263380d330f4fce9f013c0798242e31ede05fcee7fbe40ccfc20d"},
+    {file = "grpcio-1.68.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:cb400138e73969eb5e0535d1d06cae6a6f7a15f2cc74add320e2130b8179211a"},
+    {file = "grpcio-1.68.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a1b988b40f2fd9de5c820f3a701a43339d8dcf2cb2f1ca137e2c02671cc83ac1"},
+    {file = "grpcio-1.68.1-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:96f473cdacfdd506008a5d7579c9f6a7ff245a9ade92c3c0265eb76cc591914f"},
+    {file = "grpcio-1.68.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:37ea3be171f3cf3e7b7e412a98b77685eba9d4fd67421f4a34686a63a65d99f9"},
+    {file = "grpcio-1.68.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ceb56c4285754e33bb3c2fa777d055e96e6932351a3082ce3559be47f8024f0"},
+    {file = "grpcio-1.68.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:dffd29a2961f3263a16d73945b57cd44a8fd0b235740cb14056f0612329b345e"},
+    {file = "grpcio-1.68.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:025f790c056815b3bf53da850dd70ebb849fd755a4b1ac822cb65cd631e37d43"},
+    {file = "grpcio-1.68.1-cp39-cp39-win32.whl", hash = "sha256:1098f03dedc3b9810810568060dea4ac0822b4062f537b0f53aa015269be0a76"},
+    {file = "grpcio-1.68.1-cp39-cp39-win_amd64.whl", hash = "sha256:334ab917792904245a028f10e803fcd5b6f36a7b2173a820c0b5b076555825e1"},
+    {file = "grpcio-1.68.1.tar.gz", hash = "sha256:44a8502dd5de653ae6a73e2de50a401d84184f0331d0ac3daeb044e66d5c5054"},
+]
+[package.extras]
+protobuf = ["grpcio-tools (>=1.68.1)"]
 [[package]]
 name = "gto"
 [[package]]
 name = "jsonargparse"
+version = "4.34.1"
 description = "Implement minimal boilerplate CLIs derived from type hints and parse from command line, config files and environment variables."
 optional = false
 python-versions = ">=3.8"
 files = [
+    {file = "jsonargparse-4.34.1-py3-none-any.whl", hash = "sha256:1d595080e080d4581ef821b8ec71364037052be4da0f9c7bdafe95962672cee6"},
+    {file = "jsonargparse-4.34.1.tar.gz", hash = "sha256:6e0d1ab67b12b1086fe7bbe5ba429fbe9865c36493c9bb6aeb1e243047fdb58c"},
 ]
 [package.dependencies]
 [[package]]
 name = "pydrive2"
+version = "1.21.1"
 description = "Google Drive API made easy. Maintained fork of PyDrive."
 optional = false
 python-versions = ">=3.8"
 files = [
+    {file = "PyDrive2-1.21.1-py3-none-any.whl", hash = "sha256:d24b3334bc5c242e5ec58ad6ee7efbd2216aa92098c3eed353ce644f27a7e97b"},
+    {file = "pydrive2-1.21.1.tar.gz", hash = "sha256:70da0244a29a6922e28620a32e251ac6ab018449f1bb0485e9a39114a069dde0"},
 ]
 [package.dependencies]
 funcy = {version = ">=1.14", optional = true, markers = "extra == \"fsspec\""}
 google-api-python-client = ">=1.12.5"
 oauth2client = ">=4.0.0"
+pyOpenSSL = ">=19.1.0"
 PyYAML = ">=3.0"
 tqdm = {version = ">=4.0.0", optional = true, markers = "extra == \"fsspec\""}
 [[package]]
 name = "pyopenssl"
+version = "24.3.0"
 description = "Python wrapper module around the OpenSSL library"
 optional = false
+python-versions = ">=3.7"
 files = [
+    {file = "pyOpenSSL-24.3.0-py3-none-any.whl", hash = "sha256:e474f5a473cd7f92221cc04976e48f4d11502804657a08a989fb3be5514c904a"},
+    {file = "pyopenssl-24.3.0.tar.gz", hash = "sha256:49f7a019577d834746bc55c5fce6ecbcec0f2b4ec5ce1cf43a9a173b8138bb36"},
 ]
 [package.dependencies]
+cryptography = ">=41.0.5,<45"
 [package.extras]
+docs = ["sphinx (!=5.2.0,!=5.2.0.post0,!=7.2.5)", "sphinx_rtd_theme"]
+test = ["pretend", "pytest (>=3.0.1)", "pytest-rerunfailures"]
 [[package]]
 name = "pyparsing"
 [metadata]
 lock-version = "2.0"
 python-versions = "3.10.15"
+content-hash = "78d6a9383381dfba8ea585d06d83d833b85434561a2c3f39d8eda83e5e1697d9"

pyproject.toml CHANGED Viewed

@@ -77,9 +77,9 @@ nvitop = "^1.3.2"
 gradio = "5.7.1"
 gradio-client = "^1.5.0"
 accelerate = "^1.1.1"
-pyopenssl = "<23.0.0"
 cryptography = "^44.0.0"
 boto3 = "*"
 [tool.poetry.dev-dependencies]
 pytest-asyncio = "^0.20.3"

 gradio = "5.7.1"
 gradio-client = "^1.5.0"
 accelerate = "^1.1.1"
 cryptography = "^44.0.0"
 boto3 = "*"
+pyopenssl = "^24.3.0"
 [tool.poetry.dev-dependencies]
 pytest-asyncio = "^0.20.3"

src/infer.py CHANGED Viewed

@@ -5,7 +5,7 @@ import torch.nn.functional as F
 import matplotlib.pyplot as plt
 from PIL import Image
 from torchvision import transforms
-from src.models.catdog_model import ViTTinyClassifier
 from src.utils.logging_utils import setup_logger, task_wrapper, get_rich_progress
 import hydra
 from omegaconf import DictConfig, OmegaConf
@@ -13,6 +13,7 @@ from dotenv import load_dotenv, find_dotenv
 import rootutils
 import time
 from loguru import logger
 # Load environment variables
 load_dotenv(find_dotenv(".env"))
@@ -93,9 +94,15 @@ def main_infer(cfg: DictConfig):
     if flag_file.exists():
         flag_file.unlink()
     # Load the trained model
-    model = ViTTinyClassifier.load_from_checkpoint(checkpoint_path=cfg.ckpt_path)
-    classes = ["dog", "cat"]
     # Download an image for inference
     download_image(cfg)

 import matplotlib.pyplot as plt
 from PIL import Image
 from torchvision import transforms
+from src.models.catdog_model_resnet import ResnetClassifier
 from src.utils.logging_utils import setup_logger, task_wrapper, get_rich_progress
 import hydra
 from omegaconf import DictConfig, OmegaConf
 import rootutils
 import time
 from loguru import logger
+from src.utils.aws_s3_services import S3Handler
 # Load environment variables
 load_dotenv(find_dotenv(".env"))
     if flag_file.exists():
         flag_file.unlink()
+    # download the model from S3
+    s3_handler = S3Handler(bucket_name="deep-bucket-s3")
+    s3_handler.download_folder(
+        "checkpoints",
+        "checkpoints",
+    )
     # Load the trained model
+    model = ResnetClassifier.load_from_checkpoint(checkpoint_path=cfg.ckpt_path)
+    classes = cfg.labels
     # Download an image for inference
     download_image(cfg)

src/models/catdog_model_resnet.py CHANGED Viewed

@@ -9,6 +9,8 @@ import torch
 class ResnetClassifier(L.LightningModule):
     def __init__(
         self,
         num_classes: int = 2,  # Binary classification with two classes
         lr: float = 1e-3,
         weight_decay: float = 1e-5,
@@ -21,7 +23,7 @@ class ResnetClassifier(L.LightningModule):
         # Vision Transformer model initialization
         self.model = timm.create_model(
-            "efficientnet_b0", pretrained=True, num_classes=num_classes
         )
         # Define accuracy and F1 metrics for binary classification
@@ -90,5 +92,5 @@ class ResnetClassifier(L.LightningModule):
 if __name__ == "__main__":
-    model = ViTTinyClassifier()
     print(model)

 class ResnetClassifier(L.LightningModule):
     def __init__(
         self,
+        base_model: str = "efficientnet_b0",
+        pretrained: bool = True,
         num_classes: int = 2,  # Binary classification with two classes
         lr: float = 1e-3,
         weight_decay: float = 1e-5,
         # Vision Transformer model initialization
         self.model = timm.create_model(
+            base_model, pretrained=pretrained, num_classes=num_classes
         )
         # Define accuracy and F1 metrics for binary classification
 if __name__ == "__main__":
+    model = ResnetClassifier()
     print(model)

src/train_optuna_callbacks.py CHANGED Viewed

@@ -19,6 +19,7 @@ from lightning.pytorch.callbacks import Callback
 import optuna
 from lightning.pytorch import Trainer
 import json
 # Load environment variables
 load_dotenv(find_dotenv(".env"))
@@ -131,10 +132,10 @@ def objective(trial: optuna.trial.Trial, cfg: DictConfig, callbacks: List[Callba
     """Objective function for Optuna hyperparameter tuning."""
     # Sample hyperparameters for the model
-    cfg.model.embed_dim = trial.suggest_categorical("embed_dim", [64, 128, 256])
-    cfg.model.depth = trial.suggest_int("depth", 2, 6)
     cfg.model.lr = trial.suggest_loguniform("lr", 1e-5, 1e-3)
-    cfg.model.mlp_ratio = trial.suggest_float("mlp_ratio", 1.0, 4.0)
     # Initialize data module and model
     data_module: L.LightningDataModule = hydra.utils.instantiate(cfg.data)
@@ -207,6 +208,13 @@ def setup_trainer(cfg: DictConfig):
         with open(Path(cfg.paths.ckpt_dir) / "train_done.flag", "w") as f:
             f.write("Training completed successfully!")
     # Testing phase with best hyperparameters
     if cfg.get("test", False):
         best_hyperparams_path = Path(cfg.paths.ckpt_dir) / "best_hyperparams.json"

 import optuna
 from lightning.pytorch import Trainer
 import json
+from src.utils.aws_s3_services import S3Handler
 # Load environment variables
 load_dotenv(find_dotenv(".env"))
     """Objective function for Optuna hyperparameter tuning."""
     # Sample hyperparameters for the model
+    # cfg.model.embed_dim = trial.suggest_categorical("embed_dim", [64, 128, 256])
+    # cfg.model.depth = trial.suggest_int("depth", 2, 6)
     cfg.model.lr = trial.suggest_loguniform("lr", 1e-5, 1e-3)
+    # cfg.model.mlp_ratio = trial.suggest_float("mlp_ratio", 1.0, 4.0)
     # Initialize data module and model
     data_module: L.LightningDataModule = hydra.utils.instantiate(cfg.data)
         with open(Path(cfg.paths.ckpt_dir) / "train_done.flag", "w") as f:
             f.write("Training completed successfully!")
+        # upload the checkpoints to S3
+        s3_handler = S3Handler(bucket_name="deep-bucket-s3")
+        s3_handler.upload_folder(
+            "checkpoints",
+            "checkpoints",
+        )
     # Testing phase with best hyperparameters
     if cfg.get("test", False):
         best_hyperparams_path = Path(cfg.paths.ckpt_dir) / "best_hyperparams.json"

src/utils/aws_s3_services.py ADDED Viewed

	@@ -0,0 +1,78 @@

+import boto3
+import os
+from pathlib import Path
+from dotenv import load_dotenv, find_dotenv
+# Load environment variables from .env file
+load_dotenv(find_dotenv(".env"))
+class S3Handler:
+    def __init__(self, bucket_name):
+        self.bucket_name = bucket_name
+        self.s3 = boto3.client(
+            "s3",
+            aws_access_key_id=os.getenv("AWS_ACCESS_KEY_ID"),
+            aws_secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"),
+            region_name=os.getenv("AWS_REGION"),
+        )
+    def upload_folder(self, source_folder, dest_folder, filenames=None):
+        """
+        Upload specified files or all files from a local folder to an S3 folder.
+        Args:
+            source_folder (str): Local source folder path.
+            dest_folder (str): Destination folder path in S3.
+            filenames (list): List of filenames to upload (relative to source_folder). If None, uploads all files.
+        """
+        source_folder = Path(source_folder)
+        # Select files based on filenames list or all files if filenames is None
+        files_to_upload = (
+            [source_folder / file for file in filenames]
+            if filenames
+            else list(source_folder.rglob("*"))
+        )
+        for file_path in files_to_upload:
+            if file_path.is_file():
+                s3_path = f"{dest_folder}/{file_path.relative_to(source_folder)}"
+                self.s3.upload_file(str(file_path), self.bucket_name, s3_path)
+                print(f"Uploaded: {file_path} to {s3_path}")
+            else:
+                print(f"File not found: {file_path}")
+    def download_folder(self, s3_folder, dest_folder):
+        """
+        Download all files from an S3 folder to a local folder.
+        Args:
+            s3_folder (str): Source folder in S3.
+            dest_folder (str): Local destination folder path.
+        """
+        dest_folder = Path(dest_folder)
+        paginator = self.s3.get_paginator("list_objects_v2")
+        for page in paginator.paginate(Bucket=self.bucket_name, Prefix=s3_folder):
+            for obj in page.get("Contents", []):
+                s3_path = obj["Key"]
+                local_path = dest_folder / Path(s3_path).relative_to(s3_folder)
+                local_path.parent.mkdir(parents=True, exist_ok=True)
+                self.s3.download_file(self.bucket_name, s3_path, str(local_path))
+                print(f"Downloaded: {s3_path} to {local_path}")
+# Usage Example
+if __name__ == "__main__":
+    # Initialize with bucket name
+    s3_handler = S3Handler(bucket_name="deep-bucket-s3")
+    # Upload specific files
+    s3_handler.upload_folder(
+        "checkpoints",
+        "checkpoints_test",
+    )
+    # Download example
+    s3_handler.download_folder("checkpoints_test", "checkpoints")