added: docker compose

Dockerfile

@@ -1,35 +1,48 @@
-# Use an official Python base image
-FROM python:3.10.15-slim
-
-# Set environment variables for Poetry
-ENV POETRY_VERSION=1.6.1 \
-    POETRY_HOME="/opt/poetry" \
-    POETRY_VIRTUALENVS_IN_PROJECT=true \
-    PYTHONUNBUFFERED=1
-
-# Install Poetry
-RUN apt-get update && apt-get install -y curl \
-    && curl -sSL https://install.python-poetry.org | python3 - \
-    && ln -s ${POETRY_HOME}/bin/poetry /usr/local/bin/poetry \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
-
-# Set working directory
+# Stage 1: Build environment with Poetry and dependencies
+FROM python:3.10.15-slim as builder
+
+LABEL maintainer="Soutrik soutrik1991@gmail.com" \
+      description="Docker image for running a Python app with dependencies managed by Poetry."
+
+# Install Poetry and necessary system dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends curl && \
+    curl -sSL https://install.python-poetry.org | python3 - && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+# Add Poetry to the PATH explicitly
+ENV PATH="/root/.local/bin:$PATH"
+
+# Set the working directory to /app
 WORKDIR /app
 
-# Copy Poetry files for dependency installation
-COPY pyproject.toml poetry.lock ./
+# Copy pyproject.toml and poetry.lock to install dependencies
+COPY pyproject.toml poetry.lock /app/
+
+# Configure Poetry environment
+ENV POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    POETRY_CACHE_DIR=/tmp/poetry_cache
 
-# Install dependencies with Poetry (without development dependencies)
-RUN poetry install --no-root --only main
+# Install dependencies without installing the package itself
+RUN --mount=type=cache,target=/tmp/poetry_cache poetry install --only main --no-root
 
-# Copy the application source code
-COPY src/ src/
-COPY app/ app/
-COPY main.py  .
+# Stage 2: Runtime environment
+FROM python:3.10.15-slim as runner
+
+# Copy application source code and necessary files
+COPY src /app/src
+COPY configs /app/configs
+COPY .project-root /app/.project-root
+COPY main.py /app/main.py
+
+# Copy virtual environment from the builder stage
+COPY --from=builder /app/.venv /app/.venv
+
+# Set the working directory to /app
+WORKDIR /app
 
-# Expose the port for FastAPI
-EXPOSE 8000
+# Set the environment path to use the virtual environment
+ENV PATH="/app/.venv/bin:$PATH"
 
-# Command to run the FastAPI application with uvicorn
-CMD ["poetry", "run", "uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]
+# Default command
+CMD ["python", "-m", "main"]

docker-compose.yaml

@@ -1,81 +1,69 @@
 version: '3.8'
 
 services:
-  postgres:
-    image: postgres:16-alpine
-    environment:
-      POSTGRES_DB: ${POSTGRES_DB}
-      POSTGRES_USER: ${POSTGRES_USER}
-      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
-      DOCKER_ENV: 1
-    ports:
-      - "5432:5432"
+  train:
+    build:
+      context: .
+    command: |
+      python -m src.train experiment=catdog_experiment ++task_name=train ++train=True ++test=False && \
+      touch /app/checkpoints/train_done.flag
     volumes:
-      - postgres_data:/var/lib/postgresql/data
-    networks:
-      - my_network
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"]
-      interval: 30s
-      timeout: 10s
-      retries: 5
-
-  redis:
-    image: redis/redis-stack:latest
-    ports:
-      - "6379:6379"
-      - "8001:8001"
+      - ./data:/app/data
+      - ./checkpoints:/app/checkpoints
+      - ./artifacts:/app/artifacts
+      - ./logs:/app/logs
     environment:
-      DOCKER_ENV: 1
-      REDIS_PORT: ${REDIS_PORT}
-      REDIS_HOST: ${REDIS_HOST}
+      - PYTHONUNBUFFERED=1
+      - PYTHONPATH=/app
+    shm_size: '4g' 
     networks:
-      - my_network
-    healthcheck:
-      test: ["CMD", "redis-cli", "ping"]
-      interval: 30s
-      timeout: 5s
-      retries: 5
+      - default
+    env_file:
+      - .env
 
-  flower:
-    image: mher/flower:0.9.7
-    environment:
-      DOCKER_ENV: 1
-    command: ["flower", "--broker=${BROKER_URL}", "--port=5555"]
-    ports:
-      - 5557:5555
-    depends_on:
-      - redis
-    networks:
-      - my_network
-
-  app:
+  eval:
     build:
       context: .
-      dockerfile: Dockerfile  # Assumes the Dockerfile is in the root directory
+    command: |
+      sh -c 'while [ ! -f ./checkpoints/train_done.flag ]; do sleep 10; done && python -m src.train experiment=catdog_experiment ++task_name=eval ++train=False ++test=True'
+    volumes:
+      - ./data:/app/data
+      - ./checkpoints:/app/checkpoints
+      - ./artifacts:/app/artifacts
+      - ./logs:/app/logs
     environment:
-      - DATABASE_URL=${DATABASE_URL}
-      - REDIS_URL=${REDIS_URL}
-      - BROKER_URL=${BROKER_URL}
-      - FLOWER_BASIC_AUTH=${FLOWER_BASIC_AUTH}
-      - POSTGRES_DB=${POSTGRES_DB} 
-      - POSTGRES_USER=${POSTGRES_USER}
-      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
-      - DOCKER_ENV=1
-    depends_on:
-      postgres:
-        condition: service_healthy
-      redis:
-        condition: service_healthy
+      - PYTHONUNBUFFERED=1
+      - PYTHONPATH=/app
+    shm_size: '4g'
     networks:
-      - my_network
-    ports:
-      - "8000:8000"
-    command: ["poetry", "run", "uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]
+      - default
+    env_file:
+      - .env  
+
+  # inference:
+  #   build:
+  #     context: .
+  #   command: |
+  #     sh -c 'while [ ! -f ./checkpoints/train_done.flag ]; do sleep 10; done && python -m src.infer experiment=catdog_experiment'
+  #   volumes:
+  #     - ./data:/app/data
+  #     - ./checkpoints:/app/checkpoints
+  #     - ./artifacts:/app/artifacts
+  #     - ./logs:/app/logs
+  #   environment:
+  #     - PYTHONUNBUFFERED=1
+  #     - PYTHONPATH=/app
+  #   shm_size: '4g'
+  #   networks:
+  #     - default
+  #   env_file:
+  #     - .env  
 
 volumes:
-  postgres_data:
+  data:
+  checkpoints:
+  artifacts:
+  logs:
 
 networks:
-  my_network:
-    driver: bridge
+  default:

main.py

@@ -1,28 +1,3 @@
-from fastapi import FastAPI
-from app.api import chat
-from app.db.database import Base, engine
-from app.core.config import settings
-from aiocache import caches
+from loguru import logger
 
-app = FastAPI()
-
-
-# Create tables on startup
-@app.on_event("startup")
-async def startup():
-    async with engine.begin() as conn:
-        await conn.run_sync(Base.metadata.create_all)
-
-    # Configure aiocache with in-memory backend
-    caches.set_config(
-        {
-            "default": {
-                "cache": settings.cache_backend,
-                "ttl": 300,  # Default Time-To-Live for cache entries (in seconds)
-            }
-        }
-    )
-
-
-# Include the chat route
-app.include_router(chat.router)
+logger.info("Hello, World!")

src/train.py

@@ -23,12 +23,8 @@ import rootutils
 load_dotenv(find_dotenv(".env"))
 
 # Setup root directory
-try:
-    root = rootutils.setup_root(__file__, indicator=".project-root")
-except Exception as e:
-    root = os.getcwd()
 
-os.environ.setdefault("PROJECT_ROOT", str(root))
+root = rootutils.setup_root(__file__, indicator=".project-root")
 
 
 def initialize_callbacks(cfg: DictConfig) -> List[L.Callback]:

src/utils/logging_utils.py

@@ -15,7 +15,7 @@ def setup_logger(log_file):
         sys.stderr,
         format="<green>{time:YYYY-MM-DD HH:mm:ss}</green> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>",
     )
-    logger.add(log_file, rotation="1MB")
+    logger.add(log_file, rotation="1MB", retention="7 days", encoding="utf-8")
 
 
 def task_wrapper(func):