repositories/exllama/docker-compose.yml

version: "3.9"
name: exllama
services:
  web:
    build:
      context: .
      args:
        - RUN_UID=$RUN_UID
        - APPLICATION_STATE_PATH=$APPLICATION_STATE_PATH
    command: |
      --host 0.0.0.0:$PORT
    env_file:
      - .env
    volumes:
      - $MODEL_PATH:$APPLICATION_STATE_PATH/model
      - $SESSIONS_PATH:$APPLICATION_STATE_PATH/exllama_sessions
    ports:
      - "$PORT:$PORT"
    tmpfs:
      - /tmp
    stdin_open: true
    tty: true
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]