docker-compose.cuda.yml

# Optional overlay: containerised llama.cpp server with NVIDIA GPU.
# Requires NVIDIA Container Toolkit on the host:
#   https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html
#
# Usage:
#   docker compose -f docker-compose.yml -f docker-compose.cuda.yml up -d
#
# The model file path comes from LEX_LLM__MODEL_FILE in your .env.
# Make sure you've run `lex serve-llm` (or downloaded manually) first
# so the file exists in models/.
services:
  llm:
    image: ghcr.io/ggerganov/llama.cpp:server-cuda
    ports:
      - "${LEX_LLM__PORT:-8080}:8080"
    volumes:
      - ./models:/models:ro
    environment:
      - LEX_LLM__MODEL_FILE=${LEX_LLM__MODEL_FILE:-gemma-4-E4B-it-Q4_K_M.gguf}
    command: >
      --host 0.0.0.0
      --port 8080
      --model /models/${LEX_LLM__MODEL_FILE:-gemma-4-E4B-it-Q4_K_M.gguf}
      --n-gpu-layers ${LEX_LLM__N_GPU_LAYERS:--1}
      --ctx-size ${LEX_LLM__CTX_SIZE:-32768}
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: 1
              capabilities: [gpu]
    restart: unless-stopped