name: gpu-lab-vllm-modern
services:
  vllm-modern:
    image: vllm/vllm-openai:v0.8.5.post1
    container_name: gpu-lab-vllm-modern
    network_mode: host
    ipc: host
    shm_size: "32gb"
    volumes:
      - /root/models:/models:ro
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    entrypoint: ["/bin/sh", "-c"]
    command:
      - >
        exec python3 -m vllm.entrypoints.openai.api_server
        --model /models/deepseek-coder-v2-lite
        --served-model-name deepseek-coder-v2-lite
        --host 0.0.0.0 --port 8010
        --tensor-parallel-size 2
        --dtype bfloat16
        --max-model-len 8192
        --gpu-memory-utilization 0.9
        --max-num-seqs 4
        --trust-remote-code
        --enforce-eager
    restart: "no"