vllm-legacy/compose.yaml

services:
  vllm-legacy:
    image: vllm/vllm-openai:v0.4.0.post1
    container_name: vllm-legacy
    network_mode: host
    ipc: host
    shm_size: "32gb"
    env_file:
      - .env
    volumes:
      - /var/models:/models:ro
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    entrypoint: ["/bin/sh", "-c"]
    command:
      - >
        exec python3 -m vllm.entrypoints.openai.api_server
        --model "$$MODEL_PATH"
        --served-model-name "$$SERVED_NAME"
        --host 0.0.0.0 --port "$LISTEN_PORT"
        --tensor-parallel-size "$$TP_SIZE"
        --dtype "$$DTYPE"
        --max-model-len "$$MAX_MODEL_LEN"
        --gpu-memory-utilization "$$GPU_MEM_UTIL"
        --max-num-seqs "$$MAX_NUM_SEQS"
        $$VLLM_EXTRA_ARGS
    restart: "no"

volumes:
    models: