name: gpu-lab-vllm-legacy
services:
  vllm-legacy:
    image: vllm/vllm-openai:v0.4.0.post1
    container_name: gpu-lab-vllm-legacy
    network_mode: host
    ipc: host
    shm_size: "32gb"
    volumes:
      - /root/models:/models:ro
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    entrypoint: ["/bin/sh", "-c"]
    command:
      - >
        exec python3 -m vllm.entrypoints.openai.api_server
        --model /models/llama3-70b-quantized
        --served-model-name llama3-70b-gptq
        --host 0.0.0.0 --port 8010
        --tensor-parallel-size 4
        --dtype float16
        --max-model-len 8192
        --gpu-memory-utilization 0.88
        --max-num-seqs 4
        --quantization gptq
        --disable-custom-all-reduce
        --enforce-eager
    restart: "no"