34 lines
888 B
YAML
34 lines
888 B
YAML
|
name: gpu-lab-vllm-legacy
|
||
|
|
services:
|
||
|
|
vllm-legacy:
|
||
|
|
image: vllm/vllm-openai:v0.4.0.post1
|
||
|
|
container_name: gpu-lab-vllm-legacy
|
||
|
|
network_mode: host
|
||
|
|
ipc: host
|
||
|
|
shm_size: "32gb"
|
||
|
|
volumes:
|
||
|
|
- /root/models:/models:ro
|
||
|
|
deploy:
|
||
|
|
resources:
|
||
|
|
reservations:
|
||
|
|
devices:
|
||
|
|
- driver: nvidia
|
||
|
|
count: all
|
||
|
|
capabilities: [gpu]
|
||
|
|
entrypoint: ["/bin/sh", "-c"]
|
||
|
|
command:
|
||
|
|
- >
|
||
|
|
exec python3 -m vllm.entrypoints.openai.api_server
|
||
|
|
--model /models/llama3-70b-quantized
|
||
|
|
--served-model-name llama3-70b-gptq
|
||
|
|
--host 0.0.0.0 --port 8010
|
||
|
|
--tensor-parallel-size 4
|
||
|
|
--dtype float16
|
||
|
|
--max-model-len 8192
|
||
|
|
--gpu-memory-utilization 0.88
|
||
|
|
--max-num-seqs 4
|
||
|
|
--quantization gptq
|
||
|
|
--disable-custom-all-reduce
|
||
|
|
--enforce-eager
|
||
|
|
restart: "no"
|