35 lines
949 B
YAML
35 lines
949 B
YAML
|
name: gpu-lab-vllm-modern
|
||
|
|
services:
|
||
|
|
vllm-modern:
|
||
|
|
image: vllm/vllm-openai:v0.8.5.post1
|
||
|
|
container_name: gpu-lab-vllm-modern
|
||
|
|
network_mode: host
|
||
|
|
ipc: host
|
||
|
|
shm_size: "32gb"
|
||
|
|
env_file:
|
||
|
|
- active/current.env
|
||
|
|
volumes:
|
||
|
|
- /root/models:/models:ro
|
||
|
|
deploy:
|
||
|
|
resources:
|
||
|
|
reservations:
|
||
|
|
devices:
|
||
|
|
- driver: nvidia
|
||
|
|
count: all
|
||
|
|
capabilities: [gpu]
|
||
|
|
entrypoint: ["/bin/sh", "-c"]
|
||
|
|
command:
|
||
|
|
- >
|
||
|
|
exec python3 -m vllm.entrypoints.openai.api_server
|
||
|
|
--model "$$MODEL_PATH"
|
||
|
|
--served-model-name "$$SERVED_NAME"
|
||
|
|
--host 0.0.0.0 --port 8010
|
||
|
|
--tensor-parallel-size "$$TP_SIZE"
|
||
|
|
--dtype "$$DTYPE"
|
||
|
|
--max-model-len "$$MAX_MODEL_LEN"
|
||
|
|
--gpu-memory-utilization "$$GPU_MEM_UTIL"
|
||
|
|
--max-num-seqs "$$MAX_NUM_SEQS"
|
||
|
|
--trust-remote-code --enforce-eager
|
||
|
|
$$VLLM_EXTRA_ARGS
|
||
|
|
restart: "no"
|