diff --git a/park/compose.vllm-legacy.yml b/park/compose.vllm-legacy.yml new file mode 100644 index 0000000..cd0c119 --- /dev/null +++ b/park/compose.vllm-legacy.yml @@ -0,0 +1,33 @@ +name: gpu-lab-vllm-legacy +services: + vllm-legacy: + image: vllm/vllm-openai:v0.4.0.post1 + container_name: gpu-lab-vllm-legacy + network_mode: host + ipc: host + shm_size: "32gb" + volumes: + - /root/models:/models:ro + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + entrypoint: ["/bin/sh", "-c"] + command: + - > + exec python3 -m vllm.entrypoints.openai.api_server + --model /models/llama3-70b-quantized + --served-model-name llama3-70b-gptq + --host 0.0.0.0 --port 8010 + --tensor-parallel-size 4 + --dtype float16 + --max-model-len 8192 + --gpu-memory-utilization 0.88 + --max-num-seqs 4 + --quantization gptq + --disable-custom-all-reduce + --enforce-eager + restart: "no" diff --git a/park/compose.vllm-modern.yml b/park/compose.vllm-modern.yml new file mode 100644 index 0000000..21aa175 --- /dev/null +++ b/park/compose.vllm-modern.yml @@ -0,0 +1,32 @@ +name: gpu-lab-vllm-modern +services: + vllm-modern: + image: vllm/vllm-openai:v0.8.5.post1 + container_name: gpu-lab-vllm-modern + network_mode: host + ipc: host + shm_size: "32gb" + volumes: + - /root/models:/models:ro + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + entrypoint: ["/bin/sh", "-c"] + command: + - > + exec python3 -m vllm.entrypoints.openai.api_server + --model /models/deepseek-coder-v2-lite + --served-model-name deepseek-coder-v2-lite + --host 0.0.0.0 --port 8010 + --tensor-parallel-size 2 + --dtype bfloat16 + --max-model-len 8192 + --gpu-memory-utilization 0.9 + --max-num-seqs 4 + --trust-remote-code + --enforce-eager + restart: "no"