initial

.env

@@ -0,0 +1,11 @@
+#MODEL_PATH=/models/llama3-70b-quantized
+#SERVED_NAME=llama3-70b-gptq
+MODEL_PATH=/models/deepseek-coder-33b-gptq
+SERVED_NAME=deepseek-coder
+TP_SIZE=4
+DTYPE=float16
+MAX_MODEL_LEN=8192
+GPU_MEM_UTIL=0.88
+MAX_NUM_SEQS=4
+VLLM_EXTRA_ARGS=--quantization gptq --disable-custom-all-reduce --enforce-eager
+

compose.yaml

@@ -0,0 +1,35 @@
+services:
+  vllm-legacy:
+    image: vllm/vllm-openai:v0.4.0.post1
+    container_name: gpu-lab-vllm-legacy
+    network_mode: host
+    ipc: host
+    shm_size: "32gb"
+    env_file:
+      - .env
+    volumes:
+      - models:/models:ro
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: all
+              capabilities: [gpu]
+    entrypoint: ["/bin/sh", "-c"]
+    command:
+      - >
+        exec python3 -m vllm.entrypoints.openai.api_server
+        --model "$$MODEL_PATH"
+        --served-model-name "$$SERVED_NAME"
+        --host 0.0.0.0 --port 8000
+        --tensor-parallel-size "$$TP_SIZE"
+        --dtype "$$DTYPE"
+        --max-model-len "$$MAX_MODEL_LEN"
+        --gpu-memory-utilization "$$GPU_MEM_UTIL"
+        --max-num-seqs "$$MAX_NUM_SEQS"
+        $$VLLM_EXTRA_ARGS
+    restart: "no"
+
+volumes:
+    models: