update

.env

@@ -1,11 +1,12 @@
+LISTEN_PORT=8000
 #MODEL_PATH=/models/llama3-70b-quantized
 #SERVED_NAME=llama3-70b-gptq
 MODEL_PATH=/models/deepseek-coder-33b-gptq
 SERVED_NAME=deepseek-coder
 TP_SIZE=4
 DTYPE=float16
-MAX_MODEL_LEN=8192
-GPU_MEM_UTIL=0.88
+MAX_MODEL_LEN=65536
+GPU_MEM_UTIL=0.60
 MAX_NUM_SEQS=4
 VLLM_EXTRA_ARGS=--quantization gptq --disable-custom-all-reduce --enforce-eager
 

compose.yaml

@@ -1,14 +1,15 @@
 services:
   vllm-legacy:
     image: vllm/vllm-openai:v0.4.0.post1
-    container_name: gpu-lab-vllm-legacy
+    container_name: vllm-${NAME:-legacy}
+    restart: unless-stopped
     network_mode: host
     ipc: host
     shm_size: "32gb"
     env_file:
       - .env
     volumes:
-      - models:/models:ro
+      - /var/models:/models:ro
     deploy:
       resources:
         reservations:
@@ -20,16 +21,15 @@ services:
     command:
       - >
         exec python3 -m vllm.entrypoints.openai.api_server
-        --model "$$MODEL_PATH"
-        --served-model-name "$$SERVED_NAME"
-        --host 0.0.0.0 --port 8000
-        --tensor-parallel-size "$$TP_SIZE"
-        --dtype "$$DTYPE"
-        --max-model-len "$$MAX_MODEL_LEN"
-        --gpu-memory-utilization "$$GPU_MEM_UTIL"
-        --max-num-seqs "$$MAX_NUM_SEQS"
-        $$VLLM_EXTRA_ARGS
-    restart: "no"
+        --model ${MODEL_PATH}
+        --served-model-name ${SERVED_NAME}
+        --host 0.0.0.0 --port ${LISTEN_PORT}
+        --tensor-parallel-size ${TP_SIZE}
+        --dtype ${DTYPE}
+        --max-model-len ${MAX_MODEL_LEN}
+        --gpu-memory-utilization ${GPU_MEM_UTIL}
+        --max-num-seqs ${MAX_NUM_SEQS}
+        ${VLLM_EXTRA_ARGS}
 
 volumes:
     models:

park/compose.sglang.yml

@@ -0,0 +1,26 @@
+name: gpu-lab-sglang
+services:
+  sglang:
+    image: gpu-lab-sglang:v100-cu121
+    container_name: gpu-lab-sglang
+    network_mode: host
+    ipc: host
+    shm_size: "32gb"
+    env_file:
+      - active/current.env
+    environment:
+      - PYTHONPATH=/opt/v100-bootstrap
+    volumes:
+      - /root/models:/models:ro
+      - /root/model-cache:/model-cache:ro
+      - /root/gpu-lab/start-sglang-qwen.sh:/start-sglang-qwen.sh:ro
+      - /root/gpu-lab/v100-bootstrap:/opt/v100-bootstrap:ro
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: all
+              capabilities: [gpu]
+    entrypoint: ["/bin/bash", "/start-sglang-qwen.sh"]
+    restart: "no"

park/compose.vllm-legacy.yml

@@ -0,0 +1,33 @@
+name: gpu-lab-vllm-legacy
+services:
+  vllm-legacy:
+    image: vllm/vllm-openai:v0.4.0.post1
+    container_name: gpu-lab-vllm-legacy
+    network_mode: host
+    ipc: host
+    shm_size: "32gb"
+    volumes:
+      - /root/models:/models:ro
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: all
+              capabilities: [gpu]
+    entrypoint: ["/bin/sh", "-c"]
+    command:
+      - >
+        exec python3 -m vllm.entrypoints.openai.api_server
+        --model /models/llama3-70b-quantized
+        --served-model-name llama3-70b-gptq
+        --host 0.0.0.0 --port 8010
+        --tensor-parallel-size 4
+        --dtype float16
+        --max-model-len 8192
+        --gpu-memory-utilization 0.88
+        --max-num-seqs 4
+        --quantization gptq
+        --disable-custom-all-reduce
+        --enforce-eager
+    restart: "no"

park/compose.vllm-modern.yml

@@ -0,0 +1,32 @@
+name: gpu-lab-vllm-modern
+services:
+  vllm-modern:
+    image: vllm/vllm-openai:v0.8.5.post1
+    container_name: gpu-lab-vllm-modern
+    network_mode: host
+    ipc: host
+    shm_size: "32gb"
+    volumes:
+      - /root/models:/models:ro
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: all
+              capabilities: [gpu]
+    entrypoint: ["/bin/sh", "-c"]
+    command:
+      - >
+        exec python3 -m vllm.entrypoints.openai.api_server
+        --model /models/deepseek-coder-v2-lite
+        --served-model-name deepseek-coder-v2-lite
+        --host 0.0.0.0 --port 8010
+        --tensor-parallel-size 2
+        --dtype bfloat16
+        --max-model-len 8192
+        --gpu-memory-utilization 0.9
+        --max-num-seqs 4
+        --trust-remote-code
+        --enforce-eager
+    restart: "no"