update

2026-07-08 16:40:51 +02:00
parent 8a8b1db65e
commit 8ddf750b4b
2 changed files with 4 additions and 4 deletions
@@ -4,8 +4,8 @@ MODEL_PATH=/models/deepseek-coder-33b-gptq
 SERVED_NAME=deepseek-coder
 TP_SIZE=4
 DTYPE=float16
-MAX_MODEL_LEN=8192
-GPU_MEM_UTIL=0.88
+MAX_MODEL_LEN=65536
+GPU_MEM_UTIL=0.60 # 0.88
 MAX_NUM_SEQS=4
 VLLM_EXTRA_ARGS=--quantization gptq --disable-custom-all-reduce --enforce-eager

@@ -8,7 +8,7 @@ services:
    env_file:
      - .env
    volumes:
-      - models:/models:ro
+      - /var/models:/models:ro
    deploy:
      resources:
        reservations:
@@ -22,7 +22,7 @@ services:
        exec python3 -m vllm.entrypoints.openai.api_server
        --model "$$MODEL_PATH"
        --served-model-name "$$SERVED_NAME"
-        --host 0.0.0.0 --port 8000
+        --host 0.0.0.0 --port "$LISTEN_PORT"
        --tensor-parallel-size "$$TP_SIZE"
        --dtype "$$DTYPE"
        --max-model-len "$$MAX_MODEL_LEN"