ATC/vllm-legacy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
root
2026-07-08 16:40:51 +02:00
parent 8a8b1db65e
commit 8ddf750b4b
2 changed files with 4 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env
+2 -2
View File
@@ -4,8 +4,8 @@ MODEL_PATH=/models/deepseek-coder-33b-gptq
SERVED_NAME=deepseek-coder SERVED_NAME=deepseek-coder
TP_SIZE=4 TP_SIZE=4
DTYPE=float16 DTYPE=float16
MAX_MODEL_LEN=8192 MAX_MODEL_LEN=65536
GPU_MEM_UTIL=0.88 GPU_MEM_UTIL=0.60 # 0.88
MAX_NUM_SEQS=4 MAX_NUM_SEQS=4
VLLM_EXTRA_ARGS=--quantization gptq --disable-custom-all-reduce --enforce-eager VLLM_EXTRA_ARGS=--quantization gptq --disable-custom-all-reduce --enforce-eager
compose.yaml
+2 -2
View File
@@ -8,7 +8,7 @@ services:
env_file: env_file:
- .env - .env
volumes: volumes:
- models:/models:ro - /var/models:/models:ro
deploy: deploy:
resources: resources:
reservations: reservations:
@@ -22,7 +22,7 @@ services:
exec python3 -m vllm.entrypoints.openai.api_server exec python3 -m vllm.entrypoints.openai.api_server
--model "$$MODEL_PATH" --model "$$MODEL_PATH"
--served-model-name "$$SERVED_NAME" --served-model-name "$$SERVED_NAME"
--host 0.0.0.0 --port 8000 --host 0.0.0.0 --port "$LISTEN_PORT"
--tensor-parallel-size "$$TP_SIZE" --tensor-parallel-size "$$TP_SIZE"
--dtype "$$DTYPE" --dtype "$$DTYPE"
--max-model-len "$$MAX_MODEL_LEN" --max-model-len "$$MAX_MODEL_LEN"