diff --git a/.env b/.env index 1d1d8b9..b876891 100644 --- a/.env +++ b/.env @@ -4,8 +4,8 @@ MODEL_PATH=/models/deepseek-coder-33b-gptq SERVED_NAME=deepseek-coder TP_SIZE=4 DTYPE=float16 -MAX_MODEL_LEN=8192 -GPU_MEM_UTIL=0.88 +MAX_MODEL_LEN=65536 +GPU_MEM_UTIL=0.60 # 0.88 MAX_NUM_SEQS=4 VLLM_EXTRA_ARGS=--quantization gptq --disable-custom-all-reduce --enforce-eager diff --git a/compose.yaml b/compose.yaml index 8fd33d7..48e87ae 100644 --- a/compose.yaml +++ b/compose.yaml @@ -8,7 +8,7 @@ services: env_file: - .env volumes: - - models:/models:ro + - /var/models:/models:ro deploy: resources: reservations: @@ -22,7 +22,7 @@ services: exec python3 -m vllm.entrypoints.openai.api_server --model "$$MODEL_PATH" --served-model-name "$$SERVED_NAME" - --host 0.0.0.0 --port 8000 + --host 0.0.0.0 --port "$LISTEN_PORT" --tensor-parallel-size "$$TP_SIZE" --dtype "$$DTYPE" --max-model-len "$$MAX_MODEL_LEN"