2026-07-08 15:28:49 +02:00
|
|
|
#MODEL_PATH=/models/llama3-70b-quantized
|
|
|
|
|
#SERVED_NAME=llama3-70b-gptq
|
|
|
|
|
MODEL_PATH=/models/deepseek-coder-33b-gptq
|
|
|
|
|
SERVED_NAME=deepseek-coder
|
|
|
|
|
TP_SIZE=4
|
|
|
|
|
DTYPE=float16
|
2026-07-08 16:40:51 +02:00
|
|
|
MAX_MODEL_LEN=65536
|
|
|
|
|
GPU_MEM_UTIL=0.60 # 0.88
|
2026-07-08 15:28:49 +02:00
|
|
|
MAX_NUM_SEQS=4
|
|
|
|
|
VLLM_EXTRA_ARGS=--quantization gptq --disable-custom-all-reduce --enforce-eager
|
|
|
|
|
|
