root
12 lines
297 B
Bash
12 lines
297 B
Bash
#MODEL_PATH=/models/llama3-70b-quantized
|
|
#SERVED_NAME=llama3-70b-gptq
|
|
MODEL_PATH=/models/deepseek-coder-33b-gptq
|
|
SERVED_NAME=deepseek-coder
|
|
TP_SIZE=4
|
|
DTYPE=float16
|
|
MAX_MODEL_LEN=8192
|
|
GPU_MEM_UTIL=0.88
|
|
MAX_NUM_SEQS=4
|
|
VLLM_EXTRA_ARGS=--quantization gptq --disable-custom-all-reduce --enforce-eager
|
|
|