LISTEN_PORT=8000
#MODEL_PATH=/models/llama3-70b-quantized
#SERVED_NAME=llama3-70b-gptq
MODEL_PATH=/models/deepseek-coder-33b-gptq
SERVED_NAME=deepseek-coder
TP_SIZE=4
DTYPE=float16
MAX_MODEL_LEN=65536
GPU_MEM_UTIL=0.60
MAX_NUM_SEQS=4
VLLM_EXTRA_ARGS=--quantization gptq --disable-custom-all-reduce --enforce-eager