initial

2026-07-08 15:28:49 +02:00
commit 8a8b1db65e
2 changed files with 46 additions and 0 deletions
@@ -0,0 +1,11 @@
+#MODEL_PATH=/models/llama3-70b-quantized
+#SERVED_NAME=llama3-70b-gptq
+MODEL_PATH=/models/deepseek-coder-33b-gptq
+SERVED_NAME=deepseek-coder
+TP_SIZE=4
+DTYPE=float16
+MAX_MODEL_LEN=8192
+GPU_MEM_UTIL=0.88
+MAX_NUM_SEQS=4
+VLLM_EXTRA_ARGS=--quantization gptq --disable-custom-all-reduce --enforce-eager
+