mo
c154c94c6d
Mirror gpu-lab compose files and model-manager catalog from GPU-Dev; document Dockhand mapping and SGLang cutover target.
516 lines
15 KiB
JSON
516 lines
15 KiB
JSON
{
|
|
"models": [
|
|
{
|
|
"id": "llama3-70b-gptq",
|
|
"name": "Llama 3 70B GPTQ",
|
|
"description": "Meta Llama 3 open weights. Strong general coder (lab default). 4\u00d7 V100.",
|
|
"tags": [
|
|
"coding",
|
|
"opensource",
|
|
"v100_ready"
|
|
],
|
|
"license": "Llama 3 Community License",
|
|
"repo_id": null,
|
|
"local_path": "/root/models/llama3-70b-quantized",
|
|
"served_name": "llama3-70b-gptq",
|
|
"size_gb_estimate": 38,
|
|
"vllm_min_version": "0.4.0",
|
|
"vllm": {
|
|
"tensor_parallel_size": 4,
|
|
"quantization": "gptq",
|
|
"dtype": "float16",
|
|
"max_model_len": 8192,
|
|
"gpu_memory_utilization": 0.88,
|
|
"max_num_seqs": 4,
|
|
"extra_args": [
|
|
"--disable-custom-all-reduce",
|
|
"--enforce-eager"
|
|
]
|
|
},
|
|
"runtime_id": "vllm-legacy-040",
|
|
"inference_backend": "vllm",
|
|
"v100_activate": true,
|
|
"v100_block_reason": null
|
|
},
|
|
{
|
|
"id": "qwen25-coder-32b-gptq",
|
|
"name": "Qwen 2.5 Coder 32B GPTQ",
|
|
"description": "Alibaba Qwen open weights. Top open-source code LLM (GPT-4o class). 4\u00d7 V100.",
|
|
"tags": [
|
|
"coding",
|
|
"opensource",
|
|
"qwen",
|
|
"v100_ready"
|
|
],
|
|
"license": "Apache-2.0",
|
|
"repo_id": "Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4",
|
|
"local_path": "/root/models/qwen25-coder-32b-gptq",
|
|
"served_name": "qwen2.5-coder-32b-gptq",
|
|
"size_gb_estimate": 20,
|
|
"vllm_min_version": "0.4.0",
|
|
"vllm": {
|
|
"tensor_parallel_size": 4,
|
|
"quantization": "gptq",
|
|
"dtype": "float16",
|
|
"max_model_len": 8192,
|
|
"gpu_memory_utilization": 0.9,
|
|
"max_num_seqs": 4,
|
|
"extra_args": [
|
|
"--trust-remote-code",
|
|
"--enforce-eager"
|
|
]
|
|
},
|
|
"runtime_id": "vllm-legacy-040",
|
|
"inference_backend": "vllm",
|
|
"v100_activate": true,
|
|
"v100_block_reason": null
|
|
},
|
|
{
|
|
"id": "qwen25-32b-gptq",
|
|
"name": "Qwen 2.5 32B GPTQ",
|
|
"description": "Alibaba Qwen open weights (general instruct, strong coder).",
|
|
"tags": [
|
|
"coding",
|
|
"opensource",
|
|
"qwen",
|
|
"v100_ready"
|
|
],
|
|
"license": "Apache-2.0",
|
|
"repo_id": "Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4",
|
|
"local_path": "/root/models/qwen25-32b-gptq",
|
|
"served_name": "qwen2.5-32b-gptq",
|
|
"size_gb_estimate": 20,
|
|
"vllm_min_version": "0.4.0",
|
|
"vllm": {
|
|
"tensor_parallel_size": 4,
|
|
"quantization": "gptq",
|
|
"dtype": "float16",
|
|
"max_model_len": 8192,
|
|
"gpu_memory_utilization": 0.9,
|
|
"max_num_seqs": 4,
|
|
"extra_args": [
|
|
"--trust-remote-code",
|
|
"--enforce-eager"
|
|
]
|
|
},
|
|
"runtime_id": "vllm-legacy-040",
|
|
"inference_backend": "vllm",
|
|
"v100_activate": true,
|
|
"v100_block_reason": null
|
|
},
|
|
{
|
|
"id": "deepseek-coder-33b-gptq",
|
|
"name": "DeepSeek Coder 33B GPTQ",
|
|
"description": "DeepSeek open weights (TheBloke GPTQ). Top-tier coder, TP=4.",
|
|
"tags": [
|
|
"coding",
|
|
"deepseek",
|
|
"opensource",
|
|
"v100_ready"
|
|
],
|
|
"license": "DeepSeek License",
|
|
"repo_id": "TheBloke/deepseek-coder-33B-instruct-GPTQ",
|
|
"local_path": "/root/models/deepseek-coder-33b-gptq",
|
|
"served_name": "deepseek-coder-33b-gptq",
|
|
"size_gb_estimate": 18,
|
|
"vllm_min_version": "0.4.0",
|
|
"vllm": {
|
|
"tensor_parallel_size": 4,
|
|
"quantization": "gptq",
|
|
"dtype": "float16",
|
|
"max_model_len": 4096,
|
|
"gpu_memory_utilization": 0.88,
|
|
"max_num_seqs": 4,
|
|
"extra_args": [
|
|
"--trust-remote-code",
|
|
"--enforce-eager"
|
|
]
|
|
},
|
|
"runtime_id": "vllm-legacy-040",
|
|
"inference_backend": "vllm",
|
|
"v100_activate": true,
|
|
"v100_block_reason": null
|
|
},
|
|
{
|
|
"id": "qwen25-coder-7b",
|
|
"name": "Qwen 2.5 Coder 7B",
|
|
"description": "Alibaba Qwen open weights. Fast code model, 1 GPU.",
|
|
"tags": [
|
|
"coding",
|
|
"opensource",
|
|
"qwen",
|
|
"v100_ready"
|
|
],
|
|
"license": "Apache-2.0",
|
|
"repo_id": "Qwen/Qwen2.5-Coder-7B-Instruct",
|
|
"local_path": "/root/models/qwen25-coder-7b",
|
|
"served_name": "qwen2.5-coder-7b",
|
|
"size_gb_estimate": 15,
|
|
"vllm_min_version": "0.4.0",
|
|
"vllm": {
|
|
"tensor_parallel_size": 1,
|
|
"dtype": "float16",
|
|
"max_model_len": 8192,
|
|
"gpu_memory_utilization": 0.9,
|
|
"max_num_seqs": 8,
|
|
"extra_args": [
|
|
"--trust-remote-code",
|
|
"--enforce-eager"
|
|
]
|
|
},
|
|
"runtime_id": "vllm-legacy-040",
|
|
"inference_backend": "vllm",
|
|
"v100_activate": true,
|
|
"v100_block_reason": null
|
|
},
|
|
{
|
|
"id": "deepseek-coder-6.7b",
|
|
"name": "DeepSeek Coder 6.7B",
|
|
"description": "DeepSeek open weights. Fast lightweight coder, 1 GPU.",
|
|
"tags": [
|
|
"coding",
|
|
"deepseek",
|
|
"opensource",
|
|
"v100_ready"
|
|
],
|
|
"license": "DeepSeek License",
|
|
"repo_id": "deepseek-ai/deepseek-coder-6.7b-instruct",
|
|
"local_path": "/root/models/deepseek-coder-6.7b",
|
|
"served_name": "deepseek-coder-6.7b",
|
|
"size_gb_estimate": 14,
|
|
"vllm_min_version": "0.4.0",
|
|
"vllm": {
|
|
"tensor_parallel_size": 1,
|
|
"dtype": "float16",
|
|
"max_model_len": 8192,
|
|
"gpu_memory_utilization": 0.9,
|
|
"max_num_seqs": 8,
|
|
"extra_args": [
|
|
"--trust-remote-code",
|
|
"--enforce-eager"
|
|
]
|
|
},
|
|
"runtime_id": "vllm-legacy-040",
|
|
"inference_backend": "vllm",
|
|
"v100_activate": true,
|
|
"v100_block_reason": null
|
|
},
|
|
{
|
|
"id": "deepseek-coder-7b",
|
|
"name": "DeepSeek Coder 7B v1.5",
|
|
"description": "DeepSeek open weights. Balanced 7B coder.",
|
|
"tags": [
|
|
"coding",
|
|
"deepseek",
|
|
"opensource",
|
|
"v100_ready"
|
|
],
|
|
"license": "DeepSeek License",
|
|
"repo_id": "deepseek-ai/deepseek-coder-7b-instruct-v1.5",
|
|
"local_path": "/root/models/deepseek-coder-7b",
|
|
"served_name": "deepseek-coder-7b",
|
|
"size_gb_estimate": 15,
|
|
"vllm_min_version": "0.4.0",
|
|
"vllm": {
|
|
"tensor_parallel_size": 1,
|
|
"dtype": "float16",
|
|
"max_model_len": 8192,
|
|
"gpu_memory_utilization": 0.9,
|
|
"max_num_seqs": 8,
|
|
"extra_args": [
|
|
"--trust-remote-code",
|
|
"--enforce-eager"
|
|
]
|
|
},
|
|
"runtime_id": "vllm-legacy-040",
|
|
"inference_backend": "vllm",
|
|
"v100_activate": true,
|
|
"v100_block_reason": null
|
|
},
|
|
{
|
|
"id": "starcoder2-15b",
|
|
"name": "StarCoder2 15B",
|
|
"description": "BigCode/Hugging Face. Apache-2.0 code specialist, 1\u20132 GPU.",
|
|
"tags": [
|
|
"coding",
|
|
"opensource",
|
|
"v100_ready"
|
|
],
|
|
"license": "Apache-2.0",
|
|
"repo_id": "bigcode/starcoder2-15b",
|
|
"local_path": "/root/models/starcoder2-15b",
|
|
"served_name": "starcoder2-15b",
|
|
"size_gb_estimate": 30,
|
|
"vllm_min_version": "0.4.0",
|
|
"vllm": {
|
|
"tensor_parallel_size": 2,
|
|
"dtype": "float16",
|
|
"max_model_len": 8192,
|
|
"gpu_memory_utilization": 0.9,
|
|
"max_num_seqs": 4,
|
|
"extra_args": [
|
|
"--trust-remote-code",
|
|
"--enforce-eager"
|
|
]
|
|
},
|
|
"runtime_id": "vllm-legacy-040",
|
|
"inference_backend": "vllm",
|
|
"v100_activate": true,
|
|
"v100_block_reason": null
|
|
},
|
|
{
|
|
"id": "granite-code-8b",
|
|
"name": "IBM Granite Code 8B",
|
|
"description": "IBM open weights. Enterprise-friendly Apache-2.0 coder.",
|
|
"tags": [
|
|
"coding",
|
|
"opensource",
|
|
"v100_ready"
|
|
],
|
|
"license": "Apache-2.0",
|
|
"repo_id": "ibm-granite/granite-3.3-8b-instruct",
|
|
"local_path": "/root/models/granite-code-8b",
|
|
"served_name": "granite-code-8b",
|
|
"size_gb_estimate": 16,
|
|
"vllm_min_version": "0.4.0",
|
|
"vllm": {
|
|
"tensor_parallel_size": 1,
|
|
"dtype": "float16",
|
|
"max_model_len": 8192,
|
|
"gpu_memory_utilization": 0.9,
|
|
"max_num_seqs": 8,
|
|
"extra_args": [
|
|
"--trust-remote-code",
|
|
"--enforce-eager"
|
|
]
|
|
},
|
|
"runtime_id": "vllm-legacy-040",
|
|
"inference_backend": "vllm",
|
|
"v100_activate": true,
|
|
"v100_block_reason": null
|
|
},
|
|
{
|
|
"id": "yi-coder-9b",
|
|
"name": "Yi-Coder 9B",
|
|
"description": "01.AI open weights. Strong 9B code model, 1 GPU.",
|
|
"tags": [
|
|
"coding",
|
|
"opensource",
|
|
"v100_ready"
|
|
],
|
|
"license": "Apache-2.0",
|
|
"repo_id": "01-ai/Yi-Coder-9B-Chat",
|
|
"local_path": "/root/models/yi-coder-9b",
|
|
"served_name": "yi-coder-9b",
|
|
"size_gb_estimate": 18,
|
|
"vllm_min_version": "0.4.0",
|
|
"vllm": {
|
|
"tensor_parallel_size": 1,
|
|
"dtype": "float16",
|
|
"max_model_len": 8192,
|
|
"gpu_memory_utilization": 0.9,
|
|
"max_num_seqs": 8,
|
|
"extra_args": [
|
|
"--trust-remote-code",
|
|
"--enforce-eager"
|
|
]
|
|
},
|
|
"runtime_id": "vllm-legacy-040",
|
|
"inference_backend": "vllm",
|
|
"v100_activate": true,
|
|
"v100_block_reason": null
|
|
},
|
|
{
|
|
"id": "deepseek-coder-v2-lite",
|
|
"name": "DeepSeek Coder V2 Lite",
|
|
"description": "DeepSeek MoE coder (16B/2.4B active). vLLM 0.8 on V100.",
|
|
"tags": [
|
|
"coding",
|
|
"deepseek",
|
|
"opensource",
|
|
"v100_ready"
|
|
],
|
|
"license": "DeepSeek License",
|
|
"repo_id": "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct",
|
|
"local_path": "/root/models/deepseek-coder-v2-lite",
|
|
"served_name": "deepseek-coder-v2-lite",
|
|
"size_gb_estimate": 32,
|
|
"vllm_min_version": "0.6.0",
|
|
"vllm": {
|
|
"tensor_parallel_size": 2,
|
|
"dtype": "bfloat16",
|
|
"max_model_len": 8192,
|
|
"gpu_memory_utilization": 0.9,
|
|
"max_num_seqs": 4,
|
|
"extra_args": [
|
|
"--trust-remote-code",
|
|
"--enforce-eager"
|
|
]
|
|
},
|
|
"runtime_id": "vllm-modern-080",
|
|
"inference_backend": "vllm",
|
|
"v100_activate": true,
|
|
"v100_block_reason": null
|
|
},
|
|
{
|
|
"id": "qwen3-coder-next",
|
|
"name": "Qwen3 Coder Next",
|
|
"description": "Alibaba agentic coder (80B MoE, 3B active). vLLM 0.8 on V100.",
|
|
"tags": [
|
|
"coding",
|
|
"opensource",
|
|
"qwen",
|
|
"v100_ready"
|
|
],
|
|
"license": "Apache-2.0",
|
|
"repo_id": "Qwen/Qwen3-Coder-Next",
|
|
"local_path": "/root/models/qwen3-coder-next",
|
|
"served_name": "qwen3-coder-next",
|
|
"size_gb_estimate": 45,
|
|
"vllm_min_version": "0.8.0",
|
|
"vllm": {
|
|
"tensor_parallel_size": 4,
|
|
"dtype": "bfloat16",
|
|
"max_model_len": 8192,
|
|
"gpu_memory_utilization": 0.9,
|
|
"max_num_seqs": 4,
|
|
"extra_args": [
|
|
"--trust-remote-code",
|
|
"--enforce-eager"
|
|
]
|
|
},
|
|
"runtime_id": "vllm-modern-080",
|
|
"inference_backend": "vllm",
|
|
"v100_activate": true,
|
|
"v100_block_reason": null
|
|
},
|
|
{
|
|
"id": "qwen3-coder-30b",
|
|
"name": "Qwen3 Coder 30B A3B",
|
|
"description": "Alibaba MoE agentic coder. vLLM 0.8 on V100.",
|
|
"tags": [
|
|
"coding",
|
|
"opensource",
|
|
"qwen",
|
|
"v100_ready"
|
|
],
|
|
"license": "Apache-2.0",
|
|
"repo_id": "Qwen/Qwen3-Coder-30B-A3B-Instruct",
|
|
"local_path": "/root/models/qwen3-coder-30b",
|
|
"served_name": "qwen3-coder-30b",
|
|
"size_gb_estimate": 60,
|
|
"vllm_min_version": "0.8.0",
|
|
"vllm": {
|
|
"tensor_parallel_size": 4,
|
|
"dtype": "bfloat16",
|
|
"max_model_len": 8192,
|
|
"gpu_memory_utilization": 0.9,
|
|
"max_num_seqs": 4,
|
|
"extra_args": [
|
|
"--trust-remote-code",
|
|
"--enforce-eager"
|
|
]
|
|
},
|
|
"runtime_id": "vllm-modern-080",
|
|
"inference_backend": "vllm",
|
|
"v100_activate": true,
|
|
"v100_block_reason": null
|
|
},
|
|
{
|
|
"id": "qwen36-27b",
|
|
"name": "Qwen 3.6 27B",
|
|
"description": "Qwen open weights (dense). On V100 via SGLang \u2014 prefer AWQ variant.",
|
|
"tags": [
|
|
"coding",
|
|
"opensource",
|
|
"qwen",
|
|
"v100_via_sglang"
|
|
],
|
|
"license": "Apache-2.0",
|
|
"repo_id": "Qwen/Qwen3.6-27B",
|
|
"local_path": "/root/models/qwen36-27b",
|
|
"served_name": "qwen3.6-27b",
|
|
"size_gb_estimate": 54,
|
|
"vllm_min_version": "0.19.0",
|
|
"vllm": {
|
|
"tensor_parallel_size": 4,
|
|
"dtype": "bfloat16",
|
|
"max_model_len": 8192,
|
|
"gpu_memory_utilization": 0.9,
|
|
"max_num_seqs": 4,
|
|
"extra_args": [
|
|
"--trust-remote-code",
|
|
"--enforce-eager"
|
|
]
|
|
},
|
|
"runtime_id": "sglang-modern",
|
|
"inference_backend": "sglang",
|
|
"v100_activate": true,
|
|
"v100_block_reason": null
|
|
},
|
|
{
|
|
"id": "qwen36-27b-awq",
|
|
"name": "Qwen 3.6 27B AWQ",
|
|
"description": "Qwen open weights (AWQ). Primary Qwen 3.6 path on V100 via SGLang.",
|
|
"tags": [
|
|
"coding",
|
|
"opensource",
|
|
"qwen",
|
|
"v100_via_sglang"
|
|
],
|
|
"license": "Apache-2.0",
|
|
"repo_id": "QuantTrio/Qwen3.6-27B-AWQ",
|
|
"local_path": "/root/models/qwen36-27b-awq",
|
|
"served_name": "qwen3.6-27b-awq",
|
|
"size_gb_estimate": 16,
|
|
"vllm_min_version": "0.19.0",
|
|
"vllm": {
|
|
"tensor_parallel_size": 4,
|
|
"quantization": "awq",
|
|
"dtype": "float16",
|
|
"max_model_len": 8192,
|
|
"gpu_memory_utilization": 0.9,
|
|
"max_num_seqs": 8,
|
|
"extra_args": [
|
|
"--trust-remote-code",
|
|
"--enforce-eager"
|
|
]
|
|
},
|
|
"runtime_id": "sglang-modern",
|
|
"inference_backend": "sglang",
|
|
"v100_activate": true,
|
|
"v100_block_reason": null
|
|
},
|
|
{
|
|
"id": "codellama-34b",
|
|
"name": "Code Llama 34B",
|
|
"description": "Meta open weights (custom license). Classic code LLM \u2014 downloadable via HF.",
|
|
"tags": [
|
|
"coding",
|
|
"opensource",
|
|
"v100_ready"
|
|
],
|
|
"license": "Llama 3 Community License",
|
|
"repo_id": "codellama/CodeLlama-34b-Instruct-hf",
|
|
"local_path": "/root/models/codellama-34b",
|
|
"served_name": "codellama-34b",
|
|
"size_gb_estimate": 68,
|
|
"vllm_min_version": "0.4.0",
|
|
"vllm": {
|
|
"tensor_parallel_size": 4,
|
|
"dtype": "float16",
|
|
"max_model_len": 4096,
|
|
"gpu_memory_utilization": 0.9,
|
|
"max_num_seqs": 2,
|
|
"extra_args": [
|
|
"--enforce-eager"
|
|
]
|
|
},
|
|
"runtime_id": "vllm-legacy-040",
|
|
"inference_backend": "vllm",
|
|
"v100_activate": true,
|
|
"v100_block_reason": null
|
|
}
|
|
]
|
|
}
|