mo
c154c94c6d
Mirror gpu-lab compose files and model-manager catalog from GPU-Dev; document Dockhand mapping and SGLang cutover target.
39 lines
881 B
YAML
39 lines
881 B
YAML
name: gpu-lab-sglang
|
|
services:
|
|
sglang:
|
|
image: lmsysorg/sglang:v0.5.12-cu129-runtime
|
|
container_name: gpu-lab-sglang
|
|
network_mode: host
|
|
ipc: host
|
|
shm_size: "32gb"
|
|
env_file:
|
|
- active/current.env
|
|
volumes:
|
|
- /root/models:/models:ro
|
|
deploy:
|
|
resources:
|
|
reservations:
|
|
devices:
|
|
- driver: nvidia
|
|
count: all
|
|
capabilities: [gpu]
|
|
entrypoint: ["python3", "-m", "sglang.launch_server"]
|
|
command:
|
|
- --model-path
|
|
- ${MODEL_PATH}
|
|
- --served-model-name
|
|
- ${SERVED_NAME}
|
|
- --host
|
|
- "0.0.0.0"
|
|
- --port
|
|
- "8010"
|
|
- --tp-size
|
|
- ${TP_SIZE}
|
|
- --mem-fraction-static
|
|
- ${GPU_MEM_UTIL}
|
|
- --context-length
|
|
- ${MAX_MODEL_LEN}
|
|
- --disable-flashinfer
|
|
- --disable-flashinfer-sampling
|
|
restart: "no"
|