first commit

.dockerignore

@@ -0,0 +1,10 @@
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+.git/
+.agents/
+.codex/
+models/
+modelscope_cache/
+logs/

.env

@@ -0,0 +1,71 @@
+MODELSCOPE_CACHE=./modelscope_cache
+
+# 模型名称（可自定义）
+MODEL_ID=Qwen3-9B
+
+# 模型文件路径
+MODEL_DIR=./models/Qwen3.5-9B
+
+HOST=0.0.0.0
+PORT=9527
+
+# 指定加载到哪些显卡0,1,2,3,4
+CUDA_VISIBLE_DEVICES=0
+# 张量并行卡数
+TENSOR_PARALLEL_SIZE=1
+
+# 上下文长度
+MAX_MODEL_LEN=32768
+
+# 显存占用比例。默认参数0.9，多余显存分配个KV Cache以支持高并发
+GPU_MEMORY_UTILIZATION=0.4
+
+# 计算精度
+# DTYPE=bfloat16
+
+# KV Cache 精度（auto/fp8）
+# KV_CACHE_DTYPE=auto
+
+# 最大并发序列数
+MAX_NUM_SEQS=32
+
+# 单批最大 token 数，根据并发和实际上下文需求配置，默认自动分配
+MAX_NUM_BATCHED_TOKENSMAX=16384
+
+
+# 其他运行开关
+DISABLE_LOG_REQUESTS=False
+ENABLE_LOG_REQUESTS=true
+
+# VLLM运行模式DEBUG\INFO
+VLLM_LOGGING_LEVEL=INFO
+
+# Tool calling 配置，需要和模型配套，否则可能出现工具调用失败
+ENABLE_AUTO_TOOL_CHOICE=true
+TOOL_CALL_PARSER=qwen3_xml
+REASONING_PARSER=qwen3
+
+# 思考标记开关，QWEN3.5-9B不匹配
+# DEFAULT_CHAT_TEMPLATE_KWARGS='{"enable_thinking": true}'
+
+# 留空时使用模型自带 chat_template；如需官方工具模板可填绝对路径
+# CHAT_TEMPLATE=
+
+# enable-chunked-prefill分块预填空，避免瞬时大量占用内存
+# max-num-partial-prefills 1限制同一时刻最多只有 1 个 处于“分块预填充中”的请求，提高稳定性。
+
+TRUST_REMOTE_CODE=true
+API_KEY=unis123
+
+# 采样参数
+# TEMPERATURE=1.0
+# TOP_P=0.95
+# TOP_K=64
+
+LOG_DIR=./logs
+
+# Auto download model when MODEL_DIR is missing on container start.
+AUTO_DOWNLOAD_MODEL=true
+MODEL_SOURCE=Qwen/Qwen3.5-9B
+DOWNLOAD_CACHE_DIR=./modelscope_cache
+SKIP_MODEL_DOWNLOAD_IF_EXISTS=true

Dockerfile

@@ -0,0 +1,10 @@
+FROM registry.iluvatar.com.cn:10443/customer/sz/vllm0.17.0-4.4.0-x86:v5
+
+WORKDIR /workspace/vllm
+
+COPY . /workspace/vllm
+
+ENV ENV_FILE=.env
+ENV PYTHONUNBUFFERED=1
+
+CMD ["python", "serve.py"]

compose.yaml

@@ -0,0 +1,29 @@
+services:
+  vllm:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    image: local/vllm-qwen3-9b:latest
+    container_name: vllm-qwen3-9b
+    working_dir: /workspace/vllm
+    env_file:
+      - .env
+    environment:
+      ENV_FILE: .env
+      PYTHONUNBUFFERED: "1"
+    network_mode: host
+    ipc: host
+    pid: host
+    privileged: true
+    cap_add:
+      - ALL
+    restart: unless-stopped
+    volumes:
+      - ./models:/workspace/vllm/models
+      - ./modelscope_cache:/workspace/vllm/modelscope_cache
+      - ./logs:/workspace/vllm/logs
+      - /usr/src:/usr/src
+      - /lib/modules:/lib/modules
+      - /dev:/dev
+      - /home:/home
+      - /data:/data

model_download.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+"""
+Standalone model download script.
+
+Usage:
+  python model_download.py
+"""
+
+import os
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+DEFAULT_ENV_FILE = ".env"
+DEFAULT_MODEL_ID = "Qwen/Qwen3.5-9B"
+DEFAULT_MODEL_DIR = "./models/Qwen3.5-9B"
+DEFAULT_CACHE_DIR = "./modelscope_cache"
+
+
+def resolve_path(raw: str, base_dir: Path) -> Path:
+    path = Path(raw).expanduser()
+    if path.is_absolute():
+        return path.resolve()
+    return (base_dir / path).resolve()
+
+
+def load_runtime_env(script_dir: Path) -> Path:
+    env_name = (os.getenv("ENV_FILE", DEFAULT_ENV_FILE) or DEFAULT_ENV_FILE).strip()
+    env_path = (script_dir / env_name).resolve()
+    if env_path.exists():
+        load_dotenv(env_path)
+    return env_path
+
+
+def env_flag(name: str, default: bool = False) -> bool:
+    raw = os.getenv(name, "")
+    if not raw:
+        return default
+    return raw.strip().lower() in {"1", "true", "yes", "on"}
+
+
+def download_model(
+    model_id: str,
+    model_dir: Path,
+    cache_dir: Path,
+    revision: str = "",
+    skip_if_exists: bool = False,
+) -> Path:
+    try:
+        from modelscope.hub.snapshot_download import snapshot_download
+    except Exception as exc:
+        raise RuntimeError(
+            "Missing dependencies. Please install first:\n"
+            "  pip install -r requirements.txt"
+        ) from exc
+
+    model_dir.parent.mkdir(parents=True, exist_ok=True)
+    cache_dir.mkdir(parents=True, exist_ok=True)
+
+    if skip_if_exists and model_dir.exists() and any(model_dir.iterdir()):
+        print(f"[INFO] model already exists, skip download: {model_dir}")
+        return model_dir.resolve()
+
+    print(f"[INFO] model_id={model_id}")
+    print(f"[INFO] model_dir={model_dir}")
+    print(f"[INFO] cache_dir={cache_dir}")
+    if revision:
+        print(f"[INFO] revision={revision}")
+
+    kwargs = {
+        "model_id": model_id,
+        "local_dir": str(model_dir),
+        "cache_dir": str(cache_dir),
+    }
+    if revision:
+        kwargs["revision"] = revision
+
+    downloaded_path = snapshot_download(**kwargs)
+    print(f"[OK] download complete: {downloaded_path}")
+    return Path(downloaded_path).resolve()
+
+
+def main() -> None:
+    script_dir = Path(__file__).resolve().parent
+    env_path = load_runtime_env(script_dir)
+    model_id = os.getenv("DOWNLOAD_MODEL_ID", os.getenv("MODEL_SOURCE", DEFAULT_MODEL_ID)).strip()
+    model_dir_raw = os.getenv("DOWNLOAD_SAVE_DIR", os.getenv("MODEL_DIR", DEFAULT_MODEL_DIR)).strip()
+    cache_dir_raw = os.getenv("DOWNLOAD_CACHE_DIR", os.getenv("MODELSCOPE_CACHE", DEFAULT_CACHE_DIR)).strip()
+    revision = os.getenv("DOWNLOAD_REVISION", "").strip()
+    skip_if_exists = env_flag("SKIP_MODEL_DOWNLOAD_IF_EXISTS", True)
+
+    if not model_id:
+        raise ValueError("DOWNLOAD_MODEL_ID/MODEL_SOURCE is empty.")
+    if not model_dir_raw:
+        raise ValueError("DOWNLOAD_SAVE_DIR/MODEL_DIR is empty.")
+    if not cache_dir_raw:
+        raise ValueError("DOWNLOAD_CACHE_DIR/MODELSCOPE_CACHE is empty.")
+
+    model_dir = resolve_path(model_dir_raw, script_dir)
+    cache_dir = resolve_path(cache_dir_raw, script_dir)
+    print(f"[INFO] env_file={env_path}")
+    download_model(
+        model_id=model_id,
+        model_dir=model_dir,
+        cache_dir=cache_dir,
+        revision=revision,
+        skip_if_exists=skip_if_exists,
+    )
+
+
+if __name__ == "__main__":
+    main()

serve.py

@@ -0,0 +1,208 @@
+import os
+import shlex
+import subprocess
+import sys
+from pathlib import Path
+
+from dotenv import load_dotenv
+from model_download import download_model
+
+# Default env file. Override with ENV_FILE if needed.
+DEFAULT_ENV_FILE = ".env"
+
+
+def as_bool(value: str) -> bool:
+    return str(value).strip().lower() in {"1", "true", "yes", "on"}
+
+
+def resolve_path(env_name: str, default_relative: str, base_dir: Path) -> Path:
+    raw = os.getenv(env_name, "").strip()
+    if not raw:
+        return (base_dir / default_relative).resolve()
+
+    path = Path(raw).expanduser()
+    if path.is_absolute():
+        return path.resolve()
+    return (base_dir / path).resolve()
+
+
+def resolve_optional_path(raw_path: str, base_dir: Path) -> Path:
+    path = Path(raw_path).expanduser()
+    if path.is_absolute():
+        return path.resolve()
+    return (base_dir / path).resolve()
+
+
+def ensure_model_ready(script_dir: Path, model_dir: Path) -> Path:
+    auto_download = as_bool(os.getenv("AUTO_DOWNLOAD_MODEL", "false"))
+    model_source = os.getenv("MODEL_SOURCE", "").strip()
+    cache_dir_raw = os.getenv("DOWNLOAD_CACHE_DIR", os.getenv("MODELSCOPE_CACHE", "./modelscope_cache")).strip()
+    revision = os.getenv("DOWNLOAD_REVISION", "").strip()
+
+    if model_dir.exists() and any(model_dir.iterdir()):
+        return model_dir
+
+    if not auto_download:
+        raise FileNotFoundError(
+            f"Model directory does not exist: {model_dir}\n"
+            "Run `python model_download.py` first, or set AUTO_DOWNLOAD_MODEL=true."
+        )
+
+    if not model_source:
+        raise ValueError(
+            "AUTO_DOWNLOAD_MODEL=true but MODEL_SOURCE is empty.\n"
+            "Example: MODEL_SOURCE=Qwen/Qwen3.5-9B"
+        )
+
+    cache_dir = resolve_optional_path(cache_dir_raw, script_dir)
+    print("[INFO] model directory missing, start auto download")
+    download_model(
+        model_id=model_source,
+        model_dir=model_dir,
+        cache_dir=cache_dir,
+        revision=revision,
+        skip_if_exists=True,
+    )
+    return model_dir
+
+
+def main() -> None:
+    script_dir = Path(__file__).resolve().parent
+    env_path = (script_dir / (os.getenv("ENV_FILE", DEFAULT_ENV_FILE).strip() or DEFAULT_ENV_FILE)).resolve()
+    if not env_path.exists():
+        raise FileNotFoundError(f"Environment file does not exist: {env_path}")
+    load_dotenv(env_path)
+
+    cuda_visible_devices = os.getenv("CUDA_VISIBLE_DEVICES", "0").strip()
+    model_dir = resolve_path("MODEL_DIR", "models/google_gemma-4-E4B-it", script_dir)
+    host = os.getenv("HOST", "0.0.0.0")
+    port = os.getenv("PORT", "9527")
+    tensor_parallel_size = os.getenv("TENSOR_PARALLEL_SIZE", "1")
+    max_model_len = os.getenv("MAX_MODEL_LEN", "32768")
+    gpu_memory_utilization = os.getenv("GPU_MEMORY_UTILIZATION", "0.90")
+    trust_remote_code = as_bool(os.getenv("TRUST_REMOTE_CODE", "true"))
+    enable_auto_tool_choice = as_bool(os.getenv("ENABLE_AUTO_TOOL_CHOICE", "true"))
+    tool_call_parser = os.getenv("TOOL_CALL_PARSER", "auto").strip()
+    reasoning_parser = os.getenv("REASONING_PARSER", "auto").strip()
+
+    enable_log_requests_raw = os.getenv("ENABLE_LOG_REQUESTS", "").strip()
+    if enable_log_requests_raw:
+        enable_log_requests = as_bool(enable_log_requests_raw)
+    else:
+        enable_log_requests = not as_bool(os.getenv("DISABLE_LOG_REQUESTS", "false"))
+
+    vllm_logging_level = os.getenv("VLLM_LOGGING_LEVEL", "INFO").strip()
+
+    default_chat_template_kwargs = os.getenv(
+        "DEFAULT_CHAT_TEMPLATE_KWARGS", '{"enable_thinking": true}'
+    ).strip()
+
+    chat_template = os.getenv("CHAT_TEMPLATE", "").strip()
+
+    api_key = os.getenv("API_KEY", "your-secret-api-key").strip()
+    log_dir = resolve_path("LOG_DIR", "logs", script_dir)
+    max_num_seqs = os.getenv("MAX_NUM_SEQS", "64").strip()
+    max_num_batched_tokens = os.getenv("MAX_NUM_BATCHED_TOKENSMAX", "4096").strip()
+
+    model_dir = ensure_model_ready(script_dir, model_dir)
+
+    log_dir.mkdir(parents=True, exist_ok=True)
+
+    if cuda_visible_devices:
+        os.environ["CUDA_VISIBLE_DEVICES"] = cuda_visible_devices
+    if vllm_logging_level:
+        os.environ["VLLM_LOGGING_LEVEL"] = vllm_logging_level
+    # Avoid passing non-vLLM env keys through subprocess environment.
+    # These custom keys trigger "Unknown vLLM environment variable" warnings.
+
+    cmd = [
+        sys.executable,
+        "-m",
+        "vllm.entrypoints.openai.api_server",
+        "--model",
+        str(model_dir),
+        "--served-model-name",
+        os.getenv("MODEL_ID", "google/google_gemma-4-E4B-it"),
+        "--host",
+        host,
+        "--port",
+        port,
+        "--tensor-parallel-size",
+        tensor_parallel_size,
+        "--max-model-len",
+        max_model_len,
+        "--gpu-memory-utilization",
+        gpu_memory_utilization,
+    ]
+
+    if trust_remote_code:
+        cmd.append("--trust-remote-code")
+
+    if enable_log_requests:
+        cmd.append("--enable-log-requests")
+
+    if enable_auto_tool_choice:
+        cmd.append("--enable-auto-tool-choice")
+        if tool_call_parser:
+            cmd.extend(["--tool-call-parser", tool_call_parser])
+
+    if reasoning_parser:
+        cmd.extend(["--reasoning-parser", reasoning_parser])
+
+    # if default_chat_template_kwargs:
+    #     cmd.extend(["--default-chat-template-kwargs", default_chat_template_kwargs])
+
+    resolved_chat_template: Path | None = None
+    if chat_template:
+        resolved_chat_template = resolve_optional_path(chat_template, script_dir)
+        if not resolved_chat_template.exists():
+            raise FileNotFoundError(
+                f"CHAT_TEMPLATE does not exist: {resolved_chat_template}\n"
+                "Use an absolute path, or remove CHAT_TEMPLATE to let vLLM use model default template."
+            )
+
+    if resolved_chat_template is not None:
+        cmd.extend(["--chat-template", str(resolved_chat_template)])
+
+    if api_key:
+        cmd.extend(["--api-key", api_key])
+
+    if max_num_seqs:
+        cmd.extend(["--max-num-seqs", max_num_seqs])
+
+    if max_num_batched_tokens:
+        cmd.extend(["--max-num-batched-tokens", max_num_batched_tokens])
+
+
+    # Force prefill tuning flags directly in script (do not rely on env parsing).
+    cmd.extend(
+        [
+            "--enable-chunked-prefill",
+            "--max-num-partial-prefills=1",
+        ]
+    )
+
+    print("[INFO] starting vLLM server with command:")
+    print(" ".join(shlex.quote(item) for item in cmd))
+    if enable_auto_tool_choice:
+        print(f"[INFO] tool_call_parser={tool_call_parser or '(empty)'}")
+    print(f"[INFO] enable_log_requests={enable_log_requests}")
+    if vllm_logging_level:
+        print(f"[INFO] VLLM_LOGGING_LEVEL={vllm_logging_level}")
+    if reasoning_parser:
+        print(f"[INFO] reasoning_parser={reasoning_parser}")
+    if resolved_chat_template is not None:
+        print(f"[INFO] chat_template={resolved_chat_template}")
+    else:
+        print("[INFO] chat_template=(model default)")
+    if cuda_visible_devices:
+        print(f"[INFO] CUDA_VISIBLE_DEVICES={cuda_visible_devices}")
+    print(f"[INFO] resolved model_dir={model_dir}")
+    print(f"[INFO] resolved log_dir={log_dir}")
+    print(f"[INFO] env_file={env_path}")
+
+    subprocess.run(cmd, check=True)
+
+
+if __name__ == "__main__":
+    main()