diff --git a/Dockerfile b/Dockerfile index ce8d332..28aa5dc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,8 +6,10 @@ # # docker build --platform linux/amd64 -t /granite-switch-systemone: . # -# Optional ASR support (build granite-switch[audio] in, and bake in a specific speech -# model's weights at a pinned revision -- see ASR_MODEL/ASR_REVISION below): +# Optional speech input (granite-switch[audio], and a specific speech model's weights baked in +# at a pinned revision -- see ASR_MODEL/ASR_REVISION below). Decisions do not use it, so +# serve.sh switches it off unless SYSTEMONE_SPEECH_INPUT=1; build with WITH_ASR=1 for an image +# that has what it needs: # docker build --platform linux/amd64 --build-arg WITH_ASR=1 \ # -t /granite-switch-systemone:-asr . # vLLM 0.30, the version this plugin is tested on live (granite-switch 0.1.1 supports @@ -39,11 +41,16 @@ RUN python3 -m pip list --format=freeze | grep -i -E "^(vllm|torch|torchaudio|to # baked in as a Hub cache at the revision this image was tested with, without the # repo's unused .py files. The server runs with the Hub offline, so a missing/moved file # fails startup instead of silently fetching something else. +# The pinned commit is downloaded by hash, not from `main`, so the build does not change when +# the model repository moves. A download by commit writes no refs/main, and the offline lookup +# for the revision "main" reads that file, so it is written here and then checked. ARG ASR_MODEL=ibm-granite/granite-speech-5.0-470m-turboctc ARG ASR_REVISION=947f59af40db9791170a0628cf0f3f4812d720f1 RUN if [ "$WITH_ASR" = "1" ]; then \ - hf download ${ASR_MODEL} --cache-dir /opt/hf-cache --exclude "*.py" > /dev/null \ - && test "$(cat /opt/hf-cache/models--${ASR_MODEL//\//--}/refs/main)" = "${ASR_REVISION}"; \ + hf download ${ASR_MODEL} --revision ${ASR_REVISION} --cache-dir /opt/hf-cache --exclude "*.py" > /dev/null \ + && mkdir -p /opt/hf-cache/models--${ASR_MODEL//\//--}/refs \ + && printf '%s' "${ASR_REVISION}" > /opt/hf-cache/models--${ASR_MODEL//\//--}/refs/main \ + && test -d /opt/hf-cache/models--${ASR_MODEL//\//--}/snapshots/${ASR_REVISION}; \ fi ENV HF_HUB_CACHE=/opt/hf-cache HF_HUB_OFFLINE=1 diff --git a/README.md b/README.md index e6c5ec8..3dcb900 100644 --- a/README.md +++ b/README.md @@ -75,24 +75,29 @@ torch match what granite-switch declares and are the packages the live tests ran plus `granite-switch[vllm]==0.1.1`, `transformers==5.17.0` and this plugin. Weights are not baked in: mount a downloaded checkpoint at `/repository`, or set `MODEL_DIR`. -**Speech model.** The 8b and 30b checkpoints enable speech input and name no speech model, so -Granite Switch falls back to a default one (`ibm-granite/granite-speech-5.0-470m-turboctc`). -The image runs with the Hub offline, so that model must already be in its cache. The plain -build does not include it, and a server started from it with these checkpoints stops during -startup. Build with `--build-arg WITH_ASR=1`, which bakes the model in: - -```bash -docker build --platform linux/amd64 --build-arg WITH_ASR=1 -t granite-switch-systemone:asr . -``` - -Alternatively mount a cache that holds the model and set `HF_HUB_CACHE` to it, or set -`HF_HUB_OFFLINE=0` so it downloads at startup. `serve.sh` checks this before starting vLLM and -says which of these to do. - -**Status of the image: lightly tested.** It builds (amd64, plain and `WITH_ASR=1`) and the -packages and plugin entry points were checked inside it. The plain build was served on one -H100 with the speech model supplied through a mounted cache, and the wire, robustness and -accuracy checks passed; see [docs/STATUS.md](docs/STATUS.md) for what was and was not run. The +**Speech input.** The 8b and 30b checkpoints enable speech input, and Granite Switch then loads a +speech model while the engine starts. Decisions never use it, so `serve.sh` switches speech +input off (`--hf-overrides '{"asr_enabled": false}'`): the 8b and 30b start from the checkpoint +alone, with nothing else to download or mount. The same server's `/v1/chat/completions` then +takes no audio input. + +To keep speech input, set `SYSTEMONE_SPEECH_INPUT=1`. The image runs with the Hub offline, so +the speech model (`ibm-granite/granite-speech-5.0-470m-turboctc`, about 1 GB) must then be +available. Any one of: + +- build the image with it baked in (a pinned revision): + `docker build --platform linux/amd64 --build-arg WITH_ASR=1 -t granite-switch-systemone:asr .` +- mount a Hub cache that holds it and set `HF_HUB_CACHE` to that directory; +- set `HF_HUB_OFFLINE=0` so it downloads at startup. + +With `SYSTEMONE_SPEECH_INPUT=1`, `serve.sh` checks for the model first and, if it is missing, +stops with a message naming the model and these options. + +**Status of the image: lightly tested.** It builds (amd64, with and without `WITH_ASR=1`) and +the packages and plugin entry points were checked inside it. The default image was served on +H100s with only the checkpoint mounted and speech input off, the 8b on one GPU and the 30b on two with tensor +parallelism, and the wire, robustness and accuracy checks passed. That was run under enroot, +not `docker run` itself; see [docs/STATUS.md](docs/STATUS.md) for what was and was not run. The pip path above is what most of the live tests used. If you run the image, treat the first start as the test and report what you find. diff --git a/docs/STATUS.md b/docs/STATUS.md index 420e258..9abb04e 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -28,18 +28,17 @@ and `ibm-granite/granite-switch-4.1-8b` unless a row names another checkpoint. | The 30b checkpoint | `granite-switch-4.1-30b` on vLLM 0.30.0, `--tensor-parallel-size 2` on two H100s: the full battery | pass | | Tensor parallelism | the 8b with `--tensor-parallel-size 2`: the full battery | pass | | Long contexts | the 8b with `--max-model-len 131072`, `scripts/longctx_check.py` (9 checks) | 9/9: a 98k-token request answered in one engine read with the danger-zone header and rejected without it; the 131,072 backstop and the default 32k/64k caps enforced; `/v1/decisions` has no header escape | -| Docker image build | `docker build --platform linux/amd64`, plain and `WITH_ASR=1` variants | both build; the stack inside is package-identical to the live-tested one (PyPI vLLM 0.30.0, torch 2.13.0+cu130, transformers 5.17.0, granite-switch 0.1.1); entry points and imports verified in-container | -| Serving from the image | the `WITH_ASR=1` image on one H100 (vLLM 0.30.0, the 8b), run with enroot because a Docker daemon was not available on the GPU nodes, with only the checkpoint mounted: `e2e_check`, `probe_check`, `accuracy_check` | pass (11/11, 6/6, accuracy as on the pip path). The plain image with the speech model supplied through a mounted cache also passed | -| Plain image, no speech model | the plain image with only the checkpoint mounted | does not start with the 8b: the checkpoint enables speech input and the offline cache lacks the default speech model. `serve.sh` now stops before vLLM with a message naming the model and the fixes (`WITH_ASR=1`, a mounted cache, or `HF_HUB_OFFLINE=0`) | +| Docker image build | `docker build --platform linux/amd64`, default and `WITH_ASR=1` variants | both build; the stack inside is package-identical to the live-tested one (PyPI vLLM 0.30.0, torch 2.13.0+cu130, transformers 5.17.0, granite-switch 0.1.1); entry points and imports verified in-container; the `WITH_ASR=1` image holds the pinned speech-model revision as its only snapshot, with `refs/main` pointing at it | +| Serving from the image | the default image (no speech model inside) under enroot (no Docker daemon on the GPU nodes), through `serve.sh`, only the checkpoint mounted, Hub offline: the 8b on one H100 and the 30b on two H100s (`--tensor-parallel-size 2`, set from the GPU count). On each: `e2e_check`, `probe_check`, `accuracy_check` | pass on both: 11/11 and 6/6, accuracy as on the pip path; the log shows `hf_overrides: {'asr_enabled': False}` and never names the speech model | +| Speech input on (`SYSTEMONE_SPEECH_INPUT=1`) | the 8b on the default image with the speech model supplied from a mounted Hub cache (`HF_HUB_CACHE`); and, as a negative control, the same switch with no speech model anywhere | the 8b starts and passes the same three checks; the control stops in `serve.sh` with the preflight message naming the model and the fixes, before vLLM starts. The `WITH_ASR=1` image (model baked in) was served on the 8b in an earlier run | | Decision Index runs | samples of the Decision Index suite (edition 0.3) through `/v1/systemone` at `--max-model-len 131072`, on the 8b and the 30b, as an end-to-end validation of the endpoint | completes with no errors; scores are not reported here while the methodology is verified, and the model card's published figures are the reference (see the caveats below) | ## Not tested -- **Serving the image under Docker.** The image was served under enroot, on one H100, with the 8b - (see the table). That runs the same filesystem and environment as the image, but not +- **Serving the image under Docker.** The image was served under enroot, on the 8b (one H100) + and the 30b (two H100s; see the table). That runs the same filesystem and environment as the image, but not `docker run` itself: its `ENTRYPOINT` (enroot ignores it, so `serve.sh` was started explicitly), - its `--gpus all` handling and `--ipc=host` were not exercised. The 30b and tensor parallelism - were not run from the image. + its `--gpus all` handling and `--ipc=host` were not exercised. - **The 3b.** The only 3b checkpoint, `granite-switch-4.1-3b-preview`, ships twelve adapters (`citations`, `query_rewrite`, `query_clarification`, `hallucination_detection`, `answerability`, `factuality-detection`, `policy-guardrails`, `factuality-correction`, diff --git a/docs/TESTING.md b/docs/TESTING.md index fbc4f1c..742078b 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -95,11 +95,11 @@ on the old deployment and `--compare before.json` on the new one. control token inside the question turn. The usual cause is a wrong `SYSTEMONE_INVOCATION_MARKER` or `SYSTEMONE_ADAPTER_NAME`; the error names both. The probe logs nothing when it passes. -- Before `vllm serve` starts, `serve.sh` runs `python3 -m granite_switch_systemone.preflight`. If - the checkpoint enables speech input, the Hub is offline and the speech model is not in the - cache, it stops with a message naming the model and the ways to fix it, instead of failing - later inside the engine. The image sets `HF_HUB_OFFLINE=1`, so this applies to the plain - build; the `WITH_ASR=1` build bakes the model in. +- `serve.sh` switches the checkpoint's speech input off by default, which decisions do not use, so + the 8b and 30b start without a speech model. With `SYSTEMONE_SPEECH_INPUT=1` it first runs + `python3 -m granite_switch_systemone.preflight`: if the Hub is offline and the speech model is not + in the cache, it stops with a message naming the model and the ways to fix it, instead of failing + later inside the engine. The image sets `HF_HUB_OFFLINE=1`. - Each question logs the probability mass the option labels hold. Below 0.9 it logs a warning (a wrong layout or marker makes the model want a token outside the labels); otherwise it logs at debug level. Answers are never refused on it. diff --git a/serve.sh b/serve.sh index de19887..d7f649d 100755 --- a/serve.sh +++ b/serve.sh @@ -6,10 +6,18 @@ # TENSOR_PARALLEL_SIZE default: every visible GPU # MAX_MODEL_LEN (32768), GPU_MEMORY_UTILIZATION (0.90), PORT (8000), SERVED_MODEL_NAME # SYSTEMONE_ADAPTER_NAME default "decision"; SYSTEMONE_INVOCATION_MARKER, see README +# SYSTEMONE_SPEECH_INPUT default 0: speech input is switched off, so no speech model is needed. +# 1 keeps the checkpoint's speech input; see README, "Speech input". set -e MODEL_DIR=${MODEL_DIR:-/repository} -# Fail early, with the cause, if the checkpoint needs a speech model the offline cache lacks. -python3 -m granite_switch_systemone.preflight "$MODEL_DIR" +# The 8b and 30b checkpoints enable speech input (asr_enabled) and granite-switch then loads a +# speech model at startup, which decisions never use. Off by default; with it on, fail early, +# with the cause, if the speech model is not in the offline cache. +case "${SYSTEMONE_SPEECH_INPUT:-0}" in + 0) SPEECH_ARGS=(--hf-overrides '{"asr_enabled": false}') ;; + 1) SPEECH_ARGS=(); python3 -m granite_switch_systemone.preflight "$MODEL_DIR" ;; + *) echo "SYSTEMONE_SPEECH_INPUT must be 0 or 1, not '$SYSTEMONE_SPEECH_INPUT'" >&2; exit 2 ;; +esac TP=${TENSOR_PARALLEL_SIZE:-$(python3 -c "import torch; print(max(torch.cuda.device_count(), 1))")} export SYSTEMONE_ADAPTER_NAME=${SYSTEMONE_ADAPTER_NAME:-decision} # --dtype bfloat16: composed checkpoints carry torch_dtype null (granite-switch#126). @@ -29,4 +37,5 @@ exec vllm serve "$MODEL_DIR" \ --max-logprobs 256 \ --enable-prompt-tokens-details \ --enable-auto-tool-choice --tool-call-parser granite4 \ + "${SPEECH_ARGS[@]}" \ "$@" diff --git a/src/granite_switch_systemone/preflight.py b/src/granite_switch_systemone/preflight.py index 9e0960d..d7b46ce 100644 --- a/src/granite_switch_systemone/preflight.py +++ b/src/granite_switch_systemone/preflight.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 -"""Checks to run before `vllm serve` starts; `serve.sh` calls this module. +"""Checks to run before `vllm serve` starts; `serve.sh` calls this module when +SYSTEMONE_SPEECH_INPUT=1 (with speech input off the speech model is never loaded). Granite Switch's audio processor loads a speech model while the engine profiles memory. The 8b and 30b checkpoints enable it (`asr_enabled`) without naming a model, so granite-switch falls @@ -69,6 +70,7 @@ def main(argv: list[str]) -> int: f'the speech model "{model_id}", but the Hub is offline and it is not in the cache at\n' f"{hub_cache_dir(os.environ)}. Do one of:\n" " - use an image built with --build-arg WITH_ASR=1, which bakes the model in\n" + " - leave speech input off (SYSTEMONE_SPEECH_INPUT unset or 0), which decisions do not need\n" " - mount a cache that holds it and point HF_HUB_CACHE at that directory\n" " - set HF_HUB_OFFLINE=0 so the model can be downloaded at startup", file=sys.stderr, diff --git a/tests/test_preflight.py b/tests/test_preflight.py index 99326ae..41eba94 100644 --- a/tests/test_preflight.py +++ b/tests/test_preflight.py @@ -106,6 +106,7 @@ def test_main_exits_1_and_says_what_to_do(tmp_path, monkeypatch, capsys): assert DEFAULT in err assert str(tmp_path / "cache") in err assert "WITH_ASR=1" in err + assert "SYSTEMONE_SPEECH_INPUT" in err assert "HF_HUB_OFFLINE=0" in err diff --git a/tests/test_serve_sh.py b/tests/test_serve_sh.py new file mode 100644 index 0000000..71e5a55 --- /dev/null +++ b/tests/test_serve_sh.py @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: Apache-2.0 +"""No-GPU checks for serve.sh's speech-input switch, with a stub `vllm` that records its arguments. + +python3 -m pytest tests/test_serve_sh.py -v --tb=short +""" + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +SERVE_SH = Path(__file__).resolve().parent.parent / "serve.sh" +OVERRIDE = '{"asr_enabled": false}' + + +def _run(tmp_path: Path, extra_env: dict[str, str], *args: str, asr_enabled: bool = True): + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + stub = bin_dir / "vllm" + stub.write_text('#!/bin/sh\nprintf "%s\\n" "$@" > "$STUB_OUT"\n') + stub.chmod(0o755) + model_dir = tmp_path / "model" + model_dir.mkdir() + (model_dir / "config.json").write_text(json.dumps({"asr_enabled": asr_enabled})) + out = tmp_path / "args.txt" + env = { + "PATH": f"{bin_dir}{os.pathsep}{Path(sys.executable).parent}{os.pathsep}{os.environ['PATH']}", + "MODEL_DIR": str(model_dir), + "TENSOR_PARALLEL_SIZE": "1", + "STUB_OUT": str(out), + "HF_HUB_OFFLINE": "1", + "HF_HUB_CACHE": str(tmp_path / "empty-cache"), + **extra_env, + } + done = subprocess.run(["bash", str(SERVE_SH), *args], env=env, capture_output=True, text=True) + return done, (out.read_text().splitlines() if out.exists() else []) + + +def test_speech_input_is_off_by_default_and_needs_no_speech_model(tmp_path): + done, argv = _run(tmp_path, {}) + assert done.returncode == 0, done.stderr + assert argv[argv.index("--hf-overrides") + 1] == OVERRIDE + + +def test_speech_input_on_without_the_model_stops_before_vllm(tmp_path): + done, argv = _run(tmp_path, {"SYSTEMONE_SPEECH_INPUT": "1"}) + assert done.returncode == 1 + assert argv == [] # vllm never started + assert "WITH_ASR=1" in done.stderr + + +def test_speech_input_on_with_the_model_cached_starts_without_the_override(tmp_path): + cache = tmp_path / "empty-cache" / "models--ibm-granite--granite-speech-5.0-470m-turboctc" / "snapshots" / "abc" + cache.mkdir(parents=True) + (cache / "config.json").write_text("{}") + done, argv = _run(tmp_path, {"SYSTEMONE_SPEECH_INPUT": "1"}) + assert done.returncode == 0, done.stderr + assert "--hf-overrides" not in argv + + +def test_extra_arguments_still_reach_vllm(tmp_path): + done, argv = _run(tmp_path, {}, "--api-key", "k") + assert done.returncode == 0, done.stderr + assert argv[-2:] == ["--api-key", "k"] + + +@pytest.mark.parametrize("value", ["2", "yes", ""]) +def test_bad_switch_value_is_refused(tmp_path, value): + done, argv = _run(tmp_path, {"SYSTEMONE_SPEECH_INPUT": value}) + if value == "": + assert done.returncode == 0 # empty falls back to the default, off + else: + assert done.returncode == 2 + assert argv == []