From a93c5c37246f01ba6e00e1b37b6fa44e47f31f7c Mon Sep 17 00:00:00 2001 From: Nigel Jones Date: Wed, 21 Oct 2026 18:37:31 +0100 Subject: [PATCH 1/3] fix: download the pinned speech-model revision, not main The WITH_ASR=1 build downloaded the speech model's main branch and then checked that refs/main equalled the pinned commit. The model repository moved on main, so the check failed and the build stopped, for anyone building the image. Download the pinned commit with --revision instead, so the build no longer changes when the model repository does. A download by commit writes no refs/main, and the offline lookup for revision main reads it, so the Dockerfile writes it and checks that the pinned snapshot exists. Assisted-by: Claude Code Signed-off-by: Nigel Jones --- Dockerfile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index ce8d332..fc907b0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,11 +39,16 @@ RUN python3 -m pip list --format=freeze | grep -i -E "^(vllm|torch|torchaudio|to # baked in as a Hub cache at the revision this image was tested with, without the # repo's unused .py files. The server runs with the Hub offline, so a missing/moved file # fails startup instead of silently fetching something else. +# The pinned commit is downloaded by hash, not from `main`, so the build does not change when +# the model repository moves. A download by commit writes no refs/main, and the offline lookup +# for the revision "main" reads that file, so it is written here and then checked. ARG ASR_MODEL=ibm-granite/granite-speech-5.0-470m-turboctc ARG ASR_REVISION=947f59af40db9791170a0628cf0f3f4812d720f1 RUN if [ "$WITH_ASR" = "1" ]; then \ - hf download ${ASR_MODEL} --cache-dir /opt/hf-cache --exclude "*.py" > /dev/null \ - && test "$(cat /opt/hf-cache/models--${ASR_MODEL//\//--}/refs/main)" = "${ASR_REVISION}"; \ + hf download ${ASR_MODEL} --revision ${ASR_REVISION} --cache-dir /opt/hf-cache --exclude "*.py" > /dev/null \ + && mkdir -p /opt/hf-cache/models--${ASR_MODEL//\//--}/refs \ + && printf '%s' "${ASR_REVISION}" > /opt/hf-cache/models--${ASR_MODEL//\//--}/refs/main \ + && test -d /opt/hf-cache/models--${ASR_MODEL//\//--}/snapshots/${ASR_REVISION}; \ fi ENV HF_HUB_CACHE=/opt/hf-cache HF_HUB_OFFLINE=1 From 24a5d3e091dcad3a4b034781b253674bf2e69e79 Mon Sep 17 00:00:00 2001 From: Nigel Jones Date: Thu, 22 Oct 2026 16:31:31 +0100 Subject: [PATCH 2/3] feat: bake the speech model into the default image; WITH_ASR=0 for the slim one The 8b and 30b checkpoints enable speech input and the image runs with the Hub offline, so the plain image could not start with them. WITH_ASR now defaults to 1; WITH_ASR=0 builds the slim image. The README, TESTING.md and the preflight message follow. STATUS.md records the default image served from the checkpoint alone: the 8b on one H100 and the 30b on two with tensor parallelism. Assisted-by: Claude Code Signed-off-by: Nigel Jones --- Dockerfile | 14 ++++++++------ README.md | 23 ++++++++++++----------- docs/STATUS.md | 13 ++++++------- docs/TESTING.md | 4 ++-- src/granite_switch_systemone/preflight.py | 2 +- tests/test_preflight.py | 2 +- 6 files changed, 30 insertions(+), 28 deletions(-) diff --git a/Dockerfile b/Dockerfile index fc907b0..342520c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,10 +6,12 @@ # # docker build --platform linux/amd64 -t /granite-switch-systemone: . # -# Optional ASR support (build granite-switch[audio] in, and bake in a specific speech -# model's weights at a pinned revision -- see ASR_MODEL/ASR_REVISION below): -# docker build --platform linux/amd64 --build-arg WITH_ASR=1 \ -# -t /granite-switch-systemone:-asr . +# The speech model the 8b and 30b checkpoints need is baked in by default (granite-switch[audio], +# and a specific speech model's weights at a pinned revision -- see ASR_MODEL/ASR_REVISION +# below). The server runs with the Hub offline, so without it those checkpoints cannot start. +# A slim image without it, for a checkpoint that has speech input off or a mounted cache: +# docker build --platform linux/amd64 --build-arg WITH_ASR=0 \ +# -t /granite-switch-systemone:-slim . # vLLM 0.30, the version this plugin is tested on live (granite-switch 0.1.1 supports # 0.26-0.30). The default tag is the CUDA 13 build (torch 2.13.0+cu130, the PyPI vLLM # wheel) -- the same stack the live tests ran. The -cu129 tag ships torch 2.14.0, @@ -19,12 +21,12 @@ FROM vllm/vllm-openai:v0.30.0 SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG GRANITE_SWITCH_VERSION=0.1.1 -ARG WITH_ASR=0 +ARG WITH_ASR=1 # Pinned to a specific, released granite-switch version (not `main`) so this image's # behavior does not drift when upstream changes -- deliberately a tagged PyPI release, # not a floating branch. transformers 5.17 is what granite-switch supports; [audio] is -# only installed when WITH_ASR=1 (it brings librosa and soundfile, used only by the +# only installed when WITH_ASR=1, the default (it brings librosa and soundfile, used only by the # ASR cascade). The base image's vLLM, torch, and torchcodec (CUDA 13 builds) are kept # as they are: the constraint file pins them so nothing in granite-switch's dependency # tree can move them. diff --git a/README.md b/README.md index e6c5ec8..588c3a6 100644 --- a/README.md +++ b/README.md @@ -77,22 +77,23 @@ not baked in: mount a downloaded checkpoint at `/repository`, or set `MODEL_DIR` **Speech model.** The 8b and 30b checkpoints enable speech input and name no speech model, so Granite Switch falls back to a default one (`ibm-granite/granite-speech-5.0-470m-turboctc`). -The image runs with the Hub offline, so that model must already be in its cache. The plain -build does not include it, and a server started from it with these checkpoints stops during -startup. Build with `--build-arg WITH_ASR=1`, which bakes the model in: +The image runs with the Hub offline, so the build bakes that model in (a pinned revision, about +1 GB). For a checkpoint with speech input off, or to supply the model from a mounted cache +(point `HF_HUB_CACHE` at it), build the slim image: ```bash -docker build --platform linux/amd64 --build-arg WITH_ASR=1 -t granite-switch-systemone:asr . +docker build --platform linux/amd64 --build-arg WITH_ASR=0 -t granite-switch-systemone:slim . ``` -Alternatively mount a cache that holds the model and set `HF_HUB_CACHE` to it, or set -`HF_HUB_OFFLINE=0` so it downloads at startup. `serve.sh` checks this before starting vLLM and -says which of these to do. +With the slim image and these checkpoints, a server without the model stops during startup. +`serve.sh` checks this first and says what to do; setting `HF_HUB_OFFLINE=0` so the model +downloads at startup also works. -**Status of the image: lightly tested.** It builds (amd64, plain and `WITH_ASR=1`) and the -packages and plugin entry points were checked inside it. The plain build was served on one -H100 with the speech model supplied through a mounted cache, and the wire, robustness and -accuracy checks passed; see [docs/STATUS.md](docs/STATUS.md) for what was and was not run. The +**Status of the image: lightly tested.** It builds (amd64, with and without the speech model) and +the packages and plugin entry points were checked inside it. The default image was served on +H100s with only the checkpoint mounted, the 8b on one GPU and the 30b on two with tensor +parallelism, and the wire, robustness and accuracy checks passed. That was run under enroot, +not `docker run` itself; see [docs/STATUS.md](docs/STATUS.md) for what was and was not run. The pip path above is what most of the live tests used. If you run the image, treat the first start as the test and report what you find. diff --git a/docs/STATUS.md b/docs/STATUS.md index 420e258..f4accb9 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -28,18 +28,17 @@ and `ibm-granite/granite-switch-4.1-8b` unless a row names another checkpoint. | The 30b checkpoint | `granite-switch-4.1-30b` on vLLM 0.30.0, `--tensor-parallel-size 2` on two H100s: the full battery | pass | | Tensor parallelism | the 8b with `--tensor-parallel-size 2`: the full battery | pass | | Long contexts | the 8b with `--max-model-len 131072`, `scripts/longctx_check.py` (9 checks) | 9/9: a 98k-token request answered in one engine read with the danger-zone header and rejected without it; the 131,072 backstop and the default 32k/64k caps enforced; `/v1/decisions` has no header escape | -| Docker image build | `docker build --platform linux/amd64`, plain and `WITH_ASR=1` variants | both build; the stack inside is package-identical to the live-tested one (PyPI vLLM 0.30.0, torch 2.13.0+cu130, transformers 5.17.0, granite-switch 0.1.1); entry points and imports verified in-container | -| Serving from the image | the `WITH_ASR=1` image on one H100 (vLLM 0.30.0, the 8b), run with enroot because a Docker daemon was not available on the GPU nodes, with only the checkpoint mounted: `e2e_check`, `probe_check`, `accuracy_check` | pass (11/11, 6/6, accuracy as on the pip path). The plain image with the speech model supplied through a mounted cache also passed | -| Plain image, no speech model | the plain image with only the checkpoint mounted | does not start with the 8b: the checkpoint enables speech input and the offline cache lacks the default speech model. `serve.sh` now stops before vLLM with a message naming the model and the fixes (`WITH_ASR=1`, a mounted cache, or `HF_HUB_OFFLINE=0`) | +| Docker image build | `docker build --platform linux/amd64`, default (speech model baked in) and `WITH_ASR=0` (slim) variants | both build; the stack inside is package-identical to the live-tested one (PyPI vLLM 0.30.0, torch 2.13.0+cu130, transformers 5.17.0, granite-switch 0.1.1); entry points and imports verified in-container; the default image holds the pinned speech-model revision as its only snapshot, with `refs/main` pointing at it | +| Serving from the image | the default image (v0.1.2 code) under enroot (no Docker daemon on the GPU nodes), only the checkpoint mounted and no Hub cache: the 8b on one H100, then the 30b on two H100s with `--tensor-parallel-size 2` (set from the GPU count by `serve.sh`). On each: `e2e_check`, `probe_check`, `accuracy_check` | pass on both: 11/11 and 6/6, accuracy as on the pip path. The earlier `WITH_ASR=1` image on the 8b, and the plain image with a mounted speech cache, also passed | +| Slim image, no speech model | the `WITH_ASR=0` image with only the checkpoint mounted | does not start with the 8b: the checkpoint enables speech input and the offline cache lacks the default speech model. `serve.sh` now stops before vLLM with a message naming the model and the fixes (the default image, a mounted cache, or `HF_HUB_OFFLINE=0`) | | Decision Index runs | samples of the Decision Index suite (edition 0.3) through `/v1/systemone` at `--max-model-len 131072`, on the 8b and the 30b, as an end-to-end validation of the endpoint | completes with no errors; scores are not reported here while the methodology is verified, and the model card's published figures are the reference (see the caveats below) | ## Not tested -- **Serving the image under Docker.** The image was served under enroot, on one H100, with the 8b - (see the table). That runs the same filesystem and environment as the image, but not +- **Serving the image under Docker.** The image was served under enroot, on the 8b (one H100) + and the 30b (two H100s; see the table). That runs the same filesystem and environment as the image, but not `docker run` itself: its `ENTRYPOINT` (enroot ignores it, so `serve.sh` was started explicitly), - its `--gpus all` handling and `--ipc=host` were not exercised. The 30b and tensor parallelism - were not run from the image. + its `--gpus all` handling and `--ipc=host` were not exercised. - **The 3b.** The only 3b checkpoint, `granite-switch-4.1-3b-preview`, ships twelve adapters (`citations`, `query_rewrite`, `query_clarification`, `hallucination_detection`, `answerability`, `factuality-detection`, `policy-guardrails`, `factuality-correction`, diff --git a/docs/TESTING.md b/docs/TESTING.md index fbc4f1c..05b4a81 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -98,8 +98,8 @@ on the old deployment and `--compare before.json` on the new one. - Before `vllm serve` starts, `serve.sh` runs `python3 -m granite_switch_systemone.preflight`. If the checkpoint enables speech input, the Hub is offline and the speech model is not in the cache, it stops with a message naming the model and the ways to fix it, instead of failing - later inside the engine. The image sets `HF_HUB_OFFLINE=1`, so this applies to the plain - build; the `WITH_ASR=1` build bakes the model in. + later inside the engine. The image sets `HF_HUB_OFFLINE=1`, so this applies to the slim + build (`--build-arg WITH_ASR=0`); the default build bakes the model in. - Each question logs the probability mass the option labels hold. Below 0.9 it logs a warning (a wrong layout or marker makes the model want a token outside the labels); otherwise it logs at debug level. Answers are never refused on it. diff --git a/src/granite_switch_systemone/preflight.py b/src/granite_switch_systemone/preflight.py index 9e0960d..fd6c8e8 100644 --- a/src/granite_switch_systemone/preflight.py +++ b/src/granite_switch_systemone/preflight.py @@ -68,7 +68,7 @@ def main(argv: list[str]) -> int: f"granite-switch-systemone: this checkpoint enables speech input (asr_enabled in config.json) and needs\n" f'the speech model "{model_id}", but the Hub is offline and it is not in the cache at\n' f"{hub_cache_dir(os.environ)}. Do one of:\n" - " - use an image built with --build-arg WITH_ASR=1, which bakes the model in\n" + " - use the default image (built without --build-arg WITH_ASR=0), which bakes the model in\n" " - mount a cache that holds it and point HF_HUB_CACHE at that directory\n" " - set HF_HUB_OFFLINE=0 so the model can be downloaded at startup", file=sys.stderr, diff --git a/tests/test_preflight.py b/tests/test_preflight.py index 99326ae..a0f7508 100644 --- a/tests/test_preflight.py +++ b/tests/test_preflight.py @@ -105,7 +105,7 @@ def test_main_exits_1_and_says_what_to_do(tmp_path, monkeypatch, capsys): err = capsys.readouterr().err assert DEFAULT in err assert str(tmp_path / "cache") in err - assert "WITH_ASR=1" in err + assert "WITH_ASR=0" in err assert "HF_HUB_OFFLINE=0" in err From 5cf4d747c7e9c4498338dbbd4ace5976ef5dbde6 Mon Sep 17 00:00:00 2001 From: Nigel Jones Date: Thu, 22 Oct 2026 21:43:46 +0100 Subject: [PATCH 3/3] feat: switch speech input off by default so the 8b and 30b start without a speech model The 8b and 30b checkpoints enable speech input, which makes granite-switch load a speech model at startup that decisions never use. serve.sh now passes --hf-overrides '{"asr_enabled": false}' unless SYSTEMONE_SPEECH_INPUT=1, so the checkpoint alone is enough. With it set to 1 the preflight runs as before. WITH_ASR goes back to 0 (the speech model is no longer baked in by default); WITH_ASR=1 still bakes it in for the opt-in path. The README, TESTING.md and STATUS.md follow. Assisted-by: Claude Code Signed-off-by: Nigel Jones --- Dockerfile | 16 ++--- README.md | 30 +++++---- docs/STATUS.md | 6 +- docs/TESTING.md | 10 +-- serve.sh | 13 +++- src/granite_switch_systemone/preflight.py | 6 +- tests/test_preflight.py | 3 +- tests/test_serve_sh.py | 77 +++++++++++++++++++++++ 8 files changed, 127 insertions(+), 34 deletions(-) create mode 100644 tests/test_serve_sh.py diff --git a/Dockerfile b/Dockerfile index 342520c..28aa5dc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,12 +6,12 @@ # # docker build --platform linux/amd64 -t /granite-switch-systemone: . # -# The speech model the 8b and 30b checkpoints need is baked in by default (granite-switch[audio], -# and a specific speech model's weights at a pinned revision -- see ASR_MODEL/ASR_REVISION -# below). The server runs with the Hub offline, so without it those checkpoints cannot start. -# A slim image without it, for a checkpoint that has speech input off or a mounted cache: -# docker build --platform linux/amd64 --build-arg WITH_ASR=0 \ -# -t /granite-switch-systemone:-slim . +# Optional speech input (granite-switch[audio], and a specific speech model's weights baked in +# at a pinned revision -- see ASR_MODEL/ASR_REVISION below). Decisions do not use it, so +# serve.sh switches it off unless SYSTEMONE_SPEECH_INPUT=1; build with WITH_ASR=1 for an image +# that has what it needs: +# docker build --platform linux/amd64 --build-arg WITH_ASR=1 \ +# -t /granite-switch-systemone:-asr . # vLLM 0.30, the version this plugin is tested on live (granite-switch 0.1.1 supports # 0.26-0.30). The default tag is the CUDA 13 build (torch 2.13.0+cu130, the PyPI vLLM # wheel) -- the same stack the live tests ran. The -cu129 tag ships torch 2.14.0, @@ -21,12 +21,12 @@ FROM vllm/vllm-openai:v0.30.0 SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG GRANITE_SWITCH_VERSION=0.1.1 -ARG WITH_ASR=1 +ARG WITH_ASR=0 # Pinned to a specific, released granite-switch version (not `main`) so this image's # behavior does not drift when upstream changes -- deliberately a tagged PyPI release, # not a floating branch. transformers 5.17 is what granite-switch supports; [audio] is -# only installed when WITH_ASR=1, the default (it brings librosa and soundfile, used only by the +# only installed when WITH_ASR=1 (it brings librosa and soundfile, used only by the # ASR cascade). The base image's vLLM, torch, and torchcodec (CUDA 13 builds) are kept # as they are: the constraint file pins them so nothing in granite-switch's dependency # tree can move them. diff --git a/README.md b/README.md index 588c3a6..3dcb900 100644 --- a/README.md +++ b/README.md @@ -75,23 +75,27 @@ torch match what granite-switch declares and are the packages the live tests ran plus `granite-switch[vllm]==0.1.1`, `transformers==5.17.0` and this plugin. Weights are not baked in: mount a downloaded checkpoint at `/repository`, or set `MODEL_DIR`. -**Speech model.** The 8b and 30b checkpoints enable speech input and name no speech model, so -Granite Switch falls back to a default one (`ibm-granite/granite-speech-5.0-470m-turboctc`). -The image runs with the Hub offline, so the build bakes that model in (a pinned revision, about -1 GB). For a checkpoint with speech input off, or to supply the model from a mounted cache -(point `HF_HUB_CACHE` at it), build the slim image: +**Speech input.** The 8b and 30b checkpoints enable speech input, and Granite Switch then loads a +speech model while the engine starts. Decisions never use it, so `serve.sh` switches speech +input off (`--hf-overrides '{"asr_enabled": false}'`): the 8b and 30b start from the checkpoint +alone, with nothing else to download or mount. The same server's `/v1/chat/completions` then +takes no audio input. -```bash -docker build --platform linux/amd64 --build-arg WITH_ASR=0 -t granite-switch-systemone:slim . -``` +To keep speech input, set `SYSTEMONE_SPEECH_INPUT=1`. The image runs with the Hub offline, so +the speech model (`ibm-granite/granite-speech-5.0-470m-turboctc`, about 1 GB) must then be +available. Any one of: + +- build the image with it baked in (a pinned revision): + `docker build --platform linux/amd64 --build-arg WITH_ASR=1 -t granite-switch-systemone:asr .` +- mount a Hub cache that holds it and set `HF_HUB_CACHE` to that directory; +- set `HF_HUB_OFFLINE=0` so it downloads at startup. -With the slim image and these checkpoints, a server without the model stops during startup. -`serve.sh` checks this first and says what to do; setting `HF_HUB_OFFLINE=0` so the model -downloads at startup also works. +With `SYSTEMONE_SPEECH_INPUT=1`, `serve.sh` checks for the model first and, if it is missing, +stops with a message naming the model and these options. -**Status of the image: lightly tested.** It builds (amd64, with and without the speech model) and +**Status of the image: lightly tested.** It builds (amd64, with and without `WITH_ASR=1`) and the packages and plugin entry points were checked inside it. The default image was served on -H100s with only the checkpoint mounted, the 8b on one GPU and the 30b on two with tensor +H100s with only the checkpoint mounted and speech input off, the 8b on one GPU and the 30b on two with tensor parallelism, and the wire, robustness and accuracy checks passed. That was run under enroot, not `docker run` itself; see [docs/STATUS.md](docs/STATUS.md) for what was and was not run. The pip path above is what most of the live tests used. If you run the image, treat the first diff --git a/docs/STATUS.md b/docs/STATUS.md index f4accb9..9abb04e 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -28,9 +28,9 @@ and `ibm-granite/granite-switch-4.1-8b` unless a row names another checkpoint. | The 30b checkpoint | `granite-switch-4.1-30b` on vLLM 0.30.0, `--tensor-parallel-size 2` on two H100s: the full battery | pass | | Tensor parallelism | the 8b with `--tensor-parallel-size 2`: the full battery | pass | | Long contexts | the 8b with `--max-model-len 131072`, `scripts/longctx_check.py` (9 checks) | 9/9: a 98k-token request answered in one engine read with the danger-zone header and rejected without it; the 131,072 backstop and the default 32k/64k caps enforced; `/v1/decisions` has no header escape | -| Docker image build | `docker build --platform linux/amd64`, default (speech model baked in) and `WITH_ASR=0` (slim) variants | both build; the stack inside is package-identical to the live-tested one (PyPI vLLM 0.30.0, torch 2.13.0+cu130, transformers 5.17.0, granite-switch 0.1.1); entry points and imports verified in-container; the default image holds the pinned speech-model revision as its only snapshot, with `refs/main` pointing at it | -| Serving from the image | the default image (v0.1.2 code) under enroot (no Docker daemon on the GPU nodes), only the checkpoint mounted and no Hub cache: the 8b on one H100, then the 30b on two H100s with `--tensor-parallel-size 2` (set from the GPU count by `serve.sh`). On each: `e2e_check`, `probe_check`, `accuracy_check` | pass on both: 11/11 and 6/6, accuracy as on the pip path. The earlier `WITH_ASR=1` image on the 8b, and the plain image with a mounted speech cache, also passed | -| Slim image, no speech model | the `WITH_ASR=0` image with only the checkpoint mounted | does not start with the 8b: the checkpoint enables speech input and the offline cache lacks the default speech model. `serve.sh` now stops before vLLM with a message naming the model and the fixes (the default image, a mounted cache, or `HF_HUB_OFFLINE=0`) | +| Docker image build | `docker build --platform linux/amd64`, default and `WITH_ASR=1` variants | both build; the stack inside is package-identical to the live-tested one (PyPI vLLM 0.30.0, torch 2.13.0+cu130, transformers 5.17.0, granite-switch 0.1.1); entry points and imports verified in-container; the `WITH_ASR=1` image holds the pinned speech-model revision as its only snapshot, with `refs/main` pointing at it | +| Serving from the image | the default image (no speech model inside) under enroot (no Docker daemon on the GPU nodes), through `serve.sh`, only the checkpoint mounted, Hub offline: the 8b on one H100 and the 30b on two H100s (`--tensor-parallel-size 2`, set from the GPU count). On each: `e2e_check`, `probe_check`, `accuracy_check` | pass on both: 11/11 and 6/6, accuracy as on the pip path; the log shows `hf_overrides: {'asr_enabled': False}` and never names the speech model | +| Speech input on (`SYSTEMONE_SPEECH_INPUT=1`) | the 8b on the default image with the speech model supplied from a mounted Hub cache (`HF_HUB_CACHE`); and, as a negative control, the same switch with no speech model anywhere | the 8b starts and passes the same three checks; the control stops in `serve.sh` with the preflight message naming the model and the fixes, before vLLM starts. The `WITH_ASR=1` image (model baked in) was served on the 8b in an earlier run | | Decision Index runs | samples of the Decision Index suite (edition 0.3) through `/v1/systemone` at `--max-model-len 131072`, on the 8b and the 30b, as an end-to-end validation of the endpoint | completes with no errors; scores are not reported here while the methodology is verified, and the model card's published figures are the reference (see the caveats below) | ## Not tested diff --git a/docs/TESTING.md b/docs/TESTING.md index 05b4a81..742078b 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -95,11 +95,11 @@ on the old deployment and `--compare before.json` on the new one. control token inside the question turn. The usual cause is a wrong `SYSTEMONE_INVOCATION_MARKER` or `SYSTEMONE_ADAPTER_NAME`; the error names both. The probe logs nothing when it passes. -- Before `vllm serve` starts, `serve.sh` runs `python3 -m granite_switch_systemone.preflight`. If - the checkpoint enables speech input, the Hub is offline and the speech model is not in the - cache, it stops with a message naming the model and the ways to fix it, instead of failing - later inside the engine. The image sets `HF_HUB_OFFLINE=1`, so this applies to the slim - build (`--build-arg WITH_ASR=0`); the default build bakes the model in. +- `serve.sh` switches the checkpoint's speech input off by default, which decisions do not use, so + the 8b and 30b start without a speech model. With `SYSTEMONE_SPEECH_INPUT=1` it first runs + `python3 -m granite_switch_systemone.preflight`: if the Hub is offline and the speech model is not + in the cache, it stops with a message naming the model and the ways to fix it, instead of failing + later inside the engine. The image sets `HF_HUB_OFFLINE=1`. - Each question logs the probability mass the option labels hold. Below 0.9 it logs a warning (a wrong layout or marker makes the model want a token outside the labels); otherwise it logs at debug level. Answers are never refused on it. diff --git a/serve.sh b/serve.sh index de19887..d7f649d 100755 --- a/serve.sh +++ b/serve.sh @@ -6,10 +6,18 @@ # TENSOR_PARALLEL_SIZE default: every visible GPU # MAX_MODEL_LEN (32768), GPU_MEMORY_UTILIZATION (0.90), PORT (8000), SERVED_MODEL_NAME # SYSTEMONE_ADAPTER_NAME default "decision"; SYSTEMONE_INVOCATION_MARKER, see README +# SYSTEMONE_SPEECH_INPUT default 0: speech input is switched off, so no speech model is needed. +# 1 keeps the checkpoint's speech input; see README, "Speech input". set -e MODEL_DIR=${MODEL_DIR:-/repository} -# Fail early, with the cause, if the checkpoint needs a speech model the offline cache lacks. -python3 -m granite_switch_systemone.preflight "$MODEL_DIR" +# The 8b and 30b checkpoints enable speech input (asr_enabled) and granite-switch then loads a +# speech model at startup, which decisions never use. Off by default; with it on, fail early, +# with the cause, if the speech model is not in the offline cache. +case "${SYSTEMONE_SPEECH_INPUT:-0}" in + 0) SPEECH_ARGS=(--hf-overrides '{"asr_enabled": false}') ;; + 1) SPEECH_ARGS=(); python3 -m granite_switch_systemone.preflight "$MODEL_DIR" ;; + *) echo "SYSTEMONE_SPEECH_INPUT must be 0 or 1, not '$SYSTEMONE_SPEECH_INPUT'" >&2; exit 2 ;; +esac TP=${TENSOR_PARALLEL_SIZE:-$(python3 -c "import torch; print(max(torch.cuda.device_count(), 1))")} export SYSTEMONE_ADAPTER_NAME=${SYSTEMONE_ADAPTER_NAME:-decision} # --dtype bfloat16: composed checkpoints carry torch_dtype null (granite-switch#126). @@ -29,4 +37,5 @@ exec vllm serve "$MODEL_DIR" \ --max-logprobs 256 \ --enable-prompt-tokens-details \ --enable-auto-tool-choice --tool-call-parser granite4 \ + "${SPEECH_ARGS[@]}" \ "$@" diff --git a/src/granite_switch_systemone/preflight.py b/src/granite_switch_systemone/preflight.py index fd6c8e8..d7b46ce 100644 --- a/src/granite_switch_systemone/preflight.py +++ b/src/granite_switch_systemone/preflight.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 -"""Checks to run before `vllm serve` starts; `serve.sh` calls this module. +"""Checks to run before `vllm serve` starts; `serve.sh` calls this module when +SYSTEMONE_SPEECH_INPUT=1 (with speech input off the speech model is never loaded). Granite Switch's audio processor loads a speech model while the engine profiles memory. The 8b and 30b checkpoints enable it (`asr_enabled`) without naming a model, so granite-switch falls @@ -68,7 +69,8 @@ def main(argv: list[str]) -> int: f"granite-switch-systemone: this checkpoint enables speech input (asr_enabled in config.json) and needs\n" f'the speech model "{model_id}", but the Hub is offline and it is not in the cache at\n' f"{hub_cache_dir(os.environ)}. Do one of:\n" - " - use the default image (built without --build-arg WITH_ASR=0), which bakes the model in\n" + " - use an image built with --build-arg WITH_ASR=1, which bakes the model in\n" + " - leave speech input off (SYSTEMONE_SPEECH_INPUT unset or 0), which decisions do not need\n" " - mount a cache that holds it and point HF_HUB_CACHE at that directory\n" " - set HF_HUB_OFFLINE=0 so the model can be downloaded at startup", file=sys.stderr, diff --git a/tests/test_preflight.py b/tests/test_preflight.py index a0f7508..41eba94 100644 --- a/tests/test_preflight.py +++ b/tests/test_preflight.py @@ -105,7 +105,8 @@ def test_main_exits_1_and_says_what_to_do(tmp_path, monkeypatch, capsys): err = capsys.readouterr().err assert DEFAULT in err assert str(tmp_path / "cache") in err - assert "WITH_ASR=0" in err + assert "WITH_ASR=1" in err + assert "SYSTEMONE_SPEECH_INPUT" in err assert "HF_HUB_OFFLINE=0" in err diff --git a/tests/test_serve_sh.py b/tests/test_serve_sh.py new file mode 100644 index 0000000..71e5a55 --- /dev/null +++ b/tests/test_serve_sh.py @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: Apache-2.0 +"""No-GPU checks for serve.sh's speech-input switch, with a stub `vllm` that records its arguments. + +python3 -m pytest tests/test_serve_sh.py -v --tb=short +""" + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +SERVE_SH = Path(__file__).resolve().parent.parent / "serve.sh" +OVERRIDE = '{"asr_enabled": false}' + + +def _run(tmp_path: Path, extra_env: dict[str, str], *args: str, asr_enabled: bool = True): + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + stub = bin_dir / "vllm" + stub.write_text('#!/bin/sh\nprintf "%s\\n" "$@" > "$STUB_OUT"\n') + stub.chmod(0o755) + model_dir = tmp_path / "model" + model_dir.mkdir() + (model_dir / "config.json").write_text(json.dumps({"asr_enabled": asr_enabled})) + out = tmp_path / "args.txt" + env = { + "PATH": f"{bin_dir}{os.pathsep}{Path(sys.executable).parent}{os.pathsep}{os.environ['PATH']}", + "MODEL_DIR": str(model_dir), + "TENSOR_PARALLEL_SIZE": "1", + "STUB_OUT": str(out), + "HF_HUB_OFFLINE": "1", + "HF_HUB_CACHE": str(tmp_path / "empty-cache"), + **extra_env, + } + done = subprocess.run(["bash", str(SERVE_SH), *args], env=env, capture_output=True, text=True) + return done, (out.read_text().splitlines() if out.exists() else []) + + +def test_speech_input_is_off_by_default_and_needs_no_speech_model(tmp_path): + done, argv = _run(tmp_path, {}) + assert done.returncode == 0, done.stderr + assert argv[argv.index("--hf-overrides") + 1] == OVERRIDE + + +def test_speech_input_on_without_the_model_stops_before_vllm(tmp_path): + done, argv = _run(tmp_path, {"SYSTEMONE_SPEECH_INPUT": "1"}) + assert done.returncode == 1 + assert argv == [] # vllm never started + assert "WITH_ASR=1" in done.stderr + + +def test_speech_input_on_with_the_model_cached_starts_without_the_override(tmp_path): + cache = tmp_path / "empty-cache" / "models--ibm-granite--granite-speech-5.0-470m-turboctc" / "snapshots" / "abc" + cache.mkdir(parents=True) + (cache / "config.json").write_text("{}") + done, argv = _run(tmp_path, {"SYSTEMONE_SPEECH_INPUT": "1"}) + assert done.returncode == 0, done.stderr + assert "--hf-overrides" not in argv + + +def test_extra_arguments_still_reach_vllm(tmp_path): + done, argv = _run(tmp_path, {}, "--api-key", "k") + assert done.returncode == 0, done.stderr + assert argv[-2:] == ["--api-key", "k"] + + +@pytest.mark.parametrize("value", ["2", "yes", ""]) +def test_bad_switch_value_is_refused(tmp_path, value): + done, argv = _run(tmp_path, {"SYSTEMONE_SPEECH_INPUT": value}) + if value == "": + assert done.returncode == 0 # empty falls back to the default, off + else: + assert done.returncode == 2 + assert argv == []