From 0d3d3651f4df0c62f547d558917ef8eb99cd6e9c Mon Sep 17 00:00:00 2001 From: Nigel Jones Date: Fri, 23 Oct 2026 17:12:37 +0100 Subject: [PATCH 01/11] feat: publish a default and an -asr image on each release The image workflow now builds two variants from a matrix: the default (decisions only, no speech model) and an -asr variant built with WITH_ASR=1 so chat speech input works without extra setup. Tags are X.Y.Z and latest for the default, X.Y.Z-asr and latest-asr for the asr image, so the two never push the same tag. Assisted-by: Claude Code Signed-off-by: Nigel Jones --- .github/workflows/image.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index 4a2ceb0..c1cf15f 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -18,6 +18,21 @@ permissions: jobs: image: + # Two images per release: the default (decisions only, no speech model inside) and an + # `-asr` variant with the pinned speech model baked in, for chat speech input. + name: image (${{ matrix.variant }}) + strategy: + fail-fast: false + matrix: + include: + # `latest` stays the default image; the asr image gets `latest-asr` (onlatest=true), + # so the two variants never push the same tag. + - variant: default + with_asr: "0" + flavor: "latest=auto" + - variant: asr + with_asr: "1" + flavor: "suffix=-asr,onlatest=true" runs-on: ubuntu-latest permissions: contents: read @@ -45,6 +60,7 @@ jobs: uses: docker/metadata-action@v6 with: images: ${{ env.IMAGE }} + flavor: ${{ matrix.flavor }} tags: | type=semver,pattern={{version}} type=ref,event=branch @@ -53,6 +69,7 @@ jobs: with: context: . platforms: linux/amd64 + build-args: WITH_ASR=${{ matrix.with_asr }} push: ${{ env.PUSH == 'true' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From ec31ccef9ff3b3aabffead8eb74fc38b9f90c852 Mon Sep 17 00:00:00 2001 From: Nigel Jones Date: Fri, 23 Oct 2026 17:12:40 +0100 Subject: [PATCH 02/11] docs: document the build arguments and the published image Describe GRANITE_SWITCH_VERSION, WITH_ASR, ASR_MODEL and ASR_REVISION, and replace the 'no supported image is published' line with the ghcr image name and tags. The package is currently private, so the README says login is needed until it is made public. Assisted-by: Claude Code Signed-off-by: Nigel Jones --- README.md | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 6e797e1..c8b1897 100644 --- a/README.md +++ b/README.md @@ -48,24 +48,27 @@ listed, and once the variable is set it loads nothing else, so leaving out ### Docker -No supported image is published, so you build it yourself (it is large, about 30 GB). On a Linux host -with an NVIDIA GPU, the NVIDIA container toolkit and Docker: +Release images are published to GitHub Container Registry as +`ghcr.io/generative-computing/granite-switch-systemone` (linux/amd64 only; about 30 GB, so the first +pull takes a while). `latest` is the newest release and each release also has its version as a tag. +The package is currently private, so pulling needs `docker login ghcr.io` with an account that has +access; until it is made public, build the image yourself (below). On a Linux host with an NVIDIA +GPU, the NVIDIA container toolkit and Docker: ```bash -# 1. Get the code at a release tag -git clone --branch v0.1.3 https://github.com/generative-computing/granite-switch-systemone -cd granite-switch-systemone - -# 2. Download the checkpoint (public, not gated; about 18 GB) +# 1. Download the checkpoint (public, not gated; about 18 GB) hf download ibm-granite/granite-switch-4.1-8b --local-dir /srv/models/granite-switch-4.1-8b -# 3. Build and run -docker build --platform linux/amd64 -t granite-switch-systemone . +# 2. Run the image docker run -d --name gss --gpus all --ipc=host -p 8000:8000 \ -v /srv/models/granite-switch-4.1-8b:/repository:ro \ - granite-switch-systemone + ghcr.io/generative-computing/granite-switch-systemone:0.1.3 ``` +To build the image yourself instead, clone the repository at a release tag and run +`docker build --platform linux/amd64 -t granite-switch-systemone .` in it, then use that name in +`docker run`. + Then check it ([Verify the deployment](#verify-the-deployment)). Model loading takes a few minutes; follow it with `docker logs -f gss`. `--ipc=host` is vLLM's standard setting for shared memory; it matters most with `--tensor-parallel-size` above 1. @@ -75,6 +78,14 @@ torch match what granite-switch declares and are the packages the live tests ran plus `granite-switch[vllm]==0.1.1`, `transformers==5.17.0` and this plugin. Weights are not baked in: mount a downloaded checkpoint at `/repository`, or set `MODEL_DIR`. +**Build arguments.** `GRANITE_SWITCH_VERSION` (default `0.1.1`) is the granite-switch release +installed. `WITH_ASR=1` adds the speech extra and bakes in the speech model. `ASR_MODEL` and +`ASR_REVISION` name the Hub repository and the commit that is baked in; the default commit is the +one the image was tested with, so a rebuild does not change when the model repository moves. To +bake in a different speech model commit, pass +`--build-arg WITH_ASR=1 --build-arg ASR_REVISION=`. The checkpoint is never a build +argument: it is mounted when the container runs. + **Speech input.** The 8b and 30b checkpoints enable speech input, and Granite Switch then loads a speech model while the engine starts. Decisions never use it, so `serve.sh` switches speech input off (`--hf-overrides '{"asr_enabled": false}'`): the 8b and 30b start from the checkpoint @@ -94,7 +105,7 @@ With `SYSTEMONE_SPEECH_INPUT=1`, `serve.sh` checks for the model first and, if i stops with a message naming the model and these options. **Status of the image: lightly tested.** It builds (amd64, with and without `WITH_ASR=1`) and -the packages and plugin entry points were checked inside it. The default image was served on +the packages and plugin entry points were checked inside it. The published `0.1.3` image was served on H100s with only the checkpoint mounted and speech input off, the 8b on one GPU and the 30b on two with tensor parallelism, and the wire, robustness and accuracy checks passed. That was run under enroot, not `docker run` itself; see [docs/STATUS.md](docs/STATUS.md) for what was and was not run. The From 82a2670866502b73838ed4ff27c7fda0742df0c7 Mon Sep 17 00:00:00 2001 From: Nigel Jones Date: Fri, 23 Oct 2026 19:01:37 +0100 Subject: [PATCH 03/11] fix: install the audio libraries in every image and check for them at startup The default image lacked av, scipy, soundfile and soxr, so with SYSTEMONE_SPEECH_INPUT=1 and a mounted speech model the server started and passed /health, then answered every audio request with HTTP 500 "Please install vllm[audio]". The image now installs granite-switch[audio] in every variant (WITH_ASR=1 still only adds the baked-in speech model), and the speech preflight names any missing audio library and stops before vllm starts, for installs outside the image. Assisted-by: Claude Code Signed-off-by: Nigel Jones --- Dockerfile | 25 +++++++++-------- README.md | 8 ++++-- src/granite_switch_systemone/preflight.py | 30 ++++++++++++++++++++ tests/test_preflight.py | 34 +++++++++++++++++++++++ tests/test_serve_sh.py | 11 ++++++++ 5 files changed, 93 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index 28aa5dc..38707f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,10 +6,10 @@ # # docker build --platform linux/amd64 -t /granite-switch-systemone: . # -# Optional speech input (granite-switch[audio], and a specific speech model's weights baked in -# at a pinned revision -- see ASR_MODEL/ASR_REVISION below). Decisions do not use it, so -# serve.sh switches it off unless SYSTEMONE_SPEECH_INPUT=1; build with WITH_ASR=1 for an image -# that has what it needs: +# Optional speech input. The audio libraries are in every variant; WITH_ASR=1 also bakes in a +# specific speech model's weights at a pinned revision (see ASR_MODEL/ASR_REVISION below). +# Decisions do not use speech, so serve.sh switches it off unless SYSTEMONE_SPEECH_INPUT=1; +# build with WITH_ASR=1 for an image that has the speech model too: # docker build --platform linux/amd64 --build-arg WITH_ASR=1 \ # -t /granite-switch-systemone:-asr . # vLLM 0.30, the version this plugin is tested on live (granite-switch 0.1.1 supports @@ -25,17 +25,18 @@ ARG WITH_ASR=0 # Pinned to a specific, released granite-switch version (not `main`) so this image's # behavior does not drift when upstream changes -- deliberately a tagged PyPI release, -# not a floating branch. transformers 5.17 is what granite-switch supports; [audio] is -# only installed when WITH_ASR=1 (it brings librosa and soundfile, used only by the -# ASR cascade). The base image's vLLM, torch, and torchcodec (CUDA 13 builds) are kept -# as they are: the constraint file pins them so nothing in granite-switch's dependency -# tree can move them. +# not a floating branch. transformers 5.17 is what granite-switch supports. The [audio] +# extra (av, scipy, soundfile, soxr: the libraries vLLM needs to decode audio) is +# installed in every variant, so SYSTEMONE_SPEECH_INPUT=1 works with a mounted speech +# model; without them every audio request fails with HTTP 500 "install vllm[audio]" while +# health and decisions look fine. WITH_ASR=1 only adds the baked-in speech model below. +# The base image's vLLM, torch, and torchcodec (CUDA 13 builds) are kept as they are: the +# constraint file pins them so nothing in granite-switch's dependency tree can move them. RUN python3 -m pip list --format=freeze | grep -i -E "^(vllm|torch|torchaudio|torchcodec|torchvision|flashinfer-python)==" > /tmp/keep.txt \ - && if [ "$WITH_ASR" = "1" ]; then EXTRA="granite-switch[audio]"; else EXTRA="granite-switch[vllm]"; fi \ && uv pip install --system --constraint /tmp/keep.txt \ - "${EXTRA}==${GRANITE_SWITCH_VERSION}" \ + "granite-switch[audio]==${GRANITE_SWITCH_VERSION}" \ "transformers==5.17.0" \ - && python3 -c "import granite_switch.vllm, vllm, transformers, torch; print(vllm.__version__, transformers.__version__, torch.__version__)" + && python3 -c "import granite_switch.vllm, vllm, transformers, torch, av, scipy, soundfile, soxr; print(vllm.__version__, transformers.__version__, torch.__version__)" # The ASR cascade's speech model, named by Hugging Face id in the checkpoint's config, # baked in as a Hub cache at the revision this image was tested with, without the diff --git a/README.md b/README.md index c8b1897..a0b3c9a 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ plus `granite-switch[vllm]==0.1.1`, `transformers==5.17.0` and this plugin. Weig not baked in: mount a downloaded checkpoint at `/repository`, or set `MODEL_DIR`. **Build arguments.** `GRANITE_SWITCH_VERSION` (default `0.1.1`) is the granite-switch release -installed. `WITH_ASR=1` adds the speech extra and bakes in the speech model. `ASR_MODEL` and +installed, with its `audio` extra in every variant. `WITH_ASR=1` bakes in the speech model. `ASR_MODEL` and `ASR_REVISION` name the Hub repository and the commit that is baked in; the default commit is the one the image was tested with, so a rebuild does not change when the model repository moves. To bake in a different speech model commit, pass @@ -102,8 +102,10 @@ available. Any one of: - set `HF_HUB_OFFLINE=0` so it downloads at startup. With `SYSTEMONE_SPEECH_INPUT=1`, `serve.sh` checks for the model first and, if it is missing, -stops with a message naming the model and these options. - +stops with a message naming the model and these options. It also checks that the audio libraries +(`av`, `scipy`, `soundfile`, `soxr`) are installed. The release images include them. If you run +the plugin outside the image, install `vllm[audio]` too: without it the server starts and passes +`/health`, then answers every audio request with HTTP 500 "Please install vllm[audio]". **Status of the image: lightly tested.** It builds (amd64, with and without `WITH_ASR=1`) and the packages and plugin entry points were checked inside it. The published `0.1.3` image was served on H100s with only the checkpoint mounted and speech input off, the 8b on one GPU and the 30b on two with tensor diff --git a/src/granite_switch_systemone/preflight.py b/src/granite_switch_systemone/preflight.py index d7b46ce..0ee844e 100644 --- a/src/granite_switch_systemone/preflight.py +++ b/src/granite_switch_systemone/preflight.py @@ -13,12 +13,19 @@ from __future__ import annotations +import importlib.util import json import os import sys from collections.abc import Mapping from pathlib import Path +# What vLLM's `audio` extra needs to decode an audio request. Without them the server starts and +# passes /health, then answers every audio request with HTTP 500 "install vllm[audio]". +AUDIO_MODULES = ("av", "scipy", "soundfile", "soxr") +# Test hook: a comma-separated list of the modules to treat as installed (an empty value checks nothing). +_AUDIO_MODULES_ENV = "SYSTEMONE_PREFLIGHT_AUDIO_MODULES" + # granite-switch 0.1.1's DEFAULT_ASR_MODEL_ID (granite_switch/vllm/audio/asr.py), used when the # checkpoint's `asr_model_id` is null. tests/test_preflight.py compares it with the installed copy. DEFAULT_ASR_MODEL_ID = "ibm-granite/granite-speech-5.0-470m-turboctc" @@ -58,10 +65,33 @@ def missing_asr_model(model_dir: Path, env: Mapping[str, str]) -> str | None: return None if _cached(model_id, hub_cache_dir(env)) else model_id +def missing_audio_modules(model_dir: Path) -> list[str]: + """The audio libraries a speech-enabled checkpoint needs but this environment lacks.""" + config_path = model_dir / "config.json" + if not config_path.is_file() or not json.loads(config_path.read_text()).get("asr_enabled"): + return [] # not a local checkpoint, or speech input is off in it + required = AUDIO_MODULES + if _AUDIO_MODULES_ENV in os.environ: + required = tuple(n for n in os.environ[_AUDIO_MODULES_ENV].split(",") if n) + return [name for name in required if importlib.util.find_spec(name) is None] + + def main(argv: list[str]) -> int: if len(argv) != 1: print("usage: python3 -m granite_switch_systemone.preflight MODEL_DIR", file=sys.stderr) return 2 + modules = missing_audio_modules(Path(argv[0])) + if modules: + print( + "granite-switch-systemone: speech input is on, but these audio libraries are not installed: " + f"{', '.join(modules)}.\n" + "Without them the server starts and passes /health, then answers every audio request with " + 'HTTP 500 "Please install vllm[audio]". Install vllm[audio] (or granite-switch[audio]) into ' + "this environment, or use a release image, which includes them. Leave SYSTEMONE_SPEECH_INPUT " + "unset or 0 if you do not need audio.", + file=sys.stderr, + ) + return 1 model_id = missing_asr_model(Path(argv[0]), os.environ) if model_id is None: return 0 diff --git a/tests/test_preflight.py b/tests/test_preflight.py index 41eba94..ae6992a 100644 --- a/tests/test_preflight.py +++ b/tests/test_preflight.py @@ -99,6 +99,7 @@ def test_hub_cache_dir_follows_huggingface_hub_precedence(env, expected): def test_main_exits_1_and_says_what_to_do(tmp_path, monkeypatch, capsys): model_dir = _checkpoint(tmp_path, asr_enabled=True, asr_model_id=None) + monkeypatch.setenv("SYSTEMONE_PREFLIGHT_AUDIO_MODULES", "") # this test is about the speech model monkeypatch.setenv("HF_HUB_OFFLINE", "1") monkeypatch.setenv("HF_HUB_CACHE", str(tmp_path / "cache")) assert preflight.main([str(model_dir)]) == 1 @@ -110,6 +111,39 @@ def test_main_exits_1_and_says_what_to_do(tmp_path, monkeypatch, capsys): assert "HF_HUB_OFFLINE=0" in err +def test_audio_modules_missing_are_named_when_speech_is_on(tmp_path, monkeypatch): + model_dir = _checkpoint(tmp_path, asr_enabled=True) + monkeypatch.setattr( + preflight.importlib.util, "find_spec", lambda name: None if name in {"av", "soxr"} else object() + ) + assert preflight.missing_audio_modules(model_dir) == ["av", "soxr"] + + +def test_audio_modules_are_not_checked_when_the_checkpoint_has_speech_off(tmp_path, monkeypatch): + model_dir = _checkpoint(tmp_path, asr_enabled=False) + monkeypatch.setattr(preflight.importlib.util, "find_spec", lambda name: None) + assert preflight.missing_audio_modules(model_dir) == [] + + +def test_audio_modules_are_not_checked_without_a_local_config(tmp_path, monkeypatch): + monkeypatch.setattr(preflight.importlib.util, "find_spec", lambda name: None) + assert preflight.missing_audio_modules(tmp_path) == [] + + +def test_main_exits_1_and_names_the_missing_audio_libraries(tmp_path, monkeypatch, capsys): + model_dir = _checkpoint(tmp_path, asr_enabled=True, asr_model_id=None) + cache = tmp_path / "cache" + _cache_model(cache, DEFAULT) # the speech model is fine; only the libraries are missing + monkeypatch.setenv("HF_HUB_OFFLINE", "1") + monkeypatch.setenv("HF_HUB_CACHE", str(cache)) + monkeypatch.setattr(preflight.importlib.util, "find_spec", lambda name: None if name == "soundfile" else object()) + assert preflight.main([str(model_dir)]) == 1 + err = capsys.readouterr().err + assert "soundfile" in err + assert "HTTP 500" in err + assert "vllm[audio]" in err + + def test_main_exits_0_quietly_when_nothing_is_missing(tmp_path, monkeypatch, capsys): model_dir = _checkpoint(tmp_path, asr_enabled=False) monkeypatch.setenv("HF_HUB_OFFLINE", "1") diff --git a/tests/test_serve_sh.py b/tests/test_serve_sh.py index 71e5a55..17ebeca 100644 --- a/tests/test_serve_sh.py +++ b/tests/test_serve_sh.py @@ -33,6 +33,7 @@ def _run(tmp_path: Path, extra_env: dict[str, str], *args: str, asr_enabled: boo "STUB_OUT": str(out), "HF_HUB_OFFLINE": "1", "HF_HUB_CACHE": str(tmp_path / "empty-cache"), + "SYSTEMONE_PREFLIGHT_AUDIO_MODULES": "", # these tests are about the speech model, not the audio libraries **extra_env, } done = subprocess.run(["bash", str(SERVE_SH), *args], env=env, capture_output=True, text=True) @@ -61,6 +62,16 @@ def test_speech_input_on_with_the_model_cached_starts_without_the_override(tmp_p assert "--hf-overrides" not in argv +def test_speech_input_on_without_the_audio_libraries_stops_before_vllm(tmp_path): + done, argv = _run( + tmp_path, {"SYSTEMONE_SPEECH_INPUT": "1", "SYSTEMONE_PREFLIGHT_AUDIO_MODULES": "no_such_audio_module_xyz"} + ) + assert done.returncode == 1 + assert argv == [] # vllm never started + assert "no_such_audio_module_xyz" in done.stderr + assert "HTTP 500" in done.stderr + + def test_extra_arguments_still_reach_vllm(tmp_path): done, argv = _run(tmp_path, {}, "--api-key", "k") assert done.returncode == 0, done.stderr From 0e075871e621059280168af8001fb70ef3eb695e Mon Sep 17 00:00:00 2001 From: Nigel Jones Date: Fri, 23 Oct 2026 19:01:41 +0100 Subject: [PATCH 04/11] feat: bake in the multilingual speech model revision Move the default ASR_REVISION to the multilingual TurboCTC release (English, French, German, Spanish, Portuguese, Italian and Japanese, with punctuation and capitalization). The README says how to build with the earlier English-only revision instead. Assisted-by: Claude Code Signed-off-by: Nigel Jones --- Dockerfile | 2 +- README.md | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 38707f7..3e6c9dd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,7 +46,7 @@ RUN python3 -m pip list --format=freeze | grep -i -E "^(vllm|torch|torchaudio|to # the model repository moves. A download by commit writes no refs/main, and the offline lookup # for the revision "main" reads that file, so it is written here and then checked. ARG ASR_MODEL=ibm-granite/granite-speech-5.0-470m-turboctc -ARG ASR_REVISION=947f59af40db9791170a0628cf0f3f4812d720f1 +ARG ASR_REVISION=1dd23f46f6d4e6eae2531982ba2e33220086dd97 RUN if [ "$WITH_ASR" = "1" ]; then \ hf download ${ASR_MODEL} --revision ${ASR_REVISION} --cache-dir /opt/hf-cache --exclude "*.py" > /dev/null \ && mkdir -p /opt/hf-cache/models--${ASR_MODEL//\//--}/refs \ diff --git a/README.md b/README.md index a0b3c9a..70a8899 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,12 @@ stops with a message naming the model and these options. It also checks that the (`av`, `scipy`, `soundfile`, `soxr`) are installed. The release images include them. If you run the plugin outside the image, install `vllm[audio]` too: without it the server starts and passes `/health`, then answers every audio request with HTTP 500 "Please install vllm[audio]". + +The speech model in the `-asr` image is the multilingual release (English, French, German, Spanish, +Portuguese, Italian and Japanese, with punctuation and capitalization for the Latin-script +languages in our tests). Pass `--build-arg ASR_REVISION=947f59af40db9791170a0628cf0f3f4812d720f1` +to bake in the earlier English-only revision instead. + **Status of the image: lightly tested.** It builds (amd64, with and without `WITH_ASR=1`) and the packages and plugin entry points were checked inside it. The published `0.1.3` image was served on H100s with only the checkpoint mounted and speech input off, the 8b on one GPU and the 30b on two with tensor From 50ac5cf272bc58953de5b1b5258c4c7c3ba89a80 Mon Sep 17 00:00:00 2001 From: Nigel Jones Date: Fri, 23 Oct 2026 19:24:49 +0100 Subject: [PATCH 05/11] docs: rewrite the README for people deploying the plugin Reorder it so a reader gets a working server first: a quick start for the container, an existing vLLM install and Hugging Face Inference Endpoints, a first request, then securing it, speech input, configuration, behaviour and limits. Remove restated internal detail, keep every tested fact and command, and say plainly what has not been tried (Inference Endpoints). Assisted-by: Claude Code Signed-off-by: Nigel Jones --- README.md | 427 +++++++++++++++++++++++++++--------------------------- 1 file changed, 214 insertions(+), 213 deletions(-) diff --git a/README.md b/README.md index 70a8899..c0cf97b 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,58 @@ # granite-switch-systemone -A vLLM endpoint plugin that serves the decision adapter of a -[Granite Switch](https://github.com/generative-computing/granite-switch) checkpoint through -two decision APIs, on the same server and GPU cache as `/v1/chat/completions`: +Decision APIs for [Granite Switch](https://github.com/generative-computing/granite-switch), as a +vLLM plugin. Ask a question about some text and get back probabilities, from the same server and +GPU cache that already serves `/v1/chat/completions`. -- `POST /v1/decisions`: [OpenAI's Decisions API](https://developers.openai.com/api/docs/guides/decisions). - The official OpenAI SDKs work by changing `base_url`. -- `POST /v1/systemone`: [jev's system-one API](https://docs.typesafe.ai/api). +- `POST /v1/decisions` follows [OpenAI's Decisions API](https://developers.openai.com/api/docs/guides/decisions). + The official OpenAI SDKs work once you change `base_url`. +- `POST /v1/systemone` follows [jev's system-one API](https://docs.typesafe.ai/api). -Each question is answered by one engine read of the adapter's single-token option labels, -returned as softmax probabilities, with confidence computed as in jev's examples. Callers need no prompt format, marker or label scheme. +Each question is answered with one engine read of the adapter's single-token option labels, so +callers do not need a prompt format, an invocation marker or a label scheme. -What has and has not been tested is in [docs/STATUS.md](docs/STATUS.md); how to run -the tests is in [docs/TESTING.md](docs/TESTING.md). +You can run it as a container or add it to a vLLM install you already have. Start with +[Quick start](#quick-start). What has and has not been tested is in +[docs/STATUS.md](docs/STATUS.md), and how to run the tests is in [docs/TESTING.md](docs/TESTING.md). -## Install +## Quick start -`v0.1.3` is the current release. Pick one path: +You need a Linux machine with an NVIDIA GPU. The 8b checkpoint fits on one 80 GB GPU. -- **You already run vLLM:** install the plugin into that environment, below. This is the path tested most. -- **You want a container:** build the image from this repository, further below. It is - lightly tested (see the note there), so prefer the first path if you can. +### Container -### Into an existing vLLM environment (simplest) +Release images are published to GitHub Container Registry as +`ghcr.io/generative-computing/granite-switch-systemone` for linux/amd64 only. `latest` is the newest +release, and each release is also tagged with its version. The pull is about 9 GB and the image takes +about 30 GB on disk. You also need Docker and the NVIDIA container toolkit. + +```bash +# Download the checkpoint (public, not gated, about 18 GB) +hf download ibm-granite/granite-switch-4.1-8b --local-dir /srv/models/granite-switch-4.1-8b + +# Run the image +docker run -d --name gss --gpus all --ipc=host -p 8000:8000 \ + -v /srv/models/granite-switch-4.1-8b:/repository:ro \ + ghcr.io/generative-computing/granite-switch-systemone:0.1.3 +``` + +Loading takes a few minutes; follow it with `docker logs -f gss`. `--ipc=host` is vLLM's usual +setting for shared memory, and it matters most when you run with tensor parallelism. -Prerequisites: +The ghcr package is currently private, so `docker pull` needs `docker login ghcr.io` with an account +that has access. Until it is public, build the image yourself: clone this repository at a release tag +and run `docker build --platform linux/amd64 -t granite-switch-systemone .`. -- Linux with an NVIDIA GPU; the 8b fits on one 80 GB H100. -- Python 3.11+, PyTorch 2.11+ (granite-switch requires it; CUDA 13 builds tested). -- **vLLM 0.30.0 is the tested version.** Use it unless you have a reason not to. granite-switch - 0.1.1 pins `vllm>=0.26,<0.31`, and `v0.1.0` was also served live on 0.26 to 0.29, but changes - since then have been checked on 0.30.0 only. Treat 0.26 to 0.29 as unverified for this - version (see [docs/STATUS.md](docs/STATUS.md)). +The image is `vllm/vllm-openai:v0.30.0` plus `granite-switch[audio]==0.1.1`, `transformers==5.17.0` and +this plugin. It has no weights in it. It reads the checkpoint from `/repository` (override with +`MODEL_DIR`), so the same image serves the 8b and the 30b. We tested the 30b on two 80 GB GPUs; the +container uses every GPU it can see unless you set `TENSOR_PARALLEL_SIZE`. + +### Existing vLLM environment + +You need Python 3.11 or later and PyTorch 2.11 or later (CUDA 13 builds are tested). **Use vLLM +0.30.0**: it is the version the live tests ran on. granite-switch 0.1.1 accepts `vllm>=0.26,<0.31`, +and `v0.1.0` was also served on 0.26 to 0.29, but nothing since has been checked there. ```bash pip install "granite-switch[vllm]==0.1.1" "transformers>=5.16" @@ -42,245 +63,225 @@ vllm serve ibm-granite/granite-switch-4.1-8b --served-model-name granite-switch --dtype bfloat16 --attention-config.flash_attn_version=2 --enable-prompt-tokens-details ``` -`VLLM_PLUGINS` must name both plugins. vLLM loads endpoint plugins only when they are -listed, and once the variable is set it loads nothing else, so leaving out -`register_granite_switch` stops the Switch model class from registering. +List both plugins in `VLLM_PLUGINS`. vLLM loads endpoint plugins only when they are named, and once +the variable is set it loads nothing else, so leaving out `register_granite_switch` stops the Switch +model class from registering. -### Docker +### Hugging Face Inference Endpoints -Release images are published to GitHub Container Registry as -`ghcr.io/generative-computing/granite-switch-systemone` (linux/amd64 only; about 30 GB, so the first -pull takes a while). `latest` is the newest release and each release also has its version as a tag. -The package is currently private, so pulling needs `docker login ghcr.io` with an account that has -access; until it is made public, build the image yourself (below). On a Linux host with an NVIDIA -GPU, the NVIDIA container toolkit and Docker: +Create an endpoint for the model repository and choose a custom container. Hugging Face's +[custom container guide](https://huggingface.co/docs/inference-endpoints/en/guides/custom_container) +says the model weights are stored under `/repository`, which is where this image looks. The image +serves on port `8000` and answers `/health`, so those are the values for the endpoint's container port +and health route. The guide lists public images and private ones on Docker Hub, AWS ECR, Azure ACR +and Google GCR; GitHub Container Registry is not in that list, so the package has to be public. +**We have not tried this on Inference Endpoints yet.** -```bash -# 1. Download the checkpoint (public, not gated; about 18 GB) -hf download ibm-granite/granite-switch-4.1-8b --local-dir /srv/models/granite-switch-4.1-8b +### Check that it works -# 2. Run the image -docker run -d --name gss --gpus all --ipc=host -p 8000:8000 \ - -v /srv/models/granite-switch-4.1-8b:/repository:ro \ - ghcr.io/generative-computing/granite-switch-systemone:0.1.3 +```bash +curl -fsS http://localhost:8000/health && echo healthy +curl -fsS http://localhost:8000/v1/models # lists "granite-switch" +python3 scripts/e2e_check.py --base-url http://localhost:8000 ``` -To build the image yourself instead, clone the repository at a release tag and run -`docker build --platform linux/amd64 -t granite-switch-systemone .` in it, then use that name in -`docker run`. - -Then check it ([Verify the deployment](#verify-the-deployment)). Model loading takes a few -minutes; follow it with `docker logs -f gss`. `--ipc=host` is vLLM's standard setting for -shared memory; it matters most with `--tensor-parallel-size` above 1. - -The image is `vllm/vllm-openai:v0.30.0` (the default CUDA 13 tag, whose vLLM wheel and -torch match what granite-switch declares and are the packages the live tests ran) -plus `granite-switch[vllm]==0.1.1`, `transformers==5.17.0` and this plugin. Weights are -not baked in: mount a downloaded checkpoint at `/repository`, or set `MODEL_DIR`. - -**Build arguments.** `GRANITE_SWITCH_VERSION` (default `0.1.1`) is the granite-switch release -installed, with its `audio` extra in every variant. `WITH_ASR=1` bakes in the speech model. `ASR_MODEL` and -`ASR_REVISION` name the Hub repository and the commit that is baked in; the default commit is the -one the image was tested with, so a rebuild does not change when the model repository moves. To -bake in a different speech model commit, pass -`--build-arg WITH_ASR=1 --build-arg ASR_REVISION=`. The checkpoint is never a build -argument: it is mounted when the container runs. - -**Speech input.** The 8b and 30b checkpoints enable speech input, and Granite Switch then loads a -speech model while the engine starts. Decisions never use it, so `serve.sh` switches speech -input off (`--hf-overrides '{"asr_enabled": false}'`): the 8b and 30b start from the checkpoint -alone, with nothing else to download or mount. The same server's `/v1/chat/completions` then -takes no audio input. - -To keep speech input, set `SYSTEMONE_SPEECH_INPUT=1`. The image runs with the Hub offline, so -the speech model (`ibm-granite/granite-speech-5.0-470m-turboctc`, about 1 GB) must then be -available. Any one of: - -- build the image with it baked in (a pinned revision): - `docker build --platform linux/amd64 --build-arg WITH_ASR=1 -t granite-switch-systemone:asr .` -- mount a Hub cache that holds it and set `HF_HUB_CACHE` to that directory; -- set `HF_HUB_OFFLINE=0` so it downloads at startup. - -With `SYSTEMONE_SPEECH_INPUT=1`, `serve.sh` checks for the model first and, if it is missing, -stops with a message naming the model and these options. It also checks that the audio libraries -(`av`, `scipy`, `soundfile`, `soxr`) are installed. The release images include them. If you run -the plugin outside the image, install `vllm[audio]` too: without it the server starts and passes -`/health`, then answers every audio request with HTTP 500 "Please install vllm[audio]". - -The speech model in the `-asr` image is the multilingual release (English, French, German, Spanish, -Portuguese, Italian and Japanese, with punctuation and capitalization for the Latin-script -languages in our tests). Pass `--build-arg ASR_REVISION=947f59af40db9791170a0628cf0f3f4812d720f1` -to bake in the earlier English-only revision instead. - -**Status of the image: lightly tested.** It builds (amd64, with and without `WITH_ASR=1`) and -the packages and plugin entry points were checked inside it. The published `0.1.3` image was served on -H100s with only the checkpoint mounted and speech input off, the 8b on one GPU and the 30b on two with tensor -parallelism, and the wire, robustness and accuracy checks passed. That was run under enroot, -not `docker run` itself; see [docs/STATUS.md](docs/STATUS.md) for what was and was not run. The -pip path above is what most of the live tests used. If you run the image, treat the first -start as the test and report what you find. - -## Verify the deployment +`e2e_check.py` needs only the standard library and works against any host, so run it from a checkout of +this repository. It exits 0 when the wire formats on both routes, route parity, chat and prefix-cache +sharing all pass. If the server stops at startup, its log names the cause: an unknown +`SYSTEMONE_ADAPTER_NAME`, a checkpoint with no `decision` adapter, or an invocation marker that does not +activate the adapter in the chat template. + +## Make a request ```bash -curl -fsS http://localhost:8000/health && echo healthy -curl -fsS http://localhost:8000/v1/models # lists "granite-switch" -python3 scripts/e2e_check.py --base-url http://localhost:8000 # needs only the standard library -python3 scripts/accuracy_check.py --base-url http://localhost:8000 # answers are still good (needs internet) +curl http://localhost:8000/v1/decisions -H "Content-Type: application/json" -d '{ + "model": "granite-switch", + "input": "My payouts have failed three times. The bank says everything is fine.", + "questions": [ + {"type": "choice", "name": "queue", "instructions": "Which team should handle this?", + "choices": [{"value": "payments", "description": "Payout failures"}, + {"value": "account", "description": "Login issues"}]}, + {"type": "predicate", "name": "escalate", "instructions": "Does this need urgent attention?"} + ]}' ``` -`e2e_check.py` exits 0 when all its checks pass (wire formats on both routes, route -parity, chat still working, prefix-cache sharing). Run it from a checkout of this -repository; it works against any host. If the server exits at startup, the log names -the cause: an unknown `SYSTEMONE_ADAPTER_NAME`, a checkpoint with no `decision` adapter, or an -invocation marker the chat template does not activate the adapter on (checked at startup). +[examples/decisions.md](examples/decisions.md) has the OpenAI SDK, all three question types and the +equivalent `/v1/systemone` request. For equivalent questions both routes build the same prompt and +return the same probabilities. -## Securing the server +## Secure it before you expose it -The server listens on `0.0.0.0` and, as shipped, requires no authentication on any route. -Do not expose it directly. Either put it behind a gateway that authenticates callers, or -add vLLM's `--api-key`, which `serve.sh` passes through: +The server listens on `0.0.0.0` and, as shipped, asks for no credentials on any route. Put it behind a +gateway that authenticates callers, or turn on vLLM's `--api-key`, which `serve.sh` passes through: ```bash docker run -d --gpus all --ipc=host -p 8000:8000 -v … granite-switch-systemone --api-key "$KEY" ``` -vLLM's key check is app-wide middleware that rejects any request under `/v1` without -`Authorization: Bearer ` (it reads `vllm/entrypoints/serve/middleware/authenticate.py` -in v0.30.0), so it covers `/v1/decisions` and `/v1/systemone`; `/health` stays open. -Not run against a live server here: after enabling it, confirm an unauthenticated `curl` -to `/v1/decisions` returns 401. Request size limits are under Behaviour and limits. Set `SYSTEMONE_MAX_CONCURRENT_REQUESTS` -(for example 8 to 32) on a server shared with chat: it is off by default, and uncapped decision -bursts slowed chat p50 from 235 ms to 1,088 ms in our load test (see docs/STATUS.md). +In vLLM 0.30.0 that key check covers every `/v1` route, including `/v1/decisions` and +`/v1/systemone`, and leaves `/health` open (`vllm/entrypoints/serve/middleware/authenticate.py`). We +have not run it against a live server, so after enabling it, confirm that an unauthenticated `curl` to +`/v1/decisions` returns 401. -## Upstream status +If the server also handles chat, set `SYSTEMONE_MAX_CONCURRENT_REQUESTS` (8 to 32 is a sensible +range). It is off by default, and without it a burst of decisions slowed chat p50 from 235 ms to +1,088 ms in our load test (see [docs/STATUS.md](docs/STATUS.md)). -vLLM now has its own `/v1/systemone` ([#59299](https://github.com/vllm-project/vllm/pull/59299), -merged 2026-10-08) and `/v1/decisions` ([#60465](https://github.com/vllm-project/vllm/pull/60465), -merged 2026-10-13). Both are on `main` after 0.30.0, so the next vLLM release will carry them. -This package is for vLLM 0.30 (and, untested since `v0.1.0`, 0.26 to 0.29), where they are absent, and it stops being needed when -you move to a release that has them. The upstream endpoints differ from this plugin in wire -details (this one follows jev's format and OpenAI's schema as published). +## Speech input (optional) -The plugin refuses to start if either path is already registered, so running it on a vLLM -that has the upstream endpoints fails at startup instead of silently picking one. Remove -`granite_switch_systemone` from `VLLM_PLUGINS` there. +Speech is a chat feature. The decision routes take text only, and the 8b and 30b checkpoints enable +speech input in their configuration. Loading a speech model slows startup and decisions never use it, +so `serve.sh` switches it off (`--hf-overrides '{"asr_enabled": false}'`). The checkpoint then starts +with nothing else to download, and `/v1/chat/completions` accepts no audio. -Still open upstream: native Granite Switch support -([#60232](https://github.com/vllm-project/vllm/pull/60232)) and modular read strategies -([RFC #59365](https://github.com/vllm-project/vllm/issues/59365)). If the RFC lands, the -natural migration is a Granite Switch read strategy; `renderer.render(tok, state, -instructions, descs) -> (messages, label_ids)` is already close to what a strategy needs. +To turn it on, set `SYSTEMONE_SPEECH_INPUT=1`. The image runs with the Hugging Face Hub offline, so the +speech model +([`ibm-granite/granite-speech-5.0-470m-turboctc`](https://huggingface.co/ibm-granite/granite-speech-5.0-470m-turboctc), +about 1 GB) has to be available. Any one of these works: -## Server flags +- Use an image with the model built in: the `-asr` tag of a release (releases made with the current + workflow publish one), or build it yourself: + `docker build --platform linux/amd64 --build-arg WITH_ASR=1 -t granite-switch-systemone:asr .` +- Mount a Hub cache that holds the model and set `HF_HUB_CACHE` to that directory. +- Set `HF_HUB_OFFLINE=0` so the model downloads at startup. -`serve.sh` (the image's entrypoint) runs `vllm serve` with these non-default flags: +On startup `serve.sh` checks for the speech model and for the audio libraries (`av`, `scipy`, +`soundfile`, `soxr`), and stops with a message that names whatever is missing. The images already +include the libraries. If you run the plugin in your own environment, install `vllm[audio]`. Without +it the server starts and passes `/health`, then answers every audio request with HTTP 500 "Please +install vllm[audio]". -| Flag | Why | -|---|---| -| `--dtype bfloat16` | Composed checkpoints carry `torch_dtype: null` ([granite-switch#126](https://github.com/generative-computing/granite-switch/issues/126)); vLLM would otherwise load float32. | -| `--attention-config.flash_attn_version=2` | Avoids an FA3 schedule bug with CUDA graphs and prefix caching ([granite-switch#139](https://github.com/generative-computing/granite-switch/issues/139), open). With granite-switch 0.1.1 we measured FA3 the same as FA2 for answers and speed, so this is a precaution. | -| `--enable-prompt-tokens-details` | vLLM reports each request's prefix-cache hits; `/v1/decisions` returns them as `usage.input_tokens_details`. Without it both counts are 0. | -| `--max-logprobs 256` | For chat clients that read labels by top-K logprobs (vLLM's default cap is 20). The plugin does not need it. | -| `--enable-auto-tool-choice --tool-call-parser granite4` | Tool calling on the shared chat route. | -| `--served-model-name`, `--max-model-len 32768`, `--gpu-memory-utilization 0.90`, `--tensor-parallel-size` | Overridable with `SERVED_MODEL_NAME`, `MAX_MODEL_LEN`, `GPU_MEMORY_UTILIZATION`, `TENSOR_PARALLEL_SIZE`. | +The `-asr` image bakes in the multilingual release of the speech model; its model card lists the +supported languages. To bake in the earlier English-only revision instead, pass +`--build-arg ASR_REVISION=947f59af40db9791170a0628cf0f3f4812d720f1`. -There is no server-wide temperature override, so chat requests keep the model's defaults. -Prefix caching stays at vLLM's default (on). +## Configuration -### What the plugin sets on every decision request +`serve.sh`, the image's entrypoint, runs `vllm serve` with these settings. Everything after the image +name is passed on to `vllm serve`. -Decisions need a deterministic one-token read, so the plugin sends these itself and -callers cannot change them: +| Environment variable | Default | Effect | +|---|---|---| +| `MODEL_DIR` | `/repository` | Where the checkpoint is. | +| `SERVED_MODEL_NAME` | `granite-switch` | The `model` name clients send. | +| `PORT` | `8000` | Listening port. | +| `MAX_MODEL_LEN` | `32768` | Longest prompt plus output. | +| `GPU_MEMORY_UTILIZATION` | `0.90` | Share of GPU memory vLLM may use. | +| `TENSOR_PARALLEL_SIZE` | every visible GPU | GPUs to split the model across. | +| `SYSTEMONE_ADAPTER_NAME` | the only adapter, else `decision` | Which adapter answers. Checked against the checkpoint at startup; an unknown name stops the server. | +| `SYSTEMONE_INVOCATION_MARKER` | none (`### DECISION` in the image) | The text the adapter was trained to activate on. It is required: with a wrong marker the chat template activates the adapter in the wrong place without raising an error. | +| `SYSTEMONE_SPEECH_INPUT` | `0` | `1` keeps speech input; see [Speech input](#speech-input-optional). | +| `SYSTEMONE_MAX_CONCURRENT_REQUESTS` | `0` (no cap) | Decision requests allowed in flight across both routes. Past the cap a request gets 503 at once, so a burst cannot queue ahead of chat. | +| `SYSTEMONE_RENDERER` | `granite_switch_systemone.renderer:render` | A `module:function` that builds the prompt and label ids, for an adapter trained differently. | +| `SYSTEMONE_EXPERIMENTAL_SYSTEM_PROMPT` | `0` | Experimental, may be removed. `1` adds the trained decision-engine system prompt, as used for the model card's published score. It stops chat requests sharing the decision prefix cache. | -- `temperature=0.0` and `max_completion_tokens=1` -- `logprob_token_ids` set to the option labels' token ids, read exactly rather than from - the top-K. vLLM caps this at 128 ids, so a `choice` with more options runs as several - requests over the same prompt, served from the prefix cache. -- `chat_template_kwargs={"adapter_name": }`, which activates the adapter +Both routes share these settings; nothing is configured per route. -## Configuration +### Build arguments -| Env var | Default | What it does | -|---|---|---| -| `SYSTEMONE_ADAPTER_NAME` | the only adapter, else `decision` | Which adapter answers. Checked against the checkpoint's adapter list at startup; an unknown name stops the server. | -| `SYSTEMONE_INVOCATION_MARKER` | none (`### DECISION` in the image) | The text the adapter was trained to activate on. Required: with a wrong marker the chat template activates the adapter in the wrong place without raising. | -| `SYSTEMONE_RENDERER` | `granite_switch_systemone.renderer:render` | `module:function` that builds the prompt and label ids, for a differently trained adapter. | -| `SYSTEMONE_MAX_CONCURRENT_REQUESTS` | `0` (no cap) | Decision requests in flight on both routes; past it a request gets 503 at once, so decision bursts cannot queue indefinitely ahead of chat. | -| `SYSTEMONE_EXPERIMENTAL_SYSTEM_PROMPT` | `0` | Experimental, may be removed. `1` adds the trained decision-engine system prompt, as used for the model card's published score; it stops chat requests sharing the decision prefix cache. | +`GRANITE_SWITCH_VERSION` (default `0.1.1`) is the granite-switch release to install, with its `audio` +extra. `WITH_ASR=1` bakes in the speech model. `ASR_MODEL` and `ASR_REVISION` name the Hub repository +and the commit that is baked in. The default commit is pinned, so a rebuild does not change when the +model repository moves. The checkpoint is never a build argument: you mount it when the container runs. -Both routes share these; nothing is configured per route. +### Server flags -## Calling it +`serve.sh` passes these non-default flags to vLLM: -```bash -curl http://localhost:8000/v1/decisions -H "Content-Type: application/json" -d '{ - "model": "granite-switch", - "input": "My payouts have failed three times. The bank says everything is fine.", - "questions": [ - {"type": "choice", "name": "queue", "instructions": "Which team should handle this?", - "choices": [{"value": "payments", "description": "Payout failures"}, - {"value": "account", "description": "Login issues"}]}, - {"type": "predicate", "name": "escalate", "instructions": "Does this need urgent attention?"} - ]}' -``` +| Flag | Why | +|---|---| +| `--dtype bfloat16` | Composed checkpoints carry `torch_dtype: null` ([granite-switch#126](https://github.com/generative-computing/granite-switch/issues/126)), and vLLM would otherwise load float32. | +| `--attention-config.flash_attn_version=2` | Avoids an FA3 schedule bug with CUDA graphs and prefix caching ([granite-switch#139](https://github.com/generative-computing/granite-switch/issues/139), open). With granite-switch 0.1.1 we measured FA3 the same as FA2 for answers and speed, so this is a precaution. | +| `--enable-prompt-tokens-details` | vLLM reports each request's prefix-cache hits, and `/v1/decisions` returns them as `usage.input_tokens_details`. Without the flag both counts are 0. | +| `--max-logprobs 256` | For chat clients that read labels by top-K logprobs (vLLM's default cap is 20). The plugin does not need it. | +| `--enable-auto-tool-choice --tool-call-parser granite4` | Tool calling on the shared chat route. | -[examples/decisions.md](examples/decisions.md) has the OpenAI SDK, all three question types -and the equivalent `/v1/systemone` request. Both routes build the same prompt for -equivalent questions and return the same probabilities. +There is no server-wide temperature override, so chat requests keep the model's defaults. Prefix +caching stays at vLLM's default, which is on. + +On every decision request the plugin itself sets `temperature=0.0`, `max_completion_tokens=1`, +`logprob_token_ids` (the option labels' token ids, read exactly rather than from the top-K) and +`chat_template_kwargs={"adapter_name": }`. Callers cannot change them. vLLM +caps `logprob_token_ids` at 128 ids, so a `choice` with more options runs as several requests over the +same prompt, served from the prefix cache. ## Behaviour and limits -- **Prompt.** Two user turns, `State:\n` then `### DECISION` with the question and - lettered options (A–Z, then AA, AB, ... for more than 26), read as a softmax over the - label logits. Predicates render "A: No / false", "B: Yes / true". An option without a - description is described by its value. -- **No system prompt by default.** This lets a decision share the prefix cache with a chat request - whose first turn is the same `State:` turn. Any chat with its own system prompt, or - that sends the state without `State:`, shares nothing; decisions about one state always - share it with each other. The model card's published score used a system prompt; its effect - on accuracy is still being evaluated. `SYSTEMONE_EXPERIMENTAL_SYSTEM_PROMPT=1` turns it on. -- **Caller text is escaped.** Special tokens and the invocation marker inside state, - instructions or descriptions are broken with an invisible U+2060, so they cannot forge - turns or move the adapter's activation point. -- **Input.** `input` messages must have role `user`, and images are rejected; the trained - layout has one text state turn. `/v1/systemone` rejects unknown fields and `box` - questions. -- **Size.** The state, instructions and options of each question are counted before any - engine work: at most 200 questions, 32k tokens per question and 64k per request (262k with - `TypeSafe-Danger-Zone-Long-Context: true` on `/v1/systemone`), capped by - `--max-model-len`. Bodies over 32 MiB, nesting deeper than 64 and non-finite numbers - are rejected. -- **Usage.** A `choice` with more than 128 options runs as several engine requests over the same - prompt (128 labels each, so two requests at most), and `usage.input_tokens` counts the prompt - once per request, so such a question reports about twice its prompt length. `cached_tokens` shows how much was shared. -- **Speculative decoding.** Every decision read passes `logprob_token_ids`, which vLLM crashes on - when a speculative-decoding request shares the batch - ([vllm#42592](https://github.com/vllm-project/vllm/issues/42592), fix - [#44727](https://github.com/vllm-project/vllm/pull/44727) open). The plugin logs a warning at +- **Prompt.** Two user turns: `State:\n`, then `### DECISION` with the question and lettered + options (A to Z, then AA, AB and so on past 26), read as a softmax over the label logits. Predicates + render as "A: No / false" and "B: Yes / true". An option without a description is described by its + value. +- **No system prompt by default.** A decision can then share the prefix cache with a chat request + whose first turn is the same `State:` turn. A chat with its own system prompt, or one that sends the + state without `State:`, shares nothing. Decisions about one state always share with each other. The + model card's published score used a system prompt, and its effect on accuracy is still being + evaluated. +- **Caller text is escaped.** Special tokens and the invocation marker inside the state, instructions + or descriptions are broken with an invisible U+2060, so they cannot forge turns or move the adapter's + activation point. +- **Input.** `input` messages must have the role `user`, and images are rejected, because the trained + layout has one text state turn. `/v1/systemone` rejects unknown fields and `box` questions. +- **Size.** The state, instructions and options of each question are counted before any engine work: + at most 200 questions, 32k tokens per question and 64k per request (262k with + `TypeSafe-Danger-Zone-Long-Context: true` on `/v1/systemone`), all capped by `--max-model-len`. + Bodies over 32 MiB, nesting deeper than 64 and non-finite numbers are rejected. +- **Usage.** A `choice` with more than 128 options runs as two engine requests over the same prompt, + and `usage.input_tokens` counts the prompt once per request, so such a question reports about twice + its prompt length. `cached_tokens` shows how much was shared. +- **Speculative decoding.** Every decision read passes `logprob_token_ids`, which crashes vLLM when a + speculative-decoding request shares the batch ([vllm#42592](https://github.com/vllm-project/vllm/issues/42592), + fix [#44727](https://github.com/vllm-project/vllm/pull/44727) open). The plugin logs a warning at startup if `speculative_config` is set. We have not tested this combination. - **Cancellation.** A client that disconnects cancels its request's engine work. -- **Errors.** `/v1/decisions` uses OpenAI's error shape, `/v1/systemone` uses jev's +- **Errors.** `/v1/decisions` uses OpenAI's error shape and `/v1/systemone` uses jev's `HTTPValidationError` for 422s. Engine errors keep vLLM's status code. -- **Not provided.** Authentication (use vLLM's `--api-key`, which covers these routes), - rate limiting, and `refusal` answers. Engine reads are not bit-identical between - batched and solo requests, so repeated calls can differ in the third decimal place. -- **vLLM 0.32** mounts its own `/v1/systemone` (newer versions add `/v1/decisions`); - neither knows Switch adapters, and granite-switch 0.1.1 does not support those - versions yet. +- **Not provided.** Authentication (use vLLM's `--api-key`), rate limiting and `refusal` answers. + Engine reads are not bit-identical between batched and solo requests, so repeated calls can differ in + the third decimal place. + +## Upstream status + +vLLM now has its own `/v1/systemone` ([#59299](https://github.com/vllm-project/vllm/pull/59299), +merged 2026-10-08) and `/v1/decisions` ([#60465](https://github.com/vllm-project/vllm/pull/60465), +merged 2026-10-13). Both are on `main` after 0.30.0, so the next vLLM release will carry them. This +package is for vLLM 0.30, and for 0.26 to 0.29 (untested since `v0.1.0`), where those routes are +absent. You can drop it once you move to a release that has them. The upstream endpoints differ in wire +details: this plugin follows jev's format and OpenAI's schema as published. + +The plugin refuses to start if either path is already registered, so on a vLLM that has the upstream +endpoints it fails at startup and does not silently pick one. Remove `granite_switch_systemone` from +`VLLM_PLUGINS` there. vLLM 0.32 mounts its own `/v1/systemone` (newer versions add `/v1/decisions`). +Neither knows Switch adapters, and granite-switch 0.1.1 does not support those versions yet. + +Still open upstream are native Granite Switch support +([#60232](https://github.com/vllm-project/vllm/pull/60232)) and modular read strategies +([RFC #59365](https://github.com/vllm-project/vllm/issues/59365)). If the RFC is accepted, the natural +migration is a Granite Switch read strategy, and `renderer.render(tok, state, instructions, descs) -> +(messages, label_ids)` is already close to what a strategy needs. + +## How well tested is this? + +The pip path is what most of the live testing used. The container image is lightly tested: it builds +for amd64, with and without the speech model, and the packages and plugin entry points were checked +inside it. The published `0.1.3` image was served on H100 GPUs with only the checkpoint mounted and +speech input off, the 8b on one GPU and the 30b on two with tensor parallelism, and the wire, +robustness and accuracy checks passed. That was run under enroot and not with `docker run`; see +[docs/STATUS.md](docs/STATUS.md) for exactly what was and was not run. If you run the image, treat the +first start as a test and tell us what you find. -## Testing +## Development ```bash pip install -e ".[dev]" && pytest # unit and spec tests, no GPU python3 scripts/e2e_check.py --base-url http://localhost:8000 # live wire checks python3 scripts/probe_check.py --base-url http://localhost:8000 # live robustness checks +python3 scripts/accuracy_check.py --base-url http://localhost:8000 # answers are still good (needs internet) ``` -[docs/TESTING.md](docs/TESTING.md) lists every test, what runs in CI and what needs a -tokenizer or a live server. +[docs/TESTING.md](docs/TESTING.md) lists every test, what runs in CI, and what needs a tokenizer or a +live server. -## License +## Licence Apache-2.0. Parts adapted from MIT-licensed code are listed, with their notices, in [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md). From 0db3173f0eded8606dae1cfa01f6ee1a40218e5d Mon Sep 17 00:00:00 2001 From: Nigel Jones Date: Fri, 23 Oct 2026 20:39:27 +0100 Subject: [PATCH 06/11] ci: build the images for amd64 and arm64 and publish one multi-arch tag Each architecture builds on a native runner and pushes by digest. A second job joins the digests into the version tag (and latest) for both the default and the -asr image. Assisted-by: Claude Code Signed-off-by: Nigel Jones --- .github/workflows/image.yml | 104 ++++++++++++++++++++++++++++-------- 1 file changed, 82 insertions(+), 22 deletions(-) diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index c1cf15f..bdfec65 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -1,7 +1,9 @@ name: Image -# Builds the Docker image and pushes it to GitHub Container Registry. Runs on a -# version tag, or by hand against any ref (leave "push" off to build without publishing). +# Builds the Docker images and pushes them to GitHub Container Registry for linux/amd64 and +# linux/arm64. Runs on a version tag, or by hand against any ref (leave "push" off to build +# without publishing). Each architecture builds on a native runner and pushes by digest; a +# second job then joins the two into one multi-arch tag. on: push: @@ -9,37 +11,40 @@ on: workflow_dispatch: inputs: push: - description: "Push the image to ghcr.io" + description: "Push the images to ghcr.io" type: boolean default: false permissions: contents: read +env: + IMAGE: ghcr.io/${{ github.repository }} + PUSH: ${{ github.event_name == 'push' || inputs.push }} + jobs: - image: + build: # Two images per release: the default (decisions only, no speech model inside) and an # `-asr` variant with the pinned speech model baked in, for chat speech input. - name: image (${{ matrix.variant }}) + name: build (${{ matrix.variant }}, ${{ matrix.arch }}) strategy: fail-fast: false matrix: + variant: [default, asr] + arch: [amd64, arm64] include: - # `latest` stays the default image; the asr image gets `latest-asr` (onlatest=true), - # so the two variants never push the same tag. + - arch: amd64 + runner: ubuntu-latest + - arch: arm64 + runner: ubuntu-24.04-arm - variant: default with_asr: "0" - flavor: "latest=auto" - variant: asr with_asr: "1" - flavor: "suffix=-asr,onlatest=true" - runs-on: ubuntu-latest + runs-on: ${{ matrix.runner }} permissions: contents: read packages: write - env: - IMAGE: ghcr.io/${{ github.repository }} - PUSH: ${{ github.event_name == 'push' || inputs.push }} steps: # The base image is large; the runner's default free disk is not enough. - name: Free disk space @@ -55,6 +60,59 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push by digest + id: build + uses: docker/build-push-action@v7 + with: + context: . + platforms: linux/${{ matrix.arch }} + build-args: WITH_ASR=${{ matrix.with_asr }} + outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=${{ env.PUSH == 'true' }} + - name: Export digest + if: env.PUSH == 'true' + run: | + mkdir -p "$RUNNER_TEMP/digests" + digest="${{ steps.build.outputs.digest }}" + touch "$RUNNER_TEMP/digests/${digest#sha256:}" + - uses: actions/upload-artifact@v7 + if: env.PUSH == 'true' + with: + name: digests-${{ matrix.variant }}-${{ matrix.arch }} + path: ${{ runner.temp }}/digests/* + if-no-files-found: error + retention-days: 1 + + merge: + name: tag (${{ matrix.variant }}) + if: ${{ github.event_name == 'push' || inputs.push }} + needs: build + strategy: + fail-fast: false + matrix: + include: + # `latest` stays the default image; the asr image gets `latest-asr` (onlatest=true), + # so the two variants never push the same tag. + - variant: default + flavor: "latest=auto" + - variant: asr + flavor: "suffix=-asr,onlatest=true" + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/download-artifact@v8 + with: + path: ${{ runner.temp }}/digests + pattern: digests-${{ matrix.variant }}-* + merge-multiple: true + - uses: docker/setup-buildx-action@v4 + - name: Log in to ghcr.io + uses: docker/login-action@v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Tags id: meta uses: docker/metadata-action@v6 @@ -64,12 +122,14 @@ jobs: tags: | type=semver,pattern={{version}} type=ref,event=branch - - name: Build and push - uses: docker/build-push-action@v7 - with: - context: . - platforms: linux/amd64 - build-args: WITH_ASR=${{ matrix.with_asr }} - push: ${{ env.PUSH == 'true' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + - name: Create the multi-arch manifest + working-directory: ${{ runner.temp }}/digests + run: | + mapfile -t tags < <(jq -r '.tags[]' <<< "$DOCKER_METADATA_OUTPUT_JSON") + args=() + for t in "${tags[@]}"; do args+=(-t "$t"); done + refs=() + for d in *; do refs+=("${IMAGE}@sha256:${d}"); done + docker buildx imagetools create "${args[@]}" "${refs[@]}" + - name: Inspect + run: docker buildx imagetools inspect "${{ env.IMAGE }}:${{ steps.meta.outputs.version }}" From 375e0af2305ef6d3a0db35d91d2ab8b518dc86ae Mon Sep 17 00:00:00 2001 From: Nigel Jones Date: Fri, 23 Oct 2026 20:46:27 +0100 Subject: [PATCH 07/11] test: add scripts/speech_check.py for speech input on chat completions Standard library only. Sends spoken clips as input_audio and checks the reply repeats most of the words. Clips are made at run time with espeak-ng or macOS say, so no audio is stored here; --clip takes your own. Documented in docs/TESTING.md, and the stale quick-start anchor there is fixed. Assisted-by: Claude Code Signed-off-by: Nigel Jones --- docs/TESTING.md | 3 +- scripts/speech_check.py | 168 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 scripts/speech_check.py diff --git a/docs/TESTING.md b/docs/TESTING.md index fea0960..913bd47 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -74,6 +74,7 @@ Python standard library unless noted. | `python3 scripts/probe_check.py --base-url URL --load 30` | Adds a load test at 1, 8 and 32 clients with chat latency alongside. Reported, not asserted. | | `python3 scripts/longctx_check.py --base-url URL --tokenizer /path/to/granite-switch-4.1-8b` | Long-context caps, on a server started with `--max-model-len 131072` and the 8b: the danger-zone header path (a ~98k-token request answered with the header, rejected without it), the 131,072 backstop, the default 32k/64k caps, and that `/v1/decisions` has no header escape. Unlike the scripts above it needs the checkpoint's tokenizer (transformers). | | `python3 scripts/accuracy_check.py --base-url URL` | Accuracy, log-loss and Brier score on 200 BoolQ validation rows (public data, downloaded when it runs, needs internet). Fails below 75% accuracy, which catches a wrong invocation marker. To check an upgrade or a changed setting, run it on each deployment with `--save`, then `--compare` the second against the first: it reports items only one run got right and an exact McNemar p-value. | +| `python3 scripts/speech_check.py --base-url URL` | Speech input, on a server started with `SYSTEMONE_SPEECH_INPUT=1` (see the README). Sends spoken clips to chat completions as `input_audio` and checks that the reply repeats at least 80% of the words. Makes its own clips with `espeak-ng` or macOS `say`, so no audio is stored here; `--clip FILE.wav "the words"` uses your own. Exits 2 if it has no clips. A server missing the audio libraries passes `/health` and fails here with HTTP 500. | | `python3 scripts/load_test.py --base-url URL` | Throughput and latency of `/v1/decisions` at 1, 8, 32, 64 and 128 clients, with the server's prefix-cache hit rate. `--mode unique` (default) gives every request a fresh state; `--mode identical` repeats one body and flatters throughput. Run both and report both. See below. | | `SYSTEMONE_BASE_URL=URL pytest tests/test_decisions_e2e.py` | `/v1/decisions` and `/v1/systemone` return the same answers for equivalent requests. | @@ -145,7 +146,7 @@ repository: you fetch the kit, build the suite yourself, and run it against your What you need: -- A running server (see [Verify the deployment](../README.md#verify-the-deployment)). Start it +- A running server (see [Quick start](../README.md#quick-start)). Start it with `--max-model-len 131072` if you want the longest rows answered instead of rejected. - The kit, which is MIT licensed. Install it and build the suite by following its README ("Quickstart (local)"). The suite is not redistributed, so you rebuild it from public sources diff --git a/scripts/speech_check.py b/scripts/speech_check.py new file mode 100644 index 0000000..9ed26c0 --- /dev/null +++ b/scripts/speech_check.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Speech-input smoke test for a live granite-switch-systemone server started with SYSTEMONE_SPEECH_INPUT=1. + +Standard library only. Sends short spoken clips to /v1/chat/completions as `input_audio` and checks +that the model heard them: the reply has to repeat most of the words that were spoken. Decisions take +text only, so this checks the chat route the speech model feeds. + + python3 scripts/speech_check.py --base-url http://localhost:8000 + python3 scripts/speech_check.py --base-url http://localhost:8000 --clip my.wav "the words in it" + +Without --clip it makes its own clips at run time with espeak-ng (any platform) or `say` (macOS), so +no audio is stored in this repository. If neither is installed it says so and exits 2; pass --clip +with your own 16-bit WAV file instead. Exit 0 means every check passed. + +Checks: a text-only chat request still works; each clip gets HTTP 200 and a reply that repeats at +least --min-overlap of the spoken words; an audio request with no speech model behind it is reported +with its server message (the usual cause is missing `vllm[audio]` libraries: the server passes +/health, then every audio request fails with HTTP 500). +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import re +import shutil +import subprocess +import sys +import tempfile +import urllib.error +import urllib.request +from pathlib import Path + +# Plain English sentences: the check is that the model heard them, so they avoid names and numbers. +BUILTIN = ( + "the quick brown fox jumps over the lazy dog", + "please send the invoice to the finance team tomorrow morning", +) +INSTRUCTION = "Repeat the words you heard in the audio exactly, and output nothing else." + + +def post(base_url: str, body: dict, timeout: float = 180) -> tuple[int, dict | str]: + req = urllib.request.Request( + base_url.rstrip("/") + "/v1/chat/completions", + json.dumps(body).encode(), + {"Content-Type": "application/json"}, + ) + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + return r.status, json.load(r) + except urllib.error.HTTPError as e: + return e.code, e.read().decode("utf-8", "replace")[:400] + + +def words(text: str) -> list[str]: + return re.findall(r"[a-z']+", text.lower()) + + +def overlap(expected: str, reply: str) -> float: + want, got = words(expected), set(words(reply)) + return sum(w in got for w in want) / len(want) if want else 0.0 + + +def synthesize(text: str, out: Path) -> bool: + """Write a 16 kHz mono WAV of `text`; False if no speech synthesizer is installed.""" + if shutil.which("espeak-ng"): + subprocess.run(["espeak-ng", "-w", str(out), text], check=True, capture_output=True) + return True + if shutil.which("say"): + subprocess.run(["say", "--data-format=LEI16@16000", "-o", str(out), text], check=True, capture_output=True) + return True + return False + + +def ask_audio(base_url: str, wav: Path, model: str) -> tuple[int, str]: + audio = base64.b64encode(wav.read_bytes()).decode() + body = { + "model": model, + "temperature": 0, + "max_tokens": 120, + "messages": [ + { + "role": "user", + "content": [ + {"type": "input_audio", "input_audio": {"data": audio, "format": "wav"}}, + {"type": "text", "text": INSTRUCTION}, + ], + } + ], + } + status, data = post(base_url, body) + if status == 200 and isinstance(data, dict): + return status, data["choices"][0]["message"]["content"] or "" + return status, str(data) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--base-url", required=True) + ap.add_argument("--model", default="granite-switch") + ap.add_argument( + "--clip", + nargs=2, + action="append", + metavar=("WAV", "WORDS"), + default=[], + help="your own clip and the words in it (repeatable)", + ) + ap.add_argument( + "--min-overlap", type=float, default=0.8, help="share of spoken words the reply must repeat (default 0.8)" + ) + args = ap.parse_args() + + failures = 0 + + def report(ok: bool, name: str, detail: str = "") -> None: + nonlocal failures + failures += not ok + print(f"{'PASS' if ok else 'FAIL'} {name}" + (f" {detail}" if detail else "")) + + status, data = post( + args.base_url, + { + "model": args.model, + "max_tokens": 8, + "temperature": 0, + "messages": [{"role": "user", "content": "Say hello."}], + }, + ) + report(status == 200, "text-only chat still works", f"HTTP {status}") + + with tempfile.TemporaryDirectory() as tmp: + clips = [(Path(w), words_) for w, words_ in args.clip] + if not clips: + for i, text in enumerate(BUILTIN): + wav = Path(tmp) / f"clip{i}.wav" + if not synthesize(text, wav): + print( + "No speech synthesizer found (install espeak-ng, or use macOS `say`), and no --clip given.", + file=sys.stderr, + ) + return 2 + clips.append((wav, text)) + for wav, spoken in clips: + status, reply = ask_audio(args.base_url, wav, args.model) + if status != 200: + hint = ( + " (the audio libraries are probably missing: install vllm[audio])" + if "vllm[audio]" in reply + else "" + ) + report(False, f"audio clip {wav.name}", f"HTTP {status}: {reply[:200]}{hint}") + continue + share = overlap(spoken, reply) + report( + share >= args.min_overlap, + f"audio clip {wav.name}", + f"heard {share:.0%} of {len(words(spoken))} words; reply {reply!r}", + ) + + print("speech check:", "FAILED" if failures else "passed") + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) From f2d39f8481c2b730bfa9e18c9d091824d8b41dc9 Mon Sep 17 00:00:00 2001 From: Nigel Jones Date: Fri, 23 Oct 2026 20:50:29 +0100 Subject: [PATCH 08/11] docs: record the arm64 serving check, the multi-arch build and the speech check Assisted-by: Claude Code Signed-off-by: Nigel Jones --- docs/STATUS.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/STATUS.md b/docs/STATUS.md index 149b78f..f0f041c 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -2,7 +2,7 @@ How to run each test is in [TESTING.md](TESTING.md). -Last updated 2026-10-20. "Live" means a real server on H100s: granite-switch 0.1.1, +Last updated 2026-10-23. "Live" means a real server on H100s: granite-switch 0.1.1, transformers 5.17, vLLM 0.26.0 / 0.27.1 / 0.28.0 / 0.29.0 / 0.30.0 as noted per row, and `ibm-granite/granite-switch-4.1-8b` unless a row names another checkpoint. @@ -31,12 +31,15 @@ and `ibm-granite/granite-switch-4.1-8b` unless a row names another checkpoint. | Docker image build | `docker build --platform linux/amd64`, default and `WITH_ASR=1` variants | both build; the stack inside is package-identical to the live-tested one (PyPI vLLM 0.30.0, torch 2.13.0+cu130, transformers 5.17.0, granite-switch 0.1.1); entry points and imports verified in-container; the `WITH_ASR=1` image holds the pinned speech-model revision as its only snapshot, with `refs/main` pointing at it | | Serving from the image | the default image (no speech model inside) under enroot (no Docker daemon on the GPU nodes), through `serve.sh`, only the checkpoint mounted, Hub offline: the 8b on one H100 and the 30b on two H100s (`--tensor-parallel-size 2`, set from the GPU count). On each: `e2e_check`, `probe_check`, `accuracy_check` | pass on both: 11/11 and 6/6, accuracy as on the pip path; the log shows `hf_overrides: {'asr_enabled': False}` and never names the speech model | | Speech input on (`SYSTEMONE_SPEECH_INPUT=1`) | the 8b on the default image with the speech model supplied from a mounted Hub cache (`HF_HUB_CACHE`); and, as a negative control, the same switch with no speech model anywhere | the 8b starts and passes the same three checks; the control stops in `serve.sh` with the preflight message naming the model and the fixes, before vLLM starts. The `WITH_ASR=1` image (model baked in) was served on the 8b in an earlier run | +| Container on arm64 | `docker build` of the default image on an aarch64 NVIDIA GB10 host (vLLM 0.30.0, granite-switch 0.1.1 resolved, imports checked in the image); the public `granite-switch-4.1-8b` served from it with `docker run --gpus all`: `e2e_check`, `probe_check`, `accuracy_check` | pass; BoolQ accuracy 0.890 and log-loss 0.283 on arm64, against 0.890 and 0.282 on amd64 with the same checkpoint | +| Multi-arch image build | `image.yml` run by hand (build only, then with push): each of the default and `-asr` images builds natively for amd64 and arm64, and a second job joins them into one tag | see the manifest check recorded with the release | +| Speech input check | `scripts/speech_check.py` against a server with `SYSTEMONE_SPEECH_INPUT=1` and the speech model from a Hub cache: two spoken English sentences made at run time | both clips repeated back in full; without clips and without a synthesizer the script exits 2 | | Decision Index runs | samples of the Decision Index suite (edition 0.3) through `/v1/systemone` at `--max-model-len 131072`, on the 8b and the 30b, as an end-to-end validation of the endpoint | completes with no errors; scores are not reported here while the methodology is verified, and the model card's published figures are the reference (see the caveats below) | ## Not tested -- **Serving the image under Docker.** The image was served under enroot, on the 8b (one H100) - and the 30b (two H100s; see the table). That runs the same filesystem and environment as the image, but not +- **Serving the image under Docker on amd64.** The image was served under enroot, on the 8b (one H100) + and the 30b (two H100s; see the table). The arm64 image was served with `docker run` (see the table). That runs the same filesystem and environment as the image, but not `docker run` itself: its `ENTRYPOINT` (enroot ignores it, so `serve.sh` was started explicitly), its `--gpus all` handling and `--ipc=host` were not exercised. - **The 3b.** The only 3b checkpoint, `granite-switch-4.1-3b-preview`, ships twelve From 5aaa22fbbc4f3dd85e5933b6f1bf22fc00e44c06 Mon Sep 17 00:00:00 2001 From: Nigel Jones Date: Fri, 23 Oct 2026 20:56:56 +0100 Subject: [PATCH 09/11] docs: the images are multi-arch (amd64 and arm64) Confirmed from the manifest lists a workflow push run produced for both the default and -asr tags. Assisted-by: Claude Code Signed-off-by: Nigel Jones --- README.md | 14 +++++++------- docs/STATUS.md | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index c0cf97b..247e981 100644 --- a/README.md +++ b/README.md @@ -22,9 +22,9 @@ You need a Linux machine with an NVIDIA GPU. The 8b checkpoint fits on one 80 GB ### Container Release images are published to GitHub Container Registry as -`ghcr.io/generative-computing/granite-switch-systemone` for linux/amd64 only. `latest` is the newest -release, and each release is also tagged with its version. The pull is about 9 GB and the image takes -about 30 GB on disk. You also need Docker and the NVIDIA container toolkit. +`ghcr.io/generative-computing/granite-switch-systemone` for linux/amd64 and linux/arm64, so one tag +works on both. `latest` is the newest release, and each release is also tagged with its version. The +pull is about 9 GB and the image takes about 30 GB on disk (amd64 figures). You also need Docker and the NVIDIA container toolkit. ```bash # Download the checkpoint (public, not gated, about 18 GB) @@ -41,7 +41,7 @@ setting for shared memory, and it matters most when you run with tensor parallel The ghcr package is currently private, so `docker pull` needs `docker login ghcr.io` with an account that has access. Until it is public, build the image yourself: clone this repository at a release tag -and run `docker build --platform linux/amd64 -t granite-switch-systemone .`. +and run `docker build -t granite-switch-systemone .`. The image is `vllm/vllm-openai:v0.30.0` plus `granite-switch[audio]==0.1.1`, `transformers==5.17.0` and this plugin. It has no weights in it. It reads the checkpoint from `/repository` (override with @@ -141,7 +141,7 @@ about 1 GB) has to be available. Any one of these works: - Use an image with the model built in: the `-asr` tag of a release (releases made with the current workflow publish one), or build it yourself: - `docker build --platform linux/amd64 --build-arg WITH_ASR=1 -t granite-switch-systemone:asr .` + `docker build --build-arg WITH_ASR=1 -t granite-switch-systemone:asr .` - Mount a Hub cache that holds the model and set `HF_HUB_CACHE` to that directory. - Set `HF_HUB_OFFLINE=0` so the model downloads at startup. @@ -262,10 +262,10 @@ migration is a Granite Switch read strategy, and `renderer.render(tok, state, in ## How well tested is this? The pip path is what most of the live testing used. The container image is lightly tested: it builds -for amd64, with and without the speech model, and the packages and plugin entry points were checked +for amd64 and arm64, with and without the speech model, and the packages and plugin entry points were checked inside it. The published `0.1.3` image was served on H100 GPUs with only the checkpoint mounted and speech input off, the 8b on one GPU and the 30b on two with tensor parallelism, and the wire, -robustness and accuracy checks passed. That was run under enroot and not with `docker run`; see +robustness and accuracy checks passed. That was run under enroot and not with `docker run`. The arm64 image was also served with `docker run` on an NVIDIA GB10 (the 8b, default image). The arm64 `-asr` image was built but not served; see [docs/STATUS.md](docs/STATUS.md) for exactly what was and was not run. If you run the image, treat the first start as a test and tell us what you find. diff --git a/docs/STATUS.md b/docs/STATUS.md index f0f041c..1bb577e 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -32,7 +32,7 @@ and `ibm-granite/granite-switch-4.1-8b` unless a row names another checkpoint. | Serving from the image | the default image (no speech model inside) under enroot (no Docker daemon on the GPU nodes), through `serve.sh`, only the checkpoint mounted, Hub offline: the 8b on one H100 and the 30b on two H100s (`--tensor-parallel-size 2`, set from the GPU count). On each: `e2e_check`, `probe_check`, `accuracy_check` | pass on both: 11/11 and 6/6, accuracy as on the pip path; the log shows `hf_overrides: {'asr_enabled': False}` and never names the speech model | | Speech input on (`SYSTEMONE_SPEECH_INPUT=1`) | the 8b on the default image with the speech model supplied from a mounted Hub cache (`HF_HUB_CACHE`); and, as a negative control, the same switch with no speech model anywhere | the 8b starts and passes the same three checks; the control stops in `serve.sh` with the preflight message naming the model and the fixes, before vLLM starts. The `WITH_ASR=1` image (model baked in) was served on the 8b in an earlier run | | Container on arm64 | `docker build` of the default image on an aarch64 NVIDIA GB10 host (vLLM 0.30.0, granite-switch 0.1.1 resolved, imports checked in the image); the public `granite-switch-4.1-8b` served from it with `docker run --gpus all`: `e2e_check`, `probe_check`, `accuracy_check` | pass; BoolQ accuracy 0.890 and log-loss 0.283 on arm64, against 0.890 and 0.282 on amd64 with the same checkpoint | -| Multi-arch image build | `image.yml` run by hand (build only, then with push): each of the default and `-asr` images builds natively for amd64 and arm64, and a second job joins them into one tag | see the manifest check recorded with the release | +| Multi-arch image build | `image.yml` run by hand (build only, then with push): each of the default and `-asr` images builds natively for amd64 and arm64, and a second job joins them into one tag | both tags are manifest lists holding a `linux/amd64` and a `linux/arm64` image (plus attestations), read from the workflow's own `imagetools inspect` step | | Speech input check | `scripts/speech_check.py` against a server with `SYSTEMONE_SPEECH_INPUT=1` and the speech model from a Hub cache: two spoken English sentences made at run time | both clips repeated back in full; without clips and without a synthesizer the script exits 2 | | Decision Index runs | samples of the Decision Index suite (edition 0.3) through `/v1/systemone` at `--max-model-len 131072`, on the 8b and the 30b, as an end-to-end validation of the endpoint | completes with no errors; scores are not reported here while the methodology is verified, and the model card's published figures are the reference (see the caveats below) | From 15247cddd4b6cc7f8cfde82fa77b22b1d9a7dda4 Mon Sep 17 00:00:00 2001 From: Nigel Jones Date: Fri, 23 Oct 2026 21:11:56 +0100 Subject: [PATCH 10/11] docs: the arm64 -asr image serves speech Pulled the pushed arm64 -asr image and served the public 8b with SYSTEMONE_SPEECH_INPUT=1; scripts/speech_check.py passed. Assisted-by: Claude Code Signed-off-by: Nigel Jones --- README.md | 2 +- docs/STATUS.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 247e981..3df409e 100644 --- a/README.md +++ b/README.md @@ -265,7 +265,7 @@ The pip path is what most of the live testing used. The container image is light for amd64 and arm64, with and without the speech model, and the packages and plugin entry points were checked inside it. The published `0.1.3` image was served on H100 GPUs with only the checkpoint mounted and speech input off, the 8b on one GPU and the 30b on two with tensor parallelism, and the wire, -robustness and accuracy checks passed. That was run under enroot and not with `docker run`. The arm64 image was also served with `docker run` on an NVIDIA GB10 (the 8b, default image). The arm64 `-asr` image was built but not served; see +robustness and accuracy checks passed. That was run under enroot and not with `docker run`. The arm64 image was also served with `docker run` on an NVIDIA GB10 (the 8b, default image). The arm64 `-asr` image was also served with `docker run` and passed the speech check; see [docs/STATUS.md](docs/STATUS.md) for exactly what was and was not run. If you run the image, treat the first start as a test and tell us what you find. diff --git a/docs/STATUS.md b/docs/STATUS.md index 1bb577e..b50c463 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -33,7 +33,7 @@ and `ibm-granite/granite-switch-4.1-8b` unless a row names another checkpoint. | Speech input on (`SYSTEMONE_SPEECH_INPUT=1`) | the 8b on the default image with the speech model supplied from a mounted Hub cache (`HF_HUB_CACHE`); and, as a negative control, the same switch with no speech model anywhere | the 8b starts and passes the same three checks; the control stops in `serve.sh` with the preflight message naming the model and the fixes, before vLLM starts. The `WITH_ASR=1` image (model baked in) was served on the 8b in an earlier run | | Container on arm64 | `docker build` of the default image on an aarch64 NVIDIA GB10 host (vLLM 0.30.0, granite-switch 0.1.1 resolved, imports checked in the image); the public `granite-switch-4.1-8b` served from it with `docker run --gpus all`: `e2e_check`, `probe_check`, `accuracy_check` | pass; BoolQ accuracy 0.890 and log-loss 0.283 on arm64, against 0.890 and 0.282 on amd64 with the same checkpoint | | Multi-arch image build | `image.yml` run by hand (build only, then with push): each of the default and `-asr` images builds natively for amd64 and arm64, and a second job joins them into one tag | both tags are manifest lists holding a `linux/amd64` and a `linux/arm64` image (plus attestations), read from the workflow's own `imagetools inspect` step | -| Speech input check | `scripts/speech_check.py` against a server with `SYSTEMONE_SPEECH_INPUT=1` and the speech model from a Hub cache: two spoken English sentences made at run time | both clips repeated back in full; without clips and without a synthesizer the script exits 2 | +| Speech input check | `scripts/speech_check.py` against a server with `SYSTEMONE_SPEECH_INPUT=1` and the speech model from a Hub cache: two spoken English sentences made at run time | both clips repeated back in full, on the pip path and on the pushed arm64 `-asr` image (pulled from the registry, served with `docker run` on the GB10 with the public 8b); without clips and without a synthesizer the script exits 2 | | Decision Index runs | samples of the Decision Index suite (edition 0.3) through `/v1/systemone` at `--max-model-len 131072`, on the 8b and the 30b, as an end-to-end validation of the endpoint | completes with no errors; scores are not reported here while the methodology is verified, and the model card's published figures are the reference (see the caveats below) | ## Not tested From 59b1e410c4586eaddcb9ccc77c013fcadbf45a27 Mon Sep 17 00:00:00 2001 From: Nigel Jones Date: Fri, 23 Oct 2026 21:28:32 +0100 Subject: [PATCH 11/11] docs: tighten the longest comments in plugin.py and the Dockerfile Comments and docstrings only. The plugin's code is identical (checked by comparing the AST with docstrings removed), the Dockerfile diff has no non-comment lines, and the suite passes (226 passed, 4 skipped). Assisted-by: Claude Code Signed-off-by: Nigel Jones --- Dockerfile | 62 ++++++++++---------------- src/granite_switch_systemone/plugin.py | 42 +++++++---------- 2 files changed, 40 insertions(+), 64 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3e6c9dd..b5092fb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,50 +1,37 @@ -# A vLLM OpenAI-compatible server for a composed granite-switch checkpoint, serving -# BOTH the standard /v1/chat/completions API and typesafe.ai's jev system-one API -# (/v1/systemone) on the same server -- one engine, one GPU KV cache, one container. -# The checkpoint's weights are NOT baked into this image (mounted separately at -# $MODEL_DIR / /repository, e.g. by HF Inference Endpoints' own deployment mechanism). +# vLLM server for a composed granite-switch checkpoint: /v1/chat/completions plus the +# /v1/decisions and /v1/systemone plugin routes, on one engine and one KV cache. +# The checkpoint is not in the image; mount it at $MODEL_DIR (default /repository). # -# docker build --platform linux/amd64 -t /granite-switch-systemone: . +# docker build -t /granite-switch-systemone: . # -# Optional speech input. The audio libraries are in every variant; WITH_ASR=1 also bakes in a -# specific speech model's weights at a pinned revision (see ASR_MODEL/ASR_REVISION below). -# Decisions do not use speech, so serve.sh switches it off unless SYSTEMONE_SPEECH_INPUT=1; -# build with WITH_ASR=1 for an image that has the speech model too: -# docker build --platform linux/amd64 --build-arg WITH_ASR=1 \ -# -t /granite-switch-systemone:-asr . -# vLLM 0.30, the version this plugin is tested on live (granite-switch 0.1.1 supports -# 0.26-0.30). The default tag is the CUDA 13 build (torch 2.13.0+cu130, the PyPI vLLM -# wheel) -- the same stack the live tests ran. The -cu129 tag ships torch 2.14.0, -# which the vLLM wheel's own torch==2.13.0 pin contradicts, so granite-switch[vllm] -# has no resolution against it. +# The audio libraries are in every build. WITH_ASR=1 also bakes in the speech model (see +# ASR_MODEL and ASR_REVISION); serve.sh keeps speech off unless SYSTEMONE_SPEECH_INPUT=1. +# docker build --build-arg WITH_ASR=1 -t /granite-switch-systemone:-asr . +# +# vLLM 0.30 is the version tested live. The default tag is the CUDA 13 build (torch +# 2.13.0+cu130), the stack the tests ran. The -cu129 tag ships torch 2.14.0, which the +# vLLM wheel's own torch==2.13.0 pin rules out, so granite-switch[vllm] cannot resolve there. FROM vllm/vllm-openai:v0.30.0 SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG GRANITE_SWITCH_VERSION=0.1.1 ARG WITH_ASR=0 -# Pinned to a specific, released granite-switch version (not `main`) so this image's -# behavior does not drift when upstream changes -- deliberately a tagged PyPI release, -# not a floating branch. transformers 5.17 is what granite-switch supports. The [audio] -# extra (av, scipy, soundfile, soxr: the libraries vLLM needs to decode audio) is -# installed in every variant, so SYSTEMONE_SPEECH_INPUT=1 works with a mounted speech -# model; without them every audio request fails with HTTP 500 "install vllm[audio]" while -# health and decisions look fine. WITH_ASR=1 only adds the baked-in speech model below. -# The base image's vLLM, torch, and torchcodec (CUDA 13 builds) are kept as they are: the -# constraint file pins them so nothing in granite-switch's dependency tree can move them. +# granite-switch is pinned to a PyPI release so the image does not drift with upstream; +# transformers 5.17 is the version it supports. The [audio] extra (av, scipy, soundfile, +# soxr) is in every build: without it audio requests fail with HTTP 500 while /health and +# decisions look fine. The constraint file keeps the base image's vLLM, torch and +# torchcodec as they are. RUN python3 -m pip list --format=freeze | grep -i -E "^(vllm|torch|torchaudio|torchcodec|torchvision|flashinfer-python)==" > /tmp/keep.txt \ && uv pip install --system --constraint /tmp/keep.txt \ "granite-switch[audio]==${GRANITE_SWITCH_VERSION}" \ "transformers==5.17.0" \ && python3 -c "import granite_switch.vllm, vllm, transformers, torch, av, scipy, soundfile, soxr; print(vllm.__version__, transformers.__version__, torch.__version__)" -# The ASR cascade's speech model, named by Hugging Face id in the checkpoint's config, -# baked in as a Hub cache at the revision this image was tested with, without the -# repo's unused .py files. The server runs with the Hub offline, so a missing/moved file -# fails startup instead of silently fetching something else. -# The pinned commit is downloaded by hash, not from `main`, so the build does not change when -# the model repository moves. A download by commit writes no refs/main, and the offline lookup -# for the revision "main" reads that file, so it is written here and then checked. +# The speech model the checkpoint's config names, baked in as a Hub cache at the tested +# commit (without the repo's unused .py files). The server runs with the Hub offline, so a +# missing file fails startup instead of fetching something else. A download by commit +# writes no refs/main, which the offline lookup for "main" reads, so it is written here. ARG ASR_MODEL=ibm-granite/granite-speech-5.0-470m-turboctc ARG ASR_REVISION=1dd23f46f6d4e6eae2531982ba2e33220086dd97 RUN if [ "$WITH_ASR" = "1" ]; then \ @@ -55,18 +42,15 @@ RUN if [ "$WITH_ASR" = "1" ]; then \ fi ENV HF_HUB_CACHE=/opt/hf-cache HF_HUB_OFFLINE=1 -# The jev system-one API (/v1/systemone), a vllm.endpoint_plugins entry point mounted -# on this same server -- see README.md and src/granite_switch_systemone/plugin.py. +# The plugin: a vllm.endpoint_plugins entry point (see src/granite_switch_systemone/plugin.py). COPY pyproject.toml README.md /opt/granite-switch-systemone/ COPY src /opt/granite-switch-systemone/src RUN uv pip install --system --no-deps /opt/granite-switch-systemone COPY serve.sh /usr/local/bin/serve.sh RUN chmod +x /usr/local/bin/serve.sh -# Endpoint plugins only load if VLLM_PLUGINS is set AND names them (stricter than -# other vLLM plugin groups, which default to "load everything" when VLLM_PLUGINS is -# unset): must include register_granite_switch too, or the Switch model class stops -# being registered the moment this allowlist is set at all. +# Endpoint plugins load only when VLLM_PLUGINS names them, and setting it stops vLLM loading +# any others, so register_granite_switch has to be listed too. ENV VLLM_PLUGINS=register_granite_switch,granite_switch_systemone \ SYSTEMONE_ADAPTER_NAME=decision \ SYSTEMONE_INVOCATION_MARKER="### DECISION" diff --git a/src/granite_switch_systemone/plugin.py b/src/granite_switch_systemone/plugin.py index d0359a7..11d6381 100644 --- a/src/granite_switch_systemone/plugin.py +++ b/src/granite_switch_systemone/plugin.py @@ -58,10 +58,8 @@ def _load_renderer(): - """Resolve `SYSTEMONE_RENDERER` (default: this package's own placeholder) to a - callable at plugin-init time -- never imported directly by this module, so this - package has zero import-time dependency on whatever module the configured path - names (see module docstring).""" + """Resolve `SYSTEMONE_RENDERER` to a callable at plugin init. It is loaded by name, so + this module never imports the configured renderer directly.""" path = os.environ.get(_RENDERER_ENV, _DEFAULT_RENDERER) if ":" not in path: raise RuntimeError(f"{_RENDERER_ENV}={path!r} is not a valid 'module:function' path") @@ -237,10 +235,9 @@ class DecisionInputMessage(BaseModel): class DecisionRequest(BaseModel): - """OpenAI's Decisions request body. `prompt_cache_key`/`prompt_cache_options`/ - `safety_identifier` are accepted (the official SDKs send them) and ignored -- the - engine's prefix cache is shared automatically, and access control belongs to - whatever hosts this container, the same stance as the systemone route's 401/429.""" + """OpenAI's Decisions request body. `prompt_cache_key`, `prompt_cache_options` and + `safety_identifier` are accepted, since the official SDKs send them, and ignored: + the prefix cache is shared automatically and access control belongs to the host.""" model_config = ConfigDict(extra="forbid") @@ -634,10 +631,9 @@ async def _fetch_label_logprobs( msgs: list[dict[str, str]], label_ids_chunk: list[int], ): - """One `create_chat_completion` call asking for exactly `label_ids_chunk`'s - logprobs, for the given (already fully rendered) `msgs`. Returns the raw - `ChatCompletionResponse`; callers combine multiple chunks' responses when a - question needs more than `MAX_LABEL_OPTIONS` labels.""" + """One `create_chat_completion` call for the logprobs of `label_ids_chunk` over the + rendered `msgs`. Returns the raw response; a question with more than + `MAX_LABEL_OPTIONS` labels needs several.""" from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionRequest, ChatCompletionResponse, @@ -824,11 +820,9 @@ async def _cancel_pending(tasks: list[asyncio.Task]) -> None: def _decisions_input_to_state(data: str | list[DecisionInputMessage]) -> str: - """Flatten OpenAI's `input` (a string, or user messages whose content is a string - or text/image parts) into the state string both routes share: message texts joined - with blank lines, the same flattening LiteLLM uses when translating between the two - formats. Image parts are rejected with a clear error -- this is a text decision - adapter, the same stance as the systemone route's `box`.""" + """Flatten OpenAI's `input` (a string, or user messages with text/image parts) into the + state string both routes share: message texts joined with blank lines. Image parts + are rejected, as `box` questions are on the systemone route.""" if isinstance(data, str): return data texts: list[str] = [] @@ -866,11 +860,10 @@ def _decisions_question_to_internal(question: DecisionQuestion, index: int) -> t def _internal_answer_to_decisions( question: DecisionQuestion, extras: dict[str, Any], answer: dict[str, Any] ) -> dict[str, Any]: - """Map one internal (jev-shaped) answer onto OpenAI's Decisions answer shapes: - `predicate` -> `probability` (the internal noul answer is already P(true)); - `choice` -> the picked value plus per-value probabilities and the shared confidence - statistic; `score` -> the expected level over per-level probabilities, carrying the - ORIGINAL level labels (the internal legend holds the descriptions we rendered).""" + """Map one internal (jev-shaped) answer onto OpenAI's Decisions answer. `predicate` + gives P(true), `choice` the picked value with per-value probabilities and confidence, + and `score` the expected level, labelled with the caller's original level labels + (the internal legend holds the rendered descriptions).""" if question.type == "predicate": return {"type": "predicate", "name": question.name, "probability": answer["noul"]} if question.type == "choice": @@ -1078,9 +1071,8 @@ class SystemOnePlugin: required_tasks: tuple[str, ...] | None = ("generate",) def attach_router(self, app: FastAPI) -> None: - # vLLM now ships /v1/systemone and /v1/decisions itself. Whichever is registered - # first would answer and the other would be shadowed silently, with different - # wire details, so refuse to start instead. + # vLLM also ships these routes. Whichever registers first would silently shadow + # the other, with different wire details, so refuse to start. taken = sorted(path for path in _ROUTE_PATHS if _route_exists(app, path)) if taken: raise RuntimeError(