From afed9e0312392a06a7ade5c9d70012b8c3c0ea37 Mon Sep 17 00:00:00 2001 From: Krisztian Gajdar Date: Wed, 14 Oct 2026 23:53:43 +0200 Subject: [PATCH 1/3] feat(server): let the fake engine answer chat with scripted tool calls, and run the agent-loop check in CI The weightless fake had no chat template, so /v1/chat/completions answered 501 for it and the OpenCode example's conformance check could only run against a GPU-backed model. The fake adapter now implements the generation adapter's chat methods with a deterministic script: with tools declared it calls the named or first tool, passing the last user message as every required argument; after tool results it answers with them; otherwise it answers hash-derived tokens. When the profile enables thinking, one sentence of reasoning comes first and is returned only when asked. A prompt token is a word and an output token is a streamed piece, output stops at the token limit with finish_reason length, and a prompt that leaves no room in the window is refused. The single-node chat route sends fake profiles through the same adapter-owned chat path as remote profiles, so the fake's output passes the route's real validation, tool-name, usage and reasoning checks. A new sie-fake:chat profile declares thinking, a 131,072-token window and a 4,096-token output cap. The example gains a test that starts the repository's sie-server with the fake bundle on the CPU and asserts each conformance case: 9 pass, parallel_tool_calls fails because the fake makes one call per turn, and tool_choice_required is recorded as supported. --- examples/coding-agent-opencode/README.md | 56 ++++- .../tests/test_fake_engine.py | 172 +++++++++++++++ packages/sie_server/models/sie-fake.yaml | 18 +- .../src/sie_server/adapters/fake/adapter.py | 206 +++++++++++++++++- .../src/sie_server/api/openai_local.py | 18 +- .../tests/adapters/test_fake_chat.py | 111 ++++++++++ .../tests/api/test_fake_chat_route.py | 193 ++++++++++++++++ 7 files changed, 753 insertions(+), 21 deletions(-) create mode 100644 examples/coding-agent-opencode/tests/test_fake_engine.py create mode 100644 packages/sie_server/tests/adapters/test_fake_chat.py create mode 100644 packages/sie_server/tests/api/test_fake_chat_route.py diff --git a/examples/coding-agent-opencode/README.md b/examples/coding-agent-opencode/README.md index 3c118f118..c2ff3f1a0 100644 --- a/examples/coding-agent-opencode/README.md +++ b/examples/coding-agent-opencode/README.md @@ -385,13 +385,56 @@ with an output cap of 8,192 tokens and `tool_choice: "auto"`. ## Checked without a GPU -`sie-server` ships a weightless `sie-fake` model for CI, but it has no -chat-template child: `/v1/chat/completions` answers `501 -unsupported_operation` for it. So the conformance check cannot run against -`sie-fake`. +`sie-server` ships a weightless `sie-fake` model for CI. Its +`sie-fake:chat` profile answers `/v1/chat/completions` from a script, not a +model. With tools declared, a turn that ends in a user message calls the +first tool, passing that message as every required argument. A turn that ends +in a `role: tool` message answers `Tool result: ` followed by the result. Any +other turn answers with hash-derived tokens. Every turn starts with one +sentence of reasoning, returned as `reasoning_content` only with +`include_reasoning`. A token is one word of the prompt or one streamed piece +of the answer, and the profile declares a 131,072-token window. So the +conformance check runs against a real `sie-server` without a GPU or weights: + +```sh +mise run serve -- -b fake --device cpu # from the repository root +uv run python conformance.py --model sie-fake:chat +``` + +``` + tool_choice_auto pass tool_choice=auto accepted on the tools turn + streaming_tool_call_deltas pass 1 call: get_weather({"city": "What is the weather in Paris right now? Use the get_weather tool."}); 14 deltas; finish_reason=tool_calls + parallel_tool_calls fail 1 tool call in the turn: get_weather({"city": "Get the current weather in Paris and in Tokyo. Call get_weather once per city, both in this turn."}) + tool_result_round_trip pass final answer used the tool result: 'Tool result: {"city": "Paris", "temperature_c": 18, "conditions": "overcast"}' + prompt_above_64k_tokens pass 68022 prompt tokens accepted and answered; needle not found + finish_reason_stop pass finish_reason=stop after 'tok_b0437c31 tok_5aaf6099 tok_f3b6ad77 t' + finish_reason_tool_calls pass finish_reason=tool_calls on the tool-call turn + finish_reason_length pass finish_reason=length at max_tokens=8 (completion_tokens=8) + usage_on_final_chunk pass final chunk carries usage {'completion_tokens': 40, 'prompt_tokens': 7, 'total_tokens': 47} + include_reasoning pass 8 reasoning_content deltas, 40 chars; message.reasoning_content 40 chars; none without the flag + tool_choice_required recorded supported: forced get_weather({"city": "Say hello."}) + +9 pass, 1 fail, 1 recorded +``` + +- A pass here checks SIE's single-node chat wire, not a model: the streamed + call deltas, the tool result sent back, reasoning returned only on request, + the finish reasons, the usage on the last chunk and the declared window. +- `parallel_tool_calls` fails because the fake makes one call per turn. +- The fake does not read the prompt, so the needle is not found, and the + answer to "Reply with the single word: ready." is hash-derived tokens. What runs anywhere, and in this repository's CI, is the test suite: +- `tests/test_fake_engine.py` starts this repository's `sie-server` with the + fake bundle on the CPU and runs `conformance.py` against `sie-fake:chat`. It + asserts the table above case by case, the failing `parallel_tool_calls` + included, and what each case observed: the call's id and name on its first + delta, the tool result in the answer, reasoning only with the flag and the + same usage without it, the long prompt's size against the window, and + `max_tokens: 8` spending exactly 8 tokens. It needs the repository's + environment, from `mise run sync` at the root, and is skipped without it + outside CI. - `tests/test_conformance.py` starts a scripted OpenAI-compatible stub that plays a conforming wire, then breaks one thing per test: tool-call deltas without ids, one call where two were asked for, `tool_choice: "required"` @@ -415,8 +458,9 @@ What runs anywhere, and in this repository's CI, is the test suite: uv run pytest ``` -That is a check of the checker. The endpoint itself is only checked by running -`conformance.py` against it. +`tests/test_conformance.py` and `tests/test_run_gate.py` check the checker, +and `tests/test_fake_engine.py` checks SIE's chat route with a scripted model. +A model's endpoint is only checked by running `conformance.py` against it. ## What this does NOT establish diff --git a/examples/coding-agent-opencode/tests/test_fake_engine.py b/examples/coding-agent-opencode/tests/test_fake_engine.py new file mode 100644 index 000000000..0aff93034 --- /dev/null +++ b/examples/coding-agent-opencode/tests/test_fake_engine.py @@ -0,0 +1,172 @@ +"""``conformance.py`` against the repository's ``sie-server`` serving the weightless fake. + +The server runs the fake bundle on the CPU, and the check runs against its +``sie-fake:chat`` profile the way the README runs it against a real model. The +fake answers from a script: with tools declared it calls the first one, passing +the last user message as every required argument, after a ``role: tool`` +message it answers with the tool result, and it reasons in one sentence first. +So each case here checks SIE's single-node chat wire, not a model. + +The fake makes one call per turn, so ``parallel_tool_calls`` fails, and the +test asserts that failure instead of skipping the case. The fake does not read +the long prompt, so the needle is never found; the case passes on the prompt +size and the answer, as it does for any model. +""" + +from __future__ import annotations + +import json +import os +import socket +import subprocess +import time +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +import httpx +import pytest + +import conformance + +EXAMPLE = Path(__file__).resolve().parents[1] +REPOSITORY = EXAMPLE.parents[1] +SERVER = REPOSITORY / ".venv" / "bin" / "sie-server" +MODEL = "sie-fake:chat" +WINDOW = 131072 +SERVER_START_TIMEOUT_S = 300.0 +TOOL_RESULT = {"city": "Paris", "temperature_c": 18, "conditions": "overcast"} + + +def _free_port() -> int: + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + return probe.getsockname()[1] + + +def _wait_for_health(base: str, server: subprocess.Popen[bytes], log: Path) -> None: + deadline = time.monotonic() + SERVER_START_TIMEOUT_S + while time.monotonic() < deadline: + if server.poll() is not None: + pytest.fail(f"sie-server exited {server.returncode}:\n{log.read_text()[-4000:]}") + try: + if httpx.get(f"{base}/healthz", timeout=2.0).status_code == 200: + return + except httpx.HTTPError: + pass + time.sleep(0.5) + pytest.fail(f"sie-server did not answer /healthz within {SERVER_START_TIMEOUT_S}s:\n{log.read_text()[-4000:]}") + + +@pytest.fixture(scope="module") +def server(tmp_path_factory: pytest.TempPathFactory) -> Iterator[str]: + if not SERVER.exists(): + message = f"{SERVER} is missing; run `mise run sync` at the repository root" + if os.environ.get("CI"): + pytest.fail(message) + pytest.skip(message) + port = _free_port() + env = {key: value for key, value in os.environ.items() if key != "VIRTUAL_ENV"} + log = tmp_path_factory.mktemp("fake-engine") / "sie-server.log" + command = [str(SERVER), "serve", "--host", "127.0.0.1", "--port", str(port), "--device", "cpu", "-b", "fake"] + with log.open("wb") as output: + process = subprocess.Popen(command, env=env, stdout=output, stderr=subprocess.STDOUT) + try: + base = f"http://127.0.0.1:{port}" + _wait_for_health(base, process, log) + yield base + finally: + process.terminate() + try: + process.wait(timeout=30) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +@pytest.fixture(scope="module") +def report(server: str) -> dict[str, Any]: + return conformance.run(f"{server}/v1", "", MODEL) + + +def _cases(report: dict[str, Any]) -> dict[str, dict[str, Any]]: + return {case["name"]: case for case in report["cases"]} + + +def test_each_case_against_the_fake(report: dict[str, Any]) -> None: + statuses = {name: case["status"] for name, case in _cases(report).items()} + assert statuses == { + "tool_choice_auto": "pass", + "streaming_tool_call_deltas": "pass", + "parallel_tool_calls": "fail", + "tool_result_round_trip": "pass", + "prompt_above_64k_tokens": "pass", + "finish_reason_stop": "pass", + "finish_reason_tool_calls": "pass", + "finish_reason_length": "pass", + "usage_on_final_chunk": "pass", + "include_reasoning": "pass", + "tool_choice_required": "recorded", + }, report["cases"] + assert report["served_by"] == "local" + assert report["warm_up"]["error"] is None + assert report["model_entry"]["max_sequence_length"] == WINDOW + assert report["tools_turn_params"] == {"tool_choice": "auto", "extra_body": {"include_reasoning": True}} + + +def test_the_fake_makes_one_call_per_turn(report: dict[str, Any]) -> None: + case = _cases(report)["parallel_tool_calls"] + assert case["detail"].startswith("1 tool call in the turn: get_weather("), case + assert [call["name"] for call in case["observed"]["tool_calls"]] == ["get_weather"] + + +def test_the_tool_call_streams_its_id_and_name_first(report: dict[str, Any]) -> None: + observed = _cases(report)["streaming_tool_call_deltas"]["observed"] + [call] = observed["tool_calls"] + question = "What is the weather in Paris right now? Use the get_weather tool." + assert call["first_delta_has_id_and_name"] is True + assert call["deltas"] > 1 + assert json.loads(call["arguments"]) == {"city": question} + assert observed["finish_reason"] == "tool_calls" + assert observed["reasoning_deltas"] > 0 + + +def test_the_tool_result_reaches_the_answer(report: dict[str, Any]) -> None: + observed = _cases(report)["tool_result_round_trip"]["observed"] + assert observed["content"] == f"Tool result: {json.dumps(TOOL_RESULT)}" + assert observed["tool_calls"] == [] + assert observed["reasoning_content_sent_back"] + + +def test_reasoning_comes_back_only_when_asked(report: dict[str, Any]) -> None: + observed = _cases(report)["include_reasoning"]["observed"] + with_flag = observed["with_include_reasoning"] + without_flag = observed["without_include_reasoning"] + assert observed["model_reasoned"] is True + assert with_flag["reasoning_deltas"] > 0 + assert without_flag["reasoning_deltas"] == 0 + assert observed["non_streamed_with_include_reasoning"]["reasoning_chars"] == with_flag["reasoning_chars"] + assert with_flag["usage"] == without_flag["usage"] + + +def test_the_long_prompt_fits_the_declared_window(report: dict[str, Any]) -> None: + observed = _cases(report)["prompt_above_64k_tokens"]["observed"] + prompt_tokens = observed["usage"]["prompt_tokens"] + assert conformance.LONG_PROMPT_MIN_TOKENS < prompt_tokens < WINDOW + assert observed["declared_window"] == WINDOW + assert observed["needle_found"] is False + assert observed["finish_reason"] == "stop" + + +def test_max_tokens_and_usage_are_exact(report: dict[str, Any]) -> None: + cases = _cases(report) + length = cases["finish_reason_length"]["observed"] + assert length["usage"]["completion_tokens"] == 8 + usage = cases["usage_on_final_chunk"]["observed"]["usage"] + assert usage["total_tokens"] == usage["prompt_tokens"] + usage["completion_tokens"] + + +def test_tool_choice_required_forces_a_call(report: dict[str, Any]) -> None: + observed = _cases(report)["tool_choice_required"]["observed"] + assert observed["support"] == "supported" + assert json.loads(observed["tool_calls"][0]["arguments"]) == {"city": "Say hello."} diff --git a/packages/sie_server/models/sie-fake.yaml b/packages/sie_server/models/sie-fake.yaml index 071e4c727..71b72d6ed 100644 --- a/packages/sie_server/models/sie-fake.yaml +++ b/packages/sie_server/models/sie-fake.yaml @@ -1,6 +1,7 @@ ## The Fake Engine catalog entry (#1836): ONE model, every supported surface -# (encode + score + generate via the combined FakeAdapter), with profiles for -# the different cases. Non-default profiles are addressable as +# (encode + score + generate via the combined FakeAdapter, and scripted chat on +# the single-node /v1/chat/completions route), with profiles for the different +# cases. Non-default profiles are addressable as # `sie-fake:` through the loader's variant expansion; each variant is # a separate registry model with its own adapter instance, footprint, and # fault spec. @@ -12,6 +13,8 @@ # slow-load — load holds the MODEL_LOADING window for 5 s. # oom-3rd — the 3rd dispatch raises an OOM the real is_oom_error # classifier recognizes; the recovery ladder retries. +# chat — chat with reasoning (enable_thinking), a 131,072-token window +# and a 4,096-token output cap, for agent-loop wire checks. # # Note: adapter_options replacement is all-or-nothing per profile (non-empty # child loadtime REPLACES the parent's), so every profile repeats the full @@ -84,3 +87,14 @@ profiles: faults: oom_on_dispatch: 3 runtime: {} + chat: + extends: default + max_output_tokens: 4096 + chat_template_kwargs: + enable_thinking: true + adapter_options: + loadtime: + memory_footprint_bytes: 67108864 + fault_key: sie-fake:chat + max_seq_length: 131072 + runtime: {} diff --git a/packages/sie_server/src/sie_server/adapters/fake/adapter.py b/packages/sie_server/src/sie_server/adapters/fake/adapter.py index d6daba16e..4a70c120e 100644 --- a/packages/sie_server/src/sie_server/adapters/fake/adapter.py +++ b/packages/sie_server/src/sie_server/adapters/fake/adapter.py @@ -19,9 +19,10 @@ (``package_backed``, no HF weights) — served by the combined :class:`FakeAdapter` (encode + score + generate). Different cases are profiles of that one model, addressable as ``sie-fake:`` via the -loader's variant expansion. Chat completions return ``invalid_request`` — -chat-template rendering needs a real tokenizer source; ``/v1/generate`` -(raw prompt) is the supported generation surface. +loader's variant expansion. The fake has no chat template: the single-node +``/v1/chat/completions`` route gets a scripted answer from +:meth:`FakeAdapter.chat_completion`, and queued chat, which renders a chat +template, is refused. ``/v1/generate`` (raw prompt) works on every path. A model config that declares sparse or multivector dimensions, or an extract task, also gets those outputs from the same adapter, and image items are @@ -39,6 +40,7 @@ import os import re import time +import uuid from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar @@ -46,7 +48,7 @@ import numpy as np from sie_server.adapters._base_adapter import BaseAdapter -from sie_server.adapters._generation_base import GenerationAdapter, GenerationChunk +from sie_server.adapters._generation_base import GenerationAdapter, GenerationChunk, GenerationInputTooLongError from sie_server.adapters._spec import AdapterSpec from sie_server.adapters._types import ERR_NOT_LOADED from sie_server.core.inference_output import ( @@ -273,7 +275,17 @@ def _hash_unit_interval(seed: str) -> float: return int.from_bytes(digest[:8], "big") / 2**64 +def _hash_tokens(seed: str, count: int) -> list[str]: + """``count`` hash-derived tokens, each but the last followed by a space.""" + tokens = [] + for i in range(count): + fragment = hashlib.sha256(f"{seed}\x00{i}".encode()).hexdigest()[:8] + tokens.append(f"tok_{fragment}" + ("" if i == count - 1 else " ")) + return tokens + + _WORD = re.compile(r"\S+") +_PIECE = re.compile(r"\S+\s*") _SPARSE_NONZEROS = 8 _TOKENS_PER_IMAGE = 4 @@ -356,6 +368,61 @@ def _hash_extraction(item: Item, labels: list[str]) -> _FakeExtraction: ) +def _message_text(message: dict[str, Any]) -> str: + content = message.get("content") + if isinstance(content, list): + return " ".join(part.get("text") or "" for part in content) + return content or "" + + +def _chat_transcript(body: dict[str, Any]) -> str: + """Each declared tool, then each message's role, text and tool calls, one per line.""" + lines = [json.dumps(tool, sort_keys=True) for tool in body.get("tools") or []] + for message in body["messages"]: + calls = json.dumps(message["tool_calls"]) if message.get("tool_calls") else "" + lines.append(f"{message['role']} {_message_text(message)} {calls}") + return "\n".join(lines) + + +@dataclass(frozen=True, slots=True) +class _ChatTurn: + """One scripted answer cut to its token budget: one ``(kind, text)`` step per output token.""" + + steps: list[tuple[str, str]] + call_id: str + call_name: str | None + finish_reason: str + prompt_tokens: int + choices: int + + @property + def usage(self) -> dict[str, int]: + completion = len(self.steps) * self.choices + return { + "prompt_tokens": self.prompt_tokens, + "completion_tokens": completion, + "total_tokens": self.prompt_tokens + completion, + } + + def delta(self, kind: str, text: str) -> dict[str, Any]: + if kind == "call": + function = {"name": self.call_name, "arguments": ""} + return {"tool_calls": [{"index": 0, "id": self.call_id, "type": "function", "function": function}]} + if kind == "arguments": + return {"tool_calls": [{"index": 0, "function": {"arguments": text}}]} + return {"reasoning_content" if kind == "reasoning" else "content": text} + + def message(self, *, keep_reasoning: bool) -> dict[str, Any]: + text = {kind: "".join(t for k, t in self.steps if k == kind) for kind in ("reasoning", "arguments", "content")} + message: dict[str, Any] = {"role": "assistant", "content": text["content"]} + if keep_reasoning and text["reasoning"]: + message["reasoning_content"] = text["reasoning"] + if self.call_name is not None: + function = {"name": self.call_name, "arguments": text["arguments"]} + message["tool_calls"] = [{"id": self.call_id, "type": "function", "function": function}] + return message + + class FakeAdapter(BaseAdapter, GenerationAdapter): """One weightless fake serving every supported surface: hash-derived embeddings (``encode``), pair scores in ``[0, 1)`` (``score``), @@ -376,6 +443,19 @@ class FakeAdapter(BaseAdapter, GenerationAdapter): and ``logprobs`` are accepted and ignored — determinism comes from the input alone. ``finish_reason`` is ``"length"`` when truncated by ``max_new_tokens``, else ``"stop"``. + + Chat follows a script, for wire tests only. With tools declared and + ``tool_choice`` not ``"none"``, a turn calls one tool, the named one or + else the first, unless its last message is a tool result and the call is + not forced. Every required parameter gets the last user message's text. + A turn ending in tool results answers ``Tool result: `` for + each, and any other turn answers ``default_completion_tokens`` hash + tokens. When ``chat_template_kwargs`` enables thinking, one sentence of + reasoning naming the rule comes first, returned only with + ``keep_reasoning``. A prompt token is a whitespace-separated word of the + tools and messages, and an output token is one streamed piece. Output + stops at the token limit with ``"length"``, a prompt that leaves no room + in ``max_seq_length`` is refused, and ``n`` repeats the turn. """ spec: ClassVar[AdapterSpec] = AdapterSpec( @@ -402,7 +482,8 @@ def __init__( fault_key: str | None = None, **kwargs: Any, ) -> None: - _ = (model_name_or_path, max_seq_length, compute_precision, kwargs) + _ = (model_name_or_path, compute_precision, kwargs) + self._max_seq_length = max_seq_length self._dense_dim = dense_dim self._sparse_dim = sparse_dim self._multivector_dim = multivector_dim @@ -598,12 +679,10 @@ async def generate( # Char-count proxy, mirroring the reserve-estimate basis used when no # real tokenizer exists. prompt_tokens = max(1, len(prompt) // 4) - for i in range(count): + for i, token in enumerate(_hash_tokens(prompt, count)): if self._inter_token_latency_s > 0: await asyncio.sleep(self._inter_token_latency_s) - fragment = hashlib.sha256(f"{prompt}\x00{i}".encode()).hexdigest()[:8] - separator = "" if i == count - 1 else " " - yield GenerationChunk(text_delta=f"tok_{fragment}{separator}", is_first=(i == 0)) + yield GenerationChunk(text_delta=token, is_first=(i == 0)) yield GenerationChunk( text_delta="", done=True, @@ -611,3 +690,112 @@ async def generate( prompt_tokens=prompt_tokens, completion_tokens=count, ) + + async def _chat_turn(self, body: dict[str, Any]) -> _ChatTurn: + if not self._loaded: + raise RuntimeError(ERR_NOT_LOADED) + await self._fault_runtime.on_dispatch_async() + transcript = _chat_transcript(body) + prompt_tokens = len(transcript.split()) + budget = body.get("max_completion_tokens") or body["max_tokens"] + if self._max_seq_length is not None: + room = self._max_seq_length - prompt_tokens + if room < 1: + msg = f"the prompt is {prompt_tokens} tokens and leaves no room in the {self._max_seq_length}-token window" + raise GenerationInputTooLongError(msg) + budget = min(budget, room) + messages = body["messages"] + tools = [tool["function"] for tool in body.get("tools") or []] + choice = body.get("tool_choice") + named = choice.get("function", {}).get("name") if isinstance(choice, dict) else None + forced = choice == "required" or named is not None + results: list[str] = [] + for message in reversed(messages): + if message["role"] != "tool": + break + results.insert(0, _message_text(message)) + call_name: str | None = None + arguments = "" + content: list[str] = [] + if tools and choice != "none" and (forced or not results): + function = next((tool for tool in tools if tool["name"] == named), tools[0]) + call_name = function["name"] + question = next((_message_text(m) for m in reversed(messages) if m["role"] == "user"), "") + required = (function.get("parameters") or {}).get("required") or [] + arguments = json.dumps(dict.fromkeys(required, question), ensure_ascii=False) + reason = "tool_choice requires a call" if forced else "the last message is not a tool result" + thought = f"Tools are declared and {reason}, so call {call_name}." + elif results: + thought = "The last message is a tool result, so answer with it." + content = _PIECE.findall(" ".join(f"Tool result: {result}" for result in results)) + else: + thought = "No tool call applies, so answer in text." + content = _hash_tokens(transcript, self._default_completion_tokens) + steps: list[tuple[str, str]] = [] + if (body.get("chat_template_kwargs") or {}).get("enable_thinking") is True: + steps += [("reasoning", piece) for piece in _PIECE.findall(thought)] + if call_name is not None: + steps.append(("call", "")) + steps += [("arguments", piece) for piece in _PIECE.findall(arguments)] + steps += [("content", piece) for piece in content] + kept = steps[:budget] + return _ChatTurn( + steps=kept, + call_id="call_" + hashlib.sha256(f"{transcript}\x00call".encode()).hexdigest()[:24], + call_name=call_name if ("call", "") in kept else None, + finish_reason="length" if len(steps) > budget else "tool_calls" if call_name else "stop", + prompt_tokens=prompt_tokens, + choices=body.get("n") or 1, + ) + + async def chat_completion( + self, + body: dict[str, Any], + *, + requested_model: str, + max_response_bytes: int = 32 << 20, + keep_reasoning: bool = False, + ) -> dict[str, Any]: + _ = max_response_bytes + turn = await self._chat_turn(body) + return { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion", + "created": int(time.time()), + "model": requested_model, + "choices": [ + { + "index": index, + "message": turn.message(keep_reasoning=keep_reasoning), + "finish_reason": turn.finish_reason, + } + for index in range(turn.choices) + ], + "usage": turn.usage, + } + + async def chat_completion_stream( + self, + body: dict[str, Any], + *, + requested_model: str, + max_response_bytes: int = 32 << 20, + keep_reasoning: bool = False, + ) -> AsyncIterator[dict[str, Any]]: + _ = max_response_bytes + turn = await self._chat_turn(body) + envelope = { + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": requested_model, + } + for index in range(turn.choices): + deltas = [{"role": "assistant"}] + deltas += [turn.delta(kind, text) for kind, text in turn.steps if keep_reasoning or kind != "reasoning"] + for delta in deltas: + if self._inter_token_latency_s > 0: + await asyncio.sleep(self._inter_token_latency_s) + yield {**envelope, "choices": [{"index": index, "delta": delta, "finish_reason": None}]} + yield {**envelope, "choices": [{"index": index, "delta": {}, "finish_reason": turn.finish_reason}]} + yield {**envelope, "choices": [], "usage": turn.usage} diff --git a/packages/sie_server/src/sie_server/api/openai_local.py b/packages/sie_server/src/sie_server/api/openai_local.py index a43bec7d5..bb2113504 100644 --- a/packages/sie_server/src/sie_server/api/openai_local.py +++ b/packages/sie_server/src/sie_server/api/openai_local.py @@ -13,6 +13,7 @@ generation does (:mod:`sie_server.processors.schema_prompt`). Remote generation profiles use their adapter's normalized chat methods instead; the operator-configured upstream owns chat rendering, and the worker retains validation and output policy. + The weightless fake answers through the same methods from a script. - ``/v1/rerank`` wraps the in-process score adapter in the Cohere/OpenAI rerank shape (``{query, documents, top_n}`` -> ``{results: [{index, relevance_score}]}``). @@ -55,6 +56,7 @@ thinking_blocks_must_be_hidden, ) from sie_server.adapters.errors import InputTooLongError, UpstreamUnavailableError +from sie_server.adapters.fake.adapter import FakeAdapter from sie_server.adapters.mlx.generation import MLXGenerationAdapter, normalize_mlx_seed from sie_server.adapters.remote._http import RemoteUpstreamError from sie_server.adapters.sglang.generation import SGLangGenerationAdapter @@ -134,6 +136,7 @@ _MAX_CHAT_CHOICES = 128 _MAX_CHAT_VIDEOS = 1 _MAX_U32 = (1 << 32) - 1 +_FAKE_ADAPTER_PATH = f"{FakeAdapter.__module__}:{FakeAdapter.__name__}" _ALLOWED_CHAT_ROLES = frozenset({"system", "user", "assistant", "tool", "developer"}) _ANSWER_FINISH_REASONS = frozenset({"stop", "tool_calls"}) # Compatibility requests project onto the same native score bound. @@ -280,6 +283,13 @@ async def _read_json_body(http_request: Request) -> dict[str, Any]: return body +def _adapter_answers_chat(config: Any) -> bool: + """Whether the adapter answers chat through its own chat methods rather than a managed child.""" + return serves_remotely(config) or ( + "default" in config.profiles and config.resolve_profile("default").adapter_path == _FAKE_ADAPTER_PATH + ) + + def _unsupported_chat(model: str, device: object) -> HTTPException: return HTTPException( status_code=status.HTTP_501_NOT_IMPLEMENTED, @@ -1564,8 +1574,8 @@ async def _chat_completions( ) # The child decodes video on its event loop; bound that work here. await asyncio.to_thread(_probe_chat_videos, videos) - remote_chat = serves_remotely(config) - if str(device).startswith("cuda") or remote_chat: + adapter_chat = _adapter_answers_chat(config) + if str(device).startswith("cuda") or adapter_chat: _validate_cuda_chat_body(body) else: _validate_mlx_chat_body(body) @@ -1576,7 +1586,7 @@ async def _chat_completions( or ("include_usage" in options and not isinstance(options["include_usage"], bool)) ): raise _bad_request("'stream_options' must contain only boolean 'include_usage'", param="stream_options") - if remote_chat: + if adapter_chat: _validate_remote_chat_media(messages) # Normalize and reject client errors before the bridge owns a local @@ -1590,7 +1600,7 @@ async def _chat_completions( adapter = registry.get(route.key) registry.touch_lru(route.key) - if serves_remotely(registry.get_config(route.key)) and isinstance(adapter, GenerationAdapter): + if _adapter_answers_chat(registry.get_config(route.key)) and isinstance(adapter, GenerationAdapter): _validate_remote_chat_media(messages) return await _remote_chat_response( adapter, diff --git a/packages/sie_server/tests/adapters/test_fake_chat.py b/packages/sie_server/tests/adapters/test_fake_chat.py new file mode 100644 index 000000000..14a816078 --- /dev/null +++ b/packages/sie_server/tests/adapters/test_fake_chat.py @@ -0,0 +1,111 @@ +"""The fake adapter's scripted chat: properties of the script, never pinned hash tokens.""" + +from __future__ import annotations + +import json +from typing import Any + +import pytest +from sie_server.adapters._generation_base import GenerationInputTooLongError +from sie_server.adapters.fake.adapter import FakeAdapter + + +def _tool(name: str, *required: str) -> dict[str, Any]: + parameters = { + "type": "object", + "properties": {key: {"type": "string"} for key in required}, + "required": list(required), + } + return {"type": "function", "function": {"name": name, "parameters": parameters}} + + +def _body(content: str = "Look it up.", **fields: Any) -> dict[str, Any]: + return {"messages": [{"role": "user", "content": content}], "max_tokens": 256, **fields} + + +@pytest.fixture +def adapter() -> FakeAdapter: + fake = FakeAdapter(max_seq_length=1024) + fake.load("cpu") + return fake + + +async def _streamed(adapter: FakeAdapter, body: dict[str, Any], **options: Any) -> list[dict[str, Any]]: + return [event async for event in adapter.chat_completion_stream(body, requested_model="m", **options)] + + +async def test_the_stream_and_the_answer_agree(adapter: FakeAdapter) -> None: + body = _body(tools=[_tool("lookup", "query")], chat_template_kwargs={"enable_thinking": True}) + answer = await adapter.chat_completion(body, requested_model="m", keep_reasoning=True) + events = await _streamed(adapter, body, keep_reasoning=True) + deltas = [choice["delta"] for event in events for choice in event["choices"]] + pieces = [piece for delta in deltas for piece in delta.get("tool_calls", [])] + [message] = [choice["message"] for choice in answer["choices"]] + + assert "".join(delta.get("reasoning_content", "") for delta in deltas) == message["reasoning_content"] + assert pieces[0]["id"] == message["tool_calls"][0]["id"] + assert ( + "".join(piece["function"]["arguments"] for piece in pieces) == message["tool_calls"][0]["function"]["arguments"] + ) + assert events[-1]["usage"] == answer["usage"] + + +async def test_the_same_request_gets_the_same_answer_and_another_request_another(adapter: FakeAdapter) -> None: + first = await adapter.chat_completion(_body("one"), requested_model="m") + again = await adapter.chat_completion(_body("one"), requested_model="m") + other = await adapter.chat_completion(_body("two"), requested_model="m") + + assert first["choices"] == again["choices"] + assert first["choices"][0]["message"]["content"] != other["choices"][0]["message"]["content"] + + +async def test_a_named_tool_choice_calls_that_tool_with_the_last_user_message(adapter: FakeAdapter) -> None: + body = _body( + "Find flights.", + tools=[_tool("first"), _tool("second", "origin", "destination")], + tool_choice={"type": "function", "function": {"name": "second"}}, + ) + [choice] = (await adapter.chat_completion(body, requested_model="m"))["choices"] + [call] = choice["message"]["tool_calls"] + + assert call["function"]["name"] == "second" + assert json.loads(call["function"]["arguments"]) == {"origin": "Find flights.", "destination": "Find flights."} + + +async def test_a_forced_call_follows_a_tool_result(adapter: FakeAdapter) -> None: + messages = [ + {"role": "user", "content": "Look it up."}, + {"role": "assistant", "content": "", "tool_calls": [{"id": "c", "type": "function", "function": {}}]}, + {"role": "tool", "tool_call_id": "c", "content": "found"}, + ] + tools = [_tool("lookup", "query")] + auto = await adapter.chat_completion({"messages": messages, "max_tokens": 64, "tools": tools}, requested_model="m") + forced = await adapter.chat_completion( + {"messages": messages, "max_tokens": 64, "tools": tools, "tool_choice": "required"}, requested_model="m" + ) + + assert auto["choices"][0]["message"]["content"] == "Tool result: found" + assert forced["choices"][0]["finish_reason"] == "tool_calls" + + +async def test_n_repeats_the_turn_and_its_usage(adapter: FakeAdapter) -> None: + single = await adapter.chat_completion(_body(), requested_model="m") + double = await adapter.chat_completion(_body(n=2), requested_model="m") + events = await _streamed(adapter, _body(n=2)) + + assert [choice["index"] for choice in double["choices"]] == [0, 1] + assert double["choices"][1]["message"] == single["choices"][0]["message"] + assert double["usage"]["completion_tokens"] == 2 * single["usage"]["completion_tokens"] + assert sorted(c["index"] for e in events for c in e["choices"] if c["finish_reason"]) == [0, 1] + + +async def test_a_prompt_that_leaves_no_room_is_refused(adapter: FakeAdapter) -> None: + with pytest.raises(GenerationInputTooLongError): + await adapter.chat_completion(_body("w " * 1023), requested_model="m") + with pytest.raises(GenerationInputTooLongError): + await _streamed(adapter, _body("w " * 1023)) + + +async def test_chat_requires_load() -> None: + with pytest.raises(RuntimeError): + await FakeAdapter().chat_completion(_body(), requested_model="m") diff --git a/packages/sie_server/tests/api/test_fake_chat_route.py b/packages/sie_server/tests/api/test_fake_chat_route.py new file mode 100644 index 000000000..cdba26daf --- /dev/null +++ b/packages/sie_server/tests/api/test_fake_chat_route.py @@ -0,0 +1,193 @@ +"""The fake engine's scripted chat through the single-node ``/v1/chat/completions`` route. + +The model configs and the adapter come from the catalog, as ``sie-server serve`` +loads them; only the registry is a stub, so the route's validation, the +adapter-owned chat path and its output checks all run for real. +""" + +from __future__ import annotations + +import json +from functools import cache +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from sie_server.api.openai_local import router +from sie_server.config.model import ModelConfig +from sie_server.core.loader import load_adapter, load_model_configs + +MODELS_DIR = Path(__file__).resolve().parents[2] / "models" +WEATHER = { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, + }, +} +QUESTION = "What is the weather in Paris?" +RESULT = '{"city": "Paris", "temperature_c": 18}' + + +@cache +def _config(model: str) -> ModelConfig: + return load_model_configs(MODELS_DIR)[model] + + +def _client(model: str = "sie-fake:chat") -> TestClient: + config = _config(model) + adapter = load_adapter(config, MODELS_DIR, device="cpu") + adapter.load("cpu") + registry = MagicMock() + registry.device = "cpu" + registry.has_model.return_value = True + registry.get_config.return_value = config + registry.is_failed.return_value = False + registry.is_unloading.return_value = False + registry.is_loading.return_value = False + registry.is_loaded.return_value = True + registry.get.return_value = adapter + app = FastAPI() + app.include_router(router) + app.state.registry = registry + return TestClient(app) + + +def _chat(client: TestClient, messages: list[dict[str, Any]], **fields: Any) -> Any: + model = fields.pop("model", "sie-fake:chat") + return client.post("/v1/chat/completions", json={"model": model, "messages": messages, **fields}) + + +def _events(response: Any) -> list[dict[str, Any]]: + assert response.status_code == 200, response.text + data = [block.removeprefix("data: ") for block in response.text.split("\n\n") if block.startswith("data: ")] + assert data[-1] == "[DONE]" + return [json.loads(item) for item in data[:-1]] + + +def _deltas(events: list[dict[str, Any]], key: str) -> list[Any]: + return [choice["delta"][key] for event in events for choice in event["choices"] if key in choice["delta"]] + + +def _finish(events: list[dict[str, Any]]) -> list[str]: + return [choice["finish_reason"] for event in events for choice in event["choices"] if choice["finish_reason"]] + + +def test_a_tools_turn_streams_one_call_with_its_id_and_name_first() -> None: + response = _chat( + _client(), + [{"role": "user", "content": QUESTION}], + tools=[WEATHER], + tool_choice="auto", + stream=True, + stream_options={"include_usage": True}, + include_reasoning=True, + max_tokens=64, + ) + events = _events(response) + pieces = [piece for calls in _deltas(events, "tool_calls") for piece in calls] + reasoning = _deltas(events, "reasoning_content") + + assert response.headers["x-sie-served-by"] == "local" + assert pieces[0]["id"].startswith("call_") + assert pieces[0]["function"]["name"] == "get_weather" + assert all("id" not in piece and "name" not in piece["function"] for piece in pieces[1:]) + assert json.loads("".join(piece["function"]["arguments"] for piece in pieces)) == {"city": QUESTION} + assert "".join(reasoning).startswith("Tools are declared") + assert _finish(events) == ["tool_calls"] + assert events[-1]["choices"] == [] + assert events[-1]["usage"]["completion_tokens"] == len(reasoning) + len(pieces) + + +def test_a_tool_result_is_answered_in_text_that_carries_it() -> None: + call = {"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}} + messages = [ + {"role": "user", "content": QUESTION}, + {"role": "assistant", "content": "", "tool_calls": [call]}, + {"role": "tool", "tool_call_id": "call_1", "content": RESULT}, + ] + response = _chat(_client(), messages, tools=[WEATHER], max_tokens=64) + + assert response.status_code == 200, response.text + [choice] = response.json()["choices"] + assert choice["message"]["content"] == f"Tool result: {RESULT}" + assert "tool_calls" not in choice["message"] + assert choice["finish_reason"] == "stop" + + +@pytest.mark.parametrize("stream", [True, False]) +def test_reasoning_reaches_the_caller_only_with_include_reasoning(stream: bool) -> None: + client = _client() + messages = [{"role": "user", "content": "Which is larger, 9.11 or 9.9?"}] + fields = {"stream": stream, "stream_options": {"include_usage": True}} if stream else {} + answers = [_chat(client, messages, max_tokens=64, include_reasoning=flag, **fields) for flag in (True, False)] + if stream: + reasoning = ["".join(_deltas(_events(answer), "reasoning_content")) for answer in answers] + usage = [_events(answer)[-1]["usage"] for answer in answers] + else: + reasoning = [answer.json()["choices"][0]["message"].get("reasoning_content", "") for answer in answers] + usage = [answer.json()["usage"] for answer in answers] + + assert reasoning[0] == "No tool call applies, so answer in text." + assert reasoning[1] == "" + assert usage[0] == usage[1] + + +def test_required_tool_choice_forces_a_call_on_a_cpu_device() -> None: + response = _chat( + _client(), + [{"role": "user", "content": "Say hello."}], + tools=[WEATHER], + tool_choice="required", + max_tokens=64, + ) + + assert response.status_code == 200, response.text + [choice] = response.json()["choices"] + assert choice["finish_reason"] == "tool_calls" + assert json.loads(choice["message"]["tool_calls"][0]["function"]["arguments"]) == {"city": "Say hello."} + + +def test_tool_choice_none_answers_in_text() -> None: + response = _chat(_client(), [{"role": "user", "content": QUESTION}], tools=[WEATHER], tool_choice="none") + + assert response.status_code == 200, response.text + [choice] = response.json()["choices"] + assert "tool_calls" not in choice["message"] + assert choice["message"]["content"].startswith("tok_") + assert choice["finish_reason"] == "stop" + + +def test_max_tokens_cuts_the_answer_with_length() -> None: + response = _chat(_client(), [{"role": "user", "content": "Write a long answer."}], max_tokens=3) + + assert response.status_code == 200, response.text + body = response.json() + assert body["choices"][0]["finish_reason"] == "length" + assert body["usage"]["completion_tokens"] == 3 + + +def test_the_window_bounds_the_prompt_and_the_answer() -> None: + client = _client("sie-fake") + window = _config("sie-fake").max_sequence_length + assert window is not None + near = _chat(client, [{"role": "user", "content": "w " * (window - 5)}], model="sie-fake", max_tokens=64) + full = _chat(client, [{"role": "user", "content": "w " * window}], model="sie-fake", max_tokens=64) + + assert near.status_code == 200, near.text + assert near.json()["usage"] == {"prompt_tokens": window - 4, "completion_tokens": 4, "total_tokens": window} + assert near.json()["choices"][0]["finish_reason"] == "length" + assert full.status_code == 400 + assert full.json()["error"]["code"] == "INPUT_TOO_LONG" + + +def test_a_profile_without_thinking_never_returns_reasoning() -> None: + response = _chat(_client("sie-fake"), [{"role": "user", "content": "hi"}], model="sie-fake", include_reasoning=True) + + assert response.status_code == 200, response.text + [choice] = response.json()["choices"] + assert "reasoning_content" not in choice["message"] + assert response.json()["usage"]["completion_tokens"] == len(choice["message"]["content"].split()) From 335da125aa73b7081dfd3bcabd0a66cc35a9b83b Mon Sep 17 00:00:00 2001 From: Krisztian Gajdar Date: Thu, 15 Oct 2026 00:00:24 +0200 Subject: [PATCH 2/3] test(server): check the fake keeps reasoning private without keep_reasoning --- packages/sie_server/tests/adapters/test_fake_chat.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/sie_server/tests/adapters/test_fake_chat.py b/packages/sie_server/tests/adapters/test_fake_chat.py index 14a816078..ec9be4f35 100644 --- a/packages/sie_server/tests/adapters/test_fake_chat.py +++ b/packages/sie_server/tests/adapters/test_fake_chat.py @@ -50,6 +50,18 @@ async def test_the_stream_and_the_answer_agree(adapter: FakeAdapter) -> None: assert events[-1]["usage"] == answer["usage"] +async def test_reasoning_is_spent_but_kept_private_without_keep_reasoning(adapter: FakeAdapter) -> None: + body = _body(chat_template_kwargs={"enable_thinking": True}) + kept = await adapter.chat_completion(body, requested_model="m", keep_reasoning=True) + private = await adapter.chat_completion(body, requested_model="m") + events = await _streamed(adapter, body) + + assert kept["choices"][0]["message"]["reasoning_content"] + assert "reasoning_content" not in private["choices"][0]["message"] + assert not any("reasoning_content" in choice["delta"] for event in events for choice in event["choices"]) + assert private["usage"] == kept["usage"] + + async def test_the_same_request_gets_the_same_answer_and_another_request_another(adapter: FakeAdapter) -> None: first = await adapter.chat_completion(_body("one"), requested_model="m") again = await adapter.chat_completion(_body("one"), requested_model="m") From 9bf70685f6f11be02893e61e7ea38afe02e7486c Mon Sep 17 00:00:00 2001 From: Krisztian Gajdar Date: Thu, 15 Oct 2026 00:05:22 +0200 Subject: [PATCH 3/3] fix(server): refuse a fake chat turn whose tool call does not fit instead of cutting the call --- .../src/sie_server/adapters/fake/adapter.py | 24 +++++++++++++------ .../tests/api/test_fake_chat_route.py | 15 ++++++++++++ 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/packages/sie_server/src/sie_server/adapters/fake/adapter.py b/packages/sie_server/src/sie_server/adapters/fake/adapter.py index 4a70c120e..4fcfb3e32 100644 --- a/packages/sie_server/src/sie_server/adapters/fake/adapter.py +++ b/packages/sie_server/src/sie_server/adapters/fake/adapter.py @@ -48,7 +48,12 @@ import numpy as np from sie_server.adapters._base_adapter import BaseAdapter -from sie_server.adapters._generation_base import GenerationAdapter, GenerationChunk, GenerationInputTooLongError +from sie_server.adapters._generation_base import ( + GenerationAdapter, + GenerationChunk, + GenerationInputTooLongError, + GenerationInvalidRequestError, +) from sie_server.adapters._spec import AdapterSpec from sie_server.adapters._types import ERR_NOT_LOADED from sie_server.core.inference_output import ( @@ -453,9 +458,11 @@ class FakeAdapter(BaseAdapter, GenerationAdapter): tokens. When ``chat_template_kwargs`` enables thinking, one sentence of reasoning naming the rule comes first, returned only with ``keep_reasoning``. A prompt token is a whitespace-separated word of the - tools and messages, and an output token is one streamed piece. Output - stops at the token limit with ``"length"``, a prompt that leaves no room - in ``max_seq_length`` is refused, and ``n`` repeats the turn. + tools and messages, and an output token is one streamed piece. A text + answer stops at the token limit with ``"length"``. A tool call is never + cut: a turn without room for its reasoning and the whole call is refused, + as is a prompt that leaves no room in ``max_seq_length``. ``n`` repeats + the turn. """ spec: ClassVar[AdapterSpec] = AdapterSpec( @@ -738,11 +745,14 @@ async def _chat_turn(self, body: dict[str, Any]) -> _ChatTurn: steps.append(("call", "")) steps += [("arguments", piece) for piece in _PIECE.findall(arguments)] steps += [("content", piece) for piece in content] - kept = steps[:budget] + if call_name is not None and len(steps) > budget: + field = "max_completion_tokens" if body.get("max_completion_tokens") else "max_tokens" + msg = f"the scripted tool call needs {len(steps)} output tokens and {budget} are available" + raise GenerationInvalidRequestError(field, msg) return _ChatTurn( - steps=kept, + steps=steps[:budget], call_id="call_" + hashlib.sha256(f"{transcript}\x00call".encode()).hexdigest()[:24], - call_name=call_name if ("call", "") in kept else None, + call_name=call_name, finish_reason="length" if len(steps) > budget else "tool_calls" if call_name else "stop", prompt_tokens=prompt_tokens, choices=body.get("n") or 1, diff --git a/packages/sie_server/tests/api/test_fake_chat_route.py b/packages/sie_server/tests/api/test_fake_chat_route.py index cdba26daf..fe2db8994 100644 --- a/packages/sie_server/tests/api/test_fake_chat_route.py +++ b/packages/sie_server/tests/api/test_fake_chat_route.py @@ -151,6 +151,21 @@ def test_required_tool_choice_forces_a_call_on_a_cpu_device() -> None: assert json.loads(choice["message"]["tool_calls"][0]["function"]["arguments"]) == {"city": "Say hello."} +def test_a_tool_call_is_never_cut_by_max_tokens() -> None: + client = _client() + messages = [{"role": "user", "content": QUESTION}] + needed = _chat(client, messages, tools=[WEATHER], max_tokens=64).json()["usage"]["completion_tokens"] + exact = _chat(client, messages, tools=[WEATHER], max_tokens=needed) + short = _chat(client, messages, tools=[WEATHER], max_tokens=needed - 1, stream=True) + + assert exact.status_code == 200, exact.text + [choice] = exact.json()["choices"] + assert choice["finish_reason"] == "tool_calls" + assert json.loads(choice["message"]["tool_calls"][0]["function"]["arguments"]) == {"city": QUESTION} + assert short.status_code == 400 + assert short.json()["error"]["code"] == "invalid_request" + + def test_tool_choice_none_answers_in_text() -> None: response = _chat(_client(), [{"role": "user", "content": QUESTION}], tools=[WEATHER], tool_choice="none")