From 24d3dd27136e0e322fdd37939a413a0307a7c2e7 Mon Sep 17 00:00:00 2001 From: Krisztian Gajdar Date: Thu, 15 Oct 2026 17:21:48 +0200 Subject: [PATCH 1/8] feat(examples): run the OpenCode example on one GPU with Qwen3.8-27B-FP8 --- examples/coding-agent-opencode/conformance.py | 90 +++++ .../recipes/cluster/opencode.json | 10 +- .../recipes/gpu/opencode.json | 27 ++ examples/coding-agent-opencode/run_gate.sh | 4 +- .../coding-agent-opencode/session_load.py | 307 ++++++++++++++++++ .../tests/test_conformance.py | 86 ++++- .../tests/test_fake_engine.py | 13 +- .../tests/test_run_gate.py | 29 ++ .../tests/test_session_load.py | 186 +++++++++++ 9 files changed, 745 insertions(+), 7 deletions(-) create mode 100644 examples/coding-agent-opencode/recipes/gpu/opencode.json create mode 100755 examples/coding-agent-opencode/session_load.py create mode 100644 examples/coding-agent-opencode/tests/test_session_load.py diff --git a/examples/coding-agent-opencode/conformance.py b/examples/coding-agent-opencode/conformance.py index e996ff770..812fe49ea 100755 --- a/examples/coding-agent-opencode/conformance.py +++ b/examples/coding-agent-opencode/conformance.py @@ -55,6 +55,16 @@ NEEDLE = "The access code is 7741-ZEBRA." RECORDED_HEADERS = ("x-sie-served-by", "x-sie-model-revision") REASONING_QUESTION = "Which is larger, 9.11 or 9.9? Answer with just the larger number." +LONG_CALL_MIN_BYTES = 65_536 +LONG_CALL_BUDGET = 32_000 +LONG_CALL_LINES = 1_500 +LONG_CALL_SENTENCE = "The quick brown fox jumps over the lazy dog." +LONG_CALL_PROMPT = ( + "Use the write_file tool once to create the file notes/pangrams.txt. Its content must be the sentence " + f'"{LONG_CALL_SENTENCE}" on {LONG_CALL_LINES:,} lines, one copy per line, about ' + f"{LONG_CALL_LINES * (len(LONG_CALL_SENTENCE) + 1):,} characters in all. Write every line out in full in that " + "one call: no ellipsis, no placeholder, no summary and no code that generates the lines." +) WEATHER_TOOL = { "type": "function", @@ -69,6 +79,22 @@ }, } +WRITE_TOOL = { + "type": "function", + "function": { + "name": "write_file", + "description": "Create or replace one text file.", + "parameters": { + "type": "object", + "properties": { + "path": {"type": "string", "description": "File path relative to the project root"}, + "content": {"type": "string", "description": "The whole file content"}, + }, + "required": ["path", "content"], + }, + }, +} + FILLER_WORDS = [ "the", "ledger", @@ -139,6 +165,9 @@ class StreamResult: error: dict[str, Any] | None = None retries: list[dict[str, Any]] = field(default_factory=list) seconds: float = 0.0 + first_chunk_s: float | None = None + stream_s: float = 0.0 + longest_gap_s: float = 0.0 def calls(self) -> list[ToolCall]: return [self.tool_calls[index] for index in sorted(self.tool_calls)] @@ -286,6 +315,7 @@ def complete_chat(client: openai.OpenAI, model: str, messages: list[dict[str, An def _stream_once(client: openai.OpenAI, model: str, messages: list[dict[str, Any]], **params: Any) -> StreamResult: result = StreamResult() + started = previous = time.monotonic() try: raw = client.chat.completions.with_raw_response.create( model=model, @@ -296,6 +326,13 @@ def _stream_once(client: openai.OpenAI, model: str, messages: list[dict[str, Any ) _keep_headers(result, raw) for chunk in raw.parse(): + now = time.monotonic() + if result.first_chunk_s is None: + result.first_chunk_s = now - started + else: + result.longest_gap_s = max(result.longest_gap_s, now - previous) + result.stream_s = now - started - result.first_chunk_s + previous = now result.chunks += 1 result.usage_on_last_chunk = False usage = getattr(chunk, "usage", None) @@ -722,6 +759,58 @@ def case_tool_choice_required(client: openai.OpenAI, model: str) -> Case: ) +def case_long_tool_call(client: openai.OpenAI, model: str, tool_params: dict[str, Any]) -> Case: + name = "long_tool_call" + messages = [{"role": "user", "content": LONG_CALL_PROMPT}] + result = stream_chat( + client, model, messages, tools=[WRITE_TOOL], max_tokens=LONG_CALL_BUDGET, temperature=0, **tool_params + ) + calls = result.calls() + size = sum(len(call.arguments.encode()) for call in calls) + observed = result.observed() + observed.update( + max_tokens=LONG_CALL_BUDGET, + argument_bytes=size, + first_chunk_s=None if result.first_chunk_s is None else round(result.first_chunk_s, 2), + stream_s=round(result.stream_s, 2), + longest_gap_s=round(result.longest_gap_s, 2), + ) + timing = f"streamed over {result.stream_s:.0f} s, longest gap between chunks {result.longest_gap_s:.1f} s" + if result.error and _refuses_field(result.error, "max_tokens"): + error = result.error + detail = ( + f"not run: max_tokens={LONG_CALL_BUDGET} refused: {error['status']} {error['code']}: {error['message']}" + ) + return Case(name, "recorded", detail, observed) + failed = _fail_on_error(name, result) + if failed: + failed.observed = observed + failed.detail = f"{failed.detail} ({timing})" + return failed + if not calls: + detail = f"the model answered without a tool call: {result.content.strip()[:80]!r}" + return Case(name, "recorded", detail, observed) + arguments = [call.parsed_arguments() for call in calls] + if result.finish_reason == "length": + detail = f"max_tokens={LONG_CALL_BUDGET} ended the turn after {size:,} bytes of arguments ({timing})" + return Case(name, "recorded", detail, observed) + if None in arguments: + detail = f"{size:,} bytes of arguments, but not a JSON object: truncated or corrupted ({timing})" + return Case(name, "fail", detail, observed) + if result.finish_reason != "tool_calls": + detail = f"finish_reason is {result.finish_reason!r} after a complete call, expected 'tool_calls'" + return Case(name, "fail", detail, observed) + content = next((args.get("content") for args in arguments if args and isinstance(args.get("content"), str)), "") + observed["content_lines"] = len(content.splitlines()) + observed["lines_as_asked"] = sum(line.strip() == LONG_CALL_SENTENCE for line in content.splitlines()) + kib = size / 1024 + if size < LONG_CALL_MIN_BYTES: + detail = f"the model wrote {kib:.1f} KiB of arguments, under the {LONG_CALL_MIN_BYTES // 1024} KiB asked for" + return Case(name, "recorded", detail, observed) + detail = f"{kib:.1f} KiB of arguments in {len(calls)} call ({observed['content_lines']} lines), {timing}" + return Case(name, "pass", detail, observed) + + def model_entry(base_url: str, api_key: str, model: str) -> dict[str, Any] | None: headers = {"authorization": f"Bearer {api_key}"} if api_key else {} try: @@ -797,6 +886,7 @@ def run(base_url: str, api_key: str, model: str) -> dict[str, Any]: cases.append(case_usage_on_final_chunk(stop_result)) cases.append(reasoning_case) cases.append(case_tool_choice_required(client, model)) + cases.append(case_long_tool_call(client, model, tool_params)) statuses = ("pass", "fail", "recorded", "capacity") counts = {status: sum(1 for case in cases if case.status == status) for status in statuses} diff --git a/examples/coding-agent-opencode/recipes/cluster/opencode.json b/examples/coding-agent-opencode/recipes/cluster/opencode.json index 6a7e5c60e..2ba205706 100644 --- a/examples/coding-agent-opencode/recipes/cluster/opencode.json +++ b/examples/coding-agent-opencode/recipes/cluster/opencode.json @@ -1,6 +1,6 @@ { "$schema": "https://opencode.ai/config.json", - "model": "sie-cluster/zai-org/GLM-5.3-Flash", + "model": "sie-cluster/Qwen/Qwen3.8-27B-FP8:h100-256k-batch", "provider": { "sie-cluster": { "npm": "@ai-sdk/openai-compatible", @@ -28,6 +28,14 @@ "output": 4096 }, "tool_call": true + }, + "Qwen/Qwen3.8-27B-FP8:h100-256k-batch": { + "name": "Qwen3.8-27B on the cluster, 256K window", + "limit": { + "context": 262144, + "output": 32768 + }, + "tool_call": true } } } diff --git a/examples/coding-agent-opencode/recipes/gpu/opencode.json b/examples/coding-agent-opencode/recipes/gpu/opencode.json new file mode 100644 index 000000000..365fbe528 --- /dev/null +++ b/examples/coding-agent-opencode/recipes/gpu/opencode.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://opencode.ai/config.json", + "model": "sie-gpu/Qwen/Qwen3.8-27B-FP8:h100-256k-batch", + "provider": { + "sie-gpu": { + "npm": "@ai-sdk/openai-compatible", + "name": "SIE on one GPU", + "options": { + "baseURL": "http://localhost:8080/v1", + "apiKey": "local", + "timeout": 1800000, + "headerTimeout": 600000, + "chunkTimeout": 600000 + }, + "models": { + "Qwen/Qwen3.8-27B-FP8:h100-256k-batch": { + "name": "Qwen3.8-27B, 256K window, one 80 GB GPU", + "limit": { + "context": 262144, + "output": 32768 + }, + "tool_call": true + } + } + } + } +} diff --git a/examples/coding-agent-opencode/run_gate.sh b/examples/coding-agent-opencode/run_gate.sh index 33bf5d41d..17957e3e1 100755 --- a/examples/coding-agent-opencode/run_gate.sh +++ b/examples/coding-agent-opencode/run_gate.sh @@ -17,7 +17,7 @@ HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" RECIPE="${1:-cloud}" CONFIG="$HERE/recipes/$RECIPE/opencode.json" if [[ ! -f "$CONFIG" ]]; then - echo "unknown recipe '$RECIPE'; pick laptop, cluster or cloud" >&2 + echo "unknown recipe '$RECIPE'; pick laptop, gpu, cluster or cloud" >&2 exit 2 fi for tool in opencode uv; do @@ -34,7 +34,7 @@ for var in GATE_STEPS GATE_DEADLINE_SECONDS; do exit 2 fi done -if [[ "$RECIPE" != "laptop" ]]; then +if grep -q '{env:SIE_' "$CONFIG"; then for var in SIE_BASE_URL SIE_API_KEY; do if [[ -z "${!var:-}" ]]; then echo "$var is not set" >&2 diff --git a/examples/coding-agent-opencode/session_load.py b/examples/coding-agent-opencode/session_load.py new file mode 100755 index 000000000..aea7ab4e8 --- /dev/null +++ b/examples/coding-agent-opencode/session_load.py @@ -0,0 +1,307 @@ +#!/usr/bin/env python3 +"""Agent-shaped session load for an OpenAI-compatible chat endpoint. + + uv sync + uv run python session_load.py --model Qwen/Qwen3.8-27B-FP8:h100-256k-batch --sessions 1,2,4,6,8 + +A coding agent sends its whole transcript on every turn: the same system +prompt, then everything read and said so far, then the new input. Each +session here does the same. Every session shares one system prompt of +``--prefix-tokens``; each turn adds ``--turn-tokens`` of new input, unique to +the session, and asks for an answer that runs to ``--output-tokens``; the +answer goes back into the transcript as it was streamed. For each count in +``--sessions``, that many sessions run their ``--turns`` turns at once. + +Each turn records the time to the first streamed token, the decode rate after +it, the prompt tokens and, when the server reports them, the cached prompt +tokens. A returning turn is any turn after a session's first: with a working +prefix cache only its new input is prefilled. The summary for each session +count gives the median and p90 of both, where the p90 decode rate is the rate +nine turns in ten reached or beat, and whether that count meets the bar of at +least 30 tok/s and at most 3 s to the first token on returning turns, both at +p90, with no failed turn. + +The input is built from common English words in random order, sized with the endpoint's own +token count of the shared prompt, so the sizes hold on any tokenizer to within +a few percent; the report keeps the prompt tokens the server counted. +""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import random +import sys +import threading +import time +from dataclasses import asdict, dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import httpx +import openai + +from conformance import DEFAULT_BASE_URL, FILLER_WORDS, model_entry, wait_until_loaded + +DEFAULT_REPORT = Path("run-output/session-load.json") +REQUEST_TIMEOUT_S = 1800.0 +MIN_DECODE_TPS = 30.0 +MAX_RETURNING_TTFT_S = 3.0 +WORDS_PER_LINE = 16 + + +@dataclass +class Turn: + sessions: int + session: int + turn: int + prompt_tokens: int | None = None + cached_tokens: int | None = None + completion_tokens: int | None = None + ttft_s: float | None = None + decode_tps: float | None = None + seconds: float = 0.0 + finish_reason: str | None = None + error: str | None = None + + +def words_text(rng: random.Random, words: int, label: str) -> str: + lines = [label] + for start in range(0, words, WORDS_PER_LINE): + count = min(WORDS_PER_LINE, words - start) + lines.append(" ".join(rng.choice(FILLER_WORDS) for _ in range(count)) + ".") + return "\n".join(lines) + + +def _percentile(values: list[float], q: float) -> float | None: + if not values: + return None + ordered = sorted(values) + position = (len(ordered) - 1) * q + low = math.floor(position) + high = math.ceil(position) + return ordered[low] + (ordered[high] - ordered[low]) * (position - low) + + +def _round(value: float | None, digits: int = 2) -> float | None: + return None if value is None else round(value, digits) + + +def stream_turn( + client: openai.OpenAI, model: str, messages: list[dict[str, Any]], record: Turn, max_tokens: int +) -> str: + started = time.monotonic() + first = last = None + content = "" + try: + stream = client.chat.completions.create( + model=model, + messages=messages, + max_tokens=max_tokens, + stream=True, + stream_options={"include_usage": True}, + ) + for chunk in stream: + if chunk.usage is not None: + usage = chunk.usage.model_dump(exclude_none=True) + record.prompt_tokens = usage.get("prompt_tokens") + record.completion_tokens = usage.get("completion_tokens") + record.cached_tokens = (usage.get("prompt_tokens_details") or {}).get("cached_tokens") + for choice in chunk.choices or []: + if choice.finish_reason: + record.finish_reason = choice.finish_reason + text = choice.delta.content if choice.delta else None + if text: + now = time.monotonic() + first = now if first is None else first + last = now + content += text + except (openai.APIError, httpx.HTTPError) as exc: + record.error = f"{type(exc).__name__}: {exc}"[:300] + record.seconds = time.monotonic() - started + if first is not None: + record.ttft_s = first - started + if last is not None and last > first and record.completion_tokens and record.completion_tokens > 1: + record.decode_tps = (record.completion_tokens - 1) / (last - first) + return content + + +def run_session( + client: openai.OpenAI, + model: str, + system: str, + words_per_turn: int, + args: argparse.Namespace, + sessions: int, + index: int, + start: threading.Barrier, + records: list[Turn], +) -> None: + rng = random.Random(f"{args.seed}-{sessions}-{index}") + messages: list[dict[str, Any]] = [{"role": "system", "content": system}] + start.wait() + for turn in range(1, args.turns + 1): + label = f"Session {index + 1} of {sessions}, turn {turn}. The agent read this file:" + body = words_text(rng, words_per_turn, label) + ask = f"Explain what the file above changes and why, in at least {args.output_tokens * 2} words." + messages.append({"role": "user", "content": f"{body}\n\n{ask}"}) + record = Turn(sessions=sessions, session=index + 1, turn=turn) + answer = stream_turn(client, model, messages, record, args.output_tokens) + records.append(record) + if record.error: + return + messages.append({"role": "assistant", "content": answer}) + + +def summarise(sessions: int, turns: list[Turn], wall_s: float) -> dict[str, Any]: + done = [turn for turn in turns if turn.error is None] + returning = [turn for turn in done if turn.turn > 1] + rates = [turn.decode_tps for turn in done if turn.decode_tps is not None] + first_ttft = [turn.ttft_s for turn in done if turn.turn == 1 and turn.ttft_s is not None] + returning_ttft = [turn.ttft_s for turn in returning if turn.ttft_s is not None] + reported = [turn for turn in returning if turn.cached_tokens is not None and turn.prompt_tokens] + cached_share = ( + sum(turn.cached_tokens or 0 for turn in reported) / sum(turn.prompt_tokens or 0 for turn in reported) + if reported + else None + ) + decode_p90 = _percentile(rates, 0.1) + ttft_p90 = _percentile(returning_ttft, 0.9) + errors = len(turns) - len(done) + meets = ( + errors == 0 + and decode_p90 is not None + and ttft_p90 is not None + and decode_p90 >= MIN_DECODE_TPS + and ttft_p90 <= MAX_RETURNING_TTFT_S + ) + return { + "sessions": sessions, + "turns": len(turns), + "errors": errors, + "max_prompt_tokens": max((turn.prompt_tokens or 0 for turn in done), default=0), + "decode_tps_p50": _round(_percentile(rates, 0.5), 1), + "decode_tps_p90": _round(decode_p90, 1), + "first_turn_ttft_p50_s": _round(_percentile(first_ttft, 0.5)), + "returning_ttft_p50_s": _round(_percentile(returning_ttft, 0.5)), + "returning_ttft_p90_s": _round(ttft_p90), + "returning_cached_share": _round(cached_share, 3), + "wall_s": round(wall_s, 1), + "meets_bar": meets, + } + + +def run(args: argparse.Namespace) -> dict[str, Any]: + client = openai.OpenAI( + base_url=args.base_url, api_key=args.api_key or "none", timeout=REQUEST_TIMEOUT_S, max_retries=0 + ) + started = datetime.now(UTC) + entry = model_entry(args.base_url, args.api_key, args.model) + warm_up = wait_until_loaded(client, args.model) + probe_words = args.prefix_tokens + probe = words_text(random.Random(f"{args.seed}-system"), probe_words, "You are a coding agent. Project notes:") + sizing = Turn(sessions=0, session=0, turn=0) + probe_messages = [{"role": "system", "content": probe}, {"role": "user", "content": "Reply with the word ready."}] + stream_turn(client, args.model, probe_messages, sizing, 1) + if sizing.error or not sizing.prompt_tokens: + raise SystemExit(f"could not size the prompt: {sizing.error or 'no usage.prompt_tokens'}") + tokens_per_word = sizing.prompt_tokens / probe_words + system_words = round(args.prefix_tokens / tokens_per_word) + system = words_text(random.Random(f"{args.seed}-system"), system_words, "You are a coding agent. Project notes:") + words_per_turn = round(args.turn_tokens / tokens_per_word) + levels = [] + records: list[Turn] = [] + for sessions in args.sessions: + level: list[Turn] = [] + barrier = threading.Barrier(sessions) + threads = [ + threading.Thread( + target=run_session, + args=(client, args.model, system, words_per_turn, args, sessions, index, barrier, level), + ) + for index in range(sessions) + ] + began = time.monotonic() + for thread in threads: + thread.start() + for thread in threads: + thread.join() + summary = summarise(sessions, level, time.monotonic() - began) + levels.append(summary) + records.extend(level) + print_level(summary) + return { + "base_url": args.base_url, + "model": args.model, + "model_entry": entry, + "started_at": started.isoformat(), + "finished_at": datetime.now(UTC).isoformat(), + "warm_up": warm_up, + "shape": { + "prefix_tokens": args.prefix_tokens, + "turn_tokens": args.turn_tokens, + "turns": args.turns, + "output_tokens": args.output_tokens, + "tokens_per_word": round(tokens_per_word, 3), + }, + "bar": {"min_decode_tps_p90": MIN_DECODE_TPS, "max_returning_ttft_p90_s": MAX_RETURNING_TTFT_S}, + "levels": levels, + "turns": [asdict(record) for record in records], + } + + +def print_level(level: dict[str, Any]) -> None: + cached = level["returning_cached_share"] + share = "not reported" if cached is None else f"{cached:.1%}" + print( + f"{level['sessions']:>2} sessions {level['turns']:>3} turns {level['errors']} errors " + f"max prompt {level['max_prompt_tokens']:>7} decode p50/p90 {level['decode_tps_p50']}/{level['decode_tps_p90']} tok/s " + f"returning TTFT p50/p90 {level['returning_ttft_p50_s']}/{level['returning_ttft_p90_s']} s " + f"cached {share} {'meets' if level['meets_bar'] else 'misses'} the bar", + flush=True, + ) + + +def _counts(text: str) -> list[int]: + try: + counts = [int(part) for part in text.split(",")] + except ValueError as exc: + raise argparse.ArgumentTypeError(f"expected comma-separated whole numbers, got {text!r}") from exc + if not counts or min(counts) < 1: + raise argparse.ArgumentTypeError("every session count must be at least 1") + return counts + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + "--base-url", default=DEFAULT_BASE_URL, help=f"OpenAI-compatible base URL (default {DEFAULT_BASE_URL})" + ) + parser.add_argument("--api-key", default=os.environ.get("SIE_API_KEY", ""), help="Bearer key (env SIE_API_KEY)") + parser.add_argument("--model", required=True, help="model id, for example Qwen/Qwen3.8-27B-FP8:h100-256k-batch") + parser.add_argument("--sessions", type=_counts, default=[1, 2, 4], help="session counts to run, e.g. 1,2,4,6,8") + parser.add_argument("--prefix-tokens", type=int, default=16_000, help="shared system prompt size (default 16000)") + parser.add_argument("--turn-tokens", type=int, default=6_000, help="new input per turn (default 6000)") + parser.add_argument("--turns", type=int, default=7, help="turns per session (default 7, about 61K at the end)") + parser.add_argument("--output-tokens", type=int, default=500, help="answer length per turn (default 500)") + parser.add_argument("--seed", default="session-load", help="seed for the generated input") + parser.add_argument( + "--report", type=Path, default=DEFAULT_REPORT, help=f"JSON report path (default {DEFAULT_REPORT})" + ) + args = parser.parse_args() + for name in ("prefix_tokens", "turn_tokens", "turns", "output_tokens"): + if getattr(args, name) < 1: + parser.error(f"--{name.replace('_', '-')} must be at least 1") + + report = run(args) + args.report.parent.mkdir(parents=True, exist_ok=True) + args.report.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + print(f"report written to {args.report}") + return 1 if any(level["errors"] for level in report["levels"]) else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/coding-agent-opencode/tests/test_conformance.py b/examples/coding-agent-opencode/tests/test_conformance.py index 5e5839bf6..d7117bfef 100644 --- a/examples/coding-agent-opencode/tests/test_conformance.py +++ b/examples/coding-agent-opencode/tests/test_conformance.py @@ -11,6 +11,7 @@ import json import sys import threading +import time from collections.abc import Iterator from datetime import UTC, datetime, timedelta from email.utils import format_datetime @@ -76,6 +77,8 @@ def _reply(body: dict[str, Any]) -> dict[str, Any]: return {"status": 503, "code": "QUEUE_FULL", "message": "upstream busy", "headers": {"retry-after": "0"}} if KNOBS.get("midstream_error"): return {"error_event": KNOBS["midstream_error"]} + if tools and tools[0]["function"]["name"] == "write_file": + return _long_call(body) if tools and "Tokyo" in user_text: calls = [("call_1", "get_weather", {"city": "Paris"}), ("call_2", "get_weather", {"city": "Tokyo"})] if KNOBS.get("parallel") == "single": @@ -106,6 +109,28 @@ def _reply(body: dict[str, Any]) -> dict[str, Any]: return {"content": "ready", "finish": "stop"} +def _long_call(body: dict[str, Any]) -> dict[str, Any]: + SEEN.setdefault("long_calls", []) + cap = KNOBS.get("max_output") + if cap and body["max_tokens"] > cap: + message = f"max_tokens ({body['max_tokens']}) exceeds model cap ({cap})" + return {"status": 400, "code": "INVALID_INPUT", "message": message, "param": "max_tokens"} + SEEN["long_calls"].append(body) + mode = KNOBS.get("long_call", "write") + lines = 10 if mode == "short" else conformance.LONG_CALL_LINES + arguments = {"path": "notes/pangrams.txt", "content": "\n".join([conformance.LONG_CALL_SENTENCE] * lines)} + if mode == "decline": + return {"content": "That file is too long to write in one call.", "finish": "stop"} + if mode == "error": + error = {"message": "inter-chunk timeout", "type": "server_error", "param": None, "code": "TIMEOUT"} + return {"partial": "Writing the file.", "error_event": error} + raw = json.dumps(arguments) + if mode in ("truncated", "length"): + raw = raw[: len(raw) // 2] + finish = {"length": "length", "stop": "stop"}.get(mode, "tool_calls") + return {"raw_calls": [("call_long", "write_file", raw)], "finish": finish} + + def _thinking_budget_spent(body: dict[str, Any], reply: dict[str, Any]) -> dict[str, Any]: think = KNOBS.get("think_tokens", 0) max_tokens = body.get("max_tokens") @@ -153,6 +178,7 @@ def _usage(self, body: dict[str, Any], reply: dict[str, Any]) -> dict[str, int]: completion = " ".join( [reply.get("content") or "", reply.get("reasoning") or "", reply.get("hidden") or ""] + [json.dumps(arguments) for _, _, arguments in reply.get("calls") or []] + + [raw for _, _, raw in reply.get("raw_calls") or []] ) completion_tokens = max(1, _words(completion)) return { @@ -175,6 +201,8 @@ def do_POST(self) -> None: reply = _thinking_budget_spent(body, _reply(body)) if "status" in reply: error = {"message": reply["message"], "type": "invalid_request_error", "code": reply["code"]} + if reply.get("param"): + error["param"] = reply["param"] self._json(reply["status"], {"error": error}, reply.get("headers")) return if not body.get("stream"): @@ -217,6 +245,14 @@ def do_POST(self) -> None: self._chunk([{"index": 0, "delta": {"tool_calls": [head]}, "finish_reason": None}]) tail = {"index": index, "function": {"arguments": json.dumps(arguments)}} self._chunk([{"index": 0, "delta": {"tool_calls": [tail]}, "finish_reason": None}]) + for index, (call_id, name, raw) in enumerate(reply.get("raw_calls") or []): + head = {"index": index, "id": call_id, "type": "function", "function": {"name": name, "arguments": ""}} + self._chunk([{"index": 0, "delta": {"tool_calls": [head]}, "finish_reason": None}]) + for start in range(0, len(raw), 4096): + if KNOBS.get("gap_s"): + time.sleep(KNOBS["gap_s"]) + piece = {"index": index, "function": {"arguments": raw[start : start + 4096]}} + self._chunk([{"index": 0, "delta": {"tool_calls": [piece]}, "finish_reason": None}]) for word in (reply.get("content") or "").split(" "): if word: self._chunk([{"index": 0, "delta": {"content": word + " "}, "finish_reason": None}]) @@ -372,7 +408,7 @@ def test_a_request_still_busy_after_the_cap_is_a_capacity_failure( assert case["retries"] == len(expected_waits) assert waits == expected_waits assert case["detail"].startswith(f"still busy after {len(expected_waits)} retries and {sum(expected_waits):g} s") - assert report["summary"] == {"pass": 10, "fail": 0, "recorded": 0, "capacity": 1} + assert report["summary"] == {"pass": 11, "fail": 0, "recorded": 0, "capacity": 1} def test_a_retry_after_date_is_waited_until(base_url: str, waits: list[float]) -> None: @@ -455,17 +491,18 @@ def test_an_endpoint_that_stays_busy_fails_on_capacity_not_behaviour( "usage_on_final_chunk": "capacity", "include_reasoning": "capacity", "tool_choice_required": "capacity", + "long_tool_call": "capacity", } assert cases["streaming_tool_call_deltas"]["retries"] == conformance.BUSY_RETRIES assert cases["include_reasoning"]["retries"] == 3 * conformance.BUSY_RETRIES assert cases["finish_reason_tool_calls"]["retries"] == 0 assert cases["usage_on_final_chunk"]["retries"] == 0 - assert report["summary"] == {"pass": 0, "fail": 0, "recorded": 1, "capacity": 10} + assert report["summary"] == {"pass": 0, "fail": 0, "recorded": 1, "capacity": 11} argv = ["conformance.py", "--base-url", base_url, "--model", "stub-model", "--report", str(tmp_path / "r.json")] monkeypatch.setattr(sys, "argv", argv) capsys.readouterr() assert conformance.main() == 1 - assert "0 pass, 0 fail, 1 recorded, 10 capacity" in capsys.readouterr().out + assert "0 pass, 0 fail, 1 recorded, 11 capacity" in capsys.readouterr().out def test_a_failed_tools_turn_does_not_blame_tool_choice(base_url: str) -> None: @@ -581,3 +618,46 @@ def test_a_long_answer_without_reasoning_is_recorded_not_judged(base_url: str, r assert case["status"] == "recorded", case["detail"] assert "whether the model reasoned is unknown" in case["detail"] assert case["observed"]["model_reasoned"] is None + + +def test_a_long_tool_call_passes_when_the_whole_call_arrives(base_url: str) -> None: + case = _run(base_url, gap_s=0.05)["long_tool_call"] + assert case["status"] == "pass", case["detail"] + observed = case["observed"] + assert observed["argument_bytes"] > conformance.LONG_CALL_MIN_BYTES + assert observed["lines_as_asked"] == conformance.LONG_CALL_LINES + assert observed["max_tokens"] == conformance.LONG_CALL_BUDGET + assert observed["longest_gap_s"] >= 0.05 + assert len(observed["tool_calls"][0]["arguments"]) == 200 + (request,) = SEEN["long_calls"] + assert [tool["function"]["name"] for tool in request["tools"]] == ["write_file"] + assert request["tool_choice"] == "auto" + assert request["include_reasoning"] is True + + +@pytest.mark.parametrize( + ("mode", "status", "detail"), + [ + ("short", "recorded", "the model wrote 0.5 KiB of arguments, under the 64 KiB asked for"), + ("decline", "recorded", "the model answered without a tool call"), + ("length", "recorded", "max_tokens=32000 ended the turn"), + ("truncated", "fail", "but not a JSON object: truncated or corrupted"), + ("stop", "fail", "finish_reason is 'stop' after a complete call"), + ("error", "fail", "request failed: None TIMEOUT: inter-chunk timeout"), + ], +) +def test_a_long_tool_call_separates_the_model_declining_from_the_wire_breaking( + base_url: str, mode: str, status: str, detail: str +) -> None: + case = _run(base_url, long_call=mode)["long_tool_call"] + assert case["status"] == status + assert detail in case["detail"] + + +def test_a_long_tool_call_over_the_output_cap_is_recorded_not_failed(base_url: str) -> None: + case = _run(base_url, max_output=4096)["long_tool_call"] + assert case["status"] == "recorded" + assert case["detail"] == ( + "not run: max_tokens=32000 refused: 400 INVALID_INPUT: max_tokens (32000) exceeds model cap (4096)" + ) + assert SEEN["long_calls"] == [] diff --git a/examples/coding-agent-opencode/tests/test_fake_engine.py b/examples/coding-agent-opencode/tests/test_fake_engine.py index 0aff93034..3779b9ae3 100644 --- a/examples/coding-agent-opencode/tests/test_fake_engine.py +++ b/examples/coding-agent-opencode/tests/test_fake_engine.py @@ -8,7 +8,9 @@ So each case here checks SIE's single-node chat wire, not a model. The fake makes one call per turn, so ``parallel_tool_calls`` fails, and the -test asserts that failure instead of skipping the case. The fake does not read +test asserts that failure instead of skipping the case. Its 4,096-token output +cap refuses the 32,000 tokens a 64 KiB call needs, so ``long_tool_call`` is +recorded as not run. The fake does not read the long prompt, so the needle is never found; the case passes on the prompt size and the answer, as it does for any model. """ @@ -34,6 +36,7 @@ SERVER = REPOSITORY / ".venv" / "bin" / "sie-server" MODEL = "sie-fake:chat" WINDOW = 131072 +OUTPUT_CAP = 4096 SERVER_START_TIMEOUT_S = 300.0 TOOL_RESULT = {"city": "Paris", "temperature_c": 18, "conditions": "overcast"} @@ -107,6 +110,7 @@ def test_each_case_against_the_fake(report: dict[str, Any]) -> None: "usage_on_final_chunk": "pass", "include_reasoning": "pass", "tool_choice_required": "recorded", + "long_tool_call": "recorded", }, report["cases"] assert report["served_by"] == "local" assert report["warm_up"]["error"] is None @@ -166,6 +170,13 @@ def test_max_tokens_and_usage_are_exact(report: dict[str, Any]) -> None: assert usage["total_tokens"] == usage["prompt_tokens"] + usage["completion_tokens"] +def test_the_long_tool_call_is_not_sent_past_the_declared_output_cap(report: dict[str, Any]) -> None: + case = _cases(report)["long_tool_call"] + assert case["observed"]["error"]["param"] == "max_tokens" + assert case["detail"].startswith("not run: max_tokens=32000 refused: 400 INVALID_INPUT") + assert f"model cap ({OUTPUT_CAP})" in case["detail"] + + def test_tool_choice_required_forces_a_call(report: dict[str, Any]) -> None: observed = _cases(report)["tool_choice_required"]["observed"] assert observed["support"] == "supported" diff --git a/examples/coding-agent-opencode/tests/test_run_gate.py b/examples/coding-agent-opencode/tests/test_run_gate.py index 68022c55c..089b5f610 100644 --- a/examples/coding-agent-opencode/tests/test_run_gate.py +++ b/examples/coding-agent-opencode/tests/test_run_gate.py @@ -163,3 +163,32 @@ def test_a_run_past_its_deadline_is_stopped_and_fails(tmp_path: Path, ignores_te time.sleep(0.1) else: pytest.fail(f"the agent's child process {pid} outlived the deadline") + + +@pytest.mark.parametrize(("recipe", "needs_sie_variables"), [("gpu", False), ("cluster", True), ("cloud", True)]) +def test_only_a_recipe_that_reads_the_sie_variables_requires_them( + tmp_path: Path, recipe: str, needs_sie_variables: bool +) -> None: + root = tmp_path / "example" + root.mkdir() + shutil.copy(EXAMPLE / "run_gate.sh", root) + ignored = shutil.ignore_patterns(".venv", "uv.lock", "__pycache__", ".pytest_cache") + shutil.copytree(EXAMPLE / "fixture", root / "fixture", ignore=ignored) + shutil.copytree(EXAMPLE / "recipes", root / "recipes") + record = tmp_path / "opencode-config.txt" + (tmp_path / "bin").mkdir() + _executable(tmp_path / "bin" / "opencode", f'echo "$OPENCODE_CONFIG" > {record}\n{FIX}') + unset = ("VIRTUAL_ENV", "SIE_BASE_URL", "SIE_API_KEY") + env = {key: value for key, value in os.environ.items() if key not in unset} + env["PATH"] = f"{tmp_path / 'bin'}{os.pathsep}{env['PATH']}" + run = subprocess.run( + ["bash", str(root / "run_gate.sh"), recipe], check=False, capture_output=True, text=True, env=env, timeout=600 + ) + if needs_sie_variables: + assert run.returncode == 2 + assert run.stderr.strip() == "SIE_BASE_URL is not set" + assert not record.exists() + else: + assert run.returncode == 0, run.stdout + assert run.stdout.strip().splitlines()[-1].startswith("GATE PASS") + assert Path(record.read_text().strip()).parts[-3:] == ("recipes", recipe, "opencode.json") diff --git a/examples/coding-agent-opencode/tests/test_session_load.py b/examples/coding-agent-opencode/tests/test_session_load.py new file mode 100644 index 000000000..1038afbe3 --- /dev/null +++ b/examples/coding-agent-opencode/tests/test_session_load.py @@ -0,0 +1,186 @@ +"""The session load generator against a scripted OpenAI-compatible stub. + +The stub counts one prompt token per word, streams a fixed answer one word per +chunk and reports a cached prefix when a request repeats an earlier one. That +is enough to check what the generator sends, how it grows each transcript and +how it summarises the turns; capacity is measured by running it against a +real endpoint. +""" + +from __future__ import annotations + +import json +import sys +import threading +from collections.abc import Iterator +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any + +import pytest + +import session_load + +ANSWER = " ".join(f"word{i}" for i in range(40)) +REQUESTS: list[dict[str, Any]] = [] +KNOBS: dict[str, Any] = {} +LOCK = threading.Lock() + + +def _prompt_words(body: dict[str, Any]) -> list[str]: + return [word for message in body["messages"] for word in (message.get("content") or "").split()] + + +class Handler(BaseHTTPRequestHandler): + def log_message(self, *args: Any) -> None: + pass + + def _send(self, payload: dict[str, Any]) -> None: + self.wfile.write(f"data: {json.dumps(payload)}\n\n".encode()) + + def do_GET(self) -> None: + raw = json.dumps({"models": [{"name": "stub-model", "max_output_tokens": 32768}]}).encode() + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(raw))) + self.end_headers() + self.wfile.write(raw) + + def do_POST(self) -> None: + body = json.loads(self.rfile.read(int(self.headers["content-length"]))) + words = _prompt_words(body) + with LOCK: + cached = max((_shared(words, earlier) for earlier in REQUESTS), default=0) + REQUESTS.append({"body": body, "words": words}) + failing = KNOBS.get("fail_turn") and f"turn {KNOBS['fail_turn']}." in body["messages"][-1]["content"] + if failing: + raw = json.dumps({"error": {"message": "worker failed", "code": "INTERNAL_ERROR"}}).encode() + self.send_response(500) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(raw))) + self.end_headers() + self.wfile.write(raw) + return + answer = ANSWER.split()[: body["max_tokens"]] + self.send_response(200) + self.send_header("content-type", "text/event-stream") + self.end_headers() + base = {"id": "chatcmpl-stub", "object": "chat.completion.chunk", "created": 1, "model": "stub-model"} + for word in answer: + self._send({**base, "choices": [{"index": 0, "delta": {"content": word + " "}, "finish_reason": None}]}) + self._send({**base, "choices": [{"index": 0, "delta": {}, "finish_reason": "length"}]}) + usage = { + "prompt_tokens": len(words), + "completion_tokens": len(answer), + "total_tokens": len(words) + len(answer), + "prompt_tokens_details": {"cached_tokens": cached}, + } + self._send({**base, "choices": [], "usage": usage}) + self.wfile.write(b"data: [DONE]\n\n") + + +def _shared(words: list[str], earlier: dict[str, Any]) -> int: + count = 0 + for mine, theirs in zip(words, earlier["words"], strict=False): + if mine != theirs: + break + count += 1 + return count + + +@pytest.fixture(scope="module") +def base_url() -> Iterator[str]: + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_address[1]}/v1" + finally: + server.shutdown() + + +@pytest.fixture(autouse=True) +def reset() -> Iterator[None]: + REQUESTS.clear() + KNOBS.clear() + yield + REQUESTS.clear() + KNOBS.clear() + + +def _main(base_url: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *extra: str) -> tuple[int, dict[str, Any]]: + report = tmp_path / "load.json" + argv = ["session_load.py", "--base-url", base_url, "--model", "stub-model", "--report", str(report)] + small = ["--prefix-tokens", "400", "--turn-tokens", "120", "--turns", "3", "--output-tokens", "20"] + monkeypatch.setattr(sys, "argv", [*argv, *small, *extra]) + code = session_load.main() + return code, json.loads(report.read_text()) + + +def test_each_session_replays_its_whole_transcript( + base_url: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + code, _ = _main(base_url, tmp_path, monkeypatch, "--sessions", "2") + assert code == 0 + turns = [request["body"] for request in REQUESTS if len(request["body"]["messages"]) > 2] + third = [body for body in turns if "turn 3." in body["messages"][-1]["content"]] + assert len(third) == 2 + for body in third: + roles = [message["role"] for message in body["messages"]] + assert roles == ["system", "user", "assistant", "user", "assistant", "user"] + assert body["messages"][2]["content"].split() == ANSWER.split()[:20] + assert body["stream"] is True + assert body["max_tokens"] == 20 + assert "temperature" not in body + assert third[0]["messages"][0] == third[1]["messages"][0] + assert third[0]["messages"][1] != third[1]["messages"][1] + + +def test_the_input_is_sized_with_the_endpoints_token_count( + base_url: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _, report = _main(base_url, tmp_path, monkeypatch, "--sessions", "1") + assert report["shape"]["tokens_per_word"] == pytest.approx(1.0, abs=0.1) + first = next(turn for turn in report["turns"] if turn["turn"] == 1) + assert 400 + 120 <= first["prompt_tokens"] <= 400 + 120 + 80 + + +def test_the_summary_counts_turns_and_cached_prefix( + base_url: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + code, report = _main(base_url, tmp_path, monkeypatch, "--sessions", "1,3") + assert code == 0 + one, three = report["levels"] + assert (one["sessions"], one["turns"], one["errors"]) == (1, 3, 0) + assert (three["sessions"], three["turns"], three["errors"]) == (3, 9, 0) + assert 0.5 < three["returning_cached_share"] < 1.0 + assert three["max_prompt_tokens"] > one["max_prompt_tokens"] * 0.9 + for level in report["levels"]: + assert level["decode_tps_p50"] > 0 + assert level["returning_ttft_p90_s"] is not None + assert len(report["turns"]) == 12 + + +def test_a_failed_turn_ends_its_session_and_fails_the_run( + base_url: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + KNOBS["fail_turn"] = 2 + code, report = _main(base_url, tmp_path, monkeypatch, "--sessions", "2") + assert code == 1 + (level,) = report["levels"] + assert (level["turns"], level["errors"], level["meets_bar"]) == (4, 2, False) + failed = [turn for turn in report["turns"] if turn["error"]] + assert {turn["turn"] for turn in failed} == {2} + assert all("INTERNAL_ERROR" in turn["error"] or "500" in turn["error"] for turn in failed) + + +def test_the_bar_is_judged_at_p90() -> None: + def turn(number: int, ttft: float, tps: float) -> session_load.Turn: + return session_load.Turn(sessions=1, session=1, turn=number, ttft_s=ttft, decode_tps=tps, prompt_tokens=10) + + fast = [turn(1, 9.0, 60.0)] + [turn(n, 0.5, 60.0) for n in range(2, 12)] + assert session_load.summarise(1, fast, 1.0)["meets_bar"] is True + slow_ttft = fast[:-2] + [turn(11, 4.0, 60.0), turn(12, 4.0, 60.0)] + assert session_load.summarise(1, slow_ttft, 1.0)["meets_bar"] is False + slow_decode = fast[:-2] + [turn(11, 0.5, 20.0), turn(12, 0.5, 20.0)] + assert session_load.summarise(1, slow_decode, 1.0)["meets_bar"] is False From 668eb69418d59784e9f0a46a15fae7214cd52f88 Mon Sep 17 00:00:00 2001 From: Krisztian Gajdar Date: Thu, 15 Oct 2026 19:44:04 +0200 Subject: [PATCH 2/8] docs(examples): record the one-GPU OpenCode runs on Qwen3.8-27B-FP8 --- examples/coding-agent-opencode/README.md | 209 +++++++++++++++++- examples/coding-agent-opencode/conformance.py | 19 +- .../tests/test_conformance.py | 3 +- 3 files changed, 213 insertions(+), 18 deletions(-) diff --git a/examples/coding-agent-opencode/README.md b/examples/coding-agent-opencode/README.md index a53d4d11d..805735056 100644 --- a/examples/coding-agent-opencode/README.md +++ b/examples/coding-agent-opencode/README.md @@ -5,15 +5,16 @@ [OpenCode](https://opencode.ai) is an open-source coding agent. It talks to any OpenAI-compatible chat endpoint through a custom provider built on `@ai-sdk/openai-compatible`, which targets `/v1/chat/completions`. This -example points it at SIE three ways and checks that the agent loop holds up: +example points it at SIE four ways and checks that the agent loop holds up: - a laptop running `sie-server` with a small tools-capable model, +- one 80 GB NVIDIA GPU running `sie-server` with Qwen3.8-27B-FP8, - a self-hosted SIE cluster, reached through its gateway with a key, - SIE Cloud, where `zai-org/GLM-5.3-Flash:relay` is served through SIE Relay. -Three pieces are in this directory: +Four pieces are in this directory: -1. `recipes//opencode.json`, one OpenCode config per +1. `recipes//opencode.json`, one OpenCode config per setup. 2. `conformance.py`, an agent-loop conformance check. It drives `/v1/chat/completions` the way OpenCode does, streaming, with tools declared @@ -34,11 +35,15 @@ Three pieces are in this directory: and fails. OpenCode 1.19.0 treats the step budget as an instruction: at the last step it tells the model to answer in text but still declares the tools, so a model that keeps calling tools is stopped by the deadline. +4. `session_load.py`, a session load generator. It runs several agent-shaped + sessions at once against one endpoint and reports, for each session count, + the time to the first token on returning turns, the decode rate and the + prompt tokens the server served from its cache. -The conformance check uses the `openai` Python package rather than the SIE -SDK. The thing under test is the OpenAI-compatible wire surface that -OpenCode's provider speaks, so the reference client for that surface is the -right instrument. +The conformance check and the load generator use the `openai` Python package +rather than the SIE SDK. The thing under test is the OpenAI-compatible wire +surface that OpenCode's provider speaks, so the reference client for that +surface is the right instrument. ## Run it @@ -64,6 +69,53 @@ and the two variables: ./run_gate.sh ``` +### On one GPU + +On a Linux machine with one 80 GB NVIDIA GPU, start the server with +Qwen3.8-27B-FP8 loaded: + +```sh +docker run --gpus all -p 127.0.0.1:8080:8080 \ + -v sie-hf-cache:/app/.cache/huggingface \ + -v sie-kernel-cache:/app/.cache/sie-kernels \ + ghcr.io/superlinked/sie-server:latest-cuda13-sglang-cu130 \ + serve --host 0.0.0.0 --port 8080 --models-dir /app/models \ + --preload Qwen/Qwen3.8-27B-FP8:h100-256k-batch +``` + +Then, from this directory: + +```sh +uv sync +uv run python conformance.py --model Qwen/Qwen3.8-27B-FP8:h100-256k-batch +./run_gate.sh gpu +uv run python session_load.py --model Qwen/Qwen3.8-27B-FP8:h100-256k-batch --sessions 1,2,4,6,8 +``` + +and point OpenCode at it with `recipes/gpu/opencode.json`. + +- `-p 127.0.0.1:8080:8080` publishes the port on this machine only. + `sie-server` checks no key, and a coding agent sends it your source code, so + the recipe keeps it off the network. To share one GPU with a team, serve it + through the cluster gateway, which checks keys (the cluster recipe below). +- The two named volumes keep the weights and the compiled kernels across + restarts. The first start downloads about 30 GB of weights. +- `--preload` loads the model before the server answers, so OpenCode's first + turn does not meet `503 MODEL_LOADING`. +- `:h100-256k-batch` is the profile that fits an agent: a 262,144-token window, + a 32,768-token output cap, up to 32 requests decoding at once, the + `qwen3_coder` tool-call parser, no speculative decoding and no thinking. The + recipe sets `limit.context` and `limit.output` to exactly those two numbers. + OpenCode then sends `max_tokens: 32000`, so a prompt may grow to 230,144 + tokens before OpenCode compacts the session. +- The recipe names one model. Each profile is its own engine launch, so asking + the same server for a second profile unloads the first. +- For a model that thinks before it answers, `Qwen/Qwen3.8-27B-FP8:thinking` + is the same weights with reasoning on. Declare it with `"reasoning": true`, + `"interleaved": {"field": "reasoning_content"}` and `"options": + {"include_reasoning": true}`, as the Cloud recipe does. That profile runs one + request at a time, and it was not recorded here. + ### Against a laptop From a checkout of this repository: @@ -219,9 +271,13 @@ the laptop and cluster recipes. The report's `tools_turn_params` says which. | `usage_on_final_chunk` | with `stream_options.include_usage`, the last chunk carries `prompt_tokens` and `completion_tokens` | | `include_reasoning` | the flag is accepted streamed and not streamed, no raw `` tag leaks into content, and without the flag no `reasoning_content` arrives. When the model reasoned, the stream carries `delta.reasoning_content` and the non-streamed answer `message.reasoning_content`. A model reasoned when it returned reasoning, or when its completion tokens exceed the UTF-8 bytes of its visible text by more than 16, since a byte-level tokenizer spends at most one token per byte. It did not reason when no answer took more than 16 completion tokens. Between the two, whether it reasoned is unknown and the case is recorded, not judged | | `tool_choice_required` | recorded as `supported` when a call is forced or `refused` on a 400 or 422, neither a failure; accepted-but-ignored fails, and so does any other error | +| `long_tool_call` | one `write_file` call asked for 1,320 numbered lines, 67,363 bytes of arguments, arrives whole: the arguments parse as a JSON object over 64 KiB and the turn ends with `tool_calls`. The request sends `max_tokens: 32000`, as OpenCode does. A stream that breaks or arguments that do not parse fail. A model that answers in text, writes less than 64 KiB or runs out of `max_tokens` is recorded, and so is an endpoint that refuses `max_tokens: 32000`. The report keeps the argument size, how many lines came back as asked, the time to the first chunk, how long the stream ran and its longest gap between chunks | -Every request asks for up to 4,096 tokens, except the `length` case, so a -model that thinks before it answers has room to finish. The 68,000-word +Every request asks for up to 4,096 tokens, except the `length` case and the +long call, so a model that thinks before it answers has room to finish. The +long call takes several minutes on one GPU: about 20,000 tokens of output. +OpenCode's own `write` and `edit` calls carry whole files the same way, and a +server that sends nothing while a call is generated must not end the stream. The 68,000-word prompt counts at least one token per word on any tokenizer that does not merge words: 71,431 with `cl100k_base`, 71,917 with `o200k_base`, 74,835 with Qwen3.5's tokenizer and 71,432 with GLM-5.3-Flash's. @@ -238,8 +294,8 @@ that took. A request refused as busy or rate-limited is retried: a `429` or `rate_limit_exceeded`, or an error event with one of those codes that ends a stream before any output. Each wait is the `Retry-After` header, in seconds or until its future date, or else the error's `retry_after_s`, capped at 60 -seconds. Without either, the wait is 5 seconds, doubling with each retry. A request is retried at most 4 times and -waits at most 3 minutes in all. Each retry is recorded with its error and +seconds. Without either, the wait is 5 seconds, doubling with each retry. A +request is retried at most 4 times and waits at most 3 minutes in all. Each retry is recorded with its error and wait in the case's `observed`, and the case's `retries` counts them. A case whose request is still refused as busy after that, or whose stream ends in a busy error event after some output, has the status `capacity` rather than @@ -248,6 +304,37 @@ errors, including `MODEL_LOADING` or `provisioning` after the warm-up, are not retried, and an error event in the middle of a stream is recorded as that case's error. +## What `session_load.py` measures + +A coding agent sends its whole transcript on every turn, so the number of +sessions one GPU holds depends on the KV cache and on whether the server reuses +the transcript's prefix from one turn to the next. `session_load.py` runs +agent-shaped sessions against one endpoint: + +- Every session starts from one shared system prompt of `--prefix-tokens` + (16,000 by default), standing in for an agent's instructions and tool + descriptions. +- Each turn adds `--turn-tokens` (6,000) of new input, unique to the session, + standing in for a file the agent read, and asks for an answer of + `--output-tokens` (500). The answer goes back into the transcript as it was + streamed, and the whole transcript is sent again on the next turn. +- After `--turns` (7) turns a session's prompt is about 61,000 tokens. + `--turn-tokens 12000 --turns 9` grows it to about 128,000. +- For each count in `--sessions`, that many sessions start together and run + their turns back to back, with no pause for tool calls in between, which is + harder on the server than a real agent. +- No sampling fields are sent, so the profile's defaults apply, as they do to + OpenCode's requests. + +The input is sized with the endpoint's own token count of the shared prompt. +For each session count the report gives the decode rate after the first token +and the time to the first token on returning turns, at the median and at p90, +where the p90 decode rate is the rate nine turns in ten reached or beat. It +also gives the share of returning-turn prompt tokens the server reported as +cached, when it reports them. A session count meets the bar when, at p90, +every session decodes at 30 tok/s or more and a returning turn starts within +3 seconds, with no failed turn. The bar was set before the runs below. + ## What result to expect ### Laptop, Apple Silicon, recorded @@ -306,6 +393,106 @@ OpenCode took 5 steps and 4 tool calls (`bash`, `read`, `edit`, `bash`), 8 seconds from its first event to its last. An earlier run on the same server took 7 steps and 6 tool calls. +### One GPU, Qwen3.8-27B-FP8, recorded + +`Qwen/Qwen3.8-27B-FP8:h100-256k-batch` on one NVIDIA H100 80GB HBM3 with 8 CPU +cores and 64 GiB of memory, 2026-10-15. `sie-server` ran from this repository +at commit `461c5a58` with the `sglang-cu130` bundle (SGLang 0.5.20), started +as `sie-server serve -b sglang-cu130 --preload +Qwen/Qwen3.8-27B-FP8:h100-256k-batch`. The `docker run` line above starts the +same server, profile and bundle from the published image; the runs below did +not use the image. + +What the engine reported once loaded: + +- a KV cache of 325,056 tokens, shared by every request on the GPU; +- 243 slots for the recurrent state of the model's linear-attention layers; +- up to 32 requests decoding at once; +- 28.5 GB of weights and 19.8 GB of KV cache. + +The server answered its first request 197 s after it started in a new +container, with the weights already downloaded, and 76 s after a restart in the +same container, with the kernels already compiled. + +``` +model Qwen/Qwen3.8-27B-FP8:h100-256k-batch at http://localhost:8080/v1 +served by local + + tool_choice_auto pass tool_choice=auto accepted on the tools turn + streaming_tool_call_deltas pass 1 call: get_weather({"city": "Paris"}); 4 deltas; finish_reason=tool_calls + parallel_tool_calls pass 2 calls in one turn: get_weather({"city": "Paris"}), get_weather({"city": "Tokyo"}) + tool_result_round_trip pass final answer used the tool result: 'The weather in Paris right now is overcast with a temperature of 18°C.' + prompt_above_64k_tokens pass 74847 prompt tokens accepted and answered; needle found + finish_reason_stop pass finish_reason=stop after 'ready' + finish_reason_tool_calls pass finish_reason=tool_calls on the tool-call turn + finish_reason_length pass finish_reason=length at max_tokens=8 (completion_tokens=8) + usage_on_final_chunk pass final chunk carries usage {'completion_tokens': 2, 'prompt_tokens': 20, 'total_tokens': 22, 'reasoning_tokens': 0} + include_reasoning pass the model did not reason here (5 completion tokens for '9.11'); no reasoning to return + tool_choice_required recorded supported: forced get_weather({"city": "Paris"}) + long_tool_call pass 65.8 KiB of arguments in 1 call (1320 lines), streamed over 244 s, longest gap between chunks 243.4 s + +11 pass, 0 fail, 1 recorded +``` + +- The long call came back with all 1,320 lines as asked: 19,840 completion + tokens in 244 s, 81 tok/s. The engine's `qwen3_coder` parser streams a call's + name and short arguments as they are generated but holds a long argument + until it closes, so this stream sent nothing for 243 s. At this commit + `sie-server` ends a stream that sends nothing for 300 s, so one argument + that takes longer than that to generate is cut off: about 24,000 tokens for a + session alone on the GPU, fewer when other sessions share it. +- This profile does not think, so `include_reasoning` returns nothing. Its + answer, 9.11, is wrong; the case does not judge answers. +- The usage on this route carries no `cached_tokens`: the engine's cache report + is off for this profile. The session load below shows the cache at work + through the time to the first token. + +`./run_gate.sh gpu` on the same server, with OpenCode 1.19.0 and no global +OpenCode config, OpenCode's JSON event lines left out: + +``` +== before: the seeded tests +FAILED tests/test_amounts.py::test_credit_in_parentheses_is_negative - decima... +FAILED tests/test_amounts.py::test_total_nets_credits - decimal.InvalidOperat... +2 failed, 1 passed in 0.03s +== opencode run, recipe gpu, up to 50 steps and 1800 s +== after: the seeded tests must be untouched +== after: pytest in a fresh environment, pinned to the seeded config and tests +... [100%] +3 passed in 0.01s +GATE PASS (.../run-output/gate-20261015T162830Z-4g42xF) +``` + +Five runs in a row passed. Each took 5 steps and 4 tool calls (`bash`, `read`, +`edit`, `bash`), 8 to 9 seconds from OpenCode's first event to its last, with +at most 8,957 input tokens on a step. Five more runs with `"options": +{"presence_penalty": 0}` on the model also passed, with 5 or 6 steps, 4 to 8 +tool calls and at most 9,723 input tokens. + +`session_load.py` with its defaults, sessions growing to about 61,000 tokens, +and with `--turn-tokens 12000 --turns 9`, growing to about 128,000: + +| Sessions | Largest prompt | Decode p50 / p90, tok/s | Returning-turn first token p50 / p90 | Bar | +|---|---|---|---|---| +| 1 | 61,274 | 76.4 / 74.2 | 0.66 / 0.76 s | meets | +| 2 | 61,252 | 69.7 / 63.5 | 0.85 / 1.07 s | meets | +| 4 | 61,263 | 62.9 / 54.1 | 1.30 / 1.89 s | meets | +| 6 | 61,278 | 53.6 / 41.8 | 0.98 / 2.82 s | meets | +| 8 | 61,254 | 49.2 / 31.8 | 2.39 / 3.65 s | misses: first token | +| 1 | 128,203 | 71.7 / 67.5 | 1.53 / 2.00 s | meets | +| 2 | 128,288 | 64.2 / 50.5 | 2.06 / 2.60 s | meets | +| 3 | 128,214 | 47.5 / 40.4 | 2.65 / 4.21 s | misses: first token | + +- One H100 holds **6 sessions at about 61,000 tokens each, or 2 at about + 128,000**, within the bar. No turn failed at any count. +- The prefix cache carries a session from turn to turn. A returning turn with + a 61,000-token prompt reached its first token in 0.66 s with one session, + which is the time to prefill only the new input. +- Six 61,000-token sessions fit a 325,056-token cache because the shared + 16,000-token system prompt is stored once. +- The profile's 32 requests decoding at once is a scheduler limit, not a + session count. + ### SIE Cloud, GLM-5.3-Flash via SIE Relay, recorded `zai-org/GLM-5.3-Flash:relay` on an SIE Cloud endpoint that serves it, diff --git a/examples/coding-agent-opencode/conformance.py b/examples/coding-agent-opencode/conformance.py index 812fe49ea..6add6d333 100755 --- a/examples/coding-agent-opencode/conformance.py +++ b/examples/coding-agent-opencode/conformance.py @@ -57,13 +57,14 @@ REASONING_QUESTION = "Which is larger, 9.11 or 9.9? Answer with just the larger number." LONG_CALL_MIN_BYTES = 65_536 LONG_CALL_BUDGET = 32_000 -LONG_CALL_LINES = 1_500 +LONG_CALL_LINES = 1_320 LONG_CALL_SENTENCE = "The quick brown fox jumps over the lazy dog." LONG_CALL_PROMPT = ( - "Use the write_file tool once to create the file notes/pangrams.txt. Its content must be the sentence " - f'"{LONG_CALL_SENTENCE}" on {LONG_CALL_LINES:,} lines, one copy per line, about ' - f"{LONG_CALL_LINES * (len(LONG_CALL_SENTENCE) + 1):,} characters in all. Write every line out in full in that " - "one call: no ellipsis, no placeholder, no summary and no code that generates the lines." + f"Use the write_file tool once to create the file notes/pangrams.txt. Its content must be {LONG_CALL_LINES:,} " + f"lines numbered 0001 to {LONG_CALL_LINES:04d}: each line is its four-digit number, one space and the sentence " + f'"{LONG_CALL_SENTENCE}", about {LONG_CALL_LINES * (len(LONG_CALL_SENTENCE) + 6):,} characters in all. Write ' + "every line out in full in that one call: no ellipsis, no placeholder, no summary and no code that generates " + "the lines." ) WEATHER_TOOL = { @@ -759,6 +760,10 @@ def case_tool_choice_required(client: openai.OpenAI, model: str) -> Case: ) +def long_call_line(number: int) -> str: + return f"{number:04d} {LONG_CALL_SENTENCE}" + + def case_long_tool_call(client: openai.OpenAI, model: str, tool_params: dict[str, Any]) -> Case: name = "long_tool_call" messages = [{"role": "user", "content": LONG_CALL_PROMPT}] @@ -802,7 +807,9 @@ def case_long_tool_call(client: openai.OpenAI, model: str, tool_params: dict[str return Case(name, "fail", detail, observed) content = next((args.get("content") for args in arguments if args and isinstance(args.get("content"), str)), "") observed["content_lines"] = len(content.splitlines()) - observed["lines_as_asked"] = sum(line.strip() == LONG_CALL_SENTENCE for line in content.splitlines()) + observed["lines_as_asked"] = sum( + line.strip() == long_call_line(number) for number, line in enumerate(content.splitlines(), 1) + ) kib = size / 1024 if size < LONG_CALL_MIN_BYTES: detail = f"the model wrote {kib:.1f} KiB of arguments, under the {LONG_CALL_MIN_BYTES // 1024} KiB asked for" diff --git a/examples/coding-agent-opencode/tests/test_conformance.py b/examples/coding-agent-opencode/tests/test_conformance.py index d7117bfef..fb1f1e193 100644 --- a/examples/coding-agent-opencode/tests/test_conformance.py +++ b/examples/coding-agent-opencode/tests/test_conformance.py @@ -118,7 +118,8 @@ def _long_call(body: dict[str, Any]) -> dict[str, Any]: SEEN["long_calls"].append(body) mode = KNOBS.get("long_call", "write") lines = 10 if mode == "short" else conformance.LONG_CALL_LINES - arguments = {"path": "notes/pangrams.txt", "content": "\n".join([conformance.LONG_CALL_SENTENCE] * lines)} + content = "\n".join(conformance.long_call_line(number) for number in range(1, lines + 1)) + arguments = {"path": "notes/pangrams.txt", "content": content} if mode == "decline": return {"content": "That file is too long to write in one call.", "finish": "stop"} if mode == "error": From 3cdb0774e07cb75dd7bbe1f06ae4253db7762896 Mon Sep 17 00:00:00 2001 From: Krisztian Gajdar Date: Thu, 15 Oct 2026 20:22:35 +0200 Subject: [PATCH 3/8] docs(examples): record the one-GPU and cluster-route runs, and ship presence_penalty 0 --- examples/README.md | 2 +- examples/coding-agent-opencode/README.md | 212 +++++++++++++++--- .../recipes/cluster/opencode.json | 5 +- .../recipes/gpu/opencode.json | 5 +- 4 files changed, 191 insertions(+), 33 deletions(-) diff --git a/examples/README.md b/examples/README.md index cc9537708..3bd4ee0cf 100644 --- a/examples/README.md +++ b/examples/README.md @@ -20,7 +20,7 @@ to a separately maintained repository (clone and run there). | [Extract custom entities from primary sources](./named-entity-extraction) | Changing zero-shot labels across financial, healthcare, rail-safety and legal text, with the two labels that came back empty left visible | `extract` | SIE endpoint with GLiNER multi v2.1; `fetch.py` pulls the recorded run, `score.py` reproduces the counts with no key | Runnable recorded example | | [Extract the entity types you name, with no model to train](./custom-entity-types) | Asking for the 85 entity types of seven zero-shot NER sets and scoring exact spans against GPT-6 Luna, GPT-6 Sol, Claude Haiku 4.5, SIE Qwen3.8 27B, AWS Comprehend and spaCy, with a paired bootstrap and the price of each | `extract` | SIE server with GLiNER BioMed Large; `fetch.py` pulls the recorded run, `score.py` reproduces the figures with no key | Runnable recorded example | | [Answer a question from a passage, and quote the sentence you used](./chat) | Checking a generated answer for its citation, its length and whether it declines a question the passage cannot answer | `chat/completions` | SIE Cloud with Qwen3.8 27B; `fetch.py` pulls the recorded run, `score.py` reproduces the numbers with no key | Runnable recorded example | -| [Run a coding agent on SIE with OpenCode](./coding-agent-opencode) | Pointing OpenCode at a laptop server, a self-hosted cluster or SIE Cloud, and checking the agent loop the way OpenCode drives it: streaming tool-call deltas, parallel calls, tool-result round trips, a 64K-plus prompt, finish reasons, usage and the reasoning field | `chat/completions` | An SIE Cloud endpoint that lists GLM-5.3-Flash via SIE Relay, or a local server; the laptop recipe was run on Apple Silicon with Qwen3.5-4B; `conformance.py` reports every case, `run_gate.sh` makes OpenCode fix a seeded failing test | Runnable check, laptop and SIE Cloud results recorded | +| [Run a coding agent on SIE with OpenCode](./coding-agent-opencode) | Pointing OpenCode at a laptop server, one 80 GB GPU, a self-hosted cluster or SIE Cloud, and checking the agent loop the way OpenCode drives it: streaming tool-call deltas, parallel calls, tool-result round trips, a 64K-plus prompt, a 64 KiB tool call, finish reasons, usage and the reasoning field; `session_load.py` measures how many agent sessions one endpoint holds | `chat/completions` | An SIE Cloud endpoint that lists GLM-5.3-Flash via SIE Relay, or a local server; the laptop recipe was run on Apple Silicon with Qwen3.5-4B, the GPU and cluster recipes on one H100 with Qwen3.8-27B-FP8; `conformance.py` reports every case, `run_gate.sh` makes OpenCode fix a seeded failing test | Runnable check, laptop, one-GPU and SIE Cloud results recorded | | [Route requests while skipping most generation calls](./request-routing) | Assigning support, banking and assistant requests to routes with an embedding classifier and conditional generation | `encode`, `chat/completions` | SIE server with Qwen3 Embedding 4B and Qwen3.8 27B; recorded 3,000-case verification needs no API key | Runnable recorded example | | [Send each request to a cheap or a strong LLM](./request-routing/cost) | Cutting LLM spend with a bge-m3 nearest-neighbour router that picks GPT-6 Luna or GPT-6.1 Sol per request: 38.6% less than always using GPT-6.1 Sol at 95.8% of its quality on 728 held-out requests of a pre-registered study | `encode` | SIE Serverless with bge-m3; `fetch.py` pulls the recorded study, `score.py` reproduces the held-out table with no key | Runnable recorded example | | [Turn PDF pages into Markdown, scored on olmOCR-Bench](./document-to-markdown-olmocr) | Converting all 1,403 pages of Ai2's olmOCR-Bench with LightOnOCR-2-1B and scoring them with Ai2's own scorer, beside published Azure Document Intelligence and AWS Textract runs | `extract` | SIE Cloud with LightOnOCR-2-1B; `fetch.py` pulls the recorded run, `score.py` reproduces the scores with no key | Runnable recorded example | diff --git a/examples/coding-agent-opencode/README.md b/examples/coding-agent-opencode/README.md index 0b5f5e7e4..21ac0635d 100644 --- a/examples/coding-agent-opencode/README.md +++ b/examples/coding-agent-opencode/README.md @@ -111,10 +111,14 @@ and point OpenCode at it with `recipes/gpu/opencode.json`. a 32,768-token output cap, up to 32 requests decoding at once, the `qwen3_coder` tool-call parser, no speculative decoding and no thinking. The recipe sets `limit.context` and `limit.output` to exactly those two numbers. - OpenCode then sends `max_tokens: 32000`, so a prompt may grow to 230,144 - tokens before OpenCode compacts the session. + OpenCode then sends `max_tokens: 32000`, and the server accepts a prompt of + up to 230,144 tokens beside it. - The recipe names one model. Each profile is its own engine launch, so asking the same server for a second profile unloads the first. +- The recipe sends `presence_penalty: 0` with every request. The profile's own + default is 1.5, which penalises every token the answer has already used, + and code repeats its identifiers. On BFCL, 0 scored higher on both routes + below and lost no row that 1.5 got right. - For a model that thinks before it answers, `Qwen/Qwen3.8-27B-FP8:thinking` is the same weights with reasoning on. Declare it with `"reasoning": true`, `"interleaved": {"field": "reasoning_content"}` and `"options": @@ -150,20 +154,27 @@ an NVIDIA machine. ### Against your cluster -Set `limit.context` and `limit.output` in `recipes/cluster/opencode.json` to -the window your cluster advertises for the model in `/v1/models`. The recipe -reads `SIE_BASE_URL` and `SIE_API_KEY` like the Cloud one. Then: +`recipes/cluster/opencode.json` names `Qwen/Qwen3.8-27B-FP8:h100-256k-batch`, +the model and profile recorded on one GPU below, served by a cluster worker on +an 80 GB GPU. It also lists `zai-org/GLM-5.3-Flash` and +`Qwen/Qwen3-4B-Instruct-2507`. Set `limit.context` and `limit.output` for each +model to the window your cluster advertises for it in `/v1/models`. The recipe +reads `SIE_BASE_URL` and `SIE_API_KEY` like the Cloud one, and the gateway +checks the key. Then: ```sh export SIE_BASE_URL=https://your-gateway export SIE_API_KEY=... -uv run python conformance.py --base-url "$SIE_BASE_URL/v1" --model zai-org/GLM-5.3-Flash +uv run python conformance.py --base-url "$SIE_BASE_URL/v1" --model Qwen/Qwen3.8-27B-FP8:h100-256k-batch ./run_gate.sh cluster ``` -The cluster recipe is a template and has not been run. On a model whose -declared window is under 64K tokens, such as the bare `zai-org/GLM-5.3-Flash` -at 16,384, the 64K-plus prompt case is recorded rather than judged. A window +The recipe's Qwen3.8 entry was run through this repository's gateway, queue +and worker on one GPU (see +[the recorded result](#the-same-gpu-through-the-cluster-gateway-recorded)). +The GLM and Qwen3-4B entries have not been run. On a model whose declared +window is under 64K tokens, such as the bare `zai-org/GLM-5.3-Flash` at +16,384, the 64K-plus prompt case is recorded rather than judged. A window that size also runs out within a long agent session. Every path writes a JSON report, `run-output/conformance.json` by default, @@ -171,8 +182,8 @@ and prints a table. The exit code is 1 when any case fails. ## The OpenCode recipes -The three configs differ only in `baseURL`, the key and the model list. The -Cloud one in full: +The configs differ only in `baseURL`, the key and the model list. The Cloud +one in full: ```json { @@ -282,7 +293,9 @@ Every request asks for up to 4,096 tokens, except the `length` case and the long call, so a model that thinks before it answers has room to finish. The long call takes several minutes on one GPU: about 20,000 tokens of output. OpenCode's own `write` and `edit` calls carry whole files the same way, and a -server that sends nothing while a call is generated must not end the stream. The 68,000-word +server that sends nothing while a call is generated must not end the stream. +The laptop and SIE Cloud tables below were recorded before this case was +added. The 68,000-word prompt counts at least one token per word on any tokenizer that does not merge words: 71,431 with `cl100k_base`, 71,917 with `o200k_base`, 74,835 with Qwen3.5's tokenizer and 71,432 with GLM-5.3-Flash's. @@ -443,9 +456,13 @@ served by local tokens in 244 s, 81 tok/s. The engine's `qwen3_coder` parser streams a call's name and short arguments as they are generated but holds a long argument until it closes, so this stream sent nothing for 243 s. At this commit - `sie-server` ends a stream that sends nothing for 300 s, so one argument - that takes longer than that to generate is cut off: about 24,000 tokens for a - session alone on the GPU, fewer when other sessions share it. + `sie-server` ended a stream that sent nothing for 300 s, so an argument that + took longer than that to generate was cut off: about 24,000 tokens for a + session alone on the GPU, fewer when other sessions share it. From commit + `00d7d732` it writes an SSE comment every 15 s while the engine holds a + tool call, and allows the call the profile's overall budget of 1,800 s. The + case passed there too, with the same 239 s between chunks, since a comment + is not a chunk. - This profile does not think, so `include_reasoning` returns nothing. Its answer, 9.11, is wrong; the case does not judge answers. - The usage on this route carries no `cached_tokens`: the engine's cache report @@ -453,7 +470,8 @@ served by local through the time to the first token. `./run_gate.sh gpu` on the same server, with OpenCode 1.19.0 and no global -OpenCode config, OpenCode's JSON event lines left out: +OpenCode config, before the recipe set `presence_penalty`, OpenCode's JSON +event lines left out: ``` == before: the seeded tests @@ -468,11 +486,12 @@ FAILED tests/test_amounts.py::test_total_nets_credits - decimal.InvalidOperat... GATE PASS (.../run-output/gate-20261015T162830Z-4g42xF) ``` -Five runs in a row passed. Each took 5 steps and 4 tool calls (`bash`, `read`, -`edit`, `bash`), 8 to 9 seconds from OpenCode's first event to its last, with -at most 8,957 input tokens on a step. Five more runs with `"options": -{"presence_penalty": 0}` on the model also passed, with 5 or 6 steps, 4 to 8 -tool calls and at most 9,723 input tokens. +Five runs in a row passed at the profile's `presence_penalty` of 1.5. Each +took 5 steps and 4 tool calls (`bash`, `read`, `edit`, `bash`), 8 to 9 seconds +from OpenCode's first event to its last, with at most 8,957 input tokens on a +step. Five more runs with the recipe's `presence_penalty: 0` also passed, with +5 or 6 steps, 4 to 8 tool calls and at most 9,723 input tokens. The seeded +gate does not tell the two settings apart; BFCL below does. `session_load.py` with its defaults, sessions growing to about 61,000 tokens, and with `--turn-tokens 12000 --turns 9`, growing to about 128,000: @@ -498,6 +517,119 @@ and with `--turn-tokens 12000 --turns 9`, growing to about 128,000: - The profile's 32 requests decoding at once is a scheduler limit, not a session count. +BFCL v3 with the harness and settings of +[the SIE Relay record](#bfcl-through-sie-relay): the full `simple` split and +the same 200-row `multiple` subsample, one request at a time, greedy decoding, +an output cap of 8,192 tokens, `tool_choice: "auto"`, and dotted function +names sent with `_`. Both routes ran on the same GPU, the cluster route as +[recorded below](#the-same-gpu-through-the-cluster-gateway-recorded): + +| Route | `presence_penalty` | `simple`, 400 rows | `multiple`, 200 rows | +|---|---|---|---| +| Single server | 1.5, the profile's default | 359, 0.8975 | 182, 0.910 | +| Single server | 0, as the recipes set it | 370, 0.925 | 184, 0.920 | +| Cluster gateway | 1.5 | 370, 0.925 | 187, 0.935 | +| Cluster gateway | 0 | 378, 0.945 | 189, 0.945 | + +- On each route every row that passed at 1.5 also passed at 0. Setting 0 + turned 13 failed rows into passes on the single server and 10 through the + gateway, so the recipes set it. +- The two routes do not agree. They render the same tools turn to prompts of + different length, 306 tokens on the single server and 295 through the + gateway for the conformance check's weather turn, so their greedy answers + differ, and the gateway's scored higher at both settings. Which rendering is + closer to the model's training format was not established. + +One real issue on this repository, run twice through the single-server +recipe: [issue #717](https://github.com/superlinked/sie/issues/717), a +metering bug in the GLiNER adapters, on a checkout of commit `03dd3fd3`. +OpenCode was asked to fix it, add a regression test and run it, with a +100-step budget and no other help: + +- At the profile's `presence_penalty` of 1.5: 100 steps and 100 tool calls in + 370 s, prompts up to 76,868 tokens, no failed request. After reading the + code it re-ran one shell command over and over: from the 30th call on, + every tenth call and each of the last four was that command. It made no + edit. +- With `presence_penalty: 0`: 100 steps and 104 tool calls in 895 s, prompts + up to 100,258 tokens, no failed request. It changed the adapter code and + added a regression test, which still failed when the step budget ran out. +- Neither run fixed the issue. Both held a session past 75,000 tokens on one + GPU without a serving error. + +### The same GPU through the cluster gateway, recorded + +The same model, profile and H100, served the way a cluster serves it: this +repository's `sie-gateway`, NATS, `sie-server-sidecar` and one `sie-server` +worker, on one machine, with the gateway on `127.0.0.1:31000`. The worker was +ready 100 s after the stack started when the kernels were already compiled, +and 270 s after in a new container. + +The conformance check at commit `00d7d732`, which keeps a chat stream alive +while a long tool call is generated: + +``` +model Qwen/Qwen3.8-27B-FP8:h100-256k-batch at http://127.0.0.1:31000/v1 +served by local + + tool_choice_auto pass tool_choice=auto accepted on the tools turn + streaming_tool_call_deltas pass 1 call: get_weather({"city":"Paris"}); 2 deltas; finish_reason=tool_calls + parallel_tool_calls pass 2 calls in one turn: get_weather({"city":"Paris"}), get_weather({"city":"Tokyo"}) + tool_result_round_trip pass final answer used the tool result: "It's currently 18°C and overcast in Paris." + prompt_above_64k_tokens pass 74847 prompt tokens accepted and answered; needle found + finish_reason_stop pass finish_reason=stop after 'ready' + finish_reason_tool_calls pass finish_reason=tool_calls on the tool-call turn + finish_reason_length pass finish_reason=length at max_tokens=8 (completion_tokens=8) + usage_on_final_chunk pass final chunk carries usage {'completion_tokens': 2, 'prompt_tokens': 20, 'total_tokens': 22, 'prompt_tokens_details': {'cached_tokens': 0}} + include_reasoning pass the model did not reason here (5 completion tokens for '9.11'); no reasoning to return + tool_choice_required recorded supported: forced get_weather({"city":"Paris"}) + long_tool_call pass 65.8 KiB of arguments in 1 call (1320 lines), streamed over 240 s, longest gap between chunks 0.1 s + +11 pass, 0 fail, 1 recorded +``` + +- The long call came back whole: 19,840 completion tokens, all 1,320 lines as + asked. The gateway passes on an empty chunk while the worker holds a call, + so the stream never went quiet for more than 0.1 s. At commit `461c5a58`, + before that change, the same case failed after 30 s with + `inter_chunk_timeout`, with no argument bytes delivered. +- This route reports `cached_tokens` in the usage, which the session load + below reads. +- One request with a 1.1 MB body, a 196,449-token prompt, was answered in + 23 s. + +The seeded gate, `./run_gate.sh cluster` with `SIE_BASE_URL` set to the +gateway, passed 5 of 5 runs at the profile's `presence_penalty` of 1.5, at +commit `461c5a58`, and 3 of 3 with the recipe's `presence_penalty: 0`, at +commit `00d7d732`. Each run took 5 steps and 4 or 5 tool calls, 7 to 9 +seconds, with at most 9,240 input tokens on a step. + +BFCL is in [the table above](#one-gpu-qwen38-27b-fp8-recorded). The session +load below was recorded at commit `461c5a58`. + +`session_load.py` through the gateway: + +| Sessions | Largest prompt | Decode p50 / p90, tok/s | Returning-turn first token p50 / p90 | Returning-turn prompt cached | Bar | +|---|---|---|---|---|---| +| 1 | 60,992 | 76.9 / 74.2 | 0.66 / 0.75 s | 86.3% | meets | +| 2 | 61,105 | 71.9 / 63.6 | 0.80 / 1.08 s | 86.2% | meets | +| 4 | 61,265 | 62.8 / 54.7 | 1.67 / 2.03 s | 86.2% | meets | +| 6 | 61,274 | 53.4 / 38.7 | 1.23 / 2.51 s | 86.2% | meets | +| 8 | 61,266 | 46.8 / 34.7 | 2.10 / 3.59 s | 86.3% | misses: first token | +| 1 | 127,973 | 72.3 / 67.9 | 1.57 / 1.96 s | 85.6% | meets | +| 2 | 127,742 | 61.2 / 43.6 | 1.84 / 2.15 s | 85.5% | meets | + +- The gateway adds nothing measurable to the single server's capacity: 6 + sessions at about 61,000 tokens and 2 at about 128,000 meet the bar on both. +- About 86% of each returning turn's prompt came from the cache. The rest, + about 6,200 tokens a turn on average, is the size of the turn's new input, + so what the session sends again is reused. + +The same real issue as above, once through the cluster recipe with +`presence_penalty: 0` at commit `00d7d732`: 100 steps and 104 tool calls in +751 s, prompts up to 139,712 tokens, no failed request. It spent most steps on +Python probes of the word splitter and made no edit. + ### SIE Cloud, GLM-5.3-Flash via SIE Relay, recorded `zai-org/GLM-5.3-Flash:relay` on an SIE Cloud endpoint that serves it, @@ -617,14 +749,17 @@ uv run python conformance.py --model sie-fake:chat usage_on_final_chunk pass final chunk carries usage {'completion_tokens': 40, 'prompt_tokens': 7, 'total_tokens': 47} include_reasoning pass 8 reasoning_content deltas, 40 chars; message.reasoning_content 40 chars; none without the flag tool_choice_required recorded supported: forced get_weather({"city": "Say hello."}) + long_tool_call recorded not run: max_tokens=32000 refused: 400 INVALID_INPUT: max_tokens (32000) exceeds model cap (4096) -9 pass, 1 fail, 1 recorded +9 pass, 1 fail, 2 recorded ``` - A pass here checks SIE's single-node chat wire, not a model: the streamed call deltas, the tool result sent back, reasoning returned only on request, the finish reasons, the usage on the last chunk and the declared window. - `parallel_tool_calls` fails because the fake makes one call per turn. +- `long_tool_call` is not run because the profile caps output at 4,096 + tokens and refuses the 32,000 the call asks for. - The fake does not read the prompt, so the needle is not found, and the answer to "Reply with the single word: ready." is hash-derived tokens. @@ -649,8 +784,15 @@ What runs anywhere, and in this repository's CI, is the test suite: missing from the non-streamed answer, `include_reasoning` or `tool_choice: "auto"` refused, busy refusals, an error event mid-stream, a `/v1/models` answered as a bare list, a model that thinks for thousands - of tokens, and a model that answers `MODEL_LOADING` for its first requests. - Each verdict is checked for what it accepts and what it refuses. + of tokens, a model that answers `MODEL_LOADING` for its first requests, and + a long tool call that arrives whole, short, truncated, cut by an error + event, ended by the output cap or refused for its `max_tokens`. Each verdict + is checked for what it accepts and what it refuses. +- `tests/test_session_load.py` runs `session_load.py` against a scripted stub + that counts one token per word and reports a cached prefix: each session + sends its whole transcript back with the answers as streamed, the input is + sized with the endpoint's count, a failed turn ends its session and fails + the run, and the bar is judged at p90. - `tests/test_messages_conformance.py` runs `conformance.py --api messages` against a scripted Anthropic Messages stub that plays a conforming event stream, then breaks one thing per test: usage missing from @@ -662,13 +804,16 @@ What runs anywhere, and in this repository's CI, is the test suite: environment planted in the worked copy, a caller's `PYTEST_ADDOPTS` or `PYTEST_PLUGINS` that would deselect the failing tests, the step budget handed to the build agent, a run that outlives its deadline, with and - without a child that ignores TERM, and two runs started in the same second. + without a child that ignores TERM, two runs started in the same second, and + the key check, which applies only to a recipe that reads `SIE_BASE_URL` and + `SIE_API_KEY`. ```sh uv run pytest ``` -`tests/test_conformance.py` and `tests/test_run_gate.py` check the checker, +`tests/test_conformance.py`, `tests/test_session_load.py` and +`tests/test_run_gate.py` check the checkers, and `tests/test_fake_engine.py` checks SIE's chat route with a scripted model. A model's endpoint is only checked by running `conformance.py` against it. @@ -687,9 +832,16 @@ A model's endpoint is only checked by running `conformance.py` against it. - **Not a claim about every laptop.** The laptop recipe was run on one Apple Silicon machine with one 4B model. NVIDIA and other models have not been run. -- **Not a measurement of SIE's own GPU serving.** Through SIE Relay the - model runs on a partner's servers. SIE routes the request, checks the wire - and meters it. A dedicated GPU deployment of the same model is a different - serving path and has its own window and limits. +- **The SIE Relay results are not a measurement of SIE's own GPU serving.** + Through SIE Relay the model runs on a partner's servers. SIE routes the + request, checks the wire and meters it. The one-GPU results are SIE's own + serving, of a different model. +- **One GPU, one machine, one day.** The one-GPU results were recorded on one + H100 on 2026-10-15. Another card, driver or engine version can move every + number, and sessions with pauses between turns, as real agents have, load + the GPU less than `session_load.py` does. +- **Not a measure of how well Qwen3.8-27B codes.** BFCL scores single calls, + the seeded gate is a twenty-line fix, and the one real issue was not fixed + in either run. - **One endpoint, one day.** The Cloud results were recorded once, on 2026-10-14. diff --git a/examples/coding-agent-opencode/recipes/cluster/opencode.json b/examples/coding-agent-opencode/recipes/cluster/opencode.json index 2ba205706..b768d314e 100644 --- a/examples/coding-agent-opencode/recipes/cluster/opencode.json +++ b/examples/coding-agent-opencode/recipes/cluster/opencode.json @@ -35,7 +35,10 @@ "context": 262144, "output": 32768 }, - "tool_call": true + "tool_call": true, + "options": { + "presence_penalty": 0 + } } } } diff --git a/examples/coding-agent-opencode/recipes/gpu/opencode.json b/examples/coding-agent-opencode/recipes/gpu/opencode.json index 365fbe528..c607c2ea8 100644 --- a/examples/coding-agent-opencode/recipes/gpu/opencode.json +++ b/examples/coding-agent-opencode/recipes/gpu/opencode.json @@ -19,7 +19,10 @@ "context": 262144, "output": 32768 }, - "tool_call": true + "tool_call": true, + "options": { + "presence_penalty": 0 + } } } } From e7876a316fbbd20b512f509b90f1116397fa6050 Mon Sep 17 00:00:00 2001 From: Krisztian Gajdar Date: Thu, 15 Oct 2026 20:31:27 +0200 Subject: [PATCH 4/8] fix(examples): pass the long call only for the asked-for file, and time decode over what was streamed --- examples/coding-agent-opencode/README.md | 17 ++++--- examples/coding-agent-opencode/conformance.py | 19 +++++++- .../coding-agent-opencode/session_load.py | 37 ++++++++++++--- .../tests/test_conformance.py | 9 +++- .../tests/test_session_load.py | 45 ++++++++++++++++++- 5 files changed, 112 insertions(+), 15 deletions(-) diff --git a/examples/coding-agent-opencode/README.md b/examples/coding-agent-opencode/README.md index 21ac0635d..f64bb2f11 100644 --- a/examples/coding-agent-opencode/README.md +++ b/examples/coding-agent-opencode/README.md @@ -287,7 +287,7 @@ the laptop and cluster recipes. The report's `tools_turn_params` says which. | `usage_on_final_chunk` | with `stream_options.include_usage`, the last chunk carries `prompt_tokens` and `completion_tokens` | | `include_reasoning` | the flag is accepted streamed and not streamed, no raw `` tag leaks into content, and without the flag no `reasoning_content` arrives. When the model reasoned, the stream carries `delta.reasoning_content` and the non-streamed answer `message.reasoning_content`. A model reasoned when it returned reasoning, or when its completion tokens exceed the UTF-8 bytes of its visible text by more than 16, since a byte-level tokenizer spends at most one token per byte. It did not reason when no answer took more than 16 completion tokens. Between the two, whether it reasoned is unknown and the case is recorded, not judged | | `tool_choice_required` | recorded as `supported` when a call is forced or `refused` on a 400 or 422, neither a failure; accepted-but-ignored fails, and so does any other error | -| `long_tool_call` | one `write_file` call asked for 1,320 numbered lines, 67,363 bytes of arguments, arrives whole: the arguments parse as a JSON object over 64 KiB and the turn ends with `tool_calls`. The request sends `max_tokens: 32000`, as OpenCode does. A stream that breaks or arguments that do not parse fail. A model that answers in text, writes less than 64 KiB or runs out of `max_tokens` is recorded, and so is an endpoint that refuses `max_tokens: 32000`. The report keeps the argument size, how many lines came back as asked, the time to the first chunk, how long the stream ran and its longest gap between chunks | +| `long_tool_call` | one `write_file` call asked for 1,320 numbered lines, 67,363 bytes of arguments, arrives whole: the arguments parse as a JSON object over 64 KiB, the turn ends with `tool_calls`, and the one call writes `notes/pangrams.txt` with every line as asked. The request sends `max_tokens: 32000`, as OpenCode does. A stream that breaks or arguments that do not parse fail. A model that answers in text, writes less than 64 KiB, writes another file or other lines, or runs out of `max_tokens` is recorded, and so is an endpoint that refuses `max_tokens: 32000`. The report keeps the argument size, how many lines came back as asked, the time to the first chunk, how long the stream ran and its longest gap between chunks | Every request asks for up to 4,096 tokens, except the `length` case and the long call, so a model that thinks before it answers has room to finish. The @@ -347,7 +347,11 @@ agent-shaped sessions against one endpoint: The input is sized with the endpoint's own token count of the shared prompt. For each session count the report gives the decode rate after the first token and the time to the first token on returning turns, at the median and at p90, -where the p90 decode rate is the rate nine turns in ten reached or beat. It +where the p90 decode rate is the rate nine turns in ten reached or beat. The +decode rate counts the tokens after the first streamed piece, estimated by +their share of the streamed text, over the time from the first piece to the +last. A turn that ends without the finish reason `stop` or `length` counts as +failed, and answers that stopped before `--output-tokens` are counted. It also gives the share of returning-turn prompt tokens the server reported as cached, when it reports them. A session count meets the bar when, at p90, every session decodes at 30 tok/s or more and a returning turn starts within @@ -786,13 +790,16 @@ What runs anywhere, and in this repository's CI, is the test suite: a `/v1/models` answered as a bare list, a model that thinks for thousands of tokens, a model that answers `MODEL_LOADING` for its first requests, and a long tool call that arrives whole, short, truncated, cut by an error - event, ended by the output cap or refused for its `max_tokens`. Each verdict + event, ended by the output cap, refused for its `max_tokens`, or valid but + for another file or with other lines. Each verdict is checked for what it accepts and what it refuses. - `tests/test_session_load.py` runs `session_load.py` against a scripted stub that counts one token per word and reports a cached prefix: each session sends its whole transcript back with the answers as streamed, the input is - sized with the endpoint's count, a failed turn ends its session and fails - the run, and the bar is judged at p90. + sized with the endpoint's count, a failed turn or one without `stop` or + `length` ends its session and fails the run, returned reasoning counts + toward the first token and the decode window, short answers are counted, + and the bar is judged at p90. - `tests/test_messages_conformance.py` runs `conformance.py --api messages` against a scripted Anthropic Messages stub that plays a conforming event stream, then breaks one thing per test: usage missing from diff --git a/examples/coding-agent-opencode/conformance.py b/examples/coding-agent-opencode/conformance.py index 8a8550a0a..d67297923 100755 --- a/examples/coding-agent-opencode/conformance.py +++ b/examples/coding-agent-opencode/conformance.py @@ -65,9 +65,10 @@ LONG_CALL_MIN_BYTES = 65_536 LONG_CALL_BUDGET = 32_000 LONG_CALL_LINES = 1_320 +LONG_CALL_PATH = "notes/pangrams.txt" LONG_CALL_SENTENCE = "The quick brown fox jumps over the lazy dog." LONG_CALL_PROMPT = ( - f"Use the write_file tool once to create the file notes/pangrams.txt. Its content must be {LONG_CALL_LINES:,} " + f"Use the write_file tool once to create the file {LONG_CALL_PATH}. Its content must be {LONG_CALL_LINES:,} " f"lines numbered 0001 to {LONG_CALL_LINES:04d}: each line is its four-digit number, one space and the sentence " f'"{LONG_CALL_SENTENCE}", about {LONG_CALL_LINES * (len(LONG_CALL_SENTENCE) + 6):,} characters in all. Write ' "every line out in full in that one call: no ellipsis, no placeholder, no summary and no code that generates " @@ -410,6 +411,10 @@ def _complete_once(client: openai.OpenAI, model: str, messages: list[dict[str, A return result +def _describe_names(calls: list[ToolCall]) -> str: + return ", ".join(str(call.name) for call in calls) + + def _describe_calls(result: StreamResult) -> str: return ", ".join(f"{call.name}({call.arguments})" for call in result.calls()) or "no tool call" @@ -833,6 +838,18 @@ def case_long_tool_call(client: openai.OpenAI, model: str, tool_params: dict[str if size < LONG_CALL_MIN_BYTES: detail = f"the model wrote {kib:.1f} KiB of arguments, under the {LONG_CALL_MIN_BYTES // 1024} KiB asked for" return Case(name, "recorded", detail, observed) + paths = [args.get("path") for args in arguments if args] + as_asked = ( + [call.name for call in calls] == ["write_file"] + and paths == [LONG_CALL_PATH] + and observed["lines_as_asked"] == LONG_CALL_LINES == observed["content_lines"] + ) + if not as_asked: + detail = ( + f"{kib:.1f} KiB arrived as JSON, but not the file asked for: {_describe_names(calls)} to {paths}, " + f"{observed['lines_as_asked']} of {observed['content_lines']} lines as asked, {LONG_CALL_LINES} wanted" + ) + return Case(name, "recorded", detail, observed) detail = f"{kib:.1f} KiB of arguments in {len(calls)} call ({observed['content_lines']} lines), {timing}" return Case(name, "pass", detail, observed) diff --git a/examples/coding-agent-opencode/session_load.py b/examples/coding-agent-opencode/session_load.py index aea7ab4e8..5f27f8539 100755 --- a/examples/coding-agent-opencode/session_load.py +++ b/examples/coding-agent-opencode/session_load.py @@ -14,7 +14,11 @@ Each turn records the time to the first streamed token, the decode rate after it, the prompt tokens and, when the server reports them, the cached prompt -tokens. A returning turn is any turn after a session's first: with a working +tokens. The decode rate is the tokens after the first streamed piece, estimated +by their share of the streamed text, answer and any returned reasoning, over +the time from the first piece to the last. A turn whose stream ends without +the finish reason ``stop`` or ``length`` counts as failed, and the summary +counts short answers, turns that stopped before ``--output-tokens``. A returning turn is any turn after a session's first: with a working prefix cache only its new input is prefilled. The summary for each session count gives the median and p90 of both, where the p90 decode rate is the rate nine turns in ten reached or beat, and whether that count meets the bar of at @@ -65,6 +69,7 @@ class Turn: decode_tps: float | None = None seconds: float = 0.0 finish_reason: str | None = None + full_answer: bool = False error: str | None = None @@ -95,6 +100,7 @@ def stream_turn( ) -> str: started = time.monotonic() first = last = None + pieces: list[str] = [] content = "" try: stream = client.chat.completions.create( @@ -113,22 +119,39 @@ def stream_turn( for choice in chunk.choices or []: if choice.finish_reason: record.finish_reason = choice.finish_reason - text = choice.delta.content if choice.delta else None - if text: + text = (choice.delta.content if choice.delta else None) or "" + reasoning = _reasoning(choice.delta) + if text or reasoning: now = time.monotonic() first = now if first is None else first last = now + pieces.append(text + reasoning) content += text except (openai.APIError, httpx.HTTPError) as exc: record.error = f"{type(exc).__name__}: {exc}"[:300] + if record.error is None and record.finish_reason not in ("stop", "length"): + reason = record.finish_reason + record.error = "the stream ended without a finish reason" if reason is None else f"finish_reason {reason!r}" record.seconds = time.monotonic() - started - if first is not None: + record.full_answer = (record.completion_tokens or 0) >= max_tokens + streamed = sum(len(piece) for piece in pieces) + if first is not None and last is not None: record.ttft_s = first - started - if last is not None and last > first and record.completion_tokens and record.completion_tokens > 1: - record.decode_tps = (record.completion_tokens - 1) / (last - first) + if last > first and record.completion_tokens and streamed: + after_first = record.completion_tokens * (streamed - len(pieces[0])) / streamed + record.decode_tps = after_first / (last - first) return content +def _reasoning(delta: Any) -> str: + if delta is None: + return "" + value = getattr(delta, "reasoning_content", None) + if value is None and isinstance(getattr(delta, "model_extra", None), dict): + value = delta.model_extra.get("reasoning_content") + return value if isinstance(value, str) else "" + + def run_session( client: openai.OpenAI, model: str, @@ -168,6 +191,7 @@ def summarise(sessions: int, turns: list[Turn], wall_s: float) -> dict[str, Any] if reported else None ) + short = sum(1 for turn in done if not turn.full_answer) decode_p90 = _percentile(rates, 0.1) ttft_p90 = _percentile(returning_ttft, 0.9) errors = len(turns) - len(done) @@ -182,6 +206,7 @@ def summarise(sessions: int, turns: list[Turn], wall_s: float) -> dict[str, Any] "sessions": sessions, "turns": len(turns), "errors": errors, + "short_answers": short, "max_prompt_tokens": max((turn.prompt_tokens or 0 for turn in done), default=0), "decode_tps_p50": _round(_percentile(rates, 0.5), 1), "decode_tps_p90": _round(decode_p90, 1), diff --git a/examples/coding-agent-opencode/tests/test_conformance.py b/examples/coding-agent-opencode/tests/test_conformance.py index fb1f1e193..3b668033d 100644 --- a/examples/coding-agent-opencode/tests/test_conformance.py +++ b/examples/coding-agent-opencode/tests/test_conformance.py @@ -119,7 +119,12 @@ def _long_call(body: dict[str, Any]) -> dict[str, Any]: mode = KNOBS.get("long_call", "write") lines = 10 if mode == "short" else conformance.LONG_CALL_LINES content = "\n".join(conformance.long_call_line(number) for number in range(1, lines + 1)) - arguments = {"path": "notes/pangrams.txt", "content": content} + if mode == "wrong_content": + content = "\n".join( + f"{number:04d} Lorem ipsum dolor sit amet, consectetur adipiscing." for number in range(1, lines + 1) + ) + path = "notes/other.txt" if mode == "wrong_path" else conformance.LONG_CALL_PATH + arguments = {"path": path, "content": content} if mode == "decline": return {"content": "That file is too long to write in one call.", "finish": "stop"} if mode == "error": @@ -645,6 +650,8 @@ def test_a_long_tool_call_passes_when_the_whole_call_arrives(base_url: str) -> N ("truncated", "fail", "but not a JSON object: truncated or corrupted"), ("stop", "fail", "finish_reason is 'stop' after a complete call"), ("error", "fail", "request failed: None TIMEOUT: inter-chunk timeout"), + ("wrong_content", "recorded", "but not the file asked for: write_file to ['notes/pangrams.txt'], 0 of 1320"), + ("wrong_path", "recorded", "but not the file asked for: write_file to ['notes/other.txt'], 1320 of 1320"), ], ) def test_a_long_tool_call_separates_the_model_declining_from_the_wire_breaking( diff --git a/examples/coding-agent-opencode/tests/test_session_load.py b/examples/coding-agent-opencode/tests/test_session_load.py index 1038afbe3..41a604852 100644 --- a/examples/coding-agent-opencode/tests/test_session_load.py +++ b/examples/coding-agent-opencode/tests/test_session_load.py @@ -17,6 +17,7 @@ from pathlib import Path from typing import Any +import openai import pytest import session_load @@ -61,14 +62,21 @@ def do_POST(self) -> None: self.end_headers() self.wfile.write(raw) return - answer = ANSWER.split()[: body["max_tokens"]] + answer = ANSWER.split()[: KNOBS.get("answer_words", body["max_tokens"])] + thinking = ["think"] * KNOBS.get("reasoning_words", 0) if len(body["messages"]) > 2 else [] self.send_response(200) self.send_header("content-type", "text/event-stream") self.end_headers() base = {"id": "chatcmpl-stub", "object": "chat.completion.chunk", "created": 1, "model": "stub-model"} + for word in thinking: + delta = {"reasoning_content": word + " "} + self._send({**base, "choices": [{"index": 0, "delta": delta, "finish_reason": None}]}) for word in answer: self._send({**base, "choices": [{"index": 0, "delta": {"content": word + " "}, "finish_reason": None}]}) - self._send({**base, "choices": [{"index": 0, "delta": {}, "finish_reason": "length"}]}) + finish = KNOBS.get("finish", "length" if len(answer) >= body["max_tokens"] else "stop") + if finish is not None: + self._send({**base, "choices": [{"index": 0, "delta": {}, "finish_reason": finish}]}) + answer = thinking + answer usage = { "prompt_tokens": len(words), "completion_tokens": len(answer), @@ -184,3 +192,36 @@ def turn(number: int, ttft: float, tps: float) -> session_load.Turn: assert session_load.summarise(1, slow_ttft, 1.0)["meets_bar"] is False slow_decode = fast[:-2] + [turn(11, 0.5, 20.0), turn(12, 0.5, 20.0)] assert session_load.summarise(1, slow_decode, 1.0)["meets_bar"] is False + + +@pytest.mark.parametrize(("finish", "error"), [(None, "without a finish reason"), ("content_filter", "content_filter")]) +def test_a_turn_without_stop_or_length_fails( + base_url: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, finish: str | None, error: str +) -> None: + KNOBS["finish"] = finish + with pytest.raises(SystemExit, match="could not size the prompt"): + _main(base_url, tmp_path, monkeypatch, "--sessions", "1") + record = session_load.Turn(sessions=1, session=1, turn=1) + client = openai.OpenAI(base_url=base_url, api_key="none", max_retries=0) + session_load.stream_turn(client, "stub-model", [{"role": "user", "content": "hi"}], record, 20) + assert error in (record.error or "") + + +def test_reasoning_counts_toward_the_first_token_and_the_decode_window( + base_url: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + KNOBS["reasoning_words"] = 30 + _, report = _main(base_url, tmp_path, monkeypatch, "--sessions", "1") + turn = next(turn for turn in report["turns"] if turn["turn"] == 2) + assert turn["completion_tokens"] == 50 + assert turn["decode_tps"] > 0 + assert turn["error"] is None + + +def test_answers_that_stop_short_are_counted(base_url: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + KNOBS["answer_words"] = 5 + code, report = _main(base_url, tmp_path, monkeypatch, "--sessions", "1") + assert code == 0 + (level,) = report["levels"] + assert (level["errors"], level["short_answers"]) == (0, 3) + assert {turn["finish_reason"] for turn in report["turns"]} == {"stop"} From 2940e512e1117fdc9ab5f642e766ae3b523eadf2 Mon Sep 17 00:00:00 2001 From: Krisztian Gajdar Date: Thu, 15 Oct 2026 20:31:59 +0200 Subject: [PATCH 5/8] docs(examples): say how hidden reasoning affects the decode rate --- examples/coding-agent-opencode/README.md | 6 ++++-- examples/coding-agent-opencode/session_load.py | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/examples/coding-agent-opencode/README.md b/examples/coding-agent-opencode/README.md index f64bb2f11..1fe5a18c5 100644 --- a/examples/coding-agent-opencode/README.md +++ b/examples/coding-agent-opencode/README.md @@ -350,8 +350,10 @@ and the time to the first token on returning turns, at the median and at p90, where the p90 decode rate is the rate nine turns in ten reached or beat. The decode rate counts the tokens after the first streamed piece, estimated by their share of the streamed text, over the time from the first piece to the -last. A turn that ends without the finish reason `stop` or `length` counts as -failed, and answers that stopped before `--output-tokens` are counted. It +last. A model that thinks without returning its reasoning adds tokens the +text does not show, which raises that rate. A turn that ends without the +finish reason `stop` or `length` counts as failed, and answers that stopped +before `--output-tokens` are counted. It also gives the share of returning-turn prompt tokens the server reported as cached, when it reports them. A session count meets the bar when, at p90, every session decodes at 30 tok/s or more and a returning turn starts within diff --git a/examples/coding-agent-opencode/session_load.py b/examples/coding-agent-opencode/session_load.py index 5f27f8539..64ab9f8dd 100755 --- a/examples/coding-agent-opencode/session_load.py +++ b/examples/coding-agent-opencode/session_load.py @@ -16,7 +16,8 @@ it, the prompt tokens and, when the server reports them, the cached prompt tokens. The decode rate is the tokens after the first streamed piece, estimated by their share of the streamed text, answer and any returned reasoning, over -the time from the first piece to the last. A turn whose stream ends without +the time from the first piece to the last; reasoning the model does not return +is counted but not seen, and raises the rate. A turn whose stream ends without the finish reason ``stop`` or ``length`` counts as failed, and the summary counts short answers, turns that stopped before ``--output-tokens``. A returning turn is any turn after a session's first: with a working prefix cache only its new input is prefilled. The summary for each session From b77f69486630fafd62b94356efb5b1b808eca4d2 Mon Sep 17 00:00:00 2001 From: Krisztian Gajdar Date: Thu, 15 Oct 2026 20:32:28 +0200 Subject: [PATCH 6/8] docs(examples): rewrap the session load docstring --- .../coding-agent-opencode/session_load.py | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/examples/coding-agent-opencode/session_load.py b/examples/coding-agent-opencode/session_load.py index 64ab9f8dd..212c7e3ad 100755 --- a/examples/coding-agent-opencode/session_load.py +++ b/examples/coding-agent-opencode/session_load.py @@ -14,21 +14,25 @@ Each turn records the time to the first streamed token, the decode rate after it, the prompt tokens and, when the server reports them, the cached prompt -tokens. The decode rate is the tokens after the first streamed piece, estimated -by their share of the streamed text, answer and any returned reasoning, over -the time from the first piece to the last; reasoning the model does not return -is counted but not seen, and raises the rate. A turn whose stream ends without -the finish reason ``stop`` or ``length`` counts as failed, and the summary -counts short answers, turns that stopped before ``--output-tokens``. A returning turn is any turn after a session's first: with a working -prefix cache only its new input is prefilled. The summary for each session -count gives the median and p90 of both, where the p90 decode rate is the rate -nine turns in ten reached or beat, and whether that count meets the bar of at +tokens. The decode rate is the tokens after the first streamed piece, +estimated by their share of the streamed text, answer and any returned +reasoning, over the time from the first piece to the last; reasoning the model +does not return is counted but not seen, and raises the rate. A turn whose +stream ends without the finish reason ``stop`` or ``length`` counts as failed. + +A returning turn is any turn after a session's first: with a working prefix +cache only its new input is prefilled. The summary for each session count +gives the median and p90 of the decode rate and of the returning turns' time +to the first token, where the p90 decode rate is the rate nine +turns in ten reached or beat; the count of short answers, turns that stopped +before ``--output-tokens``; and whether that session count meets the bar of at least 30 tok/s and at most 3 s to the first token on returning turns, both at p90, with no failed turn. -The input is built from common English words in random order, sized with the endpoint's own -token count of the shared prompt, so the sizes hold on any tokenizer to within -a few percent; the report keeps the prompt tokens the server counted. +The input is built from common English words in random order, sized with the +endpoint's own token count of the shared prompt, so the sizes hold on any +tokenizer to within a few percent; the report keeps the prompt tokens the +server counted. """ from __future__ import annotations From af5d75f93de33b1bf8c1610433820318d19037ba Mon Sep 17 00:00:00 2001 From: Krisztian Gajdar Date: Thu, 15 Oct 2026 21:19:46 +0200 Subject: [PATCH 7/8] docs(examples): record conformance and session load at the final commit --- examples/coding-agent-opencode/README.md | 138 +++++++++++++---------- 1 file changed, 77 insertions(+), 61 deletions(-) diff --git a/examples/coding-agent-opencode/README.md b/examples/coding-agent-opencode/README.md index 1fe5a18c5..f3478030a 100644 --- a/examples/coding-agent-opencode/README.md +++ b/examples/coding-agent-opencode/README.md @@ -421,11 +421,11 @@ took 7 steps and 6 tool calls. `Qwen/Qwen3.8-27B-FP8:h100-256k-batch` on one NVIDIA H100 80GB HBM3 with 8 CPU cores and 64 GiB of memory, 2026-10-15. `sie-server` ran from this repository -at commit `461c5a58` with the `sglang-cu130` bundle (SGLang 0.5.20), started -as `sie-server serve -b sglang-cu130 --preload -Qwen/Qwen3.8-27B-FP8:h100-256k-batch`. The `docker run` line above starts the -same server, profile and bundle from the published image; the runs below did -not use the image. +with the `sglang-cu130` bundle (SGLang 0.5.20), started as `sie-server serve +-b sglang-cu130 --preload Qwen/Qwen3.8-27B-FP8:h100-256k-batch`, in three +sessions on machines of that kind. Each result below names the commit it was +recorded at. The `docker run` line above starts the same server, profile and +bundle from the published image; the runs below did not use the image. What the engine reported once loaded: @@ -434,9 +434,11 @@ What the engine reported once loaded: - up to 32 requests decoding at once; - 28.5 GB of weights and 19.8 GB of KV cache. -The server answered its first request 197 s after it started in a new -container, with the weights already downloaded, and 76 s after a restart in the -same container, with the kernels already compiled. +The server answered its first request 197 s and 229 s after it started in two +new containers, with the weights already downloaded, and 76 s after a restart +in the same container, with the kernels already compiled. + +The conformance check at commit `ac9902d3`: ``` model Qwen/Qwen3.8-27B-FP8:h100-256k-batch at http://localhost:8080/v1 @@ -453,29 +455,28 @@ served by local usage_on_final_chunk pass final chunk carries usage {'completion_tokens': 2, 'prompt_tokens': 20, 'total_tokens': 22, 'reasoning_tokens': 0} include_reasoning pass the model did not reason here (5 completion tokens for '9.11'); no reasoning to return tool_choice_required recorded supported: forced get_weather({"city": "Paris"}) - long_tool_call pass 65.8 KiB of arguments in 1 call (1320 lines), streamed over 244 s, longest gap between chunks 243.4 s + long_tool_call pass 65.8 KiB of arguments in 1 call (1320 lines), streamed over 235 s, longest gap between chunks 235.0 s 11 pass, 0 fail, 1 recorded ``` - The long call came back with all 1,320 lines as asked: 19,840 completion - tokens in 244 s, 81 tok/s. The engine's `qwen3_coder` parser streams a call's + tokens in 235 s, 84 tok/s. The engine's `qwen3_coder` parser streams a call's name and short arguments as they are generated but holds a long argument - until it closes, so this stream sent nothing for 243 s. At this commit - `sie-server` ended a stream that sent nothing for 300 s, so an argument that - took longer than that to generate was cut off: about 24,000 tokens for a - session alone on the GPU, fewer when other sessions share it. From commit - `00d7d732` it writes an SSE comment every 15 s while the engine holds a - tool call, and allows the call the profile's overall budget of 1,800 s. The - case passed there too, with the same 239 s between chunks, since a comment - is not a chunk. + until it closes, so this stream sent no chunk for 235 s. `sie-server` writes + an SSE comment every 15 s while the engine holds a tool call, which keeps a + client's idle timer running without adding a chunk, and allows the call the + profile's overall budget of 1,800 s. Before commit `00d7d732` it ended a + stream that sent nothing for 300 s instead: at commit `461c5a58` the case + passed with 243 s of silence, but an argument that took longer than 300 s to + generate was cut off. - This profile does not think, so `include_reasoning` returns nothing. Its answer, 9.11, is wrong; the case does not judge answers. - The usage on this route carries no `cached_tokens`: the engine's cache report is off for this profile. The session load below shows the cache at work through the time to the first token. -`./run_gate.sh gpu` on the same server, with OpenCode 1.19.0 and no global +`./run_gate.sh gpu` at commit `461c5a58`, with OpenCode 1.19.0 and no global OpenCode config, before the recipe set `presence_penalty`, OpenCode's JSON event lines left out: @@ -496,34 +497,44 @@ Five runs in a row passed at the profile's `presence_penalty` of 1.5. Each took 5 steps and 4 tool calls (`bash`, `read`, `edit`, `bash`), 8 to 9 seconds from OpenCode's first event to its last, with at most 8,957 input tokens on a step. Five more runs with the recipe's `presence_penalty: 0` also passed, with -5 or 6 steps, 4 to 8 tool calls and at most 9,723 input tokens. The seeded -gate does not tell the two settings apart; BFCL below does. +5 or 6 steps, 4 to 8 tool calls and at most 9,723 input tokens, and so did +three more at commit `00d7d732`. The seeded gate does not tell the two +settings apart; BFCL below does. -`session_load.py` with its defaults, sessions growing to about 61,000 tokens, -and with `--turn-tokens 12000 --turns 9`, growing to about 128,000: +`session_load.py` at commit `ac9902d3`, with its defaults, sessions growing to +about 61,000 tokens, and with `--turn-tokens 12000 --turns 9`, growing to about +128,000: | Sessions | Largest prompt | Decode p50 / p90, tok/s | Returning-turn first token p50 / p90 | Bar | |---|---|---|---|---| -| 1 | 61,274 | 76.4 / 74.2 | 0.66 / 0.76 s | meets | -| 2 | 61,252 | 69.7 / 63.5 | 0.85 / 1.07 s | meets | -| 4 | 61,263 | 62.9 / 54.1 | 1.30 / 1.89 s | meets | -| 6 | 61,278 | 53.6 / 41.8 | 0.98 / 2.82 s | meets | -| 8 | 61,254 | 49.2 / 31.8 | 2.39 / 3.65 s | misses: first token | -| 1 | 128,203 | 71.7 / 67.5 | 1.53 / 2.00 s | meets | -| 2 | 128,288 | 64.2 / 50.5 | 2.06 / 2.60 s | meets | -| 3 | 128,214 | 47.5 / 40.4 | 2.65 / 4.21 s | misses: first token | +| 1 | 61,002 | 79.1 / 77.1 | 0.73 / 1.43 s | meets | +| 2 | 60,879 | 72.0 / 60.8 | 0.97 / 1.12 s | meets | +| 4 | 61,262 | 56.9 / 51.5 | 1.18 / 1.91 s | meets | +| 6 | 61,200 | 47.5 / 40.7 | 0.88 / 2.81 s | meets | +| 8 | 61,293 | 42.4 / 33.0 | 1.10 / 3.63 s | misses: first token | +| 1 | 127,910 | 74.6 / 70.6 | 1.61 / 2.10 s | meets | +| 2 | 128,251 | 65.4 / 47.0 | 2.08 / 2.39 s | meets | +| 3 | 128,183 | 57.9 / 36.4 | 2.03 / 2.87 s | meets | - One H100 holds **6 sessions at about 61,000 tokens each, or 2 at about - 128,000**, within the bar. No turn failed at any count. + 128,000**, within the bar, on this route and through the gateway below. No + turn failed at any count. +- Three sessions at about 128,000 are at the edge. They met the bar here + with 2.87 s at p90, missed it through the gateway below with 5.14 s, and + missed it on this route in an earlier sweep at commit `461c5a58` with + 4.21 s. That sweep gave the same verdict as this one at every other count. - The prefix cache carries a session from turn to turn. A returning turn with - a 61,000-token prompt reached its first token in 0.66 s with one session, + a 61,000-token prompt reached its first token in 0.73 s with one session, which is the time to prefill only the new input. +- The model often stopped before 500 tokens on later turns: answers averaged + 410 to 455 tokens a session count at 61,000 and 350 to 440 at 128,000. The + recipes' `presence_penalty: 0` was not applied here. - Six 61,000-token sessions fit a 325,056-token cache because the shared 16,000-token system prompt is stored once. - The profile's 32 requests decoding at once is a scheduler limit, not a session count. -BFCL v3 with the harness and settings of +BFCL v3 at commit `461c5a58`, with the harness and settings of [the SIE Relay record](#bfcl-through-sie-relay): the full `simple` split and the same 200-row `multiple` subsample, one request at a time, greedy decoding, an output cap of 8,192 tokens, `tool_choice: "auto"`, and dotted function @@ -547,7 +558,8 @@ names sent with `_`. Both routes ran on the same GPU, the cluster route as closer to the model's training format was not established. One real issue on this repository, run twice through the single-server -recipe: [issue #717](https://github.com/superlinked/sie/issues/717), a +recipe at commit `461c5a58`: +[issue #717](https://github.com/superlinked/sie/issues/717), a metering bug in the GLiNER adapters, on a checkout of commit `03dd3fd3`. OpenCode was asked to fix it, add a regression test and run it, with a 100-step budget and no other help: @@ -568,11 +580,10 @@ OpenCode was asked to fix it, add a regression test and run it, with a The same model, profile and H100, served the way a cluster serves it: this repository's `sie-gateway`, NATS, `sie-server-sidecar` and one `sie-server` worker, on one machine, with the gateway on `127.0.0.1:31000`. The worker was -ready 100 s after the stack started when the kernels were already compiled, -and 270 s after in a new container. +ready 100 s and 161 s after the stack started when the kernels were already +compiled, and 270 s after in a new container. -The conformance check at commit `00d7d732`, which keeps a chat stream alive -while a long tool call is generated: +The conformance check at commit `ac9902d3`: ``` model Qwen/Qwen3.8-27B-FP8:h100-256k-batch at http://127.0.0.1:31000/v1 @@ -589,20 +600,21 @@ served by local usage_on_final_chunk pass final chunk carries usage {'completion_tokens': 2, 'prompt_tokens': 20, 'total_tokens': 22, 'prompt_tokens_details': {'cached_tokens': 0}} include_reasoning pass the model did not reason here (5 completion tokens for '9.11'); no reasoning to return tool_choice_required recorded supported: forced get_weather({"city":"Paris"}) - long_tool_call pass 65.8 KiB of arguments in 1 call (1320 lines), streamed over 240 s, longest gap between chunks 0.1 s + long_tool_call pass 65.8 KiB of arguments in 1 call (1320 lines), streamed over 235 s, longest gap between chunks 0.1 s 11 pass, 0 fail, 1 recorded ``` - The long call came back whole: 19,840 completion tokens, all 1,320 lines as asked. The gateway passes on an empty chunk while the worker holds a call, - so the stream never went quiet for more than 0.1 s. At commit `461c5a58`, - before that change, the same case failed after 30 s with - `inter_chunk_timeout`, with no argument bytes delivered. + so the stream never went quiet for more than 0.1 s. Before commit + `00d7d732` the gateway ended a stream after 30 s without a chunk: at commit + `461c5a58` the case failed after 30 s with `inter_chunk_timeout`, with no + argument bytes delivered. - This route reports `cached_tokens` in the usage, which the session load below reads. -- One request with a 1.1 MB body, a 196,449-token prompt, was answered in - 23 s. +- At commit `461c5a58`, one request with a 1.1 MB body, a 196,449-token + prompt, was answered in 23 s. The seeded gate, `./run_gate.sh cluster` with `SIE_BASE_URL` set to the gateway, passed 5 of 5 runs at the profile's `presence_penalty` of 1.5, at @@ -610,26 +622,30 @@ commit `461c5a58`, and 3 of 3 with the recipe's `presence_penalty: 0`, at commit `00d7d732`. Each run took 5 steps and 4 or 5 tool calls, 7 to 9 seconds, with at most 9,240 input tokens on a step. -BFCL is in [the table above](#one-gpu-qwen38-27b-fp8-recorded). The session -load below was recorded at commit `461c5a58`. +BFCL is in [the table above](#one-gpu-qwen38-27b-fp8-recorded). -`session_load.py` through the gateway: +`session_load.py` through the gateway at commit `ac9902d3`: | Sessions | Largest prompt | Decode p50 / p90, tok/s | Returning-turn first token p50 / p90 | Returning-turn prompt cached | Bar | |---|---|---|---|---|---| -| 1 | 60,992 | 76.9 / 74.2 | 0.66 / 0.75 s | 86.3% | meets | -| 2 | 61,105 | 71.9 / 63.6 | 0.80 / 1.08 s | 86.2% | meets | -| 4 | 61,265 | 62.8 / 54.7 | 1.67 / 2.03 s | 86.2% | meets | -| 6 | 61,274 | 53.4 / 38.7 | 1.23 / 2.51 s | 86.2% | meets | -| 8 | 61,266 | 46.8 / 34.7 | 2.10 / 3.59 s | 86.3% | misses: first token | -| 1 | 127,973 | 72.3 / 67.9 | 1.57 / 1.96 s | 85.6% | meets | -| 2 | 127,742 | 61.2 / 43.6 | 1.84 / 2.15 s | 85.5% | meets | - -- The gateway adds nothing measurable to the single server's capacity: 6 - sessions at about 61,000 tokens and 2 at about 128,000 meet the bar on both. -- About 86% of each returning turn's prompt came from the cache. The rest, - about 6,200 tokens a turn on average, is the size of the turn's new input, - so what the session sends again is reused. +| 1 | 61,106 | 79.2 / 77.0 | 0.67 / 0.77 s | 86.3% | meets | +| 2 | 61,207 | 72.9 / 65.9 | 0.78 / 1.07 s | 86.3% | meets | +| 4 | 61,263 | 66.0 / 51.7 | 1.60 / 2.06 s | 86.3% | meets | +| 6 | 61,273 | 54.3 / 37.1 | 1.30 / 2.46 s | 86.2% | meets | +| 8 | 61,294 | 48.8 / 32.9 | 2.44 / 3.85 s | 86.3% | misses: first token | +| 1 | 127,560 | 74.5 / 70.4 | 1.58 / 2.00 s | 85.5% | meets | +| 2 | 127,991 | 67.7 / 50.7 | 1.82 / 2.26 s | 85.5% | meets | +| 3 | 128,187 | 59.0 / 36.8 | 3.18 / 5.14 s | 83.7% | misses: first token | + +- The gateway gives the same capacity as the single server: 6 sessions at + about 61,000 tokens and 2 at about 128,000 meet the bar on both. A sweep at + commit `461c5a58` gave the same verdicts at the counts it ran, 1 to 8 at + 61,000 and 1 and 2 at 128,000. +- What a session sends again is served from the cache. On a returning turn the + uncached part averaged 6,150 tokens at 61,000 and 12,150 at 128,000, the + size of the turn's new input. With 3 sessions at 128,000 the three + transcripts no longer fit the cache together, some turns had up to 37,307 + tokens to prefill again, and the first token came later. The same real issue as above, once through the cluster recipe with `presence_penalty: 0` at commit `00d7d732`: 100 steps and 104 tool calls in From a1cd38aba57d2c8346b085e6d05706445982fe0c Mon Sep 17 00:00:00 2001 From: Krisztian Gajdar Date: Thu, 15 Oct 2026 21:20:09 +0200 Subject: [PATCH 8/8] docs(examples): cite the server commits on main --- examples/coding-agent-opencode/README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/examples/coding-agent-opencode/README.md b/examples/coding-agent-opencode/README.md index f3478030a..6fea6e104 100644 --- a/examples/coding-agent-opencode/README.md +++ b/examples/coding-agent-opencode/README.md @@ -423,9 +423,10 @@ took 7 steps and 6 tool calls. cores and 64 GiB of memory, 2026-10-15. `sie-server` ran from this repository with the `sglang-cu130` bundle (SGLang 0.5.20), started as `sie-server serve -b sglang-cu130 --preload Qwen/Qwen3.8-27B-FP8:h100-256k-batch`, in three -sessions on machines of that kind. Each result below names the commit it was -recorded at. The `docker run` line above starts the same server, profile and -bundle from the published image; the runs below did not use the image. +sessions on machines of that kind. Each result below names the commit of the +server it was recorded at; the checks are the ones in this directory. The +`docker run` line above starts the same server, profile and bundle from the +published image; the runs below did not use the image. What the engine reported once loaded: @@ -438,7 +439,7 @@ The server answered its first request 197 s and 229 s after it started in two new containers, with the weights already downloaded, and 76 s after a restart in the same container, with the kernels already compiled. -The conformance check at commit `ac9902d3`: +The conformance check at commit `09b62a32`: ``` model Qwen/Qwen3.8-27B-FP8:h100-256k-batch at http://localhost:8080/v1 @@ -501,7 +502,7 @@ step. Five more runs with the recipe's `presence_penalty: 0` also passed, with three more at commit `00d7d732`. The seeded gate does not tell the two settings apart; BFCL below does. -`session_load.py` at commit `ac9902d3`, with its defaults, sessions growing to +`session_load.py` at commit `09b62a32`, with its defaults, sessions growing to about 61,000 tokens, and with `--turn-tokens 12000 --turns 9`, growing to about 128,000: @@ -583,7 +584,7 @@ worker, on one machine, with the gateway on `127.0.0.1:31000`. The worker was ready 100 s and 161 s after the stack started when the kernels were already compiled, and 270 s after in a new container. -The conformance check at commit `ac9902d3`: +The conformance check at commit `09b62a32`: ``` model Qwen/Qwen3.8-27B-FP8:h100-256k-batch at http://127.0.0.1:31000/v1 @@ -624,7 +625,7 @@ seconds, with at most 9,240 input tokens on a step. BFCL is in [the table above](#one-gpu-qwen38-27b-fp8-recorded). -`session_load.py` through the gateway at commit `ac9902d3`: +`session_load.py` through the gateway at commit `09b62a32`: | Sessions | Largest prompt | Decode p50 / p90, tok/s | Returning-turn first token p50 / p90 | Returning-turn prompt cached | Bar | |---|---|---|---|---|---|