From 75c83cf9b15819ac84661db8b67890cce01c5570 Mon Sep 17 00:00:00 2001 From: Krisztian Gajdar Date: Wed, 14 Oct 2026 18:57:19 +0200 Subject: [PATCH 1/2] fix(server): keep the visible answer of a thinking model on the MLX chat path mlx_lm returns a thinking model's reasoning in its own reasoning field and the answer in content, without the closing think tag. The chat sanitizer started that content inside the prompt-seeded thinking block, never saw the close, and returned empty content; with include_reasoning the answer was appended to reasoning_content instead. A choice now starts outside the thinking block when the child returned reasoning in its own field and content does not carry it: a stream delta whose reasoning arrives before any content, or a whole message whose content never closes the block. Content that still carries a raw block is stripped as before. --- .../src/sie_server/api/openai_local.py | 24 ++-- .../sie_server/tests/api/test_openai_local.py | 104 ++++++++++++++++++ 2 files changed, 120 insertions(+), 8 deletions(-) diff --git a/packages/sie_server/src/sie_server/api/openai_local.py b/packages/sie_server/src/sie_server/api/openai_local.py index f708fdf1f..7f516876d 100644 --- a/packages/sie_server/src/sie_server/api/openai_local.py +++ b/packages/sie_server/src/sie_server/api/openai_local.py @@ -43,6 +43,7 @@ from sie_sdk.queue_types import denormalize_model_id from sie_server.adapters._generation_base import ( + REASONING_CONTROL_TOKENS, GenerationAdapter, GenerationError, GenerationUnsupportedFieldError, @@ -1037,16 +1038,23 @@ def _sanitize_chat_payload( if not hide_thinking_blocks: continue - stripper = state_map.setdefault( - index, - ThinkingBlockStripper( - inside=initial_inside_thinking, - reasoning_format=reasoning_format, - keep_reasoning=public_reasoning, - ), - ) raw_content = container.get("content") content = raw_content if isinstance(raw_content, str) else "" + stripper = state_map.get(index) + if stripper is None: + if not content and not reasoning_was_present: + continue + # A child that returns reasoning in its own field has consumed the + # prompt-seeded thinking block, unless content still carries it: a + # stream delta can close it later, a whole message has to close it. + content_holds_no_reasoning = ( + REASONING_CONTROL_TOKENS[reasoning_format][1] not in content if container is message else not content + ) + stripper = state_map[index] = ThinkingBlockStripper( + inside=initial_inside_thinking and not (reasoning_was_present and content_holds_no_reasoning), + reasoning_format=reasoning_format, + keep_reasoning=public_reasoning, + ) visible = stripper.feed(content) if content else "" is_terminal_choice = terminal or choice_obj.get("finish_reason") is not None if is_terminal_choice: diff --git a/packages/sie_server/tests/api/test_openai_local.py b/packages/sie_server/tests/api/test_openai_local.py index 0b25ccd3c..b3b1c24ff 100644 --- a/packages/sie_server/tests/api/test_openai_local.py +++ b/packages/sie_server/tests/api/test_openai_local.py @@ -2418,3 +2418,107 @@ def _handler(request: httpx.Request) -> httpx.Response: reasoning = "".join(delta.get("reasoning_content", "") for delta in deltas) assert reasoning == ("private default reasoning" if include_reasoning else "") assert "" not in response.text + + +_MLX_REASONING = "The user wants one word.\n" +_MLX_ANSWER = "\n\nready" + + +@pytest.mark.parametrize("include_reasoning", [False, True]) +@pytest.mark.parametrize( + "message", + [ + {"role": "assistant", "content": _MLX_ANSWER, "reasoning": _MLX_REASONING}, + {"role": "assistant", "content": _MLX_ANSWER, "reasoning_content": _MLX_REASONING}, + {"role": "assistant", "content": _MLX_REASONING + "" + _MLX_ANSWER}, + ], + ids=["reasoning", "reasoning_content", "raw_tags"], +) +def test_mlx_thinking_chat_keeps_the_answer_beside_split_reasoning( + monkeypatch: pytest.MonkeyPatch, message: dict[str, Any], include_reasoning: bool +) -> None: + payload = { + "id": "chatcmpl-mlx", + "system_fingerprint": "0.32.0", + "object": "chat.completion", + "model": "mlx-community/Qwen3.5-4B-4bit", + "created": 1, + "choices": [{"index": 0, "finish_reason": "stop", "message": message}], + "usage": {"prompt_tokens": 14, "completion_tokens": 12, "total_tokens": 26}, + } + client, _ = _mlx_chat_client( + monkeypatch, lambda _request: httpx.Response(200, json=payload), chat_template_kwargs={"enable_thinking": True} + ) + + response = client.post( + "/v1/chat/completions", + json={ + "model": "Qwen/Qwen3.5-4B:thinking", + "messages": [{"role": "user", "content": "Reply with one word: ready"}], + "include_reasoning": include_reasoning, + }, + ) + + assert response.status_code == 200 + returned = response.json()["choices"][0]["message"] + assert returned["content"] == _MLX_ANSWER + assert "reasoning" not in returned + assert returned.get("reasoning_content") == (_MLX_REASONING if include_reasoning else None) + assert include_reasoning or "one word" not in response.text + + +@pytest.mark.parametrize("include_reasoning", [False, True]) +@pytest.mark.parametrize( + "deltas", + [ + [ + {"role": "assistant", "reasoning": "The user wants "}, + {"role": "assistant", "reasoning": "one word.\n"}, + {"role": "assistant", "content": "\n\n"}, + {"role": "assistant", "content": "ready"}, + ], + [ + {"role": "assistant", "content": ""}, + {"reasoning_content": "The user wants "}, + {"reasoning_content": "one word.\n"}, + {"content": "\n\nready"}, + ], + [ + {"role": "assistant", "content": "The user wants "}, + {"role": "assistant", "content": "one word.\n\n\nready"}, + ], + ], + ids=["reasoning", "reasoning_content_after_role", "raw_tags"], +) +def test_mlx_thinking_chat_stream_keeps_the_answer_beside_split_reasoning( + monkeypatch: pytest.MonkeyPatch, deltas: list[dict[str, Any]], include_reasoning: bool +) -> None: + wire = _sse_wire( + [*(_stream_event(delta) for delta in deltas), _stream_event({"role": "assistant"}, finish_reason="stop")] + ) + client, _ = _mlx_chat_client( + monkeypatch, + lambda _request: httpx.Response( + 200, stream=_ChunkedAsyncStream([wire[:83], wire[83:]]), headers={"content-type": "text/event-stream"} + ), + chat_template_kwargs={"enable_thinking": True}, + ) + + response = client.post( + "/v1/chat/completions", + json={ + "model": "Qwen/Qwen3.5-4B:thinking", + "messages": [{"role": "user", "content": "Reply with one word: ready"}], + "stream": True, + "include_reasoning": include_reasoning, + }, + ) + + assert response.status_code == 200 + returned = _sse_deltas(response.text) + assert "".join(delta.get("content", "") for delta in returned) == _MLX_ANSWER + assert not any("reasoning" in delta for delta in returned) + reasoning = "".join(delta.get("reasoning_content", "") for delta in returned) + assert reasoning == (_MLX_REASONING if include_reasoning else "") + assert include_reasoning or "one word" not in response.text From f34ff02d970e918363c95382670f36e2e1b8b233 Mon Sep 17 00:00:00 2001 From: Krisztian Gajdar Date: Wed, 14 Oct 2026 19:51:42 +0200 Subject: [PATCH 2/2] fix(server): hold content beside split reasoning until the block closes or the choice stops When the prompt seeds an open thinking block and the child returns reasoning in its own field, the content of that choice is held instead of being sent through a stripper that starts outside the block. A close marker in the held content means the text before it is reasoning, which stays hidden as before. A choice that finishes with stop or tool_calls without one releases the held content as the answer. Any other finish, such as length, keeps it hidden. This makes a buffered message and the same content streamed agree, and keeps reasoning private when a child leaves the end of the block in content after a reasoning-only delta, or is cut off before it. --- .../src/sie_server/api/openai_local.py | 33 ++- .../sie_server/tests/api/test_openai_local.py | 206 +++++++++++++++--- 2 files changed, 196 insertions(+), 43 deletions(-) diff --git a/packages/sie_server/src/sie_server/api/openai_local.py b/packages/sie_server/src/sie_server/api/openai_local.py index 7f516876d..f1e135ca4 100644 --- a/packages/sie_server/src/sie_server/api/openai_local.py +++ b/packages/sie_server/src/sie_server/api/openai_local.py @@ -132,6 +132,7 @@ _MAX_CHAT_VIDEOS = 1 _MAX_U32 = (1 << 32) - 1 _ALLOWED_CHAT_ROLES = frozenset({"system", "user", "assistant", "tool", "developer"}) +_ANSWER_FINISH_REASONS = frozenset({"stop", "tool_calls"}) # Compatibility requests project onto the same native score bound. _MAX_RERANK_DOCS = MAX_SCORE_ITEMS @@ -1005,6 +1006,7 @@ def _sanitize_chat_payload( initial_inside_thinking: bool, reasoning_format: ReasoningFormat, states: dict[int, ThinkingBlockStripper] | None = None, + held_contents: dict[int, list[str]] | None = None, terminal: bool, include_reasoning: bool = False, ) -> dict[str, Any]: @@ -1017,6 +1019,7 @@ def _sanitize_chat_payload( public_reasoning = include_reasoning and hide_thinking_blocks state_map = states if states is not None else {} + held_map = held_contents if held_contents is not None else {} for position, choice in enumerate(choices): if not isinstance(choice, dict): continue @@ -1044,19 +1047,31 @@ def _sanitize_chat_payload( if stripper is None: if not content and not reasoning_was_present: continue - # A child that returns reasoning in its own field has consumed the - # prompt-seeded thinking block, unless content still carries it: a - # stream delta can close it later, a whole message has to close it. - content_holds_no_reasoning = ( - REASONING_CONTROL_TOKENS[reasoning_format][1] not in content if container is message else not content - ) stripper = state_map[index] = ThinkingBlockStripper( - inside=initial_inside_thinking and not (reasoning_was_present and content_holds_no_reasoning), + inside=initial_inside_thinking, reasoning_format=reasoning_format, keep_reasoning=public_reasoning, ) - visible = stripper.feed(content) if content else "" + if initial_inside_thinking and reasoning_was_present: + held_map[index] = [] is_terminal_choice = terminal or choice_obj.get("finish_reason") is not None + text = content + held = held_map.get(index) + if held is not None: + close = REASONING_CONTROL_TOKENS[reasoning_format][1] + closes = close in "".join(held[1 - len(close) :]) + content + if content: + held.append(content) + if closes or is_terminal_choice: + del held_map[index] + text = "".join(held) + if not closes and choice_obj.get("finish_reason") in _ANSWER_FINISH_REASONS: + stripper = state_map[index] = ThinkingBlockStripper( + reasoning_format=reasoning_format, keep_reasoning=public_reasoning + ) + else: + text = "" + visible = stripper.feed(text) if text else "" if is_terminal_choice: visible += stripper.finish() if isinstance(raw_content, str) or visible: @@ -1101,6 +1116,7 @@ async def _sanitize_sse_stream( include_reasoning: bool = False, ) -> AsyncIterator[bytes]: states: dict[int, ThinkingBlockStripper] = {} + held_contents: dict[int, list[str]] = {} strict_contents: dict[int, list[str]] = {} strict_tool_choices: set[int] = set() tool_calls: dict[int, set[object]] = {} @@ -1143,6 +1159,7 @@ async def _sanitize_sse_stream( initial_inside_thinking=initial_inside_thinking, reasoning_format=reasoning_format, states=states, + held_contents=held_contents, terminal=False, include_reasoning=include_reasoning, ) diff --git a/packages/sie_server/tests/api/test_openai_local.py b/packages/sie_server/tests/api/test_openai_local.py index b3b1c24ff..035d756a9 100644 --- a/packages/sie_server/tests/api/test_openai_local.py +++ b/packages/sie_server/tests/api/test_openai_local.py @@ -2426,16 +2426,70 @@ def _handler(request: httpx.Request) -> httpx.Response: @pytest.mark.parametrize("include_reasoning", [False, True]) @pytest.mark.parametrize( - "message", + ("message", "finish_reason", "content", "reasoning", "hidden"), [ - {"role": "assistant", "content": _MLX_ANSWER, "reasoning": _MLX_REASONING}, - {"role": "assistant", "content": _MLX_ANSWER, "reasoning_content": _MLX_REASONING}, - {"role": "assistant", "content": _MLX_REASONING + "" + _MLX_ANSWER}, + ( + {"role": "assistant", "content": _MLX_ANSWER, "reasoning": _MLX_REASONING}, + "stop", + _MLX_ANSWER, + _MLX_REASONING, + "one word", + ), + ( + {"role": "assistant", "content": _MLX_ANSWER, "reasoning_content": _MLX_REASONING}, + "stop", + _MLX_ANSWER, + _MLX_REASONING, + "one word", + ), + ( + {"role": "assistant", "content": _MLX_REASONING + "" + _MLX_ANSWER}, + "stop", + _MLX_ANSWER, + _MLX_REASONING, + "one word", + ), + ( + {"role": "assistant", "content": "one word.\n" + _MLX_ANSWER, "reasoning": "The user wants "}, + "stop", + _MLX_ANSWER, + _MLX_REASONING, + "one word", + ), + ( + {"role": "assistant", "content": "readyone word", "reasoning": _MLX_REASONING}, + "stop", + "ready", + _MLX_REASONING + "one word", + "one word", + ), + ( + {"role": "assistant", "content": "still thinking", "reasoning": _MLX_REASONING}, + "length", + "", + _MLX_REASONING + "still thinking", + "still thinking", + ), + ({"role": "assistant", "content": _MLX_ANSWER}, "stop", "", _MLX_ANSWER, "ready"), + ], + ids=[ + "reasoning", + "reasoning_content", + "raw_tags", + "split_and_raw_close", + "split_then_raw_open", + "split_cut_off", + "no_reasoning_field", ], - ids=["reasoning", "reasoning_content", "raw_tags"], ) def test_mlx_thinking_chat_keeps_the_answer_beside_split_reasoning( - monkeypatch: pytest.MonkeyPatch, message: dict[str, Any], include_reasoning: bool + monkeypatch: pytest.MonkeyPatch, + message: dict[str, Any], + finish_reason: str, + content: str, + reasoning: str, + hidden: str, + include_reasoning: bool, ) -> None: payload = { "id": "chatcmpl-mlx", @@ -2443,7 +2497,7 @@ def test_mlx_thinking_chat_keeps_the_answer_beside_split_reasoning( "object": "chat.completion", "model": "mlx-community/Qwen3.5-4B-4bit", "created": 1, - "choices": [{"index": 0, "finish_reason": "stop", "message": message}], + "choices": [{"index": 0, "finish_reason": finish_reason, "message": message}], "usage": {"prompt_tokens": 14, "completion_tokens": 12, "total_tokens": 26}, } client, _ = _mlx_chat_client( @@ -2461,41 +2515,123 @@ def test_mlx_thinking_chat_keeps_the_answer_beside_split_reasoning( assert response.status_code == 200 returned = response.json()["choices"][0]["message"] - assert returned["content"] == _MLX_ANSWER + assert returned["content"] == content assert "reasoning" not in returned - assert returned.get("reasoning_content") == (_MLX_REASONING if include_reasoning else None) - assert include_reasoning or "one word" not in response.text + assert returned.get("reasoning_content") == (reasoning if include_reasoning else None) + assert include_reasoning or hidden not in response.text @pytest.mark.parametrize("include_reasoning", [False, True]) @pytest.mark.parametrize( - "deltas", + ("deltas", "finish_reason", "content", "reasoning", "hidden"), [ - [ - {"role": "assistant", "reasoning": "The user wants "}, - {"role": "assistant", "reasoning": "one word.\n"}, - {"role": "assistant", "content": "\n\n"}, - {"role": "assistant", "content": "ready"}, - ], - [ - {"role": "assistant", "content": ""}, - {"reasoning_content": "The user wants "}, - {"reasoning_content": "one word.\n"}, - {"content": "\n\nready"}, - ], - [ - {"role": "assistant", "content": "The user wants "}, - {"role": "assistant", "content": "one word.\n\n\nready"}, - ], + ( + [ + {"role": "assistant", "reasoning": "The user wants "}, + {"role": "assistant", "reasoning": "one word.\n"}, + {"role": "assistant", "content": "\n\n"}, + {"role": "assistant", "content": "ready"}, + ], + "stop", + _MLX_ANSWER, + _MLX_REASONING, + "one word", + ), + ( + [ + {"role": "assistant", "content": ""}, + {"reasoning_content": "The user wants "}, + {"reasoning_content": "one word.\n"}, + {"content": "\n\nready"}, + ], + "stop", + _MLX_ANSWER, + _MLX_REASONING, + "one word", + ), + ( + [ + {"role": "assistant", "content": "The user wants "}, + {"role": "assistant", "content": "one word.\n\n\nready"}, + ], + "stop", + _MLX_ANSWER, + _MLX_REASONING, + "one word", + ), + ( + [ + {"role": "assistant", "content": ""}, + {"reasoning_content": "The user wants "}, + {"content": "one word.\n\n\nready"}, + ], + "stop", + _MLX_ANSWER, + _MLX_REASONING, + "one word", + ), + ( + [ + {"role": "assistant", "reasoning": "The user wants "}, + {"role": "assistant", "content": "one word.\n\n\nready"}, + ], + "stop", + _MLX_ANSWER, + _MLX_REASONING, + "one word", + ), + ( + [{"role": "assistant", "reasoning": _MLX_REASONING, "content": _MLX_ANSWER}], + "stop", + _MLX_ANSWER, + _MLX_REASONING, + "one word", + ), + ( + [ + {"role": "assistant", "reasoning": _MLX_REASONING}, + {"role": "assistant", "content": "Checking."}, + {"role": "assistant", "tool_calls": [{"index": 0, **_PARIS_CALL}]}, + ], + "tool_calls", + "Checking.", + _MLX_REASONING, + "one word", + ), + ( + [{"role": "assistant", "reasoning": _MLX_REASONING}, {"role": "assistant", "content": "still thinking"}], + "length", + "", + _MLX_REASONING + "still thinking", + "still thinking", + ), + ([{"role": "assistant", "content": _MLX_ANSWER}], "stop", "", _MLX_ANSWER, "ready"), + ], + ids=[ + "reasoning", + "reasoning_content_after_role", + "raw_tags", + "split_then_raw_close", + "split_then_raw_close_across_deltas", + "split_and_answer_in_one_delta", + "answer_before_tool_call", + "split_cut_off", + "no_reasoning_field", ], - ids=["reasoning", "reasoning_content_after_role", "raw_tags"], ) def test_mlx_thinking_chat_stream_keeps_the_answer_beside_split_reasoning( - monkeypatch: pytest.MonkeyPatch, deltas: list[dict[str, Any]], include_reasoning: bool + monkeypatch: pytest.MonkeyPatch, + deltas: list[dict[str, Any]], + finish_reason: str, + content: str, + reasoning: str, + hidden: str, + include_reasoning: bool, ) -> None: wire = _sse_wire( - [*(_stream_event(delta) for delta in deltas), _stream_event({"role": "assistant"}, finish_reason="stop")] + [*(_stream_event(delta) for delta in deltas), _stream_event({"role": "assistant"}, finish_reason=finish_reason)] ) client, _ = _mlx_chat_client( monkeypatch, @@ -2510,6 +2646,7 @@ def test_mlx_thinking_chat_stream_keeps_the_answer_beside_split_reasoning( json={ "model": "Qwen/Qwen3.5-4B:thinking", "messages": [{"role": "user", "content": "Reply with one word: ready"}], + "tools": [_WEATHER_TOOL], "stream": True, "include_reasoning": include_reasoning, }, @@ -2517,8 +2654,7 @@ def test_mlx_thinking_chat_stream_keeps_the_answer_beside_split_reasoning( assert response.status_code == 200 returned = _sse_deltas(response.text) - assert "".join(delta.get("content", "") for delta in returned) == _MLX_ANSWER + assert "".join(delta.get("content") or "" for delta in returned) == content assert not any("reasoning" in delta for delta in returned) - reasoning = "".join(delta.get("reasoning_content", "") for delta in returned) - assert reasoning == (_MLX_REASONING if include_reasoning else "") - assert include_reasoning or "one word" not in response.text + assert "".join(delta.get("reasoning_content", "") for delta in returned) == (reasoning if include_reasoning else "") + assert include_reasoning or hidden not in response.text