From dd8c819a331f0f459a6dfde8000dd0a7e225a5ef Mon Sep 17 00:00:00 2001 From: Alex Castillo Date: Sat, 17 Oct 2026 18:50:17 -0400 Subject: [PATCH 1/2] fix(bedrock): parse streamed Converse tool input like the non-streamed output converse_stream spans logged a tool_use block's input as the raw JSON string joined from the toolUse deltas, while converse spans log the dict Bedrock returns. Parse the joined string once the stream ends ("" becomes {}, input that doesn't parse, e.g. cut off by maxTokens, stays a string). Co-Authored-By: Claude Opus 5.5 --- ...ock_converse_stream_parses_tool_input.yaml | 61 +++++++++++++++++++ ...ock_converse_stream_parses_tool_input.yaml | 61 +++++++++++++++++++ .../bedrock_runtime/test_bedrock_runtime.py | 22 +++++++ .../integrations/bedrock_runtime/tracing.py | 13 +++- 4 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 py/src/braintrust/integrations/bedrock_runtime/cassettes/1.34.116/test_wrap_bedrock_converse_stream_parses_tool_input.yaml create mode 100644 py/src/braintrust/integrations/bedrock_runtime/cassettes/latest/test_wrap_bedrock_converse_stream_parses_tool_input.yaml diff --git a/py/src/braintrust/integrations/bedrock_runtime/cassettes/1.34.116/test_wrap_bedrock_converse_stream_parses_tool_input.yaml b/py/src/braintrust/integrations/bedrock_runtime/cassettes/1.34.116/test_wrap_bedrock_converse_stream_parses_tool_input.yaml new file mode 100644 index 000000000..ee6bbd481 --- /dev/null +++ b/py/src/braintrust/integrations/bedrock_runtime/cassettes/1.34.116/test_wrap_bedrock_converse_stream_parses_tool_input.yaml @@ -0,0 +1,61 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": [{"text": "What''s the weather + in Paris? Use the tool."}]}], "inferenceConfig": {"maxTokens": 200, "temperature": + 0}, "toolConfig": {"tools": [{"toolSpec": {"name": "get_weather", "description": + "Get the current weather for a city.", "inputSchema": {"json": {"type": "object", + "properties": {"city": {"type": "string", "description": "City name"}}, "required": + ["city"]}}}}], "toolChoice": {"tool": {"name": "get_weather"}}}}' + headers: + Content-Length: + - '466' + Content-Type: + - !!binary | + YXBwbGljYXRpb24vanNvbg== + User-Agent: + - !!binary | + Qm90bzMvMS40My4xMTEgbWQvQm90b2NvcmUjMS40My4xMTEgdWEvMi4xIG9zL2xpbnV4IzYuNi44 + Ny4yLW1pY3Jvc29mdC1zdGFuZGFyZC1XU0wyIG1kL2FyY2gjeDg2XzY0IGxhbmcvcHl0aG9uIzMu + MTQuOCBtZC9weWltcGwjQ1B5dGhvbiBtL0QsWixiLG4sMyBjZmcvcmV0cnktbW9kZSNsZWdhY3kg + Qm90b2NvcmUvMS40My4xMTE= + method: POST + uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/converse-stream + response: + body: + string: !!binary | + AAAAsQAAAFLN4M4VCzpldmVudC10eXBlBwAMbWVzc2FnZVN0YXJ0DTpjb250ZW50LXR5cGUHABBh + cHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsicCI6ImFiY2RlZmdoaWprbG1u + b3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVoiLCJyb2xlIjoiYXNzaXN0YW50 + In3DJvYkAAAA5gAAAFc3uSlBCzpldmVudC10eXBlBwARY29udGVudEJsb2NrU3RhcnQNOmNvbnRl + bnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJjb250ZW50 + QmxvY2tJbmRleCI6MCwicCI6ImFiY2RlZmdoaSIsInN0YXJ0Ijp7InRvb2xVc2UiOnsibmFtZSI6 + ImdldF93ZWF0aGVyIiwidG9vbFVzZUlkIjoidG9vbHVzZV9od0tPMkljSkdLSzRSTWxlSndZblRt + In19fWPJlGIAAADEAAAAV4y4VSULOmV2ZW50LXR5cGUHABFjb250ZW50QmxvY2tEZWx0YQ06Y29u + dGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImNvbnRl + bnRCbG9ja0luZGV4IjowLCJkZWx0YSI6eyJ0b29sVXNlIjp7ImlucHV0Ijoie1wiY2l0eVwiOlwi + UGFyaXNcIn0ifX0sInAiOiJhYmNkZWZnaGlqIn3jnRuKAAAArAAAAFZS/Vk/CzpldmVudC10eXBl + BwAQY29udGVudEJsb2NrU3RvcA06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVz + c2FnZS10eXBlBwAFZXZlbnR7ImNvbnRlbnRCbG9ja0luZGV4IjowLCJwIjoiYWJjZGVmZ2hpamts + bW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTiJ9MZDPXQAAAJMAAABR7+jjyws6ZXZlbnQtdHlw + ZQcAC21lc3NhZ2VTdG9wDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdl + LXR5cGUHAAVldmVudHsicCI6ImFiY2RlZmdoaWprbG1ub3BxciIsInN0b3BSZWFzb24iOiJ0b29s + X3VzZSJ9FcGaWgAAAQkAAABOMgoCKgs6ZXZlbnQtdHlwZQcACG1ldGFkYXRhDTpjb250ZW50LXR5 + cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsibWV0cmljcyI6eyJs + YXRlbmN5TXMiOjY3N30sInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpL + TE1OT1BRUlNUVVZXWCIsInVzYWdlIjp7ImlucHV0VG9rZW5zIjo0MTEsIm91dHB1dFRva2VucyI6 + MTYsInNlcnZlclRvb2xVc2FnZSI6e30sInRvdGFsVG9rZW5zIjo0Mjd9fRkMP4U= + headers: + Connection: + - keep-alive + Content-Type: + - application/vnd.amazon.eventstream + Date: + - Sat, 17 Oct 2026 22:49:01 GMT + Transfer-Encoding: + - chunked + x-amzn-RequestId: + - 5ceccdfb-1053-424c-b3d8-050a251c63bd + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/bedrock_runtime/cassettes/latest/test_wrap_bedrock_converse_stream_parses_tool_input.yaml b/py/src/braintrust/integrations/bedrock_runtime/cassettes/latest/test_wrap_bedrock_converse_stream_parses_tool_input.yaml new file mode 100644 index 000000000..ee6bbd481 --- /dev/null +++ b/py/src/braintrust/integrations/bedrock_runtime/cassettes/latest/test_wrap_bedrock_converse_stream_parses_tool_input.yaml @@ -0,0 +1,61 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": [{"text": "What''s the weather + in Paris? Use the tool."}]}], "inferenceConfig": {"maxTokens": 200, "temperature": + 0}, "toolConfig": {"tools": [{"toolSpec": {"name": "get_weather", "description": + "Get the current weather for a city.", "inputSchema": {"json": {"type": "object", + "properties": {"city": {"type": "string", "description": "City name"}}, "required": + ["city"]}}}}], "toolChoice": {"tool": {"name": "get_weather"}}}}' + headers: + Content-Length: + - '466' + Content-Type: + - !!binary | + YXBwbGljYXRpb24vanNvbg== + User-Agent: + - !!binary | + Qm90bzMvMS40My4xMTEgbWQvQm90b2NvcmUjMS40My4xMTEgdWEvMi4xIG9zL2xpbnV4IzYuNi44 + Ny4yLW1pY3Jvc29mdC1zdGFuZGFyZC1XU0wyIG1kL2FyY2gjeDg2XzY0IGxhbmcvcHl0aG9uIzMu + MTQuOCBtZC9weWltcGwjQ1B5dGhvbiBtL0QsWixiLG4sMyBjZmcvcmV0cnktbW9kZSNsZWdhY3kg + Qm90b2NvcmUvMS40My4xMTE= + method: POST + uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/converse-stream + response: + body: + string: !!binary | + AAAAsQAAAFLN4M4VCzpldmVudC10eXBlBwAMbWVzc2FnZVN0YXJ0DTpjb250ZW50LXR5cGUHABBh + cHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsicCI6ImFiY2RlZmdoaWprbG1u + b3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVoiLCJyb2xlIjoiYXNzaXN0YW50 + In3DJvYkAAAA5gAAAFc3uSlBCzpldmVudC10eXBlBwARY29udGVudEJsb2NrU3RhcnQNOmNvbnRl + bnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJjb250ZW50 + QmxvY2tJbmRleCI6MCwicCI6ImFiY2RlZmdoaSIsInN0YXJ0Ijp7InRvb2xVc2UiOnsibmFtZSI6 + ImdldF93ZWF0aGVyIiwidG9vbFVzZUlkIjoidG9vbHVzZV9od0tPMkljSkdLSzRSTWxlSndZblRt + In19fWPJlGIAAADEAAAAV4y4VSULOmV2ZW50LXR5cGUHABFjb250ZW50QmxvY2tEZWx0YQ06Y29u + dGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImNvbnRl + bnRCbG9ja0luZGV4IjowLCJkZWx0YSI6eyJ0b29sVXNlIjp7ImlucHV0Ijoie1wiY2l0eVwiOlwi + UGFyaXNcIn0ifX0sInAiOiJhYmNkZWZnaGlqIn3jnRuKAAAArAAAAFZS/Vk/CzpldmVudC10eXBl + BwAQY29udGVudEJsb2NrU3RvcA06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVz + c2FnZS10eXBlBwAFZXZlbnR7ImNvbnRlbnRCbG9ja0luZGV4IjowLCJwIjoiYWJjZGVmZ2hpamts + bW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTiJ9MZDPXQAAAJMAAABR7+jjyws6ZXZlbnQtdHlw + ZQcAC21lc3NhZ2VTdG9wDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdl + LXR5cGUHAAVldmVudHsicCI6ImFiY2RlZmdoaWprbG1ub3BxciIsInN0b3BSZWFzb24iOiJ0b29s + X3VzZSJ9FcGaWgAAAQkAAABOMgoCKgs6ZXZlbnQtdHlwZQcACG1ldGFkYXRhDTpjb250ZW50LXR5 + cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsibWV0cmljcyI6eyJs + YXRlbmN5TXMiOjY3N30sInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpL + TE1OT1BRUlNUVVZXWCIsInVzYWdlIjp7ImlucHV0VG9rZW5zIjo0MTEsIm91dHB1dFRva2VucyI6 + MTYsInNlcnZlclRvb2xVc2FnZSI6e30sInRvdGFsVG9rZW5zIjo0Mjd9fRkMP4U= + headers: + Connection: + - keep-alive + Content-Type: + - application/vnd.amazon.eventstream + Date: + - Sat, 17 Oct 2026 22:49:01 GMT + Transfer-Encoding: + - chunked + x-amzn-RequestId: + - 5ceccdfb-1053-424c-b3d8-050a251c63bd + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py b/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py index cfa227f2c..ada022360 100644 --- a/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py +++ b/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py @@ -331,6 +331,28 @@ def test_wrap_bedrock_converse_normalizes_tool_config_and_output(memory_logger): assert isinstance(tool_use.get("input"), dict) +@pytest.mark.vcr +def test_wrap_bedrock_converse_stream_parses_tool_input(memory_logger): + assert not memory_logger.pop() + client = wrap_bedrock(_bedrock_client()) + + response = client.converse_stream(**_converse_tool_kwargs(CONVERSE_STREAM_MODEL)) + tool_input = "" + for event in response["stream"]: + delta = event.get("contentBlockDelta", {}).get("delta", {}) + if "toolUse" in delta: + tool_input += delta["toolUse"]["input"] + + spans = memory_logger.pop() + assert len(spans) == 1 + content = spans[0]["output"][0]["content"] + tool_uses = [block for block in content if block.get("type") == "tool_use"] + assert tool_uses, f"expected a tool_use block in {content!r}" + # Same shape as the non-streamed converse output: the streamed JSON string is parsed. + assert tool_uses[0]["input"] == json.loads(tool_input) + assert isinstance(tool_uses[0]["input"], dict) + + @pytest.mark.vcr def test_wrap_bedrock_converse_with_image_input_materializes_attachment(memory_logger): assert not memory_logger.pop() diff --git a/py/src/braintrust/integrations/bedrock_runtime/tracing.py b/py/src/braintrust/integrations/bedrock_runtime/tracing.py index c778e359e..95d3ea927 100644 --- a/py/src/braintrust/integrations/bedrock_runtime/tracing.py +++ b/py/src/braintrust/integrations/bedrock_runtime/tracing.py @@ -560,6 +560,17 @@ def _anthropic_invoke_usage_metrics(model_id: Any, output: Any) -> dict[str, Any return metrics +def _parse_streamed_tool_input(text: str) -> Any: + """Parse streamed tool input into the JSON value the non-streamed converse output has.""" + if not text: + return {} + try: + return json.loads(text) + except ValueError: + # e.g. cut off by maxTokens: keep what was streamed + return text + + class _TracedConverseStream: """Wrap a botocore EventStream and log the final span when it drains.""" @@ -656,7 +667,7 @@ def _finish(self, error: BaseException | None) -> None: block = self._content_blocks.setdefault(idx, {}) text = "".join(pieces) if block.get("type") == "tool_use": - block["input"] = text + block["input"] = _parse_streamed_tool_input(text) else: block["text"] = text content = [self._content_blocks[idx] for idx in sorted(self._content_blocks)] From 9477af1be86fb06a046a18586d5713544556eb25 Mon Sep 17 00:00:00 2001 From: Alex Castillo Date: Sun, 18 Oct 2026 01:29:00 -0400 Subject: [PATCH 2/2] fix(bedrock): keep streamed tool input the JSON decoder rejects for depth json.loads raises RecursionError, not ValueError, for a document nested deeper than the decoder allows, so such tool input escaped the fallback and the span was never logged. Fall back to the raw string for it too. Co-Authored-By: Claude Opus 5.5 --- .../cassettes/test_auto_bedrock_runtime.yaml | 38 +++++++++++++++++ .../cassettes/test_wrap_bedrock_converse.yaml | 39 +++++++++++++++++ .../test_wrap_bedrock_converse_stream.yaml | 39 +++++++++++++++++ ...ock_converse_stream_parses_tool_input.yaml | 42 +++++++++++++++++++ ...ocument_input_materializes_attachment.yaml | 39 +++++++++++++++++ ...h_image_input_materializes_attachment.yaml | 39 +++++++++++++++++ ...preserves_response_body_and_logs_json.yaml | 41 ++++++++++++++++++ .../bedrock_runtime/test_bedrock_runtime.py | 8 ++++ .../integrations/bedrock_runtime/tracing.py | 5 ++- 9 files changed, 288 insertions(+), 2 deletions(-) create mode 100644 py/src/braintrust/integrations/bedrock_runtime/cassettes/test_auto_bedrock_runtime.yaml create mode 100644 py/src/braintrust/integrations/bedrock_runtime/cassettes/test_wrap_bedrock_converse.yaml create mode 100644 py/src/braintrust/integrations/bedrock_runtime/cassettes/test_wrap_bedrock_converse_stream.yaml create mode 100644 py/src/braintrust/integrations/bedrock_runtime/cassettes/test_wrap_bedrock_converse_stream_parses_tool_input.yaml create mode 100644 py/src/braintrust/integrations/bedrock_runtime/cassettes/test_wrap_bedrock_converse_with_document_input_materializes_attachment.yaml create mode 100644 py/src/braintrust/integrations/bedrock_runtime/cassettes/test_wrap_bedrock_converse_with_image_input_materializes_attachment.yaml create mode 100644 py/src/braintrust/integrations/bedrock_runtime/cassettes/test_wrap_bedrock_invoke_model_preserves_response_body_and_logs_json.yaml diff --git a/py/src/braintrust/integrations/bedrock_runtime/cassettes/test_auto_bedrock_runtime.yaml b/py/src/braintrust/integrations/bedrock_runtime/cassettes/test_auto_bedrock_runtime.yaml new file mode 100644 index 000000000..42748018e --- /dev/null +++ b/py/src/braintrust/integrations/bedrock_runtime/cassettes/test_auto_bedrock_runtime.yaml @@ -0,0 +1,38 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": [{"text": "Say hello in one word."}]}], + "inferenceConfig": {"maxTokens": 20, "temperature": 0}}' + headers: + Content-Length: + - '137' + Content-Type: + - !!binary | + YXBwbGljYXRpb24vanNvbg== + User-Agent: + - !!binary | + Qm90bzMvMS40My4xMTIgbWQvQm90b2NvcmUjMS40My4xMTIgdWEvMi4xIG9zL2xpbnV4IzYuNi44 + Ny4yLW1pY3Jvc29mdC1zdGFuZGFyZC1XU0wyIG1kL2FyY2gjeDg2XzY0IGxhbmcvcHl0aG9uIzMu + MTQuOCBtZC9weWltcGwjQ1B5dGhvbiBtL2IsWixnLEQgY2ZnL3JldHJ5LW1vZGUjbGVnYWN5IEJv + dG9jb3JlLzEuNDMuMTEy + method: POST + uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/converse + response: + body: + string: '{"message":"The security token included in the request is invalid."}' + headers: + Connection: + - keep-alive + Content-Length: + - '68' + Content-Type: + - application/json + Date: + - Sun, 18 Oct 2026 05:27:54 GMT + x-amzn-ErrorType: + - UnrecognizedClientException:http://internal.amazon.com/coral/com.amazon.coral.service/ + x-amzn-RequestId: + - ea0e8c40-7dc1-4df5-9a7e-efc1f69257c0 + status: + code: 403 + message: Forbidden +version: 1 diff --git a/py/src/braintrust/integrations/bedrock_runtime/cassettes/test_wrap_bedrock_converse.yaml b/py/src/braintrust/integrations/bedrock_runtime/cassettes/test_wrap_bedrock_converse.yaml new file mode 100644 index 000000000..d43e675fb --- /dev/null +++ b/py/src/braintrust/integrations/bedrock_runtime/cassettes/test_wrap_bedrock_converse.yaml @@ -0,0 +1,39 @@ +interactions: +- request: + body: '{"system": [{"text": "You answer with concise lowercase text."}], "messages": + [{"role": "user", "content": [{"text": "Say hello in one word."}]}], "inferenceConfig": + {"maxTokens": 20, "temperature": 0, "topP": 0.9, "stopSequences": ["STOP"]}}' + headers: + Content-Length: + - '242' + Content-Type: + - !!binary | + YXBwbGljYXRpb24vanNvbg== + User-Agent: + - !!binary | + Qm90bzMvMS40My4xMTIgbWQvQm90b2NvcmUjMS40My4xMTIgdWEvMi4xIG9zL2xpbnV4IzYuNi44 + Ny4yLW1pY3Jvc29mdC1zdGFuZGFyZC1XU0wyIG1kL2FyY2gjeDg2XzY0IGxhbmcvcHl0aG9uIzMu + MTQuOCBtZC9weWltcGwjQ1B5dGhvbiBtL2IsRCxaIGNmZy9yZXRyeS1tb2RlI2xlZ2FjeSBCb3Rv + Y29yZS8xLjQzLjExMg== + method: POST + uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/converse + response: + body: + string: '{"message":"The security token included in the request is invalid."}' + headers: + Connection: + - keep-alive + Content-Length: + - '68' + Content-Type: + - application/json + Date: + - Sun, 18 Oct 2026 05:27:52 GMT + x-amzn-ErrorType: + - UnrecognizedClientException:http://internal.amazon.com/coral/com.amazon.coral.service/ + x-amzn-RequestId: + - 2f5a437c-6ec6-4c41-a77c-b055d780cdfe + status: + code: 403 + message: Forbidden +version: 1 diff --git a/py/src/braintrust/integrations/bedrock_runtime/cassettes/test_wrap_bedrock_converse_stream.yaml b/py/src/braintrust/integrations/bedrock_runtime/cassettes/test_wrap_bedrock_converse_stream.yaml new file mode 100644 index 000000000..4ea4e4537 --- /dev/null +++ b/py/src/braintrust/integrations/bedrock_runtime/cassettes/test_wrap_bedrock_converse_stream.yaml @@ -0,0 +1,39 @@ +interactions: +- request: + body: '{"system": [{"text": "You answer with concise lowercase text."}], "messages": + [{"role": "user", "content": [{"text": "Say hello in one word."}]}], "inferenceConfig": + {"maxTokens": 20, "temperature": 0, "topP": 0.9, "stopSequences": ["STOP"]}}' + headers: + Content-Length: + - '242' + Content-Type: + - !!binary | + YXBwbGljYXRpb24vanNvbg== + User-Agent: + - !!binary | + Qm90bzMvMS40My4xMTIgbWQvQm90b2NvcmUjMS40My4xMTIgdWEvMi4xIG9zL2xpbnV4IzYuNi44 + Ny4yLW1pY3Jvc29mdC1zdGFuZGFyZC1XU0wyIG1kL2FyY2gjeDg2XzY0IGxhbmcvcHl0aG9uIzMu + MTQuOCBtZC9weWltcGwjQ1B5dGhvbiBtL2IsRCxaIGNmZy9yZXRyeS1tb2RlI2xlZ2FjeSBCb3Rv + Y29yZS8xLjQzLjExMg== + method: POST + uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/converse-stream + response: + body: + string: '{"message":"The security token included in the request is invalid."}' + headers: + Connection: + - keep-alive + Content-Length: + - '68' + Content-Type: + - application/json + Date: + - Sun, 18 Oct 2026 05:27:52 GMT + x-amzn-ErrorType: + - UnrecognizedClientException:http://internal.amazon.com/coral/com.amazon.coral.service/ + x-amzn-RequestId: + - fe27cd77-6f53-4862-be83-4c4fc388816a + status: + code: 403 + message: Forbidden +version: 1 diff --git a/py/src/braintrust/integrations/bedrock_runtime/cassettes/test_wrap_bedrock_converse_stream_parses_tool_input.yaml b/py/src/braintrust/integrations/bedrock_runtime/cassettes/test_wrap_bedrock_converse_stream_parses_tool_input.yaml new file mode 100644 index 000000000..a3db667e1 --- /dev/null +++ b/py/src/braintrust/integrations/bedrock_runtime/cassettes/test_wrap_bedrock_converse_stream_parses_tool_input.yaml @@ -0,0 +1,42 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": [{"text": "What''s the weather + in Paris? Use the tool."}]}], "inferenceConfig": {"maxTokens": 200, "temperature": + 0}, "toolConfig": {"tools": [{"toolSpec": {"name": "get_weather", "description": + "Get the current weather for a city.", "inputSchema": {"json": {"type": "object", + "properties": {"city": {"type": "string", "description": "City name"}}, "required": + ["city"]}}}}], "toolChoice": {"tool": {"name": "get_weather"}}}}' + headers: + Content-Length: + - '466' + Content-Type: + - !!binary | + YXBwbGljYXRpb24vanNvbg== + User-Agent: + - !!binary | + Qm90bzMvMS40My4xMTIgbWQvQm90b2NvcmUjMS40My4xMTIgdWEvMi4xIG9zL2xpbnV4IzYuNi44 + Ny4yLW1pY3Jvc29mdC1zdGFuZGFyZC1XU0wyIG1kL2FyY2gjeDg2XzY0IGxhbmcvcHl0aG9uIzMu + MTQuOCBtZC9weWltcGwjQ1B5dGhvbiBtL2IsRCxaIGNmZy9yZXRyeS1tb2RlI2xlZ2FjeSBCb3Rv + Y29yZS8xLjQzLjExMg== + method: POST + uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/converse-stream + response: + body: + string: '{"message":"The security token included in the request is invalid."}' + headers: + Connection: + - keep-alive + Content-Length: + - '68' + Content-Type: + - application/json + Date: + - Sun, 18 Oct 2026 05:27:53 GMT + x-amzn-ErrorType: + - UnrecognizedClientException:http://internal.amazon.com/coral/com.amazon.coral.service/ + x-amzn-RequestId: + - 89c89132-7c4c-427d-bfcc-3dc9c4167c4c + status: + code: 403 + message: Forbidden +version: 1 diff --git a/py/src/braintrust/integrations/bedrock_runtime/cassettes/test_wrap_bedrock_converse_with_document_input_materializes_attachment.yaml b/py/src/braintrust/integrations/bedrock_runtime/cassettes/test_wrap_bedrock_converse_with_document_input_materializes_attachment.yaml new file mode 100644 index 000000000..ed942f1fd --- /dev/null +++ b/py/src/braintrust/integrations/bedrock_runtime/cassettes/test_wrap_bedrock_converse_with_document_input_materializes_attachment.yaml @@ -0,0 +1,39 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": [{"text": "Summarize this document."}, + {"document": {"format": "pdf", "name": "tiny-pdf", "source": {"bytes": "JVBERi0xLjEKMSAwIG9iajw8L1R5cGUvQ2F0YWxvZy9QYWdlcyAyIDAgUj4+ZW5kb2JqCjIgMCBvYmo8PC9UeXBlL1BhZ2VzL0tpZHNbMyAwIFJdL0NvdW50IDE+PmVuZG9iagozIDAgb2JqPDwvVHlwZS9QYWdlL1BhcmVudCAyIDAgUi9NZWRpYUJveFswIDAgOCA4XT4+ZW5kb2JqCnhyZWYKMCA0CjAwMDAwMDAwMDAgNjU1MzUgZgowMDAwMDAwMDEwIDAwMDAwIG4KMDAwMDAwMDA1MyAwMDAwMCBuCjAwMDAwMDAwOTYgMDAwMDAgbgp0cmFpbGVyPDwvU2l6ZSA0L1Jvb3QgMSAwIFI+PgpzdGFydHhyZWYKMTQ0CiUlRU9G"}}}]}], + "inferenceConfig": {"maxTokens": 40, "temperature": 0}}' + headers: + Content-Length: + - '609' + Content-Type: + - !!binary | + YXBwbGljYXRpb24vanNvbg== + User-Agent: + - !!binary | + Qm90bzMvMS40My4xMTIgbWQvQm90b2NvcmUjMS40My4xMTIgdWEvMi4xIG9zL2xpbnV4IzYuNi44 + Ny4yLW1pY3Jvc29mdC1zdGFuZGFyZC1XU0wyIG1kL2FyY2gjeDg2XzY0IGxhbmcvcHl0aG9uIzMu + MTQuOCBtZC9weWltcGwjQ1B5dGhvbiBtL2IsRCxaIGNmZy9yZXRyeS1tb2RlI2xlZ2FjeSBCb3Rv + Y29yZS8xLjQzLjExMg== + method: POST + uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/converse + response: + body: + string: '{"message":"The security token included in the request is invalid."}' + headers: + Connection: + - keep-alive + Content-Length: + - '68' + Content-Type: + - application/json + Date: + - Sun, 18 Oct 2026 05:27:53 GMT + x-amzn-ErrorType: + - UnrecognizedClientException:http://internal.amazon.com/coral/com.amazon.coral.service/ + x-amzn-RequestId: + - a73087e1-da8c-4111-933b-f9ce27b246d4 + status: + code: 403 + message: Forbidden +version: 1 diff --git a/py/src/braintrust/integrations/bedrock_runtime/cassettes/test_wrap_bedrock_converse_with_image_input_materializes_attachment.yaml b/py/src/braintrust/integrations/bedrock_runtime/cassettes/test_wrap_bedrock_converse_with_image_input_materializes_attachment.yaml new file mode 100644 index 000000000..ee643ba53 --- /dev/null +++ b/py/src/braintrust/integrations/bedrock_runtime/cassettes/test_wrap_bedrock_converse_with_image_input_materializes_attachment.yaml @@ -0,0 +1,39 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": [{"text": "Describe this image + in one word."}, {"image": {"format": "png", "source": {"bytes": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII="}}}]}], + "inferenceConfig": {"maxTokens": 20, "temperature": 0}}' + headers: + Content-Length: + - '294' + Content-Type: + - !!binary | + YXBwbGljYXRpb24vanNvbg== + User-Agent: + - !!binary | + Qm90bzMvMS40My4xMTIgbWQvQm90b2NvcmUjMS40My4xMTIgdWEvMi4xIG9zL2xpbnV4IzYuNi44 + Ny4yLW1pY3Jvc29mdC1zdGFuZGFyZC1XU0wyIG1kL2FyY2gjeDg2XzY0IGxhbmcvcHl0aG9uIzMu + MTQuOCBtZC9weWltcGwjQ1B5dGhvbiBtL2IsRCxaIGNmZy9yZXRyeS1tb2RlI2xlZ2FjeSBCb3Rv + Y29yZS8xLjQzLjExMg== + method: POST + uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/converse + response: + body: + string: '{"message":"The security token included in the request is invalid."}' + headers: + Connection: + - keep-alive + Content-Length: + - '68' + Content-Type: + - application/json + Date: + - Sun, 18 Oct 2026 05:27:53 GMT + x-amzn-ErrorType: + - UnrecognizedClientException:http://internal.amazon.com/coral/com.amazon.coral.service/ + x-amzn-RequestId: + - 7652d08f-6121-43bb-b25a-9be92101ef6e + status: + code: 403 + message: Forbidden +version: 1 diff --git a/py/src/braintrust/integrations/bedrock_runtime/cassettes/test_wrap_bedrock_invoke_model_preserves_response_body_and_logs_json.yaml b/py/src/braintrust/integrations/bedrock_runtime/cassettes/test_wrap_bedrock_invoke_model_preserves_response_body_and_logs_json.yaml new file mode 100644 index 000000000..23be2a319 --- /dev/null +++ b/py/src/braintrust/integrations/bedrock_runtime/cassettes/test_wrap_bedrock_invoke_model_preserves_response_body_and_logs_json.yaml @@ -0,0 +1,41 @@ +interactions: +- request: + body: '{"messages": [{"role": "user", "content": [{"text": "Say hello in one word."}]}], + "inferenceConfig": {"max_new_tokens": 20, "temperature": 0}}' + headers: + Accept: + - !!binary | + YXBwbGljYXRpb24vanNvbg== + Content-Length: + - '142' + Content-Type: + - !!binary | + YXBwbGljYXRpb24vanNvbg== + User-Agent: + - !!binary | + Qm90bzMvMS40My4xMTIgbWQvQm90b2NvcmUjMS40My4xMTIgdWEvMi4xIG9zL2xpbnV4IzYuNi44 + Ny4yLW1pY3Jvc29mdC1zdGFuZGFyZC1XU0wyIG1kL2FyY2gjeDg2XzY0IGxhbmcvcHl0aG9uIzMu + MTQuOCBtZC9weWltcGwjQ1B5dGhvbiBtL2IsRCxaIGNmZy9yZXRyeS1tb2RlI2xlZ2FjeSBCb3Rv + Y29yZS8xLjQzLjExMg== + method: POST + uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/us.amazon.nova-lite-v1%3A0/invoke + response: + body: + string: '{"message":"The security token included in the request is invalid."}' + headers: + Connection: + - keep-alive + Content-Length: + - '68' + Content-Type: + - application/json + Date: + - Sun, 18 Oct 2026 05:27:52 GMT + x-amzn-ErrorType: + - UnrecognizedClientException:http://internal.amazon.com/coral/com.amazon.coral.service/ + x-amzn-RequestId: + - d648815c-2b69-4cf4-8d50-bb3c11f129fd + status: + code: 403 + message: Forbidden +version: 1 diff --git a/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py b/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py index ada022360..dbed7cf8b 100644 --- a/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py +++ b/py/src/braintrust/integrations/bedrock_runtime/test_bedrock_runtime.py @@ -290,6 +290,14 @@ def _converse_tool_kwargs(model_id=CONVERSE_MODEL): @pytest.mark.vcr +def test_parse_streamed_tool_input_keeps_input_the_decoder_rejects(): + from braintrust.integrations.bedrock_runtime.tracing import _parse_streamed_tool_input + + assert _parse_streamed_tool_input('{"city": "Par') == '{"city": "Par' + deeply_nested = "[" * 100_000 + "]" * 100_000 + assert _parse_streamed_tool_input(deeply_nested) == deeply_nested + + def test_wrap_bedrock_converse_normalizes_tool_config_and_output(memory_logger): assert not memory_logger.pop() client = wrap_bedrock(_bedrock_client()) diff --git a/py/src/braintrust/integrations/bedrock_runtime/tracing.py b/py/src/braintrust/integrations/bedrock_runtime/tracing.py index 95d3ea927..d029db192 100644 --- a/py/src/braintrust/integrations/bedrock_runtime/tracing.py +++ b/py/src/braintrust/integrations/bedrock_runtime/tracing.py @@ -566,8 +566,9 @@ def _parse_streamed_tool_input(text: str) -> Any: return {} try: return json.loads(text) - except ValueError: - # e.g. cut off by maxTokens: keep what was streamed + except (ValueError, RecursionError): + # e.g. cut off by maxTokens, or nested deeper than the decoder allows: + # keep what was streamed return text