Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions lib/ruby_llm/error.rb
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,10 @@ def initialize(type = nil)
end
end

# Raised when a message holds content blocks the selected protocol cannot send, e.g. an
# InvokeModel compaction block replayed through Bedrock Converse.
class UnsupportedContentError < StandardError; end

# Error classes for different HTTP status codes
# Raised when the API request is invalid.
class BadRequestError < Error
Expand Down
5 changes: 4 additions & 1 deletion lib/ruby_llm/message.rb
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ class Message
:CONTENT_FILTERED_FINISH_REASONS

attr_reader :role, :model_id, :tool_calls, :tool_call_id, :raw, :thinking, :tokens, :citations,
:finish_reason
:finish_reason, :provider_data
attr_writer :content

def initialize(options = {})
Expand All @@ -35,6 +35,9 @@ def initialize(options = {})
@thinking = options[:thinking]
@citations = Array(options[:citations])
@finish_reason = options[:finish_reason]
# Response fields a protocol passes through without modeling them, as plain string-keyed
# hashes (e.g. InvokeAnthropic's context_management and usage iterations).
@provider_data = options[:provider_data] || {}

ensure_valid_role
end
Expand Down
38 changes: 36 additions & 2 deletions lib/ruby_llm/protocols/converse/chat.rb
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ def format_non_tool_message(msg)

def format_message_content(msg)
if msg.content.is_a?(RubyLLM::Content::Raw)
return format_raw_content(msg.content) if msg.role == :assistant
return reject_messages_api_blocks(format_raw_content(msg.content)) if msg.role == :assistant

return sanitize_non_assistant_raw_blocks(format_raw_content(msg.content))
end
Expand Down Expand Up @@ -209,6 +209,20 @@ def format_raw_content(content)
value.is_a?(Array) ? value : [value]
end

# An assistant Raw block whose Anthropic `type` the fork does not model (compaction,
# server_tool_use, tool_search_tool_result, ...) was kept by the InvokeModel protocol
# because only InvokeModel can carry it; Converse has no field for it. Fail clearly
# rather than send a block Converse would drop or reject.
def reject_messages_api_blocks(blocks)
types = blocks.filter_map { |block| block[:type] || block['type'] if block.is_a?(Hash) }.map(&:to_s)
unsupported = (types - InvokeAnthropic::Chat::MODELED_BLOCK_TYPES).uniq
return blocks if unsupported.empty?

raise UnsupportedContentError,
"Bedrock Converse cannot send #{unsupported.join(', ')} content blocks; " \
'continue this chat with protocol: :invoke_anthropic'
end

def sanitize_non_assistant_raw_blocks(blocks)
blocks.filter_map do |block|
next unless block.is_a?(Hash)
Expand Down Expand Up @@ -395,15 +409,35 @@ def budget_reasoning_config(thinking)
# When the turn had more than one reasoning block (common under adaptive/interleaved
# thinking during tool use), thinking.blocks holds the exact original blocks and must
# be replayed verbatim instead of reconstructed from the merged text/signature.
#
# Blocks captured by the InvokeAnthropic protocol are in the Anthropic Messages shape
# (type: thinking / redacted_thinking); those are translated to reasoningContent so a
# chat that moves from InvokeModel back to Converse can still replay its reasoning.
# reasoningContent blocks pass through untouched.
def format_thinking_blocks(thinking)
return nil unless thinking

return thinking.blocks if thinking.blocks
return thinking.blocks.map { |block| converse_thinking_block(block) } if thinking.blocks

block = format_single_thinking_block(thinking)
block ? [block] : nil
end

def converse_thinking_block(block)
return block unless block.is_a?(Hash)

case (block['type'] || block[:type]).to_s
when 'thinking'
text = { text: block['thinking'] || block[:thinking] || '',
signature: block['signature'] || block[:signature] }
{ reasoningContent: { reasoningText: text.compact } }
when 'redacted_thinking'
{ reasoningContent: { redactedContent: block['data'] || block[:data] } }
else
block
end
end

# Lossy fallback for messages persisted without raw thinking blocks. `signature`
# here is always a real reasoningText signature (streaming captures it from
# reasoningContent signature deltas; redacted content is only ever captured into
Expand Down
149 changes: 149 additions & 0 deletions lib/ruby_llm/protocols/converse/event_stream.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
# frozen_string_literal: true

require 'base64'
require 'faraday'
require 'json'

module RubyLLM
module Protocols
class Converse
# AWS Event Stream plumbing shared by every bedrock-runtime streaming endpoint: ConverseStream
# (Converse::Streaming) and InvokeModelWithResponseStream (InvokeAnthropic::Streaming). Both
# frame each event as a binary eventstream message whose JSON payload may wrap the real
# event as base64 `bytes`, and both report mid-stream failures as `:exception-type`
# messages. What each endpoint puts INSIDE the payload differs, so event interpretation
# stays with the including protocol.
module EventStream
ErrorResponse = Struct.new(:body, :status)

# Bedrock stream exception type => HTTP status, so ErrorMiddleware raises the same error
# class a synchronous call with that failure would.
EXCEPTION_STATUSES = {
'throttlingException' => 429,
'validationException' => 400,
'accessDeniedException' => 401,
'unrecognizedClientException' => 401,
'serviceUnavailableException' => 503
}.freeze

private

# POSTs a SigV4-signed eventstream request and hands every raw body chunk of a 200
# response to the block. Non-200 chunks go through handle_failed_stream.
def post_event_stream(url, payload, additional_headers = {}, &on_chunk)
body = JSON.generate(payload)

@connection.post(url, payload) do |req|
req.headers.merge!(@provider.sign_headers('POST', url, body))
req.headers.merge!(additional_headers) unless additional_headers.empty?
req.headers['Accept'] = 'application/vnd.amazon.eventstream'

if Faraday::VERSION.start_with?('1')
req.options[:on_data] = proc { |chunk, _size| on_chunk.call(chunk) }
else
req.options.on_data = proc do |chunk, _bytes, env|
if env&.status == 200
on_chunk.call(chunk)
else
handle_failed_stream(chunk, env)
end
end
end
end
end

def event_stream_decoder
require 'aws-eventstream'
Aws::EventStream::Decoder.new
rescue LoadError
raise Error,
'The aws-eventstream gem is required for Bedrock streaming. ' \
'Please add it to your Gemfile: gem "aws-eventstream"'
end

def handle_failed_stream(chunk, env)
data = JSON.parse(chunk)
error_response = env.merge(body: data)
ErrorMiddleware.parse_error(provider: self, response: error_response)
rescue JSON::ParserError
RubyLLM.logger.debug { "Failed Bedrock stream error chunk: #{chunk}" }
end

def handle_non_eventstream_error_chunk(raw_chunk)
text = raw_chunk.to_s

if text.start_with?('event: error')
payload = text.lines.find { |line| line.start_with?('data:') }&.delete_prefix('data:')&.strip
raise_streaming_chunk_error(payload) if payload
return
end

return unless text.lstrip.start_with?('{') && text.include?('"error"')

raise_streaming_chunk_error(text)
end

def raise_streaming_chunk_error(payload)
parsed = JSON.parse(payload)
message = parsed.dig('error', 'message') || parsed['message'] || 'Bedrock streaming error'
response = ErrorResponse.new({ 'message' => message }, 500)
ErrorMiddleware.parse_error(provider: self, response: response)
rescue JSON::ParserError
nil
end

# Every complete eventstream message in raw_chunk (the decoder buffers partial ones
# across calls).
def decode_event_messages(decoder, raw_chunk)
messages = []
message, eof = decoder.decode_chunk(raw_chunk)

while message
messages << message
break if eof

message, eof = decoder.decode_chunk
end

messages
end

def event_type_header(message)
headers = message.headers
header = headers[':event-type'] || headers[':exception-type']
value = header.respond_to?(:value) ? header.value : header
value.is_a?(String) && !value.empty? ? value : nil
rescue StandardError
nil
end

def exception_type_header(message)
header = message.headers[':exception-type']
value = header.respond_to?(:value) ? header.value : header
value.is_a?(String) && !value.empty? ? value : nil
rescue StandardError
nil
end

def decode_event_payload(payload)
outer = JSON.parse(payload)

if outer['bytes'].is_a?(String)
JSON.parse(Base64.decode64(outer['bytes']))
else
outer
end
rescue JSON::ParserError => e
RubyLLM.logger.debug { "Failed to decode Bedrock stream event payload: #{e.message}" }
nil
end

def raise_event_stream_exception(exception_type, message)
status = EXCEPTION_STATUSES.fetch(exception_type, 500)
response = ErrorResponse.new({ 'message' => message || exception_type }, status)
ErrorMiddleware.parse_error(provider: self, response: response)
end
end
end
end
end
Loading
Loading