diff --git a/.github/workflows/session-mongodb.yml b/.github/workflows/session-mongodb.yml index b91b33e28..872bb1a49 100644 --- a/.github/workflows/session-mongodb.yml +++ b/.github/workflows/session-mongodb.yml @@ -55,4 +55,4 @@ jobs: run: .github/scripts/start-session-mongo-replset.sh 27017 - name: Run MongoDB driver conformance - run: cargo test --manifest-path integration-proofs/run-ledger-mongodb/Cargo.toml --features mongodb-storage-tests --test session_driver_conformance -- --ignored + run: cargo test --locked --manifest-path integration-proofs/run-ledger-mongodb/Cargo.toml --features mongodb-storage-tests --test session_driver_conformance -- --ignored diff --git a/crates/tinyagents-graph/src/testkit/conformance.rs b/crates/tinyagents-graph/src/testkit/conformance.rs index 835a3458c..4e45b54eb 100644 --- a/crates/tinyagents-graph/src/testkit/conformance.rs +++ b/crates/tinyagents-graph/src/testkit/conformance.rs @@ -792,11 +792,11 @@ where let short = key("g", "n", "short"); cache - .put(&short, json!(1), Some(Duration::from_millis(30))) + .put(&short, json!(1), Some(Duration::from_secs(2))) .await .unwrap(); assert_eq!(cache.get(&short).await.unwrap(), Some(json!(1))); - tokio::time::sleep(Duration::from_millis(80)).await; + tokio::time::sleep(Duration::from_millis(2_100)).await; assert_eq!(cache.get(&short).await.unwrap(), None, "expired after TTL"); assert_eq!(cache.get(&short).await.unwrap(), None, "stays expired"); cache.put(&short, json!(2), None).await.unwrap(); diff --git a/crates/tinyagents-harness-tools/README.md b/crates/tinyagents-harness-tools/README.md index e2bb3d3f2..7e9780957 100644 --- a/crates/tinyagents-harness-tools/README.md +++ b/crates/tinyagents-harness-tools/README.md @@ -29,7 +29,8 @@ run-state middleware and its required `goals` and `todos` modules. `tinyagents_orchestration::subagent::SubAgentJobRegistry`. - `thread_locks` provides the public `ThreadLockMap` concurrency primitive. - The crate root exports `CurrentTimeTool`, `ResolveTimeTool`, - `AskClarificationTool`, `PlanExitTool`, and the `per_call` binding adapter. + `AskClarificationTool`, `PlanExitTool`, `WaitTool`, `WaitLoopTool`, and the + `per_call` binding adapter. Subagent registry policy stays in `tinyagents-orchestration`; this crate owns the reusable tool declarations. The harness owns tool registration, diff --git a/crates/tinyagents-harness-tools/src/lib.rs b/crates/tinyagents-harness-tools/src/lib.rs index a8bbfcdee..8961b7772 100644 --- a/crates/tinyagents-harness-tools/src/lib.rs +++ b/crates/tinyagents-harness-tools/src/lib.rs @@ -12,6 +12,7 @@ mod plan_exit; pub mod thread_locks; mod time; mod time_parse; +mod wait; #[cfg(feature = "goals")] pub mod goals; @@ -29,6 +30,7 @@ pub mod todos; pub use ask_clarification::AskClarificationTool; pub use plan_exit::{PLAN_EXIT_MARKER, PlanExitModeLookup, PlanExitTool}; pub use time::{CurrentTimeTool, ResolveTimeTool, register_time_tools, time_tools}; +pub use wait::{WaitLoopTool, WaitTool}; #[cfg(test)] #[path = "time_parse_tests.rs"] diff --git a/crates/tinyagents-harness-tools/src/wait.rs b/crates/tinyagents-harness-tools/src/wait.rs new file mode 100644 index 000000000..9529f000e --- /dev/null +++ b/crates/tinyagents-harness-tools/src/wait.rs @@ -0,0 +1,302 @@ +//! Tool: `wait` / `wait_loop` - delayed callback ticks for the orchestrator. +//! +//! These tools intentionally do not own scheduling state. They sleep for a +//! bounded duration, then return the caller-provided message as a tool result so +//! the orchestrator can decide whether to act, poll async sub-agents, or call the +//! loop variant again. + +use std::time::Duration; + +use async_trait::async_trait; +use serde_json::json; +use tinytools::{PermissionLevel, Tool, ToolResult, ToolTimeout}; + +const DEFAULT_DURATION_SECS: u64 = 5; +const MAX_DURATION_SECS: u64 = 600; +const MILLIS_PER_SEC: u64 = 1_000; + +/// One-shot delayed callback tool for the orchestrator. +pub struct WaitTool; +/// Repeatable delayed callback tool for orchestrator-controlled polling loops. +pub struct WaitLoopTool; + +impl WaitTool { + pub fn new() -> Self { + Self + } +} + +impl Default for WaitTool { + fn default() -> Self { + Self::new() + } +} + +impl WaitLoopTool { + pub fn new() -> Self { + Self + } +} + +impl Default for WaitLoopTool { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl Tool for WaitTool { + fn name(&self) -> &str { + "wait" + } + + fn description(&self) -> &str { + "Wait for a bounded duration, then return the provided callback message \ + to the orchestrator. Use this to create a delayed tick before checking \ + async work or retrying a condition." + } + + fn parameters_schema(&self) -> serde_json::Value { + wait_schema(false) + } + + fn permission_level(&self) -> PermissionLevel { + PermissionLevel::None + } + + fn timeout_policy(&self, args: &serde_json::Value) -> ToolTimeout { + timeout_policy_for_wait(args) + } + + async fn execute(&self, args: serde_json::Value) -> anyhow::Result { + execute_wait(args, false).await + } +} + +#[async_trait] +impl Tool for WaitLoopTool { + fn name(&self) -> &str { + "wait_loop" + } + + fn description(&self) -> &str { + "Wait for a bounded duration, then return the same callback message plus \ + a ready-to-call wait_loop instruction. Use this for deliberate polling \ + loops where the orchestrator decides after each tick whether to repeat." + } + + fn parameters_schema(&self) -> serde_json::Value { + wait_schema(true) + } + + fn permission_level(&self) -> PermissionLevel { + PermissionLevel::None + } + + fn timeout_policy(&self, args: &serde_json::Value) -> ToolTimeout { + timeout_policy_for_wait(args) + } + + async fn execute(&self, args: serde_json::Value) -> anyhow::Result { + execute_wait(args, true).await + } +} + +/// Sleep for the requested duration, then return the callback tick payload. +async fn execute_wait(args: serde_json::Value, loop_mode: bool) -> anyhow::Result { + let request = match parse_wait_request(&args) { + Ok(request) => request, + Err(message) => return Ok(ToolResult::error(message)), + }; + let tool_name = if loop_mode { "wait_loop" } else { "wait" }; + tracing::info!( + "[wait] tool={} duration_ms={} iteration={} loop_key={} message_chars={}", + tool_name, + request.duration_ms, + request.iteration, + request.loop_key.as_deref().unwrap_or("none"), + request.message.chars().count() + ); + + tokio::time::sleep(Duration::from_millis(request.duration_ms)).await; + + tracing::debug!( + "[wait] elapsed tool={} duration_ms={} iteration={} loop_key={}", + tool_name, + request.duration_ms, + request.iteration, + request.loop_key.as_deref().unwrap_or("none") + ); + + Ok(ToolResult::success(format_wait_tick(&request, loop_mode))) +} + +/// Build the JSON schema shared by `wait` and `wait_loop`. +fn wait_schema(loop_mode: bool) -> serde_json::Value { + let mut properties = serde_json::Map::new(); + properties.insert( + "message".to_string(), + json!({ + "type": "string", + "description": "Callback message to return to the orchestrator after the wait elapses." + }), + ); + properties.insert( + "duration_secs".to_string(), + json!({ + "type": "integer", + "minimum": 1, + "maximum": MAX_DURATION_SECS, + "description": "Seconds to wait before returning the callback. Default 5. Ignored when duration_ms is supplied." + }), + ); + properties.insert( + "duration_ms".to_string(), + json!({ + "type": "integer", + "minimum": 1, + "maximum": MAX_DURATION_SECS * MILLIS_PER_SEC, + "description": "Milliseconds to wait before returning the callback. Use only for short test-sized waits." + }), + ); + + if loop_mode { + properties.insert( + "loop_key".to_string(), + json!({ + "type": "string", + "description": "Optional caller-defined key for the polling loop." + }), + ); + properties.insert( + "iteration".to_string(), + json!({ + "type": "integer", + "minimum": 1, + "description": "Current loop iteration. Defaults to 1; the returned instruction increments it." + }), + ); + } + + json!({ + "type": "object", + "required": ["message"], + "properties": serde_json::Value::Object(properties) + }) +} + +/// Give the harness a deadline that covers the requested wait plus grace. +fn timeout_policy_for_wait(args: &serde_json::Value) -> ToolTimeout { + let duration_ms = duration_ms_from_args(args).unwrap_or(DEFAULT_DURATION_SECS * MILLIS_PER_SEC); + let rounded_secs = duration_ms.saturating_add(MILLIS_PER_SEC - 1) / MILLIS_PER_SEC; + ToolTimeout::Millis( + rounded_secs + .saturating_add(1) + .saturating_mul(MILLIS_PER_SEC), + ) +} + +/// Extract and clamp the caller's requested wait duration in milliseconds. +fn duration_ms_from_args(args: &serde_json::Value) -> Option { + if let Some(duration_ms) = args.get("duration_ms").and_then(|v| v.as_u64()) { + return Some(duration_ms.clamp(1, MAX_DURATION_SECS * MILLIS_PER_SEC)); + } + args.get("duration_secs") + .and_then(|v| v.as_u64()) + .map(|secs| secs.clamp(1, MAX_DURATION_SECS) * MILLIS_PER_SEC) +} + +/// Validate tool arguments and normalize them into an internal request. +fn parse_wait_request(args: &serde_json::Value) -> Result { + let message = args + .get("message") + .and_then(|v| v.as_str()) + .unwrap_or("") + .trim() + .to_string(); + if message.is_empty() { + return Err("wait: `message` is required".to_string()); + } + + let duration_ms = duration_ms_from_args(args).unwrap_or(DEFAULT_DURATION_SECS * MILLIS_PER_SEC); + let iteration = args + .get("iteration") + .and_then(|v| v.as_u64()) + .unwrap_or(1) + .max(1); + let loop_key = args + .get("loop_key") + .and_then(|v| v.as_str()) + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_string); + + Ok(WaitRequest { + message, + duration_ms, + iteration, + loop_key, + }) +} + +/// Render the delayed callback as prose plus a machine-readable `[wait_tick]`. +fn format_wait_tick(request: &WaitRequest, loop_mode: bool) -> String { + let seconds = request.duration_ms as f64 / MILLIS_PER_SEC as f64; + let loop_instruction = loop_mode.then(|| { + let mut arguments = json!({ + "message": request.message, + "duration_ms": request.duration_ms, + "iteration": request.iteration.saturating_add(1) + }); + // The schema allows `loop_key` only as a string: omit it rather than + // hand back a ready-to-call instruction carrying `null`. + if let Some(loop_key) = &request.loop_key { + arguments["loop_key"] = json!(loop_key); + } + json!({ "tool": "wait_loop", "arguments": arguments }) + }); + let payload = json!({ + "status": "elapsed", + "message": request.message, + "duration_ms": request.duration_ms, + "durationMs": request.duration_ms, + "duration_secs": seconds, + "durationSecs": seconds, + "loop": loop_mode, + "loop_key": request.loop_key, + "loopKey": request.loop_key, + "iteration": request.iteration, + "instructions": { + "callback_message": request.message, + "repeat": loop_instruction + } + }); + + let prefix = if loop_mode { + format!( + "Loop tick {} elapsed after {}ms.", + request.iteration, request.duration_ms + ) + } else { + format!("Wait elapsed after {}ms.", request.duration_ms) + }; + + format!( + "{prefix}\n\n[wait_tick]\n{}\n[/wait_tick]\n\n{}", + serde_json::to_string(&payload).unwrap_or_else(|_| "{}".to_string()), + request.message + ) +} + +#[derive(Debug)] +/// Normalized wait arguments used by both wait tools. +struct WaitRequest { + message: String, + duration_ms: u64, + iteration: u64, + loop_key: Option, +} + +#[cfg(test)] +#[path = "wait_tests.rs"] +mod tests; diff --git a/crates/tinyagents-harness-tools/src/wait_tests.rs b/crates/tinyagents-harness-tools/src/wait_tests.rs new file mode 100644 index 000000000..2352b2c18 --- /dev/null +++ b/crates/tinyagents-harness-tools/src/wait_tests.rs @@ -0,0 +1,133 @@ +use super::*; + +#[test] +fn wait_schema_requires_message() { + let schema = WaitTool::new().parameters_schema(); + let required = schema + .get("required") + .and_then(|v| v.as_array()) + .expect("required list"); + assert!(required.iter().any(|v| v.as_str() == Some("message"))); + assert!(schema["properties"].get("duration_secs").is_some()); + assert!(schema["properties"].get("duration_ms").is_some()); +} + +#[test] +fn wait_loop_schema_includes_loop_controls() { + let schema = WaitLoopTool::new().parameters_schema(); + assert!(schema["properties"].get("loop_key").is_some()); + assert!(schema["properties"].get("iteration").is_some()); +} + +#[test] +fn parse_wait_request_clamps_duration_and_iteration() { + let request = parse_wait_request(&json!({ + "message": "check subagents", + "duration_secs": 9999, + "iteration": 0 + })) + .unwrap(); + assert_eq!(request.duration_ms, MAX_DURATION_SECS * MILLIS_PER_SEC); + assert_eq!(request.iteration, 1); +} + +#[test] +fn missing_message_is_rejected() { + let err = parse_wait_request(&json!({ "duration_secs": 1 })).unwrap_err(); + assert!(err.contains("message")); +} + +#[test] +fn wait_loop_tick_repeats_same_message() { + let request = parse_wait_request(&json!({ + "message": "poll async workers", + "duration_ms": 10, + "loop_key": "workers", + "iteration": 2 + })) + .unwrap(); + let output = format_wait_tick(&request, true); + + assert!(output.contains("Loop tick 2 elapsed")); + assert!(output.contains("\"tool\":\"wait_loop\"")); + assert!(output.contains("\"message\":\"poll async workers\"")); + assert!(output.contains("\"iteration\":3")); +} + +#[test] +fn wait_loop_tick_at_max_iteration_does_not_overflow() { + let request = parse_wait_request(&json!({ + "message": "poll", + "duration_ms": 10, + "iteration": u64::MAX + })) + .unwrap(); + let output = format_wait_tick(&request, true); + assert!(output.contains(&format!("\"iteration\":{}", u64::MAX))); +} + +#[test] +fn wait_loop_tick_omits_an_absent_loop_key() { + let request = parse_wait_request(&json!({ "message": "poll", "duration_ms": 10 })).unwrap(); + let output = format_wait_tick(&request, true); + let json = output + .split("[wait_tick]\n") + .nth(1) + .and_then(|rest| rest.split("\n[/wait_tick]").next()) + .expect("wait_tick payload"); + let payload: serde_json::Value = serde_json::from_str(json).unwrap(); + let arguments = &payload["instructions"]["repeat"]["arguments"]; + assert!(arguments.is_object()); + assert!( + arguments.get("loop_key").is_none(), + "the repeat call must not carry a null loop_key: {arguments}" + ); +} + +#[tokio::test] +async fn wait_execute_returns_callback_message() { + let res = WaitTool::new() + .execute(json!({ + "message": "time to check", + "duration_ms": 1 + })) + .await + .unwrap(); + assert!(!res.is_error); + assert!(res.output().contains("[wait_tick]")); + assert!(res.output().contains("time to check")); +} + +#[test] +fn delayed_wait_tools_are_available_to_read_only_turns() { + assert_eq!( + WaitTool::new().permission_level(), + tinytools::PermissionLevel::None + ); + assert_eq!( + WaitLoopTool::new().permission_level(), + tinytools::PermissionLevel::None + ); +} + +/// Wire shape is a contract with prompts and recorded transcripts: names and +/// schemas must stay byte-identical across moves. +#[test] +fn wire_schemas_are_stable() { + assert_eq!(WaitTool::new().name(), "wait"); + assert_eq!(WaitLoopTool::new().name(), "wait_loop"); + let base = json!({ + "type": "object", + "required": ["message"], + "properties": { + "message": {"type": "string", "description": "Callback message to return to the orchestrator after the wait elapses."}, + "duration_secs": {"type": "integer", "minimum": 1, "maximum": 600, "description": "Seconds to wait before returning the callback. Default 5. Ignored when duration_ms is supplied."}, + "duration_ms": {"type": "integer", "minimum": 1, "maximum": 600000, "description": "Milliseconds to wait before returning the callback. Use only for short test-sized waits."} + } + }); + assert_eq!(WaitTool::new().parameters_schema(), base); + let mut looped = base.clone(); + looped["properties"]["loop_key"] = json!({"type": "string", "description": "Optional caller-defined key for the polling loop."}); + looped["properties"]["iteration"] = json!({"type": "integer", "minimum": 1, "description": "Current loop iteration. Defaults to 1; the returned instruction increments it."}); + assert_eq!(WaitLoopTool::new().parameters_schema(), looped); +} diff --git a/crates/tinyagents-harness/src/agent_loop/README.md b/crates/tinyagents-harness/src/agent_loop/README.md index 7ee1d135b..e9bc801fd 100644 --- a/crates/tinyagents-harness/src/agent_loop/README.md +++ b/crates/tinyagents-harness/src/agent_loop/README.md @@ -147,17 +147,35 @@ as Anthropic and some gateways spell it — an empty `max_tokens` reply takes th path too) with no visible text, no tool calls, and no structured output — a result useless to every caller. Before finalizing such a turn (and before structured extraction, which would otherwise fail on the empty completion) the loop retries the model call up to -`runtime::RunPolicy::truncated_empty_retries` times (default `1`, so two -attempts total). Each retry drops the useless assistant row, doubles the -request's `max_tokens` when one was set — clamped at 4x the original cap, and a -deliberate override of the per-turn output cap that caused the truncation — or -re-issues unchanged when no budget was set (the failure is stochastic, so a -plain retry still helps). Each attempt counts as a model call and emits +`runtime::RunPolicy::truncated_empty_retries` times (default `2`, so three +attempts total). Each retry drops the useless assistant row. The first retries +at the same output cap; subsequent retries double `max_tokens` when one was +set, clamped at 4x the original cap. This deliberately overrides the per-turn +output cap that caused the truncation. With no cap, the request is reissued +unchanged: the failure is stochastic, so a plain retry still helps. +`RunPolicy::truncated_empty_constant_cap` (default `false`) instead keeps every +retry at the original cap, including later retries that would normally grow. +The wall-clock and model-call limits still bound these retries. +When truncated-empty recovery is enabled, blank length-stopped responses are +neither stored in the response cache nor accepted from existing cache entries, +so retries at an unchanged cap still reach the provider. Usable answers remain +cacheable; disabling both truncated-empty retries and nudges retains replay. Each attempt counts as a model call and emits `AgentEvent::RetryScheduled`. The retry runs *before* `RunPolicy::error_on_empty_response`; only once the retries are exhausted does that guard (if enabled) turn the still-blank final into `TinyAgentsError::EmptyResponse`. +`RunPolicy::truncated_empty_reasoning_off` (default `false`) sends the call +after a truncated-empty reply, whether a retry or a nudge, with +`ReasoningEffort::None`. This overrides the configured reasoning effort for +that one call; later calls return to the policy's reasoning configuration. +`RunPolicy::truncated_empty_reasoning_off_sticky_after` (default `0`, disabled) +instead makes reasoning-off persist for the rest of the run once that many +truncated-empty replies have occurred, across tool-call steps. The sticky +threshold applies independently of the one-call setting. These controls can +combine with constant-cap retries to avoid spending a larger budget on the +same reasoning-only failure. + A hosted reasoning model on a high effort setting can deliberate past the boosted budget too. Once the retries are spent the loop does not finish on the blank reply: up to `RunPolicy::truncated_empty_nudges` times (default `1`) it diff --git a/crates/tinyagents-harness/src/agent_loop/mod_tests.rs b/crates/tinyagents-harness/src/agent_loop/mod_tests.rs index 21849e500..e512d2988 100644 --- a/crates/tinyagents-harness/src/agent_loop/mod_tests.rs +++ b/crates/tinyagents-harness/src/agent_loop/mod_tests.rs @@ -1102,13 +1102,14 @@ async fn truncated_empty_retry_does_not_replay_a_cached_blank() { use crate::cache::InMemoryResponseCache; let model = Arc::new(crate::testkit::ScriptedModel::new(vec![ + truncated_empty_response(2048), truncated_empty_response(2048), text_response("recovered", 4, 3), ])); let mut harness: AgentHarness<()> = AgentHarness::new(); harness.register_model("mock", Arc::clone(&model) as _); harness.with_policy(RunPolicy { - // This test pins the cap ladder; keep reasoning on so the ladder runs. + // The first retry keeps its cap; only the second may grow it. ..RunPolicy::default() }); harness.with_response_cache(Arc::new(InMemoryResponseCache::new())); @@ -1122,8 +1123,12 @@ async fn truncated_empty_retry_does_not_replay_a_cached_blank() { .await .expect("length-truncated empty completion should reach the provider again"); assert_eq!(run.text(), Some("recovered".to_string())); - assert_eq!(model.requests().len(), 2); - assert_eq!(model.requests()[1].max_tokens, Some(4096)); + let caps: Vec<_> = model + .requests() + .iter() + .map(|request| request.max_tokens) + .collect(); + assert_eq!(caps, [Some(2048), Some(2048), Some(4096)]); } #[tokio::test] diff --git a/crates/tinyagents-harness/src/agent_loop/model_call.rs b/crates/tinyagents-harness/src/agent_loop/model_call.rs index 45663c6ea..6099f6a04 100644 --- a/crates/tinyagents-harness/src/agent_loop/model_call.rs +++ b/crates/tinyagents-harness/src/agent_loop/model_call.rs @@ -250,7 +250,7 @@ impl AgentHarness { } }; if let Some(mut cached) = looked_up { - if Self::should_skip_empty_response_cache(shape, &cached) { + if self.should_skip_empty_response_cache(shape, &cached) { tracing::debug!( call_id = %call_id.as_str(), "[cache] ignoring blank cached response; retry policy requires a provider call" @@ -333,7 +333,7 @@ impl AgentHarness { // reach the provider on the next identical request. Caching the // first successful-but-unusable completion would replay that blank // answer, mark it `served_from_cache`, and defeat the retry. - let blank_for_retry = Self::should_skip_empty_response_cache(shape, &response); + let blank_for_retry = self.should_skip_empty_response_cache(shape, &response); // Only the *primary* model's answer may be stored under this key. // `invoke_model_resolving` walks the fallback chain on failure and // can return a different model's response; writing that under the @@ -377,19 +377,28 @@ impl AgentHarness { /// A prior cache entry or a newly completed response must not short-circuit /// an opted-in retry when it contains no usable assistant answer. fn should_skip_empty_response_cache( + &self, shape: &super::dialect::CallShape, response: &ModelResponse, ) -> bool { // `ModelResponse` has no independent structured payload: the // StructuredExtractor reads visible text or tool-call arguments. The - // call shape still excludes structured plans so their own output retry - // policy remains the sole owner of failed extraction and caching. - shape.retry_empty_final + // call shape excludes structured plans from non-truncated empty retry; + // length-stopped blanks use the shared truncated-empty recovery before + // extraction, so they must also bypass cached dead responses. + let retryable = if crate::finish_reason::is_length_stop(response.finish_reason.as_deref()) { + // Truncated-empty recovery can resend an identical request, even + // without constant-cap mode: its first retry keeps the same cap. + // Exclude both prior entries and new writes so that retry reaches + // the provider rather than replaying the unusable cached answer. + self.policy.truncated_empty_retries > 0 || self.policy.truncated_empty_nudges > 0 + } else { + shape.retry_empty_final && response.finish_reason.as_deref() != Some("tool_calls") + }; + retryable && response.continue_turn.is_none() && response.message.tool_calls.is_empty() && response.text().trim().is_empty() - && !crate::finish_reason::is_length_stop(response.finish_reason.as_deref()) - && response.finish_reason.as_deref() != Some("tool_calls") } /// Resolves the effective [`CachePolicy`] for `request`: the per-request diff --git a/crates/tinyagents-harness/src/agent_loop/run_loop_dead_call_tests.rs b/crates/tinyagents-harness/src/agent_loop/run_loop_dead_call_tests.rs index 5e821f3b6..02defe23f 100644 --- a/crates/tinyagents-harness/src/agent_loop/run_loop_dead_call_tests.rs +++ b/crates/tinyagents-harness/src/agent_loop/run_loop_dead_call_tests.rs @@ -1,9 +1,9 @@ //! The call after a dead one (`RunPolicy::truncated_empty_reasoning_off`). +use super::*; use std::sync::Arc; -use crate::context::{RunConfig, RunContext}; -use crate::runtime::{AgentHarness, RunPolicy}; +use crate::runtime::RunPolicy; use crate::testkit::ScriptedModel; use tinyinference_llm::message::Message; use tinyinference_llm::model::{ModelResponse, ReasoningConfig, ReasoningEffort}; @@ -125,6 +125,87 @@ async fn reasoning_off_sticks_for_the_rest_of_the_run_after_repeated_dead_calls( ); } +#[tokio::test] +async fn constant_cap_retries_reach_the_provider_instead_of_replaying_cached_blanks() { + let model = Arc::new(ScriptedModel::new(vec![ + dead(2048), + crate::testkit::text_response("recovered"), + ])); + let mut harness: AgentHarness<()> = AgentHarness::new(); + harness.register_model("mock", model.clone()); + harness.with_response_cache(Arc::new(crate::cache::InMemoryResponseCache::new())); + harness.with_policy(RunPolicy { + truncated_empty_constant_cap: true, + truncated_empty_retries: 1, + truncated_empty_nudges: 0, + ..RunPolicy::default() + }); + let input = vec![Message::user("same question")]; + for run_id in ["recover", "reuse-answer"] { + let ctx = RunContext::new(RunConfig::new(run_id).with_max_turn_output_tokens(2048), ()); + let run = harness + .invoke_in_context(&(), ctx, input.clone()) + .await + .unwrap(); + assert_eq!(run.text(), Some("recovered".into())); + } + assert_eq!( + model.requests().len(), + 2, + "only the usable answer is cached" + ); +} + +#[tokio::test] +async fn truncated_empty_recovery_bypasses_a_blank_cached_before_retries_were_enabled() { + let cache = Arc::new(crate::cache::InMemoryResponseCache::new()); + let input = vec![Message::user("same question")]; + let seed_model = Arc::new(ScriptedModel::new(vec![dead(2048)])); + let mut seed: AgentHarness<()> = AgentHarness::new(); + seed.register_model("mock", seed_model.clone()); + seed.with_response_cache(cache.clone()); + seed.with_policy(RunPolicy { + truncated_empty_retries: 0, + truncated_empty_nudges: 0, + ..RunPolicy::default() + }); + for run_id in ["seed-blank", "replay-with-recovery-disabled"] { + let ctx = RunContext::new(RunConfig::new(run_id).with_max_turn_output_tokens(2048), ()); + seed.invoke_in_context(&(), ctx, input.clone()) + .await + .unwrap(); + } + assert_eq!( + seed_model.requests().len(), + 1, + "disabled recovery retains exact replay" + ); + + let model = Arc::new(ScriptedModel::new(vec![crate::testkit::text_response( + "fresh answer", + )])); + let mut harness: AgentHarness<()> = AgentHarness::new(); + harness.register_model("mock", model.clone()); + harness.with_response_cache(cache); + harness.with_policy(RunPolicy { + truncated_empty_constant_cap: true, + truncated_empty_retries: 1, + truncated_empty_nudges: 0, + ..RunPolicy::default() + }); + let ctx = RunContext::new( + RunConfig::new("recover-cached-blank").with_max_turn_output_tokens(2048), + (), + ); + let run = harness.invoke_in_context(&(), ctx, input).await.unwrap(); + assert_eq!(run.text(), Some("fresh answer".into())); + assert_eq!( + model.requests().len(), + 1, + "the prior blank cache hit is bypassed" + ); +} + /// Records, for each tool call, whether it came from a dead-call retry. #[derive(Default)] struct RetryFlags(std::sync::Mutex>); diff --git a/crates/tinyagents-registry/src/pricing/README.md b/crates/tinyagents-registry/src/pricing/README.md index 5c90d8f22..2b7c0f46b 100644 --- a/crates/tinyagents-registry/src/pricing/README.md +++ b/crates/tinyagents-registry/src/pricing/README.md @@ -15,9 +15,15 @@ This module performs no I/O and does not refresh prices. The seed catalog and its validation remain independent; this additive API preserves the migrated consumer snapshot without silently changing existing seed lookups. -Inherited compatibility limitations remain: unlisted hyphenated variants can -match a base id; some cached rates are historical assumptions; long-context -pricing other than the MiniMax M3 estimate branch is unmodeled. The flat -metadata projection deliberately preserves row rates with no tiers, so it is -not equivalent to the tier-aware M3 call estimator. These need separate -behavioral corrections rather than silent changes in this migration. +Lookup first matches an exact canonical id, then an id with a leading vendor +segment removed. Its fallback chooses the longest listed id extended only by +date/version fragments or release-channel words (`latest`, `preview`, `exp`), +such as `gpt-5.4-mini-2026-05-01` or +`gemini-2.5-pro-preview-06-05`. Unlisted variants such as `gpt-4.1-nano` and +`o3-pro` return `None`; they do not inherit the rates or context window of a +listed base model. + +Some cached rates remain historical assumptions, and long-context pricing other +than the MiniMax M3 estimate branch is unmodeled. The flat metadata projection +preserves row rates with no tiers, so it is not equivalent to the tier-aware M3 +call estimator. Those limitations need separate behavioral corrections. diff --git a/integration-proofs/run-ledger-mongodb/Cargo.lock b/integration-proofs/run-ledger-mongodb/Cargo.lock index f055059aa..0cb6a3eb2 100644 --- a/integration-proofs/run-ledger-mongodb/Cargo.lock +++ b/integration-proofs/run-ledger-mongodb/Cargo.lock @@ -2324,7 +2324,7 @@ dependencies = [ [[package]] name = "tinyagents-definition" -version = "2.2.0" +version = "2.11.0" dependencies = [ "anyhow", "async-trait", @@ -2337,7 +2337,7 @@ dependencies = [ [[package]] name = "tinyagents-harness" -version = "2.2.0" +version = "2.11.0" dependencies = [ "anyhow", "async-trait", @@ -2387,7 +2387,7 @@ dependencies = [ [[package]] name = "tinyagents-session" -version = "2.2.0" +version = "2.11.0" dependencies = [ "anyhow", "async-trait", @@ -2411,7 +2411,7 @@ dependencies = [ [[package]] name = "tinyagents-storage-kit" -version = "2.2.0" +version = "2.11.0" dependencies = [ "data-encoding", "serde", @@ -2425,7 +2425,7 @@ dependencies = [ [[package]] name = "tinyagents-tasks" -version = "2.2.0" +version = "2.11.0" dependencies = [ "serde", "serde_json",