From 1ed40853bf8862d17dc6c5e90d2941a7f821effa Mon Sep 17 00:00:00 2001 From: Waldemort Agent Date: Fri, 9 Oct 2026 02:22:00 +0000 Subject: [PATCH 1/9] replay: cancel dropped queue reads with execution-pinned policy Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: Affan Dar <1480118+affandar@users.noreply.github.com> --- Cargo.toml | 2 +- src/lib.rs | 37 +++++++++++++++++++++++++++++++++++- src/runtime/replay_engine.rs | 20 +++++++++++++++++++ src/runtime/state_helpers.rs | 6 +++--- 4 files changed, 60 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e9b0b7d..e4c7692 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "duroxide" -version = "0.1.30" +version = "0.1.31" edition = "2024" description = "Durable code execution framework for Rust" license = "MIT" diff --git a/src/lib.rs b/src/lib.rs index 42af154..7973eff 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1634,6 +1634,12 @@ pub enum CompletionResult { ExternalData(String), } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum RaceCancellationPolicy { + Legacy, + V0131, +} + #[derive(Debug)] struct CtxInner { /// Whether we're currently replaying history (true) or processing new events (false). @@ -1690,6 +1696,8 @@ struct CtxInner { cancelled_tokens: std::collections::HashSet, /// Cancelled token -> ScheduleKind mapping (for determining cancellation action) cancelled_token_kinds: std::collections::HashMap, + /// Shared execution-pinned queue, positional-wait, and CAN cancellation policy. + race_cancellation_policy: RaceCancellationPolicy, // Execution metadata execution_id: u64, @@ -1752,6 +1760,7 @@ impl CtxInner { // Cancellation tracking cancelled_tokens: Default::default(), cancelled_token_kinds: Default::default(), + race_cancellation_policy: RaceCancellationPolicy::Legacy, // Execution metadata execution_id, @@ -1792,6 +1801,14 @@ impl CtxInner { /// Bind a token to a schedule_id (called by replay engine when matching action to history). fn bind_token(&mut self, token: u64, schedule_id: u64) { self.token_bindings.insert(token, schedule_id); + if self.race_cancellation_policy == RaceCancellationPolicy::V0131 + && matches!( + self.cancelled_token_kinds.get(&token), + Some(ScheduleKind::QueueDequeue { .. }) + ) + { + self.mark_queue_subscription_cancelled(schedule_id); + } } /// Get the schedule_id bound to a token (returns None if not yet bound). @@ -1929,7 +1946,9 @@ impl CtxInner { // Our arrival index is exactly the number of active (non-cancelled) // subscriptions for this name that were created before us. - // This guarantees strict FIFO matching regardless of poll order. + // A held older subscription still reserves its position. Cancelling it + // after a newer one resolved can reindex later reads; that pattern is a + // separate known limitation of this matching model. let arrival_index: usize = self .queue_subscriptions .iter() @@ -1960,6 +1979,7 @@ impl CtxInner { ids.push(schedule_id); } } + ids.sort_unstable(); ids } @@ -1992,6 +2012,14 @@ impl CtxInner { /// Mark a token as cancelled (called by DurableFuture::drop). fn mark_token_cancelled(&mut self, token: u64, kind: ScheduleKind) { + // Matching must honor a dropped queue or positional wait before the + // next poll, not only when its later history breadcrumb is replayed. + if self.race_cancellation_policy == RaceCancellationPolicy::V0131 + && let Some(schedule_id) = self.get_bound_schedule_id(token) + && matches!(kind, ScheduleKind::QueueDequeue { .. }) + { + self.mark_queue_subscription_cancelled(schedule_id); + } self.cancelled_tokens.insert(token); self.cancelled_token_kinds.insert(token, kind); } @@ -2007,6 +2035,7 @@ impl CtxInner { ids.push(schedule_id); } } + ids.sort_unstable(); ids } @@ -2021,6 +2050,7 @@ impl CtxInner { ids.push(schedule_id); } } + ids.sort_unstable(); ids } @@ -2041,6 +2071,7 @@ impl CtxInner { // If not in mapping, the action wasn't bound yet - nothing to cancel } } + cancels.sort_by_key(|(id, _)| *id); cancels } @@ -2500,6 +2531,10 @@ impl OrchestrationContext { self.inner.lock().unwrap().mark_token_cancelled(token, kind.clone()); } + pub(crate) fn set_race_cancellation_policy(&self, policy: RaceCancellationPolicy) { + self.inner.lock().unwrap().race_cancellation_policy = policy; + } + /// Get cancelled activity schedule_ids for this turn. pub(crate) fn get_cancelled_activity_ids(&self) -> Vec { self.inner.lock().unwrap().get_cancelled_activity_ids() diff --git a/src/runtime/replay_engine.rs b/src/runtime/replay_engine.rs index 388a19e..d2b813f 100644 --- a/src/runtime/replay_engine.rs +++ b/src/runtime/replay_engine.rs @@ -16,6 +16,9 @@ use std::sync::Arc; use std::task::{Context, Poll}; use tracing::{debug, warn}; +/// This version must not be released without the queue, positional and CAN cutover. +pub(crate) const RACE_CANCELLATION_FIX_SINCE: semver::Version = semver::Version::new(0, 1, 31); + /// Result of executing an orchestration turn #[derive(Debug)] pub enum TurnResult { @@ -546,6 +549,23 @@ impl ReplayEngine { Some(worker_id.to_string()), ); + // This must be pinned for the whole execution, including its new turns. + // Applying the fix to old decisions can change a recorded race winner. + let race_cancellation_policy = match semver::Version::parse(&working_history[0].duroxide_version) { + Ok(version) if version >= RACE_CANCELLATION_FIX_SINCE => crate::RaceCancellationPolicy::V0131, + Ok(_) => crate::RaceCancellationPolicy::Legacy, + Err(error) => { + warn!( + instance = %self.instance, + version = %working_history[0].duroxide_version, + %error, + "Invalid pinned version; retaining legacy race cancellation semantics" + ); + crate::RaceCancellationPolicy::Legacy + } + }; + ctx.set_race_cancellation_policy(race_cancellation_policy); + // Seed KV state from provider snapshot before orchestration code runs. if !self.kv_snapshot.is_empty() { let mut inner = ctx.inner.lock().expect("Mutex should not be poisoned"); diff --git a/src/runtime/state_helpers.rs b/src/runtime/state_helpers.rs index 2be7da1..4704d74 100644 --- a/src/runtime/state_helpers.rs +++ b/src/runtime/state_helpers.rs @@ -307,8 +307,8 @@ impl HistoryManager { /// terminated, we delete the worker queue entries for any in-flight activities to signal /// to the worker that the activity has been cancelled. pub fn compute_inflight_activities(&self, instance: &str, execution_id: u64) -> Vec { - // Collect all scheduled activity event IDs - let scheduled: std::collections::HashSet = self + // Collect all scheduled activity event IDs (ordered, so cancellations are deterministic) + let scheduled: std::collections::BTreeSet = self .full_history_iter() .filter_map(|e| { if matches!(&e.kind, EventKind::ActivityScheduled { .. }) { @@ -320,7 +320,7 @@ impl HistoryManager { .collect(); // Collect all completed/failed activity source_event_ids (the ActivityScheduled event_id they reference) - let completed: std::collections::HashSet = self + let completed: std::collections::BTreeSet = self .full_history_iter() .filter_map(|e| { if matches!( From 7d986e071e3245be76850095c19d78d02061937e Mon Sep 17 00:00:00 2001 From: Waldemort Agent Date: Fri, 9 Oct 2026 02:22:02 +0000 Subject: [PATCH 2/9] replay: preserve bound-slot signal admission and cancellation Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: Affan Dar <1480118+affandar@users.noreply.github.com> --- src/lib.rs | 46 +++++++++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 7973eff..17d8b20 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1871,10 +1871,17 @@ impl CtxInner { /// Uses compressed (effective) indices that skip cancelled subscriptions: /// effective_index = raw_index - count_of_cancelled_subscriptions_with_lower_raw_index /// - /// Stale events are prevented from entering the arrival list by the causal - /// check in the replay loop (`has_pending_subscription_slot`), so the - /// arrival array only contains events that have a matching active slot. + /// The replay loop admits signals only for bound active slots. + /// New-policy cancellation also removes + /// an accepted value assigned to a dropped, unresolved slot. fn get_external_event(&self, schedule_id: u64) -> Option<&String> { + let (name, _) = self.external_subscriptions.get(&schedule_id)?; + let effective_index = self.external_arrival_index(schedule_id)?; + let arrivals = self.external_arrivals.get(name)?; + arrivals.get(effective_index) + } + + fn external_arrival_index(&self, schedule_id: u64) -> Option { let (name, subscription_index) = self.external_subscriptions.get(&schedule_id)?; // If this subscription is cancelled, it never resolves @@ -1885,23 +1892,29 @@ impl CtxInner { // Count cancelled subscriptions for this name with a lower raw index let cancelled_below = self .external_subscriptions - .values() - .filter(|(n, idx)| n == name && *idx < *subscription_index) - .filter(|(_, idx)| { - // Check if this subscription's schedule_id is in the cancelled set - self.external_subscriptions - .iter() - .any(|(sid, (n, i))| n == name && i == idx && self.external_cancelled_subscriptions.contains(sid)) + .iter() + .filter(|(sid, (n, idx))| { + n == name && *idx < *subscription_index && self.external_cancelled_subscriptions.contains(sid) }) .count(); - let effective_index = subscription_index - cancelled_below; - let arrivals = self.external_arrivals.get(name)?; - arrivals.get(effective_index) + Some(subscription_index - cancelled_below) } /// Mark an external subscription as cancelled. fn mark_external_subscription_cancelled(&mut self, schedule_id: u64) { + // Positional signals answer a specific active slot. If its future is + // dropped before observing an already-delivered value, discard that stale + // value instead of shifting it onto the replacement wait. Queue arrivals + // intentionally have different, persistent semantics. + if self.race_cancellation_policy == RaceCancellationPolicy::V0131 + && let Some(index) = self.external_arrival_index(schedule_id) + && let Some((name, _)) = self.external_subscriptions.get(&schedule_id) + && let Some(arrivals) = self.external_arrivals.get_mut(name) + && index < arrivals.len() + { + arrivals.remove(index); + } self.external_cancelled_subscriptions.insert(schedule_id); } @@ -2016,9 +2029,12 @@ impl CtxInner { // next poll, not only when its later history breadcrumb is replayed. if self.race_cancellation_policy == RaceCancellationPolicy::V0131 && let Some(schedule_id) = self.get_bound_schedule_id(token) - && matches!(kind, ScheduleKind::QueueDequeue { .. }) { - self.mark_queue_subscription_cancelled(schedule_id); + match &kind { + ScheduleKind::QueueDequeue { .. } => self.mark_queue_subscription_cancelled(schedule_id), + ScheduleKind::ExternalWait { .. } => self.mark_external_subscription_cancelled(schedule_id), + _ => {} + } } self.cancelled_tokens.insert(token); self.cancelled_token_kinds.insert(token, kind); From 632065da2935d3310059d7372094e8d4947b6f7e Mon Sep 17 00:00:00 2001 From: Waldemort Agent Date: Fri, 9 Oct 2026 02:22:03 +0000 Subject: [PATCH 3/9] continue-as-new: carry exactly the unread queue positions Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: Affan Dar <1480118+affandar@users.noreply.github.com> --- src/lib.rs | 33 ++++++++++++++++++++++++++++++++- src/runtime/execution.rs | 10 ++++++++-- src/runtime/replay_engine.rs | 14 +++++++++++++- 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 17d8b20..4d8adc1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1672,6 +1672,10 @@ struct CtxInner { queue_subscriptions: Vec<(u64, String)>, /// Persistent arrivals: name -> list of payloads in arrival order queue_arrivals: std::collections::HashMap>, + /// History arrival order across names as (name, per-name position), plus the + /// per-name positions actually returned. Payloads stay in `queue_arrivals`. + queue_arrival_order: Vec<(String, usize)>, + queue_consumed_positions: std::collections::HashSet<(String, usize)>, /// Persistent subscriptions that have been cancelled (dropped without completing) queue_cancelled_subscriptions: std::collections::HashSet, /// Persistent subscriptions that have already been resolved (consumed an arrival) @@ -1747,6 +1751,8 @@ impl CtxInner { external_cancelled_subscriptions: Default::default(), queue_subscriptions: Default::default(), queue_arrivals: Default::default(), + queue_arrival_order: Default::default(), + queue_consumed_positions: Default::default(), queue_cancelled_subscriptions: Default::default(), queue_resolved_subscriptions: Default::default(), #[cfg(feature = "replay-version-test")] @@ -1927,7 +1933,9 @@ impl CtxInner { /// Deliver a persistent external event (appends to arrival list for the name). fn deliver_queue_message(&mut self, name: String, data: String) { - self.queue_arrivals.entry(name).or_default().push(data); + let arrivals = self.queue_arrivals.entry(name.clone()).or_default(); + self.queue_arrival_order.push((name, arrivals.len())); + arrivals.push(data); } /// Mark a persistent subscription as cancelled (dropped without completing). @@ -1975,6 +1983,7 @@ impl CtxInner { if arrival_index < arrivals.len() { // Mark as resolved self.queue_resolved_subscriptions.insert(schedule_id); + self.queue_consumed_positions.insert((name, arrival_index)); Some(arrivals[arrival_index].clone()) } else { None @@ -2551,6 +2560,28 @@ impl OrchestrationContext { self.inner.lock().unwrap().race_cancellation_policy = policy; } + pub(crate) fn race_cancellation_fix_enabled(&self) -> bool { + self.inner.lock().unwrap().race_cancellation_policy == RaceCancellationPolicy::V0131 + } + + pub(crate) fn unconsumed_queue_arrivals(&self) -> Vec<(String, String)> { + let inner = self.inner.lock().unwrap(); + inner + .queue_arrival_order + .iter() + .filter(|position| !inner.queue_consumed_positions.contains(*position)) + .map(|(name, index)| { + // Arrival vectors are append-only, so a recorded position always exists. + let data = inner + .queue_arrivals + .get(name) + .and_then(|arrivals| arrivals.get(*index)) + .expect("queue arrival order must reference a recorded arrival"); + (name.clone(), data.clone()) + }) + .collect() + } + /// Get cancelled activity schedule_ids for this turn. pub(crate) fn get_cancelled_activity_ids(&self) -> Vec { self.inner.lock().unwrap().get_cancelled_activity_ids() diff --git a/src/runtime/execution.rs b/src/runtime/execution.rs index 8522570..d097d56 100644 --- a/src/runtime/execution.rs +++ b/src/runtime/execution.rs @@ -297,11 +297,17 @@ impl Runtime { Err(details.display_message()) } - TurnResult::ContinueAsNew { input, version } => { + TurnResult::ContinueAsNew { + input, + version, + unconsumed_queue_arrivals, + } => { // Carry forward unmatched persistent events to the new execution. // MUST be called BEFORE emit_terminal_cancellation_breadcrumbs, which appends // QueueSubscriptionCancelled events that would confuse the matching. - let mut unmatched = Self::collect_unmatched_queue_messages(history_mgr.full_history_iter(), instance); + let mut unmatched = unconsumed_queue_arrivals.unwrap_or_else(|| { + Self::collect_unmatched_queue_messages(history_mgr.full_history_iter(), instance) + }); // Emit cancellation breadcrumbs BEFORE the terminal event. // Terminal events (Completed/Failed/CAN) must be the last event in history. diff --git a/src/runtime/replay_engine.rs b/src/runtime/replay_engine.rs index d2b813f..66e8e56 100644 --- a/src/runtime/replay_engine.rs +++ b/src/runtime/replay_engine.rs @@ -29,7 +29,12 @@ pub enum TurnResult { /// Orchestration failed with error details Failed(crate::ErrorDetails), /// Orchestration requested continue-as-new - ContinueAsNew { input: String, version: Option }, + ContinueAsNew { + input: String, + version: Option, + /// New policy carries exact unread positions; None preserves legacy history counting. + unconsumed_queue_arrivals: Option>, + }, /// Orchestration was cancelled Cancelled(String), } @@ -662,6 +667,8 @@ impl ReplayEngine { } { + // Legacy turns apply dropped queue cancellation here. In immediate + // mode these IDs were already marked at drop/bind; repeating is idempotent. let cancelled_queue_waits = ctx.get_cancelled_queue_ids(); for schedule_id in cancelled_queue_waits { ctx.mark_queue_subscription_cancelled(schedule_id); @@ -1056,6 +1063,11 @@ impl ReplayEngine { return TurnResult::ContinueAsNew { input: input.clone(), version: version.clone(), + // New-policy executions carry exactly the unread arrivals; legacy + // executions keep counting subscriptions from history. + unconsumed_queue_arrivals: ctx + .race_cancellation_fix_enabled() + .then(|| ctx.unconsumed_queue_arrivals()), }; } } From 615b59d2fae5d0ea9118fbb1b7bc2de8d95865bb Mon Sep 17 00:00:00 2001 From: Waldemort Agent Date: Fri, 9 Oct 2026 02:22:04 +0000 Subject: [PATCH 4/9] dispatcher: preserve CAN admission and terminal input handling Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: Affan Dar <1480118+affandar@users.noreply.github.com> --- src/runtime/dispatchers/orchestration.rs | 84 ++++++++++++++++++++---- src/runtime/dispatchers/worker.rs | 17 ++++- src/runtime/mod.rs | 8 +++ src/runtime/state_helpers.rs | 8 +++ 4 files changed, 102 insertions(+), 15 deletions(-) diff --git a/src/runtime/dispatchers/orchestration.rs b/src/runtime/dispatchers/orchestration.rs index 0f11478..1360aa5 100644 --- a/src/runtime/dispatchers/orchestration.rs +++ b/src/runtime/dispatchers/orchestration.rs @@ -569,7 +569,7 @@ impl Runtime { // version capable of deserialising these events) a window to pick it up, // without the exponential growth that would slow down the poison path. let backoff = Duration::from_secs(1); - let remaining = self.options.max_attempts.saturating_sub(attempt_count); + let remaining = self.options.attempts_until_poison(attempt_count); warn!( instance = %instance, error = %error, @@ -589,19 +589,50 @@ impl Runtime { // Extract metadata from history and work items let temp_history_mgr = HistoryManager::from_history(&item.history); let workitem_reader = WorkItemReader::from_messages(&item.messages, &temp_history_mgr, instance); + // The poison path records a CAN successor's failure after the CAN row. + let terminal_after_can = temp_history_mgr.is_continued_as_new + && item + .history + .iter() + .any(|event| matches!(event.kind, EventKind::OrchestrationFailed { .. })); + + // Between executions nothing can run until the successor's start arrives. The start + // is committed with the CAN row, but this batch can miss it: the fetch cutoff preceded + // its visible_at, or a delayed abandon pushed it (for example an old worker without the + // successor's version). Keep every input and retry with the backoff and attempt budget + // of an unregistered handler. Successor replay drops stale completions; order against + // later arrivals is best effort, as with any delayed abandon. + if temp_history_mgr.is_continued_as_new && !terminal_after_can && !workitem_reader.is_continue_as_new { + let backoff = self.options.unregistered_backoff.delay(attempt_count); + let remaining_attempts = self.options.attempts_until_poison(attempt_count); + warn!( + instance = %instance, + attempt_count, + max_attempts = self.options.max_attempts, + remaining_attempts, + backoff_secs = %backoff.as_secs_f32(), + messages = item.messages.len(), + "CAN successor start not visible, abandoning instance inputs with {:.1}s backoff (will poison in {} more attempts)", + backoff.as_secs_f32(), + remaining_attempts + ); + // If the abandon fails, the lock expires and the inputs are redelivered. + let _ = self + .history_store + .abandon_orchestration_item(lock_token, Some(backoff), false) + .await; + return; + } - // Bail on truly terminal histories (Completed/Failed), or ContinuedAsNew without a CAN start message - if temp_history_mgr.is_completed - || temp_history_mgr.is_failed - || (temp_history_mgr.is_continued_as_new && !workitem_reader.is_continue_as_new) - { - warn!(instance = %instance, "Instance is terminal (completed/failed or CAN without start), acking batch without processing"); + // Bail on truly terminal histories (Completed/Failed) + if temp_history_mgr.is_completed || temp_history_mgr.is_failed || terminal_after_can { + warn!(instance = %instance, "Instance is terminal (completed/failed), acking batch without processing"); // If this terminal instance receives a StartOrchestration for the same child id, // the incoming start is discarded here. When that start belongs to a different // scheduling parent than the terminal child originally recorded, notify the // incoming parent with SubOrchFailed so it does not wait forever for a child // start that was never accepted. - let orchestrator_items = self.terminal_collision_notifications(&item).await; + let orchestrator_items = self.terminal_collision_notifications(&item, &temp_history_mgr).await; let _ = self .ack_orchestration_with_changes( lock_token, @@ -618,6 +649,11 @@ impl Runtime { // Extract version before moving temp_history_mgr let version = temp_history_mgr.version().unwrap_or_else(|| "unknown".to_string()); + let collision_notifications = if !item.history.is_empty() { + self.terminal_collision_notifications(&item, &temp_history_mgr).await + } else { + Vec::new() + }; // Decide execution id and history to use for this execution let (execution_id_to_use, mut history_mgr) = if workitem_reader.is_continue_as_new { @@ -685,7 +721,7 @@ impl Runtime { Err(OrchestrationProcessingError::UnregisteredOrchestration) => { // Orchestration not registered - abandon with exponential backoff let backoff = self.options.unregistered_backoff.delay(attempt_count); - let remaining_attempts = self.options.max_attempts.saturating_sub(attempt_count); + let remaining_attempts = self.options.attempts_until_poison(attempt_count); tracing::warn!( target: "duroxide::runtime", @@ -947,6 +983,7 @@ impl Runtime { } // Robust ack with basic retry on any provider error + orchestrator_items.extend(collision_notifications.iter().cloned()); match self .ack_orchestration_with_changes( lock_token, @@ -993,7 +1030,7 @@ impl Runtime { execution_id_for_ack, failure_delta.clone(), vec![], - vec![], + collision_notifications, failure_metadata, vec![], // cancelled_activities - none for failure commits ) @@ -1442,8 +1479,14 @@ impl Runtime { /// the provider contract requires `ack_orchestration_item` to delete the batch, append /// history, and enqueue outbound work in a single transaction. A child's start is /// therefore consumed exactly once, before the child can reach a terminal state. - async fn terminal_collision_notifications(&self, item: &crate::providers::OrchestrationItem) -> Vec { + /// `history` is the manager already built from `item.history` for this batch. + async fn terminal_collision_notifications( + &self, + item: &crate::providers::OrchestrationItem, + history: &HistoryManager, + ) -> Vec { let mut notifications = Vec::new(); + let terminal = history.is_completed || history.is_failed; for msg in &item.messages { if let WorkItem::StartOrchestration { parent_instance: Some(parent_instance), @@ -1452,11 +1495,24 @@ impl Runtime { .. } = msg { + // Defensive: a parent enqueues a child's start once, in its atomic ack, so a + // correct provider never redelivers it to the running child. If one does, + // keep the base behaviour and ignore it. An unknown generation cannot prove + // it is the same call, so that case is notified. + if !terminal + && history.parent_instance.as_ref() == Some(parent_instance) + && history.parent_id == Some(*parent_id) + && parent_execution_id.is_some() + && history.parent_execution_id == *parent_execution_id + { + continue; + } + let state = if terminal { " and is terminal" } else { "" }; warn!( instance = %item.instance, parent_instance = %parent_instance, parent_id = %parent_id, - "Sub-orchestration target instance id already exists and is terminal; notifying parent of failure" + "Sub-orchestration target instance id already exists; notifying colliding parent of failure" ); // Prefer the execution id stamped on the colliding start; fall back to a // durable provider read for work items produced by older runtimes. @@ -1470,8 +1526,8 @@ impl Runtime { details: crate::ErrorDetails::Application { kind: crate::AppErrorKind::OrchestrationFailed, message: format!( - "sub-orchestration instance id '{}' already exists and is terminal", - item.instance + "sub-orchestration instance id '{}' already exists{state}", + item.instance, ), retryable: false, }, diff --git a/src/runtime/dispatchers/worker.rs b/src/runtime/dispatchers/worker.rs index 8a2cdb5..9937750 100644 --- a/src/runtime/dispatchers/worker.rs +++ b/src/runtime/dispatchers/worker.rs @@ -702,7 +702,7 @@ async fn handle_activity_error( /// The poison message handling will eventually fail the activity if genuinely missing. async fn abandon_unregistered_activity(rt: &Arc, ctx: &ActivityWorkContext) { let backoff = rt.options.unregistered_backoff.delay(ctx.attempt_count); - let remaining_attempts = rt.options.max_attempts.saturating_sub(ctx.attempt_count); + let remaining_attempts = rt.options.attempts_until_poison(ctx.attempt_count); tracing::warn!( target: "duroxide::runtime", @@ -967,6 +967,21 @@ async fn run_session_manager(rt: Arc, shutdown: Arc u64 { + (u64::from(self.max_attempts) + 1).saturating_sub(u64::from(attempt_count)) + } +} + mod dispatchers; pub mod limits; pub mod observability; diff --git a/src/runtime/state_helpers.rs b/src/runtime/state_helpers.rs index 4704d74..b5fe48a 100644 --- a/src/runtime/state_helpers.rs +++ b/src/runtime/state_helpers.rs @@ -417,6 +417,14 @@ impl WorkItemReader { // Separate start/CAN from completions for work_item in messages { match work_item { + // A start for an instance with history cannot replace its pinned handler, + // and cannot take the place of the instance's CAN start. + WorkItem::StartOrchestration { .. } if !history_mgr.is_empty() => { + warn!( + instance, + "Ignoring duplicate StartOrchestration for an existing instance" + ); + } WorkItem::StartOrchestration { .. } | WorkItem::ContinueAsNew { .. } => { if start_item.is_some() { warn!(instance, "Duplicate Start/ContinueAsNew in batch - ignoring duplicate"); From e39ae3b4971e5d3c50c3038bb8291ed7e29feac7 Mon Sep 17 00:00:00 2001 From: Waldemort Agent Date: Fri, 9 Oct 2026 02:22:05 +0000 Subject: [PATCH 5/9] sqlite: scope abandon attempt undo and delay to locked rows Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: Affan Dar <1480118+affandar@users.noreply.github.com> --- src/providers/sqlite.rs | 48 ++++++++++++++++------------------------- 1 file changed, 19 insertions(+), 29 deletions(-) diff --git a/src/providers/sqlite.rs b/src/providers/sqlite.rs index 0c74856..4240f38 100644 --- a/src/providers/sqlite.rs +++ b/src/providers/sqlite.rs @@ -2216,47 +2216,37 @@ impl Provider for SqliteProvider { .await .map_err(|e| Self::sqlx_to_provider_error("abandon_orchestration_item", e))?; - // Get instance_id from lock before removing it - let instance_id: Option = - sqlx::query_scalar("SELECT instance_id FROM instance_locks WHERE lock_token = ?") - .bind(lock_token) - .fetch_optional(&mut *tx) - .await - .map_err(|e| Self::sqlx_to_provider_error("abandon_orchestration_item", e))?; - - let Some(instance_id) = instance_id else { - return Err(ProviderError::permanent( - "abandon_orchestration_item", - "Invalid lock token", - )); - }; - - // Remove instance lock - sqlx::query("DELETE FROM instance_locks WHERE lock_token = ?") + // Remove the instance lock; an unknown token is rejected + let removed = sqlx::query("DELETE FROM instance_locks WHERE lock_token = ?") .bind(lock_token) .execute(&mut *tx) .await .map_err(|e| Self::sqlx_to_provider_error("abandon_orchestration_item", e))?; + if removed.rows_affected() == 0 { + return Err(ProviderError::permanent( + "abandon_orchestration_item", + "Invalid lock token", + )); + } - // If ignore_attempt is true, decrement attempt_count on orchestrator_queue messages + // Restore only attempts incremented by this fetch, not hidden starts or + // other instance rows that were never part of the locked batch. if ignore_attempt { - sqlx::query( - "UPDATE orchestrator_queue SET attempt_count = MAX(0, attempt_count - 1) WHERE instance_id = ?", - ) - .bind(&instance_id) - .execute(&mut *tx) - .await - .map_err(|e| Self::sqlx_to_provider_error("abandon_orchestration_item", e))?; + sqlx::query("UPDATE orchestrator_queue SET attempt_count = MAX(0, attempt_count - 1) WHERE lock_token = ?") + .bind(lock_token) + .execute(&mut *tx) + .await + .map_err(|e| Self::sqlx_to_provider_error("abandon_orchestration_item", e))?; } - // Optionally delay messages for this instance + // Optionally delay only the messages this fetch locked. Rows that became visible + // after the fetch (for example a continue-as-new start) keep their visibility. if let Some(delay) = delay { let delay_ms = delay.as_millis().min(i64::MAX as u128) as i64; let visible_at = Self::now_millis().saturating_add(delay_ms); - sqlx::query("UPDATE orchestrator_queue SET visible_at = ? WHERE instance_id = ? AND visible_at <= ?") + sqlx::query("UPDATE orchestrator_queue SET visible_at = ? WHERE lock_token = ?") .bind(visible_at) - .bind(&instance_id) - .bind(Self::now_millis()) + .bind(lock_token) .execute(&mut *tx) .await .map_err(|e| Self::sqlx_to_provider_error("abandon_orchestration_item", e))?; From f60783896a6e5bf4061ce96bbdd026d6065215f4 Mon Sep 17 00:00:00 2001 From: Waldemort Agent Date: Fri, 9 Oct 2026 02:22:08 +0000 Subject: [PATCH 6/9] test: cover queue signal and CAN replay boundaries Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: Affan Dar <1480118+affandar@users.noreply.github.com> --- Cargo.toml | 2 + src/lib.rs | 209 + src/provider_validation/mod.rs | 2 + src/provider_validation/poison_message.rs | 200 + src/provider_validation/race_replay.rs | 1547 ++++ src/provider_validations.rs | 3 + src/runtime/state_helpers.rs | 141 + tests/can_abandon_failure.rs | 646 ++ tests/cancellation_corner_cases.rs | 3024 ++++++++ tests/common/cancellation_history.rs | 57 + tests/common/fault_injection.rs | 120 +- tests/fixtures/queue_race_old_engine.json | 6903 +++++++++++++++++ tests/queue_event_tests.rs | 175 + tests/queue_race_upgrade.rs | 678 ++ tests/replay_engine/helpers.rs | 8 +- tests/replay_engine/mod.rs | 2 + .../replay_engine/positional_unbound_races.rs | 318 + tests/replay_engine/queue_races.rs | 146 + tests/runtime_version_warnings.rs | 122 + tests/sqlite_provider_validations.rs | 78 + 20 files changed, 14376 insertions(+), 5 deletions(-) create mode 100644 src/provider_validation/race_replay.rs create mode 100644 tests/can_abandon_failure.rs create mode 100644 tests/cancellation_corner_cases.rs create mode 100644 tests/common/cancellation_history.rs create mode 100644 tests/fixtures/queue_race_old_engine.json create mode 100644 tests/queue_race_upgrade.rs create mode 100644 tests/replay_engine/positional_unbound_races.rs create mode 100644 tests/replay_engine/queue_races.rs create mode 100644 tests/runtime_version_warnings.rs diff --git a/Cargo.toml b/Cargo.toml index e4c7692..ae0b848 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ repository = "https://github.com/microsoft/duroxide" homepage = "https://github.com/microsoft/duroxide" documentation = "https://docs.rs/duroxide" readme = "README.md" +exclude = ["tests/fixtures/**", "tests/queue_race_upgrade.rs"] keywords = ["durable", "workflow", "orchestration", "async", "tasks"] categories = ["asynchronous", "concurrency"] @@ -41,6 +42,7 @@ sqlx = { version = "0.8", features = ["runtime-tokio-native-tls", "sqlite", "mac libsqlite3-sys = { version = ">=0.28", features = ["bundled"], optional = true } [dev-dependencies] +sha2 = "0.10" # Used in tests/examples tempfile = "3" proptest = "1.4" diff --git a/src/lib.rs b/src/lib.rs index 4d8adc1..9ca884b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4144,6 +4144,215 @@ impl OrchestrationContext { } } +#[cfg(test)] +mod queue_cancellation_tests { + use super::*; + + #[test] + fn context_defaults_to_legacy_until_the_engine_pins_a_policy() { + let ctx = OrchestrationContext::new(Vec::new(), 1, "default".into(), "Test".into(), "1.0.0".into(), None); + assert!(!ctx.race_cancellation_fix_enabled()); + } + + #[test] + fn current_build_covers_the_queue_cancellation_semantic_threshold() { + let threshold = crate::runtime::replay_engine::RACE_CANCELLATION_FIX_SINCE.clone(); + assert!(crate::providers::current_build_version() >= threshold); + } + + #[test] + fn dropping_already_resolved_dequeue_never_cancels_its_consumed_slot() { + for immediate in [false, true] { + let ctx = OrchestrationContext::new( + Vec::new(), + 1, + "resolved".into(), + "resolved".into(), + "1.0.0".into(), + None, + ); + ctx.set_race_cancellation_policy(if immediate { + RaceCancellationPolicy::V0131 + } else { + RaceCancellationPolicy::Legacy + }); + ctx.deliver_queue_message("q".into(), "first".into()); + ctx.deliver_queue_message("q".into(), "second".into()); + let mut first = ctx.dequeue_event("q"); + let token = ctx.drain_emitted_actions()[0].0; + ctx.bind_token(token, 2); + ctx.bind_queue_subscription(2, "q"); + let waker = std::task::Waker::noop(); + let mut task = std::task::Context::from_waker(waker); + assert_eq!( + std::pin::Pin::new(&mut first).poll(&mut task), + Poll::Ready("first".into()) + ); + assert!(first.completed); + drop(first); + assert!(ctx.get_cancelled_queue_ids().is_empty()); + assert!(!ctx.inner.lock().unwrap().queue_cancelled_subscriptions.contains(&2)); + let mut second = ctx.dequeue_event("q"); + let token = ctx.drain_emitted_actions()[0].0; + ctx.bind_token(token, 3); + ctx.bind_queue_subscription(3, "q"); + assert_eq!( + std::pin::Pin::new(&mut second).poll(&mut task), + Poll::Ready("second".into()) + ); + drop(second); + assert!(ctx.get_cancelled_queue_ids().is_empty()); + } + } + + #[test] + fn dropping_already_resolved_wait_never_cancels_its_slot() { + let waker = std::task::Waker::noop(); + let mut task = std::task::Context::from_waker(waker); + for immediate in [false, true] { + let ctx = OrchestrationContext::new( + Vec::new(), + 1, + "resolved".into(), + "resolved".into(), + "1.0.0".into(), + None, + ); + ctx.set_race_cancellation_policy(if immediate { + RaceCancellationPolicy::V0131 + } else { + RaceCancellationPolicy::Legacy + }); + let mut first = ctx.schedule_wait("s"); + let token = ctx.drain_emitted_actions()[0].0; + ctx.bind_token(token, 2); + ctx.bind_external_subscription(2, "s"); + ctx.deliver_external_event("s".into(), "first".into()); + assert_eq!( + std::pin::Pin::new(&mut first).poll(&mut task), + Poll::Ready("first".into()) + ); + assert!(first.completed); + drop(first); + assert!(ctx.get_cancelled_external_wait_ids().is_empty()); + assert!(!ctx.inner.lock().unwrap().external_cancelled_subscriptions.contains(&2)); + let mut second = ctx.schedule_wait("s"); + let token = ctx.drain_emitted_actions()[0].0; + ctx.bind_token(token, 3); + ctx.bind_external_subscription(3, "s"); + ctx.deliver_external_event("s".into(), "second".into()); + assert_eq!( + std::pin::Pin::new(&mut second).poll(&mut task), + Poll::Ready("second".into()), + "the resolved wait's value must stay consumed, not shift onto the next wait" + ); + } + } + + #[test] + fn dropping_an_unresolved_middle_wait_discards_only_its_own_value() { + let waker = std::task::Waker::noop(); + let mut task = std::task::Context::from_waker(waker); + let ctx = OrchestrationContext::new(Vec::new(), 1, "middle".into(), "middle".into(), "1.0.0".into(), None); + ctx.set_race_cancellation_policy(RaceCancellationPolicy::V0131); + let mut waits = Vec::new(); + for id in [2, 3, 4] { + let wait = ctx.schedule_wait("s"); + let token = ctx.drain_emitted_actions()[0].0; + ctx.bind_token(token, id); + ctx.bind_external_subscription(id, "s"); + waits.push(wait); + } + let mut last = waits.pop().expect("third wait"); + let middle = waits.pop().expect("second wait"); + let mut first = waits.pop().expect("first wait"); + ctx.deliver_external_event("s".into(), "a".into()); + ctx.deliver_external_event("s".into(), "b".into()); + assert_eq!(std::pin::Pin::new(&mut first).poll(&mut task), Poll::Ready("a".into())); + drop(first); + // "b" was accepted into the middle slot; dropping that wait discards it. + drop(middle); + assert_eq!(ctx.get_cancelled_external_wait_ids(), vec![3]); + assert_eq!(std::pin::Pin::new(&mut last).poll(&mut task), Poll::Pending); + ctx.deliver_external_event("s".into(), "c".into()); + assert_eq!(std::pin::Pin::new(&mut last).poll(&mut task), Poll::Ready("c".into())); + } + + #[test] + fn dropping_a_bound_wait_lets_the_next_bound_wait_take_a_later_signal() { + let waker = std::task::Waker::noop(); + let mut task = std::task::Context::from_waker(waker); + for immediate in [false, true] { + let ctx = OrchestrationContext::new(Vec::new(), 1, "next".into(), "next".into(), "1.0.0".into(), None); + ctx.set_race_cancellation_policy(if immediate { + RaceCancellationPolicy::V0131 + } else { + RaceCancellationPolicy::Legacy + }); + let first = ctx.schedule_wait("s"); + let token = ctx.drain_emitted_actions()[0].0; + ctx.bind_token(token, 2); + ctx.bind_external_subscription(2, "s"); + let mut second = ctx.schedule_wait("s"); + let token = ctx.drain_emitted_actions()[0].0; + ctx.bind_token(token, 3); + ctx.bind_external_subscription(3, "s"); + drop(first); + ctx.deliver_external_event("s".into(), "a".into()); + let read = std::pin::Pin::new(&mut second).poll(&mut task); + if immediate { + // "a" arrived after the first wait was dropped: it answers the open wait. + assert_eq!(read, Poll::Ready("a".into())); + } else { + // Legacy keeps the dropped wait's slot until its cancellation row. + assert_eq!(read, Poll::Pending); + } + } + } + + #[tokio::test] + async fn context_fifo_reads_characterize_delayed_and_immediate_cancellation() { + for immediate in [false, true] { + let ctx = OrchestrationContext::new(Vec::new(), 1, "fifo".into(), "fifo".into(), "1.0.0".into(), None); + ctx.set_race_cancellation_policy(if immediate { + RaceCancellationPolicy::V0131 + } else { + RaceCancellationPolicy::Legacy + }); + + let loser = ctx.dequeue_event("q"); + let token = ctx.drain_emitted_actions()[0].0; + ctx.bind_token(token, 2); + ctx.bind_queue_subscription(2, "q"); + drop(loser); + + ctx.deliver_queue_message("q".into(), "first".into()); + ctx.deliver_queue_message("q".into(), "second".into()); + let next = ctx.dequeue_event("q"); + let token = ctx.drain_emitted_actions()[0].0; + ctx.bind_token(token, 3); + ctx.bind_queue_subscription(3, "q"); + let first_read = next.await; + + // Legacy replay reaches the end-of-turn breadcrumb only after this read. + ctx.mark_queue_subscription_cancelled(2); + let following = ctx.dequeue_event("q"); + let token = ctx.drain_emitted_actions()[0].0; + ctx.bind_token(token, 4); + ctx.bind_queue_subscription(4, "q"); + let second_read = following.await; + + if immediate { + assert_eq!((first_read.as_str(), second_read.as_str()), ("first", "second")); + } else { + // The canceled loser made the first read skip index zero. Re-indexing + // cannot undo a resolved subscription, so index one is then read twice. + assert_eq!((first_read.as_str(), second_read.as_str()), ("second", "second")); + } + } + } +} + #[cfg(test)] mod system_stats_tests { use super::*; diff --git a/src/provider_validation/mod.rs b/src/provider_validation/mod.rs index 81903e3..d6af2f5 100644 --- a/src/provider_validation/mod.rs +++ b/src/provider_validation/mod.rs @@ -41,6 +41,8 @@ pub mod prune; #[cfg(feature = "provider-test")] pub mod queue_semantics; #[cfg(feature = "provider-test")] +pub mod race_replay; +#[cfg(feature = "provider-test")] pub mod sessions; #[cfg(feature = "provider-test")] pub mod tag_filtering; diff --git a/src/provider_validation/poison_message.rs b/src/provider_validation/poison_message.rs index 819d1a1..92503ed 100644 --- a/src/provider_validation/poison_message.rs +++ b/src/provider_validation/poison_message.rs @@ -14,6 +14,206 @@ use crate::providers::{TagFilter, WorkItem}; use super::ExecutionMetadata; +/// Ignoring a fetched input's attempt must not erase an unseen start's budget. +pub async fn orchestration_ignore_attempt_preserves_hidden_start(factory: &dyn ProviderFactory) { + let provider = factory.create_provider().await; + let instance = "ignore-attempt-hidden-start"; + // Persist the execution first: non-start inputs for an uncreated instance + // are correctly discarded by providers and cannot probe attempt isolation. + provider + .enqueue_for_orchestrator(super::start_item(instance), None) + .await + .expect("seed"); + let (_, seed_lock, _) = provider + .fetch_orchestration_item(factory.lock_timeout(), Duration::ZERO, None) + .await + .expect("seed fetch") + .expect("seed visible"); + let start = super::Event::with_event_id( + 1, + instance, + 1, + None, + super::EventKind::OrchestrationStarted { + name: "TestOrch".into(), + version: "1.0.0".into(), + input: "".into(), + parent_instance: None, + parent_id: None, + parent_execution_id: None, + carry_forward_events: None, + initial_custom_status: None, + }, + ); + provider + .ack_orchestration_item( + &seed_lock, + 1, + vec![start], + vec![], + vec![], + ExecutionMetadata { + orchestration_name: Some("TestOrch".into()), + orchestration_version: Some("1.0.0".into()), + ..Default::default() + }, + vec![], + ) + .await + .expect("persist execution"); + provider + .enqueue_for_orchestrator(super::start_item(instance), None) + .await + .expect("start"); + let (_, first, count) = provider + .fetch_orchestration_item(factory.lock_timeout(), Duration::ZERO, None) + .await + .expect("fetch") + .expect("visible start"); + assert_eq!(count, 1); + provider + .abandon_orchestration_item(&first, None, false) + .await + .expect("retain first attempt"); + let (_, second, count) = provider + .fetch_orchestration_item(factory.lock_timeout(), Duration::ZERO, None) + .await + .expect("fetch") + .expect("visible start"); + assert_eq!(count, 2); + provider + .abandon_orchestration_item(&second, Some(Duration::from_secs(1)), false) + .await + .expect("hide unregistered start with its two attempts"); + provider + .enqueue_for_orchestrator( + WorkItem::QueueMessage { + instance: instance.into(), + name: "q".into(), + data: "w".into(), + }, + None, + ) + .await + .expect("input"); + let (batch, input_lock, count) = provider + .fetch_orchestration_item(factory.lock_timeout(), Duration::ZERO, None) + .await + .expect("fetch input") + .expect("visible input"); + assert_eq!(batch.messages.len(), 1, "start must remain hidden"); + assert!(matches!(batch.messages[0], WorkItem::QueueMessage { .. })); + assert_eq!(count, 1); + provider + .abandon_orchestration_item(&input_lock, None, true) + .await + .expect("ignore only input attempt"); + tokio::time::sleep(Duration::from_millis(1100)).await; + let (_, lock, count) = provider + .fetch_orchestration_item(factory.lock_timeout(), Duration::ZERO, None) + .await + .expect("fetch together") + .expect("both visible"); + assert_eq!(count, 3, "the hidden start must retain both attempts before this fetch"); + provider + .ack_orchestration_item(&lock, 1, vec![], vec![], vec![], ExecutionMetadata::default(), vec![]) + .await + .expect("cleanup"); +} + +/// A delayed abandon must delay only the rows that fetch locked. A row that became +/// visible after the fetch (for example a continue-as-new start) keeps its visibility. +pub async fn orchestration_delayed_abandon_preserves_unlocked_rows(factory: &dyn ProviderFactory) { + let provider = factory.create_provider().await; + let instance = "delayed-abandon-unlocked-rows"; + // Persist the execution first: inputs for an uncreated instance are discarded. + provider + .enqueue_for_orchestrator(super::start_item(instance), None) + .await + .expect("seed"); + let (_, seed_lock, _) = provider + .fetch_orchestration_item(factory.lock_timeout(), Duration::ZERO, None) + .await + .expect("seed fetch") + .expect("seed visible"); + let start = super::Event::with_event_id( + 1, + instance, + 1, + None, + super::EventKind::OrchestrationStarted { + name: "TestOrch".into(), + version: "1.0.0".into(), + input: "".into(), + parent_instance: None, + parent_id: None, + parent_execution_id: None, + carry_forward_events: None, + initial_custom_status: None, + }, + ); + provider + .ack_orchestration_item( + &seed_lock, + 1, + vec![start], + vec![], + vec![], + ExecutionMetadata { + orchestration_name: Some("TestOrch".into()), + orchestration_version: Some("1.0.0".into()), + ..Default::default() + }, + vec![], + ) + .await + .expect("persist execution"); + let message = |data: &str| WorkItem::QueueMessage { + instance: instance.into(), + name: "q".into(), + data: data.into(), + }; + provider + .enqueue_for_orchestrator(message("locked"), None) + .await + .expect("first input"); + let (batch, lock, _) = provider + .fetch_orchestration_item(factory.lock_timeout(), Duration::ZERO, None) + .await + .expect("fetch") + .expect("first input visible"); + assert_eq!(batch.messages.len(), 1); + // Arrives while the first batch is locked, so this fetch never locked it. + provider + .enqueue_for_orchestrator(message("unlocked"), None) + .await + .expect("second input"); + provider + .abandon_orchestration_item(&lock, Some(Duration::from_secs(5)), false) + .await + .expect("delayed abandon"); + let (batch, lock, _) = provider + .fetch_orchestration_item(factory.lock_timeout(), Duration::ZERO, None) + .await + .expect("fetch after abandon") + .expect("a row the abandoned fetch did not lock must stay visible"); + assert_eq!( + batch.messages.len(), + 1, + "only the unlocked row is visible: {:?}", + batch.messages + ); + assert!( + matches!(&batch.messages[0], WorkItem::QueueMessage { data, .. } if data == "unlocked"), + "the delayed row must stay hidden: {:?}", + batch.messages + ); + provider + .ack_orchestration_item(&lock, 1, vec![], vec![], vec![], ExecutionMetadata::default(), vec![]) + .await + .expect("cleanup"); +} + /// Test that orchestration item attempt_count is 1 on first fetch pub async fn orchestration_attempt_count_starts_at_one(factory: &dyn ProviderFactory) { let provider = factory.create_provider().await; diff --git a/src/provider_validation/race_replay.rs b/src/provider_validation/race_replay.rs new file mode 100644 index 0000000..dbcd0dd --- /dev/null +++ b/src/provider_validation/race_replay.rs @@ -0,0 +1,1547 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Runtime-level queue cancellation and event-version preservation checks. + +use super::{Event, EventKind, ExecutionMetadata, ProviderFactory, start_item}; +use crate::providers::WorkItem; +use crate::runtime::registry::ActivityRegistry; +use crate::runtime::{Runtime, RuntimeOptions}; +use crate::{Client, Either3, OrchestrationContext, OrchestrationRegistry, OrchestrationStatus}; +use std::time::Duration; + +/// Duplicate client starts cannot switch a running execution to the latest +/// handler, another handler name, or a replacement input. +pub async fn test_duplicate_start_preserves_pinned_handler(factory: &F) { + for duplicate_name in ["Original", "WrongHandler"] { + let provider = factory.create_provider().await; + let instance = format!("provider-pinned-handler-{duplicate_name}"); + provider + .enqueue_for_orchestrator(start_item(&instance), None) + .await + .expect("seed start"); + let (_, lock, _) = provider + .fetch_orchestration_item(Duration::from_secs(30), Duration::ZERO, None) + .await + .expect("seed fetch") + .expect("visible seed"); + provider + .ack_orchestration_item( + &lock, + 1, + vec![ + Event::with_event_id( + 1, + &instance, + 1, + None, + EventKind::OrchestrationStarted { + name: "Original".into(), + version: "1.0.0".into(), + input: "original-input".into(), + parent_instance: None, + parent_id: None, + parent_execution_id: None, + carry_forward_events: None, + initial_custom_status: None, + }, + ), + Event::with_event_id( + 2, + &instance, + 1, + None, + EventKind::ActivityScheduled { + name: "tick".into(), + input: "v1".into(), + session_id: None, + tag: None, + }, + ), + ], + vec![], + vec![ + WorkItem::StartOrchestration { + instance: instance.clone(), + orchestration: duplicate_name.into(), + input: "wrong-input".into(), + version: None, + execution_id: 1, + parent_instance: None, + parent_id: None, + parent_execution_id: None, + }, + WorkItem::ActivityCompleted { + instance: instance.clone(), + execution_id: 1, + id: 2, + result: "ok".into(), + }, + ], + ExecutionMetadata { + status: Some("Running".into()), + orchestration_name: Some("Original".into()), + orchestration_version: Some("1.0.0".into()), + ..Default::default() + }, + vec![], + ) + .await + .expect("seed running pinned history and duplicate batch"); + let registry = OrchestrationRegistry::builder() + .register_versioned( + "Original", + "1.0.0", + |ctx: OrchestrationContext, input: String| async move { + ctx.schedule_activity("tick", "v1").await?; + Ok(format!("v1:{input}")) + }, + ) + .register_versioned("Original", "2.0.0", |ctx: OrchestrationContext, _: String| async move { + ctx.schedule_activity("tick", "v2").await?; + Ok("v2".into()) + }) + .register("WrongHandler", |ctx: OrchestrationContext, _: String| async move { + ctx.schedule_activity("tick", "wrong").await?; + Ok("wrong".into()) + }) + .build(); + let runtime = Runtime::start_with_options( + provider.clone(), + ActivityRegistry::builder().build(), + registry, + RuntimeOptions { + dispatcher_min_poll_interval: Duration::from_millis(10), + ..Default::default() + }, + ) + .await; + let result = Client::new(provider) + .wait_for_orchestration(&instance, Duration::from_secs(10)) + .await; + runtime.shutdown(None).await; + assert!(matches!(result.expect("duplicate client start must be inert"), + OrchestrationStatus::Completed { ref output, .. } if output == "v1:original-input")); + } +} + +/// During rolling upgrades an unregistered successor can hide its CAN start +/// through backoff. New inputs must survive until a compatible runtime resumes it. +pub async fn test_continue_as_new_unregistered_backoff(factory: &F) { + let provider = factory.create_provider().await; + let instance = "provider-can-unregistered-backoff"; + provider + .enqueue_for_orchestrator(start_item(instance), None) + .await + .expect("seed start"); + let (_, lock, _) = provider + .fetch_orchestration_item(Duration::from_secs(30), Duration::ZERO, None) + .await + .expect("seed fetch") + .expect("visible seed"); + provider + .ack_orchestration_item( + &lock, + 1, + vec![ + Event::with_event_id( + 1, + instance, + 1, + None, + EventKind::OrchestrationStarted { + name: "RollingSuccessor".into(), + version: "1.0.0".into(), + input: "old".into(), + parent_instance: None, + parent_id: None, + parent_execution_id: None, + carry_forward_events: None, + initial_custom_status: None, + }, + ), + Event::with_event_id( + 2, + instance, + 1, + None, + EventKind::OrchestrationContinuedAsNew { input: "next".into() }, + ), + ], + vec![], + vec![WorkItem::ContinueAsNew { + instance: instance.into(), + orchestration: "RollingSuccessor".into(), + input: "next".into(), + version: Some("1.0.0".into()), + parent_instance: None, + parent_id: None, + parent_execution_id: None, + carry_forward_events: vec![("q".into(), "carried".into())], + initial_custom_status: None, + }], + ExecutionMetadata { + status: Some("ContinuedAsNew".into()), + ..Default::default() + }, + vec![], + ) + .await + .expect("seed terminal and immediately visible successor atomically"); + let old_runtime = Runtime::start_with_options( + provider.clone(), + ActivityRegistry::builder().build(), + OrchestrationRegistry::builder().build(), + RuntimeOptions { + orchestration_concurrency: 1, + dispatcher_min_poll_interval: Duration::from_millis(10), + unregistered_backoff: crate::runtime::UnregisteredBackoffConfig { + base_delay: Duration::from_secs(2), + max_delay: Duration::from_secs(2), + }, + ..Default::default() + }, + ) + .await; + let hidden = tokio::time::timeout(Duration::from_secs(10), async { + let mut tick = tokio::time::interval(Duration::from_millis(20)); + loop { + tick.tick().await; + match provider + .fetch_orchestration_item(Duration::from_secs(30), Duration::ZERO, None) + .await + { + Ok(None) => break, + Ok(Some((item, lock, _))) => { + assert!( + item.messages + .iter() + .any(|item| matches!(item, WorkItem::ContinueAsNew { .. })) + ); + provider + .abandon_orchestration_item(&lock, None, true) + .await + .expect("release competing inspection fetch"); + } + Err(error) if error.retryable => { + tracing::warn!(%error, "Retrying backoff-test visibility inspection"); + } + Err(error) => panic!("backoff-test fetch failed: {error}"), + } + } + }) + .await; + old_runtime.shutdown(None).await; + hidden.expect("unregistered runtime must fetch and hide the start"); + assert!( + provider + .fetch_orchestration_item(Duration::from_secs(30), Duration::ZERO, None) + .await + .expect("inspect after runtime stopped") + .is_none(), + "start must be delayed by unregistered-handler backoff, not merely locked" + ); + assert_eq!( + provider + .read_with_execution(instance, 1) + .await + .expect("predecessor history") + .len(), + 2 + ); + let client = Client::new(provider.clone()); + client + .enqueue_event(instance, "q", "late") + .await + .expect("input during successor backoff"); + let registry = OrchestrationRegistry::builder() + .register("RollingSuccessor", |ctx: OrchestrationContext, _: String| async move { + assert_eq!(ctx.execution_id(), 2); + let first = ctx.dequeue_event("q").await; + let second = ctx.dequeue_event("q").await; + Ok(format!("{first},{second}")) + }) + .build(); + let new_runtime = Runtime::start_with_options( + provider.clone(), + ActivityRegistry::builder().build(), + registry, + RuntimeOptions { + dispatcher_min_poll_interval: Duration::from_millis(10), + ..Default::default() + }, + ) + .await; + let result = client.wait_for_orchestration(instance, Duration::from_secs(10)).await; + new_runtime.shutdown(None).await; + assert!( + matches!(result.expect("successor and window input must survive rolling-node backoff"), + OrchestrationStatus::Completed { ref output, .. } if output == "carried,late") + ); +} + +/// A duplicate start must not replace the durable CAN successor, even when it +/// precedes that successor in queue order. A colliding parent must be notified. +pub async fn test_continue_as_new_duplicate_start(factory: &F) { + for (stamp, collision) in [("0.1.30", false), ("0.1.30", true), ("0.1.31", false), ("0.1.31", true)] { + let provider = factory.create_provider().await; + let parent = "provider-can-duplicate-parent"; + let instance = format!("{parent}::can-child"); + let persist = |instance: String, history: Vec, messages: Vec, status: &str| { + let provider = provider.clone(); + let status = status.to_string(); + async move { + provider + .enqueue_for_orchestrator(start_item(&instance), None) + .await + .expect("seed start"); + let (_, lock, _) = provider + .fetch_orchestration_item(Duration::from_secs(30), Duration::ZERO, None) + .await + .expect("seed fetch") + .expect("visible seed"); + provider + .ack_orchestration_item( + &lock, + 1, + history, + vec![], + messages, + ExecutionMetadata { + status: Some(status), + ..Default::default() + }, + vec![], + ) + .await + .expect("seed acknowledgement"); + } + }; + let started = |instance: &str, name: &str| { + let mut event = Event::with_event_id( + 1, + instance, + 1, + None, + EventKind::OrchestrationStarted { + name: name.into(), + version: "1.0.0".into(), + input: "".into(), + parent_instance: None, + parent_id: None, + parent_execution_id: None, + carry_forward_events: None, + initial_custom_status: None, + }, + ); + event.duroxide_version = stamp.into(); + event + }; + if collision { + persist( + parent.into(), + vec![ + started(parent, "CollisionParent"), + Event::with_event_id( + 2, + parent, + 1, + None, + EventKind::SubOrchestrationScheduled { + name: "Transition".into(), + instance: "can-child".into(), + input: "duplicate".into(), + }, + ), + ], + vec![], + "Running", + ) + .await; + } + let messages = vec![ + WorkItem::StartOrchestration { + instance: instance.clone(), + orchestration: "WrongHandler".into(), + input: "duplicate".into(), + version: None, + execution_id: 1, + parent_instance: collision.then(|| parent.into()), + parent_id: collision.then_some(2), + parent_execution_id: collision.then_some(1), + }, + WorkItem::ContinueAsNew { + instance: instance.clone(), + orchestration: "Transition".into(), + input: "next".into(), + version: Some("1.0.0".into()), + parent_instance: None, + parent_id: None, + parent_execution_id: None, + carry_forward_events: vec![("q".into(), "carried".into())], + initial_custom_status: None, + }, + WorkItem::QueueMessage { + instance: instance.clone(), + name: "q".into(), + data: "late".into(), + }, + ]; + persist( + instance.clone(), + vec![ + started(&instance, "Transition"), + Event::with_event_id( + 2, + &instance, + 1, + None, + EventKind::OrchestrationContinuedAsNew { input: "next".into() }, + ), + ], + messages, + "ContinuedAsNew", + ) + .await; + let registry = OrchestrationRegistry::builder() + .register("Transition", |ctx: OrchestrationContext, _: String| async move { + assert_eq!(ctx.execution_id(), 2); + let first = ctx.dequeue_event("q").await; + let second = ctx.dequeue_event("q").await; + Ok(format!("{first},{second}")) + }) + .register("CollisionParent", |ctx: OrchestrationContext, _: String| async move { + match ctx + .schedule_sub_orchestration_with_id("Transition", "can-child", "duplicate") + .await + { + Ok(value) => Err(format!("collision unexpectedly resolved: {value}")), + Err(error) => Ok(error), + } + }) + .build(); + let runtime = Runtime::start_with_options( + provider.clone(), + ActivityRegistry::builder().build(), + registry, + RuntimeOptions { + dispatcher_min_poll_interval: Duration::from_millis(10), + ..Default::default() + }, + ) + .await; + let client = Client::new(provider.clone()); + let result = client.wait_for_orchestration(&instance, Duration::from_secs(10)).await; + let parent_result = if collision { + Some(client.wait_for_orchestration(parent, Duration::from_secs(10)).await) + } else { + None + }; + runtime.shutdown(None).await; + assert!( + matches!(result.expect("CAN successor must not livelock behind duplicate start"), + OrchestrationStatus::Completed { ref output, .. } if output == "carried,late") + ); + if let Some(parent_result) = parent_result { + assert!( + matches!(parent_result.expect("colliding parent must receive a notification"), + OrchestrationStatus::Completed { ref output, .. } + if output.contains("already exists") && !output.contains("is terminal")), + "collision notification must describe an existing logical instance, not a dead successor" + ); + } + let old = provider + .read_with_execution(&instance, 1) + .await + .expect("predecessor history"); + assert_eq!(old.len(), 2, "duplicate start must not mutate old recorded history"); + let next = provider + .read_with_execution(&instance, 2) + .await + .expect("successor history"); + assert!( + matches!(&next[0].kind, EventKind::OrchestrationStarted { name, input, .. } + if name == "Transition" && input == "next") + ); + } +} + +/// Persistent inputs must survive a CAN-terminal fetch without the successor's start item. +/// The delayed start deterministically exposes the provider's visibility-cutoff window. +pub async fn test_continue_as_new_transition_delivery(factory: &F, stamp: &str) { + for shape in [ + "queue", + "race", + "mixed-stale", + "signal", + "cancel", + "future-completion", + "stale-only", + ] { + let race = shape == "race"; + let provider = factory.create_provider().await; + let instance = format!("provider-can-window-{stamp}-{shape}"); + let mut start = Event::with_event_id( + 1, + &instance, + 1, + None, + EventKind::OrchestrationStarted { + name: "Transition".into(), + version: "1.0.0".into(), + input: "old".into(), + parent_instance: None, + parent_id: None, + parent_execution_id: None, + carry_forward_events: None, + initial_custom_status: None, + }, + ); + start.duroxide_version = stamp.into(); + provider + .enqueue_for_orchestrator(start_item(&instance), None) + .await + .expect("must enqueue the seed execution"); + let (_, lock, _) = provider + .fetch_orchestration_item(Duration::from_secs(30), Duration::ZERO, None) + .await + .expect("must fetch the seed execution") + .expect("seed execution must be visible"); + let mut inputs = vec![WorkItem::QueueMessage { + instance: instance.clone(), + name: "q".into(), + data: "late".into(), + }]; + if shape == "mixed-stale" { + inputs.extend([ + WorkItem::ActivityCompleted { + instance: instance.clone(), + execution_id: 1, + id: 99, + result: "stale".into(), + }, + WorkItem::TimerFired { + instance: instance.clone(), + execution_id: 1, + id: 100, + fire_at_ms: 1, + }, + WorkItem::SubOrchCompleted { + parent_instance: instance.clone(), + parent_execution_id: 1, + parent_id: 101, + result: "stale".into(), + }, + ]); + } + if shape == "future-completion" || shape == "stale-only" { + inputs = vec![WorkItem::ActivityCompleted { + instance: instance.clone(), + execution_id: if shape == "future-completion" { 2 } else { 1 }, + id: 99, + result: "unmatched".into(), + }]; + } + if shape == "signal" { + inputs.push(WorkItem::ExternalRaised { + instance: instance.clone(), + name: "signal".into(), + data: "answer".into(), + }); + } + if shape == "cancel" { + inputs.push(WorkItem::CancelInstance { + instance: instance.clone(), + reason: "requested-during-CAN".into(), + }); + } + provider + .ack_orchestration_item( + &lock, + 1, + vec![ + start, + Event::with_event_id( + 2, + &instance, + 1, + None, + EventKind::OrchestrationContinuedAsNew { input: "next".into() }, + ), + ], + vec![], + inputs, + ExecutionMetadata { + status: Some("ContinuedAsNew".into()), + output: Some("next".into()), + orchestration_name: Some("Transition".into()), + orchestration_version: Some("1.0.0".into()), + pinned_duroxide_version: Some(stamp.parse().expect("test stamp must parse")), + ..Default::default() + }, + vec![], + ) + .await + .expect("must persist the CAN-terminal seed and arrival atomically"); + let successor_start = WorkItem::ContinueAsNew { + instance: instance.clone(), + orchestration: "Transition".into(), + input: "next".into(), + version: Some("1.0.0".into()), + parent_instance: None, + parent_id: None, + parent_execution_id: None, + carry_forward_events: vec![("q".into(), "carried".into())], + initial_custom_status: None, + }; + provider + .enqueue_for_orchestrator(successor_start, Some(Duration::from_secs(2))) + .await + .expect("must enqueue the delayed successor start"); + let registry = OrchestrationRegistry::builder() + .register("Transition", move |ctx: OrchestrationContext, _: String| async move { + if ctx.execution_id() == 1 { + return ctx.continue_as_new("next").await; + } + if shape == "stale-only" { + return Ok(ctx.dequeue_event("q").await); + } + if shape == "signal" { + let answer = ctx.schedule_wait("signal").await; + let first = ctx.dequeue_event("q").await; + let second = ctx.dequeue_event("q").await; + return Ok(format!("{first},{second}|{answer}")); + } + let mut values = Vec::new(); + while values.len() < 2 { + if race { + if let crate::Either2::First(value) = ctx + .select2(ctx.dequeue_event("q"), ctx.schedule_timer(Duration::from_millis(10))) + .await + { + values.push(value); + } + } else { + values.push(ctx.dequeue_event("q").await); + } + } + Ok(values.join(",")) + }) + .build(); + let runtime = Runtime::start_with_options( + provider.clone(), + ActivityRegistry::builder().build(), + registry, + RuntimeOptions { + dispatcher_min_poll_interval: Duration::from_millis(10), + // Waiting inputs back off and count attempts like an unregistered handler; + // the 2 s hidden start must arrive well inside the default poison budget. + unregistered_backoff: crate::runtime::UnregisteredBackoffConfig { + base_delay: Duration::from_millis(250), + max_delay: Duration::from_secs(1), + }, + ..Default::default() + }, + ) + .await; + if shape == "signal" { + tokio::time::timeout(Duration::from_secs(10), async { + let mut tick = tokio::time::interval(Duration::from_millis(20)); + loop { + tick.tick().await; + let history = provider + .read_with_execution(&instance, 2) + .await + .expect("observe successor signal binding"); + if history + .iter() + .any(|event| matches!(&event.kind, EventKind::ExternalSubscribed { name } if name == "signal")) + && history.iter().any(|event| { + matches!(&event.kind, EventKind::ExternalEvent { name, data } + if name == "signal" && data == "answer") + }) + { + break; + } + } + }) + .await + .expect("successor must audit the preserved window signal and bind its wait"); + Client::new(provider.clone()) + .raise_event(&instance, "signal", "fresh") + .await + .expect("fresh answer after subscription"); + } + let result = Client::new(provider.clone()) + .wait_for_orchestration(&instance, Duration::from_secs(10)) + .await; + runtime.shutdown(None).await; + let successor = provider + .read_with_execution(&instance, 2) + .await + .expect("must read the successor history"); + let old = provider + .read_with_execution(&instance, 1) + .await + .expect("predecessor history"); + assert_eq!(old.len(), 2, "{stamp}/{shape}: admission cannot mutate execution 1"); + assert!(matches!(old[0].kind, EventKind::OrchestrationStarted { .. })); + assert!(matches!(old[1].kind, EventKind::OrchestrationContinuedAsNew { .. })); + assert!( + !successor.is_empty(), + "{stamp}/{shape}: successor never started; this does not prove arrival loss" + ); + assert_eq!( + successor[0] + .duroxide_version + .parse::() + .expect("successor stamp must parse"), + crate::providers::current_build_version(), + "the predecessor stamp does not select the successor's policy" + ); + let result = result.unwrap_or_else(|error| { + panic!("{stamp}/{shape}: terminal CAN must preserve the queued input: {error:?}; history={successor:?}") + }); + if shape == "future-completion" { + assert!( + matches!( + &result, + OrchestrationStatus::Failed { + details: crate::ErrorDetails::Configuration { + kind: crate::ConfigErrorKind::Nondeterminism, + .. + }, + .. + } + ), + "{stamp}/{shape}: a future-scoped malformed completion must be validated, not silently ACKed: {result:?}" + ); + continue; + } + if shape == "stale-only" { + assert!( + matches!(&result, OrchestrationStatus::Completed { output, .. } if output == "carried"), + "{stamp}/{shape}: stale-only batch must not obstruct the successor: {result:?}" + ); + continue; + } + if shape == "cancel" { + assert!( + matches!(&result, OrchestrationStatus::Failed { details: crate::ErrorDetails::Application { + kind: crate::AppErrorKind::Cancelled { reason }, .. + }, .. } if reason == "requested-during-CAN"), + "{stamp}/{shape}: logical instance cancellation must reach the successor: {result:?}" + ); + continue; + } + let expected_output = if shape == "signal" { + let binding = successor + .iter() + .find(|event| matches!(&event.kind, EventKind::ExternalSubscribed { name } if name == "signal")) + .expect("successor must bind its signal wait") + .event_id; + let answer = successor + .iter() + .find(|event| { + matches!(&event.kind, EventKind::ExternalEvent { name, data } if name == "signal" && data == "answer") + }) + .expect("preserved window signal must reach the successor audit history") + .event_id; + // Abandon visibility may split the CAN start and window inputs into + // separate fetches. Bound admission is defined at event application, + // not by the time the test enqueued the signal. + if answer < binding { + "carried,late|fresh" + } else { + "carried,late|answer" + } + } else { + "carried,late" + }; + assert!( + matches!(result, OrchestrationStatus::Completed { ref output, .. } if output == expected_output), + "{stamp}/{shape}: carried input must precede the preserved window arrival: {result:?}" + ); + if shape == "signal" { + let OrchestrationStatus::Completed { output, .. } = &result else { + unreachable!("completion was asserted above"); + }; + let consumed = output + .strip_prefix("carried,late|") + .expect("signal output must preserve queued values"); + if expected_output == "carried,late|fresh" { + assert_eq!(consumed, "fresh", "the audited answer before binding must be skipped"); + assert_ne!( + consumed, "answer", + "an early answer must not resolve the successor's wait" + ); + } else { + assert_eq!( + consumed, "answer", + "the answer applied after binding must resolve that wait" + ); + assert_ne!( + consumed, "fresh", + "fresh must not be consumed after the bound answer resolved" + ); + } + } + let delivered: Vec<_> = successor + .iter() + .filter_map(|event| match &event.kind { + EventKind::QueueEventDelivered { name, data } if name == "q" => Some(data.as_str()), + _ => None, + }) + .collect(); + assert_eq!( + delivered, + ["carried", "late"], + "{stamp}/{shape}: no loss or duplication" + ); + if shape == "signal" { + assert_eq!( + successor + .iter() + .filter(|event| matches!(&event.kind, EventKind::ExternalEvent { name, data } + if name == "signal" && data == "answer")) + .count(), + 1, + "positional input must reach successor admission exactly once" + ); + } + // Only mixed-stale injects stale completions (ids 99-101 of execution 1). In the race + // shape the successor's own timers can legitimately reach those ids while it waits. + if shape == "mixed-stale" { + assert!( + !successor.iter().any(|event| matches!( + event.kind, + EventKind::ActivityCompleted { .. } + | EventKind::TimerFired { .. } + | EventKind::SubOrchestrationCompleted { .. } + ) && event.source_event_id.is_some_and(|id| id >= 99)), + "{stamp}/{shape}: stale execution completions must not be applied to the successor" + ); + } + } +} + +/// Queue Q1/Q2/Q6 FIFO and replay checks on every provider. +pub async fn test_queue_race_cancellation_replay(factory: &F) { + for shape in ["q1", "q2", "q6"] { + let provider = factory.create_provider().await; + let instance = format!("provider-queue-race-{shape}"); + let event = |id, source, kind| Event::with_event_id(id, &instance, 1, source, kind); + let mut start = event( + 1, + None, + EventKind::OrchestrationStarted { + name: "QueueRace".into(), + version: "1.0.0".into(), + input: shape.into(), + parent_instance: None, + parent_id: None, + parent_execution_id: None, + carry_forward_events: None, + initial_custom_status: None, + }, + ); + start.duroxide_version = "0.1.31".into(); + let mut history = vec![start]; + match shape { + "q1" => { + history.extend([ + event(2, None, EventKind::QueueSubscribed { name: "q".into() }), + event(3, None, EventKind::QueueSubscribed { name: "q".into() }), + event(4, None, EventKind::TimerCreated { fire_at_ms: 1000 }), + event(5, Some(4), EventKind::TimerFired { fire_at_ms: 1000 }), + event( + 6, + None, + EventKind::QueueEventDelivered { + name: "q".into(), + data: "first".into(), + }, + ), + event( + 7, + None, + EventKind::QueueEventDelivered { + name: "q".into(), + data: "second".into(), + }, + ), + event(8, None, EventKind::QueueSubscribed { name: "q".into() }), + event(9, None, EventKind::QueueSubscribed { name: "q".into() }), + event( + 10, + Some(2), + EventKind::QueueSubscriptionCancelled { + reason: "dropped_future".into(), + }, + ), + event( + 11, + Some(3), + EventKind::QueueSubscriptionCancelled { + reason: "dropped_future".into(), + }, + ), + ]); + } + "q2" => { + history.extend([ + event(2, None, EventKind::QueueSubscribed { name: "q".into() }), + event(3, None, EventKind::QueueSubscribed { name: "q".into() }), + event(4, None, EventKind::TimerCreated { fire_at_ms: 1000 }), + event( + 5, + None, + EventKind::QueueEventDelivered { + name: "q".into(), + data: "first".into(), + }, + ), + event( + 6, + None, + EventKind::QueueEventDelivered { + name: "q".into(), + data: "second".into(), + }, + ), + event(7, None, EventKind::QueueSubscribed { name: "q".into() }), + event( + 8, + Some(3), + EventKind::QueueSubscriptionCancelled { + reason: "dropped_future".into(), + }, + ), + ]); + } + "q6" => { + history.extend([ + event(2, None, EventKind::QueueSubscribed { name: "q".into() }), + event(3, None, EventKind::TimerCreated { fire_at_ms: 1000 }), + event(4, None, EventKind::TimerCreated { fire_at_ms: 1100 }), + event(5, Some(4), EventKind::TimerFired { fire_at_ms: 1100 }), + event( + 6, + None, + EventKind::QueueEventDelivered { + name: "q".into(), + data: "first".into(), + }, + ), + event(7, None, EventKind::QueueSubscribed { name: "q".into() }), + event( + 8, + Some(2), + EventKind::QueueSubscriptionCancelled { + reason: "dropped_future".into(), + }, + ), + ]); + } + _ => unreachable!(), + } + let checkpoint_id = history.last().unwrap().event_id + 1; + let expected = if shape == "q6" { "first" } else { "first,second" }; + history.push(event( + checkpoint_id, + None, + EventKind::ActivityScheduled { + name: "checkpoint".into(), + input: expected.into(), + session_id: None, + tag: None, + }, + )); + provider + .enqueue_for_orchestrator(start_item(&instance), None) + .await + .unwrap(); + let (_, lock, _) = provider + .fetch_orchestration_item(Duration::from_secs(30), Duration::ZERO, None) + .await + .unwrap() + .unwrap(); + provider + .ack_orchestration_item( + &lock, + 1, + history.clone(), + vec![], + vec![WorkItem::ActivityCompleted { + instance: instance.clone(), + execution_id: 1, + id: checkpoint_id, + result: "ok".into(), + }], + ExecutionMetadata { + orchestration_name: Some("QueueRace".into()), + orchestration_version: Some("1.0.0".into()), + ..Default::default() + }, + vec![], + ) + .await + .unwrap(); + assert_eq!( + provider.read_with_execution(&instance, 1).await.unwrap(), + history, + "provider must preserve full event metadata and pinned semantic version" + ); + let registry = OrchestrationRegistry::builder() + .register("QueueRace", |ctx: OrchestrationContext, input: String| async move { + let mut values = Vec::new(); + if input == "q6" { + let inner = ctx.select2(ctx.dequeue_event("q"), ctx.schedule_timer(Duration::from_millis(10))); + let _ = ctx.select2(inner, ctx.schedule_timer(Duration::from_millis(100))).await; + values.push(ctx.dequeue_event("q").await); + } else { + match ctx + .select3( + ctx.dequeue_event("q"), + ctx.dequeue_event("q"), + ctx.schedule_timer(Duration::from_millis(10)), + ) + .await + { + Either3::First(value) | Either3::Second(value) => values.push(value), + Either3::Third(()) => {} + } + while values.len() < 2 { + values.push(ctx.dequeue_event("q").await); + } + } + let output = values.join(","); + ctx.schedule_activity("checkpoint", &output).await?; + Ok(output) + }) + .build(); + let runtime = Runtime::start_with_options( + provider.clone(), + ActivityRegistry::builder().build(), + registry, + RuntimeOptions { + dispatcher_min_poll_interval: Duration::from_millis(5), + ..Default::default() + }, + ) + .await; + let result = Client::new(provider) + .wait_for_orchestration(&instance, Duration::from_secs(30)) + .await; + runtime.shutdown(None).await; + let result = result.unwrap_or_else(|error| panic!("{shape}: queue replay did not complete: {error:?}")); + assert!( + matches!(result,OrchestrationStatus::Completed{ref output,..}if output==expected), + "{shape}: {result:?}" + ); + } +} + +/// Continue-as-new must carry unread arrivals, including held and non-prefix cases. +pub async fn test_continue_as_new_queue_race_replay(factory: &F) { + for shape in ["can", "can-drain", "can-held", "can-held-nonprefix"] { + let provider = factory.create_provider().await; + let instance = format!("provider-combined-race-{shape}"); + let event = |id, source, kind| Event::with_event_id(id, &instance, 1, source, kind); + let mut start = event( + 1, + None, + EventKind::OrchestrationStarted { + name: "CombinedRace".into(), + version: "1.0.0".into(), + input: shape.into(), + parent_instance: None, + parent_id: None, + parent_execution_id: None, + carry_forward_events: None, + initial_custom_status: None, + }, + ); + start.duroxide_version = "0.1.31".into(); + let mut history = vec![start]; + let incoming = { + history.extend([ + event(2, None, EventKind::QueueSubscribed { name: "q".into() }), + event(3, None, EventKind::TimerCreated { fire_at_ms: 1000 }), + ]); + let mut messages = vec![ + WorkItem::TimerFired { + instance: instance.clone(), + execution_id: 1, + id: 3, + fire_at_ms: 1000, + }, + WorkItem::QueueMessage { + instance: instance.clone(), + name: "q".into(), + data: "m1".into(), + }, + ]; + if shape == "can-drain" || shape == "can-held-nonprefix" { + messages.push(WorkItem::QueueMessage { + instance: instance.clone(), + name: "q".into(), + data: "m2".into(), + }); + } + messages + }; + provider + .enqueue_for_orchestrator(start_item(&instance), None) + .await + .unwrap(); + let (_, lock, _) = provider + .fetch_orchestration_item(Duration::from_secs(30), Duration::ZERO, None) + .await + .unwrap() + .unwrap(); + provider + .ack_orchestration_item( + &lock, + 1, + history, + vec![], + incoming, + ExecutionMetadata { + orchestration_name: Some("CombinedRace".into()), + orchestration_version: Some("1.0.0".into()), + ..Default::default() + }, + vec![], + ) + .await + .unwrap(); + let registry = OrchestrationRegistry::builder() + .register("CombinedRace", |ctx: OrchestrationContext, input: String| async move { + if ctx.execution_id() == 1 { + if input.starts_with("can-held") { + let mut held = ctx.dequeue_event("q"); + let _ = ctx + .select2(&mut held, ctx.schedule_timer(Duration::from_millis(10))) + .await; + if input == "can-held-nonprefix" { + ctx.dequeue_event("q").await; + } + return ctx.continue_as_new(input).await; + } + let _ = ctx + .select2(ctx.dequeue_event("q"), ctx.schedule_timer(Duration::from_millis(10))) + .await; + if input == "can-drain" { + ctx.dequeue_event("q").await; + } + ctx.continue_as_new(input).await + } else { + Ok(ctx.dequeue_event("q").await) + } + }) + .build(); + let runtime = Runtime::start_with_options( + provider.clone(), + ActivityRegistry::builder().build(), + registry, + RuntimeOptions { + dispatcher_min_poll_interval: Duration::from_millis(5), + ..Default::default() + }, + ) + .await; + let result = Client::new(provider) + .wait_for_orchestration(&instance, Duration::from_secs(30)) + .await; + runtime.shutdown(None).await; + let result = result.unwrap_or_else(|error| panic!("{shape}: unread CAN arrivals did not complete: {error:?}")); + let expected = match shape { + "can-drain" => "m2", + _ => "m1", + }; + assert!( + matches!(result,OrchestrationStatus::Completed{ref output,..}if output==expected), + "{shape}: {result:?}" + ); + } +} + +/// An old semantic version must round-trip without being restamped by a newer reader. +pub async fn test_queue_replay_version_stamp_roundtrip(factory: &F) { + let provider = factory.create_provider().await; + let instance = "queue-version-roundtrip"; + let mut event = Event::with_event_id( + 1, + instance, + 1, + None, + EventKind::OrchestrationStarted { + name: "TestOrch".into(), + version: "1.0.0".into(), + input: "{}".into(), + parent_instance: None, + parent_id: None, + parent_execution_id: None, + carry_forward_events: None, + initial_custom_status: None, + }, + ); + event.duroxide_version = "0.1.30".into(); + provider + .enqueue_for_orchestrator(start_item(instance), None) + .await + .unwrap(); + let (_, lock, _) = provider + .fetch_orchestration_item(Duration::from_secs(30), Duration::ZERO, None) + .await + .unwrap() + .unwrap(); + provider + .ack_orchestration_item( + &lock, + 1, + vec![event.clone()], + vec![], + vec![], + ExecutionMetadata { + orchestration_name: Some("TestOrch".into()), + orchestration_version: Some("1.0.0".into()), + ..Default::default() + }, + vec![], + ) + .await + .unwrap(); + assert_eq!(provider.read_with_execution(instance, 1).await.unwrap(), vec![event]); + provider + .enqueue_for_orchestrator( + WorkItem::ExternalRaised { + instance: instance.into(), + name: "fetch-stamp".into(), + data: "wake".into(), + }, + None, + ) + .await + .expect("enqueue fetch-path stamp probe"); + let (fetched, lock, _) = provider + .fetch_orchestration_item(Duration::from_secs(30), Duration::ZERO, None) + .await + .expect("fetch old-stamped execution") + .expect("old-stamped execution was not fetchable"); + assert_eq!( + fetched.history[0].duroxide_version, "0.1.30", + "fetch path restamped history" + ); + provider + .abandon_orchestration_item(&lock, None, true) + .await + .expect("release stamp probe lock"); +} + +/// Sequential positional round trips with a long timer loser, then identical replay. +/// The binding barrier deliberately serializes signals; this is not competitive +/// signal-race coverage. Engine tests cover same-batch signal/timer races. +pub async fn test_positional_wait_race_replay(factory: &F) { + let provider = factory.create_provider().await; + let registry = OrchestrationRegistry::builder() + .register("LiveSignalRace", |ctx: OrchestrationContext, _: String| async move { + let first = ctx + .select2( + ctx.schedule_wait("s"), + ctx.schedule_timer(Duration::from_secs(30)) + .map(|()| "timeout".to_string()), + ) + .await + .into_tuple() + .1; + Ok(format!("{first:?}:{}", ctx.schedule_wait("s").await)) + }) + .build(); + let handler = registry + .resolve_handler("LiveSignalRace") + .expect("registered live signal handler") + .1; + let runtime = Runtime::start_with_options( + provider.clone(), + ActivityRegistry::builder().build(), + registry, + RuntimeOptions::default(), + ) + .await; + let client = Client::new(provider.clone()); + client + .start_orchestration("provider-live-signal", "LiveSignalRace", "") + .await + .expect("start live signal race"); + tokio::time::timeout(Duration::from_secs(10), async { + let mut tick = tokio::time::interval(Duration::from_millis(20)); + loop { + tick.tick().await; + let history = provider + .read("provider-live-signal") + .await + .expect("read live signal history"); + if history + .iter() + .any(|e| matches!(e.kind, EventKind::ExternalSubscribed { .. })) + { + break; + } + } + }) + .await + .expect("signal subscription never appeared"); + client + .raise_event("provider-live-signal", "s", "one") + .await + .expect("raise first signal"); + tokio::time::timeout(Duration::from_secs(10), async { + let mut tick = tokio::time::interval(Duration::from_millis(20)); + loop { + tick.tick().await; + let history = provider + .read("provider-live-signal") + .await + .expect("read replacement signal binding"); + if history + .iter() + .filter(|event| matches!(&event.kind, EventKind::ExternalSubscribed { name } if name == "s")) + .count() + >= 2 + { + break; + } + } + }) + .await + .expect("replacement signal subscription never appeared"); + client + .raise_event("provider-live-signal", "s", "two") + .await + .expect("raise second signal"); + let result = client + .wait_for_orchestration("provider-live-signal", Duration::from_secs(10)) + .await; + runtime.shutdown(None).await; + let output = match result.unwrap_or_else(|error| { + panic!("live signal race did not finish after both subscriptions were bound: {error:?}") + }) { + OrchestrationStatus::Completed { output, .. } => output, + other => panic!("live signal race failed: {other:?}"), + }; + let history = provider + .read("provider-live-signal") + .await + .expect("read recorded live signal turns"); + // Terminal histories short-circuit to Continue. Replay the recorded body + // without its terminal marker, as the runtime would before completion. + let history: Vec<_> = history + .into_iter() + .filter(|event| !matches!(event.kind, EventKind::OrchestrationCompleted { .. })) + .collect(); + for _ in 0..2 { + let mut engine = + crate::runtime::replay_engine::ReplayEngine::new("provider-live-signal".into(), 1, history.clone()); + let result = engine.execute_orchestration( + handler.clone(), + "".into(), + "LiveSignalRace".into(), + "1.0.0".into(), + "validation", + ); + assert!( + matches!(result, crate::runtime::replay_engine::TurnResult::Completed(ref value) if *value == output), + "provider live/replay differs: {result:?}, live={output}, history={history:?}" + ); + assert!(engine.history_delta().is_empty(), "replay appended history"); + } +} + +/// A fetched 0.1.30 execution must take a legacy decision that differs from the new policy. +pub async fn test_legacy_queue_race_decision_preserved(factory: &F) { + use crate::runtime::replay_engine::{ReplayEngine, TurnResult}; + for stamp in ["0.1.30", "0.1.31"] { + let provider = factory.create_provider().await; + let instance = format!("provider-legacy-decision-{stamp}"); + let registry = OrchestrationRegistry::builder() + .register("LegacyCompetition", |ctx: OrchestrationContext, _: String| async move { + let first = ctx.dequeue_event("q"); + let second = ctx.dequeue_event("q"); + let t0 = ctx.schedule_timer(Duration::from_secs(30)); + let t1 = ctx.schedule_timer(Duration::from_secs(60)); + let _ = ctx.select2(first, t0).await; + let value = match ctx.select2(second, t1).await { + crate::Either2::First(value) => value, + crate::Either2::Second(()) => "second-timeout".into(), + }; + ctx.schedule_activity("checkpoint", &value).await?; + Ok(value) + }) + .build(); + let handler = registry + .resolve_handler("LegacyCompetition") + .expect("registered competition handler") + .1; + let mut start = Event::with_event_id( + 1, + &instance, + 1, + None, + EventKind::OrchestrationStarted { + name: "LegacyCompetition".into(), + version: "1.0.0".into(), + input: "".into(), + parent_instance: None, + parent_id: None, + parent_execution_id: None, + carry_forward_events: None, + initial_custom_status: None, + }, + ); + start.duroxide_version = stamp.into(); + let mut engine = ReplayEngine::new(instance.clone(), 1, vec![start.clone()]); + assert!(matches!( + engine.execute_orchestration( + handler.clone(), + "".into(), + "LegacyCompetition".into(), + "1.0.0".into(), + "validation" + ), + TurnResult::Continue + )); + let mut history = vec![start]; + history.extend_from_slice(engine.history_delta()); + // Timer completions must already be visible in the provider's queue. + // Matching checks the schedule identity, not its wall-clock deadline. + for event in &mut history { + if let EventKind::TimerCreated { fire_at_ms } = &mut event.kind { + *fire_at_ms = 1; + } + } + let timers: Vec<_> = history + .iter() + .filter_map(|e| { + if let EventKind::TimerCreated { fire_at_ms } = e.kind { + Some(WorkItem::TimerFired { + instance: instance.clone(), + execution_id: 1, + id: e.event_id(), + fire_at_ms, + }) + } else { + None + } + }) + .collect(); + assert_eq!(timers.len(), 2); + provider + .enqueue_for_orchestrator(start_item(&instance), None) + .await + .expect("seed competition"); + let (_, lock, _) = provider + .fetch_orchestration_item(Duration::from_secs(30), Duration::ZERO, None) + .await + .expect("fetch competition seed") + .expect("missing competition seed"); + provider + .ack_orchestration_item( + &lock, + 1, + history, + vec![], + vec![ + timers[0].clone(), + WorkItem::QueueMessage { + instance: instance.clone(), + name: "q".into(), + data: "first".into(), + }, + timers[1].clone(), + ], + ExecutionMetadata { + orchestration_name: Some("LegacyCompetition".into()), + orchestration_version: Some("1.0.0".into()), + ..Default::default() + }, + vec![], + ) + .await + .expect("persist competition prefix"); + let (fetched, lock, _) = provider + .fetch_orchestration_item(Duration::from_secs(30), Duration::ZERO, None) + .await + .expect("fetch competition prefix") + .expect("missing competition turn"); + assert_eq!( + fetched.history[0].duroxide_version, stamp, + "fetch path changed the execution policy" + ); + let mut turn = ReplayEngine::new(instance.clone(), 1, fetched.history.clone()); + turn.prep_completions(fetched.messages); + let live = turn.execute_orchestration( + handler, + "".into(), + "LegacyCompetition".into(), + "1.0.0".into(), + "validation", + ); + assert!( + matches!(live, TurnResult::Continue), + "competition evaluation failed: {live:?}" + ); + let checkpoint = turn + .history_delta() + .iter() + .find_map(|event| match &event.kind { + EventKind::ActivityScheduled { name, .. } if name == "checkpoint" => Some(event.event_id()), + _ => None, + }) + .expect("competition did not schedule checkpoint"); + provider + .ack_orchestration_item( + &lock, + 1, + turn.history_delta().to_vec(), + vec![], + vec![WorkItem::ActivityCompleted { + instance: instance.clone(), + execution_id: 1, + id: checkpoint, + result: "ok".into(), + }], + ExecutionMetadata::default(), + vec![], + ) + .await + .expect("persist fetched competition decisions"); + let runtime = Runtime::start_with_options( + provider.clone(), + ActivityRegistry::builder() + .register("checkpoint", |_: crate::ActivityContext, input: String| async move { + Ok(input) + }) + .build(), + registry, + RuntimeOptions::default(), + ) + .await; + let result = Client::new(provider.clone()) + .wait_for_orchestration(&instance, Duration::from_secs(10)) + .await; + let history = provider + .read(&instance) + .await + .expect("read competition diagnostic history"); + runtime.shutdown(None).await; + let result = result + .unwrap_or_else(|error| panic!("{stamp}: competition did not finish: {error:?}, history={history:?}")); + let expected = if stamp == "0.1.30" { "second-timeout" } else { "first" }; + assert!( + matches!(result,OrchestrationStatus::Completed{ref output,..}if output==expected), + "{stamp}: legacy-discriminating competition returned {result:?}" + ); + } +} diff --git a/src/provider_validations.rs b/src/provider_validations.rs index d1276a9..d937e66 100644 --- a/src/provider_validations.rs +++ b/src/provider_validations.rs @@ -32,6 +32,8 @@ // Re-export ProviderFactory from the internal module #[cfg(feature = "provider-test")] pub use crate::provider_validation::ProviderFactory; +#[cfg(feature = "provider-test")] +pub use crate::provider_validation::race_replay; /// ## Individual Test Functions /// @@ -203,6 +205,7 @@ pub mod poison_message { abandon_orchestration_item_ignore_attempt_decrements, abandon_work_item_ignore_attempt_decrements, attempt_count_is_per_message, ignore_attempt_never_goes_negative, max_attempt_count_across_message_batch, orchestration_attempt_count_increments_on_refetch, orchestration_attempt_count_starts_at_one, + orchestration_delayed_abandon_preserves_unlocked_rows, orchestration_ignore_attempt_preserves_hidden_start, worker_attempt_count_increments_on_lock_expiry, worker_attempt_count_starts_at_one, }; } diff --git a/src/runtime/state_helpers.rs b/src/runtime/state_helpers.rs index b5fe48a..4853ccc 100644 --- a/src/runtime/state_helpers.rs +++ b/src/runtime/state_helpers.rs @@ -567,6 +567,147 @@ impl WorkItemReader { mod tests { use super::*; + #[test] + fn can_start_precedes_duplicate_client_start_selection() { + let history = HistoryManager::from_history(&[ + Event::with_event_id( + 1, + "instance", + 1, + None, + EventKind::OrchestrationStarted { + name: "Original".into(), + version: "1.0.0".into(), + input: "old".into(), + parent_instance: None, + parent_id: None, + parent_execution_id: None, + carry_forward_events: None, + initial_custom_status: None, + }, + ), + Event::with_event_id( + 2, + "instance", + 1, + None, + EventKind::OrchestrationContinuedAsNew { input: "next".into() }, + ), + ]); + let duplicate = WorkItem::StartOrchestration { + instance: "instance".into(), + orchestration: "Wrong".into(), + input: "duplicate".into(), + version: None, + execution_id: 1, + parent_instance: None, + parent_id: None, + parent_execution_id: None, + }; + let can = WorkItem::ContinueAsNew { + instance: "instance".into(), + orchestration: "Original".into(), + input: "next".into(), + version: Some("1.0.0".into()), + parent_instance: None, + parent_id: None, + parent_execution_id: None, + carry_forward_events: vec![], + initial_custom_status: None, + }; + for batch in [vec![duplicate.clone(), can.clone()], vec![can, duplicate]] { + let reader = WorkItemReader::from_messages(&batch, &history, "instance"); + assert!(reader.is_continue_as_new); + assert_eq!(reader.orchestration_name, "Original"); + assert_eq!(reader.input, "next"); + } + } + + #[test] + fn duplicate_client_start_does_not_replace_pinned_running_handler() { + let history = HistoryManager::from_history(&[Event::with_event_id( + 1, + "instance", + 1, + None, + EventKind::OrchestrationStarted { + name: "Original".into(), + version: "1.0.0".into(), + input: "old".into(), + parent_instance: None, + parent_id: None, + parent_execution_id: None, + carry_forward_events: None, + initial_custom_status: None, + }, + )]); + let reader = WorkItemReader::from_messages( + &[WorkItem::StartOrchestration { + instance: "instance".into(), + orchestration: "Wrong".into(), + input: "duplicate".into(), + version: None, + execution_id: 1, + parent_instance: None, + parent_id: None, + parent_execution_id: None, + }], + &history, + "instance", + ); + assert!(!reader.has_start_item()); + assert_eq!(reader.orchestration_name, "Original"); + assert_eq!(reader.version.as_deref(), Some("1.0.0")); + assert_eq!(reader.input, "old"); + } + + #[test] + fn terminal_activity_cancellations_are_ordered() { + for stamp in ["0.1.30", "0.1.31"] { + let mut start = Event::with_event_id( + 1, + "test-inst", + 1, + None, + EventKind::OrchestrationStarted { + name: "Test".into(), + version: "1.0.0".into(), + input: "".into(), + parent_instance: None, + parent_id: None, + parent_execution_id: None, + carry_forward_events: None, + initial_custom_status: None, + }, + ); + start.duroxide_version = stamp.into(); + let mut history = vec![start]; + for id in 2..=20 { + history.push(Event::with_event_id( + id, + "test-inst", + 1, + None, + EventKind::ActivityScheduled { + name: "pending".into(), + input: "".into(), + session_id: None, + tag: None, + }, + )); + } + for _ in 0..16 { + let manager = HistoryManager::from_history(&history); + let ids: Vec<_> = manager + .compute_inflight_activities("test-inst", 1) + .into_iter() + .map(|activity| activity.activity_id) + .collect(); + assert_eq!(ids, (2..=20).collect::>(), "{stamp}"); + } + } + } + #[test] fn test_history_reader_from_empty_history() { let metadata = HistoryManager::from_history(&[]); diff --git a/tests/can_abandon_failure.rs b/tests/can_abandon_failure.rs new file mode 100644 index 0000000..7d853fe --- /dev/null +++ b/tests/can_abandon_failure.rs @@ -0,0 +1,646 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#![allow(clippy::expect_used, clippy::clone_on_ref_ptr)] + +mod common; + +use common::fault_injection::PoisonInjectingProvider; +use duroxide::providers::sqlite::SqliteProvider; +use duroxide::providers::{ExecutionMetadata, Provider, WorkItem}; +use duroxide::runtime::registry::ActivityRegistry; +use duroxide::runtime::{Runtime, RuntimeOptions, UnregisteredBackoffConfig}; +use duroxide::{Client, Event, EventKind, OrchestrationContext, OrchestrationRegistry, OrchestrationStatus}; +use std::sync::Arc; +use std::time::Duration; + +#[tokio::test] +async fn successor_first_turn_must_obey_poison_budget() { + let sqlite = Arc::new(SqliteProvider::new_in_memory().await.expect("test store")); + let provider = Arc::new(PoisonInjectingProvider::new(sqlite)); + let max_attempts = 1; + provider.inject_orchestration_poison_after_skip(1, max_attempts + 1); + let registry = OrchestrationRegistry::builder() + .register("Transition", |ctx: OrchestrationContext, _: String| async move { + if ctx.execution_id() == 1 { + ctx.continue_as_new("next").await + } else { + Ok("successor incorrectly ran".into()) + } + }) + .build(); + let client = Client::new(provider.clone()); + client + .start_orchestration("CAN-first-turn-poison", "Transition", "") + .await + .expect("start"); + let runtime = Runtime::start_with_options( + provider.clone(), + ActivityRegistry::builder().build(), + registry, + RuntimeOptions { + max_attempts, + orchestration_concurrency: 1, + dispatcher_min_poll_interval: Duration::from_millis(10), + ..Default::default() + }, + ) + .await; + let result = client + .wait_for_orchestration("CAN-first-turn-poison", Duration::from_secs(10)) + .await; + assert!( + matches!( + result.expect("poison must terminate"), + OrchestrationStatus::Failed { + details: duroxide::ErrorDetails::Poison { .. }, + .. + } + ), + "CAN successor first-turn attempts must not be exempt from poison" + ); + assert_late_inputs_are_drained(&client, provider.as_ref(), &runtime, "CAN-first-turn-poison").await; +} + +async fn assert_late_inputs_are_drained( + client: &Client, + provider: &PoisonInjectingProvider, + runtime: &Arc, + instance: &str, +) { + let before = provider.read(instance).await.expect("poisoned history"); + assert!( + before + .iter() + .any(|event| matches!(event.kind, EventKind::OrchestrationContinuedAsNew { .. })) + ); + assert!( + before + .iter() + .any(|event| matches!(event.kind, EventKind::OrchestrationFailed { .. })) + ); + client.enqueue_event(instance, "q", "late").await.expect("late queue"); + client + .raise_event(instance, "signal", "late") + .await + .expect("late signal"); + client + .cancel_instance(instance, "late cancel") + .await + .expect("late cancel"); + tokio::time::sleep(Duration::from_secs(2)).await; + runtime.clone().shutdown(None).await; + // Let an already-held batch return before inspecting the unlocked queue. + tokio::time::sleep(Duration::from_millis(1200)).await; + let batch = provider + .fetch_orchestration_item(Duration::from_secs(1), Duration::ZERO, None) + .await + .expect("inspect terminal queue"); + assert!( + batch.is_none(), + "poisoned CAN successor must drain late queue, signal and cancel inputs" + ); + assert_eq!(provider.read(instance).await.expect("unchanged history"), before); +} + +#[tokio::test] +async fn unregistered_can_successor_poison_must_drain_late_inputs() { + let sqlite = Arc::new(SqliteProvider::new_in_memory().await.expect("store")); + let provider = Arc::new(PoisonInjectingProvider::new(sqlite)); + let instance = "CAN-unregistered-poison"; + seed_waiting_can(provider.as_ref(), instance, "MissingHandler", late_input(instance)).await; + enqueue_successor(provider.as_ref(), instance, "MissingHandler", None).await; + let runtime = Runtime::start_with_options( + provider.clone(), + ActivityRegistry::builder().build(), + OrchestrationRegistry::builder().build(), + RuntimeOptions { + max_attempts: 1, + orchestration_concurrency: 1, + unregistered_backoff: UnregisteredBackoffConfig { + base_delay: Duration::from_millis(20), + max_delay: Duration::from_millis(20), + }, + dispatcher_min_poll_interval: Duration::from_millis(10), + ..Default::default() + }, + ) + .await; + let client = Client::new(provider.clone()); + let result = client + .wait_for_orchestration(instance, Duration::from_secs(10)) + .await + .expect("poison"); + assert!(matches!( + result, + OrchestrationStatus::Failed { + details: duroxide::ErrorDetails::Poison { .. }, + .. + } + )); + assert_late_inputs_are_drained(&client, provider.as_ref(), &runtime, instance).await; +} + +#[tokio::test] +async fn delayed_can_start_defers_inputs_with_counted_backoff() { + let sqlite = Arc::new(SqliteProvider::new_in_memory().await.expect("store")); + let provider = Arc::new(PoisonInjectingProvider::new(sqlite)); + let instance = "CAN-start-delayed"; + seed_waiting_can(provider.as_ref(), instance, "Transition", late_input(instance)).await; + // The successor start exists but stays hidden, as behind an old worker's backoff. + enqueue_successor( + provider.as_ref(), + instance, + "Transition", + Some(Duration::from_millis(900)), + ) + .await; + let runtime = Runtime::start_with_options( + provider.clone(), + ActivityRegistry::builder().build(), + OrchestrationRegistry::builder() + .register("Transition", |ctx: OrchestrationContext, _: String| async move { + assert_eq!(ctx.execution_id(), 2); + Ok(ctx.dequeue_event("q").await) + }) + .build(), + RuntimeOptions { + orchestration_concurrency: 1, + unregistered_backoff: UnregisteredBackoffConfig { + base_delay: Duration::from_millis(100), + max_delay: Duration::from_millis(400), + }, + dispatcher_min_poll_interval: Duration::from_millis(10), + ..Default::default() + }, + ) + .await; + let result = Client::new(provider.clone()) + .wait_for_orchestration(instance, Duration::from_secs(10)) + .await; + runtime.shutdown(None).await; + assert!( + matches!(result.expect("delayed start completion"), + OrchestrationStatus::Completed { ref output, .. } if output == "late"), + "the waiting input must reach the successor once its start is visible" + ); + let delays = provider.counted_abandon_delays(); + assert!( + delays.len() >= 2, + "the input must wait through several counted deferrals: {delays:?}" + ); + assert_eq!( + &delays[..2], + &[Duration::from_millis(100), Duration::from_millis(200)], + "deferrals use the unregistered-handler backoff for the growing attempt count" + ); + assert_eq!( + provider.successful_ignored_abandons(), + 0, + "deferral must not reset attempts" + ); + let old = provider.read_with_execution(instance, 1).await.expect("old history"); + assert_eq!(old.len(), 2, "deferral must not mutate execution 1"); +} + +/// A batch of only stale completions for the previous execution also waits for the +/// CAN start (counted, like any other batch), and the successor then drops it. +#[tokio::test] +async fn stale_only_batch_waits_for_the_can_start() { + let sqlite = Arc::new(SqliteProvider::new_in_memory().await.expect("store")); + let provider = Arc::new(PoisonInjectingProvider::new(sqlite)); + let instance = "CAN-stale-only"; + let stale = WorkItem::ActivityCompleted { + instance: instance.into(), + execution_id: 1, + id: 99, + result: "stale".into(), + }; + seed_waiting_can(provider.as_ref(), instance, "Transition", stale).await; + enqueue_successor( + provider.as_ref(), + instance, + "Transition", + Some(Duration::from_millis(400)), + ) + .await; + let runtime = Runtime::start_with_options( + provider.clone(), + ActivityRegistry::builder().build(), + OrchestrationRegistry::builder() + .register("Transition", |ctx: OrchestrationContext, _: String| async move { + assert_eq!(ctx.execution_id(), 2); + Ok("successor".into()) + }) + .build(), + RuntimeOptions { + orchestration_concurrency: 1, + unregistered_backoff: UnregisteredBackoffConfig { + base_delay: Duration::from_millis(100), + max_delay: Duration::from_millis(100), + }, + dispatcher_min_poll_interval: Duration::from_millis(10), + ..Default::default() + }, + ) + .await; + let result = Client::new(provider.clone()) + .wait_for_orchestration(instance, Duration::from_secs(10)) + .await; + runtime.shutdown(None).await; + assert!(matches!(result.expect("successor completion"), + OrchestrationStatus::Completed { ref output, .. } if output == "successor")); + assert!( + !provider.counted_abandon_delays().is_empty(), + "the stale-only batch must be deferred with a counted attempt, not acknowledged" + ); + let old = provider.read_with_execution(instance, 1).await.expect("old history"); + assert_eq!(old.len(), 2, "deferral must not mutate execution 1"); + let successor = provider + .read_with_execution(instance, 2) + .await + .expect("successor history"); + assert!( + !successor + .iter() + .any(|event| matches!(event.kind, EventKind::ActivityCompleted { .. })), + "the successor must drop the stale completion: {successor:?}" + ); +} + +/// A duplicate sub-orchestration start for a running instance fails the incoming +/// parent, unless it is the instance's own parent re-sending the same call with a +/// known parent execution id. +#[tokio::test] +async fn live_collision_ignores_only_the_same_known_parent_call() { + for (recorded_generation, incoming, notified) in [ + (Some(1), (7, Some(1)), false), // the same call: ignored, as on the base + (Some(1), (8, Some(1)), true), // another call of the same parent + (Some(1), (7, Some(2)), true), // the same call id from a later parent execution + (None, (7, None), true), // unknown generations cannot prove the same call + ] { + let sqlite = Arc::new(SqliteProvider::new_in_memory().await.expect("store")); + let provider = Arc::new(PoisonInjectingProvider::new(sqlite)); + let instance = "live-collision"; + let start = |parent_id: u64, parent_execution_id: Option| WorkItem::StartOrchestration { + instance: instance.into(), + orchestration: "Child".into(), + input: "".into(), + version: Some("1.0.0".into()), + execution_id: 1, + parent_instance: Some("parent".into()), + parent_id: Some(parent_id), + parent_execution_id, + }; + common::seed_history_turn( + provider.as_ref(), + start(7, recorded_generation), + 1, + vec![Event::with_event_id( + 1, + instance, + 1, + None, + EventKind::OrchestrationStarted { + name: "Child".into(), + version: "1.0.0".into(), + input: "".into(), + parent_instance: Some("parent".into()), + parent_id: Some(7), + parent_execution_id: recorded_generation, + carry_forward_events: None, + initial_custom_status: None, + }, + )], + vec![start(incoming.0, incoming.1), late_input(instance)], + ExecutionMetadata { + orchestration_name: Some("Child".into()), + orchestration_version: Some("1.0.0".into()), + ..Default::default() + }, + ) + .await; + let runtime = Runtime::start_with_options( + provider.clone(), + ActivityRegistry::builder().build(), + OrchestrationRegistry::builder() + .register("Child", |ctx: OrchestrationContext, _: String| async move { + Ok(ctx.dequeue_event("q").await) + }) + .build(), + RuntimeOptions { + orchestration_concurrency: 1, + dispatcher_min_poll_interval: Duration::from_millis(10), + ..Default::default() + }, + ) + .await; + // The child keeps running its original start and reads the queued input. + let result = Client::new(provider.clone()) + .wait_for_orchestration(instance, Duration::from_secs(10)) + .await; + runtime.shutdown(None).await; + assert!( + matches!(result, Ok(OrchestrationStatus::Completed { ref output, .. }) if output == "late"), + "{incoming:?}: the duplicate must not disturb the running child: {result:?}" + ); + let notices = provider.collision_notices(); + assert_eq!( + notices.iter().any(|(parent_id, _)| *parent_id == incoming.0), + notified, + "recorded generation {recorded_generation:?}, incoming {incoming:?}: notices {notices:?}" + ); + } +} + +#[tokio::test] +async fn missing_can_start_poisons_waiting_inputs_after_max_attempts() { + let sqlite = Arc::new(SqliteProvider::new_in_memory().await.expect("store")); + let provider = Arc::new(PoisonInjectingProvider::new(sqlite)); + let instance = "CAN-start-missing"; + // Corruption: execution 1 ended in continue-as-new, but its successor start is gone. + seed_waiting_can(provider.as_ref(), instance, "Transition", late_input(instance)).await; + let max_attempts = 3; + let runtime = Runtime::start_with_options( + provider.clone(), + ActivityRegistry::builder().build(), + OrchestrationRegistry::builder() + .register("Transition", |_: OrchestrationContext, _: String| async move { + Ok("successor incorrectly ran".into()) + }) + .build(), + RuntimeOptions { + max_attempts, + orchestration_concurrency: 1, + unregistered_backoff: UnregisteredBackoffConfig { + base_delay: Duration::from_millis(20), + max_delay: Duration::from_millis(20), + }, + dispatcher_min_poll_interval: Duration::from_millis(10), + ..Default::default() + }, + ) + .await; + let client = Client::new(provider.clone()); + let result = client + .wait_for_orchestration(instance, Duration::from_secs(10)) + .await + .expect("a missing start must end the wait"); + assert!( + matches!( + result, + OrchestrationStatus::Failed { + details: duroxide::ErrorDetails::Poison { .. }, + .. + } + ), + "a start that never appears must poison the waiting inputs: {result:?}" + ); + assert_eq!( + provider.counted_abandon_delays().len(), + max_attempts as usize, + "each waiting fetch up to max_attempts defers with a counted attempt" + ); + let history = provider + .read_with_execution(instance, 1) + .await + .expect("poisoned history"); + assert!(matches!( + history.last().map(|e| &e.kind), + Some(EventKind::OrchestrationFailed { .. }) + )); + assert!( + provider + .read_with_execution(instance, 2) + .await + .unwrap_or_default() + .is_empty() + ); + // Failed after CAN is terminal: later inputs are drained, not deferred forever. + assert_late_inputs_are_drained(&client, provider.as_ref(), &runtime, instance).await; +} + +#[tokio::test] +async fn failure_commit_must_preserve_collision_notification() { + let sqlite = Arc::new(SqliteProvider::new_in_memory().await.expect("store")); + let provider = Arc::new(PoisonInjectingProvider::new(sqlite)); + let instance = "collision-failure-commit"; + let start_item = |parent| WorkItem::StartOrchestration { + instance: instance.into(), + orchestration: "Transition".into(), + input: "".into(), + version: Some("1.0.0".into()), + execution_id: 1, + parent_instance: parent, + parent_id: Some(42), + parent_execution_id: Some(1), + }; + common::seed_history_turn( + provider.as_ref(), + start_item(None), + 1, + vec![Event::with_event_id( + 1, + instance, + 1, + None, + EventKind::OrchestrationStarted { + name: "Transition".into(), + version: "1.0.0".into(), + input: "".into(), + parent_instance: None, + parent_id: None, + parent_execution_id: None, + carry_forward_events: None, + initial_custom_status: None, + }, + )], + vec![start_item(Some("incoming-parent".into()))], + ExecutionMetadata { + orchestration_name: Some("Transition".into()), + orchestration_version: Some("1.0.0".into()), + ..Default::default() + }, + ) + .await; + provider.fail_next_orchestration_ack(); + let runtime = Runtime::start_with_options( + provider.clone(), + ActivityRegistry::builder().build(), + OrchestrationRegistry::builder() + .register("Transition", |_: OrchestrationContext, _: String| async { + Ok("done".into()) + }) + .build(), + RuntimeOptions { + orchestration_concurrency: 1, + ..Default::default() + }, + ) + .await; + let result = Client::new(provider.clone()) + .wait_for_orchestration(instance, Duration::from_secs(10)) + .await; + runtime.shutdown(None).await; + assert!(matches!( + result.expect("failure commit"), + OrchestrationStatus::Failed { .. } + )); + assert_eq!( + provider.failure_commit_collision_notifications(), + 1, + "the fallback transaction must atomically notify the colliding parent" + ); +} + +#[tokio::test] +async fn failed_can_abandon_leaves_waiting_inputs_for_lease_redelivery() { + let sqlite = Arc::new(SqliteProvider::new_in_memory().await.expect("test store")); + let provider = Arc::new(PoisonInjectingProvider::new(sqlite)); + let instance = "CAN-abandon-failure"; + seed_waiting_can(provider.as_ref(), instance, "Transition", late_input(instance)).await; + // Seeding fetched the instance once; count only the runtime's deliveries. + let seeded = provider.orchestration_fetches(instance).len(); + provider.fail_orchestration_abandon_persistently(); + let registry = OrchestrationRegistry::builder() + .register("Transition", |ctx: OrchestrationContext, _: String| async move { + assert_eq!(ctx.execution_id(), 2); + Ok(ctx.dequeue_event("q").await) + }) + .build(); + let runtime = Runtime::start_with_options( + provider.clone(), + ActivityRegistry::builder().build(), + registry, + RuntimeOptions { + orchestration_concurrency: 2, + orchestrator_lock_timeout: Duration::from_millis(300), + orchestrator_lock_renewal_buffer: Duration::from_millis(100), + unregistered_backoff: UnregisteredBackoffConfig { + base_delay: Duration::from_millis(50), + max_delay: Duration::from_millis(200), + }, + dispatcher_min_poll_interval: Duration::from_millis(10), + ..Default::default() + }, + ) + .await; + // An abandon that fails is left to the lease: it expires and the same input is + // fetched again under a new lock, attempt kept. + let deliveries = tokio::time::timeout(Duration::from_secs(10), async { + let mut tick = tokio::time::interval(Duration::from_millis(20)); + loop { + let fetches = provider.orchestration_fetches(instance); + if fetches.len() >= seeded + 2 { + break fetches[seeded..].to_vec(); + } + tick.tick().await; + } + }) + .await + .expect("the waiting input must be redelivered after its lease expires"); + assert_ne!(deliveries[0].0, deliveries[1].0, "redelivery must use a new lock"); + assert!( + deliveries[1].1 > deliveries[0].1, + "the failed delivery keeps its attempt: {deliveries:?}" + ); + assert!( + provider.failed_abandon_calls() >= 2, + "both deliveries must have failed to abandon" + ); + let history = provider.read(instance).await.expect("history during failure"); + assert!( + !history + .iter() + .any(|event| matches!(event.kind, EventKind::OrchestrationFailed { .. })), + "failed abandons must not poison a waiting input" + ); + provider.clear_injections(); + enqueue_successor(provider.as_ref(), instance, "Transition", None).await; + let result = Client::new(provider.clone()) + .wait_for_orchestration(instance, Duration::from_secs(10)) + .await; + runtime.shutdown(None).await; + assert!( + matches!(result.expect("waiting input must remain recoverable"), + OrchestrationStatus::Completed { ref output, .. } if output == "late"), + "waiting input must reach the successor after abandon recovery" + ); + let old = provider.read_with_execution(instance, 1).await.expect("old history"); + assert_eq!(old.len(), 2, "deferral must not mutate execution 1"); +} + +fn late_input(instance: &str) -> WorkItem { + WorkItem::QueueMessage { + instance: instance.into(), + name: "q".into(), + data: "late".into(), + } +} + +async fn seed_waiting_can(provider: &PoisonInjectingProvider, instance: &str, name: &str, waiting: WorkItem) { + common::seed_history_turn( + provider, + WorkItem::StartOrchestration { + instance: instance.into(), + orchestration: name.into(), + input: "old".into(), + version: Some("1.0.0".into()), + execution_id: 1, + parent_instance: None, + parent_id: None, + parent_execution_id: None, + }, + 1, + vec![ + Event::with_event_id( + 1, + instance, + 1, + None, + EventKind::OrchestrationStarted { + name: name.into(), + version: "1.0.0".into(), + input: "old".into(), + parent_instance: None, + parent_id: None, + parent_execution_id: None, + carry_forward_events: None, + initial_custom_status: None, + }, + ), + Event::with_event_id( + 2, + instance, + 1, + None, + EventKind::OrchestrationContinuedAsNew { input: "next".into() }, + ), + ], + vec![waiting], + ExecutionMetadata { + status: Some("ContinuedAsNew".into()), + ..Default::default() + }, + ) + .await; +} + +async fn enqueue_successor(provider: &PoisonInjectingProvider, instance: &str, name: &str, delay: Option) { + provider + .enqueue_for_orchestrator( + WorkItem::ContinueAsNew { + instance: instance.into(), + orchestration: name.into(), + input: "next".into(), + version: Some("1.0.0".into()), + parent_instance: None, + parent_id: None, + parent_execution_id: None, + carry_forward_events: vec![], + initial_custom_status: None, + }, + delay, + ) + .await + .expect("delayed existing successor"); +} diff --git a/tests/cancellation_corner_cases.rs b/tests/cancellation_corner_cases.rs new file mode 100644 index 0000000..9aa36ac --- /dev/null +++ b/tests/cancellation_corner_cases.rs @@ -0,0 +1,3024 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::clone_on_ref_ptr)] + +#[path = "replay_engine/helpers.rs"] +#[allow(dead_code)] +mod helpers; + +#[path = "common/cancellation_history.rs"] +mod cancellation_history; + +use async_trait::async_trait; +use duroxide::providers::WorkItem; +use duroxide::runtime::replay_engine::TurnResult; +use duroxide::{Action, Event, EventKind, OrchestrationContext, OrchestrationHandler}; +use helpers::*; +use std::future::Future; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +const RUNTIME_DEADLINE: Duration = Duration::from_secs(30); +const HISTORY_DEADLINE_MS: u64 = 30_000; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Arm { + Q, + Q2, + Signal, + Signal2, + Timer, + Activity, + Child, +} + +// Contract-specific expectations belong here, not in public provider validation. +// Signals require a bound open slot. Same-batch signals before binding are +// early; an unpolled dropped holder discards its assigned stale answer. +fn assert_signal_contract_read(observations: &[Observation], fresh: &str) { + let expected = fresh; + assert!( + observations + .last() + .unwrap() + .live_reads + .contains(&Read::Signal(expected.into())), + "bound-slot signal contract expected {expected}: {observations:#?}" + ); +} + +#[test] +fn ms1_positional_barrier_delivers_fresh_signal_and_replays() { + for stamp in ["0.1.30", "0.1.31"] { + let observations = record_and_replay( + stamp, + Handler::new(Shape::PositionalBarrier, 1, vec![]), + &[ + vec![Completion::Timer(0), Completion::Signal("a")], + vec![Completion::Activity(0)], + vec![Completion::Signal("b")], + ], + ); + if stamp == "0.1.31" { + assert_stable(&observations, "MS1 fixed"); + assert_signal_contract_read(&observations, "b"); + assert!( + observations + .last() + .unwrap() + .live_reads + .contains(&Read::Signal("b".into())) + ); + assert!( + !observations + .last() + .unwrap() + .live_reads + .contains(&Read::Signal("a".into())) + ); + } else { + assert!( + observations + .iter() + .any(|o| o.replay.contains("history schedule but no emitted action")) + ); + } + } +} + +#[test] +fn ms3_resolve_at_creation_discards_dropped_slots_old_value() { + for stamp in ["0.1.30", "0.1.31"] { + let observations = record_and_replay( + stamp, + Handler::new(Shape::ResolveAtCreation, 1, vec![]), + &[ + vec![Completion::Signal("a"), Completion::Activity(0)], + vec![Completion::Activity(1)], + vec![Completion::Signal("b")], + ], + ); + if stamp == "0.1.31" { + assert_stable(&observations, "MS3 fixed"); + assert_signal_contract_read(&observations, "b"); + assert!( + observations + .last() + .unwrap() + .live_reads + .contains(&Read::Signal("b".into())) + ); + } else { + assert!( + observations + .iter() + .any(|o| o.replay.contains("history schedule but no emitted action")) + ); + } + } +} + +#[tokio::test] +async fn ms4_sqlite_continue_as_new_preserves_unconsumed_queue_messages() { + use duroxide::runtime::registry::ActivityRegistry; + use duroxide::runtime::{Runtime, RuntimeOptions}; + use duroxide::{Client, OrchestrationRegistry, OrchestrationStatus}; + for stamp in ["0.1.30", "0.1.31"] { + for (drain, held) in [(false, false), (true, false), (false, true), (true, true)] { + let (store, _temporary) = common::create_sqlite_store_disk().await; + let mut start = started_event(1); + start.duroxide_version = stamp.into(); + if let EventKind::OrchestrationStarted { input, .. } = &mut start.kind { + *input = drain.to_string(); + } + common::seed_history_turn( + store.as_ref(), + WorkItem::StartOrchestration { + instance: TEST_INSTANCE.into(), + orchestration: TEST_ORCH_NAME.into(), + input: drain.to_string(), + version: Some(TEST_ORCH_VERSION.into()), + parent_instance: None, + parent_id: None, + parent_execution_id: None, + execution_id: 1, + }, + 1, + vec![ + start, + Event::with_event_id( + 2, + TEST_INSTANCE, + 1, + None, + EventKind::QueueSubscribed { name: "q".into() }, + ), + Event::with_event_id(3, TEST_INSTANCE, 1, None, EventKind::TimerCreated { fire_at_ms: 1000 }), + ], + if drain { + vec![ + timer_fired_msg(3, 1000), + WorkItem::QueueMessage { + instance: TEST_INSTANCE.into(), + name: "q".into(), + data: "m1".into(), + }, + WorkItem::QueueMessage { + instance: TEST_INSTANCE.into(), + name: "q".into(), + data: "m2".into(), + }, + ] + } else { + vec![ + timer_fired_msg(3, 1000), + WorkItem::QueueMessage { + instance: TEST_INSTANCE.into(), + name: "q".into(), + data: "m1".into(), + }, + ] + }, + duroxide::providers::ExecutionMetadata { + orchestration_name: Some(TEST_ORCH_NAME.into()), + orchestration_version: Some(TEST_ORCH_VERSION.into()), + ..Default::default() + }, + ) + .await; + let registry = OrchestrationRegistry::builder() + .register( + TEST_ORCH_NAME, + move |ctx: OrchestrationContext, input: String| async move { + if ctx.execution_id() == 1 { + if held { + let mut wait = ctx.dequeue_event("q"); + let _ = ctx + .select2(&mut wait, ctx.schedule_timer(Duration::from_millis(10))) + .await; + if drain { + ctx.dequeue_event("q").await; + } + return ctx.continue_as_new("successor").await; + } + let _ = ctx + .select2(ctx.dequeue_event("q"), ctx.schedule_timer(Duration::from_millis(10))) + .await; + if input == "true" { + ctx.dequeue_event("q").await; + } + ctx.continue_as_new("successor").await + } else { + let a = ctx.dequeue_event("q").await; + Ok(a) + } + }, + ) + .build(); + let runtime = Runtime::start_with_options( + store.clone(), + ActivityRegistry::builder().build(), + registry, + RuntimeOptions { + dispatcher_min_poll_interval: Duration::from_millis(5), + ..Default::default() + }, + ) + .await; + let client = Client::new(store.clone()); + assert!( + common::wait_for_history( + store.clone(), + TEST_INSTANCE, + |history| history.first().is_some_and(|event| event.execution_id == 2), + HISTORY_DEADLINE_MS + ) + .await + ); + if stamp == "0.1.30" { + client.enqueue_event(TEST_INSTANCE, "q", "fresh").await.unwrap(); + } + let result = client + .wait_for_orchestration(TEST_INSTANCE, RUNTIME_DEADLINE) + .await + .unwrap(); + let expected = if stamp == "0.1.30" { + "fresh" + } else if drain && !held { + "m2" + } else { + "m1" + }; + assert!( + matches!(&result,OrchestrationStatus::Completed{output,..}if output==expected), + "stamp={stamp} drain={drain}: {result:?}" + ); + let old = client.read_execution_history(TEST_INSTANCE, 1).await.unwrap(); + let cancels = old + .iter() + .filter(|event| { + matches!(&event.kind, + EventKind::QueueSubscriptionCancelled{reason}if reason=="dropped_future") + }) + .count(); + assert_eq!(cancels, 0, "continue-as-new writes no dropped_future rows"); + runtime.shutdown(None).await; + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum Read { + Queue(&'static str, String), + Signal(String), + Signal2(String), + Timer, + Activity(String), + Child(String), +} + +type Branch<'a> = Pin + Send + 'a>>; + +struct TrackedBranch<'a> { + inner: Branch<'a>, + arm: Arm, + index: Option, + ready: bool, + lifecycle: Arc>>, +} +impl Future for TrackedBranch<'_> { + type Output = Read; + fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll { + let this = self.get_mut(); + if this.index.is_none() { + let mut log = this.lifecycle.lock().unwrap(); + this.index = Some(log.len()); + log.push((this.arm, false)); + } + let result = this.inner.as_mut().poll(cx); + this.ready = result.is_ready(); + result + } +} +impl Drop for TrackedBranch<'_> { + fn drop(&mut self) { + if !self.ready + && let Some(index) = self.index + { + self.lifecycle.lock().unwrap()[index].1 = true; + } + } +} + +#[derive(Clone, Debug)] +enum Shape { + Race(Vec), + PartialJoin, + Nested, + UnboundDrop, + UnboundImmediate, + BoundDrop, + SignalBefore, + Timers, + Continue, + PositionalContinue, + PositionalBarrier, + ResolveAtCreation, + HeldContinue(bool), + QuiescenceDrop, + ContinueWithOtherDrops, + WrongNameBeforeWait, + UnboundSignalDrop, + UnboundSignalDropBarrier, + HeldSignalWait(bool), + LazySignalCompetition, + DeadlineSignals, +} + +struct Handler { + shape: Shape, + iterations: usize, + drain: Vec, + reads: Arc>>, + eager: bool, + expected_queue_counts: Option<(usize, usize)>, + lifecycle: Arc>>, + logical_cancellations: Arc>>, +} + +impl Handler { + fn new(shape: Shape, iterations: usize, drain: Vec) -> Arc { + Arc::new(Self { + shape, + iterations, + drain, + reads: Arc::new(Mutex::new(Vec::new())), + eager: false, + expected_queue_counts: None, + lifecycle: Default::default(), + logical_cancellations: Default::default(), + }) + } + + fn branch<'a>(&'a self, ctx: &'a OrchestrationContext, arm: Arm) -> Branch<'a> { + let inner = self.branch_inner(ctx, arm); + if !matches!(self.shape, Shape::Race(_)) { + return inner; + } + let index = if self.eager { + let mut log = self.lifecycle.lock().unwrap(); + let index = log.len(); + log.push((arm, false)); + Some(index) + } else { + None + }; + Box::pin(TrackedBranch { + inner, + arm, + index, + ready: false, + lifecycle: self.lifecycle.clone(), + }) + } + + fn branch_inner<'a>(&'a self, ctx: &'a OrchestrationContext, arm: Arm) -> Branch<'a> { + if self.eager { + let reads = self.reads.clone(); + let record = move |read: Read| { + reads.lock().unwrap().push(read.clone()); + read + }; + return match arm { + Arm::Q => Box::pin(ctx.dequeue_event("q").map(move |value| record(Read::Queue("q", value)))), + Arm::Q2 => Box::pin( + ctx.dequeue_event("q2") + .map(move |value| record(Read::Queue("q2", value))), + ), + Arm::Signal => Box::pin(ctx.schedule_wait("s").map(move |value| record(Read::Signal(value)))), + Arm::Signal2 => Box::pin(ctx.schedule_wait("s2").map(move |value| record(Read::Signal2(value)))), + Arm::Timer => Box::pin( + ctx.schedule_timer(Duration::from_millis(10)) + .map(move |()| record(Read::Timer)), + ), + Arm::Activity => Box::pin( + ctx.schedule_activity("a", "") + .map(move |value| record(Read::Activity(value.unwrap()))), + ), + Arm::Child => Box::pin( + ctx.schedule_sub_orchestration("child", "") + .map(move |value| record(Read::Child(value.unwrap()))), + ), + }; + } + Box::pin(async move { + let read = match arm { + Arm::Q => Read::Queue("q", ctx.dequeue_event("q").await), + Arm::Q2 => Read::Queue("q2", ctx.dequeue_event("q2").await), + Arm::Signal => Read::Signal(ctx.schedule_wait("s").await), + Arm::Signal2 => Read::Signal2(ctx.schedule_wait("s2").await), + Arm::Timer => { + ctx.schedule_timer(Duration::from_millis(10)).await; + Read::Timer + } + Arm::Activity => Read::Activity(ctx.schedule_activity("a", "").await.unwrap()), + Arm::Child => Read::Child(ctx.schedule_sub_orchestration("child", "").await.unwrap()), + }; + self.reads.lock().unwrap().push(read.clone()); + read + }) + } + + async fn race(&self, ctx: &OrchestrationContext, arms: &[Arm]) -> Read { + let winner = match arms { + [a] => self.branch(ctx, *a).await, + [a, b] => { + ctx.select2(self.branch(ctx, *a), self.branch(ctx, *b)) + .await + .into_tuple() + .1 + } + [a, b, c] => { + ctx.select3(self.branch(ctx, *a), self.branch(ctx, *b), self.branch(ctx, *c)) + .await + .into_tuple() + .1 + } + _ => panic!("one to three arms required"), + }; + // Freeze logical drops here. Disposing the in-memory replay future at + // the turn boundary must not masquerade as orchestration cancellation. + if matches!(self.shape, Shape::Race(_)) { + *self.logical_cancellations.lock().unwrap() = self + .lifecycle + .lock() + .unwrap() + .iter() + .enumerate() + .filter_map(|(index, (arm, cancelled))| (*cancelled && *arm != Arm::Timer).then_some(index)) + .collect(); + } + winner + } +} + +#[async_trait] +impl OrchestrationHandler for Handler { + async fn invoke(&self, ctx: OrchestrationContext, _: String) -> Result { + self.reads.lock().unwrap().clear(); + self.lifecycle.lock().unwrap().clear(); + self.logical_cancellations.lock().unwrap().clear(); + for _ in 0..self.iterations { + match &self.shape { + Shape::Race(arms) => { + self.race(&ctx, arms).await; + } + Shape::PartialJoin => { + let join = ctx.join2(self.branch(&ctx, Arm::Q), self.branch(&ctx, Arm::Q)); + let _ = ctx.select2(join, self.branch(&ctx, Arm::Timer)).await; + } + Shape::Nested => { + let inner = ctx.select2(self.branch(&ctx, Arm::Q), self.branch(&ctx, Arm::Timer)); + let _ = ctx + .select2(inner, async { + ctx.schedule_timer(Duration::from_millis(100)).await; + self.reads.lock().unwrap().push(Read::Timer); + }) + .await; + } + Shape::UnboundDrop => { + drop(ctx.dequeue_event("q")); + self.branch(&ctx, Arm::Timer).await; + } + Shape::UnboundImmediate => { + drop(ctx.dequeue_event("q")); + self.branch(&ctx, Arm::Q).await; + } + Shape::BoundDrop => { + let wait = ctx.dequeue_event("q"); + self.branch(&ctx, Arm::Timer).await; + drop(wait); + } + Shape::SignalBefore => { + self.branch(&ctx, Arm::Timer).await; + self.branch(&ctx, Arm::Signal).await; + } + Shape::Timers => { + let _ = ctx + .select2( + ctx.schedule_timer(Duration::from_millis(10)), + ctx.schedule_timer(Duration::from_millis(100)), + ) + .await; + } + Shape::Continue => { + let _ = ctx + .select2(ctx.dequeue_event("q"), ctx.schedule_timer(Duration::from_millis(10))) + .await; + return ctx.continue_as_new("next").await; + } + Shape::PositionalContinue => { + let _ = ctx + .select2(ctx.schedule_wait("s"), ctx.schedule_timer(Duration::from_millis(10))) + .await; + return ctx.continue_as_new("next").await; + } + Shape::PositionalBarrier => { + self.race(&ctx, &[Arm::Signal, Arm::Timer]).await; + ctx.schedule_activity("tick", "").await?; + self.branch(&ctx, Arm::Signal).await; + } + Shape::ResolveAtCreation => { + let wait = ctx.schedule_wait("s"); + ctx.schedule_activity("work", "").await?; + drop(wait); + ctx.schedule_activity("work2", "").await?; + self.branch(&ctx, Arm::Signal).await; + } + Shape::HeldContinue(read_newer) => { + let mut held = ctx.dequeue_event("q"); + let _ = ctx + .select2(&mut held, ctx.schedule_timer(Duration::from_millis(10))) + .await; + if *read_newer { + self.branch(&ctx, Arm::Q).await; + } + return ctx.continue_as_new("next").await; + } + Shape::QuiescenceDrop => { + let held = ctx.dequeue_event("q2"); + self.branch(&ctx, Arm::Q).await; + drop(held); + self.branch(&ctx, Arm::Q2).await; + } + Shape::ContinueWithOtherDrops => { + let activity = ctx.schedule_activity("held", ""); + let child = ctx.schedule_sub_orchestration("held-child", ""); + let queue = ctx.dequeue_event("q"); + let signal = ctx.schedule_wait("s"); + ctx.schedule_timer(Duration::from_millis(10)).await; + drop((activity, child, queue, signal)); + return ctx.continue_as_new("next").await; + } + Shape::WrongNameBeforeWait => { + self.branch(&ctx, Arm::Timer).await; + self.branch(&ctx, Arm::Signal).await; + self.branch(&ctx, Arm::Activity).await; + self.branch(&ctx, Arm::Signal2).await; + } + Shape::UnboundSignalDrop | Shape::UnboundSignalDropBarrier => { + self.branch(&ctx, Arm::Timer).await; + drop(ctx.schedule_wait("s")); + if matches!(self.shape, Shape::UnboundSignalDropBarrier) { + self.branch(&ctx, Arm::Activity).await; + } + self.branch(&ctx, Arm::Signal).await; + } + Shape::HeldSignalWait(drop_held) => { + let held = ctx.schedule_wait("s"); + self.branch(&ctx, Arm::Timer).await; + if *drop_held { + drop(held); + self.branch(&ctx, Arm::Signal).await; + } else { + let value = held.await; + self.reads.lock().unwrap().push(Read::Signal(value)); + } + } + Shape::LazySignalCompetition => { + let validate = ctx.schedule_activity("validate", ""); + let prepare = ctx.schedule_activity("prepare", ""); + let _ = ctx + .select2( + async { + validate.await.unwrap(); + self.branch(&ctx, Arm::Signal).await + }, + async { + prepare.await.unwrap(); + self.branch(&ctx, Arm::Signal2).await + }, + ) + .await; + } + Shape::DeadlineSignals => { + let mut deadline = ctx.schedule_timer(Duration::from_secs(30)); + loop { + if matches!( + ctx.select2(self.branch(&ctx, Arm::Signal), &mut deadline).await, + duroxide::Either2::Second(()) + ) { + break; + } + } + self.branch(&ctx, Arm::Signal).await; + } + } + } + for arm in &self.drain { + self.branch(&ctx, *arm).await; + } + if let Some((q_count, q2_count)) = self.expected_queue_counts { + for (arm, queue, total) in [(Arm::Q, "q", q_count), (Arm::Q2, "q2", q2_count)] { + while self + .reads + .lock() + .unwrap() + .iter() + .filter(|read| matches!(read,Read::Queue(name,_)if *name==queue)) + .count() + < total + { + self.branch(&ctx, arm).await; + } + } + } + // Persist consumption in an ordinary activity input so replay must match it, + // even when the handler has not yet completed. + let payload = format!("{:?}", self.reads.lock().unwrap()); + ctx.schedule_activity("checkpoint", &payload).await?; + Ok(payload) + } +} + +#[derive(Clone, Debug)] +enum Completion { + Q(&'static str), + Q2(&'static str), + Signal(&'static str), + Signal2(&'static str), + Timer(usize), + Activity(usize), + Child(usize), +} + +fn make_messages(history: &[Event], batch: &[Completion]) -> Vec { + batch + .iter() + .map(|completion| match completion { + Completion::Q(data) | Completion::Q2(data) => WorkItem::QueueMessage { + instance: TEST_INSTANCE.into(), + name: if matches!(completion, Completion::Q(_)) { + "q" + } else { + "q2" + } + .into(), + data: (*data).into(), + }, + Completion::Signal(data) => external_raised_msg("s", data), + Completion::Signal2(data) => external_raised_msg("s2", data), + Completion::Timer(index) => { + let event = history + .iter() + .filter(|e| matches!(e.kind, EventKind::TimerCreated { .. })) + .nth(*index) + .unwrap(); + let EventKind::TimerCreated { fire_at_ms } = event.kind else { + unreachable!() + }; + timer_fired_msg(event.event_id, fire_at_ms) + } + Completion::Activity(index) => { + let event = history + .iter() + .filter(|e| matches!(e.kind, EventKind::ActivityScheduled { .. })) + .nth(*index) + .unwrap(); + activity_completed_msg(event.event_id, "ok") + } + Completion::Child(index) => { + let event = history + .iter() + .filter(|e| matches!(e.kind, EventKind::SubOrchestrationScheduled { .. })) + .nth(*index) + .unwrap(); + sub_orch_completed_msg(event.event_id, "ok") + } + }) + .collect() +} + +fn outcome(result: &TurnResult) -> String { + match result { + TurnResult::Failed(error) => format!("Failed: {}", error.display_message()), + other => format!("{other:?}"), + } +} + +fn action_shapes(actions: &[Action]) -> Vec { + actions + .iter() + .map(|action| match action { + Action::CreateTimer { + scheduling_event_id, .. + } => format!("Timer:{scheduling_event_id}"), + _ => format!("{action:?}"), + }) + .collect() +} + +#[derive(Clone, Debug)] +struct Observation { + live: String, + replay: String, + live_reads: Vec, + logical_cancellations: Option>, + replay_reads: Vec, + live_actions: Vec, + replay_actions: Vec, + replay_delta: Vec, + history: Vec, +} + +fn record_and_replay(stamp: &str, handler: Arc, batches: &[Vec]) -> Vec { + record_with_initial_batch(stamp, handler, batches, &[]) +} + +fn record_with_initial_batch( + stamp: &str, + handler: Arc, + batches: &[Vec], + initial: &[Completion], +) -> Vec { + record_with_fresh_completions(stamp, handler, batches, initial, false) +} + +fn record_with_fresh_completions( + stamp: &str, + handler: Arc, + batches: &[Vec], + initial: &[Completion], + complete_last: bool, +) -> Vec { + let mut start = started_event(1); + start.duroxide_version = stamp.into(); + let mut history = vec![start]; + let mut observations = Vec::new(); + let initial = initial.to_vec(); + for (turn, batch) in std::iter::once(&initial).chain(batches.iter()).enumerate() { + let mut engine = create_engine(history.clone()); + let mut completions = batch.clone(); + if complete_last && turn == batches.len() { + let finished: std::collections::HashSet<_> = history + .iter() + .filter(|event| { + matches!( + event.kind, + EventKind::TimerFired { .. } + | EventKind::ActivityCompleted { .. } + | EventKind::ActivityFailed { .. } + | EventKind::SubOrchestrationCompleted { .. } + | EventKind::SubOrchestrationFailed { .. } + | EventKind::ActivityCancelRequested { .. } + | EventKind::SubOrchestrationCancelRequested { .. } + ) + }) + .filter_map(|event| event.source_event_id) + .collect(); + for arm in [Arm::Timer, Arm::Activity, Arm::Child] { + for (index, event) in history + .iter() + .filter(|event| match arm { + Arm::Timer => matches!(event.kind, EventKind::TimerCreated { .. }), + Arm::Activity => matches!(event.kind, EventKind::ActivityScheduled { .. }), + Arm::Child => matches!(event.kind, EventKind::SubOrchestrationScheduled { .. }), + _ => unreachable!(), + }) + .enumerate() + { + let already_supplied = completions.iter().any(|completion| match (arm, completion) { + (Arm::Timer, Completion::Timer(i)) + | (Arm::Activity, Completion::Activity(i)) + | (Arm::Child, Completion::Child(i)) => *i == index, + _ => false, + }); + if !finished.contains(&event.event_id) && !already_supplied { + completions.push(match arm { + Arm::Timer => Completion::Timer(index), + Arm::Activity => Completion::Activity(index), + Arm::Child => Completion::Child(index), + _ => unreachable!(), + }); + } + } + } + } + engine.prep_completions(make_messages(&history, &completions)); + let live = execute(&mut engine, handler.clone()); + let live_reads = handler.reads.lock().unwrap().clone(); + let logical_cancellations = + matches!(handler.shape, Shape::Race(_)).then(|| handler.logical_cancellations.lock().unwrap().clone()); + history.extend_from_slice(engine.history_delta()); + let mut replay = create_engine(history.clone()); + let replay_result = execute(&mut replay, handler.clone()); + let replay_reads = handler.reads.lock().unwrap().clone(); + observations.push(Observation { + live: outcome(&live), + replay: outcome(&replay_result), + live_reads, + logical_cancellations, + replay_reads, + live_actions: action_shapes(engine.pending_actions()), + replay_actions: action_shapes(replay.pending_actions()), + replay_delta: replay.history_delta().to_vec(), + history: history.clone(), + }); + if matches!( + live, + TurnResult::Failed(_) | TurnResult::Completed(_) | TurnResult::ContinueAsNew { .. } + ) { + break; + } + } + observations +} + +fn assert_stable(observations: &[Observation], scenario: &str) { + for observation in observations { + assert!( + !observation.live.starts_with("Failed"), + "{scenario}: live run failed: {observation:#?}" + ); + assert!( + !observation.replay.starts_with("Failed"), + "{scenario}: replay failed: {observation:#?}" + ); + assert_eq!(observation.live, observation.replay, "{scenario}: {observation:#?}"); + assert_eq!( + observation.live_reads, observation.replay_reads, + "{scenario}: read sequence changed" + ); + assert!( + observation.replay_actions.is_empty(), + "{scenario}: replay emitted new decisions: {observation:#?}" + ); + assert!( + observation.replay_delta.is_empty(), + "{scenario}: replay emitted history: {observation:#?}" + ); + let mut canceled = std::collections::HashSet::new(); + for event in &observation.history { + if matches!(&event.kind, EventKind::QueueSubscriptionCancelled { reason } | EventKind::ExternalSubscribedCancelled { reason } + | EventKind::ActivityCancelRequested {reason} | EventKind::SubOrchestrationCancelRequested {reason} + if reason == "dropped_future") + { + assert!( + canceled.insert(event.source_event_id), + "{scenario}: duplicate cancel breadcrumb" + ); + assert!( + observation + .history + .iter() + .any(|schedule| Some(schedule.event_id) == event.source_event_id), + "{scenario}: cancellation has no bound schedule" + ); + } + } + if let Some(indices) = &observation.logical_cancellations { + let schedules: Vec<_> = observation + .history + .iter() + .filter(|e| match &e.kind { + EventKind::QueueSubscribed { .. } + | EventKind::ExternalSubscribed { .. } + | EventKind::TimerCreated { .. } + | EventKind::SubOrchestrationScheduled { .. } => true, + EventKind::ActivityScheduled { name, .. } => name == "a", + _ => false, + }) + .collect(); + let expected: std::collections::HashSet<_> = indices + .iter() + .map(|index| { + Some( + schedules + .get(*index) + .expect("dropped emitted future was never bound") + .event_id(), + ) + }) + .collect(); + assert_eq!( + canceled, expected, + "{scenario}: dropped-future breadcrumb set is incomplete or unexpected" + ); + } + } +} + +fn queue_values(observations: &[Observation]) -> Vec<(&'static str, String)> { + observations + .last() + .unwrap() + .live_reads + .iter() + .filter_map(|read| { + if let Read::Queue(name, value) = read { + Some((*name, value.clone())) + } else { + None + } + }) + .collect() +} + +fn queue_scenario( + name: &str, + shape: Shape, + drain: Vec, + batches: Vec>, + expected: Vec<(&'static str, &'static str)>, +) { + for stamp in ["0.1.30", "0.1.31"] { + let observations = record_and_replay(stamp, Handler::new(shape.clone(), 1, drain.clone()), &batches); + let stable = observations + .iter() + .all(|o| o.live == o.replay && o.live_reads == o.replay_reads && o.replay_actions.is_empty()); + println!( + "SCENARIO {name} stamp={stamp} stable={stable} outcome={}", + observations.last().unwrap().replay + ); + if stamp == "0.1.31" { + assert_stable(&observations, name); + let actual = queue_values(&observations); + assert_eq!( + actual, + expected.iter().map(|(q, v)| (*q, (*v).into())).collect::>(), + "{name}: FIFO consumption" + ); + } else { + assert_eq!( + observations.last().unwrap().live, + "Continue", + "{name}: changed legacy live outcome" + ); + assert_eq!( + queue_values(&observations), + expected.iter().map(|(q, v)| (*q, (*v).into())).collect::>(), + "{name}: changed legacy live reads" + ); + let expected_replay = if name == "T1" || name == "Q1/count0" { + "Continue" + } else { + "Failed: nondeterministic: history schedule but no emitted action" + }; + assert_eq!( + observations.last().unwrap().replay, + expected_replay, + "{name}: changed retained legacy replay outcome" + ); + } + } +} + +#[test] +fn q2_first_dequeue_wins_dropped_sibling_preserves_second_message() { + queue_scenario( + "Q2", + Shape::Race(vec![Arm::Q, Arm::Q, Arm::Timer]), + vec![Arm::Q], + vec![vec![Completion::Q("first"), Completion::Q("second")]], + vec![("q", "first"), ("q", "second")], + ); +} + +#[test] +fn q3_same_queue_select2_fifo() { + queue_scenario( + "Q3", + Shape::Race(vec![Arm::Q, Arm::Q]), + vec![Arm::Q], + vec![vec![Completion::Q("first"), Completion::Q("second")]], + vec![("q", "first"), ("q", "second")], + ); +} + +#[test] +fn q4_cross_queue_cancellations_are_independent() { + queue_scenario( + "Q4", + Shape::Race(vec![Arm::Q, Arm::Q2, Arm::Timer]), + vec![Arm::Q, Arm::Q2], + vec![vec![ + Completion::Timer(0), + Completion::Q("first"), + Completion::Q2("second"), + ]], + vec![("q", "first"), ("q2", "second")], + ); +} + +#[test] +fn q5_partial_join_resolved_child_drop_does_not_duplicate() { + queue_scenario( + "Q5", + Shape::PartialJoin, + vec![Arm::Q], + vec![vec![ + Completion::Q("first"), + Completion::Timer(0), + Completion::Q("second"), + ]], + vec![("q", "first"), ("q", "second")], + ); +} + +#[test] +fn q6_nested_select_queue_and_two_timers() { + for outer_first in [false, true] { + let timer = if outer_first { 1 } else { 0 }; + queue_scenario( + "Q6", + Shape::Nested, + vec![Arm::Q], + vec![vec![Completion::Timer(timer), Completion::Q("first")]], + vec![("q", "first")], + ); + } +} + +#[test] +fn q7_queue_activity_timer_cancellation() { + queue_scenario( + "Q7", + Shape::Race(vec![Arm::Q, Arm::Activity, Arm::Timer]), + vec![Arm::Q], + vec![vec![ + Completion::Timer(0), + Completion::Activity(0), + Completion::Q("first"), + ]], + vec![("q", "first")], + ); +} + +#[test] +fn q8_queue_child_completion_same_batch() { + queue_scenario( + "Q8", + Shape::Race(vec![Arm::Q, Arm::Child]), + vec![Arm::Q], + vec![vec![Completion::Child(0), Completion::Q("first")]], + vec![("q", "first")], + ); +} + +#[test] +fn q9_unbound_drop_followed_by_message() { + for stamp in ["0.1.30", "0.1.31"] { + let obs = record_with_initial_batch( + stamp, + Handler::new(Shape::UnboundImmediate, 1, vec![]), + &[], + &[Completion::Q("first"), Completion::Q("second")], + ); + assert_stable(&obs, &format!("Q9 initial buffered bind/{stamp}")); + assert_eq!( + queue_values(&obs), + vec![("q", if stamp == "0.1.31" { "first" } else { "second" }.into())] + ); + } +} + +#[test] +fn q10_explicit_bound_drop_followed_by_message() { + queue_scenario( + "Q10", + Shape::BoundDrop, + vec![Arm::Q], + vec![vec![Completion::Timer(0), Completion::Q("first")]], + vec![("q", "first")], + ); +} + +#[test] +fn q11_catch_up_batch_fifty_arrivals() { + // Unique data plus explicit queued-read tracing checks catch-up order without wall-clock timing. + let messages: Vec<&'static str> = (0..50) + .map(|index| &*Box::leak(index.to_string().into_boxed_str())) + .collect(); + let mut batch = vec![Completion::Timer(0)]; + batch.extend(messages.iter().map(|value| Completion::Q(value))); + queue_scenario( + "Q11", + Shape::Race(vec![Arm::Q, Arm::Timer]), + vec![Arm::Q; 50], + vec![batch], + messages.into_iter().map(|value| ("q", value)).collect(), + ); +} + +#[test] +fn t1_timer_loser_does_not_consume_queue_arrival() { + queue_scenario( + "T1", + Shape::Timers, + vec![Arm::Q], + vec![vec![Completion::Timer(0), Completion::Timer(1), Completion::Q("first")]], + vec![("q", "first")], + ); +} + +#[test] +fn q14_invalid_or_missing_stamp_retains_legacy_policy() { + let mut reference = None; + for stamp in ["0.1.30", "", "not-a-version"] { + let observations = record_and_replay( + stamp, + Handler::new(Shape::Race(vec![Arm::Q, Arm::Q, Arm::Timer]), 1, vec![Arm::Q]), + &[vec![Completion::Timer(0), Completion::Q("first")]], + ); + let actual = observations + .iter() + .map(|o| { + ( + o.live.clone(), + o.replay.clone(), + o.live_reads.clone(), + o.replay_reads.clone(), + o.replay_actions.clone(), + ) + }) + .collect::>(); + if let Some(reference) = &reference { + assert_eq!(reference, &actual); + } else { + reference = Some(actual); + } + } +} + +#[test] +fn ms8_gate_boundary_prerelease_build_and_invalid_are_replay_stable() { + let (logs, _guard) = common::tracing_capture::install_tracing_capture(); + for (stamp, immediate) in [ + ("", false), + ("garbage", false), + ("0.1.30", false), + ("0.1.31", true), + ("0.1.31-rc.1", false), + ("0.1.31+build", true), + ("0.2.0", true), + ] { + logs.lock().unwrap().clear(); + let handler = Handler::new(Shape::Race(vec![Arm::Q, Arm::Q, Arm::Timer]), 1, vec![Arm::Q]); + let observations = record_and_replay( + stamp, + handler.clone(), + &[vec![Completion::Timer(0), Completion::Q("first")]], + ); + let last = observations.last().unwrap(); + if immediate { + assert_stable(&observations, stamp); + assert_eq!(queue_values(&observations), vec![("q", "first".into())]); + } else { + assert!( + last.replay.contains("history schedule but no emitted action"), + "{stamp}: {last:?}" + ); + } + for _ in 0..2 { + let mut replay = create_engine(last.history.clone()); + assert_eq!(last.replay, outcome(&execute(&mut replay, handler.clone()))); + } + assert_eq!( + logs.lock() + .unwrap() + .iter() + .any(|event| event.level == tracing::Level::WARN && event.message.contains("Invalid pinned version")), + stamp.is_empty() || stamp == "garbage", + "warning did not match stamp {stamp:?}" + ); + } +} + +#[test] +fn q14_missing_version_field_is_a_deserialization_error_not_legacy() { + let mut value = serde_json::to_value(started_event(1)).unwrap(); + value.as_object_mut().unwrap().remove("duroxide_version"); + let error = serde_json::from_value::(value).unwrap_err(); + assert!(error.to_string().contains("missing field `duroxide_version`")); +} + +#[tokio::test] +async fn q14_missing_version_is_poisoned_by_sqlite_dispatcher() { + use duroxide::OrchestrationRegistry; + use duroxide::runtime::registry::ActivityRegistry; + use duroxide::runtime::{Runtime, RuntimeOptions}; + let (store, temporary) = common::create_sqlite_store_disk().await; + common::seed_instance_with_pinned_version( + store.as_ref(), + TEST_INSTANCE, + TEST_ORCH_NAME, + semver::Version::new(0, 1, 30), + ) + .await; + let url = format!("sqlite:{}", temporary.path().join("test.db").display()); + let pool = sqlx::SqlitePool::connect(&url).await.unwrap(); + sqlx::query("UPDATE history SET event_data=json_remove(event_data,'$.duroxide_version') WHERE instance_id=?") + .bind(TEST_INSTANCE) + .execute(&pool) + .await + .unwrap(); + let runtime = Runtime::start_with_options( + store.clone(), + ActivityRegistry::builder().build(), + OrchestrationRegistry::builder().build(), + RuntimeOptions { + max_attempts: 0, + dispatcher_min_poll_interval: Duration::from_millis(5), + ..Default::default() + }, + ) + .await; + // Normal history readers must reject the corrupt start. Observe the poison + // terminal through storage instead, with bounded, paced polling. + let failed = tokio::time::timeout(RUNTIME_DEADLINE, async { + let mut tick = tokio::time::interval(Duration::from_millis(20)); + loop { + tick.tick().await; + let output: Option = + sqlx::query_scalar("SELECT output FROM executions WHERE instance_id=? AND status='Failed'") + .bind(TEST_INSTANCE) + .fetch_optional(&pool) + .await + .unwrap() + .flatten(); + if let Some(output) = output { + break output; + } + } + }) + .await + .expect("poison terminal was not persisted"); + assert!( + failed.contains("deserialization") || failed.contains("deserialize"), + "{failed}" + ); + let terminal: String = sqlx::query_scalar("SELECT event_data FROM history WHERE instance_id=? AND event_id=99999") + .bind(TEST_INSTANCE) + .fetch_one(&pool) + .await + .unwrap(); + assert!(terminal.contains("FailedDeserialization")); + runtime.shutdown(None).await; +} + +#[test] +fn ms10_legacy_hazard_remains_detectable_after_upgrade() { + let observations = record_and_replay( + "0.1.30", + Handler::new(Shape::Race(vec![Arm::Q, Arm::Timer]), 1, vec![Arm::Q]), + &[vec![Completion::Timer(0), Completion::Q("first")]], + ); + assert!( + observations + .last() + .unwrap() + .replay + .contains("history schedule but no emitted action") + ); +} + +#[test] +fn ms7_quiescence_drop_is_recorded_and_policy_pinned() { + for stamp in ["0.1.30", "0.1.31"] { + let observations = record_with_initial_batch( + stamp, + Handler::new(Shape::Race(vec![Arm::Q, Arm::Q2]), 1, vec![Arm::Q2]), + &[], + &[Completion::Q("first"), Completion::Q2("other")], + ); + assert_stable(&observations, "MS7 quiescence"); + if stamp == "0.1.31" { + assert_stable(&observations, "MS7 immediate"); + assert_eq!( + queue_values(&observations), + vec![("q", "first".into()), ("q2", "other".into())] + ); + } else { + assert_eq!( + queue_values(&observations), + vec![("q", "first".into())], + "legacy policy must keep the quiescence replacement pending" + ); + } + } +} + +#[test] +fn q12_continue_as_new_retains_policy_and_fresh_execution_selects_its_own() { + for stamp in ["0.1.30", "0.1.31"] { + let handler = Handler::new(Shape::Continue, 1, vec![]); + let mut history = vec![started_event(1)]; + history[0].duroxide_version = stamp.into(); + let mut first = create_engine(history.clone()); + assert_continue(&execute(&mut first, handler.clone())); + history.extend_from_slice(first.history_delta()); + let mut second = create_engine(history.clone()); + second.prep_completions(make_messages(&history, &[Completion::Timer(0), Completion::Q("first")])); + let result = execute(&mut second, handler.clone()); + assert!(matches!(result, TurnResult::ContinueAsNew { .. })); + history.extend_from_slice(second.history_delta()); + let mut replay = create_engine(history); + assert_eq!(outcome(&result), outcome(&execute(&mut replay, handler))); + + // A replacement execution's own start stamp chooses the policy; the + // former execution's stamp does not flow into its queue matching. + let observations = record_and_replay( + "0.1.31", + Handler::new(Shape::Race(vec![Arm::Q, Arm::Timer]), 1, vec![Arm::Q]), + &[vec![Completion::Timer(0), Completion::Q("first")]], + ); + assert_stable(&observations, "Q12/Q13 replacement"); + assert_eq!(queue_values(&observations), vec![("q", "first".into())]); + } +} + +#[test] +fn positional_drop_before_continue_as_new_replays_for_both_policies() { + for stamp in ["0.1.30", "0.1.31"] { + let handler = Handler::new(Shape::PositionalContinue, 1, vec![]); + let observations = record_and_replay( + stamp, + handler, + &[vec![Completion::Timer(0), Completion::Signal("stale")]], + ); + let observation = observations.last().unwrap(); + assert!(observation.live.starts_with("ContinueAsNew")); + assert_eq!(observation.live, observation.replay); + let canceled = observation + .history + .iter() + .filter(|event| { + matches!(&event.kind, + EventKind::ExternalSubscribedCancelled{reason}if reason=="dropped_future") + }) + .count(); + assert_eq!(canceled, 0, "continue-as-new writes no dropped_future rows"); + assert!(observation.replay_delta.is_empty()); + assert_eq!(observation.replay_actions.len(), 1); + assert!(observation.replay_actions[0].starts_with("ContinueAsNew")); + } +} + +#[test] +fn positional_s2_two_waits_timer_same_batch_preserves_policy() { + for stamp in ["0.1.30", "0.1.31"] { + let observations = record_and_replay( + stamp, + Handler::new( + Shape::Race(vec![Arm::Signal, Arm::Signal, Arm::Timer]), + 1, + vec![Arm::Signal], + ), + &[ + vec![Completion::Timer(0), Completion::Signal("first")], + vec![Completion::Signal("second")], + ], + ); + assert_stable(&observations, &format!("S2/{stamp}")); + if stamp == "0.1.31" { + assert_eq!( + observations.last().unwrap().live_reads, + vec![Read::Timer, Read::Signal("second".into())] + ); + } else { + assert_eq!(observations.last().unwrap().live_reads, vec![Read::Timer]); + } + } +} + +#[test] +fn positional_s3_signal_queue_timer_same_batch_preserves_policy() { + for stamp in ["0.1.30", "0.1.31"] { + let observations = record_and_replay( + stamp, + Handler::new( + Shape::Race(vec![Arm::Signal, Arm::Q, Arm::Timer]), + 1, + vec![Arm::Signal, Arm::Q], + ), + &[ + vec![ + Completion::Timer(0), + Completion::Signal("first"), + Completion::Q("first"), + ], + vec![Completion::Signal("second"), Completion::Q("second")], + ], + ); + if stamp == "0.1.31" { + assert_stable(&observations, &format!("S3/{stamp}")); + assert_eq!( + observations.last().unwrap().live_reads, + vec![ + Read::Timer, + Read::Signal("second".into()), + Read::Queue("q", "first".into()) + ] + ); + } else { + assert_stable(&observations, "S3 legacy"); + assert_eq!( + observations.last().unwrap().live_reads, + vec![ + Read::Timer, + Read::Signal("first".into()), + Read::Queue("q", "first".into()) + ] + ); + } + } +} + +#[test] +fn s4_signal_before_subscription_is_dropped_consistently() { + for stamp in ["0.1.30", "0.1.31"] { + let observations = record_and_replay( + stamp, + Handler::new(Shape::SignalBefore, 1, vec![]), + &[ + vec![Completion::Signal("stale"), Completion::Timer(0)], + vec![Completion::Signal("fresh")], + ], + ); + assert_stable(&observations, &format!("S4/{stamp}")); + assert!( + !observations + .last() + .unwrap() + .live_reads + .contains(&Read::Signal("stale".into())) + ); + } +} + +#[test] +fn s5_two_signal_arrivals_match_active_waits_consistently() { + for stamp in ["0.1.30", "0.1.31"] { + let observations = record_and_replay( + stamp, + Handler::new( + Shape::Race(vec![Arm::Signal, Arm::Signal, Arm::Timer]), + 1, + vec![Arm::Signal], + ), + &[vec![Completion::Signal("first"), Completion::Signal("second")]], + ); + assert_stable(&observations, &format!("S5/{stamp}")); + assert_eq!( + observations.last().unwrap().live_reads.first(), + Some(&Read::Signal("first".into())) + ); + assert_eq!( + observations.last().unwrap().live_reads, + vec![Read::Signal("first".into())] + ); + } +} + +#[test] +fn ms13_held_and_nonprefix_continue_as_new_carry_exact_unread_positions() { + for stamp in ["0.1.30", "0.1.31"] { + for read_newer in [false, true] { + let observations = record_and_replay( + stamp, + Handler::new(Shape::HeldContinue(read_newer), 1, vec![]), + &[vec![Completion::Timer(0), Completion::Q("m1"), Completion::Q("m2")]], + ); + let last = observations.last().unwrap(); + assert_eq!(last.live, last.replay, "MS13/{stamp}/{read_newer}"); + assert!(last.replay_delta.is_empty()); + let expected = if stamp == "0.1.30" { + "unconsumed_queue_arrivals: None" + } else if read_newer { + "unconsumed_queue_arrivals: Some([(\"q\", \"m1\")])" + } else { + "unconsumed_queue_arrivals: Some([(\"q\", \"m1\"), (\"q\", \"m2\")])" + }; + assert!(last.live.contains(expected), "{}", last.live); + assert_eq!( + queue_values(&observations), + if read_newer { vec![("q", "m2".into())] } else { vec![] } + ); + } + } +} + +#[test] +fn continue_as_new_exact_unread_positions_preserve_cross_name_arrival_order() { + for stamp in ["0.1.30", "0.1.31"] { + let observations = record_and_replay( + stamp, + Handler::new(Shape::HeldContinue(true), 1, vec![]), + &[vec![ + Completion::Timer(0), + Completion::Q2("other-first"), + Completion::Q("m1"), + Completion::Q2("other-second"), + Completion::Q("m2"), + ]], + ); + let last = observations.last().unwrap(); + assert_eq!(last.live, last.replay); + assert!(last.replay_delta.is_empty()); + assert_eq!(queue_values(&observations), vec![("q", "m2".into())]); + let expected = if stamp == "0.1.30" { + "unconsumed_queue_arrivals: None" + } else { + "unconsumed_queue_arrivals: Some([(\"q2\", \"other-first\"), (\"q\", \"m1\"), (\"q2\", \"other-second\")])" + }; + assert!( + last.live.contains(expected), + "cross-name carry-forward changed: {}", + last.live + ); + } +} + +#[test] +fn ms14_unbound_replacement_drops_same_batch_signal_as_early() { + for stamp in ["0.1.30", "0.1.31"] { + let observations = record_and_replay( + stamp, + Handler::new(Shape::Race(vec![Arm::Signal, Arm::Timer]), 1, vec![Arm::Signal]), + &[ + vec![Completion::Timer(0), Completion::Signal("a")], + vec![Completion::Signal("b")], + ], + ); + assert_stable(&observations, &format!("MS14/{stamp}")); + if stamp == "0.1.31" { + assert_signal_contract_read(&observations, "b"); + } + assert_eq!( + observations.last().unwrap().live_reads, + if stamp == "0.1.31" { + vec![Read::Timer, Read::Signal("b".into())] + } else { + vec![Read::Timer] + } + ); + assert!(observations.last().unwrap().replay_delta.is_empty()); + } +} + +#[test] +fn ms16_cancellation_blocks_are_deterministic_by_type_and_id() { + for (stamp, arm) in ["0.1.30", "0.1.31"] + .into_iter() + .flat_map(|stamp| [Arm::Q, Arm::Signal, Arm::Activity, Arm::Child].map(|arm| (stamp, arm))) + { + let mut baseline = None; + for _ in 0..16 { + let obs = record_and_replay( + stamp, + Handler::new(Shape::Race(vec![arm, arm, Arm::Timer]), 1, vec![]), + &[vec![Completion::Timer(0)]], + ); + assert_stable(&obs, "MS16 cancellation order"); + let ids: Vec = obs + .last() + .unwrap() + .history + .iter() + .filter(|event| { + matches!( + event.kind, + EventKind::QueueSubscriptionCancelled { .. } + | EventKind::ExternalSubscribedCancelled { .. } + | EventKind::ActivityCancelRequested { .. } + | EventKind::SubOrchestrationCancelRequested { .. } + ) + }) + .filter_map(|event| event.source_event_id) + .collect(); + assert!( + ids.len() >= 2 && ids.windows(2).all(|pair| pair[0] < pair[1]), + "{stamp} {arm:?}: dropped-future cancellations must be in ascending id order: {ids:?}" + ); + let normalized: Vec<_> = obs + .last() + .unwrap() + .history + .iter() + .map(|event| { + let mut value = event.clone(); + value.timestamp_ms = 0; + if let EventKind::TimerCreated { fire_at_ms } | EventKind::TimerFired { fire_at_ms } = + &mut value.kind + { + *fire_at_ms = 1; + } + value + }) + .collect(); + if let Some(expected) = &baseline { + assert_eq!(&normalized, expected); + } else { + baseline = Some(normalized); + } + } + } +} + +/// Independent same-type dropped-future breadcrumbs form a contiguous end-of-turn +/// block. The base enumerates them from a HashSet; replay validates source IDs as +/// sets and never polls between them. Permuting only that block preserves its +/// event-ID range and all next decisions, for both execution-pinned policies. +#[test] +fn independent_cancellation_block_permutations_preserve_replay_and_next_turn_for_both_stamps() { + for stamp in ["0.1.30", "0.1.31"] { + let handler = Handler::new(Shape::Race(vec![Arm::Q, Arm::Q, Arm::Timer]), 1, vec![]); + let observations = record_and_replay(stamp, handler.clone(), &[vec![Completion::Timer(0)]]); + assert_stable(&observations, "independent cancellation permutation"); + let original = observations.last().unwrap().history.clone(); + let indices: Vec<_> = original + .iter() + .enumerate() + .filter_map(|(index, event)| { + matches!(&event.kind, EventKind::QueueSubscriptionCancelled { reason } if reason == "dropped_future") + .then_some(index) + }) + .collect(); + assert_eq!(indices.len(), 2); + assert_eq!(indices[1], indices[0] + 1); + let mut permuted = original.clone(); + let ids = (permuted[indices[0]].event_id, permuted[indices[1]].event_id); + permuted.swap(indices[0], indices[1]); + permuted[indices[0]].event_id = ids.0; + permuted[indices[1]].event_id = ids.1; + let mut next_decisions = None; + for history in [original, permuted] { + let mut replay = create_engine(history.clone()); + assert_continue(&execute(&mut replay, handler.clone())); + assert!(replay.history_delta().is_empty()); + assert!(replay.pending_actions().is_empty()); + let mut next = create_engine(history.clone()); + next.prep_completions(make_messages(&history, &[Completion::Activity(0)])); + let result = execute(&mut next, handler.clone()); + let decisions = ( + outcome(&result), + action_shapes(next.pending_actions()), + next.history_delta() + .iter() + .map(|event| (event.event_id, event.source_event_id, event.kind.clone())) + .collect::>(), + ); + if let Some(expected) = &next_decisions { + assert_eq!(&decisions, expected); + } else { + next_decisions = Some(decisions); + } + } + } +} + +#[test] +fn continue_as_new_writes_no_drop_cancellations_or_child_side_effects() { + for stamp in ["0.1.30", "0.1.31"] { + let observations = record_and_replay( + stamp, + Handler::new(Shape::ContinueWithOtherDrops, 1, vec![]), + &[vec![Completion::Timer(0), Completion::Q("m1")]], + ); + let last = observations.last().unwrap(); + assert!(last.live.starts_with("ContinueAsNew")); + assert_eq!(last.live, last.replay); + assert!(last.replay_delta.is_empty()); + assert!(!last.history.iter().any(|event| matches!( + event.kind, + EventKind::ActivityCancelRequested { .. } | EventKind::SubOrchestrationCancelRequested { .. } + ))); + assert_eq!(last.replay_actions.len(), 1); + assert!(last.replay_actions[0].starts_with("ContinueAsNew")); + let wait_cancellations = last + .history + .iter() + .filter(|event| { + matches!( + event.kind, + EventKind::ExternalSubscribedCancelled { .. } | EventKind::QueueSubscriptionCancelled { .. } + ) + }) + .count(); + assert_eq!(wait_cancellations, 0); + } +} + +fn permutations(items: &[Completion]) -> Vec> { + if items.is_empty() { + return vec![Vec::new()]; + } + let mut result = Vec::new(); + for index in 0..items.len() { + let mut rest = items.to_vec(); + let first = rest.remove(index); + for mut tail in permutations(&rest) { + tail.insert(0, first.clone()); + result.push(tail); + } + } + result +} + +/// The old engine prints these two HashSets in randomized order in cancellation +/// mismatch errors. Normalize only those numeric diagnostic sets for the model +/// fingerprint; history decisions and the delta comparator are not altered. +fn legacy_case_fingerprint(observations: &[Observation]) -> Vec { + use sha2::{Digest, Sha256}; + let records: Vec<_> = observations + .iter() + .map(|observation| { + let history = serde_json::to_vec(&cancellation_history::decision_delta(&observation.history)) + .expect("canonical history must serialize"); + ( + legacy_model_outcome(&observation.live), + legacy_model_outcome(&observation.replay), + format!("{:?}", observation.live_reads), + format!("{:?}", observation.replay_reads), + &observation.live_actions, + &observation.replay_actions, + format!("{:x}", Sha256::digest(history)), + ) + }) + .collect(); + serde_json::to_vec(&records).expect("legacy fingerprint must serialize") +} + +#[test] +fn legacy_fingerprint_retains_history_and_live_decision_changes() { + let event = |id, source, kind| Event::with_event_id(id, TEST_INSTANCE, 1, source, kind); + let original = Observation { + live: "Continue".into(), + replay: "Continue".into(), + live_reads: vec![], + replay_reads: vec![], + logical_cancellations: None, + live_actions: vec!["next".into()], + replay_actions: vec![], + replay_delta: vec![], + history: vec![ + started_event(1), + event(2, None, EventKind::QueueSubscribed { name: "q".into() }), + event(3, None, EventKind::QueueSubscribed { name: "q".into() }), + event( + 4, + Some(2), + EventKind::QueueSubscriptionCancelled { + reason: "dropped_future".into(), + }, + ), + event( + 5, + Some(3), + EventKind::QueueSubscriptionCancelled { + reason: "dropped_future".into(), + }, + ), + ], + }; + let fingerprint = legacy_case_fingerprint(std::slice::from_ref(&original)); + let mut reordered = original.clone(); + reordered.history.swap(3, 4); + reordered.history[3].event_id = 4; + reordered.history[4].event_id = 5; + assert_eq!(fingerprint, legacy_case_fingerprint(&[reordered])); + let mut changed = original.clone(); + changed.live_actions.push("additional action".into()); + assert_ne!(fingerprint, legacy_case_fingerprint(&[changed])); + let mut changed = original.clone(); + changed.history[3].source_event_id = Some(7); + assert_ne!(fingerprint, legacy_case_fingerprint(&[changed])); + let mut changed = original.clone(); + changed.history[3].kind = EventKind::QueueSubscriptionCancelled { + reason: "changed".into(), + }; + assert_ne!(fingerprint, legacy_case_fingerprint(&[changed])); + let mut changed = original.clone(); + changed.history.push(event( + 6, + Some(2), + EventKind::QueueSubscriptionCancelled { + reason: "dropped_future".into(), + }, + )); + assert_ne!(fingerprint, legacy_case_fingerprint(&[changed])); + let mut changed = original.clone(); + changed.history[1].kind = EventKind::QueueSubscribed { name: "other".into() }; + assert_ne!(fingerprint, legacy_case_fingerprint(&[changed])); + let mut boundary = original.clone(); + boundary.history.push(event( + 6, + None, + EventKind::ActivityScheduled { + name: "barrier".into(), + input: "".into(), + session_id: None, + tag: None, + }, + )); + let before = legacy_case_fingerprint(std::slice::from_ref(&boundary)); + boundary.history.swap(4, 5); + boundary.history[4].event_id = 5; + boundary.history[5].event_id = 6; + assert_ne!(before, legacy_case_fingerprint(&[boundary])); +} + +fn legacy_model_outcome(outcome: &str) -> String { + // The new Rust result struct adds an absent legacy carry-forward field. + // Remove only that trailing field, never the same text in an output payload. + if outcome.starts_with("ContinueAsNew {") + && let Some(prefix) = outcome.strip_suffix(", unconsumed_queue_arrivals: None }") + { + return format!("{prefix} }}"); + } + for kind in ["activities", "sub-orchestrations", "external waits", "persistent waits"] { + let prefix = format!( + "Failed: nondeterministic: cancellation mismatch ({kind}): baseline_dropped_future_cancel_requests={{" + ); + let Some(sets) = outcome.strip_prefix(&prefix) else { + continue; + }; + let Some((baseline, context)) = sets.split_once("} ctx_cancelled_in_replayed_segment={") else { + return outcome.into(); + }; + let Some(context) = context.strip_suffix('}') else { + return outcome.into(); + }; + if baseline.contains(['{', '}']) || context.contains(['{', '}']) { + return outcome.into(); + } + let sorted = |set: &str| { + let mut ids: Vec = if set.is_empty() { + Vec::new() + } else { + set.split(", ") + .map(|value| { + assert!( + value.bytes().all(|byte| byte.is_ascii_digit()), + "diagnostic set must contain numeric IDs" + ); + value.parse().expect("diagnostic ID must fit u64") + }) + .collect() + }; + ids.sort_unstable(); + ids.iter().map(u64::to_string).collect::>().join(", ") + }; + return format!( + "{prefix}{}}} ctx_cancelled_in_replayed_segment={{{}}}", + sorted(baseline), + sorted(context) + ); + } + outcome.into() +} + +#[test] +fn legacy_fingerprint_normalizes_only_named_numeric_diagnostic_sets() { + use sha2::{Digest, Sha256}; + let original = "Failed: nondeterministic: cancellation mismatch (persistent waits): baseline_dropped_future_cancel_requests={3, 2} ctx_cancelled_in_replayed_segment={4, 2}"; + let reordered = "Failed: nondeterministic: cancellation mismatch (persistent waits): baseline_dropped_future_cancel_requests={2, 3} ctx_cancelled_in_replayed_segment={2, 4}"; + assert_eq!(legacy_model_outcome(original), legacy_model_outcome(reordered)); + let fingerprint = |text: &str| Sha256::digest(legacy_model_outcome(text).as_bytes()); + for changed in [ + original.replace("{3, 2}", "{3, 9}"), + original.replace("{3, 2}", "{3, 2, 9}"), + original.replace("{3, 2}", "{3}"), + original.replace("{3, 2}", "{3, 2, 2}"), + original.replace("{4, 2}", "{4, 9}"), + original.replace("{4, 2}", "{4, 2, 9}"), + original.replace("{4, 2}", "{4}"), + original.replace("{4, 2}", "{4, 2, 2}"), + original.replace("persistent waits", "activities"), + original.replace("ctx_cancelled_in_replayed_segment", "other"), + ] { + assert_ne!(legacy_model_outcome(original), legacy_model_outcome(&changed)); + assert_ne!(fingerprint(original), fingerprint(&changed)); + } + assert_eq!(legacy_model_outcome("Completed(\"{3, 2}\")"), "Completed(\"{3, 2}\")"); + let payload = "Completed(\"baseline_dropped_future_cancel_requests={3, 2}\")"; + assert_eq!(legacy_model_outcome(payload), payload); + for elsewhere in [ + format!("prefix {original}"), + format!("{original} extra"), + original.replace("nondeterministic", "activity failure"), + original.replace("persistent waits", "unknown kind"), + "Completed(\", unconsumed_queue_arrivals: None\")".into(), + ] { + assert_eq!(legacy_model_outcome(&elsewhere), elsewhere); + } +} + +#[test] +#[should_panic(expected = "diagnostic set must contain numeric IDs")] +fn legacy_diagnostic_template_rejects_non_numeric_set_elements() { + legacy_model_outcome( + "Failed: nondeterministic: cancellation mismatch (persistent waits): baseline_dropped_future_cancel_requests={2, invalid} ctx_cancelled_in_replayed_segment={4}", + ); +} + +#[test] +#[should_panic(expected = "diagnostic ID must fit u64")] +fn legacy_diagnostic_template_rejects_overflow_in_the_other_set() { + legacy_model_outcome( + "Failed: nondeterministic: cancellation mismatch (persistent waits): baseline_dropped_future_cancel_requests={2} ctx_cancelled_in_replayed_segment={18446744073709551616}", + ); +} + +/// Legacy digests: what to do when one changes. +/// +/// The two exhaustive generators fold every legacy-stamped case into a SHA-256 digest +/// (`b9ecaca1...` queue, `7f977b4e...` signal families). The expected value is the +/// unmodified 0.1.30 base's behaviour, so a mismatch means either a legacy decision +/// changed (a compatibility break) or the generated census changed. Never accept a new +/// value by copying this branch's printed digest: +/// +/// 1. Copy this test file onto the isolated base (`6a45886`, duroxide 0.1.30) and run +/// the generator there with `DUROXIDE_RECORD_LEGACY_MODEL=1`. The run checks that +/// `src/lib.rs` and the replay engine are the unmodified base, and prints the digest +/// the base actually produces. Only that value may become the expected digest. +/// 2. If the base digest and this branch's digest differ, find the cases before +/// anything else: run the queue generator on both trees with +/// `DUROXIDE_LEGACY_MODEL_TRACE=` and diff the files. Each line is one legacy +/// case (`index, shape/batch/iterations/eager/split, fingerprint`); a differing +/// fingerprint is a changed legacy decision and must be fixed, not re-recorded. +/// 3. `DUROXIDE_VERIFY_LEGACY_MODEL=1` runs only the legacy half of either generator +/// (for example 168 of the 336 signal-family cases) for a faster check. +#[test] +fn exhaustive_small_scope_queue_race_orders() { + use sha2::{Digest, Sha256}; + use std::io::Write; + let record_legacy = std::env::var_os("DUROXIDE_RECORD_LEGACY_MODEL").is_some(); + let verify_legacy = std::env::var_os("DUROXIDE_VERIFY_LEGACY_MODEL").is_some(); + let mut trace = std::env::var_os("DUROXIDE_LEGACY_MODEL_TRACE") + .map(|path| std::io::BufWriter::new(std::fs::File::create(path).expect("create legacy model trace"))); + if record_legacy { + assert_eq!(env!("CARGO_PKG_VERSION"), "0.1.30"); + assert_eq!( + format!("{:x}", Sha256::digest(include_str!("../src/lib.rs").as_bytes())), + "c0f28773f23f9f0f1bab78d3472456a45c63e5dc6fbdd21854029fee2c1129a3", + "legacy model recording requires the isolated unmodified base" + ); + assert_eq!( + format!( + "{:x}", + Sha256::digest(include_str!("../src/runtime/replay_engine.rs").as_bytes()) + ), + "b85bf87ec75d63db0befff18e9dc32d2385b721055b441b42c607477f0242deb", + "legacy model recording requires the isolated unmodified replay engine" + ); + } + let mut legacy_digest = Sha256::new(); + // Keep the model census log small; targeted warning tests capture their own + // diagnostics. Every case still runs the real engine and invariant checks. + let _quiet = tracing::subscriber::set_default(tracing::subscriber::NoSubscriber::default()); + let arms = [Arm::Q, Arm::Q2, Arm::Signal, Arm::Timer, Arm::Activity, Arm::Child]; + let mut programs = Vec::new(); + for a in arms { + programs.push(vec![a]); + for b in arms { + programs.push(vec![a, b]); + for c in arms { + programs.push(vec![a, b, c]); + } + } + } + let mut cases = 0; + let mut legacy_differences = 0; + let mut asserted = 0; + for program in programs { + let mut inputs = Vec::new(); + if program.contains(&Arm::Q) { + inputs.push(Completion::Q("first")); + inputs.push(Completion::Q("second")); + inputs.push(Completion::Q("third")); + } + if program.contains(&Arm::Q2) { + inputs.push(Completion::Q2("other")); + inputs.push(Completion::Q2("other-second")); + inputs.push(Completion::Q2("other-third")); + } + if program.contains(&Arm::Signal) { + inputs.push(Completion::Signal("first-signal")); + inputs.push(Completion::Signal("second-signal")); + inputs.push(Completion::Signal("third-signal")); + } + for index in 0..program.iter().filter(|arm| **arm == Arm::Timer).count() { + inputs.push(Completion::Timer(index)); + } + for index in 0..program.iter().filter(|arm| **arm == Arm::Activity).count() { + inputs.push(Completion::Activity(index)); + } + for index in 0..program.iter().filter(|arm| **arm == Arm::Child).count() { + inputs.push(Completion::Child(index)); + } + let mut subsets = Vec::new(); + for mask in 0..(1usize << inputs.len()) { + if mask.count_ones() > 3 { + continue; + } + let subset = inputs + .iter() + .enumerate() + .filter(|(i, _)| mask & (1 << i) != 0) + .map(|(_, completion)| completion.clone()) + .collect::>(); + subsets.extend(permutations(&subset)); + } + for batch in subsets { + let follow: Vec<_> = [ + (Arm::Q, Completion::Q("fourth")), + (Arm::Q2, Completion::Q2("other-fourth")), + (Arm::Signal, Completion::Signal("fresh-signal")), + ] + .into_iter() + .filter(|(arm, _)| program.contains(arm)) + .map(|(_, completion)| completion) + .collect(); + for iterations in 1..=3 { + for stamp in ["0.1.30", "0.1.31"] { + if (record_legacy || verify_legacy) && stamp != "0.1.30" { + continue; + } + for eager in [false, true] { + let buffered_placements = if batch.iter().all(|completion| { + matches!(completion, Completion::Q(_) | Completion::Q2(_) | Completion::Signal(_)) + }) { + 2 + } else { + 1 + }; + for placement in 0..buffered_placements { + let mut handler = Handler::new(Shape::Race(program.clone()), iterations, vec![]); + Arc::get_mut(&mut handler).unwrap().eager = eager; + Arc::get_mut(&mut handler).unwrap().expected_queue_counts = Some(( + batch + .iter() + .chain(follow.iter()) + .filter(|completion| matches!(completion, Completion::Q(_))) + .count(), + batch + .iter() + .chain(follow.iter()) + .filter(|completion| matches!(completion, Completion::Q2(_))) + .count(), + )); + let observations = if placement == 0 { + record_with_fresh_completions( + stamp, + handler, + &[batch.clone(), follow.clone()], + &[], + true, + ) + } else { + record_with_fresh_completions( + stamp, + handler, + std::slice::from_ref(&follow), + &batch, + true, + ) + }; + let stable = observations.iter().all(|o| { + o.live == o.replay && o.live_reads == o.replay_reads && o.replay_actions.is_empty() + }); + if stamp == "0.1.31" { + assert_stable(&observations, &format!("exhaustive/{program:?}/{batch:?}/{iterations}")); + if iterations >= 2 + && (program == [Arm::Signal, Arm::Timer] || program == [Arm::Timer, Arm::Signal]) + && matches!(batch.first(), Some(Completion::Timer(0))) + && batch + .iter() + .any(|completion| matches!(completion, Completion::Signal(_))) + { + assert_signal_contract_read(&observations, "fresh-signal"); + } + let reads = queue_values(&observations); + for queue in ["q", "q2"] { + let expected = batch + .iter() + .chain(follow.iter()) + .filter_map(|completion| match completion { + Completion::Q(v) if queue == "q" => Some(*v), + Completion::Q2(v) if queue == "q2" => Some(*v), + _ => None, + }) + .collect::>(); + let actual = reads + .iter() + .filter(|(name, _)| *name == queue) + .map(|(_, value)| value.as_str()) + .collect::>(); + // A pending program may not yet have read every offered value. + assert_eq!(actual, expected[..actual.len()], "FIFO prefix changed"); + let last = observations.last().unwrap(); + let canceled: std::collections::HashSet<_> = last + .history + .iter() + .filter(|event| { + matches!(event.kind, EventKind::QueueSubscriptionCancelled { .. }) + }) + .filter_map(|event| event.source_event_id) + .collect(); + let active = last + .history + .iter() + .filter(|event| { + matches!(&event.kind, EventKind::QueueSubscribed { name } if name == queue) + && !canceled.contains(&event.event_id) + }) + .count(); + if active > actual.len() { + assert_eq!( + actual.len(), + expected.len(), + "an active dequeue remained pending with an unread queued arrival" + ); + } + if observations.last().unwrap().history.iter().any(|event| + matches!(&event.kind,EventKind::ActivityScheduled{name,..}if name=="checkpoint")) { + assert_eq!(actual,expected,"completed model did not consume every queued input"); + } + } + let offered: Vec<_> = batch + .iter() + .chain(follow.iter()) + .filter_map(|c| if let Completion::Signal(v) = c { Some(*v) } else { None }) + .collect(); + let mut position = 0; + for read in &observations.last().unwrap().live_reads { + if let Read::Signal(value) = read { + let next = offered[position..] + .iter() + .position(|v| *v == value) + .expect("signal duplicated, reordered, or never offered"); + position += next + 1; + } + } + asserted += 1; + } else if !stable { + legacy_differences += 1; + } + if stamp == "0.1.30" { + let fingerprint = legacy_case_fingerprint(&observations); + legacy_digest.update(&fingerprint); + if let Some(trace) = &mut trace { + writeln!( + trace, + "{cases}\t{program:?}/{batch:?}/{iterations}/{eager}/{placement}\t{}", + std::str::from_utf8(&fingerprint).expect("JSON fingerprint must be UTF-8") + ) + .expect("write legacy model trace"); + } + } + cases += 1; + } + } + } + } + } + } + // Additional real-engine program families, including explicit/unpolled + // drops, a partial join, nested select and held/non-prefix CAN. Completion + // partitions give separate-arrival turns as well as the same-batch shapes. + for shape in [ + Shape::UnboundDrop, + Shape::BoundDrop, + Shape::PartialJoin, + Shape::Nested, + Shape::Continue, + Shape::PositionalContinue, + Shape::PositionalBarrier, + Shape::ResolveAtCreation, + Shape::HeldContinue(false), + Shape::HeldContinue(true), + Shape::QuiescenceDrop, + ] { + let inputs = vec![ + Completion::Timer(0), + Completion::Q("first"), + Completion::Signal("first-signal"), + ]; + // ResolveAtCreation has activity, not timer, at the first boundary. + let inputs = if matches!(shape, Shape::ResolveAtCreation) { + vec![ + Completion::Signal("first-signal"), + Completion::Activity(0), + Completion::Q("first"), + ] + } else if matches!(shape, Shape::QuiescenceDrop) { + vec![ + Completion::Q("first"), + Completion::Q2("other"), + Completion::Signal("first-signal"), + ] + } else { + inputs + }; + for batch in permutations(&inputs) { + for split in 0..=batch.len() { + for eager in [false, true] { + for iterations in 1..=3 { + for stamp in ["0.1.30", "0.1.31"] { + if (record_legacy || verify_legacy) && stamp != "0.1.30" { + continue; + } + let mut h = Handler::new(shape.clone(), iterations, vec![]); + Arc::get_mut(&mut h).unwrap().eager = eager; + let mut batches = vec![batch[..split].to_vec(), batch[split..].to_vec()]; + // A third completion turn exercises later binding and fresh delivery. + batches.push( + if matches!(shape, Shape::PositionalBarrier | Shape::ResolveAtCreation) { + vec![ + Completion::Activity(usize::from(matches!(shape, Shape::ResolveAtCreation))), + Completion::Signal("fresh-signal"), + Completion::Q("second"), + ] + } else { + vec![ + Completion::Q("second"), + Completion::Q2("other"), + Completion::Signal("fresh-signal"), + ] + }, + ); + if stamp == "0.1.31" && matches!(shape, Shape::PositionalBarrier | Shape::ResolveAtCreation) + { + for completion in batches.last_mut().unwrap() { + if let Completion::Signal(value) = completion { + *value = "same-batch-early"; + } + } + batches.push(vec![Completion::Signal("fresh-signal")]); + } + let obs = record_with_fresh_completions(stamp, h, &batches, &[], true); + if stamp == "0.1.31" { + if matches!(shape, Shape::PositionalBarrier | Shape::ResolveAtCreation) { + assert_signal_contract_read(&obs, "fresh-signal"); + } + // CAN has its terminal action; all other decisions must replay without emission. + if obs.last().unwrap().live.starts_with("ContinueAsNew") { + for o in &obs { + assert_eq!(o.live, o.replay); + assert_eq!(o.live_reads, o.replay_reads); + assert!(o.replay_delta.is_empty()); + } + } else { + assert_stable(&obs, &format!("family/{shape:?}/{batch:?}/{split}")); + } + asserted += 1; + } else { + if obs.iter().any(|o| o.live != o.replay || o.live_reads != o.replay_reads) { + legacy_differences += 1; + } + let fingerprint = legacy_case_fingerprint(&obs); + legacy_digest.update(&fingerprint); + if let Some(trace) = &mut trace { + writeln!( + trace, + "{cases}\t{shape:?}/{batch:?}/{iterations}/{eager}/{split}\t{}", + std::str::from_utf8(&fingerprint).expect("JSON fingerprint must be UTF-8") + ) + .expect("write legacy model trace"); + } + } + cases += 1; + } + } + } + } + } + } + let digest = format!("{:x}", legacy_digest.finalize()); + println!( + "EXHAUSTIVE enumerated={cases} individually_asserted_new={asserted} legacy_known_differences={legacy_differences} legacy_sha256={digest} arms=1..3 completions=0..3 iterations=1..3" + ); + if !record_legacy { + // On mismatch follow "Legacy digests" above; never paste this branch's value. + assert_eq!( + digest, + "b9ecaca1172c983ea9f1b1ed2b30b2544a13ca542210d02e7dcfc3de63a13d0e" + ); + assert_eq!(legacy_differences, 115_640, "isolated-base characterization changed"); + } +} + +#[path = "common/mod.rs"] +#[allow(dead_code)] +mod common; + +#[test] +fn exhaustive_signal_slot_family_orders() { + use sha2::{Digest, Sha256}; + let record_legacy = std::env::var_os("DUROXIDE_RECORD_LEGACY_MODEL").is_some(); + let verify_legacy = std::env::var_os("DUROXIDE_VERIFY_LEGACY_MODEL").is_some(); + let _quiet = tracing::subscriber::set_default(tracing::subscriber::NoSubscriber::default()); + if record_legacy { + assert_eq!(env!("CARGO_PKG_VERSION"), "0.1.30"); + assert_eq!( + format!("{:x}", Sha256::digest(include_str!("../src/lib.rs").as_bytes())), + "c0f28773f23f9f0f1bab78d3472456a45c63e5dc6fbdd21854029fee2c1129a3" + ); + } + type SignalSequence = Vec<(&'static str, &'static str)>; + let mut plans: Vec<(Shape, Vec>, SignalSequence)> = Vec::new(); + for first in permutations(&[Completion::Timer(0), Completion::Signal("first")]) { + let timer_first = matches!(first.first(), Some(Completion::Timer(0))); + plans.push(( + Shape::UnboundSignalDrop, + vec![first.clone(), vec![Completion::Signal("late")], vec![]], + vec![("s", if timer_first { "first" } else { "late" })], + )); + plans.push(( + Shape::UnboundSignalDropBarrier, + vec![ + first, + vec![Completion::Activity(0)], + vec![Completion::Signal("fresh")], + vec![], + ], + vec![("s", "fresh")], + )); + } + for first in permutations(&[Completion::Timer(0), Completion::Signal2("wrong-name-early")]) { + for third in permutations(&[Completion::Activity(0), Completion::Signal2("fresh")]) { + let accepted_fresh = matches!(third.first(), Some(Completion::Activity(0))); + plans.push(( + Shape::WrongNameBeforeWait, + vec![ + first.clone(), + vec![Completion::Signal("alpha")], + third, + vec![Completion::Signal2("late")], + vec![], + ], + vec![("s", "alpha"), ("s2", if accepted_fresh { "fresh" } else { "late" })], + )); + } + } + for first in permutations(&[Completion::Signal("held"), Completion::Timer(0)]) { + let timer_first = matches!(first.first(), Some(Completion::Timer(0))); + for drop_held in [false, true] { + plans.push(( + Shape::HeldSignalWait(drop_held), + vec![first.clone(), vec![Completion::Signal("fresh")], vec![]], + vec![("s", if drop_held && !timer_first { "fresh" } else { "held" })], + )); + } + } + for first in permutations(&[ + Completion::Activity(1), + Completion::Signal2("R"), + Completion::Activity(0), + Completion::Signal("A"), + ]) { + let position = |activity: usize| { + first + .iter() + .position(|value| matches!(value, Completion::Activity(index) if *index == activity)) + .unwrap() + }; + let a = first + .iter() + .position(|value| matches!(value, Completion::Signal("A"))) + .unwrap(); + let r = first + .iter() + .position(|value| matches!(value, Completion::Signal2("R"))) + .unwrap(); + for follow in permutations(&[Completion::Signal("fresh-A"), Completion::Signal2("fresh-R")]) { + let expected = if position(0) < a { + ("s", "A") + } else if position(1) < r { + ("s2", "R") + } else if matches!(follow.first(), Some(Completion::Signal(_))) { + ("s", "fresh-A") + } else { + ("s2", "fresh-R") + }; + plans.push(( + Shape::LazySignalCompetition, + vec![first.clone(), follow, vec![]], + vec![expected], + )); + } + } + for first in permutations(&[ + Completion::Signal("x"), + Completion::Signal("stale"), + Completion::Timer(0), + Completion::Signal("fresh"), + ]) { + let deadline = first + .iter() + .position(|value| matches!(value, Completion::Timer(0))) + .unwrap(); + let signal = |value: &Completion| match value { + Completion::Signal(value) => Some(*value), + _ => None, + }; + let mut expected = Vec::new(); + if let Some(value) = first[..deadline].iter().find_map(signal) { + expected.push(("s", value)); + } + expected.push(("s", first[deadline + 1..].iter().find_map(signal).unwrap_or("late"))); + plans.push(( + Shape::DeadlineSignals, + vec![first, vec![Completion::Signal("late")], vec![]], + expected, + )); + } + let mut legacy_digest = Sha256::new(); + let mut cases = 0; + let mut asserted = 0; + let mut legacy_differences = 0; + for (shape, batches, expected) in plans { + for eager in [false, true] { + for stamp in ["0.1.30", "0.1.31"] { + if (record_legacy || verify_legacy) && stamp != "0.1.30" { + continue; + } + let mut handler = Handler::new(shape.clone(), 1, vec![]); + Arc::get_mut(&mut handler).unwrap().eager = eager; + let expected = if stamp == "0.1.31" { + match &shape { + Shape::UnboundSignalDrop => vec![("s", "late")], + Shape::WrongNameBeforeWait => vec![("s", "alpha"), ("s2", "late")], + Shape::LazySignalCompetition => { + vec![if matches!(batches[1].first(), Some(Completion::Signal(_))) { + ("s", "fresh-A") + } else { + ("s2", "fresh-R") + }] + } + Shape::HeldSignalWait(true) => vec![("s", "fresh")], + Shape::DeadlineSignals => { + let mut values = Vec::new(); + if let Some(Completion::Signal(value)) = batches[0].first() { + values.push(("s", *value)); + } + values.push(("s", "late")); + values + } + _ => expected.clone(), + } + } else { + expected.clone() + }; + let observations = record_with_fresh_completions(stamp, handler, &batches, &[], true); + if stamp == "0.1.31" { + assert_stable(&observations, &format!("signal-family/{shape:?}/{batches:?}/{eager}")); + let actual: Vec<_> = observations + .last() + .unwrap() + .live_reads + .iter() + .filter_map(|read| match read { + Read::Signal(value) => Some(("s", value.as_str())), + Read::Signal2(value) => Some(("s2", value.as_str())), + _ => None, + }) + .collect(); + assert_eq!( + actual, expected, + "{shape:?}/{batches:?}/{eager}: admitted signal lost, duplicated or assigned to the wrong name" + ); + assert!(observations.last().unwrap().history.iter().any(|event| + matches!(&event.kind, EventKind::ActivityScheduled { name, .. } if name == "checkpoint")), + "signal-family program did not observe all required signals"); + asserted += 1; + } else { + legacy_digest.update(legacy_case_fingerprint(&observations)); + if observations + .iter() + .any(|value| value.live != value.replay || value.live_reads != value.replay_reads) + { + legacy_differences += 1; + } + } + cases += 1; + } + } + } + let digest = format!("{:x}", legacy_digest.finalize()); + println!( + "SIGNAL_FAMILIES enumerated={cases} individually_asserted_new={asserted} legacy_known_differences={legacy_differences} legacy_sha256={digest}" + ); + assert_eq!(cases, if record_legacy || verify_legacy { 168 } else { 336 }); + if !record_legacy { + // On mismatch follow "Legacy digests" above `exhaustive_small_scope_queue_race_orders`; + // never paste this branch's value. + assert_eq!( + digest, + "7f977b4e7f2c8fa329a55e6abd312bca2ea59d369adb8873523f35181f5704c0" + ); + } +} + +async fn runtime_seeded_scenario(shape: Shape, drain: Vec, batch: Vec) { + use duroxide::providers::ExecutionMetadata; + use duroxide::runtime::registry::ActivityRegistry; + use duroxide::runtime::{OrchestrationStatus, Runtime, RuntimeOptions}; + use duroxide::{Client, OrchestrationRegistry}; + let (store, _temporary) = common::create_sqlite_store_disk().await; + let handler = Handler::new(shape, 1, drain); + let observations = record_and_replay("0.1.31", handler.clone(), &[batch]); + assert_stable(&observations, "seeded SQLite runtime"); + let history = observations.last().unwrap().history.clone(); + common::seed_history_turn( + store.as_ref(), + WorkItem::StartOrchestration { + instance: TEST_INSTANCE.into(), + orchestration: TEST_ORCH_NAME.into(), + version: Some(TEST_ORCH_VERSION.into()), + input: "".into(), + parent_instance: None, + parent_id: None, + parent_execution_id: None, + execution_id: 1, + }, + 1, + history.clone(), + vec![], + ExecutionMetadata { + orchestration_name: Some(TEST_ORCH_NAME.into()), + orchestration_version: Some(TEST_ORCH_VERSION.into()), + ..Default::default() + }, + ) + .await; + let checkpoint = history + .iter() + .find(|event| matches!(&event.kind,EventKind::ActivityScheduled{name,..}if name=="checkpoint")); + if checkpoint.is_none() { + // This first turn must discard the stale positional signal. A fresh + // signal is delivered in a later turn and must resolve the replacement. + assert!( + history + .iter() + .any(|e| matches!(e.kind, EventKind::ExternalSubscribedCancelled { .. })) + ); + let handler_for_runtime = handler.clone(); + let registry = OrchestrationRegistry::builder() + .register(TEST_ORCH_NAME, move |ctx, input| { + let h = handler_for_runtime.clone(); + async move { h.invoke(ctx, input).await } + }) + .build(); + let runtime = Runtime::start_with_options( + store.clone(), + ActivityRegistry::builder().build(), + registry, + RuntimeOptions { + dispatcher_min_poll_interval: Duration::from_millis(5), + ..Default::default() + }, + ) + .await; + let client = Client::new(store.clone()); + client.raise_event(TEST_INSTANCE, "s", "fresh").await.unwrap(); + let processed = common::wait_for_history( + store.clone(), + TEST_INSTANCE, + |events| { + events + .iter() + .any(|event| matches!(&event.kind,EventKind::ExternalEvent{data,..}if data=="fresh")) + }, + HISTORY_DEADLINE_MS, + ) + .await; + assert!(processed); + assert!( + common::wait_for_history( + store.clone(), + TEST_INSTANCE, + |events| events + .iter() + .any(|event| matches!(&event.kind,EventKind::ActivityScheduled{name,input,..} + if name=="checkpoint"&&input.contains("fresh"))), + HISTORY_DEADLINE_MS + ) + .await, + "new policy did not resolve the replacement positional wait" + ); + runtime.shutdown(None).await; + return; + } + let checkpoint = checkpoint.unwrap(); + store + .enqueue_for_orchestrator(activity_completed_msg(checkpoint.event_id, "ok"), None) + .await + .unwrap(); + let registry = OrchestrationRegistry::builder() + .register(TEST_ORCH_NAME, move |ctx, input| { + let handler = handler.clone(); + async move { handler.invoke(ctx, input).await } + }) + .build(); + let runtime = Runtime::start_with_options( + store.clone(), + ActivityRegistry::builder().build(), + registry, + RuntimeOptions { + dispatcher_min_poll_interval: Duration::from_millis(5), + ..Default::default() + }, + ) + .await; + let result = Client::new(store) + .wait_for_orchestration(TEST_INSTANCE, RUNTIME_DEADLINE) + .await + .unwrap(); + assert!(matches!(result, OrchestrationStatus::Completed { .. }), "{result:?}"); + runtime.shutdown(None).await; +} + +#[tokio::test] +async fn sqlite_q1_two_dequeue_losers_restart() { + runtime_seeded_scenario( + Shape::Race(vec![Arm::Q, Arm::Q, Arm::Timer]), + vec![Arm::Q, Arm::Q], + vec![Completion::Timer(0), Completion::Q("first"), Completion::Q("second")], + ) + .await; +} + +#[tokio::test] +async fn sqlite_q2_first_dequeue_wins_restart() { + runtime_seeded_scenario( + Shape::Race(vec![Arm::Q, Arm::Q, Arm::Timer]), + vec![Arm::Q], + vec![Completion::Q("first"), Completion::Q("second")], + ) + .await; +} + +#[tokio::test] +async fn sqlite_q6_nested_select_restart() { + runtime_seeded_scenario( + Shape::Nested, + vec![Arm::Q], + vec![Completion::Timer(1), Completion::Q("first")], + ) + .await; +} + +#[tokio::test] +async fn sqlite_s1_positional_same_batch_replays_after_restart() { + runtime_seeded_scenario( + Shape::Race(vec![Arm::Signal, Arm::Timer]), + vec![Arm::Signal], + vec![Completion::Timer(0), Completion::Signal("first")], + ) + .await; +} + +#[tokio::test] +async fn sqlite_s3_mixed_same_batch_replays_after_restart() { + runtime_seeded_scenario( + Shape::Race(vec![Arm::Signal, Arm::Q, Arm::Timer]), + vec![Arm::Signal, Arm::Q], + vec![ + Completion::Timer(0), + Completion::Signal("first"), + Completion::Q("first"), + ], + ) + .await; +} + +#[tokio::test] +async fn sqlite_legacy_safe_history_keeps_old_decisions() { + use duroxide::runtime::registry::ActivityRegistry; + use duroxide::runtime::{Runtime, RuntimeOptions}; + use duroxide::{Client, OrchestrationRegistry, OrchestrationStatus}; + let (store, _temporary) = common::create_sqlite_store_disk().await; + let handler = Handler::new(Shape::Race(vec![Arm::Q, Arm::Timer]), 1, vec![]); + let observations = record_and_replay("0.1.30", handler.clone(), &[vec![Completion::Q("first")]]); + assert_stable(&observations, "legacy-safe runtime"); + let history = observations.last().unwrap().history.clone(); + let checkpoint = history + .iter() + .find(|event| matches!(&event.kind,EventKind::ActivityScheduled{name,..}if name=="checkpoint")) + .unwrap() + .event_id; + common::seed_history_turn( + store.as_ref(), + WorkItem::StartOrchestration { + instance: TEST_INSTANCE.into(), + orchestration: TEST_ORCH_NAME.into(), + input: "".into(), + version: Some(TEST_ORCH_VERSION.into()), + parent_instance: None, + parent_id: None, + parent_execution_id: None, + execution_id: 1, + }, + 1, + history, + vec![activity_completed_msg(checkpoint, "ok")], + duroxide::providers::ExecutionMetadata { + orchestration_name: Some(TEST_ORCH_NAME.into()), + orchestration_version: Some(TEST_ORCH_VERSION.into()), + ..Default::default() + }, + ) + .await; + let registry = OrchestrationRegistry::builder() + .register(TEST_ORCH_NAME, move |ctx, input| { + let h = handler.clone(); + async move { h.invoke(ctx, input).await } + }) + .build(); + let runtime = Runtime::start_with_options( + store.clone(), + ActivityRegistry::builder().build(), + registry, + RuntimeOptions { + dispatcher_min_poll_interval: Duration::from_millis(5), + ..Default::default() + }, + ) + .await; + let result = Client::new(store.clone()) + .wait_for_orchestration(TEST_INSTANCE, RUNTIME_DEADLINE) + .await + .unwrap(); + assert!(matches!(result,OrchestrationStatus::Completed{ref output,..}if output.contains("first"))); + assert_eq!(store.read(TEST_INSTANCE).await.unwrap()[0].duroxide_version, "0.1.30"); + runtime.shutdown(None).await; +} + +#[tokio::test] +async fn sqlite_legacy_hazard_keeps_the_retained_failure() { + use duroxide::runtime::registry::ActivityRegistry; + use duroxide::runtime::{Runtime, RuntimeOptions}; + use duroxide::{Client, OrchestrationRegistry, OrchestrationStatus}; + let (store, _temporary) = common::create_sqlite_store_disk().await; + let handler = Handler::new(Shape::Race(vec![Arm::Q, Arm::Timer]), 1, vec![Arm::Q]); + let observations = record_and_replay( + "0.1.30", + handler.clone(), + &[vec![Completion::Timer(0), Completion::Q("first")]], + ); + let history = observations.last().unwrap().history.clone(); + common::seed_history_turn( + store.as_ref(), + WorkItem::StartOrchestration { + instance: TEST_INSTANCE.into(), + orchestration: TEST_ORCH_NAME.into(), + input: "".into(), + version: Some(TEST_ORCH_VERSION.into()), + parent_instance: None, + parent_id: None, + parent_execution_id: None, + execution_id: 1, + }, + 1, + history, + vec![WorkItem::QueueMessage { + instance: TEST_INSTANCE.into(), + name: "q".into(), + data: "second".into(), + }], + duroxide::providers::ExecutionMetadata { + orchestration_name: Some(TEST_ORCH_NAME.into()), + orchestration_version: Some(TEST_ORCH_VERSION.into()), + ..Default::default() + }, + ) + .await; + let registry = OrchestrationRegistry::builder() + .register(TEST_ORCH_NAME, move |ctx, input| { + let h = handler.clone(); + async move { h.invoke(ctx, input).await } + }) + .build(); + let runtime = Runtime::start_with_options( + store.clone(), + ActivityRegistry::builder().build(), + registry, + RuntimeOptions { + dispatcher_min_poll_interval: Duration::from_millis(5), + ..Default::default() + }, + ) + .await; + let result = Client::new(store) + .wait_for_orchestration(TEST_INSTANCE, RUNTIME_DEADLINE) + .await + .unwrap(); + assert!( + matches!(result,OrchestrationStatus::Failed{ref details,..}if details.display_message().contains("history schedule but no emitted action")) + ); + runtime.shutdown(None).await; +} + +/// Unread queue messages survive two continue-as-new boundaries. A 0.1.30 execution +/// carries by the legacy count (one read of three arrivals leaves two); executions it +/// continues into are stamped by this runtime and carry exact unread positions. +#[tokio::test] +async fn sqlite_unread_messages_carry_across_two_continue_as_new_for_both_policies() { + use duroxide::runtime::registry::ActivityRegistry; + use duroxide::runtime::{Runtime, RuntimeOptions}; + use duroxide::{Client, OrchestrationRegistry, OrchestrationStatus}; + for stamp in [semver::Version::new(0, 1, 30), semver::Version::new(0, 1, 31)] { + let (store, _temporary) = common::create_sqlite_store_disk().await; + common::seed_instance_with_pinned_version(store.as_ref(), TEST_INSTANCE, TEST_ORCH_NAME, stamp.clone()).await; + let client = Client::new(store.clone()); + // All three arrive in execution 1's first batch; nothing is sent later. + for message in ["a", "b", "c"] { + client.enqueue_event(TEST_INSTANCE, "q", message).await.unwrap(); + } + let registry = OrchestrationRegistry::builder() + .register(TEST_ORCH_NAME, |ctx: OrchestrationContext, input: String| async move { + let read = ctx.dequeue_event("q").await; + let seen = if ctx.execution_id() == 1 { + read + } else { + format!("{input},{read}") + }; + if ctx.execution_id() < 3 { + ctx.continue_as_new(seen).await + } else { + Ok(seen) + } + }) + .build(); + let runtime = Runtime::start_with_options( + store.clone(), + ActivityRegistry::builder().build(), + registry, + RuntimeOptions { + dispatcher_min_poll_interval: Duration::from_millis(5), + ..Default::default() + }, + ) + .await; + let result = client.wait_for_orchestration(TEST_INSTANCE, RUNTIME_DEADLINE).await; + runtime.shutdown(None).await; + assert!( + matches!(result, Ok(OrchestrationStatus::Completed { ref output, .. }) if output == "a,b,c"), + "{stamp}: both unread messages must carry, then the last one again: {result:?}" + ); + } +} + +#[tokio::test] +async fn sqlite_q13_legacy_continue_as_new_gets_new_execution_policy() { + use duroxide::runtime::registry::ActivityRegistry; + use duroxide::runtime::{Runtime, RuntimeOptions}; + use duroxide::{Client, Either2, OrchestrationRegistry, OrchestrationStatus}; + let (store, _temporary) = common::create_sqlite_store_disk().await; + common::seed_instance_with_pinned_version( + store.as_ref(), + TEST_INSTANCE, + TEST_ORCH_NAME, + semver::Version::new(0, 1, 30), + ) + .await; + let registry = OrchestrationRegistry::builder() + .register(TEST_ORCH_NAME, |ctx: OrchestrationContext, _: String| async move { + if ctx.execution_id() == 1 { + let _ = ctx + .select2(ctx.dequeue_event("q"), ctx.schedule_timer(Duration::from_millis(10))) + .await; + ctx.continue_as_new("successor").await + } else { + let first = ctx.dequeue_event("q"); + let second = ctx.dequeue_event("q"); + let t0 = ctx.schedule_timer(Duration::from_secs(60)); + let t1 = ctx.schedule_timer(Duration::from_secs(120)); + let _ = ctx.select2(first, t0).await; + match ctx.select2(second, t1).await { + Either2::First(value) => Ok(value), + Either2::Second(()) => Ok("second-timeout".into()), + } + } + }) + .build(); + let runtime = Runtime::start_with_options( + store.clone(), + ActivityRegistry::builder().build(), + registry.clone(), + RuntimeOptions { + dispatcher_min_poll_interval: Duration::from_millis(5), + ..Default::default() + }, + ) + .await; + let client = Client::new(store.clone()); + assert!( + common::wait_for_history( + store.clone(), + TEST_INSTANCE, + |history| history.first().is_some_and(|event| event.execution_id == 2) + && history + .iter() + .filter(|event| matches!(event.kind, EventKind::TimerCreated { .. })) + .count() + == 2, + HISTORY_DEADLINE_MS + ) + .await + ); + runtime.shutdown(None).await; + let successor = client.read_execution_history(TEST_INSTANCE, 2).await.unwrap(); + let timers: Vec<_> = successor + .iter() + .filter_map(|e| { + if let EventKind::TimerCreated { fire_at_ms } = e.kind { + Some(WorkItem::TimerFired { + instance: TEST_INSTANCE.into(), + execution_id: 2, + id: e.event_id(), + fire_at_ms, + }) + } else { + None + } + }) + .collect(); + for message in [ + timers[0].clone(), + WorkItem::QueueMessage { + instance: TEST_INSTANCE.into(), + name: "q".into(), + data: "first".into(), + }, + timers[1].clone(), + ] { + store.enqueue_for_orchestrator(message, None).await.unwrap(); + } + let runtime = Runtime::start_with_options( + store.clone(), + ActivityRegistry::builder().build(), + registry, + RuntimeOptions::default(), + ) + .await; + let result = client + .wait_for_orchestration(TEST_INSTANCE, RUNTIME_DEADLINE) + .await + .unwrap(); + assert!(matches!(result,OrchestrationStatus::Completed{ref output,..}if output=="first")); + let old = client.read_execution_history(TEST_INSTANCE, 1).await.unwrap(); + let new = client.read_execution_history(TEST_INSTANCE, 2).await.unwrap(); + assert_eq!(old[0].duroxide_version, "0.1.30"); + assert!(semver::Version::parse(&new[0].duroxide_version).unwrap() >= semver::Version::new(0, 1, 31)); + runtime.shutdown(None).await; +} + +#[test] +fn positional_s1_same_batch_preserves_bound_slot_signal_contract() { + for stamp in ["0.1.30", "0.1.31"] { + let handler = Handler::new(Shape::Race(vec![Arm::Signal, Arm::Timer]), 1, vec![Arm::Signal]); + let observations = record_and_replay( + stamp, + handler, + &[ + vec![Completion::Timer(0), Completion::Signal("first")], + vec![Completion::Signal("second")], + ], + ); + assert_stable(&observations, &format!("S1/{stamp}")); + let reads = &observations.last().unwrap().live_reads; + if stamp == "0.1.31" { + assert!(reads.contains(&Read::Signal("second".into()))); + } else { + assert!(!reads.iter().any(|read| matches!(read, Read::Signal(_)))); + } + } +} + +#[test] +fn queue_q1_select3_two_losers_preserves_fifo_and_legacy_outcome() { + for count in 0..=3 { + let mut batch = vec![Completion::Timer(0)]; + batch.extend( + [Completion::Q("first"), Completion::Q("second"), Completion::Q("third")] + .into_iter() + .take(count), + ); + queue_scenario( + &format!("Q1/count{count}"), + Shape::Race(vec![Arm::Q, Arm::Q, Arm::Timer]), + vec![Arm::Q; count], + vec![batch], + ["first", "second", "third"] + .into_iter() + .take(count) + .map(|value| ("q", value)) + .collect(), + ); + } +} diff --git a/tests/common/cancellation_history.rs b/tests/common/cancellation_history.rs new file mode 100644 index 0000000..7e9f2f6 --- /dev/null +++ b/tests/common/cancellation_history.rs @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use duroxide::Event; + +pub fn decision_delta(events: &[Event]) -> Vec { + let mut values: Vec<_> = events + .iter() + .map(|event| { + let mut value = serde_json::to_value(event).expect("event must serialize"); + let object = value.as_object_mut().expect("event must be an object"); + object.remove("timestamp_ms"); + object.remove("duroxide_version"); + object.remove("fire_at_ms"); + value + }) + .collect(); + // Only independent contiguous dropped-future blocks have unordered IDs. + // Keep event positions, type order, reasons, source IDs and multiplicity. + let is_independent_cancel = |value: &serde_json::Value| { + matches!( + value["type"].as_str(), + Some( + "ExternalSubscribedPersistentCancelled" + | "ExternalSubscribedCancelled" + | "ActivityCancelRequested" + | "SubOrchestrationCancelRequested" + ) + ) && value["reason"].as_str() == Some("dropped_future") + }; + let mut index = 0; + while index < values.len() { + if !is_independent_cancel(&values[index]) { + index += 1; + continue; + } + let start = index; + let kind = values[start]["type"].clone(); + while index < values.len() && is_independent_cancel(&values[index]) && values[index]["type"] == kind { + index += 1; + } + let mut block = values[start..index].to_vec(); + let ids: Vec<_> = block.iter().map(|value| value["event_id"].clone()).collect(); + block.sort_by_key(|value| { + ( + value["type"].as_str().expect("cancellation type").to_string(), + value["source_event_id"].as_u64().expect("cancellation source ID"), + value["reason"].as_str().expect("cancellation reason").to_string(), + ) + }); + for (slot, mut value) in block.into_iter().enumerate() { + value["event_id"] = ids[slot].clone(); + values[start + slot] = value; + } + } + values +} diff --git a/tests/common/fault_injection.rs b/tests/common/fault_injection.rs index 49e5244..683b1d6 100644 --- a/tests/common/fault_injection.rs +++ b/tests/common/fault_injection.rs @@ -40,6 +40,14 @@ pub struct PoisonInjectingProvider { orchestration_skip_count: AtomicU32, /// Skip this many activity fetches before injecting (0 = immediate) activity_skip_count: AtomicU32, + fail_orchestration_abandon: AtomicBool, + failed_abandon_calls: AtomicU32, + fail_next_orchestration_ack: AtomicBool, + failure_commit_collision_notifications: AtomicU32, + collision_notices: std::sync::Mutex>, + successful_ignored_abandons: AtomicU32, + counted_abandon_delays: std::sync::Mutex>, + orchestration_fetches: std::sync::Mutex>, } impl PoisonInjectingProvider { @@ -52,6 +60,14 @@ impl PoisonInjectingProvider { activity_injection_persistent: AtomicBool::new(false), orchestration_skip_count: AtomicU32::new(0), activity_skip_count: AtomicU32::new(0), + fail_orchestration_abandon: AtomicBool::new(false), + failed_abandon_calls: AtomicU32::new(0), + fail_next_orchestration_ack: AtomicBool::new(false), + failure_commit_collision_notifications: AtomicU32::new(0), + collision_notices: std::sync::Mutex::new(Vec::new()), + successful_ignored_abandons: AtomicU32::new(0), + counted_abandon_delays: std::sync::Mutex::new(Vec::new()), + orchestration_fetches: std::sync::Mutex::new(Vec::new()), } } @@ -109,6 +125,49 @@ impl PoisonInjectingProvider { self.activity_injection_persistent.store(false, Ordering::SeqCst); self.orchestration_skip_count.store(0, Ordering::SeqCst); self.activity_skip_count.store(0, Ordering::SeqCst); + self.fail_orchestration_abandon.store(false, Ordering::SeqCst); + } + + pub fn fail_orchestration_abandon_persistently(&self) { + self.failed_abandon_calls.store(0, Ordering::SeqCst); + self.fail_orchestration_abandon.store(true, Ordering::SeqCst); + } + + pub fn failed_abandon_calls(&self) -> u32 { + self.failed_abandon_calls.load(Ordering::SeqCst) + } + + pub fn fail_next_orchestration_ack(&self) { + self.fail_next_orchestration_ack.store(true, Ordering::SeqCst); + } + + pub fn failure_commit_collision_notifications(&self) -> u32 { + self.failure_commit_collision_notifications.load(Ordering::SeqCst) + } + + /// (parent_id, parent_execution_id) of every committed `SubOrchFailed`. + pub fn collision_notices(&self) -> Vec<(u64, u64)> { + self.collision_notices.lock().expect("notices lock").clone() + } + + pub fn successful_ignored_abandons(&self) -> u32 { + self.successful_ignored_abandons.load(Ordering::SeqCst) + } + + /// (lock token, real attempt count) of every orchestration fetch that returned `instance`. + pub fn orchestration_fetches(&self, instance: &str) -> Vec<(String, u32)> { + self.orchestration_fetches + .lock() + .expect("fetches lock") + .iter() + .filter(|(fetched, _, _)| fetched == instance) + .map(|(_, lock, attempt)| (lock.clone(), *attempt)) + .collect() + } + + /// Delays of successful orchestration abandons that kept their attempt (`ignore_attempt = false`). + pub fn counted_abandon_delays(&self) -> Vec { + self.counted_abandon_delays.lock().expect("delays lock").clone() } } @@ -126,6 +185,11 @@ impl Provider for PoisonInjectingProvider { .await?; if let Some((item, lock_token, real_attempt_count)) = result { + self.orchestration_fetches.lock().expect("fetches lock").push(( + item.instance.clone(), + lock_token.clone(), + real_attempt_count, + )); // Check if we need to skip this fetch let skip = self.orchestration_skip_count.load(Ordering::SeqCst); if skip > 0 { @@ -189,6 +253,34 @@ impl Provider for PoisonInjectingProvider { metadata: ExecutionMetadata, cancelled_activities: Vec, ) -> Result<(), ProviderError> { + if self.fail_next_orchestration_ack.swap(false, Ordering::SeqCst) { + return Err(ProviderError::permanent( + "ack_orchestration_item", + "injected permanent turn-ack failure", + )); + } + let collision_count = if history_delta + .iter() + .any(|e| matches!(e.kind, EventKind::OrchestrationFailed { .. })) + { + orchestrator_items + .iter() + .filter(|w| matches!(w, WorkItem::SubOrchFailed { .. })) + .count() as u32 + } else { + 0 + }; + let notices: Vec<(u64, u64)> = orchestrator_items + .iter() + .filter_map(|w| match w { + WorkItem::SubOrchFailed { + parent_id, + parent_execution_id, + .. + } => Some((*parent_id, *parent_execution_id)), + _ => None, + }) + .collect(); self.inner .ack_orchestration_item( lock_token, @@ -199,7 +291,11 @@ impl Provider for PoisonInjectingProvider { metadata, cancelled_activities, ) - .await + .await?; + self.failure_commit_collision_notifications + .fetch_add(collision_count, Ordering::SeqCst); + self.collision_notices.lock().expect("notices lock").extend(notices); + Ok(()) } async fn abandon_orchestration_item( @@ -208,9 +304,27 @@ impl Provider for PoisonInjectingProvider { delay: Option, ignore_attempt: bool, ) -> Result<(), ProviderError> { - self.inner + if self.fail_orchestration_abandon.load(Ordering::SeqCst) { + self.failed_abandon_calls.fetch_add(1, Ordering::SeqCst); + return Err(ProviderError::retryable( + "abandon_orchestration_item", + "injected persistent abandon failure", + )); + } + let result = self + .inner .abandon_orchestration_item(lock_token, delay, ignore_attempt) - .await + .await; + if ignore_attempt && result.is_ok() { + self.successful_ignored_abandons.fetch_add(1, Ordering::SeqCst); + } + if !ignore_attempt + && result.is_ok() + && let Some(delay) = delay + { + self.counted_abandon_delays.lock().expect("delays lock").push(delay); + } + result } async fn ack_work_item(&self, token: &str, completion: Option) -> Result<(), ProviderError> { diff --git a/tests/fixtures/queue_race_old_engine.json b/tests/fixtures/queue_race_old_engine.json new file mode 100644 index 0000000..ba0859c --- /dev/null +++ b/tests/fixtures/queue_race_old_engine.json @@ -0,0 +1,6903 @@ +{ + "recording_source": "6a458861763a7aa5b78a7c1c97691a6f00489a8b", + "compiled_lib_sha256": "c0f28773f23f9f0f1bab78d3472456a45c63e5dc6fbdd21854029fee2c1129a3", + "compiled_replay_engine_sha256": "b85bf87ec75d63db0befff18e9dc32d2385b721055b441b42c607477f0242deb", + "recording_test_sha256": "7076f5f6024f22a8006351bc71c68571db5d62e2d5b33ddd590f6e8b93f0f173", + "recording_comparator_sha256": "0d97d689ce320357329746a0e9634615c5e25d225d177e7c297f9104034af72e", + "snapshots": [ + { + "name": "ordinary/turn-0", + "handler": { + "mode": "Ordinary", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373618, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373618, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "ordinary", + "input": "input" + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "2:activity:ordinary:input" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "ordinary/turn-1", + "handler": { + "mode": "Ordinary", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373618, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373618, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "ordinary", + "input": "input" + }, + { + "event_id": 3, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "ActivityCompleted", + "result": "ok" + }, + { + "event_id": 4, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373639 + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "4:timer" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "ordinary/turn-2", + "handler": { + "mode": "Ordinary", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373618, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373618, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "ordinary", + "input": "input" + }, + { + "event_id": 3, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "ActivityCompleted", + "result": "ok" + }, + { + "event_id": 4, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373639 + }, + { + "event_id": 5, + "source_event_id": 4, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373639 + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "6:queue:q" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "ordinary/turn-3", + "handler": { + "mode": "Ordinary", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373618, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373618, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "ordinary", + "input": "input" + }, + { + "event_id": 3, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "ActivityCompleted", + "result": "ok" + }, + { + "event_id": 4, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373639 + }, + { + "event_id": 5, + "source_event_id": 4, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373639 + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 7, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 8, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "other" + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "8:queue:other" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "ordinary/turn-4", + "handler": { + "mode": "Ordinary", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373618, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373618, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "ordinary", + "input": "input" + }, + { + "event_id": 3, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "ActivityCompleted", + "result": "ok" + }, + { + "event_id": 4, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373639 + }, + { + "event_id": 5, + "source_event_id": 4, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373639 + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 7, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 8, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "other" + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "other", + "data": "second" + } + ], + "old_live_outcome": { + "Completed": "ok:first:second" + }, + "old_live_actions": [], + "old_replay_outcome": { + "Completed": "ok:first:second" + }, + "old_replay_actions": [] + }, + { + "name": "message-first-Loop-lazy-false/turn-0", + "handler": { + "mode": "Loop", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373639 + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "2:queue:q", + "3:timer" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "message-first-Loop-lazy-false/turn-1", + "handler": { + "mode": "Loop", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373639 + }, + { + "event_id": 4, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 5, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373639 + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 7, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373639 + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "6:queue:q", + "7:timer" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "message-first-Loop-lazy-false/turn-2", + "handler": { + "mode": "Loop", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373639 + }, + { + "event_id": 4, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 5, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373639 + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 7, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373639 + }, + { + "event_id": 8, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "second" + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "checkpoint", + "input": "0:first,second" + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "9:activity:checkpoint:0:first,second" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "message-first-Loop-lazy-false/turn-3", + "handler": { + "mode": "Loop", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373639 + }, + { + "event_id": 4, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 5, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373639 + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 7, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373639 + }, + { + "event_id": 8, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "second" + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "checkpoint", + "input": "0:first,second" + }, + { + "event_id": 10, + "source_event_id": 9, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373619, + "duroxide_version": "0.1.30", + "type": "ActivityCompleted", + "result": "ok" + } + ], + "old_live_outcome": { + "Completed": "0:first,second" + }, + "old_live_actions": [], + "old_replay_outcome": { + "Completed": "0:first,second" + }, + "old_replay_actions": [] + }, + { + "name": "adjacent-turn-Loop-lazy-false/turn-0", + "handler": { + "mode": "Loop", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373640 + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "2:queue:q", + "3:timer" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "adjacent-turn-Loop-lazy-false/turn-1", + "handler": { + "mode": "Loop", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373640 + }, + { + "event_id": 4, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373640 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373640 + }, + { + "event_id": 7, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "5:queue:q", + "6:timer" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "adjacent-turn-Loop-lazy-false/turn-2", + "handler": { + "mode": "Loop", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373640 + }, + { + "event_id": 4, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373640 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373640 + }, + { + "event_id": 7, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + }, + { + "event_id": 8, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 10, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373640 + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "9:queue:q", + "10:timer" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "adjacent-turn-Loop-lazy-false/turn-3", + "handler": { + "mode": "Loop", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373640 + }, + { + "event_id": 4, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373640 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373640 + }, + { + "event_id": 7, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + }, + { + "event_id": 8, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 10, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373640 + }, + { + "event_id": 11, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "second" + }, + { + "event_id": 12, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "checkpoint", + "input": "1:first,second" + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "12:activity:checkpoint:1:first,second" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "adjacent-turn-Loop-lazy-false/turn-4", + "handler": { + "mode": "Loop", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373640 + }, + { + "event_id": 4, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373640 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373640 + }, + { + "event_id": 7, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + }, + { + "event_id": 8, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 10, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373640 + }, + { + "event_id": 11, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "second" + }, + { + "event_id": 12, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "checkpoint", + "input": "1:first,second" + }, + { + "event_id": 13, + "source_event_id": 12, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373620, + "duroxide_version": "0.1.30", + "type": "ActivityCompleted", + "result": "ok" + } + ], + "old_live_outcome": { + "Completed": "1:first,second" + }, + "old_live_actions": [], + "old_replay_outcome": { + "Completed": "1:first,second" + }, + "old_replay_actions": [] + }, + { + "name": "message-first-Drain-lazy-false/turn-0", + "handler": { + "mode": "Drain", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373641 + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "2:queue:q", + "3:timer" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "message-first-Drain-lazy-false/turn-1", + "handler": { + "mode": "Drain", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373641 + }, + { + "event_id": 4, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 5, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373641 + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 7, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373641 + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "6:queue:q", + "7:timer" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "message-first-Drain-lazy-false/turn-2", + "handler": { + "mode": "Drain", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373641 + }, + { + "event_id": 4, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 5, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373641 + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 7, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373641 + }, + { + "event_id": 8, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "second" + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "checkpoint", + "input": "0:first,second" + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "9:activity:checkpoint:0:first,second" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "message-first-Drain-lazy-false/turn-3", + "handler": { + "mode": "Drain", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373641 + }, + { + "event_id": 4, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 5, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373641 + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 7, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373641 + }, + { + "event_id": 8, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "second" + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "checkpoint", + "input": "0:first,second" + }, + { + "event_id": 10, + "source_event_id": 9, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ActivityCompleted", + "result": "ok" + } + ], + "old_live_outcome": { + "Completed": "0:first,second" + }, + "old_live_actions": [], + "old_replay_outcome": { + "Completed": "0:first,second" + }, + "old_replay_actions": [] + }, + { + "name": "adjacent-turn-Drain-lazy-false/turn-0", + "handler": { + "mode": "Drain", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373641 + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "2:queue:q", + "3:timer" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "adjacent-turn-Drain-lazy-false/turn-1", + "handler": { + "mode": "Drain", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373641 + }, + { + "event_id": 4, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373641 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 6, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "5:queue:q" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "adjacent-turn-Drain-lazy-false/turn-2", + "handler": { + "mode": "Drain", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373641 + }, + { + "event_id": 4, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373641 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 6, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + }, + { + "event_id": 7, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 8, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373641 + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "8:queue:q", + "9:timer" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "adjacent-turn-Drain-lazy-false/turn-3", + "handler": { + "mode": "Drain", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373641 + }, + { + "event_id": 4, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373641 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 6, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + }, + { + "event_id": 7, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 8, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373641 + }, + { + "event_id": 10, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "second" + }, + { + "event_id": 11, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "checkpoint", + "input": "1:first,second" + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "11:activity:checkpoint:1:first,second" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "adjacent-turn-Drain-lazy-false/turn-4", + "handler": { + "mode": "Drain", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373641 + }, + { + "event_id": 4, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373641 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 6, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + }, + { + "event_id": 7, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 8, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373641 + }, + { + "event_id": 10, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "second" + }, + { + "event_id": 11, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373621, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "checkpoint", + "input": "1:first,second" + }, + { + "event_id": 12, + "source_event_id": 11, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ActivityCompleted", + "result": "ok" + } + ], + "old_live_outcome": { + "Completed": "1:first,second" + }, + "old_live_actions": [], + "old_replay_outcome": { + "Completed": "1:first,second" + }, + "old_replay_actions": [] + }, + { + "name": "timer-first-barrier-lazy-false/turn-0", + "handler": { + "mode": "Barrier", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373642 + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "2:queue:q", + "3:timer" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "timer-first-barrier-lazy-false/turn-1", + "handler": { + "mode": "Barrier", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373642 + }, + { + "event_id": 4, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373642 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "barrier", + "input": "" + }, + { + "event_id": 7, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "6:activity:barrier:" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "timer-first-barrier-lazy-false/turn-2", + "handler": { + "mode": "Barrier", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373642 + }, + { + "event_id": 4, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373642 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "barrier", + "input": "" + }, + { + "event_id": 7, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + }, + { + "event_id": 8, + "source_event_id": 6, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ActivityCompleted", + "result": "ok" + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 10, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 11, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373642 + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "9:queue:q", + "10:queue:q", + "11:timer" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "timer-first-barrier-lazy-false/turn-3", + "handler": { + "mode": "Barrier", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373642 + }, + { + "event_id": 4, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373642 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "barrier", + "input": "" + }, + { + "event_id": 7, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + }, + { + "event_id": 8, + "source_event_id": 6, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ActivityCompleted", + "result": "ok" + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 10, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 11, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373642 + }, + { + "event_id": 12, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "second" + }, + { + "event_id": 13, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "checkpoint", + "input": "1:first,second" + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "13:activity:checkpoint:1:first,second" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "timer-first-barrier-lazy-false/turn-4", + "handler": { + "mode": "Barrier", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373642 + }, + { + "event_id": 4, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373642 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "barrier", + "input": "" + }, + { + "event_id": 7, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + }, + { + "event_id": 8, + "source_event_id": 6, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ActivityCompleted", + "result": "ok" + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 10, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 11, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373642 + }, + { + "event_id": 12, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "second" + }, + { + "event_id": 13, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "checkpoint", + "input": "1:first,second" + }, + { + "event_id": 14, + "source_event_id": 13, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373622, + "duroxide_version": "0.1.30", + "type": "ActivityCompleted", + "result": "ok" + } + ], + "old_live_outcome": { + "Completed": "1:first,second" + }, + "old_live_actions": [], + "old_replay_outcome": { + "Completed": "1:first,second" + }, + "old_replay_actions": [] + }, + { + "name": "message-first-Loop-lazy-true/turn-0", + "handler": { + "mode": "Loop", + "lazy": true + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373643 + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "2:queue:q", + "3:timer" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "message-first-Loop-lazy-true/turn-1", + "handler": { + "mode": "Loop", + "lazy": true + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373643 + }, + { + "event_id": 4, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 5, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373643 + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 7, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373643 + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "6:queue:q", + "7:timer" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "message-first-Loop-lazy-true/turn-2", + "handler": { + "mode": "Loop", + "lazy": true + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373643 + }, + { + "event_id": 4, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 5, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373643 + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 7, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373643 + }, + { + "event_id": 8, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "second" + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "checkpoint", + "input": "0:first,second" + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "9:activity:checkpoint:0:first,second" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "message-first-Loop-lazy-true/turn-3", + "handler": { + "mode": "Loop", + "lazy": true + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373643 + }, + { + "event_id": 4, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 5, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373643 + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 7, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373643 + }, + { + "event_id": 8, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "second" + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "checkpoint", + "input": "0:first,second" + }, + { + "event_id": 10, + "source_event_id": 9, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "ActivityCompleted", + "result": "ok" + } + ], + "old_live_outcome": { + "Completed": "0:first,second" + }, + "old_live_actions": [], + "old_replay_outcome": { + "Completed": "0:first,second" + }, + "old_replay_actions": [] + }, + { + "name": "adjacent-turn-Loop-lazy-true/turn-0", + "handler": { + "mode": "Loop", + "lazy": true + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373643 + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "2:queue:q", + "3:timer" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "adjacent-turn-Loop-lazy-true/turn-1", + "handler": { + "mode": "Loop", + "lazy": true + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373643 + }, + { + "event_id": 4, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373643 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373643 + }, + { + "event_id": 7, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "5:queue:q", + "6:timer" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "adjacent-turn-Loop-lazy-true/turn-2", + "handler": { + "mode": "Loop", + "lazy": true + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373643 + }, + { + "event_id": 4, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373643 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373643 + }, + { + "event_id": 7, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + }, + { + "event_id": 8, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 10, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373644 + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "9:queue:q", + "10:timer" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "adjacent-turn-Loop-lazy-true/turn-3", + "handler": { + "mode": "Loop", + "lazy": true + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373643 + }, + { + "event_id": 4, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373643 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373643 + }, + { + "event_id": 7, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + }, + { + "event_id": 8, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 10, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373644 + }, + { + "event_id": 11, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "second" + }, + { + "event_id": 12, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "checkpoint", + "input": "1:first,second" + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "12:activity:checkpoint:1:first,second" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "adjacent-turn-Loop-lazy-true/turn-4", + "handler": { + "mode": "Loop", + "lazy": true + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373643 + }, + { + "event_id": 4, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373643 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373643 + }, + { + "event_id": 7, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373623, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + }, + { + "event_id": 8, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 10, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373644 + }, + { + "event_id": 11, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "second" + }, + { + "event_id": 12, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "checkpoint", + "input": "1:first,second" + }, + { + "event_id": 13, + "source_event_id": 12, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "ActivityCompleted", + "result": "ok" + } + ], + "old_live_outcome": { + "Completed": "1:first,second" + }, + "old_live_actions": [], + "old_replay_outcome": { + "Completed": "1:first,second" + }, + "old_replay_actions": [] + }, + { + "name": "message-first-Drain-lazy-true/turn-0", + "handler": { + "mode": "Drain", + "lazy": true + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373644 + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "2:queue:q", + "3:timer" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "message-first-Drain-lazy-true/turn-1", + "handler": { + "mode": "Drain", + "lazy": true + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373644 + }, + { + "event_id": 4, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 5, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373644 + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 7, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373644 + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "6:queue:q", + "7:timer" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "message-first-Drain-lazy-true/turn-2", + "handler": { + "mode": "Drain", + "lazy": true + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373644 + }, + { + "event_id": 4, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 5, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373644 + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 7, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373644 + }, + { + "event_id": 8, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "second" + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "checkpoint", + "input": "0:first,second" + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "9:activity:checkpoint:0:first,second" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "message-first-Drain-lazy-true/turn-3", + "handler": { + "mode": "Drain", + "lazy": true + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373644 + }, + { + "event_id": 4, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 5, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373644 + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 7, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373644 + }, + { + "event_id": 8, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "second" + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373624, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "checkpoint", + "input": "0:first,second" + }, + { + "event_id": 10, + "source_event_id": 9, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "ActivityCompleted", + "result": "ok" + } + ], + "old_live_outcome": { + "Completed": "0:first,second" + }, + "old_live_actions": [], + "old_replay_outcome": { + "Completed": "0:first,second" + }, + "old_replay_actions": [] + }, + { + "name": "adjacent-turn-Drain-lazy-true/turn-0", + "handler": { + "mode": "Drain", + "lazy": true + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373645 + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "2:queue:q", + "3:timer" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "adjacent-turn-Drain-lazy-true/turn-1", + "handler": { + "mode": "Drain", + "lazy": true + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373645 + }, + { + "event_id": 4, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373645 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 6, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "5:queue:q" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "adjacent-turn-Drain-lazy-true/turn-2", + "handler": { + "mode": "Drain", + "lazy": true + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373645 + }, + { + "event_id": 4, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373645 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 6, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + }, + { + "event_id": 7, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 8, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373645 + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "8:queue:q", + "9:timer" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "adjacent-turn-Drain-lazy-true/turn-3", + "handler": { + "mode": "Drain", + "lazy": true + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373645 + }, + { + "event_id": 4, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373645 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 6, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + }, + { + "event_id": 7, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 8, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373645 + }, + { + "event_id": 10, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "second" + }, + { + "event_id": 11, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "checkpoint", + "input": "1:first,second" + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "11:activity:checkpoint:1:first,second" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "adjacent-turn-Drain-lazy-true/turn-4", + "handler": { + "mode": "Drain", + "lazy": true + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373645 + }, + { + "event_id": 4, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373645 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 6, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + }, + { + "event_id": 7, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 8, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373645 + }, + { + "event_id": 10, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "second" + }, + { + "event_id": 11, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "checkpoint", + "input": "1:first,second" + }, + { + "event_id": 12, + "source_event_id": 11, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373625, + "duroxide_version": "0.1.30", + "type": "ActivityCompleted", + "result": "ok" + } + ], + "old_live_outcome": { + "Completed": "1:first,second" + }, + "old_live_actions": [], + "old_replay_outcome": { + "Completed": "1:first,second" + }, + "old_replay_actions": [] + }, + { + "name": "timer-first-barrier-lazy-true/turn-0", + "handler": { + "mode": "Barrier", + "lazy": true + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373646 + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "2:queue:q", + "3:timer" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "timer-first-barrier-lazy-true/turn-1", + "handler": { + "mode": "Barrier", + "lazy": true + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373646 + }, + { + "event_id": 4, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373646 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "barrier", + "input": "" + }, + { + "event_id": 7, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "6:activity:barrier:" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "timer-first-barrier-lazy-true/turn-2", + "handler": { + "mode": "Barrier", + "lazy": true + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373646 + }, + { + "event_id": 4, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373646 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "barrier", + "input": "" + }, + { + "event_id": 7, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + }, + { + "event_id": 8, + "source_event_id": 6, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ActivityCompleted", + "result": "ok" + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 10, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 11, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373646 + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "9:queue:q", + "10:queue:q", + "11:timer" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "timer-first-barrier-lazy-true/turn-3", + "handler": { + "mode": "Barrier", + "lazy": true + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373646 + }, + { + "event_id": 4, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373646 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "barrier", + "input": "" + }, + { + "event_id": 7, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + }, + { + "event_id": 8, + "source_event_id": 6, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ActivityCompleted", + "result": "ok" + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 10, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 11, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373646 + }, + { + "event_id": 12, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "second" + }, + { + "event_id": 13, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "checkpoint", + "input": "1:first,second" + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "13:activity:checkpoint:1:first,second" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "timer-first-barrier-lazy-true/turn-4", + "handler": { + "mode": "Barrier", + "lazy": true + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373646 + }, + { + "event_id": 4, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373646 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "barrier", + "input": "" + }, + { + "event_id": 7, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + }, + { + "event_id": 8, + "source_event_id": 6, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ActivityCompleted", + "result": "ok" + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 10, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 11, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373646 + }, + { + "event_id": 12, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "second" + }, + { + "event_id": 13, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "checkpoint", + "input": "1:first,second" + }, + { + "event_id": 14, + "source_event_id": 13, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ActivityCompleted", + "result": "ok" + } + ], + "old_live_outcome": { + "Completed": "1:first,second" + }, + "old_live_actions": [], + "old_replay_outcome": { + "Completed": "1:first,second" + }, + "old_replay_actions": [] + }, + { + "name": "prebound-queue-versus-second-timer/turn-0", + "handler": { + "mode": "PreboundCompetition", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 4, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373646 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373656 + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "2:queue:q", + "3:queue:q", + "4:timer", + "5:timer" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "prebound-queue-versus-second-timer/turn-1", + "handler": { + "mode": "PreboundCompetition", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 4, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373646 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373656 + }, + { + "event_id": 6, + "source_event_id": 4, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373646 + }, + { + "event_id": 7, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 8, + "source_event_id": 5, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373656 + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "checkpoint", + "input": "second-timeout" + }, + { + "event_id": 10, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + }, + { + "event_id": 11, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "9:activity:checkpoint:second-timeout" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "prebound-queue-versus-second-timer/turn-2", + "handler": { + "mode": "PreboundCompetition", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 4, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373646 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373626, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373656 + }, + { + "event_id": 6, + "source_event_id": 4, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373646 + }, + { + "event_id": 7, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 8, + "source_event_id": 5, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373656 + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "checkpoint", + "input": "second-timeout" + }, + { + "event_id": 10, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + }, + { + "event_id": 11, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + }, + { + "event_id": 12, + "source_event_id": 9, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ActivityCompleted", + "result": "ok" + } + ], + "old_live_outcome": { + "Completed": "second-timeout" + }, + "old_live_actions": [], + "old_replay_outcome": { + "Completed": "second-timeout" + }, + "old_replay_actions": [] + }, + { + "name": "unbound-prearrival/turn-0", + "handler": { + "mode": "Unbound", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 4, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 5, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "3:queue:q", + "4:queue:q" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "unbound-prearrival/turn-1", + "handler": { + "mode": "Unbound", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 4, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 5, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "second" + }, + { + "event_id": 7, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "checkpoint", + "input": "first" + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "7:activity:checkpoint:first" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "unbound-prearrival/turn-2", + "handler": { + "mode": "Unbound", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "first" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 4, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 5, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistentCancelled", + "reason": "dropped_future" + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "second" + }, + { + "event_id": 7, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "checkpoint", + "input": "first" + }, + { + "event_id": 8, + "source_event_id": 7, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ActivityCompleted", + "result": "ok" + } + ], + "old_live_outcome": { + "Completed": "first" + }, + "old_live_actions": [], + "old_replay_outcome": { + "Completed": "first" + }, + "old_replay_actions": [] + }, + { + "name": "positional-barrier/turn-0", + "handler": { + "mode": "PositionalBarrier", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribed", + "name": "s" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373637 + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "2:signal:s", + "3:timer" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "positional-barrier/turn-1", + "handler": { + "mode": "PositionalBarrier", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribed", + "name": "s" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373637 + }, + { + "event_id": 4, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373637 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ExternalEvent", + "name": "s", + "data": "a" + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "tick", + "input": "" + }, + { + "event_id": 7, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedCancelled", + "reason": "dropped_future" + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "6:activity:tick:" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "positional-barrier/turn-2", + "handler": { + "mode": "PositionalBarrier", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribed", + "name": "s" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373637 + }, + { + "event_id": 4, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373637 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ExternalEvent", + "name": "s", + "data": "a" + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "tick", + "input": "" + }, + { + "event_id": 7, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373627, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedCancelled", + "reason": "dropped_future" + }, + { + "event_id": 8, + "source_event_id": 6, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ActivityCompleted", + "result": "ok" + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribed", + "name": "s" + }, + { + "event_id": 10, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "checkpoint", + "input": "a" + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "9:signal:s", + "10:activity:checkpoint:a" + ], + "old_replay_outcome": { + "Failed": "nondeterministic: history schedule but no emitted action" + }, + "old_replay_actions": [] + }, + { + "name": "positional-blocking/turn-0", + "handler": { + "mode": "PositionalBlocking", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribed", + "name": "s" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373638 + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "2:signal:s", + "3:timer" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "positional-blocking/turn-1", + "handler": { + "mode": "PositionalBlocking", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribed", + "name": "s" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373638 + }, + { + "event_id": 4, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373638 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ExternalEvent", + "name": "s", + "data": "a" + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribed", + "name": "s" + }, + { + "event_id": 7, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedCancelled", + "reason": "dropped_future" + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "6:signal:s" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "positional-blocking/turn-2", + "handler": { + "mode": "PositionalBlocking", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribed", + "name": "s" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373638 + }, + { + "event_id": 4, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373638 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ExternalEvent", + "name": "s", + "data": "a" + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribed", + "name": "s" + }, + { + "event_id": 7, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedCancelled", + "reason": "dropped_future" + }, + { + "event_id": 8, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ExternalEvent", + "name": "s", + "data": "b" + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "positional-resolve-at-creation/turn-0", + "handler": { + "mode": "ResolveAtCreation", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribed", + "name": "s" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "work", + "input": "" + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "2:signal:s", + "3:activity:work:" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "positional-resolve-at-creation/turn-1", + "handler": { + "mode": "ResolveAtCreation", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribed", + "name": "s" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "work", + "input": "" + }, + { + "event_id": 4, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ExternalEvent", + "name": "s", + "data": "a" + }, + { + "event_id": 5, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ActivityCompleted", + "result": "ok" + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "work2", + "input": "" + }, + { + "event_id": 7, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedCancelled", + "reason": "dropped_future" + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "6:activity:work2:" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "positional-resolve-at-creation/turn-2", + "handler": { + "mode": "ResolveAtCreation", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribed", + "name": "s" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "work", + "input": "" + }, + { + "event_id": 4, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ExternalEvent", + "name": "s", + "data": "a" + }, + { + "event_id": 5, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ActivityCompleted", + "result": "ok" + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "work2", + "input": "" + }, + { + "event_id": 7, + "source_event_id": 2, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedCancelled", + "reason": "dropped_future" + }, + { + "event_id": 8, + "source_event_id": 6, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ActivityCompleted", + "result": "ok" + }, + { + "event_id": 9, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribed", + "name": "s" + }, + { + "event_id": 10, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ActivityScheduled", + "name": "checkpoint", + "input": "a" + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "9:signal:s", + "10:activity:checkpoint:a" + ], + "old_replay_outcome": { + "Failed": "nondeterministic: history schedule but no emitted action" + }, + "old_replay_actions": [] + }, + { + "name": "continue-drop/turn-0", + "handler": { + "mode": "ContinueDrop", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373638 + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "2:queue:q", + "3:timer" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "continue-drop/turn-1", + "handler": { + "mode": "ContinueDrop", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373638 + }, + { + "event_id": 4, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373638 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373628, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "a" + } + ], + "old_live_outcome": { + "ContinueAsNew": "successor" + }, + "old_live_actions": [ + "continue:successor:None" + ], + "old_replay_outcome": { + "ContinueAsNew": "successor" + }, + "old_replay_actions": [ + "continue:successor:None" + ] + }, + { + "name": "continue-drain/turn-0", + "handler": { + "mode": "ContinueDrain", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373629, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373629, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373629, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373639 + } + ], + "old_live_outcome": "Continue", + "old_live_actions": [ + "2:queue:q", + "3:timer" + ], + "old_replay_outcome": "Continue", + "old_replay_actions": [] + }, + { + "name": "continue-drain/turn-1", + "handler": { + "mode": "ContinueDrain", + "lazy": false + }, + "history": [ + { + "event_id": 1, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373629, + "duroxide_version": "0.1.30", + "type": "OrchestrationStarted", + "name": "TestOrch", + "version": "1.0.0", + "input": "test-input", + "parent_instance": null, + "parent_id": null + }, + { + "event_id": 2, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373629, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + }, + { + "event_id": 3, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373629, + "duroxide_version": "0.1.30", + "type": "TimerCreated", + "fire_at_ms": 1791400373639 + }, + { + "event_id": 4, + "source_event_id": 3, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373629, + "duroxide_version": "0.1.30", + "type": "TimerFired", + "fire_at_ms": 1791400373639 + }, + { + "event_id": 5, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373629, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "a" + }, + { + "event_id": 6, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373629, + "duroxide_version": "0.1.30", + "type": "ExternalEventPersistent", + "name": "q", + "data": "b" + }, + { + "event_id": 7, + "source_event_id": null, + "instance_id": "test-instance", + "execution_id": 1, + "timestamp_ms": 1791400373629, + "duroxide_version": "0.1.30", + "type": "ExternalSubscribedPersistent", + "name": "q" + } + ], + "old_live_outcome": { + "ContinueAsNew": "successor" + }, + "old_live_actions": [ + "7:queue:q", + "continue:successor:None" + ], + "old_replay_outcome": { + "ContinueAsNew": "successor" + }, + "old_replay_actions": [ + "continue:successor:None" + ] + } + ] +} \ No newline at end of file diff --git a/tests/queue_event_tests.rs b/tests/queue_event_tests.rs index 8d01925..979b471 100644 --- a/tests/queue_event_tests.rs +++ b/tests/queue_event_tests.rs @@ -14,6 +14,95 @@ use duroxide::{ActivityContext, OrchestrationContext, OrchestrationRegistry}; mod common; use std::time::Duration; +#[tokio::test] +async fn immediate_0131_queue_race_replays_same_batch_after_restart() { + use duroxide::Event; + use duroxide::providers::{ExecutionMetadata, WorkItem}; + + let (store, _td) = common::create_sqlite_store_disk().await; + let instance = "queue-race-restart"; + common::seed_instance_with_pinned_version(store.as_ref(), instance, "QueueRace", semver::Version::new(0, 1, 31)) + .await; + + // Persist a 0.1.31 live turn that timed out, then consumed "first" using a new + // subscription, before recording the losing subscription's cancellation. + let event = |id, source, kind| Event::with_event_id(id, instance, 1, source, kind); + common::seed_history_turn( + store.as_ref(), + WorkItem::QueueMessage { + instance: instance.into(), + name: "q".into(), + data: "first".into(), + }, + 1, + vec![ + event(2, None, EventKind::QueueSubscribed { name: "q".into() }), + event(3, None, EventKind::TimerCreated { fire_at_ms: 1000 }), + event(4, Some(3), EventKind::TimerFired { fire_at_ms: 1000 }), + event( + 5, + None, + EventKind::QueueEventDelivered { + name: "q".into(), + data: "first".into(), + }, + ), + event(6, None, EventKind::QueueSubscribed { name: "q".into() }), + event(7, None, EventKind::TimerCreated { fire_at_ms: 1000 }), + event(8, None, EventKind::QueueSubscribed { name: "q".into() }), + event(9, None, EventKind::TimerCreated { fire_at_ms: 1000 }), + event( + 10, + Some(2), + EventKind::QueueSubscriptionCancelled { + reason: "dropped_future".into(), + }, + ), + ], + vec![], + ExecutionMetadata::default(), + ) + .await; + + let orchestration = |ctx: OrchestrationContext, _: String| async move { + let mut messages = Vec::new(); + let mut timers = 0; + while messages.len() < 2 { + match ctx + .select2(ctx.dequeue_event("q"), ctx.schedule_timer(Duration::from_millis(20))) + .await + { + Either2::First(message) => messages.push(message), + Either2::Second(()) => timers += 1, + } + } + Ok(format!("{timers}:{}", messages.join(","))) + }; + let rt = runtime::Runtime::start_with_options( + store.clone(), + ActivityRegistry::builder().build(), + OrchestrationRegistry::builder() + .register("QueueRace", orchestration) + .build(), + runtime::RuntimeOptions { + dispatcher_min_poll_interval: Duration::from_millis(10), + ..Default::default() + }, + ) + .await; + let client = Client::new(store); + client.enqueue_event(instance, "q", "second").await.unwrap(); + let status = client + .wait_for_orchestration(instance, Duration::from_secs(5)) + .await + .unwrap(); + assert!( + matches!(&status, runtime::OrchestrationStatus::Completed { output, .. } if output == "1:first,second"), + "Expected deterministic FIFO replay after restart, got {status:?}" + ); + rt.shutdown(None).await; +} + /// Basic persistent event delivery: raise then subscribe. #[tokio::test] async fn persistent_event_basic_delivery() { @@ -1270,3 +1359,89 @@ async fn events_enqueued_before_start_orchestration_are_dropped() { rt.shutdown(None).await; } + +/// The exact unread list is still capped at 100 at continue-as-new: the oldest 100 +/// arrivals survive in cross-name arrival order, and only the overflow is dropped with +/// a warning. +#[tokio::test] +async fn continue_as_new_carries_the_oldest_100_unread_arrivals_across_names() { + let (logs, _guard) = common::tracing_capture::install_tracing_capture(); + let (store, _td) = common::create_sqlite_store_disk().await; + let orch = |ctx: OrchestrationContext, _: String| async move { + if ctx.execution_id() == 1 { + // Hold the execution open while queue messages arrive unread. + ctx.schedule_wait("go").await; + return ctx.continue_as_new("next").await; + } + Ok("done".to_string()) + }; + let rt = runtime::Runtime::start_with_options( + store.clone(), + ActivityRegistry::builder().build(), + OrchestrationRegistry::builder().register("CarryCap", orch).build(), + runtime::RuntimeOptions { + dispatcher_min_poll_interval: Duration::from_millis(10), + ..Default::default() + }, + ) + .await; + let client = Client::new(store.clone()); + for total in [100usize, 101] { + let instance = format!("carry-cap-{total}"); + client.start_orchestration(&instance, "CarryCap", "").await.unwrap(); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + let history = store.read_with_execution(&instance, 1).await.unwrap_or_default(); + if history + .iter() + .any(|e| matches!(&e.kind, EventKind::ExternalSubscribed { name } if name == "go")) + { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("the wait must be bound before the signal is raised"); + let sent: Vec<(String, String)> = (0..total) + .map(|i| { + let name = if i % 2 == 0 { "a" } else { "b" }; + (name.to_string(), format!("{name}{i}")) + }) + .collect(); + for (name, data) in &sent { + client.enqueue_event(&instance, name, data).await.unwrap(); + } + logs.lock().unwrap().clear(); + client.raise_event(&instance, "go", "").await.unwrap(); + let status = client + .wait_for_orchestration(&instance, Duration::from_secs(10)) + .await + .unwrap(); + assert!( + matches!(&status, runtime::OrchestrationStatus::Completed { output, .. } if output == "done"), + "{status:?}" + ); + let successor = store.read_with_execution(&instance, 2).await.unwrap(); + let carried = successor + .iter() + .find_map(|e| match &e.kind { + EventKind::OrchestrationStarted { + carry_forward_events, .. + } => Some(carry_forward_events.clone().unwrap_or_default()), + _ => None, + }) + .expect("successor start"); + assert_eq!(carried, sent[..total.min(100)].to_vec(), "{total} unread arrivals"); + let dropped: Vec = logs + .lock() + .unwrap() + .iter() + .filter(|e| e.message.contains("Dropping carry-forward event beyond limit")) + .filter_map(|e| e.field("data")) + .collect(); + let expected: Vec = sent[total.min(100)..].iter().map(|(_, data)| data.clone()).collect(); + assert_eq!(dropped, expected, "only the overflow is dropped with a warning"); + } + rt.shutdown(None).await; +} diff --git a/tests/queue_race_upgrade.rs b/tests/queue_race_upgrade.rs new file mode 100644 index 0000000..9546985 --- /dev/null +++ b/tests/queue_race_upgrade.rs @@ -0,0 +1,678 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::clone_on_ref_ptr)] + +#[path = "replay_engine/helpers.rs"] +#[allow(dead_code)] +mod helpers; + +#[path = "common/cancellation_history.rs"] +mod cancellation_history; +use cancellation_history::decision_delta; + +use async_trait::async_trait; +use duroxide::providers::WorkItem; +use duroxide::runtime::replay_engine::TurnResult; +use duroxide::{Action, Either2, Event, EventKind, OrchestrationContext, OrchestrationHandler}; +use helpers::*; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use std::time::Duration; + +const OLD_SHA: &str = "6a458861763a7aa5b78a7c1c97691a6f00489a8b"; +const OLD_LIB_SHA256: &str = "c0f28773f23f9f0f1bab78d3472456a45c63e5dc6fbdd21854029fee2c1129a3"; +const OLD_ENGINE_SHA256: &str = "b85bf87ec75d63db0befff18e9dc32d2385b721055b441b42c607477f0242deb"; + +#[derive(Clone, Copy, Debug, Deserialize, Serialize)] +enum Mode { + Ordinary, + Loop, + Drain, + Barrier, + PreboundCompetition, + Unbound, + PositionalBarrier, + PositionalBlocking, + ResolveAtCreation, + ContinueDrop, + ContinueDrain, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +struct Handler { + mode: Mode, + lazy: bool, +} + +impl Handler { + async fn race(&self, ctx: &OrchestrationContext) -> Either2 { + if self.lazy { + ctx.select2(async { ctx.dequeue_event("q").await }, async { + ctx.schedule_timer(Duration::from_millis(20)).await + }) + .await + } else { + ctx.select2(ctx.dequeue_event("q"), ctx.schedule_timer(Duration::from_millis(20))) + .await + } + } +} + +#[async_trait] +impl OrchestrationHandler for Handler { + async fn invoke(&self, ctx: OrchestrationContext, _: String) -> Result { + let value = match self.mode { + Mode::PositionalBarrier | Mode::PositionalBlocking => { + let result = ctx + .select2(ctx.schedule_wait("s"), ctx.schedule_timer(Duration::from_millis(10))) + .await; + if matches!(result, Either2::Second(())) && matches!(self.mode, Mode::PositionalBarrier) { + ctx.schedule_activity("tick", "").await?; + } + ctx.schedule_wait("s").await + } + Mode::ResolveAtCreation => { + let wait = ctx.schedule_wait("s"); + ctx.schedule_activity("work", "").await?; + drop(wait); + ctx.schedule_activity("work2", "").await?; + ctx.schedule_wait("s").await + } + Mode::ContinueDrop | Mode::ContinueDrain => { + let _ = ctx + .select2(ctx.dequeue_event("q"), ctx.schedule_timer(Duration::from_millis(10))) + .await; + if matches!(self.mode, Mode::ContinueDrain) { + ctx.dequeue_event("q").await; + } + return ctx.continue_as_new("successor").await; + } + Mode::Ordinary => { + let result = ctx.schedule_activity("ordinary", "input").await?; + ctx.schedule_timer(Duration::from_millis(20)).await; + let first = ctx.dequeue_event("q").await; + let second = ctx.dequeue_event("other").await; + return Ok(format!("{result}:{first}:{second}")); + } + Mode::PreboundCompetition => { + let first = ctx.dequeue_event("q"); + let second = ctx.dequeue_event("q"); + let first_timer = ctx.schedule_timer(Duration::from_millis(20)); + let second_timer = ctx.schedule_timer(Duration::from_millis(30)); + let _ = ctx.select2(first, first_timer).await; + match ctx.select2(second, second_timer).await { + Either2::First(message) => message, + Either2::Second(()) => "second-timeout".to_string(), + } + } + Mode::Unbound => { + drop(ctx.dequeue_event("q")); + ctx.dequeue_event("q").await + } + Mode::Loop | Mode::Drain | Mode::Barrier => { + let mut messages = Vec::new(); + let mut timers = 0; + while messages.len() < 2 { + match self.race(&ctx).await { + Either2::First(message) => messages.push(message), + Either2::Second(()) => { + timers += 1; + if matches!(self.mode, Mode::Barrier) { + ctx.schedule_activity("barrier", "").await?; + } + if matches!(self.mode, Mode::Drain | Mode::Barrier) { + messages.push(ctx.dequeue_event("q").await); + } + } + } + } + format!("{timers}:{}", messages.join(",")) + } + }; + ctx.schedule_activity("checkpoint", &value).await?; + Ok(value) + } +} + +#[derive(Debug, Deserialize, Serialize, PartialEq, Eq)] +enum Outcome { + Continue, + Completed(String), + Failed(String), + ContinueAsNew(String), +} + +fn outcome(result: TurnResult) -> Outcome { + match result { + TurnResult::Continue => Outcome::Continue, + TurnResult::Completed(value) => Outcome::Completed(value), + TurnResult::Failed(error) => Outcome::Failed(error.display_message()), + TurnResult::ContinueAsNew { input, .. } => Outcome::ContinueAsNew(input), + other => panic!("unexpected result {other:?}"), + } +} + +fn action_shapes(actions: &[Action]) -> Vec { + actions + .iter() + .map(|action| match action { + Action::CallActivity { + scheduling_event_id, + name, + input, + .. + } => format!("{scheduling_event_id}:activity:{name}:{input}"), + Action::CreateTimer { + scheduling_event_id, .. + } => format!("{scheduling_event_id}:timer"), + Action::DequeueEvent { + scheduling_event_id, + name, + } => format!("{scheduling_event_id}:queue:{name}"), + Action::WaitExternal { + scheduling_event_id, + name, + } => format!("{scheduling_event_id}:signal:{name}"), + Action::ContinueAsNew { input, version } => format!("continue:{input}:{version:?}"), + other => panic!("unexpected action {other:?}"), + }) + .collect() +} + +#[derive(Debug, Deserialize, Serialize)] +struct Snapshot { + name: String, + handler: Handler, + history: Vec, + old_live_outcome: Outcome, + old_live_actions: Vec, + old_replay_outcome: Outcome, + old_replay_actions: Vec, +} + +#[derive(Debug, Deserialize, Serialize)] +struct Fixtures { + recording_source: String, + compiled_lib_sha256: String, + compiled_replay_engine_sha256: String, + recording_test_sha256: String, + recording_comparator_sha256: String, + snapshots: Vec, +} + +fn read_fixtures() -> Fixtures { + let fixtures: Fixtures = serde_json::from_str(include_str!("fixtures/queue_race_old_engine.json")).unwrap(); + assert_eq!(fixtures.compiled_lib_sha256, OLD_LIB_SHA256); + assert_eq!(fixtures.compiled_replay_engine_sha256, OLD_ENGINE_SHA256); + assert_eq!( + fixtures.recording_test_sha256, + compiled_sha256(include_str!("queue_race_upgrade.rs")) + ); + assert_eq!( + fixtures.recording_comparator_sha256, + compiled_sha256(include_str!("common/cancellation_history.rs")) + ); + fixtures +} + +fn compiled_sha256(source: &str) -> String { + use sha2::{Digest, Sha256}; + format!("{:x}", Sha256::digest(source.as_bytes())) +} + +#[test] +fn legacy_independent_cancellation_permutations_replay_identically() { + let fixtures = read_fixtures(); + let snapshot = fixtures + .snapshots + .iter() + .find(|snapshot| snapshot.name == "prebound-queue-versus-second-timer/turn-1") + .unwrap(); + let mut reversed = snapshot.history.clone(); + let indices: Vec<_> = reversed + .iter() + .enumerate() + .filter_map(|(index, event)| { + matches!(&event.kind,EventKind::QueueSubscriptionCancelled{reason}if reason=="dropped_future") + .then_some(index) + }) + .collect(); + assert_eq!(indices.len(), 2); + let a = indices[0]; + let b = indices[1]; + assert_eq!(b, a + 1); + let id_a = reversed[a].event_id; + let id_b = reversed[b].event_id; + reversed.swap(a, b); + reversed[a].event_id = id_a; + reversed[b].event_id = id_b; + assert_eq!(decision_delta(&snapshot.history), decision_delta(&reversed)); + for history in [snapshot.history.clone(), reversed] { + let mut engine = create_engine(history); + assert_eq!( + outcome(execute(&mut engine, Arc::new(snapshot.handler.clone()))), + snapshot.old_replay_outcome + ); + assert!(engine.history_delta().is_empty()); + assert_eq!(action_shapes(engine.pending_actions()), snapshot.old_replay_actions); + } + // A different cancellation source is not a harmless permutation. + let mut changed = snapshot.history.clone(); + changed[a].source_event_id = Some(999); + assert_ne!(decision_delta(&snapshot.history), decision_delta(&changed)); + let mut changed = snapshot.history.clone(); + if let EventKind::QueueSubscriptionCancelled { reason } = &mut changed[a].kind { + *reason = "continued_as_new".into(); + } + assert_ne!(decision_delta(&snapshot.history), decision_delta(&changed)); + let mut changed = snapshot.history.clone(); + changed[b].source_event_id = changed[a].source_event_id; + assert_ne!(decision_delta(&snapshot.history), decision_delta(&changed)); + let mut changed = snapshot.history.clone(); + changed.swap(a, a - 1); + assert_ne!(decision_delta(&snapshot.history), decision_delta(&changed)); +} +#[test] +fn old_engine_recorded_queue_histories_replay_identically() { + let fixtures = read_fixtures(); + assert_eq!(fixtures.snapshots.len(), 70, "missing old-engine recordings"); + let mut mismatches = Vec::new(); + for snapshot in fixtures.snapshots { + assert!(snapshot.history.iter().all(|event| event.duroxide_version == "0.1.30")); + let mut engine = create_engine(snapshot.history); + let result = outcome(execute(&mut engine, Arc::new(snapshot.handler))); + let actions = action_shapes(engine.pending_actions()); + assert!( + engine.history_delta().is_empty(), + "{}: new replay history", + snapshot.name + ); + if result != snapshot.old_replay_outcome || actions != snapshot.old_replay_actions { + mismatches.push(format!( + "{}: old={:?}/{:?}, new={result:?}/{actions:?}", + snapshot.name, snapshot.old_replay_outcome, snapshot.old_replay_actions + )); + } + } + assert!( + mismatches.is_empty(), + "old history replay changed:\n{}", + mismatches.join("\n") + ); +} + +#[test] +fn old_engine_executions_resume_identically_after_upgrade() { + let fixtures = read_fixtures(); + let mut compared = 0; + for pair in fixtures.snapshots.windows(2) { + let [before, after] = pair else { unreachable!() }; + if before.name.rsplit_once('/').unwrap().0 != after.name.rsplit_once('/').unwrap().0 { + continue; + } + assert_eq!(&after.history[..before.history.len()], before.history.as_slice()); + let completions = after.history[before.history.len()..] + .iter() + .filter_map(|event| match &event.kind { + duroxide::EventKind::QueueEventDelivered { name, data } => Some(WorkItem::QueueMessage { + instance: TEST_INSTANCE.into(), + name: name.clone(), + data: data.clone(), + }), + duroxide::EventKind::TimerFired { fire_at_ms } => { + Some(timer_fired_msg(event.source_event_id.unwrap(), *fire_at_ms)) + } + duroxide::EventKind::ActivityCompleted { result } => { + Some(activity_completed_msg(event.source_event_id.unwrap(), result)) + } + duroxide::EventKind::ExternalEvent { name, data } => Some(WorkItem::ExternalRaised { + instance: TEST_INSTANCE.into(), + name: name.clone(), + data: data.clone(), + }), + _ => None, + }) + .collect(); + let mut engine = create_engine(before.history.clone()); + engine.prep_completions(completions); + assert_eq!( + outcome(execute(&mut engine, Arc::new(before.handler.clone()))), + after.old_live_outcome, + "resuming {}", + before.name + ); + assert_eq!( + action_shapes(engine.pending_actions()), + after.old_live_actions, + "new decisions after {}", + before.name + ); + assert_eq!( + decision_delta(engine.history_delta()), + decision_delta(&after.history[before.history.len()..]), + "history delta after {}", + before.name + ); + compared += 1; + } + assert_eq!(compared, 52); +} + +#[test] +fn new_engine_executions_use_immediate_queue_cancellation() { + let handler = Handler { + mode: Mode::PreboundCompetition, + lazy: false, + }; + let mut history = vec![started_event(1)]; + history[0].duroxide_version = "0.1.31".into(); + let mut first = create_engine(history.clone()); + assert_eq!( + outcome(execute(&mut first, Arc::new(handler.clone()))), + Outcome::Continue + ); + history.extend_from_slice(first.history_delta()); + let mut second = create_engine(history.clone()); + second.prep_completions(messages( + &history, + vec![ + Completion::Timer(0), + Completion::Message("q", "first"), + Completion::Timer(1), + ], + )); + assert_eq!( + outcome(execute(&mut second, Arc::new(handler.clone()))), + Outcome::Continue + ); + assert!( + action_shapes(second.pending_actions()) + .iter() + .any(|action| action.ends_with(":checkpoint:first")) + ); + history.extend_from_slice(second.history_delta()); + let mut replay = create_engine(history.clone()); + assert_eq!( + outcome(execute(&mut replay, Arc::new(handler.clone()))), + Outcome::Continue + ); + assert!(replay.pending_actions().is_empty()); + let mut final_turn = create_engine(history.clone()); + final_turn.prep_completions(messages(&history, vec![Completion::Activity(0)])); + assert_eq!( + outcome(execute(&mut final_turn, Arc::new(handler))), + Outcome::Completed("first".into()) + ); +} + +enum Completion { + Message(&'static str, &'static str), + Timer(usize), + Activity(usize), + Signal(&'static str), +} + +fn messages(history: &[Event], completions: Vec) -> Vec { + use duroxide::EventKind; + completions + .into_iter() + .map(|completion| match completion { + Completion::Signal(data) => external_raised_msg("s", data), + Completion::Message(name, data) => WorkItem::QueueMessage { + instance: TEST_INSTANCE.into(), + name: name.into(), + data: data.into(), + }, + Completion::Timer(index) => { + let event = history + .iter() + .filter(|e| matches!(e.kind, EventKind::TimerCreated { .. })) + .nth(index) + .unwrap(); + let EventKind::TimerCreated { fire_at_ms } = event.kind else { + unreachable!() + }; + timer_fired_msg(event.event_id, fire_at_ms) + } + Completion::Activity(index) => { + let event = history + .iter() + .filter(|e| matches!(e.kind, EventKind::ActivityScheduled { .. })) + .nth(index) + .unwrap(); + activity_completed_msg(event.event_id, "ok") + } + }) + .collect() +} + +fn record(snapshots: &mut Vec, name: &str, handler: Handler, batches: Vec>) { + let mut history = vec![started_event(1)]; + history[0].duroxide_version = "0.1.30".into(); + for (index, batch) in batches.into_iter().enumerate() { + let mut engine = create_engine(history.clone()); + engine.prep_completions(messages(&history, batch)); + let live = outcome(execute(&mut engine, Arc::new(handler.clone()))); + history.extend_from_slice(engine.history_delta()); + let mut replay = create_engine(history.clone()); + let replay_result = outcome(execute(&mut replay, Arc::new(handler.clone()))); + snapshots.push(Snapshot { + name: format!("{name}/turn-{index}"), + handler: handler.clone(), + history: history.clone(), + old_live_outcome: live, + old_live_actions: action_shapes(engine.pending_actions()), + old_replay_outcome: replay_result, + old_replay_actions: action_shapes(replay.pending_actions()), + }); + } +} + +#[test] +#[ignore = "fixture recorder: run only against the pinned unmodified 0.1.30 engine"] +fn record_old_engine_queue_histories() { + let lib_hash = compiled_sha256(include_str!("../src/lib.rs")); + let engine_hash = compiled_sha256(include_str!("../src/runtime/replay_engine.rs")); + assert_eq!(lib_hash, OLD_LIB_SHA256, "recorder compiled non-base lib.rs"); + assert_eq!( + engine_hash, OLD_ENGINE_SHA256, + "recorder compiled non-base replay engine" + ); + let target = std::env::var("CARGO_TARGET_DIR").expect("fresh isolated CARGO_TARGET_DIR required"); + let marker = std::fs::read_to_string(std::path::Path::new(&target).join(".fixture-recording-fresh")) + .expect("runner must create freshness marker before the first build in an empty target"); + assert_eq!(marker.trim(), OLD_SHA); + let rlibs = std::fs::read_dir(std::path::Path::new(&target).join("debug/deps")) + .unwrap() + .map(|entry| entry.expect("read recorder dependency entry")) + .filter(|entry| { + let name = entry.file_name(); + let name = name.to_string_lossy(); + name.starts_with("libduroxide-") && name.ends_with(".rlib") + }) + .count(); + assert_eq!(rlibs, 1, "recorder target contains multiple core libraries"); + assert_eq!(env!("CARGO_PKG_VERSION"), "0.1.30"); + let destination = std::env::var("DUROXIDE_OLD_FIXTURE_OUT").unwrap(); + let mut snapshots = Vec::new(); + use Completion::{Activity, Message, Timer}; + record( + &mut snapshots, + "ordinary", + Handler { + mode: Mode::Ordinary, + lazy: false, + }, + vec![ + vec![], + vec![Activity(0)], + vec![Timer(0)], + vec![Message("q", "first")], + vec![Message("other", "second")], + ], + ); + for lazy in [false, true] { + for mode in [Mode::Loop, Mode::Drain] { + record( + &mut snapshots, + &format!("message-first-{mode:?}-lazy-{lazy}"), + Handler { mode, lazy }, + vec![ + vec![], + vec![Message("q", "first"), Timer(0)], + vec![Message("q", "second")], + vec![Activity(0)], + ], + ); + record( + &mut snapshots, + &format!("adjacent-turn-{mode:?}-lazy-{lazy}"), + Handler { mode, lazy }, + vec![ + vec![], + vec![Timer(0)], + vec![Message("q", "first")], + vec![Message("q", "second")], + vec![Activity(0)], + ], + ); + } + record( + &mut snapshots, + &format!("timer-first-barrier-lazy-{lazy}"), + Handler { + mode: Mode::Barrier, + lazy, + }, + vec![ + vec![], + vec![Timer(0), Message("q", "first")], + vec![Activity(0)], + vec![Message("q", "second")], + vec![Activity(1)], + ], + ); + } + record( + &mut snapshots, + "prebound-queue-versus-second-timer", + Handler { + mode: Mode::PreboundCompetition, + lazy: false, + }, + vec![ + vec![], + vec![Timer(0), Message("q", "first"), Timer(1)], + vec![Activity(0)], + ], + ); + record( + &mut snapshots, + "unbound-prearrival", + Handler { + mode: Mode::Unbound, + lazy: false, + }, + vec![ + vec![Message("q", "first")], + vec![Message("q", "second")], + vec![Activity(0)], + ], + ); + assert!( + snapshots.iter().any(|snapshot| { + snapshot.name == "prebound-queue-versus-second-timer/turn-2" + && snapshot.old_live_outcome == Outcome::Completed("second-timeout".into()) + }), + "the recorder did not execute legacy queue semantics" + ); + assert!( + snapshots + .iter() + .any(|snapshot| snapshot.name == "unbound-prearrival/turn-0" + && snapshot + .old_live_actions + .iter() + .all(|action| !action.contains(":checkpoint:"))), + "unbound-path canary did not use old binding semantics" + ); + record( + &mut snapshots, + "positional-barrier", + Handler { + mode: Mode::PositionalBarrier, + lazy: false, + }, + vec![vec![], vec![Timer(0), Completion::Signal("a")], vec![Activity(0)]], + ); + record( + &mut snapshots, + "positional-blocking", + Handler { + mode: Mode::PositionalBlocking, + lazy: false, + }, + vec![ + vec![], + vec![Timer(0), Completion::Signal("a")], + vec![Completion::Signal("b")], + ], + ); + record( + &mut snapshots, + "positional-resolve-at-creation", + Handler { + mode: Mode::ResolveAtCreation, + lazy: false, + }, + vec![vec![], vec![Completion::Signal("a"), Activity(0)], vec![Activity(1)]], + ); + record( + &mut snapshots, + "continue-drop", + Handler { + mode: Mode::ContinueDrop, + lazy: false, + }, + vec![vec![], vec![Timer(0), Message("q", "a")]], + ); + record( + &mut snapshots, + "continue-drain", + Handler { + mode: Mode::ContinueDrain, + lazy: false, + }, + vec![vec![], vec![Timer(0), Message("q", "a"), Message("q", "b")]], + ); + let positional = snapshots + .iter() + .find(|s| s.name == "positional-blocking/turn-2") + .unwrap(); + assert_eq!( + positional.old_live_outcome, + Outcome::Continue, + "recorder repaired the legacy positional hang" + ); + for name in ["continue-drop/turn-1", "continue-drain/turn-1"] { + let snapshot = snapshots.iter().find(|s| s.name == name).unwrap(); + assert_eq!(snapshot.old_live_outcome, Outcome::ContinueAsNew("successor".into())); + assert!( + !snapshot + .history + .iter() + .any(|e| matches!(e.kind, EventKind::QueueSubscriptionCancelled { .. })), + "recorder used corrected continue-as-new cancellation" + ); + } + let fixtures = Fixtures { + recording_source: OLD_SHA.into(), + compiled_lib_sha256: lib_hash, + compiled_replay_engine_sha256: engine_hash, + recording_test_sha256: compiled_sha256(include_str!("queue_race_upgrade.rs")), + recording_comparator_sha256: compiled_sha256(include_str!("common/cancellation_history.rs")), + snapshots, + }; + std::fs::write(destination, serde_json::to_string_pretty(&fixtures).unwrap()).unwrap(); +} diff --git a/tests/replay_engine/helpers.rs b/tests/replay_engine/helpers.rs index 5c5033e..9951725 100644 --- a/tests/replay_engine/helpers.rs +++ b/tests/replay_engine/helpers.rs @@ -30,7 +30,7 @@ pub const TEST_WORKER_ID: &str = "test-worker"; /// Create an OrchestrationStarted event pub fn started_event(event_id: u64) -> Event { - Event::with_event_id( + let mut event = Event::with_event_id( event_id, TEST_INSTANCE, TEST_EXECUTION_ID, @@ -45,7 +45,11 @@ pub fn started_event(event_id: u64) -> Event { carry_forward_events: None, initial_custom_status: None, }, - ) + ); + // This is the reviewed immediate-policy boundary, independent of the + // package version used by a future test build. + event.duroxide_version = "0.1.31".into(); + event } /// Create an ActivityScheduled event diff --git a/tests/replay_engine/mod.rs b/tests/replay_engine/mod.rs index 8e8212a..4c58ce5 100644 --- a/tests/replay_engine/mod.rs +++ b/tests/replay_engine/mod.rs @@ -23,6 +23,8 @@ mod kv; mod nondeterminism; mod panic_handling; mod partial_completion; +mod positional_unbound_races; +mod queue_races; mod replay_with_completions; mod sequential_progress; mod sub_orchestration; diff --git a/tests/replay_engine/positional_unbound_races.rs b/tests/replay_engine/positional_unbound_races.rs new file mode 100644 index 0000000..959a1db --- /dev/null +++ b/tests/replay_engine/positional_unbound_races.rs @@ -0,0 +1,318 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::helpers::*; +use async_trait::async_trait; +use duroxide::providers::WorkItem; +use duroxide::{Either2, Event, EventKind, OrchestrationContext, OrchestrationHandler}; +use std::sync::Arc; +use std::time::Duration; + +struct UnboundDropHandler; + +#[async_trait] +impl OrchestrationHandler for UnboundDropHandler { + async fn invoke(&self, ctx: OrchestrationContext, _: String) -> Result { + ctx.schedule_timer(Duration::from_millis(10)).await; + drop(ctx.schedule_wait("s")); + Ok(ctx.schedule_wait("s").await) + } +} + +fn scheduling_history(handler: Arc) -> Vec { + let mut history = vec![started_event(1)]; + let mut first = create_engine(history.clone()); + assert_continue(&execute(&mut first, handler)); + history.extend_from_slice(first.history_delta()); + history +} + +fn timer_completion(history: &[Event]) -> WorkItem { + let timer = history + .iter() + .find(|event| matches!(event.kind, EventKind::TimerCreated { .. })) + .expect("handler must schedule a timer"); + let EventKind::TimerCreated { fire_at_ms } = timer.kind else { + unreachable!() + }; + timer_fired_msg(timer.event_id, fire_at_ms) +} + +fn assert_live_and_replay( + handler: Arc, + mut history: Vec, + messages: Vec, + expected: &str, +) { + let mut live = create_engine(history.clone()); + live.prep_completions(messages); + assert_completed(&execute(&mut live, handler.clone()), expected); + history.extend_from_slice(live.history_delta()); + let mut replay = create_engine(history); + assert_completed(&execute(&mut replay, handler), expected); + assert!(replay.history_delta().is_empty()); +} + +#[test] +fn ms17a_unbound_replacement_requires_a_fresh_signal_after_binding() { + let handler: Arc = Arc::new(UnboundDropHandler); + let history = scheduling_history(handler.clone()); + let timer = timer_completion(&history); + assert_early_batch_then_fresh(handler, history, vec![timer, external_raised_msg("s", "a")], "b"); +} + +fn assert_early_batch_then_fresh( + handler: Arc, + mut history: Vec, + batch: Vec, + expected: &str, +) { + let mut live = create_engine(history.clone()); + live.prep_completions(batch); + assert_continue(&execute(&mut live, handler.clone())); + history.extend_from_slice(live.history_delta()); + let mut replay = create_engine(history.clone()); + assert_continue(&execute(&mut replay, handler.clone())); + assert!(replay.history_delta().is_empty()); + assert_live_and_replay(handler, history, vec![external_raised_msg("s", "b")], expected); +} + +struct DeadlineVotesHandler; + +#[async_trait] +impl OrchestrationHandler for DeadlineVotesHandler { + // The explicit match keeps the durable deadline-loser branch visible. + #[allow(clippy::while_let_loop)] + async fn invoke(&self, ctx: OrchestrationContext, _: String) -> Result { + let mut deadline = ctx.schedule_timer(Duration::from_secs(30)); + let mut votes = Vec::new(); + loop { + match ctx.select2(ctx.schedule_wait("s"), &mut deadline).await { + Either2::First(value) => votes.push(value), + Either2::Second(()) => break, + } + } + let ack = ctx.schedule_wait("s").await; + Ok(format!("{}|{ack}", votes.join(","))) + } +} + +#[test] +fn ms17b_dropped_unbound_holder_does_not_refuse_fresh_signal() { + let handler: Arc = Arc::new(DeadlineVotesHandler); + let history = scheduling_history(handler.clone()); + let timer = timer_completion(&history); + assert_early_batch_then_fresh( + handler, + history, + vec![ + external_raised_msg("s", "x"), + external_raised_msg("s", "stale"), + timer, + external_raised_msg("s", "fresh"), + ], + "x|b", + ); +} + +#[test] +fn ms17b_deadline_drop_before_arrival_preserves_fresh_ack() { + let handler: Arc = Arc::new(DeadlineVotesHandler); + let history = scheduling_history(handler.clone()); + let timer = timer_completion(&history); + assert_early_batch_then_fresh( + handler, + history, + vec![external_raised_msg("s", "x"), timer, external_raised_msg("s", "fresh")], + "x|b", + ); +} + +struct LazyWaitsHandler { + queue_second: bool, +} + +#[async_trait] +impl OrchestrationHandler for LazyWaitsHandler { + async fn invoke(&self, ctx: OrchestrationContext, _: String) -> Result { + let validate = ctx.schedule_activity("validate", ""); + let prepare = ctx.schedule_activity("prepare", ""); + let result = ctx + .select2( + async { + validate.await?; + Ok::<_, String>(if self.queue_second { + ctx.dequeue_event("approve").await + } else { + ctx.schedule_wait("approve").await + }) + }, + async { + prepare.await?; + Ok::<_, String>(if self.queue_second { + ctx.dequeue_event("reject").await + } else { + ctx.schedule_wait("reject").await + }) + }, + ) + .await; + let output = format!("{result:?}"); + ctx.schedule_activity("record", &output).await?; + Ok(output) + } +} + +fn check_lazy_bind_order(queue_second: bool) { + let handler: Arc = Arc::new(LazyWaitsHandler { queue_second }); + let mut history = scheduling_history(handler.clone()); + let schedule_id = |name: &str| { + history + .iter() + .find(|event| matches!(&event.kind, EventKind::ActivityScheduled { name: n, .. } if n == name)) + .expect("handler must schedule both activities") + .event_id + }; + let reject = if queue_second { + WorkItem::QueueMessage { + instance: TEST_INSTANCE.into(), + name: "reject".into(), + data: "R".into(), + } + } else { + external_raised_msg("reject", "R") + }; + let mut live = create_engine(history.clone()); + live.prep_completions(vec![ + activity_completed_msg(schedule_id("prepare"), "ok"), + reject, + activity_completed_msg(schedule_id("validate"), "ok"), + if queue_second { + WorkItem::QueueMessage { + instance: TEST_INSTANCE.into(), + name: "approve".into(), + data: "A".into(), + } + } else { + external_raised_msg("approve", "A") + }, + ]); + assert_continue(&execute(&mut live, handler.clone())); + if !queue_second { + assert!( + !live.history_delta().iter().any(|event| matches!(&event.kind, + EventKind::ActivityScheduled { name, .. } if name == "record")), + "both same-batch signals precede binding and must be dropped as early" + ); + history.extend_from_slice(live.history_delta()); + let mut replay = create_engine(history.clone()); + assert_continue(&execute(&mut replay, handler.clone())); + assert!(replay.history_delta().is_empty()); + let mut next = create_engine(history.clone()); + next.prep_completions(vec![external_raised_msg("approve", "fresh-A")]); + assert_continue(&execute(&mut next, handler.clone())); + let record_id = next + .history_delta() + .iter() + .find(|event| { + matches!(&event.kind, EventKind::ActivityScheduled { name, input, .. } + if name == "record" && input == "First(Ok(\"fresh-A\"))") + }) + .expect("fresh bound signal must choose the approve branch") + .event_id; + history.extend_from_slice(next.history_delta()); + assert_live_and_replay( + handler, + history, + vec![activity_completed_msg(record_id, "ok")], + "First(Ok(\"fresh-A\"))", + ); + return; + } + let record = live + .history_delta() + .iter() + .find(|event| matches!(&event.kind, EventKind::ActivityScheduled { name, .. } if name == "record")) + .expect("live turn must record its selected winner"); + assert!( + matches!(&record.kind, EventKind::ActivityScheduled { input, .. } if input == "First(Ok(\"A\"))"), + "live selection must respect the handler's poll order: {record:?}" + ); + history.extend_from_slice(live.history_delta()); + let mut replay = create_engine(history.clone()); + let result = execute(&mut replay, handler); + assert!( + matches!(result, duroxide::runtime::replay_engine::TurnResult::Failed( + duroxide::ErrorDetails::Configuration { + kind: duroxide::ConfigErrorKind::Nondeterminism, ref message, .. + }) if message.as_deref().is_some_and(|text| text.contains("schedule mismatch") + && text.contains("Second(Ok") && text.contains("First(Ok"))), + "D2/MS18 queue variant must retain its known creation-versus-poll ordering defect: {result:?}" + ); +} + +#[test] +fn ms18_lazy_positional_bind_order_preserves_live_winner_on_replay() { + check_lazy_bind_order(false); +} + +#[test] +fn ms18_lazy_queue_variant_pins_known_d2_replay_ordering_defect() { + check_lazy_bind_order(true); +} + +/// MS1 plus a wait dropped before it is bound. Neither policy cancels a dropped wait +/// when its subscription row binds. Doing so for a 0.1.30 execution changes its +/// recorded outcome: the stale "a" reaches the checkpoint and the next replay fails. +struct Ms1UnboundDropHandler; + +#[async_trait] +impl OrchestrationHandler for Ms1UnboundDropHandler { + async fn invoke(&self, ctx: OrchestrationContext, _: String) -> Result { + if let Either2::Second(()) = ctx + .select2(ctx.schedule_wait("s"), ctx.schedule_timer(Duration::from_millis(10))) + .await + { + ctx.schedule_activity("tick", "").await?; + } + drop(ctx.schedule_wait("s")); + let value = ctx.schedule_wait("s").await; + ctx.schedule_activity("checkpoint", &value).await?; + Ok(value) + } +} + +#[test] +fn ms1_wait_dropped_before_bind_keeps_each_policy_outcome() { + for (stamp, expected) in [("0.1.30", vec![]), ("0.1.31", vec!["b".to_string()])] { + let handler: Arc = Arc::new(Ms1UnboundDropHandler); + let mut start = started_event(1); + start.duroxide_version = stamp.into(); + let mut history = vec![start]; + let mut checkpoints = Vec::new(); + let mut turn = |history: &mut Vec, messages: Vec| { + let mut live = create_engine(history.clone()); + live.prep_completions(messages); + assert_continue(&execute(&mut live, handler.clone())); + checkpoints.extend(live.history_delta().iter().filter_map(|event| match &event.kind { + EventKind::ActivityScheduled { name, input, .. } if name == "checkpoint" => Some(input.clone()), + _ => None, + })); + history.extend_from_slice(live.history_delta()); + let mut replay = create_engine(history.clone()); + assert_continue(&execute(&mut replay, handler.clone())); + assert!(replay.history_delta().is_empty(), "{stamp}: replay must not add rows"); + }; + turn(&mut history, vec![]); + let timer = timer_completion(&history); + turn(&mut history, vec![timer, external_raised_msg("s", "a")]); + let tick = history + .iter() + .find(|event| matches!(&event.kind, EventKind::ActivityScheduled { name, .. } if name == "tick")) + .expect("the timer must win and schedule tick") + .event_id; + turn(&mut history, vec![activity_completed_msg(tick, "ok")]); + turn(&mut history, vec![external_raised_msg("s", "b")]); + assert_eq!(checkpoints, expected, "{stamp}"); + } +} diff --git a/tests/replay_engine/queue_races.rs b/tests/replay_engine/queue_races.rs new file mode 100644 index 0000000..8ee575d --- /dev/null +++ b/tests/replay_engine/queue_races.rs @@ -0,0 +1,146 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use super::helpers::*; +use async_trait::async_trait; +use duroxide::providers::WorkItem; +use duroxide::{Either2, Event, EventKind, OrchestrationContext, OrchestrationHandler}; +use std::sync::Arc; +use std::time::Duration; + +struct QueueRaceHandler { + drain: bool, + lazy: bool, +} + +#[async_trait] +impl OrchestrationHandler for QueueRaceHandler { + async fn invoke(&self, ctx: OrchestrationContext, _: String) -> Result { + let mut messages = Vec::new(); + let mut timers = 0; + while messages.len() < 2 { + let result = if self.lazy { + // The Node bridge creates the durable children when their wrappers are polled. + ctx.select2(async { ctx.dequeue_event("q").await }, async { + ctx.schedule_timer(Duration::from_millis(20)).await + }) + .await + } else { + ctx.select2(ctx.dequeue_event("q"), ctx.schedule_timer(Duration::from_millis(20))) + .await + }; + match result { + Either2::First(message) => messages.push(message), + Either2::Second(()) => { + timers += 1; + if self.drain { + // A production drain exits on timeout; the next phase blocks on a fresh dequeue. + messages.push(ctx.dequeue_event("q").await); + } + } + } + } + Ok(format!("{timers}:{}", messages.join(","))) + } +} + +fn queue_message(data: &str) -> WorkItem { + WorkItem::QueueMessage { + instance: TEST_INSTANCE.to_string(), + name: "q".to_string(), + data: data.to_string(), + } +} + +#[test] +fn queue_timer_race_replays_timer_then_message_same_batch() { + for drain in [false, true] { + for lazy in [false, true] { + let handler = Arc::new(QueueRaceHandler { drain, lazy }); + let mut history = vec![started_event(1)]; + history[0].duroxide_version = "0.1.31".into(); + let mut first = create_engine(history.clone()); + assert_continue(&execute(&mut first, handler.clone())); + history.extend_from_slice(first.history_delta()); + + let timer = history + .iter() + .find(|event| matches!(event.kind, EventKind::TimerCreated { .. })) + .unwrap(); + let timer_id = timer.event_id; + let EventKind::TimerCreated { fire_at_ms } = timer.kind else { + unreachable!(); + }; + + let mut second = create_engine(history.clone()); + second.prep_completions(vec![timer_fired_msg(timer_id, fire_at_ms), queue_message("first")]); + assert_continue(&execute(&mut second, handler.clone())); + assert!(second.history_delta().iter().any(|event| { + matches!(&event.kind, EventKind::QueueSubscriptionCancelled { reason } if reason == "dropped_future") + })); + history.extend_from_slice(second.history_delta()); + + // Replaying the persisted turn must consume the same arrival before scheduling the next race. + let mut replay = create_engine(history.clone()); + assert_continue(&execute(&mut replay, handler.clone())); + assert!(replay.history_delta().is_empty()); + + let mut third = create_engine(history); + third.prep_completions(vec![queue_message("second")]); + assert_completed(&execute(&mut third, handler), "1:first,second"); + } + } +} + +struct DroppedUnboundQueueHandler; + +#[async_trait] +impl OrchestrationHandler for DroppedUnboundQueueHandler { + async fn invoke(&self, ctx: OrchestrationContext, _: String) -> Result { + drop(ctx.dequeue_event("q")); + let first = ctx.dequeue_event("q").await; + let second = ctx.dequeue_event("q").await; + Ok(format!("{first},{second}")) + } +} + +#[test] +fn dropped_unbound_queue_does_not_reserve_an_arrival() { + let mut history = vec![ + started_event(1), + Event::with_event_id( + 2, + TEST_INSTANCE, + TEST_EXECUTION_ID, + None, + EventKind::QueueEventDelivered { + name: "q".into(), + data: "first".into(), + }, + ), + Event::with_event_id( + 3, + TEST_INSTANCE, + TEST_EXECUTION_ID, + None, + EventKind::QueueEventDelivered { + name: "q".into(), + data: "second".into(), + }, + ), + ]; + history[0].duroxide_version = "0.1.31".into(); + let mut engine = create_engine(history.clone()); + assert_completed( + &execute(&mut engine, Arc::new(DroppedUnboundQueueHandler)), + "first,second", + ); + let mut persisted = history; + persisted.extend_from_slice(engine.history_delta()); + let mut replay = create_engine(persisted); + assert_completed( + &execute(&mut replay, Arc::new(DroppedUnboundQueueHandler)), + "first,second", + ); + assert!(replay.history_delta().is_empty()); +} diff --git a/tests/runtime_version_warnings.rs b/tests/runtime_version_warnings.rs new file mode 100644 index 0000000..909e01a --- /dev/null +++ b/tests/runtime_version_warnings.rs @@ -0,0 +1,122 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +#![allow(clippy::expect_used, clippy::clone_on_ref_ptr)] + +#[path = "common/mod.rs"] +#[allow(dead_code)] +mod common; + +use duroxide::providers::{ExecutionMetadata, WorkItem}; +use duroxide::runtime::registry::ActivityRegistry; +use duroxide::runtime::{Runtime, RuntimeOptions}; +use duroxide::{Client, Event, EventKind, OrchestrationContext, OrchestrationRegistry, OrchestrationStatus}; +use std::time::Duration; + +#[tokio::test] +async fn invalid_stamp_warns_on_every_turn() { + let (logs, _guard) = common::tracing_capture::install_tracing_capture(); + for stamp in ["", "garbage"] { + let (provider, _temporary) = common::create_sqlite_store_disk().await; + let instance = format!("invalid-version-{stamp}"); + let mut start = Event::with_event_id( + 1, + &instance, + 1, + None, + EventKind::OrchestrationStarted { + name: "Warning".into(), + version: "1.0.0".into(), + input: "".into(), + parent_instance: None, + parent_id: None, + parent_execution_id: None, + carry_forward_events: None, + initial_custom_status: None, + }, + ); + start.duroxide_version = stamp.into(); + common::seed_history_turn( + provider.as_ref(), + WorkItem::StartOrchestration { + instance: instance.clone(), + orchestration: "Warning".into(), + input: "".into(), + version: Some("1.0.0".into()), + execution_id: 1, + parent_instance: None, + parent_id: None, + parent_execution_id: None, + }, + 1, + vec![start], + vec![], + ExecutionMetadata { + orchestration_name: Some("Warning".into()), + orchestration_version: Some("1.0.0".into()), + ..Default::default() + }, + ) + .await; + let registry = OrchestrationRegistry::builder() + .register("Warning", |ctx: OrchestrationContext, _: String| async move { + let first = ctx.dequeue_event("q").await; + let second = ctx.dequeue_event("q").await; + Ok(format!("{first},{second}")) + }) + .build(); + let runtime = Runtime::start_with_options( + provider.clone(), + ActivityRegistry::builder().build(), + registry, + RuntimeOptions { + dispatcher_min_poll_interval: Duration::from_millis(10), + ..Default::default() + }, + ) + .await; + let client = Client::new(provider.clone()); + client + .enqueue_event(&instance, "q", "first") + .await + .expect("first input must enqueue"); + assert!( + common::wait_for_history( + provider.clone(), + &instance, + |history| history + .iter() + .filter(|event| matches!(event.kind, EventKind::QueueSubscribed { .. })) + .count() + == 2, + 30_000, + ) + .await, + "first turn must reach its second subscription before the next input" + ); + client + .enqueue_event(&instance, "q", "second") + .await + .expect("second input must enqueue"); + let result = client.wait_for_orchestration(&instance, Duration::from_secs(30)).await; + runtime.shutdown(None).await; + assert!(matches!( + result.expect("legacy sequential workflow must complete"), + OrchestrationStatus::Completed { ref output, .. } if output == "first,second" + )); + let warnings = logs + .lock() + .expect("capture lock must not be poisoned") + .iter() + .filter(|event| { + event.message.contains("Invalid pinned version") + && event.field("instance").as_deref() == Some(instance.as_str()) + }) + .count(); + // At least two turns ran (both reads need separate inputs): the warning is not + // limited to the first. The legacy fallback itself is covered by q14 and ms8. + assert!( + warnings >= 2, + "{stamp:?}: every turn must report the invalid stamp, got {warnings}" + ); + } +} diff --git a/tests/sqlite_provider_validations.rs b/tests/sqlite_provider_validations.rs index e93c41f..93008e9 100644 --- a/tests/sqlite_provider_validations.rs +++ b/tests/sqlite_provider_validations.rs @@ -190,6 +190,84 @@ mod tests { /// Used by the vast majority of tests that don't need direct DB manipulation. struct SqliteTestFactory; + #[tokio::test] + async fn queue_race_cancellation_replay() { + duroxide::provider_validations::race_replay::test_queue_race_cancellation_replay(&SqliteTestFactory).await; + } + + #[tokio::test] + async fn queue_replay_version_stamp_roundtrip() { + duroxide::provider_validations::race_replay::test_queue_replay_version_stamp_roundtrip(&SqliteTestFactory) + .await; + } + + #[tokio::test] + async fn positional_wait_race_replay() { + duroxide::provider_validations::race_replay::test_positional_wait_race_replay(&SqliteTestFactory).await; + } + + #[tokio::test] + async fn test_sqlite_delayed_abandon_preserves_unlocked_rows() { + duroxide::provider_validations::poison_message::orchestration_delayed_abandon_preserves_unlocked_rows( + &SqliteTestFactory, + ) + .await; + } + + #[tokio::test] + async fn test_sqlite_ignore_attempt_preserves_hidden_start() { + duroxide::provider_validations::poison_message::orchestration_ignore_attempt_preserves_hidden_start( + &SqliteTestFactory, + ) + .await; + } + + #[tokio::test] + async fn continue_as_new_queue_race_replay() { + duroxide::provider_validations::race_replay::test_continue_as_new_queue_race_replay(&SqliteTestFactory).await; + } + + #[tokio::test] + async fn continue_as_new_transition_delivery_legacy_stamp() { + duroxide::provider_validations::race_replay::test_continue_as_new_transition_delivery( + &SqliteTestFactory, + "0.1.30", + ) + .await; + } + + #[tokio::test] + async fn continue_as_new_transition_delivery_new_stamp() { + duroxide::provider_validations::race_replay::test_continue_as_new_transition_delivery( + &SqliteTestFactory, + "0.1.31", + ) + .await; + } + + #[tokio::test] + async fn continue_as_new_duplicate_start() { + duroxide::provider_validations::race_replay::test_continue_as_new_duplicate_start(&SqliteTestFactory).await; + } + + #[tokio::test] + async fn continue_as_new_unregistered_backoff() { + duroxide::provider_validations::race_replay::test_continue_as_new_unregistered_backoff(&SqliteTestFactory) + .await; + } + + #[tokio::test] + async fn duplicate_start_preserves_pinned_handler() { + duroxide::provider_validations::race_replay::test_duplicate_start_preserves_pinned_handler(&SqliteTestFactory) + .await; + } + + #[tokio::test] + async fn legacy_queue_race_decision_preserved() { + duroxide::provider_validations::race_replay::test_legacy_queue_race_decision_preserved(&SqliteTestFactory) + .await; + } + #[async_trait::async_trait] impl ProviderFactory for SqliteTestFactory { async fn create_provider(&self) -> Arc { From 91589c2d22ce21869ac96a87b514c3b280daf38a Mon Sep 17 00:00:00 2001 From: Waldemort Agent Date: Fri, 9 Oct 2026 02:22:11 +0000 Subject: [PATCH 7/9] docs: describe the race policy cutover and deferred work Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: Affan Dar <1480118+affandar@users.noreply.github.com> --- .agents/skills/release-preparation/SKILL.md | 6 + CHANGELOG.md | 75 +++++++- RELEASE_POLICY.md | 8 + design/queue-race-d2-follow-ups.md | 12 ++ docs/migration-guide.md | 181 ++++++++++++++++++-- docs/provider-implementation-guide.md | 12 ++ docs/provider-testing-guide.md | 15 +- docs/versioning-best-practices.md | 56 +++--- src/lib.rs | 25 ++- src/runtime/mod.rs | 10 +- 10 files changed, 343 insertions(+), 57 deletions(-) create mode 100644 design/queue-race-d2-follow-ups.md diff --git a/.agents/skills/release-preparation/SKILL.md b/.agents/skills/release-preparation/SKILL.md index 6a88b73..1893a2b 100644 --- a/.agents/skills/release-preparation/SKILL.md +++ b/.agents/skills/release-preparation/SKILL.md @@ -27,6 +27,12 @@ Never: ## Required Input +Version 0.1.31 is reserved for the race-cancellation semantic cutover. Do not +prepare a 0.1.31+ release or hotfix from an older branch without all queue, +positional-wait and exact continue-as-new carry-forward fixes. The subsequent +release PR uses 0.1.32; never change the semantic threshold to that release +number. See the reservation in `RELEASE_POLICY.md`. + Obtain the target semantic version from the user. Do not choose a major, minor, or patch bump without confirmation. diff --git a/CHANGELOG.md b/CHANGELOG.md index e39253c..3765876 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,80 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- **Continue-as-new admission window** - A CAN-terminal fetch that cannot + yet see the successor's start no longer acknowledges instance-scoped inputs. + Queue messages, positional signals and cancellation requests are abandoned + with the unregistered-handler backoff (`unregistered_backoff`, 1 s doubling + to 60 s by default), keeping the attempt, until the start is visible. Their + order relative to later arrivals is best effort, as for any delayed abandon. + If the abandon fails, the lease expires and the inputs are fetched again. + A start that never becomes visible (for example one deleted by an unpatched + worker) ends in poison after `max_attempts`, about five minutes with the + defaults, instead of an endless wait. + A successor failure recorded after CAN makes the instance terminal, so later + inputs are acknowledged rather than waiting for a start already consumed. + Stale completions for the previous execution wait with the other inputs and + the successor discards them. This dispatcher correction is + not version-gated and changes no existing recorded replay decision. + All orchestration workers must be patched to close the ingestion window + in a mixed deployment. + A CAN start now takes priority over a duplicate client start regardless of + batch order. Duplicate starts cannot replace a running instance's pinned + handler name, version or input. A colliding sub-orchestration start now + fails its parent when the target is still running, not only when it is + finished; a running instance's own parent re-sending the same call (with a + known parent execution id) is still ignored. + +- **SQLite abandon affects only the locked rows** - `abandon_orchestration_item` + now undoes the attempt increment and applies its delay only to the rows that + fetch locked, matching PostgreSQL and the provider contract. Before, it + updated every visible row of the instance, so a delayed abandon could hide a + continue-as-new start that became visible after the fetch, repeatedly while + inputs kept arriving. New provider validation: + `orchestration_delayed_abandon_preserves_unlocked_rows`. + +- **Shared race cancellation cutover** - Positional waits now cancel at drop + and discard stale signals assigned to a dropped slot. Continue-as-new + carries exact unread arrival positions, including held and non-prefix + dequeues. + Positional signals require a bound open slot. Same-batch signals applied + before a replacement is bound are dropped as early, even if that wait was + emitted by an earlier poll. This retains the documented early-event contract. + These corrections share the 0.1.31 execution threshold with the queue fix. + Activity/child terminal cleanup and cancellation validation are unchanged by + this continue-as-new path. + Dropped-future and terminal activity cancellations are now written in ID + order within each type, for every execution. Replay compares them as sets, + so recorded histories are unaffected. + Earlier executions preserve their original policy, including known defects. + Version 0.1.31 is reserved and must never be published without these fixes; + the subsequent release PR may use 0.1.32 and retains the same threshold. + +- **Queued-event race replay** - Dropping an unresolved dequeue now + excludes its subscription from FIFO matching immediately, including when the + token has not yet been bound. A timer followed by a queue message in the same + turn replays the covered immediate-replacement race shapes consistently. + Creation-versus-poll ordering, cancellation of an older dequeue after a newer + one resolved, and poll-on-demand gaps remain separate known limitations. + The behavior is pinned by the execution's `OrchestrationStarted` core version: + executions started before 0.1.31 retain legacy cancellation semantics, and + executions started by 0.1.31 or later use the fix. Existing executions switch + only when a new execution is started by a runtime at or above the threshold, + including continue-as-new. This avoids + changing race winners in otherwise valid old histories. + The doc-hidden `bind_token` helper now applies a previously recorded dequeue + drop when binding its token to a queue schedule only under V0131; contexts + default to Legacy. + +### Changed + +- The public `TurnResult::ContinueAsNew` variant now includes + `unconsumed_queue_arrivals`. External code constructing or exhaustively + matching this variant must add the field or use `..` in its match. + This is a Rust source-compatibility change, not a persisted event-format change. + ## [0.1.30] - 2026-07-29 **Release:** @@ -1237,4 +1311,3 @@ return ctx.continue_as_new(input).await; - OpenTelemetry metrics and structured logging - Provider validation test suite - Comprehensive documentation - diff --git a/RELEASE_POLICY.md b/RELEASE_POLICY.md index ef27d0c..3e72a3e 100644 --- a/RELEASE_POLICY.md +++ b/RELEASE_POLICY.md @@ -8,6 +8,14 @@ internal pipeline instructions. ## Public Release Handoff +Version 0.1.31 is reserved for the execution-pinned race-cancellation cutover. +Never tag or publish 0.1.31 or a later version from a branch that lacks the +queue, positional-wait, and exact continue-as-new carry-forward fixes. +The initial execution stamp selects these semantics permanently. An old +hotfix branch numbered 0.1.31+ would record decisions that corrected runtimes +cannot replay with that stamp. The subsequent release preparation uses 0.1.32; +the semantic threshold remains 0.1.31. + GitHub Actions in this repository are limited to building and testing changes. 1. Prepare a release pull request that updates the package version, changelog, diff --git a/design/queue-race-d2-follow-ups.md b/design/queue-race-d2-follow-ups.md new file mode 100644 index 0000000..e0e382b --- /dev/null +++ b/design/queue-race-d2-follow-ups.md @@ -0,0 +1,12 @@ +# Queue-race follow-ups + +These items are not part of the queue-race fix and are tracked separately. + +| Area | Follow-up | +|---|---| +| Continue-as-new admission | Remove the admission window in providers: write the continue-as-new start so no fetch cutoff can exclude it, and delay abandoned instances with an instance cooldown instead of re-timing rows. Until then a waiting instance is re-fetched on the unregistered-handler backoff and poisoned after `max_attempts` if the start never appears. | +| Collision wording | Include terminal-after-continue-as-new status in collision-notification wording. | +| Dropped abandon futures | Await the pre-existing dropped `abandon_orchestration_item` futures, or remove them intentionally. | +| Collision on poison | Notify a colliding start's parent when the same batch is consumed by poison handling. | +| Schedule lookup | Replace the O(n) schedule-to-token lookup with a reverse map. | +| Successor first-turn failure | Record a continue-as-new successor's first-turn failure and poison in execution N+1 with a valid start/history shape, not in execution N. | diff --git a/docs/migration-guide.md b/docs/migration-guide.md index 8e5ec1b..5b09bb3 100644 --- a/docs/migration-guide.md +++ b/docs/migration-guide.md @@ -2,6 +2,136 @@ This guide helps you migrate between Duroxide versions and handle orchestration versioning. +## Race cancellation semantics (reserved 0.1.31) + +The queue, positional-signal, and continue-as-new fixes share one execution +version switch. Applying those changes to an old execution can change a +recorded race winner, accepted signal, or carry-forward payload. + +The runtime selects the policy from the `duroxide_version` of the first +`OrchestrationStarted` event and keeps it for the entire execution: + +| Execution started by | Queue cancellation policy | +| --- | --- | +| Core before 0.1.31 | Original queue, positional, and CAN behavior | +| Core 0.1.31 or later | Immediate cancellation at drop/bind, bound-active-slot signal admission, and exact unread-arrival carry-forward | + +Upgrading a runtime does not change an existing execution's policy. Old +histories and subsequent turns retain their original scheduling decisions. +This also retains the known old queue, signal, and CAN defects for those executions until +they finish or continue as new. A replacement execution started by a new +runtime uses the fixed policy. + +Do not rewrite the initial version stamp to enable the fix in place. A +recorded race winner may then change, causing nondeterminism. No history or +provider schema migration is needed. + +An execution whose stamp is present but is not a valid version runs with the +legacy policy and logs "Invalid pinned version" on each turn. A missing stamp +field fails deserialization, and the instance is poisoned. + +With the default range in mixed-version deployments, the dispatcher capability filter +prevents an old runtime from replaying an execution stamped 0.1.31. A start +handled by an old runtime still creates a legacy execution. Upgrade all +runtimes before expecting every new execution to use the fix. + +Do not widen `supported_replay_versions` on a pre-0.1.31 node beyond 0.1.30 +while 0.1.31+ executions exist. The override replaces both compatibility checks, +and the unchanged event schema does not prevent a legacy runtime from +misinterpreting new queue decisions. Deserialization compatibility is not +replay-semantic compatibility. + +No release numbered 0.1.31 or later may omit this semantic cutover: its number +is the recorded contract, not a runtime upgrade switch. + +Version 0.1.31 is reserved for the semantic cutover. The +subsequent release PR uses 0.1.32, which still satisfies the +same threshold. Never publish 0.1.31 from an older branch without these fixes. + +### Positional signal contract + +The implemented design-document contract discards a signal addressed to a +dropped positional wait's slot; its replacement receives a fresh signal. +This includes a value accepted into that slot while the future was not polled. +Signals are admitted only for a bound active wait. A signal applied in the same +batch after the completion that emits a replacement is still early until +the replacement is bound. That signal is dropped; the next fresh signal +can resolve the bound replacement. Batch insertion order does not establish +when a signal was raised relative to creation of an unbound wait. +Queued messages instead remain persistent after a dequeue is dropped. + +Retaining the old answer for a later wait is a documented alternative, not +the implemented contract. Such an alternative would require its own versioned +policy for future executions, not a reinterpretation of existing history. + +Continue-as-new carries up to 100 queued arrivals that no dequeue returned, in +history order across names. Newer unread arrivals beyond that limit are +dropped with a warning (`MAX_CARRY_FORWARD`). A held, unresolved dequeue is not a consumer; +consumption need not form a prefix when newer dequeues are polled first. +Activity and child cleanup keep their existing continue-as-new behavior. + +### Continue-as-new input admission + +The dispatcher also fixes a pre-existing ingestion window: an old execution +has committed continue-as-new, but the current fetch cutoff cannot yet see +the successor's start. Instance-scoped inputs in that batch are preserved +by abandoning them with the unregistered-handler backoff, not acknowledged as +terminal work. When the start is visible, carried messages +precede the preserved messages. A deferred input can be delivered after an +input that arrived later: queue order across abandons is best effort. Stale +activity, timer and child completions for the previous execution wait with the +other inputs and the successor discards them. +Positional signals still follow the successor's signal-admission contract; +preserving the work item does not make positional signals persistent. + +This correction changes no previously recorded replay decision and applies +to every version stamp. Waiting batches use the same backoff and attempt +budget as an unregistered handler: each deferral keeps its attempt, and a +start that never becomes visible ends in poison after `max_attempts` (about +five minutes with the defaults). Deferred inputs carry their attempts into the +successor's first turn. If the abandon fails, the lease expires and the +inputs are fetched again. +SQLite restores +only locked rows' attempts, matching PostgreSQL, not hidden unregistered starts. +A mixed deployment remains exposed while any unpatched worker can +fetch the affected instance; upgrade all orchestration workers to close it. +No schema migration or rewrite of the previous execution's history is needed. +The CAN start also wins over a duplicate client start independently of queue +order. A duplicate start for an instance with history does not replace its +name, version or input. A sub-orchestration start that collides with an +existing instance fails the incoming parent, whether that instance is finished +or still running. The exception is a running instance's own parent re-sending +the same call (same parent instance, call id and a known, equal parent +execution id), which is ignored as before. + +If the successor's first turn is poisoned, its failure can currently be +recorded after CAN in the previous execution. This history is terminal: +the poison transaction consumed the start, so later queue, signal and cancel +inputs are acknowledged rather than deferred. + +While a successor start stays hidden (for example behind an unregistered +handler's backoff during a rolling upgrade), each waiting instance is fetched +and abandoned on the unregistered-handler backoff schedule. A provider-level +instance cooldown and a +continue-as-new start that a fetch cutoff cannot exclude are separate +follow-ups. + +The CAN start exists after the atomic terminal acknowledgement. Successful +fetches eventually observe it once its visibility delay or registration +backoff ends. Each deferral logs a warning with the batch size, the attempt +count and the chosen backoff. A start that never becomes visible (permanent +deletion, corruption or an indefinitely unavailable handler) poisons the +waiting inputs once the attempt budget is exhausted. A warning is not evidence +that an input was delivered or that an existing deleted input was recovered. + +### Retained legacy hazards + +An old execution that already recorded an unsafe race ordering is not +repaired by upgrade. Do not rewrite its initial stamp. Operators must use +application-specific restart/recovery or let a healthy execution finish or +continue as new on an upgraded runtime. The library provides no automatic +in-place repair of a failed or hanging legacy history. + ## Reserved `sub::` instance-id marker (Unreleased) The `sub::` marker is now reserved for runtime-generated sub-orchestration instance ids. @@ -336,30 +466,57 @@ let orchestrations = OrchestrationRegistry::builder() .build(); ``` +These steps roll back application handlers on the same duroxide runtime. + +### Rolling back the duroxide runtime itself + +Rolling the duroxide crate back across the 0.1.31 race-cancellation cutover is +not a supported routine operation: fix forward instead. Executions started by +0.1.31 or later must be replayed by a 0.1.31+ runtime. + +- By default a pre-0.1.31 worker does not fetch them (its supported range ends + at its own build version), so they wait until a 0.1.31+ worker runs again. +- An instance that keeps continuing as new on 0.1.31+ workers stays on the new + policy, because each successor is stamped by the runtime that starts it. +- If a core downgrade is unavoidable, keep at least one 0.1.31+ worker with the + required handlers running until every new-policy execution has finished. + Do not widen an old worker's `supported_replay_versions` to cover them + (see [Draining Stuck Orchestrations](#draining-stuck-orchestrations-after-upgrade)). +- Downgraded workers also reopen the continue-as-new input-loss window and the + duplicate-start defect fixed in 0.1.31; those fixes apply only on patched workers. + ## Draining Stuck Orchestrations After Upgrade -If after a full upgrade some orchestrations remain pinned to an old duroxide version that no -running node supports, they will sit in the queue indefinitely. To clear them, temporarily -widen `supported_replay_versions` in `RuntimeOptions`: +Orchestrations sit in the queue when no running node supports their pinned duroxide version. +With the default range (`0.0.0` up to the node's own build version) every node supports all +older versions, so this happens only when executions are pinned to a version **newer** than +every running node (for example after a runtime rollback), or when nodes run a narrowed range. + +- **Never set the upper bound above the node's own build version** + (`duroxide::providers::current_build_version()`). History that still deserializes can carry + different replay semantics: executions stamped 0.1.31 or later replay with the race-cancellation + rules, and a pre-0.1.31 node would replay them with the old rules and change recorded decisions. +- **To process them**, run at least one node whose build version is at least the pinned version, + with the default range (or a range that ends at that node's build version). +- **To abandon them**, delete the instances explicitly (`Client::delete_instance`, + `Client::delete_instance_bulk`; pass `force` for an instance that is still running). + `Client::cancel_instance` only queues a cancel request, which a compatible node must still + process. Do not route them to an older node. ```rust +// A narrowed node can be widened back down to older versions only; the upper +// bound stays at this node's own build version. RuntimeOptions { supported_replay_versions: Some(SemverRange::new( semver::Version::new(0, 0, 0), - semver::Version::new(99, 0, 0), + duroxide::providers::current_build_version(), )), - max_attempts: 5, ..Default::default() } ``` -The wide range causes the provider filter to pass for all items. When the provider fetches -an item whose history contains unknown event types (from a newer duroxide version), -deserialization fails at the provider level, returning a permanent error. Each fetch cycle -increments the item's `attempt_count`. The item remains in the queue but is effectively -drained — it never reaches the runtime's replay engine because the provider cannot -deserialize its history. Compatible items whose history deserializes successfully are -processed normally. Revert to the default after draining. +Deserialization succeeding is not evidence of replay compatibility. Revert any temporary +range change after the backlog drains. See [Versioning Best Practices](versioning-best-practices.md#draining-stuck-orchestrations-version-mismatch) for details. diff --git a/docs/provider-implementation-guide.md b/docs/provider-implementation-guide.md index d123bd8..4221b84 100644 --- a/docs/provider-implementation-guide.md +++ b/docs/provider-implementation-guide.md @@ -178,6 +178,12 @@ Let's trace what happens when an orchestration schedules an activity: ### Event History: The Append-Only Log +Preserve every event's `duroxide_version` exactly on storage and retrieval. +This includes `read`, `read_with_execution`, and the history returned by +`fetch_orchestration_item`. Do not stamp stored events with the reader's build +version. The first `OrchestrationStarted` stamp selects replay semantics for +that execution; a provider that restamps fetch results changes its decisions. + Every orchestration instance has an **event history**—a sequence of events that records everything that happened: ``` @@ -1134,6 +1140,12 @@ WHERE lock_token = token - `delay`: Optional backoff before retry (e.g., 1 second for transient failures) - `ignore_attempt`: If true, don't count this attempt toward poison message detection +Both the delay and the attempt undo apply **only to the rows this fetch locked** +(`WHERE lock_token = token`). Rows that became visible after the fetch, such as a +continue-as-new start, keep their visibility and attempt count. Validated by +`poison_message::orchestration_delayed_abandon_preserves_unlocked_rows` and +`poison_message::orchestration_ignore_attempt_preserves_hidden_start`. + **Implementation:** ```rust async fn abandon_orchestration_item( diff --git a/docs/provider-testing-guide.md b/docs/provider-testing-guide.md index 6065346..04c20bc 100644 --- a/docs/provider-testing-guide.md +++ b/docs/provider-testing-guide.md @@ -378,6 +378,20 @@ File SQLite 2/2 281 0 100.00 25.98 ## Provider Validation Tests +The `provider_validations::race_replay` module covers queue FIFO/replay, +positional live-turn recording and replay, exact queue carry-forward across +continue-as-new, fetch-path stamp preservation and a legacy-discriminating +runtime decision. Wrap every public test function for your provider. These +validate provider behavior; positional contract choices live in engine tests. +The CAN admission checks additionally cover two predecessor stamps with a +delayed successor start, mixed stale completions, positional signals, instance +cancellation, and malformed future completions. Include the duplicate-start, +unregistered-backoff and pinned-handler wrappers. A duplicate client start must +not hide a CAN wake or change an existing execution's handler version or input. +Also wrap `poison_message::orchestration_ignore_attempt_preserves_hidden_start` +and `poison_message::orchestration_delayed_abandon_preserves_unlocked_rows`: +an abandon's attempt undo and delay must touch only the rows that fetch locked. + Duroxide includes a comprehensive suite of validation tests that validate provider behavior. These tests verify critical correctness properties like atomicity, locking, error handling, queue semantics, and management capabilities. ### Quick Start @@ -1089,4 +1103,3 @@ This generates `stress-test-results.md` with: --- **With this guide, you can thoroughly test your custom Duroxide provider!** 🎉 - diff --git a/docs/versioning-best-practices.md b/docs/versioning-best-practices.md index 8cfead0..6ae2f5c 100644 --- a/docs/versioning-best-practices.md +++ b/docs/versioning-best-practices.md @@ -288,52 +288,48 @@ ctx.continue_as_new_versioned("1.0.0", input).await ## Draining Stuck Orchestrations (Version Mismatch) -After a major upgrade, some orchestrations may be pinned to an old duroxide version that no -running node supports. These items sit in the queue indefinitely because the capability filter -excludes them. - -To clear them, temporarily widen `supported_replay_versions` on one or more nodes: +An orchestration stays in the queue when no running node supports its pinned duroxide version. +With the default range (`0.0.0` up to the node's own build version) every node supports all +older versions, so stuck items are pinned to a version **newer** than every running node (for +example after a runtime rollback), or the nodes run a narrowed range. + +**Do not widen `supported_replay_versions` above the node's own build version.** History that +deserializes is not necessarily replay-compatible. Duroxide 0.1.31 changed race-cancellation +semantics without adding event types: a pre-0.1.31 node that admits a 0.1.31+ execution +deserializes its history and replays it with the old rules, which can change recorded decisions +(nondeterminism) or deliver a different message or signal. + +To clear the backlog: + +1. **Process the items on a compatible node.** Run at least one node whose build version is at + least the pinned version, with the default range. +2. **Or abandon them explicitly.** Delete the affected instances through the client + (`delete_instance`, `delete_instance_bulk`; pass `force` for a running instance). + `cancel_instance` only queues a request that a compatible node must still process. +3. **Restore a narrowed node** only down to older versions: ```rust let runtime = Runtime::builder() .with_provider(provider) .with_options(RuntimeOptions { - // Widen range to include all possible pinned versions + // Older versions only: the upper bound is this node's own build version. supported_replay_versions: Some(SemverRange::new( semver::Version::new(0, 0, 0), - semver::Version::new(99, 0, 0), + duroxide::providers::current_build_version(), )), - max_attempts: 5, ..Default::default() }) .build() .await?; ``` -**What happens:** -1. The wide range causes the provider filter to pass for all items, regardless of pinned version. -2. The provider fetches the item and attempts to deserialize its history. If the history - contains unknown event types (from a newer duroxide version), deserialization fails at the - provider level with a permanent error. The item never reaches the runtime's replay engine. -3. Each fetch cycle increments the item's `attempt_count`. The item remains in the queue with - repeated permanent errors, effectively preventing it from being processed. -4. Items whose history deserializes successfully are processed normally by the replay engine. - -**After draining:** Revert `supported_replay_versions` to `None` (the default) or the +**After draining:** revert `supported_replay_versions` to `None` (the default) or the appropriate range for your cluster. -> **Note:** This approach is safe because items with truly incompatible history fail at -> provider-level deserialization — they never reach the replay engine and are never silently -> replayed with incorrect semantics. The items remain in the queue with escalating -> `attempt_count` but are functionally drained (permanently erroring on every fetch). - -> **Current limitation:** The drain mechanism relies entirely on serde deserialization -> failures for unknown `EventKind` variants. There is no runtime-level replay engine version -> check — the `supported_replay_versions` range controls only which items are fetched, not -> whether the replay engine can actually process them. If a future version introduces -> semantic changes without adding new event types (i.e., history still deserializes but -> replay behavior differs), the wide-range approach would not catch this. A built-in -> replay-engine compatibility check may be added in a future phase. +> **Limitation:** `supported_replay_versions` is not clamped to the build version; an override +> replaces both the provider filter and the runtime-side check. Operators own that boundary. +> Items whose history contains unknown event types still fail provider-level deserialization, +> but that protects only against new event types, not against semantic changes such as 0.1.31. --- diff --git a/src/lib.rs b/src/lib.rs index 9ca884b..9095ad7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -467,7 +467,7 @@ pub type OrchestrationHandlerRef = Arc; /// this discriminator to determine how to cancel the underlying work: /// - **Activity**: Lock stealing via provider (DELETE from worker_queue) /// - **Timer**: No-op (virtual construct, no external state) -/// - **ExternalWait**: No-op (virtual construct, no external state) +/// - **ExternalWait**: Cancel the positional slot (virtual, no provider operation) /// - **SubOrchestration**: Enqueue `CancelInstance` work item for child #[derive(Debug, Clone)] pub enum ScheduleKind { @@ -503,7 +503,8 @@ pub enum ScheduleKind { /// /// - **Activities**: Lock stealing via provider (removes from worker queue) /// - **Sub-orchestrations**: `CancelInstance` work item enqueued for child -/// - **Timers/External waits**: No-op (virtual constructs with no external state) +/// - **Timers**: No provider operation (virtual construct) +/// - **External waits**: Cancel their positional slot under the execution-pinned policy /// /// # Examples /// @@ -1804,7 +1805,8 @@ impl CtxInner { std::mem::take(&mut self.emitted_actions) } - /// Bind a token to a schedule_id (called by replay engine when matching action to history). + /// Bind a token to a schedule_id when matching an action to history. + /// A recorded queue drop is applied at bind only under V0131; contexts default to Legacy. fn bind_token(&mut self, token: u64, schedule_id: u64) { self.token_bindings.insert(token, schedule_id); if self.race_cancellation_policy == RaceCancellationPolicy::V0131 @@ -3718,9 +3720,13 @@ impl OrchestrationContext { /// Subscribe to an external event and return a cancellation-aware future. /// - /// External waits are virtual constructs - dropping the future is a no-op since - /// there's no external state to cancel. However, wrapping in `DurableFuture` - /// maintains API consistency. + /// Dropping an unresolved wait cancels its positional slot. For executions + /// stamped 0.1.31+, cancellation is immediate and a value already assigned + /// to the dropped slot is discarded. A signal is accepted only while a bound, + /// open wait for its name exists: one that arrives after a wait is dropped and + /// before its replacement is bound is dropped as early (`docs/external-events.md`). + /// Earlier executions retain legacy timing. + /// No provider-side cancellation operation is needed for this virtual wait. pub fn schedule_wait(&self, name: impl Into) -> DurableFuture { let name: String = name.into(); @@ -3765,10 +3771,11 @@ impl OrchestrationContext { /// Dequeue the next message from a named queue (FIFO mailbox semantics). /// /// Unlike `schedule_wait`, queued events use FIFO matching: - /// - No positional pairing — any unresolved subscription gets the first unmatched arrival - /// - Cancelled subscriptions are skipped (don't consume arrivals) + /// - Bound subscriptions reserve FIFO positions, including held, unpolled futures + /// - Under the 0.1.31 policy, cancellation releases the held reservation without consuming it /// - Events that arrive before a subscription are buffered until consumed - /// - Events survive `continue_as_new` boundaries (carried forward) + /// - At most 100 unread arrivals survive `continue_as_new`, in arrival order across names; + /// excess newer unread arrivals are dropped with a warning /// /// The caller enqueues messages with [`Client::enqueue_event`]. pub fn dequeue_event(&self, queue: impl Into) -> DurableFuture { diff --git a/src/runtime/mod.rs b/src/runtime/mod.rs index 6b6bdf6..a973c95 100644 --- a/src/runtime/mod.rs +++ b/src/runtime/mod.rs @@ -277,10 +277,12 @@ pub struct RuntimeOptions { /// Set this to change the range for advanced scenarios: /// - **Narrowing:** Restrict a node to only process a specific version band /// (e.g., `>=1.0.0, <=1.9.999` in a mixed-version cluster). - /// - **Widening to drain stuck items:** Set a wide range like `>=0.0.0, <=99.0.0` - /// to fetch orchestrations pinned at any version. Items with unknown event types - /// will fail at provider-level deserialization (never reaching the replay engine) - /// and remain in the queue with escalating `attempt_count`. + /// - **Widening:** Requires semantic compatibility, not only successful event + /// deserialization. Versions can change replay policy without changing event + /// types. Do not admit stamps newer than this build. In particular, never + /// widen a pre-0.1.31 runtime beyond 0.1.30 while 0.1.31+ executions exist: + /// it cannot reproduce their queue, signal or carry-forward policy. + /// The override is not clamped to this build; operators own that boundary. /// /// Default: `None` (uses `>=0.0.0, <=CURRENT_BUILD_VERSION`) pub supported_replay_versions: Option, From a33bf24300426e9e5dd4a90834e4ac343c87ac48 Mon Sep 17 00:00:00 2001 From: Waldemort Agent Date: Sat, 10 Oct 2026 19:36:58 +0000 Subject: [PATCH 8/9] kv: clear and list keys in key order prune_kv_values_updated_before and get_kv_all_keys used HashMap order, which differs between replays of the same turn, so replay failed with `kv clear mismatch`. Use key order, and let replay match a run of KeyValueCleared rows in any order so histories recorded by older runtimes still replay. Fixes #57. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: Affan Dar <1480118+affandar@users.noreply.github.com> --- CHANGELOG.md | 6 +++ src/lib.rs | 9 +++- src/runtime/replay_engine.rs | 12 ++++- tests/replay_engine/kv.rs | 87 ++++++++++++++++++++++++++++++++++++ 4 files changed, 110 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3765876..f5d7160 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 finished; a running instance's own parent re-sending the same call (with a known parent execution id) is still ignored. +- **KV key order** - `prune_kv_values_updated_before` clears keys in key order and + `get_kv_all_keys` returns them in key order, instead of `HashMap` order that + differed between replays (`nondeterministic: kv clear mismatch`, #57). Replay + accepts a run of `KeyValueCleared` rows in any order, so histories recorded by + older runtimes still replay. + - **SQLite abandon affects only the locked rows** - `abandon_orchestration_item` now undoes the attempt increment and applies its delay only to the rows that fetch locked, matching PostgreSQL and the provider contract. Before, it diff --git a/src/lib.rs b/src/lib.rs index 9095ad7..83b47a5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3442,7 +3442,10 @@ impl OrchestrationContext { /// /// Pure read from in-memory state — no provider call, no event emitted. pub fn get_kv_all_keys(&self) -> Vec { - self.inner.lock().unwrap().kv_state.keys().cloned().collect() + let mut keys: Vec = self.inner.lock().unwrap().kv_state.keys().cloned().collect(); + // Key order, not HashMap order: callers may emit actions per key. + keys.sort_unstable(); + keys } /// Return the number of KV entries. @@ -3493,7 +3496,7 @@ impl OrchestrationContext { /// /// Returns the number of keys cleared. pub fn prune_kv_values_updated_before(&self, updated_before_ms: u64) -> usize { - let keys_to_clear: Vec = { + let mut keys_to_clear: Vec = { let inner = self.inner.lock().unwrap(); inner .kv_metadata @@ -3503,6 +3506,8 @@ impl OrchestrationContext { .map(|(key, _)| key.clone()) .collect() }; + // Key order, not HashMap order, so every replay emits the same clears. + keys_to_clear.sort_unstable(); let count = keys_to_clear.len(); for key in keys_to_clear { self.clear_kv_value(key); diff --git a/src/runtime/replay_engine.rs b/src/runtime/replay_engine.rs index 66e8e56..d3f6d6c 100644 --- a/src/runtime/replay_engine.rs +++ b/src/runtime/replay_engine.rs @@ -760,8 +760,16 @@ impl ReplayEngine { )))); } } - EventKind::KeyValueCleared { .. } => { - if let Some((_, action)) = emitted_actions.pop_front() + EventKind::KeyValueCleared { key } => { + // Older runtimes recorded a prune's clears in HashMap order. Clears of + // different keys commute, so match this one anywhere in the leading run. + if let Some(index) = emitted_actions + .iter() + .take_while(|(_, action)| matches!(action, crate::Action::ClearKeyValue { .. })) + .position(|(_, action)| matches!(action, crate::Action::ClearKeyValue { key: k } if k == key)) + { + emitted_actions.remove(index); + } else if let Some((_, action)) = emitted_actions.pop_front() && !action_matches_event_kind(&action, &event.kind) { return Err(TurnResult::Failed(nondeterminism_error(&format!( diff --git a/tests/replay_engine/kv.rs b/tests/replay_engine/kv.rs index 8140dc8..17c9f44 100644 --- a/tests/replay_engine/kv.rs +++ b/tests/replay_engine/kv.rs @@ -1178,3 +1178,90 @@ fn kv_clear_single_snapshot_key_then_rmw() { let result = execute(&mut engine, Arc::new(ClearSingleRmwHandler)); assert_completed(&result, "session=gone,counter=4"); } + +/// Between two activities, sets six keys in reverse order, lists them, then prunes +/// them all. The second activity keeps the turn open, so replay checks the KV rows. +struct ListAndPruneHandler; + +#[async_trait] +impl OrchestrationHandler for ListAndPruneHandler { + async fn invoke(&self, ctx: OrchestrationContext, _input: String) -> Result { + ctx.schedule_activity("Go", "").await?; + for key in ["f", "e", "d", "c", "b", "a"] { + ctx.set_kv_value(key, "x"); + } + let keys = ctx.get_kv_all_keys().join(","); + let pruned = ctx.prune_kv_values_updated_before(u64::MAX); + ctx.schedule_activity("Done", format!("{keys}|{pruned}")).await + } +} + +/// #57: listing and pruning use key order, not HashMap order, and a run of clears +/// recorded in another order by an older runtime still replays. +#[test] +fn prune_and_key_listing_use_key_order_and_accept_recorded_clear_order() { + let summary = "a,b,c,d,e,f|6"; + let rows = |delta: &[Event]| -> Vec { + delta + .iter() + .filter_map(|event| match &event.kind { + EventKind::KeyValueSet { key, .. } => Some(format!("set {key}")), + EventKind::KeyValueCleared { key } => Some(format!("clear {key}")), + EventKind::ActivityScheduled { name, input, .. } => Some(format!("{name} {input}")), + _ => None, + }) + .collect() + }; + let started = vec![ + started_event(1), + activity_scheduled(2, "Go", ""), + activity_completed(3, 2, "ok"), + ]; + let mut expected: Vec = ["f", "e", "d", "c", "b", "a"] + .iter() + .map(|k| format!("set {k}")) + .collect(); + expected.extend(["a", "b", "c", "d", "e", "f"].iter().map(|k| format!("clear {k}"))); + expected.push(format!("Done {summary}")); + for _ in 0..8 { + let mut engine = create_engine(started.clone()); + assert_continue(&execute(&mut engine, Arc::new(ListAndPruneHandler))); + assert_eq!( + rows(engine.history_delta()), + expected, + "keys listed and cleared in key order" + ); + } + let mut history = started; + for (index, key) in ["f", "e", "d", "c", "b", "a"].into_iter().enumerate() { + history.push(Event::with_event_id( + 4 + index as u64, + TEST_INSTANCE, + TEST_EXECUTION_ID, + None, + EventKind::KeyValueSet { + key: key.to_string(), + value: "x".to_string(), + last_updated_at_ms: 0, + }, + )); + } + // An older runtime recorded the clears in HashMap order. + for (index, key) in ["c", "f", "a", "e", "b", "d"].into_iter().enumerate() { + history.push(Event::with_event_id( + 10 + index as u64, + TEST_INSTANCE, + TEST_EXECUTION_ID, + None, + EventKind::KeyValueCleared { key: key.to_string() }, + )); + } + history.push(activity_scheduled(16, "Done", summary)); + let mut engine = create_engine(history); + assert_continue(&execute(&mut engine, Arc::new(ListAndPruneHandler))); + assert!( + rows(engine.history_delta()).is_empty(), + "replay must match every recorded row: {:?}", + engine.history_delta() + ); +} From 67415d010ac45743968fc31df946429ba1b44b75 Mon Sep 17 00:00:00 2001 From: Waldemort Agent Date: Sat, 10 Oct 2026 21:21:29 +0000 Subject: [PATCH 9/9] poison: record a continue-as-new successor as its own execution When the batch of an instance whose latest execution ended in ContinuedAsNew is poisoned (the successor's first turn keeps failing, its handler is missing, or its start never appears), the failure is now committed as execution N+1: a synthetic OrchestrationStarted built from the CAN start, or from execution N's name and parent when the start is missing, followed by the poison failure. Execution N keeps ending in ContinuedAsNew, and later inputs see a failed current execution. Before, the poison path appended OrchestrationFailed to execution N after its CAN row. Such histories, written by earlier runtimes, are still treated as terminal. A duplicate client start in the poisoned batch never becomes the successor's start. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: Affan Dar <1480118+affandar@users.noreply.github.com> --- CHANGELOG.md | 7 +- design/queue-race-d2-follow-ups.md | 2 +- docs/migration-guide.md | 9 +- docs/provider-testing-guide.md | 2 +- src/provider_validation/race_replay.rs | 133 +++++++++ src/runtime/dispatchers/orchestration.rs | 37 ++- tests/can_abandon_failure.rs | 361 ++++++++++++++++++++--- tests/sqlite_provider_validations.rs | 8 + 8 files changed, 497 insertions(+), 62 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5d7160..7b98e3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,8 +19,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 A start that never becomes visible (for example one deleted by an unpatched worker) ends in poison after `max_attempts`, about five minutes with the defaults, instead of an endless wait. - A successor failure recorded after CAN makes the instance terminal, so later - inputs are acknowledged rather than waiting for a start already consumed. + A poisoned successor is recorded as its own failed execution (a start row and + the failure), so the previous execution still ends in CAN. A history written + by an earlier runtime, with the failure after the CAN row, is still terminal, + so later inputs are acknowledged rather than waiting for a start already + consumed. Stale completions for the previous execution wait with the other inputs and the successor discards them. This dispatcher correction is not version-gated and changes no existing recorded replay decision. diff --git a/design/queue-race-d2-follow-ups.md b/design/queue-race-d2-follow-ups.md index e0e382b..2403d7f 100644 --- a/design/queue-race-d2-follow-ups.md +++ b/design/queue-race-d2-follow-ups.md @@ -9,4 +9,4 @@ These items are not part of the queue-race fix and are tracked separately. | Dropped abandon futures | Await the pre-existing dropped `abandon_orchestration_item` futures, or remove them intentionally. | | Collision on poison | Notify a colliding start's parent when the same batch is consumed by poison handling. | | Schedule lookup | Replace the O(n) schedule-to-token lookup with a reverse map. | -| Successor first-turn failure | Record a continue-as-new successor's first-turn failure and poison in execution N+1 with a valid start/history shape, not in execution N. | +| Successor first-turn commit failure | When a successor's first-turn commit fails, the infrastructure-failure path still builds its failure from the previous execution's history and commits it under the successor's execution id (related to #58). Poison already records the successor as its own execution. | diff --git a/docs/migration-guide.md b/docs/migration-guide.md index 5b09bb3..a9d6947 100644 --- a/docs/migration-guide.md +++ b/docs/migration-guide.md @@ -104,10 +104,11 @@ or still running. The exception is a running instance's own parent re-sending the same call (same parent instance, call id and a known, equal parent execution id), which is ignored as before. -If the successor's first turn is poisoned, its failure can currently be -recorded after CAN in the previous execution. This history is terminal: -the poison transaction consumed the start, so later queue, signal and cancel -inputs are acknowledged rather than deferred. +If the successor's batch is poisoned, its failure is recorded as the +successor's own execution: a start row and the failure. The previous execution +still ends in CAN. Earlier runtimes recorded that failure after CAN in the +previous execution; such a history is still terminal, so later queue, signal +and cancel inputs are acknowledged rather than deferred. While a successor start stays hidden (for example behind an unregistered handler's backoff during a rolling upgrade), each waiting instance is fetched diff --git a/docs/provider-testing-guide.md b/docs/provider-testing-guide.md index 04c20bc..28dd3f4 100644 --- a/docs/provider-testing-guide.md +++ b/docs/provider-testing-guide.md @@ -386,7 +386,7 @@ validate provider behavior; positional contract choices live in engine tests. The CAN admission checks additionally cover two predecessor stamps with a delayed successor start, mixed stale completions, positional signals, instance cancellation, and malformed future completions. Include the duplicate-start, -unregistered-backoff and pinned-handler wrappers. A duplicate client start must +unregistered-backoff, poisoned-successor and pinned-handler wrappers. A duplicate client start must not hide a CAN wake or change an existing execution's handler version or input. Also wrap `poison_message::orchestration_ignore_attempt_preserves_hidden_start` and `poison_message::orchestration_delayed_abandon_preserves_unlocked_rows`: diff --git a/src/provider_validation/race_replay.rs b/src/provider_validation/race_replay.rs index dbcd0dd..46ec812 100644 --- a/src/provider_validation/race_replay.rs +++ b/src/provider_validation/race_replay.rs @@ -280,6 +280,139 @@ pub async fn test_continue_as_new_unregistered_backoff(facto ); } +/// A poisoned continue-as-new successor is committed as its own execution: the +/// poison ack creates execution 2 with a start row and the failure, and execution 1 +/// still ends in ContinuedAsNew. +pub async fn test_continue_as_new_poisoned_successor_is_own_execution(factory: &F) { + let provider = factory.create_provider().await; + let instance = "provider-can-poisoned-successor"; + provider + .enqueue_for_orchestrator(start_item(instance), None) + .await + .expect("seed start"); + let (_, lock, _) = provider + .fetch_orchestration_item(Duration::from_secs(30), Duration::ZERO, None) + .await + .expect("seed fetch") + .expect("visible seed"); + provider + .ack_orchestration_item( + &lock, + 1, + vec![ + Event::with_event_id( + 1, + instance, + 1, + None, + EventKind::OrchestrationStarted { + name: "PoisonedSuccessor".into(), + version: "1.0.0".into(), + input: "old".into(), + parent_instance: None, + parent_id: None, + parent_execution_id: None, + carry_forward_events: None, + initial_custom_status: None, + }, + ), + Event::with_event_id( + 2, + instance, + 1, + None, + EventKind::OrchestrationContinuedAsNew { input: "next".into() }, + ), + ], + vec![], + vec![WorkItem::ContinueAsNew { + instance: instance.into(), + orchestration: "PoisonedSuccessor".into(), + input: "next".into(), + version: Some("1.0.0".into()), + parent_instance: None, + parent_id: None, + parent_execution_id: None, + carry_forward_events: vec![], + initial_custom_status: None, + }], + ExecutionMetadata { + status: Some("ContinuedAsNew".into()), + ..Default::default() + }, + vec![], + ) + .await + .expect("seed terminal and successor atomically"); + // No handler is registered, so the successor's batch is poisoned after one attempt. + let runtime = Runtime::start_with_options( + provider.clone(), + ActivityRegistry::builder().build(), + OrchestrationRegistry::builder().build(), + RuntimeOptions { + max_attempts: 1, + orchestration_concurrency: 1, + dispatcher_min_poll_interval: Duration::from_millis(10), + unregistered_backoff: crate::runtime::UnregisteredBackoffConfig { + base_delay: Duration::from_millis(20), + max_delay: Duration::from_millis(20), + }, + ..Default::default() + }, + ) + .await; + let client = Client::new(provider.clone()); + let result = client.wait_for_orchestration(instance, Duration::from_secs(10)).await; + client.enqueue_event(instance, "q", "late").await.expect("late input"); + tokio::time::sleep(Duration::from_millis(500)).await; + runtime.shutdown(None).await; + assert!(matches!( + result.expect("poison must end the successor"), + OrchestrationStatus::Failed { + details: crate::ErrorDetails::Poison { .. }, + .. + } + )); + let previous = provider + .read_with_execution(instance, 1) + .await + .expect("previous execution"); + assert!(matches!( + previous.last().map(|e| &e.kind), + Some(EventKind::OrchestrationContinuedAsNew { .. }) + )); + let successor = provider.read(instance).await.expect("latest execution"); + assert_eq!( + successor, + provider + .read_with_execution(instance, 2) + .await + .expect("successor execution"), + "the poisoned successor must be the latest execution" + ); + let kinds: Vec<&EventKind> = successor.iter().map(|e| &e.kind).collect(); + assert!( + matches!( + kinds.as_slice(), + [ + EventKind::OrchestrationStarted { .. }, + EventKind::OrchestrationFailed { + details: crate::ErrorDetails::Poison { .. } + } + ] + ), + "successor history: {successor:?}" + ); + assert!( + provider + .fetch_orchestration_item(Duration::from_secs(30), Duration::ZERO, None) + .await + .expect("inspect after runtime stopped") + .is_none(), + "a late input to the failed successor is acknowledged, not left waiting" + ); +} + /// A duplicate start must not replace the durable CAN successor, even when it /// precedes that successor in queue order. A colliding parent must be notified. pub async fn test_continue_as_new_duplicate_start(factory: &F) { diff --git a/src/runtime/dispatchers/orchestration.rs b/src/runtime/dispatchers/orchestration.rs index 1360aa5..6aa49c9 100644 --- a/src/runtime/dispatchers/orchestration.rs +++ b/src/runtime/dispatchers/orchestration.rs @@ -589,7 +589,8 @@ impl Runtime { // Extract metadata from history and work items let temp_history_mgr = HistoryManager::from_history(&item.history); let workitem_reader = WorkItemReader::from_messages(&item.messages, &temp_history_mgr, instance); - // The poison path records a CAN successor's failure after the CAN row. + // Runtimes before 0.1.31 poisoned a successor into the previous execution, after its + // CAN row. That instance is terminal. let terminal_after_can = temp_history_mgr.is_continued_as_new && item .history @@ -1256,12 +1257,21 @@ impl Runtime { lock_token: &str, attempt_count: u32, ) { + // Between executions the batch belongs to the successor, which has no history yet. + // It fails as its own execution, so the previous one still ends in ContinuedAsNew. + let latest = HistoryManager::from_history(&item.history); + let successor = latest.is_continued_as_new; + let execution_id = if successor { + item.execution_id + 1 + } else { + item.execution_id + }; let error = crate::ErrorDetails::Poison { attempt_count, max_attempts: self.options.max_attempts, message_type: crate::PoisonMessageType::Orchestration { instance: item.instance.clone(), - execution_id: item.execution_id, + execution_id, }, message: serde_json::to_string(&item.messages).unwrap_or_default(), }; @@ -1324,7 +1334,7 @@ impl Runtime { } // Create failure event and commit - let mut history_mgr = HistoryManager::from_history(&item.history); + let mut history_mgr = HistoryManager::from_history(if successor { &[] } else { &item.history }); // Track parent info for sub-orchestration failure notification let mut parent_link: Option<(String, Option, u64)> = None; @@ -1336,6 +1346,7 @@ impl Runtime { item.messages .iter() .find_map(|msg| match msg { + // A successor starts only from its CAN start, never from a duplicate. WorkItem::StartOrchestration { orchestration, input, @@ -1343,7 +1354,7 @@ impl Runtime { parent_id, parent_execution_id, .. - } => Some(( + } if !successor => Some(( orchestration.clone(), input.clone(), parent_instance.clone(), @@ -1372,7 +1383,17 @@ impl Runtime { )), _ => None, }) - .unwrap_or_else(|| (item.orchestration_name.clone(), String::new(), None, None, None, None)); + // A successor whose start is missing keeps the previous execution's parent. + .unwrap_or_else(|| { + ( + item.orchestration_name.clone(), + String::new(), + latest.parent_instance.clone(), + latest.parent_id, + latest.parent_execution_id, + None, + ) + }); // Save parent link for notification if let (Some(pi), Some(pid)) = (&parent_instance, parent_id) { @@ -1382,7 +1403,7 @@ impl Runtime { history_mgr.append(Event::with_event_id( crate::INITIAL_EVENT_ID, &item.instance, - item.execution_id, + execution_id, None, EventKind::OrchestrationStarted { name: orchestration_name, @@ -1411,7 +1432,7 @@ impl Runtime { } } - history_mgr.append_failed(&item.instance, item.execution_id, error.clone()); + history_mgr.append_failed(&item.instance, execution_id, error.clone()); let metadata = ExecutionMetadata { status: Some("Failed".to_string()), @@ -1447,7 +1468,7 @@ impl Runtime { let _ = self .ack_orchestration_with_changes( lock_token, - item.execution_id, + execution_id, history_mgr.delta().to_vec(), vec![], orchestrator_items, diff --git a/tests/can_abandon_failure.rs b/tests/can_abandon_failure.rs index 7d853fe..842ea83 100644 --- a/tests/can_abandon_failure.rs +++ b/tests/can_abandon_failure.rs @@ -58,6 +58,11 @@ async fn successor_first_turn_must_obey_poison_budget() { ), "CAN successor first-turn attempts must not be exempt from poison" ); + let successor = assert_successor_failed_as_own_execution(provider.as_ref(), "CAN-first-turn-poison").await; + assert!( + matches!(&successor[0].kind, EventKind::OrchestrationStarted { input, .. } if input == "next"), + "the successor's start comes from its CAN start: {successor:?}" + ); assert_late_inputs_are_drained(&client, provider.as_ref(), &runtime, "CAN-first-turn-poison").await; } @@ -68,16 +73,6 @@ async fn assert_late_inputs_are_drained( instance: &str, ) { let before = provider.read(instance).await.expect("poisoned history"); - assert!( - before - .iter() - .any(|event| matches!(event.kind, EventKind::OrchestrationContinuedAsNew { .. })) - ); - assert!( - before - .iter() - .any(|event| matches!(event.kind, EventKind::OrchestrationFailed { .. })) - ); client.enqueue_event(instance, "q", "late").await.expect("late queue"); client .raise_event(instance, "signal", "late") @@ -137,6 +132,7 @@ async fn unregistered_can_successor_poison_must_drain_late_inputs() { .. } )); + assert_successor_failed_as_own_execution(provider.as_ref(), instance).await; assert_late_inputs_are_drained(&client, provider.as_ref(), &runtime, instance).await; } @@ -400,23 +396,265 @@ async fn missing_can_start_poisons_waiting_inputs_after_max_attempts() { max_attempts as usize, "each waiting fetch up to max_attempts defers with a counted attempt" ); - let history = provider - .read_with_execution(instance, 1) + assert_successor_failed_as_own_execution(provider.as_ref(), instance).await; + // The failed successor is terminal: later inputs are drained, not deferred forever. + assert_late_inputs_are_drained(&client, provider.as_ref(), &runtime, instance).await; +} + +#[tokio::test] +async fn missing_can_start_fails_successor_with_previous_parent_not_a_duplicate_start() { + let sqlite = Arc::new(SqliteProvider::new_in_memory().await.expect("store")); + let provider = Arc::new(PoisonInjectingProvider::new(sqlite)); + let instance = "CAN-start-missing-child"; + // The parent must exist: the provider links the child's instance row to it. + common::seed_history_turn( + provider.as_ref(), + WorkItem::StartOrchestration { + instance: "parent-orch".into(), + orchestration: "Parent".into(), + input: "".into(), + version: Some("1.0.0".into()), + execution_id: 1, + parent_instance: None, + parent_id: None, + parent_execution_id: None, + }, + 1, + vec![Event::with_event_id( + 1, + "parent-orch", + 1, + None, + EventKind::OrchestrationStarted { + name: "Parent".into(), + version: "1.0.0".into(), + input: "".into(), + parent_instance: None, + parent_id: None, + parent_execution_id: None, + carry_forward_events: None, + initial_custom_status: None, + }, + )], + vec![], + ExecutionMetadata { + orchestration_name: Some("Parent".into()), + orchestration_version: Some("1.0.0".into()), + ..Default::default() + }, + ) + .await; + seed_can_execution( + provider.as_ref(), + instance, + "Transition", + Some(("parent-orch", 7)), + false, + late_input(instance), + ) + .await; + // A duplicate client start waits with the input; it must not become the successor. + provider + .enqueue_for_orchestrator( + WorkItem::StartOrchestration { + instance: instance.into(), + orchestration: "Impostor".into(), + input: "duplicate".into(), + version: Some("1.0.0".into()), + execution_id: 1, + parent_instance: None, + parent_id: None, + parent_execution_id: None, + }, + None, + ) .await - .expect("poisoned history"); + .expect("duplicate start"); + let runtime = Runtime::start_with_options( + provider.clone(), + ActivityRegistry::builder().build(), + OrchestrationRegistry::builder().build(), + RuntimeOptions { + max_attempts: 1, + orchestration_concurrency: 1, + unregistered_backoff: UnregisteredBackoffConfig { + base_delay: Duration::from_millis(20), + max_delay: Duration::from_millis(20), + }, + dispatcher_min_poll_interval: Duration::from_millis(10), + ..Default::default() + }, + ) + .await; + let result = Client::new(provider.clone()) + .wait_for_orchestration(instance, Duration::from_secs(10)) + .await + .expect("a missing start must end the wait"); + runtime.shutdown(None).await; assert!(matches!( - history.last().map(|e| &e.kind), - Some(EventKind::OrchestrationFailed { .. }) + result, + OrchestrationStatus::Failed { + details: duroxide::ErrorDetails::Poison { .. }, + .. + } )); + let successor = assert_successor_failed_as_own_execution(provider.as_ref(), instance).await; assert!( + matches!( + &successor[0].kind, + EventKind::OrchestrationStarted { name, parent_instance: Some(parent), parent_id: Some(7), .. } + if name == "Transition" && parent == "parent-orch" + ), + "the successor keeps the previous execution's name and parent, not the duplicate's: {successor:?}" + ); +} + +#[tokio::test] +async fn poisoned_child_successor_fails_its_parent() { + let sqlite = Arc::new(SqliteProvider::new_in_memory().await.expect("store")); + let provider = Arc::new(PoisonInjectingProvider::new(sqlite)); + let registry = OrchestrationRegistry::builder() + .register("Parent", |ctx: OrchestrationContext, _: String| async move { + ctx.schedule_sub_orchestration_with_id("Child", "poisoned-child", "") + .await + }) + // The successor version is not registered, so its batch is poisoned. + .register_versioned("Child", "1.0.0", |ctx: OrchestrationContext, _: String| async move { + ctx.continue_as_new_versioned("9.9.9", "next").await + }) + .build(); + let runtime = Runtime::start_with_options( + provider.clone(), + ActivityRegistry::builder().build(), + registry, + RuntimeOptions { + max_attempts: 1, + orchestration_concurrency: 1, + unregistered_backoff: UnregisteredBackoffConfig { + base_delay: Duration::from_millis(20), + max_delay: Duration::from_millis(20), + }, + dispatcher_min_poll_interval: Duration::from_millis(10), + ..Default::default() + }, + ) + .await; + let client = Client::new(provider.clone()); + client + .start_orchestration("poisoned-child-parent", "Parent", "") + .await + .expect("start parent"); + let result = client + .wait_for_orchestration("poisoned-child-parent", Duration::from_secs(10)) + .await + .expect("the parent must hear about the poisoned child"); + runtime.shutdown(None).await; + // A child's poison is a system error: it fails the parent with the child's details. + assert!( + matches!( + &result, + OrchestrationStatus::Failed { + details: duroxide::ErrorDetails::Poison { + message_type: duroxide::PoisonMessageType::Orchestration { instance, execution_id: 2 }, + .. + }, + .. + } if instance == "poisoned-child" + ), + "the parent must fail with the poison of the child's execution 2: {result:?}" + ); + let successor = assert_successor_failed_as_own_execution(provider.as_ref(), "poisoned-child").await; + assert!( + matches!( + &successor[0].kind, + EventKind::OrchestrationStarted { parent_instance: Some(parent), .. } if parent == "poisoned-child-parent" + ), + "the child's execution 2 keeps its parent link: {successor:?}" + ); +} + +#[tokio::test] +async fn legacy_failure_after_can_stays_terminal() { + let sqlite = Arc::new(SqliteProvider::new_in_memory().await.expect("store")); + let provider = Arc::new(PoisonInjectingProvider::new(sqlite)); + let instance = "CAN-legacy-poison"; + // Runtimes before 0.1.31 recorded the successor's poison after the CAN row. + seed_can_execution( + provider.as_ref(), + instance, + "Transition", + None, + true, + late_input(instance), + ) + .await; + let runtime = Runtime::start_with_options( + provider.clone(), + ActivityRegistry::builder().build(), + OrchestrationRegistry::builder().build(), + RuntimeOptions { + max_attempts: 3, + orchestration_concurrency: 1, + unregistered_backoff: UnregisteredBackoffConfig { + base_delay: Duration::from_millis(20), + max_delay: Duration::from_millis(20), + }, + dispatcher_min_poll_interval: Duration::from_millis(10), + ..Default::default() + }, + ) + .await; + let client = Client::new(provider.clone()); + assert_late_inputs_are_drained(&client, provider.as_ref(), &runtime, instance).await; + assert!( + provider.counted_abandon_delays().is_empty(), + "inputs of a terminal legacy instance are acknowledged, never deferred" + ); + assert_eq!( provider - .read_with_execution(instance, 2) + .read_with_execution(instance, 1) .await - .unwrap_or_default() - .is_empty() + .expect("legacy history") + .len(), + 3 ); - // Failed after CAN is terminal: later inputs are drained, not deferred forever. - assert_late_inputs_are_drained(&client, provider.as_ref(), &runtime, instance).await; +} + +/// The previous execution still ends in ContinuedAsNew and the successor is its own +/// execution: a start row and the poison failure. +async fn assert_successor_failed_as_own_execution(provider: &PoisonInjectingProvider, instance: &str) -> Vec { + let previous = provider + .read_with_execution(instance, 1) + .await + .expect("previous execution"); + assert!( + matches!( + previous.last().map(|e| &e.kind), + Some(EventKind::OrchestrationContinuedAsNew { .. }) + ), + "the previous execution must still end in ContinuedAsNew: {previous:?}" + ); + let successor = provider + .read_with_execution(instance, 2) + .await + .expect("successor execution"); + let kinds: Vec<&EventKind> = successor.iter().map(|e| &e.kind).collect(); + assert!( + matches!( + kinds.as_slice(), + [ + EventKind::OrchestrationStarted { .. }, + EventKind::OrchestrationFailed { + details: duroxide::ErrorDetails::Poison { + message_type: duroxide::PoisonMessageType::Orchestration { execution_id: 2, .. }, + .. + } + } + ] + ), + "the successor must be a start row and its poison failure: {successor:?}" + ); + successor } #[tokio::test] @@ -578,6 +816,61 @@ fn late_input(instance: &str) -> WorkItem { } async fn seed_waiting_can(provider: &PoisonInjectingProvider, instance: &str, name: &str, waiting: WorkItem) { + seed_can_execution(provider, instance, name, None, false, waiting).await; +} + +async fn seed_can_execution( + provider: &PoisonInjectingProvider, + instance: &str, + name: &str, + parent: Option<(&str, u64)>, + legacy_poison: bool, + waiting: WorkItem, +) { + let mut events = vec![ + Event::with_event_id( + 1, + instance, + 1, + None, + EventKind::OrchestrationStarted { + name: name.into(), + version: "1.0.0".into(), + input: "old".into(), + parent_instance: parent.map(|(p, _)| p.into()), + parent_id: parent.map(|(_, id)| id), + parent_execution_id: parent.map(|_| 1), + carry_forward_events: None, + initial_custom_status: None, + }, + ), + Event::with_event_id( + 2, + instance, + 1, + None, + EventKind::OrchestrationContinuedAsNew { input: "next".into() }, + ), + ]; + if legacy_poison { + events.push(Event::with_event_id( + 3, + instance, + 1, + None, + EventKind::OrchestrationFailed { + details: duroxide::ErrorDetails::Poison { + attempt_count: 11, + max_attempts: 10, + message_type: duroxide::PoisonMessageType::Orchestration { + instance: instance.into(), + execution_id: 1, + }, + message: "[]".into(), + }, + }, + )); + } common::seed_history_turn( provider, WorkItem::StartOrchestration { @@ -591,34 +884,10 @@ async fn seed_waiting_can(provider: &PoisonInjectingProvider, instance: &str, na parent_execution_id: None, }, 1, - vec![ - Event::with_event_id( - 1, - instance, - 1, - None, - EventKind::OrchestrationStarted { - name: name.into(), - version: "1.0.0".into(), - input: "old".into(), - parent_instance: None, - parent_id: None, - parent_execution_id: None, - carry_forward_events: None, - initial_custom_status: None, - }, - ), - Event::with_event_id( - 2, - instance, - 1, - None, - EventKind::OrchestrationContinuedAsNew { input: "next".into() }, - ), - ], + events, vec![waiting], ExecutionMetadata { - status: Some("ContinuedAsNew".into()), + status: Some(if legacy_poison { "Failed" } else { "ContinuedAsNew" }.into()), ..Default::default() }, ) diff --git a/tests/sqlite_provider_validations.rs b/tests/sqlite_provider_validations.rs index 93008e9..d810957 100644 --- a/tests/sqlite_provider_validations.rs +++ b/tests/sqlite_provider_validations.rs @@ -256,6 +256,14 @@ mod tests { .await; } + #[tokio::test] + async fn continue_as_new_poisoned_successor_is_own_execution() { + duroxide::provider_validations::race_replay::test_continue_as_new_poisoned_successor_is_own_execution( + &SqliteTestFactory, + ) + .await; + } + #[tokio::test] async fn duplicate_start_preserves_pinned_handler() { duroxide::provider_validations::race_replay::test_duplicate_start_preserves_pinned_handler(&SqliteTestFactory)