From 8711a3e55c6472365c194f72466fa15c1e0d5441 Mon Sep 17 00:00:00 2001 From: Affan Dar Date: Sun, 11 Oct 2026 08:33:33 -0700 Subject: [PATCH] validation: retry a retryable abandon in the continue-as-new backoff test test_continue_as_new_unregistered_backoff releases its competing inspection fetch with abandon_orchestration_item while a runtime fetches from the same store. On SQLite's shared-cache in-memory store the two transactions can block each other. The provider then returns a retryable error ("database is deadlocked"), and the test panicked on it. CI hit this on the 0.1.32 release PR. Locally it failed 1 in 100 runs. Retry the abandon on retryable errors, as the fetch right above it already does. A failed abandon leaves the lock in place, so the retry is safe. With the fix, 200 of 200 runs passed and the retry fired twice. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01CuCN2hdY59UrmFYYc7DSBJ --- src/provider_validation/race_replay.rs | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/provider_validation/race_replay.rs b/src/provider_validation/race_replay.rs index 46ec812..d5377a6 100644 --- a/src/provider_validation/race_replay.rs +++ b/src/provider_validation/race_replay.rs @@ -218,10 +218,22 @@ pub async fn test_continue_as_new_unregistered_backoff(facto .iter() .any(|item| matches!(item, WorkItem::ContinueAsNew { .. })) ); - provider - .abandon_orchestration_item(&lock, None, true) - .await - .expect("release competing inspection fetch"); + // The runtime fetches from the same store at the same time, so the + // provider may report a retryable conflict (SQLite's shared-cache + // "database is deadlocked"). A failed abandon leaves the lock in + // place, so retry it. + let mut attempt = 0u64; + loop { + match provider.abandon_orchestration_item(&lock, None, true).await { + Ok(()) => break, + Err(error) if error.retryable && attempt < 5 => { + attempt += 1; + tracing::warn!(%error, attempt, "Retrying release of competing inspection fetch"); + tokio::time::sleep(Duration::from_millis(10 * attempt)).await; + } + Err(error) => panic!("release competing inspection fetch: {error}"), + } + } } Err(error) if error.retryable => { tracing::warn!(%error, "Retrying backoff-test visibility inspection");