diff --git a/ARCHITECTURE.toml b/ARCHITECTURE.toml index d10f4ddb3..b122c61d0 100644 --- a/ARCHITECTURE.toml +++ b/ARCHITECTURE.toml @@ -244,6 +244,90 @@ mode = "required" [rules.pattern_ratchet.node_manager_commands.stale_entries] mode = "required" +# sbx-mgr's node-local SQLite database (`sandboxes` and every other sbx-mgr +# table) is opened, migrated and queried only by its store, `src/db` of the +# server crate. Within sbx-mgr's crates, naming rusqlite, opening a +# connection or transaction, using the shared SQLite open and migration +# helpers or the shared job store's SQL functions, binding parameters, +# executing a statement, or writing an SQL statement literal outside the +# store is a crossing. Other services' crates use rusqlite for their own +# databases, so `paths` limits the rule to sbx-mgr's. Existing callers that +# still name the connection, error and transaction types are baselined +# until they use the store's own. +[rules.pattern_ratchet.sbxmgr_sqlite] +patterns = [ + '\brusqlite\s*::', + '\buse\s+(?:::\s*)?rusqlite\b', + '\bextern\s+crate\s+rusqlite\b', + '\b(?:Connection|Transaction)\s*::\s*(?:open\w*|new\w*)\s*\(', + '\bsql\s*::\s*sqlite\b', + '\bpplx_space_common_jobs\s*::\s*(?:db\b|\{[^}]*\bdb\b)', + '\b(?:named_)?params\s*!', + '\.\s*(?:execute|execute_batch|prepare|prepare_cached|query_row|query_row_and_then|query_one|query_map|query_and_then|transaction|unchecked_transaction|transaction_with_behavior|savepoint|pragma_update|pragma_query|pragma_query_value)\s*\(', + '\.\s*(?:query|exists)\s*\(\s*[^)\s]', + '\bimmediate_transaction\s*\(', + '"\s*(?:SELECT\b|INSERT\s+(?:OR\s+[A-Z]+\s+)?INTO\b|REPLACE\s+INTO\b|UPDATE\s+(?:OR\s+[A-Z]+\s+)?\w+(?:\s+SET\b|\s*$)|DELETE\s+FROM\b|(?:CREATE|DROP|ALTER)\s+(?:TEMP\s+|TEMPORARY\s+|UNIQUE\s+)?(?:TABLE|INDEX|TRIGGER|VIEW)\b|PRAGMA\s+\w|WITH\s+(?:RECURSIVE\s+)?\w+\s+AS\b|(?:BEGIN|COMMIT|ROLLBACK|SAVEPOINT|RELEASE|VACUUM)\b[\w\s]*;?\s*")', + '^\s*(?:SELECT\s|INSERT\s+(?:OR\s+[A-Z]+\s+)?INTO\s|UPDATE\s+(?:OR\s+[A-Z]+\s+)?\w+(?:\s+SET\s|\s*$)|DELETE\s+FROM\s|WITH\s+(?:RECURSIVE\s+)?\w+\s+AS\b)', +] +paths = ["crates/libs/sbx/mgr/**", "crates/apps/sbx/**"] +allow = [ + "crates/libs/sbx/mgr/server/src/db.rs", + "crates/libs/sbx/mgr/server/src/db/**", +] + +[rules.pattern_ratchet.sbxmgr_sqlite.no_crossing] +mode = "required" + +# Seeding the ratchet reports its initial entries against the base. +[rules.pattern_ratchet.sbxmgr_sqlite.no_new_entries] +mode = "advisory" + +[rules.pattern_ratchet.sbxmgr_sqlite.no_raised_caps] +mode = "required" + +[rules.pattern_ratchet.sbxmgr_sqlite.listed_no_growth] +mode = "required" + +[rules.pattern_ratchet.sbxmgr_sqlite.stale_entries] +mode = "required" + +# The same database's schema and location, in any crate: its migrations, its +# path and file name, and the migrator's default-path helper for it belong to +# the store, which opens and migrates it. The migrations crate defines the +# migrators and common-core the path. These names are +# specific enough to check workspace-wide, unlike `sbxmgr_sqlite`'s patterns, +# which other services' crates match for their own databases. +[rules.pattern_ratchet.sbxmgr_sqlite_schema] +patterns = [ + '\bpplx_space_sbxmgr_migrations\b', + '\bSANDBOX_MANAGER_(?:DB|MIGRATOR)\b', + '\bsqlite_sandbox_manager\b', + '\bsbx-manager\.db\b', + '\bdb/sandbox-manager\b', +] +allow = [ + "crates/libs/sbx/mgr/server/src/db.rs", + "crates/libs/sbx/mgr/server/src/db/**", + "crates/libs/sbx/mgr/migrations/src/**", + "crates/libs/common/core/src/host/linux.rs", +] + +[rules.pattern_ratchet.sbxmgr_sqlite_schema.no_crossing] +mode = "required" + +# Seeding the ratchet reports its initial entries against the base. +[rules.pattern_ratchet.sbxmgr_sqlite_schema.no_new_entries] +mode = "advisory" + +[rules.pattern_ratchet.sbxmgr_sqlite_schema.no_raised_caps] +mode = "required" + +[rules.pattern_ratchet.sbxmgr_sqlite_schema.listed_no_growth] +mode = "required" + +[rules.pattern_ratchet.sbxmgr_sqlite_schema.stale_entries] +mode = "required" + [rules.crate_names] # A crate matches if a part of its package or library name, lowercased and # split on `-` and `_`, is one of these. diff --git a/architecture-baseline.toml b/architecture-baseline.toml index 7489f3c5d..48e0fc864 100644 --- a/architecture-baseline.toml +++ b/architecture-baseline.toml @@ -75,13 +75,13 @@ "crates/libs/object-store/src/lib.rs" = 1810 "crates/libs/observability/src/lib.rs" = 1063 "crates/libs/public/api/src/lib.rs" = 1793 -"crates/libs/sbx/mgr/server/src/auto_lifecycle_delete_receipts/db.rs" = 1158 +"crates/libs/sbx/mgr/server/src/db/auto_lifecycle_delete_receipts.rs" = 1158 "crates/libs/sbx/mgr/server/src/db/cleanup_episodes.rs" = 1833 "crates/libs/sbx/mgr/server/src/db/quarantine_recovery.rs" = 1042 -"crates/libs/sbx/mgr/server/src/db/sandboxes.rs" = 1553 -"crates/libs/sbx/mgr/server/src/db/state_transitions.rs" = 1018 +"crates/libs/sbx/mgr/server/src/db/sandboxes.rs" = 1547 +"crates/libs/sbx/mgr/server/src/db/state_transitions.rs" = 1016 "crates/libs/sbx/mgr/server/src/run.rs" = 1101 -"crates/libs/sbx/mgr/server/src/sandbox_eviction.rs" = 1273 +"crates/libs/sbx/mgr/server/src/sandbox_eviction.rs" = 1253 "crates/libs/sbx/mgr/server/src/service/activation_support.rs" = 1574 "crates/libs/sbx/mgr/server/src/service/auth_lookup.rs" = 1221 "crates/libs/sbx/mgr/server/src/service/cache_cleanup.rs" = 1455 @@ -145,6 +145,51 @@ "crates/libs/node/mgr/server/src/service/artifact_fs/receive.rs" = 4 "crates/libs/node/mgr/server/src/net/fallback.rs" = 1 +# sbx-mgr code outside its SQLite store that still names rusqlite types, the +# shared SQLite helpers or the shared job store, or opens a transaction itself. +[pattern_ratchet.sbxmgr_sqlite] +"crates/libs/sbx/mgr/server/src/lifecycle_metrics.rs" = 1 +"crates/libs/sbx/mgr/server/src/local_artifacts.rs" = 28 +"crates/libs/sbx/mgr/server/src/node_state_sync.rs" = 1 +"crates/libs/sbx/mgr/server/src/route_activation_claims.rs" = 2 +"crates/libs/sbx/mgr/server/src/run.rs" = 6 +"crates/libs/sbx/mgr/server/src/runtime_reconciliation.rs" = 1 +"crates/libs/sbx/mgr/server/src/sandbox_eviction.rs" = 5 +"crates/libs/sbx/mgr/server/src/service/activation_support.rs" = 9 +"crates/libs/sbx/mgr/server/src/service/auth_lookup.rs" = 5 +"crates/libs/sbx/mgr/server/src/service/cache_cleanup.rs" = 12 +"crates/libs/sbx/mgr/server/src/service/cleanup.rs" = 15 +"crates/libs/sbx/mgr/server/src/service/cleanup_executor.rs" = 7 +"crates/libs/sbx/mgr/server/src/service/cleanup_support.rs" = 7 +"crates/libs/sbx/mgr/server/src/service/cleanup_targets.rs" = 1 +"crates/libs/sbx/mgr/server/src/service/create_support.rs" = 5 +"crates/libs/sbx/mgr/server/src/service/grpc_lifecycle/pause_resume.rs" = 2 +"crates/libs/sbx/mgr/server/src/service/grpc_lifecycle/teardown_suspend.rs" = 2 +"crates/libs/sbx/mgr/server/src/service/grpc_pool_wait_clone.rs" = 1 +"crates/libs/sbx/mgr/server/src/service/grpc_spaced_process_snapshots/rollback_state.rs" = 1 +"crates/libs/sbx/mgr/server/src/service/grpc_state_snapshot.rs" = 1 +"crates/libs/sbx/mgr/server/src/service/mod.rs" = 13 +"crates/libs/sbx/mgr/server/src/service/network_runtime.rs" = 5 +"crates/libs/sbx/mgr/server/src/service/recovery.rs" = 5 +"crates/libs/sbx/mgr/server/src/service/recovery_cross_node.rs" = 12 +"crates/libs/sbx/mgr/server/src/service/start_support.rs" = 1 +"crates/libs/sbx/mgr/server/src/service/startup_resident.rs" = 1 +"crates/libs/sbx/mgr/server/src/service/storage.rs" = 1 +"crates/libs/sbx/mgr/server/src/service/template_snapshot_support.rs" = 3 +"crates/libs/sbx/mgr/server/src/service/test_options.rs" = 1 +"crates/libs/sbx/mgr/server/src/snapshot_gc.rs" = 1 +"crates/libs/sbx/mgr/server/src/snapshot_restore.rs" = 5 +"crates/libs/sbx/mgr/server/src/snapshot_scheduler.rs" = 3 +"crates/libs/sbx/mgr/server/src/snapshot_upload_jobs.rs" = 19 +"crates/libs/sbx/mgr/server/src/watchdog.rs" = 21 + +# Code outside the store that still names sbx-mgr's migrations or database +# path: its open and migration in `run.rs`, and the migrator, until it calls +# the store. +[pattern_ratchet.sbxmgr_sqlite_schema] +"crates/libs/db/migrator/src/lib.rs" = 5 +"crates/libs/sbx/mgr/server/src/run.rs" = 3 + # [crate_names] maps each workspace package with a catch-all name (a part such # as `common`, `utils` or `core`) to its member path. Only remove entries: a pull # request that adds one is reported, and fails the check. diff --git a/crates/libs/sbx/mgr/server/src/auto_lifecycle_delete_receipts/mod.rs b/crates/libs/sbx/mgr/server/src/auto_lifecycle_delete_receipts/mod.rs index 746baab6c..392aeeeb6 100644 --- a/crates/libs/sbx/mgr/server/src/auto_lifecycle_delete_receipts/mod.rs +++ b/crates/libs/sbx/mgr/server/src/auto_lifecycle_delete_receipts/mod.rs @@ -7,7 +7,6 @@ use pplx_space_common_core::types::{ }; use pplx_space_common_svc::types::AutoLifecyclePolicyFingerprint; -pub(crate) mod db; #[cfg(test)] mod tests; diff --git a/crates/libs/sbx/mgr/server/src/auto_lifecycle_delete_receipts/tests.rs b/crates/libs/sbx/mgr/server/src/auto_lifecycle_delete_receipts/tests.rs index b7a25302b..2e51dca68 100644 --- a/crates/libs/sbx/mgr/server/src/auto_lifecycle_delete_receipts/tests.rs +++ b/crates/libs/sbx/mgr/server/src/auto_lifecycle_delete_receipts/tests.rs @@ -1,978 +1,9 @@ -use std::collections::BTreeMap; - use pplx_space_common_core::types::{ - AutoLifecycleConditionSet, AutoLifecycleConditionSetId, AutoLifecycleDeleteReceiptId, - AutoLifecycleIdleAction, AutoLifecyclePolicy, AutoLifecycleStage, AutoLifecycleWakePolicy, - ExecutionGeneration, NodeManagedBy, NodeRole, PodUid, PolicyAuthority, RuntimeKind, - SandboxActivationAttemptId, SandboxId, SandboxOwner, SandboxRuntimeLabel, - SpaceApiRouteIdentity, + ExecutionGeneration, NodeManagedBy, PodUid, SandboxActivationAttemptId, SpaceApiRouteIdentity, }; -use pplx_space_common_svc::types::AutoLifecyclePolicyFingerprint; -use rusqlite::{Connection, OptionalExtension}; use super::*; -fn delete_policy() -> AutoLifecyclePolicy { - AutoLifecyclePolicy { - wake: AutoLifecycleWakePolicy::default(), - condition_sets: [( - AutoLifecycleConditionSetId::AccessIdle, - AutoLifecycleConditionSet::access_idle(), - )] - .into(), - stages: vec![AutoLifecycleStage { - condition_set: AutoLifecycleConditionSetId::AccessIdle, - at: 900, - action: AutoLifecycleIdleAction::Delete, - }], - } -} - -fn insert_sandbox(conn: &Connection, sandbox_id: SandboxId) -> AutoLifecycleDeleteExpectation { - let owner = SandboxOwner::parse("user").unwrap(); - let runtime_label = SandboxRuntimeLabel::from_sandbox_id(sandbox_id); - let policy = delete_policy(); - crate::db::insert_sandbox( - conn, - crate::db::InsertSandboxParams { - id: &sandbox_id.to_string(), - uuid8: &runtime_label, - state: crate::state::SandboxState::Started, - template: "seed", - template_hash: Some("hash"), - user: &owner, - runtime: RuntimeKind::Firecracker.as_str(), - snapshots_enabled: false, - rootfs_size_mb: None, - disk_io_limits: Default::default(), - pool_template: None, - volume_encryption: false, - process_env: &BTreeMap::new(), - auto_lifecycle: &policy, - network_policy_override: None, - network_policy_generation: 0, - network_policy_authority: PolicyAuthority::Cluster, - auto_lifecycle_authority: PolicyAuthority::Cluster, - }, - ) - .unwrap(); - AutoLifecycleDeleteExpectation { - policy_fingerprint: AutoLifecyclePolicyFingerprint::for_policy(&policy), - authority: PolicyAuthority::Cluster, - } -} - -fn cluster_route( - generation: i64, - activation_attempt_id: SandboxActivationAttemptId, -) -> SpaceApiRouteIdentity { - SpaceApiRouteIdentity { - execution_generation: Some(ExecutionGeneration::new(generation).unwrap()), - activation_attempt_id: Some(activation_attempt_id), - substrate_kind: Some(NodeManagedBy::Kubernetes), - ..SpaceApiRouteIdentity::default() - } -} - -fn receipt( - sandbox_id: SandboxId, - expectation: AutoLifecycleDeleteExpectation, - generation: i64, - activation_attempt_id: SandboxActivationAttemptId, -) -> AutoLifecycleDeleteReceipt { - AutoLifecycleDeleteReceipt { - receipt_id: AutoLifecycleDeleteReceiptId::generate(), - sandbox_id, - execution_generation: ExecutionGeneration::new(generation).unwrap(), - route_activation_id: activation_attempt_id, - policy_fingerprint: expectation.policy_fingerprint, - } -} - -#[test] -fn delete_admission_requires_consistent_authority_and_execution_evidence() { - use AutoLifecycleDeleteBlockReason as Block; - use AutoLifecycleDeleteOrdinaryReason as Ordinary; - - #[derive(Clone, Copy, Debug)] - enum Expected { - Ordinary(Ordinary), - Receipt, - Blocked(Block), - } - let cases = [ - ( - "local", - Some(NodeManagedBy::Space), - PolicyAuthority::Local, - "UPDATE sandboxes SET space_api_route_substrate_kind = 'k8s' WHERE id = ?3", - Expected::Ordinary(Ordinary::LocalAuthority), - ), - ( - "route-less direct", - Some(NodeManagedBy::Space), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET state = 'stopped' WHERE id = ?3", - Expected::Ordinary(Ordinary::DirectUnsupported), - ), - ( - "explicit direct", - None, - PolicyAuthority::Cluster, - "UPDATE sandboxes SET space_api_route_substrate_kind = 'space' WHERE id = ?3", - Expected::Ordinary(Ordinary::DirectUnsupported), - ), - ( - "current", - Some(NodeManagedBy::Kubernetes), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET space_api_route_execution_generation = 7, space_api_route_activation_attempt_id = ?1, space_api_route_substrate_kind = 'k8s', space_api_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", - Expected::Receipt, - ), - ( - "retained", - Some(NodeManagedBy::Kubernetes), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET last_observed_execution_generation = 7, last_observed_route_activation_id = ?1, last_observed_route_substrate_kind = 'k8s', last_observed_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", - Expected::Receipt, - ), - ( - "suspended subset", - Some(NodeManagedBy::Kubernetes), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET state = 'suspended', space_api_route_execution_generation = 7, last_observed_execution_generation = 7, last_observed_route_activation_id = ?1, last_observed_route_substrate_kind = 'k8s', last_observed_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", - Expected::Receipt, - ), - ( - "missing Kubernetes", - Some(NodeManagedBy::Kubernetes), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET state = 'stopped' WHERE id = ?3", - Expected::Blocked(Block::MissingRouteIdentity), - ), - ( - "missing unknown host", - None, - PolicyAuthority::Cluster, - "UPDATE sandboxes SET state = 'stopped' WHERE id = ?3", - Expected::Blocked(Block::MissingRouteIdentity), - ), - ( - "stale retained", - Some(NodeManagedBy::Kubernetes), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET space_api_route_execution_generation = 8, space_api_route_activation_attempt_id = ?1, space_api_route_substrate_kind = 'k8s', space_api_route_identity_provenance = 'cluster_fenced', last_observed_execution_generation = 7, last_observed_route_activation_id = ?1, last_observed_route_substrate_kind = 'k8s', last_observed_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", - Expected::Blocked(Block::ConflictingRouteIdentity), - ), - ( - "conflicting attempts", - Some(NodeManagedBy::Kubernetes), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET space_api_route_execution_generation = 7, space_api_route_activation_attempt_id = ?1, space_api_route_substrate_kind = 'k8s', space_api_route_identity_provenance = 'cluster_fenced', last_observed_execution_generation = 7, last_observed_route_activation_id = ?2, last_observed_route_substrate_kind = 'k8s', last_observed_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", - Expected::Blocked(Block::ConflictingRouteIdentity), - ), - ( - "host contradicts route", - Some(NodeManagedBy::Space), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET space_api_route_substrate_kind = 'k8s' WHERE id = ?3", - Expected::Blocked(Block::ConflictingRouteIdentity), - ), - ( - "malformed current attempt", - Some(NodeManagedBy::Kubernetes), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET space_api_route_execution_generation = 7, space_api_route_activation_attempt_id = 'malformed', space_api_route_substrate_kind = 'k8s', space_api_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", - Expected::Blocked(Block::ConflictingRouteIdentity), - ), - ( - "malformed retained attempt", - Some(NodeManagedBy::Kubernetes), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET space_api_route_execution_generation = 7, space_api_route_activation_attempt_id = ?1, space_api_route_substrate_kind = 'k8s', space_api_route_identity_provenance = 'cluster_fenced', last_observed_execution_generation = 7, last_observed_route_activation_id = 'malformed', last_observed_route_substrate_kind = 'k8s', last_observed_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", - Expected::Blocked(Block::ConflictingRouteIdentity), - ), - ( - "invalid current generation", - Some(NodeManagedBy::Kubernetes), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET space_api_route_execution_generation = -1, space_api_route_activation_attempt_id = ?1, space_api_route_substrate_kind = 'k8s', space_api_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", - Expected::Blocked(Block::ConflictingRouteIdentity), - ), - ( - "invalid retained generation", - Some(NodeManagedBy::Kubernetes), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET last_observed_execution_generation = 0, last_observed_route_activation_id = ?1, last_observed_route_substrate_kind = 'k8s', last_observed_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", - Expected::Blocked(Block::ConflictingRouteIdentity), - ), - ( - "mixed substrates", - None, - PolicyAuthority::Cluster, - "UPDATE sandboxes SET space_api_route_substrate_kind = 'space', last_observed_route_substrate_kind = 'k8s' WHERE id = ?3", - Expected::Blocked(Block::ConflictingRouteIdentity), - ), - ( - "unknown substrate", - Some(NodeManagedBy::Space), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET last_observed_route_substrate_kind = 'future' WHERE id = ?3", - Expected::Blocked(Block::UnknownSubstrate), - ), - ( - "partial retained", - Some(NodeManagedBy::Kubernetes), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET space_api_route_execution_generation = 7, space_api_route_activation_attempt_id = ?1, space_api_route_substrate_kind = 'k8s', space_api_route_identity_provenance = 'cluster_fenced', last_observed_execution_generation = 7 WHERE id = ?3", - Expected::Blocked(Block::ConflictingRouteIdentity), - ), - ( - "pod contradicts direct", - Some(NodeManagedBy::Space), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET space_api_route_pod_uid = 'pod' WHERE id = ?3", - Expected::Blocked(Block::ConflictingRouteIdentity), - ), - ( - "retained placement contradicts direct", - Some(NodeManagedBy::Space), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET last_observed_route_generation_substrate_kind = 'k8s' WHERE id = ?3", - Expected::Blocked(Block::ConflictingRouteIdentity), - ), - ]; - for (name, host, authority, setup, expected) in cases { - let conn = crate::db::test_conn(); - let id = SandboxId::new(); - let mut expectation = insert_sandbox(&conn, id); - expectation.authority = authority; - let attempt = SandboxActivationAttemptId::new(); - // Seed historical malformed evidence without weakening admission writes. - conn.pragma_update(None, "ignore_check_constraints", true) - .unwrap(); - conn.execute( - setup, - rusqlite::params![ - attempt.to_string(), - SandboxActivationAttemptId::new().to_string(), - id.to_string() - ], - ) - .unwrap(); - conn.pragma_update(None, "ignore_check_constraints", false) - .unwrap(); - conn.execute( - "UPDATE sandboxes SET auto_lifecycle_authority = ?1 WHERE id = ?2", - rusqlite::params![authority.to_string(), id.to_string()], - ) - .unwrap(); - let admitted = - db::classify_and_record_admission(&conn, id, Some(NodeRole::Worker), host, expectation) - .unwrap(); - match (expected, admitted) { - (Expected::Ordinary(expected), AutoLifecycleDeleteAdmission::Ordinary(actual)) => { - assert_eq!(actual, expected, "{name}") - } - (Expected::Receipt, AutoLifecycleDeleteAdmission::Receipt(evidence)) => { - assert_eq!(evidence.execution_generation.get(), 7, "{name}"); - assert_eq!(evidence.activation_attempt_id, attempt, "{name}"); - } - (Expected::Blocked(expected), AutoLifecycleDeleteAdmission::Blocked { reason, .. }) => { - assert_eq!(reason, expected, "{name}"); - let retry: i64 = conn - .query_row( - "SELECT next_auto_event_at FROM sandboxes WHERE id = ?1", - [id.to_string()], - |row| row.get(0), - ) - .unwrap(); - assert!(retry > crate::db::now_epoch_i64().unwrap(), "{name}"); - assert!(crate::db::sandbox_exists(&conn, &id.to_string()).unwrap()); - } - pair => panic!("unexpected {name}: {pair:?}"), - } - } - for completed in [false, true] { - let conn = crate::db::test_conn(); - let id = SandboxId::new(); - let expectation = insert_sandbox(&conn, id); - let pending = receipt(id, expectation, 7, SandboxActivationAttemptId::new()); - db::insert_pending_receipt(&conn, pending).unwrap(); - if completed { - db::complete_pending_receipt(&conn, id, pending.receipt_id).unwrap(); - } - let admitted = db::classify_and_record_admission( - &conn, - id, - Some(NodeRole::Worker), - Some(NodeManagedBy::Space), - expectation, - ) - .unwrap(); - if completed { - assert_eq!( - admitted, - AutoLifecycleDeleteAdmission::Ordinary(Ordinary::DirectUnsupported) - ); - } else { - assert!(matches!( - admitted, - AutoLifecycleDeleteAdmission::Blocked { - reason: Block::ConflictingRouteIdentity, - .. - } - )); - } - assert_eq!( - conn.query_row( - "SELECT receipt_id, state FROM auto_lifecycle_delete_outbox WHERE sandbox_id = ?1", - [id.to_string()], - |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), - ) - .unwrap(), - ( - pending.receipt_id.to_string(), - if completed { "completed" } else { "pending" }.into() - ) - ); - } -} - -#[test] -fn strict_admission_requires_complete_current_cluster_fencing() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let activation_attempt_id = SandboxActivationAttemptId::new(); - crate::db::update_space_api_route_identity( - &conn, - sandbox_id, - &cluster_route(7, activation_attempt_id), - RouteIdentityProvenance::ClusterFenced, - ) - .unwrap(); - - assert!(matches!( - db::classify_and_record_admission( - &conn, - sandbox_id, - Some(NodeRole::Worker), - None, expectation, - ) - .unwrap(), - AutoLifecycleDeleteAdmission::Receipt(RouteEvidence { - execution_generation, - activation_attempt_id: observed_attempt, - .. - }) if execution_generation == ExecutionGeneration::new(7).unwrap() - && observed_attempt == activation_attempt_id - )); - - conn.execute( - "UPDATE sandboxes - SET space_api_route_execution_generation = NULL, - space_api_route_activation_attempt_id = NULL, - space_api_route_substrate_kind = NULL, - space_api_route_identity_provenance = NULL - WHERE id = ?1", - [sandbox_id.to_string()], - ) - .unwrap(); - assert!(matches!( - db::classify_and_record_admission(&conn, sandbox_id, Some(NodeRole::Worker), None, expectation,) - .unwrap(), - AutoLifecycleDeleteAdmission::Receipt(RouteEvidence { - execution_generation, - activation_attempt_id: observed_attempt, - .. - }) if execution_generation == ExecutionGeneration::new(7).unwrap() - && observed_attempt == activation_attempt_id - )); - - conn.execute( - "UPDATE sandboxes - SET space_api_route_execution_generation = 8, - space_api_route_activation_attempt_id = NULL, - space_api_route_substrate_kind = NULL, - space_api_route_identity_provenance = NULL - WHERE id = ?1", - [sandbox_id.to_string()], - ) - .unwrap(); - assert!(matches!( - db::classify_and_record_admission( - &conn, - sandbox_id, - Some(NodeRole::Worker), - None, - expectation, - ) - .unwrap(), - AutoLifecycleDeleteAdmission::Blocked { - reason: AutoLifecycleDeleteBlockReason::MissingSubstrate, - changed: true, - } - )); - let (reason, blocked_at, next_auto_event_at, runtime_state_json): ( - Option, - Option, - Option, - String, - ) = conn - .query_row( - "SELECT auto_lifecycle_delete_block_reason, - auto_lifecycle_delete_blocked_at, - next_auto_event_at, - auto_lifecycle_state_json - FROM sandboxes WHERE id = ?1", - [sandbox_id.to_string()], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), - ) - .unwrap(); - assert_eq!(reason.as_deref(), Some("missing_substrate")); - assert!(blocked_at.is_some()); - assert!(next_auto_event_at.unwrap() > crate::db::now_epoch_i64().unwrap()); - let runtime_state: pplx_space_common_core::types::AutoLifecycleRuntimeState = - serde_json::from_str(&runtime_state_json).unwrap(); - assert!( - runtime_state - .last_auto_error - .as_deref() - .is_some_and(|error| error.contains("missing_substrate")) - ); - - crate::db::update_space_api_route_identity( - &conn, - sandbox_id, - &cluster_route(8, SandboxActivationAttemptId::new()), - RouteIdentityProvenance::ClusterFenced, - ) - .unwrap(); - let (reason, blocked_at, runtime_state_json): (Option, Option, String) = conn - .query_row( - "SELECT auto_lifecycle_delete_block_reason, - auto_lifecycle_delete_blocked_at, - auto_lifecycle_state_json - FROM sandboxes WHERE id = ?1", - [sandbox_id.to_string()], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), - ) - .unwrap(); - assert_eq!((reason, blocked_at), (None, None)); - let runtime_state: pplx_space_common_core::types::AutoLifecycleRuntimeState = - serde_json::from_str(&runtime_state_json).unwrap(); - assert_eq!(runtime_state.last_auto_error, None); -} - -#[test] -fn dormant_suspend_identity_uses_matching_retained_route_evidence() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let activation_attempt_id = SandboxActivationAttemptId::new(); - crate::db::update_space_api_route_identity( - &conn, - sandbox_id, - &cluster_route(7, activation_attempt_id), - RouteIdentityProvenance::ClusterFenced, - ) - .unwrap(); - - // Suspend completion retains generation for central publication but clears - // the remaining current route fields. The matching complete retained tuple - // is still the exact route that produced this dormant sandbox. - conn.execute( - "UPDATE sandboxes - SET state = 'suspended', - space_api_route_activation_attempt_id = NULL, - space_api_route_substrate_kind = NULL, - space_api_route_identity_provenance = NULL - WHERE id = ?1", - [sandbox_id.to_string()], - ) - .unwrap(); - - assert!(matches!( - db::classify_and_record_admission( - &conn, - sandbox_id, - Some(NodeRole::Worker), - None, expectation, - ) - .unwrap(), - AutoLifecycleDeleteAdmission::Receipt(RouteEvidence { - execution_generation, - activation_attempt_id: observed_attempt, - .. - }) if execution_generation == ExecutionGeneration::new(7).unwrap() - && observed_attempt == activation_attempt_id - )); -} - -#[test] -fn non_dormant_partial_route_does_not_reuse_retained_evidence() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let activation_attempt_id = SandboxActivationAttemptId::new(); - crate::db::update_space_api_route_identity( - &conn, - sandbox_id, - &cluster_route(7, activation_attempt_id), - RouteIdentityProvenance::ClusterFenced, - ) - .unwrap(); - - conn.execute( - "UPDATE sandboxes - SET space_api_route_activation_attempt_id = NULL, - space_api_route_substrate_kind = NULL, - space_api_route_identity_provenance = NULL - WHERE id = ?1", - [sandbox_id.to_string()], - ) - .unwrap(); - - assert!(matches!( - db::classify_and_record_admission( - &conn, - sandbox_id, - Some(NodeRole::Worker), - None, - expectation, - ) - .unwrap(), - AutoLifecycleDeleteAdmission::Blocked { - reason: AutoLifecycleDeleteBlockReason::MissingSubstrate, - changed: true, - } - )); -} - -#[test] -fn contradictory_current_and_retained_routes_block_deletion() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let retained_attempt = SandboxActivationAttemptId::new(); - crate::db::update_space_api_route_identity( - &conn, - sandbox_id, - &cluster_route(8, retained_attempt), - RouteIdentityProvenance::ClusterFenced, - ) - .unwrap(); - - // Model a stale or corrupt current-route writer without updating the - // retained tuple. Receipt admission must not choose either side of the - // contradiction and delete local state that central fencing will retain. - conn.execute( - "UPDATE sandboxes - SET space_api_route_execution_generation = 7, - space_api_route_activation_attempt_id = ?1, - space_api_route_substrate_kind = 'k8s', - space_api_route_identity_provenance = 'cluster_fenced' - WHERE id = ?2", - rusqlite::params![ - SandboxActivationAttemptId::new().to_string(), - sandbox_id.to_string() - ], - ) - .unwrap(); - - assert!(matches!( - db::classify_and_record_admission( - &conn, - sandbox_id, - Some(NodeRole::Worker), - None, expectation, - ) - .unwrap(), - AutoLifecycleDeleteAdmission::Blocked { reason, changed: true } - if reason.as_str() == "conflicting_route_identity" - )); -} - -#[test] -fn later_current_attempt_blocks_receipt_admission() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let retained_attempt = SandboxActivationAttemptId::new(); - crate::db::update_space_api_route_identity( - &conn, - sandbox_id, - &cluster_route(8, retained_attempt), - RouteIdentityProvenance::ClusterFenced, - ) - .unwrap(); - - let current_attempt = SandboxActivationAttemptId::new_after(retained_attempt).unwrap(); - conn.execute( - "UPDATE sandboxes - SET space_api_route_activation_attempt_id = ?1 - WHERE id = ?2", - rusqlite::params![current_attempt.to_string(), sandbox_id.to_string()], - ) - .unwrap(); - - assert!(matches!( - db::classify_and_record_admission( - &conn, - sandbox_id, - Some(NodeRole::Worker), - None, expectation, - ) - .unwrap(), - AutoLifecycleDeleteAdmission::Blocked { reason, changed: true } - if reason == AutoLifecycleDeleteBlockReason::ConflictingRouteIdentity - )); -} - -#[test] -fn delayed_current_attempt_blocks_receipt_admission() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let delayed_attempt = SandboxActivationAttemptId::new(); - let retained_attempt = SandboxActivationAttemptId::new_after(delayed_attempt).unwrap(); - crate::db::update_space_api_route_identity( - &conn, - sandbox_id, - &cluster_route(8, retained_attempt), - RouteIdentityProvenance::ClusterFenced, - ) - .unwrap(); - - conn.execute( - "UPDATE sandboxes - SET space_api_route_activation_attempt_id = ?1 - WHERE id = ?2", - rusqlite::params![delayed_attempt.to_string(), sandbox_id.to_string()], - ) - .unwrap(); - - assert!(matches!( - db::classify_and_record_admission( - &conn, - sandbox_id, - Some(NodeRole::Worker), - None, expectation, - ) - .unwrap(), - AutoLifecycleDeleteAdmission::Blocked { reason, changed: true } - if reason == AutoLifecycleDeleteBlockReason::ConflictingRouteIdentity - )); -} - -#[test] -fn receipt_eligibility_excludes_non_worker_and_non_kubernetes_populations() { - let conn = crate::db::test_conn(); - - let direct_id = SandboxId::new(); - let direct_expectation = insert_sandbox(&conn, direct_id); - let direct_identity = SpaceApiRouteIdentity { - execution_generation: Some(ExecutionGeneration::new(1).unwrap()), - activation_attempt_id: Some(SandboxActivationAttemptId::new()), - substrate_kind: Some(NodeManagedBy::Space), - ..SpaceApiRouteIdentity::default() - }; - crate::db::update_space_api_route_identity( - &conn, - direct_id, - &direct_identity, - RouteIdentityProvenance::Unfenced, - ) - .unwrap(); - conn.execute( - "UPDATE sandboxes - SET space_api_route_activation_attempt_id = 'not-an-attempt' - WHERE id = ?1", - [direct_id.to_string()], - ) - .unwrap(); - assert_eq!( - db::classify_and_record_admission( - &conn, - direct_id, - Some(NodeRole::Worker), - None, - direct_expectation, - ) - .unwrap(), - AutoLifecycleDeleteAdmission::Ordinary( - AutoLifecycleDeleteOrdinaryReason::DirectUnsupported, - ) - ); - - let builder_id = SandboxId::new(); - let builder_expectation = insert_sandbox(&conn, builder_id); - crate::db::update_space_api_route_identity( - &conn, - builder_id, - &cluster_route(2, SandboxActivationAttemptId::new()), - RouteIdentityProvenance::ClusterFenced, - ) - .unwrap(); - conn.execute( - "UPDATE sandboxes - SET space_api_route_activation_attempt_id = 'not-an-attempt' - WHERE id = ?1", - [builder_id.to_string()], - ) - .unwrap(); - assert!(matches!( - db::classify_and_record_admission(&conn, builder_id, None, None, builder_expectation,) - .unwrap(), - AutoLifecycleDeleteAdmission::Blocked { - reason: AutoLifecycleDeleteBlockReason::NodeRoleUnavailable, - .. - } - )); - assert!(matches!( - db::classify_and_record_admission( - &conn, - builder_id, - Some(NodeRole::Builder), - None, - builder_expectation, - ) - .unwrap(), - AutoLifecycleDeleteAdmission::Blocked { - reason: AutoLifecycleDeleteBlockReason::NodeRoleNotWorker, - .. - } - )); - let blocked_at: String = conn - .query_row( - "SELECT auto_lifecycle_delete_blocked_at FROM sandboxes WHERE id = ?1", - [builder_id.to_string()], - |row| row.get(0), - ) - .unwrap(); - db::record_retained_route_evidence( - &conn, - builder_id, - RouteEvidence::from_identity( - &cluster_route(2, SandboxActivationAttemptId::new()), - RouteIdentityProvenance::ClusterFenced, - ) - .unwrap(), - ) - .unwrap(); - assert_eq!( - conn.query_row( - "SELECT auto_lifecycle_delete_block_reason, - auto_lifecycle_delete_blocked_at - FROM sandboxes WHERE id = ?1", - [builder_id.to_string()], - |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), - ) - .unwrap(), - ("node_role_not_worker".to_string(), blocked_at), - "route repair cannot clear a publisher-role block" - ); - - let local_id = SandboxId::new(); - let mut local_expectation = insert_sandbox(&conn, local_id); - conn.execute( - "UPDATE sandboxes SET auto_lifecycle_authority = 'local' WHERE id = ?1", - [local_id.to_string()], - ) - .unwrap(); - local_expectation.authority = PolicyAuthority::Local; - assert_eq!( - db::classify_and_record_admission(&conn, local_id, None, None, local_expectation,).unwrap(), - AutoLifecycleDeleteAdmission::Ordinary(AutoLifecycleDeleteOrdinaryReason::LocalAuthority,) - ); - - let provisional_id = SandboxId::new(); - let provisional_expectation = insert_sandbox(&conn, provisional_id); - crate::db::update_space_api_route_identity( - &conn, - provisional_id, - &cluster_route(3, SandboxActivationAttemptId::new()), - RouteIdentityProvenance::Unfenced, - ) - .unwrap(); - assert!(matches!( - db::classify_and_record_admission( - &conn, - provisional_id, - Some(NodeRole::Worker), - None, - provisional_expectation, - ) - .unwrap(), - AutoLifecycleDeleteAdmission::Blocked { - reason: AutoLifecycleDeleteBlockReason::UnfencedRoute, - .. - } - )); - - let missing_route_id = SandboxId::new(); - let missing_route_expectation = insert_sandbox(&conn, missing_route_id); - assert!(matches!( - db::classify_and_record_admission( - &conn, - missing_route_id, - Some(NodeRole::Worker), - None, - missing_route_expectation, - ) - .unwrap(), - AutoLifecycleDeleteAdmission::Blocked { - reason: AutoLifecycleDeleteBlockReason::MissingRouteIdentity, - .. - } - )); - crate::db::update_space_api_route_identity( - &conn, - missing_route_id, - &cluster_route(4, SandboxActivationAttemptId::new()), - RouteIdentityProvenance::ClusterFenced, - ) - .unwrap(); - assert!(matches!( - db::classify_and_record_admission( - &conn, - missing_route_id, - Some(NodeRole::Worker), - None, - missing_route_expectation, - ) - .unwrap(), - AutoLifecycleDeleteAdmission::Receipt(_) - )); -} - -#[test] -fn matching_receipt_completes_atomically_with_row_deletion() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let activation_attempt_id = SandboxActivationAttemptId::new(); - let claimed_receipt = receipt(sandbox_id, expectation, 9, activation_attempt_id); - db::insert_pending_receipt(&conn, claimed_receipt).unwrap(); - conn.execute( - "UPDATE sandboxes - SET state = 'deleting', - cleanup_completion = 'delete_row', - cleanup_keep_filesystem = 0 - WHERE id = ?1", - [sandbox_id.to_string()], - ) - .unwrap(); - - crate::db::complete_row_intent_cleanup_with_receipt( - &conn, - &sandbox_id.to_string(), - crate::db::CleanupCompletion::DeleteRow, - None, - Some(claimed_receipt), - ) - .unwrap(); - - assert!( - crate::db::get_sandbox(&conn, &sandbox_id.to_string()) - .unwrap() - .is_none() - ); - let state: String = conn - .query_row( - "SELECT state FROM auto_lifecycle_delete_outbox WHERE receipt_id = ?1", - [claimed_receipt.receipt_id.to_string()], - |row| row.get(0), - ) - .unwrap(); - assert_eq!(state, "completed"); - - let recovered_sandbox_id = SandboxId::new(); - let recovered_expectation = insert_sandbox(&conn, recovered_sandbox_id); - let recovered_attempt = SandboxActivationAttemptId::new(); - let recovered_receipt = receipt( - recovered_sandbox_id, - recovered_expectation, - 10, - recovered_attempt, - ); - db::insert_pending_receipt(&conn, recovered_receipt).unwrap(); - conn.execute( - "UPDATE sandboxes - SET state = 'deleting', - cleanup_completion = 'delete_row', - cleanup_keep_filesystem = 0 - WHERE id = ?1", - [recovered_sandbox_id.to_string()], - ) - .unwrap(); - crate::db::complete_row_intent_cleanup_with_receipt( - &conn, - &recovered_sandbox_id.to_string(), - crate::db::CleanupCompletion::DeleteRow, - None, - None, - ) - .unwrap(); - assert!( - crate::db::get_sandbox(&conn, &recovered_sandbox_id.to_string()) - .unwrap() - .is_none() - ); - assert_eq!( - conn.query_row( - "SELECT route_activation_id, state - FROM auto_lifecycle_delete_outbox - WHERE receipt_id = ?1", - [recovered_receipt.receipt_id.to_string()], - |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), - ) - .unwrap(), - (recovered_attempt.to_string(), "completed".to_string()) - ); -} - -#[test] -fn pending_receipt_does_not_silently_survive_lost_delete_intent() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let pending = receipt( - sandbox_id, - expectation, - 11, - SandboxActivationAttemptId::new(), - ); - db::insert_pending_receipt(&conn, pending).unwrap(); - - let error = crate::db::complete_row_intent_cleanup_with_receipt( - &conn, - &sandbox_id.to_string(), - crate::db::CleanupCompletion::DeleteRow, - None, - Some(pending), - ) - .unwrap_err(); - - assert!( - crate::db::get_sandbox(&conn, &sandbox_id.to_string()) - .unwrap() - .is_some() - ); - assert!( - db::pending_receipt_for_sandbox(&conn, sandbox_id) - .unwrap() - .is_some() - ); - assert!(matches!( - error, - crate::db::CompleteRowIntentCleanupError::PendingReceiptLostDeleteIntent { - sandbox_id: observed, - } if observed == sandbox_id - )); -} - #[test] fn route_evidence_uses_the_effective_kubernetes_substrate() { let activation_attempt_id = SandboxActivationAttemptId::new(); @@ -993,1001 +24,3 @@ fn route_evidence_uses_the_effective_kubernetes_substrate() { }) ); } - -#[test] -fn pending_receipt_guard_aborts_only_the_delete_statement() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - db::insert_pending_receipt( - &conn, - receipt( - sandbox_id, - expectation, - 3, - SandboxActivationAttemptId::new(), - ), - ) - .unwrap(); - - let tx = crate::db::immediate_transaction(&conn).unwrap(); - tx.execute( - "UPDATE sandboxes SET failure_reason = 'before guarded delete' WHERE id = ?1", - [sandbox_id.to_string()], - ) - .unwrap(); - let error = tx - .execute( - "DELETE FROM sandboxes WHERE id = ?1", - [sandbox_id.to_string()], - ) - .unwrap_err(); - let error = db::map_pending_receipt_delete_error(error); - let rusqlite::Error::ToSqlConversionFailure(error) = error else { - panic!("pending-receipt guard must map to a typed database error") - }; - assert!( - error - .downcast_ref::() - .is_some() - ); - tx.commit().unwrap(); - - let failure_reason: Option = conn - .query_row( - "SELECT failure_reason FROM sandboxes WHERE id = ?1", - [sandbox_id.to_string()], - |row| row.get(0), - ) - .unwrap(); - assert_eq!(failure_reason.as_deref(), Some("before guarded delete")); -} - -#[test] -fn list_and_ack_touch_only_completed_receipts() { - let conn = crate::db::test_conn(); - let completed_sandbox = SandboxId::new(); - let pending_sandbox = SandboxId::new(); - let completed_expectation = insert_sandbox(&conn, completed_sandbox); - let pending_expectation = insert_sandbox(&conn, pending_sandbox); - let completed = receipt( - completed_sandbox, - completed_expectation, - 1, - SandboxActivationAttemptId::new(), - ); - let pending = receipt( - pending_sandbox, - pending_expectation, - 1, - SandboxActivationAttemptId::new(), - ); - db::insert_pending_receipt(&conn, completed).unwrap(); - db::insert_pending_receipt(&conn, pending).unwrap(); - db::complete_pending_receipt(&conn, completed_sandbox, completed.receipt_id).unwrap(); - - assert_eq!(db::list_completed_receipts(&conn).unwrap(), vec![completed]); - assert_eq!( - db::acknowledge_completed_receipts(&conn, &[completed.receipt_id, pending.receipt_id]) - .unwrap(), - vec![completed.receipt_id] - ); - assert!( - db::pending_receipt_for_sandbox(&conn, pending_sandbox) - .unwrap() - .is_some() - ); - assert!( - conn.query_row( - "SELECT receipt_id FROM auto_lifecycle_delete_outbox WHERE sandbox_id = ?1", - [completed_sandbox.to_string()], - |row| row.get::<_, String>(0), - ) - .optional() - .unwrap() - .is_none() - ); -} - -#[test] -fn completed_receipt_listing_is_bounded_without_starving_fresh_evidence() { - let conn = crate::db::test_conn(); - let receipts = (0..17) - .map(|generation| { - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let receipt = receipt( - sandbox_id, - expectation, - generation + 1, - SandboxActivationAttemptId::new(), - ); - db::insert_pending_receipt(&conn, receipt).unwrap(); - db::complete_pending_receipt(&conn, sandbox_id, receipt.receipt_id).unwrap(); - receipt - }) - .collect::>(); - - let first = db::list_completed_receipts(&conn).unwrap(); - assert_eq!(first.len(), 16); - let omitted = receipts - .iter() - .copied() - .find(|receipt| !first.contains(receipt)) - .expect("the bounded first batch omits one fresh receipt"); - - let second = db::list_completed_receipts(&conn).unwrap(); - assert_eq!(second.len(), 16); - assert_eq!(second.first(), Some(&omitted)); -} - -#[test] -fn pending_receipt_rejects_regressing_and_completed_evidence() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let older_attempt = SandboxActivationAttemptId::new(); - let first_attempt = SandboxActivationAttemptId::new_after(older_attempt).unwrap(); - let first = receipt(sandbox_id, expectation, 4, first_attempt); - assert_eq!(db::insert_pending_receipt(&conn, first).unwrap(), first); - - let retry = AutoLifecycleDeleteReceipt { - receipt_id: AutoLifecycleDeleteReceiptId::generate(), - ..first - }; - assert_eq!(db::insert_pending_receipt(&conn, retry).unwrap(), first); - - let conflicting = AutoLifecycleDeleteReceipt { - route_activation_id: older_attempt, - ..retry - }; - db::insert_pending_receipt(&conn, conflicting).expect_err("conflicting receipt rejected"); - - db::complete_pending_receipt(&conn, sandbox_id, first.receipt_id).unwrap(); - db::insert_pending_receipt(&conn, retry).expect_err("conflicting receipt rejected"); -} - -#[test] -fn newer_attempt_conflicts_with_a_pending_receipt() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let first_attempt = SandboxActivationAttemptId::new(); - let newer_attempt = SandboxActivationAttemptId::new_after(first_attempt).unwrap(); - let first = receipt(sandbox_id, expectation, 4, first_attempt); - db::insert_pending_receipt(&conn, first).unwrap(); - - let replacement = AutoLifecycleDeleteReceipt { - receipt_id: AutoLifecycleDeleteReceiptId::generate(), - route_activation_id: newer_attempt, - ..first - }; - - db::insert_pending_receipt(&conn, replacement).expect_err("conflicting receipt rejected"); - assert_eq!( - db::pending_receipt_for_sandbox(&conn, sandbox_id).unwrap(), - Some(first) - ); -} - -#[test] -fn pending_receipt_blocks_same_generation_route_adoption() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let admitted_attempt = SandboxActivationAttemptId::new(); - let pending = receipt(sandbox_id, expectation, 4, admitted_attempt); - db::insert_pending_receipt(&conn, pending).unwrap(); - - let replacement_attempt = SandboxActivationAttemptId::new_after(admitted_attempt).unwrap(); - assert!( - crate::db::update_space_api_route_identity( - &conn, - sandbox_id, - &cluster_route(4, replacement_attempt), - RouteIdentityProvenance::ClusterFenced, - ) - .is_err() - ); - assert_eq!( - crate::db::get_sandbox(&conn, &sandbox_id.to_string()) - .unwrap() - .unwrap() - .space_api_route_identity - .activation_attempt_id, - None - ); - assert_eq!( - db::pending_receipt_for_sandbox(&conn, sandbox_id).unwrap(), - Some(pending) - ); -} - -#[test] -fn current_policy_refreshes_a_pending_receipt_without_hiding_its_age() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let activation_attempt_id = SandboxActivationAttemptId::new(); - let first = receipt(sandbox_id, expectation, 4, activation_attempt_id); - db::insert_pending_receipt(&conn, first).unwrap(); - conn.execute( - "UPDATE auto_lifecycle_delete_outbox - SET created_at = '1' - WHERE sandbox_id = ?1", - [sandbox_id.to_string()], - ) - .unwrap(); - - let mut revised_policy = delete_policy(); - revised_policy.stages[0].at += 1; - let replacement = AutoLifecycleDeleteReceipt { - receipt_id: AutoLifecycleDeleteReceiptId::generate(), - policy_fingerprint: AutoLifecyclePolicyFingerprint::for_policy(&revised_policy), - ..first - }; - - assert_eq!( - db::insert_pending_receipt(&conn, replacement).unwrap(), - replacement, - ); - assert_eq!( - conn.query_row( - "SELECT receipt_id, auto_lifecycle_policy_fingerprint, created_at - FROM auto_lifecycle_delete_outbox - WHERE sandbox_id = ?1", - [sandbox_id.to_string()], - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - )) - }, - ) - .unwrap(), - ( - replacement.receipt_id.to_string(), - replacement.policy_fingerprint.to_string(), - "1".to_string(), - ), - ); -} - -#[test] -fn only_generation_supersession_discards_a_pending_receipt() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let pending = receipt( - sandbox_id, - expectation, - 4, - SandboxActivationAttemptId::new(), - ); - db::insert_pending_receipt(&conn, pending).unwrap(); - - assert!( - db::discard_pending_receipt_superseded_by_generation( - &conn, - sandbox_id, - pending.execution_generation, - ) - .unwrap() - .is_none() - ); - assert_eq!( - db::pending_receipt_for_sandbox(&conn, sandbox_id).unwrap(), - Some(pending) - ); - - let next_generation = ExecutionGeneration::new(5).unwrap(); - assert_eq!( - db::discard_pending_receipt_superseded_by_generation(&conn, sandbox_id, next_generation,) - .unwrap(), - Some(db::SupersededPendingReceipt { - receipt_id: pending.receipt_id, - execution_generation: pending.execution_generation, - }) - ); - assert!( - db::pending_receipt_for_sandbox(&conn, sandbox_id) - .unwrap() - .is_none() - ); -} - -#[test] -fn pending_receipt_fence_is_distinct_from_a_missing_sandbox() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let generation = ExecutionGeneration::new(4).unwrap(); - db::insert_pending_receipt( - &conn, - receipt( - sandbox_id, - expectation, - generation.get(), - SandboxActivationAttemptId::new(), - ), - ) - .unwrap(); - - let error = crate::db::update_space_api_route_identity( - &conn, - sandbox_id, - &cluster_route(generation.get(), SandboxActivationAttemptId::new()), - RouteIdentityProvenance::ClusterFenced, - ) - .unwrap_err(); - - assert!(db::is_pending_receipt_execution_fence(&error)); - - let missing_id = SandboxId::new(); - let missing_error = crate::db::update_space_api_route_identity( - &conn, - missing_id, - &cluster_route(generation.get(), SandboxActivationAttemptId::new()), - RouteIdentityProvenance::ClusterFenced, - ) - .unwrap_err(); - assert!(matches!( - missing_error, - rusqlite::Error::QueryReturnedNoRows - )); -} - -#[test] -fn deleting_row_rejects_runtime_attachment_without_retiring_its_receipt() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let pending = receipt( - sandbox_id, - expectation, - 4, - SandboxActivationAttemptId::new(), - ); - db::insert_pending_receipt(&conn, pending).unwrap(); - conn.execute( - "UPDATE sandboxes - SET state = 'deleting', - cleanup_completion = 'delete_row', - cleanup_keep_filesystem = 0 - WHERE id = ?1", - [sandbox_id.to_string()], - ) - .unwrap(); - - let error = crate::db::update_runtime_fields( - &conn, - sandbox_id, - crate::db::RuntimeFields { - uid: 10_000, - slot: 7, - guest_cid: 37, - pid: std::process::id(), - spaced_host_pid: 10_001, - netgtw_pid: 10_002, - vsock_uds_path: "/space/run/firecracker/sandboxes/test/root/v.sock", - api_socket_path: "/space/run/firecracker/sandboxes/test/api.sock", - spaced_endpoint_proof: None, - space_api_route_identity: SpaceApiRouteIdentity { - execution_generation: Some(ExecutionGeneration::new(5).unwrap()), - ..SpaceApiRouteIdentity::default() - }, - route_identity_provenance: RouteIdentityProvenance::Unfenced, - }, - ) - .expect_err("runtime attachment must not supersede cleanup ownership"); - - assert!(db::is_lifecycle_execution_fence(&error)); - assert!(!db::is_pending_receipt_execution_fence(&error)); - assert_eq!( - db::pending_receipt_for_sandbox(&conn, sandbox_id).unwrap(), - Some(pending) - ); -} - -#[test] -fn beginning_a_new_execution_retires_an_older_pending_receipt() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - conn.execute( - "UPDATE sandboxes SET state = 'created' WHERE id = ?1", - [sandbox_id.to_string()], - ) - .unwrap(); - let pending = receipt( - sandbox_id, - expectation, - 4, - SandboxActivationAttemptId::new(), - ); - db::insert_pending_receipt(&conn, pending).unwrap(); - let next_generation = ExecutionGeneration::new(5).unwrap(); - let next_attempt = SandboxActivationAttemptId::new(); - - let episode = crate::db::begin_activation_with_route_evidence( - &conn, - &sandbox_id.to_string(), - crate::state::SandboxState::Created, - crate::db::ActivationKind::ColdStart, - Some(next_generation), - Some(next_attempt), - crate::db::ActivationRouteAdmission { - auto_lifecycle: None, - operation_activation_attempt_id: None, - evidence: Some(RouteEvidence { - execution_generation: next_generation, - activation_attempt_id: next_attempt, - substrate: NodeManagedBy::Kubernetes, - provenance: RouteIdentityProvenance::ClusterFenced, - }), - open_claim: None, - }, - ) - .unwrap(); - - assert!(episode.is_some()); - assert!( - db::pending_receipt_for_sandbox(&conn, sandbox_id) - .unwrap() - .is_none(), - "the activation claim must retire stale cleanup evidence atomically" - ); -} - -#[test] -fn beginning_an_execution_cannot_supersede_a_current_pending_receipt() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - conn.execute( - "UPDATE sandboxes SET state = 'created' WHERE id = ?1", - [sandbox_id.to_string()], - ) - .unwrap(); - let generation = ExecutionGeneration::new(5).unwrap(); - let pending = receipt( - sandbox_id, - expectation, - generation.get(), - SandboxActivationAttemptId::new(), - ); - db::insert_pending_receipt(&conn, pending).unwrap(); - - let episode = crate::db::begin_activation_with_route_evidence( - &conn, - &sandbox_id.to_string(), - crate::state::SandboxState::Created, - crate::db::ActivationKind::ColdStart, - Some(generation), - Some(SandboxActivationAttemptId::new()), - crate::db::ActivationRouteAdmission::default(), - ) - .unwrap(); - - assert!(episode.is_none()); - assert_eq!( - crate::db::get_sandbox(&conn, &sandbox_id.to_string()) - .unwrap() - .unwrap() - .state, - crate::state::SandboxState::Created - ); - assert_eq!( - db::pending_receipt_for_sandbox(&conn, sandbox_id).unwrap(), - Some(pending) - ); -} - -#[test] -fn generation_writers_discard_an_older_pending_receipt_without_complete_route_evidence() { - let conn = crate::db::test_conn(); - - let route_sandbox_id = SandboxId::new(); - let route_expectation = insert_sandbox(&conn, route_sandbox_id); - let route_pending = receipt( - route_sandbox_id, - route_expectation, - 4, - SandboxActivationAttemptId::new(), - ); - db::insert_pending_receipt(&conn, route_pending).unwrap(); - crate::db::update_space_api_route_identity( - &conn, - route_sandbox_id, - &SpaceApiRouteIdentity { - execution_generation: Some(ExecutionGeneration::new(5).unwrap()), - ..SpaceApiRouteIdentity::default() - }, - RouteIdentityProvenance::Unfenced, - ) - .unwrap(); - assert!( - db::pending_receipt_for_sandbox(&conn, route_sandbox_id) - .unwrap() - .is_none() - ); - - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let pending = receipt( - sandbox_id, - expectation, - 4, - SandboxActivationAttemptId::new(), - ); - db::insert_pending_receipt(&conn, pending).unwrap(); - crate::db::update_runtime_fields( - &conn, - sandbox_id, - crate::db::RuntimeFields { - uid: 10_000, - slot: 7, - guest_cid: 37, - pid: std::process::id(), - spaced_host_pid: 10_001, - netgtw_pid: 10_002, - vsock_uds_path: "/space/run/firecracker/sandboxes/test/root/v.sock", - api_socket_path: "/space/run/firecracker/sandboxes/test/api.sock", - spaced_endpoint_proof: None, - space_api_route_identity: SpaceApiRouteIdentity { - execution_generation: Some(ExecutionGeneration::new(5).unwrap()), - ..SpaceApiRouteIdentity::default() - }, - route_identity_provenance: RouteIdentityProvenance::Unfenced, - }, - ) - .unwrap(); - assert!( - db::pending_receipt_for_sandbox(&conn, sandbox_id) - .unwrap() - .is_none() - ); -} - -#[test] -fn completion_reports_a_typed_missing_expected_receipt() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let pending = receipt( - sandbox_id, - expectation, - 4, - SandboxActivationAttemptId::new(), - ); - db::insert_pending_receipt(&conn, pending).unwrap(); - conn.execute( - "UPDATE sandboxes - SET state = 'deleting', - cleanup_completion = 'delete_row', - cleanup_keep_filesystem = 0 - WHERE id = ?1", - [sandbox_id.to_string()], - ) - .unwrap(); - db::discard_pending_receipt_superseded_by_generation( - &conn, - sandbox_id, - ExecutionGeneration::new(5).unwrap(), - ) - .unwrap(); - - let error = crate::db::complete_row_intent_cleanup_with_receipt( - &conn, - &sandbox_id.to_string(), - crate::db::CleanupCompletion::DeleteRow, - None, - Some(pending), - ) - .unwrap_err(); - - assert!(matches!( - error, - crate::db::CompleteRowIntentCleanupError::ExpectedReceiptMissingOrChanged { - sandbox_id: observed, - } if observed == sandbox_id - )); -} - -#[test] -fn newer_route_generation_supersedes_receipt_completion() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let pending = receipt( - sandbox_id, - expectation, - 4, - SandboxActivationAttemptId::new(), - ); - db::insert_pending_receipt(&conn, pending).unwrap(); - conn.execute( - "UPDATE sandboxes - SET state = 'deleting', - cleanup_completion = 'delete_row', - cleanup_keep_filesystem = 0 - WHERE id = ?1", - [sandbox_id.to_string()], - ) - .unwrap(); - db::record_retained_route_evidence( - &conn, - sandbox_id, - RouteEvidence { - execution_generation: ExecutionGeneration::new(5).unwrap(), - activation_attempt_id: SandboxActivationAttemptId::new(), - substrate: NodeManagedBy::Kubernetes, - provenance: RouteIdentityProvenance::ClusterFenced, - }, - ) - .unwrap(); - db::discard_pending_receipt_superseded_by_generation( - &conn, - sandbox_id, - ExecutionGeneration::new(5).unwrap(), - ) - .unwrap(); - - assert_eq!( - crate::db::complete_row_intent_cleanup_with_receipt( - &conn, - &sandbox_id.to_string(), - crate::db::CleanupCompletion::DeleteRow, - None, - Some(pending), - ) - .unwrap(), - crate::db::CompleteRowIntentCleanupDisposition::Superseded, - ); - assert!( - crate::db::get_sandbox(&conn, &sandbox_id.to_string()) - .unwrap() - .is_some(), - "the superseded cleanup must not delete the replacement execution" - ); -} - -#[test] -fn newer_current_execution_generation_supersedes_receipt_completion() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let pending = receipt( - sandbox_id, - expectation, - 4, - SandboxActivationAttemptId::new(), - ); - db::insert_pending_receipt(&conn, pending).unwrap(); - assert!( - db::discard_pending_receipt_superseded_by_generation( - &conn, - sandbox_id, - ExecutionGeneration::new(5).unwrap(), - ) - .unwrap() - .is_some() - ); - crate::db::update_space_api_route_identity( - &conn, - sandbox_id, - &SpaceApiRouteIdentity { - execution_generation: Some(ExecutionGeneration::new(5).unwrap()), - ..SpaceApiRouteIdentity::default() - }, - RouteIdentityProvenance::Unfenced, - ) - .unwrap(); - conn.execute( - "UPDATE sandboxes - SET state = 'started', - cleanup_completion = NULL, - cleanup_keep_filesystem = NULL - WHERE id = ?1", - [sandbox_id.to_string()], - ) - .unwrap(); - - assert_eq!( - crate::db::complete_row_intent_cleanup_with_receipt( - &conn, - &sandbox_id.to_string(), - crate::db::CleanupCompletion::DeleteRow, - None, - Some(pending), - ) - .unwrap(), - crate::db::CompleteRowIntentCleanupDisposition::Superseded, - ); - assert!( - crate::db::get_sandbox(&conn, &sandbox_id.to_string()) - .unwrap() - .is_some(), - "the superseded cleanup must not delete the replacement execution" - ); - assert!( - db::pending_receipt_for_sandbox(&conn, sandbox_id) - .unwrap() - .is_none() - ); -} - -#[test] -fn equal_current_execution_generation_does_not_supersede_receipt_completion() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let generation = ExecutionGeneration::new(4).unwrap(); - crate::db::update_space_api_route_identity( - &conn, - sandbox_id, - &SpaceApiRouteIdentity { - execution_generation: Some(generation), - ..SpaceApiRouteIdentity::default() - }, - RouteIdentityProvenance::Unfenced, - ) - .unwrap(); - let pending = receipt( - sandbox_id, - expectation, - generation.get(), - SandboxActivationAttemptId::new(), - ); - db::insert_pending_receipt(&conn, pending).unwrap(); - conn.execute( - "UPDATE sandboxes - SET state = 'deleting', - cleanup_completion = 'delete_row', - cleanup_keep_filesystem = 0 - WHERE id = ?1", - [sandbox_id.to_string()], - ) - .unwrap(); - let mismatched = AutoLifecycleDeleteReceipt { - receipt_id: AutoLifecycleDeleteReceiptId::generate(), - ..pending - }; - - let error = crate::db::complete_row_intent_cleanup_with_receipt( - &conn, - &sandbox_id.to_string(), - crate::db::CleanupCompletion::DeleteRow, - None, - Some(mismatched), - ) - .unwrap_err(); - - assert!(matches!( - error, - crate::db::CompleteRowIntentCleanupError::ExpectedReceiptMissingOrChanged { - sandbox_id: observed, - } if observed == sandbox_id - )); - assert!( - crate::db::get_sandbox(&conn, &sandbox_id.to_string()) - .unwrap() - .is_some() - ); - assert_eq!( - db::pending_receipt_for_sandbox(&conn, sandbox_id).unwrap(), - Some(pending) - ); -} - -#[test] -fn equal_generation_retained_evidence_cannot_be_downgraded() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - insert_sandbox(&conn, sandbox_id); - let first_attempt = SandboxActivationAttemptId::new(); - let replacement_attempt = SandboxActivationAttemptId::new(); - db::record_retained_route_evidence( - &conn, - sandbox_id, - RouteEvidence { - execution_generation: ExecutionGeneration::new(4).unwrap(), - activation_attempt_id: first_attempt, - substrate: NodeManagedBy::Kubernetes, - provenance: RouteIdentityProvenance::ClusterFenced, - }, - ) - .unwrap(); - - assert!( - !db::record_retained_route_evidence( - &conn, - sandbox_id, - RouteEvidence { - execution_generation: ExecutionGeneration::new(4).unwrap(), - activation_attempt_id: replacement_attempt, - substrate: NodeManagedBy::Kubernetes, - provenance: RouteIdentityProvenance::Unfenced, - }, - ) - .unwrap() - ); - assert_eq!( - conn.query_row( - "SELECT last_observed_route_activation_id, - last_observed_route_identity_provenance - FROM sandboxes WHERE id = ?1", - [sandbox_id.to_string()], - |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), - ) - .unwrap(), - (first_attempt.to_string(), "cluster_fenced".to_string()) - ); - - assert!( - db::record_retained_route_evidence( - &conn, - sandbox_id, - RouteEvidence { - execution_generation: ExecutionGeneration::new(4).unwrap(), - activation_attempt_id: replacement_attempt, - substrate: NodeManagedBy::Kubernetes, - provenance: RouteIdentityProvenance::ClusterFenced, - }, - ) - .unwrap() - ); - assert_eq!( - conn.query_row( - "SELECT last_observed_route_activation_id, - last_observed_route_identity_provenance - FROM sandboxes WHERE id = ?1", - [sandbox_id.to_string()], - |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), - ) - .unwrap(), - ( - replacement_attempt.to_string(), - "cluster_fenced".to_string() - ) - ); - - assert!( - !db::record_retained_route_evidence( - &conn, - sandbox_id, - RouteEvidence { - execution_generation: ExecutionGeneration::new(4).unwrap(), - activation_attempt_id: first_attempt, - substrate: NodeManagedBy::Kubernetes, - provenance: RouteIdentityProvenance::ClusterFenced, - }, - ) - .unwrap(), - "delayed same-generation evidence must not replace the newer attempt" - ); - assert_eq!( - conn.query_row( - "SELECT last_observed_route_activation_id - FROM sandboxes WHERE id = ?1", - [sandbox_id.to_string()], - |row| row.get::<_, String>(0), - ) - .unwrap(), - replacement_attempt.to_string() - ); -} - -#[test] -fn receipt_conflict_uses_bounded_retry_without_claiming_a_durable_block() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - insert_sandbox(&conn, sandbox_id); - - let storage_id = sandbox_id.to_string(); - assert!(db::record_receipt_conflict_retry(&conn, &storage_id).unwrap()); - assert!(!db::record_receipt_conflict_retry(&conn, &storage_id).unwrap()); - - let (block_reason, retry_at, runtime_state_json): (Option, i64, String) = conn - .query_row( - "SELECT auto_lifecycle_delete_block_reason, - next_auto_event_at, - auto_lifecycle_state_json - FROM sandboxes - WHERE id = ?1", - [sandbox_id.to_string()], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), - ) - .unwrap(); - let runtime_state: pplx_space_common_core::types::AutoLifecycleRuntimeState = - serde_json::from_str(&runtime_state_json).unwrap(); - assert!(block_reason.is_none()); - assert!(retry_at > crate::db::now_epoch_i64().unwrap()); - assert_eq!( - runtime_state.next_auto_event_at, - Some(u64::try_from(retry_at).unwrap()) - ); - assert!( - runtime_state - .last_auto_error - .as_deref() - .is_some_and(|error| error.contains("conflicting receipt evidence")) - ); -} - -#[test] -fn policy_replacement_retires_obsolete_delete_block() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - - assert!(matches!( - db::classify_and_record_admission( - &conn, - sandbox_id, - Some(NodeRole::Worker), - None, - expectation, - ) - .unwrap(), - AutoLifecycleDeleteAdmission::Blocked { - reason: AutoLifecycleDeleteBlockReason::MissingRouteIdentity, - .. - } - )); - - crate::db::update_auto_lifecycle( - &conn, - &sandbox_id.to_string(), - &AutoLifecyclePolicy::disabled(), - PolicyAuthority::Cluster, - ) - .unwrap(); - - let (reason, blocked_at): (Option, Option) = conn - .query_row( - "SELECT auto_lifecycle_delete_block_reason, - auto_lifecycle_delete_blocked_at - FROM sandboxes - WHERE id = ?1", - [sandbox_id.to_string()], - |row| Ok((row.get(0)?, row.get(1)?)), - ) - .unwrap(); - assert_eq!((reason, blocked_at), (None, None)); -} - -#[test] -fn unknown_substrate_is_durably_blocked() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - conn.execute( - "UPDATE sandboxes - SET space_api_route_execution_generation = 7, - space_api_route_activation_attempt_id = ?1, - space_api_route_substrate_kind = 'future_substrate', - space_api_route_identity_provenance = 'cluster_fenced' - WHERE id = ?2", - rusqlite::params![ - SandboxActivationAttemptId::new().to_string(), - sandbox_id.to_string() - ], - ) - .unwrap(); - - let admission = db::classify_and_record_admission( - &conn, - sandbox_id, - Some(NodeRole::Worker), - None, - expectation, - ) - .expect("unknown substrate must fail closed as a durable block"); - let AutoLifecycleDeleteAdmission::Blocked { reason, changed } = admission else { - panic!("unknown substrate must not admit deletion: {admission:?}"); - }; - assert_eq!(reason.as_str(), "unknown_substrate"); - assert!(changed); -} diff --git a/crates/libs/sbx/mgr/server/src/db.rs b/crates/libs/sbx/mgr/server/src/db.rs index d05965bf9..9ca82fb87 100644 --- a/crates/libs/sbx/mgr/server/src/db.rs +++ b/crates/libs/sbx/mgr/server/src/db.rs @@ -32,12 +32,18 @@ use rusqlite::types::Type; use pplx_space_common_core::sql; use rusqlite::{Connection, OptionalExtension, Result, params}; +pub(crate) mod auto_lifecycle_delete_receipts; pub(crate) mod cleanup_jobs; pub(crate) mod cleanup_target_scan; +pub(crate) mod eviction_scan; +pub(crate) mod guest_health_scan; pub(crate) mod inventory; pub(crate) mod inventory_read; +pub(crate) mod local_artifact_leases; pub(crate) mod quarantine_history; +pub(crate) mod route_activation_claims; pub(crate) mod snapshot_gc; +pub(crate) mod snapshot_upload_jobs; pub(crate) mod timestamps; pub(crate) mod transfer; @@ -51,6 +57,7 @@ const DB_PATH: &str = pplx_space_common_core::host::linux::paths::SANDBOX_MANAGE mod quarantine_scan; mod sandbox_row; mod teardown_jobs; +use auto_lifecycle_delete_receipts::discard_pending_receipt_for_execution_advance; pub(crate) use teardown_jobs::{ TeardownSuspendFailure, defer_teardown_suspend_retry, fail_teardown_suspend_backup_operation, fail_teardown_suspend_operation, reconcile_terminal_teardown_suspend_jobs, @@ -307,7 +314,7 @@ fn emit_sandbox_quarantined_event( fn emit_auto_lifecycle_delete_receipt_superseded( sandbox_id: SandboxId, - superseded_receipt: crate::auto_lifecycle_delete_receipts::db::SupersededPendingReceipt, + superseded_receipt: crate::db::auto_lifecycle_delete_receipts::SupersededPendingReceipt, current_generation: ExecutionGeneration, ) { emit_sandbox_event( @@ -328,7 +335,7 @@ fn emit_auto_lifecycle_delete_receipt_superseded( pub(crate) fn emit_auto_lifecycle_delete_receipt_superseded_if_present( sandbox_id: SandboxId, - supersession: Option, + supersession: Option, ) { if let Some(supersession) = supersession { emit_auto_lifecycle_delete_receipt_superseded( @@ -345,7 +352,7 @@ pub(crate) fn emit_auto_lifecycle_delete_receipt_superseded_if_present( /// add new child tables here, not at the individual delete call sites. fn delete_sandbox_child_rows(tx: &Connection, id: &str) -> Result<()> { teardown_jobs::fail_active(tx, id, TeardownSuspendFailure::Deleted)?; - crate::route_activation_claims::close_claim(tx, id)?; + crate::db::route_activation_claims::close_claim(tx, id)?; tx.execute( "DELETE FROM sandbox_cleanup_episodes WHERE sandbox_id = ?1", params![id], diff --git a/crates/libs/sbx/mgr/server/src/db/auto_lifecycle.rs b/crates/libs/sbx/mgr/server/src/db/auto_lifecycle.rs index ecf8f066d..a5e189c58 100644 --- a/crates/libs/sbx/mgr/server/src/db/auto_lifecycle.rs +++ b/crates/libs/sbx/mgr/server/src/db/auto_lifecycle.rs @@ -270,7 +270,7 @@ pub fn update_auto_lifecycle_with_state( /// /// One reader moves the eager way. The disk-pressure eviction ages a /// `suspended` row with no activity clocks by `updated_at` -/// (`sandbox_eviction::IDLE_WITNESS_SQL`), so such a row whose stage keeps +/// (`db::eviction_scan::IDLE_WITNESS_SQL`), so such a row whose stage keeps /// failing identically now ages from its last real change rather than its /// last repeated record, and can become a candidate sooner than before. That /// is the intended meaning of idle, and the eviction's backup proof and claim diff --git a/crates/libs/sbx/mgr/server/src/auto_lifecycle_delete_receipts/db.rs b/crates/libs/sbx/mgr/server/src/db/auto_lifecycle_delete_receipts.rs similarity index 99% rename from crates/libs/sbx/mgr/server/src/auto_lifecycle_delete_receipts/db.rs rename to crates/libs/sbx/mgr/server/src/db/auto_lifecycle_delete_receipts.rs index 6a0da4f10..83a577845 100644 --- a/crates/libs/sbx/mgr/server/src/auto_lifecycle_delete_receipts/db.rs +++ b/crates/libs/sbx/mgr/server/src/db/auto_lifecycle_delete_receipts.rs @@ -4,12 +4,10 @@ use pplx_space_common_core::types::{ AutoLifecycleDeleteReceiptId, ExecutionGeneration, NodeManagedBy, NodeRole, PolicyAuthority, SandboxActivationAttemptId, SandboxId, }; -use pplx_space_common_svc::sql::sqlite::SqliteU64; -use pplx_space_common_svc::types::AutoLifecyclePolicyFingerprint; -use rusqlite::types::Type; -use rusqlite::{Connection, OptionalExtension, params}; +use pplx_space_common_svc::{sql::sqlite::SqliteU64, types::AutoLifecyclePolicyFingerprint}; +use rusqlite::{Connection, OptionalExtension, params, types::Type}; -use super::{ +use crate::auto_lifecycle_delete_receipts::{ AutoLifecycleDeleteAdmission, AutoLifecycleDeleteBlockReason, AutoLifecycleDeleteBlockedStats, AutoLifecycleDeleteOrdinaryReason, AutoLifecycleDeleteOutboxState, AutoLifecycleDeleteOutboxStats, AutoLifecycleDeleteReceipt, RouteEvidence, @@ -183,7 +181,7 @@ pub(crate) fn classify_and_record_admission( sandbox_id: SandboxId, node_role: Option, node_managed_by: Option, - expectation: super::AutoLifecycleDeleteExpectation, + expectation: crate::auto_lifecycle_delete_receipts::AutoLifecycleDeleteExpectation, ) -> Result { let values = conn .query_row( @@ -1229,3 +1227,6 @@ where rusqlite::Error::FromSqlConversionFailure(column, Type::Text, Box::new(error)) }) } + +#[cfg(test)] +mod tests; diff --git a/crates/libs/sbx/mgr/server/src/db/auto_lifecycle_delete_receipts/tests.rs b/crates/libs/sbx/mgr/server/src/db/auto_lifecycle_delete_receipts/tests.rs new file mode 100644 index 000000000..650b8ea56 --- /dev/null +++ b/crates/libs/sbx/mgr/server/src/db/auto_lifecycle_delete_receipts/tests.rs @@ -0,0 +1,1962 @@ +use std::collections::BTreeMap; + +use pplx_space_common_core::types::{ + AutoLifecycleConditionSet, AutoLifecycleConditionSetId, AutoLifecycleDeleteReceiptId, + AutoLifecycleIdleAction, AutoLifecyclePolicy, AutoLifecycleStage, AutoLifecycleWakePolicy, + ExecutionGeneration, NodeManagedBy, NodeRole, PolicyAuthority, RuntimeKind, + SandboxActivationAttemptId, SandboxId, SandboxOwner, SandboxRuntimeLabel, + SpaceApiRouteIdentity, +}; +use pplx_space_common_svc::types::AutoLifecyclePolicyFingerprint; +use rusqlite::{Connection, OptionalExtension}; + +use super::*; +use crate::auto_lifecycle_delete_receipts::*; + +fn delete_policy() -> AutoLifecyclePolicy { + AutoLifecyclePolicy { + wake: AutoLifecycleWakePolicy::default(), + condition_sets: [( + AutoLifecycleConditionSetId::AccessIdle, + AutoLifecycleConditionSet::access_idle(), + )] + .into(), + stages: vec![AutoLifecycleStage { + condition_set: AutoLifecycleConditionSetId::AccessIdle, + at: 900, + action: AutoLifecycleIdleAction::Delete, + }], + } +} + +fn insert_sandbox(conn: &Connection, sandbox_id: SandboxId) -> AutoLifecycleDeleteExpectation { + let owner = SandboxOwner::parse("user").unwrap(); + let runtime_label = SandboxRuntimeLabel::from_sandbox_id(sandbox_id); + let policy = delete_policy(); + crate::db::insert_sandbox( + conn, + crate::db::InsertSandboxParams { + id: &sandbox_id.to_string(), + uuid8: &runtime_label, + state: crate::state::SandboxState::Started, + template: "seed", + template_hash: Some("hash"), + user: &owner, + runtime: RuntimeKind::Firecracker.as_str(), + snapshots_enabled: false, + rootfs_size_mb: None, + disk_io_limits: Default::default(), + pool_template: None, + volume_encryption: false, + process_env: &BTreeMap::new(), + auto_lifecycle: &policy, + network_policy_override: None, + network_policy_generation: 0, + network_policy_authority: PolicyAuthority::Cluster, + auto_lifecycle_authority: PolicyAuthority::Cluster, + }, + ) + .unwrap(); + AutoLifecycleDeleteExpectation { + policy_fingerprint: AutoLifecyclePolicyFingerprint::for_policy(&policy), + authority: PolicyAuthority::Cluster, + } +} + +fn cluster_route( + generation: i64, + activation_attempt_id: SandboxActivationAttemptId, +) -> SpaceApiRouteIdentity { + SpaceApiRouteIdentity { + execution_generation: Some(ExecutionGeneration::new(generation).unwrap()), + activation_attempt_id: Some(activation_attempt_id), + substrate_kind: Some(NodeManagedBy::Kubernetes), + ..SpaceApiRouteIdentity::default() + } +} + +fn receipt( + sandbox_id: SandboxId, + expectation: AutoLifecycleDeleteExpectation, + generation: i64, + activation_attempt_id: SandboxActivationAttemptId, +) -> AutoLifecycleDeleteReceipt { + AutoLifecycleDeleteReceipt { + receipt_id: AutoLifecycleDeleteReceiptId::generate(), + sandbox_id, + execution_generation: ExecutionGeneration::new(generation).unwrap(), + route_activation_id: activation_attempt_id, + policy_fingerprint: expectation.policy_fingerprint, + } +} + +#[test] +fn delete_admission_requires_consistent_authority_and_execution_evidence() { + use AutoLifecycleDeleteBlockReason as Block; + use AutoLifecycleDeleteOrdinaryReason as Ordinary; + + #[derive(Clone, Copy, Debug)] + enum Expected { + Ordinary(Ordinary), + Receipt, + Blocked(Block), + } + let cases = [ + ( + "local", + Some(NodeManagedBy::Space), + PolicyAuthority::Local, + "UPDATE sandboxes SET space_api_route_substrate_kind = 'k8s' WHERE id = ?3", + Expected::Ordinary(Ordinary::LocalAuthority), + ), + ( + "route-less direct", + Some(NodeManagedBy::Space), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET state = 'stopped' WHERE id = ?3", + Expected::Ordinary(Ordinary::DirectUnsupported), + ), + ( + "explicit direct", + None, + PolicyAuthority::Cluster, + "UPDATE sandboxes SET space_api_route_substrate_kind = 'space' WHERE id = ?3", + Expected::Ordinary(Ordinary::DirectUnsupported), + ), + ( + "current", + Some(NodeManagedBy::Kubernetes), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET space_api_route_execution_generation = 7, space_api_route_activation_attempt_id = ?1, space_api_route_substrate_kind = 'k8s', space_api_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", + Expected::Receipt, + ), + ( + "retained", + Some(NodeManagedBy::Kubernetes), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET last_observed_execution_generation = 7, last_observed_route_activation_id = ?1, last_observed_route_substrate_kind = 'k8s', last_observed_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", + Expected::Receipt, + ), + ( + "suspended subset", + Some(NodeManagedBy::Kubernetes), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET state = 'suspended', space_api_route_execution_generation = 7, last_observed_execution_generation = 7, last_observed_route_activation_id = ?1, last_observed_route_substrate_kind = 'k8s', last_observed_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", + Expected::Receipt, + ), + ( + "missing Kubernetes", + Some(NodeManagedBy::Kubernetes), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET state = 'stopped' WHERE id = ?3", + Expected::Blocked(Block::MissingRouteIdentity), + ), + ( + "missing unknown host", + None, + PolicyAuthority::Cluster, + "UPDATE sandboxes SET state = 'stopped' WHERE id = ?3", + Expected::Blocked(Block::MissingRouteIdentity), + ), + ( + "stale retained", + Some(NodeManagedBy::Kubernetes), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET space_api_route_execution_generation = 8, space_api_route_activation_attempt_id = ?1, space_api_route_substrate_kind = 'k8s', space_api_route_identity_provenance = 'cluster_fenced', last_observed_execution_generation = 7, last_observed_route_activation_id = ?1, last_observed_route_substrate_kind = 'k8s', last_observed_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", + Expected::Blocked(Block::ConflictingRouteIdentity), + ), + ( + "conflicting attempts", + Some(NodeManagedBy::Kubernetes), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET space_api_route_execution_generation = 7, space_api_route_activation_attempt_id = ?1, space_api_route_substrate_kind = 'k8s', space_api_route_identity_provenance = 'cluster_fenced', last_observed_execution_generation = 7, last_observed_route_activation_id = ?2, last_observed_route_substrate_kind = 'k8s', last_observed_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", + Expected::Blocked(Block::ConflictingRouteIdentity), + ), + ( + "host contradicts route", + Some(NodeManagedBy::Space), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET space_api_route_substrate_kind = 'k8s' WHERE id = ?3", + Expected::Blocked(Block::ConflictingRouteIdentity), + ), + ( + "malformed current attempt", + Some(NodeManagedBy::Kubernetes), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET space_api_route_execution_generation = 7, space_api_route_activation_attempt_id = 'malformed', space_api_route_substrate_kind = 'k8s', space_api_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", + Expected::Blocked(Block::ConflictingRouteIdentity), + ), + ( + "malformed retained attempt", + Some(NodeManagedBy::Kubernetes), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET space_api_route_execution_generation = 7, space_api_route_activation_attempt_id = ?1, space_api_route_substrate_kind = 'k8s', space_api_route_identity_provenance = 'cluster_fenced', last_observed_execution_generation = 7, last_observed_route_activation_id = 'malformed', last_observed_route_substrate_kind = 'k8s', last_observed_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", + Expected::Blocked(Block::ConflictingRouteIdentity), + ), + ( + "invalid current generation", + Some(NodeManagedBy::Kubernetes), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET space_api_route_execution_generation = -1, space_api_route_activation_attempt_id = ?1, space_api_route_substrate_kind = 'k8s', space_api_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", + Expected::Blocked(Block::ConflictingRouteIdentity), + ), + ( + "invalid retained generation", + Some(NodeManagedBy::Kubernetes), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET last_observed_execution_generation = 0, last_observed_route_activation_id = ?1, last_observed_route_substrate_kind = 'k8s', last_observed_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", + Expected::Blocked(Block::ConflictingRouteIdentity), + ), + ( + "mixed substrates", + None, + PolicyAuthority::Cluster, + "UPDATE sandboxes SET space_api_route_substrate_kind = 'space', last_observed_route_substrate_kind = 'k8s' WHERE id = ?3", + Expected::Blocked(Block::ConflictingRouteIdentity), + ), + ( + "unknown substrate", + Some(NodeManagedBy::Space), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET last_observed_route_substrate_kind = 'future' WHERE id = ?3", + Expected::Blocked(Block::UnknownSubstrate), + ), + ( + "partial retained", + Some(NodeManagedBy::Kubernetes), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET space_api_route_execution_generation = 7, space_api_route_activation_attempt_id = ?1, space_api_route_substrate_kind = 'k8s', space_api_route_identity_provenance = 'cluster_fenced', last_observed_execution_generation = 7 WHERE id = ?3", + Expected::Blocked(Block::ConflictingRouteIdentity), + ), + ( + "pod contradicts direct", + Some(NodeManagedBy::Space), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET space_api_route_pod_uid = 'pod' WHERE id = ?3", + Expected::Blocked(Block::ConflictingRouteIdentity), + ), + ( + "retained placement contradicts direct", + Some(NodeManagedBy::Space), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET last_observed_route_generation_substrate_kind = 'k8s' WHERE id = ?3", + Expected::Blocked(Block::ConflictingRouteIdentity), + ), + ]; + for (name, host, authority, setup, expected) in cases { + let conn = crate::db::test_conn(); + let id = SandboxId::new(); + let mut expectation = insert_sandbox(&conn, id); + expectation.authority = authority; + let attempt = SandboxActivationAttemptId::new(); + // Seed historical malformed evidence without weakening admission writes. + conn.pragma_update(None, "ignore_check_constraints", true) + .unwrap(); + conn.execute( + setup, + rusqlite::params![ + attempt.to_string(), + SandboxActivationAttemptId::new().to_string(), + id.to_string() + ], + ) + .unwrap(); + conn.pragma_update(None, "ignore_check_constraints", false) + .unwrap(); + conn.execute( + "UPDATE sandboxes SET auto_lifecycle_authority = ?1 WHERE id = ?2", + rusqlite::params![authority.to_string(), id.to_string()], + ) + .unwrap(); + let admitted = + classify_and_record_admission(&conn, id, Some(NodeRole::Worker), host, expectation) + .unwrap(); + match (expected, admitted) { + (Expected::Ordinary(expected), AutoLifecycleDeleteAdmission::Ordinary(actual)) => { + assert_eq!(actual, expected, "{name}") + } + (Expected::Receipt, AutoLifecycleDeleteAdmission::Receipt(evidence)) => { + assert_eq!(evidence.execution_generation.get(), 7, "{name}"); + assert_eq!(evidence.activation_attempt_id, attempt, "{name}"); + } + (Expected::Blocked(expected), AutoLifecycleDeleteAdmission::Blocked { reason, .. }) => { + assert_eq!(reason, expected, "{name}"); + let retry: i64 = conn + .query_row( + "SELECT next_auto_event_at FROM sandboxes WHERE id = ?1", + [id.to_string()], + |row| row.get(0), + ) + .unwrap(); + assert!(retry > crate::db::now_epoch_i64().unwrap(), "{name}"); + assert!(crate::db::sandbox_exists(&conn, &id.to_string()).unwrap()); + } + pair => panic!("unexpected {name}: {pair:?}"), + } + } + for completed in [false, true] { + let conn = crate::db::test_conn(); + let id = SandboxId::new(); + let expectation = insert_sandbox(&conn, id); + let pending = receipt(id, expectation, 7, SandboxActivationAttemptId::new()); + insert_pending_receipt(&conn, pending).unwrap(); + if completed { + complete_pending_receipt(&conn, id, pending.receipt_id).unwrap(); + } + let admitted = classify_and_record_admission( + &conn, + id, + Some(NodeRole::Worker), + Some(NodeManagedBy::Space), + expectation, + ) + .unwrap(); + if completed { + assert_eq!( + admitted, + AutoLifecycleDeleteAdmission::Ordinary(Ordinary::DirectUnsupported) + ); + } else { + assert!(matches!( + admitted, + AutoLifecycleDeleteAdmission::Blocked { + reason: Block::ConflictingRouteIdentity, + .. + } + )); + } + assert_eq!( + conn.query_row( + "SELECT receipt_id, state FROM auto_lifecycle_delete_outbox WHERE sandbox_id = ?1", + [id.to_string()], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ) + .unwrap(), + ( + pending.receipt_id.to_string(), + if completed { "completed" } else { "pending" }.into() + ) + ); + } +} + +#[test] +fn strict_admission_requires_complete_current_cluster_fencing() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let activation_attempt_id = SandboxActivationAttemptId::new(); + crate::db::update_space_api_route_identity( + &conn, + sandbox_id, + &cluster_route(7, activation_attempt_id), + RouteIdentityProvenance::ClusterFenced, + ) + .unwrap(); + + assert!(matches!( + classify_and_record_admission( + &conn, + sandbox_id, + Some(NodeRole::Worker), + None, expectation, + ) + .unwrap(), + AutoLifecycleDeleteAdmission::Receipt(RouteEvidence { + execution_generation, + activation_attempt_id: observed_attempt, + .. + }) if execution_generation == ExecutionGeneration::new(7).unwrap() + && observed_attempt == activation_attempt_id + )); + + conn.execute( + "UPDATE sandboxes + SET space_api_route_execution_generation = NULL, + space_api_route_activation_attempt_id = NULL, + space_api_route_substrate_kind = NULL, + space_api_route_identity_provenance = NULL + WHERE id = ?1", + [sandbox_id.to_string()], + ) + .unwrap(); + assert!(matches!( + classify_and_record_admission(&conn, sandbox_id, Some(NodeRole::Worker), None, expectation,) + .unwrap(), + AutoLifecycleDeleteAdmission::Receipt(RouteEvidence { + execution_generation, + activation_attempt_id: observed_attempt, + .. + }) if execution_generation == ExecutionGeneration::new(7).unwrap() + && observed_attempt == activation_attempt_id + )); + + conn.execute( + "UPDATE sandboxes + SET space_api_route_execution_generation = 8, + space_api_route_activation_attempt_id = NULL, + space_api_route_substrate_kind = NULL, + space_api_route_identity_provenance = NULL + WHERE id = ?1", + [sandbox_id.to_string()], + ) + .unwrap(); + assert!(matches!( + classify_and_record_admission( + &conn, + sandbox_id, + Some(NodeRole::Worker), + None, + expectation, + ) + .unwrap(), + AutoLifecycleDeleteAdmission::Blocked { + reason: AutoLifecycleDeleteBlockReason::MissingSubstrate, + changed: true, + } + )); + let (reason, blocked_at, next_auto_event_at, runtime_state_json): ( + Option, + Option, + Option, + String, + ) = conn + .query_row( + "SELECT auto_lifecycle_delete_block_reason, + auto_lifecycle_delete_blocked_at, + next_auto_event_at, + auto_lifecycle_state_json + FROM sandboxes WHERE id = ?1", + [sandbox_id.to_string()], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ) + .unwrap(); + assert_eq!(reason.as_deref(), Some("missing_substrate")); + assert!(blocked_at.is_some()); + assert!(next_auto_event_at.unwrap() > crate::db::now_epoch_i64().unwrap()); + let runtime_state: pplx_space_common_core::types::AutoLifecycleRuntimeState = + serde_json::from_str(&runtime_state_json).unwrap(); + assert!( + runtime_state + .last_auto_error + .as_deref() + .is_some_and(|error| error.contains("missing_substrate")) + ); + + crate::db::update_space_api_route_identity( + &conn, + sandbox_id, + &cluster_route(8, SandboxActivationAttemptId::new()), + RouteIdentityProvenance::ClusterFenced, + ) + .unwrap(); + let (reason, blocked_at, runtime_state_json): (Option, Option, String) = conn + .query_row( + "SELECT auto_lifecycle_delete_block_reason, + auto_lifecycle_delete_blocked_at, + auto_lifecycle_state_json + FROM sandboxes WHERE id = ?1", + [sandbox_id.to_string()], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + assert_eq!((reason, blocked_at), (None, None)); + let runtime_state: pplx_space_common_core::types::AutoLifecycleRuntimeState = + serde_json::from_str(&runtime_state_json).unwrap(); + assert_eq!(runtime_state.last_auto_error, None); +} + +#[test] +fn dormant_suspend_identity_uses_matching_retained_route_evidence() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let activation_attempt_id = SandboxActivationAttemptId::new(); + crate::db::update_space_api_route_identity( + &conn, + sandbox_id, + &cluster_route(7, activation_attempt_id), + RouteIdentityProvenance::ClusterFenced, + ) + .unwrap(); + + // Suspend completion retains generation for central publication but clears + // the remaining current route fields. The matching complete retained tuple + // is still the exact route that produced this dormant sandbox. + conn.execute( + "UPDATE sandboxes + SET state = 'suspended', + space_api_route_activation_attempt_id = NULL, + space_api_route_substrate_kind = NULL, + space_api_route_identity_provenance = NULL + WHERE id = ?1", + [sandbox_id.to_string()], + ) + .unwrap(); + + assert!(matches!( + classify_and_record_admission( + &conn, + sandbox_id, + Some(NodeRole::Worker), + None, expectation, + ) + .unwrap(), + AutoLifecycleDeleteAdmission::Receipt(RouteEvidence { + execution_generation, + activation_attempt_id: observed_attempt, + .. + }) if execution_generation == ExecutionGeneration::new(7).unwrap() + && observed_attempt == activation_attempt_id + )); +} + +#[test] +fn non_dormant_partial_route_does_not_reuse_retained_evidence() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let activation_attempt_id = SandboxActivationAttemptId::new(); + crate::db::update_space_api_route_identity( + &conn, + sandbox_id, + &cluster_route(7, activation_attempt_id), + RouteIdentityProvenance::ClusterFenced, + ) + .unwrap(); + + conn.execute( + "UPDATE sandboxes + SET space_api_route_activation_attempt_id = NULL, + space_api_route_substrate_kind = NULL, + space_api_route_identity_provenance = NULL + WHERE id = ?1", + [sandbox_id.to_string()], + ) + .unwrap(); + + assert!(matches!( + classify_and_record_admission( + &conn, + sandbox_id, + Some(NodeRole::Worker), + None, + expectation, + ) + .unwrap(), + AutoLifecycleDeleteAdmission::Blocked { + reason: AutoLifecycleDeleteBlockReason::MissingSubstrate, + changed: true, + } + )); +} + +#[test] +fn contradictory_current_and_retained_routes_block_deletion() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let retained_attempt = SandboxActivationAttemptId::new(); + crate::db::update_space_api_route_identity( + &conn, + sandbox_id, + &cluster_route(8, retained_attempt), + RouteIdentityProvenance::ClusterFenced, + ) + .unwrap(); + + // Model a stale or corrupt current-route writer without updating the + // retained tuple. Receipt admission must not choose either side of the + // contradiction and delete local state that central fencing will retain. + conn.execute( + "UPDATE sandboxes + SET space_api_route_execution_generation = 7, + space_api_route_activation_attempt_id = ?1, + space_api_route_substrate_kind = 'k8s', + space_api_route_identity_provenance = 'cluster_fenced' + WHERE id = ?2", + rusqlite::params![ + SandboxActivationAttemptId::new().to_string(), + sandbox_id.to_string() + ], + ) + .unwrap(); + + assert!(matches!( + classify_and_record_admission( + &conn, + sandbox_id, + Some(NodeRole::Worker), + None, expectation, + ) + .unwrap(), + AutoLifecycleDeleteAdmission::Blocked { reason, changed: true } + if reason.as_str() == "conflicting_route_identity" + )); +} + +#[test] +fn later_current_attempt_blocks_receipt_admission() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let retained_attempt = SandboxActivationAttemptId::new(); + crate::db::update_space_api_route_identity( + &conn, + sandbox_id, + &cluster_route(8, retained_attempt), + RouteIdentityProvenance::ClusterFenced, + ) + .unwrap(); + + let current_attempt = SandboxActivationAttemptId::new_after(retained_attempt).unwrap(); + conn.execute( + "UPDATE sandboxes + SET space_api_route_activation_attempt_id = ?1 + WHERE id = ?2", + rusqlite::params![current_attempt.to_string(), sandbox_id.to_string()], + ) + .unwrap(); + + assert!(matches!( + classify_and_record_admission( + &conn, + sandbox_id, + Some(NodeRole::Worker), + None, expectation, + ) + .unwrap(), + AutoLifecycleDeleteAdmission::Blocked { reason, changed: true } + if reason == AutoLifecycleDeleteBlockReason::ConflictingRouteIdentity + )); +} + +#[test] +fn delayed_current_attempt_blocks_receipt_admission() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let delayed_attempt = SandboxActivationAttemptId::new(); + let retained_attempt = SandboxActivationAttemptId::new_after(delayed_attempt).unwrap(); + crate::db::update_space_api_route_identity( + &conn, + sandbox_id, + &cluster_route(8, retained_attempt), + RouteIdentityProvenance::ClusterFenced, + ) + .unwrap(); + + conn.execute( + "UPDATE sandboxes + SET space_api_route_activation_attempt_id = ?1 + WHERE id = ?2", + rusqlite::params![delayed_attempt.to_string(), sandbox_id.to_string()], + ) + .unwrap(); + + assert!(matches!( + classify_and_record_admission( + &conn, + sandbox_id, + Some(NodeRole::Worker), + None, expectation, + ) + .unwrap(), + AutoLifecycleDeleteAdmission::Blocked { reason, changed: true } + if reason == AutoLifecycleDeleteBlockReason::ConflictingRouteIdentity + )); +} + +#[test] +fn receipt_eligibility_excludes_non_worker_and_non_kubernetes_populations() { + let conn = crate::db::test_conn(); + + let direct_id = SandboxId::new(); + let direct_expectation = insert_sandbox(&conn, direct_id); + let direct_identity = SpaceApiRouteIdentity { + execution_generation: Some(ExecutionGeneration::new(1).unwrap()), + activation_attempt_id: Some(SandboxActivationAttemptId::new()), + substrate_kind: Some(NodeManagedBy::Space), + ..SpaceApiRouteIdentity::default() + }; + crate::db::update_space_api_route_identity( + &conn, + direct_id, + &direct_identity, + RouteIdentityProvenance::Unfenced, + ) + .unwrap(); + conn.execute( + "UPDATE sandboxes + SET space_api_route_activation_attempt_id = 'not-an-attempt' + WHERE id = ?1", + [direct_id.to_string()], + ) + .unwrap(); + assert_eq!( + classify_and_record_admission( + &conn, + direct_id, + Some(NodeRole::Worker), + None, + direct_expectation, + ) + .unwrap(), + AutoLifecycleDeleteAdmission::Ordinary( + AutoLifecycleDeleteOrdinaryReason::DirectUnsupported, + ) + ); + + let builder_id = SandboxId::new(); + let builder_expectation = insert_sandbox(&conn, builder_id); + crate::db::update_space_api_route_identity( + &conn, + builder_id, + &cluster_route(2, SandboxActivationAttemptId::new()), + RouteIdentityProvenance::ClusterFenced, + ) + .unwrap(); + conn.execute( + "UPDATE sandboxes + SET space_api_route_activation_attempt_id = 'not-an-attempt' + WHERE id = ?1", + [builder_id.to_string()], + ) + .unwrap(); + assert!(matches!( + classify_and_record_admission(&conn, builder_id, None, None, builder_expectation,).unwrap(), + AutoLifecycleDeleteAdmission::Blocked { + reason: AutoLifecycleDeleteBlockReason::NodeRoleUnavailable, + .. + } + )); + assert!(matches!( + classify_and_record_admission( + &conn, + builder_id, + Some(NodeRole::Builder), + None, + builder_expectation, + ) + .unwrap(), + AutoLifecycleDeleteAdmission::Blocked { + reason: AutoLifecycleDeleteBlockReason::NodeRoleNotWorker, + .. + } + )); + let blocked_at: String = conn + .query_row( + "SELECT auto_lifecycle_delete_blocked_at FROM sandboxes WHERE id = ?1", + [builder_id.to_string()], + |row| row.get(0), + ) + .unwrap(); + record_retained_route_evidence( + &conn, + builder_id, + RouteEvidence::from_identity( + &cluster_route(2, SandboxActivationAttemptId::new()), + RouteIdentityProvenance::ClusterFenced, + ) + .unwrap(), + ) + .unwrap(); + assert_eq!( + conn.query_row( + "SELECT auto_lifecycle_delete_block_reason, + auto_lifecycle_delete_blocked_at + FROM sandboxes WHERE id = ?1", + [builder_id.to_string()], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ) + .unwrap(), + ("node_role_not_worker".to_string(), blocked_at), + "route repair cannot clear a publisher-role block" + ); + + let local_id = SandboxId::new(); + let mut local_expectation = insert_sandbox(&conn, local_id); + conn.execute( + "UPDATE sandboxes SET auto_lifecycle_authority = 'local' WHERE id = ?1", + [local_id.to_string()], + ) + .unwrap(); + local_expectation.authority = PolicyAuthority::Local; + assert_eq!( + classify_and_record_admission(&conn, local_id, None, None, local_expectation,).unwrap(), + AutoLifecycleDeleteAdmission::Ordinary(AutoLifecycleDeleteOrdinaryReason::LocalAuthority,) + ); + + let provisional_id = SandboxId::new(); + let provisional_expectation = insert_sandbox(&conn, provisional_id); + crate::db::update_space_api_route_identity( + &conn, + provisional_id, + &cluster_route(3, SandboxActivationAttemptId::new()), + RouteIdentityProvenance::Unfenced, + ) + .unwrap(); + assert!(matches!( + classify_and_record_admission( + &conn, + provisional_id, + Some(NodeRole::Worker), + None, + provisional_expectation, + ) + .unwrap(), + AutoLifecycleDeleteAdmission::Blocked { + reason: AutoLifecycleDeleteBlockReason::UnfencedRoute, + .. + } + )); + + let missing_route_id = SandboxId::new(); + let missing_route_expectation = insert_sandbox(&conn, missing_route_id); + assert!(matches!( + classify_and_record_admission( + &conn, + missing_route_id, + Some(NodeRole::Worker), + None, + missing_route_expectation, + ) + .unwrap(), + AutoLifecycleDeleteAdmission::Blocked { + reason: AutoLifecycleDeleteBlockReason::MissingRouteIdentity, + .. + } + )); + crate::db::update_space_api_route_identity( + &conn, + missing_route_id, + &cluster_route(4, SandboxActivationAttemptId::new()), + RouteIdentityProvenance::ClusterFenced, + ) + .unwrap(); + assert!(matches!( + classify_and_record_admission( + &conn, + missing_route_id, + Some(NodeRole::Worker), + None, + missing_route_expectation, + ) + .unwrap(), + AutoLifecycleDeleteAdmission::Receipt(_) + )); +} + +#[test] +fn matching_receipt_completes_atomically_with_row_deletion() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let activation_attempt_id = SandboxActivationAttemptId::new(); + let claimed_receipt = receipt(sandbox_id, expectation, 9, activation_attempt_id); + insert_pending_receipt(&conn, claimed_receipt).unwrap(); + conn.execute( + "UPDATE sandboxes + SET state = 'deleting', + cleanup_completion = 'delete_row', + cleanup_keep_filesystem = 0 + WHERE id = ?1", + [sandbox_id.to_string()], + ) + .unwrap(); + + crate::db::complete_row_intent_cleanup_with_receipt( + &conn, + &sandbox_id.to_string(), + crate::db::CleanupCompletion::DeleteRow, + None, + Some(claimed_receipt), + ) + .unwrap(); + + assert!( + crate::db::get_sandbox(&conn, &sandbox_id.to_string()) + .unwrap() + .is_none() + ); + let state: String = conn + .query_row( + "SELECT state FROM auto_lifecycle_delete_outbox WHERE receipt_id = ?1", + [claimed_receipt.receipt_id.to_string()], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(state, "completed"); + + let recovered_sandbox_id = SandboxId::new(); + let recovered_expectation = insert_sandbox(&conn, recovered_sandbox_id); + let recovered_attempt = SandboxActivationAttemptId::new(); + let recovered_receipt = receipt( + recovered_sandbox_id, + recovered_expectation, + 10, + recovered_attempt, + ); + insert_pending_receipt(&conn, recovered_receipt).unwrap(); + conn.execute( + "UPDATE sandboxes + SET state = 'deleting', + cleanup_completion = 'delete_row', + cleanup_keep_filesystem = 0 + WHERE id = ?1", + [recovered_sandbox_id.to_string()], + ) + .unwrap(); + crate::db::complete_row_intent_cleanup_with_receipt( + &conn, + &recovered_sandbox_id.to_string(), + crate::db::CleanupCompletion::DeleteRow, + None, + None, + ) + .unwrap(); + assert!( + crate::db::get_sandbox(&conn, &recovered_sandbox_id.to_string()) + .unwrap() + .is_none() + ); + assert_eq!( + conn.query_row( + "SELECT route_activation_id, state + FROM auto_lifecycle_delete_outbox + WHERE receipt_id = ?1", + [recovered_receipt.receipt_id.to_string()], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ) + .unwrap(), + (recovered_attempt.to_string(), "completed".to_string()) + ); +} + +#[test] +fn pending_receipt_does_not_silently_survive_lost_delete_intent() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let pending = receipt( + sandbox_id, + expectation, + 11, + SandboxActivationAttemptId::new(), + ); + insert_pending_receipt(&conn, pending).unwrap(); + + let error = crate::db::complete_row_intent_cleanup_with_receipt( + &conn, + &sandbox_id.to_string(), + crate::db::CleanupCompletion::DeleteRow, + None, + Some(pending), + ) + .unwrap_err(); + + assert!( + crate::db::get_sandbox(&conn, &sandbox_id.to_string()) + .unwrap() + .is_some() + ); + assert!( + pending_receipt_for_sandbox(&conn, sandbox_id) + .unwrap() + .is_some() + ); + assert!(matches!( + error, + crate::db::CompleteRowIntentCleanupError::PendingReceiptLostDeleteIntent { + sandbox_id: observed, + } if observed == sandbox_id + )); +} + +#[test] +fn pending_receipt_guard_aborts_only_the_delete_statement() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + insert_pending_receipt( + &conn, + receipt( + sandbox_id, + expectation, + 3, + SandboxActivationAttemptId::new(), + ), + ) + .unwrap(); + + let tx = crate::db::immediate_transaction(&conn).unwrap(); + tx.execute( + "UPDATE sandboxes SET failure_reason = 'before guarded delete' WHERE id = ?1", + [sandbox_id.to_string()], + ) + .unwrap(); + let error = tx + .execute( + "DELETE FROM sandboxes WHERE id = ?1", + [sandbox_id.to_string()], + ) + .unwrap_err(); + let error = map_pending_receipt_delete_error(error); + let rusqlite::Error::ToSqlConversionFailure(error) = error else { + panic!("pending-receipt guard must map to a typed database error") + }; + assert!(error.downcast_ref::().is_some()); + tx.commit().unwrap(); + + let failure_reason: Option = conn + .query_row( + "SELECT failure_reason FROM sandboxes WHERE id = ?1", + [sandbox_id.to_string()], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(failure_reason.as_deref(), Some("before guarded delete")); +} + +#[test] +fn list_and_ack_touch_only_completed_receipts() { + let conn = crate::db::test_conn(); + let completed_sandbox = SandboxId::new(); + let pending_sandbox = SandboxId::new(); + let completed_expectation = insert_sandbox(&conn, completed_sandbox); + let pending_expectation = insert_sandbox(&conn, pending_sandbox); + let completed = receipt( + completed_sandbox, + completed_expectation, + 1, + SandboxActivationAttemptId::new(), + ); + let pending = receipt( + pending_sandbox, + pending_expectation, + 1, + SandboxActivationAttemptId::new(), + ); + insert_pending_receipt(&conn, completed).unwrap(); + insert_pending_receipt(&conn, pending).unwrap(); + complete_pending_receipt(&conn, completed_sandbox, completed.receipt_id).unwrap(); + + assert_eq!(list_completed_receipts(&conn).unwrap(), vec![completed]); + assert_eq!( + acknowledge_completed_receipts(&conn, &[completed.receipt_id, pending.receipt_id]).unwrap(), + vec![completed.receipt_id] + ); + assert!( + pending_receipt_for_sandbox(&conn, pending_sandbox) + .unwrap() + .is_some() + ); + assert!( + conn.query_row( + "SELECT receipt_id FROM auto_lifecycle_delete_outbox WHERE sandbox_id = ?1", + [completed_sandbox.to_string()], + |row| row.get::<_, String>(0), + ) + .optional() + .unwrap() + .is_none() + ); +} + +#[test] +fn completed_receipt_listing_is_bounded_without_starving_fresh_evidence() { + let conn = crate::db::test_conn(); + let receipts = (0..17) + .map(|generation| { + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let receipt = receipt( + sandbox_id, + expectation, + generation + 1, + SandboxActivationAttemptId::new(), + ); + insert_pending_receipt(&conn, receipt).unwrap(); + complete_pending_receipt(&conn, sandbox_id, receipt.receipt_id).unwrap(); + receipt + }) + .collect::>(); + + let first = list_completed_receipts(&conn).unwrap(); + assert_eq!(first.len(), 16); + let omitted = receipts + .iter() + .copied() + .find(|receipt| !first.contains(receipt)) + .expect("the bounded first batch omits one fresh receipt"); + + let second = list_completed_receipts(&conn).unwrap(); + assert_eq!(second.len(), 16); + assert_eq!(second.first(), Some(&omitted)); +} + +#[test] +fn pending_receipt_rejects_regressing_and_completed_evidence() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let older_attempt = SandboxActivationAttemptId::new(); + let first_attempt = SandboxActivationAttemptId::new_after(older_attempt).unwrap(); + let first = receipt(sandbox_id, expectation, 4, first_attempt); + assert_eq!(insert_pending_receipt(&conn, first).unwrap(), first); + + let retry = AutoLifecycleDeleteReceipt { + receipt_id: AutoLifecycleDeleteReceiptId::generate(), + ..first + }; + assert_eq!(insert_pending_receipt(&conn, retry).unwrap(), first); + + let conflicting = AutoLifecycleDeleteReceipt { + route_activation_id: older_attempt, + ..retry + }; + insert_pending_receipt(&conn, conflicting).expect_err("conflicting receipt rejected"); + + complete_pending_receipt(&conn, sandbox_id, first.receipt_id).unwrap(); + insert_pending_receipt(&conn, retry).expect_err("conflicting receipt rejected"); +} + +#[test] +fn newer_attempt_conflicts_with_a_pending_receipt() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let first_attempt = SandboxActivationAttemptId::new(); + let newer_attempt = SandboxActivationAttemptId::new_after(first_attempt).unwrap(); + let first = receipt(sandbox_id, expectation, 4, first_attempt); + insert_pending_receipt(&conn, first).unwrap(); + + let replacement = AutoLifecycleDeleteReceipt { + receipt_id: AutoLifecycleDeleteReceiptId::generate(), + route_activation_id: newer_attempt, + ..first + }; + + insert_pending_receipt(&conn, replacement).expect_err("conflicting receipt rejected"); + assert_eq!( + pending_receipt_for_sandbox(&conn, sandbox_id).unwrap(), + Some(first) + ); +} + +#[test] +fn pending_receipt_blocks_same_generation_route_adoption() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let admitted_attempt = SandboxActivationAttemptId::new(); + let pending = receipt(sandbox_id, expectation, 4, admitted_attempt); + insert_pending_receipt(&conn, pending).unwrap(); + + let replacement_attempt = SandboxActivationAttemptId::new_after(admitted_attempt).unwrap(); + assert!( + crate::db::update_space_api_route_identity( + &conn, + sandbox_id, + &cluster_route(4, replacement_attempt), + RouteIdentityProvenance::ClusterFenced, + ) + .is_err() + ); + assert_eq!( + crate::db::get_sandbox(&conn, &sandbox_id.to_string()) + .unwrap() + .unwrap() + .space_api_route_identity + .activation_attempt_id, + None + ); + assert_eq!( + pending_receipt_for_sandbox(&conn, sandbox_id).unwrap(), + Some(pending) + ); +} + +#[test] +fn current_policy_refreshes_a_pending_receipt_without_hiding_its_age() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let activation_attempt_id = SandboxActivationAttemptId::new(); + let first = receipt(sandbox_id, expectation, 4, activation_attempt_id); + insert_pending_receipt(&conn, first).unwrap(); + conn.execute( + "UPDATE auto_lifecycle_delete_outbox + SET created_at = '1' + WHERE sandbox_id = ?1", + [sandbox_id.to_string()], + ) + .unwrap(); + + let mut revised_policy = delete_policy(); + revised_policy.stages[0].at += 1; + let replacement = AutoLifecycleDeleteReceipt { + receipt_id: AutoLifecycleDeleteReceiptId::generate(), + policy_fingerprint: AutoLifecyclePolicyFingerprint::for_policy(&revised_policy), + ..first + }; + + assert_eq!( + insert_pending_receipt(&conn, replacement).unwrap(), + replacement, + ); + assert_eq!( + conn.query_row( + "SELECT receipt_id, auto_lifecycle_policy_fingerprint, created_at + FROM auto_lifecycle_delete_outbox + WHERE sandbox_id = ?1", + [sandbox_id.to_string()], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + )) + }, + ) + .unwrap(), + ( + replacement.receipt_id.to_string(), + replacement.policy_fingerprint.to_string(), + "1".to_string(), + ), + ); +} + +#[test] +fn only_generation_supersession_discards_a_pending_receipt() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let pending = receipt( + sandbox_id, + expectation, + 4, + SandboxActivationAttemptId::new(), + ); + insert_pending_receipt(&conn, pending).unwrap(); + + assert!( + discard_pending_receipt_superseded_by_generation( + &conn, + sandbox_id, + pending.execution_generation, + ) + .unwrap() + .is_none() + ); + assert_eq!( + pending_receipt_for_sandbox(&conn, sandbox_id).unwrap(), + Some(pending) + ); + + let next_generation = ExecutionGeneration::new(5).unwrap(); + assert_eq!( + discard_pending_receipt_superseded_by_generation(&conn, sandbox_id, next_generation,) + .unwrap(), + Some(SupersededPendingReceipt { + receipt_id: pending.receipt_id, + execution_generation: pending.execution_generation, + }) + ); + assert!( + pending_receipt_for_sandbox(&conn, sandbox_id) + .unwrap() + .is_none() + ); +} + +#[test] +fn pending_receipt_fence_is_distinct_from_a_missing_sandbox() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let generation = ExecutionGeneration::new(4).unwrap(); + insert_pending_receipt( + &conn, + receipt( + sandbox_id, + expectation, + generation.get(), + SandboxActivationAttemptId::new(), + ), + ) + .unwrap(); + + let error = crate::db::update_space_api_route_identity( + &conn, + sandbox_id, + &cluster_route(generation.get(), SandboxActivationAttemptId::new()), + RouteIdentityProvenance::ClusterFenced, + ) + .unwrap_err(); + + assert!(is_pending_receipt_execution_fence(&error)); + + let missing_id = SandboxId::new(); + let missing_error = crate::db::update_space_api_route_identity( + &conn, + missing_id, + &cluster_route(generation.get(), SandboxActivationAttemptId::new()), + RouteIdentityProvenance::ClusterFenced, + ) + .unwrap_err(); + assert!(matches!( + missing_error, + rusqlite::Error::QueryReturnedNoRows + )); +} + +#[test] +fn deleting_row_rejects_runtime_attachment_without_retiring_its_receipt() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let pending = receipt( + sandbox_id, + expectation, + 4, + SandboxActivationAttemptId::new(), + ); + insert_pending_receipt(&conn, pending).unwrap(); + conn.execute( + "UPDATE sandboxes + SET state = 'deleting', + cleanup_completion = 'delete_row', + cleanup_keep_filesystem = 0 + WHERE id = ?1", + [sandbox_id.to_string()], + ) + .unwrap(); + + let error = crate::db::update_runtime_fields( + &conn, + sandbox_id, + crate::db::RuntimeFields { + uid: 10_000, + slot: 7, + guest_cid: 37, + pid: std::process::id(), + spaced_host_pid: 10_001, + netgtw_pid: 10_002, + vsock_uds_path: "/space/run/firecracker/sandboxes/test/root/v.sock", + api_socket_path: "/space/run/firecracker/sandboxes/test/api.sock", + spaced_endpoint_proof: None, + space_api_route_identity: SpaceApiRouteIdentity { + execution_generation: Some(ExecutionGeneration::new(5).unwrap()), + ..SpaceApiRouteIdentity::default() + }, + route_identity_provenance: RouteIdentityProvenance::Unfenced, + }, + ) + .expect_err("runtime attachment must not supersede cleanup ownership"); + + assert!(is_lifecycle_execution_fence(&error)); + assert!(!is_pending_receipt_execution_fence(&error)); + assert_eq!( + pending_receipt_for_sandbox(&conn, sandbox_id).unwrap(), + Some(pending) + ); +} + +#[test] +fn beginning_a_new_execution_retires_an_older_pending_receipt() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + conn.execute( + "UPDATE sandboxes SET state = 'created' WHERE id = ?1", + [sandbox_id.to_string()], + ) + .unwrap(); + let pending = receipt( + sandbox_id, + expectation, + 4, + SandboxActivationAttemptId::new(), + ); + insert_pending_receipt(&conn, pending).unwrap(); + let next_generation = ExecutionGeneration::new(5).unwrap(); + let next_attempt = SandboxActivationAttemptId::new(); + + let episode = crate::db::begin_activation_with_route_evidence( + &conn, + &sandbox_id.to_string(), + crate::state::SandboxState::Created, + crate::db::ActivationKind::ColdStart, + Some(next_generation), + Some(next_attempt), + crate::db::ActivationRouteAdmission { + auto_lifecycle: None, + operation_activation_attempt_id: None, + evidence: Some(RouteEvidence { + execution_generation: next_generation, + activation_attempt_id: next_attempt, + substrate: NodeManagedBy::Kubernetes, + provenance: RouteIdentityProvenance::ClusterFenced, + }), + open_claim: None, + }, + ) + .unwrap(); + + assert!(episode.is_some()); + assert!( + pending_receipt_for_sandbox(&conn, sandbox_id) + .unwrap() + .is_none(), + "the activation claim must retire stale cleanup evidence atomically" + ); +} + +#[test] +fn beginning_an_execution_cannot_supersede_a_current_pending_receipt() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + conn.execute( + "UPDATE sandboxes SET state = 'created' WHERE id = ?1", + [sandbox_id.to_string()], + ) + .unwrap(); + let generation = ExecutionGeneration::new(5).unwrap(); + let pending = receipt( + sandbox_id, + expectation, + generation.get(), + SandboxActivationAttemptId::new(), + ); + insert_pending_receipt(&conn, pending).unwrap(); + + let episode = crate::db::begin_activation_with_route_evidence( + &conn, + &sandbox_id.to_string(), + crate::state::SandboxState::Created, + crate::db::ActivationKind::ColdStart, + Some(generation), + Some(SandboxActivationAttemptId::new()), + crate::db::ActivationRouteAdmission::default(), + ) + .unwrap(); + + assert!(episode.is_none()); + assert_eq!( + crate::db::get_sandbox(&conn, &sandbox_id.to_string()) + .unwrap() + .unwrap() + .state, + crate::state::SandboxState::Created + ); + assert_eq!( + pending_receipt_for_sandbox(&conn, sandbox_id).unwrap(), + Some(pending) + ); +} + +#[test] +fn generation_writers_discard_an_older_pending_receipt_without_complete_route_evidence() { + let conn = crate::db::test_conn(); + + let route_sandbox_id = SandboxId::new(); + let route_expectation = insert_sandbox(&conn, route_sandbox_id); + let route_pending = receipt( + route_sandbox_id, + route_expectation, + 4, + SandboxActivationAttemptId::new(), + ); + insert_pending_receipt(&conn, route_pending).unwrap(); + crate::db::update_space_api_route_identity( + &conn, + route_sandbox_id, + &SpaceApiRouteIdentity { + execution_generation: Some(ExecutionGeneration::new(5).unwrap()), + ..SpaceApiRouteIdentity::default() + }, + RouteIdentityProvenance::Unfenced, + ) + .unwrap(); + assert!( + pending_receipt_for_sandbox(&conn, route_sandbox_id) + .unwrap() + .is_none() + ); + + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let pending = receipt( + sandbox_id, + expectation, + 4, + SandboxActivationAttemptId::new(), + ); + insert_pending_receipt(&conn, pending).unwrap(); + crate::db::update_runtime_fields( + &conn, + sandbox_id, + crate::db::RuntimeFields { + uid: 10_000, + slot: 7, + guest_cid: 37, + pid: std::process::id(), + spaced_host_pid: 10_001, + netgtw_pid: 10_002, + vsock_uds_path: "/space/run/firecracker/sandboxes/test/root/v.sock", + api_socket_path: "/space/run/firecracker/sandboxes/test/api.sock", + spaced_endpoint_proof: None, + space_api_route_identity: SpaceApiRouteIdentity { + execution_generation: Some(ExecutionGeneration::new(5).unwrap()), + ..SpaceApiRouteIdentity::default() + }, + route_identity_provenance: RouteIdentityProvenance::Unfenced, + }, + ) + .unwrap(); + assert!( + pending_receipt_for_sandbox(&conn, sandbox_id) + .unwrap() + .is_none() + ); +} + +#[test] +fn completion_reports_a_typed_missing_expected_receipt() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let pending = receipt( + sandbox_id, + expectation, + 4, + SandboxActivationAttemptId::new(), + ); + insert_pending_receipt(&conn, pending).unwrap(); + conn.execute( + "UPDATE sandboxes + SET state = 'deleting', + cleanup_completion = 'delete_row', + cleanup_keep_filesystem = 0 + WHERE id = ?1", + [sandbox_id.to_string()], + ) + .unwrap(); + discard_pending_receipt_superseded_by_generation( + &conn, + sandbox_id, + ExecutionGeneration::new(5).unwrap(), + ) + .unwrap(); + + let error = crate::db::complete_row_intent_cleanup_with_receipt( + &conn, + &sandbox_id.to_string(), + crate::db::CleanupCompletion::DeleteRow, + None, + Some(pending), + ) + .unwrap_err(); + + assert!(matches!( + error, + crate::db::CompleteRowIntentCleanupError::ExpectedReceiptMissingOrChanged { + sandbox_id: observed, + } if observed == sandbox_id + )); +} + +#[test] +fn newer_route_generation_supersedes_receipt_completion() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let pending = receipt( + sandbox_id, + expectation, + 4, + SandboxActivationAttemptId::new(), + ); + insert_pending_receipt(&conn, pending).unwrap(); + conn.execute( + "UPDATE sandboxes + SET state = 'deleting', + cleanup_completion = 'delete_row', + cleanup_keep_filesystem = 0 + WHERE id = ?1", + [sandbox_id.to_string()], + ) + .unwrap(); + record_retained_route_evidence( + &conn, + sandbox_id, + RouteEvidence { + execution_generation: ExecutionGeneration::new(5).unwrap(), + activation_attempt_id: SandboxActivationAttemptId::new(), + substrate: NodeManagedBy::Kubernetes, + provenance: RouteIdentityProvenance::ClusterFenced, + }, + ) + .unwrap(); + discard_pending_receipt_superseded_by_generation( + &conn, + sandbox_id, + ExecutionGeneration::new(5).unwrap(), + ) + .unwrap(); + + assert_eq!( + crate::db::complete_row_intent_cleanup_with_receipt( + &conn, + &sandbox_id.to_string(), + crate::db::CleanupCompletion::DeleteRow, + None, + Some(pending), + ) + .unwrap(), + crate::db::CompleteRowIntentCleanupDisposition::Superseded, + ); + assert!( + crate::db::get_sandbox(&conn, &sandbox_id.to_string()) + .unwrap() + .is_some(), + "the superseded cleanup must not delete the replacement execution" + ); +} + +#[test] +fn newer_current_execution_generation_supersedes_receipt_completion() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let pending = receipt( + sandbox_id, + expectation, + 4, + SandboxActivationAttemptId::new(), + ); + insert_pending_receipt(&conn, pending).unwrap(); + assert!( + discard_pending_receipt_superseded_by_generation( + &conn, + sandbox_id, + ExecutionGeneration::new(5).unwrap(), + ) + .unwrap() + .is_some() + ); + crate::db::update_space_api_route_identity( + &conn, + sandbox_id, + &SpaceApiRouteIdentity { + execution_generation: Some(ExecutionGeneration::new(5).unwrap()), + ..SpaceApiRouteIdentity::default() + }, + RouteIdentityProvenance::Unfenced, + ) + .unwrap(); + conn.execute( + "UPDATE sandboxes + SET state = 'started', + cleanup_completion = NULL, + cleanup_keep_filesystem = NULL + WHERE id = ?1", + [sandbox_id.to_string()], + ) + .unwrap(); + + assert_eq!( + crate::db::complete_row_intent_cleanup_with_receipt( + &conn, + &sandbox_id.to_string(), + crate::db::CleanupCompletion::DeleteRow, + None, + Some(pending), + ) + .unwrap(), + crate::db::CompleteRowIntentCleanupDisposition::Superseded, + ); + assert!( + crate::db::get_sandbox(&conn, &sandbox_id.to_string()) + .unwrap() + .is_some(), + "the superseded cleanup must not delete the replacement execution" + ); + assert!( + pending_receipt_for_sandbox(&conn, sandbox_id) + .unwrap() + .is_none() + ); +} + +#[test] +fn equal_current_execution_generation_does_not_supersede_receipt_completion() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let generation = ExecutionGeneration::new(4).unwrap(); + crate::db::update_space_api_route_identity( + &conn, + sandbox_id, + &SpaceApiRouteIdentity { + execution_generation: Some(generation), + ..SpaceApiRouteIdentity::default() + }, + RouteIdentityProvenance::Unfenced, + ) + .unwrap(); + let pending = receipt( + sandbox_id, + expectation, + generation.get(), + SandboxActivationAttemptId::new(), + ); + insert_pending_receipt(&conn, pending).unwrap(); + conn.execute( + "UPDATE sandboxes + SET state = 'deleting', + cleanup_completion = 'delete_row', + cleanup_keep_filesystem = 0 + WHERE id = ?1", + [sandbox_id.to_string()], + ) + .unwrap(); + let mismatched = AutoLifecycleDeleteReceipt { + receipt_id: AutoLifecycleDeleteReceiptId::generate(), + ..pending + }; + + let error = crate::db::complete_row_intent_cleanup_with_receipt( + &conn, + &sandbox_id.to_string(), + crate::db::CleanupCompletion::DeleteRow, + None, + Some(mismatched), + ) + .unwrap_err(); + + assert!(matches!( + error, + crate::db::CompleteRowIntentCleanupError::ExpectedReceiptMissingOrChanged { + sandbox_id: observed, + } if observed == sandbox_id + )); + assert!( + crate::db::get_sandbox(&conn, &sandbox_id.to_string()) + .unwrap() + .is_some() + ); + assert_eq!( + pending_receipt_for_sandbox(&conn, sandbox_id).unwrap(), + Some(pending) + ); +} + +#[test] +fn equal_generation_retained_evidence_cannot_be_downgraded() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + insert_sandbox(&conn, sandbox_id); + let first_attempt = SandboxActivationAttemptId::new(); + let replacement_attempt = SandboxActivationAttemptId::new(); + record_retained_route_evidence( + &conn, + sandbox_id, + RouteEvidence { + execution_generation: ExecutionGeneration::new(4).unwrap(), + activation_attempt_id: first_attempt, + substrate: NodeManagedBy::Kubernetes, + provenance: RouteIdentityProvenance::ClusterFenced, + }, + ) + .unwrap(); + + assert!( + !record_retained_route_evidence( + &conn, + sandbox_id, + RouteEvidence { + execution_generation: ExecutionGeneration::new(4).unwrap(), + activation_attempt_id: replacement_attempt, + substrate: NodeManagedBy::Kubernetes, + provenance: RouteIdentityProvenance::Unfenced, + }, + ) + .unwrap() + ); + assert_eq!( + conn.query_row( + "SELECT last_observed_route_activation_id, + last_observed_route_identity_provenance + FROM sandboxes WHERE id = ?1", + [sandbox_id.to_string()], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ) + .unwrap(), + (first_attempt.to_string(), "cluster_fenced".to_string()) + ); + + assert!( + record_retained_route_evidence( + &conn, + sandbox_id, + RouteEvidence { + execution_generation: ExecutionGeneration::new(4).unwrap(), + activation_attempt_id: replacement_attempt, + substrate: NodeManagedBy::Kubernetes, + provenance: RouteIdentityProvenance::ClusterFenced, + }, + ) + .unwrap() + ); + assert_eq!( + conn.query_row( + "SELECT last_observed_route_activation_id, + last_observed_route_identity_provenance + FROM sandboxes WHERE id = ?1", + [sandbox_id.to_string()], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ) + .unwrap(), + ( + replacement_attempt.to_string(), + "cluster_fenced".to_string() + ) + ); + + assert!( + !record_retained_route_evidence( + &conn, + sandbox_id, + RouteEvidence { + execution_generation: ExecutionGeneration::new(4).unwrap(), + activation_attempt_id: first_attempt, + substrate: NodeManagedBy::Kubernetes, + provenance: RouteIdentityProvenance::ClusterFenced, + }, + ) + .unwrap(), + "delayed same-generation evidence must not replace the newer attempt" + ); + assert_eq!( + conn.query_row( + "SELECT last_observed_route_activation_id + FROM sandboxes WHERE id = ?1", + [sandbox_id.to_string()], + |row| row.get::<_, String>(0), + ) + .unwrap(), + replacement_attempt.to_string() + ); +} + +#[test] +fn receipt_conflict_uses_bounded_retry_without_claiming_a_durable_block() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + insert_sandbox(&conn, sandbox_id); + + let storage_id = sandbox_id.to_string(); + assert!(record_receipt_conflict_retry(&conn, &storage_id).unwrap()); + assert!(!record_receipt_conflict_retry(&conn, &storage_id).unwrap()); + + let (block_reason, retry_at, runtime_state_json): (Option, i64, String) = conn + .query_row( + "SELECT auto_lifecycle_delete_block_reason, + next_auto_event_at, + auto_lifecycle_state_json + FROM sandboxes + WHERE id = ?1", + [sandbox_id.to_string()], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + let runtime_state: pplx_space_common_core::types::AutoLifecycleRuntimeState = + serde_json::from_str(&runtime_state_json).unwrap(); + assert!(block_reason.is_none()); + assert!(retry_at > crate::db::now_epoch_i64().unwrap()); + assert_eq!( + runtime_state.next_auto_event_at, + Some(u64::try_from(retry_at).unwrap()) + ); + assert!( + runtime_state + .last_auto_error + .as_deref() + .is_some_and(|error| error.contains("conflicting receipt evidence")) + ); +} + +#[test] +fn policy_replacement_retires_obsolete_delete_block() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + + assert!(matches!( + classify_and_record_admission( + &conn, + sandbox_id, + Some(NodeRole::Worker), + None, + expectation, + ) + .unwrap(), + AutoLifecycleDeleteAdmission::Blocked { + reason: AutoLifecycleDeleteBlockReason::MissingRouteIdentity, + .. + } + )); + + crate::db::update_auto_lifecycle( + &conn, + &sandbox_id.to_string(), + &AutoLifecyclePolicy::disabled(), + PolicyAuthority::Cluster, + ) + .unwrap(); + + let (reason, blocked_at): (Option, Option) = conn + .query_row( + "SELECT auto_lifecycle_delete_block_reason, + auto_lifecycle_delete_blocked_at + FROM sandboxes + WHERE id = ?1", + [sandbox_id.to_string()], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + assert_eq!((reason, blocked_at), (None, None)); +} + +#[test] +fn unknown_substrate_is_durably_blocked() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + conn.execute( + "UPDATE sandboxes + SET space_api_route_execution_generation = 7, + space_api_route_activation_attempt_id = ?1, + space_api_route_substrate_kind = 'future_substrate', + space_api_route_identity_provenance = 'cluster_fenced' + WHERE id = ?2", + rusqlite::params![ + SandboxActivationAttemptId::new().to_string(), + sandbox_id.to_string() + ], + ) + .unwrap(); + + let admission = + classify_and_record_admission(&conn, sandbox_id, Some(NodeRole::Worker), None, expectation) + .expect("unknown substrate must fail closed as a durable block"); + let AutoLifecycleDeleteAdmission::Blocked { reason, changed } = admission else { + panic!("unknown substrate must not admit deletion: {admission:?}"); + }; + assert_eq!(reason.as_str(), "unknown_substrate"); + assert!(changed); +} diff --git a/crates/libs/sbx/mgr/server/src/db/cleanup_episodes.rs b/crates/libs/sbx/mgr/server/src/db/cleanup_episodes.rs index 183de45ae..20774ed10 100644 --- a/crates/libs/sbx/mgr/server/src/db/cleanup_episodes.rs +++ b/crates/libs/sbx/mgr/server/src/db/cleanup_episodes.rs @@ -154,7 +154,7 @@ pub(crate) fn insert_sandbox_with_cleanup_episode_and_route_evidence( params: InsertSandboxParams<'_>, episode: CleanupEpisodeSpec, route_evidence: Option, - open_route_activation: Option<&crate::route_activation_claims::OpenRouteActivationClaim>, + open_route_activation: Option<&route_activation_claims::OpenRouteActivationClaim>, execution: Option<&ExecutionMetadata>, ) -> std::result::Result<(), InsertSandboxWithCleanupEpisodeError> { if params.state != episode.activation_state { @@ -172,7 +172,7 @@ pub(crate) fn insert_sandbox_with_cleanup_episode_and_route_evidence( upsert_sandbox_execution_metadata_in_transaction(&tx, &sandbox_id, execution)?; } if let Some(route_evidence) = route_evidence { - crate::auto_lifecycle_delete_receipts::db::record_retained_route_evidence( + auto_lifecycle_delete_receipts::record_retained_route_evidence( &tx, sandbox_id .parse() @@ -181,11 +181,7 @@ pub(crate) fn insert_sandbox_with_cleanup_episode_and_route_evidence( )?; } if let Some(open_route_activation) = open_route_activation { - crate::route_activation_claims::insert_create_claim( - &tx, - &sandbox_id, - open_route_activation, - )?; + route_activation_claims::insert_create_claim(&tx, &sandbox_id, open_route_activation)?; } insert_cleanup_episode(&tx, &sandbox_id, episode)?; tx.commit()?; @@ -1037,7 +1033,7 @@ pub(crate) fn complete_cleanup_episode_for_job( record_execution_delete_receipt(&tx, sandbox_id, execution_generation)?; delete_sandbox_child_rows(&tx, sandbox_id)?; tx.execute("DELETE FROM sandboxes WHERE id = ?1", params![sandbox_id]) - .map_err(crate::auto_lifecycle_delete_receipts::db::map_pending_receipt_delete_error)?; + .map_err(auto_lifecycle_delete_receipts::map_pending_receipt_delete_error)?; } else { if !complete_state_cleanup(&tx, sandbox_id, cleanup_id, &episode, recovery.as_ref())? { tx.commit()?; @@ -1050,7 +1046,7 @@ pub(crate) fn complete_cleanup_episode_for_job( // completed lifecycle teardown. Keep the exact attempt's continuation // claim so a transiently failed RPC can retry after runtime cleanup. if episode.completion == CleanupCompletion::Quarantine { - crate::route_activation_claims::close_claim(&tx, sandbox_id)?; + route_activation_claims::close_claim(&tx, sandbox_id)?; } tx.execute( "DELETE FROM sandbox_cleanup_episodes @@ -1133,9 +1129,7 @@ pub(crate) enum CompleteRowIntentCleanupError { #[error(transparent)] Database(#[from] rusqlite::Error), #[error(transparent)] - ReceiptCompletion( - #[from] crate::auto_lifecycle_delete_receipts::db::CompletePendingReceiptError, - ), + ReceiptCompletion(#[from] auto_lifecycle_delete_receipts::CompletePendingReceiptError), #[error( "expected auto-lifecycle delete receipt is missing or changed for sandbox {sandbox_id}" )] @@ -1260,7 +1254,7 @@ pub(crate) fn complete_row_intent_cleanup_for_job( .transpose()?; let pending_delete_receipt = parsed_delete_sandbox_id .map(|sandbox_id| { - crate::auto_lifecycle_delete_receipts::db::pending_receipt_for_sandbox(&tx, sandbox_id) + auto_lifecycle_delete_receipts::pending_receipt_for_sandbox(&tx, sandbox_id) }) .transpose()? .flatten(); @@ -1322,7 +1316,7 @@ pub(crate) fn complete_row_intent_cleanup_for_job( // A caller-supplied expected receipt was matched by the hoisted // ownership check. Startup and operator recovery instead derive // ownership from this durable receipt plus the DeleteRow intent. - crate::auto_lifecycle_delete_receipts::db::complete_pending_receipt( + auto_lifecycle_delete_receipts::complete_pending_receipt( &tx, parsed_sandbox_id, receipt.receipt_id, @@ -1335,7 +1329,7 @@ pub(crate) fn complete_row_intent_cleanup_for_job( )?; delete_sandbox_child_rows(&tx, sandbox_id)?; tx.execute("DELETE FROM sandboxes WHERE id = ?1", params![sandbox_id]) - .map_err(crate::auto_lifecycle_delete_receipts::db::map_pending_receipt_delete_error)?; + .map_err(auto_lifecycle_delete_receipts::map_pending_receipt_delete_error)?; cleanup_jobs::finish_in_transaction(&tx, job_id, &cleanup_jobs::CleanupOutcome::Applied)?; tx.commit()?; return Ok(CompleteRowIntentCleanupDisposition::Applied); @@ -1413,7 +1407,7 @@ pub(crate) fn complete_row_intent_cleanup_for_job( // `apply_cleanup_state` clears the live identity and fires the retained // identity trigger. Close the claim only after that observation, but // before this transaction commits. - crate::route_activation_claims::close_claim(&tx, sandbox_id)?; + route_activation_claims::close_claim(&tx, sandbox_id)?; let outcome = if missing_suspend_artifact { cleanup_jobs::CleanupOutcome::Quarantined( SUSPEND_PUBLISH_FAILED_NO_ARTIFACT_RUNTIME_GONE.to_owned(), @@ -1514,7 +1508,7 @@ pub(crate) fn quarantine_cleanup_episode_for_job( ], )?; if changed == 1 { - crate::route_activation_claims::close_claim(&tx, sandbox_id)?; + route_activation_claims::close_claim(&tx, sandbox_id)?; insert_new_quarantine_episode( &tx, &QuarantineEpisode { diff --git a/crates/libs/sbx/mgr/server/src/db/cleanup_jobs.rs b/crates/libs/sbx/mgr/server/src/db/cleanup_jobs.rs index 982514ca9..7849a8510 100644 --- a/crates/libs/sbx/mgr/server/src/db/cleanup_jobs.rs +++ b/crates/libs/sbx/mgr/server/src/db/cleanup_jobs.rs @@ -278,7 +278,7 @@ pub(crate) fn current(conn: &Connection, work: &CleanupWork) -> rusqlite::Result return cleanup_episode_current(conn, &id, episode); } if work.completion == CleanupCompletion::DeleteRow - && !crate::auto_lifecycle_delete_receipts::db::pending_receipt_matches_observed_execution( + && !auto_lifecycle_delete_receipts::pending_receipt_matches_observed_execution( conn, work.sandbox_id, )? diff --git a/crates/libs/sbx/mgr/server/src/db/eviction.rs b/crates/libs/sbx/mgr/server/src/db/eviction.rs index 25974bba8..7475c8a12 100644 --- a/crates/libs/sbx/mgr/server/src/db/eviction.rs +++ b/crates/libs/sbx/mgr/server/src/db/eviction.rs @@ -309,7 +309,7 @@ fn cancel_live_upload_job(conn: &Connection, artifact: LocalArtifactId) -> Resul use pplx_space_common_jobs::{JobStatus, db as jobs}; let Some(job) = jobs::find_job_by_dedup_key( conn, - crate::snapshot_upload_jobs::SNAPSHOT_UPLOAD_JOB_KIND, + super::snapshot_upload_jobs::SNAPSHOT_UPLOAD_JOB_KIND, &artifact.to_string(), )? else { diff --git a/crates/libs/sbx/mgr/server/src/db/eviction_scan.rs b/crates/libs/sbx/mgr/server/src/db/eviction_scan.rs new file mode 100644 index 000000000..c23167607 --- /dev/null +++ b/crates/libs/sbx/mgr/server/src/db/eviction_scan.rs @@ -0,0 +1,218 @@ +//! Suspended-sandbox reads behind `crate::sandbox_eviction`'s candidate +//! scans and local eviction evidence. + +use pplx_space_common_svc::sql::sqlite::SqliteU64; +use rusqlite::{Connection, OptionalExtension, Result, params}; + +use crate::db::timestamps::EpochSeconds; + +const VALID_SUSPENDED_UPDATED_AT_SQL: &str = + "s.updated_at <> '' AND s.updated_at NOT GLOB '*[^0-9]*'"; + +/// Idle witness for eviction candidacy: the sandbox's last real +/// activity, not the row's last write. `updated_at` has an unbounded +/// writer set (periodic reconcilers rewrite rows value-identically and +/// stamp it), so a row can look seconds-old for weeks while its bytes +/// sit cold — the activity clocks inside `auto_lifecycle_state_json` +/// only advance on genuine touches and transitions. NULL-safe two-way +/// max of `last_access_at`/`last_command_at`; rows carrying neither +/// clock (operator/legacy/orphan-adopted rows) fall back to +/// `updated_at`, preserving their previous behavior exactly. The +/// `json_valid` gate is load-bearing: `json_extract` aborts the whole +/// statement on malformed JSON, so without it one corrupt row would +/// fail every scan and shield the valid cold rows around it — a +/// malformed blob falls back to `updated_at` like a missing one. Each +/// clock is additionally type-gated to a non-negative integer: our +/// writers only ever store u64 epoch seconds, so anything else +/// (negative, fractional, string) is corruption and must read as +/// absent rather than fail the `SqliteU64` row decode. The CAS claim +/// deliberately keeps raw `updated_at`: "any write means the row is in +/// flux" is the right semantic there and the wrong one here. +/// +/// The partial eviction index prunes to suspended rows before evaluating +/// this expression. If scan cost grows with node row counts, the escape +/// hatch is an index on this deterministic expression or a virtual generated +/// column — not a writer-maintained copy, which would reintroduce the drift +/// this witness exists to escape. +/// Scalar `MAX` returns NULL if ANY argument is NULL, so absent or +/// corrupt clocks are mapped to a -1 sentinel first (real clocks are +/// type-gated non-negative, so the sentinel cannot collide) and +/// `NULLIF` restores NULL when both were absent. +const IDLE_WITNESS_SQL: &str = "COALESCE( + CASE + WHEN json_valid(s.auto_lifecycle_state_json) THEN + NULLIF( + MAX( + COALESCE(CASE + WHEN typeof(json_extract(s.auto_lifecycle_state_json, '$.last_access_at')) = 'integer' + AND json_extract(s.auto_lifecycle_state_json, '$.last_access_at') >= 0 + THEN json_extract(s.auto_lifecycle_state_json, '$.last_access_at') + END, -1), + COALESCE(CASE + WHEN typeof(json_extract(s.auto_lifecycle_state_json, '$.last_command_at')) = 'integer' + AND json_extract(s.auto_lifecycle_state_json, '$.last_command_at') >= 0 + THEN json_extract(s.auto_lifecycle_state_json, '$.last_command_at') + END, -1) + ), + -1) + END, + CAST(s.updated_at AS INTEGER) + )"; + +/// `(sandbox_id, idle witness)` of cold suspended sandboxes after the +/// `(cursor_witness, cursor_sandbox_id)` keyset position, in witness then id +/// order. +pub(crate) fn idle_suspended_candidates( + conn: &Connection, + cutoff: i64, + cursor_witness: Option, + cursor_sandbox_id: Option<&str>, + limit: usize, +) -> Result> { + let sql = format!( + "SELECT s.id, {IDLE_WITNESS_SQL} FROM sandboxes s + WHERE s.state = 'suspended' + AND {VALID_SUSPENDED_UPDATED_AT_SQL} + AND {IDLE_WITNESS_SQL} <= ?1 + AND ( + ?2 IS NULL + OR {IDLE_WITNESS_SQL} > ?2 + OR ({IDLE_WITNESS_SQL} = ?2 AND s.id > ?3) + ) + AND ( + s.current_local_artifact_id IS NOT NULL + OR EXISTS ( + SELECT 1 FROM jobs j + WHERE j.kind = 'snapshot_upload' + AND j.subject_id = s.id + AND json_extract(j.payload, '$.snapshot_id') IS NOT NULL + ) + ) + AND NOT EXISTS ( + SELECT 1 FROM jobs j + WHERE j.kind = 'snapshot_upload' + AND j.subject_id = s.id + AND j.status IN ('pending', 'running') + ) + ORDER BY {IDLE_WITNESS_SQL} ASC, s.id ASC + LIMIT ?4" + ); + let mut stmt = conn.prepare(&sql)?; + let rows = stmt + .query_map( + params![cutoff, cursor_witness, cursor_sandbox_id, limit as i64], + |row| { + let sandbox_id: String = row.get(0)?; + let idle_witness_secs = row.get::<_, SqliteU64>(1)?.0; + Ok((sandbox_id, idle_witness_secs)) + }, + )? + .collect::>()?; + Ok(rows) +} + +/// Ids of cold suspended sandboxes with neither a pinned artifact nor an +/// upload job carrying a snapshot id, oldest witness first. +pub(crate) fn legacy_reupload_candidates( + conn: &Connection, + idle_cutoff_epoch_secs: u64, + limit: usize, +) -> Result> { + let sql = format!( + "SELECT s.id FROM sandboxes s + WHERE s.state = 'suspended' + AND {VALID_SUSPENDED_UPDATED_AT_SQL} + AND {IDLE_WITNESS_SQL} <= ?1 + AND s.current_local_artifact_id IS NULL + AND NOT EXISTS ( + SELECT 1 FROM jobs j + WHERE j.kind = 'snapshot_upload' + AND j.subject_id = s.id + AND j.status IN ('pending', 'running') + ) + AND NOT EXISTS ( + SELECT 1 FROM jobs j + WHERE j.kind = 'snapshot_upload' + AND j.subject_id = s.id + AND json_extract(j.payload, '$.snapshot_id') IS NOT NULL + ) + ORDER BY {IDLE_WITNESS_SQL} ASC + LIMIT ?2" + ); + let mut stmt = conn.prepare(&sql)?; + let rows = stmt + .query_map( + params![SqliteU64(idle_cutoff_epoch_secs), limit as i64], + |r| r.get(0), + )? + .collect::>()?; + Ok(rows) +} + +pub(crate) fn count_malformed_suspended_updated_at(conn: &Connection) -> Result { + let sql = format!( + "SELECT COUNT(*) FROM sandboxes s + WHERE s.state = 'suspended' + AND NOT ({VALID_SUSPENDED_UPDATED_AT_SQL})" + ); + conn.query_row(&sql, [], |row| { + let count: i64 = row.get(0)?; + Ok(usize::try_from(count).unwrap_or_default()) + }) +} + +/// Every local `sandboxes` row, whatever its state. node-ctl's full census +/// publishes one entry per row, so this is the count that sizes it. +pub(crate) fn count_resident_rows(conn: &Connection) -> Result { + conn.query_row("SELECT COUNT(*) FROM sandboxes", [], |row| { + let count: i64 = row.get(0)?; + usize::try_from(count).map_err(|_| rusqlite::Error::IntegralValueOutOfRange(0, count)) + }) +} + +/// The sandbox's pinned artifact id and `updated_at`, `None` without a row. +pub(crate) fn artifact_pin_and_updated_at( + conn: &Connection, + sandbox_id: &str, +) -> Result, EpochSeconds)>> { + conn.query_row( + "SELECT current_local_artifact_id, updated_at FROM sandboxes WHERE id = ?1", + params![sandbox_id], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .optional() +} + +/// `(artifact_kind, local_artifact_id, backed up)` of the artifact `pin` +/// names, only if `sandbox_id` owns it. +pub(crate) fn owned_artifact_evidence( + conn: &Connection, + pin: &str, + sandbox_id: &str, +) -> Result> { + conn.query_row( + "SELECT artifact_kind, local_artifact_id, backup_completed_at IS NOT NULL + FROM local_artifacts + WHERE local_artifact_id = ?1 AND sandbox_id = ?2", + params![pin, sandbox_id], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), + ) + .optional() +} + +/// The payload `snapshot_id` and `created_at` of the sandbox's newest +/// snapshot-upload job, whatever its status. +pub(crate) fn latest_upload_job( + conn: &Connection, + sandbox_id: &str, +) -> Result, String)>> { + conn.query_row( + "SELECT json_extract(j.payload, '$.snapshot_id'), j.created_at FROM jobs j + WHERE j.kind = 'snapshot_upload' AND j.subject_id = ?1 + ORDER BY j.created_at DESC, j.job_id DESC + LIMIT 1", + params![sandbox_id], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .optional() +} diff --git a/crates/libs/sbx/mgr/server/src/db/guest_health_scan.rs b/crates/libs/sbx/mgr/server/src/db/guest_health_scan.rs new file mode 100644 index 000000000..29f3ae51e --- /dev/null +++ b/crates/libs/sbx/mgr/server/src/db/guest_health_scan.rs @@ -0,0 +1,66 @@ +//! Sandbox rows the watchdog's guest-health and probe passes read. + +use rusqlite::types::Type; +use rusqlite::{Connection, OptionalExtension, Result}; + +use pplx_space_common_core::sql; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct SandboxGuestHealthRow { + pub(crate) id: String, + pub(crate) state: crate::state::SandboxState, + pub(crate) template_content_hash: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +#[error("unknown sandbox state '{value}'")] +struct SandboxStateParseError { + value: String, +} + +pub(crate) fn list_guest_health_rows(conn: &Connection) -> Result> { + let mut stmt = conn.prepare(sql!("SELECT id, state, template_hash FROM sandboxes"))?; + let rows = stmt.query_map([], |row| { + let state_value = row.get::<_, String>(1)?; + let state = state_value + .parse::() + .map_err(|_| { + rusqlite::Error::FromSqlConversionFailure( + 1, + Type::Text, + Box::new(SandboxStateParseError { value: state_value }), + ) + })?; + Ok(SandboxGuestHealthRow { + id: row.get(0)?, + state, + template_content_hash: row.get(2)?, + }) + })?; + rows.collect() +} + +/// One sandbox's `state` label, `None` without a row. +pub(crate) type StateLabelRead = Result>; + +/// The `state` label of each of `sandbox_ids` through one prepared +/// statement, `None` for a sandbox without a row. Only preparing the +/// statement fails the whole lookup; each id's read fails on its own. +pub(crate) fn state_labels_for_ids<'a>( + conn: &Connection, + sandbox_ids: &'a [String], +) -> Result> { + let mut statement = conn.prepare("SELECT state FROM sandboxes WHERE id = ?1")?; + Ok(sandbox_ids + .iter() + .map(|sandbox_id| { + let label = statement + .query_row([sandbox_id], |row| row.get::<_, String>(0)) + .optional(); + (sandbox_id, label) + }) + .collect()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/libs/sbx/mgr/server/src/db/guest_health_scan/tests.rs b/crates/libs/sbx/mgr/server/src/db/guest_health_scan/tests.rs new file mode 100644 index 000000000..de5401770 --- /dev/null +++ b/crates/libs/sbx/mgr/server/src/db/guest_health_scan/tests.rs @@ -0,0 +1,30 @@ +use pplx_space_common_core::types::SandboxId; + +use crate::service::tests::service_tests::insert_test_sandbox; +use crate::state::SandboxState; + +use super::*; + +#[test] +fn state_labels_follow_the_ids_and_report_missing_rows_as_none() { + let conn = crate::db::test_conn(); + let [started, missing, stopped] = [(); 3].map(|()| SandboxId::new().to_string()); + insert_test_sandbox(&conn, &started, SandboxState::Started); + insert_test_sandbox(&conn, &stopped, SandboxState::Stopped); + let ids = [stopped.clone(), missing.clone(), started.clone()]; + + let labels: Vec<_> = state_labels_for_ids(&conn, &ids) + .unwrap() + .into_iter() + .map(|(id, label)| (id.clone(), label.unwrap())) + .collect(); + + assert_eq!( + labels, + [ + (stopped, Some("stopped".to_owned())), + (missing, None), + (started, Some("started".to_owned())), + ] + ); +} diff --git a/crates/libs/sbx/mgr/server/src/db/local_artifact_leases.rs b/crates/libs/sbx/mgr/server/src/db/local_artifact_leases.rs new file mode 100644 index 000000000..df8b54e6e --- /dev/null +++ b/crates/libs/sbx/mgr/server/src/db/local_artifact_leases.rs @@ -0,0 +1,382 @@ +//! Statements behind `crate::local_artifacts::LocalArtifacts` and its +//! restore and delete leases. Each runs on the connection or transaction its +//! caller holds, so a checkout's reads and its CAS share one transaction. + +use pplx_space_common_core::sql; +use pplx_space_common_core::types::{ArtifactKind, LocalArtifactId, SnapshotName}; +use pplx_space_common_svc::sql::sqlite::SqliteU64; +use rusqlite::{Connection, OptionalExtension, Result, params}; + +use crate::db::timestamps::EpochSeconds; + +/// One sandbox's current lifecycle artifact that this node cannot prove has +/// a durable remote copy. +/// +/// Identity comes from the `sandboxes` pin rather than from the artifact +/// row, because the case where there *is* no artifact row is one of the +/// cases this reports. Migration 026's CHECK keeps `current_local_artifact_id` +/// and `current_local_artifact_kind` set or unset together, so the pin alone +/// always carries a whole identity. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct PendingBackup { + pub sandbox_id: String, + pub local_artifact_id: LocalArtifactId, + pub artifact_kind: ArtifactKind, +} + +/// Logical state of a `local_artifacts` row. Encoded on the row's +/// `state` column; transitions are CAS via +/// `LocalArtifacts::check_out_for_restore` / +/// `LocalArtifacts::check_out_for_delete`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ArtifactState { + /// Published and available. The only state from which a checkout + /// (restore or delete) can succeed. + Ready, + /// A restore is currently reading this artifact's `local_path`. + /// Reapers must skip; deleters must wait or fail with NotReady. + Restoring, + /// A delete has reserved this row. The on-disk directory has + /// usually been renamed to a `.gc-pending-` sibling. Readers + /// should treat the artifact as gone. + Deleting, +} + +impl ArtifactState { + pub fn as_str(self) -> &'static str { + match self { + ArtifactState::Ready => "ready", + ArtifactState::Restoring => "restoring", + ArtifactState::Deleting => "deleting", + } + } +} + +/// Error parsing an [`ArtifactState`] from its wire/DB string. The +/// CHECK constraint on `local_artifacts.state` should make this +/// unreachable for rows the schema accepted; it surfaces table +/// corruption or migration drift. +#[derive(Debug, thiserror::Error)] +#[error("unrecognized artifact state {0:?}")] +pub struct ParseArtifactStateError(String); + +impl std::str::FromStr for ArtifactState { + type Err = ParseArtifactStateError; + + fn from_str(s: &str) -> Result { + match s { + "ready" => Ok(ArtifactState::Ready), + "restoring" => Ok(ArtifactState::Restoring), + "deleting" => Ok(ArtifactState::Deleting), + _ => Err(ParseArtifactStateError(s.to_string())), + } + } +} + +impl std::fmt::Display for ArtifactState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// Point-in-time publication evidence for a named snapshot, not a checkout lease. +pub(crate) struct NamedArtifactPublication { + pub snapshot_name: SnapshotName, + pub local_path: String, + pub artifact_kind: ArtifactKind, + pub state: ArtifactState, +} + +/// Every sandbox whose pinned artifact has no backup-completed stamp, or +/// whose pin names no row it owns. The caller, +/// `LocalArtifacts::list_pending_backups`, documents the contract. +pub(crate) fn list_pending_backups(conn: &Connection) -> Result> { + conn.prepare( + "SELECT s.id, s.current_local_artifact_id, s.current_local_artifact_kind + FROM sandboxes AS s + LEFT JOIN local_artifacts AS a + ON a.local_artifact_id = s.current_local_artifact_id + AND a.sandbox_id = s.id + WHERE s.current_local_artifact_id IS NOT NULL + AND (a.local_artifact_id IS NULL OR a.backup_completed_at IS NULL) + ORDER BY s.id", + )? + .query_map([], |row| { + Ok(PendingBackup { + sandbox_id: row.get(0)?, + local_artifact_id: row.get::<_, String>(1)?.parse().map_err(|error| { + rusqlite::Error::FromSqlConversionFailure( + 1, + rusqlite::types::Type::Text, + Box::new(error), + ) + })?, + artifact_kind: row.get::<_, String>(2)?.parse().map_err(|error| { + rusqlite::Error::FromSqlConversionFailure( + 2, + rusqlite::types::Type::Text, + Box::new(error), + ) + })?, + }) + })? + .collect() +} + +/// A sandbox's named-snapshot artifacts, within the inventory read budget. +pub(crate) fn list_named_publications( + conn: &Connection, + sandbox_id: &str, +) -> std::result::Result, crate::db::inventory_read::InventoryReadError> +{ + use pplx_space_common_core::inventory::{InventoryBudget, MAX_ITEMS, MAX_RAW_BYTES}; + let mut statement = conn.prepare(sql!( + "SELECT snapshot_name, local_path, artifact_kind, state + FROM local_artifacts WHERE sandbox_id = ?1 AND artifact_origin = 'snapshot'" + ))?; + let mut rows = statement.query([sandbox_id])?; + let mut budget = InventoryBudget::new(MAX_ITEMS, MAX_RAW_BYTES); + let mut publications = Vec::new(); + while let Some(row) = rows.next()? { + crate::db::inventory_read::check_row(row, &mut budget)?; + publications.push(NamedArtifactPublication { + snapshot_name: row.get::<_, String>(0)?.parse().map_err(|error| { + rusqlite::Error::FromSqlConversionFailure( + 0, + rusqlite::types::Type::Text, + Box::new(error), + ) + })?, + local_path: row.get(1)?, + artifact_kind: row.get::<_, String>(2)?.parse().map_err(|error| { + rusqlite::Error::FromSqlConversionFailure( + 2, + rusqlite::types::Type::Text, + Box::new(error), + ) + })?, + state: row.get::<_, String>(3)?.parse().map_err(|error| { + rusqlite::Error::FromSqlConversionFailure( + 3, + rusqlite::types::Type::Text, + Box::new(error), + ) + })?, + }); + } + Ok(publications) +} + +/// `(local_path, state)` of an artifact row, as stored. +pub(crate) fn artifact_path_and_state( + conn: &Connection, + local_artifact_id: &str, +) -> Result> { + conn.query_row( + "SELECT local_path, state FROM local_artifacts WHERE local_artifact_id = ?1", + params![local_artifact_id], + |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)), + ) + .optional() +} + +/// CAS `ready` → `restoring`; the number of rows changed. +pub(crate) fn mark_artifact_restoring(conn: &Connection, local_artifact_id: &str) -> Result { + conn.execute( + "UPDATE local_artifacts SET state = 'restoring' + WHERE local_artifact_id = ?1 AND state = 'ready'", + params![local_artifact_id], + ) +} + +/// CAS `ready` → `deleting`; the number of rows changed. +pub(crate) fn mark_artifact_deleting(conn: &Connection, local_artifact_id: &str) -> Result { + conn.execute( + "UPDATE local_artifacts SET state = 'deleting' + WHERE local_artifact_id = ?1 AND state = 'ready'", + params![local_artifact_id], + ) +} + +/// The GC checkout's evidence for an artifact `sandbox_id` owns: its +/// creation and last-touch times, whether a sandbox pins it, and whether its +/// backup completed. +pub(crate) fn gc_checkout_evidence( + conn: &Connection, + local_artifact_id: &str, + sandbox_id: &str, +) -> Result> { + conn.query_row( + "SELECT created_at, COALESCE(last_touched_at, created_at), + EXISTS( + SELECT 1 FROM sandboxes + WHERE current_local_artifact_id = ?1 + OR suspend_predecessor_artifact_id = ?1 + ), + backup_completed_at IS NOT NULL + FROM local_artifacts WHERE local_artifact_id = ?1 AND sandbox_id = ?2", + params![local_artifact_id, sandbox_id], + |row| { + Ok(( + row.get::<_, EpochSeconds>(0)?, + row.get::<_, EpochSeconds>(1)?, + row.get::<_, bool>(2)?, + row.get::<_, bool>(3)?, + )) + }, + ) + .optional() +} + +/// Whether a sandbox pins the artifact as its current or suspend-predecessor +/// artifact. +pub(crate) fn artifact_is_pinned(conn: &Connection, local_artifact_id: &str) -> Result { + conn.query_row( + sql!( + "SELECT EXISTS(SELECT 1 FROM sandboxes + WHERE current_local_artifact_id = ?1 OR suspend_predecessor_artifact_id = ?1)" + ), + [local_artifact_id], + |row| row.get(0), + ) +} + +/// The number of artifact rows at `local_path`. +pub(crate) fn count_artifacts_at_path(conn: &Connection, local_path: &str) -> Result { + conn.query_row( + "SELECT COUNT(*) FROM local_artifacts WHERE local_path = ?1", + params![local_path], + |r| Ok(r.get::<_, SqliteU64>(0)?.0), + ) +} + +/// Reset every `restoring` row to `ready`; the number reset. +pub(crate) fn reset_restoring_artifacts(conn: &Connection) -> Result { + conn.execute( + "UPDATE local_artifacts SET state = 'ready' WHERE state = 'restoring'", + [], + ) +} + +/// `(local_artifact_id, local_path)` of every `deleting` row. +pub(crate) fn list_deleting_artifacts(conn: &Connection) -> Result> { + conn.prepare( + "SELECT local_artifact_id, local_path FROM local_artifacts + WHERE state = 'deleting'", + )? + .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? + .collect::>() +} + +/// CAS `deleting` → `ready`; the number of rows changed. +pub(crate) fn roll_back_deleting_artifact( + conn: &Connection, + local_artifact_id: &str, +) -> Result { + conn.execute( + "UPDATE local_artifacts SET state = 'ready' + WHERE local_artifact_id = ?1 AND state = 'deleting'", + params![local_artifact_id], + ) +} + +/// Delete the row if it is still `deleting`; the number deleted. +pub(crate) fn delete_deleting_artifact( + conn: &Connection, + local_artifact_id: &str, +) -> Result { + conn.execute( + "DELETE FROM local_artifacts + WHERE local_artifact_id = ?1 AND state = 'deleting'", + params![local_artifact_id], + ) +} + +/// CAS `restoring` → `ready`, touching the artifact now. +pub(crate) fn release_restoring_artifact_with_touch( + conn: &Connection, + local_artifact_id: &str, +) -> Result { + conn.execute( + "UPDATE local_artifacts + SET state = 'ready', last_touched_at = ?1 + WHERE local_artifact_id = ?2 AND state = 'restoring'", + params![crate::db::now()?, local_artifact_id], + ) +} + +/// CAS `restoring` → `ready`, leaving the touch time alone. +pub(crate) fn release_restoring_artifact( + conn: &Connection, + local_artifact_id: &str, +) -> Result { + conn.execute( + "UPDATE local_artifacts SET state = 'ready' + WHERE local_artifact_id = ?1 AND state = 'restoring'", + params![local_artifact_id], + ) +} + +/// Whether neither an artifact row nor a sandbox pin names the id. +pub(crate) fn gc_staging_is_unowned(conn: &Connection, local_artifact_id: &str) -> Result { + conn.query_row( + sql!( + "SELECT NOT EXISTS(SELECT 1 FROM local_artifacts WHERE local_artifact_id = ?1) + AND NOT EXISTS(SELECT 1 FROM sandboxes + WHERE current_local_artifact_id = ?1 OR suspend_predecessor_artifact_id = ?1)" + ), + [local_artifact_id], + |row| row.get(0), + ) +} + +/// The highest artifact rowid, `None` for an empty table. +pub(crate) fn max_artifact_rowid(conn: &Connection) -> Result> { + conn.query_row("SELECT MAX(rowid) FROM local_artifacts", [], |r| r.get(0)) +} + +/// An orphan-scan row's `(local_artifact_id, local_path, owned)`, or the +/// error decoding them. +pub(crate) type OrphanScanRow = Result<(String, String, bool)>; + +/// Up to `limit` artifact rows with rowids in `next..=through`, in rowid +/// order: each rowid, and its `(local_artifact_id, local_path, owned)` or the +/// error decoding them, so one malformed row does not fail the scan. +pub(crate) fn orphan_scan_rows( + conn: &Connection, + next: i64, + through: i64, + limit: i64, +) -> Result> { + conn.prepare( + "SELECT la.rowid, la.local_artifact_id, la.local_path, + EXISTS (SELECT 1 FROM sandboxes s WHERE s.id = la.sandbox_id) + FROM local_artifacts la + WHERE la.rowid >= ?1 AND la.rowid <= ?2 + ORDER BY la.rowid LIMIT ?3", + )? + .query_map(params![next, through, limit], |row| { + let decoded = (|| -> rusqlite::Result<(String, String, bool)> { + Ok((row.get(1)?, row.get(2)?, row.get(3)?)) + })(); + Ok((row.get::<_, i64>(0)?, decoded)) + })? + .collect::>>() +} + +/// Delete the artifact row if its sandbox row is gone; the number deleted. +pub(crate) fn delete_orphan_artifact_row( + conn: &Connection, + local_artifact_id: &str, +) -> Result { + conn.execute( + "DELETE FROM local_artifacts + WHERE local_artifact_id = ?1 + AND NOT EXISTS ( + SELECT 1 + FROM sandboxes s + WHERE s.id = local_artifacts.sandbox_id + )", + params![local_artifact_id], + ) +} diff --git a/crates/libs/sbx/mgr/server/src/db/local_artifacts.rs b/crates/libs/sbx/mgr/server/src/db/local_artifacts.rs index c84e446f5..1b364915c 100644 --- a/crates/libs/sbx/mgr/server/src/db/local_artifacts.rs +++ b/crates/libs/sbx/mgr/server/src/db/local_artifacts.rs @@ -465,7 +465,7 @@ pub fn publish_stopped_artifact( } // Clear the runtime first so the retention trigger observes its complete // identity, then close the exact attempt in the same transaction. - crate::route_activation_claims::close_claim(&tx, sandbox_id)?; + route_activation_claims::close_claim(&tx, sandbox_id)?; tx.commit()?; Ok(()) } diff --git a/crates/libs/sbx/mgr/server/src/db/pool.rs b/crates/libs/sbx/mgr/server/src/db/pool.rs index bc77641f7..7213c4ed5 100644 --- a/crates/libs/sbx/mgr/server/src/db/pool.rs +++ b/crates/libs/sbx/mgr/server/src/db/pool.rs @@ -132,7 +132,7 @@ fn restore_pool_donor( conn: &Connection, sandbox_id: SandboxId, donor: &SandboxRow, -) -> Result> { +) -> Result> { let id = sandbox_id.to_string(); set_pool_template(conn, &id, donor.pool_template.as_deref())?; update_user(conn, &id, &donor.user)?; @@ -189,16 +189,12 @@ pub fn restore_pool_donor_if_adopted( let receipt_supersession = match restore_pool_donor(&tx, sandbox_id, donor) { Ok(receipt_supersession) => receipt_supersession, Err(error) - if crate::auto_lifecycle_delete_receipts::db::is_pending_receipt_execution_fence( - &error, - ) => + if auto_lifecycle_delete_receipts::is_pending_receipt_execution_fence(&error) => { tx.rollback()?; return Ok(PoolDonorRestoreDisposition::ReceiptOwned); } - Err(error) - if crate::auto_lifecycle_delete_receipts::db::is_lifecycle_execution_fence(&error) => - { + Err(error) if auto_lifecycle_delete_receipts::is_lifecycle_execution_fence(&error) => { tx.rollback()?; return Ok(PoolDonorRestoreDisposition::LifecycleOwned); } diff --git a/crates/libs/sbx/mgr/server/src/db/quarantine.rs b/crates/libs/sbx/mgr/server/src/db/quarantine.rs index b4c360a2c..497e20b62 100644 --- a/crates/libs/sbx/mgr/server/src/db/quarantine.rs +++ b/crates/libs/sbx/mgr/server/src/db/quarantine.rs @@ -849,7 +849,7 @@ fn mark_quarantined_inner( ); } if changed != 0 { - crate::route_activation_claims::close_claim(&tx, id)?; + route_activation_claims::close_claim(&tx, id)?; let episode = QuarantineEpisode { sandbox_id: id, entered_from_state: previous_state.as_deref(), diff --git a/crates/libs/sbx/mgr/server/src/db/route_activation_claims.rs b/crates/libs/sbx/mgr/server/src/db/route_activation_claims.rs new file mode 100644 index 000000000..a220902b4 --- /dev/null +++ b/crates/libs/sbx/mgr/server/src/db/route_activation_claims.rs @@ -0,0 +1,142 @@ +//! Persistence of open Kubernetes route activation claims +//! (`sandbox_open_route_activations`). + +use pplx_space_common_core::types::{ + ExecutionGeneration, NodeId, NodeManagedBy, PodUid, RouteGeneration, + SandboxActivationAttemptId, SpaceApiRouteIdentity, +}; +use pplx_space_common_svc::sql::sqlite::SqliteU64; +use rusqlite::{Connection, params}; + +use crate::auto_lifecycle_delete_receipts::RouteIdentityProvenance; + +#[derive(Clone, Debug)] +/// Durable idempotency record for one exact Kubernetes activation. +/// +/// This is not an exclusive lease: the matching pod and attempt may continue +/// after a manager restart. A later activation is fenced by the retained +/// generation recorded when that activation actually begins. +pub(crate) struct OpenRouteActivationClaim { + route_generation: RouteGeneration, + node_id: NodeId, + execution_generation: ExecutionGeneration, + pod_uid: PodUid, + activation_attempt_id: SandboxActivationAttemptId, +} + +impl OpenRouteActivationClaim { + pub(crate) fn for_cluster_fenced_identity( + identity: &SpaceApiRouteIdentity, + provenance: RouteIdentityProvenance, + ) -> Option { + if provenance != RouteIdentityProvenance::ClusterFenced + || identity.substrate_kind != Some(NodeManagedBy::Kubernetes) + { + return None; + } + Some(Self { + route_generation: identity.route_generation?, + node_id: identity.current_node_id?, + execution_generation: identity.execution_generation?, + pod_uid: identity.pod_uid.clone()?, + activation_attempt_id: identity.activation_attempt_id?, + }) + } + + pub(crate) fn requested_generation(&self) -> u64 { + // ExecutionGeneration admits only positive i64 values. + self.route_generation + .get() + .min(self.execution_generation.get().unsigned_abs()) + } +} + +fn write_claim( + conn: &Connection, + sandbox_id: &str, + claim: &OpenRouteActivationClaim, +) -> rusqlite::Result<()> { + conn.execute( + "INSERT INTO sandbox_open_route_activations ( + sandbox_id, route_generation, node_id, execution_generation, + pod_uid, substrate_kind, activation_attempt_id, + identity_provenance + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) + ON CONFLICT(sandbox_id) DO UPDATE SET + route_generation = excluded.route_generation, + node_id = excluded.node_id, + execution_generation = excluded.execution_generation, + pod_uid = excluded.pod_uid, + substrate_kind = excluded.substrate_kind, + activation_attempt_id = excluded.activation_attempt_id, + identity_provenance = excluded.identity_provenance", + params![ + sandbox_id, + SqliteU64(claim.route_generation.get()), + claim.node_id.to_string(), + claim.execution_generation.get(), + claim.pod_uid.as_str(), + NodeManagedBy::Kubernetes.to_string(), + claim.activation_attempt_id.to_string(), + RouteIdentityProvenance::ClusterFenced.as_str(), + ], + )?; + Ok(()) +} + +pub(crate) fn insert_create_claim( + conn: &Connection, + sandbox_id: &str, + claim: &OpenRouteActivationClaim, +) -> rusqlite::Result<()> { + write_claim(conn, sandbox_id, claim) +} + +pub(crate) fn claim_matches( + conn: &Connection, + sandbox_id: &str, + claim: &OpenRouteActivationClaim, +) -> rusqlite::Result { + conn.query_row( + "SELECT EXISTS ( + SELECT 1 + FROM sandbox_open_route_activations + WHERE sandbox_id = ?1 + AND route_generation = ?2 + AND node_id = ?3 + AND execution_generation = ?4 + AND pod_uid = ?5 + AND substrate_kind = 'k8s' + AND activation_attempt_id = ?6 + AND identity_provenance = ?7 + )", + params![ + sandbox_id, + SqliteU64(claim.route_generation.get()), + claim.node_id.to_string(), + claim.execution_generation.get(), + claim.pod_uid.as_str(), + claim.activation_attempt_id.to_string(), + RouteIdentityProvenance::ClusterFenced.as_str(), + ], + |row| row.get(0), + ) +} + +pub(crate) fn close_claim(conn: &Connection, sandbox_id: &str) -> rusqlite::Result { + Ok(conn.execute( + "DELETE FROM sandbox_open_route_activations WHERE sandbox_id = ?1", + [sandbox_id], + )? == 1) +} + +#[cfg(test)] +pub(crate) fn claim_exists(conn: &Connection, sandbox_id: &str) -> rusqlite::Result { + conn.query_row( + "SELECT EXISTS ( + SELECT 1 FROM sandbox_open_route_activations WHERE sandbox_id = ?1 + )", + [sandbox_id], + |row| row.get(0), + ) +} diff --git a/crates/libs/sbx/mgr/server/src/db/sandboxes.rs b/crates/libs/sbx/mgr/server/src/db/sandboxes.rs index bea736b19..8538c7d4b 100644 --- a/crates/libs/sbx/mgr/server/src/db/sandboxes.rs +++ b/crates/libs/sbx/mgr/server/src/db/sandboxes.rs @@ -757,7 +757,7 @@ pub(crate) fn update_space_api_route_identity_in_transaction( sandbox_id: SandboxId, identity: &SpaceApiRouteIdentity, provenance: crate::auto_lifecycle_delete_receipts::RouteIdentityProvenance, -) -> Result> { +) -> Result> { let sandbox_id_string = sandbox_id.to_string(); let previous_generation = runtime_execution_generation(conn, &sandbox_id_string)?; let updated = conn.execute( @@ -800,13 +800,11 @@ pub(crate) fn update_space_api_route_identity_in_transaction( ], )?; if updated != 1 { - return Err( - crate::auto_lifecycle_delete_receipts::db::execution_write_miss_error( - conn, - sandbox_id, - identity.execution_generation, - ), - ); + return Err(auto_lifecycle_delete_receipts::execution_write_miss_error( + conn, + sandbox_id, + identity.execution_generation, + )); } reset_access_floor_for_execution_change( conn, @@ -817,15 +815,13 @@ pub(crate) fn update_space_api_route_identity_in_transaction( if let Some(evidence) = crate::auto_lifecycle_delete_receipts::RouteEvidence::from_identity(identity, provenance) { - crate::auto_lifecycle_delete_receipts::db::record_retained_route_evidence( - conn, sandbox_id, evidence, - )?; + auto_lifecycle_delete_receipts::record_retained_route_evidence(conn, sandbox_id, evidence)?; } // Provenance controls whether route evidence may mint a receipt. Once this // transaction persists a newer execution generation, an older receipt no // longer describes the row, including when the new route is incomplete. let receipt_supersession = - crate::auto_lifecycle_delete_receipts::db::discard_pending_receipt_for_execution_advance( + auto_lifecycle_delete_receipts::discard_pending_receipt_for_execution_advance( conn, sandbox_id, identity.execution_generation, @@ -954,13 +950,11 @@ fn update_runtime_fields_with_activation_owner( ], )?; if updated != 1 { - return Err( - crate::auto_lifecycle_delete_receipts::db::execution_write_miss_error( - &tx, - sandbox_id, - space_api_route_execution_generation, - ), - ); + return Err(auto_lifecycle_delete_receipts::execution_write_miss_error( + &tx, + sandbox_id, + space_api_route_execution_generation, + )); } reset_access_floor_for_execution_change(&tx, &id, previous_generation, execution_generation)?; if let (Some(execution_generation), Some(activation_attempt_id), Some(substrate)) = ( @@ -968,7 +962,7 @@ fn update_runtime_fields_with_activation_owner( space_api_route_activation_attempt_id, space_api_route_substrate_kind, ) { - crate::auto_lifecycle_delete_receipts::db::record_retained_route_evidence( + auto_lifecycle_delete_receipts::record_retained_route_evidence( &tx, sandbox_id, crate::auto_lifecycle_delete_receipts::RouteEvidence { @@ -980,7 +974,7 @@ fn update_runtime_fields_with_activation_owner( )?; } let receipt_supersession = - crate::auto_lifecycle_delete_receipts::db::discard_pending_receipt_for_execution_advance( + auto_lifecycle_delete_receipts::discard_pending_receipt_for_execution_advance( &tx, sandbox_id, space_api_route_execution_generation, @@ -1408,7 +1402,7 @@ pub fn delete_sandbox(conn: &Connection, id: &str) -> Result<()> { let tx = immediate_transaction(conn)?; delete_sandbox_child_rows(&tx, id)?; tx.execute("DELETE FROM sandboxes WHERE id = ?1", params![id]) - .map_err(crate::auto_lifecycle_delete_receipts::db::map_pending_receipt_delete_error)?; + .map_err(auto_lifecycle_delete_receipts::map_pending_receipt_delete_error)?; tx.commit()?; Ok(()) } @@ -1506,7 +1500,7 @@ pub(crate) fn delete_sandbox_if_inventory_witness_matches( delete_sandbox_child_rows(&tx, &observed.id)?; let deleted = tx .execute("DELETE FROM sandboxes WHERE id = ?1", params![&observed.id]) - .map_err(crate::auto_lifecycle_delete_receipts::db::map_pending_receipt_delete_error)?; + .map_err(auto_lifecycle_delete_receipts::map_pending_receipt_delete_error)?; if deleted != 1 { return Ok(InventoryStaleRowMutation::WitnessChanged); } diff --git a/crates/libs/sbx/mgr/server/src/db/snapshot_upload_jobs.rs b/crates/libs/sbx/mgr/server/src/db/snapshot_upload_jobs.rs new file mode 100644 index 000000000..7b4c8c33f --- /dev/null +++ b/crates/libs/sbx/mgr/server/src/db/snapshot_upload_jobs.rs @@ -0,0 +1,87 @@ +//! `jobs` rows of the snapshot-upload pipeline, and the backup markers its +//! completed jobs prove. + +use pplx_space_common_core::types::JobId; +use rusqlite::{Connection, Result, params}; + +/// The `jobs.kind` of the snapshot-upload pipeline's jobs. +pub const SNAPSHOT_UPLOAD_JOB_KIND: &str = "snapshot_upload"; + +/// Give back the attempt a restart interrupted: every running job of `kind` +/// with attempts left loses one; the number refunded. +pub(crate) fn refund_interrupted_running_attempts(conn: &Connection, kind: &str) -> Result { + conn.execute( + "UPDATE jobs + SET attempts = attempts - 1 + WHERE kind = ?1 AND status = 'running' AND attempts > 0", + params![kind], + ) +} + +/// Replace a pending or running job's payload; the number of rows changed. +pub(crate) fn replace_active_job_payload( + conn: &Connection, + payload: &str, + job_id: &JobId, + kind: &str, +) -> Result { + conn.execute( + "UPDATE jobs SET payload = ?1 + WHERE job_id = ?2 AND kind = ?3 AND status IN ('pending', 'running')", + params![payload, job_id, kind], + ) +} + +/// The highest job rowid, `None` for an empty table. +pub(crate) fn max_job_rowid(conn: &Connection) -> Result> { + conn.query_row("SELECT MAX(rowid) FROM jobs", [], |r| r.get(0)) +} + +/// Visit up to `limit` completed snapshot-upload jobs with rowids in +/// `(after, through]`, in rowid order, with the scan's timestamp, each rowid +/// and its payload or the error reading it. The timestamp is taken once the +/// statement is bound, before the first row is read. The statement stays open +/// while `visit` runs, so `visit` may write through the same connection; its +/// error stops the scan. +pub(crate) fn visit_completed_upload_payloads( + conn: &Connection, + after: i64, + through: i64, + limit: i64, + mut visit: impl FnMut(&str, i64, Result) -> Result<()>, +) -> Result<()> { + let mut statement = conn.prepare( + "SELECT rowid, payload FROM jobs + WHERE kind = 'snapshot_upload' AND status = 'completed' + AND rowid > ?1 AND rowid <= ?2 ORDER BY rowid LIMIT ?3", + )?; + let mut rows = statement.query(params![after, through, limit])?; + let now = crate::db::now()?; + while let Some(row) = rows.next()? { + visit(&now, row.get(0)?, row.get::<_, String>(1))?; + } + Ok(()) +} + +/// Stamp `backup_completed_at` on an artifact that has none; the number of +/// rows changed. +pub(crate) fn stamp_backup_completed_if_unset( + conn: &Connection, + now: &str, + local_artifact_id: &str, +) -> Result { + conn.execute( + "UPDATE local_artifacts SET backup_completed_at = ?1 + WHERE local_artifact_id = ?2 AND backup_completed_at IS NULL", + params![now, local_artifact_id], + ) +} + +/// Whether an artifact row exists. +pub(crate) fn local_artifact_exists(conn: &Connection, local_artifact_id: &str) -> Result { + conn.query_row( + "SELECT EXISTS(SELECT 1 FROM local_artifacts WHERE local_artifact_id = ?1)", + params![local_artifact_id], + |r| r.get(0), + ) +} diff --git a/crates/libs/sbx/mgr/server/src/db/startup_recovery.rs b/crates/libs/sbx/mgr/server/src/db/startup_recovery.rs index e4f4ef8f0..132c9682b 100644 --- a/crates/libs/sbx/mgr/server/src/db/startup_recovery.rs +++ b/crates/libs/sbx/mgr/server/src/db/startup_recovery.rs @@ -125,7 +125,7 @@ pub(crate) fn settle_startup_resident( message, } = settlement { - crate::route_activation_claims::close_claim(&tx, &observed.id)?; + route_activation_claims::close_claim(&tx, &observed.id)?; insert_new_quarantine_episode( &tx, &QuarantineEpisode { diff --git a/crates/libs/sbx/mgr/server/src/db/state_transitions.rs b/crates/libs/sbx/mgr/server/src/db/state_transitions.rs index d373b145b..7ab22008a 100644 --- a/crates/libs/sbx/mgr/server/src/db/state_transitions.rs +++ b/crates/libs/sbx/mgr/server/src/db/state_transitions.rs @@ -229,7 +229,7 @@ pub(crate) struct ActivationRouteAdmission<'a> { pub(crate) auto_lifecycle: Option<&'a AutoLifecyclePolicy>, pub(crate) operation_activation_attempt_id: Option, pub(crate) evidence: Option, - pub(crate) open_claim: Option<&'a crate::route_activation_claims::OpenRouteActivationClaim>, + pub(crate) open_claim: Option<&'a route_activation_claims::OpenRouteActivationClaim>, } pub(crate) fn begin_activation_with_route_evidence( @@ -322,28 +322,26 @@ pub(crate) fn begin_activation_with_route_evidence( .transpose()?; match route_admission.open_claim { Some(open_route_activation) => { - crate::route_activation_claims::insert_create_claim(&tx, id, open_route_activation)?; + route_activation_claims::insert_create_claim(&tx, id, open_route_activation)?; } None => { // Beginning any other activation supersedes an older Kubernetes // continuation ticket in the same transaction that claims the row. - crate::route_activation_claims::close_claim(&tx, id)?; + route_activation_claims::close_claim(&tx, id)?; } } if let Some(route_evidence) = route_admission.evidence { let sandbox_id = sandbox_id.ok_or(rusqlite::Error::InvalidQuery)?; - crate::auto_lifecycle_delete_receipts::db::record_retained_route_evidence( + auto_lifecycle_delete_receipts::record_retained_route_evidence( &tx, sandbox_id, route_evidence, )?; } let receipt_supersession = match sandbox_id { - Some(sandbox_id) => crate::auto_lifecycle_delete_receipts::db::discard_pending_receipt_for_execution_advance( - &tx, - sandbox_id, - execution_generation, - )?, + Some(sandbox_id) => { + discard_pending_receipt_for_execution_advance(&tx, sandbox_id, execution_generation)? + } None => None, }; let cleanup_id = SandboxCleanupEpisodeId::new(); @@ -659,7 +657,7 @@ pub fn mark_start_succeeded( arm_auto_lifecycle_after_activation(&tx, id)?; } if changed == 1 { - crate::route_activation_claims::close_claim(&tx, id)?; + route_activation_claims::close_claim(&tx, id)?; } tx.commit()?; Ok(changed == 1) @@ -792,7 +790,7 @@ pub fn mark_restore_succeeded( arm_auto_lifecycle_after_activation(&tx, id)?; } if changed == 1 { - crate::route_activation_claims::close_claim(&tx, id)?; + route_activation_claims::close_claim(&tx, id)?; } tx.commit()?; if changed == 1 { diff --git a/crates/libs/sbx/mgr/server/src/local_artifacts.rs b/crates/libs/sbx/mgr/server/src/local_artifacts.rs index 48ceb2efc..db5970302 100644 --- a/crates/libs/sbx/mgr/server/src/local_artifacts.rs +++ b/crates/libs/sbx/mgr/server/src/local_artifacts.rs @@ -61,82 +61,13 @@ use std::path::Path; use std::sync::{Arc, Mutex, MutexGuard}; -use pplx_space_common_core::sql; use pplx_space_common_core::types::{ArtifactKind, LocalArtifactId, SnapshotName}; -use pplx_space_common_svc::sql::sqlite::SqliteU64; -use rusqlite::{Connection, OptionalExtension, params}; +use rusqlite::Connection; use crate::db::LocalArtifactRow; - -/// One sandbox's current lifecycle artifact that this node cannot prove has -/// a durable remote copy. -/// -/// Identity comes from the `sandboxes` pin rather than from the artifact -/// row, because the case where there *is* no artifact row is one of the -/// cases this reports. Migration 026's CHECK keeps `current_local_artifact_id` -/// and `current_local_artifact_kind` set or unset together, so the pin alone -/// always carries a whole identity. -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct PendingBackup { - pub sandbox_id: String, - pub local_artifact_id: LocalArtifactId, - pub artifact_kind: ArtifactKind, -} - -/// Logical state of a `local_artifacts` row. Encoded on the row's -/// `state` column; transitions are CAS via -/// [`LocalArtifacts::check_out_for_restore`] / -/// [`LocalArtifacts::check_out_for_delete`]. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ArtifactState { - /// Published and available. The only state from which a checkout - /// (restore or delete) can succeed. - Ready, - /// A restore is currently reading this artifact's `local_path`. - /// Reapers must skip; deleters must wait or fail with NotReady. - Restoring, - /// A delete has reserved this row. The on-disk directory has - /// usually been renamed to a `.gc-pending-` sibling. Readers - /// should treat the artifact as gone. - Deleting, -} - -impl ArtifactState { - pub fn as_str(self) -> &'static str { - match self { - ArtifactState::Ready => "ready", - ArtifactState::Restoring => "restoring", - ArtifactState::Deleting => "deleting", - } - } -} - -/// Error parsing an [`ArtifactState`] from its wire/DB string. The -/// CHECK constraint on `local_artifacts.state` should make this -/// unreachable for rows the schema accepted; it surfaces table -/// corruption or migration drift. -#[derive(Debug, thiserror::Error)] -#[error("unrecognized artifact state {0:?}")] -pub struct ParseArtifactStateError(String); - -impl std::str::FromStr for ArtifactState { - type Err = ParseArtifactStateError; - - fn from_str(s: &str) -> Result { - match s { - "ready" => Ok(ArtifactState::Ready), - "restoring" => Ok(ArtifactState::Restoring), - "deleting" => Ok(ArtifactState::Deleting), - _ => Err(ParseArtifactStateError(s.to_string())), - } - } -} - -impl std::fmt::Display for ArtifactState { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(self.as_str()) - } -} +use crate::db::local_artifact_leases as leases; +pub(crate) use crate::db::local_artifact_leases::NamedArtifactPublication; +pub use crate::db::local_artifact_leases::{ArtifactState, ParseArtifactStateError, PendingBackup}; /// Reason a checkout (restore or delete) failed. #[derive(Debug, thiserror::Error)] @@ -208,14 +139,6 @@ pub struct LocalArtifacts { orphan_scan: Arc>, } -/// Point-in-time publication evidence for a named snapshot, not a checkout lease. -pub(crate) struct NamedArtifactPublication { - pub snapshot_name: SnapshotName, - pub local_path: String, - pub artifact_kind: ArtifactKind, - pub state: ArtifactState, -} - #[derive(Clone, Copy)] struct OrphanScan { next: i64, @@ -371,36 +294,7 @@ impl LocalArtifacts { /// with no report, which every consumer already has to handle. pub fn list_pending_backups(&self) -> rusqlite::Result> { let conn = self.db_conn()?; - conn.prepare( - "SELECT s.id, s.current_local_artifact_id, s.current_local_artifact_kind - FROM sandboxes AS s - LEFT JOIN local_artifacts AS a - ON a.local_artifact_id = s.current_local_artifact_id - AND a.sandbox_id = s.id - WHERE s.current_local_artifact_id IS NOT NULL - AND (a.local_artifact_id IS NULL OR a.backup_completed_at IS NULL) - ORDER BY s.id", - )? - .query_map([], |row| { - Ok(PendingBackup { - sandbox_id: row.get(0)?, - local_artifact_id: row.get::<_, String>(1)?.parse().map_err(|error| { - rusqlite::Error::FromSqlConversionFailure( - 1, - rusqlite::types::Type::Text, - Box::new(error), - ) - })?, - artifact_kind: row.get::<_, String>(2)?.parse().map_err(|error| { - rusqlite::Error::FromSqlConversionFailure( - 2, - rusqlite::types::Type::Text, - Box::new(error), - ) - })?, - }) - })? - .collect() + leases::list_pending_backups(&conn) } /// Inventory-only observation. Callers must still acquire a checkout before @@ -409,43 +303,8 @@ impl LocalArtifacts { &self, sandbox_id: &str, ) -> Result, crate::db::inventory_read::InventoryReadError> { - use pplx_space_common_core::inventory::{InventoryBudget, MAX_ITEMS, MAX_RAW_BYTES}; let conn = self.db_conn()?; - let mut statement = conn.prepare(sql!( - "SELECT snapshot_name, local_path, artifact_kind, state - FROM local_artifacts WHERE sandbox_id = ?1 AND artifact_origin = 'snapshot'" - ))?; - let mut rows = statement.query([sandbox_id])?; - let mut budget = InventoryBudget::new(MAX_ITEMS, MAX_RAW_BYTES); - let mut publications = Vec::new(); - while let Some(row) = rows.next()? { - crate::db::inventory_read::check_row(row, &mut budget)?; - publications.push(NamedArtifactPublication { - snapshot_name: row.get::<_, String>(0)?.parse().map_err(|error| { - rusqlite::Error::FromSqlConversionFailure( - 0, - rusqlite::types::Type::Text, - Box::new(error), - ) - })?, - local_path: row.get(1)?, - artifact_kind: row.get::<_, String>(2)?.parse().map_err(|error| { - rusqlite::Error::FromSqlConversionFailure( - 2, - rusqlite::types::Type::Text, - Box::new(error), - ) - })?, - state: row.get::<_, String>(3)?.parse().map_err(|error| { - rusqlite::Error::FromSqlConversionFailure( - 3, - rusqlite::types::Type::Text, - Box::new(error), - ) - })?, - }); - } - Ok(publications) + leases::list_named_publications(&conn, sandbox_id) } /// Mark the artifact's remote backup as durably completed. This @@ -551,13 +410,7 @@ impl LocalArtifacts { let tx = rusqlite::Transaction::new_unchecked(&conn, rusqlite::TransactionBehavior::Immediate)?; let id_str = local_artifact_id.to_string(); - let row: Option<(String, String)> = tx - .query_row( - "SELECT local_path, state FROM local_artifacts WHERE local_artifact_id = ?1", - params![id_str], - |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)), - ) - .optional()?; + let row = leases::artifact_path_and_state(&tx, &id_str)?; let Some((local_path, state)) = row else { return Err(CheckoutError::NotFound(local_artifact_id)); }; @@ -576,11 +429,7 @@ impl LocalArtifacts { // CAS — only flip if still ready. If a concurrent path won // the race between the SELECT and the UPDATE, the UPDATE // affects 0 rows and we surface NotReady. - let changed = tx.execute( - "UPDATE local_artifacts SET state = 'restoring' - WHERE local_artifact_id = ?1 AND state = 'ready'", - params![id_str], - )?; + let changed = leases::mark_artifact_restoring(&tx, &id_str)?; if changed == 0 { return Err(CheckoutError::NotReady { id: local_artifact_id, @@ -631,32 +480,10 @@ impl LocalArtifacts { sandbox_id: &str, eligibility: GcEligibility, ) -> Result { - use crate::db::timestamps::EpochSeconds; - let conn = self.db_conn()?; let tx = rusqlite::Transaction::new_unchecked(&conn, rusqlite::TransactionBehavior::Immediate)?; - let row = tx - .query_row( - "SELECT created_at, COALESCE(last_touched_at, created_at), - EXISTS( - SELECT 1 FROM sandboxes - WHERE current_local_artifact_id = ?1 - OR suspend_predecessor_artifact_id = ?1 - ), - backup_completed_at IS NOT NULL - FROM local_artifacts WHERE local_artifact_id = ?1 AND sandbox_id = ?2", - params![local_artifact_id.to_string(), sandbox_id], - |row| { - Ok(( - row.get::<_, EpochSeconds>(0)?, - row.get::<_, EpochSeconds>(1)?, - row.get::<_, bool>(2)?, - row.get::<_, bool>(3)?, - )) - }, - ) - .optional()?; + let row = leases::gc_checkout_evidence(&tx, &local_artifact_id.to_string(), sandbox_id)?; let Some((created_at, touched_at, pinned, backed_up)) = row else { return Err(CheckoutError::NotFound(local_artifact_id)); }; @@ -694,24 +521,11 @@ impl LocalArtifacts { let id_str = local_artifact_id.to_string(); // The IMMEDIATE transaction serializes pin changes with checkout for // every deleter, including named-snapshot deletion outside GC. - let pinned: bool = tx.query_row( - sql!( - "SELECT EXISTS(SELECT 1 FROM sandboxes - WHERE current_local_artifact_id = ?1 OR suspend_predecessor_artifact_id = ?1)" - ), - [&id_str], - |row| row.get(0), - )?; + let pinned = leases::artifact_is_pinned(tx, &id_str)?; if pinned { return Err(CheckoutError::Pinned(local_artifact_id)); } - let row: Option<(String, String)> = tx - .query_row( - "SELECT local_path, state FROM local_artifacts WHERE local_artifact_id = ?1", - params![id_str], - |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)), - ) - .optional()?; + let row = leases::artifact_path_and_state(tx, &id_str)?; let Some((local_path, state)) = row else { return Err(CheckoutError::NotFound(local_artifact_id)); }; @@ -729,22 +543,14 @@ impl LocalArtifacts { } // Shared-path safety gate. The reaper's old plan-time // check moved here so DeleteSnapshots gets it too. - let path_count: u64 = tx.query_row( - "SELECT COUNT(*) FROM local_artifacts WHERE local_path = ?1", - params![local_path], - |r| Ok(r.get::<_, SqliteU64>(0)?.0), - )?; + let path_count = leases::count_artifacts_at_path(tx, &local_path)?; if path_count != 1 { return Err(CheckoutError::SharedLocalPath { id: local_artifact_id, sibling_count: path_count.saturating_sub(1), }); } - let changed = tx.execute( - "UPDATE local_artifacts SET state = 'deleting' - WHERE local_artifact_id = ?1 AND state = 'ready'", - params![id_str], - )?; + let changed = leases::mark_artifact_deleting(tx, &id_str)?; if changed == 0 { return Err(CheckoutError::NotReady { id: local_artifact_id, @@ -814,17 +620,8 @@ impl LocalArtifacts { let conn = self.db_conn()?; let tx = rusqlite::Transaction::new_unchecked(&conn, rusqlite::TransactionBehavior::Immediate)?; - let restoring_reset = tx.execute( - "UPDATE local_artifacts SET state = 'ready' WHERE state = 'restoring'", - [], - )?; - let stuck_deleting: Vec<(String, String)> = tx - .prepare( - "SELECT local_artifact_id, local_path FROM local_artifacts - WHERE state = 'deleting'", - )? - .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? - .collect::>()?; + let restoring_reset = leases::reset_restoring_artifacts(&tx)?; + let stuck_deleting = leases::list_deleting_artifacts(&tx)?; let mut deleting_rolled_back = 0usize; let mut deleting_finished = 0usize; let mut deleting_deferred = 0usize; @@ -842,32 +639,16 @@ impl LocalArtifacts { continue; } }; - let pinned: bool = tx.query_row( - "SELECT EXISTS( - SELECT 1 FROM sandboxes - WHERE current_local_artifact_id = ?1 - OR suspend_predecessor_artifact_id = ?1 - )", - params![id], - |r| r.get(0), - )?; + let pinned = leases::artifact_is_pinned(&tx, &id)?; if original_present && !staging_present { - tx.execute( - "UPDATE local_artifacts SET state = 'ready' - WHERE local_artifact_id = ?1 AND state = 'deleting'", - params![id], - )?; + leases::roll_back_deleting_artifact(&tx, &id)?; deleting_rolled_back += 1; } else if pinned { tracing::warn!(artifact_id = %id, original_present, staging_present, "retaining pinned deleting artifact evidence; explicit restore or cleanup required"); deleting_deferred += 1; } else { - tx.execute( - "DELETE FROM local_artifacts - WHERE local_artifact_id = ?1 AND state = 'deleting'", - params![id], - )?; + leases::delete_deleting_artifact(&tx, &id)?; deleting_finished += 1; } } @@ -883,15 +664,8 @@ impl LocalArtifacts { /// GC staging is disposable only after both its registry and pin are gone. /// Retain rows in every state, including historical deletion conflicts. pub(crate) fn gc_staging_is_unowned(&self, id: LocalArtifactId) -> rusqlite::Result { - self.db_conn()?.query_row( - sql!( - "SELECT NOT EXISTS(SELECT 1 FROM local_artifacts WHERE local_artifact_id = ?1) - AND NOT EXISTS(SELECT 1 FROM sandboxes - WHERE current_local_artifact_id = ?1 OR suspend_predecessor_artifact_id = ?1)" - ), - [id.to_string()], - |row| row.get(0), - ) + let conn = self.db_conn()?; + leases::gc_staging_is_unowned(&conn, &id.to_string()) } /// Examine at most `max_records` raw rows, advancing past owned, present, @@ -911,34 +685,17 @@ impl LocalArtifacts { let tx = rusqlite::Transaction::new_unchecked(&conn, rusqlite::TransactionBehavior::Immediate)?; if scan.through.is_none() { - scan.through = - tx.query_row("SELECT MAX(rowid) FROM local_artifacts", [], |r| r.get(0))?; + scan.through = leases::max_artifact_rowid(&tx)?; } let Some(through) = scan.through else { return Ok(0); }; - let rows = tx - .prepare( - "SELECT la.rowid, la.local_artifact_id, la.local_path, - EXISTS (SELECT 1 FROM sandboxes s WHERE s.id = la.sandbox_id) - FROM local_artifacts la - WHERE la.rowid >= ?1 AND la.rowid <= ?2 - ORDER BY la.rowid LIMIT ?3", - )? - .query_map( - params![ - scan.next, - through, - i64::try_from(max_records).unwrap_or(i64::MAX) - ], - |row| { - let decoded = (|| -> rusqlite::Result<(String, String, bool)> { - Ok((row.get(1)?, row.get(2)?, row.get(3)?)) - })(); - Ok((row.get::<_, i64>(0)?, decoded)) - }, - )? - .collect::>>()?; + let rows = leases::orphan_scan_rows( + &tx, + scan.next, + through, + i64::try_from(max_records).unwrap_or(i64::MAX), + )?; let mut deleted = 0usize; let examined = rows.len(); for (rowid, decoded) in rows { @@ -964,16 +721,7 @@ impl LocalArtifacts { continue; } } - deleted += tx.execute( - "DELETE FROM local_artifacts - WHERE local_artifact_id = ?1 - AND NOT EXISTS ( - SELECT 1 - FROM sandboxes s - WHERE s.id = local_artifacts.sandbox_id - )", - params![id], - )?; + deleted += leases::delete_orphan_artifact_row(&tx, &id)?; } if examined < max_records { scan = OrphanScan::default(); @@ -1055,12 +803,7 @@ impl RestoreLease { fn release_with_touch(&self) -> rusqlite::Result<()> { let conn = lock_db(&self.db)?; - conn.execute( - "UPDATE local_artifacts - SET state = 'ready', last_touched_at = ?1 - WHERE local_artifact_id = ?2 AND state = 'restoring'", - params![crate::db::now()?, self.id.to_string()], - )?; + leases::release_restoring_artifact_with_touch(&conn, &self.id.to_string())?; Ok(()) } @@ -1071,11 +814,7 @@ impl RestoreLease { /// idle clock should reflect that. fn release_without_touch(&self) -> rusqlite::Result<()> { let conn = lock_db(&self.db)?; - conn.execute( - "UPDATE local_artifacts SET state = 'ready' - WHERE local_artifact_id = ?1 AND state = 'restoring'", - params![self.id.to_string()], - )?; + leases::release_restoring_artifact(&conn, &self.id.to_string())?; Ok(()) } } @@ -1121,11 +860,7 @@ impl DeleteLease { /// than a foot-gun. pub fn commit(mut self) -> rusqlite::Result<()> { let conn = lock_db(&self.db)?; - let deleted = conn.execute( - "DELETE FROM local_artifacts - WHERE local_artifact_id = ?1 AND state = 'deleting'", - params![self.id.to_string()], - )?; + let deleted = leases::delete_deleting_artifact(&conn, &self.id.to_string())?; drop(conn); if deleted == 0 { tracing::warn!( @@ -1154,11 +889,7 @@ impl Drop for DeleteLease { return; } }; - if let Err(error) = conn.execute( - "UPDATE local_artifacts SET state = 'ready' - WHERE local_artifact_id = ?1 AND state = 'deleting'", - params![self.id.to_string()], - ) { + if let Err(error) = leases::roll_back_deleting_artifact(&conn, &self.id.to_string()) { tracing::warn!( local_artifact_id = %self.id, %error, diff --git a/crates/libs/sbx/mgr/server/src/local_artifacts/refusal_tests.rs b/crates/libs/sbx/mgr/server/src/local_artifacts/refusal_tests.rs index 38a4b00f0..8d9e76790 100644 --- a/crates/libs/sbx/mgr/server/src/local_artifacts/refusal_tests.rs +++ b/crates/libs/sbx/mgr/server/src/local_artifacts/refusal_tests.rs @@ -3,6 +3,8 @@ use std::sync::PoisonError; +use rusqlite::{OptionalExtension, params}; + use super::*; const ID: &str = "01939e3e-aaaa-7000-8000-000000000001"; diff --git a/crates/libs/sbx/mgr/server/src/local_artifacts/tests.rs b/crates/libs/sbx/mgr/server/src/local_artifacts/tests.rs index 7780fddca..3dcac7287 100644 --- a/crates/libs/sbx/mgr/server/src/local_artifacts/tests.rs +++ b/crates/libs/sbx/mgr/server/src/local_artifacts/tests.rs @@ -1,4 +1,5 @@ use pplx_space_common_core::types::SandboxId; +use rusqlite::params; use super::*; diff --git a/crates/libs/sbx/mgr/server/src/route_activation_claims.rs b/crates/libs/sbx/mgr/server/src/route_activation_claims.rs index cd084ef45..6c49856c5 100644 --- a/crates/libs/sbx/mgr/server/src/route_activation_claims.rs +++ b/crates/libs/sbx/mgr/server/src/route_activation_claims.rs @@ -1,52 +1,6 @@ -use pplx_space_common_core::types::{ - ExecutionGeneration, NodeId, NodeManagedBy, PodUid, RouteGeneration, - SandboxActivationAttemptId, SpaceApiRouteIdentity, -}; -use pplx_space_common_svc::sql::sqlite::SqliteU64; -use rusqlite::{Connection, params}; +use rusqlite::Connection; -use crate::auto_lifecycle_delete_receipts::RouteIdentityProvenance; - -#[derive(Clone, Debug)] -/// Durable idempotency record for one exact Kubernetes activation. -/// -/// This is not an exclusive lease: the matching pod and attempt may continue -/// after a manager restart. A later activation is fenced by the retained -/// generation recorded when that activation actually begins. -pub(crate) struct OpenRouteActivationClaim { - route_generation: RouteGeneration, - node_id: NodeId, - execution_generation: ExecutionGeneration, - pod_uid: PodUid, - activation_attempt_id: SandboxActivationAttemptId, -} - -impl OpenRouteActivationClaim { - pub(crate) fn for_cluster_fenced_identity( - identity: &SpaceApiRouteIdentity, - provenance: RouteIdentityProvenance, - ) -> Option { - if provenance != RouteIdentityProvenance::ClusterFenced - || identity.substrate_kind != Some(NodeManagedBy::Kubernetes) - { - return None; - } - Some(Self { - route_generation: identity.route_generation?, - node_id: identity.current_node_id?, - execution_generation: identity.execution_generation?, - pod_uid: identity.pod_uid.clone()?, - activation_attempt_id: identity.activation_attempt_id?, - }) - } - - fn requested_generation(&self) -> u64 { - // ExecutionGeneration admits only positive i64 values. - self.route_generation - .get() - .min(self.execution_generation.get().unsigned_abs()) - } -} +use crate::db::route_activation_claims::{OpenRouteActivationClaim, claim_matches}; #[derive(Debug, thiserror::Error)] pub(crate) enum OpenRouteActivationAdmissionError { @@ -58,47 +12,6 @@ pub(crate) enum OpenRouteActivationAdmissionError { StaleOrConflicting { retained: u64 }, } -fn write_claim( - conn: &Connection, - sandbox_id: &str, - claim: &OpenRouteActivationClaim, -) -> rusqlite::Result<()> { - conn.execute( - "INSERT INTO sandbox_open_route_activations ( - sandbox_id, route_generation, node_id, execution_generation, - pod_uid, substrate_kind, activation_attempt_id, - identity_provenance - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) - ON CONFLICT(sandbox_id) DO UPDATE SET - route_generation = excluded.route_generation, - node_id = excluded.node_id, - execution_generation = excluded.execution_generation, - pod_uid = excluded.pod_uid, - substrate_kind = excluded.substrate_kind, - activation_attempt_id = excluded.activation_attempt_id, - identity_provenance = excluded.identity_provenance", - params![ - sandbox_id, - SqliteU64(claim.route_generation.get()), - claim.node_id.to_string(), - claim.execution_generation.get(), - claim.pod_uid.as_str(), - NodeManagedBy::Kubernetes.to_string(), - claim.activation_attempt_id.to_string(), - RouteIdentityProvenance::ClusterFenced.as_str(), - ], - )?; - Ok(()) -} - -pub(crate) fn insert_create_claim( - conn: &Connection, - sandbox_id: &str, - claim: &OpenRouteActivationClaim, -) -> rusqlite::Result<()> { - write_claim(conn, sandbox_id, claim) -} - /// Highest live or retained generation observed on this node. /// /// Cluster-fenced claims reject a different attempt at the live generation; @@ -121,37 +34,6 @@ fn observed_generation_floor(provenance: crate::db::ActivationRouteProvenance) - .max() } -fn claim_matches( - conn: &Connection, - sandbox_id: &str, - claim: &OpenRouteActivationClaim, -) -> rusqlite::Result { - conn.query_row( - "SELECT EXISTS ( - SELECT 1 - FROM sandbox_open_route_activations - WHERE sandbox_id = ?1 - AND route_generation = ?2 - AND node_id = ?3 - AND execution_generation = ?4 - AND pod_uid = ?5 - AND substrate_kind = 'k8s' - AND activation_attempt_id = ?6 - AND identity_provenance = ?7 - )", - params![ - sandbox_id, - SqliteU64(claim.route_generation.get()), - claim.node_id.to_string(), - claim.execution_generation.get(), - claim.pod_uid.as_str(), - claim.activation_attempt_id.to_string(), - RouteIdentityProvenance::ClusterFenced.as_str(), - ], - |row| row.get(0), - ) -} - pub(crate) fn validate_cluster_fenced_activation( conn: &Connection, sandbox_id: &str, @@ -171,21 +53,3 @@ pub(crate) fn validate_cluster_fenced_activation( Ok(()) } - -pub(crate) fn close_claim(conn: &Connection, sandbox_id: &str) -> rusqlite::Result { - Ok(conn.execute( - "DELETE FROM sandbox_open_route_activations WHERE sandbox_id = ?1", - [sandbox_id], - )? == 1) -} - -#[cfg(test)] -pub(crate) fn claim_exists(conn: &Connection, sandbox_id: &str) -> rusqlite::Result { - conn.query_row( - "SELECT EXISTS ( - SELECT 1 FROM sandbox_open_route_activations WHERE sandbox_id = ?1 - )", - [sandbox_id], - |row| row.get(0), - ) -} diff --git a/crates/libs/sbx/mgr/server/src/sandbox_eviction.rs b/crates/libs/sbx/mgr/server/src/sandbox_eviction.rs index 92846525a..89666b80d 100644 --- a/crates/libs/sbx/mgr/server/src/sandbox_eviction.rs +++ b/crates/libs/sbx/mgr/server/src/sandbox_eviction.rs @@ -90,13 +90,13 @@ use std::collections::HashMap; use std::path::Path; use std::time::{Duration, Instant}; -use pplx_space_common_svc::sql::sqlite::SqliteU64; -use rusqlite::{Connection, OptionalExtension, params}; +use rusqlite::Connection; use tokio_util::sync::CancellationToken; use tracing::{info, warn}; use uuid::Uuid; pub(crate) use crate::auto_lifecycle::now_secs; +use crate::db::eviction_scan::{count_malformed_suspended_updated_at, count_resident_rows}; use crate::service::SandboxManagerService; /// Runtime configuration for the eviction loop. Loaded from env on @@ -517,59 +517,6 @@ impl CandidateScanState { } } -const VALID_SUSPENDED_UPDATED_AT_SQL: &str = - "s.updated_at <> '' AND s.updated_at NOT GLOB '*[^0-9]*'"; - -/// Idle witness for eviction candidacy: the sandbox's last real -/// activity, not the row's last write. `updated_at` has an unbounded -/// writer set (periodic reconcilers rewrite rows value-identically and -/// stamp it), so a row can look seconds-old for weeks while its bytes -/// sit cold — the activity clocks inside `auto_lifecycle_state_json` -/// only advance on genuine touches and transitions. NULL-safe two-way -/// max of `last_access_at`/`last_command_at`; rows carrying neither -/// clock (operator/legacy/orphan-adopted rows) fall back to -/// `updated_at`, preserving their previous behavior exactly. The -/// `json_valid` gate is load-bearing: `json_extract` aborts the whole -/// statement on malformed JSON, so without it one corrupt row would -/// fail every scan and shield the valid cold rows around it — a -/// malformed blob falls back to `updated_at` like a missing one. Each -/// clock is additionally type-gated to a non-negative integer: our -/// writers only ever store u64 epoch seconds, so anything else -/// (negative, fractional, string) is corruption and must read as -/// absent rather than fail the `SqliteU64` row decode. The CAS claim -/// deliberately keeps raw `updated_at`: "any write means the row is in -/// flux" is the right semantic there and the wrong one here. -/// -/// The partial eviction index prunes to suspended rows before evaluating -/// this expression. If scan cost grows with node row counts, the escape -/// hatch is an index on this deterministic expression or a virtual generated -/// column — not a writer-maintained copy, which would reintroduce the drift -/// this witness exists to escape. -/// Scalar `MAX` returns NULL if ANY argument is NULL, so absent or -/// corrupt clocks are mapped to a -1 sentinel first (real clocks are -/// type-gated non-negative, so the sentinel cannot collide) and -/// `NULLIF` restores NULL when both were absent. -const IDLE_WITNESS_SQL: &str = "COALESCE( - CASE - WHEN json_valid(s.auto_lifecycle_state_json) THEN - NULLIF( - MAX( - COALESCE(CASE - WHEN typeof(json_extract(s.auto_lifecycle_state_json, '$.last_access_at')) = 'integer' - AND json_extract(s.auto_lifecycle_state_json, '$.last_access_at') >= 0 - THEN json_extract(s.auto_lifecycle_state_json, '$.last_access_at') - END, -1), - COALESCE(CASE - WHEN typeof(json_extract(s.auto_lifecycle_state_json, '$.last_command_at')) = 'integer' - AND json_extract(s.auto_lifecycle_state_json, '$.last_command_at') >= 0 - THEN json_extract(s.auto_lifecycle_state_json, '$.last_command_at') - END, -1) - ), - -1) - END, - CAST(s.updated_at AS INTEGER) - )"; - fn query_candidates_after( conn: &Connection, idle_cutoff_epoch_secs: u64, @@ -580,52 +527,23 @@ fn query_candidates_after( cursor.map(|cursor| i64::try_from(cursor.idle_witness_secs).unwrap_or(i64::MAX)); let cursor_sandbox_id = cursor.map(|cursor| cursor.sandbox_id.as_str()); let cutoff = i64::try_from(idle_cutoff_epoch_secs).unwrap_or(i64::MAX); - let sql = format!( - "SELECT s.id, {IDLE_WITNESS_SQL} FROM sandboxes s - WHERE s.state = 'suspended' - AND {VALID_SUSPENDED_UPDATED_AT_SQL} - AND {IDLE_WITNESS_SQL} <= ?1 - AND ( - ?2 IS NULL - OR {IDLE_WITNESS_SQL} > ?2 - OR ({IDLE_WITNESS_SQL} = ?2 AND s.id > ?3) - ) - AND ( - s.current_local_artifact_id IS NOT NULL - OR EXISTS ( - SELECT 1 FROM jobs j - WHERE j.kind = 'snapshot_upload' - AND j.subject_id = s.id - AND json_extract(j.payload, '$.snapshot_id') IS NOT NULL - ) - ) - AND NOT EXISTS ( - SELECT 1 FROM jobs j - WHERE j.kind = 'snapshot_upload' - AND j.subject_id = s.id - AND j.status IN ('pending', 'running') - ) - ORDER BY {IDLE_WITNESS_SQL} ASC, s.id ASC - LIMIT ?4" - ); - let mut stmt = conn.prepare(&sql)?; - let rows = stmt - .query_map( - params![cutoff, cursor_witness, cursor_sandbox_id, limit as i64], - |row| { - let sandbox_id: String = row.get(0)?; - let idle_witness_secs = row.get::<_, SqliteU64>(1)?.0; - Ok(EvictionCandidate { - cursor: CandidateCursor { - idle_witness_secs, - sandbox_id: sandbox_id.clone(), - }, - sandbox_id, - }) + let rows = crate::db::eviction_scan::idle_suspended_candidates( + conn, + cutoff, + cursor_witness, + cursor_sandbox_id, + limit, + )?; + Ok(rows + .into_iter() + .map(|(sandbox_id, idle_witness_secs)| EvictionCandidate { + cursor: CandidateCursor { + idle_witness_secs, + sandbox_id: sandbox_id.clone(), }, - )? - .collect::>()?; - Ok(rows) + sandbox_id, + }) + .collect()) } /// Cold suspended sandboxes eligible for the per-candidate cluster check: @@ -655,56 +573,7 @@ pub(crate) fn list_legacy_reupload_candidates( idle_cutoff_epoch_secs: u64, limit: usize, ) -> rusqlite::Result> { - let sql = format!( - "SELECT s.id FROM sandboxes s - WHERE s.state = 'suspended' - AND {VALID_SUSPENDED_UPDATED_AT_SQL} - AND {IDLE_WITNESS_SQL} <= ?1 - AND s.current_local_artifact_id IS NULL - AND NOT EXISTS ( - SELECT 1 FROM jobs j - WHERE j.kind = 'snapshot_upload' - AND j.subject_id = s.id - AND j.status IN ('pending', 'running') - ) - AND NOT EXISTS ( - SELECT 1 FROM jobs j - WHERE j.kind = 'snapshot_upload' - AND j.subject_id = s.id - AND json_extract(j.payload, '$.snapshot_id') IS NOT NULL - ) - ORDER BY {IDLE_WITNESS_SQL} ASC - LIMIT ?2" - ); - let mut stmt = conn.prepare(&sql)?; - let rows = stmt - .query_map( - params![SqliteU64(idle_cutoff_epoch_secs), limit as i64], - |r| r.get(0), - )? - .collect::>()?; - Ok(rows) -} - -fn count_malformed_suspended_updated_at(conn: &Connection) -> rusqlite::Result { - let sql = format!( - "SELECT COUNT(*) FROM sandboxes s - WHERE s.state = 'suspended' - AND NOT ({VALID_SUSPENDED_UPDATED_AT_SQL})" - ); - conn.query_row(&sql, [], |row| { - let count: i64 = row.get(0)?; - Ok(usize::try_from(count).unwrap_or_default()) - }) -} - -/// Every local `sandboxes` row, whatever its state. node-ctl's full census -/// publishes one entry per row, so this is the count that sizes it. -fn count_resident_rows(conn: &Connection) -> rusqlite::Result { - conn.query_row("SELECT COUNT(*) FROM sandboxes", [], |row| { - let count: i64 = row.get(0)?; - usize::try_from(count).map_err(|_| rusqlite::Error::IntegralValueOutOfRange(0, count)) - }) + crate::db::eviction_scan::legacy_reupload_candidates(conn, idle_cutoff_epoch_secs, limit) } /// Gather the local half of the eviction evidence plus the sandbox's @@ -723,28 +592,14 @@ pub(crate) fn gather_local_evidence( conn: &Connection, sandbox_id: &str, ) -> rusqlite::Result> { - let row: Option<(Option, crate::db::timestamps::EpochSeconds)> = conn - .query_row( - "SELECT current_local_artifact_id, updated_at FROM sandboxes WHERE id = ?1", - params![sandbox_id], - |r| Ok((r.get(0)?, r.get(1)?)), - ) - .optional()?; + let row = crate::db::eviction_scan::artifact_pin_and_updated_at(conn, sandbox_id)?; let Some((pin, updated_at)) = row else { return Ok(None); }; let updated_at_secs = updated_at.unsigned(); if let Some(pin) = pin { - let artifact: Option<(String, String, bool)> = conn - .query_row( - "SELECT artifact_kind, local_artifact_id, backup_completed_at IS NOT NULL - FROM local_artifacts - WHERE local_artifact_id = ?1 AND sandbox_id = ?2", - params![pin, sandbox_id], - |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), - ) - .optional()?; + let artifact = crate::db::eviction_scan::owned_artifact_evidence(conn, &pin, sandbox_id)?; let evidence = match artifact { Some((kind, id, backup_marker)) => { match (id.parse(), kind.parse()) { @@ -766,16 +621,7 @@ pub(crate) fn gather_local_evidence( // Unpinned: the newest upload job regardless of status (a failed // newest upload must still count — it proves newer-than-backed // local state exists). - let job: Option<(Option, String)> = conn - .query_row( - "SELECT json_extract(j.payload, '$.snapshot_id'), j.created_at FROM jobs j - WHERE j.kind = 'snapshot_upload' AND j.subject_id = ?1 - ORDER BY j.created_at DESC, j.job_id DESC - LIMIT 1", - params![sandbox_id], - |r| Ok((r.get(0)?, r.get(1)?)), - ) - .optional()?; + let job = crate::db::eviction_scan::latest_upload_job(conn, sandbox_id)?; let latest_job = job.and_then(|(snapshot_id, created_at)| { let snapshot_id = snapshot_id?.trim().parse::().ok()?; // Invalid time cannot prove recency, even when the sandbox itself diff --git a/crates/libs/sbx/mgr/server/src/sandbox_eviction/tests.rs b/crates/libs/sbx/mgr/server/src/sandbox_eviction/tests.rs index 48a44b293..a2ac37a8d 100644 --- a/crates/libs/sbx/mgr/server/src/sandbox_eviction/tests.rs +++ b/crates/libs/sbx/mgr/server/src/sandbox_eviction/tests.rs @@ -1,4 +1,5 @@ use pplx_space_common_core::types::{LocalArtifactId, SnapshotId}; +use rusqlite::params; use super::*; diff --git a/crates/libs/sbx/mgr/server/src/service/activation_support.rs b/crates/libs/sbx/mgr/server/src/service/activation_support.rs index a54a1c8be..947a6fe2f 100644 --- a/crates/libs/sbx/mgr/server/src/service/activation_support.rs +++ b/crates/libs/sbx/mgr/server/src/service/activation_support.rs @@ -360,7 +360,7 @@ impl SandboxManagerService { match crate::db::activation_route_provenance(&db, sandbox_id) { Ok(provenance) => { if requested_can_open_claim - && let Some(claim) = crate::route_activation_claims::OpenRouteActivationClaim::for_cluster_fenced_identity( + && let Some(claim) = crate::db::route_activation_claims::OpenRouteActivationClaim::for_cluster_fenced_identity( &identity.identity, identity.provenance, ) diff --git a/crates/libs/sbx/mgr/server/src/service/cleanup.rs b/crates/libs/sbx/mgr/server/src/service/cleanup.rs index 8a81a38cb..64d5581b6 100644 --- a/crates/libs/sbx/mgr/server/src/service/cleanup.rs +++ b/crates/libs/sbx/mgr/server/src/service/cleanup.rs @@ -1114,7 +1114,7 @@ impl SandboxManagerService { } if completion.deletes_row() && mode.auto_lifecycle_delete_expectation().is_none() - && crate::auto_lifecycle_delete_receipts::db::pending_receipt_for_sandbox( + && crate::db::auto_lifecycle_delete_receipts::pending_receipt_for_sandbox( &transaction, sandbox_id, ) @@ -1134,7 +1134,7 @@ impl SandboxManagerService { mode.auto_lifecycle_delete_expectation() { let admission = match - crate::auto_lifecycle_delete_receipts::db::classify_and_record_admission( + crate::db::auto_lifecycle_delete_receipts::classify_and_record_admission( &transaction, sandbox_id, self.node_role, @@ -1143,7 +1143,7 @@ impl SandboxManagerService { ) { Ok(admission) => admission, - Err(error @ crate::auto_lifecycle_delete_receipts::db::AutoLifecycleDeleteEvidenceError::PolicyChanged) => { + Err(error @ crate::db::auto_lifecycle_delete_receipts::AutoLifecycleDeleteEvidenceError::PolicyChanged) => { pplx_space_metrics::sandbox::worker::auto_lifecycle_delete_admission( pplx_space_metrics::sandbox::worker::AutoLifecycleDeleteAdmissionDisposition::Failed, pplx_space_metrics::sandbox::worker::AutoLifecycleDeleteAdmissionReason::PolicyChanged, @@ -1163,7 +1163,7 @@ impl SandboxManagerService { reason: AutoLifecycleDeleteCasLostReason::PolicyChanged, }); } - Err(error @ crate::auto_lifecycle_delete_receipts::db::AutoLifecycleDeleteEvidenceError::AuthorityChanged { .. }) => { + Err(error @ crate::db::auto_lifecycle_delete_receipts::AutoLifecycleDeleteEvidenceError::AuthorityChanged { .. }) => { pplx_space_metrics::sandbox::worker::auto_lifecycle_delete_admission( pplx_space_metrics::sandbox::worker::AutoLifecycleDeleteAdmissionDisposition::Failed, pplx_space_metrics::sandbox::worker::AutoLifecycleDeleteAdmissionReason::AuthorityChanged, @@ -1322,7 +1322,7 @@ impl SandboxManagerService { "receipt admission is missing its auto-lifecycle delete expectation", ) })?; - let receipt = match crate::auto_lifecycle_delete_receipts::db::insert_pending_receipt( + let receipt = match crate::db::auto_lifecycle_delete_receipts::insert_pending_receipt( &transaction, crate::auto_lifecycle_delete_receipts::AutoLifecycleDeleteReceipt { receipt_id: pplx_space_common_core::types::AutoLifecycleDeleteReceiptId::generate(), @@ -1333,7 +1333,7 @@ impl SandboxManagerService { }, ) { Ok(receipt) => receipt, - Err(crate::auto_lifecycle_delete_receipts::db::InsertPendingReceiptError::Conflict(error)) => { + Err(crate::db::auto_lifecycle_delete_receipts::InsertPendingReceiptError::Conflict(error)) => { pplx_space_metrics::sandbox::worker::auto_lifecycle_delete_admission( pplx_space_metrics::sandbox::worker::AutoLifecycleDeleteAdmissionDisposition::Failed, pplx_space_metrics::sandbox::worker::AutoLifecycleDeleteAdmissionReason::ReceiptConflict, @@ -1343,7 +1343,7 @@ impl SandboxManagerService { sandbox_id, }); } - Err(crate::auto_lifecycle_delete_receipts::db::InsertPendingReceiptError::Database(error)) => { + Err(crate::db::auto_lifecycle_delete_receipts::InsertPendingReceiptError::Database(error)) => { return Err(Status::unavailable(format!( "persist auto-lifecycle delete receipt: {error}" )) @@ -1697,7 +1697,7 @@ impl SandboxManagerService { let sandbox_id = sandbox_id .parse::() .map_err(|error| error.to_string())?; - if !crate::auto_lifecycle_delete_receipts::db::pending_receipt_matches_observed_execution( + if !crate::db::auto_lifecycle_delete_receipts::pending_receipt_matches_observed_execution( &db, sandbox_id, ) diff --git a/crates/libs/sbx/mgr/server/src/service/cleanup_executor.rs b/crates/libs/sbx/mgr/server/src/service/cleanup_executor.rs index cc62fd862..449780eb2 100644 --- a/crates/libs/sbx/mgr/server/src/service/cleanup_executor.rs +++ b/crates/libs/sbx/mgr/server/src/service/cleanup_executor.rs @@ -636,7 +636,7 @@ impl SandboxManagerService { return Ok(None); } let receipt = - crate::auto_lifecycle_delete_receipts::db::pending_receipt_for_sandbox( + crate::db::auto_lifecycle_delete_receipts::pending_receipt_for_sandbox( db, work.sandbox_id, ) diff --git a/crates/libs/sbx/mgr/server/src/service/create.rs b/crates/libs/sbx/mgr/server/src/service/create.rs index 438e7fb67..002df47be 100644 --- a/crates/libs/sbx/mgr/server/src/service/create.rs +++ b/crates/libs/sbx/mgr/server/src/service/create.rs @@ -335,7 +335,7 @@ impl SandboxManagerService { let create_route_provenance = normalized_create_route_identity.provenance; let open_route_activation = (!req.autostart && requested_create_route_identity_can_open_claim).then(|| { - crate::route_activation_claims::OpenRouteActivationClaim::for_cluster_fenced_identity( + crate::db::route_activation_claims::OpenRouteActivationClaim::for_cluster_fenced_identity( &create_route_identity, create_route_provenance, ) @@ -733,7 +733,7 @@ impl SandboxManagerService { .await?; let open_route_activation = open_activation_claim_eligible .then(|| { - crate::route_activation_claims::OpenRouteActivationClaim::for_cluster_fenced_identity( + crate::db::route_activation_claims::OpenRouteActivationClaim::for_cluster_fenced_identity( &space_api_route_identity, route_identity_provenance, ) diff --git a/crates/libs/sbx/mgr/server/src/service/grpc_auto_lifecycle_delete_receipts.rs b/crates/libs/sbx/mgr/server/src/service/grpc_auto_lifecycle_delete_receipts.rs index 49afe52c0..cdbff70b6 100644 --- a/crates/libs/sbx/mgr/server/src/service/grpc_auto_lifecycle_delete_receipts.rs +++ b/crates/libs/sbx/mgr/server/src/service/grpc_auto_lifecycle_delete_receipts.rs @@ -15,7 +15,7 @@ impl SandboxManagerService { .db .lock() .map_err(|_| Status::internal("database mutex poisoned"))?; - crate::auto_lifecycle_delete_receipts::db::list_completed_receipts(&db) + crate::db::auto_lifecycle_delete_receipts::list_completed_receipts(&db) .map_err(|error| Status::internal(format!("list delete receipts: {error}")))? }; Ok(Response::new(ListAutoLifecycleDeleteReceiptsResponse { @@ -58,7 +58,7 @@ impl SandboxManagerService { .db .lock() .map_err(|_| Status::internal("database mutex poisoned"))?; - crate::auto_lifecycle_delete_receipts::db::acknowledge_completed_receipts( + crate::db::auto_lifecycle_delete_receipts::acknowledge_completed_receipts( &db, &receipt_ids, ) diff --git a/crates/libs/sbx/mgr/server/src/service/grpc_core/tests.rs b/crates/libs/sbx/mgr/server/src/service/grpc_core/tests.rs index ee5ac0f02..193800287 100644 --- a/crates/libs/sbx/mgr/server/src/service/grpc_core/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/grpc_core/tests.rs @@ -947,7 +947,7 @@ async fn auto_lifecycle_delete_requires_one_canonical_id_and_the_expected_policy ); fixture.assert_untouched(&id, SandboxState::Started, case.name); assert_eq!( - crate::auto_lifecycle_delete_receipts::db::pending_receipt_for_sandbox( + crate::db::auto_lifecycle_delete_receipts::pending_receipt_for_sandbox( &fixture.service.db_conn().unwrap(), sandbox_id, ) diff --git a/crates/libs/sbx/mgr/server/src/service/mod.rs b/crates/libs/sbx/mgr/server/src/service/mod.rs index 39402324a..e58d6610e 100644 --- a/crates/libs/sbx/mgr/server/src/service/mod.rs +++ b/crates/libs/sbx/mgr/server/src/service/mod.rs @@ -949,10 +949,10 @@ fn persisted_activation_owns_route_identity( } fn map_runtime_fields_update_error(error: rusqlite::Error) -> Status { - if crate::auto_lifecycle_delete_receipts::db::is_pending_receipt_execution_fence(&error) { + if crate::db::auto_lifecycle_delete_receipts::is_pending_receipt_execution_fence(&error) { return Status::aborted("pending auto-lifecycle delete receipt owns the sandbox execution"); } - if crate::auto_lifecycle_delete_receipts::db::is_lifecycle_execution_fence(&error) { + if crate::db::auto_lifecycle_delete_receipts::is_lifecycle_execution_fence(&error) { return Status::aborted("sandbox lifecycle or cleanup owns the sandbox execution"); } match error { @@ -966,9 +966,9 @@ fn route_identity_adoption_write_rejection_reason( error: &rusqlite::Error, ) -> pplx_space_metrics::sandbox::worker::RouteIdentityAdoptionReason { use pplx_space_metrics::sandbox::worker::RouteIdentityAdoptionReason; - if crate::auto_lifecycle_delete_receipts::db::is_pending_receipt_execution_fence(error) { + if crate::db::auto_lifecycle_delete_receipts::is_pending_receipt_execution_fence(error) { RouteIdentityAdoptionReason::PendingReceiptOwnsExecution - } else if crate::auto_lifecycle_delete_receipts::db::is_lifecycle_execution_fence(error) { + } else if crate::db::auto_lifecycle_delete_receipts::is_lifecycle_execution_fence(error) { RouteIdentityAdoptionReason::LifecycleOwnsExecution } else { match error { @@ -1255,7 +1255,7 @@ impl SandboxManagerService { local.current_node_id = local.current_node_id.or(self.local_node_id); } let merged = merge_authoritative_route_identity(&local, incoming); - let metadata = crate::auto_lifecycle_delete_receipts::db::route_identity_adoption_metadata( + let metadata = crate::db::auto_lifecycle_delete_receipts::route_identity_adoption_metadata( &db, sandbox_id, ) .map_err(|error| Status::internal(format!("inspect route identity metadata: {error}"))) @@ -1294,7 +1294,7 @@ impl SandboxManagerService { // provenance and receipt mode are current. Persist an authoritative // observation even when it does not advance the stream epoch. let pending_receipt = - crate::auto_lifecycle_delete_receipts::db::pending_receipt_for_sandbox( + crate::db::auto_lifecycle_delete_receipts::pending_receipt_for_sandbox( &db, sandbox_id, ) .map_err(|error| { @@ -1651,7 +1651,7 @@ impl SandboxManagerService { .as_ref() .map(|bundle| bundle.generation.get()); if options.node_role != Some(NodeRole::Worker) { - match crate::auto_lifecycle_delete_receipts::db::outbox_stats(&conn) { + match crate::db::auto_lifecycle_delete_receipts::outbox_stats(&conn) { Ok(stats) if stats.pending_count.saturating_add(stats.completed_count) != 0 => { tracing::warn!( node_role = ?options.node_role, diff --git a/crates/libs/sbx/mgr/server/src/service/reconcile.rs b/crates/libs/sbx/mgr/server/src/service/reconcile.rs index fb7c207a0..bc30b1c05 100644 --- a/crates/libs/sbx/mgr/server/src/service/reconcile.rs +++ b/crates/libs/sbx/mgr/server/src/service/reconcile.rs @@ -219,7 +219,7 @@ impl SandboxManagerService { let Ok(db) = self.db_conn() else { return; }; - match crate::auto_lifecycle_delete_receipts::db::record_receipt_conflict_retry( + match crate::db::auto_lifecycle_delete_receipts::record_receipt_conflict_retry( &db, storage_id, ) { Ok(true) => emit_sandbox_event( diff --git a/crates/libs/sbx/mgr/server/src/service/recovery_cross_node.rs b/crates/libs/sbx/mgr/server/src/service/recovery_cross_node.rs index 448f392cf..2f9d95eae 100644 --- a/crates/libs/sbx/mgr/server/src/service/recovery_cross_node.rs +++ b/crates/libs/sbx/mgr/server/src/service/recovery_cross_node.rs @@ -685,17 +685,10 @@ fn adopt_restoring_row_under_recovery_claim( // The generation CAS below sees the newly written Restoring state, so it // cannot independently prove the prior row was still eligible for recovery. if recovery.is_some() { - use rusqlite::OptionalExtension; - let existing_state: Option = tx - .query_row( - pplx_space_common_core::sql!("SELECT state FROM sandboxes WHERE id = ?1"), - rusqlite::params![canonical_id], - |row| row.get(0), - ) - .optional() - .map_err( - |source| crate::snapshot_restore::RestoreError::AdvanceRestorePhase { source }, - )?; + let existing_state = + crate::db::current_state_label(&tx, canonical_id).map_err(|source| { + crate::snapshot_restore::RestoreError::AdvanceRestorePhase { source } + })?; // An absent row is the orphan case this path exists for; the upsert // inserts it. Only an existing row in an unexpected state is a refusal. // diff --git a/crates/libs/sbx/mgr/server/src/service/route_identity_tests.rs b/crates/libs/sbx/mgr/server/src/service/route_identity_tests.rs index 26bdd4554..cd9cddc27 100644 --- a/crates/libs/sbx/mgr/server/src/service/route_identity_tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/route_identity_tests.rs @@ -190,7 +190,7 @@ fn route_identity_provenance( service: &SandboxManagerService, sandbox_id: SandboxId, ) -> Option { - crate::auto_lifecycle_delete_receipts::db::route_identity_adoption_metadata( + crate::db::auto_lifecycle_delete_receipts::route_identity_adoption_metadata( &service.db_conn().unwrap(), sandbox_id, ) @@ -315,7 +315,7 @@ async fn identity_equal_adoption_does_not_overwrite_an_owned_execution() { SandboxState::Stopping } ExecutionOwner::PendingDeleteReceipt => { - crate::auto_lifecycle_delete_receipts::db::insert_pending_receipt( + crate::db::auto_lifecycle_delete_receipts::insert_pending_receipt( &conn, crate::auto_lifecycle_delete_receipts::AutoLifecycleDeleteReceipt { receipt_id: diff --git a/crates/libs/sbx/mgr/server/src/service/start.rs b/crates/libs/sbx/mgr/server/src/service/start.rs index 6ac9476dd..c511a3b2f 100644 --- a/crates/libs/sbx/mgr/server/src/service/start.rs +++ b/crates/libs/sbx/mgr/server/src/service/start.rs @@ -1731,7 +1731,7 @@ impl SandboxManagerService { let open_route_activation = start_ctx .open_activation_claim_eligible .then(|| { - crate::route_activation_claims::OpenRouteActivationClaim::for_cluster_fenced_identity( + crate::db::route_activation_claims::OpenRouteActivationClaim::for_cluster_fenced_identity( &start_ctx.space_api_route_identity, start_ctx.route_identity_provenance, ) diff --git a/crates/libs/sbx/mgr/server/src/service/tests.rs b/crates/libs/sbx/mgr/server/src/service/tests.rs index fd44acfd7..1d76da9b9 100644 --- a/crates/libs/sbx/mgr/server/src/service/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/tests.rs @@ -1084,7 +1084,7 @@ pub(crate) mod service_tests { assert!(resolved.current_node_id.is_some()); assert_eq!(resolved.substrate_kind, Some(NodeManagedBy::Space)); assert!( - !crate::route_activation_claims::claim_exists( + !crate::db::route_activation_claims::claim_exists( &service.db.lock().unwrap(), &sandbox_id.to_string() ) @@ -1821,7 +1821,7 @@ pub(crate) mod service_tests { normalized.provenance, true, ) { - crate::auto_lifecycle_delete_receipts::db::record_retained_route_evidence( + crate::db::auto_lifecycle_delete_receipts::record_retained_route_evidence( &service.db.lock().unwrap(), sandbox_id, evidence, @@ -1899,7 +1899,7 @@ pub(crate) mod service_tests { activation_attempt_id: identity.activation_attempt_id, }, Some(evidence), - crate::route_activation_claims::OpenRouteActivationClaim::for_cluster_fenced_identity( + crate::db::route_activation_claims::OpenRouteActivationClaim::for_cluster_fenced_identity( &identity, crate::auto_lifecycle_delete_receipts::RouteIdentityProvenance::ClusterFenced, ) @@ -1918,7 +1918,9 @@ pub(crate) mod service_tests { &crate::db::edges::CREATE_BASE_SNAPSHOT_COMPLETE, ); assert!(crate::db::complete_create(&db, &sandbox_id_string, cleanup_id, true).unwrap()); - assert!(crate::route_activation_claims::claim_exists(&db, &sandbox_id_string).unwrap()); + assert!( + crate::db::route_activation_claims::claim_exists(&db, &sandbox_id_string).unwrap() + ); } (service, sandbox_id, identity) } @@ -1977,7 +1979,7 @@ pub(crate) mod service_tests { ) .unwrap(); assert!( - !crate::route_activation_claims::claim_exists(&db, &sandbox_id.to_string()) + !crate::db::route_activation_claims::claim_exists(&db, &sandbox_id.to_string()) .unwrap() ); } @@ -2054,7 +2056,7 @@ pub(crate) mod service_tests { .unwrap() .expect("the later Direct activation must claim the sandbox row"); assert!( - !crate::route_activation_claims::claim_exists(&db, &sandbox_id.to_string()) + !crate::db::route_activation_claims::claim_exists(&db, &sandbox_id.to_string()) .unwrap(), "a later non-Kubernetes activation must supersede the open Kubernetes claim" ); @@ -2097,7 +2099,7 @@ pub(crate) mod service_tests { }; let provenance = crate::auto_lifecycle_delete_receipts::RouteIdentityProvenance::ClusterFenced; - let open_claim = crate::route_activation_claims::OpenRouteActivationClaim::for_cluster_fenced_identity(&route, provenance).unwrap(); + let open_claim = crate::db::route_activation_claims::OpenRouteActivationClaim::for_cluster_fenced_identity(&route, provenance).unwrap(); let episode = crate::db::begin_activation_with_route_evidence( &conn, &id, @@ -2196,7 +2198,7 @@ pub(crate) mod service_tests { if receipt.kind() == SandboxTransitionReceiptKind::ActivationRollback && receipt.activation_attempt_id() == Some(if persist_operation { operation } else { route_attempt })) ); - assert!(crate::route_activation_claims::claim_exists(&db, &id).unwrap()); + assert!(crate::db::route_activation_claims::claim_exists(&db, &id).unwrap()); assert!( crate::route_activation_claims::validate_cluster_fenced_activation( &db, @@ -2267,7 +2269,7 @@ pub(crate) mod service_tests { .resolve_runtime_activation_identity(&sandbox_id.to_string(), identity, false) .expect("the surviving pod and attempt may continue after manager restart"); assert!( - crate::route_activation_claims::claim_exists( + crate::db::route_activation_claims::claim_exists( &service.db.lock().unwrap(), &sandbox_id.to_string() ) @@ -2281,7 +2283,7 @@ pub(crate) mod service_tests { service_with_base_snapshot_create_route_evidence(); { let db = service.db.lock().unwrap(); - crate::route_activation_claims::close_claim(&db, &sandbox_id.to_string()).unwrap(); + crate::db::route_activation_claims::close_claim(&db, &sandbox_id.to_string()).unwrap(); } identity.route_generation = Some(RouteGeneration::new(2).unwrap()); identity.execution_generation = Some(ExecutionGeneration::new(2).unwrap()); @@ -2292,7 +2294,7 @@ pub(crate) mod service_tests { .resolve_runtime_activation_identity(&sandbox_id.to_string(), identity, false) .expect("a newer rollout-compatible identity remains on the existing path"); assert!( - !crate::route_activation_claims::claim_exists( + !crate::db::route_activation_claims::claim_exists( &service.db.lock().unwrap(), &sandbox_id.to_string() ) @@ -3477,7 +3479,7 @@ pub(crate) mod service_tests { crate::auto_lifecycle_delete_receipts::RouteIdentityProvenance::ClusterFenced, ) .unwrap(); - crate::auto_lifecycle_delete_receipts::db::insert_pending_receipt( + crate::db::auto_lifecycle_delete_receipts::insert_pending_receipt( &db, crate::auto_lifecycle_delete_receipts::AutoLifecycleDeleteReceipt { receipt_id: pending_receipt_id, @@ -3503,7 +3505,7 @@ pub(crate) mod service_tests { let db = service.db.lock().unwrap(); assert!( - crate::auto_lifecycle_delete_receipts::db::pending_receipt_for_sandbox( + crate::db::auto_lifecycle_delete_receipts::pending_receipt_for_sandbox( &db, sandbox_id, ) .unwrap() @@ -3568,7 +3570,7 @@ pub(crate) mod service_tests { crate::auto_lifecycle_delete_receipts::RouteIdentityProvenance::ClusterFenced, ) .unwrap(); - crate::auto_lifecycle_delete_receipts::db::insert_pending_receipt(&db, pending) + crate::db::auto_lifecycle_delete_receipts::insert_pending_receipt(&db, pending) .unwrap(); db.execute_batch( "CREATE TRIGGER reject_redundant_route_identity_adoption @@ -3601,7 +3603,7 @@ pub(crate) mod service_tests { let db = service.db.lock().unwrap(); assert_eq!( - crate::auto_lifecycle_delete_receipts::db::pending_receipt_for_sandbox( + crate::db::auto_lifecycle_delete_receipts::pending_receipt_for_sandbox( &db, sandbox_id, ) .unwrap(), @@ -3679,7 +3681,7 @@ pub(crate) mod service_tests { assert_eq!( route_identity_adoption_write_rejection_reason( &rusqlite::Error::ToSqlConversionFailure(Box::new( - crate::auto_lifecycle_delete_receipts::db::PendingReceiptOwnsExecution, + crate::db::auto_lifecycle_delete_receipts::PendingReceiptOwnsExecution, )), ), RouteIdentityAdoptionReason::PendingReceiptOwnsExecution @@ -3687,7 +3689,7 @@ pub(crate) mod service_tests { assert_eq!( route_identity_adoption_write_rejection_reason( &rusqlite::Error::ToSqlConversionFailure(Box::new( - crate::auto_lifecycle_delete_receipts::db::LifecycleOwnsExecution, + crate::db::auto_lifecycle_delete_receipts::LifecycleOwnsExecution, )), ), RouteIdentityAdoptionReason::LifecycleOwnsExecution @@ -6894,7 +6896,7 @@ pub(crate) mod service_tests { .is_none() ); assert_eq!( - crate::auto_lifecycle_delete_receipts::db::pending_receipt_for_sandbox( + crate::db::auto_lifecycle_delete_receipts::pending_receipt_for_sandbox( &db, sandbox_id, ) .unwrap(), @@ -6924,7 +6926,7 @@ pub(crate) mod service_tests { assert_eq!(row.cleanup_completion, Some(CleanupCompletion::DeleteRow)); assert_eq!(row.cleanup_keep_filesystem, Some(false)); assert_eq!( - crate::auto_lifecycle_delete_receipts::db::pending_receipt_for_sandbox( + crate::db::auto_lifecycle_delete_receipts::pending_receipt_for_sandbox( &db, sandbox_id, ) .unwrap(), @@ -8267,7 +8269,7 @@ pub(crate) mod service_tests { &AutoLifecyclePolicy::disabled(), ), }; - crate::auto_lifecycle_delete_receipts::db::insert_pending_receipt(&conn, receipt).unwrap(); + crate::db::auto_lifecycle_delete_receipts::insert_pending_receipt(&conn, receipt).unwrap(); let service = SandboxManagerService::for_test(conn); let error = service @@ -8309,7 +8311,7 @@ pub(crate) mod service_tests { crate::auto_lifecycle_delete_receipts::RouteIdentityProvenance::ClusterFenced, ) .unwrap(); - crate::auto_lifecycle_delete_receipts::db::insert_pending_receipt( + crate::db::auto_lifecycle_delete_receipts::insert_pending_receipt( &conn, crate::auto_lifecycle_delete_receipts::AutoLifecycleDeleteReceipt { receipt_id: pplx_space_common_core::types::AutoLifecycleDeleteReceiptId::generate(), @@ -9973,7 +9975,7 @@ pub(crate) mod service_tests { crate::auto_lifecycle_delete_receipts::RouteIdentityProvenance::Unfenced, ) .unwrap(); - crate::auto_lifecycle_delete_receipts::db::insert_pending_receipt(&conn, receipt).unwrap(); + crate::db::auto_lifecycle_delete_receipts::insert_pending_receipt(&conn, receipt).unwrap(); conn.execute( "UPDATE sandboxes SET state = 'deleting', @@ -9983,7 +9985,7 @@ pub(crate) mod service_tests { [sandbox_id.to_string()], ) .unwrap(); - crate::auto_lifecycle_delete_receipts::db::record_retained_route_evidence( + crate::db::auto_lifecycle_delete_receipts::record_retained_route_evidence( &conn, sandbox_id, crate::auto_lifecycle_delete_receipts::RouteEvidence { @@ -10853,7 +10855,7 @@ pub(crate) mod service_tests { SandboxState::Started ); assert_eq!( - crate::auto_lifecycle_delete_receipts::db::pending_receipt_for_sandbox( + crate::db::auto_lifecycle_delete_receipts::pending_receipt_for_sandbox( &db, sandbox_id, ) .unwrap(), @@ -17718,7 +17720,7 @@ pub(crate) mod service_tests { &AutoLifecyclePolicy::disabled(), ), }; - crate::auto_lifecycle_delete_receipts::db::insert_pending_receipt(&conn, receipt).unwrap(); + crate::db::auto_lifecycle_delete_receipts::insert_pending_receipt(&conn, receipt).unwrap(); let service = SandboxManagerService::for_test(conn); let targets = service.recover_requested_cleanup_targets_for_ids(vec![sandbox_id]); diff --git a/crates/libs/sbx/mgr/server/src/snapshot_upload_jobs.rs b/crates/libs/sbx/mgr/server/src/snapshot_upload_jobs.rs index b75c0ce50..0fef00ab8 100644 --- a/crates/libs/sbx/mgr/server/src/snapshot_upload_jobs.rs +++ b/crates/libs/sbx/mgr/server/src/snapshot_upload_jobs.rs @@ -53,7 +53,7 @@ use crate::snapshot_upload::{ format_from_meta, upload_artifacts_for_meta, }; -pub const SNAPSHOT_UPLOAD_JOB_KIND: &str = "snapshot_upload"; +pub use crate::db::snapshot_upload_jobs::SNAPSHOT_UPLOAD_JOB_KIND; const DEFAULT_CONCURRENCY: usize = 4; const JOB_TIMEOUT: Duration = Duration::from_secs(7200); @@ -89,11 +89,9 @@ pub(crate) fn build_worker( } pub fn refund_interrupted_running_attempts(conn: &rusqlite::Connection) -> rusqlite::Result { - conn.execute( - "UPDATE jobs - SET attempts = attempts - 1 - WHERE kind = ?1 AND status = 'running' AND attempts > 0", - rusqlite::params![SNAPSHOT_UPLOAD_JOB_KIND], + crate::db::snapshot_upload_jobs::refund_interrupted_running_attempts( + conn, + SNAPSHOT_UPLOAD_JOB_KIND, ) } @@ -517,17 +515,17 @@ pub(crate) fn persist_submitted_transfer_ids( "snapshot-upload database mutex poisoned while persisting transfer IDs for {job_id}" )) })?; - let changed = conn - .execute( - "UPDATE jobs SET payload = ?1 - WHERE job_id = ?2 AND kind = ?3 AND status IN ('pending', 'running')", - rusqlite::params![payload, job_id, SNAPSHOT_UPLOAD_JOB_KIND], - ) - .map_err(|error| { - JobError::retriable(format!( - "persist submitted transfer IDs for snapshot-upload job {job_id}: {error}" - )) - })?; + let changed = crate::db::snapshot_upload_jobs::replace_active_job_payload( + &conn, + &payload, + job_id, + SNAPSHOT_UPLOAD_JOB_KIND, + ) + .map_err(|error| { + JobError::retriable(format!( + "persist submitted transfer IDs for snapshot-upload job {job_id}: {error}" + )) + })?; if changed != 1 { return Err(JobError::permanent(format!( "snapshot-upload job {job_id} was not pending or running while persisting transfer IDs" @@ -1426,8 +1424,7 @@ pub fn backfill_backup_completed_markers( let through = match scan.through { Some(through) => through, None => { - let through: Option = - conn.query_row("SELECT MAX(rowid) FROM jobs", [], |r| r.get(0))?; + let through = crate::db::snapshot_upload_jobs::max_job_rowid(conn)?; let Some(through) = through else { return Ok(report); }; @@ -1435,67 +1432,59 @@ pub fn backfill_backup_completed_markers( through } }; - let mut statement = conn.prepare( - "SELECT rowid, payload FROM jobs - WHERE kind = 'snapshot_upload' AND status = 'completed' - AND rowid > ?1 AND rowid <= ?2 ORDER BY rowid LIMIT ?3", - )?; - let mut rows = statement.query(rusqlite::params![ + crate::db::snapshot_upload_jobs::visit_completed_upload_payloads( + conn, scan.after.unwrap_or(i64::MIN), through, BACKUP_MARKER_BATCH as i64, - ])?; - let now = crate::db::now()?; - while let Some(row) = rows.next()? { - scan.after = Some(row.get(0)?); - report.examined += 1; - let Ok(payload) = row.get::<_, String>(1) else { - report.malformed_payload += 1; - continue; - }; - // Extract via `serde_json::Value` rather than the full - // `SnapshotUploadJobs` struct: historical payloads predate - // several of its fields, and the backfill needs only the - // artifact id. - let Ok(value) = serde_json::from_str::(&payload) else { - report.malformed_payload += 1; - continue; - }; - let Some(artifact_id) = value - .get("meta") - .and_then(|m| m.get("local_artifact_id")) - .and_then(|v| v.as_str()) - else { - report.payload_missing_artifact_id += 1; - continue; - }; - let changed = match conn.execute( - "UPDATE local_artifacts SET backup_completed_at = ?1 - WHERE local_artifact_id = ?2 AND backup_completed_at IS NULL", - rusqlite::params![now, artifact_id], - ) { - Ok(changed) => changed, - Err(error) => { - report.marker_write_failed += 1; - tracing::warn!(artifact_id, %error, "backup marker repair failed; evidence retained for the next sweep"); - continue; + |now, rowid, payload| { + scan.after = Some(rowid); + report.examined += 1; + let Ok(payload) = payload else { + report.malformed_payload += 1; + return Ok(()); + }; + // Extract via `serde_json::Value` rather than the full + // `SnapshotUploadJobs` struct: historical payloads predate + // several of its fields, and the backfill needs only the + // artifact id. + let Ok(value) = serde_json::from_str::(&payload) else { + report.malformed_payload += 1; + return Ok(()); + }; + let Some(artifact_id) = value + .get("meta") + .and_then(|m| m.get("local_artifact_id")) + .and_then(|v| v.as_str()) + else { + report.payload_missing_artifact_id += 1; + return Ok(()); + }; + let changed = match crate::db::snapshot_upload_jobs::stamp_backup_completed_if_unset( + conn, + now, + artifact_id, + ) { + Ok(changed) => changed, + Err(error) => { + report.marker_write_failed += 1; + tracing::warn!(artifact_id, %error, "backup marker repair failed; evidence retained for the next sweep"); + return Ok(()); + } + }; + if changed == 1 { + report.stamped += 1; + return Ok(()); } - }; - if changed == 1 { - report.stamped += 1; - continue; - } - let exists: bool = conn.query_row( - "SELECT EXISTS(SELECT 1 FROM local_artifacts WHERE local_artifact_id = ?1)", - rusqlite::params![artifact_id], - |r| r.get(0), - )?; - if exists { - report.already_stamped += 1; - } else { - report.artifact_row_missing += 1; - } - } + let exists = crate::db::snapshot_upload_jobs::local_artifact_exists(conn, artifact_id)?; + if exists { + report.already_stamped += 1; + } else { + report.artifact_row_missing += 1; + } + Ok(()) + }, + )?; if report.examined < BACKUP_MARKER_BATCH || scan.after == Some(through) { *scan = BackupMarkerScan::default(); } diff --git a/crates/libs/sbx/mgr/server/src/watchdog.rs b/crates/libs/sbx/mgr/server/src/watchdog.rs index 4288c6fdf..030d49009 100644 --- a/crates/libs/sbx/mgr/server/src/watchdog.rs +++ b/crates/libs/sbx/mgr/server/src/watchdog.rs @@ -17,12 +17,10 @@ use std::time::{Duration, Instant, SystemTime}; use futures_util::future::BoxFuture; use futures_util::stream::FuturesUnordered; use futures_util::{FutureExt, StreamExt}; -use rusqlite::types::Type; use tokio::sync::Mutex; use tokio_util::sync::CancellationToken; use tokio_util::task::TaskTracker; -use pplx_space_common_core::sql; use pplx_space_common_core::types::{LinuxPid, SandboxId, SemVer}; use pplx_space_metrics::sandbox::worker::{ NetGatewayRouteVerificationStatus, NetGatewayTlsInterceptionMode, @@ -30,6 +28,7 @@ use pplx_space_metrics::sandbox::worker::{ use pplx_space_node_runtime_core::fc_api::FirecrackerVmState; use crate::auto_lifecycle::AutoLifecycleCoordinator; +use crate::db::guest_health_scan::{SandboxGuestHealthRow, list_guest_health_rows}; use crate::node_manager_transport::NodeManagerConnector; use crate::node_state_sync::mark_sandbox_dirty; use crate::service::network_policy::NetworkPolicyReconcileQueue; @@ -87,13 +86,6 @@ const GUEST_HEALTH_CATEGORIES: [GuestHealthCategory; 3] = [ GuestHealthCategory::Unreachable, ]; -#[derive(Clone, Debug, PartialEq, Eq)] -struct SandboxGuestHealthRow { - id: String, - state: crate::state::SandboxState, - template_content_hash: Option, -} - #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] enum NetGatewayLifecycle { Known(crate::state::SandboxState), @@ -241,12 +233,6 @@ struct GuestHealthGaugeValue { count: usize, } -#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] -#[error("unknown sandbox state '{value}'")] -struct SandboxStateParseError { - value: String, -} - #[derive(Clone)] pub struct SidecarInfo { pub node_execution_id: Option, @@ -1571,14 +1557,14 @@ fn emit_auto_lifecycle_delete_gauges(db: &Arc stats, Err(error) => { tracing::warn!(%error, "failed to inspect auto-lifecycle delete outbox gauges"); return; } }; - let blocked = match crate::auto_lifecycle_delete_receipts::db::blocked_stats(&db) { + let blocked = match crate::db::auto_lifecycle_delete_receipts::blocked_stats(&db) { Ok(stats) => stats, Err(error) => { tracing::warn!(%error, "failed to inspect auto-lifecycle delete block gauges"); @@ -1899,30 +1885,6 @@ async fn emit_net_gateway_route_verification_gauges( } } -fn list_guest_health_rows( - conn: &rusqlite::Connection, -) -> rusqlite::Result> { - let mut stmt = conn.prepare(sql!("SELECT id, state, template_hash FROM sandboxes"))?; - let rows = stmt.query_map([], |row| { - let state_value = row.get::<_, String>(1)?; - let state = state_value - .parse::() - .map_err(|_| { - rusqlite::Error::FromSqlConversionFailure( - 1, - Type::Text, - Box::new(SandboxStateParseError { value: state_value }), - ) - })?; - Ok(SandboxGuestHealthRow { - id: row.get(0)?, - state, - template_content_hash: row.get(2)?, - }) - })?; - rows.collect() -} - fn sandbox_states_for_ids( db: &Arc>, sandbox_ids: &[String], @@ -1934,18 +1896,18 @@ fn sandbox_states_for_ids( return None; } }; - let mut statement = match db.prepare("SELECT state FROM sandboxes WHERE id = ?1") { - Ok(statement) => statement, + let labels = match crate::db::guest_health_scan::state_labels_for_ids(&db, sandbox_ids) { + Ok(labels) => labels, Err(error) => { tracing::warn!(%error, "failed to prepare watchdog probe-state lookup"); return None; } }; let mut states = HashMap::with_capacity(sandbox_ids.len()); - for sandbox_id in sandbox_ids { - let state = match statement.query_row([sandbox_id], |row| row.get::<_, String>(0)) { - Ok(state) => state, - Err(rusqlite::Error::QueryReturnedNoRows) => continue, + for (sandbox_id, label) in labels { + let state = match label { + Ok(Some(state)) => state, + Ok(None) => continue, Err(error) => { tracing::warn!(sandbox_id, %error, "failed to load watchdog probe state"); continue; diff --git a/crates/tools/archcheck/README.md b/crates/tools/archcheck/README.md index 8dfb9ea47..3853138c6 100644 --- a/crates/tools/archcheck/README.md +++ b/crates/tools/archcheck/README.md @@ -68,8 +68,9 @@ table: ```toml [rules.pattern_ratchet.] patterns = ['\bsome_name\b'] # regexes, matched against each code line -include = ["crates/x/src/**"] # optional; omitted means all production files +include = ["crates/x/src/**"] # optional; omitted means all production files allow = ["crates/x/src/owner.rs", "crates/x/src/owner/**"] +paths = ["crates/x/**"] # optional: the files checked; all when absent [rules.pattern_ratchet..no_crossing] mode = "advisory" @@ -97,6 +98,12 @@ name so that longer identifiers containing it do not match. Patterns are case-sensitive unless they use an inline `(?i)` flag. `allow` globs match repository-relative paths: `*` and `?` stay within one path component and `**` spans any number. Files they match are never counted. +`paths`, when given, takes globs of the same form and limits the instance to +the production files they match, for a name that other crates use for their +own purposes; files outside them are never counted, and an entry for one is +stale. An empty `paths` would check nothing and exits 2; omit it to check +every production file. An empty `allow` is fine. An instance may give both +`include` and `paths`; it then checks only the files that match both. - `no_crossing`: a file outside the allowed paths and the instance's baseline table has a matching line; @@ -259,11 +266,11 @@ becomes `support.rs`) without changing what the module holds. Each behaviour has a `mode`: `advisory` findings are reported, `required` ones also fail the check, so behaviours can graduate one at a time. Unknown keys, sections or modes, a missing file, bad TOML, an invalid regex, glob or instance -name, an empty `patterns` list, a baseline table for an instance that is not -configured, and a failing `cargo metadata` exit 2. So does a `--base-baseline`, -`--renames` or `--base-architecture` that can't be read, or one combined with -`--write-baseline`, and `--renames` input that is not git's (see [Renamed -files](#renamed-files)). +name, an empty `patterns` or `paths` list, a baseline table for an instance +that is not configured, and a failing `cargo metadata` exit 2. So does a +`--base-baseline`, `--renames` or `--base-architecture` that can't be read, or +one combined with `--write-baseline`, and `--renames` input that is not git's +(see [Renamed files](#renamed-files)). ## Baseline diff --git a/crates/tools/archcheck/src/config.rs b/crates/tools/archcheck/src/config.rs index f1c2a2841..fb400d9e1 100644 --- a/crates/tools/archcheck/src/config.rs +++ b/crates/tools/archcheck/src/config.rs @@ -59,11 +59,15 @@ pub struct FileSizeConfig { pub struct PatternRatchetConfig { /// Optional repository-relative scope; absent means all production files. #[serde(default)] - pub include: Option, + pub include: Option, /// A code line matching any of these regexes is restricted. pub patterns: Patterns, + /// Repository-relative path globs of the files the instance checks; + /// every production file when absent. + #[serde(default)] + pub paths: Option, /// Repository-relative path globs whose files may match freely. - pub allow: AllowList, + pub allow: PathGlobs, pub no_crossing: BehaviourConfig, pub no_new_entries: BehaviourConfig, pub no_raised_caps: BehaviourConfig, @@ -152,15 +156,15 @@ impl TryFrom> for Patterns { /// within one path component; `**` spans any number of them. #[derive(Debug, Clone, Deserialize)] #[serde(try_from = "Vec")] -pub struct AllowList(GlobSet); +pub struct PathGlobs(GlobSet); -impl AllowList { +impl PathGlobs { pub fn is_match(&self, path: &str) -> bool { self.0.is_match(path) } } -impl TryFrom> for AllowList { +impl TryFrom> for PathGlobs { type Error = globset::Error; fn try_from(globs: Vec) -> Result { @@ -172,6 +176,37 @@ impl TryFrom> for AllowList { } } +/// An instance's `paths`: [`PathGlobs`] with at least one glob, since an +/// empty list would check nothing. +#[derive(Debug, Clone, Deserialize)] +#[serde(try_from = "Vec")] +pub struct CheckedPaths(PathGlobs); + +#[derive(Debug, Error)] +pub enum CheckedPathsError { + #[error("paths must list at least one glob; omit it to check every production file")] + Empty, + #[error(transparent)] + Glob(#[from] globset::Error), +} + +impl CheckedPaths { + pub fn is_match(&self, path: &str) -> bool { + self.0.is_match(path) + } +} + +impl TryFrom> for CheckedPaths { + type Error = CheckedPathsError; + + fn try_from(globs: Vec) -> Result { + if globs.is_empty() { + return Err(CheckedPathsError::Empty); + } + Ok(Self(PathGlobs::try_from(globs)?)) + } +} + #[derive(Debug, Clone, Deserialize)] #[serde(deny_unknown_fields)] pub struct CrateNamesConfig { diff --git a/crates/tools/archcheck/src/rules/pattern_ratchet.rs b/crates/tools/archcheck/src/rules/pattern_ratchet.rs index 748e9ba8e..e9e5db26a 100644 --- a/crates/tools/archcheck/src/rules/pattern_ratchet.rs +++ b/crates/tools/archcheck/src/rules/pattern_ratchet.rs @@ -4,8 +4,10 @@ //! any of its `patterns` to the files its `allow` globs match: a file outside //! them with a matching line violates it, and the instance's own //! `[pattern_ratchet.]` baseline table caps each listed file's -//! matching lines; [`super::ratchet`] has the behaviours. Instances are -//! independent: each reads only its own configuration and tables. +//! matching lines; [`super::ratchet`] has the behaviours. An optional `paths` +//! list limits the instance to the files its globs match; it checks every +//! production file without one. Instances are independent: each reads only +//! its own configuration and tables. use crate::{ baseline::BASELINE_FILE, @@ -38,6 +40,14 @@ struct Instance<'a> { } impl Instance<'_> { + /// Whether `file` is among the paths the instance checks. + fn checks(&self, file: &ProductionFile) -> bool { + self.config + .paths + .as_ref() + .is_none_or(|paths| paths.is_match(&file.path)) + } + /// The instance's baseline table and where it is. fn table(&self) -> String { format!("[{RULE}.{}] in {BASELINE_FILE}", self.instance) @@ -61,13 +71,15 @@ impl CountRatchet for Instance<'_> { } /// `file`'s code lines that match the patterns, unless its path is - /// allowed or none do. Findings about them are shown on the first. + /// outside the checked paths, allowed, or none do. Findings about them + /// are shown on the first. fn measure(&self, file: &ProductionFile) -> Option { if self .config .include .as_ref() .is_some_and(|scope| !scope.is_match(&file.path)) + || !self.checks(file) || self.config.allow.is_match(&file.path) { return None; @@ -107,11 +119,15 @@ impl CountRatchet for Instance<'_> { ) } - fn cleared(&self, _file: &ProductionFile) -> String { - format!( - " has no code lines left matching the {} patterns outside its allowed paths", - self.instance - ) + fn cleared(&self, file: &ProductionFile) -> String { + let instance = self.instance; + if self.checks(file) { + format!( + " has no code lines left matching the {instance} patterns outside its allowed paths" + ) + } else { + format!(" is outside the paths [rules.{RULE}.{instance}] in ARCHITECTURE.toml checks") + } } fn counted(&self, count: usize) -> String { diff --git a/crates/tools/archcheck/tests/cli.rs b/crates/tools/archcheck/tests/cli.rs index 05999128c..dbb0dc0eb 100644 --- a/crates/tools/archcheck/tests/cli.rs +++ b/crates/tools/archcheck/tests/cli.rs @@ -1904,6 +1904,111 @@ fn crate_names_rdjson_and_exit_codes() { assert_eq!(document["diagnostics"][0]["severity"], "WARNING"); } +/// `paths` limits an instance to the files it matches: others are neither +/// counted nor kept in its table, and `allow` still applies inside them. +#[test] +fn pattern_ratchet_paths() { + let config = format!( + "{}{}", + config(ADVISORY), + ratchet( + "old", + &[r"\bold_table\b"], + &["crates/a/src/allowed.rs"], + ADVISORY + ), + ) + .replace( + "allow = ['crates/a/src/allowed.rs']\n", + "allow = ['crates/a/src/allowed.rs']\npaths = ['crates/a/**']\n", + ); + type Case<'a> = ( + &'a str, + &'a [(&'a str, usize)], + &'a [(&'a str, &'a str)], + &'a [(&'a str, &'a str)], + ); + let cases: &[Case] = &[ + ( + "a file inside the paths is checked", + &[], + &[("crates/a/src/x.rs", "old_table\n")], + &[("no_crossing", "crates/a/src/x.rs: 1 code line matches")], + ), + ( + "a file outside the paths is not", + &[], + &[("crates/b/src/x.rs", "old_table\n")], + &[], + ), + ( + "allowed paths inside the paths may match", + &[], + &[("crates/a/src/allowed.rs", "old_table\n")], + &[], + ), + ( + "an entry for a file outside the paths is stale", + &[("crates/b/src/x.rs", 1)], + &[("crates/b/src/x.rs", "old_table\n")], + &[( + "stale_entries", + "crates/b/src/x.rs is outside the paths [rules.pattern_ratchet.old] in \ + ARCHITECTURE.toml checks. Remove it from [pattern_ratchet.old] in \ + architecture-baseline.toml.", + )], + ), + ]; + for (name, entries, files, expected) in cases { + let mut files: Vec<(&str, String)> = files + .iter() + .map(|(path, text)| (*path, (*text).to_owned())) + .collect(); + files.push(("ARCHITECTURE.toml", config.clone())); + files.push(( + "architecture-baseline.toml", + ratchet_baseline(&[("old", entries)]), + )); + let ws = workspace(&files); + + let found = ratchet_findings(ws.path()); + + assert_eq!(found.len(), expected.len(), "{name}: {found:#?}"); + for ((code, message), (want_behaviour, want_message)) in found.iter().zip(*expected) { + assert_eq!( + code, + &format!("pattern_ratchet.old.{want_behaviour}"), + "{name}" + ); + assert!(message.starts_with(want_message), "{name}: {message}"); + } + } + + // An empty `paths` would check nothing, so it is rejected; an empty + // `allow` is still legal. + let no_allow = config.replace("allow = ['crates/a/src/allowed.rs']", "allow = []"); + let baseline = ratchet_baseline(&[("old", &[])]); + let ws = workspace(&[ + ("ARCHITECTURE.toml", no_allow.clone()), + ("architecture-baseline.toml", baseline.clone()), + ]); + Context::load(ws.path()).expect("an empty allow list loads"); + let ws = workspace(&[ + ( + "ARCHITECTURE.toml", + no_allow.replace("paths = ['crates/a/**']", "paths = []"), + ), + ("architecture-baseline.toml", baseline), + ]); + let error = Context::load(ws.path()).expect_err("an empty paths list is rejected"); + assert!( + error + .to_string() + .contains("paths must list at least one glob; omit it to check every production file"), + "{error}" + ); +} + /// One finding per behaviour, each reported in its own mode; only required /// findings exit 1. Without `--base-baseline`, no_new_entries is skipped and /// the summary says so. diff --git a/crates/tools/archcheck/tests/repository.rs b/crates/tools/archcheck/tests/repository.rs index 4aad40761..32f1e69ff 100644 --- a/crates/tools/archcheck/tests/repository.rs +++ b/crates/tools/archcheck/tests/repository.rs @@ -84,3 +84,205 @@ fn node_manager_commands_reject_direct_spawns_without_restricting_other_services assert!(check(&ctx).iter().any(|finding| finding.path == owner && finding.code() == "pattern_ratchet.node_manager_commands.listed_no_growth")); } + +/// Repository-relative path of the one crate the `sbxmgr_sqlite` workspace +/// below has: inside the instance's `paths`, outside its `allow`. +const SBXMGR_SRC: &str = "crates/libs/sbx/mgr/server/src"; + +/// The database migrator's sources, outside `sbxmgr_sqlite_schema`'s `allow`. +const MIGRATOR_SRC: &str = "crates/libs/db/migrator/src"; + +/// [`ratchet_findings`] for `sbxmgr_sqlite` and a crate at [`SBXMGR_SRC`]. +fn sbxmgr_sqlite_findings( + entries: &[(&str, usize)], + files: &[(&str, &str)], +) -> Vec<(&'static str, String)> { + ratchet_findings("sbxmgr_sqlite", SBXMGR_SRC, entries, files) +} + +/// A workspace with one crate whose sources are at `src`, checked by the +/// repository's own `[rules.pattern_ratchet.]`, its +/// `[rules.file_size]` and `[rules.crate_names]`, `entries` as the instance's +/// baseline, and `files` under `src`. Returns the instance's findings as +/// `(behaviour, message)`. +fn ratchet_findings( + instance: &str, + src: &str, + entries: &[(&str, usize)], + files: &[(&str, &str)], +) -> Vec<(&'static str, String)> { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../.."); + let text = std::fs::read_to_string(root.join("ARCHITECTURE.toml")) + .expect("read the repository's ARCHITECTURE.toml"); + let mut config: toml::Table = toml::from_str(&text).expect("ARCHITECTURE.toml is TOML"); + config + .get_mut("rules") + .and_then(|rules| rules.get_mut("pattern_ratchet")) + .and_then(toml::Value::as_table_mut) + .expect("ARCHITECTURE.toml has [rules.pattern_ratchet]") + .retain(|name, _| name == instance); + + let dir = tempfile::tempdir().expect("create a temporary workspace"); + let member = src.trim_end_matches("/src"); + let mut baseline = format!("[file_size]\n[crate_names]\n[pattern_ratchet.{instance}]\n"); + for (path, cap) in entries { + baseline.push_str(&format!("\"{src}/{path}\" = {cap}\n")); + } + let mut written = vec![ + ( + "Cargo.toml".to_owned(), + format!("[workspace]\nmembers = [\"{member}\"]\n"), + ), + ( + format!("{member}/Cargo.toml"), + "[package]\nname = \"sbx\"\n[lib]\npath = \"src/lib.rs\"\n".to_owned(), + ), + ( + "ARCHITECTURE.toml".to_owned(), + toml::to_string(&config).expect("serialize the configuration"), + ), + ("architecture-baseline.toml".to_owned(), baseline), + ]; + written.extend( + files + .iter() + .map(|(path, text)| (format!("{src}/{path}"), (*text).to_owned())), + ); + for (path, text) in written { + let path = dir.path().join(path); + std::fs::create_dir_all(path.parent().expect("a parent directory")).expect("create it"); + std::fs::write(path, text).expect("write a workspace file"); + } + let ctx = Context::load(dir.path()).expect("the workspace loads"); + check(&ctx) + .into_iter() + .filter(|finding| finding.instance.as_deref() == Some(instance)) + .map(|finding| (finding.behaviour, finding.message())) + .collect() +} + +/// Every way sbx-mgr code can reach its SQLite database outside the store is +/// a matching line, and ordinary code that shares a word with one is not. +#[test] +fn sbxmgr_sqlite_patterns_match_database_access_only() { + let crossings = [ + "use rusqlite::Connection;", + "use ::rusqlite::Connection;", + "use rusqlite as sqlite;", + "extern crate rusqlite;", + "let conn = Connection::open(path)?;", + "let conn = Connection::open_in_memory()?;", + "let tx = Transaction::new_unchecked(&conn, behavior)?;", + "let conn = pplx_space_common_svc::sql::sqlite::open_rusqlite(path, options)?;", + "use pplx_space_common_jobs::db as jobs;", + "use pplx_space_common_jobs::{JobStatus, db as jobs};", + "let bound = params![id];", + "let bound = named_params! { \":id\": id };", + "conn.execute(sql, bound)?;", + "let n: i64 = conn.query_one(sql, [], |row| row.get(0))?;", + "conn.query_row_and_then(sql, [], decode)?;", + "let mut rows = statement.query([id])?;", + "let present = statement.exists([id])?;", + "let tx = conn.unchecked_transaction()?;", + "let tx = crate::db::immediate_transaction(&conn)?;", + "let sql = \"SELECT state FROM sandboxes WHERE id = ?1\";", + "let sql = \"UPDATE sandboxes", + " DELETE FROM sandboxes WHERE id = ?1", + ]; + for line in crossings { + let found = sbxmgr_sqlite_findings(&[], &[("lib.rs", &format!("{line}\n"))]); + assert_eq!(found.len(), 1, "{line}: {found:#?}"); + assert_eq!(found[0].0, "no_crossing", "{line}"); + } + + let ordinary = [ + "use pplx_space_common_jobs::{JobPriority, JobStatus};", + "let db = crate::db::SharedConnection::new(conn);", + "if path.exists() {", + "let query = uri.query();", + "let worker = executor.execute_one(job);", + "let method = \"DELETE\";", + "let message = format!(\"select a sandbox: {id}\");", + "tracing::warn!(%error, \"update failed\");", + ]; + let text: String = ordinary.iter().map(|line| format!("{line}\n")).collect(); + let found = sbxmgr_sqlite_findings(&[], &[("lib.rs", &text)]); + assert!(found.is_empty(), "{found:#?}"); +} + +/// A listed file grows when it opens a connection through a type it already +/// imported, as does a child module through its parent's import. +#[test] +fn sbxmgr_sqlite_counts_opens_through_imported_names() { + let found = sbxmgr_sqlite_findings( + &[("lib.rs", 1)], + &[( + "lib.rs", + "use rusqlite::Connection;\nfn open() { let conn = Connection::open_in_memory(); }\n", + )], + ); + assert_eq!(found.len(), 1, "{found:#?}"); + assert_eq!(found[0].0, "listed_no_growth"); + assert!(found[0].1.contains("grew from 1 to 2"), "{}", found[0].1); + + let found = sbxmgr_sqlite_findings( + &[("parent.rs", 1)], + &[ + ("parent.rs", "use rusqlite::Connection;\nmod child;\n"), + ( + "parent/child.rs", + "use super::Connection;\nfn open(path: &str) { let conn = Connection::open(path); }\n", + ), + ], + ); + assert_eq!(found.len(), 1, "{found:#?}"); + assert_eq!(found[0].0, "no_crossing"); + assert!( + found[0] + .1 + .starts_with(&format!("{SBXMGR_SRC}/parent/child.rs: 1 code line")), + "{}", + found[0].1 + ); +} + +/// Outside the store, the migrator may not name sbx-mgr's database: its +/// migrations, its path or file, or its own default-path helper. The other +/// services' databases it migrates are not matches. +#[test] +fn sbxmgr_sqlite_schema_matches_the_migrators_sandbox_manager_database() { + let crossings = [ + "use pplx_space_sbxmgr_migrations::SANDBOX_MANAGER_MIGRATOR;", + "default_path: paths::SANDBOX_MANAGER_DB,", + "Database::SandboxManager(args) => run_sqlite(sqlite_sandbox_manager(), args).await,", + "fn sqlite_sandbox_manager() -> SqliteDatabase {", + "default_path: \"/space/run/sandbox-manager/sbx-manager.db\",", + "static MIGRATOR: Migrator = sqlx::migrate!(\"../../../../db/sandbox-manager\");", + ]; + for line in crossings { + let found = ratchet_findings( + "sbxmgr_sqlite_schema", + MIGRATOR_SRC, + &[], + &[("lib.rs", &format!("{line}\n"))], + ); + assert_eq!(found.len(), 1, "{line}: {found:#?}"); + assert_eq!(found[0].0, "no_crossing", "{line}"); + } + + let ordinary = [ + "Database::TemplateManager(args) => run_sqlite(sqlite_template_manager(), args).await,", + "default_path: paths::TEMPLATE_MANAGER_DB,", + "default_path: \"/space/run/template-manager/template-manager.db\",", + "static MIGRATOR: Migrator = sqlx::migrate!(\"../../../../db/template-manager\");", + "SandboxManager(SqliteArgs),", + ]; + let text: String = ordinary.iter().map(|line| format!("{line}\n")).collect(); + let found = ratchet_findings( + "sbxmgr_sqlite_schema", + MIGRATOR_SRC, + &[], + &[("lib.rs", &text)], + ); + assert!(found.is_empty(), "{found:#?}"); +}