From 16016bbb7a0ec1fb56de5489435023b8e02b5b87 Mon Sep 17 00:00:00 2001 From: Abhinav Chordia Date: Fri, 16 Oct 2026 05:30:04 +0000 Subject: [PATCH] sbx-mgr: route every SQLite statement through db/ and fence the rest with cargo arch sbx-mgr's node-local SQLite database (`sandboxes` and every other sbx-mgr table) was queried from seven modules outside `src/db`, and `db/` had two-way dependencies with `auto_lifecycle_delete_receipts` and `route_activation_claims`. Every statement now lives in a named `db/` function, and new `cargo arch` ratchets count what is left and trip on the common ways of adding a crossing. They are line-based tripwires, not a guarantee: aliases, wrapped imports and calls through names imported earlier escape them. The compile-time boundary is the store crate that follows. Store moves, with callers keeping their transactions, locks and error mapping: - `auto_lifecycle_delete_receipts/db.rs` moves to `db/auto_lifecycle_delete_receipts.rs`, with its tests. The parent module keeps only the receipt types, so it is a leaf `db/` depends on. - The claim type and statements of `route_activation_claims` move to `db/route_activation_claims.rs`; the admission check that reads them stays outside and depends on `db/` one way. - `local_artifacts.rs`'s statements and row types move to `db/local_artifact_leases.rs`; `LocalArtifacts` and its leases keep their transactions, mutex and poisoning behaviour, and re-export the types. - `sandbox_eviction`'s scans and evidence reads move to `db/eviction_scan.rs`, the watchdog's to `db/guest_health_scan.rs`, and the snapshot-upload jobs' to `db/snapshot_upload_jobs.rs`. The marker backfill visits rows through the open cursor and samples its timestamp after binding the read, as before, so its writes interleave with the read in the same order. The watchdog's per-id state lookup gains a test. - Cross-node recovery reuses `db::current_state_label`. `db/` now depends only on the leaf type modules `state`, `auto_lifecycle`, `auto_lifecycle_delete_receipts`, `endpoint_proof` and `restore_operation`. archcheck's `pattern_ratchet` gains an optional, non-empty `paths` list that limits an instance to the files it matches, since other services' crates use rusqlite for their own databases. `[rules.pattern_ratchet.sbxmgr_sqlite]` covers `crates/libs/sbx/mgr/**` and `crates/apps/sbx/**` and allows only `src/db`. It matches rusqlite paths and imports, connection and transaction constructors, the shared SQLite open helpers by their qualified path, the shared job store's SQL functions, parameter macros, statement and transaction calls, and SQL statement literals. Its baseline is seeded with the 34 files that still name rusqlite types, open transactions or import the job store (203 lines); no SQL statement remains outside `db/`. `[rules.pattern_ratchet.sbxmgr_sqlite_schema]` matches the sbx-mgr migrations crate, the database's path, file name, migrator and migrations directory names, and the database migrator's default-path helper, in any crate outside the store, the migrations crate and the path's definition. Its baseline lists `run.rs` and the database migrator (8 lines). --- ARCHITECTURE.toml | 84 + architecture-baseline.toml | 53 +- .../src/auto_lifecycle_delete_receipts/mod.rs | 1 - .../auto_lifecycle_delete_receipts/tests.rs | 1969 +---------------- crates/libs/sbx/mgr/server/src/db.rs | 13 +- .../sbx/mgr/server/src/db/auto_lifecycle.rs | 2 +- .../auto_lifecycle_delete_receipts.rs} | 13 +- .../auto_lifecycle_delete_receipts/tests.rs | 1962 ++++++++++++++++ .../sbx/mgr/server/src/db/cleanup_episodes.rs | 28 +- .../sbx/mgr/server/src/db/cleanup_jobs.rs | 2 +- crates/libs/sbx/mgr/server/src/db/eviction.rs | 2 +- .../sbx/mgr/server/src/db/eviction_scan.rs | 218 ++ .../mgr/server/src/db/guest_health_scan.rs | 66 + .../server/src/db/guest_health_scan/tests.rs | 30 + .../server/src/db/local_artifact_leases.rs | 382 ++++ .../sbx/mgr/server/src/db/local_artifacts.rs | 2 +- crates/libs/sbx/mgr/server/src/db/pool.rs | 10 +- .../libs/sbx/mgr/server/src/db/quarantine.rs | 2 +- .../server/src/db/route_activation_claims.rs | 142 ++ .../libs/sbx/mgr/server/src/db/sandboxes.rs | 40 +- .../mgr/server/src/db/snapshot_upload_jobs.rs | 87 + .../sbx/mgr/server/src/db/startup_recovery.rs | 2 +- .../mgr/server/src/db/state_transitions.rs | 20 +- .../sbx/mgr/server/src/local_artifacts.rs | 333 +-- .../src/local_artifacts/refusal_tests.rs | 2 + .../mgr/server/src/local_artifacts/tests.rs | 1 + .../mgr/server/src/route_activation_claims.rs | 140 +- .../sbx/mgr/server/src/sandbox_eviction.rs | 198 +- .../mgr/server/src/sandbox_eviction/tests.rs | 1 + .../server/src/service/activation_support.rs | 2 +- .../sbx/mgr/server/src/service/cleanup.rs | 16 +- .../server/src/service/cleanup_executor.rs | 2 +- .../libs/sbx/mgr/server/src/service/create.rs | 4 +- .../grpc_auto_lifecycle_delete_receipts.rs | 4 +- .../mgr/server/src/service/grpc_core/tests.rs | 2 +- crates/libs/sbx/mgr/server/src/service/mod.rs | 14 +- .../sbx/mgr/server/src/service/reconcile.rs | 2 +- .../server/src/service/recovery_cross_node.rs | 15 +- .../src/service/route_identity_tests.rs | 4 +- .../libs/sbx/mgr/server/src/service/start.rs | 2 +- .../libs/sbx/mgr/server/src/service/tests.rs | 52 +- .../mgr/server/src/snapshot_upload_jobs.rs | 141 +- crates/libs/sbx/mgr/server/src/watchdog.rs | 56 +- crates/tools/archcheck/README.md | 19 +- crates/tools/archcheck/src/config.rs | 45 +- .../archcheck/src/rules/pattern_ratchet.rs | 32 +- crates/tools/archcheck/tests/cli.rs | 105 + crates/tools/archcheck/tests/repository.rs | 202 ++ 48 files changed, 3659 insertions(+), 2865 deletions(-) rename crates/libs/sbx/mgr/server/src/{auto_lifecycle_delete_receipts/db.rs => db/auto_lifecycle_delete_receipts.rs} (99%) create mode 100644 crates/libs/sbx/mgr/server/src/db/auto_lifecycle_delete_receipts/tests.rs create mode 100644 crates/libs/sbx/mgr/server/src/db/eviction_scan.rs create mode 100644 crates/libs/sbx/mgr/server/src/db/guest_health_scan.rs create mode 100644 crates/libs/sbx/mgr/server/src/db/guest_health_scan/tests.rs create mode 100644 crates/libs/sbx/mgr/server/src/db/local_artifact_leases.rs create mode 100644 crates/libs/sbx/mgr/server/src/db/route_activation_claims.rs create mode 100644 crates/libs/sbx/mgr/server/src/db/snapshot_upload_jobs.rs diff --git a/ARCHITECTURE.toml b/ARCHITECTURE.toml index d10f4ddb3..b122c61d0 100644 --- a/ARCHITECTURE.toml +++ b/ARCHITECTURE.toml @@ -244,6 +244,90 @@ mode = "required" [rules.pattern_ratchet.node_manager_commands.stale_entries] mode = "required" +# sbx-mgr's node-local SQLite database (`sandboxes` and every other sbx-mgr +# table) is opened, migrated and queried only by its store, `src/db` of the +# server crate. Within sbx-mgr's crates, naming rusqlite, opening a +# connection or transaction, using the shared SQLite open and migration +# helpers or the shared job store's SQL functions, binding parameters, +# executing a statement, or writing an SQL statement literal outside the +# store is a crossing. Other services' crates use rusqlite for their own +# databases, so `paths` limits the rule to sbx-mgr's. Existing callers that +# still name the connection, error and transaction types are baselined +# until they use the store's own. +[rules.pattern_ratchet.sbxmgr_sqlite] +patterns = [ + '\brusqlite\s*::', + '\buse\s+(?:::\s*)?rusqlite\b', + '\bextern\s+crate\s+rusqlite\b', + '\b(?:Connection|Transaction)\s*::\s*(?:open\w*|new\w*)\s*\(', + '\bsql\s*::\s*sqlite\b', + '\bpplx_space_common_jobs\s*::\s*(?:db\b|\{[^}]*\bdb\b)', + '\b(?:named_)?params\s*!', + '\.\s*(?:execute|execute_batch|prepare|prepare_cached|query_row|query_row_and_then|query_one|query_map|query_and_then|transaction|unchecked_transaction|transaction_with_behavior|savepoint|pragma_update|pragma_query|pragma_query_value)\s*\(', + '\.\s*(?:query|exists)\s*\(\s*[^)\s]', + '\bimmediate_transaction\s*\(', + '"\s*(?:SELECT\b|INSERT\s+(?:OR\s+[A-Z]+\s+)?INTO\b|REPLACE\s+INTO\b|UPDATE\s+(?:OR\s+[A-Z]+\s+)?\w+(?:\s+SET\b|\s*$)|DELETE\s+FROM\b|(?:CREATE|DROP|ALTER)\s+(?:TEMP\s+|TEMPORARY\s+|UNIQUE\s+)?(?:TABLE|INDEX|TRIGGER|VIEW)\b|PRAGMA\s+\w|WITH\s+(?:RECURSIVE\s+)?\w+\s+AS\b|(?:BEGIN|COMMIT|ROLLBACK|SAVEPOINT|RELEASE|VACUUM)\b[\w\s]*;?\s*")', + '^\s*(?:SELECT\s|INSERT\s+(?:OR\s+[A-Z]+\s+)?INTO\s|UPDATE\s+(?:OR\s+[A-Z]+\s+)?\w+(?:\s+SET\s|\s*$)|DELETE\s+FROM\s|WITH\s+(?:RECURSIVE\s+)?\w+\s+AS\b)', +] +paths = ["crates/libs/sbx/mgr/**", "crates/apps/sbx/**"] +allow = [ + "crates/libs/sbx/mgr/server/src/db.rs", + "crates/libs/sbx/mgr/server/src/db/**", +] + +[rules.pattern_ratchet.sbxmgr_sqlite.no_crossing] +mode = "required" + +# Seeding the ratchet reports its initial entries against the base. +[rules.pattern_ratchet.sbxmgr_sqlite.no_new_entries] +mode = "advisory" + +[rules.pattern_ratchet.sbxmgr_sqlite.no_raised_caps] +mode = "required" + +[rules.pattern_ratchet.sbxmgr_sqlite.listed_no_growth] +mode = "required" + +[rules.pattern_ratchet.sbxmgr_sqlite.stale_entries] +mode = "required" + +# The same database's schema and location, in any crate: its migrations, its +# path and file name, and the migrator's default-path helper for it belong to +# the store, which opens and migrates it. The migrations crate defines the +# migrators and common-core the path. These names are +# specific enough to check workspace-wide, unlike `sbxmgr_sqlite`'s patterns, +# which other services' crates match for their own databases. +[rules.pattern_ratchet.sbxmgr_sqlite_schema] +patterns = [ + '\bpplx_space_sbxmgr_migrations\b', + '\bSANDBOX_MANAGER_(?:DB|MIGRATOR)\b', + '\bsqlite_sandbox_manager\b', + '\bsbx-manager\.db\b', + '\bdb/sandbox-manager\b', +] +allow = [ + "crates/libs/sbx/mgr/server/src/db.rs", + "crates/libs/sbx/mgr/server/src/db/**", + "crates/libs/sbx/mgr/migrations/src/**", + "crates/libs/common/core/src/host/linux.rs", +] + +[rules.pattern_ratchet.sbxmgr_sqlite_schema.no_crossing] +mode = "required" + +# Seeding the ratchet reports its initial entries against the base. +[rules.pattern_ratchet.sbxmgr_sqlite_schema.no_new_entries] +mode = "advisory" + +[rules.pattern_ratchet.sbxmgr_sqlite_schema.no_raised_caps] +mode = "required" + +[rules.pattern_ratchet.sbxmgr_sqlite_schema.listed_no_growth] +mode = "required" + +[rules.pattern_ratchet.sbxmgr_sqlite_schema.stale_entries] +mode = "required" + [rules.crate_names] # A crate matches if a part of its package or library name, lowercased and # split on `-` and `_`, is one of these. diff --git a/architecture-baseline.toml b/architecture-baseline.toml index 7489f3c5d..48e0fc864 100644 --- a/architecture-baseline.toml +++ b/architecture-baseline.toml @@ -75,13 +75,13 @@ "crates/libs/object-store/src/lib.rs" = 1810 "crates/libs/observability/src/lib.rs" = 1063 "crates/libs/public/api/src/lib.rs" = 1793 -"crates/libs/sbx/mgr/server/src/auto_lifecycle_delete_receipts/db.rs" = 1158 +"crates/libs/sbx/mgr/server/src/db/auto_lifecycle_delete_receipts.rs" = 1158 "crates/libs/sbx/mgr/server/src/db/cleanup_episodes.rs" = 1833 "crates/libs/sbx/mgr/server/src/db/quarantine_recovery.rs" = 1042 -"crates/libs/sbx/mgr/server/src/db/sandboxes.rs" = 1553 -"crates/libs/sbx/mgr/server/src/db/state_transitions.rs" = 1018 +"crates/libs/sbx/mgr/server/src/db/sandboxes.rs" = 1547 +"crates/libs/sbx/mgr/server/src/db/state_transitions.rs" = 1016 "crates/libs/sbx/mgr/server/src/run.rs" = 1101 -"crates/libs/sbx/mgr/server/src/sandbox_eviction.rs" = 1273 +"crates/libs/sbx/mgr/server/src/sandbox_eviction.rs" = 1253 "crates/libs/sbx/mgr/server/src/service/activation_support.rs" = 1574 "crates/libs/sbx/mgr/server/src/service/auth_lookup.rs" = 1221 "crates/libs/sbx/mgr/server/src/service/cache_cleanup.rs" = 1455 @@ -145,6 +145,51 @@ "crates/libs/node/mgr/server/src/service/artifact_fs/receive.rs" = 4 "crates/libs/node/mgr/server/src/net/fallback.rs" = 1 +# sbx-mgr code outside its SQLite store that still names rusqlite types, the +# shared SQLite helpers or the shared job store, or opens a transaction itself. +[pattern_ratchet.sbxmgr_sqlite] +"crates/libs/sbx/mgr/server/src/lifecycle_metrics.rs" = 1 +"crates/libs/sbx/mgr/server/src/local_artifacts.rs" = 28 +"crates/libs/sbx/mgr/server/src/node_state_sync.rs" = 1 +"crates/libs/sbx/mgr/server/src/route_activation_claims.rs" = 2 +"crates/libs/sbx/mgr/server/src/run.rs" = 6 +"crates/libs/sbx/mgr/server/src/runtime_reconciliation.rs" = 1 +"crates/libs/sbx/mgr/server/src/sandbox_eviction.rs" = 5 +"crates/libs/sbx/mgr/server/src/service/activation_support.rs" = 9 +"crates/libs/sbx/mgr/server/src/service/auth_lookup.rs" = 5 +"crates/libs/sbx/mgr/server/src/service/cache_cleanup.rs" = 12 +"crates/libs/sbx/mgr/server/src/service/cleanup.rs" = 15 +"crates/libs/sbx/mgr/server/src/service/cleanup_executor.rs" = 7 +"crates/libs/sbx/mgr/server/src/service/cleanup_support.rs" = 7 +"crates/libs/sbx/mgr/server/src/service/cleanup_targets.rs" = 1 +"crates/libs/sbx/mgr/server/src/service/create_support.rs" = 5 +"crates/libs/sbx/mgr/server/src/service/grpc_lifecycle/pause_resume.rs" = 2 +"crates/libs/sbx/mgr/server/src/service/grpc_lifecycle/teardown_suspend.rs" = 2 +"crates/libs/sbx/mgr/server/src/service/grpc_pool_wait_clone.rs" = 1 +"crates/libs/sbx/mgr/server/src/service/grpc_spaced_process_snapshots/rollback_state.rs" = 1 +"crates/libs/sbx/mgr/server/src/service/grpc_state_snapshot.rs" = 1 +"crates/libs/sbx/mgr/server/src/service/mod.rs" = 13 +"crates/libs/sbx/mgr/server/src/service/network_runtime.rs" = 5 +"crates/libs/sbx/mgr/server/src/service/recovery.rs" = 5 +"crates/libs/sbx/mgr/server/src/service/recovery_cross_node.rs" = 12 +"crates/libs/sbx/mgr/server/src/service/start_support.rs" = 1 +"crates/libs/sbx/mgr/server/src/service/startup_resident.rs" = 1 +"crates/libs/sbx/mgr/server/src/service/storage.rs" = 1 +"crates/libs/sbx/mgr/server/src/service/template_snapshot_support.rs" = 3 +"crates/libs/sbx/mgr/server/src/service/test_options.rs" = 1 +"crates/libs/sbx/mgr/server/src/snapshot_gc.rs" = 1 +"crates/libs/sbx/mgr/server/src/snapshot_restore.rs" = 5 +"crates/libs/sbx/mgr/server/src/snapshot_scheduler.rs" = 3 +"crates/libs/sbx/mgr/server/src/snapshot_upload_jobs.rs" = 19 +"crates/libs/sbx/mgr/server/src/watchdog.rs" = 21 + +# Code outside the store that still names sbx-mgr's migrations or database +# path: its open and migration in `run.rs`, and the migrator, until it calls +# the store. +[pattern_ratchet.sbxmgr_sqlite_schema] +"crates/libs/db/migrator/src/lib.rs" = 5 +"crates/libs/sbx/mgr/server/src/run.rs" = 3 + # [crate_names] maps each workspace package with a catch-all name (a part such # as `common`, `utils` or `core`) to its member path. Only remove entries: a pull # request that adds one is reported, and fails the check. diff --git a/crates/libs/sbx/mgr/server/src/auto_lifecycle_delete_receipts/mod.rs b/crates/libs/sbx/mgr/server/src/auto_lifecycle_delete_receipts/mod.rs index 746baab6c..392aeeeb6 100644 --- a/crates/libs/sbx/mgr/server/src/auto_lifecycle_delete_receipts/mod.rs +++ b/crates/libs/sbx/mgr/server/src/auto_lifecycle_delete_receipts/mod.rs @@ -7,7 +7,6 @@ use pplx_space_common_core::types::{ }; use pplx_space_common_svc::types::AutoLifecyclePolicyFingerprint; -pub(crate) mod db; #[cfg(test)] mod tests; diff --git a/crates/libs/sbx/mgr/server/src/auto_lifecycle_delete_receipts/tests.rs b/crates/libs/sbx/mgr/server/src/auto_lifecycle_delete_receipts/tests.rs index b7a25302b..2e51dca68 100644 --- a/crates/libs/sbx/mgr/server/src/auto_lifecycle_delete_receipts/tests.rs +++ b/crates/libs/sbx/mgr/server/src/auto_lifecycle_delete_receipts/tests.rs @@ -1,978 +1,9 @@ -use std::collections::BTreeMap; - use pplx_space_common_core::types::{ - AutoLifecycleConditionSet, AutoLifecycleConditionSetId, AutoLifecycleDeleteReceiptId, - AutoLifecycleIdleAction, AutoLifecyclePolicy, AutoLifecycleStage, AutoLifecycleWakePolicy, - ExecutionGeneration, NodeManagedBy, NodeRole, PodUid, PolicyAuthority, RuntimeKind, - SandboxActivationAttemptId, SandboxId, SandboxOwner, SandboxRuntimeLabel, - SpaceApiRouteIdentity, + ExecutionGeneration, NodeManagedBy, PodUid, SandboxActivationAttemptId, SpaceApiRouteIdentity, }; -use pplx_space_common_svc::types::AutoLifecyclePolicyFingerprint; -use rusqlite::{Connection, OptionalExtension}; use super::*; -fn delete_policy() -> AutoLifecyclePolicy { - AutoLifecyclePolicy { - wake: AutoLifecycleWakePolicy::default(), - condition_sets: [( - AutoLifecycleConditionSetId::AccessIdle, - AutoLifecycleConditionSet::access_idle(), - )] - .into(), - stages: vec![AutoLifecycleStage { - condition_set: AutoLifecycleConditionSetId::AccessIdle, - at: 900, - action: AutoLifecycleIdleAction::Delete, - }], - } -} - -fn insert_sandbox(conn: &Connection, sandbox_id: SandboxId) -> AutoLifecycleDeleteExpectation { - let owner = SandboxOwner::parse("user").unwrap(); - let runtime_label = SandboxRuntimeLabel::from_sandbox_id(sandbox_id); - let policy = delete_policy(); - crate::db::insert_sandbox( - conn, - crate::db::InsertSandboxParams { - id: &sandbox_id.to_string(), - uuid8: &runtime_label, - state: crate::state::SandboxState::Started, - template: "seed", - template_hash: Some("hash"), - user: &owner, - runtime: RuntimeKind::Firecracker.as_str(), - snapshots_enabled: false, - rootfs_size_mb: None, - disk_io_limits: Default::default(), - pool_template: None, - volume_encryption: false, - process_env: &BTreeMap::new(), - auto_lifecycle: &policy, - network_policy_override: None, - network_policy_generation: 0, - network_policy_authority: PolicyAuthority::Cluster, - auto_lifecycle_authority: PolicyAuthority::Cluster, - }, - ) - .unwrap(); - AutoLifecycleDeleteExpectation { - policy_fingerprint: AutoLifecyclePolicyFingerprint::for_policy(&policy), - authority: PolicyAuthority::Cluster, - } -} - -fn cluster_route( - generation: i64, - activation_attempt_id: SandboxActivationAttemptId, -) -> SpaceApiRouteIdentity { - SpaceApiRouteIdentity { - execution_generation: Some(ExecutionGeneration::new(generation).unwrap()), - activation_attempt_id: Some(activation_attempt_id), - substrate_kind: Some(NodeManagedBy::Kubernetes), - ..SpaceApiRouteIdentity::default() - } -} - -fn receipt( - sandbox_id: SandboxId, - expectation: AutoLifecycleDeleteExpectation, - generation: i64, - activation_attempt_id: SandboxActivationAttemptId, -) -> AutoLifecycleDeleteReceipt { - AutoLifecycleDeleteReceipt { - receipt_id: AutoLifecycleDeleteReceiptId::generate(), - sandbox_id, - execution_generation: ExecutionGeneration::new(generation).unwrap(), - route_activation_id: activation_attempt_id, - policy_fingerprint: expectation.policy_fingerprint, - } -} - -#[test] -fn delete_admission_requires_consistent_authority_and_execution_evidence() { - use AutoLifecycleDeleteBlockReason as Block; - use AutoLifecycleDeleteOrdinaryReason as Ordinary; - - #[derive(Clone, Copy, Debug)] - enum Expected { - Ordinary(Ordinary), - Receipt, - Blocked(Block), - } - let cases = [ - ( - "local", - Some(NodeManagedBy::Space), - PolicyAuthority::Local, - "UPDATE sandboxes SET space_api_route_substrate_kind = 'k8s' WHERE id = ?3", - Expected::Ordinary(Ordinary::LocalAuthority), - ), - ( - "route-less direct", - Some(NodeManagedBy::Space), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET state = 'stopped' WHERE id = ?3", - Expected::Ordinary(Ordinary::DirectUnsupported), - ), - ( - "explicit direct", - None, - PolicyAuthority::Cluster, - "UPDATE sandboxes SET space_api_route_substrate_kind = 'space' WHERE id = ?3", - Expected::Ordinary(Ordinary::DirectUnsupported), - ), - ( - "current", - Some(NodeManagedBy::Kubernetes), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET space_api_route_execution_generation = 7, space_api_route_activation_attempt_id = ?1, space_api_route_substrate_kind = 'k8s', space_api_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", - Expected::Receipt, - ), - ( - "retained", - Some(NodeManagedBy::Kubernetes), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET last_observed_execution_generation = 7, last_observed_route_activation_id = ?1, last_observed_route_substrate_kind = 'k8s', last_observed_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", - Expected::Receipt, - ), - ( - "suspended subset", - Some(NodeManagedBy::Kubernetes), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET state = 'suspended', space_api_route_execution_generation = 7, last_observed_execution_generation = 7, last_observed_route_activation_id = ?1, last_observed_route_substrate_kind = 'k8s', last_observed_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", - Expected::Receipt, - ), - ( - "missing Kubernetes", - Some(NodeManagedBy::Kubernetes), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET state = 'stopped' WHERE id = ?3", - Expected::Blocked(Block::MissingRouteIdentity), - ), - ( - "missing unknown host", - None, - PolicyAuthority::Cluster, - "UPDATE sandboxes SET state = 'stopped' WHERE id = ?3", - Expected::Blocked(Block::MissingRouteIdentity), - ), - ( - "stale retained", - Some(NodeManagedBy::Kubernetes), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET space_api_route_execution_generation = 8, space_api_route_activation_attempt_id = ?1, space_api_route_substrate_kind = 'k8s', space_api_route_identity_provenance = 'cluster_fenced', last_observed_execution_generation = 7, last_observed_route_activation_id = ?1, last_observed_route_substrate_kind = 'k8s', last_observed_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", - Expected::Blocked(Block::ConflictingRouteIdentity), - ), - ( - "conflicting attempts", - Some(NodeManagedBy::Kubernetes), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET space_api_route_execution_generation = 7, space_api_route_activation_attempt_id = ?1, space_api_route_substrate_kind = 'k8s', space_api_route_identity_provenance = 'cluster_fenced', last_observed_execution_generation = 7, last_observed_route_activation_id = ?2, last_observed_route_substrate_kind = 'k8s', last_observed_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", - Expected::Blocked(Block::ConflictingRouteIdentity), - ), - ( - "host contradicts route", - Some(NodeManagedBy::Space), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET space_api_route_substrate_kind = 'k8s' WHERE id = ?3", - Expected::Blocked(Block::ConflictingRouteIdentity), - ), - ( - "malformed current attempt", - Some(NodeManagedBy::Kubernetes), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET space_api_route_execution_generation = 7, space_api_route_activation_attempt_id = 'malformed', space_api_route_substrate_kind = 'k8s', space_api_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", - Expected::Blocked(Block::ConflictingRouteIdentity), - ), - ( - "malformed retained attempt", - Some(NodeManagedBy::Kubernetes), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET space_api_route_execution_generation = 7, space_api_route_activation_attempt_id = ?1, space_api_route_substrate_kind = 'k8s', space_api_route_identity_provenance = 'cluster_fenced', last_observed_execution_generation = 7, last_observed_route_activation_id = 'malformed', last_observed_route_substrate_kind = 'k8s', last_observed_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", - Expected::Blocked(Block::ConflictingRouteIdentity), - ), - ( - "invalid current generation", - Some(NodeManagedBy::Kubernetes), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET space_api_route_execution_generation = -1, space_api_route_activation_attempt_id = ?1, space_api_route_substrate_kind = 'k8s', space_api_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", - Expected::Blocked(Block::ConflictingRouteIdentity), - ), - ( - "invalid retained generation", - Some(NodeManagedBy::Kubernetes), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET last_observed_execution_generation = 0, last_observed_route_activation_id = ?1, last_observed_route_substrate_kind = 'k8s', last_observed_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", - Expected::Blocked(Block::ConflictingRouteIdentity), - ), - ( - "mixed substrates", - None, - PolicyAuthority::Cluster, - "UPDATE sandboxes SET space_api_route_substrate_kind = 'space', last_observed_route_substrate_kind = 'k8s' WHERE id = ?3", - Expected::Blocked(Block::ConflictingRouteIdentity), - ), - ( - "unknown substrate", - Some(NodeManagedBy::Space), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET last_observed_route_substrate_kind = 'future' WHERE id = ?3", - Expected::Blocked(Block::UnknownSubstrate), - ), - ( - "partial retained", - Some(NodeManagedBy::Kubernetes), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET space_api_route_execution_generation = 7, space_api_route_activation_attempt_id = ?1, space_api_route_substrate_kind = 'k8s', space_api_route_identity_provenance = 'cluster_fenced', last_observed_execution_generation = 7 WHERE id = ?3", - Expected::Blocked(Block::ConflictingRouteIdentity), - ), - ( - "pod contradicts direct", - Some(NodeManagedBy::Space), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET space_api_route_pod_uid = 'pod' WHERE id = ?3", - Expected::Blocked(Block::ConflictingRouteIdentity), - ), - ( - "retained placement contradicts direct", - Some(NodeManagedBy::Space), - PolicyAuthority::Cluster, - "UPDATE sandboxes SET last_observed_route_generation_substrate_kind = 'k8s' WHERE id = ?3", - Expected::Blocked(Block::ConflictingRouteIdentity), - ), - ]; - for (name, host, authority, setup, expected) in cases { - let conn = crate::db::test_conn(); - let id = SandboxId::new(); - let mut expectation = insert_sandbox(&conn, id); - expectation.authority = authority; - let attempt = SandboxActivationAttemptId::new(); - // Seed historical malformed evidence without weakening admission writes. - conn.pragma_update(None, "ignore_check_constraints", true) - .unwrap(); - conn.execute( - setup, - rusqlite::params![ - attempt.to_string(), - SandboxActivationAttemptId::new().to_string(), - id.to_string() - ], - ) - .unwrap(); - conn.pragma_update(None, "ignore_check_constraints", false) - .unwrap(); - conn.execute( - "UPDATE sandboxes SET auto_lifecycle_authority = ?1 WHERE id = ?2", - rusqlite::params![authority.to_string(), id.to_string()], - ) - .unwrap(); - let admitted = - db::classify_and_record_admission(&conn, id, Some(NodeRole::Worker), host, expectation) - .unwrap(); - match (expected, admitted) { - (Expected::Ordinary(expected), AutoLifecycleDeleteAdmission::Ordinary(actual)) => { - assert_eq!(actual, expected, "{name}") - } - (Expected::Receipt, AutoLifecycleDeleteAdmission::Receipt(evidence)) => { - assert_eq!(evidence.execution_generation.get(), 7, "{name}"); - assert_eq!(evidence.activation_attempt_id, attempt, "{name}"); - } - (Expected::Blocked(expected), AutoLifecycleDeleteAdmission::Blocked { reason, .. }) => { - assert_eq!(reason, expected, "{name}"); - let retry: i64 = conn - .query_row( - "SELECT next_auto_event_at FROM sandboxes WHERE id = ?1", - [id.to_string()], - |row| row.get(0), - ) - .unwrap(); - assert!(retry > crate::db::now_epoch_i64().unwrap(), "{name}"); - assert!(crate::db::sandbox_exists(&conn, &id.to_string()).unwrap()); - } - pair => panic!("unexpected {name}: {pair:?}"), - } - } - for completed in [false, true] { - let conn = crate::db::test_conn(); - let id = SandboxId::new(); - let expectation = insert_sandbox(&conn, id); - let pending = receipt(id, expectation, 7, SandboxActivationAttemptId::new()); - db::insert_pending_receipt(&conn, pending).unwrap(); - if completed { - db::complete_pending_receipt(&conn, id, pending.receipt_id).unwrap(); - } - let admitted = db::classify_and_record_admission( - &conn, - id, - Some(NodeRole::Worker), - Some(NodeManagedBy::Space), - expectation, - ) - .unwrap(); - if completed { - assert_eq!( - admitted, - AutoLifecycleDeleteAdmission::Ordinary(Ordinary::DirectUnsupported) - ); - } else { - assert!(matches!( - admitted, - AutoLifecycleDeleteAdmission::Blocked { - reason: Block::ConflictingRouteIdentity, - .. - } - )); - } - assert_eq!( - conn.query_row( - "SELECT receipt_id, state FROM auto_lifecycle_delete_outbox WHERE sandbox_id = ?1", - [id.to_string()], - |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), - ) - .unwrap(), - ( - pending.receipt_id.to_string(), - if completed { "completed" } else { "pending" }.into() - ) - ); - } -} - -#[test] -fn strict_admission_requires_complete_current_cluster_fencing() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let activation_attempt_id = SandboxActivationAttemptId::new(); - crate::db::update_space_api_route_identity( - &conn, - sandbox_id, - &cluster_route(7, activation_attempt_id), - RouteIdentityProvenance::ClusterFenced, - ) - .unwrap(); - - assert!(matches!( - db::classify_and_record_admission( - &conn, - sandbox_id, - Some(NodeRole::Worker), - None, expectation, - ) - .unwrap(), - AutoLifecycleDeleteAdmission::Receipt(RouteEvidence { - execution_generation, - activation_attempt_id: observed_attempt, - .. - }) if execution_generation == ExecutionGeneration::new(7).unwrap() - && observed_attempt == activation_attempt_id - )); - - conn.execute( - "UPDATE sandboxes - SET space_api_route_execution_generation = NULL, - space_api_route_activation_attempt_id = NULL, - space_api_route_substrate_kind = NULL, - space_api_route_identity_provenance = NULL - WHERE id = ?1", - [sandbox_id.to_string()], - ) - .unwrap(); - assert!(matches!( - db::classify_and_record_admission(&conn, sandbox_id, Some(NodeRole::Worker), None, expectation,) - .unwrap(), - AutoLifecycleDeleteAdmission::Receipt(RouteEvidence { - execution_generation, - activation_attempt_id: observed_attempt, - .. - }) if execution_generation == ExecutionGeneration::new(7).unwrap() - && observed_attempt == activation_attempt_id - )); - - conn.execute( - "UPDATE sandboxes - SET space_api_route_execution_generation = 8, - space_api_route_activation_attempt_id = NULL, - space_api_route_substrate_kind = NULL, - space_api_route_identity_provenance = NULL - WHERE id = ?1", - [sandbox_id.to_string()], - ) - .unwrap(); - assert!(matches!( - db::classify_and_record_admission( - &conn, - sandbox_id, - Some(NodeRole::Worker), - None, - expectation, - ) - .unwrap(), - AutoLifecycleDeleteAdmission::Blocked { - reason: AutoLifecycleDeleteBlockReason::MissingSubstrate, - changed: true, - } - )); - let (reason, blocked_at, next_auto_event_at, runtime_state_json): ( - Option, - Option, - Option, - String, - ) = conn - .query_row( - "SELECT auto_lifecycle_delete_block_reason, - auto_lifecycle_delete_blocked_at, - next_auto_event_at, - auto_lifecycle_state_json - FROM sandboxes WHERE id = ?1", - [sandbox_id.to_string()], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), - ) - .unwrap(); - assert_eq!(reason.as_deref(), Some("missing_substrate")); - assert!(blocked_at.is_some()); - assert!(next_auto_event_at.unwrap() > crate::db::now_epoch_i64().unwrap()); - let runtime_state: pplx_space_common_core::types::AutoLifecycleRuntimeState = - serde_json::from_str(&runtime_state_json).unwrap(); - assert!( - runtime_state - .last_auto_error - .as_deref() - .is_some_and(|error| error.contains("missing_substrate")) - ); - - crate::db::update_space_api_route_identity( - &conn, - sandbox_id, - &cluster_route(8, SandboxActivationAttemptId::new()), - RouteIdentityProvenance::ClusterFenced, - ) - .unwrap(); - let (reason, blocked_at, runtime_state_json): (Option, Option, String) = conn - .query_row( - "SELECT auto_lifecycle_delete_block_reason, - auto_lifecycle_delete_blocked_at, - auto_lifecycle_state_json - FROM sandboxes WHERE id = ?1", - [sandbox_id.to_string()], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), - ) - .unwrap(); - assert_eq!((reason, blocked_at), (None, None)); - let runtime_state: pplx_space_common_core::types::AutoLifecycleRuntimeState = - serde_json::from_str(&runtime_state_json).unwrap(); - assert_eq!(runtime_state.last_auto_error, None); -} - -#[test] -fn dormant_suspend_identity_uses_matching_retained_route_evidence() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let activation_attempt_id = SandboxActivationAttemptId::new(); - crate::db::update_space_api_route_identity( - &conn, - sandbox_id, - &cluster_route(7, activation_attempt_id), - RouteIdentityProvenance::ClusterFenced, - ) - .unwrap(); - - // Suspend completion retains generation for central publication but clears - // the remaining current route fields. The matching complete retained tuple - // is still the exact route that produced this dormant sandbox. - conn.execute( - "UPDATE sandboxes - SET state = 'suspended', - space_api_route_activation_attempt_id = NULL, - space_api_route_substrate_kind = NULL, - space_api_route_identity_provenance = NULL - WHERE id = ?1", - [sandbox_id.to_string()], - ) - .unwrap(); - - assert!(matches!( - db::classify_and_record_admission( - &conn, - sandbox_id, - Some(NodeRole::Worker), - None, expectation, - ) - .unwrap(), - AutoLifecycleDeleteAdmission::Receipt(RouteEvidence { - execution_generation, - activation_attempt_id: observed_attempt, - .. - }) if execution_generation == ExecutionGeneration::new(7).unwrap() - && observed_attempt == activation_attempt_id - )); -} - -#[test] -fn non_dormant_partial_route_does_not_reuse_retained_evidence() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let activation_attempt_id = SandboxActivationAttemptId::new(); - crate::db::update_space_api_route_identity( - &conn, - sandbox_id, - &cluster_route(7, activation_attempt_id), - RouteIdentityProvenance::ClusterFenced, - ) - .unwrap(); - - conn.execute( - "UPDATE sandboxes - SET space_api_route_activation_attempt_id = NULL, - space_api_route_substrate_kind = NULL, - space_api_route_identity_provenance = NULL - WHERE id = ?1", - [sandbox_id.to_string()], - ) - .unwrap(); - - assert!(matches!( - db::classify_and_record_admission( - &conn, - sandbox_id, - Some(NodeRole::Worker), - None, - expectation, - ) - .unwrap(), - AutoLifecycleDeleteAdmission::Blocked { - reason: AutoLifecycleDeleteBlockReason::MissingSubstrate, - changed: true, - } - )); -} - -#[test] -fn contradictory_current_and_retained_routes_block_deletion() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let retained_attempt = SandboxActivationAttemptId::new(); - crate::db::update_space_api_route_identity( - &conn, - sandbox_id, - &cluster_route(8, retained_attempt), - RouteIdentityProvenance::ClusterFenced, - ) - .unwrap(); - - // Model a stale or corrupt current-route writer without updating the - // retained tuple. Receipt admission must not choose either side of the - // contradiction and delete local state that central fencing will retain. - conn.execute( - "UPDATE sandboxes - SET space_api_route_execution_generation = 7, - space_api_route_activation_attempt_id = ?1, - space_api_route_substrate_kind = 'k8s', - space_api_route_identity_provenance = 'cluster_fenced' - WHERE id = ?2", - rusqlite::params![ - SandboxActivationAttemptId::new().to_string(), - sandbox_id.to_string() - ], - ) - .unwrap(); - - assert!(matches!( - db::classify_and_record_admission( - &conn, - sandbox_id, - Some(NodeRole::Worker), - None, expectation, - ) - .unwrap(), - AutoLifecycleDeleteAdmission::Blocked { reason, changed: true } - if reason.as_str() == "conflicting_route_identity" - )); -} - -#[test] -fn later_current_attempt_blocks_receipt_admission() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let retained_attempt = SandboxActivationAttemptId::new(); - crate::db::update_space_api_route_identity( - &conn, - sandbox_id, - &cluster_route(8, retained_attempt), - RouteIdentityProvenance::ClusterFenced, - ) - .unwrap(); - - let current_attempt = SandboxActivationAttemptId::new_after(retained_attempt).unwrap(); - conn.execute( - "UPDATE sandboxes - SET space_api_route_activation_attempt_id = ?1 - WHERE id = ?2", - rusqlite::params![current_attempt.to_string(), sandbox_id.to_string()], - ) - .unwrap(); - - assert!(matches!( - db::classify_and_record_admission( - &conn, - sandbox_id, - Some(NodeRole::Worker), - None, expectation, - ) - .unwrap(), - AutoLifecycleDeleteAdmission::Blocked { reason, changed: true } - if reason == AutoLifecycleDeleteBlockReason::ConflictingRouteIdentity - )); -} - -#[test] -fn delayed_current_attempt_blocks_receipt_admission() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let delayed_attempt = SandboxActivationAttemptId::new(); - let retained_attempt = SandboxActivationAttemptId::new_after(delayed_attempt).unwrap(); - crate::db::update_space_api_route_identity( - &conn, - sandbox_id, - &cluster_route(8, retained_attempt), - RouteIdentityProvenance::ClusterFenced, - ) - .unwrap(); - - conn.execute( - "UPDATE sandboxes - SET space_api_route_activation_attempt_id = ?1 - WHERE id = ?2", - rusqlite::params![delayed_attempt.to_string(), sandbox_id.to_string()], - ) - .unwrap(); - - assert!(matches!( - db::classify_and_record_admission( - &conn, - sandbox_id, - Some(NodeRole::Worker), - None, expectation, - ) - .unwrap(), - AutoLifecycleDeleteAdmission::Blocked { reason, changed: true } - if reason == AutoLifecycleDeleteBlockReason::ConflictingRouteIdentity - )); -} - -#[test] -fn receipt_eligibility_excludes_non_worker_and_non_kubernetes_populations() { - let conn = crate::db::test_conn(); - - let direct_id = SandboxId::new(); - let direct_expectation = insert_sandbox(&conn, direct_id); - let direct_identity = SpaceApiRouteIdentity { - execution_generation: Some(ExecutionGeneration::new(1).unwrap()), - activation_attempt_id: Some(SandboxActivationAttemptId::new()), - substrate_kind: Some(NodeManagedBy::Space), - ..SpaceApiRouteIdentity::default() - }; - crate::db::update_space_api_route_identity( - &conn, - direct_id, - &direct_identity, - RouteIdentityProvenance::Unfenced, - ) - .unwrap(); - conn.execute( - "UPDATE sandboxes - SET space_api_route_activation_attempt_id = 'not-an-attempt' - WHERE id = ?1", - [direct_id.to_string()], - ) - .unwrap(); - assert_eq!( - db::classify_and_record_admission( - &conn, - direct_id, - Some(NodeRole::Worker), - None, - direct_expectation, - ) - .unwrap(), - AutoLifecycleDeleteAdmission::Ordinary( - AutoLifecycleDeleteOrdinaryReason::DirectUnsupported, - ) - ); - - let builder_id = SandboxId::new(); - let builder_expectation = insert_sandbox(&conn, builder_id); - crate::db::update_space_api_route_identity( - &conn, - builder_id, - &cluster_route(2, SandboxActivationAttemptId::new()), - RouteIdentityProvenance::ClusterFenced, - ) - .unwrap(); - conn.execute( - "UPDATE sandboxes - SET space_api_route_activation_attempt_id = 'not-an-attempt' - WHERE id = ?1", - [builder_id.to_string()], - ) - .unwrap(); - assert!(matches!( - db::classify_and_record_admission(&conn, builder_id, None, None, builder_expectation,) - .unwrap(), - AutoLifecycleDeleteAdmission::Blocked { - reason: AutoLifecycleDeleteBlockReason::NodeRoleUnavailable, - .. - } - )); - assert!(matches!( - db::classify_and_record_admission( - &conn, - builder_id, - Some(NodeRole::Builder), - None, - builder_expectation, - ) - .unwrap(), - AutoLifecycleDeleteAdmission::Blocked { - reason: AutoLifecycleDeleteBlockReason::NodeRoleNotWorker, - .. - } - )); - let blocked_at: String = conn - .query_row( - "SELECT auto_lifecycle_delete_blocked_at FROM sandboxes WHERE id = ?1", - [builder_id.to_string()], - |row| row.get(0), - ) - .unwrap(); - db::record_retained_route_evidence( - &conn, - builder_id, - RouteEvidence::from_identity( - &cluster_route(2, SandboxActivationAttemptId::new()), - RouteIdentityProvenance::ClusterFenced, - ) - .unwrap(), - ) - .unwrap(); - assert_eq!( - conn.query_row( - "SELECT auto_lifecycle_delete_block_reason, - auto_lifecycle_delete_blocked_at - FROM sandboxes WHERE id = ?1", - [builder_id.to_string()], - |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), - ) - .unwrap(), - ("node_role_not_worker".to_string(), blocked_at), - "route repair cannot clear a publisher-role block" - ); - - let local_id = SandboxId::new(); - let mut local_expectation = insert_sandbox(&conn, local_id); - conn.execute( - "UPDATE sandboxes SET auto_lifecycle_authority = 'local' WHERE id = ?1", - [local_id.to_string()], - ) - .unwrap(); - local_expectation.authority = PolicyAuthority::Local; - assert_eq!( - db::classify_and_record_admission(&conn, local_id, None, None, local_expectation,).unwrap(), - AutoLifecycleDeleteAdmission::Ordinary(AutoLifecycleDeleteOrdinaryReason::LocalAuthority,) - ); - - let provisional_id = SandboxId::new(); - let provisional_expectation = insert_sandbox(&conn, provisional_id); - crate::db::update_space_api_route_identity( - &conn, - provisional_id, - &cluster_route(3, SandboxActivationAttemptId::new()), - RouteIdentityProvenance::Unfenced, - ) - .unwrap(); - assert!(matches!( - db::classify_and_record_admission( - &conn, - provisional_id, - Some(NodeRole::Worker), - None, - provisional_expectation, - ) - .unwrap(), - AutoLifecycleDeleteAdmission::Blocked { - reason: AutoLifecycleDeleteBlockReason::UnfencedRoute, - .. - } - )); - - let missing_route_id = SandboxId::new(); - let missing_route_expectation = insert_sandbox(&conn, missing_route_id); - assert!(matches!( - db::classify_and_record_admission( - &conn, - missing_route_id, - Some(NodeRole::Worker), - None, - missing_route_expectation, - ) - .unwrap(), - AutoLifecycleDeleteAdmission::Blocked { - reason: AutoLifecycleDeleteBlockReason::MissingRouteIdentity, - .. - } - )); - crate::db::update_space_api_route_identity( - &conn, - missing_route_id, - &cluster_route(4, SandboxActivationAttemptId::new()), - RouteIdentityProvenance::ClusterFenced, - ) - .unwrap(); - assert!(matches!( - db::classify_and_record_admission( - &conn, - missing_route_id, - Some(NodeRole::Worker), - None, - missing_route_expectation, - ) - .unwrap(), - AutoLifecycleDeleteAdmission::Receipt(_) - )); -} - -#[test] -fn matching_receipt_completes_atomically_with_row_deletion() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let activation_attempt_id = SandboxActivationAttemptId::new(); - let claimed_receipt = receipt(sandbox_id, expectation, 9, activation_attempt_id); - db::insert_pending_receipt(&conn, claimed_receipt).unwrap(); - conn.execute( - "UPDATE sandboxes - SET state = 'deleting', - cleanup_completion = 'delete_row', - cleanup_keep_filesystem = 0 - WHERE id = ?1", - [sandbox_id.to_string()], - ) - .unwrap(); - - crate::db::complete_row_intent_cleanup_with_receipt( - &conn, - &sandbox_id.to_string(), - crate::db::CleanupCompletion::DeleteRow, - None, - Some(claimed_receipt), - ) - .unwrap(); - - assert!( - crate::db::get_sandbox(&conn, &sandbox_id.to_string()) - .unwrap() - .is_none() - ); - let state: String = conn - .query_row( - "SELECT state FROM auto_lifecycle_delete_outbox WHERE receipt_id = ?1", - [claimed_receipt.receipt_id.to_string()], - |row| row.get(0), - ) - .unwrap(); - assert_eq!(state, "completed"); - - let recovered_sandbox_id = SandboxId::new(); - let recovered_expectation = insert_sandbox(&conn, recovered_sandbox_id); - let recovered_attempt = SandboxActivationAttemptId::new(); - let recovered_receipt = receipt( - recovered_sandbox_id, - recovered_expectation, - 10, - recovered_attempt, - ); - db::insert_pending_receipt(&conn, recovered_receipt).unwrap(); - conn.execute( - "UPDATE sandboxes - SET state = 'deleting', - cleanup_completion = 'delete_row', - cleanup_keep_filesystem = 0 - WHERE id = ?1", - [recovered_sandbox_id.to_string()], - ) - .unwrap(); - crate::db::complete_row_intent_cleanup_with_receipt( - &conn, - &recovered_sandbox_id.to_string(), - crate::db::CleanupCompletion::DeleteRow, - None, - None, - ) - .unwrap(); - assert!( - crate::db::get_sandbox(&conn, &recovered_sandbox_id.to_string()) - .unwrap() - .is_none() - ); - assert_eq!( - conn.query_row( - "SELECT route_activation_id, state - FROM auto_lifecycle_delete_outbox - WHERE receipt_id = ?1", - [recovered_receipt.receipt_id.to_string()], - |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), - ) - .unwrap(), - (recovered_attempt.to_string(), "completed".to_string()) - ); -} - -#[test] -fn pending_receipt_does_not_silently_survive_lost_delete_intent() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let pending = receipt( - sandbox_id, - expectation, - 11, - SandboxActivationAttemptId::new(), - ); - db::insert_pending_receipt(&conn, pending).unwrap(); - - let error = crate::db::complete_row_intent_cleanup_with_receipt( - &conn, - &sandbox_id.to_string(), - crate::db::CleanupCompletion::DeleteRow, - None, - Some(pending), - ) - .unwrap_err(); - - assert!( - crate::db::get_sandbox(&conn, &sandbox_id.to_string()) - .unwrap() - .is_some() - ); - assert!( - db::pending_receipt_for_sandbox(&conn, sandbox_id) - .unwrap() - .is_some() - ); - assert!(matches!( - error, - crate::db::CompleteRowIntentCleanupError::PendingReceiptLostDeleteIntent { - sandbox_id: observed, - } if observed == sandbox_id - )); -} - #[test] fn route_evidence_uses_the_effective_kubernetes_substrate() { let activation_attempt_id = SandboxActivationAttemptId::new(); @@ -993,1001 +24,3 @@ fn route_evidence_uses_the_effective_kubernetes_substrate() { }) ); } - -#[test] -fn pending_receipt_guard_aborts_only_the_delete_statement() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - db::insert_pending_receipt( - &conn, - receipt( - sandbox_id, - expectation, - 3, - SandboxActivationAttemptId::new(), - ), - ) - .unwrap(); - - let tx = crate::db::immediate_transaction(&conn).unwrap(); - tx.execute( - "UPDATE sandboxes SET failure_reason = 'before guarded delete' WHERE id = ?1", - [sandbox_id.to_string()], - ) - .unwrap(); - let error = tx - .execute( - "DELETE FROM sandboxes WHERE id = ?1", - [sandbox_id.to_string()], - ) - .unwrap_err(); - let error = db::map_pending_receipt_delete_error(error); - let rusqlite::Error::ToSqlConversionFailure(error) = error else { - panic!("pending-receipt guard must map to a typed database error") - }; - assert!( - error - .downcast_ref::() - .is_some() - ); - tx.commit().unwrap(); - - let failure_reason: Option = conn - .query_row( - "SELECT failure_reason FROM sandboxes WHERE id = ?1", - [sandbox_id.to_string()], - |row| row.get(0), - ) - .unwrap(); - assert_eq!(failure_reason.as_deref(), Some("before guarded delete")); -} - -#[test] -fn list_and_ack_touch_only_completed_receipts() { - let conn = crate::db::test_conn(); - let completed_sandbox = SandboxId::new(); - let pending_sandbox = SandboxId::new(); - let completed_expectation = insert_sandbox(&conn, completed_sandbox); - let pending_expectation = insert_sandbox(&conn, pending_sandbox); - let completed = receipt( - completed_sandbox, - completed_expectation, - 1, - SandboxActivationAttemptId::new(), - ); - let pending = receipt( - pending_sandbox, - pending_expectation, - 1, - SandboxActivationAttemptId::new(), - ); - db::insert_pending_receipt(&conn, completed).unwrap(); - db::insert_pending_receipt(&conn, pending).unwrap(); - db::complete_pending_receipt(&conn, completed_sandbox, completed.receipt_id).unwrap(); - - assert_eq!(db::list_completed_receipts(&conn).unwrap(), vec![completed]); - assert_eq!( - db::acknowledge_completed_receipts(&conn, &[completed.receipt_id, pending.receipt_id]) - .unwrap(), - vec![completed.receipt_id] - ); - assert!( - db::pending_receipt_for_sandbox(&conn, pending_sandbox) - .unwrap() - .is_some() - ); - assert!( - conn.query_row( - "SELECT receipt_id FROM auto_lifecycle_delete_outbox WHERE sandbox_id = ?1", - [completed_sandbox.to_string()], - |row| row.get::<_, String>(0), - ) - .optional() - .unwrap() - .is_none() - ); -} - -#[test] -fn completed_receipt_listing_is_bounded_without_starving_fresh_evidence() { - let conn = crate::db::test_conn(); - let receipts = (0..17) - .map(|generation| { - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let receipt = receipt( - sandbox_id, - expectation, - generation + 1, - SandboxActivationAttemptId::new(), - ); - db::insert_pending_receipt(&conn, receipt).unwrap(); - db::complete_pending_receipt(&conn, sandbox_id, receipt.receipt_id).unwrap(); - receipt - }) - .collect::>(); - - let first = db::list_completed_receipts(&conn).unwrap(); - assert_eq!(first.len(), 16); - let omitted = receipts - .iter() - .copied() - .find(|receipt| !first.contains(receipt)) - .expect("the bounded first batch omits one fresh receipt"); - - let second = db::list_completed_receipts(&conn).unwrap(); - assert_eq!(second.len(), 16); - assert_eq!(second.first(), Some(&omitted)); -} - -#[test] -fn pending_receipt_rejects_regressing_and_completed_evidence() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let older_attempt = SandboxActivationAttemptId::new(); - let first_attempt = SandboxActivationAttemptId::new_after(older_attempt).unwrap(); - let first = receipt(sandbox_id, expectation, 4, first_attempt); - assert_eq!(db::insert_pending_receipt(&conn, first).unwrap(), first); - - let retry = AutoLifecycleDeleteReceipt { - receipt_id: AutoLifecycleDeleteReceiptId::generate(), - ..first - }; - assert_eq!(db::insert_pending_receipt(&conn, retry).unwrap(), first); - - let conflicting = AutoLifecycleDeleteReceipt { - route_activation_id: older_attempt, - ..retry - }; - db::insert_pending_receipt(&conn, conflicting).expect_err("conflicting receipt rejected"); - - db::complete_pending_receipt(&conn, sandbox_id, first.receipt_id).unwrap(); - db::insert_pending_receipt(&conn, retry).expect_err("conflicting receipt rejected"); -} - -#[test] -fn newer_attempt_conflicts_with_a_pending_receipt() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let first_attempt = SandboxActivationAttemptId::new(); - let newer_attempt = SandboxActivationAttemptId::new_after(first_attempt).unwrap(); - let first = receipt(sandbox_id, expectation, 4, first_attempt); - db::insert_pending_receipt(&conn, first).unwrap(); - - let replacement = AutoLifecycleDeleteReceipt { - receipt_id: AutoLifecycleDeleteReceiptId::generate(), - route_activation_id: newer_attempt, - ..first - }; - - db::insert_pending_receipt(&conn, replacement).expect_err("conflicting receipt rejected"); - assert_eq!( - db::pending_receipt_for_sandbox(&conn, sandbox_id).unwrap(), - Some(first) - ); -} - -#[test] -fn pending_receipt_blocks_same_generation_route_adoption() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let admitted_attempt = SandboxActivationAttemptId::new(); - let pending = receipt(sandbox_id, expectation, 4, admitted_attempt); - db::insert_pending_receipt(&conn, pending).unwrap(); - - let replacement_attempt = SandboxActivationAttemptId::new_after(admitted_attempt).unwrap(); - assert!( - crate::db::update_space_api_route_identity( - &conn, - sandbox_id, - &cluster_route(4, replacement_attempt), - RouteIdentityProvenance::ClusterFenced, - ) - .is_err() - ); - assert_eq!( - crate::db::get_sandbox(&conn, &sandbox_id.to_string()) - .unwrap() - .unwrap() - .space_api_route_identity - .activation_attempt_id, - None - ); - assert_eq!( - db::pending_receipt_for_sandbox(&conn, sandbox_id).unwrap(), - Some(pending) - ); -} - -#[test] -fn current_policy_refreshes_a_pending_receipt_without_hiding_its_age() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let activation_attempt_id = SandboxActivationAttemptId::new(); - let first = receipt(sandbox_id, expectation, 4, activation_attempt_id); - db::insert_pending_receipt(&conn, first).unwrap(); - conn.execute( - "UPDATE auto_lifecycle_delete_outbox - SET created_at = '1' - WHERE sandbox_id = ?1", - [sandbox_id.to_string()], - ) - .unwrap(); - - let mut revised_policy = delete_policy(); - revised_policy.stages[0].at += 1; - let replacement = AutoLifecycleDeleteReceipt { - receipt_id: AutoLifecycleDeleteReceiptId::generate(), - policy_fingerprint: AutoLifecyclePolicyFingerprint::for_policy(&revised_policy), - ..first - }; - - assert_eq!( - db::insert_pending_receipt(&conn, replacement).unwrap(), - replacement, - ); - assert_eq!( - conn.query_row( - "SELECT receipt_id, auto_lifecycle_policy_fingerprint, created_at - FROM auto_lifecycle_delete_outbox - WHERE sandbox_id = ?1", - [sandbox_id.to_string()], - |row| { - Ok(( - row.get::<_, String>(0)?, - row.get::<_, String>(1)?, - row.get::<_, String>(2)?, - )) - }, - ) - .unwrap(), - ( - replacement.receipt_id.to_string(), - replacement.policy_fingerprint.to_string(), - "1".to_string(), - ), - ); -} - -#[test] -fn only_generation_supersession_discards_a_pending_receipt() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let pending = receipt( - sandbox_id, - expectation, - 4, - SandboxActivationAttemptId::new(), - ); - db::insert_pending_receipt(&conn, pending).unwrap(); - - assert!( - db::discard_pending_receipt_superseded_by_generation( - &conn, - sandbox_id, - pending.execution_generation, - ) - .unwrap() - .is_none() - ); - assert_eq!( - db::pending_receipt_for_sandbox(&conn, sandbox_id).unwrap(), - Some(pending) - ); - - let next_generation = ExecutionGeneration::new(5).unwrap(); - assert_eq!( - db::discard_pending_receipt_superseded_by_generation(&conn, sandbox_id, next_generation,) - .unwrap(), - Some(db::SupersededPendingReceipt { - receipt_id: pending.receipt_id, - execution_generation: pending.execution_generation, - }) - ); - assert!( - db::pending_receipt_for_sandbox(&conn, sandbox_id) - .unwrap() - .is_none() - ); -} - -#[test] -fn pending_receipt_fence_is_distinct_from_a_missing_sandbox() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let generation = ExecutionGeneration::new(4).unwrap(); - db::insert_pending_receipt( - &conn, - receipt( - sandbox_id, - expectation, - generation.get(), - SandboxActivationAttemptId::new(), - ), - ) - .unwrap(); - - let error = crate::db::update_space_api_route_identity( - &conn, - sandbox_id, - &cluster_route(generation.get(), SandboxActivationAttemptId::new()), - RouteIdentityProvenance::ClusterFenced, - ) - .unwrap_err(); - - assert!(db::is_pending_receipt_execution_fence(&error)); - - let missing_id = SandboxId::new(); - let missing_error = crate::db::update_space_api_route_identity( - &conn, - missing_id, - &cluster_route(generation.get(), SandboxActivationAttemptId::new()), - RouteIdentityProvenance::ClusterFenced, - ) - .unwrap_err(); - assert!(matches!( - missing_error, - rusqlite::Error::QueryReturnedNoRows - )); -} - -#[test] -fn deleting_row_rejects_runtime_attachment_without_retiring_its_receipt() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let pending = receipt( - sandbox_id, - expectation, - 4, - SandboxActivationAttemptId::new(), - ); - db::insert_pending_receipt(&conn, pending).unwrap(); - conn.execute( - "UPDATE sandboxes - SET state = 'deleting', - cleanup_completion = 'delete_row', - cleanup_keep_filesystem = 0 - WHERE id = ?1", - [sandbox_id.to_string()], - ) - .unwrap(); - - let error = crate::db::update_runtime_fields( - &conn, - sandbox_id, - crate::db::RuntimeFields { - uid: 10_000, - slot: 7, - guest_cid: 37, - pid: std::process::id(), - spaced_host_pid: 10_001, - netgtw_pid: 10_002, - vsock_uds_path: "/space/run/firecracker/sandboxes/test/root/v.sock", - api_socket_path: "/space/run/firecracker/sandboxes/test/api.sock", - spaced_endpoint_proof: None, - space_api_route_identity: SpaceApiRouteIdentity { - execution_generation: Some(ExecutionGeneration::new(5).unwrap()), - ..SpaceApiRouteIdentity::default() - }, - route_identity_provenance: RouteIdentityProvenance::Unfenced, - }, - ) - .expect_err("runtime attachment must not supersede cleanup ownership"); - - assert!(db::is_lifecycle_execution_fence(&error)); - assert!(!db::is_pending_receipt_execution_fence(&error)); - assert_eq!( - db::pending_receipt_for_sandbox(&conn, sandbox_id).unwrap(), - Some(pending) - ); -} - -#[test] -fn beginning_a_new_execution_retires_an_older_pending_receipt() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - conn.execute( - "UPDATE sandboxes SET state = 'created' WHERE id = ?1", - [sandbox_id.to_string()], - ) - .unwrap(); - let pending = receipt( - sandbox_id, - expectation, - 4, - SandboxActivationAttemptId::new(), - ); - db::insert_pending_receipt(&conn, pending).unwrap(); - let next_generation = ExecutionGeneration::new(5).unwrap(); - let next_attempt = SandboxActivationAttemptId::new(); - - let episode = crate::db::begin_activation_with_route_evidence( - &conn, - &sandbox_id.to_string(), - crate::state::SandboxState::Created, - crate::db::ActivationKind::ColdStart, - Some(next_generation), - Some(next_attempt), - crate::db::ActivationRouteAdmission { - auto_lifecycle: None, - operation_activation_attempt_id: None, - evidence: Some(RouteEvidence { - execution_generation: next_generation, - activation_attempt_id: next_attempt, - substrate: NodeManagedBy::Kubernetes, - provenance: RouteIdentityProvenance::ClusterFenced, - }), - open_claim: None, - }, - ) - .unwrap(); - - assert!(episode.is_some()); - assert!( - db::pending_receipt_for_sandbox(&conn, sandbox_id) - .unwrap() - .is_none(), - "the activation claim must retire stale cleanup evidence atomically" - ); -} - -#[test] -fn beginning_an_execution_cannot_supersede_a_current_pending_receipt() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - conn.execute( - "UPDATE sandboxes SET state = 'created' WHERE id = ?1", - [sandbox_id.to_string()], - ) - .unwrap(); - let generation = ExecutionGeneration::new(5).unwrap(); - let pending = receipt( - sandbox_id, - expectation, - generation.get(), - SandboxActivationAttemptId::new(), - ); - db::insert_pending_receipt(&conn, pending).unwrap(); - - let episode = crate::db::begin_activation_with_route_evidence( - &conn, - &sandbox_id.to_string(), - crate::state::SandboxState::Created, - crate::db::ActivationKind::ColdStart, - Some(generation), - Some(SandboxActivationAttemptId::new()), - crate::db::ActivationRouteAdmission::default(), - ) - .unwrap(); - - assert!(episode.is_none()); - assert_eq!( - crate::db::get_sandbox(&conn, &sandbox_id.to_string()) - .unwrap() - .unwrap() - .state, - crate::state::SandboxState::Created - ); - assert_eq!( - db::pending_receipt_for_sandbox(&conn, sandbox_id).unwrap(), - Some(pending) - ); -} - -#[test] -fn generation_writers_discard_an_older_pending_receipt_without_complete_route_evidence() { - let conn = crate::db::test_conn(); - - let route_sandbox_id = SandboxId::new(); - let route_expectation = insert_sandbox(&conn, route_sandbox_id); - let route_pending = receipt( - route_sandbox_id, - route_expectation, - 4, - SandboxActivationAttemptId::new(), - ); - db::insert_pending_receipt(&conn, route_pending).unwrap(); - crate::db::update_space_api_route_identity( - &conn, - route_sandbox_id, - &SpaceApiRouteIdentity { - execution_generation: Some(ExecutionGeneration::new(5).unwrap()), - ..SpaceApiRouteIdentity::default() - }, - RouteIdentityProvenance::Unfenced, - ) - .unwrap(); - assert!( - db::pending_receipt_for_sandbox(&conn, route_sandbox_id) - .unwrap() - .is_none() - ); - - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let pending = receipt( - sandbox_id, - expectation, - 4, - SandboxActivationAttemptId::new(), - ); - db::insert_pending_receipt(&conn, pending).unwrap(); - crate::db::update_runtime_fields( - &conn, - sandbox_id, - crate::db::RuntimeFields { - uid: 10_000, - slot: 7, - guest_cid: 37, - pid: std::process::id(), - spaced_host_pid: 10_001, - netgtw_pid: 10_002, - vsock_uds_path: "/space/run/firecracker/sandboxes/test/root/v.sock", - api_socket_path: "/space/run/firecracker/sandboxes/test/api.sock", - spaced_endpoint_proof: None, - space_api_route_identity: SpaceApiRouteIdentity { - execution_generation: Some(ExecutionGeneration::new(5).unwrap()), - ..SpaceApiRouteIdentity::default() - }, - route_identity_provenance: RouteIdentityProvenance::Unfenced, - }, - ) - .unwrap(); - assert!( - db::pending_receipt_for_sandbox(&conn, sandbox_id) - .unwrap() - .is_none() - ); -} - -#[test] -fn completion_reports_a_typed_missing_expected_receipt() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let pending = receipt( - sandbox_id, - expectation, - 4, - SandboxActivationAttemptId::new(), - ); - db::insert_pending_receipt(&conn, pending).unwrap(); - conn.execute( - "UPDATE sandboxes - SET state = 'deleting', - cleanup_completion = 'delete_row', - cleanup_keep_filesystem = 0 - WHERE id = ?1", - [sandbox_id.to_string()], - ) - .unwrap(); - db::discard_pending_receipt_superseded_by_generation( - &conn, - sandbox_id, - ExecutionGeneration::new(5).unwrap(), - ) - .unwrap(); - - let error = crate::db::complete_row_intent_cleanup_with_receipt( - &conn, - &sandbox_id.to_string(), - crate::db::CleanupCompletion::DeleteRow, - None, - Some(pending), - ) - .unwrap_err(); - - assert!(matches!( - error, - crate::db::CompleteRowIntentCleanupError::ExpectedReceiptMissingOrChanged { - sandbox_id: observed, - } if observed == sandbox_id - )); -} - -#[test] -fn newer_route_generation_supersedes_receipt_completion() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let pending = receipt( - sandbox_id, - expectation, - 4, - SandboxActivationAttemptId::new(), - ); - db::insert_pending_receipt(&conn, pending).unwrap(); - conn.execute( - "UPDATE sandboxes - SET state = 'deleting', - cleanup_completion = 'delete_row', - cleanup_keep_filesystem = 0 - WHERE id = ?1", - [sandbox_id.to_string()], - ) - .unwrap(); - db::record_retained_route_evidence( - &conn, - sandbox_id, - RouteEvidence { - execution_generation: ExecutionGeneration::new(5).unwrap(), - activation_attempt_id: SandboxActivationAttemptId::new(), - substrate: NodeManagedBy::Kubernetes, - provenance: RouteIdentityProvenance::ClusterFenced, - }, - ) - .unwrap(); - db::discard_pending_receipt_superseded_by_generation( - &conn, - sandbox_id, - ExecutionGeneration::new(5).unwrap(), - ) - .unwrap(); - - assert_eq!( - crate::db::complete_row_intent_cleanup_with_receipt( - &conn, - &sandbox_id.to_string(), - crate::db::CleanupCompletion::DeleteRow, - None, - Some(pending), - ) - .unwrap(), - crate::db::CompleteRowIntentCleanupDisposition::Superseded, - ); - assert!( - crate::db::get_sandbox(&conn, &sandbox_id.to_string()) - .unwrap() - .is_some(), - "the superseded cleanup must not delete the replacement execution" - ); -} - -#[test] -fn newer_current_execution_generation_supersedes_receipt_completion() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let pending = receipt( - sandbox_id, - expectation, - 4, - SandboxActivationAttemptId::new(), - ); - db::insert_pending_receipt(&conn, pending).unwrap(); - assert!( - db::discard_pending_receipt_superseded_by_generation( - &conn, - sandbox_id, - ExecutionGeneration::new(5).unwrap(), - ) - .unwrap() - .is_some() - ); - crate::db::update_space_api_route_identity( - &conn, - sandbox_id, - &SpaceApiRouteIdentity { - execution_generation: Some(ExecutionGeneration::new(5).unwrap()), - ..SpaceApiRouteIdentity::default() - }, - RouteIdentityProvenance::Unfenced, - ) - .unwrap(); - conn.execute( - "UPDATE sandboxes - SET state = 'started', - cleanup_completion = NULL, - cleanup_keep_filesystem = NULL - WHERE id = ?1", - [sandbox_id.to_string()], - ) - .unwrap(); - - assert_eq!( - crate::db::complete_row_intent_cleanup_with_receipt( - &conn, - &sandbox_id.to_string(), - crate::db::CleanupCompletion::DeleteRow, - None, - Some(pending), - ) - .unwrap(), - crate::db::CompleteRowIntentCleanupDisposition::Superseded, - ); - assert!( - crate::db::get_sandbox(&conn, &sandbox_id.to_string()) - .unwrap() - .is_some(), - "the superseded cleanup must not delete the replacement execution" - ); - assert!( - db::pending_receipt_for_sandbox(&conn, sandbox_id) - .unwrap() - .is_none() - ); -} - -#[test] -fn equal_current_execution_generation_does_not_supersede_receipt_completion() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - let generation = ExecutionGeneration::new(4).unwrap(); - crate::db::update_space_api_route_identity( - &conn, - sandbox_id, - &SpaceApiRouteIdentity { - execution_generation: Some(generation), - ..SpaceApiRouteIdentity::default() - }, - RouteIdentityProvenance::Unfenced, - ) - .unwrap(); - let pending = receipt( - sandbox_id, - expectation, - generation.get(), - SandboxActivationAttemptId::new(), - ); - db::insert_pending_receipt(&conn, pending).unwrap(); - conn.execute( - "UPDATE sandboxes - SET state = 'deleting', - cleanup_completion = 'delete_row', - cleanup_keep_filesystem = 0 - WHERE id = ?1", - [sandbox_id.to_string()], - ) - .unwrap(); - let mismatched = AutoLifecycleDeleteReceipt { - receipt_id: AutoLifecycleDeleteReceiptId::generate(), - ..pending - }; - - let error = crate::db::complete_row_intent_cleanup_with_receipt( - &conn, - &sandbox_id.to_string(), - crate::db::CleanupCompletion::DeleteRow, - None, - Some(mismatched), - ) - .unwrap_err(); - - assert!(matches!( - error, - crate::db::CompleteRowIntentCleanupError::ExpectedReceiptMissingOrChanged { - sandbox_id: observed, - } if observed == sandbox_id - )); - assert!( - crate::db::get_sandbox(&conn, &sandbox_id.to_string()) - .unwrap() - .is_some() - ); - assert_eq!( - db::pending_receipt_for_sandbox(&conn, sandbox_id).unwrap(), - Some(pending) - ); -} - -#[test] -fn equal_generation_retained_evidence_cannot_be_downgraded() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - insert_sandbox(&conn, sandbox_id); - let first_attempt = SandboxActivationAttemptId::new(); - let replacement_attempt = SandboxActivationAttemptId::new(); - db::record_retained_route_evidence( - &conn, - sandbox_id, - RouteEvidence { - execution_generation: ExecutionGeneration::new(4).unwrap(), - activation_attempt_id: first_attempt, - substrate: NodeManagedBy::Kubernetes, - provenance: RouteIdentityProvenance::ClusterFenced, - }, - ) - .unwrap(); - - assert!( - !db::record_retained_route_evidence( - &conn, - sandbox_id, - RouteEvidence { - execution_generation: ExecutionGeneration::new(4).unwrap(), - activation_attempt_id: replacement_attempt, - substrate: NodeManagedBy::Kubernetes, - provenance: RouteIdentityProvenance::Unfenced, - }, - ) - .unwrap() - ); - assert_eq!( - conn.query_row( - "SELECT last_observed_route_activation_id, - last_observed_route_identity_provenance - FROM sandboxes WHERE id = ?1", - [sandbox_id.to_string()], - |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), - ) - .unwrap(), - (first_attempt.to_string(), "cluster_fenced".to_string()) - ); - - assert!( - db::record_retained_route_evidence( - &conn, - sandbox_id, - RouteEvidence { - execution_generation: ExecutionGeneration::new(4).unwrap(), - activation_attempt_id: replacement_attempt, - substrate: NodeManagedBy::Kubernetes, - provenance: RouteIdentityProvenance::ClusterFenced, - }, - ) - .unwrap() - ); - assert_eq!( - conn.query_row( - "SELECT last_observed_route_activation_id, - last_observed_route_identity_provenance - FROM sandboxes WHERE id = ?1", - [sandbox_id.to_string()], - |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), - ) - .unwrap(), - ( - replacement_attempt.to_string(), - "cluster_fenced".to_string() - ) - ); - - assert!( - !db::record_retained_route_evidence( - &conn, - sandbox_id, - RouteEvidence { - execution_generation: ExecutionGeneration::new(4).unwrap(), - activation_attempt_id: first_attempt, - substrate: NodeManagedBy::Kubernetes, - provenance: RouteIdentityProvenance::ClusterFenced, - }, - ) - .unwrap(), - "delayed same-generation evidence must not replace the newer attempt" - ); - assert_eq!( - conn.query_row( - "SELECT last_observed_route_activation_id - FROM sandboxes WHERE id = ?1", - [sandbox_id.to_string()], - |row| row.get::<_, String>(0), - ) - .unwrap(), - replacement_attempt.to_string() - ); -} - -#[test] -fn receipt_conflict_uses_bounded_retry_without_claiming_a_durable_block() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - insert_sandbox(&conn, sandbox_id); - - let storage_id = sandbox_id.to_string(); - assert!(db::record_receipt_conflict_retry(&conn, &storage_id).unwrap()); - assert!(!db::record_receipt_conflict_retry(&conn, &storage_id).unwrap()); - - let (block_reason, retry_at, runtime_state_json): (Option, i64, String) = conn - .query_row( - "SELECT auto_lifecycle_delete_block_reason, - next_auto_event_at, - auto_lifecycle_state_json - FROM sandboxes - WHERE id = ?1", - [sandbox_id.to_string()], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), - ) - .unwrap(); - let runtime_state: pplx_space_common_core::types::AutoLifecycleRuntimeState = - serde_json::from_str(&runtime_state_json).unwrap(); - assert!(block_reason.is_none()); - assert!(retry_at > crate::db::now_epoch_i64().unwrap()); - assert_eq!( - runtime_state.next_auto_event_at, - Some(u64::try_from(retry_at).unwrap()) - ); - assert!( - runtime_state - .last_auto_error - .as_deref() - .is_some_and(|error| error.contains("conflicting receipt evidence")) - ); -} - -#[test] -fn policy_replacement_retires_obsolete_delete_block() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - - assert!(matches!( - db::classify_and_record_admission( - &conn, - sandbox_id, - Some(NodeRole::Worker), - None, - expectation, - ) - .unwrap(), - AutoLifecycleDeleteAdmission::Blocked { - reason: AutoLifecycleDeleteBlockReason::MissingRouteIdentity, - .. - } - )); - - crate::db::update_auto_lifecycle( - &conn, - &sandbox_id.to_string(), - &AutoLifecyclePolicy::disabled(), - PolicyAuthority::Cluster, - ) - .unwrap(); - - let (reason, blocked_at): (Option, Option) = conn - .query_row( - "SELECT auto_lifecycle_delete_block_reason, - auto_lifecycle_delete_blocked_at - FROM sandboxes - WHERE id = ?1", - [sandbox_id.to_string()], - |row| Ok((row.get(0)?, row.get(1)?)), - ) - .unwrap(); - assert_eq!((reason, blocked_at), (None, None)); -} - -#[test] -fn unknown_substrate_is_durably_blocked() { - let conn = crate::db::test_conn(); - let sandbox_id = SandboxId::new(); - let expectation = insert_sandbox(&conn, sandbox_id); - conn.execute( - "UPDATE sandboxes - SET space_api_route_execution_generation = 7, - space_api_route_activation_attempt_id = ?1, - space_api_route_substrate_kind = 'future_substrate', - space_api_route_identity_provenance = 'cluster_fenced' - WHERE id = ?2", - rusqlite::params![ - SandboxActivationAttemptId::new().to_string(), - sandbox_id.to_string() - ], - ) - .unwrap(); - - let admission = db::classify_and_record_admission( - &conn, - sandbox_id, - Some(NodeRole::Worker), - None, - expectation, - ) - .expect("unknown substrate must fail closed as a durable block"); - let AutoLifecycleDeleteAdmission::Blocked { reason, changed } = admission else { - panic!("unknown substrate must not admit deletion: {admission:?}"); - }; - assert_eq!(reason.as_str(), "unknown_substrate"); - assert!(changed); -} diff --git a/crates/libs/sbx/mgr/server/src/db.rs b/crates/libs/sbx/mgr/server/src/db.rs index d05965bf9..9ca82fb87 100644 --- a/crates/libs/sbx/mgr/server/src/db.rs +++ b/crates/libs/sbx/mgr/server/src/db.rs @@ -32,12 +32,18 @@ use rusqlite::types::Type; use pplx_space_common_core::sql; use rusqlite::{Connection, OptionalExtension, Result, params}; +pub(crate) mod auto_lifecycle_delete_receipts; pub(crate) mod cleanup_jobs; pub(crate) mod cleanup_target_scan; +pub(crate) mod eviction_scan; +pub(crate) mod guest_health_scan; pub(crate) mod inventory; pub(crate) mod inventory_read; +pub(crate) mod local_artifact_leases; pub(crate) mod quarantine_history; +pub(crate) mod route_activation_claims; pub(crate) mod snapshot_gc; +pub(crate) mod snapshot_upload_jobs; pub(crate) mod timestamps; pub(crate) mod transfer; @@ -51,6 +57,7 @@ const DB_PATH: &str = pplx_space_common_core::host::linux::paths::SANDBOX_MANAGE mod quarantine_scan; mod sandbox_row; mod teardown_jobs; +use auto_lifecycle_delete_receipts::discard_pending_receipt_for_execution_advance; pub(crate) use teardown_jobs::{ TeardownSuspendFailure, defer_teardown_suspend_retry, fail_teardown_suspend_backup_operation, fail_teardown_suspend_operation, reconcile_terminal_teardown_suspend_jobs, @@ -307,7 +314,7 @@ fn emit_sandbox_quarantined_event( fn emit_auto_lifecycle_delete_receipt_superseded( sandbox_id: SandboxId, - superseded_receipt: crate::auto_lifecycle_delete_receipts::db::SupersededPendingReceipt, + superseded_receipt: crate::db::auto_lifecycle_delete_receipts::SupersededPendingReceipt, current_generation: ExecutionGeneration, ) { emit_sandbox_event( @@ -328,7 +335,7 @@ fn emit_auto_lifecycle_delete_receipt_superseded( pub(crate) fn emit_auto_lifecycle_delete_receipt_superseded_if_present( sandbox_id: SandboxId, - supersession: Option, + supersession: Option, ) { if let Some(supersession) = supersession { emit_auto_lifecycle_delete_receipt_superseded( @@ -345,7 +352,7 @@ pub(crate) fn emit_auto_lifecycle_delete_receipt_superseded_if_present( /// add new child tables here, not at the individual delete call sites. fn delete_sandbox_child_rows(tx: &Connection, id: &str) -> Result<()> { teardown_jobs::fail_active(tx, id, TeardownSuspendFailure::Deleted)?; - crate::route_activation_claims::close_claim(tx, id)?; + crate::db::route_activation_claims::close_claim(tx, id)?; tx.execute( "DELETE FROM sandbox_cleanup_episodes WHERE sandbox_id = ?1", params![id], diff --git a/crates/libs/sbx/mgr/server/src/db/auto_lifecycle.rs b/crates/libs/sbx/mgr/server/src/db/auto_lifecycle.rs index ecf8f066d..a5e189c58 100644 --- a/crates/libs/sbx/mgr/server/src/db/auto_lifecycle.rs +++ b/crates/libs/sbx/mgr/server/src/db/auto_lifecycle.rs @@ -270,7 +270,7 @@ pub fn update_auto_lifecycle_with_state( /// /// One reader moves the eager way. The disk-pressure eviction ages a /// `suspended` row with no activity clocks by `updated_at` -/// (`sandbox_eviction::IDLE_WITNESS_SQL`), so such a row whose stage keeps +/// (`db::eviction_scan::IDLE_WITNESS_SQL`), so such a row whose stage keeps /// failing identically now ages from its last real change rather than its /// last repeated record, and can become a candidate sooner than before. That /// is the intended meaning of idle, and the eviction's backup proof and claim diff --git a/crates/libs/sbx/mgr/server/src/auto_lifecycle_delete_receipts/db.rs b/crates/libs/sbx/mgr/server/src/db/auto_lifecycle_delete_receipts.rs similarity index 99% rename from crates/libs/sbx/mgr/server/src/auto_lifecycle_delete_receipts/db.rs rename to crates/libs/sbx/mgr/server/src/db/auto_lifecycle_delete_receipts.rs index 6a0da4f10..83a577845 100644 --- a/crates/libs/sbx/mgr/server/src/auto_lifecycle_delete_receipts/db.rs +++ b/crates/libs/sbx/mgr/server/src/db/auto_lifecycle_delete_receipts.rs @@ -4,12 +4,10 @@ use pplx_space_common_core::types::{ AutoLifecycleDeleteReceiptId, ExecutionGeneration, NodeManagedBy, NodeRole, PolicyAuthority, SandboxActivationAttemptId, SandboxId, }; -use pplx_space_common_svc::sql::sqlite::SqliteU64; -use pplx_space_common_svc::types::AutoLifecyclePolicyFingerprint; -use rusqlite::types::Type; -use rusqlite::{Connection, OptionalExtension, params}; +use pplx_space_common_svc::{sql::sqlite::SqliteU64, types::AutoLifecyclePolicyFingerprint}; +use rusqlite::{Connection, OptionalExtension, params, types::Type}; -use super::{ +use crate::auto_lifecycle_delete_receipts::{ AutoLifecycleDeleteAdmission, AutoLifecycleDeleteBlockReason, AutoLifecycleDeleteBlockedStats, AutoLifecycleDeleteOrdinaryReason, AutoLifecycleDeleteOutboxState, AutoLifecycleDeleteOutboxStats, AutoLifecycleDeleteReceipt, RouteEvidence, @@ -183,7 +181,7 @@ pub(crate) fn classify_and_record_admission( sandbox_id: SandboxId, node_role: Option, node_managed_by: Option, - expectation: super::AutoLifecycleDeleteExpectation, + expectation: crate::auto_lifecycle_delete_receipts::AutoLifecycleDeleteExpectation, ) -> Result { let values = conn .query_row( @@ -1229,3 +1227,6 @@ where rusqlite::Error::FromSqlConversionFailure(column, Type::Text, Box::new(error)) }) } + +#[cfg(test)] +mod tests; diff --git a/crates/libs/sbx/mgr/server/src/db/auto_lifecycle_delete_receipts/tests.rs b/crates/libs/sbx/mgr/server/src/db/auto_lifecycle_delete_receipts/tests.rs new file mode 100644 index 000000000..650b8ea56 --- /dev/null +++ b/crates/libs/sbx/mgr/server/src/db/auto_lifecycle_delete_receipts/tests.rs @@ -0,0 +1,1962 @@ +use std::collections::BTreeMap; + +use pplx_space_common_core::types::{ + AutoLifecycleConditionSet, AutoLifecycleConditionSetId, AutoLifecycleDeleteReceiptId, + AutoLifecycleIdleAction, AutoLifecyclePolicy, AutoLifecycleStage, AutoLifecycleWakePolicy, + ExecutionGeneration, NodeManagedBy, NodeRole, PolicyAuthority, RuntimeKind, + SandboxActivationAttemptId, SandboxId, SandboxOwner, SandboxRuntimeLabel, + SpaceApiRouteIdentity, +}; +use pplx_space_common_svc::types::AutoLifecyclePolicyFingerprint; +use rusqlite::{Connection, OptionalExtension}; + +use super::*; +use crate::auto_lifecycle_delete_receipts::*; + +fn delete_policy() -> AutoLifecyclePolicy { + AutoLifecyclePolicy { + wake: AutoLifecycleWakePolicy::default(), + condition_sets: [( + AutoLifecycleConditionSetId::AccessIdle, + AutoLifecycleConditionSet::access_idle(), + )] + .into(), + stages: vec![AutoLifecycleStage { + condition_set: AutoLifecycleConditionSetId::AccessIdle, + at: 900, + action: AutoLifecycleIdleAction::Delete, + }], + } +} + +fn insert_sandbox(conn: &Connection, sandbox_id: SandboxId) -> AutoLifecycleDeleteExpectation { + let owner = SandboxOwner::parse("user").unwrap(); + let runtime_label = SandboxRuntimeLabel::from_sandbox_id(sandbox_id); + let policy = delete_policy(); + crate::db::insert_sandbox( + conn, + crate::db::InsertSandboxParams { + id: &sandbox_id.to_string(), + uuid8: &runtime_label, + state: crate::state::SandboxState::Started, + template: "seed", + template_hash: Some("hash"), + user: &owner, + runtime: RuntimeKind::Firecracker.as_str(), + snapshots_enabled: false, + rootfs_size_mb: None, + disk_io_limits: Default::default(), + pool_template: None, + volume_encryption: false, + process_env: &BTreeMap::new(), + auto_lifecycle: &policy, + network_policy_override: None, + network_policy_generation: 0, + network_policy_authority: PolicyAuthority::Cluster, + auto_lifecycle_authority: PolicyAuthority::Cluster, + }, + ) + .unwrap(); + AutoLifecycleDeleteExpectation { + policy_fingerprint: AutoLifecyclePolicyFingerprint::for_policy(&policy), + authority: PolicyAuthority::Cluster, + } +} + +fn cluster_route( + generation: i64, + activation_attempt_id: SandboxActivationAttemptId, +) -> SpaceApiRouteIdentity { + SpaceApiRouteIdentity { + execution_generation: Some(ExecutionGeneration::new(generation).unwrap()), + activation_attempt_id: Some(activation_attempt_id), + substrate_kind: Some(NodeManagedBy::Kubernetes), + ..SpaceApiRouteIdentity::default() + } +} + +fn receipt( + sandbox_id: SandboxId, + expectation: AutoLifecycleDeleteExpectation, + generation: i64, + activation_attempt_id: SandboxActivationAttemptId, +) -> AutoLifecycleDeleteReceipt { + AutoLifecycleDeleteReceipt { + receipt_id: AutoLifecycleDeleteReceiptId::generate(), + sandbox_id, + execution_generation: ExecutionGeneration::new(generation).unwrap(), + route_activation_id: activation_attempt_id, + policy_fingerprint: expectation.policy_fingerprint, + } +} + +#[test] +fn delete_admission_requires_consistent_authority_and_execution_evidence() { + use AutoLifecycleDeleteBlockReason as Block; + use AutoLifecycleDeleteOrdinaryReason as Ordinary; + + #[derive(Clone, Copy, Debug)] + enum Expected { + Ordinary(Ordinary), + Receipt, + Blocked(Block), + } + let cases = [ + ( + "local", + Some(NodeManagedBy::Space), + PolicyAuthority::Local, + "UPDATE sandboxes SET space_api_route_substrate_kind = 'k8s' WHERE id = ?3", + Expected::Ordinary(Ordinary::LocalAuthority), + ), + ( + "route-less direct", + Some(NodeManagedBy::Space), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET state = 'stopped' WHERE id = ?3", + Expected::Ordinary(Ordinary::DirectUnsupported), + ), + ( + "explicit direct", + None, + PolicyAuthority::Cluster, + "UPDATE sandboxes SET space_api_route_substrate_kind = 'space' WHERE id = ?3", + Expected::Ordinary(Ordinary::DirectUnsupported), + ), + ( + "current", + Some(NodeManagedBy::Kubernetes), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET space_api_route_execution_generation = 7, space_api_route_activation_attempt_id = ?1, space_api_route_substrate_kind = 'k8s', space_api_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", + Expected::Receipt, + ), + ( + "retained", + Some(NodeManagedBy::Kubernetes), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET last_observed_execution_generation = 7, last_observed_route_activation_id = ?1, last_observed_route_substrate_kind = 'k8s', last_observed_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", + Expected::Receipt, + ), + ( + "suspended subset", + Some(NodeManagedBy::Kubernetes), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET state = 'suspended', space_api_route_execution_generation = 7, last_observed_execution_generation = 7, last_observed_route_activation_id = ?1, last_observed_route_substrate_kind = 'k8s', last_observed_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", + Expected::Receipt, + ), + ( + "missing Kubernetes", + Some(NodeManagedBy::Kubernetes), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET state = 'stopped' WHERE id = ?3", + Expected::Blocked(Block::MissingRouteIdentity), + ), + ( + "missing unknown host", + None, + PolicyAuthority::Cluster, + "UPDATE sandboxes SET state = 'stopped' WHERE id = ?3", + Expected::Blocked(Block::MissingRouteIdentity), + ), + ( + "stale retained", + Some(NodeManagedBy::Kubernetes), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET space_api_route_execution_generation = 8, space_api_route_activation_attempt_id = ?1, space_api_route_substrate_kind = 'k8s', space_api_route_identity_provenance = 'cluster_fenced', last_observed_execution_generation = 7, last_observed_route_activation_id = ?1, last_observed_route_substrate_kind = 'k8s', last_observed_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", + Expected::Blocked(Block::ConflictingRouteIdentity), + ), + ( + "conflicting attempts", + Some(NodeManagedBy::Kubernetes), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET space_api_route_execution_generation = 7, space_api_route_activation_attempt_id = ?1, space_api_route_substrate_kind = 'k8s', space_api_route_identity_provenance = 'cluster_fenced', last_observed_execution_generation = 7, last_observed_route_activation_id = ?2, last_observed_route_substrate_kind = 'k8s', last_observed_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", + Expected::Blocked(Block::ConflictingRouteIdentity), + ), + ( + "host contradicts route", + Some(NodeManagedBy::Space), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET space_api_route_substrate_kind = 'k8s' WHERE id = ?3", + Expected::Blocked(Block::ConflictingRouteIdentity), + ), + ( + "malformed current attempt", + Some(NodeManagedBy::Kubernetes), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET space_api_route_execution_generation = 7, space_api_route_activation_attempt_id = 'malformed', space_api_route_substrate_kind = 'k8s', space_api_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", + Expected::Blocked(Block::ConflictingRouteIdentity), + ), + ( + "malformed retained attempt", + Some(NodeManagedBy::Kubernetes), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET space_api_route_execution_generation = 7, space_api_route_activation_attempt_id = ?1, space_api_route_substrate_kind = 'k8s', space_api_route_identity_provenance = 'cluster_fenced', last_observed_execution_generation = 7, last_observed_route_activation_id = 'malformed', last_observed_route_substrate_kind = 'k8s', last_observed_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", + Expected::Blocked(Block::ConflictingRouteIdentity), + ), + ( + "invalid current generation", + Some(NodeManagedBy::Kubernetes), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET space_api_route_execution_generation = -1, space_api_route_activation_attempt_id = ?1, space_api_route_substrate_kind = 'k8s', space_api_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", + Expected::Blocked(Block::ConflictingRouteIdentity), + ), + ( + "invalid retained generation", + Some(NodeManagedBy::Kubernetes), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET last_observed_execution_generation = 0, last_observed_route_activation_id = ?1, last_observed_route_substrate_kind = 'k8s', last_observed_route_identity_provenance = 'cluster_fenced' WHERE id = ?3", + Expected::Blocked(Block::ConflictingRouteIdentity), + ), + ( + "mixed substrates", + None, + PolicyAuthority::Cluster, + "UPDATE sandboxes SET space_api_route_substrate_kind = 'space', last_observed_route_substrate_kind = 'k8s' WHERE id = ?3", + Expected::Blocked(Block::ConflictingRouteIdentity), + ), + ( + "unknown substrate", + Some(NodeManagedBy::Space), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET last_observed_route_substrate_kind = 'future' WHERE id = ?3", + Expected::Blocked(Block::UnknownSubstrate), + ), + ( + "partial retained", + Some(NodeManagedBy::Kubernetes), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET space_api_route_execution_generation = 7, space_api_route_activation_attempt_id = ?1, space_api_route_substrate_kind = 'k8s', space_api_route_identity_provenance = 'cluster_fenced', last_observed_execution_generation = 7 WHERE id = ?3", + Expected::Blocked(Block::ConflictingRouteIdentity), + ), + ( + "pod contradicts direct", + Some(NodeManagedBy::Space), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET space_api_route_pod_uid = 'pod' WHERE id = ?3", + Expected::Blocked(Block::ConflictingRouteIdentity), + ), + ( + "retained placement contradicts direct", + Some(NodeManagedBy::Space), + PolicyAuthority::Cluster, + "UPDATE sandboxes SET last_observed_route_generation_substrate_kind = 'k8s' WHERE id = ?3", + Expected::Blocked(Block::ConflictingRouteIdentity), + ), + ]; + for (name, host, authority, setup, expected) in cases { + let conn = crate::db::test_conn(); + let id = SandboxId::new(); + let mut expectation = insert_sandbox(&conn, id); + expectation.authority = authority; + let attempt = SandboxActivationAttemptId::new(); + // Seed historical malformed evidence without weakening admission writes. + conn.pragma_update(None, "ignore_check_constraints", true) + .unwrap(); + conn.execute( + setup, + rusqlite::params![ + attempt.to_string(), + SandboxActivationAttemptId::new().to_string(), + id.to_string() + ], + ) + .unwrap(); + conn.pragma_update(None, "ignore_check_constraints", false) + .unwrap(); + conn.execute( + "UPDATE sandboxes SET auto_lifecycle_authority = ?1 WHERE id = ?2", + rusqlite::params![authority.to_string(), id.to_string()], + ) + .unwrap(); + let admitted = + classify_and_record_admission(&conn, id, Some(NodeRole::Worker), host, expectation) + .unwrap(); + match (expected, admitted) { + (Expected::Ordinary(expected), AutoLifecycleDeleteAdmission::Ordinary(actual)) => { + assert_eq!(actual, expected, "{name}") + } + (Expected::Receipt, AutoLifecycleDeleteAdmission::Receipt(evidence)) => { + assert_eq!(evidence.execution_generation.get(), 7, "{name}"); + assert_eq!(evidence.activation_attempt_id, attempt, "{name}"); + } + (Expected::Blocked(expected), AutoLifecycleDeleteAdmission::Blocked { reason, .. }) => { + assert_eq!(reason, expected, "{name}"); + let retry: i64 = conn + .query_row( + "SELECT next_auto_event_at FROM sandboxes WHERE id = ?1", + [id.to_string()], + |row| row.get(0), + ) + .unwrap(); + assert!(retry > crate::db::now_epoch_i64().unwrap(), "{name}"); + assert!(crate::db::sandbox_exists(&conn, &id.to_string()).unwrap()); + } + pair => panic!("unexpected {name}: {pair:?}"), + } + } + for completed in [false, true] { + let conn = crate::db::test_conn(); + let id = SandboxId::new(); + let expectation = insert_sandbox(&conn, id); + let pending = receipt(id, expectation, 7, SandboxActivationAttemptId::new()); + insert_pending_receipt(&conn, pending).unwrap(); + if completed { + complete_pending_receipt(&conn, id, pending.receipt_id).unwrap(); + } + let admitted = classify_and_record_admission( + &conn, + id, + Some(NodeRole::Worker), + Some(NodeManagedBy::Space), + expectation, + ) + .unwrap(); + if completed { + assert_eq!( + admitted, + AutoLifecycleDeleteAdmission::Ordinary(Ordinary::DirectUnsupported) + ); + } else { + assert!(matches!( + admitted, + AutoLifecycleDeleteAdmission::Blocked { + reason: Block::ConflictingRouteIdentity, + .. + } + )); + } + assert_eq!( + conn.query_row( + "SELECT receipt_id, state FROM auto_lifecycle_delete_outbox WHERE sandbox_id = ?1", + [id.to_string()], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ) + .unwrap(), + ( + pending.receipt_id.to_string(), + if completed { "completed" } else { "pending" }.into() + ) + ); + } +} + +#[test] +fn strict_admission_requires_complete_current_cluster_fencing() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let activation_attempt_id = SandboxActivationAttemptId::new(); + crate::db::update_space_api_route_identity( + &conn, + sandbox_id, + &cluster_route(7, activation_attempt_id), + RouteIdentityProvenance::ClusterFenced, + ) + .unwrap(); + + assert!(matches!( + classify_and_record_admission( + &conn, + sandbox_id, + Some(NodeRole::Worker), + None, expectation, + ) + .unwrap(), + AutoLifecycleDeleteAdmission::Receipt(RouteEvidence { + execution_generation, + activation_attempt_id: observed_attempt, + .. + }) if execution_generation == ExecutionGeneration::new(7).unwrap() + && observed_attempt == activation_attempt_id + )); + + conn.execute( + "UPDATE sandboxes + SET space_api_route_execution_generation = NULL, + space_api_route_activation_attempt_id = NULL, + space_api_route_substrate_kind = NULL, + space_api_route_identity_provenance = NULL + WHERE id = ?1", + [sandbox_id.to_string()], + ) + .unwrap(); + assert!(matches!( + classify_and_record_admission(&conn, sandbox_id, Some(NodeRole::Worker), None, expectation,) + .unwrap(), + AutoLifecycleDeleteAdmission::Receipt(RouteEvidence { + execution_generation, + activation_attempt_id: observed_attempt, + .. + }) if execution_generation == ExecutionGeneration::new(7).unwrap() + && observed_attempt == activation_attempt_id + )); + + conn.execute( + "UPDATE sandboxes + SET space_api_route_execution_generation = 8, + space_api_route_activation_attempt_id = NULL, + space_api_route_substrate_kind = NULL, + space_api_route_identity_provenance = NULL + WHERE id = ?1", + [sandbox_id.to_string()], + ) + .unwrap(); + assert!(matches!( + classify_and_record_admission( + &conn, + sandbox_id, + Some(NodeRole::Worker), + None, + expectation, + ) + .unwrap(), + AutoLifecycleDeleteAdmission::Blocked { + reason: AutoLifecycleDeleteBlockReason::MissingSubstrate, + changed: true, + } + )); + let (reason, blocked_at, next_auto_event_at, runtime_state_json): ( + Option, + Option, + Option, + String, + ) = conn + .query_row( + "SELECT auto_lifecycle_delete_block_reason, + auto_lifecycle_delete_blocked_at, + next_auto_event_at, + auto_lifecycle_state_json + FROM sandboxes WHERE id = ?1", + [sandbox_id.to_string()], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ) + .unwrap(); + assert_eq!(reason.as_deref(), Some("missing_substrate")); + assert!(blocked_at.is_some()); + assert!(next_auto_event_at.unwrap() > crate::db::now_epoch_i64().unwrap()); + let runtime_state: pplx_space_common_core::types::AutoLifecycleRuntimeState = + serde_json::from_str(&runtime_state_json).unwrap(); + assert!( + runtime_state + .last_auto_error + .as_deref() + .is_some_and(|error| error.contains("missing_substrate")) + ); + + crate::db::update_space_api_route_identity( + &conn, + sandbox_id, + &cluster_route(8, SandboxActivationAttemptId::new()), + RouteIdentityProvenance::ClusterFenced, + ) + .unwrap(); + let (reason, blocked_at, runtime_state_json): (Option, Option, String) = conn + .query_row( + "SELECT auto_lifecycle_delete_block_reason, + auto_lifecycle_delete_blocked_at, + auto_lifecycle_state_json + FROM sandboxes WHERE id = ?1", + [sandbox_id.to_string()], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + assert_eq!((reason, blocked_at), (None, None)); + let runtime_state: pplx_space_common_core::types::AutoLifecycleRuntimeState = + serde_json::from_str(&runtime_state_json).unwrap(); + assert_eq!(runtime_state.last_auto_error, None); +} + +#[test] +fn dormant_suspend_identity_uses_matching_retained_route_evidence() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let activation_attempt_id = SandboxActivationAttemptId::new(); + crate::db::update_space_api_route_identity( + &conn, + sandbox_id, + &cluster_route(7, activation_attempt_id), + RouteIdentityProvenance::ClusterFenced, + ) + .unwrap(); + + // Suspend completion retains generation for central publication but clears + // the remaining current route fields. The matching complete retained tuple + // is still the exact route that produced this dormant sandbox. + conn.execute( + "UPDATE sandboxes + SET state = 'suspended', + space_api_route_activation_attempt_id = NULL, + space_api_route_substrate_kind = NULL, + space_api_route_identity_provenance = NULL + WHERE id = ?1", + [sandbox_id.to_string()], + ) + .unwrap(); + + assert!(matches!( + classify_and_record_admission( + &conn, + sandbox_id, + Some(NodeRole::Worker), + None, expectation, + ) + .unwrap(), + AutoLifecycleDeleteAdmission::Receipt(RouteEvidence { + execution_generation, + activation_attempt_id: observed_attempt, + .. + }) if execution_generation == ExecutionGeneration::new(7).unwrap() + && observed_attempt == activation_attempt_id + )); +} + +#[test] +fn non_dormant_partial_route_does_not_reuse_retained_evidence() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let activation_attempt_id = SandboxActivationAttemptId::new(); + crate::db::update_space_api_route_identity( + &conn, + sandbox_id, + &cluster_route(7, activation_attempt_id), + RouteIdentityProvenance::ClusterFenced, + ) + .unwrap(); + + conn.execute( + "UPDATE sandboxes + SET space_api_route_activation_attempt_id = NULL, + space_api_route_substrate_kind = NULL, + space_api_route_identity_provenance = NULL + WHERE id = ?1", + [sandbox_id.to_string()], + ) + .unwrap(); + + assert!(matches!( + classify_and_record_admission( + &conn, + sandbox_id, + Some(NodeRole::Worker), + None, + expectation, + ) + .unwrap(), + AutoLifecycleDeleteAdmission::Blocked { + reason: AutoLifecycleDeleteBlockReason::MissingSubstrate, + changed: true, + } + )); +} + +#[test] +fn contradictory_current_and_retained_routes_block_deletion() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let retained_attempt = SandboxActivationAttemptId::new(); + crate::db::update_space_api_route_identity( + &conn, + sandbox_id, + &cluster_route(8, retained_attempt), + RouteIdentityProvenance::ClusterFenced, + ) + .unwrap(); + + // Model a stale or corrupt current-route writer without updating the + // retained tuple. Receipt admission must not choose either side of the + // contradiction and delete local state that central fencing will retain. + conn.execute( + "UPDATE sandboxes + SET space_api_route_execution_generation = 7, + space_api_route_activation_attempt_id = ?1, + space_api_route_substrate_kind = 'k8s', + space_api_route_identity_provenance = 'cluster_fenced' + WHERE id = ?2", + rusqlite::params![ + SandboxActivationAttemptId::new().to_string(), + sandbox_id.to_string() + ], + ) + .unwrap(); + + assert!(matches!( + classify_and_record_admission( + &conn, + sandbox_id, + Some(NodeRole::Worker), + None, expectation, + ) + .unwrap(), + AutoLifecycleDeleteAdmission::Blocked { reason, changed: true } + if reason.as_str() == "conflicting_route_identity" + )); +} + +#[test] +fn later_current_attempt_blocks_receipt_admission() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let retained_attempt = SandboxActivationAttemptId::new(); + crate::db::update_space_api_route_identity( + &conn, + sandbox_id, + &cluster_route(8, retained_attempt), + RouteIdentityProvenance::ClusterFenced, + ) + .unwrap(); + + let current_attempt = SandboxActivationAttemptId::new_after(retained_attempt).unwrap(); + conn.execute( + "UPDATE sandboxes + SET space_api_route_activation_attempt_id = ?1 + WHERE id = ?2", + rusqlite::params![current_attempt.to_string(), sandbox_id.to_string()], + ) + .unwrap(); + + assert!(matches!( + classify_and_record_admission( + &conn, + sandbox_id, + Some(NodeRole::Worker), + None, expectation, + ) + .unwrap(), + AutoLifecycleDeleteAdmission::Blocked { reason, changed: true } + if reason == AutoLifecycleDeleteBlockReason::ConflictingRouteIdentity + )); +} + +#[test] +fn delayed_current_attempt_blocks_receipt_admission() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let delayed_attempt = SandboxActivationAttemptId::new(); + let retained_attempt = SandboxActivationAttemptId::new_after(delayed_attempt).unwrap(); + crate::db::update_space_api_route_identity( + &conn, + sandbox_id, + &cluster_route(8, retained_attempt), + RouteIdentityProvenance::ClusterFenced, + ) + .unwrap(); + + conn.execute( + "UPDATE sandboxes + SET space_api_route_activation_attempt_id = ?1 + WHERE id = ?2", + rusqlite::params![delayed_attempt.to_string(), sandbox_id.to_string()], + ) + .unwrap(); + + assert!(matches!( + classify_and_record_admission( + &conn, + sandbox_id, + Some(NodeRole::Worker), + None, expectation, + ) + .unwrap(), + AutoLifecycleDeleteAdmission::Blocked { reason, changed: true } + if reason == AutoLifecycleDeleteBlockReason::ConflictingRouteIdentity + )); +} + +#[test] +fn receipt_eligibility_excludes_non_worker_and_non_kubernetes_populations() { + let conn = crate::db::test_conn(); + + let direct_id = SandboxId::new(); + let direct_expectation = insert_sandbox(&conn, direct_id); + let direct_identity = SpaceApiRouteIdentity { + execution_generation: Some(ExecutionGeneration::new(1).unwrap()), + activation_attempt_id: Some(SandboxActivationAttemptId::new()), + substrate_kind: Some(NodeManagedBy::Space), + ..SpaceApiRouteIdentity::default() + }; + crate::db::update_space_api_route_identity( + &conn, + direct_id, + &direct_identity, + RouteIdentityProvenance::Unfenced, + ) + .unwrap(); + conn.execute( + "UPDATE sandboxes + SET space_api_route_activation_attempt_id = 'not-an-attempt' + WHERE id = ?1", + [direct_id.to_string()], + ) + .unwrap(); + assert_eq!( + classify_and_record_admission( + &conn, + direct_id, + Some(NodeRole::Worker), + None, + direct_expectation, + ) + .unwrap(), + AutoLifecycleDeleteAdmission::Ordinary( + AutoLifecycleDeleteOrdinaryReason::DirectUnsupported, + ) + ); + + let builder_id = SandboxId::new(); + let builder_expectation = insert_sandbox(&conn, builder_id); + crate::db::update_space_api_route_identity( + &conn, + builder_id, + &cluster_route(2, SandboxActivationAttemptId::new()), + RouteIdentityProvenance::ClusterFenced, + ) + .unwrap(); + conn.execute( + "UPDATE sandboxes + SET space_api_route_activation_attempt_id = 'not-an-attempt' + WHERE id = ?1", + [builder_id.to_string()], + ) + .unwrap(); + assert!(matches!( + classify_and_record_admission(&conn, builder_id, None, None, builder_expectation,).unwrap(), + AutoLifecycleDeleteAdmission::Blocked { + reason: AutoLifecycleDeleteBlockReason::NodeRoleUnavailable, + .. + } + )); + assert!(matches!( + classify_and_record_admission( + &conn, + builder_id, + Some(NodeRole::Builder), + None, + builder_expectation, + ) + .unwrap(), + AutoLifecycleDeleteAdmission::Blocked { + reason: AutoLifecycleDeleteBlockReason::NodeRoleNotWorker, + .. + } + )); + let blocked_at: String = conn + .query_row( + "SELECT auto_lifecycle_delete_blocked_at FROM sandboxes WHERE id = ?1", + [builder_id.to_string()], + |row| row.get(0), + ) + .unwrap(); + record_retained_route_evidence( + &conn, + builder_id, + RouteEvidence::from_identity( + &cluster_route(2, SandboxActivationAttemptId::new()), + RouteIdentityProvenance::ClusterFenced, + ) + .unwrap(), + ) + .unwrap(); + assert_eq!( + conn.query_row( + "SELECT auto_lifecycle_delete_block_reason, + auto_lifecycle_delete_blocked_at + FROM sandboxes WHERE id = ?1", + [builder_id.to_string()], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ) + .unwrap(), + ("node_role_not_worker".to_string(), blocked_at), + "route repair cannot clear a publisher-role block" + ); + + let local_id = SandboxId::new(); + let mut local_expectation = insert_sandbox(&conn, local_id); + conn.execute( + "UPDATE sandboxes SET auto_lifecycle_authority = 'local' WHERE id = ?1", + [local_id.to_string()], + ) + .unwrap(); + local_expectation.authority = PolicyAuthority::Local; + assert_eq!( + classify_and_record_admission(&conn, local_id, None, None, local_expectation,).unwrap(), + AutoLifecycleDeleteAdmission::Ordinary(AutoLifecycleDeleteOrdinaryReason::LocalAuthority,) + ); + + let provisional_id = SandboxId::new(); + let provisional_expectation = insert_sandbox(&conn, provisional_id); + crate::db::update_space_api_route_identity( + &conn, + provisional_id, + &cluster_route(3, SandboxActivationAttemptId::new()), + RouteIdentityProvenance::Unfenced, + ) + .unwrap(); + assert!(matches!( + classify_and_record_admission( + &conn, + provisional_id, + Some(NodeRole::Worker), + None, + provisional_expectation, + ) + .unwrap(), + AutoLifecycleDeleteAdmission::Blocked { + reason: AutoLifecycleDeleteBlockReason::UnfencedRoute, + .. + } + )); + + let missing_route_id = SandboxId::new(); + let missing_route_expectation = insert_sandbox(&conn, missing_route_id); + assert!(matches!( + classify_and_record_admission( + &conn, + missing_route_id, + Some(NodeRole::Worker), + None, + missing_route_expectation, + ) + .unwrap(), + AutoLifecycleDeleteAdmission::Blocked { + reason: AutoLifecycleDeleteBlockReason::MissingRouteIdentity, + .. + } + )); + crate::db::update_space_api_route_identity( + &conn, + missing_route_id, + &cluster_route(4, SandboxActivationAttemptId::new()), + RouteIdentityProvenance::ClusterFenced, + ) + .unwrap(); + assert!(matches!( + classify_and_record_admission( + &conn, + missing_route_id, + Some(NodeRole::Worker), + None, + missing_route_expectation, + ) + .unwrap(), + AutoLifecycleDeleteAdmission::Receipt(_) + )); +} + +#[test] +fn matching_receipt_completes_atomically_with_row_deletion() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let activation_attempt_id = SandboxActivationAttemptId::new(); + let claimed_receipt = receipt(sandbox_id, expectation, 9, activation_attempt_id); + insert_pending_receipt(&conn, claimed_receipt).unwrap(); + conn.execute( + "UPDATE sandboxes + SET state = 'deleting', + cleanup_completion = 'delete_row', + cleanup_keep_filesystem = 0 + WHERE id = ?1", + [sandbox_id.to_string()], + ) + .unwrap(); + + crate::db::complete_row_intent_cleanup_with_receipt( + &conn, + &sandbox_id.to_string(), + crate::db::CleanupCompletion::DeleteRow, + None, + Some(claimed_receipt), + ) + .unwrap(); + + assert!( + crate::db::get_sandbox(&conn, &sandbox_id.to_string()) + .unwrap() + .is_none() + ); + let state: String = conn + .query_row( + "SELECT state FROM auto_lifecycle_delete_outbox WHERE receipt_id = ?1", + [claimed_receipt.receipt_id.to_string()], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(state, "completed"); + + let recovered_sandbox_id = SandboxId::new(); + let recovered_expectation = insert_sandbox(&conn, recovered_sandbox_id); + let recovered_attempt = SandboxActivationAttemptId::new(); + let recovered_receipt = receipt( + recovered_sandbox_id, + recovered_expectation, + 10, + recovered_attempt, + ); + insert_pending_receipt(&conn, recovered_receipt).unwrap(); + conn.execute( + "UPDATE sandboxes + SET state = 'deleting', + cleanup_completion = 'delete_row', + cleanup_keep_filesystem = 0 + WHERE id = ?1", + [recovered_sandbox_id.to_string()], + ) + .unwrap(); + crate::db::complete_row_intent_cleanup_with_receipt( + &conn, + &recovered_sandbox_id.to_string(), + crate::db::CleanupCompletion::DeleteRow, + None, + None, + ) + .unwrap(); + assert!( + crate::db::get_sandbox(&conn, &recovered_sandbox_id.to_string()) + .unwrap() + .is_none() + ); + assert_eq!( + conn.query_row( + "SELECT route_activation_id, state + FROM auto_lifecycle_delete_outbox + WHERE receipt_id = ?1", + [recovered_receipt.receipt_id.to_string()], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ) + .unwrap(), + (recovered_attempt.to_string(), "completed".to_string()) + ); +} + +#[test] +fn pending_receipt_does_not_silently_survive_lost_delete_intent() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let pending = receipt( + sandbox_id, + expectation, + 11, + SandboxActivationAttemptId::new(), + ); + insert_pending_receipt(&conn, pending).unwrap(); + + let error = crate::db::complete_row_intent_cleanup_with_receipt( + &conn, + &sandbox_id.to_string(), + crate::db::CleanupCompletion::DeleteRow, + None, + Some(pending), + ) + .unwrap_err(); + + assert!( + crate::db::get_sandbox(&conn, &sandbox_id.to_string()) + .unwrap() + .is_some() + ); + assert!( + pending_receipt_for_sandbox(&conn, sandbox_id) + .unwrap() + .is_some() + ); + assert!(matches!( + error, + crate::db::CompleteRowIntentCleanupError::PendingReceiptLostDeleteIntent { + sandbox_id: observed, + } if observed == sandbox_id + )); +} + +#[test] +fn pending_receipt_guard_aborts_only_the_delete_statement() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + insert_pending_receipt( + &conn, + receipt( + sandbox_id, + expectation, + 3, + SandboxActivationAttemptId::new(), + ), + ) + .unwrap(); + + let tx = crate::db::immediate_transaction(&conn).unwrap(); + tx.execute( + "UPDATE sandboxes SET failure_reason = 'before guarded delete' WHERE id = ?1", + [sandbox_id.to_string()], + ) + .unwrap(); + let error = tx + .execute( + "DELETE FROM sandboxes WHERE id = ?1", + [sandbox_id.to_string()], + ) + .unwrap_err(); + let error = map_pending_receipt_delete_error(error); + let rusqlite::Error::ToSqlConversionFailure(error) = error else { + panic!("pending-receipt guard must map to a typed database error") + }; + assert!(error.downcast_ref::().is_some()); + tx.commit().unwrap(); + + let failure_reason: Option = conn + .query_row( + "SELECT failure_reason FROM sandboxes WHERE id = ?1", + [sandbox_id.to_string()], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(failure_reason.as_deref(), Some("before guarded delete")); +} + +#[test] +fn list_and_ack_touch_only_completed_receipts() { + let conn = crate::db::test_conn(); + let completed_sandbox = SandboxId::new(); + let pending_sandbox = SandboxId::new(); + let completed_expectation = insert_sandbox(&conn, completed_sandbox); + let pending_expectation = insert_sandbox(&conn, pending_sandbox); + let completed = receipt( + completed_sandbox, + completed_expectation, + 1, + SandboxActivationAttemptId::new(), + ); + let pending = receipt( + pending_sandbox, + pending_expectation, + 1, + SandboxActivationAttemptId::new(), + ); + insert_pending_receipt(&conn, completed).unwrap(); + insert_pending_receipt(&conn, pending).unwrap(); + complete_pending_receipt(&conn, completed_sandbox, completed.receipt_id).unwrap(); + + assert_eq!(list_completed_receipts(&conn).unwrap(), vec![completed]); + assert_eq!( + acknowledge_completed_receipts(&conn, &[completed.receipt_id, pending.receipt_id]).unwrap(), + vec![completed.receipt_id] + ); + assert!( + pending_receipt_for_sandbox(&conn, pending_sandbox) + .unwrap() + .is_some() + ); + assert!( + conn.query_row( + "SELECT receipt_id FROM auto_lifecycle_delete_outbox WHERE sandbox_id = ?1", + [completed_sandbox.to_string()], + |row| row.get::<_, String>(0), + ) + .optional() + .unwrap() + .is_none() + ); +} + +#[test] +fn completed_receipt_listing_is_bounded_without_starving_fresh_evidence() { + let conn = crate::db::test_conn(); + let receipts = (0..17) + .map(|generation| { + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let receipt = receipt( + sandbox_id, + expectation, + generation + 1, + SandboxActivationAttemptId::new(), + ); + insert_pending_receipt(&conn, receipt).unwrap(); + complete_pending_receipt(&conn, sandbox_id, receipt.receipt_id).unwrap(); + receipt + }) + .collect::>(); + + let first = list_completed_receipts(&conn).unwrap(); + assert_eq!(first.len(), 16); + let omitted = receipts + .iter() + .copied() + .find(|receipt| !first.contains(receipt)) + .expect("the bounded first batch omits one fresh receipt"); + + let second = list_completed_receipts(&conn).unwrap(); + assert_eq!(second.len(), 16); + assert_eq!(second.first(), Some(&omitted)); +} + +#[test] +fn pending_receipt_rejects_regressing_and_completed_evidence() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let older_attempt = SandboxActivationAttemptId::new(); + let first_attempt = SandboxActivationAttemptId::new_after(older_attempt).unwrap(); + let first = receipt(sandbox_id, expectation, 4, first_attempt); + assert_eq!(insert_pending_receipt(&conn, first).unwrap(), first); + + let retry = AutoLifecycleDeleteReceipt { + receipt_id: AutoLifecycleDeleteReceiptId::generate(), + ..first + }; + assert_eq!(insert_pending_receipt(&conn, retry).unwrap(), first); + + let conflicting = AutoLifecycleDeleteReceipt { + route_activation_id: older_attempt, + ..retry + }; + insert_pending_receipt(&conn, conflicting).expect_err("conflicting receipt rejected"); + + complete_pending_receipt(&conn, sandbox_id, first.receipt_id).unwrap(); + insert_pending_receipt(&conn, retry).expect_err("conflicting receipt rejected"); +} + +#[test] +fn newer_attempt_conflicts_with_a_pending_receipt() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let first_attempt = SandboxActivationAttemptId::new(); + let newer_attempt = SandboxActivationAttemptId::new_after(first_attempt).unwrap(); + let first = receipt(sandbox_id, expectation, 4, first_attempt); + insert_pending_receipt(&conn, first).unwrap(); + + let replacement = AutoLifecycleDeleteReceipt { + receipt_id: AutoLifecycleDeleteReceiptId::generate(), + route_activation_id: newer_attempt, + ..first + }; + + insert_pending_receipt(&conn, replacement).expect_err("conflicting receipt rejected"); + assert_eq!( + pending_receipt_for_sandbox(&conn, sandbox_id).unwrap(), + Some(first) + ); +} + +#[test] +fn pending_receipt_blocks_same_generation_route_adoption() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let admitted_attempt = SandboxActivationAttemptId::new(); + let pending = receipt(sandbox_id, expectation, 4, admitted_attempt); + insert_pending_receipt(&conn, pending).unwrap(); + + let replacement_attempt = SandboxActivationAttemptId::new_after(admitted_attempt).unwrap(); + assert!( + crate::db::update_space_api_route_identity( + &conn, + sandbox_id, + &cluster_route(4, replacement_attempt), + RouteIdentityProvenance::ClusterFenced, + ) + .is_err() + ); + assert_eq!( + crate::db::get_sandbox(&conn, &sandbox_id.to_string()) + .unwrap() + .unwrap() + .space_api_route_identity + .activation_attempt_id, + None + ); + assert_eq!( + pending_receipt_for_sandbox(&conn, sandbox_id).unwrap(), + Some(pending) + ); +} + +#[test] +fn current_policy_refreshes_a_pending_receipt_without_hiding_its_age() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let activation_attempt_id = SandboxActivationAttemptId::new(); + let first = receipt(sandbox_id, expectation, 4, activation_attempt_id); + insert_pending_receipt(&conn, first).unwrap(); + conn.execute( + "UPDATE auto_lifecycle_delete_outbox + SET created_at = '1' + WHERE sandbox_id = ?1", + [sandbox_id.to_string()], + ) + .unwrap(); + + let mut revised_policy = delete_policy(); + revised_policy.stages[0].at += 1; + let replacement = AutoLifecycleDeleteReceipt { + receipt_id: AutoLifecycleDeleteReceiptId::generate(), + policy_fingerprint: AutoLifecyclePolicyFingerprint::for_policy(&revised_policy), + ..first + }; + + assert_eq!( + insert_pending_receipt(&conn, replacement).unwrap(), + replacement, + ); + assert_eq!( + conn.query_row( + "SELECT receipt_id, auto_lifecycle_policy_fingerprint, created_at + FROM auto_lifecycle_delete_outbox + WHERE sandbox_id = ?1", + [sandbox_id.to_string()], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + )) + }, + ) + .unwrap(), + ( + replacement.receipt_id.to_string(), + replacement.policy_fingerprint.to_string(), + "1".to_string(), + ), + ); +} + +#[test] +fn only_generation_supersession_discards_a_pending_receipt() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let pending = receipt( + sandbox_id, + expectation, + 4, + SandboxActivationAttemptId::new(), + ); + insert_pending_receipt(&conn, pending).unwrap(); + + assert!( + discard_pending_receipt_superseded_by_generation( + &conn, + sandbox_id, + pending.execution_generation, + ) + .unwrap() + .is_none() + ); + assert_eq!( + pending_receipt_for_sandbox(&conn, sandbox_id).unwrap(), + Some(pending) + ); + + let next_generation = ExecutionGeneration::new(5).unwrap(); + assert_eq!( + discard_pending_receipt_superseded_by_generation(&conn, sandbox_id, next_generation,) + .unwrap(), + Some(SupersededPendingReceipt { + receipt_id: pending.receipt_id, + execution_generation: pending.execution_generation, + }) + ); + assert!( + pending_receipt_for_sandbox(&conn, sandbox_id) + .unwrap() + .is_none() + ); +} + +#[test] +fn pending_receipt_fence_is_distinct_from_a_missing_sandbox() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let generation = ExecutionGeneration::new(4).unwrap(); + insert_pending_receipt( + &conn, + receipt( + sandbox_id, + expectation, + generation.get(), + SandboxActivationAttemptId::new(), + ), + ) + .unwrap(); + + let error = crate::db::update_space_api_route_identity( + &conn, + sandbox_id, + &cluster_route(generation.get(), SandboxActivationAttemptId::new()), + RouteIdentityProvenance::ClusterFenced, + ) + .unwrap_err(); + + assert!(is_pending_receipt_execution_fence(&error)); + + let missing_id = SandboxId::new(); + let missing_error = crate::db::update_space_api_route_identity( + &conn, + missing_id, + &cluster_route(generation.get(), SandboxActivationAttemptId::new()), + RouteIdentityProvenance::ClusterFenced, + ) + .unwrap_err(); + assert!(matches!( + missing_error, + rusqlite::Error::QueryReturnedNoRows + )); +} + +#[test] +fn deleting_row_rejects_runtime_attachment_without_retiring_its_receipt() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let pending = receipt( + sandbox_id, + expectation, + 4, + SandboxActivationAttemptId::new(), + ); + insert_pending_receipt(&conn, pending).unwrap(); + conn.execute( + "UPDATE sandboxes + SET state = 'deleting', + cleanup_completion = 'delete_row', + cleanup_keep_filesystem = 0 + WHERE id = ?1", + [sandbox_id.to_string()], + ) + .unwrap(); + + let error = crate::db::update_runtime_fields( + &conn, + sandbox_id, + crate::db::RuntimeFields { + uid: 10_000, + slot: 7, + guest_cid: 37, + pid: std::process::id(), + spaced_host_pid: 10_001, + netgtw_pid: 10_002, + vsock_uds_path: "/space/run/firecracker/sandboxes/test/root/v.sock", + api_socket_path: "/space/run/firecracker/sandboxes/test/api.sock", + spaced_endpoint_proof: None, + space_api_route_identity: SpaceApiRouteIdentity { + execution_generation: Some(ExecutionGeneration::new(5).unwrap()), + ..SpaceApiRouteIdentity::default() + }, + route_identity_provenance: RouteIdentityProvenance::Unfenced, + }, + ) + .expect_err("runtime attachment must not supersede cleanup ownership"); + + assert!(is_lifecycle_execution_fence(&error)); + assert!(!is_pending_receipt_execution_fence(&error)); + assert_eq!( + pending_receipt_for_sandbox(&conn, sandbox_id).unwrap(), + Some(pending) + ); +} + +#[test] +fn beginning_a_new_execution_retires_an_older_pending_receipt() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + conn.execute( + "UPDATE sandboxes SET state = 'created' WHERE id = ?1", + [sandbox_id.to_string()], + ) + .unwrap(); + let pending = receipt( + sandbox_id, + expectation, + 4, + SandboxActivationAttemptId::new(), + ); + insert_pending_receipt(&conn, pending).unwrap(); + let next_generation = ExecutionGeneration::new(5).unwrap(); + let next_attempt = SandboxActivationAttemptId::new(); + + let episode = crate::db::begin_activation_with_route_evidence( + &conn, + &sandbox_id.to_string(), + crate::state::SandboxState::Created, + crate::db::ActivationKind::ColdStart, + Some(next_generation), + Some(next_attempt), + crate::db::ActivationRouteAdmission { + auto_lifecycle: None, + operation_activation_attempt_id: None, + evidence: Some(RouteEvidence { + execution_generation: next_generation, + activation_attempt_id: next_attempt, + substrate: NodeManagedBy::Kubernetes, + provenance: RouteIdentityProvenance::ClusterFenced, + }), + open_claim: None, + }, + ) + .unwrap(); + + assert!(episode.is_some()); + assert!( + pending_receipt_for_sandbox(&conn, sandbox_id) + .unwrap() + .is_none(), + "the activation claim must retire stale cleanup evidence atomically" + ); +} + +#[test] +fn beginning_an_execution_cannot_supersede_a_current_pending_receipt() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + conn.execute( + "UPDATE sandboxes SET state = 'created' WHERE id = ?1", + [sandbox_id.to_string()], + ) + .unwrap(); + let generation = ExecutionGeneration::new(5).unwrap(); + let pending = receipt( + sandbox_id, + expectation, + generation.get(), + SandboxActivationAttemptId::new(), + ); + insert_pending_receipt(&conn, pending).unwrap(); + + let episode = crate::db::begin_activation_with_route_evidence( + &conn, + &sandbox_id.to_string(), + crate::state::SandboxState::Created, + crate::db::ActivationKind::ColdStart, + Some(generation), + Some(SandboxActivationAttemptId::new()), + crate::db::ActivationRouteAdmission::default(), + ) + .unwrap(); + + assert!(episode.is_none()); + assert_eq!( + crate::db::get_sandbox(&conn, &sandbox_id.to_string()) + .unwrap() + .unwrap() + .state, + crate::state::SandboxState::Created + ); + assert_eq!( + pending_receipt_for_sandbox(&conn, sandbox_id).unwrap(), + Some(pending) + ); +} + +#[test] +fn generation_writers_discard_an_older_pending_receipt_without_complete_route_evidence() { + let conn = crate::db::test_conn(); + + let route_sandbox_id = SandboxId::new(); + let route_expectation = insert_sandbox(&conn, route_sandbox_id); + let route_pending = receipt( + route_sandbox_id, + route_expectation, + 4, + SandboxActivationAttemptId::new(), + ); + insert_pending_receipt(&conn, route_pending).unwrap(); + crate::db::update_space_api_route_identity( + &conn, + route_sandbox_id, + &SpaceApiRouteIdentity { + execution_generation: Some(ExecutionGeneration::new(5).unwrap()), + ..SpaceApiRouteIdentity::default() + }, + RouteIdentityProvenance::Unfenced, + ) + .unwrap(); + assert!( + pending_receipt_for_sandbox(&conn, route_sandbox_id) + .unwrap() + .is_none() + ); + + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let pending = receipt( + sandbox_id, + expectation, + 4, + SandboxActivationAttemptId::new(), + ); + insert_pending_receipt(&conn, pending).unwrap(); + crate::db::update_runtime_fields( + &conn, + sandbox_id, + crate::db::RuntimeFields { + uid: 10_000, + slot: 7, + guest_cid: 37, + pid: std::process::id(), + spaced_host_pid: 10_001, + netgtw_pid: 10_002, + vsock_uds_path: "/space/run/firecracker/sandboxes/test/root/v.sock", + api_socket_path: "/space/run/firecracker/sandboxes/test/api.sock", + spaced_endpoint_proof: None, + space_api_route_identity: SpaceApiRouteIdentity { + execution_generation: Some(ExecutionGeneration::new(5).unwrap()), + ..SpaceApiRouteIdentity::default() + }, + route_identity_provenance: RouteIdentityProvenance::Unfenced, + }, + ) + .unwrap(); + assert!( + pending_receipt_for_sandbox(&conn, sandbox_id) + .unwrap() + .is_none() + ); +} + +#[test] +fn completion_reports_a_typed_missing_expected_receipt() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let pending = receipt( + sandbox_id, + expectation, + 4, + SandboxActivationAttemptId::new(), + ); + insert_pending_receipt(&conn, pending).unwrap(); + conn.execute( + "UPDATE sandboxes + SET state = 'deleting', + cleanup_completion = 'delete_row', + cleanup_keep_filesystem = 0 + WHERE id = ?1", + [sandbox_id.to_string()], + ) + .unwrap(); + discard_pending_receipt_superseded_by_generation( + &conn, + sandbox_id, + ExecutionGeneration::new(5).unwrap(), + ) + .unwrap(); + + let error = crate::db::complete_row_intent_cleanup_with_receipt( + &conn, + &sandbox_id.to_string(), + crate::db::CleanupCompletion::DeleteRow, + None, + Some(pending), + ) + .unwrap_err(); + + assert!(matches!( + error, + crate::db::CompleteRowIntentCleanupError::ExpectedReceiptMissingOrChanged { + sandbox_id: observed, + } if observed == sandbox_id + )); +} + +#[test] +fn newer_route_generation_supersedes_receipt_completion() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let pending = receipt( + sandbox_id, + expectation, + 4, + SandboxActivationAttemptId::new(), + ); + insert_pending_receipt(&conn, pending).unwrap(); + conn.execute( + "UPDATE sandboxes + SET state = 'deleting', + cleanup_completion = 'delete_row', + cleanup_keep_filesystem = 0 + WHERE id = ?1", + [sandbox_id.to_string()], + ) + .unwrap(); + record_retained_route_evidence( + &conn, + sandbox_id, + RouteEvidence { + execution_generation: ExecutionGeneration::new(5).unwrap(), + activation_attempt_id: SandboxActivationAttemptId::new(), + substrate: NodeManagedBy::Kubernetes, + provenance: RouteIdentityProvenance::ClusterFenced, + }, + ) + .unwrap(); + discard_pending_receipt_superseded_by_generation( + &conn, + sandbox_id, + ExecutionGeneration::new(5).unwrap(), + ) + .unwrap(); + + assert_eq!( + crate::db::complete_row_intent_cleanup_with_receipt( + &conn, + &sandbox_id.to_string(), + crate::db::CleanupCompletion::DeleteRow, + None, + Some(pending), + ) + .unwrap(), + crate::db::CompleteRowIntentCleanupDisposition::Superseded, + ); + assert!( + crate::db::get_sandbox(&conn, &sandbox_id.to_string()) + .unwrap() + .is_some(), + "the superseded cleanup must not delete the replacement execution" + ); +} + +#[test] +fn newer_current_execution_generation_supersedes_receipt_completion() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let pending = receipt( + sandbox_id, + expectation, + 4, + SandboxActivationAttemptId::new(), + ); + insert_pending_receipt(&conn, pending).unwrap(); + assert!( + discard_pending_receipt_superseded_by_generation( + &conn, + sandbox_id, + ExecutionGeneration::new(5).unwrap(), + ) + .unwrap() + .is_some() + ); + crate::db::update_space_api_route_identity( + &conn, + sandbox_id, + &SpaceApiRouteIdentity { + execution_generation: Some(ExecutionGeneration::new(5).unwrap()), + ..SpaceApiRouteIdentity::default() + }, + RouteIdentityProvenance::Unfenced, + ) + .unwrap(); + conn.execute( + "UPDATE sandboxes + SET state = 'started', + cleanup_completion = NULL, + cleanup_keep_filesystem = NULL + WHERE id = ?1", + [sandbox_id.to_string()], + ) + .unwrap(); + + assert_eq!( + crate::db::complete_row_intent_cleanup_with_receipt( + &conn, + &sandbox_id.to_string(), + crate::db::CleanupCompletion::DeleteRow, + None, + Some(pending), + ) + .unwrap(), + crate::db::CompleteRowIntentCleanupDisposition::Superseded, + ); + assert!( + crate::db::get_sandbox(&conn, &sandbox_id.to_string()) + .unwrap() + .is_some(), + "the superseded cleanup must not delete the replacement execution" + ); + assert!( + pending_receipt_for_sandbox(&conn, sandbox_id) + .unwrap() + .is_none() + ); +} + +#[test] +fn equal_current_execution_generation_does_not_supersede_receipt_completion() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + let generation = ExecutionGeneration::new(4).unwrap(); + crate::db::update_space_api_route_identity( + &conn, + sandbox_id, + &SpaceApiRouteIdentity { + execution_generation: Some(generation), + ..SpaceApiRouteIdentity::default() + }, + RouteIdentityProvenance::Unfenced, + ) + .unwrap(); + let pending = receipt( + sandbox_id, + expectation, + generation.get(), + SandboxActivationAttemptId::new(), + ); + insert_pending_receipt(&conn, pending).unwrap(); + conn.execute( + "UPDATE sandboxes + SET state = 'deleting', + cleanup_completion = 'delete_row', + cleanup_keep_filesystem = 0 + WHERE id = ?1", + [sandbox_id.to_string()], + ) + .unwrap(); + let mismatched = AutoLifecycleDeleteReceipt { + receipt_id: AutoLifecycleDeleteReceiptId::generate(), + ..pending + }; + + let error = crate::db::complete_row_intent_cleanup_with_receipt( + &conn, + &sandbox_id.to_string(), + crate::db::CleanupCompletion::DeleteRow, + None, + Some(mismatched), + ) + .unwrap_err(); + + assert!(matches!( + error, + crate::db::CompleteRowIntentCleanupError::ExpectedReceiptMissingOrChanged { + sandbox_id: observed, + } if observed == sandbox_id + )); + assert!( + crate::db::get_sandbox(&conn, &sandbox_id.to_string()) + .unwrap() + .is_some() + ); + assert_eq!( + pending_receipt_for_sandbox(&conn, sandbox_id).unwrap(), + Some(pending) + ); +} + +#[test] +fn equal_generation_retained_evidence_cannot_be_downgraded() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + insert_sandbox(&conn, sandbox_id); + let first_attempt = SandboxActivationAttemptId::new(); + let replacement_attempt = SandboxActivationAttemptId::new(); + record_retained_route_evidence( + &conn, + sandbox_id, + RouteEvidence { + execution_generation: ExecutionGeneration::new(4).unwrap(), + activation_attempt_id: first_attempt, + substrate: NodeManagedBy::Kubernetes, + provenance: RouteIdentityProvenance::ClusterFenced, + }, + ) + .unwrap(); + + assert!( + !record_retained_route_evidence( + &conn, + sandbox_id, + RouteEvidence { + execution_generation: ExecutionGeneration::new(4).unwrap(), + activation_attempt_id: replacement_attempt, + substrate: NodeManagedBy::Kubernetes, + provenance: RouteIdentityProvenance::Unfenced, + }, + ) + .unwrap() + ); + assert_eq!( + conn.query_row( + "SELECT last_observed_route_activation_id, + last_observed_route_identity_provenance + FROM sandboxes WHERE id = ?1", + [sandbox_id.to_string()], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ) + .unwrap(), + (first_attempt.to_string(), "cluster_fenced".to_string()) + ); + + assert!( + record_retained_route_evidence( + &conn, + sandbox_id, + RouteEvidence { + execution_generation: ExecutionGeneration::new(4).unwrap(), + activation_attempt_id: replacement_attempt, + substrate: NodeManagedBy::Kubernetes, + provenance: RouteIdentityProvenance::ClusterFenced, + }, + ) + .unwrap() + ); + assert_eq!( + conn.query_row( + "SELECT last_observed_route_activation_id, + last_observed_route_identity_provenance + FROM sandboxes WHERE id = ?1", + [sandbox_id.to_string()], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ) + .unwrap(), + ( + replacement_attempt.to_string(), + "cluster_fenced".to_string() + ) + ); + + assert!( + !record_retained_route_evidence( + &conn, + sandbox_id, + RouteEvidence { + execution_generation: ExecutionGeneration::new(4).unwrap(), + activation_attempt_id: first_attempt, + substrate: NodeManagedBy::Kubernetes, + provenance: RouteIdentityProvenance::ClusterFenced, + }, + ) + .unwrap(), + "delayed same-generation evidence must not replace the newer attempt" + ); + assert_eq!( + conn.query_row( + "SELECT last_observed_route_activation_id + FROM sandboxes WHERE id = ?1", + [sandbox_id.to_string()], + |row| row.get::<_, String>(0), + ) + .unwrap(), + replacement_attempt.to_string() + ); +} + +#[test] +fn receipt_conflict_uses_bounded_retry_without_claiming_a_durable_block() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + insert_sandbox(&conn, sandbox_id); + + let storage_id = sandbox_id.to_string(); + assert!(record_receipt_conflict_retry(&conn, &storage_id).unwrap()); + assert!(!record_receipt_conflict_retry(&conn, &storage_id).unwrap()); + + let (block_reason, retry_at, runtime_state_json): (Option, i64, String) = conn + .query_row( + "SELECT auto_lifecycle_delete_block_reason, + next_auto_event_at, + auto_lifecycle_state_json + FROM sandboxes + WHERE id = ?1", + [sandbox_id.to_string()], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + let runtime_state: pplx_space_common_core::types::AutoLifecycleRuntimeState = + serde_json::from_str(&runtime_state_json).unwrap(); + assert!(block_reason.is_none()); + assert!(retry_at > crate::db::now_epoch_i64().unwrap()); + assert_eq!( + runtime_state.next_auto_event_at, + Some(u64::try_from(retry_at).unwrap()) + ); + assert!( + runtime_state + .last_auto_error + .as_deref() + .is_some_and(|error| error.contains("conflicting receipt evidence")) + ); +} + +#[test] +fn policy_replacement_retires_obsolete_delete_block() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + + assert!(matches!( + classify_and_record_admission( + &conn, + sandbox_id, + Some(NodeRole::Worker), + None, + expectation, + ) + .unwrap(), + AutoLifecycleDeleteAdmission::Blocked { + reason: AutoLifecycleDeleteBlockReason::MissingRouteIdentity, + .. + } + )); + + crate::db::update_auto_lifecycle( + &conn, + &sandbox_id.to_string(), + &AutoLifecyclePolicy::disabled(), + PolicyAuthority::Cluster, + ) + .unwrap(); + + let (reason, blocked_at): (Option, Option) = conn + .query_row( + "SELECT auto_lifecycle_delete_block_reason, + auto_lifecycle_delete_blocked_at + FROM sandboxes + WHERE id = ?1", + [sandbox_id.to_string()], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + assert_eq!((reason, blocked_at), (None, None)); +} + +#[test] +fn unknown_substrate_is_durably_blocked() { + let conn = crate::db::test_conn(); + let sandbox_id = SandboxId::new(); + let expectation = insert_sandbox(&conn, sandbox_id); + conn.execute( + "UPDATE sandboxes + SET space_api_route_execution_generation = 7, + space_api_route_activation_attempt_id = ?1, + space_api_route_substrate_kind = 'future_substrate', + space_api_route_identity_provenance = 'cluster_fenced' + WHERE id = ?2", + rusqlite::params![ + SandboxActivationAttemptId::new().to_string(), + sandbox_id.to_string() + ], + ) + .unwrap(); + + let admission = + classify_and_record_admission(&conn, sandbox_id, Some(NodeRole::Worker), None, expectation) + .expect("unknown substrate must fail closed as a durable block"); + let AutoLifecycleDeleteAdmission::Blocked { reason, changed } = admission else { + panic!("unknown substrate must not admit deletion: {admission:?}"); + }; + assert_eq!(reason.as_str(), "unknown_substrate"); + assert!(changed); +} diff --git a/crates/libs/sbx/mgr/server/src/db/cleanup_episodes.rs b/crates/libs/sbx/mgr/server/src/db/cleanup_episodes.rs index 183de45ae..20774ed10 100644 --- a/crates/libs/sbx/mgr/server/src/db/cleanup_episodes.rs +++ b/crates/libs/sbx/mgr/server/src/db/cleanup_episodes.rs @@ -154,7 +154,7 @@ pub(crate) fn insert_sandbox_with_cleanup_episode_and_route_evidence( params: InsertSandboxParams<'_>, episode: CleanupEpisodeSpec, route_evidence: Option, - open_route_activation: Option<&crate::route_activation_claims::OpenRouteActivationClaim>, + open_route_activation: Option<&route_activation_claims::OpenRouteActivationClaim>, execution: Option<&ExecutionMetadata>, ) -> std::result::Result<(), InsertSandboxWithCleanupEpisodeError> { if params.state != episode.activation_state { @@ -172,7 +172,7 @@ pub(crate) fn insert_sandbox_with_cleanup_episode_and_route_evidence( upsert_sandbox_execution_metadata_in_transaction(&tx, &sandbox_id, execution)?; } if let Some(route_evidence) = route_evidence { - crate::auto_lifecycle_delete_receipts::db::record_retained_route_evidence( + auto_lifecycle_delete_receipts::record_retained_route_evidence( &tx, sandbox_id .parse() @@ -181,11 +181,7 @@ pub(crate) fn insert_sandbox_with_cleanup_episode_and_route_evidence( )?; } if let Some(open_route_activation) = open_route_activation { - crate::route_activation_claims::insert_create_claim( - &tx, - &sandbox_id, - open_route_activation, - )?; + route_activation_claims::insert_create_claim(&tx, &sandbox_id, open_route_activation)?; } insert_cleanup_episode(&tx, &sandbox_id, episode)?; tx.commit()?; @@ -1037,7 +1033,7 @@ pub(crate) fn complete_cleanup_episode_for_job( record_execution_delete_receipt(&tx, sandbox_id, execution_generation)?; delete_sandbox_child_rows(&tx, sandbox_id)?; tx.execute("DELETE FROM sandboxes WHERE id = ?1", params![sandbox_id]) - .map_err(crate::auto_lifecycle_delete_receipts::db::map_pending_receipt_delete_error)?; + .map_err(auto_lifecycle_delete_receipts::map_pending_receipt_delete_error)?; } else { if !complete_state_cleanup(&tx, sandbox_id, cleanup_id, &episode, recovery.as_ref())? { tx.commit()?; @@ -1050,7 +1046,7 @@ pub(crate) fn complete_cleanup_episode_for_job( // completed lifecycle teardown. Keep the exact attempt's continuation // claim so a transiently failed RPC can retry after runtime cleanup. if episode.completion == CleanupCompletion::Quarantine { - crate::route_activation_claims::close_claim(&tx, sandbox_id)?; + route_activation_claims::close_claim(&tx, sandbox_id)?; } tx.execute( "DELETE FROM sandbox_cleanup_episodes @@ -1133,9 +1129,7 @@ pub(crate) enum CompleteRowIntentCleanupError { #[error(transparent)] Database(#[from] rusqlite::Error), #[error(transparent)] - ReceiptCompletion( - #[from] crate::auto_lifecycle_delete_receipts::db::CompletePendingReceiptError, - ), + ReceiptCompletion(#[from] auto_lifecycle_delete_receipts::CompletePendingReceiptError), #[error( "expected auto-lifecycle delete receipt is missing or changed for sandbox {sandbox_id}" )] @@ -1260,7 +1254,7 @@ pub(crate) fn complete_row_intent_cleanup_for_job( .transpose()?; let pending_delete_receipt = parsed_delete_sandbox_id .map(|sandbox_id| { - crate::auto_lifecycle_delete_receipts::db::pending_receipt_for_sandbox(&tx, sandbox_id) + auto_lifecycle_delete_receipts::pending_receipt_for_sandbox(&tx, sandbox_id) }) .transpose()? .flatten(); @@ -1322,7 +1316,7 @@ pub(crate) fn complete_row_intent_cleanup_for_job( // A caller-supplied expected receipt was matched by the hoisted // ownership check. Startup and operator recovery instead derive // ownership from this durable receipt plus the DeleteRow intent. - crate::auto_lifecycle_delete_receipts::db::complete_pending_receipt( + auto_lifecycle_delete_receipts::complete_pending_receipt( &tx, parsed_sandbox_id, receipt.receipt_id, @@ -1335,7 +1329,7 @@ pub(crate) fn complete_row_intent_cleanup_for_job( )?; delete_sandbox_child_rows(&tx, sandbox_id)?; tx.execute("DELETE FROM sandboxes WHERE id = ?1", params![sandbox_id]) - .map_err(crate::auto_lifecycle_delete_receipts::db::map_pending_receipt_delete_error)?; + .map_err(auto_lifecycle_delete_receipts::map_pending_receipt_delete_error)?; cleanup_jobs::finish_in_transaction(&tx, job_id, &cleanup_jobs::CleanupOutcome::Applied)?; tx.commit()?; return Ok(CompleteRowIntentCleanupDisposition::Applied); @@ -1413,7 +1407,7 @@ pub(crate) fn complete_row_intent_cleanup_for_job( // `apply_cleanup_state` clears the live identity and fires the retained // identity trigger. Close the claim only after that observation, but // before this transaction commits. - crate::route_activation_claims::close_claim(&tx, sandbox_id)?; + route_activation_claims::close_claim(&tx, sandbox_id)?; let outcome = if missing_suspend_artifact { cleanup_jobs::CleanupOutcome::Quarantined( SUSPEND_PUBLISH_FAILED_NO_ARTIFACT_RUNTIME_GONE.to_owned(), @@ -1514,7 +1508,7 @@ pub(crate) fn quarantine_cleanup_episode_for_job( ], )?; if changed == 1 { - crate::route_activation_claims::close_claim(&tx, sandbox_id)?; + route_activation_claims::close_claim(&tx, sandbox_id)?; insert_new_quarantine_episode( &tx, &QuarantineEpisode { diff --git a/crates/libs/sbx/mgr/server/src/db/cleanup_jobs.rs b/crates/libs/sbx/mgr/server/src/db/cleanup_jobs.rs index 982514ca9..7849a8510 100644 --- a/crates/libs/sbx/mgr/server/src/db/cleanup_jobs.rs +++ b/crates/libs/sbx/mgr/server/src/db/cleanup_jobs.rs @@ -278,7 +278,7 @@ pub(crate) fn current(conn: &Connection, work: &CleanupWork) -> rusqlite::Result return cleanup_episode_current(conn, &id, episode); } if work.completion == CleanupCompletion::DeleteRow - && !crate::auto_lifecycle_delete_receipts::db::pending_receipt_matches_observed_execution( + && !auto_lifecycle_delete_receipts::pending_receipt_matches_observed_execution( conn, work.sandbox_id, )? diff --git a/crates/libs/sbx/mgr/server/src/db/eviction.rs b/crates/libs/sbx/mgr/server/src/db/eviction.rs index 25974bba8..7475c8a12 100644 --- a/crates/libs/sbx/mgr/server/src/db/eviction.rs +++ b/crates/libs/sbx/mgr/server/src/db/eviction.rs @@ -309,7 +309,7 @@ fn cancel_live_upload_job(conn: &Connection, artifact: LocalArtifactId) -> Resul use pplx_space_common_jobs::{JobStatus, db as jobs}; let Some(job) = jobs::find_job_by_dedup_key( conn, - crate::snapshot_upload_jobs::SNAPSHOT_UPLOAD_JOB_KIND, + super::snapshot_upload_jobs::SNAPSHOT_UPLOAD_JOB_KIND, &artifact.to_string(), )? else { diff --git a/crates/libs/sbx/mgr/server/src/db/eviction_scan.rs b/crates/libs/sbx/mgr/server/src/db/eviction_scan.rs new file mode 100644 index 000000000..c23167607 --- /dev/null +++ b/crates/libs/sbx/mgr/server/src/db/eviction_scan.rs @@ -0,0 +1,218 @@ +//! Suspended-sandbox reads behind `crate::sandbox_eviction`'s candidate +//! scans and local eviction evidence. + +use pplx_space_common_svc::sql::sqlite::SqliteU64; +use rusqlite::{Connection, OptionalExtension, Result, params}; + +use crate::db::timestamps::EpochSeconds; + +const VALID_SUSPENDED_UPDATED_AT_SQL: &str = + "s.updated_at <> '' AND s.updated_at NOT GLOB '*[^0-9]*'"; + +/// Idle witness for eviction candidacy: the sandbox's last real +/// activity, not the row's last write. `updated_at` has an unbounded +/// writer set (periodic reconcilers rewrite rows value-identically and +/// stamp it), so a row can look seconds-old for weeks while its bytes +/// sit cold — the activity clocks inside `auto_lifecycle_state_json` +/// only advance on genuine touches and transitions. NULL-safe two-way +/// max of `last_access_at`/`last_command_at`; rows carrying neither +/// clock (operator/legacy/orphan-adopted rows) fall back to +/// `updated_at`, preserving their previous behavior exactly. The +/// `json_valid` gate is load-bearing: `json_extract` aborts the whole +/// statement on malformed JSON, so without it one corrupt row would +/// fail every scan and shield the valid cold rows around it — a +/// malformed blob falls back to `updated_at` like a missing one. Each +/// clock is additionally type-gated to a non-negative integer: our +/// writers only ever store u64 epoch seconds, so anything else +/// (negative, fractional, string) is corruption and must read as +/// absent rather than fail the `SqliteU64` row decode. The CAS claim +/// deliberately keeps raw `updated_at`: "any write means the row is in +/// flux" is the right semantic there and the wrong one here. +/// +/// The partial eviction index prunes to suspended rows before evaluating +/// this expression. If scan cost grows with node row counts, the escape +/// hatch is an index on this deterministic expression or a virtual generated +/// column — not a writer-maintained copy, which would reintroduce the drift +/// this witness exists to escape. +/// Scalar `MAX` returns NULL if ANY argument is NULL, so absent or +/// corrupt clocks are mapped to a -1 sentinel first (real clocks are +/// type-gated non-negative, so the sentinel cannot collide) and +/// `NULLIF` restores NULL when both were absent. +const IDLE_WITNESS_SQL: &str = "COALESCE( + CASE + WHEN json_valid(s.auto_lifecycle_state_json) THEN + NULLIF( + MAX( + COALESCE(CASE + WHEN typeof(json_extract(s.auto_lifecycle_state_json, '$.last_access_at')) = 'integer' + AND json_extract(s.auto_lifecycle_state_json, '$.last_access_at') >= 0 + THEN json_extract(s.auto_lifecycle_state_json, '$.last_access_at') + END, -1), + COALESCE(CASE + WHEN typeof(json_extract(s.auto_lifecycle_state_json, '$.last_command_at')) = 'integer' + AND json_extract(s.auto_lifecycle_state_json, '$.last_command_at') >= 0 + THEN json_extract(s.auto_lifecycle_state_json, '$.last_command_at') + END, -1) + ), + -1) + END, + CAST(s.updated_at AS INTEGER) + )"; + +/// `(sandbox_id, idle witness)` of cold suspended sandboxes after the +/// `(cursor_witness, cursor_sandbox_id)` keyset position, in witness then id +/// order. +pub(crate) fn idle_suspended_candidates( + conn: &Connection, + cutoff: i64, + cursor_witness: Option, + cursor_sandbox_id: Option<&str>, + limit: usize, +) -> Result> { + let sql = format!( + "SELECT s.id, {IDLE_WITNESS_SQL} FROM sandboxes s + WHERE s.state = 'suspended' + AND {VALID_SUSPENDED_UPDATED_AT_SQL} + AND {IDLE_WITNESS_SQL} <= ?1 + AND ( + ?2 IS NULL + OR {IDLE_WITNESS_SQL} > ?2 + OR ({IDLE_WITNESS_SQL} = ?2 AND s.id > ?3) + ) + AND ( + s.current_local_artifact_id IS NOT NULL + OR EXISTS ( + SELECT 1 FROM jobs j + WHERE j.kind = 'snapshot_upload' + AND j.subject_id = s.id + AND json_extract(j.payload, '$.snapshot_id') IS NOT NULL + ) + ) + AND NOT EXISTS ( + SELECT 1 FROM jobs j + WHERE j.kind = 'snapshot_upload' + AND j.subject_id = s.id + AND j.status IN ('pending', 'running') + ) + ORDER BY {IDLE_WITNESS_SQL} ASC, s.id ASC + LIMIT ?4" + ); + let mut stmt = conn.prepare(&sql)?; + let rows = stmt + .query_map( + params![cutoff, cursor_witness, cursor_sandbox_id, limit as i64], + |row| { + let sandbox_id: String = row.get(0)?; + let idle_witness_secs = row.get::<_, SqliteU64>(1)?.0; + Ok((sandbox_id, idle_witness_secs)) + }, + )? + .collect::>()?; + Ok(rows) +} + +/// Ids of cold suspended sandboxes with neither a pinned artifact nor an +/// upload job carrying a snapshot id, oldest witness first. +pub(crate) fn legacy_reupload_candidates( + conn: &Connection, + idle_cutoff_epoch_secs: u64, + limit: usize, +) -> Result> { + let sql = format!( + "SELECT s.id FROM sandboxes s + WHERE s.state = 'suspended' + AND {VALID_SUSPENDED_UPDATED_AT_SQL} + AND {IDLE_WITNESS_SQL} <= ?1 + AND s.current_local_artifact_id IS NULL + AND NOT EXISTS ( + SELECT 1 FROM jobs j + WHERE j.kind = 'snapshot_upload' + AND j.subject_id = s.id + AND j.status IN ('pending', 'running') + ) + AND NOT EXISTS ( + SELECT 1 FROM jobs j + WHERE j.kind = 'snapshot_upload' + AND j.subject_id = s.id + AND json_extract(j.payload, '$.snapshot_id') IS NOT NULL + ) + ORDER BY {IDLE_WITNESS_SQL} ASC + LIMIT ?2" + ); + let mut stmt = conn.prepare(&sql)?; + let rows = stmt + .query_map( + params![SqliteU64(idle_cutoff_epoch_secs), limit as i64], + |r| r.get(0), + )? + .collect::>()?; + Ok(rows) +} + +pub(crate) fn count_malformed_suspended_updated_at(conn: &Connection) -> Result { + let sql = format!( + "SELECT COUNT(*) FROM sandboxes s + WHERE s.state = 'suspended' + AND NOT ({VALID_SUSPENDED_UPDATED_AT_SQL})" + ); + conn.query_row(&sql, [], |row| { + let count: i64 = row.get(0)?; + Ok(usize::try_from(count).unwrap_or_default()) + }) +} + +/// Every local `sandboxes` row, whatever its state. node-ctl's full census +/// publishes one entry per row, so this is the count that sizes it. +pub(crate) fn count_resident_rows(conn: &Connection) -> Result { + conn.query_row("SELECT COUNT(*) FROM sandboxes", [], |row| { + let count: i64 = row.get(0)?; + usize::try_from(count).map_err(|_| rusqlite::Error::IntegralValueOutOfRange(0, count)) + }) +} + +/// The sandbox's pinned artifact id and `updated_at`, `None` without a row. +pub(crate) fn artifact_pin_and_updated_at( + conn: &Connection, + sandbox_id: &str, +) -> Result, EpochSeconds)>> { + conn.query_row( + "SELECT current_local_artifact_id, updated_at FROM sandboxes WHERE id = ?1", + params![sandbox_id], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .optional() +} + +/// `(artifact_kind, local_artifact_id, backed up)` of the artifact `pin` +/// names, only if `sandbox_id` owns it. +pub(crate) fn owned_artifact_evidence( + conn: &Connection, + pin: &str, + sandbox_id: &str, +) -> Result> { + conn.query_row( + "SELECT artifact_kind, local_artifact_id, backup_completed_at IS NOT NULL + FROM local_artifacts + WHERE local_artifact_id = ?1 AND sandbox_id = ?2", + params![pin, sandbox_id], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), + ) + .optional() +} + +/// The payload `snapshot_id` and `created_at` of the sandbox's newest +/// snapshot-upload job, whatever its status. +pub(crate) fn latest_upload_job( + conn: &Connection, + sandbox_id: &str, +) -> Result, String)>> { + conn.query_row( + "SELECT json_extract(j.payload, '$.snapshot_id'), j.created_at FROM jobs j + WHERE j.kind = 'snapshot_upload' AND j.subject_id = ?1 + ORDER BY j.created_at DESC, j.job_id DESC + LIMIT 1", + params![sandbox_id], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .optional() +} diff --git a/crates/libs/sbx/mgr/server/src/db/guest_health_scan.rs b/crates/libs/sbx/mgr/server/src/db/guest_health_scan.rs new file mode 100644 index 000000000..29f3ae51e --- /dev/null +++ b/crates/libs/sbx/mgr/server/src/db/guest_health_scan.rs @@ -0,0 +1,66 @@ +//! Sandbox rows the watchdog's guest-health and probe passes read. + +use rusqlite::types::Type; +use rusqlite::{Connection, OptionalExtension, Result}; + +use pplx_space_common_core::sql; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct SandboxGuestHealthRow { + pub(crate) id: String, + pub(crate) state: crate::state::SandboxState, + pub(crate) template_content_hash: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +#[error("unknown sandbox state '{value}'")] +struct SandboxStateParseError { + value: String, +} + +pub(crate) fn list_guest_health_rows(conn: &Connection) -> Result> { + let mut stmt = conn.prepare(sql!("SELECT id, state, template_hash FROM sandboxes"))?; + let rows = stmt.query_map([], |row| { + let state_value = row.get::<_, String>(1)?; + let state = state_value + .parse::() + .map_err(|_| { + rusqlite::Error::FromSqlConversionFailure( + 1, + Type::Text, + Box::new(SandboxStateParseError { value: state_value }), + ) + })?; + Ok(SandboxGuestHealthRow { + id: row.get(0)?, + state, + template_content_hash: row.get(2)?, + }) + })?; + rows.collect() +} + +/// One sandbox's `state` label, `None` without a row. +pub(crate) type StateLabelRead = Result>; + +/// The `state` label of each of `sandbox_ids` through one prepared +/// statement, `None` for a sandbox without a row. Only preparing the +/// statement fails the whole lookup; each id's read fails on its own. +pub(crate) fn state_labels_for_ids<'a>( + conn: &Connection, + sandbox_ids: &'a [String], +) -> Result> { + let mut statement = conn.prepare("SELECT state FROM sandboxes WHERE id = ?1")?; + Ok(sandbox_ids + .iter() + .map(|sandbox_id| { + let label = statement + .query_row([sandbox_id], |row| row.get::<_, String>(0)) + .optional(); + (sandbox_id, label) + }) + .collect()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/libs/sbx/mgr/server/src/db/guest_health_scan/tests.rs b/crates/libs/sbx/mgr/server/src/db/guest_health_scan/tests.rs new file mode 100644 index 000000000..de5401770 --- /dev/null +++ b/crates/libs/sbx/mgr/server/src/db/guest_health_scan/tests.rs @@ -0,0 +1,30 @@ +use pplx_space_common_core::types::SandboxId; + +use crate::service::tests::service_tests::insert_test_sandbox; +use crate::state::SandboxState; + +use super::*; + +#[test] +fn state_labels_follow_the_ids_and_report_missing_rows_as_none() { + let conn = crate::db::test_conn(); + let [started, missing, stopped] = [(); 3].map(|()| SandboxId::new().to_string()); + insert_test_sandbox(&conn, &started, SandboxState::Started); + insert_test_sandbox(&conn, &stopped, SandboxState::Stopped); + let ids = [stopped.clone(), missing.clone(), started.clone()]; + + let labels: Vec<_> = state_labels_for_ids(&conn, &ids) + .unwrap() + .into_iter() + .map(|(id, label)| (id.clone(), label.unwrap())) + .collect(); + + assert_eq!( + labels, + [ + (stopped, Some("stopped".to_owned())), + (missing, None), + (started, Some("started".to_owned())), + ] + ); +} diff --git a/crates/libs/sbx/mgr/server/src/db/local_artifact_leases.rs b/crates/libs/sbx/mgr/server/src/db/local_artifact_leases.rs new file mode 100644 index 000000000..df8b54e6e --- /dev/null +++ b/crates/libs/sbx/mgr/server/src/db/local_artifact_leases.rs @@ -0,0 +1,382 @@ +//! Statements behind `crate::local_artifacts::LocalArtifacts` and its +//! restore and delete leases. Each runs on the connection or transaction its +//! caller holds, so a checkout's reads and its CAS share one transaction. + +use pplx_space_common_core::sql; +use pplx_space_common_core::types::{ArtifactKind, LocalArtifactId, SnapshotName}; +use pplx_space_common_svc::sql::sqlite::SqliteU64; +use rusqlite::{Connection, OptionalExtension, Result, params}; + +use crate::db::timestamps::EpochSeconds; + +/// One sandbox's current lifecycle artifact that this node cannot prove has +/// a durable remote copy. +/// +/// Identity comes from the `sandboxes` pin rather than from the artifact +/// row, because the case where there *is* no artifact row is one of the +/// cases this reports. Migration 026's CHECK keeps `current_local_artifact_id` +/// and `current_local_artifact_kind` set or unset together, so the pin alone +/// always carries a whole identity. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct PendingBackup { + pub sandbox_id: String, + pub local_artifact_id: LocalArtifactId, + pub artifact_kind: ArtifactKind, +} + +/// Logical state of a `local_artifacts` row. Encoded on the row's +/// `state` column; transitions are CAS via +/// `LocalArtifacts::check_out_for_restore` / +/// `LocalArtifacts::check_out_for_delete`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ArtifactState { + /// Published and available. The only state from which a checkout + /// (restore or delete) can succeed. + Ready, + /// A restore is currently reading this artifact's `local_path`. + /// Reapers must skip; deleters must wait or fail with NotReady. + Restoring, + /// A delete has reserved this row. The on-disk directory has + /// usually been renamed to a `.gc-pending-` sibling. Readers + /// should treat the artifact as gone. + Deleting, +} + +impl ArtifactState { + pub fn as_str(self) -> &'static str { + match self { + ArtifactState::Ready => "ready", + ArtifactState::Restoring => "restoring", + ArtifactState::Deleting => "deleting", + } + } +} + +/// Error parsing an [`ArtifactState`] from its wire/DB string. The +/// CHECK constraint on `local_artifacts.state` should make this +/// unreachable for rows the schema accepted; it surfaces table +/// corruption or migration drift. +#[derive(Debug, thiserror::Error)] +#[error("unrecognized artifact state {0:?}")] +pub struct ParseArtifactStateError(String); + +impl std::str::FromStr for ArtifactState { + type Err = ParseArtifactStateError; + + fn from_str(s: &str) -> Result { + match s { + "ready" => Ok(ArtifactState::Ready), + "restoring" => Ok(ArtifactState::Restoring), + "deleting" => Ok(ArtifactState::Deleting), + _ => Err(ParseArtifactStateError(s.to_string())), + } + } +} + +impl std::fmt::Display for ArtifactState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// Point-in-time publication evidence for a named snapshot, not a checkout lease. +pub(crate) struct NamedArtifactPublication { + pub snapshot_name: SnapshotName, + pub local_path: String, + pub artifact_kind: ArtifactKind, + pub state: ArtifactState, +} + +/// Every sandbox whose pinned artifact has no backup-completed stamp, or +/// whose pin names no row it owns. The caller, +/// `LocalArtifacts::list_pending_backups`, documents the contract. +pub(crate) fn list_pending_backups(conn: &Connection) -> Result> { + conn.prepare( + "SELECT s.id, s.current_local_artifact_id, s.current_local_artifact_kind + FROM sandboxes AS s + LEFT JOIN local_artifacts AS a + ON a.local_artifact_id = s.current_local_artifact_id + AND a.sandbox_id = s.id + WHERE s.current_local_artifact_id IS NOT NULL + AND (a.local_artifact_id IS NULL OR a.backup_completed_at IS NULL) + ORDER BY s.id", + )? + .query_map([], |row| { + Ok(PendingBackup { + sandbox_id: row.get(0)?, + local_artifact_id: row.get::<_, String>(1)?.parse().map_err(|error| { + rusqlite::Error::FromSqlConversionFailure( + 1, + rusqlite::types::Type::Text, + Box::new(error), + ) + })?, + artifact_kind: row.get::<_, String>(2)?.parse().map_err(|error| { + rusqlite::Error::FromSqlConversionFailure( + 2, + rusqlite::types::Type::Text, + Box::new(error), + ) + })?, + }) + })? + .collect() +} + +/// A sandbox's named-snapshot artifacts, within the inventory read budget. +pub(crate) fn list_named_publications( + conn: &Connection, + sandbox_id: &str, +) -> std::result::Result, crate::db::inventory_read::InventoryReadError> +{ + use pplx_space_common_core::inventory::{InventoryBudget, MAX_ITEMS, MAX_RAW_BYTES}; + let mut statement = conn.prepare(sql!( + "SELECT snapshot_name, local_path, artifact_kind, state + FROM local_artifacts WHERE sandbox_id = ?1 AND artifact_origin = 'snapshot'" + ))?; + let mut rows = statement.query([sandbox_id])?; + let mut budget = InventoryBudget::new(MAX_ITEMS, MAX_RAW_BYTES); + let mut publications = Vec::new(); + while let Some(row) = rows.next()? { + crate::db::inventory_read::check_row(row, &mut budget)?; + publications.push(NamedArtifactPublication { + snapshot_name: row.get::<_, String>(0)?.parse().map_err(|error| { + rusqlite::Error::FromSqlConversionFailure( + 0, + rusqlite::types::Type::Text, + Box::new(error), + ) + })?, + local_path: row.get(1)?, + artifact_kind: row.get::<_, String>(2)?.parse().map_err(|error| { + rusqlite::Error::FromSqlConversionFailure( + 2, + rusqlite::types::Type::Text, + Box::new(error), + ) + })?, + state: row.get::<_, String>(3)?.parse().map_err(|error| { + rusqlite::Error::FromSqlConversionFailure( + 3, + rusqlite::types::Type::Text, + Box::new(error), + ) + })?, + }); + } + Ok(publications) +} + +/// `(local_path, state)` of an artifact row, as stored. +pub(crate) fn artifact_path_and_state( + conn: &Connection, + local_artifact_id: &str, +) -> Result> { + conn.query_row( + "SELECT local_path, state FROM local_artifacts WHERE local_artifact_id = ?1", + params![local_artifact_id], + |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)), + ) + .optional() +} + +/// CAS `ready` → `restoring`; the number of rows changed. +pub(crate) fn mark_artifact_restoring(conn: &Connection, local_artifact_id: &str) -> Result { + conn.execute( + "UPDATE local_artifacts SET state = 'restoring' + WHERE local_artifact_id = ?1 AND state = 'ready'", + params![local_artifact_id], + ) +} + +/// CAS `ready` → `deleting`; the number of rows changed. +pub(crate) fn mark_artifact_deleting(conn: &Connection, local_artifact_id: &str) -> Result { + conn.execute( + "UPDATE local_artifacts SET state = 'deleting' + WHERE local_artifact_id = ?1 AND state = 'ready'", + params![local_artifact_id], + ) +} + +/// The GC checkout's evidence for an artifact `sandbox_id` owns: its +/// creation and last-touch times, whether a sandbox pins it, and whether its +/// backup completed. +pub(crate) fn gc_checkout_evidence( + conn: &Connection, + local_artifact_id: &str, + sandbox_id: &str, +) -> Result> { + conn.query_row( + "SELECT created_at, COALESCE(last_touched_at, created_at), + EXISTS( + SELECT 1 FROM sandboxes + WHERE current_local_artifact_id = ?1 + OR suspend_predecessor_artifact_id = ?1 + ), + backup_completed_at IS NOT NULL + FROM local_artifacts WHERE local_artifact_id = ?1 AND sandbox_id = ?2", + params![local_artifact_id, sandbox_id], + |row| { + Ok(( + row.get::<_, EpochSeconds>(0)?, + row.get::<_, EpochSeconds>(1)?, + row.get::<_, bool>(2)?, + row.get::<_, bool>(3)?, + )) + }, + ) + .optional() +} + +/// Whether a sandbox pins the artifact as its current or suspend-predecessor +/// artifact. +pub(crate) fn artifact_is_pinned(conn: &Connection, local_artifact_id: &str) -> Result { + conn.query_row( + sql!( + "SELECT EXISTS(SELECT 1 FROM sandboxes + WHERE current_local_artifact_id = ?1 OR suspend_predecessor_artifact_id = ?1)" + ), + [local_artifact_id], + |row| row.get(0), + ) +} + +/// The number of artifact rows at `local_path`. +pub(crate) fn count_artifacts_at_path(conn: &Connection, local_path: &str) -> Result { + conn.query_row( + "SELECT COUNT(*) FROM local_artifacts WHERE local_path = ?1", + params![local_path], + |r| Ok(r.get::<_, SqliteU64>(0)?.0), + ) +} + +/// Reset every `restoring` row to `ready`; the number reset. +pub(crate) fn reset_restoring_artifacts(conn: &Connection) -> Result { + conn.execute( + "UPDATE local_artifacts SET state = 'ready' WHERE state = 'restoring'", + [], + ) +} + +/// `(local_artifact_id, local_path)` of every `deleting` row. +pub(crate) fn list_deleting_artifacts(conn: &Connection) -> Result> { + conn.prepare( + "SELECT local_artifact_id, local_path FROM local_artifacts + WHERE state = 'deleting'", + )? + .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? + .collect::>() +} + +/// CAS `deleting` → `ready`; the number of rows changed. +pub(crate) fn roll_back_deleting_artifact( + conn: &Connection, + local_artifact_id: &str, +) -> Result { + conn.execute( + "UPDATE local_artifacts SET state = 'ready' + WHERE local_artifact_id = ?1 AND state = 'deleting'", + params![local_artifact_id], + ) +} + +/// Delete the row if it is still `deleting`; the number deleted. +pub(crate) fn delete_deleting_artifact( + conn: &Connection, + local_artifact_id: &str, +) -> Result { + conn.execute( + "DELETE FROM local_artifacts + WHERE local_artifact_id = ?1 AND state = 'deleting'", + params![local_artifact_id], + ) +} + +/// CAS `restoring` → `ready`, touching the artifact now. +pub(crate) fn release_restoring_artifact_with_touch( + conn: &Connection, + local_artifact_id: &str, +) -> Result { + conn.execute( + "UPDATE local_artifacts + SET state = 'ready', last_touched_at = ?1 + WHERE local_artifact_id = ?2 AND state = 'restoring'", + params![crate::db::now()?, local_artifact_id], + ) +} + +/// CAS `restoring` → `ready`, leaving the touch time alone. +pub(crate) fn release_restoring_artifact( + conn: &Connection, + local_artifact_id: &str, +) -> Result { + conn.execute( + "UPDATE local_artifacts SET state = 'ready' + WHERE local_artifact_id = ?1 AND state = 'restoring'", + params![local_artifact_id], + ) +} + +/// Whether neither an artifact row nor a sandbox pin names the id. +pub(crate) fn gc_staging_is_unowned(conn: &Connection, local_artifact_id: &str) -> Result { + conn.query_row( + sql!( + "SELECT NOT EXISTS(SELECT 1 FROM local_artifacts WHERE local_artifact_id = ?1) + AND NOT EXISTS(SELECT 1 FROM sandboxes + WHERE current_local_artifact_id = ?1 OR suspend_predecessor_artifact_id = ?1)" + ), + [local_artifact_id], + |row| row.get(0), + ) +} + +/// The highest artifact rowid, `None` for an empty table. +pub(crate) fn max_artifact_rowid(conn: &Connection) -> Result> { + conn.query_row("SELECT MAX(rowid) FROM local_artifacts", [], |r| r.get(0)) +} + +/// An orphan-scan row's `(local_artifact_id, local_path, owned)`, or the +/// error decoding them. +pub(crate) type OrphanScanRow = Result<(String, String, bool)>; + +/// Up to `limit` artifact rows with rowids in `next..=through`, in rowid +/// order: each rowid, and its `(local_artifact_id, local_path, owned)` or the +/// error decoding them, so one malformed row does not fail the scan. +pub(crate) fn orphan_scan_rows( + conn: &Connection, + next: i64, + through: i64, + limit: i64, +) -> Result> { + conn.prepare( + "SELECT la.rowid, la.local_artifact_id, la.local_path, + EXISTS (SELECT 1 FROM sandboxes s WHERE s.id = la.sandbox_id) + FROM local_artifacts la + WHERE la.rowid >= ?1 AND la.rowid <= ?2 + ORDER BY la.rowid LIMIT ?3", + )? + .query_map(params![next, through, limit], |row| { + let decoded = (|| -> rusqlite::Result<(String, String, bool)> { + Ok((row.get(1)?, row.get(2)?, row.get(3)?)) + })(); + Ok((row.get::<_, i64>(0)?, decoded)) + })? + .collect::>>() +} + +/// Delete the artifact row if its sandbox row is gone; the number deleted. +pub(crate) fn delete_orphan_artifact_row( + conn: &Connection, + local_artifact_id: &str, +) -> Result { + conn.execute( + "DELETE FROM local_artifacts + WHERE local_artifact_id = ?1 + AND NOT EXISTS ( + SELECT 1 + FROM sandboxes s + WHERE s.id = local_artifacts.sandbox_id + )", + params![local_artifact_id], + ) +} diff --git a/crates/libs/sbx/mgr/server/src/db/local_artifacts.rs b/crates/libs/sbx/mgr/server/src/db/local_artifacts.rs index c84e446f5..1b364915c 100644 --- a/crates/libs/sbx/mgr/server/src/db/local_artifacts.rs +++ b/crates/libs/sbx/mgr/server/src/db/local_artifacts.rs @@ -465,7 +465,7 @@ pub fn publish_stopped_artifact( } // Clear the runtime first so the retention trigger observes its complete // identity, then close the exact attempt in the same transaction. - crate::route_activation_claims::close_claim(&tx, sandbox_id)?; + route_activation_claims::close_claim(&tx, sandbox_id)?; tx.commit()?; Ok(()) } diff --git a/crates/libs/sbx/mgr/server/src/db/pool.rs b/crates/libs/sbx/mgr/server/src/db/pool.rs index bc77641f7..7213c4ed5 100644 --- a/crates/libs/sbx/mgr/server/src/db/pool.rs +++ b/crates/libs/sbx/mgr/server/src/db/pool.rs @@ -132,7 +132,7 @@ fn restore_pool_donor( conn: &Connection, sandbox_id: SandboxId, donor: &SandboxRow, -) -> Result> { +) -> Result> { let id = sandbox_id.to_string(); set_pool_template(conn, &id, donor.pool_template.as_deref())?; update_user(conn, &id, &donor.user)?; @@ -189,16 +189,12 @@ pub fn restore_pool_donor_if_adopted( let receipt_supersession = match restore_pool_donor(&tx, sandbox_id, donor) { Ok(receipt_supersession) => receipt_supersession, Err(error) - if crate::auto_lifecycle_delete_receipts::db::is_pending_receipt_execution_fence( - &error, - ) => + if auto_lifecycle_delete_receipts::is_pending_receipt_execution_fence(&error) => { tx.rollback()?; return Ok(PoolDonorRestoreDisposition::ReceiptOwned); } - Err(error) - if crate::auto_lifecycle_delete_receipts::db::is_lifecycle_execution_fence(&error) => - { + Err(error) if auto_lifecycle_delete_receipts::is_lifecycle_execution_fence(&error) => { tx.rollback()?; return Ok(PoolDonorRestoreDisposition::LifecycleOwned); } diff --git a/crates/libs/sbx/mgr/server/src/db/quarantine.rs b/crates/libs/sbx/mgr/server/src/db/quarantine.rs index b4c360a2c..497e20b62 100644 --- a/crates/libs/sbx/mgr/server/src/db/quarantine.rs +++ b/crates/libs/sbx/mgr/server/src/db/quarantine.rs @@ -849,7 +849,7 @@ fn mark_quarantined_inner( ); } if changed != 0 { - crate::route_activation_claims::close_claim(&tx, id)?; + route_activation_claims::close_claim(&tx, id)?; let episode = QuarantineEpisode { sandbox_id: id, entered_from_state: previous_state.as_deref(), diff --git a/crates/libs/sbx/mgr/server/src/db/route_activation_claims.rs b/crates/libs/sbx/mgr/server/src/db/route_activation_claims.rs new file mode 100644 index 000000000..a220902b4 --- /dev/null +++ b/crates/libs/sbx/mgr/server/src/db/route_activation_claims.rs @@ -0,0 +1,142 @@ +//! Persistence of open Kubernetes route activation claims +//! (`sandbox_open_route_activations`). + +use pplx_space_common_core::types::{ + ExecutionGeneration, NodeId, NodeManagedBy, PodUid, RouteGeneration, + SandboxActivationAttemptId, SpaceApiRouteIdentity, +}; +use pplx_space_common_svc::sql::sqlite::SqliteU64; +use rusqlite::{Connection, params}; + +use crate::auto_lifecycle_delete_receipts::RouteIdentityProvenance; + +#[derive(Clone, Debug)] +/// Durable idempotency record for one exact Kubernetes activation. +/// +/// This is not an exclusive lease: the matching pod and attempt may continue +/// after a manager restart. A later activation is fenced by the retained +/// generation recorded when that activation actually begins. +pub(crate) struct OpenRouteActivationClaim { + route_generation: RouteGeneration, + node_id: NodeId, + execution_generation: ExecutionGeneration, + pod_uid: PodUid, + activation_attempt_id: SandboxActivationAttemptId, +} + +impl OpenRouteActivationClaim { + pub(crate) fn for_cluster_fenced_identity( + identity: &SpaceApiRouteIdentity, + provenance: RouteIdentityProvenance, + ) -> Option { + if provenance != RouteIdentityProvenance::ClusterFenced + || identity.substrate_kind != Some(NodeManagedBy::Kubernetes) + { + return None; + } + Some(Self { + route_generation: identity.route_generation?, + node_id: identity.current_node_id?, + execution_generation: identity.execution_generation?, + pod_uid: identity.pod_uid.clone()?, + activation_attempt_id: identity.activation_attempt_id?, + }) + } + + pub(crate) fn requested_generation(&self) -> u64 { + // ExecutionGeneration admits only positive i64 values. + self.route_generation + .get() + .min(self.execution_generation.get().unsigned_abs()) + } +} + +fn write_claim( + conn: &Connection, + sandbox_id: &str, + claim: &OpenRouteActivationClaim, +) -> rusqlite::Result<()> { + conn.execute( + "INSERT INTO sandbox_open_route_activations ( + sandbox_id, route_generation, node_id, execution_generation, + pod_uid, substrate_kind, activation_attempt_id, + identity_provenance + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) + ON CONFLICT(sandbox_id) DO UPDATE SET + route_generation = excluded.route_generation, + node_id = excluded.node_id, + execution_generation = excluded.execution_generation, + pod_uid = excluded.pod_uid, + substrate_kind = excluded.substrate_kind, + activation_attempt_id = excluded.activation_attempt_id, + identity_provenance = excluded.identity_provenance", + params![ + sandbox_id, + SqliteU64(claim.route_generation.get()), + claim.node_id.to_string(), + claim.execution_generation.get(), + claim.pod_uid.as_str(), + NodeManagedBy::Kubernetes.to_string(), + claim.activation_attempt_id.to_string(), + RouteIdentityProvenance::ClusterFenced.as_str(), + ], + )?; + Ok(()) +} + +pub(crate) fn insert_create_claim( + conn: &Connection, + sandbox_id: &str, + claim: &OpenRouteActivationClaim, +) -> rusqlite::Result<()> { + write_claim(conn, sandbox_id, claim) +} + +pub(crate) fn claim_matches( + conn: &Connection, + sandbox_id: &str, + claim: &OpenRouteActivationClaim, +) -> rusqlite::Result { + conn.query_row( + "SELECT EXISTS ( + SELECT 1 + FROM sandbox_open_route_activations + WHERE sandbox_id = ?1 + AND route_generation = ?2 + AND node_id = ?3 + AND execution_generation = ?4 + AND pod_uid = ?5 + AND substrate_kind = 'k8s' + AND activation_attempt_id = ?6 + AND identity_provenance = ?7 + )", + params![ + sandbox_id, + SqliteU64(claim.route_generation.get()), + claim.node_id.to_string(), + claim.execution_generation.get(), + claim.pod_uid.as_str(), + claim.activation_attempt_id.to_string(), + RouteIdentityProvenance::ClusterFenced.as_str(), + ], + |row| row.get(0), + ) +} + +pub(crate) fn close_claim(conn: &Connection, sandbox_id: &str) -> rusqlite::Result { + Ok(conn.execute( + "DELETE FROM sandbox_open_route_activations WHERE sandbox_id = ?1", + [sandbox_id], + )? == 1) +} + +#[cfg(test)] +pub(crate) fn claim_exists(conn: &Connection, sandbox_id: &str) -> rusqlite::Result { + conn.query_row( + "SELECT EXISTS ( + SELECT 1 FROM sandbox_open_route_activations WHERE sandbox_id = ?1 + )", + [sandbox_id], + |row| row.get(0), + ) +} diff --git a/crates/libs/sbx/mgr/server/src/db/sandboxes.rs b/crates/libs/sbx/mgr/server/src/db/sandboxes.rs index bea736b19..8538c7d4b 100644 --- a/crates/libs/sbx/mgr/server/src/db/sandboxes.rs +++ b/crates/libs/sbx/mgr/server/src/db/sandboxes.rs @@ -757,7 +757,7 @@ pub(crate) fn update_space_api_route_identity_in_transaction( sandbox_id: SandboxId, identity: &SpaceApiRouteIdentity, provenance: crate::auto_lifecycle_delete_receipts::RouteIdentityProvenance, -) -> Result> { +) -> Result> { let sandbox_id_string = sandbox_id.to_string(); let previous_generation = runtime_execution_generation(conn, &sandbox_id_string)?; let updated = conn.execute( @@ -800,13 +800,11 @@ pub(crate) fn update_space_api_route_identity_in_transaction( ], )?; if updated != 1 { - return Err( - crate::auto_lifecycle_delete_receipts::db::execution_write_miss_error( - conn, - sandbox_id, - identity.execution_generation, - ), - ); + return Err(auto_lifecycle_delete_receipts::execution_write_miss_error( + conn, + sandbox_id, + identity.execution_generation, + )); } reset_access_floor_for_execution_change( conn, @@ -817,15 +815,13 @@ pub(crate) fn update_space_api_route_identity_in_transaction( if let Some(evidence) = crate::auto_lifecycle_delete_receipts::RouteEvidence::from_identity(identity, provenance) { - crate::auto_lifecycle_delete_receipts::db::record_retained_route_evidence( - conn, sandbox_id, evidence, - )?; + auto_lifecycle_delete_receipts::record_retained_route_evidence(conn, sandbox_id, evidence)?; } // Provenance controls whether route evidence may mint a receipt. Once this // transaction persists a newer execution generation, an older receipt no // longer describes the row, including when the new route is incomplete. let receipt_supersession = - crate::auto_lifecycle_delete_receipts::db::discard_pending_receipt_for_execution_advance( + auto_lifecycle_delete_receipts::discard_pending_receipt_for_execution_advance( conn, sandbox_id, identity.execution_generation, @@ -954,13 +950,11 @@ fn update_runtime_fields_with_activation_owner( ], )?; if updated != 1 { - return Err( - crate::auto_lifecycle_delete_receipts::db::execution_write_miss_error( - &tx, - sandbox_id, - space_api_route_execution_generation, - ), - ); + return Err(auto_lifecycle_delete_receipts::execution_write_miss_error( + &tx, + sandbox_id, + space_api_route_execution_generation, + )); } reset_access_floor_for_execution_change(&tx, &id, previous_generation, execution_generation)?; if let (Some(execution_generation), Some(activation_attempt_id), Some(substrate)) = ( @@ -968,7 +962,7 @@ fn update_runtime_fields_with_activation_owner( space_api_route_activation_attempt_id, space_api_route_substrate_kind, ) { - crate::auto_lifecycle_delete_receipts::db::record_retained_route_evidence( + auto_lifecycle_delete_receipts::record_retained_route_evidence( &tx, sandbox_id, crate::auto_lifecycle_delete_receipts::RouteEvidence { @@ -980,7 +974,7 @@ fn update_runtime_fields_with_activation_owner( )?; } let receipt_supersession = - crate::auto_lifecycle_delete_receipts::db::discard_pending_receipt_for_execution_advance( + auto_lifecycle_delete_receipts::discard_pending_receipt_for_execution_advance( &tx, sandbox_id, space_api_route_execution_generation, @@ -1408,7 +1402,7 @@ pub fn delete_sandbox(conn: &Connection, id: &str) -> Result<()> { let tx = immediate_transaction(conn)?; delete_sandbox_child_rows(&tx, id)?; tx.execute("DELETE FROM sandboxes WHERE id = ?1", params![id]) - .map_err(crate::auto_lifecycle_delete_receipts::db::map_pending_receipt_delete_error)?; + .map_err(auto_lifecycle_delete_receipts::map_pending_receipt_delete_error)?; tx.commit()?; Ok(()) } @@ -1506,7 +1500,7 @@ pub(crate) fn delete_sandbox_if_inventory_witness_matches( delete_sandbox_child_rows(&tx, &observed.id)?; let deleted = tx .execute("DELETE FROM sandboxes WHERE id = ?1", params![&observed.id]) - .map_err(crate::auto_lifecycle_delete_receipts::db::map_pending_receipt_delete_error)?; + .map_err(auto_lifecycle_delete_receipts::map_pending_receipt_delete_error)?; if deleted != 1 { return Ok(InventoryStaleRowMutation::WitnessChanged); } diff --git a/crates/libs/sbx/mgr/server/src/db/snapshot_upload_jobs.rs b/crates/libs/sbx/mgr/server/src/db/snapshot_upload_jobs.rs new file mode 100644 index 000000000..7b4c8c33f --- /dev/null +++ b/crates/libs/sbx/mgr/server/src/db/snapshot_upload_jobs.rs @@ -0,0 +1,87 @@ +//! `jobs` rows of the snapshot-upload pipeline, and the backup markers its +//! completed jobs prove. + +use pplx_space_common_core::types::JobId; +use rusqlite::{Connection, Result, params}; + +/// The `jobs.kind` of the snapshot-upload pipeline's jobs. +pub const SNAPSHOT_UPLOAD_JOB_KIND: &str = "snapshot_upload"; + +/// Give back the attempt a restart interrupted: every running job of `kind` +/// with attempts left loses one; the number refunded. +pub(crate) fn refund_interrupted_running_attempts(conn: &Connection, kind: &str) -> Result { + conn.execute( + "UPDATE jobs + SET attempts = attempts - 1 + WHERE kind = ?1 AND status = 'running' AND attempts > 0", + params![kind], + ) +} + +/// Replace a pending or running job's payload; the number of rows changed. +pub(crate) fn replace_active_job_payload( + conn: &Connection, + payload: &str, + job_id: &JobId, + kind: &str, +) -> Result { + conn.execute( + "UPDATE jobs SET payload = ?1 + WHERE job_id = ?2 AND kind = ?3 AND status IN ('pending', 'running')", + params![payload, job_id, kind], + ) +} + +/// The highest job rowid, `None` for an empty table. +pub(crate) fn max_job_rowid(conn: &Connection) -> Result> { + conn.query_row("SELECT MAX(rowid) FROM jobs", [], |r| r.get(0)) +} + +/// Visit up to `limit` completed snapshot-upload jobs with rowids in +/// `(after, through]`, in rowid order, with the scan's timestamp, each rowid +/// and its payload or the error reading it. The timestamp is taken once the +/// statement is bound, before the first row is read. The statement stays open +/// while `visit` runs, so `visit` may write through the same connection; its +/// error stops the scan. +pub(crate) fn visit_completed_upload_payloads( + conn: &Connection, + after: i64, + through: i64, + limit: i64, + mut visit: impl FnMut(&str, i64, Result) -> Result<()>, +) -> Result<()> { + let mut statement = conn.prepare( + "SELECT rowid, payload FROM jobs + WHERE kind = 'snapshot_upload' AND status = 'completed' + AND rowid > ?1 AND rowid <= ?2 ORDER BY rowid LIMIT ?3", + )?; + let mut rows = statement.query(params![after, through, limit])?; + let now = crate::db::now()?; + while let Some(row) = rows.next()? { + visit(&now, row.get(0)?, row.get::<_, String>(1))?; + } + Ok(()) +} + +/// Stamp `backup_completed_at` on an artifact that has none; the number of +/// rows changed. +pub(crate) fn stamp_backup_completed_if_unset( + conn: &Connection, + now: &str, + local_artifact_id: &str, +) -> Result { + conn.execute( + "UPDATE local_artifacts SET backup_completed_at = ?1 + WHERE local_artifact_id = ?2 AND backup_completed_at IS NULL", + params![now, local_artifact_id], + ) +} + +/// Whether an artifact row exists. +pub(crate) fn local_artifact_exists(conn: &Connection, local_artifact_id: &str) -> Result { + conn.query_row( + "SELECT EXISTS(SELECT 1 FROM local_artifacts WHERE local_artifact_id = ?1)", + params![local_artifact_id], + |r| r.get(0), + ) +} diff --git a/crates/libs/sbx/mgr/server/src/db/startup_recovery.rs b/crates/libs/sbx/mgr/server/src/db/startup_recovery.rs index e4f4ef8f0..132c9682b 100644 --- a/crates/libs/sbx/mgr/server/src/db/startup_recovery.rs +++ b/crates/libs/sbx/mgr/server/src/db/startup_recovery.rs @@ -125,7 +125,7 @@ pub(crate) fn settle_startup_resident( message, } = settlement { - crate::route_activation_claims::close_claim(&tx, &observed.id)?; + route_activation_claims::close_claim(&tx, &observed.id)?; insert_new_quarantine_episode( &tx, &QuarantineEpisode { diff --git a/crates/libs/sbx/mgr/server/src/db/state_transitions.rs b/crates/libs/sbx/mgr/server/src/db/state_transitions.rs index d373b145b..7ab22008a 100644 --- a/crates/libs/sbx/mgr/server/src/db/state_transitions.rs +++ b/crates/libs/sbx/mgr/server/src/db/state_transitions.rs @@ -229,7 +229,7 @@ pub(crate) struct ActivationRouteAdmission<'a> { pub(crate) auto_lifecycle: Option<&'a AutoLifecyclePolicy>, pub(crate) operation_activation_attempt_id: Option, pub(crate) evidence: Option, - pub(crate) open_claim: Option<&'a crate::route_activation_claims::OpenRouteActivationClaim>, + pub(crate) open_claim: Option<&'a route_activation_claims::OpenRouteActivationClaim>, } pub(crate) fn begin_activation_with_route_evidence( @@ -322,28 +322,26 @@ pub(crate) fn begin_activation_with_route_evidence( .transpose()?; match route_admission.open_claim { Some(open_route_activation) => { - crate::route_activation_claims::insert_create_claim(&tx, id, open_route_activation)?; + route_activation_claims::insert_create_claim(&tx, id, open_route_activation)?; } None => { // Beginning any other activation supersedes an older Kubernetes // continuation ticket in the same transaction that claims the row. - crate::route_activation_claims::close_claim(&tx, id)?; + route_activation_claims::close_claim(&tx, id)?; } } if let Some(route_evidence) = route_admission.evidence { let sandbox_id = sandbox_id.ok_or(rusqlite::Error::InvalidQuery)?; - crate::auto_lifecycle_delete_receipts::db::record_retained_route_evidence( + auto_lifecycle_delete_receipts::record_retained_route_evidence( &tx, sandbox_id, route_evidence, )?; } let receipt_supersession = match sandbox_id { - Some(sandbox_id) => crate::auto_lifecycle_delete_receipts::db::discard_pending_receipt_for_execution_advance( - &tx, - sandbox_id, - execution_generation, - )?, + Some(sandbox_id) => { + discard_pending_receipt_for_execution_advance(&tx, sandbox_id, execution_generation)? + } None => None, }; let cleanup_id = SandboxCleanupEpisodeId::new(); @@ -659,7 +657,7 @@ pub fn mark_start_succeeded( arm_auto_lifecycle_after_activation(&tx, id)?; } if changed == 1 { - crate::route_activation_claims::close_claim(&tx, id)?; + route_activation_claims::close_claim(&tx, id)?; } tx.commit()?; Ok(changed == 1) @@ -792,7 +790,7 @@ pub fn mark_restore_succeeded( arm_auto_lifecycle_after_activation(&tx, id)?; } if changed == 1 { - crate::route_activation_claims::close_claim(&tx, id)?; + route_activation_claims::close_claim(&tx, id)?; } tx.commit()?; if changed == 1 { diff --git a/crates/libs/sbx/mgr/server/src/local_artifacts.rs b/crates/libs/sbx/mgr/server/src/local_artifacts.rs index 48ceb2efc..db5970302 100644 --- a/crates/libs/sbx/mgr/server/src/local_artifacts.rs +++ b/crates/libs/sbx/mgr/server/src/local_artifacts.rs @@ -61,82 +61,13 @@ use std::path::Path; use std::sync::{Arc, Mutex, MutexGuard}; -use pplx_space_common_core::sql; use pplx_space_common_core::types::{ArtifactKind, LocalArtifactId, SnapshotName}; -use pplx_space_common_svc::sql::sqlite::SqliteU64; -use rusqlite::{Connection, OptionalExtension, params}; +use rusqlite::Connection; use crate::db::LocalArtifactRow; - -/// One sandbox's current lifecycle artifact that this node cannot prove has -/// a durable remote copy. -/// -/// Identity comes from the `sandboxes` pin rather than from the artifact -/// row, because the case where there *is* no artifact row is one of the -/// cases this reports. Migration 026's CHECK keeps `current_local_artifact_id` -/// and `current_local_artifact_kind` set or unset together, so the pin alone -/// always carries a whole identity. -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct PendingBackup { - pub sandbox_id: String, - pub local_artifact_id: LocalArtifactId, - pub artifact_kind: ArtifactKind, -} - -/// Logical state of a `local_artifacts` row. Encoded on the row's -/// `state` column; transitions are CAS via -/// [`LocalArtifacts::check_out_for_restore`] / -/// [`LocalArtifacts::check_out_for_delete`]. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ArtifactState { - /// Published and available. The only state from which a checkout - /// (restore or delete) can succeed. - Ready, - /// A restore is currently reading this artifact's `local_path`. - /// Reapers must skip; deleters must wait or fail with NotReady. - Restoring, - /// A delete has reserved this row. The on-disk directory has - /// usually been renamed to a `.gc-pending-` sibling. Readers - /// should treat the artifact as gone. - Deleting, -} - -impl ArtifactState { - pub fn as_str(self) -> &'static str { - match self { - ArtifactState::Ready => "ready", - ArtifactState::Restoring => "restoring", - ArtifactState::Deleting => "deleting", - } - } -} - -/// Error parsing an [`ArtifactState`] from its wire/DB string. The -/// CHECK constraint on `local_artifacts.state` should make this -/// unreachable for rows the schema accepted; it surfaces table -/// corruption or migration drift. -#[derive(Debug, thiserror::Error)] -#[error("unrecognized artifact state {0:?}")] -pub struct ParseArtifactStateError(String); - -impl std::str::FromStr for ArtifactState { - type Err = ParseArtifactStateError; - - fn from_str(s: &str) -> Result { - match s { - "ready" => Ok(ArtifactState::Ready), - "restoring" => Ok(ArtifactState::Restoring), - "deleting" => Ok(ArtifactState::Deleting), - _ => Err(ParseArtifactStateError(s.to_string())), - } - } -} - -impl std::fmt::Display for ArtifactState { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(self.as_str()) - } -} +use crate::db::local_artifact_leases as leases; +pub(crate) use crate::db::local_artifact_leases::NamedArtifactPublication; +pub use crate::db::local_artifact_leases::{ArtifactState, ParseArtifactStateError, PendingBackup}; /// Reason a checkout (restore or delete) failed. #[derive(Debug, thiserror::Error)] @@ -208,14 +139,6 @@ pub struct LocalArtifacts { orphan_scan: Arc>, } -/// Point-in-time publication evidence for a named snapshot, not a checkout lease. -pub(crate) struct NamedArtifactPublication { - pub snapshot_name: SnapshotName, - pub local_path: String, - pub artifact_kind: ArtifactKind, - pub state: ArtifactState, -} - #[derive(Clone, Copy)] struct OrphanScan { next: i64, @@ -371,36 +294,7 @@ impl LocalArtifacts { /// with no report, which every consumer already has to handle. pub fn list_pending_backups(&self) -> rusqlite::Result> { let conn = self.db_conn()?; - conn.prepare( - "SELECT s.id, s.current_local_artifact_id, s.current_local_artifact_kind - FROM sandboxes AS s - LEFT JOIN local_artifacts AS a - ON a.local_artifact_id = s.current_local_artifact_id - AND a.sandbox_id = s.id - WHERE s.current_local_artifact_id IS NOT NULL - AND (a.local_artifact_id IS NULL OR a.backup_completed_at IS NULL) - ORDER BY s.id", - )? - .query_map([], |row| { - Ok(PendingBackup { - sandbox_id: row.get(0)?, - local_artifact_id: row.get::<_, String>(1)?.parse().map_err(|error| { - rusqlite::Error::FromSqlConversionFailure( - 1, - rusqlite::types::Type::Text, - Box::new(error), - ) - })?, - artifact_kind: row.get::<_, String>(2)?.parse().map_err(|error| { - rusqlite::Error::FromSqlConversionFailure( - 2, - rusqlite::types::Type::Text, - Box::new(error), - ) - })?, - }) - })? - .collect() + leases::list_pending_backups(&conn) } /// Inventory-only observation. Callers must still acquire a checkout before @@ -409,43 +303,8 @@ impl LocalArtifacts { &self, sandbox_id: &str, ) -> Result, crate::db::inventory_read::InventoryReadError> { - use pplx_space_common_core::inventory::{InventoryBudget, MAX_ITEMS, MAX_RAW_BYTES}; let conn = self.db_conn()?; - let mut statement = conn.prepare(sql!( - "SELECT snapshot_name, local_path, artifact_kind, state - FROM local_artifacts WHERE sandbox_id = ?1 AND artifact_origin = 'snapshot'" - ))?; - let mut rows = statement.query([sandbox_id])?; - let mut budget = InventoryBudget::new(MAX_ITEMS, MAX_RAW_BYTES); - let mut publications = Vec::new(); - while let Some(row) = rows.next()? { - crate::db::inventory_read::check_row(row, &mut budget)?; - publications.push(NamedArtifactPublication { - snapshot_name: row.get::<_, String>(0)?.parse().map_err(|error| { - rusqlite::Error::FromSqlConversionFailure( - 0, - rusqlite::types::Type::Text, - Box::new(error), - ) - })?, - local_path: row.get(1)?, - artifact_kind: row.get::<_, String>(2)?.parse().map_err(|error| { - rusqlite::Error::FromSqlConversionFailure( - 2, - rusqlite::types::Type::Text, - Box::new(error), - ) - })?, - state: row.get::<_, String>(3)?.parse().map_err(|error| { - rusqlite::Error::FromSqlConversionFailure( - 3, - rusqlite::types::Type::Text, - Box::new(error), - ) - })?, - }); - } - Ok(publications) + leases::list_named_publications(&conn, sandbox_id) } /// Mark the artifact's remote backup as durably completed. This @@ -551,13 +410,7 @@ impl LocalArtifacts { let tx = rusqlite::Transaction::new_unchecked(&conn, rusqlite::TransactionBehavior::Immediate)?; let id_str = local_artifact_id.to_string(); - let row: Option<(String, String)> = tx - .query_row( - "SELECT local_path, state FROM local_artifacts WHERE local_artifact_id = ?1", - params![id_str], - |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)), - ) - .optional()?; + let row = leases::artifact_path_and_state(&tx, &id_str)?; let Some((local_path, state)) = row else { return Err(CheckoutError::NotFound(local_artifact_id)); }; @@ -576,11 +429,7 @@ impl LocalArtifacts { // CAS — only flip if still ready. If a concurrent path won // the race between the SELECT and the UPDATE, the UPDATE // affects 0 rows and we surface NotReady. - let changed = tx.execute( - "UPDATE local_artifacts SET state = 'restoring' - WHERE local_artifact_id = ?1 AND state = 'ready'", - params![id_str], - )?; + let changed = leases::mark_artifact_restoring(&tx, &id_str)?; if changed == 0 { return Err(CheckoutError::NotReady { id: local_artifact_id, @@ -631,32 +480,10 @@ impl LocalArtifacts { sandbox_id: &str, eligibility: GcEligibility, ) -> Result { - use crate::db::timestamps::EpochSeconds; - let conn = self.db_conn()?; let tx = rusqlite::Transaction::new_unchecked(&conn, rusqlite::TransactionBehavior::Immediate)?; - let row = tx - .query_row( - "SELECT created_at, COALESCE(last_touched_at, created_at), - EXISTS( - SELECT 1 FROM sandboxes - WHERE current_local_artifact_id = ?1 - OR suspend_predecessor_artifact_id = ?1 - ), - backup_completed_at IS NOT NULL - FROM local_artifacts WHERE local_artifact_id = ?1 AND sandbox_id = ?2", - params![local_artifact_id.to_string(), sandbox_id], - |row| { - Ok(( - row.get::<_, EpochSeconds>(0)?, - row.get::<_, EpochSeconds>(1)?, - row.get::<_, bool>(2)?, - row.get::<_, bool>(3)?, - )) - }, - ) - .optional()?; + let row = leases::gc_checkout_evidence(&tx, &local_artifact_id.to_string(), sandbox_id)?; let Some((created_at, touched_at, pinned, backed_up)) = row else { return Err(CheckoutError::NotFound(local_artifact_id)); }; @@ -694,24 +521,11 @@ impl LocalArtifacts { let id_str = local_artifact_id.to_string(); // The IMMEDIATE transaction serializes pin changes with checkout for // every deleter, including named-snapshot deletion outside GC. - let pinned: bool = tx.query_row( - sql!( - "SELECT EXISTS(SELECT 1 FROM sandboxes - WHERE current_local_artifact_id = ?1 OR suspend_predecessor_artifact_id = ?1)" - ), - [&id_str], - |row| row.get(0), - )?; + let pinned = leases::artifact_is_pinned(tx, &id_str)?; if pinned { return Err(CheckoutError::Pinned(local_artifact_id)); } - let row: Option<(String, String)> = tx - .query_row( - "SELECT local_path, state FROM local_artifacts WHERE local_artifact_id = ?1", - params![id_str], - |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)), - ) - .optional()?; + let row = leases::artifact_path_and_state(tx, &id_str)?; let Some((local_path, state)) = row else { return Err(CheckoutError::NotFound(local_artifact_id)); }; @@ -729,22 +543,14 @@ impl LocalArtifacts { } // Shared-path safety gate. The reaper's old plan-time // check moved here so DeleteSnapshots gets it too. - let path_count: u64 = tx.query_row( - "SELECT COUNT(*) FROM local_artifacts WHERE local_path = ?1", - params![local_path], - |r| Ok(r.get::<_, SqliteU64>(0)?.0), - )?; + let path_count = leases::count_artifacts_at_path(tx, &local_path)?; if path_count != 1 { return Err(CheckoutError::SharedLocalPath { id: local_artifact_id, sibling_count: path_count.saturating_sub(1), }); } - let changed = tx.execute( - "UPDATE local_artifacts SET state = 'deleting' - WHERE local_artifact_id = ?1 AND state = 'ready'", - params![id_str], - )?; + let changed = leases::mark_artifact_deleting(tx, &id_str)?; if changed == 0 { return Err(CheckoutError::NotReady { id: local_artifact_id, @@ -814,17 +620,8 @@ impl LocalArtifacts { let conn = self.db_conn()?; let tx = rusqlite::Transaction::new_unchecked(&conn, rusqlite::TransactionBehavior::Immediate)?; - let restoring_reset = tx.execute( - "UPDATE local_artifacts SET state = 'ready' WHERE state = 'restoring'", - [], - )?; - let stuck_deleting: Vec<(String, String)> = tx - .prepare( - "SELECT local_artifact_id, local_path FROM local_artifacts - WHERE state = 'deleting'", - )? - .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, String>(1)?)))? - .collect::>()?; + let restoring_reset = leases::reset_restoring_artifacts(&tx)?; + let stuck_deleting = leases::list_deleting_artifacts(&tx)?; let mut deleting_rolled_back = 0usize; let mut deleting_finished = 0usize; let mut deleting_deferred = 0usize; @@ -842,32 +639,16 @@ impl LocalArtifacts { continue; } }; - let pinned: bool = tx.query_row( - "SELECT EXISTS( - SELECT 1 FROM sandboxes - WHERE current_local_artifact_id = ?1 - OR suspend_predecessor_artifact_id = ?1 - )", - params![id], - |r| r.get(0), - )?; + let pinned = leases::artifact_is_pinned(&tx, &id)?; if original_present && !staging_present { - tx.execute( - "UPDATE local_artifacts SET state = 'ready' - WHERE local_artifact_id = ?1 AND state = 'deleting'", - params![id], - )?; + leases::roll_back_deleting_artifact(&tx, &id)?; deleting_rolled_back += 1; } else if pinned { tracing::warn!(artifact_id = %id, original_present, staging_present, "retaining pinned deleting artifact evidence; explicit restore or cleanup required"); deleting_deferred += 1; } else { - tx.execute( - "DELETE FROM local_artifacts - WHERE local_artifact_id = ?1 AND state = 'deleting'", - params![id], - )?; + leases::delete_deleting_artifact(&tx, &id)?; deleting_finished += 1; } } @@ -883,15 +664,8 @@ impl LocalArtifacts { /// GC staging is disposable only after both its registry and pin are gone. /// Retain rows in every state, including historical deletion conflicts. pub(crate) fn gc_staging_is_unowned(&self, id: LocalArtifactId) -> rusqlite::Result { - self.db_conn()?.query_row( - sql!( - "SELECT NOT EXISTS(SELECT 1 FROM local_artifacts WHERE local_artifact_id = ?1) - AND NOT EXISTS(SELECT 1 FROM sandboxes - WHERE current_local_artifact_id = ?1 OR suspend_predecessor_artifact_id = ?1)" - ), - [id.to_string()], - |row| row.get(0), - ) + let conn = self.db_conn()?; + leases::gc_staging_is_unowned(&conn, &id.to_string()) } /// Examine at most `max_records` raw rows, advancing past owned, present, @@ -911,34 +685,17 @@ impl LocalArtifacts { let tx = rusqlite::Transaction::new_unchecked(&conn, rusqlite::TransactionBehavior::Immediate)?; if scan.through.is_none() { - scan.through = - tx.query_row("SELECT MAX(rowid) FROM local_artifacts", [], |r| r.get(0))?; + scan.through = leases::max_artifact_rowid(&tx)?; } let Some(through) = scan.through else { return Ok(0); }; - let rows = tx - .prepare( - "SELECT la.rowid, la.local_artifact_id, la.local_path, - EXISTS (SELECT 1 FROM sandboxes s WHERE s.id = la.sandbox_id) - FROM local_artifacts la - WHERE la.rowid >= ?1 AND la.rowid <= ?2 - ORDER BY la.rowid LIMIT ?3", - )? - .query_map( - params![ - scan.next, - through, - i64::try_from(max_records).unwrap_or(i64::MAX) - ], - |row| { - let decoded = (|| -> rusqlite::Result<(String, String, bool)> { - Ok((row.get(1)?, row.get(2)?, row.get(3)?)) - })(); - Ok((row.get::<_, i64>(0)?, decoded)) - }, - )? - .collect::>>()?; + let rows = leases::orphan_scan_rows( + &tx, + scan.next, + through, + i64::try_from(max_records).unwrap_or(i64::MAX), + )?; let mut deleted = 0usize; let examined = rows.len(); for (rowid, decoded) in rows { @@ -964,16 +721,7 @@ impl LocalArtifacts { continue; } } - deleted += tx.execute( - "DELETE FROM local_artifacts - WHERE local_artifact_id = ?1 - AND NOT EXISTS ( - SELECT 1 - FROM sandboxes s - WHERE s.id = local_artifacts.sandbox_id - )", - params![id], - )?; + deleted += leases::delete_orphan_artifact_row(&tx, &id)?; } if examined < max_records { scan = OrphanScan::default(); @@ -1055,12 +803,7 @@ impl RestoreLease { fn release_with_touch(&self) -> rusqlite::Result<()> { let conn = lock_db(&self.db)?; - conn.execute( - "UPDATE local_artifacts - SET state = 'ready', last_touched_at = ?1 - WHERE local_artifact_id = ?2 AND state = 'restoring'", - params![crate::db::now()?, self.id.to_string()], - )?; + leases::release_restoring_artifact_with_touch(&conn, &self.id.to_string())?; Ok(()) } @@ -1071,11 +814,7 @@ impl RestoreLease { /// idle clock should reflect that. fn release_without_touch(&self) -> rusqlite::Result<()> { let conn = lock_db(&self.db)?; - conn.execute( - "UPDATE local_artifacts SET state = 'ready' - WHERE local_artifact_id = ?1 AND state = 'restoring'", - params![self.id.to_string()], - )?; + leases::release_restoring_artifact(&conn, &self.id.to_string())?; Ok(()) } } @@ -1121,11 +860,7 @@ impl DeleteLease { /// than a foot-gun. pub fn commit(mut self) -> rusqlite::Result<()> { let conn = lock_db(&self.db)?; - let deleted = conn.execute( - "DELETE FROM local_artifacts - WHERE local_artifact_id = ?1 AND state = 'deleting'", - params![self.id.to_string()], - )?; + let deleted = leases::delete_deleting_artifact(&conn, &self.id.to_string())?; drop(conn); if deleted == 0 { tracing::warn!( @@ -1154,11 +889,7 @@ impl Drop for DeleteLease { return; } }; - if let Err(error) = conn.execute( - "UPDATE local_artifacts SET state = 'ready' - WHERE local_artifact_id = ?1 AND state = 'deleting'", - params![self.id.to_string()], - ) { + if let Err(error) = leases::roll_back_deleting_artifact(&conn, &self.id.to_string()) { tracing::warn!( local_artifact_id = %self.id, %error, diff --git a/crates/libs/sbx/mgr/server/src/local_artifacts/refusal_tests.rs b/crates/libs/sbx/mgr/server/src/local_artifacts/refusal_tests.rs index 38a4b00f0..8d9e76790 100644 --- a/crates/libs/sbx/mgr/server/src/local_artifacts/refusal_tests.rs +++ b/crates/libs/sbx/mgr/server/src/local_artifacts/refusal_tests.rs @@ -3,6 +3,8 @@ use std::sync::PoisonError; +use rusqlite::{OptionalExtension, params}; + use super::*; const ID: &str = "01939e3e-aaaa-7000-8000-000000000001"; diff --git a/crates/libs/sbx/mgr/server/src/local_artifacts/tests.rs b/crates/libs/sbx/mgr/server/src/local_artifacts/tests.rs index 7780fddca..3dcac7287 100644 --- a/crates/libs/sbx/mgr/server/src/local_artifacts/tests.rs +++ b/crates/libs/sbx/mgr/server/src/local_artifacts/tests.rs @@ -1,4 +1,5 @@ use pplx_space_common_core::types::SandboxId; +use rusqlite::params; use super::*; diff --git a/crates/libs/sbx/mgr/server/src/route_activation_claims.rs b/crates/libs/sbx/mgr/server/src/route_activation_claims.rs index cd084ef45..6c49856c5 100644 --- a/crates/libs/sbx/mgr/server/src/route_activation_claims.rs +++ b/crates/libs/sbx/mgr/server/src/route_activation_claims.rs @@ -1,52 +1,6 @@ -use pplx_space_common_core::types::{ - ExecutionGeneration, NodeId, NodeManagedBy, PodUid, RouteGeneration, - SandboxActivationAttemptId, SpaceApiRouteIdentity, -}; -use pplx_space_common_svc::sql::sqlite::SqliteU64; -use rusqlite::{Connection, params}; +use rusqlite::Connection; -use crate::auto_lifecycle_delete_receipts::RouteIdentityProvenance; - -#[derive(Clone, Debug)] -/// Durable idempotency record for one exact Kubernetes activation. -/// -/// This is not an exclusive lease: the matching pod and attempt may continue -/// after a manager restart. A later activation is fenced by the retained -/// generation recorded when that activation actually begins. -pub(crate) struct OpenRouteActivationClaim { - route_generation: RouteGeneration, - node_id: NodeId, - execution_generation: ExecutionGeneration, - pod_uid: PodUid, - activation_attempt_id: SandboxActivationAttemptId, -} - -impl OpenRouteActivationClaim { - pub(crate) fn for_cluster_fenced_identity( - identity: &SpaceApiRouteIdentity, - provenance: RouteIdentityProvenance, - ) -> Option { - if provenance != RouteIdentityProvenance::ClusterFenced - || identity.substrate_kind != Some(NodeManagedBy::Kubernetes) - { - return None; - } - Some(Self { - route_generation: identity.route_generation?, - node_id: identity.current_node_id?, - execution_generation: identity.execution_generation?, - pod_uid: identity.pod_uid.clone()?, - activation_attempt_id: identity.activation_attempt_id?, - }) - } - - fn requested_generation(&self) -> u64 { - // ExecutionGeneration admits only positive i64 values. - self.route_generation - .get() - .min(self.execution_generation.get().unsigned_abs()) - } -} +use crate::db::route_activation_claims::{OpenRouteActivationClaim, claim_matches}; #[derive(Debug, thiserror::Error)] pub(crate) enum OpenRouteActivationAdmissionError { @@ -58,47 +12,6 @@ pub(crate) enum OpenRouteActivationAdmissionError { StaleOrConflicting { retained: u64 }, } -fn write_claim( - conn: &Connection, - sandbox_id: &str, - claim: &OpenRouteActivationClaim, -) -> rusqlite::Result<()> { - conn.execute( - "INSERT INTO sandbox_open_route_activations ( - sandbox_id, route_generation, node_id, execution_generation, - pod_uid, substrate_kind, activation_attempt_id, - identity_provenance - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) - ON CONFLICT(sandbox_id) DO UPDATE SET - route_generation = excluded.route_generation, - node_id = excluded.node_id, - execution_generation = excluded.execution_generation, - pod_uid = excluded.pod_uid, - substrate_kind = excluded.substrate_kind, - activation_attempt_id = excluded.activation_attempt_id, - identity_provenance = excluded.identity_provenance", - params![ - sandbox_id, - SqliteU64(claim.route_generation.get()), - claim.node_id.to_string(), - claim.execution_generation.get(), - claim.pod_uid.as_str(), - NodeManagedBy::Kubernetes.to_string(), - claim.activation_attempt_id.to_string(), - RouteIdentityProvenance::ClusterFenced.as_str(), - ], - )?; - Ok(()) -} - -pub(crate) fn insert_create_claim( - conn: &Connection, - sandbox_id: &str, - claim: &OpenRouteActivationClaim, -) -> rusqlite::Result<()> { - write_claim(conn, sandbox_id, claim) -} - /// Highest live or retained generation observed on this node. /// /// Cluster-fenced claims reject a different attempt at the live generation; @@ -121,37 +34,6 @@ fn observed_generation_floor(provenance: crate::db::ActivationRouteProvenance) - .max() } -fn claim_matches( - conn: &Connection, - sandbox_id: &str, - claim: &OpenRouteActivationClaim, -) -> rusqlite::Result { - conn.query_row( - "SELECT EXISTS ( - SELECT 1 - FROM sandbox_open_route_activations - WHERE sandbox_id = ?1 - AND route_generation = ?2 - AND node_id = ?3 - AND execution_generation = ?4 - AND pod_uid = ?5 - AND substrate_kind = 'k8s' - AND activation_attempt_id = ?6 - AND identity_provenance = ?7 - )", - params![ - sandbox_id, - SqliteU64(claim.route_generation.get()), - claim.node_id.to_string(), - claim.execution_generation.get(), - claim.pod_uid.as_str(), - claim.activation_attempt_id.to_string(), - RouteIdentityProvenance::ClusterFenced.as_str(), - ], - |row| row.get(0), - ) -} - pub(crate) fn validate_cluster_fenced_activation( conn: &Connection, sandbox_id: &str, @@ -171,21 +53,3 @@ pub(crate) fn validate_cluster_fenced_activation( Ok(()) } - -pub(crate) fn close_claim(conn: &Connection, sandbox_id: &str) -> rusqlite::Result { - Ok(conn.execute( - "DELETE FROM sandbox_open_route_activations WHERE sandbox_id = ?1", - [sandbox_id], - )? == 1) -} - -#[cfg(test)] -pub(crate) fn claim_exists(conn: &Connection, sandbox_id: &str) -> rusqlite::Result { - conn.query_row( - "SELECT EXISTS ( - SELECT 1 FROM sandbox_open_route_activations WHERE sandbox_id = ?1 - )", - [sandbox_id], - |row| row.get(0), - ) -} diff --git a/crates/libs/sbx/mgr/server/src/sandbox_eviction.rs b/crates/libs/sbx/mgr/server/src/sandbox_eviction.rs index 92846525a..89666b80d 100644 --- a/crates/libs/sbx/mgr/server/src/sandbox_eviction.rs +++ b/crates/libs/sbx/mgr/server/src/sandbox_eviction.rs @@ -90,13 +90,13 @@ use std::collections::HashMap; use std::path::Path; use std::time::{Duration, Instant}; -use pplx_space_common_svc::sql::sqlite::SqliteU64; -use rusqlite::{Connection, OptionalExtension, params}; +use rusqlite::Connection; use tokio_util::sync::CancellationToken; use tracing::{info, warn}; use uuid::Uuid; pub(crate) use crate::auto_lifecycle::now_secs; +use crate::db::eviction_scan::{count_malformed_suspended_updated_at, count_resident_rows}; use crate::service::SandboxManagerService; /// Runtime configuration for the eviction loop. Loaded from env on @@ -517,59 +517,6 @@ impl CandidateScanState { } } -const VALID_SUSPENDED_UPDATED_AT_SQL: &str = - "s.updated_at <> '' AND s.updated_at NOT GLOB '*[^0-9]*'"; - -/// Idle witness for eviction candidacy: the sandbox's last real -/// activity, not the row's last write. `updated_at` has an unbounded -/// writer set (periodic reconcilers rewrite rows value-identically and -/// stamp it), so a row can look seconds-old for weeks while its bytes -/// sit cold — the activity clocks inside `auto_lifecycle_state_json` -/// only advance on genuine touches and transitions. NULL-safe two-way -/// max of `last_access_at`/`last_command_at`; rows carrying neither -/// clock (operator/legacy/orphan-adopted rows) fall back to -/// `updated_at`, preserving their previous behavior exactly. The -/// `json_valid` gate is load-bearing: `json_extract` aborts the whole -/// statement on malformed JSON, so without it one corrupt row would -/// fail every scan and shield the valid cold rows around it — a -/// malformed blob falls back to `updated_at` like a missing one. Each -/// clock is additionally type-gated to a non-negative integer: our -/// writers only ever store u64 epoch seconds, so anything else -/// (negative, fractional, string) is corruption and must read as -/// absent rather than fail the `SqliteU64` row decode. The CAS claim -/// deliberately keeps raw `updated_at`: "any write means the row is in -/// flux" is the right semantic there and the wrong one here. -/// -/// The partial eviction index prunes to suspended rows before evaluating -/// this expression. If scan cost grows with node row counts, the escape -/// hatch is an index on this deterministic expression or a virtual generated -/// column — not a writer-maintained copy, which would reintroduce the drift -/// this witness exists to escape. -/// Scalar `MAX` returns NULL if ANY argument is NULL, so absent or -/// corrupt clocks are mapped to a -1 sentinel first (real clocks are -/// type-gated non-negative, so the sentinel cannot collide) and -/// `NULLIF` restores NULL when both were absent. -const IDLE_WITNESS_SQL: &str = "COALESCE( - CASE - WHEN json_valid(s.auto_lifecycle_state_json) THEN - NULLIF( - MAX( - COALESCE(CASE - WHEN typeof(json_extract(s.auto_lifecycle_state_json, '$.last_access_at')) = 'integer' - AND json_extract(s.auto_lifecycle_state_json, '$.last_access_at') >= 0 - THEN json_extract(s.auto_lifecycle_state_json, '$.last_access_at') - END, -1), - COALESCE(CASE - WHEN typeof(json_extract(s.auto_lifecycle_state_json, '$.last_command_at')) = 'integer' - AND json_extract(s.auto_lifecycle_state_json, '$.last_command_at') >= 0 - THEN json_extract(s.auto_lifecycle_state_json, '$.last_command_at') - END, -1) - ), - -1) - END, - CAST(s.updated_at AS INTEGER) - )"; - fn query_candidates_after( conn: &Connection, idle_cutoff_epoch_secs: u64, @@ -580,52 +527,23 @@ fn query_candidates_after( cursor.map(|cursor| i64::try_from(cursor.idle_witness_secs).unwrap_or(i64::MAX)); let cursor_sandbox_id = cursor.map(|cursor| cursor.sandbox_id.as_str()); let cutoff = i64::try_from(idle_cutoff_epoch_secs).unwrap_or(i64::MAX); - let sql = format!( - "SELECT s.id, {IDLE_WITNESS_SQL} FROM sandboxes s - WHERE s.state = 'suspended' - AND {VALID_SUSPENDED_UPDATED_AT_SQL} - AND {IDLE_WITNESS_SQL} <= ?1 - AND ( - ?2 IS NULL - OR {IDLE_WITNESS_SQL} > ?2 - OR ({IDLE_WITNESS_SQL} = ?2 AND s.id > ?3) - ) - AND ( - s.current_local_artifact_id IS NOT NULL - OR EXISTS ( - SELECT 1 FROM jobs j - WHERE j.kind = 'snapshot_upload' - AND j.subject_id = s.id - AND json_extract(j.payload, '$.snapshot_id') IS NOT NULL - ) - ) - AND NOT EXISTS ( - SELECT 1 FROM jobs j - WHERE j.kind = 'snapshot_upload' - AND j.subject_id = s.id - AND j.status IN ('pending', 'running') - ) - ORDER BY {IDLE_WITNESS_SQL} ASC, s.id ASC - LIMIT ?4" - ); - let mut stmt = conn.prepare(&sql)?; - let rows = stmt - .query_map( - params![cutoff, cursor_witness, cursor_sandbox_id, limit as i64], - |row| { - let sandbox_id: String = row.get(0)?; - let idle_witness_secs = row.get::<_, SqliteU64>(1)?.0; - Ok(EvictionCandidate { - cursor: CandidateCursor { - idle_witness_secs, - sandbox_id: sandbox_id.clone(), - }, - sandbox_id, - }) + let rows = crate::db::eviction_scan::idle_suspended_candidates( + conn, + cutoff, + cursor_witness, + cursor_sandbox_id, + limit, + )?; + Ok(rows + .into_iter() + .map(|(sandbox_id, idle_witness_secs)| EvictionCandidate { + cursor: CandidateCursor { + idle_witness_secs, + sandbox_id: sandbox_id.clone(), }, - )? - .collect::>()?; - Ok(rows) + sandbox_id, + }) + .collect()) } /// Cold suspended sandboxes eligible for the per-candidate cluster check: @@ -655,56 +573,7 @@ pub(crate) fn list_legacy_reupload_candidates( idle_cutoff_epoch_secs: u64, limit: usize, ) -> rusqlite::Result> { - let sql = format!( - "SELECT s.id FROM sandboxes s - WHERE s.state = 'suspended' - AND {VALID_SUSPENDED_UPDATED_AT_SQL} - AND {IDLE_WITNESS_SQL} <= ?1 - AND s.current_local_artifact_id IS NULL - AND NOT EXISTS ( - SELECT 1 FROM jobs j - WHERE j.kind = 'snapshot_upload' - AND j.subject_id = s.id - AND j.status IN ('pending', 'running') - ) - AND NOT EXISTS ( - SELECT 1 FROM jobs j - WHERE j.kind = 'snapshot_upload' - AND j.subject_id = s.id - AND json_extract(j.payload, '$.snapshot_id') IS NOT NULL - ) - ORDER BY {IDLE_WITNESS_SQL} ASC - LIMIT ?2" - ); - let mut stmt = conn.prepare(&sql)?; - let rows = stmt - .query_map( - params![SqliteU64(idle_cutoff_epoch_secs), limit as i64], - |r| r.get(0), - )? - .collect::>()?; - Ok(rows) -} - -fn count_malformed_suspended_updated_at(conn: &Connection) -> rusqlite::Result { - let sql = format!( - "SELECT COUNT(*) FROM sandboxes s - WHERE s.state = 'suspended' - AND NOT ({VALID_SUSPENDED_UPDATED_AT_SQL})" - ); - conn.query_row(&sql, [], |row| { - let count: i64 = row.get(0)?; - Ok(usize::try_from(count).unwrap_or_default()) - }) -} - -/// Every local `sandboxes` row, whatever its state. node-ctl's full census -/// publishes one entry per row, so this is the count that sizes it. -fn count_resident_rows(conn: &Connection) -> rusqlite::Result { - conn.query_row("SELECT COUNT(*) FROM sandboxes", [], |row| { - let count: i64 = row.get(0)?; - usize::try_from(count).map_err(|_| rusqlite::Error::IntegralValueOutOfRange(0, count)) - }) + crate::db::eviction_scan::legacy_reupload_candidates(conn, idle_cutoff_epoch_secs, limit) } /// Gather the local half of the eviction evidence plus the sandbox's @@ -723,28 +592,14 @@ pub(crate) fn gather_local_evidence( conn: &Connection, sandbox_id: &str, ) -> rusqlite::Result> { - let row: Option<(Option, crate::db::timestamps::EpochSeconds)> = conn - .query_row( - "SELECT current_local_artifact_id, updated_at FROM sandboxes WHERE id = ?1", - params![sandbox_id], - |r| Ok((r.get(0)?, r.get(1)?)), - ) - .optional()?; + let row = crate::db::eviction_scan::artifact_pin_and_updated_at(conn, sandbox_id)?; let Some((pin, updated_at)) = row else { return Ok(None); }; let updated_at_secs = updated_at.unsigned(); if let Some(pin) = pin { - let artifact: Option<(String, String, bool)> = conn - .query_row( - "SELECT artifact_kind, local_artifact_id, backup_completed_at IS NOT NULL - FROM local_artifacts - WHERE local_artifact_id = ?1 AND sandbox_id = ?2", - params![pin, sandbox_id], - |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)), - ) - .optional()?; + let artifact = crate::db::eviction_scan::owned_artifact_evidence(conn, &pin, sandbox_id)?; let evidence = match artifact { Some((kind, id, backup_marker)) => { match (id.parse(), kind.parse()) { @@ -766,16 +621,7 @@ pub(crate) fn gather_local_evidence( // Unpinned: the newest upload job regardless of status (a failed // newest upload must still count — it proves newer-than-backed // local state exists). - let job: Option<(Option, String)> = conn - .query_row( - "SELECT json_extract(j.payload, '$.snapshot_id'), j.created_at FROM jobs j - WHERE j.kind = 'snapshot_upload' AND j.subject_id = ?1 - ORDER BY j.created_at DESC, j.job_id DESC - LIMIT 1", - params![sandbox_id], - |r| Ok((r.get(0)?, r.get(1)?)), - ) - .optional()?; + let job = crate::db::eviction_scan::latest_upload_job(conn, sandbox_id)?; let latest_job = job.and_then(|(snapshot_id, created_at)| { let snapshot_id = snapshot_id?.trim().parse::().ok()?; // Invalid time cannot prove recency, even when the sandbox itself diff --git a/crates/libs/sbx/mgr/server/src/sandbox_eviction/tests.rs b/crates/libs/sbx/mgr/server/src/sandbox_eviction/tests.rs index 48a44b293..a2ac37a8d 100644 --- a/crates/libs/sbx/mgr/server/src/sandbox_eviction/tests.rs +++ b/crates/libs/sbx/mgr/server/src/sandbox_eviction/tests.rs @@ -1,4 +1,5 @@ use pplx_space_common_core::types::{LocalArtifactId, SnapshotId}; +use rusqlite::params; use super::*; diff --git a/crates/libs/sbx/mgr/server/src/service/activation_support.rs b/crates/libs/sbx/mgr/server/src/service/activation_support.rs index a54a1c8be..947a6fe2f 100644 --- a/crates/libs/sbx/mgr/server/src/service/activation_support.rs +++ b/crates/libs/sbx/mgr/server/src/service/activation_support.rs @@ -360,7 +360,7 @@ impl SandboxManagerService { match crate::db::activation_route_provenance(&db, sandbox_id) { Ok(provenance) => { if requested_can_open_claim - && let Some(claim) = crate::route_activation_claims::OpenRouteActivationClaim::for_cluster_fenced_identity( + && let Some(claim) = crate::db::route_activation_claims::OpenRouteActivationClaim::for_cluster_fenced_identity( &identity.identity, identity.provenance, ) diff --git a/crates/libs/sbx/mgr/server/src/service/cleanup.rs b/crates/libs/sbx/mgr/server/src/service/cleanup.rs index 8a81a38cb..64d5581b6 100644 --- a/crates/libs/sbx/mgr/server/src/service/cleanup.rs +++ b/crates/libs/sbx/mgr/server/src/service/cleanup.rs @@ -1114,7 +1114,7 @@ impl SandboxManagerService { } if completion.deletes_row() && mode.auto_lifecycle_delete_expectation().is_none() - && crate::auto_lifecycle_delete_receipts::db::pending_receipt_for_sandbox( + && crate::db::auto_lifecycle_delete_receipts::pending_receipt_for_sandbox( &transaction, sandbox_id, ) @@ -1134,7 +1134,7 @@ impl SandboxManagerService { mode.auto_lifecycle_delete_expectation() { let admission = match - crate::auto_lifecycle_delete_receipts::db::classify_and_record_admission( + crate::db::auto_lifecycle_delete_receipts::classify_and_record_admission( &transaction, sandbox_id, self.node_role, @@ -1143,7 +1143,7 @@ impl SandboxManagerService { ) { Ok(admission) => admission, - Err(error @ crate::auto_lifecycle_delete_receipts::db::AutoLifecycleDeleteEvidenceError::PolicyChanged) => { + Err(error @ crate::db::auto_lifecycle_delete_receipts::AutoLifecycleDeleteEvidenceError::PolicyChanged) => { pplx_space_metrics::sandbox::worker::auto_lifecycle_delete_admission( pplx_space_metrics::sandbox::worker::AutoLifecycleDeleteAdmissionDisposition::Failed, pplx_space_metrics::sandbox::worker::AutoLifecycleDeleteAdmissionReason::PolicyChanged, @@ -1163,7 +1163,7 @@ impl SandboxManagerService { reason: AutoLifecycleDeleteCasLostReason::PolicyChanged, }); } - Err(error @ crate::auto_lifecycle_delete_receipts::db::AutoLifecycleDeleteEvidenceError::AuthorityChanged { .. }) => { + Err(error @ crate::db::auto_lifecycle_delete_receipts::AutoLifecycleDeleteEvidenceError::AuthorityChanged { .. }) => { pplx_space_metrics::sandbox::worker::auto_lifecycle_delete_admission( pplx_space_metrics::sandbox::worker::AutoLifecycleDeleteAdmissionDisposition::Failed, pplx_space_metrics::sandbox::worker::AutoLifecycleDeleteAdmissionReason::AuthorityChanged, @@ -1322,7 +1322,7 @@ impl SandboxManagerService { "receipt admission is missing its auto-lifecycle delete expectation", ) })?; - let receipt = match crate::auto_lifecycle_delete_receipts::db::insert_pending_receipt( + let receipt = match crate::db::auto_lifecycle_delete_receipts::insert_pending_receipt( &transaction, crate::auto_lifecycle_delete_receipts::AutoLifecycleDeleteReceipt { receipt_id: pplx_space_common_core::types::AutoLifecycleDeleteReceiptId::generate(), @@ -1333,7 +1333,7 @@ impl SandboxManagerService { }, ) { Ok(receipt) => receipt, - Err(crate::auto_lifecycle_delete_receipts::db::InsertPendingReceiptError::Conflict(error)) => { + Err(crate::db::auto_lifecycle_delete_receipts::InsertPendingReceiptError::Conflict(error)) => { pplx_space_metrics::sandbox::worker::auto_lifecycle_delete_admission( pplx_space_metrics::sandbox::worker::AutoLifecycleDeleteAdmissionDisposition::Failed, pplx_space_metrics::sandbox::worker::AutoLifecycleDeleteAdmissionReason::ReceiptConflict, @@ -1343,7 +1343,7 @@ impl SandboxManagerService { sandbox_id, }); } - Err(crate::auto_lifecycle_delete_receipts::db::InsertPendingReceiptError::Database(error)) => { + Err(crate::db::auto_lifecycle_delete_receipts::InsertPendingReceiptError::Database(error)) => { return Err(Status::unavailable(format!( "persist auto-lifecycle delete receipt: {error}" )) @@ -1697,7 +1697,7 @@ impl SandboxManagerService { let sandbox_id = sandbox_id .parse::() .map_err(|error| error.to_string())?; - if !crate::auto_lifecycle_delete_receipts::db::pending_receipt_matches_observed_execution( + if !crate::db::auto_lifecycle_delete_receipts::pending_receipt_matches_observed_execution( &db, sandbox_id, ) diff --git a/crates/libs/sbx/mgr/server/src/service/cleanup_executor.rs b/crates/libs/sbx/mgr/server/src/service/cleanup_executor.rs index cc62fd862..449780eb2 100644 --- a/crates/libs/sbx/mgr/server/src/service/cleanup_executor.rs +++ b/crates/libs/sbx/mgr/server/src/service/cleanup_executor.rs @@ -636,7 +636,7 @@ impl SandboxManagerService { return Ok(None); } let receipt = - crate::auto_lifecycle_delete_receipts::db::pending_receipt_for_sandbox( + crate::db::auto_lifecycle_delete_receipts::pending_receipt_for_sandbox( db, work.sandbox_id, ) diff --git a/crates/libs/sbx/mgr/server/src/service/create.rs b/crates/libs/sbx/mgr/server/src/service/create.rs index 438e7fb67..002df47be 100644 --- a/crates/libs/sbx/mgr/server/src/service/create.rs +++ b/crates/libs/sbx/mgr/server/src/service/create.rs @@ -335,7 +335,7 @@ impl SandboxManagerService { let create_route_provenance = normalized_create_route_identity.provenance; let open_route_activation = (!req.autostart && requested_create_route_identity_can_open_claim).then(|| { - crate::route_activation_claims::OpenRouteActivationClaim::for_cluster_fenced_identity( + crate::db::route_activation_claims::OpenRouteActivationClaim::for_cluster_fenced_identity( &create_route_identity, create_route_provenance, ) @@ -733,7 +733,7 @@ impl SandboxManagerService { .await?; let open_route_activation = open_activation_claim_eligible .then(|| { - crate::route_activation_claims::OpenRouteActivationClaim::for_cluster_fenced_identity( + crate::db::route_activation_claims::OpenRouteActivationClaim::for_cluster_fenced_identity( &space_api_route_identity, route_identity_provenance, ) diff --git a/crates/libs/sbx/mgr/server/src/service/grpc_auto_lifecycle_delete_receipts.rs b/crates/libs/sbx/mgr/server/src/service/grpc_auto_lifecycle_delete_receipts.rs index 49afe52c0..cdbff70b6 100644 --- a/crates/libs/sbx/mgr/server/src/service/grpc_auto_lifecycle_delete_receipts.rs +++ b/crates/libs/sbx/mgr/server/src/service/grpc_auto_lifecycle_delete_receipts.rs @@ -15,7 +15,7 @@ impl SandboxManagerService { .db .lock() .map_err(|_| Status::internal("database mutex poisoned"))?; - crate::auto_lifecycle_delete_receipts::db::list_completed_receipts(&db) + crate::db::auto_lifecycle_delete_receipts::list_completed_receipts(&db) .map_err(|error| Status::internal(format!("list delete receipts: {error}")))? }; Ok(Response::new(ListAutoLifecycleDeleteReceiptsResponse { @@ -58,7 +58,7 @@ impl SandboxManagerService { .db .lock() .map_err(|_| Status::internal("database mutex poisoned"))?; - crate::auto_lifecycle_delete_receipts::db::acknowledge_completed_receipts( + crate::db::auto_lifecycle_delete_receipts::acknowledge_completed_receipts( &db, &receipt_ids, ) diff --git a/crates/libs/sbx/mgr/server/src/service/grpc_core/tests.rs b/crates/libs/sbx/mgr/server/src/service/grpc_core/tests.rs index ee5ac0f02..193800287 100644 --- a/crates/libs/sbx/mgr/server/src/service/grpc_core/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/grpc_core/tests.rs @@ -947,7 +947,7 @@ async fn auto_lifecycle_delete_requires_one_canonical_id_and_the_expected_policy ); fixture.assert_untouched(&id, SandboxState::Started, case.name); assert_eq!( - crate::auto_lifecycle_delete_receipts::db::pending_receipt_for_sandbox( + crate::db::auto_lifecycle_delete_receipts::pending_receipt_for_sandbox( &fixture.service.db_conn().unwrap(), sandbox_id, ) diff --git a/crates/libs/sbx/mgr/server/src/service/mod.rs b/crates/libs/sbx/mgr/server/src/service/mod.rs index 39402324a..e58d6610e 100644 --- a/crates/libs/sbx/mgr/server/src/service/mod.rs +++ b/crates/libs/sbx/mgr/server/src/service/mod.rs @@ -949,10 +949,10 @@ fn persisted_activation_owns_route_identity( } fn map_runtime_fields_update_error(error: rusqlite::Error) -> Status { - if crate::auto_lifecycle_delete_receipts::db::is_pending_receipt_execution_fence(&error) { + if crate::db::auto_lifecycle_delete_receipts::is_pending_receipt_execution_fence(&error) { return Status::aborted("pending auto-lifecycle delete receipt owns the sandbox execution"); } - if crate::auto_lifecycle_delete_receipts::db::is_lifecycle_execution_fence(&error) { + if crate::db::auto_lifecycle_delete_receipts::is_lifecycle_execution_fence(&error) { return Status::aborted("sandbox lifecycle or cleanup owns the sandbox execution"); } match error { @@ -966,9 +966,9 @@ fn route_identity_adoption_write_rejection_reason( error: &rusqlite::Error, ) -> pplx_space_metrics::sandbox::worker::RouteIdentityAdoptionReason { use pplx_space_metrics::sandbox::worker::RouteIdentityAdoptionReason; - if crate::auto_lifecycle_delete_receipts::db::is_pending_receipt_execution_fence(error) { + if crate::db::auto_lifecycle_delete_receipts::is_pending_receipt_execution_fence(error) { RouteIdentityAdoptionReason::PendingReceiptOwnsExecution - } else if crate::auto_lifecycle_delete_receipts::db::is_lifecycle_execution_fence(error) { + } else if crate::db::auto_lifecycle_delete_receipts::is_lifecycle_execution_fence(error) { RouteIdentityAdoptionReason::LifecycleOwnsExecution } else { match error { @@ -1255,7 +1255,7 @@ impl SandboxManagerService { local.current_node_id = local.current_node_id.or(self.local_node_id); } let merged = merge_authoritative_route_identity(&local, incoming); - let metadata = crate::auto_lifecycle_delete_receipts::db::route_identity_adoption_metadata( + let metadata = crate::db::auto_lifecycle_delete_receipts::route_identity_adoption_metadata( &db, sandbox_id, ) .map_err(|error| Status::internal(format!("inspect route identity metadata: {error}"))) @@ -1294,7 +1294,7 @@ impl SandboxManagerService { // provenance and receipt mode are current. Persist an authoritative // observation even when it does not advance the stream epoch. let pending_receipt = - crate::auto_lifecycle_delete_receipts::db::pending_receipt_for_sandbox( + crate::db::auto_lifecycle_delete_receipts::pending_receipt_for_sandbox( &db, sandbox_id, ) .map_err(|error| { @@ -1651,7 +1651,7 @@ impl SandboxManagerService { .as_ref() .map(|bundle| bundle.generation.get()); if options.node_role != Some(NodeRole::Worker) { - match crate::auto_lifecycle_delete_receipts::db::outbox_stats(&conn) { + match crate::db::auto_lifecycle_delete_receipts::outbox_stats(&conn) { Ok(stats) if stats.pending_count.saturating_add(stats.completed_count) != 0 => { tracing::warn!( node_role = ?options.node_role, diff --git a/crates/libs/sbx/mgr/server/src/service/reconcile.rs b/crates/libs/sbx/mgr/server/src/service/reconcile.rs index fb7c207a0..bc30b1c05 100644 --- a/crates/libs/sbx/mgr/server/src/service/reconcile.rs +++ b/crates/libs/sbx/mgr/server/src/service/reconcile.rs @@ -219,7 +219,7 @@ impl SandboxManagerService { let Ok(db) = self.db_conn() else { return; }; - match crate::auto_lifecycle_delete_receipts::db::record_receipt_conflict_retry( + match crate::db::auto_lifecycle_delete_receipts::record_receipt_conflict_retry( &db, storage_id, ) { Ok(true) => emit_sandbox_event( diff --git a/crates/libs/sbx/mgr/server/src/service/recovery_cross_node.rs b/crates/libs/sbx/mgr/server/src/service/recovery_cross_node.rs index 448f392cf..2f9d95eae 100644 --- a/crates/libs/sbx/mgr/server/src/service/recovery_cross_node.rs +++ b/crates/libs/sbx/mgr/server/src/service/recovery_cross_node.rs @@ -685,17 +685,10 @@ fn adopt_restoring_row_under_recovery_claim( // The generation CAS below sees the newly written Restoring state, so it // cannot independently prove the prior row was still eligible for recovery. if recovery.is_some() { - use rusqlite::OptionalExtension; - let existing_state: Option = tx - .query_row( - pplx_space_common_core::sql!("SELECT state FROM sandboxes WHERE id = ?1"), - rusqlite::params![canonical_id], - |row| row.get(0), - ) - .optional() - .map_err( - |source| crate::snapshot_restore::RestoreError::AdvanceRestorePhase { source }, - )?; + let existing_state = + crate::db::current_state_label(&tx, canonical_id).map_err(|source| { + crate::snapshot_restore::RestoreError::AdvanceRestorePhase { source } + })?; // An absent row is the orphan case this path exists for; the upsert // inserts it. Only an existing row in an unexpected state is a refusal. // diff --git a/crates/libs/sbx/mgr/server/src/service/route_identity_tests.rs b/crates/libs/sbx/mgr/server/src/service/route_identity_tests.rs index 26bdd4554..cd9cddc27 100644 --- a/crates/libs/sbx/mgr/server/src/service/route_identity_tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/route_identity_tests.rs @@ -190,7 +190,7 @@ fn route_identity_provenance( service: &SandboxManagerService, sandbox_id: SandboxId, ) -> Option { - crate::auto_lifecycle_delete_receipts::db::route_identity_adoption_metadata( + crate::db::auto_lifecycle_delete_receipts::route_identity_adoption_metadata( &service.db_conn().unwrap(), sandbox_id, ) @@ -315,7 +315,7 @@ async fn identity_equal_adoption_does_not_overwrite_an_owned_execution() { SandboxState::Stopping } ExecutionOwner::PendingDeleteReceipt => { - crate::auto_lifecycle_delete_receipts::db::insert_pending_receipt( + crate::db::auto_lifecycle_delete_receipts::insert_pending_receipt( &conn, crate::auto_lifecycle_delete_receipts::AutoLifecycleDeleteReceipt { receipt_id: diff --git a/crates/libs/sbx/mgr/server/src/service/start.rs b/crates/libs/sbx/mgr/server/src/service/start.rs index 6ac9476dd..c511a3b2f 100644 --- a/crates/libs/sbx/mgr/server/src/service/start.rs +++ b/crates/libs/sbx/mgr/server/src/service/start.rs @@ -1731,7 +1731,7 @@ impl SandboxManagerService { let open_route_activation = start_ctx .open_activation_claim_eligible .then(|| { - crate::route_activation_claims::OpenRouteActivationClaim::for_cluster_fenced_identity( + crate::db::route_activation_claims::OpenRouteActivationClaim::for_cluster_fenced_identity( &start_ctx.space_api_route_identity, start_ctx.route_identity_provenance, ) diff --git a/crates/libs/sbx/mgr/server/src/service/tests.rs b/crates/libs/sbx/mgr/server/src/service/tests.rs index fd44acfd7..1d76da9b9 100644 --- a/crates/libs/sbx/mgr/server/src/service/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/tests.rs @@ -1084,7 +1084,7 @@ pub(crate) mod service_tests { assert!(resolved.current_node_id.is_some()); assert_eq!(resolved.substrate_kind, Some(NodeManagedBy::Space)); assert!( - !crate::route_activation_claims::claim_exists( + !crate::db::route_activation_claims::claim_exists( &service.db.lock().unwrap(), &sandbox_id.to_string() ) @@ -1821,7 +1821,7 @@ pub(crate) mod service_tests { normalized.provenance, true, ) { - crate::auto_lifecycle_delete_receipts::db::record_retained_route_evidence( + crate::db::auto_lifecycle_delete_receipts::record_retained_route_evidence( &service.db.lock().unwrap(), sandbox_id, evidence, @@ -1899,7 +1899,7 @@ pub(crate) mod service_tests { activation_attempt_id: identity.activation_attempt_id, }, Some(evidence), - crate::route_activation_claims::OpenRouteActivationClaim::for_cluster_fenced_identity( + crate::db::route_activation_claims::OpenRouteActivationClaim::for_cluster_fenced_identity( &identity, crate::auto_lifecycle_delete_receipts::RouteIdentityProvenance::ClusterFenced, ) @@ -1918,7 +1918,9 @@ pub(crate) mod service_tests { &crate::db::edges::CREATE_BASE_SNAPSHOT_COMPLETE, ); assert!(crate::db::complete_create(&db, &sandbox_id_string, cleanup_id, true).unwrap()); - assert!(crate::route_activation_claims::claim_exists(&db, &sandbox_id_string).unwrap()); + assert!( + crate::db::route_activation_claims::claim_exists(&db, &sandbox_id_string).unwrap() + ); } (service, sandbox_id, identity) } @@ -1977,7 +1979,7 @@ pub(crate) mod service_tests { ) .unwrap(); assert!( - !crate::route_activation_claims::claim_exists(&db, &sandbox_id.to_string()) + !crate::db::route_activation_claims::claim_exists(&db, &sandbox_id.to_string()) .unwrap() ); } @@ -2054,7 +2056,7 @@ pub(crate) mod service_tests { .unwrap() .expect("the later Direct activation must claim the sandbox row"); assert!( - !crate::route_activation_claims::claim_exists(&db, &sandbox_id.to_string()) + !crate::db::route_activation_claims::claim_exists(&db, &sandbox_id.to_string()) .unwrap(), "a later non-Kubernetes activation must supersede the open Kubernetes claim" ); @@ -2097,7 +2099,7 @@ pub(crate) mod service_tests { }; let provenance = crate::auto_lifecycle_delete_receipts::RouteIdentityProvenance::ClusterFenced; - let open_claim = crate::route_activation_claims::OpenRouteActivationClaim::for_cluster_fenced_identity(&route, provenance).unwrap(); + let open_claim = crate::db::route_activation_claims::OpenRouteActivationClaim::for_cluster_fenced_identity(&route, provenance).unwrap(); let episode = crate::db::begin_activation_with_route_evidence( &conn, &id, @@ -2196,7 +2198,7 @@ pub(crate) mod service_tests { if receipt.kind() == SandboxTransitionReceiptKind::ActivationRollback && receipt.activation_attempt_id() == Some(if persist_operation { operation } else { route_attempt })) ); - assert!(crate::route_activation_claims::claim_exists(&db, &id).unwrap()); + assert!(crate::db::route_activation_claims::claim_exists(&db, &id).unwrap()); assert!( crate::route_activation_claims::validate_cluster_fenced_activation( &db, @@ -2267,7 +2269,7 @@ pub(crate) mod service_tests { .resolve_runtime_activation_identity(&sandbox_id.to_string(), identity, false) .expect("the surviving pod and attempt may continue after manager restart"); assert!( - crate::route_activation_claims::claim_exists( + crate::db::route_activation_claims::claim_exists( &service.db.lock().unwrap(), &sandbox_id.to_string() ) @@ -2281,7 +2283,7 @@ pub(crate) mod service_tests { service_with_base_snapshot_create_route_evidence(); { let db = service.db.lock().unwrap(); - crate::route_activation_claims::close_claim(&db, &sandbox_id.to_string()).unwrap(); + crate::db::route_activation_claims::close_claim(&db, &sandbox_id.to_string()).unwrap(); } identity.route_generation = Some(RouteGeneration::new(2).unwrap()); identity.execution_generation = Some(ExecutionGeneration::new(2).unwrap()); @@ -2292,7 +2294,7 @@ pub(crate) mod service_tests { .resolve_runtime_activation_identity(&sandbox_id.to_string(), identity, false) .expect("a newer rollout-compatible identity remains on the existing path"); assert!( - !crate::route_activation_claims::claim_exists( + !crate::db::route_activation_claims::claim_exists( &service.db.lock().unwrap(), &sandbox_id.to_string() ) @@ -3477,7 +3479,7 @@ pub(crate) mod service_tests { crate::auto_lifecycle_delete_receipts::RouteIdentityProvenance::ClusterFenced, ) .unwrap(); - crate::auto_lifecycle_delete_receipts::db::insert_pending_receipt( + crate::db::auto_lifecycle_delete_receipts::insert_pending_receipt( &db, crate::auto_lifecycle_delete_receipts::AutoLifecycleDeleteReceipt { receipt_id: pending_receipt_id, @@ -3503,7 +3505,7 @@ pub(crate) mod service_tests { let db = service.db.lock().unwrap(); assert!( - crate::auto_lifecycle_delete_receipts::db::pending_receipt_for_sandbox( + crate::db::auto_lifecycle_delete_receipts::pending_receipt_for_sandbox( &db, sandbox_id, ) .unwrap() @@ -3568,7 +3570,7 @@ pub(crate) mod service_tests { crate::auto_lifecycle_delete_receipts::RouteIdentityProvenance::ClusterFenced, ) .unwrap(); - crate::auto_lifecycle_delete_receipts::db::insert_pending_receipt(&db, pending) + crate::db::auto_lifecycle_delete_receipts::insert_pending_receipt(&db, pending) .unwrap(); db.execute_batch( "CREATE TRIGGER reject_redundant_route_identity_adoption @@ -3601,7 +3603,7 @@ pub(crate) mod service_tests { let db = service.db.lock().unwrap(); assert_eq!( - crate::auto_lifecycle_delete_receipts::db::pending_receipt_for_sandbox( + crate::db::auto_lifecycle_delete_receipts::pending_receipt_for_sandbox( &db, sandbox_id, ) .unwrap(), @@ -3679,7 +3681,7 @@ pub(crate) mod service_tests { assert_eq!( route_identity_adoption_write_rejection_reason( &rusqlite::Error::ToSqlConversionFailure(Box::new( - crate::auto_lifecycle_delete_receipts::db::PendingReceiptOwnsExecution, + crate::db::auto_lifecycle_delete_receipts::PendingReceiptOwnsExecution, )), ), RouteIdentityAdoptionReason::PendingReceiptOwnsExecution @@ -3687,7 +3689,7 @@ pub(crate) mod service_tests { assert_eq!( route_identity_adoption_write_rejection_reason( &rusqlite::Error::ToSqlConversionFailure(Box::new( - crate::auto_lifecycle_delete_receipts::db::LifecycleOwnsExecution, + crate::db::auto_lifecycle_delete_receipts::LifecycleOwnsExecution, )), ), RouteIdentityAdoptionReason::LifecycleOwnsExecution @@ -6894,7 +6896,7 @@ pub(crate) mod service_tests { .is_none() ); assert_eq!( - crate::auto_lifecycle_delete_receipts::db::pending_receipt_for_sandbox( + crate::db::auto_lifecycle_delete_receipts::pending_receipt_for_sandbox( &db, sandbox_id, ) .unwrap(), @@ -6924,7 +6926,7 @@ pub(crate) mod service_tests { assert_eq!(row.cleanup_completion, Some(CleanupCompletion::DeleteRow)); assert_eq!(row.cleanup_keep_filesystem, Some(false)); assert_eq!( - crate::auto_lifecycle_delete_receipts::db::pending_receipt_for_sandbox( + crate::db::auto_lifecycle_delete_receipts::pending_receipt_for_sandbox( &db, sandbox_id, ) .unwrap(), @@ -8267,7 +8269,7 @@ pub(crate) mod service_tests { &AutoLifecyclePolicy::disabled(), ), }; - crate::auto_lifecycle_delete_receipts::db::insert_pending_receipt(&conn, receipt).unwrap(); + crate::db::auto_lifecycle_delete_receipts::insert_pending_receipt(&conn, receipt).unwrap(); let service = SandboxManagerService::for_test(conn); let error = service @@ -8309,7 +8311,7 @@ pub(crate) mod service_tests { crate::auto_lifecycle_delete_receipts::RouteIdentityProvenance::ClusterFenced, ) .unwrap(); - crate::auto_lifecycle_delete_receipts::db::insert_pending_receipt( + crate::db::auto_lifecycle_delete_receipts::insert_pending_receipt( &conn, crate::auto_lifecycle_delete_receipts::AutoLifecycleDeleteReceipt { receipt_id: pplx_space_common_core::types::AutoLifecycleDeleteReceiptId::generate(), @@ -9973,7 +9975,7 @@ pub(crate) mod service_tests { crate::auto_lifecycle_delete_receipts::RouteIdentityProvenance::Unfenced, ) .unwrap(); - crate::auto_lifecycle_delete_receipts::db::insert_pending_receipt(&conn, receipt).unwrap(); + crate::db::auto_lifecycle_delete_receipts::insert_pending_receipt(&conn, receipt).unwrap(); conn.execute( "UPDATE sandboxes SET state = 'deleting', @@ -9983,7 +9985,7 @@ pub(crate) mod service_tests { [sandbox_id.to_string()], ) .unwrap(); - crate::auto_lifecycle_delete_receipts::db::record_retained_route_evidence( + crate::db::auto_lifecycle_delete_receipts::record_retained_route_evidence( &conn, sandbox_id, crate::auto_lifecycle_delete_receipts::RouteEvidence { @@ -10853,7 +10855,7 @@ pub(crate) mod service_tests { SandboxState::Started ); assert_eq!( - crate::auto_lifecycle_delete_receipts::db::pending_receipt_for_sandbox( + crate::db::auto_lifecycle_delete_receipts::pending_receipt_for_sandbox( &db, sandbox_id, ) .unwrap(), @@ -17718,7 +17720,7 @@ pub(crate) mod service_tests { &AutoLifecyclePolicy::disabled(), ), }; - crate::auto_lifecycle_delete_receipts::db::insert_pending_receipt(&conn, receipt).unwrap(); + crate::db::auto_lifecycle_delete_receipts::insert_pending_receipt(&conn, receipt).unwrap(); let service = SandboxManagerService::for_test(conn); let targets = service.recover_requested_cleanup_targets_for_ids(vec![sandbox_id]); diff --git a/crates/libs/sbx/mgr/server/src/snapshot_upload_jobs.rs b/crates/libs/sbx/mgr/server/src/snapshot_upload_jobs.rs index b75c0ce50..0fef00ab8 100644 --- a/crates/libs/sbx/mgr/server/src/snapshot_upload_jobs.rs +++ b/crates/libs/sbx/mgr/server/src/snapshot_upload_jobs.rs @@ -53,7 +53,7 @@ use crate::snapshot_upload::{ format_from_meta, upload_artifacts_for_meta, }; -pub const SNAPSHOT_UPLOAD_JOB_KIND: &str = "snapshot_upload"; +pub use crate::db::snapshot_upload_jobs::SNAPSHOT_UPLOAD_JOB_KIND; const DEFAULT_CONCURRENCY: usize = 4; const JOB_TIMEOUT: Duration = Duration::from_secs(7200); @@ -89,11 +89,9 @@ pub(crate) fn build_worker( } pub fn refund_interrupted_running_attempts(conn: &rusqlite::Connection) -> rusqlite::Result { - conn.execute( - "UPDATE jobs - SET attempts = attempts - 1 - WHERE kind = ?1 AND status = 'running' AND attempts > 0", - rusqlite::params![SNAPSHOT_UPLOAD_JOB_KIND], + crate::db::snapshot_upload_jobs::refund_interrupted_running_attempts( + conn, + SNAPSHOT_UPLOAD_JOB_KIND, ) } @@ -517,17 +515,17 @@ pub(crate) fn persist_submitted_transfer_ids( "snapshot-upload database mutex poisoned while persisting transfer IDs for {job_id}" )) })?; - let changed = conn - .execute( - "UPDATE jobs SET payload = ?1 - WHERE job_id = ?2 AND kind = ?3 AND status IN ('pending', 'running')", - rusqlite::params![payload, job_id, SNAPSHOT_UPLOAD_JOB_KIND], - ) - .map_err(|error| { - JobError::retriable(format!( - "persist submitted transfer IDs for snapshot-upload job {job_id}: {error}" - )) - })?; + let changed = crate::db::snapshot_upload_jobs::replace_active_job_payload( + &conn, + &payload, + job_id, + SNAPSHOT_UPLOAD_JOB_KIND, + ) + .map_err(|error| { + JobError::retriable(format!( + "persist submitted transfer IDs for snapshot-upload job {job_id}: {error}" + )) + })?; if changed != 1 { return Err(JobError::permanent(format!( "snapshot-upload job {job_id} was not pending or running while persisting transfer IDs" @@ -1426,8 +1424,7 @@ pub fn backfill_backup_completed_markers( let through = match scan.through { Some(through) => through, None => { - let through: Option = - conn.query_row("SELECT MAX(rowid) FROM jobs", [], |r| r.get(0))?; + let through = crate::db::snapshot_upload_jobs::max_job_rowid(conn)?; let Some(through) = through else { return Ok(report); }; @@ -1435,67 +1432,59 @@ pub fn backfill_backup_completed_markers( through } }; - let mut statement = conn.prepare( - "SELECT rowid, payload FROM jobs - WHERE kind = 'snapshot_upload' AND status = 'completed' - AND rowid > ?1 AND rowid <= ?2 ORDER BY rowid LIMIT ?3", - )?; - let mut rows = statement.query(rusqlite::params![ + crate::db::snapshot_upload_jobs::visit_completed_upload_payloads( + conn, scan.after.unwrap_or(i64::MIN), through, BACKUP_MARKER_BATCH as i64, - ])?; - let now = crate::db::now()?; - while let Some(row) = rows.next()? { - scan.after = Some(row.get(0)?); - report.examined += 1; - let Ok(payload) = row.get::<_, String>(1) else { - report.malformed_payload += 1; - continue; - }; - // Extract via `serde_json::Value` rather than the full - // `SnapshotUploadJobs` struct: historical payloads predate - // several of its fields, and the backfill needs only the - // artifact id. - let Ok(value) = serde_json::from_str::(&payload) else { - report.malformed_payload += 1; - continue; - }; - let Some(artifact_id) = value - .get("meta") - .and_then(|m| m.get("local_artifact_id")) - .and_then(|v| v.as_str()) - else { - report.payload_missing_artifact_id += 1; - continue; - }; - let changed = match conn.execute( - "UPDATE local_artifacts SET backup_completed_at = ?1 - WHERE local_artifact_id = ?2 AND backup_completed_at IS NULL", - rusqlite::params![now, artifact_id], - ) { - Ok(changed) => changed, - Err(error) => { - report.marker_write_failed += 1; - tracing::warn!(artifact_id, %error, "backup marker repair failed; evidence retained for the next sweep"); - continue; + |now, rowid, payload| { + scan.after = Some(rowid); + report.examined += 1; + let Ok(payload) = payload else { + report.malformed_payload += 1; + return Ok(()); + }; + // Extract via `serde_json::Value` rather than the full + // `SnapshotUploadJobs` struct: historical payloads predate + // several of its fields, and the backfill needs only the + // artifact id. + let Ok(value) = serde_json::from_str::(&payload) else { + report.malformed_payload += 1; + return Ok(()); + }; + let Some(artifact_id) = value + .get("meta") + .and_then(|m| m.get("local_artifact_id")) + .and_then(|v| v.as_str()) + else { + report.payload_missing_artifact_id += 1; + return Ok(()); + }; + let changed = match crate::db::snapshot_upload_jobs::stamp_backup_completed_if_unset( + conn, + now, + artifact_id, + ) { + Ok(changed) => changed, + Err(error) => { + report.marker_write_failed += 1; + tracing::warn!(artifact_id, %error, "backup marker repair failed; evidence retained for the next sweep"); + return Ok(()); + } + }; + if changed == 1 { + report.stamped += 1; + return Ok(()); } - }; - if changed == 1 { - report.stamped += 1; - continue; - } - let exists: bool = conn.query_row( - "SELECT EXISTS(SELECT 1 FROM local_artifacts WHERE local_artifact_id = ?1)", - rusqlite::params![artifact_id], - |r| r.get(0), - )?; - if exists { - report.already_stamped += 1; - } else { - report.artifact_row_missing += 1; - } - } + let exists = crate::db::snapshot_upload_jobs::local_artifact_exists(conn, artifact_id)?; + if exists { + report.already_stamped += 1; + } else { + report.artifact_row_missing += 1; + } + Ok(()) + }, + )?; if report.examined < BACKUP_MARKER_BATCH || scan.after == Some(through) { *scan = BackupMarkerScan::default(); } diff --git a/crates/libs/sbx/mgr/server/src/watchdog.rs b/crates/libs/sbx/mgr/server/src/watchdog.rs index 4288c6fdf..030d49009 100644 --- a/crates/libs/sbx/mgr/server/src/watchdog.rs +++ b/crates/libs/sbx/mgr/server/src/watchdog.rs @@ -17,12 +17,10 @@ use std::time::{Duration, Instant, SystemTime}; use futures_util::future::BoxFuture; use futures_util::stream::FuturesUnordered; use futures_util::{FutureExt, StreamExt}; -use rusqlite::types::Type; use tokio::sync::Mutex; use tokio_util::sync::CancellationToken; use tokio_util::task::TaskTracker; -use pplx_space_common_core::sql; use pplx_space_common_core::types::{LinuxPid, SandboxId, SemVer}; use pplx_space_metrics::sandbox::worker::{ NetGatewayRouteVerificationStatus, NetGatewayTlsInterceptionMode, @@ -30,6 +28,7 @@ use pplx_space_metrics::sandbox::worker::{ use pplx_space_node_runtime_core::fc_api::FirecrackerVmState; use crate::auto_lifecycle::AutoLifecycleCoordinator; +use crate::db::guest_health_scan::{SandboxGuestHealthRow, list_guest_health_rows}; use crate::node_manager_transport::NodeManagerConnector; use crate::node_state_sync::mark_sandbox_dirty; use crate::service::network_policy::NetworkPolicyReconcileQueue; @@ -87,13 +86,6 @@ const GUEST_HEALTH_CATEGORIES: [GuestHealthCategory; 3] = [ GuestHealthCategory::Unreachable, ]; -#[derive(Clone, Debug, PartialEq, Eq)] -struct SandboxGuestHealthRow { - id: String, - state: crate::state::SandboxState, - template_content_hash: Option, -} - #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] enum NetGatewayLifecycle { Known(crate::state::SandboxState), @@ -241,12 +233,6 @@ struct GuestHealthGaugeValue { count: usize, } -#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] -#[error("unknown sandbox state '{value}'")] -struct SandboxStateParseError { - value: String, -} - #[derive(Clone)] pub struct SidecarInfo { pub node_execution_id: Option, @@ -1571,14 +1557,14 @@ fn emit_auto_lifecycle_delete_gauges(db: &Arc stats, Err(error) => { tracing::warn!(%error, "failed to inspect auto-lifecycle delete outbox gauges"); return; } }; - let blocked = match crate::auto_lifecycle_delete_receipts::db::blocked_stats(&db) { + let blocked = match crate::db::auto_lifecycle_delete_receipts::blocked_stats(&db) { Ok(stats) => stats, Err(error) => { tracing::warn!(%error, "failed to inspect auto-lifecycle delete block gauges"); @@ -1899,30 +1885,6 @@ async fn emit_net_gateway_route_verification_gauges( } } -fn list_guest_health_rows( - conn: &rusqlite::Connection, -) -> rusqlite::Result> { - let mut stmt = conn.prepare(sql!("SELECT id, state, template_hash FROM sandboxes"))?; - let rows = stmt.query_map([], |row| { - let state_value = row.get::<_, String>(1)?; - let state = state_value - .parse::() - .map_err(|_| { - rusqlite::Error::FromSqlConversionFailure( - 1, - Type::Text, - Box::new(SandboxStateParseError { value: state_value }), - ) - })?; - Ok(SandboxGuestHealthRow { - id: row.get(0)?, - state, - template_content_hash: row.get(2)?, - }) - })?; - rows.collect() -} - fn sandbox_states_for_ids( db: &Arc>, sandbox_ids: &[String], @@ -1934,18 +1896,18 @@ fn sandbox_states_for_ids( return None; } }; - let mut statement = match db.prepare("SELECT state FROM sandboxes WHERE id = ?1") { - Ok(statement) => statement, + let labels = match crate::db::guest_health_scan::state_labels_for_ids(&db, sandbox_ids) { + Ok(labels) => labels, Err(error) => { tracing::warn!(%error, "failed to prepare watchdog probe-state lookup"); return None; } }; let mut states = HashMap::with_capacity(sandbox_ids.len()); - for sandbox_id in sandbox_ids { - let state = match statement.query_row([sandbox_id], |row| row.get::<_, String>(0)) { - Ok(state) => state, - Err(rusqlite::Error::QueryReturnedNoRows) => continue, + for (sandbox_id, label) in labels { + let state = match label { + Ok(Some(state)) => state, + Ok(None) => continue, Err(error) => { tracing::warn!(sandbox_id, %error, "failed to load watchdog probe state"); continue; diff --git a/crates/tools/archcheck/README.md b/crates/tools/archcheck/README.md index 8dfb9ea47..3853138c6 100644 --- a/crates/tools/archcheck/README.md +++ b/crates/tools/archcheck/README.md @@ -68,8 +68,9 @@ table: ```toml [rules.pattern_ratchet.] patterns = ['\bsome_name\b'] # regexes, matched against each code line -include = ["crates/x/src/**"] # optional; omitted means all production files +include = ["crates/x/src/**"] # optional; omitted means all production files allow = ["crates/x/src/owner.rs", "crates/x/src/owner/**"] +paths = ["crates/x/**"] # optional: the files checked; all when absent [rules.pattern_ratchet..no_crossing] mode = "advisory" @@ -97,6 +98,12 @@ name so that longer identifiers containing it do not match. Patterns are case-sensitive unless they use an inline `(?i)` flag. `allow` globs match repository-relative paths: `*` and `?` stay within one path component and `**` spans any number. Files they match are never counted. +`paths`, when given, takes globs of the same form and limits the instance to +the production files they match, for a name that other crates use for their +own purposes; files outside them are never counted, and an entry for one is +stale. An empty `paths` would check nothing and exits 2; omit it to check +every production file. An empty `allow` is fine. An instance may give both +`include` and `paths`; it then checks only the files that match both. - `no_crossing`: a file outside the allowed paths and the instance's baseline table has a matching line; @@ -259,11 +266,11 @@ becomes `support.rs`) without changing what the module holds. Each behaviour has a `mode`: `advisory` findings are reported, `required` ones also fail the check, so behaviours can graduate one at a time. Unknown keys, sections or modes, a missing file, bad TOML, an invalid regex, glob or instance -name, an empty `patterns` list, a baseline table for an instance that is not -configured, and a failing `cargo metadata` exit 2. So does a `--base-baseline`, -`--renames` or `--base-architecture` that can't be read, or one combined with -`--write-baseline`, and `--renames` input that is not git's (see [Renamed -files](#renamed-files)). +name, an empty `patterns` or `paths` list, a baseline table for an instance +that is not configured, and a failing `cargo metadata` exit 2. So does a +`--base-baseline`, `--renames` or `--base-architecture` that can't be read, or +one combined with `--write-baseline`, and `--renames` input that is not git's +(see [Renamed files](#renamed-files)). ## Baseline diff --git a/crates/tools/archcheck/src/config.rs b/crates/tools/archcheck/src/config.rs index f1c2a2841..fb400d9e1 100644 --- a/crates/tools/archcheck/src/config.rs +++ b/crates/tools/archcheck/src/config.rs @@ -59,11 +59,15 @@ pub struct FileSizeConfig { pub struct PatternRatchetConfig { /// Optional repository-relative scope; absent means all production files. #[serde(default)] - pub include: Option, + pub include: Option, /// A code line matching any of these regexes is restricted. pub patterns: Patterns, + /// Repository-relative path globs of the files the instance checks; + /// every production file when absent. + #[serde(default)] + pub paths: Option, /// Repository-relative path globs whose files may match freely. - pub allow: AllowList, + pub allow: PathGlobs, pub no_crossing: BehaviourConfig, pub no_new_entries: BehaviourConfig, pub no_raised_caps: BehaviourConfig, @@ -152,15 +156,15 @@ impl TryFrom> for Patterns { /// within one path component; `**` spans any number of them. #[derive(Debug, Clone, Deserialize)] #[serde(try_from = "Vec")] -pub struct AllowList(GlobSet); +pub struct PathGlobs(GlobSet); -impl AllowList { +impl PathGlobs { pub fn is_match(&self, path: &str) -> bool { self.0.is_match(path) } } -impl TryFrom> for AllowList { +impl TryFrom> for PathGlobs { type Error = globset::Error; fn try_from(globs: Vec) -> Result { @@ -172,6 +176,37 @@ impl TryFrom> for AllowList { } } +/// An instance's `paths`: [`PathGlobs`] with at least one glob, since an +/// empty list would check nothing. +#[derive(Debug, Clone, Deserialize)] +#[serde(try_from = "Vec")] +pub struct CheckedPaths(PathGlobs); + +#[derive(Debug, Error)] +pub enum CheckedPathsError { + #[error("paths must list at least one glob; omit it to check every production file")] + Empty, + #[error(transparent)] + Glob(#[from] globset::Error), +} + +impl CheckedPaths { + pub fn is_match(&self, path: &str) -> bool { + self.0.is_match(path) + } +} + +impl TryFrom> for CheckedPaths { + type Error = CheckedPathsError; + + fn try_from(globs: Vec) -> Result { + if globs.is_empty() { + return Err(CheckedPathsError::Empty); + } + Ok(Self(PathGlobs::try_from(globs)?)) + } +} + #[derive(Debug, Clone, Deserialize)] #[serde(deny_unknown_fields)] pub struct CrateNamesConfig { diff --git a/crates/tools/archcheck/src/rules/pattern_ratchet.rs b/crates/tools/archcheck/src/rules/pattern_ratchet.rs index 748e9ba8e..e9e5db26a 100644 --- a/crates/tools/archcheck/src/rules/pattern_ratchet.rs +++ b/crates/tools/archcheck/src/rules/pattern_ratchet.rs @@ -4,8 +4,10 @@ //! any of its `patterns` to the files its `allow` globs match: a file outside //! them with a matching line violates it, and the instance's own //! `[pattern_ratchet.]` baseline table caps each listed file's -//! matching lines; [`super::ratchet`] has the behaviours. Instances are -//! independent: each reads only its own configuration and tables. +//! matching lines; [`super::ratchet`] has the behaviours. An optional `paths` +//! list limits the instance to the files its globs match; it checks every +//! production file without one. Instances are independent: each reads only +//! its own configuration and tables. use crate::{ baseline::BASELINE_FILE, @@ -38,6 +40,14 @@ struct Instance<'a> { } impl Instance<'_> { + /// Whether `file` is among the paths the instance checks. + fn checks(&self, file: &ProductionFile) -> bool { + self.config + .paths + .as_ref() + .is_none_or(|paths| paths.is_match(&file.path)) + } + /// The instance's baseline table and where it is. fn table(&self) -> String { format!("[{RULE}.{}] in {BASELINE_FILE}", self.instance) @@ -61,13 +71,15 @@ impl CountRatchet for Instance<'_> { } /// `file`'s code lines that match the patterns, unless its path is - /// allowed or none do. Findings about them are shown on the first. + /// outside the checked paths, allowed, or none do. Findings about them + /// are shown on the first. fn measure(&self, file: &ProductionFile) -> Option { if self .config .include .as_ref() .is_some_and(|scope| !scope.is_match(&file.path)) + || !self.checks(file) || self.config.allow.is_match(&file.path) { return None; @@ -107,11 +119,15 @@ impl CountRatchet for Instance<'_> { ) } - fn cleared(&self, _file: &ProductionFile) -> String { - format!( - " has no code lines left matching the {} patterns outside its allowed paths", - self.instance - ) + fn cleared(&self, file: &ProductionFile) -> String { + let instance = self.instance; + if self.checks(file) { + format!( + " has no code lines left matching the {instance} patterns outside its allowed paths" + ) + } else { + format!(" is outside the paths [rules.{RULE}.{instance}] in ARCHITECTURE.toml checks") + } } fn counted(&self, count: usize) -> String { diff --git a/crates/tools/archcheck/tests/cli.rs b/crates/tools/archcheck/tests/cli.rs index 05999128c..dbb0dc0eb 100644 --- a/crates/tools/archcheck/tests/cli.rs +++ b/crates/tools/archcheck/tests/cli.rs @@ -1904,6 +1904,111 @@ fn crate_names_rdjson_and_exit_codes() { assert_eq!(document["diagnostics"][0]["severity"], "WARNING"); } +/// `paths` limits an instance to the files it matches: others are neither +/// counted nor kept in its table, and `allow` still applies inside them. +#[test] +fn pattern_ratchet_paths() { + let config = format!( + "{}{}", + config(ADVISORY), + ratchet( + "old", + &[r"\bold_table\b"], + &["crates/a/src/allowed.rs"], + ADVISORY + ), + ) + .replace( + "allow = ['crates/a/src/allowed.rs']\n", + "allow = ['crates/a/src/allowed.rs']\npaths = ['crates/a/**']\n", + ); + type Case<'a> = ( + &'a str, + &'a [(&'a str, usize)], + &'a [(&'a str, &'a str)], + &'a [(&'a str, &'a str)], + ); + let cases: &[Case] = &[ + ( + "a file inside the paths is checked", + &[], + &[("crates/a/src/x.rs", "old_table\n")], + &[("no_crossing", "crates/a/src/x.rs: 1 code line matches")], + ), + ( + "a file outside the paths is not", + &[], + &[("crates/b/src/x.rs", "old_table\n")], + &[], + ), + ( + "allowed paths inside the paths may match", + &[], + &[("crates/a/src/allowed.rs", "old_table\n")], + &[], + ), + ( + "an entry for a file outside the paths is stale", + &[("crates/b/src/x.rs", 1)], + &[("crates/b/src/x.rs", "old_table\n")], + &[( + "stale_entries", + "crates/b/src/x.rs is outside the paths [rules.pattern_ratchet.old] in \ + ARCHITECTURE.toml checks. Remove it from [pattern_ratchet.old] in \ + architecture-baseline.toml.", + )], + ), + ]; + for (name, entries, files, expected) in cases { + let mut files: Vec<(&str, String)> = files + .iter() + .map(|(path, text)| (*path, (*text).to_owned())) + .collect(); + files.push(("ARCHITECTURE.toml", config.clone())); + files.push(( + "architecture-baseline.toml", + ratchet_baseline(&[("old", entries)]), + )); + let ws = workspace(&files); + + let found = ratchet_findings(ws.path()); + + assert_eq!(found.len(), expected.len(), "{name}: {found:#?}"); + for ((code, message), (want_behaviour, want_message)) in found.iter().zip(*expected) { + assert_eq!( + code, + &format!("pattern_ratchet.old.{want_behaviour}"), + "{name}" + ); + assert!(message.starts_with(want_message), "{name}: {message}"); + } + } + + // An empty `paths` would check nothing, so it is rejected; an empty + // `allow` is still legal. + let no_allow = config.replace("allow = ['crates/a/src/allowed.rs']", "allow = []"); + let baseline = ratchet_baseline(&[("old", &[])]); + let ws = workspace(&[ + ("ARCHITECTURE.toml", no_allow.clone()), + ("architecture-baseline.toml", baseline.clone()), + ]); + Context::load(ws.path()).expect("an empty allow list loads"); + let ws = workspace(&[ + ( + "ARCHITECTURE.toml", + no_allow.replace("paths = ['crates/a/**']", "paths = []"), + ), + ("architecture-baseline.toml", baseline), + ]); + let error = Context::load(ws.path()).expect_err("an empty paths list is rejected"); + assert!( + error + .to_string() + .contains("paths must list at least one glob; omit it to check every production file"), + "{error}" + ); +} + /// One finding per behaviour, each reported in its own mode; only required /// findings exit 1. Without `--base-baseline`, no_new_entries is skipped and /// the summary says so. diff --git a/crates/tools/archcheck/tests/repository.rs b/crates/tools/archcheck/tests/repository.rs index 4aad40761..32f1e69ff 100644 --- a/crates/tools/archcheck/tests/repository.rs +++ b/crates/tools/archcheck/tests/repository.rs @@ -84,3 +84,205 @@ fn node_manager_commands_reject_direct_spawns_without_restricting_other_services assert!(check(&ctx).iter().any(|finding| finding.path == owner && finding.code() == "pattern_ratchet.node_manager_commands.listed_no_growth")); } + +/// Repository-relative path of the one crate the `sbxmgr_sqlite` workspace +/// below has: inside the instance's `paths`, outside its `allow`. +const SBXMGR_SRC: &str = "crates/libs/sbx/mgr/server/src"; + +/// The database migrator's sources, outside `sbxmgr_sqlite_schema`'s `allow`. +const MIGRATOR_SRC: &str = "crates/libs/db/migrator/src"; + +/// [`ratchet_findings`] for `sbxmgr_sqlite` and a crate at [`SBXMGR_SRC`]. +fn sbxmgr_sqlite_findings( + entries: &[(&str, usize)], + files: &[(&str, &str)], +) -> Vec<(&'static str, String)> { + ratchet_findings("sbxmgr_sqlite", SBXMGR_SRC, entries, files) +} + +/// A workspace with one crate whose sources are at `src`, checked by the +/// repository's own `[rules.pattern_ratchet.]`, its +/// `[rules.file_size]` and `[rules.crate_names]`, `entries` as the instance's +/// baseline, and `files` under `src`. Returns the instance's findings as +/// `(behaviour, message)`. +fn ratchet_findings( + instance: &str, + src: &str, + entries: &[(&str, usize)], + files: &[(&str, &str)], +) -> Vec<(&'static str, String)> { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../.."); + let text = std::fs::read_to_string(root.join("ARCHITECTURE.toml")) + .expect("read the repository's ARCHITECTURE.toml"); + let mut config: toml::Table = toml::from_str(&text).expect("ARCHITECTURE.toml is TOML"); + config + .get_mut("rules") + .and_then(|rules| rules.get_mut("pattern_ratchet")) + .and_then(toml::Value::as_table_mut) + .expect("ARCHITECTURE.toml has [rules.pattern_ratchet]") + .retain(|name, _| name == instance); + + let dir = tempfile::tempdir().expect("create a temporary workspace"); + let member = src.trim_end_matches("/src"); + let mut baseline = format!("[file_size]\n[crate_names]\n[pattern_ratchet.{instance}]\n"); + for (path, cap) in entries { + baseline.push_str(&format!("\"{src}/{path}\" = {cap}\n")); + } + let mut written = vec![ + ( + "Cargo.toml".to_owned(), + format!("[workspace]\nmembers = [\"{member}\"]\n"), + ), + ( + format!("{member}/Cargo.toml"), + "[package]\nname = \"sbx\"\n[lib]\npath = \"src/lib.rs\"\n".to_owned(), + ), + ( + "ARCHITECTURE.toml".to_owned(), + toml::to_string(&config).expect("serialize the configuration"), + ), + ("architecture-baseline.toml".to_owned(), baseline), + ]; + written.extend( + files + .iter() + .map(|(path, text)| (format!("{src}/{path}"), (*text).to_owned())), + ); + for (path, text) in written { + let path = dir.path().join(path); + std::fs::create_dir_all(path.parent().expect("a parent directory")).expect("create it"); + std::fs::write(path, text).expect("write a workspace file"); + } + let ctx = Context::load(dir.path()).expect("the workspace loads"); + check(&ctx) + .into_iter() + .filter(|finding| finding.instance.as_deref() == Some(instance)) + .map(|finding| (finding.behaviour, finding.message())) + .collect() +} + +/// Every way sbx-mgr code can reach its SQLite database outside the store is +/// a matching line, and ordinary code that shares a word with one is not. +#[test] +fn sbxmgr_sqlite_patterns_match_database_access_only() { + let crossings = [ + "use rusqlite::Connection;", + "use ::rusqlite::Connection;", + "use rusqlite as sqlite;", + "extern crate rusqlite;", + "let conn = Connection::open(path)?;", + "let conn = Connection::open_in_memory()?;", + "let tx = Transaction::new_unchecked(&conn, behavior)?;", + "let conn = pplx_space_common_svc::sql::sqlite::open_rusqlite(path, options)?;", + "use pplx_space_common_jobs::db as jobs;", + "use pplx_space_common_jobs::{JobStatus, db as jobs};", + "let bound = params![id];", + "let bound = named_params! { \":id\": id };", + "conn.execute(sql, bound)?;", + "let n: i64 = conn.query_one(sql, [], |row| row.get(0))?;", + "conn.query_row_and_then(sql, [], decode)?;", + "let mut rows = statement.query([id])?;", + "let present = statement.exists([id])?;", + "let tx = conn.unchecked_transaction()?;", + "let tx = crate::db::immediate_transaction(&conn)?;", + "let sql = \"SELECT state FROM sandboxes WHERE id = ?1\";", + "let sql = \"UPDATE sandboxes", + " DELETE FROM sandboxes WHERE id = ?1", + ]; + for line in crossings { + let found = sbxmgr_sqlite_findings(&[], &[("lib.rs", &format!("{line}\n"))]); + assert_eq!(found.len(), 1, "{line}: {found:#?}"); + assert_eq!(found[0].0, "no_crossing", "{line}"); + } + + let ordinary = [ + "use pplx_space_common_jobs::{JobPriority, JobStatus};", + "let db = crate::db::SharedConnection::new(conn);", + "if path.exists() {", + "let query = uri.query();", + "let worker = executor.execute_one(job);", + "let method = \"DELETE\";", + "let message = format!(\"select a sandbox: {id}\");", + "tracing::warn!(%error, \"update failed\");", + ]; + let text: String = ordinary.iter().map(|line| format!("{line}\n")).collect(); + let found = sbxmgr_sqlite_findings(&[], &[("lib.rs", &text)]); + assert!(found.is_empty(), "{found:#?}"); +} + +/// A listed file grows when it opens a connection through a type it already +/// imported, as does a child module through its parent's import. +#[test] +fn sbxmgr_sqlite_counts_opens_through_imported_names() { + let found = sbxmgr_sqlite_findings( + &[("lib.rs", 1)], + &[( + "lib.rs", + "use rusqlite::Connection;\nfn open() { let conn = Connection::open_in_memory(); }\n", + )], + ); + assert_eq!(found.len(), 1, "{found:#?}"); + assert_eq!(found[0].0, "listed_no_growth"); + assert!(found[0].1.contains("grew from 1 to 2"), "{}", found[0].1); + + let found = sbxmgr_sqlite_findings( + &[("parent.rs", 1)], + &[ + ("parent.rs", "use rusqlite::Connection;\nmod child;\n"), + ( + "parent/child.rs", + "use super::Connection;\nfn open(path: &str) { let conn = Connection::open(path); }\n", + ), + ], + ); + assert_eq!(found.len(), 1, "{found:#?}"); + assert_eq!(found[0].0, "no_crossing"); + assert!( + found[0] + .1 + .starts_with(&format!("{SBXMGR_SRC}/parent/child.rs: 1 code line")), + "{}", + found[0].1 + ); +} + +/// Outside the store, the migrator may not name sbx-mgr's database: its +/// migrations, its path or file, or its own default-path helper. The other +/// services' databases it migrates are not matches. +#[test] +fn sbxmgr_sqlite_schema_matches_the_migrators_sandbox_manager_database() { + let crossings = [ + "use pplx_space_sbxmgr_migrations::SANDBOX_MANAGER_MIGRATOR;", + "default_path: paths::SANDBOX_MANAGER_DB,", + "Database::SandboxManager(args) => run_sqlite(sqlite_sandbox_manager(), args).await,", + "fn sqlite_sandbox_manager() -> SqliteDatabase {", + "default_path: \"/space/run/sandbox-manager/sbx-manager.db\",", + "static MIGRATOR: Migrator = sqlx::migrate!(\"../../../../db/sandbox-manager\");", + ]; + for line in crossings { + let found = ratchet_findings( + "sbxmgr_sqlite_schema", + MIGRATOR_SRC, + &[], + &[("lib.rs", &format!("{line}\n"))], + ); + assert_eq!(found.len(), 1, "{line}: {found:#?}"); + assert_eq!(found[0].0, "no_crossing", "{line}"); + } + + let ordinary = [ + "Database::TemplateManager(args) => run_sqlite(sqlite_template_manager(), args).await,", + "default_path: paths::TEMPLATE_MANAGER_DB,", + "default_path: \"/space/run/template-manager/template-manager.db\",", + "static MIGRATOR: Migrator = sqlx::migrate!(\"../../../../db/template-manager\");", + "SandboxManager(SqliteArgs),", + ]; + let text: String = ordinary.iter().map(|line| format!("{line}\n")).collect(); + let found = ratchet_findings( + "sbxmgr_sqlite_schema", + MIGRATOR_SRC, + &[], + &[("lib.rs", &text)], + ); + assert!(found.is_empty(), "{found:#?}"); +}