diff --git a/crates/libs/node/ctl/server/src/facets.rs b/crates/libs/node/ctl/server/src/facets.rs index ffc91d905..8f618ece0 100644 --- a/crates/libs/node/ctl/server/src/facets.rs +++ b/crates/libs/node/ctl/server/src/facets.rs @@ -45,6 +45,22 @@ use crate::dirty_state::{ }; use crate::{InboundFacetResult, ReconcileStats, connect_cluster_api, local_manager_request, util}; +mod cluster_deleted_eviction; +mod inventory_publish; +mod local_state; +mod network_policy; +mod port_exposure; +mod sandbox_desired_state; +mod template_desired_state; + +use cluster_deleted_eviction::*; +use inventory_publish::*; +use local_state::*; +pub use network_policy::*; +use port_exposure::*; +use sandbox_desired_state::*; +use template_desired_state::*; + #[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] pub enum FacetKind { NetworkPolicy, @@ -123,39 +139,6 @@ impl SandboxPublishPayload { } } -#[derive(Debug, Default)] -struct LocalSandboxNetworkPolicies { - policies: BTreeMap, - missing: BTreeSet, -} - -impl LocalSandboxNetworkPolicies { - fn from_policies(policies: BTreeMap) -> Self { - Self { - policies, - missing: BTreeSet::new(), - } - } - - fn get(&self, sandbox_id: &SandboxId) -> Option<&NetworkPolicyDetail> { - self.policies.get(sandbox_id) - } - - fn insert_missing(&mut self, sandbox_id: SandboxId) { - self.missing.insert(sandbox_id); - } - - fn is_missing(&self, sandbox_id: &SandboxId) -> bool { - self.missing.contains(sandbox_id) - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum SandboxPolicyReconcileAction { - Apply, - Skip, -} - #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum SandboxDesiredReconcileAction { ApplyAll, @@ -163,22 +146,6 @@ enum SandboxDesiredReconcileAction { SkipNonLocal, } -fn sandbox_desired_reconcile_action( - has_local_state: bool, - has_port_exposures: bool, - has_local_allocation: bool, -) -> SandboxDesiredReconcileAction { - if has_port_exposures && !has_local_allocation { - SandboxDesiredReconcileAction::SkipNonLocal - } else if has_local_state { - SandboxDesiredReconcileAction::ApplyAll - } else if has_port_exposures { - SandboxDesiredReconcileAction::ApplyPortExposuresAndRetry - } else { - SandboxDesiredReconcileAction::SkipNonLocal - } -} - impl FacetScope { pub fn from_optional_items(items: Option>) -> Self { match items { @@ -419,2833 +386,75 @@ pub fn periodic_outbound_refs() -> Vec { .collect() } -fn split_network_policy_items(items: Vec) -> Result<(Vec, Vec), String> { - let mut templates = Vec::new(); - let mut sandbox_ids = Vec::new(); - for item in util::dedupe_non_empty(items) { - match item - .parse::() - .map_err(|error| error.to_string())? - { - NetworkPolicyTarget::Template(template) => templates.push(template.to_string()), - NetworkPolicyTarget::Sandbox(sandbox_id) => sandbox_ids.push(sandbox_id.to_string()), - } - } - Ok(( - util::dedupe_non_empty(templates), - util::dedupe_non_empty(sandbox_ids), - )) -} - fn proto_node_role(role: Option) -> Option { role.map(|role| role as i32) } -/// Splits one node's network-policy fetch into bounded serial RPCs: templates -/// (few) ride the first request, sandbox ids are batched at `max_batch_size` -/// per request. Serial batches keep at most one in-flight query per node on -/// cluster-api, and each batch is applied as soon as it arrives, so a timeout -/// or error loses only the remaining batches instead of the whole node. -fn network_policy_state_requests( - node_id: &str, - node_role: Option, - templates: Vec, - sandbox_ids: Vec, - max_batch_size: usize, -) -> Vec { - let max_batch_size = max_batch_size.max(1); - let mut batches: Vec> = sandbox_ids - .chunks(max_batch_size) - .map(<[String]>::to_vec) - .collect(); - if batches.is_empty() { - batches.push(Vec::new()); - } - let mut templates = Some(templates); - batches - .into_iter() - .map(|sandbox_ids| GetNodeNetworkPolicyStateRequest { - node_id: node_id.to_string(), - templates: templates.take().unwrap_or_default(), - sandbox_ids, - role: proto_node_role(node_role), +fn record_element_error(errors: &mut BTreeMap, element: &str, error: &str) { + errors + .entry(element.to_string()) + .and_modify(|existing| { + existing.push_str("; "); + existing.push_str(error); }) - .collect() -} - -async fn refresh_network_policy( - node_id: &str, - node_role: Option, - scope: FacetScope, - progress: crate::FacetProgress, -) -> Result { - let (templates, sandbox_ids) = resolve_network_policy_scope(scope).await?; - let mut stats = ReconcileStats { - templates_seen: templates.len() as u64, - sandboxes_seen: sandbox_ids.len() as u64, - ..ReconcileStats::default() - }; - if templates.is_empty() && sandbox_ids.is_empty() { - return Ok(stats); - } - - let requests = network_policy_state_requests( - node_id, - node_role, - templates, - sandbox_ids, - crate::CLUSTER_RPC_MAX_BATCH_SIZE, - ); - let mut client = connect_cluster_api() - .await - .map_err(|error| error.to_string())?; - for request in requests { - let response = client - .get_node_network_policy_state(request) - .await - .map_err(|error| format!("cluster network policy refresh failed: {error}"))? - .into_inner(); - apply_network_policy_state(response, &mut stats).await?; - // Each fetched-and-applied batch is forward progress; the facet - // watchdog budgets time-per-batch, not time-per-run, so large nodes - // don't hit a fixed whole-run deadline. - progress.touch(); - } - Ok(stats) -} - -async fn apply_network_policy_state( - response: GetNodeNetworkPolicyStateResponse, - stats: &mut ReconcileStats, -) -> Result<(), String> { - for template in response.templates { - let Some(policy) = template.policy else { - continue; - }; - match apply_template_policy(&template.template, policy, template.generation, false).await { - Ok(decision) => { - if apply_decision_counts_as_applied(decision) { - stats.templates_applied = stats.templates_applied.saturating_add(1); - } - } - Err(error) => { - return Err(format!( - "failed to apply template network policy {}: {error}", - template.template - )); - } - } - } - let sandboxes = response.sandboxes; - let local_sandbox_policies = fetch_local_sandbox_network_policies(&sandboxes).await?; - for sandbox in sandboxes { - let sandbox_id = - parse_sandbox_id(&sandbox.sandbox_id).map_err(|error| error.to_string())?; - if sandbox_policy_reconcile_action(&sandbox, &sandbox_id, &local_sandbox_policies) - == SandboxPolicyReconcileAction::Skip - { - continue; - } - let source = NetworkPolicySource::try_from(sandbox.source) - .unwrap_or(NetworkPolicySource::Unspecified); - let inherit_template = source == NetworkPolicySource::Template; - let policy = if inherit_template { - None - } else { - sandbox.policy - }; - match apply_sandbox_policy( - &sandbox.sandbox_id, - policy, - inherit_template, - sandbox.generation, - false, - ) - .await - { - Ok(decision) => { - if apply_decision_counts_as_applied(decision) { - stats.sandboxes_applied = stats.sandboxes_applied.saturating_add(1); - } - } - Err(error) => { - return Err(format!( - "failed to apply sandbox network policy {}: {error}", - sandbox.sandbox_id - )); - } - } - } - Ok(()) -} - -async fn fetch_local_sandbox_network_policies( - sandboxes: &[SandboxNetworkPolicyState], -) -> Result { - if sandboxes.is_empty() { - return Ok(LocalSandboxNetworkPolicies::default()); - } - - let sandbox_ids = sandboxes - .iter() - .map(|sandbox| parse_sandbox_id(&sandbox.sandbox_id)) - .collect::, _>>() - .map_err(|error| error.to_string())?; - let mut client = pplx_space_sbx_mgr_client::connect() - .await - .map_err(|error| format!("connect sbx-manager: {error}"))?; - match fetch_sandbox_network_policies(&mut client, &sandbox_ids).await { - Ok(policies) => Ok(LocalSandboxNetworkPolicies::from_policies(policies)), - Err(error) if should_fallback_to_individual_policy_fetch(&error) => { - tracing::debug!( - %error, - "batched sandbox network policy fetch hit a missing sandbox; retrying individually" - ); - fetch_local_sandbox_network_policies_individually(&mut client, &sandbox_ids).await - } - Err(error) => Err(error.to_string()), - } -} - -async fn fetch_local_sandbox_network_policies_individually( - client: &mut pplx_space_sbx_mgr_client::SandboxManagerGrpcClient, - sandbox_ids: &[SandboxId], -) -> Result { - let mut local = LocalSandboxNetworkPolicies::default(); - for sandbox_id in sandbox_ids { - match fetch_sandbox_network_policies(client, std::slice::from_ref(sandbox_id)).await { - Ok(mut policies) => { - if let Some(policy) = policies.remove(sandbox_id) { - local.policies.insert(*sandbox_id, policy); - } else { - tracing::debug!( - %sandbox_id, - "sandbox-manager returned no network policy for local sandbox" - ); - local.insert_missing(*sandbox_id); - } - } - Err(error) if should_fallback_to_individual_policy_fetch(&error) => { - tracing::debug!( - %sandbox_id, - "skipping network policy reconcile for sandbox missing locally" - ); - local.insert_missing(*sandbox_id); - } - Err(error) => return Err(error.to_string()), - } - } - Ok(local) -} - -fn should_fallback_to_individual_policy_fetch(error: &SandboxInventoryEnrichmentError) -> bool { - matches!( - error, - SandboxInventoryEnrichmentError::GetNetworkPolicies(status) - if status.code() == tonic::Code::NotFound - ) -} - -fn sandbox_policy_reconcile_action( - desired: &SandboxNetworkPolicyState, - sandbox_id: &SandboxId, - local: &LocalSandboxNetworkPolicies, -) -> SandboxPolicyReconcileAction { - if local.is_missing(sandbox_id) { - return SandboxPolicyReconcileAction::Skip; - } - if sandbox_policy_needs_apply(desired, local.get(sandbox_id)) { - SandboxPolicyReconcileAction::Apply - } else { - SandboxPolicyReconcileAction::Skip - } -} - -fn sandbox_policy_needs_apply( - desired: &SandboxNetworkPolicyState, - local: Option<&NetworkPolicyDetail>, -) -> bool { - let Some(local) = local else { - return true; - }; - - let source = - NetworkPolicySource::try_from(desired.source).unwrap_or(NetworkPolicySource::Unspecified); - let inherit_template = source == NetworkPolicySource::Template; - let expected_applied_source = if inherit_template { - NetworkPolicySource::Template - } else { - NetworkPolicySource::Override - }; - - local.authority != PolicyAuthority::Cluster.as_proto_i32() - || local.inherited != inherit_template - || local.effective_generation != desired.generation - || desired.policy != local.effective_policy - || local.applied_generation != desired.generation - || local.applied_source != expected_applied_source as i32 -} - -fn apply_decision_from_proto(value: i32) -> NetworkPolicyApplyDecision { - NetworkPolicyApplyDecision::try_from(value).unwrap_or(NetworkPolicyApplyDecision::Unspecified) -} - -fn apply_decision_counts_as_applied(decision: NetworkPolicyApplyDecision) -> bool { - matches!( - decision, - NetworkPolicyApplyDecision::Apply | NetworkPolicyApplyDecision::Repair - ) + .or_insert_with(|| error.to_string()); } -async fn refresh_template_base_snapshot_policy( +async fn publish_node_state( node_id: &str, node_role: Option, - scope: FacetScope, + state: NodeStatePublish, ) -> Result { - let templates = resolve_template_scope(scope).await?; - let mut stats = ReconcileStats { - templates_seen: templates.len() as u64, - ..ReconcileStats::default() - }; - if templates.is_empty() { - return Ok(stats); + if state.sandboxes.is_empty() + && state.templates.is_empty() + && !state.sandbox_inventory_complete + && !state.template_inventory_complete + && state.node_status.is_none() + && state.execution_inventory.is_none() + && state.auto_lifecycle_delete_completions.is_empty() + { + return Ok(ReconcileStats { + sandboxes_seen: state.sandboxes_seen, + templates_seen: state.templates_seen, + ..ReconcileStats::default() + }); } + let templates_seen = state.templates_seen; + let sandboxes_seen = state.sandboxes_seen; let mut client = connect_cluster_api() .await .map_err(|error| error.to_string())?; let response = client - .get_node_template_base_snapshot_policy_state( - GetNodeTemplateBaseSnapshotPolicyStateRequest { - node_id: node_id.to_string(), - templates, - role: proto_node_role(node_role), - }, - ) - .await - .map_err(|error| format!("cluster base-snapshot policy refresh failed: {error}"))? - .into_inner(); - - let mut tmpl = pplx_space_template_mgr_client::connect() - .await - .map_err(|error| format!("connect template-manager: {error}"))?; - let mut failures = 0_u64; - for template in response.templates { - if let Err(error) = tmpl - .update_template_snapshot_policy(local_manager_request( - UpdateTemplateSnapshotPolicyRequest { - template: template.template.clone(), - ensure_base_snapshot: template.ensure_base_snapshot, - snapshot_vcpu: Some(template.snapshot_vcpu), - snapshot_memory_mib: Some(template.snapshot_memory_mib), - }, - false, - )) - .await - { - tracing::warn!( - template = %template.template, - %error, - "failed to apply template base-snapshot policy" - ); - failures = failures.saturating_add(1); - continue; - } - stats.templates_applied = stats.templates_applied.saturating_add(1); - } - if failures > 0 { - return Err(format!( - "failed to apply {failures} template base-snapshot policy item(s)" - )); - } - Ok(stats) -} - -async fn refresh_template_desired_state( - node_id: &str, - node_role: Option, - scope: FacetScope, -) -> Result { - let templates = resolve_template_scope(scope).await?; - let mut stats = ReconcileStats { - templates_seen: templates.len() as u64, - ..ReconcileStats::default() - }; - if templates.is_empty() { - return Ok(stats); - } - - let mut cluster = connect_cluster_api() - .await - .map_err(|error| error.to_string())?; - let response = cluster - .get_node_template_desired_state(GetNodeTemplateDesiredStateRequest { - node_id: node_id.to_string(), - templates, - role: proto_node_role(node_role), - }) - .await - .map_err(|error| format!("cluster template desired-state refresh failed: {error}"))? - .into_inner(); - - let mut tmpl = pplx_space_template_mgr_client::connect() - .await - .map_err(|error| format!("connect template-manager: {error}"))?; - let local_templates = tmpl - .list_templates(Request::new(ListTemplatesRequest {})) - .await - .map_err(|error| format!("list local templates: {error}"))? - .into_inner() - .templates - .into_iter() - .map(|template| (template.name.clone(), template.default_auto_lifecycle)) - .collect::>(); - let local_pools = tmpl - .pool_status(Request::new(PoolStatusRequest { template: None })) - .await - .map(|response| { - response - .into_inner() - .pools - .into_iter() - .map(|pool| (pool.template.clone(), pool)) - .collect::>() - }) - .unwrap_or_default(); - - let mut failures = 0_u64; - for template in response.templates { - let mut applied = false; - if template.present == Some(false) { - if local_pools.contains_key(&template.template) { - match tmpl - .delete_pool(local_manager_request( - DeletePoolRequest { - template: template.template.clone(), - }, - true, - )) - .await - { - Ok(_) => { - applied = true; - } - Err(error) => { - tracing::warn!( - template = %template.template, - %error, - "failed to delete undesired template pool" - ); - failures = failures.saturating_add(1); - } - } - } - if local_templates.contains_key(&template.template) { - match tmpl - .delete_template(local_manager_request( - DeleteTemplateRequest { - template: template.template.clone(), - }, - true, - )) - .await - { - Ok(_) => { - applied = true; - } - Err(error) if error.code() == Code::NotFound => {} - Err(error) => { - tracing::warn!( - template = %template.template, - %error, - "failed to delete undesired template" - ); - failures = failures.saturating_add(1); - } - } - } - if applied { - stats.templates_applied = stats.templates_applied.saturating_add(1); - } - continue; - } - - if !local_templates.contains_key(&template.template) { - if local_pools.contains_key(&template.template) { - match tmpl - .delete_pool(local_manager_request( - DeletePoolRequest { - template: template.template.clone(), - }, - true, - )) - .await - { - Ok(_) => { - stats.templates_applied = stats.templates_applied.saturating_add(1); - } - Err(error) => { - tracing::warn!( - template = %template.template, - %error, - "failed to delete pool for absent local template" - ); - failures = failures.saturating_add(1); - } - } - } - continue; - } - - if let Some(policy) = template.default_auto_lifecycle.clone() { - let current = local_templates - .get(&template.template) - .and_then(|policy| policy.as_ref()); - if current != Some(&policy) { - match tmpl - .update_template_settings(local_manager_request( - UpdateTemplateSettingsRequest { - template: template.template.clone(), - default_auto_lifecycle: Some(policy), - network_policy: None, - network_policy_generation: 0, - network_policy_authority: PolicyAuthority::Cluster.as_proto_i32(), - auto_lifecycle_authority: PolicyAuthority::Cluster.as_proto_i32(), - }, - true, - )) - .await - { - Ok(_) => { - applied = true; - } - Err(error) => { - tracing::warn!( - template = %template.template, - %error, - "failed to apply template default auto-lifecycle policy" - ); - failures = failures.saturating_add(1); - } - } - } - } - - match template.pool { - Some(pool) => match tmpl - .create_pool(local_manager_request( - CreatePoolRequest { - template: template.template.clone(), - target_count: Some(pool.target_count), - boot_mode: pool.boot_mode, - vcpu: pool.vcpu, - memory_mib: pool.memory_mib, - }, - true, - )) - .await - { - Ok(_) => { - applied = true; - } - Err(error) => { - tracing::warn!( - template = %template.template, - %error, - "failed to apply template pool policy" - ); - failures = failures.saturating_add(1); - } - }, - None => { - if local_pools.contains_key(&template.template) { - match tmpl - .delete_pool(local_manager_request( - DeletePoolRequest { - template: template.template.clone(), - }, - true, - )) - .await - { - Ok(_) => { - applied = true; - } - Err(error) => { - tracing::warn!( - template = %template.template, - %error, - "failed to delete undesired template pool" - ); - failures = failures.saturating_add(1); - } - } - } - } - } - - if applied { - stats.templates_applied = stats.templates_applied.saturating_add(1); - } - } - - if failures > 0 { - return Err(format!( - "failed to apply {failures} template desired-state item(s)" - )); - } - Ok(stats) -} - -async fn refresh_sandbox_desired_state( - node_id: &str, - node_role: Option, - scope: FacetScope, -) -> Result { - let sandbox_ids = resolve_sandbox_scope(scope).await?; - let mut stats = ReconcileStats { - sandboxes_seen: sandbox_ids.len() as u64, - ..ReconcileStats::default() - }; - if sandbox_ids.is_empty() { - return Ok(InboundFacetResult::from_stats(stats)); - } - - let mut cluster = connect_cluster_api() - .await - .map_err(|error| error.to_string())?; - let response = cluster - .get_node_sandbox_desired_state(GetNodeSandboxDesiredStateRequest { + .publish_node_state(PublishNodeStateRequest { node_id: node_id.to_string(), - sandbox_ids, + sandbox_inventory_complete: state.sandbox_inventory_complete, + sandboxes: state.sandboxes, + template_inventory_complete: state.template_inventory_complete, + templates: state.templates, role: proto_node_role(node_role), + node_status: state.node_status, + execution_inventory: state.execution_inventory, + auto_lifecycle_delete_completions: state.auto_lifecycle_delete_completions, + pending_local_backups: state.pending_local_backups, }) .await - .map_err(|error| format!("cluster sandbox desired-state refresh failed: {error}"))? + .map_err(|error| format!("cluster node state publish failed: {error}"))? .into_inner(); - let has_port_exposures = response - .sandboxes - .iter() - .any(|sandbox| !sandbox.port_exposures.is_empty()); - let mut sbx = pplx_space_sbx_mgr_client::connect() - .await - .map_err(|error| format!("connect sbx-manager: {error}"))?; - // Deleted ids are included so the eviction step below can gate on the - // sandbox's current local state without a second State round-trip. - let local_state_lookup_ids = response - .sandboxes - .iter() - .map(|sandbox| sandbox.sandbox_id.clone()) - .chain(response.deleted_sandbox_ids.iter().cloned()) - .collect::>(); - let local_sandbox_state_by_id = fetch_local_sandbox_states(&mut sbx, local_state_lookup_ids) - .await - .map_err(|error| format!("sandbox desired-state shared state lookup failed: {error}"))?; - let mut node = has_port_exposures.then(pplx_space_node_mgr_client::lazy); - let local_sandbox_allocations = if let Some(node) = node.as_mut() { - local_sandbox_allocation_ids(node).await.map_err(|error| { - format!("sandbox desired-state shared allocation lookup failed: {error}") - })? - } else { - BTreeSet::new() - }; - let mut element_errors = BTreeMap::new(); - let mut exposure_apply_states = Vec::new(); - for sandbox in response.sandboxes { - let sandbox_id = sandbox - .sandbox_id - .parse::() - .map_err(|error| format!("invalid sandbox desired-state id: {error}"))?; - let has_sandbox_port_exposures = !sandbox.port_exposures.is_empty(); - let local_sandbox_state = local_sandbox_state_by_id.get(&sandbox.sandbox_id); - let action = sandbox_desired_reconcile_action( - local_sandbox_state.is_some(), - has_sandbox_port_exposures, - local_sandbox_allocations.contains(&sandbox_id), - ); - if action == SandboxDesiredReconcileAction::SkipNonLocal { - tracing::debug!( - sandbox_id = %sandbox.sandbox_id, - exposures = sandbox.port_exposures.len(), - "skipping desired state for sandbox that is no longer local" - ); - continue; - } - if let Some(local_sandbox_state) = local_sandbox_state { - handle_local_desired_state_apply( - apply_sandbox_snapshot_desired( - &mut sbx, - &sandbox.sandbox_id, - local_sandbox_state, - sandbox.snapshots_enabled_desired, - ) - .await, - &sandbox.sandbox_id, - &mut stats, - &mut element_errors, - ); - if let Some(policy) = sandbox.auto_lifecycle { - handle_local_desired_state_apply( - apply_sandbox_auto_lifecycle_desired( - &mut sbx, - &sandbox.sandbox_id, - local_sandbox_state, - policy, - ) - .await, - &sandbox.sandbox_id, - &mut stats, - &mut element_errors, - ); - } - } else { - tracing::warn!( - sandbox_id = %sandbox.sandbox_id, - exposures = sandbox.port_exposures.len(), - "local sandbox state is unavailable; applying port exposures and retrying the ref" - ); - } - if let Some(node) = node.as_mut() { - reconcile_sandbox_port_exposures( - PortExposureReconcileMode { action }, - node, - &sandbox.sandbox_id, - sandbox.port_exposures, - &mut stats, - &mut element_errors, - &mut exposure_apply_states, - ) - .await; - } - } - if !exposure_apply_states.is_empty() { - match cluster - .publish_sandbox_port_exposure_states(PublishSandboxPortExposureStatesRequest { - node_id: node_id.to_string(), - states: exposure_apply_states, - role: proto_node_role(node_role), - }) - .await - { - Ok(response) => { - for error in response.into_inner().errors { - tracing::warn!( - sandbox_id = %error.sandbox_id, - protocol = port_protocol_label(error.protocol), - guest_port = error.guest_port, - error = %error.error, - "failed to publish sandbox port exposure apply state" - ); - record_element_error( - &mut element_errors, - &error.sandbox_id, - &format!("publish port exposure apply state: {}", error.error), - ); - } - } - Err(error) => { - return Err(format!( - "publish sandbox port exposure apply states: {error}" - )); - } - } - } - if !response.deleted_sandbox_ids.is_empty() { - let evict_cap = CLUSTER_DELETED_EVICTS_PER_SYNC; - let now = std::time::Instant::now(); - CLUSTER_DELETED_EVICT_BACKOFF_UNTIL.retain(|_, until| *until > now); - let candidates = select_cluster_deleted_evict_candidates( - &response.deleted_sandbox_ids, - &local_sandbox_state_by_id, - CLUSTER_DELETED_CANDIDATE_WINDOW, - |sandbox_id| CLUSTER_DELETED_EVICT_BACKOFF_UNTIL.contains_key(sandbox_id), - ); - let started = std::time::Instant::now(); - let mut evicted = 0_u64; - let mut deferred = 0_u64; - let mut budget_exhausted = false; - for sandbox_id in candidates { - if evicted >= evict_cap as u64 { - break; - } - if started.elapsed() > CLUSTER_DELETED_EVICT_SYNC_BUDGET { - budget_exhausted = true; - break; - } - let call = sbx.evict_cluster_deleted_sandbox(local_manager_request( - EvictClusterDeletedSandboxRequest { - sandbox_id: sandbox_id.to_string(), - }, - true, - )); - let result = match tokio::time::timeout(CLUSTER_DELETED_EVICT_RPC_TIMEOUT, call).await { - Ok(result) => result, - // The teardown runs detached in sbx-manager, so an - // elapsed deadline means "still in progress", not - // failure; the next sync observes the outcome. - Err(_elapsed) => Err(tonic::Status::deadline_exceeded( - "eviction still in progress at the per-RPC deadline", - )), - }; - let status = match &result { - Ok(_) => { - evicted = evicted.saturating_add(1); - stats.sandboxes_applied = stats.sandboxes_applied.saturating_add(1); - pplx_space_metrics::reconcile::cluster_deleted_evict( - pplx_space_metrics::reconcile::ClusterDeletedEvictResult::Evicted, - ); - continue; - } - Err(status) => status, - }; - match cluster_deleted_evict_refusal_disposition(status) { - ClusterDeletedEvictDisposition::RetryNextSync => { - deferred = deferred.saturating_add(1); - pplx_space_metrics::reconcile::cluster_deleted_evict( - pplx_space_metrics::reconcile::ClusterDeletedEvictResult::Deferred, - ); - tracing::debug!( - sandbox_id, - status = %status, - "cluster-deleted eviction deferred to the next sync: restore staging in flight" - ); - } - ClusterDeletedEvictDisposition::Deferred(backoff) => { - deferred = deferred.saturating_add(1); - CLUSTER_DELETED_EVICT_BACKOFF_UNTIL - .insert(sandbox_id.to_string(), std::time::Instant::now() + backoff); - pplx_space_metrics::reconcile::cluster_deleted_evict( - pplx_space_metrics::reconcile::ClusterDeletedEvictResult::Deferred, - ); - tracing::debug!( - sandbox_id, - status = %status, - backoff_secs = backoff.as_secs(), - "cluster-deleted eviction deferred" - ); - } - ClusterDeletedEvictDisposition::Failed(backoff) => { - CLUSTER_DELETED_EVICT_BACKOFF_UNTIL - .insert(sandbox_id.to_string(), std::time::Instant::now() + backoff); - pplx_space_metrics::reconcile::cluster_deleted_evict( - pplx_space_metrics::reconcile::ClusterDeletedEvictResult::Failed, - ); - tracing::warn!( - sandbox_id, - status = %status, - "cluster-deleted eviction failed" - ); - record_element_error( - &mut element_errors, - sandbox_id, - &format!("cluster-deleted eviction failed: {status}"), - ); - } - } - } - if evicted > 0 || deferred > 0 || budget_exhausted { - tracing::info!( - deleted_reported = response.deleted_sandbox_ids.len(), - evicted, - deferred, - budget_exhausted, - per_sync_cap = evict_cap, - "evicted local presence of cluster-deleted sandboxes" - ); - } - } + acknowledge_auto_lifecycle_delete_completions( + node_role, + &response.acknowledged_auto_lifecycle_delete_receipt_ids, + ) + .await; - Ok(InboundFacetResult { - stats, - element_errors, + Ok(ReconcileStats { + templates_seen, + sandboxes_seen, + templates_applied: response.templates_accepted, + sandboxes_applied: response.sandboxes_accepted, }) } -fn record_element_error(errors: &mut BTreeMap, element: &str, error: &str) { - errors - .entry(element.to_string()) - .and_modify(|existing| { - existing.push_str("; "); - existing.push_str(error); - }) - .or_insert_with(|| error.to_string()); -} - -/// Cap on successful cluster-deleted evictions per sync tick. -/// Bounds both the teardown work added to one facet pass and the blast -/// radius of a bad cluster answer; the backlog drains across syncs. -const CLUSTER_DELETED_EVICTS_PER_SYNC: usize = 25; - -/// How many candidates one sync may examine while filling the eviction -/// cap. Wider than the cap so rows that defer within one sync cannot -/// monopolize the window; the per-candidate backoff below handles rows -/// that keep deferring across syncs. -const CLUSTER_DELETED_CANDIDATE_WINDOW: usize = 100; - -/// Per-candidate retry backoff: a candidate that deferred or failed is -/// skipped by selection for this long, mirroring the disk-pressure -/// eviction loop's candidate backoff, so a permanently-deferring oldest -/// prefix (e.g. malformed `updated_at` rows) rotates out of the window -/// instead of starving the drainable backlog sorted behind it. Process -/// lifetime, in-memory: a node-ctl restart merely retries earlier. -const CLUSTER_DELETED_EVICT_BACKOFF: std::time::Duration = std::time::Duration::from_secs(30 * 60); - -/// Slack added to a server-supplied retry horizon so the retry lands on a -/// sync *after* the horizon passes rather than racing it. Sized to the -/// deployed default poll interval as a fixed safety margin: the interval is -/// runtime-configurable, so a shorter one just skips a few more syncs and a -/// longer one still adds less than one interval of margin; neither breaks -/// the schedule, it only shifts the retry by at most one extra sync. -const CLUSTER_DELETED_EVICT_RETRY_SLACK: std::time::Duration = - std::time::Duration::from_secs(crate::DEFAULT_POLL_INTERVAL_SECS); - -/// What one refused `EvictClusterDeletedSandbox` means for the per-id retry -/// schedule (a successful claim needs no schedule: the teardown runs -/// detached). Pure so the routing is testable without a live sbx-manager. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum ClusterDeletedEvictDisposition { - /// Retry on the very next sync, no back-off: sbx-manager found a - /// restore staging pull live on the row at claim time (`Aborted`), - /// which ends on the pull's own timescale. - RetryNextSync, - /// Skip the id for this long, then retry. Not an element failure. - Deferred(std::time::Duration), - /// The eviction failed outright; back off and report an element error. - Failed(std::time::Duration), -} - -/// Route a refused eviction RPC. `NotFound` (the local row vanished), -/// `FailedPrecondition` (row transitioned / inside the quiescence window / -/// refused as local-provenance or pool), `Unimplemented` (rollout skew), -/// `Unavailable` (sbx-manager draining) and `DeadlineExceeded` (teardown -/// still running detached) all defer. A refusal that carries a retry -/// horizon — sbx-manager's quiescence refusal says how long the window has -/// left — defers exactly that long plus one sync of slack, so a fresh -/// cluster-deleted row is reclaimed at quiescence end (~11 min after the -/// delete) instead of after the flat 30 min back-off that stays in place -/// for refusals with no horizon. Everything else is a failure. -fn cluster_deleted_evict_refusal_disposition( - status: &tonic::Status, -) -> ClusterDeletedEvictDisposition { - match status.code() { - Code::Aborted => ClusterDeletedEvictDisposition::RetryNextSync, - Code::NotFound - | Code::FailedPrecondition - | Code::Unimplemented - | Code::Unavailable - | Code::DeadlineExceeded => ClusterDeletedEvictDisposition::Deferred( - pplx_space_common_grpc::retry_after(status) - .map(|horizon| horizon.saturating_add(CLUSTER_DELETED_EVICT_RETRY_SLACK)) - .map_or(CLUSTER_DELETED_EVICT_BACKOFF, |until| { - until.min(CLUSTER_DELETED_EVICT_BACKOFF) - }), - ), - _ => ClusterDeletedEvictDisposition::Failed(CLUSTER_DELETED_EVICT_BACKOFF), - } -} - -static CLUSTER_DELETED_EVICT_BACKOFF_UNTIL: std::sync::LazyLock< - dashmap::DashMap, -> = std::sync::LazyLock::new(dashmap::DashMap::new); - -/// Wall-clock budget for the eviction pass inside one facet sync. The -/// whole inbound facet races a 30s cancel-by-drop timeout in the work -/// loop; stopping well inside it leaves room for the desired-state applies -/// that share the tick and carries the remainder to the next sync. -const CLUSTER_DELETED_EVICT_SYNC_BUDGET: std::time::Duration = std::time::Duration::from_secs(15); - -/// Per-eviction RPC deadline. The teardown is detached server-side, so -/// this bounds slot occupancy, not the teardown itself. -const CLUSTER_DELETED_EVICT_RPC_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); - -/// Pick which cluster-deleted sandbox ids to evict locally this tick: -/// only ids whose local sbx-manager row is currently `suspended` (the -/// leaked wake-cache shape), `restoring` with a persisted staging -/// failure (the row a failed cross-node restore retains on the target -/// "for retry" — a retry can never come once the cluster row is gone, and -/// no other path reclaims it), or `suspending` with a permanently refused -/// teardown backup (the row a pod teardown leaves when the sandbox was -/// deleted mid-activation, whose suspend can never publish), owned by a -/// platform user (`user:` — -/// positive evidence the cluster minted the sandbox; local-provenance -/// rows like `root@local` or `tmplmgr@local` never had a cluster row, so -/// its absence proves nothing about them), and not a pool slot. A -/// `restoring` row without the failure marker is a first-attempt restore -/// and is left alone, as is a `suspending` row without the terminal -/// teardown-backup marker, and anything else mid-transition, failed, or -/// already absent locally. Neither marker is a liveness proof — a retry of -/// the failed restore reuses the row without clearing it. sbx-manager re-checks the owner, pool and quiescence guards, takes -/// the staging lock non-blocking to prove no retry pull is live, and claims -/// the row at the exact version it read before tearing anything down — but -/// it makes no cluster call of its own: the `deleted_sandbox_ids` report -/// this selection is drawn from IS the authority that the cluster row is -/// gone, so getting that report right is this caller's responsibility, not -/// something sbx-manager can recover from. -/// Oldest first — UUIDv7 ids sort by creation time — so the largest -/// accumulated dead weight drains ahead of fresh arrivals. -fn select_cluster_deleted_evict_candidates<'a>( - deleted_sandbox_ids: &'a [String], - local_states: &BTreeMap, - window: usize, - is_backed_off: impl Fn(&str) -> bool, -) -> Vec<&'a str> { - let mut candidates = deleted_sandbox_ids - .iter() - .filter(|sandbox_id| !is_backed_off(sandbox_id)) - .filter(|sandbox_id| { - local_states.get(*sandbox_id).is_some_and(|state| { - local_row_is_cluster_deleted_reclaimable(state) - && state.pool_template.is_none() - && state - .owner - .as_deref() - .and_then(|owner| SandboxOwner::parse(owner).ok()) - .is_some_and(|owner| owner.is_platform_user()) - }) - }) - .map(String::as_str) - .collect::>(); - candidates.sort_unstable(); - candidates.dedup(); - candidates.truncate(window); - candidates -} - -/// The local row shapes a cluster-deleted report may reclaim: a -/// stopped rootfs, a suspended wake-cache copy, a failed cross-node restore's retained -/// `restoring` row (identified by any persisted `failure_reason`; a -/// whitespace-only reason is treated as no marker, matching sbx-manager), -/// or a `suspending` row whose teardown suspend had its durable backup -/// permanently refused, or a quarantined row retained for a deleted sandbox. -/// -/// The `suspending` shape matches one exact reason rather than "any -/// failure", because that is the only `suspending` failure that ends the -/// row's own publication path for good. Other reasons sit on suspends -/// recovery can still finish, and offering them would spend eviction -/// budget and back-off slots on ids sbx-manager is bound to refuse. -fn local_row_is_cluster_deleted_reclaimable(state: &SandboxInfo) -> bool { - use pplx_space_sbx_mgr_client::proto::SandboxLifecycleState; - match state.observed_lifecycle().ok() { - Some( - SandboxLifecycleState::Stopped - | SandboxLifecycleState::Suspended - | SandboxLifecycleState::Quarantined, - ) => true, - Some(SandboxLifecycleState::Restoring) => state - .failure_reason - .as_deref() - .is_some_and(|reason| !reason.trim().is_empty()), - Some(SandboxLifecycleState::Suspending) => { - state.failure_reason.as_deref() - == Some( - pplx_space_sbx_mgr_client::TEARDOWN_SUSPEND_BACKUP_PERMANENTLY_FAILED_REASON, - ) - } - _ => false, - } -} - -async fn local_sandbox_allocation_ids( - client: &mut pplx_space_node_mgr_client::NodeManagerGrpcClient, -) -> Result, String> { - let response = client - .list_network_leases(Request::new(ListNetworkLeasesRequest {})) - .await - .map_err(|error| format!("list local network leases: {error}"))? - .into_inner(); - let mut sandbox_ids = BTreeSet::new(); - for lease in response.leases { - if !network_slot_state_has_live_allocation(lease.slot_state) - || lease.sandbox_id.trim().is_empty() - { - continue; - } - let sandbox_id = lease - .sandbox_id - .parse::() - .map_err(|error| format!("invalid local network lease sandbox id: {error}"))?; - sandbox_ids.insert(sandbox_id); - } - Ok(sandbox_ids) -} - -async fn fetch_local_sandbox_states( - client: &mut pplx_space_sbx_mgr_client::SandboxManagerGrpcClient, - sandbox_ids: Vec, -) -> Result, String> { - if sandbox_ids.is_empty() { - return Ok(BTreeMap::new()); - } - let response = fetch_sandbox_states(client, &sandbox_ids).await?; - Ok(response - .sandboxes - .into_iter() - .filter(|sandbox| !sandbox.sandbox_id.trim().is_empty()) - .map(|sandbox| (sandbox.sandbox_id.clone(), sandbox)) - .collect()) -} - -fn network_slot_state_has_live_allocation(slot_state: i32) -> bool { - matches!( - NetworkSlotState::try_from(slot_state).unwrap_or(NetworkSlotState::Unspecified), - NetworkSlotState::Leasing | NetworkSlotState::Leased - ) -} - -async fn publish_network_policy( - node_id: &str, - node_role: Option, - scope: FacetScope, - node_status: Option, -) -> Result { - let (templates, sandbox_ids) = resolve_network_policy_publish_scope(scope)?; - let (sandboxes, sandboxes_seen) = - collect_sandbox_states(sandbox_ids, SandboxPublishPayload::NetworkPolicy).await?; - let (templates, templates_seen) = collect_template_states(templates).await?; - publish_node_state( - node_id, - node_role, - NodeStatePublish { - sandbox_inventory_complete: false, - sandboxes, - template_inventory_complete: false, - templates, - sandboxes_seen, - templates_seen, - node_status, - execution_inventory: None, - auto_lifecycle_delete_completions: Vec::new(), - pending_local_backups: None, - }, - ) - .await -} - -async fn publish_sandbox_inventory( - node_id: &str, - node_role: Option, - scope: FacetScope, - node_status: Option, -) -> Result { - let complete = matches!(scope, FacetScope::Full); - let (sandboxes, sandboxes_seen) = - collect_sandbox_states(scope, SandboxPublishPayload::Inventory).await?; - let auto_lifecycle_delete_completions = - collect_auto_lifecycle_delete_completions(node_role).await; - // The report rides the Full census only. Not because a delta's list - // would be short — `ListPendingLocalBackups` answers for the whole - // node either way — but because the contract this field is published - // under (see `PublishNodeStateRequest` in - // `proto/node/api/v1/cluster.proto`) lets a consumer cross-check the - // report against the sandbox rows in the same publication, and a delta - // carries only a slice of those. - // - // The report is also strictly the newest thing in the publication: - // `collect_sandbox_states` and the delete-receipt collector each made - // their own round trip before this one. A sandbox that reaches - // suspended-with-pin in between is still *present* in this - // publication's `sandboxes` — the Full scope enumerates from sbx-mgr - // `List`, whose query is `FROM sandboxes ORDER BY created_at` with no - // state filter, so the row was picked up while it was still running. - // Present, not missing — but its lifecycle is whatever the `State` - // call saw, and `State` is only the second of several round trips - // before the report: disk usage, snapshots and the delete receipts - // all follow it. A sandbox that suspends after `State` therefore - // ships a Running row beside a pending entry naming it. Genuine - // absence is narrower still, needing a sandbox created *and* pinned - // entirely inside the gap: a create plus a stop or suspend between - // two consecutive RPCs. - // - // The order is still the better one. Reversed, a sandbox that suspends - // after the report but before `collect_sandbox_states` ships a - // freshly-suspended row whose pending entry is missing — a publication - // that contradicts itself in the direction of under-reporting. As - // written, the contradiction runs the other way: an entry whose row is - // a moment stale, which is the better shape to hand a reader than an - // absence. It does put an obligation on that reader, though, and this - // is the only place the obligation is visible: **a pending entry can - // name an artifact newer than anything its own sandbox row reflects**, - // so the row's backup marker must never be read as evidence about the - // reported artifact. The next census reconciles either way. - let pending_local_backups = if complete { - collect_pending_local_backups().await - } else { - None - }; - publish_node_state( - node_id, - node_role, - NodeStatePublish { - sandbox_inventory_complete: complete, - sandboxes, - template_inventory_complete: false, - templates: Vec::new(), - sandboxes_seen, - templates_seen: 0, - node_status, - execution_inventory: None, - auto_lifecycle_delete_completions, - pending_local_backups, - }, - ) - .await -} - -async fn publish_template_inventory( - node_id: &str, - node_role: Option, - scope: FacetScope, - node_status: Option, -) -> Result { - let complete = matches!(scope, FacetScope::Full); - let (templates, templates_seen) = collect_template_states(scope).await?; - publish_node_state( - node_id, - node_role, - NodeStatePublish { - sandbox_inventory_complete: false, - sandboxes: Vec::new(), - template_inventory_complete: complete, - templates, - sandboxes_seen: 0, - templates_seen, - node_status, - execution_inventory: None, - auto_lifecycle_delete_completions: Vec::new(), - pending_local_backups: None, - }, - ) - .await -} - -async fn publish_node_status( - node_id: &str, - node_role: Option, - node_status: Option, -) -> Result { - let execution_inventory = match pplx_space_node_mgr_client::connect().await { - Ok(mut node_manager) => match node_manager - .get_execution_inventory(local_manager_request( - GetExecutionInventoryRequest {}, - false, - )) - .await - { - Ok(response) => Some(response.into_inner()), - Err(error) => { - tracing::warn!( - %error, - "skipping execution inventory publish because node-manager inventory is unavailable" - ); - None - } - }, - Err(error) => { - tracing::warn!( - %error, - "skipping execution inventory publish because node-manager is unavailable" - ); - None - } - }; - publish_node_state( - node_id, - node_role, - NodeStatePublish { - sandbox_inventory_complete: false, - sandboxes: Vec::new(), - template_inventory_complete: false, - templates: Vec::new(), - sandboxes_seen: 0, - templates_seen: 0, - node_status, - execution_inventory, - auto_lifecycle_delete_completions: Vec::new(), - pending_local_backups: None, - }, - ) - .await -} - -async fn publish_node_state( - node_id: &str, - node_role: Option, - state: NodeStatePublish, -) -> Result { - if state.sandboxes.is_empty() - && state.templates.is_empty() - && !state.sandbox_inventory_complete - && !state.template_inventory_complete - && state.node_status.is_none() - && state.execution_inventory.is_none() - && state.auto_lifecycle_delete_completions.is_empty() - { - return Ok(ReconcileStats { - sandboxes_seen: state.sandboxes_seen, - templates_seen: state.templates_seen, - ..ReconcileStats::default() - }); - } - - let templates_seen = state.templates_seen; - let sandboxes_seen = state.sandboxes_seen; - let mut client = connect_cluster_api() - .await - .map_err(|error| error.to_string())?; - let response = client - .publish_node_state(PublishNodeStateRequest { - node_id: node_id.to_string(), - sandbox_inventory_complete: state.sandbox_inventory_complete, - sandboxes: state.sandboxes, - template_inventory_complete: state.template_inventory_complete, - templates: state.templates, - role: proto_node_role(node_role), - node_status: state.node_status, - execution_inventory: state.execution_inventory, - auto_lifecycle_delete_completions: state.auto_lifecycle_delete_completions, - pending_local_backups: state.pending_local_backups, - }) - .await - .map_err(|error| format!("cluster node state publish failed: {error}"))? - .into_inner(); - - acknowledge_auto_lifecycle_delete_completions( - node_role, - &response.acknowledged_auto_lifecycle_delete_receipt_ids, - ) - .await; - - Ok(ReconcileStats { - templates_seen, - sandboxes_seen, - templates_applied: response.templates_accepted, - sandboxes_applied: response.sandboxes_accepted, - }) -} - -/// How long one census will spend obtaining its pending-backup report. -/// -/// Both legs below are otherwise generous: `connect()` carries the shared -/// 10-second connect-retry budget, and the sandbox-manager channel's own -/// request timeout is 120 seconds. Those are the right numbers for a -/// payload that has to arrive. This one does not — every failure yields -/// `None`, the census publishes anyway, and the next tick regenerates the -/// report (60 s by default, `poll_interval_secs`). Spending ten of the -/// facet watchdog's ninety — its default, `CLUSTER_RPC_TIMEOUT` × 3 — -/// let alone all of them, to produce the same `None` an immediate failure -/// produces would starve the publish the report rides on. -/// -/// Two seconds still covers several of `connect()`'s retry attempts, so a -/// momentary socket blip costs nothing; only a genuinely unavailable -/// manager costs the node a cycle of gate freshness. -const PENDING_LOCAL_BACKUP_BUDGET: Duration = Duration::from_secs(2); - -/// Ask sandbox-manager which of this node's current lifecycle artifacts -/// still have no durable remote copy. -/// -/// Every failure — sandbox-manager unreachable, the RPC erroring, the whole -/// exchange outrunning [`PENDING_LOCAL_BACKUP_BUDGET`], an older -/// sandbox-manager answering `Unimplemented` — yields `None`, which is -/// byte-identical on the wire to what a node built before the field existed -/// sends. Deliberately not an empty report: an empty *report* asserts this -/// disk holds nothing that needs rescuing, and a failed enumeration has not -/// established that. -async fn collect_pending_local_backups() -> Option { - bounded_pending_local_backups(fetch_pending_local_backups()).await -} - -/// The budget, split from the fetch so the expiry arm can be exercised -/// without a sandbox-manager: a test supplies its own slow future. -async fn bounded_pending_local_backups( - fetch: impl Future>, -) -> Option { - match tokio::time::timeout(PENDING_LOCAL_BACKUP_BUDGET, fetch).await { - Ok(report) => report, - Err(_elapsed) => { - tracing::warn!( - budget = ?PENDING_LOCAL_BACKUP_BUDGET, - "gave up listing pending local backups within the census budget" - ); - None - } - } -} - -async fn fetch_pending_local_backups() -> Option { - let mut client = match pplx_space_sbx_mgr_client::connect().await { - Ok(client) => client, - Err(error) => { - tracing::warn!(%error, "cannot connect to sbx-manager to list pending local backups"); - return None; - } - }; - match client - .list_pending_local_backups(Request::new(ListPendingLocalBackupsRequest {})) - .await - { - Ok(response) => { - // `cluster.proto` carries sandbox-manager's own - // `PendingLocalBackup`, so the artifact identities move up - // without being re-encoded — there is no field-by-field copy - // here to fall out of step with the message. - let pending = response.into_inner().pending; - // The counterpart to the warn arms above: they say the node - // could not answer, this says it did and with how many. Without - // it a node-local log cannot tell an empty report apart from an - // enumeration that never ran. - tracing::debug!( - pending_local_backups = pending.len(), - "reporting pending local backups in the full census" - ); - Some(PendingLocalBackupReport { pending }) - } - Err(error) if error.code() == Code::Unimplemented => { - // Not a warning — an older sandbox-manager during a rollout is - // expected. Logged all the same so a census that quietly stops - // carrying the report has a node-local explanation. - tracing::debug!(%error, "sbx-manager does not serve the pending-backup inventory"); - None - } - Err(error) => { - tracing::warn!(%error, "cannot list pending local backups"); - None - } - } -} - -async fn collect_auto_lifecycle_delete_completions( - node_role: Option, -) -> Vec { - if node_role != Some(NodeRole::Worker) { - return Vec::new(); - } - let mut client = match pplx_space_sbx_mgr_client::connect().await { - Ok(client) => client, - Err(error) => { - pplx_space_metrics::reconcile::auto_lifecycle_delete_delivery( - pplx_space_metrics::reconcile::AutoLifecycleDeleteDeliveryOperation::List, - pplx_space_metrics::reconcile::AutoLifecycleDeleteDeliveryOutcome::Failed, - ); - tracing::warn!(%error, "cannot connect to sbx-manager to list delete receipts"); - return Vec::new(); - } - }; - match client - .list_auto_lifecycle_delete_receipts(Request::new( - ListAutoLifecycleDeleteReceiptsRequest {}, - )) - .await - { - Ok(response) => { - pplx_space_metrics::reconcile::auto_lifecycle_delete_delivery( - pplx_space_metrics::reconcile::AutoLifecycleDeleteDeliveryOperation::List, - pplx_space_metrics::reconcile::AutoLifecycleDeleteDeliveryOutcome::Succeeded, - ); - response - .into_inner() - .receipts - .into_iter() - .map(|receipt| AutoLifecycleDeleteCompletion { - receipt_id: receipt.receipt_id, - sandbox_id: receipt.sandbox_id, - execution_generation: receipt.execution_generation, - route_activation_id: receipt.route_activation_id, - auto_lifecycle_policy_fingerprint: receipt.auto_lifecycle_policy_fingerprint, - }) - .collect() - } - Err(error) if error.code() == Code::Unimplemented => { - pplx_space_metrics::reconcile::auto_lifecycle_delete_delivery( - pplx_space_metrics::reconcile::AutoLifecycleDeleteDeliveryOperation::List, - pplx_space_metrics::reconcile::AutoLifecycleDeleteDeliveryOutcome::Unimplemented, - ); - Vec::new() - } - Err(error) => { - pplx_space_metrics::reconcile::auto_lifecycle_delete_delivery( - pplx_space_metrics::reconcile::AutoLifecycleDeleteDeliveryOperation::List, - pplx_space_metrics::reconcile::AutoLifecycleDeleteDeliveryOutcome::Failed, - ); - tracing::warn!(%error, "cannot list auto-lifecycle delete receipts"); - Vec::new() - } - } -} - -async fn acknowledge_auto_lifecycle_delete_completions( - node_role: Option, - receipt_ids: &[String], -) { - if node_role != Some(NodeRole::Worker) || receipt_ids.is_empty() { - return; - } - let mut client = match pplx_space_sbx_mgr_client::connect().await { - Ok(client) => client, - Err(error) => { - pplx_space_metrics::reconcile::auto_lifecycle_delete_delivery( - pplx_space_metrics::reconcile::AutoLifecycleDeleteDeliveryOperation::Acknowledge, - pplx_space_metrics::reconcile::AutoLifecycleDeleteDeliveryOutcome::Failed, - ); - tracing::warn!(%error, "cannot connect to sbx-manager to acknowledge delete receipts"); - return; - } - }; - match client - .acknowledge_auto_lifecycle_delete_receipts(Request::new( - AcknowledgeAutoLifecycleDeleteReceiptsRequest { - receipt_ids: receipt_ids.to_vec(), - }, - )) - .await - { - Ok(_) => { - pplx_space_metrics::reconcile::auto_lifecycle_delete_delivery( - pplx_space_metrics::reconcile::AutoLifecycleDeleteDeliveryOperation::Acknowledge, - pplx_space_metrics::reconcile::AutoLifecycleDeleteDeliveryOutcome::Succeeded, - ); - } - Err(error) => { - pplx_space_metrics::reconcile::auto_lifecycle_delete_delivery( - pplx_space_metrics::reconcile::AutoLifecycleDeleteDeliveryOperation::Acknowledge, - pplx_space_metrics::reconcile::AutoLifecycleDeleteDeliveryOutcome::Failed, - ); - tracing::warn!(%error, "cannot acknowledge auto-lifecycle delete receipts"); - } - } -} - -async fn resolve_network_policy_scope( - scope: FacetScope, -) -> Result<(Vec, Vec), String> { - match scope { - FacetScope::Full => { - let templates = list_local_template_names().await?; - let sandbox_ids = list_local_sandbox_ids().await?; - Ok((templates, sandbox_ids)) - } - FacetScope::Items(items) => split_network_policy_items(items), - } -} - -fn resolve_network_policy_publish_scope( - scope: FacetScope, -) -> Result<(FacetScope, FacetScope), String> { - match scope { - FacetScope::Full => Ok((FacetScope::Full, FacetScope::Full)), - FacetScope::Items(items) => { - let (templates, sandbox_ids) = split_network_policy_items(items)?; - Ok((FacetScope::Items(templates), FacetScope::Items(sandbox_ids))) - } - } -} - -async fn resolve_template_scope(scope: FacetScope) -> Result, String> { - match scope { - FacetScope::Items(items) => Ok(util::dedupe_non_empty(items)), - FacetScope::Full => list_local_template_names().await, - } -} - -async fn resolve_sandbox_scope(scope: FacetScope) -> Result, String> { - match scope { - FacetScope::Items(items) => Ok(util::dedupe_non_empty(items)), - FacetScope::Full => list_local_sandbox_ids().await, - } -} - -async fn list_local_template_names() -> Result, String> { - let mut tmpl = pplx_space_template_mgr_client::connect() - .await - .map_err(|error| format!("connect template-manager: {error}"))?; - let response = tmpl - .list_templates(Request::new(ListTemplatesRequest {})) - .await - .map_err(|error| format!("list templates: {error}"))? - .into_inner(); - Ok(util::dedupe_non_empty( - response - .templates - .into_iter() - .map(|template| template.name) - .collect(), - )) -} - -async fn list_local_sandbox_ids() -> Result, String> { - let mut sbx = pplx_space_sbx_mgr_client::connect() - .await - .map_err(|error| format!("connect sbx-manager: {error}"))?; - let sandboxes = pplx_space_sbx_mgr_client::list_all_sandboxes(&mut sbx) - .await - .map_err(|error| format!("list sandboxes: {error}"))?; - Ok(util::dedupe_non_empty( - sandboxes - .into_iter() - .map(|sandbox| sandbox.sandbox_id) - .collect(), - )) -} - -async fn collect_sandbox_states( - scope: FacetScope, - payload: SandboxPublishPayload, -) -> Result<(Vec, u64), String> { - let mut client = pplx_space_sbx_mgr_client::connect() - .await - .map_err(|error| format!("connect sbx-manager: {error}"))?; - let mut out = Vec::new(); - match scope { - FacetScope::Items(items) => { - let ids = util::dedupe_non_empty(items); - if ids.is_empty() { - return Ok((out, 0)); - } - out.reserve(ids.len()); - let response = fetch_sandbox_states(&mut client, &ids).await?; - let by_id = index_sandbox_state_response(response.sandboxes, &mut out); - let disk_usage_by_id = fetch_sandbox_disk_usage(&mut client, &ids).await; - let policy_by_id = if payload.includes_network_policy() { - let enrichment_ids = by_id - .iter() - .filter_map(|(sandbox_id, sandbox)| { - (!sandbox.state_degraded).then_some(*sandbox_id) - }) - .collect::>(); - fetch_sandbox_network_policies(&mut client, &enrichment_ids) - .await - .map_err(|error| error.to_string())? - } else { - BTreeMap::new() - }; - let snapshot_ids = by_id - .iter() - .filter(|(_, sandbox)| sandbox.snapshots_enabled) - .map(|(sandbox_id, _)| *sandbox_id) - .collect::>(); - let snapshots_by_id = fetch_sandbox_snapshots(&mut client, &snapshot_ids) - .await - .map_err(|error| error.to_string())?; - for (sandbox_id, sandbox) in by_id { - let policy = if payload.includes_network_policy() && !sandbox.state_degraded { - Some( - policy_by_id - .get(&sandbox_id) - .ok_or(SandboxInventoryEnrichmentError::MissingNetworkPolicy( - sandbox_id, - )) - .map_err(|error| error.to_string())?, - ) - } else { - None - }; - let snapshots = snapshots_by_id - .get(&sandbox_id) - .cloned() - .unwrap_or_default(); - out.push(published_sandbox_state( - &sandbox, - disk_usage_by_id.get(&sandbox.sandbox_id).copied(), - policy, - snapshots, - )); - } - for sandbox_id in response.missing_ids { - out.push(absent_sandbox_state(sandbox_id)); - } - } - FacetScope::Full => { - let ids = pplx_space_sbx_mgr_client::list_all_sandboxes(&mut client) - .await - .map_err(|error| format!("list sandboxes: {error}"))? - .into_iter() - .map(|sandbox| sandbox.sandbox_id) - .collect::>(); - if ids.is_empty() { - return Ok((out, 0)); - } - let response = fetch_sandbox_states(&mut client, &ids).await?; - let by_id = index_sandbox_state_response(response.sandboxes, &mut out); - let disk_ids = by_id.keys().map(ToString::to_string).collect::>(); - let disk_usage_by_id = fetch_sandbox_disk_usage(&mut client, &disk_ids).await; - let policy_by_id = if payload.includes_network_policy() { - let enrichment_ids = by_id - .iter() - .filter_map(|(sandbox_id, sandbox)| { - (!sandbox.state_degraded).then_some(*sandbox_id) - }) - .collect::>(); - fetch_sandbox_network_policies(&mut client, &enrichment_ids) - .await - .map_err(|error| error.to_string())? - } else { - BTreeMap::new() - }; - let snapshot_ids = by_id - .iter() - .filter(|(_, sandbox)| sandbox.snapshots_enabled) - .map(|(sandbox_id, _)| *sandbox_id) - .collect::>(); - let snapshots_by_id = fetch_sandbox_snapshots(&mut client, &snapshot_ids) - .await - .map_err(|error| error.to_string())?; - out.reserve(by_id.len()); - for (sandbox_id, sandbox) in by_id { - let policy = if payload.includes_network_policy() && !sandbox.state_degraded { - Some( - policy_by_id - .get(&sandbox_id) - .ok_or(SandboxInventoryEnrichmentError::MissingNetworkPolicy( - sandbox_id, - )) - .map_err(|error| error.to_string())?, - ) - } else { - None - }; - let snapshots = snapshots_by_id - .get(&sandbox_id) - .cloned() - .unwrap_or_default(); - out.push(published_sandbox_state( - &sandbox, - disk_usage_by_id.get(&sandbox.sandbox_id).copied(), - policy, - snapshots, - )); - } - for sandbox_id in response.missing_ids { - out.push(absent_sandbox_state(sandbox_id)); - } - } - } - let seen = out.len() as u64; - Ok((out, seen)) -} - -async fn fetch_sandbox_states( - client: &mut pplx_space_sbx_mgr_client::SandboxManagerGrpcClient, - ids: &[String], -) -> Result { - fetch_sandbox_state_batches(ids, |sandbox_ids| { - let mut client = client.clone(); - async move { - client - .state(Request::new(SbxStateRequest { sandbox_ids })) - .await - .map(tonic::Response::into_inner) - } - }) - .await -} - -/// No partial result escapes on a failed batch: the caller may mark the final -/// publication as a complete census only after every batch has succeeded. -async fn fetch_sandbox_state_batches( - ids: &[String], - mut fetch: impl FnMut(Vec) -> Fut, -) -> Result -where - Fut: Future>, -{ - let mut response = SbxStateResponse::default(); - for ids in ids.chunks(pplx_space_common_core::inventory::READ_BATCH_ITEMS) { - let batch = fetch(ids.to_vec()) - .await - .map_err(|error| format!("get sandbox state: {error}"))?; - response.sandboxes.extend(batch.sandboxes); - response.missing_ids.extend(batch.missing_ids); - } - Ok(response) -} - -fn index_sandbox_state_response( - sandboxes: Vec, - out: &mut Vec, -) -> BTreeMap { - let mut by_id = BTreeMap::new(); - for sandbox in sandboxes { - match parse_sandbox_id(&sandbox.sandbox_id) { - Ok(sandbox_id) => { - by_id.insert(sandbox_id, sandbox); - } - Err(error) => { - tracing::warn!( - sandbox_id = %sandbox.sandbox_id, - %error, - "forwarding malformed sandbox identity so the control plane disables absence pruning" - ); - out.push(published_sandbox_state(&sandbox, None, None, Vec::new())); - } - } - } - by_id -} - -async fn fetch_sandbox_network_policies( - client: &mut pplx_space_sbx_mgr_client::SandboxManagerGrpcClient, - sandbox_ids: &[SandboxId], -) -> Result, SandboxInventoryEnrichmentError> { - if sandbox_ids.is_empty() { - return Ok(BTreeMap::new()); - } - let response = client - .get_network_policy(Request::new(SbxGetNetworkPolicyRequest { - sandbox_ids: sandbox_ids.iter().map(ToString::to_string).collect(), - })) - .await - .map_err(SandboxInventoryEnrichmentError::GetNetworkPolicies)? - .into_inner(); - let mut policies = BTreeMap::new(); - for policy in response.policies { - policies.insert(parse_sandbox_id(&policy.sandbox_id)?, policy); - } - Ok(policies) -} - -async fn fetch_sandbox_snapshots( - client: &mut pplx_space_sbx_mgr_client::SandboxManagerGrpcClient, - sandbox_ids: &[SandboxId], -) -> Result>, SandboxInventoryEnrichmentError> { - if sandbox_ids.is_empty() { - return Ok(BTreeMap::new()); - } - let response = client - .list_snapshots(Request::new(ListSnapshotsRequest { - sandbox_ids: sandbox_ids.iter().map(ToString::to_string).collect(), - })) - .await - .map_err(SandboxInventoryEnrichmentError::ListSnapshots)? - .into_inner(); - let mut snapshots_by_id = BTreeMap::>::new(); - for snapshot in response.snapshots { - snapshots_by_id - .entry(parse_sandbox_id(&snapshot.sandbox_id)?) - .or_default() - .push(snapshot); - } - Ok(snapshots_by_id) -} - -fn parse_sandbox_id(sandbox_id: &str) -> Result { - sandbox_id - .parse() - .map_err(|source| SandboxInventoryEnrichmentError::InvalidSandboxId { - value: sandbox_id.to_string(), - source, - }) -} - -async fn fetch_sandbox_disk_usage( - client: &mut pplx_space_sbx_mgr_client::SandboxManagerGrpcClient, - sandbox_ids: &[String], -) -> BTreeMap { - if sandbox_ids.is_empty() { - return BTreeMap::new(); - } - let mut usage = BTreeMap::new(); - for sandbox_ids in sandbox_ids.chunks(pplx_space_common_core::inventory::READ_BATCH_ITEMS) { - match client - .get_disk_usage(Request::new(SbxGetDiskUsageRequest { - sandbox_ids: sandbox_ids.to_vec(), - })) - .await - { - Ok(response) => usage.extend(response.into_inner().usage_bytes), - Err(error) => { - tracing::warn!(%error, "sandbox disk usage unavailable during inventory publish"); - return BTreeMap::new(); - } - } - } - usage -} - -fn published_sandbox_state( - sandbox: &SandboxInfo, - disk_usage_bytes: Option, - policy: Option<&NetworkPolicyDetail>, - snapshots: Vec, -) -> PublishedSandboxState { - let lifecycle = sandbox.observed_lifecycle(); - if let Err(error) = &lifecycle { - tracing::warn!(sandbox_id = %sandbox.sandbox_id, %error, "preserving cluster lifecycle after unknown node observation"); - } - PublishedSandboxState { - sandbox_id: sandbox.sandbox_id.clone(), - present: true, - template: sandbox.template.clone(), - status: lifecycle - .as_ref() - .ok() - .map(|state| state.label().to_string()), - content_hash: sandbox.image_hash.clone(), - runtime: sandbox.runtime.clone(), - created_at: sandbox.created_at.clone(), - owner: sandbox.owner.clone(), - observed_network_policy: policy.and_then(|policy| policy.effective_policy.clone()), - observed_network_policy_generation: policy.map_or(0, |policy| policy.applied_generation), - observed_network_policy_source: policy.map_or(0, |policy| policy.applied_source), - observed_network_policy_authority: policy.map_or(0, |policy| policy.authority), - auto_lifecycle_authority: sandbox.auto_lifecycle_authority, - network_tx_bytes: sandbox.network_tx_bytes, - network_rx_bytes: sandbox.network_rx_bytes, - disk_usage_bytes, - snapshots_enabled: sandbox.snapshots_enabled, - pool_template: sandbox.pool_template.clone(), - failure_reason: sandbox.failure_reason.clone(), - quarantine_reason_code: sandbox.quarantine_reason_code.clone(), - transition_receipt: sandbox.transition_receipt.clone(), - state_degraded: sandbox.state_degraded || lifecycle.is_err(), - current_local_artifact_id: sandbox.current_local_artifact_id.clone(), - current_local_artifact_kind: sandbox.current_local_artifact_kind, - current_local_artifact_origin: sandbox.current_local_artifact_origin, - state_changed_at: lifecycle.as_ref().ok().and(sandbox.state_changed_at), - quarantine_recovery: sandbox.quarantine_recovery.clone(), - execution_generation: sandbox.execution_generation, - pod_uid: sandbox.pod_uid.clone(), - activation_attempt_id: sandbox.activation_attempt_id.clone(), - snapshots, - pid: sandbox.pid, - namespace: sandbox.namespace.clone(), - uptime: sandbox.uptime.clone(), - execution: sandbox.execution.clone(), - last_used_at: sandbox_last_used_at(sandbox), - last_access_at: sandbox_last_access_at(sandbox), - last_command_at: sandbox_last_command_at(sandbox), - spaced_version: sandbox_spaced_version(sandbox), - } -} - -fn sandbox_spaced_version(sandbox: &SandboxInfo) -> Option { - let raw = sandbox.spaced_version.as_deref()?; - match SemVer::parse(raw) { - Ok(version) => Some(version.to_string()), - Err(error) => { - tracing::warn!( - sandbox_id = %sandbox.sandbox_id, - error = %error, - "sandbox inventory contained invalid spaced version" - ); - None - } - } -} - -fn sandbox_last_used_at(sandbox: &SandboxInfo) -> Option { - max_proto_timestamp( - sandbox_last_access_at(sandbox), - sandbox_last_command_at(sandbox), - ) -} - -fn sandbox_last_access_at(sandbox: &SandboxInfo) -> Option { - sandbox - .auto_lifecycle_status - .as_ref() - .and_then(|status| status.last_access_at) -} - -fn sandbox_last_command_at(sandbox: &SandboxInfo) -> Option { - sandbox - .auto_lifecycle_status - .as_ref() - .and_then(|status| status.last_command_at) -} - -fn max_proto_timestamp( - left: Option, - right: Option, -) -> Option { - match (left, right) { - (Some(left), Some(right)) => { - if (right.seconds, right.nanos) >= (left.seconds, left.nanos) { - Some(right) - } else { - Some(left) - } - } - (Some(value), None) | (None, Some(value)) => Some(value), - (None, None) => None, - } -} - -fn absent_sandbox_state(sandbox_id: String) -> PublishedSandboxState { - PublishedSandboxState { - sandbox_id, - present: false, - ..PublishedSandboxState::default() - } -} - -async fn collect_template_states( - scope: FacetScope, -) -> Result<(Vec, u64), String> { - let mut client = pplx_space_template_mgr_client::connect() - .await - .map_err(|error| format!("connect template-manager: {error}"))?; - let templates = client - .list_templates(Request::new(ListTemplatesRequest {})) - .await - .map_err(|error| format!("list templates: {error}"))? - .into_inner() - .templates; - let pools = client - .pool_status(Request::new(PoolStatusRequest { template: None })) - .await - .map(|response| response.into_inner().pools) - .unwrap_or_default(); - let pools_by_name = pools - .into_iter() - .map(|pool| (pool.template.clone(), pool)) - .collect::>(); - let mut by_name = BTreeMap::new(); - for template in templates { - by_name.insert(template.name.clone(), template); - } - - let names: BTreeSet = match scope { - FacetScope::Items(items) => util::dedupe_non_empty(items).into_iter().collect(), - FacetScope::Full => by_name.keys().cloned().collect(), - }; - let mut out = Vec::with_capacity(names.len()); - for name in names { - let Some(template) = by_name.get(&name) else { - out.push(PublishedTemplateState { - template: name, - present: false, - ..PublishedTemplateState::default() - }); - continue; - }; - let pool = pools_by_name.get(&template.name); - out.push(PublishedTemplateState { - template: template.name.clone(), - present: true, - content_hash: template.content_hash.clone(), - runtime: template.runtime.clone(), - arch: template.arch.clone(), - has_base_snapshot: template.has_base_snapshot, - network_policy: template.network_policy.clone(), - ensure_base_snapshot: template.ensure_base_snapshot, - snapshot_vcpu: template.snapshot_vcpu, - snapshot_memory_mib: template.snapshot_memory_mib, - built_at: template.built_at.clone(), - artifact_size_bytes: template.artifact_size_bytes, - cache_size_bytes: template.cache_size_bytes, - base_snapshot_size_bytes: template.base_snapshot_size_bytes, - pool_target: pool.map(|pool| pool.target_count).or(template.pool_target), - pool_current: pool.map_or(template.pool_current, |pool| pool.current_count), - pool_pending: pool.map_or(0, |pool| pool.pending_count), - pool_sandbox_ids: pool - .map(|pool| pool.sandbox_ids.clone()) - .unwrap_or_default(), - snapshots: template.snapshots.clone(), - default_auto_lifecycle: template.default_auto_lifecycle.clone(), - network_policy_generation: template.network_policy_generation, - network_policy_authority: template.network_policy_authority, - auto_lifecycle_authority: template.auto_lifecycle_authority, - vcpu: template.vcpu, - memory_mib: template.memory_mib, - disk_size_mb: template.disk_size_mb, - execution: template.execution.clone(), - }); - } - let seen = out.len() as u64; - Ok((out, seen)) -} - -pub async fn apply_template_policy( - template: &str, - policy: ProtoNetworkPolicyV1, - generation: u64, - sync_cluster: bool, -) -> Result { - let mut client = match pplx_space_template_mgr_client::connect().await { - Ok(client) => client, - Err(error) => { - tracing::warn!(template, %error, "cannot connect to template-manager"); - return Err(format!("connect template-manager: {error}")); - } - }; - match client - .update_template_settings(local_manager_request( - UpdateTemplateSettingsRequest { - template: template.to_string(), - default_auto_lifecycle: None, - network_policy: Some(policy), - network_policy_generation: generation, - network_policy_authority: PolicyAuthority::Cluster.as_proto_i32(), - auto_lifecycle_authority: PolicyAuthority::Cluster.as_proto_i32(), - }, - sync_cluster, - )) - .await - { - Ok(response) => Ok(apply_decision_from_proto( - response.into_inner().network_policy_apply_decision, - )), - Err(error) => { - tracing::warn!( - template, - generation, - %error, - "failed to apply template network policy" - ); - Err(error.to_string()) - } - } -} - -pub async fn apply_sandbox_policy( - sandbox_id: &str, - policy: Option, - inherit_template: bool, - generation: u64, - sync_cluster: bool, -) -> Result { - let mut client = match pplx_space_sbx_mgr_client::connect().await { - Ok(client) => client, - Err(error) => { - tracing::warn!(sandbox_id, %error, "cannot connect to sbx-manager"); - return Err(format!("connect sbx-manager: {error}")); - } - }; - match client - .set_network_policy(local_manager_request( - SetNetworkPolicyRequest { - sandbox_id: sandbox_id.to_string(), - policy, - inherit_template, - policy_generation: generation, - authority: PolicyAuthority::Cluster.as_proto_i32(), - sync_to_gateway: false, - }, - sync_cluster, - )) - .await - { - Ok(response) => Ok(apply_decision_from_proto( - response.into_inner().apply_decision, - )), - Err(error) => { - tracing::warn!( - sandbox_id, - generation, - %error, - "failed to apply sandbox network policy" - ); - Err(error.to_string()) - } - } -} - -/// Node-manager port RPCs issued while reconciling one exposure. Kept as an -/// enum so terminal-code classification keys off the exact operation instead -/// of substring-matching a formatted message. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum PortExposureRpc { - Allow, - Expose, - Unexpose, - Deny, -} - -impl PortExposureRpc { - fn as_str(self) -> &'static str { - match self { - Self::Allow => "allow port", - Self::Expose => "expose port", - Self::Unexpose => "unexpose port", - Self::Deny => "deny port", - } - } -} - -impl std::fmt::Display for PortExposureRpc { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str(self.as_str()) - } -} - -/// Terminal-vs-retryable disposition for a tonic `Status` returned by an -/// sbx-manager desired-state apply sub-op (loopback-bridge open, snapshot, -/// auto-lifecycle). Terminal means retrying on this node cannot converge while -/// state is unchanged, so the queue ref must complete; a lifecycle or cluster -/// repair changes state and periodic desired-state sync re-applies afterward. -/// -/// Terminal **only** when sbx-manager marks the status. sbx-manager answers -/// `FailedPrecondition` for both terminal cases (guest unreachable in a -/// settled non-reachable state; auto-lifecycle target stuck in -/// `Failed`/`CleanupRequired`/`CleanupFailed`/`Quarantined`) and transient ones (a guest still -/// initializing — `guest_not_ready`; a mid-transition lifecycle state that -/// settles), and `NotFound` for both a gone row and the startup race where the -/// local row has not appeared yet (`ApplyPortExposuresAndRetry`). Neither bare -/// code can distinguish these, so classification defers entirely to the marker -/// the server sets at its own precondition gate; everything unmarked — -/// transport, `Unavailable`, `Internal`, timeouts, rollout-skew -/// `Unimplemented`, unmarked `FailedPrecondition`/`NotFound` — stays retryable. -/// -/// The node-manager `ExposePort` path does **not** use this: its -/// `FailedPrecondition`/`NotFound` are unambiguously the terminal stale-netns / -/// gone-allocation cases, so it classifies by code directly (see -/// `SandboxPortExposureApplyError`). -fn sbx_mgr_desired_state_apply_is_terminal(status: &tonic::Status) -> bool { - pplx_space_common_grpc::is_desired_state_apply_terminal(status) -} - -/// Failure applying one sandbox port exposure. RPC failures preserve the -/// tonic `Status` (code + message) instead of flattening it to a `String` so -/// the reconcile loop can distinguish terminal dispositions from retryable -/// ones. -#[derive(Debug, thiserror::Error)] -enum SandboxPortExposureApplyError { - /// Local pre-validation failed before any node-manager RPC. - #[error("{0}")] - Invalid(String), - /// A node-manager port RPC failed. - #[error("{operation}: {status}")] - Rpc { - operation: PortExposureRpc, - #[source] - status: tonic::Status, - }, - /// The in-VM loopback bridge re-arm failed after the exposure RPCs - /// succeeded. - #[error(transparent)] - Bridge(OpenLoopbackBridgeError), -} - -impl SandboxPortExposureApplyError { - /// Terminal: this node cannot make progress by retrying while cluster - /// state is unchanged. Completion policy is handled separately because - /// stale-netns and unreachable-guest failures have different repair paths. - /// - /// The two terminal sub-ops classify differently because their servers do. - /// - /// `ExposePort` (node-manager): terminal on `NotFound` (no allocation for - /// this sandbox — authoritatively absent from this node) or a - /// `FailedPrecondition` **tagged with the stale-pod-netns marker** (the CNI - /// namespace behind the allocation is gone). `FailedPrecondition` is *not* - /// terminal on its own — the RPC also returns it for the root-namespace - /// ingress-safety rejection, which has a live allocation the reconciler - /// won't repair — so keying on the exact marker keeps the two apart. - /// - /// The loopback-bridge open (sbx-manager) answers `FailedPrecondition` for - /// both the terminal guest-unreachable gate and the transient - /// `guest_not_ready` case, so it defers to - /// [`sbx_mgr_desired_state_apply_is_terminal`], which trusts the server-set - /// terminal marker rather than the bare code. - /// - /// Transport errors, `Internal` (e.g. netlink timeouts), the bridge - /// not-yet-ready status, and every other RPC stay retryable. - fn is_terminal(&self) -> bool { - match self { - Self::Rpc { - operation: PortExposureRpc::Expose, - status, - } => status.code() == Code::NotFound || Self::has_stale_pod_netns_marker(status), - Self::Bridge(OpenLoopbackBridgeError::Rpc(status)) => { - sbx_mgr_desired_state_apply_is_terminal(status) - } - _ => false, - } - } - - /// Whether this classified failure may complete the enclosing sandbox ref. - /// - /// A stale-netns `ExposePort` failure never completes the ref: the ref stays - /// retryable so a stale-netns sandbox cannot be stranded with no repair - /// owner. A marked - /// sbx-manager bridge failure is different: local state exists but the - /// guest cannot become reachable without a separate lifecycle transition, - /// so retrying this ref cannot help. Complete it immediately and let the - /// periodic full reconcile reapply after the lifecycle changes. - fn completes_ref(&self) -> bool { - match self { - Self::Bridge(OpenLoopbackBridgeError::Rpc(status)) => { - sbx_mgr_desired_state_apply_is_terminal(status) - } - _ => false, - } - } - - fn has_stale_pod_netns_marker(status: &tonic::Status) -> bool { - status - .metadata() - .get(EXPOSE_TERMINAL_REASON_METADATA_KEY) - .and_then(|value| value.to_str().ok()) - == Some(EXPOSE_TERMINAL_REASON_STALE_POD_NETNS) - } -} - -/// Outcome of applying one sandbox port exposure. The `result` decides -/// `apply_state` (Ok → applied, Err → failed); `bridge_status` and -/// `service_generation` are observation metadata persisted alongside so the -/// reconciler can skip steady-state rows and a future UI can render -/// reachability without a live node probe. -struct SandboxPortExposureApplyOutcome { - result: Result<(), SandboxPortExposureApplyError>, - bridge_status: Option, - service_generation: Option, -} - -trait SandboxPortExposureApplier { - fn apply<'a>( - &'a mut self, - sandbox_id: &'a str, - exposure: &'a SandboxPortExposureDesiredState, - ) -> Pin + Send + 'a>>; -} - -impl SandboxPortExposureApplier for pplx_space_node_mgr_client::NodeManagerGrpcClient { - fn apply<'a>( - &'a mut self, - sandbox_id: &'a str, - exposure: &'a SandboxPortExposureDesiredState, - ) -> Pin + Send + 'a>> { - Box::pin(apply_sandbox_port_exposure_desired( - self, sandbox_id, exposure, - )) - } -} - -/// How one sandbox's port-exposure batch is reconciled. -#[derive(Clone, Copy)] -struct PortExposureReconcileMode { - action: SandboxDesiredReconcileAction, -} - -async fn reconcile_sandbox_port_exposures( - mode: PortExposureReconcileMode, - applier: &mut A, - sandbox_id: &str, - exposures: Vec, - stats: &mut ReconcileStats, - element_errors: &mut BTreeMap, - exposure_apply_states: &mut Vec, -) { - let action = mode.action; - if action == SandboxDesiredReconcileAction::SkipNonLocal { - return; - } - - let mut terminal_failure = false; - for exposure in exposures { - let outcome = applier.apply(sandbox_id, &exposure).await; - let (apply_state, error) = match outcome.result { - Ok(()) => { - stats.sandboxes_applied = stats.sandboxes_applied.saturating_add(1); - ( - SandboxPortApplyStateKind::SandboxPortApplyStateApplied, - None, - ) - } - Err(error) => { - // Completion policy depends on the failing boundary. A stale - // node-manager netns needs the rollout-gated orphan repair - // owner, while a marked sbx-manager bridge failure has local - // state and completes until a lifecycle change makes the guest - // reachable again. Keep classification and completion separate - // so the stale-netns safety gate cannot suppress the actual - // unreachable-guest poison-ref fix. - let classified_terminal = error.is_terminal(); - let complete_ref = error.completes_ref(); - tracing::warn!( - sandbox_id, - protocol = port_protocol_label(exposure.protocol), - guest_port = exposure.guest_port, - desired_state = port_desired_state_label(exposure.desired_state), - classified_terminal, - complete_ref, - %error, - "failed to apply sandbox port exposure desired state" - ); - let error = error.to_string(); - let result_kind = if complete_ref { - // Publish the failed observation but do not fail the - // sandbox ref: retrying on this node can never succeed - // (the pod netns / allocation is gone, or the guest is - // unreachable). The owning reconciler/lifecycle transition - // repairs durable state; periodic desired-state sync then - // re-applies anything that becomes actionable. - terminal_failure = true; - pplx_space_metrics::reconcile::DesiredStateApplyResult::Terminal - } else { - record_element_error(element_errors, sandbox_id, &error); - pplx_space_metrics::reconcile::DesiredStateApplyResult::Retryable - }; - pplx_space_metrics::reconcile::desired_state_apply(result_kind); - ( - SandboxPortApplyStateKind::SandboxPortApplyStateFailed, - Some(error), - ) - } - }; - exposure_apply_states.push(SandboxPortExposureApplyState { - sandbox_id: sandbox_id.to_string(), - protocol: exposure.protocol, - guest_port: exposure.guest_port, - apply_state: apply_state as i32, - error, - bridge_status: outcome.bridge_status, - service_generation: outcome.service_generation, - }); - } - - // Missing local sandbox state normally keeps the ref failing so the node - // retries until sbx-manager state shows up. A terminal exposure failure - // proves that state never will (the netns behind the allocation is gone), - // so recording the retry error would poison the queue with a ref that can - // never converge. - if action == SandboxDesiredReconcileAction::ApplyPortExposuresAndRetry && !terminal_failure { - record_element_error( - element_errors, - sandbox_id, - "local sandbox state unavailable", - ); - } -} - -async fn apply_sandbox_port_exposure_desired( - client: &mut pplx_space_node_mgr_client::NodeManagerGrpcClient, - sandbox_id: &str, - exposure: &SandboxPortExposureDesiredState, -) -> SandboxPortExposureApplyOutcome { - let mut outcome = SandboxPortExposureApplyOutcome { - result: Ok(()), - bridge_status: None, - service_generation: None, - }; - if SandboxPortProtocol::try_from(exposure.protocol).unwrap_or(SandboxPortProtocol::Unspecified) - != SandboxPortProtocol::Tcp - { - outcome.result = Err(SandboxPortExposureApplyError::Invalid(format!( - "unsupported sandbox port exposure protocol {:?}", - port_protocol_label(exposure.protocol) - ))); - return outcome; - } - let port = match u16::try_from(exposure.guest_port) - .ok() - .filter(|port| *port > 0) - { - Some(port) => u32::from(port), - None => { - outcome.result = Err(SandboxPortExposureApplyError::Invalid(format!( - "invalid sandbox port {}", - exposure.guest_port - ))); - return outcome; - } - }; - match SandboxPortDesiredStateKind::try_from(exposure.desired_state) - .unwrap_or(SandboxPortDesiredStateKind::SandboxPortDesiredStateUnspecified) - { - SandboxPortDesiredStateKind::SandboxPortDesiredStateOpen => { - if let Err(status) = client - .allow_port(Request::new(AllowPortRequest { - sandbox_id: sandbox_id.to_string(), - port, - protocol: NetworkProtocol::Tcp as i32, - })) - .await - { - outcome.result = Err(SandboxPortExposureApplyError::Rpc { - operation: PortExposureRpc::Allow, - status, - }); - return outcome; - } - if let Err(status) = client - .expose_port(Request::new(ExposePortRequest { - sandbox_id: sandbox_id.to_string(), - port, - protocol: NetworkProtocol::Tcp as i32, - guest_port: Some(port), - })) - .await - { - outcome.result = Err(SandboxPortExposureApplyError::Rpc { - operation: PortExposureRpc::Expose, - status, - }); - return outcome; - } - // Re-arm the in-VM loopback bridge. This matters on (a) node - // restart / state recovery — `LoopbackBridgeManager` starts - // empty so authorized ports must be re-installed — and (b) - // sandbox guest restart, where spaced-svc forgets the - // authorized set. The observed bridge status and service - // generation are persisted regardless of result so the - // reconciler's steady-state skip and the future UI both have - // current data. - let bridge = open_loopback_bridge_for_exposure(sandbox_id, port).await; - // `bridge.bridge_status` and `bridge.service_generation` are - // already `Option`s — `None` for transport failures (no - // observation possible) so the publish path's `COALESCE` - // preserves any prior observation rather than overwriting - // with `UNSPECIFIED` (which the cluster parser rejects). - outcome.bridge_status = bridge.bridge_status; - outcome.service_generation = bridge.service_generation; - outcome.result = bridge.result.map_err(SandboxPortExposureApplyError::Bridge); - } - SandboxPortDesiredStateKind::SandboxPortDesiredStateClosed => { - if let Err(status) = client - .unexpose_port(Request::new(UnexposePortRequest { - sandbox_id: sandbox_id.to_string(), - port: Some(port), - protocol: NetworkProtocol::Tcp as i32, - })) - .await - { - outcome.result = Err(SandboxPortExposureApplyError::Rpc { - operation: PortExposureRpc::Unexpose, - status, - }); - return outcome; - } - if let Err(status) = client - .deny_port(Request::new(DenyPortRequest { - sandbox_id: sandbox_id.to_string(), - port: Some(port), - protocol: NetworkProtocol::Tcp as i32, - })) - .await - { - outcome.result = Err(SandboxPortExposureApplyError::Rpc { - operation: PortExposureRpc::Deny, - status, - }); - return outcome; - } - // Tear down the in-VM bridge symmetric to the open arm. This - // is purely housekeeping: the bridge dies when the sandbox - // does, but if the sandbox keeps running across many open/ - // close cycles, leaving stale forwarders authorized burns one - // tokio task per stale port. Best-effort. - if let Err(error) = close_loopback_bridge_for_exposure(sandbox_id, port).await { - tracing::warn!( - sandbox_id, - port, - %error, - "loopback bridge close failed during reconcile (forwarder dies with sandbox)" - ); - } - } - SandboxPortDesiredStateKind::SandboxPortDesiredStateUnspecified => { - outcome.result = Err(SandboxPortExposureApplyError::Invalid( - "sandbox port desired state is required".to_string(), - )); - } - } - outcome -} - -/// Failure opening the in-VM loopback bridge for one exposure. The RPC arm -/// preserves the tonic `Status` so a guest-unreachable `FailedPrecondition` -/// (sandbox failed/suspending/suspended/stopped) is distinguishable from -/// transient failures; `Connect` and `NotReady` are always retryable. -#[derive(Debug, thiserror::Error)] -enum OpenLoopbackBridgeError { - /// Could not connect to the local sbx-manager socket — transport, so - /// retryable. - #[error("connect sbx-mgr: {0}")] - Connect(String), - /// The `SpacedOpenLoopbackBridge` RPC returned an error status. The tonic - /// `Status` is preserved so the guest-unreachable `FailedPrecondition` - /// case can be classified terminal by the shared classifier. - #[error("spaced_open_loopback_bridge: {0}")] - Rpc(#[source] tonic::Status), - /// The RPC succeeded but the bridge is not in a serving status yet - /// (`WaitingForListener`/`BindConflict`/`Failed`/`Unspecified`). The - /// reconciler re-ticks to promote it, so this stays retryable. - #[error("loopback bridge not ready: {0}")] - NotReady(String), -} - -/// Outcome of one `SpacedOpenLoopbackBridge` call. `bridge_status` is -/// `Some(numeric proto enum value)` only when we received a response — -/// transport failures (sbx-mgr connect, RPC error) return `None` so the -/// publish path leaves the column unchanged via `COALESCE` instead of -/// overwriting with `UNSPECIFIED` (which the cluster parser rejects and -/// would fail the entire batch). `service_generation` is the -/// spaced-guest-svc per-process random id — likewise `None` on transport -/// failure. -struct OpenLoopbackBridgeOutcome { - result: Result<(), OpenLoopbackBridgeError>, - bridge_status: Option, - service_generation: Option, -} - -async fn open_loopback_bridge_for_exposure( - sandbox_id: &str, - port: u32, -) -> OpenLoopbackBridgeOutcome { - use pplx_space_sbx_mgr_client::proto::SpacedLoopbackBridgeStatus as S; - - let mut client = match pplx_space_sbx_mgr_client::connect().await { - Ok(client) => client, - Err(error) => { - return OpenLoopbackBridgeOutcome { - result: Err(OpenLoopbackBridgeError::Connect(error.to_string())), - bridge_status: None, - service_generation: None, - }; - } - }; - let response = match client - .spaced_open_loopback_bridge( - pplx_space_sbx_mgr_client::proto::SpacedOpenLoopbackBridgeRequest { - sandbox_id: sandbox_id.to_string(), - operation: Some( - pplx_space_sbx_mgr_client::operations::OpenLoopbackBridgeRequest { - port, - wait_timeout: Some(prost_types::Duration { - seconds: 2, - nanos: 0, - }), - }, - ), - }, - ) - .await - { - Ok(response) => response.into_inner(), - Err(status) => { - return OpenLoopbackBridgeOutcome { - result: Err(OpenLoopbackBridgeError::Rpc(status)), - bridge_status: None, - service_generation: None, - }; - } - }; - let status = S::try_from(response.status).unwrap_or(S::Unspecified); - let service_generation = response.service_generation.clone(); - // Only `Direct` and `LoopbackForwarded` count as `applied` — the - // exposure represents the whole externally visible port contract, not - // just the host DNAT half. `WaitingForListener` means the bridge is - // armed but no loopback listener has appeared yet, so the port still - // returns 502 to clients; that's not `applied`. The reconciler ticks - // every poll interval and will promote the row once the watcher sees - // the listener and transitions to `LoopbackForwarded`. `BindConflict` - // / `Failed` / `Unspecified` are propagated for the same reason. - // Symmetric with the api-side caller in `controllers/sandbox/ports.rs`. - let result = match status { - S::Direct | S::LoopbackForwarded => { - tracing::debug!( - sandbox_id, - port, - ?status, - detail = %response.detail, - service_generation = %service_generation, - "reconcile installed loopback bridge" - ); - Ok(()) - } - S::WaitingForListener | S::BindConflict | S::Failed | S::Unspecified => { - Err(OpenLoopbackBridgeError::NotReady(format!( - "status={status:?} detail={} service_generation={service_generation}", - response.detail - ))) - } - }; - OpenLoopbackBridgeOutcome { - result, - bridge_status: Some(response.status), - service_generation: Some(service_generation), - } -} - -async fn close_loopback_bridge_for_exposure(sandbox_id: &str, port: u32) -> Result<(), String> { - let mut client = pplx_space_sbx_mgr_client::connect() - .await - .map_err(|error| format!("connect sbx-mgr: {error}"))?; - let _ = client - .spaced_close_loopback_bridge( - pplx_space_sbx_mgr_client::proto::SpacedCloseLoopbackBridgeRequest { - sandbox_id: sandbox_id.to_string(), - operation: Some( - pplx_space_sbx_mgr_client::operations::CloseLoopbackBridgeRequest { port }, - ), - }, - ) - .await - .map_err(|error| format!("spaced_close_loopback_bridge: {error}"))?; - Ok(()) -} - -fn port_protocol_label(value: i32) -> &'static str { - match SandboxPortProtocol::try_from(value).unwrap_or(SandboxPortProtocol::Unspecified) { - SandboxPortProtocol::Tcp => "tcp", - SandboxPortProtocol::Unspecified => "unspecified", - } -} - -fn port_desired_state_label(value: i32) -> &'static str { - match SandboxPortDesiredStateKind::try_from(value) - .unwrap_or(SandboxPortDesiredStateKind::SandboxPortDesiredStateUnspecified) - { - SandboxPortDesiredStateKind::SandboxPortDesiredStateOpen => "open", - SandboxPortDesiredStateKind::SandboxPortDesiredStateClosed => "closed", - SandboxPortDesiredStateKind::SandboxPortDesiredStateUnspecified => "unspecified", - } -} - -/// sbx-manager RPCs the snapshot / auto-lifecycle desired-state apply sub-ops -/// issue. Kept as an enum so terminal-code classification keys off the exact -/// operation instead of substring-matching a formatted message. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum LocalDesiredStateRpc { - EnableSnapshots, - DisableSnapshots, - SetAutoLifecyclePolicy, -} - -impl LocalDesiredStateRpc { - fn as_str(self) -> &'static str { - match self { - Self::EnableSnapshots => "enable snapshots", - Self::DisableSnapshots => "disable snapshots", - Self::SetAutoLifecyclePolicy => "set auto-lifecycle policy", - } - } -} - -impl std::fmt::Display for LocalDesiredStateRpc { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str(self.as_str()) - } -} - -/// Failure applying a snapshot / auto-lifecycle desired-state sub-op against -/// the local sbx-manager. RPC failures preserve the tonic `Status` so a -/// guest-unreachable `FailedPrecondition` (sandbox failed/suspending/ -/// suspended/stopped) is distinguishable from transient failures. -#[derive(Debug, thiserror::Error)] -enum LocalDesiredStateApplyError { - /// A sbx-manager RPC failed. - #[error("{operation}: {status}")] - Rpc { - operation: LocalDesiredStateRpc, - #[source] - status: tonic::Status, - }, -} - -impl LocalDesiredStateApplyError { - /// See [`sbx_mgr_desired_state_apply_is_terminal`]. - fn is_terminal(&self) -> bool { - match self { - Self::Rpc { status, .. } => sbx_mgr_desired_state_apply_is_terminal(status), - } - } -} - -async fn apply_sandbox_snapshot_desired( - client: &mut pplx_space_sbx_mgr_client::SandboxManagerGrpcClient, - sandbox_id: &str, - state: &SandboxInfo, - snapshots_enabled_desired: bool, -) -> Result<(), LocalDesiredStateApplyError> { - if state.snapshots_enabled == snapshots_enabled_desired { - return Ok(()); - } - - let (operation, result) = if snapshots_enabled_desired { - ( - LocalDesiredStateRpc::EnableSnapshots, - client - .enable_snapshots(local_manager_request( - EnableSnapshotsRequest { - sandbox_id: sandbox_id.to_string(), - }, - true, - )) - .await - .map(|_| ()), - ) - } else { - ( - LocalDesiredStateRpc::DisableSnapshots, - client - .disable_snapshots(local_manager_request( - DisableSnapshotsRequest { - sandbox_id: sandbox_id.to_string(), - }, - true, - )) - .await - .map(|_| ()), - ) - }; - - result.map_err(|status| LocalDesiredStateApplyError::Rpc { operation, status }) -} - -async fn apply_sandbox_auto_lifecycle_desired( - client: &mut pplx_space_sbx_mgr_client::SandboxManagerGrpcClient, - sandbox_id: &str, - state: &SandboxInfo, - policy: pplx_space_sbx_mgr_client::proto::AutoLifecyclePolicy, -) -> Result<(), LocalDesiredStateApplyError> { - if state.auto_lifecycle.as_ref() == Some(&policy) { - return Ok(()); - } - - client - .set_auto_lifecycle_policy(local_manager_request( - SetAutoLifecyclePolicyRequest { - sandbox_id: sandbox_id.to_string(), - policy: Some(policy), - authority: PolicyAuthority::Cluster.as_proto_i32(), - }, - true, - )) - .await - .map(|_| ()) - .map_err(|status| LocalDesiredStateApplyError::Rpc { - operation: LocalDesiredStateRpc::SetAutoLifecyclePolicy, - status, - }) -} - -/// Fold a snapshot / auto-lifecycle apply outcome into the reconcile stats and -/// element errors. `Ok` (applied or already in desired state) counts as -/// applied. A server-marked terminal state-gate failure publishes -/// the diagnostic and metric but records **no** element error, so the sandbox -/// desired-state ref completes. A separate lifecycle/cluster repair changes -/// the state, and periodic desired-state sync re-applies afterward; retrying -/// this unchanged ref could never converge and would pin the queue-lag gauge. -/// A transient failure records an element error so the queue re-drives it. -fn handle_local_desired_state_apply( - result: Result<(), LocalDesiredStateApplyError>, - sandbox_id: &str, - stats: &mut ReconcileStats, - element_errors: &mut BTreeMap, -) { - let Err(error) = result else { - stats.sandboxes_applied = stats.sandboxes_applied.saturating_add(1); - return; - }; - let result_kind = if error.is_terminal() { - tracing::info!( - sandbox_id, - %error, - "terminal sandbox desired-state apply; completing ref until state changes" - ); - pplx_space_metrics::reconcile::DesiredStateApplyResult::Terminal - } else { - tracing::warn!( - sandbox_id, - %error, - "retryable sandbox desired-state apply failure" - ); - record_element_error(element_errors, sandbox_id, &error.to_string()); - pplx_space_metrics::reconcile::DesiredStateApplyResult::Retryable - }; - pplx_space_metrics::reconcile::desired_state_apply(result_kind); -} - #[cfg(test)] mod tests; diff --git a/crates/libs/node/ctl/server/src/facets/cluster_deleted_eviction.rs b/crates/libs/node/ctl/server/src/facets/cluster_deleted_eviction.rs new file mode 100644 index 000000000..e6cfea001 --- /dev/null +++ b/crates/libs/node/ctl/server/src/facets/cluster_deleted_eviction.rs @@ -0,0 +1,182 @@ +//! Which local sandboxes the cluster has deleted may be evicted, and how fast: the per-sync budget, +//! backoff and candidate selection. The eviction loop itself runs in the sandbox desired-state +//! facet. + +use super::*; + +/// Cap on successful cluster-deleted evictions per sync tick. +/// Bounds both the teardown work added to one facet pass and the blast +/// radius of a bad cluster answer; the backlog drains across syncs. +pub(super) const CLUSTER_DELETED_EVICTS_PER_SYNC: usize = 25; + +/// How many candidates one sync may examine while filling the eviction +/// cap. Wider than the cap so rows that defer within one sync cannot +/// monopolize the window; the per-candidate backoff below handles rows +/// that keep deferring across syncs. +pub(super) const CLUSTER_DELETED_CANDIDATE_WINDOW: usize = 100; + +/// Per-candidate retry backoff: a candidate that deferred or failed is +/// skipped by selection for this long, mirroring the disk-pressure +/// eviction loop's candidate backoff, so a permanently-deferring oldest +/// prefix (e.g. malformed `updated_at` rows) rotates out of the window +/// instead of starving the drainable backlog sorted behind it. Process +/// lifetime, in-memory: a node-ctl restart merely retries earlier. +pub(super) const CLUSTER_DELETED_EVICT_BACKOFF: std::time::Duration = + std::time::Duration::from_secs(30 * 60); + +/// Slack added to a server-supplied retry horizon so the retry lands on a +/// sync *after* the horizon passes rather than racing it. Sized to the +/// deployed default poll interval as a fixed safety margin: the interval is +/// runtime-configurable, so a shorter one just skips a few more syncs and a +/// longer one still adds less than one interval of margin; neither breaks +/// the schedule, it only shifts the retry by at most one extra sync. +pub(super) const CLUSTER_DELETED_EVICT_RETRY_SLACK: std::time::Duration = + std::time::Duration::from_secs(crate::DEFAULT_POLL_INTERVAL_SECS); + +/// What one refused `EvictClusterDeletedSandbox` means for the per-id retry +/// schedule (a successful claim needs no schedule: the teardown runs +/// detached). Pure so the routing is testable without a live sbx-manager. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum ClusterDeletedEvictDisposition { + /// Retry on the very next sync, no back-off: sbx-manager found a + /// restore staging pull live on the row at claim time (`Aborted`), + /// which ends on the pull's own timescale. + RetryNextSync, + /// Skip the id for this long, then retry. Not an element failure. + Deferred(std::time::Duration), + /// The eviction failed outright; back off and report an element error. + Failed(std::time::Duration), +} + +/// Route a refused eviction RPC. `NotFound` (the local row vanished), +/// `FailedPrecondition` (row transitioned / inside the quiescence window / +/// refused as local-provenance or pool), `Unimplemented` (rollout skew), +/// `Unavailable` (sbx-manager draining) and `DeadlineExceeded` (teardown +/// still running detached) all defer. A refusal that carries a retry +/// horizon — sbx-manager's quiescence refusal says how long the window has +/// left — defers exactly that long plus one sync of slack, so a fresh +/// cluster-deleted row is reclaimed at quiescence end (~11 min after the +/// delete) instead of after the flat 30 min back-off that stays in place +/// for refusals with no horizon. Everything else is a failure. +pub(super) fn cluster_deleted_evict_refusal_disposition( + status: &tonic::Status, +) -> ClusterDeletedEvictDisposition { + match status.code() { + Code::Aborted => ClusterDeletedEvictDisposition::RetryNextSync, + Code::NotFound + | Code::FailedPrecondition + | Code::Unimplemented + | Code::Unavailable + | Code::DeadlineExceeded => ClusterDeletedEvictDisposition::Deferred( + pplx_space_common_grpc::retry_after(status) + .map(|horizon| horizon.saturating_add(CLUSTER_DELETED_EVICT_RETRY_SLACK)) + .map_or(CLUSTER_DELETED_EVICT_BACKOFF, |until| { + until.min(CLUSTER_DELETED_EVICT_BACKOFF) + }), + ), + _ => ClusterDeletedEvictDisposition::Failed(CLUSTER_DELETED_EVICT_BACKOFF), + } +} + +pub(super) static CLUSTER_DELETED_EVICT_BACKOFF_UNTIL: std::sync::LazyLock< + dashmap::DashMap, +> = std::sync::LazyLock::new(dashmap::DashMap::new); + +/// Wall-clock budget for the eviction pass inside one facet sync. The +/// whole inbound facet races a 30s cancel-by-drop timeout in the work +/// loop; stopping well inside it leaves room for the desired-state applies +/// that share the tick and carries the remainder to the next sync. +pub(super) const CLUSTER_DELETED_EVICT_SYNC_BUDGET: std::time::Duration = + std::time::Duration::from_secs(15); + +/// Per-eviction RPC deadline. The teardown is detached server-side, so +/// this bounds slot occupancy, not the teardown itself. +pub(super) const CLUSTER_DELETED_EVICT_RPC_TIMEOUT: std::time::Duration = + std::time::Duration::from_secs(10); + +/// Pick which cluster-deleted sandbox ids to evict locally this tick: +/// only ids whose local sbx-manager row is currently `suspended` (the +/// leaked wake-cache shape), `restoring` with a persisted staging +/// failure (the row a failed cross-node restore retains on the target +/// "for retry" — a retry can never come once the cluster row is gone, and +/// no other path reclaims it), or `suspending` with a permanently refused +/// teardown backup (the row a pod teardown leaves when the sandbox was +/// deleted mid-activation, whose suspend can never publish), owned by a +/// platform user (`user:` — +/// positive evidence the cluster minted the sandbox; local-provenance +/// rows like `root@local` or `tmplmgr@local` never had a cluster row, so +/// its absence proves nothing about them), and not a pool slot. A +/// `restoring` row without the failure marker is a first-attempt restore +/// and is left alone, as is a `suspending` row without the terminal +/// teardown-backup marker, and anything else mid-transition, failed, or +/// already absent locally. Neither marker is a liveness proof — a retry of +/// the failed restore reuses the row without clearing it. sbx-manager re-checks the owner, pool and quiescence guards, takes +/// the staging lock non-blocking to prove no retry pull is live, and claims +/// the row at the exact version it read before tearing anything down — but +/// it makes no cluster call of its own: the `deleted_sandbox_ids` report +/// this selection is drawn from IS the authority that the cluster row is +/// gone, so getting that report right is this caller's responsibility, not +/// something sbx-manager can recover from. +/// Oldest first — UUIDv7 ids sort by creation time — so the largest +/// accumulated dead weight drains ahead of fresh arrivals. +pub(super) fn select_cluster_deleted_evict_candidates<'a>( + deleted_sandbox_ids: &'a [String], + local_states: &BTreeMap, + window: usize, + is_backed_off: impl Fn(&str) -> bool, +) -> Vec<&'a str> { + let mut candidates = deleted_sandbox_ids + .iter() + .filter(|sandbox_id| !is_backed_off(sandbox_id)) + .filter(|sandbox_id| { + local_states.get(*sandbox_id).is_some_and(|state| { + local_row_is_cluster_deleted_reclaimable(state) + && state.pool_template.is_none() + && state + .owner + .as_deref() + .and_then(|owner| SandboxOwner::parse(owner).ok()) + .is_some_and(|owner| owner.is_platform_user()) + }) + }) + .map(String::as_str) + .collect::>(); + candidates.sort_unstable(); + candidates.dedup(); + candidates.truncate(window); + candidates +} + +/// The local row shapes a cluster-deleted report may reclaim: a +/// stopped rootfs, a suspended wake-cache copy, a failed cross-node restore's retained +/// `restoring` row (identified by any persisted `failure_reason`; a +/// whitespace-only reason is treated as no marker, matching sbx-manager), +/// or a `suspending` row whose teardown suspend had its durable backup +/// permanently refused, or a quarantined row retained for a deleted sandbox. +/// +/// The `suspending` shape matches one exact reason rather than "any +/// failure", because that is the only `suspending` failure that ends the +/// row's own publication path for good. Other reasons sit on suspends +/// recovery can still finish, and offering them would spend eviction +/// budget and back-off slots on ids sbx-manager is bound to refuse. +pub(super) fn local_row_is_cluster_deleted_reclaimable(state: &SandboxInfo) -> bool { + use pplx_space_sbx_mgr_client::proto::SandboxLifecycleState; + match state.observed_lifecycle().ok() { + Some( + SandboxLifecycleState::Stopped + | SandboxLifecycleState::Suspended + | SandboxLifecycleState::Quarantined, + ) => true, + Some(SandboxLifecycleState::Restoring) => state + .failure_reason + .as_deref() + .is_some_and(|reason| !reason.trim().is_empty()), + Some(SandboxLifecycleState::Suspending) => { + state.failure_reason.as_deref() + == Some( + pplx_space_sbx_mgr_client::TEARDOWN_SUSPEND_BACKUP_PERMANENTLY_FAILED_REASON, + ) + } + _ => false, + } +} diff --git a/crates/libs/node/ctl/server/src/facets/inventory_publish.rs b/crates/libs/node/ctl/server/src/facets/inventory_publish.rs new file mode 100644 index 000000000..ed02127e3 --- /dev/null +++ b/crates/libs/node/ctl/server/src/facets/inventory_publish.rs @@ -0,0 +1,341 @@ +//! Outbound publishes of sandbox and template inventory, node status, pending backups and +//! auto-lifecycle delete completions. + +use super::*; + +pub(super) async fn publish_sandbox_inventory( + node_id: &str, + node_role: Option, + scope: FacetScope, + node_status: Option, +) -> Result { + let complete = matches!(scope, FacetScope::Full); + let (sandboxes, sandboxes_seen) = + collect_sandbox_states(scope, SandboxPublishPayload::Inventory).await?; + let auto_lifecycle_delete_completions = + collect_auto_lifecycle_delete_completions(node_role).await; + // The report rides the Full census only. Not because a delta's list + // would be short — `ListPendingLocalBackups` answers for the whole + // node either way — but because the contract this field is published + // under (see `PublishNodeStateRequest` in + // `proto/node/api/v1/cluster.proto`) lets a consumer cross-check the + // report against the sandbox rows in the same publication, and a delta + // carries only a slice of those. + // + // The report is also strictly the newest thing in the publication: + // `collect_sandbox_states` and the delete-receipt collector each made + // their own round trip before this one. A sandbox that reaches + // suspended-with-pin in between is still *present* in this + // publication's `sandboxes` — the Full scope enumerates from sbx-mgr + // `List`, whose query is `FROM sandboxes ORDER BY created_at` with no + // state filter, so the row was picked up while it was still running. + // Present, not missing — but its lifecycle is whatever the `State` + // call saw, and `State` is only the second of several round trips + // before the report: disk usage, snapshots and the delete receipts + // all follow it. A sandbox that suspends after `State` therefore + // ships a Running row beside a pending entry naming it. Genuine + // absence is narrower still, needing a sandbox created *and* pinned + // entirely inside the gap: a create plus a stop or suspend between + // two consecutive RPCs. + // + // The order is still the better one. Reversed, a sandbox that suspends + // after the report but before `collect_sandbox_states` ships a + // freshly-suspended row whose pending entry is missing — a publication + // that contradicts itself in the direction of under-reporting. As + // written, the contradiction runs the other way: an entry whose row is + // a moment stale, which is the better shape to hand a reader than an + // absence. It does put an obligation on that reader, though, and this + // is the only place the obligation is visible: **a pending entry can + // name an artifact newer than anything its own sandbox row reflects**, + // so the row's backup marker must never be read as evidence about the + // reported artifact. The next census reconciles either way. + let pending_local_backups = if complete { + collect_pending_local_backups().await + } else { + None + }; + publish_node_state( + node_id, + node_role, + NodeStatePublish { + sandbox_inventory_complete: complete, + sandboxes, + template_inventory_complete: false, + templates: Vec::new(), + sandboxes_seen, + templates_seen: 0, + node_status, + execution_inventory: None, + auto_lifecycle_delete_completions, + pending_local_backups, + }, + ) + .await +} + +pub(super) async fn publish_template_inventory( + node_id: &str, + node_role: Option, + scope: FacetScope, + node_status: Option, +) -> Result { + let complete = matches!(scope, FacetScope::Full); + let (templates, templates_seen) = collect_template_states(scope).await?; + publish_node_state( + node_id, + node_role, + NodeStatePublish { + sandbox_inventory_complete: false, + sandboxes: Vec::new(), + template_inventory_complete: complete, + templates, + sandboxes_seen: 0, + templates_seen, + node_status, + execution_inventory: None, + auto_lifecycle_delete_completions: Vec::new(), + pending_local_backups: None, + }, + ) + .await +} + +pub(super) async fn publish_node_status( + node_id: &str, + node_role: Option, + node_status: Option, +) -> Result { + let execution_inventory = match pplx_space_node_mgr_client::connect().await { + Ok(mut node_manager) => match node_manager + .get_execution_inventory(local_manager_request( + GetExecutionInventoryRequest {}, + false, + )) + .await + { + Ok(response) => Some(response.into_inner()), + Err(error) => { + tracing::warn!( + %error, + "skipping execution inventory publish because node-manager inventory is unavailable" + ); + None + } + }, + Err(error) => { + tracing::warn!( + %error, + "skipping execution inventory publish because node-manager is unavailable" + ); + None + } + }; + publish_node_state( + node_id, + node_role, + NodeStatePublish { + sandbox_inventory_complete: false, + sandboxes: Vec::new(), + template_inventory_complete: false, + templates: Vec::new(), + sandboxes_seen: 0, + templates_seen: 0, + node_status, + execution_inventory, + auto_lifecycle_delete_completions: Vec::new(), + pending_local_backups: None, + }, + ) + .await +} + +/// How long one census will spend obtaining its pending-backup report. +/// +/// Both legs below are otherwise generous: `connect()` carries the shared +/// 10-second connect-retry budget, and the sandbox-manager channel's own +/// request timeout is 120 seconds. Those are the right numbers for a +/// payload that has to arrive. This one does not — every failure yields +/// `None`, the census publishes anyway, and the next tick regenerates the +/// report (60 s by default, `poll_interval_secs`). Spending ten of the +/// facet watchdog's ninety — its default, `CLUSTER_RPC_TIMEOUT` × 3 — +/// let alone all of them, to produce the same `None` an immediate failure +/// produces would starve the publish the report rides on. +/// +/// Two seconds still covers several of `connect()`'s retry attempts, so a +/// momentary socket blip costs nothing; only a genuinely unavailable +/// manager costs the node a cycle of gate freshness. +pub(super) const PENDING_LOCAL_BACKUP_BUDGET: Duration = Duration::from_secs(2); + +/// Ask sandbox-manager which of this node's current lifecycle artifacts +/// still have no durable remote copy. +/// +/// Every failure — sandbox-manager unreachable, the RPC erroring, the whole +/// exchange outrunning [`PENDING_LOCAL_BACKUP_BUDGET`], an older +/// sandbox-manager answering `Unimplemented` — yields `None`, which is +/// byte-identical on the wire to what a node built before the field existed +/// sends. Deliberately not an empty report: an empty *report* asserts this +/// disk holds nothing that needs rescuing, and a failed enumeration has not +/// established that. +pub(super) async fn collect_pending_local_backups() -> Option { + bounded_pending_local_backups(fetch_pending_local_backups()).await +} + +/// The budget, split from the fetch so the expiry arm can be exercised +/// without a sandbox-manager: a test supplies its own slow future. +pub(super) async fn bounded_pending_local_backups( + fetch: impl Future>, +) -> Option { + match tokio::time::timeout(PENDING_LOCAL_BACKUP_BUDGET, fetch).await { + Ok(report) => report, + Err(_elapsed) => { + tracing::warn!( + budget = ?PENDING_LOCAL_BACKUP_BUDGET, + "gave up listing pending local backups within the census budget" + ); + None + } + } +} + +pub(super) async fn fetch_pending_local_backups() -> Option { + let mut client = match pplx_space_sbx_mgr_client::connect().await { + Ok(client) => client, + Err(error) => { + tracing::warn!(%error, "cannot connect to sbx-manager to list pending local backups"); + return None; + } + }; + match client + .list_pending_local_backups(Request::new(ListPendingLocalBackupsRequest {})) + .await + { + Ok(response) => { + // `cluster.proto` carries sandbox-manager's own + // `PendingLocalBackup`, so the artifact identities move up + // without being re-encoded — there is no field-by-field copy + // here to fall out of step with the message. + let pending = response.into_inner().pending; + // The counterpart to the warn arms above: they say the node + // could not answer, this says it did and with how many. Without + // it a node-local log cannot tell an empty report apart from an + // enumeration that never ran. + tracing::debug!( + pending_local_backups = pending.len(), + "reporting pending local backups in the full census" + ); + Some(PendingLocalBackupReport { pending }) + } + Err(error) if error.code() == Code::Unimplemented => { + // Not a warning — an older sandbox-manager during a rollout is + // expected. Logged all the same so a census that quietly stops + // carrying the report has a node-local explanation. + tracing::debug!(%error, "sbx-manager does not serve the pending-backup inventory"); + None + } + Err(error) => { + tracing::warn!(%error, "cannot list pending local backups"); + None + } + } +} + +pub(super) async fn collect_auto_lifecycle_delete_completions( + node_role: Option, +) -> Vec { + if node_role != Some(NodeRole::Worker) { + return Vec::new(); + } + let mut client = match pplx_space_sbx_mgr_client::connect().await { + Ok(client) => client, + Err(error) => { + pplx_space_metrics::reconcile::auto_lifecycle_delete_delivery( + pplx_space_metrics::reconcile::AutoLifecycleDeleteDeliveryOperation::List, + pplx_space_metrics::reconcile::AutoLifecycleDeleteDeliveryOutcome::Failed, + ); + tracing::warn!(%error, "cannot connect to sbx-manager to list delete receipts"); + return Vec::new(); + } + }; + match client + .list_auto_lifecycle_delete_receipts(Request::new( + ListAutoLifecycleDeleteReceiptsRequest {}, + )) + .await + { + Ok(response) => { + pplx_space_metrics::reconcile::auto_lifecycle_delete_delivery( + pplx_space_metrics::reconcile::AutoLifecycleDeleteDeliveryOperation::List, + pplx_space_metrics::reconcile::AutoLifecycleDeleteDeliveryOutcome::Succeeded, + ); + response + .into_inner() + .receipts + .into_iter() + .map(|receipt| AutoLifecycleDeleteCompletion { + receipt_id: receipt.receipt_id, + sandbox_id: receipt.sandbox_id, + execution_generation: receipt.execution_generation, + route_activation_id: receipt.route_activation_id, + auto_lifecycle_policy_fingerprint: receipt.auto_lifecycle_policy_fingerprint, + }) + .collect() + } + Err(error) if error.code() == Code::Unimplemented => { + pplx_space_metrics::reconcile::auto_lifecycle_delete_delivery( + pplx_space_metrics::reconcile::AutoLifecycleDeleteDeliveryOperation::List, + pplx_space_metrics::reconcile::AutoLifecycleDeleteDeliveryOutcome::Unimplemented, + ); + Vec::new() + } + Err(error) => { + pplx_space_metrics::reconcile::auto_lifecycle_delete_delivery( + pplx_space_metrics::reconcile::AutoLifecycleDeleteDeliveryOperation::List, + pplx_space_metrics::reconcile::AutoLifecycleDeleteDeliveryOutcome::Failed, + ); + tracing::warn!(%error, "cannot list auto-lifecycle delete receipts"); + Vec::new() + } + } +} + +pub(super) async fn acknowledge_auto_lifecycle_delete_completions( + node_role: Option, + receipt_ids: &[String], +) { + if node_role != Some(NodeRole::Worker) || receipt_ids.is_empty() { + return; + } + let mut client = match pplx_space_sbx_mgr_client::connect().await { + Ok(client) => client, + Err(error) => { + pplx_space_metrics::reconcile::auto_lifecycle_delete_delivery( + pplx_space_metrics::reconcile::AutoLifecycleDeleteDeliveryOperation::Acknowledge, + pplx_space_metrics::reconcile::AutoLifecycleDeleteDeliveryOutcome::Failed, + ); + tracing::warn!(%error, "cannot connect to sbx-manager to acknowledge delete receipts"); + return; + } + }; + match client + .acknowledge_auto_lifecycle_delete_receipts(Request::new( + AcknowledgeAutoLifecycleDeleteReceiptsRequest { + receipt_ids: receipt_ids.to_vec(), + }, + )) + .await + { + Ok(_) => { + pplx_space_metrics::reconcile::auto_lifecycle_delete_delivery( + pplx_space_metrics::reconcile::AutoLifecycleDeleteDeliveryOperation::Acknowledge, + pplx_space_metrics::reconcile::AutoLifecycleDeleteDeliveryOutcome::Succeeded, + ); + } + Err(error) => { + pplx_space_metrics::reconcile::auto_lifecycle_delete_delivery( + pplx_space_metrics::reconcile::AutoLifecycleDeleteDeliveryOperation::Acknowledge, + pplx_space_metrics::reconcile::AutoLifecycleDeleteDeliveryOutcome::Failed, + ); + tracing::warn!(%error, "cannot acknowledge auto-lifecycle delete receipts"); + } + } +} diff --git a/crates/libs/node/ctl/server/src/facets/local_state.rs b/crates/libs/node/ctl/server/src/facets/local_state.rs new file mode 100644 index 000000000..c29065d36 --- /dev/null +++ b/crates/libs/node/ctl/server/src/facets/local_state.rs @@ -0,0 +1,567 @@ +//! Reads of the node's local sandbox and template state that the facets reconcile and publish. + +use super::*; + +pub(super) async fn local_sandbox_allocation_ids( + client: &mut pplx_space_node_mgr_client::NodeManagerGrpcClient, +) -> Result, String> { + let response = client + .list_network_leases(Request::new(ListNetworkLeasesRequest {})) + .await + .map_err(|error| format!("list local network leases: {error}"))? + .into_inner(); + let mut sandbox_ids = BTreeSet::new(); + for lease in response.leases { + if !network_slot_state_has_live_allocation(lease.slot_state) + || lease.sandbox_id.trim().is_empty() + { + continue; + } + let sandbox_id = lease + .sandbox_id + .parse::() + .map_err(|error| format!("invalid local network lease sandbox id: {error}"))?; + sandbox_ids.insert(sandbox_id); + } + Ok(sandbox_ids) +} + +pub(super) async fn fetch_local_sandbox_states( + client: &mut pplx_space_sbx_mgr_client::SandboxManagerGrpcClient, + sandbox_ids: Vec, +) -> Result, String> { + if sandbox_ids.is_empty() { + return Ok(BTreeMap::new()); + } + let response = fetch_sandbox_states(client, &sandbox_ids).await?; + Ok(response + .sandboxes + .into_iter() + .filter(|sandbox| !sandbox.sandbox_id.trim().is_empty()) + .map(|sandbox| (sandbox.sandbox_id.clone(), sandbox)) + .collect()) +} + +pub(super) fn network_slot_state_has_live_allocation(slot_state: i32) -> bool { + matches!( + NetworkSlotState::try_from(slot_state).unwrap_or(NetworkSlotState::Unspecified), + NetworkSlotState::Leasing | NetworkSlotState::Leased + ) +} + +pub(super) async fn resolve_template_scope(scope: FacetScope) -> Result, String> { + match scope { + FacetScope::Items(items) => Ok(util::dedupe_non_empty(items)), + FacetScope::Full => list_local_template_names().await, + } +} + +pub(super) async fn resolve_sandbox_scope(scope: FacetScope) -> Result, String> { + match scope { + FacetScope::Items(items) => Ok(util::dedupe_non_empty(items)), + FacetScope::Full => list_local_sandbox_ids().await, + } +} + +pub(super) async fn list_local_template_names() -> Result, String> { + let mut tmpl = pplx_space_template_mgr_client::connect() + .await + .map_err(|error| format!("connect template-manager: {error}"))?; + let response = tmpl + .list_templates(Request::new(ListTemplatesRequest {})) + .await + .map_err(|error| format!("list templates: {error}"))? + .into_inner(); + Ok(util::dedupe_non_empty( + response + .templates + .into_iter() + .map(|template| template.name) + .collect(), + )) +} + +pub(super) async fn list_local_sandbox_ids() -> Result, String> { + let mut sbx = pplx_space_sbx_mgr_client::connect() + .await + .map_err(|error| format!("connect sbx-manager: {error}"))?; + let sandboxes = pplx_space_sbx_mgr_client::list_all_sandboxes(&mut sbx) + .await + .map_err(|error| format!("list sandboxes: {error}"))?; + Ok(util::dedupe_non_empty( + sandboxes + .into_iter() + .map(|sandbox| sandbox.sandbox_id) + .collect(), + )) +} + +pub(super) async fn collect_sandbox_states( + scope: FacetScope, + payload: SandboxPublishPayload, +) -> Result<(Vec, u64), String> { + let mut client = pplx_space_sbx_mgr_client::connect() + .await + .map_err(|error| format!("connect sbx-manager: {error}"))?; + let mut out = Vec::new(); + match scope { + FacetScope::Items(items) => { + let ids = util::dedupe_non_empty(items); + if ids.is_empty() { + return Ok((out, 0)); + } + out.reserve(ids.len()); + let response = fetch_sandbox_states(&mut client, &ids).await?; + let by_id = index_sandbox_state_response(response.sandboxes, &mut out); + let disk_usage_by_id = fetch_sandbox_disk_usage(&mut client, &ids).await; + let policy_by_id = if payload.includes_network_policy() { + let enrichment_ids = by_id + .iter() + .filter_map(|(sandbox_id, sandbox)| { + (!sandbox.state_degraded).then_some(*sandbox_id) + }) + .collect::>(); + fetch_sandbox_network_policies(&mut client, &enrichment_ids) + .await + .map_err(|error| error.to_string())? + } else { + BTreeMap::new() + }; + let snapshot_ids = by_id + .iter() + .filter(|(_, sandbox)| sandbox.snapshots_enabled) + .map(|(sandbox_id, _)| *sandbox_id) + .collect::>(); + let snapshots_by_id = fetch_sandbox_snapshots(&mut client, &snapshot_ids) + .await + .map_err(|error| error.to_string())?; + for (sandbox_id, sandbox) in by_id { + let policy = if payload.includes_network_policy() && !sandbox.state_degraded { + Some( + policy_by_id + .get(&sandbox_id) + .ok_or(SandboxInventoryEnrichmentError::MissingNetworkPolicy( + sandbox_id, + )) + .map_err(|error| error.to_string())?, + ) + } else { + None + }; + let snapshots = snapshots_by_id + .get(&sandbox_id) + .cloned() + .unwrap_or_default(); + out.push(published_sandbox_state( + &sandbox, + disk_usage_by_id.get(&sandbox.sandbox_id).copied(), + policy, + snapshots, + )); + } + for sandbox_id in response.missing_ids { + out.push(absent_sandbox_state(sandbox_id)); + } + } + FacetScope::Full => { + let ids = pplx_space_sbx_mgr_client::list_all_sandboxes(&mut client) + .await + .map_err(|error| format!("list sandboxes: {error}"))? + .into_iter() + .map(|sandbox| sandbox.sandbox_id) + .collect::>(); + if ids.is_empty() { + return Ok((out, 0)); + } + let response = fetch_sandbox_states(&mut client, &ids).await?; + let by_id = index_sandbox_state_response(response.sandboxes, &mut out); + let disk_ids = by_id.keys().map(ToString::to_string).collect::>(); + let disk_usage_by_id = fetch_sandbox_disk_usage(&mut client, &disk_ids).await; + let policy_by_id = if payload.includes_network_policy() { + let enrichment_ids = by_id + .iter() + .filter_map(|(sandbox_id, sandbox)| { + (!sandbox.state_degraded).then_some(*sandbox_id) + }) + .collect::>(); + fetch_sandbox_network_policies(&mut client, &enrichment_ids) + .await + .map_err(|error| error.to_string())? + } else { + BTreeMap::new() + }; + let snapshot_ids = by_id + .iter() + .filter(|(_, sandbox)| sandbox.snapshots_enabled) + .map(|(sandbox_id, _)| *sandbox_id) + .collect::>(); + let snapshots_by_id = fetch_sandbox_snapshots(&mut client, &snapshot_ids) + .await + .map_err(|error| error.to_string())?; + out.reserve(by_id.len()); + for (sandbox_id, sandbox) in by_id { + let policy = if payload.includes_network_policy() && !sandbox.state_degraded { + Some( + policy_by_id + .get(&sandbox_id) + .ok_or(SandboxInventoryEnrichmentError::MissingNetworkPolicy( + sandbox_id, + )) + .map_err(|error| error.to_string())?, + ) + } else { + None + }; + let snapshots = snapshots_by_id + .get(&sandbox_id) + .cloned() + .unwrap_or_default(); + out.push(published_sandbox_state( + &sandbox, + disk_usage_by_id.get(&sandbox.sandbox_id).copied(), + policy, + snapshots, + )); + } + for sandbox_id in response.missing_ids { + out.push(absent_sandbox_state(sandbox_id)); + } + } + } + let seen = out.len() as u64; + Ok((out, seen)) +} + +pub(super) async fn fetch_sandbox_states( + client: &mut pplx_space_sbx_mgr_client::SandboxManagerGrpcClient, + ids: &[String], +) -> Result { + fetch_sandbox_state_batches(ids, |sandbox_ids| { + let mut client = client.clone(); + async move { + client + .state(Request::new(SbxStateRequest { sandbox_ids })) + .await + .map(tonic::Response::into_inner) + } + }) + .await +} + +/// No partial result escapes on a failed batch: the caller may mark the final +/// publication as a complete census only after every batch has succeeded. +pub(super) async fn fetch_sandbox_state_batches( + ids: &[String], + mut fetch: impl FnMut(Vec) -> Fut, +) -> Result +where + Fut: Future>, +{ + let mut response = SbxStateResponse::default(); + for ids in ids.chunks(pplx_space_common_core::inventory::READ_BATCH_ITEMS) { + let batch = fetch(ids.to_vec()) + .await + .map_err(|error| format!("get sandbox state: {error}"))?; + response.sandboxes.extend(batch.sandboxes); + response.missing_ids.extend(batch.missing_ids); + } + Ok(response) +} + +pub(super) fn index_sandbox_state_response( + sandboxes: Vec, + out: &mut Vec, +) -> BTreeMap { + let mut by_id = BTreeMap::new(); + for sandbox in sandboxes { + match parse_sandbox_id(&sandbox.sandbox_id) { + Ok(sandbox_id) => { + by_id.insert(sandbox_id, sandbox); + } + Err(error) => { + tracing::warn!( + sandbox_id = %sandbox.sandbox_id, + %error, + "forwarding malformed sandbox identity so the control plane disables absence pruning" + ); + out.push(published_sandbox_state(&sandbox, None, None, Vec::new())); + } + } + } + by_id +} + +pub(super) async fn fetch_sandbox_network_policies( + client: &mut pplx_space_sbx_mgr_client::SandboxManagerGrpcClient, + sandbox_ids: &[SandboxId], +) -> Result, SandboxInventoryEnrichmentError> { + if sandbox_ids.is_empty() { + return Ok(BTreeMap::new()); + } + let response = client + .get_network_policy(Request::new(SbxGetNetworkPolicyRequest { + sandbox_ids: sandbox_ids.iter().map(ToString::to_string).collect(), + })) + .await + .map_err(SandboxInventoryEnrichmentError::GetNetworkPolicies)? + .into_inner(); + let mut policies = BTreeMap::new(); + for policy in response.policies { + policies.insert(parse_sandbox_id(&policy.sandbox_id)?, policy); + } + Ok(policies) +} + +pub(super) async fn fetch_sandbox_snapshots( + client: &mut pplx_space_sbx_mgr_client::SandboxManagerGrpcClient, + sandbox_ids: &[SandboxId], +) -> Result>, SandboxInventoryEnrichmentError> { + if sandbox_ids.is_empty() { + return Ok(BTreeMap::new()); + } + let response = client + .list_snapshots(Request::new(ListSnapshotsRequest { + sandbox_ids: sandbox_ids.iter().map(ToString::to_string).collect(), + })) + .await + .map_err(SandboxInventoryEnrichmentError::ListSnapshots)? + .into_inner(); + let mut snapshots_by_id = BTreeMap::>::new(); + for snapshot in response.snapshots { + snapshots_by_id + .entry(parse_sandbox_id(&snapshot.sandbox_id)?) + .or_default() + .push(snapshot); + } + Ok(snapshots_by_id) +} + +pub(super) fn parse_sandbox_id( + sandbox_id: &str, +) -> Result { + sandbox_id + .parse() + .map_err(|source| SandboxInventoryEnrichmentError::InvalidSandboxId { + value: sandbox_id.to_string(), + source, + }) +} + +pub(super) async fn fetch_sandbox_disk_usage( + client: &mut pplx_space_sbx_mgr_client::SandboxManagerGrpcClient, + sandbox_ids: &[String], +) -> BTreeMap { + if sandbox_ids.is_empty() { + return BTreeMap::new(); + } + let mut usage = BTreeMap::new(); + for sandbox_ids in sandbox_ids.chunks(pplx_space_common_core::inventory::READ_BATCH_ITEMS) { + match client + .get_disk_usage(Request::new(SbxGetDiskUsageRequest { + sandbox_ids: sandbox_ids.to_vec(), + })) + .await + { + Ok(response) => usage.extend(response.into_inner().usage_bytes), + Err(error) => { + tracing::warn!(%error, "sandbox disk usage unavailable during inventory publish"); + return BTreeMap::new(); + } + } + } + usage +} + +pub(super) fn published_sandbox_state( + sandbox: &SandboxInfo, + disk_usage_bytes: Option, + policy: Option<&NetworkPolicyDetail>, + snapshots: Vec, +) -> PublishedSandboxState { + let lifecycle = sandbox.observed_lifecycle(); + if let Err(error) = &lifecycle { + tracing::warn!(sandbox_id = %sandbox.sandbox_id, %error, "preserving cluster lifecycle after unknown node observation"); + } + PublishedSandboxState { + sandbox_id: sandbox.sandbox_id.clone(), + present: true, + template: sandbox.template.clone(), + status: lifecycle + .as_ref() + .ok() + .map(|state| state.label().to_string()), + content_hash: sandbox.image_hash.clone(), + runtime: sandbox.runtime.clone(), + created_at: sandbox.created_at.clone(), + owner: sandbox.owner.clone(), + observed_network_policy: policy.and_then(|policy| policy.effective_policy.clone()), + observed_network_policy_generation: policy.map_or(0, |policy| policy.applied_generation), + observed_network_policy_source: policy.map_or(0, |policy| policy.applied_source), + observed_network_policy_authority: policy.map_or(0, |policy| policy.authority), + auto_lifecycle_authority: sandbox.auto_lifecycle_authority, + network_tx_bytes: sandbox.network_tx_bytes, + network_rx_bytes: sandbox.network_rx_bytes, + disk_usage_bytes, + snapshots_enabled: sandbox.snapshots_enabled, + pool_template: sandbox.pool_template.clone(), + failure_reason: sandbox.failure_reason.clone(), + quarantine_reason_code: sandbox.quarantine_reason_code.clone(), + transition_receipt: sandbox.transition_receipt.clone(), + state_degraded: sandbox.state_degraded || lifecycle.is_err(), + current_local_artifact_id: sandbox.current_local_artifact_id.clone(), + current_local_artifact_kind: sandbox.current_local_artifact_kind, + current_local_artifact_origin: sandbox.current_local_artifact_origin, + state_changed_at: lifecycle.as_ref().ok().and(sandbox.state_changed_at), + quarantine_recovery: sandbox.quarantine_recovery.clone(), + execution_generation: sandbox.execution_generation, + pod_uid: sandbox.pod_uid.clone(), + activation_attempt_id: sandbox.activation_attempt_id.clone(), + snapshots, + pid: sandbox.pid, + namespace: sandbox.namespace.clone(), + uptime: sandbox.uptime.clone(), + execution: sandbox.execution.clone(), + last_used_at: sandbox_last_used_at(sandbox), + last_access_at: sandbox_last_access_at(sandbox), + last_command_at: sandbox_last_command_at(sandbox), + spaced_version: sandbox_spaced_version(sandbox), + } +} + +pub(super) fn sandbox_spaced_version(sandbox: &SandboxInfo) -> Option { + let raw = sandbox.spaced_version.as_deref()?; + match SemVer::parse(raw) { + Ok(version) => Some(version.to_string()), + Err(error) => { + tracing::warn!( + sandbox_id = %sandbox.sandbox_id, + error = %error, + "sandbox inventory contained invalid spaced version" + ); + None + } + } +} + +pub(super) fn sandbox_last_used_at(sandbox: &SandboxInfo) -> Option { + max_proto_timestamp( + sandbox_last_access_at(sandbox), + sandbox_last_command_at(sandbox), + ) +} + +pub(super) fn sandbox_last_access_at(sandbox: &SandboxInfo) -> Option { + sandbox + .auto_lifecycle_status + .as_ref() + .and_then(|status| status.last_access_at) +} + +pub(super) fn sandbox_last_command_at(sandbox: &SandboxInfo) -> Option { + sandbox + .auto_lifecycle_status + .as_ref() + .and_then(|status| status.last_command_at) +} + +pub(super) fn max_proto_timestamp( + left: Option, + right: Option, +) -> Option { + match (left, right) { + (Some(left), Some(right)) => { + if (right.seconds, right.nanos) >= (left.seconds, left.nanos) { + Some(right) + } else { + Some(left) + } + } + (Some(value), None) | (None, Some(value)) => Some(value), + (None, None) => None, + } +} + +pub(super) fn absent_sandbox_state(sandbox_id: String) -> PublishedSandboxState { + PublishedSandboxState { + sandbox_id, + present: false, + ..PublishedSandboxState::default() + } +} + +pub(super) async fn collect_template_states( + scope: FacetScope, +) -> Result<(Vec, u64), String> { + let mut client = pplx_space_template_mgr_client::connect() + .await + .map_err(|error| format!("connect template-manager: {error}"))?; + let templates = client + .list_templates(Request::new(ListTemplatesRequest {})) + .await + .map_err(|error| format!("list templates: {error}"))? + .into_inner() + .templates; + let pools = client + .pool_status(Request::new(PoolStatusRequest { template: None })) + .await + .map(|response| response.into_inner().pools) + .unwrap_or_default(); + let pools_by_name = pools + .into_iter() + .map(|pool| (pool.template.clone(), pool)) + .collect::>(); + let mut by_name = BTreeMap::new(); + for template in templates { + by_name.insert(template.name.clone(), template); + } + + let names: BTreeSet = match scope { + FacetScope::Items(items) => util::dedupe_non_empty(items).into_iter().collect(), + FacetScope::Full => by_name.keys().cloned().collect(), + }; + let mut out = Vec::with_capacity(names.len()); + for name in names { + let Some(template) = by_name.get(&name) else { + out.push(PublishedTemplateState { + template: name, + present: false, + ..PublishedTemplateState::default() + }); + continue; + }; + let pool = pools_by_name.get(&template.name); + out.push(PublishedTemplateState { + template: template.name.clone(), + present: true, + content_hash: template.content_hash.clone(), + runtime: template.runtime.clone(), + arch: template.arch.clone(), + has_base_snapshot: template.has_base_snapshot, + network_policy: template.network_policy.clone(), + ensure_base_snapshot: template.ensure_base_snapshot, + snapshot_vcpu: template.snapshot_vcpu, + snapshot_memory_mib: template.snapshot_memory_mib, + built_at: template.built_at.clone(), + artifact_size_bytes: template.artifact_size_bytes, + cache_size_bytes: template.cache_size_bytes, + base_snapshot_size_bytes: template.base_snapshot_size_bytes, + pool_target: pool.map(|pool| pool.target_count).or(template.pool_target), + pool_current: pool.map_or(template.pool_current, |pool| pool.current_count), + pool_pending: pool.map_or(0, |pool| pool.pending_count), + pool_sandbox_ids: pool + .map(|pool| pool.sandbox_ids.clone()) + .unwrap_or_default(), + snapshots: template.snapshots.clone(), + default_auto_lifecycle: template.default_auto_lifecycle.clone(), + network_policy_generation: template.network_policy_generation, + network_policy_authority: template.network_policy_authority, + auto_lifecycle_authority: template.auto_lifecycle_authority, + vcpu: template.vcpu, + memory_mib: template.memory_mib, + disk_size_mb: template.disk_size_mb, + execution: template.execution.clone(), + }); + } + let seen = out.len() as u64; + Ok((out, seen)) +} diff --git a/crates/libs/node/ctl/server/src/facets/network_policy.rs b/crates/libs/node/ctl/server/src/facets/network_policy.rs new file mode 100644 index 000000000..a279721d0 --- /dev/null +++ b/crates/libs/node/ctl/server/src/facets/network_policy.rs @@ -0,0 +1,453 @@ +//! The network-policy facet: inbound refresh and apply, and outbound publish. + +use super::*; + +#[derive(Debug, Default)] +pub(super) struct LocalSandboxNetworkPolicies { + pub(super) policies: BTreeMap, + pub(super) missing: BTreeSet, +} + +impl LocalSandboxNetworkPolicies { + pub(super) fn from_policies(policies: BTreeMap) -> Self { + Self { + policies, + missing: BTreeSet::new(), + } + } + + pub(super) fn get(&self, sandbox_id: &SandboxId) -> Option<&NetworkPolicyDetail> { + self.policies.get(sandbox_id) + } + + pub(super) fn insert_missing(&mut self, sandbox_id: SandboxId) { + self.missing.insert(sandbox_id); + } + + pub(super) fn is_missing(&self, sandbox_id: &SandboxId) -> bool { + self.missing.contains(sandbox_id) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum SandboxPolicyReconcileAction { + Apply, + Skip, +} + +pub(super) fn split_network_policy_items( + items: Vec, +) -> Result<(Vec, Vec), String> { + let mut templates = Vec::new(); + let mut sandbox_ids = Vec::new(); + for item in util::dedupe_non_empty(items) { + match item + .parse::() + .map_err(|error| error.to_string())? + { + NetworkPolicyTarget::Template(template) => templates.push(template.to_string()), + NetworkPolicyTarget::Sandbox(sandbox_id) => sandbox_ids.push(sandbox_id.to_string()), + } + } + Ok(( + util::dedupe_non_empty(templates), + util::dedupe_non_empty(sandbox_ids), + )) +} + +/// Splits one node's network-policy fetch into bounded serial RPCs: templates +/// (few) ride the first request, sandbox ids are batched at `max_batch_size` +/// per request. Serial batches keep at most one in-flight query per node on +/// cluster-api, and each batch is applied as soon as it arrives, so a timeout +/// or error loses only the remaining batches instead of the whole node. +pub(super) fn network_policy_state_requests( + node_id: &str, + node_role: Option, + templates: Vec, + sandbox_ids: Vec, + max_batch_size: usize, +) -> Vec { + let max_batch_size = max_batch_size.max(1); + let mut batches: Vec> = sandbox_ids + .chunks(max_batch_size) + .map(<[String]>::to_vec) + .collect(); + if batches.is_empty() { + batches.push(Vec::new()); + } + let mut templates = Some(templates); + batches + .into_iter() + .map(|sandbox_ids| GetNodeNetworkPolicyStateRequest { + node_id: node_id.to_string(), + templates: templates.take().unwrap_or_default(), + sandbox_ids, + role: proto_node_role(node_role), + }) + .collect() +} + +pub(super) async fn refresh_network_policy( + node_id: &str, + node_role: Option, + scope: FacetScope, + progress: crate::FacetProgress, +) -> Result { + let (templates, sandbox_ids) = resolve_network_policy_scope(scope).await?; + let mut stats = ReconcileStats { + templates_seen: templates.len() as u64, + sandboxes_seen: sandbox_ids.len() as u64, + ..ReconcileStats::default() + }; + if templates.is_empty() && sandbox_ids.is_empty() { + return Ok(stats); + } + + let requests = network_policy_state_requests( + node_id, + node_role, + templates, + sandbox_ids, + crate::CLUSTER_RPC_MAX_BATCH_SIZE, + ); + let mut client = connect_cluster_api() + .await + .map_err(|error| error.to_string())?; + for request in requests { + let response = client + .get_node_network_policy_state(request) + .await + .map_err(|error| format!("cluster network policy refresh failed: {error}"))? + .into_inner(); + apply_network_policy_state(response, &mut stats).await?; + // Each fetched-and-applied batch is forward progress; the facet + // watchdog budgets time-per-batch, not time-per-run, so large nodes + // don't hit a fixed whole-run deadline. + progress.touch(); + } + Ok(stats) +} + +pub(super) async fn apply_network_policy_state( + response: GetNodeNetworkPolicyStateResponse, + stats: &mut ReconcileStats, +) -> Result<(), String> { + for template in response.templates { + let Some(policy) = template.policy else { + continue; + }; + match apply_template_policy(&template.template, policy, template.generation, false).await { + Ok(decision) => { + if apply_decision_counts_as_applied(decision) { + stats.templates_applied = stats.templates_applied.saturating_add(1); + } + } + Err(error) => { + return Err(format!( + "failed to apply template network policy {}: {error}", + template.template + )); + } + } + } + let sandboxes = response.sandboxes; + let local_sandbox_policies = fetch_local_sandbox_network_policies(&sandboxes).await?; + for sandbox in sandboxes { + let sandbox_id = + parse_sandbox_id(&sandbox.sandbox_id).map_err(|error| error.to_string())?; + if sandbox_policy_reconcile_action(&sandbox, &sandbox_id, &local_sandbox_policies) + == SandboxPolicyReconcileAction::Skip + { + continue; + } + let source = NetworkPolicySource::try_from(sandbox.source) + .unwrap_or(NetworkPolicySource::Unspecified); + let inherit_template = source == NetworkPolicySource::Template; + let policy = if inherit_template { + None + } else { + sandbox.policy + }; + match apply_sandbox_policy( + &sandbox.sandbox_id, + policy, + inherit_template, + sandbox.generation, + false, + ) + .await + { + Ok(decision) => { + if apply_decision_counts_as_applied(decision) { + stats.sandboxes_applied = stats.sandboxes_applied.saturating_add(1); + } + } + Err(error) => { + return Err(format!( + "failed to apply sandbox network policy {}: {error}", + sandbox.sandbox_id + )); + } + } + } + Ok(()) +} + +pub(super) async fn fetch_local_sandbox_network_policies( + sandboxes: &[SandboxNetworkPolicyState], +) -> Result { + if sandboxes.is_empty() { + return Ok(LocalSandboxNetworkPolicies::default()); + } + + let sandbox_ids = sandboxes + .iter() + .map(|sandbox| parse_sandbox_id(&sandbox.sandbox_id)) + .collect::, _>>() + .map_err(|error| error.to_string())?; + let mut client = pplx_space_sbx_mgr_client::connect() + .await + .map_err(|error| format!("connect sbx-manager: {error}"))?; + match fetch_sandbox_network_policies(&mut client, &sandbox_ids).await { + Ok(policies) => Ok(LocalSandboxNetworkPolicies::from_policies(policies)), + Err(error) if should_fallback_to_individual_policy_fetch(&error) => { + tracing::debug!( + %error, + "batched sandbox network policy fetch hit a missing sandbox; retrying individually" + ); + fetch_local_sandbox_network_policies_individually(&mut client, &sandbox_ids).await + } + Err(error) => Err(error.to_string()), + } +} + +pub(super) async fn fetch_local_sandbox_network_policies_individually( + client: &mut pplx_space_sbx_mgr_client::SandboxManagerGrpcClient, + sandbox_ids: &[SandboxId], +) -> Result { + let mut local = LocalSandboxNetworkPolicies::default(); + for sandbox_id in sandbox_ids { + match fetch_sandbox_network_policies(client, std::slice::from_ref(sandbox_id)).await { + Ok(mut policies) => { + if let Some(policy) = policies.remove(sandbox_id) { + local.policies.insert(*sandbox_id, policy); + } else { + tracing::debug!( + %sandbox_id, + "sandbox-manager returned no network policy for local sandbox" + ); + local.insert_missing(*sandbox_id); + } + } + Err(error) if should_fallback_to_individual_policy_fetch(&error) => { + tracing::debug!( + %sandbox_id, + "skipping network policy reconcile for sandbox missing locally" + ); + local.insert_missing(*sandbox_id); + } + Err(error) => return Err(error.to_string()), + } + } + Ok(local) +} + +pub(super) fn should_fallback_to_individual_policy_fetch( + error: &SandboxInventoryEnrichmentError, +) -> bool { + matches!( + error, + SandboxInventoryEnrichmentError::GetNetworkPolicies(status) + if status.code() == tonic::Code::NotFound + ) +} + +pub(super) fn sandbox_policy_reconcile_action( + desired: &SandboxNetworkPolicyState, + sandbox_id: &SandboxId, + local: &LocalSandboxNetworkPolicies, +) -> SandboxPolicyReconcileAction { + if local.is_missing(sandbox_id) { + return SandboxPolicyReconcileAction::Skip; + } + if sandbox_policy_needs_apply(desired, local.get(sandbox_id)) { + SandboxPolicyReconcileAction::Apply + } else { + SandboxPolicyReconcileAction::Skip + } +} + +pub(super) fn sandbox_policy_needs_apply( + desired: &SandboxNetworkPolicyState, + local: Option<&NetworkPolicyDetail>, +) -> bool { + let Some(local) = local else { + return true; + }; + + let source = + NetworkPolicySource::try_from(desired.source).unwrap_or(NetworkPolicySource::Unspecified); + let inherit_template = source == NetworkPolicySource::Template; + let expected_applied_source = if inherit_template { + NetworkPolicySource::Template + } else { + NetworkPolicySource::Override + }; + + local.authority != PolicyAuthority::Cluster.as_proto_i32() + || local.inherited != inherit_template + || local.effective_generation != desired.generation + || desired.policy != local.effective_policy + || local.applied_generation != desired.generation + || local.applied_source != expected_applied_source as i32 +} + +pub(super) fn apply_decision_from_proto(value: i32) -> NetworkPolicyApplyDecision { + NetworkPolicyApplyDecision::try_from(value).unwrap_or(NetworkPolicyApplyDecision::Unspecified) +} + +pub(super) fn apply_decision_counts_as_applied(decision: NetworkPolicyApplyDecision) -> bool { + matches!( + decision, + NetworkPolicyApplyDecision::Apply | NetworkPolicyApplyDecision::Repair + ) +} + +pub(super) async fn publish_network_policy( + node_id: &str, + node_role: Option, + scope: FacetScope, + node_status: Option, +) -> Result { + let (templates, sandbox_ids) = resolve_network_policy_publish_scope(scope)?; + let (sandboxes, sandboxes_seen) = + collect_sandbox_states(sandbox_ids, SandboxPublishPayload::NetworkPolicy).await?; + let (templates, templates_seen) = collect_template_states(templates).await?; + publish_node_state( + node_id, + node_role, + NodeStatePublish { + sandbox_inventory_complete: false, + sandboxes, + template_inventory_complete: false, + templates, + sandboxes_seen, + templates_seen, + node_status, + execution_inventory: None, + auto_lifecycle_delete_completions: Vec::new(), + pending_local_backups: None, + }, + ) + .await +} + +pub(super) async fn resolve_network_policy_scope( + scope: FacetScope, +) -> Result<(Vec, Vec), String> { + match scope { + FacetScope::Full => { + let templates = list_local_template_names().await?; + let sandbox_ids = list_local_sandbox_ids().await?; + Ok((templates, sandbox_ids)) + } + FacetScope::Items(items) => split_network_policy_items(items), + } +} + +pub(super) fn resolve_network_policy_publish_scope( + scope: FacetScope, +) -> Result<(FacetScope, FacetScope), String> { + match scope { + FacetScope::Full => Ok((FacetScope::Full, FacetScope::Full)), + FacetScope::Items(items) => { + let (templates, sandbox_ids) = split_network_policy_items(items)?; + Ok((FacetScope::Items(templates), FacetScope::Items(sandbox_ids))) + } + } +} + +pub async fn apply_template_policy( + template: &str, + policy: ProtoNetworkPolicyV1, + generation: u64, + sync_cluster: bool, +) -> Result { + let mut client = match pplx_space_template_mgr_client::connect().await { + Ok(client) => client, + Err(error) => { + tracing::warn!(template, %error, "cannot connect to template-manager"); + return Err(format!("connect template-manager: {error}")); + } + }; + match client + .update_template_settings(local_manager_request( + UpdateTemplateSettingsRequest { + template: template.to_string(), + default_auto_lifecycle: None, + network_policy: Some(policy), + network_policy_generation: generation, + network_policy_authority: PolicyAuthority::Cluster.as_proto_i32(), + auto_lifecycle_authority: PolicyAuthority::Cluster.as_proto_i32(), + }, + sync_cluster, + )) + .await + { + Ok(response) => Ok(apply_decision_from_proto( + response.into_inner().network_policy_apply_decision, + )), + Err(error) => { + tracing::warn!( + template, + generation, + %error, + "failed to apply template network policy" + ); + Err(error.to_string()) + } + } +} + +pub async fn apply_sandbox_policy( + sandbox_id: &str, + policy: Option, + inherit_template: bool, + generation: u64, + sync_cluster: bool, +) -> Result { + let mut client = match pplx_space_sbx_mgr_client::connect().await { + Ok(client) => client, + Err(error) => { + tracing::warn!(sandbox_id, %error, "cannot connect to sbx-manager"); + return Err(format!("connect sbx-manager: {error}")); + } + }; + match client + .set_network_policy(local_manager_request( + SetNetworkPolicyRequest { + sandbox_id: sandbox_id.to_string(), + policy, + inherit_template, + policy_generation: generation, + authority: PolicyAuthority::Cluster.as_proto_i32(), + sync_to_gateway: false, + }, + sync_cluster, + )) + .await + { + Ok(response) => Ok(apply_decision_from_proto( + response.into_inner().apply_decision, + )), + Err(error) => { + tracing::warn!( + sandbox_id, + generation, + %error, + "failed to apply sandbox network policy" + ); + Err(error.to_string()) + } + } +} diff --git a/crates/libs/node/ctl/server/src/facets/port_exposure.rs b/crates/libs/node/ctl/server/src/facets/port_exposure.rs new file mode 100644 index 000000000..9b84004fa --- /dev/null +++ b/crates/libs/node/ctl/server/src/facets/port_exposure.rs @@ -0,0 +1,558 @@ +//! Reconciliation of a sandbox's exposed ports, including the loopback bridges behind them. + +use super::*; + +/// Node-manager port RPCs issued while reconciling one exposure. Kept as an +/// enum so terminal-code classification keys off the exact operation instead +/// of substring-matching a formatted message. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum PortExposureRpc { + Allow, + Expose, + Unexpose, + Deny, +} + +impl PortExposureRpc { + pub(super) fn as_str(self) -> &'static str { + match self { + Self::Allow => "allow port", + Self::Expose => "expose port", + Self::Unexpose => "unexpose port", + Self::Deny => "deny port", + } + } +} + +impl std::fmt::Display for PortExposureRpc { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(self.as_str()) + } +} + +/// Terminal-vs-retryable disposition for a tonic `Status` returned by an +/// sbx-manager desired-state apply sub-op (loopback-bridge open, snapshot, +/// auto-lifecycle). Terminal means retrying on this node cannot converge while +/// state is unchanged, so the queue ref must complete; a lifecycle or cluster +/// repair changes state and periodic desired-state sync re-applies afterward. +/// +/// Terminal **only** when sbx-manager marks the status. sbx-manager answers +/// `FailedPrecondition` for both terminal cases (guest unreachable in a +/// settled non-reachable state; auto-lifecycle target stuck in +/// `Failed`/`CleanupRequired`/`CleanupFailed`/`Quarantined`) and transient ones (a guest still +/// initializing — `guest_not_ready`; a mid-transition lifecycle state that +/// settles), and `NotFound` for both a gone row and the startup race where the +/// local row has not appeared yet (`ApplyPortExposuresAndRetry`). Neither bare +/// code can distinguish these, so classification defers entirely to the marker +/// the server sets at its own precondition gate; everything unmarked — +/// transport, `Unavailable`, `Internal`, timeouts, rollout-skew +/// `Unimplemented`, unmarked `FailedPrecondition`/`NotFound` — stays retryable. +/// +/// The node-manager `ExposePort` path does **not** use this: its +/// `FailedPrecondition`/`NotFound` are unambiguously the terminal stale-netns / +/// gone-allocation cases, so it classifies by code directly (see +/// `SandboxPortExposureApplyError`). +pub(super) fn sbx_mgr_desired_state_apply_is_terminal(status: &tonic::Status) -> bool { + pplx_space_common_grpc::is_desired_state_apply_terminal(status) +} + +/// Failure applying one sandbox port exposure. RPC failures preserve the +/// tonic `Status` (code + message) instead of flattening it to a `String` so +/// the reconcile loop can distinguish terminal dispositions from retryable +/// ones. +#[derive(Debug, thiserror::Error)] +pub(super) enum SandboxPortExposureApplyError { + /// Local pre-validation failed before any node-manager RPC. + #[error("{0}")] + Invalid(String), + /// A node-manager port RPC failed. + #[error("{operation}: {status}")] + Rpc { + operation: PortExposureRpc, + #[source] + status: tonic::Status, + }, + /// The in-VM loopback bridge re-arm failed after the exposure RPCs + /// succeeded. + #[error(transparent)] + Bridge(OpenLoopbackBridgeError), +} + +impl SandboxPortExposureApplyError { + /// Terminal: this node cannot make progress by retrying while cluster + /// state is unchanged. Completion policy is handled separately because + /// stale-netns and unreachable-guest failures have different repair paths. + /// + /// The two terminal sub-ops classify differently because their servers do. + /// + /// `ExposePort` (node-manager): terminal on `NotFound` (no allocation for + /// this sandbox — authoritatively absent from this node) or a + /// `FailedPrecondition` **tagged with the stale-pod-netns marker** (the CNI + /// namespace behind the allocation is gone). `FailedPrecondition` is *not* + /// terminal on its own — the RPC also returns it for the root-namespace + /// ingress-safety rejection, which has a live allocation the reconciler + /// won't repair — so keying on the exact marker keeps the two apart. + /// + /// The loopback-bridge open (sbx-manager) answers `FailedPrecondition` for + /// both the terminal guest-unreachable gate and the transient + /// `guest_not_ready` case, so it defers to + /// [`sbx_mgr_desired_state_apply_is_terminal`], which trusts the server-set + /// terminal marker rather than the bare code. + /// + /// Transport errors, `Internal` (e.g. netlink timeouts), the bridge + /// not-yet-ready status, and every other RPC stay retryable. + pub(super) fn is_terminal(&self) -> bool { + match self { + Self::Rpc { + operation: PortExposureRpc::Expose, + status, + } => status.code() == Code::NotFound || Self::has_stale_pod_netns_marker(status), + Self::Bridge(OpenLoopbackBridgeError::Rpc(status)) => { + sbx_mgr_desired_state_apply_is_terminal(status) + } + _ => false, + } + } + + /// Whether this classified failure may complete the enclosing sandbox ref. + /// + /// A stale-netns `ExposePort` failure never completes the ref: the ref stays + /// retryable so a stale-netns sandbox cannot be stranded with no repair + /// owner. A marked + /// sbx-manager bridge failure is different: local state exists but the + /// guest cannot become reachable without a separate lifecycle transition, + /// so retrying this ref cannot help. Complete it immediately and let the + /// periodic full reconcile reapply after the lifecycle changes. + pub(super) fn completes_ref(&self) -> bool { + match self { + Self::Bridge(OpenLoopbackBridgeError::Rpc(status)) => { + sbx_mgr_desired_state_apply_is_terminal(status) + } + _ => false, + } + } + + pub(super) fn has_stale_pod_netns_marker(status: &tonic::Status) -> bool { + status + .metadata() + .get(EXPOSE_TERMINAL_REASON_METADATA_KEY) + .and_then(|value| value.to_str().ok()) + == Some(EXPOSE_TERMINAL_REASON_STALE_POD_NETNS) + } +} + +/// Outcome of applying one sandbox port exposure. The `result` decides +/// `apply_state` (Ok → applied, Err → failed); `bridge_status` and +/// `service_generation` are observation metadata persisted alongside so the +/// reconciler can skip steady-state rows and a future UI can render +/// reachability without a live node probe. +pub(super) struct SandboxPortExposureApplyOutcome { + pub(super) result: Result<(), SandboxPortExposureApplyError>, + pub(super) bridge_status: Option, + pub(super) service_generation: Option, +} + +pub(super) trait SandboxPortExposureApplier { + fn apply<'a>( + &'a mut self, + sandbox_id: &'a str, + exposure: &'a SandboxPortExposureDesiredState, + ) -> Pin + Send + 'a>>; +} + +impl SandboxPortExposureApplier for pplx_space_node_mgr_client::NodeManagerGrpcClient { + fn apply<'a>( + &'a mut self, + sandbox_id: &'a str, + exposure: &'a SandboxPortExposureDesiredState, + ) -> Pin + Send + 'a>> { + Box::pin(apply_sandbox_port_exposure_desired( + self, sandbox_id, exposure, + )) + } +} + +/// How one sandbox's port-exposure batch is reconciled. +#[derive(Clone, Copy)] +pub(super) struct PortExposureReconcileMode { + pub(super) action: SandboxDesiredReconcileAction, +} + +pub(super) async fn reconcile_sandbox_port_exposures( + mode: PortExposureReconcileMode, + applier: &mut A, + sandbox_id: &str, + exposures: Vec, + stats: &mut ReconcileStats, + element_errors: &mut BTreeMap, + exposure_apply_states: &mut Vec, +) { + let action = mode.action; + if action == SandboxDesiredReconcileAction::SkipNonLocal { + return; + } + + let mut terminal_failure = false; + for exposure in exposures { + let outcome = applier.apply(sandbox_id, &exposure).await; + let (apply_state, error) = match outcome.result { + Ok(()) => { + stats.sandboxes_applied = stats.sandboxes_applied.saturating_add(1); + ( + SandboxPortApplyStateKind::SandboxPortApplyStateApplied, + None, + ) + } + Err(error) => { + // Completion policy depends on the failing boundary. A stale + // node-manager netns needs the rollout-gated orphan repair + // owner, while a marked sbx-manager bridge failure has local + // state and completes until a lifecycle change makes the guest + // reachable again. Keep classification and completion separate + // so the stale-netns safety gate cannot suppress the actual + // unreachable-guest poison-ref fix. + let classified_terminal = error.is_terminal(); + let complete_ref = error.completes_ref(); + tracing::warn!( + sandbox_id, + protocol = port_protocol_label(exposure.protocol), + guest_port = exposure.guest_port, + desired_state = port_desired_state_label(exposure.desired_state), + classified_terminal, + complete_ref, + %error, + "failed to apply sandbox port exposure desired state" + ); + let error = error.to_string(); + let result_kind = if complete_ref { + // Publish the failed observation but do not fail the + // sandbox ref: retrying on this node can never succeed + // (the pod netns / allocation is gone, or the guest is + // unreachable). The owning reconciler/lifecycle transition + // repairs durable state; periodic desired-state sync then + // re-applies anything that becomes actionable. + terminal_failure = true; + pplx_space_metrics::reconcile::DesiredStateApplyResult::Terminal + } else { + record_element_error(element_errors, sandbox_id, &error); + pplx_space_metrics::reconcile::DesiredStateApplyResult::Retryable + }; + pplx_space_metrics::reconcile::desired_state_apply(result_kind); + ( + SandboxPortApplyStateKind::SandboxPortApplyStateFailed, + Some(error), + ) + } + }; + exposure_apply_states.push(SandboxPortExposureApplyState { + sandbox_id: sandbox_id.to_string(), + protocol: exposure.protocol, + guest_port: exposure.guest_port, + apply_state: apply_state as i32, + error, + bridge_status: outcome.bridge_status, + service_generation: outcome.service_generation, + }); + } + + // Missing local sandbox state normally keeps the ref failing so the node + // retries until sbx-manager state shows up. A terminal exposure failure + // proves that state never will (the netns behind the allocation is gone), + // so recording the retry error would poison the queue with a ref that can + // never converge. + if action == SandboxDesiredReconcileAction::ApplyPortExposuresAndRetry && !terminal_failure { + record_element_error( + element_errors, + sandbox_id, + "local sandbox state unavailable", + ); + } +} + +pub(super) async fn apply_sandbox_port_exposure_desired( + client: &mut pplx_space_node_mgr_client::NodeManagerGrpcClient, + sandbox_id: &str, + exposure: &SandboxPortExposureDesiredState, +) -> SandboxPortExposureApplyOutcome { + let mut outcome = SandboxPortExposureApplyOutcome { + result: Ok(()), + bridge_status: None, + service_generation: None, + }; + if SandboxPortProtocol::try_from(exposure.protocol).unwrap_or(SandboxPortProtocol::Unspecified) + != SandboxPortProtocol::Tcp + { + outcome.result = Err(SandboxPortExposureApplyError::Invalid(format!( + "unsupported sandbox port exposure protocol {:?}", + port_protocol_label(exposure.protocol) + ))); + return outcome; + } + let port = match u16::try_from(exposure.guest_port) + .ok() + .filter(|port| *port > 0) + { + Some(port) => u32::from(port), + None => { + outcome.result = Err(SandboxPortExposureApplyError::Invalid(format!( + "invalid sandbox port {}", + exposure.guest_port + ))); + return outcome; + } + }; + match SandboxPortDesiredStateKind::try_from(exposure.desired_state) + .unwrap_or(SandboxPortDesiredStateKind::SandboxPortDesiredStateUnspecified) + { + SandboxPortDesiredStateKind::SandboxPortDesiredStateOpen => { + if let Err(status) = client + .allow_port(Request::new(AllowPortRequest { + sandbox_id: sandbox_id.to_string(), + port, + protocol: NetworkProtocol::Tcp as i32, + })) + .await + { + outcome.result = Err(SandboxPortExposureApplyError::Rpc { + operation: PortExposureRpc::Allow, + status, + }); + return outcome; + } + if let Err(status) = client + .expose_port(Request::new(ExposePortRequest { + sandbox_id: sandbox_id.to_string(), + port, + protocol: NetworkProtocol::Tcp as i32, + guest_port: Some(port), + })) + .await + { + outcome.result = Err(SandboxPortExposureApplyError::Rpc { + operation: PortExposureRpc::Expose, + status, + }); + return outcome; + } + // Re-arm the in-VM loopback bridge. This matters on (a) node + // restart / state recovery — `LoopbackBridgeManager` starts + // empty so authorized ports must be re-installed — and (b) + // sandbox guest restart, where spaced-svc forgets the + // authorized set. The observed bridge status and service + // generation are persisted regardless of result so the + // reconciler's steady-state skip and the future UI both have + // current data. + let bridge = open_loopback_bridge_for_exposure(sandbox_id, port).await; + // `bridge.bridge_status` and `bridge.service_generation` are + // already `Option`s — `None` for transport failures (no + // observation possible) so the publish path's `COALESCE` + // preserves any prior observation rather than overwriting + // with `UNSPECIFIED` (which the cluster parser rejects). + outcome.bridge_status = bridge.bridge_status; + outcome.service_generation = bridge.service_generation; + outcome.result = bridge.result.map_err(SandboxPortExposureApplyError::Bridge); + } + SandboxPortDesiredStateKind::SandboxPortDesiredStateClosed => { + if let Err(status) = client + .unexpose_port(Request::new(UnexposePortRequest { + sandbox_id: sandbox_id.to_string(), + port: Some(port), + protocol: NetworkProtocol::Tcp as i32, + })) + .await + { + outcome.result = Err(SandboxPortExposureApplyError::Rpc { + operation: PortExposureRpc::Unexpose, + status, + }); + return outcome; + } + if let Err(status) = client + .deny_port(Request::new(DenyPortRequest { + sandbox_id: sandbox_id.to_string(), + port: Some(port), + protocol: NetworkProtocol::Tcp as i32, + })) + .await + { + outcome.result = Err(SandboxPortExposureApplyError::Rpc { + operation: PortExposureRpc::Deny, + status, + }); + return outcome; + } + // Tear down the in-VM bridge symmetric to the open arm. This + // is purely housekeeping: the bridge dies when the sandbox + // does, but if the sandbox keeps running across many open/ + // close cycles, leaving stale forwarders authorized burns one + // tokio task per stale port. Best-effort. + if let Err(error) = close_loopback_bridge_for_exposure(sandbox_id, port).await { + tracing::warn!( + sandbox_id, + port, + %error, + "loopback bridge close failed during reconcile (forwarder dies with sandbox)" + ); + } + } + SandboxPortDesiredStateKind::SandboxPortDesiredStateUnspecified => { + outcome.result = Err(SandboxPortExposureApplyError::Invalid( + "sandbox port desired state is required".to_string(), + )); + } + } + outcome +} + +/// Failure opening the in-VM loopback bridge for one exposure. The RPC arm +/// preserves the tonic `Status` so a guest-unreachable `FailedPrecondition` +/// (sandbox failed/suspending/suspended/stopped) is distinguishable from +/// transient failures; `Connect` and `NotReady` are always retryable. +#[derive(Debug, thiserror::Error)] +pub(super) enum OpenLoopbackBridgeError { + /// Could not connect to the local sbx-manager socket — transport, so + /// retryable. + #[error("connect sbx-mgr: {0}")] + Connect(String), + /// The `SpacedOpenLoopbackBridge` RPC returned an error status. The tonic + /// `Status` is preserved so the guest-unreachable `FailedPrecondition` + /// case can be classified terminal by the shared classifier. + #[error("spaced_open_loopback_bridge: {0}")] + Rpc(#[source] tonic::Status), + /// The RPC succeeded but the bridge is not in a serving status yet + /// (`WaitingForListener`/`BindConflict`/`Failed`/`Unspecified`). The + /// reconciler re-ticks to promote it, so this stays retryable. + #[error("loopback bridge not ready: {0}")] + NotReady(String), +} + +/// Outcome of one `SpacedOpenLoopbackBridge` call. `bridge_status` is +/// `Some(numeric proto enum value)` only when we received a response — +/// transport failures (sbx-mgr connect, RPC error) return `None` so the +/// publish path leaves the column unchanged via `COALESCE` instead of +/// overwriting with `UNSPECIFIED` (which the cluster parser rejects and +/// would fail the entire batch). `service_generation` is the +/// spaced-guest-svc per-process random id — likewise `None` on transport +/// failure. +pub(super) struct OpenLoopbackBridgeOutcome { + pub(super) result: Result<(), OpenLoopbackBridgeError>, + pub(super) bridge_status: Option, + pub(super) service_generation: Option, +} + +pub(super) async fn open_loopback_bridge_for_exposure( + sandbox_id: &str, + port: u32, +) -> OpenLoopbackBridgeOutcome { + use pplx_space_sbx_mgr_client::proto::SpacedLoopbackBridgeStatus as S; + + let mut client = match pplx_space_sbx_mgr_client::connect().await { + Ok(client) => client, + Err(error) => { + return OpenLoopbackBridgeOutcome { + result: Err(OpenLoopbackBridgeError::Connect(error.to_string())), + bridge_status: None, + service_generation: None, + }; + } + }; + let response = match client + .spaced_open_loopback_bridge( + pplx_space_sbx_mgr_client::proto::SpacedOpenLoopbackBridgeRequest { + sandbox_id: sandbox_id.to_string(), + operation: Some( + pplx_space_sbx_mgr_client::operations::OpenLoopbackBridgeRequest { + port, + wait_timeout: Some(prost_types::Duration { + seconds: 2, + nanos: 0, + }), + }, + ), + }, + ) + .await + { + Ok(response) => response.into_inner(), + Err(status) => { + return OpenLoopbackBridgeOutcome { + result: Err(OpenLoopbackBridgeError::Rpc(status)), + bridge_status: None, + service_generation: None, + }; + } + }; + let status = S::try_from(response.status).unwrap_or(S::Unspecified); + let service_generation = response.service_generation.clone(); + // Only `Direct` and `LoopbackForwarded` count as `applied` — the + // exposure represents the whole externally visible port contract, not + // just the host DNAT half. `WaitingForListener` means the bridge is + // armed but no loopback listener has appeared yet, so the port still + // returns 502 to clients; that's not `applied`. The reconciler ticks + // every poll interval and will promote the row once the watcher sees + // the listener and transitions to `LoopbackForwarded`. `BindConflict` + // / `Failed` / `Unspecified` are propagated for the same reason. + // Symmetric with the api-side caller in `controllers/sandbox/ports.rs`. + let result = match status { + S::Direct | S::LoopbackForwarded => { + tracing::debug!( + sandbox_id, + port, + ?status, + detail = %response.detail, + service_generation = %service_generation, + "reconcile installed loopback bridge" + ); + Ok(()) + } + S::WaitingForListener | S::BindConflict | S::Failed | S::Unspecified => { + Err(OpenLoopbackBridgeError::NotReady(format!( + "status={status:?} detail={} service_generation={service_generation}", + response.detail + ))) + } + }; + OpenLoopbackBridgeOutcome { + result, + bridge_status: Some(response.status), + service_generation: Some(service_generation), + } +} + +pub(super) async fn close_loopback_bridge_for_exposure( + sandbox_id: &str, + port: u32, +) -> Result<(), String> { + let mut client = pplx_space_sbx_mgr_client::connect() + .await + .map_err(|error| format!("connect sbx-mgr: {error}"))?; + let _ = client + .spaced_close_loopback_bridge( + pplx_space_sbx_mgr_client::proto::SpacedCloseLoopbackBridgeRequest { + sandbox_id: sandbox_id.to_string(), + operation: Some( + pplx_space_sbx_mgr_client::operations::CloseLoopbackBridgeRequest { port }, + ), + }, + ) + .await + .map_err(|error| format!("spaced_close_loopback_bridge: {error}"))?; + Ok(()) +} + +pub(super) fn port_protocol_label(value: i32) -> &'static str { + match SandboxPortProtocol::try_from(value).unwrap_or(SandboxPortProtocol::Unspecified) { + SandboxPortProtocol::Tcp => "tcp", + SandboxPortProtocol::Unspecified => "unspecified", + } +} + +pub(super) fn port_desired_state_label(value: i32) -> &'static str { + match SandboxPortDesiredStateKind::try_from(value) + .unwrap_or(SandboxPortDesiredStateKind::SandboxPortDesiredStateUnspecified) + { + SandboxPortDesiredStateKind::SandboxPortDesiredStateOpen => "open", + SandboxPortDesiredStateKind::SandboxPortDesiredStateClosed => "closed", + SandboxPortDesiredStateKind::SandboxPortDesiredStateUnspecified => "unspecified", + } +} diff --git a/crates/libs/node/ctl/server/src/facets/sandbox_desired_state.rs b/crates/libs/node/ctl/server/src/facets/sandbox_desired_state.rs new file mode 100644 index 000000000..0a5aef708 --- /dev/null +++ b/crates/libs/node/ctl/server/src/facets/sandbox_desired_state.rs @@ -0,0 +1,439 @@ +//! The sandbox desired-state facet and the local desired-state apply it drives. + +use super::*; + +pub(super) fn sandbox_desired_reconcile_action( + has_local_state: bool, + has_port_exposures: bool, + has_local_allocation: bool, +) -> SandboxDesiredReconcileAction { + if has_port_exposures && !has_local_allocation { + SandboxDesiredReconcileAction::SkipNonLocal + } else if has_local_state { + SandboxDesiredReconcileAction::ApplyAll + } else if has_port_exposures { + SandboxDesiredReconcileAction::ApplyPortExposuresAndRetry + } else { + SandboxDesiredReconcileAction::SkipNonLocal + } +} + +pub(super) async fn refresh_sandbox_desired_state( + node_id: &str, + node_role: Option, + scope: FacetScope, +) -> Result { + let sandbox_ids = resolve_sandbox_scope(scope).await?; + let mut stats = ReconcileStats { + sandboxes_seen: sandbox_ids.len() as u64, + ..ReconcileStats::default() + }; + if sandbox_ids.is_empty() { + return Ok(InboundFacetResult::from_stats(stats)); + } + + let mut cluster = connect_cluster_api() + .await + .map_err(|error| error.to_string())?; + let response = cluster + .get_node_sandbox_desired_state(GetNodeSandboxDesiredStateRequest { + node_id: node_id.to_string(), + sandbox_ids, + role: proto_node_role(node_role), + }) + .await + .map_err(|error| format!("cluster sandbox desired-state refresh failed: {error}"))? + .into_inner(); + + let has_port_exposures = response + .sandboxes + .iter() + .any(|sandbox| !sandbox.port_exposures.is_empty()); + let mut sbx = pplx_space_sbx_mgr_client::connect() + .await + .map_err(|error| format!("connect sbx-manager: {error}"))?; + // Deleted ids are included so the eviction step below can gate on the + // sandbox's current local state without a second State round-trip. + let local_state_lookup_ids = response + .sandboxes + .iter() + .map(|sandbox| sandbox.sandbox_id.clone()) + .chain(response.deleted_sandbox_ids.iter().cloned()) + .collect::>(); + let local_sandbox_state_by_id = fetch_local_sandbox_states(&mut sbx, local_state_lookup_ids) + .await + .map_err(|error| format!("sandbox desired-state shared state lookup failed: {error}"))?; + let mut node = has_port_exposures.then(pplx_space_node_mgr_client::lazy); + let local_sandbox_allocations = if let Some(node) = node.as_mut() { + local_sandbox_allocation_ids(node).await.map_err(|error| { + format!("sandbox desired-state shared allocation lookup failed: {error}") + })? + } else { + BTreeSet::new() + }; + let mut element_errors = BTreeMap::new(); + let mut exposure_apply_states = Vec::new(); + for sandbox in response.sandboxes { + let sandbox_id = sandbox + .sandbox_id + .parse::() + .map_err(|error| format!("invalid sandbox desired-state id: {error}"))?; + let has_sandbox_port_exposures = !sandbox.port_exposures.is_empty(); + let local_sandbox_state = local_sandbox_state_by_id.get(&sandbox.sandbox_id); + let action = sandbox_desired_reconcile_action( + local_sandbox_state.is_some(), + has_sandbox_port_exposures, + local_sandbox_allocations.contains(&sandbox_id), + ); + if action == SandboxDesiredReconcileAction::SkipNonLocal { + tracing::debug!( + sandbox_id = %sandbox.sandbox_id, + exposures = sandbox.port_exposures.len(), + "skipping desired state for sandbox that is no longer local" + ); + continue; + } + if let Some(local_sandbox_state) = local_sandbox_state { + handle_local_desired_state_apply( + apply_sandbox_snapshot_desired( + &mut sbx, + &sandbox.sandbox_id, + local_sandbox_state, + sandbox.snapshots_enabled_desired, + ) + .await, + &sandbox.sandbox_id, + &mut stats, + &mut element_errors, + ); + if let Some(policy) = sandbox.auto_lifecycle { + handle_local_desired_state_apply( + apply_sandbox_auto_lifecycle_desired( + &mut sbx, + &sandbox.sandbox_id, + local_sandbox_state, + policy, + ) + .await, + &sandbox.sandbox_id, + &mut stats, + &mut element_errors, + ); + } + } else { + tracing::warn!( + sandbox_id = %sandbox.sandbox_id, + exposures = sandbox.port_exposures.len(), + "local sandbox state is unavailable; applying port exposures and retrying the ref" + ); + } + if let Some(node) = node.as_mut() { + reconcile_sandbox_port_exposures( + PortExposureReconcileMode { action }, + node, + &sandbox.sandbox_id, + sandbox.port_exposures, + &mut stats, + &mut element_errors, + &mut exposure_apply_states, + ) + .await; + } + } + if !exposure_apply_states.is_empty() { + match cluster + .publish_sandbox_port_exposure_states(PublishSandboxPortExposureStatesRequest { + node_id: node_id.to_string(), + states: exposure_apply_states, + role: proto_node_role(node_role), + }) + .await + { + Ok(response) => { + for error in response.into_inner().errors { + tracing::warn!( + sandbox_id = %error.sandbox_id, + protocol = port_protocol_label(error.protocol), + guest_port = error.guest_port, + error = %error.error, + "failed to publish sandbox port exposure apply state" + ); + record_element_error( + &mut element_errors, + &error.sandbox_id, + &format!("publish port exposure apply state: {}", error.error), + ); + } + } + Err(error) => { + return Err(format!( + "publish sandbox port exposure apply states: {error}" + )); + } + } + } + if !response.deleted_sandbox_ids.is_empty() { + let evict_cap = CLUSTER_DELETED_EVICTS_PER_SYNC; + let now = std::time::Instant::now(); + CLUSTER_DELETED_EVICT_BACKOFF_UNTIL.retain(|_, until| *until > now); + let candidates = select_cluster_deleted_evict_candidates( + &response.deleted_sandbox_ids, + &local_sandbox_state_by_id, + CLUSTER_DELETED_CANDIDATE_WINDOW, + |sandbox_id| CLUSTER_DELETED_EVICT_BACKOFF_UNTIL.contains_key(sandbox_id), + ); + let started = std::time::Instant::now(); + let mut evicted = 0_u64; + let mut deferred = 0_u64; + let mut budget_exhausted = false; + for sandbox_id in candidates { + if evicted >= evict_cap as u64 { + break; + } + if started.elapsed() > CLUSTER_DELETED_EVICT_SYNC_BUDGET { + budget_exhausted = true; + break; + } + let call = sbx.evict_cluster_deleted_sandbox(local_manager_request( + EvictClusterDeletedSandboxRequest { + sandbox_id: sandbox_id.to_string(), + }, + true, + )); + let result = match tokio::time::timeout(CLUSTER_DELETED_EVICT_RPC_TIMEOUT, call).await { + Ok(result) => result, + // The teardown runs detached in sbx-manager, so an + // elapsed deadline means "still in progress", not + // failure; the next sync observes the outcome. + Err(_elapsed) => Err(tonic::Status::deadline_exceeded( + "eviction still in progress at the per-RPC deadline", + )), + }; + let status = match &result { + Ok(_) => { + evicted = evicted.saturating_add(1); + stats.sandboxes_applied = stats.sandboxes_applied.saturating_add(1); + pplx_space_metrics::reconcile::cluster_deleted_evict( + pplx_space_metrics::reconcile::ClusterDeletedEvictResult::Evicted, + ); + continue; + } + Err(status) => status, + }; + match cluster_deleted_evict_refusal_disposition(status) { + ClusterDeletedEvictDisposition::RetryNextSync => { + deferred = deferred.saturating_add(1); + pplx_space_metrics::reconcile::cluster_deleted_evict( + pplx_space_metrics::reconcile::ClusterDeletedEvictResult::Deferred, + ); + tracing::debug!( + sandbox_id, + status = %status, + "cluster-deleted eviction deferred to the next sync: restore staging in flight" + ); + } + ClusterDeletedEvictDisposition::Deferred(backoff) => { + deferred = deferred.saturating_add(1); + CLUSTER_DELETED_EVICT_BACKOFF_UNTIL + .insert(sandbox_id.to_string(), std::time::Instant::now() + backoff); + pplx_space_metrics::reconcile::cluster_deleted_evict( + pplx_space_metrics::reconcile::ClusterDeletedEvictResult::Deferred, + ); + tracing::debug!( + sandbox_id, + status = %status, + backoff_secs = backoff.as_secs(), + "cluster-deleted eviction deferred" + ); + } + ClusterDeletedEvictDisposition::Failed(backoff) => { + CLUSTER_DELETED_EVICT_BACKOFF_UNTIL + .insert(sandbox_id.to_string(), std::time::Instant::now() + backoff); + pplx_space_metrics::reconcile::cluster_deleted_evict( + pplx_space_metrics::reconcile::ClusterDeletedEvictResult::Failed, + ); + tracing::warn!( + sandbox_id, + status = %status, + "cluster-deleted eviction failed" + ); + record_element_error( + &mut element_errors, + sandbox_id, + &format!("cluster-deleted eviction failed: {status}"), + ); + } + } + } + if evicted > 0 || deferred > 0 || budget_exhausted { + tracing::info!( + deleted_reported = response.deleted_sandbox_ids.len(), + evicted, + deferred, + budget_exhausted, + per_sync_cap = evict_cap, + "evicted local presence of cluster-deleted sandboxes" + ); + } + } + + Ok(InboundFacetResult { + stats, + element_errors, + }) +} + +/// sbx-manager RPCs the snapshot / auto-lifecycle desired-state apply sub-ops +/// issue. Kept as an enum so terminal-code classification keys off the exact +/// operation instead of substring-matching a formatted message. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum LocalDesiredStateRpc { + EnableSnapshots, + DisableSnapshots, + SetAutoLifecyclePolicy, +} + +impl LocalDesiredStateRpc { + pub(super) fn as_str(self) -> &'static str { + match self { + Self::EnableSnapshots => "enable snapshots", + Self::DisableSnapshots => "disable snapshots", + Self::SetAutoLifecyclePolicy => "set auto-lifecycle policy", + } + } +} + +impl std::fmt::Display for LocalDesiredStateRpc { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(self.as_str()) + } +} + +/// Failure applying a snapshot / auto-lifecycle desired-state sub-op against +/// the local sbx-manager. RPC failures preserve the tonic `Status` so a +/// guest-unreachable `FailedPrecondition` (sandbox failed/suspending/ +/// suspended/stopped) is distinguishable from transient failures. +#[derive(Debug, thiserror::Error)] +pub(super) enum LocalDesiredStateApplyError { + /// A sbx-manager RPC failed. + #[error("{operation}: {status}")] + Rpc { + operation: LocalDesiredStateRpc, + #[source] + status: tonic::Status, + }, +} + +impl LocalDesiredStateApplyError { + /// See [`sbx_mgr_desired_state_apply_is_terminal`]. + pub(super) fn is_terminal(&self) -> bool { + match self { + Self::Rpc { status, .. } => sbx_mgr_desired_state_apply_is_terminal(status), + } + } +} + +pub(super) async fn apply_sandbox_snapshot_desired( + client: &mut pplx_space_sbx_mgr_client::SandboxManagerGrpcClient, + sandbox_id: &str, + state: &SandboxInfo, + snapshots_enabled_desired: bool, +) -> Result<(), LocalDesiredStateApplyError> { + if state.snapshots_enabled == snapshots_enabled_desired { + return Ok(()); + } + + let (operation, result) = if snapshots_enabled_desired { + ( + LocalDesiredStateRpc::EnableSnapshots, + client + .enable_snapshots(local_manager_request( + EnableSnapshotsRequest { + sandbox_id: sandbox_id.to_string(), + }, + true, + )) + .await + .map(|_| ()), + ) + } else { + ( + LocalDesiredStateRpc::DisableSnapshots, + client + .disable_snapshots(local_manager_request( + DisableSnapshotsRequest { + sandbox_id: sandbox_id.to_string(), + }, + true, + )) + .await + .map(|_| ()), + ) + }; + + result.map_err(|status| LocalDesiredStateApplyError::Rpc { operation, status }) +} + +pub(super) async fn apply_sandbox_auto_lifecycle_desired( + client: &mut pplx_space_sbx_mgr_client::SandboxManagerGrpcClient, + sandbox_id: &str, + state: &SandboxInfo, + policy: pplx_space_sbx_mgr_client::proto::AutoLifecyclePolicy, +) -> Result<(), LocalDesiredStateApplyError> { + if state.auto_lifecycle.as_ref() == Some(&policy) { + return Ok(()); + } + + client + .set_auto_lifecycle_policy(local_manager_request( + SetAutoLifecyclePolicyRequest { + sandbox_id: sandbox_id.to_string(), + policy: Some(policy), + authority: PolicyAuthority::Cluster.as_proto_i32(), + }, + true, + )) + .await + .map(|_| ()) + .map_err(|status| LocalDesiredStateApplyError::Rpc { + operation: LocalDesiredStateRpc::SetAutoLifecyclePolicy, + status, + }) +} + +/// Fold a snapshot / auto-lifecycle apply outcome into the reconcile stats and +/// element errors. `Ok` (applied or already in desired state) counts as +/// applied. A server-marked terminal state-gate failure publishes +/// the diagnostic and metric but records **no** element error, so the sandbox +/// desired-state ref completes. A separate lifecycle/cluster repair changes +/// the state, and periodic desired-state sync re-applies afterward; retrying +/// this unchanged ref could never converge and would pin the queue-lag gauge. +/// A transient failure records an element error so the queue re-drives it. +pub(super) fn handle_local_desired_state_apply( + result: Result<(), LocalDesiredStateApplyError>, + sandbox_id: &str, + stats: &mut ReconcileStats, + element_errors: &mut BTreeMap, +) { + let Err(error) = result else { + stats.sandboxes_applied = stats.sandboxes_applied.saturating_add(1); + return; + }; + let result_kind = if error.is_terminal() { + tracing::info!( + sandbox_id, + %error, + "terminal sandbox desired-state apply; completing ref until state changes" + ); + pplx_space_metrics::reconcile::DesiredStateApplyResult::Terminal + } else { + tracing::warn!( + sandbox_id, + %error, + "retryable sandbox desired-state apply failure" + ); + record_element_error(element_errors, sandbox_id, &error.to_string()); + pplx_space_metrics::reconcile::DesiredStateApplyResult::Retryable + }; + pplx_space_metrics::reconcile::desired_state_apply(result_kind); +} diff --git a/crates/libs/node/ctl/server/src/facets/template_desired_state.rs b/crates/libs/node/ctl/server/src/facets/template_desired_state.rs new file mode 100644 index 000000000..71d611c2c --- /dev/null +++ b/crates/libs/node/ctl/server/src/facets/template_desired_state.rs @@ -0,0 +1,303 @@ +//! The template desired-state facet and the template base-snapshot policy refresh. + +use super::*; + +pub(super) async fn refresh_template_base_snapshot_policy( + node_id: &str, + node_role: Option, + scope: FacetScope, +) -> Result { + let templates = resolve_template_scope(scope).await?; + let mut stats = ReconcileStats { + templates_seen: templates.len() as u64, + ..ReconcileStats::default() + }; + if templates.is_empty() { + return Ok(stats); + } + + let mut client = connect_cluster_api() + .await + .map_err(|error| error.to_string())?; + let response = client + .get_node_template_base_snapshot_policy_state( + GetNodeTemplateBaseSnapshotPolicyStateRequest { + node_id: node_id.to_string(), + templates, + role: proto_node_role(node_role), + }, + ) + .await + .map_err(|error| format!("cluster base-snapshot policy refresh failed: {error}"))? + .into_inner(); + + let mut tmpl = pplx_space_template_mgr_client::connect() + .await + .map_err(|error| format!("connect template-manager: {error}"))?; + let mut failures = 0_u64; + for template in response.templates { + if let Err(error) = tmpl + .update_template_snapshot_policy(local_manager_request( + UpdateTemplateSnapshotPolicyRequest { + template: template.template.clone(), + ensure_base_snapshot: template.ensure_base_snapshot, + snapshot_vcpu: Some(template.snapshot_vcpu), + snapshot_memory_mib: Some(template.snapshot_memory_mib), + }, + false, + )) + .await + { + tracing::warn!( + template = %template.template, + %error, + "failed to apply template base-snapshot policy" + ); + failures = failures.saturating_add(1); + continue; + } + stats.templates_applied = stats.templates_applied.saturating_add(1); + } + if failures > 0 { + return Err(format!( + "failed to apply {failures} template base-snapshot policy item(s)" + )); + } + Ok(stats) +} + +pub(super) async fn refresh_template_desired_state( + node_id: &str, + node_role: Option, + scope: FacetScope, +) -> Result { + let templates = resolve_template_scope(scope).await?; + let mut stats = ReconcileStats { + templates_seen: templates.len() as u64, + ..ReconcileStats::default() + }; + if templates.is_empty() { + return Ok(stats); + } + + let mut cluster = connect_cluster_api() + .await + .map_err(|error| error.to_string())?; + let response = cluster + .get_node_template_desired_state(GetNodeTemplateDesiredStateRequest { + node_id: node_id.to_string(), + templates, + role: proto_node_role(node_role), + }) + .await + .map_err(|error| format!("cluster template desired-state refresh failed: {error}"))? + .into_inner(); + + let mut tmpl = pplx_space_template_mgr_client::connect() + .await + .map_err(|error| format!("connect template-manager: {error}"))?; + let local_templates = tmpl + .list_templates(Request::new(ListTemplatesRequest {})) + .await + .map_err(|error| format!("list local templates: {error}"))? + .into_inner() + .templates + .into_iter() + .map(|template| (template.name.clone(), template.default_auto_lifecycle)) + .collect::>(); + let local_pools = tmpl + .pool_status(Request::new(PoolStatusRequest { template: None })) + .await + .map(|response| { + response + .into_inner() + .pools + .into_iter() + .map(|pool| (pool.template.clone(), pool)) + .collect::>() + }) + .unwrap_or_default(); + + let mut failures = 0_u64; + for template in response.templates { + let mut applied = false; + if template.present == Some(false) { + if local_pools.contains_key(&template.template) { + match tmpl + .delete_pool(local_manager_request( + DeletePoolRequest { + template: template.template.clone(), + }, + true, + )) + .await + { + Ok(_) => { + applied = true; + } + Err(error) => { + tracing::warn!( + template = %template.template, + %error, + "failed to delete undesired template pool" + ); + failures = failures.saturating_add(1); + } + } + } + if local_templates.contains_key(&template.template) { + match tmpl + .delete_template(local_manager_request( + DeleteTemplateRequest { + template: template.template.clone(), + }, + true, + )) + .await + { + Ok(_) => { + applied = true; + } + Err(error) if error.code() == Code::NotFound => {} + Err(error) => { + tracing::warn!( + template = %template.template, + %error, + "failed to delete undesired template" + ); + failures = failures.saturating_add(1); + } + } + } + if applied { + stats.templates_applied = stats.templates_applied.saturating_add(1); + } + continue; + } + + if !local_templates.contains_key(&template.template) { + if local_pools.contains_key(&template.template) { + match tmpl + .delete_pool(local_manager_request( + DeletePoolRequest { + template: template.template.clone(), + }, + true, + )) + .await + { + Ok(_) => { + stats.templates_applied = stats.templates_applied.saturating_add(1); + } + Err(error) => { + tracing::warn!( + template = %template.template, + %error, + "failed to delete pool for absent local template" + ); + failures = failures.saturating_add(1); + } + } + } + continue; + } + + if let Some(policy) = template.default_auto_lifecycle.clone() { + let current = local_templates + .get(&template.template) + .and_then(|policy| policy.as_ref()); + if current != Some(&policy) { + match tmpl + .update_template_settings(local_manager_request( + UpdateTemplateSettingsRequest { + template: template.template.clone(), + default_auto_lifecycle: Some(policy), + network_policy: None, + network_policy_generation: 0, + network_policy_authority: PolicyAuthority::Cluster.as_proto_i32(), + auto_lifecycle_authority: PolicyAuthority::Cluster.as_proto_i32(), + }, + true, + )) + .await + { + Ok(_) => { + applied = true; + } + Err(error) => { + tracing::warn!( + template = %template.template, + %error, + "failed to apply template default auto-lifecycle policy" + ); + failures = failures.saturating_add(1); + } + } + } + } + + match template.pool { + Some(pool) => match tmpl + .create_pool(local_manager_request( + CreatePoolRequest { + template: template.template.clone(), + target_count: Some(pool.target_count), + boot_mode: pool.boot_mode, + vcpu: pool.vcpu, + memory_mib: pool.memory_mib, + }, + true, + )) + .await + { + Ok(_) => { + applied = true; + } + Err(error) => { + tracing::warn!( + template = %template.template, + %error, + "failed to apply template pool policy" + ); + failures = failures.saturating_add(1); + } + }, + None => { + if local_pools.contains_key(&template.template) { + match tmpl + .delete_pool(local_manager_request( + DeletePoolRequest { + template: template.template.clone(), + }, + true, + )) + .await + { + Ok(_) => { + applied = true; + } + Err(error) => { + tracing::warn!( + template = %template.template, + %error, + "failed to delete undesired template pool" + ); + failures = failures.saturating_add(1); + } + } + } + } + } + + if applied { + stats.templates_applied = stats.templates_applied.saturating_add(1); + } + } + + if failures > 0 { + return Err(format!( + "failed to apply {failures} template desired-state item(s)" + )); + } + Ok(stats) +}