diff --git a/.sqlx/query-09f4f824ed960a6bfbf6b8c3c28cd3c88cb041e76f1af34b79751cca2266174f.json b/.sqlx/query-09f4f824ed960a6bfbf6b8c3c28cd3c88cb041e76f1af34b79751cca2266174f.json new file mode 100644 index 0000000..7e92adb --- /dev/null +++ b/.sqlx/query-09f4f824ed960a6bfbf6b8c3c28cd3c88cb041e76f1af34b79751cca2266174f.json @@ -0,0 +1,64 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT app_id, url, checked_at, last_error FROM app_sources WHERE app_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "app_id", + "type_info": "Text", + "origin": { + "Table": { + "table": "app_sources", + "name": "app_id" + } + } + }, + { + "ordinal": 1, + "name": "url", + "type_info": "Text", + "origin": { + "Table": { + "table": "app_sources", + "name": "url" + } + } + }, + { + "ordinal": 2, + "name": "checked_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "app_sources", + "name": "checked_at" + } + } + }, + { + "ordinal": 3, + "name": "last_error", + "type_info": "Text", + "origin": { + "Table": { + "table": "app_sources", + "name": "last_error" + } + } + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + true, + true + ] + }, + "hash": "09f4f824ed960a6bfbf6b8c3c28cd3c88cb041e76f1af34b79751cca2266174f" +} diff --git a/.sqlx/query-33fe8be23d0952bcd9e3e3e312aecfbb2922625384ec931684f05b3aaf7adcdb.json b/.sqlx/query-33fe8be23d0952bcd9e3e3e312aecfbb2922625384ec931684f05b3aaf7adcdb.json new file mode 100644 index 0000000..e99cfa3 --- /dev/null +++ b/.sqlx/query-33fe8be23d0952bcd9e3e3e312aecfbb2922625384ec931684f05b3aaf7adcdb.json @@ -0,0 +1,65 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO app_sources (app_id, url) VALUES ($1, $2)\n ON CONFLICT (app_id) DO UPDATE\n SET url = EXCLUDED.url,\n checked_at = CASE WHEN app_sources.url = EXCLUDED.url\n THEN app_sources.checked_at END,\n last_error = CASE WHEN app_sources.url = EXCLUDED.url\n THEN app_sources.last_error END\n RETURNING app_id, url, checked_at, last_error", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "app_id", + "type_info": "Text", + "origin": { + "Table": { + "table": "app_sources", + "name": "app_id" + } + } + }, + { + "ordinal": 1, + "name": "url", + "type_info": "Text", + "origin": { + "Table": { + "table": "app_sources", + "name": "url" + } + } + }, + { + "ordinal": 2, + "name": "checked_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "app_sources", + "name": "checked_at" + } + } + }, + { + "ordinal": 3, + "name": "last_error", + "type_info": "Text", + "origin": { + "Table": { + "table": "app_sources", + "name": "last_error" + } + } + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + true, + true + ] + }, + "hash": "33fe8be23d0952bcd9e3e3e312aecfbb2922625384ec931684f05b3aaf7adcdb" +} diff --git a/.sqlx/query-6099560f8e8294d1455186bc5f5fd91b7b516018bbb6d29da278131a56a55b4b.json b/.sqlx/query-6099560f8e8294d1455186bc5f5fd91b7b516018bbb6d29da278131a56a55b4b.json new file mode 100644 index 0000000..e52836f --- /dev/null +++ b/.sqlx/query-6099560f8e8294d1455186bc5f5fd91b7b516018bbb6d29da278131a56a55b4b.json @@ -0,0 +1,62 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT app_id, url, checked_at, last_error FROM app_sources ORDER BY app_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "app_id", + "type_info": "Text", + "origin": { + "Table": { + "table": "app_sources", + "name": "app_id" + } + } + }, + { + "ordinal": 1, + "name": "url", + "type_info": "Text", + "origin": { + "Table": { + "table": "app_sources", + "name": "url" + } + } + }, + { + "ordinal": 2, + "name": "checked_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "app_sources", + "name": "checked_at" + } + } + }, + { + "ordinal": 3, + "name": "last_error", + "type_info": "Text", + "origin": { + "Table": { + "table": "app_sources", + "name": "last_error" + } + } + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + true, + true + ] + }, + "hash": "6099560f8e8294d1455186bc5f5fd91b7b516018bbb6d29da278131a56a55b4b" +} diff --git a/.sqlx/query-9fed993e8dfbc0e7eeca1b6c01683e8341aba5e59f72e1a58025e7b13583a69a.json b/.sqlx/query-9fed993e8dfbc0e7eeca1b6c01683e8341aba5e59f72e1a58025e7b13583a69a.json new file mode 100644 index 0000000..cb90e95 --- /dev/null +++ b/.sqlx/query-9fed993e8dfbc0e7eeca1b6c01683e8341aba5e59f72e1a58025e7b13583a69a.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE app_sources SET checked_at = $2, last_error = $3 WHERE app_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Timestamptz", + "Text" + ] + }, + "nullable": [] + }, + "hash": "9fed993e8dfbc0e7eeca1b6c01683e8341aba5e59f72e1a58025e7b13583a69a" +} diff --git a/.sqlx/query-bc2e79109d949ae99fb1df140bf37550105763beacd61bdc17551bbaeae3aacc.json b/.sqlx/query-bc2e79109d949ae99fb1df140bf37550105763beacd61bdc17551bbaeae3aacc.json new file mode 100644 index 0000000..e62c3c2 --- /dev/null +++ b/.sqlx/query-bc2e79109d949ae99fb1df140bf37550105763beacd61bdc17551bbaeae3aacc.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM app_sources WHERE app_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "bc2e79109d949ae99fb1df140bf37550105763beacd61bdc17551bbaeae3aacc" +} diff --git a/CHANGELOG.md b/CHANGELOG.md index c35868c..3455e33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,36 @@ All notable changes to Setu. Release notes are cut from the section matching the tag. +## 0.1.5 — 2026-10-21 + +Spec: `docs/specs/2026-10-21-setu-0.1.5-design.md`. + +### Added + +- **Apps follow their manifest URL.** An app published from a URL is + checked again every 6 hours, and any new version is published, so it + appears in each site's Updates tab without publishing it again by hand. + Sites are never updated on their own, and a check never replaces a + version that is already published. The Apps page shows the URL and the + last check, with **Check now** and **Stop following**. The API is + `POST /api/v1/apps/{id}/check` and `DELETE /api/v1/apps/{id}/source`, + and `AppOut` gains `source`. +- **`setu upgrade [--version ]`.** Run as root, it upgrades this + server to a release (the latest by default). It verifies the checksum, + backs up the control plane, installs the binary, restarts `setu` and + `setu-agent`, and waits for `/healthz`. Older releases are refused + unless `--allow-downgrade`. + +### Changed + +- `setu` reads `/etc/setu/setu.toml` when the current directory has no + `setu.toml`, so `setu backup` and `setu upgrade` work from any + directory. + +### Migrations + +- `0004_app_sources.sql` (forward-only): the `app_sources` table. + ## 0.1.4 — 2026-10-20 Spec: `docs/specs/2026-10-20-setu-0.1.4-design.md`. diff --git a/Cargo.lock b/Cargo.lock index a10fc9e..3cfb77a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -977,7 +977,7 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hello" -version = "0.1.4" +version = "0.1.5" dependencies = [ "axum", "reqwest", @@ -2170,7 +2170,7 @@ dependencies = [ [[package]] name = "setu-agent" -version = "0.1.4" +version = "0.1.5" dependencies = [ "anyhow", "async-trait", @@ -2204,7 +2204,7 @@ dependencies = [ [[package]] name = "setu-api" -version = "0.1.4" +version = "0.1.5" dependencies = [ "argon2", "async-trait", @@ -2212,6 +2212,7 @@ dependencies = [ "base64 0.22.1", "chrono", "clap", + "flate2", "futures", "hex", "hostname", @@ -2223,6 +2224,7 @@ dependencies = [ "rand 0.9.5", "reqwest", "rust-embed", + "semver", "serde", "serde_json", "setu-agent", @@ -2234,6 +2236,7 @@ dependencies = [ "setu-testkit", "sha2 0.10.9", "sqlx", + "tar", "tempfile", "tokio", "tower", @@ -2245,7 +2248,7 @@ dependencies = [ [[package]] name = "setu-common" -version = "0.1.4" +version = "0.1.5" dependencies = [ "anyhow", "axum", @@ -2271,7 +2274,7 @@ dependencies = [ [[package]] name = "setu-db" -version = "0.1.4" +version = "0.1.5" dependencies = [ "anyhow", "base64 0.22.1", @@ -2293,7 +2296,7 @@ dependencies = [ [[package]] name = "setu-jobs" -version = "0.1.4" +version = "0.1.5" dependencies = [ "async-trait", "chrono", @@ -2314,7 +2317,7 @@ dependencies = [ [[package]] name = "setu-manifest" -version = "0.1.4" +version = "0.1.5" dependencies = [ "hex", "rand 0.9.5", @@ -2328,7 +2331,7 @@ dependencies = [ [[package]] name = "setu-proto" -version = "0.1.4" +version = "0.1.5" dependencies = [ "chrono", "serde", @@ -2338,7 +2341,7 @@ dependencies = [ [[package]] name = "setu-testkit" -version = "0.1.4" +version = "0.1.5" dependencies = [ "sqlx", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 9843b13..5305e19 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ members = [ ] [workspace.package] -version = "0.1.4" +version = "0.1.5" edition = "2021" license = "AGPL-3.0-only" # Never published to crates.io (single binary + private repo); also diff --git a/crates/api/Cargo.toml b/crates/api/Cargo.toml index c2e05ac..982aca9 100644 --- a/crates/api/Cargo.toml +++ b/crates/api/Cargo.toml @@ -47,6 +47,9 @@ base64.workspace = true ipnet.workspace = true nix.workspace = true tempfile.workspace = true +semver.workspace = true +tar.workspace = true +flate2.workspace = true tracing.workspace = true tracing-subscriber.workspace = true setu-common = { path = "../common" } diff --git a/crates/api/src/follow.rs b/crates/api/src/follow.rs new file mode 100644 index 0000000..e2bc85b --- /dev/null +++ b/crates/api/src/follow.rs @@ -0,0 +1,126 @@ +//! Followed manifest URLs (0.1.5 §1): an app published from a URL is +//! fetched again every [`CHECK_EVERY`] and any version not yet published +//! is published. A check never replaces a published version: a manifest +//! names binaries and their checksums, and an unattended re-publish must +//! not swap what a running version installs. + +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use setu_common::AppError; +use setu_db::{AppSource, Db}; +use setu_jobs::Clock; + +use crate::auth::audit; + +/// How often every followed app is checked. +pub const CHECK_EVERY: Duration = Duration::from_hours(6); + +/// Fetches a manifest by URL; the real one applies the publish guards. +#[async_trait] +pub trait ManifestFetcher: Send + Sync + std::fmt::Debug { + async fn fetch(&self, url: &str) -> Result; +} + +/// The guarded HTTP fetch `POST /api/v1/apps` uses. +#[derive(Debug, Clone, Copy, Default)] +pub struct HttpFetcher; + +#[async_trait] +impl ManifestFetcher for HttpFetcher { + async fn fetch(&self, url: &str) -> Result { + crate::rest::apps::fetch_manifest_url(url).await + } +} + +/// Fetch `source.url` once and publish its version when it is new. +/// Returns the version published, or `None` when it was already +/// published. The outcome is recorded on the source either way. +pub async fn check( + db: &Db, + fetcher: &dyn ManifestFetcher, + clock: &dyn Clock, + source: &AppSource, +) -> Result, AppError> { + let outcome = check_inner(db, fetcher, source).await; + let error = outcome.as_ref().err().map(ToString::to_string); + db.app_sources() + .record_check(&source.app_id, clock.now(), error.as_deref()) + .await?; + outcome +} + +async fn check_inner( + db: &Db, + fetcher: &dyn ManifestFetcher, + source: &AppSource, +) -> Result, AppError> { + let src = fetcher.fetch(&source.url).await?; + let manifest = crate::rest::apps::parse_or_validation(&src)?; + if manifest.app.id != source.app_id { + return Err(AppError::validation(format!( + "manifest is for app {}, not {}", + manifest.app.id, source.app_id + ))); + } + match db + .apps() + .version(&source.app_id, &manifest.app.version) + .await + { + Ok(_) => return Ok(None), + Err(e) if e.code == setu_common::ErrorCode::NotFound => {} + Err(e) => return Err(e), + } + let json = serde_json::to_value(&manifest).map_err(|e| { + tracing::error!(error = %e, "manifest re-serialisation failed"); + AppError::internal() + })?; + let row = db.apps().upsert_version(&json).await?; + audit( + db, + None, + "app.publish", + Some(&format!("{}/{} (followed)", row.app_id, row.version)), + None, + ) + .await; + tracing::info!(app = %row.app_id, version = %row.version, "followed URL published a new version"); + Ok(Some(row.version)) +} + +/// Check every followed app now, one after another. +pub async fn check_all(db: &Db, fetcher: &dyn ManifestFetcher, clock: &dyn Clock) { + let sources = match db.app_sources().list().await { + Ok(s) => s, + Err(e) => { + tracing::error!(error = %e, "app sources unreadable"); + return; + } + }; + for source in &sources { + if let Err(e) = check(db, fetcher, clock, source).await { + tracing::warn!(app = %source.app_id, error = %e, "followed URL check failed"); + } + } +} + +/// The background loop: a round every [`CHECK_EVERY`], the first one +/// `first_after` after start. +pub fn spawn( + db: Db, + fetcher: Arc, + clock: Arc, + first_after: Duration, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + let mut every = + tokio::time::interval_at(tokio::time::Instant::now() + first_after, CHECK_EVERY); + every.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + every.tick().await; + check_all(&db, fetcher.as_ref(), clock.as_ref()).await; + } + }) +} diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index 58245cc..ff987c2 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -4,18 +4,21 @@ //! sessions, the client IP, role checks, the `Origin` guard and password //! hashing; [`rest`] assembles the router; [`listen`] binds the unix //! socket or TCP listener; [`notifier`] wakes agent long polls; [`cli`] -//! backs `setu operator create` and `setu backup`; +//! backs `setu operator create` and `setu backup`; [`upgrade`] is +//! `setu upgrade`; [`follow`] re-checks followed manifest URLs; //! [`testing::TestServer`] boots the whole stack against a throwaway //! database for integration tests. pub mod auth; pub mod cli; pub mod embed; +pub mod follow; pub mod listen; pub mod notifier; pub mod rest; pub mod self_backup; pub mod state; pub mod testing; +pub mod upgrade; pub use state::AppState; diff --git a/crates/api/src/main.rs b/crates/api/src/main.rs index c631fb9..03e4861 100644 --- a/crates/api/src/main.rs +++ b/crates/api/src/main.rs @@ -1,7 +1,8 @@ //! The one `setu` binary (spec §8): `serve` is the control plane, //! `agent …` the per-server agent and `helper …` its privileged writer //! (both from the `setu-agent` library crate), plus operator management, -//! the control plane's own database backup and OpenAPI export. +//! the control plane's own database backup, OpenAPI export and +//! `upgrade` (0.1.5 §2). use std::sync::Arc; @@ -49,6 +50,16 @@ enum Cmd { #[arg(long, default_value = "openapi.json")] out: std::path::PathBuf, }, + /// Upgrade this server's `setu` to a release (default: the latest): + /// verify, back up the control plane, install, restart. Run as root. + Upgrade { + /// Release version, e.g. `0.1.5`. + #[arg(long)] + version: Option, + /// Allow installing an older release (migrations are forward-only). + #[arg(long)] + allow_downgrade: bool, + }, /// Per-server agent (library crate `setu-agent`). Agent { #[command(subcommand)] @@ -91,6 +102,16 @@ async fn main() { cmd: SettingCmd::Set { key, value }, } => setting_set(&key, &value).await, Cmd::Openapi { out } => openapi(&out).await, + Cmd::Upgrade { + version, + allow_downgrade, + } => { + setu_api::upgrade::run(setu_api::upgrade::UpgradeArgs { + version, + allow_downgrade, + }) + .await + } Cmd::Agent { cmd } => { // The agent's own log (journald captures stderr); stdout // stays clean for `doctor`'s table and install output. @@ -172,6 +193,14 @@ async fn serve() -> Result<(), String> { clock.clone(), setu_jobs::SchedulerConfig::default(), ); + // Followed manifest URLs, every 6 h (0.1.5 §1); the first round a + // minute after start so a restart picks up new versions soon. + let _follow = setu_api::follow::spawn( + db.clone(), + state.fetcher.clone(), + clock.clone(), + std::time::Duration::from_mins(1), + ); // Nightly pg_dump of Setu's own database (0.1.1 #9, R145). let _self_backup = setu_api::self_backup::spawn( db.clone(), diff --git a/crates/api/src/rest/apps.rs b/crates/api/src/rest/apps.rs index bd5602f..08a25b0 100644 --- a/crates/api/src/rest/apps.rs +++ b/crates/api/src/rest/apps.rs @@ -1,11 +1,13 @@ //! `GET /api/v1/apps`, `POST /apps/validate`, `POST /apps`, -//! `GET /api/v1/apps/{id}/versions`. +//! `GET /api/v1/apps/{id}/versions`, `POST /apps/{id}/check`, +//! `DELETE /apps/{id}/source`. //! //! Publishing accepts either pasted TOML or a URL. URL fetches carry the //! same guard rules as the installer: public hosts only (private, loopback, //! link-local and other special ranges are rejected before connecting), a //! 1 MiB response cap and a 10 s timeout, with redirects disabled so a -//! public URL cannot bounce to an internal one. +//! public URL cannot bounce to an internal one. An app published from a +//! URL follows it (0.1.5 §1, [`crate::follow`]). use std::net::{IpAddr, SocketAddr}; use std::time::Duration; @@ -35,6 +37,33 @@ pub struct AppOut { pub id: String, pub name: String, pub homepage: Option, + /// The manifest URL the app follows, if any. + pub source: Option, +} + +/// A followed manifest URL and its last check. +#[derive(Debug, Serialize, ToSchema)] +pub struct AppSourceOut { + pub url: String, + pub checked_at: Option>, + pub last_error: Option, +} + +impl From for AppSourceOut { + fn from(s: setu_db::AppSource) -> Self { + Self { + url: s.url, + checked_at: s.checked_at, + last_error: s.last_error, + } + } +} + +/// `POST /apps/{id}/check`: the version the check published, if any. +#[derive(Debug, Serialize, ToSchema)] +pub struct CheckOut { + pub published: Option, + pub source: AppSourceOut, } #[derive(Debug, Serialize, ToSchema)] @@ -103,10 +132,19 @@ pub async fn list( ) -> Result { require_role(&session, Role::Viewer).await?; let apps = state.db.apps().list().await?; + let mut sources: std::collections::HashMap = state + .db + .app_sources() + .list() + .await? + .into_iter() + .map(|s| (s.app_id.clone(), s)) + .collect(); Ok(Json(AppListOut { apps: apps .into_iter() .map(|a| AppOut { + source: sources.remove(&a.id).map(Into::into), id: a.id, name: a.name, homepage: a.homepage, @@ -154,9 +192,9 @@ pub async fn publish( ApiJson(body): ApiJson, ) -> Result { let caller = require_role(&session, Role::Operator).await?; - let src = match (body.toml, body.url) { + let src = match (body.toml, body.url.as_deref()) { (Some(toml), None) => toml, - (None, Some(url)) => fetch_manifest_url(&url).await?, + (None, Some(url)) => state.fetcher.fetch(url).await?, (Some(_), Some(_)) => { return Err(AppError::validation("provide exactly one of toml, url")); } @@ -171,6 +209,10 @@ pub async fn publish( })?; let row = state.db.apps().upsert_version(&json).await?; let app = state.db.apps().get(&row.app_id).await?; + let source = match body.url.as_deref() { + Some(url) => Some(state.db.app_sources().set(&app.id, url).await?), + None => state.db.app_sources().get(&app.id).await?, + }; audit( &state.db, Some(&caller.id), @@ -186,6 +228,7 @@ pub async fn publish( id: app.id, name: app.name, homepage: app.homepage, + source: source.map(Into::into), }, version: AppVersionOut { id: row.id, @@ -231,6 +274,84 @@ pub async fn versions( })) } +/// Fetch the app's followed URL now and publish its version when new +/// (operator and above). +#[utoipa::path( + post, + path = "/api/v1/apps/{id}/check", + params(("id" = String, Path, description = "App id")), + responses( + (status = 200, body = CheckOut), + (status = 400, body = super::ErrorBody), + (status = 401, body = super::ErrorBody), + (status = 403, body = super::ErrorBody), + (status = 404, body = super::ErrorBody), + ) +)] +pub async fn check( + State(state): State, + session: Session, + Path(id): Path, +) -> Result { + require_role(&session, Role::Operator).await?; + let source = state + .db + .app_sources() + .get(&id) + .await? + .ok_or_else(|| AppError::not_found("app follows no manifest URL"))?; + let published = crate::follow::check( + &state.db, + state.fetcher.as_ref(), + state.clock.as_ref(), + &source, + ) + .await?; + let source = state + .db + .app_sources() + .get(&id) + .await? + .ok_or_else(|| AppError::not_found("app follows no manifest URL"))?; + Ok(Json(CheckOut { + published, + source: source.into(), + })) +} + +/// Stop following the app's manifest URL (operator and above). +#[utoipa::path( + delete, + path = "/api/v1/apps/{id}/source", + params(("id" = String, Path, description = "App id")), + responses( + (status = 204), + (status = 401, body = super::ErrorBody), + (status = 403, body = super::ErrorBody), + (status = 404, body = super::ErrorBody), + ) +)] +pub async fn clear_source( + State(state): State, + session: Session, + ip: ClientIp, + Path(id): Path, +) -> Result { + let caller = require_role(&session, Role::Operator).await?; + if !state.db.app_sources().clear(&id).await? { + return Err(AppError::not_found("app follows no manifest URL")); + } + audit( + &state.db, + Some(&caller.id), + "app.source_clear", + Some(&id), + ip.as_deref(), + ) + .await; + Ok(StatusCode::NO_CONTENT) +} + fn validate_toml(src: &str) -> ValidateResponse { match setu_manifest::Manifest::parse(src) { Ok(_) => ValidateResponse { @@ -251,7 +372,7 @@ fn validate_toml(src: &str) -> ValidateResponse { } } -fn parse_or_validation(src: &str) -> Result { +pub(crate) fn parse_or_validation(src: &str) -> Result { setu_manifest::Manifest::parse(src).map_err(|errors| { let first = errors .iter() @@ -325,7 +446,7 @@ fn is_non_public(ip: IpAddr) -> bool { /// the URL keeps its host name, so https still verifies the certificate /// against it (SNI). This closes the TOCTOU the Task 7 ruling accepted /// (M3, R138). -async fn fetch_manifest_url(url_str: &str) -> Result { +pub(crate) async fn fetch_manifest_url(url_str: &str) -> Result { let url = url::Url::parse(url_str).map_err(|_| AppError::validation("url is not a valid URL"))?; if url.scheme() != "http" && url.scheme() != "https" { diff --git a/crates/api/src/rest/docs.rs b/crates/api/src/rest/docs.rs index 1705711..652b1c6 100644 --- a/crates/api/src/rest/docs.rs +++ b/crates/api/src/rest/docs.rs @@ -10,8 +10,8 @@ use utoipa::OpenApi; use crate::auth::{require_role, Role, Session}; use super::apps::{ - AppListOut, AppOut, AppVersionListOut, AppVersionOut, ManifestErrorOut, PromptListOut, - PromptOut, PublishRequest, PublishResponse, ValidateRequest, ValidateResponse, + AppListOut, AppOut, AppSourceOut, AppVersionListOut, AppVersionOut, CheckOut, ManifestErrorOut, + PromptListOut, PromptOut, PublishRequest, PublishResponse, ValidateRequest, ValidateResponse, }; use super::auth::{LoginRequest, OperatorOut}; use super::backups::{ @@ -63,6 +63,8 @@ use super::{ErrorBody, ErrorDetail, HealthOut, ReadyOut}; super::apps::validate, super::apps::publish, super::apps::versions, + super::apps::check, + super::apps::clear_source, super::apps::version_prompts, super::panel::backup_status, super::plans::list, @@ -124,6 +126,8 @@ use super::{ErrorBody, ErrorDetail, HealthOut, ReadyOut}; LocalServerResponse, AppOut, AppListOut, + AppSourceOut, + CheckOut, AppVersionOut, AppVersionListOut, ValidateRequest, diff --git a/crates/api/src/rest/mod.rs b/crates/api/src/rest/mod.rs index 803ff4d..d368bb9 100644 --- a/crates/api/src/rest/mod.rs +++ b/crates/api/src/rest/mod.rs @@ -84,6 +84,8 @@ pub fn router(state: AppState) -> Router { .route("/apps", get(apps::list).post(apps::publish)) .route("/apps/validate", post(apps::validate)) .route("/apps/{id}/versions", get(apps::versions)) + .route("/apps/{id}/check", post(apps::check)) + .route("/apps/{id}/source", delete(apps::clear_source)) .route( "/apps/{id}/versions/{version}/prompts", get(apps::version_prompts), diff --git a/crates/api/src/state.rs b/crates/api/src/state.rs index 735f22d..4d53980 100644 --- a/crates/api/src/state.rs +++ b/crates/api/src/state.rs @@ -40,6 +40,8 @@ pub struct AppState { pub agent: Arc, /// Wakes `GET /agent/v1/commands` long polls on enqueue. pub notifier: Notifier, + /// Fetches followed manifest URLs (tests swap in a fake). + pub fetcher: Arc, login_failures: Arc>>>, agent_polls: Arc>>>, agent_poll_limit: usize, @@ -76,6 +78,7 @@ impl AppState { clock, agent, notifier, + fetcher: Arc::new(crate::follow::HttpFetcher), login_failures: Arc::new(tokio::sync::Mutex::new(HashMap::new())), agent_polls: Arc::new(tokio::sync::Mutex::new(HashMap::new())), agent_poll_limit: MAX_AGENT_POLLS, @@ -90,6 +93,13 @@ impl AppState { self } + /// Replace the manifest fetcher (tests serve manifests from memory). + #[must_use] + pub fn with_fetcher(mut self, fetcher: Arc) -> Self { + self.fetcher = fetcher; + self + } + /// Count one password attempt for the IP *before* the password is /// verified, in the same critical section as the check: at /// [`MAX_LOGIN_FAILURES`] attempts inside the window the attempt is diff --git a/crates/api/src/testing.rs b/crates/api/src/testing.rs index 811b022..c5cf511 100644 --- a/crates/api/src/testing.rs +++ b/crates/api/src/testing.rs @@ -41,6 +41,8 @@ pub struct TestOptions { /// TCP; this only changes what handlers read, e.g. the local install /// command). pub config_listen: Option, + /// The manifest fetcher (default: the guarded HTTP fetch). + pub fetcher: Option>, } /// A running control plane for integration tests. @@ -157,6 +159,9 @@ impl TestServer { if let Some(limit) = opts.agent_poll_limit { state = state.with_agent_poll_limit(limit); } + if let Some(fetcher) = opts.fetcher { + state = state.with_fetcher(fetcher); + } let app = crate::rest::router(state.clone()); let serve_app = app.clone(); tokio::spawn(async move { diff --git a/crates/api/src/upgrade.rs b/crates/api/src/upgrade.rs new file mode 100644 index 0000000..e5d4bef --- /dev/null +++ b/crates/api/src/upgrade.rs @@ -0,0 +1,379 @@ +//! `setu upgrade [--version ]` (0.1.5 §2): moves this server's +//! binary to a release with the installer's checks, then restarts what +//! runs it. On a control plane it first takes a self-backup (the same +//! dump and rotation as the nightly one) and afterwards waits for +//! `/healthz` on the configured listener. + +use std::io::{Read as _, Write as _}; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::Arc; +use std::time::Duration; + +use sha2::{Digest as _, Sha256}; + +/// Where releases are published (`install.sh`'s `REPO`). +const REPO: &str = "setucp/setu"; +/// The installed binary (`install.sh`). +const INSTALL_PATH: &str = "/opt/setu/setu"; +/// Longest wait for `/healthz` after the restart. +const HEALTH_WAIT: Duration = Duration::from_secs(90); +const DOWNLOAD_TIMEOUT: Duration = Duration::from_mins(5); + +/// What `setu upgrade` was asked for. +#[derive(Debug, Clone, Default)] +pub struct UpgradeArgs { + pub version: Option, + pub allow_downgrade: bool, +} + +/// The release target of this build (`install.sh`'s `select_archive`). +pub fn target() -> Result<&'static str, String> { + match std::env::consts::ARCH { + "x86_64" => Ok("x86_64-unknown-linux-musl"), + "aarch64" => Ok("aarch64-unknown-linux-musl"), + other => Err(format!("no release is built for {other}")), + } +} + +/// `setu--.tar.gz`. +pub fn archive_name(version: &str, target: &str) -> String { + format!("setu-{version}-{target}.tar.gz") +} + +/// The checksum `SHA256SUMS` lists for `file` (` `, or the +/// binary-mode ` *`). +pub fn sum_for(sums: &str, file: &str) -> Option { + sums.lines().find_map(|line| { + let mut parts = line.split_whitespace(); + let hex = parts.next()?; + let name = parts.next()?; + let ok = hex.len() == 64 && hex.bytes().all(|b| b.is_ascii_hexdigit()); + (ok && name.trim_start_matches('*') == file).then(|| hex.to_ascii_lowercase()) + }) +} + +/// What to do going from `current` to `wanted`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Plan { + AlreadyCurrent, + Upgrade, + Downgrade, +} + +/// Compares two release versions (semver; a leading `v` is ignored). +pub fn plan(current: &str, wanted: &str) -> Result { + let parse = |v: &str| { + semver::Version::parse(v.trim_start_matches('v')) + .map_err(|e| format!("version {v:?} is not semver: {e}")) + }; + let (cur, want) = (parse(current)?, parse(wanted)?); + Ok(match want.cmp(&cur) { + std::cmp::Ordering::Equal => Plan::AlreadyCurrent, + std::cmp::Ordering::Greater => Plan::Upgrade, + std::cmp::Ordering::Less => Plan::Downgrade, + }) +} + +fn say(msg: &str) { + println!("==> {msg}"); +} + +fn http() -> Result { + reqwest::Client::builder() + .user_agent(concat!("setu/", env!("CARGO_PKG_VERSION"))) + .timeout(DOWNLOAD_TIMEOUT) + .build() + .map_err(|e| format!("http client: {e}")) +} + +async fn get_bytes(client: &reqwest::Client, url: &str) -> Result, String> { + let resp = client + .get(url) + .send() + .await + .map_err(|e| format!("download failed: {url}: {e}"))?; + if !resp.status().is_success() { + return Err(format!("download failed: {url}: {}", resp.status())); + } + Ok(resp + .bytes() + .await + .map_err(|e| format!("download failed: {url}: {e}"))? + .to_vec()) +} + +async fn latest_version(client: &reqwest::Client) -> Result { + #[derive(serde::Deserialize)] + struct Release { + tag_name: String, + } + let url = format!("https://api.github.com/repos/{REPO}/releases/latest"); + let body = get_bytes(client, &url).await?; + let release: Release = serde_json::from_slice(&body) + .map_err(|e| format!("cannot read the latest release: {e}"))?; + Ok(release.tag_name.trim_start_matches('v').to_string()) +} + +/// Extracts the archive's top-level `setu` into `dir`. +fn extract_binary(archive: &[u8], dir: &Path) -> Result { + let mut tar = tar::Archive::new(flate2::read::GzDecoder::new(archive)); + let entries = tar.entries().map_err(|e| format!("bad archive: {e}"))?; + for entry in entries { + let mut entry = entry.map_err(|e| format!("bad archive: {e}"))?; + let path = entry.path().map_err(|e| format!("bad archive: {e}"))?; + if path.as_os_str() == "setu" || path.as_os_str() == "./setu" { + let out = dir.join("setu"); + let mut bytes = Vec::new(); + entry + .read_to_end(&mut bytes) + .map_err(|e| format!("bad archive: {e}"))?; + write_executable(&out, &bytes)?; + return Ok(out); + } + } + Err("the archive has no top-level setu binary".into()) +} + +fn write_executable(path: &Path, bytes: &[u8]) -> Result<(), String> { + use std::os::unix::fs::OpenOptionsExt as _; + let mut f = std::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .mode(0o755) + .open(path) + .map_err(|e| format!("cannot write {}: {e}", path.display()))?; + f.write_all(bytes) + .and_then(|()| f.sync_all()) + .map_err(|e| format!("cannot write {}: {e}", path.display())) +} + +fn systemctl(args: &[&str]) -> bool { + Command::new("systemctl") + .args(args) + .status() + .is_ok_and(|s| s.success()) +} + +fn unit_enabled(unit: &str) -> bool { + systemctl(&["is-enabled", "--quiet", unit]) +} + +/// Takes a control-plane backup into the self-backup directory; the dump +/// is handed to the owner of that directory (`setu`), since root wrote it. +async fn backup_control_plane() -> Result { + let config = setu_common::Config::load().map_err(|e| e.to_string())?; + let db = setu_db::Db::connect(&config.database_url) + .await + .map_err(|e| format!("cannot connect to database: {}", e.message))?; + let rec = crate::self_backup::run_once( + &db, + Arc::new(crate::cli::TimedProcessRunner { + timeout: crate::self_backup::DUMP_TIMEOUT, + }), + &config.database_url, + &config.data_dir, + chrono::Utc::now(), + ) + .await + .map_err(|e| e.message)?; + if let Some(err) = rec.last_error { + return Err(err); + } + let dir = crate::self_backup::backup_dir(&config.data_dir); + let file = dir.join(rec.file.unwrap_or_default()); + if let Ok(meta) = std::fs::metadata(&dir) { + use std::os::unix::fs::MetadataExt as _; + let _ = std::os::unix::fs::chown(&file, Some(meta.uid()), Some(meta.gid())); + } + Ok(file.display().to_string()) +} + +/// `GET /healthz` on the configured listener answers 200. +async fn healthy(listen: &setu_common::Listen) -> bool { + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + const REQ: &[u8] = b"GET /healthz HTTP/1.0\r\nHost: localhost\r\n\r\n"; + async fn ask(mut s: S) -> bool { + let mut head = [0u8; 12]; + s.write_all(REQ).await.is_ok() + && s.read_exact(&mut head).await.is_ok() + && head.ends_with(b" 200") + } + match listen { + setu_common::Listen::Unix(path) => match tokio::net::UnixStream::connect(path).await { + Ok(s) => ask(s).await, + Err(_) => false, + }, + setu_common::Listen::Tcp(addr) => match tokio::net::TcpStream::connect(addr).await { + Ok(s) => ask(s).await, + Err(_) => false, + }, + } +} + +/// Runs `setu upgrade`. +pub async fn run(args: UpgradeArgs) -> Result<(), String> { + if !nix::unistd::Uid::effective().is_root() { + return Err("setu upgrade must run as root".into()); + } + let current = env!("CARGO_PKG_VERSION"); + let target = target()?; + let client = http()?; + let version = match args.version { + Some(v) => v.trim_start_matches('v').to_string(), + None => latest_version(&client).await?, + }; + match plan(current, &version)? { + Plan::AlreadyCurrent => { + say(&format!("Already on setu {current}")); + return Ok(()); + } + Plan::Downgrade if !args.allow_downgrade => { + return Err(format!( + "{version} is older than the installed {current}; migrations are forward-only \ + (pass --allow-downgrade only if the database never ran a newer version)" + )); + } + Plan::Downgrade | Plan::Upgrade => {} + } + + let base = format!("https://github.com/{REPO}/releases/download/v{version}"); + let archive = archive_name(&version, target); + say(&format!("Downloading {archive}")); + let bytes = get_bytes(&client, &format!("{base}/{archive}")).await?; + let sums = get_bytes(&client, &format!("{base}/SHA256SUMS")).await?; + let sums = String::from_utf8(sums).map_err(|_| "SHA256SUMS is not text".to_string())?; + let expected = + sum_for(&sums, &archive).ok_or(format!("SHA256SUMS has no entry for {archive}"))?; + if hex::encode(Sha256::digest(&bytes)) != expected { + return Err(format!("checksum mismatch for {archive}")); + } + say("Checksum verified"); + + let tmp = tempfile::tempdir().map_err(|e| format!("temp dir: {e}"))?; + let new_bin = extract_binary(&bytes, tmp.path())?; + let reported = Command::new(&new_bin) + .arg("--version") + .output() + .map_err(|e| format!("cannot run the new binary: {e}"))?; + let reported = String::from_utf8_lossy(&reported.stdout).trim().to_string(); + if reported != format!("setu {version}") { + return Err(format!( + "the new binary reports {reported:?}, not setu {version}" + )); + } + + let control_plane = unit_enabled("setu.service"); + if control_plane { + say("Backing up the control-plane database"); + let file = backup_control_plane() + .await + .map_err(|e| format!("backup failed, nothing changed: {e}"))?; + say(&format!("Backup written: {file}")); + } + + let staged = Path::new(INSTALL_PATH).with_file_name(".setu.new"); + std::fs::copy(&new_bin, &staged).map_err(|e| format!("cannot stage the binary: {e}"))?; + std::fs::rename(&staged, INSTALL_PATH) + .map_err(|e| format!("cannot install {INSTALL_PATH}: {e}"))?; + say(&format!("Installed setu {version}")); + + if control_plane { + say("Restarting setu.service"); + if !systemctl(&["restart", "setu.service"]) { + return Err("systemctl restart setu.service failed (journalctl -u setu)".into()); + } + } + if unit_enabled("setu-agent.service") { + say("Restarting setu-agent.service"); + if !systemctl(&["restart", "setu-agent.service"]) { + return Err( + "systemctl restart setu-agent.service failed (journalctl -u setu-agent)".into(), + ); + } + } + if control_plane { + let config = setu_common::Config::load().map_err(|e| e.to_string())?; + let deadline = tokio::time::Instant::now() + HEALTH_WAIT; + while !healthy(&config.listen).await { + if tokio::time::Instant::now() > deadline { + return Err(format!( + "setu {version} did not answer /healthz within {}s (journalctl -u setu)", + HEALTH_WAIT.as_secs() + )); + } + tokio::time::sleep(Duration::from_secs(1)).await; + } + say("Control plane healthy"); + } + say(&format!("Upgraded setu {current} -> {version}")); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sums_match_text_and_binary_mode() { + let a = "a".repeat(64); + let b = "B".repeat(64); + let sums = format!( + "{a} setu-0.1.5-x86_64-unknown-linux-musl.tar.gz\n{b} *install.sh\nbad line\n" + ); + assert_eq!( + sum_for(&sums, "setu-0.1.5-x86_64-unknown-linux-musl.tar.gz"), + Some(a) + ); + assert_eq!(sum_for(&sums, "install.sh"), Some("b".repeat(64))); + assert_eq!( + sum_for(&sums, "setu-0.1.5-aarch64-unknown-linux-musl.tar.gz"), + None + ); + assert_eq!(sum_for("xyz install.sh", "install.sh"), None); + } + + #[test] + fn plans_compare_semver() { + assert_eq!(plan("0.1.4", "0.1.5"), Ok(Plan::Upgrade)); + assert_eq!(plan("0.1.4", "v0.1.4"), Ok(Plan::AlreadyCurrent)); + assert_eq!(plan("0.1.10", "0.1.9"), Ok(Plan::Downgrade)); + assert_eq!(plan("0.1.4", "0.2.0-rc.1"), Ok(Plan::Upgrade)); + assert!(plan("0.1.4", "latest").is_err()); + } + + #[test] + fn archive_names_follow_the_release() { + assert_eq!( + archive_name("0.1.5", "aarch64-unknown-linux-musl"), + "setu-0.1.5-aarch64-unknown-linux-musl.tar.gz" + ); + } + + #[test] + fn extracts_only_the_top_level_binary() { + use std::os::unix::fs::PermissionsExt as _; + let mut gz = Vec::new(); + { + let enc = flate2::write::GzEncoder::new(&mut gz, flate2::Compression::fast()); + let mut tar = tar::Builder::new(enc); + for (name, body) in [("README.md", &b"hi"[..]), ("setu", &b"#!/bin/sh\n"[..])] { + let mut h = tar::Header::new_gnu(); + h.set_size(body.len() as u64); + h.set_mode(0o644); + h.set_cksum(); + tar.append_data(&mut h, name, body).expect("append"); + } + tar.into_inner().expect("tar").finish().expect("gz"); + } + let dir = tempfile::tempdir().expect("tmp"); + let bin = extract_binary(&gz, dir.path()).expect("extracts"); + assert_eq!(std::fs::read(&bin).expect("read"), b"#!/bin/sh\n"); + assert_eq!( + std::fs::metadata(&bin).expect("meta").permissions().mode() & 0o777, + 0o755 + ); + assert!(extract_binary(b"not gzip", dir.path()).is_err()); + } +} diff --git a/crates/api/tests/apps.rs b/crates/api/tests/apps.rs index 5f6999d..79da6ee 100644 --- a/crates/api/tests/apps.rs +++ b/crates/api/tests/apps.rs @@ -161,3 +161,186 @@ async fn unknown_route_uses_the_not_found_envelope() { assert_eq!(body["error"]["code"], serde_json::json!("not_found")); } } + +/// Serves manifests from memory by URL (the real fetcher refuses local +/// addresses), so followed URLs can change between checks. +#[derive(Debug, Default)] +struct MemFetcher(std::sync::Mutex>); + +impl MemFetcher { + fn put(&self, url: &str, toml: &str) { + self.0.lock().unwrap().insert(url.into(), toml.into()); + } +} + +#[async_trait::async_trait] +impl setu_api::follow::ManifestFetcher for MemFetcher { + async fn fetch(&self, url: &str) -> Result { + self.0.lock().unwrap().get(url).cloned().ok_or_else(|| { + setu_common::AppError::validation("manifest fetch failed with status 404") + }) + } +} + +fn hello(version: &str, sha: &str) -> String { + HELLO_TOML + .replace("version = \"0.1.0\"", &format!("version = \"{version}\"")) + .replace( + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + sha, + ) +} + +const URL: &str = "https://manifests.example/hello.toml"; +const SHA_A: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; +const SHA_B: &str = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + +async fn follow_server() -> (TestServer, std::sync::Arc) { + let fetcher = std::sync::Arc::new(MemFetcher::default()); + let srv = TestServer::spawn_with( + TestDb::new().await, + FakeAgent::new(), + setu_api::testing::TestOptions { + fetcher: Some(fetcher.clone()), + ..Default::default() + }, + ) + .await; + (srv, fetcher) +} + +/// 0.1.5 §1: publishing from a URL follows it; a check publishes a new +/// version, never replaces a published one, and records its outcome. +#[tokio::test] +async fn published_url_is_followed_and_checked() { + let (srv, fetcher) = follow_server().await; + let owner = srv.login_as_owner().await; + fetcher.put(URL, &hello("0.1.0", SHA_A)); + + let resp = owner + .post(srv.url("/api/v1/apps")) + .json(&serde_json::json!({ "url": URL })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201); + let body: serde_json::Value = resp.json().await.unwrap(); + assert_eq!(body["app"]["source"]["url"], URL); + + // Nothing new: no version published, the check is recorded. + let check = |o: &reqwest::Client| o.post(srv.url("/api/v1/apps/hello/check")).send(); + let out: serde_json::Value = check(&owner).await.unwrap().json().await.unwrap(); + assert_eq!(out["published"], serde_json::Value::Null); + assert!(out["source"]["checked_at"].is_string()); + assert_eq!(out["source"]["last_error"], serde_json::Value::Null); + + // The same version with other binaries is not re-published. + fetcher.put(URL, &hello("0.1.0", SHA_B)); + let out: serde_json::Value = check(&owner).await.unwrap().json().await.unwrap(); + assert_eq!(out["published"], serde_json::Value::Null); + let v = srv.db.apps().version("hello", "0.1.0").await.unwrap(); + assert!( + v.manifest.to_string().contains(SHA_A), + "published manifest kept" + ); + + // A new version is published and audited without an operator. + fetcher.put(URL, &hello("0.2.0", SHA_B)); + let out: serde_json::Value = check(&owner).await.unwrap().json().await.unwrap(); + assert_eq!(out["published"], "0.2.0"); + srv.db.apps().version("hello", "0.2.0").await.unwrap(); + let audit = srv.db.audit().list_recent(5).await.unwrap(); + let entry = audit + .iter() + .find(|e| e.target_id.as_deref() == Some("hello/0.2.0 (followed)")); + assert!(entry.is_some_and(|e| e.action == "app.publish" && e.operator_id.is_none())); + + // The app list carries the source. + let list: serde_json::Value = owner + .get(srv.url("/api/v1/apps")) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(list["apps"][0]["source"]["url"], URL); + + // A failing fetch is recorded and reported. + fetcher.0.lock().unwrap().clear(); + let resp = check(&owner).await.unwrap(); + assert!(resp.status().is_client_error()); + let src = srv.db.app_sources().get("hello").await.unwrap().unwrap(); + assert!(src.last_error.unwrap().contains("404")); + + // Stop following: the source is gone, a second clear is 404. + let del = |o: &reqwest::Client| o.delete(srv.url("/api/v1/apps/hello/source")).send(); + assert_eq!(del(&owner).await.unwrap().status(), 204); + assert_eq!(del(&owner).await.unwrap().status(), 404); + assert_eq!(check(&owner).await.unwrap().status(), 404); +} + +/// A followed URL that starts serving another app is refused. +#[tokio::test] +async fn followed_url_must_keep_its_app_id() { + let (srv, fetcher) = follow_server().await; + let owner = srv.login_as_owner().await; + fetcher.put(URL, &hello("0.1.0", SHA_A)); + let resp = owner + .post(srv.url("/api/v1/apps")) + .json(&serde_json::json!({ "url": URL })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201); + + fetcher.put( + URL, + &hello("0.2.0", SHA_A).replace("id = \"hello\"", "id = \"other\""), + ); + let resp = owner + .post(srv.url("/api/v1/apps/hello/check")) + .send() + .await + .unwrap(); + assert!(resp.status().is_client_error()); + let src = srv.db.app_sources().get("hello").await.unwrap().unwrap(); + assert!(src.last_error.unwrap().contains("not hello")); + assert!(srv.db.apps().version("other", "0.2.0").await.is_err()); +} + +/// The scheduled round checks every followed app. +#[tokio::test] +async fn check_all_publishes_new_versions() { + let (srv, fetcher) = follow_server().await; + let owner = srv.login_as_owner().await; + fetcher.put(URL, &hello("0.1.0", SHA_A)); + owner + .post(srv.url("/api/v1/apps")) + .json(&serde_json::json!({ "url": URL })) + .send() + .await + .unwrap(); + fetcher.put(URL, &hello("0.3.0", SHA_A)); + setu_api::follow::check_all(&srv.db, fetcher.as_ref(), &setu_jobs::SystemClock).await; + srv.db.apps().version("hello", "0.3.0").await.unwrap(); +} + +/// Viewers cannot check or stop following. +#[tokio::test] +async fn follow_endpoints_need_an_operator() { + let (srv, _fetcher) = follow_server().await; + let viewer = srv.login_as("viewer").await; + let resp = viewer + .post(srv.url("/api/v1/apps/hello/check")) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 403); + let resp = viewer + .delete(srv.url("/api/v1/apps/hello/source")) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 403); +} diff --git a/crates/common/src/config.rs b/crates/common/src/config.rs index f696d54..6dda09c 100644 --- a/crates/common/src/config.rs +++ b/crates/common/src/config.rs @@ -1,8 +1,9 @@ //! Control-plane configuration (`setu.toml` + `SETU_*` env). //! //! Sources, in increasing precedence: built-in defaults, `setu.toml` in the -//! current directory when present, then `SETU_*` environment variables. -//! Environment always wins over the file. +//! current directory when present (else `/etc/setu/setu.toml` when it is +//! readable, 0.1.5), then `SETU_*` environment variables. Environment +//! always wins over the file. use std::{fmt, net::SocketAddr, path::PathBuf, str::FromStr}; @@ -128,6 +129,22 @@ pub enum ConfigError { Load(String), } +/// The installed config, used when the current directory has none. +const SYSTEM_CONFIG: &str = "/etc/setu/setu.toml"; + +/// `setu.toml` in the current directory, else [`SYSTEM_CONFIG`] when this +/// user can read it (it is 0600: root or `setu`), else the current +/// directory's (absent, so defaults and env only). +fn config_file() -> &'static str { + if std::path::Path::new("setu.toml").exists() { + "setu.toml" + } else if std::fs::File::open(SYSTEM_CONFIG).is_ok() { + SYSTEM_CONFIG + } else { + "setu.toml" + } +} + impl Config { /// Load from `setu.toml` (if present) with `SETU_*` env overriding. pub fn load() -> Result { @@ -140,7 +157,7 @@ impl Config { // site paths from the short id, never from this. .set_default("data_dir", "/var/lib/setu") .map_err(|e| ConfigError::Load(e.to_string()))? - .add_source(config::File::with_name("setu").required(false)) + .add_source(config::File::with_name(config_file()).required(false)) .add_source( config::Environment::with_prefix("SETU") .prefix_separator("_") diff --git a/crates/db/migrations/0004_app_sources.sql b/crates/db/migrations/0004_app_sources.sql new file mode 100644 index 0000000..61a1383 --- /dev/null +++ b/crates/db/migrations/0004_app_sources.sql @@ -0,0 +1,8 @@ +-- Followed manifest URLs (0.1.5 §1): an app published from a URL is +-- checked again on a schedule. One row per app; dropping the app drops it. +CREATE TABLE app_sources ( + app_id TEXT PRIMARY KEY REFERENCES apps (id) ON DELETE CASCADE, + url TEXT NOT NULL, + checked_at TIMESTAMPTZ, + last_error TEXT +); diff --git a/crates/db/src/lib.rs b/crates/db/src/lib.rs index 4ac9742..34e4416 100644 --- a/crates/db/src/lib.rs +++ b/crates/db/src/lib.rs @@ -12,11 +12,11 @@ mod repo; pub static MIGRATOR: sqlx::migrate::Migrator = sqlx::migrate!("./migrations"); pub use repo::{ - AgentCommand, Allocation, App, AppVersion, Apps, Audit, AuditEntry, Backup, Backups, Commands, - CreateError, HeartbeatInput, HeartbeatOutcome, Job, Jobs, JoinError, JoinInput, LatestMetric, - Metrics, NewPlan, NewSite, Operator, Operators, Plan, Plans, Resets, Server, ServerStatus, - Servers, Session, Sessions, Settings, Site, SiteReset, Sites, WaitError, NOTIFY_CHANNEL, - SELF_BACKUP, SETUP_TOKEN_HASH, TLS_INTERNAL, + AgentCommand, Allocation, App, AppSource, AppSources, AppVersion, Apps, Audit, AuditEntry, + Backup, Backups, Commands, CreateError, HeartbeatInput, HeartbeatOutcome, Job, Jobs, JoinError, + JoinInput, LatestMetric, Metrics, NewPlan, NewSite, Operator, Operators, Plan, Plans, Resets, + Server, ServerStatus, Servers, Session, Sessions, Settings, Site, SiteReset, Sites, WaitError, + NOTIFY_CHANNEL, SELF_BACKUP, SETUP_TOKEN_HASH, TLS_INTERNAL, }; pub use setu_common::{AppError, ErrorCode}; @@ -113,6 +113,10 @@ impl Db { Backups::new(&self.pool) } + pub fn app_sources(&self) -> AppSources<'_> { + AppSources::new(&self.pool) + } + pub fn resets(&self) -> Resets<'_> { Resets::new(&self.pool) } diff --git a/crates/db/src/repo/app_sources.rs b/crates/db/src/repo/app_sources.rs new file mode 100644 index 0000000..710e9ea --- /dev/null +++ b/crates/db/src/repo/app_sources.rs @@ -0,0 +1,97 @@ +//! `app_sources` repo (0.1.5 §1): the manifest URL an app follows and the +//! outcome of its last check. + +use chrono::{DateTime, Utc}; +use sqlx::PgPool; + +use crate::{map_db_error, AppError}; + +/// The URL an app follows. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AppSource { + pub app_id: String, + pub url: String, + pub checked_at: Option>, + pub last_error: Option, +} + +pub struct AppSources<'a> { + pool: &'a PgPool, +} + +impl<'a> AppSources<'a> { + pub fn new(pool: &'a PgPool) -> Self { + Self { pool } + } + + /// The app's source, if it follows one. + pub async fn get(&self, app_id: &str) -> Result, AppError> { + sqlx::query_as!( + AppSource, + "SELECT app_id, url, checked_at, last_error FROM app_sources WHERE app_id = $1", + app_id, + ) + .fetch_optional(self.pool) + .await + .map_err(map_db_error) + } + + /// Every source. + pub async fn list(&self) -> Result, AppError> { + sqlx::query_as!( + AppSource, + "SELECT app_id, url, checked_at, last_error FROM app_sources ORDER BY app_id", + ) + .fetch_all(self.pool) + .await + .map_err(map_db_error) + } + + /// Sets (or replaces) the app's URL; a new URL starts unchecked. + pub async fn set(&self, app_id: &str, url: &str) -> Result { + sqlx::query_as!( + AppSource, + "INSERT INTO app_sources (app_id, url) VALUES ($1, $2) + ON CONFLICT (app_id) DO UPDATE + SET url = EXCLUDED.url, + checked_at = CASE WHEN app_sources.url = EXCLUDED.url + THEN app_sources.checked_at END, + last_error = CASE WHEN app_sources.url = EXCLUDED.url + THEN app_sources.last_error END + RETURNING app_id, url, checked_at, last_error", + app_id, + url, + ) + .fetch_one(self.pool) + .await + .map_err(map_db_error) + } + + /// Records a finished check: `error` is `None` on success. + pub async fn record_check( + &self, + app_id: &str, + now: DateTime, + error: Option<&str>, + ) -> Result<(), AppError> { + sqlx::query!( + "UPDATE app_sources SET checked_at = $2, last_error = $3 WHERE app_id = $1", + app_id, + now, + error, + ) + .execute(self.pool) + .await + .map_err(map_db_error)?; + Ok(()) + } + + /// Stops following; `false` when the app followed nothing. + pub async fn clear(&self, app_id: &str) -> Result { + let done = sqlx::query!("DELETE FROM app_sources WHERE app_id = $1", app_id) + .execute(self.pool) + .await + .map_err(map_db_error)?; + Ok(done.rows_affected() > 0) + } +} diff --git a/crates/db/src/repo/mod.rs b/crates/db/src/repo/mod.rs index f95a5c2..eba4514 100644 --- a/crates/db/src/repo/mod.rs +++ b/crates/db/src/repo/mod.rs @@ -1,3 +1,4 @@ +pub mod app_sources; pub mod apps; pub mod audit; pub mod backups; @@ -12,6 +13,7 @@ pub mod sessions; pub mod settings; pub mod sites; +pub use app_sources::{AppSource, AppSources}; pub use apps::{App, AppVersion, Apps}; pub use audit::{Audit, AuditEntry}; pub use backups::{Backup, Backups}; diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 1928f1e..03e2dc9 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -134,7 +134,8 @@ the first line of stdin. ## Upgrading -Re-run `install.sh --version …` with the same flags: it verifies +Run `setu upgrade` as root (0.1.5; [OPERATIONS.md](OPERATIONS.md#upgrading)), +or re-run `install.sh --version …` with the same flags: it verifies the new archive, replaces the binary and restarts `setu.service` (migrations run on boot) and, when the local agent is installed, `setu-agent.service` (only when the binary changed; R139). Upgrade other diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index dc23941..c6e52f3 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -95,10 +95,13 @@ database is sealed with `secret_key` from `setu.toml`, and without that key they cannot be opened. ```sh -cd /etc/setu sudo -u setu setu backup --out /var/lib/setu/setu-$(date +%F).dump ``` +`setu` reads `setu.toml` from the current directory, else +`/etc/setu/setu.toml` when the user running it can read it (root or +`setu`; since 0.1.5, so no `cd /etc/setu` is needed). + `setu backup --out ` runs `pg_dump -Fc` against the configured `database_url`, passing the password in `pg_dump`'s environment, never on its command line. Copy the dump **and** a copy of `setu.toml` (it @@ -138,21 +141,34 @@ stored step once the lease expires. ## Upgrading -Back up the control plane first (migrations are forward-only). Then, -on the control-plane box, re-run the installer with the same flags and -the new version: +On each server, as root: ```sh -bash install.sh --panel-host setu.example.com --version 0.1.2 +setu upgrade # the latest release +setu upgrade --version 0.1.5 # a given release ``` -The installer verifies the archive, replaces `/opt/setu/setu`, and -restarts `setu.service` (migrations run on boot) and the local -`setu-agent.service` when it is installed — only when the binary -changed, so a re-run of the same version restarts nothing (R139). On other -servers, replace `/opt/setu/setu` with the new binary from the same -release (root:root 0755) and `systemctl restart setu-agent`. Site apps -are updated separately, per site, from the UI. +`setu upgrade` (0.1.5) downloads the release archive for this machine +and `SHA256SUMS`, refuses a checksum mismatch, and checks the new binary +reports the version asked for. On the control-plane box +(`setu.service` enabled) it then takes a control-plane backup into +`/var/lib/setu/backups/setu/` (the nightly dump's directory and +rotation) and stops if that fails. It installs `/opt/setu/setu`, +restarts `setu.service` (migrations run on boot) and, when enabled, +`setu-agent.service`, and on the control plane waits up to 90 s for +`/healthz`. Upgrade the control plane first, then the other servers. +Older releases are refused (migrations are forward-only) unless +`--allow-downgrade`. + +Re-running the installer with the same flags and `--version ` still +works and does the same install and restarts, without the backup. + +Site apps are updated separately, per site, from the UI. An app +published from a URL follows it: Setu fetches the URL every 6 hours and +publishes any new version, which then appears in each site's Updates +tab. **Check now** on the Apps page checks at once. Sites are never +updated on their own, and a version already published is never +replaced by a check (publish by hand to replace it). ## Changing the TLS mode or the panel host diff --git a/docs/specs/2026-10-21-setu-0.1.5-design.md b/docs/specs/2026-10-21-setu-0.1.5-design.md new file mode 100644 index 0000000..e31d63f --- /dev/null +++ b/docs/specs/2026-10-21-setu-0.1.5-design.md @@ -0,0 +1,96 @@ +# Setu 0.1.5: followed manifest URLs and `setu upgrade` + +Two changes that remove manual steps from keeping a Setu install current. +Today a new app version reaches the Updates tab only after an operator +publishes its manifest again, and Setu itself is upgraded by re-running +`install.sh` with the flags of the first install. + +## 1. Apps follow their manifest URL + +An app published from a URL **follows** that URL: Setu fetches it again on +a schedule and publishes any new version it finds. Nothing is installed or +updated on a site; a new version only appears in the site's Updates tab. + +- **Schema.** Migration `0004_app_sources.sql` adds a table `app_sources` + with one row per app: + - `app_id`, the primary key, references `apps (id)` with + `ON DELETE CASCADE`; + - `url`, the manifest URL; + - `checked_at`, when the last check finished, and `last_error`, its + error or NULL. + + A separate table keeps every existing `apps` query unchanged. +- **Publish.** `POST /api/v1/apps` with `url` stores the URL as the app's + source (replacing an earlier one). Publishing pasted TOML leaves the + source as it is. +- **Check.** One check of one app: + 1. fetches the URL with the publish guards (public addresses only, no + redirects, 10 s, 1 MiB); + 2. parses and validates the manifest; + 3. refuses a manifest whose `app.id` is not the followed app's; + 4. when the version is **not yet published**, publishes it, audited as + `app.publish` with no operator and the detail `/ + (followed)`; + 5. records `checked_at` and `last_error`. + + A check never replaces a version that is already published, even when + the manifest at the URL changed: a manifest names binaries and their + checksums, and an unattended re-publish must not swap what a running + version installs. Re-publishing by hand still replaces it, as today. +- **Schedule.** The control plane checks every followed app every 6 hours + (on the scheduler tick, the first check one tick after start). Failures + are recorded and retried at the next round; they are not errors of the + panel. +- **API.** + - `AppOut` gains `source: { url, checked_at, last_error } | null`. + - `POST /api/v1/apps/{id}/check` (operator and above) runs a check now + and returns `{ published: | null, source }`; 404 when the + app follows no URL. + - `DELETE /api/v1/apps/{id}/source` (operator and above) stops + following; 404 when there is none. Audited as `app.source_clear`. +- **UI.** The Apps page shows, per app, "Follows · checked