diff --git a/Cargo.lock b/Cargo.lock index 3557739c..7b0055cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2400,6 +2400,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "quick-xml" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" +dependencies = [ + "memchr", +] + [[package]] name = "quinn" version = "0.11.11" @@ -3721,6 +3730,7 @@ dependencies = [ "tw-types", "tw-watch", "tw-yaml", + "url", "windows-sys 0.61.2", ] @@ -3763,6 +3773,7 @@ dependencies = [ "tw-pricing", "tw-secret", "tw-store", + "tw-sync", "tw-types", "tw-watch", "tw-yaml", @@ -3961,6 +3972,32 @@ dependencies = [ "tw-secret", ] +[[package]] +name = "tw-sync" +version = "0.76.0" +dependencies = [ + "age", + "axum", + "base64 0.22.1", + "percent-encoding", + "quick-xml", + "rand 0.10.2", + "reqwest", + "serde", + "serde_json", + "sha2 0.11.0", + "tempfile", + "thiserror 2.0.20", + "tokio", + "tracing", + "tw-bedrock", + "tw-config", + "tw-pack", + "tw-types", + "tw-watch", + "url", +] + [[package]] name = "tw-types" version = "0.76.0" diff --git a/Cargo.toml b/Cargo.toml index a2d085af..6d818724 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,8 @@ members = [ # 配置包(.twconfig):导出、导入、本机备份与同步共用的格式,以及把一份配置拆成 # 项目与合并单位的模型 "crates/tw-pack", + # 多设备同步:同步位置的存储(文件夹、WebDAV、S3)、keyring、三方合并 + "crates/tw-sync", # 控制通道的握手与加密。桌面端按 tag 依赖它,所以只依赖契约层和 tw-yaml "crates/tw-link", # ── bin ────────────────────────────────────────────── @@ -80,6 +82,7 @@ tw-bedrock = { path = "crates/tw-bedrock" } tw-link = { path = "crates/tw-link" } tw-plugin = { path = "crates/tw-plugin" } tw-pack = { path = "crates/tw-pack" } +tw-sync = { path = "crates/tw-sync" } # ── 版本与企业版 workspace 对齐,便于反向依赖时不打架 ──── tokio = { version = "1", features = ["rt-multi-thread", "macros", "net", "time", "sync", "io-util", "signal"] } @@ -146,6 +149,10 @@ aws-credential-types = "1" aws-smithy-eventstream = "0.60" aws-smithy-types = "1" percent-encoding = "2" +# 地址解析(同步位置的 `http://` 只给内网)。reqwest 本来就依赖它 +url = "2" +# WebDAV 的 PROPFIND、S3 的 ListObjectsV2 回的是 XML +quick-xml = "0.42" thiserror = "2" anyhow = "1" diff --git a/bin/twcore/Cargo.toml b/bin/twcore/Cargo.toml index 8455b64c..fc898f4b 100644 --- a/bin/twcore/Cargo.toml +++ b/bin/twcore/Cargo.toml @@ -25,6 +25,8 @@ tw-observe = { workspace = true } tw-store = { workspace = true } tw-pricing = { workspace = true } chrono = { workspace = true } +# `twcore sync` 和控制面说 JSON +serde_json = { workspace = true } tw-secret = { workspace = true } anyhow = { workspace = true } clap = { workspace = true } diff --git a/bin/twcore/src/call.rs b/bin/twcore/src/call.rs index 9c8299c7..19084ab1 100644 --- a/bin/twcore/src/call.rs +++ b/bin/twcore/src/call.rs @@ -18,6 +18,27 @@ pub fn run( data: Option, out: Option, ) -> Result<()> { + let (status, body) = request(config, method, endpoint, data)?; + match out { + Some(file) => { + std::fs::write(&file, &body).with_context(|| format!("writing {}", file.display()))?; + println!("{status}"); + } + None => { + use std::io::Write; + std::io::stdout().write_all(&body)?; + } + } + Ok(()) +} + +/// 发一个请求,拿回状态码和响应体。`twcore sync` 这类命令也走它 +pub fn request( + config: &Path, + method: &str, + endpoint: &str, + data: Option, +) -> Result<(u16, Bytes)> { let dir = config .parent() .map(PathBuf::from) @@ -36,7 +57,7 @@ pub fn run( let rt = tokio::runtime::Builder::new_current_thread() .enable_all() .build()?; - let (status, body) = rt.block_on(async move { + rt.block_on(async move { match Address::in_dir(&dir) { #[cfg(unix)] Address::Socket(p) => { @@ -59,18 +80,7 @@ pub fn run( send(s, &key, method, &endpoint, data).await } } - })?; - match out { - Some(file) => { - std::fs::write(&file, &body).with_context(|| format!("writing {}", file.display()))?; - println!("{status}"); - } - None => { - use std::io::Write; - std::io::stdout().write_all(&body)?; - } - } - Ok(()) + }) } async fn send( diff --git a/bin/twcore/src/main.rs b/bin/twcore/src/main.rs index 391ab9d1..1a2e251e 100644 --- a/bin/twcore/src/main.rs +++ b/bin/twcore/src/main.rs @@ -13,6 +13,7 @@ mod call; mod lockfile; mod pack_cli; mod proc; +mod sync_cmd; mod upgrade; use lockfile::{LockFile, LockOutcome}; @@ -138,6 +139,13 @@ enum Command { #[command(subcommand)] what: BackupCmd, }, + /// Show or change multi-device sync. Talks to the running core + // + // 给没有界面的服务器(远程 core)用:和界面走同一套控制面端点 + Sync { + #[command(subcommand)] + what: SyncCmd, + }, /// Replace this twcore with another release from GitHub; the configuration is left alone // // 只给单独装的 twcore 用(服务器上)。桌面应用包里的那份由应用更新,见 upgrade.rs @@ -154,6 +162,22 @@ enum Command { }, } +#[derive(Subcommand)] +enum SyncCmd { + /// Print the sync state, the location and the devices + Status, + /// Sync once now + Now, + /// Turn sync on; asks for the location and the passphrase + Enable, + /// Turn sync off; this device keeps its configuration + Disable { + /// Also delete every version, device record and the keyring at the sync location + #[arg(long)] + purge: bool, + }, +} + #[derive(Subcommand)] enum RemoteCmd { /// Print whether the remote control port is open and where to connect @@ -260,6 +284,12 @@ fn main() -> Result<()> { data, out, } => call::run(&path, &endpoint, &method, data, out), + Command::Sync { what } => match what { + SyncCmd::Status => sync_cmd::status(&path), + SyncCmd::Now => sync_cmd::now(&path), + SyncCmd::Enable => sync_cmd::enable(&path), + SyncCmd::Disable { purge } => sync_cmd::disable(&path, purge), + }, Command::Upgrade { check, restart, @@ -964,6 +994,8 @@ fn cmd_serve(path: &Path, port: Option, safe: bool, parent: Option) -> // 这是一种(另一种是上面的默认插件)—— 理由是服务器换发新 refresh // token 的那一刻旧的就作废了,不写回等于让配置文件从那一秒起就是坏的。 tw_control::rotation::spawn(control.clone()); + // 多设备同步:启动时一轮,之后按本机改动、定时、文件夹变化、唤醒。安全模式下不自动同步 + let sync = tw_control::sync::start(&control, safe); // 定期刷新默认价目表(`pricing.auto_update`,默认开) tw_control::pricing::spawn(control.clone()); @@ -1032,6 +1064,8 @@ fn cmd_serve(path: &Path, port: Option, safe: bool, parent: Option) -> } _ = shutdown_requested(&shutdown) => { tracing::info!("got a shutdown signal"); + // 还没上传的本机改动,退出前上传(等不了太久:没传成的下次启动时传) + sync.flush().await; Ok(()) } } diff --git a/bin/twcore/src/sync_cmd.rs b/bin/twcore/src/sync_cmd.rs new file mode 100644 index 00000000..c52ff7b8 --- /dev/null +++ b/bin/twcore/src/sync_cmd.rs @@ -0,0 +1,293 @@ +//! `twcore sync status | now | enable | disable`:给没有界面的服务器(远程 core)用。 +//! +//! 和界面走同一套控制面端点(正在跑的 core 来做),这里只是把它们问成终端里的几个问题。 + +use std::io::{BufRead, Write}; +use std::path::Path; + +use anyhow::{Context, Result, bail}; + +use crate::call; + +fn ask(prompt: &str, default: Option<&str>) -> Result { + match default { + Some(d) if !d.is_empty() => eprint!("{prompt} [{d}]: "), + _ => eprint!("{prompt}: "), + } + std::io::stderr().flush()?; + let mut line = String::new(); + std::io::stdin().lock().read_line(&mut line)?; + let line = line.trim().to_string(); + Ok(if line.is_empty() { + default.unwrap_or_default().to_string() + } else { + line + }) +} + +fn yes(prompt: &str, default: bool) -> Result { + let d = if default { "Y/n" } else { "y/N" }; + let a = ask(&format!("{prompt} ({d})"), None)?; + Ok(match a.to_ascii_lowercase().as_str() { + "" => default, + "y" | "yes" => true, + _ => false, + }) +} + +/// 发一个请求:2xx 时把响应体解析成 `T`,别的把控制面那句话说出来 +fn send( + config: &Path, + method: &str, + path: &str, + body: Option, +) -> Result { + let (status, bytes) = call::request(config, method, path, body.map(|b| b.to_string())) + .context("asking the running core (start it with twcore serve)")?; + if !(200..300).contains(&status) { + let m: Option = serde_json::from_slice(&bytes).ok(); + match m { + Some(m) => bail!("{}", m.text), + None => bail!( + "the core answered {status}: {}", + String::from_utf8_lossy(&bytes) + ), + } + } + serde_json::from_slice(&bytes).context("reading the answer") +} + +fn when(ms: u64) -> String { + let secs = (ms / 1000) as i64; + chrono::DateTime::from_timestamp(secs, 0) + .map(|t| { + t.with_timezone(&chrono::Local) + .format("%Y-%m-%d %H:%M") + .to_string() + }) + .unwrap_or_else(|| ms.to_string()) +} + +fn print_view(v: &tw_api::SyncView) { + println!("state: {}", v.state.slug()); + if let Some(t) = &v.target { + let place = match t { + tw_api::SyncTarget::Folder { path, .. } => format!("folder {path}"), + tw_api::SyncTarget::Webdav { url, .. } => format!("WebDAV {url}"), + tw_api::SyncTarget::S3 { + endpoint, + bucket, + prefix, + .. + } => format!("S3 {endpoint} bucket {bucket} prefix {prefix}"), + }; + println!("location: {place}"); + println!("encrypted: {}", if v.encrypted { "yes" } else { "no" }); + } + if let Some(d) = &v.device { + println!("this device: {} ({})", d.name, &d.id[..8.min(d.id.len())]); + } + if let Some(ms) = v.last_synced_ms { + println!("last synced: {}", when(ms)); + } + if let Some(e) = &v.error { + println!("problem: {}", e.text); + } + if v.conflicts > 0 { + println!( + "conflicts: {} (resolve them in ThinkWatch Lite, Settings → Sync and backup)", + v.conflicts + ); + } + if !v.excluded.is_empty() { + println!("not synced: {}", v.excluded.join(", ")); + } + for d in &v.devices { + let mut notes = Vec::new(); + if d.this { + notes.push("this device"); + } + if d.remote_core { + notes.push("remote core"); + } + if d.paused_update { + notes.push("paused: needs an update"); + } + let notes = if notes.is_empty() { + String::new() + } else { + format!(" ({})", notes.join(", ")) + }; + println!( + " {} core {} last synced {}{notes}", + d.name, + d.core_version, + when(d.last_seen_ms) + ); + } +} + +pub fn status(config: &Path) -> Result<()> { + let v: tw_api::SyncView = send(config, "GET", "/sync", None)?; + print_view(&v); + Ok(()) +} + +pub fn now(config: &Path) -> Result<()> { + let v: tw_api::SyncView = send(config, "POST", "/sync/now", None)?; + print_view(&v); + Ok(()) +} + +pub fn disable(config: &Path, purge: bool) -> Result<()> { + if purge + && !yes( + "Delete every version, device record and the keyring at the sync location? Other \ + devices stop syncing", + false, + )? + { + bail!("nothing was changed"); + } + let _: () = send( + config, + "POST", + "/sync/disable", + Some(serde_json::json!({ "purge": purge })), + )?; + println!("sync is off; this device keeps its configuration"); + Ok(()) +} + +fn host_name() -> String { + for var in ["COMPUTERNAME", "HOSTNAME"] { + if let Ok(v) = std::env::var(var) + && !v.trim().is_empty() + { + return v.trim().to_string(); + } + } + std::fs::read_to_string("/etc/hostname") + .map(|s| s.trim().to_string()) + .ok() + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| "server".into()) +} + +pub fn enable(config: &Path) -> Result<()> { + let kind = ask("Sync location: folder, webdav or s3", Some("folder"))?; + let proxy = || -> Result { + ask( + "Outbound proxy: direct, system or a proxy name", + Some("direct"), + ) + }; + let target = match kind.to_ascii_lowercase().as_str() { + "folder" => serde_json::json!({ + "type": "folder", + "path": ask("Folder path on this machine", None)?, + "proxy": "direct", + }), + "webdav" => serde_json::json!({ + "type": "webdav", + "url": ask("WebDAV address (https://…)", None)?, + "user": ask("User name", None)?, + "pass": ask("Password or app password", None)?, + "proxy": proxy()?, + }), + "s3" => serde_json::json!({ + "type": "s3", + "endpoint": ask("Endpoint (https://…)", None)?, + "region": ask("Region", Some("auto"))?, + "bucket": ask("Bucket", None)?, + "prefix": ask("Prefix", Some(""))?, + "access_key_id": ask("Access key ID", None)?, + "secret_access_key": ask("Secret access key", None)?, + "proxy": proxy()?, + }), + other => bail!("{other} is not one of folder, webdav, s3"), + }; + let probe: tw_api::SyncProbe = send( + config, + "POST", + "/sync/test", + Some(serde_json::json!({ "target": target })), + )?; + if !probe.writable { + bail!( + "the sync location cannot be written: {}", + probe.error.map(|e| e.text).unwrap_or_default() + ); + } + let device_name = ask("Name of this device", Some(&host_name()))?; + let mut body = serde_json::json!({ + "target": target, + "device_name": device_name, + "encrypt": true, + "join": "use_remote", + "decisions": [], + }); + match &probe.existing { + None => { + let encrypt = yes("Encrypt the sync data with a passphrase", true)?; + body["encrypt"] = encrypt.into(); + if encrypt { + eprintln!( + "The passphrase is typed in the clear here. At least {} characters; every \ + device joining needs it", + tw_config::sync::PASSPHRASE_MIN + ); + let p = ask("Sync passphrase", None)?; + let again = ask("Sync passphrase again", None)?; + if p != again { + bail!("the two passphrases differ"); + } + body["passphrase"] = p.into(); + } + } + Some(e) => { + if e.encrypted { + body["passphrase"] = ask("Sync passphrase", None)?.into(); + } + let preview: tw_api::SyncJoinPreview = send( + config, + "POST", + "/sync/enable/preview", + Some(serde_json::json!({ "target": target, "passphrase": body.get("passphrase") })), + )?; + if !preview.initial { + let conflicts: Vec<&tw_api::ImportRow> = preview + .rows + .iter() + .filter(|r| r.status == tw_api::ImportStatus::Conflict) + .collect(); + let new = preview + .rows + .iter() + .filter(|r| r.status == tw_api::ImportStatus::New) + .count(); + eprintln!( + "This device already has a configuration: {new} item(s) arrive from the sync \ + data, {} differ", + conflicts.len() + ); + let replace = yes( + "Replace this device's configuration with the sync data (no keeps this \ + device's version of every item that differs)", + false, + )?; + if !replace { + body["join"] = "merge".into(); + body["decisions"] = conflicts + .iter() + .map(|r| serde_json::json!({ "item": r.item, "action": "skip" })) + .collect(); + } + } + } + } + let _: tw_api::ConfigWritten = send(config, "POST", "/sync/enable", Some(body))?; + let v: tw_api::SyncView = send(config, "GET", "/sync", None)?; + print_view(&v); + Ok(()) +} diff --git a/crates/tw-api/msg-codes.txt b/crates/tw-api/msg-codes.txt index b86e8b00..5dd966db 100644 --- a/crates/tw-api/msg-codes.txt +++ b/crates/tw-api/msg-codes.txt @@ -542,6 +542,56 @@ security.unknown_action security.unknown_custom_rule security.unknown_guard security.unknown_rule +sync.already_on +sync.both_name_taken +sync.both_not_allowed +sync.config.address +sync.config.blank +sync.config.blank_exclude +sync.config.bucket +sync.config.device_id +sync.config.device_name +sync.config.folder_relative +sync.config.multiline +sync.config.passphrase_missing +sync.config.unknown_proxy +sync.emptied +sync.folder.denied +sync.folder.io +sync.folder.missing +sync.folder.task +sync.http_client +sync.join_undecided +sync.keyring_corrupt +sync.keyring_too_new +sync.local +sync.local.io +sync.location_taken +sync.locked_new +sync.merge_invalid +sync.not_arrived +sync.not_encrypted +sync.off +sync.passphrase_changed +sync.passphrase_short +sync.passphrase_wrong +sync.place_changed +sync.proxy_unusable +sync.s3.bad_listing +sync.s3.sign +sync.shutting_down +sync.stale +sync.store.bad_path +sync.store.denied +sync.store.exists +sync.store.server_error +sync.store.unexpected +sync.store.unreachable +sync.too_new +sync.version_gone +sync.webdav.bad_listing +sync.webdav.bad_url +sync.write_section t.auth test t.boom test t.broke test diff --git a/crates/tw-api/src/ep.rs b/crates/tw-api/src/ep.rs index 49d9c18a..138591a6 100644 --- a/crates/tw-api/src/ep.rs +++ b/crates/tw-api/src/ep.rs @@ -257,6 +257,32 @@ endpoints! { /// 最近的日志,老的在前。**网页可以调** PluginLogs: GET "/plugins/{id}/logs" [id], () => Vec; + // ─────────────────────────────────────────────── 多设备同步 + // + // 同步设置写在 config.yaml 的 `sync` 一节(本机设置,不随导出和同步带出)。改设置的几个 + // 端点(开启、改设置、加密、解锁、停止)回的是写完之后的版本号或同步状态 + /// 同步此刻怎么样:状态、同步位置、设备、冲突数、不同步的上游和随之不同步的项目 + SyncStatus: GET "/sync", () => api::SyncView; + /// 连上去看看:读写行不行,是空的还是已经有同步数据(加没加密、有哪些设备)。**不写配置** + SyncTest: POST "/sync/test", api::SyncTestRequest => api::SyncProbe; + /// 加入已有的同步时,本机配置和同步数据怎么合(规则和导入预览一样)。**不写配置** + SyncJoinPreview: POST "/sync/enable/preview", api::SyncTestRequest => api::SyncJoinPreview; + /// 开启同步:空的同步位置新建,有数据的加入 + SyncEnable: POST "/sync/enable", api::SyncEnable => api::ConfigWritten; + /// 改连接参数、设备名称、不同步的上游 + SyncSave: PUT "/sync/settings", api::SyncSettingsSave => api::ConfigWritten; + /// 立即同步一轮(打开主窗口时 Lite 也调它) + SyncNow: POST "/sync/now", () => api::SyncView; + SyncConflicts: GET "/sync/conflicts", () => Vec; + /// 处理冲突:结果作为新版本上传,别的设备随之清除 + SyncResolve: POST "/sync/conflicts", api::SyncResolve => api::SyncView; + /// 开启、关闭加密,更改同步口令 + SyncEncryption: POST "/sync/encryption", api::SyncEncryption => api::SyncView; + /// 其他设备改了口令之后,在这台设备上输入新口令 + SyncUnlock: POST "/sync/unlock", api::SyncUnlock => api::SyncView; + /// 停止同步;`purge` 时连同步位置上的数据一起删掉 + SyncDisable: POST "/sync/disable", api::SyncDisable => (); + // ─────────────────────────────────────────────── 账号登录 StartChatgptLogin: POST "/chatgpt/login", api::ChatgptLoginStart => api::ChatgptLogin; ChatgptLoginStatus: GET "/chatgpt/login/{id}" [id], () => api::ChatgptLoginStatus; diff --git a/crates/tw-api/src/lib.rs b/crates/tw-api/src/lib.rs index 90f1cc86..d545dc17 100644 --- a/crates/tw-api/src/lib.rs +++ b/crates/tw-api/src/lib.rs @@ -94,6 +94,9 @@ macro_rules! slug_enum { // 配置包那一半:导出、导入、备份与恢复(`slug_enum!` 在上面定义,所以在这里声明) mod pack; pub use pack::*; +// 多设备同步那一半(`slug_enum!` 在上面定义,所以在这里声明) +mod sync; +pub use sync::*; // ---------------------------------------------------------------- 取值固定的词 // @@ -977,6 +980,17 @@ pub const MSG_CODES: &str = include_str!("../msg-codes.txt"); /// 和 [`RestorePreview`] 的 `repairs` 列出修了哪几处。OAuth 上游换发来的令牌(刷新令牌会轮换) /// 和 ChatGPT 的登录一样留在本机,不随导出带走、导入覆盖时留着本机的。一键修复修不了时的码换成 /// `control.config_unrepairable`(句子改说从备份恢复)。照 49 写的界面调不到删掉的那两个端点。 +/// +/// **50 起有多设备同步**:`GET /sync`([`SyncView`])、`POST /sync/test`([`SyncTestRequest`] → +/// [`SyncProbe`])、`POST /sync/enable/preview`(→ [`SyncJoinPreview`],规则和导入预览一样)、 +/// `POST /sync/enable`([`SyncEnable`]:空的同步位置新建,有数据的加入,`merge` 或 `use_remote`)、 +/// `PUT /sync/settings`([`SyncSettingsSave`])、`POST /sync/now`、`GET` / `POST /sync/conflicts` +/// ([`SyncConflict`]、[`SyncResolve`]:采用本机、另一边或两者都保留)、`POST /sync/encryption` +/// ([`SyncEncryption`]:开启、关闭、更改口令)、`POST /sync/unlock`([`SyncUnlock`])、 +/// `POST /sync/disable`([`SyncDisable`])。状态变了发 [`Event::SyncStatus`];应用别的设备的 +/// 版本时配置的来源是 `sync`,改了网关密钥时发 `GatewayKeysChanged`。[`ProviderView`] 多了 +/// `sync_excluded`(不同步的上游)和 `sync_url_changed`(地址经同步变更,改之前的地址)。 +/// 同步的码是 `sync.*`。安全模式下不自动同步(`paused`) pub const CONTROL_API_VERSION: u32 = 50; /// `/live` 的速度看最近多久(见 [`LiveView::tokens_per_sec`]),毫秒 @@ -1710,6 +1724,9 @@ pub enum Event { /// 「进行中」的计数和每一行的状态都靠事件,丢一个结局,那一行就永远停在 /// 「进行中」。 EventsDropped { id: u64, count: u64, at_ms: u64 }, + /// 同步的状态、冲突数或设备列表变了(`GET /sync` 的那一份)。正常运转的同步不发系统通知, + /// 要人处理的状态(冲突、连不上、拒绝访问、口令、版本过新)由界面按通知总开关提醒一次 + SyncStatus { id: u64, view: SyncView, at_ms: u64 }, /// 这几把网关密钥的值变了或者没了:导入覆盖、恢复、同步改的。 /// /// **本机接管的客户端配置里写着密钥的值**:界面收到它,照「更换密钥后更新客户端」那一套 @@ -1982,6 +1999,7 @@ impl Event { | Event::RequestCancelled { id, .. } | Event::LocallyAnswered { id, .. } | Event::EventsDropped { id, .. } + | Event::SyncStatus { id, .. } | Event::GatewayKeysChanged { id, .. } | Event::ClientPrefsChanged { id, .. } | Event::ConfigReloaded { id, .. } @@ -2372,6 +2390,14 @@ pub struct ProviderView { /// 密钥是用哪一家的账号登录换来的(配置里的 `signed_in:`):`zai` / `bigmodel`。手填的 /// 密钥、别的上游、这一项出现之前登录的都是空。在界面里手动换掉密钥之后也是空 pub signed_in: Option, + /// 不同步的上游(同步设置里选的):不上传,也不被别的设备的同名上游覆盖。没开同步是 false + #[serde(default)] + pub sync_excluded: bool, + /// 地址经同步变更:不加密的同步最近一次把这家的地址改了(改之前的地址、哪台设备、什么 + /// 时候)。「提示一次」由界面自己记已读;在本机改过这家的地址就没有了。加密的同步不给 + /// (没有口令的人写不进来) + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sync_url_changed: Option, } /// 一行请求头,配置里写的原样。 diff --git a/crates/tw-api/src/sync.rs b/crates/tw-api/src/sync.rs new file mode 100644 index 00000000..18800f47 --- /dev/null +++ b/crates/tw-api/src/sync.rs @@ -0,0 +1,340 @@ +//! 多设备同步那一半的契约:同步位置、状态、开启与加入、冲突、加密、停止。 +//! +//! 同步怎么合并、同步位置上放什么在 core 的 `tw-sync` 里;这里只有线上的样子。冲突、随之 +//! 不同步的项目按项目([`ItemRef`])说,和导入、恢复是同一套。 + +use serde::{Deserialize, Serialize}; + +use crate::{ImportDecision, ImportRow, ItemRef, Msg}; + +fn direct() -> String { + "direct".to_string() +} + +/// 同步位置。`proxy` 是 `direct`、`system`(跟随系统)或者出站代理的名字:WebDAV、S3 的请求经 +/// 它发出,文件夹用不到。 +/// +/// **口令、密码、密钥原样来回**(界面不打码配置值):设置表单回填的就是它们。 +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +#[serde(tag = "type", rename_all = "lowercase")] +pub enum SyncTarget { + /// 本机(远程 core 时是服务器上)的一个文件夹:iCloud Drive、Dropbox、OneDrive、Syncthing、 + /// NAS 共享目录 + Folder { + path: String, + #[serde(default = "direct")] + proxy: String, + }, + /// 坚果云、Nextcloud、群晖 NAS……。`http://` 只给本机和内网 + Webdav { + url: String, + #[serde(default)] + user: String, + #[serde(default)] + pass: String, + #[serde(default = "direct")] + proxy: String, + }, + /// Amazon S3、Cloudflare R2、阿里云 OSS、腾讯云 COS、MinIO。寻址方式(桶名进域名还是进 + /// 路径)由端点决定,不用选 + S3 { + endpoint: String, + region: String, + bucket: String, + #[serde(default)] + prefix: String, + access_key_id: String, + secret_access_key: String, + #[serde(default = "direct")] + proxy: String, + }, +} + +slug_enum! { + /// 同步此刻怎么样。 + pub enum SyncState { + /// 没开 + Off = "off", + /// 已同步 + Idle = "idle", + /// 正在同步 + Syncing = "syncing", + /// 有冲突等着处理(其余照常同步) + Conflict = "conflict", + /// 连续 2 小时连不上同步位置。恢复后自动继续,本机的修改那时上传 + Unreachable = "unreachable", + /// 同步位置拒绝访问:改连接参数 + AuthFailed = "auth_failed", + /// 同步口令已在其他设备上更改(或者同步位置开了加密):输入口令 + Locked = "locked", + /// 其他设备用的是较新的版本,本机读不了:暂停,更新之后继续 + TooNew = "too_new", + /// 安全模式:不自动同步 + Paused = "paused", + } +} + +/// 这台设备 +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncDeviceRef { + pub id: String, + pub name: String, +} + +/// 参与同步的一台设备 +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncDevice { + pub id: String, + pub name: String, + pub core_version: String, + /// 最后一次同步成功的时间 + pub last_seen_ms: u64, + /// 是这台设备 + pub this: bool, + /// 「已暂停:需更新」:同步位置上的最新版本它读不了 + pub paused_update: bool, + /// 它的 core 开着远程控制端口(界面标「远程 core」) + pub remote_core: bool, +} + +/// `GET /sync`,也是 [`crate::Event::SyncStatus`] 带的那一份。 +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncView { + pub state: SyncState, + /// 没开是空的 + pub target: Option, + /// 同步位置上的内容加没加密 + pub encrypted: bool, + pub device: Option, + pub last_synced_ms: Option, + /// 从这一刻起一直没同步成。成功一次就清掉 + pub failing_since_ms: Option, + /// 要人处理的状态(拒绝访问、口令、版本过新、连不上)的原因 + pub error: Option, + pub devices: Vec, + /// 还没处理的冲突有几处 + pub conflicts: u32, + /// 不同步的上游(名字) + pub excluded: Vec, + /// 随之不同步的项目:引用了不同步的上游的路由、别名、分组 + pub excluded_cascade: Vec, +} + +/// `POST /sync/test`、`POST /sync/enable/preview`:连上去看看。 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncTestRequest { + pub target: SyncTarget, + /// 已经有加密的同步数据时,给口令才看得到设备列表 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub passphrase: Option, +} + +/// 同步位置上已经有的同步数据 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncExisting { + pub encrypted: bool, + /// 加密的:口令对了。不加密的总是 `true` + pub unlocked: bool, + /// 已经在同步的设备。加密而口令不对时是空的 + pub devices: Vec, +} + +/// `POST /sync/test` 的结果。 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncProbe { + /// 读写都行 + pub writable: bool, + /// 不行的原因 + pub error: Option, + /// 空的同步位置是 `null`(开启同步走「新建」);已经有同步数据走「加入」 + pub existing: Option, +} + +/// `POST /sync/enable/preview` 的结果:加入时本机配置和同步数据怎么合,规则和导入一样。 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncJoinPreview { + /// 本机配置还是初始配置(新安装后没改过):不出现合并这一步,直接用同步数据 + pub initial: bool, + /// 同步数据里的每个项目和本机比([`ImportRow`])。`initial` 时是空的 + pub rows: Vec, +} + +slug_enum! { + /// 加入时怎么处理本机已有的配置。 + pub enum SyncJoin { + /// 按导入的规则合并,冲突项逐项选择(`decisions`) + Merge = "merge", + /// 用同步数据替换本机配置。本机已接管的客户端正在用的网关密钥并入同步数据 + UseRemote = "use_remote", + } +} + +/// `POST /sync/enable`:开启同步。同步位置空着就新建,已经有同步数据就加入。 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncEnable { + pub target: SyncTarget, + /// 新建时加密(缺省)还是不加密。加入时按同步位置上的,这一项不看 + pub encrypt: bool, + /// 新建加密的同步时至少 12 个字符;加入加密的同步时是那个口令 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub passphrase: Option, + pub device_name: String, + pub join: SyncJoin, + /// `merge` 时逐项的处理([`ImportDecision`]),没列的新项目照常导入 + #[serde(default)] + pub decisions: Vec, +} + +/// `PUT /sync/settings`:改连接参数(密码、密钥、代理)、设备名称、不同步的上游。只改给了的。 +/// +/// **换成另一个同步位置(另一个文件夹、地址、存储桶、前缀)不在这里**:那是先停止同步、再 +/// 开启同步(`sync.place_changed`) +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncSettingsSave { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub target: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub device_name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub excluded: Option>, +} + +/// 冲突的一边 +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncConflictSide { + /// 设备名称 + pub device: String, + pub at_ms: u64, + /// 这一边删了这个对象 + pub deleted: bool, +} + +slug_enum! { + /// 冲突不是因为两边改得不一样,而是: + pub enum SyncConflictReason { + /// 另一边删了一把网关密钥,而本机已接管的客户端正在用它:没删 + KeyInUse = "key_in_use", + /// 合并的结果过不了配置校验(比如路由引用的上游被另一边删了) + Invalid = "invalid", + } +} + +/// 冲突里的一个合并单位,两边的值(规范化之后的 JSON)。一边删了对象时那一边是 `null` +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncConflictDiff { + pub unit: String, + #[cfg_attr(feature = "ts", ts(type = "unknown"))] + pub local: Option, + #[cfg_attr(feature = "ts", ts(type = "unknown"))] + pub remote: Option, +} + +/// `GET /sync/conflicts` 的一项。 +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncConflict { + pub id: String, + pub item: ItemRef, + /// 不同的合并单位:上游是 `conn`、`models`、`billing`、`other`,插件是 `script`、`settings`、 + /// `enabled`,全局设置是字段的路径,别的是 `all` + pub units: Vec, + /// 本机这一边(冲突处理之前本机一直用它) + pub local: SyncConflictSide, + /// 另一边 + pub remote: SyncConflictSide, + /// 两边不同的地方,逐个单位:点对象名时显示的差异 + pub diff: Vec, + /// 能不能「两者都保留」:两边都还有这个对象,而且它有名字 + pub both_allowed: bool, + pub reason: Option, +} + +slug_enum! { + /// 处理一处冲突:采用哪一边。 + pub enum SyncTake { + Local = "local", + Remote = "remote", + /// 两者都保留:本机的留原名称,另一边的以「名称 (设备名)」新增 + Both = "both", + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncConflictDecision { + pub id: String, + pub take: SyncTake, + /// 两者都保留时另一份的名字。不给就是「名称 (设备名)」 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub new_name: Option, +} + +/// `POST /sync/conflicts` +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncResolve { + pub decisions: Vec, +} + +slug_enum! { + pub enum SyncEncryptionAction { + /// 设口令、开启加密:重新上传最新版本,删掉未加密的版本 + Enable = "enable", + /// 关闭加密:要当前的口令 + Disable = "disable", + /// 更改口令:只重新加密 keyring + Change = "change", + } +} + +/// `POST /sync/encryption` +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncEncryption { + pub action: SyncEncryptionAction, + /// 开启时是新口令(至少 12 个字符),关闭、更改时是当前的口令 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub passphrase: Option, + /// 更改时的新口令(至少 12 个字符) + #[serde(default, skip_serializing_if = "Option::is_none")] + pub new_passphrase: Option, +} + +/// `POST /sync/unlock`:其他设备改了口令(或者开了加密)之后,在这台设备上输入 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncUnlock { + pub passphrase: String, +} + +/// `POST /sync/disable` +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncDisable { + /// 连同步位置上的全部版本、设备记录和 keyring 一起删掉。其他设备随后提示同步位置为空 + #[serde(default)] + pub purge: bool, +} + +/// 不加密的同步改了一个上游的地址([`crate::ProviderView::sync_url_changed`]):最近一次的 +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncUrlChange { + /// 改之前的地址 + pub from: String, + /// 改的那台设备的名称 + pub device: Option, + pub at_ms: u64, +} diff --git a/crates/tw-bedrock/src/lib.rs b/crates/tw-bedrock/src/lib.rs index 0b48b678..0d3e5748 100644 --- a/crates/tw-bedrock/src/lib.rs +++ b/crates/tw-bedrock/src/lib.rs @@ -2,7 +2,8 @@ //! //! - [`endpoint`]:区域、runtime 和控制面的地址、模型 id 怎么写进路径 //! - [`beta`]:客户端的 `anthropic-beta` 里哪些能带给 Bedrock 上的 Claude -//! - [`sign`]:SigV4 签名,以及「带 API Key 就不签」这条规则 +//! - [`sign`]:SigV4 签名,以及「带 API Key 就不签」这条规则;S3 兼容存储(core 的同步位置) +//! 也借它签 //! - [`eventstream`]:ConverseStream 的二进制帧 → SSE //! - [`catalog`]:一个区域能路由到哪些模型 //! - [`error`]:AWS 的错误里能拿出来什么、能给谁看 @@ -21,4 +22,4 @@ pub mod error; pub mod eventstream; pub mod sign; -pub use sign::{Credentials, SignError, carries_api_key}; +pub use sign::{Credentials, SignError, carries_api_key, sign_service}; diff --git a/crates/tw-bedrock/src/sign.rs b/crates/tw-bedrock/src/sign.rs index c8619653..14817659 100644 --- a/crates/tw-bedrock/src/sign.rs +++ b/crates/tw-bedrock/src/sign.rs @@ -140,6 +140,78 @@ pub fn sign( .collect()) } +/// 给别的 AWS 服务签名:S3 兼容存储(同步位置)也是 SigV4,只是服务名、要一起签的请求头不同。 +/// +/// - `service` 是签名用的服务名(`s3`)。S3 的规矩和别的服务不一样:路径**不再编一遍、 +/// 不规范化**(`a/./b`、`%2F` 原样签),这里按服务名照办。 +/// - `headers` 是除了 host 以外要一起签的请求头(`content-type`、`if-none-match`……),发请求时 +/// 一个不差地带上;host 从 `url` 里取。 +/// - `body` 是发出去的请求体,一个字节都不能差;没有请求体时给空的。 +/// +/// 返回的头和 [`sign`] 一样:`authorization`、`x-amz-date`、`x-amz-content-sha256`,临时凭证 +/// 还有 `x-amz-security-token`。 +#[allow(clippy::too_many_arguments)] +pub fn sign_service( + credentials: &Credentials, + region: &str, + service: &str, + method: &str, + url: &str, + headers: &[(&str, &str)], + body: &[u8], + at: SystemTime, +) -> Result, SignError> { + use aws_sigv4::http_request::{ + PayloadChecksumKind, PercentEncodingMode, SignableBody, SignableRequest, SignatureLocation, + SigningSettings, UriPathNormalizationMode, sign, + }; + use aws_sigv4::sign::v4; + + let fail = |e: &dyn fmt::Display| SignError(e.to_string()); + let identity = aws_credential_types::Credentials::new( + &credentials.access_key_id, + &credentials.secret_access_key, + credentials.session_token.clone(), + None, + "thinkwatch", + ) + .into(); + + let mut settings = SigningSettings::default(); + settings.payload_checksum_kind = PayloadChecksumKind::XAmzSha256; + settings.signature_location = SignatureLocation::Headers; + if service == "s3" { + settings.percent_encoding_mode = PercentEncodingMode::Single; + settings.uri_path_normalization_mode = UriPathNormalizationMode::Disabled; + } + + let params = v4::SigningParams::builder() + .identity(&identity) + .region(region) + .name(service) + .time(at) + .settings(settings) + .build() + .map_err(|e| fail(&e))?; + + let signable = SignableRequest::new( + method, + url, + headers.iter().copied(), + SignableBody::Bytes(body), + ) + .map_err(|e| fail(&e))?; + + let (instructions, _signature) = sign(signable, ¶ms.into()) + .map_err(|e| fail(&e))? + .into_parts(); + let (headers, _query) = instructions.into_parts(); + Ok(headers + .into_iter() + .map(|h| (h.name().to_ascii_lowercase(), h.value().to_string())) + .collect()) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/tw-bedrock/tests/common/mod.rs b/crates/tw-bedrock/tests/common/mod.rs index 2f966f25..f14de3f6 100644 --- a/crates/tw-bedrock/tests/common/mod.rs +++ b/crates/tw-bedrock/tests/common/mod.rs @@ -144,6 +144,11 @@ pub fn aws_encode(s: &str) -> String { /// 在规范请求里是 `%252F`),查询参数解码后重新编码、排序,签过的头按名字写出来, /// 请求体的哈希取自 `x-amz-content-sha256`,并核对它就是收到的请求体的哈希。 pub fn signature_of(req: &Seen, secret: &str, region: &str) -> (String, String) { + signature_for(req, secret, region, "bedrock") +} + +/// 同上,换一个服务名。`s3` 照 S3 的规矩:路径不再编一遍,收到什么就签什么 +pub fn signature_for(req: &Seen, secret: &str, region: &str, service: &str) -> (String, String) { let header = |name: &str| { req.header(name) .unwrap_or_else(|| panic!("no {name} on {} {}", req.method, req.path)) @@ -166,12 +171,15 @@ pub fn signature_of(req: &Seen, secret: &str, region: &str) -> (String, String) "the signed payload hash is not the hash of what arrived" ); - let path = req - .path - .split('/') - .map(aws_encode) - .collect::>() - .join("/"); + let path = if service == "s3" { + req.path.clone() + } else { + req.path + .split('/') + .map(aws_encode) + .collect::>() + .join("/") + }; let mut query: Vec<(String, String)> = req .params() .iter() @@ -193,7 +201,7 @@ pub fn signature_of(req: &Seen, secret: &str, region: &str) -> (String, String) ); let date = &amz_date[..8]; - let scope = format!("{date}/{region}/bedrock/aws4_request"); + let scope = format!("{date}/{region}/{service}/aws4_request"); let credential = field("Credential"); assert!( credential.ends_with(&format!("/{scope}")), @@ -205,7 +213,7 @@ pub fn signature_of(req: &Seen, secret: &str, region: &str) -> (String, String) ); let key = hmac(format!("AWS4{secret}").as_bytes(), date); let key = hmac(&key, region); - let key = hmac(&key, "bedrock"); + let key = hmac(&key, service); let key = hmac(&key, "aws4_request"); (field("Signature"), hex::encode(hmac(&key, &string_to_sign))) } diff --git a/crates/tw-bedrock/tests/sign.rs b/crates/tw-bedrock/tests/sign.rs index 35e1b552..3f617caa 100644 --- a/crates/tw-bedrock/tests/sign.rs +++ b/crates/tw-bedrock/tests/sign.rs @@ -5,7 +5,7 @@ mod common; use std::time::{Duration, SystemTime}; use axum::http::StatusCode; -use common::{AK, Fake, SK, keys, signature_of}; +use common::{AK, Fake, SK, keys, signature_for, signature_of}; use serde_json::json; use tw_bedrock::sign::sign; @@ -185,3 +185,62 @@ async fn aws_computes_the_same_signature_from_what_arrives() { assert_eq!(claimed, expected); assert!(seen[0].header("authorization").unwrap().contains(AK)); } + +/// S3 兼容存储(core 的同步位置)也用这一份签名:服务名是 `s3`,路径收到什么签什么(不再 +/// 编一遍),带上调用方要一起签的头。签过的请求真的发出去,在收到的一端按 S3 的规矩重算 +#[tokio::test] +async fn an_s3_request_signs_the_way_s3_checks_it() { + let fake = Fake::start(|_| (StatusCode::OK, vec![], json!({}))).await; + let path = "/tw-sync/me/versions/0000000001-3f9a2c1e.twconfig"; + let url = reqwest::Url::parse(&format!("{}{path}", fake.uri())).unwrap(); + let body = b"PK\x03\x04 not really a zip"; + let extra = [ + ("content-type", "application/octet-stream"), + ("if-none-match", "*"), + ]; + let signed = tw_bedrock::sign_service( + &keys(), + "auto", + "s3", + "PUT", + url.as_str(), + &extra, + body, + now(), + ) + .unwrap(); + assert_eq!( + signed_headers(&signed), + "content-type;host;if-none-match;x-amz-content-sha256;x-amz-date" + ); + let mut req = reqwest::Client::new().put(url); + for (k, v) in extra { + req = req.header(k, v); + } + for (k, v) in signed { + req = req.header(k, v); + } + assert_eq!(req.body(body.to_vec()).send().await.unwrap().status(), 200); + + let seen = fake.seen(); + let (claimed, expected) = signature_for(&seen[0], SK, "auto", "s3"); + assert_eq!(claimed, expected); + + // 列目录:查询串里的斜杠和等号签的是编过一遍的样子 + let list = reqwest::Url::parse(&format!( + "{}/tw-sync?list-type=2&prefix=me%2Fversions%2F&delimiter=%2F", + fake.uri() + )) + .unwrap(); + let signed = + tw_bedrock::sign_service(&keys(), "auto", "s3", "GET", list.as_str(), &[], b"", now()) + .unwrap(); + let mut req = reqwest::Client::new().get(list); + for (k, v) in signed { + req = req.header(k, v); + } + assert_eq!(req.send().await.unwrap().status(), 200); + let seen = fake.seen(); + let (claimed, expected) = signature_for(&seen[1], SK, "auto", "s3"); + assert_eq!(claimed, expected); +} diff --git a/crates/tw-config/Cargo.toml b/crates/tw-config/Cargo.toml index 15f8a800..af8e4831 100644 --- a/crates/tw-config/Cargo.toml +++ b/crates/tw-config/Cargo.toml @@ -29,6 +29,8 @@ serde = { workspace = true } serde_yaml_ng = { workspace = true } thiserror = { workspace = true } rand = { workspace = true } +# 同步位置的地址:`http://` 只收本机和内网,要认出主机是 IP 还是域名 +url = { workspace = true } blake3 = { workspace = true } tw-watch = { workspace = true } # 网卡枚举:`bind` 认网卡名,要问系统它现在是哪个地址。 diff --git a/crates/tw-config/src/lib.rs b/crates/tw-config/src/lib.rs index a97f99aa..ec17e088 100644 --- a/crates/tw-config/src/lib.rs +++ b/crates/tw-config/src/lib.rs @@ -29,6 +29,7 @@ pub mod repair; mod retention; mod security; pub mod store; +pub mod sync; mod validate; pub mod watch; mod wire; @@ -129,6 +130,10 @@ pub struct Config { /// 脚本插件,**从上到下就是运行的顺序**。不写就没有 #[serde(default, skip_serializing_if = "Vec::is_empty")] pub plugins: Vec, + /// 多设备同步(见 [`sync`])。**本机设置**:配置包不带它,恢复备份时保留当前值。 + /// 没开同步就没有这一节 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sync: Option, } /// 便于构造,**不代表一份可用的配置** —— `providers` 和 `clients` 都是 @@ -153,6 +158,7 @@ impl Default for Config { default_route: None, default_key: None, plugins: Vec::new(), + sync: None, } } } diff --git a/crates/tw-config/src/sync.rs b/crates/tw-config/src/sync.rs new file mode 100644 index 00000000..7a460ee4 --- /dev/null +++ b/crates/tw-config/src/sync.rs @@ -0,0 +1,682 @@ +//! `sync` 一节:这台设备怎么参与多设备同步。 +//! +//! ```yaml +//! sync: +//! device_id: 3f9a2c1e-… # 开启同步时生成 +//! device_name: Mac mini +//! encrypt: true # 不写就是 true +//! passphrase: "…" # encrypt 为 true 时必填 +//! exclude: [本机 Ollama] # 不同步的上游 +//! target: +//! type: webdav # folder | webdav | s3 +//! url: https://dav.jianguoyun.com/dav/ThinkWatch +//! user: name@example.com +//! pass: "…" +//! proxy: direct # direct 或出站代理名称 +//! ``` +//! +//! **这是本机设置**:配置包(导出、同步、分享)不带它,恢复备份时保留当前值。同步口令和 +//! 同步位置的凭据和别的密钥一样明文写在这里 —— 这份文件本来就是 0600、只有本机用户读得到。 +//! +//! 同步位置就是数据所在的那一层:`keyring`、`versions/`、`devices/` 直接放在配置的文件夹、 +//! WebDAV 地址、S3 前缀底下,不再另套一层目录。 + +use serde::{Deserialize, Serialize}; +use tw_types::{Msg, msg}; + +use crate::DIRECT; + +/// 同步口令最少几个字符。**只在设口令的那一刻查**(开启同步、改口令):加入别人建的同步时 +/// 照原样输入,配置里读到的也照原样收 +pub const PASSPHRASE_MIN: usize = 12; + +/// 设备名称最长几个字符 +pub const DEVICE_NAME_MAX: usize = 64; + +/// 设备标识至少要有几个十六进制字符:同步位置上的文件名用的是它的前 8 个 +pub const DEVICE_ID_PREFIX: usize = 8; + +#[derive(Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Sync { + /// 随机生成,开启同步时写下。同步位置上的文件名只用它的前 8 个十六进制字符 + pub device_id: String, + /// 设备列表里显示的名字。缺省为计算机名称 + pub device_name: String, + /// 同一同步位置上的所有设备采用同一设置。**总是写出来**:这是一件要能一眼看清的事, + /// 不写时按 `true` 读 + #[serde(default = "yes")] + pub encrypt: bool, + /// 同步口令。加密时必填 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub passphrase: Option, + /// 不同步的上游,按名字。引用它们的路由、别名、分组随之不同步 + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub exclude: Vec, + pub target: Target, +} + +fn yes() -> bool { + true +} + +fn direct() -> String { + DIRECT.to_string() +} + +fn is_direct(v: &str) -> bool { + v == DIRECT +} + +/// 同步位置。 +#[derive(Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "lowercase", deny_unknown_fields)] +pub enum Target { + /// 本机(或服务器上)的一个文件夹,由 iCloud Drive、Dropbox、Syncthing 这类客户端在 + /// 设备之间传送 + Folder { + path: String, + /// 文件夹用不到,为了三种写法一致才有 + #[serde(default = "direct", skip_serializing_if = "is_direct")] + proxy: String, + }, + Webdav { + url: String, + #[serde(default)] + user: String, + #[serde(default)] + pass: String, + #[serde(default = "direct", skip_serializing_if = "is_direct")] + proxy: String, + }, + S3 { + endpoint: String, + region: String, + bucket: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + prefix: String, + access_key_id: String, + secret_access_key: String, + #[serde(default = "direct", skip_serializing_if = "is_direct")] + proxy: String, + }, +} + +impl Target { + /// 出站请求走哪个代理:`direct`、`system`,或 `proxies` 里的一个名字 + pub fn proxy(&self) -> &str { + match self { + Target::Folder { proxy, .. } + | Target::Webdav { proxy, .. } + | Target::S3 { proxy, .. } => proxy, + } + } + + /// `folder` / `webdav` / `s3` + pub fn kind(&self) -> &'static str { + match self { + Target::Folder { .. } => "folder", + Target::Webdav { .. } => "webdav", + Target::S3 { .. } => "s3", + } + } + + /// 两个设置指的是不是同一个同步位置(凭据和代理不算)。换了位置要重新加入, + /// 只换了凭据不用 + pub fn same_place(&self, other: &Target) -> bool { + match (self, other) { + (Target::Folder { path: a, .. }, Target::Folder { path: b, .. }) => { + a.trim_end_matches(['/', '\\']) == b.trim_end_matches(['/', '\\']) + } + (Target::Webdav { url: a, .. }, Target::Webdav { url: b, .. }) => { + a.trim_end_matches('/') == b.trim_end_matches('/') + } + ( + Target::S3 { + endpoint: e1, + bucket: b1, + prefix: p1, + .. + }, + Target::S3 { + endpoint: e2, + bucket: b2, + prefix: p2, + .. + }, + ) => { + e1.trim_end_matches('/') == e2.trim_end_matches('/') + && b1 == b2 + && p1.trim_matches('/') == p2.trim_matches('/') + } + _ => false, + } + } +} + +/// **口令和凭据不打印。**`Config` 会整个落进 Debug 输出,而日志是会被贴进 issue 的 +impl std::fmt::Debug for Sync { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Sync") + .field("device_id", &self.device_id) + .field("device_name", &self.device_name) + .field("encrypt", &self.encrypt) + .field( + "passphrase", + &self.passphrase.as_ref().map(|_| ""), + ) + .field("exclude", &self.exclude) + .field("target", &self.target) + .finish() + } +} + +impl std::fmt::Debug for Target { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Target::Folder { path, .. } => f.debug_struct("Folder").field("path", path).finish(), + Target::Webdav { + url, user, proxy, .. + } => f + .debug_struct("Webdav") + .field("url", url) + .field("user", user) + .field("pass", &"") + .field("proxy", proxy) + .finish(), + Target::S3 { + endpoint, + region, + bucket, + prefix, + proxy, + .. + } => f + .debug_struct("S3") + .field("endpoint", endpoint) + .field("region", region) + .field("bucket", bucket) + .field("prefix", prefix) + .field("access_key_id", &"") + .field("secret_access_key", &"") + .field("proxy", proxy) + .finish(), + } + } +} + +/// `sync` 一节写得不对。**英文只写一遍**:`Display` 就是 [`SyncError::msg`] 的原句。 +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum SyncError { + #[error("{}", self.msg())] + DeviceId, + #[error("{}", self.msg())] + DeviceName, + #[error("{}", self.msg())] + PassphraseMissing, + /// `field` 是那个字段在 `sync.target` 下的名字(`path`、`url`、`bucket`……) + #[error("{}", self.msg())] + Blank { field: &'static str }, + #[error("{}", self.msg())] + FolderNotAbsolute { path: String }, + /// 地址不是 `https://`,或者是 `http://` 而主机不在本机或内网 + #[error("{}", self.msg())] + Address { field: &'static str, url: String }, + #[error("{}", self.msg())] + Bucket { bucket: String }, + #[error("{}", self.msg())] + UnknownProxy { proxy: String }, + #[error("{}", self.msg())] + BlankExclude, + #[error("{}", self.msg())] + MultiLine { field: &'static str }, +} + +impl SyncError { + pub fn msg(&self) -> Msg { + match self { + SyncError::DeviceId => msg!( + "sync.config.device_id" => + "sync.device_id has to start with at least 8 lowercase hexadecimal characters. \ + It is generated when sync is turned on" + ), + SyncError::DeviceName => msg!( + "sync.config.device_name", max = DEVICE_NAME_MAX => + "sync.device_name has to be between 1 and {max} characters on one line" + ), + SyncError::PassphraseMissing => msg!( + "sync.config.passphrase_missing" => + "sync.encrypt is on, so sync.passphrase is required" + ), + SyncError::Blank { field } => msg!( + "sync.config.blank", field = field => + "sync.target.{field} cannot be empty" + ), + SyncError::FolderNotAbsolute { path } => msg!( + "sync.config.folder_relative", path = path => + "the sync folder {path} has to be an absolute path" + ), + SyncError::Address { field, url } => msg!( + "sync.config.address", field = field, url = url => + "sync.target.{field} is {url}. Sync locations take https:// addresses; http:// \ + works only for this machine and the local network" + ), + SyncError::Bucket { bucket } => msg!( + "sync.config.bucket", bucket = bucket => + "{bucket} is not a bucket name: use lowercase letters, digits, dots and hyphens" + ), + SyncError::UnknownProxy { proxy } => msg!( + "sync.config.unknown_proxy", proxy = proxy => + "the sync location goes through proxy `{proxy}`, which is not defined under \ + `proxies`; the built-in choices are direct and system" + ), + SyncError::BlankExclude => msg!( + "sync.config.blank_exclude" => + "sync.exclude has an empty upstream name" + ), + SyncError::MultiLine { field } => msg!( + "sync.config.multiline", field = field => + "sync.{field} has to be on one line" + ), + } + } +} + +impl Sync { + /// 只看这一节自己就能判断的。代理名对不对得上 `proxies` 由 [`check_proxy`] 查 + pub fn check(&self) -> Result<(), SyncError> { + if !device_id_ok(&self.device_id) { + return Err(SyncError::DeviceId); + } + let name = self.device_name.trim(); + if name.is_empty() + || self.device_name.chars().count() > DEVICE_NAME_MAX + || self.device_name.chars().any(char::is_control) + { + return Err(SyncError::DeviceName); + } + if self.encrypt && self.passphrase.as_deref().is_none_or(str::is_empty) { + return Err(SyncError::PassphraseMissing); + } + if self + .passphrase + .as_deref() + .is_some_and(|p| p.contains(['\n', '\r'])) + { + return Err(SyncError::MultiLine { + field: "passphrase", + }); + } + if self.exclude.iter().any(|e| e.trim().is_empty()) { + return Err(SyncError::BlankExclude); + } + self.target.check() + } +} + +impl Target { + pub fn check(&self) -> Result<(), SyncError> { + let blank = |field: &'static str, v: &str| { + if v.trim().is_empty() { + Err(SyncError::Blank { field }) + } else if v.contains(['\n', '\r']) { + Err(SyncError::MultiLine { field }) + } else { + Ok(()) + } + }; + match self { + Target::Folder { path, .. } => { + blank("path", path)?; + if !is_absolute(path) { + return Err(SyncError::FolderNotAbsolute { path: path.clone() }); + } + } + Target::Webdav { + url, user, pass, .. + } => { + blank("url", url)?; + if user.contains(['\n', '\r']) { + return Err(SyncError::MultiLine { field: "user" }); + } + if pass.contains(['\n', '\r']) { + return Err(SyncError::MultiLine { field: "pass" }); + } + if !address_ok(url) { + return Err(SyncError::Address { + field: "url", + url: url.clone(), + }); + } + } + Target::S3 { + endpoint, + region, + bucket, + prefix, + access_key_id, + secret_access_key, + .. + } => { + blank("endpoint", endpoint)?; + blank("region", region)?; + blank("bucket", bucket)?; + blank("access_key_id", access_key_id)?; + blank("secret_access_key", secret_access_key)?; + if prefix.contains(['\n', '\r']) { + return Err(SyncError::MultiLine { field: "prefix" }); + } + if !address_ok(endpoint) { + return Err(SyncError::Address { + field: "endpoint", + url: endpoint.clone(), + }); + } + if !bucket_ok(bucket) { + return Err(SyncError::Bucket { + bucket: bucket.clone(), + }); + } + } + } + Ok(()) + } +} + +/// 代理名对得上吗:`direct`、`system`,或者 `proxies` 里有的那一个 +pub fn check_proxy(target: &Target, proxies: &[crate::Proxy]) -> Result<(), SyncError> { + let p = target.proxy(); + if p == DIRECT || p == crate::SYSTEM || proxies.iter().any(|x| x.name == p) { + return Ok(()); + } + Err(SyncError::UnknownProxy { + proxy: p.to_string(), + }) +} + +/// 设备标识:开头至少 8 个小写十六进制字符,其余是十六进制或连字符(UUID 的写法) +pub fn device_id_ok(id: &str) -> bool { + id.len() >= DEVICE_ID_PREFIX + && id.len() <= 64 + && id + .bytes() + .take(DEVICE_ID_PREFIX) + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + && id + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b) || b == b'-') +} + +/// 生成一个设备标识(UUID v4 的写法) +pub fn generate_device_id() -> String { + let b: [u8; 16] = rand::random(); + let h: String = b.iter().map(|x| format!("{x:02x}")).collect(); + format!( + "{}-{}-4{}-{}-{}", + &h[0..8], + &h[8..12], + &h[13..16], + &h[16..20], + &h[20..32] + ) +} + +/// 文件夹路径是不是绝对路径。**两个平台都认**:远程 core 可能是 Linux,而填路径的那一端 +/// 在 Windows 上 —— 判的是写法,不是这台机器 +fn is_absolute(p: &str) -> bool { + let b = p.as_bytes(); + p.starts_with('/') + || p.starts_with("\\\\") + || (b.len() >= 3 + && b[0].is_ascii_alphabetic() + && b[1] == b':' + && matches!(b[2], b'\\' | b'/')) +} + +/// 存储桶名:S3 的规矩(3 到 63 个字符,小写字母、数字、点和连字符,字母或数字开头结尾) +fn bucket_ok(b: &str) -> bool { + (3..=63).contains(&b.len()) + && b.bytes() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == b'.' || c == b'-') + && b.bytes().next().is_some_and(|c| c.is_ascii_alphanumeric()) + && b.bytes().last().is_some_and(|c| c.is_ascii_alphanumeric()) +} + +/// 同步位置的地址收不收:`https://`,或者主机在本机、内网的 `http://`([`plain_http_ok`])。 +/// 不带账号密码、查询串和片段 —— 凭据有自己的字段 +pub fn address_ok(raw: &str) -> bool { + let Ok(u) = url::Url::parse(raw.trim()) else { + return false; + }; + if !u.username().is_empty() + || u.password().is_some() + || u.query().is_some() + || u.fragment().is_some() + { + return false; + } + let Some(host) = u.host() else { return false }; + match u.scheme() { + "https" => true, + "http" => plain_http_ok(&host), + _ => false, + } +} + +/// 内网的主机名后缀:`http://` 收这几种。和桌面端导入链接的规则是同一份 +const INTRANET_SUFFIXES: &[&str] = &[ + ".local", + ".lan", + ".internal", + ".intranet", + ".corp", + ".home.arpa", + ".localhost", +]; + +/// `http://` 收不收这个主机:本机回环地址(`localhost`、`127.0.0.1`、`[::1]`);私有网段的 +/// IPv4(`10/8`、`172.16/12`、`192.168/16`、`100.64/10`、`169.254/16`)和 IPv6(`fc00::/7`、 +/// `fe80::/10`);没有点的单段主机名,或者以内网后缀结尾的。别的一律只收 `https://` —— +/// 明文传过去的是同步位置的密码,和可能没加密的整份配置 +pub fn plain_http_ok(host: &url::Host<&str>) -> bool { + match host { + url::Host::Domain(d) => { + let d = d.to_ascii_lowercase(); + d == "localhost" || !d.contains('.') || INTRANET_SUFFIXES.iter().any(|s| d.ends_with(s)) + } + url::Host::Ipv4(ip) => { + let [a, b, ..] = ip.octets(); + *ip == std::net::Ipv4Addr::LOCALHOST + || a == 10 + || (a == 172 && (16..=31).contains(&b)) + || (a == 192 && b == 168) + || (a == 100 && (64..=127).contains(&b)) + || (a == 169 && b == 254) + } + url::Host::Ipv6(ip) => { + let first = ip.segments()[0]; + *ip == std::net::Ipv6Addr::LOCALHOST + || first & 0xfe00 == 0xfc00 + || first & 0xffc0 == 0xfe80 + } + } +} + +/// 从一份**读不进来**的配置原文里尽量找出 `sync` 一节(和它引用的出站代理)。 +/// +/// 安全模式下在服务的是一份临时的空配置,而出错页要「从同步位置恢复」:同步位置的连接参数 +/// 只在磁盘上那份写坏了的原文里。这里只要 `sync` 和 `proxies` 两节各自读得进来就行,别处 +/// 坏成什么样都不管。找不到、这一节本身也写坏了是 `None` +pub fn from_raw(text: &str) -> Option<(Sync, Vec)> { + let doc: serde_yaml_ng::Value = serde_yaml_ng::from_str(text).ok()?; + let s: Sync = serde_yaml_ng::from_value(doc.get("sync")?.clone()).ok()?; + s.check().ok()?; + let proxies = doc + .get("proxies") + .and_then(|p| serde_yaml_ng::from_value::>(p.clone()).ok()) + .unwrap_or_default(); + Some((s, proxies)) +} + +#[cfg(test)] +mod tests { + use super::*; + + const ID: &str = "3f9a2c1e-0b1d-4e2f-9a3b-5c6d7e8f9a0b"; + + fn webdav(url: &str) -> Sync { + Sync { + device_id: ID.into(), + device_name: "Mac mini".into(), + encrypt: true, + passphrase: Some("correct horse battery".into()), + exclude: vec![], + target: Target::Webdav { + url: url.into(), + user: "u".into(), + pass: "p".into(), + proxy: DIRECT.into(), + }, + } + } + + #[test] + fn the_section_round_trips_and_keeps_defaults_out() { + let text = format!( + "device_id: {ID}\ndevice_name: Mac mini\npassphrase: secret-secret\ntarget:\n type: folder\n path: /Users/x/Sync\n" + ); + let s: Sync = serde_yaml_ng::from_str(&text).unwrap(); + assert!(s.encrypt, "不写 encrypt 就是加密"); + assert_eq!(s.target.proxy(), DIRECT); + s.check().unwrap(); + let out = serde_yaml_ng::to_string(&s).unwrap(); + assert!(out.contains("encrypt: true"), "{out}"); + assert!(!out.contains("proxy"), "{out}"); + assert!(!out.contains("exclude"), "{out}"); + let back: Sync = serde_yaml_ng::from_str(&out).unwrap(); + assert!(back == s); + } + + #[test] + fn every_target_kind_parses_and_a_typo_is_refused() { + let s3 = "type: s3\nendpoint: https://abc.r2.cloudflarestorage.com\nregion: auto\nbucket: tw-sync\nprefix: me/\naccess_key_id: AK\nsecret_access_key: SK\nproxy: corp\n"; + let t: Target = serde_yaml_ng::from_str(s3).unwrap(); + assert_eq!(t.kind(), "s3"); + assert_eq!(t.proxy(), "corp"); + t.check().unwrap(); + let typo = "type: webdav\nurl: https://dav.example.com/x\nusr: me\n"; + assert!(serde_yaml_ng::from_str::(typo).is_err()); + assert!(serde_yaml_ng::from_str::("type: ftp\npath: /x\n").is_err()); + } + + #[test] + fn a_plain_http_address_is_only_taken_for_this_machine_and_the_local_network() { + for ok in [ + "https://dav.jianguoyun.com/dav/ThinkWatch", + "http://127.0.0.1:5005/dav", + "http://192.168.1.10:9000", + "http://10.0.0.2/webdav", + "http://nas/webdav", + "http://nas.local:5005", + "http://[::1]:8080/", + "http://[fd00::1]/", + ] { + assert!(address_ok(ok), "{ok}"); + webdav(ok).check().unwrap(); + } + for bad in [ + "http://dav.example.com/dav", + "http://8.8.8.8/", + "ftp://nas/x", + "https://user:pw@dav.example.com/", + "https://dav.example.com/?a=1", + "dav.example.com", + ] { + assert!(!address_ok(bad), "{bad}"); + assert!( + matches!(webdav(bad).check(), Err(SyncError::Address { .. })), + "{bad}" + ); + } + } + + #[test] + fn encryption_needs_a_passphrase_and_the_device_id_needs_its_prefix() { + let mut s = webdav("https://dav.example.com/"); + s.passphrase = None; + assert_eq!(s.check(), Err(SyncError::PassphraseMissing)); + s.encrypt = false; + s.check().unwrap(); + s.device_id = "XYZ".into(); + assert_eq!(s.check(), Err(SyncError::DeviceId)); + assert!(device_id_ok(&generate_device_id())); + assert_ne!(generate_device_id(), generate_device_id()); + } + + #[test] + fn folder_paths_have_to_be_absolute_on_either_platform() { + for ok in [ + "/Users/x/iCloud/TW", + "C:\\Users\\x\\Dropbox\\TW", + "D:/Sync", + "\\\\nas\\share", + ] { + assert!(is_absolute(ok), "{ok}"); + } + for bad in ["Sync", "./Sync", "C:Sync", "~/Sync"] { + assert!(!is_absolute(bad), "{bad}"); + } + } + + #[test] + fn debug_never_prints_a_credential() { + let mut s = webdav("https://dav.example.com/"); + s.passphrase = Some("the-passphrase-itself".into()); + s.target = Target::Webdav { + url: "https://dav.example.com/".into(), + user: "me".into(), + pass: "the-password-itself".into(), + proxy: DIRECT.into(), + }; + let shown = format!("{s:?}"); + assert!(!shown.contains("the-passphrase-itself"), "{shown}"); + assert!(!shown.contains("the-password-itself"), "{shown}"); + let s3 = Target::S3 { + endpoint: "https://s3.example.com".into(), + region: "auto".into(), + bucket: "b-1".into(), + prefix: String::new(), + access_key_id: "AKIA-ID-VALUE".into(), + secret_access_key: "SECRET-VALUE".into(), + proxy: DIRECT.into(), + }; + let shown = format!("{s3:?}"); + assert!( + !shown.contains("AKIA-ID-VALUE") && !shown.contains("SECRET-VALUE"), + "{shown}" + ); + } + + #[test] + fn the_section_is_found_in_a_config_that_does_not_load() { + let text = format!( + "version: 1\nnot_a_field: 1\nproxies:\n - name: corp\n addr: 127.0.0.1:7890\nsync:\n device_id: {ID}\n device_name: srv\n encrypt: false\n target:\n type: folder\n path: /srv/sync\n proxy: corp\n" + ); + let (s, proxies) = from_raw(&text).expect("sync 一节读得进来"); + assert_eq!(s.device_name, "srv"); + assert_eq!(proxies.len(), 1); + assert!(from_raw("version: 1\n").is_none()); + assert!(from_raw("version: [\n").is_none()); + } + + #[test] + fn the_same_place_ignores_credentials_and_trailing_slashes() { + let a = webdav("https://dav.example.com/tw/").target; + let mut b = webdav("https://dav.example.com/tw").target; + if let Target::Webdav { pass, .. } = &mut b { + *pass = "other".into(); + } + assert!(a.same_place(&b)); + assert!(!a.same_place(&webdav("https://dav.example.com/other").target)); + } +} diff --git a/crates/tw-config/src/validate.rs b/crates/tw-config/src/validate.rs index f245ed71..5c53b263 100644 --- a/crates/tw-config/src/validate.rs +++ b/crates/tw-config/src/validate.rs @@ -156,6 +156,9 @@ pub enum ValidationError { days: u64, min: u64, }, + /// `sync` 一节写得不对([`crate::sync::SyncError`]) + #[error("{}", self.msg())] + Sync(crate::sync::SyncError), } impl ValidationError { @@ -443,6 +446,7 @@ impl ValidationError { After a restart the total for the {per} is added up again from the request \ records, so row_days has to be at least {min}" ), + Sync(e) => e.msg(), } } } @@ -677,6 +681,12 @@ pub fn validate(cfg: &Config) -> Result<(), ValidationError> { } } } + // 同步。**代理名要对得上** `proxies`:同步位置的请求经它发出,指到一个不存在的代理上, + // 表现是每一次同步都失败,而原因不在同步设置里 + if let Some(sync) = &cfg.sync { + sync.check().map_err(ValidationError::Sync)?; + crate::sync::check_proxy(&sync.target, &cfg.proxies).map_err(ValidationError::Sync)?; + } Ok(()) } diff --git a/crates/tw-config/tests/manual.rs b/crates/tw-config/tests/manual.rs index 96f55e87..e6fa2f35 100644 --- a/crates/tw-config/tests/manual.rs +++ b/crates/tw-config/tests/manual.rs @@ -378,8 +378,10 @@ fn check_section(s: &Section, all: &[Section], errs: &mut Vec) { s.path, r.name )); } - // 一个列表可以是必填的(策略组的成员):它自己是值,不是一个有默认值的对象 - let list = matches!(r.kind, Kind::StrsOrObjs(_)) && matches!(r.def, Def::Required); + // 一个列表可以是必填的(策略组的成员):它自己是值,不是一个有默认值的对象。 + // 一个对象也可以是必填的(同步位置):不写它这一节就不成立 + let list = matches!(r.kind, Kind::StrsOrObjs(_) | Kind::Obj(_)) + && matches!(r.def, Def::Required); if !list && !matches!(r.def, Def::Section | Def::Is(_) | Def::Unset) { errs.push(format!( "{}.{} is an object; its default is the section's own", diff --git a/crates/tw-config/tests/manual/schema.rs b/crates/tw-config/tests/manual/schema.rs index b67aa589..d885ec39 100644 --- a/crates/tw-config/tests/manual/schema.rs +++ b/crates/tw-config/tests/manual/schema.rs @@ -7,7 +7,7 @@ //! 说明写给手写配置文件的人:这个字段管什么、不写是什么意思、写错了会怎样。 //! 两种语言各写一遍,**不是互译的字面对照**,各自按各自的习惯说。 -use super::{Def, Kind, Lang, Row, Section, T2}; +use super::{Def, Kind, Lang, Row, Section, T2, Ty}; use tw_config::proxy::ProxyAuth; use tw_config::*; use tw_engine::rule::When; @@ -87,6 +87,49 @@ fn limit_pers() -> Vec<&'static str> { super::fields::() } +/// 同步位置三种写法各自的字段。`target` 是按 `type` 分的枚举,问 serde 要字段的办法问不出 +/// 枚举每一种的字段,所以借它的报错:写一个不存在的字段,serde 说出这一种认得的全部字段 +fn target_fields(kind: &str) -> Vec<&'static str> { + let e = serde_yaml_ng::from_str::(&format!( + "type: {kind}\n__probe__: 1\n" + )) + .expect_err("an unknown field is refused") + .to_string(); + let expected = e + .split("expected") + .nth(1) + .unwrap_or_else(|| panic!("serde did not list the fields: {e}")); + let mut out: Vec<&'static str> = vec!["type"]; + out.extend( + expected + .split('`') + .skip(1) + .step_by(2) + .map(|f| &*Box::leak(f.to_string().into_boxed_str())), + ); + out +} +fn folder_fields() -> Vec<&'static str> { + target_fields("folder") +} +fn webdav_fields() -> Vec<&'static str> { + target_fields("webdav") +} +fn s3_fields() -> Vec<&'static str> { + target_fields("s3") +} +fn target_type() -> Vec<&'static str> { + vec!["type"] +} +fn target_types() -> Vec<&'static str> { + vec!["folder", "webdav", "s3"] +} + +const PROXY_DOC: T2 = t( + "Where requests to the sync location go out: `direct`, `system` (the system proxy), or the name of a proxy under `proxies`.", + "访问同步位置的请求从哪里出去:`direct`(直连)、`system`(跟随系统代理),或者 `proxies` 里一个代理的名字。", +); + const RULE_ID: T2 = t("built-in rule id", "内置规则 id"); const MODE_DOC: T2 = t( "`off` does nothing (Off in the app); `observe` detects and records only, and changes nothing \ @@ -263,6 +306,15 @@ pub fn sections() -> Vec
{ "脚本插件,按运行的顺序。由应用安装,每个插件的代码和设置是本文件旁边的一个文件。", ), ), + row( + "sync", + Kind::Obj("sync"), + Def::Unset, + t( + "Multi-device sync. A local setting: configuration packs and other devices never carry it, and restoring a backup keeps the current one. Unset: sync is off.", + "多设备同步。属于本机设置:配置包和其他设备不会带走它,恢复备份时保留当前值。不写:不同步。", + ), + ), ], }, // ── listen ──────────────────────────────────────────── @@ -1829,6 +1881,210 @@ pub fn sections() -> Vec
{ ), ], }, + // ── sync ────────────────────────────────────────────── + Section { + path: "sync", + ty: checked!( + tw_config::sync::Sync, + "{device_id: 3f9a2c1e-0b1d-4e2f-9a3b-5c6d7e8f9a0b, device_name: Mac mini, target: {type: folder, path: /sync}}" + ), + rows: vec![ + row( + "device_id", + Kind::Str, + Def::Required, + t( + "Generated when sync is turned on. File names at the sync location show its first 8 characters.", + "开启同步时生成。同步位置上的文件名里出现的是它的前 8 个字符。", + ), + ), + row( + "device_name", + Kind::Str, + Def::Required, + t( + "Shown in the device list of every device that syncs here.", + "在每台参与同步的设备的设备列表里显示的名字。", + ), + ), + row( + "encrypt", + Kind::Bool, + Def::Is("true"), + t( + "Encrypt what goes to the sync location with the sync passphrase. Every device at one location uses the same setting; change it in the app, which re-uploads the latest version.", + "以同步口令加密存到同步位置的内容。同一同步位置上的所有设备采用同一设置;在应用里更改,应用会重新上传最新版本。", + ), + ), + row( + "passphrase", + Kind::Str, + Def::Unset, + t( + "The sync passphrase; required when `encrypt` is on. Kept here in the clear, like the other keys in this file.", + "同步口令,`encrypt` 开着时必填。和本文件里的其他密钥一样以明文保存。", + ), + ), + row( + "exclude", + Kind::Strs, + Def::Is("[]"), + t( + "Upstreams, by name, that are not synced: they are not uploaded and another device's upstream of the same name does not replace them. Routes, aliases and groups that refer to them are not synced either.", + "不同步的上游,按名字:不上传,也不被其他设备的同名上游覆盖。引用它们的路由、别名、分组随之不同步。", + ), + ), + row( + "target", + Kind::Obj("sync.target"), + Def::Required, + t( + "The sync location. The keyring, `versions/` and `devices/` live directly in it.", + "同步位置。`keyring`、`versions/`、`devices/` 直接放在它里面。", + ), + ), + ], + }, + Section { + path: "sync.target", + ty: Ty::Code { + fields: target_type, + same: super::same_as_omitted::, + parses_without: super::parses_without::, + minimal: "{type: folder}", + }, + rows: vec![row( + "type", + Kind::Enum(target_types), + Def::Required, + t( + "Which kind of location; the other fields depend on it, see the three tables below.", + "同步位置的种类;其余字段随之不同,见下面三张表。", + ), + )], + }, + Section { + path: "sync.target (folder)", + ty: Ty::Code { + fields: folder_fields, + same: super::same_as_omitted::, + parses_without: super::parses_without::, + minimal: "{type: folder, path: /sync}", + }, + rows: vec![ + row("type", Kind::Str, Def::Required, t("`folder`", "`folder`")), + row( + "path", + Kind::Str, + Def::Required, + t( + "An absolute path on the machine core runs on: a folder that iCloud Drive, Dropbox, OneDrive, Syncthing or a NAS share carries between devices.", + "core 所在机器上的绝对路径:由 iCloud Drive、Dropbox、OneDrive、Syncthing 或 NAS 共享在设备之间传送的文件夹。", + ), + ), + row( + "proxy", + Kind::Str, + Def::Is("direct"), + t("Not used for a folder.", "文件夹用不到。"), + ), + ], + }, + Section { + path: "sync.target (webdav)", + ty: Ty::Code { + fields: webdav_fields, + same: super::same_as_omitted::, + parses_without: super::parses_without::, + minimal: "{type: webdav, url: 'https://dav.example.com/ThinkWatch'}", + }, + rows: vec![ + row("type", Kind::Str, Def::Required, t("`webdav`", "`webdav`")), + row( + "url", + Kind::Str, + Def::Required, + t( + "The WebDAV address of the folder. `https://`, with the certificate checked; `http://` only for this machine and the local network.", + "文件夹的 WebDAV 地址。`https://`,校验证书;`http://` 只用于本机和局域网。", + ), + ), + row( + "user", + Kind::Str, + Def::Is("''"), + t("User name.", "用户名。"), + ), + row( + "pass", + Kind::Str, + Def::Is("''"), + t( + "Password, or the app password the service issues for WebDAV.", + "密码,或者服务为 WebDAV 签发的应用密码。", + ), + ), + row("proxy", Kind::Str, Def::Is("direct"), PROXY_DOC), + ], + }, + Section { + path: "sync.target (s3)", + ty: Ty::Code { + fields: s3_fields, + same: super::same_as_omitted::, + parses_without: super::parses_without::, + minimal: "{type: s3, endpoint: 'https://s3.us-east-1.amazonaws.com', region: us-east-1, bucket: tw-sync, access_key_id: AK, secret_access_key: SK}", + }, + rows: vec![ + row("type", Kind::Str, Def::Required, t("`s3`", "`s3`")), + row( + "endpoint", + Kind::Str, + Def::Required, + t( + "The S3 endpoint. Hosts ending in `amazonaws.com`, `aliyuncs.com` or `myqcloud.com` put the bucket in the host name; every other endpoint (Cloudflare R2, MinIO, an address) puts it in the path. `http://` only for this machine and the local network.", + "S3 端点。主机以 `amazonaws.com`、`aliyuncs.com`、`myqcloud.com` 结尾的,存储桶名放进域名;其余端点(Cloudflare R2、MinIO、IP 地址)放进路径。`http://` 只用于本机和局域网。", + ), + ), + row( + "region", + Kind::Str, + Def::Required, + t( + "The signing region; `auto` for Cloudflare R2.", + "签名用的区域;Cloudflare R2 填 `auto`。", + ), + ), + row( + "bucket", + Kind::Str, + Def::Required, + t("The bucket. Keep it private.", "存储桶,建议设为私有。"), + ), + row( + "prefix", + Kind::Str, + Def::Is("''"), + t( + "A folder inside the bucket.", + "存储桶里的一个前缀(文件夹)。", + ), + ), + row( + "access_key_id", + Kind::Str, + Def::Required, + t("Access key ID.", "Access Key ID。"), + ), + row( + "secret_access_key", + Kind::Str, + Def::Required, + t("Secret access key.", "Secret Access Key。"), + ), + row("proxy", Kind::Str, Def::Is("direct"), PROXY_DOC), + ], + }, ] } diff --git a/crates/tw-control/Cargo.toml b/crates/tw-control/Cargo.toml index 725d07ae..69eb079c 100644 --- a/crates/tw-control/Cargo.toml +++ b/crates/tw-control/Cargo.toml @@ -23,6 +23,8 @@ libc = { workspace = true } tw-pack = { workspace = true } base64 = { workspace = true } tw-config = { workspace = true } +# 多设备同步:同步位置、合并、加密 +tw-sync = { workspace = true } tw-guard = { workspace = true } bytes = { workspace = true } tw-engine = { workspace = true } diff --git a/crates/tw-control/src/backups.rs b/crates/tw-control/src/backups.rs index 3bf7b060..c1fccdec 100644 --- a/crates/tw-control/src/backups.rs +++ b/crates/tw-control/src/backups.rs @@ -432,7 +432,7 @@ async fn export( // ─────────────────────────────────────────────────────────── 导入 -fn import_ctx() -> tw_pack::import::Context<'static> { +pub(crate) fn import_ctx() -> tw_pack::import::Context<'static> { tw_pack::import::Context { has_aws_profile: &|p| tw_config::aws_profile::credentials(p).is_ok(), new_key: &tw_config::generate_key, @@ -452,7 +452,7 @@ fn import_fail(e: tw_pack::import::ImportError, pack: &Pack) -> Fail { } } -fn row_view(r: tw_pack::import::Row) -> tw_api::ImportRow { +pub(crate) fn row_view(r: tw_pack::import::Row) -> tw_api::ImportRow { use tw_pack::import::{Flag, Status}; tw_api::ImportRow { item: item_ref(&r.item), @@ -568,7 +568,8 @@ async fn backups(State(s): State) -> Result>, /// 同步位置上的备份版本。没开同步是 None remote: std::sync::RwLock>>, + /// 这份配置的同步服务,第一次用到时建(见 `crate::sync::service`) + sync: std::sync::OnceLock>, } /// 一次配置改动没成的原因。 @@ -127,6 +129,7 @@ impl ConfigManager { applied: tokio::sync::Notify::new(), pinned: std::sync::Mutex::new(pinned), remote: std::sync::RwLock::new(None), + sync: std::sync::OnceLock::new(), } } @@ -149,6 +152,16 @@ impl ConfigManager { self } + /// 同步服务放在这里(`crate::sync::service`) + pub(crate) fn sync_slot(&self) -> &std::sync::OnceLock> { + &self.sync + } + + /// 事件总线 + pub fn bus(&self) -> &tw_observe::EventBus { + &self.bus + } + /// 数据面。管理面里要碰插件文件、编插件的那几处从这里拿 pub fn gateway(&self) -> &tw_gateway::AppState { &self.gateway diff --git a/crates/tw-control/src/lib.rs b/crates/tw-control/src/lib.rs index a5c3eaec..3e2b89a0 100644 --- a/crates/tw-control/src/lib.rs +++ b/crates/tw-control/src/lib.rs @@ -43,6 +43,7 @@ pub mod rotation; pub mod routes; pub mod security; pub mod shutdown; +pub mod sync; pub mod zai; pub use config::{ApplyError, ConfigManager, resolve_path, spawn_watcher}; pub use shutdown::Shutdown; @@ -153,6 +154,7 @@ pub fn router(state: ControlState) -> Router { .merge(pricing::router()) .merge(chatgpt::router()) .merge(zai::router()) + .merge(sync::router()) .fallback(contract::no_such_endpoint) .layer(axum::middleware::from_fn(contract::errors_are_messages)) .with_state(state) @@ -444,7 +446,13 @@ fn provider_view( p: &tw_config::Provider, ) -> tw_api::ProviderView { let listing = s.gateway.models.listing(p); + let synced = sync::service(s); tw_api::ProviderView { + sync_excluded: cfg + .sync + .as_ref() + .is_some_and(|y| y.exclude.iter().any(|n| n == &p.name)), + sync_url_changed: synced.url_changed(&p.name), name: p.name.clone(), base_url: p.base_url.clone(), key: p.key.as_ref().map(|k| k.raw().to_string()), diff --git a/crates/tw-control/src/sync.rs b/crates/tw-control/src/sync.rs new file mode 100644 index 00000000..41e27f91 --- /dev/null +++ b/crates/tw-control/src/sync.rs @@ -0,0 +1,1682 @@ +//! 多设备同步的控制面一半:什么时候同步、同步设置怎么写、端点、状态事件。 +//! +//! 合并规则、同步位置上的文件、加密都在 `tw-sync`;这里把它接到配置的写入流程上: +//! +//! | 时机 | 行为 | +//! |---|---| +//! | 本机配置变更 | 合并最近 10 秒内的变更后上传一个版本。退出前立即上传还没上传的 | +//! | 定时 | WebDAV、S3 每 30 分钟看一次;文件夹位置盯着文件变化,新版本一到就合并 | +//! | 事件 | 启动、从睡眠中唤醒、网络恢复(失败之后按 1、2、5、10 分钟重试)、`POST /sync/now` | +//! +//! 应用别的设备的版本走现有的写入流程(校验、热更新、记一个本机版本),来源记为 `sync`; +//! 这种写入不会再触发上传。**安全模式下不自动同步**(`paused`)。 +//! +//! 连不上同步位置的头 2 小时里状态照旧(已同步 / 冲突),过了 2 小时才是 `unreachable`: +//! 笔记本合上盖子、换个网络都会失败几次,那不该是一条要人处理的提示。 + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::sync::{Arc, Weak}; +use std::time::Duration; + +use axum::Json; +use axum::extract::State; +use axum::http::StatusCode; +use tokio::sync::Notify; +use tw_api::ep; +use tw_config::history::Origin; +use tw_pack::model::{self, ItemId, ItemKind, Snapshot}; +use tw_pack::{Pack, PackOrigin}; +use tw_sync::engine::{self, Current, Decision, Engine, Host, Report, RoundError, Take}; +use tw_sync::{AnyStore, StoreError}; +use tw_types::{Msg, msg}; + +use crate::backups::{BoxFuture, RemoteBackups, RemoteVersionInfo}; +use crate::contract::RouterExt; +use crate::{ApplyError, ConfigManager, ControlState, Fail, apply_fail, fail}; + +/// 本机配置改了之后等多久再上传:这期间的改动合成一个版本 +pub const SETTLE: Duration = Duration::from_secs(10); +/// WebDAV、S3 多久看一次 +pub const POLL: Duration = Duration::from_secs(30 * 60); +/// 连续失败多久才算「同步位置不可用」 +pub const UNREACHABLE_AFTER_MS: u64 = 2 * 3600 * 1000; +/// 失败之后隔多久重试:网络恢复后几分钟内就接上 +const RETRY: &[u64] = &[60, 120, 300, 600]; +/// 退出前上传最多等多久 +const FLUSH_WITHIN: Duration = Duration::from_secs(8); + +pub fn router() -> axum::Router { + axum::Router::new() + .at(ep::SyncStatus, status) + .at(ep::SyncTest, test) + .at(ep::SyncJoinPreview, join_preview) + .at(ep::SyncEnable, enable) + .at(ep::SyncSave, save) + .at(ep::SyncNow, now) + .at(ep::SyncConflicts, conflicts) + .at(ep::SyncResolve, resolve) + .at(ep::SyncEncryption, encryption) + .at(ep::SyncUnlock, unlock) + .at(ep::SyncDisable, disable) +} + +fn now_ms() -> u64 { + crate::config::now_ms() +} + +// ─────────────────────────────────────────────────────────── 本机这一边 + +/// 本机的配置:读成配置包、照配置包写回去 +#[derive(Clone)] +struct CoreHost { + mgr: Weak, +} + +impl CoreHost { + fn mgr(&self) -> Result, Msg> { + self.mgr + .upgrade() + .ok_or_else(|| msg!("sync.shutting_down" => "the core is shutting down")) + } +} + +/// 本机此刻的完整配置包:配置原文、插件文件、Lite 的通用偏好(和备份、导出读的是同一份, +/// `tw_pack::disk::current`) +pub fn current_pack(mgr: &ConfigManager) -> Result { + let loaded = mgr.current().map_err(|e| e.msg())?; + let pack = tw_pack::disk::current(mgr.path(), &loaded.text, PackOrigin::Ui); + let changed_ms = loaded + .fingerprint + .mtime + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|d| d.as_millis() as u64) + .unwrap_or_else(now_ms); + Ok(Current { + version: loaded.version(), + pack, + changed_ms, + }) +} + +/// 照一个配置包写本机,和导入、恢复同一个顺序:攥着插件的编辑锁,先写插件文件和通用偏好,再 +/// 经写入流程写配置(来源记为同步,留一个本机版本,记下是哪台设备传来的);配置没写成就把 +/// 文件还原 +async fn apply_pack( + mgr: &ConfigManager, + after: Pack, + base_version: &str, + from: &str, +) -> Result<(), engine::ApplyError> { + let path = mgr.path().to_path_buf(); + let _edit = mgr.gateway().plugins.edits.lock().await; + let cur = mgr + .current() + .map_err(|e| engine::ApplyError::Rejected(e.msg()))?; + if cur.version() != base_version { + return Err(engine::ApplyError::Stale); + } + let before = tw_pack::disk::current(&path, &cur.text, PackOrigin::Sync); + let io = |p: &Path, e: std::io::Error| { + engine::ApplyError::Rejected(msg!( + "sync.local.io", path = p.display(), detail = e => + "{path} could not be used: {detail}" + )) + }; + let undo = + tw_pack::disk::write_plugins(&path, &before.plugins, &after.plugins).map_err(|e| { + io( + &tw_config::plugins::dir_in(&tw_pack::disk::dir_of(&path)), + e, + ) + })?; + // 远程连接下偏好在 Lite 那边;同步这一路跑在 core 自己的任务里,不是远程连接 + let prefs_changed = after.client_prefs.is_some() && after.client_prefs != before.client_prefs; + if prefs_changed && let Err(e) = tw_pack::disk::write_prefs(&path, after.client_prefs.as_ref()) + { + undo.restore(); + return Err(io(&tw_pack::disk::prefs_path(&path), e)); + } + let started = now_ms(); + if let Err(e) = mgr + .write(&after.config_yaml, Some(base_version), Origin::Sync) + .await + { + undo.restore(); + if prefs_changed { + let _ = tw_pack::disk::write_prefs(&path, before.client_prefs.as_ref()); + } + return Err(match e { + ApplyError::Stale { .. } + | ApplyError::Store(tw_config::StoreError::Conflict { .. }) => { + engine::ApplyError::Stale + } + other => engine::ApplyError::Rejected(other.msg()), + }); + } + // 本机这一版是从哪台设备来的:备份列表里写「同步自 MacBook Air」 + if !from.is_empty() + && let Ok(all) = tw_pack::backup::list(&path) + && let Some(b) = all + .iter() + .rev() + .find(|b| b.origin == Origin::Sync && b.at_ms + 1000 >= started) + && let Err(e) = tw_pack::backup::note_device(b, from) + { + tracing::debug!("the backup version could not be labelled with its device: {e}"); + } + if prefs_changed { + mgr.bus().emit(tw_api::Event::ClientPrefsChanged { + id: mgr.bus().next_id(), + at_ms: now_ms(), + }); + } + Ok(()) +} + +impl Host for CoreHost { + fn current(&self) -> Result { + current_pack(&*self.mgr()?) + } + async fn apply( + &self, + pack: Pack, + base_version: &str, + from: &str, + ) -> Result<(), engine::ApplyError> { + let mgr = self.mgr().map_err(engine::ApplyError::Rejected)?; + apply_pack(&mgr, pack, base_version, from).await + } + fn pinned_keys(&self) -> BTreeSet { + self.mgr + .upgrade() + .map(|m| m.pinned_keys()) + .unwrap_or_default() + } + /// 本机备份里内容一样的那一版(找合并基准)。新的在前 + fn backup(&self, content_id: &str) -> Option { + let mgr = self.mgr.upgrade()?; + let all = tw_pack::backup::list(mgr.path()).ok()?; + all.iter() + .rev() + .filter_map(|b| tw_pack::backup::open(b).ok()) + .find(|p| p.content_id() == content_id) + } +} + +// ─────────────────────────────────────────────────────────── 同步位置 + +/// 发 WebDAV、S3 请求用的客户端:经同步设置里选的出站代理,**不跟重定向**(跟到别的主机, +/// 同步位置的密码就跟着去了),证书照常校验 +fn http(cfg: &tw_config::Config, proxy: &str) -> Result { + let mut b = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(10)) + .redirect(reqwest::redirect::Policy::none()) + .user_agent(tw_gateway::user_agent()); + match proxy { + tw_config::DIRECT => b = b.no_proxy(), + tw_config::SYSTEM => {} + name => { + let p = cfg.proxies.iter().find(|x| x.name == name).ok_or_else(|| { + tw_config::sync::SyncError::UnknownProxy { + proxy: name.to_string(), + } + .msg() + })?; + let url = p.url().map_err(|e| e.msg())?; + let px = reqwest::Proxy::all(&url).map_err(|e| { + msg!( + "sync.proxy_unusable", proxy = name, detail = e => + "proxy `{proxy}` cannot be used for the sync location: {detail}" + ) + })?; + b = b.proxy(px); + } + } + b.build().map_err(|e| { + msg!( + "sync.http_client", detail = e => + "the connection to the sync location could not be set up: {detail}" + ) + }) +} + +fn store_for(cfg: &tw_config::Config, t: &tw_config::sync::Target) -> Result { + t.check().map_err(|e| e.msg())?; + tw_config::sync::check_proxy(t, &cfg.proxies).map_err(|e| e.msg())?; + AnyStore::new(t, http(cfg, t.proxy())?).map_err(|e| e.msg()) +} + +pub fn target_to_config(t: &tw_api::SyncTarget) -> tw_config::sync::Target { + use tw_api::SyncTarget as A; + use tw_config::sync::Target as C; + let proxy = |p: &str| { + if p.trim().is_empty() { + tw_config::DIRECT.to_string() + } else { + p.trim().to_string() + } + }; + match t { + A::Folder { path, proxy: p } => C::Folder { + path: path.trim().to_string(), + proxy: proxy(p), + }, + A::Webdav { + url, + user, + pass, + proxy: p, + } => C::Webdav { + url: url.trim().to_string(), + user: user.clone(), + pass: pass.clone(), + proxy: proxy(p), + }, + A::S3 { + endpoint, + region, + bucket, + prefix, + access_key_id, + secret_access_key, + proxy: p, + } => C::S3 { + endpoint: endpoint.trim().to_string(), + region: region.trim().to_string(), + bucket: bucket.trim().to_string(), + prefix: prefix.trim().trim_matches('/').to_string(), + access_key_id: access_key_id.trim().to_string(), + secret_access_key: secret_access_key.trim().to_string(), + proxy: proxy(p), + }, + } +} + +pub fn target_to_api(t: &tw_config::sync::Target) -> tw_api::SyncTarget { + use tw_api::SyncTarget as A; + use tw_config::sync::Target as C; + match t.clone() { + C::Folder { path, proxy } => A::Folder { path, proxy }, + C::Webdav { + url, + user, + pass, + proxy, + } => A::Webdav { + url, + user, + pass, + proxy, + }, + C::S3 { + endpoint, + region, + bucket, + prefix, + access_key_id, + secret_access_key, + proxy, + } => A::S3 { + endpoint, + region, + bucket, + prefix, + access_key_id, + secret_access_key, + proxy, + }, + } +} + +/// 写 `sync` 一节(`None` 是删掉)。这是本机设置:同步一轮里比较的快照不含它,写它不会 +/// 触发上传 +async fn write_section( + mgr: &ConfigManager, + sync: Option<&tw_config::sync::Sync>, + origin: Origin, +) -> Result { + let value = match sync { + Some(s) => Some(serde_yaml_ng::to_value(s).map_err(|e| { + ApplyError::Invalid(msg!( + "sync.write_section", detail = e => + "the sync settings could not be written: {detail}" + )) + })?), + None => None, + }; + mgr.transform(None, origin, |text, _| { + Ok(tw_config::edit::set( + text, + &[tw_yaml::Step::key("sync")], + value.as_ref(), + )?) + }) + .await +} + +/// 引擎的设置:`sync` 一节,加上这台设备是不是远程 core(开着远程控制端口) +fn settings_of(cfg: &tw_config::Config, sync: &tw_config::sync::Sync) -> engine::Settings { + let mut s = engine::Settings::of(sync); + s.remote_core = cfg + .listen + .control + .remote + .as_ref() + .is_some_and(|r| r.enabled); + s +} + +// ─────────────────────────────────────────────────────────── 服务 + +type Eng = Engine; + +struct Running { + engine: Eng, + /// 建它时的 `sync` 一节和出站代理:变了就重建 + config: tw_config::sync::Sync, + proxies: String, + /// 文件夹位置的监听,留着 + _watch: Option, +} + +/// 同步服务。每份配置一个([`service`]) +pub struct Sync { + /// 自己(装进备份列表的那扇门拿着它) + me: Weak, + mgr: Weak, + run: tokio::sync::Mutex>, + view: std::sync::Mutex>, + /// 经同步改了地址的上游(不加密时)。上游视图读它 + url_changed: std::sync::Mutex>, + kick: Arc, + /// 本机改了配置、还没上传:这个时刻之后上传 + dirty: std::sync::Mutex>, + paused: AtomicBool, + started: AtomicBool, + /// 连续失败了几次(重试的间隔) + failures: AtomicU32, +} + +/// 这份配置的同步服务。第一次用的时候建(端点、启动、测试都走这里) +pub fn service(s: &ControlState) -> Arc { + of(&s.cfg) +} + +pub fn of(mgr: &Arc) -> Arc { + mgr.sync_slot() + .get_or_init(|| { + let dir = crate::plugins::dir_of(mgr.path()); + let url_changed = tw_sync::local::Local::new(&dir) + .state() + .url_changed + .into_iter() + .map(|(k, v)| (k, url_change(&v))) + .collect(); + Arc::new_cyclic(|me| Sync { + me: me.clone(), + mgr: Arc::downgrade(mgr), + run: tokio::sync::Mutex::new(None), + view: std::sync::Mutex::new(None), + url_changed: std::sync::Mutex::new(url_changed), + kick: Arc::new(Notify::new()), + dirty: std::sync::Mutex::new(None), + paused: AtomicBool::new(false), + started: AtomicBool::new(false), + failures: AtomicU32::new(0), + }) + }) + .clone() +} + +fn url_change(v: &tw_sync::local::UrlChange) -> tw_api::SyncUrlChange { + tw_api::SyncUrlChange { + from: v.from.clone(), + device: v.device.clone(), + at_ms: v.at_ms, + } +} + +/// 一轮或者一次设置操作没成,作为控制面的失败 +fn round_fail(e: &RoundError) -> Fail { + let code = match e { + RoundError::Store(StoreError::AuthFailed(_)) => StatusCode::FORBIDDEN, + RoundError::Store(_) => StatusCode::BAD_GATEWAY, + RoundError::Locked(_) => StatusCode::BAD_REQUEST, + RoundError::TooNew { .. } | RoundError::Stale | RoundError::Emptied => StatusCode::CONFLICT, + RoundError::NotArrived => StatusCode::SERVICE_UNAVAILABLE, + RoundError::Local(_) => StatusCode::BAD_REQUEST, + }; + fail(code, e.msg()) +} + +fn sync_off() -> Fail { + fail( + StatusCode::CONFLICT, + msg!("sync.off" => "sync is not turned on"), + ) +} + +/// 口令够长吗(设口令的那一刻才查) +fn check_passphrase(p: Option<&str>) -> Result<&str, Fail> { + match p { + Some(p) if p.chars().count() >= tw_config::sync::PASSPHRASE_MIN => Ok(p), + _ => Err(fail( + StatusCode::BAD_REQUEST, + msg!( + "sync.passphrase_short", min = tw_config::sync::PASSPHRASE_MIN => + "the sync passphrase has to be at least {min} characters" + ), + )), + } +} + +use crate::backups::item_ref; + +impl Sync { + fn mgr(&self) -> Option> { + self.mgr.upgrade() + } + + fn data_dir(&self) -> Option { + self.mgr().map(|m| crate::plugins::dir_of(m.path())) + } + + /// 某个上游的地址最近一次经同步改过:改之前的地址、哪台设备、什么时候 + pub fn url_changed(&self, provider: &str) -> Option { + self.url_changed + .lock() + .ok() + .and_then(|m| m.get(provider).cloned()) + } + + /// 本机配置变了:`SETTLE` 之后上传 + pub fn touch(&self) { + if let Ok(mut d) = self.dirty.lock() + && d.is_none() + { + *d = Some(tokio::time::Instant::now() + SETTLE); + self.kick.notify_one(); + } + } + + /// 马上来一轮(不等定时) + pub fn kick(&self) { + self.kick.notify_one(); + } + + /// 配置里的 `sync` 一节和此刻在跑的对得上;对不上就照配置重建(同步位置、口令、不同步 + /// 的上游、代理变了)。没开同步是 `None` + fn ensure<'a>(&self, run: &'a mut Option) -> Result, Msg> { + let Some(mgr) = self.mgr() else { + *run = None; + return Ok(None); + }; + let cfg = mgr.gateway().config(); + let Some(sync) = cfg.sync.clone() else { + *run = None; + // 安全模式下配置里没有这一节是因为在服务的是临时配置:门留着(见 `start`) + if !self.paused.load(Ordering::Relaxed) { + self.register_backups(false); + } + return Ok(None); + }; + self.register_backups(true); + let proxies = format!("{:?}", cfg.proxies); + let fresh = match run.as_ref() { + Some(r) => r.config == sync && r.proxies == proxies, + None => false, + }; + if !fresh { + let store = store_for(&cfg, &sync.target)?; + let watch = match store.folder_root() { + Some(root) => match tw_sync::store::folder::watch(root) { + Ok((w, mut rx)) => { + let kick = self.kick.clone(); + tokio::spawn(async move { + while rx.recv().await.is_some() { + kick.notify_one(); + } + }); + Some(w) + } + Err(e) => { + tracing::warn!( + "the sync folder cannot be watched, so it is checked every 30 minutes: {e}" + ); + None + } + }, + None => None, + }; + let dir = crate::plugins::dir_of(mgr.path()); + let mut engine = Engine::new( + store, + CoreHost { + mgr: Arc::downgrade(&mgr), + }, + &dir, + settings_of(&cfg, &sync), + ); + // 同一个位置、同一个口令:缓存和捷径留着也没错,但重建了就从头看一遍 + engine.invalidate(); + *run = Some(Running { + engine, + config: sync, + proxies, + _watch: watch, + }); + } + Ok(run.as_mut()) + } + + /// 此刻给界面看的样子 + pub fn view(&self) -> tw_api::SyncView { + let on = self + .mgr() + .map(|m| m.gateway().config().sync.is_some()) + .unwrap_or(false); + if let Ok(v) = self.view.lock() + && let Some(v) = v.as_ref() + // 配置里开着、记下的却是关着(刚开启),或者反过来(手改文件关掉了):重新拼 + && on == (v.state != tw_api::SyncState::Off) + { + return v.clone(); + } + self.compute_view(None, None) + } + + /// 照配置、本机的同步状态和上一轮的结果拼出视图 + fn compute_view( + &self, + report: Option<&Report>, + state: Option, + ) -> tw_api::SyncView { + let off = tw_api::SyncView { + state: tw_api::SyncState::Off, + target: None, + encrypted: false, + device: None, + last_synced_ms: None, + failing_since_ms: None, + error: None, + devices: Vec::new(), + conflicts: 0, + excluded: Vec::new(), + excluded_cascade: Vec::new(), + }; + let Some(mgr) = self.mgr() else { return off }; + let cfg = mgr.gateway().config(); + let paused = self.paused.load(Ordering::Relaxed); + // 安全模式下在服务的是一份临时配置:同步设置在磁盘上那份读不进来的原文里 + let raw = (paused && cfg.sync.is_none()) + .then(|| mgr.current().ok()) + .flatten() + .and_then(|l| tw_config::sync::from_raw(&l.text)) + .map(|(s, _)| s); + let Some(sync) = cfg.sync.as_ref().or(raw.as_ref()) else { + return off; + }; + let local = tw_sync::local::Local::new(&crate::plugins::dir_of(mgr.path())); + let st = local.state(); + let report = report.cloned(); + let prev = self.view.lock().ok().and_then(|v| v.clone()); + let conflicts = report + .as_ref() + .map(|r| r.conflicts.len() as u32) + .or(prev.as_ref().map(|p| p.conflicts)) + .unwrap_or(0); + let devices = match &report { + Some(r) => r + .devices + .iter() + .map(|d| tw_api::SyncDevice { + id: d.id.clone(), + name: d.name.clone(), + core_version: d.core_version.clone(), + last_seen_ms: d.last_seen_ms, + this: d.id == sync.device_id, + paused_update: d.paused_update, + remote_core: d.remote_core, + }) + .collect(), + None => prev.as_ref().map(|p| p.devices.clone()).unwrap_or_default(), + }; + let state = if paused { + tw_api::SyncState::Paused + } else { + state.unwrap_or(if conflicts > 0 { + tw_api::SyncState::Conflict + } else { + tw_api::SyncState::Idle + }) + }; + let cascade = current_pack(&mgr) + .ok() + .and_then(|c| model::snapshot(&c.pack).ok()) + .map(|snap| { + let names: BTreeSet = sync.exclude.iter().cloned().collect(); + engine::excluded(&names, &[&snap]) + .into_iter() + .filter(|id| !(id.kind == ItemKind::Provider && names.contains(&id.name))) + .filter(|id| snap.items.contains_key(id)) + .map(|id| item_ref(&id)) + .collect() + }) + .unwrap_or_default(); + tw_api::SyncView { + state, + target: Some(target_to_api(&sync.target)), + encrypted: report.as_ref().map(|r| r.encrypted).unwrap_or(sync.encrypt), + device: Some(tw_api::SyncDeviceRef { + id: sync.device_id.clone(), + name: sync.device_name.clone(), + }), + last_synced_ms: st.last_synced_ms, + failing_since_ms: st.failing_since_ms, + error: st.error.clone(), + devices, + conflicts, + excluded: sync.exclude.clone(), + excluded_cascade: cascade, + } + } + + fn publish(&self, v: tw_api::SyncView) { + let changed = self + .view + .lock() + .map(|mut g| { + let changed = g.as_ref() != Some(&v); + *g = Some(v.clone()); + changed + }) + .unwrap_or(false); + if changed && let Some(mgr) = self.mgr() { + mgr.bus().emit(tw_api::Event::SyncStatus { + id: mgr.bus().next_id(), + view: v, + at_ms: now_ms(), + }); + } + } + + /// 记一次失败:从什么时候开始一直失败、原因 + fn record_failure(&self, r: &Running, e: &RoundError) { + let local = r.engine.local(); + let mut st = local.state(); + st.failing_since_ms.get_or_insert_with(now_ms); + st.error = Some(e.msg()); + let _ = local.save_state(&st); + self.failures.fetch_add(1, Ordering::Relaxed); + } + + /// 同步一轮(顺带处理几处冲突),更新状态、发事件 + pub async fn round(&self, decisions: &[Decision]) -> Result { + let mut guard = self.run.lock().await; + let r = match self.ensure(&mut guard) { + Ok(Some(r)) => r, + Ok(None) => { + let v = self.compute_view(None, None); + self.publish(v.clone()); + return Ok(v); + } + Err(m) => { + let v = tw_api::SyncView { + error: Some(m.clone()), + ..self.compute_view(None, Some(tw_api::SyncState::AuthFailed)) + }; + self.publish(v.clone()); + return Err(fail(StatusCode::BAD_REQUEST, m)); + } + }; + if self.paused.load(Ordering::Relaxed) { + let v = self.compute_view(None, None); + self.publish(v.clone()); + return Ok(v); + } + if let Ok(mut d) = self.dirty.lock() { + *d = None; + } + if let Ok(mut g) = self.view.lock() + && let Some(v) = g.as_mut() + { + v.state = tw_api::SyncState::Syncing; + } + let mut result = r.engine.round_with(decisions).await; + for _ in 0..2 { + if matches!(result, Err(RoundError::Stale)) { + result = r.engine.round_with(decisions).await; + } + } + let outcome = match result { + Ok(report) => { + self.failures.store(0, Ordering::Relaxed); + let encrypted = report.encrypted; + let st = r.engine.local().state(); + if let Ok(mut m) = self.url_changed.lock() { + *m = st + .url_changed + .iter() + .map(|(k, v)| (k.clone(), url_change(v))) + .collect(); + } + if !report.keys_changed.is_empty() + && let Some(mgr) = self.mgr() + { + mgr.bus().emit(tw_api::Event::GatewayKeysChanged { + id: mgr.bus().next_id(), + names: report.keys_changed.clone(), + at_ms: now_ms(), + }); + } + // 别的设备关掉了加密:这台跟着改设置 + let follow = (encrypted != r.config.encrypt && !encrypted).then(|| { + let mut s = r.config.clone(); + s.encrypt = false; + s.passphrase = None; + s + }); + Ok((report, follow)) + } + Err(e) => Err(e), + }; + let v = match outcome { + Ok((report, follow)) => { + let v = self.compute_view(Some(&report), None); + drop(guard); + if let (Some(s), Some(mgr)) = (follow, self.mgr()) { + let _ = write_section(&mgr, Some(&s), Origin::Sync).await; + } + v + } + Err(RoundError::NotArrived) => { + drop(guard); + self.view() + } + Err(RoundError::Emptied) => { + // 别的设备停止同步并删除了同步数据:这台也停下,说一声 + r.engine.local().clear(); + drop(guard); + if let Some(mgr) = self.mgr() { + let _ = write_section(&mgr, None, Origin::Sync).await; + } + *self.run.lock().await = None; + self.register_backups(false); + tw_api::SyncView { + error: Some(RoundError::Emptied.msg()), + ..self.compute_view(None, Some(tw_api::SyncState::Off)) + } + } + Err(e) => { + self.record_failure(r, &e); + let st = r.engine.local().state(); + let state = match &e { + RoundError::Store(StoreError::AuthFailed(_)) => tw_api::SyncState::AuthFailed, + RoundError::Locked(_) => tw_api::SyncState::Locked, + RoundError::TooNew { .. } => tw_api::SyncState::TooNew, + _ => { + let since = st.failing_since_ms.unwrap_or_else(now_ms); + if now_ms().saturating_sub(since) >= UNREACHABLE_AFTER_MS { + tw_api::SyncState::Unreachable + } else { + let prev = self.view.lock().ok().and_then(|v| v.clone()); + match prev.map(|p| p.conflicts) { + Some(n) if n > 0 => tw_api::SyncState::Conflict, + _ => tw_api::SyncState::Idle, + } + } + } + }; + drop(guard); + self.compute_view(None, Some(state)) + } + }; + self.publish(v.clone()); + Ok(v) + } + + /// 退出前:还有没上传的改动就马上传 + pub async fn flush(&self) { + let pending = self.dirty.lock().map(|d| d.is_some()).unwrap_or(false); + if !pending || self.paused.load(Ordering::Relaxed) { + return; + } + let _ = tokio::time::timeout(FLUSH_WITHIN, self.round(&[])).await; + } + + /// 下一次定时检查隔多久:失败着就按重试的间隔 + fn next_poll(&self) -> Duration { + let n = self.failures.load(Ordering::Relaxed) as usize; + if n == 0 { + return POLL; + } + Duration::from_secs(RETRY[(n - 1).min(RETRY.len() - 1)]) + } + + /// 冲突列表 + pub async fn conflicts(&self) -> Vec { + let guard = self.run.lock().await; + let Some(r) = guard.as_ref() else { + return Vec::new(); + }; + let me = r.config.device_id.clone(); + let Some(mgr) = self.mgr() else { + return Vec::new(); + }; + let local = current_pack(&mgr) + .ok() + .and_then(|c| model::snapshot(&c.pack).ok()) + .unwrap_or_default(); + r.engine + .last_report() + .conflicts + .iter() + .map(|c| { + let units = local.items.get(&c.item).map(|it| c.pick(&it.units)); + let (mine, theirs) = c.mine(&me, units.as_ref()); + tw_api::SyncConflict { + id: c.id.clone(), + item: item_ref(&c.item), + units: c.units.clone(), + local: tw_api::SyncConflictSide { + device: mine.device_name.clone(), + at_ms: mine.at_ms, + deleted: mine.units.is_none(), + }, + remote: tw_api::SyncConflictSide { + device: theirs.device_name.clone(), + at_ms: theirs.at_ms, + deleted: theirs.units.is_none(), + }, + diff: c + .units + .iter() + .map(|u| tw_api::SyncConflictDiff { + unit: u.clone(), + local: mine.units.as_ref().and_then(|m| m.get(u).cloned()), + remote: theirs.units.as_ref().and_then(|m| m.get(u).cloned()), + }) + .collect(), + both_allowed: c.both_allowed(), + reason: c.reason.map(|r| match r { + tw_sync::record::Reason::KeyInUse => tw_api::SyncConflictReason::KeyInUse, + tw_sync::record::Reason::Invalid => tw_api::SyncConflictReason::Invalid, + }), + } + }) + .collect() + } +} + +// ─────────────────────────────────────────────────────────── 同步位置上的备份 + +impl Sync { + /// 读同步位置用的引擎:开着同步就是在跑的那一个。**安全模式下**在服务的是一份临时配置, + /// 没有 `sync` 一节:照磁盘上那份读不进来的原文里的 `sync` 一节临时建一个 + async fn with_engine( + &self, + f: impl AsyncFnOnce(&mut Eng) -> Result, + ) -> Option> { + let mut guard = self.run.lock().await; + if let Ok(Some(r)) = self.ensure(&mut guard) { + return Some(f(&mut r.engine).await); + } + drop(guard); + let mgr = self.mgr()?; + let text = mgr.current().ok()?.text; + let (sync, proxies) = tw_config::sync::from_raw(&text)?; + let cfg = tw_config::Config { + proxies, + ..Default::default() + }; + let store = store_for(&cfg, &sync.target).ok()?; + let mut e = Engine::new( + store, + CoreHost { + mgr: Arc::downgrade(&mgr), + }, + &crate::plugins::dir_of(mgr.path()), + settings_of(&cfg, &sync), + ); + Some(f(&mut e).await) + } + + /// 同步位置上读得出来的版本。没开同步、连不上是空的 + pub async fn remote_versions(&self) -> Vec { + let got = self + .with_engine(async |e: &mut Eng| e.remote_versions().await) + .await; + match got { + Some(Ok(vs)) => vs + .iter() + .map(|v| RemoteVersionInfo { + content_id: v.content.clone(), + at_ms: v.pack.manifest.created_at_ms, + device_name: v.meta.device_name.clone(), + core_version: v.pack.manifest.core_version.clone(), + counts: v.pack.manifest.counts, + }) + .collect(), + Some(Err(e)) => { + tracing::debug!( + "the sync location could not be listed for backups: {}", + e.msg() + ); + Vec::new() + } + None => Vec::new(), + } + } + + /// 同步位置上内容标识是这个的版本(不加密的配置包,不含本机设置) + pub async fn remote_pack(&self, content_id: &str) -> Result, Msg> { + let id = content_id.to_string(); + match self + .with_engine(async move |e: &mut Eng| e.remote_version(&id).await) + .await + { + Some(Ok(p)) => Ok(p), + Some(Err(e)) => Err(e.msg()), + None => Ok(None), + } + } +} + +/// 备份列表看同步位置的那一扇门(`crate::backups::RemoteBackups`)。开着同步时装在配置管理上 +/// ([`ConfigManager::set_remote_backups`]),关掉时拿掉 +struct Backups(Weak); + +impl RemoteBackups for Backups { + fn list(&self) -> BoxFuture<'_, Vec> { + Box::pin(async move { + match self.0.upgrade() { + Some(s) => s.remote_versions().await, + None => Vec::new(), + } + }) + } + + fn fetch<'a>(&'a self, content_id: &'a str) -> BoxFuture<'a, Result> { + Box::pin(async move { + let Some(s) = self.0.upgrade() else { + return Err(msg!("sync.shutting_down" => "the core is shutting down")); + }; + s.remote_pack(content_id).await?.ok_or_else(|| { + msg!( + "sync.version_gone", id = content_id => + "the sync location no longer has version {id}" + ) + }) + }) + } + + fn other_devices(&self) -> BoxFuture<'_, u32> { + Box::pin(async move { + self.0 + .upgrade() + .map(|s| s.view().devices.iter().filter(|d| !d.this).count() as u32) + .unwrap_or(0) + }) + } +} + +impl Sync { + /// 开着同步(配置里有 `sync` 一节,安全模式下原文里有)就把这扇门装上,没开就拿掉 + fn register_backups(&self, on: bool) { + let Some(mgr) = self.mgr() else { return }; + let have = mgr.remote_backups().is_some(); + if on && !have { + mgr.set_remote_backups(Some(Arc::new(Backups(self.me.clone())))); + } else if !on && have { + mgr.set_remote_backups(None); + } + } +} + +// ─────────────────────────────────────────────────────────── 调度 + +/// 起同步的后台任务:启动时一轮,之后按上面那张表。`safe` 是安全模式:不自动同步,状态是 +/// `paused`,备份列表照样能读同步位置 +pub fn start(s: &ControlState, safe: bool) -> Arc { + let sync = service(s); + if safe { + sync.paused.store(true, Ordering::Relaxed); + // 出错页的「从备份恢复」要列同步位置上的版本:原文里有同步设置就把门装上 + let on = s.config().sync.is_some() + || s.cfg + .current() + .ok() + .and_then(|l| tw_config::sync::from_raw(&l.text)) + .is_some(); + sync.register_backups(on); + sync.publish(sync.compute_view(None, None)); + return sync; + } + if sync.started.swap(true, Ordering::Relaxed) { + return sync; + } + let bus = s.bus().clone(); + let me = sync.clone(); + tokio::spawn(async move { + let mut rx = bus.subscribe(); + // 从睡眠中醒来:墙上的钟比单调钟多走了一大截 + let mut wall = std::time::SystemTime::now(); + let mut mono = tokio::time::Instant::now(); + let mut tick = tokio::time::interval(Duration::from_secs(60)); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let mut next = tokio::time::Instant::now(); + loop { + let dirty = me.dirty.lock().ok().and_then(|d| *d); + let due = match dirty { + Some(d) if d < next => d, + _ => next, + }; + tokio::select! { + _ = me.kick.notified() => { + // 本机刚改了配置:等它稳下来 + if let Some(d) = me.dirty.lock().ok().and_then(|d| *d) + && d > tokio::time::Instant::now() + { + continue; + } + } + _ = tokio::time::sleep_until(due) => {} + ev = rx.recv() => { + match ev { + Ok(tw_api::Event::ConfigReloaded { origin, .. }) + if origin != tw_api::ConfigOrigin::Sync => me.touch(), + Ok(tw_api::Event::ClientPrefsChanged { .. }) => me.touch(), + Err(tokio::sync::broadcast::error::RecvError::Closed) => return, + _ => {} + } + continue; + } + _ = tick.tick() => { + let wall_gap = std::time::SystemTime::now() + .duration_since(wall) + .unwrap_or_default(); + let mono_gap = mono.elapsed(); + wall = std::time::SystemTime::now(); + mono = tokio::time::Instant::now(); + if wall_gap < mono_gap + Duration::from_secs(120) { + continue; + } + tracing::debug!("woke from sleep; checking the sync location"); + } + } + if me.mgr().is_none() { + return; + } + let _ = me.round(&[]).await; + next = tokio::time::Instant::now() + me.next_poll(); + } + }); + sync.kick(); + sync +} + +// ─────────────────────────────────────────────────────────── 加入 + +/// 本机配置还是初始配置(新安装后没改过):只有一把网关密钥,没有上游、路由这些,全局设置 +/// 都是默认值。随 core 发的默认插件(停用着)不算 +fn is_initial(snap: &Snapshot) -> bool { + let defaults: BTreeSet<&str> = tw_gateway::plugin::defaults::ALL + .iter() + .map(|(id, _)| *id) + .collect(); + let mut keys = 0; + for (id, it) in &snap.items { + match id.kind { + ItemKind::Key => keys += 1, + ItemKind::ClientPrefs | ItemKind::PluginOrder => {} + ItemKind::Plugin + if defaults.contains(id.name.as_str()) + && it.units.get(model::unit::ENABLED) + != Some(&serde_json::Value::Bool(true)) => {} + _ => return false, + } + } + keys <= 1 +} + +/// 加入时「合并」:和导入同一套规矩(`tw_pack::import`),本机打底,同步数据逐项并进来。 +/// 新增的默认导入,冲突的必须选(覆盖、改名导入、跳过),没选是 `sync.join_undecided`。 +/// +/// **两处和导入不一样**:导入进来的插件按导入的规矩是停用的,加入同步时照同步数据里的开关 +/// (各设备一致,见方案 6.2;不加密时同步一轮里另有停用的规矩);导入预览里只有本机有的 +/// 项目不出现,这里也一样 —— 它们会随这次加入上传 +fn merge_join( + local: &Pack, + remote: &Pack, + decisions: &[tw_api::ImportDecision], +) -> Result { + let ctx = crate::backups::import_ctx(); + let rows = tw_pack::import::preview(local, remote, &ctx).map_err(|e| import_fail(&e))?; + let decided: BTreeMap = decisions + .iter() + .map(|d| (crate::backups::item_id(&d.item), d)) + .collect(); + let mut out = Vec::new(); + for r in &rows { + let action = match decided.get(&r.item) { + Some(d) => match d.action { + tw_api::ImportAction::Import => tw_pack::import::Action::Import, + tw_api::ImportAction::Overwrite => tw_pack::import::Action::Overwrite, + tw_api::ImportAction::Rename => { + tw_pack::import::Action::Rename(d.new_name.clone().unwrap_or_default()) + } + tw_api::ImportAction::Skip => tw_pack::import::Action::Skip, + }, + None => match r.status { + tw_pack::import::Status::New => tw_pack::import::Action::Import, + tw_pack::import::Status::Same | tw_pack::import::Status::Error => { + tw_pack::import::Action::Skip + } + tw_pack::import::Status::Conflict => { + return Err(fail( + StatusCode::BAD_REQUEST, + msg!( + "sync.join_undecided", item = &r.item => + "{item} differs between this device and the sync data; choose to \ + overwrite, import under a new name, or skip" + ), + )); + } + }, + }; + out.push(tw_pack::import::Decision { + item: r.item.clone(), + action, + }); + } + let applied = tw_pack::import::apply(local, remote, &out, &ctx).map_err(|e| import_fail(&e))?; + // 插件照同步数据里的开关,不按导入的规矩停用 + let mut snap = + model::snapshot(&applied.pack).map_err(|e| fail(StatusCode::BAD_REQUEST, e.msg()))?; + let theirs = model::snapshot(remote).map_err(|e| fail(StatusCode::BAD_REQUEST, e.msg()))?; + let mut changed = false; + for d in &out { + if d.item.kind != ItemKind::Plugin || matches!(d.action, tw_pack::import::Action::Skip) { + continue; + } + let name = match &d.action { + tw_pack::import::Action::Rename(n) => n.clone(), + _ => d.item.name.clone(), + }; + let id = ItemId::new(ItemKind::Plugin, name); + let want = theirs + .items + .get(&d.item) + .and_then(|it| it.units.get(model::unit::ENABLED)) + .cloned(); + if let (Some(it), Some(want)) = (snap.items.get_mut(&id), want) + && it.units.get(model::unit::ENABLED) != Some(&want) + { + it.units.insert(model::unit::ENABLED.to_string(), want); + changed = true; + } + } + if !changed { + return Ok(applied.pack); + } + snap.relink(); + model::materialize(local, &snap).map_err(|e| fail(StatusCode::BAD_REQUEST, e.msg())) +} + +fn import_fail(e: &tw_pack::import::ImportError) -> Fail { + use tw_pack::import::ImportError as E; + let code = match e { + E::Exists(_) | E::NameTaken(..) => StatusCode::CONFLICT, + E::NotInPack(_) => StatusCode::NOT_FOUND, + E::Item(..) => StatusCode::UNPROCESSABLE_ENTITY, + E::Model(_) | E::Rejected(_) | E::TooNew(_) => StatusCode::BAD_REQUEST, + }; + fail(code, e.msg()) +} + +// ─────────────────────────────────────────────────────────── 端点 + +async fn status(State(s): State) -> Json { + Json(service(&s).view()) +} + +fn config_target( + s: &ControlState, + t: &tw_api::SyncTarget, +) -> Result<(tw_config::sync::Target, AnyStore), Fail> { + let target = target_to_config(t); + let store = store_for(&s.config(), &target).map_err(|m| fail(StatusCode::BAD_REQUEST, m))?; + Ok((target, store)) +} + +fn device_view(d: &tw_sync::device::DeviceRecord, me: Option<&str>) -> tw_api::SyncDevice { + tw_api::SyncDevice { + id: d.id.clone(), + name: d.name.clone(), + core_version: d.core_version.clone(), + last_seen_ms: d.last_seen_ms, + this: Some(d.id.as_str()) == me, + paused_update: d.paused_update, + remote_core: d.remote_core, + } +} + +async fn test( + State(s): State, + Json(req): Json, +) -> Result, Fail> { + let (_, store) = config_target(&s, &req.target)?; + let p = Eng::probe(&store, req.passphrase.as_deref()).await; + let me = s.config().sync.as_ref().map(|y| y.device_id.clone()); + Ok(Json(tw_api::SyncProbe { + writable: p.writable.is_ok(), + error: p.writable.err().map(|e| e.msg()), + existing: p.existing.map(|e| tw_api::SyncExisting { + encrypted: e.encrypted, + unlocked: e.unlocked, + devices: e + .devices + .iter() + .map(|d| device_view(d, me.as_deref())) + .collect(), + }), + })) +} + +/// 一台还没开同步的设备对着某个同步位置的引擎(加入预览、开启) +fn temp_engine( + s: &ControlState, + target: &tw_config::sync::Target, + store: AnyStore, + sync: &tw_config::sync::Sync, +) -> Eng { + let _ = target; + Engine::new( + store, + CoreHost { + mgr: Arc::downgrade(&s.cfg), + }, + &crate::plugins::dir_of(s.config_path()), + settings_of(&s.config(), sync), + ) +} + +fn draft( + target: tw_config::sync::Target, + encrypt: bool, + passphrase: Option, + device_name: &str, +) -> tw_config::sync::Sync { + tw_config::sync::Sync { + device_id: tw_config::sync::generate_device_id(), + device_name: device_name.trim().to_string(), + encrypt, + passphrase, + exclude: Vec::new(), + target, + } +} + +async fn join_preview( + State(s): State, + Json(req): Json, +) -> Result, Fail> { + let (target, store) = config_target(&s, &req.target)?; + let cur = current_pack(&s.cfg).map_err(|m| fail(StatusCode::INTERNAL_SERVER_ERROR, m))?; + let local = model::snapshot(&cur.pack).map_err(|e| fail(StatusCode::BAD_REQUEST, e.msg()))?; + if is_initial(&local) { + return Ok(Json(tw_api::SyncJoinPreview { + initial: true, + rows: Vec::new(), + })); + } + let probe = Eng::probe(&store, req.passphrase.as_deref()).await; + if probe.existing.is_none() { + return Ok(Json(tw_api::SyncJoinPreview { + initial: false, + rows: Vec::new(), + })); + } + let encrypted = probe.existing.as_ref().is_some_and(|e| e.encrypted); + let mut e = temp_engine( + &s, + &target, + store, + &draft(target.clone(), encrypted, req.passphrase.clone(), "preview"), + ); + let remote = e.latest().await.map_err(|e| round_fail(&e))?; + // 和导入预览同一套规矩 + let rows = tw_pack::import::preview(&cur.pack, &remote, &crate::backups::import_ctx()) + .map_err(|e| import_fail(&e))?; + Ok(Json(tw_api::SyncJoinPreview { + initial: false, + rows: rows.into_iter().map(crate::backups::row_view).collect(), + })) +} + +async fn enable( + State(s): State, + Json(req): Json, +) -> Result, Fail> { + let sync = service(&s); + let mut guard = sync.run.lock().await; + if s.config().sync.is_some() { + return Err(fail( + StatusCode::CONFLICT, + msg!("sync.already_on" => "sync is already on. Stop it before using another location"), + )); + } + let (target, store) = config_target(&s, &req.target)?; + let probe = Eng::probe(&store, req.passphrase.as_deref()).await; + if let Err(e) = &probe.writable { + return Err(round_fail(&RoundError::Store(e.clone()))); + } + let mut settings = match &probe.existing { + None => { + let pass = if req.encrypt { + Some(check_passphrase(req.passphrase.as_deref())?.to_string()) + } else { + None + }; + draft(target.clone(), req.encrypt, pass, &req.device_name) + } + Some(e) => { + if e.encrypted && !e.unlocked { + return Err(fail( + StatusCode::BAD_REQUEST, + tw_sync::keyring::KeyringError::WrongPassphrase.msg(), + )); + } + draft( + target.clone(), + e.encrypted, + e.encrypted.then(|| req.passphrase.clone()).flatten(), + &req.device_name, + ) + } + }; + settings + .check() + .map_err(|e| fail(StatusCode::BAD_REQUEST, e.msg()))?; + let mut engine = temp_engine(&s, &target, store, &settings); + match &probe.existing { + None => { + if !engine.create().await.map_err(|e| round_fail(&e))? { + return Err(fail( + StatusCode::CONFLICT, + msg!( + "sync.location_taken" => + "another device has just started syncing at this location. Check the \ + location again and join it" + ), + )); + } + } + Some(_) => { + let cur = + current_pack(&s.cfg).map_err(|m| fail(StatusCode::INTERNAL_SERVER_ERROR, m))?; + let local = + model::snapshot(&cur.pack).map_err(|e| fail(StatusCode::BAD_REQUEST, e.msg()))?; + let merged = if is_initial(&local) || req.join == tw_api::SyncJoin::UseRemote { + None + } else { + let remote_pack = engine.latest().await.map_err(|e| round_fail(&e))?; + Some(merge_join(&cur.pack, &remote_pack, &req.decisions)?) + }; + let report = engine.join(merged).await.map_err(|e| round_fail(&e))?; + if !report.keys_changed.is_empty() { + s.bus().emit(tw_api::Event::GatewayKeysChanged { + id: s.bus().next_id(), + names: report.keys_changed.clone(), + at_ms: now_ms(), + }); + } + settings.encrypt = report.encrypted; + } + } + let version = write_section(&s.cfg, Some(&settings), Origin::Ui) + .await + .map_err(apply_fail)?; + let cfg = s.config(); + *guard = Some(Running { + engine, + config: settings, + proxies: format!("{:?}", cfg.proxies), + _watch: None, + }); + // 文件夹的监听在下一轮由 ensure 补上:重建一次 + if let Some(r) = guard.as_mut() { + r.proxies.clear(); + } + drop(guard); + sync.failures.store(0, Ordering::Relaxed); + sync.register_backups(true); + sync.publish(sync.compute_view(None, None)); + sync.kick(); + Ok(Json(tw_api::ConfigWritten { version })) +} + +async fn save( + State(s): State, + Json(req): Json, +) -> Result, Fail> { + let sync = service(&s); + let _guard = sync.run.lock().await; + let Some(mut cur) = s.config().sync.clone() else { + return Err(sync_off()); + }; + if let Some(t) = &req.target { + let t = target_to_config(t); + if !t.same_place(&cur.target) { + return Err(fail( + StatusCode::BAD_REQUEST, + msg!( + "sync.place_changed" => + "this is a different sync location. Stop syncing, then turn sync on with the \ + new location" + ), + )); + } + cur.target = t; + } + if let Some(n) = &req.device_name { + cur.device_name = n.trim().to_string(); + } + if let Some(ex) = &req.excluded { + let mut v: Vec = ex.iter().map(|e| e.trim().to_string()).collect(); + v.dedup(); + cur.exclude = v; + } + cur.check() + .map_err(|e| fail(StatusCode::BAD_REQUEST, e.msg()))?; + let version = write_section(&s.cfg, Some(&cur), Origin::Ui) + .await + .map_err(apply_fail)?; + drop(_guard); + sync.kick(); + Ok(Json(tw_api::ConfigWritten { version })) +} + +async fn now(State(s): State) -> Result, Fail> { + let sync = service(&s); + if s.config().sync.is_none() { + return Ok(Json(sync.view())); + } + sync.round(&[]).await.map(Json) +} + +async fn conflicts(State(s): State) -> Json> { + Json(service(&s).conflicts().await) +} + +async fn resolve( + State(s): State, + Json(req): Json, +) -> Result, Fail> { + if s.config().sync.is_none() { + return Err(sync_off()); + } + let decisions: Vec = req + .decisions + .iter() + .map(|d| Decision { + id: d.id.clone(), + take: match d.take { + tw_api::SyncTake::Local => Take::Local, + tw_api::SyncTake::Remote => Take::Remote, + tw_api::SyncTake::Both => Take::Both, + }, + new_name: d.new_name.clone(), + }) + .collect(); + let sync = service(&s); + let v = sync.round(&decisions).await?; + if let Some(e) = v + .error + .as_ref() + .filter(|_| v.state != tw_api::SyncState::Conflict && v.state != tw_api::SyncState::Idle) + { + return Err(fail(StatusCode::BAD_GATEWAY, e.clone())); + } + Ok(Json(v)) +} + +async fn encryption( + State(s): State, + Json(req): Json, +) -> Result, Fail> { + let sync = service(&s); + let mut guard = sync.run.lock().await; + let r = match sync.ensure(&mut guard) { + Ok(Some(r)) => r, + Ok(None) => return Err(sync_off()), + Err(m) => return Err(fail(StatusCode::BAD_REQUEST, m)), + }; + let mut next = r.config.clone(); + let res = match req.action { + tw_api::SyncEncryptionAction::Enable => { + let p = check_passphrase(req.passphrase.as_deref())?.to_string(); + next.encrypt = true; + next.passphrase = Some(p.clone()); + r.engine.enable_encryption(&p).await + } + tw_api::SyncEncryptionAction::Disable => { + let p = req.passphrase.clone().unwrap_or_default(); + next.encrypt = false; + next.passphrase = None; + r.engine.disable_encryption(&p).await + } + tw_api::SyncEncryptionAction::Change => { + let new = check_passphrase(req.new_passphrase.as_deref())?.to_string(); + let old = req.passphrase.clone().unwrap_or_default(); + next.passphrase = Some(new.clone()); + r.engine.change_passphrase(&old, &new).await + } + }; + res.map_err(|e| round_fail(&e))?; + r.config = next.clone(); + r.engine.set_settings(settings_of(&s.config(), &next)); + drop(guard); + write_section(&s.cfg, Some(&next), Origin::Ui) + .await + .map_err(apply_fail)?; + sync.round(&[]).await.map(Json) +} + +async fn unlock( + State(s): State, + Json(req): Json, +) -> Result, Fail> { + let sync = service(&s); + let mut guard = sync.run.lock().await; + let r = match sync.ensure(&mut guard) { + Ok(Some(r)) => r, + Ok(None) => return Err(sync_off()), + Err(m) => return Err(fail(StatusCode::BAD_REQUEST, m)), + }; + let encrypted = r + .engine + .unlock(&req.passphrase) + .await + .map_err(|e| round_fail(&e))?; + let mut next = r.config.clone(); + if encrypted { + next.encrypt = true; + next.passphrase = Some(req.passphrase.clone()); + } + r.config = next.clone(); + r.engine.set_settings(settings_of(&s.config(), &next)); + drop(guard); + write_section(&s.cfg, Some(&next), Origin::Ui) + .await + .map_err(apply_fail)?; + sync.round(&[]).await.map(Json) +} + +async fn disable( + State(s): State, + Json(req): Json, +) -> Result, Fail> { + let sync = service(&s); + let mut guard = sync.run.lock().await; + if let Ok(Some(r)) = sync.ensure(&mut guard) { + match r.engine.stop(req.purge).await { + Ok(()) => {} + // 只是移除自己的设备记录时连不上也照样停:半年之后它自己会过期 + Err(e) if !req.purge => { + tracing::warn!( + "the device record could not be removed from the sync location: {}", + e.msg() + ); + r.engine.local().clear(); + } + Err(e) => return Err(round_fail(&e)), + } + } else if let Some(dir) = sync.data_dir() { + tw_sync::local::Local::new(&dir).clear(); + } + *guard = None; + drop(guard); + if s.config().sync.is_some() { + write_section(&s.cfg, None, Origin::Ui) + .await + .map_err(apply_fail)?; + } + if let Ok(mut m) = sync.url_changed.lock() { + m.clear(); + } + sync.register_backups(false); + sync.publish(sync.compute_view(None, None)); + Ok(Json(())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn targets_convert_both_ways_and_blank_proxies_are_direct() { + let api = tw_api::SyncTarget::S3 { + endpoint: " https://abc.r2.cloudflarestorage.com ".into(), + region: "auto".into(), + bucket: "tw".into(), + prefix: "/me/".into(), + access_key_id: "AK".into(), + secret_access_key: "SK".into(), + proxy: String::new(), + }; + let c = target_to_config(&api); + assert_eq!(c.proxy(), "direct"); + match target_to_api(&c) { + tw_api::SyncTarget::S3 { + endpoint, prefix, .. + } => { + assert_eq!(endpoint, "https://abc.r2.cloudflarestorage.com"); + assert_eq!(prefix, "me"); + } + other => panic!("{other:?}"), + } + } +} diff --git a/crates/tw-control/tests/sync_devices.rs b/crates/tw-control/tests/sync_devices.rs new file mode 100644 index 00000000..4ac190da --- /dev/null +++ b/crates/tw-control/tests/sync_devices.rs @@ -0,0 +1,683 @@ +//! 多设备同步走控制面:两台设备(两个数据目录、两份配置、两个控制面)经同一个文件夹位置。 +//! +//! 合并规则本身在 `tw-sync` 的测试里逐条测过;这里测的是接缝:开启与加入、经写入流程应用 +//! (来源记为同步、热更新)、状态事件、密钥变了的事件、冲突经端点处理、安全模式暂停、 +//! 本机改动 10 秒后上传而另一台的文件夹监听即时收到。 + +use std::sync::Arc; +use std::time::Duration; + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use serde_json::{Value, json}; +use tower::ServiceExt; +use tw_control::{ConfigManager, ControlState}; + +const PASS: &str = "correct horse battery staple"; + +fn base(key: char) -> String { + format!( + "version: 1\nlisten:\n control:\n key: {}\nclients:\n - name: default\n key: tw-default-{key}\n", + key.to_string().repeat(64) + ) +} + +struct Dev { + dir: tempfile::TempDir, + state: ControlState, + app: axum::Router, +} + +impl Dev { + fn new(key: char) -> Dev { + tw_sync::keyring::set_work_factor_for_tests(4); + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join("config.yaml"); + let text = base(key); + std::fs::write(&p, &text).unwrap(); + let cfg = tw_config::try_parse(&text).unwrap(); + let gw = tw_gateway::AppState::new(cfg).unwrap(); + // 插件引擎用假的:照真的那一套读 manifest,不起沙箱 + gw.set_plugin_engine(Arc::new(tw_gateway::plugin::fake::FakeEngine)); + let bus = gw.bus.clone(); + let state = ControlState { + shutdown: Default::default(), + remote: Default::default(), + cfg: Arc::new(ConfigManager::new(p, gw.clone(), bus)), + gateway: gw, + store: None, + started: std::time::Instant::now(), + price_updater: Default::default(), + chatgpt: Default::default(), + zai: Default::default(), + }; + Dev { + app: tw_control::router(state.clone()), + state, + dir, + } + } + + async fn call(&self, method: &str, path: &str, body: Value) -> (StatusCode, Value) { + let r = self + .app + .clone() + .oneshot( + Request::builder() + .method(method) + .uri(path) + .header("content-type", "application/json") + .body(Body::from(if body.is_null() { + String::new() + } else { + body.to_string() + })) + .unwrap(), + ) + .await + .unwrap(); + let status = r.status(); + let bytes = axum::body::to_bytes(r.into_body(), usize::MAX) + .await + .unwrap(); + ( + status, + serde_json::from_slice(&bytes).unwrap_or(Value::Null), + ) + } + + async fn ok(&self, method: &str, path: &str, body: Value) -> Value { + let (s, v) = self.call(method, path, body).await; + assert!(s.is_success(), "{method} {path}: {s} {v}"); + v + } + + fn text(&self) -> String { + std::fs::read_to_string(self.dir.path().join("config.yaml")).unwrap() + } + + fn config(&self) -> tw_config::Config { + self.state.config().as_ref().clone() + } + + /// 用户在界面上改了配置(`PUT /config`) + async fn edit(&self, f: impl FnOnce(&str) -> String) { + let cur = self.ok("GET", "/config", Value::Null).await; + let next = f(cur["text"].as_str().unwrap()); + self.ok( + "PUT", + "/config", + json!({ "base_version": cur["version"], "text": next }), + ) + .await; + } +} + +fn folder(loc: &std::path::Path) -> Value { + json!({ "type": "folder", "path": loc.display().to_string(), "proxy": "direct" }) +} + +fn add_provider(text: &str, name: &str, url: &str, key: &str) -> String { + let p = format!(" - name: {name}\n base_url: {url}\n key: {key}\n"); + if text.contains("\nproviders:\n") { + text.replacen("\nproviders:\n", &format!("\nproviders:\n{p}"), 1) + } else { + format!("{text}providers:\n{p}") + } +} + +/// A 开启(同步位置空着,加密),B 加入(新安装:直接用同步数据) +async fn pair(loc: &std::path::Path) -> (Dev, Dev) { + let (a, b) = (Dev::new('a'), Dev::new('b')); + let probe = a + .ok("POST", "/sync/test", json!({ "target": folder(loc) })) + .await; + assert_eq!(probe["writable"], true); + assert!(probe["existing"].is_null(), "空的同步位置:{probe}"); + // 口令太短不收 + let (s, v) = a + .call( + "POST", + "/sync/enable", + json!({ "target": folder(loc), "encrypt": true, "passphrase": "short", "device_name": "Mac mini", "join": "use_remote" }), + ) + .await; + assert_eq!(s, StatusCode::BAD_REQUEST, "{v}"); + assert_eq!(v["code"], "sync.passphrase_short"); + a.ok( + "POST", + "/sync/enable", + json!({ "target": folder(loc), "encrypt": true, "passphrase": PASS, "device_name": "Mac mini", "join": "use_remote" }), + ) + .await; + assert!(a.config().sync.is_some(), "同步设置写进了配置"); + let v = a.ok("GET", "/sync", Value::Null).await; + assert_ne!(v["state"], "off", "开启之后马上就不是关着:{v}"); + assert_eq!(v["device"]["name"], "Mac mini"); + + let probe = b + .ok( + "POST", + "/sync/test", + json!({ "target": folder(loc), "passphrase": PASS }), + ) + .await; + assert_eq!(probe["existing"]["encrypted"], true); + assert_eq!(probe["existing"]["unlocked"], true); + assert_eq!(probe["existing"]["devices"][0]["name"], "Mac mini"); + let preview = b + .ok( + "POST", + "/sync/enable/preview", + json!({ "target": folder(loc), "passphrase": PASS }), + ) + .await; + assert_eq!(preview["initial"], true, "新安装的配置不出现合并这一步"); + // 口令不对加入不了 + let (s, _) = b + .call( + "POST", + "/sync/enable", + json!({ "target": folder(loc), "encrypt": true, "passphrase": "wrong passphrase!", "device_name": "MacBook Air", "join": "use_remote" }), + ) + .await; + assert_eq!(s, StatusCode::BAD_REQUEST); + b.ok( + "POST", + "/sync/enable", + json!({ "target": folder(loc), "encrypt": true, "passphrase": PASS, "device_name": "MacBook Air", "join": "use_remote" }), + ) + .await; + (a, b) +} + +#[tokio::test] +async fn two_devices_turn_sync_on_and_changes_flow_both_ways() { + let loc = tempfile::tempdir().unwrap(); + let (a, b) = pair(loc.path()).await; + let mut events = b.state.bus().subscribe(); + + a.edit(|t| add_provider(t, "relay", "https://relay.example/v1", "sk-relay")) + .await; + a.ok("POST", "/sync/now", Value::Null).await; + let v = b.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["state"], "idle", "{v}"); + assert_eq!(v["encrypted"], true); + assert_eq!(v["devices"].as_array().unwrap().len(), 2, "{v}"); + // B 收到了,而且已经生效(网关换入了新配置) + let p = b.config().providers.into_iter().find(|p| p.name == "relay"); + assert!(p.is_some(), "{}", b.text()); + assert!( + b.text().contains(&"b".repeat(64)), + "B 的控制面钥匙是它自己的" + ); + // 来源记为同步 + let mut saw_sync = false; + let mut saw_status = false; + while let Ok(ev) = events.try_recv() { + match ev { + tw_api::Event::ConfigReloaded { + origin: tw_api::ConfigOrigin::Sync, + .. + } => saw_sync = true, + tw_api::Event::SyncStatus { view, .. } => { + saw_status = true; + assert_eq!(view.device.unwrap().name, "MacBook Air"); + } + _ => {} + } + } + assert!(saw_sync, "应用别的设备的版本,来源是 sync"); + assert!(saw_status, "状态变了要发 sync_status"); + + // 用同步数据加入:B 的网关密钥就是 A 的那一把 + assert_eq!(b.config().clients[0].key, "tw-default-a"); + // B 换了网关密钥的值:A 那边发 gateway_keys_changed + let mut a_events = a.state.bus().subscribe(); + b.edit(|t| t.replace("tw-default-a", "tw-default-rotated")) + .await; + b.ok("POST", "/sync/now", Value::Null).await; + a.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(a.config().clients[0].key, "tw-default-rotated"); + let mut names = None; + while let Ok(ev) = a_events.try_recv() { + if let tw_api::Event::GatewayKeysChanged { names: n, .. } = ev { + names = Some(n); + } + } + assert_eq!(names, Some(vec!["default".to_string()])); +} + +#[tokio::test] +async fn a_conflict_is_listed_and_resolved_through_the_endpoints() { + let loc = tempfile::tempdir().unwrap(); + let (a, b) = pair(loc.path()).await; + a.edit(|t| add_provider(t, "relay", "https://relay.example", "sk-base")) + .await; + a.ok("POST", "/sync/now", Value::Null).await; + b.ok("POST", "/sync/now", Value::Null).await; + a.edit(|t| t.replace("sk-base", "sk-a")).await; + b.edit(|t| t.replace("sk-base", "sk-b")).await; + a.ok("POST", "/sync/now", Value::Null).await; + let v = b.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["state"], "conflict", "{v}"); + assert_eq!(v["conflicts"], 1); + let list = b.ok("GET", "/sync/conflicts", Value::Null).await; + let c = &list[0]; + assert_eq!(c["item"], json!({ "kind": "provider", "name": "relay" })); + assert_eq!(c["units"], json!(["conn"])); + assert_eq!(c["local"]["device"], "MacBook Air"); + assert_eq!(c["remote"]["device"], "Mac mini"); + assert_eq!(c["both_allowed"], true); + assert_eq!(c["diff"][0]["unit"], "conn"); + assert_eq!(c["diff"][0]["local"]["key"], "sk-b"); + assert_eq!(c["diff"][0]["remote"]["key"], "sk-a"); + // A 看到的是同一处,本机那一边是 A + a.ok("POST", "/sync/now", Value::Null).await; + let la = a.ok("GET", "/sync/conflicts", Value::Null).await; + assert_eq!(la[0]["id"], c["id"]); + assert_eq!(la[0]["local"]["device"], "Mac mini"); + + let v = b + .ok( + "POST", + "/sync/conflicts", + json!({ "decisions": [{ "id": c["id"], "take": "remote" }] }), + ) + .await; + assert_eq!(v["conflicts"], 0, "{v}"); + let key = |d: &Dev| { + d.config() + .providers + .into_iter() + .find(|p| p.name == "relay") + .unwrap() + .key + .unwrap() + .raw() + .to_string() + }; + assert_eq!(key(&b), "sk-a"); + let v = a.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["conflicts"], 0, "A 上的冲突随之清除:{v}"); + assert_eq!(key(&a), "sk-a"); +} + +#[tokio::test] +async fn joining_with_a_configuration_merges_by_the_import_rules() { + let loc = tempfile::tempdir().unwrap(); + let a = Dev::new('a'); + a.edit(|t| add_provider(t, "shared", "https://a.example", "sk-a")) + .await; + a.ok( + "POST", + "/sync/enable", + json!({ "target": folder(loc.path()), "encrypt": false, "device_name": "A", "join": "use_remote" }), + ) + .await; + let b = Dev::new('b'); + b.edit(|t| { + let t = add_provider(t, "shared", "https://b.example", "sk-b"); + add_provider(&t, "mine", "https://mine.example", "sk-m") + }) + .await; + let preview = b + .ok( + "POST", + "/sync/enable/preview", + json!({ "target": folder(loc.path()) }), + ) + .await; + assert_eq!(preview["initial"], false); + let row = preview["rows"] + .as_array() + .unwrap() + .iter() + .find(|r| r["item"]["name"] == "shared") + .unwrap() + .clone(); + assert_eq!(row["status"], "conflict"); + assert_eq!(row["suggested_name"], "shared (2)"); + // 冲突没选:不收 + let (s, v) = b + .call( + "POST", + "/sync/enable", + json!({ "target": folder(loc.path()), "encrypt": false, "device_name": "B", "join": "merge" }), + ) + .await; + assert_eq!(s, StatusCode::BAD_REQUEST, "{v}"); + assert_eq!(v["code"], "sync.join_undecided"); + b.ok( + "POST", + "/sync/enable", + json!({ + "target": folder(loc.path()), "encrypt": false, "device_name": "B", "join": "merge", + "decisions": [ + { "item": { "kind": "provider", "name": "shared" }, "action": "rename", "new_name": "shared (A)" }, + // 两边各自生成的默认密钥值不同:留本机的 + { "item": { "kind": "key", "name": "default" }, "action": "skip" }, + ], + }), + ) + .await; + let names = |d: &Dev| { + let mut v: Vec = d.config().providers.into_iter().map(|p| p.name).collect(); + v.sort(); + v + }; + assert_eq!(names(&b), ["mine", "shared", "shared (A)"]); + a.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(names(&a), ["mine", "shared", "shared (A)"], "{}", a.text()); + assert_eq!( + a.config().clients[0].key, + "tw-default-b", + "选了留本机的那一把,A 跟着换" + ); +} + +#[tokio::test] +async fn an_excluded_upstream_is_marked_and_stays_local() { + let loc = tempfile::tempdir().unwrap(); + let (a, b) = pair(loc.path()).await; + a.edit(|t| add_provider(t, "ollama", "http://127.0.0.1:11434", "none")) + .await; + a.ok("PUT", "/sync/settings", json!({ "excluded": ["ollama"] })) + .await; + let v = a.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["excluded"], json!(["ollama"])); + b.ok("POST", "/sync/now", Value::Null).await; + assert!(b.config().providers.is_empty(), "{}", b.text()); + let ov = a.ok("GET", "/overview", Value::Null).await; + assert_eq!(ov["providers"][0]["sync_excluded"], true); + // 换成另一个同步位置不在这里改 + let other = tempfile::tempdir().unwrap(); + let (s, v) = a + .call( + "PUT", + "/sync/settings", + json!({ "target": folder(other.path()) }), + ) + .await; + assert_eq!(s, StatusCode::BAD_REQUEST); + assert_eq!(v["code"], "sync.place_changed"); +} + +#[tokio::test] +async fn safe_mode_pauses_sync() { + let loc = tempfile::tempdir().unwrap(); + let (a, _b) = pair(loc.path()).await; + tw_control::sync::start(&a.state, true); + a.edit(|t| add_provider(t, "relay", "https://relay.example", "sk")) + .await; + let v = a.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["state"], "paused"); + let names: Vec = std::fs::read_dir(loc.path().join("versions")) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().to_string()) + .collect(); + assert_eq!(names.len(), 2, "暂停时不上传:{names:?}"); +} + +#[tokio::test] +async fn turning_sync_off_keeps_the_configuration_and_purging_empties_the_location() { + let loc = tempfile::tempdir().unwrap(); + let (a, b) = pair(loc.path()).await; + a.edit(|t| add_provider(t, "relay", "https://relay.example", "sk")) + .await; + a.ok("POST", "/sync/now", Value::Null).await; + a.ok("POST", "/sync/disable", json!({ "purge": true })) + .await; + assert!(a.config().sync.is_none()); + assert!(!a.config().providers.is_empty(), "本机保留当前配置"); + assert!(!a.dir.path().join("sync").exists(), "本机的同步状态删掉了"); + assert!(!loc.path().join("keyring").exists()); + // B 发现同步位置空了:停下,说一声 + let v = b.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["state"], "off", "{v}"); + assert_eq!(v["error"]["code"], "sync.emptied"); + assert!(b.config().sync.is_none()); +} + +/// 调度:本机改了配置,10 秒合并之后上传;另一台的文件夹监听一收到就合并 +#[tokio::test] +async fn a_local_change_is_uploaded_after_it_settles_and_the_folder_watcher_picks_it_up() { + let loc = tempfile::tempdir().unwrap(); + let (a, b) = pair(loc.path()).await; + tw_control::sync::start(&a.state, false); + tw_control::sync::start(&b.state, false); + // 启动那一轮先跑完 + tokio::time::sleep(Duration::from_millis(500)).await; + a.edit(|t| add_provider(t, "first", "https://first.example", "sk-1")) + .await; + a.edit(|t| add_provider(t, "second", "https://second.example", "sk-2")) + .await; + let started = std::time::Instant::now(); + loop { + let names: Vec = b.config().providers.into_iter().map(|p| p.name).collect(); + if names.len() == 2 { + break; + } + assert!( + started.elapsed() < Duration::from_secs(40), + "40 秒内没收到:{names:?}" + ); + tokio::time::sleep(Duration::from_millis(200)).await; + } + assert!( + started.elapsed() >= tw_control::sync::SETTLE - Duration::from_millis(500), + "改动要稳下来才上传" + ); + // 两次改动合成了一个版本:开启、加入、这一次 + let names: Vec = std::fs::read_dir(loc.path().join("versions")) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().to_string()) + .collect(); + assert!(names.len() <= 2, "{names:?}"); + // A 那一轮的尾巴(写设备记录)可能还没走完:不该一直停在「正在同步」 + let mut v = a.ok("GET", "/sync", Value::Null).await; + for _ in 0..50 { + if v["state"] != "syncing" { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + v = a.ok("GET", "/sync", Value::Null).await; + } + assert_eq!(v["state"], "idle", "{v}"); +} + +/// 不加密的同步:经同步新增的插件在本设备停用着,要在本设备启用;上游的地址经同步改了要 +/// 标出来(改之前的地址、哪台设备) +#[tokio::test] +async fn without_encryption_arriving_plugins_start_disabled_and_address_changes_are_flagged() { + let loc = tempfile::tempdir().unwrap(); + let (a, b) = (Dev::new('a'), Dev::new('b')); + a.edit(|t| add_provider(t, "relay", "https://relay.example", "sk")) + .await; + a.ok( + "POST", + "/sync/enable", + json!({ "target": folder(loc.path()), "encrypt": false, "device_name": "Mac mini", "join": "use_remote" }), + ) + .await; + b.ok( + "POST", + "/sync/enable", + json!({ "target": folder(loc.path()), "encrypt": false, "device_name": "MacBook Air", "join": "use_remote" }), + ) + .await; + let src = tw_gateway::plugin::fake::source( + json!({ "name": "Shout", "api": 1, "permissions": ["reply.text"] }), + &["onReplyText"], + ); + let ov = a.ok("GET", "/overview", Value::Null).await; + a.ok( + "POST", + "/plugins", + json!({ "source": src, "enabled": true, "base_version": ov["config_version"] }), + ) + .await; + a.edit(|t| t.replace("https://relay.example", "https://moved.example")) + .await; + a.ok("POST", "/sync/now", Value::Null).await; + b.ok("POST", "/sync/now", Value::Null).await; + let bp = b.config().plugins; + assert_eq!(bp.len(), 1, "{}", b.text()); + assert!( + !bp[0].enabled, + "经不加密的同步新来的插件,在这台设备上停用着" + ); + assert!(a.config().plugins[0].enabled, "A 上照旧开着"); + let ov = b.ok("GET", "/overview", Value::Null).await; + let relay = ov["providers"] + .as_array() + .unwrap() + .iter() + .find(|p| p["name"] == "relay") + .unwrap() + .clone(); + assert_eq!(relay["base_url"], "https://moved.example"); + assert_eq!(relay["sync_url_changed"]["from"], "https://relay.example"); + assert_eq!(relay["sync_url_changed"]["device"], "Mac mini"); + // 再同步几轮:B 的停用不会传回去 + b.ok("POST", "/sync/now", Value::Null).await; + a.ok("POST", "/sync/now", Value::Null).await; + assert!(a.config().plugins[0].enabled); + // B 在本机启用:之后就和别的设备一样了 + let id = bp[0].id.clone(); + let cur = b.ok("GET", "/config", Value::Null).await; + let text = cur["text"] + .as_str() + .unwrap() + .replace("enabled: false", "enabled: true"); + b.ok( + "PUT", + "/config", + json!({ "base_version": cur["version"], "text": text }), + ) + .await; + b.ok("POST", "/sync/now", Value::Null).await; + a.ok("POST", "/sync/now", Value::Null).await; + assert!(b.config().plugins.iter().any(|p| p.id == id && p.enabled)); + assert!(a.config().plugins[0].enabled); +} + +/// 安全模式:配置读不进来,在服务的是一份临时配置。同步不自动跑,状态是 `paused`;同步位置上 +/// 的版本照样列得出来(出错页的「从备份恢复」要用) +#[tokio::test] +async fn in_safe_mode_the_sync_location_still_lists_its_versions() { + let loc = tempfile::tempdir().unwrap(); + let (a, _b) = pair(loc.path()).await; + let path = a.dir.path().join("config.yaml"); + let broken = format!("{}a_field_from_nowhere: 1\n", a.text()); + std::fs::write(&path, &broken).unwrap(); + let (cfg, rejected) = tw_config::stand_in(&broken).expect("钥匙还在,顶得上"); + let gw = tw_gateway::AppState::new(cfg).unwrap(); + let bus = gw.bus.clone(); + let state = ControlState { + shutdown: Default::default(), + remote: Default::default(), + cfg: Arc::new(ConfigManager::new(path, gw.clone(), bus).standing_in(&rejected)), + gateway: gw, + store: None, + started: std::time::Instant::now(), + price_updater: Default::default(), + chatgpt: Default::default(), + zai: Default::default(), + }; + let sync = tw_control::sync::start(&state, true); + let v = sync.view(); + assert_eq!(v.state, tw_api::SyncState::Paused); + assert!(v.target.is_some(), "同步位置照原文里的"); + let versions = sync.remote_versions().await; + assert_eq!(versions.len(), 2, "{versions:?}"); + assert!(versions.iter().any(|v| v.device_name == "Mac mini")); + let pack = sync + .remote_pack(&versions[0].content_id) + .await + .unwrap() + .expect("按内容标识取得到"); + assert!(!pack.config_yaml.contains("listen"), "同步版本不带本机设置"); +} + +/// 备份列表里有同步位置上的版本(本机也有的同一份合成一行,只在同步位置上的是 +/// `sync-`);经同步应用的本机版本记着「同步自」哪台设备;恢复预览说得出会同步到 +/// 几台别的设备;接管的客户端在用的密钥(`PUT /pinned-keys`)不被别的设备删掉 +#[tokio::test] +async fn backups_see_the_sync_location_and_pinned_keys_survive() { + let loc = tempfile::tempdir().unwrap(); + let (a, b) = pair(loc.path()).await; + a.edit(|t| { + let t = add_provider(t, "relay", "https://relay.example", "sk"); + t.replace( + "clients:\n", + "clients:\n - name: codex\n key: tw-codex-0002\n", + ) + }) + .await; + a.ok("POST", "/sync/now", Value::Null).await; + b.ok("POST", "/sync/now", Value::Null).await; + + let list = b.ok("GET", "/backups", Value::Null).await; + let rows = list.as_array().unwrap(); + let synced = rows + .iter() + .find(|r| r["origin"] == "sync" && r["local"] == true) + .unwrap_or_else(|| panic!("经同步应用的那一版:{list}")); + assert_eq!(synced["device"], "Mac mini", "同步自哪台设备:{synced}"); + assert_eq!(synced["sync"], true, "同步位置上也有这一份:{synced}"); + assert_eq!(synced["current"], true); + // A 那边看:同步位置上只有、本机没有的版本(B 加入时上传的那一版)是 sync- + let la = a.ok("GET", "/backups", Value::Null).await; + let remote_only = la + .as_array() + .unwrap() + .iter() + .find(|r| r["local"] == false) + .cloned(); + if let Some(r) = remote_only { + assert!(r["id"].as_str().unwrap().starts_with("sync-"), "{r}"); + let file = a + .ok("POST", "/backups/file", json!({ "id": r["id"] })) + .await; + assert!(file["data_b64"].as_str().is_some_and(|d| !d.is_empty())); + } + let mine = la + .as_array() + .unwrap() + .iter() + .find(|r| r["local"] == true) + .unwrap() + .clone(); + let preview = a + .ok("POST", "/backups/preview", json!({ "id": mine["id"] })) + .await; + assert_eq!( + preview["sync_devices"], 1, + "恢复之后会同步到另一台:{preview}" + ); + + // B 上接管的客户端在用 codex;A 删了它:B 不删,作为冲突列出 + b.ok("PUT", "/pinned-keys", json!({ "names": ["codex"] })) + .await; + a.edit(|t| t.replace(" - name: codex\n key: tw-codex-0002\n", "")) + .await; + a.ok("POST", "/sync/now", Value::Null).await; + let v = b.ok("POST", "/sync/now", Value::Null).await; + assert!( + b.config().clients.iter().any(|c| c.name == "codex"), + "{}", + b.text() + ); + assert_eq!(v["conflicts"], 1, "{v}"); + let c = b.ok("GET", "/sync/conflicts", Value::Null).await; + assert_eq!(c[0]["reason"], "key_in_use"); + // 关掉同步:备份列表不再看同步位置 + a.ok("POST", "/sync/disable", json!({ "purge": false })) + .await; + let la = a.ok("GET", "/backups", Value::Null).await; + assert!( + la.as_array().unwrap().iter().all(|r| r["sync"] == false), + "{la}" + ); +} diff --git a/crates/tw-gateway/tests/passthrough.rs b/crates/tw-gateway/tests/passthrough.rs index 58e7193f..e55726b4 100644 --- a/crates/tw-gateway/tests/passthrough.rs +++ b/crates/tw-gateway/tests/passthrough.rs @@ -424,6 +424,7 @@ async fn a_rule_sends_opus_to_one_upstream_and_everything_else_to_another() { default_route: None, default_key: None, plugins: Vec::new(), + sync: None, version: 1, listen: Listen::default(), clients: vec![Client { diff --git a/crates/tw-observe/src/bus.rs b/crates/tw-observe/src/bus.rs index 840b791d..8925909e 100644 --- a/crates/tw-observe/src/bus.rs +++ b/crates/tw-observe/src/bus.rs @@ -127,6 +127,8 @@ fn about_the_request(ev: &tw_api::Event) -> bool { | E::ConfigRejected { .. } | E::GatewayKeysChanged { .. } | E::ClientPrefsChanged { .. } + // 同步的状态:说的是配置,不是请求 + | E::SyncStatus { .. } | E::EventsDropped { .. } => false, } } diff --git a/crates/tw-pack/src/backup.rs b/crates/tw-pack/src/backup.rs index c954f17e..0c6c7292 100644 --- a/crates/tw-pack/src/backup.rs +++ b/crates/tw-pack/src/backup.rs @@ -207,6 +207,25 @@ fn save( }) } +/// 同步应用的那一版是从哪台设备来的:记在这一版的 manifest 里(`sync.device_name`),备份列表 +/// 里写「同步自 MacBook Air」。**只改这一个字段**,别的字节照旧 +pub fn note_device(b: &Backup, device: &str) -> Result<(), BackupError> { + let mut pack = open(b)?; + pack.manifest.sync = Some(serde_json::json!({ "device_name": device })); + let bytes = crate::write(&pack, Seal::None).map_err(BackupError::Pack)?; + disk::write_private(&b.path, &bytes).map_err(io(&b.path)) +} + +/// 一版是从哪台设备同步来的([`note_device`] 记下的) +pub fn device_of(pack: &Pack) -> Option { + pack.manifest + .sync + .as_ref()? + .get("device_name")? + .as_str() + .map(str::to_string) +} + /// 超出 [`KEEP`] 的老版本删掉,以前的修改记录(`*.yaml`)也删掉 fn prune(config_path: &Path) -> Result<(), BackupError> { let all = list(config_path)?; @@ -375,6 +394,22 @@ mod tests { ); } + /// 经同步应用的那一版记着来源设备,别的照旧 + #[test] + fn a_synced_version_remembers_which_device_it_came_from() { + let d = tempfile::tempdir().unwrap(); + let p = d.path().join("config.yaml"); + let text = "version: 1\nclients:\n - name: default\n key: tw-k\n"; + std::fs::write(&p, text).unwrap(); + let b = record(&p, text, Origin::Sync).unwrap().unwrap(); + assert_eq!(device_of(&open(&b).unwrap()), None); + note_device(&b, "MacBook Air").unwrap(); + let back = open(&b).unwrap(); + assert_eq!(device_of(&back).as_deref(), Some("MacBook Air")); + assert_eq!(back.config_yaml, open(&b).unwrap().config_yaml); + assert_eq!(list(&p).unwrap().len(), 1, "不多出一版"); + } + #[test] fn a_rotation_leaves_no_version_and_does_not_count_as_a_change() { let (_d, p) = setup(); diff --git a/crates/tw-secret/src/mask.rs b/crates/tw-secret/src/mask.rs index bed0eceb..84583aba 100644 --- a/crates/tw-secret/src/mask.rs +++ b/crates/tw-secret/src/mask.rs @@ -280,6 +280,8 @@ fn is_secret_name(name: &str) -> bool { "access_key_id", "secret_access_key", "session_token", + // 同步口令(`sync:` 那一段):解得开同步位置上的每一个版本 + "passphrase", ]; NAMES.contains(&name) } @@ -754,6 +756,23 @@ mod body_tests { assert!(!mask_body("the key is ASIAIOSFODNN7EXAMPLE here").contains("IOSFODNN7")); } + /// 同步那一节:口令、WebDAV 的密码、S3 的密钥都不能进诊断包 + #[test] + fn the_sync_sections_secrets_are_masked() { + let cfg = "sync:\n device_id: 3f9a2c1e-0b1d\n device_name: Mac mini\n encrypt: true\n passphrase: horse battery staple correct\n target:\n type: webdav\n url: https://dav.example.com/tw\n user: me\n pass: app-password-value\n"; + let out = mask_config_yaml(cfg); + for secret in ["battery staple", "app-password-value"] { + assert!(!out.contains(secret), "{secret} leaked: {out}"); + } + assert!(out.contains("device_name: Mac mini"), "{out}"); + let s3 = "sync:\n target:\n type: s3\n access_key_id: AKIDEXAMPLEVALUE1\n secret_access_key: plainlookingsecretvalue\n"; + let out = mask_config_yaml(s3); + assert!( + !out.contains("plainlookingsecretvalue") && !out.contains("EXAMPLEVALUE1"), + "{out}" + ); + } + #[test] fn a_field_that_merely_ends_in_key_is_not_a_secret() { // `api_key_env` 之类装的是环境变量名,不是密钥。多打码的代价小, diff --git a/crates/tw-store/src/recorder.rs b/crates/tw-store/src/recorder.rs index daa14eda..e94b1dcc 100644 --- a/crates/tw-store/src/recorder.rs +++ b/crates/tw-store/src/recorder.rs @@ -699,7 +699,8 @@ impl Recorder { | Event::GatewayKeysChanged { .. } | Event::ClientPrefsChanged { .. } // 只发给事件流上掉队的那一个订阅者,从不进总线 - | Event::EventsDropped { .. } => {} + | Event::EventsDropped { .. } + | Event::SyncStatus { .. } => {} } } diff --git a/crates/tw-sync/Cargo.toml b/crates/tw-sync/Cargo.toml new file mode 100644 index 00000000..62dbc2de --- /dev/null +++ b/crates/tw-sync/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "tw-sync" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +documentation.workspace = true +readme.workspace = true +description = "Multi-device configuration sync: folder, WebDAV and S3 locations, the keyring, version naming and the three-way merge" + +[dependencies] +# `sync` 一节的形状、地址规则 +tw-config = { workspace = true } +tw-types = { workspace = true } +# 配置包的格式与加密、把一份配置拆成项目与合并单位 +tw-pack = { workspace = true } +# S3 的 SigV4 和 Bedrock 用的是同一份 +tw-bedrock = { workspace = true } +# 文件夹位置:版本一到就合并,不等定时检查 +tw-watch = { workspace = true } +reqwest = { workspace = true } +tokio = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +age = { workspace = true } +quick-xml = { workspace = true } +sha2 = { workspace = true } +rand = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } +percent-encoding = { workspace = true } +url = { workspace = true } + +[dev-dependencies] +# 假的 WebDAV、假的 S3:和 core 别处的测试一样手写在 axum 上 +axum = { workspace = true } +tempfile = "3" +# 假 WebDAV 核对 Basic 认证 +base64 = { workspace = true } +# 两台设备的测试直接读写配置 +tw-config = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time"] } diff --git a/crates/tw-sync/src/device.rs b/crates/tw-sync/src/device.rs new file mode 100644 index 00000000..86350912 --- /dev/null +++ b/crates/tw-sync/src/device.rs @@ -0,0 +1,93 @@ +//! 设备记录:`devices/<设备标识前 8 位>`,每台设备只写自己那一份。 +//! +//! 加密同步时它也加密(给版本密钥的公钥):设备名称常常就是人名(「张三的 MacBook Air」)。 + +use serde::{Deserialize, Serialize}; + +use crate::keyring::Keyring; + +/// 设备记录多久没更新就从同步位置上移除 +pub const STALE_AFTER_MS: u64 = 180 * 24 * 3600 * 1000; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DeviceRecord { + /// 完整的设备标识 + pub id: String, + pub name: String, + pub core_version: String, + /// 最后一次同步成功的时间 + pub last_seen_ms: u64, + /// 同步位置的最新版本这台设备读不了(由较新的 Core 写的),暂停着等更新 + #[serde(default)] + pub paused_update: bool, + /// 它的 core 开着远程控制端口(远程 core) + #[serde(default)] + pub remote_core: bool, +} + +impl DeviceRecord { + /// 写进同步位置的样子:加密同步时加密给版本密钥 + pub fn seal(&self, keyring: Option<&Keyring>) -> Vec { + let json = serde_json::to_vec(self).expect("plain data serializes"); + match keyring { + Some(k) => k.encrypt(&json), + None => json, + } + } + + /// 读回来。解不开、读不懂的是 `None`(别的设备刚改了加密设置、还没传完、被人改坏了) + pub fn open(bytes: &[u8], keyring: Option<&Keyring>) -> Option { + let json = if crate::keyring::is_age(bytes) { + keyring?.decrypt(bytes)? + } else { + bytes.to_vec() + }; + serde_json::from_slice(&json).ok() + } + + pub fn is_stale(&self, now_ms: u64) -> bool { + now_ms.saturating_sub(self.last_seen_ms) > STALE_AFTER_MS + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn rec() -> DeviceRecord { + DeviceRecord { + id: "3f9a2c1e-0b1d-4e2f-9a3b-5c6d7e8f9a0b".into(), + name: "张三的 MacBook Air".into(), + core_version: "0.77.0".into(), + last_seen_ms: 1_000, + paused_update: false, + remote_core: false, + } + } + + #[test] + fn a_record_reads_back_plain_or_sealed() { + let r = rec(); + assert_eq!(DeviceRecord::open(&r.seal(None), None), Some(r.clone())); + let k = Keyring::generate(); + let sealed = r.seal(Some(&k)); + assert!( + !String::from_utf8_lossy(&sealed).contains("张三"), + "加密时名字不能是明文" + ); + assert_eq!(DeviceRecord::open(&sealed, Some(&k)), Some(r.clone())); + assert_eq!(DeviceRecord::open(&sealed, None), None); + assert_eq!( + DeviceRecord::open(&sealed, Some(&Keyring::generate())), + None + ); + assert_eq!(DeviceRecord::open(b"{not json", None), None); + } + + #[test] + fn a_record_goes_stale_after_half_a_year() { + let r = rec(); + assert!(!r.is_stale(r.last_seen_ms + STALE_AFTER_MS)); + assert!(r.is_stale(r.last_seen_ms + STALE_AFTER_MS + 1)); + } +} diff --git a/crates/tw-sync/src/engine.rs b/crates/tw-sync/src/engine.rs new file mode 100644 index 00000000..d2828f86 --- /dev/null +++ b/crates/tw-sync/src/engine.rs @@ -0,0 +1,2015 @@ +//! 一轮同步:看同步位置上有什么新版本,和本机合并,该应用的应用、该上传的上传。 +//! +//! # 一轮里做什么 +//! +//! 1. 列出 `versions/`,读出每个版本(加密的用版本密钥解开)。读不出来的当作尚未到达:最新 +//! 那一层有没到的就等下一轮,不拿一半去合并。 +//! 2. 找出所有的**头**(没有别的版本依据它的版本)。不止一个时(两台设备同时上传、云盘晚到 +//! 的版本)先以它们共同的上一版本为基准两两合并,得到「同步位置的最新」。 +//! 3. 以本机基准(上次同步的那一版)为基准,和本机当前配置三方合并([`crate::merge`])。 +//! 冲突照常往下走:本机留本机的,上传的留同步位置的,记进冲突记录。 +//! 4. 本机的结果和现在不一样就应用(经控制面的写入流程,来源记为同步);上传的结果和同步 +//! 位置上的不一样(或者合并了几个头、冲突记录变了)就上传一个新版本,然后删掉更早的。 +//! +//! # 冲突记录随版本走 +//! +//! 一处冲突在被处理之前,**两边各留各的**:参与的每台设备用自己的内容,同步版本里是先上传 +//! 的那一边([`Conflict::shared`])。任一设备处理之后,冲突从记录里去掉、结果上传,别的设备 +//! 看到记录没了,就照同步版本改成处理之后的样子。 + +use std::collections::{BTreeMap, BTreeSet}; +use std::future::Future; +use std::path::Path; +use std::sync::Arc; + +use serde_json::Value; +use sha2::{Digest, Sha256}; +use tw_pack::model::{self, Item, ItemId, ItemKind, Snapshot, unit}; +use tw_pack::{Pack, PackError, PackKind, PackOrigin, Seal, Unseal}; +use tw_types::{Msg, msg}; + +use crate::device::DeviceRecord; +use crate::keyring::{Keyring, KeyringError}; +use crate::layout::{self, DEVICES, KEYRING, Listed, VERSIONS, VersionName}; +use crate::local::{Local, State, UrlChange}; +use crate::merge::{self, Objects}; +use crate::order; +use crate::record::{self, Conflict, Meta, Reason, Side}; +use crate::store::{Store, StoreError}; + +/// 设备记录多久没写就写一次(最后同步时间),即便这一轮什么都没变 +const DEVICE_RECORD_EVERY_MS: u64 = 6 * 3600 * 1000; + +/// 本机这一边:配置从哪儿读、往哪儿写。控制面实现它(`tw-control` 的 `sync`),测试里换成 +/// 两个目录里的两份配置 +pub trait Host: Send + Sync { + /// 本机此刻的完整配置包(含本机设置、插件、通用偏好) + fn current(&self) -> Result; + /// 照这个包写本机配置:经现有的写入流程(校验、热更新、记一个本机版本),来源记为同步。 + /// `base_version` 是 [`Current::version`]:读完之后配置被别人改过就是 [`ApplyError::Stale`] + fn apply( + &self, + pack: Pack, + base_version: &str, + from: &str, + ) -> impl Future> + Send; + /// 本机已接管的客户端正在用的网关密钥(名字) + fn pinned_keys(&self) -> BTreeSet; + /// 本机备份里内容标识是这个的那一版 + fn backup(&self, content_id: &str) -> Option; +} + +/// 本机此刻的配置 +pub struct Current { + pub pack: Pack, + /// 配置文件的版本号(`tw_config::version_of`),写的时候拿它核对 + pub version: String, + /// 配置最后改动的时间(冲突里「本机改的时间」) + pub changed_ms: u64, +} + +#[derive(Debug)] +pub enum ApplyError { + /// 读完之后配置被改过了:下一轮再来 + Stale, + Rejected(Msg), +} + +/// 这台设备在同步里的设置 +#[derive(Debug, Clone)] +pub struct Settings { + pub device_id: String, + pub device_name: String, + pub encrypt: bool, + pub passphrase: Option, + /// 不同步的上游 + pub exclude: BTreeSet, + /// 同步位置的指纹([`crate::local::place`])。本机的同步状态是对着它的:换了位置就作废 + pub place: String, + /// 这台设备的 core 开着远程控制端口:设备记录里写上(界面标「远程 core」) + pub remote_core: bool, +} + +impl Settings { + pub fn of(s: &tw_config::sync::Sync) -> Self { + Self { + device_id: s.device_id.clone(), + device_name: s.device_name.clone(), + encrypt: s.encrypt, + passphrase: s.passphrase.clone(), + exclude: s.exclude.iter().cloned().collect(), + place: crate::local::place(&s.target), + remote_core: false, + } + } +} + +/// 一轮没成 +#[derive(Debug)] +pub enum RoundError { + /// 同步位置连不上、拒绝访问…… + Store(StoreError), + /// 同步口令不对(别的设备改了口令),或者同步位置开了加密而这台设备还没有口令 + Locked(Msg), + /// 同步位置的最新版本由较新的 Core 写,本机读不了:暂停 + TooNew { core_version: String }, + /// 最新的版本还没传完(云盘还在下载):下一轮再来,不算失败 + NotArrived, + /// 同步位置被清空了(别的设备停止同步并删除了同步数据) + Emptied, + /// 本机的配置读不了、写不进去 + Local(Msg), + /// 这一轮里本机配置被改了:马上再来一轮 + Stale, +} + +impl From for RoundError { + fn from(e: StoreError) -> Self { + RoundError::Store(e) + } +} + +impl RoundError { + pub fn msg(&self) -> Msg { + match self { + RoundError::Store(e) => e.msg(), + RoundError::Locked(m) | RoundError::Local(m) => m.clone(), + RoundError::TooNew { core_version } => msg!( + "sync.too_new", version = core_version => + "another device uses a newer version (Core {version}). Update ThinkWatch Lite to \ + keep syncing" + ), + RoundError::NotArrived => msg!( + "sync.not_arrived" => + "the latest version has not finished arriving at the sync location" + ), + RoundError::Emptied => msg!( + "sync.emptied" => + "the sync location is empty: another device stopped syncing and deleted the \ + sync data" + ), + RoundError::Stale => msg!( + "sync.stale" => "the configuration changed during the sync; it will run again" + ), + } + } +} + +/// 一轮做了什么 +#[derive(Debug, Clone, Default)] +pub struct Report { + /// 上传的新版本 + pub uploaded: Option, + /// 应用了别的设备的改动:那些设备的名字 + pub applied_from: Vec, + /// 还没处理的冲突 + pub conflicts: Vec, + pub devices: Vec, + /// 同步位置此刻加没加密。和设置不一样时(别的设备关了加密)调用方改设置 + pub encrypted: bool, + /// 本机应用之后值变了、或者被删掉的网关密钥 + pub keys_changed: Vec, +} + +/// 读出来的一个版本 +#[derive(Clone)] +pub struct Version { + pub name: VersionName, + pub pack: Pack, + pub meta: Meta, + pub snap: Snapshot, + pub content: String, +} + +#[derive(Clone)] +enum Fetched { + Ok(Arc), + TooNew(String), +} + +/// 同步位置的最新(几个头合并之后) +struct Remote { + snap: Snapshot, + conflicts: Vec, + heads: Vec>, + /// 写最新那个头的设备 + device: String, + device_name: String, + at_ms: u64, +} + +/// 同步位置是不是空的、加没加密、有哪些设备(开启同步时的「测试连接」) +#[derive(Debug, Clone)] +pub struct Probe { + pub writable: Result<(), StoreError>, + pub existing: Option, +} + +#[derive(Debug, Clone)] +pub struct Existing { + pub encrypted: bool, + /// 加密时口令对不对(没给口令是 `false`) + pub unlocked: bool, + pub devices: Vec, +} + +pub struct Engine { + pub store: S, + pub host: H, + local: Local, + settings: Settings, + /// 解开了的版本密钥,和 keyring 文件的哈希(文件变了要重新解) + keyring: Option<(Keyring, [u8; 32])>, + cache: BTreeMap, + /// 上一轮结束时同步位置上的名字和本机的内容:都没变就什么都不用做 + quiet: Option<(Vec, String)>, + last_report: Report, +} + +fn now_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +fn local_err(e: impl std::fmt::Display) -> RoundError { + RoundError::Local(msg!( + "sync.local", detail = e => + "the local configuration could not be used for sync: {detail}" + )) +} + +/// 内容的 SHA-256:配置原文、插件、通用偏好(不加密的版本带着它,发现传输损坏) +fn content_sha256(p: &Pack) -> String { + let mut h = Sha256::new(); + h.update(p.config_yaml.as_bytes()); + for (id, f) in &p.plugins { + h.update(id.as_bytes()); + h.update([0]); + h.update(&f.script); + h.update([0]); + if let Some(a) = &f.approved { + h.update(a); + } + h.update([0]); + } + if let Some(c) = &p.client_prefs { + h.update(c.to_string().as_bytes()); + } + let out: [u8; 32] = h.finalize().into(); + out.iter().map(|b| format!("{b:02x}")).collect() +} + +fn objects(s: &Snapshot) -> Objects { + s.items + .iter() + .map(|(k, v)| (k.clone(), v.units.clone())) + .collect() +} + +/// 合并出来的单位拼回项目:插件的原文件从脚本一样的那一边拿 +fn rebuild(objs: &Objects, sides: &[&Snapshot]) -> Snapshot { + let mut items = BTreeMap::new(); + for (id, units) in objs { + let files = sides.iter().find_map(|s| { + let it = s.items.get(id)?; + (it.units.get(unit::SCRIPT) == units.get(unit::SCRIPT)) + .then(|| it.files.clone()) + .flatten() + }); + items.insert( + id.clone(), + Item { + units: units.clone(), + refs: Default::default(), + files, + }, + ); + } + let mut s = Snapshot { items }; + s.relink(); + s +} + +/// 一种顺序项目的名字列表 +fn order_list(s: Option<&Snapshot>, kind: ItemKind) -> Option> { + let it = s?.items.get(&ItemId::new(kind, ""))?; + match it.units.get(unit::ALL) { + Some(Value::Array(a)) => Some( + a.iter() + .filter_map(|v| v.as_str().map(str::to_string)) + .collect(), + ), + _ => None, + } +} + +fn names_of(s: &Snapshot, kind: ItemKind) -> BTreeSet { + s.of_kind(kind).map(|(id, _)| id.name.clone()).collect() +} + +fn set_order(s: &mut Snapshot, kind: ItemKind, list: Vec) { + let id = ItemId::new(kind, ""); + if list.is_empty() { + s.items.remove(&id); + return; + } + let item = s.items.entry(id).or_default(); + item.units.insert( + unit::ALL.to_string(), + Value::Array(list.into_iter().map(Value::String).collect()), + ); +} + +const ORDERS: &[(ItemKind, ItemKind)] = &[ + (ItemKind::ProviderOrder, ItemKind::Provider), + (ItemKind::RouteOrder, ItemKind::Route), + (ItemKind::PluginOrder, ItemKind::Plugin), +]; + +/// 对象里冲突的那几个单位此刻的值;对象不在是 `None` +fn side_units(s: &Snapshot, c: &Conflict) -> Option> { + s.items.get(&c.item).map(|it| c.pick(&it.units)) +} + +/// 合并新发现的一处冲突写成记录:`shared` 是同步版本那一边(快照和它的设备),`other` 是另一边 +fn found( + f: &merge::Found, + shared: (&Snapshot, Side), + other: (&Snapshot, Side), +) -> Conflict { + // 两边都改了:记冲突的那几个单位;一边删了:记整个对象 + let placeholder = (f.kind == merge::Kind::Modified).then(BTreeMap::new); + let mut c = Conflict { + id: record::new_id(), + item: f.key.clone(), + units: f.units.iter().cloned().collect(), + shared: Side { + units: placeholder.clone(), + ..shared.1 + }, + other: Side { + units: placeholder, + ..other.1 + }, + reason: None, + }; + c.shared.units = side_units(shared.0, &c); + c.other.units = side_units(other.0, &c); + c +} + +/// 把冲突的那一部分改成 `src` 里的样子 +fn force(dst: &mut Snapshot, src: &Snapshot, c: &Conflict) { + match src.items.get(&c.item) { + None => { + dst.items.remove(&c.item); + } + Some(it) if c.whole_item() => { + dst.items.insert(c.item.clone(), it.clone()); + } + Some(it) => { + let entry = dst + .items + .entry(c.item.clone()) + .or_insert_with(|| it.clone()); + for u in &c.units { + match it.units.get(u) { + Some(v) => { + entry.units.insert(u.clone(), v.clone()); + } + None => { + entry.units.remove(u); + } + } + } + if it.units.get(unit::SCRIPT) == entry.units.get(unit::SCRIPT) { + entry.files = it.files.clone(); + } + } + } +} + +/// 把冲突那一部分写成一边记录下来的值 +fn set_side(dst: &mut Snapshot, c: &Conflict, side: &Side, from: &Snapshot) { + match &side.units { + None => { + dst.items.remove(&c.item); + } + Some(units) => { + let entry = dst + .items + .entry(c.item.clone()) + .or_insert_with(|| from.items.get(&c.item).cloned().unwrap_or_default()); + for (u, v) in units { + entry.units.insert(u.clone(), v.clone()); + } + for u in &c.units { + if !units.contains_key(u) { + entry.units.remove(u); + } + } + } + } +} + +/// 处理一处冲突:采用哪一边 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Take { + /// 本机的 + Local, + /// 另一边的 + Remote, + /// 两者都保留:本机的留原名称,另一边的以「名称 (设备名)」新增 + Both, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Decision { + pub id: String, + pub take: Take, + /// 两者都保留时另一份的名字。不给就是「名称 (设备名)」,被占了往后加序号 + pub new_name: Option, +} + +/// 「两者都保留」那一份的名字。自定义安全规则的名字带着分项(`content/项目代号`),只换规则名 +fn both_name(id: &ItemId, given: Option<&str>, device_name: &str) -> String { + let (prefix, plain) = match (id.kind, id.name.split_once('/')) { + (ItemKind::SecurityRule, Some((guard, rule))) => (format!("{guard}/"), rule), + _ => (String::new(), id.name.as_str()), + }; + match given.map(str::trim).filter(|g| !g.is_empty()) { + Some(g) if g.contains('/') && id.kind == ItemKind::SecurityRule => g.to_string(), + Some(g) => format!("{prefix}{g}"), + None => format!("{prefix}{plain} ({device_name})"), + } +} + +/// 不同步的上游,连同直接或间接引用它们的项目(路由、别名、分组) +pub fn excluded(names: &BTreeSet, snaps: &[&Snapshot]) -> BTreeSet { + let mut out: BTreeSet = names + .iter() + .map(|n| ItemId::new(ItemKind::Provider, n.clone())) + .collect(); + if out.is_empty() { + return out; + } + loop { + let before = out.len(); + for s in snaps { + for (id, it) in &s.items { + if !id.kind.is_order() && it.refs.iter().any(|r| out.contains(r)) { + out.insert(id.clone()); + } + } + } + if out.len() == before { + return out; + } + } +} + +/// 一个上游的地址(`conn.base_url`) +fn base_url(it: &Item) -> Option<&str> { + it.units.get(unit::CONN)?.get("base_url")?.as_str() +} + +/// 一把网关密钥的值 +fn key_value(it: &Item) -> Option<&Value> { + it.units.get(unit::ALL)?.get("key") +} + +impl Engine { + /// `data_dir` 是配置文件所在的目录:本机的同步状态放在它底下的 `sync/` + pub fn new(store: S, host: H, data_dir: &Path, settings: Settings) -> Self { + Self { + store, + host, + local: Local::new(data_dir), + settings, + keyring: None, + cache: BTreeMap::new(), + quiet: None, + last_report: Report::default(), + } + } + + pub fn settings(&self) -> &Settings { + &self.settings + } + + pub fn set_settings(&mut self, s: Settings) { + if s.passphrase != self.settings.passphrase || s.encrypt != self.settings.encrypt { + self.keyring = None; + self.cache.clear(); + } + self.quiet = None; + self.settings = s; + } + + pub fn local(&self) -> &Local { + &self.local + } + + pub fn state(&self) -> State { + self.local.state() + } + + /// 上一轮的结果(冲突、设备列表) + pub fn last_report(&self) -> &Report { + &self.last_report + } + + /// 下一轮不走捷径:哪怕同步位置和本机都没变,也从头看一遍 + pub fn invalidate(&mut self) { + self.quiet = None; + } + + // ─────────────────────────────────────────────── 读 + + /// 拿到版本密钥。没加密是 `Ok(None)` + async fn keyring(&mut self) -> Result, RoundError> { + let Some(bytes) = self.store.get(KEYRING).await? else { + self.keyring = None; + return Ok(None); + }; + let hash: [u8; 32] = Sha256::digest(&bytes).into(); + if self.keyring.as_ref().is_some_and(|(_, h)| *h == hash) { + return Ok(self.keyring.as_ref().map(|(k, _)| k)); + } + let Some(pass) = self.settings.passphrase.clone().filter(|p| !p.is_empty()) else { + return Err(RoundError::Locked(msg!( + "sync.locked_new" => + "the sync location is now encrypted. Enter the sync passphrase to keep syncing" + ))); + }; + let k = tokio::task::spawn_blocking(move || Keyring::open(&bytes, &pass)) + .await + .map_err(local_err)?; + match k { + Ok(k) => { + self.cache.clear(); + self.keyring = Some((k, hash)); + Ok(self.keyring.as_ref().map(|(k, _)| k)) + } + Err(KeyringError::WrongPassphrase) => Err(RoundError::Locked(msg!( + "sync.passphrase_changed" => + "the sync passphrase was changed on another device. Enter the new passphrase" + ))), + Err(KeyringError::TooNew) => Err(RoundError::TooNew { + core_version: String::new(), + }), + Err(KeyringError::Corrupt) => Err(RoundError::NotArrived), + } + } + + /// 读一个版本。读不出来(没传完、坏了、不是这把密钥加密的)是 `None` + async fn fetch(&mut self, name: &VersionName) -> Result, RoundError> { + if let Some(f) = self.cache.get(name) { + return Ok(Some(f.clone())); + } + let Some(bytes) = self.store.get(&name.path()).await? else { + return Ok(None); + }; + let identity = self.keyring.as_ref().map(|(k, _)| k.identity().clone()); + let decoded = tokio::task::spawn_blocking(move || { + let unseal = match &identity { + Some(id) => Unseal::Identity(id), + None => Unseal::None, + }; + let pack = tw_pack::read(&bytes, unseal)?; + let snap = model::snapshot(&pack); + Ok::<_, PackError>((pack, snap)) + }) + .await + .map_err(local_err)?; + let fetched = match decoded { + Err(PackError::TooNew { core_version, .. }) => Fetched::TooNew(core_version), + Err(_) => return Ok(None), + Ok((pack, Err(e))) if e.is_too_new() => Fetched::TooNew(pack.manifest.core_version), + Ok((_, Err(_))) => return Ok(None), + Ok((pack, Ok(snap))) => { + let Some(meta) = pack + .manifest + .sync + .clone() + .and_then(|v| serde_json::from_value::(v).ok()) + else { + return Ok(None); + }; + // 名字和内容对不上(被人挪过、改过名),或者不加密的版本校验和对不上:当作坏了 + if meta.seq != name.seq || layout::device_tag(&meta.device) != name.device { + return Ok(None); + } + if !pack.manifest.encrypted + && meta.sha256.as_deref() != Some(content_sha256(&pack).as_str()) + { + return Ok(None); + } + let content = pack.content_id(); + Fetched::Ok(Arc::new(Version { + name: name.clone(), + pack, + meta, + snap, + content, + })) + } + }; + self.cache.insert(name.clone(), fetched.clone()); + Ok(Some(fetched)) + } + + fn base(&self) -> Option { + let bytes = self.local.base()?; + let pack = tw_pack::read(&bytes, Unseal::None).ok()?; + let meta: Meta = serde_json::from_value(pack.manifest.sync.clone()?).ok()?; + let snap = model::snapshot(&pack).ok()?; + let name = VersionName::parse(&self.local.state().base?)?; + let content = pack.content_id(); + Some(Version { + name, + pack, + meta, + snap, + content, + }) + } + + /// 同步位置上的版本:读得出来的、读不出来的名字 + async fn versions( + &mut self, + ) -> Result< + ( + Vec, + Vec>, + Vec<(VersionName, String)>, + ), + RoundError, + > { + let mut names: Vec = self + .store + .list(VERSIONS) + .await? + .into_iter() + .filter_map(|e| VersionName::parse(&e.name)) + .collect(); + names.sort(); + self.cache.retain(|n, _| names.contains(n)); + let mut ok = Vec::new(); + let mut too_new = Vec::new(); + for n in &names { + match self.fetch(n).await? { + Some(Fetched::Ok(v)) => ok.push(v), + Some(Fetched::TooNew(cv)) => too_new.push((n.clone(), cv)), + None => {} + } + } + Ok((names, ok, too_new)) + } + + /// 同步位置上的设备记录。半年没更新的顺手删掉 + async fn devices(&mut self) -> Result, RoundError> { + let kr = self.keyring.as_ref().map(|(k, _)| k); + let now = now_ms(); + let mut out = Vec::new(); + for e in self.store.list(DEVICES).await? { + let path = format!("{DEVICES}/{}", e.name); + let Some(bytes) = self.store.get(&path).await? else { + continue; + }; + let Some(rec) = DeviceRecord::open(&bytes, kr) else { + continue; + }; + if rec.is_stale(now) && rec.id != self.settings.device_id { + let _ = self.store.delete(&path).await; + continue; + } + out.push(rec); + } + out.sort_by(|a, b| a.name.cmp(&b.name).then(a.id.cmp(&b.id))); + Ok(out) + } + + async fn write_device(&mut self, paused: bool) -> Result<(), RoundError> { + let rec = DeviceRecord { + id: self.settings.device_id.clone(), + name: self.settings.device_name.clone(), + core_version: tw_pack::CORE_VERSION.to_string(), + last_seen_ms: now_ms(), + paused_update: paused, + remote_core: self.settings.remote_core, + }; + let kr = self.keyring.as_ref().map(|(k, _)| k); + self.store + .put( + &layout::device_path(&self.settings.device_id), + &rec.seal(kr), + ) + .await?; + let mut st = self.local.state(); + st.device_record_ms = rec.last_seen_ms; + st.device_record_paused = paused; + self.local.save_state(&st).map_err(RoundError::Local)?; + Ok(()) + } + + // ─────────────────────────────────────────────── 合并几个头 + + /// `a`、`b` 共同的上一版本 + fn common_ancestor( + &self, + a: &[Arc], + b: &Version, + all: &[Arc], + base: Option<&Version>, + ) -> Option { + // 版本标识 → (上一版本, 它们的内容标识) + let mut parents: BTreeMap, Vec)> = BTreeMap::new(); + for v in all.iter().map(|v| v.as_ref()).chain(base) { + parents.insert( + v.name.id(), + (v.meta.parents.clone(), v.meta.parent_content.clone()), + ); + } + let mut content_of: BTreeMap = BTreeMap::new(); + for (ps, cs) in parents.values() { + for (p, c) in ps.iter().zip(cs) { + content_of.insert(p.clone(), c.clone()); + } + } + let ancestors = |start: &str| -> BTreeSet { + let mut out = BTreeSet::new(); + let mut todo = vec![start.to_string()]; + while let Some(id) = todo.pop() { + if !out.insert(id.clone()) { + continue; + } + if let Some((ps, _)) = parents.get(&id) { + todo.extend(ps.iter().cloned()); + } + } + out + }; + let left: BTreeSet = a.iter().flat_map(|v| ancestors(&v.name.id())).collect(); + let right = ancestors(&b.name.id()); + let mut common: Vec = left + .intersection(&right) + .filter_map(|id| VersionName::parse(id)) + .collect(); + common.sort(); + let id = common.pop()?.id(); + if let Some(v) = all.iter().find(|v| v.name.id() == id) { + return Some((**v).clone()); + } + if let Some(b) = base.filter(|b| b.name.id() == id) { + return Some(b.clone()); + } + let content = content_of.get(&id)?; + let pack = self.host.backup(content)?; + let snap = model::snapshot(&pack).ok()?; + Some(Version { + name: VersionName::parse(&id)?, + meta: Meta::default(), + content: content.clone(), + snap, + pack, + }) + } + + /// 几个头合并成一份 + fn fold(&self, heads: &[Arc], all: &[Arc], base: Option<&Version>) -> Remote { + let first = &heads[0]; + let mut acc = Remote { + snap: first.snap.clone(), + conflicts: first.meta.conflicts.clone(), + heads: vec![first.clone()], + device: first.meta.device.clone(), + device_name: first.meta.device_name.clone(), + at_ms: first.pack.manifest.created_at_ms, + }; + for h in &heads[1..] { + let anc = self.common_ancestor(&acc.heads, h, all, base); + let anc_conf: BTreeSet<&str> = anc + .as_ref() + .map(|a| a.meta.conflicts.iter().map(|c| c.id.as_str()).collect()) + .unwrap_or_default(); + // 两边的冲突记录合起来,减去在共同的上一版本里有、而某一边已经处理掉了的 + let ids_a: BTreeSet<&str> = acc.conflicts.iter().map(|c| c.id.as_str()).collect(); + let ids_b: BTreeSet<&str> = h.meta.conflicts.iter().map(|c| c.id.as_str()).collect(); + let mut open: Vec = Vec::new(); + for c in acc.conflicts.iter().chain(&h.meta.conflicts) { + let resolved = anc_conf.contains(c.id.as_str()) + && !(ids_a.contains(c.id.as_str()) && ids_b.contains(c.id.as_str())); + if !resolved && !open.iter().any(|o| o.id == c.id) { + open.push(c.clone()); + } + } + let (mut hold_items, hold_units) = record::holds(&open); + for (o, _) in ORDERS { + hold_items.insert(ItemId::new(*o, "")); + } + let base_objs = anc.as_ref().map(|a| objects(&a.snap)); + let (lo, ro) = (objects(&acc.snap), objects(&h.snap)); + let out = merge::merge(merge::Input { + base: base_objs.as_ref(), + local: &lo, + remote: &ro, + hold_items: &hold_items, + hold_units: &hold_units, + }); + let mut snap = rebuild(&out.shared, &[&h.snap, &acc.snap]); + for (o, k) in ORDERS { + let m = order::merge( + order_list(anc.as_ref().map(|a| &a.snap), *o).as_deref(), + &order_list(Some(&acc.snap), *o).unwrap_or_default(), + &order_list(Some(&h.snap), *o).unwrap_or_default(), + &names_of(&snap, *k), + &names_of(&snap, *k), + open.iter().any(|c| c.item.kind == *o), + ); + set_order(&mut snap, *o, m.shared); + if m.conflict { + let id = ItemId::new(*o, ""); + open.push(Conflict { + id: record::new_id(), + units: vec![unit::ALL.to_string()], + shared: Side { + device: h.meta.device.clone(), + device_name: h.meta.device_name.clone(), + at_ms: h.pack.manifest.created_at_ms, + units: h.snap.items.get(&id).map(|i| i.units.clone()), + }, + other: Side { + device: acc.device.clone(), + device_name: acc.device_name.clone(), + at_ms: acc.at_ms, + units: acc.snap.items.get(&id).map(|i| i.units.clone()), + }, + item: id, + reason: None, + }); + } + } + for f in &out.conflicts { + open.push(found( + f, + ( + &h.snap, + Side { + device: h.meta.device.clone(), + device_name: h.meta.device_name.clone(), + at_ms: h.pack.manifest.created_at_ms, + units: None, + }, + ), + ( + &acc.snap, + Side { + device: acc.device.clone(), + device_name: acc.device_name.clone(), + at_ms: acc.at_ms, + units: None, + }, + ), + )); + } + // 同步版本里冲突的那一部分是「同步版本那一边」记下的值 + for c in &open { + set_side(&mut snap, c, &c.shared, &h.snap); + } + snap.relink(); + let later = h.pack.manifest.created_at_ms >= acc.at_ms; + acc = Remote { + snap, + conflicts: open, + heads: acc.heads.iter().cloned().chain([h.clone()]).collect(), + device: if later { + h.meta.device.clone() + } else { + acc.device + }, + device_name: if later { + h.meta.device_name.clone() + } else { + acc.device_name + }, + at_ms: acc.at_ms.max(h.pack.manifest.created_at_ms), + }; + } + acc + } + + // ─────────────────────────────────────────────── 写 + + /// 上传一个新版本:内容是 `shared`,依据 `parents` + async fn publish( + &mut self, + shared: &Snapshot, + seq: u64, + parents: &[Arc], + conflicts: Vec, + ) -> Result { + let name = VersionName::new(seq, &self.settings.device_id); + let empty = Pack { + manifest: tw_pack::Manifest::new(PackKind::Full, PackOrigin::Sync), + config_yaml: format!("version: {}\n", tw_config::SCHEMA_VERSION), + plugins: Default::default(), + client_prefs: None, + }; + let mut pack = + model::materialize(&empty, shared).map_err(|e| RoundError::Local(e.msg()))?; + pack.manifest = tw_pack::Manifest::new(PackKind::Full, PackOrigin::Sync); + let mut meta = Meta { + seq, + device: self.settings.device_id.clone(), + device_name: self.settings.device_name.clone(), + parents: parents.iter().map(|p| p.name.id()).collect(), + parent_content: parents.iter().map(|p| p.content.clone()).collect(), + sha256: None, + conflicts, + }; + let encrypted = self.keyring.is_some(); + if !encrypted { + meta.sha256 = Some(content_sha256(&pack)); + } + pack.manifest.sync = Some(serde_json::to_value(&meta).expect("plain data serializes")); + let bytes = { + let recipient = self.keyring.as_ref().map(|(k, _)| k.recipient()); + let seal = match &recipient { + Some(r) => Seal::Recipient(r), + None => Seal::None, + }; + tw_pack::write(&pack, seal).map_err(|e| RoundError::Local(e.msg()))? + }; + match self.store.put_new(&name.path(), &bytes).await { + Ok(()) => {} + Err(StoreError::Exists { .. }) => return Err(RoundError::Stale), + Err(e) => return Err(e.into()), + } + let plain = tw_pack::write(&pack, Seal::None).map_err(|e| RoundError::Local(e.msg()))?; + let pack = tw_pack::read(&plain, Unseal::None).map_err(|e| RoundError::Local(e.msg()))?; + let snap = model::snapshot(&pack).map_err(|e| RoundError::Local(e.msg()))?; + let content = pack.content_id(); + let v = Version { + name: name.clone(), + pack, + meta, + snap, + content, + }; + self.cache.insert(name, Fetched::Ok(Arc::new(v.clone()))); + Ok(v) + } + + fn save_base(&self, v: &Version, st: &mut State) -> Result<(), RoundError> { + let bytes = tw_pack::write(&v.pack, Seal::None).map_err(|e| RoundError::Local(e.msg()))?; + self.local.save_base(&bytes).map_err(RoundError::Local)?; + st.base = Some(v.name.id()); + Ok(()) + } + + async fn prune(&mut self, names: &[VersionName], ok: &[Arc]) { + let listed: Vec = names + .iter() + .map(|n| Listed { + name: n.clone(), + parents: ok + .iter() + .find(|v| &v.name == n) + .map(|v| { + v.meta + .parents + .iter() + .filter_map(|p| VersionName::parse(p)) + .collect() + }) + .unwrap_or_default(), + }) + .collect(); + for n in layout::to_delete(&listed) { + let _ = self.store.delete(&n.path()).await; + self.cache.remove(&n); + } + } + + // ─────────────────────────────────────────────── 一轮 + + /// 同步一轮。冲突不算失败:在 [`Report::conflicts`] 里 + pub async fn round(&mut self) -> Result { + self.round_with(&[]).await + } + + /// 同步一轮,顺带处理几处冲突。处理的结果作为新版本上传,别的设备随之清除这几处冲突。 + /// 已经被别的设备处理掉了的那几处不管 + pub async fn round_with(&mut self, decisions: &[Decision]) -> Result { + if !decisions.is_empty() { + self.quiet = None; + } + match self.round_inner(decisions).await { + Ok(r) => { + self.last_report = r.clone(); + Ok(r) + } + Err(e) => Err(e), + } + } + + async fn round_inner(&mut self, decisions: &[Decision]) -> Result { + let mut st = self.local.state(); + if st.place != self.settings.place && !st.place.is_empty() { + // 换了同步位置:原来的基准是那个位置上的,作废 + self.local.clear(); + st = State { + place: self.settings.place.clone(), + ..Default::default() + }; + self.local.save_state(&st).map_err(RoundError::Local)?; + self.cache.clear(); + } + st.place.clone_from(&self.settings.place); + let base = self.base(); + let kr = self.keyring().await?.is_some(); + let (names, ok, too_new) = self.versions().await?; + if names.is_empty() { + if base.is_some() { + return Err(RoundError::Emptied); + } + // 开启同步时上传第一个版本没成(中途断了):这一轮补上 + self.publish_first().await?; + return self.finish_quiet(Vec::new(), kr).await; + } + let cur = self.host.current().map_err(RoundError::Local)?; + let listing: Vec = names.iter().map(|n| n.id()).collect(); + let local_content = cur.pack.content_id(); + if self.quiet.as_ref() == Some(&(listing.clone(), local_content.clone())) { + // 什么都没变:冲突还是上一轮的。照样记一次同步成功,设备记录隔一阵写一次,设备 + // 列表重读(别的设备的记录会变) + st.last_synced_ms = Some(now_ms()); + st.failing_since_ms = None; + st.error = None; + self.local.save_state(&st).map_err(RoundError::Local)?; + if now_ms().saturating_sub(st.device_record_ms) > DEVICE_RECORD_EVERY_MS { + self.write_device(false).await?; + } + let r = Report { + encrypted: kr, + conflicts: self.last_report.conflicts.clone(), + devices: self.devices().await?, + ..Default::default() + }; + return Ok(r); + } + // 头:没有别的版本依据它的版本 + let referenced: BTreeSet = ok + .iter() + .flat_map(|v| v.meta.parents.iter().cloned()) + .collect(); + if let Some((_, cv)) = too_new.iter().find(|(n, _)| !referenced.contains(&n.id())) { + if !st.device_record_paused { + self.write_device(true).await?; + } + return Err(RoundError::TooNew { + core_version: cv.clone(), + }); + } + let top_ok = ok.iter().map(|v| v.name.seq).max(); + let top_all = names.iter().map(|n| n.seq).max(); + if top_ok.is_none() || top_ok < top_all { + return Err(RoundError::NotArrived); + } + let heads: Vec> = ok + .iter() + .filter(|v| !referenced.contains(&v.name.id())) + .cloned() + .collect(); + // 同步位置没有 keyring、版本却是加密的:keyring 还没传到 + if !kr && heads.iter().any(|h| h.pack.manifest.encrypted) { + return Err(RoundError::NotArrived); + } + let remote = self.fold(&heads, &ok, base.as_ref()); + let local_snap = model::snapshot(&cur.pack).map_err(|e| RoundError::Local(e.msg()))?; + + let me = self.settings.device_id.clone(); + let mut local = local_snap.clone(); + // 别的设备处理掉了的冲突:照同步版本改 + let open_ids: BTreeSet<&str> = remote.conflicts.iter().map(|c| c.id.as_str()).collect(); + if let Some(b) = &base { + for c in &b.meta.conflicts { + if !open_ids.contains(c.id.as_str()) { + force(&mut local, &remote.snap, c); + } + } + } + // 自己参与的冲突:本机又改了就更新记录;改得和另一边一样了,冲突就过去了 + let mut open: Vec = Vec::new(); + let mut i_moved: Vec = Vec::new(); + for c in &remote.conflicts { + let now = side_units(&local, c); + let mut c = c.clone(); + if c.other.device == me { + if now == c.shared.units { + continue; + } + if now != c.other.units { + c.other.units = now; + c.other.at_ms = cur.changed_ms; + } + } else if c.shared.device == me { + if now == c.other.units { + continue; + } + if now != c.shared.units { + c.shared.units = now; + c.shared.at_ms = cur.changed_ms; + i_moved.push(c.clone()); + } + } + open.push(c); + } + + // 这一轮要处理的冲突:从记录里拿掉(不再各留各的),本机按选的那一边改 + for d in decisions { + let Some(pos) = open.iter().position(|c| c.id == d.id) else { + continue; + }; + let c = open.remove(pos); + let local_units = local.items.get(&c.item).map(|it| c.pick(&it.units)); + let (_, theirs) = c.mine(&me, local_units.as_ref()); + let theirs = theirs.clone(); + match d.take { + Take::Local => {} + Take::Remote => set_side(&mut local, &c, &theirs, &remote.snap), + Take::Both => { + if !c.both_allowed() { + return Err(RoundError::Local(msg!( + "sync.both_not_allowed" => + "keeping both is not possible for this conflict: one side deleted \ + it, or it has no name" + ))); + } + let name = both_name(&c.item, d.new_name.as_deref(), &theirs.device_name); + let id = ItemId::new(c.item.kind, name); + let taken = + local.items.contains_key(&id) || remote.snap.items.contains_key(&id); + let id = match (taken, d.new_name.is_some()) { + (false, _) => id, + (true, true) => { + return Err(RoundError::Local(msg!( + "sync.both_name_taken", name = id.name => + "the name {name} is already taken" + ))); + } + (true, false) => { + let mut all = local.clone(); + all.items.extend(remote.snap.items.clone()); + ItemId::new(c.item.kind, model::free_name(&all, c.item.kind, &id.name)) + } + }; + let mut copy = local + .items + .get(&c.item) + .or(remote.snap.items.get(&c.item)) + .cloned() + .unwrap_or_default(); + for (u, v) in theirs.units.iter().flatten() { + copy.units.insert(u.clone(), v.clone()); + } + local.items.insert(id, copy); + local.relink(); + } + } + } + + let (mut hold_items, mut hold_units) = record::holds(&open); + let ex = excluded( + &self.settings.exclude, + &[&local, &remote.snap] + .into_iter() + .chain(base.as_ref().map(|b| &b.snap)) + .collect::>(), + ); + hold_items.extend(ex.iter().cloned()); + for (o, _) in ORDERS { + hold_items.insert(ItemId::new(*o, "")); + } + // 不加密时经同步停用的插件:本机启用了就放开 + st.held_plugins.retain(|id, _| { + let item = local.items.get(&ItemId::new(ItemKind::Plugin, id.clone())); + item.is_some_and(|it| it.units.get(unit::ENABLED) != Some(&Value::Bool(true))) + }); + for id in st.held_plugins.keys() { + hold_units.insert(( + ItemId::new(ItemKind::Plugin, id.clone()), + unit::ENABLED.to_string(), + )); + } + + let base_objs = base.as_ref().map(|b| objects(&b.snap)); + let (lo, ro) = (objects(&local), objects(&remote.snap)); + let out = merge::merge(merge::Input { + base: base_objs.as_ref(), + local: &lo, + remote: &ro, + hold_items: &hold_items, + hold_units: &hold_units, + }); + let mut target = rebuild(&out.local, &[&local, &remote.snap]); + let mut shared = rebuild(&out.shared, &[&remote.snap, &local]); + // 不同步的上游、随之不同步的项目:上传的里面是同步位置原有的,不是本机的 + for id in &ex { + match remote.snap.items.get(id) { + Some(it) => shared.items.insert(id.clone(), it.clone()), + None => shared.items.remove(id), + }; + } + // 本机改了自己是「同步版本那一边」的冲突:同步版本跟着改 + for c in &i_moved { + set_side(&mut shared, c, &c.shared, &local); + } + + let side_me = |units: Option>| Side { + device: me.clone(), + device_name: self.settings.device_name.clone(), + at_ms: cur.changed_ms, + units, + }; + let side_remote = |units: Option>| Side { + device: remote.device.clone(), + device_name: remote.device_name.clone(), + at_ms: remote.at_ms, + units, + }; + let mut fresh: Vec = Vec::new(); + for f in &out.conflicts { + fresh.push(found( + f, + (&remote.snap, side_remote(None)), + (&local, side_me(None)), + )); + } + // 已接管的客户端正在用的网关密钥:另一边删了,这边不删,作为冲突列出 + let pinned = self.host.pinned_keys(); + for (id, it) in local.of_kind(ItemKind::Key) { + if pinned.contains(&id.name) + && !target.items.contains_key(id) + && !hold_items.contains(id) + { + target.items.insert(id.clone(), it.clone()); + fresh.push(Conflict { + id: record::new_id(), + item: id.clone(), + units: it.units.keys().cloned().collect(), + shared: side_remote(None), + other: side_me(Some(it.units.clone())), + reason: Some(Reason::KeyInUse), + }); + } + } + // 顺序 + for (o, k) in ORDERS { + let held = open.iter().chain(&fresh).any(|c| c.item.kind == *o); + let m = order::merge( + order_list(base.as_ref().map(|b| &b.snap), *o).as_deref(), + &order_list(Some(&local), *o).unwrap_or_default(), + &order_list(Some(&remote.snap), *o).unwrap_or_default(), + &names_of(&target, *k), + &names_of(&shared, *k), + held, + ); + set_order(&mut target, *o, m.local); + set_order(&mut shared, *o, m.shared); + if m.conflict { + let id = ItemId::new(*o, ""); + fresh.push(Conflict { + id: record::new_id(), + units: vec![unit::ALL.to_string()], + shared: side_remote(remote.snap.items.get(&id).map(|i| i.units.clone())), + other: side_me(local.items.get(&id).map(|i| i.units.clone())), + item: id, + reason: None, + }); + } + } + target.relink(); + shared.relink(); + + // 合并的结果要过配置校验:过不了的那几项留本机的,作为冲突列出 + self.check_target( + &cur.pack, + &local, + &mut target, + &mut shared, + &remote, + &mut fresh, + &side_me, + &side_remote, + )?; + + // 不加密:经同步新增或脚本变了的插件在本机停用;上游的地址变了记一笔 + if !kr { + for (id, it) in target + .of_kind(ItemKind::Plugin) + .map(|(a, b)| (a.clone(), b.clone())) + .collect::>() + { + let before = local.items.get(&id); + let script = it.units.get(unit::SCRIPT); + let changed = before.is_none_or(|b| b.units.get(unit::SCRIPT) != script); + let from_remote = remote + .snap + .items + .get(&id) + .is_some_and(|r| r.units.get(unit::SCRIPT) == script); + if changed && from_remote && it.units.get(unit::ENABLED) == Some(&Value::Bool(true)) + { + if let Some(t) = target.items.get_mut(&id) { + t.units + .insert(unit::ENABLED.to_string(), Value::Bool(false)); + } + st.held_plugins + .insert(id.name.clone(), script.cloned().unwrap_or(Value::Null)); + } + } + for (id, it) in target.of_kind(ItemKind::Provider) { + if let (Some(before), Some(to)) = + (local.items.get(id).and_then(base_url), base_url(it)) + && before != to + { + st.url_changed.insert( + id.name.clone(), + UrlChange { + from: before.to_string(), + to: to.to_string(), + device: Some(remote.device_name.clone()), + at_ms: remote.at_ms, + }, + ); + } + } + } + // 地址又被本机改过、上游没了:那条提示过去了 + st.url_changed.retain(|name, ch| { + target + .items + .get(&ItemId::new(ItemKind::Provider, name.clone())) + .and_then(base_url) + == Some(ch.to.as_str()) + }); + + let mut conflicts = open.clone(); + conflicts.extend(fresh); + + let mut report = Report { + encrypted: kr, + ..Default::default() + }; + // 应用 + if model::normalized(&target) != model::normalized(&local_snap) { + report.keys_changed = local_snap + .of_kind(ItemKind::Key) + .filter(|(id, it)| { + target.items.get(*id).map(|t| key_value(t)) != Some(key_value(it)) + }) + .map(|(id, _)| id.name.clone()) + .collect(); + let pack = + model::materialize(&cur.pack, &target).map_err(|e| RoundError::Local(e.msg()))?; + let from: Vec = { + let mut v: Vec = remote + .heads + .iter() + .filter(|h| h.meta.device != me) + .map(|h| h.meta.device_name.clone()) + .collect(); + v.dedup(); + v + }; + let label = from.join(", "); + match self.host.apply(pack, &cur.version, &label).await { + Ok(()) => {} + Err(ApplyError::Stale) => return Err(RoundError::Stale), + Err(ApplyError::Rejected(m)) => return Err(RoundError::Local(m)), + } + report.applied_from = from; + } + // 上传 + let need_upload = heads.len() > 1 + || model::normalized(&shared) != model::normalized(&remote.snap) + || conflicts != remote.conflicts; + let next_base = if need_upload { + let seq = names.iter().map(|n| n.seq).max().unwrap_or(0) + 1; + let v = self + .publish(&shared, seq, &heads, conflicts.clone()) + .await?; + report.uploaded = Some(v.name.id()); + let mut all: Vec = names.clone(); + all.push(v.name.clone()); + let mut ok_all = ok.clone(); + ok_all.push(Arc::new(v.clone())); + self.prune(&all, &ok_all).await; + v + } else { + (*heads[0]).clone() + }; + self.save_base(&next_base, &mut st)?; + st.last_synced_ms = Some(now_ms()); + st.failing_since_ms = None; + st.error = None; + self.local.save_state(&st).map_err(RoundError::Local)?; + report.conflicts = conflicts; + + let changed = report.uploaded.is_some() || !report.applied_from.is_empty(); + if changed + || st.device_record_paused + || now_ms().saturating_sub(st.device_record_ms) > DEVICE_RECORD_EVERY_MS + { + self.write_device(false).await?; + } + report.devices = self.devices().await?; + // 这一轮之后同步位置上的样子 + let after: Vec = self + .store + .list(VERSIONS) + .await? + .into_iter() + .filter_map(|e| VersionName::parse(&e.name).map(|n| n.id())) + .collect(); + let local_after = self + .host + .current() + .map(|c| c.pack.content_id()) + .unwrap_or_default(); + self.quiet = Some((after, local_after)); + Ok(report) + } + + async fn finish_quiet( + &mut self, + conflicts: Vec, + encrypted: bool, + ) -> Result { + let devices = self.devices().await?; + let r = Report { + conflicts, + devices, + encrypted, + ..Default::default() + }; + Ok(r) + } + + /// 合并出来的本机结果过一遍配置校验。过不了时一项一项试:从本机现在的样子出发,把变了的 + /// 项目一个个换过去,换了就过不了的那一项留本机的,记成冲突(`invalid`),上传的里面也留 + /// 本机的 —— 不然别的设备收到的也是一份过不了校验的配置 + #[allow(clippy::too_many_arguments)] + fn check_target( + &self, + local_pack: &Pack, + local: &Snapshot, + target: &mut Snapshot, + shared: &mut Snapshot, + remote: &Remote, + fresh: &mut Vec, + side_me: &dyn Fn(Option>) -> Side, + side_remote: &dyn Fn(Option>) -> Side, + ) -> Result<(), RoundError> { + let valid = |s: &Snapshot| -> bool { + model::materialize(local_pack, s) + .ok() + .is_some_and(|p| tw_config::try_parse(&p.config_yaml).is_ok()) + }; + if valid(target) { + return Ok(()); + } + let changed: Vec = local + .items + .keys() + .chain(target.items.keys()) + .filter(|id| !id.kind.is_order() && local.items.get(*id) != target.items.get(*id)) + .cloned() + .collect::>() + .into_iter() + .collect(); + let mut t = local.clone(); + let fix_orders = |s: &mut Snapshot, from: &Snapshot| { + for (o, k) in ORDERS { + let names = names_of(s, *k); + let mut list: Vec = order_list(Some(from), *o) + .unwrap_or_default() + .into_iter() + .filter(|n| names.contains(n)) + .collect(); + for n in &names { + if !list.contains(n) { + list.push(n.clone()); + } + } + set_order(s, *o, list); + } + s.relink(); + }; + for id in changed { + let mut next = t.clone(); + match target.items.get(&id) { + Some(it) => next.items.insert(id.clone(), it.clone()), + None => next.items.remove(&id), + }; + fix_orders(&mut next, target); + if valid(&next) { + t = next; + continue; + } + // 这一项留本机的 + match local.items.get(&id) { + Some(it) => shared.items.insert(id.clone(), it.clone()), + None => shared.items.remove(&id), + }; + let units: Vec = local + .items + .get(&id) + .or(target.items.get(&id)) + .map(|it| it.units.keys().cloned().collect()) + .unwrap_or_default(); + fresh.retain(|c| c.item != id); + fresh.push(Conflict { + id: record::new_id(), + item: id.clone(), + units, + shared: side_remote(remote.snap.items.get(&id).map(|i| i.units.clone())), + other: side_me(local.items.get(&id).map(|i| i.units.clone())), + reason: Some(Reason::Invalid), + }); + } + fix_orders(&mut t, target); + let before = shared.clone(); + fix_orders(shared, &before); + *target = t; + if !valid(target) { + return Err(RoundError::Local(msg!( + "sync.merge_invalid" => + "the merged configuration does not pass validation, so nothing was applied" + ))); + } + Ok(()) + } + + // ─────────────────────────────────────────────── 开启、加入 + + /// 看一眼同步位置:写得进去吗,空的还是已经有数据,加没加密,有哪些设备(加密时给了 + /// 对的口令才列得出) + pub async fn probe(store: &S, passphrase: Option<&str>) -> Probe { + let r: u64 = rand::random(); + let path = format!("probe-{r:016x}"); + let writable = match store.put_new(&path, b"ThinkWatch sync probe").await { + Ok(()) => store.delete(&path).await, + Err(e) => Err(e), + }; + if let Err(e) = &writable { + return Probe { + writable: Err(e.clone()), + existing: None, + }; + } + let versions = store.list(VERSIONS).await.unwrap_or_default(); + let keyring = store.get(KEYRING).await.ok().flatten(); + if versions.is_empty() && keyring.is_none() { + return Probe { + writable, + existing: None, + }; + } + let opened = match (&keyring, passphrase) { + (Some(b), Some(p)) => { + let (b, p) = (b.clone(), p.to_string()); + tokio::task::spawn_blocking(move || Keyring::open(&b, &p).ok()) + .await + .ok() + .flatten() + } + _ => None, + }; + let encrypted = keyring.is_some(); + let unlocked = !encrypted || opened.is_some(); + let mut devices = Vec::new(); + if unlocked { + for e in store.list(DEVICES).await.unwrap_or_default() { + if let Ok(Some(b)) = store.get(&format!("{DEVICES}/{}", e.name)).await + && let Some(rec) = DeviceRecord::open(&b, opened.as_ref()) + { + devices.push(rec); + } + } + devices.sort_by(|a, b| a.name.cmp(&b.name)); + } + Probe { + writable, + existing: Some(Existing { + encrypted, + unlocked, + devices, + }), + } + } + + /// 同步位置空着时开启:加密的先建 keyring(**仅当不存在时**:两台设备同时开启时后写的 + /// 那一台拿到 `Ok(false)`,转去加入),然后以本机配置上传第一个版本 + pub async fn create(&mut self) -> Result { + if self.settings.encrypt { + let pass = self + .settings + .passphrase + .clone() + .filter(|p| !p.is_empty()) + .ok_or_else(|| { + RoundError::Local(tw_config::sync::SyncError::PassphraseMissing.msg()) + })?; + let k = Keyring::generate(); + let sealed = tokio::task::spawn_blocking({ + let pass = pass.clone(); + move || (k.seal(&pass), k) + }) + .await + .map_err(local_err)?; + let (sealed, k) = sealed; + if !self.store.create_exclusive(KEYRING, &sealed).await? { + return Ok(false); + } + let hash: [u8; 32] = Sha256::digest(&sealed).into(); + self.keyring = Some((k, hash)); + } else if self.store.get(KEYRING).await?.is_some() { + return Ok(false); + } + if !self.store.list(VERSIONS).await?.is_empty() { + return Ok(false); + } + self.local.clear(); + let st = State { + place: self.settings.place.clone(), + ..Default::default() + }; + self.local.save_state(&st).map_err(RoundError::Local)?; + self.publish_first().await?; + self.quiet = None; + Ok(true) + } + + /// 同步位置的最新(几个头合并之后),写成一个配置包:加入时给导入预览比对。不含本机设置 + pub async fn latest(&mut self) -> Result { + let remote = self.latest_remote().await?.0; + self.pack_of(&remote.snap) + } + + fn pack_of(&self, snap: &Snapshot) -> Result { + let empty = Pack { + manifest: tw_pack::Manifest::new(PackKind::Full, PackOrigin::Sync), + config_yaml: format!("version: {}\n", tw_config::SCHEMA_VERSION), + plugins: Default::default(), + client_prefs: None, + }; + model::materialize(&empty, snap).map_err(|e| RoundError::Local(e.msg())) + } + + async fn latest_remote( + &mut self, + ) -> Result<(Remote, Vec, Vec>), RoundError> { + self.keyring().await?; + let (names, ok, too_new) = self.versions().await?; + let referenced: BTreeSet = ok + .iter() + .flat_map(|v| v.meta.parents.iter().cloned()) + .collect(); + if let Some((_, cv)) = too_new.iter().find(|(n, _)| !referenced.contains(&n.id())) { + return Err(RoundError::TooNew { + core_version: cv.clone(), + }); + } + let heads: Vec> = ok + .iter() + .filter(|v| !referenced.contains(&v.name.id())) + .cloned() + .collect(); + let top_ok = ok.iter().map(|v| v.name.seq).max(); + if heads.is_empty() || top_ok < names.iter().map(|n| n.seq).max() { + return Err(RoundError::NotArrived); + } + let base = self.base(); + Ok((self.fold(&heads, &ok, base.as_ref()), names, ok)) + } + + /// 加入已有的同步。`merged` 是本机按导入的规则合并之后的完整配置包(「合并」);`None` 是 + /// 用同步数据替换本机配置(新安装、或者选了「使用同步数据」)—— 那时本机已接管的客户端 + /// 正在用的网关密钥并入同步数据,不同步的上游留在本机。 + /// + /// 本机先照结果改,再上传一个依据同步位置最新版本的新版本,作为这台设备的基准 + pub async fn join(&mut self, merged: Option) -> Result { + let (remote, names, _) = self.latest_remote().await?; + let cur = self.host.current().map_err(RoundError::Local)?; + let local = model::snapshot(&cur.pack).map_err(|e| RoundError::Local(e.msg()))?; + let ex = excluded(&self.settings.exclude, &[&local, &remote.snap]); + let mut target = match &merged { + Some(p) => model::snapshot(p).map_err(|e| RoundError::Local(e.msg()))?, + None => { + let mut t = remote.snap.clone(); + let pinned = self.host.pinned_keys(); + for (id, it) in local.of_kind(ItemKind::Key) { + if pinned.contains(&id.name) { + t.items.insert(id.clone(), it.clone()); + } + } + t + } + }; + for id in &ex { + match local.items.get(id) { + Some(it) => target.items.insert(id.clone(), it.clone()), + None => target.items.remove(id), + }; + } + let mut shared = target.clone(); + for id in &ex { + match remote.snap.items.get(id) { + Some(it) => shared.items.insert(id.clone(), it.clone()), + None => shared.items.remove(id), + }; + } + for (o, k) in ORDERS { + let m = order::merge( + order_list(Some(&remote.snap), *o).as_deref(), + &order_list(Some(&target), *o) + .or_else(|| order_list(Some(&local), *o)) + .unwrap_or_default(), + &order_list(Some(&remote.snap), *o).unwrap_or_default(), + &names_of(&target, *k), + &names_of(&shared, *k), + false, + ); + set_order(&mut target, *o, m.local); + set_order(&mut shared, *o, m.shared); + } + target.relink(); + shared.relink(); + // 不加密:经同步新来、或者脚本变了的插件在本机停用着,要在本机启用(同步一轮里是同一条) + let mut held = BTreeMap::new(); + if self.keyring.is_none() { + let ids: Vec = target + .of_kind(ItemKind::Plugin) + .map(|(id, _)| id.clone()) + .collect(); + for id in ids { + let script = target.items[&id].units.get(unit::SCRIPT).cloned(); + let changed = local + .items + .get(&id) + .is_none_or(|b| b.units.get(unit::SCRIPT) != script.as_ref()); + let from_remote = remote + .snap + .items + .get(&id) + .is_some_and(|r| r.units.get(unit::SCRIPT) == script.as_ref()); + let on = target.items[&id].units.get(unit::ENABLED) == Some(&Value::Bool(true)); + if changed && from_remote && on { + if let Some(t) = target.items.get_mut(&id) { + t.units + .insert(unit::ENABLED.to_string(), Value::Bool(false)); + } + held.insert(id.name.clone(), script.unwrap_or(Value::Null)); + } + } + } + let mut report = Report { + encrypted: self.keyring.is_some(), + ..Default::default() + }; + if model::normalized(&target) != model::normalized(&local) { + report.keys_changed = local + .of_kind(ItemKind::Key) + .filter(|(id, it)| { + target.items.get(*id).map(|t| key_value(t)) != Some(key_value(it)) + }) + .map(|(id, _)| id.name.clone()) + .collect(); + let pack = + model::materialize(&cur.pack, &target).map_err(|e| RoundError::Local(e.msg()))?; + if tw_config::try_parse(&pack.config_yaml).is_err() { + return Err(RoundError::Local(msg!( + "sync.merge_invalid" => + "the merged configuration does not pass validation, so nothing was applied" + ))); + } + match self + .host + .apply(pack, &cur.version, &remote.device_name) + .await + { + Ok(()) => {} + Err(ApplyError::Stale) => return Err(RoundError::Stale), + Err(ApplyError::Rejected(m)) => return Err(RoundError::Local(m)), + } + report.applied_from = vec![remote.device_name.clone()]; + } + let seq = names.iter().map(|n| n.seq).max().unwrap_or(0) + 1; + let v = self + .publish(&shared, seq, &remote.heads, remote.conflicts.clone()) + .await?; + report.uploaded = Some(v.name.id()); + let mut st = State { + place: self.settings.place.clone(), + held_plugins: held, + ..Default::default() + }; + self.save_base(&v, &mut st)?; + st.last_synced_ms = Some(now_ms()); + st.failing_since_ms = None; + st.error = None; + self.local.save_state(&st).map_err(RoundError::Local)?; + self.write_device(false).await?; + report.devices = self.devices().await?; + report.conflicts = remote.conflicts; + self.quiet = None; + self.last_report = report.clone(); + Ok(report) + } + + // ─────────────────────────────────────────────── 加密 + + /// 之后开启加密:建 keyring,以它重新上传最新版本,删掉未加密的版本。其他设备下次同步时 + /// 要输入口令 + pub async fn enable_encryption(&mut self, passphrase: &str) -> Result<(), RoundError> { + let (remote, names, _) = self.latest_remote().await?; + if self.keyring.is_some() { + return Ok(()); + } + let k = Keyring::generate(); + let sealed = k.seal(passphrase); + if !self.store.create_exclusive(KEYRING, &sealed).await? { + return Err(RoundError::Locked(msg!( + "sync.locked_new" => + "the sync location is now encrypted. Enter the sync passphrase to keep syncing" + ))); + } + let hash: [u8; 32] = Sha256::digest(&sealed).into(); + self.keyring = Some((k, hash)); + self.settings.encrypt = true; + self.settings.passphrase = Some(passphrase.to_string()); + self.reupload(remote, &names).await + } + + /// 关闭加密:口令要对。重新上传不加密的最新版本,删掉 keyring 和加密过的版本 + pub async fn disable_encryption(&mut self, passphrase: &str) -> Result<(), RoundError> { + let Some(bytes) = self.store.get(KEYRING).await? else { + self.keyring = None; + return Ok(()); + }; + let hash: [u8; 32] = Sha256::digest(&bytes).into(); + let pass = passphrase.to_string(); + let opened = tokio::task::spawn_blocking(move || Keyring::open(&bytes, &pass)) + .await + .map_err(local_err)? + .map_err(|e| RoundError::Locked(e.msg()))?; + self.settings.passphrase = Some(passphrase.to_string()); + self.keyring = Some((opened, hash)); + let (remote, names, _) = self.latest_remote().await?; + // 之后写的一律不加密 + self.keyring = None; + self.settings.encrypt = false; + self.reupload(remote, &names).await?; + self.store.delete(KEYRING).await?; + Ok(()) + } + + /// 以现在的加密设置重新上传最新版本,删掉别的版本(它们是另一种加密设置下写的) + async fn reupload(&mut self, remote: Remote, names: &[VersionName]) -> Result<(), RoundError> { + let seq = names.iter().map(|n| n.seq).max().unwrap_or(0) + 1; + let v = self + .publish(&remote.snap, seq, &remote.heads, remote.conflicts.clone()) + .await?; + for n in names { + let _ = self.store.delete(&n.path()).await; + self.cache.remove(n); + } + let mut st = self.local.state(); + self.save_base(&v, &mut st)?; + st.last_synced_ms = Some(now_ms()); + self.local.save_state(&st).map_err(RoundError::Local)?; + self.write_device(false).await?; + self.quiet = None; + Ok(()) + } + + /// 更改同步口令:只重新加密 keyring,已有的版本不变。其他设备下次同步时要输入新口令 + pub async fn change_passphrase(&mut self, old: &str, new: &str) -> Result<(), RoundError> { + let Some(bytes) = self.store.get(KEYRING).await? else { + return Err(RoundError::Local(msg!( + "sync.not_encrypted" => "the sync location is not encrypted" + ))); + }; + let (old, new2) = (old.to_string(), new.to_string()); + let sealed = tokio::task::spawn_blocking(move || { + Keyring::open(&bytes, &old).map(|k| (k.seal(&new2), k)) + }) + .await + .map_err(local_err)? + .map_err(|e| RoundError::Locked(e.msg()))?; + let (sealed, k) = sealed; + self.store.put(KEYRING, &sealed).await?; + let hash: [u8; 32] = Sha256::digest(&sealed).into(); + self.keyring = Some((k, hash)); + self.settings.passphrase = Some(new.to_string()); + Ok(()) + } + + /// 口令对不对得上同步位置的 keyring(别的设备改了口令之后重新输入)。没加密是 `Ok(false)` + pub async fn unlock(&mut self, passphrase: &str) -> Result { + let Some(bytes) = self.store.get(KEYRING).await? else { + return Ok(false); + }; + let pass = passphrase.to_string(); + let k = tokio::task::spawn_blocking(move || Keyring::open(&bytes, &pass)) + .await + .map_err(local_err)?; + match k { + Ok(_) => { + self.settings.passphrase = Some(passphrase.to_string()); + self.settings.encrypt = true; + self.keyring = None; + self.quiet = None; + Ok(true) + } + Err(e) => Err(RoundError::Locked(e.msg())), + } + } + + // ─────────────────────────────────────────────── 停止 + + /// 停止同步:从同步位置移除本设备的设备记录;`purge` 时连全部版本、设备记录和 keyring 一起 + /// 删掉(其他设备随后发现同步位置空了,停止同步)。本机的同步状态目录删掉 + pub async fn stop(&mut self, purge: bool) -> Result<(), RoundError> { + let mine = self + .store + .delete(&layout::device_path(&self.settings.device_id)) + .await; + if purge { + for e in self.store.list(VERSIONS).await? { + self.store.delete(&format!("{VERSIONS}/{}", e.name)).await?; + } + for e in self.store.list(DEVICES).await? { + self.store.delete(&format!("{DEVICES}/{}", e.name)).await?; + } + self.store.delete(KEYRING).await?; + } else { + mine?; + } + self.local.clear(); + self.cache.clear(); + self.quiet = None; + Ok(()) + } + + // ─────────────────────────────────────────────── 同步位置上的备份 + + /// 同步位置上读得出来的版本(最新与上一个):给备份列表 + pub async fn remote_versions(&mut self) -> Result>, RoundError> { + self.keyring().await?; + let (_, ok, _) = self.versions().await?; + Ok(ok) + } + + /// 同步位置上内容标识是这个的版本 + pub async fn remote_version(&mut self, content_id: &str) -> Result, RoundError> { + Ok(self + .remote_versions() + .await? + .into_iter() + .find(|v| v.content == content_id) + .map(|v| v.pack.clone())) + } + + /// 同步位置空着:以本机配置上传第一个版本(加密时先建 keyring) + async fn publish_first(&mut self) -> Result { + let cur = self.host.current().map_err(RoundError::Local)?; + let snap = model::snapshot(&cur.pack).map_err(|e| RoundError::Local(e.msg()))?; + let mut shared = snap.clone(); + let ex = excluded(&self.settings.exclude, &[&snap]); + shared.items.retain(|id, _| !ex.contains(id)); + for (o, k) in ORDERS { + let names = names_of(&shared, *k); + let list: Vec = order_list(Some(&snap), *o) + .unwrap_or_default() + .into_iter() + .filter(|n| names.contains(n)) + .collect(); + set_order(&mut shared, *o, list); + } + shared.relink(); + let v = self.publish(&shared, 1, &[], Vec::new()).await?; + let mut st = self.local.state(); + st.place.clone_from(&self.settings.place); + self.save_base(&v, &mut st)?; + st.last_synced_ms = Some(now_ms()); + self.local.save_state(&st).map_err(RoundError::Local)?; + self.write_device(false).await?; + Ok(v) + } +} diff --git a/crates/tw-sync/src/keyring.rs b/crates/tw-sync/src/keyring.rs new file mode 100644 index 00000000..b17f5d78 --- /dev/null +++ b/crates/tw-sync/src/keyring.rs @@ -0,0 +1,225 @@ +//! 加密同步的版本密钥。 +//! +//! 首台设备生成一把随机的 age x25519 密钥,**以同步口令加密**(age 的口令模式,scrypt 派生) +//! 后存成同步位置上的 `keyring`。各版本、各设备记录都加密给这把密钥的公钥。于是: +//! +//! - 改口令只重新加密 keyring,已有的版本不用动; +//! - 没有口令的一方解不开 keyring,也就既读不了版本、也写不出别的设备认的版本(age 的认证 +//! 加密,见 `tw_pack`)。 +//! +//! scrypt 的强度是**写死的**([`WORK_FACTOR`]),不按这台机器的速度现定:一台快的 Mac 按自己 +//! 定的强度加密,慢的服务器可能解不开(age 拒绝超过本机上限的强度)。 + +use std::sync::atomic::{AtomicU8, Ordering}; + +use age::secrecy::{ExposeSecret, SecretString}; +use serde::{Deserialize, Serialize}; +use tw_types::{Msg, msg}; + +/// scrypt 的 N = 2^17:一次约 128 MiB 内存、零点几秒。只在打开 keyring 时算一次 +pub const WORK_FACTOR: u8 = 17; + +/// 解 keyring 时接受的最高强度。**比写的那个只高一点**:对同步位置有写权限的人能放一个 +/// 强度极高的 keyring 进来,让每台设备一打开就耗尽内存 +const MAX_WORK_FACTOR: u8 = WORK_FACTOR + 1; + +static WORK: AtomicU8 = AtomicU8::new(WORK_FACTOR); + +/// **只给测试用**:把 scrypt 的强度调低,一条测试不必每次都算零点几秒。解的时候照样认 +pub fn set_work_factor_for_tests(log_n: u8) { + WORK.store(log_n.clamp(1, WORK_FACTOR), Ordering::Relaxed); +} + +/// keyring 解开之后的内容 +#[derive(Serialize, Deserialize)] +struct Plain { + /// 格式版本 + v: u32, + /// `AGE-SECRET-KEY-1…` + identity: String, +} + +/// 一把解开了的版本密钥。**Debug 不打印它** +pub struct Keyring { + identity: age::x25519::Identity, +} + +impl std::fmt::Debug for Keyring { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Keyring") + .field("recipient", &self.identity.to_public().to_string()) + .finish() + } +} + +#[derive(Debug, thiserror::Error)] +pub enum KeyringError { + /// 口令不对。**和「坏了」分开**:前者要人重新输入(别的设备可能刚改过口令) + #[error("{}", self.msg())] + WrongPassphrase, + #[error("{}", self.msg())] + Corrupt, + /// 格式版本比这个 core 认得的新 + #[error("{}", self.msg())] + TooNew, +} + +impl KeyringError { + pub fn msg(&self) -> Msg { + match self { + KeyringError::WrongPassphrase => msg!( + "sync.passphrase_wrong" => "the sync passphrase is not correct" + ), + KeyringError::Corrupt => msg!( + "sync.keyring_corrupt" => + "the keyring at the sync location cannot be read. It may still be arriving, or \ + it is damaged" + ), + KeyringError::TooNew => msg!( + "sync.keyring_too_new" => + "the keyring at the sync location was written by a newer version. Update \ + ThinkWatch Lite to keep syncing" + ), + } + } +} + +impl Keyring { + /// 新的一把 + pub fn generate() -> Self { + Self { + identity: age::x25519::Identity::generate(), + } + } + + /// 以口令加密,存成 `keyring` 文件的内容 + pub fn seal(&self, passphrase: &str) -> Vec { + let plain = Plain { + v: 1, + identity: self.identity.to_string().expose_secret().to_string(), + }; + let json = serde_json::to_vec(&plain).expect("plain data serializes"); + let mut r = age::scrypt::Recipient::new(SecretString::from(passphrase.to_string())); + r.set_work_factor(WORK.load(Ordering::Relaxed)); + age::encrypt(&r, &json).expect("encrypting to a passphrase does not fail") + } + + /// 用口令解开 `keyring` 文件 + pub fn open(bytes: &[u8], passphrase: &str) -> Result { + let mut id = age::scrypt::Identity::new(SecretString::from(passphrase.to_string())); + id.set_max_work_factor(MAX_WORK_FACTOR); + let json = age::decrypt(&id, bytes).map_err(|e| match e { + age::DecryptError::DecryptionFailed + | age::DecryptError::KeyDecryptionFailed + | age::DecryptError::NoMatchingKeys => KeyringError::WrongPassphrase, + age::DecryptError::UnknownFormat => KeyringError::TooNew, + _ => KeyringError::Corrupt, + })?; + let plain: Plain = serde_json::from_slice(&json).map_err(|_| KeyringError::Corrupt)?; + if plain.v > 1 { + return Err(KeyringError::TooNew); + } + let identity = plain + .identity + .parse::() + .map_err(|_| KeyringError::Corrupt)?; + Ok(Self { identity }) + } + + /// 加密用的公钥:版本、设备记录都加密给它 + pub fn recipient(&self) -> age::x25519::Recipient { + self.identity.to_public() + } + + /// 解密用的私钥 + pub fn identity(&self) -> &age::x25519::Identity { + &self.identity + } + + /// 加密一小段数据(设备记录) + pub fn encrypt(&self, plain: &[u8]) -> Vec { + age::encrypt(&self.recipient(), plain).expect("encrypting to an x25519 key does not fail") + } + + /// 解密一小段数据。解不开(不是这把密钥加密的、被改过、没传完)是 `None` + pub fn decrypt(&self, bytes: &[u8]) -> Option> { + age::decrypt(&self.identity, bytes).ok() + } + + /// 两把是不是同一把(公钥相同) + pub fn same_as(&self, other: &Keyring) -> bool { + self.recipient().to_string() == other.recipient().to_string() + } +} + +/// 一段字节是不是 age 加密过的(看文件头) +pub fn is_age(bytes: &[u8]) -> bool { + bytes.starts_with(b"age-encryption.org/") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_keyring_opens_with_its_passphrase_and_only_with_it() { + set_work_factor_for_tests(4); + let k = Keyring::generate(); + let sealed = k.seal("correct horse battery staple"); + assert!(is_age(&sealed)); + let back = Keyring::open(&sealed, "correct horse battery staple").unwrap(); + assert!(back.same_as(&k)); + assert!(matches!( + Keyring::open(&sealed, "wrong passphrase!!"), + Err(KeyringError::WrongPassphrase) + )); + assert!(matches!( + Keyring::open(b"not a keyring", "x"), + Err(KeyringError::Corrupt) + )); + } + + #[test] + fn changing_the_passphrase_keeps_the_same_key() { + set_work_factor_for_tests(4); + let k = Keyring::generate(); + let sealed = k.seal("old passphrase 1"); + let opened = Keyring::open(&sealed, "old passphrase 1").unwrap(); + let resealed = opened.seal("new passphrase 2"); + let again = Keyring::open(&resealed, "new passphrase 2").unwrap(); + assert!(again.same_as(&k), "改口令只重新加密 keyring,密钥本身不变"); + let data = k.encrypt(b"device record"); + assert_eq!(again.decrypt(&data).unwrap(), b"device record"); + assert!(Keyring::generate().decrypt(&data).is_none()); + } + + #[test] + fn a_keyring_that_demands_too_much_work_is_refused() { + set_work_factor_for_tests(4); + // 按强度 4 加密,再把文件头里写的强度改成 30:age 在算 scrypt 之前就按上限拒绝, + // 测试不用真的去算一遍 2^30 + let sealed = Keyring::generate().seal("p"); + let text = String::from_utf8_lossy(&sealed).to_string(); + let line = text + .lines() + .find(|l| l.starts_with("-> scrypt ")) + .expect("口令模式的文件头有一行 scrypt") + .to_string(); + let tampered = line.trim_end_matches(char::is_numeric).to_string() + "30"; + let mut bytes = sealed.clone(); + let at = text.find(&line).unwrap(); + bytes.splice(at..at + line.len(), tampered.bytes()); + assert!(matches!( + Keyring::open(&bytes, "p"), + Err(KeyringError::Corrupt) + )); + } + + #[test] + fn debug_shows_only_the_public_half() { + let k = Keyring::generate(); + let shown = format!("{k:?}"); + assert!(shown.contains("age1"), "{shown}"); + assert!(!shown.contains("AGE-SECRET-KEY"), "{shown}"); + } +} diff --git a/crates/tw-sync/src/layout.rs b/crates/tw-sync/src/layout.rs new file mode 100644 index 00000000..6f01ac26 --- /dev/null +++ b/crates/tw-sync/src/layout.rs @@ -0,0 +1,259 @@ +//! 同步位置上有什么、叫什么、留哪些。 +//! +//! ```text +//! <同步位置>/ +//! keyring 以同步口令加密的版本密钥(仅加密时存在) +//! versions/ +//! 0000000041-3f9a2c1e.twconfig 序号-设备标识,只新增、不覆盖 +//! 0000000042-3f9a2c1e.twconfig +//! 0000000042-b71d04aa.twconfig 两台设备同时上传,序号相同,下次同步时合并 +//! devices/ +//! 3f9a2c1e 设备记录(名称、Core 版本、最后同步时间) +//! ``` +//! +//! 同步位置能看到的只有文件名(版本序号和设备标识的前 8 位)、大小和修改时间。 + +use std::collections::BTreeSet; + +pub const KEYRING: &str = "keyring"; +pub const VERSIONS: &str = "versions"; +pub const DEVICES: &str = "devices"; +/// 版本文件的扩展名,和导出的配置包一样 +pub const EXT: &str = ".twconfig"; +/// 序号写几位 +const SEQ_DIGITS: usize = 10; + +/// 一个版本的名字:序号和写它的设备(设备标识的前 8 位)。**按序号排,同序号按设备排** +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct VersionName { + pub seq: u64, + pub device: String, +} + +impl VersionName { + pub fn new(seq: u64, device_id: &str) -> Self { + Self { + seq, + device: device_tag(device_id), + } + } + + /// `0000000042-3f9a2c1e`,不带扩展名。版本互相引用(上一版本)时用它 + pub fn id(&self) -> String { + format!("{:0width$}-{}", self.seq, self.device, width = SEQ_DIGITS) + } + + /// 文件名 + pub fn file(&self) -> String { + format!("{}{EXT}", self.id()) + } + + /// 在同步位置里的路径 + pub fn path(&self) -> String { + format!("{VERSIONS}/{}", self.file()) + } + + /// 认文件名(带扩展名)或者版本标识(不带)。别的名字(临时文件、云盘的冲突副本 + /// `… (conflicted copy).twconfig`、手放进去的文件)是 `None` + pub fn parse(name: &str) -> Option { + let id = name.strip_suffix(EXT).unwrap_or(name); + let (seq, dev) = id.split_once('-')?; + if seq.len() != SEQ_DIGITS || !seq.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + if dev.len() != tw_config::sync::DEVICE_ID_PREFIX + || !dev + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + { + return None; + } + Some(Self { + seq: seq.parse().ok()?, + device: dev.to_string(), + }) + } +} + +impl std::fmt::Display for VersionName { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.id()) + } +} + +/// 设备标识的前 8 位:文件名、设备记录名里用它 +pub fn device_tag(device_id: &str) -> String { + device_id + .chars() + .take(tw_config::sync::DEVICE_ID_PREFIX) + .collect() +} + +/// 设备记录的路径 +pub fn device_path(device_id: &str) -> String { + format!("{DEVICES}/{}", device_tag(device_id)) +} + +/// 同步位置上的一个版本:名字,以及它说自己依据的上一版本(读出来之后才知道) +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Listed { + pub name: VersionName, + pub parents: Vec, +} + +/// 上传新版本之后要删掉哪些。 +/// +/// **保留最新版本与上一个版本**:每一个「头」(没有别的版本依据它的版本 —— 通常只有最新的 +/// 那一个),加上它们依据的上一版本。两台设备同时上传、还没合并时,头有两个,它们和它们 +/// 共同的上一版本都留着,等下一台设备合并:合并要用那个上一版本当基准。晚到的版本(云盘 +/// 后传到的)也是头,**不会因为序号小就被删掉**。别的都删。 +/// +/// `listed` 里的 `parents` 只有读出来了的版本才有。读不出来的(尚未到达、坏了)当作头留着; +/// 最新那一层有读不出来的时,序号紧挨着的那一层也留着 +pub fn to_delete(listed: &[Listed]) -> Vec { + let Some(top) = listed.iter().map(|l| l.name.seq).max() else { + return Vec::new(); + }; + let referenced: BTreeSet<&VersionName> = listed.iter().flat_map(|l| &l.parents).collect(); + let heads: Vec<&Listed> = listed + .iter() + .filter(|l| !referenced.contains(&l.name)) + .collect(); + let mut keep: BTreeSet = heads.iter().map(|l| l.name.clone()).collect(); + for l in &heads { + keep.extend(l.parents.iter().cloned()); + } + // 不知道上一版本是谁(读不出来)时,至少留住序号紧挨着的那一层 + if listed + .iter() + .any(|l| l.name.seq == top && l.parents.is_empty()) + && let Some(prev) = listed.iter().map(|l| l.name.seq).filter(|s| *s < top).max() + { + keep.extend( + listed + .iter() + .filter(|l| l.name.seq == prev) + .map(|l| l.name.clone()), + ); + } + listed + .iter() + .map(|l| l.name.clone()) + .filter(|n| !keep.contains(n)) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn v(seq: u64, dev: &str) -> VersionName { + VersionName { + seq, + device: dev.into(), + } + } + + fn l(seq: u64, dev: &str, parents: &[(u64, &str)]) -> Listed { + Listed { + name: v(seq, dev), + parents: parents.iter().map(|(s, d)| v(*s, d)).collect(), + } + } + + #[test] + fn names_round_trip_and_strangers_are_ignored() { + let n = VersionName::new(42, "3f9a2c1e-0b1d-4e2f-9a3b-5c6d7e8f9a0b"); + assert_eq!(n.file(), "0000000042-3f9a2c1e.twconfig"); + assert_eq!(n.path(), "versions/0000000042-3f9a2c1e.twconfig"); + assert_eq!(VersionName::parse(&n.file()), Some(n.clone())); + assert_eq!(VersionName::parse(&n.id()), Some(n)); + for stranger in [ + "0000000042-3f9a2c1e (conflicted copy).twconfig", + "42-3f9a2c1e.twconfig", + "0000000042-3F9A2C1E.twconfig", + "0000000042-3f9a2c.twconfig", + "notes.txt", + ".tmp-1-0000000042-3f9a2c1e.twconfig", + ] { + assert_eq!(VersionName::parse(stranger), None, "{stranger}"); + } + } + + #[test] + fn versions_sort_by_sequence_then_device() { + let mut all = vec![v(10, "bbbbbbbb"), v(9, "cccccccc"), v(10, "aaaaaaaa")]; + all.sort(); + assert_eq!( + all, + [v(9, "cccccccc"), v(10, "aaaaaaaa"), v(10, "bbbbbbbb")] + ); + } + + #[test] + fn the_latest_and_the_one_before_it_are_kept() { + let listed = [ + l(1, "aaaaaaaa", &[]), + l(2, "aaaaaaaa", &[(1, "aaaaaaaa")]), + l(3, "bbbbbbbb", &[(2, "aaaaaaaa")]), + ]; + assert_eq!(to_delete(&listed), [v(1, "aaaaaaaa")]); + } + + #[test] + fn unmerged_versions_keep_their_common_parent_until_someone_merges_them() { + let listed = [ + l(4, "aaaaaaaa", &[(3, "aaaaaaaa")]), + l(3, "aaaaaaaa", &[(2, "aaaaaaaa")]), + l(5, "aaaaaaaa", &[(4, "aaaaaaaa")]), + l(5, "bbbbbbbb", &[(4, "aaaaaaaa")]), + ]; + assert_eq!(to_delete(&listed), [v(3, "aaaaaaaa")]); + // 合并之后:新版本依据那两个,它们留着,共同的上一版本可以删了 + let merged = [ + l(4, "aaaaaaaa", &[(3, "aaaaaaaa")]), + l(5, "aaaaaaaa", &[(4, "aaaaaaaa")]), + l(5, "bbbbbbbb", &[(4, "aaaaaaaa")]), + l(6, "cccccccc", &[(5, "aaaaaaaa"), (5, "bbbbbbbb")]), + ]; + assert_eq!(to_delete(&merged), [v(4, "aaaaaaaa")]); + } + + /// 云盘后传到的版本:序号比最新的小,但没有谁依据它 —— 它是一个还没合并的头,留着 + #[test] + fn a_late_arrival_is_a_head_and_is_kept_with_its_parent() { + let listed = [ + l(4, "aaaaaaaa", &[(3, "aaaaaaaa")]), + l(5, "aaaaaaaa", &[(4, "aaaaaaaa")]), + l(6, "aaaaaaaa", &[(5, "aaaaaaaa")]), + l(5, "bbbbbbbb", &[(4, "aaaaaaaa")]), + ]; + assert!(to_delete(&listed).is_empty(), "{:?}", to_delete(&listed)); + } + + #[test] + fn a_version_that_has_not_arrived_still_keeps_the_layer_below() { + // 3 读不出来,不知道它依据谁:2 也还是头,2 和它依据的 1 都留着 + let listed = [ + l(1, "aaaaaaaa", &[]), + l(2, "aaaaaaaa", &[(1, "aaaaaaaa")]), + l(3, "bbbbbbbb", &[]), + ]; + assert!(to_delete(&listed).is_empty()); + // 3 读出来了、依据 2:1 可以删了 + let listed = [ + l(1, "aaaaaaaa", &[]), + l(2, "aaaaaaaa", &[(1, "aaaaaaaa")]), + l(3, "bbbbbbbb", &[(2, "aaaaaaaa")]), + ]; + assert_eq!(to_delete(&listed), [v(1, "aaaaaaaa")]); + // 都读不出来:最新的和紧挨着的那一层留着 + let listed = [ + l(1, "aaaaaaaa", &[]), + l(2, "aaaaaaaa", &[]), + l(3, "bbbbbbbb", &[]), + ]; + assert_eq!(to_delete(&listed), Vec::::new()); + assert!(to_delete(&[]).is_empty()); + } +} diff --git a/crates/tw-sync/src/lib.rs b/crates/tw-sync/src/lib.rs new file mode 100644 index 00000000..e4e9d72c --- /dev/null +++ b/crates/tw-sync/src/lib.rs @@ -0,0 +1,24 @@ +//! 多设备配置同步。 +//! +//! 多台设备(本机 Core、远程 core)通过用户自选的同步位置共用一份配置: +//! +//! - **同步位置只追加新版本、不覆盖已有文件**([`layout`])。不依赖存储服务的条件写入,文件夹、 +//! WebDAV、S3 是同一套机制([`store`]); +//! - **按对象三方合并**,以共同的上一版本为基准([`merge`])。不同设备改不同对象、同一对象的 +//! 不同部分时自动合并,只有同一处被改成不同内容才是冲突; +//! - 内容可选加密:一把随机的版本密钥,以同步口令加密后存在同步位置上([`keyring`]); +//! - 每台设备在同步位置上有一份设备记录([`device`])。 +//! +//! 这里不碰控制面:调度、写配置、事件在 `tw-control` 的 `sync` 模块。 + +pub mod device; +pub mod engine; +pub mod keyring; +pub mod layout; +pub mod local; +pub mod merge; +pub mod order; +pub mod record; +pub mod store; + +pub use store::{AnyStore, Entry, Store, StoreError}; diff --git a/crates/tw-sync/src/local.rs b/crates/tw-sync/src/local.rs new file mode 100644 index 00000000..4d8eac11 --- /dev/null +++ b/crates/tw-sync/src/local.rs @@ -0,0 +1,207 @@ +//! 这台设备自己记着的同步状态:数据目录里的 `sync/`。 +//! +//! ```text +//! sync/ +//! base.twconfig 上次同步的那一版(合并基准),不加密,不计入本机保留的 10 个版本 +//! state.json 上次同步的时间、失败从什么时候开始、因同步停用的插件、地址经同步变更的上游 +//! ``` +//! +//! 同步位置上的版本被删掉(设备离线期间)不影响合并:基准在这里。停止同步时整个目录删掉。 + +use std::collections::BTreeMap; +use std::io::Write; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use tw_types::{Msg, msg}; + +pub const DIR: &str = "sync"; +const BASE: &str = "base.twconfig"; +const STATE: &str = "state.json"; + +/// `state.json` +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct State { + /// 这份状态是对着哪个同步位置的([`place`])。换了位置就作废 + pub place: String, + /// 基准是哪一版(版本标识) + pub base: Option, + pub last_synced_ms: Option, + /// 从这个时刻起一直没同步成(连续失败)。成功一次就清掉 + pub failing_since_ms: Option, + /// 最近一次失败的原因 + pub error: Option, + /// 不加密时经同步新增、或者脚本变了的插件,在这台设备上停用着:插件 id → 停用时的脚本 + /// (合并单位的值)。在本机启用之后去掉 + pub held_plugins: BTreeMap, + /// 不加密时经同步改了地址的上游:名字 → 改之前和改之后的地址 + pub url_changed: BTreeMap, + /// 上次写自己的设备记录是什么时候 + pub device_record_ms: u64, + /// 上次写设备记录时是不是暂停着(版本过新) + pub device_record_paused: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct UrlChange { + pub from: String, + pub to: String, + /// 改它的设备的名称 + #[serde(default)] + pub device: Option, + #[serde(default)] + pub at_ms: u64, +} + +/// 一个同步位置的指纹:种类加地址,不含凭据(换了密码还是同一个位置) +pub fn place(target: &tw_config::sync::Target) -> String { + use tw_config::sync::Target; + match target { + Target::Folder { path, .. } => format!("folder:{}", path.trim_end_matches(['/', '\\'])), + Target::Webdav { url, .. } => format!("webdav:{}", url.trim_end_matches('/')), + Target::S3 { + endpoint, + bucket, + prefix, + .. + } => format!( + "s3:{}/{bucket}/{}", + endpoint.trim_end_matches('/'), + prefix.trim_matches('/') + ), + } +} + +fn io(path: &Path, e: std::io::Error) -> Msg { + msg!( + "sync.local.io", path = path.display(), detail = e => + "{path} could not be used: {detail}" + ) +} + +/// 原子写一个只有本机用户读得到的文件 +fn write_private(path: &Path, bytes: &[u8]) -> Result<(), Msg> { + let dir = path.parent().unwrap_or(Path::new(".")); + tw_config::private_dir::create(dir).map_err(|e| io(dir, e))?; + let r: u64 = rand::random(); + let tmp = dir.join(format!(".tmp-{r:016x}")); + let mut opts = std::fs::OpenOptions::new(); + opts.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + opts.mode(0o600); + } + let res = opts + .open(&tmp) + .and_then(|mut f| f.write_all(bytes).and_then(|()| f.sync_all())) + .and_then(|()| std::fs::rename(&tmp, path)); + if let Err(e) = res { + let _ = std::fs::remove_file(&tmp); + return Err(io(path, e)); + } + Ok(()) +} + +/// 这台设备的同步状态目录 +#[derive(Debug, Clone)] +pub struct Local { + dir: PathBuf, +} + +impl Local { + /// `data_dir` 是配置文件所在的目录 + pub fn new(data_dir: &Path) -> Self { + Self { + dir: data_dir.join(DIR), + } + } + + pub fn dir(&self) -> &Path { + &self.dir + } + + pub fn state(&self) -> State { + std::fs::read(self.dir.join(STATE)) + .ok() + .and_then(|b| serde_json::from_slice(&b).ok()) + .unwrap_or_default() + } + + pub fn save_state(&self, s: &State) -> Result<(), Msg> { + write_private( + &self.dir.join(STATE), + &serde_json::to_vec_pretty(s).expect("plain data serializes"), + ) + } + + /// 基准版本的字节(不加密的配置包)。没有是 `None` + pub fn base(&self) -> Option> { + std::fs::read(self.dir.join(BASE)).ok() + } + + pub fn save_base(&self, bytes: &[u8]) -> Result<(), Msg> { + write_private(&self.dir.join(BASE), bytes) + } + + /// 停止同步:整个目录删掉 + pub fn clear(&self) { + let _ = std::fs::remove_dir_all(&self.dir); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn state_and_base_survive_a_restart_and_are_private() { + let d = tempfile::tempdir().unwrap(); + let l = Local::new(d.path()); + assert_eq!(l.state(), State::default()); + assert!(l.base().is_none()); + let s = State { + place: "folder:/x".into(), + base: Some("0000000001-aaaaaaaa".into()), + last_synced_ms: Some(5), + ..Default::default() + }; + l.save_state(&s).unwrap(); + l.save_base(b"PK").unwrap(); + let again = Local::new(d.path()); + assert_eq!(again.state(), s); + assert_eq!(again.base().as_deref(), Some(&b"PK"[..])); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(d.path().join("sync/base.twconfig")) + .unwrap() + .permissions() + .mode() + & 0o777; + assert_eq!(mode, 0o600); + } + again.clear(); + assert!(!d.path().join("sync").exists()); + } + + #[test] + fn the_place_ignores_credentials() { + use tw_config::sync::Target; + let a = Target::Webdav { + url: "https://h/dav/".into(), + user: "a".into(), + pass: "1".into(), + proxy: "direct".into(), + }; + let b = Target::Webdav { + url: "https://h/dav".into(), + user: "b".into(), + pass: "2".into(), + proxy: "corp".into(), + }; + assert_eq!(place(&a), place(&b)); + } +} diff --git a/crates/tw-sync/src/merge.rs b/crates/tw-sync/src/merge.rs new file mode 100644 index 00000000..0c215a59 --- /dev/null +++ b/crates/tw-sync/src/merge.rs @@ -0,0 +1,583 @@ +//! 三方合并:以共同的上一版本为基准,本机当前配置和同步位置的最新版本按**合并单位**逐一比较。 +//! +//! 一个对象(上游、路由、插件……)拆成几个合并单位(上游是连接、模型、计费、其他四组;全局 +//! 设置逐个字段;别的对象整个一个单位),拆法由 `tw_pack::model` 定。这里只认「对象 → 单位名 +//! → 规范化的值」,所以和配置长什么样无关,表驱动测试直接写值。 +//! +//! | 本机相对基准 | 同步位置相对基准 | 结果 | +//! |---|---|---| +//! | 未变 | 未变 | 不变 | +//! | 已修改或新增 | 未变 | 采用本机 | +//! | 未变 | 已修改或新增 | 采用同步位置 | +//! | 已删除 | 未变 | 删除 | +//! | 未变 | 已删除 | 删除 | +//! | 已修改 | 已修改,内容相同 | 不变 | +//! | 已修改或新增 | 已修改或新增,内容不同 | 冲突 | +//! | 已删除 | 已修改 | 冲突 | +//! | 已修改 | 已删除 | 冲突 | +//! +//! **冲突不阻断合并**:其余单位照常合并。冲突的那几个单位(或者那个被删的对象)两边各留各的 +//! —— 本机的结果里是本机的,要上传的结果里是同步位置上已有的,等人处理。 +//! +//! 没有基准(基准版本哪里都找不到)时按两方比较:两边都有、内容不同的单位是冲突;只有一边 +//! 有的对象当作那一边新增的,不当作另一边删掉的 —— 分不清的时候,留着比删掉好。 + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::Value; + +/// 一个对象:合并单位 → 规范化的值 +pub type Units = BTreeMap; + +/// 一份配置拆成的对象 +pub type Objects = BTreeMap; + +/// 冲突的形状 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Kind { + /// 两边都改了(或者都新增了)同一个单位,内容不同 + Modified, + /// 本机删了,同步位置改了 + LocalDeleted, + /// 本机改了,同步位置删了 + RemoteDeleted, +} + +/// 这次合并新发现的一处冲突 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Found { + pub key: K, + /// 冲突的单位。一边删了对象时是另一边改过的那几个 + pub units: BTreeSet, + pub kind: Kind, +} + +/// 合并的输入 +pub struct Input<'a, K> { + /// 共同的上一版本。`None` 是哪里都找不到,按两方比较 + pub base: Option<&'a Objects>, + pub local: &'a Objects, + pub remote: &'a Objects, + /// 整个对象各留各的,不比较:不同步的上游和随之不同步的项目,以及一边删了对象、还没 + /// 处理的冲突 + pub hold_items: &'a BTreeSet, + /// 这几个单位各留各的:还没处理的冲突 + pub hold_units: &'a BTreeSet<(K, String)>, +} + +/// 合并的结果 +#[derive(Debug, Clone, PartialEq)] +pub struct Outcome { + /// 本机应用的样子 + pub local: Objects, + /// 上传的样子 + pub shared: Objects, + pub conflicts: Vec>, +} + +/// 一个单位在一边的样子:没有这个对象、或者对象里没有这个单位都是 `None` +fn unit<'a>(o: Option<&'a Units>, u: &str) -> Option<&'a Value> { + o.and_then(|m| m.get(u)) +} + +pub fn merge(i: Input<'_, K>) -> Outcome { + let empty = Objects::new(); + let base = i.base.unwrap_or(&empty); + let mut out = Outcome { + local: Objects::new(), + shared: Objects::new(), + conflicts: Vec::new(), + }; + let keys: BTreeSet<&K> = base + .keys() + .chain(i.local.keys()) + .chain(i.remote.keys()) + .collect(); + for k in keys { + let (b, l, r) = (base.get(k), i.local.get(k), i.remote.get(k)); + let mut put = |local: Option<&Units>, shared: Option<&Units>| { + if let Some(v) = local { + out.local.insert(k.clone(), v.clone()); + } + if let Some(v) = shared { + out.shared.insert(k.clone(), v.clone()); + } + }; + if i.hold_items.contains(k) { + put(l, r); + continue; + } + match (b, l, r) { + (_, None, None) => {} + // 有基准,一边删了 + (Some(b), None, Some(r)) => { + if r == b { + // 同步位置没动、本机删了:删 + } else { + put(None, Some(r)); + out.conflicts.push(Found { + key: k.clone(), + units: changed(b, r), + kind: Kind::LocalDeleted, + }); + } + } + (Some(b), Some(l), None) => { + if l == b { + // 本机没动、同步位置删了:删 + } else { + put(Some(l), None); + out.conflicts.push(Found { + key: k.clone(), + units: changed(b, l), + kind: Kind::RemoteDeleted, + }); + } + } + // 没有基准、只有一边有:那一边新增的 + (None, Some(l), None) => put(Some(l), Some(l)), + (None, None, Some(r)) => put(Some(r), Some(r)), + // 两边都有:逐个单位比 + (b, Some(l), Some(r)) => { + let names: BTreeSet<&String> = b + .into_iter() + .flat_map(|m| m.keys()) + .chain(l.keys()) + .chain(r.keys()) + .collect(); + let (mut lo, mut sh) = (Units::new(), Units::new()); + let mut clash = BTreeSet::new(); + for u in names { + let (bv, lv, rv) = (unit(b, u), l.get(u), r.get(u)); + let mut set = |lo_v: Option<&Value>, sh_v: Option<&Value>| { + if let Some(v) = lo_v { + lo.insert(u.clone(), v.clone()); + } + if let Some(v) = sh_v { + sh.insert(u.clone(), v.clone()); + } + }; + if i.hold_units.contains(&(k.clone(), u.clone())) { + set(lv, rv); + } else if lv == rv || rv == bv { + // 两边一样,或者同步位置没动:本机的 + set(lv, lv); + } else if lv == bv { + // 本机没动:同步位置的 + set(rv, rv); + } else { + set(lv, rv); + clash.insert(u.clone()); + } + } + put(Some(&lo), Some(&sh)); + if !clash.is_empty() { + out.conflicts.push(Found { + key: k.clone(), + units: clash, + kind: Kind::Modified, + }); + } + } + } + } + out +} + +/// 两个样子之间变了的单位 +pub fn changed(a: &Units, b: &Units) -> BTreeSet { + a.keys() + .chain(b.keys()) + .filter(|u| a.get(*u) != b.get(*u)) + .cloned() + .collect() +} + +/// 一份配置相对基准有没有变(不看 `hold_items`、`hold_units` 里的)。决定要不要上传 +pub fn differs( + base: &Objects, + now: &Objects, + hold_items: &BTreeSet, + hold_units: &BTreeSet<(K, String)>, +) -> bool { + let keys: BTreeSet<&K> = base.keys().chain(now.keys()).collect(); + keys.into_iter().any(|k| { + if hold_items.contains(k) { + return false; + } + match (base.get(k), now.get(k)) { + (None, None) => false, + (Some(_), None) | (None, Some(_)) => true, + (Some(a), Some(b)) => changed(a, b) + .iter() + .any(|u| !hold_units.contains(&(k.clone(), u.clone()))), + } + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + /// 一个对象:单位名和值成对写 + fn obj(units: &[(&str, Value)]) -> Units { + units + .iter() + .map(|(u, v)| (u.to_string(), v.clone())) + .collect() + } + + fn objs(items: &[(&str, Option)]) -> Objects { + items + .iter() + .filter_map(|(k, v)| v.clone().map(|v| (k.to_string(), v))) + .collect() + } + + fn run( + base: Option>, + local: Option, + remote: Option, + ) -> Outcome { + let b = base.map(|b| objs(&[("p", b)])); + let l = objs(&[("p", local)]); + let r = objs(&[("p", remote)]); + merge(Input { + base: b.as_ref(), + local: &l, + remote: &r, + hold_items: &BTreeSet::new(), + hold_units: &BTreeSet::new(), + }) + } + + /// 方案 6.6 的每一行,在一个合并单位上,外加两边都删、两边都新增、没有基准。 + /// + /// 每一行:基准、本机、同步位置(`None` 是没有这个对象),期望本机的结果、上传的结果、 + /// 冲突的形状。 + #[test] + fn every_row_of_the_merge_table() { + let a = || Some(obj(&[("conn", json!("A"))])); + let b2 = || Some(obj(&[("conn", json!("B"))])); + let c = || Some(obj(&[("conn", json!("C"))])); + type Row = ( + &'static str, + Option>, + Option, + Option, + Option, + Option, + Option, + ); + let rows: Vec = vec![ + ("未变 / 未变 → 不变", Some(a()), a(), a(), a(), a(), None), + ( + "本机修改 / 未变 → 采用本机", + Some(a()), + b2(), + a(), + b2(), + b2(), + None, + ), + ( + "本机新增 / 未变 → 采用本机", + Some(None), + b2(), + None, + b2(), + b2(), + None, + ), + ( + "未变 / 同步位置修改 → 采用同步位置", + Some(a()), + a(), + b2(), + b2(), + b2(), + None, + ), + ( + "未变 / 同步位置新增 → 采用同步位置", + Some(None), + None, + b2(), + b2(), + b2(), + None, + ), + ( + "本机删除 / 未变 → 删除", + Some(a()), + None, + a(), + None, + None, + None, + ), + ( + "未变 / 同步位置删除 → 删除", + Some(a()), + a(), + None, + None, + None, + None, + ), + ("两边都删 → 删除", Some(a()), None, None, None, None, None), + ( + "两边改成一样 → 不变", + Some(a()), + b2(), + b2(), + b2(), + b2(), + None, + ), + ( + "两边新增成一样 → 不变", + Some(None), + b2(), + b2(), + b2(), + b2(), + None, + ), + ( + "两边改得不一样 → 冲突", + Some(a()), + b2(), + c(), + b2(), + c(), + Some(Kind::Modified), + ), + ( + "两边新增得不一样 → 冲突", + Some(None), + b2(), + c(), + b2(), + c(), + Some(Kind::Modified), + ), + ( + "本机删除 / 同步位置修改 → 冲突", + Some(a()), + None, + b2(), + None, + b2(), + Some(Kind::LocalDeleted), + ), + ( + "本机修改 / 同步位置删除 → 冲突", + Some(a()), + b2(), + None, + b2(), + None, + Some(Kind::RemoteDeleted), + ), + ("没有基准,一样 → 不变", None, a(), a(), a(), a(), None), + ( + "没有基准,不一样 → 冲突", + None, + a(), + b2(), + a(), + b2(), + Some(Kind::Modified), + ), + ( + "没有基准,只有本机有 → 留着", + None, + a(), + None, + a(), + a(), + None, + ), + ( + "没有基准,只有同步位置有 → 拿来", + None, + None, + a(), + a(), + a(), + None, + ), + ]; + for (what, base, local, remote, want_local, want_shared, want_conflict) in rows { + let o = run(base, local, remote); + assert_eq!(o.local.get("p").cloned(), want_local, "{what}:本机"); + assert_eq!(o.shared.get("p").cloned(), want_shared, "{what}:上传"); + assert_eq!( + o.conflicts.first().map(|c| c.kind), + want_conflict, + "{what}:冲突" + ); + if want_conflict.is_some() { + assert_eq!(o.conflicts.len(), 1, "{what}"); + assert_eq!(o.conflicts[0].key, "p", "{what}"); + assert_eq!( + o.conflicts[0].units, + BTreeSet::from(["conn".to_string()]), + "{what}" + ); + } + } + } + + /// 同一个上游,两边改的是不同的单位:自动合并,没有冲突 + #[test] + fn changes_to_different_units_of_one_object_merge_by_themselves() { + let base = obj(&[ + ("conn", json!("A")), + ("models", json!(["m1"])), + ("billing", json!("x")), + ]); + let local = obj(&[ + ("conn", json!("A2")), + ("models", json!(["m1"])), + ("billing", json!("x")), + ]); + let remote = obj(&[ + ("conn", json!("A")), + ("models", json!(["m1", "m2"])), + ("billing", json!("x")), + ]); + let o = run(Some(Some(base)), Some(local), Some(remote)); + let want = obj(&[ + ("conn", json!("A2")), + ("models", json!(["m1", "m2"])), + ("billing", json!("x")), + ]); + assert!(o.conflicts.is_empty(), "{:?}", o.conflicts); + assert_eq!(o.local["p"], want); + assert_eq!(o.shared["p"], want); + } + + /// 一个单位冲突,其余单位照常合并;冲突的那一个本机留本机的、上传留同步位置的 + #[test] + fn a_conflict_on_one_unit_does_not_hold_back_the_others() { + let base = obj(&[("conn", json!("A")), ("models", json!(1))]); + let local = obj(&[("conn", json!("L")), ("models", json!(1))]); + let remote = obj(&[("conn", json!("R")), ("models", json!(2))]); + let o = run(Some(Some(base)), Some(local), Some(remote)); + assert_eq!( + o.local["p"], + obj(&[("conn", json!("L")), ("models", json!(2))]) + ); + assert_eq!( + o.shared["p"], + obj(&[("conn", json!("R")), ("models", json!(2))]) + ); + assert_eq!(o.conflicts.len(), 1); + assert_eq!(o.conflicts[0].units, BTreeSet::from(["conn".to_string()])); + } + + /// 删了的一边对改了的一边:冲突列出的是改了的那几个单位 + #[test] + fn a_delete_against_an_edit_names_the_edited_units() { + let base = obj(&[ + ("conn", json!("A")), + ("models", json!(1)), + ("other", json!(0)), + ]); + let remote = obj(&[ + ("conn", json!("A")), + ("models", json!(2)), + ("other", json!(5)), + ]); + let o = run(Some(Some(base)), None, Some(remote.clone())); + assert_eq!(o.local.get("p"), None); + assert_eq!(o.shared.get("p"), Some(&remote)); + assert_eq!( + o.conflicts[0].units, + BTreeSet::from(["models".to_string(), "other".to_string()]) + ); + } + + /// 不同步的项目、还没处理的冲突:两边各留各的,不比较,也不再报一次冲突 + #[test] + fn held_items_and_units_pass_through_on_each_side() { + let base = objs(&[ + ("ollama", Some(obj(&[("conn", json!("base"))]))), + ( + "relay", + Some(obj(&[("conn", json!("A")), ("models", json!(1))])), + ), + ]); + let local = objs(&[ + ("ollama", Some(obj(&[("conn", json!("local"))]))), + ( + "relay", + Some(obj(&[("conn", json!("L")), ("models", json!(1))])), + ), + ]); + let remote = objs(&[ + ("ollama", Some(obj(&[("conn", json!("remote"))]))), + ( + "relay", + Some(obj(&[("conn", json!("R")), ("models", json!(2))])), + ), + ]); + let hold_items = BTreeSet::from(["ollama".to_string()]); + let hold_units = BTreeSet::from([("relay".to_string(), "conn".to_string())]); + let o = merge(Input { + base: Some(&base), + local: &local, + remote: &remote, + hold_items: &hold_items, + hold_units: &hold_units, + }); + assert!(o.conflicts.is_empty(), "{:?}", o.conflicts); + assert_eq!(o.local["ollama"], obj(&[("conn", json!("local"))])); + assert_eq!(o.shared["ollama"], obj(&[("conn", json!("remote"))])); + assert_eq!( + o.local["relay"], + obj(&[("conn", json!("L")), ("models", json!(2))]) + ); + assert_eq!( + o.shared["relay"], + obj(&[("conn", json!("R")), ("models", json!(2))]) + ); + // 只有本机有、不同步的:不上传 + let local_only = objs(&[("ollama", Some(obj(&[("conn", json!("local"))])))]); + let o = merge(Input { + base: None, + local: &local_only, + remote: &Objects::new(), + hold_items: &hold_items, + hold_units: &BTreeSet::new(), + }); + assert!(o.shared.is_empty()); + assert_eq!(o.local.len(), 1); + } + + #[test] + fn whether_there_is_anything_to_upload() { + let base = objs(&[("a", Some(obj(&[("conn", json!(1)), ("models", json!(1))])))]); + let same = base.clone(); + let none = BTreeSet::new(); + assert!(!differs(&base, &same, &none, &BTreeSet::new())); + let edited = objs(&[("a", Some(obj(&[("conn", json!(2)), ("models", json!(1))])))]); + assert!(differs(&base, &edited, &none, &BTreeSet::new())); + // 改的正是还没处理的冲突那一个单位:不算 + let held = BTreeSet::from([("a".to_string(), "conn".to_string())]); + assert!(!differs(&base, &edited, &none, &held)); + // 新增了一个不同步的上游:不算 + let mut more = base.clone(); + more.insert("ollama".into(), obj(&[("conn", json!(1))])); + assert!(differs(&base, &more, &none, &BTreeSet::new())); + assert!(!differs( + &base, + &more, + &BTreeSet::from(["ollama".to_string()]), + &BTreeSet::new() + )); + } +} diff --git a/crates/tw-sync/src/order.rs b/crates/tw-sync/src/order.rs new file mode 100644 index 00000000..82e7ae4e --- /dev/null +++ b/crates/tw-sync/src/order.rs @@ -0,0 +1,192 @@ +//! 上游、路由、插件的先后顺序:**内容和顺序分开合并**。 +//! +//! 一边新增了一条路由、另一边新增了另一条,两边的顺序都变了 —— 这不该是冲突。所以顺序不按 +//! 单位整个比,而是: +//! +//! 1. 两边都有、基准里也有的那些名字,比它们的相对先后。只有一边改了就用那一边的;两边改得 +//! 一样也行;两边改得不一样是冲突(本机留本机的,上传的留同步位置的)。 +//! 2. 新来的名字插在它在自己那一边的前一个名字后面。 +//! 3. 最后只留下合并之后还在的那些项目。 + +use std::collections::BTreeSet; + +/// 顺序合并的结果 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Merged { + pub local: Vec, + pub shared: Vec, + /// 两边把同一批名字排成了不同的先后 + pub conflict: bool, +} + +fn restrict(list: &[String], keep: &BTreeSet<&str>) -> Vec { + list.iter() + .filter(|n| keep.contains(n.as_str())) + .cloned() + .collect() +} + +/// `primary` 打底,把 `present` 里还没排上的名字按 `sources`(先看的在前)里各自的前一个 +/// 名字插进去,最后只留 `present` 里有的 +fn arrange(primary: &[String], sources: &[&[String]], present: &BTreeSet) -> Vec { + let mut out: Vec = primary + .iter() + .filter(|n| present.contains(*n)) + .cloned() + .collect(); + for src in sources { + for (i, n) in src.iter().enumerate() { + if !present.contains(n) || out.contains(n) { + continue; + } + // 它在这一边的前一个、已经排上了的名字 + let after = src[..i].iter().rev().find(|p| out.contains(*p)); + match after { + Some(p) => { + let at = out.iter().position(|x| x == p).map_or(out.len(), |i| i + 1); + out.insert(at, n.clone()); + } + None => out.insert(0, n.clone()), + } + } + } + // 哪一边的顺序里都没有的(不该有):接在后面 + for n in present { + if !out.contains(n) { + out.push(n.clone()); + } + } + out +} + +/// 合并一种顺序。`local_names`、`shared_names` 是合并之后两份结果里各自还在的那些项目 +pub fn merge( + base: Option<&[String]>, + local: &[String], + remote: &[String], + local_names: &BTreeSet, + shared_names: &BTreeSet, + hold: bool, +) -> Merged { + let base = base.unwrap_or(&[]); + let common: BTreeSet<&str> = local + .iter() + .filter(|n| remote.contains(n) && (base.is_empty() || base.contains(n))) + .map(String::as_str) + .collect(); + let (b, l, r) = ( + restrict(base, &common), + restrict(local, &common), + restrict(remote, &common), + ); + let (l_changed, r_changed) = (!base.is_empty() && l != b, !base.is_empty() && r != b); + let no_base_disagree = base.is_empty() && l != r; + let conflict = !hold && ((l_changed && r_changed && l != r) || no_base_disagree); + let (local_primary, shared_primary): (&[String], &[String]) = if hold || conflict { + (local, remote) + } else if l_changed && !r_changed { + (local, local) + } else { + (remote, remote) + }; + Merged { + local: arrange(local_primary, &[local, remote], local_names), + shared: arrange(shared_primary, &[remote, local], shared_names), + conflict, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn v(names: &[&str]) -> Vec { + names.iter().map(|s| s.to_string()).collect() + } + fn set(names: &[&str]) -> BTreeSet { + names.iter().map(|s| s.to_string()).collect() + } + + #[test] + fn two_sides_adding_different_items_is_not_a_conflict() { + let m = merge( + Some(&v(&["a", "b"])), + &v(&["a", "b", "l"]), + &v(&["r", "a", "b"]), + &set(&["a", "b", "l", "r"]), + &set(&["a", "b", "l", "r"]), + false, + ); + assert!(!m.conflict); + assert_eq!(m.local, v(&["r", "a", "b", "l"])); + assert_eq!(m.shared, m.local); + } + + #[test] + fn one_side_reordering_wins_and_both_reordering_differently_conflicts() { + let all = set(&["a", "b", "c"]); + let m = merge( + Some(&v(&["a", "b", "c"])), + &v(&["c", "a", "b"]), + &v(&["a", "b", "c"]), + &all, + &all, + false, + ); + assert_eq!((m.local.clone(), m.conflict), (v(&["c", "a", "b"]), false)); + assert_eq!(m.shared, m.local); + let m = merge( + Some(&v(&["a", "b", "c"])), + &v(&["c", "a", "b"]), + &v(&["b", "a", "c"]), + &all, + &all, + false, + ); + assert!(m.conflict); + assert_eq!(m.local, v(&["c", "a", "b"])); + assert_eq!(m.shared, v(&["b", "a", "c"])); + // 两边排成一样的:不冲突 + let m = merge( + Some(&v(&["a", "b"])), + &v(&["b", "a"]), + &v(&["b", "a"]), + &set(&["a", "b"]), + &set(&["a", "b"]), + false, + ); + assert!(!m.conflict); + } + + #[test] + fn deleted_and_local_only_names_drop_out_of_the_upload() { + // 本机有一个不同步的 ollama:本机的顺序里留着,上传的没有 + let m = merge( + Some(&v(&["ollama", "relay", "official"])), + &v(&["ollama", "relay", "official"]), + &v(&["relay", "official"]), + &set(&["ollama", "relay", "official"]), + &set(&["relay", "official"]), + false, + ); + assert!(!m.conflict); + assert_eq!(m.local, v(&["ollama", "relay", "official"])); + assert_eq!(m.shared, v(&["relay", "official"])); + } + + #[test] + fn a_held_order_keeps_each_side_without_raising_it_again() { + let all = set(&["a", "b"]); + let m = merge( + Some(&v(&["a", "b"])), + &v(&["b", "a"]), + &v(&["a", "b"]), + &all, + &all, + true, + ); + assert!(!m.conflict); + assert_eq!(m.local, v(&["b", "a"])); + assert_eq!(m.shared, v(&["a", "b"])); + } +} diff --git a/crates/tw-sync/src/record.rs b/crates/tw-sync/src/record.rs new file mode 100644 index 00000000..dc5d3c78 --- /dev/null +++ b/crates/tw-sync/src/record.rs @@ -0,0 +1,226 @@ +//! 每个同步版本自己说的那几件事(配置包 manifest 里的 `sync`):第几版、谁写的、依据的 +//! 上一版本、还没处理的冲突。 +//! +//! **冲突记录随版本走**:所有设备看到的是同一份冲突,任一设备处理之后,处理结果作为新版本 +//! 上传,冲突从记录里去掉,别的设备随之清除,不重复处理。 + +use std::collections::{BTreeMap, BTreeSet}; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use tw_pack::model::ItemId; + +/// `manifest.sync` +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct Meta { + pub seq: u64, + /// 写它的设备,完整的标识 + pub device: String, + pub device_name: String, + /// 依据的上一版本(版本标识,见 [`crate::layout::VersionName::id`])。第一个版本没有; + /// 合并了并列的版本时有几个 + #[serde(default)] + pub parents: Vec, + /// 上一版本各自的内容标识(`Pack::content_id`),和 `parents` 一一对应。上一版本在同步 + /// 位置上已经删了时,按它去本机备份里找基准 + #[serde(default)] + pub parent_content: Vec, + /// 内容的 SHA-256(只在不加密时写):发现传输中损坏。加密时 age 的认证加密已经管了 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sha256: Option, + /// 还没处理的冲突 + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub conflicts: Vec, +} + +/// 一处冲突:同一个对象(的几个单位)在两台设备上被改成了不同的内容。 +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Conflict { + /// 随机生成 + pub id: String, + pub item: ItemId, + /// 冲突的单位 + pub units: Vec, + /// 同步版本里现在是这一边的内容 + pub shared: Side, + /// 另一边:它的设备在冲突处理之前一直用自己的 + pub other: Side, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, +} + +/// 冲突的一边 +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Side { + pub device: String, + pub device_name: String, + /// 这一边改的时间 + pub at_ms: u64, + /// 冲突的那几个单位在这一边的值。**这一边删了这个对象**是 `None` + pub units: Option>, +} + +/// 冲突不是因为两边改得不一样,而是: +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Reason { + /// 另一边删了一把网关密钥,而这台设备上已接管的客户端正在用它:没有删 + KeyInUse, + /// 合并出来的结果过不了配置校验(比如路由引用的上游被另一边删了) + Invalid, +} + +impl Conflict { + /// 有一边删了整个对象:两边各留各的是整个对象,不只是几个单位 + pub fn whole_item(&self) -> bool { + self.shared.units.is_none() || self.other.units.is_none() + } + + /// 「两者都保留」能不能选:两边都还有这个对象,而且它有名字(全局设置、顺序、通用偏好 + /// 没有第二份可言) + pub fn both_allowed(&self) -> bool { + !self.whole_item() && self.item.kind.is_named() && self.reason.is_none() + } + + /// 这台设备看这处冲突时,哪一边是「本机」:自己写的那一边;不是参与的设备时,是和本机 + /// 现在内容一样的那一边,都不一样就是同步版本里的那一边 + pub fn mine<'a>( + &'a self, + device: &str, + local_units: Option<&BTreeMap>, + ) -> (&'a Side, &'a Side) { + if self.other.device == device { + return (&self.other, &self.shared); + } + if self.shared.device == device { + return (&self.shared, &self.other); + } + let mine_now = local_units.map(|l| self.pick(l)); + if mine_now.as_ref() == self.other.units.as_ref() { + (&self.other, &self.shared) + } else { + (&self.shared, &self.other) + } + } + + /// 一个对象里冲突的那几个单位。有一边删了整个对象时是整个对象(处理冲突选了「有」的 + /// 那一边,要拿它整个恢复回来) + pub fn pick(&self, units: &BTreeMap) -> BTreeMap { + if self.whole_item() { + return units.clone(); + } + self.units + .iter() + .filter_map(|u| units.get(u).map(|v| (u.clone(), v.clone()))) + .collect() + } +} + +/// 一组冲突各留各的那部分:整个对象,和单个单位 +pub fn holds(conflicts: &[Conflict]) -> (BTreeSet, BTreeSet<(ItemId, String)>) { + let mut items = BTreeSet::new(); + let mut units = BTreeSet::new(); + for c in conflicts { + if c.whole_item() { + items.insert(c.item.clone()); + } else { + for u in &c.units { + units.insert((c.item.clone(), u.clone())); + } + } + } + (items, units) +} + +/// 新的冲突标识 +pub fn new_id() -> String { + let r: u64 = rand::random(); + format!("{r:016x}") +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use tw_pack::model::ItemKind; + + fn side(device: &str, v: Option) -> Side { + Side { + device: device.into(), + device_name: device.to_uppercase(), + at_ms: 1, + units: v.map(|v| BTreeMap::from([("conn".to_string(), v)])), + } + } + + fn c(shared: Option, other: Option, kind: ItemKind) -> Conflict { + Conflict { + id: new_id(), + item: ItemId::new(kind, "relay"), + units: vec!["conn".into()], + shared: side("aaaa", shared), + other: side("bbbb", other), + reason: None, + } + } + + #[test] + fn each_device_sees_its_own_side_as_local() { + let x = c(Some(json!("A")), Some(json!("B")), ItemKind::Provider); + assert_eq!(x.mine("aaaa", None).0.device, "aaaa"); + assert_eq!(x.mine("bbbb", None).0.device, "bbbb"); + // 第三台:和本机内容一样的那一边 + let local = BTreeMap::from([ + ("conn".to_string(), json!("B")), + ("models".into(), json!(1)), + ]); + assert_eq!(x.mine("cccc", Some(&local)).0.device, "bbbb"); + assert_eq!(x.mine("cccc", None).0.device, "aaaa"); + } + + #[test] + fn keeping_both_needs_two_living_named_objects() { + assert!(c(Some(json!(1)), Some(json!(2)), ItemKind::Provider).both_allowed()); + assert!(!c(None, Some(json!(2)), ItemKind::Provider).both_allowed()); + assert!(!c(Some(json!(1)), None, ItemKind::Route).both_allowed()); + assert!(!c(Some(json!(1)), Some(json!(2)), ItemKind::Global).both_allowed()); + assert!(!c(Some(json!(1)), Some(json!(2)), ItemKind::RouteOrder).both_allowed()); + let mut k = c(Some(json!(1)), None, ItemKind::Key); + k.reason = Some(Reason::KeyInUse); + assert!(!k.both_allowed()); + } + + #[test] + fn holds_split_into_whole_objects_and_units() { + let (items, units) = holds(&[c(Some(json!(1)), Some(json!(2)), ItemKind::Provider), { + let mut d = c(None, Some(json!(2)), ItemKind::Route); + d.item = ItemId::new(ItemKind::Route, "long"); + d + }]); + assert_eq!( + items, + BTreeSet::from([ItemId::new(ItemKind::Route, "long")]) + ); + assert_eq!( + units, + BTreeSet::from([(ItemId::new(ItemKind::Provider, "relay"), "conn".to_string())]) + ); + } + + #[test] + fn a_record_survives_json() { + let m = Meta { + seq: 7, + device: "aaaa".into(), + device_name: "Mac mini".into(), + parents: vec!["0000000006-aaaaaaaa".into()], + parent_content: vec!["abc".into()], + sha256: None, + conflicts: vec![c(Some(json!({"base_url": "x"})), None, ItemKind::Key)], + }; + let v = serde_json::to_value(&m).unwrap(); + assert_eq!(v["conflicts"][0]["item"]["kind"], "key"); + let back: Meta = serde_json::from_value(v).unwrap(); + assert_eq!(back, m); + } +} diff --git a/crates/tw-sync/src/store/folder.rs b/crates/tw-sync/src/store/folder.rs new file mode 100644 index 00000000..a8fca449 --- /dev/null +++ b/crates/tw-sync/src/store/folder.rs @@ -0,0 +1,398 @@ +//! 文件夹位置:本机的一个目录,由 iCloud Drive、Dropbox、OneDrive、Syncthing 或者 NAS +//! 共享在设备之间传送。 +//! +//! **写入先写临时名再改名**:第三方客户端随时可能把写了一半的文件传走,另一台设备读到半个 +//! 版本。临时名以点开头,列目录时跳过;云盘客户端多数也不传点开头的临时文件。 +//! +//! 云盘里还没下载到本机的文件(iCloud 的 `.名字.icloud` 占位文件、Dropbox 的在线文件)列目录 +//! 时要么不出现、要么读不出来 —— 调用方把读不出来的版本当成「尚未到达」,到了再合并。 + +use std::io::Write; +use std::path::{Path, PathBuf}; + +use tw_types::msg; + +use super::{Entry, Store, StoreError, check_path, split, temp_name}; + +pub struct Folder { + root: PathBuf, +} + +impl Folder { + pub fn new(root: impl Into) -> Self { + Self { root: root.into() } + } + + pub fn root(&self) -> &Path { + &self.root + } + + fn at(&self, path: &str) -> Result { + check_path(path)?; + let mut p = self.root.clone(); + for seg in path.split('/') { + p.push(seg); + } + Ok(p) + } +} + +fn io(path: &Path, e: std::io::Error) -> StoreError { + let detail = e.to_string(); + let path = path.display().to_string(); + // 根目录不在(外接盘没插、网络共享没挂上):和网络位置连不上是同一种情况,等它回来 + if e.kind() == std::io::ErrorKind::NotFound { + return StoreError::Unreachable(msg!( + "sync.folder.missing", path = path => + "the sync folder {path} is not there. If it is on an external drive or a network \ + share, connect it" + )); + } + if e.kind() == std::io::ErrorKind::PermissionDenied { + return StoreError::AuthFailed(msg!( + "sync.folder.denied", path = path, detail = detail => + "{path} cannot be written: {detail}" + )); + } + StoreError::Other(msg!( + "sync.folder.io", path = path, detail = detail => + "{path} could not be used: {detail}" + )) +} + +/// 建一个只有本机用户读得到的新文件,写完、落盘 +fn write_new(path: &Path, bytes: &[u8]) -> std::io::Result<()> { + let mut opts = std::fs::OpenOptions::new(); + opts.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + opts.mode(0o600); + } + let mut f = opts.open(path)?; + f.write_all(bytes)?; + f.sync_all() +} + +/// 父目录:要在根目录底下。**根目录本身不替用户建**:它不在说明外接盘、共享没挂上,在一个 +/// 空目录里写进去的版本谁也收不到 +fn ensure_parent(root: &Path, p: &Path) -> Result<(), StoreError> { + if !root.is_dir() { + return Err(io( + root, + std::io::Error::new(std::io::ErrorKind::NotFound, "missing"), + )); + } + if let Some(dir) = p.parent() + && dir != root + { + std::fs::create_dir_all(dir).map_err(|e| io(dir, e))?; + } + Ok(()) +} + +async fn blocking( + f: impl FnOnce() -> Result + Send + 'static, +) -> Result { + tokio::task::spawn_blocking(f).await.unwrap_or_else(|e| { + Err(StoreError::Other(msg!( + "sync.folder.task", detail = e => + "the sync folder could not be used: {detail}" + ))) + }) +} + +/// 改名成最终的名字,**目标已经有了就不改**。硬链接是原子的「不存在才建」;不支持硬链接的 +/// 文件系统(FAT、有的网络共享)退回到先看一眼再改名 —— 名字归写它的那台设备独占,看一眼 +/// 和改名之间不会有别人来写同一个名字 +fn link_or_rename(tmp: &Path, dest: &Path) -> std::io::Result { + match std::fs::hard_link(tmp, dest) { + Ok(()) => { + let _ = std::fs::remove_file(tmp); + Ok(true) + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { + let _ = std::fs::remove_file(tmp); + Ok(false) + } + Err(_) => { + if dest.exists() { + let _ = std::fs::remove_file(tmp); + return Ok(false); + } + std::fs::rename(tmp, dest).map(|()| true) + } + } +} + +impl Store for Folder { + async fn list(&self, dir: &str) -> Result, StoreError> { + let root = self.root.clone(); + let d = self.at(dir)?; + blocking(move || { + if !root.is_dir() { + return Err(io( + &root, + std::io::Error::new(std::io::ErrorKind::NotFound, "missing"), + )); + } + let rd = match std::fs::read_dir(&d) { + Ok(r) => r, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(e) => return Err(io(&d, e)), + }; + let mut out = Vec::new(); + for e in rd.flatten() { + let name = e.file_name().to_string_lossy().to_string(); + // 临时文件、云盘的占位文件、系统的 .DS_Store + if name.starts_with('.') { + continue; + } + let Ok(meta) = e.metadata() else { continue }; + if !meta.is_file() { + continue; + } + out.push(Entry { + name, + size: meta.len(), + }); + } + out.sort_by(|a, b| a.name.cmp(&b.name)); + Ok(out) + }) + .await + } + + async fn get(&self, path: &str) -> Result>, StoreError> { + let root = self.root.clone(); + let p = self.at(path)?; + blocking(move || match std::fs::read(&p) { + Ok(b) => Ok(Some(b)), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + if root.is_dir() { + Ok(None) + } else { + Err(io(&root, e)) + } + } + Err(e) => Err(io(&p, e)), + }) + .await + } + + async fn put_new(&self, path: &str, bytes: &[u8]) -> Result<(), StoreError> { + let root = self.root.clone(); + let dest = self.at(path)?; + let (_, name) = split(path); + let tmp = dest.with_file_name(temp_name(name)); + let bytes = bytes.to_vec(); + let shown = path.to_string(); + blocking(move || { + ensure_parent(&root, &dest)?; + if dest.exists() { + return Err(StoreError::Exists { path: shown }); + } + write_new(&tmp, &bytes).map_err(|e| io(&tmp, e))?; + match link_or_rename(&tmp, &dest) { + Ok(true) => Ok(()), + Ok(false) => Err(StoreError::Exists { path: shown }), + Err(e) => { + let _ = std::fs::remove_file(&tmp); + Err(io(&dest, e)) + } + } + }) + .await + } + + async fn put(&self, path: &str, bytes: &[u8]) -> Result<(), StoreError> { + let root = self.root.clone(); + let dest = self.at(path)?; + let (_, name) = split(path); + let tmp = dest.with_file_name(temp_name(name)); + let bytes = bytes.to_vec(); + blocking(move || { + ensure_parent(&root, &dest)?; + write_new(&tmp, &bytes).map_err(|e| io(&tmp, e))?; + // rename 在两个平台上都换掉已有的目标(Windows 上 std 用的是 MOVEFILE_REPLACE_EXISTING) + std::fs::rename(&tmp, &dest).map_err(|e| { + let _ = std::fs::remove_file(&tmp); + io(&dest, e) + }) + }) + .await + } + + async fn delete(&self, path: &str) -> Result<(), StoreError> { + let p = self.at(path)?; + blocking(move || match std::fs::remove_file(&p) { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(io(&p, e)), + }) + .await + } + + async fn create_exclusive(&self, path: &str, bytes: &[u8]) -> Result { + let root = self.root.clone(); + let dest = self.at(path)?; + let (_, name) = split(path); + let tmp = dest.with_file_name(temp_name(name)); + let bytes = bytes.to_vec(); + blocking(move || { + ensure_parent(&root, &dest)?; + if dest.exists() { + return Ok(false); + } + write_new(&tmp, &bytes).map_err(|e| io(&tmp, e))?; + link_or_rename(&tmp, &dest).map_err(|e| { + let _ = std::fs::remove_file(&tmp); + io(&dest, e) + }) + }) + .await + } +} + +/// 盯着同步文件夹:新版本、新的 keyring 一到就发一个信号(见 [`tw_watch::watch`])。盯的是 +/// 根目录和 `versions/`,不递归;`versions/` 还没有就先建出来,不然盯不住。 +/// +/// **设备记录不盯**:每台设备同步完都会写自己那一份,盯着它的话,两台设备会你写一份、我 +/// 同步一轮、我再写一份地来回转 +pub fn watch( + root: &Path, +) -> Result<(tw_watch::Watch, tokio::sync::mpsc::Receiver<()>), tw_watch::WatchError> { + let versions = root.join(crate::layout::VERSIONS); + // 根目录不在(外接盘没插)时不替它建 + if root.is_dir() { + let _ = std::fs::create_dir(&versions); + } + let dirs: Vec = [root.to_path_buf(), versions] + .into_iter() + .filter(|d| d.is_dir()) + .collect(); + tw_watch::watch(&dirs, std::time::Duration::from_millis(500), |p| { + // 我们自己的临时文件、云盘的占位文件不算:改名成正式的名字那一下才算 + p.file_name() + .and_then(|n| n.to_str()) + .is_some_and(|n| !n.starts_with('.')) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn bed() -> (tempfile::TempDir, Folder) { + let d = tempfile::tempdir().unwrap(); + let f = Folder::new(d.path()); + (d, f) + } + + #[tokio::test] + async fn put_new_never_overwrites_and_leaves_no_temporary_file() { + let (d, f) = bed(); + f.put_new("versions/0000000001-aaaaaaaa.twconfig", b"one") + .await + .unwrap(); + let e = f + .put_new("versions/0000000001-aaaaaaaa.twconfig", b"two") + .await + .unwrap_err(); + assert!(matches!(e, StoreError::Exists { .. }), "{e:?}"); + assert_eq!( + f.get("versions/0000000001-aaaaaaaa.twconfig") + .await + .unwrap(), + Some(b"one".to_vec()) + ); + let names: Vec = std::fs::read_dir(d.path().join("versions")) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().to_string()) + .collect(); + assert_eq!(names, ["0000000001-aaaaaaaa.twconfig"]); + } + + #[tokio::test] + async fn create_exclusive_says_whether_it_made_the_file() { + let (_d, f) = bed(); + assert!(f.create_exclusive("keyring", b"first").await.unwrap()); + assert!(!f.create_exclusive("keyring", b"second").await.unwrap()); + assert_eq!(f.get("keyring").await.unwrap(), Some(b"first".to_vec())); + f.put("keyring", b"changed").await.unwrap(); + assert_eq!(f.get("keyring").await.unwrap(), Some(b"changed".to_vec())); + } + + #[tokio::test] + async fn listing_skips_hidden_and_placeholder_files() { + let (d, f) = bed(); + assert!(f.list("versions").await.unwrap().is_empty()); + f.put_new("versions/0000000002-bbbbbbbb.twconfig", b"x") + .await + .unwrap(); + std::fs::write( + d.path() + .join("versions/.0000000003-cccccccc.twconfig.icloud"), + b"placeholder", + ) + .unwrap(); + std::fs::create_dir(d.path().join("versions/sub")).unwrap(); + let got = f.list("versions").await.unwrap(); + assert_eq!( + got, + [Entry { + name: "0000000002-bbbbbbbb.twconfig".into(), + size: 1 + }] + ); + f.delete("versions/0000000002-bbbbbbbb.twconfig") + .await + .unwrap(); + f.delete("versions/0000000002-bbbbbbbb.twconfig") + .await + .unwrap(); + assert!(f.list("versions").await.unwrap().is_empty()); + } + + #[tokio::test] + async fn a_missing_root_is_unreachable_and_never_created() { + let d = tempfile::tempdir().unwrap(); + let gone = d.path().join("not-mounted"); + let f = Folder::new(&gone); + let e = f.put_new("versions/x.twconfig", b"x").await.unwrap_err(); + assert!(matches!(e, StoreError::Unreachable(_)), "{e:?}"); + assert!(matches!( + f.list("versions").await.unwrap_err(), + StoreError::Unreachable(_) + )); + assert!(!gone.exists(), "根目录不该替用户建出来"); + } + + #[cfg(unix)] + #[tokio::test] + async fn files_are_readable_only_by_this_user() { + use std::os::unix::fs::PermissionsExt; + let (d, f) = bed(); + f.put_new("devices/aaaaaaaa", b"x").await.unwrap(); + let mode = std::fs::metadata(d.path().join("devices/aaaaaaaa")) + .unwrap() + .permissions() + .mode() + & 0o777; + assert_eq!(mode, 0o600); + } + + #[tokio::test] + async fn a_new_version_arriving_wakes_the_watcher() { + let (d, f) = bed(); + let (_w, mut rx) = watch(d.path()).unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + f.put_new("versions/0000000009-dddddddd.twconfig", b"v") + .await + .unwrap(); + let got = tokio::time::timeout(std::time::Duration::from_secs(5), rx.recv()) + .await + .expect("五秒内没收到信号"); + assert_eq!(got, Some(())); + } +} diff --git a/crates/tw-sync/src/store/mod.rs b/crates/tw-sync/src/store/mod.rs new file mode 100644 index 00000000..cd56a9c7 --- /dev/null +++ b/crates/tw-sync/src/store/mod.rs @@ -0,0 +1,296 @@ +//! 同步位置的存储:文件夹、WebDAV、S3 兼容存储,**同一套五个动作**。 +//! +//! 同步只追加新版本、不覆盖已有文件,所以不依赖存储服务的条件写入:版本的文件名 +//! (序号-设备标识)由写它的那台设备独占,两台设备永远不会去写同一个名字。只有两样东西 +//! 不是这样: +//! +//! - `keyring` **仅当不存在时创建**([`Store::create_exclusive`]):两台设备同时开启加密同步时, +//! 后写入的一方要知道自己晚了一步,转入加入流程; +//! - 设备记录和改口令后的 keyring 要覆盖([`Store::put`]):设备记录只有它自己那台设备写。 +//! +//! 路径一律相对同步位置的根,用 `/` 分隔,只有两层(`versions/<名字>`、`devices/<名字>`、 +//! `keyring`)。名字由这个 crate 生成,只含 ASCII 字母、数字、连字符和点。 + +use std::future::Future; + +use tw_types::{Msg, msg}; + +pub mod folder; +pub mod s3; +pub mod webdav; +mod xml; + +pub use folder::Folder; +pub use s3::S3; +pub use webdav::WebDav; + +/// 目录里的一个文件。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Entry { + pub name: String, + pub size: u64, +} + +/// 一次存储操作没成。 +/// +/// **按「接下来怎么办」分类**,不按出错的那一层:界面上同步位置不可用(等它恢复)、 +/// 拒绝访问(改连接参数)是两种不同的提示。 +#[derive(Debug, Clone, thiserror::Error)] +pub enum StoreError { + /// 连不上、超时、服务端 5xx。**过一会儿再试** + #[error("{}", .0.text)] + Unreachable(Msg), + /// 401 / 403:用户名、密码、密钥不对,或者没有这个位置的权限 + #[error("{}", .0.text)] + AuthFailed(Msg), + /// `put_new`:这个名字已经有文件了 + #[error("{}", self.msg())] + Exists { path: String }, + /// 别的:服务端回了一个看不懂的答复、本机磁盘写不进去 + #[error("{}", .0.text)] + Other(Msg), +} + +impl StoreError { + pub fn msg(&self) -> Msg { + match self { + StoreError::Unreachable(m) | StoreError::AuthFailed(m) | StoreError::Other(m) => { + m.clone() + } + StoreError::Exists { path } => msg!( + "sync.store.exists", path = path => + "the sync location already has {path}" + ), + } + } +} + +/// 同步位置。实现见 [`Folder`]、[`WebDav`]、[`S3`];运行时按配置选一个,用 [`AnyStore`]。 +pub trait Store: Send + Sync { + /// 一个目录里的文件(不递归,不含子目录)。目录不存在是空的,不是错误 + fn list(&self, dir: &str) -> impl Future, StoreError>> + Send; + /// 读一个文件。不存在是 `None` + fn get(&self, path: &str) -> impl Future>, StoreError>> + Send; + /// **只新增**:已有同名文件时是 [`StoreError::Exists`],原来那个不动。写的过程中别的设备 + /// 读不到半个文件(文件夹和 WebDAV 先写临时名再改名) + fn put_new( + &self, + path: &str, + bytes: &[u8], + ) -> impl Future> + Send; + /// 覆盖写。只用于设备记录(只有它自己那台设备写)和改口令后的 keyring + fn put(&self, path: &str, bytes: &[u8]) -> impl Future> + Send; + /// 删掉。本来就没有不算错 + fn delete(&self, path: &str) -> impl Future> + Send; + /// 仅当不存在时创建:建成是 `true`,已经有了是 `false`(原来那个不动) + fn create_exclusive( + &self, + path: &str, + bytes: &[u8], + ) -> impl Future> + Send; +} + +/// 按配置选出来的那一种。 +pub enum AnyStore { + Folder(Folder), + WebDav(WebDav), + S3(S3), +} + +impl AnyStore { + /// 按 `sync.target` 建一个。`http` 是发 WebDAV、S3 请求用的客户端(带着出站代理), + /// 文件夹用不到它 + pub fn new( + target: &tw_config::sync::Target, + http: reqwest::Client, + ) -> Result { + use tw_config::sync::Target; + Ok(match target { + Target::Folder { path, .. } => AnyStore::Folder(Folder::new(path)), + Target::Webdav { + url, user, pass, .. + } => AnyStore::WebDav(WebDav::new(url, user, pass, http)?), + Target::S3 { + endpoint, + region, + bucket, + prefix, + access_key_id, + secret_access_key, + .. + } => AnyStore::S3(S3::new( + s3::Config { + endpoint: endpoint.clone(), + region: region.clone(), + bucket: bucket.clone(), + prefix: prefix.clone(), + access_key_id: access_key_id.clone(), + secret_access_key: secret_access_key.clone(), + }, + http, + )?), + }) + } + + /// 文件夹位置的根目录。别的是 `None` + pub fn folder_root(&self) -> Option<&std::path::Path> { + match self { + AnyStore::Folder(f) => Some(f.root()), + _ => None, + } + } +} + +macro_rules! each { + ($self:ident, $s:ident => $e:expr) => { + match $self { + AnyStore::Folder($s) => $e, + AnyStore::WebDav($s) => $e, + AnyStore::S3($s) => $e, + } + }; +} + +impl Store for AnyStore { + async fn list(&self, dir: &str) -> Result, StoreError> { + each!(self, s => s.list(dir).await) + } + async fn get(&self, path: &str) -> Result>, StoreError> { + each!(self, s => s.get(path).await) + } + async fn put_new(&self, path: &str, bytes: &[u8]) -> Result<(), StoreError> { + each!(self, s => s.put_new(path, bytes).await) + } + async fn put(&self, path: &str, bytes: &[u8]) -> Result<(), StoreError> { + each!(self, s => s.put(path, bytes).await) + } + async fn delete(&self, path: &str) -> Result<(), StoreError> { + each!(self, s => s.delete(path).await) + } + async fn create_exclusive(&self, path: &str, bytes: &[u8]) -> Result { + each!(self, s => s.create_exclusive(path, bytes).await) + } +} + +/// 临时文件名:点开头(列目录时跳过),带随机串(两个进程不撞) +pub(crate) fn temp_name(final_name: &str) -> String { + let r: u64 = rand::random(); + format!(".tmp-{r:016x}-{final_name}") +} + +/// 一个路径拆成目录和文件名。只有一层目录 +pub(crate) fn split(path: &str) -> (&str, &str) { + match path.rsplit_once('/') { + Some((d, f)) => (d, f), + None => ("", path), + } +} + +/// 路径里只许出现我们自己生成的那些字符:拼进 URL 和文件系统都不用转义,也走不出根目录 +pub(crate) fn check_path(path: &str) -> Result<(), StoreError> { + let ok = !path.is_empty() + && path.split('/').all(|seg| { + !seg.is_empty() + && seg != "." + && seg != ".." + && seg + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.')) + }); + if ok { + Ok(()) + } else { + Err(StoreError::Other(msg!( + "sync.store.bad_path", path = path => + "{path} is not a path inside the sync location" + ))) + } +} + +/// HTTP 层的失败按「接下来怎么办」归类。`what` 是这次在做什么(`GET versions/…`),只进英文原句 +pub(crate) fn http_status(status: reqwest::StatusCode, what: &str) -> StoreError { + let code = status.as_u16(); + if code == 401 || code == 403 { + return StoreError::AuthFailed(msg!( + "sync.store.denied", status = code => + "the sync location refused access ({status}). Check the user name, password or keys" + )); + } + if status.is_server_error() || code == 408 || code == 429 { + return StoreError::Unreachable(msg!( + "sync.store.server_error", status = code, what = what => + "the sync location answered {status} to {what}" + )); + } + StoreError::Other(msg!( + "sync.store.unexpected", status = code, what = what => + "the sync location answered {status} to {what}" + )) +} + +/// 请求没发出去或者没收到答复 +pub(crate) fn http_error(e: &reqwest::Error) -> StoreError { + let mut detail = e.to_string(); + let mut cur = std::error::Error::source(e); + while let Some(c) = cur { + detail.push_str(": "); + detail.push_str(&c.to_string()); + cur = c.source(); + } + StoreError::Unreachable(msg!( + "sync.store.unreachable", detail = detail => + "the sync location could not be reached: {detail}" + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_paths_we_generate_are_taken() { + for ok in [ + "keyring", + "versions/0000000041-3f9a2c1e.twconfig", + "devices/3f9a2c1e", + ] { + check_path(ok).unwrap(); + } + for bad in [ + "", + "/keyring", + "versions/../x", + "a//b", + "a b", + "versions/%2e%2e", + "a\\b", + ] { + assert!(check_path(bad).is_err(), "{bad}"); + } + } + + #[test] + fn statuses_are_sorted_by_what_to_do_next() { + use reqwest::StatusCode as S; + assert!(matches!( + http_status(S::UNAUTHORIZED, "x"), + StoreError::AuthFailed(_) + )); + assert!(matches!( + http_status(S::FORBIDDEN, "x"), + StoreError::AuthFailed(_) + )); + assert!(matches!( + http_status(S::BAD_GATEWAY, "x"), + StoreError::Unreachable(_) + )); + assert!(matches!( + http_status(S::TOO_MANY_REQUESTS, "x"), + StoreError::Unreachable(_) + )); + assert!(matches!( + http_status(S::CONFLICT, "x"), + StoreError::Other(_) + )); + } +} diff --git a/crates/tw-sync/src/store/s3.rs b/crates/tw-sync/src/store/s3.rs new file mode 100644 index 00000000..a823ee1f --- /dev/null +++ b/crates/tw-sync/src/store/s3.rs @@ -0,0 +1,456 @@ +//! S3 兼容存储:Amazon S3、Cloudflare R2、阿里云 OSS、腾讯云 COS、MinIO。 +//! +//! 签名是 SigV4,和 Bedrock 用的同一份([`tw_bedrock::sign_service`])。四个动作: +//! `ListObjectsV2`(列目录,带 `delimiter=/`,不递归)、`GetObject`、`PutObject`、`DeleteObject`。 +//! 「只新增」和「仅当不存在时创建」都带 `If-None-Match: *`:S3、R2、MinIO 认它;不认的服务端 +//! (回 501)退回到先看一眼再写 —— 版本的名字归写它的那台设备独占,只有 keyring 会有两台设备 +//! 抢,那一刻两台设备同时开启加密同步,退回之后后写的那台覆盖了先写的,先写的那台下一次同步 +//! 时发现口令对不上(keyring 换了)就转入加入流程。 +//! +//! **地址怎么拼**([`addressing`]):端点主机以 `amazonaws.com`、`aliyuncs.com`、`myqcloud.com` +//! 结尾的用虚拟主机(`<桶>.<端点>/<键>`,OSS、COS 要求如此),别的(R2、MinIO、IP 地址、自定义 +//! 域名)用路径(`<端点>/<桶>/<键>`)。 + +use std::time::{Duration, SystemTime}; + +use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, utf8_percent_encode}; +use reqwest::Method; +use tw_types::msg; + +use super::{Entry, Store, StoreError, check_path, http_error, http_status}; + +const TIMEOUT: Duration = Duration::from_secs(120); + +/// 路径里的一段:RFC 3986 的非保留字符不编,别的都编 +const SEGMENT: &AsciiSet = &NON_ALPHANUMERIC + .remove(b'-') + .remove(b'.') + .remove(b'_') + .remove(b'~'); + +/// 查询参数的值:同上,斜杠也编 +const QUERY: &AsciiSet = SEGMENT; + +/// 这几家只认(或者推荐)虚拟主机的写法 +const VIRTUAL_HOSTED: &[&str] = &["amazonaws.com", "aliyuncs.com", "myqcloud.com"]; + +#[derive(Clone)] +pub struct Config { + pub endpoint: String, + pub region: String, + pub bucket: String, + pub prefix: String, + pub access_key_id: String, + pub secret_access_key: String, +} + +pub struct S3 { + /// 桶的地址,以 `/` 结尾:`https://b.s3.us-east-1.amazonaws.com/` 或 `http://nas:9000/b/` + bucket_url: String, + /// 键的前缀,空的或者以 `/` 结尾 + prefix: String, + region: String, + creds: tw_bedrock::Credentials, + http: reqwest::Client, +} + +fn bad(endpoint: &str) -> StoreError { + StoreError::Other( + tw_config::sync::SyncError::Address { + field: "endpoint", + url: endpoint.to_string(), + } + .msg(), + ) +} + +/// 寻址方式:端点主机以 `amazonaws.com`、`aliyuncs.com`、`myqcloud.com` 结尾的用虚拟主机 +/// (桶名进域名,阿里云 OSS、腾讯云 COS 要求如此);别的(R2、MinIO、IP 地址、自定义域名)用 +/// 路径。不给界面加选项。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Addressing { + VirtualHost, + Path, +} + +/// 按端点的主机选寻址方式(纯函数) +pub fn addressing(host: &str) -> Addressing { + let host = host.trim_end_matches('.').to_ascii_lowercase(); + if VIRTUAL_HOSTED + .iter() + .any(|d| host == *d || host.ends_with(&format!(".{d}"))) + { + Addressing::VirtualHost + } else { + Addressing::Path + } +} + +/// 桶的地址,以 `/` 结尾(纯函数)。虚拟主机方式下,端点本身已经以 `<桶>.` 开头的(照着控制台 +/// 抄来的桶域名)不再加一遍。地址规则(`http://` 只给内网)在这之前查 +pub fn bucket_url(endpoint: &str, bucket: &str) -> Option { + let mut u = url::Url::parse(endpoint.trim()).ok()?; + let host = u.host_str()?.to_ascii_lowercase(); + let base_path = u.path().trim_end_matches('/').to_string(); + match addressing(&host) { + Addressing::VirtualHost => { + if !host.starts_with(&format!("{bucket}.")) { + u.set_host(Some(&format!("{bucket}.{host}"))).ok()?; + } + u.set_path(&format!("{base_path}/")); + } + Addressing::Path => { + let seg = utf8_percent_encode(bucket, SEGMENT).to_string(); + u.set_path(&format!("{base_path}/{seg}/")); + } + } + Some(u.to_string()) +} + +impl S3 { + pub fn new(c: Config, http: reqwest::Client) -> Result { + // `http://` 只给本机和内网,和 WebDAV 同一条规则;证书由客户端照常校验 + if !tw_config::sync::address_ok(&c.endpoint) { + return Err(bad(&c.endpoint)); + } + let bucket_url = bucket_url(&c.endpoint, &c.bucket).ok_or_else(|| bad(&c.endpoint))?; + let p = c.prefix.trim().trim_matches('/'); + let prefix = if p.is_empty() { + String::new() + } else { + format!("{p}/") + }; + Ok(Self { + bucket_url, + prefix, + region: c.region.trim().to_string(), + creds: tw_bedrock::Credentials { + access_key_id: c.access_key_id, + secret_access_key: c.secret_access_key, + session_token: None, + }, + http, + }) + } + + fn key(&self, path: &str) -> String { + format!("{}{path}", self.prefix) + } + + fn object_url(&self, key: &str) -> String { + let enc: Vec = key + .split('/') + .map(|s| utf8_percent_encode(s, SEGMENT).to_string()) + .collect(); + format!("{}{}", self.bucket_url, enc.join("/")) + } + + /// 签好名、发出去。`headers` 一起签 + async fn send( + &self, + m: Method, + url: &str, + headers: &[(&str, &str)], + body: &[u8], + ) -> Result { + let signed = tw_bedrock::sign_service( + &self.creds, + &self.region, + "s3", + m.as_str(), + url, + headers, + body, + SystemTime::now(), + ) + .map_err(|e| { + StoreError::Other(msg!( + "sync.s3.sign", detail = e => + "the request to the S3 location could not be signed: {detail}" + )) + })?; + let mut r = self.http.request(m, url).timeout(TIMEOUT); + for (k, v) in headers { + r = r.header(*k, *v); + } + for (k, v) in signed { + r = r.header(k, v); + } + if !body.is_empty() { + r = r.body(body.to_vec()); + } + r.send().await.map_err(|e| http_error(&e)) + } + + /// S3 错误答复里的 `` + async fn error_code(r: reqwest::Response) -> String { + let body = r.text().await.unwrap_or_default(); + super::xml::parse(&body, "Error") + .and_then(|(rs, top)| { + rs.first() + .and_then(|e| e.get("Code").map(str::to_string)) + .or_else(|| top.get("Code").cloned()) + }) + .unwrap_or_default() + } + + async fn exists(&self, path: &str) -> Result { + let url = self.object_url(&self.key(path)); + let r = self.send(Method::HEAD, &url, &[], b"").await?; + match r.status().as_u16() { + 404 => Ok(false), + s if (200..300).contains(&s) => Ok(true), + _ => Err(http_status(r.status(), "HEAD")), + } + } + + /// 写一个对象。`exclusive`:带 `If-None-Match: *`,已经有了是 `Ok(false)`;服务端不认这个 + /// 头时退回到先看一眼再写 + async fn put_object( + &self, + path: &str, + bytes: &[u8], + exclusive: bool, + ) -> Result { + let url = self.object_url(&self.key(path)); + let ct = ("content-type", "application/octet-stream"); + if !exclusive { + return self.put_plain(&url, bytes).await.map(|()| true); + } + let r = self + .send(Method::PUT, &url, &[ct, ("if-none-match", "*")], bytes) + .await?; + let st = r.status(); + match st.as_u16() { + s if (200..300).contains(&s) => Ok(true), + // 已经有了;409 是另一个条件写入正在写同一个键 + 412 | 409 => Ok(false), + 501 => self.put_checked(path, bytes).await, + 400 => { + let code = Self::error_code(r).await; + if code == "NotImplemented" || code == "InvalidArgument" { + self.put_checked(path, bytes).await + } else { + Err(http_status(st, "PUT")) + } + } + _ => Err(http_status(st, "PUT")), + } + } + + async fn put_checked(&self, path: &str, bytes: &[u8]) -> Result { + if self.exists(path).await? { + return Ok(false); + } + let url = self.object_url(&self.key(path)); + self.put_plain(&url, bytes).await.map(|()| true) + } + + async fn put_plain(&self, url: &str, bytes: &[u8]) -> Result<(), StoreError> { + let ct = ("content-type", "application/octet-stream"); + let r = self.send(Method::PUT, url, &[ct], bytes).await?; + if r.status().is_success() { + Ok(()) + } else { + Err(http_status(r.status(), "PUT")) + } + } +} + +impl Store for S3 { + async fn list(&self, dir: &str) -> Result, StoreError> { + check_path(dir)?; + let prefix = format!("{}{dir}/", self.prefix); + let mut out = Vec::new(); + let mut token: Option = None; + // 一页一千个;同步位置上最多几个版本、几十台设备,翻页只是兜底 + for _ in 0..100 { + let mut url = format!( + "{}?delimiter=%2F&list-type=2&prefix={}", + self.bucket_url, + utf8_percent_encode(&prefix, QUERY) + ); + if let Some(t) = &token { + url.push_str(&format!( + "&continuation-token={}", + utf8_percent_encode(t, QUERY) + )); + } + let r = self.send(Method::GET, &url, &[], b"").await?; + if !r.status().is_success() { + return Err(http_status(r.status(), "ListObjectsV2")); + } + let body = r.text().await.map_err(|e| http_error(&e))?; + let Some((records, top)) = super::xml::parse(&body, "Contents") else { + return Err(StoreError::Other(msg!( + "sync.s3.bad_listing" => + "the S3 location answered a listing that could not be read" + ))); + }; + for r in records { + let Some(key) = r.get("Key") else { continue }; + let Some(name) = key.strip_prefix(&prefix) else { + continue; + }; + if name.is_empty() || name.contains('/') || name.starts_with('.') { + continue; + } + out.push(Entry { + name: name.to_string(), + size: r.get("Size").and_then(|s| s.parse().ok()).unwrap_or(0), + }); + } + match ( + top.get("IsTruncated").map(String::as_str), + top.get("NextContinuationToken"), + ) { + (Some("true"), Some(t)) => token = Some(t.clone()), + _ => break, + } + } + out.sort_by(|a, b| a.name.cmp(&b.name)); + Ok(out) + } + + async fn get(&self, path: &str) -> Result>, StoreError> { + check_path(path)?; + let url = self.object_url(&self.key(path)); + let r = self.send(Method::GET, &url, &[], b"").await?; + match r.status().as_u16() { + 404 => Ok(None), + s if (200..300).contains(&s) => { + Ok(Some(r.bytes().await.map_err(|e| http_error(&e))?.to_vec())) + } + _ => Err(http_status(r.status(), "GET")), + } + } + + async fn put_new(&self, path: &str, bytes: &[u8]) -> Result<(), StoreError> { + check_path(path)?; + if self.put_object(path, bytes, true).await? { + Ok(()) + } else { + Err(StoreError::Exists { + path: path.to_string(), + }) + } + } + + async fn put(&self, path: &str, bytes: &[u8]) -> Result<(), StoreError> { + check_path(path)?; + self.put_object(path, bytes, false).await.map(|_| ()) + } + + async fn delete(&self, path: &str) -> Result<(), StoreError> { + check_path(path)?; + let url = self.object_url(&self.key(path)); + let r = self.send(Method::DELETE, &url, &[], b"").await?; + match r.status().as_u16() { + 404 => Ok(()), + s if (200..300).contains(&s) => Ok(()), + _ => Err(http_status(r.status(), "DELETE")), + } + } + + async fn create_exclusive(&self, path: &str, bytes: &[u8]) -> Result { + check_path(path)?; + self.put_object(path, bytes, true).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn s3(endpoint: &str, bucket: &str, prefix: &str) -> S3 { + S3::new( + Config { + endpoint: endpoint.into(), + region: "auto".into(), + bucket: bucket.into(), + prefix: prefix.into(), + access_key_id: "AK".into(), + secret_access_key: "SK".into(), + }, + reqwest::Client::new(), + ) + .unwrap() + } + + #[test] + fn the_address_style_follows_the_provider() { + let aws = s3("https://s3.us-east-1.amazonaws.com", "tw-b", ""); + assert_eq!(aws.bucket_url, "https://tw-b.s3.us-east-1.amazonaws.com/"); + let oss = s3("https://oss-cn-hangzhou.aliyuncs.com", "tw-b", "me"); + assert_eq!(oss.bucket_url, "https://tw-b.oss-cn-hangzhou.aliyuncs.com/"); + assert_eq!(oss.key("keyring"), "me/keyring"); + let pasted = s3( + "https://tw-b.cos.ap-guangzhou.myqcloud.com", + "tw-b", + "/a/b/", + ); + assert_eq!( + pasted.bucket_url, + "https://tw-b.cos.ap-guangzhou.myqcloud.com/" + ); + assert_eq!(pasted.key("versions/x"), "a/b/versions/x"); + let r2 = s3("https://abc123.r2.cloudflarestorage.com", "tw-b", ""); + assert_eq!( + r2.bucket_url, + "https://abc123.r2.cloudflarestorage.com/tw-b/" + ); + let minio = s3("http://192.168.1.10:9000", "tw-b", ""); + assert_eq!(minio.bucket_url, "http://192.168.1.10:9000/tw-b/"); + assert_eq!( + minio.object_url("versions/0000000001-aaaaaaaa.twconfig"), + "http://192.168.1.10:9000/tw-b/versions/0000000001-aaaaaaaa.twconfig" + ); + } + + #[test] + fn the_addressing_is_chosen_by_the_endpoint_host_alone() { + use Addressing::*; + for (host, want) in [ + ("s3.us-east-1.amazonaws.com", VirtualHost), + ("s3.amazonaws.com", VirtualHost), + ("amazonaws.com", VirtualHost), + ("oss-cn-hangzhou.aliyuncs.com", VirtualHost), + ("cos.ap-guangzhou.myqcloud.com", VirtualHost), + ("S3.EU-WEST-1.AMAZONAWS.COM", VirtualHost), + ("abc123.r2.cloudflarestorage.com", Path), + ("192.168.1.10", Path), + ("nas", Path), + ("minio.example.com", Path), + ("tw-b.minio.lan", Path), + ("notamazonaws.com", Path), + ("amazonaws.com.evil.example", Path), + ] { + assert_eq!(addressing(host), want, "{host}"); + } + assert_eq!( + bucket_url("http://tw-b.minio.lan:9000", "tw-b").unwrap(), + "http://tw-b.minio.lan:9000/tw-b/", + "路径方式不猜桶名是不是已经在域名里" + ); + assert_eq!( + bucket_url("https://minio.example.com/s3/", "tw-b").unwrap(), + "https://minio.example.com/s3/tw-b/" + ); + } + + #[test] + fn a_public_plain_http_endpoint_is_refused() { + let r = S3::new( + Config { + endpoint: "http://s3.example.com".into(), + region: "auto".into(), + bucket: "b-1".into(), + prefix: String::new(), + access_key_id: "AK".into(), + secret_access_key: "SK".into(), + }, + reqwest::Client::new(), + ); + assert!(r.is_err()); + } +} diff --git a/crates/tw-sync/src/store/webdav.rs b/crates/tw-sync/src/store/webdav.rs new file mode 100644 index 00000000..3b6a85fa --- /dev/null +++ b/crates/tw-sync/src/store/webdav.rs @@ -0,0 +1,366 @@ +//! WebDAV 位置:坚果云、Nextcloud、群晖 NAS、InfiniCloud。 +//! +//! 只用五个方法:`PROPFIND`(列目录,Depth 1)、`GET`、`PUT`、`DELETE`、`MOVE`,外加建目录的 +//! `MKCOL`。写入先 `PUT` 到一个点开头的临时名,再 `MOVE` 过去(只新增时带 `Overwrite: F`); +//! 「仅当不存在时创建」用 `If-None-Match: *`。 +//! +//! **证书一律校验**,`http://` 只给本机和内网(见 [`tw_config::sync::address_ok`]):这条 +//! 连接上传的是同步位置的密码,和可能没加密的整份配置。**不跟重定向**:跟到别的主机, +//! 密码就跟着去了。 + +use std::time::Duration; + +use reqwest::{Method, StatusCode}; +use tw_types::msg; + +use super::{Entry, Store, StoreError, check_path, http_error, http_status, split, temp_name}; + +/// 一个请求最多等多久。版本最大 16 MiB,慢的网盘也够了 +const TIMEOUT: Duration = Duration::from_secs(120); + +pub struct WebDav { + /// 以 `/` 结尾 + base: url::Url, + user: String, + pass: String, + http: reqwest::Client, +} + +fn method(m: &'static str) -> Method { + Method::from_bytes(m.as_bytes()).expect("a valid method name") +} + +impl WebDav { + pub fn new( + url: &str, + user: &str, + pass: &str, + http: reqwest::Client, + ) -> Result { + if !tw_config::sync::address_ok(url) { + return Err(StoreError::Other( + tw_config::sync::SyncError::Address { + field: "url", + url: url.to_string(), + } + .msg(), + )); + } + let mut s = url.trim().to_string(); + if !s.ends_with('/') { + s.push('/'); + } + let base = url::Url::parse(&s).map_err(|e| { + StoreError::Other(msg!( + "sync.webdav.bad_url", url = url, detail = e => + "{url} is not a WebDAV address: {detail}" + )) + })?; + Ok(Self { + base, + user: user.to_string(), + pass: pass.to_string(), + http, + }) + } + + /// 文件的地址。路径只含我们自己生成的字符,拼上去不用转义 + fn file(&self, path: &str) -> Result { + check_path(path)?; + self.base.join(path).map_err(|e| { + StoreError::Other(msg!( + "sync.webdav.bad_url", url = path, detail = e => + "{url} is not a WebDAV address: {detail}" + )) + }) + } + + /// 目录的地址,以 `/` 结尾。空的是根 + fn dir(&self, dir: &str) -> Result { + if dir.is_empty() { + return Ok(self.base.clone()); + } + let mut u = self.file(dir)?; + u.set_path(&format!("{}/", u.path().trim_end_matches('/'))); + Ok(u) + } + + fn req(&self, m: Method, url: url::Url) -> reqwest::RequestBuilder { + let r = self.http.request(m, url).timeout(TIMEOUT); + if self.user.is_empty() && self.pass.is_empty() { + r + } else { + r.basic_auth(&self.user, Some(&self.pass)) + } + } + + async fn send(&self, r: reqwest::RequestBuilder) -> Result { + r.send().await.map_err(|e| http_error(&e)) + } + + /// 建目录。已经有了(405)不算错;上一级没有(409)先建上一级(同步位置的根) + async fn mkcol(&self, dir: &str) -> Result<(), StoreError> { + let url = self.dir(dir)?; + let r = self.send(self.req(method("MKCOL"), url.clone())).await?; + match r.status().as_u16() { + 200 | 201 | 204 | 405 => Ok(()), + 409 if !dir.is_empty() => { + Box::pin(self.mkcol("")).await?; + let r = self.send(self.req(method("MKCOL"), url)).await?; + match r.status().as_u16() { + 200 | 201 | 204 | 405 => Ok(()), + _ => Err(http_status(r.status(), "MKCOL")), + } + } + _ => Err(http_status(r.status(), "MKCOL")), + } + } + + async fn put_raw( + &self, + url: url::Url, + bytes: &[u8], + if_none_match: bool, + ) -> Result { + let mut r = self + .req(Method::PUT, url) + .header("content-type", "application/octet-stream") + .body(bytes.to_vec()); + if if_none_match { + r = r.header("if-none-match", "*"); + } + Ok(self.send(r).await?.status()) + } + + /// 写临时名、改名过去。`overwrite` 是 MOVE 的 `Overwrite`。返回 MOVE 的状态,服务端 + /// 不支持 MOVE(405、501)时是 `None`,临时文件已经删掉 + async fn via_temp( + &self, + path: &str, + bytes: &[u8], + overwrite: bool, + ) -> Result, StoreError> { + let (dir, name) = split(path); + let tmp_path = if dir.is_empty() { + temp_name(name) + } else { + format!("{dir}/{}", temp_name(name)) + }; + let tmp = self.file(&tmp_path)?; + let dest = self.file(path)?; + let st = self.put_raw(tmp.clone(), bytes, false).await?; + if !st.is_success() { + return Err(http_status(st, "PUT")); + } + let r = self + .send( + self.req(method("MOVE"), tmp.clone()) + .header("destination", dest.as_str()) + .header("overwrite", if overwrite { "T" } else { "F" }), + ) + .await; + let st = match r { + Ok(r) => r.status(), + Err(e) => { + let _ = self.send(self.req(Method::DELETE, tmp)).await; + return Err(e); + } + }; + if st.is_success() { + return Ok(Some(st)); + } + let _ = self.send(self.req(Method::DELETE, tmp)).await; + if matches!(st.as_u16(), 405 | 501) { + return Ok(None); + } + Ok(Some(st)) + } + + async fn exists(&self, path: &str) -> Result { + let r = self.send(self.req(Method::HEAD, self.file(path)?)).await?; + match r.status().as_u16() { + 404 => Ok(false), + s if (200..300).contains(&s) => Ok(true), + _ => Err(http_status(r.status(), "HEAD")), + } + } +} + +/// `href` 里最后一段,解码之后。目录以 `/` 结尾,去掉再取 +fn last_segment(href: &str) -> String { + let path = match url::Url::parse(href) { + Ok(u) => u.path().to_string(), + Err(_) => href.to_string(), + }; + let seg = path.trim_end_matches('/').rsplit('/').next().unwrap_or(""); + percent_encoding::percent_decode_str(seg) + .decode_utf8_lossy() + .to_string() +} + +const PROPFIND: &str = r#" +"#; + +impl Store for WebDav { + async fn list(&self, dir: &str) -> Result, StoreError> { + let url = self.dir(dir)?; + let own = url.path().trim_end_matches('/').to_string(); + let r = self + .send( + self.req(method("PROPFIND"), url) + .header("depth", "1") + .header("content-type", "application/xml; charset=utf-8") + .body(PROPFIND), + ) + .await?; + match r.status().as_u16() { + 207 | 200 => {} + 404 => return Ok(Vec::new()), + _ => return Err(http_status(r.status(), "PROPFIND")), + } + let body = r.text().await.map_err(|e| http_error(&e))?; + let Some((records, _)) = super::xml::parse(&body, "response") else { + return Err(StoreError::Other(msg!( + "sync.webdav.bad_listing" => + "the WebDAV server answered a directory listing that could not be read" + ))); + }; + let mut out = Vec::new(); + for r in records { + let Some(href) = r.get("href") else { continue }; + let path = match url::Url::parse(href) { + Ok(u) => u.path().to_string(), + Err(_) => href.to_string(), + }; + if r.has("collection") || path.trim_end_matches('/') == own { + continue; + } + let name = last_segment(href); + if name.is_empty() || name.starts_with('.') { + continue; + } + out.push(Entry { + name, + size: r + .get("getcontentlength") + .and_then(|s| s.parse().ok()) + .unwrap_or(0), + }); + } + out.sort_by(|a, b| a.name.cmp(&b.name)); + Ok(out) + } + + async fn get(&self, path: &str) -> Result>, StoreError> { + let r = self.send(self.req(Method::GET, self.file(path)?)).await?; + match r.status().as_u16() { + 404 | 410 => Ok(None), + s if (200..300).contains(&s) => { + Ok(Some(r.bytes().await.map_err(|e| http_error(&e))?.to_vec())) + } + _ => Err(http_status(r.status(), "GET")), + } + } + + async fn put_new(&self, path: &str, bytes: &[u8]) -> Result<(), StoreError> { + let (dir, _) = split(path); + self.mkcol(dir).await?; + let exists = || StoreError::Exists { + path: path.to_string(), + }; + match self.via_temp(path, bytes, false).await? { + Some(st) if st.is_success() => Ok(()), + Some(st) if st.as_u16() == 412 => Err(exists()), + Some(st) => Err(http_status(st, "MOVE")), + None => { + // 不支持 MOVE 的服务器:直接写,带上「不存在才写」 + let st = self.put_raw(self.file(path)?, bytes, true).await?; + match st.as_u16() { + 412 => Err(exists()), + s if (200..300).contains(&s) => Ok(()), + _ => Err(http_status(st, "PUT")), + } + } + } + } + + async fn put(&self, path: &str, bytes: &[u8]) -> Result<(), StoreError> { + let (dir, _) = split(path); + self.mkcol(dir).await?; + match self.via_temp(path, bytes, true).await? { + Some(st) if st.is_success() => Ok(()), + Some(st) => Err(http_status(st, "MOVE")), + None => { + let st = self.put_raw(self.file(path)?, bytes, false).await?; + if st.is_success() { + Ok(()) + } else { + Err(http_status(st, "PUT")) + } + } + } + } + + async fn delete(&self, path: &str) -> Result<(), StoreError> { + let r = self + .send(self.req(Method::DELETE, self.file(path)?)) + .await?; + match r.status().as_u16() { + 404 | 410 => Ok(()), + s if (200..300).contains(&s) => Ok(()), + _ => Err(http_status(r.status(), "DELETE")), + } + } + + async fn create_exclusive(&self, path: &str, bytes: &[u8]) -> Result { + let (dir, _) = split(path); + self.mkcol(dir).await?; + // 先看一眼:有的服务器不认 If-None-Match,会直接覆盖 + if self.exists(path).await? { + return Ok(false); + } + let st = self.put_raw(self.file(path)?, bytes, true).await?; + match st.as_u16() { + 412 => Ok(false), + s if (200..300).contains(&s) => Ok(true), + _ => Err(http_status(st, "PUT")), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_plain_http_address_off_the_local_network_is_refused() { + let http = reqwest::Client::new(); + assert!(WebDav::new("http://dav.example.com/x", "u", "p", http.clone()).is_err()); + assert!(WebDav::new("https://dav.example.com/x", "u", "p", http.clone()).is_ok()); + assert!(WebDav::new("http://192.168.1.2:5005/x", "u", "p", http).is_ok()); + } + + #[test] + fn hrefs_are_decoded_whether_absolute_or_not() { + assert_eq!(last_segment("/dav/a%20b/versions/x.twconfig"), "x.twconfig"); + assert_eq!(last_segment("https://h/dav/tw/devices/"), "devices"); + assert_eq!(last_segment("/dav/%E5%90%8C%E6%AD%A5/"), "同步"); + } + + #[test] + fn files_and_directories_resolve_under_the_base() { + let d = WebDav::new("https://h/dav/ThinkWatch", "u", "p", reqwest::Client::new()).unwrap(); + assert_eq!( + d.file("versions/0000000001-aaaaaaaa.twconfig") + .unwrap() + .as_str(), + "https://h/dav/ThinkWatch/versions/0000000001-aaaaaaaa.twconfig" + ); + assert_eq!( + d.dir("versions").unwrap().as_str(), + "https://h/dav/ThinkWatch/versions/" + ); + assert_eq!(d.dir("").unwrap().as_str(), "https://h/dav/ThinkWatch/"); + assert!(d.file("../x").is_err()); + } +} diff --git a/crates/tw-sync/src/store/xml.rs b/crates/tw-sync/src/store/xml.rs new file mode 100644 index 00000000..7d9642aa --- /dev/null +++ b/crates/tw-sync/src/store/xml.rs @@ -0,0 +1,160 @@ +//! 读 WebDAV 的 PROPFIND 和 S3 的 ListObjectsV2、错误答复:只要几个元素的文字。 +//! +//! **按本地名认元素,不管命名空间前缀**:同一个 `DAV:` 命名空间,有的服务器写 `D:href`, +//! 有的写 `d:href`、`lp1:getcontentlength`,还有的不带前缀。 + +use std::collections::BTreeMap; + +use quick_xml::events::Event; + +/// 一条记录:一个 `record` 元素底下各个元素的文字(同名的取第一个),以及出现过的空元素 +/// (`` 这种只看在不在的) +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct Record { + pub text: BTreeMap, + pub present: std::collections::BTreeSet, +} + +impl Record { + pub fn get(&self, name: &str) -> Option<&str> { + self.text.get(name).map(String::as_str) + } + pub fn has(&self, name: &str) -> bool { + self.present.contains(name) || self.text.contains_key(name) + } +} + +/// 预定义的五个实体和字符引用。认不出的原样留着 +fn entity(name: &str) -> String { + match name { + "amp" => "&".into(), + "lt" => "<".into(), + "gt" => ">".into(), + "quot" => "\"".into(), + "apos" => "'".into(), + _ => { + let n = name + .strip_prefix("#x") + .or_else(|| name.strip_prefix("#X")) + .and_then(|h| u32::from_str_radix(h, 16).ok()) + .or_else(|| name.strip_prefix('#').and_then(|d| d.parse().ok())); + match n.and_then(char::from_u32) { + Some(c) => c.to_string(), + None => format!("&{name};"), + } + } + } +} + +/// 文档里每个本地名是 `record` 的元素一条记录;`top` 是不在任何记录里的那些元素的文字。 +/// 读不了的 XML 返回 `None` +pub fn parse(xml: &str, record: &str) -> Option<(Vec, BTreeMap)> { + let mut reader = quick_xml::Reader::from_str(xml); + let mut stack: Vec = Vec::new(); + let mut records = Vec::new(); + let mut top = BTreeMap::new(); + // 正在读的那一条记录,以及它在栈里的深度 + let mut cur: Option<(usize, Record)> = None; + let mut text = String::new(); + loop { + match reader.read_event().ok()? { + Event::Start(e) => { + let name = e.local_name().as_ref().to_string(); + stack.push(name.clone()); + text.clear(); + if name == record && cur.is_none() { + cur = Some((stack.len(), Record::default())); + } + } + Event::Empty(e) => { + let name = e.local_name().as_ref().to_string(); + if let Some((_, r)) = cur.as_mut() { + r.present.insert(name); + } + } + Event::Text(t) => text.push_str(&t), + Event::CData(t) => text.push_str(&t), + Event::GeneralRef(r) => text.push_str(&entity(&r)), + Event::End(_) => { + let name = stack.pop()?; + let value = std::mem::take(&mut text).trim().to_string(); + match cur.as_mut() { + Some((depth, _)) if *depth == stack.len() + 1 => { + if let Some((_, r)) = cur.take() { + records.push(r); + } + } + Some((_, r)) => { + r.present.insert(name.clone()); + if !value.is_empty() { + r.text.entry(name).or_insert(value); + } + } + None => { + if !value.is_empty() { + top.entry(name).or_insert(value); + } + } + } + } + Event::Eof => break, + _ => {} + } + } + Some((records, top)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn webdav_multistatus_with_any_prefix() { + let xml = r#" + + + /dav/tw/versions/ + + + + /dav/tw/versions/0000000001-aaaaaaaa.twconfig + + 1234 + + HTTP/1.1 200 OK + +"#; + let (rs, _) = parse(xml, "response").unwrap(); + assert_eq!(rs.len(), 2); + assert!(rs[0].has("collection")); + assert_eq!( + rs[1].get("href"), + Some("/dav/tw/versions/0000000001-aaaaaaaa.twconfig") + ); + assert_eq!(rs[1].get("getcontentlength"), Some("1234")); + assert!(!rs[1].has("collection")); + } + + #[test] + fn s3_listing_and_its_continuation() { + let xml = r#" +bp/versions/true +tok&en +p/versions/a3 +p/versions/b&c4 +"#; + let (rs, top) = parse(xml, "Contents").unwrap(); + assert_eq!(rs.len(), 2); + assert_eq!(rs[1].get("Key"), Some("p/versions/b&c")); + assert_eq!(top.get("IsTruncated").map(String::as_str), Some("true")); + assert_eq!( + top.get("NextContinuationToken").map(String::as_str), + Some("tok&en") + ); + } + + #[test] + fn broken_xml_is_none_not_a_panic() { + assert!(parse("", "b").is_none()); + } +} diff --git a/crates/tw-sync/tests/common/mod.rs b/crates/tw-sync/tests/common/mod.rs new file mode 100644 index 00000000..8f758d12 --- /dev/null +++ b/crates/tw-sync/tests/common/mod.rs @@ -0,0 +1,323 @@ +//! 进程内的假 WebDAV、假 S3:内容放在内存里,几十行。 +//! +//! **只实现同步用到的那几样**,但按规矩答:`PUT` 认 `If-None-Match: *`、`MOVE` 认 +//! `Overwrite: F`、`PROPFIND` 只列一层、S3 的列目录认 `prefix` 和 `delimiter`。凭据不对回 401 / +//! 403。可以让它「坏掉」:回 503,或者不认 `MOVE`、`If-None-Match`(有的服务器就是这样)。 + +#![allow(dead_code)] + +use std::collections::BTreeMap; +use std::net::SocketAddr; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; + +use axum::body::Body; +use axum::http::{HeaderMap, Request, Response, StatusCode}; + +pub const USER: &str = "tester"; +pub const PASS: &str = "app-password"; +pub const AK: &str = "AKIDFAKESYNCTEST"; +pub const SK: &str = "fake-secret-for-tests-only"; + +/// 假服务端的开关 +#[derive(Default)] +pub struct Knobs { + /// 一律回 503 + pub down: AtomicBool, + /// WebDAV 不认 MOVE(405) + pub no_move: AtomicBool, + /// 不认 `If-None-Match`:照写不误(WebDAV)或者回 501(S3) + pub no_conditional: AtomicBool, + /// 收到的请求数 + pub requests: AtomicUsize, +} + +pub struct Fake { + pub addr: SocketAddr, + /// 路径(不带开头的 `/`,WebDAV 是去掉根之后的,S3 是桶内的键)→ 内容 + pub files: Arc>>>, + pub knobs: Arc, +} + +impl Fake { + pub fn url(&self) -> String { + format!("http://{}", self.addr) + } + pub fn names(&self) -> Vec { + self.files.lock().unwrap().keys().cloned().collect() + } + pub fn get(&self, path: &str) -> Option> { + self.files.lock().unwrap().get(path).cloned() + } +} + +async fn serve( + handler: impl Fn(Request, Vec) -> Response + Send + Sync + 'static, +) -> SocketAddr { + let handler = Arc::new(handler); + let app = axum::Router::new().fallback(move |req: Request| { + let handler = handler.clone(); + async move { + let (parts, body) = req.into_parts(); + let bytes = axum::body::to_bytes(body, usize::MAX) + .await + .unwrap_or_default() + .to_vec(); + handler(Request::from_parts(parts, Body::empty()), bytes) + } + }); + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = l.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(l, app).await.unwrap() }); + addr +} + +fn status(s: u16) -> Response { + Response::builder() + .status(StatusCode::from_u16(s).unwrap()) + .body(Body::empty()) + .unwrap() +} + +fn header<'a>(h: &'a HeaderMap, name: &str) -> Option<&'a str> { + h.get(name).and_then(|v| v.to_str().ok()) +} + +/// WebDAV,根在 `/dav/root/`。 +pub async fn webdav() -> Fake { + let files: Arc>>> = Default::default(); + let dirs: Arc>> = Default::default(); + let knobs: Arc = Default::default(); + let (f, d, k) = (files.clone(), dirs.clone(), knobs.clone()); + let addr = serve(move |req, body| { + k.requests.fetch_add(1, Ordering::Relaxed); + if k.down.load(Ordering::Relaxed) { + return status(503); + } + use base64::Engine as _; + let want = format!( + "Basic {}", + base64::engine::general_purpose::STANDARD.encode(format!("{USER}:{PASS}")) + ); + if header(req.headers(), "authorization") != Some(want.as_str()) { + return status(401); + } + let Some(rel) = req.uri().path().strip_prefix("/dav/root/") else { + // 根本身 + if req.uri().path().trim_end_matches('/') == "/dav/root" { + return match req.method().as_str() { + "MKCOL" => status(405), + "PROPFIND" => status(207), + _ => status(405), + }; + } + return status(404); + }; + let rel = percent_encoding::percent_decode_str(rel) + .decode_utf8_lossy() + .to_string(); + let mut files = f.lock().unwrap(); + let mut dirs = d.lock().unwrap(); + match req.method().as_str() { + "MKCOL" => { + let dir = rel.trim_end_matches('/').to_string(); + if dir.contains('/') && !dirs.contains(dir.rsplit_once('/').unwrap().0) { + return status(409); + } + if dirs.insert(dir) { status(201) } else { status(405) } + } + "PROPFIND" => { + let dir = rel.trim_end_matches('/').to_string(); + if !dirs.contains(&dir) { + return status(404); + } + let mut xml = String::from( + r#""#, + ); + xml.push_str(&format!( + "/dav/root/{dir}/\ + " + )); + for (name, bytes) in files.iter() { + if let Some((parent, leaf)) = name.rsplit_once('/') + && parent == dir + { + xml.push_str(&format!( + "/dav/root/{parent}/{leaf}\ + {}\ + HTTP/1.1 200 OK", + bytes.len() + )); + } + } + xml.push_str(""); + Response::builder() + .status(207) + .header("content-type", "application/xml") + .body(Body::from(xml)) + .unwrap() + } + "GET" | "HEAD" => match files.get(&rel) { + Some(b) if req.method() == "GET" => Response::new(Body::from(b.clone())), + Some(_) => status(200), + None => status(404), + }, + "PUT" => { + let parent = rel.rsplit_once('/').map(|(p, _)| p).unwrap_or(""); + if !parent.is_empty() && !dirs.contains(parent) { + return status(409); + } + if header(req.headers(), "if-none-match") == Some("*") + && !k.no_conditional.load(Ordering::Relaxed) + && files.contains_key(&rel) + { + return status(412); + } + let existed = files.insert(rel, body).is_some(); + status(if existed { 204 } else { 201 }) + } + "DELETE" => match files.remove(&rel) { + Some(_) => status(204), + None => status(404), + }, + "MOVE" => { + if k.no_move.load(Ordering::Relaxed) { + return status(405); + } + let Some(dest) = header(req.headers(), "destination") else { + return status(400); + }; + let dest_path = url::Url::parse(dest) + .map(|u| u.path().to_string()) + .unwrap_or_else(|_| dest.to_string()); + let Some(dest_rel) = dest_path.strip_prefix("/dav/root/") else { + return status(502); + }; + let dest_rel = dest_rel.to_string(); + let overwrite = header(req.headers(), "overwrite") != Some("F"); + if !overwrite && files.contains_key(&dest_rel) { + return status(412); + } + let Some(b) = files.remove(&rel) else { + return status(404); + }; + let existed = files.insert(dest_rel, b).is_some(); + status(if existed { 204 } else { 201 }) + } + _ => status(405), + } + }) + .await; + // 根在 + Fake { addr, files, knobs } +} + +/// S3,路径方式,桶叫 `tw-bucket`。 +pub async fn s3() -> Fake { + let files: Arc>>> = Default::default(); + let knobs: Arc = Default::default(); + let (f, k) = (files.clone(), knobs.clone()); + let addr = serve(move |req, body| { + k.requests.fetch_add(1, Ordering::Relaxed); + if k.down.load(Ordering::Relaxed) { + return status(503); + } + let auth = header(req.headers(), "authorization").unwrap_or_default(); + if !auth.starts_with(&format!("AWS4-HMAC-SHA256 Credential={AK}/")) + || !auth.contains("/s3/aws4_request") + || header(req.headers(), "x-amz-content-sha256").is_none() + { + return Response::builder() + .status(403) + .body(Body::from( + "SignatureDoesNotMatch", + )) + .unwrap(); + } + let path = req.uri().path().to_string(); + let Some(key) = path.strip_prefix("/tw-bucket") else { + return status(404); + }; + let key = percent_encoding::percent_decode_str(key.trim_start_matches('/')) + .decode_utf8_lossy() + .to_string(); + let mut files = f.lock().unwrap(); + match req.method().as_str() { + "GET" if key.is_empty() => { + let q: BTreeMap = url::form_urlencoded::parse( + req.uri().query().unwrap_or_default().as_bytes(), + ) + .into_owned() + .collect(); + assert_eq!(q.get("list-type").map(String::as_str), Some("2")); + let prefix = q.get("prefix").cloned().unwrap_or_default(); + let delim = q.get("delimiter").cloned(); + let mut xml = String::from( + r#"false"#, + ); + for (name, bytes) in files.iter() { + let Some(rest) = name.strip_prefix(&prefix) else { + continue; + }; + if delim.as_deref() == Some("/") && rest.contains('/') { + continue; + } + xml.push_str(&format!( + "{name}{}", + bytes.len() + )); + } + xml.push_str(""); + Response::new(Body::from(xml)) + } + "GET" => match files.get(&key) { + Some(b) => Response::new(Body::from(b.clone())), + None => Response::builder() + .status(404) + .body(Body::from("NoSuchKey")) + .unwrap(), + }, + "HEAD" => status(if files.contains_key(&key) { 200 } else { 404 }), + "PUT" => { + if header(req.headers(), "if-none-match") == Some("*") { + if k.no_conditional.load(Ordering::Relaxed) { + return Response::builder() + .status(501) + .body(Body::from("NotImplemented")) + .unwrap(); + } + if files.contains_key(&key) { + return status(412); + } + } + files.insert(key, body); + status(200) + } + "DELETE" => { + files.remove(&key); + status(204) + } + _ => status(405), + } + }) + .await; + Fake { addr, files, knobs } +} + +/// 一个没人听的地址:连上去就被拒绝 +pub fn nowhere() -> String { + let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let a = l.local_addr().unwrap(); + drop(l); + format!("http://{a}") +} + +/// 测试里的 HTTP 客户端:不走系统代理,User-Agent 用通用值 +pub fn http() -> reqwest::Client { + reqwest::Client::builder() + .no_proxy() + .user_agent("sync-test") + .redirect(reqwest::redirect::Policy::none()) + .build() + .unwrap() +} diff --git a/crates/tw-sync/tests/devices.rs b/crates/tw-sync/tests/devices.rs new file mode 100644 index 00000000..add592cf --- /dev/null +++ b/crates/tw-sync/tests/devices.rs @@ -0,0 +1,747 @@ +//! 两台设备(两个独立的数据目录、两份配置)经同一个同步位置交替修改、同时上传、冲突、离线、 +//! 加密开关与改口令、版本过新、不同步的上游、已接管客户端的密钥。 +//! +//! 设备这一边([`Host`])是最简单的那种:配置就是数据目录里的 `config.yaml`,应用就是校验 +//! 之后写回去。控制面那一层(写入流程、事件)在 `tw-control` 的测试里另测。 + +mod common; + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; + +use tw_pack::{Pack, PackKind, PackOrigin}; +use tw_sync::engine::{ApplyError, Current, Decision, Engine, Host, RoundError, Settings, Take}; +use tw_sync::store::{AnyStore, Folder}; +use tw_sync::{Store, StoreError}; +use tw_types::Msg; + +const PASS: &str = "correct horse battery staple"; + +/// 一台设备的配置:控制面钥匙各不相同(本机设置,不该被同步走) +fn base_config(control_key_byte: char) -> String { + format!( + "version: 1\nlisten:\n control:\n key: {}\nclients:\n - name: default\n key: tw-default-key-0001\n", + control_key_byte.to_string().repeat(64) + ) +} + +#[derive(Clone)] +struct Dev { + dir: Arc, + pinned: Arc>>, + /// 应用过几次,每次是从哪些设备来的 + applied: Arc>>, +} + +impl Dev { + fn new(key: char) -> Self { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("config.yaml"), base_config(key)).unwrap(); + Self { + dir: Arc::new(dir), + pinned: Default::default(), + applied: Default::default(), + } + } + fn path(&self) -> PathBuf { + self.dir.path().join("config.yaml") + } + fn text(&self) -> String { + std::fs::read_to_string(self.path()).unwrap() + } + fn config(&self) -> tw_config::Config { + tw_config::try_parse(&self.text()).unwrap_or_else(|e| panic!("{e}\n{}", self.text())) + } + /// 用户改配置:在原文上追加或者整段改 + fn edit(&self, f: impl FnOnce(&str) -> String) { + let next = f(&self.text()); + tw_config::try_parse(&next).unwrap_or_else(|e| panic!("测试写坏了配置:{e}\n{next}")); + std::fs::write(self.path(), next).unwrap(); + } + fn provider(&self, name: &str) -> Option { + self.config().providers.into_iter().find(|p| p.name == name) + } +} + +impl Host for Dev { + fn current(&self) -> Result { + let text = self.text(); + Ok(Current { + pack: Pack { + manifest: tw_pack::Manifest::new(PackKind::Full, PackOrigin::Ui), + config_yaml: text.clone(), + plugins: Default::default(), + client_prefs: None, + }, + version: tw_config::version_of(&text), + changed_ms: 1, + }) + } + async fn apply(&self, pack: Pack, base_version: &str, from: &str) -> Result<(), ApplyError> { + if tw_config::version_of(&self.text()) != base_version { + return Err(ApplyError::Stale); + } + tw_config::try_parse(&pack.config_yaml).map_err(|r| ApplyError::Rejected(r.msg()))?; + std::fs::write(self.path(), &pack.config_yaml).unwrap(); + self.applied.lock().unwrap().push(from.to_string()); + Ok(()) + } + fn pinned_keys(&self) -> BTreeSet { + self.pinned.lock().unwrap().clone() + } + fn backup(&self, _content_id: &str) -> Option { + None + } +} + +fn settings(id: &str, name: &str, encrypt: bool, place: &Path) -> Settings { + Settings { + device_id: id.into(), + device_name: name.into(), + encrypt, + passphrase: encrypt.then(|| PASS.to_string()), + exclude: BTreeSet::new(), + place: format!("folder:{}", place.display()), + remote_core: false, + } +} + +const A_ID: &str = "aaaaaaaa-0000-4000-8000-000000000001"; +const B_ID: &str = "bbbbbbbb-0000-4000-8000-000000000002"; + +struct Bed { + loc: tempfile::TempDir, + a: Dev, + b: Dev, + ea: Engine, + eb: Engine, +} + +fn folder(loc: &Path) -> AnyStore { + AnyStore::Folder(Folder::new(loc)) +} + +/// A 开启同步(同步位置空着),B 用同步数据加入 +async fn bed(encrypt: bool) -> Bed { + tw_sync::keyring::set_work_factor_for_tests(4); + let loc = tempfile::tempdir().unwrap(); + let (a, b) = (Dev::new('a'), Dev::new('b')); + let mut ea = Engine::new( + folder(loc.path()), + a.clone(), + a.dir.path(), + settings(A_ID, "Mac mini", encrypt, loc.path()), + ); + let mut eb = Engine::new( + folder(loc.path()), + b.clone(), + b.dir.path(), + settings(B_ID, "MacBook Air", encrypt, loc.path()), + ); + assert!(ea.create().await.unwrap(), "空的同步位置开得起来"); + eb.join(None).await.unwrap(); + Bed { loc, a, b, ea, eb } +} + +fn provider_yaml(name: &str, url: &str, key: &str) -> String { + format!(" - name: {name}\n base_url: {url}\n key: {key}\n") +} + +fn add_provider(text: &str, name: &str, url: &str, key: &str) -> String { + if text.contains("\nproviders:\n") { + text.replacen( + "\nproviders:\n", + &format!("\nproviders:\n{}", provider_yaml(name, url, key)), + 1, + ) + } else { + format!("{text}providers:\n{}", provider_yaml(name, url, key)) + } +} + +fn versions(loc: &Path) -> Vec { + let mut v: Vec = std::fs::read_dir(loc.join("versions")) + .map(|rd| { + rd.filter_map(|e| e.ok()) + .map(|e| e.file_name().to_string_lossy().to_string()) + .filter(|n| !n.starts_with('.')) + .collect() + }) + .unwrap_or_default(); + v.sort(); + v +} + +#[tokio::test] +async fn edits_on_either_device_reach_the_other_and_local_settings_stay_put() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider(s, "relay", "https://relay.example/v1", "sk-relay")); + let r = t.ea.round().await.unwrap(); + assert!(r.uploaded.is_some(), "A 的改动要上传"); + let r = t.eb.round().await.unwrap(); + assert_eq!(r.applied_from, ["Mac mini"]); + let p = t.b.provider("relay").expect("B 收到了 relay"); + assert_eq!( + p.key.as_ref().map(|k| k.raw()), + Some("sk-relay"), + "密钥随同步带过去" + ); + assert!( + t.b.text().contains(&"b".repeat(64)), + "B 的控制面钥匙是它自己的" + ); + + // B 改,A 收 + t.b.edit(|s| s.replace("https://relay.example/v1", "https://relay2.example/v1")); + assert!(t.eb.round().await.unwrap().uploaded.is_some()); + t.ea.round().await.unwrap(); + assert_eq!( + t.a.provider("relay").unwrap().base_url, + "https://relay2.example/v1" + ); + assert!(t.a.text().contains(&"a".repeat(64))); + + // 都没变:不再上传、不再应用 + let before = versions(t.loc.path()); + let ra = t.ea.round().await.unwrap(); + let rb = t.eb.round().await.unwrap(); + assert!(ra.uploaded.is_none() && rb.uploaded.is_none()); + assert!(ra.applied_from.is_empty() && rb.applied_from.is_empty()); + assert_eq!(versions(t.loc.path()), before); + // 同步位置只留最新与上一个 + assert!(before.len() <= 2, "{before:?}"); + // 两台设备都在设备列表里 + let names: Vec = rb.devices.iter().map(|d| d.name.clone()).collect(); + assert_eq!(names, ["Mac mini", "MacBook Air"]); +} + +#[tokio::test] +async fn different_objects_changed_on_both_sides_merge_without_a_conflict() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider(s, "one", "https://one.example", "sk-1")); + t.b.edit(|s| add_provider(s, "two", "https://two.example", "sk-2")); + t.ea.round().await.unwrap(); + let r = t.eb.round().await.unwrap(); + assert!(r.conflicts.is_empty(), "{:?}", r.conflicts); + assert!(r.uploaded.is_some() && !r.applied_from.is_empty()); + t.ea.round().await.unwrap(); + for d in [&t.a, &t.b] { + assert!( + d.provider("one").is_some() && d.provider("two").is_some(), + "{}", + d.text() + ); + } +} + +/// 一个存储,**假装还没收到某些文件**:云盘里别的设备的版本还没传到这台设备上 +struct Lagging { + inner: AnyStore, + hidden: Arc>>, +} + +impl Store for Lagging { + async fn list(&self, dir: &str) -> Result, StoreError> { + let hidden = self.hidden.lock().unwrap().clone(); + Ok(self + .inner + .list(dir) + .await? + .into_iter() + .filter(|e| !hidden.contains(&e.name)) + .collect()) + } + async fn get(&self, path: &str) -> Result>, StoreError> { + if self + .hidden + .lock() + .unwrap() + .iter() + .any(|h| path.ends_with(h.as_str())) + { + return Ok(None); + } + self.inner.get(path).await + } + async fn put_new(&self, path: &str, bytes: &[u8]) -> Result<(), StoreError> { + self.inner.put_new(path, bytes).await + } + async fn put(&self, path: &str, bytes: &[u8]) -> Result<(), StoreError> { + self.inner.put(path, bytes).await + } + async fn delete(&self, path: &str) -> Result<(), StoreError> { + self.inner.delete(path).await + } + async fn create_exclusive(&self, path: &str, bytes: &[u8]) -> Result { + self.inner.create_exclusive(path, bytes).await + } +} + +/// 两台设备同时上传:序号相同的两个版本,第三台(这里是 A 自己的下一轮)以它们共同的上一版本 +/// 为基准合并;合并之前它们和共同的上一版本都留着 +#[tokio::test] +async fn simultaneous_uploads_are_merged_on_their_common_parent() { + tw_sync::keyring::set_work_factor_for_tests(4); + let t = bed(false).await; + let hidden: Arc>> = Default::default(); + let mut eb = Engine::new( + Lagging { + inner: folder(t.loc.path()), + hidden: hidden.clone(), + }, + t.b.clone(), + t.b.dir.path(), + settings(B_ID, "MacBook Air", false, t.loc.path()), + ); + let mut ea = t.ea; + t.a.edit(|s| add_provider(s, "one", "https://one.example", "sk-1")); + t.b.edit(|s| add_provider(s, "two", "https://two.example", "sk-2")); + let up_a = ea.round().await.unwrap().uploaded.unwrap(); + // B 看不到 A 刚传的那一版:它也传一个同序号的 + hidden.lock().unwrap().insert(format!("{up_a}.twconfig")); + let up_b = eb.round().await.unwrap().uploaded.unwrap(); + assert_eq!(up_a[..10], up_b[..10], "同一个序号:{up_a} / {up_b}"); + let listed = versions(t.loc.path()); + assert_eq!( + listed.len(), + 3, + "两个并列的头和它们共同的上一版本都在:{listed:?}" + ); + + // 文件到了。A 的下一轮看到两个头,合并成一个新版本 + hidden.lock().unwrap().clear(); + let r = ea.round().await.unwrap(); + assert!(r.conflicts.is_empty(), "{:?}", r.conflicts); + assert!(r.uploaded.is_some(), "并列的版本要合并成一个"); + assert!(t.a.provider("one").is_some() && t.a.provider("two").is_some()); + eb.round().await.unwrap(); + assert!(t.b.provider("one").is_some() && t.b.provider("two").is_some()); + let listed = versions(t.loc.path()); + assert!( + listed.len() <= 3, + "合并之后共同的上一版本删掉了:{listed:?}" + ); +} + +/// 同一个上游的连接两边改得不一样:冲突。两台设备看到同一处冲突;冲突期间各用各的; +/// 一台处理之后,另一台随之清除,内容一致 +#[tokio::test] +async fn a_conflict_is_the_same_on_both_devices_and_handled_once() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider(s, "relay", "https://relay.example", "sk-base")); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + + t.a.edit(|s| s.replace("sk-base", "sk-from-a")); + t.b.edit(|s| s.replace("sk-base", "sk-from-b")); + // 另一个上游上互不相干的改动照常合并 + t.b.edit(|s| add_provider(s, "other", "https://other.example", "sk-o")); + t.ea.round().await.unwrap(); + let rb = t.eb.round().await.unwrap(); + assert_eq!(rb.conflicts.len(), 1, "{:?}", rb.conflicts); + let c = &rb.conflicts[0]; + assert_eq!(c.item.name, "relay"); + assert_eq!(c.units, ["conn"]); + // 冲突期间各用各的 + assert_eq!( + t.b.provider("relay").unwrap().key.unwrap().raw(), + "sk-from-b" + ); + let ra = t.ea.round().await.unwrap(); + assert_eq!(ra.conflicts.len(), 1); + assert_eq!(ra.conflicts[0].id, c.id, "两台设备看到的是同一处冲突"); + assert_eq!( + t.a.provider("relay").unwrap().key.unwrap().raw(), + "sk-from-a" + ); + assert!(t.a.provider("other").is_some(), "不相干的改动照常合并"); + // 再同步几轮也不会反复上传 + let before = versions(t.loc.path()); + t.eb.round().await.unwrap(); + t.ea.round().await.unwrap(); + assert_eq!(versions(t.loc.path()), before); + + // A 选 B 的 + let ra = + t.ea.round_with(&[Decision { + id: c.id.clone(), + take: Take::Remote, + new_name: None, + }]) + .await + .unwrap(); + assert!(ra.conflicts.is_empty()); + assert_eq!( + t.a.provider("relay").unwrap().key.unwrap().raw(), + "sk-from-b" + ); + let rb = t.eb.round().await.unwrap(); + assert!(rb.conflicts.is_empty(), "B 上的这处冲突随之清除"); + assert_eq!( + t.b.provider("relay").unwrap().key.unwrap().raw(), + "sk-from-b" + ); +} + +#[tokio::test] +async fn keeping_both_adds_the_other_side_under_the_device_name() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider(s, "relay", "https://relay.example", "sk-base")); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + t.a.edit(|s| s.replace("https://relay.example", "https://a.example")); + t.b.edit(|s| s.replace("https://relay.example", "https://b.example")); + t.ea.round().await.unwrap(); + let rb = t.eb.round().await.unwrap(); + let id = rb.conflicts[0].id.clone(); + let rb = + t.eb.round_with(&[Decision { + id, + take: Take::Both, + new_name: None, + }]) + .await + .unwrap(); + assert!(rb.conflicts.is_empty()); + assert_eq!(t.b.provider("relay").unwrap().base_url, "https://b.example"); + assert_eq!( + t.b.provider("relay (Mac mini)").unwrap().base_url, + "https://a.example" + ); + t.ea.round().await.unwrap(); + assert_eq!(t.a.provider("relay").unwrap().base_url, "https://b.example"); + assert_eq!( + t.a.provider("relay (Mac mini)").unwrap().base_url, + "https://a.example" + ); +} + +/// 一边删了、另一边改了:冲突;删了的一边选「本机」就是删 +#[tokio::test] +async fn a_delete_against_an_edit_is_a_conflict() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider(s, "relay", "https://relay.example", "sk-base")); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + t.a.edit(|s| { + s.replace( + &provider_yaml("relay", "https://relay.example", "sk-base"), + "", + ) + .replace("providers:\n", "") + }); + assert!(t.a.provider("relay").is_none()); + t.b.edit(|s| s.replace("sk-base", "sk-new")); + t.ea.round().await.unwrap(); + let rb = t.eb.round().await.unwrap(); + assert_eq!(rb.conflicts.len(), 1, "{:?}", rb.conflicts); + assert!(rb.conflicts[0].whole_item()); + assert!(!rb.conflicts[0].both_allowed()); + let ra = t.ea.round().await.unwrap(); + let id = ra.conflicts[0].id.clone(); + t.ea.round_with(&[Decision { + id, + take: Take::Local, + new_name: None, + }]) + .await + .unwrap(); + t.eb.round().await.unwrap(); + assert!(t.b.provider("relay").is_none(), "{}", t.b.text()); +} + +/// 设备离线期间别的设备传了好几版,它当时的基准在同步位置上早删了:恢复连接后照本机基准合并, +/// 不报假冲突、不丢本机的改动 +#[tokio::test] +async fn coming_back_after_the_base_was_deleted_still_merges() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider(s, "relay", "https://relay.example", "sk-1")); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + // B 离线。A 改了好几次 + for i in 2..6 { + t.a.edit(|s| s.replace(&format!("sk-{}", i - 1), &format!("sk-{i}"))); + t.ea.round().await.unwrap(); + } + let b_base = t.eb.state().base.unwrap(); + assert!( + !versions(t.loc.path()).contains(&format!("{b_base}.twconfig")), + "B 的基准在同步位置上已经删了" + ); + t.b.edit(|s| add_provider(s, "offline", "https://offline.example", "sk-off")); + let r = t.eb.round().await.unwrap(); + assert!(r.conflicts.is_empty(), "{:?}", r.conflicts); + assert_eq!(t.b.provider("relay").unwrap().key.unwrap().raw(), "sk-5"); + t.ea.round().await.unwrap(); + assert!(t.a.provider("offline").is_some()); +} + +#[tokio::test] +async fn encryption_can_be_turned_on_off_and_its_passphrase_changed() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider(s, "relay", "https://relay.example", "sk-secret-value")); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + + // A 开启加密:B 下一轮要口令 + t.ea.enable_encryption(PASS).await.unwrap(); + assert!(t.loc.path().join("keyring").exists()); + for v in versions(t.loc.path()) { + let bytes = std::fs::read(t.loc.path().join("versions").join(&v)).unwrap(); + assert!(tw_pack::is_encrypted(&bytes), "未加密的版本删了:{v}"); + assert!(!String::from_utf8_lossy(&bytes).contains("sk-secret-value")); + } + assert!(matches!(t.eb.round().await, Err(RoundError::Locked(_)))); + assert!(t.eb.unlock("wrong passphrase").await.is_err()); + assert!(t.eb.unlock(PASS).await.unwrap()); + t.b.edit(|s| s.replace("sk-secret-value", "sk-after-encryption")); + let r = t.eb.round().await.unwrap(); + assert!(r.encrypted && r.uploaded.is_some()); + t.ea.round().await.unwrap(); + assert_eq!( + t.a.provider("relay").unwrap().key.unwrap().raw(), + "sk-after-encryption" + ); + + // A 改口令:已有版本不变,B 要输入新口令 + let before = versions(t.loc.path()); + t.ea.change_passphrase(PASS, "a brand new passphrase") + .await + .unwrap(); + assert_eq!(versions(t.loc.path()), before, "改口令只重新加密 keyring"); + assert!(matches!(t.eb.round().await, Err(RoundError::Locked(_)))); + assert!(t.eb.unlock("a brand new passphrase").await.unwrap()); + t.eb.round().await.unwrap(); + + // A 关闭加密:B 跟着变成不加密 + t.ea.disable_encryption("a brand new passphrase") + .await + .unwrap(); + assert!(!t.loc.path().join("keyring").exists()); + let r = t.eb.round().await.unwrap(); + assert!(!r.encrypted, "同步位置不加密了,B 照着改设置"); +} + +/// 两台设备同时开启加密同步:keyring 只建得成一份,后写的那台转去加入 +#[tokio::test] +async fn only_one_keyring_is_ever_created() { + tw_sync::keyring::set_work_factor_for_tests(4); + let loc = tempfile::tempdir().unwrap(); + let (a, b) = (Dev::new('a'), Dev::new('b')); + let mut ea = Engine::new( + folder(loc.path()), + a.clone(), + a.dir.path(), + settings(A_ID, "A", true, loc.path()), + ); + let mut eb = Engine::new( + folder(loc.path()), + b.clone(), + b.dir.path(), + settings(B_ID, "B", true, loc.path()), + ); + assert!(ea.create().await.unwrap()); + assert!(!eb.create().await.unwrap(), "后到的一方要转入加入流程"); + eb.join(None).await.unwrap(); + assert!(eb.round().await.unwrap().encrypted); +} + +/// 同步位置的最新版本由更新的 Core 写、本机读不了:暂停,不下载也不上传;设备记录里写明 +#[tokio::test] +async fn a_version_from_a_newer_core_pauses_sync() { + let mut t = bed(false).await; + // 伪造一个更新的 Core 写的版本:配置里有本机不认识的字段 + let latest = versions(t.loc.path()).pop().unwrap(); + let bytes = std::fs::read(t.loc.path().join("versions").join(&latest)).unwrap(); + let mut pack = tw_pack::read(&bytes, tw_pack::Unseal::None).unwrap(); + let mut meta: tw_sync::record::Meta = + serde_json::from_value(pack.manifest.sync.clone().unwrap()).unwrap(); + let parent = latest.trim_end_matches(".twconfig").to_string(); + meta.seq += 1; + meta.device = "cccccccc-0000-4000-8000-000000000003".into(); + meta.parents = vec![parent]; + pack.config_yaml.push_str("a_field_from_the_future: true\n"); + pack.manifest.core_version = "9.0.0".into(); + pack.manifest.sync = Some(serde_json::to_value(&meta).unwrap()); + let name = format!("{:010}-cccccccc.twconfig", meta.seq); + std::fs::write( + t.loc.path().join("versions").join(&name), + tw_pack::write(&pack, tw_pack::Seal::None).unwrap(), + ) + .unwrap(); + + t.b.edit(|s| add_provider(s, "local-only-for-now", "https://x.example", "sk")); + let e = t.eb.round().await.unwrap_err(); + assert!(matches!(e, RoundError::TooNew { .. }), "{e:?}"); + assert!( + !versions(t.loc.path()) + .iter() + .any(|v| v.ends_with("-bbbbbbbb.twconfig") && v > &name), + "暂停时不上传" + ); + let ra = t.ea.round().await; + assert!(matches!(ra, Err(RoundError::TooNew { .. }))); + // A 看得到 B 暂停着 + let probe = Engine::::probe(&folder(t.loc.path()), None).await; + let devs = probe.existing.unwrap().devices; + assert!(devs.iter().all(|d| d.paused_update), "{devs:?}"); +} + +/// 不同步的上游:不上传,也不被别的设备的同名上游覆盖;引用它的路由随之不同步 +#[tokio::test] +async fn an_excluded_upstream_and_what_refers_to_it_stay_local() { + let mut t = bed(false).await; + t.a.edit(|s| { + let s = add_provider(s, "ollama", "http://127.0.0.1:11434", "none"); + format!( + "{s}routes:\n - name: local-first\n rules:\n - name: r\n to: ollama\n" + ) + }); + t.b.edit(|s| add_provider(s, "ollama", "http://192.168.1.9:11434", "none")); + let mut sa = settings(A_ID, "Mac mini", false, t.loc.path()); + sa.exclude.insert("ollama".into()); + t.ea.set_settings(sa); + let mut sb = settings(B_ID, "MacBook Air", false, t.loc.path()); + sb.exclude.insert("ollama".into()); + t.eb.set_settings(sb); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + t.ea.round().await.unwrap(); + assert_eq!( + t.a.provider("ollama").unwrap().base_url, + "http://127.0.0.1:11434" + ); + assert_eq!( + t.b.provider("ollama").unwrap().base_url, + "http://192.168.1.9:11434" + ); + assert!( + t.b.config().routes.is_empty(), + "引用不同步上游的路由不上传:{}", + t.b.text() + ); + for v in versions(t.loc.path()) { + let p = tw_pack::read( + &std::fs::read(t.loc.path().join("versions").join(v)).unwrap(), + tw_pack::Unseal::None, + ) + .unwrap(); + assert!(!p.config_yaml.contains("ollama"), "{}", p.config_yaml); + } +} + +/// 本机已接管的客户端正在用的网关密钥被另一台设备删了:不删,作为冲突列出 +#[tokio::test] +async fn a_key_in_use_here_is_not_deleted_by_another_device() { + let mut t = bed(false).await; + t.a.edit(|s| { + s.replace( + "clients:\n", + "clients:\n - name: codex\n key: tw-codex-key-0002\n", + ) + }); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + t.b.pinned.lock().unwrap().insert("codex".into()); + t.a.edit(|s| s.replace(" - name: codex\n key: tw-codex-key-0002\n", "")); + t.ea.round().await.unwrap(); + let r = t.eb.round().await.unwrap(); + assert!( + t.b.config().clients.iter().any(|c| c.name == "codex"), + "{}", + t.b.text() + ); + assert_eq!(r.conflicts.len(), 1); + assert_eq!( + r.conflicts[0].reason, + Some(tw_sync::record::Reason::KeyInUse) + ); +} + +/// 另一台设备停止同步并删除了同步数据:这台设备发现同步位置空了 +#[tokio::test] +async fn a_purged_location_is_noticed() { + let mut t = bed(true).await; + t.ea.stop(true).await.unwrap(); + assert!(versions(t.loc.path()).is_empty()); + assert!(!t.loc.path().join("keyring").exists()); + let r = t.eb.round().await; + assert!( + matches!(r, Err(RoundError::Emptied)), + "{:?}", + r.map(|r| r.uploaded) + ); +} + +/// 同一套流程走 WebDAV 和 S3(进程内的假服务) +#[tokio::test] +async fn the_same_two_device_flow_over_webdav_and_s3() { + tw_sync::keyring::set_work_factor_for_tests(4); + let dav = common::webdav().await; + let s3 = common::s3().await; + let stores: Vec AnyStore>> = vec![ + Box::new({ + let url = format!("{}/dav/root", dav.url()); + move || { + AnyStore::WebDav( + tw_sync::store::WebDav::new(&url, common::USER, common::PASS, common::http()) + .unwrap(), + ) + } + }), + Box::new({ + let endpoint = s3.url(); + move || { + AnyStore::S3( + tw_sync::store::S3::new( + tw_sync::store::s3::Config { + endpoint: endpoint.clone(), + region: "auto".into(), + bucket: "tw-bucket".into(), + prefix: "tw".into(), + access_key_id: common::AK.into(), + secret_access_key: common::SK.into(), + }, + common::http(), + ) + .unwrap(), + ) + } + }), + ]; + for (i, mk) in stores.iter().enumerate() { + let (a, b) = (Dev::new('a'), Dev::new('b')); + let place = Path::new(if i == 0 { "/webdav" } else { "/s3" }); + let mut ea = Engine::new( + mk(), + a.clone(), + a.dir.path(), + settings(A_ID, "A", true, place), + ); + let mut eb = Engine::new( + mk(), + b.clone(), + b.dir.path(), + settings(B_ID, "B", true, place), + ); + assert!(ea.create().await.unwrap()); + eb.join(None).await.unwrap(); + a.edit(|s| add_provider(s, "relay", "https://relay.example", "sk-over-the-wire")); + ea.round().await.unwrap(); + eb.round().await.unwrap(); + assert_eq!( + b.provider("relay").unwrap().key.unwrap().raw(), + "sk-over-the-wire" + ); + b.edit(|s| s.replace("sk-over-the-wire", "sk-back")); + eb.round().await.unwrap(); + ea.round().await.unwrap(); + assert_eq!(a.provider("relay").unwrap().key.unwrap().raw(), "sk-back"); + } + // 同步位置上看不到明文密钥 + for f in dav.names().iter().chain(s3.names().iter()) { + let bytes = dav.get(f).or_else(|| s3.get(f)).unwrap(); + assert!(!String::from_utf8_lossy(&bytes).contains("sk-back"), "{f}"); + } +} diff --git a/crates/tw-sync/tests/stores.rs b/crates/tw-sync/tests/stores.rs new file mode 100644 index 00000000..a26be7a4 --- /dev/null +++ b/crates/tw-sync/tests/stores.rs @@ -0,0 +1,198 @@ +//! 三种同步位置答同一套问题:只新增不覆盖、仅当不存在时创建、覆盖写、列目录跳过临时文件、 +//! 删两次不报错、凭据不对是「拒绝访问」、连不上是「不可用」。 +//! +//! 文件夹用临时目录,WebDAV 和 S3 用进程内的假服务(`common`)。 + +mod common; + +use tw_sync::store::{self, Folder, S3, WebDav}; +use tw_sync::{Store, StoreError}; + +const V1: &str = "versions/0000000001-aaaaaaaa.twconfig"; +const V2: &str = "versions/0000000002-bbbbbbbb.twconfig"; + +async fn conformance(s: &impl Store) { + assert!( + s.list("versions").await.unwrap().is_empty(), + "空的位置列出来是空的" + ); + assert_eq!(s.get(V1).await.unwrap(), None); + + s.put_new(V1, b"one").await.unwrap(); + let e = s.put_new(V1, b"two").await.unwrap_err(); + assert!(matches!(e, StoreError::Exists { .. }), "{e:?}"); + assert_eq!( + s.get(V1).await.unwrap().as_deref(), + Some(&b"one"[..]), + "只新增不覆盖" + ); + + s.put_new(V2, b"twotwo").await.unwrap(); + let names: Vec<(String, u64)> = s + .list("versions") + .await + .unwrap() + .into_iter() + .map(|e| (e.name, e.size)) + .collect(); + assert_eq!( + names, + [ + ("0000000001-aaaaaaaa.twconfig".to_string(), 3), + ("0000000002-bbbbbbbb.twconfig".to_string(), 6) + ], + "临时文件不出现在列表里" + ); + + assert!(s.create_exclusive("keyring", b"k1").await.unwrap()); + assert!(!s.create_exclusive("keyring", b"k2").await.unwrap()); + assert_eq!(s.get("keyring").await.unwrap().as_deref(), Some(&b"k1"[..])); + s.put("keyring", b"k3").await.unwrap(); + assert_eq!(s.get("keyring").await.unwrap().as_deref(), Some(&b"k3"[..])); + + s.put("devices/aaaaaaaa", b"d1").await.unwrap(); + s.put("devices/aaaaaaaa", b"d2").await.unwrap(); + assert_eq!( + s.get("devices/aaaaaaaa").await.unwrap().as_deref(), + Some(&b"d2"[..]) + ); + assert_eq!(s.list("devices").await.unwrap().len(), 1); + + s.delete(V1).await.unwrap(); + s.delete(V1).await.unwrap(); + assert_eq!(s.get(V1).await.unwrap(), None); + assert_eq!(s.list("versions").await.unwrap().len(), 1); +} + +#[tokio::test] +async fn a_folder_answers_like_the_others() { + let d = tempfile::tempdir().unwrap(); + conformance(&Folder::new(d.path())).await; + // 只剩正式的文件,没有半截的临时文件 + let left: Vec = std::fs::read_dir(d.path().join("versions")) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().to_string()) + .collect(); + assert_eq!(left, ["0000000002-bbbbbbbb.twconfig"]); +} + +#[tokio::test] +async fn webdav_answers_like_the_others() { + let fake = common::webdav().await; + let s = WebDav::new( + &format!("{}/dav/root", fake.url()), + common::USER, + common::PASS, + common::http(), + ) + .unwrap(); + conformance(&s).await; + assert!( + fake.names().iter().all(|n| !n.contains(".tmp-")), + "临时文件改名之后不该留下:{:?}", + fake.names() + ); +} + +#[tokio::test] +async fn webdav_without_move_or_conditional_put_still_never_overwrites_a_version() { + let fake = common::webdav().await; + fake.knobs + .no_move + .store(true, std::sync::atomic::Ordering::Relaxed); + let s = WebDav::new( + &format!("{}/dav/root", fake.url()), + common::USER, + common::PASS, + common::http(), + ) + .unwrap(); + conformance(&s).await; + // 连 If-None-Match 都不认的:keyring 先看一眼再写 + fake.knobs + .no_conditional + .store(true, std::sync::atomic::Ordering::Relaxed); + assert!(!s.create_exclusive("keyring", b"other").await.unwrap()); + assert_eq!(fake.get("keyring").as_deref(), Some(&b"k3"[..])); +} + +fn s3_store(fake: &common::Fake, ak: &str) -> S3 { + S3::new( + store::s3::Config { + endpoint: fake.url(), + region: "auto".into(), + bucket: "tw-bucket".into(), + prefix: "team/tw".into(), + access_key_id: ak.into(), + secret_access_key: common::SK.into(), + }, + common::http(), + ) + .unwrap() +} + +#[tokio::test] +async fn s3_answers_like_the_others() { + let fake = common::s3().await; + conformance(&s3_store(&fake, common::AK)).await; + assert!( + fake.names().iter().all(|n| n.starts_with("team/tw/")), + "键都在前缀底下:{:?}", + fake.names() + ); +} + +#[tokio::test] +async fn s3_without_conditional_writes_falls_back_to_looking_first() { + let fake = common::s3().await; + fake.knobs + .no_conditional + .store(true, std::sync::atomic::Ordering::Relaxed); + conformance(&s3_store(&fake, common::AK)).await; +} + +#[tokio::test] +async fn wrong_credentials_are_a_refusal_and_a_dead_server_is_unreachable() { + let dav = common::webdav().await; + let wrong = WebDav::new( + &format!("{}/dav/root", dav.url()), + common::USER, + "wrong", + common::http(), + ) + .unwrap(); + assert!(matches!( + wrong.list("versions").await.unwrap_err(), + StoreError::AuthFailed(_) + )); + let s3 = common::s3().await; + assert!(matches!( + s3_store(&s3, "AKIDSOMEONEELSE").get(V1).await.unwrap_err(), + StoreError::AuthFailed(_) + )); + dav.knobs + .down + .store(true, std::sync::atomic::Ordering::Relaxed); + let right = WebDav::new( + &format!("{}/dav/root", dav.url()), + common::USER, + common::PASS, + common::http(), + ) + .unwrap(); + assert!(matches!( + right.get(V1).await.unwrap_err(), + StoreError::Unreachable(_) + )); + let gone = WebDav::new( + &format!("{}/dav/root", common::nowhere()), + common::USER, + common::PASS, + common::http(), + ) + .unwrap(); + assert!(matches!( + gone.list("versions").await.unwrap_err(), + StoreError::Unreachable(_) + )); +} diff --git a/docs/config.md b/docs/config.md index 75a91421..5ac3653a 100644 --- a/docs/config.md +++ b/docs/config.md @@ -193,6 +193,7 @@ means. | `default_route` | string | — | The route for keys that do not name one. Unset: the route named `default`, or the built-in failover when there is none. | | `default_key` | string | — | The gateway key for clients that were not given a key of their own. Unset: the key named `default`, or the first key. It cannot be disabled. | | `plugins` | list of [`plugins[]`](#cfg-plugins) | `[]` | Script plugins, in the order they run. The app installs them; each one's code and settings are a file next to this one. | +| `sync` | object, [`sync`](#cfg-sync) | — | Multi-device sync. A local setting: configuration packs and other devices never carry it, and restoring a backup keeps the current one. Unset: sync is off. | ### `listen` @@ -1502,6 +1503,95 @@ export const manifest = { }; ``` +### `sync` + +Several devices (this Mac, a laptop, a server running a remote core) can keep +one configuration in step through a sync location you choose: a folder that a +cloud drive carries between devices, a WebDAV server, or S3-compatible +storage. Each change is merged object by object with what the other devices +did since the last sync; only when two devices change the same part of the +same object to different values is there a conflict, which the app shows on +every device until one of them resolves it. Turn sync on, join a location and +resolve conflicts in the app (Settings, Sync and backup) or with +`twcore sync enable` on a server; this section records the result. + +What syncs is the configuration a configuration pack carries: upstreams, +proxies, gateway keys, aliases, groups, routes, price sheets, security rules, +plugins and the general settings, keys included. `listen`, `retention` and +`sync` itself stay on this device, and so does a ChatGPT account's sign-in. + + + + +| Field | Type | Default | Description | +|---|---|---|---| +| `device_id` | string | **required** | Generated when sync is turned on. File names at the sync location show its first 8 characters. | +| `device_name` | string | **required** | Shown in the device list of every device that syncs here. | +| `encrypt` | bool | `true` | Encrypt what goes to the sync location with the sync passphrase. Every device at one location uses the same setting; change it in the app, which re-uploads the latest version. | +| `passphrase` | string | — | The sync passphrase; required when `encrypt` is on. Kept here in the clear, like the other keys in this file. | +| `exclude` | list of strings | `[]` | Upstreams, by name, that are not synced: they are not uploaded and another device's upstream of the same name does not replace them. Routes, aliases and groups that refer to them are not synced either. | +| `target` | object, [`sync.target`](#cfg-sync-target) | **required** | The sync location. The keyring, `versions/` and `devices/` live directly in it. | + + + + + +| Field | Type | Default | Description | +|---|---|---|---| +| `type` | `folder` \| `webdav` \| `s3` | **required** | Which kind of location; the other fields depend on it, see the three tables below. | + + + + + +| Field | Type | Default | Description | +|---|---|---|---| +| `type` | string | **required** | `folder` | +| `path` | string | **required** | An absolute path on the machine core runs on: a folder that iCloud Drive, Dropbox, OneDrive, Syncthing or a NAS share carries between devices. | +| `proxy` | string | `direct` | Not used for a folder. | + + + + + +| Field | Type | Default | Description | +|---|---|---|---| +| `type` | string | **required** | `webdav` | +| `url` | string | **required** | The WebDAV address of the folder. `https://`, with the certificate checked; `http://` only for this machine and the local network. | +| `user` | string | `''` | User name. | +| `pass` | string | `''` | Password, or the app password the service issues for WebDAV. | +| `proxy` | string | `direct` | Where requests to the sync location go out: `direct`, `system` (the system proxy), or the name of a proxy under `proxies`. | + + + + + +| Field | Type | Default | Description | +|---|---|---|---| +| `type` | string | **required** | `s3` | +| `endpoint` | string | **required** | The S3 endpoint. Hosts ending in `amazonaws.com`, `aliyuncs.com` or `myqcloud.com` put the bucket in the host name; every other endpoint (Cloudflare R2, MinIO, an address) puts it in the path. `http://` only for this machine and the local network. | +| `region` | string | **required** | The signing region; `auto` for Cloudflare R2. | +| `bucket` | string | **required** | The bucket. Keep it private. | +| `prefix` | string | `''` | A folder inside the bucket. | +| `access_key_id` | string | **required** | Access key ID. | +| `secret_access_key` | string | **required** | Secret access key. | +| `proxy` | string | `direct` | Where requests to the sync location go out: `direct`, `system` (the system proxy), or the name of a proxy under `proxies`. | + + +```yaml +sync: + device_id: 3f9a2c1e-0b1d-4e2f-9a3b-5c6d7e8f9a0b + device_name: Mac mini + encrypt: true + passphrase: "correct horse battery staple" + exclude: [local ollama] + target: + type: webdav + url: https://dav.example.com/dav/ThinkWatch + user: name@example.com + pass: "app password" +``` + ## Environment variables | Variable | Effect | diff --git a/docs/config.zh-CN.md b/docs/config.zh-CN.md index e6ba2936..9a471081 100644 --- a/docs/config.zh-CN.md +++ b/docs/config.zh-CN.md @@ -118,6 +118,7 @@ twcore config set /listen/gateway/port 8790 --int | `default_route` | 字符串 | — | 未指定路由的密钥走哪条路由。不写:名为 `default` 的路由;没有这条路由时走内置的故障转移。 | | `default_key` | 字符串 | — | 没有专用密钥的客户端使用哪一把。不写:名为 `default` 的那把,没有则取第一把。这把密钥不能停用。 | | `plugins` | 对象列表,见 [`plugins[]`](#cfg-plugins) | `[]` | 脚本插件,按运行的顺序。由应用安装,每个插件的代码和设置是本文件旁边的一个文件。 | +| `sync` | 对象,见 [`sync`](#cfg-sync) | — | 多设备同步。属于本机设置:配置包和其他设备不会带走它,恢复备份时保留当前值。不写:不同步。 | ### `listen` @@ -1100,6 +1101,84 @@ export const manifest = { }; ``` +### `sync` + +多台设备(这台 Mac、一台笔记本、一台运行远程 core 的服务器)可以通过自选的同步位置保持同一份配置:由云盘在设备之间传送的文件夹、WebDAV 服务器或 S3 兼容存储。每次改动都按对象与其他设备自上次同步以来的改动合并;只有两台设备把同一对象的同一部分改成不同内容时才构成冲突,冲突在每台设备上显示,任一设备处理之后随之清除。开启同步、加入已有的同步和处理冲突在应用里进行(设置 → 同步与备份),服务器上用 `twcore sync enable`;本节记录的是结果。 + +同步的内容即配置包带的内容:上游、出站代理、网关密钥、别名、分组、路由、价目表、安全规则、插件和全局设置,包括密钥。`listen`、`retention` 和 `sync` 本身留在本机,ChatGPT 账号的登录也一样。 + + + + +| 字段 | 类型 | 默认值 | 说明 | +|---|---|---|---| +| `device_id` | 字符串 | **必填** | 开启同步时生成。同步位置上的文件名里出现的是它的前 8 个字符。 | +| `device_name` | 字符串 | **必填** | 在每台参与同步的设备的设备列表里显示的名字。 | +| `encrypt` | 布尔 | `true` | 以同步口令加密存到同步位置的内容。同一同步位置上的所有设备采用同一设置;在应用里更改,应用会重新上传最新版本。 | +| `passphrase` | 字符串 | — | 同步口令,`encrypt` 开着时必填。和本文件里的其他密钥一样以明文保存。 | +| `exclude` | 字符串列表 | `[]` | 不同步的上游,按名字:不上传,也不被其他设备的同名上游覆盖。引用它们的路由、别名、分组随之不同步。 | +| `target` | 对象,见 [`sync.target`](#cfg-sync-target) | **必填** | 同步位置。`keyring`、`versions/`、`devices/` 直接放在它里面。 | + + + + + +| 字段 | 类型 | 默认值 | 说明 | +|---|---|---|---| +| `type` | `folder` \| `webdav` \| `s3` | **必填** | 同步位置的种类;其余字段随之不同,见下面三张表。 | + + + + + +| 字段 | 类型 | 默认值 | 说明 | +|---|---|---|---| +| `type` | 字符串 | **必填** | `folder` | +| `path` | 字符串 | **必填** | core 所在机器上的绝对路径:由 iCloud Drive、Dropbox、OneDrive、Syncthing 或 NAS 共享在设备之间传送的文件夹。 | +| `proxy` | 字符串 | `direct` | 文件夹用不到。 | + + + + + +| 字段 | 类型 | 默认值 | 说明 | +|---|---|---|---| +| `type` | 字符串 | **必填** | `webdav` | +| `url` | 字符串 | **必填** | 文件夹的 WebDAV 地址。`https://`,校验证书;`http://` 只用于本机和局域网。 | +| `user` | 字符串 | `''` | 用户名。 | +| `pass` | 字符串 | `''` | 密码,或者服务为 WebDAV 签发的应用密码。 | +| `proxy` | 字符串 | `direct` | 访问同步位置的请求从哪里出去:`direct`(直连)、`system`(跟随系统代理),或者 `proxies` 里一个代理的名字。 | + + + + + +| 字段 | 类型 | 默认值 | 说明 | +|---|---|---|---| +| `type` | 字符串 | **必填** | `s3` | +| `endpoint` | 字符串 | **必填** | S3 端点。主机以 `amazonaws.com`、`aliyuncs.com`、`myqcloud.com` 结尾的,存储桶名放进域名;其余端点(Cloudflare R2、MinIO、IP 地址)放进路径。`http://` 只用于本机和局域网。 | +| `region` | 字符串 | **必填** | 签名用的区域;Cloudflare R2 填 `auto`。 | +| `bucket` | 字符串 | **必填** | 存储桶,建议设为私有。 | +| `prefix` | 字符串 | `''` | 存储桶里的一个前缀(文件夹)。 | +| `access_key_id` | 字符串 | **必填** | Access Key ID。 | +| `secret_access_key` | 字符串 | **必填** | Secret Access Key。 | +| `proxy` | 字符串 | `direct` | 访问同步位置的请求从哪里出去:`direct`(直连)、`system`(跟随系统代理),或者 `proxies` 里一个代理的名字。 | + + +```yaml +sync: + device_id: 3f9a2c1e-0b1d-4e2f-9a3b-5c6d7e8f9a0b + device_name: Mac mini + encrypt: true + passphrase: "correct horse battery staple" + exclude: [本机 Ollama] + target: + type: webdav + url: https://dav.example.com/dav/ThinkWatch + user: name@example.com + pass: "应用密码" +``` + ## 环境变量 | 变量 | 作用 |