diff --git a/Cargo.lock b/Cargo.lock index 7b0055cb..f9224567 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3985,6 +3985,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "serde_yaml_ng", "sha2 0.11.0", "tempfile", "thiserror 2.0.20", diff --git a/bin/twcore/src/main.rs b/bin/twcore/src/main.rs index 1a2e251e..dc03ad21 100644 --- a/bin/twcore/src/main.rs +++ b/bin/twcore/src/main.rs @@ -176,6 +176,9 @@ enum SyncCmd { #[arg(long)] purge: bool, }, + /// After the sync location was emptied or lost its keyring: upload this device's + /// configuration again (and put back the keyring this device keeps) + Republish, } #[derive(Subcommand)] @@ -289,6 +292,7 @@ fn main() -> Result<()> { SyncCmd::Now => sync_cmd::now(&path), SyncCmd::Enable => sync_cmd::enable(&path), SyncCmd::Disable { purge } => sync_cmd::disable(&path, purge), + SyncCmd::Republish => sync_cmd::republish(&path), }, Command::Upgrade { check, diff --git a/bin/twcore/src/sync_cmd.rs b/bin/twcore/src/sync_cmd.rs index c52ff7b8..b9965132 100644 --- a/bin/twcore/src/sync_cmd.rs +++ b/bin/twcore/src/sync_cmd.rs @@ -102,6 +102,19 @@ fn print_view(v: &tw_api::SyncView) { if !v.excluded.is_empty() { println!("not synced: {}", v.excluded.join(", ")); } + if v.skipped > 0 { + println!( + "skipped: {} file(s) at the sync location could not be read", + v.skipped + ); + } + match v.state { + tw_api::SyncState::Emptied | tw_api::SyncState::KeyringMissing => println!( + "to go on: `twcore sync republish` uploads this device's configuration again, \ + `twcore sync disable` stops syncing" + ), + _ => {} + } for d in &v.devices { let mut notes = Vec::new(); if d.this { @@ -139,11 +152,18 @@ pub fn now(config: &Path) -> Result<()> { Ok(()) } +/// 同步位置空了、keyring 不见了之后：以本机配置重新上传（keyring 放回本机记着的那一份） +pub fn republish(config: &Path) -> Result<()> { + let v: tw_api::SyncView = send(config, "POST", "/sync/republish", None)?; + print_view(&v); + Ok(()) +} + pub fn disable(config: &Path, purge: bool) -> Result<()> { if purge && !yes( "Delete every version, device record and the keyring at the sync location? Other \ - devices stop syncing", + devices are told the location is empty", false, )? { diff --git a/crates/tw-api/msg-codes.txt b/crates/tw-api/msg-codes.txt index 3f3a4c23..8a1cda17 100644 --- a/crates/tw-api/msg-codes.txt +++ b/crates/tw-api/msg-codes.txt @@ -555,8 +555,9 @@ sync.config.device_name sync.config.folder_relative sync.config.multiline sync.config.passphrase_missing +sync.config.plain_http_proxy +sync.config.prefix sync.config.unknown_proxy -sync.emptied sync.folder.denied sync.folder.io sync.folder.missing @@ -564,11 +565,15 @@ sync.folder.task sync.http_client sync.join_undecided sync.keyring_corrupt +sync.keyring_missing +sync.keyring_rolled_back sync.keyring_too_new sync.local sync.local.io +sync.location_empty sync.location_taken sync.locked_new +sync.looks_empty sync.merge_invalid sync.not_arrived sync.not_encrypted @@ -586,12 +591,16 @@ sync.store.bad_path sync.store.denied sync.store.exists sync.store.server_error +sync.store.too_large sync.store.unexpected sync.store.unreachable +sync.take_not_allowed sync.too_new +sync.too_new_unknown sync.version_gone sync.webdav.bad_listing sync.webdav.bad_url +sync.webdav.root_missing sync.write_section t.auth test t.boom test diff --git a/crates/tw-api/src/ep.rs b/crates/tw-api/src/ep.rs index 138591a6..34735dc5 100644 --- a/crates/tw-api/src/ep.rs +++ b/crates/tw-api/src/ep.rs @@ -282,6 +282,10 @@ endpoints! { SyncUnlock: POST "/sync/unlock", api::SyncUnlock => api::SyncView; /// 停止同步；`purge` 时连同步位置上的数据一起删掉 SyncDisable: POST "/sync/disable", api::SyncDisable => (); + /// 同步位置空了（`emptied`）、keyring 不见了（`keyring_missing`）之后由人选的「重新上传本机 + /// 配置」：keyring 放回这台设备记着的那一份（没有就新建），没有读得出来的版本时以本机配置 + /// 上传一个，然后同步一轮 + SyncRepublish: POST "/sync/republish", () => api::SyncView; // ─────────────────────────────────────────────── 账号登录 StartChatgptLogin: POST "/chatgpt/login", api::ChatgptLoginStart => api::ChatgptLogin; diff --git a/crates/tw-api/src/lib.rs b/crates/tw-api/src/lib.rs index 24d27632..16bd6778 100644 --- a/crates/tw-api/src/lib.rs +++ b/crates/tw-api/src/lib.rs @@ -990,7 +990,10 @@ pub const MSG_CODES: &str = include_str!("../msg-codes.txt"); /// `PUT /sync/settings`（[`SyncSettingsSave`]）、`POST /sync/now`、`GET` / `POST /sync/conflicts` /// （[`SyncConflict`]、[`SyncResolve`]：采用本机、另一边或两者都保留）、`POST /sync/encryption` /// （[`SyncEncryption`]：开启、关闭、更改口令）、`POST /sync/unlock`（[`SyncUnlock`]）、 -/// `POST /sync/disable`（[`SyncDisable`]）。状态变了发 [`Event::SyncStatus`]；应用别的设备的 +/// `POST /sync/disable`（[`SyncDisable`]）、`POST /sync/republish`（同步位置空了、keyring 不见了 +/// 之后由人选的「重新上传本机配置」，→ [`SyncView`]）。[`SyncState`] 里要人处理的有 +/// `keyring_missing`、`emptied`、`misconfigured`，[`SyncView`] 的 `skipped` 是同步位置上跳过了的 +/// 文件数，[`SyncConflict`] 的 `options` 是能选的处理。状态变了发 [`Event::SyncStatus`]；应用别的设备的 /// 版本时配置的来源是 `sync`，改了网关密钥时发 `GatewayKeysChanged`。[`ProviderView`] 多了 /// `sync_excluded`（不同步的上游）和 `sync_url_changed`（地址经同步变更，改之前的地址）。 /// 同步的码是 `sync.*`。安全模式下不自动同步（`paused`） diff --git a/crates/tw-api/src/sync.rs b/crates/tw-api/src/sync.rs index 18800f47..c7dd837f 100644 --- a/crates/tw-api/src/sync.rs +++ b/crates/tw-api/src/sync.rs @@ -72,6 +72,17 @@ slug_enum! { TooNew = "too_new", /// 安全模式：不自动同步 Paused = "paused", + /// 同步位置上的 keyring 不见了、被换成了更早的、一直读不出来。**不会自己关掉加密**： + /// 由人选「重新上传本机配置」（`POST /sync/republish`：放回这台设备记着的 keyring）或者 + /// 停止同步。原因在 `error` + KeyringMissing = "keyring_missing", + /// 同步位置空了（连着几轮、过了一段时间都是空的）：其他设备删除了同步数据，或者它已经 + /// 不是原来那个位置。同步设置不动，由人选「重新上传本机配置」（`POST /sync/republish`） + /// 或者停止同步 + Emptied = "emptied", + /// 同步设置用不了（比如出站代理被删了、不加密的 `http://` 地址设了代理）：改设置。 + /// 原因在 `error` + Misconfigured = "misconfigured", } } @@ -122,6 +133,9 @@ pub struct SyncView { pub excluded: Vec, /// 随之不同步的项目：引用了不同步的上游的路由、别名、分组 pub excluded_cascade: Vec, + /// 同步位置上跳过了的文件：坏了的、不是这把密钥加密的、加密的位置上没加密的、等了 + /// 30 分钟还没传完的。不是 0 时界面提一句 + pub skipped: u32, } /// `POST /sync/test`、`POST /sync/enable/preview`：连上去看看。 @@ -256,9 +270,14 @@ pub struct SyncConflict { pub remote: SyncConflictSide, /// 两边不同的地方，逐个单位：点对象名时显示的差异 pub diff: Vec, - /// 能不能「两者都保留」：两边都还有这个对象，而且它有名字 + /// 能不能「两者都保留」：两边都还有这个对象，而且它有名字（和 `options` 里有没有 `both` + /// 是一回事） pub both_allowed: bool, pub reason: Option, + /// 这处冲突能选哪几种处理，界面只给这几个按钮：`local` 总在；`remote` 在合并结果过不了 + /// 校验（`invalid`）时没有；`both` 同 `both_allowed`。网关密钥的「两者都保留」新增的那一把 + /// 是一个新的值 + pub options: Vec, } slug_enum! { diff --git a/crates/tw-config/src/sync.rs b/crates/tw-config/src/sync.rs index 7a460ee4..4179739f 100644 --- a/crates/tw-config/src/sync.rs +++ b/crates/tw-config/src/sync.rs @@ -230,6 +230,12 @@ pub enum SyncError { BlankExclude, #[error("{}", self.msg())] MultiLine { field: &'static str }, + /// S3 前缀里有 `..`、`.` 这样的段，或者以 `/` 开头 + #[error("{}", self.msg())] + Prefix { prefix: String }, + /// `http://` 的内网地址经出站代理：同步位置的密码会明文交给代理 + #[error("{}", self.msg())] + PlainHttpProxy { proxy: String }, } impl SyncError { @@ -278,6 +284,16 @@ impl SyncError { "sync.config.multiline", field = field => "sync.{field} has to be on one line" ), + SyncError::Prefix { prefix } => msg!( + "sync.config.prefix", prefix = prefix => + "{prefix} is not a folder inside the bucket: no leading /, and no . or .. parts" + ), + SyncError::PlainHttpProxy { proxy } => msg!( + "sync.config.plain_http_proxy", proxy = proxy => + "an http:// sync location on the local network cannot go through proxy \ + `{proxy}`: its password would reach the proxy in the clear. Use direct, or an \ + https:// address" + ), } } } @@ -366,6 +382,11 @@ impl Target { if prefix.contains(['\n', '\r']) { return Err(SyncError::MultiLine { field: "prefix" }); } + if !prefix_ok(prefix) { + return Err(SyncError::Prefix { + prefix: prefix.clone(), + }); + } if !address_ok(endpoint) { return Err(SyncError::Address { field: "endpoint", @@ -383,9 +404,36 @@ impl Target { } } -/// 代理名对得上吗：`direct`、`system`，或者 `proxies` 里有的那一个 +/// S3 前缀：桶里的一个文件夹。不以 `/` 开头，没有 `.`、`..` 段和反斜杠（结尾的 `/` 无所谓） +fn prefix_ok(p: &str) -> bool { + let p = p.trim(); + !p.starts_with('/') + && !p.contains('\\') + && p.trim_end_matches('/') + .split('/') + .filter(|s| !p.is_empty() || !s.is_empty()) + .all(|seg| p.is_empty() || (!seg.is_empty() && seg != "." && seg != "..")) +} + +/// 地址是不是 `http://`（只给内网的那种） +fn plain_http(t: &Target) -> bool { + let url = match t { + Target::Webdav { url, .. } => url, + Target::S3 { endpoint, .. } => endpoint, + Target::Folder { .. } => return false, + }; + url.trim().to_ascii_lowercase().starts_with("http://") +} + +/// 代理名对得上吗：`direct`、`system`，或者 `proxies` 里有的那一个。`http://` 的内网地址 +/// 只能直连 pub fn check_proxy(target: &Target, proxies: &[crate::Proxy]) -> Result<(), SyncError> { let p = target.proxy(); + if p != DIRECT && plain_http(target) { + return Err(SyncError::PlainHttpProxy { + proxy: p.to_string(), + }); + } if p == DIRECT || p == crate::SYSTEM || proxies.iter().any(|x| x.name == p) { return Ok(()); } @@ -613,6 +661,34 @@ mod tests { assert_ne!(generate_device_id(), generate_device_id()); } + #[test] + fn an_s3_prefix_stays_inside_the_bucket() { + for ok in ["", "me", "team/tw", "team/tw/"] { + assert!(prefix_ok(ok), "{ok:?}"); + } + for bad in ["/abs", "a/../b", "..", "./x", "a//b", "a\\b"] { + assert!(!prefix_ok(bad), "{bad:?}"); + } + } + + #[test] + fn a_plain_http_location_never_goes_through_a_proxy() { + let mut t = webdav("http://192.168.1.2:5005/dav").target; + check_proxy(&t, &[]).unwrap(); + if let Target::Webdav { proxy, .. } = &mut t { + *proxy = "system".into(); + } + assert!(matches!( + check_proxy(&t, &[]), + Err(SyncError::PlainHttpProxy { .. }) + )); + let mut https = webdav("https://dav.example.com/").target; + if let Target::Webdav { proxy, .. } = &mut https { + *proxy = "system".into(); + } + check_proxy(&https, &[]).unwrap(); + } + #[test] fn folder_paths_have_to_be_absolute_on_either_platform() { for ok in [ diff --git a/crates/tw-config/tests/manual/schema.rs b/crates/tw-config/tests/manual/schema.rs index d885ec39..7d01198d 100644 --- a/crates/tw-config/tests/manual/schema.rs +++ b/crates/tw-config/tests/manual/schema.rs @@ -126,8 +126,8 @@ fn target_types() -> Vec<&'static str> { } const PROXY_DOC: T2 = t( - "Where requests to the sync location go out: `direct`, `system` (the system proxy), or the name of a proxy under `proxies`.", - "访问同步位置的请求从哪里出去：`direct`（直连）、`system`（跟随系统代理），或者 `proxies` 里一个代理的名字。", + "Where requests to the sync location go out: `direct`, `system` (the system proxy), or the name of a proxy under `proxies`. An `http://` address on this machine or the local network has to be `direct`.", + "访问同步位置的请求从哪里出去：`direct`（直连）、`system`（跟随系统代理），或者 `proxies` 里一个代理的名字。本机、内网的 `http://` 地址只能是 `direct`。", ); const RULE_ID: T2 = t("built-in rule id", "内置规则 id"); @@ -1921,8 +1921,8 @@ pub fn sections() -> Vec
{ Kind::Str, Def::Unset, t( - "The sync passphrase; required when `encrypt` is on. Kept here in the clear, like the other keys in this file.", - "同步口令，`encrypt` 开着时必填。和本文件里的其他密钥一样以明文保存。", + "The sync passphrase; required when `encrypt` is on, and kept after encryption is turned off: the keyring that says it is off is checked with it. Kept here in the clear, like the other keys in this file.", + "同步口令，`encrypt` 开着时必填；关闭加密之后也留着：同步位置上写着「已关闭」的 keyring 要用它核对。和本文件里的其他密钥一样以明文保存。", ), ), row( @@ -2066,8 +2066,8 @@ pub fn sections() -> Vec
{ Kind::Str, Def::Is("''"), t( - "A folder inside the bucket.", - "存储桶里的一个前缀（文件夹）。", + "A folder inside the bucket. No `..` parts, and not starting with `/`.", + "存储桶里的一个前缀（文件夹）。不能有 `..`，不以 `/` 开头。", ), ), row( diff --git a/crates/tw-control/src/sync.rs b/crates/tw-control/src/sync.rs index 41e27f91..3a6883b2 100644 --- a/crates/tw-control/src/sync.rs +++ b/crates/tw-control/src/sync.rs @@ -60,6 +60,7 @@ pub fn router() -> axum::Router { .at(ep::SyncEncryption, encryption) .at(ep::SyncUnlock, unlock) .at(ep::SyncDisable, disable) + .at(ep::SyncRepublish, republish) } fn now_ms() -> u64 { @@ -212,6 +213,9 @@ impl Host for CoreHost { fn http(cfg: &tw_config::Config, proxy: &str) -> Result { let mut b = reqwest::Client::builder() .connect_timeout(Duration::from_secs(10)) + // 一轮里的请求是接连发的；两轮之间的空闲连接不留：很多服务端几秒就关掉空闲连接， + // 留着的下一轮一发就断（发请求那一处另有一次重发） + .pool_idle_timeout(Duration::from_secs(4)) .redirect(reqwest::redirect::Policy::none()) .user_agent(tw_gateway::user_agent()); match proxy { @@ -444,8 +448,11 @@ fn round_fail(e: &RoundError) -> Fail { RoundError::Store(StoreError::AuthFailed(_)) => StatusCode::FORBIDDEN, RoundError::Store(_) => StatusCode::BAD_GATEWAY, RoundError::Locked(_) => StatusCode::BAD_REQUEST, - RoundError::TooNew { .. } | RoundError::Stale | RoundError::Emptied => StatusCode::CONFLICT, - RoundError::NotArrived => StatusCode::SERVICE_UNAVAILABLE, + RoundError::TooNew { .. } + | RoundError::Stale + | RoundError::Emptied + | RoundError::KeyringMissing(_) => StatusCode::CONFLICT, + RoundError::NotArrived | RoundError::LooksEmpty => StatusCode::SERVICE_UNAVAILABLE, RoundError::Local(_) => StatusCode::BAD_REQUEST, }; fail(code, e.msg()) @@ -605,6 +612,7 @@ impl Sync { conflicts: 0, excluded: Vec::new(), excluded_cascade: Vec::new(), + skipped: 0, }; let Some(mgr) = self.mgr() else { return off }; let cfg = mgr.gateway().config(); @@ -627,6 +635,11 @@ impl Sync { .map(|r| r.conflicts.len() as u32) .or(prev.as_ref().map(|p| p.conflicts)) .unwrap_or(0); + let skipped = report + .as_ref() + .map(|r| r.skipped) + .or(prev.as_ref().map(|p| p.skipped)) + .unwrap_or(0); let devices = match &report { Some(r) => r .devices @@ -680,6 +693,7 @@ impl Sync { conflicts, excluded: sync.exclude.clone(), excluded_cascade: cascade, + skipped, } } @@ -723,9 +737,10 @@ impl Sync { return Ok(v); } Err(m) => { + // 照设置建不起来（出站代理没了、地址不合规矩）：改设置，不是拒绝访问 let v = tw_api::SyncView { error: Some(m.clone()), - ..self.compute_view(None, Some(tw_api::SyncState::AuthFailed)) + ..self.compute_view(None, Some(tw_api::SyncState::Misconfigured)) }; self.publish(v.clone()); return Err(fail(StatusCode::BAD_REQUEST, m)); @@ -771,11 +786,13 @@ impl Sync { at_ms: now_ms(), }); } - // 别的设备关掉了加密：这台跟着改设置 - let follow = (encrypted != r.config.encrypt && !encrypted).then(|| { + // 别的设备关掉了加密（keyring 写着已关闭，口令解得开）、或者又开了（口令还是 + // 这台记着的那个）：这台跟着改设置。**口令留着**：靠它认出以后的 keyring + let follow = (encrypted != r.config.encrypt + && (!encrypted || r.config.passphrase.is_some())) + .then(|| { let mut s = r.config.clone(); - s.encrypt = false; - s.passphrase = None; + s.encrypt = encrypted; s }); Ok((report, follow)) @@ -795,27 +812,16 @@ impl Sync { drop(guard); self.view() } - Err(RoundError::Emptied) => { - // 别的设备停止同步并删除了同步数据：这台也停下，说一声 - r.engine.local().clear(); - drop(guard); - if let Some(mgr) = self.mgr() { - let _ = write_section(&mgr, None, Origin::Sync).await; - } - *self.run.lock().await = None; - self.register_backups(false); - tw_api::SyncView { - error: Some(RoundError::Emptied.msg()), - ..self.compute_view(None, Some(tw_api::SyncState::Off)) - } - } Err(e) => { + // 同步位置空了、keyring 不见了也一样：记下来、等人处理，**同步设置不动** self.record_failure(r, &e); let st = r.engine.local().state(); let state = match &e { RoundError::Store(StoreError::AuthFailed(_)) => tw_api::SyncState::AuthFailed, RoundError::Locked(_) => tw_api::SyncState::Locked, RoundError::TooNew { .. } => tw_api::SyncState::TooNew, + RoundError::Emptied => tw_api::SyncState::Emptied, + RoundError::KeyringMissing(_) => tw_api::SyncState::KeyringMissing, _ => { let since = st.failing_since_ms.unwrap_or_else(now_ms); if now_ms().saturating_sub(since) >= UNREACHABLE_AFTER_MS { @@ -904,6 +910,14 @@ impl Sync { tw_sync::record::Reason::KeyInUse => tw_api::SyncConflictReason::KeyInUse, tw_sync::record::Reason::Invalid => tw_api::SyncConflictReason::Invalid, }), + options: engine::options(c) + .into_iter() + .map(|t| match t { + Take::Local => tw_api::SyncTake::Local, + Take::Remote => tw_api::SyncTake::Remote, + Take::Both => tw_api::SyncTake::Both, + }) + .collect(), } }) .collect() @@ -1563,9 +1577,10 @@ async fn encryption( r.engine.enable_encryption(&p).await } tw_api::SyncEncryptionAction::Disable => { + // 口令留着：同步位置上的 keyring 写着「已关闭」，靠它认出来 let p = req.passphrase.clone().unwrap_or_default(); next.encrypt = false; - next.passphrase = None; + next.passphrase = Some(p.clone()).filter(|p| !p.is_empty()); r.engine.disable_encryption(&p).await } tw_api::SyncEncryptionAction::Change => { @@ -1602,10 +1617,8 @@ async fn unlock( .await .map_err(|e| round_fail(&e))?; let mut next = r.config.clone(); - if encrypted { - next.encrypt = true; - next.passphrase = Some(req.passphrase.clone()); - } + next.encrypt = encrypted; + next.passphrase = Some(req.passphrase.clone()); r.config = next.clone(); r.engine.set_settings(settings_of(&s.config(), &next)); drop(guard); @@ -1615,6 +1628,24 @@ async fn unlock( sync.round(&[]).await.map(Json) } +/// 同步位置空了、keyring 不见了之后，由人选的「重新上传本机配置」 +async fn republish(State(s): State) -> Result, Fail> { + let sync = service(&s); + let mut guard = sync.run.lock().await; + let r = match sync.ensure(&mut guard) { + Ok(Some(r)) => r, + Ok(None) => return Err(sync_off()), + Err(m) => return Err(fail(StatusCode::BAD_REQUEST, m)), + }; + let res = r.engine.republish().await; + drop(guard); + if let Err(e) = res { + return Err(round_fail(&e)); + } + sync.failures.store(0, Ordering::Relaxed); + sync.round(&[]).await.map(Json) +} + async fn disable( State(s): State, Json(req): Json, diff --git a/crates/tw-control/tests/sync_devices.rs b/crates/tw-control/tests/sync_devices.rs index 4ac190da..038dcf7e 100644 --- a/crates/tw-control/tests/sync_devices.rs +++ b/crates/tw-control/tests/sync_devices.rs @@ -432,11 +432,32 @@ async fn turning_sync_off_keeps_the_configuration_and_purging_empties_the_locati assert!(!a.config().providers.is_empty(), "本机保留当前配置"); assert!(!a.dir.path().join("sync").exists(), "本机的同步状态删掉了"); assert!(!loc.path().join("keyring").exists()); - // B 发现同步位置空了：停下，说一声 + // B：读到空的先不下结论；连着几轮、过了一段时间才是「同步位置空了」，同步设置不动，由人 + // 选重新上传或者停止 + for _ in 0..3 { + let v = b.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["error"]["code"], "sync.looks_empty", "{v}"); + assert_ne!(v["state"], "emptied"); + } + let local = tw_sync::local::Local::new(b.dir.path()); + let mut st = local.state(); + st.empty_since_ms = st.empty_since_ms.map(|t| t - 11 * 60 * 1000); + local.save_state(&st).unwrap(); let v = b.ok("POST", "/sync/now", Value::Null).await; - assert_eq!(v["state"], "off", "{v}"); - assert_eq!(v["error"]["code"], "sync.emptied"); - assert!(b.config().sync.is_none()); + assert_eq!(v["state"], "emptied", "{v}"); + assert_eq!(v["error"]["code"], "sync.location_empty"); + assert!(b.config().sync.is_some(), "同步设置不动"); + // 重新上传本机配置：keyring 放回 B 记着的那一份，版本以 B 的配置上传 + let v = b.ok("POST", "/sync/republish", Value::Null).await; + assert_eq!(v["state"], "idle", "{v}"); + assert!(v["error"].is_null(), "{v}"); + assert!(loc.path().join("keyring").exists()); + assert_eq!( + std::fs::read_dir(loc.path().join("versions")) + .unwrap() + .count(), + 1 + ); } /// 调度：本机改了配置，10 秒合并之后上传；另一台的文件夹监听一收到就合并 diff --git a/crates/tw-control/tests/sync_regressions.rs b/crates/tw-control/tests/sync_regressions.rs new file mode 100644 index 00000000..5473cf7f --- /dev/null +++ b/crates/tw-control/tests/sync_regressions.rs @@ -0,0 +1,347 @@ +//! 审查里找出来的同步问题在控制面这一层钉住：等登录的上游（加入的两种方式、登录之后）、要人 +//! 处理的状态（keyring 不见了）、冲突给出能选的处理、关闭加密之后口令留着。 + +use std::sync::Arc; + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use serde_json::{Value, json}; +use tower::ServiceExt; +use tw_control::{ConfigManager, ControlState}; + +const PASS: &str = "correct horse battery staple"; + +fn base(key: char) -> String { + format!( + "version: 1\nlisten:\n control:\n key: {}\nclients:\n - name: default\n key: tw-default-{key}\n", + key.to_string().repeat(64) + ) +} + +struct Dev { + /// 数据目录：活到测试结束 + _dir: tempfile::TempDir, + state: ControlState, + app: axum::Router, +} + +impl Dev { + fn new(key: char) -> Dev { + tw_sync::keyring::set_work_factor_for_tests(4); + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join("config.yaml"); + let text = base(key); + std::fs::write(&p, &text).unwrap(); + let cfg = tw_config::try_parse(&text).unwrap(); + let gw = tw_gateway::AppState::new(cfg).unwrap(); + gw.set_plugin_engine(Arc::new(tw_gateway::plugin::fake::FakeEngine)); + let bus = gw.bus.clone(); + let state = ControlState { + shutdown: Default::default(), + remote: Default::default(), + cfg: Arc::new(ConfigManager::new(p, gw.clone(), bus)), + gateway: gw, + store: None, + started: std::time::Instant::now(), + price_updater: Default::default(), + chatgpt: Default::default(), + zai: Default::default(), + }; + Dev { + app: tw_control::router(state.clone()), + state, + _dir: dir, + } + } + + async fn call(&self, method: &str, path: &str, body: Value) -> (StatusCode, Value) { + let r = self + .app + .clone() + .oneshot( + Request::builder() + .method(method) + .uri(path) + .header("content-type", "application/json") + .body(Body::from(if body.is_null() { + String::new() + } else { + body.to_string() + })) + .unwrap(), + ) + .await + .unwrap(); + let status = r.status(); + let bytes = axum::body::to_bytes(r.into_body(), usize::MAX) + .await + .unwrap(); + ( + status, + serde_json::from_slice(&bytes).unwrap_or(Value::Null), + ) + } + + async fn ok(&self, method: &str, path: &str, body: Value) -> Value { + let (s, v) = self.call(method, path, body).await; + assert!(s.is_success(), "{method} {path}: {s} {v}"); + v + } + + fn config(&self) -> tw_config::Config { + self.state.config().as_ref().clone() + } + + fn provider(&self, name: &str) -> Option { + self.config().providers.into_iter().find(|p| p.name == name) + } + + /// 用户在界面上改了配置（`PUT /config`） + async fn edit(&self, f: impl FnOnce(&str) -> String) { + let cur = self.ok("GET", "/config", Value::Null).await; + let next = f(cur["text"].as_str().unwrap()); + self.ok( + "PUT", + "/config", + json!({ "base_version": cur["version"], "text": next }), + ) + .await; + } + + async fn enable(&self, loc: &std::path::Path, encrypt: bool, name: &str, join: Value) { + let mut body = json!({ + "target": folder(loc), "encrypt": encrypt, "device_name": name, "join": "use_remote", + }); + if encrypt { + body["passphrase"] = json!(PASS); + } + if let Value::Object(extra) = join { + for (k, v) in extra { + body[k] = v; + } + } + self.ok("POST", "/sync/enable", body).await; + } +} + +fn folder(loc: &std::path::Path) -> Value { + json!({ "type": "folder", "path": loc.display().to_string(), "proxy": "direct" }) +} + +fn add_raw(text: &str, yaml: &str) -> String { + if text.contains("\nproviders:\n") { + text.replacen("\nproviders:\n", &format!("\nproviders:\n{yaml}"), 1) + } else { + format!("{text}providers:\n{yaml}") + } +} + +const CHATGPT: &str = " - name: chatgpt\n base_url: https://chatgpt.com/backend-api/codex\n oauth:\n refresh: rt-a\n endpoint: https://auth.openai.com/oauth/token\n headers:\n ChatGPT-Account-Id: acct-a\n"; + +/// 用 YAML 改配置里的一个上游 +fn with_provider(text: &str, name: &str, f: impl FnOnce(&mut serde_yaml_ng::Value)) -> String { + let mut cfg: serde_yaml_ng::Value = serde_yaml_ng::from_str(text).unwrap(); + let p = cfg["providers"] + .as_sequence_mut() + .unwrap() + .iter_mut() + .find(|p| p["name"] == name) + .unwrap(); + f(p); + serde_yaml_ng::to_string(&cfg).unwrap() +} + +/// 新安装加入一个有 ChatGPT 上游（A 上登录着）的同步位置：加入得了，上游在 B 上停用着，A 上 +/// 照旧启用 +#[tokio::test] +async fn a_fresh_device_joins_a_location_with_a_signed_in_upstream() { + let loc = tempfile::tempdir().unwrap(); + let a = Dev::new('a'); + a.edit(|t| add_raw(t, CHATGPT)).await; + a.enable(loc.path(), false, "A", Value::Null).await; + let b = Dev::new('b'); + let preview = b + .ok( + "POST", + "/sync/enable/preview", + json!({ "target": folder(loc.path()) }), + ) + .await; + assert_eq!(preview["initial"], true); + b.enable(loc.path(), false, "B", Value::Null).await; + let p = b.provider("chatgpt").expect("到了 B"); + assert!(p.disabled && p.oauth.is_none()); + let v = a.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["conflicts"], 0, "{v}"); + let p = a.provider("chatgpt").unwrap(); + assert!(!p.disabled, "A 上没被停用"); + assert_eq!(p.oauth.unwrap().refresh, "rt-a"); +} + +/// 已有配置的设备以「合并」加入：导入的规矩在 B 上停用了 ChatGPT 上游，这个停用不随加入 +/// 上传、不回流到 A。之后 B 上登录：照同步的样子启用 +#[tokio::test] +async fn merging_in_keeps_the_upstream_on_where_it_is_signed_in() { + let loc = tempfile::tempdir().unwrap(); + let a = Dev::new('a'); + a.edit(|t| add_raw(t, CHATGPT)).await; + a.enable(loc.path(), false, "A", Value::Null).await; + let b = Dev::new('b'); + b.edit(|t| { + add_raw( + t, + " - name: mine\n base_url: https://mine.example\n key: sk-m\n", + ) + }) + .await; + b.enable( + loc.path(), + false, + "B", + json!({ + "join": "merge", + "decisions": [ { "item": { "kind": "key", "name": "default" }, "action": "skip" } ], + }), + ) + .await; + assert!(b.provider("chatgpt").unwrap().disabled); + let v = a.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["conflicts"], 0, "{v}"); + let p = a.provider("chatgpt").unwrap(); + assert!(!p.disabled, "B 的合并加入没有停掉 A 上的"); + assert!(a.provider("mine").is_some()); + + // A 改并发：B 收到，还停用着 + a.edit(|t| with_provider(t, "chatgpt", |p| p["max_concurrent"] = 4.into())) + .await; + a.ok("POST", "/sync/now", Value::Null).await; + b.ok("POST", "/sync/now", Value::Null).await; + let p = b.provider("chatgpt").unwrap(); + assert_eq!(p.max_concurrent, Some(4)); + assert!(p.disabled); + + // B 上登录（写上自己的登录，停用还在） + b.edit(|t| { + with_provider(t, "chatgpt", |p| { + p["oauth"] = serde_yaml_ng::from_str( + "{ refresh: rt-b, endpoint: 'https://auth.openai.com/oauth/token' }", + ) + .unwrap(); + p["headers"] = serde_yaml_ng::from_str("{ ChatGPT-Account-Id: acct-b }").unwrap(); + }) + }) + .await; + let v = b.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["conflicts"], 0, "{v}"); + let p = b.provider("chatgpt").unwrap(); + assert!(!p.disabled, "登录之后照同步的样子启用"); + assert_eq!(p.oauth.unwrap().refresh, "rt-b"); + let v = a.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["conflicts"], 0, "{v}"); + let p = a.provider("chatgpt").unwrap(); + assert!(!p.disabled); + assert_eq!(p.oauth.unwrap().refresh, "rt-a", "A 的登录还是 A 的"); +} + +/// keyring 不见了：要人处理的状态（`keyring_missing`），同步设置和口令都不动；「重新上传」 +/// 放回这台记着的 keyring 之后照常 +#[tokio::test] +async fn a_missing_keyring_waits_for_a_person() { + let loc = tempfile::tempdir().unwrap(); + let a = Dev::new('a'); + a.enable(loc.path(), true, "A", Value::Null).await; + a.ok("POST", "/sync/now", Value::Null).await; + std::fs::remove_file(loc.path().join("keyring")).unwrap(); + let v = a.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["state"], "keyring_missing", "{v}"); + assert_eq!(v["error"]["code"], "sync.keyring_missing"); + let s = a.config().sync.unwrap(); + assert!(s.encrypt && s.passphrase.as_deref() == Some(PASS)); + let v = a.ok("POST", "/sync/republish", Value::Null).await; + assert_eq!(v["state"], "idle", "{v}"); + assert_eq!(v["encrypted"], true); + assert!(loc.path().join("keyring").exists()); +} + +/// 冲突带着能选的处理：网关密钥三种都能选，「两者都保留」那一份是一把新的密钥 +#[tokio::test] +async fn each_conflict_says_which_choices_it_has() { + let loc = tempfile::tempdir().unwrap(); + let (a, b) = (Dev::new('a'), Dev::new('b')); + a.edit(|t| { + t.replace( + "clients:\n", + "clients:\n - name: k2\n key: tw-k2-0001\n", + ) + }) + .await; + a.enable(loc.path(), false, "A", Value::Null).await; + b.enable(loc.path(), false, "B", Value::Null).await; + a.edit(|t| { + t.replace( + " key: tw-k2-0001\n", + " key: tw-k2-0001\n max_concurrent: 2\n", + ) + }) + .await; + b.edit(|t| { + t.replace( + " key: tw-k2-0001\n", + " key: tw-k2-0001\n max_concurrent: 3\n", + ) + }) + .await; + a.ok("POST", "/sync/now", Value::Null).await; + b.ok("POST", "/sync/now", Value::Null).await; + let list = b.ok("GET", "/sync/conflicts", Value::Null).await; + let c = list + .as_array() + .unwrap() + .iter() + .find(|c| c["item"]["name"] == "k2") + .unwrap_or_else(|| panic!("{list}")); + assert_eq!(c["options"], json!(["local", "remote", "both"])); + b.ok( + "POST", + "/sync/conflicts", + json!({ "decisions": [ { "id": c["id"], "take": "both" } ] }), + ) + .await; + let clients = b.config().clients; + let copy = clients.iter().find(|c| c.name.starts_with("k2 (")).unwrap(); + assert_ne!(copy.key, "tw-k2-0001"); +} + +/// 关闭加密之后两台设备的口令都留着（同步位置上写着「已关闭」的 keyring 靠它核对） +#[tokio::test] +async fn turning_encryption_off_keeps_the_passphrase() { + let loc = tempfile::tempdir().unwrap(); + let (a, b) = (Dev::new('a'), Dev::new('b')); + a.enable(loc.path(), true, "A", Value::Null).await; + b.enable(loc.path(), true, "B", Value::Null).await; + let v = a + .ok( + "POST", + "/sync/encryption", + json!({ "action": "disable", "passphrase": PASS }), + ) + .await; + assert_eq!(v["encrypted"], false, "{v}"); + let s = a.config().sync.unwrap(); + assert!(!s.encrypt && s.passphrase.as_deref() == Some(PASS)); + let v = b.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["encrypted"], false, "{v}"); + let s = b.config().sync.unwrap(); + assert!(!s.encrypt, "B 跟着改成不加密"); + assert_eq!(s.passphrase.as_deref(), Some(PASS), "口令留着"); + // 再开回来：B 用记着的口令跟着改回加密 + a.ok( + "POST", + "/sync/encryption", + json!({ "action": "enable", "passphrase": PASS }), + ) + .await; + let v = b.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["encrypted"], true, "{v}"); + assert!(b.config().sync.unwrap().encrypt); +} diff --git a/crates/tw-sync/Cargo.toml b/crates/tw-sync/Cargo.toml index 62dbc2de..085d9243 100644 --- a/crates/tw-sync/Cargo.toml +++ b/crates/tw-sync/Cargo.toml @@ -41,4 +41,5 @@ tempfile = "3" base64 = { workspace = true } # 两台设备的测试直接读写配置 tw-config = { workspace = true } +serde_yaml_ng = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time"] } diff --git a/crates/tw-sync/src/device.rs b/crates/tw-sync/src/device.rs index 86350912..ff31e9d6 100644 --- a/crates/tw-sync/src/device.rs +++ b/crates/tw-sync/src/device.rs @@ -6,6 +6,9 @@ use serde::{Deserialize, Serialize}; use crate::keyring::Keyring; +/// 一份设备记录最大多少字节：几百字节的 JSON，再大的不是我们写的 +pub const MAX_BYTES: usize = 64 * 1024; + /// 设备记录多久没更新就从同步位置上移除 pub const STALE_AFTER_MS: u64 = 180 * 24 * 3600 * 1000; @@ -35,8 +38,15 @@ impl DeviceRecord { } } - /// 读回来。解不开、读不懂的是 `None`（别的设备刚改了加密设置、还没传完、被人改坏了） - pub fn open(bytes: &[u8], keyring: Option<&Keyring>) -> Option { + /// 读回来。解不开、读不懂的是 `None`（别的设备刚改了加密设置、还没传完、被人改坏了）。 + /// `sealed_only`：同步位置加着密 —— **不加密的记录一律不认**，那是没有口令的人放进来的 + pub fn open(bytes: &[u8], keyring: Option<&Keyring>, sealed_only: bool) -> Option { + if bytes.len() > MAX_BYTES { + return None; + } + if sealed_only && !crate::keyring::is_age(bytes) { + return None; + } let json = if crate::keyring::is_age(bytes) { keyring?.decrypt(bytes)? } else { @@ -68,20 +78,28 @@ mod tests { #[test] fn a_record_reads_back_plain_or_sealed() { let r = rec(); - assert_eq!(DeviceRecord::open(&r.seal(None), None), Some(r.clone())); - let k = Keyring::generate(); + assert_eq!( + DeviceRecord::open(&r.seal(None), None, false), + Some(r.clone()) + ); + // 加着密的同步位置：明文的记录不认 + assert_eq!( + DeviceRecord::open(&r.seal(None), Some(&Keyring::generate(0)), true), + None + ); + let k = Keyring::generate(0); let sealed = r.seal(Some(&k)); assert!( !String::from_utf8_lossy(&sealed).contains("张三"), "加密时名字不能是明文" ); - assert_eq!(DeviceRecord::open(&sealed, Some(&k)), Some(r.clone())); - assert_eq!(DeviceRecord::open(&sealed, None), None); + assert_eq!(DeviceRecord::open(&sealed, Some(&k), true), Some(r.clone())); + assert_eq!(DeviceRecord::open(&sealed, None, false), None); assert_eq!( - DeviceRecord::open(&sealed, Some(&Keyring::generate())), + DeviceRecord::open(&sealed, Some(&Keyring::generate(0)), true), None ); - assert_eq!(DeviceRecord::open(b"{not json", None), None); + assert_eq!(DeviceRecord::open(b"{not json", None, false), None); } #[test] diff --git a/crates/tw-sync/src/engine.rs b/crates/tw-sync/src/engine.rs index d2828f86..e92f7870 100644 --- a/crates/tw-sync/src/engine.rs +++ b/crates/tw-sync/src/engine.rs @@ -2,13 +2,20 @@ //! //! # 一轮里做什么 //! -//! 1. 列出 `versions/`，读出每个版本（加密的用版本密钥解开）。读不出来的当作尚未到达：最新 -//! 那一层有没到的就等下一轮，不拿一半去合并。 -//! 2. 找出所有的**头**（没有别的版本依据它的版本）。不止一个时（两台设备同时上传、云盘晚到 +//! 1. 看 keyring 定这一轮加不加密（[`Engine::security`]）。**加密不可降级**：本机设置为加密、 +//! 或者同步位置上有开着的 keyring 时，不加密的版本和设备记录一律不认；keyring 不见了、被换成 +//! 旧的不当作「关掉了加密」，要人处理（[`RoundError::KeyringMissing`]）。只有用口令解得开、 +//! 写着已关闭的 keyring 才算关了加密。 +//! 2. 列出 `versions/`，读出每个版本（加密的用版本密钥解开）。读不出来的分两种：**坏了**的（不是 +//! 配置包、不是这把密钥加密的、加密的位置上没加密）马上跳过；**可能没传完**的当作尚未到达， +//! 最新那一层有没到的就等下一轮，等了 30 分钟还读不出来也跳过。跳过的计数报给界面。 +//! 3. 找出所有的**头**（没有别的版本依据它的版本）。不止一个时（两台设备同时上传、云盘晚到 //! 的版本）先以它们共同的上一版本为基准两两合并，得到「同步位置的最新」。 -//! 3. 以本机基准（上次同步的那一版）为基准，和本机当前配置三方合并（[`crate::merge`]）。 -//! 冲突照常往下走：本机留本机的，上传的留同步位置的，记进冲突记录。 -//! 4. 本机的结果和现在不一样就应用（经控制面的写入流程，来源记为同步）；上传的结果和同步 +//! 4. 以本机基准（上次同步的那一版）为基准，和本机当前配置三方合并（[`crate::merge`]）。基准 +//! **得是同步位置最新的祖先**（版本记着自己的祖先，见 [`Meta::history`]）；不是的话（同步位置 +//! 被别的设备重新上传过、keyring 换过）按两方比较。冲突照常往下走：本机留本机的，上传的留 +//! 同步位置的，记进冲突记录。 +//! 5. 本机的结果和现在不一样就应用（经控制面的写入流程，来源记为同步）；上传的结果和同步 //! 位置上的不一样（或者合并了几个头、冲突记录变了）就上传一个新版本，然后删掉更早的。 //! //! # 冲突记录随版本走 @@ -16,6 +23,14 @@ //! 一处冲突在被处理之前，**两边各留各的**：参与的每台设备用自己的内容，同步版本里是先上传 //! 的那一边（[`Conflict::shared`]）。任一设备处理之后，冲突从记录里去掉、结果上传，别的设备 //! 看到记录没了，就照同步版本改成处理之后的样子。 +//! +//! # 等登录的上游 +//! +//! ChatGPT 账号的登录、OAuth 的令牌属于各自的设备，不随同步走。一台设备上登录着的上游到了 +//! 另一台没登录的设备上，**在那台停用着**（不然过不了校验）。这个停用是那台设备自己的事 +//! （[`State::held_login`]），和插件的停用一样：合并之前换回同步的样子，落地时再停用，上传的 +//! 里面永远是同步的样子 —— 不会因为一台没登录的设备把别的设备上用着的上游停掉。那台登录之后 +//! 照同步的样子启用。 use std::collections::{BTreeMap, BTreeSet}; use std::future::Future; @@ -29,17 +44,24 @@ use tw_pack::{Pack, PackError, PackKind, PackOrigin, Seal, Unseal}; use tw_types::{Msg, msg}; use crate::device::DeviceRecord; -use crate::keyring::{Keyring, KeyringError}; +use crate::keyring::{self, Keyring, KeyringError}; use crate::layout::{self, DEVICES, KEYRING, Listed, VERSIONS, VersionName}; use crate::local::{Local, State, UrlChange}; use crate::merge::{self, Objects}; use crate::order; -use crate::record::{self, Conflict, Meta, Reason, Side}; +use crate::record::{self, Conflict, HISTORY, Meta, Reason, Side}; use crate::store::{Store, StoreError}; /// 设备记录多久没写就写一次（最后同步时间），即便这一轮什么都没变 const DEVICE_RECORD_EVERY_MS: u64 = 6 * 3600 * 1000; +/// 一个可能没传完的版本（或者 keyring）等多久：之后跳过，不再让它挡着 +pub const SKIP_AFTER_MS: u64 = 30 * 60 * 1000; + +/// 同步位置看上去空了：连着这么多轮、过了这么久、自己的设备记录也不在了，才算真的清空了 +const EMPTY_ROUNDS: u32 = 3; +const EMPTY_AFTER_MS: u64 = 10 * 60 * 1000; + /// 本机这一边：配置从哪儿读、往哪儿写。控制面实现它（`tw-control` 的 `sync`），测试里换成 /// 两个目录里的两份配置 pub trait Host: Send + Sync { @@ -111,11 +133,17 @@ pub enum RoundError { Store(StoreError), /// 同步口令不对（别的设备改了口令），或者同步位置开了加密而这台设备还没有口令 Locked(Msg), + /// keyring 不见了、被换成了更早的那一份、一直读不出来：**要人处理**（放回这台设备记着的那 + /// 一份，或者停止同步）。不会因此关掉加密，也不删本机的口令 + KeyringMissing(Msg), /// 同步位置的最新版本由较新的 Core 写，本机读不了：暂停 TooNew { core_version: String }, /// 最新的版本还没传完（云盘还在下载）：下一轮再来，不算失败 NotArrived, - /// 同步位置被清空了（别的设备停止同步并删除了同步数据） + /// 同步位置看上去空了（网络盘没挂上时挂载点是个空目录）：再看几轮才下结论 + LooksEmpty, + /// 同步位置确实空了（别的设备停止同步并删除了同步数据，或者换成了别的位置）：**要人处理** + /// （重新上传本机配置，或者停止同步）。本机的同步设置不动 Emptied, /// 本机的配置读不了、写不进去 Local(Msg), @@ -133,7 +161,13 @@ impl RoundError { pub fn msg(&self) -> Msg { match self { RoundError::Store(e) => e.msg(), - RoundError::Locked(m) | RoundError::Local(m) => m.clone(), + RoundError::Locked(m) | RoundError::Local(m) | RoundError::KeyringMissing(m) => { + m.clone() + } + RoundError::TooNew { core_version } if core_version.is_empty() => msg!( + "sync.too_new_unknown" => + "another device uses a newer version. Update ThinkWatch Lite to keep syncing" + ), RoundError::TooNew { core_version } => msg!( "sync.too_new", version = core_version => "another device uses a newer version (Core {version}). Update ThinkWatch Lite to \ @@ -143,10 +177,15 @@ impl RoundError { "sync.not_arrived" => "the latest version has not finished arriving at the sync location" ), + RoundError::LooksEmpty => msg!( + "sync.looks_empty" => + "the sync location looks empty. It is checked a few more times before anything \ + is done" + ), RoundError::Emptied => msg!( - "sync.emptied" => - "the sync location is empty: another device stopped syncing and deleted the \ - sync data" + "sync.location_empty" => + "the sync location is empty: its sync data was deleted, or it is not the \ + location it was. Upload this device's configuration to it again, or stop syncing" ), RoundError::Stale => msg!( "sync.stale" => "the configuration changed during the sync; it will run again" @@ -155,6 +194,21 @@ impl RoundError { } } +fn locked_new() -> RoundError { + RoundError::Locked(msg!( + "sync.locked_new" => + "the sync location is now encrypted. Enter the sync passphrase to keep syncing" + )) +} + +fn keyring_missing() -> RoundError { + RoundError::KeyringMissing(msg!( + "sync.keyring_missing" => + "the keyring is missing from the sync location, so nothing was synced. Put back the \ + copy this device keeps, or stop syncing" + )) +} + /// 一轮做了什么 #[derive(Debug, Clone, Default)] pub struct Report { @@ -165,10 +219,14 @@ pub struct Report { /// 还没处理的冲突 pub conflicts: Vec, pub devices: Vec, - /// 同步位置此刻加没加密。和设置不一样时（别的设备关了加密）调用方改设置 + /// 同步位置此刻加没加密。和设置不一样时（别的设备开了、或者经口令认证关了加密）调用方 + /// 改设置 pub encrypted: bool, /// 本机应用之后值变了、或者被删掉的网关密钥 pub keys_changed: Vec, + /// 同步位置上跳过了的文件：坏了的、不是这把密钥加密的、加密的位置上没加密的、等了太久 + /// 还没传完的 + pub skipped: u32, } /// 读出来的一个版本 @@ -181,10 +239,40 @@ pub struct Version { pub content: String, } +/// 读过的版本（缓存）。可能没传完的不缓存，下一轮再读 #[derive(Clone)] enum Fetched { Ok(Arc), TooNew(String), + /// 读不出来、也不会自己好 + Broken, +} + +enum Got { + Ok(Arc), + TooNew(String), + Broken, + /// 可能还没传完 + Pending, + /// 列出来之后又没了 + Gone, +} + +/// 同步位置上的版本，读过一遍之后 +#[derive(Default)] +struct Scan { + /// 参与这一轮的：读出来的、版本过新的、还在等的 + names: Vec, + ok: Vec>, + too_new: Vec<(VersionName, String)>, + /// 可能没传完、还在等的 + pending: Vec, + /// 跳过了的 + skipped: Vec, + /// 跳过了的里面这台设备自己写的（keyring 换过之后解不开了）：重新上传之后删掉 + own_broken: Vec, + /// 同步位置上全部的版本名 + all: Vec, } /// 同步位置的最新（几个头合并之后） @@ -213,17 +301,33 @@ pub struct Existing { pub devices: Vec, } +/// 上一轮结束时同步位置上参与的版本、本机的内容、加没加密：都没变就什么都不用做 +#[derive(Debug, Clone, PartialEq, Eq)] +struct Quiet { + listing: Vec, + content: String, + sealed: bool, +} + pub struct Engine { pub store: S, pub host: H, local: Local, settings: Settings, - /// 解开了的版本密钥，和 keyring 文件的哈希（文件变了要重新解） - keyring: Option<(Keyring, [u8; 32])>, + /// 解开了的版本密钥（开着或关着的） + keyring: Option, + /// 这一份 keyring 文件原样的字节：同步成功之后存进本机（[`Local::save_keyring`]） + keyring_raw: Option>, + /// 同步位置开着加密：只认加密的版本和设备记录 + sealed: bool, + /// 上面这几项是照哪一份 keyring 定的（文件的哈希，没有 keyring 是 `Some(None)`）。没定过是 + /// `None` + mode_of: Option>, cache: BTreeMap, - /// 上一轮结束时同步位置上的名字和本机的内容：都没变就什么都不用做 - quiet: Option<(Vec, String)>, + quiet: Option, last_report: Report, + /// 这台设备以前留下的临时文件清过了 + swept: bool, } fn now_ms() -> u64 { @@ -433,6 +537,19 @@ pub struct Decision { pub new_name: Option, } +/// 一处冲突能怎么处理：本机的总能选；另一边的、两者都保留看冲突（[`Conflict::remote_allowed`]、 +/// [`Conflict::both_allowed`]） +pub fn options(c: &Conflict) -> Vec { + let mut out = vec![Take::Local]; + if c.remote_allowed() { + out.push(Take::Remote); + } + if c.both_allowed() { + out.push(Take::Both); + } + out +} + /// 「两者都保留」那一份的名字。自定义安全规则的名字带着分项（`content/项目代号`），只换规则名 fn both_name(id: &ItemId, given: Option<&str>, device_name: &str) -> String { let (prefix, plain) = match (id.kind, id.name.split_once('/')) { @@ -470,6 +587,11 @@ pub fn excluded(names: &BTreeSet, snaps: &[&Snapshot]) -> BTreeSet bool { + matches!(id.kind, ItemKind::Global | ItemKind::ClientPrefs) +} + /// 一个上游的地址（`conn.base_url`） fn base_url(it: &Item) -> Option<&str> { it.units.get(unit::CONN)?.get("base_url")?.as_str() @@ -480,6 +602,145 @@ fn key_value(it: &Item) -> Option<&Value> { it.units.get(unit::ALL)?.get("key") } +// ─────────────────────────────────────────────── 等登录的上游 + +/// 本机配置里的上游（带着这台设备自己的登录） +fn local_providers(p: &Pack) -> BTreeMap { + tw_config::try_parse(&p.config_yaml) + .map(|c| { + c.providers + .into_iter() + .map(|p| (p.name.clone(), p)) + .collect() + }) + .unwrap_or_default() +} + +/// 这个上游落到这台设备上没有登录可用：ChatGPT 账号这台没登录，OAuth 这台没有同一个令牌端点 +/// 的刷新令牌（落地时令牌取本机的，见 `tw_pack::model`） +fn needs_login(id: &ItemId, it: &Item, here: &BTreeMap) -> bool { + if id.kind != ItemKind::Provider { + return false; + } + let Ok(p) = model::provider_of(id, it) else { + return false; + }; + let mine = here.get(&id.name); + if model::is_chatgpt(&p) { + return !mine.is_some_and(|m| model::is_chatgpt(m) && m.oauth.is_some()); + } + match &p.oauth { + Some(o) => !mine + .and_then(|m| m.oauth.as_ref()) + .is_some_and(|m| m.endpoint == o.endpoint && !m.refresh.trim().is_empty()), + None => false, + } +} + +fn disabled(it: &Item) -> bool { + it.units + .get(unit::OTHER) + .and_then(|o| o.get("disabled")) + .and_then(Value::as_bool) + .unwrap_or(false) +} + +/// 停用一个上游 +fn force_disabled(it: &mut Item) { + let other = it + .units + .entry(unit::OTHER.to_string()) + .or_insert_with(|| Value::Object(Default::default())); + if let Value::Object(o) = other { + o.insert("disabled".into(), Value::Bool(true)); + } +} + +/// 一个上游的停用照 `from` 的样子 +fn copy_disabled(it: &mut Item, from: &Item) { + let want = from + .units + .get(unit::OTHER) + .and_then(|o| o.get("disabled")) + .cloned(); + let other = it + .units + .entry(unit::OTHER.to_string()) + .or_insert_with(|| Value::Object(Default::default())); + if let Value::Object(o) = other { + match want { + Some(v) => o.insert("disabled".into(), v), + None => o.remove("disabled"), + }; + } +} + +/// 落地之前：没登录、在这台停用着的上游停用掉，记下哪些是这台自己停用的（同步的样子是启用的） +fn hold_logins( + target: &mut Snapshot, + here: &BTreeMap, +) -> BTreeSet { + let mut held = BTreeSet::new(); + let ids: Vec = target + .of_kind(ItemKind::Provider) + .filter(|(id, it)| needs_login(id, it, here) && !disabled(it)) + .map(|(id, _)| id.clone()) + .collect(); + for id in ids { + if let Some(it) = target.items.get_mut(&id) { + force_disabled(it); + } + held.insert(id.name); + } + held +} + +// ─────────────────────────────────────────────── 版本的谱系 + +/// 版本标识 → 它记着的上一版本和更早的祖先 +type Index = BTreeMap, Vec)>; + +fn index_of<'a>(vs: impl Iterator) -> Index { + vs.map(|v| { + ( + v.name.id(), + (v.meta.parents.clone(), v.meta.history.clone()), + ) + }) + .collect() +} + +/// `start` 自己和它的全部祖先（按谱系能找到的） +fn ancestors(index: &Index, start: &str) -> BTreeSet { + let mut out = BTreeSet::new(); + let mut todo = vec![start.to_string()]; + while let Some(id) = todo.pop() { + if !out.insert(id.clone()) { + continue; + } + if let Some((ps, hs)) = index.get(&id) { + todo.extend(ps.iter().cloned()); + todo.extend(hs.iter().cloned()); + } + } + out +} + +/// 依据 `parents` 的新版本要记的更早的祖先：父版本们的祖先，最近的 [`HISTORY`] 个 +fn history_of(parents: &[Arc]) -> Vec { + let direct: BTreeSet = parents.iter().map(|p| p.name.id()).collect(); + let mut all: BTreeSet = parents + .iter() + .flat_map(|p| p.meta.history.iter().chain(&p.meta.parents)) + .filter(|id| !direct.contains(*id)) + .filter_map(|id| VersionName::parse(id)) + .collect(); + while all.len() > HISTORY { + all.pop_first(); + } + all.into_iter().map(|n| n.id()).collect() +} + impl Engine { /// `data_dir` 是配置文件所在的目录：本机的同步状态放在它底下的 `sync/` pub fn new(store: S, host: H, data_dir: &Path, settings: Settings) -> Self { @@ -489,9 +750,13 @@ impl Engine { local: Local::new(data_dir), settings, keyring: None, + keyring_raw: None, + sealed: false, + mode_of: None, cache: BTreeMap::new(), quiet: None, last_report: Report::default(), + swept: false, } } @@ -501,6 +766,7 @@ impl Engine { pub fn set_settings(&mut self, s: Settings) { if s.passphrase != self.settings.passphrase || s.encrypt != self.settings.encrypt { + self.mode_of = None; self.keyring = None; self.cache.clear(); } @@ -526,32 +792,73 @@ impl Engine { self.quiet = None; } - // ─────────────────────────────────────────────── 读 + // ─────────────────────────────────────────────── 加不加密 + + /// 照 keyring 定下来：开着、关着、没有。`of` 是 keyring 文件的哈希 + fn set_mode( + &mut self, + of: Option<[u8; 32]>, + keyring: Option, + raw: Option>, + ) -> bool { + if self.mode_of != Some(of) { + self.cache.clear(); + self.quiet = None; + } + self.mode_of = Some(of); + self.sealed = keyring.as_ref().is_some_and(|k| !k.off()); + self.keyring = keyring; + self.keyring_raw = raw; + self.sealed + } - /// 拿到版本密钥。没加密是 `Ok(None)` - async fn keyring(&mut self) -> Result, RoundError> { + /// 这一轮加不加密：keyring 在、开着是 `true`（之后只认加密的版本和设备记录）。 + /// + /// - 没有 keyring：本机设置为加密、或者以前见过 keyring 时是 [`RoundError::KeyringMissing`] + /// —— **不当作关掉了加密**； + /// - keyring 解得开：开着就加密；写着已关闭（有口令的人关的）就不加密，代号比见过的小 + /// （有人放回了一份旧的）是 [`RoundError::KeyringMissing`]； + /// - 本来就不加密、**没有口令**的设备认明文的「已关闭」标记（见 [`crate::keyring`]）；有口令 + /// 的设备只认解开之后写着的 + async fn security(&mut self, st: &mut State) -> Result { let Some(bytes) = self.store.get(KEYRING).await? else { - self.keyring = None; - return Ok(None); + st.unreadable.remove(KEYRING); + if self.settings.encrypt || st.keyring_gen > 0 { + return Err(keyring_missing()); + } + return Ok(self.set_mode(None, None, None)); }; let hash: [u8; 32] = Sha256::digest(&bytes).into(); - if self.keyring.as_ref().is_some_and(|(_, h)| *h == hash) { - return Ok(self.keyring.as_ref().map(|(k, _)| k)); + if self.mode_of == Some(Some(hash)) { + return Ok(self.sealed); } let Some(pass) = self.settings.passphrase.clone().filter(|p| !p.is_empty()) else { - return Err(RoundError::Locked(msg!( - "sync.locked_new" => - "the sync location is now encrypted. Enter the sync passphrase to keep syncing" - ))); + // 没有口令、本来就不加密的设备认明文的「已关闭」标记。**有口令的设备不认它**，只认 + // 解开之后写着的：标记谁都加得上 + if keyring::marked_off(&bytes) && !self.settings.encrypt { + return Ok(self.set_mode(Some(hash), None, None)); + } + return Err(locked_new()); }; - let k = tokio::task::spawn_blocking(move || Keyring::open(&bytes, &pass)) - .await - .map_err(local_err)?; - match k { + let opened = { + let b = bytes.clone(); + tokio::task::spawn_blocking(move || Keyring::open(&b, &pass)) + .await + .map_err(local_err)? + }; + match opened { Ok(k) => { - self.cache.clear(); - self.keyring = Some((k, hash)); - Ok(self.keyring.as_ref().map(|(k, _)| k)) + st.unreadable.remove(KEYRING); + if k.off() && k.generation() < st.keyring_gen { + return Err(RoundError::KeyringMissing(msg!( + "sync.keyring_rolled_back" => + "the keyring at the sync location is older than the one this device \ + saw last, so nothing was synced. Put back the copy this device keeps, \ + or stop syncing" + ))); + } + st.keyring_gen = st.keyring_gen.max(k.generation()); + Ok(self.set_mode(Some(hash), Some(k), Some(bytes))) } Err(KeyringError::WrongPassphrase) => Err(RoundError::Locked(msg!( "sync.passphrase_changed" => @@ -560,25 +867,57 @@ impl Engine { Err(KeyringError::TooNew) => Err(RoundError::TooNew { core_version: String::new(), }), - Err(KeyringError::Corrupt) => Err(RoundError::NotArrived), + Err(KeyringError::Corrupt) => { + let since = *st + .unreadable + .entry(KEYRING.to_string()) + .or_insert_with(now_ms); + if now_ms().saturating_sub(since) >= SKIP_AFTER_MS { + Err(RoundError::KeyringMissing(KeyringError::Corrupt.msg())) + } else { + Err(RoundError::NotArrived) + } + } } } - /// 读一个版本。读不出来（没传完、坏了、不是这把密钥加密的）是 `None` - async fn fetch(&mut self, name: &VersionName) -> Result, RoundError> { + /// 解开了的 keyring 写设备记录、版本时用不用：加密开着才用 + fn sealing(&self) -> Option<&Keyring> { + self.keyring.as_ref().filter(|_| self.sealed) + } + + // ─────────────────────────────────────────────── 读 + + /// 读一个版本 + async fn fetch(&mut self, name: &VersionName) -> Result { if let Some(f) = self.cache.get(name) { - return Ok(Some(f.clone())); + return Ok(match f.clone() { + Fetched::Ok(v) => Got::Ok(v), + Fetched::TooNew(c) => Got::TooNew(c), + Fetched::Broken => Got::Broken, + }); } let Some(bytes) = self.store.get(&name.path()).await? else { - return Ok(None); + return Ok(Got::Gone); }; - let identity = self.keyring.as_ref().map(|(k, _)| k.identity().clone()); + // 空的：云盘的占位文件、正在写 + if bytes.is_empty() { + return Ok(Got::Pending); + } + // 认得出是配置包（ZIP）或者加密过的：读不出来多半是没传完 + let shaped = keyring::is_age(&bytes) || bytes.starts_with(b"PK\x03\x04"); + let identity = self.keyring.as_ref().map(|k| k.identity().clone()); + let strict = self.sealed; let decoded = tokio::task::spawn_blocking(move || { let unseal = match &identity { Some(id) => Unseal::Identity(id), None => Unseal::None, }; - let pack = tw_pack::read(&bytes, unseal)?; + let pack = if strict { + tw_pack::read_strict(&bytes, unseal)? + } else { + tw_pack::read(&bytes, unseal)? + }; let snap = model::snapshot(&pack); Ok::<_, PackError>((pack, snap)) }) @@ -586,47 +925,54 @@ impl Engine { .map_err(local_err)?; let fetched = match decoded { Err(PackError::TooNew { core_version, .. }) => Fetched::TooNew(core_version), - Err(_) => return Ok(None), + // 加密的版本，而 keyring 还没到 + Err(PackError::NeedKey) => return Ok(Got::Pending), + Err(PackError::Corrupt { .. }) if shaped => return Ok(Got::Pending), + Err(_) => Fetched::Broken, Ok((pack, Err(e))) if e.is_too_new() => Fetched::TooNew(pack.manifest.core_version), - Ok((_, Err(_))) => return Ok(None), + Ok((_, Err(_))) => Fetched::Broken, Ok((pack, Ok(snap))) => { - let Some(meta) = pack + let meta = pack .manifest .sync .clone() - .and_then(|v| serde_json::from_value::(v).ok()) - else { - return Ok(None); - }; - // 名字和内容对不上（被人挪过、改过名），或者不加密的版本校验和对不上：当作坏了 - if meta.seq != name.seq || layout::device_tag(&meta.device) != name.device { - return Ok(None); - } - if !pack.manifest.encrypted - && meta.sha256.as_deref() != Some(content_sha256(&pack).as_str()) - { - return Ok(None); + .and_then(|v| serde_json::from_value::(v).ok()); + match meta { + // 名字和内容对不上（被人挪过、改过名），或者不加密的版本校验和对不上：坏了 + Some(meta) + if meta.seq == name.seq + && layout::device_tag(&meta.device) == name.device + && (pack.manifest.encrypted + || meta.sha256.as_deref() + == Some(content_sha256(&pack).as_str())) => + { + let content = pack.content_id(); + Fetched::Ok(Arc::new(Version { + name: name.clone(), + pack, + meta, + snap, + content, + })) + } + _ => Fetched::Broken, } - let content = pack.content_id(); - Fetched::Ok(Arc::new(Version { - name: name.clone(), - pack, - meta, - snap, - content, - })) } }; self.cache.insert(name.clone(), fetched.clone()); - Ok(Some(fetched)) + Ok(match fetched { + Fetched::Ok(v) => Got::Ok(v), + Fetched::TooNew(c) => Got::TooNew(c), + Fetched::Broken => Got::Broken, + }) } - fn base(&self) -> Option { - let bytes = self.local.base()?; - let pack = tw_pack::read(&bytes, Unseal::None).ok()?; + /// 这一份基准（不加密的配置包）读成版本 + fn version_of(bytes: &[u8], id: &str) -> Option { + let pack = tw_pack::read(bytes, Unseal::None).ok()?; let meta: Meta = serde_json::from_value(pack.manifest.sync.clone()?).ok()?; let snap = model::snapshot(&pack).ok()?; - let name = VersionName::parse(&self.local.state().base?)?; + let name = VersionName::parse(id)?; let content = pack.content_id(); Some(Version { name, @@ -637,17 +983,12 @@ impl Engine { }) } - /// 同步位置上的版本：读得出来的、读不出来的名字 - async fn versions( - &mut self, - ) -> Result< - ( - Vec, - Vec>, - Vec<(VersionName, String)>, - ), - RoundError, - > { + fn base(&self) -> Option { + Self::version_of(&self.local.base()?, &self.local.state().base?) + } + + /// 同步位置上的版本名 + async fn listing(&self) -> Result, RoundError> { let mut names: Vec = self .store .list(VERSIONS) @@ -656,30 +997,97 @@ impl Engine { .filter_map(|e| VersionName::parse(&e.name)) .collect(); names.sort(); - self.cache.retain(|n, _| names.contains(n)); - let mut ok = Vec::new(); - let mut too_new = Vec::new(); - for n in &names { + names.dedup(); + Ok(names) + } + + /// 读一遍同步位置上的版本，分出读得出来的、过新的、还在等的、跳过的 + async fn scan(&mut self, all: Vec, st: &mut State) -> Result { + self.cache.retain(|n, _| all.contains(n)); + let me = layout::device_tag(&self.settings.device_id); + let now = now_ms(); + let mut sc = Scan::default(); + for n in &all { match self.fetch(n).await? { - Some(Fetched::Ok(v)) => ok.push(v), - Some(Fetched::TooNew(cv)) => too_new.push((n.clone(), cv)), - None => {} + Got::Ok(v) => { + st.unreadable.remove(&n.id()); + sc.names.push(n.clone()); + sc.ok.push(v); + } + Got::TooNew(cv) => { + st.unreadable.remove(&n.id()); + sc.names.push(n.clone()); + sc.too_new.push((n.clone(), cv)); + } + Got::Broken => { + st.unreadable.remove(&n.id()); + sc.skipped.push(n.clone()); + if n.device == me { + sc.own_broken.push(n.clone()); + } + } + Got::Pending => { + let since = *st.unreadable.entry(n.id()).or_insert(now); + if now.saturating_sub(since) >= SKIP_AFTER_MS { + sc.skipped.push(n.clone()); + } else { + sc.names.push(n.clone()); + sc.pending.push(n.clone()); + } + } + Got::Gone => {} } } - Ok((names, ok, too_new)) + st.unreadable + .retain(|k, _| k == KEYRING || all.iter().any(|n| &n.id() == k)); + sc.all = all; + Ok(sc) } - /// 同步位置上的设备记录。半年没更新的顺手删掉 + /// 头：没有别的版本依据它的版本。最新的是较新的 Core 写的是 [`RoundError::TooNew`]，最新那 + /// 一层还没到齐是 [`RoundError::NotArrived`] + fn heads(sc: &Scan) -> Result>, RoundError> { + let referenced: BTreeSet = sc + .ok + .iter() + .flat_map(|v| v.meta.parents.iter().cloned()) + .collect(); + if let Some((_, cv)) = sc + .too_new + .iter() + .find(|(n, _)| !referenced.contains(&n.id())) + { + return Err(RoundError::TooNew { + core_version: cv.clone(), + }); + } + let top_ok = sc.ok.iter().map(|v| v.name.seq).max(); + let top_all = sc.names.iter().map(|n| n.seq).max(); + if top_ok.is_none() || top_ok < top_all { + return Err(RoundError::NotArrived); + } + Ok(sc + .ok + .iter() + .filter(|v| !referenced.contains(&v.name.id())) + .cloned() + .collect()) + } + + /// 同步位置上的设备记录。半年没更新的顺手删掉。加密着时不加密的记录不认 async fn devices(&mut self) -> Result, RoundError> { - let kr = self.keyring.as_ref().map(|(k, _)| k); + let kr = self.keyring.as_ref(); let now = now_ms(); let mut out = Vec::new(); for e in self.store.list(DEVICES).await? { + if !layout::is_device_file(&e.name) { + continue; + } let path = format!("{DEVICES}/{}", e.name); let Some(bytes) = self.store.get(&path).await? else { continue; }; - let Some(rec) = DeviceRecord::open(&bytes, kr) else { + let Some(rec) = DeviceRecord::open(&bytes, kr, self.sealed) else { continue; }; if rec.is_stale(now) && rec.id != self.settings.device_id { @@ -692,7 +1100,7 @@ impl Engine { Ok(out) } - async fn write_device(&mut self, paused: bool) -> Result<(), RoundError> { + async fn write_device(&mut self, st: &mut State, paused: bool) -> Result<(), RoundError> { let rec = DeviceRecord { id: self.settings.device_id.clone(), name: self.settings.device_name.clone(), @@ -701,22 +1109,47 @@ impl Engine { paused_update: paused, remote_core: self.settings.remote_core, }; - let kr = self.keyring.as_ref().map(|(k, _)| k); + let bytes = rec.seal(self.sealing()); self.store - .put( - &layout::device_path(&self.settings.device_id), - &rec.seal(kr), - ) + .put(&layout::device_path(&self.settings.device_id), &bytes) .await?; - let mut st = self.local.state(); st.device_record_ms = rec.last_seen_ms; st.device_record_paused = paused; - self.local.save_state(&st).map_err(RoundError::Local)?; + Ok(()) + } + + /// 存下本机的同步状态，连同这一轮解开的 keyring 的原样（留着放回去用） + fn commit(&self, st: &State) -> Result<(), RoundError> { + self.local.save_state(st).map_err(RoundError::Local)?; + if let Some(raw) = &self.keyring_raw { + self.local.save_keyring(raw).map_err(RoundError::Local)?; + } Ok(()) } // ─────────────────────────────────────────────── 合并几个头 + /// 同步位置上早删了的版本：本机之前的基准里找，本机备份里按内容找 + fn elsewhere(&self, id: &str, content: Option<&String>) -> Option { + if let Some(v) = self + .local + .base_by_id(id) + .and_then(|b| Self::version_of(&b, id)) + { + return Some(v); + } + let content = content?; + let pack = self.host.backup(content)?; + let snap = model::snapshot(&pack).ok()?; + Some(Version { + name: VersionName::parse(id)?, + meta: Meta::default(), + content: content.clone(), + snap, + pack, + }) + } + /// `a`、`b` 共同的上一版本 fn common_ancestor( &self, @@ -725,35 +1158,18 @@ impl Engine { all: &[Arc], base: Option<&Version>, ) -> Option { - // 版本标识 → (上一版本, 它们的内容标识) - let mut parents: BTreeMap, Vec)> = BTreeMap::new(); - for v in all.iter().map(|v| v.as_ref()).chain(base) { - parents.insert( - v.name.id(), - (v.meta.parents.clone(), v.meta.parent_content.clone()), - ); - } + let index = index_of(all.iter().map(|v| v.as_ref()).chain(base)); let mut content_of: BTreeMap = BTreeMap::new(); - for (ps, cs) in parents.values() { - for (p, c) in ps.iter().zip(cs) { + for v in all.iter().map(|v| v.as_ref()).chain(base) { + for (p, c) in v.meta.parents.iter().zip(&v.meta.parent_content) { content_of.insert(p.clone(), c.clone()); } } - let ancestors = |start: &str| -> BTreeSet { - let mut out = BTreeSet::new(); - let mut todo = vec![start.to_string()]; - while let Some(id) = todo.pop() { - if !out.insert(id.clone()) { - continue; - } - if let Some((ps, _)) = parents.get(&id) { - todo.extend(ps.iter().cloned()); - } - } - out - }; - let left: BTreeSet = a.iter().flat_map(|v| ancestors(&v.name.id())).collect(); - let right = ancestors(&b.name.id()); + let left: BTreeSet = a + .iter() + .flat_map(|v| ancestors(&index, &v.name.id())) + .collect(); + let right = ancestors(&index, &b.name.id()); let mut common: Vec = left .intersection(&right) .filter_map(|id| VersionName::parse(id)) @@ -766,16 +1182,7 @@ impl Engine { if let Some(b) = base.filter(|b| b.name.id() == id) { return Some(b.clone()); } - let content = content_of.get(&id)?; - let pack = self.host.backup(content)?; - let snap = model::snapshot(&pack).ok()?; - Some(Version { - name: VersionName::parse(&id)?, - meta: Meta::default(), - content: content.clone(), - snap, - pack, - }) + self.elsewhere(&id, content_of.get(&id)) } /// 几个头合并成一份 @@ -818,6 +1225,7 @@ impl Engine { remote: &ro, hold_items: &hold_items, hold_units: &hold_units, + fieldwise: &fieldwise, }); let mut snap = rebuild(&out.shared, &[&h.snap, &acc.snap]); for (o, k) in ORDERS { @@ -903,6 +1311,16 @@ impl Engine { // ─────────────────────────────────────────────── 写 + /// 新版本的序号：比参与的版本都大，和同步位置上已有的名字不重 + fn next_seq(&self, sc_names: &[VersionName], all: &[VersionName]) -> u64 { + let mut seq = sc_names.iter().map(|n| n.seq).max().unwrap_or(0) + 1; + let tag = layout::device_tag(&self.settings.device_id); + while all.iter().any(|n| n.seq == seq && n.device == tag) { + seq += 1; + } + seq + } + /// 上传一个新版本：内容是 `shared`，依据 `parents` async fn publish( &mut self, @@ -927,16 +1345,17 @@ impl Engine { device_name: self.settings.device_name.clone(), parents: parents.iter().map(|p| p.name.id()).collect(), parent_content: parents.iter().map(|p| p.content.clone()).collect(), + history: history_of(parents), sha256: None, conflicts, }; - let encrypted = self.keyring.is_some(); + let encrypted = self.sealed; if !encrypted { meta.sha256 = Some(content_sha256(&pack)); } pack.manifest.sync = Some(serde_json::to_value(&meta).expect("plain data serializes")); let bytes = { - let recipient = self.keyring.as_ref().map(|(k, _)| k.recipient()); + let recipient = self.sealing().map(|k| k.recipient()); let seal = match &recipient { Some(r) => Seal::Recipient(r), None => Seal::None, @@ -965,33 +1384,52 @@ impl Engine { fn save_base(&self, v: &Version, st: &mut State) -> Result<(), RoundError> { let bytes = tw_pack::write(&v.pack, Seal::None).map_err(|e| RoundError::Local(e.msg()))?; - self.local.save_base(&bytes).map_err(RoundError::Local)?; + self.local + .save_base(&v.name.id(), &bytes) + .map_err(RoundError::Local)?; st.base = Some(v.name.id()); Ok(()) } - async fn prune(&mut self, names: &[VersionName], ok: &[Arc]) { + /// 上传之后删掉更早的版本（[`layout::to_delete`]），连同比留下的都早的、跳过了的文件。删了 + /// 哪些 + async fn prune( + &mut self, + names: &[VersionName], + ok: &[Arc], + skipped: &[VersionName], + ) -> BTreeSet { + let ids = |v: &[String]| -> Vec { + v.iter().filter_map(|p| VersionName::parse(p)).collect() + }; let listed: Vec = names .iter() - .map(|n| Listed { - name: n.clone(), - parents: ok - .iter() - .find(|v| &v.name == n) - .map(|v| { - v.meta - .parents - .iter() - .filter_map(|p| VersionName::parse(p)) - .collect() - }) - .unwrap_or_default(), + .map(|n| { + let meta = ok.iter().find(|v| &v.name == n).map(|v| &v.meta); + Listed { + name: n.clone(), + parents: meta.map(|m| ids(&m.parents)).unwrap_or_default(), + history: meta.map(|m| ids(&m.history)).unwrap_or_default(), + } }) .collect(); - for n in layout::to_delete(&listed) { - let _ = self.store.delete(&n.path()).await; - self.cache.remove(&n); + let del: BTreeSet = layout::to_delete(&listed).into_iter().collect(); + let oldest_kept = names + .iter() + .filter(|n| !del.contains(*n)) + .map(|n| n.seq) + .min(); + let mut gone = BTreeSet::new(); + let below = skipped + .iter() + .filter(|n| oldest_kept.is_some_and(|min| n.seq < min)); + for n in del.iter().chain(below) { + if self.store.delete(&n.path()).await.is_ok() { + self.cache.remove(n); + gone.insert(n.clone()); + } } + gone } // ─────────────────────────────────────────────── 一轮 @@ -1007,92 +1445,147 @@ impl Engine { if !decisions.is_empty() { self.quiet = None; } - match self.round_inner(decisions).await { - Ok(r) => { - self.last_report = r.clone(); - Ok(r) - } - Err(e) => Err(e), - } - } - - async fn round_inner(&mut self, decisions: &[Decision]) -> Result { let mut st = self.local.state(); if st.place != self.settings.place && !st.place.is_empty() { // 换了同步位置：原来的基准是那个位置上的，作废 self.local.clear(); - st = State { - place: self.settings.place.clone(), - ..Default::default() - }; - self.local.save_state(&st).map_err(RoundError::Local)?; + st = State::default(); self.cache.clear(); + self.mode_of = None; } st.place.clone_from(&self.settings.place); - let base = self.base(); - let kr = self.keyring().await?.is_some(); - let (names, ok, too_new) = self.versions().await?; - if names.is_empty() { - if base.is_some() { - return Err(RoundError::Emptied); - } + if !self.swept { + self.store + .sweep(&layout::device_tag(&self.settings.device_id)) + .await; + self.swept = true; + } + let r = self.round_inner(decisions, &mut st).await; + // 成没成都存：读不出来的文件从什么时候起、见过的 keyring 代号、看上去空了几轮 + self.commit(&st)?; + if let Ok(r) = &r { + self.last_report = r.clone(); + } + r + } + + /// 同步位置上没有可用的版本 + async fn on_empty(&mut self, st: &mut State, has_base: bool) -> Result { + if !has_base { // 开启同步时上传第一个版本没成（中途断了）：这一轮补上 - self.publish_first().await?; - return self.finish_quiet(Vec::new(), kr).await; + self.security(st).await?; + let all = self.listing().await?; + let seq = self.next_seq(&[], &all); + self.publish_first(seq, st).await?; + st.empty_rounds = 0; + st.empty_since_ms = None; + return Ok(Report { + encrypted: self.sealed, + devices: self.devices().await?, + ..Default::default() + }); + } + let now = now_ms(); + st.empty_rounds = st.empty_rounds.saturating_add(1); + let since = *st.empty_since_ms.get_or_insert(now); + let mine = self + .store + .get(&layout::device_path(&self.settings.device_id)) + .await?; + if st.empty_rounds >= EMPTY_ROUNDS + && now.saturating_sub(since) >= EMPTY_AFTER_MS + && mine.is_none() + { + return Err(RoundError::Emptied); } + Err(RoundError::LooksEmpty) + } + + async fn round_inner( + &mut self, + decisions: &[Decision], + st: &mut State, + ) -> Result { + let base = self.base(); + let listed = self.listing().await?; + if listed.is_empty() { + return self.on_empty(st, base.is_some()).await; + } + let kr = self.security(st).await?; + let sc = self.scan(listed, st).await?; + if sc.names.is_empty() { + return self.on_empty(st, base.is_some()).await; + } + st.empty_rounds = 0; + st.empty_since_ms = None; + let skipped = sc.skipped.len() as u32; let cur = self.host.current().map_err(RoundError::Local)?; - let listing: Vec = names.iter().map(|n| n.id()).collect(); - let local_content = cur.pack.content_id(); - if self.quiet.as_ref() == Some(&(listing.clone(), local_content.clone())) { + // 本机的内容按原文算（含这台设备自己的登录：登录了要放开等登录的上游） + let quiet = Quiet { + listing: sc.names.iter().map(|n| n.id()).collect(), + content: content_sha256(&cur.pack), + sealed: kr, + }; + if sc.pending.is_empty() && sc.own_broken.is_empty() && self.quiet.as_ref() == Some(&quiet) + { // 什么都没变：冲突还是上一轮的。照样记一次同步成功，设备记录隔一阵写一次，设备 // 列表重读（别的设备的记录会变） st.last_synced_ms = Some(now_ms()); st.failing_since_ms = None; st.error = None; - self.local.save_state(&st).map_err(RoundError::Local)?; if now_ms().saturating_sub(st.device_record_ms) > DEVICE_RECORD_EVERY_MS { - self.write_device(false).await?; + self.write_device(st, false).await?; } - let r = Report { + return Ok(Report { encrypted: kr, conflicts: self.last_report.conflicts.clone(), devices: self.devices().await?, + skipped, ..Default::default() - }; - return Ok(r); - } - // 头：没有别的版本依据它的版本 - let referenced: BTreeSet = ok - .iter() - .flat_map(|v| v.meta.parents.iter().cloned()) - .collect(); - if let Some((_, cv)) = too_new.iter().find(|(n, _)| !referenced.contains(&n.id())) { - if !st.device_record_paused { - self.write_device(true).await?; - } - return Err(RoundError::TooNew { - core_version: cv.clone(), }); } - let top_ok = ok.iter().map(|v| v.name.seq).max(); - let top_all = names.iter().map(|n| n.seq).max(); - if top_ok.is_none() || top_ok < top_all { - return Err(RoundError::NotArrived); - } - let heads: Vec> = ok - .iter() - .filter(|v| !referenced.contains(&v.name.id())) - .cloned() - .collect(); - // 同步位置没有 keyring、版本却是加密的：keyring 还没传到 - if !kr && heads.iter().any(|h| h.pack.manifest.encrypted) { - return Err(RoundError::NotArrived); - } - let remote = self.fold(&heads, &ok, base.as_ref()); + let heads = match Self::heads(&sc) { + Ok(h) => h, + Err(e @ RoundError::TooNew { .. }) => { + if !st.device_record_paused { + self.write_device(st, true).await?; + } + return Err(e); + } + Err(e) => return Err(e), + }; + // 基准得是同步位置最新的祖先：不是的话（同步位置被重新上传过、keyring 换过）按两方比较 + let index = index_of(sc.ok.iter().map(|v| v.as_ref()).chain(base.as_ref())); + let base = base.filter(|b| { + heads + .iter() + .any(|h| ancestors(&index, &h.name.id()).contains(&b.name.id())) + }); + let remote = self.fold(&heads, &sc.ok, base.as_ref()); let local_snap = model::snapshot(&cur.pack).map_err(|e| RoundError::Local(e.msg()))?; + let here = local_providers(&cur.pack); let me = self.settings.device_id.clone(); let mut local = local_snap.clone(); + // 等登录的上游：这台的停用不算本机的改动，换回同步的样子再合并（登录了的这一轮也换回 + // 去，落地时就照同步的样子启用） + let neutral: BTreeSet = local + .of_kind(ItemKind::Provider) + .filter(|(id, it)| needs_login(id, it, &here)) + .map(|(id, _)| id.name.clone()) + .chain(st.held_login.iter().cloned()) + .collect(); + for name in &neutral { + let id = ItemId::new(ItemKind::Provider, name.clone()); + let synced = base + .as_ref() + .and_then(|b| b.snap.items.get(&id)) + .or(remote.snap.items.get(&id)) + .cloned(); + if let (Some(it), Some(s)) = (local.items.get_mut(&id), synced) { + copy_disabled(it, &s); + } + } // 别的设备处理掉了的冲突：照同步版本改 let open_ids: BTreeSet<&str> = remote.conflicts.iter().map(|c| c.id.as_str()).collect(); if let Some(b) = &base { @@ -1134,6 +1627,13 @@ impl Engine { let Some(pos) = open.iter().position(|c| c.id == d.id) else { continue; }; + if d.take == Take::Remote && !open[pos].remote_allowed() { + return Err(RoundError::Local(msg!( + "sync.take_not_allowed" => + "using the other side is not possible for this conflict: the result would \ + not pass validation" + ))); + } let c = open.remove(pos); let local_units = local.items.get(&c.item).map(|it| c.pick(&it.units)); let (_, theirs) = c.mine(&me, local_units.as_ref()); @@ -1176,6 +1676,14 @@ impl Engine { for (u, v) in theirs.units.iter().flatten() { copy.units.insert(u.clone(), v.clone()); } + // 网关密钥的另一份是一把新的密钥：同一个值出现两次，客户端用的是哪一把 + // 就说不清了。它也不是为哪个客户端生成的 + if c.item.kind == ItemKind::Key + && let Some(Value::Object(all)) = copy.units.get_mut(unit::ALL) + { + all.insert("key".into(), Value::String(tw_config::generate_key())); + all.remove("client"); + } local.items.insert(id, copy); local.relink(); } @@ -1214,6 +1722,7 @@ impl Engine { remote: &ro, hold_items: &hold_items, hold_units: &hold_units, + fieldwise: &fieldwise, }); let mut target = rebuild(&out.local, &[&local, &remote.snap]); let mut shared = rebuild(&out.shared, &[&remote.snap, &local]); @@ -1294,10 +1803,15 @@ impl Engine { } target.relink(); shared.relink(); + // 没登录的上游在这台停用着 + st.held_login = hold_logins(&mut target, &here); + let mut start = local.clone(); + hold_logins(&mut start, &here); // 合并的结果要过配置校验：过不了的那几项留本机的，作为冲突列出 self.check_target( &cur.pack, + &start, &local, &mut target, &mut shared, @@ -1363,8 +1877,10 @@ impl Engine { let mut report = Report { encrypted: kr, + skipped, ..Default::default() }; + let mut content_after = quiet.content.clone(); // 应用 if model::normalized(&target) != model::normalized(&local_snap) { report.keys_changed = local_snap @@ -1376,6 +1892,7 @@ impl Engine { .collect(); let pack = model::materialize(&cur.pack, &target).map_err(|e| RoundError::Local(e.msg()))?; + content_after = content_sha256(&pack); let from: Vec = { let mut v: Vec = remote .heads @@ -1396,28 +1913,36 @@ impl Engine { } // 上传 let need_upload = heads.len() > 1 + || !sc.own_broken.is_empty() || model::normalized(&shared) != model::normalized(&remote.snap) || conflicts != remote.conflicts; + let mut listing: BTreeSet = sc.names.iter().cloned().collect(); let next_base = if need_upload { - let seq = names.iter().map(|n| n.seq).max().unwrap_or(0) + 1; + let seq = self.next_seq(&sc.names, &sc.all); let v = self .publish(&shared, seq, &heads, conflicts.clone()) .await?; report.uploaded = Some(v.name.id()); - let mut all: Vec = names.clone(); - all.push(v.name.clone()); - let mut ok_all = ok.clone(); + listing.insert(v.name.clone()); + let all: Vec = listing.iter().cloned().collect(); + let mut ok_all = sc.ok.clone(); ok_all.push(Arc::new(v.clone())); - self.prune(&all, &ok_all).await; + let gone = self.prune(&all, &ok_all, &sc.skipped).await; + // 这台设备自己写的、换了 keyring 之后解不开的：内容已经在新版本里了 + for n in &sc.own_broken { + if !gone.contains(n) && self.store.delete(&n.path()).await.is_ok() { + self.cache.remove(n); + } + } + listing.retain(|n| !gone.contains(n)); v } else { (*heads[0]).clone() }; - self.save_base(&next_base, &mut st)?; + self.save_base(&next_base, st)?; st.last_synced_ms = Some(now_ms()); st.failing_since_ms = None; st.error = None; - self.local.save_state(&st).map_err(RoundError::Local)?; report.conflicts = conflicts; let changed = report.uploaded.is_some() || !report.applied_from.is_empty(); @@ -1425,49 +1950,30 @@ impl Engine { || st.device_record_paused || now_ms().saturating_sub(st.device_record_ms) > DEVICE_RECORD_EVERY_MS { - self.write_device(false).await?; + self.write_device(st, false).await?; } report.devices = self.devices().await?; - // 这一轮之后同步位置上的样子 - let after: Vec = self - .store - .list(VERSIONS) - .await? - .into_iter() - .filter_map(|e| VersionName::parse(&e.name).map(|n| n.id())) - .collect(); - let local_after = self - .host - .current() - .map(|c| c.pack.content_id()) - .unwrap_or_default(); - self.quiet = Some((after, local_after)); + // 下一轮的捷径照这一轮**开始时**看到的算，加上这一轮自己做的：这一轮中途别的设备传上 + // 来的、本机被改的，下一轮都看得出来 + self.quiet = sc.pending.is_empty().then(|| Quiet { + listing: listing.iter().map(|n| n.id()).collect(), + content: content_after, + sealed: kr, + }); Ok(report) } - async fn finish_quiet( - &mut self, - conflicts: Vec, - encrypted: bool, - ) -> Result { - let devices = self.devices().await?; - let r = Report { - conflicts, - devices, - encrypted, - ..Default::default() - }; - Ok(r) - } - - /// 合并出来的本机结果过一遍配置校验。过不了时一项一项试：从本机现在的样子出发，把变了的 - /// 项目一个个换过去，换了就过不了的那一项留本机的，记成冲突（`invalid`），上传的里面也留 - /// 本机的 —— 不然别的设备收到的也是一份过不了校验的配置 + /// 合并出来的本机结果过一遍配置校验。过不了时一项一项试：从本机现在的样子（`start`）出发， + /// 把变了的项目一个个换过去，换了就过不了的那一项留本机的，记成冲突（`invalid`），上传的 + /// 里面也留本机的（`mine`：这台停用着的上游是同步的样子）—— 不然别的设备收到的也是一份 + /// 过不了校验的配置。**先试别的、后试上游**：一个新上游用着另一边新加的出站代理，出站代理 + /// 先换过去，上游才过得了 #[allow(clippy::too_many_arguments)] fn check_target( &self, local_pack: &Pack, - local: &Snapshot, + start: &Snapshot, + mine: &Snapshot, target: &mut Snapshot, shared: &mut Snapshot, remote: &Remote, @@ -1483,16 +1989,33 @@ impl Engine { if valid(target) { return Ok(()); } - let changed: Vec = local + let mut changed: Vec = start .items .keys() .chain(target.items.keys()) - .filter(|id| !id.kind.is_order() && local.items.get(*id) != target.items.get(*id)) + .filter(|id| !id.kind.is_order() && start.items.get(*id) != target.items.get(*id)) .cloned() .collect::>() .into_iter() .collect(); - let mut t = local.clone(); + // 新增和修改按引用的先后（被引用的先到位），删除倒过来（引用它的先走）。一遍下来还有 + // 过不了的，再来一遍：前面换过去的可能正是它缺的 + let rank = |id: &ItemId| -> u8 { + let r = match id.kind { + ItemKind::Provider => 1, + ItemKind::Alias | ItemKind::Group => 2, + ItemKind::Route => 3, + ItemKind::Key | ItemKind::Global => 4, + _ => 0, + }; + if target.items.contains_key(id) { + r + } else { + 10 - r + } + }; + changed.sort_by_key(|id| rank(id)); + let mut t = start.clone(); let fix_orders = |s: &mut Snapshot, from: &Snapshot| { for (o, k) in ORDERS { let names = names_of(s, *k); @@ -1510,23 +2033,35 @@ impl Engine { } s.relink(); }; - for id in changed { - let mut next = t.clone(); - match target.items.get(&id) { - Some(it) => next.items.insert(id.clone(), it.clone()), - None => next.items.remove(&id), - }; - fix_orders(&mut next, target); - if valid(&next) { - t = next; - continue; + let mut left = changed; + loop { + let before = left.len(); + let mut still = Vec::new(); + for id in left { + let mut next = t.clone(); + match target.items.get(&id) { + Some(it) => next.items.insert(id.clone(), it.clone()), + None => next.items.remove(&id), + }; + fix_orders(&mut next, target); + if valid(&next) { + t = next; + } else { + still.push(id); + } } + left = still; + if left.is_empty() || left.len() == before { + break; + } + } + for id in left { // 这一项留本机的 - match local.items.get(&id) { + match mine.items.get(&id) { Some(it) => shared.items.insert(id.clone(), it.clone()), None => shared.items.remove(&id), }; - let units: Vec = local + let units: Vec = mine .items .get(&id) .or(target.items.get(&id)) @@ -1538,7 +2073,7 @@ impl Engine { item: id.clone(), units, shared: side_remote(remote.snap.items.get(&id).map(|i| i.units.clone())), - other: side_me(local.items.get(&id).map(|i| i.units.clone())), + other: side_me(mine.items.get(&id).map(|i| i.units.clone())), reason: Some(Reason::Invalid), }); } @@ -1590,13 +2125,20 @@ impl Engine { } _ => None, }; - let encrypted = keyring.is_some(); + let encrypted = match (&keyring, &opened) { + (None, _) => false, + (Some(_), Some(k)) => !k.off(), + (Some(b), None) => !keyring::marked_off(b), + }; let unlocked = !encrypted || opened.is_some(); let mut devices = Vec::new(); if unlocked { for e in store.list(DEVICES).await.unwrap_or_default() { + if !layout::is_device_file(&e.name) { + continue; + } if let Ok(Some(b)) = store.get(&format!("{DEVICES}/{}", e.name)).await - && let Some(rec) = DeviceRecord::open(&b, opened.as_ref()) + && let Some(rec) = DeviceRecord::open(&b, opened.as_ref(), encrypted) { devices.push(rec); } @@ -1613,9 +2155,24 @@ impl Engine { } } - /// 同步位置空着时开启：加密的先建 keyring（**仅当不存在时**：两台设备同时开启时后写的 - /// 那一台拿到 `Ok(false)`，转去加入），然后以本机配置上传第一个版本 + /// 同步位置空着时开启：加密的先建 keyring（**仅当不存在时**），然后以本机配置上传第一个 + /// 版本。已经有了版本（两台设备同时开启、后写的那一台）是 `Ok(false)`，转去加入。 + /// + /// 只有 keyring、没有版本（之前开启加密时第一个版本没传上去）时，口令解得开就接着用它 pub async fn create(&mut self) -> Result { + self.local.clear(); + let mut st = State { + place: self.settings.place.clone(), + ..Default::default() + }; + let r = self.create_inner(&mut st).await; + if matches!(r, Ok(true)) { + self.commit(&st)?; + } + r + } + + async fn create_inner(&mut self, st: &mut State) -> Result { if self.settings.encrypt { let pass = self .settings @@ -1625,40 +2182,47 @@ impl Engine { .ok_or_else(|| { RoundError::Local(tw_config::sync::SyncError::PassphraseMissing.msg()) })?; - let k = Keyring::generate(); - let sealed = tokio::task::spawn_blocking({ + let (sealed, k) = tokio::task::spawn_blocking({ let pass = pass.clone(); - move || (k.seal(&pass), k) + move || { + let k = Keyring::generate(0); + (k.seal(&pass), k) + } }) .await .map_err(local_err)?; - let (sealed, k) = sealed; - if !self.store.create_exclusive(KEYRING, &sealed).await? { - return Ok(false); + if self.store.create_exclusive(KEYRING, &sealed).await? { + st.keyring_gen = k.generation(); + let hash: [u8; 32] = Sha256::digest(&sealed).into(); + self.set_mode(Some(hash), Some(k), Some(sealed)); + } else { + // 已经有一份：别的设备的（转去加入），或者上次开启到一半留下的 + self.security(st).await?; + if !self.sealed { + return Ok(false); + } } - let hash: [u8; 32] = Sha256::digest(&sealed).into(); - self.keyring = Some((k, hash)); } else if self.store.get(KEYRING).await?.is_some() { return Ok(false); + } else { + self.set_mode(None, None, None); } - if !self.store.list(VERSIONS).await?.is_empty() { + if !self.listing().await?.is_empty() { return Ok(false); } - self.local.clear(); - let st = State { - place: self.settings.place.clone(), - ..Default::default() - }; - self.local.save_state(&st).map_err(RoundError::Local)?; - self.publish_first().await?; + self.publish_first(1, st).await?; self.quiet = None; Ok(true) } - /// 同步位置的最新（几个头合并之后），写成一个配置包：加入时给导入预览比对。不含本机设置 + /// 同步位置的最新（几个头合并之后），写成一个配置包：加入时给导入预览比对。不含本机设置。 + /// 同步位置上还没有版本（只有 keyring）时是一份空的配置 pub async fn latest(&mut self) -> Result { - let remote = self.latest_remote().await?.0; - self.pack_of(&remote.snap) + let mut st = self.local.state(); + match self.latest_remote(&mut st).await? { + Some((remote, _)) => self.pack_of(&remote.snap), + None => self.pack_of(&Snapshot::default()), + } } fn pack_of(&self, snap: &Snapshot) -> Result { @@ -1671,42 +2235,64 @@ impl Engine { model::materialize(&empty, snap).map_err(|e| RoundError::Local(e.msg())) } + /// 同步位置的最新。没有可用的版本是 `None` async fn latest_remote( &mut self, - ) -> Result<(Remote, Vec, Vec>), RoundError> { - self.keyring().await?; - let (names, ok, too_new) = self.versions().await?; - let referenced: BTreeSet = ok - .iter() - .flat_map(|v| v.meta.parents.iter().cloned()) - .collect(); - if let Some((_, cv)) = too_new.iter().find(|(n, _)| !referenced.contains(&n.id())) { - return Err(RoundError::TooNew { - core_version: cv.clone(), - }); - } - let heads: Vec> = ok - .iter() - .filter(|v| !referenced.contains(&v.name.id())) - .cloned() - .collect(); - let top_ok = ok.iter().map(|v| v.name.seq).max(); - if heads.is_empty() || top_ok < names.iter().map(|n| n.seq).max() { - return Err(RoundError::NotArrived); + st: &mut State, + ) -> Result, RoundError> { + self.security(st).await?; + let listed = self.listing().await?; + let sc = self.scan(listed, st).await?; + if sc.names.is_empty() { + return Ok(None); } + let heads = Self::heads(&sc)?; let base = self.base(); - Ok((self.fold(&heads, &ok, base.as_ref()), names, ok)) + let remote = self.fold(&heads, &sc.ok, base.as_ref()); + Ok(Some((remote, sc))) } /// 加入已有的同步。`merged` 是本机按导入的规则合并之后的完整配置包（「合并」）；`None` 是 /// 用同步数据替换本机配置（新安装、或者选了「使用同步数据」）—— 那时本机已接管的客户端 /// 正在用的网关密钥并入同步数据，不同步的上游留在本机。 /// - /// 本机先照结果改，再上传一个依据同步位置最新版本的新版本，作为这台设备的基准 + /// 本机先照结果改，再上传一个依据同步位置最新版本的新版本，作为这台设备的基准。两种方式 + /// 下，别的设备上登录着、这台没登录的上游都在这台停用着，**上传的是同步的样子**（启用着）。 + /// 同步位置上只有 keyring、还没有版本时，以本机配置上传第一个版本 pub async fn join(&mut self, merged: Option) -> Result { - let (remote, names, _) = self.latest_remote().await?; + self.local.clear(); + let mut st = State { + place: self.settings.place.clone(), + ..Default::default() + }; + let r = self.join_inner(merged, &mut st).await; + if r.is_ok() { + self.commit(&st)?; + } + r + } + + async fn join_inner( + &mut self, + merged: Option, + st: &mut State, + ) -> Result { + let Some((remote, sc)) = self.latest_remote(st).await? else { + let all = self.listing().await?; + let seq = self.next_seq(&[], &all); + self.publish_first(seq, st).await?; + let report = Report { + encrypted: self.sealed, + devices: self.devices().await?, + ..Default::default() + }; + self.quiet = None; + self.last_report = report.clone(); + return Ok(report); + }; let cur = self.host.current().map_err(RoundError::Local)?; let local = model::snapshot(&cur.pack).map_err(|e| RoundError::Local(e.msg()))?; + let here = local_providers(&cur.pack); let ex = excluded(&self.settings.exclude, &[&local, &remote.snap]); let mut target = match &merged { Some(p) => model::snapshot(p).map_err(|e| RoundError::Local(e.msg()))?, @@ -1734,6 +2320,17 @@ impl Engine { None => shared.items.remove(id), }; } + // 没登录的上游：导入的规矩在本机停用了它，上传的照同步数据里的样子 + let waiting: Vec = shared + .of_kind(ItemKind::Provider) + .filter(|(id, it)| !ex.contains(*id) && needs_login(id, it, &here)) + .map(|(id, _)| id.clone()) + .collect(); + for id in &waiting { + if let (Some(it), Some(r)) = (shared.items.get_mut(id), remote.snap.items.get(id)) { + copy_disabled(it, r); + } + } for (o, k) in ORDERS { let m = order::merge( order_list(Some(&remote.snap), *o).as_deref(), @@ -1750,9 +2347,15 @@ impl Engine { } target.relink(); shared.relink(); + // 本机这一份：没登录的停用着；同步的样子是启用着的，记成等登录 + hold_logins(&mut target, &here); + st.held_login = waiting + .iter() + .filter(|id| shared.items.get(*id).is_some_and(|it| !disabled(it))) + .map(|id| id.name.clone()) + .collect(); // 不加密：经同步新来、或者脚本变了的插件在本机停用着，要在本机启用（同步一轮里是同一条） - let mut held = BTreeMap::new(); - if self.keyring.is_none() { + if !self.sealed { let ids: Vec = target .of_kind(ItemKind::Plugin) .map(|(id, _)| id.clone()) @@ -1774,12 +2377,14 @@ impl Engine { t.units .insert(unit::ENABLED.to_string(), Value::Bool(false)); } - held.insert(id.name.clone(), script.unwrap_or(Value::Null)); + st.held_plugins + .insert(id.name.clone(), script.unwrap_or(Value::Null)); } } } let mut report = Report { - encrypted: self.keyring.is_some(), + encrypted: self.sealed, + skipped: sc.skipped.len() as u32, ..Default::default() }; if model::normalized(&target) != model::normalized(&local) { @@ -1809,22 +2414,16 @@ impl Engine { } report.applied_from = vec![remote.device_name.clone()]; } - let seq = names.iter().map(|n| n.seq).max().unwrap_or(0) + 1; + let seq = self.next_seq(&sc.names, &sc.all); let v = self .publish(&shared, seq, &remote.heads, remote.conflicts.clone()) .await?; report.uploaded = Some(v.name.id()); - let mut st = State { - place: self.settings.place.clone(), - held_plugins: held, - ..Default::default() - }; - self.save_base(&v, &mut st)?; + self.save_base(&v, st)?; st.last_synced_ms = Some(now_ms()); st.failing_since_ms = None; st.error = None; - self.local.save_state(&st).map_err(RoundError::Local)?; - self.write_device(false).await?; + self.write_device(st, false).await?; report.devices = self.devices().await?; report.conflicts = remote.conflicts; self.quiet = None; @@ -1834,93 +2433,176 @@ impl Engine { // ─────────────────────────────────────────────── 加密 - /// 之后开启加密：建 keyring，以它重新上传最新版本，删掉未加密的版本。其他设备下次同步时 - /// 要输入口令 + /// 之后开启加密：建 keyring（同步位置上有一份已关闭的就换掉它），以它重新上传最新版本， + /// 删掉未加密的版本和设备记录。其他设备下次同步时要输入口令 pub async fn enable_encryption(&mut self, passphrase: &str) -> Result<(), RoundError> { - let (remote, names, _) = self.latest_remote().await?; - if self.keyring.is_some() { - return Ok(()); + let mut st = self.local.state(); + let r = self.enable_inner(passphrase, &mut st).await; + self.commit(&st)?; + r + } + + async fn enable_inner(&mut self, passphrase: &str, st: &mut State) -> Result<(), RoundError> { + let existing = self.store.get(KEYRING).await?; + let mut after = st.keyring_gen; + if let Some(b) = &existing { + // 给的口令、记着的口令，哪个解得开（第一个是给的那个） + let tries: Vec = [ + Some(passphrase.to_string()), + self.settings.passphrase.clone(), + ] + .into_iter() + .flatten() + .collect(); + let b2 = b.clone(); + let opened = tokio::task::spawn_blocking(move || { + tries + .iter() + .enumerate() + .find_map(|(i, p)| Keyring::open(&b2, p).ok().map(|k| (i == 0, k))) + }) + .await + .map_err(local_err)?; + match opened { + // 已经开着：同一个口令就当作开好了 + Some((true, k)) if !k.off() => { + self.settings.encrypt = true; + self.settings.passphrase = Some(passphrase.to_string()); + self.mode_of = None; + return Ok(()); + } + Some((_, k)) if !k.off() => return Err(locked_new()), + Some((_, k)) => after = after.max(k.generation()), + None if keyring::marked_off(b) => {} + None => return Err(locked_new()), + } } - let k = Keyring::generate(); + let remote = self.latest_remote(st).await?; + let all = self.listing().await?; + let k = Keyring::generate(after); let sealed = k.seal(passphrase); - if !self.store.create_exclusive(KEYRING, &sealed).await? { - return Err(RoundError::Locked(msg!( - "sync.locked_new" => - "the sync location is now encrypted. Enter the sync passphrase to keep syncing" - ))); + if existing.is_some() { + self.store.put(KEYRING, &sealed).await?; + } else if !self.store.create_exclusive(KEYRING, &sealed).await? { + return Err(locked_new()); } + st.keyring_gen = k.generation(); let hash: [u8; 32] = Sha256::digest(&sealed).into(); - self.keyring = Some((k, hash)); self.settings.encrypt = true; self.settings.passphrase = Some(passphrase.to_string()); - self.reupload(remote, &names).await + self.set_mode(Some(hash), Some(k), Some(sealed)); + self.reupload(remote, &all, st).await?; + // 不加密的设备记录：加密的位置上不认它们，各设备下次同步时加密着重写 + for e in self.store.list(DEVICES).await? { + if !layout::is_device_file(&e.name) { + continue; + } + let path = format!("{DEVICES}/{}", e.name); + if let Some(b) = self.store.get(&path).await? + && !keyring::is_age(&b) + { + let _ = self.store.delete(&path).await; + } + } + Ok(()) } - /// 关闭加密：口令要对。重新上传不加密的最新版本，删掉 keyring 和加密过的版本 + /// 关闭加密：口令要对。keyring 改写成「已关闭」（**不删**：别的设备靠它认出这是有口令的人 + /// 关的），重新上传不加密的最新版本，删掉加密过的版本 pub async fn disable_encryption(&mut self, passphrase: &str) -> Result<(), RoundError> { + let mut st = self.local.state(); + let r = self.disable_inner(passphrase, &mut st).await; + self.commit(&st)?; + r + } + + async fn disable_inner(&mut self, passphrase: &str, st: &mut State) -> Result<(), RoundError> { let Some(bytes) = self.store.get(KEYRING).await? else { - self.keyring = None; - return Ok(()); + return Err(keyring_missing()); }; - let hash: [u8; 32] = Sha256::digest(&bytes).into(); let pass = passphrase.to_string(); let opened = tokio::task::spawn_blocking(move || Keyring::open(&bytes, &pass)) .await .map_err(local_err)? .map_err(|e| RoundError::Locked(e.msg()))?; + if opened.off() { + self.settings.encrypt = false; + self.mode_of = None; + return Ok(()); + } self.settings.passphrase = Some(passphrase.to_string()); - self.keyring = Some((opened, hash)); - let (remote, names, _) = self.latest_remote().await?; - // 之后写的一律不加密 - self.keyring = None; + self.settings.encrypt = true; + self.mode_of = None; + let remote = self.latest_remote(st).await?; + let all = self.listing().await?; + let off = opened.next(true); + let sealed = off.seal(passphrase); + self.store.put(KEYRING, &sealed).await?; + st.keyring_gen = st.keyring_gen.max(off.generation()); + let hash: [u8; 32] = Sha256::digest(&sealed).into(); self.settings.encrypt = false; - self.reupload(remote, &names).await?; - self.store.delete(KEYRING).await?; - Ok(()) + self.set_mode(Some(hash), Some(off), Some(sealed)); + self.reupload(remote, &all, st).await } /// 以现在的加密设置重新上传最新版本，删掉别的版本（它们是另一种加密设置下写的） - async fn reupload(&mut self, remote: Remote, names: &[VersionName]) -> Result<(), RoundError> { - let seq = names.iter().map(|n| n.seq).max().unwrap_or(0) + 1; - let v = self - .publish(&remote.snap, seq, &remote.heads, remote.conflicts.clone()) - .await?; + async fn reupload( + &mut self, + remote: Option<(Remote, Scan)>, + names: &[VersionName], + st: &mut State, + ) -> Result<(), RoundError> { + let v = match remote { + Some((remote, sc)) => { + let seq = self.next_seq(&sc.names, names); + self.publish(&remote.snap, seq, &remote.heads, remote.conflicts.clone()) + .await? + } + None => { + let seq = self.next_seq(&[], names); + return self.publish_first(seq, st).await.map(|_| ()); + } + }; for n in names { let _ = self.store.delete(&n.path()).await; self.cache.remove(n); } - let mut st = self.local.state(); - self.save_base(&v, &mut st)?; + self.save_base(&v, st)?; st.last_synced_ms = Some(now_ms()); - self.local.save_state(&st).map_err(RoundError::Local)?; - self.write_device(false).await?; + self.write_device(st, false).await?; self.quiet = None; Ok(()) } /// 更改同步口令：只重新加密 keyring，已有的版本不变。其他设备下次同步时要输入新口令 pub async fn change_passphrase(&mut self, old: &str, new: &str) -> Result<(), RoundError> { + let mut st = self.local.state(); let Some(bytes) = self.store.get(KEYRING).await? else { return Err(RoundError::Local(msg!( "sync.not_encrypted" => "the sync location is not encrypted" ))); }; let (old, new2) = (old.to_string(), new.to_string()); - let sealed = tokio::task::spawn_blocking(move || { - Keyring::open(&bytes, &old).map(|k| (k.seal(&new2), k)) + let (sealed, k) = tokio::task::spawn_blocking(move || { + Keyring::open(&bytes, &old).map(|k| { + let off = k.off(); + let k = k.next(off); + (k.seal(&new2), k) + }) }) .await .map_err(local_err)? .map_err(|e| RoundError::Locked(e.msg()))?; - let (sealed, k) = sealed; self.store.put(KEYRING, &sealed).await?; + st.keyring_gen = st.keyring_gen.max(k.generation()); let hash: [u8; 32] = Sha256::digest(&sealed).into(); - self.keyring = Some((k, hash)); self.settings.passphrase = Some(new.to_string()); - Ok(()) + self.set_mode(Some(hash), Some(k), Some(sealed)); + self.commit(&st) } - /// 口令对不对得上同步位置的 keyring（别的设备改了口令之后重新输入）。没加密是 `Ok(false)` + /// 口令对不对得上同步位置的 keyring（别的设备改了口令、开了加密之后重新输入）。加密开着是 + /// `Ok(true)`；没有 keyring、或者是已关闭的是 `Ok(false)` pub async fn unlock(&mut self, passphrase: &str) -> Result { let Some(bytes) = self.store.get(KEYRING).await? else { return Ok(false); @@ -1930,21 +2612,109 @@ impl Engine { .await .map_err(local_err)?; match k { - Ok(_) => { + Ok(k) => { self.settings.passphrase = Some(passphrase.to_string()); - self.settings.encrypt = true; + self.settings.encrypt = !k.off(); + self.mode_of = None; self.keyring = None; self.quiet = None; - Ok(true) + Ok(!k.off()) } Err(e) => Err(RoundError::Locked(e.msg())), } } + // ─────────────────────────────────────────────── 要人处理的状态 + + /// 同步位置清空了（[`RoundError::Emptied`]）、keyring 不见了或者被换成旧的 + /// （[`RoundError::KeyringMissing`]）之后，由人选的「重新上传本机配置」： + /// + /// - 加密着时 keyring 放回去：这台设备记着的那一份（口令解得开的话），没有就新建一把 —— + /// 之前的版本这时谁也解不开了，各设备跳过它们； + /// - 同步位置上没有读得出来的版本时，以本机配置上传一个新版本； + /// - 然后照常同步一轮 + pub async fn republish(&mut self) -> Result { + let mut st = self.local.state(); + let r = self.republish_inner(&mut st).await; + self.commit(&st)?; + r?; + self.quiet = None; + self.round().await + } + + async fn republish_inner(&mut self, st: &mut State) -> Result<(), RoundError> { + let pass = self.settings.passphrase.clone().filter(|p| !p.is_empty()); + let open = |b: Vec| { + let pass = pass.clone(); + async move { + let Some(p) = pass else { + return Err(KeyringError::WrongPassphrase); + }; + tokio::task::spawn_blocking(move || Keyring::open(&b, &p)) + .await + .unwrap_or(Err(KeyringError::Corrupt)) + } + }; + let seen = st.keyring_gen; + let usable = |k: &Keyring| !(k.off() && k.generation() < seen); + let here = match self.store.get(KEYRING).await? { + Some(b) => Some(open(b).await), + None => None, + }; + let fine = matches!(&here, Some(Ok(k)) if usable(k)); + if !fine && (self.settings.encrypt || st.keyring_gen > 0) { + match &here { + // 口令不对的那一份不是丢了：别的设备改了口令，要输入新口令 + Some(Err(KeyringError::WrongPassphrase)) => { + return Err(RoundError::Locked(KeyringError::WrongPassphrase.msg())); + } + Some(Err(KeyringError::TooNew)) => { + return Err(RoundError::TooNew { + core_version: String::new(), + }); + } + _ => {} + } + let copy = match self.local.keyring() { + Some(b) => match open(b.clone()).await { + Ok(k) if usable(&k) => Some(b), + _ => None, + }, + None => None, + }; + match (copy, &pass) { + (Some(b), _) => self.store.put(KEYRING, &b).await?, + (None, Some(p)) if self.settings.encrypt => { + let p = p.clone(); + let k = Keyring::generate(seen); + st.keyring_gen = k.generation(); + let sealed = tokio::task::spawn_blocking(move || k.seal(&p)) + .await + .map_err(local_err)?; + self.store.put(KEYRING, &sealed).await?; + } + // 不加密、也没有记着的 keyring：当作从没加密过 + _ => st.keyring_gen = 0, + } + } + self.mode_of = None; + self.security(st).await?; + st.empty_rounds = 0; + st.empty_since_ms = None; + let all = self.listing().await?; + let sc = self.scan(all.clone(), st).await?; + if sc.ok.is_empty() { + let seq = self.next_seq(&[], &all); + self.publish_first(seq, st).await?; + } + Ok(()) + } + // ─────────────────────────────────────────────── 停止 /// 停止同步：从同步位置移除本设备的设备记录；`purge` 时连全部版本、设备记录和 keyring 一起 - /// 删掉（其他设备随后发现同步位置空了，停止同步）。本机的同步状态目录删掉 + /// 删掉（其他设备随后发现同步位置空了，由人决定怎么办）。**只删认得出的同步文件**：用户 + /// 自己放进 `versions/`、`devices/` 的不动。本机的同步状态目录删掉 pub async fn stop(&mut self, purge: bool) -> Result<(), RoundError> { let mine = self .store @@ -1952,10 +2722,14 @@ impl Engine { .await; if purge { for e in self.store.list(VERSIONS).await? { - self.store.delete(&format!("{VERSIONS}/{}", e.name)).await?; + if VersionName::parse(&e.name).is_some() { + self.store.delete(&format!("{VERSIONS}/{}", e.name)).await?; + } } for e in self.store.list(DEVICES).await? { - self.store.delete(&format!("{DEVICES}/{}", e.name)).await?; + if layout::is_device_file(&e.name) { + self.store.delete(&format!("{DEVICES}/{}", e.name)).await?; + } } self.store.delete(KEYRING).await?; } else { @@ -1964,6 +2738,7 @@ impl Engine { self.local.clear(); self.cache.clear(); self.quiet = None; + self.mode_of = None; Ok(()) } @@ -1971,9 +2746,10 @@ impl Engine { /// 同步位置上读得出来的版本（最新与上一个）：给备份列表 pub async fn remote_versions(&mut self) -> Result>, RoundError> { - self.keyring().await?; - let (_, ok, _) = self.versions().await?; - Ok(ok) + let mut st = self.local.state(); + self.security(&mut st).await?; + let listed = self.listing().await?; + Ok(self.scan(listed, &mut st).await?.ok) } /// 同步位置上内容标识是这个的版本 @@ -1986,8 +2762,8 @@ impl Engine { .map(|v| v.pack.clone())) } - /// 同步位置空着：以本机配置上传第一个版本（加密时先建 keyring） - async fn publish_first(&mut self) -> Result { + /// 以本机配置上传一个不依据别的版本的版本（同步位置空着时） + async fn publish_first(&mut self, seq: u64, st: &mut State) -> Result { let cur = self.host.current().map_err(RoundError::Local)?; let snap = model::snapshot(&cur.pack).map_err(|e| RoundError::Local(e.msg()))?; let mut shared = snap.clone(); @@ -2003,13 +2779,13 @@ impl Engine { set_order(&mut shared, *o, list); } shared.relink(); - let v = self.publish(&shared, 1, &[], Vec::new()).await?; - let mut st = self.local.state(); + let v = self.publish(&shared, seq, &[], Vec::new()).await?; st.place.clone_from(&self.settings.place); - self.save_base(&v, &mut st)?; + self.save_base(&v, st)?; st.last_synced_ms = Some(now_ms()); - self.local.save_state(&st).map_err(RoundError::Local)?; - self.write_device(false).await?; + st.failing_since_ms = None; + st.error = None; + self.write_device(st, false).await?; Ok(v) } } diff --git a/crates/tw-sync/src/keyring.rs b/crates/tw-sync/src/keyring.rs index b17f5d78..aa0eb0b5 100644 --- a/crates/tw-sync/src/keyring.rs +++ b/crates/tw-sync/src/keyring.rs @@ -7,6 +7,16 @@ //! - 没有口令的一方解不开 keyring，也就既读不了版本、也写不出别的设备认的版本（age 的认证 //! 加密，见 `tw_pack`）。 //! +//! **关闭加密也写在 keyring 里**（[`Keyring::off`]）：keyring 以口令加密、带认证，没有口令的人 +//! 造不出一个「已关闭」的 keyring。于是同步位置上的 keyring 被删掉、被换成别的，设备都不会把 +//! 这当成「关掉了加密」：只有用口令解得开、写着已关闭的 keyring 才算。 +//! +//! 每写一次 keyring 带一个更大的**代号**（[`Keyring::generation`]，取写的时刻，至少比上一个 +//! 大 1）。设备记着见过的最大代号：把一个旧的「已关闭」keyring 放回来（它当初是真的）也不算。 +//! +//! 已关闭的 keyring 文件开头另有一行明文的标记（[`OFF_MARK`]），给**没有口令**的设备看：它们 +//! 本来就不加密，看到标记就照常同步；有口令的设备不看标记，只认解开之后写着的。 +//! //! scrypt 的强度是**写死的**（[`WORK_FACTOR`]），不按这台机器的速度现定：一台快的 Mac 按自己 //! 定的强度加密，慢的服务器可能解不开（age 拒绝超过本机上限的强度）。 @@ -25,6 +35,9 @@ const MAX_WORK_FACTOR: u8 = WORK_FACTOR + 1; static WORK: AtomicU8 = AtomicU8::new(WORK_FACTOR); +/// 已关闭的 keyring 文件的第一行（明文，见模块说明） +pub const OFF_MARK: &[u8] = b"thinkwatch-sync: encryption off\n"; + /// **只给测试用**：把 scrypt 的强度调低，一条测试不必每次都算零点几秒。解的时候照样认 pub fn set_work_factor_for_tests(log_n: u8) { WORK.store(log_n.clamp(1, WORK_FACTOR), Ordering::Relaxed); @@ -37,17 +50,27 @@ struct Plain { v: u32, /// `AGE-SECRET-KEY-1…` identity: String, + /// 加密已关闭：版本和设备记录不再加密。keyring 留着，好让各设备认得出这是有口令的人关的 + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + off: bool, + /// 代号：每写一次就更大 + #[serde(default)] + generation: u64, } /// 一把解开了的版本密钥。**Debug 不打印它** pub struct Keyring { identity: age::x25519::Identity, + off: bool, + generation: u64, } impl std::fmt::Debug for Keyring { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Keyring") .field("recipient", &self.identity.to_public().to_string()) + .field("off", &self.off) + .field("generation", &self.generation) .finish() } } @@ -85,10 +108,31 @@ impl KeyringError { } impl Keyring { - /// 新的一把 - pub fn generate() -> Self { + /// 新的一把。`after` 是见过的最大代号：新的这把比它大 + pub fn generate(after: u64) -> Self { Self { identity: age::x25519::Identity::generate(), + off: false, + generation: next_generation(after), + } + } + + /// 加密已经关了（有口令的人关的） + pub fn off(&self) -> bool { + self.off + } + + pub fn generation(&self) -> u64 { + self.generation + } + + /// 同一把密钥再写一次（关加密、开回来、换口令）：开着还是关着，代号更大 + pub fn next(self, off: bool) -> Self { + let generation = next_generation(self.generation); + Self { + off, + generation, + ..self } } @@ -97,15 +141,28 @@ impl Keyring { let plain = Plain { v: 1, identity: self.identity.to_string().expose_secret().to_string(), + off: self.off, + generation: self.generation, }; let json = serde_json::to_vec(&plain).expect("plain data serializes"); let mut r = age::scrypt::Recipient::new(SecretString::from(passphrase.to_string())); r.set_work_factor(WORK.load(Ordering::Relaxed)); - age::encrypt(&r, &json).expect("encrypting to a passphrase does not fail") + let sealed = age::encrypt(&r, &json).expect("encrypting to a passphrase does not fail"); + if self.off { + [OFF_MARK, &sealed].concat() + } else { + sealed + } } - /// 用口令解开 `keyring` 文件 + /// 用口令解开 `keyring` 文件。开头的标记和里面写的对不上（有人加上或去掉了标记）算坏了 pub fn open(bytes: &[u8], passphrase: &str) -> Result { + let marked = marked_off(bytes); + let bytes = if marked { + &bytes[OFF_MARK.len()..] + } else { + bytes + }; let mut id = age::scrypt::Identity::new(SecretString::from(passphrase.to_string())); id.set_max_work_factor(MAX_WORK_FACTOR); let json = age::decrypt(&id, bytes).map_err(|e| match e { @@ -119,11 +176,18 @@ impl Keyring { if plain.v > 1 { return Err(KeyringError::TooNew); } + if plain.off != marked { + return Err(KeyringError::Corrupt); + } let identity = plain .identity .parse::() .map_err(|_| KeyringError::Corrupt)?; - Ok(Self { identity }) + Ok(Self { + identity, + off: plain.off, + generation: plain.generation, + }) } /// 加密用的公钥：版本、设备记录都加密给它 @@ -157,6 +221,20 @@ pub fn is_age(bytes: &[u8]) -> bool { bytes.starts_with(b"age-encryption.org/") } +/// `keyring` 文件开头有「已关闭」的标记。**没有认证**：只给没有口令的设备用（见模块说明） +pub fn marked_off(bytes: &[u8]) -> bool { + bytes.starts_with(OFF_MARK) +} + +/// 下一个代号：此刻的毫秒数，至少比 `after` 大 1（时钟慢的设备照样往上走） +fn next_generation(after: u64) -> u64 { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0); + now.max(after.saturating_add(1)) +} + #[cfg(test)] mod tests { use super::*; @@ -164,7 +242,7 @@ mod tests { #[test] fn a_keyring_opens_with_its_passphrase_and_only_with_it() { set_work_factor_for_tests(4); - let k = Keyring::generate(); + let k = Keyring::generate(0); let sealed = k.seal("correct horse battery staple"); assert!(is_age(&sealed)); let back = Keyring::open(&sealed, "correct horse battery staple").unwrap(); @@ -182,7 +260,7 @@ mod tests { #[test] fn changing_the_passphrase_keeps_the_same_key() { set_work_factor_for_tests(4); - let k = Keyring::generate(); + let k = Keyring::generate(0); let sealed = k.seal("old passphrase 1"); let opened = Keyring::open(&sealed, "old passphrase 1").unwrap(); let resealed = opened.seal("new passphrase 2"); @@ -190,7 +268,41 @@ mod tests { assert!(again.same_as(&k), "改口令只重新加密 keyring，密钥本身不变"); let data = k.encrypt(b"device record"); assert_eq!(again.decrypt(&data).unwrap(), b"device record"); - assert!(Keyring::generate().decrypt(&data).is_none()); + assert!(Keyring::generate(0).decrypt(&data).is_none()); + } + + #[test] + fn turning_encryption_off_is_written_in_the_keyring_itself() { + set_work_factor_for_tests(4); + let k = Keyring::generate(0); + let first = k.generation(); + assert!(!k.off()); + let sealed = k.next(true).seal("pass phrase 12"); + assert!(marked_off(&sealed), "没有口令的设备看得到标记"); + let back = Keyring::open(&sealed, "pass phrase 12").unwrap(); + assert!(back.off(), "关了就是关了"); + assert!(back.generation() > first, "每写一次代号更大"); + // 没有口令的人改不了它：口令不对就解不开，也就看不到「关了」 + assert!(Keyring::open(&sealed, "someone else!!").is_err()); + // 标记被去掉、或者加到一个开着的 keyring 上：和里面写的对不上，算坏了 + assert!(matches!( + Keyring::open(&sealed[OFF_MARK.len()..], "pass phrase 12"), + Err(KeyringError::Corrupt) + )); + let on_sealed = back.next(false).seal("pass phrase 12"); + assert!(!marked_off(&on_sealed)); + assert!(matches!( + Keyring::open(&[OFF_MARK, &on_sealed].concat(), "pass phrase 12"), + Err(KeyringError::Corrupt) + )); + let on = Keyring::open(&on_sealed, "pass phrase 12").unwrap(); + assert!(!on.off()); + } + + #[test] + fn a_new_keyring_always_comes_after_the_ones_seen() { + let far = u64::MAX / 2; + assert!(Keyring::generate(far).generation() > far, "时钟慢也往上走"); } #[test] @@ -198,7 +310,7 @@ mod tests { set_work_factor_for_tests(4); // 按强度 4 加密，再把文件头里写的强度改成 30：age 在算 scrypt 之前就按上限拒绝， // 测试不用真的去算一遍 2^30 - let sealed = Keyring::generate().seal("p"); + let sealed = Keyring::generate(0).seal("p"); let text = String::from_utf8_lossy(&sealed).to_string(); let line = text .lines() @@ -217,7 +329,7 @@ mod tests { #[test] fn debug_shows_only_the_public_half() { - let k = Keyring::generate(); + let k = Keyring::generate(0); let shown = format!("{k:?}"); assert!(shown.contains("age1"), "{shown}"); assert!(!shown.contains("AGE-SECRET-KEY"), "{shown}"); diff --git a/crates/tw-sync/src/layout.rs b/crates/tw-sync/src/layout.rs index 6f01ac26..e4a7a2d7 100644 --- a/crates/tw-sync/src/layout.rs +++ b/crates/tw-sync/src/layout.rs @@ -94,11 +94,21 @@ pub fn device_path(device_id: &str) -> String { format!("{DEVICES}/{}", device_tag(device_id)) } +/// `devices/` 里的这个名字是不是一份设备记录（设备标识的前 8 位）。别的是用户自己放的 +pub fn is_device_file(name: &str) -> bool { + name.len() == tw_config::sync::DEVICE_ID_PREFIX + && name + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + /// 同步位置上的一个版本：名字，以及它说自己依据的上一版本（读出来之后才知道） #[derive(Debug, Clone, PartialEq, Eq)] pub struct Listed { pub name: VersionName, pub parents: Vec, + /// 更早的祖先（版本自己记着的） + pub history: Vec, } /// 上传新版本之后要删掉哪些。 @@ -123,6 +133,27 @@ pub fn to_delete(listed: &[Listed]) -> Vec { for l in &heads { keep.extend(l.parents.iter().cloned()); } + // 不止一个头：它们共同的祖先也留着，连同从它到每个头之间的版本 —— 下一台设备合并它们要 + // 用它当基准，删了就只能两方比较 + if heads.len() > 1 { + let lineage = |l: &Listed| -> BTreeSet { + l.history + .iter() + .chain(&l.parents) + .cloned() + .chain([l.name.clone()]) + .collect() + }; + let sets: Vec> = heads.iter().map(|l| lineage(l)).collect(); + let common = sets.iter().skip(1).fold(sets[0].clone(), |acc, s| { + acc.intersection(s).cloned().collect() + }); + if let Some(ca) = common.iter().max() { + for s in &sets { + keep.extend(s.iter().filter(|n| n.seq >= ca.seq).cloned()); + } + } + } // 不知道上一版本是谁（读不出来）时，至少留住序号紧挨着的那一层 if listed .iter() @@ -158,6 +189,7 @@ mod tests { Listed { name: v(seq, dev), parents: parents.iter().map(|(s, d)| v(*s, d)).collect(), + history: Vec::new(), } } @@ -231,6 +263,23 @@ mod tests { assert!(to_delete(&listed).is_empty(), "{:?}", to_delete(&listed)); } + /// 两个头的共同祖先比它们的上一版本还早：它和中间的版本都留着 + #[test] + fn the_common_ancestor_of_two_heads_is_kept() { + let mut late = l(3, "bbbbbbbb", &[(2, "aaaaaaaa")]); + late.history = vec![v(1, "aaaaaaaa")]; + let mut top = l(4, "aaaaaaaa", &[(3, "aaaaaaaa")]); + top.history = vec![v(1, "aaaaaaaa"), v(2, "aaaaaaaa")]; + let listed = [ + l(1, "aaaaaaaa", &[]), + l(2, "aaaaaaaa", &[(1, "aaaaaaaa")]), + l(3, "aaaaaaaa", &[(2, "aaaaaaaa")]), + top, + late, + ]; + assert_eq!(to_delete(&listed), [v(1, "aaaaaaaa")]); + } + #[test] fn a_version_that_has_not_arrived_still_keeps_the_layer_below() { // 3 读不出来，不知道它依据谁：2 也还是头，2 和它依据的 1 都留着 diff --git a/crates/tw-sync/src/local.rs b/crates/tw-sync/src/local.rs index 4d8eac11..904d5c09 100644 --- a/crates/tw-sync/src/local.rs +++ b/crates/tw-sync/src/local.rs @@ -3,12 +3,17 @@ //! ```text //! sync/ //! base.twconfig 上次同步的那一版（合并基准），不加密，不计入本机保留的 10 个版本 -//! state.json 上次同步的时间、失败从什么时候开始、因同步停用的插件、地址经同步变更的上游 +//! bases/ 之前的几个基准（最近 8 个，按版本标识）：同步位置上早删了的共同上一版本 +//! 从这里找 +//! keyring 同步位置上的 keyring 最近一次解得开的那一份（以口令加密，原样）：同步位置 +//! 上的丢了、被换成旧的时，由人决定放回去 +//! state.json 上次同步的时间、失败从什么时候开始、因同步停用的插件和上游、地址经同步 +//! 变更的上游、读不出来的文件 //! ``` //! //! 同步位置上的版本被删掉（设备离线期间）不影响合并：基准在这里。停止同步时整个目录删掉。 -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::io::Write; use std::path::{Path, PathBuf}; @@ -18,7 +23,11 @@ use tw_types::{Msg, msg}; pub const DIR: &str = "sync"; const BASE: &str = "base.twconfig"; +const BASES: &str = "bases"; +const KEYRING: &str = "keyring"; const STATE: &str = "state.json"; +/// `bases/` 里留几个 +const KEEP_BASES: usize = 8; /// `state.json` #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] @@ -42,6 +51,18 @@ pub struct State { pub device_record_ms: u64, /// 上次写设备记录时是不是暂停着（版本过新） pub device_record_paused: bool, + /// 等这台设备登录的上游（名字）：别的设备上登录着、这台没有，于是在这台停用着。停用是 + /// 这台自己的事，不上传；登录之后照同步的样子启用 + #[serde(skip_serializing_if = "BTreeSet::is_empty")] + pub held_login: BTreeSet, + /// 见过的 keyring 的最大代号（[`crate::keyring::Keyring::generation`]） + pub keyring_gen: u64, + /// 同步位置上读不出来的文件：名字 → 第一次读不出来的时刻。读出来了就去掉 + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + pub unreadable: BTreeMap, + /// 同步位置看上去是空的：从什么时候起、连着几轮 + pub empty_since_ms: Option, + pub empty_rounds: u32, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -142,8 +163,45 @@ impl Local { std::fs::read(self.dir.join(BASE)).ok() } - pub fn save_base(&self, bytes: &[u8]) -> Result<(), Msg> { - write_private(&self.dir.join(BASE), bytes) + /// 换基准。`id` 是它的版本标识：同时留一份在 `bases/`，只留最近的几个 + pub fn save_base(&self, id: &str, bytes: &[u8]) -> Result<(), Msg> { + write_private(&self.dir.join(BASE), bytes)?; + let dir = self.dir.join(BASES); + write_private(&dir.join(format!("{id}.twconfig")), bytes)?; + let mut all: Vec<(std::time::SystemTime, PathBuf)> = std::fs::read_dir(&dir) + .map(|rd| { + rd.filter_map(|e| e.ok()) + .filter(|e| e.file_name().to_string_lossy().ends_with(".twconfig")) + .filter_map(|e| Some((e.metadata().ok()?.modified().ok()?, e.path()))) + .collect() + }) + .unwrap_or_default(); + all.sort(); + let extra = all.len().saturating_sub(KEEP_BASES); + for (_, p) in all.into_iter().take(extra) { + let _ = std::fs::remove_file(p); + } + Ok(()) + } + + /// 之前的基准里版本标识是 `id` 的那一个 + pub fn base_by_id(&self, id: &str) -> Option> { + if id.contains(['/', '\\']) || id.starts_with('.') { + return None; + } + std::fs::read(self.dir.join(BASES).join(format!("{id}.twconfig"))).ok() + } + + /// 记着的 keyring（同步位置上最近一次解得开的那一份） + pub fn keyring(&self) -> Option> { + std::fs::read(self.dir.join(KEYRING)).ok() + } + + pub fn save_keyring(&self, bytes: &[u8]) -> Result<(), Msg> { + if self.keyring().as_deref() == Some(bytes) { + return Ok(()); + } + write_private(&self.dir.join(KEYRING), bytes) } /// 停止同步：整个目录删掉 @@ -169,10 +227,14 @@ mod tests { ..Default::default() }; l.save_state(&s).unwrap(); - l.save_base(b"PK").unwrap(); + l.save_base("0000000001-aaaaaaaa", b"PK").unwrap(); let again = Local::new(d.path()); assert_eq!(again.state(), s); assert_eq!(again.base().as_deref(), Some(&b"PK"[..])); + assert_eq!( + again.base_by_id("0000000001-aaaaaaaa").as_deref(), + Some(&b"PK"[..]) + ); #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; @@ -187,6 +249,29 @@ mod tests { assert!(!d.path().join("sync").exists()); } + #[test] + fn only_the_latest_few_bases_are_kept() { + let d = tempfile::tempdir().unwrap(); + let l = Local::new(d.path()); + for i in 1..=12u64 { + l.save_base(&format!("{i:010}-aaaaaaaa"), format!("v{i}").as_bytes()) + .unwrap(); + // 修改时间要分得出先后 + std::thread::sleep(std::time::Duration::from_millis(5)); + } + assert!(l.base_by_id("0000000001-aaaaaaaa").is_none()); + assert!(l.base_by_id("0000000004-aaaaaaaa").is_none()); + assert_eq!( + l.base_by_id("0000000005-aaaaaaaa").as_deref(), + Some(&b"v5"[..]) + ); + assert_eq!( + l.base_by_id("0000000012-aaaaaaaa").as_deref(), + Some(&b"v12"[..]) + ); + assert!(l.base_by_id("../state.json").is_none()); + } + #[test] fn the_place_ignores_credentials() { use tw_config::sync::Target; diff --git a/crates/tw-sync/src/merge.rs b/crates/tw-sync/src/merge.rs index 0c215a59..f35e34d4 100644 --- a/crates/tw-sync/src/merge.rs +++ b/crates/tw-sync/src/merge.rs @@ -63,6 +63,10 @@ pub struct Input<'a, K> { pub hold_items: &'a BTreeSet, /// 这几个单位各留各的：还没处理的冲突 pub hold_units: &'a BTreeSet<(K, String)>, + /// 逐字段合并、全是默认值时整个对象不在的那几种（全局设置的一节、Lite 的通用偏好）： + /// **不在就是每个字段都是默认值**，不是被删了。一边把一节改回默认、另一边改了这一节的 + /// 另一个字段，各管各的字段，不是「删了对改了」 + pub fieldwise: &'a dyn Fn(&K) -> bool, } /// 合并的结果 @@ -93,13 +97,22 @@ pub fn merge(i: Input<'_, K>) -> Outcome { .chain(i.local.keys()) .chain(i.remote.keys()) .collect(); + let none = Units::new(); for k in keys { - let (b, l, r) = (base.get(k), i.local.get(k), i.remote.get(k)); + let (mut b, mut l, mut r) = (base.get(k), i.local.get(k), i.remote.get(k)); + let fieldwise = (i.fieldwise)(k); + if fieldwise { + // 没有基准时也照样：不在就是全是默认值 + b = b.or(Some(&none)); + l = l.or(Some(&none)); + r = r.or(Some(&none)); + } let mut put = |local: Option<&Units>, shared: Option<&Units>| { - if let Some(v) = local { + // 逐字段的那几种合并出来一个字段都没有：全是默认值，不写 + if let Some(v) = local.filter(|v| !(fieldwise && v.is_empty())) { out.local.insert(k.clone(), v.clone()); } - if let Some(v) = shared { + if let Some(v) = shared.filter(|v| !(fieldwise && v.is_empty())) { out.shared.insert(k.clone(), v.clone()); } }; @@ -249,6 +262,7 @@ mod tests { remote: &r, hold_items: &BTreeSet::new(), hold_units: &BTreeSet::new(), + fieldwise: &|_| false, }) } @@ -533,6 +547,7 @@ mod tests { remote: &remote, hold_items: &hold_items, hold_units: &hold_units, + fieldwise: &|_| false, }); assert!(o.conflicts.is_empty(), "{:?}", o.conflicts); assert_eq!(o.local["ollama"], obj(&[("conn", json!("local"))])); @@ -553,11 +568,49 @@ mod tests { remote: &Objects::new(), hold_items: &hold_items, hold_units: &BTreeSet::new(), + fieldwise: &|_| false, }); assert!(o.shared.is_empty()); assert_eq!(o.local.len(), 1); } + /// 全局设置一节：一边改回了默认（整节不在了），另一边改了这一节的另一个字段 —— 各管各的 + /// 字段，不是删了对改了 + #[test] + fn a_field_level_object_reset_to_defaults_is_not_a_delete() { + let base = objs(&[("failover", Some(obj(&[("pause_secs", json!(100))])))]); + let local = objs(&[( + "failover", + Some(obj(&[ + ("pause_secs", json!(100)), + ("failures_to_pause", json!(7)), + ])), + )]); + let remote = Objects::new(); + let o = merge(Input { + base: Some(&base), + local: &local, + remote: &remote, + hold_items: &BTreeSet::new(), + hold_units: &BTreeSet::new(), + fieldwise: &|k: &String| k == "failover", + }); + assert!(o.conflicts.is_empty(), "{:?}", o.conflicts); + let want = obj(&[("failures_to_pause", json!(7))]); + assert_eq!(o.local["failover"], want); + assert_eq!(o.shared["failover"], want); + // 两边都改回默认：整节不在 + let o = merge(Input { + base: Some(&base), + local: &Objects::new(), + remote: &Objects::new(), + hold_items: &BTreeSet::new(), + hold_units: &BTreeSet::new(), + fieldwise: &|_: &String| true, + }); + assert!(o.local.is_empty() && o.shared.is_empty() && o.conflicts.is_empty()); + } + #[test] fn whether_there_is_anything_to_upload() { let base = objs(&[("a", Some(obj(&[("conn", json!(1)), ("models", json!(1))])))]); diff --git a/crates/tw-sync/src/record.rs b/crates/tw-sync/src/record.rs index dc5d3c78..6b8ea3e4 100644 --- a/crates/tw-sync/src/record.rs +++ b/crates/tw-sync/src/record.rs @@ -25,6 +25,10 @@ pub struct Meta { /// 位置上已经删了时，按它去本机备份里找基准 #[serde(default)] pub parent_content: Vec, + /// 更早的祖先（版本标识，老的在前，最多 [`HISTORY`] 个）：基准在同步位置上早删了时，靠它 + /// 判断基准是不是这个版本的祖先 + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub history: Vec, /// 内容的 SHA-256（只在不加密时写）：发现传输中损坏。加密时 age 的认证加密已经管了 #[serde(default, skip_serializing_if = "Option::is_none")] pub sha256: Option, @@ -33,6 +37,21 @@ pub struct Meta { pub conflicts: Vec, } +/// 一个版本记多少个祖先 +pub const HISTORY: usize = 200; + +impl Meta { + /// 这个版本自己和它记得的全部祖先 + pub fn lineage(&self, own_id: &str) -> BTreeSet { + self.history + .iter() + .chain(&self.parents) + .cloned() + .chain([own_id.to_string()]) + .collect() + } +} + /// 一处冲突：同一个对象（的几个单位）在两台设备上被改成了不同的内容。 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct Conflict { @@ -82,6 +101,11 @@ impl Conflict { !self.whole_item() && self.item.kind.is_named() && self.reason.is_none() } + /// 「使用另一边」能不能选。合并结果过不了校验的（`invalid`）不能：选了还是过不了 + pub fn remote_allowed(&self) -> bool { + self.reason != Some(Reason::Invalid) + } + /// 这台设备看这处冲突时，哪一边是「本机」：自己写的那一边；不是参与的设备时，是和本机 /// 现在内容一样的那一边，都不一样就是同步版本里的那一边 pub fn mine<'a>( @@ -215,6 +239,7 @@ mod tests { device_name: "Mac mini".into(), parents: vec!["0000000006-aaaaaaaa".into()], parent_content: vec!["abc".into()], + history: vec!["0000000005-aaaaaaaa".into()], sha256: None, conflicts: vec![c(Some(json!({"base_url": "x"})), None, ItemKind::Key)], }; @@ -222,5 +247,13 @@ mod tests { assert_eq!(v["conflicts"][0]["item"]["kind"], "key"); let back: Meta = serde_json::from_value(v).unwrap(); assert_eq!(back, m); + assert_eq!( + m.lineage("0000000007-aaaaaaaa"), + BTreeSet::from([ + "0000000005-aaaaaaaa".to_string(), + "0000000006-aaaaaaaa".to_string(), + "0000000007-aaaaaaaa".to_string() + ]) + ); } } diff --git a/crates/tw-sync/src/store/folder.rs b/crates/tw-sync/src/store/folder.rs index a8fca449..a6ae4891 100644 --- a/crates/tw-sync/src/store/folder.rs +++ b/crates/tw-sync/src/store/folder.rs @@ -5,14 +5,34 @@ //! 版本。临时名以点开头，列目录时跳过；云盘客户端多数也不传点开头的临时文件。 //! //! 云盘里还没下载到本机的文件（iCloud 的 `.名字.icloud` 占位文件、Dropbox 的在线文件）列目录 -//! 时要么不出现、要么读不出来 —— 调用方把读不出来的版本当成「尚未到达」，到了再合并。 +//! 时照真名列出、读出来是空的（iCloud）或者读不出来 —— 调用方把读不出来的版本当成「尚未到达」， +//! 到了再合并。 use std::io::Write; use std::path::{Path, PathBuf}; use tw_types::msg; -use super::{Entry, Store, StoreError, check_path, split, temp_name}; +use super::{ + Entry, MAX_FILE, Store, StoreError, TEMP_PREFIX, check_path, own_temp, split, temp_name, + too_large, +}; + +/// iCloud Drive 还没下载到本机的文件，在目录里是一个 `.名字.icloud` 的占位文件 +const ICLOUD: &str = ".icloud"; + +/// 读一个文件，**边读边数**：超过上限就不读了 +fn read_limited(p: &Path) -> std::io::Result, ()>> { + use std::io::Read; + let f = std::fs::File::open(p)?; + let mut out = Vec::new(); + f.take(MAX_FILE as u64 + 1).read_to_end(&mut out)?; + Ok(if out.len() > MAX_FILE { + Err(()) + } else { + Ok(out) + }) +} pub struct Folder { root: PathBuf, @@ -144,7 +164,19 @@ impl Store for Folder { let mut out = Vec::new(); for e in rd.flatten() { let name = e.file_name().to_string_lossy().to_string(); - // 临时文件、云盘的占位文件、系统的 .DS_Store + // iCloud 还没下载的文件：照它的真名列出来，读的时候读不到 —— 当作尚未到达 + if let Some(real) = name + .strip_prefix('.') + .and_then(|n| n.strip_suffix(ICLOUD)) + .filter(|n| !n.starts_with('.') && !d.join(n).exists()) + { + out.push(Entry { + name: real.to_string(), + size: 0, + }); + continue; + } + // 临时文件、系统的 .DS_Store if name.starts_with('.') { continue; } @@ -166,13 +198,22 @@ impl Store for Folder { async fn get(&self, path: &str) -> Result>, StoreError> { let root = self.root.clone(); let p = self.at(path)?; - blocking(move || match std::fs::read(&p) { - Ok(b) => Ok(Some(b)), + let shown = path.to_string(); + blocking(move || match read_limited(&p) { + Ok(Ok(b)) => Ok(Some(b)), + Ok(Err(())) => Err(too_large(&shown, MAX_FILE)), Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - if root.is_dir() { - Ok(None) + if !root.is_dir() { + return Err(io(&root, e)); + } + // iCloud 还没下载的：有，只是还没到 —— 读成空的（调用方当作尚未到达） + let placeholder = p + .file_name() + .map(|n| p.with_file_name(format!(".{}{ICLOUD}", n.to_string_lossy()))); + if placeholder.is_some_and(|ph| ph.is_file()) { + Ok(Some(Vec::new())) } else { - Err(io(&root, e)) + Ok(None) } } Err(e) => Err(io(&p, e)), @@ -252,6 +293,41 @@ impl Store for Folder { }) .await } + + /// 这台设备写到一半断了留下的临时文件：一小时以前的删掉（还在写的那一个不动） + async fn sweep(&self, tag: &str) { + let root = self.root.clone(); + let tag = tag.to_string(); + let _ = blocking(move || { + let hour = std::time::Duration::from_secs(3600); + for d in [ + root.clone(), + root.join(crate::layout::VERSIONS), + root.join(crate::layout::DEVICES), + ] { + let Ok(rd) = std::fs::read_dir(&d) else { + continue; + }; + for e in rd.flatten() { + let name = e.file_name().to_string_lossy().to_string(); + if !name.starts_with(TEMP_PREFIX) || !own_temp(&name, &tag) { + continue; + } + let old = e + .metadata() + .and_then(|m| m.modified()) + .ok() + .and_then(|t| t.elapsed().ok()) + .is_some_and(|age| age > hour); + if old { + let _ = std::fs::remove_file(e.path()); + } + } + } + Ok(()) + }) + .await; + } } /// 盯着同步文件夹：新版本、新的 keyring 一到就发一个信号（见 [`tw_watch::watch`]）。盯的是 @@ -336,15 +412,27 @@ mod tests { b"placeholder", ) .unwrap(); + std::fs::write(d.path().join("versions/.DS_Store"), b"x").unwrap(); std::fs::create_dir(d.path().join("versions/sub")).unwrap(); let got = f.list("versions").await.unwrap(); assert_eq!( got, - [Entry { - name: "0000000002-bbbbbbbb.twconfig".into(), - size: 1 - }] + [ + Entry { + name: "0000000002-bbbbbbbb.twconfig".into(), + size: 1 + }, + Entry { + name: "0000000003-cccccccc.twconfig".into(), + size: 0 + } + ] ); + std::fs::remove_file( + d.path() + .join("versions/.0000000003-cccccccc.twconfig.icloud"), + ) + .unwrap(); f.delete("versions/0000000002-bbbbbbbb.twconfig") .await .unwrap(); @@ -354,6 +442,82 @@ mod tests { assert!(f.list("versions").await.unwrap().is_empty()); } + /// iCloud 还没下载的版本：列得出名字，读不到 —— 当作尚未到达，不是没有 + #[tokio::test] + async fn an_icloud_placeholder_is_a_version_that_has_not_arrived() { + let (d, f) = bed(); + std::fs::create_dir(d.path().join("versions")).unwrap(); + std::fs::write( + d.path() + .join("versions/.0000000003-cccccccc.twconfig.icloud"), + b"placeholder", + ) + .unwrap(); + let got = f.list("versions").await.unwrap(); + assert_eq!(got.len(), 1); + assert_eq!(got[0].name, "0000000003-cccccccc.twconfig"); + assert_eq!( + f.get("versions/0000000003-cccccccc.twconfig") + .await + .unwrap(), + Some(Vec::new()), + "读成空的：有，还没到" + ); + assert_eq!( + f.get("versions/0000000004-cccccccc.twconfig") + .await + .unwrap(), + None + ); + } + + #[tokio::test] + async fn an_oversized_file_is_not_read() { + let (d, f) = bed(); + std::fs::create_dir(d.path().join("versions")).unwrap(); + let big = vec![0u8; MAX_FILE + 1]; + std::fs::write(d.path().join("versions/0000000001-aaaaaaaa.twconfig"), big).unwrap(); + assert!( + f.get("versions/0000000001-aaaaaaaa.twconfig") + .await + .is_err() + ); + } + + /// 写到一半断了留下的临时文件：自己的、一小时以前的删掉，别人的和刚写的不动 + #[tokio::test] + async fn leftover_temporary_files_of_this_device_are_swept() { + let (d, f) = bed(); + std::fs::create_dir(d.path().join("versions")).unwrap(); + let mine = d + .path() + .join("versions") + .join(temp_name("0000000002-aaaaaaaa.twconfig")); + let theirs = d + .path() + .join("versions") + .join(temp_name("0000000002-bbbbbbbb.twconfig")); + let fresh = d + .path() + .join("versions") + .join(temp_name("0000000003-aaaaaaaa.twconfig")); + for p in [&mine, &theirs, &fresh] { + std::fs::write(p, b"x").unwrap(); + } + let old = std::time::SystemTime::now() - std::time::Duration::from_secs(7200); + for p in [&mine, &theirs] { + std::fs::File::options() + .write(true) + .open(p) + .unwrap() + .set_modified(old) + .unwrap(); + } + f.sweep("aaaaaaaa").await; + assert!(!mine.exists()); + assert!(theirs.exists() && fresh.exists()); + } + #[tokio::test] async fn a_missing_root_is_unreachable_and_never_created() { let d = tempfile::tempdir().unwrap(); diff --git a/crates/tw-sync/src/store/mod.rs b/crates/tw-sync/src/store/mod.rs index cd56a9c7..4e03c4a1 100644 --- a/crates/tw-sync/src/store/mod.rs +++ b/crates/tw-sync/src/store/mod.rs @@ -88,6 +88,46 @@ pub trait Store: Send + Sync { path: &str, bytes: &[u8], ) -> impl Future> + Send; + /// 清掉这台设备以前留下的临时文件（写到一半断了的）。`tag` 是设备标识的前 8 位。没有临时 + /// 文件可言的存储什么都不做 + fn sweep(&self, tag: &str) -> impl Future + Send { + let _ = tag; + async {} + } +} + +/// 从同步位置读一个文件最多多少字节：配置包的上限，加上加密和余量。**边读边数**，超过就 +/// 不读了 —— 对同步位置有写权限的人能放一个很大的文件进来 +pub const MAX_FILE: usize = tw_pack::MAX_BYTES + 1024 * 1024; + +/// 一个目录列表（PROPFIND、ListObjectsV2）的答复最多多少字节 +pub const MAX_LISTING: usize = 8 * 1024 * 1024; + +/// 太大了 +pub(crate) fn too_large(what: &str, max: usize) -> StoreError { + StoreError::Other(msg!( + "sync.store.too_large", what = what, limit = max / (1024 * 1024) => + "{what} at the sync location is larger than {limit} MiB, so it was not read" + )) +} + +/// 读一个 HTTP 答复的正文，**边读边数**：超过 `max` 就停 +pub(crate) async fn read_limited( + mut r: reqwest::Response, + max: usize, + what: &str, +) -> Result, StoreError> { + if r.content_length().is_some_and(|n| n as usize > max) { + return Err(too_large(what, max)); + } + let mut out = Vec::new(); + while let Some(chunk) = r.chunk().await.map_err(|e| http_error(&e))? { + if out.len() + chunk.len() > max { + return Err(too_large(what, max)); + } + out.extend_from_slice(&chunk); + } + Ok(out) } /// 按配置选出来的那一种。 @@ -170,12 +210,31 @@ impl Store for AnyStore { async fn create_exclusive(&self, path: &str, bytes: &[u8]) -> Result { each!(self, s => s.create_exclusive(path, bytes).await) } + async fn sweep(&self, tag: &str) { + each!(self, s => s.sweep(tag).await) + } } -/// 临时文件名：点开头（列目录时跳过），带随机串（两个进程不撞） +/// 临时文件名：点开头（列目录时跳过），带随机串（两个进程不撞），以最终的名字结尾（版本和 +/// 设备记录的名字里有设备标识，清理时认得出哪些是自己的） pub(crate) fn temp_name(final_name: &str) -> String { let r: u64 = rand::random(); - format!(".tmp-{r:016x}-{final_name}") + format!("{TEMP_PREFIX}{r:016x}-{final_name}") +} + +pub(crate) const TEMP_PREFIX: &str = ".tmp-"; + +/// 一个临时文件是不是这台设备留下的：最终的名字是它的版本、它的设备记录，或者 keyring +pub(crate) fn own_temp(name: &str, tag: &str) -> bool { + let Some(rest) = name.strip_prefix(TEMP_PREFIX) else { + return false; + }; + let Some((_, final_name)) = rest.split_once('-') else { + return false; + }; + final_name == tag + || final_name == crate::layout::KEYRING + || crate::layout::VersionName::parse(final_name).is_some_and(|v| v.device == tag) } /// 一个路径拆成目录和文件名。只有一层目录 @@ -229,6 +288,21 @@ pub(crate) fn http_status(status: reqwest::StatusCode, what: &str) -> StoreError } /// 请求没发出去或者没收到答复 +/// 发一个请求。**连接池里那条空闲连接刚被服务端关掉**（Apache 默认空闲 5 秒就关，发出去 +/// 就收到「连接在答复之前关了」）时换一条连接再发一次：请求根本没到服务端那里。超时、连不上 +/// 不重发 +pub(crate) async fn send(r: reqwest::RequestBuilder) -> Result { + let again = r.try_clone(); + match r.send().await { + Ok(resp) => Ok(resp), + Err(e) if e.is_request() && !e.is_timeout() && !e.is_connect() => match again { + Some(r) => r.send().await.map_err(|e| http_error(&e)), + None => Err(http_error(&e)), + }, + Err(e) => Err(http_error(&e)), + } +} + pub(crate) fn http_error(e: &reqwest::Error) -> StoreError { let mut detail = e.to_string(); let mut cur = std::error::Error::source(e); @@ -269,6 +343,16 @@ mod tests { } } + #[test] + fn a_temporary_file_is_recognised_as_this_devices() { + let t = temp_name("0000000003-aaaaaaaa.twconfig"); + assert!(own_temp(&t, "aaaaaaaa")); + assert!(!own_temp(&t, "bbbbbbbb")); + assert!(own_temp(&temp_name("aaaaaaaa"), "aaaaaaaa")); + assert!(own_temp(&temp_name("keyring"), "aaaaaaaa")); + assert!(!own_temp("notes.txt", "aaaaaaaa")); + } + #[test] fn statuses_are_sorted_by_what_to_do_next() { use reqwest::StatusCode as S; diff --git a/crates/tw-sync/src/store/s3.rs b/crates/tw-sync/src/store/s3.rs index a823ee1f..c23db4f8 100644 --- a/crates/tw-sync/src/store/s3.rs +++ b/crates/tw-sync/src/store/s3.rs @@ -17,7 +17,9 @@ use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, utf8_percent_encode}; use reqwest::Method; use tw_types::msg; -use super::{Entry, Store, StoreError, check_path, http_error, http_status}; +use super::{ + Entry, MAX_FILE, MAX_LISTING, Store, StoreError, check_path, http_status, read_limited, +}; const TIMEOUT: Duration = Duration::from_secs(120); @@ -179,12 +181,15 @@ impl S3 { if !body.is_empty() { r = r.body(body.to_vec()); } - r.send().await.map_err(|e| http_error(&e)) + super::send(r).await } /// S3 错误答复里的 `` async fn error_code(r: reqwest::Response) -> String { - let body = r.text().await.unwrap_or_default(); + let body = read_limited(r, 64 * 1024, "the error") + .await + .map(|b| String::from_utf8_lossy(&b).to_string()) + .unwrap_or_default(); super::xml::parse(&body, "Error") .and_then(|(rs, top)| { rs.first() @@ -280,7 +285,8 @@ impl Store for S3 { if !r.status().is_success() { return Err(http_status(r.status(), "ListObjectsV2")); } - let body = r.text().await.map_err(|e| http_error(&e))?; + let body = read_limited(r, MAX_LISTING, "the listing").await?; + let body = String::from_utf8_lossy(&body).to_string(); let Some((records, top)) = super::xml::parse(&body, "Contents") else { return Err(StoreError::Other(msg!( "sync.s3.bad_listing" => @@ -318,9 +324,7 @@ impl Store for S3 { let r = self.send(Method::GET, &url, &[], b"").await?; match r.status().as_u16() { 404 => Ok(None), - s if (200..300).contains(&s) => { - Ok(Some(r.bytes().await.map_err(|e| http_error(&e))?.to_vec())) - } + s if (200..300).contains(&s) => Ok(Some(read_limited(r, MAX_FILE, path).await?)), _ => Err(http_status(r.status(), "GET")), } } diff --git a/crates/tw-sync/src/store/webdav.rs b/crates/tw-sync/src/store/webdav.rs index 3b6a85fa..809f47f2 100644 --- a/crates/tw-sync/src/store/webdav.rs +++ b/crates/tw-sync/src/store/webdav.rs @@ -13,7 +13,10 @@ use std::time::Duration; use reqwest::{Method, StatusCode}; use tw_types::msg; -use super::{Entry, Store, StoreError, check_path, http_error, http_status, split, temp_name}; +use super::{ + Entry, MAX_FILE, MAX_LISTING, Store, StoreError, TEMP_PREFIX, check_path, http_status, + own_temp, read_limited, split, temp_name, +}; /// 一个请求最多等多久。版本最大 16 MiB，慢的网盘也够了 const TIMEOUT: Duration = Duration::from_secs(120); @@ -95,7 +98,7 @@ impl WebDav { } async fn send(&self, r: reqwest::RequestBuilder) -> Result { - r.send().await.map_err(|e| http_error(&e)) + super::send(r).await } /// 建目录。已经有了（405）不算错；上一级没有（409）先建上一级（同步位置的根） @@ -201,8 +204,9 @@ fn last_segment(href: &str) -> String { const PROPFIND: &str = r#" "#; -impl Store for WebDav { - async fn list(&self, dir: &str) -> Result, StoreError> { +impl WebDav { + /// 列一个目录，**连点开头的一起**（清临时文件要看到它们）。目录不在是 `Ok(None)` + async fn list_raw(&self, dir: &str) -> Result>, StoreError> { let url = self.dir(dir)?; let own = url.path().trim_end_matches('/').to_string(); let r = self @@ -215,10 +219,11 @@ impl Store for WebDav { .await?; match r.status().as_u16() { 207 | 200 => {} - 404 => return Ok(Vec::new()), + 404 => return Ok(None), _ => return Err(http_status(r.status(), "PROPFIND")), } - let body = r.text().await.map_err(|e| http_error(&e))?; + let body = read_limited(r, MAX_LISTING, "the directory listing").await?; + let body = String::from_utf8_lossy(&body); let Some((records, _)) = super::xml::parse(&body, "response") else { return Err(StoreError::Other(msg!( "sync.webdav.bad_listing" => @@ -236,7 +241,7 @@ impl Store for WebDav { continue; } let name = last_segment(href); - if name.is_empty() || name.starts_with('.') { + if name.is_empty() { continue; } out.push(Entry { @@ -248,16 +253,77 @@ impl Store for WebDav { }); } out.sort_by(|a, b| a.name.cmp(&b.name)); - Ok(out) + Ok(Some(out)) + } + + async fn list_raw_root(&self) -> Result>, StoreError> { + self.list_raw("").await + } + + /// 同步位置的根在不在。**目录列不出来（404）不等于是空的**：根本身不在时那是地址写错了、 + /// 或者服务器上的文件夹被挪走了，不能当成同步数据被清空了 + async fn root_exists(&self) -> Result { + let r = self + .send( + self.req(method("PROPFIND"), self.base.clone()) + .header("depth", "0") + .header("content-type", "application/xml; charset=utf-8") + .body(PROPFIND), + ) + .await?; + match r.status().as_u16() { + 207 | 200 => Ok(true), + 404 => Ok(false), + _ => Err(http_status(r.status(), "PROPFIND")), + } + } +} + +impl Store for WebDav { + async fn list(&self, dir: &str) -> Result, StoreError> { + match self.list_raw(dir).await? { + Some(all) => Ok(all + .into_iter() + .filter(|e| !e.name.starts_with('.')) + .collect()), + None if self.root_exists().await? => Ok(Vec::new()), + None => Err(StoreError::Unreachable(msg!( + "sync.webdav.root_missing", url = self.base.as_str() => + "the WebDAV folder {url} is not there" + ))), + } + } + + /// 这台设备写到一半断了留下的临时文件（`PUT` 了、没来得及 `MOVE`）。只在开始同步时清一次， + /// 那时这台设备没有在写的 + async fn sweep(&self, tag: &str) { + for dir in ["", crate::layout::VERSIONS, crate::layout::DEVICES] { + let listed = if dir.is_empty() { + self.list_raw_root().await + } else { + self.list_raw(dir).await + }; + let Ok(Some(all)) = listed else { continue }; + for e in all { + if e.name.starts_with(TEMP_PREFIX) && own_temp(&e.name, tag) { + let path = if dir.is_empty() { + e.name.clone() + } else { + format!("{dir}/{}", e.name) + }; + if let Ok(u) = self.file(&path) { + let _ = self.send(self.req(Method::DELETE, u)).await; + } + } + } + } } async fn get(&self, path: &str) -> Result>, StoreError> { let r = self.send(self.req(Method::GET, self.file(path)?)).await?; match r.status().as_u16() { 404 | 410 => Ok(None), - s if (200..300).contains(&s) => { - Ok(Some(r.bytes().await.map_err(|e| http_error(&e))?.to_vec())) - } + s if (200..300).contains(&s) => Ok(Some(read_limited(r, MAX_FILE, path).await?)), _ => Err(http_status(r.status(), "GET")), } } diff --git a/crates/tw-sync/tests/common/mod.rs b/crates/tw-sync/tests/common/mod.rs index 8f758d12..60086f52 100644 --- a/crates/tw-sync/tests/common/mod.rs +++ b/crates/tw-sync/tests/common/mod.rs @@ -3,6 +3,9 @@ //! **只实现同步用到的那几样**，但按规矩答：`PUT` 认 `If-None-Match: *`、`MOVE` 认 //! `Overwrite: F`、`PROPFIND` 只列一层、S3 的列目录认 `prefix` 和 `delimiter`。凭据不对回 401 / //! 403。可以让它「坏掉」：回 503，或者不认 `MOVE`、`If-None-Match`（有的服务器就是这样）。 +//! +//! 假 S3 **自己核对 SigV4 签名**（[`sigv4`]，照 AWS 的规范另写一份，不借被测的那一份；用 AWS +//! 文档里的示例核对过它自己，见 `stores.rs`）：签名算错了照样回 403。 #![allow(dead_code)] @@ -13,6 +16,7 @@ use std::sync::{Arc, Mutex}; use axum::body::Body; use axum::http::{HeaderMap, Request, Response, StatusCode}; +use sha2::{Digest, Sha256}; pub const USER: &str = "tester"; pub const PASS: &str = "app-password"; @@ -102,17 +106,16 @@ pub async fn webdav() -> Fake { if header(req.headers(), "authorization") != Some(want.as_str()) { return status(401); } - let Some(rel) = req.uri().path().strip_prefix("/dav/root/") else { - // 根本身 - if req.uri().path().trim_end_matches('/') == "/dav/root" { - return match req.method().as_str() { - "MKCOL" => status(405), - "PROPFIND" => status(207), - _ => status(405), - }; - } - return status(404); + let path = req.uri().path().to_string(); + let rel = match path.strip_prefix("/dav/root/") { + Some(r) => r, + // 根本身：用户选的那个文件夹，在 + None if path == "/dav/root" => "", + None => return status(404), }; + if rel.is_empty() && req.method().as_str() == "MKCOL" { + return status(405); + } let rel = percent_encoding::percent_decode_str(rel) .decode_utf8_lossy() .to_string(); @@ -128,26 +131,36 @@ pub async fn webdav() -> Fake { } "PROPFIND" => { let dir = rel.trim_end_matches('/').to_string(); - if !dirs.contains(&dir) { + if !dir.is_empty() && !dirs.contains(&dir) { return status(404); } + let at = |p: &str| { + if p.is_empty() { + "/dav/root/".to_string() + } else { + format!("/dav/root/{p}/") + } + }; let mut xml = String::from( r#""#, ); xml.push_str(&format!( - "/dav/root/{dir}/\ - " + "{}\ + ", + at(&dir) )); - for (name, bytes) in files.iter() { - if let Some((parent, leaf)) = name.rsplit_once('/') - && parent == dir - { - xml.push_str(&format!( - "/dav/root/{parent}/{leaf}\ - {}\ - HTTP/1.1 200 OK", - bytes.len() - )); + if header(req.headers(), "depth") != Some("0") { + for (name, bytes) in files.iter() { + let (parent, leaf) = name.rsplit_once('/').unwrap_or(("", name)); + if parent == dir { + xml.push_str(&format!( + "{}{leaf}\ + {}\ + HTTP/1.1 200 OK", + at(parent), + bytes.len() + )); + } } } xml.push_str(""); @@ -223,15 +236,22 @@ pub async fn s3() -> Fake { return status(503); } let auth = header(req.headers(), "authorization").unwrap_or_default(); - if !auth.starts_with(&format!("AWS4-HMAC-SHA256 Credential={AK}/")) - || !auth.contains("/s3/aws4_request") - || header(req.headers(), "x-amz-content-sha256").is_none() - { + let signed = sigv4( + req.method().as_str(), + req.uri().path(), + req.uri().query().unwrap_or_default(), + req.headers(), + &body, + AK, + SK, + ); + if !auth.starts_with(&format!("AWS4-HMAC-SHA256 Credential={AK}/")) || signed.is_err() { return Response::builder() .status(403) - .body(Body::from( - "SignatureDoesNotMatch", - )) + .body(Body::from(format!( + "SignatureDoesNotMatch{}", + signed.err().unwrap_or_default() + ))) .unwrap(); } let path = req.uri().path().to_string(); @@ -304,6 +324,146 @@ pub async fn s3() -> Fake { Fake { addr, files, knobs } } +// ─────────────────────────────────────────────── SigV4（服务端核对） + +/// HMAC-SHA256（RFC 2104），照定义写 +fn hmac(key: &[u8], msg: &[u8]) -> [u8; 32] { + let mut k = [0u8; 64]; + if key.len() > 64 { + let d: [u8; 32] = Sha256::digest(key).into(); + k[..32].copy_from_slice(&d); + } else { + k[..key.len()].copy_from_slice(key); + } + let mut inner = Sha256::new(); + inner.update(k.map(|b| b ^ 0x36)); + inner.update(msg); + let inner: [u8; 32] = inner.finalize().into(); + let mut outer = Sha256::new(); + outer.update(k.map(|b| b ^ 0x5c)); + outer.update(inner); + outer.finalize().into() +} + +fn hex(b: &[u8]) -> String { + b.iter().map(|x| format!("{x:02x}")).collect() +} + +/// AWS 的 URI 编码：只有 `A-Z a-z 0-9 - _ . ~` 不编码，别的一律 `%XX`（大写） +fn aws_encode(s: &str, keep_slash: bool) -> String { + let mut out = String::new(); + for b in s.bytes() { + match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + out.push(b as char) + } + b'/' if keep_slash => out.push('/'), + _ => out.push_str(&format!("%{b:02X}")), + } + } + out +} + +fn decode(s: &str) -> String { + percent_encoding::percent_decode_str(s) + .decode_utf8_lossy() + .to_string() +} + +/// 照 AWS SigV4 的规范从收到的请求重算签名，和 `Authorization` 里的比。不对是 `Err(原因)` +pub fn sigv4( + method: &str, + path: &str, + query: &str, + headers: &HeaderMap, + body: &[u8], + access_key: &str, + secret: &str, +) -> Result<(), String> { + let auth = header(headers, "authorization").ok_or("no Authorization")?; + let rest = auth + .strip_prefix("AWS4-HMAC-SHA256 ") + .ok_or("not AWS4-HMAC-SHA256")?; + let mut fields: BTreeMap<&str, &str> = BTreeMap::new(); + for part in rest.split(',') { + let (k, v) = part.trim().split_once('=').ok_or("bad Authorization")?; + fields.insert(k, v); + } + let cred = fields.get("Credential").ok_or("no Credential")?; + let signed = fields.get("SignedHeaders").ok_or("no SignedHeaders")?; + let got = fields.get("Signature").ok_or("no Signature")?; + let mut scope_parts = cred.splitn(2, '/'); + if scope_parts.next() != Some(access_key) { + return Err("unknown access key".into()); + } + let scope = scope_parts.next().ok_or("no scope")?; + let s: Vec<&str> = scope.split('/').collect(); + if s.len() != 4 || s[2] != "s3" || s[3] != "aws4_request" { + return Err(format!("bad scope {scope}")); + } + let amz_date = header(headers, "x-amz-date").ok_or("no x-amz-date")?; + if !amz_date.starts_with(s[0]) { + return Err("x-amz-date is not the scope's date".into()); + } + let payload = header(headers, "x-amz-content-sha256").ok_or("no x-amz-content-sha256")?; + if payload != "UNSIGNED-PAYLOAD" && payload != hex(&Sha256::digest(body)) { + return Err("x-amz-content-sha256 does not match the body".into()); + } + let names: Vec<&str> = signed.split(';').collect(); + if !names.contains(&"host") { + return Err("host is not signed".into()); + } + let mut canon_headers = String::new(); + for n in &names { + let v = headers + .get_all(*n) + .iter() + .filter_map(|v| v.to_str().ok()) + .map(|v| v.split_whitespace().collect::>().join(" ")) + .collect::>() + .join(","); + if v.is_empty() && headers.get(*n).is_none() { + return Err(format!("signed header {n} is not there")); + } + canon_headers.push_str(&format!("{n}:{v}\n")); + } + let uri: String = path + .split('/') + .map(|seg| aws_encode(&decode(seg), false)) + .collect::>() + .join("/"); + let mut pairs: Vec<(String, String)> = query + .split('&') + .filter(|p| !p.is_empty()) + .map(|p| { + let (k, v) = p.split_once('=').unwrap_or((p, "")); + (aws_encode(&decode(k), false), aws_encode(&decode(v), false)) + }) + .collect(); + pairs.sort(); + let canon_query = pairs + .iter() + .map(|(k, v)| format!("{k}={v}")) + .collect::>() + .join("&"); + let canonical = format!("{method}\n{uri}\n{canon_query}\n{canon_headers}\n{signed}\n{payload}"); + let to_sign = format!( + "AWS4-HMAC-SHA256\n{amz_date}\n{scope}\n{}", + hex(&Sha256::digest(canonical.as_bytes())) + ); + let k_date = hmac(format!("AWS4{secret}").as_bytes(), s[0].as_bytes()); + let k_region = hmac(&k_date, s[1].as_bytes()); + let k_service = hmac(&k_region, b"s3"); + let k_signing = hmac(&k_service, b"aws4_request"); + let want = hex(&hmac(&k_signing, to_sign.as_bytes())); + if want != *got { + return Err(format!( + "signature does not match; canonical request:\n{canonical}" + )); + } + Ok(()) +} + /// 一个没人听的地址：连上去就被拒绝 pub fn nowhere() -> String { let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); diff --git a/crates/tw-sync/tests/devices.rs b/crates/tw-sync/tests/devices.rs index add592cf..415c718c 100644 --- a/crates/tw-sync/tests/devices.rs +++ b/crates/tw-sync/tests/devices.rs @@ -514,13 +514,41 @@ async fn encryption_can_be_turned_on_off_and_its_passphrase_changed() { assert!(t.eb.unlock("a brand new passphrase").await.unwrap()); t.eb.round().await.unwrap(); - // A 关闭加密：B 跟着变成不加密 + // A 关闭加密：keyring 留着、改写成「已关闭」（有口令的人关的），版本不再加密；B 跟着变成 + // 不加密 t.ea.disable_encryption("a brand new passphrase") .await .unwrap(); - assert!(!t.loc.path().join("keyring").exists()); + let kr = std::fs::read(t.loc.path().join("keyring")).unwrap(); + assert!(tw_sync::keyring::marked_off(&kr)); + assert!( + tw_sync::keyring::Keyring::open(&kr, "a brand new passphrase") + .unwrap() + .off() + ); + for v in versions(t.loc.path()) { + let bytes = std::fs::read(t.loc.path().join("versions").join(&v)).unwrap(); + assert!(!tw_pack::is_encrypted(&bytes), "加密过的版本删了：{v}"); + } let r = t.eb.round().await.unwrap(); assert!(!r.encrypted, "同步位置不加密了，B 照着改设置"); + t.b.edit(|s| s.replace("sk-after-encryption", "sk-after-off")); + assert!(t.eb.round().await.unwrap().uploaded.is_some()); + t.ea.round().await.unwrap(); + assert_eq!( + t.a.provider("relay").unwrap().key.unwrap().raw(), + "sk-after-off" + ); + + // 再开回来：B 用同一个口令接着同步 + t.ea.enable_encryption("a brand new passphrase") + .await + .unwrap(); + assert!(!tw_sync::keyring::marked_off( + &std::fs::read(t.loc.path().join("keyring")).unwrap() + )); + let r = t.eb.round().await.unwrap(); + assert!(r.encrypted, "B 记着口令，解得开新的 keyring"); } /// 两台设备同时开启加密同步：keyring 只建得成一份，后写的那台转去加入 @@ -659,19 +687,48 @@ async fn a_key_in_use_here_is_not_deleted_by_another_device() { ); } -/// 另一台设备停止同步并删除了同步数据：这台设备发现同步位置空了 +/// 另一台设备停止同步并删除了同步数据：这台设备几轮、一段时间之后确认同步位置空了，由人 +/// 决定（重新上传本机配置），同步设置和口令都不动 #[tokio::test] async fn a_purged_location_is_noticed() { let mut t = bed(true).await; t.ea.stop(true).await.unwrap(); assert!(versions(t.loc.path()).is_empty()); assert!(!t.loc.path().join("keyring").exists()); + for _ in 0..3 { + let r = t.eb.round().await; + assert!( + matches!(r, Err(RoundError::LooksEmpty)), + "一轮读到空的不下结论：{:?}", + r.map(|r| r.uploaded) + ); + } + // 过了十分钟（改本机记着的时刻） + let mut st = t.eb.local().state(); + st.empty_since_ms = st.empty_since_ms.map(|t| t - 11 * 60 * 1000); + t.eb.local().save_state(&st).unwrap(); let r = t.eb.round().await; assert!( matches!(r, Err(RoundError::Emptied)), "{:?}", r.map(|r| r.uploaded) ); + assert!(t.eb.local().base().is_some(), "本机的同步状态不删"); + // 重新上传本机配置：keyring 放回这台记着的那一份，版本以本机配置上传 + t.b.edit(|s| add_provider(s, "kept", "https://kept.example", "sk-kept")); + let r = t.eb.republish().await.unwrap(); + assert!(r.encrypted); + assert!(t.loc.path().join("keyring").exists()); + assert_eq!(versions(t.loc.path()).len(), 1); + // A 重新加入，拿到 B 的配置 + let mut ea = Engine::new( + folder(t.loc.path()), + t.a.clone(), + t.a.dir.path(), + settings(A_ID, "Mac mini", true, t.loc.path()), + ); + ea.join(None).await.unwrap(); + assert!(t.a.provider("kept").is_some()); } /// 同一套流程走 WebDAV 和 S3（进程内的假服务） diff --git a/crates/tw-sync/tests/regressions.rs b/crates/tw-sync/tests/regressions.rs new file mode 100644 index 00000000..08440fb7 --- /dev/null +++ b/crates/tw-sync/tests/regressions.rs @@ -0,0 +1,1118 @@ +//! 审查里找出来的同步问题，一条一条钉住：加密不可降级、等登录的上游、捷径不吞改动、基准不是 +//! 祖先、晚到的头、一个坏文件挡住所有设备、同步位置读起来是空的、清除只删同步文件。 +//! +//! 设备这一边和 `devices.rs` 一样是最简单的那种（配置就是数据目录里的 `config.yaml`），多一个 +//! 钩子：一轮的中途（读完本机配置之后、上传之前）跑一次，模拟用户此刻改配置、别的设备此刻上传。 + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; + +use sha2::Digest; +use tw_pack::{Pack, PackKind, PackOrigin, Seal}; +use tw_sync::engine::{ + ApplyError, Current, Decision, Engine, Host, RoundError, SKIP_AFTER_MS, Settings, Take, +}; +use tw_sync::keyring::Keyring; +use tw_sync::store::{AnyStore, Folder}; +use tw_sync::{Entry, Store, StoreError}; +use tw_types::Msg; + +const PASS: &str = "correct horse battery staple"; + +fn base_config(c: char) -> String { + format!( + "version: 1\nlisten:\n control:\n key: {}\nclients:\n - name: default\n key: tw-default-key-0001\n", + c.to_string().repeat(64) + ) +} + +type Hook = Arc>>>; + +#[derive(Clone)] +struct Dev { + dir: Arc, + hook: Hook, +} + +impl Dev { + fn new(key: char) -> Self { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("config.yaml"), base_config(key)).unwrap(); + Self { + dir: Arc::new(dir), + hook: Default::default(), + } + } + fn path(&self) -> PathBuf { + self.dir.path().join("config.yaml") + } + fn text(&self) -> String { + std::fs::read_to_string(self.path()).unwrap() + } + fn config(&self) -> tw_config::Config { + tw_config::try_parse(&self.text()).unwrap_or_else(|e| panic!("{e}\n{}", self.text())) + } + fn edit(&self, f: impl FnOnce(&str) -> String) { + let next = f(&self.text()); + tw_config::try_parse(&next).unwrap_or_else(|e| panic!("测试写坏了配置：{e}\n{next}")); + std::fs::write(self.path(), next).unwrap(); + } + fn provider(&self, name: &str) -> Option { + self.config().providers.into_iter().find(|p| p.name == name) + } + /// 一轮的中途跑一次 + fn during_next_round(&self, f: impl FnOnce() + Send + 'static) { + *self.hook.lock().unwrap() = Some(Box::new(f)); + } +} + +impl Host for Dev { + fn current(&self) -> Result { + let text = self.text(); + Ok(Current { + pack: Pack { + manifest: tw_pack::Manifest::new(PackKind::Full, PackOrigin::Ui), + config_yaml: text.clone(), + plugins: Default::default(), + client_prefs: None, + }, + version: tw_config::version_of(&text), + changed_ms: 1, + }) + } + async fn apply(&self, pack: Pack, base_version: &str, _from: &str) -> Result<(), ApplyError> { + if tw_config::version_of(&self.text()) != base_version { + return Err(ApplyError::Stale); + } + tw_config::try_parse(&pack.config_yaml).map_err(|r| ApplyError::Rejected(r.msg()))?; + std::fs::write(self.path(), &pack.config_yaml).unwrap(); + Ok(()) + } + fn pinned_keys(&self) -> BTreeSet { + if let Some(f) = self.hook.lock().unwrap().take() { + f(); + } + BTreeSet::new() + } + fn backup(&self, _content_id: &str) -> Option { + None + } +} + +fn settings(id: &str, name: &str, encrypt: bool, place: &Path) -> Settings { + Settings { + device_id: id.into(), + device_name: name.into(), + encrypt, + passphrase: encrypt.then(|| PASS.to_string()), + exclude: BTreeSet::new(), + place: format!("folder:{}", place.display()), + remote_core: false, + } +} + +const A_ID: &str = "aaaaaaaa-0000-4000-8000-000000000001"; +const B_ID: &str = "bbbbbbbb-0000-4000-8000-000000000002"; + +struct Bed { + loc: tempfile::TempDir, + a: Dev, + b: Dev, + ea: Engine, + eb: Engine, +} + +fn folder(loc: &Path) -> AnyStore { + AnyStore::Folder(Folder::new(loc)) +} + +async fn bed(encrypt: bool) -> Bed { + tw_sync::keyring::set_work_factor_for_tests(4); + let loc = tempfile::tempdir().unwrap(); + let (a, b) = (Dev::new('a'), Dev::new('b')); + let mut ea = Engine::new( + folder(loc.path()), + a.clone(), + a.dir.path(), + settings(A_ID, "Mac mini", encrypt, loc.path()), + ); + let mut eb = Engine::new( + folder(loc.path()), + b.clone(), + b.dir.path(), + settings(B_ID, "MacBook Air", encrypt, loc.path()), + ); + assert!(ea.create().await.unwrap()); + eb.join(None).await.unwrap(); + Bed { loc, a, b, ea, eb } +} + +fn add_provider_raw(text: &str, yaml: &str) -> String { + if text.contains("\nproviders:\n") { + text.replacen("\nproviders:\n", &format!("\nproviders:\n{yaml}"), 1) + } else { + format!("{text}providers:\n{yaml}") + } +} + +fn add_provider(text: &str, name: &str, url: &str, key: &str) -> String { + add_provider_raw( + text, + &format!(" - name: {name}\n base_url: {url}\n key: {key}\n"), + ) +} + +fn versions(loc: &Path) -> Vec { + let mut v: Vec = std::fs::read_dir(loc.join("versions")) + .map(|rd| { + rd.filter_map(|e| e.ok()) + .map(|e| e.file_name().to_string_lossy().to_string()) + .filter(|n| !n.starts_with('.')) + .collect() + }) + .unwrap_or_default(); + v.sort(); + v +} + +fn read_version(loc: &Path, id: &str) -> Vec { + std::fs::read(loc.join("versions").join(format!("{id}.twconfig"))).unwrap() +} + +/// 没有口令的人往同步位置里放一个不加密的版本（manifest.sync 照格式写、带 SHA-256），依据 +/// 最新的那一个 +fn forge_plain_version(loc: &Path, config_yaml: String) -> String { + let head = versions(loc) + .last() + .unwrap() + .trim_end_matches(".twconfig") + .to_string(); + let seq: u64 = head[..10].parse().unwrap(); + let mut pack = Pack { + manifest: tw_pack::Manifest::new(PackKind::Full, PackOrigin::Sync), + config_yaml, + plugins: Default::default(), + client_prefs: None, + }; + let sha: [u8; 32] = sha2::Sha256::digest(pack.config_yaml.as_bytes()).into(); + let sha: String = sha.iter().map(|b| format!("{b:02x}")).collect(); + let meta = tw_sync::record::Meta { + seq: seq + 1, + device: "eeeeeeee-0000-4000-8000-00000000000e".into(), + device_name: "Eve".into(), + parents: vec![head], + sha256: Some(sha), + ..Default::default() + }; + pack.manifest.sync = Some(serde_json::to_value(&meta).unwrap()); + let bytes = tw_pack::write(&pack, Seal::None).unwrap(); + let name = format!("{:010}-eeeeeeee.twconfig", seq + 1); + std::fs::write(loc.join("versions").join(&name), bytes).unwrap(); + name +} + +/// 把本机记着的「从什么时候起」往前拨，模拟时间过去了 +fn age_state(e: &Engine, by_ms: u64) { + let mut st = e.local().state(); + for t in st.unreadable.values_mut() { + *t = t.saturating_sub(by_ms); + } + st.empty_since_ms = st.empty_since_ms.map(|t| t.saturating_sub(by_ms)); + e.local().save_state(&st).unwrap(); +} + +// ─────────────────────────────────────────────── 加密不可降级 + +/// 加密的同步位置：没有口令的人放进来一个不加密的版本 —— 不读、不合并，计入跳过 +#[tokio::test] +async fn a_plaintext_version_in_an_encrypted_location_is_never_read() { + let mut t = bed(true).await; + t.a.edit(|s| add_provider(s, "relay", "https://relay.example", "sk-secret-relay")); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + assert!(t.b.provider("relay").is_some()); + let evil = add_provider(&t.a.text(), "evil", "https://evil.example", "sk-attacker"); + forge_plain_version(t.loc.path(), evil); + let r = t.eb.round().await.unwrap(); + assert!(t.b.provider("evil").is_none(), "不加密的版本被合并了"); + assert!(r.encrypted); + assert_eq!(r.skipped, 1, "跳过了，界面看得到"); +} + +/// keyring 被删、再放进一个不加密的版本：不降级、不上传明文、口令不删，要人处理。放回这台 +/// 设备记着的 keyring 之后照常同步，伪造的那个版本照样不读 +#[tokio::test] +async fn a_deleted_keyring_does_not_turn_encryption_off() { + let mut t = bed(true).await; + t.a.edit(|s| add_provider(s, "relay", "https://relay.example", "sk-secret-relay")); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + std::fs::remove_file(t.loc.path().join("keyring")).unwrap(); + forge_plain_version(t.loc.path(), t.a.text()); + t.b.edit(|s| add_provider(s, "bnew", "https://bnew.example", "sk-b-secret")); + let before = versions(t.loc.path()); + let r = t.eb.round().await; + assert!( + matches!(r, Err(RoundError::KeyringMissing(_))), + "{:?}", + r.map(|r| r.uploaded) + ); + assert_eq!(versions(t.loc.path()), before, "什么都没上传"); + assert!(t.eb.settings().encrypt && t.eb.settings().passphrase.is_some()); + // 再来一轮还是一样：不会过一会儿自己转成不加密 + assert!(matches!( + t.eb.round().await, + Err(RoundError::KeyringMissing(_)) + )); + + // 由人选「重新上传」：keyring 放回 B 记着的那一份 + let r = t.eb.republish().await.unwrap(); + assert!(r.encrypted); + assert!(t.loc.path().join("keyring").exists()); + let up = r.uploaded.expect("B 的改动这时上传"); + assert!(tw_pack::is_encrypted(&read_version(t.loc.path(), &up))); + assert!(t.b.provider("bnew").is_some()); + // A 用的还是同一把密钥，照常同步 + t.ea.round().await.unwrap(); + assert!(t.a.provider("bnew").is_some()); + assert!(t.a.provider("evil").is_none()); +} + +/// 关过一次加密、又开回来之后，有人把当年那份「已关闭」的 keyring 放回来（它当初是真的， +/// 口令也解得开）：代号比见过的小，不认 +#[tokio::test] +async fn an_old_keyring_that_says_off_is_not_believed() { + let mut t = bed(true).await; + t.ea.disable_encryption(PASS).await.unwrap(); + let off = std::fs::read(t.loc.path().join("keyring")).unwrap(); + assert!(tw_sync::keyring::marked_off(&off)); + let r = t.eb.round().await.unwrap(); + assert!(!r.encrypted, "有口令的人关的：认"); + t.ea.enable_encryption(PASS).await.unwrap(); + assert!(t.eb.round().await.unwrap().encrypted); + + std::fs::write(t.loc.path().join("keyring"), &off).unwrap(); + t.b.edit(|s| add_provider(s, "secret", "https://s.example", "sk-not-for-plaintext")); + let before = versions(t.loc.path()); + let r = t.eb.round().await; + assert!( + matches!(&r, Err(RoundError::KeyringMissing(m)) if m.code == "sync.keyring_rolled_back"), + "{:?}", + r.map(|r| r.encrypted) + ); + assert_eq!( + versions(t.loc.path()), + before, + "没有按旧的 keyring 上传明文" + ); + // 放回 B 记着的那一份（较新的） + let r = t.eb.republish().await.unwrap(); + assert!(r.encrypted); + for v in versions(t.loc.path()) { + let bytes = std::fs::read(t.loc.path().join("versions").join(&v)).unwrap(); + assert!(!String::from_utf8_lossy(&bytes).contains("sk-not-for-plaintext")); + } +} + +/// 关了加密之后有人放进来一个只有明文标记的假 keyring（解不开）：记着口令的设备不认标记， +/// 要输入口令；没有口令的设备（本来就不加密）照常不加密同步 +#[tokio::test] +async fn a_fake_off_marker_does_not_fool_a_device_with_the_passphrase() { + let mut t = bed(true).await; + t.ea.disable_encryption(PASS).await.unwrap(); + assert!(!t.eb.round().await.unwrap().encrypted); + let mut fake = tw_sync::keyring::OFF_MARK.to_vec(); + fake.extend(Keyring::generate(0).seal("someone else's phrase")); + std::fs::write(t.loc.path().join("keyring"), fake).unwrap(); + let r = t.eb.round().await; + assert!( + matches!(r, Err(RoundError::Locked(_))), + "{:?}", + r.map(|r| r.encrypted) + ); + // 一台从没有过口令的设备照常 + let c = Dev::new('c'); + let mut s = settings( + "cccccccc-0000-4000-8000-000000000003", + "Mini PC", + false, + t.loc.path(), + ); + s.passphrase = None; + let mut ec = Engine::new(folder(t.loc.path()), c.clone(), c.dir.path(), s); + assert!(!ec.join(None).await.unwrap().encrypted); +} + +/// 加密的同步位置上，不加密的设备记录不列出来；之后开启加密时删掉不加密的设备记录 +#[tokio::test] +async fn plaintext_device_records_do_not_count_once_encrypted() { + let mut t = bed(false).await; + let fake = serde_json::json!({ + "id": "cccccccc-0000-4000-8000-00000000000c", "name": "Fake", "core_version": "0.77.0", + "last_seen_ms": 4102444800000u64, "paused_update": false, "remote_core": false + }); + std::fs::write( + t.loc.path().join("devices/cccccccc"), + serde_json::to_vec(&fake).unwrap(), + ) + .unwrap(); + let names: Vec = + t.ea.round() + .await + .unwrap() + .devices + .into_iter() + .map(|d| d.name) + .collect(); + assert!( + names.contains(&"Fake".to_string()), + "不加密时是一份普通的记录" + ); + t.ea.enable_encryption(PASS).await.unwrap(); + assert!( + !t.loc.path().join("devices/cccccccc").exists(), + "开启加密时删掉不加密的设备记录" + ); + std::fs::write( + t.loc.path().join("devices/cccccccc"), + serde_json::to_vec(&fake).unwrap(), + ) + .unwrap(); + t.a.edit(|s| add_provider(s, "x", "https://x.example", "sk-x")); + let names: Vec = + t.ea.round() + .await + .unwrap() + .devices + .into_iter() + .map(|d| d.name) + .collect(); + assert!(!names.contains(&"Fake".to_string()), "{names:?}"); + assert!(names.contains(&"Mac mini".to_string())); +} + +// ─────────────────────────────────────────────── 捷径不吞改动 + +/// 一轮中途用户改了配置：下一轮照样上传 +#[tokio::test] +async fn an_edit_made_during_a_round_is_uploaded_next_round() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider(s, "one", "https://one.example", "sk-1")); + let a = t.a.clone(); + t.a.during_next_round(move || { + a.edit(|s| add_provider(s, "two", "https://two.example", "sk-2")); + }); + assert!(t.ea.round().await.unwrap().uploaded.is_some()); + assert!(t.a.provider("two").is_some()); + assert!( + t.ea.round().await.unwrap().uploaded.is_some(), + "中途的改动这一轮上传" + ); + t.eb.round().await.unwrap(); + assert!(t.b.provider("two").is_some()); +} + +/// 一轮中途别的设备传了新版本：下一轮照样合并 +#[tokio::test] +async fn a_version_arriving_during_a_round_is_merged_next_round() { + let mut t = bed(false).await; + t.b.edit(|s| add_provider(s, "fromb", "https://b.example", "sk-b")); + let up_b = t.eb.round().await.unwrap().uploaded.unwrap(); + let parked = tempfile::tempdir().unwrap(); + let file = format!("{up_b}.twconfig"); + let in_loc = t.loc.path().join("versions").join(&file); + let out = parked.path().join(&file); + std::fs::rename(&in_loc, &out).unwrap(); + t.a.edit(|s| add_provider(s, "froma", "https://a.example", "sk-a")); + t.a.during_next_round(move || std::fs::rename(&out, &in_loc).unwrap()); + t.ea.round().await.unwrap(); + assert!(t.a.provider("fromb").is_none(), "这一轮开始时还没到"); + t.ea.round().await.unwrap(); + assert!(t.a.provider("fromb").is_some()); + t.eb.round().await.unwrap(); + assert!(t.b.provider("froma").is_some()); +} + +// ─────────────────────────────────────────────── 等登录的上游 + +const CHATGPT: &str = " - name: chatgpt\n base_url: https://chatgpt.com/backend-api/codex\n oauth:\n refresh: rt-a\n endpoint: https://auth.openai.com/oauth/token\n headers:\n ChatGPT-Account-Id: acct-a\n"; +const CORP: &str = " - name: corp\n base_url: https://corp.example\n oauth:\n refresh: rt-corp-a\n endpoint: https://auth.example/token\n"; + +/// A 上登录着的 ChatGPT 账号和 OAuth 上游到了没登录的 B：在 B 上停用着，没有冲突；A 上照旧 +/// 启用、登录还在。B 登录之后照同步的样子启用 +#[tokio::test] +async fn an_upstream_signed_in_elsewhere_waits_for_a_sign_in_here() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider_raw(&add_provider_raw(s, CHATGPT), CORP)); + t.ea.round().await.unwrap(); + let r = t.eb.round().await.unwrap(); + assert!(r.conflicts.is_empty(), "{:?}", r.conflicts); + for n in ["chatgpt", "corp"] { + let p = t.b.provider(n).unwrap_or_else(|| panic!("{n} 没到 B")); + assert!(p.disabled, "{n} 在 B 上停用着"); + } + assert!(t.b.provider("chatgpt").unwrap().oauth.is_none()); + assert_eq!( + t.eb.state().held_login, + BTreeSet::from(["chatgpt".to_string(), "corp".to_string()]) + ); + // 停用没有传回 A，也不会来回上传 + let listing = versions(t.loc.path()); + let ra = t.ea.round().await.unwrap(); + assert!(ra.uploaded.is_none() && ra.applied_from.is_empty()); + assert!(t.eb.round().await.unwrap().uploaded.is_none()); + assert_eq!(versions(t.loc.path()), listing); + let a = t.a.provider("chatgpt").unwrap(); + assert!(!a.disabled); + assert_eq!(a.oauth.unwrap().refresh, "rt-a"); + + // A 改了并发：B 收到，还停用着 + t.a.edit(|s| { + s.replace( + " base_url: https://corp.example\n", + " base_url: https://corp.example\n max_concurrent: 3\n", + ) + }); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + let corp = t.b.provider("corp").unwrap(); + assert_eq!(corp.max_concurrent, Some(3)); + assert!(corp.disabled); + + // B 上登录 corp（填上刷新令牌）：照同步的样子启用，A 不受影响 + t.b.edit(|s| { + let mut cfg: serde_yaml_ng::Value = serde_yaml_ng::from_str(s).unwrap(); + for p in cfg["providers"].as_sequence_mut().unwrap() { + if p["name"] == "corp" { + p["oauth"]["refresh"] = "rt-corp-b".into(); + } + } + serde_yaml_ng::to_string(&cfg).unwrap() + }); + let r = t.eb.round().await.unwrap(); + assert!(r.uploaded.is_none(), "登录是这台的事，不上传"); + let corp = t.b.provider("corp").unwrap(); + assert!(!corp.disabled, "登录之后启用"); + assert_eq!(corp.oauth.unwrap().refresh, "rt-corp-b"); + assert_eq!( + t.eb.state().held_login, + BTreeSet::from(["chatgpt".to_string()]) + ); + t.ea.round().await.unwrap(); + assert_eq!( + t.a.provider("corp").unwrap().oauth.unwrap().refresh, + "rt-corp-a" + ); +} + +/// B 上自己改了等登录上游的别的设置：照常上传，停用不跟着走 +#[tokio::test] +async fn editing_a_waiting_upstream_here_does_not_turn_it_off_elsewhere() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider_raw(s, CHATGPT)); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + assert!(t.b.provider("chatgpt").unwrap().disabled); + t.b.edit(|s| { + s.replace( + " base_url: https://chatgpt.com/backend-api/codex\n", + " base_url: https://chatgpt.com/backend-api/codex\n max_concurrent: 2\n", + ) + }); + assert!(t.eb.round().await.unwrap().uploaded.is_some()); + t.ea.round().await.unwrap(); + let a = t.a.provider("chatgpt").unwrap(); + assert_eq!(a.max_concurrent, Some(2)); + assert!(!a.disabled, "B 上的停用没传过来"); + assert!(a.oauth.is_some()); +} + +/// 两台都登录了同名的 ChatGPT 上游（各自的账号）：A 改并发，B 收到后登录还是 B 自己的 +#[tokio::test] +async fn two_devices_signed_in_to_one_chatgpt_upstream_keep_their_own_login() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider_raw(s, CHATGPT)); + t.b.edit(|s| { + add_provider_raw( + s, + &CHATGPT.replace("rt-a", "rt-b").replace("acct-a", "acct-b"), + ) + }); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + t.a.edit(|s| { + s.replace( + " base_url: https://chatgpt.com/backend-api/codex\n", + " base_url: https://chatgpt.com/backend-api/codex\n max_concurrent: 4\n", + ) + }); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + let p = t.b.provider("chatgpt").unwrap(); + assert_eq!(p.max_concurrent, Some(4)); + assert!(!p.disabled); + assert_eq!(p.oauth.unwrap().refresh, "rt-b"); +} + +// ─────────────────────────────────────────────── 冲突 + +/// 网关密钥的冲突选「两者都保留」：另一份是一把新的密钥，不是同一个值，也不归哪个客户端 +#[tokio::test] +async fn keeping_both_gateway_keys_makes_the_copy_a_new_key() { + let mut t = bed(false).await; + t.a.edit(|s| { + s.replace( + "clients:\n", + "clients:\n - name: k2\n key: tw-k2-value-0001\n client: codex\n", + ) + }); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + t.a.edit(|s| { + s.replace( + " client: codex\n", + " client: codex\n max_concurrent: 2\n", + ) + }); + t.b.edit(|s| { + s.replace( + " client: codex\n", + " client: codex\n max_concurrent: 3\n", + ) + }); + t.ea.round().await.unwrap(); + let rb = t.eb.round().await.unwrap(); + let c = rb.conflicts.iter().find(|c| c.item.name == "k2").unwrap(); + assert_eq!( + tw_sync::engine::options(c), + [Take::Local, Take::Remote, Take::Both] + ); + let r = + t.eb.round_with(&[Decision { + id: c.id.clone(), + take: Take::Both, + new_name: None, + }]) + .await + .unwrap(); + assert!(r.conflicts.is_empty()); + let clients = t.b.config().clients; + let orig = clients.iter().find(|c| c.name == "k2").unwrap(); + let copy = clients + .iter() + .find(|c| c.name.starts_with("k2 (")) + .unwrap_or_else(|| panic!("{clients:?}")); + assert_eq!(orig.key, "tw-k2-value-0001"); + assert_ne!(copy.key, orig.key, "另一份是一把新的密钥"); + assert!(copy.key.starts_with("tw-")); + assert_eq!(copy.max_concurrent, Some(2), "另一份是另一边的设置"); + assert!(copy.client.is_none(), "新的那把不归哪个客户端"); +} + +/// 合并结果过不了校验的冲突（`invalid`）只能选本机的：选另一边是一个明确的错误 +#[tokio::test] +async fn an_invalid_conflict_only_offers_this_side() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider(s, "old", "https://old.example", "sk-old")); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + t.a.edit(|s| { + s.replace( + " - name: old\n base_url: https://old.example\n key: sk-old\n", + "", + ) + .replace("providers:\n", "") + }); + t.b.edit(|s| { + format!("{s}routes:\n - name: viaold\n rules:\n - name: r\n to: old\n") + }); + t.eb.round().await.unwrap(); + let ra = t.ea.round().await.unwrap(); + let c = ra + .conflicts + .iter() + .find(|c| c.reason == Some(tw_sync::record::Reason::Invalid)) + .unwrap_or_else(|| panic!("{:?}", ra.conflicts)); + assert_eq!(tw_sync::engine::options(c), [Take::Local]); + let r = + t.ea.round_with(&[Decision { + id: c.id.clone(), + take: Take::Remote, + new_name: None, + }]) + .await; + assert!( + matches!(&r, Err(RoundError::Local(m)) if m.code == "sync.take_not_allowed"), + "{:?}", + r.map(|r| r.conflicts.len()) + ); + let r = + t.ea.round_with(&[Decision { + id: c.id.clone(), + take: Take::Local, + new_name: None, + }]) + .await + .unwrap(); + assert!(r.conflicts.is_empty()); +} + +/// 全局设置：一边把一节改回默认（这一项从快照里消失），另一边改了这一节的另一个字段 —— +/// 各管各的字段，不是冲突 +#[tokio::test] +async fn a_global_section_reset_to_defaults_merges_field_by_field() { + let mut t = bed(false).await; + t.a.edit(|s| format!("{s}failover:\n pause_secs: 100\n")); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + assert_eq!(t.b.config().failover.pause_secs, 100); + let default_pause = tw_config::Config::default().failover.pause_secs; + t.a.edit(|s| s.replace("failover:\n pause_secs: 100\n", "")); + t.b.edit(|s| { + s.replace( + "failover:\n pause_secs: 100\n", + "failover:\n pause_secs: 100\n failures_to_pause: 7\n", + ) + }); + t.ea.round().await.unwrap(); + let rb = t.eb.round().await.unwrap(); + assert!(rb.conflicts.is_empty(), "{:?}", rb.conflicts); + let f = t.b.config().failover; + assert_eq!((f.pause_secs, f.failures_to_pause), (default_pause, 7)); + t.ea.round().await.unwrap(); + let f = t.a.config().failover; + assert_eq!((f.pause_secs, f.failures_to_pause), (default_pause, 7)); +} + +/// 合并结果过不了校验时逐项回退：另一边新增的「上游 + 它用的出站代理」，出站代理先到位，上游 +/// 不算过不了校验 +#[tokio::test] +async fn a_new_upstream_with_its_new_proxy_is_not_taken_for_invalid() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider(s, "old", "https://old.example", "sk-old")); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + t.a.edit(|s| { + s.replace( + " - name: old\n base_url: https://old.example\n key: sk-old\n", + "", + ) + .replace("providers:\n", "") + }); + t.b.edit(|s| { + let s = add_provider_raw( + s, + " - name: q\n base_url: https://q.example\n key: sk-q\n proxy: hk\n", + ); + format!( + "{s}proxies:\n - name: hk\n type: socks5h\n addr: 127.0.0.1:7890\nroutes:\n - name: viaold\n rules:\n - name: r\n to: old\n" + ) + }); + t.eb.round().await.unwrap(); + let ra = t.ea.round().await.unwrap(); + assert!( + !ra.conflicts + .iter() + .any(|c| c.item.name == "q" || c.item.name == "hk"), + "{:?}", + ra.conflicts + ); + assert!(t.a.provider("q").is_some()); +} + +// ─────────────────────────────────────────────── 谱系 + +/// 云盘晚到：B 依据 v3 传了一个版本，A 那边还没看到时又传了两个、删掉了 v3。B 的版本到了 +/// 之后，两个头共同的上一版本只在 A 之前的基准里：照样三方合并，两边改同一个上游的不同单位 +/// 不是冲突 +#[tokio::test] +async fn a_late_head_merges_three_way_after_its_parent_was_pruned() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider(s, "relay", "https://relay.example", "sk-r")); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + let hidden: Arc>> = Default::default(); + let mut ea = Engine::new( + Lagging { + inner: folder(t.loc.path()), + hidden: hidden.clone(), + }, + t.a.clone(), + t.a.dir.path(), + settings(A_ID, "Mac mini", false, t.loc.path()), + ); + t.b.edit(|s| s.replace(" key: sk-r\n", " key: sk-r\n max_concurrent: 3\n")); + let up_b = t.eb.round().await.unwrap().uploaded.unwrap(); + hidden.lock().unwrap().insert(format!("{up_b}.twconfig")); + t.a.edit(|s| s.replace(" key: sk-r\n", " key: sk-r\n models: [m1]\n")); + ea.round().await.unwrap(); + t.a.edit(|s| s.replace(" models: [m1]\n", " models: [m1, m2]\n")); + ea.round().await.unwrap(); + hidden.lock().unwrap().clear(); + let r = ea.round().await.unwrap(); + assert!(r.conflicts.is_empty(), "{:?}", r.conflicts); + let p = t.a.provider("relay").unwrap(); + assert_eq!(p.max_concurrent, Some(3)); +} + +/// 不止一个头时，删旧版本留着它们共同的上一版本 +#[tokio::test] +async fn pruning_keeps_the_common_parent_of_two_heads() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider(s, "relay", "https://relay.example", "sk-r")); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + let parent = versions(t.loc.path()).last().unwrap().clone(); + // 两台同时上传：两个头，依据同一个版本 + t.a.edit(|s| s.replace(" key: sk-r\n", " key: sk-r\n models: [m1]\n")); + t.b.edit(|s| s.replace(" key: sk-r\n", " key: sk-r\n max_concurrent: 3\n")); + let hidden: Arc>> = Default::default(); + let mut eb = Engine::new( + Lagging { + inner: folder(t.loc.path()), + hidden: hidden.clone(), + }, + t.b.clone(), + t.b.dir.path(), + settings(B_ID, "MacBook Air", false, t.loc.path()), + ); + let up_a = t.ea.round().await.unwrap().uploaded.unwrap(); + hidden.lock().unwrap().insert(format!("{up_a}.twconfig")); + eb.round().await.unwrap(); + hidden.lock().unwrap().clear(); + assert!(versions(t.loc.path()).contains(&parent)); + // A 合并两个头之后：共同的上一版本还在，A 的合并没有冲突 + t.a.edit(|s| s.replace(" models: [m1]\n", " models: [m1, m2]\n")); + let r = t.ea.round().await.unwrap(); + assert!(r.conflicts.is_empty(), "{:?}", r.conflicts); + assert_eq!(t.a.provider("relay").unwrap().max_concurrent, Some(3)); +} + +/// 云盘上两台几乎同时开启加密同步：B 那一刻还没收到 A 的 keyring 和第一个版本，于是新建， +/// 覆盖了 A 的 keyring。A 自己的版本这时解不开了：A 照两方比较合并、以新的 keyring 重新上传， +/// 删掉自己那个解不开的版本 —— 两台的配置都在 +#[tokio::test] +async fn a_keyring_race_on_a_cloud_folder_loses_nothing() { + tw_sync::keyring::set_work_factor_for_tests(4); + let loc = tempfile::tempdir().unwrap(); + let (a, b) = (Dev::new('a'), Dev::new('b')); + a.edit(|s| add_provider(s, "one", "https://one.example", "sk-1")); + b.edit(|s| add_provider(s, "two", "https://two.example", "sk-2")); + let hidden: Arc>> = Default::default(); + let mut ea = Engine::new( + folder(loc.path()), + a.clone(), + a.dir.path(), + settings(A_ID, "Mac mini", true, loc.path()), + ); + let mut eb = Engine::new( + Racy { + inner: folder(loc.path()), + hidden: hidden.clone(), + }, + b.clone(), + b.dir.path(), + settings(B_ID, "MacBook Air", true, loc.path()), + ); + assert!(ea.create().await.unwrap()); + hidden.lock().unwrap().extend([ + "keyring".to_string(), + "0000000001-aaaaaaaa.twconfig".to_string(), + ]); + assert!(eb.create().await.unwrap(), "B 没看到 A 的，自己新建"); + hidden.lock().unwrap().clear(); + let r = ea.round().await.unwrap(); + assert!(r.uploaded.is_some()); + assert!(a.provider("one").is_some(), "A 自己的上游还在"); + assert!(a.provider("two").is_some()); + assert!( + !versions(loc.path()).contains(&"0000000001-aaaaaaaa.twconfig".to_string()), + "解不开的那个自己的版本删了" + ); + eb.round().await.unwrap(); + assert!(b.provider("one").is_some()); + assert!(b.provider("two").is_some()); +} + +/// 两台几乎同时开启不加密的同步：两个不依据别的版本的第一个版本，按两方比较合并 +#[tokio::test] +async fn two_first_versions_are_merged_two_way() { + let loc = tempfile::tempdir().unwrap(); + let (a, b) = (Dev::new('a'), Dev::new('b')); + a.edit(|s| add_provider(s, "one", "https://one.example", "sk-1")); + b.edit(|s| add_provider(s, "two", "https://two.example", "sk-2")); + let hidden: Arc>> = Default::default(); + let mut ea = Engine::new( + folder(loc.path()), + a.clone(), + a.dir.path(), + settings(A_ID, "Mac mini", false, loc.path()), + ); + let mut eb = Engine::new( + Lagging { + inner: folder(loc.path()), + hidden: hidden.clone(), + }, + b.clone(), + b.dir.path(), + settings(B_ID, "MacBook Air", false, loc.path()), + ); + assert!(ea.create().await.unwrap()); + hidden + .lock() + .unwrap() + .insert("0000000001-aaaaaaaa.twconfig".to_string()); + assert!(eb.create().await.unwrap()); + hidden.lock().unwrap().clear(); + let r = ea.round().await.unwrap(); + assert!(r.conflicts.is_empty(), "{:?}", r.conflicts); + assert!(a.provider("two").is_some()); + eb.round().await.unwrap(); + assert!(b.provider("one").is_some()); +} + +/// 开启加密时 keyring 建好了、第一个版本没传上去：开启（口令对）接着传，加入也一样 +#[tokio::test] +async fn a_location_with_only_a_keyring_can_be_turned_on() { + tw_sync::keyring::set_work_factor_for_tests(4); + for join in [false, true] { + let loc = tempfile::tempdir().unwrap(); + let a = Dev::new('a'); + a.edit(|s| add_provider(s, "one", "https://one.example", "sk-1")); + folder(loc.path()) + .create_exclusive("keyring", &Keyring::generate(0).seal(PASS)) + .await + .unwrap(); + let mut ea = Engine::new( + folder(loc.path()), + a.clone(), + a.dir.path(), + settings(A_ID, "Mac mini", true, loc.path()), + ); + if join { + assert!(ea.join(None).await.unwrap().encrypted); + } else { + assert!(ea.create().await.unwrap()); + } + assert_eq!(versions(loc.path()).len(), 1); + let (b, bdir) = (Dev::new('b'), tempfile::tempdir().unwrap()); + let mut eb = Engine::new( + folder(loc.path()), + b.clone(), + bdir.path(), + settings(B_ID, "MacBook Air", true, loc.path()), + ); + eb.join(None).await.unwrap(); + assert!(b.provider("one").is_some()); + } +} + +// ─────────────────────────────────────────────── 读不出来的文件 + +/// 最新一层有一个坏文件（不是配置包）：马上跳过，计数；本机改动照常上传，别的设备照常收到 +#[tokio::test] +async fn a_broken_file_on_top_does_not_stop_sync() { + let mut t = bed(true).await; + std::fs::write( + t.loc.path().join("versions/9999999999-cccccccc.twconfig"), + b"garbage", + ) + .unwrap(); + t.a.edit(|s| add_provider(s, "one", "https://one.example", "sk-1")); + let r = t.ea.round().await.unwrap(); + assert!(r.uploaded.is_some()); + assert_eq!(r.skipped, 1); + let r = t.eb.round().await.unwrap(); + assert_eq!(r.skipped, 1); + assert!(t.b.provider("one").is_some()); +} + +/// 最新一层有一个看上去没传完的版本：等着（不合并一半）；等了 30 分钟还读不出来就跳过 +#[tokio::test] +async fn a_version_that_never_finishes_arriving_is_skipped_after_a_while() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider(s, "one", "https://one.example", "sk-1")); + let up = t.ea.round().await.unwrap().uploaded.unwrap(); + // B 那边：最新的那个只到了一半 + let bytes = read_version(t.loc.path(), &up); + let parked = tempfile::tempdir().unwrap(); + std::fs::write(parked.path().join("whole"), &bytes).unwrap(); + let seq: u64 = up[..10].parse().unwrap(); + let half = format!("{:010}-cccccccc.twconfig", seq + 1); + std::fs::write( + t.loc.path().join("versions").join(&half), + &bytes[..bytes.len() / 2], + ) + .unwrap(); + assert!(matches!(t.eb.round().await, Err(RoundError::NotArrived))); + assert!(t.b.provider("one").is_none(), "不拿一半去合并"); + age_state(&t.eb, SKIP_AFTER_MS + 1000); + let r = t.eb.round().await.unwrap(); + assert_eq!(r.skipped, 1); + assert!(t.b.provider("one").is_some()); +} + +/// iCloud 还没下载的最新版本（只有占位文件）：当作尚未到达 +#[tokio::test] +async fn an_icloud_placeholder_on_top_is_waited_for() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider(s, "one", "https://one.example", "sk-1")); + let up = t.ea.round().await.unwrap().uploaded.unwrap(); + let file = format!("{up}.twconfig"); + let real = t.loc.path().join("versions").join(&file); + let parked = tempfile::tempdir().unwrap(); + std::fs::rename(&real, parked.path().join(&file)).unwrap(); + std::fs::write( + t.loc + .path() + .join("versions") + .join(format!(".{file}.icloud")), + b"placeholder", + ) + .unwrap(); + assert!(matches!(t.eb.round().await, Err(RoundError::NotArrived))); + std::fs::rename(parked.path().join(&file), &real).unwrap(); + t.eb.round().await.unwrap(); + assert!(t.b.provider("one").is_some()); +} + +// ─────────────────────────────────────────────── 空的同步位置 + +/// 文件夹位置的根目录还在、里面暂时是空的（网络盘没挂上时挂载点是个空目录）：不当作清空， +/// 挂回来之后照常同步 +#[tokio::test] +async fn an_empty_mount_point_is_not_taken_for_a_purge() { + let mut t = bed(false).await; + let parked = tempfile::tempdir().unwrap(); + for name in ["versions", "devices"] { + std::fs::rename(t.loc.path().join(name), parked.path().join(name)).unwrap(); + } + for _ in 0..5 { + assert!(matches!(t.ea.round().await, Err(RoundError::LooksEmpty))); + } + // 过了很久也一样：自己的设备记录不在只是没挂上的一种样子，结论要人下 + age_state(&t.ea, 60 * 60 * 1000); + assert!(matches!(t.ea.round().await, Err(RoundError::Emptied))); + assert!(t.ea.local().base().is_some(), "本机的同步状态不删"); + for name in ["versions", "devices"] { + std::fs::rename(parked.path().join(name), t.loc.path().join(name)).unwrap(); + } + t.a.edit(|s| add_provider(s, "back", "https://back.example", "sk-b")); + assert!(t.ea.round().await.unwrap().uploaded.is_some()); + t.eb.round().await.unwrap(); + assert!(t.b.provider("back").is_some()); +} + +/// 停止同步并删除同步数据：只删认得出的同步文件，用户自己放进去的不动 +#[tokio::test] +async fn purging_leaves_files_that_are_not_sync_data() { + let mut t = bed(false).await; + std::fs::write(t.loc.path().join("devices/notes.txt"), b"mine").unwrap(); + std::fs::write(t.loc.path().join("versions/readme.md"), b"mine").unwrap(); + t.ea.stop(true).await.unwrap(); + assert!(versions(t.loc.path()).iter().all(|n| n == "readme.md")); + assert!(t.loc.path().join("devices/notes.txt").exists()); + assert!(t.loc.path().join("versions/readme.md").exists()); + assert!(!t.loc.path().join("devices/bbbbbbbb").exists()); +} + +// ─────────────────────────────────────────────── 测试用的存储 + +/// 云盘晚到：藏起来的文件列不出、读不到 +struct Lagging { + inner: AnyStore, + hidden: Arc>>, +} + +impl Lagging { + fn hides(&self, path: &str) -> bool { + self.hidden + .lock() + .unwrap() + .iter() + .any(|h| path.ends_with(h.as_str())) + } +} + +impl Store for Lagging { + async fn list(&self, dir: &str) -> Result, StoreError> { + let hidden = self.hidden.lock().unwrap().clone(); + Ok(self + .inner + .list(dir) + .await? + .into_iter() + .filter(|e| !hidden.contains(&e.name)) + .collect()) + } + async fn get(&self, path: &str) -> Result>, StoreError> { + if self.hides(path) { + return Ok(None); + } + self.inner.get(path).await + } + async fn put_new(&self, path: &str, bytes: &[u8]) -> Result<(), StoreError> { + self.inner.put_new(path, bytes).await + } + async fn put(&self, path: &str, bytes: &[u8]) -> Result<(), StoreError> { + self.inner.put(path, bytes).await + } + async fn delete(&self, path: &str) -> Result<(), StoreError> { + self.inner.delete(path).await + } + async fn create_exclusive(&self, path: &str, bytes: &[u8]) -> Result { + self.inner.create_exclusive(path, bytes).await + } +} + +/// 云盘上的「仅当不存在时创建」管不了别的设备：藏起来的文件当作不存在，照写不误 +struct Racy { + inner: AnyStore, + hidden: Arc>>, +} + +impl Racy { + fn hides(&self, path: &str) -> bool { + self.hidden + .lock() + .unwrap() + .iter() + .any(|h| path.ends_with(h.as_str())) + } +} + +impl Store for Racy { + async fn list(&self, dir: &str) -> Result, StoreError> { + let hidden = self.hidden.lock().unwrap().clone(); + Ok(self + .inner + .list(dir) + .await? + .into_iter() + .filter(|e| !hidden.contains(&e.name)) + .collect()) + } + async fn get(&self, path: &str) -> Result>, StoreError> { + if self.hides(path) { + return Ok(None); + } + self.inner.get(path).await + } + async fn put_new(&self, path: &str, bytes: &[u8]) -> Result<(), StoreError> { + self.inner.put_new(path, bytes).await + } + async fn put(&self, path: &str, bytes: &[u8]) -> Result<(), StoreError> { + self.inner.put(path, bytes).await + } + async fn delete(&self, path: &str) -> Result<(), StoreError> { + self.inner.delete(path).await + } + async fn create_exclusive(&self, path: &str, bytes: &[u8]) -> Result { + if self.hides(path) { + self.inner.put(path, bytes).await?; + return Ok(true); + } + self.inner.create_exclusive(path, bytes).await + } +} diff --git a/crates/tw-sync/tests/stores.rs b/crates/tw-sync/tests/stores.rs index a26be7a4..cd86f9f9 100644 --- a/crates/tw-sync/tests/stores.rs +++ b/crates/tw-sync/tests/stores.rs @@ -117,6 +117,10 @@ async fn webdav_without_move_or_conditional_put_still_never_overwrites_a_version } fn s3_store(fake: &common::Fake, ak: &str) -> S3 { + s3_store_with(fake, ak, common::SK) +} + +fn s3_store_with(fake: &common::Fake, ak: &str, sk: &str) -> S3 { S3::new( store::s3::Config { endpoint: fake.url(), @@ -124,13 +128,119 @@ fn s3_store(fake: &common::Fake, ak: &str) -> S3 { bucket: "tw-bucket".into(), prefix: "team/tw".into(), access_key_id: ak.into(), - secret_access_key: common::SK.into(), + secret_access_key: sk.into(), }, common::http(), ) .unwrap() } +/// 假 S3 核对签名的那一份按 AWS 文档里的两个示例对得上（GET Object、ListObjects 带查询参数）： +/// 它核对得了，`S3` 的签名在它那里过得去才算数 +#[test] +fn the_fake_s3_checks_signatures_the_way_aws_does() { + use axum::http::{HeaderMap, HeaderValue}; + let secret = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; + let empty = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + let req = |auth: &str, range: bool| { + let mut h = HeaderMap::new(); + h.insert( + "host", + HeaderValue::from_static("examplebucket.s3.amazonaws.com"), + ); + if range { + h.insert("range", HeaderValue::from_static("bytes=0-9")); + } + h.insert("x-amz-content-sha256", HeaderValue::from_static(empty)); + h.insert("x-amz-date", HeaderValue::from_static("20130524T000000Z")); + h.insert("authorization", HeaderValue::from_str(auth).unwrap()); + h + }; + let get = req( + "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,\ + SignedHeaders=host;range;x-amz-content-sha256;x-amz-date,\ + Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41", + true, + ); + common::sigv4( + "GET", + "/test.txt", + "", + &get, + b"", + "AKIAIOSFODNN7EXAMPLE", + secret, + ) + .unwrap(); + let list = req( + "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,\ + SignedHeaders=host;x-amz-content-sha256;x-amz-date,\ + Signature=34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7", + false, + ); + common::sigv4( + "GET", + "/", + "max-keys=2&prefix=J", + &list, + b"", + "AKIAIOSFODNN7EXAMPLE", + secret, + ) + .unwrap(); + // 改一点就不认 + assert!( + common::sigv4( + "GET", + "/", + "max-keys=3&prefix=J", + &list, + b"", + "AKIAIOSFODNN7EXAMPLE", + secret + ) + .is_err() + ); + assert!( + common::sigv4( + "GET", + "/test.txt", + "", + &get, + b"", + "AKIAIOSFODNN7EXAMPLE", + "other" + ) + .is_err() + ); + assert!( + common::sigv4( + "GET", + "/test.txt", + "", + &get, + b"body", + "AKIAIOSFODNN7EXAMPLE", + secret + ) + .is_err() + ); +} + +/// 密钥 ID 对、密钥不对：签名对不上，拒绝访问 +#[tokio::test] +async fn s3_with_the_wrong_secret_is_refused() { + let fake = common::s3().await; + assert!(matches!( + s3_store_with(&fake, common::AK, "not-the-secret") + .list("versions") + .await + .unwrap_err(), + StoreError::AuthFailed(_) + )); + assert!(fake.names().is_empty()); +} + #[tokio::test] async fn s3_answers_like_the_others() { let fake = common::s3().await; @@ -196,3 +306,62 @@ async fn wrong_credentials_are_a_refusal_and_a_dead_server_is_unreachable() { StoreError::Unreachable(_) )); } + +/// 服务端关掉了连接池里那条空闲连接（Apache 空闲 5 秒就关）：发出去的请求收到「连接在答复 +/// 之前关了」，换一条连接再发一次就好。这里的服务端每条连接只答第一个请求，第二个请求一到 +/// 就关连接 —— 没有重发的话第二次读一定失败 +#[tokio::test] +async fn a_pooled_connection_the_server_closed_is_retried_once() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = l.local_addr().unwrap(); + let conns = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let n = conns.clone(); + tokio::spawn(async move { + loop { + let Ok((mut sock, _)) = l.accept().await else { + return; + }; + n.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + tokio::spawn(async move { + let mut buf = Vec::new(); + let mut served = 0; + loop { + let mut chunk = [0u8; 4096]; + let Ok(got) = sock.read(&mut chunk).await else { + return; + }; + if got == 0 { + return; + } + buf.extend_from_slice(&chunk[..got]); + while let Some(end) = buf.windows(4).position(|w| w == b"\r\n\r\n") { + buf.drain(..end + 4); + if served == 1 { + // 第二个请求：一个字也不答就关 + return; + } + served += 1; + let _ = sock + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhello") + .await; + } + } + }); + } + }); + let s = WebDav::new( + &format!("http://{addr}/dav/root"), + common::USER, + common::PASS, + common::http(), + ) + .unwrap(); + for _ in 0..3 { + assert_eq!(s.get(V1).await.unwrap().as_deref(), Some(&b"hello"[..])); + } + assert!( + conns.load(std::sync::atomic::Ordering::Relaxed) >= 3, + "每次都换了一条连接" + ); +} diff --git a/docs/config.md b/docs/config.md index 5ac3653a..63b3b118 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1518,7 +1518,15 @@ resolve conflicts in the app (Settings, Sync and backup) or with What syncs is the configuration a configuration pack carries: upstreams, proxies, gateway keys, aliases, groups, routes, price sheets, security rules, plugins and the general settings, keys included. `listen`, `retention` and -`sync` itself stay on this device, and so does a ChatGPT account's sign-in. +`sync` itself stay on this device, and so do a ChatGPT account's sign-in and +OAuth tokens. An upstream signed in on another device stays turned off on this +one until it is signed in here; that does not turn it off anywhere else. + +With encryption on, versions that are not encrypted are never read. A keyring +that went missing, or an older one put back, does not turn encryption off: the +app asks whether to put back the copy this device keeps. A location that reads +as empty is checked a few more times before the app says so, and turning sync +off is still up to you. @@ -1528,7 +1536,7 @@ plugins and the general settings, keys included. `listen`, `retention` and | `device_id` | string | **required** | Generated when sync is turned on. File names at the sync location show its first 8 characters. | | `device_name` | string | **required** | Shown in the device list of every device that syncs here. | | `encrypt` | bool | `true` | Encrypt what goes to the sync location with the sync passphrase. Every device at one location uses the same setting; change it in the app, which re-uploads the latest version. | -| `passphrase` | string | — | The sync passphrase; required when `encrypt` is on. Kept here in the clear, like the other keys in this file. | +| `passphrase` | string | — | The sync passphrase; required when `encrypt` is on, and kept after encryption is turned off: the keyring that says it is off is checked with it. Kept here in the clear, like the other keys in this file. | | `exclude` | list of strings | `[]` | Upstreams, by name, that are not synced: they are not uploaded and another device's upstream of the same name does not replace them. Routes, aliases and groups that refer to them are not synced either. | | `target` | object, [`sync.target`](#cfg-sync-target) | **required** | The sync location. The keyring, `versions/` and `devices/` live directly in it. | @@ -1560,7 +1568,7 @@ plugins and the general settings, keys included. `listen`, `retention` and | `url` | string | **required** | The WebDAV address of the folder. `https://`, with the certificate checked; `http://` only for this machine and the local network. | | `user` | string | `''` | User name. | | `pass` | string | `''` | Password, or the app password the service issues for WebDAV. | -| `proxy` | string | `direct` | Where requests to the sync location go out: `direct`, `system` (the system proxy), or the name of a proxy under `proxies`. | +| `proxy` | string | `direct` | Where requests to the sync location go out: `direct`, `system` (the system proxy), or the name of a proxy under `proxies`. An `http://` address on this machine or the local network has to be `direct`. | @@ -1572,10 +1580,10 @@ plugins and the general settings, keys included. `listen`, `retention` and | `endpoint` | string | **required** | The S3 endpoint. Hosts ending in `amazonaws.com`, `aliyuncs.com` or `myqcloud.com` put the bucket in the host name; every other endpoint (Cloudflare R2, MinIO, an address) puts it in the path. `http://` only for this machine and the local network. | | `region` | string | **required** | The signing region; `auto` for Cloudflare R2. | | `bucket` | string | **required** | The bucket. Keep it private. | -| `prefix` | string | `''` | A folder inside the bucket. | +| `prefix` | string | `''` | A folder inside the bucket. No `..` parts, and not starting with `/`. | | `access_key_id` | string | **required** | Access key ID. | | `secret_access_key` | string | **required** | Secret access key. | -| `proxy` | string | `direct` | Where requests to the sync location go out: `direct`, `system` (the system proxy), or the name of a proxy under `proxies`. | +| `proxy` | string | `direct` | Where requests to the sync location go out: `direct`, `system` (the system proxy), or the name of a proxy under `proxies`. An `http://` address on this machine or the local network has to be `direct`. | ```yaml diff --git a/docs/config.zh-CN.md b/docs/config.zh-CN.md index 9a471081..930fa9c0 100644 --- a/docs/config.zh-CN.md +++ b/docs/config.zh-CN.md @@ -1105,7 +1105,9 @@ export const manifest = { 多台设备（这台 Mac、一台笔记本、一台运行远程 core 的服务器）可以通过自选的同步位置保持同一份配置：由云盘在设备之间传送的文件夹、WebDAV 服务器或 S3 兼容存储。每次改动都按对象与其他设备自上次同步以来的改动合并；只有两台设备把同一对象的同一部分改成不同内容时才构成冲突，冲突在每台设备上显示，任一设备处理之后随之清除。开启同步、加入已有的同步和处理冲突在应用里进行（设置 → 同步与备份），服务器上用 `twcore sync enable`；本节记录的是结果。 -同步的内容即配置包带的内容：上游、出站代理、网关密钥、别名、分组、路由、价目表、安全规则、插件和全局设置，包括密钥。`listen`、`retention` 和 `sync` 本身留在本机，ChatGPT 账号的登录也一样。 +同步的内容即配置包带的内容：上游、出站代理、网关密钥、别名、分组、路由、价目表、安全规则、插件和全局设置，包括密钥。`listen`、`retention` 和 `sync` 本身留在本机，ChatGPT 账号的登录、OAuth 的令牌也一样。别的设备上登录着的上游在这台设备上停用着，在这台登录之后才启用；这不会让它在别的设备上停用。 + +开着加密时，不加密的版本一概不读。keyring 不见了、被换成更早的那一份，都不会关掉加密：应用会问要不要放回这台设备记着的那一份。同步位置读起来是空的时，要再看几次才提示，停不停止同步由你决定。 @@ -1115,7 +1117,7 @@ export const manifest = { | `device_id` | 字符串 | **必填** | 开启同步时生成。同步位置上的文件名里出现的是它的前 8 个字符。 | | `device_name` | 字符串 | **必填** | 在每台参与同步的设备的设备列表里显示的名字。 | | `encrypt` | 布尔 | `true` | 以同步口令加密存到同步位置的内容。同一同步位置上的所有设备采用同一设置；在应用里更改，应用会重新上传最新版本。 | -| `passphrase` | 字符串 | — | 同步口令，`encrypt` 开着时必填。和本文件里的其他密钥一样以明文保存。 | +| `passphrase` | 字符串 | — | 同步口令，`encrypt` 开着时必填；关闭加密之后也留着：同步位置上写着「已关闭」的 keyring 要用它核对。和本文件里的其他密钥一样以明文保存。 | | `exclude` | 字符串列表 | `[]` | 不同步的上游，按名字：不上传，也不被其他设备的同名上游覆盖。引用它们的路由、别名、分组随之不同步。 | | `target` | 对象，见 [`sync.target`](#cfg-sync-target) | **必填** | 同步位置。`keyring`、`versions/`、`devices/` 直接放在它里面。 | @@ -1147,7 +1149,7 @@ export const manifest = { | `url` | 字符串 | **必填** | 文件夹的 WebDAV 地址。`https://`，校验证书；`http://` 只用于本机和局域网。 | | `user` | 字符串 | `''` | 用户名。 | | `pass` | 字符串 | `''` | 密码，或者服务为 WebDAV 签发的应用密码。 | -| `proxy` | 字符串 | `direct` | 访问同步位置的请求从哪里出去：`direct`（直连）、`system`（跟随系统代理），或者 `proxies` 里一个代理的名字。 | +| `proxy` | 字符串 | `direct` | 访问同步位置的请求从哪里出去：`direct`（直连）、`system`（跟随系统代理），或者 `proxies` 里一个代理的名字。本机、内网的 `http://` 地址只能是 `direct`。 | @@ -1159,10 +1161,10 @@ export const manifest = { | `endpoint` | 字符串 | **必填** | S3 端点。主机以 `amazonaws.com`、`aliyuncs.com`、`myqcloud.com` 结尾的，存储桶名放进域名；其余端点（Cloudflare R2、MinIO、IP 地址）放进路径。`http://` 只用于本机和局域网。 | | `region` | 字符串 | **必填** | 签名用的区域；Cloudflare R2 填 `auto`。 | | `bucket` | 字符串 | **必填** | 存储桶，建议设为私有。 | -| `prefix` | 字符串 | `''` | 存储桶里的一个前缀（文件夹）。 | +| `prefix` | 字符串 | `''` | 存储桶里的一个前缀（文件夹）。不能有 `..`，不以 `/` 开头。 | | `access_key_id` | 字符串 | **必填** | Access Key ID。 | | `secret_access_key` | 字符串 | **必填** | Secret Access Key。 | -| `proxy` | 字符串 | `direct` | 访问同步位置的请求从哪里出去：`direct`（直连）、`system`（跟随系统代理），或者 `proxies` 里一个代理的名字。 | +| `proxy` | 字符串 | `direct` | 访问同步位置的请求从哪里出去：`direct`（直连）、`system`（跟随系统代理），或者 `proxies` 里一个代理的名字。本机、内网的 `http://` 地址只能是 `direct`。 | ```yaml