From a334fd5f69c117759e390325f27e8721c96a07fe Mon Sep 17 00:00:00 2001 From: fylorn <249551762+fylorn@users.noreply.github.com> Date: Mon, 19 Oct 2026 01:26:36 +0800 Subject: [PATCH 01/13] tw-pack: configuration pack format and the item model (#348) * tw-pack: configuration pack format and the item model A new crate for the .twconfig configuration pack that export, sharing, local backups and sync versions share: - Format: a ZIP with manifest.json, config.yaml, plugin scripts and their approved copies, and client-prefs.json; optionally sealed with age (scrypt passphrase, or an X25519 recipient for sync). Reading treats the file as untrusted: size limits on the file and on what it unpacks to, only the known paths, valid plugin ids, no duplicates. Errors carry their own message codes (pack.*). Same content writes the same bytes. - Model: a configuration split into items (upstreams, proxies, keys, aliases, groups, routes, price sheets, custom security rules, plugins, the order of upstreams, routes and plugins, each global section, client preferences) and merge units (upstream: conn/models/billing/other; plugin: script/settings/enabled; global settings and preferences field by field). Values are normalized, so comments, layout and written-out defaults are not differences. Local settings (listen, retention, sync, the ChatGPT login and account header, the script of a shipped default plugin) stay out of the snapshot and are kept on materialize. - materialize: lands a target snapshot on a local config by editing it item by item, so untouched items keep their bytes and comments; falls back to a full rewrite when an edit cannot apply, and verifies the result either way. A plugin that can rewrite tool calls is not approved when it arrives enabled from elsewhere. - rename follows every reference; content_id identifies the shareable content (12 hex chars of blake3 over the normalized snapshot). Plugin source splitting needs the plugin runtime, which this crate does not depend on; the core installs it (install_codec), and without it a script is compared whole. Co-Authored-By: Claude Opus 5.5 * tw-pack: zip 9 (RUSTSEC-2026-0367/0368/0369); a clippy fix for Rust 1.99 The advisories are about extracting archives to disk, which this crate never does, but the patched release is the one to depend on. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- Cargo.lock | 695 +++++++++++++++++-- Cargo.toml | 9 + README.md | 1 + README.zh-CN.md | 1 + crates/tw-api/msg-codes.txt | 12 + crates/tw-pack/Cargo.toml | 30 + crates/tw-pack/src/lib.rs | 696 +++++++++++++++++++ crates/tw-pack/src/model/codec.rs | 47 ++ crates/tw-pack/src/model/items.rs | 728 ++++++++++++++++++++ crates/tw-pack/src/model/json.rs | 131 ++++ crates/tw-pack/src/model/mod.rs | 533 +++++++++++++++ crates/tw-pack/src/model/rename.rs | 241 +++++++ crates/tw-pack/src/model/tests.rs | 1015 ++++++++++++++++++++++++++++ crates/tw-pack/src/model/write.rs | 800 ++++++++++++++++++++++ crates/tw-pack/src/tests.rs | 334 +++++++++ 15 files changed, 5235 insertions(+), 38 deletions(-) create mode 100644 crates/tw-pack/Cargo.toml create mode 100644 crates/tw-pack/src/lib.rs create mode 100644 crates/tw-pack/src/model/codec.rs create mode 100644 crates/tw-pack/src/model/items.rs create mode 100644 crates/tw-pack/src/model/json.rs create mode 100644 crates/tw-pack/src/model/mod.rs create mode 100644 crates/tw-pack/src/model/rename.rs create mode 100644 crates/tw-pack/src/model/tests.rs create mode 100644 crates/tw-pack/src/model/write.rs create mode 100644 crates/tw-pack/src/tests.rs diff --git a/Cargo.lock b/Cargo.lock index aa6d27e0..03cbac05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -27,6 +27,49 @@ dependencies = [ "generic-array", ] +[[package]] +name = "age" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "047a482d1843edf1ce76ada63183698144030fe1191bd5ddba6e41e164e0bc43" +dependencies = [ + "age-core", + "base64 0.21.7", + "bech32", + "chacha20poly1305", + "cookie-factory", + "hmac 0.12.1", + "i18n-embed", + "i18n-embed-fl", + "lazy_static", + "nom", + "pin-project", + "rand 0.8.8", + "rust-embed", + "scrypt", + "sha2 0.10.9", + "subtle", + "x25519-dalek", + "zeroize", +] + +[[package]] +name = "age-core" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2bf6a89c984ca9d850913ece2da39e1d200563b0a94b002b253beee4c5acf99" +dependencies = [ + "base64 0.21.7", + "chacha20poly1305", + "cookie-factory", + "hkdf", + "io_tee", + "nom", + "rand 0.8.8", + "secrecy", + "sha2 0.10.9", +] + [[package]] name = "aho-corasick" version = "1.1.5" @@ -227,7 +270,7 @@ dependencies = [ "bytes", "form_urlencoded", "hex", - "hmac", + "hmac 0.13.0", "http", "percent-encoding", "sha2 0.11.0", @@ -335,7 +378,7 @@ checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ "axum-core", "axum-macros", - "base64", + "base64 0.22.1", "bytes", "form_urlencoded", "futures-util", @@ -394,6 +437,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + [[package]] name = "base64" version = "0.22.1" @@ -410,6 +459,21 @@ dependencies = [ "vsimd", ] +[[package]] +name = "basic-toml" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba62675e8242a4c4e806d12f11d136e626e6c8361d6b829310732241652a178a" +dependencies = [ + "serde", +] + +[[package]] +name = "bech32" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d86b93f97252c47b41663388e6d155714a9d0c398b99f1005cbc5f978b29f445" + [[package]] name = "bitflags" version = "2.13.1" @@ -626,7 +690,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" dependencies = [ - "thiserror", + "thiserror 2.0.20", ] [[package]] @@ -657,6 +721,15 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" +[[package]] +name = "cookie-factory" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9885fa71e26b8ab7855e2ec7cae6e9b380edff76cd052e07c683a0319d51b3a2" +dependencies = [ + "futures", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -761,7 +834,7 @@ dependencies = [ "postcard", "pulley-interpreter", "regalloc2", - "rustc-hash", + "rustc-hash 2.1.3", "serde", "serde_derive", "sha2 0.10.9", @@ -920,6 +993,7 @@ dependencies = [ "fiat-crypto", "rustc_version", "subtle", + "zeroize", ] [[package]] @@ -1043,6 +1117,15 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" +[[package]] +name = "find-crate" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59a98bbaacea1c0eb6a0876280051b892eb73594fd90cf3b20e9c817029c57d2" +dependencies = [ + "toml", +] + [[package]] name = "find-msvc-tools" version = "0.1.12" @@ -1060,6 +1143,50 @@ dependencies = [ "zlib-rs", ] +[[package]] +name = "fluent" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb74634707bebd0ce645a981148e8fb8c7bccd4c33c652aeffd28bf2f96d555a" +dependencies = [ + "fluent-bundle", + "unic-langid", +] + +[[package]] +name = "fluent-bundle" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe0a21ee80050c678013f82edf4b705fe2f26f1f9877593d13198612503f493" +dependencies = [ + "fluent-langneg", + "fluent-syntax", + "intl-memoizer", + "intl_pluralrules", + "rustc-hash 1.1.0", + "self_cell 0.10.3", + "smallvec", + "unic-langid", +] + +[[package]] +name = "fluent-langneg" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eebbe59450baee8282d71676f3bfed5689aeab00b27545e83e5f14b1195e8b0" +dependencies = [ + "unic-langid", +] + +[[package]] +name = "fluent-syntax" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a530c4694a6a8d528794ee9bbd8ba0122e779629ac908d15ad5a7ae7763a33d" +dependencies = [ + "thiserror 1.0.69", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1305,6 +1432,24 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac 0.12.1", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + [[package]] name = "hmac" version = "0.13.0" @@ -1411,7 +1556,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -1428,6 +1573,72 @@ dependencies = [ "tracing", ] +[[package]] +name = "i18n-config" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e06b90c8a0d252e203c94344b21e35a30f3a3a85dc7db5af8f8df9f3e0c63ef" +dependencies = [ + "basic-toml", + "log", + "serde", + "serde_derive", + "thiserror 1.0.69", + "unic-langid", +] + +[[package]] +name = "i18n-embed" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "669ffc2c93f97e6ddf06ddbe999fcd6782e3342978bb85f7d3c087c7978404c4" +dependencies = [ + "arc-swap", + "fluent", + "fluent-langneg", + "fluent-syntax", + "i18n-embed-impl", + "intl-memoizer", + "log", + "parking_lot", + "rust-embed", + "thiserror 1.0.69", + "unic-langid", + "walkdir", +] + +[[package]] +name = "i18n-embed-fl" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04b2969d0b3fc6143776c535184c19722032b43e6a642d710fa3f88faec53c2d" +dependencies = [ + "find-crate", + "fluent", + "fluent-syntax", + "i18n-config", + "i18n-embed", + "proc-macro-error2", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", + "unic-langid", +] + +[[package]] +name = "i18n-embed-impl" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f2cc0e0523d1fe6fc2c6f66e5038624ea8091b3e7748b5e8e0c84b1698db6c2" +dependencies = [ + "find-crate", + "i18n-config", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -1597,6 +1808,31 @@ dependencies = [ "generic-array", ] +[[package]] +name = "intl-memoizer" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "310da2e345f5eb861e7a07ee182262e94975051db9e4223e909ba90f392f163f" +dependencies = [ + "type-map", + "unic-langid", +] + +[[package]] +name = "intl_pluralrules" +version = "7.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "078ea7b7c29a2b4df841a7f6ac8775ff6074020c6776d48491ce2268e068f972" +dependencies = [ + "unic-langid", +] + +[[package]] +name = "io_tee" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b3f7cef34251886990511df1c61443aa928499d598a9473929ab5a90a527304" + [[package]] name = "ipnet" version = "2.12.2" @@ -1636,7 +1872,7 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror", + "thiserror 2.0.20", "walkdir", "windows-link", ] @@ -1761,6 +1997,15 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.34" @@ -1815,6 +2060,22 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.9.1" @@ -1837,6 +2098,28 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "notify" version = "8.2.0" @@ -1939,12 +2222,65 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest 0.10.7", + "hmac 0.12.1", +] + [[package]] name = "percent-encoding" version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "pin-project-lite" version = "0.2.17" @@ -2010,6 +2346,28 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -2053,10 +2411,10 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", + "rustc-hash 2.1.3", "rustls", "socket2", - "thiserror", + "thiserror 2.0.20", "tokio", "tracing", "web-time", @@ -2075,11 +2433,11 @@ dependencies = [ "rand 0.10.2", "rand_pcg", "ring", - "rustc-hash", + "rustc-hash 2.1.3", "rustls", "rustls-pki-types", "slab", - "thiserror", + "thiserror 2.0.20", "tinyvec", "tracing", "web-time", @@ -2120,13 +2478,24 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + [[package]] name = "rand" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" dependencies = [ - "rand_chacha", + "rand_chacha 0.9.0", "rand_core 0.9.5", ] @@ -2141,6 +2510,16 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + [[package]] name = "rand_chacha" version = "0.9.0" @@ -2151,6 +2530,15 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + [[package]] name = "rand_core" version = "0.9.5" @@ -2195,6 +2583,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + [[package]] name = "regalloc2" version = "0.15.2" @@ -2205,7 +2602,7 @@ dependencies = [ "bumpalo", "hashbrown 0.17.1", "log", - "rustc-hash", + "rustc-hash 2.1.3", "serde", "smallvec", ] @@ -2245,7 +2642,7 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", "futures-util", @@ -2302,7 +2699,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" dependencies = [ "hashbrown 0.16.1", - "thiserror", + "thiserror 2.0.20", ] [[package]] @@ -2320,12 +2717,53 @@ dependencies = [ "sqlite-wasm-rs", ] +[[package]] +name = "rust-embed" +version = "8.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19afa5b4b6a611de00bd1bdae6ae6f39084c9399f0679c3f52d8469cf335cc23" +dependencies = [ + "rust-embed-impl", + "rust-embed-utils", + "walkdir", +] + +[[package]] +name = "rust-embed-impl" +version = "8.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0d8afda6374eac59e066abee06d265247ebbaf3006cf878e2879e8356e34053" +dependencies = [ + "mime_guess", + "proc-macro2", + "quote", + "rust-embed-utils", + "syn 2.0.119", + "walkdir", +] + +[[package]] +name = "rust-embed-utils" +version = "8.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d84e8ba78bd384263e5922f084cbe1b081c3b7e69add59c8fb097b879ba968a" +dependencies = [ + "sha2 0.11.0", + "walkdir", +] + [[package]] name = "rustc-demangle" version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-hash" version = "2.1.3" @@ -2442,6 +2880,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher", +] + [[package]] name = "same-file" version = "1.0.6" @@ -2458,7 +2905,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0b1b76852c5643f558e7347e4de549474c3ecb7154295a189f4668428c1b453" dependencies = [ "arraydeque", - "thiserror", + "thiserror 2.0.20", ] [[package]] @@ -2470,6 +2917,32 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "scrypt" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +dependencies = [ + "pbkdf2", + "salsa20", + "sha2 0.10.9", +] + +[[package]] +name = "secrecy" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e891af845473308773346dc847b2c23ee78fe442e0472ac50e22a18a93d3ae5a" +dependencies = [ + "zeroize", +] + [[package]] name = "security-framework" version = "3.7.0" @@ -2493,6 +2966,21 @@ dependencies = [ "libc", ] +[[package]] +name = "self_cell" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14e4d63b804dc0c7ec4a1e52bcb63f02c7ac94476755aa579edac21e01f915d" +dependencies = [ + "self_cell 1.3.0", +] + +[[package]] +name = "self_cell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab42ca02749e120097e328d91d415325bdf43b1c72c4c8badf37375fe40a813" + [[package]] name = "semver" version = "1.0.28" @@ -2786,7 +3274,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.61.2", @@ -2801,13 +3289,33 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + [[package]] name = "thiserror" version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.20", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] @@ -2867,6 +3375,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", + "serde_core", "zerovec", ] @@ -2972,6 +3481,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + [[package]] name = "tower" version = "0.5.3" @@ -3092,7 +3610,7 @@ version = "12.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "756050066659291d47a554a9f558125db17428b073c5ffce1daf5dcb0f7231d8" dependencies = [ - "thiserror", + "thiserror 2.0.20", "ts-rs-macros", ] @@ -3121,7 +3639,7 @@ dependencies = [ "log", "rand 0.9.5", "sha1", - "thiserror", + "thiserror 2.0.20", "utf-8", ] @@ -3138,7 +3656,7 @@ dependencies = [ "log", "rand 0.9.5", "sha1", - "thiserror", + "thiserror 2.0.20", ] [[package]] @@ -3164,7 +3682,7 @@ dependencies = [ "bytes", "futures", "hex", - "hmac", + "hmac 0.13.0", "percent-encoding", "reqwest", "serde_json", @@ -3192,7 +3710,7 @@ dependencies = [ "serde", "serde_yaml_ng", "tempfile", - "thiserror", + "thiserror 2.0.20", "tokio", "tw-api", "tw-bedrock", @@ -3211,7 +3729,7 @@ name = "tw-control" version = "0.76.0" dependencies = [ "axum", - "base64", + "base64 0.22.1", "bytes", "chrono", "futures", @@ -3226,7 +3744,7 @@ dependencies = [ "serde_yaml_ng", "sha2 0.11.0", "tempfile", - "thiserror", + "thiserror 2.0.20", "tokio", "tokio-stream", "tokio-tungstenite 0.28.0", @@ -3266,7 +3784,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml_ng", - "thiserror", + "thiserror 2.0.20", "tracing", "tw-dialect", "tw-types", @@ -3281,7 +3799,7 @@ dependencies = [ "aws-smithy-eventstream", "aws-smithy-types", "axum", - "base64", + "base64 0.22.1", "blake3", "bytes", "chrono", @@ -3301,7 +3819,7 @@ dependencies = [ "serde_yaml_ng", "sha2 0.11.0", "tempfile", - "thiserror", + "thiserror 2.0.20", "tokio", "tokio-rustls", "tokio-tungstenite 0.28.0", @@ -3328,7 +3846,7 @@ name = "tw-guard" version = "0.76.0" dependencies = [ "aho-corasick", - "base64", + "base64 0.22.1", "bytes", "memchr", "regex", @@ -3336,7 +3854,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml_ng", - "thiserror", + "thiserror 2.0.20", "ts-rs", "tw-dialect", ] @@ -3349,7 +3867,7 @@ dependencies = [ "serde_json", "snow", "tempfile", - "thiserror", + "thiserror 2.0.20", "tokio", "tw-api", "tw-yaml", @@ -3364,6 +3882,26 @@ dependencies = [ "tw-api", ] +[[package]] +name = "tw-pack" +version = "0.76.0" +dependencies = [ + "age", + "blake3", + "serde", + "serde_json", + "serde_yaml_ng", + "sha2 0.11.0", + "tempfile", + "thiserror 2.0.20", + "tw-config", + "tw-engine", + "tw-pricing", + "tw-types", + "tw-yaml", + "zip", +] + [[package]] name = "tw-plugin" version = "0.76.0" @@ -3373,7 +3911,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.11.0", - "thiserror", + "thiserror 2.0.20", "wasm-encoder", "wasmparser", "wasmtime", @@ -3388,7 +3926,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml_ng", - "thiserror", + "thiserror 2.0.20", "tw-types", ] @@ -3396,7 +3934,7 @@ dependencies = [ name = "tw-secret" version = "0.76.0" dependencies = [ - "thiserror", + "thiserror 2.0.20", ] [[package]] @@ -3409,7 +3947,7 @@ dependencies = [ "serde", "serde_json", "tempfile", - "thiserror", + "thiserror 2.0.20", "tokio", "tracing", "tw-api", @@ -3435,7 +3973,7 @@ dependencies = [ "libc", "notify", "tempfile", - "thiserror", + "thiserror 2.0.20", "tokio", ] @@ -3445,7 +3983,7 @@ version = "0.76.0" dependencies = [ "saphyr-parser", "serde_yaml_ng", - "thiserror", + "thiserror 2.0.20", "tw-types", ] @@ -3484,12 +4022,52 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "type-map" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb30dbbd9036155e74adad6812e9898d03ec374946234fbcebd5dfc7b9187b90" +dependencies = [ + "rustc-hash 2.1.3", +] + +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + [[package]] name = "typenum" version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "unic-langid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ba52c9b05311f4f6e62d5d9d46f094bd6e84cb8df7b3ef952748d752a7d05" +dependencies = [ + "unic-langid-impl", +] + +[[package]] +name = "unic-langid-impl" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce1bf08044d4b7a94028c93786f8566047edc11110595914de93362559bc658" +dependencies = [ + "serde", + "tinystr", +] + +[[package]] +name = "unicase" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357cc3acc6a036009fd6c973ed009037c732d60d0b4f6c673e9041497482a28f" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -3825,7 +4403,7 @@ dependencies = [ "pulley-interpreter", "smallvec", "target-lexicon", - "thiserror", + "thiserror 2.0.20", "wasmparser", "wasmtime-environ", "wasmtime-internal-core", @@ -3958,7 +4536,7 @@ dependencies = [ "regalloc2", "smallvec", "target-lexicon", - "thiserror", + "thiserror 2.0.20", "wasmparser", "wasmtime-environ", "wasmtime-internal-core", @@ -4192,6 +4770,18 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" +[[package]] +name = "x25519-dalek" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" +dependencies = [ + "curve25519-dalek", + "rand_core 0.6.4", + "serde", + "zeroize", +] + [[package]] name = "yoke" version = "0.8.3" @@ -4261,6 +4851,20 @@ name = "zeroize" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] [[package]] name = "zerotrie" @@ -4279,6 +4883,7 @@ version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" dependencies = [ + "serde", "yoke", "zerofrom", "zerovec-derive", @@ -4295,6 +4900,20 @@ dependencies = [ "syn 3.0.5", ] +[[package]] +name = "zip" +version = "9.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e04976da629d3b04204ba8105b396a193af8ad2583b2c79bfb3ab8af580a11e" +dependencies = [ + "crc32fast", + "flate2", + "indexmap", + "memchr", + "nix", + "typed-path", +] + [[package]] name = "zlib-rs" version = "0.6.7" diff --git a/Cargo.toml b/Cargo.toml index 8d4796e5..a2d085af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,9 @@ members = [ # crate、企业版用的第一层都不能沾上，tw-plugin 的 tests/boundary.rs 守着 "crates/tw-plugin", "crates/tw-control", + # 配置包（.twconfig）：导出、导入、本机备份与同步共用的格式，以及把一份配置拆成 + # 项目与合并单位的模型 + "crates/tw-pack", # 控制通道的握手与加密。桌面端按 tag 依赖它，所以只依赖契约层和 tw-yaml "crates/tw-link", # ── bin ────────────────────────────────────────────── @@ -76,6 +79,7 @@ tw-dialect = { path = "crates/tw-dialect" } tw-bedrock = { path = "crates/tw-bedrock" } tw-link = { path = "crates/tw-link" } tw-plugin = { path = "crates/tw-plugin" } +tw-pack = { path = "crates/tw-pack" } # ── 版本与企业版 workspace 对齐，便于反向依赖时不打架 ──── tokio = { version = "1", features = ["rt-multi-thread", "macros", "net", "time", "sync", "io-util", "signal"] } @@ -161,6 +165,11 @@ rand = "0.10" zstd = { version = "0.13", default-features = false } clap = { version = "4", features = ["derive"] } +# 配置包的口令加密（scrypt）与同步版本的密钥加密（X25519）。默认特性是空的：不要 +# SSH 密钥、插件、命令行那几样 +age = { version = "0.11", default-features = false } +# 配置包本身是 ZIP。只要 deflate（走已经在依赖树里的 flate2），不要别的压缩算法和加密 +zip = { version = "9", default-features = false, features = ["deflate-flate2"] } # 控制面契约导出成 TypeScript（tw-api 的 `ts` feature）。**不开任何 serde 以外的 # 类型实现**：契约里只有 std 和 serde 的类型 ts-rs = { version = "12", default-features = false, features = ["serde-compat"] } diff --git a/README.md b/README.md index e64940f1..53af4952 100644 --- a/README.md +++ b/README.md @@ -156,6 +156,7 @@ a time, and cannot stop core, take the diagnostic bundle or change | `tw-observe` | Event bus | | `tw-gateway` | Data plane: the life of a request | | `tw-plugin` | Script-plugin sandbox: QuickJS compiled to WebAssembly, run by Wasmtime | +| `tw-pack` | Configuration packs (`.twconfig`): format, encryption, local backups, and the item model import and sync compare on | | `tw-control` | Control-plane server | ThinkWatch Enterprise depends only on the first four, which depend only on one diff --git a/README.zh-CN.md b/README.zh-CN.md index d07fc7b5..1da1df98 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -93,6 +93,7 @@ twcore control-key --rotate # 更换密钥；用旧密钥建立的连接随 | `tw-observe` | 事件总线 | | `tw-gateway` | 数据面：一个请求的完整生命周期 | | `tw-plugin` | 脚本插件的沙箱：编成 WebAssembly 的 QuickJS，由 Wasmtime 运行 | +| `tw-pack` | 配置包（`.twconfig`）：格式、加密、本机备份，以及导入与同步共同比较的项目模型 | | `tw-control` | 控制面服务 | ThinkWatch 企业版只依赖前四个 crate，这四个 crate 也只相互依赖；CI 会针对它们的每一次改动检查企业版能否编译。ThinkWatch Lite 把 `tw-api`、`tw-types`、`tw-yaml`、`tw-guard`、`tw-watch` 和 `tw-link` 固定在某个 Release 的 tag 上，并打包同一 Release 的 `twcore`。接管 AI 客户端和扫描其配置在 Lite 中完成，作用于应用所在的机器；`twcore` 只为每个客户端签发专用的网关密钥。二进制的源码位于 `bin/twcore`。只有 `tw-gateway` 和 `twcore` 可以依赖 `tw-plugin`——它是唯一一个构建时除了 Rust 还需要别的工具的 crate（见下文），因此用这些 crate 构建 Lite 或企业版时都不需要。 diff --git a/crates/tw-api/msg-codes.txt b/crates/tw-api/msg-codes.txt index a1c07e9c..41d57420 100644 --- a/crates/tw-api/msg-codes.txt +++ b/crates/tw-api/msg-codes.txt @@ -492,6 +492,18 @@ l3.not_streamed l3.refused l3.sign_failed l3.stream_failed +pack.bad_entry +pack.corrupt +pack.item_shape +pack.missing_entry +pack.need_key +pack.need_passphrase +pack.schema_too_new +pack.too_large +pack.too_new +pack.verify_failed +pack.wrong_key +pack.wrong_passphrase pricing.sheet.bad_multiplier pricing.sheet.bad_price pricing.sheet.duplicate_name diff --git a/crates/tw-pack/Cargo.toml b/crates/tw-pack/Cargo.toml new file mode 100644 index 00000000..a2a8625c --- /dev/null +++ b/crates/tw-pack/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "tw-pack" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +documentation.workspace = true +readme.workspace = true +description = "Configuration packs (.twconfig): format, encryption, local backups, and the item model that import and sync compare on" + +[dependencies] +# 配置的形状、校验、按名字改一项（保留注释）都在这里 +tw-config = { workspace = true } +tw-engine = { workspace = true } +tw-pricing = { workspace = true } +tw-types = { workspace = true } +tw-yaml = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +serde_yaml_ng = { workspace = true } +thiserror = { workspace = true } +blake3 = { workspace = true } +sha2 = { workspace = true } +age = { workspace = true } +zip = { workspace = true } + +[dev-dependencies] +tempfile = "3" diff --git a/crates/tw-pack/src/lib.rs b/crates/tw-pack/src/lib.rs new file mode 100644 index 00000000..99c1e4c6 --- /dev/null +++ b/crates/tw-pack/src/lib.rs @@ -0,0 +1,696 @@ +//! 配置包（`.twconfig`）：导出、分享、本机备份与同步版本共用的一种文件。 +//! +//! # 文件里有什么 +//! +//! 未加密的配置包是一个 ZIP，里面只许有这几样： +//! +//! ```text +//! manifest.json 格式版本与内容说明（[`Manifest`]） +//! config.yaml 配置 +//! plugins/.js 插件脚本 +//! plugins/.approved/.js 批准过的那一份（底稿） +//! client-prefs.json Lite 的通用偏好，Core 不解析 +//! ``` +//! +//! 加密的配置包是 age 格式：口令加密（scrypt），或者加密给一把 X25519 公钥（同步位置的 +//! 版本密钥）。解开之后是同样的 ZIP。两者按文件头区分（[`is_encrypted`]），扩展名相同。 +//! +//! # 读的时候不信任文件 +//! +//! 配置包会从别人那里来（分享、导入），所以读的时候按不可信的输入对待：整个文件和解压之后 +//! 的总大小都不超过 [`MAX_BYTES`]，ZIP 里出现上面以外的路径（绝对路径、`..`、别的文件名） +//! 整包拒绝，插件 id 必须合规矩，同一个路径出现两次也拒绝。 +//! +//! # 同样的内容，同样的字节 +//! +//! 条目按固定的顺序写、时间戳固定，所以内容相同的两个包只差 `created_at_ms`。比较两个包是 +//! 不是同一份配置用 [`Pack::content_id`]：它只看可以带走的内容（本机设置不算），按 +//! [`model`] 规范化之后再算，注释、排版、默认值写没写都不影响它。 + +use std::collections::BTreeMap; +use std::io::{Read, Write}; + +use serde::{Deserialize, Serialize}; +use tw_types::{Msg, msg}; + +pub mod model; + +/// 配置包格式的版本。**读到更大的就拒绝**（[`PackError::TooNew`]）：新版本的格式旧版本 +/// 猜不准，猜错了会悄悄丢东西 +pub const FORMAT: u32 = 1; + +/// 一个配置包（连同解压之后的内容）最大多少字节 +pub const MAX_BYTES: usize = 16 * 1024 * 1024; + +/// 扩展名，不带点 +pub const EXTENSION: &str = "twconfig"; + +/// 写进 manifest 的 Core 版本：这几个 crate 的版本，和 `Status.version` 的 CalVer 不是一回事 +pub const CORE_VERSION: &str = env!("CARGO_PKG_VERSION"); + +/// ZIP 里的固定路径 +const MANIFEST: &str = "manifest.json"; +const CONFIG: &str = "config.yaml"; +const PREFS: &str = "client-prefs.json"; +const PLUGINS: &str = "plugins/"; +const APPROVED: &str = "plugins/.approved/"; + +/// ZIP 里最多多少个条目：清单、配置、偏好，加上每个插件两份。再多只会是乱塞的 +const MAX_ENTRIES: usize = 2048; + +/// 一个配置包，解开之后的样子。 +#[derive(Debug, Clone, PartialEq)] +pub struct Pack { + pub manifest: Manifest, + pub config_yaml: String, + /// 插件 id → 文件 + pub plugins: BTreeMap, + /// Lite 的通用偏好。**不解析**：Lite 定义它，Core 只是存着、带着走 + pub client_prefs: Option, +} + +/// 一个插件的两份文件。 +#[derive(Clone, PartialEq, Eq)] +pub struct PluginFiles { + pub script: Vec, + /// 批准过的那一份。和脚本一样（或者没有）就是没有待批准的改动 + pub approved: Option>, +} + +/// 脚本可能有几百 KB，Debug 里只说多长 +impl std::fmt::Debug for PluginFiles { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PluginFiles") + .field("script", &format_args!("{} bytes", self.script.len())) + .field( + "approved", + &self.approved.as_ref().map(|a| format!("{} bytes", a.len())), + ) + .finish() + } +} + +/// `manifest.json`。 +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Manifest { + pub format: u32, + pub kind: PackKind, + pub created_at_ms: u64, + pub core_version: String, + /// 配置文件格式的版本（`tw_config::SCHEMA_VERSION`） + pub schema: u32, + /// 带没带密钥。不带时上游密钥、请求头的值、AWS 凭据、代理密码、网关密钥都是空的 + pub secrets: bool, + /// **读出来时由文件头决定**，文件里写的不算 + #[serde(default)] + pub encrypted: bool, + pub origin: PackOrigin, + /// 写的时候按内容重算，读的时候也重算：文件里写的只是给人看的 + #[serde(default)] + pub counts: Counts, + /// 同步用的那几项（序号、设备、上一版本、冲突记录），同步那一路自己定义、自己读 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sync: Option, +} + +impl Manifest { + /// 一份新的说明：当前的格式和 Core 版本、此刻的时间、带着密钥、不加密 + pub fn new(kind: PackKind, origin: PackOrigin) -> Self { + Self { + format: FORMAT, + kind, + created_at_ms: now_ms(), + core_version: CORE_VERSION.to_string(), + schema: tw_config::SCHEMA_VERSION, + secrets: true, + encrypted: false, + origin, + counts: Counts::default(), + sync: None, + } + } + + /// 由比这一份 Core 新的版本写的 + pub fn from_newer_core(&self) -> bool { + newer_than_us(&self.core_version) + } +} + +/// 完整的配置，还是选出来的几项（选择项目导出、分享）。 +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PackKind { + Full, + Partial, +} + +/// 这个包是怎么来的。导出和分享之外，和配置的一次改动的来源是同一套词。 +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PackOrigin { + Export, + Share, + Ui, + Cli, + External, + Defaults, + Sync, + Import, + Restore, + Repair, + /// 更新的版本才有的来源。**读得进来**：一个新词不该让整个包读不了 + #[serde(other)] + Other, +} + +impl PackOrigin { + pub fn slug(&self) -> &'static str { + match self { + PackOrigin::Export => "export", + PackOrigin::Share => "share", + PackOrigin::Ui => "ui", + PackOrigin::Cli => "cli", + PackOrigin::External => "external", + PackOrigin::Defaults => "defaults", + PackOrigin::Sync => "sync", + PackOrigin::Import => "import", + PackOrigin::Restore => "restore", + PackOrigin::Repair => "repair", + PackOrigin::Other => "other", + } + } + + pub fn from_slug(s: &str) -> PackOrigin { + serde_json::from_value(serde_json::Value::String(s.to_string())) + .unwrap_or(PackOrigin::Other) + } +} + +impl From for PackOrigin { + fn from(o: tw_config::history::Origin) -> Self { + use tw_config::history::Origin as O; + match o { + O::Ui => PackOrigin::Ui, + O::Cli => PackOrigin::Cli, + O::External => PackOrigin::External, + O::Defaults => PackOrigin::Defaults, + O::Rollback => PackOrigin::Restore, + // 轮换不留版本，走到这里也只是一个说不出来源的版本 + O::Rotation => PackOrigin::Other, + } + } +} + +/// 包里有多少东西。给列表和预览看。 +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct Counts { + pub providers: u32, + pub proxies: u32, + pub clients: u32, + pub aliases: u32, + pub groups: u32, + pub routes: u32, + pub price_sheets: u32, + pub security_rules: u32, + pub plugins: u32, +} + +impl Counts { + /// 按配置原文数。**读不成 YAML 也不报错**：数不出来就是 0 + pub fn of(config_yaml: &str) -> Counts { + let Ok(doc) = serde_yaml_ng::from_str::(config_yaml) else { + return Counts::default(); + }; + let len = |path: &[&str]| -> u32 { + let mut cur = &doc; + for k in path { + match cur.get(*k) { + Some(v) => cur = v, + None => return 0, + } + } + match cur { + serde_yaml_ng::Value::Sequence(s) => s.len() as u32, + serde_yaml_ng::Value::Mapping(m) => m.len() as u32, + _ => 0, + } + }; + Counts { + providers: len(&["providers"]), + proxies: len(&["proxies"]), + clients: len(&["clients"]), + aliases: len(&["aliases"]), + groups: len(&["groups"]), + routes: len(&["routes"]), + price_sheets: len(&["pricing", "sheets"]), + security_rules: len(&["security", "redact", "custom"]) + + len(&["security", "inspect_tools", "custom"]) + + len(&["security", "content", "custom"]), + plugins: len(&["plugins"]), + } + } +} + +/// 写的时候怎么封。 +pub enum Seal<'a> { + None, + Passphrase(&'a str), + Recipient(&'a age::x25519::Recipient), +} + +/// 读的时候拿什么解。 +pub enum Unseal<'a> { + None, + Passphrase(&'a str), + Identity(&'a age::x25519::Identity), +} + +/// 读写配置包时的失败。**英文只写一遍**：`Display` 就是 [`PackError::msg`] 的原句。 +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum PackError { + /// 口令加密的包，没给口令 + #[error("{}", self.msg())] + NeedPassphrase, + #[error("{}", self.msg())] + WrongPassphrase, + /// 加密给一把密钥的包（同步版本），没给密钥 + #[error("{}", self.msg())] + NeedKey, + #[error("{}", self.msg())] + WrongKey, + /// 格式版本比这一份 Core 认得的新 + #[error("{}", self.msg())] + TooNew { format: u32, core_version: String }, + /// 读不成一个配置包：不是 ZIP、ZIP 坏了、清单读不懂、配置不是 UTF-8…… + #[error("{}", self.msg())] + Corrupt { detail: String }, + /// ZIP 里有不该有的路径 + #[error("{}", self.msg())] + BadEntry { name: String }, + /// 少了必须有的那一项（清单、配置） + #[error("{}", self.msg())] + MissingEntry { name: String }, + #[error("{}", self.msg())] + TooLarge { limit: usize }, +} + +impl PackError { + pub fn msg(&self) -> Msg { + match self { + PackError::NeedPassphrase => msg!( + "pack.need_passphrase" => + "This configuration pack is encrypted with a passphrase. Enter the passphrase to open it." + ), + PackError::WrongPassphrase => msg!( + "pack.wrong_passphrase" => + "The passphrase is not correct." + ), + PackError::NeedKey => msg!( + "pack.need_key" => + "This configuration pack is encrypted for a sync location and cannot be opened with a passphrase." + ), + PackError::WrongKey => msg!( + "pack.wrong_key" => + "This configuration pack is encrypted with a different key." + ), + PackError::TooNew { + format, + core_version, + } => msg!( + "pack.too_new", format = format, version = core_version => + "This configuration pack was exported by a newer version (Core {version}, format \ + {format}). Update ThinkWatch Lite first." + ), + PackError::Corrupt { detail } => msg!( + "pack.corrupt", detail = detail => + "This file is not a readable configuration pack: {detail}" + ), + PackError::BadEntry { name } => msg!( + "pack.bad_entry", name = name => + "This configuration pack contains `{name}`, which a configuration pack may not \ + contain, so it was not opened." + ), + PackError::MissingEntry { name } => msg!( + "pack.missing_entry", name = name => + "This configuration pack has no {name}." + ), + PackError::TooLarge { limit } => msg!( + "pack.too_large", limit = limit / (1024 * 1024) => + "This configuration pack is larger than {limit} MiB." + ), + } + } +} + +fn corrupt(detail: impl std::fmt::Display) -> PackError { + PackError::Corrupt { + detail: detail.to_string(), + } +} + +// ─────────────────────────────────────────────────────────── 写 + +/// 把一个包写成文件的字节。`counts` 和 `encrypted` 按内容重算，别的照 `pack.manifest`。 +pub fn write(pack: &Pack, seal: Seal) -> Result, PackError> { + let mut manifest = pack.manifest.clone(); + manifest.counts = Counts::of(&pack.config_yaml); + manifest.encrypted = !matches!(seal, Seal::None); + let plain = zip_bytes(pack, &manifest)?; + if plain.len() > MAX_BYTES { + return Err(PackError::TooLarge { limit: MAX_BYTES }); + } + let out = match seal { + Seal::None => plain, + Seal::Passphrase(p) => { + let mut r = + age::scrypt::Recipient::new(age::secrecy::SecretString::from(p.to_string())); + r.set_work_factor(work_factor()); + encrypt(&plain, &r)? + } + Seal::Recipient(r) => encrypt(&plain, r)?, + }; + if out.len() > MAX_BYTES { + return Err(PackError::TooLarge { limit: MAX_BYTES }); + } + Ok(out) +} + +fn zip_bytes(pack: &Pack, manifest: &Manifest) -> Result, PackError> { + // **解压之后的总量也不超过上限**：读的时候按它挡，写出去的就得读得回来 + let unpacked = pack.config_yaml.len() + + pack + .plugins + .values() + .map(|f| f.script.len() + f.approved.as_ref().map_or(0, Vec::len)) + .sum::() + + pack + .client_prefs + .as_ref() + .map_or(0, |p| p.to_string().len() * 2); + if unpacked > MAX_BYTES { + return Err(PackError::TooLarge { limit: MAX_BYTES }); + } + let mut z = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + // **时间戳固定**：同样的内容写出同样的字节 + let opts = zip::write::SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Deflated) + .last_modified_time(zip::DateTime::DEFAULT) + .unix_permissions(0o600); + let mut put = |name: &str, bytes: &[u8]| -> Result<(), PackError> { + z.start_file(name, opts).map_err(corrupt)?; + z.write_all(bytes).map_err(corrupt) + }; + let manifest_json = serde_json::to_vec_pretty(manifest).map_err(corrupt)?; + put(MANIFEST, &manifest_json)?; + put(CONFIG, pack.config_yaml.as_bytes())?; + if let Some(p) = &pack.client_prefs { + put(PREFS, &serde_json::to_vec_pretty(p).map_err(corrupt)?)?; + } + for (id, files) in &pack.plugins { + if !tw_config::plugins::valid_id(id) { + return Err(PackError::BadEntry { + name: format!("{PLUGINS}{id}.js"), + }); + } + put(&format!("{PLUGINS}{id}.js"), &files.script)?; + if let Some(a) = &files.approved { + put(&format!("{APPROVED}{id}.js"), a)?; + } + } + Ok(z.finish().map_err(corrupt)?.into_inner()) +} + +fn encrypt(plain: &[u8], r: &dyn age::Recipient) -> Result, PackError> { + let e = age::Encryptor::with_recipients(std::iter::once(r)).map_err(corrupt)?; + let mut out = Vec::with_capacity(plain.len() + 256); + let mut w = e.wrap_output(&mut out).map_err(corrupt)?; + w.write_all(plain).map_err(corrupt)?; + w.finish().map_err(corrupt)?; + Ok(out) +} + +/// scrypt 的工作量（`N = 2^log_n`）。**固定，不按这台机器的快慢挑**：同步位置上的版本由 +/// 哪台设备写、哪台设备读都不一定，快的那台挑出来的工作量慢的那台可能解不开。18 是 age +/// 自己的下限，一台普通的电脑一秒以内 +fn work_factor() -> u8 { + match TEST_WORK_FACTOR.load(std::sync::atomic::Ordering::Relaxed) { + 0 => 18, + n => n, + } +} + +/// 解开时最多接受多大的工作量。比写的时候大一些：将来调高了，旧版本照样读得了新包 +const MAX_WORK_FACTOR: u8 = 22; + +/// **只给测试用**：把口令加密的工作量调低，测试不用每次等一秒。0 是不改 +#[doc(hidden)] +pub static TEST_WORK_FACTOR: std::sync::atomic::AtomicU8 = std::sync::atomic::AtomicU8::new(0); + +// ─────────────────────────────────────────────────────────── 读 + +/// 这些字节是不是加密的配置包（age 的二进制格式）。 +pub fn is_encrypted(bytes: &[u8]) -> bool { + bytes.starts_with(b"age-encryption.org/") +} + +/// 读一个配置包。 +pub fn read(bytes: &[u8], unseal: Unseal) -> Result { + if bytes.len() > MAX_BYTES { + return Err(PackError::TooLarge { limit: MAX_BYTES }); + } + let encrypted = is_encrypted(bytes); + let plain; + let zip_bytes: &[u8] = if encrypted { + plain = decrypt(bytes, unseal)?; + &plain + } else { + bytes + }; + let mut pack = unzip(zip_bytes)?; + pack.manifest.encrypted = encrypted; + pack.manifest.counts = Counts::of(&pack.config_yaml); + Ok(pack) +} + +fn decrypt(bytes: &[u8], unseal: Unseal) -> Result, PackError> { + let d = age::Decryptor::new_buffered(bytes).map_err(|e| match e { + age::DecryptError::UnknownFormat => PackError::TooNew { + format: FORMAT + 1, + core_version: String::new(), + }, + other => corrupt(other), + })?; + let reader = if d.is_scrypt() { + let Unseal::Passphrase(p) = unseal else { + return Err(PackError::NeedPassphrase); + }; + let mut id = age::scrypt::Identity::new(age::secrecy::SecretString::from(p.to_string())); + id.set_max_work_factor(MAX_WORK_FACTOR); + d.decrypt(std::iter::once(&id as &dyn age::Identity)) + .map_err(|e| match e { + age::DecryptError::DecryptionFailed | age::DecryptError::NoMatchingKeys => { + PackError::WrongPassphrase + } + other => corrupt(other), + })? + } else { + let Unseal::Identity(id) = unseal else { + return Err(PackError::NeedKey); + }; + d.decrypt(std::iter::once(id as &dyn age::Identity)) + .map_err(|e| match e { + age::DecryptError::NoMatchingKeys | age::DecryptError::DecryptionFailed => { + PackError::WrongKey + } + other => corrupt(other), + })? + }; + let mut out = Vec::new(); + reader + .take(MAX_BYTES as u64 + 1) + .read_to_end(&mut out) + .map_err(corrupt)?; + if out.len() > MAX_BYTES { + return Err(PackError::TooLarge { limit: MAX_BYTES }); + } + Ok(out) +} + +/// ZIP 里的一个路径是什么。 +enum Entry { + Manifest, + Config, + Prefs, + Script(String), + Approved(String), + /// `plugins/`、`plugins/.approved/` 这两个目录条目：别的工具打包时会带上，无害 + Dir, +} + +fn entry_of(name: &str) -> Option { + let plugin = |rest: &str| { + let id = rest.strip_suffix(".js")?; + tw_config::plugins::valid_id(id).then(|| id.to_string()) + }; + match name { + MANIFEST => Some(Entry::Manifest), + CONFIG => Some(Entry::Config), + PREFS => Some(Entry::Prefs), + PLUGINS | APPROVED => Some(Entry::Dir), + _ => { + if let Some(rest) = name.strip_prefix(APPROVED) { + plugin(rest).map(Entry::Approved) + } else if let Some(rest) = name.strip_prefix(PLUGINS) { + plugin(rest).map(Entry::Script) + } else { + None + } + } + } +} + +fn unzip(bytes: &[u8]) -> Result { + let mut z = zip::ZipArchive::new(std::io::Cursor::new(bytes)).map_err(corrupt)?; + if z.len() > MAX_ENTRIES { + return Err(corrupt(format!("it has {} entries", z.len()))); + } + let mut manifest: Option = None; + let mut config: Option = None; + let mut prefs: Option = None; + let mut scripts: BTreeMap> = BTreeMap::new(); + let mut approved: BTreeMap> = BTreeMap::new(); + let mut seen = std::collections::BTreeSet::new(); + // 解压出来的总量。**按真读到的算**：条目头里写的大小可以是假的 + let mut budget = MAX_BYTES; + for i in 0..z.len() { + let mut f = z.by_index(i).map_err(corrupt)?; + let name = f.name().map_err(corrupt)?.to_string(); + let Some(kind) = entry_of(&name) else { + return Err(PackError::BadEntry { name }); + }; + if !seen.insert(name.clone()) { + return Err(PackError::BadEntry { name }); + } + if matches!(kind, Entry::Dir) { + continue; + } + if f.is_dir() { + return Err(PackError::BadEntry { name }); + } + let mut buf = Vec::new(); + (&mut f) + .take(budget as u64 + 1) + .read_to_end(&mut buf) + .map_err(corrupt)?; + if buf.len() > budget { + return Err(PackError::TooLarge { limit: MAX_BYTES }); + } + budget -= buf.len(); + match kind { + Entry::Manifest => { + // 先按最宽的读出格式版本：更新的格式整份读不懂时，要说的是「太新」 + let raw: serde_json::Value = serde_json::from_slice(&buf) + .map_err(|e| corrupt(format!("{MANIFEST}: {e}")))?; + let format = raw.get("format").and_then(serde_json::Value::as_u64); + if let Some(f) = format + && f > u64::from(FORMAT) + { + return Err(PackError::TooNew { + format: f as u32, + core_version: raw + .get("core_version") + .and_then(serde_json::Value::as_str) + .unwrap_or_default() + .to_string(), + }); + } + manifest = Some( + serde_json::from_value(raw).map_err(|e| corrupt(format!("{MANIFEST}: {e}")))?, + ); + } + Entry::Config => { + config = Some( + String::from_utf8(buf) + .map_err(|_| corrupt(format!("{CONFIG} is not UTF-8")))?, + ); + } + Entry::Prefs => { + prefs = Some( + serde_json::from_slice(&buf).map_err(|e| corrupt(format!("{PREFS}: {e}")))?, + ); + } + Entry::Script(id) => { + scripts.insert(id, buf); + } + Entry::Approved(id) => { + approved.insert(id, buf); + } + Entry::Dir => {} + } + } + let manifest = manifest.ok_or_else(|| PackError::MissingEntry { + name: MANIFEST.to_string(), + })?; + let config_yaml = config.ok_or_else(|| PackError::MissingEntry { + name: CONFIG.to_string(), + })?; + // 只有底稿、没有脚本的不是一个插件 + if let Some(id) = approved.keys().find(|id| !scripts.contains_key(*id)) { + return Err(PackError::BadEntry { + name: format!("{APPROVED}{id}.js"), + }); + } + let plugins = scripts + .into_iter() + .map(|(id, script)| { + let approved = approved.remove(&id); + (id, PluginFiles { script, approved }) + }) + .collect(); + Ok(Pack { + manifest, + config_yaml, + plugins, + client_prefs: prefs, + }) +} + +// ─────────────────────────────────────────────────────────── 杂项 + +impl Pack { + /// 这份内容的标识：可以带走的内容（[`model::snapshot`]）规范化之后的 blake3，前 12 位 + /// 十六进制。**本机设置不算**，所以本机备份和它上传到同步位置的那一版对得上；注释、 + /// 排版、默认值写没写都不影响它。读不成配置时按原文算 + pub fn content_id(&self) -> String { + model::content_id(self) + } +} + +pub(crate) fn now_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +/// `0.77.0` 这样的版本号，比这一份 Core 新吗。读不懂的不算新 +pub fn newer_than_us(version: &str) -> bool { + match (parse_version(version), parse_version(CORE_VERSION)) { + (Some(theirs), Some(ours)) => theirs > ours, + _ => false, + } +} + +fn parse_version(v: &str) -> Option<(u64, u64, u64)> { + let core = v.trim().split(['-', '+']).next()?; + let mut it = core.split('.').map(|p| p.parse::().ok()); + Some(( + it.next()??, + it.next().flatten().unwrap_or(0), + it.next().flatten().unwrap_or(0), + )) +} + +#[cfg(test)] +mod tests; diff --git a/crates/tw-pack/src/model/codec.rs b/crates/tw-pack/src/model/codec.rs new file mode 100644 index 00000000..e354a9a1 --- /dev/null +++ b/crates/tw-pack/src/model/codec.rs @@ -0,0 +1,47 @@ +//! 插件源码怎么拆成「代码」和「数据」。 +//! +//! 插件的设置写在它自己的文件里（manifest 字面量里的出错时怎么办、范围、设置的值）。比较 +//! 时这两样分开：一台设备改了设置、另一台改了代码，各改各的，不算冲突。读 manifest 要 +//! 插件运行时那一套（编它要一个能出 wasm 的 clang），这个 crate 不依赖它：Core 启动时把 +//! 真的拆法装进来（[`super::install_codec`]），没装时整份文件当代码（[`WholeFile`]）。 + +use serde_json::Value as Json; + +/// 插件源码的拆法。 +pub trait PluginCodec: Send + Sync { + /// 拆成两份：**代码**是数据换成缺省之后的完整源码（manifest 那一段按固定的样子重写， + /// 只差数据的两份源码拆出来的代码一字不差），**数据**是出错时怎么办、范围和设置的值。 + /// 读不出 manifest 的是 None：整份当代码 + fn split(&self, source: &str) -> Option<(String, Json)>; + + /// 把数据写进源码，别的字节不动。写不进去（数据和源码声明的对不上）是 None + fn join(&self, source: &str, data: &Json) -> Option; + + /// 随这一份 Core 发的默认插件的源码。不是默认插件是 None + fn shipped(&self, id: &str) -> Option; + + /// 改得了回答里的工具调用（权限有 `reply_tool_calls`）：这种插件每台设备分别批准， + /// 别处来的版本落到本机时先不批准。读不出来的按要批准算 + fn needs_approval(&self, source: &str) -> bool; +} + +/// 不拆：整份文件是代码，没有数据，没有默认插件，一律要批准。 +pub struct WholeFile; + +impl PluginCodec for WholeFile { + fn split(&self, _source: &str) -> Option<(String, Json)> { + None + } + + fn join(&self, source: &str, data: &Json) -> Option { + data.is_null().then(|| source.to_string()) + } + + fn shipped(&self, _id: &str) -> Option { + None + } + + fn needs_approval(&self, _source: &str) -> bool { + true + } +} diff --git a/crates/tw-pack/src/model/items.rs b/crates/tw-pack/src/model/items.rs new file mode 100644 index 00000000..08615a64 --- /dev/null +++ b/crates/tw-pack/src/model/items.rs @@ -0,0 +1,728 @@ +//! 配置 → 项目，项目 → 配置里的一项，以及项目之间的引用。 + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::{Map, Value as Json}; +use serde_yaml_ng::{Mapping, Value as Yaml}; +use tw_config::{Config, Protocol, Provider}; + +use super::json::{key, to_json, to_yaml, typed_mapping}; +use super::{GUARDS, Item, ItemId, ItemKind, ModelError, PluginCodec, global, unit}; +use crate::{Pack, PluginFiles}; + +/// ChatGPT 账号登录所得的账户 ID 所在的请求头：跟着登录走，不随配置带走 +pub(crate) const ACCOUNT_HEADER: &str = tw_config::chatgpt::ACCOUNT_HEADER; + +/// 一个上游是 ChatGPT 账号。它的登录（`oauth`）和账户 ID 请求头属于这台设备 +pub(crate) fn is_chatgpt(p: &Provider) -> bool { + p.effective_protocol() == Some(Protocol::Chatgpt) +} + +fn shape(item: &ItemId, detail: impl std::fmt::Display) -> ModelError { + ModelError::Shape { + item: item.to_string(), + detail: detail.to_string(), + } +} + +// ─────────────────────────────────────────────────────────── 配置 → 项目 + +pub(crate) fn of_config( + cfg: &Config, + pack: &Pack, + codec: &dyn PluginCodec, +) -> Result, ModelError> { + let mut out = BTreeMap::new(); + let mut put = |id: ItemId, units: BTreeMap| { + out.insert( + id, + Item { + units, + ..Default::default() + }, + ); + }; + + for p in &cfg.providers { + let id = ItemId::new(ItemKind::Provider, &p.name); + let units = provider_units(p).map_err(|e| shape(&id, e))?; + put(id, units); + } + order( + &mut put, + ItemKind::ProviderOrder, + cfg.providers.iter().map(|p| &p.name), + ); + + for px in &cfg.proxies { + let id = ItemId::new(ItemKind::Proxy, &px.name); + let units = whole(px, "name").map_err(|e| shape(&id, e))?; + put(id, units); + } + for c in &cfg.clients { + let id = ItemId::new(ItemKind::Key, &c.name); + let units = whole(c, "name").map_err(|e| shape(&id, e))?; + put(id, units); + } + for a in cfg.aliases.iter() { + let id = ItemId::new(ItemKind::Alias, &a.name); + put( + id, + one(Json::Array( + a.models.iter().cloned().map(Json::String).collect(), + )), + ); + } + for g in &cfg.groups { + let id = ItemId::new(ItemKind::Group, &g.name); + let units = whole(g, "name").map_err(|e| shape(&id, e))?; + put(id, units); + } + for r in &cfg.routes { + let id = ItemId::new(ItemKind::Route, &r.name); + let units = whole(r, "name").map_err(|e| shape(&id, e))?; + put(id, units); + } + order( + &mut put, + ItemKind::RouteOrder, + cfg.routes.iter().map(|r| &r.name), + ); + for s in &cfg.pricing.sheets { + let id = ItemId::new(ItemKind::PriceSheet, &s.name); + let units = whole(s, "name").map_err(|e| shape(&id, e))?; + put(id, units); + } + let rules: [(&str, Vec); 3] = [ + ("redact", rule_values(&cfg.security.redact.custom)), + ( + "inspect_tools", + rule_values(&cfg.security.inspect_tools.custom), + ), + ("content", rule_values(&cfg.security.content.custom)), + ]; + for (guard, list) in rules { + for v in list { + let Json::Object(mut o) = v else { continue }; + let name = o + .remove("name") + .and_then(|n| n.as_str().map(str::to_string)) + .unwrap_or_default(); + put( + ItemId::new(ItemKind::SecurityRule, format!("{guard}/{name}")), + one(Json::Object(o)), + ); + } + } + + for p in &cfg.plugins { + let id = ItemId::new(ItemKind::Plugin, &p.id); + let files = pack.plugins.get(&p.id); + let (units, files) = plugin_units(p, files, codec); + out.insert( + id, + Item { + units, + files, + ..Default::default() + }, + ); + } + let mut put = |id: ItemId, units: BTreeMap| { + out.insert( + id, + Item { + units, + ..Default::default() + }, + ); + }; + order( + &mut put, + ItemKind::PluginOrder, + cfg.plugins.iter().map(|p| &p.id), + ); + + for (name, units) in + global_units(cfg).map_err(|e| shape(&ItemId::new(ItemKind::Global, ""), e))? + { + if !units.is_empty() { + put(ItemId::new(ItemKind::Global, name), units); + } + } + + if let Some(prefs) = &pack.client_prefs { + let mut units = BTreeMap::new(); + super::json::flatten("", prefs, &mut units); + if !units.is_empty() && prefs != &Json::Object(Default::default()) { + put(ItemId::new(ItemKind::ClientPrefs, ""), units); + } + } + Ok(out) +} + +fn one(v: Json) -> BTreeMap { + BTreeMap::from([(unit::ALL.to_string(), v)]) +} + +fn order<'a>( + put: &mut impl FnMut(ItemId, BTreeMap), + kind: ItemKind, + names: impl Iterator, +) { + let list: Vec = names.cloned().map(Json::String).collect(); + if !list.is_empty() { + put(ItemId::new(kind, ""), one(Json::Array(list))); + } +} + +/// 整个对象是一个单位，名字不在里面（名字是项目的标识） +fn whole(v: &T, name_key: &str) -> Result, String> { + match to_json(v)? { + Json::Object(mut o) => { + o.remove(name_key); + Ok(one(Json::Object(o))) + } + _ => Err("it does not serialize to an object".into()), + } +} + +fn rule_values(rules: &[T]) -> Vec { + rules.iter().filter_map(|r| to_json(r).ok()).collect() +} + +// ─────────────────────────────────────────────────────────── 上游 + +/// 上游的一个字段归哪个单位。**不认识的归 `other`**：将来新加的字段照样参与比较 +fn provider_unit(field: &str) -> &'static str { + match field { + "base_url" | "protocol" | "key" | "headers" | "oauth" | "aws" | "proxy" + | "on_proxy_fail" | "signed_in" => unit::CONN, + "models" | "models_only" | "model_specs" => unit::MODELS, + "billing" | "pricing" => unit::BILLING, + _ => unit::OTHER, + } +} + +pub(crate) const PROVIDER_UNITS: &[&str] = &[unit::CONN, unit::MODELS, unit::BILLING, unit::OTHER]; + +/// 去掉这台设备自己的部分：ChatGPT 的登录和账户 ID 请求头 +pub(crate) fn shareable(p: &Provider) -> Provider { + let mut p = p.clone(); + if is_chatgpt(&p) { + p.oauth = None; + p.headers = tw_config::Headers::new( + p.headers + .iter() + .filter(|h| !h.name.eq_ignore_ascii_case(ACCOUNT_HEADER)) + .cloned() + .collect(), + ); + } + p +} + +fn provider_units(p: &Provider) -> Result, String> { + let p = shareable(p); + let Json::Object(mut obj) = to_json(&p)? else { + return Err("it does not serialize to an object".into()); + }; + obj.remove("name"); + // **请求头按写的顺序**：JSON 对象的键是排过序的，所以写成一串 [名字, 值] + if obj.contains_key("headers") { + obj.insert( + "headers".into(), + Json::Array( + p.headers + .iter() + .map(|h| { + Json::Array(vec![ + Json::String(h.name.clone()), + Json::String(h.value.raw().to_string()), + ]) + }) + .collect(), + ), + ); + } + let mut units: BTreeMap> = PROVIDER_UNITS + .iter() + .map(|u| (u.to_string(), Map::new())) + .collect(); + for (k, v) in obj { + units + .get_mut(provider_unit(&k)) + .expect("every unit is there") + .insert(k, v); + } + Ok(units + .into_iter() + .map(|(k, v)| (k, Json::Object(v))) + .collect()) +} + +/// 上游的单位拼回配置里的那一项。`local` 是本机同名的那一家：它是 ChatGPT 账号、拼出来 +/// 的也是的话，登录和账户 ID 请求头取它的 +pub(crate) fn provider_mapping( + id: &ItemId, + units: &BTreeMap, + local: Option<&Provider>, +) -> Result<(Provider, Mapping), ModelError> { + let mut m = Mapping::new(); + m.insert(key("name"), Yaml::String(id.name.clone())); + for (u, v) in units { + let Json::Object(o) = v else { + return Err(shape(id, format!("unit `{u}` is not an object"))); + }; + for (k, v) in o { + let value = if k == "headers" { + headers_yaml(v).map_err(|e| shape(id, e))? + } else { + to_yaml(v) + }; + m.insert(key(k), value); + } + } + let (mut p, _) = typed_mapping::(Yaml::Mapping(m)).map_err(|e| shape(id, e))?; + if let Some(local) = local + && is_chatgpt(&p) + && is_chatgpt(local) + { + p.oauth = local.oauth.clone(); + if let Some(h) = local.headers.get(ACCOUNT_HEADER) { + let mut all: Vec = p + .headers + .iter() + .filter(|x| !x.name.eq_ignore_ascii_case(ACCOUNT_HEADER)) + .cloned() + .collect(); + all.push(h.clone()); + p.headers = tw_config::Headers::new(all); + } + } + let m = match serde_yaml_ng::to_value(&p) { + Ok(Yaml::Mapping(m)) => m, + _ => return Err(shape(id, "it does not serialize to a mapping")), + }; + Ok((p, m)) +} + +/// `[[名字, 值], …]` → 按这个顺序的 YAML 映射 +fn headers_yaml(v: &Json) -> Result { + let Json::Array(pairs) = v else { + return Err("headers is not a list of pairs".into()); + }; + let mut m = Mapping::new(); + for p in pairs { + match p.as_array().map(Vec::as_slice) { + Some([Json::String(n), Json::String(v)]) => { + m.insert(key(n), Yaml::String(v.clone())); + } + _ => return Err("headers is not a list of pairs".into()), + } + } + Ok(Yaml::Mapping(m)) +} + +// ─────────────────────────────────────────────────────────── 别的有名字的 + +/// 一个只有 `all` 的项目拼回配置里的那一项（名字键叫 `name_key`），按类型 `T` 核过、 +/// 写成它的字段顺序 +pub(crate) fn whole_mapping( + id: &ItemId, + name: &str, + units: &BTreeMap, +) -> Result<(T, Mapping), ModelError> +where + T: serde::de::DeserializeOwned + serde::Serialize, +{ + let mut m = Mapping::new(); + m.insert(key("name"), Yaml::String(name.to_string())); + match units.get(unit::ALL) { + Some(Json::Object(o)) => { + for (k, v) in o { + m.insert(key(k), to_yaml(v)); + } + } + Some(_) => return Err(shape(id, "unit `all` is not an object")), + None => return Err(shape(id, "unit `all` is missing")), + } + typed_mapping::(Yaml::Mapping(m)).map_err(|e| shape(id, e)) +} + +/// 别名的模型列表 +pub(crate) fn alias_models( + id: &ItemId, + units: &BTreeMap, +) -> Result, ModelError> { + match units.get(unit::ALL) { + Some(Json::Array(a)) => a + .iter() + .map(|m| { + m.as_str() + .map(str::to_string) + .ok_or_else(|| shape(id, "a model is not a string")) + }) + .collect(), + _ => Err(shape(id, "unit `all` is not a list of models")), + } +} + +/// 顺序项目里的名字 +pub(crate) fn order_names(units: &BTreeMap) -> Vec { + match units.get(unit::ALL) { + Some(Json::Array(a)) => a + .iter() + .filter_map(|n| n.as_str().map(str::to_string)) + .collect(), + _ => Vec::new(), + } +} + +// ─────────────────────────────────────────────────────────── 插件 + +/// 一个插件的单位，和带着走的文件。 +/// +/// **比较的是批准过的那一份**：文件被人改了、还没批准时，脚本那个单位按底稿算 —— 没批准的 +/// 改动不该借导出或同步跑到别的设备上，在那里被当成批准过的。底稿和配置里的哈希都对不上时 +/// 按文件算（导入时这样的插件是错误） +fn plugin_units( + entry: &tw_config::Plugin, + files: Option<&PluginFiles>, + codec: &dyn PluginCodec, +) -> (BTreeMap, Option) { + let mut units = BTreeMap::new(); + units.insert(unit::ENABLED.to_string(), Json::Bool(entry.enabled)); + let Some(files) = files else { + return (units, None); + }; + let source = approved_source(entry, files); + let text = String::from_utf8_lossy(source).into_owned(); + let (code, data) = match codec.split(&text) { + Some((code, data)) => (code, Some(data)), + None => (text, None), + }; + let shipped = codec + .shipped(&entry.id) + .map(|s| codec.split(&s).map_or(s, |(c, _)| c)); + // 随 Core 发的默认插件、代码没动过：脚本由各设备自己的 Core 给，不比较 + if shipped.as_deref() != Some(code.as_str()) { + units.insert(unit::SCRIPT.to_string(), Json::String(code)); + } + if let Some(d) = data { + units.insert(unit::SETTINGS.to_string(), d); + } + (units, Some(files.clone())) +} + +/// 批准过的那一份：哈希对得上配置的脚本或底稿。都对不上是脚本 +pub(crate) fn approved_source<'a>(entry: &tw_config::Plugin, files: &'a PluginFiles) -> &'a [u8] { + if sha256_hex(&files.script) == entry.sha256 { + return &files.script; + } + match &files.approved { + Some(a) if sha256_hex(a) == entry.sha256 => a, + _ => &files.script, + } +} + +pub(crate) fn sha256_hex(bytes: &[u8]) -> String { + use sha2::Digest; + let d = sha2::Sha256::digest(bytes); + d.iter().map(|b| format!("{b:02x}")).collect() +} + +// ─────────────────────────────────────────────────────────── 全局设置 + +/// 每一节的单位：等于默认值的字段不在里面 +/// 一节全局设置的单位 +type Units = BTreeMap; + +fn global_units(cfg: &Config) -> Result, String> { + let mut out = Vec::new(); + out.push(( + global::FAILOVER, + fields(global::FAILOVER, &to_json(&cfg.failover)?), + )); + let mut sec = BTreeMap::new(); + if let Json::Object(o) = to_json(&cfg.security)? { + for (guard, v) in o { + let Json::Object(fields) = v else { continue }; + for (f, v) in fields { + if f == "custom" { + continue; + } + match (f.as_str(), v) { + ("actions", Json::Object(actions)) => { + for (rule, a) in actions { + sec.insert(format!("{}.{guard}.actions.{rule}", global::SECURITY), a); + } + } + (_, v) => { + sec.insert(format!("{}.{guard}.{f}", global::SECURITY), v); + } + } + } + } + } + out.push((global::SECURITY, sec)); + let scalar = |name: &str, v: Option<&String>| -> BTreeMap { + v.map(|v| BTreeMap::from([(name.to_string(), Json::String(v.clone()))])) + .unwrap_or_default() + }; + out.push(( + global::DEFAULT_ROUTE, + scalar(global::DEFAULT_ROUTE, cfg.default_route.as_ref()), + )); + out.push(( + global::DEFAULT_KEY, + scalar(global::DEFAULT_KEY, cfg.default_key.as_ref()), + )); + out.push(( + global::CLIENT_PROBES, + fields(global::CLIENT_PROBES, &to_json(&cfg.client_probes)?), + )); + let mut pricing = to_json(&cfg.pricing)?; + if let Json::Object(o) = &mut pricing { + o.remove("sheets"); + } + out.push((global::PRICING, fields(global::PRICING, &pricing))); + for (_, units) in out.iter_mut() { + units.retain(|path, v| !is_global_default(path, v)); + } + Ok(out) +} + +fn fields(section: &str, v: &Json) -> BTreeMap { + match v { + Json::Object(o) => o + .iter() + .map(|(k, v)| (format!("{section}.{k}"), v.clone())) + .collect(), + _ => BTreeMap::new(), + } +} + +/// 全局设置的一个单位按路径拆开：节名，和节里的那几步 +pub(crate) fn global_path(path: &str) -> Vec { + let limit = if path.starts_with("security.") { 4 } else { 2 }; + path.splitn(limit, '.').map(str::to_string).collect() +} + +/// 一节的默认值写成 JSON，和读进它的类型。节名不认识是 None +fn section_default(section: &str) -> Option { + match section { + global::FAILOVER => to_json(&tw_config::Failover::default()).ok(), + global::SECURITY => to_json(&tw_config::Security::default()).ok(), + global::CLIENT_PROBES => to_json(&tw_config::ClientProbes::default()).ok(), + global::PRICING => to_json(&tw_pricing_default()).ok(), + _ => None, + } +} + +fn tw_pricing_default() -> Json { + let mut d = to_json(&tw_pricing::PricingConfig::default()).unwrap_or(Json::Null); + if let Json::Object(o) = &mut d { + o.remove("sheets"); + } + d +} + +/// 一节的 JSON 读进它的类型，再写出来：默认值去掉。读不进去是 Err +pub(crate) fn section_normalized(section: &str, v: &Json) -> Result { + fn round(v: &Json) -> Result { + let t: T = serde_json::from_value(v.clone()).map_err(|e| e.to_string())?; + to_json(&t) + } + match section { + global::FAILOVER => round::(v), + global::SECURITY => round::(v), + global::CLIENT_PROBES => round::(v), + global::PRICING => { + let mut out = round::(v)?; + if let Json::Object(o) = &mut out { + o.remove("sheets"); + } + Ok(out) + } + other => Err(format!("there is no section `{other}`")), + } +} + +/// 全局设置的一个单位是不是等于默认值。**按类型判断**：把它写进默认的那一节、读进类型 +/// 再写出来，和默认的那一节一样就是默认值 —— 序列化时省掉的默认值也认得出 +pub(crate) fn is_global_default(path: &str, v: &Json) -> bool { + let parts = global_path(path); + match parts.as_slice() { + [s] if s == global::DEFAULT_ROUTE => v.as_str() == Some(tw_engine::DEFAULT_ROUTE), + [s] if s == global::DEFAULT_KEY => v.as_str() == Some(tw_config::DEFAULT_KEY), + [section, rest @ ..] if !rest.is_empty() => { + let Some(default) = section_default(section) else { + return false; + }; + let mut with = default.clone(); + set_path(&mut with, rest, v.clone()); + match section_normalized(section, &with) { + Ok(n) => n == default, + Err(_) => false, + } + } + _ => false, + } +} + +/// 在一棵 JSON 里按路径写一个值，缺的对象补出来 +pub(crate) fn set_path(root: &mut Json, path: &[String], v: Json) { + let mut cur = root; + for (i, p) in path.iter().enumerate() { + if !cur.is_object() { + *cur = Json::Object(Default::default()); + } + let o = cur.as_object_mut().expect("just made it an object"); + if i + 1 == path.len() { + o.insert(p.clone(), v); + return; + } + cur = o + .entry(p.clone()) + .or_insert_with(|| Json::Object(Default::default())); + } +} + +// ─────────────────────────────────────────────────────────── 引用 + +pub(crate) fn relink(items: &mut BTreeMap) { + let names = |kind: ItemKind| -> BTreeSet { + items + .keys() + .filter(|id| id.kind == kind) + .map(|id| id.name.clone()) + .collect() + }; + let groups = names(ItemKind::Group); + let aliases = names(ItemKind::Alias); + for (id, item) in items.iter_mut() { + item.refs = refs_of(id, &item.units, &groups, &aliases); + } +} + +fn refs_of( + id: &ItemId, + units: &BTreeMap, + groups: &BTreeSet, + aliases: &BTreeSet, +) -> BTreeSet { + let mut out = BTreeSet::new(); + let s = |v: Option<&Json>| v.and_then(Json::as_str).map(str::to_string); + let all = units.get(unit::ALL); + let get = |v: Option<&Json>, k: &str| v.and_then(|v| v.get(k)).cloned(); + match id.kind { + ItemKind::Provider => { + if let Some(px) = s(units.get(unit::CONN).and_then(|c| c.get("proxy"))) + && px != tw_config::DIRECT + && px != tw_config::SYSTEM + { + out.insert(ItemId::new(ItemKind::Proxy, px)); + } + if let Some(sheet) = s(units.get(unit::BILLING).and_then(|c| c.get("pricing"))) { + out.insert(ItemId::new(ItemKind::PriceSheet, sheet)); + } + } + ItemKind::Key => { + if let Some(r) = s(get(all, "route").as_ref()) { + out.insert(ItemId::new(ItemKind::Route, r)); + } + for m in get(all, "allow") + .and_then(|a| a.as_array().cloned()) + .unwrap_or_default() + { + if let Some(m) = m.as_str() + && aliases.contains(m) + { + out.insert(ItemId::new(ItemKind::Alias, m)); + } + } + } + ItemKind::Group => { + for m in get(all, "providers") + .and_then(|a| a.as_array().cloned()) + .unwrap_or_default() + { + let name = m.as_str().map(str::to_string).or_else(|| s(m.get("name"))); + if let Some(n) = name { + out.insert(ItemId::new(ItemKind::Provider, n)); + } + } + if let Some(sel) = s(get(all, "selected").as_ref()) { + out.insert(ItemId::new(ItemKind::Provider, sel)); + } + } + ItemKind::Route => { + for rule in get(all, "rules") + .and_then(|a| a.as_array().cloned()) + .unwrap_or_default() + { + match rule.get("to") { + Some(Json::String(n)) if n == tw_engine::ALL_UPSTREAMS => {} + Some(Json::String(n)) if groups.contains(n) => { + out.insert(ItemId::new(ItemKind::Group, n)); + } + Some(Json::String(n)) => { + out.insert(ItemId::new(ItemKind::Provider, n)); + } + Some(Json::Array(pinned)) => { + for p in pinned { + if let Some(n) = s(p.get("provider")) { + out.insert(ItemId::new(ItemKind::Provider, n)); + } + } + } + _ => {} + } + let when = rule.get("when"); + let phase_two = when.and_then(|w| w.get("provider_would_be")); + match phase_two { + Some(Json::String(n)) => { + out.insert(ItemId::new(ItemKind::Provider, n)); + } + Some(Json::Array(list)) => { + for n in list.iter().filter_map(Json::as_str) { + out.insert(ItemId::new(ItemKind::Provider, n)); + } + } + _ => {} + } + if let Some(k) = s(when.and_then(|w| w.get("client"))) { + out.insert(ItemId::new(ItemKind::Key, k)); + } + if let Some(m) = s(when.and_then(|w| w.get("model"))) + && aliases.contains(&m) + { + out.insert(ItemId::new(ItemKind::Alias, m)); + } + if phase_two.is_none() + && let Some(m) = s(rule.get("set").and_then(|w| w.get("model"))) + && aliases.contains(&m) + { + out.insert(ItemId::new(ItemKind::Alias, m)); + } + } + } + ItemKind::Global => { + if let Some(r) = s(units.get(global::DEFAULT_ROUTE)) { + out.insert(ItemId::new(ItemKind::Route, r)); + } + if let Some(k) = s(units.get(global::DEFAULT_KEY)) { + out.insert(ItemId::new(ItemKind::Key, k)); + } + } + _ => {} + } + out +} + +/// 自定义安全规则的项目名拆成分项和规则名 +pub(crate) fn rule_name(name: &str) -> Option<(&str, &str)> { + let (guard, rule) = name.split_once('/')?; + GUARDS.contains(&guard).then_some((guard, rule)) +} diff --git a/crates/tw-pack/src/model/json.rs b/crates/tw-pack/src/model/json.rs new file mode 100644 index 00000000..fb759d5c --- /dev/null +++ b/crates/tw-pack/src/model/json.rs @@ -0,0 +1,131 @@ +//! JSON 与 YAML 之间来回，和按路径摊平一棵 JSON。 +//! +//! 单位的值是 JSON（规范化、键有序）；写回配置用的是 YAML 的映射（键按结构体的字段顺序）。 +//! 两边的标量一一对应：字符串、整数、浮点数、布尔、空。 + +use serde_json::Value as Json; +use serde_yaml_ng::{Mapping, Value as Yaml}; + +/// 一个可序列化的值 → JSON。失败是我们自己的类型写坏了 +pub(crate) fn to_json(v: &T) -> Result { + serde_json::to_value(v).map_err(|e| e.to_string()) +} + +/// JSON → YAML。对象的键按 JSON 里的顺序（也就是字母序） +pub(crate) fn to_yaml(v: &Json) -> Yaml { + match v { + Json::Null => Yaml::Null, + Json::Bool(b) => Yaml::Bool(*b), + Json::Number(n) => { + if let Some(u) = n.as_u64() { + Yaml::Number(u.into()) + } else if let Some(i) = n.as_i64() { + Yaml::Number(i.into()) + } else { + Yaml::Number(n.as_f64().unwrap_or(0.0).into()) + } + } + Json::String(s) => Yaml::String(s.clone()), + Json::Array(a) => Yaml::Sequence(a.iter().map(to_yaml).collect()), + Json::Object(m) => Yaml::Mapping( + m.iter() + .map(|(k, v)| (Yaml::String(k.clone()), to_yaml(v))) + .collect(), + ), + } +} + +/// 一个 JSON 对象（单位拼起来的样子）→ 某个配置类型，再写成 YAML 映射：键按结构体的 +/// 字段顺序，默认值不写。读不进那个类型就是这些单位不成形 +pub(crate) fn typed_mapping(fields: Yaml) -> Result<(T, Mapping), String> +where + T: serde::de::DeserializeOwned + serde::Serialize, +{ + let typed: T = serde_yaml_ng::from_value(fields).map_err(|e| e.to_string())?; + match serde_yaml_ng::to_value(&typed).map_err(|e| e.to_string())? { + Yaml::Mapping(m) => Ok((typed, m)), + _ => Err("it does not serialize to a mapping".into()), + } +} + +/// 摊平：对象一层层展开成 `a.b.c` → 值；数组和别的标量是叶子，空对象也是叶子。 +/// 键里的 `~` 和 `.` 转义成 `~0`、`~1` +pub(crate) fn flatten(prefix: &str, v: &Json, out: &mut std::collections::BTreeMap) { + match v { + Json::Object(m) if !m.is_empty() => { + for (k, v) in m { + let k = k.replace('~', "~0").replace('.', "~1"); + let path = if prefix.is_empty() { + k + } else { + format!("{prefix}.{k}") + }; + flatten(&path, v, out); + } + } + other => { + out.insert(prefix.to_string(), other.clone()); + } + } +} + +/// [`flatten`] 反过来。空路径是整棵树本身 +pub(crate) fn unflatten(units: &std::collections::BTreeMap) -> Json { + if let Some(root) = units.get("") { + return root.clone(); + } + let mut root = serde_json::Map::new(); + for (path, v) in units { + let parts: Vec = path + .split('.') + .map(|p| p.replace("~1", ".").replace("~0", "~")) + .collect(); + let mut cur = &mut root; + for (i, p) in parts.iter().enumerate() { + if i + 1 == parts.len() { + cur.insert(p.clone(), v.clone()); + } else { + let next = cur + .entry(p.clone()) + .or_insert_with(|| Json::Object(Default::default())); + if !next.is_object() { + *next = Json::Object(Default::default()); + } + cur = next.as_object_mut().expect("just made it an object"); + } + } + } + Json::Object(root) +} + +/// 映射里的一个键 +pub(crate) fn key(k: &str) -> Yaml { + Yaml::String(k.to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn flattening_and_back_gives_the_same_tree() { + for v in [ + json!({"language": "zh", "menubar": {"show": "tokens", "rows": 2}, "list": [1, 2], "empty": {}}), + json!({"a.b": {"c~d": true}}), + json!("just a string"), + ] { + let mut flat = Default::default(); + flatten("", &v, &mut flat); + assert_eq!(unflatten(&flat), v, "{flat:?}"); + } + } + + #[test] + fn numbers_keep_their_kind_through_yaml() { + let v = json!({"u": 3, "i": -2, "f": 1.5}); + let y = to_yaml(&v); + let back: Json = serde_json::to_value(&y).unwrap(); + assert_eq!(back, v); + } +} diff --git a/crates/tw-pack/src/model/mod.rs b/crates/tw-pack/src/model/mod.rs new file mode 100644 index 00000000..ce9efe29 --- /dev/null +++ b/crates/tw-pack/src/model/mod.rs @@ -0,0 +1,533 @@ +//! 一份配置拆成「项目」与「合并单位」，以及反过来把目标状态落回配置文本。 +//! +//! 导入（两方比较）、同步（三方合并）、恢复（按项目挑）都只在这一层上比较：比的是 +//! [`Snapshot`]，不是 YAML 原文。原文里的注释、排版、默认值写没写都不算差别。 +//! +//! # 项目 +//! +//! 一个项目是配置里有名字的一样东西（[`ItemId`]）：上游、出站代理、网关密钥、别名、策略组、 +//! 路由、价目表、自定义安全规则、插件，加上几样没有名字的：上游、路由、插件各自的先后顺序 +//! （顺序有意义：没写路由时按上游的顺序故障转移，插件按顺序运行），全局设置的每一节， +//! Lite 的通用偏好。 +//! +//! # 合并单位 +//! +//! 一个项目里能各自合并的部分（[`Item::units`]）。值是**规范化**之后的 JSON：按配置的类型 +//! 读进来再写出去，默认值不写，同样的内容总是同样的值。 +//! +//! - 上游：`conn`（地址、协议、密钥、请求头、出站代理、AWS、认证方式）、`models`、 +//! `billing`、`other`（并发、转发客户端身份、余额来源、停用，以及将来新加的字段） +//! - 插件：`script`（数据换成缺省之后的源码）、`settings`（出错时怎么办、范围、设置的值）、 +//! `enabled` +//! - 全局设置、通用偏好：逐个字段，单位名是字段的路径（`failover.pause_secs`）。**等于 +//! 默认值的字段不在里面**，一节全是默认值时这一项就不在快照里 +//! - 其余：一个单位 `all` +//! +//! # 本机设置 +//! +//! 不进快照，[`materialize`] 时保留本机的值：`listen`、`retention`、`sync` 三节，ChatGPT +//! 上游的登录（`oauth`）和登录所得的 `ChatGPT-Account-Id` 请求头，Core 自带的默认插件的 +//! 脚本（它的设置和开关照常比较）。 + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::OnceLock; + +use serde::{Deserialize, Serialize}; +use serde_json::Value as Json; +use tw_types::{Msg, msg}; + +use crate::{Pack, PluginFiles}; + +mod codec; +mod items; +mod json; +mod rename; +mod write; + +pub use codec::{PluginCodec, WholeFile}; +pub use rename::{free_name, rename}; +pub use write::{materialize, materialize_with}; + +/// 配置里属于这台设备、不随导出和同步带走的几节 +pub const LOCAL_SECTIONS: &[&str] = &["listen", "retention", "sync"]; + +/// 合并单位的名字。 +pub mod unit { + /// 整个项目是一个单位 + pub const ALL: &str = "all"; + /// 上游：服务类型、地址、协议、认证方式、密钥、请求头、出站代理、AWS + pub const CONN: &str = "conn"; + /// 上游：模型列表、只用这些模型、手动规格 + pub const MODELS: &str = "models"; + /// 上游：计费方式、价目表 + pub const BILLING: &str = "billing"; + /// 上游：并发上限、转发客户端身份、余额来源、停用 + pub const OTHER: &str = "other"; + /// 插件：数据换成缺省之后的源码 + pub const SCRIPT: &str = "script"; + /// 插件：出错时怎么办、范围、设置的值 + pub const SETTINGS: &str = "settings"; + /// 插件：开着没有 + pub const ENABLED: &str = "enabled"; +} + +/// 全局设置各节的名字（[`ItemKind::Global`] 的 `name`）。 +pub mod global { + pub const FAILOVER: &str = "failover"; + pub const SECURITY: &str = "security"; + pub const DEFAULT_ROUTE: &str = "default_route"; + pub const DEFAULT_KEY: &str = "default_key"; + pub const CLIENT_PROBES: &str = "client_probes"; + pub const PRICING: &str = "pricing"; + /// 全部，按这个顺序 + pub const ALL: &[&str] = &[ + FAILOVER, + SECURITY, + DEFAULT_ROUTE, + DEFAULT_KEY, + CLIENT_PROBES, + PRICING, + ]; +} + +/// 自定义安全规则分在哪一项防护下：`security.<分项>.custom` +pub const GUARDS: &[&str] = &["redact", "inspect_tools", "content"]; + +/// 项目的种类。**先后就是排序**：列表、预览里按这个顺序。 +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ItemKind { + Provider, + Proxy, + /// 网关密钥（配置里的 `clients`） + Key, + Alias, + Group, + Route, + PriceSheet, + /// `name` 是 `<分项>/<规则名>`，如 `content/项目代号` + SecurityRule, + Plugin, + /// 上游的先后。`name` 是空的 + ProviderOrder, + /// 路由的先后。`name` 是空的 + RouteOrder, + /// 插件的先后（运行的顺序）。`name` 是空的 + PluginOrder, + /// `name` 是节名，见 [`global`] + Global, + /// Lite 的通用偏好。`name` 是空的 + ClientPrefs, +} + +impl ItemKind { + pub const ALL: &'static [ItemKind] = &[ + ItemKind::Provider, + ItemKind::Proxy, + ItemKind::Key, + ItemKind::Alias, + ItemKind::Group, + ItemKind::Route, + ItemKind::PriceSheet, + ItemKind::SecurityRule, + ItemKind::Plugin, + ItemKind::ProviderOrder, + ItemKind::RouteOrder, + ItemKind::PluginOrder, + ItemKind::Global, + ItemKind::ClientPrefs, + ]; + + pub fn slug(&self) -> &'static str { + match self { + ItemKind::Provider => "provider", + ItemKind::Proxy => "proxy", + ItemKind::Key => "key", + ItemKind::Alias => "alias", + ItemKind::Group => "group", + ItemKind::Route => "route", + ItemKind::PriceSheet => "price_sheet", + ItemKind::SecurityRule => "security_rule", + ItemKind::Plugin => "plugin", + ItemKind::ProviderOrder => "provider_order", + ItemKind::RouteOrder => "route_order", + ItemKind::PluginOrder => "plugin_order", + ItemKind::Global => "global", + ItemKind::ClientPrefs => "client_prefs", + } + } + + pub fn from_slug(s: &str) -> Option { + ItemKind::ALL.iter().copied().find(|k| k.slug() == s) + } + + /// 一种先后顺序。它们不是用户挑的项目：导入时新加的项目接在后面，恢复、同步时整个比 + pub fn is_order(&self) -> bool { + matches!( + self, + ItemKind::ProviderOrder | ItemKind::RouteOrder | ItemKind::PluginOrder + ) + } + + /// 有名字、可以改名的那几种 + pub fn is_named(&self) -> bool { + !self.is_order() && !matches!(self, ItemKind::Global | ItemKind::ClientPrefs) + } + + /// 这种项目的先后由哪一种顺序管 + pub fn order(&self) -> Option { + match self { + ItemKind::Provider => Some(ItemKind::ProviderOrder), + ItemKind::Route => Some(ItemKind::RouteOrder), + ItemKind::Plugin => Some(ItemKind::PluginOrder), + _ => None, + } + } +} + +/// 一个项目是谁：种类加名字。 +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +pub struct ItemId { + pub kind: ItemKind, + pub name: String, +} + +impl ItemId { + pub fn new(kind: ItemKind, name: impl Into) -> Self { + Self { + kind, + name: name.into(), + } + } +} + +impl std::fmt::Display for ItemId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if self.name.is_empty() { + f.write_str(self.kind.slug()) + } else { + write!(f, "{} `{}`", self.kind.slug(), self.name) + } + } +} + +/// 一个项目：它的合并单位，和它引用的项目。 +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct Item { + /// 单位名 → 规范化的值 + pub units: BTreeMap, + /// 它引用的项目（路由 → 上游、上游 → 出站代理……）。**按单位算出来的**，单位改了要 + /// [`Snapshot::relink`]。被引用的那一项不一定在快照里：引用了不存在的东西，导入时 + /// 按这个说出缺的是什么 + pub refs: BTreeSet, + /// 插件的原文件，**不参与比较**：落盘时优先用它，字节原样；和单位对不上、或者没有时 + /// 按单位重新拼出源码（manifest 那一段按固定的样子写） + #[serde(skip)] + pub files: Option, +} + +/// 比较只看单位和引用：同样的内容从两份文件读出来，原文件的字节可以不同 +impl PartialEq for Item { + fn eq(&self, other: &Self) -> bool { + self.units == other.units && self.refs == other.refs + } +} + +/// 一份配置拆开之后的样子。 +#[derive(Debug, Clone, Default, PartialEq)] +pub struct Snapshot { + pub items: BTreeMap, +} + +/// 写成 JSON 时是一个列表：`ItemId` 当不了 JSON 对象的键 +impl Serialize for Snapshot { + fn serialize(&self, s: S) -> Result { + #[derive(Serialize)] + struct Entry<'a> { + kind: ItemKind, + name: &'a str, + units: &'a BTreeMap, + #[serde(skip_serializing_if = "BTreeSet::is_empty")] + refs: &'a BTreeSet, + } + s.collect_seq(self.items.iter().map(|(id, it)| Entry { + kind: id.kind, + name: &id.name, + units: &it.units, + refs: &it.refs, + })) + } +} + +impl<'de> Deserialize<'de> for Snapshot { + fn deserialize>(d: D) -> Result { + #[derive(Deserialize)] + struct Entry { + kind: ItemKind, + name: String, + #[serde(default)] + units: BTreeMap, + #[serde(default)] + refs: BTreeSet, + } + let entries = Vec::::deserialize(d)?; + Ok(Snapshot { + items: entries + .into_iter() + .map(|e| { + ( + ItemId::new(e.kind, e.name), + Item { + units: e.units, + refs: e.refs, + files: None, + }, + ) + }) + .collect(), + }) + } +} + +impl Snapshot { + pub fn get(&self, id: &ItemId) -> Option<&Item> { + self.items.get(id) + } + + /// 某一种的全部项目，按名字 + pub fn of_kind(&self, kind: ItemKind) -> impl Iterator { + self.items.iter().filter(move |(id, _)| id.kind == kind) + } + + /// 这一项连同它（直接或间接）引用的、快照里有的项目。导出所选项目时用：路由带上 + /// 它的上游，上游带上它的出站代理 + pub fn closure(&self, roots: &[ItemId]) -> BTreeSet { + let mut out = BTreeSet::new(); + let mut todo: Vec = roots.to_vec(); + while let Some(id) = todo.pop() { + let Some(item) = self.items.get(&id) else { + continue; + }; + if !out.insert(id) { + continue; + } + todo.extend(item.refs.iter().cloned()); + } + out + } + + /// 按单位重算每一项的引用。改了单位之后（合并、改名）调它 + pub fn relink(&mut self) { + items::relink(&mut self.items); + } + + /// 只留下这几项，**顺序项目跟着裁**：只剩留下来的那些名字 + pub fn retain(&mut self, keep: &BTreeSet) { + let names = |kind: ItemKind| -> BTreeSet { + keep.iter() + .filter(|id| id.kind == kind) + .map(|id| id.name.clone()) + .collect() + }; + let mut orders = BTreeMap::new(); + for kind in [ItemKind::Provider, ItemKind::Route, ItemKind::Plugin] { + orders.insert(kind.order().unwrap_or(kind), names(kind)); + } + self.items.retain(|id, item| { + if let Some(allowed) = orders.get(&id.kind) { + if let Some(Json::Array(list)) = item.units.get_mut(unit::ALL) { + list.retain(|n| n.as_str().is_some_and(|n| allowed.contains(n))); + return !list.is_empty(); + } + return false; + } + keep.contains(id) + }); + } +} + +/// 拆不开、或者落不回去。**英文只写一遍**：`Display` 就是 [`ModelError::msg`] 的原句。 +#[derive(Debug, Clone, thiserror::Error)] +pub enum ModelError { + /// 配置读不进来（语法错、字段不认识）。里面是和换入配置时同一句话 + #[error("{}", self.msg())] + Config(Box), + /// 配置的格式版本比这一份 Core 新 + #[error("{}", self.msg())] + TooNew { schema: u32 }, + /// 一项的单位拼不回配置里的样子（同步合并出来的东西不成形） + #[error("{}", self.msg())] + Shape { item: String, detail: String }, + /// 按名字改配置时没成 + #[error("{}", self.msg())] + Edit(String, Box), + /// 写完读回来和要的不一样。是我们自己的错 + #[error("{}", self.msg())] + Verify { detail: String }, +} + +impl ModelError { + pub fn msg(&self) -> Msg { + match self { + ModelError::Config(r) => r.msg(), + ModelError::TooNew { schema } => msg!( + "pack.schema_too_new", schema = schema, supported = tw_config::SCHEMA_VERSION => + "The configuration is schema version {schema}, and this version supports up to \ + {supported}. Update ThinkWatch Lite first." + ), + ModelError::Shape { item, detail } => msg!( + "pack.item_shape", item = item, detail = detail => + "{item} cannot be written into the configuration: {detail}" + ), + ModelError::Edit(_, m) => (**m).clone(), + ModelError::Verify { detail } => msg!( + "pack.verify_failed", detail = detail => + "The configuration that was put together does not read back as intended \ + ({detail}), so nothing was written." + ), + } + } + + /// 读不进来是因为它由更新的版本写的（不认识的字段、取值，或者格式版本更大） + pub fn is_too_new(&self) -> bool { + match self { + ModelError::TooNew { .. } => true, + ModelError::Config(r) => matches!( + r.message.code.as_str(), + "config.unknown_field" | "config.unknown_variant" + ), + _ => false, + } + } +} + +impl From for ModelError { + fn from(e: tw_config::edit::EditError) -> Self { + ModelError::Edit(e.to_string(), Box::new(e.msg())) + } +} + +impl From for ModelError { + fn from(e: tw_yaml::PatchError) -> Self { + ModelError::Edit(e.to_string(), Box::new(e.msg())) + } +} + +// ─────────────────────────────────────────────────────────── 插件的拆法 + +static CODEC: OnceLock> = OnceLock::new(); + +/// 装上插件源码的拆法（[`PluginCodec`]）。**进程里装一次**：Core 启动时装上真的那一套 +/// （它要编插件的那几个 crate）；没装时按 [`WholeFile`] 整份比较。装过了返回 false +pub fn install_codec(codec: Box) -> bool { + CODEC.set(codec).is_ok() +} + +/// 此刻用的拆法 +pub fn codec() -> &'static dyn PluginCodec { + match CODEC.get() { + Some(c) => c.as_ref(), + None => &WholeFile, + } +} + +// ─────────────────────────────────────────────────────────── 拆 + +/// 把一个包拆成项目。本机设置不在里面。用进程里装的插件拆法（[`install_codec`]） +pub fn snapshot(pack: &Pack) -> Result { + snapshot_with(pack, codec()) +} + +/// 同 [`snapshot`]，插件拆法由调用方给 +pub fn snapshot_with(pack: &Pack, codec: &dyn PluginCodec) -> Result { + let cfg = parse(&pack.config_yaml)?; + let mut items = items::of_config(&cfg, pack, codec)?; + items::relink(&mut items); + Ok(Snapshot { items }) +} + +/// 读一份配置原文，**不做整份校验**：一个只有一个上游的部分包没有网关密钥，也没有 +/// 控制面的钥匙。本机那几节先去掉再读，同步那一节的写法这里不必认得 +pub(crate) fn parse(text: &str) -> Result { + let reject = || -> ModelError { + match tw_config::try_parse(text) { + Err(r) if r.stage != tw_config::Stage::Semantics => ModelError::Config(Box::new(r)), + _ => ModelError::Verify { + detail: "the configuration could not be read".into(), + }, + } + }; + let mut doc: serde_yaml_ng::Value = serde_yaml_ng::from_str(text).map_err(|_| reject())?; + if doc.is_null() { + doc = serde_yaml_ng::Value::Mapping(Default::default()); + } + if let serde_yaml_ng::Value::Mapping(m) = &mut doc { + for k in LOCAL_SECTIONS { + m.remove(*k); + } + } + let cfg: tw_config::Config = serde_yaml_ng::from_value(doc).map_err(|e| { + // 本机那几节以外读不进来：原文整份读一遍，借它的行号和那句带码的话 + match tw_config::try_parse(text) { + Err(r) if r.stage != tw_config::Stage::Semantics => ModelError::Config(Box::new(r)), + _ => ModelError::Config(Box::new(tw_config::Rejected { + stage: tw_config::Stage::Schema, + message: Box::new(msg!(tw_config::reload::UNPARSABLE, detail = e => "{detail}")), + line: None, + column: None, + excerpt: None, + })), + } + })?; + if cfg.version > tw_config::SCHEMA_VERSION { + return Err(ModelError::TooNew { + schema: cfg.version, + }); + } + Ok(cfg) +} + +/// [`Pack::content_id`] +pub(crate) fn content_id(pack: &Pack) -> String { + let hash = match snapshot(pack) { + Ok(s) => blake3::hash( + serde_json::to_string(&normalized(&s)) + .unwrap_or_default() + .as_bytes(), + ), + Err(_) => { + let mut h = blake3::Hasher::new(); + h.update(pack.config_yaml.as_bytes()); + for (id, f) in &pack.plugins { + h.update(id.as_bytes()); + h.update(&f.script); + } + if let Some(p) = &pack.client_prefs { + h.update(p.to_string().as_bytes()); + } + h.finalize() + } + }; + hash.to_hex()[..12].to_string() +} + +/// 只留单位（引用是算出来的），全局设置里等于默认值的单位去掉、空了的项目去掉 —— +/// 同步合并出来的快照可能把默认值写明了，落回配置之后它们读出来就是没写 +pub fn normalized(s: &Snapshot) -> Snapshot { + let mut out = s.clone(); + for (id, item) in out.items.iter_mut() { + item.refs.clear(); + item.files = None; + if id.kind == ItemKind::Global { + item.units + .retain(|path, v| !items::is_global_default(path, v)); + } + } + out.items.retain(|id, item| { + !(matches!(id.kind, ItemKind::Global | ItemKind::ClientPrefs) && item.units.is_empty()) + }); + out +} + +#[cfg(test)] +mod tests; diff --git a/crates/tw-pack/src/model/rename.rs b/crates/tw-pack/src/model/rename.rs new file mode 100644 index 00000000..3a84ec3e --- /dev/null +++ b/crates/tw-pack/src/model/rename.rs @@ -0,0 +1,241 @@ +//! 在快照里给一个项目改名，引用它的地方跟着改。 +//! +//! 和按名字编辑配置时的改名（`tw_config::refs`）是同一份规矩，只是改的是单位而不是原文： +//! 导入时选「改名导入」、同步时选「两者都保留」，改名的是配置包里的那一份，包里引用它的 +//! 路由、别名、分组随之改用新名称。 + +use serde_json::Value as Json; + +use super::items::rule_name; +use super::{ItemId, ItemKind, Snapshot, global, unit}; + +/// 把 `id` 改名成 `new_name`，连同快照里引用它的地方。没有这一项、名字没变、这种项目 +/// 没有名字（顺序、全局设置、通用偏好）时什么都不做。新名字已经被占了也不做：调用方 +/// 先挑一个没人用的（[`free_name`]）。 +/// +/// 自定义安全规则的 `new_name` 可以只写规则名（分项不变），也可以写成 `<分项>/<规则名>` +pub fn rename(snap: &mut Snapshot, id: &ItemId, new_name: &str) { + if !id.kind.is_named() || id.name == new_name { + return; + } + let new_name = match id.kind { + ItemKind::SecurityRule if !new_name.contains('/') => match rule_name(&id.name) { + Some((guard, _)) => format!("{guard}/{new_name}"), + None => return, + }, + _ => new_name.to_string(), + }; + let new_id = ItemId::new(id.kind, new_name.clone()); + if snap.items.contains_key(&new_id) { + return; + } + let Some(item) = snap.items.remove(id) else { + return; + }; + snap.items.insert(new_id, item); + let old = id.name.as_str(); + let new = new_name.as_str(); + + for (other, item) in snap.items.iter_mut() { + let all = item.units.get_mut(unit::ALL); + match (id.kind, other.kind) { + (ItemKind::Provider, ItemKind::Route) => { + if let Some(all) = all { + for rule in rules(all) { + match rule.get_mut("to") { + Some(Json::Array(pinned)) => { + for p in pinned { + swap(p.get_mut("provider"), old, new); + } + } + to => swap(to, old, new), + } + if let Some(when) = rule.get_mut("when") { + swap_one_or_many(when.get_mut("provider_would_be"), old, new); + } + } + } + } + (ItemKind::Provider, ItemKind::Group) => { + if let Some(all) = all { + if let Some(Json::Array(members)) = all.get_mut("providers") { + for m in members { + match m { + Json::String(_) => swap(Some(m), old, new), + Json::Object(o) => swap(o.get_mut("name"), old, new), + _ => {} + } + } + } + swap(all.get_mut("selected"), old, new); + } + } + (ItemKind::Provider, ItemKind::ProviderOrder) + | (ItemKind::Route, ItemKind::RouteOrder) + | (ItemKind::Plugin, ItemKind::PluginOrder) => { + if let Some(Json::Array(list)) = all { + for n in list { + swap(Some(n), old, new); + } + } + } + (ItemKind::Proxy, ItemKind::Provider) => { + swap( + item.units + .get_mut(unit::CONN) + .and_then(|c| c.get_mut("proxy")), + old, + new, + ); + } + (ItemKind::PriceSheet, ItemKind::Provider) => { + swap( + item.units + .get_mut(unit::BILLING) + .and_then(|c| c.get_mut("pricing")), + old, + new, + ); + } + (ItemKind::Key, ItemKind::Route) => { + if let Some(all) = all { + for rule in rules(all) { + swap( + rule.get_mut("when").and_then(|w| w.get_mut("client")), + old, + new, + ); + } + } + } + (ItemKind::Key, ItemKind::Global) if other.name == global::DEFAULT_KEY => { + swap(item.units.get_mut(global::DEFAULT_KEY), old, new); + } + (ItemKind::Alias, ItemKind::Key) => { + if let Some(Json::Array(allow)) = all.and_then(|a| a.get_mut("allow")) { + for m in allow { + swap(Some(m), old, new); + } + } + } + (ItemKind::Alias, ItemKind::Route) => { + if let Some(all) = all { + for rule in rules(all) { + let phase_two = rule + .get("when") + .is_some_and(|w| w.get("provider_would_be").is_some()); + swap( + rule.get_mut("when").and_then(|w| w.get_mut("model")), + old, + new, + ); + if !phase_two { + swap( + rule.get_mut("set").and_then(|s| s.get_mut("model")), + old, + new, + ); + } + } + } + } + (ItemKind::Group, ItemKind::Route) => { + if let Some(all) = all { + for rule in rules(all) { + swap(rule.get_mut("to"), old, new); + } + } + } + (ItemKind::Route, ItemKind::Key) => { + swap(all.and_then(|a| a.get_mut("route")), old, new); + } + _ => {} + } + } + + // 默认路由：没写明时说的是叫「默认」的那条。改的正是它，要把新名字写明；改成的正是 + // 「默认」，写明的那一项就不用了 + if id.kind == ItemKind::Route { + let at = ItemId::new(ItemKind::Global, global::DEFAULT_ROUTE); + let current = snap + .items + .get(&at) + .and_then(|i| i.units.get(global::DEFAULT_ROUTE)) + .and_then(Json::as_str) + .unwrap_or(tw_engine::DEFAULT_ROUTE) + .to_string(); + if current == old { + if new == tw_engine::DEFAULT_ROUTE { + snap.items.remove(&at); + } else { + snap.items.entry(at).or_default().units.insert( + global::DEFAULT_ROUTE.to_string(), + Json::String(new.to_string()), + ); + } + } + } + // 默认密钥同理 + if id.kind == ItemKind::Key { + let at = ItemId::new(ItemKind::Global, global::DEFAULT_KEY); + let written = snap + .items + .get(&at) + .and_then(|i| i.units.get(global::DEFAULT_KEY)) + .is_some(); + if !written && old == tw_config::DEFAULT_KEY { + snap.items.entry(at).or_default().units.insert( + global::DEFAULT_KEY.to_string(), + Json::String(new.to_string()), + ); + } else if written && new == tw_config::DEFAULT_KEY { + let now = snap + .items + .get(&at) + .and_then(|i| i.units.get(global::DEFAULT_KEY)); + if now.and_then(Json::as_str) == Some(new) { + snap.items.remove(&at); + } + } + } + snap.relink(); +} + +/// `base` 没被占就是它，被占了是「base (2)」「base (3)」…… +pub fn free_name(snap: &Snapshot, kind: ItemKind, base: &str) -> String { + let taken = |n: &str| snap.items.contains_key(&ItemId::new(kind, n)); + if !taken(base) { + return base.to_string(); + } + (2..) + .map(|i| format!("{base} ({i})")) + .find(|n| !taken(n)) + .expect("an unbounded range always finds one") +} + +fn rules(all: &mut Json) -> impl Iterator { + all.get_mut("rules") + .and_then(Json::as_array_mut) + .into_iter() + .flatten() +} + +/// 这个位置上的字符串正是 `old` 就换成 `new` +fn swap(at: Option<&mut Json>, old: &str, new: &str) { + if let Some(v) = at + && v.as_str() == Some(old) + { + *v = Json::String(new.to_string()); + } +} + +fn swap_one_or_many(at: Option<&mut Json>, old: &str, new: &str) { + match at { + Some(Json::Array(list)) => { + for v in list { + swap(Some(v), old, new); + } + } + other => swap(other, old, new), + } +} diff --git a/crates/tw-pack/src/model/tests.rs b/crates/tw-pack/src/model/tests.rs new file mode 100644 index 00000000..a09ce6c1 --- /dev/null +++ b/crates/tw-pack/src/model/tests.rs @@ -0,0 +1,1015 @@ +//! 拆开、落回、改名。表驱动：每一种项目的每一种改法，落回去再拆开都是要的那个样子。 + +use std::collections::BTreeMap; + +use serde_json::{Value as Json, json}; + +use super::*; +use crate::{Manifest, Pack, PackKind, PackOrigin, PluginFiles}; + +const KEY: &str = "c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00"; + +/// 一份什么都有的配置。注释散在各处：没改的项目落回去之后它们还在 +fn rich() -> String { + format!( + r#"version: 1 +listen: + control: + key: {KEY} + gateway: + port: 9000 +retention: + body_days: 3 +clients: + - name: default + key: tw-aaa + # 给 Codex 的 + - name: codex + key: tw-bbb + route: 长上下文 + client: codex + allow: [fast] +proxies: + - name: hk + type: socks5h + addr: 127.0.0.1:7890 +providers: + # 官方直连 + - name: 官方 + base_url: https://api.anthropic.com # 不走代理 + key: sk-a + proxy: direct + headers: + X-Second: b + X-First: a + - name: relay + base_url: https://relay.example/v1 + key: sk-b + proxy: hk + pricing: 中转 + models: [m1] + max_concurrent: 4 + - name: chatgpt + base_url: https://chatgpt.com/backend-api/codex + oauth: + refresh: rt-1 + endpoint: https://auth.openai.com/oauth/token + client_id: app_EMoamEEZ73f0CkXaXp7hrann + headers: + ChatGPT-Account-Id: acct-1 +pricing: + auto_update: false + sheets: + - name: 中转 + multiplier: 0.5 +failover: + pause_secs: 120 +security: + redact: + mode: enforce + custom: + - name: 项目代号 + pattern: 北极星 + content: + custom: + - name: 脏话 + pattern: xx +aliases: + fast: [claude-haiku, gpt-mini] +groups: + - name: pool + type: fallback + providers: [官方, relay] +routes: + - name: default + rules: + - name: 兜底 + to: pool + - name: 长上下文 + rules: + - name: big + when: {{ input_tokens: '>200k', client: codex }} + to: relay +default_key: default +"# + ) +} + +fn pack(text: &str) -> Pack { + Pack { + manifest: Manifest::new(PackKind::Full, PackOrigin::Ui), + config_yaml: text.to_string(), + plugins: BTreeMap::new(), + client_prefs: None, + } +} + +fn id(kind: ItemKind, name: &str) -> ItemId { + ItemId::new(kind, name) +} + +/// 插件源码的测试拆法：第一行是 `// data: `，其余是代码。名字带 `steer` 的要批准 +struct Fake { + shipped: BTreeMap, +} + +impl Fake { + fn new() -> Self { + Self { + shipped: BTreeMap::from([( + "reply-language".to_string(), + "// data: {\"lang\":\"\"}\nshipped code v1\n".to_string(), + )]), + } + } +} + +impl PluginCodec for Fake { + fn split(&self, source: &str) -> Option<(String, Json)> { + let (first, rest) = source.split_once('\n')?; + let data: Json = serde_json::from_str(first.strip_prefix("// data: ")?).ok()?; + Some((format!("// data: null\n{rest}"), data)) + } + fn join(&self, source: &str, data: &Json) -> Option { + let (_, rest) = source.split_once('\n')?; + Some(format!("// data: {data}\n{rest}")) + } + fn shipped(&self, id: &str) -> Option { + self.shipped.get(id).cloned() + } + fn needs_approval(&self, source: &str) -> bool { + source.contains("steer") + } +} + +fn snap(p: &Pack) -> Snapshot { + snapshot_with(p, &Fake::new()).unwrap_or_else(|e| panic!("{e}\n{}", p.config_yaml)) +} + +fn mat(local: &Pack, target: &Snapshot) -> Pack { + materialize_with(local, target, &Fake::new()).unwrap_or_else(|e| panic!("{e}")) +} + +fn without_orders(s: &Snapshot) -> Snapshot { + let mut s = s.clone(); + s.items.retain(|id, _| !id.kind.is_order()); + s +} + +// ─────────────────────────────────────────────────────────── 拆 + +#[test] +fn every_named_thing_is_an_item_and_local_settings_are_not() { + let s = snap(&pack(&rich())); + let ids: Vec = s.items.keys().map(|i| i.to_string()).collect(); + for want in [ + "provider `官方`", + "provider `relay`", + "provider `chatgpt`", + "proxy `hk`", + "key `default`", + "key `codex`", + "alias `fast`", + "group `pool`", + "route `default`", + "route `长上下文`", + "price_sheet `中转`", + "security_rule `redact/项目代号`", + "security_rule `content/脏话`", + "provider_order", + "route_order", + "global `failover`", + "global `security`", + "global `pricing`", + ] { + assert!(ids.contains(&want.to_string()), "{want} 不在：{ids:?}"); + } + // 写成默认值的那一节不算（default_key: default 就是没写） + assert!(!ids.iter().any(|i| i.contains("default_key")), "{ids:?}"); + let all = serde_json::to_string(&s).unwrap(); + for local in [KEY, "9000", "body_days", "rt-1", "acct-1"] { + assert!(!all.contains(local), "本机设置进了快照：{local}"); + } +} + +#[test] +fn an_upstream_splits_into_four_units_by_what_its_fields_are_about() { + let s = snap(&pack(&rich())); + let relay = &s.items[&id(ItemKind::Provider, "relay")]; + assert_eq!( + relay.units.keys().collect::>(), + ["billing", "conn", "models", "other"] + ); + assert_eq!( + relay.units["conn"], + json!({"base_url": "https://relay.example/v1", "key": "sk-b", "proxy": "hk"}) + ); + assert_eq!(relay.units["models"], json!({"models": ["m1"]})); + assert_eq!(relay.units["billing"], json!({"pricing": "中转"})); + assert_eq!(relay.units["other"], json!({"max_concurrent": 4})); + // 写明的默认值（proxy: direct）和没写一样；请求头按写的顺序 + let official = &s.items[&id(ItemKind::Provider, "官方")]; + assert_eq!( + official.units["conn"], + json!({"base_url": "https://api.anthropic.com", "key": "sk-a", + "headers": [["X-Second", "b"], ["X-First", "a"]]}) + ); + // ChatGPT 的登录和账户 ID 请求头不在里面 + let chatgpt = &s.items[&id(ItemKind::Provider, "chatgpt")]; + assert_eq!( + chatgpt.units["conn"], + json!({"base_url": "https://chatgpt.com/backend-api/codex"}) + ); +} + +#[test] +fn references_point_at_what_each_item_uses() { + let s = snap(&pack(&rich())); + let refs = |kind, name: &str| -> Vec { + s.items[&id(kind, name)] + .refs + .iter() + .map(|r| r.to_string()) + .collect() + }; + assert_eq!( + refs(ItemKind::Provider, "relay"), + ["proxy `hk`", "price_sheet `中转`"] + ); + assert_eq!( + refs(ItemKind::Key, "codex"), + ["alias `fast`", "route `长上下文`"] + ); + assert_eq!( + refs(ItemKind::Group, "pool"), + ["provider `relay`", "provider `官方`"] + ); + assert_eq!(refs(ItemKind::Route, "default"), ["group `pool`"]); + assert_eq!( + refs(ItemKind::Route, "长上下文"), + ["provider `relay`", "key `codex`"] + ); + // 选一条路由导出：带上它的上游、上游的代理和价目表、按它分流的密钥 + let picked: Vec = s + .closure(&[id(ItemKind::Route, "长上下文")]) + .iter() + .map(|i| i.to_string()) + .collect(); + assert_eq!( + picked, + [ + "provider `relay`", + "proxy `hk`", + "key `codex`", + "alias `fast`", + "route `长上下文`", + "price_sheet `中转`" + ] + ); +} + +/// 注释、排版、写没写默认值都不是差别；本机设置也不是 +#[test] +fn the_same_content_written_differently_is_the_same_snapshot_and_id() { + let a = pack(&rich()); + let mut b = pack( + &rich() + .replace(" # 官方直连\n", "") + .replace(" proxy: direct\n", "") + .replace(" gateway:\n port: 9000\n", "") + .replace("retention:\n body_days: 3\n", "") + .replace("default_key: default\n", "") + .replace("rt-1", "rt-2-after-a-rotation"), + ); + assert_eq!(snap(&a), snap(&b)); + assert_eq!(a.content_id(), b.content_id()); + assert_eq!(a.content_id().len(), 12); + b.config_yaml = b.config_yaml.replace("sk-b", "sk-changed"); + assert_ne!(a.content_id(), b.content_id()); +} + +#[test] +fn global_settings_hold_only_what_differs_from_the_default() { + let s = snap(&pack(&rich())); + assert_eq!( + s.items[&id(ItemKind::Global, "failover")].units, + BTreeMap::from([("failover.pause_secs".to_string(), json!(120))]) + ); + assert_eq!( + s.items[&id(ItemKind::Global, "security")].units, + BTreeMap::from([("security.redact.mode".to_string(), json!("enforce"))]) + ); + assert_eq!( + s.items[&id(ItemKind::Global, "pricing")].units, + BTreeMap::from([("pricing.auto_update".to_string(), json!(false))]) + ); + for (path, v, default) in [ + ("failover.pause_secs", json!(60), true), + ("failover.pause_secs", json!(61), false), + ("security.redact.mode", json!("observe"), true), + ("security.content.mode", json!("off"), false), + ("client_probes.titling", json!("intercept"), false), + ("pricing.auto_update", json!(true), true), + ("default_route", json!("default"), true), + ("default_route", json!("other"), false), + ("default_key", json!("default"), true), + ] { + assert_eq!(items::is_global_default(path, &v), default, "{path} = {v}"); + } +} + +#[test] +fn a_pack_from_a_newer_version_is_too_new() { + for text in [ + "version: 1\nsomething_new: true\n", + "version: 1\nproviders:\n - { name: a, base_url: https://a, balance: unheard_of }\n", + ] { + let e = snapshot_with(&pack(text), &Fake::new()).unwrap_err(); + assert!(e.is_too_new(), "{text}: {e:?}"); + } + let e = snapshot_with(&pack("version: 99\n"), &Fake::new()).unwrap_err(); + assert!(e.is_too_new()); + assert_eq!(e.msg().code, "pack.schema_too_new"); + // 读不成 YAML 的不算太新 + let e = snapshot_with(&pack("version: 1\nproviders: [\n"), &Fake::new()).unwrap_err(); + assert!(!e.is_too_new()); + // 只有一个上游的部分包：没有网关密钥、没有钥匙，照样拆得开 + snap(&pack( + "version: 1\nproviders:\n - { name: a, base_url: https://a.example }\n", + )); + // 同步那一节这里不必认得 + snap(&pack( + "version: 1\nsync:\n device_id: x\n whatever: [1]\n", + )); +} + +#[test] +fn a_snapshot_survives_json() { + let s = snap(&pack(&rich())); + let back: Snapshot = serde_json::from_str(&serde_json::to_string(&s).unwrap()).unwrap(); + assert_eq!(back, s); +} + +// ─────────────────────────────────────────────────────────── 落回 + +/// 原样落回：一个字节都不变 +#[test] +fn materializing_a_config_onto_itself_changes_nothing() { + let p = pack(&rich()); + let out = mat(&p, &snap(&p)); + assert_eq!(out.config_yaml, p.config_yaml); +} + +/// 改一处：别的原样，注释也在；本机设置和 ChatGPT 的登录还在 +#[test] +fn changing_one_item_leaves_the_others_byte_for_byte() { + let p = pack(&rich()); + let mut t = snap(&p); + t.items + .get_mut(&id(ItemKind::Provider, "relay")) + .unwrap() + .units + .insert( + "other".into(), + json!({"max_concurrent": 8, "disabled": true}), + ); + let out = mat(&p, &t); + assert!( + out.config_yaml.contains(" max_concurrent: 8\n"), + "{}", + out.config_yaml + ); + for kept in [ + " # 官方直连\n", + "base_url: https://api.anthropic.com # 不走代理\n", + " # 给 Codex 的\n", + " port: 9000\n", + " body_days: 3\n", + " refresh: rt-1\n", + " ChatGPT-Account-Id: acct-1\n", + " proxy: direct\n", + ] { + assert!( + out.config_yaml.contains(kept), + "{kept:?} 没了：\n{}", + out.config_yaml + ); + } + assert_eq!(normalized(&snap(&out)), normalized(&t)); + tw_config::try_parse(&out.config_yaml).unwrap_or_else(|e| panic!("{e}\n{}", out.config_yaml)); +} + +/// 一种项目一张表：加、删、改，落回去再拆开就是要的样子，配置照样过得了校验 +#[test] +fn every_kind_of_change_lands_as_intended() { + type Change = fn(&mut Snapshot); + let cases: Vec<(&str, Change)> = vec![ + ("add an upstream", |s| { + s.items.insert( + ItemId::new(ItemKind::Provider, "new"), + Item { + units: BTreeMap::from([ + ( + "conn".into(), + json!({"base_url": "https://new.example", "key": "sk-n"}), + ), + ("models".into(), json!({})), + ("billing".into(), json!({"billing": "free"})), + ("other".into(), json!({})), + ]), + ..Default::default() + }, + ); + }), + ("drop an upstream nobody uses", |s| { + s.items.remove(&ItemId::new(ItemKind::Provider, "chatgpt")); + }), + ("change an upstream's headers and order", |s| { + let p = s + .items + .get_mut(&ItemId::new(ItemKind::Provider, "官方")) + .unwrap(); + p.units.insert( + "conn".into(), + json!({"base_url": "https://api.anthropic.com", "key": "sk-new", + "headers": [["X-First", "a"], ["X-Third", "c"]]}), + ); + }), + ("add a proxy", |s| { + s.items.insert( + ItemId::new(ItemKind::Proxy, "sg"), + Item { + units: BTreeMap::from([( + "all".into(), + json!({"type": "http", "addr": "10.0.0.1:8080"}), + )]), + ..Default::default() + }, + ); + }), + ("change a key", |s| { + s.items + .get_mut(&ItemId::new(ItemKind::Key, "default")) + .unwrap() + .units + .insert("all".into(), json!({"key": "tw-zzz", "max_concurrent": 2})); + }), + ("drop an alias and its use", |s| { + s.items.remove(&ItemId::new(ItemKind::Alias, "fast")); + s.items + .get_mut(&ItemId::new(ItemKind::Key, "codex")) + .unwrap() + .units + .insert( + "all".into(), + json!({"key": "tw-bbb", "route": "长上下文", "client": "codex"}), + ); + }), + ("add an alias", |s| { + s.items.insert( + ItemId::new(ItemKind::Alias, "big"), + Item { + units: BTreeMap::from([("all".into(), json!(["claude-opus"]))]), + ..Default::default() + }, + ); + }), + ("change a group", |s| { + s.items + .get_mut(&ItemId::new(ItemKind::Group, "pool")) + .unwrap() + .units + .insert( + "all".into(), + json!({"type": "load-balance", "providers": ["官方", {"name": "relay", "weight": 3}]}), + ); + }), + ("add a route and reorder", |s| { + s.items.insert( + ItemId::new(ItemKind::Route, "a-first"), + Item { + units: BTreeMap::from([( + "all".into(), + json!({"rules": [{"name": "all", "to": "官方"}]}), + )]), + ..Default::default() + }, + ); + s.items.insert( + ItemId::new(ItemKind::RouteOrder, ""), + Item { + units: BTreeMap::from([( + "all".into(), + json!(["长上下文", "a-first", "default"]), + )]), + ..Default::default() + }, + ); + }), + ("reorder upstreams", |s| { + s.items.insert( + ItemId::new(ItemKind::ProviderOrder, ""), + Item { + units: BTreeMap::from([("all".into(), json!(["chatgpt", "relay", "官方"]))]), + ..Default::default() + }, + ); + }), + ("change a price sheet", |s| { + s.items + .get_mut(&ItemId::new(ItemKind::PriceSheet, "中转")) + .unwrap() + .units + .insert("all".into(), json!({"multiplier": 0.8})); + }), + ("add and drop custom rules", |s| { + s.items + .remove(&ItemId::new(ItemKind::SecurityRule, "content/脏话")); + s.items.insert( + ItemId::new(ItemKind::SecurityRule, "inspect_tools/rm"), + Item { + units: BTreeMap::from([("all".into(), json!({"pattern": "rm -rf"}))]), + ..Default::default() + }, + ); + }), + ("change and reset global fields", |s| { + let f = s + .items + .get_mut(&ItemId::new(ItemKind::Global, "failover")) + .unwrap(); + f.units.clear(); + f.units + .insert("failover.idle_timeout_secs".into(), json!(600)); + s.items.remove(&ItemId::new(ItemKind::Global, "pricing")); + s.items.insert( + ItemId::new(ItemKind::Global, "security"), + Item { + units: BTreeMap::from([ + ("security.content.mode".into(), json!("enforce")), + ("security.redact.actions.jwt".into(), json!("record")), + ]), + ..Default::default() + }, + ); + s.items.insert( + ItemId::new(ItemKind::Global, "client_probes"), + Item { + units: BTreeMap::from([("client_probes.titling".into(), json!("forward"))]), + ..Default::default() + }, + ); + }), + ("set the default route and key", |s| { + s.items.insert( + ItemId::new(ItemKind::Global, "default_route"), + Item { + units: BTreeMap::from([("default_route".into(), json!("长上下文"))]), + ..Default::default() + }, + ); + s.items.insert( + ItemId::new(ItemKind::Global, "default_key"), + Item { + units: BTreeMap::from([("default_key".into(), json!("codex"))]), + ..Default::default() + }, + ); + }), + ("an explicit default is the same as none", |s| { + s.items.insert( + ItemId::new(ItemKind::Global, "client_probes"), + Item { + units: BTreeMap::from([("client_probes.warmup".into(), json!("intercept"))]), + ..Default::default() + }, + ); + }), + ]; + for (what, change) in cases { + let p = pack(&rich()); + let mut t = snap(&p); + change(&mut t); + t.relink(); + let out = materialize_with(&p, &t, &Fake::new()).unwrap_or_else(|e| panic!("{what}: {e}")); + let got = normalized(&snap(&out)); + assert_eq!( + without_orders(&got), + without_orders(&normalized(&t)), + "{what}:\n{}", + out.config_yaml + ); + // 目标里列着的顺序照着排；没列的（新加的上游）接在后面 + for (oid, o) in t.items.iter().filter(|(i, _)| i.kind.is_order()) { + let have = got.items[oid].units["all"].as_array().unwrap(); + let want: Vec<&Json> = o.units["all"] + .as_array() + .unwrap() + .iter() + .filter(|n| have.contains(n)) + .collect(); + let have: Vec<&Json> = have.iter().take(want.len()).collect(); + assert_eq!(have, want, "{what}: {oid}"); + } + tw_config::try_parse(&out.config_yaml) + .unwrap_or_else(|e| panic!("{what}: {e}\n{}", out.config_yaml)); + // 本机设置都还在；ChatGPT 那一家还在的话，它的登录也在 + // 注释也在：走的是按项目改原文那条路，不是整份重写 + let mut kept = vec![ + " port: 9000\n", + " body_days: 3\n", + KEY, + " # 给 Codex 的\n", + " # 官方直连\n", + ]; + if t.items.contains_key(&id(ItemKind::Provider, "chatgpt")) { + kept.push(" refresh: rt-1\n"); + } + for kept in kept { + assert!( + out.config_yaml.contains(kept), + "{what}: {kept:?} 没了\n{}", + out.config_yaml + ); + } + } +} + +/// 按项目改不动的写法（行内的一节删不了键）：整份重写，内容照样对 +#[test] +fn a_shape_the_edits_cannot_touch_is_rewritten_whole() { + let text = format!( + "version: 1\nlisten: {{ control: {{ key: {KEY} }} }}\nclients: [{{ name: default, key: tw-a }}]\nfailover: {{ pause_secs: 120, idle_timeout_secs: 600 }}\n" + ); + let p = pack(&text); + let mut t = snap(&p); + t.items + .get_mut(&id(ItemKind::Global, "failover")) + .unwrap() + .units + .remove("failover.idle_timeout_secs"); + let out = mat(&p, &t); + assert_eq!( + normalized(&snap(&out)), + normalized(&t), + "{}", + out.config_yaml + ); + assert!(out.config_yaml.contains(KEY)); + tw_config::try_parse(&out.config_yaml).unwrap(); +} + +/// 从空的配置开始落：导出所选项目就是这么生成配置包的 +#[test] +fn materializing_onto_an_empty_config_builds_just_those_items() { + let full = snap(&pack(&rich())); + let mut picked = full.clone(); + picked.retain(&full.closure(&[id(ItemKind::Route, "长上下文")])); + let out = mat(&pack("version: 1\n"), &picked); + assert_eq!( + normalized(&snap(&out)), + normalized(&picked), + "{}", + out.config_yaml + ); + assert!(!out.config_yaml.contains("官方"), "{}", out.config_yaml); +} + +/// ChatGPT 账号上游：落到别的设备上时没有登录；落回本机时留着本机的登录 +#[test] +fn a_chatgpt_upstream_keeps_the_login_of_the_device_it_lands_on() { + let p = pack(&rich()); + let mut t = snap(&p); + t.items + .get_mut(&id(ItemKind::Provider, "chatgpt")) + .unwrap() + .units + .insert("other".into(), json!({"disabled": true})); + let out = mat(&p, &t); + let cfg = tw_config::try_parse(&out.config_yaml).unwrap(); + let c = cfg.providers.iter().find(|p| p.name == "chatgpt").unwrap(); + assert!(c.disabled); + assert_eq!(c.oauth.as_ref().unwrap().refresh, "rt-1"); + assert_eq!( + c.headers.get("ChatGPT-Account-Id").unwrap().value.raw(), + "acct-1" + ); + // 别的设备上：只有上游本身 + let other = mat(&pack("version: 1\n"), &t); + assert!(!other.config_yaml.contains("rt-1"), "{}", other.config_yaml); + assert!( + !other.config_yaml.contains("acct-1"), + "{}", + other.config_yaml + ); +} + +// ─────────────────────────────────────────────────────────── 插件 + +fn with_plugins(entries: &[(&str, &str, bool)]) -> Pack { + let mut text = format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n key: tw-a\nplugins:\n" + ); + let mut plugins = BTreeMap::new(); + for (id, source, enabled) in entries { + let sha = items::sha256_hex(source.as_bytes()); + text.push_str(&format!( + " - id: {id}\n file: plugins/{id}.js\n sha256: {sha}\n enabled: {enabled}\n" + )); + plugins.insert( + id.to_string(), + PluginFiles { + script: source.as_bytes().to_vec(), + approved: Some(source.as_bytes().to_vec()), + }, + ); + } + Pack { + plugins, + ..pack(&text) + } +} + +#[test] +fn a_plugin_splits_into_code_settings_and_switch() { + let p = with_plugins(&[ + ("note", "// data: {\"v\":1}\nmy code\n", true), + ( + "reply-language", + "// data: {\"lang\":\"zh\"}\nshipped code v1\n", + false, + ), + ]); + let s = snap(&p); + let note = &s.items[&id(ItemKind::Plugin, "note")]; + assert_eq!(note.units["script"], json!("// data: null\nmy code\n")); + assert_eq!(note.units["settings"], json!({"v": 1})); + assert_eq!(note.units["enabled"], json!(true)); + // 默认插件：脚本不比较，设置和开关照常 + let def = &s.items[&id(ItemKind::Plugin, "reply-language")]; + assert!(!def.units.contains_key("script")); + assert_eq!(def.units["settings"], json!({"lang": "zh"})); + assert_eq!( + s.items[&id(ItemKind::PluginOrder, "")].units["all"], + json!(["note", "reply-language"]) + ); +} + +#[test] +fn plugin_changes_land_on_the_files_and_the_entry() { + let codec = Fake::new(); + let p = with_plugins(&[ + ("note", "// data: {\"v\":1}\nmy code\n", true), + ( + "reply-language", + "// data: {\"lang\":\"zh\"}\nshipped code v1\n", + false, + ), + ]); + // 只改设置：本机的文件上改，照样批准着 + let mut t = snap(&p); + t.items + .get_mut(&id(ItemKind::Plugin, "note")) + .unwrap() + .units + .insert("settings".into(), json!({"v": 2})); + let out = materialize_with(&p, &t, &codec).unwrap(); + let f = &out.plugins["note"]; + assert_eq!(f.script, b"// data: {\"v\":2}\nmy code\n"); + assert_eq!(f.approved.as_deref(), Some(&f.script[..])); + let cfg = tw_config::try_parse(&out.config_yaml).unwrap(); + assert_eq!(cfg.plugins[0].sha256, items::sha256_hex(&f.script)); + // 默认插件在本机没有：用这一份 Core 自带的，设置照目标 + let bare = with_plugins(&[]); + let out = materialize_with(&bare, &snap(&p), &codec).unwrap(); + assert_eq!( + out.plugins["reply-language"].script, + b"// data: {\"lang\":\"zh\"}\nshipped code v1\n" + ); + assert_eq!(out.plugins["note"].script, b"// data: {\"v\":1}\nmy code\n"); + // 删掉一个 + let mut t = snap(&p); + t.items.remove(&id(ItemKind::Plugin, "note")); + t.items.insert( + id(ItemKind::PluginOrder, ""), + Item { + units: BTreeMap::from([("all".into(), json!(["reply-language"]))]), + ..Default::default() + }, + ); + let out = materialize_with(&p, &t, &codec).unwrap(); + assert!(!out.plugins.contains_key("note")); + assert!(!out.config_yaml.contains("id: note"), "{}", out.config_yaml); +} + +/// 改得了工具调用的插件：开着、而代码在本机没批准过时，落下去也不批准 +#[test] +fn a_steering_plugin_from_elsewhere_waits_for_approval_here() { + let codec = Fake::new(); + let src = "// data: {}\nsteer the tool calls\n"; + let theirs = with_plugins(&[("steerer", src, true)]); + let mut t = snap(&theirs); + // 带着原文件来（导入、同步都带着） + t.items + .get_mut(&id(ItemKind::Plugin, "steerer")) + .unwrap() + .files = theirs.plugins.get("steerer").cloned(); + let out = materialize_with(&with_plugins(&[]), &t, &codec).unwrap(); + let cfg = tw_config::try_parse(&out.config_yaml).unwrap(); + let e = &cfg.plugins[0]; + assert!(e.enabled); + assert_eq!(out.plugins["steerer"].script, src.as_bytes()); + assert_ne!( + e.sha256, + items::sha256_hex(src.as_bytes()), + "批准了别处来的代码" + ); + assert_eq!(out.plugins["steerer"].approved, None); + // 停用着来的照常批准：打开时才问 + let mut off = t.clone(); + off.items + .get_mut(&id(ItemKind::Plugin, "steerer")) + .unwrap() + .units + .insert("enabled".into(), json!(false)); + let out = materialize_with(&with_plugins(&[]), &off, &codec).unwrap(); + let cfg = tw_config::try_parse(&out.config_yaml).unwrap(); + assert_eq!(cfg.plugins[0].sha256, items::sha256_hex(src.as_bytes())); + // 本机停用着、批准过同一份代码，别处把它打开了：这里照样要点头 + let off_here = with_plugins(&[("steerer", src, false)]); + let mut t = snap(&off_here); + t.items + .get_mut(&id(ItemKind::Plugin, "steerer")) + .unwrap() + .units + .insert("enabled".into(), json!(true)); + let out = materialize_with(&off_here, &t, &codec).unwrap(); + let cfg = tw_config::try_parse(&out.config_yaml).unwrap(); + assert!(cfg.plugins[0].enabled); + assert_ne!( + cfg.plugins[0].sha256, + items::sha256_hex(&out.plugins["steerer"].script) + ); + // 读回来和目标一样：不会因为待批准就当成本机又改了一次 + assert_eq!( + snap(&out).items[&id(ItemKind::Plugin, "steerer")], + t.items[&id(ItemKind::Plugin, "steerer")] + ); + // 本机开着、批准过同一份代码：只改设置不用再批准 + let mine = with_plugins(&[("steerer", src, true)]); + let mut t = snap(&mine); + t.items + .get_mut(&id(ItemKind::Plugin, "steerer")) + .unwrap() + .units + .insert("settings".into(), json!({"x": 1})); + let out = materialize_with(&mine, &t, &codec).unwrap(); + let cfg = tw_config::try_parse(&out.config_yaml).unwrap(); + assert_eq!( + cfg.plugins[0].sha256, + items::sha256_hex(&out.plugins["steerer"].script) + ); +} + +/// 本机文件改了还没批准：比较按批准过的那一份，没批准的改动不往外带 +#[test] +fn an_unapproved_edit_is_neither_exported_nor_lost() { + let approved = "// data: {\"v\":1}\napproved code\n"; + let mut p = with_plugins(&[("note", approved, true)]); + p.plugins.get_mut("note").unwrap().script = + b"// data: {\"v\":1}\nedited, not approved\n".to_vec(); + let s = snap(&p); + assert_eq!( + s.items[&id(ItemKind::Plugin, "note")].units["script"], + json!("// data: null\napproved code\n") + ); + // 别处改了设置：本机那处没批准的改动留着，底稿和哈希换成新的 + let mut t = s.clone(); + t.items + .get_mut(&id(ItemKind::Plugin, "note")) + .unwrap() + .units + .insert("settings".into(), json!({"v": 2})); + let out = materialize_with(&p, &t, &Fake::new()).unwrap(); + let f = &out.plugins["note"]; + assert_eq!(f.script, b"// data: {\"v\":1}\nedited, not approved\n"); + assert_eq!( + f.approved.as_deref(), + Some(&b"// data: {\"v\":2}\napproved code\n"[..]) + ); +} + +#[test] +fn client_preferences_merge_field_by_field() { + let mut p = pack(&rich()); + p.client_prefs = Some(json!({"language": "zh", "menubar": {"show": "tokens", "rows": 2}})); + let s = snap(&p); + let prefs = &s.items[&id(ItemKind::ClientPrefs, "")]; + assert_eq!( + prefs.units.keys().collect::>(), + ["language", "menubar.rows", "menubar.show"] + ); + let mut t = s.clone(); + t.items + .get_mut(&id(ItemKind::ClientPrefs, "")) + .unwrap() + .units + .insert("menubar.show".into(), json!("speed")); + let out = mat(&p, &t); + assert_eq!( + out.client_prefs, + Some(json!({"language": "zh", "menubar": {"show": "speed", "rows": 2}})) + ); +} + +// ─────────────────────────────────────────────────────────── 改名 + +#[test] +fn renaming_follows_every_reference() { + let p = pack(&rich()); + type Check = fn(&Snapshot); + let cases: Vec<(ItemKind, &str, &str, Check)> = vec![ + (ItemKind::Provider, "relay", "relay (2)", |s| { + let g = &s.items[&ItemId::new(ItemKind::Group, "pool")].units["all"]; + assert_eq!(g["providers"], json!(["官方", "relay (2)"])); + let r = &s.items[&ItemId::new(ItemKind::Route, "长上下文")].units["all"]; + assert_eq!(r["rules"][0]["to"], json!("relay (2)")); + let o = &s.items[&ItemId::new(ItemKind::ProviderOrder, "")].units["all"]; + assert_eq!(o, &json!(["官方", "relay (2)", "chatgpt"])); + }), + (ItemKind::Proxy, "hk", "hk2", |s| { + let c = &s.items[&ItemId::new(ItemKind::Provider, "relay")].units["conn"]; + assert_eq!(c["proxy"], json!("hk2")); + }), + (ItemKind::PriceSheet, "中转", "中转 (2)", |s| { + let b = &s.items[&ItemId::new(ItemKind::Provider, "relay")].units["billing"]; + assert_eq!(b["pricing"], json!("中转 (2)")); + }), + (ItemKind::Key, "codex", "codex-2", |s| { + let r = &s.items[&ItemId::new(ItemKind::Route, "长上下文")].units["all"]; + assert_eq!(r["rules"][0]["when"]["client"], json!("codex-2")); + }), + (ItemKind::Key, "default", "main", |s| { + // 默认密钥原来是「叫 default 的那把」：改名之后要写明 + let g = &s.items[&ItemId::new(ItemKind::Global, "default_key")]; + assert_eq!(g.units["default_key"], json!("main")); + }), + (ItemKind::Alias, "fast", "quick", |s| { + let k = &s.items[&ItemId::new(ItemKind::Key, "codex")].units["all"]; + assert_eq!(k["allow"], json!(["quick"])); + }), + (ItemKind::Group, "pool", "pool2", |s| { + let r = &s.items[&ItemId::new(ItemKind::Route, "default")].units["all"]; + assert_eq!(r["rules"][0]["to"], json!("pool2")); + }), + (ItemKind::Route, "长上下文", "long", |s| { + let k = &s.items[&ItemId::new(ItemKind::Key, "codex")].units["all"]; + assert_eq!(k["route"], json!("long")); + let o = &s.items[&ItemId::new(ItemKind::RouteOrder, "")].units["all"]; + assert_eq!(o, &json!(["default", "long"])); + }), + (ItemKind::Route, "default", "main", |s| { + let g = &s.items[&ItemId::new(ItemKind::Global, "default_route")]; + assert_eq!(g.units["default_route"], json!("main")); + }), + (ItemKind::SecurityRule, "content/脏话", "粗话", |s| { + assert!( + s.items + .contains_key(&ItemId::new(ItemKind::SecurityRule, "content/粗话")) + ); + }), + ]; + for (kind, old, new, check) in cases { + let mut s = snap(&p); + rename(&mut s, &id(kind, old), new); + assert!(!s.items.contains_key(&id(kind, old)), "{old}"); + check(&s); + // 改了名的那一份落得回去、过得了校验 + let out = materialize_with(&p, &s, &Fake::new()).unwrap_or_else(|e| panic!("{old}: {e}")); + tw_config::try_parse(&out.config_yaml) + .unwrap_or_else(|e| panic!("{old} → {new}: {e}\n{}", out.config_yaml)); + assert_eq!(normalized(&snap(&out)), normalized(&s), "{old} → {new}"); + } + // 名字被占了、没有名字的项目：不动 + let mut s = snap(&p); + let before = s.clone(); + rename(&mut s, &id(ItemKind::Provider, "relay"), "官方"); + rename(&mut s, &id(ItemKind::Global, "failover"), "x"); + assert_eq!(s, before); + assert_eq!(free_name(&s, ItemKind::Provider, "relay"), "relay (2)"); + assert_eq!(free_name(&s, ItemKind::Provider, "new"), "new"); +} + +#[test] +fn retain_trims_the_orders_too() { + let mut s = snap(&pack(&rich())); + s.retain(&[id(ItemKind::Provider, "relay")].into_iter().collect()); + assert_eq!( + s.items.keys().map(|i| i.to_string()).collect::>(), + ["provider `relay`", "provider_order"] + ); + assert_eq!( + s.items[&id(ItemKind::ProviderOrder, "")].units["all"], + json!(["relay"]) + ); +} diff --git a/crates/tw-pack/src/model/write.rs b/crates/tw-pack/src/model/write.rs new file mode 100644 index 00000000..ff3355bc --- /dev/null +++ b/crates/tw-pack/src/model/write.rs @@ -0,0 +1,800 @@ +//! 把目标状态落回配置文本：[`materialize`]。 +//! +//! # 两条路 +//! +//! 先**按项目改原文**（`tw_config::edit`）：删掉目标里没有的，改掉变了的，加上新的，再排 +//! 顺序、改全局设置的字段。没变的项目一个字节都不动，旁边的注释也留着。改完读回来拆一遍， +//! 和目标对得上才算数。 +//! +//! 对不上（手写成了行内写法、锚点这类按项目改不动的）就**整份重写**：本机那几节原样留着， +//! 其余照目标生成。注释没了，但内容一定对。两条路都对不上就报错，什么都不写。 + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::Value as Json; +use serde_yaml_ng::{Mapping, Value as Yaml}; +use tw_config::edit::{self, Section}; +use tw_config::{Config, Plugin}; +use tw_yaml::Step; + +use super::items::{ + alias_models, approved_source, global_path, order_names, provider_mapping, rule_name, + sha256_hex, whole_mapping, +}; +use super::json::{key, to_yaml, unflatten}; +use super::{ + Item, ItemId, ItemKind, LOCAL_SECTIONS, ModelError, PluginCodec, Snapshot, codec, global, + normalized, parse, snapshot_with, unit, +}; +use crate::{Pack, PluginFiles}; + +const CLIENTS: Section = Section { + path: &["clients"], + what: "gateway key", + key: "name", +}; +const REDACT_RULES: Section = Section { + path: &["security", "redact", "custom"], + what: "custom rule", + key: "name", +}; +const TOOL_RULES: Section = Section { + path: &["security", "inspect_tools", "custom"], + what: "custom rule", + key: "name", +}; +const CONTENT_RULES: Section = Section { + path: &["security", "content", "custom"], + what: "custom rule", + key: "name", +}; + +/// 没有批准过的插件在配置里写的哈希：空文件的 SHA-256。哪个插件文件都对不上它，插件就 +/// 停在「文件变了、待批准」 +const NOT_APPROVED: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + +/// 以 `local` 为底，落成 `target` 那个样子：本机设置原样保留，其余换成 `target`；没变的 +/// 项目保留原文里的注释与顺序。用进程里装的插件拆法 +pub fn materialize(local: &Pack, target: &Snapshot) -> Result { + materialize_with(local, target, codec()) +} + +/// 同 [`materialize`]，插件拆法由调用方给。 +/// +/// 插件： +/// +/// - 代码没变的（包括默认插件）在本机的文件上改数据，字节尽量不动；代码变了的、新来的用 +/// 目标里带着的原文件（[`Item::files`]），没有就按单位拼出源码；默认插件在本机没有时 +/// 用这一份 Core 自带的那一份，也没有就不加 +/// - **改得了工具调用的插件，开着、而代码在本机没批准过（新来的、代码变了的、本机原来 +/// 停用着的）时不批准**：文件写下去，配置里的哈希留着本机原来批准的那一份（没有就是 +/// 一个谁都对不上的），插件停在「待批准」，要在本机点过头才跑 +/// - 别的照目标批准：哈希就是写下去的文件的 +pub fn materialize_with( + local: &Pack, + target: &Snapshot, + codec: &dyn PluginCodec, +) -> Result { + let cfg = parse(&local.config_yaml)?; + let current = snapshot_with(local, codec)?; + let plugins = plugin_outcomes(local, &cfg, ¤t, target, codec)?; + let want = complete_orders(normalized(target), &cfg, &plugins); + let prefs = target + .get(&ItemId::new(ItemKind::ClientPrefs, "")) + .map(|i| unflatten(&i.units)); + let files: BTreeMap = plugins + .iter() + .map(|(id, o)| (id.clone(), o.files.clone())) + .collect(); + let (files, prefs) = (&files, &prefs); + let finish = |text: String| Pack { + manifest: local.manifest.clone(), + config_yaml: text, + plugins: files.clone(), + client_prefs: prefs.clone(), + }; + + let edited = by_edits(&local.config_yaml, &cfg, ¤t, target, &want, &plugins) + .map(finish) + .ok() + .filter(|p| matches(p, &want, codec)); + if let Some(p) = edited { + return Ok(p); + } + let rebuilt = finish(rebuild(&local.config_yaml, &cfg, target, &want, &plugins)?); + if matches(&rebuilt, &want, codec) { + return Ok(rebuilt); + } + Err(ModelError::Verify { + detail: first_difference(&rebuilt, &want, codec), + }) +} + +/// 落下去的配置读回来拆一遍，和目标一样吗。**插件的脚本和设置不比**：待批准的插件读回来 +/// 按本机原来批准的那一份算，那正是要的样子；它们的文件由这里直接写 +fn matches(p: &Pack, want: &Snapshot, codec: &dyn PluginCodec) -> bool { + match snapshot_with(p, codec) { + Ok(got) => comparable(&normalized(&got)) == comparable(want), + Err(_) => false, + } +} + +fn comparable(s: &Snapshot) -> Snapshot { + let mut s = s.clone(); + for (id, item) in s.items.iter_mut() { + if id.kind == ItemKind::Plugin { + item.units.retain(|u, _| u == unit::ENABLED); + } + } + s +} + +fn first_difference(p: &Pack, want: &Snapshot, codec: &dyn PluginCodec) -> String { + let got = match snapshot_with(p, codec) { + Ok(s) => comparable(&normalized(&s)), + Err(e) => return e.to_string(), + }; + let want = comparable(want); + for (id, w) in &want.items { + match got.items.get(id) { + None => return format!("{id} is missing"), + Some(g) if g.units != w.units => return format!("{id} differs"), + _ => {} + } + } + match got.items.keys().find(|id| !want.items.contains_key(id)) { + Some(id) => format!("{id} is extra"), + None => "the order differs".into(), + } +} + +/// 落下去之后各种项目的先后：本机原来就有的按原来的先后、新的接在后面（按名字），再照 +/// 目标里的顺序项目排 —— 顺序项目里没列出来的接在列出来的后面。目标里的顺序项目缺了谁、 +/// 多了谁，落下去之后读回来都是这个样子，比较时按它 +fn complete_orders( + mut want: Snapshot, + cfg: &Config, + plugins: &BTreeMap, +) -> Snapshot { + for (kind, before) in [ + ( + ItemKind::Provider, + cfg.providers + .iter() + .map(|p| p.name.clone()) + .collect::>(), + ), + ( + ItemKind::Route, + cfg.routes.iter().map(|r| r.name.clone()).collect(), + ), + ( + ItemKind::Plugin, + cfg.plugins.iter().map(|p| p.id.clone()).collect(), + ), + ] { + let order_id = ItemId::new(kind.order().expect("these kinds have an order"), ""); + let present = |n: &String| { + want.items.contains_key(&ItemId::new(kind, n.clone())) + && (kind != ItemKind::Plugin || plugins.contains_key(n)) + }; + let mut all: Vec = before.into_iter().filter(|n| present(n)).collect(); + for (id, _) in want.of_kind(kind) { + if !all.contains(&id.name) && present(&id.name) { + all.push(id.name.clone()); + } + } + let listed = want + .get(&order_id) + .map(|o| order_names(&o.units)) + .unwrap_or_default(); + let fin = arrange(&all, &listed); + if fin.is_empty() { + want.items.remove(&order_id); + } else { + want.items.insert( + order_id, + Item { + units: BTreeMap::from([( + unit::ALL.to_string(), + Json::Array(fin.into_iter().map(Json::String).collect()), + )]), + ..Default::default() + }, + ); + } + } + want.items + .retain(|id, _| id.kind != ItemKind::Plugin || plugins.contains_key(&id.name)); + want +} + +// ─────────────────────────────────────────────────────────── 插件 + +struct PluginOut { + entry: Plugin, + files: PluginFiles, + /// 配置里那一条和本机的不一样（或者是新的） + changed: bool, +} + +fn plugin_outcomes( + local: &Pack, + cfg: &Config, + current: &Snapshot, + target: &Snapshot, + codec: &dyn PluginCodec, +) -> Result, ModelError> { + let mut out = BTreeMap::new(); + for (id, t) in target.of_kind(ItemKind::Plugin) { + let name = id.name.as_str(); + let le = cfg.plugins.iter().find(|p| p.id == name); + let lf = local.plugins.get(name); + let li = current.items.get(id); + if let (Some(le), Some(lf), Some(li)) = (le, lf, li) + && li.units == t.units + { + out.insert( + name.to_string(), + PluginOut { + entry: le.clone(), + files: lf.clone(), + changed: false, + }, + ); + continue; + } + let enabled = t + .units + .get(unit::ENABLED) + .and_then(Json::as_bool) + .unwrap_or(true); + let t_script = t.units.get(unit::SCRIPT); + let local_approved = match (le, lf) { + (Some(le), Some(lf)) => { + let a = approved_source(le, lf); + (sha256_hex(a) == le.sha256).then(|| a.to_vec()) + } + _ => None, + }; + let same_code = li.is_some_and(|li| li.units.get(unit::SCRIPT) == t_script) && lf.is_some(); + let text = |b: &[u8]| String::from_utf8_lossy(b).into_owned(); + let base: Option = if same_code { + lf.map(|lf| text(le.map_or(&lf.script[..], |le| approved_source(le, lf)))) + } else if let Some(f) = t.files.as_ref().filter(|f| { + let src = text(&f.script); + let code = codec.split(&src).map_or(src, |(c, _)| c); + t_script.and_then(Json::as_str) == Some(code.as_str()) + }) { + Some(text(&f.script)) + } else if let Some(s) = t_script.and_then(Json::as_str) { + Some(s.to_string()) + } else if let (Some(le), Some(lf)) = (le, lf) { + // 默认插件：本机的那一份 + Some(text(approved_source(le, lf))) + } else { + codec.shipped(name) + }; + let Some(base) = base else { + // 默认插件，而这一份 Core 没有它：不加 + continue; + }; + let source = match t.units.get(unit::SETTINGS) { + Some(d) => codec.join(&base, d).ok_or_else(|| ModelError::Shape { + item: id.to_string(), + detail: "its settings do not fit its script".into(), + })?, + None => base, + }; + let approved_here = + le.is_some_and(|le| le.enabled) && same_code && local_approved.is_some(); + let gated = enabled && codec.needs_approval(&source) && !approved_here; + let pending_edit = match (le, lf) { + (Some(le), Some(lf)) => sha256_hex(&lf.script) != le.sha256, + _ => false, + }; + let (files, sha256) = if gated { + // 哈希留着本机原来批准的那一份（界面拿它和底稿比出改了什么）；它恰好就是要写下去 + // 的这一份（本机停用着、批准过同一份代码）时也不能留：留着就等于替人点了头 + let sha = match le { + Some(le) if le.sha256 != sha256_hex(source.as_bytes()) => le.sha256.clone(), + _ => NOT_APPROVED.to_string(), + }; + ( + PluginFiles { + script: source.into_bytes(), + approved: local_approved.clone(), + }, + sha, + ) + } else if same_code && pending_edit { + // 本机有一处还没批准的改动：留着它，底稿和哈希换成新的 + let sha = sha256_hex(source.as_bytes()); + ( + PluginFiles { + script: lf.map(|f| f.script.clone()).unwrap_or_default(), + approved: Some(source.into_bytes()), + }, + sha, + ) + } else { + let sha = sha256_hex(source.as_bytes()); + let bytes = source.into_bytes(); + ( + PluginFiles { + script: bytes.clone(), + approved: Some(bytes), + }, + sha, + ) + }; + let entry = Plugin { + id: name.to_string(), + file: Plugin::file_for(name), + sha256, + enabled, + }; + let changed = le != Some(&entry); + out.insert( + name.to_string(), + PluginOut { + entry, + files, + changed, + }, + ); + } + Ok(out) +} + +// ─────────────────────────────────────────────────────────── 按项目改 + +fn section_of(id: &ItemId) -> Option<(Section, String)> { + let s = match id.kind { + ItemKind::Provider => edit::PROVIDERS, + ItemKind::Proxy => edit::PROXIES, + ItemKind::Key => CLIENTS, + ItemKind::Group => edit::GROUPS, + ItemKind::Route => edit::ROUTES, + ItemKind::PriceSheet => edit::PRICE_SHEETS, + ItemKind::Plugin => edit::PLUGINS, + ItemKind::SecurityRule => { + let (guard, rule) = rule_name(&id.name)?; + let s = match guard { + "redact" => REDACT_RULES, + "inspect_tools" => TOOL_RULES, + _ => CONTENT_RULES, + }; + return Some((s, rule.to_string())); + } + _ => return None, + }; + Some((s, id.name.clone())) +} + +/// 一项在配置里的样子（名字键在里面） +fn mapping_of( + id: &ItemId, + item: &Item, + cfg: &Config, + plugins: &BTreeMap, +) -> Result { + let m = match id.kind { + ItemKind::Provider => { + let local = cfg.providers.iter().find(|p| p.name == id.name); + provider_mapping(id, &item.units, local)?.1 + } + ItemKind::Proxy => whole_mapping::(id, &id.name, &item.units)?.1, + ItemKind::Key => whole_mapping::(id, &id.name, &item.units)?.1, + ItemKind::Group => whole_mapping::(id, &id.name, &item.units)?.1, + ItemKind::Route => whole_mapping::(id, &id.name, &item.units)?.1, + ItemKind::PriceSheet => whole_mapping::(id, &id.name, &item.units)?.1, + ItemKind::SecurityRule => { + let Some((guard, rule)) = rule_name(&id.name) else { + return Err(ModelError::Shape { + item: id.to_string(), + detail: "the name is not /".into(), + }); + }; + match guard { + "redact" => whole_mapping::(id, rule, &item.units)?.1, + "inspect_tools" => { + whole_mapping::(id, rule, &item.units)?.1 + } + _ => whole_mapping::(id, rule, &item.units)?.1, + } + } + ItemKind::Plugin => { + let Some(o) = plugins.get(&id.name) else { + return Err(ModelError::Shape { + item: id.to_string(), + detail: "there is no script for it".into(), + }); + }; + match serde_yaml_ng::to_value(&o.entry) { + Ok(Yaml::Mapping(m)) => m, + _ => { + return Err(ModelError::Shape { + item: id.to_string(), + detail: "it does not serialize to a mapping".into(), + }); + } + } + } + _ => { + return Err(ModelError::Shape { + item: id.to_string(), + detail: "it is not a list entry".into(), + }); + } + }; + Ok(m) +} + +fn by_edits( + text: &str, + cfg: &Config, + current: &Snapshot, + target: &Snapshot, + want: &Snapshot, + plugins: &BTreeMap, +) -> Result { + let mut text = text.to_string(); + let wanted = + |id: &ItemId| -> bool { id.kind != ItemKind::Plugin || plugins.contains_key(&id.name) }; + + // 删：先删引用别人的，再删被引用的（文本上无所谓，读起来顺） + for id in current.items.keys().rev() { + if !id.kind.is_named() || (target.items.contains_key(id) && wanted(id)) { + continue; + } + text = remove(&text, id)?; + } + // 改、加 + for (id, t) in &target.items { + if !id.kind.is_named() || !wanted(id) { + continue; + } + let existing = current.items.get(id); + let same = match id.kind { + ItemKind::Plugin => existing.is_some() && !plugins[&id.name].changed, + _ => existing.is_some_and(|c| c.units == t.units), + }; + if same { + continue; + } + text = upsert(&text, id, t, existing.is_some(), cfg, plugins)?; + } + // 顺序 + for (kind, section) in [ + (ItemKind::Provider, edit::PROVIDERS), + (ItemKind::Route, edit::ROUTES), + (ItemKind::Plugin, edit::PLUGINS), + ] { + let Some(order) = kind.order().and_then(|o| target.get(&ItemId::new(o, ""))) else { + continue; + }; + let doc = edit::parse(&text)?; + let now = names_in(&doc, section); + let desired = arrange(&now, &order_names(&order.units)); + if desired != now { + text = edit::reorder(&text, section, &desired)?; + } + } + // 全局设置：逐个字段 + for section in global::ALL { + let id = ItemId::new(ItemKind::Global, *section); + let empty = BTreeMap::new(); + let l = current.get(&id).map_or(&empty, |i| &i.units); + let t = want.get(&id).map_or(&empty, |i| &i.units); + if l == t { + continue; + } + let paths: BTreeSet<&String> = l.keys().chain(t.keys()).collect(); + for p in paths { + let tv = t.get(p); + if l.get(p) == tv { + continue; + } + let steps: Vec = global_path(p).iter().map(Step::key).collect(); + text = edit::set(&text, &steps, tv.map(to_yaml).as_ref())?; + } + } + Ok(text) +} + +fn remove(text: &str, id: &ItemId) -> Result { + if id.kind == ItemKind::Alias { + return Ok(edit::remove_alias(text, &id.name)?); + } + let Some((section, name)) = section_of(id) else { + return Ok(text.to_string()); + }; + Ok(edit::remove(text, section, &name)?) +} + +fn upsert( + text: &str, + id: &ItemId, + item: &Item, + exists: bool, + cfg: &Config, + plugins: &BTreeMap, +) -> Result { + if id.kind == ItemKind::Alias { + let alias = tw_config::Alias { + name: id.name.clone(), + models: alias_models(id, &item.units)?, + }; + let current = exists.then_some(id.name.as_str()); + return Ok(edit::upsert_alias(text, current, &alias)?); + } + let Some((section, name)) = section_of(id) else { + return Ok(text.to_string()); + }; + let m = mapping_of(id, item, cfg, plugins)?; + let current = exists.then_some(name.as_str()); + Ok(edit::upsert(text, section, current, &m)?) +} + +/// 一段列表里各项的名字，按写的顺序 +fn names_in(doc: &Yaml, section: Section) -> Vec { + let mut cur = doc; + for k in section.path { + match cur.get(*k) { + Some(v) => cur = v, + None => return Vec::new(), + } + } + cur.as_sequence() + .into_iter() + .flatten() + .filter_map(|it| { + it.get(section.key) + .and_then(Yaml::as_str) + .map(str::to_string) + }) + .collect() +} + +/// 按 `order` 排 `now`：`order` 里有的按它的先后排在前面，没有的按原来的先后接在后面 +fn arrange(now: &[String], order: &[String]) -> Vec { + let present: BTreeSet<&String> = now.iter().collect(); + let mut out: Vec = order + .iter() + .filter(|n| present.contains(n)) + .cloned() + .collect(); + let mut seen: BTreeSet = out.iter().cloned().collect(); + for n in now { + if seen.insert(n.clone()) { + out.push(n.clone()); + } + } + // 同一个名字在 order 里写了两次：只留第一次 + let mut once = BTreeSet::new(); + out.retain(|n| once.insert(n.clone())); + out +} + +// ─────────────────────────────────────────────────────────── 整份重写 + +fn rebuild( + text: &str, + cfg: &Config, + target: &Snapshot, + want: &Snapshot, + plugins: &BTreeMap, +) -> Result { + let doc: Yaml = serde_yaml_ng::from_str(text).unwrap_or(Yaml::Null); + let mut out = Mapping::new(); + out.insert( + key("version"), + doc.get("version") + .cloned() + .unwrap_or(Yaml::Number(tw_config::SCHEMA_VERSION.into())), + ); + let local = |k: &str| doc.get(k).cloned(); + + // 一种项目在目标里的名字：原来就有的按原来的先后，新的接在后面；有顺序项目的照它排 + let names = |kind: ItemKind, before: Vec| -> Vec { + let mut all: Vec = before + .into_iter() + .filter(|n| target.items.contains_key(&ItemId::new(kind, n.clone()))) + .collect(); + for (id, _) in target.of_kind(kind) { + if !all.contains(&id.name) { + all.push(id.name.clone()); + } + } + if kind == ItemKind::Plugin { + all.retain(|n| plugins.contains_key(n)); + } + match kind.order().and_then(|o| target.get(&ItemId::new(o, ""))) { + Some(o) => arrange(&all, &order_names(&o.units)), + None => all, + } + }; + let list = |kind: ItemKind, before: Vec| -> Result, ModelError> { + names(kind, before) + .into_iter() + .map(|n| { + let id = ItemId::new(kind, n); + let item = &target.items[&id]; + mapping_of(&id, item, cfg, plugins).map(Yaml::Mapping) + }) + .collect() + }; + + if let Some(v) = local("listen") { + out.insert(key("listen"), v); + } + let seq = |v: Vec| (!v.is_empty()).then_some(Yaml::Sequence(v)); + if let Some(v) = seq(list( + ItemKind::Key, + cfg.clients.iter().map(|c| c.name.clone()).collect(), + )?) { + out.insert(key("clients"), v); + } + if let Some(v) = seq(list( + ItemKind::Provider, + cfg.providers.iter().map(|p| p.name.clone()).collect(), + )?) { + out.insert(key("providers"), v); + } + if let Some(v) = seq(list( + ItemKind::Proxy, + cfg.proxies.iter().map(|p| p.name.clone()).collect(), + )?) { + out.insert(key("proxies"), v); + } + + // 全局设置：每一节按单位拼出来，读进类型核过 + let section = |name: &str| -> Result { + let mut v = Json::Object(Default::default()); + if let Some(item) = want.get(&ItemId::new(ItemKind::Global, name)) { + for (path, value) in &item.units { + let parts = global_path(path); + super::items::set_path(&mut v, &parts[1..], value.clone()); + } + } + super::items::section_normalized(name, &v).map_err(|detail| ModelError::Shape { + item: format!("global `{name}`"), + detail, + }) + }; + let mut pricing = section(global::PRICING)?; + let sheets = list( + ItemKind::PriceSheet, + cfg.pricing.sheets.iter().map(|s| s.name.clone()).collect(), + )?; + let mut pricing_yaml = match to_yaml(&pricing.take()) { + Yaml::Mapping(m) => m, + _ => Mapping::new(), + }; + if !sheets.is_empty() { + pricing_yaml.insert(key("sheets"), Yaml::Sequence(sheets)); + } + if !pricing_yaml.is_empty() { + out.insert(key("pricing"), Yaml::Mapping(pricing_yaml)); + } + let probes = to_yaml(§ion(global::CLIENT_PROBES)?); + if probes.as_mapping().is_some_and(|m| !m.is_empty()) { + out.insert(key("client_probes"), probes); + } + let mut security = match to_yaml(§ion(global::SECURITY)?) { + Yaml::Mapping(m) => m, + _ => Mapping::new(), + }; + for (guard, before) in [ + ( + "redact", + cfg.security + .redact + .custom + .iter() + .map(|r| r.name.clone()) + .collect::>(), + ), + ( + "inspect_tools", + cfg.security + .inspect_tools + .custom + .iter() + .map(|r| r.name.clone()) + .collect(), + ), + ( + "content", + cfg.security + .content + .custom + .iter() + .map(|r| r.name.clone()) + .collect(), + ), + ] { + let prefix = format!("{guard}/"); + let before = before.into_iter().map(|n| format!("{prefix}{n}")).collect(); + let own: Vec = names(ItemKind::SecurityRule, before) + .into_iter() + .filter(|n| n.starts_with(&prefix)) + .map(|n| { + let id = ItemId::new(ItemKind::SecurityRule, n); + mapping_of(&id, &target.items[&id], cfg, plugins).map(Yaml::Mapping) + }) + .collect::>()?; + if !own.is_empty() { + let entry = security + .entry(key(guard)) + .or_insert_with(|| Yaml::Mapping(Mapping::new())); + if let Yaml::Mapping(g) = entry { + g.insert(key("custom"), Yaml::Sequence(own)); + } + } + } + if !security.is_empty() { + out.insert(key("security"), Yaml::Mapping(security)); + } + if let Some(v) = local("retention") { + out.insert(key("retention"), v); + } + let failover = to_yaml(§ion(global::FAILOVER)?); + if failover.as_mapping().is_some_and(|m| !m.is_empty()) { + out.insert(key("failover"), failover); + } + // 别名表：名字 → 一个模型是字符串，几个是列表 + let mut aliases = Mapping::new(); + for n in names( + ItemKind::Alias, + cfg.aliases.iter().map(|a| a.name.clone()).collect(), + ) { + let id = ItemId::new(ItemKind::Alias, n.clone()); + let models = alias_models(&id, &target.items[&id].units)?; + let v = match models.as_slice() { + [one] => Yaml::String(one.clone()), + many => Yaml::Sequence(many.iter().cloned().map(Yaml::String).collect()), + }; + aliases.insert(key(&n), v); + } + if !aliases.is_empty() { + out.insert(key("aliases"), Yaml::Mapping(aliases)); + } + if let Some(v) = seq(list( + ItemKind::Group, + cfg.groups.iter().map(|g| g.name.clone()).collect(), + )?) { + out.insert(key("groups"), v); + } + if let Some(v) = seq(list( + ItemKind::Route, + cfg.routes.iter().map(|r| r.name.clone()).collect(), + )?) { + out.insert(key("routes"), v); + } + for name in [global::DEFAULT_ROUTE, global::DEFAULT_KEY] { + if let Some(v) = want + .get(&ItemId::new(ItemKind::Global, name)) + .and_then(|i| i.units.get(name)) + { + out.insert(key(name), to_yaml(v)); + } + } + if let Some(v) = seq(list( + ItemKind::Plugin, + cfg.plugins.iter().map(|p| p.id.clone()).collect(), + )?) { + out.insert(key("plugins"), v); + } + for k in LOCAL_SECTIONS { + if !out.contains_key(key(k)) + && let Some(v) = local(k) + { + out.insert(key(k), v); + } + } + serde_yaml_ng::to_string(&Yaml::Mapping(out)).map_err(|e| ModelError::Verify { + detail: e.to_string(), + }) +} diff --git a/crates/tw-pack/src/tests.rs b/crates/tw-pack/src/tests.rs new file mode 100644 index 00000000..d4e9e03b --- /dev/null +++ b/crates/tw-pack/src/tests.rs @@ -0,0 +1,334 @@ +//! 配置包的格式：写出去读得回来，读的时候不信任文件。 + +use std::io::Write; + +use super::*; + +fn fast() { + // 口令加密的工作量调低：测试不用每次等一秒 + TEST_WORK_FACTOR.store(10, std::sync::atomic::Ordering::Relaxed); +} + +fn sample() -> Pack { + let mut plugins = BTreeMap::new(); + plugins.insert( + "add-date".to_string(), + PluginFiles { + script: b"export const manifest = { name: \"x\", api: 1 };\n".to_vec(), + approved: Some(b"export const manifest = { name: \"x\", api: 1 };\n".to_vec()), + }, + ); + plugins.insert( + "only-script".to_string(), + PluginFiles { + script: b"// no approved copy\n".to_vec(), + approved: None, + }, + ); + Pack { + manifest: Manifest::new(PackKind::Full, PackOrigin::Export), + config_yaml: "version: 1\n# 注释\nproviders:\n - { name: a, base_url: https://a.example, key: sk-a }\nclients:\n - { name: default, key: tw-k }\n".into(), + plugins, + client_prefs: Some(serde_json::json!({"language": "zh", "menubar": {"show": "tokens"}})), + } +} + +#[test] +fn a_pack_reads_back_exactly_as_written_with_each_seal() { + fast(); + let pack = sample(); + let id = age::x25519::Identity::generate(); + let recipient = id.to_public(); + for (seal, unseal) in [ + (Seal::None, Unseal::None), + ( + Seal::Passphrase("correct horse"), + Unseal::Passphrase("correct horse"), + ), + (Seal::Recipient(&recipient), Unseal::Identity(&id)), + ] { + let encrypted = !matches!(seal, Seal::None); + let bytes = write(&pack, seal).unwrap(); + assert_eq!(is_encrypted(&bytes), encrypted); + let back = read(&bytes, unseal).unwrap(); + assert_eq!(back.config_yaml, pack.config_yaml); + assert_eq!(back.plugins, pack.plugins); + assert_eq!(back.client_prefs, pack.client_prefs); + assert_eq!(back.manifest.encrypted, encrypted); + assert_eq!(back.manifest.counts.providers, 1); + assert_eq!(back.manifest.counts.clients, 1); + assert_eq!(back.manifest.origin, PackOrigin::Export); + assert_eq!(back.manifest.core_version, CORE_VERSION); + } +} + +/// 内容一样，字节就一样（只差创建时间）：备份去重、同步比较靠得住 +#[test] +fn the_same_content_writes_the_same_bytes() { + let mut a = sample(); + a.manifest.created_at_ms = 1; + let b = a.clone(); + assert_eq!( + write(&a, Seal::None).unwrap(), + write(&b, Seal::None).unwrap() + ); +} + +#[test] +fn opening_an_encrypted_pack_needs_the_right_secret() { + fast(); + let pack = sample(); + let by_pass = write(&pack, Seal::Passphrase("right one")).unwrap(); + let id = age::x25519::Identity::generate(); + let by_key = write(&pack, Seal::Recipient(&id.to_public())).unwrap(); + let other = age::x25519::Identity::generate(); + let cases: Vec<(&[u8], Unseal, PackError)> = vec![ + (&by_pass, Unseal::None, PackError::NeedPassphrase), + ( + &by_pass, + Unseal::Passphrase("wrong one"), + PackError::WrongPassphrase, + ), + (&by_pass, Unseal::Identity(&id), PackError::NeedPassphrase), + (&by_key, Unseal::None, PackError::NeedKey), + (&by_key, Unseal::Passphrase("right one"), PackError::NeedKey), + (&by_key, Unseal::Identity(&other), PackError::WrongKey), + ]; + for (bytes, unseal, want) in cases { + assert_eq!(read(bytes, unseal).unwrap_err(), want); + } +} + +/// 手工做一个 ZIP：测「读的时候不信任文件」 +fn zip_of(entries: &[(&str, &[u8])]) -> Vec { + let mut z = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + let opts = zip::write::SimpleFileOptions::default(); + for (name, bytes) in entries { + z.start_file(*name, opts).unwrap(); + z.write_all(bytes).unwrap(); + } + z.finish().unwrap().into_inner() +} + +fn manifest_json(format: u32) -> Vec { + let mut m = Manifest::new(PackKind::Full, PackOrigin::Export); + m.format = format; + serde_json::to_vec(&m).unwrap() +} + +#[test] +fn a_pack_with_anything_unexpected_inside_is_refused_whole() { + let m = manifest_json(FORMAT); + let cfg: &[u8] = b"version: 1\n"; + for (bad, why) in [ + ("../config.yaml", "climbs out"), + ("/etc/passwd", "absolute"), + ("plugins/../../x.js", "climbs out through plugins"), + ("plugins/Bad_Id.js", "an id that is not one"), + ("plugins/a.txt", "not a script"), + ( + "plugins/.approved/../a.js", + "climbs out of the approved copies", + ), + ("README.md", "a file a pack does not have"), + ("plugins/sub/a.js", "a nested directory"), + ] { + let bytes = zip_of(&[("manifest.json", &m), ("config.yaml", cfg), (bad, b"x")]); + match read(&bytes, Unseal::None) { + Err(PackError::BadEntry { name }) => assert_eq!(name, bad, "{why}"), + other => panic!("{why}: {other:?}"), + } + } + // 只有底稿没有脚本 + let bytes = zip_of(&[ + ("manifest.json", &m), + ("config.yaml", cfg), + ("plugins/.approved/a.js", b"x"), + ]); + assert!(matches!( + read(&bytes, Unseal::None), + Err(PackError::BadEntry { .. }) + )); + // 目录条目无害 + let bytes = zip_of(&[ + ("manifest.json", &m), + ("config.yaml", cfg), + ("plugins/", b""), + ("plugins/.approved/", b""), + ("plugins/a.js", b"x"), + ]); + assert_eq!(read(&bytes, Unseal::None).unwrap().plugins.len(), 1); +} + +#[test] +fn a_pack_missing_its_manifest_or_config_says_which() { + let m = manifest_json(FORMAT); + for (entries, missing) in [ + (vec![("config.yaml", &b"version: 1\n"[..])], "manifest.json"), + (vec![("manifest.json", &m[..])], "config.yaml"), + ] { + let e = read(&zip_of(&entries), Unseal::None).unwrap_err(); + assert_eq!( + e, + PackError::MissingEntry { + name: missing.into() + } + ); + assert_eq!(e.msg().code, "pack.missing_entry"); + } +} + +#[test] +fn a_newer_format_is_too_new_and_not_corrupt() { + let m = manifest_json(FORMAT + 1); + let bytes = zip_of(&[("manifest.json", &m), ("config.yaml", b"version: 1\n")]); + match read(&bytes, Unseal::None) { + Err(PackError::TooNew { + format, + core_version, + }) => { + assert_eq!(format, FORMAT + 1); + assert_eq!(core_version, CORE_VERSION); + } + other => panic!("{other:?}"), + } + // 新的格式里清单多了不认识的字段，照样先说太新 + let raw = serde_json::json!({"format": FORMAT + 1, "whatever": [1, 2]}); + let bytes = zip_of(&[ + ("manifest.json", raw.to_string().as_bytes()), + ("config.yaml", b"version: 1\n"), + ]); + assert!(matches!( + read(&bytes, Unseal::None), + Err(PackError::TooNew { .. }) + )); + // 同一个格式里多出来的字段不碍事：新一点的 Core 写的、格式没变的包读得进来 + let mut raw: serde_json::Value = serde_json::from_slice(&manifest_json(FORMAT)).unwrap(); + raw["something_new"] = serde_json::json!(true); + raw["origin"] = serde_json::json!("a_new_origin"); + let bytes = zip_of(&[ + ("manifest.json", raw.to_string().as_bytes()), + ("config.yaml", b"version: 1\n"), + ]); + let p = read(&bytes, Unseal::None).unwrap(); + assert_eq!(p.manifest.origin, PackOrigin::Other); +} + +#[test] +fn garbage_is_corrupt_and_says_so() { + for bytes in [&b"not a zip at all"[..], b"PK\x03\x04broken", b""] { + let e = read(bytes, Unseal::None).unwrap_err(); + assert!(matches!(e, PackError::Corrupt { .. }), "{e:?}"); + assert_eq!(e.msg().code, "pack.corrupt"); + } + let m = manifest_json(FORMAT); + let bytes = zip_of(&[("manifest.json", &m), ("config.yaml", &[0xff, 0xfe])]); + assert!(matches!( + read(&bytes, Unseal::None), + Err(PackError::Corrupt { .. }) + )); + let bytes = zip_of(&[ + ("manifest.json", b"{not json"), + ("config.yaml", b"version: 1\n"), + ]); + assert!(matches!( + read(&bytes, Unseal::None), + Err(PackError::Corrupt { .. }) + )); +} + +/// 解压之后的大小按真读到的算：一个压得很小、解开很大的包挡在门口 +#[test] +fn a_pack_that_unpacks_to_more_than_the_limit_is_refused() { + let m = manifest_json(FORMAT); + let big = vec![b'a'; MAX_BYTES + 1]; + let mut z = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + let opts = zip::write::SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Deflated) + .large_file(true); + z.start_file("manifest.json", opts).unwrap(); + z.write_all(&m).unwrap(); + z.start_file("config.yaml", opts).unwrap(); + z.write_all(&big).unwrap(); + let bytes = z.finish().unwrap().into_inner(); + assert!(bytes.len() < MAX_BYTES, "the compressed file is small"); + assert_eq!( + read(&bytes, Unseal::None).unwrap_err(), + PackError::TooLarge { limit: MAX_BYTES } + ); + // 本身就太大的文件不打开 + assert_eq!( + read(&vec![0u8; MAX_BYTES + 1], Unseal::None).unwrap_err(), + PackError::TooLarge { limit: MAX_BYTES } + ); + // 写的时候也挡 + let mut p = sample(); + p.config_yaml = "a".repeat(MAX_BYTES * 2); + // 压得很小的内容照样按解压后的大小算：写得出去就得读得回来 + assert_eq!( + write(&p, Seal::None).unwrap_err(), + PackError::TooLarge { limit: MAX_BYTES } + ); +} + +#[test] +fn every_error_speaks_with_its_own_code() { + let all = [ + (PackError::NeedPassphrase, "pack.need_passphrase"), + (PackError::WrongPassphrase, "pack.wrong_passphrase"), + (PackError::NeedKey, "pack.need_key"), + (PackError::WrongKey, "pack.wrong_key"), + ( + PackError::TooNew { + format: 2, + core_version: "9.0.0".into(), + }, + "pack.too_new", + ), + (PackError::Corrupt { detail: "x".into() }, "pack.corrupt"), + (PackError::BadEntry { name: "x".into() }, "pack.bad_entry"), + ( + PackError::MissingEntry { name: "x".into() }, + "pack.missing_entry", + ), + (PackError::TooLarge { limit: MAX_BYTES }, "pack.too_large"), + ]; + for (e, code) in all { + assert_eq!(e.msg().code, code); + assert_eq!(e.to_string(), e.msg().text); + } + assert_eq!( + PackError::TooLarge { limit: MAX_BYTES }.msg().arg("limit"), + "16" + ); +} + +#[test] +fn versions_compare_as_numbers() { + assert!(!newer_than_us(CORE_VERSION)); + assert!(newer_than_us("999.0.0")); + assert!(!newer_than_us("0.0.1")); + assert!(!newer_than_us("not a version")); + assert_eq!(parse_version("0.77.0-rc.1"), Some((0, 77, 0))); + assert_eq!(parse_version("1.2"), Some((1, 2, 0))); +} + +#[test] +fn origins_keep_their_words() { + for o in [ + PackOrigin::Export, + PackOrigin::Share, + PackOrigin::Ui, + PackOrigin::Cli, + PackOrigin::External, + PackOrigin::Defaults, + PackOrigin::Sync, + PackOrigin::Import, + PackOrigin::Restore, + PackOrigin::Repair, + ] { + assert_eq!(serde_json::to_value(o).unwrap(), o.slug()); + assert_eq!(PackOrigin::from_slug(o.slug()), o); + } + assert_eq!(PackOrigin::from_slug("nope"), PackOrigin::Other); +} From 13dc22a891f8b372dc9f0ba2f8c368f15e1be197 Mon Sep 17 00:00:00 2001 From: fylorn <249551762+fylorn@users.noreply.github.com> Date: Mon, 19 Oct 2026 09:58:01 +0800 Subject: [PATCH 02/13] Export, import and local backups on the control plane and the CLI (#349) * Export, import and local backups on the control plane and the CLI - Endpoints: /config/items, /config/export, /config/import/preview, /config/import, /backups, /backups/file, /backups/preview, /backups/restore, /client-prefs, /pinned-keys. Import goes item by item (new, same, conflict, error; import, overwrite, rename, skip) and says what each row refers to; restore can take a subset of items, keeps the local settings, ChatGPT sign-ins and pinned keys, and in safe mode brings a version back whole with the current control key. - Local backups replace the change history: every write leaves a full configuration pack (plugins and client preferences included, control key masked) before and after, the last 10 are kept, rotations make none, and the old history files go. /config/history and /config/rollback are removed; ConfigOrigin drops rollback and gains sync, import, restore, repair. - New events gateway_keys_changed and client_prefs_changed. Over a remote connection the client preferences stay in the app. - RemoteBackups lets sync add the versions of the sync location to the list; pinned keys are kept in the data directory. - A disabled upstream may lack its credential, which is how imported ones arrive. A plugin that can rewrite tool calls is never approved by an import or a restore. - CLI: twcore export / import / backup list / backup restore; config history and config rollback are gone. - Control API version 50. Co-Authored-By: Claude Opus 5.5 * smoke: the backup endpoints instead of the removed history one Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- Cargo.lock | 4 + bin/twcore/Cargo.toml | 2 + bin/twcore/src/main.rs | 117 +-- bin/twcore/src/pack_cli.rs | 403 +++++++++++ crates/tw-api/msg-codes.txt | 17 +- crates/tw-api/src/ep.rs | 30 +- crates/tw-api/src/lib.rs | 76 +- crates/tw-api/src/pack.rs | 382 ++++++++++ crates/tw-api/src/ts.rs | 2 +- crates/tw-config/src/control_key.rs | 3 +- crates/tw-config/src/credential.rs | 40 +- crates/tw-config/src/history.rs | 498 ++----------- crates/tw-config/src/remote.rs | 7 +- crates/tw-config/src/wire.rs | 7 +- crates/tw-control/Cargo.toml | 3 + crates/tw-control/src/backups.rs | 794 +++++++++++++++++++++ crates/tw-control/src/config.rs | 86 ++- crates/tw-control/src/lib.rs | 50 +- crates/tw-control/tests/backups.rs | 566 +++++++++++++++ crates/tw-control/tests/contract.rs | 8 +- crates/tw-control/tests/gate.rs | 20 +- crates/tw-control/tests/m2_acceptance.rs | 38 +- crates/tw-control/tests/plugin_defaults.rs | 4 +- crates/tw-control/tests/plugins.rs | 20 +- crates/tw-control/tests/pricing.rs | 2 +- crates/tw-control/tests/resources.rs | 15 +- crates/tw-control/tests/sync.rs | 32 +- crates/tw-gateway/src/plugin/source.rs | 135 ++++ crates/tw-observe/src/bus.rs | 2 + crates/tw-pack/Cargo.toml | 2 + crates/tw-pack/src/backup.rs | 445 ++++++++++++ crates/tw-pack/src/disk.rs | 303 ++++++++ crates/tw-pack/src/export.rs | 227 ++++++ crates/tw-pack/src/import.rs | 589 +++++++++++++++ crates/tw-pack/src/import/tests.rs | 524 ++++++++++++++ crates/tw-pack/src/lib.rs | 13 +- crates/tw-pack/src/model/items.rs | 11 +- crates/tw-pack/src/model/mod.rs | 12 + crates/tw-pack/src/model/tests.rs | 7 +- crates/tw-pack/src/model/write.rs | 49 +- crates/tw-pack/src/restore.rs | 187 +++++ crates/tw-pack/src/restore/tests.rs | 105 +++ crates/tw-store/src/recorder.rs | 3 + docs/config.md | 37 +- docs/config.zh-CN.md | 20 +- scripts/smoke.sh | 2 +- 46 files changed, 5188 insertions(+), 711 deletions(-) create mode 100644 bin/twcore/src/pack_cli.rs create mode 100644 crates/tw-api/src/pack.rs create mode 100644 crates/tw-control/src/backups.rs create mode 100644 crates/tw-control/tests/backups.rs create mode 100644 crates/tw-pack/src/backup.rs create mode 100644 crates/tw-pack/src/disk.rs create mode 100644 crates/tw-pack/src/export.rs create mode 100644 crates/tw-pack/src/import.rs create mode 100644 crates/tw-pack/src/import/tests.rs create mode 100644 crates/tw-pack/src/restore.rs create mode 100644 crates/tw-pack/src/restore/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 03cbac05..7f74297d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3759,6 +3759,7 @@ dependencies = [ "tw-guard", "tw-link", "tw-observe", + "tw-pack", "tw-pricing", "tw-secret", "tw-store", @@ -3894,6 +3895,8 @@ dependencies = [ "sha2 0.11.0", "tempfile", "thiserror 2.0.20", + "tracing", + "tw-api", "tw-config", "tw-engine", "tw-pricing", @@ -4015,6 +4018,7 @@ dependencies = [ "tw-guard", "tw-link", "tw-observe", + "tw-pack", "tw-pricing", "tw-secret", "tw-store", diff --git a/bin/twcore/Cargo.toml b/bin/twcore/Cargo.toml index 3d2d9e65..8455b64c 100644 --- a/bin/twcore/Cargo.toml +++ b/bin/twcore/Cargo.toml @@ -16,6 +16,8 @@ categories.workspace = true [dependencies] tw-config = { workspace = true } tw-control = { workspace = true } +# `twcore export` / `import` / `backup`：配置包和本机备份 +tw-pack = { workspace = true } tw-gateway = { workspace = true } tw-yaml = { workspace = true } tw-api = { workspace = true } diff --git a/bin/twcore/src/main.rs b/bin/twcore/src/main.rs index 22466920..391ab9d1 100644 --- a/bin/twcore/src/main.rs +++ b/bin/twcore/src/main.rs @@ -11,6 +11,7 @@ use clap::{Parser, Subcommand}; mod call; mod lockfile; +mod pack_cli; mod proc; mod upgrade; @@ -103,6 +104,40 @@ enum Command { #[arg(short, long)] out: Option, }, + /// Export the configuration as a configuration pack (.twconfig) + Export { + /// Export only these upstreams (comma-separated), with what they refer to + #[arg(long, value_delimiter = ',')] + providers: Vec, + /// Leave the keys out: upstream keys, header values, AWS keys, proxy passwords and + /// gateway keys are left empty + #[arg(long)] + no_secrets: bool, + /// Encrypt the file with a passphrase, asked for or taken from TWCORE_PASSPHRASE + #[arg(long)] + encrypt: bool, + /// The file to write + #[arg(short, long)] + out: PathBuf, + }, + /// Import a configuration pack: what is not here is added, and what differs is chosen item + /// by item + Import { + /// The configuration pack (.twconfig) + file: PathBuf, + /// Show what would happen and change nothing + #[arg(long)] + dry_run: bool, + /// What to do with every item that differs from the one here; asked item by item + /// without it + #[arg(long, value_enum)] + on_conflict: Option, + }, + /// The last versions of the configuration, and restoring one + Backup { + #[command(subcommand)] + what: BackupCmd, + }, /// Replace this twcore with another release from GitHub; the configuration is left alone // // 只给单独装的 twcore 用（服务器上）。桌面应用包里的那份由应用更新，见 upgrade.rs @@ -140,6 +175,21 @@ enum RemoteCmd { Disable, } +#[derive(Subcommand)] +enum BackupCmd { + /// List the versions, newest first + List, + /// Go back to a version. Settings that belong to this machine, ChatGPT sign-ins and the + /// gateway keys clients set up here use are kept + Restore { + /// The id from `twcore backup list` + id: String, + /// Show what would change and change nothing + #[arg(long)] + dry_run: bool, + }, +} + #[derive(Subcommand)] enum ConfigCmd { /// Print the configuration as it is, with its version @@ -159,10 +209,6 @@ enum ConfigCmd { #[arg(long, conflicts_with_all = ["int", "bool_value"])] null: bool, }, - /// List the versions in the history - History, - /// Roll back to a version - Rollback { version: String }, } fn main() -> Result<()> { @@ -182,6 +228,8 @@ fn main() -> Result<()> { let cli = Cli::parse(); let path = cli.config.clone().unwrap_or_else(tw_config::default_path); + // 配置包、备份按「代码」「数据」分开比较插件：拆法要插件那一套 + tw_control::backups::install_codec(); match cli.command { Command::Init { force } => cmd_init(&path, force), @@ -191,6 +239,21 @@ fn main() -> Result<()> { Command::Config { what } => cmd_config(&path, what), Command::ControlKey { rotate } => cmd_control_key(&path, rotate), Command::Remote { what } => cmd_remote(&path, what.unwrap_or(RemoteCmd::Show)), + Command::Export { + providers, + no_secrets, + encrypt, + out, + } => pack_cli::export(&path, providers, no_secrets, encrypt, out), + Command::Import { + file, + dry_run, + on_conflict, + } => pack_cli::import(&path, &file, dry_run, on_conflict), + Command::Backup { what } => match what { + BackupCmd::List => pack_cli::list(&path), + BackupCmd::Restore { id, dry_run } => pack_cli::restore(&path, &id, dry_run), + }, Command::Call { path: endpoint, method, @@ -212,7 +275,7 @@ fn main() -> Result<()> { /// 改配置。 /// /// **不连控制面，直接操作文件。**理由是这个命令必须在 core 没跑的时候 -/// 也能用 —— 「配置写坏了导致 core 起不来」正是最需要 `config rollback` +/// 也能用 —— 「配置写坏了导致 core 起不来」正是最需要 `backup restore` /// 的时刻，而那时控制面根本不存在。 /// /// 代价是 core 正在跑时，改动要等它的文件监听发现（几百毫秒）。那条路 @@ -225,40 +288,6 @@ fn cmd_config(path: &Path, what: ConfigCmd) -> Result<()> { eprintln!("── {} · {}", c.path.display(), c.version()); Ok(()) } - ConfigCmd::History => { - let all = tw_config::history::list(path)?; - if all.is_empty() { - println!("No versions yet; the first change to the configuration makes one."); - return Ok(()); - } - let now = tw_config::store::read(path).map(|c| c.version()).ok(); - // 新的在前 —— 要找的几乎总是最近那几版 - for v in all.iter().rev() { - let mark = if Some(&v.version) == now.as_ref() { - "← current" - } else { - " " - }; - println!( - "{mark} {} {:<10} {} bytes {}", - v.version, - v.origin.label(), - v.bytes, - fmt_time(v.at_ms) - ); - } - println!(); - println!( - "To roll back: twcore config rollback (the first few characters are enough)" - ); - Ok(()) - } - ConfigCmd::Rollback { version } => { - let text = tw_config::history::rollback(path, &version)?; - println!("rolled back to {}", tw_config::store::version_of(&text)); - eprintln!("(a running core picks this up within a second)"); - Ok(()) - } ConfigCmd::Set { path: pointer, value, @@ -293,10 +322,11 @@ fn cmd_config(path: &Path, what: ConfigCmd) -> Result<()> { let next = tw_yaml::set(&cur.text, &steps, &scalar)?; // **先校验再写。**写完才发现读不回来，那份坏配置已经在盘上了。 tw_config::try_parse(&next).map_err(|r| anyhow::anyhow!("{r}"))?; - // 改之前那一版进历史，这样这条命令也能被 rollback 撤销 - let _ = tw_config::history::snapshot(path, &cur.text, tw_config::history::Origin::Cli); + // 前后各留一个备份版本，这样这条命令也能被 `twcore backup restore` 撤销 + let _ = + tw_pack::backup::record_before(path, &cur.text, tw_config::history::Origin::Cli); tw_config::store::write_if_unchanged(path, &cur.fingerprint, &next)?; - let _ = tw_config::history::snapshot(path, &next, tw_config::history::Origin::Cli); + let _ = tw_pack::backup::record(path, &next, tw_config::history::Origin::Cli); println!( "changed; the new version is {}", tw_config::store::version_of(&next) @@ -884,6 +914,9 @@ fn cmd_serve(path: &Path, port: Option, safe: bool, parent: Option) -> Some(r) => manager.standing_in(r), None => manager, }); + // 磁盘上此刻的样子和最新的备份版本不一样就留一版：第一次运行、升级之后、core 没在跑时 + // 命令行改过。配置读不进来时也留：一键修复之前的那一份在备份里 + manager.record_startup(); // 默认插件（随 core 发的那几个）：没给过的装上（停用着），没动过的换成新版。 // 启动时在控制面起来之前走一遍，界面第一次取插件就看得到它们；之后每换入一份 // 配置再走一遍。**不挡启动**：哪个没办成只记一行、说一声。安全模式不走 —— diff --git a/bin/twcore/src/pack_cli.rs b/bin/twcore/src/pack_cli.rs new file mode 100644 index 00000000..dcc4c79e --- /dev/null +++ b/bin/twcore/src/pack_cli.rs @@ -0,0 +1,403 @@ +//! `twcore export` / `import` / `backup`：配置包和本机备份的命令行。 +//! +//! **不连控制面，直接操作文件**，和 `twcore config` 一样：没有界面的服务器（远程 core）、 +//! 配置坏到 core 起不来时都用得上。core 正在跑时，它的文件监听在一秒之内换入写下去的配置， +//! 插件目录的监听重读插件。 +//! +//! 规矩和界面上的一样（同一份 `tw-pack`）：导入逐项加、冲突逐项选，恢复留着本机设置、 +//! ChatGPT 的登录和接管的客户端在用的密钥。 + +use std::io::{BufRead, Write}; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, anyhow, bail}; +use tw_config::history::Origin; +use tw_pack::model::{ItemId, ItemKind}; +use tw_pack::{Pack, PackOrigin}; + +/// 冲突怎么处理（`--on-conflict`） +#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)] +pub enum OnConflict { + /// Replace the item here with the one in the pack + Overwrite, + /// Import it under a new name, such as `relay (2)` + Rename, + /// Keep the item here + Skip, +} + +/// 口令：环境变量 `TWCORE_PASSPHRASE`，没有就在终端上问（不回显） +fn passphrase(confirm: bool) -> Result { + if let Ok(p) = std::env::var("TWCORE_PASSPHRASE") { + return Ok(p); + } + let first = ask_hidden("Passphrase: ")?; + if confirm { + if first.chars().count() < tw_pack::MIN_PASSPHRASE { + bail!( + "the passphrase needs at least {} characters", + tw_pack::MIN_PASSPHRASE + ); + } + let again = ask_hidden("Passphrase again: ")?; + if again != first { + bail!("the two passphrases differ"); + } + } + Ok(first) +} + +fn ask_hidden(prompt: &str) -> Result { + eprint!("{prompt}"); + std::io::stderr().flush().ok(); + let _quiet = Quiet::on(); + let mut line = String::new(); + std::io::stdin().lock().read_line(&mut line)?; + eprintln!(); + Ok(line.trim_end_matches(['\r', '\n']).to_string()) +} + +/// 读口令时关掉终端的回显，走的时候恢复。不是终端（管道）就什么都不做。 +/// **只在 unix 上关**：Windows 的控制台照常回显，要藏起来就用环境变量 +struct Quiet { + #[cfg(unix)] + saved: Option, +} + +impl Quiet { + fn on() -> Self { + #[cfg(unix)] + { + // SAFETY: 只对标准输入调 tcgetattr / tcsetattr，传进去的是栈上的结构体 + unsafe { + let mut t: libc::termios = std::mem::zeroed(); + if libc::isatty(0) == 1 && libc::tcgetattr(0, &mut t) == 0 { + let saved = t; + t.c_lflag &= !libc::ECHO; + libc::tcsetattr(0, libc::TCSANOW, &t); + return Quiet { saved: Some(saved) }; + } + } + Quiet { saved: None } + } + #[cfg(not(unix))] + { + Quiet {} + } + } +} + +impl Drop for Quiet { + fn drop(&mut self) { + #[cfg(unix)] + if let Some(t) = self.saved { + // SAFETY: 同上 + unsafe { + libc::tcsetattr(0, libc::TCSANOW, &t); + } + } + } +} + +fn ask(prompt: &str) -> Result { + eprint!("{prompt}"); + std::io::stderr().flush().ok(); + let mut line = String::new(); + std::io::stdin().lock().read_line(&mut line)?; + Ok(line.trim().to_lowercase()) +} + +fn current(path: &Path, origin: PackOrigin) -> Result<(tw_config::Loaded, Pack)> { + let cur = tw_config::store::read(path)?; + let pack = tw_pack::disk::current(path, &cur.text, origin); + Ok((cur, pack)) +} + +fn model(e: tw_pack::model::ModelError) -> anyhow::Error { + anyhow!("{e}") +} + +/// 写下去：插件文件、通用偏好、配置，配置没写成就把前两样还原。前后各留一个备份版本 +fn write_back( + path: &Path, + cur: &tw_config::Loaded, + before: &Pack, + after: &Pack, + origin: Origin, +) -> Result<()> { + let _ = tw_pack::backup::record_before(path, &cur.text, origin); + let undo = tw_pack::disk::write_plugins(path, &before.plugins, &after.plugins) + .context("writing the plugin files")?; + let prefs_changed = after.client_prefs != before.client_prefs; + if prefs_changed && let Err(e) = tw_pack::disk::write_prefs(path, after.client_prefs.as_ref()) { + undo.restore(); + return Err(e).context("writing the client preferences"); + } + if let Err(e) = tw_config::store::write_if_unchanged(path, &cur.fingerprint, &after.config_yaml) + { + undo.restore(); + if prefs_changed { + let _ = tw_pack::disk::write_prefs(path, before.client_prefs.as_ref()); + } + return Err(e.into()); + } + let _ = tw_pack::backup::record(path, &after.config_yaml, origin); + Ok(()) +} + +fn describe(id: &ItemId) -> String { + let kind = match id.kind { + ItemKind::Provider => "upstream", + ItemKind::Proxy => "proxy", + ItemKind::Key => "gateway key", + ItemKind::Alias => "alias", + ItemKind::Group => "group", + ItemKind::Route => "route", + ItemKind::PriceSheet => "price sheet", + ItemKind::SecurityRule => "custom rule", + ItemKind::Plugin => "plugin", + ItemKind::ProviderOrder => "the order of upstreams", + ItemKind::RouteOrder => "the order of routes", + ItemKind::PluginOrder => "the order of plugins", + ItemKind::Global => "settings", + ItemKind::ClientPrefs => "the app's preferences", + }; + if id.name.is_empty() { + kind.to_string() + } else { + format!("{kind} `{}`", id.name) + } +} + +// ─────────────────────────────────────────────────────────── export + +pub fn export( + path: &Path, + providers: Vec, + no_secrets: bool, + encrypt: bool, + out: PathBuf, +) -> Result<()> { + let (_, local) = current(path, PackOrigin::Export)?; + let scope = (!providers.is_empty()).then(|| { + providers + .iter() + .map(|p| ItemId::new(ItemKind::Provider, p.trim())) + .collect::>() + }); + if let Some(ids) = &scope { + let snap = tw_pack::model::snapshot(&local).map_err(model)?; + if let Some(missing) = ids.iter().find(|id| !snap.items.contains_key(*id)) { + bail!("there is no upstream named `{}`", missing.name); + } + } + let exported = tw_pack::export::export( + &local, + tw_pack::export::Options { + scope, + secrets: !no_secrets, + client_prefs: None, + origin: PackOrigin::Cli, + }, + ) + .map_err(model)?; + let pass = encrypt.then(|| passphrase(true)).transpose()?; + let seal = match &pass { + Some(p) => tw_pack::Seal::Passphrase(p), + None => tw_pack::Seal::None, + }; + let bytes = tw_pack::write(&exported.pack, seal).map_err(|e| anyhow!("{e}"))?; + tw_pack::disk::write_private(&out, &bytes) + .with_context(|| format!("writing {}", out.display()))?; + println!( + "exported {} items to {}", + exported.included.len(), + out.display() + ); + if !exported.omitted_logins.is_empty() { + println!( + "not exported: the ChatGPT sign-in of {}; sign in again after importing", + exported.omitted_logins.join(", ") + ); + } + if !no_secrets && pass.is_none() { + eprintln!("the file holds keys in plain text; keep it safe"); + } + Ok(()) +} + +// ─────────────────────────────────────────────────────────── import + +fn read_pack(file: &Path) -> Result { + let bytes = std::fs::read(file).with_context(|| format!("reading {}", file.display()))?; + let pass = tw_pack::is_encrypted(&bytes) + .then(|| passphrase(false)) + .transpose()?; + let unseal = match &pass { + Some(p) => tw_pack::Unseal::Passphrase(p), + None => tw_pack::Unseal::None, + }; + tw_pack::read(&bytes, unseal).map_err(|e| anyhow!("{e}")) +} + +pub fn import( + path: &Path, + file: &Path, + dry_run: bool, + on_conflict: Option, +) -> Result<()> { + use tw_pack::import::{Action, Decision, Status}; + let incoming = read_pack(file)?; + let (cur, local) = current(path, PackOrigin::Import)?; + let ctx = tw_pack::import::Context { + has_aws_profile: &|p| tw_config::aws_profile::credentials(p).is_ok(), + new_key: &tw_config::generate_key, + with_prefs: true, + }; + let rows = tw_pack::import::preview(&local, &incoming, &ctx).map_err(|e| anyhow!("{e}"))?; + let mut decisions = Vec::new(); + for r in &rows { + let what = describe(&r.item); + let action = match r.status { + Status::Same => { + println!(" same {what}"); + continue; + } + Status::Error => { + let why = r.error.as_ref().map(|e| e.text.clone()).unwrap_or_default(); + println!(" error {what}: {why}"); + continue; + } + Status::New => { + println!(" new {what}"); + Action::Import + } + Status::Conflict => { + println!(" differs {what} ({})", r.differs.join(", ")); + let choice = match on_conflict { + Some(c) => c, + None if dry_run => continue, + None => loop { + let a = if r.global { + ask(" [o]verwrite or [s]kip? ")? + } else { + ask(" [o]verwrite, [r]ename or [s]kip? ")? + }; + match a.as_str() { + "o" | "overwrite" => break OnConflict::Overwrite, + "r" | "rename" if !r.global => break OnConflict::Rename, + "s" | "skip" => break OnConflict::Skip, + _ => {} + } + }, + }; + match (choice, &r.suggested_name) { + (OnConflict::Overwrite, _) => Action::Overwrite, + (OnConflict::Rename, Some(n)) => Action::Rename(n.clone()), + _ => Action::Skip, + } + } + }; + decisions.push(Decision { + item: r.item.clone(), + action, + }); + } + if dry_run { + println!("(dry run: nothing was written)"); + return Ok(()); + } + let applied = + tw_pack::import::apply(&local, &incoming, &decisions, &ctx).map_err(|e| anyhow!("{e}"))?; + write_back(path, &cur, &local, &applied.pack, Origin::Import)?; + println!( + "imported; the new version is {}", + tw_config::store::version_of(&applied.pack.config_yaml) + ); + eprintln!("(a running core picks this up within a second)"); + Ok(()) +} + +// ─────────────────────────────────────────────────────────── backup + +pub fn list(path: &Path) -> Result<()> { + let all = tw_pack::backup::list(path).map_err(|e| anyhow!("{e}"))?; + if all.is_empty() { + println!("No versions yet; the first change to the configuration makes one."); + return Ok(()); + } + let now = current(path, PackOrigin::Ui) + .ok() + .filter(|(_, p)| tw_pack::model::snapshot(p).is_ok()) + .map(|(_, p)| p.content_id()); + let mut marked = false; + for b in all.iter().rev() { + let Ok(pack) = tw_pack::backup::open(b) else { + continue; + }; + let current = !marked && Some(pack.content_id()) == now; + marked |= current; + println!( + "{} {} {} {:<22} upstreams {}, plugins {} core {}", + if current { "← current" } else { " " }, + b.id, + crate::fmt_time(b.at_ms), + b.origin.label(), + pack.manifest.counts.providers, + pack.manifest.counts.plugins, + pack.manifest.core_version, + ); + } + println!(); + println!("To restore one: twcore backup restore "); + Ok(()) +} + +pub fn restore(path: &Path, id: &str, dry_run: bool) -> Result<()> { + use tw_pack::restore::{Change, Kept}; + let b = tw_pack::backup::find(path, id).map_err(|e| anyhow!("{e}"))?; + let backup = tw_pack::backup::open(&b).map_err(|e| anyhow!("{e}"))?; + let (cur, local) = current(path, PackOrigin::Restore)?; + let pinned = tw_pack::disk::read_pinned(path); + let after = if tw_pack::model::snapshot(&local).is_err() { + println!("the configuration here does not load; the whole version comes back"); + tw_pack::restore::whole(&backup, &cur.text) + } else { + let rows = tw_pack::restore::preview(&local, &backup, &pinned, true).map_err(model)?; + if rows.is_empty() { + println!("this version is the same as the configuration now"); + return Ok(()); + } + for r in &rows { + let change = match r.change { + Change::Add => "add ", + Change::Remove => "remove", + Change::Modify => "change", + }; + let kept = match r.kept { + Some(Kept::KeyInUse) => " (kept: a client set up on this machine uses it)", + None => "", + }; + println!(" {change} {}{kept}", describe(&r.item)); + } + if dry_run { + println!("(dry run: nothing was written)"); + return Ok(()); + } + tw_pack::restore::apply(&local, &backup, None, &pinned, true) + .map_err(model)? + .pack + }; + if dry_run { + println!("(dry run: nothing was written)"); + return Ok(()); + } + tw_config::try_parse(&after.config_yaml).map_err(|r| anyhow!("{r}"))?; + write_back(path, &cur, &local, &after, Origin::Restore)?; + println!( + "restored; the new version is {}", + tw_config::store::version_of(&after.config_yaml) + ); + eprintln!("(a running core picks this up within a second)"); + Ok(()) +} diff --git a/crates/tw-api/msg-codes.txt b/crates/tw-api/msg-codes.txt index 41d57420..ee9b9edd 100644 --- a/crates/tw-api/msg-codes.txt +++ b/crates/tw-api/msg-codes.txt @@ -3,6 +3,10 @@ # A second column marks codes a UI does not translate: # passthrough the sentence is only placeholders (system or upstream text); show `text` # test only produced by tests; never reaches a UI +backup.io +backup.not_found +backup.prefs_too_large +backup.write_failed config.alias_blank_model config.alias_chained config.alias_duplicate @@ -174,7 +178,6 @@ control.name_taken_not_zai control.name_whitespace control.no_keys control.no_such_endpoint -control.no_such_version control.not_a_chatgpt_account control.patch.no_entry control.patch.not_an_entry @@ -432,6 +435,15 @@ gw.ws.proxy_unsupported gw.ws.send_failed gw.ws.upstream_broke gw.ws.upstream_closed +import.exists +import.key_value_taken +import.missing_reference +import.name_taken +import.no_aws_profile +import.not_in_pack +import.plugin_mismatch +import.plugin_missing +import.skipped_reference l1.config.bad_url l1.config.no_host l1.config.proxy_addr_form @@ -498,6 +510,9 @@ pack.item_shape pack.missing_entry pack.need_key pack.need_passphrase +pack.no_such_item +pack.not_base64 +pack.passphrase_short pack.schema_too_new pack.too_large pack.too_new diff --git a/crates/tw-api/src/ep.rs b/crates/tw-api/src/ep.rs index bb99fd01..49d9c18a 100644 --- a/crates/tw-api/src/ep.rs +++ b/crates/tw-api/src/ep.rs @@ -28,13 +28,37 @@ endpoints! { /// 诊断包（Markdown，已脱敏） Diagnostics: GET "/diagnostics", () => String, text; - // ─────────────────────────────────────────────── 配置原文与历史 + // ─────────────────────────────────────────────── 配置原文 GetConfig: GET "/config", () => api::ConfigText; PatchConfig: PATCH "/config", api::ConfigPatch => api::ConfigWritten; PutConfig: PUT "/config", api::ConfigWrite => api::ConfigWritten; - ConfigHistory: GET "/config/history", () => Vec; ConfigAt: GET "/config/at", api::ConfigAtQuery => api::ConfigAt; - ConfigRollback: POST "/config/rollback", api::RollbackRequest => api::ConfigWritten; + // ─────────────────────────────────────────────── 配置包、导入导出、备份 + // + // 一个项目是配置里有名字的一样东西（上游、密钥、路由……），加上全局设置的每一节和 + // Lite 的通用偏好。导入、恢复都按项目列、按项目选。**备份的几个在安全模式下照样可用**： + // 配置读不进来时从备份恢复。 + /// 配置里的每个项目，和它引用的项目 + ConfigItems: GET "/config/items", () => Vec; + /// 导出成配置包：全部，或者选的几项连同它们引用的。**不写任何东西** + ExportConfig: POST "/config/export", api::ExportRequest => api::ExportedFile; + /// 导入会怎么做：逐项新增、相同、冲突、错误。**不写任何东西** + ImportPreview: POST "/config/import/preview", api::ImportSource => api::ImportPreview; + /// 照逐项的处理导入。预览之后配置变了是 409，重新预览 + ImportConfig: POST "/config/import", api::ImportRequest => api::ConfigWritten; + /// 备份版本，新的在前 + Backups: GET "/backups", () => Vec; + /// 一个版本另存为配置包 + BackupFile: POST "/backups/file", api::BackupFileRequest => api::ExportedFile; + /// 恢复到一个版本会改什么。**不写任何东西** + RestorePreview: POST "/backups/preview", api::RestorePreviewRequest => api::RestorePreview; + /// 恢复到一个版本（全部，或者选的几项）。恢复本身也留一个版本 + Restore: POST "/backups/restore", api::RestoreRequest => api::ConfigWritten; + /// Lite 的通用偏好 + GetClientPrefs: GET "/client-prefs", () => api::ClientPrefs; + PutClientPrefs: PUT "/client-prefs", api::ClientPrefs => (); + /// 本机接管的客户端在用的网关密钥 + PinKeys: PUT "/pinned-keys", api::PinnedKeys => (); /// 配置读不进来时，一键修复会改哪几处 ConfigRepairPlan: GET "/config/repair", () => api::ConfigRepair; /// 照那几处修好写回：取值改回默认值、删掉不认识的字段 diff --git a/crates/tw-api/src/lib.rs b/crates/tw-api/src/lib.rs index e80b0174..b1924591 100644 --- a/crates/tw-api/src/lib.rs +++ b/crates/tw-api/src/lib.rs @@ -91,6 +91,10 @@ macro_rules! slug_enum { }; } +// 配置包那一半：导出、导入、备份与恢复（`slug_enum!` 在上面定义，所以在这里声明） +mod pack; +pub use pack::*; + // ---------------------------------------------------------------- 取值固定的词 // // 下面这些是散在各处的字段共用的词表，按主题排。只在一个地方用到的集合写在 @@ -368,12 +372,18 @@ slug_enum! { Cli = "cli", /// 别的程序或者编辑器 External = "external", - /// 回滚 - Rollback = "rollback", - /// OAuth 凭据轮换之后写回 + /// OAuth 凭据轮换之后写回。**不留备份版本** Rotation = "rotation", /// core 自己：装上它自带的默认插件，或者把没动过的默认插件换成新版 Defaults = "defaults", + /// 应用了另一台设备经同步位置传来的版本 + Sync = "sync", + /// 导入了一个配置包 + Import = "import", + /// 恢复了一个备份版本 + Restore = "restore", + /// 一键修复 + Repair = "repair", } } @@ -946,7 +956,25 @@ pub const MSG_CODES: &str = include_str!("../msg-codes.txt"); /// 没有在跑的请求是 null，有请求却都还没开始生成是 0。[`RunningView`] 多了 `tokens_per_sec` /// （这一个请求此刻的速度）和 `output_tokens`（到目前为止估的输出 token），第一个 token 之前、 /// 非流式的是 null。 -pub const CONTROL_API_VERSION: u32 = 49; +/// +/// **50 起有配置包（`.twconfig`）和本机备份**，修改记录换成了备份：`GET /config/history`、 +/// `POST /config/rollback`（`ConfigVersion`、`RollbackRequest`）删了，换成 `GET /backups` +/// （[`BackupEntry`]：本机最近 10 个版本，开着同步时连同同步位置上的，同一个版本一行）、 +/// `POST /backups/file`（另存为配置包）、`POST /backups/preview`（[`RestorePreview`]：按项目 +/// 列出差异）、`POST /backups/restore`（可以只恢复几项，本机设置和 ChatGPT 的登录留着，接管的 +/// 客户端在用的密钥不删）。一版是一个完整的配置包，连同插件和 Lite 的通用偏好。新端点 +/// `GET /config/items`（[`ItemNode`]）、`POST /config/export`（[`ExportRequest`] → +/// [`ExportedFile`]）、`POST /config/import/preview`（[`ImportSource`] → [`ImportPreview`]：逐项 +/// 新增、相同、冲突、错误）和 `POST /config/import`（[`ImportRequest`]：逐项导入、覆盖、改名 +/// 导入或跳过）。Lite 的通用偏好由 core 存：`GET` / `PUT /client-prefs`（[`ClientPrefs`]， +/// 不解析，最大 64 KiB），变了发 [`Event::ClientPrefsChanged`]。`PUT /pinned-keys` +/// （[`PinnedKeys`]）报告接管的客户端在用哪几把密钥。导入覆盖、恢复改了或删了网关密钥时发 +/// [`Event::GatewayKeysChanged`]。[`ConfigOrigin`] 删了 `rollback`，多了 `sync`、`import`、 +/// `restore`、`repair`（一键修复以前记成 `ui`）。备份相关的端点在安全模式下照样可用。配置包的 +/// 码是 `pack.*`，导入的是 `import.*`，备份与恢复的是 `backup.*`。停用着的上游可以缺凭据 +/// （密钥、OAuth、AWS 访问密钥空着，ChatGPT 账号没登录）：导入时就是这样进来的。照 49 写的 +/// 界面调不到删掉的那两个端点。 +pub const CONTROL_API_VERSION: u32 = 50; /// `/live` 的速度看最近多久（见 [`LiveView::tokens_per_sec`]），毫秒 pub const LIVE_RATE_WINDOW_MS: u64 = 3_000; @@ -1601,7 +1629,7 @@ pub enum Event { id: u64, /// 内容版本号，和 `PATCH /config` 的 `base_version` 是同一个 version: String, - /// `ui` / `cli` / `external` / `rollback` / `rotation` / `defaults` + /// 谁写的，见 [`ConfigOrigin`] origin: ConfigOrigin, at_ms: u64, }, @@ -1619,7 +1647,7 @@ pub enum Event { line: Option, /// 出错那一行的原文，**已脱敏** excerpt: Option, - /// 这一版是谁写的：`ui` / `cli` / `external` / `rollback` / `rotation` / `defaults`。 + /// 这一版是谁写的（[`ConfigOrigin`]）。 /// /// **界面靠它区分「用户在编辑器里写错了」和「界面自己刚写坏了」** —— /// 前者要提醒，后者是保存失败，那条路自己会报。 @@ -1679,6 +1707,17 @@ pub enum Event { /// 「进行中」的计数和每一行的状态都靠事件，丢一个结局，那一行就永远停在 /// 「进行中」。 EventsDropped { id: u64, count: u64, at_ms: u64 }, + /// 这几把网关密钥的值变了或者没了：导入覆盖、恢复、同步改的。 + /// + /// **本机接管的客户端配置里写着密钥的值**：界面收到它，照「更换密钥后更新客户端」那一套 + /// 改写用着这几把的客户端。名字是变之前的那些 + GatewayKeysChanged { + id: u64, + names: Vec, + at_ms: u64, + }, + /// Lite 的通用偏好变了（导入、恢复、同步、别的界面存的）：重读 `GET /client-prefs` 并应用 + ClientPrefsChanged { id: u64, at_ms: u64 }, } slug_enum! { @@ -1940,6 +1979,8 @@ impl Event { | Event::RequestCancelled { id, .. } | Event::LocallyAnswered { id, .. } | Event::EventsDropped { id, .. } + | Event::GatewayKeysChanged { id, .. } + | Event::ClientPrefsChanged { id, .. } | Event::ConfigReloaded { id, .. } | Event::ListenChanged { id, .. } | Event::ConfigRejected { id, .. } @@ -4003,29 +4044,6 @@ pub struct BaseVersion { pub base_version: Option, } -/// 历史里的一版。 -#[derive(Debug, Clone, Serialize, Deserialize)] -#[cfg_attr(feature = "ts", derive(ts_rs::TS))] -pub struct ConfigVersion { - pub version: String, - pub at_ms: u64, - /// `ui` / `cli` / `external` / `rollback` / `rotation` / `defaults` - pub origin: ConfigOrigin, - pub bytes: u64, - /// 这一版是现在跑着的那一版吗。 - /// - /// **历史里包括当前版本**，所以列表最上面那条通常就是它 —— 不标 - /// 出来的话，用户会以为第一条是「上一版」然后回滚到自己身上。 - #[serde(default)] - pub current: bool, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[cfg_attr(feature = "ts", derive(ts_rs::TS))] -pub struct RollbackRequest { - pub version: String, -} - /// 一段时间的汇总。 /// /// **实测和估算分开，没有价格的单独数。**「今日 $12.40 实测 + ~$0.80 diff --git a/crates/tw-api/src/pack.rs b/crates/tw-api/src/pack.rs new file mode 100644 index 00000000..b5b2b242 --- /dev/null +++ b/crates/tw-api/src/pack.rs @@ -0,0 +1,382 @@ +//! 配置包那一半的契约：导出、导入、本机备份与恢复、Lite 的通用偏好、固定密钥。 +//! +//! 配置包（`.twconfig`）的格式和「项目」的划分在 core 的 `tw-pack` 里；这里只有线上的样子。 +//! 一个项目是配置里有名字的一样东西（[`ItemRef`]），导入和恢复都按项目列、按项目选。 + +use serde::{Deserialize, Serialize}; + +use crate::{ConfigOrigin, Msg}; + +slug_enum! { + /// 配置里的一样东西是哪一种。 + pub enum ItemKind { + /// 上游 + Provider = "provider", + /// 出站代理 + Proxy = "proxy", + /// 网关密钥 + Key = "key", + /// 模型别名 + Alias = "alias", + /// 策略组 + Group = "group", + Route = "route", + /// 价目表 + PriceSheet = "price_sheet", + /// 自定义安全规则。名字是 `<分项>/<规则名>`（`content/项目代号`） + SecurityRule = "security_rule", + Plugin = "plugin", + /// 上游的先后（没写路由时就是故障转移的顺序）。名字是空的 + ProviderOrder = "provider_order", + /// 路由的先后。名字是空的 + RouteOrder = "route_order", + /// 插件的先后（运行的顺序）。名字是空的 + PluginOrder = "plugin_order", + /// 全局设置的一节：`failover` / `security` / `default_route` / `default_key` / + /// `client_probes` / `pricing` + Global = "global", + /// Lite 的通用偏好。名字是空的 + ClientPrefs = "client_prefs", + } +} + +/// 一个项目：种类加名字。 +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct ItemRef { + pub kind: ItemKind, + pub name: String, +} + +/// `GET /config/items`：配置里的每个项目，和它引用的项目（导出所选项目时随附的那些）。 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct ItemNode { + pub item: ItemRef, + /// 直接引用的项目：路由 → 上游、策略组；上游 → 出站代理、价目表…… + pub refs: Vec, +} + +/// 导出哪些。线上是 `"all"`，或者 `{ "items": [ItemRef, …] }`。 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +#[serde(rename_all = "snake_case")] +pub enum ExportScope { + /// 全部配置 + All, + /// 这几项，**连同它们引用的项目**（core 补齐） + Items(Vec), +} + +/// `POST /config/export`。 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct ExportRequest { + pub scope: ExportScope, + /// 带上密钥：上游密钥、请求头的值、AWS 凭据、代理密码、网关密钥。不带时这些值是空的， + /// 写成 `${变量名}` 的照样带着（那不是密钥） + pub secrets: bool, + /// 用口令加密。至少 8 个字符；不给是不加密 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub passphrase: Option, + /// Lite 的通用偏好，**经远程连接导出时由 Lite 附上**（那时偏好在 Lite 那边）。经本机连接 + /// 导出时不看它：用 core 存着的那一份 + #[serde(default, skip_serializing_if = "Option::is_none")] + #[cfg_attr(feature = "ts", ts(type = "unknown"))] + pub client_prefs: Option, +} + +/// 导出（或另存）的文件。 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct ExportedFile { + /// 建议的文件名，`ThinkWatch-2026-10-18.twconfig` + pub file_name: String, + /// 文件内容，base64 + pub data_b64: String, + /// 包里有哪些项目（随附的也在里面） + pub included: Vec, + /// 登录没有随文件导出的上游（ChatGPT 账号）：导入之后要重新登录 + pub omitted_logins: Vec, +} + +/// 一个配置包的内容，导入之前看。 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct ImportSource { + pub data_b64: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub passphrase: Option, +} + +slug_enum! { + /// 完整的配置，还是选出来的几项。 + pub enum PackKind { + Full = "full", + /// 选择项目导出、分享 + Partial = "partial", + } +} + +slug_enum! { + /// 配置包是怎么来的。 + pub enum PackOrigin { + Export = "export", + Share = "share", + Ui = "ui", + Cli = "cli", + External = "external", + Defaults = "defaults", + Sync = "sync", + Import = "import", + Restore = "restore", + Repair = "repair", + /// 更新的版本才有的来源 + Other = "other", + } +} + +/// 配置包里有多少东西。 +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct PackCounts { + pub providers: u32, + pub proxies: u32, + pub clients: u32, + pub aliases: u32, + pub groups: u32, + pub routes: u32, + pub price_sheets: u32, + pub security_rules: u32, + pub plugins: u32, +} + +/// 配置包的说明（`manifest.json`）。 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct PackInfo { + pub kind: PackKind, + pub created_at_ms: u64, + /// 哪个版本的 Core 写的 + pub core_version: String, + /// 带没带密钥 + pub secrets: bool, + pub encrypted: bool, + pub origin: PackOrigin, + pub counts: PackCounts, +} + +/// `POST /config/import/preview`：导入会怎么做。**什么都不写**。 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct ImportPreview { + pub manifest: PackInfo, + /// 按磁盘上哪一版算的。导入时带回来：配置在这中间变了就是 409，重新预览 + pub base_version: String, + /// 包里的每个项目，按种类、名字排。顺序项目不在里面：新加的接在后面 + pub rows: Vec, + /// 包里的 Lite 通用偏好。**经远程连接导入时由 Lite 照它在本机应用**，core 不写；经本机 + /// 连接导入时它是 `client_prefs` 那一行，core 写下并发 [`crate::Event::ClientPrefsChanged`] + #[serde(default, skip_serializing_if = "Option::is_none")] + #[cfg_attr(feature = "ts", ts(type = "unknown"))] + pub client_prefs: Option, +} + +slug_enum! { + /// 包里的一项和本机比。 + pub enum ImportStatus { + /// 本机没有：直接导入，可以不选 + New = "new", + /// 本机有，内容一样：不处理 + Same = "same", + /// 本机有，内容不同：必须选覆盖、改名导入或跳过（全局设置只有覆盖、跳过） + Conflict = "conflict", + /// 导入不了，原因在 `error` + Error = "error", + } +} + +slug_enum! { + /// 导入时要知道的一件事。 + pub enum ImportFlag { + /// 包里不带密钥：上游导入为停用，填上密钥之后启用；网关密钥重新生成 + MissingKey = "missing_key", + /// ChatGPT 账号上游：登录不随文件带，导入为停用，登录之后启用 + NeedsLogin = "needs_login", + /// 插件：导入为停用。改得了工具调用的插件启用时要批准 + PluginDisabled = "plugin_disabled", + } +} + +/// 包里的一项。 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct ImportRow { + pub item: ItemRef, + pub status: ImportStatus, + /// 全局设置、通用偏好：冲突时只能覆盖或跳过 + pub global: bool, + pub flags: Vec, + /// 冲突时哪几个合并单位不同：上游是 `conn`（地址、协议、密钥、请求头、代理）、`models`、 + /// `billing`、`other`（并发、身份、余额、停用）；插件是 `script`、`settings`、`enabled`； + /// 全局设置和通用偏好是字段的路径（`failover.pause_secs`）；别的是 `all` + pub differs: Vec, + /// 改名导入时的缺省名字（`公司中转 (2)`）。冲突的、有名字的项目才有 + pub suggested_name: Option, + pub error: Option, + /// 它引用的、包里也有的项目。**被跳过的项目被谁引用着，谁就导入不了**（除非引用的那一项 + /// 是改名导入）：界面按它把那几行标成错误 + pub refs: Vec, +} + +slug_enum! { + /// 对一项怎么处理。 + pub enum ImportAction { + /// 本机没有的：导入 + Import = "import", + /// 用包里的替换本机同名的那一项。本机引用它的地方不变 + Overwrite = "overwrite", + /// 用新名字导入，包里引用它的项目跟着改用新名字 + Rename = "rename", + Skip = "skip", + } +} + +/// 对一项的处理。 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct ImportDecision { + pub item: ItemRef, + pub action: ImportAction, + /// 改名导入的新名字。自定义安全规则只写规则名 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub new_name: Option, +} + +/// `POST /config/import`。没有写进 `decisions` 的项目不导入。 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct ImportRequest { + pub data_b64: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub passphrase: Option, + /// 预览时的那一版 + pub base_version: String, + pub decisions: Vec, +} + +/// 一个版本里有多少东西。 +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct BackupCounts { + pub providers: u32, + pub plugins: u32, +} + +/// 一个备份版本（`GET /backups`），新的在前。本机的和同步位置上的同一个版本是一行。 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct BackupEntry { + /// 恢复、另存时带上 + pub id: String, + pub at_ms: u64, + pub origin: ConfigOrigin, + /// 同步来的版本：哪台设备上传的 + pub device: Option, + /// 存在本机 + pub local: bool, + /// 存在同步位置 + pub sync: bool, + pub counts: BackupCounts, + pub core_version: String, + /// 和现在的配置是同一份内容 + pub current: bool, +} + +/// `POST /backups/file`：一个版本另存为配置包。 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct BackupFileRequest { + pub id: String, + /// 用口令加密。至少 8 个字符；不给是不加密 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub passphrase: Option, +} + +/// `POST /backups/preview`。 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct RestorePreviewRequest { + pub id: String, +} + +/// 恢复会改什么。**什么都不写**。 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct RestorePreview { + /// 恢复时带回来 + pub base_version: String, + /// 和现在不同的项目。缺省全选；取消的保持现在的样子 + pub rows: Vec, + /// 开着同步时，恢复的结果会同步到几台别的设备。没开同步是 0 + pub sync_devices: u32, +} + +slug_enum! { + /// 恢复对一项做什么。 + pub enum RestoreChange { + /// 那一版有、现在没有 + Add = "add", + /// 现在有、那一版没有 + Remove = "remove", + Modify = "modify", + } +} + +slug_enum! { + /// 恢复时为什么留着它。 + pub enum KeptReason { + /// 本机接管的客户端正在用这把网关密钥：不删 + KeyInUse = "key_in_use", + } +} + +/// 恢复里的一项。 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct RestoreRow { + pub item: ItemRef, + pub change: RestoreChange, + /// 不照那一版改的原因。有的话这一项保持现在的样子 + pub kept: Option, +} + +/// `POST /backups/restore`。 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct RestoreRequest { + pub id: String, + pub base_version: String, + /// 只恢复这几项。不给是全部 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub items: Option>, +} + +/// Lite 的通用偏好（语言、外观、菜单栏、通知、检查更新），**core 不解析**，最大 64 KiB。 +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct ClientPrefs { + /// 还没存过是空 + #[cfg_attr(feature = "ts", ts(type = "unknown"))] + pub prefs: Option, +} + +/// Lite 的通用偏好最大多少字节（写成 JSON 之后） +pub const CLIENT_PREFS_MAX: usize = 64 * 1024; + +/// `PUT /pinned-keys`：本机接管的客户端正在用的网关密钥。恢复、导入、同步不删它们。 +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct PinnedKeys { + pub names: Vec, +} diff --git a/crates/tw-api/src/ts.rs b/crates/tw-api/src/ts.rs index 258d6142..308cbf1c 100644 --- a/crates/tw-api/src/ts.rs +++ b/crates/tw-api/src/ts.rs @@ -446,7 +446,7 @@ mod tests { // core 自己写配置（默认插件）的那一版有自己的来源 assert_eq!( decl_of(&ts, "ConfigOrigin"), - "export type ConfigOrigin = \"ui\" | \"cli\" | \"external\" | \"rollback\" | \"rotation\" | \"defaults\"" + "export type ConfigOrigin = \"ui\" | \"cli\" | \"external\" | \"rotation\" | \"defaults\" | \"sync\" | \"import\" | \"restore\" | \"repair\"" ); } diff --git a/crates/tw-config/src/control_key.rs b/crates/tw-config/src/control_key.rs index 28ddcf8c..9bfbd1ad 100644 --- a/crates/tw-config/src/control_key.rs +++ b/crates/tw-config/src/control_key.rs @@ -126,9 +126,8 @@ pub fn rotate_file(path: &Path) -> Result { let cur = store::read(path)?; let (next, key) = rotate(&cur.text)?; crate::try_parse(&next).map_err(|r| EnsureError::Rejected(Box::new(r)))?; - let _ = crate::history::snapshot(path, &cur.text, crate::history::Origin::Cli); + // **不留备份版本**：备份里的钥匙是打码的，换钥匙前后的两版存下来一模一样 store::write_if_unchanged(path, &cur.fingerprint, &next)?; - let _ = crate::history::snapshot(path, &next, crate::history::Origin::Cli); Ok(key) } diff --git a/crates/tw-config/src/credential.rs b/crates/tw-config/src/credential.rs index 37051a85..35454503 100644 --- a/crates/tw-config/src/credential.rs +++ b/crates/tw-config/src/credential.rs @@ -541,9 +541,16 @@ impl Provider { } /// 凭据写法对不对。**不联网，不读环境变量** —— 那是转发时的事。 + /// + /// **停用着的上游可以缺凭据**：密钥、OAuth、AWS 访问密钥空着，ChatGPT 账号还没登录。 + /// 导入一个不带密钥的配置包、别的设备传来一家 ChatGPT 账号上游时就是这样 —— 上游先 + /// 停用着进来，填上密钥（登录）之后才启用得了。写法上的毛病（请求头写错、两种凭据 + /// 都写了）照样不收 pub fn check_credential(&self) -> Result<(), CredentialError> { + let waiting = self.disabled; if let Some(k) = &self.key && k.is_blank() + && !waiting { return Err(CredentialError::EmptyKey); } @@ -552,7 +559,7 @@ impl Provider { if self.key.is_some() { return Err(CredentialError::KeyAndOauth); } - if o.refresh.trim().is_empty() || o.endpoint.trim().is_empty() { + if (o.refresh.trim().is_empty() && !waiting) || o.endpoint.trim().is_empty() { return Err(CredentialError::EmptyOauth); } // **订阅账号的令牌不接。**Anthropic 规定 Claude 订阅的 OAuth 令牌不得由 @@ -574,7 +581,8 @@ impl Provider { } // **Codex 后端只认登录拿到的凭据**，API 密钥发过去只会是一个 401 let chatgpt = self.effective_protocol() == Some(Protocol::Chatgpt); - if chatgpt && crate::chatgpt::is_backend(&self.base_url) && self.oauth.is_none() { + if chatgpt && crate::chatgpt::is_backend(&self.base_url) && self.oauth.is_none() && !waiting + { return Err(CredentialError::ChatgptWithoutLogin); } // 接 ChatGPT 账号的前提是如实说明请求来自 ThinkWatch：转发客户端的身份就是冒充它 @@ -657,9 +665,10 @@ impl Provider { Some(_) => {} None => { let blank = |s: &Option| s.as_ref().is_none_or(Secret::is_blank); - if blank(&aws.access_key_id) - || blank(&aws.secret_access_key) - || aws.session_token.as_ref().is_some_and(Secret::is_blank) + if !self.disabled + && (blank(&aws.access_key_id) + || blank(&aws.secret_access_key) + || aws.session_token.as_ref().is_some_and(Secret::is_blank)) { return Err(CredentialError::AwsEmpty); } @@ -964,6 +973,27 @@ mod tests { lookalike.check_credential().unwrap(); } + /// 停用着的上游可以缺凭据（导入时没带密钥、别的设备上登录的 ChatGPT 账号）；启用就不行。 + /// 写法上的毛病照样不收 + #[test] + fn a_disabled_upstream_may_wait_for_its_credential() { + for yaml in [ + "name: k\nbase_url: https://relay.example\nkey: ''\n", + "name: c\nbase_url: https://chatgpt.com/backend-api/codex\n", + "name: o\nbase_url: https://relay.example\noauth: { refresh: '', endpoint: https://t.example }\n", + "name: b\nbase_url: https://bedrock-runtime.us-east-1.amazonaws.com\naws: { access_key_id: '', secret_access_key: '' }\n", + ] { + assert!(p(yaml).check_credential().is_err(), "{yaml}"); + let off = p(&format!("{yaml}disabled: true\n")); + off.check_credential() + .unwrap_or_else(|e| panic!("{yaml}: {e}")); + } + let both = p( + "name: x\nbase_url: https://relay.example\nkey: ''\noauth: { refresh: r, endpoint: https://t.example }\ndisabled: true\n", + ); + assert_eq!(both.check_credential(), Err(CredentialError::KeyAndOauth)); + } + #[test] fn a_chatgpt_account_takes_only_a_login_and_its_identity_is_not_configurable() { let bare = p("name: c\nbase_url: https://chatgpt.com/backend-api/codex\n"); diff --git a/crates/tw-config/src/history.rs b/crates/tw-config/src/history.rs index 48015ca5..11a845ac 100644 --- a/crates/tw-config/src/history.rs +++ b/crates/tw-config/src/history.rs @@ -1,28 +1,10 @@ -//! 变更历史与回滚（M2 验收「改坏能一键回滚」）。 +//! 一次配置改动的来源。 //! -//! **存的是每一版，包括当前这一版。**写之前存一次「改之前」、生效之后 -//! 存一次「改之后」，去重之后的效果就是「每个存在过的版本各一条」，而 -//! 最新那条就是现在跑着的。 -//! -//! 为什么不只存「改之前」：那样最新的一版永远不在历史里，于是「改坏了 -//! 想回到上一版」和「回到上上版」在列表上长得一模一样，用户要自己数。 -//! 而且外部改动那条路径根本没有「写之前」可言。 -//! -//! 存的是整份文本，不是 diff。理由很实际：一份配置几 KB，五十个版本也 -//! 就几百 KB，而**用 diff 存会让回滚依赖一条完整的链** —— 中间任何一环 -//! 坏掉，后面全部作废。回滚是出事时用的功能，它自己不能有脆弱的前提。 - -use std::path::{Path, PathBuf}; +//! 每一版配置存在哪儿、留几版、怎么恢复，在 `tw-pack` 的本机备份里（`tw_pack::backup`）： +//! 一版是一个完整的配置包，连同插件和 Lite 的通用偏好。这里只剩「谁改的」这个词 —— +//! 写配置的每一条路都要说它，而那几条路不都够得着 `tw-pack`。 -use crate::store::{self, StoreError}; - -/// 留多少版。 -/// -/// 五十版在一份几 KB 的配置上是几百 KB，而**回滚要找的那一版几乎总在 -/// 最近几次里** —— 更早的版本留着只是安慰。 -pub const KEEP: usize = 50; - -/// 一次改动的来源。**记下来是为了让历史列表能读懂** —— +/// 一次改动的来源。**记下来是为了让备份列表能读懂** —— /// 「界面改的」和「你自己在编辑器里改的」在排查时是完全不同的线索。 #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Origin { @@ -30,462 +12,90 @@ pub enum Origin { Cli, /// 外部编辑（编辑器、脚本），我们是从文件监听发现的 External, - Rollback, - /// token 端点换发了新的 refresh token，我们把它写回去了。 + /// core 自己装上它自带的默认插件、或者把没动过的默认插件换成新版。不是人发起的， + /// 要一眼认得出来 + Defaults, + /// 应用了另一台设备经同步位置传来的版本 + Sync, + /// 导入了一个配置包 + Import, + /// 恢复了一个备份版本 + Restore, + /// 一键修复 + Repair, + /// token 端点换发了新的凭据，我们把它写回去了。 /// - /// **不是人发起的写入**，所以它在历史里要能一眼认出来 —— 用户看到「配置 - /// 变了」时，第一个问题是「谁改的」。 + /// **不是人发起的写入，也不留备份版本**：恢复到一次轮换之前，拿到的是一个已经作废的 + /// token；会轮换的服务器一小时一次，几天就把用户自己改过的那些版本挤出去了 Rotation, - /// core 自己装上它自带的默认插件、或者把没动过的默认插件换成新版。同样不是 - /// 人发起的，同样要一眼认得出来 - Defaults, } impl Origin { - /// 控制面发给界面的值，也是历史文件名里的那一段。 + /// 控制面发给界面的值，也是备份文件名里的那一段。 pub fn slug(&self) -> &'static str { match self { Origin::Ui => "ui", Origin::Cli => "cli", Origin::External => "external", - Origin::Rollback => "rollback", - Origin::Rotation => "rotation", Origin::Defaults => "defaults", + Origin::Sync => "sync", + Origin::Import => "import", + Origin::Restore => "restore", + Origin::Repair => "repair", + Origin::Rotation => "rotation", } } - fn parse(s: &str) -> Origin { + + /// 反过来。认不出的是外部编辑：一个说不出来源的版本，最可能就是有人直接改了文件 + pub fn parse(s: &str) -> Origin { match s { "ui" => Origin::Ui, "cli" => Origin::Cli, - "rollback" => Origin::Rollback, - "rotation" => Origin::Rotation, "defaults" => Origin::Defaults, + "sync" => Origin::Sync, + "import" => Origin::Import, + "restore" => Origin::Restore, + "repair" => Origin::Repair, + "rotation" => Origin::Rotation, _ => Origin::External, } } + + /// 命令行里的说法 pub fn label(&self) -> &'static str { match self { Origin::Ui => "the interface", Origin::Cli => "the command line", Origin::External => "an outside edit", - Origin::Rollback => "a rollback", - Origin::Rotation => "a credential rotation", Origin::Defaults => "the default plugins", + Origin::Sync => "sync", + Origin::Import => "an import", + Origin::Restore => "a restore", + Origin::Repair => "a repair", + Origin::Rotation => "a credential rotation", } } } -/// 历史里的一版。 -#[derive(Debug, Clone)] -pub struct Version { - pub file: PathBuf, - /// 毫秒时间戳。**文件名里就带着它** —— 历史目录直接按名字排序就是 - /// 时间序，不必读每个文件的 mtime（备份工具会把 mtime 全改成同一天） - pub at_ms: u64, - pub origin: Origin, - /// 内容版本号，和 `store::version_of` 是同一个 - pub version: String, - pub bytes: u64, -} - -fn history_dir(config_path: &Path) -> PathBuf { - config_path - .parent() - .unwrap_or(Path::new(".")) - .join("history") -} - -fn now_ms() -> u64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0) -} - -/// 把当前这一版存进历史。 -/// -/// **内容和上一版一模一样时什么都不做。**否则一次「保存但没改动」会在 -/// 历史里堆一版噪音，而回滚列表里全是同一份内容的时候，它就没用了。 -pub fn snapshot( - config_path: &Path, - text: &str, - origin: Origin, -) -> Result, StoreError> { - // **凭据轮换不进历史**。三条理由，第一条就够了： - // - // 1. **回滚到一次轮换之前，拿到的是一个已经作废的 token。**服务器 - // 换发新的那一刻就把旧的废了 —— 这一版不是「一个可以退回去的 - // 状态」，是个陷阱。 - // 2. 会轮换的服务器一小时一次，两天就把 50 版全占满了，用户自己 - // 改过的那些全被挤出去 —— 而那才是他要回滚的东西。 - // 3. 每一版都是一份明文密钥的副本，而这几份副本永远派不上用场。 - // - // 「谁改的、改了什么」由 `ConfigReloaded` 事件和日志记着，不靠这里。 - if matches!(origin, Origin::Rotation) { - return Ok(None); - } - let dir = history_dir(config_path); - let version = store::version_of(text); - let all = list(config_path)?; - if let Some(last) = all.last() - && last.version == version - { - return Ok(None); - } - std::fs::create_dir_all(&dir).map_err(|source| StoreError::Io { - path: dir.clone(), - source, - })?; - // **时间戳必须严格递增。**同一毫秒里连着存两版（表单自动保存、 - // 脚本连改）会得到同一个 at_ms，而那时「哪一版更新」就只能听 - // `read_dir` 的顺序 —— 于是 prune 可能删掉最新的那一版，回滚也会 - // 回到一个说不清的地方。 - // - // 代价是一次连写里的时间戳可能比真实时刻晚几毫秒。**顺序永远不 - // 会错，而顺序才是回滚依赖的东西**，时间只是给人看的。 - let at_ms = match all.last() { - Some(last) if now_ms() <= last.at_ms => last.at_ms + 1, - _ => now_ms(), - }; - // 名字里带上时间、来源和版本号 —— 光看文件名就能读懂这一版是什么。 - let file = dir.join(format!( - "{at_ms}-{}-{}.yaml", - origin.slug(), - &version["blake3:".len()..] - )); - // **钥匙不进历史**：存的是打码的原文，版本号仍按真原文算（文件名里那一段）， - // 「哪一版是现在这版」照样对得上。回滚到这一版时打码换回当时磁盘上的钥匙 - // （见 [`crate::control_key::unmask`]）—— 回滚不该顺手把门锁换掉 - store::write_atomic(&file, &crate::control_key::mask(text))?; - prune(config_path)?; - Ok(Some(Version { - file, - at_ms, - origin, - version, - bytes: text.len() as u64, - })) -} - -/// 历史列表，**从旧到新**。 -pub fn list(config_path: &Path) -> Result, StoreError> { - let dir = history_dir(config_path); - let rd = match std::fs::read_dir(&dir) { - Ok(r) => r, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), - Err(source) => return Err(StoreError::Io { path: dir, source }), - }; - let mut out = Vec::new(); - for e in rd.flatten() { - let name = e.file_name().to_string_lossy().to_string(); - let Some(v) = parse_name(&name) else { continue }; - let bytes = e.metadata().map(|m| m.len()).unwrap_or(0); - out.push(Version { - file: e.path(), - bytes, - ..v - }); - } - // 按时间戳排，不按文件名的字典序 —— 时间戳位数会变（跨过 10 的 - // 幂次），那时字典序和时间序就分家了。 - out.sort_by_key(|v| v.at_ms); - Ok(out) -} - -fn parse_name(name: &str) -> Option { - let stem = name.strip_suffix(".yaml")?; - let mut it = stem.splitn(3, '-'); - let at_ms: u64 = it.next()?.parse().ok()?; - let origin = Origin::parse(it.next()?); - let hash = it.next()?; - Some(Version { - file: PathBuf::new(), - at_ms, - origin, - version: format!("blake3:{hash}"), - bytes: 0, - }) -} - -/// 超出上限的老版本删掉。 -fn prune(config_path: &Path) -> Result<(), StoreError> { - let all = list(config_path)?; - if all.len() <= KEEP { - return Ok(()); - } - for v in &all[..all.len() - KEEP] { - let _ = std::fs::remove_file(&v.file); - } - Ok(()) -} - -/// 取某一版的内容。 -pub fn read(v: &Version) -> Result { - std::fs::read_to_string(&v.file).map_err(|source| StoreError::Io { - path: v.file.clone(), - source, - }) -} - -#[derive(Debug, thiserror::Error)] -pub enum RollbackError { - #[error("the version history has no {0}")] - NoSuchVersion(String), - #[error(transparent)] - Store(#[from] StoreError), -} - -/// 回到某一版。 -/// -/// **回滚本身也是一次改动，也要进历史。**否则「回滚之后发现回错了」 -/// 就没有退路了 —— 而那是回滚最常见的用法之一。 -pub fn rollback(config_path: &Path, version: &str) -> Result { - let all = list(config_path)?; - let target = all - .iter() - .rev() - .find(|v| v.version == version || v.version.ends_with(version)) - .ok_or_else(|| RollbackError::NoSuchVersion(version.to_string()))?; - let mut text = read(target)?; - // 先把「现在这一版」存进历史，再覆盖 - if let Ok(cur) = store::read(config_path) { - snapshot(config_path, &cur.text, Origin::Rollback)?; - // 历史里的钥匙是打码的，换回现在这一把 - let real = crate::control_key::raw_in(&cur.text).ok().flatten(); - if let Ok(t) = crate::control_key::unmask(&text, real.as_deref()) { - text = t; - } - } - store::write_atomic(config_path, &text)?; - Ok(text) -} - #[cfg(test)] mod tests { use super::*; - fn setup() -> (tempfile::TempDir, PathBuf) { - let d = tempfile::tempdir().unwrap(); - let p = d.path().join("config.yaml"); - std::fs::write(&p, "version: 1\nport: 8788\n").unwrap(); - (d, p) - } - - #[test] - fn a_snapshot_can_be_read_back_byte_for_byte() { - let (_d, p) = setup(); - let text = "version: 1\n# 注释也要留着\nport: 8788\n"; - let v = snapshot(&p, text, Origin::Ui).unwrap().unwrap(); - assert_eq!(read(&v).unwrap(), text); - } - #[test] - fn saving_the_same_content_twice_does_not_add_a_second_version() { - // 一次「保存但没改动」在历史里堆一版噪音，而回滚列表里全是同一 - // 份内容的时候它就没用了。 - let (_d, p) = setup(); - snapshot(&p, "a: 1\n", Origin::Ui).unwrap().unwrap(); - assert!(snapshot(&p, "a: 1\n", Origin::Ui).unwrap().is_none()); - assert_eq!(list(&p).unwrap().len(), 1); - } - - #[test] - fn the_origin_of_each_change_survives_a_restart() { - // 「界面改的」和「你自己在编辑器里改的」在排查时是完全不同的线索， - // 而它必须能从磁盘上读回来 —— 内存里记着的东西活不过一次重启。 - let (_d, p) = setup(); - snapshot(&p, "a: 1\n", Origin::Ui).unwrap(); - snapshot(&p, "a: 2\n", Origin::External).unwrap(); - snapshot(&p, "a: 3\n", Origin::Cli).unwrap(); - let got: Vec = list(&p).unwrap().iter().map(|v| v.origin).collect(); - assert_eq!(got, vec![Origin::Ui, Origin::External, Origin::Cli]); - } - - #[test] - fn history_is_ordered_oldest_first_even_across_a_digit_boundary() { - // 按文件名的字典序排会在时间戳位数变化时和时间序分家。 - let (_d, p) = setup(); - let dir = history_dir(&p); - std::fs::create_dir_all(&dir).unwrap(); - for (ms, body) in [(999u64, "a: 1\n"), (1000, "a: 2\n"), (10000, "a: 3\n")] { - std::fs::write( - dir.join(format!("{ms}-ui-{}.yaml", &store::hash_of(body)[..12])), - body, - ) - .unwrap(); - } - let got: Vec = list(&p).unwrap().iter().map(|v| v.at_ms).collect(); - assert_eq!(got, vec![999, 1000, 10000]); - } - - #[test] - fn rolling_back_restores_the_old_text_exactly() { - let (_d, p) = setup(); - let old = "version: 1\n# 我要找回这个注释\nport: 8788\n"; - let v = snapshot(&p, old, Origin::Ui).unwrap().unwrap(); - std::fs::write(&p, "version: 1\nport: 9999\n").unwrap(); - let back = rollback(&p, &v.version).unwrap(); - assert_eq!(back, old); - assert_eq!(std::fs::read_to_string(&p).unwrap(), old); - } - - #[test] - fn rolling_back_is_itself_undoable() { - // **「回滚之后发现回错了」是回滚最常见的用法之一。** - let (_d, p) = setup(); - let v1 = snapshot(&p, "a: 1\n", Origin::Ui).unwrap().unwrap(); - std::fs::write(&p, "a: 2\n").unwrap(); - rollback(&p, &v1.version).unwrap(); - assert_eq!(std::fs::read_to_string(&p).unwrap(), "a: 1\n"); - // 回滚前那一版（a: 2）进了历史，所以能再回去 - let v2 = list(&p) - .unwrap() - .into_iter() - .find(|v| v.origin == Origin::Rollback) - .expect("回滚没留下退路"); - assert_eq!(read(&v2).unwrap(), "a: 2\n"); - rollback(&p, &v2.version).unwrap(); - assert_eq!(std::fs::read_to_string(&p).unwrap(), "a: 2\n"); - } - - #[test] - fn a_short_version_prefix_is_enough_to_name_a_version() { - // 用户会从界面上抄一小截，或者只记得前几位。 - let (_d, p) = setup(); - let v = snapshot(&p, "a: 1\n", Origin::Ui).unwrap().unwrap(); - std::fs::write(&p, "a: 2\n").unwrap(); - let short = &v.version["blake3:".len()..]; - assert_eq!(rollback(&p, short).unwrap(), "a: 1\n"); - } - - #[test] - fn asking_for_a_version_that_is_not_there_says_so() { - let (_d, p) = setup(); - let e = rollback(&p, "blake3:deadbeef").unwrap_err(); - assert!(matches!(e, RollbackError::NoSuchVersion(_)), "{e:?}"); - } - - #[test] - fn a_rotation_never_enters_the_history() { - // **回滚到一次轮换之前，拿到的是一个已经作废的 token** —— - // 那一版不是可以退回去的状态，是个陷阱。而且一小时一次的轮换 - // 两天就能把用户自己改过的那些全挤出去。 - let (_d, p) = setup(); - snapshot(&p, "version: 1\nport: 1\n", Origin::Ui).unwrap(); - for i in 0..5 { - let text = format!("version: 1\nport: 1\n# rot {i}\n"); - assert!( - snapshot(&p, &text, Origin::Rotation).unwrap().is_none(), - "轮换进历史了" - ); + fn every_origin_reads_back_from_its_word() { + for o in [ + Origin::Ui, + Origin::Cli, + Origin::External, + Origin::Defaults, + Origin::Sync, + Origin::Import, + Origin::Restore, + Origin::Repair, + Origin::Rotation, + ] { + assert_eq!(Origin::parse(o.slug()), o); } - let all = list(&p).unwrap(); - assert_eq!(all.len(), 1, "历史里不该有轮换那几版：{all:?}"); - assert_eq!(all[0].origin.label(), "the interface"); - } - - #[test] - fn old_versions_are_pruned_but_the_recent_ones_all_survive() { - let (_d, p) = setup(); - for i in 0..KEEP + 10 { - snapshot(&p, &format!("a: {i}\n"), Origin::Ui).unwrap(); - } - let all = list(&p).unwrap(); - assert_eq!(all.len(), KEEP); - // 留下的是最近的那些 - assert_eq!( - read(all.last().unwrap()).unwrap(), - format!("a: {}\n", KEEP + 9) - ); - } - - #[test] - fn history_files_are_0600_too_because_they_contain_the_same_keys() { - // 一份历史版本和当前配置一样有明文密钥。权限漏在这里等于没漏在 - // 那里 —— 而这类遗漏最容易发生在「顺手加的辅助功能」上。 - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let (_d, p) = setup(); - let v = snapshot(&p, "key: sk-secret\n", Origin::Ui) - .unwrap() - .unwrap(); - let mode = std::fs::metadata(&v.file).unwrap().permissions().mode() & 0o777; - assert_eq!(mode, 0o600, "历史文件权限是 {mode:o}"); - } - } - - #[test] - fn an_empty_history_is_not_an_error() { - // 第一次运行时历史目录还不存在。那是正常状态。 - let (_d, p) = setup(); - assert!(list(&p).unwrap().is_empty()); - } - - #[test] - fn a_stray_file_in_the_history_directory_is_ignored_not_fatal() { - // 用户往那个目录里丢过东西、或者别的工具留下过东西。 - let (_d, p) = setup(); - let dir = history_dir(&p); - std::fs::create_dir_all(&dir).unwrap(); - std::fs::write(dir.join("README.txt"), "hi").unwrap(); - std::fs::write(dir.join("不是时间戳-ui-abc.yaml"), "a: 1\n").unwrap(); - snapshot(&p, "a: 1\n", Origin::Ui).unwrap(); - assert_eq!(list(&p).unwrap().len(), 1); - } -} - -#[cfg(test)] -mod ordering_tests { - use super::*; - - #[test] - fn versions_written_in_the_same_millisecond_keep_their_order() { - // 表单自动保存、脚本连改都会撞在同一毫秒里。撞上之后「哪一版 - // 更新」如果只能听 read_dir 的顺序，prune 会删掉最新的那一版， - // 而回滚会回到一个说不清的地方。 - let d = tempfile::tempdir().unwrap(); - let p = d.path().join("config.yaml"); - let mut stamps = Vec::new(); - for i in 0..200 { - let v = snapshot(&p, &format!("a: {i}\n"), Origin::Ui) - .unwrap() - .unwrap(); - stamps.push(v.at_ms); - } - assert!( - stamps.windows(2).all(|w| w[0] < w[1]), - "时间戳没有严格递增：{stamps:?}" - ); - let listed = list(&p).unwrap(); - assert_eq!( - read(listed.last().unwrap()).unwrap(), - "a: 199\n", - "最新的那一版不是最后一个" - ); - } - - /// 历史里没有钥匙；回滚回去，钥匙是现在这一把，不是当时那一把。 - #[test] - fn history_never_holds_the_control_key_and_a_rollback_keeps_the_current_one() { - let d = tempfile::tempdir().unwrap(); - let p = d.path().join("config.yaml"); - let old_key = "a".repeat(64); - let new_key = "b".repeat(64); - let with = |k: &str, port: u16| { - format!("version: 1\nlisten:\n control:\n key: {k}\nport: {port}\n") - }; - let v1 = snapshot(&p, &with(&old_key, 1), Origin::Ui) - .unwrap() - .unwrap(); - assert_eq!(v1.version, store::version_of(&with(&old_key, 1))); - let stored = read(&v1).unwrap(); - assert!(!stored.contains(&old_key), "{stored}"); - assert!(stored.contains(tw_api::control::KEY_MASK), "{stored}"); - - std::fs::write(&p, with(&new_key, 2)).unwrap(); - let back = rollback(&p, &v1.version).unwrap(); - assert_eq!(back, with(&new_key, 1)); - assert_eq!(std::fs::read_to_string(&p).unwrap(), with(&new_key, 1)); + assert_eq!(Origin::parse("rollback"), Origin::External); } } diff --git a/crates/tw-config/src/remote.rs b/crates/tw-config/src/remote.rs index 56f5f278..bc307c6c 100644 --- a/crates/tw-config/src/remote.rs +++ b/crates/tw-config/src/remote.rs @@ -96,7 +96,10 @@ pub fn disable(text: &str) -> Result, PatchError> { } } -/// 改完先校验、再写，前后两版进历史。和 `control_key::rotate_file` 同一条路。 +/// 改完先校验、再写。和 `control_key::rotate_file` 同一条路。 +/// +/// **备份版本不在这里留**：备份在 `tw-pack`，这个 crate 够不着它。core 正在跑时文件监听 +/// 换入这一版、留下它；没在跑时下一次启动留（`twcore` 起来时先记一版磁盘上的样子） fn write_checked( path: &Path, f: impl FnOnce(&str) -> Result, PatchError>, @@ -106,9 +109,7 @@ fn write_checked( return Ok(current(&cur.text)); }; let cfg = crate::try_parse(&next).map_err(|r| EnsureError::Rejected(Box::new(r)))?; - let _ = crate::history::snapshot(path, &cur.text, crate::history::Origin::Cli); store::write_if_unchanged(path, &cur.fingerprint, &next)?; - let _ = crate::history::snapshot(path, &next, crate::history::Origin::Cli); Ok(cfg.listen.control.remote) } diff --git a/crates/tw-config/src/wire.rs b/crates/tw-config/src/wire.rs index 1ee7917b..e5e59c84 100644 --- a/crates/tw-config/src/wire.rs +++ b/crates/tw-config/src/wire.rs @@ -104,9 +104,12 @@ impl From for tw_api::ConfigOrigin { Origin::Ui => Self::Ui, Origin::Cli => Self::Cli, Origin::External => Self::External, - Origin::Rollback => Self::Rollback, - Origin::Rotation => Self::Rotation, Origin::Defaults => Self::Defaults, + Origin::Sync => Self::Sync, + Origin::Import => Self::Import, + Origin::Restore => Self::Restore, + Origin::Repair => Self::Repair, + Origin::Rotation => Self::Rotation, } } } diff --git a/crates/tw-control/Cargo.toml b/crates/tw-control/Cargo.toml index eb339a88..725d07ae 100644 --- a/crates/tw-control/Cargo.toml +++ b/crates/tw-control/Cargo.toml @@ -19,6 +19,9 @@ tw-types = { workspace = true } tw-link = { workspace = true } # 列网卡用 getifaddrs —— 比引一个 crate 便宜 libc = { workspace = true } +# 配置包（导出、导入、本机备份、恢复）。文件内容在线上是 base64 +tw-pack = { workspace = true } +base64 = { workspace = true } tw-config = { workspace = true } tw-guard = { workspace = true } bytes = { workspace = true } diff --git a/crates/tw-control/src/backups.rs b/crates/tw-control/src/backups.rs new file mode 100644 index 00000000..286a91a9 --- /dev/null +++ b/crates/tw-control/src/backups.rs @@ -0,0 +1,794 @@ +//! 配置包、导入导出、本机备份与恢复、Lite 的通用偏好、固定密钥。 +//! +//! 格式、项目的划分、导入和恢复的规矩都在 `tw-pack`；这里把它们接到控制面上：读磁盘上此刻 +//! 的样子、写插件文件和配置（同一条写入路径：核版本、先校验再写、留备份版本、换入）、发事件。 +//! +//! # 写的顺序 +//! +//! 和插件页一样**先写文件、再写配置**，攥着 `Plugins::edits`：插件文件和通用偏好先落盘，配置 +//! 一换入网关就照它重读插件；配置没写成（版本对不上、校验没过），刚写的文件按写之前的样子 +//! 还原。 +//! +//! # 远程连接 +//! +//! Lite 的通用偏好经远程连接时在 Lite 那边：导出用请求里带来的那一份，导入、恢复不写 core +//! 这边的偏好（导入预览把包里的偏好交给 Lite，由它在本机应用）。 +//! +//! # 安全模式 +//! +//! 备份的几个端点在安全模式下照样可用：配置读不进来时，恢复把那一版整份写回去，只有控制面 +//! 的钥匙换成现在这一把（[`tw_pack::restore::whole`]）。 + +use std::collections::BTreeSet; +use std::future::Future; +use std::pin::Pin; + +use axum::Json; +use axum::extract::State; +use axum::http::StatusCode; +use base64::Engine; +use tw_api::ep; +use tw_config::history::Origin; +use tw_pack::model::{self, ItemId, ItemKind}; +use tw_pack::{Pack, PackError, PackOrigin, Seal, Unseal}; +use tw_types::{Msg, msg}; + +use crate::contract::RouterExt; +use crate::{ApplyError, ControlState, Fail, apply_fail, fail, unreadable_config}; + +pub fn router() -> axum::Router { + axum::Router::new() + .at(ep::ConfigItems, items) + .at(ep::ExportConfig, export) + .at(ep::ImportPreview, import_preview) + .at(ep::ImportConfig, import) + .at(ep::Backups, backups) + .at(ep::BackupFile, backup_file) + .at(ep::RestorePreview, restore_preview) + .at(ep::Restore, restore) + .at(ep::GetClientPrefs, get_prefs) + .at(ep::PutClientPrefs, put_prefs) + .at(ep::PinKeys, pin_keys) +} + +// ─────────────────────────────────────────────────────────── 同步位置上的版本 + +/// 一个装箱的 future。 +pub type BoxFuture<'a, T> = Pin + Send + 'a>>; + +/// 同步位置上的一个版本。 +#[derive(Debug, Clone, PartialEq)] +pub struct RemoteVersionInfo { + /// [`Pack::content_id`]：和本机同一份内容的版本靠它合成一行 + pub content_id: String, + pub at_ms: u64, + /// 哪台设备上传的 + pub device_name: String, + pub core_version: String, + pub counts: tw_pack::Counts, +} + +/// 同步位置上的备份版本。**同步那一路实现并装上**（[`crate::ConfigManager::set_remote_backups`]）， +/// 没开同步时没有。 +pub trait RemoteBackups: Send + Sync { + /// 同步位置上留着的版本。连不上就是空的：列表照样列本机的 + fn list(&self) -> BoxFuture<'_, Vec>; + /// 取一个版本（已经解开） + fn fetch<'a>(&'a self, content_id: &'a str) -> BoxFuture<'a, Result>; + /// 恢复之后会同步到几台别的设备 + fn other_devices(&self) -> BoxFuture<'_, u32> { + Box::pin(async { 0 }) + } +} + +/// 同步位置上的版本在 `BackupEntry.id` 里写成这样：`sync-` +const SYNC_ID: &str = "sync-"; + +// ─────────────────────────────────────────────────────────── 插件的拆法 + +/// 插件源码的拆法：代码和数据分开（`tw_gateway::plugin::source`）。**进程里装一次**， +/// `ConfigManager::new` 和命令行都会调，装过了什么都不做 +pub fn install_codec() { + tw_pack::model::install_codec(Box::new(Codec)); +} + +struct Codec; + +impl tw_pack::model::PluginCodec for Codec { + fn split(&self, source: &str) -> Option<(String, serde_json::Value)> { + tw_gateway::plugin::source::split_data(source) + } + fn join(&self, source: &str, data: &serde_json::Value) -> Option { + tw_gateway::plugin::source::join_data(source, data) + } + fn shipped(&self, id: &str) -> Option { + tw_gateway::plugin::defaults::ALL + .iter() + .find(|(i, _)| *i == id) + .map(|(_, s)| s.to_string()) + } + fn needs_approval(&self, source: &str) -> bool { + tw_gateway::plugin::source::steers_declared(source.as_bytes()) + } +} + +// ─────────────────────────────────────────────────────────── 换成线上的样子 + +pub(crate) fn kind_to_api(k: ItemKind) -> tw_api::ItemKind { + use tw_api::ItemKind as A; + match k { + ItemKind::Provider => A::Provider, + ItemKind::Proxy => A::Proxy, + ItemKind::Key => A::Key, + ItemKind::Alias => A::Alias, + ItemKind::Group => A::Group, + ItemKind::Route => A::Route, + ItemKind::PriceSheet => A::PriceSheet, + ItemKind::SecurityRule => A::SecurityRule, + ItemKind::Plugin => A::Plugin, + ItemKind::ProviderOrder => A::ProviderOrder, + ItemKind::RouteOrder => A::RouteOrder, + ItemKind::PluginOrder => A::PluginOrder, + ItemKind::Global => A::Global, + ItemKind::ClientPrefs => A::ClientPrefs, + } +} + +pub(crate) fn kind_from_api(k: tw_api::ItemKind) -> ItemKind { + // 两边的词一样 + ItemKind::from_slug(k.slug()).expect("the two kinds of item share their words") +} + +pub(crate) fn item_ref(id: &ItemId) -> tw_api::ItemRef { + tw_api::ItemRef { + kind: kind_to_api(id.kind), + name: id.name.clone(), + } +} + +pub(crate) fn item_id(r: &tw_api::ItemRef) -> ItemId { + ItemId::new(kind_from_api(r.kind), r.name.clone()) +} + +fn origin_to_api(o: PackOrigin) -> tw_api::PackOrigin { + tw_api::PackOrigin::from_slug(o.slug()).unwrap_or(tw_api::PackOrigin::Other) +} + +fn counts(c: tw_pack::Counts) -> tw_api::PackCounts { + tw_api::PackCounts { + providers: c.providers, + proxies: c.proxies, + clients: c.clients, + aliases: c.aliases, + groups: c.groups, + routes: c.routes, + price_sheets: c.price_sheets, + security_rules: c.security_rules, + plugins: c.plugins, + } +} + +fn pack_info(m: &tw_pack::Manifest) -> tw_api::PackInfo { + tw_api::PackInfo { + kind: match m.kind { + tw_pack::PackKind::Full => tw_api::PackKind::Full, + tw_pack::PackKind::Partial => tw_api::PackKind::Partial, + }, + created_at_ms: m.created_at_ms, + core_version: m.core_version.clone(), + secrets: m.secrets, + encrypted: m.encrypted, + origin: origin_to_api(m.origin), + counts: counts(m.counts), + } +} + +// ─────────────────────────────────────────────────────────── 失败 + +fn bad_pack(e: PackError) -> Fail { + fail(StatusCode::BAD_REQUEST, e.msg()) +} + +/// 配置（本机的、包里的）拆不开。**由更新的版本写的**说成「太新」 +fn model_fail(e: model::ModelError, pack: Option<&Pack>) -> Fail { + if let Some(p) = pack + && e.is_too_new() + { + return bad_pack(PackError::TooNew { + format: p.manifest.format, + core_version: p.manifest.core_version.clone(), + }); + } + match e { + model::ModelError::Config(r) => apply_fail(ApplyError::Rejected(*r)), + other => fail(StatusCode::UNPROCESSABLE_ENTITY, other.msg()), + } +} + +fn io_fail(path: &std::path::Path, e: std::io::Error) -> Fail { + fail( + StatusCode::INTERNAL_SERVER_ERROR, + msg!( + "backup.write_failed", path = path.display(), detail = e => + "{path} could not be written: {detail}" + ), + ) +} + +fn backup_fail(e: tw_pack::backup::BackupError) -> Fail { + let code = match &e { + tw_pack::backup::BackupError::NotFound { .. } => StatusCode::NOT_FOUND, + _ => StatusCode::INTERNAL_SERVER_ERROR, + }; + fail(code, e.msg()) +} + +fn decode(b64: &str) -> Result, Fail> { + base64::engine::general_purpose::STANDARD + .decode(b64.trim()) + .map_err(|e| { + fail( + StatusCode::BAD_REQUEST, + msg!( + "pack.not_base64", detail = e => + "The file content is not valid base64: {detail}" + ), + ) + }) +} + +/// 口令：至少 8 个字符 +fn passphrase(p: Option<&str>) -> Result, Fail> { + match p { + None => Ok(None), + Some(p) if p.chars().count() >= tw_pack::MIN_PASSPHRASE => Ok(Some(p)), + Some(_) => Err(fail( + StatusCode::BAD_REQUEST, + msg!( + "pack.passphrase_short", min = tw_pack::MIN_PASSPHRASE => + "The passphrase needs at least {min} characters." + ), + )), + } +} + +fn open_pack(data_b64: &str, pass: Option<&str>) -> Result { + let bytes = decode(data_b64)?; + let unseal = match pass { + Some(p) => Unseal::Passphrase(p), + None => Unseal::None, + }; + tw_pack::read(&bytes, unseal).map_err(bad_pack) +} + +fn file_of(pack: &Pack, pass: Option<&str>) -> Result<(String, String), Fail> { + let seal = match pass { + Some(p) => Seal::Passphrase(p), + None => Seal::None, + }; + let bytes = tw_pack::write(pack, seal).map_err(bad_pack)?; + Ok(( + tw_pack::export::file_name(pack.manifest.created_at_ms), + base64::engine::general_purpose::STANDARD.encode(bytes), + )) +} + +// ─────────────────────────────────────────────────────────── 此刻的样子 + +/// 磁盘上此刻的配置（原文和版本）连同它的插件、通用偏好 +fn current(s: &ControlState, origin: PackOrigin) -> Result<(tw_config::Loaded, Pack), Fail> { + let cur = s.cfg.current().map_err(unreadable_config)?; + let pack = tw_pack::disk::current(s.config_path(), &cur.text, origin); + Ok((cur, pack)) +} + +/// 通用偏好由 core 存着吗：经本机连接是，经远程连接不是（在 Lite 那边） +fn core_keeps_prefs() -> bool { + !crate::remote::is_remote() +} + +/// 写下去：插件文件、通用偏好、配置，**按这个顺序**，配置没写成就把前两样还原。之后报 +/// 网关密钥换了、偏好变了 +async fn land( + s: &ControlState, + before: &Pack, + after: &Pack, + base_version: &str, + origin: Origin, + changed_keys: Vec, +) -> Result { + let path = s.config_path().to_path_buf(); + let _edit = s.gateway.plugins.edits.lock().await; + let undo = + tw_pack::disk::write_plugins(&path, &before.plugins, &after.plugins).map_err(|e| { + io_fail( + &tw_config::plugins::dir_in(&tw_pack::disk::dir_of(&path)), + e, + ) + })?; + let prefs_changed = core_keeps_prefs() && after.client_prefs != before.client_prefs; + if prefs_changed && let Err(e) = tw_pack::disk::write_prefs(&path, after.client_prefs.as_ref()) + { + undo.restore(); + return Err(io_fail(&tw_pack::disk::prefs_path(&path), e)); + } + let written = s + .cfg + .write(&after.config_yaml, Some(base_version), origin) + .await; + let version = match written { + Ok(v) => v, + Err(e) => { + undo.restore(); + if prefs_changed { + let _ = tw_pack::disk::write_prefs(&path, before.client_prefs.as_ref()); + } + return Err(apply_fail(e)); + } + }; + if prefs_changed { + s.bus().emit(tw_api::Event::ClientPrefsChanged { + id: s.bus().next_id(), + at_ms: crate::config::now_ms(), + }); + } + if !changed_keys.is_empty() { + s.bus().emit(tw_api::Event::GatewayKeysChanged { + id: s.bus().next_id(), + names: changed_keys, + at_ms: crate::config::now_ms(), + }); + } + Ok(version) +} + +// ─────────────────────────────────────────────────────────── 项目 + +async fn items(State(s): State) -> Result>, Fail> { + let (_, pack) = current(&s, PackOrigin::Export)?; + let snap = model::snapshot(&pack).map_err(|e| model_fail(e, None))?; + Ok(Json( + snap.items + .iter() + .filter(|(id, _)| !id.kind.is_order()) + .filter(|(id, _)| core_keeps_prefs() || id.kind != ItemKind::ClientPrefs) + .map(|(id, item)| tw_api::ItemNode { + item: item_ref(id), + refs: item + .refs + .iter() + .filter(|r| !r.kind.is_order()) + .map(item_ref) + .collect(), + }) + .collect(), + )) +} + +// ─────────────────────────────────────────────────────────── 导出 + +async fn export( + State(s): State, + Json(req): Json, +) -> Result, Fail> { + let pass = passphrase(req.passphrase.as_deref())?; + let (_, mut local) = current(&s, PackOrigin::Export)?; + if !core_keeps_prefs() { + local.client_prefs = req.client_prefs.clone(); + } + let scope = match &req.scope { + tw_api::ExportScope::All => None, + tw_api::ExportScope::Items(items) => { + let snap = model::snapshot(&local).map_err(|e| model_fail(e, None))?; + let ids: Vec = items.iter().map(item_id).collect(); + if let Some(missing) = ids.iter().find(|id| !snap.items.contains_key(*id)) { + return Err(fail( + StatusCode::NOT_FOUND, + msg!( + "pack.no_such_item", kind = missing.kind.slug(), name = &missing.name => + "There is no {kind} named `{name}`." + ), + )); + } + Some(ids) + } + }; + let out = tw_pack::export::export( + &local, + tw_pack::export::Options { + scope, + secrets: req.secrets, + client_prefs: None, + origin: PackOrigin::Export, + }, + ) + .map_err(|e| model_fail(e, None))?; + let (file_name, data_b64) = file_of(&out.pack, pass)?; + Ok(Json(tw_api::ExportedFile { + file_name, + data_b64, + included: out.included.iter().map(item_ref).collect(), + omitted_logins: out.omitted_logins, + })) +} + +// ─────────────────────────────────────────────────────────── 导入 + +fn import_ctx() -> tw_pack::import::Context<'static> { + tw_pack::import::Context { + has_aws_profile: &|p| tw_config::aws_profile::credentials(p).is_ok(), + new_key: &tw_config::generate_key, + with_prefs: core_keeps_prefs(), + } +} + +fn import_fail(e: tw_pack::import::ImportError, pack: &Pack) -> Fail { + use tw_pack::import::ImportError as E; + match e { + E::Model(m) => model_fail(m, Some(pack)), + E::Rejected(r) => apply_fail(ApplyError::Rejected(*r)), + E::Exists(_) | E::NameTaken(..) => fail(StatusCode::CONFLICT, e.msg()), + E::NotInPack(_) => fail(StatusCode::NOT_FOUND, e.msg()), + E::Item(..) => fail(StatusCode::UNPROCESSABLE_ENTITY, e.msg()), + } +} + +fn row_view(r: tw_pack::import::Row) -> tw_api::ImportRow { + use tw_pack::import::{Flag, Status}; + tw_api::ImportRow { + item: item_ref(&r.item), + status: match r.status { + Status::New => tw_api::ImportStatus::New, + Status::Same => tw_api::ImportStatus::Same, + Status::Conflict => tw_api::ImportStatus::Conflict, + Status::Error => tw_api::ImportStatus::Error, + }, + global: r.global, + flags: r + .flags + .into_iter() + .map(|f| match f { + Flag::MissingKey => tw_api::ImportFlag::MissingKey, + Flag::NeedsLogin => tw_api::ImportFlag::NeedsLogin, + Flag::PluginDisabled => tw_api::ImportFlag::PluginDisabled, + }) + .collect(), + differs: r.differs, + suggested_name: r.suggested_name, + error: r.error, + refs: r.refs.iter().map(item_ref).collect(), + } +} + +async fn import_preview( + State(s): State, + Json(req): Json, +) -> Result, Fail> { + let incoming = open_pack(&req.data_b64, req.passphrase.as_deref())?; + let (cur, local) = current(&s, PackOrigin::Import)?; + let rows = tw_pack::import::preview(&local, &incoming, &import_ctx()) + .map_err(|e| import_fail(e, &incoming))?; + Ok(Json(tw_api::ImportPreview { + manifest: pack_info(&incoming.manifest), + base_version: cur.version(), + rows: rows.into_iter().map(row_view).collect(), + client_prefs: incoming.client_prefs.clone(), + })) +} + +async fn import( + State(s): State, + Json(req): Json, +) -> Result, Fail> { + let incoming = open_pack(&req.data_b64, req.passphrase.as_deref())?; + let (cur, local) = current(&s, PackOrigin::Import)?; + if cur.version() != req.base_version { + return Err(apply_fail(ApplyError::Stale { + base: req.base_version, + current: cur.version(), + })); + } + let decisions: Vec = req + .decisions + .iter() + .map(|d| tw_pack::import::Decision { + item: item_id(&d.item), + action: match d.action { + tw_api::ImportAction::Import => tw_pack::import::Action::Import, + tw_api::ImportAction::Overwrite => tw_pack::import::Action::Overwrite, + tw_api::ImportAction::Rename => { + tw_pack::import::Action::Rename(d.new_name.clone().unwrap_or_default()) + } + tw_api::ImportAction::Skip => tw_pack::import::Action::Skip, + }, + }) + .collect(); + let applied = tw_pack::import::apply(&local, &incoming, &decisions, &import_ctx()) + .map_err(|e| import_fail(e, &incoming))?; + // 直接写原文的那几条路一样：装上、打开、批准一个改工具调用的插件要点头。导入落下去时这种 + // 插件停在待批准，这一关照样过一遍 + crate::plugins::guard_raw_write(&s, &cur.text, &applied.pack.config_yaml).await?; + let version = land( + &s, + &local, + &applied.pack, + &cur.version(), + Origin::Import, + applied.changed_keys, + ) + .await?; + Ok(Json(tw_api::ConfigWritten { version })) +} + +// ─────────────────────────────────────────────────────────── 备份 + +async fn backups(State(s): State) -> Result>, Fail> { + let path = s.config_path().to_path_buf(); + let all = tw_pack::backup::list(&path).map_err(backup_fail)?; + // 现在的内容：配置读不进来（安全模式）时没有哪一版算「当前」 + let now = s + .cfg + .current() + .ok() + .map(|cur| tw_pack::disk::current(&path, &cur.text, PackOrigin::Ui)) + .filter(|p| model::snapshot(p).is_ok()) + .map(|p| p.content_id()); + let mut rows: Vec<(tw_api::BackupEntry, String)> = Vec::new(); + for b in all.iter().rev() { + let Ok(pack) = tw_pack::backup::open(b) else { + continue; + }; + let content = pack.content_id(); + rows.push(( + tw_api::BackupEntry { + id: b.id.clone(), + at_ms: b.at_ms, + origin: b.origin.into(), + device: None, + local: true, + sync: false, + counts: tw_api::BackupCounts { + providers: pack.manifest.counts.providers, + plugins: pack.manifest.counts.plugins, + }, + core_version: pack.manifest.core_version.clone(), + current: false, + }, + content, + )); + } + if let Some(remote) = s.cfg.remote_backups() { + for v in remote.list().await { + // 同一份内容只并进本机最新的那一行 + match rows.iter_mut().find(|(e, c)| *c == v.content_id && !e.sync) { + Some((e, _)) => { + e.sync = true; + e.device = Some(v.device_name.clone()); + } + None => rows.push(( + tw_api::BackupEntry { + id: format!("{SYNC_ID}{}", v.content_id), + at_ms: v.at_ms, + origin: tw_api::ConfigOrigin::Sync, + device: Some(v.device_name.clone()), + local: false, + sync: true, + counts: tw_api::BackupCounts { + providers: v.counts.providers, + plugins: v.counts.plugins, + }, + core_version: v.core_version.clone(), + current: false, + }, + v.content_id.clone(), + )), + } + } + } + // 「当前」只标一行：内容和现在一样的版本里最新的那个 + if let Some(now) = &now + && let Some((e, _)) = rows + .iter_mut() + .filter(|(_, c)| c == now) + .max_by_key(|(e, _)| e.at_ms) + { + e.current = true; + } + let mut out: Vec = rows.into_iter().map(|(e, _)| e).collect(); + out.sort_by_key(|e| std::cmp::Reverse(e.at_ms)); + Ok(Json(out)) +} + +/// 按标识取一个版本：本机的读文件，同步位置上的去取 +async fn backup_pack(s: &ControlState, id: &str) -> Result { + if let Some(content) = id.strip_prefix(SYNC_ID) { + let Some(remote) = s.cfg.remote_backups() else { + return Err(backup_fail(tw_pack::backup::BackupError::NotFound { + id: id.to_string(), + })); + }; + return remote + .fetch(content) + .await + .map_err(|m| fail(StatusCode::BAD_GATEWAY, m)); + } + let b = tw_pack::backup::find(s.config_path(), id).map_err(backup_fail)?; + tw_pack::backup::open(&b).map_err(backup_fail) +} + +async fn backup_file( + State(s): State, + Json(req): Json, +) -> Result, Fail> { + let pass = passphrase(req.passphrase.as_deref())?; + let pack = backup_pack(&s, &req.id).await?; + // 另存出去的和导出的一样：本机设置、ChatGPT 的登录不跟着走 + let out = tw_pack::export::export( + &pack, + tw_pack::export::Options { + scope: None, + secrets: true, + client_prefs: None, + origin: PackOrigin::Export, + }, + ) + .map_err(|e| model_fail(e, Some(&pack)))?; + let mut exported = out.pack; + exported.manifest.created_at_ms = pack.manifest.created_at_ms; + let (file_name, data_b64) = file_of(&exported, pass)?; + Ok(Json(tw_api::ExportedFile { + file_name, + data_b64, + included: out.included.iter().map(item_ref).collect(), + omitted_logins: out.omitted_logins, + })) +} + +fn restore_row(r: tw_pack::restore::Row) -> tw_api::RestoreRow { + use tw_pack::restore::{Change, Kept}; + tw_api::RestoreRow { + item: item_ref(&r.item), + change: match r.change { + Change::Add => tw_api::RestoreChange::Add, + Change::Remove => tw_api::RestoreChange::Remove, + Change::Modify => tw_api::RestoreChange::Modify, + }, + kept: r.kept.map(|k| match k { + Kept::KeyInUse => tw_api::KeptReason::KeyInUse, + }), + } +} + +async fn restore_preview( + State(s): State, + Json(req): Json, +) -> Result, Fail> { + let backup = backup_pack(&s, &req.id).await?; + let (cur, local) = current(&s, PackOrigin::Restore)?; + let sync_devices = match s.cfg.remote_backups() { + Some(r) => r.other_devices().await, + None => 0, + }; + // 配置读不进来：整份换回那一版，每一项都是「新增」 + let rows = match tw_pack::restore::preview( + &local, + &backup, + &s.cfg.pinned_keys(), + core_keeps_prefs(), + ) { + Ok(rows) => rows.into_iter().map(restore_row).collect(), + Err(_) if model::snapshot(&local).is_err() => model::snapshot(&backup) + .map_err(|e| model_fail(e, Some(&backup)))? + .items + .keys() + .map(|id| tw_api::RestoreRow { + item: item_ref(id), + change: tw_api::RestoreChange::Add, + kept: None, + }) + .collect(), + Err(e) => return Err(model_fail(e, Some(&backup))), + }; + Ok(Json(tw_api::RestorePreview { + base_version: cur.version(), + rows, + sync_devices, + })) +} + +async fn restore( + State(s): State, + Json(req): Json, +) -> Result, Fail> { + let backup = backup_pack(&s, &req.id).await?; + let (cur, local) = current(&s, PackOrigin::Restore)?; + if cur.version() != req.base_version { + return Err(apply_fail(ApplyError::Stale { + base: req.base_version, + current: cur.version(), + })); + } + let picked: Option> = req + .items + .as_ref() + .map(|xs| xs.iter().map(item_id).collect()); + let (after, changed) = if model::snapshot(&local).is_err() { + // 安全模式：现在这份读不进来，整份换回去 + let mut whole = tw_pack::restore::whole(&backup, &cur.text); + if !core_keeps_prefs() { + whole.client_prefs = local.client_prefs.clone(); + } + (whole, Vec::new()) + } else { + let applied = tw_pack::restore::apply( + &local, + &backup, + picked.as_deref(), + &s.cfg.pinned_keys(), + core_keeps_prefs(), + ) + .map_err(|e| model_fail(e, Some(&backup)))?; + (applied.pack, applied.changed_keys) + }; + crate::plugins::guard_raw_write(&s, &cur.text, &after.config_yaml).await?; + let version = land(&s, &local, &after, &cur.version(), Origin::Restore, changed).await?; + Ok(Json(tw_api::ConfigWritten { version })) +} + +// ─────────────────────────────────────────────────────────── 通用偏好、固定密钥 + +async fn get_prefs(State(s): State) -> Json { + Json(tw_api::ClientPrefs { + prefs: tw_pack::disk::read_prefs(s.config_path()), + }) +} + +async fn put_prefs( + State(s): State, + Json(req): Json, +) -> Result, Fail> { + if let Some(p) = &req.prefs + && p.to_string().len() > tw_api::CLIENT_PREFS_MAX + { + return Err(fail( + StatusCode::PAYLOAD_TOO_LARGE, + msg!( + "backup.prefs_too_large", limit = tw_api::CLIENT_PREFS_MAX / 1024 => + "The client preferences are larger than {limit} KiB." + ), + )); + } + let path = s.config_path().to_path_buf(); + if tw_pack::disk::read_prefs(&path) == req.prefs { + return Ok(Json(())); + } + tw_pack::disk::write_prefs(&path, req.prefs.as_ref()) + .map_err(|e| io_fail(&tw_pack::disk::prefs_path(&path), e))?; + // 偏好也是备份的一部分：只改了它也留一版 + if let Ok(cur) = s.cfg.current() + && let Err(e) = tw_pack::backup::record(&path, &cur.text, Origin::Ui) + { + tracing::warn!("a backup version could not be kept: {e}"); + } + s.bus().emit(tw_api::Event::ClientPrefsChanged { + id: s.bus().next_id(), + at_ms: crate::config::now_ms(), + }); + Ok(Json(())) +} + +async fn pin_keys( + State(s): State, + Json(req): Json, +) -> Result, Fail> { + let names: BTreeSet = req.names.into_iter().collect(); + s.cfg + .set_pinned_keys(names) + .map_err(|e| io_fail(&tw_pack::disk::pinned_path(s.config_path()), e))?; + Ok(Json(())) +} + +/// 给同步那一路用：固定密钥 +pub fn pinned_keys(s: &ControlState) -> BTreeSet { + s.cfg.pinned_keys() +} diff --git a/crates/tw-control/src/config.rs b/crates/tw-control/src/config.rs index d0879663..4f02ff61 100644 --- a/crates/tw-control/src/config.rs +++ b/crates/tw-control/src/config.rs @@ -27,6 +27,10 @@ pub struct ConfigManager { rejected: std::sync::Mutex>, /// 每换入一份配置响一次（[`Self::applied`]）。默认插件那一路等着它 applied: tokio::sync::Notify, + /// 固定密钥：本机接管的客户端在用的网关密钥，桌面端报上来（`PUT /pinned-keys`） + pinned: std::sync::Mutex>, + /// 同步位置上的备份版本。没开同步是 None + remote: std::sync::RwLock>>, } /// 一次配置改动没成的原因。 @@ -111,6 +115,9 @@ impl ConfigManager { let seen = store::read(&path).ok().map(|l| l.fingerprint); // 插件文件的路径相对配置文件所在的目录：**知道配置在哪儿的是这里**，告诉网关一声 gateway.set_config_dir(crate::plugins::dir_of(&path)); + // 配置包、备份按「代码」「数据」分开比较插件，要插件那一套来拆 + crate::backups::install_codec(); + let pinned = tw_pack::disk::read_pinned(&path); Self { path, gateway, @@ -118,6 +125,8 @@ impl ConfigManager { seen: Mutex::new(seen), rejected: std::sync::Mutex::new(None), applied: tokio::sync::Notify::new(), + pinned: std::sync::Mutex::new(pinned), + remote: std::sync::RwLock::new(None), } } @@ -243,7 +252,7 @@ impl ConfigManager { // 之后的效果是「每个存在过的版本各一条」，最新那条就是现在跑 // 着的 —— 于是「回到上一版」在列表上就是第二条，不用数。 self.set_rejected(None); - let _ = tw_config::history::snapshot(&self.path, text, origin); + self.backup(tw_pack::backup::record(&self.path, text, origin)); let version = store::version_of(text); tracing::info!(%version, origin = origin.slug(), "the configuration is in effect"); self.applied.notify_one(); @@ -299,9 +308,11 @@ impl ConfigManager { { return Err(ApplyError::RemoteControlLocked); } - // 改之前那一版进历史。**这一步在写盘之前** —— 写完再存的话， + // 改之前那一版留一个备份版本。**这一步在写盘之前** —— 写完再存的话， // 中间崩一次就永远丢了那一版，而那恰恰是最需要它的时刻。 - let _ = tw_config::history::snapshot(&self.path, &cur.text, origin); + self.backup(tw_pack::backup::record_before( + &self.path, &cur.text, origin, + )); // 写盘之前再确认一次磁盘还是我们读到的那份 —— 绝不静默覆盖手改 let fp = store::write_if_unchanged(&self.path, &cur.fingerprint, new_text)?; // **先记指纹再换入。**顺序反了的话，文件事件可能在记指纹之前就 @@ -338,34 +349,51 @@ impl ConfigManager { self.write(&text, Some(&cur.version()), origin).await } - /// 回到某一版。 - pub async fn rollback(&self, version: &str) -> Result { - let text = self.rollback_text(version)?; - let cur = self.current().ok(); - // 回滚也走同一条写入路径，所以它同样会：校验、存历史、防回环。 - self.write( - &text, - cur.as_ref().map(|c| c.version()).as_deref(), - Origin::Rollback, - ) - .await + /// 留备份版本没成：记一行，不挡这次写入 —— 配置本身已经写下去了 + fn backup(&self, r: Result, tw_pack::backup::BackupError>) { + if let Err(e) = r { + tracing::warn!("a backup version could not be kept: {e}"); + } + } + + /// 启动时：磁盘上此刻的样子和最新的备份版本不一样就留一版（第一次运行、升级之后、core + /// 没在跑时命令行改过）。来源记成外部改动 —— 说不清是谁改的 + pub fn record_startup(&self) { + if let Ok(cur) = self.current() { + self.backup(tw_pack::backup::record( + &self.path, + &cur.text, + Origin::External, + )); + } + } + + /// 本机接管的客户端在用的网关密钥（固定密钥）。恢复、导入、同步不删它们 + pub fn pinned_keys(&self) -> std::collections::BTreeSet { + self.pinned.lock().map(|g| g.clone()).unwrap_or_default() + } + + /// 换掉固定密钥，存进数据目录（`pinned-keys.json`），重启之后还在 + pub fn set_pinned_keys( + &self, + names: std::collections::BTreeSet, + ) -> std::io::Result<()> { + tw_pack::disk::write_pinned(&self.path, &names)?; + if let Ok(mut g) = self.pinned.lock() { + *g = names; + } + Ok(()) + } + + /// 同步位置上的版本（开着同步时由同步那一路装上，关掉时换成 None） + pub fn set_remote_backups(&self, r: Option>) { + if let Ok(mut g) = self.remote.write() { + *g = r; + } } - /// 回滚要写回去的那一版原文。**不写**：控制面先看过它（插件那一节，见 - /// `plugins::guard_raw_write`）再写 - pub fn rollback_text(&self, version: &str) -> Result { - let all = tw_config::history::list(&self.path)?; - let target = all - .iter() - .rev() - .find(|v| v.version == version || v.version.ends_with(version)) - .ok_or_else(|| { - ApplyError::BadPath(msg!( - "control.no_such_version", version = version => - "the version history has no {version}" - )) - })?; - Ok(tw_config::history::read(target)?) + pub fn remote_backups(&self) -> Option> { + self.remote.read().ok().and_then(|g| g.clone()) } /// 记下一个不是我们写的指纹。**首次运行生成配置之后要调它** —— diff --git a/crates/tw-control/src/lib.rs b/crates/tw-control/src/lib.rs index fbfb3d23..f6720f71 100644 --- a/crates/tw-control/src/lib.rs +++ b/crates/tw-control/src/lib.rs @@ -23,6 +23,7 @@ use tw_types::{Msg, msg}; use contract::RouterExt; pub mod aliases; +pub mod backups; pub mod chatgpt; pub mod clients; pub mod config; @@ -104,9 +105,8 @@ pub fn router(state: ControlState) -> Router { .at(ep::GetConfig, get_config) .at(ep::PatchConfig, patch_config) .at(ep::PutConfig, put_config) - .at(ep::ConfigHistory, config_history) .at(ep::ConfigAt, config::path_at) - .at(ep::ConfigRollback, config_rollback) + .merge(backups::router()) .at(ep::ConfigRepairPlan, config_repair_plan) .at(ep::RepairConfig, repair_config) .at(ep::Summary, summary) @@ -1515,50 +1515,6 @@ async fn put_config( Ok(Json(tw_api::ConfigWritten { version })) } -async fn config_history( - State(s): State, -) -> Result>, Fail> { - let all = tw_config::history::list(s.config_path()).map_err(unreadable_config)?; - let now = s.cfg.current().map(|c| c.version()).unwrap_or_default(); - // **新的在前。**用户找的几乎总是最近那几版。 - Ok(Json( - all.into_iter() - .rev() - .map(|v| tw_api::ConfigVersion { - current: v.version == now, - version: v.version, - at_ms: v.at_ms, - origin: v.origin.into(), - bytes: v.bytes, - }) - .collect(), - )) -} - -async fn config_rollback( - State(s): State, - Json(req): Json, -) -> Result, Fail> { - let cur = s - .cfg - .current() - .map_err(|e| apply_fail(ApplyError::Store(e)))?; - let text = s.cfg.rollback_text(&req.version).map_err(apply_fail)?; - // 回到一份打开着工具调用插件的旧配置，同样要点头 - plugins::guard_raw_write(&s, &cur.text, &text).await?; - // 回滚也走同一条写入路径：校验、存历史、防回环 - let version = s - .cfg - .write( - &text, - Some(&cur.version()), - tw_config::history::Origin::Rollback, - ) - .await - .map_err(apply_fail)?; - Ok(Json(tw_api::ConfigWritten { version })) -} - /// 一键修复会改哪几处。修不了、或者本来就读得进来，`fixes` 是空的 async fn config_repair_plan( State(s): State, @@ -1606,7 +1562,7 @@ async fn repair_config( .write( &r.text, Some(&cur.version()), - tw_config::history::Origin::Ui, + tw_config::history::Origin::Repair, ) .await .map_err(apply_fail)?; diff --git a/crates/tw-control/tests/backups.rs b/crates/tw-control/tests/backups.rs new file mode 100644 index 00000000..dfab9e05 --- /dev/null +++ b/crates/tw-control/tests/backups.rs @@ -0,0 +1,566 @@ +//! 配置包、导入导出、备份与恢复、通用偏好、固定密钥：经控制面走一遍，断言落在文件上。 + +use std::sync::Arc; + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use base64::Engine; +use tower::ServiceExt; +use tw_control::{ConfigManager, ControlState}; + +const KEY: &str = "c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00"; + +fn mine() -> String { + format!( + "version: 1 +listen: + control: + key: {KEY} +clients: + - name: default + key: tw-aaaa + - name: claude-code + key: tw-cccc + client: claude-code +# 我的上游 +providers: + - name: 官方 + base_url: https://api.anthropic.com + key: sk-official +routes: + - name: default + rules: + - name: 兜底 + to: 官方 +" + ) +} + +fn theirs() -> String { + "version: 1 +listen: + control: + key: 1111111111111111111111111111111111111111111111111111111111111111 +clients: + - name: default + key: tw-theirs +providers: + - name: 官方 + base_url: https://api.anthropic.com/v2 + key: sk-theirs + - name: relay + base_url: https://relay.example + key: sk-relay +failover: + pause_secs: 30 +" + .to_string() +} + +struct Bed { + dir: tempfile::TempDir, + app: axum::Router, + state: ControlState, +} + +impl Bed { + fn path(&self) -> std::path::PathBuf { + self.dir.path().join("config.yaml") + } + fn file(&self) -> String { + std::fs::read_to_string(self.path()).unwrap() + } + fn version(&self) -> String { + tw_config::store::version_of(&self.file()) + } +} + +fn state_for( + p: std::path::PathBuf, + mgr: Arc, + gw: tw_gateway::AppState, +) -> ControlState { + let _ = p; + ControlState { + shutdown: Default::default(), + remote: Default::default(), + cfg: mgr, + gateway: gw, + store: None, + started: std::time::Instant::now(), + price_updater: Default::default(), + chatgpt: Default::default(), + zai: Default::default(), + } +} + +fn bed(yaml: &str) -> Bed { + let d = tempfile::tempdir().unwrap(); + let p = d.path().join("config.yaml"); + std::fs::write(&p, yaml).unwrap(); + let cfg = tw_config::try_parse(yaml).unwrap(); + let gw = tw_gateway::AppState::new(cfg).unwrap(); + let bus = gw.bus.clone(); + let mgr = Arc::new(ConfigManager::new(p.clone(), gw.clone(), bus)); + let state = state_for(p, mgr, gw); + Bed { + app: tw_control::router(state.clone()), + state, + dir: d, + } +} + +async fn call( + app: &axum::Router, + method: &str, + path: &str, + body: serde_json::Value, +) -> (StatusCode, serde_json::Value) { + let r = app + .clone() + .oneshot( + Request::builder() + .method(method) + .uri(path) + .header("content-type", "application/json") + .body(Body::from(body.to_string())) + .unwrap(), + ) + .await + .unwrap(); + let st = r.status(); + let b = axum::body::to_bytes(r.into_body(), 1 << 24).await.unwrap(); + let text = String::from_utf8_lossy(&b).to_string(); + let v = serde_json::from_str(&text).unwrap_or(serde_json::Value::String(text)); + (st, v) +} + +async fn export_all(b: &Bed, secrets: bool) -> serde_json::Value { + let (st, v) = call( + &b.app, + "POST", + "/config/export", + serde_json::json!({ "scope": "all", "secrets": secrets }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{v}"); + v +} + +#[tokio::test] +async fn an_export_carries_no_local_settings_and_imports_item_by_item() { + let src = bed(&theirs()); + let file = export_all(&src, true).await; + assert!(file["file_name"].as_str().unwrap().ends_with(".twconfig")); + let bytes = base64::engine::general_purpose::STANDARD + .decode(file["data_b64"].as_str().unwrap()) + .unwrap(); + let pack = tw_pack::read(&bytes, tw_pack::Unseal::None).unwrap(); + assert!( + !pack.config_yaml.contains("1111111111"), + "{}", + pack.config_yaml + ); + assert!(!pack.config_yaml.contains("listen"), "{}", pack.config_yaml); + + let dst = bed(&mine()); + let (st, preview) = call( + &dst.app, + "POST", + "/config/import/preview", + serde_json::json!({ "data_b64": file["data_b64"] }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{preview}"); + let status = |kind: &str, name: &str| -> String { + preview["rows"] + .as_array() + .unwrap() + .iter() + .find(|r| r["item"]["kind"] == kind && r["item"]["name"] == name) + .map(|r| r["status"].as_str().unwrap().to_string()) + .unwrap_or_else(|| panic!("no row {kind} {name}: {preview}")) + }; + assert_eq!(status("provider", "relay"), "new"); + assert_eq!(status("provider", "官方"), "conflict"); + assert_eq!(status("global", "failover"), "new"); + assert_eq!(preview["manifest"]["kind"], "full"); + assert_eq!(preview["base_version"], dst.version()); + + let (st, v) = call( + &dst.app, + "POST", + "/config/import", + serde_json::json!({ + "data_b64": file["data_b64"], + "base_version": preview["base_version"], + "decisions": [ + { "item": { "kind": "provider", "name": "relay" }, "action": "import" }, + { "item": { "kind": "provider", "name": "官方" }, "action": "rename", "new_name": "官方 (2)" }, + { "item": { "kind": "global", "name": "failover" }, "action": "import" }, + ], + }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{v}"); + let text = dst.file(); + let cfg = tw_config::try_parse(&text).unwrap(); + let names: Vec<&str> = cfg.providers.iter().map(|p| p.name.as_str()).collect(); + assert_eq!(names, ["官方", "官方 (2)", "relay"]); + assert_eq!(cfg.failover.pause_secs, 30); + assert!( + text.contains("# 我的上游"), + "untouched lines keep their comments: {text}" + ); + assert!(text.contains(KEY)); + // 导入留了一个版本，来源是导入 + let (st, list) = call(&dst.app, "GET", "/backups", serde_json::json!(null)).await; + assert_eq!(st, StatusCode::OK, "{list}"); + assert_eq!(list[0]["origin"], "import"); + assert_eq!(list[0]["current"], true); + // 预览之后配置变了：409 + let (st, _) = call( + &dst.app, + "POST", + "/config/import", + serde_json::json!({ + "data_b64": file["data_b64"], + "base_version": preview["base_version"], + "decisions": [], + }), + ) + .await; + assert_eq!(st, StatusCode::CONFLICT); +} + +#[tokio::test] +async fn an_encrypted_export_needs_its_passphrase_to_preview() { + tw_pack::TEST_WORK_FACTOR.store(10, std::sync::atomic::Ordering::Relaxed); + let src = bed(&theirs()); + let (st, v) = call( + &src.app, + "POST", + "/config/export", + serde_json::json!({ "scope": "all", "secrets": false, "passphrase": "short" }), + ) + .await; + assert_eq!(st, StatusCode::BAD_REQUEST); + assert_eq!(v["code"], "pack.passphrase_short"); + let (st, file) = call( + &src.app, + "POST", + "/config/export", + serde_json::json!({ "scope": "all", "secrets": false, "passphrase": "long enough" }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{file}"); + let dst = bed(&mine()); + for (pass, code) in [ + (serde_json::Value::Null, "pack.need_passphrase"), + (serde_json::json!("wrong one!"), "pack.wrong_passphrase"), + ] { + let (st, v) = call( + &dst.app, + "POST", + "/config/import/preview", + serde_json::json!({ "data_b64": file["data_b64"], "passphrase": pass }), + ) + .await; + assert_eq!(st, StatusCode::BAD_REQUEST); + assert_eq!(v["code"], code); + } + let (st, preview) = call( + &dst.app, + "POST", + "/config/import/preview", + serde_json::json!({ "data_b64": file["data_b64"], "passphrase": "long enough" }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{preview}"); + assert_eq!(preview["manifest"]["secrets"], false); + assert_eq!(preview["manifest"]["encrypted"], true); + let relay = preview["rows"] + .as_array() + .unwrap() + .iter() + .find(|r| r["item"]["name"] == "relay") + .unwrap(); + assert_eq!(relay["flags"], serde_json::json!(["missing_key"])); +} + +#[tokio::test] +async fn exporting_chosen_items_brings_what_they_refer_to() { + let b = bed(&mine()); + let (st, v) = call( + &b.app, + "POST", + "/config/export", + serde_json::json!({ + "scope": { "items": [{ "kind": "route", "name": "default" }] }, + "secrets": true, + }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{v}"); + let included: Vec = v["included"] + .as_array() + .unwrap() + .iter() + .map(|i| { + format!( + "{}/{}", + i["kind"].as_str().unwrap(), + i["name"].as_str().unwrap() + ) + }) + .collect(); + assert_eq!(included, ["provider/官方", "route/default"]); + let (st, v) = call( + &b.app, + "POST", + "/config/export", + serde_json::json!({ + "scope": { "items": [{ "kind": "provider", "name": "nope" }] }, + "secrets": true, + }), + ) + .await; + assert_eq!(st, StatusCode::NOT_FOUND); + assert_eq!(v["code"], "pack.no_such_item"); + // 项目清单 + let (st, items) = call(&b.app, "GET", "/config/items", serde_json::json!(null)).await; + assert_eq!(st, StatusCode::OK, "{items}"); + let route = items + .as_array() + .unwrap() + .iter() + .find(|n| n["item"]["kind"] == "route") + .unwrap(); + assert_eq!( + route["refs"], + serde_json::json!([{ "kind": "provider", "name": "官方" }]) + ); +} + +#[tokio::test] +async fn a_restore_brings_back_items_keeps_pinned_keys_and_is_itself_a_version() { + let b = bed(&mine()); + // 记下现在这一版，再改两处：删掉一把密钥、换掉上游地址 + b.state.cfg.record_startup(); + let changed = mine() + .replace( + " - name: claude-code\n key: tw-cccc\n client: claude-code\n", + "", + ) + .replace("https://api.anthropic.com\n", "https://changed.example\n"); + b.state + .cfg + .write(&changed, None, tw_config::history::Origin::Ui) + .await + .unwrap(); + // 接管着 claude-code 的那把在现在的配置里已经没了；固定的是 default + let (st, _) = call( + &b.app, + "PUT", + "/pinned-keys", + serde_json::json!({ "names": ["default"] }), + ) + .await; + assert_eq!(st, StatusCode::OK); + assert_eq!( + tw_pack::disk::read_pinned(&b.path()), + ["default".to_string()].into_iter().collect() + ); + let (_, list) = call(&b.app, "GET", "/backups", serde_json::json!(null)).await; + let oldest = list.as_array().unwrap().last().unwrap()["id"].clone(); + let (st, preview) = call( + &b.app, + "POST", + "/backups/preview", + serde_json::json!({ "id": oldest }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{preview}"); + let rows: Vec = preview["rows"] + .as_array() + .unwrap() + .iter() + .map(|r| { + format!( + "{} {}/{}", + r["change"].as_str().unwrap(), + r["item"]["kind"].as_str().unwrap(), + r["item"]["name"].as_str().unwrap() + ) + }) + .collect(); + assert_eq!(rows, ["modify provider/官方", "add key/claude-code"]); + // 只恢复上游 + let (st, v) = call( + &b.app, + "POST", + "/backups/restore", + serde_json::json!({ + "id": oldest, + "base_version": preview["base_version"], + "items": [{ "kind": "provider", "name": "官方" }], + }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{v}"); + let cfg = tw_config::try_parse(&b.file()).unwrap(); + assert_eq!(cfg.providers[0].base_url, "https://api.anthropic.com"); + assert!(!cfg.clients.iter().any(|c| c.name == "claude-code")); + let (_, list) = call(&b.app, "GET", "/backups", serde_json::json!(null)).await; + assert_eq!(list[0]["origin"], "restore"); + // 另存为配置包：本机设置不在里面 + let (st, file) = call( + &b.app, + "POST", + "/backups/file", + serde_json::json!({ "id": oldest }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{file}"); + let bytes = base64::engine::general_purpose::STANDARD + .decode(file["data_b64"].as_str().unwrap()) + .unwrap(); + let pack = tw_pack::read(&bytes, tw_pack::Unseal::None).unwrap(); + assert!(!pack.config_yaml.contains("listen"), "{}", pack.config_yaml); + let (st, v) = call( + &b.app, + "POST", + "/backups/preview", + serde_json::json!({ "id": "123" }), + ) + .await; + assert_eq!(st, StatusCode::NOT_FOUND, "{v}"); + assert_eq!(v["code"], "backup.not_found"); +} + +#[tokio::test] +async fn client_preferences_are_kept_announced_and_part_of_a_version() { + let b = bed(&mine()); + let mut events = b.state.bus().subscribe(); + let (st, v) = call(&b.app, "GET", "/client-prefs", serde_json::json!(null)).await; + assert_eq!(st, StatusCode::OK); + assert_eq!(v, serde_json::json!({ "prefs": null })); + let prefs = serde_json::json!({ "language": "en", "appearance": "dark" }); + let (st, _) = call( + &b.app, + "PUT", + "/client-prefs", + serde_json::json!({ "prefs": prefs }), + ) + .await; + assert_eq!(st, StatusCode::OK); + let (_, v) = call(&b.app, "GET", "/client-prefs", serde_json::json!(null)).await; + assert_eq!(v["prefs"], prefs); + let got = tokio::time::timeout(std::time::Duration::from_secs(1), async { + loop { + if let Ok(tw_api::Event::ClientPrefsChanged { .. }) = events.recv().await { + return; + } + } + }) + .await; + assert!(got.is_ok(), "no client_prefs_changed"); + let last = tw_pack::backup::list(&b.path()).unwrap().pop().unwrap(); + assert_eq!( + tw_pack::backup::open(&last).unwrap().client_prefs, + Some(prefs) + ); + let big = "x".repeat(tw_api::CLIENT_PREFS_MAX + 1); + let (st, v) = call( + &b.app, + "PUT", + "/client-prefs", + serde_json::json!({ "prefs": { "big": big } }), + ) + .await; + assert_eq!(st, StatusCode::PAYLOAD_TOO_LARGE, "{v}"); +} + +#[tokio::test] +async fn overwriting_a_key_by_import_announces_it() { + let src = bed(&theirs()); + let file = export_all(&src, true).await; + let dst = bed(&mine()); + let mut events = dst.state.bus().subscribe(); + let (_, preview) = call( + &dst.app, + "POST", + "/config/import/preview", + serde_json::json!({ "data_b64": file["data_b64"] }), + ) + .await; + let (st, v) = call( + &dst.app, + "POST", + "/config/import", + serde_json::json!({ + "data_b64": file["data_b64"], + "base_version": preview["base_version"], + "decisions": [{ "item": { "kind": "key", "name": "default" }, "action": "overwrite" }], + }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{v}"); + let names = tokio::time::timeout(std::time::Duration::from_secs(1), async { + loop { + if let Ok(tw_api::Event::GatewayKeysChanged { names, .. }) = events.recv().await { + return names; + } + } + }) + .await + .expect("no gateway_keys_changed"); + assert_eq!(names, ["default"]); +} + +/// 安全模式：配置读不进来，从备份恢复整份换回去，钥匙是现在这一把 +#[tokio::test] +async fn in_safe_mode_a_backup_comes_back_whole() { + let d = tempfile::tempdir().unwrap(); + let p = d.path().join("config.yaml"); + std::fs::write(&p, mine()).unwrap(); + // 先有一版好的备份 + tw_pack::backup::record(&p, &mine(), tw_config::history::Origin::Ui).unwrap(); + let broken = format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n kye: x\n" + ); + std::fs::write(&p, &broken).unwrap(); + let (cfg, r) = tw_config::stand_in(&broken).unwrap(); + let gw = tw_gateway::AppState::new(cfg).unwrap(); + let bus = gw.bus.clone(); + let mgr = Arc::new(ConfigManager::new(p.clone(), gw.clone(), bus).standing_in(&r)); + let app = tw_control::router(state_for(p.clone(), mgr, gw)); + let (st, list) = call(&app, "GET", "/backups", serde_json::json!(null)).await; + assert_eq!(st, StatusCode::OK, "{list}"); + let id = list[0]["id"].clone(); + let (st, preview) = call( + &app, + "POST", + "/backups/preview", + serde_json::json!({ "id": id }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{preview}"); + assert!( + preview["rows"] + .as_array() + .unwrap() + .iter() + .all(|r| r["change"] == "add") + ); + let (st, v) = call( + &app, + "POST", + "/backups/restore", + serde_json::json!({ "id": id, "base_version": preview["base_version"] }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{v}"); + let text = std::fs::read_to_string(&p).unwrap(); + assert_eq!(text, mine()); +} diff --git a/crates/tw-control/tests/contract.rs b/crates/tw-control/tests/contract.rs index 7ada07bd..45a5228c 100644 --- a/crates/tw-control/tests/contract.rs +++ b/crates/tw-control/tests/contract.rs @@ -130,13 +130,7 @@ async fn failures_the_framework_answers_are_messages_too() { assert_eq!(error(&body.unwrap()).code, "control.method_not_allowed"); // 请求体读不成请求类型 - let (st, body) = send( - &app, - "POST", - ep::ConfigRollback::PATH, - Some("{\"version\":5}"), - ) - .await; + let (st, body) = send(&app, "POST", ep::RestorePreview::PATH, Some("{\"id\":5}")).await; assert!(st.is_client_error(), "{st}"); let m = error(&body.unwrap()); assert_eq!(m.code, "control.request_rejected"); diff --git a/crates/tw-control/tests/gate.rs b/crates/tw-control/tests/gate.rs index 8f30731f..71c45f50 100644 --- a/crates/tw-control/tests/gate.rs +++ b/crates/tw-control/tests/gate.rs @@ -373,27 +373,31 @@ async fn the_key_cannot_be_changed_or_removed_through_the_control_plane() { /// 历史里没有钥匙；回滚回去，钥匙是现在这一把。 #[tokio::test] -async fn a_rollback_through_the_control_plane_keeps_the_key() { +async fn a_restore_through_the_control_plane_keeps_the_key() { let b = bed(); let app = b.app(); let path = b.dir.path().join("config.yaml"); - let v0 = tw_config::store::version_of(&b.file()); b.state .cfg .write(&yaml(KEY).replace("tw-aaaa", "tw-bbbb"), None, Origin::Ui) .await .unwrap(); - for v in tw_config::history::list(&path).unwrap() { - let t = tw_config::history::read(&v).unwrap(); - assert!(!t.contains(KEY), "历史里有钥匙：{t}"); + let all = tw_pack::backup::list(&path).unwrap(); + for v in &all { + let t = tw_pack::backup::open(v).unwrap().config_yaml; + assert!(!t.contains(KEY), "备份里有钥匙：{t}"); } + let first = &all[0]; + let base = tw_config::store::version_of(&b.file()); let (st, body) = call( &app, "POST", - "/config/rollback", - serde_json::json!({ "version": v0 }), + "/backups/restore", + serde_json::json!({ "id": first.id, "base_version": base }), ) .await; assert_eq!(st, StatusCode::OK, "{body}"); - assert_eq!(b.file(), yaml(KEY)); + let now = b.file(); + assert!(now.contains(KEY), "{now}"); + assert!(now.contains("tw-aaaa"), "{now}"); } diff --git a/crates/tw-control/tests/m2_acceptance.rs b/crates/tw-control/tests/m2_acceptance.rs index 8182e1da..fdc4c6d4 100644 --- a/crates/tw-control/tests/m2_acceptance.rs +++ b/crates/tw-control/tests/m2_acceptance.rs @@ -127,6 +127,32 @@ async fn two_b_a_path_by_index_and_a_path_by_name_mean_the_same_thing() { assert_eq!(by_name, by_index); } +/// 回到备份里内容是 `version` 那一版：和界面上「恢复…」同一套（`tw_pack::restore`） +async fn restore(mgr: &tw_control::ConfigManager, version: &str) { + let path = mgr.path().to_path_buf(); + let b = tw_pack::backup::list(&path) + .unwrap() + .into_iter() + .rev() + .find(|b| { + let text = tw_pack::backup::open(b).unwrap().config_yaml; + let real = tw_config::control_key::unmask( + &text, + tw_config::control_key::raw_in(REAL).unwrap().as_deref(), + ) + .unwrap(); + tw_config::store::version_of(&real) == version + }) + .expect("这一版在备份里"); + let backup = tw_pack::backup::open(&b).unwrap(); + let cur = mgr.current().unwrap(); + let local = tw_pack::disk::current(&path, &cur.text, tw_pack::PackOrigin::Restore); + let out = tw_pack::restore::apply(&local, &backup, None, &Default::default(), true).unwrap(); + mgr.write(&out.pack.config_yaml, Some(&cur.version()), Origin::Restore) + .await + .unwrap(); +} + #[tokio::test] async fn three_a_bad_edit_can_be_undone_in_one_step() { let (_d, mgr) = setup(); @@ -145,17 +171,17 @@ async fn three_a_bad_edit_can_be_undone_in_one_step() { .contains("坏的") ); - mgr.rollback(&good).await.unwrap(); + restore(&mgr, &good).await; assert_eq!( std::fs::read_to_string(mgr.path()).unwrap(), REAL, - "回滚没有精确还原" + "恢复没有精确还原" ); } #[tokio::test] -async fn three_b_a_rollback_can_itself_be_rolled_back() { - // 「回滚之后发现回错了」是回滚最常见的用法之一。 +async fn three_b_a_restore_can_itself_be_undone() { + // 「恢复之后发现回错了」是恢复最常见的用法之一：恢复本身也留一个版本 let (_d, mgr) = setup(); let v0 = mgr.current().unwrap().version(); mgr.patch(&patch("/providers/官方/key", "sk-new"), None, Origin::Ui) @@ -163,14 +189,14 @@ async fn three_b_a_rollback_can_itself_be_rolled_back() { .unwrap(); let v1 = mgr.current().unwrap().version(); - mgr.rollback(&v0).await.unwrap(); + restore(&mgr, &v0).await; assert!( !std::fs::read_to_string(mgr.path()) .unwrap() .contains("sk-new") ); - mgr.rollback(&v1).await.unwrap(); + restore(&mgr, &v1).await; assert!( std::fs::read_to_string(mgr.path()) .unwrap() diff --git a/crates/tw-control/tests/plugin_defaults.rs b/crates/tw-control/tests/plugin_defaults.rs index d9b1edef..83e2bf37 100644 --- a/crates/tw-control/tests/plugin_defaults.rs +++ b/crates/tw-control/tests/plugin_defaults.rs @@ -314,8 +314,8 @@ async fn the_first_run_adds_every_default_turned_off() { .is_some() ); - // 写配置的这一版来源是 defaults，一版、一条历史 - let (_, history) = call(&b.app, "GET", "/config/history", None).await; + // 写配置的这一版来源是 defaults，一版、一个备份版本 + let (_, history) = call(&b.app, "GET", "/backups", None).await; let now = history .as_array() .unwrap() diff --git a/crates/tw-control/tests/plugins.rs b/crates/tw-control/tests/plugins.rs index 30a5d9ed..e9ea6a69 100644 --- a/crates/tw-control/tests/plugins.rs +++ b/crates/tw-control/tests/plugins.rs @@ -1588,22 +1588,28 @@ async fn raw_configuration_writes_cannot_bypass_the_confirmation() { assert_eq!(b.config(), before); std::fs::write(b.file(&id), rewrite_calls()).unwrap(); - // 回滚到它开着的那一版：拒绝 + // 恢复到它开着的那一版：恢复了，但插件停在待批准，跑不起来 —— 打开它要在这里点头 let (st, v) = b.save_confirmed(&id, &rewrite_calls(), true).await; assert_eq!(st, StatusCode::OK, "{v}"); - let on = b.version().await; + let on_backup = tw_pack::backup::list(&b.dir.join("config.yaml")) + .unwrap() + .last() + .unwrap() + .id + .clone(); let (st, v) = b.save(&id, &rewrite_calls(), false).await; assert_eq!(st, StatusCode::OK, "{v}"); - let before = b.config(); let (st, v) = call( &b.app, "POST", - "/config/rollback", - Some(json!({"version": on})), + "/backups/restore", + Some(json!({"id": on_backup, "base_version": b.version().await})), ) .await; - refused(st, &v, "rolling back to a version where it was on"); - assert_eq!(b.config(), before); + assert_eq!(st, StatusCode::OK, "{v}"); + let p = b.parsed().plugins.into_iter().find(|p| p.id == id).unwrap(); + assert!(p.enabled); + assert_ne!(p.sha256, sha(&rewrite_calls()), "恢复替人批准了"); // 和插件无关的改动照常 let (st, v) = put(shown().await.replace("# 默认那把", "# 改了一个注释")).await; diff --git a/crates/tw-control/tests/pricing.rs b/crates/tw-control/tests/pricing.rs index 838c150d..b8312987 100644 --- a/crates/tw-control/tests/pricing.rs +++ b/crates/tw-control/tests/pricing.rs @@ -211,7 +211,7 @@ async fn a_sheet_is_created_chosen_renamed_and_deleted_only_once_nobody_uses_it( async fn a_sheet_saved_with_its_upstreams_assigns_them_in_the_same_version() { let b = bed(BASE); let versions = || { - tw_config::history::list(&b.dir.path().join("config.yaml")) + tw_pack::backup::list(&b.dir.path().join("config.yaml")) .unwrap() .len() }; diff --git a/crates/tw-control/tests/resources.rs b/crates/tw-control/tests/resources.rs index 3fd30884..481eb3ce 100644 --- a/crates/tw-control/tests/resources.rs +++ b/crates/tw-control/tests/resources.rs @@ -443,14 +443,15 @@ async fn renaming_an_upstream_moves_its_references_in_the_same_version() { let cfg = b.parsed(); assert_eq!(cfg.providers[0].name, "anthropic"); assert_eq!(cfg.groups[0].names(), ["anthropic"]); - // 一次保存一个版本：历史里只多了改之前的那一份 - let history = tw_config::history::list(&b.dir.path().join("config.yaml")).unwrap(); + // 一次保存一个版本：备份里是改之前（来源说不清，记成外部改动）和改之后的那两份 + let history = tw_pack::backup::list(&b.dir.path().join("config.yaml")).unwrap(); + let origins: Vec<_> = history.iter().map(|v| v.origin).collect(); assert_eq!( - history - .iter() - .filter(|v| v.origin == tw_config::history::Origin::Ui) - .count(), - 2 + origins, + [ + tw_config::history::Origin::External, + tw_config::history::Origin::Ui + ] ); } diff --git a/crates/tw-control/tests/sync.rs b/crates/tw-control/tests/sync.rs index 88f4365f..12cfb1f9 100644 --- a/crates/tw-control/tests/sync.rs +++ b/crates/tw-control/tests/sync.rs @@ -158,26 +158,6 @@ async fn a_write_that_would_not_parse_never_reaches_the_disk() { assert_eq!(std::fs::read_to_string(mgr.path()).unwrap(), BASE); } -#[tokio::test] -async fn a_rollback_goes_through_the_same_door_as_everything_else() { - // 三条路各写一遍，就会有两条忘了存历史、一条忘了防回环。 - let (_d, mgr, _bus) = setup(); - let v0 = mgr.current().unwrap().version(); - mgr.write(&format!("{BASE}providers: []\n"), None, Origin::Ui) - .await - .unwrap(); - assert!( - std::fs::read_to_string(mgr.path()) - .unwrap() - .contains("providers") - ); - - mgr.rollback(&v0).await.unwrap(); - let back = std::fs::read_to_string(mgr.path()).unwrap(); - assert_eq!(back, BASE); - assert!(back.contains("别动我这句注释"), "注释没回来"); -} - #[tokio::test] async fn every_write_leaves_the_previous_version_in_history() { // 「回滚」这个动作要的是「回到我动它之前」。 @@ -185,15 +165,15 @@ async fn every_write_leaves_the_previous_version_in_history() { mgr.write(&format!("{BASE}providers: []\n"), None, Origin::Ui) .await .unwrap(); - let hist = tw_config::history::list(mgr.path()).unwrap(); + let hist = tw_pack::backup::list(mgr.path()).unwrap(); let texts: Vec = hist .iter() - .map(|v| tw_config::history::read(v).unwrap()) + .map(|v| tw_pack::backup::open(v).unwrap().config_yaml) .collect(); - // 历史里存的钥匙是打码的 + // 备份里存的钥匙是打码的 assert!( texts.contains(&tw_config::control_key::mask(BASE)), - "改之前那一版没进历史：{texts:?}" + "改之前那一版没进备份：{texts:?}" ); } @@ -208,10 +188,10 @@ async fn an_external_edit_also_lands_in_history_so_it_can_be_undone() { .await .unwrap() .expect("该被当成外部改动"); - let hist = tw_config::history::list(mgr.path()).unwrap(); + let hist = tw_pack::backup::list(mgr.path()).unwrap(); assert!( hist.iter().any(|v| v.origin == Origin::External), - "外部改动没进历史" + "外部改动没进备份" ); } diff --git a/crates/tw-gateway/src/plugin/source.rs b/crates/tw-gateway/src/plugin/source.rs index 56e2dc77..76bde169 100644 --- a/crates/tw-gateway/src/plugin/source.rs +++ b/crates/tw-gateway/src/plugin/source.rs @@ -184,6 +184,107 @@ pub fn with_values(m: &Manifest, source: &str) -> Option { Some(out) } +/// 把一份源码拆成**代码**和**数据**，给配置包、同步按这两样分开比较（`tw_pack::model`）。 +/// +/// 数据是出错时怎么办、范围、每个设置的值；代码是完整的源码，manifest 字面量里的数据去掉， +/// 字面量按固定的样子重写（[`literal::write`]）—— 只差数据的两份源码拆出来的代码一字不差， +/// 字面量原来怎么排版都一样。代码本身还是一份能读的源码：数据都是不写时的值。读不出 manifest +/// 是 None +pub fn split_data(source: &str) -> Option<(String, serde_json::Value)> { + let lit = literal::find(source).ok()?; + let values = lit.values()?; + let literal::Data::Object(fields) = &lit.data else { + return None; + }; + let code_fields = fields + .iter() + .filter(|(k, _)| k != "on_error" && k != "match") + .map(|(k, v)| { + let v = match (k.as_str(), v) { + ("settings", literal::Data::Object(specs)) => literal::Data::Object( + specs + .iter() + .map(|(name, spec)| { + let spec = match spec { + literal::Data::Object(f) => literal::Data::Object( + f.iter().filter(|(k, _)| k != "value").cloned().collect(), + ), + other => other.clone(), + }; + (name.clone(), spec) + }) + .collect(), + ), + _ => v.clone(), + }; + (k.clone(), v) + }) + .collect(); + let newline = if source.contains("\r\n") { + "\r\n" + } else { + "\n" + }; + let code = format!( + "{}{}{}", + &source[..lit.span.start], + literal::write(&literal::Data::Object(code_fields), "", newline), + &source[lit.span.end..] + ); + let data = serde_json::json!({ + "on_error": values.on_error.as_str(), + "match": { + "clients": values.scope.clients, + "models": values.scope.models, + "upstreams": values.scope.upstreams, + }, + "settings": values.settings.into_iter().collect::>(), + }); + Some((code, data)) +} + +/// [`split_data`] 反过来：把数据写进源码，**只换 manifest 那一段**。数据和源码声明的对不上 +/// （设置不在、类型不对）是 None +pub fn join_data(source: &str, data: &serde_json::Value) -> Option { + let strings = |v: Option<&serde_json::Value>| -> Option> { + match v { + None | Some(serde_json::Value::Null) => Some(Vec::new()), + Some(serde_json::Value::Array(a)) => { + a.iter().map(|x| x.as_str().map(str::to_string)).collect() + } + _ => None, + } + }; + let on_error = match data.get("on_error").and_then(serde_json::Value::as_str) { + Some(s) => tw_plugin::OnError::from_manifest(s)?, + None => tw_plugin::OnError::Reject, + }; + let m = data.get("match"); + let values = Values { + on_error, + scope: tw_plugin::Scope { + clients: strings(m.and_then(|m| m.get("clients")))?, + models: strings(m.and_then(|m| m.get("models")))?, + upstreams: strings(m.and_then(|m| m.get("upstreams")))?, + }, + settings: match data.get("settings") { + Some(serde_json::Value::Object(o)) => { + o.iter().map(|(k, v)| (k.clone(), v.clone())).collect() + } + None | Some(serde_json::Value::Null) => Vec::new(), + Some(_) => return None, + }, + }; + literal::rewrite(source, &values).ok() +} + +/// 这份源码的插件改不改得了回答里的工具调用（权限有 `reply_tool_calls`），**不编译**，照 +/// 文件里写着的 manifest 判断。读不出来的按改得了算 +pub fn steers_declared(source: &[u8]) -> bool { + display_manifest(source) + .is_none_or(|m| m.permissions.contains(&tw_api::Permission::ReplyToolCalls)) +} + #[cfg(test)] mod tests { use super::*; @@ -332,4 +433,38 @@ mod tests { assert_eq!(patched.on_error, OnError::Skip); assert_eq!(patched.settings[0].value, json!("b")); } + + /// 只差数据的两份源码拆出来代码一样；数据写回去就是原来的数据 + #[test] + fn code_and_data_split_apart_and_join_back() { + let a = SRC; + let b = rewrite( + SRC, + OnError::Skip, + &scope(&["claude-*"]), + &BTreeMap::from([("note".to_string(), SettingValue::String("b".into()))]), + ) + .unwrap(); + let (code_a, data_a) = split_data(a).unwrap(); + let (code_b, data_b) = split_data(&b).unwrap(); + assert_eq!(code_a, code_b, "只差数据，代码就该一样"); + assert_ne!(data_a, data_b); + assert_eq!(data_b["on_error"], "skip"); + assert_eq!(data_b["match"]["models"], json!(["claude-*"])); + assert_eq!(data_b["settings"]["note"], "b"); + // 代码本身读得出 manifest，数据都是不写时的值 + let (_, empty) = split_data(&code_a).unwrap(); + assert_eq!(empty["settings"]["note"], ""); + // 数据写回代码：和原来那一份数据一样 + let back = join_data(&code_a, &data_b).unwrap(); + assert_eq!(split_data(&back).unwrap().1, data_b); + assert!(back.contains("export function onRequest(req) { return req; }")); + // 数据写回原文件：只差数据时原样返回 + assert_eq!(join_data(a, &data_a).unwrap(), a); + // 数据和源码对不上 + assert!(join_data(a, &json!({"settings": {"unknown": 1}})).is_none()); + assert!(split_data("no manifest here").is_none()); + assert!(steers_declared(b"no manifest here")); + assert!(!steers_declared(SRC.as_bytes())); + } } diff --git a/crates/tw-observe/src/bus.rs b/crates/tw-observe/src/bus.rs index feff35b9..840b791d 100644 --- a/crates/tw-observe/src/bus.rs +++ b/crates/tw-observe/src/bus.rs @@ -125,6 +125,8 @@ fn about_the_request(ev: &tw_api::Event) -> bool { | E::ListenChanged { .. } | E::ConfigReloaded { .. } | E::ConfigRejected { .. } + | E::GatewayKeysChanged { .. } + | E::ClientPrefsChanged { .. } | E::EventsDropped { .. } => false, } } diff --git a/crates/tw-pack/Cargo.toml b/crates/tw-pack/Cargo.toml index a2a8625c..d842fd96 100644 --- a/crates/tw-pack/Cargo.toml +++ b/crates/tw-pack/Cargo.toml @@ -13,6 +13,7 @@ description = "Configuration packs (.twconfig): format, encryption, local backup [dependencies] # 配置的形状、校验、按名字改一项（保留注释）都在这里 tw-config = { workspace = true } +tw-api = { workspace = true } tw-engine = { workspace = true } tw-pricing = { workspace = true } tw-types = { workspace = true } @@ -21,6 +22,7 @@ serde = { workspace = true } serde_json = { workspace = true } serde_yaml_ng = { workspace = true } thiserror = { workspace = true } +tracing = { workspace = true } blake3 = { workspace = true } sha2 = { workspace = true } age = { workspace = true } diff --git a/crates/tw-pack/src/backup.rs b/crates/tw-pack/src/backup.rs new file mode 100644 index 00000000..c954f17e --- /dev/null +++ b/crates/tw-pack/src/backup.rs @@ -0,0 +1,445 @@ +//! 本机备份：每次配置变更留一个版本，最近 [`KEEP`] 个。取代以前的修改记录。 +//! +//! # 一版是什么 +//! +//! 一个**完整的配置包**，不加密：配置原文（本机设置、ChatGPT 的登录都在里面，和 config.yaml +//! 本身的保护级别一样），配置里列着的插件的文件，Lite 的通用偏好。**控制面的钥匙不在里面**： +//! 存的是打码的原文 —— 恢复时本机设置本来就留着现在的。 +//! +//! 文件在数据目录的 `history/` 里，名字是 `<毫秒时间戳>-<来源>.twconfig`，时间戳就是这一版 +//! 的标识。目录 0700、文件 0600。 +//! +//! # 什么时候留 +//! +//! 每个写入配置的地方在写之前（[`record_before`]）和生效之后（[`record`]）各问一次。和最新 +//! 那一版内容一样就不留：效果是「每个存在过的版本各一条」，最新那条就是现在跑着的。 +//! +//! - **写之前那一次只看配置原文**：插件的写法是先写文件、再写配置，写之前那一刻磁盘上的插件 +//! 已经是新的了 —— 拿旧的原文配新的插件存一版，那一版从来没有存在过 +//! - **凭据轮换不留**（`Origin::Rotation`）：恢复到一次轮换之前，拿到的是一个已经作废的 +//! token。比较时 OAuth 的 token 也不算：轮换之后的下一次修改不会因此多出一版 +//! - 以前的修改记录（`history/*.yaml`）在第一次留版本时删掉 + +use std::path::{Path, PathBuf}; + +use tw_config::history::Origin; +use tw_types::{Msg, msg}; + +use crate::{Pack, PackError, PackOrigin, Seal, Unseal, disk}; + +/// 本机留多少版 +pub const KEEP: usize = 10; + +/// 备份目录，相对配置文件所在的目录 +pub const DIR: &str = "history"; + +/// 一个备份版本。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Backup { + /// 标识：毫秒时间戳，严格递增 + pub id: String, + pub at_ms: u64, + pub origin: Origin, + pub path: PathBuf, +} + +/// 备份读写失败。**英文只写一遍**：`Display` 就是 [`BackupError::msg`] 的原句。 +#[derive(Debug, thiserror::Error)] +pub enum BackupError { + #[error("{}", self.msg())] + Io { + path: PathBuf, + source: std::io::Error, + }, + #[error("{}", self.msg())] + NotFound { id: String }, + #[error("{}", self.msg())] + Pack(PackError), +} + +impl BackupError { + pub fn msg(&self) -> Msg { + match self { + BackupError::Io { path, source } => msg!( + "backup.io", path = path.display(), detail = source => + "{path} could not be read or written: {detail}" + ), + BackupError::NotFound { id } => msg!( + "backup.not_found", id = id => + "There is no backup version {id}." + ), + BackupError::Pack(e) => e.msg(), + } + } +} + +fn io(path: &Path) -> impl FnOnce(std::io::Error) -> BackupError + '_ { + move |source| BackupError::Io { + path: path.to_path_buf(), + source, + } +} + +pub fn dir(config_path: &Path) -> PathBuf { + disk::dir_of(config_path).join(DIR) +} + +/// 全部版本，**从旧到新**。目录还不在是空的 +pub fn list(config_path: &Path) -> Result, BackupError> { + let d = dir(config_path); + let rd = match std::fs::read_dir(&d) { + Ok(r) => r, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(e) => return Err(io(&d)(e)), + }; + let mut out: Vec = rd + .flatten() + .filter_map(|e| { + let name = e.file_name().to_string_lossy().to_string(); + let stem = name.strip_suffix(&format!(".{}", crate::EXTENSION))?; + let (at, origin) = stem.split_once('-')?; + let at_ms: u64 = at.parse().ok()?; + Some(Backup { + id: at_ms.to_string(), + at_ms, + origin: Origin::parse(origin), + path: e.path(), + }) + }) + .collect(); + // 按时间戳排，不按名字的字典序：位数会变 + out.sort_by_key(|b| b.at_ms); + Ok(out) +} + +/// 按标识找一版 +pub fn find(config_path: &Path, id: &str) -> Result { + list(config_path)? + .into_iter() + .find(|b| b.id == id) + .ok_or_else(|| BackupError::NotFound { id: id.to_string() }) +} + +/// 读出一版 +pub fn open(b: &Backup) -> Result { + let bytes = std::fs::read(&b.path).map_err(io(&b.path))?; + crate::read(&bytes, Unseal::None).map_err(BackupError::Pack) +} + +/// 生效之后：此刻磁盘上的样子（`text` 加上它列着的插件、通用偏好）和最新那一版不一样就 +/// 留一版。返回留下的那一版 +pub fn record( + config_path: &Path, + text: &str, + origin: Origin, +) -> Result, BackupError> { + if origin == Origin::Rotation { + return Ok(None); + } + let pack = disk::current(config_path, text, PackOrigin::from(origin)); + let all = list(config_path)?; + if let Some(last) = all.last() + && let Ok(prev) = open(last) + && fingerprint(&prev) == fingerprint(&masked(&pack)) + { + return Ok(None); + } + save(config_path, &pack, origin, all.last()).map(Some) +} + +/// 写之前：`text`（磁盘上此刻的配置原文）和最新那一版的原文不一样才留一版（外部改动没来得 +/// 及留、第一次运行）。**只比原文**，理由见模块说明。 +/// +/// 留下的这一版**来源记成外部改动**，不是 `origin`：`origin` 说的是接下来这次写，而写之前的 +/// 这个样子不是它造成的 —— 说不清是谁改的，最可能就是有人直接改了文件 +pub fn record_before( + config_path: &Path, + text: &str, + origin: Origin, +) -> Result, BackupError> { + if origin == Origin::Rotation { + return Ok(None); + } + let all = list(config_path)?; + if let Some(last) = all.last() + && let Ok(prev) = open(last) + && text_fingerprint(&prev.config_yaml) + == text_fingerprint(&tw_config::control_key::mask(text)) + { + return Ok(None); + } + let pack = disk::current(config_path, text, PackOrigin::External); + save(config_path, &pack, Origin::External, all.last()).map(Some) +} + +fn masked(pack: &Pack) -> Pack { + Pack { + config_yaml: tw_config::control_key::mask(&pack.config_yaml), + ..pack.clone() + } +} + +fn save( + config_path: &Path, + pack: &Pack, + origin: Origin, + last: Option<&Backup>, +) -> Result { + let d = dir(config_path); + tw_config::private_dir::create(&d).map_err(io(&d))?; + // **时间戳严格递增**：同一毫秒里连着留两版时，「哪一版更新」不能听 read_dir 的 + let now = crate::now_ms(); + let at_ms = match last { + Some(l) if now <= l.at_ms => l.at_ms + 1, + _ => now, + }; + let mut pack = masked(pack); + pack.manifest.created_at_ms = at_ms; + let bytes = crate::write(&pack, Seal::None).map_err(BackupError::Pack)?; + let path = d.join(format!("{at_ms}-{}.{}", origin.slug(), crate::EXTENSION)); + disk::write_private(&path, &bytes).map_err(io(&path))?; + prune(config_path)?; + Ok(Backup { + id: at_ms.to_string(), + at_ms, + origin, + path, + }) +} + +/// 超出 [`KEEP`] 的老版本删掉，以前的修改记录（`*.yaml`）也删掉 +fn prune(config_path: &Path) -> Result<(), BackupError> { + let all = list(config_path)?; + if all.len() > KEEP { + for b in &all[..all.len() - KEEP] { + let _ = std::fs::remove_file(&b.path); + } + } + let d = dir(config_path); + if let Ok(rd) = std::fs::read_dir(&d) { + for e in rd.flatten() { + let name = e.file_name().to_string_lossy().to_string(); + let legacy = name.ends_with(".yaml") + && name + .split('-') + .next() + .is_some_and(|t| t.parse::().is_ok()); + if legacy { + let _ = std::fs::remove_file(e.path()); + } + } + } + Ok(()) +} + +/// 比较两版用的：原文里 OAuth 的 token 和过期时间去掉（轮换不算改动），加上插件、偏好 +fn fingerprint(pack: &Pack) -> String { + let mut h = blake3::Hasher::new(); + h.update(text_fingerprint(&pack.config_yaml).as_bytes()); + for (id, f) in &pack.plugins { + h.update(id.as_bytes()); + h.update(&[0]); + h.update(&f.script); + h.update(&[1]); + if let Some(a) = &f.approved { + h.update(a); + } + h.update(&[2]); + } + if let Some(p) = &pack.client_prefs { + h.update(p.to_string().as_bytes()); + } + h.finalize().to_hex().to_string() +} + +fn text_fingerprint(text: &str) -> String { + let Ok(mut doc) = serde_yaml_ng::from_str::(text) else { + return blake3::hash(text.as_bytes()).to_hex().to_string(); + }; + if let Some(list) = doc + .get_mut("providers") + .and_then(serde_yaml_ng::Value::as_sequence_mut) + { + for p in list { + if let Some(serde_yaml_ng::Value::Mapping(o)) = p.get_mut("oauth") { + for k in ["access", "refresh", "expires_at"] { + o.remove(k); + } + } + } + } + // 只比内容：注释和排版变了也算变了（那是用户改的），所以原文也要算进去 —— 去掉 token + // 那几行之外的原文 + let mut lines = String::new(); + for l in text.lines() { + let t = l.trim_start(); + if !(t.starts_with("access:") || t.starts_with("refresh:") || t.starts_with("expires_at:")) + { + lines.push_str(l); + lines.push('\n'); + } + } + let canon = serde_json::to_string(&doc).unwrap_or_default(); + blake3::hash(format!("{canon}\n{lines}").as_bytes()) + .to_hex() + .to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + const KEY: &str = "c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00"; + + fn cfg(port: u16) -> String { + format!( + "version: 1\nlisten:\n control:\n key: {KEY}\n gateway:\n port: {port}\nclients:\n - name: default\n key: tw-a\n" + ) + } + + fn setup() -> (tempfile::TempDir, PathBuf) { + let d = tempfile::tempdir().unwrap(); + let p = d.path().join("config.yaml"); + (d, p) + } + + #[test] + fn each_distinct_version_is_kept_once_and_reads_back_whole() { + let (_d, p) = setup(); + let a = record(&p, &cfg(1), Origin::Ui).unwrap().unwrap(); + assert!( + record(&p, &cfg(1), Origin::Ui).unwrap().is_none(), + "same content twice" + ); + assert!(record_before(&p, &cfg(1), Origin::Ui).unwrap().is_none()); + let b = record(&p, &cfg(2), Origin::Cli).unwrap().unwrap(); + assert!(b.at_ms > a.at_ms); + let all = list(&p).unwrap(); + assert_eq!(all, vec![a.clone(), b.clone()]); + let back = open(&a).unwrap(); + assert!(back.config_yaml.contains("port: 1")); + // 钥匙不进备份 + assert!(!back.config_yaml.contains(KEY), "{}", back.config_yaml); + assert_eq!(back.manifest.origin, PackOrigin::Ui); + assert_eq!(find(&p, &b.id).unwrap(), b); + assert!(matches!( + find(&p, "nope"), + Err(BackupError::NotFound { .. }) + )); + } + + #[test] + fn plugins_and_preferences_are_part_of_a_version() { + let (_d, p) = setup(); + let text = format!( + "{}plugins:\n - id: a\n file: plugins/a.js\n sha256: {}\n", + cfg(1), + "0".repeat(64) + ); + let files = crate::PluginFiles { + script: b"v1".to_vec(), + approved: None, + }; + disk::write_plugins( + &p, + &Default::default(), + &[("a".to_string(), files)].into_iter().collect(), + ) + .unwrap(); + record(&p, &text, Origin::Ui).unwrap().unwrap(); + // 只改了插件文件：又一版 + disk::write_plugins( + &p, + &Default::default(), + &[( + "a".to_string(), + crate::PluginFiles { + script: b"v2".to_vec(), + approved: None, + }, + )] + .into_iter() + .collect(), + ) + .unwrap(); + // 写之前那一次只看原文：原文没变，不留 + assert!(record_before(&p, &text, Origin::Ui).unwrap().is_none()); + let v2 = record(&p, &text, Origin::Ui).unwrap().unwrap(); + assert_eq!(open(&v2).unwrap().plugins["a"].script, b"v2"); + // 偏好变了：又一版 + disk::write_prefs(&p, Some(&serde_json::json!({"language": "en"}))).unwrap(); + let v3 = record(&p, &text, Origin::Ui).unwrap().unwrap(); + assert_eq!( + open(&v3).unwrap().client_prefs, + Some(serde_json::json!({"language": "en"})) + ); + } + + #[test] + fn a_rotation_leaves_no_version_and_does_not_count_as_a_change() { + let (_d, p) = setup(); + let with = |rt: &str| { + format!( + "{}providers:\n - name: c\n base_url: https://x\n oauth:\n refresh: {rt}\n endpoint: https://t\n", + cfg(1) + ) + }; + record(&p, &with("rt-1"), Origin::Ui).unwrap().unwrap(); + assert!( + record(&p, &with("rt-2"), Origin::Rotation) + .unwrap() + .is_none() + ); + // 轮换之后的下一次写：写之前那一次不因为 token 变了多留一版 + assert!( + record_before(&p, &with("rt-2"), Origin::Ui) + .unwrap() + .is_none() + ); + assert_eq!(list(&p).unwrap().len(), 1); + } + + #[test] + fn only_the_latest_versions_stay_and_the_old_history_goes() { + let (_d, p) = setup(); + let d = dir(&p); + std::fs::create_dir_all(&d).unwrap(); + std::fs::write(d.join("1700000000000-ui-abcdef012345.yaml"), "a: 1\n").unwrap(); + std::fs::write(d.join("README.txt"), "hi").unwrap(); + for i in 0..KEEP + 5 { + record(&p, &cfg(i as u16 + 1), Origin::Ui).unwrap().unwrap(); + } + let all = list(&p).unwrap(); + assert_eq!(all.len(), KEEP); + assert!( + open(all.last().unwrap()) + .unwrap() + .config_yaml + .contains(&format!("port: {}", KEEP + 5)) + ); + assert!(!d.join("1700000000000-ui-abcdef012345.yaml").exists()); + assert!( + d.join("README.txt").exists(), + "files that are not ours stay" + ); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(&all[0].path) + .unwrap() + .permissions() + .mode() + & 0o777; + assert_eq!(mode, 0o600); + } + } + + #[test] + fn versions_made_in_the_same_millisecond_keep_their_order() { + let (_d, p) = setup(); + let stamps: Vec = (0..30) + .map(|i| record(&p, &cfg(i + 1), Origin::Ui).unwrap().unwrap().at_ms) + .collect(); + assert!(stamps.windows(2).all(|w| w[0] < w[1]), "{stamps:?}"); + } +} diff --git a/crates/tw-pack/src/disk.rs b/crates/tw-pack/src/disk.rs new file mode 100644 index 00000000..6dbc74a9 --- /dev/null +++ b/crates/tw-pack/src/disk.rs @@ -0,0 +1,303 @@ +//! 数据目录里和配置包有关的几样文件：插件、Lite 的通用偏好、固定密钥。 +//! +//! 路径都相对配置文件所在的目录，和插件文件一样（`tw_config::plugins`）：远程 core 也是， +//! 文件在 core 那台机器上。**只给自己看**：目录 0700、文件 0600（unix；Windows 上数据目录 +//! 的 ACL 由 `tw_config::private_dir` 管）。 + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; + +use crate::{Manifest, Pack, PackKind, PackOrigin, PluginFiles}; + +/// Lite 的通用偏好存在哪个文件里 +pub const PREFS_FILE: &str = "client-prefs.json"; + +/// 固定密钥（本机接管的客户端在用的网关密钥）存在哪个文件里 +pub const PINNED_FILE: &str = "pinned-keys.json"; + +/// 配置文件所在的目录 +pub fn dir_of(config_path: &Path) -> PathBuf { + match config_path.parent() { + Some(d) if !d.as_os_str().is_empty() => d.to_path_buf(), + _ => PathBuf::from("."), + } +} + +pub fn prefs_path(config_path: &Path) -> PathBuf { + dir_of(config_path).join(PREFS_FILE) +} + +pub fn pinned_path(config_path: &Path) -> PathBuf { + dir_of(config_path).join(PINNED_FILE) +} + +/// 存着的通用偏好。没有、读不成 JSON 都是 None：偏好丢了不该挡住别的 +pub fn read_prefs(config_path: &Path) -> Option { + let bytes = std::fs::read(prefs_path(config_path)).ok()?; + serde_json::from_slice(&bytes).ok() +} + +/// 存下通用偏好。None 是删掉 +pub fn write_prefs(config_path: &Path, prefs: Option<&serde_json::Value>) -> std::io::Result<()> { + let path = prefs_path(config_path); + match prefs { + Some(p) => { + let bytes = serde_json::to_vec_pretty(p).map_err(std::io::Error::other)?; + write_private(&path, &bytes) + } + None => match std::fs::remove_file(&path) { + Err(e) if e.kind() != std::io::ErrorKind::NotFound => Err(e), + _ => Ok(()), + }, + } +} + +/// 存着的固定密钥。没有、读不懂都是空的 +pub fn read_pinned(config_path: &Path) -> BTreeSet { + #[derive(serde::Deserialize)] + struct File { + names: BTreeSet, + } + std::fs::read(pinned_path(config_path)) + .ok() + .and_then(|b| serde_json::from_slice::(&b).ok()) + .map(|f| f.names) + .unwrap_or_default() +} + +pub fn write_pinned(config_path: &Path, names: &BTreeSet) -> std::io::Result<()> { + let bytes = serde_json::to_vec_pretty(&serde_json::json!({ "names": names })) + .map_err(std::io::Error::other)?; + write_private(&pinned_path(config_path), &bytes) +} + +/// 配置原文里列着的插件 id，按顺序。读不成 YAML 是空的 +pub fn plugin_ids(text: &str) -> Vec { + let Ok(doc) = serde_yaml_ng::from_str::(text) else { + return Vec::new(); + }; + doc.get(tw_config::plugins::DIR) + .and_then(serde_yaml_ng::Value::as_sequence) + .into_iter() + .flatten() + .filter_map(|p| p.get("id").and_then(serde_yaml_ng::Value::as_str)) + .filter(|id| tw_config::plugins::valid_id(id)) + .map(str::to_string) + .collect() +} + +/// 一个插件在磁盘上的两份文件。脚本不在就是 None +pub fn read_plugin(config_path: &Path, id: &str) -> Option { + let dir = dir_of(config_path); + let script = std::fs::read(tw_config::plugins::file_path(&dir, id)).ok()?; + let approved = std::fs::read(tw_config::plugins::approved_path(&dir, id)).ok(); + Some(PluginFiles { script, approved }) +} + +/// 此刻的样子配成一个包：`text` 是配置原文，插件是它列着的那几个在磁盘上的文件，加上 +/// 存着的通用偏好。**原文原样放进去**：要拿去哪儿（备份、导出）由调用方决定剔掉什么 +pub fn current(config_path: &Path, text: &str, origin: PackOrigin) -> Pack { + let plugins: BTreeMap = plugin_ids(text) + .into_iter() + .filter_map(|id| read_plugin(config_path, &id).map(|f| (id, f))) + .collect(); + Pack { + manifest: Manifest::new(PackKind::Full, origin), + config_yaml: text.to_string(), + plugins, + client_prefs: read_prefs(config_path), + } +} + +/// 写之前的样子。配置没写成时照它还原 +#[derive(Debug, Default)] +pub struct Undo(Vec<(PathBuf, Option>)>); + +impl Undo { + /// 还原。哪个文件还原不了只记一行：手里已经有一个更要紧的错误 + pub fn restore(self) { + for (path, before) in self.0.into_iter().rev() { + let r = match before { + Some(b) => write_private(&path, &b), + None => match std::fs::remove_file(&path) { + Err(e) if e.kind() != std::io::ErrorKind::NotFound => Err(e), + _ => Ok(()), + }, + }; + if let Err(e) = r { + tracing::warn!(path = %path.display(), "a file could not be put back: {e}"); + } + } + } +} + +/// 插件文件换成 `after` 的样子：有变化的写下去，`before` 里有、`after` 里没有的删掉 +/// （连同底稿）。`after` 里没有底稿的，原来的底稿删掉。返回还原用的 +pub fn write_plugins( + config_path: &Path, + before: &BTreeMap, + after: &BTreeMap, +) -> std::io::Result { + let dir = dir_of(config_path); + let plugins = tw_config::plugins::dir_in(&dir); + let approved = plugins.join(tw_config::plugins::APPROVED_DIR); + let mut undo = Undo::default(); + let put = |path: PathBuf, bytes: Option<&[u8]>, undo: &mut Undo| -> std::io::Result<()> { + let old = match std::fs::read(&path) { + Ok(b) => Some(b), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => None, + Err(e) => return Err(e), + }; + if old.as_deref() == bytes { + return Ok(()); + } + match bytes { + Some(b) => { + for d in [&plugins, &approved] { + tw_config::private_dir::create(d)?; + } + write_private(&path, b)?; + } + None => std::fs::remove_file(&path)?, + } + undo.0.push((path, old)); + Ok(()) + }; + let result = (|| { + for (id, files) in after { + put( + tw_config::plugins::file_path(&dir, id), + Some(&files.script), + &mut undo, + )?; + put( + tw_config::plugins::approved_path(&dir, id), + files.approved.as_deref(), + &mut undo, + )?; + } + for id in before.keys().filter(|id| !after.contains_key(*id)) { + put(tw_config::plugins::file_path(&dir, id), None, &mut undo)?; + put(tw_config::plugins::approved_path(&dir, id), None, &mut undo)?; + } + Ok(()) + })(); + match result { + Ok(()) => Ok(undo), + Err(e) => { + undo.restore(); + Err(e) + } + } +} + +/// 原子地写一个只给自己看的文件：**建的那一刻就是 0600**，写完再改名过去 +pub fn write_private(path: &Path, bytes: &[u8]) -> std::io::Result<()> { + use std::io::Write; + if let Some(dir) = path.parent() + && !dir.as_os_str().is_empty() + { + tw_config::private_dir::create(dir)?; + } + let tmp = path.with_extension(format!("tmp{}", std::process::id())); + let _ = std::fs::remove_file(&tmp); + let mut opts = std::fs::OpenOptions::new(); + opts.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + opts.mode(0o600); + } + let written = opts.open(&tmp).and_then(|mut f| { + f.write_all(bytes)?; + f.sync_all() + }); + if let Err(e) = written.and_then(|()| std::fs::rename(&tmp, path)) { + let _ = std::fs::remove_file(&tmp); + return Err(e); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn files(s: &str) -> PluginFiles { + PluginFiles { + script: s.as_bytes().to_vec(), + approved: Some(s.as_bytes().to_vec()), + } + } + + #[test] + fn plugin_files_are_swapped_in_and_put_back_on_undo() { + let d = tempfile::tempdir().unwrap(); + let cfg = d.path().join("config.yaml"); + let before = BTreeMap::from([ + ("a".to_string(), files("a1")), + ("b".to_string(), files("b1")), + ]); + write_plugins(&cfg, &BTreeMap::new(), &before).unwrap(); + assert_eq!(read_plugin(&cfg, "a").unwrap(), files("a1")); + let after = BTreeMap::from([ + ("a".to_string(), files("a2")), + ("c".to_string(), files("c1")), + ]); + let undo = write_plugins(&cfg, &before, &after).unwrap(); + assert_eq!(read_plugin(&cfg, "a").unwrap(), files("a2")); + assert!(read_plugin(&cfg, "b").is_none(), "removed plugins go"); + assert_eq!(read_plugin(&cfg, "c").unwrap(), files("c1")); + undo.restore(); + assert_eq!(read_plugin(&cfg, "a").unwrap(), files("a1")); + assert_eq!(read_plugin(&cfg, "b").unwrap(), files("b1")); + assert!(read_plugin(&cfg, "c").is_none()); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let p = tw_config::plugins::file_path(d.path(), "a"); + assert_eq!( + std::fs::metadata(p).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } + } + + #[test] + fn prefs_and_pinned_keys_round_trip_and_a_bad_file_reads_as_none() { + let d = tempfile::tempdir().unwrap(); + let cfg = d.path().join("config.yaml"); + assert_eq!(read_prefs(&cfg), None); + let p = serde_json::json!({"language": "zh"}); + write_prefs(&cfg, Some(&p)).unwrap(); + assert_eq!(read_prefs(&cfg), Some(p)); + write_prefs(&cfg, None).unwrap(); + assert_eq!(read_prefs(&cfg), None); + write_prefs(&cfg, None).unwrap(); + std::fs::write(prefs_path(&cfg), b"{not json").unwrap(); + assert_eq!(read_prefs(&cfg), None); + + let names: BTreeSet = ["codex".to_string()].into_iter().collect(); + write_pinned(&cfg, &names).unwrap(); + assert_eq!(read_pinned(&cfg), names); + } + + #[test] + fn the_current_state_is_the_config_and_the_plugins_it_lists() { + let d = tempfile::tempdir().unwrap(); + let cfg = d.path().join("config.yaml"); + write_plugins( + &cfg, + &BTreeMap::new(), + &BTreeMap::from([ + ("a".to_string(), files("a")), + ("stray".to_string(), files("s")), + ]), + ) + .unwrap(); + let text = "version: 1\nplugins:\n - { id: a, file: plugins/a.js, sha256: x }\n - { id: gone, file: plugins/gone.js, sha256: x }\n"; + let p = current(&cfg, text, PackOrigin::Ui); + assert_eq!(p.plugins.keys().collect::>(), ["a"]); + assert_eq!(p.config_yaml, text); + } +} diff --git a/crates/tw-pack/src/export.rs b/crates/tw-pack/src/export.rs new file mode 100644 index 00000000..17d50c47 --- /dev/null +++ b/crates/tw-pack/src/export.rs @@ -0,0 +1,227 @@ +//! 导出：把本机的配置（全部，或者选的几项连同它们引用的）做成一个配置包。 +//! +//! 包里**没有本机设置**（[`crate::model::LOCAL_SECTIONS`]、ChatGPT 的登录），配置按项目 +//! 重新生成（[`crate::model::materialize`] 落在一份空配置上）。不带密钥时，密钥类的值换成 +//! 空字符串：写成 `${变量名}` 的不是密钥，照样带着。 + +use std::collections::BTreeSet; + +use serde_json::Value as Json; + +use crate::model::{self, ItemId, ItemKind, ModelError, PluginCodec, Snapshot, unit}; +use crate::{Manifest, Pack, PackKind, PackOrigin}; + +/// 导出什么、怎么导出。 +#[derive(Debug, Clone)] +pub struct Options { + /// None 是全部配置；否则是这几项，连同它们引用的 + pub scope: Option>, + /// 带上密钥 + pub secrets: bool, + /// 换掉通用偏好（经远程连接导出时 Lite 给的那一份）。None 是用包里本来的 + pub client_prefs: Option, + pub origin: PackOrigin, +} + +/// 导出的结果。 +#[derive(Debug, Clone)] +pub struct Exported { + pub pack: Pack, + /// 包里有哪些项目（随附的也在），顺序项目不算 + pub included: Vec, + /// 登录没有随包带走的上游（ChatGPT 账号） + pub omitted_logins: Vec, +} + +/// 导出。`local` 是本机此刻的样子（[`crate::disk::current`]） +pub fn export(local: &Pack, opts: Options) -> Result { + export_with(local, opts, model::codec()) +} + +pub fn export_with( + local: &Pack, + opts: Options, + codec: &dyn PluginCodec, +) -> Result { + let mut snap = model::snapshot_with(local, codec)?; + let prefs_id = ItemId::new(ItemKind::ClientPrefs, ""); + if let Some(p) = &opts.client_prefs { + let mut units = Default::default(); + model::json_flatten(p, &mut units); + snap.items.remove(&prefs_id); + if !units.is_empty() { + snap.items.insert( + prefs_id, + model::Item { + units, + ..Default::default() + }, + ); + } + } + if let Some(roots) = &opts.scope { + let keep = snap.closure(roots); + snap.retain(&keep); + } + if !opts.secrets { + strip_secrets(&mut snap); + } + let cfg = model::parse(&local.config_yaml)?; + let included: Vec = snap + .items + .keys() + .filter(|id| !id.kind.is_order()) + .cloned() + .collect(); + let picked: BTreeSet<&str> = included + .iter() + .filter(|id| id.kind == ItemKind::Provider) + .map(|id| id.name.as_str()) + .collect(); + let omitted_logins = cfg + .providers + .iter() + .filter(|p| picked.contains(p.name.as_str()) && model::is_chatgpt(p)) + .map(|p| p.name.clone()) + .collect(); + let kind = if opts.scope.is_some() { + PackKind::Partial + } else { + PackKind::Full + }; + let mut manifest = Manifest::new(kind, opts.origin); + manifest.secrets = opts.secrets; + let base = Pack { + manifest, + config_yaml: format!("version: {}\n", tw_config::SCHEMA_VERSION), + plugins: Default::default(), + client_prefs: None, + }; + let pack = model::materialize_for_export(&base, &snap, codec)?; + Ok(Exported { + pack, + included, + omitted_logins, + }) +} + +/// 不是 `${变量名}` 的就是密钥 +fn is_literal(v: &Json) -> bool { + v.as_str() + .is_some_and(|s| !s.is_empty() && !s.contains("${")) +} + +fn blank(at: Option<&mut Json>) { + if let Some(v) = at + && is_literal(v) + { + *v = Json::String(String::new()); + } +} + +/// 去掉密钥：上游的密钥、请求头的值、AWS 访问密钥、OAuth 的 refresh token 和客户端密钥， +/// 代理的密码，网关密钥的值。换成空字符串，导入的一方据此知道这里本来有一个 +pub fn strip_secrets(snap: &mut Snapshot) { + for (id, item) in snap.items.iter_mut() { + match id.kind { + ItemKind::Provider => { + let Some(conn) = item.units.get_mut(unit::CONN) else { + continue; + }; + blank(conn.get_mut("key")); + if let Some(Json::Array(headers)) = conn.get_mut("headers") { + for h in headers { + blank(h.get_mut(1)); + } + } + if let Some(aws) = conn.get_mut("aws") { + for k in ["access_key_id", "secret_access_key", "session_token"] { + blank(aws.get_mut(k)); + } + } + if let Some(Json::Object(o)) = conn.get_mut("oauth") { + o.remove("access"); + o.remove("expires_at"); + blank(o.get_mut("refresh")); + blank(o.get_mut("client_secret")); + } + } + ItemKind::Proxy => { + if let Some(auth) = item + .units + .get_mut(unit::ALL) + .and_then(|a| a.get_mut("auth")) + { + blank(auth.get_mut("pass")); + } + } + ItemKind::Key => { + blank(item.units.get_mut(unit::ALL).and_then(|a| a.get_mut("key"))); + } + _ => {} + } + } +} + +/// 一项里有没有被去掉的密钥（空字符串） +pub fn missing_secret(id: &ItemId, item: &model::Item) -> bool { + let empty = |v: Option<&Json>| v.and_then(Json::as_str) == Some(""); + match id.kind { + ItemKind::Provider => { + let Some(conn) = item.units.get(unit::CONN) else { + return false; + }; + empty(conn.get("key")) + || conn + .get("headers") + .and_then(Json::as_array) + .is_some_and(|hs| hs.iter().any(|h| empty(h.get(1)))) + || conn.get("aws").is_some_and(|aws| { + ["access_key_id", "secret_access_key", "session_token"] + .iter() + .any(|k| empty(aws.get(*k))) + }) + || conn + .get("oauth") + .is_some_and(|o| empty(o.get("refresh")) || empty(o.get("client_secret"))) + } + ItemKind::Key => empty(item.units.get(unit::ALL).and_then(|a| a.get("key"))), + _ => false, + } +} + +/// 导出文件的建议名字：`ThinkWatch-2026-10-18.twconfig`（UTC 的日期） +pub fn file_name(created_at_ms: u64) -> String { + let (y, m, d) = civil(created_at_ms / 86_400_000); + format!("ThinkWatch-{y:04}-{m:02}-{d:02}.{}", crate::EXTENSION) +} + +/// 1970-01-01 起的第几天 → 年月日（公历） +fn civil(days: u64) -> (i64, u32, u32) { + let z = days as i64 + 719_468; + let era = z.div_euclid(146_097); + let doe = z.rem_euclid(146_097); + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = (doy - (153 * mp + 2) / 5 + 1) as u32; + let m = if mp < 10 { mp + 3 } else { mp - 9 } as u32; + let y = yoe + era * 400 + i64::from(m <= 2); + (y, m, d) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_file_name_carries_the_date() { + assert_eq!(file_name(0), "ThinkWatch-1970-01-01.twconfig"); + // 2026-10-18 09:30 UTC + assert_eq!( + file_name(1_792_315_800_000), + "ThinkWatch-2026-10-18.twconfig" + ); + assert_eq!(file_name(951_782_400_000), "ThinkWatch-2000-02-29.twconfig"); + } +} diff --git a/crates/tw-pack/src/import.rs b/crates/tw-pack/src/import.rs new file mode 100644 index 00000000..646f8280 --- /dev/null +++ b/crates/tw-pack/src/import.rs @@ -0,0 +1,589 @@ +//! 导入：把配置包里的项目逐项加进本机的配置。 +//! +//! # 规则 +//! +//! - 本机没有的直接导入；同名且内容不同的是冲突，由人选覆盖、改名导入或跳过。全局设置 +//! 的每一节、通用偏好是一项：本机还是默认值（没有这一项）时直接采用，否则不同就是冲突， +//! 只能覆盖或跳过 +//! - 本机有、包里没有的不动。整份替换不是导入，是恢复 +//! - 导入进来的插件停用着；包里不带密钥的上游停用着（等填上密钥），网关密钥重新生成； +//! ChatGPT 账号上游停用着（等登录）—— 本机同名的那一家已经登录着、选了覆盖的除外，登录 +//! 留着本机的 +//! - 改名导入：包里引用它的路由、别名、分组跟着改用新名字（[`crate::model::rename`]） +//! - 不能导入（错误）：引用了本机和包里都没有的东西、本机没有它要的 AWS profile、插件文件 +//! 和配置里批准的哈希对不上、网关密钥的值和本机另一把相同、或者导进来配置就过不了校验 + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::Value as Json; +use tw_types::{Msg, msg}; + +use crate::model::{ + self, Item, ItemId, ItemKind, ModelError, PluginCodec, Snapshot, free_name, global, rename, + unit, +}; +use crate::{Pack, export}; + +/// 包里的一项和本机比。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Status { + New, + Same, + Conflict, + Error, +} + +/// 导入时要知道的一件事。 +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Flag { + /// 包里不带密钥 + MissingKey, + /// ChatGPT 账号上游，登录不随包带 + NeedsLogin, + /// 插件导入为停用 + PluginDisabled, +} + +/// 预览里的一行。 +#[derive(Debug, Clone, PartialEq)] +pub struct Row { + pub item: ItemId, + pub status: Status, + /// 全局设置、通用偏好：冲突只能覆盖或跳过 + pub global: bool, + pub flags: Vec, + /// 不同的合并单位 + pub differs: Vec, + pub suggested_name: Option, + pub error: Option, + /// 它引用的、包里也有的项目 + pub refs: Vec, +} + +/// 对一项怎么处理。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Action { + Import, + Overwrite, + Rename(String), + Skip, +} + +#[derive(Debug, Clone)] +pub struct Decision { + pub item: ItemId, + pub action: Action, +} + +/// 本机的情况：外面才知道的几件事。 +pub struct Context<'a> { + /// 这台机器的 AWS 凭证文件里有没有这个 profile + pub has_aws_profile: &'a dyn Fn(&str) -> bool, + /// 新的网关密钥（包里不带密钥时重新生成） + pub new_key: &'a dyn Fn() -> String, + /// 通用偏好要不要导入（经远程连接导入时偏好由 Lite 自己应用，core 不写） + pub with_prefs: bool, +} + +/// 导入没成。**英文只写一遍**：`Display` 就是 [`ImportError::msg`] 的原句。 +#[derive(Debug, Clone, thiserror::Error)] +pub enum ImportError { + #[error("{}", self.msg())] + Model(ModelError), + /// 这一项导入不了（预览里的错误那一行） + #[error("{}", self.msg())] + Item(ItemId, Box), + /// 本机已经有同名的：导入要选覆盖或改名 + #[error("{}", self.msg())] + Exists(ItemId), + #[error("{}", self.msg())] + NameTaken(ItemId, String), + #[error("{}", self.msg())] + NotInPack(ItemId), + /// 导进来之后配置过不了校验 + #[error("{}", self.msg())] + Rejected(Box), +} + +impl From for ImportError { + fn from(e: ModelError) -> Self { + ImportError::Model(e) + } +} + +impl ImportError { + pub fn msg(&self) -> Msg { + match self { + ImportError::Model(e) => e.msg(), + ImportError::Item(_, m) => (**m).clone(), + ImportError::Exists(id) => msg!( + "import.exists", kind = id.kind.slug(), name = &id.name => + "There is already a {kind} named `{name}` here; choose to overwrite it, import it \ + under another name, or skip it." + ), + ImportError::NameTaken(id, name) => msg!( + "import.name_taken", kind = id.kind.slug(), name = name => + "There is already a {kind} named `{name}`." + ), + ImportError::NotInPack(id) => msg!( + "import.not_in_pack", kind = id.kind.slug(), name = &id.name => + "The configuration pack has no {kind} named `{name}`." + ), + ImportError::Rejected(r) => r.msg(), + } + } +} + +fn missing_ref(id: &ItemId, target: &ItemId) -> Msg { + msg!( + "import.missing_reference", kind = id.kind.slug(), name = &id.name, + target_kind = target.kind.slug(), target = &target.name => + "{kind} `{name}` refers to {target_kind} `{target}`, which is neither here nor in the \ + configuration pack." + ) +} + +fn skipped_ref(id: &ItemId, target: &ItemId) -> Msg { + msg!( + "import.skipped_reference", kind = id.kind.slug(), name = &id.name, + target_kind = target.kind.slug(), target = &target.name => + "{kind} `{name}` refers to {target_kind} `{target}`, which is skipped." + ) +} + +/// 全局设置和通用偏好：冲突只能覆盖或跳过 +fn is_global(kind: ItemKind) -> bool { + matches!(kind, ItemKind::Global | ItemKind::ClientPrefs) +} + +/// 预览。**什么都不写** +pub fn preview(local: &Pack, incoming: &Pack, ctx: &Context) -> Result, ImportError> { + preview_with(local, incoming, ctx, model::codec()) +} + +pub fn preview_with( + local: &Pack, + incoming: &Pack, + ctx: &Context, + codec: &dyn PluginCodec, +) -> Result, ImportError> { + let l = model::snapshot_with(local, codec)?; + let i = model::snapshot_with(incoming, codec)?; + let both = union(&l, &i); + let mut rows = Vec::new(); + for (id, item) in &i.items { + if id.kind.is_order() || (id.kind == ItemKind::ClientPrefs && !ctx.with_prefs) { + continue; + } + let global = is_global(id.kind); + let mut row = Row { + item: id.clone(), + status: Status::New, + global, + flags: flags(id, item, incoming, &l), + differs: Vec::new(), + suggested_name: None, + error: None, + refs: both.items[id] + .refs + .iter() + .filter(|r| i.items.contains_key(*r)) + .cloned() + .collect(), + }; + if let Some(mine) = l.items.get(id) { + if mine.units == item.units { + row.status = Status::Same; + } else { + row.status = Status::Conflict; + row.differs = differing(&mine.units, &item.units); + if !global { + row.suggested_name = Some(free_name(&both, id.kind, &suggest_base(id))); + } + } + } + if row.status != Status::Same { + row.error = problem(id, item, local, incoming, &l, &i, &both, ctx, codec); + if row.error.is_some() { + row.status = Status::Error; + } + } + rows.push(row); + } + Ok(rows) +} + +/// 改名导入时缺省名字的底：自定义安全规则是完整的项目名（`content/规则`） +fn suggest_base(id: &ItemId) -> String { + id.name.clone() +} + +fn differing(a: &BTreeMap, b: &BTreeMap) -> Vec { + let keys: BTreeSet<&String> = a.keys().chain(b.keys()).collect(); + keys.into_iter() + .filter(|k| a.get(*k) != b.get(*k)) + .cloned() + .collect() +} + +fn union(l: &Snapshot, i: &Snapshot) -> Snapshot { + let mut both = l.clone(); + for (id, item) in &i.items { + both.items.insert(id.clone(), item.clone()); + } + both.relink(); + both +} + +fn flags(id: &ItemId, item: &Item, incoming: &Pack, l: &Snapshot) -> Vec { + let mut out = Vec::new(); + if !incoming.manifest.secrets && export::missing_secret(id, item) { + out.push(Flag::MissingKey); + } + if id.kind == ItemKind::Provider && provider_is_chatgpt(id, item) { + // 本机同名的那一家也是 ChatGPT 账号：覆盖时登录留着本机的 + let here = l + .items + .get(id) + .is_some_and(|mine| provider_is_chatgpt(id, mine)); + if !here { + out.push(Flag::NeedsLogin); + } + } + if id.kind == ItemKind::Plugin { + out.push(Flag::PluginDisabled); + } + out +} + +fn provider_is_chatgpt(id: &ItemId, item: &Item) -> bool { + model::provider_of(id, item).is_ok_and(|p| model::is_chatgpt(&p)) +} + +/// 这一项导入不了的原因。没有是 None +#[allow(clippy::too_many_arguments)] +fn problem( + id: &ItemId, + item: &Item, + local: &Pack, + incoming: &Pack, + l: &Snapshot, + i: &Snapshot, + both: &Snapshot, + ctx: &Context, + codec: &dyn PluginCodec, +) -> Option { + // 引用了本机和包里都没有的 + for r in &both.items[id].refs { + if !both.items.contains_key(r) && !builtin(r) { + return Some(missing_ref(id, r)); + } + } + match id.kind { + ItemKind::Provider => { + if let Some(profile) = item + .units + .get(unit::CONN) + .and_then(|c| c.get("aws")) + .and_then(|a| a.get("profile")) + .and_then(Json::as_str) + && !(ctx.has_aws_profile)(profile) + { + return Some(msg!( + "import.no_aws_profile", name = &id.name, profile = profile => + "Upstream `{name}` uses the AWS profile `{profile}`, which this machine does \ + not have." + )); + } + } + ItemKind::Plugin => { + let entry = model::parse(&incoming.config_yaml) + .ok() + .and_then(|c| c.plugins.into_iter().find(|p| p.id == id.name)); + match (entry, incoming.plugins.get(&id.name)) { + (_, None) => { + return Some(msg!( + "import.plugin_missing", plugin = &id.name => + "The configuration pack has no script for plugin `{plugin}`." + )); + } + (Some(e), Some(f)) => { + let ok = model::sha256_hex(&f.script) == e.sha256 + || f.approved + .as_ref() + .is_some_and(|a| model::sha256_hex(a) == e.sha256); + if !ok { + return Some(msg!( + "import.plugin_mismatch", plugin = &id.name => + "The script of plugin `{plugin}` in the configuration pack is not \ + the one that was approved." + )); + } + } + (None, Some(_)) => {} + } + } + ItemKind::Key => { + if let Some(v) = key_value(item) + && !v.is_empty() + && let Some((other, _)) = l + .of_kind(ItemKind::Key) + .find(|(o, it)| o.name != id.name && key_value(it).as_deref() == Some(&v)) + { + return Some(msg!( + "import.key_value_taken", key = &id.name, other = &other.name => + "Gateway key `{key}` has the same value as gateway key `{other}` here." + )); + } + } + _ => {} + } + // 连同它引用的、本机没有的那些一起导进来，配置过得了校验吗 + let mut target = l.clone(); + let mut todo = vec![id.clone()]; + let mut seen = BTreeSet::new(); + while let Some(next) = todo.pop() { + if !seen.insert(next.clone()) { + continue; + } + if next != *id && l.items.contains_key(&next) { + continue; + } + if let Some(it) = i.items.get(&next) { + target + .items + .insert(next.clone(), prepared(&next, it, incoming, l, ctx)); + todo.extend(both.items[&next].refs.iter().cloned()); + } + } + target.relink(); + match land(local, &target, codec) { + Ok(_) => None, + Err(e) => Some(e.msg()), + } +} + +/// 引擎自己有的东西：叫「默认」的那条路由（没写出来时由引擎合成） +fn builtin(r: &ItemId) -> bool { + r.kind == ItemKind::Route && r.name == tw_engine::DEFAULT_ROUTE +} + +fn key_value(item: &Item) -> Option { + item.units + .get(unit::ALL) + .and_then(|a| a.get("key")) + .and_then(Json::as_str) + .map(str::to_string) +} + +/// 包里的一项按导入的规矩调整：插件停用，缺密钥、没登录的上游停用，空的网关密钥重新生成 +fn prepared(id: &ItemId, item: &Item, incoming: &Pack, l: &Snapshot, ctx: &Context) -> Item { + let mut item = item.clone(); + match id.kind { + ItemKind::Plugin => { + item.units + .insert(unit::ENABLED.to_string(), Json::Bool(false)); + } + ItemKind::Provider => { + let f = flags(id, &item, incoming, l); + if (f.contains(&Flag::MissingKey) || f.contains(&Flag::NeedsLogin)) + && let Some(Json::Object(other)) = item.units.get_mut(unit::OTHER) + { + other.insert("disabled".into(), Json::Bool(true)); + } + } + ItemKind::Key => { + if key_value(&item).as_deref() == Some("") + && let Some(Json::Object(all)) = item.units.get_mut(unit::ALL) + { + all.insert("key".into(), Json::String((ctx.new_key)())); + } + } + _ => {} + } + item +} + +/// 落下去并校验。校验和换入配置时是同一套（`tw_config::try_parse`） +fn land(local: &Pack, target: &Snapshot, codec: &dyn PluginCodec) -> Result { + let pack = model::materialize_with(local, target, codec)?; + tw_config::try_parse(&pack.config_yaml).map_err(|r| ImportError::Rejected(Box::new(r)))?; + Ok(pack) +} + +/// 导入的结果。 +#[derive(Debug, Clone)] +pub struct Applied { + pub pack: Pack, + /// 值变了的网关密钥（覆盖进来的），名字是本机原来的 + pub changed_keys: Vec, +} + +/// 照逐项的处理导入。没有写进 `decisions` 的项目不导入 +pub fn apply( + local: &Pack, + incoming: &Pack, + decisions: &[Decision], + ctx: &Context, +) -> Result { + apply_with(local, incoming, decisions, ctx, model::codec()) +} + +pub fn apply_with( + local: &Pack, + incoming: &Pack, + decisions: &[Decision], + ctx: &Context, + codec: &dyn PluginCodec, +) -> Result { + let l = model::snapshot_with(local, codec)?; + let mut i = model::snapshot_with(incoming, codec)?; + let rows = preview_with(local, incoming, ctx, codec)?; + let errors: BTreeMap<&ItemId, &Msg> = rows + .iter() + .filter_map(|r| r.error.as_ref().map(|e| (&r.item, e))) + .collect(); + + // 先改名（包里引用它的跟着改），再逐项落 + let mut names: BTreeMap = BTreeMap::new(); + for d in decisions { + if let Action::Rename(new) = &d.action { + if !d.item.kind.is_named() { + return Err(ImportError::NotInPack(d.item.clone())); + } + let new_id = match d.item.kind { + ItemKind::SecurityRule if !new.contains('/') => { + let guard = d.item.name.split('/').next().unwrap_or_default(); + ItemId::new(ItemKind::SecurityRule, format!("{guard}/{new}")) + } + k => ItemId::new(k, new.clone()), + }; + if new.trim().is_empty() || new.trim() != new { + return Err(ImportError::NameTaken(d.item.clone(), new.clone())); + } + if l.items.contains_key(&new_id) || i.items.contains_key(&new_id) { + return Err(ImportError::NameTaken(d.item.clone(), new.clone())); + } + rename(&mut i, &d.item, &new_id.name); + names.insert(d.item.clone(), new_id); + } + } + + let mut target = l.clone(); + let mut taken: Vec = Vec::new(); + for d in decisions { + if d.item.kind.is_order() || (d.item.kind == ItemKind::ClientPrefs && !ctx.with_prefs) { + continue; + } + let id = names + .get(&d.item) + .cloned() + .unwrap_or_else(|| d.item.clone()); + if d.action != Action::Skip + && let Some(e) = errors.get(&d.item) + { + return Err(ImportError::Item(d.item.clone(), Box::new((*e).clone()))); + } + let Some(item) = i.items.get(&id) else { + if d.action == Action::Skip { + continue; + } + return Err(ImportError::NotInPack(d.item.clone())); + }; + match &d.action { + Action::Skip => continue, + Action::Import if l.items.contains_key(&id) && !is_global(id.kind) => { + // 内容一样的「导入」什么都不做；不一样的要选 + if l.items[&id].units != item.units { + return Err(ImportError::Exists(id)); + } + continue; + } + Action::Import | Action::Overwrite | Action::Rename(_) => {} + } + let item = prepared(&id, item, incoming, &l, ctx); + target.items.insert(id.clone(), item); + taken.push(id); + } + + // 新加的上游、路由、插件接在后面，按包里的先后 + for kind in [ItemKind::Provider, ItemKind::Route, ItemKind::Plugin] { + let order_id = ItemId::new(kind.order().expect("has an order"), ""); + let pack_order: Vec = i + .get(&order_id) + .map(|o| names_of(&o.units)) + .unwrap_or_default(); + let mut list: Vec = target + .get(&order_id) + .map(|o| names_of(&o.units)) + .unwrap_or_default(); + let new: Vec<&ItemId> = taken + .iter() + .filter(|id| id.kind == kind && !list.contains(&id.name)) + .collect(); + if new.is_empty() { + continue; + } + let mut added: Vec = new.iter().map(|id| id.name.clone()).collect(); + added.sort_by_key(|n| pack_order.iter().position(|p| p == n).unwrap_or(usize::MAX)); + list.extend(added); + target.items.insert( + order_id, + Item { + units: BTreeMap::from([( + unit::ALL.to_string(), + Json::Array(list.into_iter().map(Json::String).collect()), + )]), + ..Default::default() + }, + ); + } + target.relink(); + + // 导进来的项目引用的东西都得在：被跳过的那些不在 + for id in &taken { + for r in &target.items[id].refs { + if !target.items.contains_key(r) && !builtin(r) { + return Err(ImportError::Item(id.clone(), Box::new(skipped_ref(id, r)))); + } + } + } + + let pack = land(local, &target, codec)?; + Ok(Applied { + changed_keys: changed_keys(local, &pack), + pack, + }) +} + +fn names_of(units: &BTreeMap) -> Vec { + match units.get(unit::ALL) { + Some(Json::Array(a)) => a + .iter() + .filter_map(|n| n.as_str().map(str::to_string)) + .collect(), + _ => Vec::new(), + } +} + +/// 值变了或者没了的网关密钥：名字是 `before` 里的 +pub fn changed_keys(before: &Pack, after: &Pack) -> Vec { + let keys = |p: &Pack| -> BTreeMap { + model::parse(&p.config_yaml) + .map(|c| c.clients.into_iter().map(|c| (c.name, c.key)).collect()) + .unwrap_or_default() + }; + let (b, a) = (keys(before), keys(after)); + b.into_iter() + .filter(|(name, v)| a.get(name) != Some(v)) + .map(|(name, _)| name) + .collect() +} + +/// 全局设置的一节在导入时叫什么（给界面列在「全局设置」下） +pub fn is_global_section(id: &ItemId) -> bool { + id.kind == ItemKind::Global && global::ALL.contains(&id.name.as_str()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/tw-pack/src/import/tests.rs b/crates/tw-pack/src/import/tests.rs new file mode 100644 index 00000000..f66a9396 --- /dev/null +++ b/crates/tw-pack/src/import/tests.rs @@ -0,0 +1,524 @@ +//! 导入：5.4 节的每一种状态，和改名、跳过、覆盖之后的样子。 + +use std::collections::BTreeMap; + +use serde_json::json; + +use super::*; +use crate::model::PluginCodec; + +/// 整份比较、只有带 `steer` 的要批准 +struct WholeFile; + +impl PluginCodec for WholeFile { + fn split(&self, _: &str) -> Option<(String, serde_json::Value)> { + None + } + fn join(&self, source: &str, data: &serde_json::Value) -> Option { + data.is_null().then(|| source.to_string()) + } + fn shipped(&self, _: &str) -> Option { + None + } + fn needs_approval(&self, source: &str) -> bool { + source.contains("steer") + } +} +use crate::{Manifest, PackKind, PackOrigin, PluginFiles}; + +const KEY: &str = "c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00"; + +fn local_text(extra: &str) -> String { + format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n key: tw-local\n - name: codex\n key: tw-codex\nproviders:\n - name: 公司中转\n base_url: https://relay.example\n key: sk-old\n - name: chatgpt\n base_url: https://chatgpt.com/backend-api/codex\n oauth:\n refresh: rt-mine\n endpoint: https://auth.openai.com/oauth/token\n headers:\n ChatGPT-Account-Id: acct-mine\n{extra}" + ) +} + +fn pack_of(text: &str, secrets: bool) -> Pack { + let mut manifest = Manifest::new(PackKind::Full, PackOrigin::Export); + manifest.secrets = secrets; + Pack { + manifest, + config_yaml: text.to_string(), + plugins: BTreeMap::new(), + client_prefs: None, + } +} + +const INCOMING: &str = r#"version: 1 +clients: + - name: default + key: tw-local + - name: shared + key: tw-shared +providers: + - name: 公司中转 + base_url: https://relay.example/v2 + key: sk-new + - name: OpenRouter + base_url: https://openrouter.ai/api/v1 + key: sk-or + - name: chatgpt + base_url: https://chatgpt.com/backend-api/codex + - name: 新的 ChatGPT + base_url: https://chatgpt.com/backend-api/codex + - name: Bedrock 东京 + base_url: https://bedrock-runtime.ap-northeast-1.amazonaws.com + aws: { profile: tokyo } +routes: + - name: 走中转 + rules: + - { name: all, to: 公司中转 } + - name: 缺上游 + rules: + - { name: all, to: 不存在 } +failover: + pause_secs: 30 +"#; + +fn ctx() -> Context<'static> { + Context { + has_aws_profile: &|p| p == "default", + new_key: &|| "tw-regenerated".to_string(), + with_prefs: true, + } +} + +fn rows(local: &Pack, incoming: &Pack) -> BTreeMap { + preview_with(local, incoming, &ctx(), &WholeFile) + .unwrap() + .into_iter() + .map(|r| (r.item.to_string(), r)) + .collect() +} + +#[test] +fn every_item_gets_its_status() { + let local = pack_of(&local_text(""), true); + let incoming = pack_of(INCOMING, true); + let r = rows(&local, &incoming); + let status = |k: &str| { + r.get(k) + .unwrap_or_else(|| panic!("no row {k}: {:?}", r.keys())) + .status + }; + assert_eq!(status("provider `OpenRouter`"), Status::New); + assert_eq!(status("key `shared`"), Status::New); + assert_eq!(status("key `default`"), Status::Same); + assert_eq!(status("provider `公司中转`"), Status::Conflict); + assert_eq!(r["provider `公司中转`"].differs, ["conn"]); + assert_eq!( + r["provider `公司中转`"].suggested_name.as_deref(), + Some("公司中转 (2)") + ); + // 本机的全局设置还是默认值：直接采用 + assert_eq!(status("global `failover`"), Status::New); + assert!(r["global `failover`"].global); + // 顺序项目不是一行 + assert!(!r.keys().any(|k| k.contains("order")), "{:?}", r.keys()); + // 引用了两边都没有的上游 + assert_eq!(status("route `缺上游`"), Status::Error); + let e = r["route `缺上游`"].error.as_ref().unwrap(); + assert_eq!(e.code, "import.missing_reference"); + assert_eq!(e.arg("target"), "不存在"); + // 本机没有它要的 AWS profile + assert_eq!(status("provider `Bedrock 东京`"), Status::Error); + assert_eq!( + r["provider `Bedrock 东京`"].error.as_ref().unwrap().code, + "import.no_aws_profile" + ); + // ChatGPT：本机没有同名的要登录；有同名、登录着的不用 + assert_eq!(r["provider `新的 ChatGPT`"].flags, [Flag::NeedsLogin]); + assert_eq!(r["provider `chatgpt`"].status, Status::Same); + // 引用：路由引用的、包里也有的上游 + assert_eq!( + r["route `走中转`"] + .refs + .iter() + .map(|r| r.to_string()) + .collect::>(), + ["provider `公司中转`"] + ); +} + +#[test] +fn a_changed_global_section_is_a_conflict_that_only_overwrites_or_skips() { + let local = pack_of(&local_text("failover:\n pause_secs: 120\n"), true); + let r = rows(&local, &pack_of(INCOMING, true)); + let f = &r["global `failover`"]; + assert_eq!(f.status, Status::Conflict); + assert!(f.global); + assert_eq!(f.differs, ["failover.pause_secs"]); + assert_eq!(f.suggested_name, None); +} + +#[test] +fn a_pack_without_secrets_imports_disabled_upstreams_and_new_keys() { + let local = pack_of(&local_text(""), true); + let mut full = pack_of(INCOMING, true); + full.config_yaml = full.config_yaml.replace(" - name: Bedrock 东京\n base_url: https://bedrock-runtime.ap-northeast-1.amazonaws.com\n aws: { profile: tokyo }\n", ""); + let exported = export::export_with( + &full, + export::Options { + scope: None, + secrets: false, + client_prefs: None, + origin: PackOrigin::Export, + }, + &WholeFile, + ) + .unwrap(); + let incoming = crate::read( + &crate::write(&exported.pack, crate::Seal::None).unwrap(), + crate::Unseal::None, + ) + .unwrap(); + assert!(!incoming.manifest.secrets); + assert!( + !incoming.config_yaml.contains("sk-or"), + "{}", + incoming.config_yaml + ); + assert!( + !incoming.config_yaml.contains("tw-shared"), + "{}", + incoming.config_yaml + ); + let r = rows(&local, &incoming); + assert_eq!(r["provider `OpenRouter`"].flags, [Flag::MissingKey]); + assert_eq!(r["provider `OpenRouter`"].status, Status::New); + assert_eq!(r["key `shared`"].flags, [Flag::MissingKey]); + let applied = apply_with( + &local, + &incoming, + &[ + Decision { + item: ItemId::new(ItemKind::Provider, "OpenRouter"), + action: Action::Import, + }, + Decision { + item: ItemId::new(ItemKind::Key, "shared"), + action: Action::Import, + }, + ], + &ctx(), + &WholeFile, + ) + .unwrap(); + let cfg = tw_config::try_parse(&applied.pack.config_yaml).unwrap(); + let or = cfg + .providers + .iter() + .find(|p| p.name == "OpenRouter") + .unwrap(); + assert!(or.disabled, "缺密钥的上游导入为停用"); + assert_eq!(or.key.as_ref().map(|k| k.raw()), Some("")); + let shared = cfg.clients.iter().find(|c| c.name == "shared").unwrap(); + assert_eq!(shared.key, "tw-regenerated"); +} + +#[test] +fn import_overwrite_rename_and_skip_land_as_chosen() { + let local = pack_of(&local_text(""), true); + let incoming = pack_of(INCOMING, true); + let d = |kind, name: &str, action| Decision { + item: ItemId::new(kind, name), + action, + }; + let applied = apply_with( + &local, + &incoming, + &[ + d(ItemKind::Provider, "OpenRouter", Action::Import), + d( + ItemKind::Provider, + "公司中转", + Action::Rename("公司中转 (2)".into()), + ), + d(ItemKind::Route, "走中转", Action::Import), + d(ItemKind::Provider, "新的 ChatGPT", Action::Import), + d(ItemKind::Global, "failover", Action::Import), + d(ItemKind::Key, "shared", Action::Skip), + ], + &ctx(), + &WholeFile, + ) + .unwrap(); + let cfg = tw_config::try_parse(&applied.pack.config_yaml) + .unwrap_or_else(|e| panic!("{e}\n{}", applied.pack.config_yaml)); + let names: Vec<&str> = cfg.providers.iter().map(|p| p.name.as_str()).collect(); + // 本机的在前，新加的按包里的先后接在后面 + assert_eq!( + names, + [ + "公司中转", + "chatgpt", + "公司中转 (2)", + "OpenRouter", + "新的 ChatGPT" + ] + ); + // 本机那一家没动 + let mine = &cfg.providers[0]; + assert_eq!(mine.base_url, "https://relay.example"); + // 改名导入的那一家，包里引用它的路由跟着改 + let route = cfg.routes.iter().find(|r| r.name == "走中转").unwrap(); + assert_eq!( + route.rules[0].to.as_ref().and_then(|t| t.name()), + Some("公司中转 (2)") + ); + // 没登录的 ChatGPT 账号上游停用着 + let gpt = cfg + .providers + .iter() + .find(|p| p.name == "新的 ChatGPT") + .unwrap(); + assert!(gpt.disabled && gpt.oauth.is_none()); + assert_eq!(cfg.failover.pause_secs, 30); + assert!(!cfg.clients.iter().any(|c| c.name == "shared")); + assert!(applied.changed_keys.is_empty()); + // 本机设置和登录都在 + assert!(applied.pack.config_yaml.contains(KEY)); + assert!(applied.pack.config_yaml.contains("rt-mine")); +} + +#[test] +fn overwriting_keeps_the_local_login_and_reports_changed_keys() { + let local = pack_of(&local_text(""), true); + let incoming = pack_of( + &INCOMING + .replace(" key: tw-local\n", " key: tw-local-new\n") + .replace( + " - name: chatgpt\n base_url: https://chatgpt.com/backend-api/codex\n", + " - name: chatgpt\n base_url: https://chatgpt.com/backend-api/codex\n max_concurrent: 3\n", + ), + true, + ); + let r = rows(&local, &incoming); + assert_eq!(r["provider `chatgpt`"].status, Status::Conflict); + assert_eq!(r["provider `chatgpt`"].flags, Vec::::new()); + let applied = apply_with( + &local, + &incoming, + &[ + Decision { + item: ItemId::new(ItemKind::Provider, "chatgpt"), + action: Action::Overwrite, + }, + Decision { + item: ItemId::new(ItemKind::Key, "default"), + action: Action::Overwrite, + }, + ], + &ctx(), + &WholeFile, + ) + .unwrap(); + let cfg = tw_config::try_parse(&applied.pack.config_yaml).unwrap(); + let gpt = cfg.providers.iter().find(|p| p.name == "chatgpt").unwrap(); + assert_eq!(gpt.max_concurrent, Some(3)); + assert!(!gpt.disabled); + assert_eq!(gpt.oauth.as_ref().unwrap().refresh, "rt-mine"); + assert_eq!(applied.changed_keys, ["default"]); +} + +#[test] +fn a_reference_to_a_skipped_item_is_refused() { + let local = pack_of(&local_text(""), true); + let incoming = pack_of(&INCOMING.replace("to: 公司中转", "to: OpenRouter"), true); + let e = apply_with( + &local, + &incoming, + &[ + Decision { + item: ItemId::new(ItemKind::Route, "走中转"), + action: Action::Import, + }, + Decision { + item: ItemId::new(ItemKind::Provider, "OpenRouter"), + action: Action::Skip, + }, + ], + &ctx(), + &WholeFile, + ) + .unwrap_err(); + assert_eq!(e.msg().code, "import.skipped_reference"); + // 选「导入」一个本机已有、内容不同的：要选覆盖或改名 + let e = apply_with( + &local, + &incoming, + &[Decision { + item: ItemId::new(ItemKind::Provider, "公司中转"), + action: Action::Import, + }], + &ctx(), + &WholeFile, + ) + .unwrap_err(); + assert_eq!(e.msg().code, "import.exists"); + // 改名改成一个被占的名字 + let e = apply_with( + &local, + &incoming, + &[Decision { + item: ItemId::new(ItemKind::Provider, "公司中转"), + action: Action::Rename("chatgpt".into()), + }], + &ctx(), + &WholeFile, + ) + .unwrap_err(); + assert_eq!(e.msg().code, "import.name_taken"); + // 导入一个出错的那一行 + let e = apply_with( + &local, + &incoming, + &[Decision { + item: ItemId::new(ItemKind::Route, "缺上游"), + action: Action::Import, + }], + &ctx(), + &WholeFile, + ) + .unwrap_err(); + assert_eq!(e.msg().code, "import.missing_reference"); +} + +#[test] +fn plugins_are_checked_against_their_approved_hash_and_arrive_disabled() { + let script = b"export const manifest = {};\n".to_vec(); + let sha = model::sha256_hex(&script); + let text = format!( + "version: 1\nplugins:\n - id: good\n file: plugins/good.js\n sha256: {sha}\n - id: tampered\n file: plugins/tampered.js\n sha256: {sha}\n - id: absent\n file: plugins/absent.js\n sha256: {sha}\n" + ); + let mut incoming = pack_of(&text, true); + incoming.plugins.insert( + "good".into(), + PluginFiles { + script: script.clone(), + approved: Some(script.clone()), + }, + ); + incoming.plugins.insert( + "tampered".into(), + PluginFiles { + script: b"something else".to_vec(), + approved: None, + }, + ); + let local = pack_of(&local_text(""), true); + let r = rows(&local, &incoming); + assert_eq!(r["plugin `good`"].status, Status::New); + assert_eq!(r["plugin `good`"].flags, [Flag::PluginDisabled]); + assert_eq!( + r["plugin `tampered`"].error.as_ref().unwrap().code, + "import.plugin_mismatch" + ); + assert_eq!( + r["plugin `absent`"].error.as_ref().unwrap().code, + "import.plugin_missing" + ); + let applied = apply_with( + &local, + &incoming, + &[Decision { + item: ItemId::new(ItemKind::Plugin, "good"), + action: Action::Import, + }], + &ctx(), + &WholeFile, + ) + .unwrap(); + let cfg = tw_config::try_parse(&applied.pack.config_yaml).unwrap(); + assert_eq!(cfg.plugins.len(), 1); + assert!(!cfg.plugins[0].enabled); + assert_eq!(cfg.plugins[0].sha256, sha); + assert_eq!(applied.pack.plugins["good"].script, script); + + // 改得了工具调用的插件：停用着进来，也不批准 —— 要在插件页上点头 + let steer = b"export const manifest = {}; // steer\n".to_vec(); + let sha = model::sha256_hex(&steer); + let mut incoming = pack_of( + &format!("version: 1\nplugins:\n - id: s\n file: plugins/s.js\n sha256: {sha}\n"), + true, + ); + incoming.plugins.insert( + "s".into(), + PluginFiles { + script: steer.clone(), + approved: Some(steer), + }, + ); + let applied = apply_with( + &local, + &incoming, + &[Decision { + item: ItemId::new(ItemKind::Plugin, "s"), + action: Action::Import, + }], + &ctx(), + &WholeFile, + ) + .unwrap(); + let cfg = tw_config::try_parse(&applied.pack.config_yaml).unwrap(); + assert_ne!(cfg.plugins[0].sha256, sha); + assert!(!cfg.plugins[0].enabled); +} + +#[test] +fn a_key_with_the_value_of_another_key_here_is_an_error() { + let local = pack_of(&local_text(""), true); + let incoming = pack_of( + "version: 1\nclients:\n - name: copy\n key: tw-codex\n", + true, + ); + let r = rows(&local, &incoming); + assert_eq!( + r["key `copy`"].error.as_ref().unwrap().code, + "import.key_value_taken" + ); +} + +/// 导进来过不了校验的：那一行就是错误，带着校验的那句话 +#[test] +fn an_item_that_would_not_validate_is_an_error_with_the_reason() { + let local = pack_of(&local_text(""), true); + let incoming = pack_of( + "version: 1\nproviders:\n - name: bad\n base_url: ftp://nope\n", + true, + ); + let r = rows(&local, &incoming); + assert_eq!(r["provider `bad`"].status, Status::Error); + assert_eq!( + r["provider `bad`"].error.as_ref().unwrap().code, + "config.bad_base_url" + ); +} + +#[test] +fn client_preferences_import_only_when_core_keeps_them() { + let local = pack_of(&local_text(""), true); + let mut incoming = pack_of("version: 1\n", true); + incoming.client_prefs = Some(json!({"language": "en"})); + let r = rows(&local, &incoming); + assert_eq!(r["client_prefs"].status, Status::New); + let remote = Context { + with_prefs: false, + ..ctx() + }; + let r = preview_with(&local, &incoming, &remote, &WholeFile).unwrap(); + assert!(r.is_empty(), "{r:?}"); + let applied = apply_with( + &local, + &incoming, + &[Decision { + item: ItemId::new(ItemKind::ClientPrefs, ""), + action: Action::Import, + }], + &ctx(), + &WholeFile, + ) + .unwrap(); + assert_eq!(applied.pack.client_prefs, Some(json!({"language": "en"}))); +} diff --git a/crates/tw-pack/src/lib.rs b/crates/tw-pack/src/lib.rs index 99c1e4c6..789b1400 100644 --- a/crates/tw-pack/src/lib.rs +++ b/crates/tw-pack/src/lib.rs @@ -33,7 +33,12 @@ use std::io::{Read, Write}; use serde::{Deserialize, Serialize}; use tw_types::{Msg, msg}; +pub mod backup; +pub mod disk; +pub mod export; +pub mod import; pub mod model; +pub mod restore; /// 配置包格式的版本。**读到更大的就拒绝**（[`PackError::TooNew`]）：新版本的格式旧版本 /// 猜不准，猜错了会悄悄丢东西 @@ -42,6 +47,9 @@ pub const FORMAT: u32 = 1; /// 一个配置包（连同解压之后的内容）最大多少字节 pub const MAX_BYTES: usize = 16 * 1024 * 1024; +/// 口令最少几个字符 +pub const MIN_PASSPHRASE: usize = 8; + /// 扩展名，不带点 pub const EXTENSION: &str = "twconfig"; @@ -194,7 +202,10 @@ impl From for PackOrigin { O::Cli => PackOrigin::Cli, O::External => PackOrigin::External, O::Defaults => PackOrigin::Defaults, - O::Rollback => PackOrigin::Restore, + O::Sync => PackOrigin::Sync, + O::Import => PackOrigin::Import, + O::Restore => PackOrigin::Restore, + O::Repair => PackOrigin::Repair, // 轮换不留版本，走到这里也只是一个说不出来源的版本 O::Rotation => PackOrigin::Other, } diff --git a/crates/tw-pack/src/model/items.rs b/crates/tw-pack/src/model/items.rs index 08615a64..3f200741 100644 --- a/crates/tw-pack/src/model/items.rs +++ b/crates/tw-pack/src/model/items.rs @@ -14,7 +14,7 @@ use crate::{Pack, PluginFiles}; pub(crate) const ACCOUNT_HEADER: &str = tw_config::chatgpt::ACCOUNT_HEADER; /// 一个上游是 ChatGPT 账号。它的登录（`oauth`）和账户 ID 请求头属于这台设备 -pub(crate) fn is_chatgpt(p: &Provider) -> bool { +pub fn is_chatgpt(p: &Provider) -> bool { p.effective_protocol() == Some(Protocol::Chatgpt) } @@ -412,7 +412,12 @@ fn plugin_units( if let Some(d) = data { units.insert(unit::SETTINGS.to_string(), d); } - (units, Some(files.clone())) + // 带着走的是批准过的那一份：和单位说的是同一份代码 + let carried = PluginFiles { + script: source.to_vec(), + approved: Some(source.to_vec()), + }; + (units, Some(carried)) } /// 批准过的那一份：哈希对得上配置的脚本或底稿。都对不上是脚本 @@ -426,7 +431,7 @@ pub(crate) fn approved_source<'a>(entry: &tw_config::Plugin, files: &'a PluginFi } } -pub(crate) fn sha256_hex(bytes: &[u8]) -> String { +pub fn sha256_hex(bytes: &[u8]) -> String { use sha2::Digest; let d = sha2::Sha256::digest(bytes); d.iter().map(|b| format!("{b:02x}")).collect() diff --git a/crates/tw-pack/src/model/mod.rs b/crates/tw-pack/src/model/mod.rs index ce9efe29..28974ee1 100644 --- a/crates/tw-pack/src/model/mod.rs +++ b/crates/tw-pack/src/model/mod.rs @@ -45,7 +45,9 @@ mod rename; mod write; pub use codec::{PluginCodec, WholeFile}; +pub use items::{is_chatgpt, sha256_hex}; pub use rename::{free_name, rename}; +pub(crate) use write::materialize_for_export; pub use write::{materialize, materialize_with}; /// 配置里属于这台设备、不随导出和同步带走的几节 @@ -487,6 +489,16 @@ pub(crate) fn parse(text: &str) -> Result { Ok(cfg) } +/// 一个上游项目的单位读成配置里的上游（没有本机的登录） +pub fn provider_of(id: &ItemId, item: &Item) -> Result { + items::provider_mapping(id, &item.units, None).map(|(p, _)| p) +} + +/// 一棵 JSON 摊平成通用偏好那样的单位（`a.b` → 值） +pub(crate) fn json_flatten(v: &Json, out: &mut BTreeMap) { + json::flatten("", v, out); +} + /// [`Pack::content_id`] pub(crate) fn content_id(pack: &Pack) -> String { let hash = match snapshot(pack) { diff --git a/crates/tw-pack/src/model/tests.rs b/crates/tw-pack/src/model/tests.rs index a09ce6c1..3bbedfde 100644 --- a/crates/tw-pack/src/model/tests.rs +++ b/crates/tw-pack/src/model/tests.rs @@ -825,7 +825,7 @@ fn a_steering_plugin_from_elsewhere_waits_for_approval_here() { "批准了别处来的代码" ); assert_eq!(out.plugins["steerer"].approved, None); - // 停用着来的照常批准：打开时才问 + // 停用着来的也不批准：装上这种插件本身就要点头 let mut off = t.clone(); off.items .get_mut(&id(ItemKind::Plugin, "steerer")) @@ -834,6 +834,11 @@ fn a_steering_plugin_from_elsewhere_waits_for_approval_here() { .insert("enabled".into(), json!(false)); let out = materialize_with(&with_plugins(&[]), &off, &codec).unwrap(); let cfg = tw_config::try_parse(&out.config_yaml).unwrap(); + assert!(!cfg.plugins[0].enabled); + assert_ne!(cfg.plugins[0].sha256, items::sha256_hex(src.as_bytes())); + // 导出不管这些：包里的是那一边批准过的样子 + let exported = super::materialize_for_export(&with_plugins(&[]), &t, &codec).unwrap(); + let cfg = tw_config::try_parse(&exported.config_yaml).unwrap(); assert_eq!(cfg.plugins[0].sha256, items::sha256_hex(src.as_bytes())); // 本机停用着、批准过同一份代码，别处把它打开了：这里照样要点头 let off_here = with_plugins(&[("steerer", src, false)]); diff --git a/crates/tw-pack/src/model/write.rs b/crates/tw-pack/src/model/write.rs index ff3355bc..793ef2d6 100644 --- a/crates/tw-pack/src/model/write.rs +++ b/crates/tw-pack/src/model/write.rs @@ -66,18 +66,38 @@ pub fn materialize(local: &Pack, target: &Snapshot) -> Result /// - 代码没变的（包括默认插件）在本机的文件上改数据，字节尽量不动；代码变了的、新来的用 /// 目标里带着的原文件（[`Item::files`]），没有就按单位拼出源码；默认插件在本机没有时 /// 用这一份 Core 自带的那一份，也没有就不加 -/// - **改得了工具调用的插件，开着、而代码在本机没批准过（新来的、代码变了的、本机原来 -/// 停用着的）时不批准**：文件写下去，配置里的哈希留着本机原来批准的那一份（没有就是 -/// 一个谁都对不上的），插件停在「待批准」，要在本机点过头才跑 +/// - **改得了工具调用的插件，代码在本机没批准过（新来的、代码变了的），或者在本机原来停用着、 +/// 落下去是开着的，就不批准**：文件写下去，配置里的哈希是一个谁都对不上的，插件停在 +/// 「待批准」，要在本机点过头（确认过的那条路）才跑。和插件页上一样：装上、打开、改代码都要 +/// 点头 /// - 别的照目标批准：哈希就是写下去的文件的 pub fn materialize_with( local: &Pack, target: &Snapshot, codec: &dyn PluginCodec, +) -> Result { + land(local, target, codec, true) +} + +/// 同 [`materialize_with`]，**插件照目标批准，不管它改不改得了工具调用**。只给生成配置包用 +/// （导出）：落的是包里的内容，不是这台设备要跑的东西 +pub(crate) fn materialize_for_export( + local: &Pack, + target: &Snapshot, + codec: &dyn PluginCodec, +) -> Result { + land(local, target, codec, false) +} + +fn land( + local: &Pack, + target: &Snapshot, + codec: &dyn PluginCodec, + gate: bool, ) -> Result { let cfg = parse(&local.config_yaml)?; let current = snapshot_with(local, codec)?; - let plugins = plugin_outcomes(local, &cfg, ¤t, target, codec)?; + let plugins = plugin_outcomes(local, &cfg, ¤t, target, codec, gate)?; let want = complete_orders(normalized(target), &cfg, &plugins); let prefs = target .get(&ItemId::new(ItemKind::ClientPrefs, "")) @@ -224,6 +244,7 @@ fn plugin_outcomes( current: &Snapshot, target: &Snapshot, codec: &dyn PluginCodec, + gate: bool, ) -> Result, ModelError> { let mut out = BTreeMap::new(); for (id, t) in target.of_kind(ItemKind::Plugin) { @@ -264,7 +285,8 @@ fn plugin_outcomes( } else if let Some(f) = t.files.as_ref().filter(|f| { let src = text(&f.script); let code = codec.split(&src).map_or(src, |(c, _)| c); - t_script.and_then(Json::as_str) == Some(code.as_str()) + // 默认插件没有脚本这个单位：带着的就是它在那一边的文件 + t_script.is_none_or(|s| s.as_str() == Some(code.as_str())) }) { Some(text(&f.script)) } else if let Some(s) = t_script.and_then(Json::as_str) { @@ -286,26 +308,23 @@ fn plugin_outcomes( })?, None => base, }; - let approved_here = - le.is_some_and(|le| le.enabled) && same_code && local_approved.is_some(); - let gated = enabled && codec.needs_approval(&source) && !approved_here; + // 本机批准过这份代码；打开它也是本机的人打开的 + let approved_here = same_code && local_approved.is_some(); + let turned_on = enabled && !le.is_some_and(|le| le.enabled); + let gated = gate && codec.needs_approval(&source) && (!approved_here || turned_on); let pending_edit = match (le, lf) { (Some(le), Some(lf)) => sha256_hex(&lf.script) != le.sha256, _ => false, }; let (files, sha256) = if gated { - // 哈希留着本机原来批准的那一份（界面拿它和底稿比出改了什么）；它恰好就是要写下去 - // 的这一份（本机停用着、批准过同一份代码）时也不能留：留着就等于替人点了头 - let sha = match le { - Some(le) if le.sha256 != sha256_hex(source.as_bytes()) => le.sha256.clone(), - _ => NOT_APPROVED.to_string(), - }; + // 哈希写成一个谁都对不上的：插件停在「文件变了、待批准」，批准要走确认过的那条路。 + // 本机原来批准的底稿留着，界面拿它比出改了什么 ( PluginFiles { script: source.into_bytes(), approved: local_approved.clone(), }, - sha, + NOT_APPROVED.to_string(), ) } else if same_code && pending_edit { // 本机有一处还没批准的改动：留着它，底稿和哈希换成新的 diff --git a/crates/tw-pack/src/restore.rs b/crates/tw-pack/src/restore.rs new file mode 100644 index 00000000..9ae33160 --- /dev/null +++ b/crates/tw-pack/src/restore.rs @@ -0,0 +1,187 @@ +//! 恢复：把配置换回某一个备份版本，全部或者其中几项。 +//! +//! 按项目列出那一版和现在的差异（新增、删除、修改），缺省全选，也就是整体换回那一版； +//! 没选的保持现在的样子。 +//! +//! - **本机设置留着现在的**（监听、保留期、同步设置、控制面的钥匙），ChatGPT 上游的登录也是 +//! 现在的 —— 那一版里的旧 token 可能早就因为刷新作废了。那一版有、现在没有的 ChatGPT 上游 +//! 恢复成停用：它的登录随删除吊销了，要重新登录 +//! - **本机接管的客户端在用的网关密钥不删**（固定密钥）：那一行标成留着 +//! - 恢复本身也留一个版本，所以可以撤销 + +use std::collections::BTreeSet; + +use serde_json::Value as Json; + +use crate::model::{self, ItemId, ItemKind, ModelError, PluginCodec, Snapshot, unit}; +use crate::{Pack, import}; + +/// 恢复对一项做什么。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Change { + Add, + Remove, + Modify, +} + +/// 为什么留着它。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Kept { + /// 接管的客户端在用这把网关密钥 + KeyInUse, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Row { + pub item: ItemId, + pub change: Change, + pub kept: Option, +} + +/// 那一版和现在的差异。`pinned` 是固定密钥的名字。`with_prefs` 为 false 时通用偏好不算 +/// （经远程连接恢复：偏好在 Lite 那边） +pub fn preview( + local: &Pack, + backup: &Pack, + pinned: &BTreeSet, + with_prefs: bool, +) -> Result, ModelError> { + preview_with(local, backup, pinned, with_prefs, model::codec()) +} + +pub fn preview_with( + local: &Pack, + backup: &Pack, + pinned: &BTreeSet, + with_prefs: bool, + codec: &dyn PluginCodec, +) -> Result, ModelError> { + let l = model::snapshot_with(local, codec)?; + let b = model::snapshot_with(backup, codec)?; + Ok(rows(&l, &b, pinned, with_prefs)) +} + +fn rows(l: &Snapshot, b: &Snapshot, pinned: &BTreeSet, with_prefs: bool) -> Vec { + let (l, b) = (model::normalized(l), model::normalized(b)); + let ids: BTreeSet<&ItemId> = l.items.keys().chain(b.items.keys()).collect(); + ids.into_iter() + .filter(|id| with_prefs || id.kind != ItemKind::ClientPrefs) + .filter_map(|id| { + let change = match (l.items.get(id), b.items.get(id)) { + (None, Some(_)) => Change::Add, + (Some(_), None) => Change::Remove, + (Some(x), Some(y)) if x.units != y.units => Change::Modify, + _ => return None, + }; + let kept = + (change == Change::Remove && id.kind == ItemKind::Key && pinned.contains(&id.name)) + .then_some(Kept::KeyInUse); + Some(Row { + item: id.clone(), + change, + kept, + }) + }) + .collect() +} + +/// 恢复的结果。 +#[derive(Debug, Clone)] +pub struct Applied { + pub pack: Pack, + /// 值变了或者没了的网关密钥 + pub changed_keys: Vec, + /// 因为固定密钥留着没删的 + pub kept: Vec, +} + +/// 恢复。`items` 是只恢复这几项，None 是全部 +pub fn apply( + local: &Pack, + backup: &Pack, + items: Option<&[ItemId]>, + pinned: &BTreeSet, + with_prefs: bool, +) -> Result { + apply_with(local, backup, items, pinned, with_prefs, model::codec()) +} + +pub fn apply_with( + local: &Pack, + backup: &Pack, + items: Option<&[ItemId]>, + pinned: &BTreeSet, + with_prefs: bool, + codec: &dyn PluginCodec, +) -> Result { + let l = model::snapshot_with(local, codec)?; + let b = model::snapshot_with(backup, codec)?; + let picked: Option> = items.map(|xs| xs.iter().collect()); + let mut target = l.clone(); + let mut kept = Vec::new(); + for row in rows(&l, &b, pinned, with_prefs) { + if picked.as_ref().is_some_and(|p| !p.contains(&row.item)) { + continue; + } + if row.kept.is_some() { + kept.push(row.item.clone()); + continue; + } + match b.items.get(&row.item) { + Some(it) => { + let mut it = it.clone(); + // 那一版有、现在没有的 ChatGPT 账号上游：没有登录，停用着回来 + if row.item.kind == ItemKind::Provider + && !l.items.contains_key(&row.item) + && model::provider_of(&row.item, &it).is_ok_and(|p| model::is_chatgpt(&p)) + && let Some(Json::Object(other)) = it.units.get_mut(unit::OTHER) + { + other.insert("disabled".into(), Json::Bool(true)); + } + target.items.insert(row.item.clone(), it); + } + None => { + target.items.remove(&row.item); + } + } + } + target.relink(); + let pack = model::materialize_with(local, &target, codec)?; + tw_config::try_parse(&pack.config_yaml).map_err(|r| ModelError::Config(Box::new(r)))?; + Ok(Applied { + changed_keys: import::changed_keys(local, &pack), + pack, + kept, + }) +} + +/// 配置读不进来（安全模式）时的恢复：整份换成那一版，**只有控制面的钥匙换成现在这一把** +/// （`current_text` 是磁盘上现在那份读不进来的原文）。插件和通用偏好照那一版 +pub fn whole(backup: &Pack, current_text: &str) -> Pack { + let real = tw_config::control_key::raw_in(current_text) + .ok() + .flatten() + .or_else(|| raw_key_line(current_text)); + let text = tw_config::control_key::unmask(&backup.config_yaml, real.as_deref()) + .unwrap_or_else(|_| backup.config_yaml.clone()); + Pack { + config_yaml: text, + ..backup.clone() + } +} + +/// YAML 坏到按路径找不到时，退一步在原文里找 `key: <64 个十六进制字符>` 那一行 +fn raw_key_line(text: &str) -> Option { + text.lines().find_map(|l| { + let v = l + .trim() + .strip_prefix("key:")? + .trim() + .trim_matches(['"', '\'']); + (v.len() == tw_api::control::KEY_HEX_LEN && v.bytes().all(|b| b.is_ascii_hexdigit())) + .then(|| v.to_string()) + }) +} + +#[cfg(test)] +mod tests; diff --git a/crates/tw-pack/src/restore/tests.rs b/crates/tw-pack/src/restore/tests.rs new file mode 100644 index 00000000..3ecaeb0d --- /dev/null +++ b/crates/tw-pack/src/restore/tests.rs @@ -0,0 +1,105 @@ +//! 恢复：按项目列差异、挑几项恢复、本机设置和登录留着、固定密钥不删。 + +use std::collections::{BTreeMap, BTreeSet}; + +use super::*; +use crate::model::WholeFile; +use crate::{Manifest, PackKind, PackOrigin}; + +const KEY: &str = "c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00"; +const OLD_KEY: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + +fn pack(text: &str) -> Pack { + Pack { + manifest: Manifest::new(PackKind::Full, PackOrigin::Ui), + config_yaml: text.to_string(), + plugins: BTreeMap::new(), + client_prefs: None, + } +} + +fn now() -> Pack { + pack(&format!( + "version: 1\nlisten:\n control:\n key: {KEY}\n gateway:\n port: 9000\nclients:\n - name: default\n key: tw-a\n - name: claude-code\n key: tw-cc\n client: claude-code\nproviders:\n - name: kept\n base_url: https://kept.example\n - name: chatgpt\n base_url: https://chatgpt.com/backend-api/codex\n oauth:\n refresh: rt-now\n endpoint: https://auth.openai.com/oauth/token\n" + )) +} + +/// 一个旧版本：密钥 claude-code 那时还没有，有一个后来删掉的上游，chatgpt 的 token 是旧的 +fn then() -> Pack { + pack(&format!( + "version: 1\nlisten:\n control:\n key: {}\n gateway:\n port: 8000\nclients:\n - name: default\n key: tw-old\nproviders:\n - name: kept\n base_url: https://kept.example/v0\n - name: deleted\n base_url: https://deleted.example\n - name: chatgpt\n base_url: https://chatgpt.com/backend-api/codex\n oauth:\n refresh: rt-then\n endpoint: https://auth.openai.com/oauth/token\n - name: old-gpt\n base_url: https://chatgpt.com/backend-api/codex\n oauth:\n refresh: rt-gone\n endpoint: https://auth.openai.com/oauth/token\n", + tw_api::control::KEY_MASK + )) +} + +fn pinned() -> BTreeSet { + ["claude-code".to_string()].into_iter().collect() +} + +#[test] +fn the_preview_lists_what_differs_by_item() { + let rows = preview_with(&now(), &then(), &pinned(), true, &WholeFile).unwrap(); + let got: Vec<(String, Change, Option)> = rows + .iter() + .map(|r| (r.item.to_string(), r.change, r.kept)) + .collect(); + assert_eq!( + got, + vec![ + ("provider `deleted`".into(), Change::Add, None), + ("provider `kept`".into(), Change::Modify, None), + ("provider `old-gpt`".into(), Change::Add, None), + ( + "key `claude-code`".into(), + Change::Remove, + Some(Kept::KeyInUse) + ), + ("key `default`".into(), Change::Modify, None), + ("provider_order".into(), Change::Modify, None), + ] + ); +} + +#[test] +fn restoring_everything_keeps_local_settings_logins_and_keys_in_use() { + let out = apply_with(&now(), &then(), None, &pinned(), true, &WholeFile).unwrap(); + let cfg = tw_config::try_parse(&out.pack.config_yaml) + .unwrap_or_else(|e| panic!("{e}\n{}", out.pack.config_yaml)); + assert_eq!(cfg.listen.gateway.port, 9000, "本机设置留着现在的"); + assert!(out.pack.config_yaml.contains(KEY)); + let p = |n: &str| cfg.providers.iter().find(|p| p.name == n); + assert_eq!(p("kept").unwrap().base_url, "https://kept.example/v0"); + assert!(p("deleted").is_some()); + // 登录是现在的；回来的那一家停用着，旧 token 不写回 + assert_eq!( + p("chatgpt").unwrap().oauth.as_ref().unwrap().refresh, + "rt-now" + ); + let old = p("old-gpt").unwrap(); + assert!(old.disabled && old.oauth.is_none()); + assert!(!out.pack.config_yaml.contains("rt-gone")); + // 接管在用的密钥没删 + assert!(cfg.clients.iter().any(|c| c.name == "claude-code")); + assert_eq!(out.kept, [ItemId::new(ItemKind::Key, "claude-code")]); + assert_eq!(out.changed_keys, ["default"]); +} + +#[test] +fn restoring_a_few_items_leaves_the_rest_as_it_is() { + let pick = [ItemId::new(ItemKind::Provider, "deleted")]; + let out = apply_with(&now(), &then(), Some(&pick), &pinned(), true, &WholeFile).unwrap(); + let cfg = tw_config::try_parse(&out.pack.config_yaml).unwrap(); + let names: Vec<&str> = cfg.providers.iter().map(|p| p.name.as_str()).collect(); + assert_eq!(names, ["kept", "chatgpt", "deleted"]); + assert_eq!(cfg.providers[0].base_url, "https://kept.example"); + assert!(out.changed_keys.is_empty()); +} + +#[test] +fn in_safe_mode_the_whole_version_comes_back_with_todays_key() { + let broken = format!("version: 1\nlisten:\n control:\n key: {OLD_KEY}\nproviders: [\n"); + let out = whole(&then(), &broken); + assert!(out.config_yaml.contains(OLD_KEY), "{}", out.config_yaml); + assert!(!out.config_yaml.contains(tw_api::control::KEY_MASK)); + assert!(out.config_yaml.contains("port: 8000")); +} diff --git a/crates/tw-store/src/recorder.rs b/crates/tw-store/src/recorder.rs index b292a279..daa14eda 100644 --- a/crates/tw-store/src/recorder.rs +++ b/crates/tw-store/src/recorder.rs @@ -695,6 +695,9 @@ impl Recorder { | Event::PluginFailed { .. } // 自己刚报出去的那条。**不能再处理一遍** —— 那是一个回路 | Event::RequestPriced { .. } + // 配置那一边的事：哪几把密钥换了、界面的偏好变了 + | Event::GatewayKeysChanged { .. } + | Event::ClientPrefsChanged { .. } // 只发给事件流上掉队的那一个订阅者，从不进总线 | Event::EventsDropped { .. } => {} } diff --git a/docs/config.md b/docs/config.md index 7ef1c664..75a91421 100644 --- a/docs/config.md +++ b/docs/config.md @@ -19,7 +19,7 @@ when they disagree, so a field listed here is a field the binary reads. `THINKWATCH_HOME` replaces the directory, and `--config ` names the file for a single command. The directory holds everything else core keeps -as well: the request database (`data.db`), the configuration history +as well: the request database (`data.db`), the backups of the configuration (`history/`), the downloaded price table (`model_prices.json`) and the local control socket (`twcore.sock`; on Windows a loopback port recorded in `control.port`). The directory is private to its owner (`0700`), the file is @@ -116,23 +116,38 @@ reports why. Retention changes are applied on the next hourly clean-up. When two writers edit at once (the app and a hand edit), the second write is refused with a version mismatch instead of overwriting the first. -### History and rollback +### Backups, export and import -Every version that was in effect is kept in `history/` beside the file, -with where it came from (the app, the command line, an outside edit, a -rollback, a credential rotation). The last 50 are kept. +Every change to the configuration leaves a version in `history/` beside the +file: a configuration pack (`.twconfig`) with the plugins and the app's +preferences, recording where the change came from (the app, the command +line, an outside edit, an import, a restore, sync). The last 10 are kept; a +credential rotation does not make one. The control key is masked in them. ```sh twcore check # validate the file without starting anything twcore config show # print it, with its version -twcore config history # list the versions, newest first -twcore config rollback 3f9a2c # go back to a version (a prefix is enough) +twcore backup list # list the versions, newest first +twcore backup restore 1792315800000 --dry-run # what going back would change +twcore backup restore 1792315800000 # go back to a version +twcore export -o all.twconfig # the whole configuration as a pack +twcore export --providers relay --no-secrets --encrypt -o relay.twconfig +twcore import all.twconfig --dry-run # what importing would do twcore config set /listen/gateway/port 8790 --int ``` -These commands work on the file directly, so they work when core is not -running, which is when a rollback is most needed. A running core picks -their changes up like any other save. +A restore keeps the settings that belong to this machine (listening, the +control key, retention, sync), the ChatGPT sign-ins and the gateway keys that +clients set up on this machine use. An import adds what is not here and asks, +item by item, what to do with what differs (`--on-conflict` answers for all). +A pack carries no settings of this machine and no ChatGPT sign-in; without +`--no-secrets` it holds keys in plain text unless `--encrypt` is given (the +passphrase is asked for, or read from `TWCORE_PASSPHRASE`). + +These commands work on the files directly, so they work when core is not +running, which is when a restore is most needed; when the file does not +load at all, a restore brings the whole version back with the current +control key. A running core picks their changes up like any other save. `twcore config set ` changes one value that is already written in the file. The path names list items by their `name` @@ -244,7 +259,7 @@ complete the first message. `key` is written by `twcore serve` before the control channel starts listening, if it is missing, and by `twcore init`. It is masked wherever the -configuration is shown or kept in the history; saving a masked value back +configuration is shown or kept in a backup; saving a masked value back keeps the real one. A missing or malformed key (not 64 hexadecimal characters) makes the whole file invalid, so it cannot be replaced by a short, guessable one. diff --git a/docs/config.zh-CN.md b/docs/config.zh-CN.md index 1940b245..e6ba2936 100644 --- a/docs/config.zh-CN.md +++ b/docs/config.zh-CN.md @@ -13,7 +13,7 @@ ThinkWatch Core 只读一个文件：`config.yaml`。本文逐项说明其中每 | macOS、Linux | `~/.thinkwatch/config.yaml` | | Windows | `%APPDATA%\ThinkWatch\config.yaml` | -`THINKWATCH_HOME` 替换整个目录；`--config <路径>` 为单条命令指定文件。core 的其余数据也在这个目录里：请求数据库（`data.db`）、配置历史（`history/`）、下载的价目表（`model_prices.json`），以及本地控制通道的 socket 文件（`twcore.sock`；Windows 上是回环端口，记录在 `control.port` 中）。目录只有所有者可访问（`0700`），配置文件权限为 `0600`：其中以明文保存密钥。 +`THINKWATCH_HOME` 替换整个目录；`--config <路径>` 为单条命令指定文件。core 的其余数据也在这个目录里：请求数据库（`data.db`）、配置的备份（`history/`）、下载的价目表（`model_prices.json`），以及本地控制通道的 socket 文件（`twcore.sock`；Windows 上是回环端口，记录在 `control.port` 中）。目录只有所有者可访问（`0700`），配置文件权限为 `0600`：其中以明文保存密钥。 这两项权限就是这份文件在本机上的全部保护：挡得住其他用户，挡不住以同一用户身份运行的程序。这样的程序能读到文件里的每一把密钥，也能用其中的控制通道密钥经控制通道修改配置。出站脱敏不改变这一点：它保护的是请求带出本机的内容，不是磁盘上的文件。显示配置时给控制通道密钥打码，是为了让经控制通道的写入改不了它（见 [`listen.control`](#cfg-listen-control)），而不是对本机程序隐藏它；上游密钥和网关密钥按原样显示。 @@ -65,19 +65,25 @@ providers: 两方同时修改时（应用和手工编辑），后写入的一方因版本不一致被拒绝，不会覆盖先写入的内容。 -### 历史与回滚 +### 备份、导出与导入 -每个生效过的版本都保存在配置文件旁边的 `history/` 目录中，并记录来源（应用、命令行、外部编辑、回滚、凭据轮换）。保留最近 50 个版本。 +配置每改动一次，配置文件旁边的 `history/` 目录中就留下一个版本：一个配置包（`.twconfig`），连同插件和应用的通用偏好，并记录来源（应用、命令行、外部编辑、导入、恢复、同步）。保留最近 10 个版本；凭据轮换不生成版本。版本中的控制面密钥一律打码。 ```sh twcore check # 只校验配置，不启动任何服务 twcore config show # 打印当前配置及其版本 -twcore config history # 列出历史版本，最新的在前 -twcore config rollback 3f9a2c # 回滚到某个版本（写前几位即可） +twcore backup list # 列出备份版本，最新的在前 +twcore backup restore 1792315800000 --dry-run # 恢复会改动什么 +twcore backup restore 1792315800000 # 恢复到某个版本 +twcore export -o all.twconfig # 把全部配置导出为配置包 +twcore export --providers relay --no-secrets --encrypt -o relay.twconfig +twcore import all.twconfig --dry-run # 导入会做什么 twcore config set /listen/gateway/port 8790 --int ``` -这些命令直接操作文件，因此 core 没有运行时也能使用，而那往往正是最需要回滚的时候。正在运行的 core 会像对待其他保存一样接收这些改动。 +恢复时保留本机设置（监听、控制面密钥、保留期限、同步设置）、ChatGPT 的登录，以及本机已接管的客户端正在使用的网关密钥。导入时加入本机没有的项目，与本机不同的项目逐项选择处理方式（`--on-conflict` 一次指定全部）。配置包不含本机设置与 ChatGPT 登录；未指定 `--no-secrets` 时其中以明文保存密钥，除非指定 `--encrypt`（口令在终端输入，或取自环境变量 `TWCORE_PASSPHRASE`）。 + +这些命令直接操作文件，因此 core 没有运行时也能使用，而那往往正是最需要恢复的时候；配置完全无法读取时，恢复把整个版本写回，控制面密钥保持现在的值。正在运行的 core 会像对待其他保存一样接收这些改动。 `twcore config set <路径> <值>` 修改文件中已经写出的一个值。路径中的列表项按其 `name` 定位（`/providers/anthropic/base_url`）；数字表示下标（`/routes/0/rules/1/to`）。值默认按字符串写入，`--int`、`--bool`、`--null` 另作指定。写入前先校验结果。要添加文件中还没有的字段，请直接编辑文件。 @@ -164,7 +170,7 @@ listen: | `remote` | 对象，见 [`listen.control.remote`](#cfg-listen-control-remote) | — | 供另一台机器上的桌面应用连接的网络端口。它是本地通道之外额外开的，不取代本地通道。 | -`key` 缺失时，由 `twcore serve` 在控制通道开始监听之前写入；`twcore init` 生成的配置也带有它。凡是显示配置或写入配置历史的地方，这个字段一律打码；把打码值原样存回时保留原值。密钥缺失或格式不对（不是 64 个十六进制字符）时整份配置无效，因此无法把它换成一把容易猜到的短密钥。 +`key` 缺失时，由 `twcore serve` 在控制通道开始监听之前写入；`twcore init` 生成的配置也带有它。凡是显示配置或写入备份的地方，这个字段一律打码；把打码值原样存回时保留原值。密钥缺失或格式不对（不是 64 个十六进制字符）时整份配置无效，因此无法把它换成一把容易猜到的短密钥。 ```sh twcore control-key # 显示密钥，用于粘贴到桌面应用 diff --git a/scripts/smoke.sh b/scripts/smoke.sh index 2f18b3fa..b2d3768e 100755 --- a/scripts/smoke.sh +++ b/scripts/smoke.sh @@ -524,7 +524,7 @@ for ep in "/summary?from_ms=$DAY" "/summary/buckets?from_ms=$DAY&bucket_ms=36000 done for ep in /status /overview /summary /history /latency /latency/provider /latency/client /storage /quota /security \ - /security/events /sessions /diagnostics /config /config/history /models /in-flight /live \ + /security/events /sessions /diagnostics /config /config/items /backups /client-prefs /models /in-flight /live \ /upstreams/health /plugins; do C=$(get "$ep") [ "$C" = "200" ] && ok "GET $ep" || bad "GET $ep 返回 $C" From 23d64a489e24bfad7516f60da2bb513c44e83924 Mon Sep 17 00:00:00 2001 From: fylorn <249551762+fylorn@users.noreply.github.com> Date: Mon, 19 Oct 2026 10:43:32 +0800 Subject: [PATCH 03/13] Packs: repair older packs, plugin scope refs, OAuth tokens stay local (#351) - A pack exported by an older version whose config does not read is repaired in memory by the one-click repair rules (unknown fields removed, unknown values back to the default) before preview and import; ImportPreview and RestorePreview list the repairs. A pack from a newer core is still too new. - A plugin's scope (match.upstreams) refers to the upstreams it names, so an export brings them, skipping one refuses the plugin on import, and a rename follows. Wildcard patterns are not references, and an upstream that is nowhere is not an error for a plugin. - OAuth upstreams: the refresh token rotates (the gateway writes back the new one), so the tokens stay on the device like a ChatGPT sign-in. They are not exported, an overwrite keeps the local ones when the token endpoint is the same, a new one arrives disabled and flagged missing_key, and a restore brings back the tokens of an upstream that is no longer here from the version restored. - control.config_not_repairable becomes control.config_unrepairable and points at restoring a backup. Co-authored-by: Claude Opus 5.5 --- Cargo.lock | 1 + bin/twcore/src/pack_cli.rs | 20 +++ crates/tw-api/msg-codes.txt | 2 +- crates/tw-api/src/lib.rs | 7 +- crates/tw-api/src/pack.rs | 6 + crates/tw-control/src/backups.rs | 39 ++++- crates/tw-control/src/lib.rs | 4 +- crates/tw-control/tests/backups.rs | 61 ++++++++ crates/tw-control/tests/safe_mode.rs | 2 +- crates/tw-pack/Cargo.toml | 2 + crates/tw-pack/src/export.rs | 4 +- crates/tw-pack/src/import.rs | 60 +++++++- crates/tw-pack/src/import/tests.rs | 165 ++++++++++++++++++++++ crates/tw-pack/src/lib.rs | 1 + crates/tw-pack/src/model/items.rs | 52 ++++++- crates/tw-pack/src/model/mod.rs | 7 +- crates/tw-pack/src/model/rename.rs | 12 ++ crates/tw-pack/src/model/tests.rs | 99 +++++++++++++ crates/tw-pack/src/repair.rs | 204 +++++++++++++++++++++++++++ crates/tw-pack/src/restore.rs | 85 +++++++++-- crates/tw-pack/src/restore/tests.rs | 31 ++++ 21 files changed, 834 insertions(+), 30 deletions(-) create mode 100644 crates/tw-pack/src/repair.rs diff --git a/Cargo.lock b/Cargo.lock index 7f74297d..3557739c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3900,6 +3900,7 @@ dependencies = [ "tw-config", "tw-engine", "tw-pricing", + "tw-secret", "tw-types", "tw-yaml", "zip", diff --git a/bin/twcore/src/pack_cli.rs b/bin/twcore/src/pack_cli.rs index dcc4c79e..8287bcf8 100644 --- a/bin/twcore/src/pack_cli.rs +++ b/bin/twcore/src/pack_cli.rs @@ -169,6 +169,22 @@ fn describe(id: &ItemId) -> String { } } +/// 较旧版本写的、读不进来的字段在内存里修过的那几处 +fn print_repairs(fixes: &[tw_config::repair::Fix]) { + if fixes.is_empty() { + return; + } + println!("written by an older version; repaired before reading:"); + for f in fixes { + let what = match f.kind { + tw_config::repair::FixKind::UnknownValue => "back to the default", + tw_config::repair::FixKind::UnknownField => "removed", + }; + let line = f.line.map(|l| format!(" (line {l})")).unwrap_or_default(); + println!(" {}{line}: {what}", f.field); + } +} + // ─────────────────────────────────────────────────────────── export pub fn export( @@ -248,6 +264,8 @@ pub fn import( ) -> Result<()> { use tw_pack::import::{Action, Decision, Status}; let incoming = read_pack(file)?; + let (incoming, repairs) = tw_pack::import::prepare(incoming).map_err(|e| anyhow!("{e}"))?; + print_repairs(&repairs); let (cur, local) = current(path, PackOrigin::Import)?; let ctx = tw_pack::import::Context { has_aws_profile: &|p| tw_config::aws_profile::credentials(p).is_ok(), @@ -357,6 +375,8 @@ pub fn restore(path: &Path, id: &str, dry_run: bool) -> Result<()> { use tw_pack::restore::{Change, Kept}; let b = tw_pack::backup::find(path, id).map_err(|e| anyhow!("{e}"))?; let backup = tw_pack::backup::open(&b).map_err(|e| anyhow!("{e}"))?; + let (backup, repairs) = tw_pack::import::prepare(backup).map_err(|e| anyhow!("{e}"))?; + print_repairs(&repairs); let (cur, local) = current(path, PackOrigin::Restore)?; let pinned = tw_pack::disk::read_pinned(path); let after = if tw_pack::model::snapshot(&local).is_err() { diff --git a/crates/tw-api/msg-codes.txt b/crates/tw-api/msg-codes.txt index ee9b9edd..b86e8b00 100644 --- a/crates/tw-api/msg-codes.txt +++ b/crates/tw-api/msg-codes.txt @@ -141,8 +141,8 @@ control.chatgpt_login.expired control.chatgpt_login.no_code control.chatgpt_login.page_error control.client_unknown -control.config_not_repairable control.config_stale +control.config_unrepairable control.control_key_locked control.default_key_cannot_delete control.default_key_cannot_disable diff --git a/crates/tw-api/src/lib.rs b/crates/tw-api/src/lib.rs index b1924591..90f1cc86 100644 --- a/crates/tw-api/src/lib.rs +++ b/crates/tw-api/src/lib.rs @@ -972,8 +972,11 @@ pub const MSG_CODES: &str = include_str!("../msg-codes.txt"); /// [`Event::GatewayKeysChanged`]。[`ConfigOrigin`] 删了 `rollback`，多了 `sync`、`import`、 /// `restore`、`repair`（一键修复以前记成 `ui`）。备份相关的端点在安全模式下照样可用。配置包的 /// 码是 `pack.*`，导入的是 `import.*`，备份与恢复的是 `backup.*`。停用着的上游可以缺凭据 -/// （密钥、OAuth、AWS 访问密钥空着，ChatGPT 账号没登录）：导入时就是这样进来的。照 49 写的 -/// 界面调不到删掉的那两个端点。 +/// （密钥、OAuth、AWS 访问密钥空着，ChatGPT 账号没登录）：导入时就是这样进来的。较旧版本 +/// 导出的包、升级之前留的版本里读不进来的字段按一键修复的规矩在内存里修好，[`ImportPreview`] +/// 和 [`RestorePreview`] 的 `repairs` 列出修了哪几处。OAuth 上游换发来的令牌（刷新令牌会轮换） +/// 和 ChatGPT 的登录一样留在本机，不随导出带走、导入覆盖时留着本机的。一键修复修不了时的码换成 +/// `control.config_unrepairable`（句子改说从备份恢复）。照 49 写的界面调不到删掉的那两个端点。 pub const CONTROL_API_VERSION: u32 = 50; /// `/live` 的速度看最近多久（见 [`LiveView::tokens_per_sec`]），毫秒 diff --git a/crates/tw-api/src/pack.rs b/crates/tw-api/src/pack.rs index b5b2b242..689c940a 100644 --- a/crates/tw-api/src/pack.rs +++ b/crates/tw-api/src/pack.rs @@ -180,6 +180,9 @@ pub struct ImportPreview { #[serde(default, skip_serializing_if = "Option::is_none")] #[cfg_attr(feature = "ts", ts(type = "unknown"))] pub client_prefs: Option, + /// 较旧版本导出的包里读不进来的字段，按一键修复的规矩（不认识的字段删掉、取值不合法的 + /// 回到默认值）**在内存里修过了**：预览和导入都按修好的算，包本身不变。没修过是空的 + pub repairs: Vec, } slug_enum! { @@ -320,6 +323,9 @@ pub struct RestorePreview { pub rows: Vec, /// 开着同步时，恢复的结果会同步到几台别的设备。没开同步是 0 pub sync_devices: u32, + /// 那一版里这一版 Core 读不进来的字段（升级之前留的版本），按一键修复的规矩在内存里修过了。 + /// 没修过是空的 + pub repairs: Vec, } slug_enum! { diff --git a/crates/tw-control/src/backups.rs b/crates/tw-control/src/backups.rs index 286a91a9..3bf7b060 100644 --- a/crates/tw-control/src/backups.rs +++ b/crates/tw-control/src/backups.rs @@ -191,8 +191,12 @@ fn bad_pack(e: PackError) -> Fail { /// 配置（本机的、包里的）拆不开。**由更新的版本写的**说成「太新」 fn model_fail(e: model::ModelError, pack: Option<&Pack>) -> Fail { + let newer = |p: &Pack| { + matches!(e, model::ModelError::TooNew { .. }) + || (e.is_too_new() && p.manifest.from_newer_core()) + }; if let Some(p) = pack - && e.is_too_new() + && newer(p) { return bad_pack(PackError::TooNew { format: p.manifest.format, @@ -273,6 +277,20 @@ fn file_of(pack: &Pack, pass: Option<&str>) -> Result<(String, String), Fail> { )) } +fn fix_view(f: tw_config::repair::Fix) -> tw_api::ConfigFix { + use tw_config::repair::FixKind; + tw_api::ConfigFix { + kind: match f.kind { + FixKind::UnknownValue => tw_api::ConfigFixKind::UnknownValue, + FixKind::UnknownField => tw_api::ConfigFixKind::UnknownField, + }, + field: f.field, + line: f.line, + value: f.value, + now: f.now, + } +} + // ─────────────────────────────────────────────────────────── 此刻的样子 /// 磁盘上此刻的配置（原文和版本）连同它的插件、通用偏好 @@ -430,6 +448,7 @@ fn import_fail(e: tw_pack::import::ImportError, pack: &Pack) -> Fail { E::Exists(_) | E::NameTaken(..) => fail(StatusCode::CONFLICT, e.msg()), E::NotInPack(_) => fail(StatusCode::NOT_FOUND, e.msg()), E::Item(..) => fail(StatusCode::UNPROCESSABLE_ENTITY, e.msg()), + E::TooNew(p) => bad_pack(p), } } @@ -464,7 +483,9 @@ async fn import_preview( State(s): State, Json(req): Json, ) -> Result, Fail> { - let incoming = open_pack(&req.data_b64, req.passphrase.as_deref())?; + let raw = open_pack(&req.data_b64, req.passphrase.as_deref())?; + let (incoming, repairs) = + tw_pack::import::prepare(raw.clone()).map_err(|e| import_fail(e, &raw))?; let (cur, local) = current(&s, PackOrigin::Import)?; let rows = tw_pack::import::preview(&local, &incoming, &import_ctx()) .map_err(|e| import_fail(e, &incoming))?; @@ -473,6 +494,7 @@ async fn import_preview( base_version: cur.version(), rows: rows.into_iter().map(row_view).collect(), client_prefs: incoming.client_prefs.clone(), + repairs: repairs.into_iter().map(fix_view).collect(), })) } @@ -480,7 +502,9 @@ async fn import( State(s): State, Json(req): Json, ) -> Result, Fail> { - let incoming = open_pack(&req.data_b64, req.passphrase.as_deref())?; + let raw = open_pack(&req.data_b64, req.passphrase.as_deref())?; + // 预览时修过的那几处，这里照样修一遍（同一份字节，修出来一样） + let (incoming, _) = tw_pack::import::prepare(raw.clone()).map_err(|e| import_fail(e, &raw))?; let (cur, local) = current(&s, PackOrigin::Import)?; if cur.version() != req.base_version { return Err(apply_fail(ApplyError::Stale { @@ -663,7 +687,10 @@ async fn restore_preview( State(s): State, Json(req): Json, ) -> Result, Fail> { - let backup = backup_pack(&s, &req.id).await?; + let raw = backup_pack(&s, &req.id).await?; + // 升级之前留的版本可能有这一版不认识的写法：和导入一样在内存里修好 + let (backup, repairs) = + tw_pack::import::prepare(raw.clone()).map_err(|e| import_fail(e, &raw))?; let (cur, local) = current(&s, PackOrigin::Restore)?; let sync_devices = match s.cfg.remote_backups() { Some(r) => r.other_devices().await, @@ -693,6 +720,7 @@ async fn restore_preview( base_version: cur.version(), rows, sync_devices, + repairs: repairs.into_iter().map(fix_view).collect(), })) } @@ -700,7 +728,8 @@ async fn restore( State(s): State, Json(req): Json, ) -> Result, Fail> { - let backup = backup_pack(&s, &req.id).await?; + let raw = backup_pack(&s, &req.id).await?; + let (backup, _) = tw_pack::import::prepare(raw.clone()).map_err(|e| import_fail(e, &raw))?; let (cur, local) = current(&s, PackOrigin::Restore)?; if cur.version() != req.base_version { return Err(apply_fail(ApplyError::Stale { diff --git a/crates/tw-control/src/lib.rs b/crates/tw-control/src/lib.rs index f6720f71..a5c3eaec 100644 --- a/crates/tw-control/src/lib.rs +++ b/crates/tw-control/src/lib.rs @@ -1549,9 +1549,9 @@ async fn repair_config( return Err(fail( StatusCode::BAD_REQUEST, msg!( - "control.config_not_repairable" => + "control.config_unrepairable" => "The configuration has no error that can be repaired automatically. Edit the \ - configuration file, or roll back to an earlier version" + configuration file, or restore an earlier version from the backups" ), )); }; diff --git a/crates/tw-control/tests/backups.rs b/crates/tw-control/tests/backups.rs index dfab9e05..69b8e6d3 100644 --- a/crates/tw-control/tests/backups.rs +++ b/crates/tw-control/tests/backups.rs @@ -564,3 +564,64 @@ async fn in_safe_mode_a_backup_comes_back_whole() { let text = std::fs::read_to_string(&p).unwrap(); assert_eq!(text, mine()); } + +/// 较旧版本导出的包：读不进来的字段在内存里修好，预览说修了哪几处；更新的版本写的说太新 +#[tokio::test] +async fn an_older_pack_is_previewed_repaired_and_a_newer_one_is_too_new() { + let dst = bed(&mine()); + let pack_b64 = |core: &str| { + let mut m = tw_pack::Manifest::new(tw_pack::PackKind::Full, tw_pack::PackOrigin::Export); + m.core_version = core.to_string(); + let p = tw_pack::Pack { + manifest: m, + config_yaml: "version: 1\nproviders:\n - name: old\n base_url: https://old.example\n retired_field: 1\nclient_probes:\n titling: passthrough\n".into(), + plugins: Default::default(), + client_prefs: None, + }; + base64::engine::general_purpose::STANDARD + .encode(tw_pack::write(&p, tw_pack::Seal::None).unwrap()) + }; + let old = pack_b64("0.1.0"); + let (st, preview) = call( + &dst.app, + "POST", + "/config/import/preview", + serde_json::json!({ "data_b64": old }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{preview}"); + let fields: Vec<&str> = preview["repairs"] + .as_array() + .unwrap() + .iter() + .map(|f| f["field"].as_str().unwrap()) + .collect(); + assert_eq!( + fields, + ["providers[0].retired_field", "client_probes.titling"] + ); + assert_eq!(preview["repairs"][1]["kind"], "unknown_value"); + let (st, v) = call( + &dst.app, + "POST", + "/config/import", + serde_json::json!({ + "data_b64": old, + "base_version": preview["base_version"], + "decisions": [{ "item": { "kind": "provider", "name": "old" }, "action": "import" }], + }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{v}"); + assert!(dst.file().contains("https://old.example")); + assert!(!dst.file().contains("retired_field")); + let (st, v) = call( + &dst.app, + "POST", + "/config/import/preview", + serde_json::json!({ "data_b64": pack_b64("999.0.0") }), + ) + .await; + assert_eq!(st, StatusCode::BAD_REQUEST, "{v}"); + assert_eq!(v["code"], "pack.too_new"); +} diff --git a/crates/tw-control/tests/safe_mode.rs b/crates/tw-control/tests/safe_mode.rs index 6c4f009c..fb2fd5af 100644 --- a/crates/tw-control/tests/safe_mode.rs +++ b/crates/tw-control/tests/safe_mode.rs @@ -165,7 +165,7 @@ async fn an_error_that_is_not_fixable_offers_nothing() { ) .await; assert_eq!(st, StatusCode::BAD_REQUEST, "{e}"); - assert_eq!(e["code"], "control.config_not_repairable", "{e}"); + assert_eq!(e["code"], "control.config_unrepairable", "{e}"); } #[tokio::test] diff --git a/crates/tw-pack/Cargo.toml b/crates/tw-pack/Cargo.toml index d842fd96..ba3b604b 100644 --- a/crates/tw-pack/Cargo.toml +++ b/crates/tw-pack/Cargo.toml @@ -18,6 +18,8 @@ tw-engine = { workspace = true } tw-pricing = { workspace = true } tw-types = { workspace = true } tw-yaml = { workspace = true } +# 修导入的旧配置包时，原值给人看之前脱敏 +tw-secret = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } serde_yaml_ng = { workspace = true } diff --git a/crates/tw-pack/src/export.rs b/crates/tw-pack/src/export.rs index 17d50c47..30d72f81 100644 --- a/crates/tw-pack/src/export.rs +++ b/crates/tw-pack/src/export.rs @@ -29,7 +29,7 @@ pub struct Exported { pub pack: Pack, /// 包里有哪些项目（随附的也在），顺序项目不算 pub included: Vec, - /// 登录没有随包带走的上游（ChatGPT 账号） + /// 登录没有随包带走的上游：ChatGPT 账号，和令牌留在本机的 OAuth 上游 pub omitted_logins: Vec, } @@ -81,7 +81,7 @@ pub fn export_with( let omitted_logins = cfg .providers .iter() - .filter(|p| picked.contains(p.name.as_str()) && model::is_chatgpt(p)) + .filter(|p| picked.contains(p.name.as_str()) && model::holds_login(p)) .map(|p| p.name.clone()) .collect(); let kind = if opts.scope.is_some() { diff --git a/crates/tw-pack/src/import.rs b/crates/tw-pack/src/import.rs index 646f8280..4621a094 100644 --- a/crates/tw-pack/src/import.rs +++ b/crates/tw-pack/src/import.rs @@ -103,6 +103,36 @@ pub enum ImportError { /// 导进来之后配置过不了校验 #[error("{}", self.msg())] Rejected(Box), + /// 由更新的版本写的 + #[error("{}", self.msg())] + TooNew(crate::PackError), +} + +/// 导入之前：包里的配置读得进来吗。由更新的版本写的是「太新」；较旧版本写的、有读不进来的 +/// 字段的，按一键修复的规矩在内存里修好（[`crate::repair`]），连同修了哪几处一起交回去 +pub fn prepare(incoming: Pack) -> Result<(Pack, Vec), ImportError> { + let e = match model::snapshot(&incoming) { + Ok(_) => return Ok((incoming, Vec::new())), + Err(e) => e, + }; + let newer = matches!(e, ModelError::TooNew { .. }) + || (e.is_too_new() && incoming.manifest.from_newer_core()); + if newer { + return Err(ImportError::TooNew(crate::PackError::TooNew { + format: incoming.manifest.format, + core_version: incoming.manifest.core_version.clone(), + })); + } + match crate::repair::repair(&incoming.config_yaml) { + Some((text, fixes)) => Ok(( + Pack { + config_yaml: text, + ..incoming + }, + fixes, + )), + None => Err(ImportError::Model(e)), + } } impl From for ImportError { @@ -130,6 +160,7 @@ impl ImportError { "The configuration pack has no {kind} named `{name}`." ), ImportError::Rejected(r) => r.msg(), + ImportError::TooNew(e) => e.msg(), } } } @@ -249,6 +280,15 @@ fn flags(id: &ItemId, item: &Item, incoming: &Pack, l: &Snapshot) -> Vec { if !here { out.push(Flag::NeedsLogin); } + } else if id.kind == ItemKind::Provider + && let Some(endpoint) = oauth_endpoint(item) + { + // OAuth 的令牌不随包带（会轮换，见 `model`）：本机同名的那一家用着同一个令牌端点时， + // 覆盖留着本机的令牌；否则要填上刷新令牌 + let here = l.items.get(id).and_then(oauth_endpoint) == Some(endpoint); + if !here && !out.contains(&Flag::MissingKey) { + out.push(Flag::MissingKey); + } } if id.kind == ItemKind::Plugin { out.push(Flag::PluginDisabled); @@ -256,6 +296,16 @@ fn flags(id: &ItemId, item: &Item, incoming: &Pack, l: &Snapshot) -> Vec { out } +/// 一个上游项目的 OAuth 令牌端点。不是 OAuth 的是 None +fn oauth_endpoint(item: &Item) -> Option { + item.units + .get(unit::CONN)? + .get("oauth")? + .get("endpoint")? + .as_str() + .map(str::to_string) +} + fn provider_is_chatgpt(id: &ItemId, item: &Item) -> bool { model::provider_of(id, item).is_ok_and(|p| model::is_chatgpt(&p)) } @@ -273,9 +323,10 @@ fn problem( ctx: &Context, codec: &dyn PluginCodec, ) -> Option { - // 引用了本机和包里都没有的 + // 引用了本机和包里都没有的。**插件不算**：它范围里写的上游这里没有，只是管不着什么， + // 不是导入不了 for r in &both.items[id].refs { - if !both.items.contains_key(r) && !builtin(r) { + if id.kind != ItemKind::Plugin && !both.items.contains_key(r) && !builtin(r) { return Some(missing_ref(id, r)); } } @@ -540,10 +591,11 @@ pub fn apply_with( } target.relink(); - // 导进来的项目引用的东西都得在：被跳过的那些不在 + // 导进来的项目引用的东西都得在：被跳过的那些不在。插件只看包里有、却被跳过的那些 for id in &taken { for r in &target.items[id].refs { - if !target.items.contains_key(r) && !builtin(r) { + let soft = id.kind == ItemKind::Plugin && !i.items.contains_key(r); + if !soft && !target.items.contains_key(r) && !builtin(r) { return Err(ImportError::Item(id.clone(), Box::new(skipped_ref(id, r)))); } } diff --git a/crates/tw-pack/src/import/tests.rs b/crates/tw-pack/src/import/tests.rs index f66a9396..242a2b5e 100644 --- a/crates/tw-pack/src/import/tests.rs +++ b/crates/tw-pack/src/import/tests.rs @@ -522,3 +522,168 @@ fn client_preferences_import_only_when_core_keeps_them() { .unwrap(); assert_eq!(applied.pack.client_prefs, Some(json!({"language": "en"}))); } + +/// 第一行是 `// data: ` 的插件源码：数据单独拆出来（同 `model` 的测试） +struct DataLine; + +impl PluginCodec for DataLine { + fn split(&self, source: &str) -> Option<(String, serde_json::Value)> { + let (first, rest) = source.split_once('\n')?; + let data = serde_json::from_str(first.strip_prefix("// data: ")?).ok()?; + Some((format!("// data: null\n{rest}"), data)) + } + fn join(&self, source: &str, data: &serde_json::Value) -> Option { + let (_, rest) = source.split_once('\n')?; + Some(format!("// data: {data}\n{rest}")) + } + fn shipped(&self, _: &str) -> Option { + None + } + fn needs_approval(&self, _: &str) -> bool { + false + } +} + +/// OAuth 上游的令牌不随包带：新来的停用着、标成缺密钥；覆盖本机同一个令牌端点的留着本机的令牌 +#[test] +fn an_oauth_upstream_arrives_without_its_tokens() { + let oauth = " - name: corp\n base_url: https://corp.example\n oauth:\n refresh: rt-theirs\n endpoint: https://auth.example/token\n"; + let incoming = pack_of( + &format!( + "version: 1\nproviders:\n{oauth} - name: corp2\n base_url: https://corp2.example\n oauth:\n refresh: rt-2\n endpoint: https://auth.example/token\n max_concurrent: 2\n" + ), + true, + ); + let local = pack_of( + &local_text("").replace( + "providers:\n", + "providers:\n - name: corp2\n base_url: https://corp2.example\n oauth:\n refresh: rt-mine\n endpoint: https://auth.example/token\n", + ), + true, + ); + let r = rows(&local, &incoming); + assert_eq!(r["provider `corp`"].flags, [Flag::MissingKey]); + assert_eq!(r["provider `corp2`"].flags, Vec::::new()); + assert_eq!(r["provider `corp2`"].status, Status::Conflict); + let applied = apply_with( + &local, + &incoming, + &[ + Decision { + item: ItemId::new(ItemKind::Provider, "corp"), + action: Action::Import, + }, + Decision { + item: ItemId::new(ItemKind::Provider, "corp2"), + action: Action::Overwrite, + }, + ], + &ctx(), + &WholeFile, + ) + .unwrap(); + let cfg = tw_config::try_parse(&applied.pack.config_yaml).unwrap(); + let p = |n: &str| cfg.providers.iter().find(|p| p.name == n).unwrap(); + assert!(p("corp").disabled); + assert_eq!(p("corp").oauth.as_ref().unwrap().refresh, ""); + assert!(!applied.pack.config_yaml.contains("rt-theirs")); + assert_eq!(p("corp2").oauth.as_ref().unwrap().refresh, "rt-mine"); + assert_eq!(p("corp2").max_concurrent, Some(2)); + assert!(!p("corp2").disabled); +} + +/// 插件范围里写明的上游：跳过它，插件就导入不了；本机和包里都没有的不碍事 +#[test] +fn skipping_an_upstream_a_plugin_names_refuses_the_plugin() { + let src = "// data: {\"match\":{\"upstreams\":[\"relay\",\"nowhere\"]}}\nscoped\n"; + let sha = model::sha256_hex(src.as_bytes()); + let mut incoming = pack_of( + &format!( + "version: 1\nproviders:\n - name: relay\n base_url: https://relay.example\nplugins:\n - id: scoped\n file: plugins/scoped.js\n sha256: {sha}\n" + ), + true, + ); + incoming.plugins.insert( + "scoped".into(), + PluginFiles { + script: src.as_bytes().to_vec(), + approved: Some(src.as_bytes().to_vec()), + }, + ); + let local = pack_of(&local_text(""), true); + let r: BTreeMap = preview_with(&local, &incoming, &ctx(), &DataLine) + .unwrap() + .into_iter() + .map(|r| (r.item.to_string(), r)) + .collect(); + let plugin = &r["plugin `scoped`"]; + assert_eq!(plugin.status, Status::New, "{plugin:?}"); + assert_eq!( + plugin + .refs + .iter() + .map(|r| r.to_string()) + .collect::>(), + ["provider `relay`"] + ); + let d = |kind, name: &str, action| Decision { + item: ItemId::new(kind, name), + action, + }; + let e = apply_with( + &local, + &incoming, + &[ + d(ItemKind::Plugin, "scoped", Action::Import), + d(ItemKind::Provider, "relay", Action::Skip), + ], + &ctx(), + &DataLine, + ) + .unwrap_err(); + assert_eq!(e.msg().code, "import.skipped_reference"); + // 一起导入就成；改名导入的上游，插件的范围跟着改 + let applied = apply_with( + &local, + &incoming, + &[ + d(ItemKind::Plugin, "scoped", Action::Import), + d( + ItemKind::Provider, + "relay", + Action::Rename("relay-cn".into()), + ), + ], + &ctx(), + &DataLine, + ) + .unwrap(); + let script = String::from_utf8(applied.pack.plugins["scoped"].script.clone()).unwrap(); + assert!(script.contains("relay-cn"), "{script}"); +} + +/// 较旧版本导出的、有这一版不认识的写法的包：修好再看；更新的版本写的说太新 +#[test] +fn an_older_pack_is_repaired_and_a_newer_one_is_too_new() { + let text = + "version: 1\nproviders:\n - name: a\n base_url: https://a.example\n retired: 1\n"; + let mut old = pack_of(text, true); + old.manifest.core_version = "0.1.0".into(); + let (fixed, fixes) = prepare(old.clone()).unwrap(); + assert_eq!(fixes.len(), 1); + assert_eq!(fixes[0].field, "providers[0].retired"); + assert!(!fixed.config_yaml.contains("retired")); + let r = rows(&pack_of(&local_text(""), true), &fixed); + assert_eq!(r["provider `a`"].status, Status::New); + // 读得进来的不动 + let (same, none) = prepare(pack_of("version: 1\n", true)).unwrap(); + assert!(none.is_empty()); + assert_eq!(same.config_yaml, "version: 1\n"); + // 更新的 Core 写的 + let mut new = old.clone(); + new.manifest.core_version = "999.0.0".into(); + assert_eq!(prepare(new).unwrap_err().msg().code, "pack.too_new"); + // 修不了的照旧是读不进来的那句话 + let broken = pack_of("version: 1\nproviders: [\n", true); + assert!(prepare(broken).is_err()); +} diff --git a/crates/tw-pack/src/lib.rs b/crates/tw-pack/src/lib.rs index 789b1400..3ede6064 100644 --- a/crates/tw-pack/src/lib.rs +++ b/crates/tw-pack/src/lib.rs @@ -38,6 +38,7 @@ pub mod disk; pub mod export; pub mod import; pub mod model; +pub mod repair; pub mod restore; /// 配置包格式的版本。**读到更大的就拒绝**（[`PackError::TooNew`]）：新版本的格式旧版本 diff --git a/crates/tw-pack/src/model/items.rs b/crates/tw-pack/src/model/items.rs index 3f200741..8fcb73ad 100644 --- a/crates/tw-pack/src/model/items.rs +++ b/crates/tw-pack/src/model/items.rs @@ -206,7 +206,19 @@ fn provider_unit(field: &str) -> &'static str { pub(crate) const PROVIDER_UNITS: &[&str] = &[unit::CONN, unit::MODELS, unit::BILLING, unit::OTHER]; -/// 去掉这台设备自己的部分：ChatGPT 的登录和账户 ID 请求头 +/// OAuth 里属于这台设备的那几项：换发来的令牌。**刷新令牌会轮换**：很多 OAuth 服务器每刷新 +/// 一次就换发一个新的、作废旧的（`tw_gateway::oauth` 收到新的就写回配置），同一个刷新令牌 +/// 在两台设备上用，先刷新的那台让另一台的失效。所以它和 ChatGPT 的登录一样留在本机， +/// 端点、客户端 ID 和密钥照常比较、带走 +pub(crate) const OAUTH_TOKENS: &[&str] = &["refresh", "access", "expires_at"]; + +/// 这一家的凭据里有属于这台设备的登录：ChatGPT 账号，或者 OAuth 的令牌 +pub fn holds_login(p: &Provider) -> bool { + is_chatgpt(p) || p.oauth.is_some() +} + +/// 去掉这台设备自己的部分：ChatGPT 的登录和账户 ID 请求头（OAuth 的令牌在 +/// [`provider_units`] 里去掉：`refresh` 是必填的，结构体里去不掉） pub(crate) fn shareable(p: &Provider) -> Provider { let mut p = p.clone(); if is_chatgpt(&p) { @@ -228,6 +240,11 @@ fn provider_units(p: &Provider) -> Result, String> { return Err("it does not serialize to an object".into()); }; obj.remove("name"); + if let Some(Json::Object(o)) = obj.get_mut("oauth") { + for k in OAUTH_TOKENS { + o.remove(*k); + } + } // **请求头按写的顺序**：JSON 对象的键是排过序的，所以写成一串 [名字, 值] if obj.contains_key("headers") { obj.insert( @@ -262,7 +279,8 @@ fn provider_units(p: &Provider) -> Result, String> { } /// 上游的单位拼回配置里的那一项。`local` 是本机同名的那一家：它是 ChatGPT 账号、拼出来 -/// 的也是的话，登录和账户 ID 请求头取它的 +/// 的也是的话，登录和账户 ID 请求头取它的；两边都是 OAuth、令牌端点一样的话，令牌取它的。 +/// 本机没有令牌可取的 OAuth 上游，刷新令牌是空的（停用着才过得了校验，见导入、恢复） pub(crate) fn provider_mapping( id: &ItemId, units: &BTreeMap, @@ -283,7 +301,21 @@ pub(crate) fn provider_mapping( m.insert(key(k), value); } } + if let Some(Yaml::Mapping(o)) = m.get_mut("oauth") + && !o.contains_key("refresh") + { + o.insert(key("refresh"), Yaml::String(String::new())); + } let (mut p, _) = typed_mapping::(Yaml::Mapping(m)).map_err(|e| shape(id, e))?; + let chatgpt = is_chatgpt(&p); + if let (Some(mine), Some(theirs)) = (local.and_then(|l| l.oauth.as_ref()), p.oauth.as_mut()) + && !chatgpt + && mine.endpoint == theirs.endpoint + { + theirs.refresh = mine.refresh.clone(); + theirs.access = mine.access.clone(); + theirs.expires_at = mine.expires_at.clone(); + } if let Some(local) = local && is_chatgpt(&p) && is_chatgpt(local) @@ -713,6 +745,22 @@ fn refs_of( } } } + ItemKind::Plugin => { + // 范围里写明的上游（`match.upstreams`）。**通配的不算**：`relay-*` 说的是一批名字， + // 不是哪一个上游 + let listed = units + .get(unit::SETTINGS) + .and_then(|d| d.get("match")) + .and_then(|m| m.get("upstreams")) + .and_then(Json::as_array) + .cloned() + .unwrap_or_default(); + for n in listed.iter().filter_map(Json::as_str).map(str::trim) { + if !n.is_empty() && !n.contains(['*', '?', '[']) { + out.insert(ItemId::new(ItemKind::Provider, n)); + } + } + } ItemKind::Global => { if let Some(r) = s(units.get(global::DEFAULT_ROUTE)) { out.insert(ItemId::new(ItemKind::Route, r)); diff --git a/crates/tw-pack/src/model/mod.rs b/crates/tw-pack/src/model/mod.rs index 28974ee1..1691795e 100644 --- a/crates/tw-pack/src/model/mod.rs +++ b/crates/tw-pack/src/model/mod.rs @@ -26,8 +26,9 @@ //! # 本机设置 //! //! 不进快照，[`materialize`] 时保留本机的值：`listen`、`retention`、`sync` 三节，ChatGPT -//! 上游的登录（`oauth`）和登录所得的 `ChatGPT-Account-Id` 请求头，Core 自带的默认插件的 -//! 脚本（它的设置和开关照常比较）。 +//! 上游的登录（`oauth`）和登录所得的 `ChatGPT-Account-Id` 请求头，别的 OAuth 上游换发来的 +//! 令牌（`oauth.refresh` / `access` / `expires_at`：刷新令牌会轮换，两台设备共用一个会互相 +//! 作废），Core 自带的默认插件的脚本（它的设置和开关照常比较）。 use std::collections::{BTreeMap, BTreeSet}; use std::sync::OnceLock; @@ -45,7 +46,7 @@ mod rename; mod write; pub use codec::{PluginCodec, WholeFile}; -pub use items::{is_chatgpt, sha256_hex}; +pub use items::{holds_login, is_chatgpt, sha256_hex}; pub use rename::{free_name, rename}; pub(crate) use write::materialize_for_export; pub use write::{materialize, materialize_with}; diff --git a/crates/tw-pack/src/model/rename.rs b/crates/tw-pack/src/model/rename.rs index 3a84ec3e..5e84e1cb 100644 --- a/crates/tw-pack/src/model/rename.rs +++ b/crates/tw-pack/src/model/rename.rs @@ -79,6 +79,18 @@ pub fn rename(snap: &mut Snapshot, id: &ItemId, new_name: &str) { } } } + (ItemKind::Provider, ItemKind::Plugin) => { + if let Some(Json::Array(list)) = item + .units + .get_mut(unit::SETTINGS) + .and_then(|d| d.get_mut("match")) + .and_then(|m| m.get_mut("upstreams")) + { + for n in list { + swap(Some(n), old, new); + } + } + } (ItemKind::Proxy, ItemKind::Provider) => { swap( item.units diff --git a/crates/tw-pack/src/model/tests.rs b/crates/tw-pack/src/model/tests.rs index 3bbedfde..79c0af6c 100644 --- a/crates/tw-pack/src/model/tests.rs +++ b/crates/tw-pack/src/model/tests.rs @@ -1018,3 +1018,102 @@ fn retain_trims_the_orders_too() { json!(["relay"]) ); } + +// ─────────────────────────────────────────────────────────── OAuth 的令牌、插件的范围 + +const OAUTH: &str = " oauth:\n refresh: rt-mine\n access: at-mine\n expires_at: '2026-10-19T00:00:00Z'\n endpoint: https://auth.example/token\n client_id: cid\n"; + +fn with_oauth(endpoint: &str) -> String { + format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n key: tw-a\nproviders:\n - name: corp\n base_url: https://corp.example\n{}", + OAUTH.replace("https://auth.example/token", endpoint) + ) +} + +/// 刷新令牌会轮换：令牌留在本机，端点和客户端 ID 照常比较 +#[test] +fn oauth_tokens_stay_on_this_device() { + let p = pack(&with_oauth("https://auth.example/token")); + let s = snap(&p); + let conn = &s.items[&id(ItemKind::Provider, "corp")].units["conn"]; + assert_eq!( + conn["oauth"], + json!({"endpoint": "https://auth.example/token", "client_id": "cid"}) + ); + // 轮换不算改动 + let rotated = pack(&with_oauth("https://auth.example/token").replace("rt-mine", "rt-rotated")); + assert_eq!(snap(&rotated), s); + assert_eq!(rotated.content_id(), p.content_id()); + // 别处来的（同一个端点）改了别的：令牌留着本机的 + let mut t = s.clone(); + t.items + .get_mut(&id(ItemKind::Provider, "corp")) + .unwrap() + .units + .insert("other".into(), json!({"max_concurrent": 2})); + let out = mat(&p, &t); + assert!( + out.config_yaml.contains("refresh: rt-mine"), + "{}", + out.config_yaml + ); + assert!( + out.config_yaml.contains("access: at-mine"), + "{}", + out.config_yaml + ); + // 换了令牌端点：本机的令牌不是那个服务器发的，不留 + let mut t = snap(&pack(&with_oauth("https://other.example/token"))); + t.items + .get_mut(&id(ItemKind::Provider, "corp")) + .unwrap() + .units + .insert("other".into(), json!({"disabled": true})); + let out = mat(&p, &t); + let cfg = tw_config::try_parse(&out.config_yaml).unwrap(); + let o = cfg.providers[0].oauth.as_ref().unwrap(); + assert_eq!(o.endpoint, "https://other.example/token"); + assert_eq!(o.refresh, ""); + // 落到一份空配置上（导出）：令牌不在 + let out = mat(&pack("version: 1\n"), &s); + assert!(!out.config_yaml.contains("rt-mine"), "{}", out.config_yaml); +} + +fn with_scoped_plugin(upstreams: &str) -> Pack { + let src = format!("// data: {{\"match\":{{\"upstreams\":{upstreams}}}}}\nscoped code\n"); + let base = with_plugins(&[("scoped", &src, true)]); + Pack { + config_yaml: base.config_yaml.replace( + "plugins:\n", + "providers:\n - name: relay\n base_url: https://relay.example\n - name: other\n base_url: https://other.example\nplugins:\n", + ), + ..base + } +} + +/// 插件范围里写明的上游是它的引用：导出带上它，改名跟着改；通配的不算 +#[test] +fn a_plugin_refers_to_the_upstreams_its_scope_names() { + let p = with_scoped_plugin(r#"["relay", "rel*", " "]"#); + let s = snap(&p); + let refs: Vec = s.items[&id(ItemKind::Plugin, "scoped")] + .refs + .iter() + .map(|r| r.to_string()) + .collect(); + assert_eq!(refs, ["provider `relay`"]); + let picked = s.closure(&[id(ItemKind::Plugin, "scoped")]); + assert!(picked.contains(&id(ItemKind::Provider, "relay"))); + assert!(!picked.contains(&id(ItemKind::Provider, "other"))); + let mut s2 = s.clone(); + rename(&mut s2, &id(ItemKind::Provider, "relay"), "relay (2)"); + let settings = &s2.items[&id(ItemKind::Plugin, "scoped")].units["settings"]; + assert_eq!( + settings["match"]["upstreams"], + json!(["relay (2)", "rel*", " "]) + ); + // 改了名的落下去，插件文件里的范围跟着改 + let out = mat(&p, &s2); + let script = String::from_utf8(out.plugins["scoped"].script.clone()).unwrap(); + assert!(script.contains("relay (2)"), "{script}"); +} diff --git a/crates/tw-pack/src/repair.rs b/crates/tw-pack/src/repair.rs new file mode 100644 index 00000000..cdfad38b --- /dev/null +++ b/crates/tw-pack/src/repair.rs @@ -0,0 +1,204 @@ +//! 较旧版本导出的配置包：读不进来的字段按「一键修复」的规矩在内存里修好再导入。 +//! +//! 规矩和 `tw_config::repair` 一样，**只修两种、都只是删掉一个键**：取值不在可选范围里的 +//! （删掉就回到默认值）、不认识的字段（删掉）。不同的是这里只要求「读得进来」，不要求 +//! 「是一份完整、能用的配置」：一个配置包可以只有一个上游，没有网关密钥、没有控制面的钥匙， +//! 拿整份校验去要求它，什么都修不了。修完之后能不能用由导入那一步照常校验。 +//! +//! **更新的版本写的不修**：那时不认识的字段是旧版本不懂的新东西，删掉就丢了，要说的是 +//! 「请先更新」（[`crate::Manifest::from_newer_core`]）。 + +use tw_config::repair::{Fix, FixKind}; +use tw_yaml::Step; + +use crate::model::{self, ModelError}; + +/// 一份配置最多修几处。删一处冒一处的修不完 +const MAX_FIXES: usize = 20; + +/// 修好的原文和修了哪几处。读得进来（不用修）、或者修不了，都是 `None` +pub fn repair(text: &str) -> Option<(String, Vec)> { + let mut out = text.to_string(); + let mut fixes: Vec = Vec::new(); + for _ in 0..=MAX_FIXES { + let r = match model::parse(&out) { + Ok(cfg) => { + if fixes.is_empty() { + return None; + } + for f in fixes.iter_mut() { + if f.kind == FixKind::UnknownValue { + f.now = default_at(&cfg, &f.field); + } + } + return Some((out, fixes)); + } + Err(ModelError::Config(r)) => r, + Err(_) => return None, + }; + let kind = match r.message.code.as_str() { + "config.unknown_variant" => FixKind::UnknownValue, + "config.unknown_field" => FixKind::UnknownField, + _ => return None, + }; + let field = r.message.args.get("field")?.clone(); + let path = steps(&field)?; + // 行号和原值按**修之前的原文**找：前面删过行，`r.line` 已经对不上了 + let before = tw_yaml::find(text, &path).ok(); + let line = before + .as_ref() + .map(|f| text[..f.bytes.start].matches('\n').count() + 1) + .or(r.line); + let value = before.map(|f| masked(&path, &f.value)); + out = tw_yaml::remove_key(&out, &path).ok()?; + if fixes.len() == MAX_FIXES { + return None; + } + fixes.push(Fix { + kind, + field, + line, + value, + now: None, + }); + } + None +} + +/// 原值给人看之前脱敏：拼错名字的那个字段完全可能就是一把密钥。连着键名一起判断 +fn masked(path: &[Step], v: &str) -> String { + let name = match path.last() { + Some(Step::Key(k)) => k.as_str(), + _ => "value", + }; + let line = tw_secret::mask_line(&mask_hex_runs(&format!("{name}: {v}"))); + match line.split_once(": ") { + Some((_, v)) => v.to_string(), + None => line, + } +} + +/// 正好 64 个十六进制字符的一段（控制面的钥匙那样的）换成打码 +fn mask_hex_runs(s: &str) -> String { + let b = s.as_bytes(); + let (mut out, mut i, mut last) = (String::new(), 0, 0); + while i < b.len() { + if !b[i].is_ascii_hexdigit() { + i += 1; + continue; + } + let start = i; + while i < b.len() && b[i].is_ascii_hexdigit() { + i += 1; + } + if i - start == tw_api::control::KEY_HEX_LEN { + out.push_str(&s[last..start]); + out.push_str(tw_api::control::KEY_MASK); + last = i; + } + } + out.push_str(&s[last..]); + out +} + +/// 修完之后这个字段的值（也就是默认值），写成一个标量。写不成的是 None +fn default_at(cfg: &tw_config::Config, field: &str) -> Option { + use serde_yaml_ng::to_value; + let path = steps(field)?; + let (first, rest) = path.split_first()?; + let Step::Key(name) = first else { return None }; + // **一节单独序列化**：整份序列化时还是默认值的那几节整节不写，改回默认值的恰恰是它们 + let section = match name.as_str() { + "clients" => to_value(&cfg.clients), + "providers" => to_value(&cfg.providers), + "proxies" => to_value(&cfg.proxies), + "pricing" => to_value(&cfg.pricing), + "client_probes" => to_value(&cfg.client_probes), + "security" => to_value(&cfg.security), + "failover" => to_value(&cfg.failover), + "aliases" => to_value(&cfg.aliases), + "groups" => to_value(&cfg.groups), + "routes" => to_value(&cfg.routes), + "plugins" => to_value(&cfg.plugins), + _ => return None, + } + .ok()?; + let mut cur = §ion; + for s in rest { + cur = match s { + Step::Key(k) => cur.get(k.as_str())?, + Step::Index(i) => cur.get(*i)?, + }; + } + match cur { + serde_yaml_ng::Value::String(s) => Some(s.clone()), + serde_yaml_ng::Value::Bool(b) => Some(b.to_string()), + serde_yaml_ng::Value::Number(n) => Some(n.to_string()), + _ => None, + } +} + +/// `providers[0].protocol` → 键、下标、键。认不出（键里带点、方括号）就是 None：那一处 +/// 不修 —— 删错一个键比不修糟得多 +fn steps(field: &str) -> Option> { + let mut out = Vec::new(); + for part in field.split('.') { + let (key, rest) = match part.find('[') { + Some(i) => (&part[..i], &part[i..]), + None => (part, ""), + }; + if key.is_empty() || key.contains(']') { + return None; + } + out.push(Step::key(key)); + let mut rest = rest; + while !rest.is_empty() { + let inner = rest.strip_prefix('[')?; + let (n, after) = inner.split_once(']')?; + out.push(Step::Index(n.parse().ok()?)); + rest = after; + } + } + Some(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_partial_pack_with_an_old_field_and_an_old_value_is_repaired_in_memory() { + let text = "version: 1\nproviders:\n - name: a\n base_url: https://a.example\n balance: retired_kind\n gone_field: sk-ant-api03-secretsecretsecret\nclient_probes:\n titling: passthrough\n"; + let (out, fixes) = repair(text).expect("修得了"); + assert!(model::parse(&out).is_ok(), "{out}"); + let got: Vec<(FixKind, &str, Option)> = fixes + .iter() + .map(|f| (f.kind, f.field.as_str(), f.line)) + .collect(); + assert_eq!( + got, + vec![ + (FixKind::UnknownValue, "providers[0].balance", Some(5)), + (FixKind::UnknownField, "providers[0].gone_field", Some(6)), + (FixKind::UnknownValue, "client_probes.titling", Some(8)), + ] + ); + // 默认值不写出来的（余额来源缺省是 auto）说不出来 + assert_eq!(fixes[0].now, None); + assert_eq!(fixes[2].now.as_deref(), Some("forward")); + // 原值脱敏 + let v = fixes[1].value.as_deref().unwrap(); + assert!(!v.contains("secretsecret"), "{v}"); + // 别的原样留着 + assert!(out.contains(" base_url: https://a.example\n"), "{out}"); + } + + #[test] + fn what_reads_or_cannot_be_repaired_is_none() { + assert!(repair("version: 1\n").is_none()); + // 语法错不修 + assert!(repair("version: 1\nproviders: [\n").is_none()); + // 缺字段（没有默认值可回）不修 + assert!(repair("version: 1\nproviders:\n - name: a\n").is_none()); + } +} diff --git a/crates/tw-pack/src/restore.rs b/crates/tw-pack/src/restore.rs index 9ae33160..e53b7bb2 100644 --- a/crates/tw-pack/src/restore.rs +++ b/crates/tw-pack/src/restore.rs @@ -119,6 +119,8 @@ pub fn apply_with( let picked: Option> = items.map(|xs| xs.iter().collect()); let mut target = l.clone(); let mut kept = Vec::new(); + // 那一版有、现在没有的 OAuth 上游：令牌不在快照里，落下去之后从那一版里取回来 + let mut tokens_from_backup: Vec = Vec::new(); for row in rows(&l, &b, pinned, with_prefs) { if picked.as_ref().is_some_and(|p| !p.contains(&row.item)) { continue; @@ -130,13 +132,20 @@ pub fn apply_with( match b.items.get(&row.item) { Some(it) => { let mut it = it.clone(); - // 那一版有、现在没有的 ChatGPT 账号上游：没有登录，停用着回来 - if row.item.kind == ItemKind::Provider - && !l.items.contains_key(&row.item) - && model::provider_of(&row.item, &it).is_ok_and(|p| model::is_chatgpt(&p)) - && let Some(Json::Object(other)) = it.units.get_mut(unit::OTHER) - { - other.insert("disabled".into(), Json::Bool(true)); + // 那一版有、现在没有的 ChatGPT 账号上游：它的登录随删除吊销了，停用着回来。 + // 别的 OAuth 上游：令牌从那一版里取回来（落下去之后补上） + let back = !l.items.contains_key(&row.item); + let p = (row.item.kind == ItemKind::Provider) + .then(|| model::provider_of(&row.item, &it).ok()) + .flatten(); + if back && let Some(p) = &p { + if model::is_chatgpt(p) { + if let Some(Json::Object(other)) = it.units.get_mut(unit::OTHER) { + other.insert("disabled".into(), Json::Bool(true)); + } + } else if p.oauth.is_some() { + tokens_from_backup.push(row.item.name.clone()); + } } target.items.insert(row.item.clone(), it); } @@ -146,7 +155,10 @@ pub fn apply_with( } } target.relink(); - let pack = model::materialize_with(local, &target, codec)?; + let mut pack = model::materialize_with(local, &target, codec)?; + if !tokens_from_backup.is_empty() { + pack.config_yaml = tokens_back(&pack.config_yaml, backup, &tokens_from_backup)?; + } tw_config::try_parse(&pack.config_yaml).map_err(|r| ModelError::Config(Box::new(r)))?; Ok(Applied { changed_keys: import::changed_keys(local, &pack), @@ -155,6 +167,63 @@ pub fn apply_with( }) } +/// 把那一版里这几家 OAuth 上游的令牌写回 `text`。那一版里没有令牌（空的）的那家停用着回来 +fn tokens_back(text: &str, backup: &Pack, names: &[String]) -> Result { + let then = model::parse(&backup.config_yaml)?; + let mut out = text.to_string(); + for name in names { + let Some(o) = then + .providers + .iter() + .find(|p| &p.name == name) + .and_then(|p| p.oauth.as_ref()) + else { + continue; + }; + if o.refresh.trim().is_empty() { + out = tw_config::edit::set( + &out, + &provider_field(&out, name, "disabled")?, + Some(&serde_yaml_ng::Value::Bool(true)), + )?; + continue; + } + let tokens = tw_config::RenewedTokens { + access: o.access.as_deref().unwrap_or_default(), + expires_at: o.expires_at.as_deref(), + refresh: Some(&o.refresh), + }; + out = tw_config::patch_oauth_tokens(&out, name, tokens).map_err(|e| ModelError::Shape { + item: format!("provider `{name}`"), + detail: e.to_string(), + })?; + // 没有 access token 的：去掉写进去的空值，下一次用到时拿 refresh 换 + if o.access.is_none() { + let at = provider_field(&out, name, "oauth")?; + let mut p = at.clone(); + p.push(tw_yaml::Step::key("access")); + out = tw_config::edit::set(&out, &p, None)?; + } + } + Ok(out) +} + +/// 配置里上游 `name` 的某个字段的路径 +fn provider_field(text: &str, name: &str, field: &str) -> Result, ModelError> { + let doc = tw_config::edit::parse(text)?; + let i = tw_config::edit::PROVIDERS + .index_of(&doc, name) + .ok_or_else(|| ModelError::Shape { + item: format!("provider `{name}`"), + detail: "it is not in the configuration".into(), + })?; + Ok(vec![ + tw_yaml::Step::key("providers"), + tw_yaml::Step::Index(i), + tw_yaml::Step::key(field), + ]) +} + /// 配置读不进来（安全模式）时的恢复：整份换成那一版，**只有控制面的钥匙换成现在这一把** /// （`current_text` 是磁盘上现在那份读不进来的原文）。插件和通用偏好照那一版 pub fn whole(backup: &Pack, current_text: &str) -> Pack { diff --git a/crates/tw-pack/src/restore/tests.rs b/crates/tw-pack/src/restore/tests.rs index 3ecaeb0d..409f9076 100644 --- a/crates/tw-pack/src/restore/tests.rs +++ b/crates/tw-pack/src/restore/tests.rs @@ -103,3 +103,34 @@ fn in_safe_mode_the_whole_version_comes_back_with_todays_key() { assert!(!out.config_yaml.contains(tw_api::control::KEY_MASK)); assert!(out.config_yaml.contains("port: 8000")); } + +/// OAuth 上游：现在还在的留着现在的令牌；那一版有、现在没有的，令牌从那一版里取回来 +#[test] +fn oauth_tokens_come_from_now_or_from_the_version_it_comes_back_from() { + let oauth = |name: &str, rt: &str| { + format!( + " - name: {name}\n base_url: https://{name}.example\n oauth:\n refresh: {rt}\n access: at-{rt}\n endpoint: https://auth.example/token\n" + ) + }; + let head = format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n key: tw-a\nproviders:\n" + ); + let now = pack(&format!("{head}{}", oauth("kept", "rt-now"))); + let then = pack(&format!( + "{head}{}{}{}", + oauth("kept", "rt-then").replace("kept.example", "kept.example/v0"), + oauth("gone", "rt-gone"), + " - name: blank\n base_url: https://blank.example\n disabled: true\n oauth:\n refresh: ''\n endpoint: https://auth.example/token\n", + )); + let out = apply_with(&now, &then, None, &BTreeSet::new(), true, &WholeFile).unwrap(); + let cfg = tw_config::try_parse(&out.pack.config_yaml) + .unwrap_or_else(|e| panic!("{e}\n{}", out.pack.config_yaml)); + let p = |n: &str| cfg.providers.iter().find(|p| p.name == n).unwrap(); + assert_eq!(p("kept").base_url, "https://kept.example/v0"); + assert_eq!(p("kept").oauth.as_ref().unwrap().refresh, "rt-now"); + let gone = p("gone").oauth.as_ref().unwrap(); + assert_eq!(gone.refresh, "rt-gone"); + assert_eq!(gone.access.as_deref(), Some("at-rt-gone")); + assert!(!p("gone").disabled); + assert!(p("blank").disabled); +} From 908e1fd1dcfbec3d91ad9d0ba2df606fced6f660 Mon Sep 17 00:00:00 2001 From: fylorn <249551762+fylorn@users.noreply.github.com> Date: Mon, 19 Oct 2026 11:15:16 +0800 Subject: [PATCH 04/13] Multi-device sync: folder, WebDAV and S3 locations, three-way merge, conflicts that travel with versions (#350) - tw-config: a local-only `sync` section (device, encryption, passphrase, excluded upstreams, target) with validation; http:// only for this machine and the local network. Documented in the configuration manual. - tw-sync (new): a Store trait (list, get, put_new, put, delete, create_exclusive) with folder, WebDAV and S3 implementations (S3 addressing chosen from the endpoint host); an age X25519 keyring sealed with the passphrase; version naming and retention that keeps every head and its parents; device records; the unit-wise three-way merge with table tests; order merging; conflict records; the sync round (heads merged on their common parent, base from the local base, local backups or the location), joining, encryption on/off and passphrase change, stop and purge. - tw-control: the sync service (10-second settle, upload before exit, 30-minute polls for WebDAV and S3, a folder watcher, checks at start, after wake and with retries; paused in safe mode; unreachable only after two hours); the /sync endpoints and Event::SyncStatus; local writes through the same order as import and restore, with the source device kept on the local backup; RemoteBackups registered while sync is on; pinned keys from the configuration manager; joining with the import preview and apply of tw-pack. - tw-pack: a backup version can record the device it was synced from. - tw-bedrock: sign_service, additive SigV4 for other services (S3). - tw-secret: the sync passphrase is masked in diagnostics. - twcore: sync status, now, enable and disable; the service started and flushed with serve. - CONTROL_API_VERSION 50; message codes sync.*. Co-authored-by: Claude Opus 5.5 --- Cargo.lock | 37 + Cargo.toml | 7 + bin/twcore/Cargo.toml | 2 + bin/twcore/src/call.rs | 36 +- bin/twcore/src/main.rs | 34 + bin/twcore/src/sync_cmd.rs | 293 ++++ crates/tw-api/msg-codes.txt | 50 + crates/tw-api/src/ep.rs | 26 + crates/tw-api/src/lib.rs | 26 + crates/tw-api/src/sync.rs | 340 ++++ crates/tw-bedrock/src/lib.rs | 5 +- crates/tw-bedrock/src/sign.rs | 72 + crates/tw-bedrock/tests/common/mod.rs | 24 +- crates/tw-bedrock/tests/sign.rs | 61 +- crates/tw-config/Cargo.toml | 2 + crates/tw-config/src/lib.rs | 6 + crates/tw-config/src/sync.rs | 682 ++++++++ crates/tw-config/src/validate.rs | 10 + crates/tw-config/tests/manual.rs | 6 +- crates/tw-config/tests/manual/schema.rs | 258 ++- crates/tw-control/Cargo.toml | 2 + crates/tw-control/src/backups.rs | 7 +- crates/tw-control/src/config.rs | 13 + crates/tw-control/src/lib.rs | 8 + crates/tw-control/src/sync.rs | 1682 +++++++++++++++++++ crates/tw-control/tests/sync_devices.rs | 683 ++++++++ crates/tw-gateway/tests/passthrough.rs | 1 + crates/tw-observe/src/bus.rs | 2 + crates/tw-pack/src/backup.rs | 35 + crates/tw-secret/src/mask.rs | 19 + crates/tw-store/src/recorder.rs | 3 +- crates/tw-sync/Cargo.toml | 44 + crates/tw-sync/src/device.rs | 93 ++ crates/tw-sync/src/engine.rs | 2015 +++++++++++++++++++++++ crates/tw-sync/src/keyring.rs | 225 +++ crates/tw-sync/src/layout.rs | 259 +++ crates/tw-sync/src/lib.rs | 24 + crates/tw-sync/src/local.rs | 207 +++ crates/tw-sync/src/merge.rs | 583 +++++++ crates/tw-sync/src/order.rs | 192 +++ crates/tw-sync/src/record.rs | 226 +++ crates/tw-sync/src/store/folder.rs | 398 +++++ crates/tw-sync/src/store/mod.rs | 296 ++++ crates/tw-sync/src/store/s3.rs | 456 +++++ crates/tw-sync/src/store/webdav.rs | 366 ++++ crates/tw-sync/src/store/xml.rs | 160 ++ crates/tw-sync/tests/common/mod.rs | 323 ++++ crates/tw-sync/tests/devices.rs | 747 +++++++++ crates/tw-sync/tests/stores.rs | 198 +++ docs/config.md | 90 + docs/config.zh-CN.md | 79 + 51 files changed, 11382 insertions(+), 31 deletions(-) create mode 100644 bin/twcore/src/sync_cmd.rs create mode 100644 crates/tw-api/src/sync.rs create mode 100644 crates/tw-config/src/sync.rs create mode 100644 crates/tw-control/src/sync.rs create mode 100644 crates/tw-control/tests/sync_devices.rs create mode 100644 crates/tw-sync/Cargo.toml create mode 100644 crates/tw-sync/src/device.rs create mode 100644 crates/tw-sync/src/engine.rs create mode 100644 crates/tw-sync/src/keyring.rs create mode 100644 crates/tw-sync/src/layout.rs create mode 100644 crates/tw-sync/src/lib.rs create mode 100644 crates/tw-sync/src/local.rs create mode 100644 crates/tw-sync/src/merge.rs create mode 100644 crates/tw-sync/src/order.rs create mode 100644 crates/tw-sync/src/record.rs create mode 100644 crates/tw-sync/src/store/folder.rs create mode 100644 crates/tw-sync/src/store/mod.rs create mode 100644 crates/tw-sync/src/store/s3.rs create mode 100644 crates/tw-sync/src/store/webdav.rs create mode 100644 crates/tw-sync/src/store/xml.rs create mode 100644 crates/tw-sync/tests/common/mod.rs create mode 100644 crates/tw-sync/tests/devices.rs create mode 100644 crates/tw-sync/tests/stores.rs diff --git a/Cargo.lock b/Cargo.lock index 3557739c..7b0055cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2400,6 +2400,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "quick-xml" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" +dependencies = [ + "memchr", +] + [[package]] name = "quinn" version = "0.11.11" @@ -3721,6 +3730,7 @@ dependencies = [ "tw-types", "tw-watch", "tw-yaml", + "url", "windows-sys 0.61.2", ] @@ -3763,6 +3773,7 @@ dependencies = [ "tw-pricing", "tw-secret", "tw-store", + "tw-sync", "tw-types", "tw-watch", "tw-yaml", @@ -3961,6 +3972,32 @@ dependencies = [ "tw-secret", ] +[[package]] +name = "tw-sync" +version = "0.76.0" +dependencies = [ + "age", + "axum", + "base64 0.22.1", + "percent-encoding", + "quick-xml", + "rand 0.10.2", + "reqwest", + "serde", + "serde_json", + "sha2 0.11.0", + "tempfile", + "thiserror 2.0.20", + "tokio", + "tracing", + "tw-bedrock", + "tw-config", + "tw-pack", + "tw-types", + "tw-watch", + "url", +] + [[package]] name = "tw-types" version = "0.76.0" diff --git a/Cargo.toml b/Cargo.toml index a2d085af..6d818724 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,8 @@ members = [ # 配置包（.twconfig）：导出、导入、本机备份与同步共用的格式，以及把一份配置拆成 # 项目与合并单位的模型 "crates/tw-pack", + # 多设备同步：同步位置的存储（文件夹、WebDAV、S3）、keyring、三方合并 + "crates/tw-sync", # 控制通道的握手与加密。桌面端按 tag 依赖它，所以只依赖契约层和 tw-yaml "crates/tw-link", # ── bin ────────────────────────────────────────────── @@ -80,6 +82,7 @@ tw-bedrock = { path = "crates/tw-bedrock" } tw-link = { path = "crates/tw-link" } tw-plugin = { path = "crates/tw-plugin" } tw-pack = { path = "crates/tw-pack" } +tw-sync = { path = "crates/tw-sync" } # ── 版本与企业版 workspace 对齐，便于反向依赖时不打架 ──── tokio = { version = "1", features = ["rt-multi-thread", "macros", "net", "time", "sync", "io-util", "signal"] } @@ -146,6 +149,10 @@ aws-credential-types = "1" aws-smithy-eventstream = "0.60" aws-smithy-types = "1" percent-encoding = "2" +# 地址解析（同步位置的 `http://` 只给内网）。reqwest 本来就依赖它 +url = "2" +# WebDAV 的 PROPFIND、S3 的 ListObjectsV2 回的是 XML +quick-xml = "0.42" thiserror = "2" anyhow = "1" diff --git a/bin/twcore/Cargo.toml b/bin/twcore/Cargo.toml index 8455b64c..fc898f4b 100644 --- a/bin/twcore/Cargo.toml +++ b/bin/twcore/Cargo.toml @@ -25,6 +25,8 @@ tw-observe = { workspace = true } tw-store = { workspace = true } tw-pricing = { workspace = true } chrono = { workspace = true } +# `twcore sync` 和控制面说 JSON +serde_json = { workspace = true } tw-secret = { workspace = true } anyhow = { workspace = true } clap = { workspace = true } diff --git a/bin/twcore/src/call.rs b/bin/twcore/src/call.rs index 9c8299c7..19084ab1 100644 --- a/bin/twcore/src/call.rs +++ b/bin/twcore/src/call.rs @@ -18,6 +18,27 @@ pub fn run( data: Option, out: Option, ) -> Result<()> { + let (status, body) = request(config, method, endpoint, data)?; + match out { + Some(file) => { + std::fs::write(&file, &body).with_context(|| format!("writing {}", file.display()))?; + println!("{status}"); + } + None => { + use std::io::Write; + std::io::stdout().write_all(&body)?; + } + } + Ok(()) +} + +/// 发一个请求，拿回状态码和响应体。`twcore sync` 这类命令也走它 +pub fn request( + config: &Path, + method: &str, + endpoint: &str, + data: Option, +) -> Result<(u16, Bytes)> { let dir = config .parent() .map(PathBuf::from) @@ -36,7 +57,7 @@ pub fn run( let rt = tokio::runtime::Builder::new_current_thread() .enable_all() .build()?; - let (status, body) = rt.block_on(async move { + rt.block_on(async move { match Address::in_dir(&dir) { #[cfg(unix)] Address::Socket(p) => { @@ -59,18 +80,7 @@ pub fn run( send(s, &key, method, &endpoint, data).await } } - })?; - match out { - Some(file) => { - std::fs::write(&file, &body).with_context(|| format!("writing {}", file.display()))?; - println!("{status}"); - } - None => { - use std::io::Write; - std::io::stdout().write_all(&body)?; - } - } - Ok(()) + }) } async fn send( diff --git a/bin/twcore/src/main.rs b/bin/twcore/src/main.rs index 391ab9d1..1a2e251e 100644 --- a/bin/twcore/src/main.rs +++ b/bin/twcore/src/main.rs @@ -13,6 +13,7 @@ mod call; mod lockfile; mod pack_cli; mod proc; +mod sync_cmd; mod upgrade; use lockfile::{LockFile, LockOutcome}; @@ -138,6 +139,13 @@ enum Command { #[command(subcommand)] what: BackupCmd, }, + /// Show or change multi-device sync. Talks to the running core + // + // 给没有界面的服务器（远程 core）用：和界面走同一套控制面端点 + Sync { + #[command(subcommand)] + what: SyncCmd, + }, /// Replace this twcore with another release from GitHub; the configuration is left alone // // 只给单独装的 twcore 用（服务器上）。桌面应用包里的那份由应用更新，见 upgrade.rs @@ -154,6 +162,22 @@ enum Command { }, } +#[derive(Subcommand)] +enum SyncCmd { + /// Print the sync state, the location and the devices + Status, + /// Sync once now + Now, + /// Turn sync on; asks for the location and the passphrase + Enable, + /// Turn sync off; this device keeps its configuration + Disable { + /// Also delete every version, device record and the keyring at the sync location + #[arg(long)] + purge: bool, + }, +} + #[derive(Subcommand)] enum RemoteCmd { /// Print whether the remote control port is open and where to connect @@ -260,6 +284,12 @@ fn main() -> Result<()> { data, out, } => call::run(&path, &endpoint, &method, data, out), + Command::Sync { what } => match what { + SyncCmd::Status => sync_cmd::status(&path), + SyncCmd::Now => sync_cmd::now(&path), + SyncCmd::Enable => sync_cmd::enable(&path), + SyncCmd::Disable { purge } => sync_cmd::disable(&path, purge), + }, Command::Upgrade { check, restart, @@ -964,6 +994,8 @@ fn cmd_serve(path: &Path, port: Option, safe: bool, parent: Option) -> // 这是一种（另一种是上面的默认插件）—— 理由是服务器换发新 refresh // token 的那一刻旧的就作废了，不写回等于让配置文件从那一秒起就是坏的。 tw_control::rotation::spawn(control.clone()); + // 多设备同步：启动时一轮，之后按本机改动、定时、文件夹变化、唤醒。安全模式下不自动同步 + let sync = tw_control::sync::start(&control, safe); // 定期刷新默认价目表（`pricing.auto_update`，默认开） tw_control::pricing::spawn(control.clone()); @@ -1032,6 +1064,8 @@ fn cmd_serve(path: &Path, port: Option, safe: bool, parent: Option) -> } _ = shutdown_requested(&shutdown) => { tracing::info!("got a shutdown signal"); + // 还没上传的本机改动，退出前上传（等不了太久：没传成的下次启动时传） + sync.flush().await; Ok(()) } } diff --git a/bin/twcore/src/sync_cmd.rs b/bin/twcore/src/sync_cmd.rs new file mode 100644 index 00000000..c52ff7b8 --- /dev/null +++ b/bin/twcore/src/sync_cmd.rs @@ -0,0 +1,293 @@ +//! `twcore sync status | now | enable | disable`：给没有界面的服务器（远程 core）用。 +//! +//! 和界面走同一套控制面端点（正在跑的 core 来做），这里只是把它们问成终端里的几个问题。 + +use std::io::{BufRead, Write}; +use std::path::Path; + +use anyhow::{Context, Result, bail}; + +use crate::call; + +fn ask(prompt: &str, default: Option<&str>) -> Result { + match default { + Some(d) if !d.is_empty() => eprint!("{prompt} [{d}]: "), + _ => eprint!("{prompt}: "), + } + std::io::stderr().flush()?; + let mut line = String::new(); + std::io::stdin().lock().read_line(&mut line)?; + let line = line.trim().to_string(); + Ok(if line.is_empty() { + default.unwrap_or_default().to_string() + } else { + line + }) +} + +fn yes(prompt: &str, default: bool) -> Result { + let d = if default { "Y/n" } else { "y/N" }; + let a = ask(&format!("{prompt} ({d})"), None)?; + Ok(match a.to_ascii_lowercase().as_str() { + "" => default, + "y" | "yes" => true, + _ => false, + }) +} + +/// 发一个请求：2xx 时把响应体解析成 `T`，别的把控制面那句话说出来 +fn send( + config: &Path, + method: &str, + path: &str, + body: Option, +) -> Result { + let (status, bytes) = call::request(config, method, path, body.map(|b| b.to_string())) + .context("asking the running core (start it with twcore serve)")?; + if !(200..300).contains(&status) { + let m: Option = serde_json::from_slice(&bytes).ok(); + match m { + Some(m) => bail!("{}", m.text), + None => bail!( + "the core answered {status}: {}", + String::from_utf8_lossy(&bytes) + ), + } + } + serde_json::from_slice(&bytes).context("reading the answer") +} + +fn when(ms: u64) -> String { + let secs = (ms / 1000) as i64; + chrono::DateTime::from_timestamp(secs, 0) + .map(|t| { + t.with_timezone(&chrono::Local) + .format("%Y-%m-%d %H:%M") + .to_string() + }) + .unwrap_or_else(|| ms.to_string()) +} + +fn print_view(v: &tw_api::SyncView) { + println!("state: {}", v.state.slug()); + if let Some(t) = &v.target { + let place = match t { + tw_api::SyncTarget::Folder { path, .. } => format!("folder {path}"), + tw_api::SyncTarget::Webdav { url, .. } => format!("WebDAV {url}"), + tw_api::SyncTarget::S3 { + endpoint, + bucket, + prefix, + .. + } => format!("S3 {endpoint} bucket {bucket} prefix {prefix}"), + }; + println!("location: {place}"); + println!("encrypted: {}", if v.encrypted { "yes" } else { "no" }); + } + if let Some(d) = &v.device { + println!("this device: {} ({})", d.name, &d.id[..8.min(d.id.len())]); + } + if let Some(ms) = v.last_synced_ms { + println!("last synced: {}", when(ms)); + } + if let Some(e) = &v.error { + println!("problem: {}", e.text); + } + if v.conflicts > 0 { + println!( + "conflicts: {} (resolve them in ThinkWatch Lite, Settings → Sync and backup)", + v.conflicts + ); + } + if !v.excluded.is_empty() { + println!("not synced: {}", v.excluded.join(", ")); + } + for d in &v.devices { + let mut notes = Vec::new(); + if d.this { + notes.push("this device"); + } + if d.remote_core { + notes.push("remote core"); + } + if d.paused_update { + notes.push("paused: needs an update"); + } + let notes = if notes.is_empty() { + String::new() + } else { + format!(" ({})", notes.join(", ")) + }; + println!( + " {} core {} last synced {}{notes}", + d.name, + d.core_version, + when(d.last_seen_ms) + ); + } +} + +pub fn status(config: &Path) -> Result<()> { + let v: tw_api::SyncView = send(config, "GET", "/sync", None)?; + print_view(&v); + Ok(()) +} + +pub fn now(config: &Path) -> Result<()> { + let v: tw_api::SyncView = send(config, "POST", "/sync/now", None)?; + print_view(&v); + Ok(()) +} + +pub fn disable(config: &Path, purge: bool) -> Result<()> { + if purge + && !yes( + "Delete every version, device record and the keyring at the sync location? Other \ + devices stop syncing", + false, + )? + { + bail!("nothing was changed"); + } + let _: () = send( + config, + "POST", + "/sync/disable", + Some(serde_json::json!({ "purge": purge })), + )?; + println!("sync is off; this device keeps its configuration"); + Ok(()) +} + +fn host_name() -> String { + for var in ["COMPUTERNAME", "HOSTNAME"] { + if let Ok(v) = std::env::var(var) + && !v.trim().is_empty() + { + return v.trim().to_string(); + } + } + std::fs::read_to_string("/etc/hostname") + .map(|s| s.trim().to_string()) + .ok() + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| "server".into()) +} + +pub fn enable(config: &Path) -> Result<()> { + let kind = ask("Sync location: folder, webdav or s3", Some("folder"))?; + let proxy = || -> Result { + ask( + "Outbound proxy: direct, system or a proxy name", + Some("direct"), + ) + }; + let target = match kind.to_ascii_lowercase().as_str() { + "folder" => serde_json::json!({ + "type": "folder", + "path": ask("Folder path on this machine", None)?, + "proxy": "direct", + }), + "webdav" => serde_json::json!({ + "type": "webdav", + "url": ask("WebDAV address (https://…)", None)?, + "user": ask("User name", None)?, + "pass": ask("Password or app password", None)?, + "proxy": proxy()?, + }), + "s3" => serde_json::json!({ + "type": "s3", + "endpoint": ask("Endpoint (https://…)", None)?, + "region": ask("Region", Some("auto"))?, + "bucket": ask("Bucket", None)?, + "prefix": ask("Prefix", Some(""))?, + "access_key_id": ask("Access key ID", None)?, + "secret_access_key": ask("Secret access key", None)?, + "proxy": proxy()?, + }), + other => bail!("{other} is not one of folder, webdav, s3"), + }; + let probe: tw_api::SyncProbe = send( + config, + "POST", + "/sync/test", + Some(serde_json::json!({ "target": target })), + )?; + if !probe.writable { + bail!( + "the sync location cannot be written: {}", + probe.error.map(|e| e.text).unwrap_or_default() + ); + } + let device_name = ask("Name of this device", Some(&host_name()))?; + let mut body = serde_json::json!({ + "target": target, + "device_name": device_name, + "encrypt": true, + "join": "use_remote", + "decisions": [], + }); + match &probe.existing { + None => { + let encrypt = yes("Encrypt the sync data with a passphrase", true)?; + body["encrypt"] = encrypt.into(); + if encrypt { + eprintln!( + "The passphrase is typed in the clear here. At least {} characters; every \ + device joining needs it", + tw_config::sync::PASSPHRASE_MIN + ); + let p = ask("Sync passphrase", None)?; + let again = ask("Sync passphrase again", None)?; + if p != again { + bail!("the two passphrases differ"); + } + body["passphrase"] = p.into(); + } + } + Some(e) => { + if e.encrypted { + body["passphrase"] = ask("Sync passphrase", None)?.into(); + } + let preview: tw_api::SyncJoinPreview = send( + config, + "POST", + "/sync/enable/preview", + Some(serde_json::json!({ "target": target, "passphrase": body.get("passphrase") })), + )?; + if !preview.initial { + let conflicts: Vec<&tw_api::ImportRow> = preview + .rows + .iter() + .filter(|r| r.status == tw_api::ImportStatus::Conflict) + .collect(); + let new = preview + .rows + .iter() + .filter(|r| r.status == tw_api::ImportStatus::New) + .count(); + eprintln!( + "This device already has a configuration: {new} item(s) arrive from the sync \ + data, {} differ", + conflicts.len() + ); + let replace = yes( + "Replace this device's configuration with the sync data (no keeps this \ + device's version of every item that differs)", + false, + )?; + if !replace { + body["join"] = "merge".into(); + body["decisions"] = conflicts + .iter() + .map(|r| serde_json::json!({ "item": r.item, "action": "skip" })) + .collect(); + } + } + } + } + let _: tw_api::ConfigWritten = send(config, "POST", "/sync/enable", Some(body))?; + let v: tw_api::SyncView = send(config, "GET", "/sync", None)?; + print_view(&v); + Ok(()) +} diff --git a/crates/tw-api/msg-codes.txt b/crates/tw-api/msg-codes.txt index b86e8b00..5dd966db 100644 --- a/crates/tw-api/msg-codes.txt +++ b/crates/tw-api/msg-codes.txt @@ -542,6 +542,56 @@ security.unknown_action security.unknown_custom_rule security.unknown_guard security.unknown_rule +sync.already_on +sync.both_name_taken +sync.both_not_allowed +sync.config.address +sync.config.blank +sync.config.blank_exclude +sync.config.bucket +sync.config.device_id +sync.config.device_name +sync.config.folder_relative +sync.config.multiline +sync.config.passphrase_missing +sync.config.unknown_proxy +sync.emptied +sync.folder.denied +sync.folder.io +sync.folder.missing +sync.folder.task +sync.http_client +sync.join_undecided +sync.keyring_corrupt +sync.keyring_too_new +sync.local +sync.local.io +sync.location_taken +sync.locked_new +sync.merge_invalid +sync.not_arrived +sync.not_encrypted +sync.off +sync.passphrase_changed +sync.passphrase_short +sync.passphrase_wrong +sync.place_changed +sync.proxy_unusable +sync.s3.bad_listing +sync.s3.sign +sync.shutting_down +sync.stale +sync.store.bad_path +sync.store.denied +sync.store.exists +sync.store.server_error +sync.store.unexpected +sync.store.unreachable +sync.too_new +sync.version_gone +sync.webdav.bad_listing +sync.webdav.bad_url +sync.write_section t.auth test t.boom test t.broke test diff --git a/crates/tw-api/src/ep.rs b/crates/tw-api/src/ep.rs index 49d9c18a..138591a6 100644 --- a/crates/tw-api/src/ep.rs +++ b/crates/tw-api/src/ep.rs @@ -257,6 +257,32 @@ endpoints! { /// 最近的日志，老的在前。**网页可以调** PluginLogs: GET "/plugins/{id}/logs" [id], () => Vec; + // ─────────────────────────────────────────────── 多设备同步 + // + // 同步设置写在 config.yaml 的 `sync` 一节（本机设置，不随导出和同步带出）。改设置的几个 + // 端点（开启、改设置、加密、解锁、停止）回的是写完之后的版本号或同步状态 + /// 同步此刻怎么样：状态、同步位置、设备、冲突数、不同步的上游和随之不同步的项目 + SyncStatus: GET "/sync", () => api::SyncView; + /// 连上去看看：读写行不行，是空的还是已经有同步数据（加没加密、有哪些设备）。**不写配置** + SyncTest: POST "/sync/test", api::SyncTestRequest => api::SyncProbe; + /// 加入已有的同步时，本机配置和同步数据怎么合（规则和导入预览一样）。**不写配置** + SyncJoinPreview: POST "/sync/enable/preview", api::SyncTestRequest => api::SyncJoinPreview; + /// 开启同步：空的同步位置新建，有数据的加入 + SyncEnable: POST "/sync/enable", api::SyncEnable => api::ConfigWritten; + /// 改连接参数、设备名称、不同步的上游 + SyncSave: PUT "/sync/settings", api::SyncSettingsSave => api::ConfigWritten; + /// 立即同步一轮（打开主窗口时 Lite 也调它） + SyncNow: POST "/sync/now", () => api::SyncView; + SyncConflicts: GET "/sync/conflicts", () => Vec; + /// 处理冲突：结果作为新版本上传，别的设备随之清除 + SyncResolve: POST "/sync/conflicts", api::SyncResolve => api::SyncView; + /// 开启、关闭加密，更改同步口令 + SyncEncryption: POST "/sync/encryption", api::SyncEncryption => api::SyncView; + /// 其他设备改了口令之后，在这台设备上输入新口令 + SyncUnlock: POST "/sync/unlock", api::SyncUnlock => api::SyncView; + /// 停止同步；`purge` 时连同步位置上的数据一起删掉 + SyncDisable: POST "/sync/disable", api::SyncDisable => (); + // ─────────────────────────────────────────────── 账号登录 StartChatgptLogin: POST "/chatgpt/login", api::ChatgptLoginStart => api::ChatgptLogin; ChatgptLoginStatus: GET "/chatgpt/login/{id}" [id], () => api::ChatgptLoginStatus; diff --git a/crates/tw-api/src/lib.rs b/crates/tw-api/src/lib.rs index 90f1cc86..d545dc17 100644 --- a/crates/tw-api/src/lib.rs +++ b/crates/tw-api/src/lib.rs @@ -94,6 +94,9 @@ macro_rules! slug_enum { // 配置包那一半：导出、导入、备份与恢复（`slug_enum!` 在上面定义，所以在这里声明） mod pack; pub use pack::*; +// 多设备同步那一半（`slug_enum!` 在上面定义，所以在这里声明） +mod sync; +pub use sync::*; // ---------------------------------------------------------------- 取值固定的词 // @@ -977,6 +980,17 @@ pub const MSG_CODES: &str = include_str!("../msg-codes.txt"); /// 和 [`RestorePreview`] 的 `repairs` 列出修了哪几处。OAuth 上游换发来的令牌（刷新令牌会轮换） /// 和 ChatGPT 的登录一样留在本机，不随导出带走、导入覆盖时留着本机的。一键修复修不了时的码换成 /// `control.config_unrepairable`（句子改说从备份恢复）。照 49 写的界面调不到删掉的那两个端点。 +/// +/// **50 起有多设备同步**：`GET /sync`（[`SyncView`]）、`POST /sync/test`（[`SyncTestRequest`] → +/// [`SyncProbe`]）、`POST /sync/enable/preview`（→ [`SyncJoinPreview`]，规则和导入预览一样）、 +/// `POST /sync/enable`（[`SyncEnable`]：空的同步位置新建，有数据的加入，`merge` 或 `use_remote`）、 +/// `PUT /sync/settings`（[`SyncSettingsSave`]）、`POST /sync/now`、`GET` / `POST /sync/conflicts` +/// （[`SyncConflict`]、[`SyncResolve`]：采用本机、另一边或两者都保留）、`POST /sync/encryption` +/// （[`SyncEncryption`]：开启、关闭、更改口令）、`POST /sync/unlock`（[`SyncUnlock`]）、 +/// `POST /sync/disable`（[`SyncDisable`]）。状态变了发 [`Event::SyncStatus`]；应用别的设备的 +/// 版本时配置的来源是 `sync`，改了网关密钥时发 `GatewayKeysChanged`。[`ProviderView`] 多了 +/// `sync_excluded`（不同步的上游）和 `sync_url_changed`（地址经同步变更，改之前的地址）。 +/// 同步的码是 `sync.*`。安全模式下不自动同步（`paused`） pub const CONTROL_API_VERSION: u32 = 50; /// `/live` 的速度看最近多久（见 [`LiveView::tokens_per_sec`]），毫秒 @@ -1710,6 +1724,9 @@ pub enum Event { /// 「进行中」的计数和每一行的状态都靠事件，丢一个结局，那一行就永远停在 /// 「进行中」。 EventsDropped { id: u64, count: u64, at_ms: u64 }, + /// 同步的状态、冲突数或设备列表变了（`GET /sync` 的那一份）。正常运转的同步不发系统通知， + /// 要人处理的状态（冲突、连不上、拒绝访问、口令、版本过新）由界面按通知总开关提醒一次 + SyncStatus { id: u64, view: SyncView, at_ms: u64 }, /// 这几把网关密钥的值变了或者没了：导入覆盖、恢复、同步改的。 /// /// **本机接管的客户端配置里写着密钥的值**：界面收到它，照「更换密钥后更新客户端」那一套 @@ -1982,6 +1999,7 @@ impl Event { | Event::RequestCancelled { id, .. } | Event::LocallyAnswered { id, .. } | Event::EventsDropped { id, .. } + | Event::SyncStatus { id, .. } | Event::GatewayKeysChanged { id, .. } | Event::ClientPrefsChanged { id, .. } | Event::ConfigReloaded { id, .. } @@ -2372,6 +2390,14 @@ pub struct ProviderView { /// 密钥是用哪一家的账号登录换来的（配置里的 `signed_in:`）：`zai` / `bigmodel`。手填的 /// 密钥、别的上游、这一项出现之前登录的都是空。在界面里手动换掉密钥之后也是空 pub signed_in: Option, + /// 不同步的上游（同步设置里选的）：不上传，也不被别的设备的同名上游覆盖。没开同步是 false + #[serde(default)] + pub sync_excluded: bool, + /// 地址经同步变更：不加密的同步最近一次把这家的地址改了（改之前的地址、哪台设备、什么 + /// 时候）。「提示一次」由界面自己记已读；在本机改过这家的地址就没有了。加密的同步不给 + /// （没有口令的人写不进来） + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sync_url_changed: Option, } /// 一行请求头，配置里写的原样。 diff --git a/crates/tw-api/src/sync.rs b/crates/tw-api/src/sync.rs new file mode 100644 index 00000000..18800f47 --- /dev/null +++ b/crates/tw-api/src/sync.rs @@ -0,0 +1,340 @@ +//! 多设备同步那一半的契约：同步位置、状态、开启与加入、冲突、加密、停止。 +//! +//! 同步怎么合并、同步位置上放什么在 core 的 `tw-sync` 里；这里只有线上的样子。冲突、随之 +//! 不同步的项目按项目（[`ItemRef`]）说，和导入、恢复是同一套。 + +use serde::{Deserialize, Serialize}; + +use crate::{ImportDecision, ImportRow, ItemRef, Msg}; + +fn direct() -> String { + "direct".to_string() +} + +/// 同步位置。`proxy` 是 `direct`、`system`（跟随系统）或者出站代理的名字：WebDAV、S3 的请求经 +/// 它发出，文件夹用不到。 +/// +/// **口令、密码、密钥原样来回**（界面不打码配置值）：设置表单回填的就是它们。 +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +#[serde(tag = "type", rename_all = "lowercase")] +pub enum SyncTarget { + /// 本机（远程 core 时是服务器上）的一个文件夹：iCloud Drive、Dropbox、OneDrive、Syncthing、 + /// NAS 共享目录 + Folder { + path: String, + #[serde(default = "direct")] + proxy: String, + }, + /// 坚果云、Nextcloud、群晖 NAS……。`http://` 只给本机和内网 + Webdav { + url: String, + #[serde(default)] + user: String, + #[serde(default)] + pass: String, + #[serde(default = "direct")] + proxy: String, + }, + /// Amazon S3、Cloudflare R2、阿里云 OSS、腾讯云 COS、MinIO。寻址方式（桶名进域名还是进 + /// 路径）由端点决定，不用选 + S3 { + endpoint: String, + region: String, + bucket: String, + #[serde(default)] + prefix: String, + access_key_id: String, + secret_access_key: String, + #[serde(default = "direct")] + proxy: String, + }, +} + +slug_enum! { + /// 同步此刻怎么样。 + pub enum SyncState { + /// 没开 + Off = "off", + /// 已同步 + Idle = "idle", + /// 正在同步 + Syncing = "syncing", + /// 有冲突等着处理（其余照常同步） + Conflict = "conflict", + /// 连续 2 小时连不上同步位置。恢复后自动继续，本机的修改那时上传 + Unreachable = "unreachable", + /// 同步位置拒绝访问：改连接参数 + AuthFailed = "auth_failed", + /// 同步口令已在其他设备上更改（或者同步位置开了加密）：输入口令 + Locked = "locked", + /// 其他设备用的是较新的版本，本机读不了：暂停，更新之后继续 + TooNew = "too_new", + /// 安全模式：不自动同步 + Paused = "paused", + } +} + +/// 这台设备 +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncDeviceRef { + pub id: String, + pub name: String, +} + +/// 参与同步的一台设备 +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncDevice { + pub id: String, + pub name: String, + pub core_version: String, + /// 最后一次同步成功的时间 + pub last_seen_ms: u64, + /// 是这台设备 + pub this: bool, + /// 「已暂停：需更新」：同步位置上的最新版本它读不了 + pub paused_update: bool, + /// 它的 core 开着远程控制端口（界面标「远程 core」） + pub remote_core: bool, +} + +/// `GET /sync`，也是 [`crate::Event::SyncStatus`] 带的那一份。 +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncView { + pub state: SyncState, + /// 没开是空的 + pub target: Option, + /// 同步位置上的内容加没加密 + pub encrypted: bool, + pub device: Option, + pub last_synced_ms: Option, + /// 从这一刻起一直没同步成。成功一次就清掉 + pub failing_since_ms: Option, + /// 要人处理的状态（拒绝访问、口令、版本过新、连不上）的原因 + pub error: Option, + pub devices: Vec, + /// 还没处理的冲突有几处 + pub conflicts: u32, + /// 不同步的上游（名字） + pub excluded: Vec, + /// 随之不同步的项目：引用了不同步的上游的路由、别名、分组 + pub excluded_cascade: Vec, +} + +/// `POST /sync/test`、`POST /sync/enable/preview`：连上去看看。 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncTestRequest { + pub target: SyncTarget, + /// 已经有加密的同步数据时，给口令才看得到设备列表 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub passphrase: Option, +} + +/// 同步位置上已经有的同步数据 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncExisting { + pub encrypted: bool, + /// 加密的：口令对了。不加密的总是 `true` + pub unlocked: bool, + /// 已经在同步的设备。加密而口令不对时是空的 + pub devices: Vec, +} + +/// `POST /sync/test` 的结果。 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncProbe { + /// 读写都行 + pub writable: bool, + /// 不行的原因 + pub error: Option, + /// 空的同步位置是 `null`（开启同步走「新建」）；已经有同步数据走「加入」 + pub existing: Option, +} + +/// `POST /sync/enable/preview` 的结果：加入时本机配置和同步数据怎么合，规则和导入一样。 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncJoinPreview { + /// 本机配置还是初始配置（新安装后没改过）：不出现合并这一步，直接用同步数据 + pub initial: bool, + /// 同步数据里的每个项目和本机比（[`ImportRow`]）。`initial` 时是空的 + pub rows: Vec, +} + +slug_enum! { + /// 加入时怎么处理本机已有的配置。 + pub enum SyncJoin { + /// 按导入的规则合并，冲突项逐项选择（`decisions`） + Merge = "merge", + /// 用同步数据替换本机配置。本机已接管的客户端正在用的网关密钥并入同步数据 + UseRemote = "use_remote", + } +} + +/// `POST /sync/enable`：开启同步。同步位置空着就新建，已经有同步数据就加入。 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncEnable { + pub target: SyncTarget, + /// 新建时加密（缺省）还是不加密。加入时按同步位置上的，这一项不看 + pub encrypt: bool, + /// 新建加密的同步时至少 12 个字符；加入加密的同步时是那个口令 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub passphrase: Option, + pub device_name: String, + pub join: SyncJoin, + /// `merge` 时逐项的处理（[`ImportDecision`]），没列的新项目照常导入 + #[serde(default)] + pub decisions: Vec, +} + +/// `PUT /sync/settings`：改连接参数（密码、密钥、代理）、设备名称、不同步的上游。只改给了的。 +/// +/// **换成另一个同步位置（另一个文件夹、地址、存储桶、前缀）不在这里**：那是先停止同步、再 +/// 开启同步（`sync.place_changed`） +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncSettingsSave { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub target: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub device_name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub excluded: Option>, +} + +/// 冲突的一边 +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncConflictSide { + /// 设备名称 + pub device: String, + pub at_ms: u64, + /// 这一边删了这个对象 + pub deleted: bool, +} + +slug_enum! { + /// 冲突不是因为两边改得不一样，而是： + pub enum SyncConflictReason { + /// 另一边删了一把网关密钥，而本机已接管的客户端正在用它：没删 + KeyInUse = "key_in_use", + /// 合并的结果过不了配置校验（比如路由引用的上游被另一边删了） + Invalid = "invalid", + } +} + +/// 冲突里的一个合并单位，两边的值（规范化之后的 JSON）。一边删了对象时那一边是 `null` +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncConflictDiff { + pub unit: String, + #[cfg_attr(feature = "ts", ts(type = "unknown"))] + pub local: Option, + #[cfg_attr(feature = "ts", ts(type = "unknown"))] + pub remote: Option, +} + +/// `GET /sync/conflicts` 的一项。 +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncConflict { + pub id: String, + pub item: ItemRef, + /// 不同的合并单位：上游是 `conn`、`models`、`billing`、`other`，插件是 `script`、`settings`、 + /// `enabled`，全局设置是字段的路径，别的是 `all` + pub units: Vec, + /// 本机这一边（冲突处理之前本机一直用它） + pub local: SyncConflictSide, + /// 另一边 + pub remote: SyncConflictSide, + /// 两边不同的地方，逐个单位：点对象名时显示的差异 + pub diff: Vec, + /// 能不能「两者都保留」：两边都还有这个对象，而且它有名字 + pub both_allowed: bool, + pub reason: Option, +} + +slug_enum! { + /// 处理一处冲突：采用哪一边。 + pub enum SyncTake { + Local = "local", + Remote = "remote", + /// 两者都保留：本机的留原名称，另一边的以「名称 (设备名)」新增 + Both = "both", + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncConflictDecision { + pub id: String, + pub take: SyncTake, + /// 两者都保留时另一份的名字。不给就是「名称 (设备名)」 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub new_name: Option, +} + +/// `POST /sync/conflicts` +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncResolve { + pub decisions: Vec, +} + +slug_enum! { + pub enum SyncEncryptionAction { + /// 设口令、开启加密：重新上传最新版本，删掉未加密的版本 + Enable = "enable", + /// 关闭加密：要当前的口令 + Disable = "disable", + /// 更改口令：只重新加密 keyring + Change = "change", + } +} + +/// `POST /sync/encryption` +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncEncryption { + pub action: SyncEncryptionAction, + /// 开启时是新口令（至少 12 个字符），关闭、更改时是当前的口令 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub passphrase: Option, + /// 更改时的新口令（至少 12 个字符） + #[serde(default, skip_serializing_if = "Option::is_none")] + pub new_passphrase: Option, +} + +/// `POST /sync/unlock`：其他设备改了口令（或者开了加密）之后，在这台设备上输入 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncUnlock { + pub passphrase: String, +} + +/// `POST /sync/disable` +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncDisable { + /// 连同步位置上的全部版本、设备记录和 keyring 一起删掉。其他设备随后提示同步位置为空 + #[serde(default)] + pub purge: bool, +} + +/// 不加密的同步改了一个上游的地址（[`crate::ProviderView::sync_url_changed`]）：最近一次的 +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncUrlChange { + /// 改之前的地址 + pub from: String, + /// 改的那台设备的名称 + pub device: Option, + pub at_ms: u64, +} diff --git a/crates/tw-bedrock/src/lib.rs b/crates/tw-bedrock/src/lib.rs index 0b48b678..0d3e5748 100644 --- a/crates/tw-bedrock/src/lib.rs +++ b/crates/tw-bedrock/src/lib.rs @@ -2,7 +2,8 @@ //! //! - [`endpoint`]：区域、runtime 和控制面的地址、模型 id 怎么写进路径 //! - [`beta`]：客户端的 `anthropic-beta` 里哪些能带给 Bedrock 上的 Claude -//! - [`sign`]：SigV4 签名，以及「带 API Key 就不签」这条规则 +//! - [`sign`]：SigV4 签名，以及「带 API Key 就不签」这条规则；S3 兼容存储（core 的同步位置） +//! 也借它签 //! - [`eventstream`]：ConverseStream 的二进制帧 → SSE //! - [`catalog`]：一个区域能路由到哪些模型 //! - [`error`]：AWS 的错误里能拿出来什么、能给谁看 @@ -21,4 +22,4 @@ pub mod error; pub mod eventstream; pub mod sign; -pub use sign::{Credentials, SignError, carries_api_key}; +pub use sign::{Credentials, SignError, carries_api_key, sign_service}; diff --git a/crates/tw-bedrock/src/sign.rs b/crates/tw-bedrock/src/sign.rs index c8619653..14817659 100644 --- a/crates/tw-bedrock/src/sign.rs +++ b/crates/tw-bedrock/src/sign.rs @@ -140,6 +140,78 @@ pub fn sign( .collect()) } +/// 给别的 AWS 服务签名：S3 兼容存储（同步位置）也是 SigV4，只是服务名、要一起签的请求头不同。 +/// +/// - `service` 是签名用的服务名（`s3`）。S3 的规矩和别的服务不一样：路径**不再编一遍、 +/// 不规范化**（`a/./b`、`%2F` 原样签），这里按服务名照办。 +/// - `headers` 是除了 host 以外要一起签的请求头（`content-type`、`if-none-match`……），发请求时 +/// 一个不差地带上；host 从 `url` 里取。 +/// - `body` 是发出去的请求体，一个字节都不能差；没有请求体时给空的。 +/// +/// 返回的头和 [`sign`] 一样：`authorization`、`x-amz-date`、`x-amz-content-sha256`，临时凭证 +/// 还有 `x-amz-security-token`。 +#[allow(clippy::too_many_arguments)] +pub fn sign_service( + credentials: &Credentials, + region: &str, + service: &str, + method: &str, + url: &str, + headers: &[(&str, &str)], + body: &[u8], + at: SystemTime, +) -> Result, SignError> { + use aws_sigv4::http_request::{ + PayloadChecksumKind, PercentEncodingMode, SignableBody, SignableRequest, SignatureLocation, + SigningSettings, UriPathNormalizationMode, sign, + }; + use aws_sigv4::sign::v4; + + let fail = |e: &dyn fmt::Display| SignError(e.to_string()); + let identity = aws_credential_types::Credentials::new( + &credentials.access_key_id, + &credentials.secret_access_key, + credentials.session_token.clone(), + None, + "thinkwatch", + ) + .into(); + + let mut settings = SigningSettings::default(); + settings.payload_checksum_kind = PayloadChecksumKind::XAmzSha256; + settings.signature_location = SignatureLocation::Headers; + if service == "s3" { + settings.percent_encoding_mode = PercentEncodingMode::Single; + settings.uri_path_normalization_mode = UriPathNormalizationMode::Disabled; + } + + let params = v4::SigningParams::builder() + .identity(&identity) + .region(region) + .name(service) + .time(at) + .settings(settings) + .build() + .map_err(|e| fail(&e))?; + + let signable = SignableRequest::new( + method, + url, + headers.iter().copied(), + SignableBody::Bytes(body), + ) + .map_err(|e| fail(&e))?; + + let (instructions, _signature) = sign(signable, ¶ms.into()) + .map_err(|e| fail(&e))? + .into_parts(); + let (headers, _query) = instructions.into_parts(); + Ok(headers + .into_iter() + .map(|h| (h.name().to_ascii_lowercase(), h.value().to_string())) + .collect()) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/tw-bedrock/tests/common/mod.rs b/crates/tw-bedrock/tests/common/mod.rs index 2f966f25..f14de3f6 100644 --- a/crates/tw-bedrock/tests/common/mod.rs +++ b/crates/tw-bedrock/tests/common/mod.rs @@ -144,6 +144,11 @@ pub fn aws_encode(s: &str) -> String { /// 在规范请求里是 `%252F`），查询参数解码后重新编码、排序，签过的头按名字写出来， /// 请求体的哈希取自 `x-amz-content-sha256`，并核对它就是收到的请求体的哈希。 pub fn signature_of(req: &Seen, secret: &str, region: &str) -> (String, String) { + signature_for(req, secret, region, "bedrock") +} + +/// 同上，换一个服务名。`s3` 照 S3 的规矩：路径不再编一遍，收到什么就签什么 +pub fn signature_for(req: &Seen, secret: &str, region: &str, service: &str) -> (String, String) { let header = |name: &str| { req.header(name) .unwrap_or_else(|| panic!("no {name} on {} {}", req.method, req.path)) @@ -166,12 +171,15 @@ pub fn signature_of(req: &Seen, secret: &str, region: &str) -> (String, String) "the signed payload hash is not the hash of what arrived" ); - let path = req - .path - .split('/') - .map(aws_encode) - .collect::>() - .join("/"); + let path = if service == "s3" { + req.path.clone() + } else { + req.path + .split('/') + .map(aws_encode) + .collect::>() + .join("/") + }; let mut query: Vec<(String, String)> = req .params() .iter() @@ -193,7 +201,7 @@ pub fn signature_of(req: &Seen, secret: &str, region: &str) -> (String, String) ); let date = &amz_date[..8]; - let scope = format!("{date}/{region}/bedrock/aws4_request"); + let scope = format!("{date}/{region}/{service}/aws4_request"); let credential = field("Credential"); assert!( credential.ends_with(&format!("/{scope}")), @@ -205,7 +213,7 @@ pub fn signature_of(req: &Seen, secret: &str, region: &str) -> (String, String) ); let key = hmac(format!("AWS4{secret}").as_bytes(), date); let key = hmac(&key, region); - let key = hmac(&key, "bedrock"); + let key = hmac(&key, service); let key = hmac(&key, "aws4_request"); (field("Signature"), hex::encode(hmac(&key, &string_to_sign))) } diff --git a/crates/tw-bedrock/tests/sign.rs b/crates/tw-bedrock/tests/sign.rs index 35e1b552..3f617caa 100644 --- a/crates/tw-bedrock/tests/sign.rs +++ b/crates/tw-bedrock/tests/sign.rs @@ -5,7 +5,7 @@ mod common; use std::time::{Duration, SystemTime}; use axum::http::StatusCode; -use common::{AK, Fake, SK, keys, signature_of}; +use common::{AK, Fake, SK, keys, signature_for, signature_of}; use serde_json::json; use tw_bedrock::sign::sign; @@ -185,3 +185,62 @@ async fn aws_computes_the_same_signature_from_what_arrives() { assert_eq!(claimed, expected); assert!(seen[0].header("authorization").unwrap().contains(AK)); } + +/// S3 兼容存储（core 的同步位置）也用这一份签名：服务名是 `s3`，路径收到什么签什么（不再 +/// 编一遍），带上调用方要一起签的头。签过的请求真的发出去，在收到的一端按 S3 的规矩重算 +#[tokio::test] +async fn an_s3_request_signs_the_way_s3_checks_it() { + let fake = Fake::start(|_| (StatusCode::OK, vec![], json!({}))).await; + let path = "/tw-sync/me/versions/0000000001-3f9a2c1e.twconfig"; + let url = reqwest::Url::parse(&format!("{}{path}", fake.uri())).unwrap(); + let body = b"PK\x03\x04 not really a zip"; + let extra = [ + ("content-type", "application/octet-stream"), + ("if-none-match", "*"), + ]; + let signed = tw_bedrock::sign_service( + &keys(), + "auto", + "s3", + "PUT", + url.as_str(), + &extra, + body, + now(), + ) + .unwrap(); + assert_eq!( + signed_headers(&signed), + "content-type;host;if-none-match;x-amz-content-sha256;x-amz-date" + ); + let mut req = reqwest::Client::new().put(url); + for (k, v) in extra { + req = req.header(k, v); + } + for (k, v) in signed { + req = req.header(k, v); + } + assert_eq!(req.body(body.to_vec()).send().await.unwrap().status(), 200); + + let seen = fake.seen(); + let (claimed, expected) = signature_for(&seen[0], SK, "auto", "s3"); + assert_eq!(claimed, expected); + + // 列目录：查询串里的斜杠和等号签的是编过一遍的样子 + let list = reqwest::Url::parse(&format!( + "{}/tw-sync?list-type=2&prefix=me%2Fversions%2F&delimiter=%2F", + fake.uri() + )) + .unwrap(); + let signed = + tw_bedrock::sign_service(&keys(), "auto", "s3", "GET", list.as_str(), &[], b"", now()) + .unwrap(); + let mut req = reqwest::Client::new().get(list); + for (k, v) in signed { + req = req.header(k, v); + } + assert_eq!(req.send().await.unwrap().status(), 200); + let seen = fake.seen(); + let (claimed, expected) = signature_for(&seen[1], SK, "auto", "s3"); + assert_eq!(claimed, expected); +} diff --git a/crates/tw-config/Cargo.toml b/crates/tw-config/Cargo.toml index 15f8a800..af8e4831 100644 --- a/crates/tw-config/Cargo.toml +++ b/crates/tw-config/Cargo.toml @@ -29,6 +29,8 @@ serde = { workspace = true } serde_yaml_ng = { workspace = true } thiserror = { workspace = true } rand = { workspace = true } +# 同步位置的地址：`http://` 只收本机和内网，要认出主机是 IP 还是域名 +url = { workspace = true } blake3 = { workspace = true } tw-watch = { workspace = true } # 网卡枚举：`bind` 认网卡名，要问系统它现在是哪个地址。 diff --git a/crates/tw-config/src/lib.rs b/crates/tw-config/src/lib.rs index a97f99aa..ec17e088 100644 --- a/crates/tw-config/src/lib.rs +++ b/crates/tw-config/src/lib.rs @@ -29,6 +29,7 @@ pub mod repair; mod retention; mod security; pub mod store; +pub mod sync; mod validate; pub mod watch; mod wire; @@ -129,6 +130,10 @@ pub struct Config { /// 脚本插件，**从上到下就是运行的顺序**。不写就没有 #[serde(default, skip_serializing_if = "Vec::is_empty")] pub plugins: Vec, + /// 多设备同步（见 [`sync`]）。**本机设置**：配置包不带它，恢复备份时保留当前值。 + /// 没开同步就没有这一节 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sync: Option, } /// 便于构造，**不代表一份可用的配置** —— `providers` 和 `clients` 都是 @@ -153,6 +158,7 @@ impl Default for Config { default_route: None, default_key: None, plugins: Vec::new(), + sync: None, } } } diff --git a/crates/tw-config/src/sync.rs b/crates/tw-config/src/sync.rs new file mode 100644 index 00000000..7a460ee4 --- /dev/null +++ b/crates/tw-config/src/sync.rs @@ -0,0 +1,682 @@ +//! `sync` 一节：这台设备怎么参与多设备同步。 +//! +//! ```yaml +//! sync: +//! device_id: 3f9a2c1e-… # 开启同步时生成 +//! device_name: Mac mini +//! encrypt: true # 不写就是 true +//! passphrase: "…" # encrypt 为 true 时必填 +//! exclude: [本机 Ollama] # 不同步的上游 +//! target: +//! type: webdav # folder | webdav | s3 +//! url: https://dav.jianguoyun.com/dav/ThinkWatch +//! user: name@example.com +//! pass: "…" +//! proxy: direct # direct 或出站代理名称 +//! ``` +//! +//! **这是本机设置**：配置包（导出、同步、分享）不带它，恢复备份时保留当前值。同步口令和 +//! 同步位置的凭据和别的密钥一样明文写在这里 —— 这份文件本来就是 0600、只有本机用户读得到。 +//! +//! 同步位置就是数据所在的那一层：`keyring`、`versions/`、`devices/` 直接放在配置的文件夹、 +//! WebDAV 地址、S3 前缀底下，不再另套一层目录。 + +use serde::{Deserialize, Serialize}; +use tw_types::{Msg, msg}; + +use crate::DIRECT; + +/// 同步口令最少几个字符。**只在设口令的那一刻查**（开启同步、改口令）：加入别人建的同步时 +/// 照原样输入，配置里读到的也照原样收 +pub const PASSPHRASE_MIN: usize = 12; + +/// 设备名称最长几个字符 +pub const DEVICE_NAME_MAX: usize = 64; + +/// 设备标识至少要有几个十六进制字符：同步位置上的文件名用的是它的前 8 个 +pub const DEVICE_ID_PREFIX: usize = 8; + +#[derive(Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Sync { + /// 随机生成，开启同步时写下。同步位置上的文件名只用它的前 8 个十六进制字符 + pub device_id: String, + /// 设备列表里显示的名字。缺省为计算机名称 + pub device_name: String, + /// 同一同步位置上的所有设备采用同一设置。**总是写出来**：这是一件要能一眼看清的事， + /// 不写时按 `true` 读 + #[serde(default = "yes")] + pub encrypt: bool, + /// 同步口令。加密时必填 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub passphrase: Option, + /// 不同步的上游，按名字。引用它们的路由、别名、分组随之不同步 + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub exclude: Vec, + pub target: Target, +} + +fn yes() -> bool { + true +} + +fn direct() -> String { + DIRECT.to_string() +} + +fn is_direct(v: &str) -> bool { + v == DIRECT +} + +/// 同步位置。 +#[derive(Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "lowercase", deny_unknown_fields)] +pub enum Target { + /// 本机（或服务器上）的一个文件夹，由 iCloud Drive、Dropbox、Syncthing 这类客户端在 + /// 设备之间传送 + Folder { + path: String, + /// 文件夹用不到，为了三种写法一致才有 + #[serde(default = "direct", skip_serializing_if = "is_direct")] + proxy: String, + }, + Webdav { + url: String, + #[serde(default)] + user: String, + #[serde(default)] + pass: String, + #[serde(default = "direct", skip_serializing_if = "is_direct")] + proxy: String, + }, + S3 { + endpoint: String, + region: String, + bucket: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + prefix: String, + access_key_id: String, + secret_access_key: String, + #[serde(default = "direct", skip_serializing_if = "is_direct")] + proxy: String, + }, +} + +impl Target { + /// 出站请求走哪个代理：`direct`、`system`，或 `proxies` 里的一个名字 + pub fn proxy(&self) -> &str { + match self { + Target::Folder { proxy, .. } + | Target::Webdav { proxy, .. } + | Target::S3 { proxy, .. } => proxy, + } + } + + /// `folder` / `webdav` / `s3` + pub fn kind(&self) -> &'static str { + match self { + Target::Folder { .. } => "folder", + Target::Webdav { .. } => "webdav", + Target::S3 { .. } => "s3", + } + } + + /// 两个设置指的是不是同一个同步位置（凭据和代理不算）。换了位置要重新加入， + /// 只换了凭据不用 + pub fn same_place(&self, other: &Target) -> bool { + match (self, other) { + (Target::Folder { path: a, .. }, Target::Folder { path: b, .. }) => { + a.trim_end_matches(['/', '\\']) == b.trim_end_matches(['/', '\\']) + } + (Target::Webdav { url: a, .. }, Target::Webdav { url: b, .. }) => { + a.trim_end_matches('/') == b.trim_end_matches('/') + } + ( + Target::S3 { + endpoint: e1, + bucket: b1, + prefix: p1, + .. + }, + Target::S3 { + endpoint: e2, + bucket: b2, + prefix: p2, + .. + }, + ) => { + e1.trim_end_matches('/') == e2.trim_end_matches('/') + && b1 == b2 + && p1.trim_matches('/') == p2.trim_matches('/') + } + _ => false, + } + } +} + +/// **口令和凭据不打印。**`Config` 会整个落进 Debug 输出，而日志是会被贴进 issue 的 +impl std::fmt::Debug for Sync { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Sync") + .field("device_id", &self.device_id) + .field("device_name", &self.device_name) + .field("encrypt", &self.encrypt) + .field( + "passphrase", + &self.passphrase.as_ref().map(|_| ""), + ) + .field("exclude", &self.exclude) + .field("target", &self.target) + .finish() + } +} + +impl std::fmt::Debug for Target { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Target::Folder { path, .. } => f.debug_struct("Folder").field("path", path).finish(), + Target::Webdav { + url, user, proxy, .. + } => f + .debug_struct("Webdav") + .field("url", url) + .field("user", user) + .field("pass", &"") + .field("proxy", proxy) + .finish(), + Target::S3 { + endpoint, + region, + bucket, + prefix, + proxy, + .. + } => f + .debug_struct("S3") + .field("endpoint", endpoint) + .field("region", region) + .field("bucket", bucket) + .field("prefix", prefix) + .field("access_key_id", &"") + .field("secret_access_key", &"") + .field("proxy", proxy) + .finish(), + } + } +} + +/// `sync` 一节写得不对。**英文只写一遍**：`Display` 就是 [`SyncError::msg`] 的原句。 +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum SyncError { + #[error("{}", self.msg())] + DeviceId, + #[error("{}", self.msg())] + DeviceName, + #[error("{}", self.msg())] + PassphraseMissing, + /// `field` 是那个字段在 `sync.target` 下的名字（`path`、`url`、`bucket`……） + #[error("{}", self.msg())] + Blank { field: &'static str }, + #[error("{}", self.msg())] + FolderNotAbsolute { path: String }, + /// 地址不是 `https://`，或者是 `http://` 而主机不在本机或内网 + #[error("{}", self.msg())] + Address { field: &'static str, url: String }, + #[error("{}", self.msg())] + Bucket { bucket: String }, + #[error("{}", self.msg())] + UnknownProxy { proxy: String }, + #[error("{}", self.msg())] + BlankExclude, + #[error("{}", self.msg())] + MultiLine { field: &'static str }, +} + +impl SyncError { + pub fn msg(&self) -> Msg { + match self { + SyncError::DeviceId => msg!( + "sync.config.device_id" => + "sync.device_id has to start with at least 8 lowercase hexadecimal characters. \ + It is generated when sync is turned on" + ), + SyncError::DeviceName => msg!( + "sync.config.device_name", max = DEVICE_NAME_MAX => + "sync.device_name has to be between 1 and {max} characters on one line" + ), + SyncError::PassphraseMissing => msg!( + "sync.config.passphrase_missing" => + "sync.encrypt is on, so sync.passphrase is required" + ), + SyncError::Blank { field } => msg!( + "sync.config.blank", field = field => + "sync.target.{field} cannot be empty" + ), + SyncError::FolderNotAbsolute { path } => msg!( + "sync.config.folder_relative", path = path => + "the sync folder {path} has to be an absolute path" + ), + SyncError::Address { field, url } => msg!( + "sync.config.address", field = field, url = url => + "sync.target.{field} is {url}. Sync locations take https:// addresses; http:// \ + works only for this machine and the local network" + ), + SyncError::Bucket { bucket } => msg!( + "sync.config.bucket", bucket = bucket => + "{bucket} is not a bucket name: use lowercase letters, digits, dots and hyphens" + ), + SyncError::UnknownProxy { proxy } => msg!( + "sync.config.unknown_proxy", proxy = proxy => + "the sync location goes through proxy `{proxy}`, which is not defined under \ + `proxies`; the built-in choices are direct and system" + ), + SyncError::BlankExclude => msg!( + "sync.config.blank_exclude" => + "sync.exclude has an empty upstream name" + ), + SyncError::MultiLine { field } => msg!( + "sync.config.multiline", field = field => + "sync.{field} has to be on one line" + ), + } + } +} + +impl Sync { + /// 只看这一节自己就能判断的。代理名对不对得上 `proxies` 由 [`check_proxy`] 查 + pub fn check(&self) -> Result<(), SyncError> { + if !device_id_ok(&self.device_id) { + return Err(SyncError::DeviceId); + } + let name = self.device_name.trim(); + if name.is_empty() + || self.device_name.chars().count() > DEVICE_NAME_MAX + || self.device_name.chars().any(char::is_control) + { + return Err(SyncError::DeviceName); + } + if self.encrypt && self.passphrase.as_deref().is_none_or(str::is_empty) { + return Err(SyncError::PassphraseMissing); + } + if self + .passphrase + .as_deref() + .is_some_and(|p| p.contains(['\n', '\r'])) + { + return Err(SyncError::MultiLine { + field: "passphrase", + }); + } + if self.exclude.iter().any(|e| e.trim().is_empty()) { + return Err(SyncError::BlankExclude); + } + self.target.check() + } +} + +impl Target { + pub fn check(&self) -> Result<(), SyncError> { + let blank = |field: &'static str, v: &str| { + if v.trim().is_empty() { + Err(SyncError::Blank { field }) + } else if v.contains(['\n', '\r']) { + Err(SyncError::MultiLine { field }) + } else { + Ok(()) + } + }; + match self { + Target::Folder { path, .. } => { + blank("path", path)?; + if !is_absolute(path) { + return Err(SyncError::FolderNotAbsolute { path: path.clone() }); + } + } + Target::Webdav { + url, user, pass, .. + } => { + blank("url", url)?; + if user.contains(['\n', '\r']) { + return Err(SyncError::MultiLine { field: "user" }); + } + if pass.contains(['\n', '\r']) { + return Err(SyncError::MultiLine { field: "pass" }); + } + if !address_ok(url) { + return Err(SyncError::Address { + field: "url", + url: url.clone(), + }); + } + } + Target::S3 { + endpoint, + region, + bucket, + prefix, + access_key_id, + secret_access_key, + .. + } => { + blank("endpoint", endpoint)?; + blank("region", region)?; + blank("bucket", bucket)?; + blank("access_key_id", access_key_id)?; + blank("secret_access_key", secret_access_key)?; + if prefix.contains(['\n', '\r']) { + return Err(SyncError::MultiLine { field: "prefix" }); + } + if !address_ok(endpoint) { + return Err(SyncError::Address { + field: "endpoint", + url: endpoint.clone(), + }); + } + if !bucket_ok(bucket) { + return Err(SyncError::Bucket { + bucket: bucket.clone(), + }); + } + } + } + Ok(()) + } +} + +/// 代理名对得上吗：`direct`、`system`，或者 `proxies` 里有的那一个 +pub fn check_proxy(target: &Target, proxies: &[crate::Proxy]) -> Result<(), SyncError> { + let p = target.proxy(); + if p == DIRECT || p == crate::SYSTEM || proxies.iter().any(|x| x.name == p) { + return Ok(()); + } + Err(SyncError::UnknownProxy { + proxy: p.to_string(), + }) +} + +/// 设备标识：开头至少 8 个小写十六进制字符，其余是十六进制或连字符（UUID 的写法） +pub fn device_id_ok(id: &str) -> bool { + id.len() >= DEVICE_ID_PREFIX + && id.len() <= 64 + && id + .bytes() + .take(DEVICE_ID_PREFIX) + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + && id + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b) || b == b'-') +} + +/// 生成一个设备标识（UUID v4 的写法） +pub fn generate_device_id() -> String { + let b: [u8; 16] = rand::random(); + let h: String = b.iter().map(|x| format!("{x:02x}")).collect(); + format!( + "{}-{}-4{}-{}-{}", + &h[0..8], + &h[8..12], + &h[13..16], + &h[16..20], + &h[20..32] + ) +} + +/// 文件夹路径是不是绝对路径。**两个平台都认**：远程 core 可能是 Linux，而填路径的那一端 +/// 在 Windows 上 —— 判的是写法，不是这台机器 +fn is_absolute(p: &str) -> bool { + let b = p.as_bytes(); + p.starts_with('/') + || p.starts_with("\\\\") + || (b.len() >= 3 + && b[0].is_ascii_alphabetic() + && b[1] == b':' + && matches!(b[2], b'\\' | b'/')) +} + +/// 存储桶名：S3 的规矩（3 到 63 个字符，小写字母、数字、点和连字符，字母或数字开头结尾） +fn bucket_ok(b: &str) -> bool { + (3..=63).contains(&b.len()) + && b.bytes() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == b'.' || c == b'-') + && b.bytes().next().is_some_and(|c| c.is_ascii_alphanumeric()) + && b.bytes().last().is_some_and(|c| c.is_ascii_alphanumeric()) +} + +/// 同步位置的地址收不收：`https://`，或者主机在本机、内网的 `http://`（[`plain_http_ok`]）。 +/// 不带账号密码、查询串和片段 —— 凭据有自己的字段 +pub fn address_ok(raw: &str) -> bool { + let Ok(u) = url::Url::parse(raw.trim()) else { + return false; + }; + if !u.username().is_empty() + || u.password().is_some() + || u.query().is_some() + || u.fragment().is_some() + { + return false; + } + let Some(host) = u.host() else { return false }; + match u.scheme() { + "https" => true, + "http" => plain_http_ok(&host), + _ => false, + } +} + +/// 内网的主机名后缀：`http://` 收这几种。和桌面端导入链接的规则是同一份 +const INTRANET_SUFFIXES: &[&str] = &[ + ".local", + ".lan", + ".internal", + ".intranet", + ".corp", + ".home.arpa", + ".localhost", +]; + +/// `http://` 收不收这个主机：本机回环地址（`localhost`、`127.0.0.1`、`[::1]`）；私有网段的 +/// IPv4（`10/8`、`172.16/12`、`192.168/16`、`100.64/10`、`169.254/16`）和 IPv6（`fc00::/7`、 +/// `fe80::/10`）；没有点的单段主机名，或者以内网后缀结尾的。别的一律只收 `https://` —— +/// 明文传过去的是同步位置的密码，和可能没加密的整份配置 +pub fn plain_http_ok(host: &url::Host<&str>) -> bool { + match host { + url::Host::Domain(d) => { + let d = d.to_ascii_lowercase(); + d == "localhost" || !d.contains('.') || INTRANET_SUFFIXES.iter().any(|s| d.ends_with(s)) + } + url::Host::Ipv4(ip) => { + let [a, b, ..] = ip.octets(); + *ip == std::net::Ipv4Addr::LOCALHOST + || a == 10 + || (a == 172 && (16..=31).contains(&b)) + || (a == 192 && b == 168) + || (a == 100 && (64..=127).contains(&b)) + || (a == 169 && b == 254) + } + url::Host::Ipv6(ip) => { + let first = ip.segments()[0]; + *ip == std::net::Ipv6Addr::LOCALHOST + || first & 0xfe00 == 0xfc00 + || first & 0xffc0 == 0xfe80 + } + } +} + +/// 从一份**读不进来**的配置原文里尽量找出 `sync` 一节（和它引用的出站代理）。 +/// +/// 安全模式下在服务的是一份临时的空配置，而出错页要「从同步位置恢复」：同步位置的连接参数 +/// 只在磁盘上那份写坏了的原文里。这里只要 `sync` 和 `proxies` 两节各自读得进来就行，别处 +/// 坏成什么样都不管。找不到、这一节本身也写坏了是 `None` +pub fn from_raw(text: &str) -> Option<(Sync, Vec)> { + let doc: serde_yaml_ng::Value = serde_yaml_ng::from_str(text).ok()?; + let s: Sync = serde_yaml_ng::from_value(doc.get("sync")?.clone()).ok()?; + s.check().ok()?; + let proxies = doc + .get("proxies") + .and_then(|p| serde_yaml_ng::from_value::>(p.clone()).ok()) + .unwrap_or_default(); + Some((s, proxies)) +} + +#[cfg(test)] +mod tests { + use super::*; + + const ID: &str = "3f9a2c1e-0b1d-4e2f-9a3b-5c6d7e8f9a0b"; + + fn webdav(url: &str) -> Sync { + Sync { + device_id: ID.into(), + device_name: "Mac mini".into(), + encrypt: true, + passphrase: Some("correct horse battery".into()), + exclude: vec![], + target: Target::Webdav { + url: url.into(), + user: "u".into(), + pass: "p".into(), + proxy: DIRECT.into(), + }, + } + } + + #[test] + fn the_section_round_trips_and_keeps_defaults_out() { + let text = format!( + "device_id: {ID}\ndevice_name: Mac mini\npassphrase: secret-secret\ntarget:\n type: folder\n path: /Users/x/Sync\n" + ); + let s: Sync = serde_yaml_ng::from_str(&text).unwrap(); + assert!(s.encrypt, "不写 encrypt 就是加密"); + assert_eq!(s.target.proxy(), DIRECT); + s.check().unwrap(); + let out = serde_yaml_ng::to_string(&s).unwrap(); + assert!(out.contains("encrypt: true"), "{out}"); + assert!(!out.contains("proxy"), "{out}"); + assert!(!out.contains("exclude"), "{out}"); + let back: Sync = serde_yaml_ng::from_str(&out).unwrap(); + assert!(back == s); + } + + #[test] + fn every_target_kind_parses_and_a_typo_is_refused() { + let s3 = "type: s3\nendpoint: https://abc.r2.cloudflarestorage.com\nregion: auto\nbucket: tw-sync\nprefix: me/\naccess_key_id: AK\nsecret_access_key: SK\nproxy: corp\n"; + let t: Target = serde_yaml_ng::from_str(s3).unwrap(); + assert_eq!(t.kind(), "s3"); + assert_eq!(t.proxy(), "corp"); + t.check().unwrap(); + let typo = "type: webdav\nurl: https://dav.example.com/x\nusr: me\n"; + assert!(serde_yaml_ng::from_str::(typo).is_err()); + assert!(serde_yaml_ng::from_str::("type: ftp\npath: /x\n").is_err()); + } + + #[test] + fn a_plain_http_address_is_only_taken_for_this_machine_and_the_local_network() { + for ok in [ + "https://dav.jianguoyun.com/dav/ThinkWatch", + "http://127.0.0.1:5005/dav", + "http://192.168.1.10:9000", + "http://10.0.0.2/webdav", + "http://nas/webdav", + "http://nas.local:5005", + "http://[::1]:8080/", + "http://[fd00::1]/", + ] { + assert!(address_ok(ok), "{ok}"); + webdav(ok).check().unwrap(); + } + for bad in [ + "http://dav.example.com/dav", + "http://8.8.8.8/", + "ftp://nas/x", + "https://user:pw@dav.example.com/", + "https://dav.example.com/?a=1", + "dav.example.com", + ] { + assert!(!address_ok(bad), "{bad}"); + assert!( + matches!(webdav(bad).check(), Err(SyncError::Address { .. })), + "{bad}" + ); + } + } + + #[test] + fn encryption_needs_a_passphrase_and_the_device_id_needs_its_prefix() { + let mut s = webdav("https://dav.example.com/"); + s.passphrase = None; + assert_eq!(s.check(), Err(SyncError::PassphraseMissing)); + s.encrypt = false; + s.check().unwrap(); + s.device_id = "XYZ".into(); + assert_eq!(s.check(), Err(SyncError::DeviceId)); + assert!(device_id_ok(&generate_device_id())); + assert_ne!(generate_device_id(), generate_device_id()); + } + + #[test] + fn folder_paths_have_to_be_absolute_on_either_platform() { + for ok in [ + "/Users/x/iCloud/TW", + "C:\\Users\\x\\Dropbox\\TW", + "D:/Sync", + "\\\\nas\\share", + ] { + assert!(is_absolute(ok), "{ok}"); + } + for bad in ["Sync", "./Sync", "C:Sync", "~/Sync"] { + assert!(!is_absolute(bad), "{bad}"); + } + } + + #[test] + fn debug_never_prints_a_credential() { + let mut s = webdav("https://dav.example.com/"); + s.passphrase = Some("the-passphrase-itself".into()); + s.target = Target::Webdav { + url: "https://dav.example.com/".into(), + user: "me".into(), + pass: "the-password-itself".into(), + proxy: DIRECT.into(), + }; + let shown = format!("{s:?}"); + assert!(!shown.contains("the-passphrase-itself"), "{shown}"); + assert!(!shown.contains("the-password-itself"), "{shown}"); + let s3 = Target::S3 { + endpoint: "https://s3.example.com".into(), + region: "auto".into(), + bucket: "b-1".into(), + prefix: String::new(), + access_key_id: "AKIA-ID-VALUE".into(), + secret_access_key: "SECRET-VALUE".into(), + proxy: DIRECT.into(), + }; + let shown = format!("{s3:?}"); + assert!( + !shown.contains("AKIA-ID-VALUE") && !shown.contains("SECRET-VALUE"), + "{shown}" + ); + } + + #[test] + fn the_section_is_found_in_a_config_that_does_not_load() { + let text = format!( + "version: 1\nnot_a_field: 1\nproxies:\n - name: corp\n addr: 127.0.0.1:7890\nsync:\n device_id: {ID}\n device_name: srv\n encrypt: false\n target:\n type: folder\n path: /srv/sync\n proxy: corp\n" + ); + let (s, proxies) = from_raw(&text).expect("sync 一节读得进来"); + assert_eq!(s.device_name, "srv"); + assert_eq!(proxies.len(), 1); + assert!(from_raw("version: 1\n").is_none()); + assert!(from_raw("version: [\n").is_none()); + } + + #[test] + fn the_same_place_ignores_credentials_and_trailing_slashes() { + let a = webdav("https://dav.example.com/tw/").target; + let mut b = webdav("https://dav.example.com/tw").target; + if let Target::Webdav { pass, .. } = &mut b { + *pass = "other".into(); + } + assert!(a.same_place(&b)); + assert!(!a.same_place(&webdav("https://dav.example.com/other").target)); + } +} diff --git a/crates/tw-config/src/validate.rs b/crates/tw-config/src/validate.rs index f245ed71..5c53b263 100644 --- a/crates/tw-config/src/validate.rs +++ b/crates/tw-config/src/validate.rs @@ -156,6 +156,9 @@ pub enum ValidationError { days: u64, min: u64, }, + /// `sync` 一节写得不对（[`crate::sync::SyncError`]） + #[error("{}", self.msg())] + Sync(crate::sync::SyncError), } impl ValidationError { @@ -443,6 +446,7 @@ impl ValidationError { After a restart the total for the {per} is added up again from the request \ records, so row_days has to be at least {min}" ), + Sync(e) => e.msg(), } } } @@ -677,6 +681,12 @@ pub fn validate(cfg: &Config) -> Result<(), ValidationError> { } } } + // 同步。**代理名要对得上** `proxies`：同步位置的请求经它发出，指到一个不存在的代理上， + // 表现是每一次同步都失败，而原因不在同步设置里 + if let Some(sync) = &cfg.sync { + sync.check().map_err(ValidationError::Sync)?; + crate::sync::check_proxy(&sync.target, &cfg.proxies).map_err(ValidationError::Sync)?; + } Ok(()) } diff --git a/crates/tw-config/tests/manual.rs b/crates/tw-config/tests/manual.rs index 96f55e87..e6fa2f35 100644 --- a/crates/tw-config/tests/manual.rs +++ b/crates/tw-config/tests/manual.rs @@ -378,8 +378,10 @@ fn check_section(s: &Section, all: &[Section], errs: &mut Vec) { s.path, r.name )); } - // 一个列表可以是必填的（策略组的成员）：它自己是值，不是一个有默认值的对象 - let list = matches!(r.kind, Kind::StrsOrObjs(_)) && matches!(r.def, Def::Required); + // 一个列表可以是必填的（策略组的成员）：它自己是值，不是一个有默认值的对象。 + // 一个对象也可以是必填的（同步位置）：不写它这一节就不成立 + let list = matches!(r.kind, Kind::StrsOrObjs(_) | Kind::Obj(_)) + && matches!(r.def, Def::Required); if !list && !matches!(r.def, Def::Section | Def::Is(_) | Def::Unset) { errs.push(format!( "{}.{} is an object; its default is the section's own", diff --git a/crates/tw-config/tests/manual/schema.rs b/crates/tw-config/tests/manual/schema.rs index b67aa589..d885ec39 100644 --- a/crates/tw-config/tests/manual/schema.rs +++ b/crates/tw-config/tests/manual/schema.rs @@ -7,7 +7,7 @@ //! 说明写给手写配置文件的人：这个字段管什么、不写是什么意思、写错了会怎样。 //! 两种语言各写一遍，**不是互译的字面对照**，各自按各自的习惯说。 -use super::{Def, Kind, Lang, Row, Section, T2}; +use super::{Def, Kind, Lang, Row, Section, T2, Ty}; use tw_config::proxy::ProxyAuth; use tw_config::*; use tw_engine::rule::When; @@ -87,6 +87,49 @@ fn limit_pers() -> Vec<&'static str> { super::fields::() } +/// 同步位置三种写法各自的字段。`target` 是按 `type` 分的枚举，问 serde 要字段的办法问不出 +/// 枚举每一种的字段，所以借它的报错：写一个不存在的字段，serde 说出这一种认得的全部字段 +fn target_fields(kind: &str) -> Vec<&'static str> { + let e = serde_yaml_ng::from_str::(&format!( + "type: {kind}\n__probe__: 1\n" + )) + .expect_err("an unknown field is refused") + .to_string(); + let expected = e + .split("expected") + .nth(1) + .unwrap_or_else(|| panic!("serde did not list the fields: {e}")); + let mut out: Vec<&'static str> = vec!["type"]; + out.extend( + expected + .split('`') + .skip(1) + .step_by(2) + .map(|f| &*Box::leak(f.to_string().into_boxed_str())), + ); + out +} +fn folder_fields() -> Vec<&'static str> { + target_fields("folder") +} +fn webdav_fields() -> Vec<&'static str> { + target_fields("webdav") +} +fn s3_fields() -> Vec<&'static str> { + target_fields("s3") +} +fn target_type() -> Vec<&'static str> { + vec!["type"] +} +fn target_types() -> Vec<&'static str> { + vec!["folder", "webdav", "s3"] +} + +const PROXY_DOC: T2 = t( + "Where requests to the sync location go out: `direct`, `system` (the system proxy), or the name of a proxy under `proxies`.", + "访问同步位置的请求从哪里出去：`direct`（直连）、`system`（跟随系统代理），或者 `proxies` 里一个代理的名字。", +); + const RULE_ID: T2 = t("built-in rule id", "内置规则 id"); const MODE_DOC: T2 = t( "`off` does nothing (Off in the app); `observe` detects and records only, and changes nothing \ @@ -263,6 +306,15 @@ pub fn sections() -> Vec
{ "脚本插件，按运行的顺序。由应用安装，每个插件的代码和设置是本文件旁边的一个文件。", ), ), + row( + "sync", + Kind::Obj("sync"), + Def::Unset, + t( + "Multi-device sync. A local setting: configuration packs and other devices never carry it, and restoring a backup keeps the current one. Unset: sync is off.", + "多设备同步。属于本机设置：配置包和其他设备不会带走它，恢复备份时保留当前值。不写：不同步。", + ), + ), ], }, // ── listen ──────────────────────────────────────────── @@ -1829,6 +1881,210 @@ pub fn sections() -> Vec
{ ), ], }, + // ── sync ────────────────────────────────────────────── + Section { + path: "sync", + ty: checked!( + tw_config::sync::Sync, + "{device_id: 3f9a2c1e-0b1d-4e2f-9a3b-5c6d7e8f9a0b, device_name: Mac mini, target: {type: folder, path: /sync}}" + ), + rows: vec![ + row( + "device_id", + Kind::Str, + Def::Required, + t( + "Generated when sync is turned on. File names at the sync location show its first 8 characters.", + "开启同步时生成。同步位置上的文件名里出现的是它的前 8 个字符。", + ), + ), + row( + "device_name", + Kind::Str, + Def::Required, + t( + "Shown in the device list of every device that syncs here.", + "在每台参与同步的设备的设备列表里显示的名字。", + ), + ), + row( + "encrypt", + Kind::Bool, + Def::Is("true"), + t( + "Encrypt what goes to the sync location with the sync passphrase. Every device at one location uses the same setting; change it in the app, which re-uploads the latest version.", + "以同步口令加密存到同步位置的内容。同一同步位置上的所有设备采用同一设置；在应用里更改，应用会重新上传最新版本。", + ), + ), + row( + "passphrase", + Kind::Str, + Def::Unset, + t( + "The sync passphrase; required when `encrypt` is on. Kept here in the clear, like the other keys in this file.", + "同步口令，`encrypt` 开着时必填。和本文件里的其他密钥一样以明文保存。", + ), + ), + row( + "exclude", + Kind::Strs, + Def::Is("[]"), + t( + "Upstreams, by name, that are not synced: they are not uploaded and another device's upstream of the same name does not replace them. Routes, aliases and groups that refer to them are not synced either.", + "不同步的上游，按名字：不上传，也不被其他设备的同名上游覆盖。引用它们的路由、别名、分组随之不同步。", + ), + ), + row( + "target", + Kind::Obj("sync.target"), + Def::Required, + t( + "The sync location. The keyring, `versions/` and `devices/` live directly in it.", + "同步位置。`keyring`、`versions/`、`devices/` 直接放在它里面。", + ), + ), + ], + }, + Section { + path: "sync.target", + ty: Ty::Code { + fields: target_type, + same: super::same_as_omitted::, + parses_without: super::parses_without::, + minimal: "{type: folder}", + }, + rows: vec![row( + "type", + Kind::Enum(target_types), + Def::Required, + t( + "Which kind of location; the other fields depend on it, see the three tables below.", + "同步位置的种类；其余字段随之不同，见下面三张表。", + ), + )], + }, + Section { + path: "sync.target (folder)", + ty: Ty::Code { + fields: folder_fields, + same: super::same_as_omitted::, + parses_without: super::parses_without::, + minimal: "{type: folder, path: /sync}", + }, + rows: vec![ + row("type", Kind::Str, Def::Required, t("`folder`", "`folder`")), + row( + "path", + Kind::Str, + Def::Required, + t( + "An absolute path on the machine core runs on: a folder that iCloud Drive, Dropbox, OneDrive, Syncthing or a NAS share carries between devices.", + "core 所在机器上的绝对路径：由 iCloud Drive、Dropbox、OneDrive、Syncthing 或 NAS 共享在设备之间传送的文件夹。", + ), + ), + row( + "proxy", + Kind::Str, + Def::Is("direct"), + t("Not used for a folder.", "文件夹用不到。"), + ), + ], + }, + Section { + path: "sync.target (webdav)", + ty: Ty::Code { + fields: webdav_fields, + same: super::same_as_omitted::, + parses_without: super::parses_without::, + minimal: "{type: webdav, url: 'https://dav.example.com/ThinkWatch'}", + }, + rows: vec![ + row("type", Kind::Str, Def::Required, t("`webdav`", "`webdav`")), + row( + "url", + Kind::Str, + Def::Required, + t( + "The WebDAV address of the folder. `https://`, with the certificate checked; `http://` only for this machine and the local network.", + "文件夹的 WebDAV 地址。`https://`，校验证书；`http://` 只用于本机和局域网。", + ), + ), + row( + "user", + Kind::Str, + Def::Is("''"), + t("User name.", "用户名。"), + ), + row( + "pass", + Kind::Str, + Def::Is("''"), + t( + "Password, or the app password the service issues for WebDAV.", + "密码，或者服务为 WebDAV 签发的应用密码。", + ), + ), + row("proxy", Kind::Str, Def::Is("direct"), PROXY_DOC), + ], + }, + Section { + path: "sync.target (s3)", + ty: Ty::Code { + fields: s3_fields, + same: super::same_as_omitted::, + parses_without: super::parses_without::, + minimal: "{type: s3, endpoint: 'https://s3.us-east-1.amazonaws.com', region: us-east-1, bucket: tw-sync, access_key_id: AK, secret_access_key: SK}", + }, + rows: vec![ + row("type", Kind::Str, Def::Required, t("`s3`", "`s3`")), + row( + "endpoint", + Kind::Str, + Def::Required, + t( + "The S3 endpoint. Hosts ending in `amazonaws.com`, `aliyuncs.com` or `myqcloud.com` put the bucket in the host name; every other endpoint (Cloudflare R2, MinIO, an address) puts it in the path. `http://` only for this machine and the local network.", + "S3 端点。主机以 `amazonaws.com`、`aliyuncs.com`、`myqcloud.com` 结尾的，存储桶名放进域名；其余端点（Cloudflare R2、MinIO、IP 地址）放进路径。`http://` 只用于本机和局域网。", + ), + ), + row( + "region", + Kind::Str, + Def::Required, + t( + "The signing region; `auto` for Cloudflare R2.", + "签名用的区域；Cloudflare R2 填 `auto`。", + ), + ), + row( + "bucket", + Kind::Str, + Def::Required, + t("The bucket. Keep it private.", "存储桶，建议设为私有。"), + ), + row( + "prefix", + Kind::Str, + Def::Is("''"), + t( + "A folder inside the bucket.", + "存储桶里的一个前缀（文件夹）。", + ), + ), + row( + "access_key_id", + Kind::Str, + Def::Required, + t("Access key ID.", "Access Key ID。"), + ), + row( + "secret_access_key", + Kind::Str, + Def::Required, + t("Secret access key.", "Secret Access Key。"), + ), + row("proxy", Kind::Str, Def::Is("direct"), PROXY_DOC), + ], + }, ] } diff --git a/crates/tw-control/Cargo.toml b/crates/tw-control/Cargo.toml index 725d07ae..69eb079c 100644 --- a/crates/tw-control/Cargo.toml +++ b/crates/tw-control/Cargo.toml @@ -23,6 +23,8 @@ libc = { workspace = true } tw-pack = { workspace = true } base64 = { workspace = true } tw-config = { workspace = true } +# 多设备同步：同步位置、合并、加密 +tw-sync = { workspace = true } tw-guard = { workspace = true } bytes = { workspace = true } tw-engine = { workspace = true } diff --git a/crates/tw-control/src/backups.rs b/crates/tw-control/src/backups.rs index 3bf7b060..c1fccdec 100644 --- a/crates/tw-control/src/backups.rs +++ b/crates/tw-control/src/backups.rs @@ -432,7 +432,7 @@ async fn export( // ─────────────────────────────────────────────────────────── 导入 -fn import_ctx() -> tw_pack::import::Context<'static> { +pub(crate) fn import_ctx() -> tw_pack::import::Context<'static> { tw_pack::import::Context { has_aws_profile: &|p| tw_config::aws_profile::credentials(p).is_ok(), new_key: &tw_config::generate_key, @@ -452,7 +452,7 @@ fn import_fail(e: tw_pack::import::ImportError, pack: &Pack) -> Fail { } } -fn row_view(r: tw_pack::import::Row) -> tw_api::ImportRow { +pub(crate) fn row_view(r: tw_pack::import::Row) -> tw_api::ImportRow { use tw_pack::import::{Flag, Status}; tw_api::ImportRow { item: item_ref(&r.item), @@ -568,7 +568,8 @@ async fn backups(State(s): State) -> Result>, /// 同步位置上的备份版本。没开同步是 None remote: std::sync::RwLock>>, + /// 这份配置的同步服务，第一次用到时建（见 `crate::sync::service`） + sync: std::sync::OnceLock>, } /// 一次配置改动没成的原因。 @@ -127,6 +129,7 @@ impl ConfigManager { applied: tokio::sync::Notify::new(), pinned: std::sync::Mutex::new(pinned), remote: std::sync::RwLock::new(None), + sync: std::sync::OnceLock::new(), } } @@ -149,6 +152,16 @@ impl ConfigManager { self } + /// 同步服务放在这里（`crate::sync::service`） + pub(crate) fn sync_slot(&self) -> &std::sync::OnceLock> { + &self.sync + } + + /// 事件总线 + pub fn bus(&self) -> &tw_observe::EventBus { + &self.bus + } + /// 数据面。管理面里要碰插件文件、编插件的那几处从这里拿 pub fn gateway(&self) -> &tw_gateway::AppState { &self.gateway diff --git a/crates/tw-control/src/lib.rs b/crates/tw-control/src/lib.rs index a5c3eaec..3e2b89a0 100644 --- a/crates/tw-control/src/lib.rs +++ b/crates/tw-control/src/lib.rs @@ -43,6 +43,7 @@ pub mod rotation; pub mod routes; pub mod security; pub mod shutdown; +pub mod sync; pub mod zai; pub use config::{ApplyError, ConfigManager, resolve_path, spawn_watcher}; pub use shutdown::Shutdown; @@ -153,6 +154,7 @@ pub fn router(state: ControlState) -> Router { .merge(pricing::router()) .merge(chatgpt::router()) .merge(zai::router()) + .merge(sync::router()) .fallback(contract::no_such_endpoint) .layer(axum::middleware::from_fn(contract::errors_are_messages)) .with_state(state) @@ -444,7 +446,13 @@ fn provider_view( p: &tw_config::Provider, ) -> tw_api::ProviderView { let listing = s.gateway.models.listing(p); + let synced = sync::service(s); tw_api::ProviderView { + sync_excluded: cfg + .sync + .as_ref() + .is_some_and(|y| y.exclude.iter().any(|n| n == &p.name)), + sync_url_changed: synced.url_changed(&p.name), name: p.name.clone(), base_url: p.base_url.clone(), key: p.key.as_ref().map(|k| k.raw().to_string()), diff --git a/crates/tw-control/src/sync.rs b/crates/tw-control/src/sync.rs new file mode 100644 index 00000000..41e27f91 --- /dev/null +++ b/crates/tw-control/src/sync.rs @@ -0,0 +1,1682 @@ +//! 多设备同步的控制面一半：什么时候同步、同步设置怎么写、端点、状态事件。 +//! +//! 合并规则、同步位置上的文件、加密都在 `tw-sync`；这里把它接到配置的写入流程上： +//! +//! | 时机 | 行为 | +//! |---|---| +//! | 本机配置变更 | 合并最近 10 秒内的变更后上传一个版本。退出前立即上传还没上传的 | +//! | 定时 | WebDAV、S3 每 30 分钟看一次；文件夹位置盯着文件变化，新版本一到就合并 | +//! | 事件 | 启动、从睡眠中唤醒、网络恢复（失败之后按 1、2、5、10 分钟重试）、`POST /sync/now` | +//! +//! 应用别的设备的版本走现有的写入流程（校验、热更新、记一个本机版本），来源记为 `sync`； +//! 这种写入不会再触发上传。**安全模式下不自动同步**（`paused`）。 +//! +//! 连不上同步位置的头 2 小时里状态照旧（已同步 / 冲突），过了 2 小时才是 `unreachable`： +//! 笔记本合上盖子、换个网络都会失败几次，那不该是一条要人处理的提示。 + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::sync::{Arc, Weak}; +use std::time::Duration; + +use axum::Json; +use axum::extract::State; +use axum::http::StatusCode; +use tokio::sync::Notify; +use tw_api::ep; +use tw_config::history::Origin; +use tw_pack::model::{self, ItemId, ItemKind, Snapshot}; +use tw_pack::{Pack, PackOrigin}; +use tw_sync::engine::{self, Current, Decision, Engine, Host, Report, RoundError, Take}; +use tw_sync::{AnyStore, StoreError}; +use tw_types::{Msg, msg}; + +use crate::backups::{BoxFuture, RemoteBackups, RemoteVersionInfo}; +use crate::contract::RouterExt; +use crate::{ApplyError, ConfigManager, ControlState, Fail, apply_fail, fail}; + +/// 本机配置改了之后等多久再上传：这期间的改动合成一个版本 +pub const SETTLE: Duration = Duration::from_secs(10); +/// WebDAV、S3 多久看一次 +pub const POLL: Duration = Duration::from_secs(30 * 60); +/// 连续失败多久才算「同步位置不可用」 +pub const UNREACHABLE_AFTER_MS: u64 = 2 * 3600 * 1000; +/// 失败之后隔多久重试：网络恢复后几分钟内就接上 +const RETRY: &[u64] = &[60, 120, 300, 600]; +/// 退出前上传最多等多久 +const FLUSH_WITHIN: Duration = Duration::from_secs(8); + +pub fn router() -> axum::Router { + axum::Router::new() + .at(ep::SyncStatus, status) + .at(ep::SyncTest, test) + .at(ep::SyncJoinPreview, join_preview) + .at(ep::SyncEnable, enable) + .at(ep::SyncSave, save) + .at(ep::SyncNow, now) + .at(ep::SyncConflicts, conflicts) + .at(ep::SyncResolve, resolve) + .at(ep::SyncEncryption, encryption) + .at(ep::SyncUnlock, unlock) + .at(ep::SyncDisable, disable) +} + +fn now_ms() -> u64 { + crate::config::now_ms() +} + +// ─────────────────────────────────────────────────────────── 本机这一边 + +/// 本机的配置：读成配置包、照配置包写回去 +#[derive(Clone)] +struct CoreHost { + mgr: Weak, +} + +impl CoreHost { + fn mgr(&self) -> Result, Msg> { + self.mgr + .upgrade() + .ok_or_else(|| msg!("sync.shutting_down" => "the core is shutting down")) + } +} + +/// 本机此刻的完整配置包：配置原文、插件文件、Lite 的通用偏好（和备份、导出读的是同一份， +/// `tw_pack::disk::current`） +pub fn current_pack(mgr: &ConfigManager) -> Result { + let loaded = mgr.current().map_err(|e| e.msg())?; + let pack = tw_pack::disk::current(mgr.path(), &loaded.text, PackOrigin::Ui); + let changed_ms = loaded + .fingerprint + .mtime + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|d| d.as_millis() as u64) + .unwrap_or_else(now_ms); + Ok(Current { + version: loaded.version(), + pack, + changed_ms, + }) +} + +/// 照一个配置包写本机，和导入、恢复同一个顺序：攥着插件的编辑锁，先写插件文件和通用偏好，再 +/// 经写入流程写配置（来源记为同步，留一个本机版本，记下是哪台设备传来的）；配置没写成就把 +/// 文件还原 +async fn apply_pack( + mgr: &ConfigManager, + after: Pack, + base_version: &str, + from: &str, +) -> Result<(), engine::ApplyError> { + let path = mgr.path().to_path_buf(); + let _edit = mgr.gateway().plugins.edits.lock().await; + let cur = mgr + .current() + .map_err(|e| engine::ApplyError::Rejected(e.msg()))?; + if cur.version() != base_version { + return Err(engine::ApplyError::Stale); + } + let before = tw_pack::disk::current(&path, &cur.text, PackOrigin::Sync); + let io = |p: &Path, e: std::io::Error| { + engine::ApplyError::Rejected(msg!( + "sync.local.io", path = p.display(), detail = e => + "{path} could not be used: {detail}" + )) + }; + let undo = + tw_pack::disk::write_plugins(&path, &before.plugins, &after.plugins).map_err(|e| { + io( + &tw_config::plugins::dir_in(&tw_pack::disk::dir_of(&path)), + e, + ) + })?; + // 远程连接下偏好在 Lite 那边；同步这一路跑在 core 自己的任务里，不是远程连接 + let prefs_changed = after.client_prefs.is_some() && after.client_prefs != before.client_prefs; + if prefs_changed && let Err(e) = tw_pack::disk::write_prefs(&path, after.client_prefs.as_ref()) + { + undo.restore(); + return Err(io(&tw_pack::disk::prefs_path(&path), e)); + } + let started = now_ms(); + if let Err(e) = mgr + .write(&after.config_yaml, Some(base_version), Origin::Sync) + .await + { + undo.restore(); + if prefs_changed { + let _ = tw_pack::disk::write_prefs(&path, before.client_prefs.as_ref()); + } + return Err(match e { + ApplyError::Stale { .. } + | ApplyError::Store(tw_config::StoreError::Conflict { .. }) => { + engine::ApplyError::Stale + } + other => engine::ApplyError::Rejected(other.msg()), + }); + } + // 本机这一版是从哪台设备来的：备份列表里写「同步自 MacBook Air」 + if !from.is_empty() + && let Ok(all) = tw_pack::backup::list(&path) + && let Some(b) = all + .iter() + .rev() + .find(|b| b.origin == Origin::Sync && b.at_ms + 1000 >= started) + && let Err(e) = tw_pack::backup::note_device(b, from) + { + tracing::debug!("the backup version could not be labelled with its device: {e}"); + } + if prefs_changed { + mgr.bus().emit(tw_api::Event::ClientPrefsChanged { + id: mgr.bus().next_id(), + at_ms: now_ms(), + }); + } + Ok(()) +} + +impl Host for CoreHost { + fn current(&self) -> Result { + current_pack(&*self.mgr()?) + } + async fn apply( + &self, + pack: Pack, + base_version: &str, + from: &str, + ) -> Result<(), engine::ApplyError> { + let mgr = self.mgr().map_err(engine::ApplyError::Rejected)?; + apply_pack(&mgr, pack, base_version, from).await + } + fn pinned_keys(&self) -> BTreeSet { + self.mgr + .upgrade() + .map(|m| m.pinned_keys()) + .unwrap_or_default() + } + /// 本机备份里内容一样的那一版（找合并基准）。新的在前 + fn backup(&self, content_id: &str) -> Option { + let mgr = self.mgr.upgrade()?; + let all = tw_pack::backup::list(mgr.path()).ok()?; + all.iter() + .rev() + .filter_map(|b| tw_pack::backup::open(b).ok()) + .find(|p| p.content_id() == content_id) + } +} + +// ─────────────────────────────────────────────────────────── 同步位置 + +/// 发 WebDAV、S3 请求用的客户端：经同步设置里选的出站代理，**不跟重定向**（跟到别的主机， +/// 同步位置的密码就跟着去了），证书照常校验 +fn http(cfg: &tw_config::Config, proxy: &str) -> Result { + let mut b = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(10)) + .redirect(reqwest::redirect::Policy::none()) + .user_agent(tw_gateway::user_agent()); + match proxy { + tw_config::DIRECT => b = b.no_proxy(), + tw_config::SYSTEM => {} + name => { + let p = cfg.proxies.iter().find(|x| x.name == name).ok_or_else(|| { + tw_config::sync::SyncError::UnknownProxy { + proxy: name.to_string(), + } + .msg() + })?; + let url = p.url().map_err(|e| e.msg())?; + let px = reqwest::Proxy::all(&url).map_err(|e| { + msg!( + "sync.proxy_unusable", proxy = name, detail = e => + "proxy `{proxy}` cannot be used for the sync location: {detail}" + ) + })?; + b = b.proxy(px); + } + } + b.build().map_err(|e| { + msg!( + "sync.http_client", detail = e => + "the connection to the sync location could not be set up: {detail}" + ) + }) +} + +fn store_for(cfg: &tw_config::Config, t: &tw_config::sync::Target) -> Result { + t.check().map_err(|e| e.msg())?; + tw_config::sync::check_proxy(t, &cfg.proxies).map_err(|e| e.msg())?; + AnyStore::new(t, http(cfg, t.proxy())?).map_err(|e| e.msg()) +} + +pub fn target_to_config(t: &tw_api::SyncTarget) -> tw_config::sync::Target { + use tw_api::SyncTarget as A; + use tw_config::sync::Target as C; + let proxy = |p: &str| { + if p.trim().is_empty() { + tw_config::DIRECT.to_string() + } else { + p.trim().to_string() + } + }; + match t { + A::Folder { path, proxy: p } => C::Folder { + path: path.trim().to_string(), + proxy: proxy(p), + }, + A::Webdav { + url, + user, + pass, + proxy: p, + } => C::Webdav { + url: url.trim().to_string(), + user: user.clone(), + pass: pass.clone(), + proxy: proxy(p), + }, + A::S3 { + endpoint, + region, + bucket, + prefix, + access_key_id, + secret_access_key, + proxy: p, + } => C::S3 { + endpoint: endpoint.trim().to_string(), + region: region.trim().to_string(), + bucket: bucket.trim().to_string(), + prefix: prefix.trim().trim_matches('/').to_string(), + access_key_id: access_key_id.trim().to_string(), + secret_access_key: secret_access_key.trim().to_string(), + proxy: proxy(p), + }, + } +} + +pub fn target_to_api(t: &tw_config::sync::Target) -> tw_api::SyncTarget { + use tw_api::SyncTarget as A; + use tw_config::sync::Target as C; + match t.clone() { + C::Folder { path, proxy } => A::Folder { path, proxy }, + C::Webdav { + url, + user, + pass, + proxy, + } => A::Webdav { + url, + user, + pass, + proxy, + }, + C::S3 { + endpoint, + region, + bucket, + prefix, + access_key_id, + secret_access_key, + proxy, + } => A::S3 { + endpoint, + region, + bucket, + prefix, + access_key_id, + secret_access_key, + proxy, + }, + } +} + +/// 写 `sync` 一节（`None` 是删掉）。这是本机设置：同步一轮里比较的快照不含它，写它不会 +/// 触发上传 +async fn write_section( + mgr: &ConfigManager, + sync: Option<&tw_config::sync::Sync>, + origin: Origin, +) -> Result { + let value = match sync { + Some(s) => Some(serde_yaml_ng::to_value(s).map_err(|e| { + ApplyError::Invalid(msg!( + "sync.write_section", detail = e => + "the sync settings could not be written: {detail}" + )) + })?), + None => None, + }; + mgr.transform(None, origin, |text, _| { + Ok(tw_config::edit::set( + text, + &[tw_yaml::Step::key("sync")], + value.as_ref(), + )?) + }) + .await +} + +/// 引擎的设置：`sync` 一节，加上这台设备是不是远程 core（开着远程控制端口） +fn settings_of(cfg: &tw_config::Config, sync: &tw_config::sync::Sync) -> engine::Settings { + let mut s = engine::Settings::of(sync); + s.remote_core = cfg + .listen + .control + .remote + .as_ref() + .is_some_and(|r| r.enabled); + s +} + +// ─────────────────────────────────────────────────────────── 服务 + +type Eng = Engine; + +struct Running { + engine: Eng, + /// 建它时的 `sync` 一节和出站代理：变了就重建 + config: tw_config::sync::Sync, + proxies: String, + /// 文件夹位置的监听，留着 + _watch: Option, +} + +/// 同步服务。每份配置一个（[`service`]） +pub struct Sync { + /// 自己（装进备份列表的那扇门拿着它） + me: Weak, + mgr: Weak, + run: tokio::sync::Mutex>, + view: std::sync::Mutex>, + /// 经同步改了地址的上游（不加密时）。上游视图读它 + url_changed: std::sync::Mutex>, + kick: Arc, + /// 本机改了配置、还没上传：这个时刻之后上传 + dirty: std::sync::Mutex>, + paused: AtomicBool, + started: AtomicBool, + /// 连续失败了几次（重试的间隔） + failures: AtomicU32, +} + +/// 这份配置的同步服务。第一次用的时候建（端点、启动、测试都走这里） +pub fn service(s: &ControlState) -> Arc { + of(&s.cfg) +} + +pub fn of(mgr: &Arc) -> Arc { + mgr.sync_slot() + .get_or_init(|| { + let dir = crate::plugins::dir_of(mgr.path()); + let url_changed = tw_sync::local::Local::new(&dir) + .state() + .url_changed + .into_iter() + .map(|(k, v)| (k, url_change(&v))) + .collect(); + Arc::new_cyclic(|me| Sync { + me: me.clone(), + mgr: Arc::downgrade(mgr), + run: tokio::sync::Mutex::new(None), + view: std::sync::Mutex::new(None), + url_changed: std::sync::Mutex::new(url_changed), + kick: Arc::new(Notify::new()), + dirty: std::sync::Mutex::new(None), + paused: AtomicBool::new(false), + started: AtomicBool::new(false), + failures: AtomicU32::new(0), + }) + }) + .clone() +} + +fn url_change(v: &tw_sync::local::UrlChange) -> tw_api::SyncUrlChange { + tw_api::SyncUrlChange { + from: v.from.clone(), + device: v.device.clone(), + at_ms: v.at_ms, + } +} + +/// 一轮或者一次设置操作没成，作为控制面的失败 +fn round_fail(e: &RoundError) -> Fail { + let code = match e { + RoundError::Store(StoreError::AuthFailed(_)) => StatusCode::FORBIDDEN, + RoundError::Store(_) => StatusCode::BAD_GATEWAY, + RoundError::Locked(_) => StatusCode::BAD_REQUEST, + RoundError::TooNew { .. } | RoundError::Stale | RoundError::Emptied => StatusCode::CONFLICT, + RoundError::NotArrived => StatusCode::SERVICE_UNAVAILABLE, + RoundError::Local(_) => StatusCode::BAD_REQUEST, + }; + fail(code, e.msg()) +} + +fn sync_off() -> Fail { + fail( + StatusCode::CONFLICT, + msg!("sync.off" => "sync is not turned on"), + ) +} + +/// 口令够长吗（设口令的那一刻才查） +fn check_passphrase(p: Option<&str>) -> Result<&str, Fail> { + match p { + Some(p) if p.chars().count() >= tw_config::sync::PASSPHRASE_MIN => Ok(p), + _ => Err(fail( + StatusCode::BAD_REQUEST, + msg!( + "sync.passphrase_short", min = tw_config::sync::PASSPHRASE_MIN => + "the sync passphrase has to be at least {min} characters" + ), + )), + } +} + +use crate::backups::item_ref; + +impl Sync { + fn mgr(&self) -> Option> { + self.mgr.upgrade() + } + + fn data_dir(&self) -> Option { + self.mgr().map(|m| crate::plugins::dir_of(m.path())) + } + + /// 某个上游的地址最近一次经同步改过：改之前的地址、哪台设备、什么时候 + pub fn url_changed(&self, provider: &str) -> Option { + self.url_changed + .lock() + .ok() + .and_then(|m| m.get(provider).cloned()) + } + + /// 本机配置变了：`SETTLE` 之后上传 + pub fn touch(&self) { + if let Ok(mut d) = self.dirty.lock() + && d.is_none() + { + *d = Some(tokio::time::Instant::now() + SETTLE); + self.kick.notify_one(); + } + } + + /// 马上来一轮（不等定时） + pub fn kick(&self) { + self.kick.notify_one(); + } + + /// 配置里的 `sync` 一节和此刻在跑的对得上；对不上就照配置重建（同步位置、口令、不同步 + /// 的上游、代理变了）。没开同步是 `None` + fn ensure<'a>(&self, run: &'a mut Option) -> Result, Msg> { + let Some(mgr) = self.mgr() else { + *run = None; + return Ok(None); + }; + let cfg = mgr.gateway().config(); + let Some(sync) = cfg.sync.clone() else { + *run = None; + // 安全模式下配置里没有这一节是因为在服务的是临时配置：门留着（见 `start`） + if !self.paused.load(Ordering::Relaxed) { + self.register_backups(false); + } + return Ok(None); + }; + self.register_backups(true); + let proxies = format!("{:?}", cfg.proxies); + let fresh = match run.as_ref() { + Some(r) => r.config == sync && r.proxies == proxies, + None => false, + }; + if !fresh { + let store = store_for(&cfg, &sync.target)?; + let watch = match store.folder_root() { + Some(root) => match tw_sync::store::folder::watch(root) { + Ok((w, mut rx)) => { + let kick = self.kick.clone(); + tokio::spawn(async move { + while rx.recv().await.is_some() { + kick.notify_one(); + } + }); + Some(w) + } + Err(e) => { + tracing::warn!( + "the sync folder cannot be watched, so it is checked every 30 minutes: {e}" + ); + None + } + }, + None => None, + }; + let dir = crate::plugins::dir_of(mgr.path()); + let mut engine = Engine::new( + store, + CoreHost { + mgr: Arc::downgrade(&mgr), + }, + &dir, + settings_of(&cfg, &sync), + ); + // 同一个位置、同一个口令：缓存和捷径留着也没错，但重建了就从头看一遍 + engine.invalidate(); + *run = Some(Running { + engine, + config: sync, + proxies, + _watch: watch, + }); + } + Ok(run.as_mut()) + } + + /// 此刻给界面看的样子 + pub fn view(&self) -> tw_api::SyncView { + let on = self + .mgr() + .map(|m| m.gateway().config().sync.is_some()) + .unwrap_or(false); + if let Ok(v) = self.view.lock() + && let Some(v) = v.as_ref() + // 配置里开着、记下的却是关着（刚开启），或者反过来（手改文件关掉了）：重新拼 + && on == (v.state != tw_api::SyncState::Off) + { + return v.clone(); + } + self.compute_view(None, None) + } + + /// 照配置、本机的同步状态和上一轮的结果拼出视图 + fn compute_view( + &self, + report: Option<&Report>, + state: Option, + ) -> tw_api::SyncView { + let off = tw_api::SyncView { + state: tw_api::SyncState::Off, + target: None, + encrypted: false, + device: None, + last_synced_ms: None, + failing_since_ms: None, + error: None, + devices: Vec::new(), + conflicts: 0, + excluded: Vec::new(), + excluded_cascade: Vec::new(), + }; + let Some(mgr) = self.mgr() else { return off }; + let cfg = mgr.gateway().config(); + let paused = self.paused.load(Ordering::Relaxed); + // 安全模式下在服务的是一份临时配置：同步设置在磁盘上那份读不进来的原文里 + let raw = (paused && cfg.sync.is_none()) + .then(|| mgr.current().ok()) + .flatten() + .and_then(|l| tw_config::sync::from_raw(&l.text)) + .map(|(s, _)| s); + let Some(sync) = cfg.sync.as_ref().or(raw.as_ref()) else { + return off; + }; + let local = tw_sync::local::Local::new(&crate::plugins::dir_of(mgr.path())); + let st = local.state(); + let report = report.cloned(); + let prev = self.view.lock().ok().and_then(|v| v.clone()); + let conflicts = report + .as_ref() + .map(|r| r.conflicts.len() as u32) + .or(prev.as_ref().map(|p| p.conflicts)) + .unwrap_or(0); + let devices = match &report { + Some(r) => r + .devices + .iter() + .map(|d| tw_api::SyncDevice { + id: d.id.clone(), + name: d.name.clone(), + core_version: d.core_version.clone(), + last_seen_ms: d.last_seen_ms, + this: d.id == sync.device_id, + paused_update: d.paused_update, + remote_core: d.remote_core, + }) + .collect(), + None => prev.as_ref().map(|p| p.devices.clone()).unwrap_or_default(), + }; + let state = if paused { + tw_api::SyncState::Paused + } else { + state.unwrap_or(if conflicts > 0 { + tw_api::SyncState::Conflict + } else { + tw_api::SyncState::Idle + }) + }; + let cascade = current_pack(&mgr) + .ok() + .and_then(|c| model::snapshot(&c.pack).ok()) + .map(|snap| { + let names: BTreeSet = sync.exclude.iter().cloned().collect(); + engine::excluded(&names, &[&snap]) + .into_iter() + .filter(|id| !(id.kind == ItemKind::Provider && names.contains(&id.name))) + .filter(|id| snap.items.contains_key(id)) + .map(|id| item_ref(&id)) + .collect() + }) + .unwrap_or_default(); + tw_api::SyncView { + state, + target: Some(target_to_api(&sync.target)), + encrypted: report.as_ref().map(|r| r.encrypted).unwrap_or(sync.encrypt), + device: Some(tw_api::SyncDeviceRef { + id: sync.device_id.clone(), + name: sync.device_name.clone(), + }), + last_synced_ms: st.last_synced_ms, + failing_since_ms: st.failing_since_ms, + error: st.error.clone(), + devices, + conflicts, + excluded: sync.exclude.clone(), + excluded_cascade: cascade, + } + } + + fn publish(&self, v: tw_api::SyncView) { + let changed = self + .view + .lock() + .map(|mut g| { + let changed = g.as_ref() != Some(&v); + *g = Some(v.clone()); + changed + }) + .unwrap_or(false); + if changed && let Some(mgr) = self.mgr() { + mgr.bus().emit(tw_api::Event::SyncStatus { + id: mgr.bus().next_id(), + view: v, + at_ms: now_ms(), + }); + } + } + + /// 记一次失败：从什么时候开始一直失败、原因 + fn record_failure(&self, r: &Running, e: &RoundError) { + let local = r.engine.local(); + let mut st = local.state(); + st.failing_since_ms.get_or_insert_with(now_ms); + st.error = Some(e.msg()); + let _ = local.save_state(&st); + self.failures.fetch_add(1, Ordering::Relaxed); + } + + /// 同步一轮（顺带处理几处冲突），更新状态、发事件 + pub async fn round(&self, decisions: &[Decision]) -> Result { + let mut guard = self.run.lock().await; + let r = match self.ensure(&mut guard) { + Ok(Some(r)) => r, + Ok(None) => { + let v = self.compute_view(None, None); + self.publish(v.clone()); + return Ok(v); + } + Err(m) => { + let v = tw_api::SyncView { + error: Some(m.clone()), + ..self.compute_view(None, Some(tw_api::SyncState::AuthFailed)) + }; + self.publish(v.clone()); + return Err(fail(StatusCode::BAD_REQUEST, m)); + } + }; + if self.paused.load(Ordering::Relaxed) { + let v = self.compute_view(None, None); + self.publish(v.clone()); + return Ok(v); + } + if let Ok(mut d) = self.dirty.lock() { + *d = None; + } + if let Ok(mut g) = self.view.lock() + && let Some(v) = g.as_mut() + { + v.state = tw_api::SyncState::Syncing; + } + let mut result = r.engine.round_with(decisions).await; + for _ in 0..2 { + if matches!(result, Err(RoundError::Stale)) { + result = r.engine.round_with(decisions).await; + } + } + let outcome = match result { + Ok(report) => { + self.failures.store(0, Ordering::Relaxed); + let encrypted = report.encrypted; + let st = r.engine.local().state(); + if let Ok(mut m) = self.url_changed.lock() { + *m = st + .url_changed + .iter() + .map(|(k, v)| (k.clone(), url_change(v))) + .collect(); + } + if !report.keys_changed.is_empty() + && let Some(mgr) = self.mgr() + { + mgr.bus().emit(tw_api::Event::GatewayKeysChanged { + id: mgr.bus().next_id(), + names: report.keys_changed.clone(), + at_ms: now_ms(), + }); + } + // 别的设备关掉了加密：这台跟着改设置 + let follow = (encrypted != r.config.encrypt && !encrypted).then(|| { + let mut s = r.config.clone(); + s.encrypt = false; + s.passphrase = None; + s + }); + Ok((report, follow)) + } + Err(e) => Err(e), + }; + let v = match outcome { + Ok((report, follow)) => { + let v = self.compute_view(Some(&report), None); + drop(guard); + if let (Some(s), Some(mgr)) = (follow, self.mgr()) { + let _ = write_section(&mgr, Some(&s), Origin::Sync).await; + } + v + } + Err(RoundError::NotArrived) => { + drop(guard); + self.view() + } + Err(RoundError::Emptied) => { + // 别的设备停止同步并删除了同步数据：这台也停下，说一声 + r.engine.local().clear(); + drop(guard); + if let Some(mgr) = self.mgr() { + let _ = write_section(&mgr, None, Origin::Sync).await; + } + *self.run.lock().await = None; + self.register_backups(false); + tw_api::SyncView { + error: Some(RoundError::Emptied.msg()), + ..self.compute_view(None, Some(tw_api::SyncState::Off)) + } + } + Err(e) => { + self.record_failure(r, &e); + let st = r.engine.local().state(); + let state = match &e { + RoundError::Store(StoreError::AuthFailed(_)) => tw_api::SyncState::AuthFailed, + RoundError::Locked(_) => tw_api::SyncState::Locked, + RoundError::TooNew { .. } => tw_api::SyncState::TooNew, + _ => { + let since = st.failing_since_ms.unwrap_or_else(now_ms); + if now_ms().saturating_sub(since) >= UNREACHABLE_AFTER_MS { + tw_api::SyncState::Unreachable + } else { + let prev = self.view.lock().ok().and_then(|v| v.clone()); + match prev.map(|p| p.conflicts) { + Some(n) if n > 0 => tw_api::SyncState::Conflict, + _ => tw_api::SyncState::Idle, + } + } + } + }; + drop(guard); + self.compute_view(None, Some(state)) + } + }; + self.publish(v.clone()); + Ok(v) + } + + /// 退出前：还有没上传的改动就马上传 + pub async fn flush(&self) { + let pending = self.dirty.lock().map(|d| d.is_some()).unwrap_or(false); + if !pending || self.paused.load(Ordering::Relaxed) { + return; + } + let _ = tokio::time::timeout(FLUSH_WITHIN, self.round(&[])).await; + } + + /// 下一次定时检查隔多久：失败着就按重试的间隔 + fn next_poll(&self) -> Duration { + let n = self.failures.load(Ordering::Relaxed) as usize; + if n == 0 { + return POLL; + } + Duration::from_secs(RETRY[(n - 1).min(RETRY.len() - 1)]) + } + + /// 冲突列表 + pub async fn conflicts(&self) -> Vec { + let guard = self.run.lock().await; + let Some(r) = guard.as_ref() else { + return Vec::new(); + }; + let me = r.config.device_id.clone(); + let Some(mgr) = self.mgr() else { + return Vec::new(); + }; + let local = current_pack(&mgr) + .ok() + .and_then(|c| model::snapshot(&c.pack).ok()) + .unwrap_or_default(); + r.engine + .last_report() + .conflicts + .iter() + .map(|c| { + let units = local.items.get(&c.item).map(|it| c.pick(&it.units)); + let (mine, theirs) = c.mine(&me, units.as_ref()); + tw_api::SyncConflict { + id: c.id.clone(), + item: item_ref(&c.item), + units: c.units.clone(), + local: tw_api::SyncConflictSide { + device: mine.device_name.clone(), + at_ms: mine.at_ms, + deleted: mine.units.is_none(), + }, + remote: tw_api::SyncConflictSide { + device: theirs.device_name.clone(), + at_ms: theirs.at_ms, + deleted: theirs.units.is_none(), + }, + diff: c + .units + .iter() + .map(|u| tw_api::SyncConflictDiff { + unit: u.clone(), + local: mine.units.as_ref().and_then(|m| m.get(u).cloned()), + remote: theirs.units.as_ref().and_then(|m| m.get(u).cloned()), + }) + .collect(), + both_allowed: c.both_allowed(), + reason: c.reason.map(|r| match r { + tw_sync::record::Reason::KeyInUse => tw_api::SyncConflictReason::KeyInUse, + tw_sync::record::Reason::Invalid => tw_api::SyncConflictReason::Invalid, + }), + } + }) + .collect() + } +} + +// ─────────────────────────────────────────────────────────── 同步位置上的备份 + +impl Sync { + /// 读同步位置用的引擎：开着同步就是在跑的那一个。**安全模式下**在服务的是一份临时配置， + /// 没有 `sync` 一节：照磁盘上那份读不进来的原文里的 `sync` 一节临时建一个 + async fn with_engine( + &self, + f: impl AsyncFnOnce(&mut Eng) -> Result, + ) -> Option> { + let mut guard = self.run.lock().await; + if let Ok(Some(r)) = self.ensure(&mut guard) { + return Some(f(&mut r.engine).await); + } + drop(guard); + let mgr = self.mgr()?; + let text = mgr.current().ok()?.text; + let (sync, proxies) = tw_config::sync::from_raw(&text)?; + let cfg = tw_config::Config { + proxies, + ..Default::default() + }; + let store = store_for(&cfg, &sync.target).ok()?; + let mut e = Engine::new( + store, + CoreHost { + mgr: Arc::downgrade(&mgr), + }, + &crate::plugins::dir_of(mgr.path()), + settings_of(&cfg, &sync), + ); + Some(f(&mut e).await) + } + + /// 同步位置上读得出来的版本。没开同步、连不上是空的 + pub async fn remote_versions(&self) -> Vec { + let got = self + .with_engine(async |e: &mut Eng| e.remote_versions().await) + .await; + match got { + Some(Ok(vs)) => vs + .iter() + .map(|v| RemoteVersionInfo { + content_id: v.content.clone(), + at_ms: v.pack.manifest.created_at_ms, + device_name: v.meta.device_name.clone(), + core_version: v.pack.manifest.core_version.clone(), + counts: v.pack.manifest.counts, + }) + .collect(), + Some(Err(e)) => { + tracing::debug!( + "the sync location could not be listed for backups: {}", + e.msg() + ); + Vec::new() + } + None => Vec::new(), + } + } + + /// 同步位置上内容标识是这个的版本（不加密的配置包，不含本机设置） + pub async fn remote_pack(&self, content_id: &str) -> Result, Msg> { + let id = content_id.to_string(); + match self + .with_engine(async move |e: &mut Eng| e.remote_version(&id).await) + .await + { + Some(Ok(p)) => Ok(p), + Some(Err(e)) => Err(e.msg()), + None => Ok(None), + } + } +} + +/// 备份列表看同步位置的那一扇门（`crate::backups::RemoteBackups`）。开着同步时装在配置管理上 +/// （[`ConfigManager::set_remote_backups`]），关掉时拿掉 +struct Backups(Weak); + +impl RemoteBackups for Backups { + fn list(&self) -> BoxFuture<'_, Vec> { + Box::pin(async move { + match self.0.upgrade() { + Some(s) => s.remote_versions().await, + None => Vec::new(), + } + }) + } + + fn fetch<'a>(&'a self, content_id: &'a str) -> BoxFuture<'a, Result> { + Box::pin(async move { + let Some(s) = self.0.upgrade() else { + return Err(msg!("sync.shutting_down" => "the core is shutting down")); + }; + s.remote_pack(content_id).await?.ok_or_else(|| { + msg!( + "sync.version_gone", id = content_id => + "the sync location no longer has version {id}" + ) + }) + }) + } + + fn other_devices(&self) -> BoxFuture<'_, u32> { + Box::pin(async move { + self.0 + .upgrade() + .map(|s| s.view().devices.iter().filter(|d| !d.this).count() as u32) + .unwrap_or(0) + }) + } +} + +impl Sync { + /// 开着同步（配置里有 `sync` 一节，安全模式下原文里有）就把这扇门装上，没开就拿掉 + fn register_backups(&self, on: bool) { + let Some(mgr) = self.mgr() else { return }; + let have = mgr.remote_backups().is_some(); + if on && !have { + mgr.set_remote_backups(Some(Arc::new(Backups(self.me.clone())))); + } else if !on && have { + mgr.set_remote_backups(None); + } + } +} + +// ─────────────────────────────────────────────────────────── 调度 + +/// 起同步的后台任务：启动时一轮，之后按上面那张表。`safe` 是安全模式：不自动同步，状态是 +/// `paused`，备份列表照样能读同步位置 +pub fn start(s: &ControlState, safe: bool) -> Arc { + let sync = service(s); + if safe { + sync.paused.store(true, Ordering::Relaxed); + // 出错页的「从备份恢复」要列同步位置上的版本：原文里有同步设置就把门装上 + let on = s.config().sync.is_some() + || s.cfg + .current() + .ok() + .and_then(|l| tw_config::sync::from_raw(&l.text)) + .is_some(); + sync.register_backups(on); + sync.publish(sync.compute_view(None, None)); + return sync; + } + if sync.started.swap(true, Ordering::Relaxed) { + return sync; + } + let bus = s.bus().clone(); + let me = sync.clone(); + tokio::spawn(async move { + let mut rx = bus.subscribe(); + // 从睡眠中醒来：墙上的钟比单调钟多走了一大截 + let mut wall = std::time::SystemTime::now(); + let mut mono = tokio::time::Instant::now(); + let mut tick = tokio::time::interval(Duration::from_secs(60)); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let mut next = tokio::time::Instant::now(); + loop { + let dirty = me.dirty.lock().ok().and_then(|d| *d); + let due = match dirty { + Some(d) if d < next => d, + _ => next, + }; + tokio::select! { + _ = me.kick.notified() => { + // 本机刚改了配置：等它稳下来 + if let Some(d) = me.dirty.lock().ok().and_then(|d| *d) + && d > tokio::time::Instant::now() + { + continue; + } + } + _ = tokio::time::sleep_until(due) => {} + ev = rx.recv() => { + match ev { + Ok(tw_api::Event::ConfigReloaded { origin, .. }) + if origin != tw_api::ConfigOrigin::Sync => me.touch(), + Ok(tw_api::Event::ClientPrefsChanged { .. }) => me.touch(), + Err(tokio::sync::broadcast::error::RecvError::Closed) => return, + _ => {} + } + continue; + } + _ = tick.tick() => { + let wall_gap = std::time::SystemTime::now() + .duration_since(wall) + .unwrap_or_default(); + let mono_gap = mono.elapsed(); + wall = std::time::SystemTime::now(); + mono = tokio::time::Instant::now(); + if wall_gap < mono_gap + Duration::from_secs(120) { + continue; + } + tracing::debug!("woke from sleep; checking the sync location"); + } + } + if me.mgr().is_none() { + return; + } + let _ = me.round(&[]).await; + next = tokio::time::Instant::now() + me.next_poll(); + } + }); + sync.kick(); + sync +} + +// ─────────────────────────────────────────────────────────── 加入 + +/// 本机配置还是初始配置（新安装后没改过）：只有一把网关密钥，没有上游、路由这些，全局设置 +/// 都是默认值。随 core 发的默认插件（停用着）不算 +fn is_initial(snap: &Snapshot) -> bool { + let defaults: BTreeSet<&str> = tw_gateway::plugin::defaults::ALL + .iter() + .map(|(id, _)| *id) + .collect(); + let mut keys = 0; + for (id, it) in &snap.items { + match id.kind { + ItemKind::Key => keys += 1, + ItemKind::ClientPrefs | ItemKind::PluginOrder => {} + ItemKind::Plugin + if defaults.contains(id.name.as_str()) + && it.units.get(model::unit::ENABLED) + != Some(&serde_json::Value::Bool(true)) => {} + _ => return false, + } + } + keys <= 1 +} + +/// 加入时「合并」：和导入同一套规矩（`tw_pack::import`），本机打底，同步数据逐项并进来。 +/// 新增的默认导入，冲突的必须选（覆盖、改名导入、跳过），没选是 `sync.join_undecided`。 +/// +/// **两处和导入不一样**：导入进来的插件按导入的规矩是停用的，加入同步时照同步数据里的开关 +/// （各设备一致，见方案 6.2；不加密时同步一轮里另有停用的规矩）；导入预览里只有本机有的 +/// 项目不出现，这里也一样 —— 它们会随这次加入上传 +fn merge_join( + local: &Pack, + remote: &Pack, + decisions: &[tw_api::ImportDecision], +) -> Result { + let ctx = crate::backups::import_ctx(); + let rows = tw_pack::import::preview(local, remote, &ctx).map_err(|e| import_fail(&e))?; + let decided: BTreeMap = decisions + .iter() + .map(|d| (crate::backups::item_id(&d.item), d)) + .collect(); + let mut out = Vec::new(); + for r in &rows { + let action = match decided.get(&r.item) { + Some(d) => match d.action { + tw_api::ImportAction::Import => tw_pack::import::Action::Import, + tw_api::ImportAction::Overwrite => tw_pack::import::Action::Overwrite, + tw_api::ImportAction::Rename => { + tw_pack::import::Action::Rename(d.new_name.clone().unwrap_or_default()) + } + tw_api::ImportAction::Skip => tw_pack::import::Action::Skip, + }, + None => match r.status { + tw_pack::import::Status::New => tw_pack::import::Action::Import, + tw_pack::import::Status::Same | tw_pack::import::Status::Error => { + tw_pack::import::Action::Skip + } + tw_pack::import::Status::Conflict => { + return Err(fail( + StatusCode::BAD_REQUEST, + msg!( + "sync.join_undecided", item = &r.item => + "{item} differs between this device and the sync data; choose to \ + overwrite, import under a new name, or skip" + ), + )); + } + }, + }; + out.push(tw_pack::import::Decision { + item: r.item.clone(), + action, + }); + } + let applied = tw_pack::import::apply(local, remote, &out, &ctx).map_err(|e| import_fail(&e))?; + // 插件照同步数据里的开关，不按导入的规矩停用 + let mut snap = + model::snapshot(&applied.pack).map_err(|e| fail(StatusCode::BAD_REQUEST, e.msg()))?; + let theirs = model::snapshot(remote).map_err(|e| fail(StatusCode::BAD_REQUEST, e.msg()))?; + let mut changed = false; + for d in &out { + if d.item.kind != ItemKind::Plugin || matches!(d.action, tw_pack::import::Action::Skip) { + continue; + } + let name = match &d.action { + tw_pack::import::Action::Rename(n) => n.clone(), + _ => d.item.name.clone(), + }; + let id = ItemId::new(ItemKind::Plugin, name); + let want = theirs + .items + .get(&d.item) + .and_then(|it| it.units.get(model::unit::ENABLED)) + .cloned(); + if let (Some(it), Some(want)) = (snap.items.get_mut(&id), want) + && it.units.get(model::unit::ENABLED) != Some(&want) + { + it.units.insert(model::unit::ENABLED.to_string(), want); + changed = true; + } + } + if !changed { + return Ok(applied.pack); + } + snap.relink(); + model::materialize(local, &snap).map_err(|e| fail(StatusCode::BAD_REQUEST, e.msg())) +} + +fn import_fail(e: &tw_pack::import::ImportError) -> Fail { + use tw_pack::import::ImportError as E; + let code = match e { + E::Exists(_) | E::NameTaken(..) => StatusCode::CONFLICT, + E::NotInPack(_) => StatusCode::NOT_FOUND, + E::Item(..) => StatusCode::UNPROCESSABLE_ENTITY, + E::Model(_) | E::Rejected(_) | E::TooNew(_) => StatusCode::BAD_REQUEST, + }; + fail(code, e.msg()) +} + +// ─────────────────────────────────────────────────────────── 端点 + +async fn status(State(s): State) -> Json { + Json(service(&s).view()) +} + +fn config_target( + s: &ControlState, + t: &tw_api::SyncTarget, +) -> Result<(tw_config::sync::Target, AnyStore), Fail> { + let target = target_to_config(t); + let store = store_for(&s.config(), &target).map_err(|m| fail(StatusCode::BAD_REQUEST, m))?; + Ok((target, store)) +} + +fn device_view(d: &tw_sync::device::DeviceRecord, me: Option<&str>) -> tw_api::SyncDevice { + tw_api::SyncDevice { + id: d.id.clone(), + name: d.name.clone(), + core_version: d.core_version.clone(), + last_seen_ms: d.last_seen_ms, + this: Some(d.id.as_str()) == me, + paused_update: d.paused_update, + remote_core: d.remote_core, + } +} + +async fn test( + State(s): State, + Json(req): Json, +) -> Result, Fail> { + let (_, store) = config_target(&s, &req.target)?; + let p = Eng::probe(&store, req.passphrase.as_deref()).await; + let me = s.config().sync.as_ref().map(|y| y.device_id.clone()); + Ok(Json(tw_api::SyncProbe { + writable: p.writable.is_ok(), + error: p.writable.err().map(|e| e.msg()), + existing: p.existing.map(|e| tw_api::SyncExisting { + encrypted: e.encrypted, + unlocked: e.unlocked, + devices: e + .devices + .iter() + .map(|d| device_view(d, me.as_deref())) + .collect(), + }), + })) +} + +/// 一台还没开同步的设备对着某个同步位置的引擎（加入预览、开启） +fn temp_engine( + s: &ControlState, + target: &tw_config::sync::Target, + store: AnyStore, + sync: &tw_config::sync::Sync, +) -> Eng { + let _ = target; + Engine::new( + store, + CoreHost { + mgr: Arc::downgrade(&s.cfg), + }, + &crate::plugins::dir_of(s.config_path()), + settings_of(&s.config(), sync), + ) +} + +fn draft( + target: tw_config::sync::Target, + encrypt: bool, + passphrase: Option, + device_name: &str, +) -> tw_config::sync::Sync { + tw_config::sync::Sync { + device_id: tw_config::sync::generate_device_id(), + device_name: device_name.trim().to_string(), + encrypt, + passphrase, + exclude: Vec::new(), + target, + } +} + +async fn join_preview( + State(s): State, + Json(req): Json, +) -> Result, Fail> { + let (target, store) = config_target(&s, &req.target)?; + let cur = current_pack(&s.cfg).map_err(|m| fail(StatusCode::INTERNAL_SERVER_ERROR, m))?; + let local = model::snapshot(&cur.pack).map_err(|e| fail(StatusCode::BAD_REQUEST, e.msg()))?; + if is_initial(&local) { + return Ok(Json(tw_api::SyncJoinPreview { + initial: true, + rows: Vec::new(), + })); + } + let probe = Eng::probe(&store, req.passphrase.as_deref()).await; + if probe.existing.is_none() { + return Ok(Json(tw_api::SyncJoinPreview { + initial: false, + rows: Vec::new(), + })); + } + let encrypted = probe.existing.as_ref().is_some_and(|e| e.encrypted); + let mut e = temp_engine( + &s, + &target, + store, + &draft(target.clone(), encrypted, req.passphrase.clone(), "preview"), + ); + let remote = e.latest().await.map_err(|e| round_fail(&e))?; + // 和导入预览同一套规矩 + let rows = tw_pack::import::preview(&cur.pack, &remote, &crate::backups::import_ctx()) + .map_err(|e| import_fail(&e))?; + Ok(Json(tw_api::SyncJoinPreview { + initial: false, + rows: rows.into_iter().map(crate::backups::row_view).collect(), + })) +} + +async fn enable( + State(s): State, + Json(req): Json, +) -> Result, Fail> { + let sync = service(&s); + let mut guard = sync.run.lock().await; + if s.config().sync.is_some() { + return Err(fail( + StatusCode::CONFLICT, + msg!("sync.already_on" => "sync is already on. Stop it before using another location"), + )); + } + let (target, store) = config_target(&s, &req.target)?; + let probe = Eng::probe(&store, req.passphrase.as_deref()).await; + if let Err(e) = &probe.writable { + return Err(round_fail(&RoundError::Store(e.clone()))); + } + let mut settings = match &probe.existing { + None => { + let pass = if req.encrypt { + Some(check_passphrase(req.passphrase.as_deref())?.to_string()) + } else { + None + }; + draft(target.clone(), req.encrypt, pass, &req.device_name) + } + Some(e) => { + if e.encrypted && !e.unlocked { + return Err(fail( + StatusCode::BAD_REQUEST, + tw_sync::keyring::KeyringError::WrongPassphrase.msg(), + )); + } + draft( + target.clone(), + e.encrypted, + e.encrypted.then(|| req.passphrase.clone()).flatten(), + &req.device_name, + ) + } + }; + settings + .check() + .map_err(|e| fail(StatusCode::BAD_REQUEST, e.msg()))?; + let mut engine = temp_engine(&s, &target, store, &settings); + match &probe.existing { + None => { + if !engine.create().await.map_err(|e| round_fail(&e))? { + return Err(fail( + StatusCode::CONFLICT, + msg!( + "sync.location_taken" => + "another device has just started syncing at this location. Check the \ + location again and join it" + ), + )); + } + } + Some(_) => { + let cur = + current_pack(&s.cfg).map_err(|m| fail(StatusCode::INTERNAL_SERVER_ERROR, m))?; + let local = + model::snapshot(&cur.pack).map_err(|e| fail(StatusCode::BAD_REQUEST, e.msg()))?; + let merged = if is_initial(&local) || req.join == tw_api::SyncJoin::UseRemote { + None + } else { + let remote_pack = engine.latest().await.map_err(|e| round_fail(&e))?; + Some(merge_join(&cur.pack, &remote_pack, &req.decisions)?) + }; + let report = engine.join(merged).await.map_err(|e| round_fail(&e))?; + if !report.keys_changed.is_empty() { + s.bus().emit(tw_api::Event::GatewayKeysChanged { + id: s.bus().next_id(), + names: report.keys_changed.clone(), + at_ms: now_ms(), + }); + } + settings.encrypt = report.encrypted; + } + } + let version = write_section(&s.cfg, Some(&settings), Origin::Ui) + .await + .map_err(apply_fail)?; + let cfg = s.config(); + *guard = Some(Running { + engine, + config: settings, + proxies: format!("{:?}", cfg.proxies), + _watch: None, + }); + // 文件夹的监听在下一轮由 ensure 补上：重建一次 + if let Some(r) = guard.as_mut() { + r.proxies.clear(); + } + drop(guard); + sync.failures.store(0, Ordering::Relaxed); + sync.register_backups(true); + sync.publish(sync.compute_view(None, None)); + sync.kick(); + Ok(Json(tw_api::ConfigWritten { version })) +} + +async fn save( + State(s): State, + Json(req): Json, +) -> Result, Fail> { + let sync = service(&s); + let _guard = sync.run.lock().await; + let Some(mut cur) = s.config().sync.clone() else { + return Err(sync_off()); + }; + if let Some(t) = &req.target { + let t = target_to_config(t); + if !t.same_place(&cur.target) { + return Err(fail( + StatusCode::BAD_REQUEST, + msg!( + "sync.place_changed" => + "this is a different sync location. Stop syncing, then turn sync on with the \ + new location" + ), + )); + } + cur.target = t; + } + if let Some(n) = &req.device_name { + cur.device_name = n.trim().to_string(); + } + if let Some(ex) = &req.excluded { + let mut v: Vec = ex.iter().map(|e| e.trim().to_string()).collect(); + v.dedup(); + cur.exclude = v; + } + cur.check() + .map_err(|e| fail(StatusCode::BAD_REQUEST, e.msg()))?; + let version = write_section(&s.cfg, Some(&cur), Origin::Ui) + .await + .map_err(apply_fail)?; + drop(_guard); + sync.kick(); + Ok(Json(tw_api::ConfigWritten { version })) +} + +async fn now(State(s): State) -> Result, Fail> { + let sync = service(&s); + if s.config().sync.is_none() { + return Ok(Json(sync.view())); + } + sync.round(&[]).await.map(Json) +} + +async fn conflicts(State(s): State) -> Json> { + Json(service(&s).conflicts().await) +} + +async fn resolve( + State(s): State, + Json(req): Json, +) -> Result, Fail> { + if s.config().sync.is_none() { + return Err(sync_off()); + } + let decisions: Vec = req + .decisions + .iter() + .map(|d| Decision { + id: d.id.clone(), + take: match d.take { + tw_api::SyncTake::Local => Take::Local, + tw_api::SyncTake::Remote => Take::Remote, + tw_api::SyncTake::Both => Take::Both, + }, + new_name: d.new_name.clone(), + }) + .collect(); + let sync = service(&s); + let v = sync.round(&decisions).await?; + if let Some(e) = v + .error + .as_ref() + .filter(|_| v.state != tw_api::SyncState::Conflict && v.state != tw_api::SyncState::Idle) + { + return Err(fail(StatusCode::BAD_GATEWAY, e.clone())); + } + Ok(Json(v)) +} + +async fn encryption( + State(s): State, + Json(req): Json, +) -> Result, Fail> { + let sync = service(&s); + let mut guard = sync.run.lock().await; + let r = match sync.ensure(&mut guard) { + Ok(Some(r)) => r, + Ok(None) => return Err(sync_off()), + Err(m) => return Err(fail(StatusCode::BAD_REQUEST, m)), + }; + let mut next = r.config.clone(); + let res = match req.action { + tw_api::SyncEncryptionAction::Enable => { + let p = check_passphrase(req.passphrase.as_deref())?.to_string(); + next.encrypt = true; + next.passphrase = Some(p.clone()); + r.engine.enable_encryption(&p).await + } + tw_api::SyncEncryptionAction::Disable => { + let p = req.passphrase.clone().unwrap_or_default(); + next.encrypt = false; + next.passphrase = None; + r.engine.disable_encryption(&p).await + } + tw_api::SyncEncryptionAction::Change => { + let new = check_passphrase(req.new_passphrase.as_deref())?.to_string(); + let old = req.passphrase.clone().unwrap_or_default(); + next.passphrase = Some(new.clone()); + r.engine.change_passphrase(&old, &new).await + } + }; + res.map_err(|e| round_fail(&e))?; + r.config = next.clone(); + r.engine.set_settings(settings_of(&s.config(), &next)); + drop(guard); + write_section(&s.cfg, Some(&next), Origin::Ui) + .await + .map_err(apply_fail)?; + sync.round(&[]).await.map(Json) +} + +async fn unlock( + State(s): State, + Json(req): Json, +) -> Result, Fail> { + let sync = service(&s); + let mut guard = sync.run.lock().await; + let r = match sync.ensure(&mut guard) { + Ok(Some(r)) => r, + Ok(None) => return Err(sync_off()), + Err(m) => return Err(fail(StatusCode::BAD_REQUEST, m)), + }; + let encrypted = r + .engine + .unlock(&req.passphrase) + .await + .map_err(|e| round_fail(&e))?; + let mut next = r.config.clone(); + if encrypted { + next.encrypt = true; + next.passphrase = Some(req.passphrase.clone()); + } + r.config = next.clone(); + r.engine.set_settings(settings_of(&s.config(), &next)); + drop(guard); + write_section(&s.cfg, Some(&next), Origin::Ui) + .await + .map_err(apply_fail)?; + sync.round(&[]).await.map(Json) +} + +async fn disable( + State(s): State, + Json(req): Json, +) -> Result, Fail> { + let sync = service(&s); + let mut guard = sync.run.lock().await; + if let Ok(Some(r)) = sync.ensure(&mut guard) { + match r.engine.stop(req.purge).await { + Ok(()) => {} + // 只是移除自己的设备记录时连不上也照样停：半年之后它自己会过期 + Err(e) if !req.purge => { + tracing::warn!( + "the device record could not be removed from the sync location: {}", + e.msg() + ); + r.engine.local().clear(); + } + Err(e) => return Err(round_fail(&e)), + } + } else if let Some(dir) = sync.data_dir() { + tw_sync::local::Local::new(&dir).clear(); + } + *guard = None; + drop(guard); + if s.config().sync.is_some() { + write_section(&s.cfg, None, Origin::Ui) + .await + .map_err(apply_fail)?; + } + if let Ok(mut m) = sync.url_changed.lock() { + m.clear(); + } + sync.register_backups(false); + sync.publish(sync.compute_view(None, None)); + Ok(Json(())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn targets_convert_both_ways_and_blank_proxies_are_direct() { + let api = tw_api::SyncTarget::S3 { + endpoint: " https://abc.r2.cloudflarestorage.com ".into(), + region: "auto".into(), + bucket: "tw".into(), + prefix: "/me/".into(), + access_key_id: "AK".into(), + secret_access_key: "SK".into(), + proxy: String::new(), + }; + let c = target_to_config(&api); + assert_eq!(c.proxy(), "direct"); + match target_to_api(&c) { + tw_api::SyncTarget::S3 { + endpoint, prefix, .. + } => { + assert_eq!(endpoint, "https://abc.r2.cloudflarestorage.com"); + assert_eq!(prefix, "me"); + } + other => panic!("{other:?}"), + } + } +} diff --git a/crates/tw-control/tests/sync_devices.rs b/crates/tw-control/tests/sync_devices.rs new file mode 100644 index 00000000..4ac190da --- /dev/null +++ b/crates/tw-control/tests/sync_devices.rs @@ -0,0 +1,683 @@ +//! 多设备同步走控制面：两台设备（两个数据目录、两份配置、两个控制面）经同一个文件夹位置。 +//! +//! 合并规则本身在 `tw-sync` 的测试里逐条测过；这里测的是接缝：开启与加入、经写入流程应用 +//! （来源记为同步、热更新）、状态事件、密钥变了的事件、冲突经端点处理、安全模式暂停、 +//! 本机改动 10 秒后上传而另一台的文件夹监听即时收到。 + +use std::sync::Arc; +use std::time::Duration; + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use serde_json::{Value, json}; +use tower::ServiceExt; +use tw_control::{ConfigManager, ControlState}; + +const PASS: &str = "correct horse battery staple"; + +fn base(key: char) -> String { + format!( + "version: 1\nlisten:\n control:\n key: {}\nclients:\n - name: default\n key: tw-default-{key}\n", + key.to_string().repeat(64) + ) +} + +struct Dev { + dir: tempfile::TempDir, + state: ControlState, + app: axum::Router, +} + +impl Dev { + fn new(key: char) -> Dev { + tw_sync::keyring::set_work_factor_for_tests(4); + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join("config.yaml"); + let text = base(key); + std::fs::write(&p, &text).unwrap(); + let cfg = tw_config::try_parse(&text).unwrap(); + let gw = tw_gateway::AppState::new(cfg).unwrap(); + // 插件引擎用假的：照真的那一套读 manifest，不起沙箱 + gw.set_plugin_engine(Arc::new(tw_gateway::plugin::fake::FakeEngine)); + let bus = gw.bus.clone(); + let state = ControlState { + shutdown: Default::default(), + remote: Default::default(), + cfg: Arc::new(ConfigManager::new(p, gw.clone(), bus)), + gateway: gw, + store: None, + started: std::time::Instant::now(), + price_updater: Default::default(), + chatgpt: Default::default(), + zai: Default::default(), + }; + Dev { + app: tw_control::router(state.clone()), + state, + dir, + } + } + + async fn call(&self, method: &str, path: &str, body: Value) -> (StatusCode, Value) { + let r = self + .app + .clone() + .oneshot( + Request::builder() + .method(method) + .uri(path) + .header("content-type", "application/json") + .body(Body::from(if body.is_null() { + String::new() + } else { + body.to_string() + })) + .unwrap(), + ) + .await + .unwrap(); + let status = r.status(); + let bytes = axum::body::to_bytes(r.into_body(), usize::MAX) + .await + .unwrap(); + ( + status, + serde_json::from_slice(&bytes).unwrap_or(Value::Null), + ) + } + + async fn ok(&self, method: &str, path: &str, body: Value) -> Value { + let (s, v) = self.call(method, path, body).await; + assert!(s.is_success(), "{method} {path}: {s} {v}"); + v + } + + fn text(&self) -> String { + std::fs::read_to_string(self.dir.path().join("config.yaml")).unwrap() + } + + fn config(&self) -> tw_config::Config { + self.state.config().as_ref().clone() + } + + /// 用户在界面上改了配置（`PUT /config`） + async fn edit(&self, f: impl FnOnce(&str) -> String) { + let cur = self.ok("GET", "/config", Value::Null).await; + let next = f(cur["text"].as_str().unwrap()); + self.ok( + "PUT", + "/config", + json!({ "base_version": cur["version"], "text": next }), + ) + .await; + } +} + +fn folder(loc: &std::path::Path) -> Value { + json!({ "type": "folder", "path": loc.display().to_string(), "proxy": "direct" }) +} + +fn add_provider(text: &str, name: &str, url: &str, key: &str) -> String { + let p = format!(" - name: {name}\n base_url: {url}\n key: {key}\n"); + if text.contains("\nproviders:\n") { + text.replacen("\nproviders:\n", &format!("\nproviders:\n{p}"), 1) + } else { + format!("{text}providers:\n{p}") + } +} + +/// A 开启（同步位置空着，加密），B 加入（新安装：直接用同步数据） +async fn pair(loc: &std::path::Path) -> (Dev, Dev) { + let (a, b) = (Dev::new('a'), Dev::new('b')); + let probe = a + .ok("POST", "/sync/test", json!({ "target": folder(loc) })) + .await; + assert_eq!(probe["writable"], true); + assert!(probe["existing"].is_null(), "空的同步位置：{probe}"); + // 口令太短不收 + let (s, v) = a + .call( + "POST", + "/sync/enable", + json!({ "target": folder(loc), "encrypt": true, "passphrase": "short", "device_name": "Mac mini", "join": "use_remote" }), + ) + .await; + assert_eq!(s, StatusCode::BAD_REQUEST, "{v}"); + assert_eq!(v["code"], "sync.passphrase_short"); + a.ok( + "POST", + "/sync/enable", + json!({ "target": folder(loc), "encrypt": true, "passphrase": PASS, "device_name": "Mac mini", "join": "use_remote" }), + ) + .await; + assert!(a.config().sync.is_some(), "同步设置写进了配置"); + let v = a.ok("GET", "/sync", Value::Null).await; + assert_ne!(v["state"], "off", "开启之后马上就不是关着：{v}"); + assert_eq!(v["device"]["name"], "Mac mini"); + + let probe = b + .ok( + "POST", + "/sync/test", + json!({ "target": folder(loc), "passphrase": PASS }), + ) + .await; + assert_eq!(probe["existing"]["encrypted"], true); + assert_eq!(probe["existing"]["unlocked"], true); + assert_eq!(probe["existing"]["devices"][0]["name"], "Mac mini"); + let preview = b + .ok( + "POST", + "/sync/enable/preview", + json!({ "target": folder(loc), "passphrase": PASS }), + ) + .await; + assert_eq!(preview["initial"], true, "新安装的配置不出现合并这一步"); + // 口令不对加入不了 + let (s, _) = b + .call( + "POST", + "/sync/enable", + json!({ "target": folder(loc), "encrypt": true, "passphrase": "wrong passphrase!", "device_name": "MacBook Air", "join": "use_remote" }), + ) + .await; + assert_eq!(s, StatusCode::BAD_REQUEST); + b.ok( + "POST", + "/sync/enable", + json!({ "target": folder(loc), "encrypt": true, "passphrase": PASS, "device_name": "MacBook Air", "join": "use_remote" }), + ) + .await; + (a, b) +} + +#[tokio::test] +async fn two_devices_turn_sync_on_and_changes_flow_both_ways() { + let loc = tempfile::tempdir().unwrap(); + let (a, b) = pair(loc.path()).await; + let mut events = b.state.bus().subscribe(); + + a.edit(|t| add_provider(t, "relay", "https://relay.example/v1", "sk-relay")) + .await; + a.ok("POST", "/sync/now", Value::Null).await; + let v = b.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["state"], "idle", "{v}"); + assert_eq!(v["encrypted"], true); + assert_eq!(v["devices"].as_array().unwrap().len(), 2, "{v}"); + // B 收到了，而且已经生效（网关换入了新配置） + let p = b.config().providers.into_iter().find(|p| p.name == "relay"); + assert!(p.is_some(), "{}", b.text()); + assert!( + b.text().contains(&"b".repeat(64)), + "B 的控制面钥匙是它自己的" + ); + // 来源记为同步 + let mut saw_sync = false; + let mut saw_status = false; + while let Ok(ev) = events.try_recv() { + match ev { + tw_api::Event::ConfigReloaded { + origin: tw_api::ConfigOrigin::Sync, + .. + } => saw_sync = true, + tw_api::Event::SyncStatus { view, .. } => { + saw_status = true; + assert_eq!(view.device.unwrap().name, "MacBook Air"); + } + _ => {} + } + } + assert!(saw_sync, "应用别的设备的版本，来源是 sync"); + assert!(saw_status, "状态变了要发 sync_status"); + + // 用同步数据加入：B 的网关密钥就是 A 的那一把 + assert_eq!(b.config().clients[0].key, "tw-default-a"); + // B 换了网关密钥的值：A 那边发 gateway_keys_changed + let mut a_events = a.state.bus().subscribe(); + b.edit(|t| t.replace("tw-default-a", "tw-default-rotated")) + .await; + b.ok("POST", "/sync/now", Value::Null).await; + a.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(a.config().clients[0].key, "tw-default-rotated"); + let mut names = None; + while let Ok(ev) = a_events.try_recv() { + if let tw_api::Event::GatewayKeysChanged { names: n, .. } = ev { + names = Some(n); + } + } + assert_eq!(names, Some(vec!["default".to_string()])); +} + +#[tokio::test] +async fn a_conflict_is_listed_and_resolved_through_the_endpoints() { + let loc = tempfile::tempdir().unwrap(); + let (a, b) = pair(loc.path()).await; + a.edit(|t| add_provider(t, "relay", "https://relay.example", "sk-base")) + .await; + a.ok("POST", "/sync/now", Value::Null).await; + b.ok("POST", "/sync/now", Value::Null).await; + a.edit(|t| t.replace("sk-base", "sk-a")).await; + b.edit(|t| t.replace("sk-base", "sk-b")).await; + a.ok("POST", "/sync/now", Value::Null).await; + let v = b.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["state"], "conflict", "{v}"); + assert_eq!(v["conflicts"], 1); + let list = b.ok("GET", "/sync/conflicts", Value::Null).await; + let c = &list[0]; + assert_eq!(c["item"], json!({ "kind": "provider", "name": "relay" })); + assert_eq!(c["units"], json!(["conn"])); + assert_eq!(c["local"]["device"], "MacBook Air"); + assert_eq!(c["remote"]["device"], "Mac mini"); + assert_eq!(c["both_allowed"], true); + assert_eq!(c["diff"][0]["unit"], "conn"); + assert_eq!(c["diff"][0]["local"]["key"], "sk-b"); + assert_eq!(c["diff"][0]["remote"]["key"], "sk-a"); + // A 看到的是同一处，本机那一边是 A + a.ok("POST", "/sync/now", Value::Null).await; + let la = a.ok("GET", "/sync/conflicts", Value::Null).await; + assert_eq!(la[0]["id"], c["id"]); + assert_eq!(la[0]["local"]["device"], "Mac mini"); + + let v = b + .ok( + "POST", + "/sync/conflicts", + json!({ "decisions": [{ "id": c["id"], "take": "remote" }] }), + ) + .await; + assert_eq!(v["conflicts"], 0, "{v}"); + let key = |d: &Dev| { + d.config() + .providers + .into_iter() + .find(|p| p.name == "relay") + .unwrap() + .key + .unwrap() + .raw() + .to_string() + }; + assert_eq!(key(&b), "sk-a"); + let v = a.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["conflicts"], 0, "A 上的冲突随之清除：{v}"); + assert_eq!(key(&a), "sk-a"); +} + +#[tokio::test] +async fn joining_with_a_configuration_merges_by_the_import_rules() { + let loc = tempfile::tempdir().unwrap(); + let a = Dev::new('a'); + a.edit(|t| add_provider(t, "shared", "https://a.example", "sk-a")) + .await; + a.ok( + "POST", + "/sync/enable", + json!({ "target": folder(loc.path()), "encrypt": false, "device_name": "A", "join": "use_remote" }), + ) + .await; + let b = Dev::new('b'); + b.edit(|t| { + let t = add_provider(t, "shared", "https://b.example", "sk-b"); + add_provider(&t, "mine", "https://mine.example", "sk-m") + }) + .await; + let preview = b + .ok( + "POST", + "/sync/enable/preview", + json!({ "target": folder(loc.path()) }), + ) + .await; + assert_eq!(preview["initial"], false); + let row = preview["rows"] + .as_array() + .unwrap() + .iter() + .find(|r| r["item"]["name"] == "shared") + .unwrap() + .clone(); + assert_eq!(row["status"], "conflict"); + assert_eq!(row["suggested_name"], "shared (2)"); + // 冲突没选：不收 + let (s, v) = b + .call( + "POST", + "/sync/enable", + json!({ "target": folder(loc.path()), "encrypt": false, "device_name": "B", "join": "merge" }), + ) + .await; + assert_eq!(s, StatusCode::BAD_REQUEST, "{v}"); + assert_eq!(v["code"], "sync.join_undecided"); + b.ok( + "POST", + "/sync/enable", + json!({ + "target": folder(loc.path()), "encrypt": false, "device_name": "B", "join": "merge", + "decisions": [ + { "item": { "kind": "provider", "name": "shared" }, "action": "rename", "new_name": "shared (A)" }, + // 两边各自生成的默认密钥值不同：留本机的 + { "item": { "kind": "key", "name": "default" }, "action": "skip" }, + ], + }), + ) + .await; + let names = |d: &Dev| { + let mut v: Vec = d.config().providers.into_iter().map(|p| p.name).collect(); + v.sort(); + v + }; + assert_eq!(names(&b), ["mine", "shared", "shared (A)"]); + a.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(names(&a), ["mine", "shared", "shared (A)"], "{}", a.text()); + assert_eq!( + a.config().clients[0].key, + "tw-default-b", + "选了留本机的那一把，A 跟着换" + ); +} + +#[tokio::test] +async fn an_excluded_upstream_is_marked_and_stays_local() { + let loc = tempfile::tempdir().unwrap(); + let (a, b) = pair(loc.path()).await; + a.edit(|t| add_provider(t, "ollama", "http://127.0.0.1:11434", "none")) + .await; + a.ok("PUT", "/sync/settings", json!({ "excluded": ["ollama"] })) + .await; + let v = a.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["excluded"], json!(["ollama"])); + b.ok("POST", "/sync/now", Value::Null).await; + assert!(b.config().providers.is_empty(), "{}", b.text()); + let ov = a.ok("GET", "/overview", Value::Null).await; + assert_eq!(ov["providers"][0]["sync_excluded"], true); + // 换成另一个同步位置不在这里改 + let other = tempfile::tempdir().unwrap(); + let (s, v) = a + .call( + "PUT", + "/sync/settings", + json!({ "target": folder(other.path()) }), + ) + .await; + assert_eq!(s, StatusCode::BAD_REQUEST); + assert_eq!(v["code"], "sync.place_changed"); +} + +#[tokio::test] +async fn safe_mode_pauses_sync() { + let loc = tempfile::tempdir().unwrap(); + let (a, _b) = pair(loc.path()).await; + tw_control::sync::start(&a.state, true); + a.edit(|t| add_provider(t, "relay", "https://relay.example", "sk")) + .await; + let v = a.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["state"], "paused"); + let names: Vec = std::fs::read_dir(loc.path().join("versions")) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().to_string()) + .collect(); + assert_eq!(names.len(), 2, "暂停时不上传：{names:?}"); +} + +#[tokio::test] +async fn turning_sync_off_keeps_the_configuration_and_purging_empties_the_location() { + let loc = tempfile::tempdir().unwrap(); + let (a, b) = pair(loc.path()).await; + a.edit(|t| add_provider(t, "relay", "https://relay.example", "sk")) + .await; + a.ok("POST", "/sync/now", Value::Null).await; + a.ok("POST", "/sync/disable", json!({ "purge": true })) + .await; + assert!(a.config().sync.is_none()); + assert!(!a.config().providers.is_empty(), "本机保留当前配置"); + assert!(!a.dir.path().join("sync").exists(), "本机的同步状态删掉了"); + assert!(!loc.path().join("keyring").exists()); + // B 发现同步位置空了：停下，说一声 + let v = b.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["state"], "off", "{v}"); + assert_eq!(v["error"]["code"], "sync.emptied"); + assert!(b.config().sync.is_none()); +} + +/// 调度：本机改了配置，10 秒合并之后上传；另一台的文件夹监听一收到就合并 +#[tokio::test] +async fn a_local_change_is_uploaded_after_it_settles_and_the_folder_watcher_picks_it_up() { + let loc = tempfile::tempdir().unwrap(); + let (a, b) = pair(loc.path()).await; + tw_control::sync::start(&a.state, false); + tw_control::sync::start(&b.state, false); + // 启动那一轮先跑完 + tokio::time::sleep(Duration::from_millis(500)).await; + a.edit(|t| add_provider(t, "first", "https://first.example", "sk-1")) + .await; + a.edit(|t| add_provider(t, "second", "https://second.example", "sk-2")) + .await; + let started = std::time::Instant::now(); + loop { + let names: Vec = b.config().providers.into_iter().map(|p| p.name).collect(); + if names.len() == 2 { + break; + } + assert!( + started.elapsed() < Duration::from_secs(40), + "40 秒内没收到：{names:?}" + ); + tokio::time::sleep(Duration::from_millis(200)).await; + } + assert!( + started.elapsed() >= tw_control::sync::SETTLE - Duration::from_millis(500), + "改动要稳下来才上传" + ); + // 两次改动合成了一个版本：开启、加入、这一次 + let names: Vec = std::fs::read_dir(loc.path().join("versions")) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().to_string()) + .collect(); + assert!(names.len() <= 2, "{names:?}"); + // A 那一轮的尾巴（写设备记录）可能还没走完：不该一直停在「正在同步」 + let mut v = a.ok("GET", "/sync", Value::Null).await; + for _ in 0..50 { + if v["state"] != "syncing" { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + v = a.ok("GET", "/sync", Value::Null).await; + } + assert_eq!(v["state"], "idle", "{v}"); +} + +/// 不加密的同步：经同步新增的插件在本设备停用着，要在本设备启用；上游的地址经同步改了要 +/// 标出来（改之前的地址、哪台设备） +#[tokio::test] +async fn without_encryption_arriving_plugins_start_disabled_and_address_changes_are_flagged() { + let loc = tempfile::tempdir().unwrap(); + let (a, b) = (Dev::new('a'), Dev::new('b')); + a.edit(|t| add_provider(t, "relay", "https://relay.example", "sk")) + .await; + a.ok( + "POST", + "/sync/enable", + json!({ "target": folder(loc.path()), "encrypt": false, "device_name": "Mac mini", "join": "use_remote" }), + ) + .await; + b.ok( + "POST", + "/sync/enable", + json!({ "target": folder(loc.path()), "encrypt": false, "device_name": "MacBook Air", "join": "use_remote" }), + ) + .await; + let src = tw_gateway::plugin::fake::source( + json!({ "name": "Shout", "api": 1, "permissions": ["reply.text"] }), + &["onReplyText"], + ); + let ov = a.ok("GET", "/overview", Value::Null).await; + a.ok( + "POST", + "/plugins", + json!({ "source": src, "enabled": true, "base_version": ov["config_version"] }), + ) + .await; + a.edit(|t| t.replace("https://relay.example", "https://moved.example")) + .await; + a.ok("POST", "/sync/now", Value::Null).await; + b.ok("POST", "/sync/now", Value::Null).await; + let bp = b.config().plugins; + assert_eq!(bp.len(), 1, "{}", b.text()); + assert!( + !bp[0].enabled, + "经不加密的同步新来的插件，在这台设备上停用着" + ); + assert!(a.config().plugins[0].enabled, "A 上照旧开着"); + let ov = b.ok("GET", "/overview", Value::Null).await; + let relay = ov["providers"] + .as_array() + .unwrap() + .iter() + .find(|p| p["name"] == "relay") + .unwrap() + .clone(); + assert_eq!(relay["base_url"], "https://moved.example"); + assert_eq!(relay["sync_url_changed"]["from"], "https://relay.example"); + assert_eq!(relay["sync_url_changed"]["device"], "Mac mini"); + // 再同步几轮：B 的停用不会传回去 + b.ok("POST", "/sync/now", Value::Null).await; + a.ok("POST", "/sync/now", Value::Null).await; + assert!(a.config().plugins[0].enabled); + // B 在本机启用：之后就和别的设备一样了 + let id = bp[0].id.clone(); + let cur = b.ok("GET", "/config", Value::Null).await; + let text = cur["text"] + .as_str() + .unwrap() + .replace("enabled: false", "enabled: true"); + b.ok( + "PUT", + "/config", + json!({ "base_version": cur["version"], "text": text }), + ) + .await; + b.ok("POST", "/sync/now", Value::Null).await; + a.ok("POST", "/sync/now", Value::Null).await; + assert!(b.config().plugins.iter().any(|p| p.id == id && p.enabled)); + assert!(a.config().plugins[0].enabled); +} + +/// 安全模式：配置读不进来，在服务的是一份临时配置。同步不自动跑，状态是 `paused`；同步位置上 +/// 的版本照样列得出来（出错页的「从备份恢复」要用） +#[tokio::test] +async fn in_safe_mode_the_sync_location_still_lists_its_versions() { + let loc = tempfile::tempdir().unwrap(); + let (a, _b) = pair(loc.path()).await; + let path = a.dir.path().join("config.yaml"); + let broken = format!("{}a_field_from_nowhere: 1\n", a.text()); + std::fs::write(&path, &broken).unwrap(); + let (cfg, rejected) = tw_config::stand_in(&broken).expect("钥匙还在，顶得上"); + let gw = tw_gateway::AppState::new(cfg).unwrap(); + let bus = gw.bus.clone(); + let state = ControlState { + shutdown: Default::default(), + remote: Default::default(), + cfg: Arc::new(ConfigManager::new(path, gw.clone(), bus).standing_in(&rejected)), + gateway: gw, + store: None, + started: std::time::Instant::now(), + price_updater: Default::default(), + chatgpt: Default::default(), + zai: Default::default(), + }; + let sync = tw_control::sync::start(&state, true); + let v = sync.view(); + assert_eq!(v.state, tw_api::SyncState::Paused); + assert!(v.target.is_some(), "同步位置照原文里的"); + let versions = sync.remote_versions().await; + assert_eq!(versions.len(), 2, "{versions:?}"); + assert!(versions.iter().any(|v| v.device_name == "Mac mini")); + let pack = sync + .remote_pack(&versions[0].content_id) + .await + .unwrap() + .expect("按内容标识取得到"); + assert!(!pack.config_yaml.contains("listen"), "同步版本不带本机设置"); +} + +/// 备份列表里有同步位置上的版本（本机也有的同一份合成一行，只在同步位置上的是 +/// `sync-`）；经同步应用的本机版本记着「同步自」哪台设备；恢复预览说得出会同步到 +/// 几台别的设备；接管的客户端在用的密钥（`PUT /pinned-keys`）不被别的设备删掉 +#[tokio::test] +async fn backups_see_the_sync_location_and_pinned_keys_survive() { + let loc = tempfile::tempdir().unwrap(); + let (a, b) = pair(loc.path()).await; + a.edit(|t| { + let t = add_provider(t, "relay", "https://relay.example", "sk"); + t.replace( + "clients:\n", + "clients:\n - name: codex\n key: tw-codex-0002\n", + ) + }) + .await; + a.ok("POST", "/sync/now", Value::Null).await; + b.ok("POST", "/sync/now", Value::Null).await; + + let list = b.ok("GET", "/backups", Value::Null).await; + let rows = list.as_array().unwrap(); + let synced = rows + .iter() + .find(|r| r["origin"] == "sync" && r["local"] == true) + .unwrap_or_else(|| panic!("经同步应用的那一版：{list}")); + assert_eq!(synced["device"], "Mac mini", "同步自哪台设备：{synced}"); + assert_eq!(synced["sync"], true, "同步位置上也有这一份：{synced}"); + assert_eq!(synced["current"], true); + // A 那边看：同步位置上只有、本机没有的版本（B 加入时上传的那一版）是 sync- + let la = a.ok("GET", "/backups", Value::Null).await; + let remote_only = la + .as_array() + .unwrap() + .iter() + .find(|r| r["local"] == false) + .cloned(); + if let Some(r) = remote_only { + assert!(r["id"].as_str().unwrap().starts_with("sync-"), "{r}"); + let file = a + .ok("POST", "/backups/file", json!({ "id": r["id"] })) + .await; + assert!(file["data_b64"].as_str().is_some_and(|d| !d.is_empty())); + } + let mine = la + .as_array() + .unwrap() + .iter() + .find(|r| r["local"] == true) + .unwrap() + .clone(); + let preview = a + .ok("POST", "/backups/preview", json!({ "id": mine["id"] })) + .await; + assert_eq!( + preview["sync_devices"], 1, + "恢复之后会同步到另一台：{preview}" + ); + + // B 上接管的客户端在用 codex；A 删了它：B 不删，作为冲突列出 + b.ok("PUT", "/pinned-keys", json!({ "names": ["codex"] })) + .await; + a.edit(|t| t.replace(" - name: codex\n key: tw-codex-0002\n", "")) + .await; + a.ok("POST", "/sync/now", Value::Null).await; + let v = b.ok("POST", "/sync/now", Value::Null).await; + assert!( + b.config().clients.iter().any(|c| c.name == "codex"), + "{}", + b.text() + ); + assert_eq!(v["conflicts"], 1, "{v}"); + let c = b.ok("GET", "/sync/conflicts", Value::Null).await; + assert_eq!(c[0]["reason"], "key_in_use"); + // 关掉同步：备份列表不再看同步位置 + a.ok("POST", "/sync/disable", json!({ "purge": false })) + .await; + let la = a.ok("GET", "/backups", Value::Null).await; + assert!( + la.as_array().unwrap().iter().all(|r| r["sync"] == false), + "{la}" + ); +} diff --git a/crates/tw-gateway/tests/passthrough.rs b/crates/tw-gateway/tests/passthrough.rs index 58e7193f..e55726b4 100644 --- a/crates/tw-gateway/tests/passthrough.rs +++ b/crates/tw-gateway/tests/passthrough.rs @@ -424,6 +424,7 @@ async fn a_rule_sends_opus_to_one_upstream_and_everything_else_to_another() { default_route: None, default_key: None, plugins: Vec::new(), + sync: None, version: 1, listen: Listen::default(), clients: vec![Client { diff --git a/crates/tw-observe/src/bus.rs b/crates/tw-observe/src/bus.rs index 840b791d..8925909e 100644 --- a/crates/tw-observe/src/bus.rs +++ b/crates/tw-observe/src/bus.rs @@ -127,6 +127,8 @@ fn about_the_request(ev: &tw_api::Event) -> bool { | E::ConfigRejected { .. } | E::GatewayKeysChanged { .. } | E::ClientPrefsChanged { .. } + // 同步的状态：说的是配置，不是请求 + | E::SyncStatus { .. } | E::EventsDropped { .. } => false, } } diff --git a/crates/tw-pack/src/backup.rs b/crates/tw-pack/src/backup.rs index c954f17e..0c6c7292 100644 --- a/crates/tw-pack/src/backup.rs +++ b/crates/tw-pack/src/backup.rs @@ -207,6 +207,25 @@ fn save( }) } +/// 同步应用的那一版是从哪台设备来的：记在这一版的 manifest 里（`sync.device_name`），备份列表 +/// 里写「同步自 MacBook Air」。**只改这一个字段**，别的字节照旧 +pub fn note_device(b: &Backup, device: &str) -> Result<(), BackupError> { + let mut pack = open(b)?; + pack.manifest.sync = Some(serde_json::json!({ "device_name": device })); + let bytes = crate::write(&pack, Seal::None).map_err(BackupError::Pack)?; + disk::write_private(&b.path, &bytes).map_err(io(&b.path)) +} + +/// 一版是从哪台设备同步来的（[`note_device`] 记下的） +pub fn device_of(pack: &Pack) -> Option { + pack.manifest + .sync + .as_ref()? + .get("device_name")? + .as_str() + .map(str::to_string) +} + /// 超出 [`KEEP`] 的老版本删掉，以前的修改记录（`*.yaml`）也删掉 fn prune(config_path: &Path) -> Result<(), BackupError> { let all = list(config_path)?; @@ -375,6 +394,22 @@ mod tests { ); } + /// 经同步应用的那一版记着来源设备，别的照旧 + #[test] + fn a_synced_version_remembers_which_device_it_came_from() { + let d = tempfile::tempdir().unwrap(); + let p = d.path().join("config.yaml"); + let text = "version: 1\nclients:\n - name: default\n key: tw-k\n"; + std::fs::write(&p, text).unwrap(); + let b = record(&p, text, Origin::Sync).unwrap().unwrap(); + assert_eq!(device_of(&open(&b).unwrap()), None); + note_device(&b, "MacBook Air").unwrap(); + let back = open(&b).unwrap(); + assert_eq!(device_of(&back).as_deref(), Some("MacBook Air")); + assert_eq!(back.config_yaml, open(&b).unwrap().config_yaml); + assert_eq!(list(&p).unwrap().len(), 1, "不多出一版"); + } + #[test] fn a_rotation_leaves_no_version_and_does_not_count_as_a_change() { let (_d, p) = setup(); diff --git a/crates/tw-secret/src/mask.rs b/crates/tw-secret/src/mask.rs index bed0eceb..84583aba 100644 --- a/crates/tw-secret/src/mask.rs +++ b/crates/tw-secret/src/mask.rs @@ -280,6 +280,8 @@ fn is_secret_name(name: &str) -> bool { "access_key_id", "secret_access_key", "session_token", + // 同步口令（`sync:` 那一段）：解得开同步位置上的每一个版本 + "passphrase", ]; NAMES.contains(&name) } @@ -754,6 +756,23 @@ mod body_tests { assert!(!mask_body("the key is ASIAIOSFODNN7EXAMPLE here").contains("IOSFODNN7")); } + /// 同步那一节：口令、WebDAV 的密码、S3 的密钥都不能进诊断包 + #[test] + fn the_sync_sections_secrets_are_masked() { + let cfg = "sync:\n device_id: 3f9a2c1e-0b1d\n device_name: Mac mini\n encrypt: true\n passphrase: horse battery staple correct\n target:\n type: webdav\n url: https://dav.example.com/tw\n user: me\n pass: app-password-value\n"; + let out = mask_config_yaml(cfg); + for secret in ["battery staple", "app-password-value"] { + assert!(!out.contains(secret), "{secret} leaked: {out}"); + } + assert!(out.contains("device_name: Mac mini"), "{out}"); + let s3 = "sync:\n target:\n type: s3\n access_key_id: AKIDEXAMPLEVALUE1\n secret_access_key: plainlookingsecretvalue\n"; + let out = mask_config_yaml(s3); + assert!( + !out.contains("plainlookingsecretvalue") && !out.contains("EXAMPLEVALUE1"), + "{out}" + ); + } + #[test] fn a_field_that_merely_ends_in_key_is_not_a_secret() { // `api_key_env` 之类装的是环境变量名，不是密钥。多打码的代价小， diff --git a/crates/tw-store/src/recorder.rs b/crates/tw-store/src/recorder.rs index daa14eda..e94b1dcc 100644 --- a/crates/tw-store/src/recorder.rs +++ b/crates/tw-store/src/recorder.rs @@ -699,7 +699,8 @@ impl Recorder { | Event::GatewayKeysChanged { .. } | Event::ClientPrefsChanged { .. } // 只发给事件流上掉队的那一个订阅者，从不进总线 - | Event::EventsDropped { .. } => {} + | Event::EventsDropped { .. } + | Event::SyncStatus { .. } => {} } } diff --git a/crates/tw-sync/Cargo.toml b/crates/tw-sync/Cargo.toml new file mode 100644 index 00000000..62dbc2de --- /dev/null +++ b/crates/tw-sync/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "tw-sync" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +documentation.workspace = true +readme.workspace = true +description = "Multi-device configuration sync: folder, WebDAV and S3 locations, the keyring, version naming and the three-way merge" + +[dependencies] +# `sync` 一节的形状、地址规则 +tw-config = { workspace = true } +tw-types = { workspace = true } +# 配置包的格式与加密、把一份配置拆成项目与合并单位 +tw-pack = { workspace = true } +# S3 的 SigV4 和 Bedrock 用的是同一份 +tw-bedrock = { workspace = true } +# 文件夹位置：版本一到就合并，不等定时检查 +tw-watch = { workspace = true } +reqwest = { workspace = true } +tokio = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +age = { workspace = true } +quick-xml = { workspace = true } +sha2 = { workspace = true } +rand = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } +percent-encoding = { workspace = true } +url = { workspace = true } + +[dev-dependencies] +# 假的 WebDAV、假的 S3：和 core 别处的测试一样手写在 axum 上 +axum = { workspace = true } +tempfile = "3" +# 假 WebDAV 核对 Basic 认证 +base64 = { workspace = true } +# 两台设备的测试直接读写配置 +tw-config = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time"] } diff --git a/crates/tw-sync/src/device.rs b/crates/tw-sync/src/device.rs new file mode 100644 index 00000000..86350912 --- /dev/null +++ b/crates/tw-sync/src/device.rs @@ -0,0 +1,93 @@ +//! 设备记录：`devices/<设备标识前 8 位>`，每台设备只写自己那一份。 +//! +//! 加密同步时它也加密（给版本密钥的公钥）：设备名称常常就是人名（「张三的 MacBook Air」）。 + +use serde::{Deserialize, Serialize}; + +use crate::keyring::Keyring; + +/// 设备记录多久没更新就从同步位置上移除 +pub const STALE_AFTER_MS: u64 = 180 * 24 * 3600 * 1000; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DeviceRecord { + /// 完整的设备标识 + pub id: String, + pub name: String, + pub core_version: String, + /// 最后一次同步成功的时间 + pub last_seen_ms: u64, + /// 同步位置的最新版本这台设备读不了（由较新的 Core 写的），暂停着等更新 + #[serde(default)] + pub paused_update: bool, + /// 它的 core 开着远程控制端口（远程 core） + #[serde(default)] + pub remote_core: bool, +} + +impl DeviceRecord { + /// 写进同步位置的样子：加密同步时加密给版本密钥 + pub fn seal(&self, keyring: Option<&Keyring>) -> Vec { + let json = serde_json::to_vec(self).expect("plain data serializes"); + match keyring { + Some(k) => k.encrypt(&json), + None => json, + } + } + + /// 读回来。解不开、读不懂的是 `None`（别的设备刚改了加密设置、还没传完、被人改坏了） + pub fn open(bytes: &[u8], keyring: Option<&Keyring>) -> Option { + let json = if crate::keyring::is_age(bytes) { + keyring?.decrypt(bytes)? + } else { + bytes.to_vec() + }; + serde_json::from_slice(&json).ok() + } + + pub fn is_stale(&self, now_ms: u64) -> bool { + now_ms.saturating_sub(self.last_seen_ms) > STALE_AFTER_MS + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn rec() -> DeviceRecord { + DeviceRecord { + id: "3f9a2c1e-0b1d-4e2f-9a3b-5c6d7e8f9a0b".into(), + name: "张三的 MacBook Air".into(), + core_version: "0.77.0".into(), + last_seen_ms: 1_000, + paused_update: false, + remote_core: false, + } + } + + #[test] + fn a_record_reads_back_plain_or_sealed() { + let r = rec(); + assert_eq!(DeviceRecord::open(&r.seal(None), None), Some(r.clone())); + let k = Keyring::generate(); + let sealed = r.seal(Some(&k)); + assert!( + !String::from_utf8_lossy(&sealed).contains("张三"), + "加密时名字不能是明文" + ); + assert_eq!(DeviceRecord::open(&sealed, Some(&k)), Some(r.clone())); + assert_eq!(DeviceRecord::open(&sealed, None), None); + assert_eq!( + DeviceRecord::open(&sealed, Some(&Keyring::generate())), + None + ); + assert_eq!(DeviceRecord::open(b"{not json", None), None); + } + + #[test] + fn a_record_goes_stale_after_half_a_year() { + let r = rec(); + assert!(!r.is_stale(r.last_seen_ms + STALE_AFTER_MS)); + assert!(r.is_stale(r.last_seen_ms + STALE_AFTER_MS + 1)); + } +} diff --git a/crates/tw-sync/src/engine.rs b/crates/tw-sync/src/engine.rs new file mode 100644 index 00000000..d2828f86 --- /dev/null +++ b/crates/tw-sync/src/engine.rs @@ -0,0 +1,2015 @@ +//! 一轮同步：看同步位置上有什么新版本，和本机合并，该应用的应用、该上传的上传。 +//! +//! # 一轮里做什么 +//! +//! 1. 列出 `versions/`，读出每个版本（加密的用版本密钥解开）。读不出来的当作尚未到达：最新 +//! 那一层有没到的就等下一轮，不拿一半去合并。 +//! 2. 找出所有的**头**（没有别的版本依据它的版本）。不止一个时（两台设备同时上传、云盘晚到 +//! 的版本）先以它们共同的上一版本为基准两两合并，得到「同步位置的最新」。 +//! 3. 以本机基准（上次同步的那一版）为基准，和本机当前配置三方合并（[`crate::merge`]）。 +//! 冲突照常往下走：本机留本机的，上传的留同步位置的，记进冲突记录。 +//! 4. 本机的结果和现在不一样就应用（经控制面的写入流程，来源记为同步）；上传的结果和同步 +//! 位置上的不一样（或者合并了几个头、冲突记录变了）就上传一个新版本，然后删掉更早的。 +//! +//! # 冲突记录随版本走 +//! +//! 一处冲突在被处理之前，**两边各留各的**：参与的每台设备用自己的内容，同步版本里是先上传 +//! 的那一边（[`Conflict::shared`]）。任一设备处理之后，冲突从记录里去掉、结果上传，别的设备 +//! 看到记录没了，就照同步版本改成处理之后的样子。 + +use std::collections::{BTreeMap, BTreeSet}; +use std::future::Future; +use std::path::Path; +use std::sync::Arc; + +use serde_json::Value; +use sha2::{Digest, Sha256}; +use tw_pack::model::{self, Item, ItemId, ItemKind, Snapshot, unit}; +use tw_pack::{Pack, PackError, PackKind, PackOrigin, Seal, Unseal}; +use tw_types::{Msg, msg}; + +use crate::device::DeviceRecord; +use crate::keyring::{Keyring, KeyringError}; +use crate::layout::{self, DEVICES, KEYRING, Listed, VERSIONS, VersionName}; +use crate::local::{Local, State, UrlChange}; +use crate::merge::{self, Objects}; +use crate::order; +use crate::record::{self, Conflict, Meta, Reason, Side}; +use crate::store::{Store, StoreError}; + +/// 设备记录多久没写就写一次（最后同步时间），即便这一轮什么都没变 +const DEVICE_RECORD_EVERY_MS: u64 = 6 * 3600 * 1000; + +/// 本机这一边：配置从哪儿读、往哪儿写。控制面实现它（`tw-control` 的 `sync`），测试里换成 +/// 两个目录里的两份配置 +pub trait Host: Send + Sync { + /// 本机此刻的完整配置包（含本机设置、插件、通用偏好） + fn current(&self) -> Result; + /// 照这个包写本机配置：经现有的写入流程（校验、热更新、记一个本机版本），来源记为同步。 + /// `base_version` 是 [`Current::version`]：读完之后配置被别人改过就是 [`ApplyError::Stale`] + fn apply( + &self, + pack: Pack, + base_version: &str, + from: &str, + ) -> impl Future> + Send; + /// 本机已接管的客户端正在用的网关密钥（名字） + fn pinned_keys(&self) -> BTreeSet; + /// 本机备份里内容标识是这个的那一版 + fn backup(&self, content_id: &str) -> Option; +} + +/// 本机此刻的配置 +pub struct Current { + pub pack: Pack, + /// 配置文件的版本号（`tw_config::version_of`），写的时候拿它核对 + pub version: String, + /// 配置最后改动的时间（冲突里「本机改的时间」） + pub changed_ms: u64, +} + +#[derive(Debug)] +pub enum ApplyError { + /// 读完之后配置被改过了：下一轮再来 + Stale, + Rejected(Msg), +} + +/// 这台设备在同步里的设置 +#[derive(Debug, Clone)] +pub struct Settings { + pub device_id: String, + pub device_name: String, + pub encrypt: bool, + pub passphrase: Option, + /// 不同步的上游 + pub exclude: BTreeSet, + /// 同步位置的指纹（[`crate::local::place`]）。本机的同步状态是对着它的：换了位置就作废 + pub place: String, + /// 这台设备的 core 开着远程控制端口：设备记录里写上（界面标「远程 core」） + pub remote_core: bool, +} + +impl Settings { + pub fn of(s: &tw_config::sync::Sync) -> Self { + Self { + device_id: s.device_id.clone(), + device_name: s.device_name.clone(), + encrypt: s.encrypt, + passphrase: s.passphrase.clone(), + exclude: s.exclude.iter().cloned().collect(), + place: crate::local::place(&s.target), + remote_core: false, + } + } +} + +/// 一轮没成 +#[derive(Debug)] +pub enum RoundError { + /// 同步位置连不上、拒绝访问…… + Store(StoreError), + /// 同步口令不对（别的设备改了口令），或者同步位置开了加密而这台设备还没有口令 + Locked(Msg), + /// 同步位置的最新版本由较新的 Core 写，本机读不了：暂停 + TooNew { core_version: String }, + /// 最新的版本还没传完（云盘还在下载）：下一轮再来，不算失败 + NotArrived, + /// 同步位置被清空了（别的设备停止同步并删除了同步数据） + Emptied, + /// 本机的配置读不了、写不进去 + Local(Msg), + /// 这一轮里本机配置被改了：马上再来一轮 + Stale, +} + +impl From for RoundError { + fn from(e: StoreError) -> Self { + RoundError::Store(e) + } +} + +impl RoundError { + pub fn msg(&self) -> Msg { + match self { + RoundError::Store(e) => e.msg(), + RoundError::Locked(m) | RoundError::Local(m) => m.clone(), + RoundError::TooNew { core_version } => msg!( + "sync.too_new", version = core_version => + "another device uses a newer version (Core {version}). Update ThinkWatch Lite to \ + keep syncing" + ), + RoundError::NotArrived => msg!( + "sync.not_arrived" => + "the latest version has not finished arriving at the sync location" + ), + RoundError::Emptied => msg!( + "sync.emptied" => + "the sync location is empty: another device stopped syncing and deleted the \ + sync data" + ), + RoundError::Stale => msg!( + "sync.stale" => "the configuration changed during the sync; it will run again" + ), + } + } +} + +/// 一轮做了什么 +#[derive(Debug, Clone, Default)] +pub struct Report { + /// 上传的新版本 + pub uploaded: Option, + /// 应用了别的设备的改动：那些设备的名字 + pub applied_from: Vec, + /// 还没处理的冲突 + pub conflicts: Vec, + pub devices: Vec, + /// 同步位置此刻加没加密。和设置不一样时（别的设备关了加密）调用方改设置 + pub encrypted: bool, + /// 本机应用之后值变了、或者被删掉的网关密钥 + pub keys_changed: Vec, +} + +/// 读出来的一个版本 +#[derive(Clone)] +pub struct Version { + pub name: VersionName, + pub pack: Pack, + pub meta: Meta, + pub snap: Snapshot, + pub content: String, +} + +#[derive(Clone)] +enum Fetched { + Ok(Arc), + TooNew(String), +} + +/// 同步位置的最新（几个头合并之后） +struct Remote { + snap: Snapshot, + conflicts: Vec, + heads: Vec>, + /// 写最新那个头的设备 + device: String, + device_name: String, + at_ms: u64, +} + +/// 同步位置是不是空的、加没加密、有哪些设备（开启同步时的「测试连接」） +#[derive(Debug, Clone)] +pub struct Probe { + pub writable: Result<(), StoreError>, + pub existing: Option, +} + +#[derive(Debug, Clone)] +pub struct Existing { + pub encrypted: bool, + /// 加密时口令对不对（没给口令是 `false`） + pub unlocked: bool, + pub devices: Vec, +} + +pub struct Engine { + pub store: S, + pub host: H, + local: Local, + settings: Settings, + /// 解开了的版本密钥，和 keyring 文件的哈希（文件变了要重新解） + keyring: Option<(Keyring, [u8; 32])>, + cache: BTreeMap, + /// 上一轮结束时同步位置上的名字和本机的内容：都没变就什么都不用做 + quiet: Option<(Vec, String)>, + last_report: Report, +} + +fn now_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +fn local_err(e: impl std::fmt::Display) -> RoundError { + RoundError::Local(msg!( + "sync.local", detail = e => + "the local configuration could not be used for sync: {detail}" + )) +} + +/// 内容的 SHA-256：配置原文、插件、通用偏好（不加密的版本带着它，发现传输损坏） +fn content_sha256(p: &Pack) -> String { + let mut h = Sha256::new(); + h.update(p.config_yaml.as_bytes()); + for (id, f) in &p.plugins { + h.update(id.as_bytes()); + h.update([0]); + h.update(&f.script); + h.update([0]); + if let Some(a) = &f.approved { + h.update(a); + } + h.update([0]); + } + if let Some(c) = &p.client_prefs { + h.update(c.to_string().as_bytes()); + } + let out: [u8; 32] = h.finalize().into(); + out.iter().map(|b| format!("{b:02x}")).collect() +} + +fn objects(s: &Snapshot) -> Objects { + s.items + .iter() + .map(|(k, v)| (k.clone(), v.units.clone())) + .collect() +} + +/// 合并出来的单位拼回项目：插件的原文件从脚本一样的那一边拿 +fn rebuild(objs: &Objects, sides: &[&Snapshot]) -> Snapshot { + let mut items = BTreeMap::new(); + for (id, units) in objs { + let files = sides.iter().find_map(|s| { + let it = s.items.get(id)?; + (it.units.get(unit::SCRIPT) == units.get(unit::SCRIPT)) + .then(|| it.files.clone()) + .flatten() + }); + items.insert( + id.clone(), + Item { + units: units.clone(), + refs: Default::default(), + files, + }, + ); + } + let mut s = Snapshot { items }; + s.relink(); + s +} + +/// 一种顺序项目的名字列表 +fn order_list(s: Option<&Snapshot>, kind: ItemKind) -> Option> { + let it = s?.items.get(&ItemId::new(kind, ""))?; + match it.units.get(unit::ALL) { + Some(Value::Array(a)) => Some( + a.iter() + .filter_map(|v| v.as_str().map(str::to_string)) + .collect(), + ), + _ => None, + } +} + +fn names_of(s: &Snapshot, kind: ItemKind) -> BTreeSet { + s.of_kind(kind).map(|(id, _)| id.name.clone()).collect() +} + +fn set_order(s: &mut Snapshot, kind: ItemKind, list: Vec) { + let id = ItemId::new(kind, ""); + if list.is_empty() { + s.items.remove(&id); + return; + } + let item = s.items.entry(id).or_default(); + item.units.insert( + unit::ALL.to_string(), + Value::Array(list.into_iter().map(Value::String).collect()), + ); +} + +const ORDERS: &[(ItemKind, ItemKind)] = &[ + (ItemKind::ProviderOrder, ItemKind::Provider), + (ItemKind::RouteOrder, ItemKind::Route), + (ItemKind::PluginOrder, ItemKind::Plugin), +]; + +/// 对象里冲突的那几个单位此刻的值；对象不在是 `None` +fn side_units(s: &Snapshot, c: &Conflict) -> Option> { + s.items.get(&c.item).map(|it| c.pick(&it.units)) +} + +/// 合并新发现的一处冲突写成记录：`shared` 是同步版本那一边（快照和它的设备），`other` 是另一边 +fn found( + f: &merge::Found, + shared: (&Snapshot, Side), + other: (&Snapshot, Side), +) -> Conflict { + // 两边都改了：记冲突的那几个单位；一边删了：记整个对象 + let placeholder = (f.kind == merge::Kind::Modified).then(BTreeMap::new); + let mut c = Conflict { + id: record::new_id(), + item: f.key.clone(), + units: f.units.iter().cloned().collect(), + shared: Side { + units: placeholder.clone(), + ..shared.1 + }, + other: Side { + units: placeholder, + ..other.1 + }, + reason: None, + }; + c.shared.units = side_units(shared.0, &c); + c.other.units = side_units(other.0, &c); + c +} + +/// 把冲突的那一部分改成 `src` 里的样子 +fn force(dst: &mut Snapshot, src: &Snapshot, c: &Conflict) { + match src.items.get(&c.item) { + None => { + dst.items.remove(&c.item); + } + Some(it) if c.whole_item() => { + dst.items.insert(c.item.clone(), it.clone()); + } + Some(it) => { + let entry = dst + .items + .entry(c.item.clone()) + .or_insert_with(|| it.clone()); + for u in &c.units { + match it.units.get(u) { + Some(v) => { + entry.units.insert(u.clone(), v.clone()); + } + None => { + entry.units.remove(u); + } + } + } + if it.units.get(unit::SCRIPT) == entry.units.get(unit::SCRIPT) { + entry.files = it.files.clone(); + } + } + } +} + +/// 把冲突那一部分写成一边记录下来的值 +fn set_side(dst: &mut Snapshot, c: &Conflict, side: &Side, from: &Snapshot) { + match &side.units { + None => { + dst.items.remove(&c.item); + } + Some(units) => { + let entry = dst + .items + .entry(c.item.clone()) + .or_insert_with(|| from.items.get(&c.item).cloned().unwrap_or_default()); + for (u, v) in units { + entry.units.insert(u.clone(), v.clone()); + } + for u in &c.units { + if !units.contains_key(u) { + entry.units.remove(u); + } + } + } + } +} + +/// 处理一处冲突：采用哪一边 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Take { + /// 本机的 + Local, + /// 另一边的 + Remote, + /// 两者都保留：本机的留原名称，另一边的以「名称 (设备名)」新增 + Both, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Decision { + pub id: String, + pub take: Take, + /// 两者都保留时另一份的名字。不给就是「名称 (设备名)」，被占了往后加序号 + pub new_name: Option, +} + +/// 「两者都保留」那一份的名字。自定义安全规则的名字带着分项（`content/项目代号`），只换规则名 +fn both_name(id: &ItemId, given: Option<&str>, device_name: &str) -> String { + let (prefix, plain) = match (id.kind, id.name.split_once('/')) { + (ItemKind::SecurityRule, Some((guard, rule))) => (format!("{guard}/"), rule), + _ => (String::new(), id.name.as_str()), + }; + match given.map(str::trim).filter(|g| !g.is_empty()) { + Some(g) if g.contains('/') && id.kind == ItemKind::SecurityRule => g.to_string(), + Some(g) => format!("{prefix}{g}"), + None => format!("{prefix}{plain} ({device_name})"), + } +} + +/// 不同步的上游，连同直接或间接引用它们的项目（路由、别名、分组） +pub fn excluded(names: &BTreeSet, snaps: &[&Snapshot]) -> BTreeSet { + let mut out: BTreeSet = names + .iter() + .map(|n| ItemId::new(ItemKind::Provider, n.clone())) + .collect(); + if out.is_empty() { + return out; + } + loop { + let before = out.len(); + for s in snaps { + for (id, it) in &s.items { + if !id.kind.is_order() && it.refs.iter().any(|r| out.contains(r)) { + out.insert(id.clone()); + } + } + } + if out.len() == before { + return out; + } + } +} + +/// 一个上游的地址（`conn.base_url`） +fn base_url(it: &Item) -> Option<&str> { + it.units.get(unit::CONN)?.get("base_url")?.as_str() +} + +/// 一把网关密钥的值 +fn key_value(it: &Item) -> Option<&Value> { + it.units.get(unit::ALL)?.get("key") +} + +impl Engine { + /// `data_dir` 是配置文件所在的目录：本机的同步状态放在它底下的 `sync/` + pub fn new(store: S, host: H, data_dir: &Path, settings: Settings) -> Self { + Self { + store, + host, + local: Local::new(data_dir), + settings, + keyring: None, + cache: BTreeMap::new(), + quiet: None, + last_report: Report::default(), + } + } + + pub fn settings(&self) -> &Settings { + &self.settings + } + + pub fn set_settings(&mut self, s: Settings) { + if s.passphrase != self.settings.passphrase || s.encrypt != self.settings.encrypt { + self.keyring = None; + self.cache.clear(); + } + self.quiet = None; + self.settings = s; + } + + pub fn local(&self) -> &Local { + &self.local + } + + pub fn state(&self) -> State { + self.local.state() + } + + /// 上一轮的结果（冲突、设备列表） + pub fn last_report(&self) -> &Report { + &self.last_report + } + + /// 下一轮不走捷径：哪怕同步位置和本机都没变，也从头看一遍 + pub fn invalidate(&mut self) { + self.quiet = None; + } + + // ─────────────────────────────────────────────── 读 + + /// 拿到版本密钥。没加密是 `Ok(None)` + async fn keyring(&mut self) -> Result, RoundError> { + let Some(bytes) = self.store.get(KEYRING).await? else { + self.keyring = None; + return Ok(None); + }; + let hash: [u8; 32] = Sha256::digest(&bytes).into(); + if self.keyring.as_ref().is_some_and(|(_, h)| *h == hash) { + return Ok(self.keyring.as_ref().map(|(k, _)| k)); + } + let Some(pass) = self.settings.passphrase.clone().filter(|p| !p.is_empty()) else { + return Err(RoundError::Locked(msg!( + "sync.locked_new" => + "the sync location is now encrypted. Enter the sync passphrase to keep syncing" + ))); + }; + let k = tokio::task::spawn_blocking(move || Keyring::open(&bytes, &pass)) + .await + .map_err(local_err)?; + match k { + Ok(k) => { + self.cache.clear(); + self.keyring = Some((k, hash)); + Ok(self.keyring.as_ref().map(|(k, _)| k)) + } + Err(KeyringError::WrongPassphrase) => Err(RoundError::Locked(msg!( + "sync.passphrase_changed" => + "the sync passphrase was changed on another device. Enter the new passphrase" + ))), + Err(KeyringError::TooNew) => Err(RoundError::TooNew { + core_version: String::new(), + }), + Err(KeyringError::Corrupt) => Err(RoundError::NotArrived), + } + } + + /// 读一个版本。读不出来（没传完、坏了、不是这把密钥加密的）是 `None` + async fn fetch(&mut self, name: &VersionName) -> Result, RoundError> { + if let Some(f) = self.cache.get(name) { + return Ok(Some(f.clone())); + } + let Some(bytes) = self.store.get(&name.path()).await? else { + return Ok(None); + }; + let identity = self.keyring.as_ref().map(|(k, _)| k.identity().clone()); + let decoded = tokio::task::spawn_blocking(move || { + let unseal = match &identity { + Some(id) => Unseal::Identity(id), + None => Unseal::None, + }; + let pack = tw_pack::read(&bytes, unseal)?; + let snap = model::snapshot(&pack); + Ok::<_, PackError>((pack, snap)) + }) + .await + .map_err(local_err)?; + let fetched = match decoded { + Err(PackError::TooNew { core_version, .. }) => Fetched::TooNew(core_version), + Err(_) => return Ok(None), + Ok((pack, Err(e))) if e.is_too_new() => Fetched::TooNew(pack.manifest.core_version), + Ok((_, Err(_))) => return Ok(None), + Ok((pack, Ok(snap))) => { + let Some(meta) = pack + .manifest + .sync + .clone() + .and_then(|v| serde_json::from_value::(v).ok()) + else { + return Ok(None); + }; + // 名字和内容对不上（被人挪过、改过名），或者不加密的版本校验和对不上：当作坏了 + if meta.seq != name.seq || layout::device_tag(&meta.device) != name.device { + return Ok(None); + } + if !pack.manifest.encrypted + && meta.sha256.as_deref() != Some(content_sha256(&pack).as_str()) + { + return Ok(None); + } + let content = pack.content_id(); + Fetched::Ok(Arc::new(Version { + name: name.clone(), + pack, + meta, + snap, + content, + })) + } + }; + self.cache.insert(name.clone(), fetched.clone()); + Ok(Some(fetched)) + } + + fn base(&self) -> Option { + let bytes = self.local.base()?; + let pack = tw_pack::read(&bytes, Unseal::None).ok()?; + let meta: Meta = serde_json::from_value(pack.manifest.sync.clone()?).ok()?; + let snap = model::snapshot(&pack).ok()?; + let name = VersionName::parse(&self.local.state().base?)?; + let content = pack.content_id(); + Some(Version { + name, + pack, + meta, + snap, + content, + }) + } + + /// 同步位置上的版本：读得出来的、读不出来的名字 + async fn versions( + &mut self, + ) -> Result< + ( + Vec, + Vec>, + Vec<(VersionName, String)>, + ), + RoundError, + > { + let mut names: Vec = self + .store + .list(VERSIONS) + .await? + .into_iter() + .filter_map(|e| VersionName::parse(&e.name)) + .collect(); + names.sort(); + self.cache.retain(|n, _| names.contains(n)); + let mut ok = Vec::new(); + let mut too_new = Vec::new(); + for n in &names { + match self.fetch(n).await? { + Some(Fetched::Ok(v)) => ok.push(v), + Some(Fetched::TooNew(cv)) => too_new.push((n.clone(), cv)), + None => {} + } + } + Ok((names, ok, too_new)) + } + + /// 同步位置上的设备记录。半年没更新的顺手删掉 + async fn devices(&mut self) -> Result, RoundError> { + let kr = self.keyring.as_ref().map(|(k, _)| k); + let now = now_ms(); + let mut out = Vec::new(); + for e in self.store.list(DEVICES).await? { + let path = format!("{DEVICES}/{}", e.name); + let Some(bytes) = self.store.get(&path).await? else { + continue; + }; + let Some(rec) = DeviceRecord::open(&bytes, kr) else { + continue; + }; + if rec.is_stale(now) && rec.id != self.settings.device_id { + let _ = self.store.delete(&path).await; + continue; + } + out.push(rec); + } + out.sort_by(|a, b| a.name.cmp(&b.name).then(a.id.cmp(&b.id))); + Ok(out) + } + + async fn write_device(&mut self, paused: bool) -> Result<(), RoundError> { + let rec = DeviceRecord { + id: self.settings.device_id.clone(), + name: self.settings.device_name.clone(), + core_version: tw_pack::CORE_VERSION.to_string(), + last_seen_ms: now_ms(), + paused_update: paused, + remote_core: self.settings.remote_core, + }; + let kr = self.keyring.as_ref().map(|(k, _)| k); + self.store + .put( + &layout::device_path(&self.settings.device_id), + &rec.seal(kr), + ) + .await?; + let mut st = self.local.state(); + st.device_record_ms = rec.last_seen_ms; + st.device_record_paused = paused; + self.local.save_state(&st).map_err(RoundError::Local)?; + Ok(()) + } + + // ─────────────────────────────────────────────── 合并几个头 + + /// `a`、`b` 共同的上一版本 + fn common_ancestor( + &self, + a: &[Arc], + b: &Version, + all: &[Arc], + base: Option<&Version>, + ) -> Option { + // 版本标识 → (上一版本, 它们的内容标识) + let mut parents: BTreeMap, Vec)> = BTreeMap::new(); + for v in all.iter().map(|v| v.as_ref()).chain(base) { + parents.insert( + v.name.id(), + (v.meta.parents.clone(), v.meta.parent_content.clone()), + ); + } + let mut content_of: BTreeMap = BTreeMap::new(); + for (ps, cs) in parents.values() { + for (p, c) in ps.iter().zip(cs) { + content_of.insert(p.clone(), c.clone()); + } + } + let ancestors = |start: &str| -> BTreeSet { + let mut out = BTreeSet::new(); + let mut todo = vec![start.to_string()]; + while let Some(id) = todo.pop() { + if !out.insert(id.clone()) { + continue; + } + if let Some((ps, _)) = parents.get(&id) { + todo.extend(ps.iter().cloned()); + } + } + out + }; + let left: BTreeSet = a.iter().flat_map(|v| ancestors(&v.name.id())).collect(); + let right = ancestors(&b.name.id()); + let mut common: Vec = left + .intersection(&right) + .filter_map(|id| VersionName::parse(id)) + .collect(); + common.sort(); + let id = common.pop()?.id(); + if let Some(v) = all.iter().find(|v| v.name.id() == id) { + return Some((**v).clone()); + } + if let Some(b) = base.filter(|b| b.name.id() == id) { + return Some(b.clone()); + } + let content = content_of.get(&id)?; + let pack = self.host.backup(content)?; + let snap = model::snapshot(&pack).ok()?; + Some(Version { + name: VersionName::parse(&id)?, + meta: Meta::default(), + content: content.clone(), + snap, + pack, + }) + } + + /// 几个头合并成一份 + fn fold(&self, heads: &[Arc], all: &[Arc], base: Option<&Version>) -> Remote { + let first = &heads[0]; + let mut acc = Remote { + snap: first.snap.clone(), + conflicts: first.meta.conflicts.clone(), + heads: vec![first.clone()], + device: first.meta.device.clone(), + device_name: first.meta.device_name.clone(), + at_ms: first.pack.manifest.created_at_ms, + }; + for h in &heads[1..] { + let anc = self.common_ancestor(&acc.heads, h, all, base); + let anc_conf: BTreeSet<&str> = anc + .as_ref() + .map(|a| a.meta.conflicts.iter().map(|c| c.id.as_str()).collect()) + .unwrap_or_default(); + // 两边的冲突记录合起来，减去在共同的上一版本里有、而某一边已经处理掉了的 + let ids_a: BTreeSet<&str> = acc.conflicts.iter().map(|c| c.id.as_str()).collect(); + let ids_b: BTreeSet<&str> = h.meta.conflicts.iter().map(|c| c.id.as_str()).collect(); + let mut open: Vec = Vec::new(); + for c in acc.conflicts.iter().chain(&h.meta.conflicts) { + let resolved = anc_conf.contains(c.id.as_str()) + && !(ids_a.contains(c.id.as_str()) && ids_b.contains(c.id.as_str())); + if !resolved && !open.iter().any(|o| o.id == c.id) { + open.push(c.clone()); + } + } + let (mut hold_items, hold_units) = record::holds(&open); + for (o, _) in ORDERS { + hold_items.insert(ItemId::new(*o, "")); + } + let base_objs = anc.as_ref().map(|a| objects(&a.snap)); + let (lo, ro) = (objects(&acc.snap), objects(&h.snap)); + let out = merge::merge(merge::Input { + base: base_objs.as_ref(), + local: &lo, + remote: &ro, + hold_items: &hold_items, + hold_units: &hold_units, + }); + let mut snap = rebuild(&out.shared, &[&h.snap, &acc.snap]); + for (o, k) in ORDERS { + let m = order::merge( + order_list(anc.as_ref().map(|a| &a.snap), *o).as_deref(), + &order_list(Some(&acc.snap), *o).unwrap_or_default(), + &order_list(Some(&h.snap), *o).unwrap_or_default(), + &names_of(&snap, *k), + &names_of(&snap, *k), + open.iter().any(|c| c.item.kind == *o), + ); + set_order(&mut snap, *o, m.shared); + if m.conflict { + let id = ItemId::new(*o, ""); + open.push(Conflict { + id: record::new_id(), + units: vec![unit::ALL.to_string()], + shared: Side { + device: h.meta.device.clone(), + device_name: h.meta.device_name.clone(), + at_ms: h.pack.manifest.created_at_ms, + units: h.snap.items.get(&id).map(|i| i.units.clone()), + }, + other: Side { + device: acc.device.clone(), + device_name: acc.device_name.clone(), + at_ms: acc.at_ms, + units: acc.snap.items.get(&id).map(|i| i.units.clone()), + }, + item: id, + reason: None, + }); + } + } + for f in &out.conflicts { + open.push(found( + f, + ( + &h.snap, + Side { + device: h.meta.device.clone(), + device_name: h.meta.device_name.clone(), + at_ms: h.pack.manifest.created_at_ms, + units: None, + }, + ), + ( + &acc.snap, + Side { + device: acc.device.clone(), + device_name: acc.device_name.clone(), + at_ms: acc.at_ms, + units: None, + }, + ), + )); + } + // 同步版本里冲突的那一部分是「同步版本那一边」记下的值 + for c in &open { + set_side(&mut snap, c, &c.shared, &h.snap); + } + snap.relink(); + let later = h.pack.manifest.created_at_ms >= acc.at_ms; + acc = Remote { + snap, + conflicts: open, + heads: acc.heads.iter().cloned().chain([h.clone()]).collect(), + device: if later { + h.meta.device.clone() + } else { + acc.device + }, + device_name: if later { + h.meta.device_name.clone() + } else { + acc.device_name + }, + at_ms: acc.at_ms.max(h.pack.manifest.created_at_ms), + }; + } + acc + } + + // ─────────────────────────────────────────────── 写 + + /// 上传一个新版本：内容是 `shared`，依据 `parents` + async fn publish( + &mut self, + shared: &Snapshot, + seq: u64, + parents: &[Arc], + conflicts: Vec, + ) -> Result { + let name = VersionName::new(seq, &self.settings.device_id); + let empty = Pack { + manifest: tw_pack::Manifest::new(PackKind::Full, PackOrigin::Sync), + config_yaml: format!("version: {}\n", tw_config::SCHEMA_VERSION), + plugins: Default::default(), + client_prefs: None, + }; + let mut pack = + model::materialize(&empty, shared).map_err(|e| RoundError::Local(e.msg()))?; + pack.manifest = tw_pack::Manifest::new(PackKind::Full, PackOrigin::Sync); + let mut meta = Meta { + seq, + device: self.settings.device_id.clone(), + device_name: self.settings.device_name.clone(), + parents: parents.iter().map(|p| p.name.id()).collect(), + parent_content: parents.iter().map(|p| p.content.clone()).collect(), + sha256: None, + conflicts, + }; + let encrypted = self.keyring.is_some(); + if !encrypted { + meta.sha256 = Some(content_sha256(&pack)); + } + pack.manifest.sync = Some(serde_json::to_value(&meta).expect("plain data serializes")); + let bytes = { + let recipient = self.keyring.as_ref().map(|(k, _)| k.recipient()); + let seal = match &recipient { + Some(r) => Seal::Recipient(r), + None => Seal::None, + }; + tw_pack::write(&pack, seal).map_err(|e| RoundError::Local(e.msg()))? + }; + match self.store.put_new(&name.path(), &bytes).await { + Ok(()) => {} + Err(StoreError::Exists { .. }) => return Err(RoundError::Stale), + Err(e) => return Err(e.into()), + } + let plain = tw_pack::write(&pack, Seal::None).map_err(|e| RoundError::Local(e.msg()))?; + let pack = tw_pack::read(&plain, Unseal::None).map_err(|e| RoundError::Local(e.msg()))?; + let snap = model::snapshot(&pack).map_err(|e| RoundError::Local(e.msg()))?; + let content = pack.content_id(); + let v = Version { + name: name.clone(), + pack, + meta, + snap, + content, + }; + self.cache.insert(name, Fetched::Ok(Arc::new(v.clone()))); + Ok(v) + } + + fn save_base(&self, v: &Version, st: &mut State) -> Result<(), RoundError> { + let bytes = tw_pack::write(&v.pack, Seal::None).map_err(|e| RoundError::Local(e.msg()))?; + self.local.save_base(&bytes).map_err(RoundError::Local)?; + st.base = Some(v.name.id()); + Ok(()) + } + + async fn prune(&mut self, names: &[VersionName], ok: &[Arc]) { + let listed: Vec = names + .iter() + .map(|n| Listed { + name: n.clone(), + parents: ok + .iter() + .find(|v| &v.name == n) + .map(|v| { + v.meta + .parents + .iter() + .filter_map(|p| VersionName::parse(p)) + .collect() + }) + .unwrap_or_default(), + }) + .collect(); + for n in layout::to_delete(&listed) { + let _ = self.store.delete(&n.path()).await; + self.cache.remove(&n); + } + } + + // ─────────────────────────────────────────────── 一轮 + + /// 同步一轮。冲突不算失败：在 [`Report::conflicts`] 里 + pub async fn round(&mut self) -> Result { + self.round_with(&[]).await + } + + /// 同步一轮，顺带处理几处冲突。处理的结果作为新版本上传，别的设备随之清除这几处冲突。 + /// 已经被别的设备处理掉了的那几处不管 + pub async fn round_with(&mut self, decisions: &[Decision]) -> Result { + if !decisions.is_empty() { + self.quiet = None; + } + match self.round_inner(decisions).await { + Ok(r) => { + self.last_report = r.clone(); + Ok(r) + } + Err(e) => Err(e), + } + } + + async fn round_inner(&mut self, decisions: &[Decision]) -> Result { + let mut st = self.local.state(); + if st.place != self.settings.place && !st.place.is_empty() { + // 换了同步位置：原来的基准是那个位置上的，作废 + self.local.clear(); + st = State { + place: self.settings.place.clone(), + ..Default::default() + }; + self.local.save_state(&st).map_err(RoundError::Local)?; + self.cache.clear(); + } + st.place.clone_from(&self.settings.place); + let base = self.base(); + let kr = self.keyring().await?.is_some(); + let (names, ok, too_new) = self.versions().await?; + if names.is_empty() { + if base.is_some() { + return Err(RoundError::Emptied); + } + // 开启同步时上传第一个版本没成（中途断了）：这一轮补上 + self.publish_first().await?; + return self.finish_quiet(Vec::new(), kr).await; + } + let cur = self.host.current().map_err(RoundError::Local)?; + let listing: Vec = names.iter().map(|n| n.id()).collect(); + let local_content = cur.pack.content_id(); + if self.quiet.as_ref() == Some(&(listing.clone(), local_content.clone())) { + // 什么都没变：冲突还是上一轮的。照样记一次同步成功，设备记录隔一阵写一次，设备 + // 列表重读（别的设备的记录会变） + st.last_synced_ms = Some(now_ms()); + st.failing_since_ms = None; + st.error = None; + self.local.save_state(&st).map_err(RoundError::Local)?; + if now_ms().saturating_sub(st.device_record_ms) > DEVICE_RECORD_EVERY_MS { + self.write_device(false).await?; + } + let r = Report { + encrypted: kr, + conflicts: self.last_report.conflicts.clone(), + devices: self.devices().await?, + ..Default::default() + }; + return Ok(r); + } + // 头：没有别的版本依据它的版本 + let referenced: BTreeSet = ok + .iter() + .flat_map(|v| v.meta.parents.iter().cloned()) + .collect(); + if let Some((_, cv)) = too_new.iter().find(|(n, _)| !referenced.contains(&n.id())) { + if !st.device_record_paused { + self.write_device(true).await?; + } + return Err(RoundError::TooNew { + core_version: cv.clone(), + }); + } + let top_ok = ok.iter().map(|v| v.name.seq).max(); + let top_all = names.iter().map(|n| n.seq).max(); + if top_ok.is_none() || top_ok < top_all { + return Err(RoundError::NotArrived); + } + let heads: Vec> = ok + .iter() + .filter(|v| !referenced.contains(&v.name.id())) + .cloned() + .collect(); + // 同步位置没有 keyring、版本却是加密的：keyring 还没传到 + if !kr && heads.iter().any(|h| h.pack.manifest.encrypted) { + return Err(RoundError::NotArrived); + } + let remote = self.fold(&heads, &ok, base.as_ref()); + let local_snap = model::snapshot(&cur.pack).map_err(|e| RoundError::Local(e.msg()))?; + + let me = self.settings.device_id.clone(); + let mut local = local_snap.clone(); + // 别的设备处理掉了的冲突：照同步版本改 + let open_ids: BTreeSet<&str> = remote.conflicts.iter().map(|c| c.id.as_str()).collect(); + if let Some(b) = &base { + for c in &b.meta.conflicts { + if !open_ids.contains(c.id.as_str()) { + force(&mut local, &remote.snap, c); + } + } + } + // 自己参与的冲突：本机又改了就更新记录；改得和另一边一样了，冲突就过去了 + let mut open: Vec = Vec::new(); + let mut i_moved: Vec = Vec::new(); + for c in &remote.conflicts { + let now = side_units(&local, c); + let mut c = c.clone(); + if c.other.device == me { + if now == c.shared.units { + continue; + } + if now != c.other.units { + c.other.units = now; + c.other.at_ms = cur.changed_ms; + } + } else if c.shared.device == me { + if now == c.other.units { + continue; + } + if now != c.shared.units { + c.shared.units = now; + c.shared.at_ms = cur.changed_ms; + i_moved.push(c.clone()); + } + } + open.push(c); + } + + // 这一轮要处理的冲突：从记录里拿掉（不再各留各的），本机按选的那一边改 + for d in decisions { + let Some(pos) = open.iter().position(|c| c.id == d.id) else { + continue; + }; + let c = open.remove(pos); + let local_units = local.items.get(&c.item).map(|it| c.pick(&it.units)); + let (_, theirs) = c.mine(&me, local_units.as_ref()); + let theirs = theirs.clone(); + match d.take { + Take::Local => {} + Take::Remote => set_side(&mut local, &c, &theirs, &remote.snap), + Take::Both => { + if !c.both_allowed() { + return Err(RoundError::Local(msg!( + "sync.both_not_allowed" => + "keeping both is not possible for this conflict: one side deleted \ + it, or it has no name" + ))); + } + let name = both_name(&c.item, d.new_name.as_deref(), &theirs.device_name); + let id = ItemId::new(c.item.kind, name); + let taken = + local.items.contains_key(&id) || remote.snap.items.contains_key(&id); + let id = match (taken, d.new_name.is_some()) { + (false, _) => id, + (true, true) => { + return Err(RoundError::Local(msg!( + "sync.both_name_taken", name = id.name => + "the name {name} is already taken" + ))); + } + (true, false) => { + let mut all = local.clone(); + all.items.extend(remote.snap.items.clone()); + ItemId::new(c.item.kind, model::free_name(&all, c.item.kind, &id.name)) + } + }; + let mut copy = local + .items + .get(&c.item) + .or(remote.snap.items.get(&c.item)) + .cloned() + .unwrap_or_default(); + for (u, v) in theirs.units.iter().flatten() { + copy.units.insert(u.clone(), v.clone()); + } + local.items.insert(id, copy); + local.relink(); + } + } + } + + let (mut hold_items, mut hold_units) = record::holds(&open); + let ex = excluded( + &self.settings.exclude, + &[&local, &remote.snap] + .into_iter() + .chain(base.as_ref().map(|b| &b.snap)) + .collect::>(), + ); + hold_items.extend(ex.iter().cloned()); + for (o, _) in ORDERS { + hold_items.insert(ItemId::new(*o, "")); + } + // 不加密时经同步停用的插件：本机启用了就放开 + st.held_plugins.retain(|id, _| { + let item = local.items.get(&ItemId::new(ItemKind::Plugin, id.clone())); + item.is_some_and(|it| it.units.get(unit::ENABLED) != Some(&Value::Bool(true))) + }); + for id in st.held_plugins.keys() { + hold_units.insert(( + ItemId::new(ItemKind::Plugin, id.clone()), + unit::ENABLED.to_string(), + )); + } + + let base_objs = base.as_ref().map(|b| objects(&b.snap)); + let (lo, ro) = (objects(&local), objects(&remote.snap)); + let out = merge::merge(merge::Input { + base: base_objs.as_ref(), + local: &lo, + remote: &ro, + hold_items: &hold_items, + hold_units: &hold_units, + }); + let mut target = rebuild(&out.local, &[&local, &remote.snap]); + let mut shared = rebuild(&out.shared, &[&remote.snap, &local]); + // 不同步的上游、随之不同步的项目：上传的里面是同步位置原有的，不是本机的 + for id in &ex { + match remote.snap.items.get(id) { + Some(it) => shared.items.insert(id.clone(), it.clone()), + None => shared.items.remove(id), + }; + } + // 本机改了自己是「同步版本那一边」的冲突：同步版本跟着改 + for c in &i_moved { + set_side(&mut shared, c, &c.shared, &local); + } + + let side_me = |units: Option>| Side { + device: me.clone(), + device_name: self.settings.device_name.clone(), + at_ms: cur.changed_ms, + units, + }; + let side_remote = |units: Option>| Side { + device: remote.device.clone(), + device_name: remote.device_name.clone(), + at_ms: remote.at_ms, + units, + }; + let mut fresh: Vec = Vec::new(); + for f in &out.conflicts { + fresh.push(found( + f, + (&remote.snap, side_remote(None)), + (&local, side_me(None)), + )); + } + // 已接管的客户端正在用的网关密钥：另一边删了，这边不删，作为冲突列出 + let pinned = self.host.pinned_keys(); + for (id, it) in local.of_kind(ItemKind::Key) { + if pinned.contains(&id.name) + && !target.items.contains_key(id) + && !hold_items.contains(id) + { + target.items.insert(id.clone(), it.clone()); + fresh.push(Conflict { + id: record::new_id(), + item: id.clone(), + units: it.units.keys().cloned().collect(), + shared: side_remote(None), + other: side_me(Some(it.units.clone())), + reason: Some(Reason::KeyInUse), + }); + } + } + // 顺序 + for (o, k) in ORDERS { + let held = open.iter().chain(&fresh).any(|c| c.item.kind == *o); + let m = order::merge( + order_list(base.as_ref().map(|b| &b.snap), *o).as_deref(), + &order_list(Some(&local), *o).unwrap_or_default(), + &order_list(Some(&remote.snap), *o).unwrap_or_default(), + &names_of(&target, *k), + &names_of(&shared, *k), + held, + ); + set_order(&mut target, *o, m.local); + set_order(&mut shared, *o, m.shared); + if m.conflict { + let id = ItemId::new(*o, ""); + fresh.push(Conflict { + id: record::new_id(), + units: vec![unit::ALL.to_string()], + shared: side_remote(remote.snap.items.get(&id).map(|i| i.units.clone())), + other: side_me(local.items.get(&id).map(|i| i.units.clone())), + item: id, + reason: None, + }); + } + } + target.relink(); + shared.relink(); + + // 合并的结果要过配置校验：过不了的那几项留本机的，作为冲突列出 + self.check_target( + &cur.pack, + &local, + &mut target, + &mut shared, + &remote, + &mut fresh, + &side_me, + &side_remote, + )?; + + // 不加密：经同步新增或脚本变了的插件在本机停用；上游的地址变了记一笔 + if !kr { + for (id, it) in target + .of_kind(ItemKind::Plugin) + .map(|(a, b)| (a.clone(), b.clone())) + .collect::>() + { + let before = local.items.get(&id); + let script = it.units.get(unit::SCRIPT); + let changed = before.is_none_or(|b| b.units.get(unit::SCRIPT) != script); + let from_remote = remote + .snap + .items + .get(&id) + .is_some_and(|r| r.units.get(unit::SCRIPT) == script); + if changed && from_remote && it.units.get(unit::ENABLED) == Some(&Value::Bool(true)) + { + if let Some(t) = target.items.get_mut(&id) { + t.units + .insert(unit::ENABLED.to_string(), Value::Bool(false)); + } + st.held_plugins + .insert(id.name.clone(), script.cloned().unwrap_or(Value::Null)); + } + } + for (id, it) in target.of_kind(ItemKind::Provider) { + if let (Some(before), Some(to)) = + (local.items.get(id).and_then(base_url), base_url(it)) + && before != to + { + st.url_changed.insert( + id.name.clone(), + UrlChange { + from: before.to_string(), + to: to.to_string(), + device: Some(remote.device_name.clone()), + at_ms: remote.at_ms, + }, + ); + } + } + } + // 地址又被本机改过、上游没了：那条提示过去了 + st.url_changed.retain(|name, ch| { + target + .items + .get(&ItemId::new(ItemKind::Provider, name.clone())) + .and_then(base_url) + == Some(ch.to.as_str()) + }); + + let mut conflicts = open.clone(); + conflicts.extend(fresh); + + let mut report = Report { + encrypted: kr, + ..Default::default() + }; + // 应用 + if model::normalized(&target) != model::normalized(&local_snap) { + report.keys_changed = local_snap + .of_kind(ItemKind::Key) + .filter(|(id, it)| { + target.items.get(*id).map(|t| key_value(t)) != Some(key_value(it)) + }) + .map(|(id, _)| id.name.clone()) + .collect(); + let pack = + model::materialize(&cur.pack, &target).map_err(|e| RoundError::Local(e.msg()))?; + let from: Vec = { + let mut v: Vec = remote + .heads + .iter() + .filter(|h| h.meta.device != me) + .map(|h| h.meta.device_name.clone()) + .collect(); + v.dedup(); + v + }; + let label = from.join(", "); + match self.host.apply(pack, &cur.version, &label).await { + Ok(()) => {} + Err(ApplyError::Stale) => return Err(RoundError::Stale), + Err(ApplyError::Rejected(m)) => return Err(RoundError::Local(m)), + } + report.applied_from = from; + } + // 上传 + let need_upload = heads.len() > 1 + || model::normalized(&shared) != model::normalized(&remote.snap) + || conflicts != remote.conflicts; + let next_base = if need_upload { + let seq = names.iter().map(|n| n.seq).max().unwrap_or(0) + 1; + let v = self + .publish(&shared, seq, &heads, conflicts.clone()) + .await?; + report.uploaded = Some(v.name.id()); + let mut all: Vec = names.clone(); + all.push(v.name.clone()); + let mut ok_all = ok.clone(); + ok_all.push(Arc::new(v.clone())); + self.prune(&all, &ok_all).await; + v + } else { + (*heads[0]).clone() + }; + self.save_base(&next_base, &mut st)?; + st.last_synced_ms = Some(now_ms()); + st.failing_since_ms = None; + st.error = None; + self.local.save_state(&st).map_err(RoundError::Local)?; + report.conflicts = conflicts; + + let changed = report.uploaded.is_some() || !report.applied_from.is_empty(); + if changed + || st.device_record_paused + || now_ms().saturating_sub(st.device_record_ms) > DEVICE_RECORD_EVERY_MS + { + self.write_device(false).await?; + } + report.devices = self.devices().await?; + // 这一轮之后同步位置上的样子 + let after: Vec = self + .store + .list(VERSIONS) + .await? + .into_iter() + .filter_map(|e| VersionName::parse(&e.name).map(|n| n.id())) + .collect(); + let local_after = self + .host + .current() + .map(|c| c.pack.content_id()) + .unwrap_or_default(); + self.quiet = Some((after, local_after)); + Ok(report) + } + + async fn finish_quiet( + &mut self, + conflicts: Vec, + encrypted: bool, + ) -> Result { + let devices = self.devices().await?; + let r = Report { + conflicts, + devices, + encrypted, + ..Default::default() + }; + Ok(r) + } + + /// 合并出来的本机结果过一遍配置校验。过不了时一项一项试：从本机现在的样子出发，把变了的 + /// 项目一个个换过去，换了就过不了的那一项留本机的，记成冲突（`invalid`），上传的里面也留 + /// 本机的 —— 不然别的设备收到的也是一份过不了校验的配置 + #[allow(clippy::too_many_arguments)] + fn check_target( + &self, + local_pack: &Pack, + local: &Snapshot, + target: &mut Snapshot, + shared: &mut Snapshot, + remote: &Remote, + fresh: &mut Vec, + side_me: &dyn Fn(Option>) -> Side, + side_remote: &dyn Fn(Option>) -> Side, + ) -> Result<(), RoundError> { + let valid = |s: &Snapshot| -> bool { + model::materialize(local_pack, s) + .ok() + .is_some_and(|p| tw_config::try_parse(&p.config_yaml).is_ok()) + }; + if valid(target) { + return Ok(()); + } + let changed: Vec = local + .items + .keys() + .chain(target.items.keys()) + .filter(|id| !id.kind.is_order() && local.items.get(*id) != target.items.get(*id)) + .cloned() + .collect::>() + .into_iter() + .collect(); + let mut t = local.clone(); + let fix_orders = |s: &mut Snapshot, from: &Snapshot| { + for (o, k) in ORDERS { + let names = names_of(s, *k); + let mut list: Vec = order_list(Some(from), *o) + .unwrap_or_default() + .into_iter() + .filter(|n| names.contains(n)) + .collect(); + for n in &names { + if !list.contains(n) { + list.push(n.clone()); + } + } + set_order(s, *o, list); + } + s.relink(); + }; + for id in changed { + let mut next = t.clone(); + match target.items.get(&id) { + Some(it) => next.items.insert(id.clone(), it.clone()), + None => next.items.remove(&id), + }; + fix_orders(&mut next, target); + if valid(&next) { + t = next; + continue; + } + // 这一项留本机的 + match local.items.get(&id) { + Some(it) => shared.items.insert(id.clone(), it.clone()), + None => shared.items.remove(&id), + }; + let units: Vec = local + .items + .get(&id) + .or(target.items.get(&id)) + .map(|it| it.units.keys().cloned().collect()) + .unwrap_or_default(); + fresh.retain(|c| c.item != id); + fresh.push(Conflict { + id: record::new_id(), + item: id.clone(), + units, + shared: side_remote(remote.snap.items.get(&id).map(|i| i.units.clone())), + other: side_me(local.items.get(&id).map(|i| i.units.clone())), + reason: Some(Reason::Invalid), + }); + } + fix_orders(&mut t, target); + let before = shared.clone(); + fix_orders(shared, &before); + *target = t; + if !valid(target) { + return Err(RoundError::Local(msg!( + "sync.merge_invalid" => + "the merged configuration does not pass validation, so nothing was applied" + ))); + } + Ok(()) + } + + // ─────────────────────────────────────────────── 开启、加入 + + /// 看一眼同步位置：写得进去吗，空的还是已经有数据，加没加密，有哪些设备（加密时给了 + /// 对的口令才列得出） + pub async fn probe(store: &S, passphrase: Option<&str>) -> Probe { + let r: u64 = rand::random(); + let path = format!("probe-{r:016x}"); + let writable = match store.put_new(&path, b"ThinkWatch sync probe").await { + Ok(()) => store.delete(&path).await, + Err(e) => Err(e), + }; + if let Err(e) = &writable { + return Probe { + writable: Err(e.clone()), + existing: None, + }; + } + let versions = store.list(VERSIONS).await.unwrap_or_default(); + let keyring = store.get(KEYRING).await.ok().flatten(); + if versions.is_empty() && keyring.is_none() { + return Probe { + writable, + existing: None, + }; + } + let opened = match (&keyring, passphrase) { + (Some(b), Some(p)) => { + let (b, p) = (b.clone(), p.to_string()); + tokio::task::spawn_blocking(move || Keyring::open(&b, &p).ok()) + .await + .ok() + .flatten() + } + _ => None, + }; + let encrypted = keyring.is_some(); + let unlocked = !encrypted || opened.is_some(); + let mut devices = Vec::new(); + if unlocked { + for e in store.list(DEVICES).await.unwrap_or_default() { + if let Ok(Some(b)) = store.get(&format!("{DEVICES}/{}", e.name)).await + && let Some(rec) = DeviceRecord::open(&b, opened.as_ref()) + { + devices.push(rec); + } + } + devices.sort_by(|a, b| a.name.cmp(&b.name)); + } + Probe { + writable, + existing: Some(Existing { + encrypted, + unlocked, + devices, + }), + } + } + + /// 同步位置空着时开启：加密的先建 keyring（**仅当不存在时**：两台设备同时开启时后写的 + /// 那一台拿到 `Ok(false)`，转去加入），然后以本机配置上传第一个版本 + pub async fn create(&mut self) -> Result { + if self.settings.encrypt { + let pass = self + .settings + .passphrase + .clone() + .filter(|p| !p.is_empty()) + .ok_or_else(|| { + RoundError::Local(tw_config::sync::SyncError::PassphraseMissing.msg()) + })?; + let k = Keyring::generate(); + let sealed = tokio::task::spawn_blocking({ + let pass = pass.clone(); + move || (k.seal(&pass), k) + }) + .await + .map_err(local_err)?; + let (sealed, k) = sealed; + if !self.store.create_exclusive(KEYRING, &sealed).await? { + return Ok(false); + } + let hash: [u8; 32] = Sha256::digest(&sealed).into(); + self.keyring = Some((k, hash)); + } else if self.store.get(KEYRING).await?.is_some() { + return Ok(false); + } + if !self.store.list(VERSIONS).await?.is_empty() { + return Ok(false); + } + self.local.clear(); + let st = State { + place: self.settings.place.clone(), + ..Default::default() + }; + self.local.save_state(&st).map_err(RoundError::Local)?; + self.publish_first().await?; + self.quiet = None; + Ok(true) + } + + /// 同步位置的最新（几个头合并之后），写成一个配置包：加入时给导入预览比对。不含本机设置 + pub async fn latest(&mut self) -> Result { + let remote = self.latest_remote().await?.0; + self.pack_of(&remote.snap) + } + + fn pack_of(&self, snap: &Snapshot) -> Result { + let empty = Pack { + manifest: tw_pack::Manifest::new(PackKind::Full, PackOrigin::Sync), + config_yaml: format!("version: {}\n", tw_config::SCHEMA_VERSION), + plugins: Default::default(), + client_prefs: None, + }; + model::materialize(&empty, snap).map_err(|e| RoundError::Local(e.msg())) + } + + async fn latest_remote( + &mut self, + ) -> Result<(Remote, Vec, Vec>), RoundError> { + self.keyring().await?; + let (names, ok, too_new) = self.versions().await?; + let referenced: BTreeSet = ok + .iter() + .flat_map(|v| v.meta.parents.iter().cloned()) + .collect(); + if let Some((_, cv)) = too_new.iter().find(|(n, _)| !referenced.contains(&n.id())) { + return Err(RoundError::TooNew { + core_version: cv.clone(), + }); + } + let heads: Vec> = ok + .iter() + .filter(|v| !referenced.contains(&v.name.id())) + .cloned() + .collect(); + let top_ok = ok.iter().map(|v| v.name.seq).max(); + if heads.is_empty() || top_ok < names.iter().map(|n| n.seq).max() { + return Err(RoundError::NotArrived); + } + let base = self.base(); + Ok((self.fold(&heads, &ok, base.as_ref()), names, ok)) + } + + /// 加入已有的同步。`merged` 是本机按导入的规则合并之后的完整配置包（「合并」）；`None` 是 + /// 用同步数据替换本机配置（新安装、或者选了「使用同步数据」）—— 那时本机已接管的客户端 + /// 正在用的网关密钥并入同步数据，不同步的上游留在本机。 + /// + /// 本机先照结果改，再上传一个依据同步位置最新版本的新版本，作为这台设备的基准 + pub async fn join(&mut self, merged: Option) -> Result { + let (remote, names, _) = self.latest_remote().await?; + let cur = self.host.current().map_err(RoundError::Local)?; + let local = model::snapshot(&cur.pack).map_err(|e| RoundError::Local(e.msg()))?; + let ex = excluded(&self.settings.exclude, &[&local, &remote.snap]); + let mut target = match &merged { + Some(p) => model::snapshot(p).map_err(|e| RoundError::Local(e.msg()))?, + None => { + let mut t = remote.snap.clone(); + let pinned = self.host.pinned_keys(); + for (id, it) in local.of_kind(ItemKind::Key) { + if pinned.contains(&id.name) { + t.items.insert(id.clone(), it.clone()); + } + } + t + } + }; + for id in &ex { + match local.items.get(id) { + Some(it) => target.items.insert(id.clone(), it.clone()), + None => target.items.remove(id), + }; + } + let mut shared = target.clone(); + for id in &ex { + match remote.snap.items.get(id) { + Some(it) => shared.items.insert(id.clone(), it.clone()), + None => shared.items.remove(id), + }; + } + for (o, k) in ORDERS { + let m = order::merge( + order_list(Some(&remote.snap), *o).as_deref(), + &order_list(Some(&target), *o) + .or_else(|| order_list(Some(&local), *o)) + .unwrap_or_default(), + &order_list(Some(&remote.snap), *o).unwrap_or_default(), + &names_of(&target, *k), + &names_of(&shared, *k), + false, + ); + set_order(&mut target, *o, m.local); + set_order(&mut shared, *o, m.shared); + } + target.relink(); + shared.relink(); + // 不加密：经同步新来、或者脚本变了的插件在本机停用着，要在本机启用（同步一轮里是同一条） + let mut held = BTreeMap::new(); + if self.keyring.is_none() { + let ids: Vec = target + .of_kind(ItemKind::Plugin) + .map(|(id, _)| id.clone()) + .collect(); + for id in ids { + let script = target.items[&id].units.get(unit::SCRIPT).cloned(); + let changed = local + .items + .get(&id) + .is_none_or(|b| b.units.get(unit::SCRIPT) != script.as_ref()); + let from_remote = remote + .snap + .items + .get(&id) + .is_some_and(|r| r.units.get(unit::SCRIPT) == script.as_ref()); + let on = target.items[&id].units.get(unit::ENABLED) == Some(&Value::Bool(true)); + if changed && from_remote && on { + if let Some(t) = target.items.get_mut(&id) { + t.units + .insert(unit::ENABLED.to_string(), Value::Bool(false)); + } + held.insert(id.name.clone(), script.unwrap_or(Value::Null)); + } + } + } + let mut report = Report { + encrypted: self.keyring.is_some(), + ..Default::default() + }; + if model::normalized(&target) != model::normalized(&local) { + report.keys_changed = local + .of_kind(ItemKind::Key) + .filter(|(id, it)| { + target.items.get(*id).map(|t| key_value(t)) != Some(key_value(it)) + }) + .map(|(id, _)| id.name.clone()) + .collect(); + let pack = + model::materialize(&cur.pack, &target).map_err(|e| RoundError::Local(e.msg()))?; + if tw_config::try_parse(&pack.config_yaml).is_err() { + return Err(RoundError::Local(msg!( + "sync.merge_invalid" => + "the merged configuration does not pass validation, so nothing was applied" + ))); + } + match self + .host + .apply(pack, &cur.version, &remote.device_name) + .await + { + Ok(()) => {} + Err(ApplyError::Stale) => return Err(RoundError::Stale), + Err(ApplyError::Rejected(m)) => return Err(RoundError::Local(m)), + } + report.applied_from = vec![remote.device_name.clone()]; + } + let seq = names.iter().map(|n| n.seq).max().unwrap_or(0) + 1; + let v = self + .publish(&shared, seq, &remote.heads, remote.conflicts.clone()) + .await?; + report.uploaded = Some(v.name.id()); + let mut st = State { + place: self.settings.place.clone(), + held_plugins: held, + ..Default::default() + }; + self.save_base(&v, &mut st)?; + st.last_synced_ms = Some(now_ms()); + st.failing_since_ms = None; + st.error = None; + self.local.save_state(&st).map_err(RoundError::Local)?; + self.write_device(false).await?; + report.devices = self.devices().await?; + report.conflicts = remote.conflicts; + self.quiet = None; + self.last_report = report.clone(); + Ok(report) + } + + // ─────────────────────────────────────────────── 加密 + + /// 之后开启加密：建 keyring，以它重新上传最新版本，删掉未加密的版本。其他设备下次同步时 + /// 要输入口令 + pub async fn enable_encryption(&mut self, passphrase: &str) -> Result<(), RoundError> { + let (remote, names, _) = self.latest_remote().await?; + if self.keyring.is_some() { + return Ok(()); + } + let k = Keyring::generate(); + let sealed = k.seal(passphrase); + if !self.store.create_exclusive(KEYRING, &sealed).await? { + return Err(RoundError::Locked(msg!( + "sync.locked_new" => + "the sync location is now encrypted. Enter the sync passphrase to keep syncing" + ))); + } + let hash: [u8; 32] = Sha256::digest(&sealed).into(); + self.keyring = Some((k, hash)); + self.settings.encrypt = true; + self.settings.passphrase = Some(passphrase.to_string()); + self.reupload(remote, &names).await + } + + /// 关闭加密：口令要对。重新上传不加密的最新版本，删掉 keyring 和加密过的版本 + pub async fn disable_encryption(&mut self, passphrase: &str) -> Result<(), RoundError> { + let Some(bytes) = self.store.get(KEYRING).await? else { + self.keyring = None; + return Ok(()); + }; + let hash: [u8; 32] = Sha256::digest(&bytes).into(); + let pass = passphrase.to_string(); + let opened = tokio::task::spawn_blocking(move || Keyring::open(&bytes, &pass)) + .await + .map_err(local_err)? + .map_err(|e| RoundError::Locked(e.msg()))?; + self.settings.passphrase = Some(passphrase.to_string()); + self.keyring = Some((opened, hash)); + let (remote, names, _) = self.latest_remote().await?; + // 之后写的一律不加密 + self.keyring = None; + self.settings.encrypt = false; + self.reupload(remote, &names).await?; + self.store.delete(KEYRING).await?; + Ok(()) + } + + /// 以现在的加密设置重新上传最新版本，删掉别的版本（它们是另一种加密设置下写的） + async fn reupload(&mut self, remote: Remote, names: &[VersionName]) -> Result<(), RoundError> { + let seq = names.iter().map(|n| n.seq).max().unwrap_or(0) + 1; + let v = self + .publish(&remote.snap, seq, &remote.heads, remote.conflicts.clone()) + .await?; + for n in names { + let _ = self.store.delete(&n.path()).await; + self.cache.remove(n); + } + let mut st = self.local.state(); + self.save_base(&v, &mut st)?; + st.last_synced_ms = Some(now_ms()); + self.local.save_state(&st).map_err(RoundError::Local)?; + self.write_device(false).await?; + self.quiet = None; + Ok(()) + } + + /// 更改同步口令：只重新加密 keyring，已有的版本不变。其他设备下次同步时要输入新口令 + pub async fn change_passphrase(&mut self, old: &str, new: &str) -> Result<(), RoundError> { + let Some(bytes) = self.store.get(KEYRING).await? else { + return Err(RoundError::Local(msg!( + "sync.not_encrypted" => "the sync location is not encrypted" + ))); + }; + let (old, new2) = (old.to_string(), new.to_string()); + let sealed = tokio::task::spawn_blocking(move || { + Keyring::open(&bytes, &old).map(|k| (k.seal(&new2), k)) + }) + .await + .map_err(local_err)? + .map_err(|e| RoundError::Locked(e.msg()))?; + let (sealed, k) = sealed; + self.store.put(KEYRING, &sealed).await?; + let hash: [u8; 32] = Sha256::digest(&sealed).into(); + self.keyring = Some((k, hash)); + self.settings.passphrase = Some(new.to_string()); + Ok(()) + } + + /// 口令对不对得上同步位置的 keyring（别的设备改了口令之后重新输入）。没加密是 `Ok(false)` + pub async fn unlock(&mut self, passphrase: &str) -> Result { + let Some(bytes) = self.store.get(KEYRING).await? else { + return Ok(false); + }; + let pass = passphrase.to_string(); + let k = tokio::task::spawn_blocking(move || Keyring::open(&bytes, &pass)) + .await + .map_err(local_err)?; + match k { + Ok(_) => { + self.settings.passphrase = Some(passphrase.to_string()); + self.settings.encrypt = true; + self.keyring = None; + self.quiet = None; + Ok(true) + } + Err(e) => Err(RoundError::Locked(e.msg())), + } + } + + // ─────────────────────────────────────────────── 停止 + + /// 停止同步：从同步位置移除本设备的设备记录；`purge` 时连全部版本、设备记录和 keyring 一起 + /// 删掉（其他设备随后发现同步位置空了，停止同步）。本机的同步状态目录删掉 + pub async fn stop(&mut self, purge: bool) -> Result<(), RoundError> { + let mine = self + .store + .delete(&layout::device_path(&self.settings.device_id)) + .await; + if purge { + for e in self.store.list(VERSIONS).await? { + self.store.delete(&format!("{VERSIONS}/{}", e.name)).await?; + } + for e in self.store.list(DEVICES).await? { + self.store.delete(&format!("{DEVICES}/{}", e.name)).await?; + } + self.store.delete(KEYRING).await?; + } else { + mine?; + } + self.local.clear(); + self.cache.clear(); + self.quiet = None; + Ok(()) + } + + // ─────────────────────────────────────────────── 同步位置上的备份 + + /// 同步位置上读得出来的版本（最新与上一个）：给备份列表 + pub async fn remote_versions(&mut self) -> Result>, RoundError> { + self.keyring().await?; + let (_, ok, _) = self.versions().await?; + Ok(ok) + } + + /// 同步位置上内容标识是这个的版本 + pub async fn remote_version(&mut self, content_id: &str) -> Result, RoundError> { + Ok(self + .remote_versions() + .await? + .into_iter() + .find(|v| v.content == content_id) + .map(|v| v.pack.clone())) + } + + /// 同步位置空着：以本机配置上传第一个版本（加密时先建 keyring） + async fn publish_first(&mut self) -> Result { + let cur = self.host.current().map_err(RoundError::Local)?; + let snap = model::snapshot(&cur.pack).map_err(|e| RoundError::Local(e.msg()))?; + let mut shared = snap.clone(); + let ex = excluded(&self.settings.exclude, &[&snap]); + shared.items.retain(|id, _| !ex.contains(id)); + for (o, k) in ORDERS { + let names = names_of(&shared, *k); + let list: Vec = order_list(Some(&snap), *o) + .unwrap_or_default() + .into_iter() + .filter(|n| names.contains(n)) + .collect(); + set_order(&mut shared, *o, list); + } + shared.relink(); + let v = self.publish(&shared, 1, &[], Vec::new()).await?; + let mut st = self.local.state(); + st.place.clone_from(&self.settings.place); + self.save_base(&v, &mut st)?; + st.last_synced_ms = Some(now_ms()); + self.local.save_state(&st).map_err(RoundError::Local)?; + self.write_device(false).await?; + Ok(v) + } +} diff --git a/crates/tw-sync/src/keyring.rs b/crates/tw-sync/src/keyring.rs new file mode 100644 index 00000000..b17f5d78 --- /dev/null +++ b/crates/tw-sync/src/keyring.rs @@ -0,0 +1,225 @@ +//! 加密同步的版本密钥。 +//! +//! 首台设备生成一把随机的 age x25519 密钥，**以同步口令加密**（age 的口令模式，scrypt 派生） +//! 后存成同步位置上的 `keyring`。各版本、各设备记录都加密给这把密钥的公钥。于是： +//! +//! - 改口令只重新加密 keyring，已有的版本不用动； +//! - 没有口令的一方解不开 keyring，也就既读不了版本、也写不出别的设备认的版本（age 的认证 +//! 加密，见 `tw_pack`）。 +//! +//! scrypt 的强度是**写死的**（[`WORK_FACTOR`]），不按这台机器的速度现定：一台快的 Mac 按自己 +//! 定的强度加密，慢的服务器可能解不开（age 拒绝超过本机上限的强度）。 + +use std::sync::atomic::{AtomicU8, Ordering}; + +use age::secrecy::{ExposeSecret, SecretString}; +use serde::{Deserialize, Serialize}; +use tw_types::{Msg, msg}; + +/// scrypt 的 N = 2^17：一次约 128 MiB 内存、零点几秒。只在打开 keyring 时算一次 +pub const WORK_FACTOR: u8 = 17; + +/// 解 keyring 时接受的最高强度。**比写的那个只高一点**：对同步位置有写权限的人能放一个 +/// 强度极高的 keyring 进来，让每台设备一打开就耗尽内存 +const MAX_WORK_FACTOR: u8 = WORK_FACTOR + 1; + +static WORK: AtomicU8 = AtomicU8::new(WORK_FACTOR); + +/// **只给测试用**：把 scrypt 的强度调低，一条测试不必每次都算零点几秒。解的时候照样认 +pub fn set_work_factor_for_tests(log_n: u8) { + WORK.store(log_n.clamp(1, WORK_FACTOR), Ordering::Relaxed); +} + +/// keyring 解开之后的内容 +#[derive(Serialize, Deserialize)] +struct Plain { + /// 格式版本 + v: u32, + /// `AGE-SECRET-KEY-1…` + identity: String, +} + +/// 一把解开了的版本密钥。**Debug 不打印它** +pub struct Keyring { + identity: age::x25519::Identity, +} + +impl std::fmt::Debug for Keyring { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Keyring") + .field("recipient", &self.identity.to_public().to_string()) + .finish() + } +} + +#[derive(Debug, thiserror::Error)] +pub enum KeyringError { + /// 口令不对。**和「坏了」分开**：前者要人重新输入（别的设备可能刚改过口令） + #[error("{}", self.msg())] + WrongPassphrase, + #[error("{}", self.msg())] + Corrupt, + /// 格式版本比这个 core 认得的新 + #[error("{}", self.msg())] + TooNew, +} + +impl KeyringError { + pub fn msg(&self) -> Msg { + match self { + KeyringError::WrongPassphrase => msg!( + "sync.passphrase_wrong" => "the sync passphrase is not correct" + ), + KeyringError::Corrupt => msg!( + "sync.keyring_corrupt" => + "the keyring at the sync location cannot be read. It may still be arriving, or \ + it is damaged" + ), + KeyringError::TooNew => msg!( + "sync.keyring_too_new" => + "the keyring at the sync location was written by a newer version. Update \ + ThinkWatch Lite to keep syncing" + ), + } + } +} + +impl Keyring { + /// 新的一把 + pub fn generate() -> Self { + Self { + identity: age::x25519::Identity::generate(), + } + } + + /// 以口令加密，存成 `keyring` 文件的内容 + pub fn seal(&self, passphrase: &str) -> Vec { + let plain = Plain { + v: 1, + identity: self.identity.to_string().expose_secret().to_string(), + }; + let json = serde_json::to_vec(&plain).expect("plain data serializes"); + let mut r = age::scrypt::Recipient::new(SecretString::from(passphrase.to_string())); + r.set_work_factor(WORK.load(Ordering::Relaxed)); + age::encrypt(&r, &json).expect("encrypting to a passphrase does not fail") + } + + /// 用口令解开 `keyring` 文件 + pub fn open(bytes: &[u8], passphrase: &str) -> Result { + let mut id = age::scrypt::Identity::new(SecretString::from(passphrase.to_string())); + id.set_max_work_factor(MAX_WORK_FACTOR); + let json = age::decrypt(&id, bytes).map_err(|e| match e { + age::DecryptError::DecryptionFailed + | age::DecryptError::KeyDecryptionFailed + | age::DecryptError::NoMatchingKeys => KeyringError::WrongPassphrase, + age::DecryptError::UnknownFormat => KeyringError::TooNew, + _ => KeyringError::Corrupt, + })?; + let plain: Plain = serde_json::from_slice(&json).map_err(|_| KeyringError::Corrupt)?; + if plain.v > 1 { + return Err(KeyringError::TooNew); + } + let identity = plain + .identity + .parse::() + .map_err(|_| KeyringError::Corrupt)?; + Ok(Self { identity }) + } + + /// 加密用的公钥：版本、设备记录都加密给它 + pub fn recipient(&self) -> age::x25519::Recipient { + self.identity.to_public() + } + + /// 解密用的私钥 + pub fn identity(&self) -> &age::x25519::Identity { + &self.identity + } + + /// 加密一小段数据（设备记录） + pub fn encrypt(&self, plain: &[u8]) -> Vec { + age::encrypt(&self.recipient(), plain).expect("encrypting to an x25519 key does not fail") + } + + /// 解密一小段数据。解不开（不是这把密钥加密的、被改过、没传完）是 `None` + pub fn decrypt(&self, bytes: &[u8]) -> Option> { + age::decrypt(&self.identity, bytes).ok() + } + + /// 两把是不是同一把（公钥相同） + pub fn same_as(&self, other: &Keyring) -> bool { + self.recipient().to_string() == other.recipient().to_string() + } +} + +/// 一段字节是不是 age 加密过的（看文件头） +pub fn is_age(bytes: &[u8]) -> bool { + bytes.starts_with(b"age-encryption.org/") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_keyring_opens_with_its_passphrase_and_only_with_it() { + set_work_factor_for_tests(4); + let k = Keyring::generate(); + let sealed = k.seal("correct horse battery staple"); + assert!(is_age(&sealed)); + let back = Keyring::open(&sealed, "correct horse battery staple").unwrap(); + assert!(back.same_as(&k)); + assert!(matches!( + Keyring::open(&sealed, "wrong passphrase!!"), + Err(KeyringError::WrongPassphrase) + )); + assert!(matches!( + Keyring::open(b"not a keyring", "x"), + Err(KeyringError::Corrupt) + )); + } + + #[test] + fn changing_the_passphrase_keeps_the_same_key() { + set_work_factor_for_tests(4); + let k = Keyring::generate(); + let sealed = k.seal("old passphrase 1"); + let opened = Keyring::open(&sealed, "old passphrase 1").unwrap(); + let resealed = opened.seal("new passphrase 2"); + let again = Keyring::open(&resealed, "new passphrase 2").unwrap(); + assert!(again.same_as(&k), "改口令只重新加密 keyring，密钥本身不变"); + let data = k.encrypt(b"device record"); + assert_eq!(again.decrypt(&data).unwrap(), b"device record"); + assert!(Keyring::generate().decrypt(&data).is_none()); + } + + #[test] + fn a_keyring_that_demands_too_much_work_is_refused() { + set_work_factor_for_tests(4); + // 按强度 4 加密，再把文件头里写的强度改成 30：age 在算 scrypt 之前就按上限拒绝， + // 测试不用真的去算一遍 2^30 + let sealed = Keyring::generate().seal("p"); + let text = String::from_utf8_lossy(&sealed).to_string(); + let line = text + .lines() + .find(|l| l.starts_with("-> scrypt ")) + .expect("口令模式的文件头有一行 scrypt") + .to_string(); + let tampered = line.trim_end_matches(char::is_numeric).to_string() + "30"; + let mut bytes = sealed.clone(); + let at = text.find(&line).unwrap(); + bytes.splice(at..at + line.len(), tampered.bytes()); + assert!(matches!( + Keyring::open(&bytes, "p"), + Err(KeyringError::Corrupt) + )); + } + + #[test] + fn debug_shows_only_the_public_half() { + let k = Keyring::generate(); + let shown = format!("{k:?}"); + assert!(shown.contains("age1"), "{shown}"); + assert!(!shown.contains("AGE-SECRET-KEY"), "{shown}"); + } +} diff --git a/crates/tw-sync/src/layout.rs b/crates/tw-sync/src/layout.rs new file mode 100644 index 00000000..6f01ac26 --- /dev/null +++ b/crates/tw-sync/src/layout.rs @@ -0,0 +1,259 @@ +//! 同步位置上有什么、叫什么、留哪些。 +//! +//! ```text +//! <同步位置>/ +//! keyring 以同步口令加密的版本密钥（仅加密时存在） +//! versions/ +//! 0000000041-3f9a2c1e.twconfig 序号-设备标识，只新增、不覆盖 +//! 0000000042-3f9a2c1e.twconfig +//! 0000000042-b71d04aa.twconfig 两台设备同时上传，序号相同，下次同步时合并 +//! devices/ +//! 3f9a2c1e 设备记录（名称、Core 版本、最后同步时间） +//! ``` +//! +//! 同步位置能看到的只有文件名（版本序号和设备标识的前 8 位）、大小和修改时间。 + +use std::collections::BTreeSet; + +pub const KEYRING: &str = "keyring"; +pub const VERSIONS: &str = "versions"; +pub const DEVICES: &str = "devices"; +/// 版本文件的扩展名，和导出的配置包一样 +pub const EXT: &str = ".twconfig"; +/// 序号写几位 +const SEQ_DIGITS: usize = 10; + +/// 一个版本的名字：序号和写它的设备（设备标识的前 8 位）。**按序号排，同序号按设备排** +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct VersionName { + pub seq: u64, + pub device: String, +} + +impl VersionName { + pub fn new(seq: u64, device_id: &str) -> Self { + Self { + seq, + device: device_tag(device_id), + } + } + + /// `0000000042-3f9a2c1e`，不带扩展名。版本互相引用（上一版本）时用它 + pub fn id(&self) -> String { + format!("{:0width$}-{}", self.seq, self.device, width = SEQ_DIGITS) + } + + /// 文件名 + pub fn file(&self) -> String { + format!("{}{EXT}", self.id()) + } + + /// 在同步位置里的路径 + pub fn path(&self) -> String { + format!("{VERSIONS}/{}", self.file()) + } + + /// 认文件名（带扩展名）或者版本标识（不带）。别的名字（临时文件、云盘的冲突副本 + /// `… (conflicted copy).twconfig`、手放进去的文件）是 `None` + pub fn parse(name: &str) -> Option { + let id = name.strip_suffix(EXT).unwrap_or(name); + let (seq, dev) = id.split_once('-')?; + if seq.len() != SEQ_DIGITS || !seq.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + if dev.len() != tw_config::sync::DEVICE_ID_PREFIX + || !dev + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + { + return None; + } + Some(Self { + seq: seq.parse().ok()?, + device: dev.to_string(), + }) + } +} + +impl std::fmt::Display for VersionName { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.id()) + } +} + +/// 设备标识的前 8 位：文件名、设备记录名里用它 +pub fn device_tag(device_id: &str) -> String { + device_id + .chars() + .take(tw_config::sync::DEVICE_ID_PREFIX) + .collect() +} + +/// 设备记录的路径 +pub fn device_path(device_id: &str) -> String { + format!("{DEVICES}/{}", device_tag(device_id)) +} + +/// 同步位置上的一个版本：名字，以及它说自己依据的上一版本（读出来之后才知道） +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Listed { + pub name: VersionName, + pub parents: Vec, +} + +/// 上传新版本之后要删掉哪些。 +/// +/// **保留最新版本与上一个版本**：每一个「头」（没有别的版本依据它的版本 —— 通常只有最新的 +/// 那一个），加上它们依据的上一版本。两台设备同时上传、还没合并时，头有两个，它们和它们 +/// 共同的上一版本都留着，等下一台设备合并：合并要用那个上一版本当基准。晚到的版本（云盘 +/// 后传到的）也是头，**不会因为序号小就被删掉**。别的都删。 +/// +/// `listed` 里的 `parents` 只有读出来了的版本才有。读不出来的（尚未到达、坏了）当作头留着； +/// 最新那一层有读不出来的时，序号紧挨着的那一层也留着 +pub fn to_delete(listed: &[Listed]) -> Vec { + let Some(top) = listed.iter().map(|l| l.name.seq).max() else { + return Vec::new(); + }; + let referenced: BTreeSet<&VersionName> = listed.iter().flat_map(|l| &l.parents).collect(); + let heads: Vec<&Listed> = listed + .iter() + .filter(|l| !referenced.contains(&l.name)) + .collect(); + let mut keep: BTreeSet = heads.iter().map(|l| l.name.clone()).collect(); + for l in &heads { + keep.extend(l.parents.iter().cloned()); + } + // 不知道上一版本是谁（读不出来）时，至少留住序号紧挨着的那一层 + if listed + .iter() + .any(|l| l.name.seq == top && l.parents.is_empty()) + && let Some(prev) = listed.iter().map(|l| l.name.seq).filter(|s| *s < top).max() + { + keep.extend( + listed + .iter() + .filter(|l| l.name.seq == prev) + .map(|l| l.name.clone()), + ); + } + listed + .iter() + .map(|l| l.name.clone()) + .filter(|n| !keep.contains(n)) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn v(seq: u64, dev: &str) -> VersionName { + VersionName { + seq, + device: dev.into(), + } + } + + fn l(seq: u64, dev: &str, parents: &[(u64, &str)]) -> Listed { + Listed { + name: v(seq, dev), + parents: parents.iter().map(|(s, d)| v(*s, d)).collect(), + } + } + + #[test] + fn names_round_trip_and_strangers_are_ignored() { + let n = VersionName::new(42, "3f9a2c1e-0b1d-4e2f-9a3b-5c6d7e8f9a0b"); + assert_eq!(n.file(), "0000000042-3f9a2c1e.twconfig"); + assert_eq!(n.path(), "versions/0000000042-3f9a2c1e.twconfig"); + assert_eq!(VersionName::parse(&n.file()), Some(n.clone())); + assert_eq!(VersionName::parse(&n.id()), Some(n)); + for stranger in [ + "0000000042-3f9a2c1e (conflicted copy).twconfig", + "42-3f9a2c1e.twconfig", + "0000000042-3F9A2C1E.twconfig", + "0000000042-3f9a2c.twconfig", + "notes.txt", + ".tmp-1-0000000042-3f9a2c1e.twconfig", + ] { + assert_eq!(VersionName::parse(stranger), None, "{stranger}"); + } + } + + #[test] + fn versions_sort_by_sequence_then_device() { + let mut all = vec![v(10, "bbbbbbbb"), v(9, "cccccccc"), v(10, "aaaaaaaa")]; + all.sort(); + assert_eq!( + all, + [v(9, "cccccccc"), v(10, "aaaaaaaa"), v(10, "bbbbbbbb")] + ); + } + + #[test] + fn the_latest_and_the_one_before_it_are_kept() { + let listed = [ + l(1, "aaaaaaaa", &[]), + l(2, "aaaaaaaa", &[(1, "aaaaaaaa")]), + l(3, "bbbbbbbb", &[(2, "aaaaaaaa")]), + ]; + assert_eq!(to_delete(&listed), [v(1, "aaaaaaaa")]); + } + + #[test] + fn unmerged_versions_keep_their_common_parent_until_someone_merges_them() { + let listed = [ + l(4, "aaaaaaaa", &[(3, "aaaaaaaa")]), + l(3, "aaaaaaaa", &[(2, "aaaaaaaa")]), + l(5, "aaaaaaaa", &[(4, "aaaaaaaa")]), + l(5, "bbbbbbbb", &[(4, "aaaaaaaa")]), + ]; + assert_eq!(to_delete(&listed), [v(3, "aaaaaaaa")]); + // 合并之后：新版本依据那两个，它们留着，共同的上一版本可以删了 + let merged = [ + l(4, "aaaaaaaa", &[(3, "aaaaaaaa")]), + l(5, "aaaaaaaa", &[(4, "aaaaaaaa")]), + l(5, "bbbbbbbb", &[(4, "aaaaaaaa")]), + l(6, "cccccccc", &[(5, "aaaaaaaa"), (5, "bbbbbbbb")]), + ]; + assert_eq!(to_delete(&merged), [v(4, "aaaaaaaa")]); + } + + /// 云盘后传到的版本：序号比最新的小，但没有谁依据它 —— 它是一个还没合并的头，留着 + #[test] + fn a_late_arrival_is_a_head_and_is_kept_with_its_parent() { + let listed = [ + l(4, "aaaaaaaa", &[(3, "aaaaaaaa")]), + l(5, "aaaaaaaa", &[(4, "aaaaaaaa")]), + l(6, "aaaaaaaa", &[(5, "aaaaaaaa")]), + l(5, "bbbbbbbb", &[(4, "aaaaaaaa")]), + ]; + assert!(to_delete(&listed).is_empty(), "{:?}", to_delete(&listed)); + } + + #[test] + fn a_version_that_has_not_arrived_still_keeps_the_layer_below() { + // 3 读不出来，不知道它依据谁：2 也还是头，2 和它依据的 1 都留着 + let listed = [ + l(1, "aaaaaaaa", &[]), + l(2, "aaaaaaaa", &[(1, "aaaaaaaa")]), + l(3, "bbbbbbbb", &[]), + ]; + assert!(to_delete(&listed).is_empty()); + // 3 读出来了、依据 2：1 可以删了 + let listed = [ + l(1, "aaaaaaaa", &[]), + l(2, "aaaaaaaa", &[(1, "aaaaaaaa")]), + l(3, "bbbbbbbb", &[(2, "aaaaaaaa")]), + ]; + assert_eq!(to_delete(&listed), [v(1, "aaaaaaaa")]); + // 都读不出来：最新的和紧挨着的那一层留着 + let listed = [ + l(1, "aaaaaaaa", &[]), + l(2, "aaaaaaaa", &[]), + l(3, "bbbbbbbb", &[]), + ]; + assert_eq!(to_delete(&listed), Vec::::new()); + assert!(to_delete(&[]).is_empty()); + } +} diff --git a/crates/tw-sync/src/lib.rs b/crates/tw-sync/src/lib.rs new file mode 100644 index 00000000..e4e9d72c --- /dev/null +++ b/crates/tw-sync/src/lib.rs @@ -0,0 +1,24 @@ +//! 多设备配置同步。 +//! +//! 多台设备（本机 Core、远程 core）通过用户自选的同步位置共用一份配置： +//! +//! - **同步位置只追加新版本、不覆盖已有文件**（[`layout`]）。不依赖存储服务的条件写入，文件夹、 +//! WebDAV、S3 是同一套机制（[`store`]）； +//! - **按对象三方合并**，以共同的上一版本为基准（[`merge`]）。不同设备改不同对象、同一对象的 +//! 不同部分时自动合并，只有同一处被改成不同内容才是冲突； +//! - 内容可选加密：一把随机的版本密钥，以同步口令加密后存在同步位置上（[`keyring`]）； +//! - 每台设备在同步位置上有一份设备记录（[`device`]）。 +//! +//! 这里不碰控制面：调度、写配置、事件在 `tw-control` 的 `sync` 模块。 + +pub mod device; +pub mod engine; +pub mod keyring; +pub mod layout; +pub mod local; +pub mod merge; +pub mod order; +pub mod record; +pub mod store; + +pub use store::{AnyStore, Entry, Store, StoreError}; diff --git a/crates/tw-sync/src/local.rs b/crates/tw-sync/src/local.rs new file mode 100644 index 00000000..4d8eac11 --- /dev/null +++ b/crates/tw-sync/src/local.rs @@ -0,0 +1,207 @@ +//! 这台设备自己记着的同步状态：数据目录里的 `sync/`。 +//! +//! ```text +//! sync/ +//! base.twconfig 上次同步的那一版（合并基准），不加密，不计入本机保留的 10 个版本 +//! state.json 上次同步的时间、失败从什么时候开始、因同步停用的插件、地址经同步变更的上游 +//! ``` +//! +//! 同步位置上的版本被删掉（设备离线期间）不影响合并：基准在这里。停止同步时整个目录删掉。 + +use std::collections::BTreeMap; +use std::io::Write; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use tw_types::{Msg, msg}; + +pub const DIR: &str = "sync"; +const BASE: &str = "base.twconfig"; +const STATE: &str = "state.json"; + +/// `state.json` +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct State { + /// 这份状态是对着哪个同步位置的（[`place`]）。换了位置就作废 + pub place: String, + /// 基准是哪一版（版本标识） + pub base: Option, + pub last_synced_ms: Option, + /// 从这个时刻起一直没同步成（连续失败）。成功一次就清掉 + pub failing_since_ms: Option, + /// 最近一次失败的原因 + pub error: Option, + /// 不加密时经同步新增、或者脚本变了的插件，在这台设备上停用着：插件 id → 停用时的脚本 + /// （合并单位的值）。在本机启用之后去掉 + pub held_plugins: BTreeMap, + /// 不加密时经同步改了地址的上游：名字 → 改之前和改之后的地址 + pub url_changed: BTreeMap, + /// 上次写自己的设备记录是什么时候 + pub device_record_ms: u64, + /// 上次写设备记录时是不是暂停着（版本过新） + pub device_record_paused: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct UrlChange { + pub from: String, + pub to: String, + /// 改它的设备的名称 + #[serde(default)] + pub device: Option, + #[serde(default)] + pub at_ms: u64, +} + +/// 一个同步位置的指纹：种类加地址，不含凭据（换了密码还是同一个位置） +pub fn place(target: &tw_config::sync::Target) -> String { + use tw_config::sync::Target; + match target { + Target::Folder { path, .. } => format!("folder:{}", path.trim_end_matches(['/', '\\'])), + Target::Webdav { url, .. } => format!("webdav:{}", url.trim_end_matches('/')), + Target::S3 { + endpoint, + bucket, + prefix, + .. + } => format!( + "s3:{}/{bucket}/{}", + endpoint.trim_end_matches('/'), + prefix.trim_matches('/') + ), + } +} + +fn io(path: &Path, e: std::io::Error) -> Msg { + msg!( + "sync.local.io", path = path.display(), detail = e => + "{path} could not be used: {detail}" + ) +} + +/// 原子写一个只有本机用户读得到的文件 +fn write_private(path: &Path, bytes: &[u8]) -> Result<(), Msg> { + let dir = path.parent().unwrap_or(Path::new(".")); + tw_config::private_dir::create(dir).map_err(|e| io(dir, e))?; + let r: u64 = rand::random(); + let tmp = dir.join(format!(".tmp-{r:016x}")); + let mut opts = std::fs::OpenOptions::new(); + opts.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + opts.mode(0o600); + } + let res = opts + .open(&tmp) + .and_then(|mut f| f.write_all(bytes).and_then(|()| f.sync_all())) + .and_then(|()| std::fs::rename(&tmp, path)); + if let Err(e) = res { + let _ = std::fs::remove_file(&tmp); + return Err(io(path, e)); + } + Ok(()) +} + +/// 这台设备的同步状态目录 +#[derive(Debug, Clone)] +pub struct Local { + dir: PathBuf, +} + +impl Local { + /// `data_dir` 是配置文件所在的目录 + pub fn new(data_dir: &Path) -> Self { + Self { + dir: data_dir.join(DIR), + } + } + + pub fn dir(&self) -> &Path { + &self.dir + } + + pub fn state(&self) -> State { + std::fs::read(self.dir.join(STATE)) + .ok() + .and_then(|b| serde_json::from_slice(&b).ok()) + .unwrap_or_default() + } + + pub fn save_state(&self, s: &State) -> Result<(), Msg> { + write_private( + &self.dir.join(STATE), + &serde_json::to_vec_pretty(s).expect("plain data serializes"), + ) + } + + /// 基准版本的字节（不加密的配置包）。没有是 `None` + pub fn base(&self) -> Option> { + std::fs::read(self.dir.join(BASE)).ok() + } + + pub fn save_base(&self, bytes: &[u8]) -> Result<(), Msg> { + write_private(&self.dir.join(BASE), bytes) + } + + /// 停止同步：整个目录删掉 + pub fn clear(&self) { + let _ = std::fs::remove_dir_all(&self.dir); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn state_and_base_survive_a_restart_and_are_private() { + let d = tempfile::tempdir().unwrap(); + let l = Local::new(d.path()); + assert_eq!(l.state(), State::default()); + assert!(l.base().is_none()); + let s = State { + place: "folder:/x".into(), + base: Some("0000000001-aaaaaaaa".into()), + last_synced_ms: Some(5), + ..Default::default() + }; + l.save_state(&s).unwrap(); + l.save_base(b"PK").unwrap(); + let again = Local::new(d.path()); + assert_eq!(again.state(), s); + assert_eq!(again.base().as_deref(), Some(&b"PK"[..])); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(d.path().join("sync/base.twconfig")) + .unwrap() + .permissions() + .mode() + & 0o777; + assert_eq!(mode, 0o600); + } + again.clear(); + assert!(!d.path().join("sync").exists()); + } + + #[test] + fn the_place_ignores_credentials() { + use tw_config::sync::Target; + let a = Target::Webdav { + url: "https://h/dav/".into(), + user: "a".into(), + pass: "1".into(), + proxy: "direct".into(), + }; + let b = Target::Webdav { + url: "https://h/dav".into(), + user: "b".into(), + pass: "2".into(), + proxy: "corp".into(), + }; + assert_eq!(place(&a), place(&b)); + } +} diff --git a/crates/tw-sync/src/merge.rs b/crates/tw-sync/src/merge.rs new file mode 100644 index 00000000..0c215a59 --- /dev/null +++ b/crates/tw-sync/src/merge.rs @@ -0,0 +1,583 @@ +//! 三方合并：以共同的上一版本为基准，本机当前配置和同步位置的最新版本按**合并单位**逐一比较。 +//! +//! 一个对象（上游、路由、插件……）拆成几个合并单位（上游是连接、模型、计费、其他四组；全局 +//! 设置逐个字段；别的对象整个一个单位），拆法由 `tw_pack::model` 定。这里只认「对象 → 单位名 +//! → 规范化的值」，所以和配置长什么样无关，表驱动测试直接写值。 +//! +//! | 本机相对基准 | 同步位置相对基准 | 结果 | +//! |---|---|---| +//! | 未变 | 未变 | 不变 | +//! | 已修改或新增 | 未变 | 采用本机 | +//! | 未变 | 已修改或新增 | 采用同步位置 | +//! | 已删除 | 未变 | 删除 | +//! | 未变 | 已删除 | 删除 | +//! | 已修改 | 已修改，内容相同 | 不变 | +//! | 已修改或新增 | 已修改或新增，内容不同 | 冲突 | +//! | 已删除 | 已修改 | 冲突 | +//! | 已修改 | 已删除 | 冲突 | +//! +//! **冲突不阻断合并**：其余单位照常合并。冲突的那几个单位（或者那个被删的对象）两边各留各的 +//! —— 本机的结果里是本机的，要上传的结果里是同步位置上已有的，等人处理。 +//! +//! 没有基准（基准版本哪里都找不到）时按两方比较：两边都有、内容不同的单位是冲突；只有一边 +//! 有的对象当作那一边新增的，不当作另一边删掉的 —— 分不清的时候，留着比删掉好。 + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::Value; + +/// 一个对象：合并单位 → 规范化的值 +pub type Units = BTreeMap; + +/// 一份配置拆成的对象 +pub type Objects = BTreeMap; + +/// 冲突的形状 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Kind { + /// 两边都改了（或者都新增了）同一个单位，内容不同 + Modified, + /// 本机删了，同步位置改了 + LocalDeleted, + /// 本机改了，同步位置删了 + RemoteDeleted, +} + +/// 这次合并新发现的一处冲突 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Found { + pub key: K, + /// 冲突的单位。一边删了对象时是另一边改过的那几个 + pub units: BTreeSet, + pub kind: Kind, +} + +/// 合并的输入 +pub struct Input<'a, K> { + /// 共同的上一版本。`None` 是哪里都找不到，按两方比较 + pub base: Option<&'a Objects>, + pub local: &'a Objects, + pub remote: &'a Objects, + /// 整个对象各留各的，不比较：不同步的上游和随之不同步的项目，以及一边删了对象、还没 + /// 处理的冲突 + pub hold_items: &'a BTreeSet, + /// 这几个单位各留各的：还没处理的冲突 + pub hold_units: &'a BTreeSet<(K, String)>, +} + +/// 合并的结果 +#[derive(Debug, Clone, PartialEq)] +pub struct Outcome { + /// 本机应用的样子 + pub local: Objects, + /// 上传的样子 + pub shared: Objects, + pub conflicts: Vec>, +} + +/// 一个单位在一边的样子：没有这个对象、或者对象里没有这个单位都是 `None` +fn unit<'a>(o: Option<&'a Units>, u: &str) -> Option<&'a Value> { + o.and_then(|m| m.get(u)) +} + +pub fn merge(i: Input<'_, K>) -> Outcome { + let empty = Objects::new(); + let base = i.base.unwrap_or(&empty); + let mut out = Outcome { + local: Objects::new(), + shared: Objects::new(), + conflicts: Vec::new(), + }; + let keys: BTreeSet<&K> = base + .keys() + .chain(i.local.keys()) + .chain(i.remote.keys()) + .collect(); + for k in keys { + let (b, l, r) = (base.get(k), i.local.get(k), i.remote.get(k)); + let mut put = |local: Option<&Units>, shared: Option<&Units>| { + if let Some(v) = local { + out.local.insert(k.clone(), v.clone()); + } + if let Some(v) = shared { + out.shared.insert(k.clone(), v.clone()); + } + }; + if i.hold_items.contains(k) { + put(l, r); + continue; + } + match (b, l, r) { + (_, None, None) => {} + // 有基准，一边删了 + (Some(b), None, Some(r)) => { + if r == b { + // 同步位置没动、本机删了：删 + } else { + put(None, Some(r)); + out.conflicts.push(Found { + key: k.clone(), + units: changed(b, r), + kind: Kind::LocalDeleted, + }); + } + } + (Some(b), Some(l), None) => { + if l == b { + // 本机没动、同步位置删了：删 + } else { + put(Some(l), None); + out.conflicts.push(Found { + key: k.clone(), + units: changed(b, l), + kind: Kind::RemoteDeleted, + }); + } + } + // 没有基准、只有一边有：那一边新增的 + (None, Some(l), None) => put(Some(l), Some(l)), + (None, None, Some(r)) => put(Some(r), Some(r)), + // 两边都有：逐个单位比 + (b, Some(l), Some(r)) => { + let names: BTreeSet<&String> = b + .into_iter() + .flat_map(|m| m.keys()) + .chain(l.keys()) + .chain(r.keys()) + .collect(); + let (mut lo, mut sh) = (Units::new(), Units::new()); + let mut clash = BTreeSet::new(); + for u in names { + let (bv, lv, rv) = (unit(b, u), l.get(u), r.get(u)); + let mut set = |lo_v: Option<&Value>, sh_v: Option<&Value>| { + if let Some(v) = lo_v { + lo.insert(u.clone(), v.clone()); + } + if let Some(v) = sh_v { + sh.insert(u.clone(), v.clone()); + } + }; + if i.hold_units.contains(&(k.clone(), u.clone())) { + set(lv, rv); + } else if lv == rv || rv == bv { + // 两边一样，或者同步位置没动：本机的 + set(lv, lv); + } else if lv == bv { + // 本机没动：同步位置的 + set(rv, rv); + } else { + set(lv, rv); + clash.insert(u.clone()); + } + } + put(Some(&lo), Some(&sh)); + if !clash.is_empty() { + out.conflicts.push(Found { + key: k.clone(), + units: clash, + kind: Kind::Modified, + }); + } + } + } + } + out +} + +/// 两个样子之间变了的单位 +pub fn changed(a: &Units, b: &Units) -> BTreeSet { + a.keys() + .chain(b.keys()) + .filter(|u| a.get(*u) != b.get(*u)) + .cloned() + .collect() +} + +/// 一份配置相对基准有没有变（不看 `hold_items`、`hold_units` 里的）。决定要不要上传 +pub fn differs( + base: &Objects, + now: &Objects, + hold_items: &BTreeSet, + hold_units: &BTreeSet<(K, String)>, +) -> bool { + let keys: BTreeSet<&K> = base.keys().chain(now.keys()).collect(); + keys.into_iter().any(|k| { + if hold_items.contains(k) { + return false; + } + match (base.get(k), now.get(k)) { + (None, None) => false, + (Some(_), None) | (None, Some(_)) => true, + (Some(a), Some(b)) => changed(a, b) + .iter() + .any(|u| !hold_units.contains(&(k.clone(), u.clone()))), + } + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + /// 一个对象：单位名和值成对写 + fn obj(units: &[(&str, Value)]) -> Units { + units + .iter() + .map(|(u, v)| (u.to_string(), v.clone())) + .collect() + } + + fn objs(items: &[(&str, Option)]) -> Objects { + items + .iter() + .filter_map(|(k, v)| v.clone().map(|v| (k.to_string(), v))) + .collect() + } + + fn run( + base: Option>, + local: Option, + remote: Option, + ) -> Outcome { + let b = base.map(|b| objs(&[("p", b)])); + let l = objs(&[("p", local)]); + let r = objs(&[("p", remote)]); + merge(Input { + base: b.as_ref(), + local: &l, + remote: &r, + hold_items: &BTreeSet::new(), + hold_units: &BTreeSet::new(), + }) + } + + /// 方案 6.6 的每一行，在一个合并单位上，外加两边都删、两边都新增、没有基准。 + /// + /// 每一行：基准、本机、同步位置（`None` 是没有这个对象），期望本机的结果、上传的结果、 + /// 冲突的形状。 + #[test] + fn every_row_of_the_merge_table() { + let a = || Some(obj(&[("conn", json!("A"))])); + let b2 = || Some(obj(&[("conn", json!("B"))])); + let c = || Some(obj(&[("conn", json!("C"))])); + type Row = ( + &'static str, + Option>, + Option, + Option, + Option, + Option, + Option, + ); + let rows: Vec = vec![ + ("未变 / 未变 → 不变", Some(a()), a(), a(), a(), a(), None), + ( + "本机修改 / 未变 → 采用本机", + Some(a()), + b2(), + a(), + b2(), + b2(), + None, + ), + ( + "本机新增 / 未变 → 采用本机", + Some(None), + b2(), + None, + b2(), + b2(), + None, + ), + ( + "未变 / 同步位置修改 → 采用同步位置", + Some(a()), + a(), + b2(), + b2(), + b2(), + None, + ), + ( + "未变 / 同步位置新增 → 采用同步位置", + Some(None), + None, + b2(), + b2(), + b2(), + None, + ), + ( + "本机删除 / 未变 → 删除", + Some(a()), + None, + a(), + None, + None, + None, + ), + ( + "未变 / 同步位置删除 → 删除", + Some(a()), + a(), + None, + None, + None, + None, + ), + ("两边都删 → 删除", Some(a()), None, None, None, None, None), + ( + "两边改成一样 → 不变", + Some(a()), + b2(), + b2(), + b2(), + b2(), + None, + ), + ( + "两边新增成一样 → 不变", + Some(None), + b2(), + b2(), + b2(), + b2(), + None, + ), + ( + "两边改得不一样 → 冲突", + Some(a()), + b2(), + c(), + b2(), + c(), + Some(Kind::Modified), + ), + ( + "两边新增得不一样 → 冲突", + Some(None), + b2(), + c(), + b2(), + c(), + Some(Kind::Modified), + ), + ( + "本机删除 / 同步位置修改 → 冲突", + Some(a()), + None, + b2(), + None, + b2(), + Some(Kind::LocalDeleted), + ), + ( + "本机修改 / 同步位置删除 → 冲突", + Some(a()), + b2(), + None, + b2(), + None, + Some(Kind::RemoteDeleted), + ), + ("没有基准，一样 → 不变", None, a(), a(), a(), a(), None), + ( + "没有基准，不一样 → 冲突", + None, + a(), + b2(), + a(), + b2(), + Some(Kind::Modified), + ), + ( + "没有基准，只有本机有 → 留着", + None, + a(), + None, + a(), + a(), + None, + ), + ( + "没有基准，只有同步位置有 → 拿来", + None, + None, + a(), + a(), + a(), + None, + ), + ]; + for (what, base, local, remote, want_local, want_shared, want_conflict) in rows { + let o = run(base, local, remote); + assert_eq!(o.local.get("p").cloned(), want_local, "{what}：本机"); + assert_eq!(o.shared.get("p").cloned(), want_shared, "{what}：上传"); + assert_eq!( + o.conflicts.first().map(|c| c.kind), + want_conflict, + "{what}：冲突" + ); + if want_conflict.is_some() { + assert_eq!(o.conflicts.len(), 1, "{what}"); + assert_eq!(o.conflicts[0].key, "p", "{what}"); + assert_eq!( + o.conflicts[0].units, + BTreeSet::from(["conn".to_string()]), + "{what}" + ); + } + } + } + + /// 同一个上游，两边改的是不同的单位：自动合并，没有冲突 + #[test] + fn changes_to_different_units_of_one_object_merge_by_themselves() { + let base = obj(&[ + ("conn", json!("A")), + ("models", json!(["m1"])), + ("billing", json!("x")), + ]); + let local = obj(&[ + ("conn", json!("A2")), + ("models", json!(["m1"])), + ("billing", json!("x")), + ]); + let remote = obj(&[ + ("conn", json!("A")), + ("models", json!(["m1", "m2"])), + ("billing", json!("x")), + ]); + let o = run(Some(Some(base)), Some(local), Some(remote)); + let want = obj(&[ + ("conn", json!("A2")), + ("models", json!(["m1", "m2"])), + ("billing", json!("x")), + ]); + assert!(o.conflicts.is_empty(), "{:?}", o.conflicts); + assert_eq!(o.local["p"], want); + assert_eq!(o.shared["p"], want); + } + + /// 一个单位冲突，其余单位照常合并；冲突的那一个本机留本机的、上传留同步位置的 + #[test] + fn a_conflict_on_one_unit_does_not_hold_back_the_others() { + let base = obj(&[("conn", json!("A")), ("models", json!(1))]); + let local = obj(&[("conn", json!("L")), ("models", json!(1))]); + let remote = obj(&[("conn", json!("R")), ("models", json!(2))]); + let o = run(Some(Some(base)), Some(local), Some(remote)); + assert_eq!( + o.local["p"], + obj(&[("conn", json!("L")), ("models", json!(2))]) + ); + assert_eq!( + o.shared["p"], + obj(&[("conn", json!("R")), ("models", json!(2))]) + ); + assert_eq!(o.conflicts.len(), 1); + assert_eq!(o.conflicts[0].units, BTreeSet::from(["conn".to_string()])); + } + + /// 删了的一边对改了的一边：冲突列出的是改了的那几个单位 + #[test] + fn a_delete_against_an_edit_names_the_edited_units() { + let base = obj(&[ + ("conn", json!("A")), + ("models", json!(1)), + ("other", json!(0)), + ]); + let remote = obj(&[ + ("conn", json!("A")), + ("models", json!(2)), + ("other", json!(5)), + ]); + let o = run(Some(Some(base)), None, Some(remote.clone())); + assert_eq!(o.local.get("p"), None); + assert_eq!(o.shared.get("p"), Some(&remote)); + assert_eq!( + o.conflicts[0].units, + BTreeSet::from(["models".to_string(), "other".to_string()]) + ); + } + + /// 不同步的项目、还没处理的冲突：两边各留各的，不比较，也不再报一次冲突 + #[test] + fn held_items_and_units_pass_through_on_each_side() { + let base = objs(&[ + ("ollama", Some(obj(&[("conn", json!("base"))]))), + ( + "relay", + Some(obj(&[("conn", json!("A")), ("models", json!(1))])), + ), + ]); + let local = objs(&[ + ("ollama", Some(obj(&[("conn", json!("local"))]))), + ( + "relay", + Some(obj(&[("conn", json!("L")), ("models", json!(1))])), + ), + ]); + let remote = objs(&[ + ("ollama", Some(obj(&[("conn", json!("remote"))]))), + ( + "relay", + Some(obj(&[("conn", json!("R")), ("models", json!(2))])), + ), + ]); + let hold_items = BTreeSet::from(["ollama".to_string()]); + let hold_units = BTreeSet::from([("relay".to_string(), "conn".to_string())]); + let o = merge(Input { + base: Some(&base), + local: &local, + remote: &remote, + hold_items: &hold_items, + hold_units: &hold_units, + }); + assert!(o.conflicts.is_empty(), "{:?}", o.conflicts); + assert_eq!(o.local["ollama"], obj(&[("conn", json!("local"))])); + assert_eq!(o.shared["ollama"], obj(&[("conn", json!("remote"))])); + assert_eq!( + o.local["relay"], + obj(&[("conn", json!("L")), ("models", json!(2))]) + ); + assert_eq!( + o.shared["relay"], + obj(&[("conn", json!("R")), ("models", json!(2))]) + ); + // 只有本机有、不同步的：不上传 + let local_only = objs(&[("ollama", Some(obj(&[("conn", json!("local"))])))]); + let o = merge(Input { + base: None, + local: &local_only, + remote: &Objects::new(), + hold_items: &hold_items, + hold_units: &BTreeSet::new(), + }); + assert!(o.shared.is_empty()); + assert_eq!(o.local.len(), 1); + } + + #[test] + fn whether_there_is_anything_to_upload() { + let base = objs(&[("a", Some(obj(&[("conn", json!(1)), ("models", json!(1))])))]); + let same = base.clone(); + let none = BTreeSet::new(); + assert!(!differs(&base, &same, &none, &BTreeSet::new())); + let edited = objs(&[("a", Some(obj(&[("conn", json!(2)), ("models", json!(1))])))]); + assert!(differs(&base, &edited, &none, &BTreeSet::new())); + // 改的正是还没处理的冲突那一个单位：不算 + let held = BTreeSet::from([("a".to_string(), "conn".to_string())]); + assert!(!differs(&base, &edited, &none, &held)); + // 新增了一个不同步的上游：不算 + let mut more = base.clone(); + more.insert("ollama".into(), obj(&[("conn", json!(1))])); + assert!(differs(&base, &more, &none, &BTreeSet::new())); + assert!(!differs( + &base, + &more, + &BTreeSet::from(["ollama".to_string()]), + &BTreeSet::new() + )); + } +} diff --git a/crates/tw-sync/src/order.rs b/crates/tw-sync/src/order.rs new file mode 100644 index 00000000..82e7ae4e --- /dev/null +++ b/crates/tw-sync/src/order.rs @@ -0,0 +1,192 @@ +//! 上游、路由、插件的先后顺序：**内容和顺序分开合并**。 +//! +//! 一边新增了一条路由、另一边新增了另一条，两边的顺序都变了 —— 这不该是冲突。所以顺序不按 +//! 单位整个比，而是： +//! +//! 1. 两边都有、基准里也有的那些名字，比它们的相对先后。只有一边改了就用那一边的；两边改得 +//! 一样也行；两边改得不一样是冲突（本机留本机的，上传的留同步位置的）。 +//! 2. 新来的名字插在它在自己那一边的前一个名字后面。 +//! 3. 最后只留下合并之后还在的那些项目。 + +use std::collections::BTreeSet; + +/// 顺序合并的结果 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Merged { + pub local: Vec, + pub shared: Vec, + /// 两边把同一批名字排成了不同的先后 + pub conflict: bool, +} + +fn restrict(list: &[String], keep: &BTreeSet<&str>) -> Vec { + list.iter() + .filter(|n| keep.contains(n.as_str())) + .cloned() + .collect() +} + +/// `primary` 打底，把 `present` 里还没排上的名字按 `sources`（先看的在前）里各自的前一个 +/// 名字插进去，最后只留 `present` 里有的 +fn arrange(primary: &[String], sources: &[&[String]], present: &BTreeSet) -> Vec { + let mut out: Vec = primary + .iter() + .filter(|n| present.contains(*n)) + .cloned() + .collect(); + for src in sources { + for (i, n) in src.iter().enumerate() { + if !present.contains(n) || out.contains(n) { + continue; + } + // 它在这一边的前一个、已经排上了的名字 + let after = src[..i].iter().rev().find(|p| out.contains(*p)); + match after { + Some(p) => { + let at = out.iter().position(|x| x == p).map_or(out.len(), |i| i + 1); + out.insert(at, n.clone()); + } + None => out.insert(0, n.clone()), + } + } + } + // 哪一边的顺序里都没有的（不该有）：接在后面 + for n in present { + if !out.contains(n) { + out.push(n.clone()); + } + } + out +} + +/// 合并一种顺序。`local_names`、`shared_names` 是合并之后两份结果里各自还在的那些项目 +pub fn merge( + base: Option<&[String]>, + local: &[String], + remote: &[String], + local_names: &BTreeSet, + shared_names: &BTreeSet, + hold: bool, +) -> Merged { + let base = base.unwrap_or(&[]); + let common: BTreeSet<&str> = local + .iter() + .filter(|n| remote.contains(n) && (base.is_empty() || base.contains(n))) + .map(String::as_str) + .collect(); + let (b, l, r) = ( + restrict(base, &common), + restrict(local, &common), + restrict(remote, &common), + ); + let (l_changed, r_changed) = (!base.is_empty() && l != b, !base.is_empty() && r != b); + let no_base_disagree = base.is_empty() && l != r; + let conflict = !hold && ((l_changed && r_changed && l != r) || no_base_disagree); + let (local_primary, shared_primary): (&[String], &[String]) = if hold || conflict { + (local, remote) + } else if l_changed && !r_changed { + (local, local) + } else { + (remote, remote) + }; + Merged { + local: arrange(local_primary, &[local, remote], local_names), + shared: arrange(shared_primary, &[remote, local], shared_names), + conflict, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn v(names: &[&str]) -> Vec { + names.iter().map(|s| s.to_string()).collect() + } + fn set(names: &[&str]) -> BTreeSet { + names.iter().map(|s| s.to_string()).collect() + } + + #[test] + fn two_sides_adding_different_items_is_not_a_conflict() { + let m = merge( + Some(&v(&["a", "b"])), + &v(&["a", "b", "l"]), + &v(&["r", "a", "b"]), + &set(&["a", "b", "l", "r"]), + &set(&["a", "b", "l", "r"]), + false, + ); + assert!(!m.conflict); + assert_eq!(m.local, v(&["r", "a", "b", "l"])); + assert_eq!(m.shared, m.local); + } + + #[test] + fn one_side_reordering_wins_and_both_reordering_differently_conflicts() { + let all = set(&["a", "b", "c"]); + let m = merge( + Some(&v(&["a", "b", "c"])), + &v(&["c", "a", "b"]), + &v(&["a", "b", "c"]), + &all, + &all, + false, + ); + assert_eq!((m.local.clone(), m.conflict), (v(&["c", "a", "b"]), false)); + assert_eq!(m.shared, m.local); + let m = merge( + Some(&v(&["a", "b", "c"])), + &v(&["c", "a", "b"]), + &v(&["b", "a", "c"]), + &all, + &all, + false, + ); + assert!(m.conflict); + assert_eq!(m.local, v(&["c", "a", "b"])); + assert_eq!(m.shared, v(&["b", "a", "c"])); + // 两边排成一样的：不冲突 + let m = merge( + Some(&v(&["a", "b"])), + &v(&["b", "a"]), + &v(&["b", "a"]), + &set(&["a", "b"]), + &set(&["a", "b"]), + false, + ); + assert!(!m.conflict); + } + + #[test] + fn deleted_and_local_only_names_drop_out_of_the_upload() { + // 本机有一个不同步的 ollama：本机的顺序里留着，上传的没有 + let m = merge( + Some(&v(&["ollama", "relay", "official"])), + &v(&["ollama", "relay", "official"]), + &v(&["relay", "official"]), + &set(&["ollama", "relay", "official"]), + &set(&["relay", "official"]), + false, + ); + assert!(!m.conflict); + assert_eq!(m.local, v(&["ollama", "relay", "official"])); + assert_eq!(m.shared, v(&["relay", "official"])); + } + + #[test] + fn a_held_order_keeps_each_side_without_raising_it_again() { + let all = set(&["a", "b"]); + let m = merge( + Some(&v(&["a", "b"])), + &v(&["b", "a"]), + &v(&["a", "b"]), + &all, + &all, + true, + ); + assert!(!m.conflict); + assert_eq!(m.local, v(&["b", "a"])); + assert_eq!(m.shared, v(&["a", "b"])); + } +} diff --git a/crates/tw-sync/src/record.rs b/crates/tw-sync/src/record.rs new file mode 100644 index 00000000..dc5d3c78 --- /dev/null +++ b/crates/tw-sync/src/record.rs @@ -0,0 +1,226 @@ +//! 每个同步版本自己说的那几件事（配置包 manifest 里的 `sync`）：第几版、谁写的、依据的 +//! 上一版本、还没处理的冲突。 +//! +//! **冲突记录随版本走**：所有设备看到的是同一份冲突，任一设备处理之后，处理结果作为新版本 +//! 上传，冲突从记录里去掉，别的设备随之清除，不重复处理。 + +use std::collections::{BTreeMap, BTreeSet}; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use tw_pack::model::ItemId; + +/// `manifest.sync` +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct Meta { + pub seq: u64, + /// 写它的设备，完整的标识 + pub device: String, + pub device_name: String, + /// 依据的上一版本（版本标识，见 [`crate::layout::VersionName::id`]）。第一个版本没有； + /// 合并了并列的版本时有几个 + #[serde(default)] + pub parents: Vec, + /// 上一版本各自的内容标识（`Pack::content_id`），和 `parents` 一一对应。上一版本在同步 + /// 位置上已经删了时，按它去本机备份里找基准 + #[serde(default)] + pub parent_content: Vec, + /// 内容的 SHA-256（只在不加密时写）：发现传输中损坏。加密时 age 的认证加密已经管了 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sha256: Option, + /// 还没处理的冲突 + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub conflicts: Vec, +} + +/// 一处冲突：同一个对象（的几个单位）在两台设备上被改成了不同的内容。 +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Conflict { + /// 随机生成 + pub id: String, + pub item: ItemId, + /// 冲突的单位 + pub units: Vec, + /// 同步版本里现在是这一边的内容 + pub shared: Side, + /// 另一边：它的设备在冲突处理之前一直用自己的 + pub other: Side, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, +} + +/// 冲突的一边 +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Side { + pub device: String, + pub device_name: String, + /// 这一边改的时间 + pub at_ms: u64, + /// 冲突的那几个单位在这一边的值。**这一边删了这个对象**是 `None` + pub units: Option>, +} + +/// 冲突不是因为两边改得不一样，而是： +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Reason { + /// 另一边删了一把网关密钥，而这台设备上已接管的客户端正在用它：没有删 + KeyInUse, + /// 合并出来的结果过不了配置校验（比如路由引用的上游被另一边删了） + Invalid, +} + +impl Conflict { + /// 有一边删了整个对象：两边各留各的是整个对象，不只是几个单位 + pub fn whole_item(&self) -> bool { + self.shared.units.is_none() || self.other.units.is_none() + } + + /// 「两者都保留」能不能选：两边都还有这个对象，而且它有名字（全局设置、顺序、通用偏好 + /// 没有第二份可言） + pub fn both_allowed(&self) -> bool { + !self.whole_item() && self.item.kind.is_named() && self.reason.is_none() + } + + /// 这台设备看这处冲突时，哪一边是「本机」：自己写的那一边；不是参与的设备时，是和本机 + /// 现在内容一样的那一边，都不一样就是同步版本里的那一边 + pub fn mine<'a>( + &'a self, + device: &str, + local_units: Option<&BTreeMap>, + ) -> (&'a Side, &'a Side) { + if self.other.device == device { + return (&self.other, &self.shared); + } + if self.shared.device == device { + return (&self.shared, &self.other); + } + let mine_now = local_units.map(|l| self.pick(l)); + if mine_now.as_ref() == self.other.units.as_ref() { + (&self.other, &self.shared) + } else { + (&self.shared, &self.other) + } + } + + /// 一个对象里冲突的那几个单位。有一边删了整个对象时是整个对象（处理冲突选了「有」的 + /// 那一边，要拿它整个恢复回来） + pub fn pick(&self, units: &BTreeMap) -> BTreeMap { + if self.whole_item() { + return units.clone(); + } + self.units + .iter() + .filter_map(|u| units.get(u).map(|v| (u.clone(), v.clone()))) + .collect() + } +} + +/// 一组冲突各留各的那部分：整个对象，和单个单位 +pub fn holds(conflicts: &[Conflict]) -> (BTreeSet, BTreeSet<(ItemId, String)>) { + let mut items = BTreeSet::new(); + let mut units = BTreeSet::new(); + for c in conflicts { + if c.whole_item() { + items.insert(c.item.clone()); + } else { + for u in &c.units { + units.insert((c.item.clone(), u.clone())); + } + } + } + (items, units) +} + +/// 新的冲突标识 +pub fn new_id() -> String { + let r: u64 = rand::random(); + format!("{r:016x}") +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use tw_pack::model::ItemKind; + + fn side(device: &str, v: Option) -> Side { + Side { + device: device.into(), + device_name: device.to_uppercase(), + at_ms: 1, + units: v.map(|v| BTreeMap::from([("conn".to_string(), v)])), + } + } + + fn c(shared: Option, other: Option, kind: ItemKind) -> Conflict { + Conflict { + id: new_id(), + item: ItemId::new(kind, "relay"), + units: vec!["conn".into()], + shared: side("aaaa", shared), + other: side("bbbb", other), + reason: None, + } + } + + #[test] + fn each_device_sees_its_own_side_as_local() { + let x = c(Some(json!("A")), Some(json!("B")), ItemKind::Provider); + assert_eq!(x.mine("aaaa", None).0.device, "aaaa"); + assert_eq!(x.mine("bbbb", None).0.device, "bbbb"); + // 第三台：和本机内容一样的那一边 + let local = BTreeMap::from([ + ("conn".to_string(), json!("B")), + ("models".into(), json!(1)), + ]); + assert_eq!(x.mine("cccc", Some(&local)).0.device, "bbbb"); + assert_eq!(x.mine("cccc", None).0.device, "aaaa"); + } + + #[test] + fn keeping_both_needs_two_living_named_objects() { + assert!(c(Some(json!(1)), Some(json!(2)), ItemKind::Provider).both_allowed()); + assert!(!c(None, Some(json!(2)), ItemKind::Provider).both_allowed()); + assert!(!c(Some(json!(1)), None, ItemKind::Route).both_allowed()); + assert!(!c(Some(json!(1)), Some(json!(2)), ItemKind::Global).both_allowed()); + assert!(!c(Some(json!(1)), Some(json!(2)), ItemKind::RouteOrder).both_allowed()); + let mut k = c(Some(json!(1)), None, ItemKind::Key); + k.reason = Some(Reason::KeyInUse); + assert!(!k.both_allowed()); + } + + #[test] + fn holds_split_into_whole_objects_and_units() { + let (items, units) = holds(&[c(Some(json!(1)), Some(json!(2)), ItemKind::Provider), { + let mut d = c(None, Some(json!(2)), ItemKind::Route); + d.item = ItemId::new(ItemKind::Route, "long"); + d + }]); + assert_eq!( + items, + BTreeSet::from([ItemId::new(ItemKind::Route, "long")]) + ); + assert_eq!( + units, + BTreeSet::from([(ItemId::new(ItemKind::Provider, "relay"), "conn".to_string())]) + ); + } + + #[test] + fn a_record_survives_json() { + let m = Meta { + seq: 7, + device: "aaaa".into(), + device_name: "Mac mini".into(), + parents: vec!["0000000006-aaaaaaaa".into()], + parent_content: vec!["abc".into()], + sha256: None, + conflicts: vec![c(Some(json!({"base_url": "x"})), None, ItemKind::Key)], + }; + let v = serde_json::to_value(&m).unwrap(); + assert_eq!(v["conflicts"][0]["item"]["kind"], "key"); + let back: Meta = serde_json::from_value(v).unwrap(); + assert_eq!(back, m); + } +} diff --git a/crates/tw-sync/src/store/folder.rs b/crates/tw-sync/src/store/folder.rs new file mode 100644 index 00000000..a8fca449 --- /dev/null +++ b/crates/tw-sync/src/store/folder.rs @@ -0,0 +1,398 @@ +//! 文件夹位置：本机的一个目录，由 iCloud Drive、Dropbox、OneDrive、Syncthing 或者 NAS +//! 共享在设备之间传送。 +//! +//! **写入先写临时名再改名**：第三方客户端随时可能把写了一半的文件传走，另一台设备读到半个 +//! 版本。临时名以点开头，列目录时跳过；云盘客户端多数也不传点开头的临时文件。 +//! +//! 云盘里还没下载到本机的文件（iCloud 的 `.名字.icloud` 占位文件、Dropbox 的在线文件）列目录 +//! 时要么不出现、要么读不出来 —— 调用方把读不出来的版本当成「尚未到达」，到了再合并。 + +use std::io::Write; +use std::path::{Path, PathBuf}; + +use tw_types::msg; + +use super::{Entry, Store, StoreError, check_path, split, temp_name}; + +pub struct Folder { + root: PathBuf, +} + +impl Folder { + pub fn new(root: impl Into) -> Self { + Self { root: root.into() } + } + + pub fn root(&self) -> &Path { + &self.root + } + + fn at(&self, path: &str) -> Result { + check_path(path)?; + let mut p = self.root.clone(); + for seg in path.split('/') { + p.push(seg); + } + Ok(p) + } +} + +fn io(path: &Path, e: std::io::Error) -> StoreError { + let detail = e.to_string(); + let path = path.display().to_string(); + // 根目录不在（外接盘没插、网络共享没挂上）：和网络位置连不上是同一种情况，等它回来 + if e.kind() == std::io::ErrorKind::NotFound { + return StoreError::Unreachable(msg!( + "sync.folder.missing", path = path => + "the sync folder {path} is not there. If it is on an external drive or a network \ + share, connect it" + )); + } + if e.kind() == std::io::ErrorKind::PermissionDenied { + return StoreError::AuthFailed(msg!( + "sync.folder.denied", path = path, detail = detail => + "{path} cannot be written: {detail}" + )); + } + StoreError::Other(msg!( + "sync.folder.io", path = path, detail = detail => + "{path} could not be used: {detail}" + )) +} + +/// 建一个只有本机用户读得到的新文件，写完、落盘 +fn write_new(path: &Path, bytes: &[u8]) -> std::io::Result<()> { + let mut opts = std::fs::OpenOptions::new(); + opts.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + opts.mode(0o600); + } + let mut f = opts.open(path)?; + f.write_all(bytes)?; + f.sync_all() +} + +/// 父目录：要在根目录底下。**根目录本身不替用户建**：它不在说明外接盘、共享没挂上，在一个 +/// 空目录里写进去的版本谁也收不到 +fn ensure_parent(root: &Path, p: &Path) -> Result<(), StoreError> { + if !root.is_dir() { + return Err(io( + root, + std::io::Error::new(std::io::ErrorKind::NotFound, "missing"), + )); + } + if let Some(dir) = p.parent() + && dir != root + { + std::fs::create_dir_all(dir).map_err(|e| io(dir, e))?; + } + Ok(()) +} + +async fn blocking( + f: impl FnOnce() -> Result + Send + 'static, +) -> Result { + tokio::task::spawn_blocking(f).await.unwrap_or_else(|e| { + Err(StoreError::Other(msg!( + "sync.folder.task", detail = e => + "the sync folder could not be used: {detail}" + ))) + }) +} + +/// 改名成最终的名字，**目标已经有了就不改**。硬链接是原子的「不存在才建」；不支持硬链接的 +/// 文件系统（FAT、有的网络共享）退回到先看一眼再改名 —— 名字归写它的那台设备独占，看一眼 +/// 和改名之间不会有别人来写同一个名字 +fn link_or_rename(tmp: &Path, dest: &Path) -> std::io::Result { + match std::fs::hard_link(tmp, dest) { + Ok(()) => { + let _ = std::fs::remove_file(tmp); + Ok(true) + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { + let _ = std::fs::remove_file(tmp); + Ok(false) + } + Err(_) => { + if dest.exists() { + let _ = std::fs::remove_file(tmp); + return Ok(false); + } + std::fs::rename(tmp, dest).map(|()| true) + } + } +} + +impl Store for Folder { + async fn list(&self, dir: &str) -> Result, StoreError> { + let root = self.root.clone(); + let d = self.at(dir)?; + blocking(move || { + if !root.is_dir() { + return Err(io( + &root, + std::io::Error::new(std::io::ErrorKind::NotFound, "missing"), + )); + } + let rd = match std::fs::read_dir(&d) { + Ok(r) => r, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(e) => return Err(io(&d, e)), + }; + let mut out = Vec::new(); + for e in rd.flatten() { + let name = e.file_name().to_string_lossy().to_string(); + // 临时文件、云盘的占位文件、系统的 .DS_Store + if name.starts_with('.') { + continue; + } + let Ok(meta) = e.metadata() else { continue }; + if !meta.is_file() { + continue; + } + out.push(Entry { + name, + size: meta.len(), + }); + } + out.sort_by(|a, b| a.name.cmp(&b.name)); + Ok(out) + }) + .await + } + + async fn get(&self, path: &str) -> Result>, StoreError> { + let root = self.root.clone(); + let p = self.at(path)?; + blocking(move || match std::fs::read(&p) { + Ok(b) => Ok(Some(b)), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + if root.is_dir() { + Ok(None) + } else { + Err(io(&root, e)) + } + } + Err(e) => Err(io(&p, e)), + }) + .await + } + + async fn put_new(&self, path: &str, bytes: &[u8]) -> Result<(), StoreError> { + let root = self.root.clone(); + let dest = self.at(path)?; + let (_, name) = split(path); + let tmp = dest.with_file_name(temp_name(name)); + let bytes = bytes.to_vec(); + let shown = path.to_string(); + blocking(move || { + ensure_parent(&root, &dest)?; + if dest.exists() { + return Err(StoreError::Exists { path: shown }); + } + write_new(&tmp, &bytes).map_err(|e| io(&tmp, e))?; + match link_or_rename(&tmp, &dest) { + Ok(true) => Ok(()), + Ok(false) => Err(StoreError::Exists { path: shown }), + Err(e) => { + let _ = std::fs::remove_file(&tmp); + Err(io(&dest, e)) + } + } + }) + .await + } + + async fn put(&self, path: &str, bytes: &[u8]) -> Result<(), StoreError> { + let root = self.root.clone(); + let dest = self.at(path)?; + let (_, name) = split(path); + let tmp = dest.with_file_name(temp_name(name)); + let bytes = bytes.to_vec(); + blocking(move || { + ensure_parent(&root, &dest)?; + write_new(&tmp, &bytes).map_err(|e| io(&tmp, e))?; + // rename 在两个平台上都换掉已有的目标（Windows 上 std 用的是 MOVEFILE_REPLACE_EXISTING） + std::fs::rename(&tmp, &dest).map_err(|e| { + let _ = std::fs::remove_file(&tmp); + io(&dest, e) + }) + }) + .await + } + + async fn delete(&self, path: &str) -> Result<(), StoreError> { + let p = self.at(path)?; + blocking(move || match std::fs::remove_file(&p) { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(io(&p, e)), + }) + .await + } + + async fn create_exclusive(&self, path: &str, bytes: &[u8]) -> Result { + let root = self.root.clone(); + let dest = self.at(path)?; + let (_, name) = split(path); + let tmp = dest.with_file_name(temp_name(name)); + let bytes = bytes.to_vec(); + blocking(move || { + ensure_parent(&root, &dest)?; + if dest.exists() { + return Ok(false); + } + write_new(&tmp, &bytes).map_err(|e| io(&tmp, e))?; + link_or_rename(&tmp, &dest).map_err(|e| { + let _ = std::fs::remove_file(&tmp); + io(&dest, e) + }) + }) + .await + } +} + +/// 盯着同步文件夹：新版本、新的 keyring 一到就发一个信号（见 [`tw_watch::watch`]）。盯的是 +/// 根目录和 `versions/`，不递归；`versions/` 还没有就先建出来，不然盯不住。 +/// +/// **设备记录不盯**：每台设备同步完都会写自己那一份，盯着它的话，两台设备会你写一份、我 +/// 同步一轮、我再写一份地来回转 +pub fn watch( + root: &Path, +) -> Result<(tw_watch::Watch, tokio::sync::mpsc::Receiver<()>), tw_watch::WatchError> { + let versions = root.join(crate::layout::VERSIONS); + // 根目录不在（外接盘没插）时不替它建 + if root.is_dir() { + let _ = std::fs::create_dir(&versions); + } + let dirs: Vec = [root.to_path_buf(), versions] + .into_iter() + .filter(|d| d.is_dir()) + .collect(); + tw_watch::watch(&dirs, std::time::Duration::from_millis(500), |p| { + // 我们自己的临时文件、云盘的占位文件不算：改名成正式的名字那一下才算 + p.file_name() + .and_then(|n| n.to_str()) + .is_some_and(|n| !n.starts_with('.')) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn bed() -> (tempfile::TempDir, Folder) { + let d = tempfile::tempdir().unwrap(); + let f = Folder::new(d.path()); + (d, f) + } + + #[tokio::test] + async fn put_new_never_overwrites_and_leaves_no_temporary_file() { + let (d, f) = bed(); + f.put_new("versions/0000000001-aaaaaaaa.twconfig", b"one") + .await + .unwrap(); + let e = f + .put_new("versions/0000000001-aaaaaaaa.twconfig", b"two") + .await + .unwrap_err(); + assert!(matches!(e, StoreError::Exists { .. }), "{e:?}"); + assert_eq!( + f.get("versions/0000000001-aaaaaaaa.twconfig") + .await + .unwrap(), + Some(b"one".to_vec()) + ); + let names: Vec = std::fs::read_dir(d.path().join("versions")) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().to_string()) + .collect(); + assert_eq!(names, ["0000000001-aaaaaaaa.twconfig"]); + } + + #[tokio::test] + async fn create_exclusive_says_whether_it_made_the_file() { + let (_d, f) = bed(); + assert!(f.create_exclusive("keyring", b"first").await.unwrap()); + assert!(!f.create_exclusive("keyring", b"second").await.unwrap()); + assert_eq!(f.get("keyring").await.unwrap(), Some(b"first".to_vec())); + f.put("keyring", b"changed").await.unwrap(); + assert_eq!(f.get("keyring").await.unwrap(), Some(b"changed".to_vec())); + } + + #[tokio::test] + async fn listing_skips_hidden_and_placeholder_files() { + let (d, f) = bed(); + assert!(f.list("versions").await.unwrap().is_empty()); + f.put_new("versions/0000000002-bbbbbbbb.twconfig", b"x") + .await + .unwrap(); + std::fs::write( + d.path() + .join("versions/.0000000003-cccccccc.twconfig.icloud"), + b"placeholder", + ) + .unwrap(); + std::fs::create_dir(d.path().join("versions/sub")).unwrap(); + let got = f.list("versions").await.unwrap(); + assert_eq!( + got, + [Entry { + name: "0000000002-bbbbbbbb.twconfig".into(), + size: 1 + }] + ); + f.delete("versions/0000000002-bbbbbbbb.twconfig") + .await + .unwrap(); + f.delete("versions/0000000002-bbbbbbbb.twconfig") + .await + .unwrap(); + assert!(f.list("versions").await.unwrap().is_empty()); + } + + #[tokio::test] + async fn a_missing_root_is_unreachable_and_never_created() { + let d = tempfile::tempdir().unwrap(); + let gone = d.path().join("not-mounted"); + let f = Folder::new(&gone); + let e = f.put_new("versions/x.twconfig", b"x").await.unwrap_err(); + assert!(matches!(e, StoreError::Unreachable(_)), "{e:?}"); + assert!(matches!( + f.list("versions").await.unwrap_err(), + StoreError::Unreachable(_) + )); + assert!(!gone.exists(), "根目录不该替用户建出来"); + } + + #[cfg(unix)] + #[tokio::test] + async fn files_are_readable_only_by_this_user() { + use std::os::unix::fs::PermissionsExt; + let (d, f) = bed(); + f.put_new("devices/aaaaaaaa", b"x").await.unwrap(); + let mode = std::fs::metadata(d.path().join("devices/aaaaaaaa")) + .unwrap() + .permissions() + .mode() + & 0o777; + assert_eq!(mode, 0o600); + } + + #[tokio::test] + async fn a_new_version_arriving_wakes_the_watcher() { + let (d, f) = bed(); + let (_w, mut rx) = watch(d.path()).unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + f.put_new("versions/0000000009-dddddddd.twconfig", b"v") + .await + .unwrap(); + let got = tokio::time::timeout(std::time::Duration::from_secs(5), rx.recv()) + .await + .expect("五秒内没收到信号"); + assert_eq!(got, Some(())); + } +} diff --git a/crates/tw-sync/src/store/mod.rs b/crates/tw-sync/src/store/mod.rs new file mode 100644 index 00000000..cd56a9c7 --- /dev/null +++ b/crates/tw-sync/src/store/mod.rs @@ -0,0 +1,296 @@ +//! 同步位置的存储：文件夹、WebDAV、S3 兼容存储，**同一套五个动作**。 +//! +//! 同步只追加新版本、不覆盖已有文件，所以不依赖存储服务的条件写入：版本的文件名 +//! （序号-设备标识）由写它的那台设备独占，两台设备永远不会去写同一个名字。只有两样东西 +//! 不是这样： +//! +//! - `keyring` **仅当不存在时创建**（[`Store::create_exclusive`]）：两台设备同时开启加密同步时， +//! 后写入的一方要知道自己晚了一步，转入加入流程； +//! - 设备记录和改口令后的 keyring 要覆盖（[`Store::put`]）：设备记录只有它自己那台设备写。 +//! +//! 路径一律相对同步位置的根，用 `/` 分隔，只有两层（`versions/<名字>`、`devices/<名字>`、 +//! `keyring`）。名字由这个 crate 生成，只含 ASCII 字母、数字、连字符和点。 + +use std::future::Future; + +use tw_types::{Msg, msg}; + +pub mod folder; +pub mod s3; +pub mod webdav; +mod xml; + +pub use folder::Folder; +pub use s3::S3; +pub use webdav::WebDav; + +/// 目录里的一个文件。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Entry { + pub name: String, + pub size: u64, +} + +/// 一次存储操作没成。 +/// +/// **按「接下来怎么办」分类**，不按出错的那一层：界面上同步位置不可用（等它恢复）、 +/// 拒绝访问（改连接参数）是两种不同的提示。 +#[derive(Debug, Clone, thiserror::Error)] +pub enum StoreError { + /// 连不上、超时、服务端 5xx。**过一会儿再试** + #[error("{}", .0.text)] + Unreachable(Msg), + /// 401 / 403：用户名、密码、密钥不对，或者没有这个位置的权限 + #[error("{}", .0.text)] + AuthFailed(Msg), + /// `put_new`：这个名字已经有文件了 + #[error("{}", self.msg())] + Exists { path: String }, + /// 别的：服务端回了一个看不懂的答复、本机磁盘写不进去 + #[error("{}", .0.text)] + Other(Msg), +} + +impl StoreError { + pub fn msg(&self) -> Msg { + match self { + StoreError::Unreachable(m) | StoreError::AuthFailed(m) | StoreError::Other(m) => { + m.clone() + } + StoreError::Exists { path } => msg!( + "sync.store.exists", path = path => + "the sync location already has {path}" + ), + } + } +} + +/// 同步位置。实现见 [`Folder`]、[`WebDav`]、[`S3`]；运行时按配置选一个，用 [`AnyStore`]。 +pub trait Store: Send + Sync { + /// 一个目录里的文件（不递归，不含子目录）。目录不存在是空的，不是错误 + fn list(&self, dir: &str) -> impl Future, StoreError>> + Send; + /// 读一个文件。不存在是 `None` + fn get(&self, path: &str) -> impl Future>, StoreError>> + Send; + /// **只新增**：已有同名文件时是 [`StoreError::Exists`]，原来那个不动。写的过程中别的设备 + /// 读不到半个文件（文件夹和 WebDAV 先写临时名再改名） + fn put_new( + &self, + path: &str, + bytes: &[u8], + ) -> impl Future> + Send; + /// 覆盖写。只用于设备记录（只有它自己那台设备写）和改口令后的 keyring + fn put(&self, path: &str, bytes: &[u8]) -> impl Future> + Send; + /// 删掉。本来就没有不算错 + fn delete(&self, path: &str) -> impl Future> + Send; + /// 仅当不存在时创建：建成是 `true`，已经有了是 `false`（原来那个不动） + fn create_exclusive( + &self, + path: &str, + bytes: &[u8], + ) -> impl Future> + Send; +} + +/// 按配置选出来的那一种。 +pub enum AnyStore { + Folder(Folder), + WebDav(WebDav), + S3(S3), +} + +impl AnyStore { + /// 按 `sync.target` 建一个。`http` 是发 WebDAV、S3 请求用的客户端（带着出站代理）， + /// 文件夹用不到它 + pub fn new( + target: &tw_config::sync::Target, + http: reqwest::Client, + ) -> Result { + use tw_config::sync::Target; + Ok(match target { + Target::Folder { path, .. } => AnyStore::Folder(Folder::new(path)), + Target::Webdav { + url, user, pass, .. + } => AnyStore::WebDav(WebDav::new(url, user, pass, http)?), + Target::S3 { + endpoint, + region, + bucket, + prefix, + access_key_id, + secret_access_key, + .. + } => AnyStore::S3(S3::new( + s3::Config { + endpoint: endpoint.clone(), + region: region.clone(), + bucket: bucket.clone(), + prefix: prefix.clone(), + access_key_id: access_key_id.clone(), + secret_access_key: secret_access_key.clone(), + }, + http, + )?), + }) + } + + /// 文件夹位置的根目录。别的是 `None` + pub fn folder_root(&self) -> Option<&std::path::Path> { + match self { + AnyStore::Folder(f) => Some(f.root()), + _ => None, + } + } +} + +macro_rules! each { + ($self:ident, $s:ident => $e:expr) => { + match $self { + AnyStore::Folder($s) => $e, + AnyStore::WebDav($s) => $e, + AnyStore::S3($s) => $e, + } + }; +} + +impl Store for AnyStore { + async fn list(&self, dir: &str) -> Result, StoreError> { + each!(self, s => s.list(dir).await) + } + async fn get(&self, path: &str) -> Result>, StoreError> { + each!(self, s => s.get(path).await) + } + async fn put_new(&self, path: &str, bytes: &[u8]) -> Result<(), StoreError> { + each!(self, s => s.put_new(path, bytes).await) + } + async fn put(&self, path: &str, bytes: &[u8]) -> Result<(), StoreError> { + each!(self, s => s.put(path, bytes).await) + } + async fn delete(&self, path: &str) -> Result<(), StoreError> { + each!(self, s => s.delete(path).await) + } + async fn create_exclusive(&self, path: &str, bytes: &[u8]) -> Result { + each!(self, s => s.create_exclusive(path, bytes).await) + } +} + +/// 临时文件名：点开头（列目录时跳过），带随机串（两个进程不撞） +pub(crate) fn temp_name(final_name: &str) -> String { + let r: u64 = rand::random(); + format!(".tmp-{r:016x}-{final_name}") +} + +/// 一个路径拆成目录和文件名。只有一层目录 +pub(crate) fn split(path: &str) -> (&str, &str) { + match path.rsplit_once('/') { + Some((d, f)) => (d, f), + None => ("", path), + } +} + +/// 路径里只许出现我们自己生成的那些字符：拼进 URL 和文件系统都不用转义，也走不出根目录 +pub(crate) fn check_path(path: &str) -> Result<(), StoreError> { + let ok = !path.is_empty() + && path.split('/').all(|seg| { + !seg.is_empty() + && seg != "." + && seg != ".." + && seg + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.')) + }); + if ok { + Ok(()) + } else { + Err(StoreError::Other(msg!( + "sync.store.bad_path", path = path => + "{path} is not a path inside the sync location" + ))) + } +} + +/// HTTP 层的失败按「接下来怎么办」归类。`what` 是这次在做什么（`GET versions/…`），只进英文原句 +pub(crate) fn http_status(status: reqwest::StatusCode, what: &str) -> StoreError { + let code = status.as_u16(); + if code == 401 || code == 403 { + return StoreError::AuthFailed(msg!( + "sync.store.denied", status = code => + "the sync location refused access ({status}). Check the user name, password or keys" + )); + } + if status.is_server_error() || code == 408 || code == 429 { + return StoreError::Unreachable(msg!( + "sync.store.server_error", status = code, what = what => + "the sync location answered {status} to {what}" + )); + } + StoreError::Other(msg!( + "sync.store.unexpected", status = code, what = what => + "the sync location answered {status} to {what}" + )) +} + +/// 请求没发出去或者没收到答复 +pub(crate) fn http_error(e: &reqwest::Error) -> StoreError { + let mut detail = e.to_string(); + let mut cur = std::error::Error::source(e); + while let Some(c) = cur { + detail.push_str(": "); + detail.push_str(&c.to_string()); + cur = c.source(); + } + StoreError::Unreachable(msg!( + "sync.store.unreachable", detail = detail => + "the sync location could not be reached: {detail}" + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_paths_we_generate_are_taken() { + for ok in [ + "keyring", + "versions/0000000041-3f9a2c1e.twconfig", + "devices/3f9a2c1e", + ] { + check_path(ok).unwrap(); + } + for bad in [ + "", + "/keyring", + "versions/../x", + "a//b", + "a b", + "versions/%2e%2e", + "a\\b", + ] { + assert!(check_path(bad).is_err(), "{bad}"); + } + } + + #[test] + fn statuses_are_sorted_by_what_to_do_next() { + use reqwest::StatusCode as S; + assert!(matches!( + http_status(S::UNAUTHORIZED, "x"), + StoreError::AuthFailed(_) + )); + assert!(matches!( + http_status(S::FORBIDDEN, "x"), + StoreError::AuthFailed(_) + )); + assert!(matches!( + http_status(S::BAD_GATEWAY, "x"), + StoreError::Unreachable(_) + )); + assert!(matches!( + http_status(S::TOO_MANY_REQUESTS, "x"), + StoreError::Unreachable(_) + )); + assert!(matches!( + http_status(S::CONFLICT, "x"), + StoreError::Other(_) + )); + } +} diff --git a/crates/tw-sync/src/store/s3.rs b/crates/tw-sync/src/store/s3.rs new file mode 100644 index 00000000..a823ee1f --- /dev/null +++ b/crates/tw-sync/src/store/s3.rs @@ -0,0 +1,456 @@ +//! S3 兼容存储：Amazon S3、Cloudflare R2、阿里云 OSS、腾讯云 COS、MinIO。 +//! +//! 签名是 SigV4，和 Bedrock 用的同一份（[`tw_bedrock::sign_service`]）。四个动作： +//! `ListObjectsV2`（列目录，带 `delimiter=/`，不递归）、`GetObject`、`PutObject`、`DeleteObject`。 +//! 「只新增」和「仅当不存在时创建」都带 `If-None-Match: *`：S3、R2、MinIO 认它；不认的服务端 +//! （回 501）退回到先看一眼再写 —— 版本的名字归写它的那台设备独占，只有 keyring 会有两台设备 +//! 抢，那一刻两台设备同时开启加密同步，退回之后后写的那台覆盖了先写的，先写的那台下一次同步 +//! 时发现口令对不上（keyring 换了）就转入加入流程。 +//! +//! **地址怎么拼**（[`addressing`]）：端点主机以 `amazonaws.com`、`aliyuncs.com`、`myqcloud.com` +//! 结尾的用虚拟主机（`<桶>.<端点>/<键>`，OSS、COS 要求如此），别的（R2、MinIO、IP 地址、自定义 +//! 域名）用路径（`<端点>/<桶>/<键>`）。 + +use std::time::{Duration, SystemTime}; + +use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, utf8_percent_encode}; +use reqwest::Method; +use tw_types::msg; + +use super::{Entry, Store, StoreError, check_path, http_error, http_status}; + +const TIMEOUT: Duration = Duration::from_secs(120); + +/// 路径里的一段：RFC 3986 的非保留字符不编，别的都编 +const SEGMENT: &AsciiSet = &NON_ALPHANUMERIC + .remove(b'-') + .remove(b'.') + .remove(b'_') + .remove(b'~'); + +/// 查询参数的值：同上，斜杠也编 +const QUERY: &AsciiSet = SEGMENT; + +/// 这几家只认（或者推荐）虚拟主机的写法 +const VIRTUAL_HOSTED: &[&str] = &["amazonaws.com", "aliyuncs.com", "myqcloud.com"]; + +#[derive(Clone)] +pub struct Config { + pub endpoint: String, + pub region: String, + pub bucket: String, + pub prefix: String, + pub access_key_id: String, + pub secret_access_key: String, +} + +pub struct S3 { + /// 桶的地址，以 `/` 结尾：`https://b.s3.us-east-1.amazonaws.com/` 或 `http://nas:9000/b/` + bucket_url: String, + /// 键的前缀，空的或者以 `/` 结尾 + prefix: String, + region: String, + creds: tw_bedrock::Credentials, + http: reqwest::Client, +} + +fn bad(endpoint: &str) -> StoreError { + StoreError::Other( + tw_config::sync::SyncError::Address { + field: "endpoint", + url: endpoint.to_string(), + } + .msg(), + ) +} + +/// 寻址方式：端点主机以 `amazonaws.com`、`aliyuncs.com`、`myqcloud.com` 结尾的用虚拟主机 +/// （桶名进域名，阿里云 OSS、腾讯云 COS 要求如此）；别的（R2、MinIO、IP 地址、自定义域名）用 +/// 路径。不给界面加选项。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Addressing { + VirtualHost, + Path, +} + +/// 按端点的主机选寻址方式（纯函数） +pub fn addressing(host: &str) -> Addressing { + let host = host.trim_end_matches('.').to_ascii_lowercase(); + if VIRTUAL_HOSTED + .iter() + .any(|d| host == *d || host.ends_with(&format!(".{d}"))) + { + Addressing::VirtualHost + } else { + Addressing::Path + } +} + +/// 桶的地址，以 `/` 结尾（纯函数）。虚拟主机方式下，端点本身已经以 `<桶>.` 开头的（照着控制台 +/// 抄来的桶域名）不再加一遍。地址规则（`http://` 只给内网）在这之前查 +pub fn bucket_url(endpoint: &str, bucket: &str) -> Option { + let mut u = url::Url::parse(endpoint.trim()).ok()?; + let host = u.host_str()?.to_ascii_lowercase(); + let base_path = u.path().trim_end_matches('/').to_string(); + match addressing(&host) { + Addressing::VirtualHost => { + if !host.starts_with(&format!("{bucket}.")) { + u.set_host(Some(&format!("{bucket}.{host}"))).ok()?; + } + u.set_path(&format!("{base_path}/")); + } + Addressing::Path => { + let seg = utf8_percent_encode(bucket, SEGMENT).to_string(); + u.set_path(&format!("{base_path}/{seg}/")); + } + } + Some(u.to_string()) +} + +impl S3 { + pub fn new(c: Config, http: reqwest::Client) -> Result { + // `http://` 只给本机和内网，和 WebDAV 同一条规则；证书由客户端照常校验 + if !tw_config::sync::address_ok(&c.endpoint) { + return Err(bad(&c.endpoint)); + } + let bucket_url = bucket_url(&c.endpoint, &c.bucket).ok_or_else(|| bad(&c.endpoint))?; + let p = c.prefix.trim().trim_matches('/'); + let prefix = if p.is_empty() { + String::new() + } else { + format!("{p}/") + }; + Ok(Self { + bucket_url, + prefix, + region: c.region.trim().to_string(), + creds: tw_bedrock::Credentials { + access_key_id: c.access_key_id, + secret_access_key: c.secret_access_key, + session_token: None, + }, + http, + }) + } + + fn key(&self, path: &str) -> String { + format!("{}{path}", self.prefix) + } + + fn object_url(&self, key: &str) -> String { + let enc: Vec = key + .split('/') + .map(|s| utf8_percent_encode(s, SEGMENT).to_string()) + .collect(); + format!("{}{}", self.bucket_url, enc.join("/")) + } + + /// 签好名、发出去。`headers` 一起签 + async fn send( + &self, + m: Method, + url: &str, + headers: &[(&str, &str)], + body: &[u8], + ) -> Result { + let signed = tw_bedrock::sign_service( + &self.creds, + &self.region, + "s3", + m.as_str(), + url, + headers, + body, + SystemTime::now(), + ) + .map_err(|e| { + StoreError::Other(msg!( + "sync.s3.sign", detail = e => + "the request to the S3 location could not be signed: {detail}" + )) + })?; + let mut r = self.http.request(m, url).timeout(TIMEOUT); + for (k, v) in headers { + r = r.header(*k, *v); + } + for (k, v) in signed { + r = r.header(k, v); + } + if !body.is_empty() { + r = r.body(body.to_vec()); + } + r.send().await.map_err(|e| http_error(&e)) + } + + /// S3 错误答复里的 `` + async fn error_code(r: reqwest::Response) -> String { + let body = r.text().await.unwrap_or_default(); + super::xml::parse(&body, "Error") + .and_then(|(rs, top)| { + rs.first() + .and_then(|e| e.get("Code").map(str::to_string)) + .or_else(|| top.get("Code").cloned()) + }) + .unwrap_or_default() + } + + async fn exists(&self, path: &str) -> Result { + let url = self.object_url(&self.key(path)); + let r = self.send(Method::HEAD, &url, &[], b"").await?; + match r.status().as_u16() { + 404 => Ok(false), + s if (200..300).contains(&s) => Ok(true), + _ => Err(http_status(r.status(), "HEAD")), + } + } + + /// 写一个对象。`exclusive`：带 `If-None-Match: *`，已经有了是 `Ok(false)`；服务端不认这个 + /// 头时退回到先看一眼再写 + async fn put_object( + &self, + path: &str, + bytes: &[u8], + exclusive: bool, + ) -> Result { + let url = self.object_url(&self.key(path)); + let ct = ("content-type", "application/octet-stream"); + if !exclusive { + return self.put_plain(&url, bytes).await.map(|()| true); + } + let r = self + .send(Method::PUT, &url, &[ct, ("if-none-match", "*")], bytes) + .await?; + let st = r.status(); + match st.as_u16() { + s if (200..300).contains(&s) => Ok(true), + // 已经有了；409 是另一个条件写入正在写同一个键 + 412 | 409 => Ok(false), + 501 => self.put_checked(path, bytes).await, + 400 => { + let code = Self::error_code(r).await; + if code == "NotImplemented" || code == "InvalidArgument" { + self.put_checked(path, bytes).await + } else { + Err(http_status(st, "PUT")) + } + } + _ => Err(http_status(st, "PUT")), + } + } + + async fn put_checked(&self, path: &str, bytes: &[u8]) -> Result { + if self.exists(path).await? { + return Ok(false); + } + let url = self.object_url(&self.key(path)); + self.put_plain(&url, bytes).await.map(|()| true) + } + + async fn put_plain(&self, url: &str, bytes: &[u8]) -> Result<(), StoreError> { + let ct = ("content-type", "application/octet-stream"); + let r = self.send(Method::PUT, url, &[ct], bytes).await?; + if r.status().is_success() { + Ok(()) + } else { + Err(http_status(r.status(), "PUT")) + } + } +} + +impl Store for S3 { + async fn list(&self, dir: &str) -> Result, StoreError> { + check_path(dir)?; + let prefix = format!("{}{dir}/", self.prefix); + let mut out = Vec::new(); + let mut token: Option = None; + // 一页一千个；同步位置上最多几个版本、几十台设备，翻页只是兜底 + for _ in 0..100 { + let mut url = format!( + "{}?delimiter=%2F&list-type=2&prefix={}", + self.bucket_url, + utf8_percent_encode(&prefix, QUERY) + ); + if let Some(t) = &token { + url.push_str(&format!( + "&continuation-token={}", + utf8_percent_encode(t, QUERY) + )); + } + let r = self.send(Method::GET, &url, &[], b"").await?; + if !r.status().is_success() { + return Err(http_status(r.status(), "ListObjectsV2")); + } + let body = r.text().await.map_err(|e| http_error(&e))?; + let Some((records, top)) = super::xml::parse(&body, "Contents") else { + return Err(StoreError::Other(msg!( + "sync.s3.bad_listing" => + "the S3 location answered a listing that could not be read" + ))); + }; + for r in records { + let Some(key) = r.get("Key") else { continue }; + let Some(name) = key.strip_prefix(&prefix) else { + continue; + }; + if name.is_empty() || name.contains('/') || name.starts_with('.') { + continue; + } + out.push(Entry { + name: name.to_string(), + size: r.get("Size").and_then(|s| s.parse().ok()).unwrap_or(0), + }); + } + match ( + top.get("IsTruncated").map(String::as_str), + top.get("NextContinuationToken"), + ) { + (Some("true"), Some(t)) => token = Some(t.clone()), + _ => break, + } + } + out.sort_by(|a, b| a.name.cmp(&b.name)); + Ok(out) + } + + async fn get(&self, path: &str) -> Result>, StoreError> { + check_path(path)?; + let url = self.object_url(&self.key(path)); + let r = self.send(Method::GET, &url, &[], b"").await?; + match r.status().as_u16() { + 404 => Ok(None), + s if (200..300).contains(&s) => { + Ok(Some(r.bytes().await.map_err(|e| http_error(&e))?.to_vec())) + } + _ => Err(http_status(r.status(), "GET")), + } + } + + async fn put_new(&self, path: &str, bytes: &[u8]) -> Result<(), StoreError> { + check_path(path)?; + if self.put_object(path, bytes, true).await? { + Ok(()) + } else { + Err(StoreError::Exists { + path: path.to_string(), + }) + } + } + + async fn put(&self, path: &str, bytes: &[u8]) -> Result<(), StoreError> { + check_path(path)?; + self.put_object(path, bytes, false).await.map(|_| ()) + } + + async fn delete(&self, path: &str) -> Result<(), StoreError> { + check_path(path)?; + let url = self.object_url(&self.key(path)); + let r = self.send(Method::DELETE, &url, &[], b"").await?; + match r.status().as_u16() { + 404 => Ok(()), + s if (200..300).contains(&s) => Ok(()), + _ => Err(http_status(r.status(), "DELETE")), + } + } + + async fn create_exclusive(&self, path: &str, bytes: &[u8]) -> Result { + check_path(path)?; + self.put_object(path, bytes, true).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn s3(endpoint: &str, bucket: &str, prefix: &str) -> S3 { + S3::new( + Config { + endpoint: endpoint.into(), + region: "auto".into(), + bucket: bucket.into(), + prefix: prefix.into(), + access_key_id: "AK".into(), + secret_access_key: "SK".into(), + }, + reqwest::Client::new(), + ) + .unwrap() + } + + #[test] + fn the_address_style_follows_the_provider() { + let aws = s3("https://s3.us-east-1.amazonaws.com", "tw-b", ""); + assert_eq!(aws.bucket_url, "https://tw-b.s3.us-east-1.amazonaws.com/"); + let oss = s3("https://oss-cn-hangzhou.aliyuncs.com", "tw-b", "me"); + assert_eq!(oss.bucket_url, "https://tw-b.oss-cn-hangzhou.aliyuncs.com/"); + assert_eq!(oss.key("keyring"), "me/keyring"); + let pasted = s3( + "https://tw-b.cos.ap-guangzhou.myqcloud.com", + "tw-b", + "/a/b/", + ); + assert_eq!( + pasted.bucket_url, + "https://tw-b.cos.ap-guangzhou.myqcloud.com/" + ); + assert_eq!(pasted.key("versions/x"), "a/b/versions/x"); + let r2 = s3("https://abc123.r2.cloudflarestorage.com", "tw-b", ""); + assert_eq!( + r2.bucket_url, + "https://abc123.r2.cloudflarestorage.com/tw-b/" + ); + let minio = s3("http://192.168.1.10:9000", "tw-b", ""); + assert_eq!(minio.bucket_url, "http://192.168.1.10:9000/tw-b/"); + assert_eq!( + minio.object_url("versions/0000000001-aaaaaaaa.twconfig"), + "http://192.168.1.10:9000/tw-b/versions/0000000001-aaaaaaaa.twconfig" + ); + } + + #[test] + fn the_addressing_is_chosen_by_the_endpoint_host_alone() { + use Addressing::*; + for (host, want) in [ + ("s3.us-east-1.amazonaws.com", VirtualHost), + ("s3.amazonaws.com", VirtualHost), + ("amazonaws.com", VirtualHost), + ("oss-cn-hangzhou.aliyuncs.com", VirtualHost), + ("cos.ap-guangzhou.myqcloud.com", VirtualHost), + ("S3.EU-WEST-1.AMAZONAWS.COM", VirtualHost), + ("abc123.r2.cloudflarestorage.com", Path), + ("192.168.1.10", Path), + ("nas", Path), + ("minio.example.com", Path), + ("tw-b.minio.lan", Path), + ("notamazonaws.com", Path), + ("amazonaws.com.evil.example", Path), + ] { + assert_eq!(addressing(host), want, "{host}"); + } + assert_eq!( + bucket_url("http://tw-b.minio.lan:9000", "tw-b").unwrap(), + "http://tw-b.minio.lan:9000/tw-b/", + "路径方式不猜桶名是不是已经在域名里" + ); + assert_eq!( + bucket_url("https://minio.example.com/s3/", "tw-b").unwrap(), + "https://minio.example.com/s3/tw-b/" + ); + } + + #[test] + fn a_public_plain_http_endpoint_is_refused() { + let r = S3::new( + Config { + endpoint: "http://s3.example.com".into(), + region: "auto".into(), + bucket: "b-1".into(), + prefix: String::new(), + access_key_id: "AK".into(), + secret_access_key: "SK".into(), + }, + reqwest::Client::new(), + ); + assert!(r.is_err()); + } +} diff --git a/crates/tw-sync/src/store/webdav.rs b/crates/tw-sync/src/store/webdav.rs new file mode 100644 index 00000000..3b6a85fa --- /dev/null +++ b/crates/tw-sync/src/store/webdav.rs @@ -0,0 +1,366 @@ +//! WebDAV 位置：坚果云、Nextcloud、群晖 NAS、InfiniCloud。 +//! +//! 只用五个方法：`PROPFIND`（列目录，Depth 1）、`GET`、`PUT`、`DELETE`、`MOVE`，外加建目录的 +//! `MKCOL`。写入先 `PUT` 到一个点开头的临时名，再 `MOVE` 过去（只新增时带 `Overwrite: F`）； +//! 「仅当不存在时创建」用 `If-None-Match: *`。 +//! +//! **证书一律校验**，`http://` 只给本机和内网（见 [`tw_config::sync::address_ok`]）：这条 +//! 连接上传的是同步位置的密码，和可能没加密的整份配置。**不跟重定向**：跟到别的主机， +//! 密码就跟着去了。 + +use std::time::Duration; + +use reqwest::{Method, StatusCode}; +use tw_types::msg; + +use super::{Entry, Store, StoreError, check_path, http_error, http_status, split, temp_name}; + +/// 一个请求最多等多久。版本最大 16 MiB，慢的网盘也够了 +const TIMEOUT: Duration = Duration::from_secs(120); + +pub struct WebDav { + /// 以 `/` 结尾 + base: url::Url, + user: String, + pass: String, + http: reqwest::Client, +} + +fn method(m: &'static str) -> Method { + Method::from_bytes(m.as_bytes()).expect("a valid method name") +} + +impl WebDav { + pub fn new( + url: &str, + user: &str, + pass: &str, + http: reqwest::Client, + ) -> Result { + if !tw_config::sync::address_ok(url) { + return Err(StoreError::Other( + tw_config::sync::SyncError::Address { + field: "url", + url: url.to_string(), + } + .msg(), + )); + } + let mut s = url.trim().to_string(); + if !s.ends_with('/') { + s.push('/'); + } + let base = url::Url::parse(&s).map_err(|e| { + StoreError::Other(msg!( + "sync.webdav.bad_url", url = url, detail = e => + "{url} is not a WebDAV address: {detail}" + )) + })?; + Ok(Self { + base, + user: user.to_string(), + pass: pass.to_string(), + http, + }) + } + + /// 文件的地址。路径只含我们自己生成的字符，拼上去不用转义 + fn file(&self, path: &str) -> Result { + check_path(path)?; + self.base.join(path).map_err(|e| { + StoreError::Other(msg!( + "sync.webdav.bad_url", url = path, detail = e => + "{url} is not a WebDAV address: {detail}" + )) + }) + } + + /// 目录的地址，以 `/` 结尾。空的是根 + fn dir(&self, dir: &str) -> Result { + if dir.is_empty() { + return Ok(self.base.clone()); + } + let mut u = self.file(dir)?; + u.set_path(&format!("{}/", u.path().trim_end_matches('/'))); + Ok(u) + } + + fn req(&self, m: Method, url: url::Url) -> reqwest::RequestBuilder { + let r = self.http.request(m, url).timeout(TIMEOUT); + if self.user.is_empty() && self.pass.is_empty() { + r + } else { + r.basic_auth(&self.user, Some(&self.pass)) + } + } + + async fn send(&self, r: reqwest::RequestBuilder) -> Result { + r.send().await.map_err(|e| http_error(&e)) + } + + /// 建目录。已经有了（405）不算错；上一级没有（409）先建上一级（同步位置的根） + async fn mkcol(&self, dir: &str) -> Result<(), StoreError> { + let url = self.dir(dir)?; + let r = self.send(self.req(method("MKCOL"), url.clone())).await?; + match r.status().as_u16() { + 200 | 201 | 204 | 405 => Ok(()), + 409 if !dir.is_empty() => { + Box::pin(self.mkcol("")).await?; + let r = self.send(self.req(method("MKCOL"), url)).await?; + match r.status().as_u16() { + 200 | 201 | 204 | 405 => Ok(()), + _ => Err(http_status(r.status(), "MKCOL")), + } + } + _ => Err(http_status(r.status(), "MKCOL")), + } + } + + async fn put_raw( + &self, + url: url::Url, + bytes: &[u8], + if_none_match: bool, + ) -> Result { + let mut r = self + .req(Method::PUT, url) + .header("content-type", "application/octet-stream") + .body(bytes.to_vec()); + if if_none_match { + r = r.header("if-none-match", "*"); + } + Ok(self.send(r).await?.status()) + } + + /// 写临时名、改名过去。`overwrite` 是 MOVE 的 `Overwrite`。返回 MOVE 的状态，服务端 + /// 不支持 MOVE（405、501）时是 `None`，临时文件已经删掉 + async fn via_temp( + &self, + path: &str, + bytes: &[u8], + overwrite: bool, + ) -> Result, StoreError> { + let (dir, name) = split(path); + let tmp_path = if dir.is_empty() { + temp_name(name) + } else { + format!("{dir}/{}", temp_name(name)) + }; + let tmp = self.file(&tmp_path)?; + let dest = self.file(path)?; + let st = self.put_raw(tmp.clone(), bytes, false).await?; + if !st.is_success() { + return Err(http_status(st, "PUT")); + } + let r = self + .send( + self.req(method("MOVE"), tmp.clone()) + .header("destination", dest.as_str()) + .header("overwrite", if overwrite { "T" } else { "F" }), + ) + .await; + let st = match r { + Ok(r) => r.status(), + Err(e) => { + let _ = self.send(self.req(Method::DELETE, tmp)).await; + return Err(e); + } + }; + if st.is_success() { + return Ok(Some(st)); + } + let _ = self.send(self.req(Method::DELETE, tmp)).await; + if matches!(st.as_u16(), 405 | 501) { + return Ok(None); + } + Ok(Some(st)) + } + + async fn exists(&self, path: &str) -> Result { + let r = self.send(self.req(Method::HEAD, self.file(path)?)).await?; + match r.status().as_u16() { + 404 => Ok(false), + s if (200..300).contains(&s) => Ok(true), + _ => Err(http_status(r.status(), "HEAD")), + } + } +} + +/// `href` 里最后一段，解码之后。目录以 `/` 结尾，去掉再取 +fn last_segment(href: &str) -> String { + let path = match url::Url::parse(href) { + Ok(u) => u.path().to_string(), + Err(_) => href.to_string(), + }; + let seg = path.trim_end_matches('/').rsplit('/').next().unwrap_or(""); + percent_encoding::percent_decode_str(seg) + .decode_utf8_lossy() + .to_string() +} + +const PROPFIND: &str = r#" +"#; + +impl Store for WebDav { + async fn list(&self, dir: &str) -> Result, StoreError> { + let url = self.dir(dir)?; + let own = url.path().trim_end_matches('/').to_string(); + let r = self + .send( + self.req(method("PROPFIND"), url) + .header("depth", "1") + .header("content-type", "application/xml; charset=utf-8") + .body(PROPFIND), + ) + .await?; + match r.status().as_u16() { + 207 | 200 => {} + 404 => return Ok(Vec::new()), + _ => return Err(http_status(r.status(), "PROPFIND")), + } + let body = r.text().await.map_err(|e| http_error(&e))?; + let Some((records, _)) = super::xml::parse(&body, "response") else { + return Err(StoreError::Other(msg!( + "sync.webdav.bad_listing" => + "the WebDAV server answered a directory listing that could not be read" + ))); + }; + let mut out = Vec::new(); + for r in records { + let Some(href) = r.get("href") else { continue }; + let path = match url::Url::parse(href) { + Ok(u) => u.path().to_string(), + Err(_) => href.to_string(), + }; + if r.has("collection") || path.trim_end_matches('/') == own { + continue; + } + let name = last_segment(href); + if name.is_empty() || name.starts_with('.') { + continue; + } + out.push(Entry { + name, + size: r + .get("getcontentlength") + .and_then(|s| s.parse().ok()) + .unwrap_or(0), + }); + } + out.sort_by(|a, b| a.name.cmp(&b.name)); + Ok(out) + } + + async fn get(&self, path: &str) -> Result>, StoreError> { + let r = self.send(self.req(Method::GET, self.file(path)?)).await?; + match r.status().as_u16() { + 404 | 410 => Ok(None), + s if (200..300).contains(&s) => { + Ok(Some(r.bytes().await.map_err(|e| http_error(&e))?.to_vec())) + } + _ => Err(http_status(r.status(), "GET")), + } + } + + async fn put_new(&self, path: &str, bytes: &[u8]) -> Result<(), StoreError> { + let (dir, _) = split(path); + self.mkcol(dir).await?; + let exists = || StoreError::Exists { + path: path.to_string(), + }; + match self.via_temp(path, bytes, false).await? { + Some(st) if st.is_success() => Ok(()), + Some(st) if st.as_u16() == 412 => Err(exists()), + Some(st) => Err(http_status(st, "MOVE")), + None => { + // 不支持 MOVE 的服务器：直接写，带上「不存在才写」 + let st = self.put_raw(self.file(path)?, bytes, true).await?; + match st.as_u16() { + 412 => Err(exists()), + s if (200..300).contains(&s) => Ok(()), + _ => Err(http_status(st, "PUT")), + } + } + } + } + + async fn put(&self, path: &str, bytes: &[u8]) -> Result<(), StoreError> { + let (dir, _) = split(path); + self.mkcol(dir).await?; + match self.via_temp(path, bytes, true).await? { + Some(st) if st.is_success() => Ok(()), + Some(st) => Err(http_status(st, "MOVE")), + None => { + let st = self.put_raw(self.file(path)?, bytes, false).await?; + if st.is_success() { + Ok(()) + } else { + Err(http_status(st, "PUT")) + } + } + } + } + + async fn delete(&self, path: &str) -> Result<(), StoreError> { + let r = self + .send(self.req(Method::DELETE, self.file(path)?)) + .await?; + match r.status().as_u16() { + 404 | 410 => Ok(()), + s if (200..300).contains(&s) => Ok(()), + _ => Err(http_status(r.status(), "DELETE")), + } + } + + async fn create_exclusive(&self, path: &str, bytes: &[u8]) -> Result { + let (dir, _) = split(path); + self.mkcol(dir).await?; + // 先看一眼：有的服务器不认 If-None-Match，会直接覆盖 + if self.exists(path).await? { + return Ok(false); + } + let st = self.put_raw(self.file(path)?, bytes, true).await?; + match st.as_u16() { + 412 => Ok(false), + s if (200..300).contains(&s) => Ok(true), + _ => Err(http_status(st, "PUT")), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_plain_http_address_off_the_local_network_is_refused() { + let http = reqwest::Client::new(); + assert!(WebDav::new("http://dav.example.com/x", "u", "p", http.clone()).is_err()); + assert!(WebDav::new("https://dav.example.com/x", "u", "p", http.clone()).is_ok()); + assert!(WebDav::new("http://192.168.1.2:5005/x", "u", "p", http).is_ok()); + } + + #[test] + fn hrefs_are_decoded_whether_absolute_or_not() { + assert_eq!(last_segment("/dav/a%20b/versions/x.twconfig"), "x.twconfig"); + assert_eq!(last_segment("https://h/dav/tw/devices/"), "devices"); + assert_eq!(last_segment("/dav/%E5%90%8C%E6%AD%A5/"), "同步"); + } + + #[test] + fn files_and_directories_resolve_under_the_base() { + let d = WebDav::new("https://h/dav/ThinkWatch", "u", "p", reqwest::Client::new()).unwrap(); + assert_eq!( + d.file("versions/0000000001-aaaaaaaa.twconfig") + .unwrap() + .as_str(), + "https://h/dav/ThinkWatch/versions/0000000001-aaaaaaaa.twconfig" + ); + assert_eq!( + d.dir("versions").unwrap().as_str(), + "https://h/dav/ThinkWatch/versions/" + ); + assert_eq!(d.dir("").unwrap().as_str(), "https://h/dav/ThinkWatch/"); + assert!(d.file("../x").is_err()); + } +} diff --git a/crates/tw-sync/src/store/xml.rs b/crates/tw-sync/src/store/xml.rs new file mode 100644 index 00000000..7d9642aa --- /dev/null +++ b/crates/tw-sync/src/store/xml.rs @@ -0,0 +1,160 @@ +//! 读 WebDAV 的 PROPFIND 和 S3 的 ListObjectsV2、错误答复：只要几个元素的文字。 +//! +//! **按本地名认元素，不管命名空间前缀**：同一个 `DAV:` 命名空间，有的服务器写 `D:href`， +//! 有的写 `d:href`、`lp1:getcontentlength`，还有的不带前缀。 + +use std::collections::BTreeMap; + +use quick_xml::events::Event; + +/// 一条记录：一个 `record` 元素底下各个元素的文字（同名的取第一个），以及出现过的空元素 +/// （`` 这种只看在不在的） +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct Record { + pub text: BTreeMap, + pub present: std::collections::BTreeSet, +} + +impl Record { + pub fn get(&self, name: &str) -> Option<&str> { + self.text.get(name).map(String::as_str) + } + pub fn has(&self, name: &str) -> bool { + self.present.contains(name) || self.text.contains_key(name) + } +} + +/// 预定义的五个实体和字符引用。认不出的原样留着 +fn entity(name: &str) -> String { + match name { + "amp" => "&".into(), + "lt" => "<".into(), + "gt" => ">".into(), + "quot" => "\"".into(), + "apos" => "'".into(), + _ => { + let n = name + .strip_prefix("#x") + .or_else(|| name.strip_prefix("#X")) + .and_then(|h| u32::from_str_radix(h, 16).ok()) + .or_else(|| name.strip_prefix('#').and_then(|d| d.parse().ok())); + match n.and_then(char::from_u32) { + Some(c) => c.to_string(), + None => format!("&{name};"), + } + } + } +} + +/// 文档里每个本地名是 `record` 的元素一条记录；`top` 是不在任何记录里的那些元素的文字。 +/// 读不了的 XML 返回 `None` +pub fn parse(xml: &str, record: &str) -> Option<(Vec, BTreeMap)> { + let mut reader = quick_xml::Reader::from_str(xml); + let mut stack: Vec = Vec::new(); + let mut records = Vec::new(); + let mut top = BTreeMap::new(); + // 正在读的那一条记录，以及它在栈里的深度 + let mut cur: Option<(usize, Record)> = None; + let mut text = String::new(); + loop { + match reader.read_event().ok()? { + Event::Start(e) => { + let name = e.local_name().as_ref().to_string(); + stack.push(name.clone()); + text.clear(); + if name == record && cur.is_none() { + cur = Some((stack.len(), Record::default())); + } + } + Event::Empty(e) => { + let name = e.local_name().as_ref().to_string(); + if let Some((_, r)) = cur.as_mut() { + r.present.insert(name); + } + } + Event::Text(t) => text.push_str(&t), + Event::CData(t) => text.push_str(&t), + Event::GeneralRef(r) => text.push_str(&entity(&r)), + Event::End(_) => { + let name = stack.pop()?; + let value = std::mem::take(&mut text).trim().to_string(); + match cur.as_mut() { + Some((depth, _)) if *depth == stack.len() + 1 => { + if let Some((_, r)) = cur.take() { + records.push(r); + } + } + Some((_, r)) => { + r.present.insert(name.clone()); + if !value.is_empty() { + r.text.entry(name).or_insert(value); + } + } + None => { + if !value.is_empty() { + top.entry(name).or_insert(value); + } + } + } + } + Event::Eof => break, + _ => {} + } + } + Some((records, top)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn webdav_multistatus_with_any_prefix() { + let xml = r#" + + + /dav/tw/versions/ + + + + /dav/tw/versions/0000000001-aaaaaaaa.twconfig + + 1234 + + HTTP/1.1 200 OK + +"#; + let (rs, _) = parse(xml, "response").unwrap(); + assert_eq!(rs.len(), 2); + assert!(rs[0].has("collection")); + assert_eq!( + rs[1].get("href"), + Some("/dav/tw/versions/0000000001-aaaaaaaa.twconfig") + ); + assert_eq!(rs[1].get("getcontentlength"), Some("1234")); + assert!(!rs[1].has("collection")); + } + + #[test] + fn s3_listing_and_its_continuation() { + let xml = r#" +bp/versions/true +tok&en +p/versions/a3 +p/versions/b&c4 +"#; + let (rs, top) = parse(xml, "Contents").unwrap(); + assert_eq!(rs.len(), 2); + assert_eq!(rs[1].get("Key"), Some("p/versions/b&c")); + assert_eq!(top.get("IsTruncated").map(String::as_str), Some("true")); + assert_eq!( + top.get("NextContinuationToken").map(String::as_str), + Some("tok&en") + ); + } + + #[test] + fn broken_xml_is_none_not_a_panic() { + assert!(parse("", "b").is_none()); + } +} diff --git a/crates/tw-sync/tests/common/mod.rs b/crates/tw-sync/tests/common/mod.rs new file mode 100644 index 00000000..8f758d12 --- /dev/null +++ b/crates/tw-sync/tests/common/mod.rs @@ -0,0 +1,323 @@ +//! 进程内的假 WebDAV、假 S3：内容放在内存里，几十行。 +//! +//! **只实现同步用到的那几样**，但按规矩答：`PUT` 认 `If-None-Match: *`、`MOVE` 认 +//! `Overwrite: F`、`PROPFIND` 只列一层、S3 的列目录认 `prefix` 和 `delimiter`。凭据不对回 401 / +//! 403。可以让它「坏掉」：回 503，或者不认 `MOVE`、`If-None-Match`（有的服务器就是这样）。 + +#![allow(dead_code)] + +use std::collections::BTreeMap; +use std::net::SocketAddr; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; + +use axum::body::Body; +use axum::http::{HeaderMap, Request, Response, StatusCode}; + +pub const USER: &str = "tester"; +pub const PASS: &str = "app-password"; +pub const AK: &str = "AKIDFAKESYNCTEST"; +pub const SK: &str = "fake-secret-for-tests-only"; + +/// 假服务端的开关 +#[derive(Default)] +pub struct Knobs { + /// 一律回 503 + pub down: AtomicBool, + /// WebDAV 不认 MOVE（405） + pub no_move: AtomicBool, + /// 不认 `If-None-Match`：照写不误（WebDAV）或者回 501（S3） + pub no_conditional: AtomicBool, + /// 收到的请求数 + pub requests: AtomicUsize, +} + +pub struct Fake { + pub addr: SocketAddr, + /// 路径（不带开头的 `/`，WebDAV 是去掉根之后的，S3 是桶内的键）→ 内容 + pub files: Arc>>>, + pub knobs: Arc, +} + +impl Fake { + pub fn url(&self) -> String { + format!("http://{}", self.addr) + } + pub fn names(&self) -> Vec { + self.files.lock().unwrap().keys().cloned().collect() + } + pub fn get(&self, path: &str) -> Option> { + self.files.lock().unwrap().get(path).cloned() + } +} + +async fn serve( + handler: impl Fn(Request, Vec) -> Response + Send + Sync + 'static, +) -> SocketAddr { + let handler = Arc::new(handler); + let app = axum::Router::new().fallback(move |req: Request| { + let handler = handler.clone(); + async move { + let (parts, body) = req.into_parts(); + let bytes = axum::body::to_bytes(body, usize::MAX) + .await + .unwrap_or_default() + .to_vec(); + handler(Request::from_parts(parts, Body::empty()), bytes) + } + }); + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = l.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(l, app).await.unwrap() }); + addr +} + +fn status(s: u16) -> Response { + Response::builder() + .status(StatusCode::from_u16(s).unwrap()) + .body(Body::empty()) + .unwrap() +} + +fn header<'a>(h: &'a HeaderMap, name: &str) -> Option<&'a str> { + h.get(name).and_then(|v| v.to_str().ok()) +} + +/// WebDAV，根在 `/dav/root/`。 +pub async fn webdav() -> Fake { + let files: Arc>>> = Default::default(); + let dirs: Arc>> = Default::default(); + let knobs: Arc = Default::default(); + let (f, d, k) = (files.clone(), dirs.clone(), knobs.clone()); + let addr = serve(move |req, body| { + k.requests.fetch_add(1, Ordering::Relaxed); + if k.down.load(Ordering::Relaxed) { + return status(503); + } + use base64::Engine as _; + let want = format!( + "Basic {}", + base64::engine::general_purpose::STANDARD.encode(format!("{USER}:{PASS}")) + ); + if header(req.headers(), "authorization") != Some(want.as_str()) { + return status(401); + } + let Some(rel) = req.uri().path().strip_prefix("/dav/root/") else { + // 根本身 + if req.uri().path().trim_end_matches('/') == "/dav/root" { + return match req.method().as_str() { + "MKCOL" => status(405), + "PROPFIND" => status(207), + _ => status(405), + }; + } + return status(404); + }; + let rel = percent_encoding::percent_decode_str(rel) + .decode_utf8_lossy() + .to_string(); + let mut files = f.lock().unwrap(); + let mut dirs = d.lock().unwrap(); + match req.method().as_str() { + "MKCOL" => { + let dir = rel.trim_end_matches('/').to_string(); + if dir.contains('/') && !dirs.contains(dir.rsplit_once('/').unwrap().0) { + return status(409); + } + if dirs.insert(dir) { status(201) } else { status(405) } + } + "PROPFIND" => { + let dir = rel.trim_end_matches('/').to_string(); + if !dirs.contains(&dir) { + return status(404); + } + let mut xml = String::from( + r#""#, + ); + xml.push_str(&format!( + "/dav/root/{dir}/\ + " + )); + for (name, bytes) in files.iter() { + if let Some((parent, leaf)) = name.rsplit_once('/') + && parent == dir + { + xml.push_str(&format!( + "/dav/root/{parent}/{leaf}\ + {}\ + HTTP/1.1 200 OK", + bytes.len() + )); + } + } + xml.push_str(""); + Response::builder() + .status(207) + .header("content-type", "application/xml") + .body(Body::from(xml)) + .unwrap() + } + "GET" | "HEAD" => match files.get(&rel) { + Some(b) if req.method() == "GET" => Response::new(Body::from(b.clone())), + Some(_) => status(200), + None => status(404), + }, + "PUT" => { + let parent = rel.rsplit_once('/').map(|(p, _)| p).unwrap_or(""); + if !parent.is_empty() && !dirs.contains(parent) { + return status(409); + } + if header(req.headers(), "if-none-match") == Some("*") + && !k.no_conditional.load(Ordering::Relaxed) + && files.contains_key(&rel) + { + return status(412); + } + let existed = files.insert(rel, body).is_some(); + status(if existed { 204 } else { 201 }) + } + "DELETE" => match files.remove(&rel) { + Some(_) => status(204), + None => status(404), + }, + "MOVE" => { + if k.no_move.load(Ordering::Relaxed) { + return status(405); + } + let Some(dest) = header(req.headers(), "destination") else { + return status(400); + }; + let dest_path = url::Url::parse(dest) + .map(|u| u.path().to_string()) + .unwrap_or_else(|_| dest.to_string()); + let Some(dest_rel) = dest_path.strip_prefix("/dav/root/") else { + return status(502); + }; + let dest_rel = dest_rel.to_string(); + let overwrite = header(req.headers(), "overwrite") != Some("F"); + if !overwrite && files.contains_key(&dest_rel) { + return status(412); + } + let Some(b) = files.remove(&rel) else { + return status(404); + }; + let existed = files.insert(dest_rel, b).is_some(); + status(if existed { 204 } else { 201 }) + } + _ => status(405), + } + }) + .await; + // 根在 + Fake { addr, files, knobs } +} + +/// S3，路径方式，桶叫 `tw-bucket`。 +pub async fn s3() -> Fake { + let files: Arc>>> = Default::default(); + let knobs: Arc = Default::default(); + let (f, k) = (files.clone(), knobs.clone()); + let addr = serve(move |req, body| { + k.requests.fetch_add(1, Ordering::Relaxed); + if k.down.load(Ordering::Relaxed) { + return status(503); + } + let auth = header(req.headers(), "authorization").unwrap_or_default(); + if !auth.starts_with(&format!("AWS4-HMAC-SHA256 Credential={AK}/")) + || !auth.contains("/s3/aws4_request") + || header(req.headers(), "x-amz-content-sha256").is_none() + { + return Response::builder() + .status(403) + .body(Body::from( + "SignatureDoesNotMatch", + )) + .unwrap(); + } + let path = req.uri().path().to_string(); + let Some(key) = path.strip_prefix("/tw-bucket") else { + return status(404); + }; + let key = percent_encoding::percent_decode_str(key.trim_start_matches('/')) + .decode_utf8_lossy() + .to_string(); + let mut files = f.lock().unwrap(); + match req.method().as_str() { + "GET" if key.is_empty() => { + let q: BTreeMap = url::form_urlencoded::parse( + req.uri().query().unwrap_or_default().as_bytes(), + ) + .into_owned() + .collect(); + assert_eq!(q.get("list-type").map(String::as_str), Some("2")); + let prefix = q.get("prefix").cloned().unwrap_or_default(); + let delim = q.get("delimiter").cloned(); + let mut xml = String::from( + r#"false"#, + ); + for (name, bytes) in files.iter() { + let Some(rest) = name.strip_prefix(&prefix) else { + continue; + }; + if delim.as_deref() == Some("/") && rest.contains('/') { + continue; + } + xml.push_str(&format!( + "{name}{}", + bytes.len() + )); + } + xml.push_str(""); + Response::new(Body::from(xml)) + } + "GET" => match files.get(&key) { + Some(b) => Response::new(Body::from(b.clone())), + None => Response::builder() + .status(404) + .body(Body::from("NoSuchKey")) + .unwrap(), + }, + "HEAD" => status(if files.contains_key(&key) { 200 } else { 404 }), + "PUT" => { + if header(req.headers(), "if-none-match") == Some("*") { + if k.no_conditional.load(Ordering::Relaxed) { + return Response::builder() + .status(501) + .body(Body::from("NotImplemented")) + .unwrap(); + } + if files.contains_key(&key) { + return status(412); + } + } + files.insert(key, body); + status(200) + } + "DELETE" => { + files.remove(&key); + status(204) + } + _ => status(405), + } + }) + .await; + Fake { addr, files, knobs } +} + +/// 一个没人听的地址：连上去就被拒绝 +pub fn nowhere() -> String { + let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let a = l.local_addr().unwrap(); + drop(l); + format!("http://{a}") +} + +/// 测试里的 HTTP 客户端：不走系统代理，User-Agent 用通用值 +pub fn http() -> reqwest::Client { + reqwest::Client::builder() + .no_proxy() + .user_agent("sync-test") + .redirect(reqwest::redirect::Policy::none()) + .build() + .unwrap() +} diff --git a/crates/tw-sync/tests/devices.rs b/crates/tw-sync/tests/devices.rs new file mode 100644 index 00000000..add592cf --- /dev/null +++ b/crates/tw-sync/tests/devices.rs @@ -0,0 +1,747 @@ +//! 两台设备（两个独立的数据目录、两份配置）经同一个同步位置交替修改、同时上传、冲突、离线、 +//! 加密开关与改口令、版本过新、不同步的上游、已接管客户端的密钥。 +//! +//! 设备这一边（[`Host`]）是最简单的那种：配置就是数据目录里的 `config.yaml`，应用就是校验 +//! 之后写回去。控制面那一层（写入流程、事件）在 `tw-control` 的测试里另测。 + +mod common; + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; + +use tw_pack::{Pack, PackKind, PackOrigin}; +use tw_sync::engine::{ApplyError, Current, Decision, Engine, Host, RoundError, Settings, Take}; +use tw_sync::store::{AnyStore, Folder}; +use tw_sync::{Store, StoreError}; +use tw_types::Msg; + +const PASS: &str = "correct horse battery staple"; + +/// 一台设备的配置：控制面钥匙各不相同（本机设置，不该被同步走） +fn base_config(control_key_byte: char) -> String { + format!( + "version: 1\nlisten:\n control:\n key: {}\nclients:\n - name: default\n key: tw-default-key-0001\n", + control_key_byte.to_string().repeat(64) + ) +} + +#[derive(Clone)] +struct Dev { + dir: Arc, + pinned: Arc>>, + /// 应用过几次，每次是从哪些设备来的 + applied: Arc>>, +} + +impl Dev { + fn new(key: char) -> Self { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("config.yaml"), base_config(key)).unwrap(); + Self { + dir: Arc::new(dir), + pinned: Default::default(), + applied: Default::default(), + } + } + fn path(&self) -> PathBuf { + self.dir.path().join("config.yaml") + } + fn text(&self) -> String { + std::fs::read_to_string(self.path()).unwrap() + } + fn config(&self) -> tw_config::Config { + tw_config::try_parse(&self.text()).unwrap_or_else(|e| panic!("{e}\n{}", self.text())) + } + /// 用户改配置：在原文上追加或者整段改 + fn edit(&self, f: impl FnOnce(&str) -> String) { + let next = f(&self.text()); + tw_config::try_parse(&next).unwrap_or_else(|e| panic!("测试写坏了配置：{e}\n{next}")); + std::fs::write(self.path(), next).unwrap(); + } + fn provider(&self, name: &str) -> Option { + self.config().providers.into_iter().find(|p| p.name == name) + } +} + +impl Host for Dev { + fn current(&self) -> Result { + let text = self.text(); + Ok(Current { + pack: Pack { + manifest: tw_pack::Manifest::new(PackKind::Full, PackOrigin::Ui), + config_yaml: text.clone(), + plugins: Default::default(), + client_prefs: None, + }, + version: tw_config::version_of(&text), + changed_ms: 1, + }) + } + async fn apply(&self, pack: Pack, base_version: &str, from: &str) -> Result<(), ApplyError> { + if tw_config::version_of(&self.text()) != base_version { + return Err(ApplyError::Stale); + } + tw_config::try_parse(&pack.config_yaml).map_err(|r| ApplyError::Rejected(r.msg()))?; + std::fs::write(self.path(), &pack.config_yaml).unwrap(); + self.applied.lock().unwrap().push(from.to_string()); + Ok(()) + } + fn pinned_keys(&self) -> BTreeSet { + self.pinned.lock().unwrap().clone() + } + fn backup(&self, _content_id: &str) -> Option { + None + } +} + +fn settings(id: &str, name: &str, encrypt: bool, place: &Path) -> Settings { + Settings { + device_id: id.into(), + device_name: name.into(), + encrypt, + passphrase: encrypt.then(|| PASS.to_string()), + exclude: BTreeSet::new(), + place: format!("folder:{}", place.display()), + remote_core: false, + } +} + +const A_ID: &str = "aaaaaaaa-0000-4000-8000-000000000001"; +const B_ID: &str = "bbbbbbbb-0000-4000-8000-000000000002"; + +struct Bed { + loc: tempfile::TempDir, + a: Dev, + b: Dev, + ea: Engine, + eb: Engine, +} + +fn folder(loc: &Path) -> AnyStore { + AnyStore::Folder(Folder::new(loc)) +} + +/// A 开启同步（同步位置空着），B 用同步数据加入 +async fn bed(encrypt: bool) -> Bed { + tw_sync::keyring::set_work_factor_for_tests(4); + let loc = tempfile::tempdir().unwrap(); + let (a, b) = (Dev::new('a'), Dev::new('b')); + let mut ea = Engine::new( + folder(loc.path()), + a.clone(), + a.dir.path(), + settings(A_ID, "Mac mini", encrypt, loc.path()), + ); + let mut eb = Engine::new( + folder(loc.path()), + b.clone(), + b.dir.path(), + settings(B_ID, "MacBook Air", encrypt, loc.path()), + ); + assert!(ea.create().await.unwrap(), "空的同步位置开得起来"); + eb.join(None).await.unwrap(); + Bed { loc, a, b, ea, eb } +} + +fn provider_yaml(name: &str, url: &str, key: &str) -> String { + format!(" - name: {name}\n base_url: {url}\n key: {key}\n") +} + +fn add_provider(text: &str, name: &str, url: &str, key: &str) -> String { + if text.contains("\nproviders:\n") { + text.replacen( + "\nproviders:\n", + &format!("\nproviders:\n{}", provider_yaml(name, url, key)), + 1, + ) + } else { + format!("{text}providers:\n{}", provider_yaml(name, url, key)) + } +} + +fn versions(loc: &Path) -> Vec { + let mut v: Vec = std::fs::read_dir(loc.join("versions")) + .map(|rd| { + rd.filter_map(|e| e.ok()) + .map(|e| e.file_name().to_string_lossy().to_string()) + .filter(|n| !n.starts_with('.')) + .collect() + }) + .unwrap_or_default(); + v.sort(); + v +} + +#[tokio::test] +async fn edits_on_either_device_reach_the_other_and_local_settings_stay_put() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider(s, "relay", "https://relay.example/v1", "sk-relay")); + let r = t.ea.round().await.unwrap(); + assert!(r.uploaded.is_some(), "A 的改动要上传"); + let r = t.eb.round().await.unwrap(); + assert_eq!(r.applied_from, ["Mac mini"]); + let p = t.b.provider("relay").expect("B 收到了 relay"); + assert_eq!( + p.key.as_ref().map(|k| k.raw()), + Some("sk-relay"), + "密钥随同步带过去" + ); + assert!( + t.b.text().contains(&"b".repeat(64)), + "B 的控制面钥匙是它自己的" + ); + + // B 改，A 收 + t.b.edit(|s| s.replace("https://relay.example/v1", "https://relay2.example/v1")); + assert!(t.eb.round().await.unwrap().uploaded.is_some()); + t.ea.round().await.unwrap(); + assert_eq!( + t.a.provider("relay").unwrap().base_url, + "https://relay2.example/v1" + ); + assert!(t.a.text().contains(&"a".repeat(64))); + + // 都没变：不再上传、不再应用 + let before = versions(t.loc.path()); + let ra = t.ea.round().await.unwrap(); + let rb = t.eb.round().await.unwrap(); + assert!(ra.uploaded.is_none() && rb.uploaded.is_none()); + assert!(ra.applied_from.is_empty() && rb.applied_from.is_empty()); + assert_eq!(versions(t.loc.path()), before); + // 同步位置只留最新与上一个 + assert!(before.len() <= 2, "{before:?}"); + // 两台设备都在设备列表里 + let names: Vec = rb.devices.iter().map(|d| d.name.clone()).collect(); + assert_eq!(names, ["Mac mini", "MacBook Air"]); +} + +#[tokio::test] +async fn different_objects_changed_on_both_sides_merge_without_a_conflict() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider(s, "one", "https://one.example", "sk-1")); + t.b.edit(|s| add_provider(s, "two", "https://two.example", "sk-2")); + t.ea.round().await.unwrap(); + let r = t.eb.round().await.unwrap(); + assert!(r.conflicts.is_empty(), "{:?}", r.conflicts); + assert!(r.uploaded.is_some() && !r.applied_from.is_empty()); + t.ea.round().await.unwrap(); + for d in [&t.a, &t.b] { + assert!( + d.provider("one").is_some() && d.provider("two").is_some(), + "{}", + d.text() + ); + } +} + +/// 一个存储，**假装还没收到某些文件**：云盘里别的设备的版本还没传到这台设备上 +struct Lagging { + inner: AnyStore, + hidden: Arc>>, +} + +impl Store for Lagging { + async fn list(&self, dir: &str) -> Result, StoreError> { + let hidden = self.hidden.lock().unwrap().clone(); + Ok(self + .inner + .list(dir) + .await? + .into_iter() + .filter(|e| !hidden.contains(&e.name)) + .collect()) + } + async fn get(&self, path: &str) -> Result>, StoreError> { + if self + .hidden + .lock() + .unwrap() + .iter() + .any(|h| path.ends_with(h.as_str())) + { + return Ok(None); + } + self.inner.get(path).await + } + async fn put_new(&self, path: &str, bytes: &[u8]) -> Result<(), StoreError> { + self.inner.put_new(path, bytes).await + } + async fn put(&self, path: &str, bytes: &[u8]) -> Result<(), StoreError> { + self.inner.put(path, bytes).await + } + async fn delete(&self, path: &str) -> Result<(), StoreError> { + self.inner.delete(path).await + } + async fn create_exclusive(&self, path: &str, bytes: &[u8]) -> Result { + self.inner.create_exclusive(path, bytes).await + } +} + +/// 两台设备同时上传：序号相同的两个版本，第三台（这里是 A 自己的下一轮）以它们共同的上一版本 +/// 为基准合并；合并之前它们和共同的上一版本都留着 +#[tokio::test] +async fn simultaneous_uploads_are_merged_on_their_common_parent() { + tw_sync::keyring::set_work_factor_for_tests(4); + let t = bed(false).await; + let hidden: Arc>> = Default::default(); + let mut eb = Engine::new( + Lagging { + inner: folder(t.loc.path()), + hidden: hidden.clone(), + }, + t.b.clone(), + t.b.dir.path(), + settings(B_ID, "MacBook Air", false, t.loc.path()), + ); + let mut ea = t.ea; + t.a.edit(|s| add_provider(s, "one", "https://one.example", "sk-1")); + t.b.edit(|s| add_provider(s, "two", "https://two.example", "sk-2")); + let up_a = ea.round().await.unwrap().uploaded.unwrap(); + // B 看不到 A 刚传的那一版：它也传一个同序号的 + hidden.lock().unwrap().insert(format!("{up_a}.twconfig")); + let up_b = eb.round().await.unwrap().uploaded.unwrap(); + assert_eq!(up_a[..10], up_b[..10], "同一个序号：{up_a} / {up_b}"); + let listed = versions(t.loc.path()); + assert_eq!( + listed.len(), + 3, + "两个并列的头和它们共同的上一版本都在：{listed:?}" + ); + + // 文件到了。A 的下一轮看到两个头，合并成一个新版本 + hidden.lock().unwrap().clear(); + let r = ea.round().await.unwrap(); + assert!(r.conflicts.is_empty(), "{:?}", r.conflicts); + assert!(r.uploaded.is_some(), "并列的版本要合并成一个"); + assert!(t.a.provider("one").is_some() && t.a.provider("two").is_some()); + eb.round().await.unwrap(); + assert!(t.b.provider("one").is_some() && t.b.provider("two").is_some()); + let listed = versions(t.loc.path()); + assert!( + listed.len() <= 3, + "合并之后共同的上一版本删掉了：{listed:?}" + ); +} + +/// 同一个上游的连接两边改得不一样：冲突。两台设备看到同一处冲突；冲突期间各用各的； +/// 一台处理之后，另一台随之清除，内容一致 +#[tokio::test] +async fn a_conflict_is_the_same_on_both_devices_and_handled_once() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider(s, "relay", "https://relay.example", "sk-base")); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + + t.a.edit(|s| s.replace("sk-base", "sk-from-a")); + t.b.edit(|s| s.replace("sk-base", "sk-from-b")); + // 另一个上游上互不相干的改动照常合并 + t.b.edit(|s| add_provider(s, "other", "https://other.example", "sk-o")); + t.ea.round().await.unwrap(); + let rb = t.eb.round().await.unwrap(); + assert_eq!(rb.conflicts.len(), 1, "{:?}", rb.conflicts); + let c = &rb.conflicts[0]; + assert_eq!(c.item.name, "relay"); + assert_eq!(c.units, ["conn"]); + // 冲突期间各用各的 + assert_eq!( + t.b.provider("relay").unwrap().key.unwrap().raw(), + "sk-from-b" + ); + let ra = t.ea.round().await.unwrap(); + assert_eq!(ra.conflicts.len(), 1); + assert_eq!(ra.conflicts[0].id, c.id, "两台设备看到的是同一处冲突"); + assert_eq!( + t.a.provider("relay").unwrap().key.unwrap().raw(), + "sk-from-a" + ); + assert!(t.a.provider("other").is_some(), "不相干的改动照常合并"); + // 再同步几轮也不会反复上传 + let before = versions(t.loc.path()); + t.eb.round().await.unwrap(); + t.ea.round().await.unwrap(); + assert_eq!(versions(t.loc.path()), before); + + // A 选 B 的 + let ra = + t.ea.round_with(&[Decision { + id: c.id.clone(), + take: Take::Remote, + new_name: None, + }]) + .await + .unwrap(); + assert!(ra.conflicts.is_empty()); + assert_eq!( + t.a.provider("relay").unwrap().key.unwrap().raw(), + "sk-from-b" + ); + let rb = t.eb.round().await.unwrap(); + assert!(rb.conflicts.is_empty(), "B 上的这处冲突随之清除"); + assert_eq!( + t.b.provider("relay").unwrap().key.unwrap().raw(), + "sk-from-b" + ); +} + +#[tokio::test] +async fn keeping_both_adds_the_other_side_under_the_device_name() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider(s, "relay", "https://relay.example", "sk-base")); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + t.a.edit(|s| s.replace("https://relay.example", "https://a.example")); + t.b.edit(|s| s.replace("https://relay.example", "https://b.example")); + t.ea.round().await.unwrap(); + let rb = t.eb.round().await.unwrap(); + let id = rb.conflicts[0].id.clone(); + let rb = + t.eb.round_with(&[Decision { + id, + take: Take::Both, + new_name: None, + }]) + .await + .unwrap(); + assert!(rb.conflicts.is_empty()); + assert_eq!(t.b.provider("relay").unwrap().base_url, "https://b.example"); + assert_eq!( + t.b.provider("relay (Mac mini)").unwrap().base_url, + "https://a.example" + ); + t.ea.round().await.unwrap(); + assert_eq!(t.a.provider("relay").unwrap().base_url, "https://b.example"); + assert_eq!( + t.a.provider("relay (Mac mini)").unwrap().base_url, + "https://a.example" + ); +} + +/// 一边删了、另一边改了：冲突；删了的一边选「本机」就是删 +#[tokio::test] +async fn a_delete_against_an_edit_is_a_conflict() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider(s, "relay", "https://relay.example", "sk-base")); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + t.a.edit(|s| { + s.replace( + &provider_yaml("relay", "https://relay.example", "sk-base"), + "", + ) + .replace("providers:\n", "") + }); + assert!(t.a.provider("relay").is_none()); + t.b.edit(|s| s.replace("sk-base", "sk-new")); + t.ea.round().await.unwrap(); + let rb = t.eb.round().await.unwrap(); + assert_eq!(rb.conflicts.len(), 1, "{:?}", rb.conflicts); + assert!(rb.conflicts[0].whole_item()); + assert!(!rb.conflicts[0].both_allowed()); + let ra = t.ea.round().await.unwrap(); + let id = ra.conflicts[0].id.clone(); + t.ea.round_with(&[Decision { + id, + take: Take::Local, + new_name: None, + }]) + .await + .unwrap(); + t.eb.round().await.unwrap(); + assert!(t.b.provider("relay").is_none(), "{}", t.b.text()); +} + +/// 设备离线期间别的设备传了好几版，它当时的基准在同步位置上早删了：恢复连接后照本机基准合并， +/// 不报假冲突、不丢本机的改动 +#[tokio::test] +async fn coming_back_after_the_base_was_deleted_still_merges() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider(s, "relay", "https://relay.example", "sk-1")); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + // B 离线。A 改了好几次 + for i in 2..6 { + t.a.edit(|s| s.replace(&format!("sk-{}", i - 1), &format!("sk-{i}"))); + t.ea.round().await.unwrap(); + } + let b_base = t.eb.state().base.unwrap(); + assert!( + !versions(t.loc.path()).contains(&format!("{b_base}.twconfig")), + "B 的基准在同步位置上已经删了" + ); + t.b.edit(|s| add_provider(s, "offline", "https://offline.example", "sk-off")); + let r = t.eb.round().await.unwrap(); + assert!(r.conflicts.is_empty(), "{:?}", r.conflicts); + assert_eq!(t.b.provider("relay").unwrap().key.unwrap().raw(), "sk-5"); + t.ea.round().await.unwrap(); + assert!(t.a.provider("offline").is_some()); +} + +#[tokio::test] +async fn encryption_can_be_turned_on_off_and_its_passphrase_changed() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider(s, "relay", "https://relay.example", "sk-secret-value")); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + + // A 开启加密：B 下一轮要口令 + t.ea.enable_encryption(PASS).await.unwrap(); + assert!(t.loc.path().join("keyring").exists()); + for v in versions(t.loc.path()) { + let bytes = std::fs::read(t.loc.path().join("versions").join(&v)).unwrap(); + assert!(tw_pack::is_encrypted(&bytes), "未加密的版本删了：{v}"); + assert!(!String::from_utf8_lossy(&bytes).contains("sk-secret-value")); + } + assert!(matches!(t.eb.round().await, Err(RoundError::Locked(_)))); + assert!(t.eb.unlock("wrong passphrase").await.is_err()); + assert!(t.eb.unlock(PASS).await.unwrap()); + t.b.edit(|s| s.replace("sk-secret-value", "sk-after-encryption")); + let r = t.eb.round().await.unwrap(); + assert!(r.encrypted && r.uploaded.is_some()); + t.ea.round().await.unwrap(); + assert_eq!( + t.a.provider("relay").unwrap().key.unwrap().raw(), + "sk-after-encryption" + ); + + // A 改口令：已有版本不变，B 要输入新口令 + let before = versions(t.loc.path()); + t.ea.change_passphrase(PASS, "a brand new passphrase") + .await + .unwrap(); + assert_eq!(versions(t.loc.path()), before, "改口令只重新加密 keyring"); + assert!(matches!(t.eb.round().await, Err(RoundError::Locked(_)))); + assert!(t.eb.unlock("a brand new passphrase").await.unwrap()); + t.eb.round().await.unwrap(); + + // A 关闭加密：B 跟着变成不加密 + t.ea.disable_encryption("a brand new passphrase") + .await + .unwrap(); + assert!(!t.loc.path().join("keyring").exists()); + let r = t.eb.round().await.unwrap(); + assert!(!r.encrypted, "同步位置不加密了，B 照着改设置"); +} + +/// 两台设备同时开启加密同步：keyring 只建得成一份，后写的那台转去加入 +#[tokio::test] +async fn only_one_keyring_is_ever_created() { + tw_sync::keyring::set_work_factor_for_tests(4); + let loc = tempfile::tempdir().unwrap(); + let (a, b) = (Dev::new('a'), Dev::new('b')); + let mut ea = Engine::new( + folder(loc.path()), + a.clone(), + a.dir.path(), + settings(A_ID, "A", true, loc.path()), + ); + let mut eb = Engine::new( + folder(loc.path()), + b.clone(), + b.dir.path(), + settings(B_ID, "B", true, loc.path()), + ); + assert!(ea.create().await.unwrap()); + assert!(!eb.create().await.unwrap(), "后到的一方要转入加入流程"); + eb.join(None).await.unwrap(); + assert!(eb.round().await.unwrap().encrypted); +} + +/// 同步位置的最新版本由更新的 Core 写、本机读不了：暂停，不下载也不上传；设备记录里写明 +#[tokio::test] +async fn a_version_from_a_newer_core_pauses_sync() { + let mut t = bed(false).await; + // 伪造一个更新的 Core 写的版本：配置里有本机不认识的字段 + let latest = versions(t.loc.path()).pop().unwrap(); + let bytes = std::fs::read(t.loc.path().join("versions").join(&latest)).unwrap(); + let mut pack = tw_pack::read(&bytes, tw_pack::Unseal::None).unwrap(); + let mut meta: tw_sync::record::Meta = + serde_json::from_value(pack.manifest.sync.clone().unwrap()).unwrap(); + let parent = latest.trim_end_matches(".twconfig").to_string(); + meta.seq += 1; + meta.device = "cccccccc-0000-4000-8000-000000000003".into(); + meta.parents = vec![parent]; + pack.config_yaml.push_str("a_field_from_the_future: true\n"); + pack.manifest.core_version = "9.0.0".into(); + pack.manifest.sync = Some(serde_json::to_value(&meta).unwrap()); + let name = format!("{:010}-cccccccc.twconfig", meta.seq); + std::fs::write( + t.loc.path().join("versions").join(&name), + tw_pack::write(&pack, tw_pack::Seal::None).unwrap(), + ) + .unwrap(); + + t.b.edit(|s| add_provider(s, "local-only-for-now", "https://x.example", "sk")); + let e = t.eb.round().await.unwrap_err(); + assert!(matches!(e, RoundError::TooNew { .. }), "{e:?}"); + assert!( + !versions(t.loc.path()) + .iter() + .any(|v| v.ends_with("-bbbbbbbb.twconfig") && v > &name), + "暂停时不上传" + ); + let ra = t.ea.round().await; + assert!(matches!(ra, Err(RoundError::TooNew { .. }))); + // A 看得到 B 暂停着 + let probe = Engine::::probe(&folder(t.loc.path()), None).await; + let devs = probe.existing.unwrap().devices; + assert!(devs.iter().all(|d| d.paused_update), "{devs:?}"); +} + +/// 不同步的上游：不上传，也不被别的设备的同名上游覆盖；引用它的路由随之不同步 +#[tokio::test] +async fn an_excluded_upstream_and_what_refers_to_it_stay_local() { + let mut t = bed(false).await; + t.a.edit(|s| { + let s = add_provider(s, "ollama", "http://127.0.0.1:11434", "none"); + format!( + "{s}routes:\n - name: local-first\n rules:\n - name: r\n to: ollama\n" + ) + }); + t.b.edit(|s| add_provider(s, "ollama", "http://192.168.1.9:11434", "none")); + let mut sa = settings(A_ID, "Mac mini", false, t.loc.path()); + sa.exclude.insert("ollama".into()); + t.ea.set_settings(sa); + let mut sb = settings(B_ID, "MacBook Air", false, t.loc.path()); + sb.exclude.insert("ollama".into()); + t.eb.set_settings(sb); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + t.ea.round().await.unwrap(); + assert_eq!( + t.a.provider("ollama").unwrap().base_url, + "http://127.0.0.1:11434" + ); + assert_eq!( + t.b.provider("ollama").unwrap().base_url, + "http://192.168.1.9:11434" + ); + assert!( + t.b.config().routes.is_empty(), + "引用不同步上游的路由不上传：{}", + t.b.text() + ); + for v in versions(t.loc.path()) { + let p = tw_pack::read( + &std::fs::read(t.loc.path().join("versions").join(v)).unwrap(), + tw_pack::Unseal::None, + ) + .unwrap(); + assert!(!p.config_yaml.contains("ollama"), "{}", p.config_yaml); + } +} + +/// 本机已接管的客户端正在用的网关密钥被另一台设备删了：不删，作为冲突列出 +#[tokio::test] +async fn a_key_in_use_here_is_not_deleted_by_another_device() { + let mut t = bed(false).await; + t.a.edit(|s| { + s.replace( + "clients:\n", + "clients:\n - name: codex\n key: tw-codex-key-0002\n", + ) + }); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + t.b.pinned.lock().unwrap().insert("codex".into()); + t.a.edit(|s| s.replace(" - name: codex\n key: tw-codex-key-0002\n", "")); + t.ea.round().await.unwrap(); + let r = t.eb.round().await.unwrap(); + assert!( + t.b.config().clients.iter().any(|c| c.name == "codex"), + "{}", + t.b.text() + ); + assert_eq!(r.conflicts.len(), 1); + assert_eq!( + r.conflicts[0].reason, + Some(tw_sync::record::Reason::KeyInUse) + ); +} + +/// 另一台设备停止同步并删除了同步数据：这台设备发现同步位置空了 +#[tokio::test] +async fn a_purged_location_is_noticed() { + let mut t = bed(true).await; + t.ea.stop(true).await.unwrap(); + assert!(versions(t.loc.path()).is_empty()); + assert!(!t.loc.path().join("keyring").exists()); + let r = t.eb.round().await; + assert!( + matches!(r, Err(RoundError::Emptied)), + "{:?}", + r.map(|r| r.uploaded) + ); +} + +/// 同一套流程走 WebDAV 和 S3（进程内的假服务） +#[tokio::test] +async fn the_same_two_device_flow_over_webdav_and_s3() { + tw_sync::keyring::set_work_factor_for_tests(4); + let dav = common::webdav().await; + let s3 = common::s3().await; + let stores: Vec AnyStore>> = vec![ + Box::new({ + let url = format!("{}/dav/root", dav.url()); + move || { + AnyStore::WebDav( + tw_sync::store::WebDav::new(&url, common::USER, common::PASS, common::http()) + .unwrap(), + ) + } + }), + Box::new({ + let endpoint = s3.url(); + move || { + AnyStore::S3( + tw_sync::store::S3::new( + tw_sync::store::s3::Config { + endpoint: endpoint.clone(), + region: "auto".into(), + bucket: "tw-bucket".into(), + prefix: "tw".into(), + access_key_id: common::AK.into(), + secret_access_key: common::SK.into(), + }, + common::http(), + ) + .unwrap(), + ) + } + }), + ]; + for (i, mk) in stores.iter().enumerate() { + let (a, b) = (Dev::new('a'), Dev::new('b')); + let place = Path::new(if i == 0 { "/webdav" } else { "/s3" }); + let mut ea = Engine::new( + mk(), + a.clone(), + a.dir.path(), + settings(A_ID, "A", true, place), + ); + let mut eb = Engine::new( + mk(), + b.clone(), + b.dir.path(), + settings(B_ID, "B", true, place), + ); + assert!(ea.create().await.unwrap()); + eb.join(None).await.unwrap(); + a.edit(|s| add_provider(s, "relay", "https://relay.example", "sk-over-the-wire")); + ea.round().await.unwrap(); + eb.round().await.unwrap(); + assert_eq!( + b.provider("relay").unwrap().key.unwrap().raw(), + "sk-over-the-wire" + ); + b.edit(|s| s.replace("sk-over-the-wire", "sk-back")); + eb.round().await.unwrap(); + ea.round().await.unwrap(); + assert_eq!(a.provider("relay").unwrap().key.unwrap().raw(), "sk-back"); + } + // 同步位置上看不到明文密钥 + for f in dav.names().iter().chain(s3.names().iter()) { + let bytes = dav.get(f).or_else(|| s3.get(f)).unwrap(); + assert!(!String::from_utf8_lossy(&bytes).contains("sk-back"), "{f}"); + } +} diff --git a/crates/tw-sync/tests/stores.rs b/crates/tw-sync/tests/stores.rs new file mode 100644 index 00000000..a26be7a4 --- /dev/null +++ b/crates/tw-sync/tests/stores.rs @@ -0,0 +1,198 @@ +//! 三种同步位置答同一套问题：只新增不覆盖、仅当不存在时创建、覆盖写、列目录跳过临时文件、 +//! 删两次不报错、凭据不对是「拒绝访问」、连不上是「不可用」。 +//! +//! 文件夹用临时目录，WebDAV 和 S3 用进程内的假服务（`common`）。 + +mod common; + +use tw_sync::store::{self, Folder, S3, WebDav}; +use tw_sync::{Store, StoreError}; + +const V1: &str = "versions/0000000001-aaaaaaaa.twconfig"; +const V2: &str = "versions/0000000002-bbbbbbbb.twconfig"; + +async fn conformance(s: &impl Store) { + assert!( + s.list("versions").await.unwrap().is_empty(), + "空的位置列出来是空的" + ); + assert_eq!(s.get(V1).await.unwrap(), None); + + s.put_new(V1, b"one").await.unwrap(); + let e = s.put_new(V1, b"two").await.unwrap_err(); + assert!(matches!(e, StoreError::Exists { .. }), "{e:?}"); + assert_eq!( + s.get(V1).await.unwrap().as_deref(), + Some(&b"one"[..]), + "只新增不覆盖" + ); + + s.put_new(V2, b"twotwo").await.unwrap(); + let names: Vec<(String, u64)> = s + .list("versions") + .await + .unwrap() + .into_iter() + .map(|e| (e.name, e.size)) + .collect(); + assert_eq!( + names, + [ + ("0000000001-aaaaaaaa.twconfig".to_string(), 3), + ("0000000002-bbbbbbbb.twconfig".to_string(), 6) + ], + "临时文件不出现在列表里" + ); + + assert!(s.create_exclusive("keyring", b"k1").await.unwrap()); + assert!(!s.create_exclusive("keyring", b"k2").await.unwrap()); + assert_eq!(s.get("keyring").await.unwrap().as_deref(), Some(&b"k1"[..])); + s.put("keyring", b"k3").await.unwrap(); + assert_eq!(s.get("keyring").await.unwrap().as_deref(), Some(&b"k3"[..])); + + s.put("devices/aaaaaaaa", b"d1").await.unwrap(); + s.put("devices/aaaaaaaa", b"d2").await.unwrap(); + assert_eq!( + s.get("devices/aaaaaaaa").await.unwrap().as_deref(), + Some(&b"d2"[..]) + ); + assert_eq!(s.list("devices").await.unwrap().len(), 1); + + s.delete(V1).await.unwrap(); + s.delete(V1).await.unwrap(); + assert_eq!(s.get(V1).await.unwrap(), None); + assert_eq!(s.list("versions").await.unwrap().len(), 1); +} + +#[tokio::test] +async fn a_folder_answers_like_the_others() { + let d = tempfile::tempdir().unwrap(); + conformance(&Folder::new(d.path())).await; + // 只剩正式的文件，没有半截的临时文件 + let left: Vec = std::fs::read_dir(d.path().join("versions")) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().to_string()) + .collect(); + assert_eq!(left, ["0000000002-bbbbbbbb.twconfig"]); +} + +#[tokio::test] +async fn webdav_answers_like_the_others() { + let fake = common::webdav().await; + let s = WebDav::new( + &format!("{}/dav/root", fake.url()), + common::USER, + common::PASS, + common::http(), + ) + .unwrap(); + conformance(&s).await; + assert!( + fake.names().iter().all(|n| !n.contains(".tmp-")), + "临时文件改名之后不该留下：{:?}", + fake.names() + ); +} + +#[tokio::test] +async fn webdav_without_move_or_conditional_put_still_never_overwrites_a_version() { + let fake = common::webdav().await; + fake.knobs + .no_move + .store(true, std::sync::atomic::Ordering::Relaxed); + let s = WebDav::new( + &format!("{}/dav/root", fake.url()), + common::USER, + common::PASS, + common::http(), + ) + .unwrap(); + conformance(&s).await; + // 连 If-None-Match 都不认的：keyring 先看一眼再写 + fake.knobs + .no_conditional + .store(true, std::sync::atomic::Ordering::Relaxed); + assert!(!s.create_exclusive("keyring", b"other").await.unwrap()); + assert_eq!(fake.get("keyring").as_deref(), Some(&b"k3"[..])); +} + +fn s3_store(fake: &common::Fake, ak: &str) -> S3 { + S3::new( + store::s3::Config { + endpoint: fake.url(), + region: "auto".into(), + bucket: "tw-bucket".into(), + prefix: "team/tw".into(), + access_key_id: ak.into(), + secret_access_key: common::SK.into(), + }, + common::http(), + ) + .unwrap() +} + +#[tokio::test] +async fn s3_answers_like_the_others() { + let fake = common::s3().await; + conformance(&s3_store(&fake, common::AK)).await; + assert!( + fake.names().iter().all(|n| n.starts_with("team/tw/")), + "键都在前缀底下：{:?}", + fake.names() + ); +} + +#[tokio::test] +async fn s3_without_conditional_writes_falls_back_to_looking_first() { + let fake = common::s3().await; + fake.knobs + .no_conditional + .store(true, std::sync::atomic::Ordering::Relaxed); + conformance(&s3_store(&fake, common::AK)).await; +} + +#[tokio::test] +async fn wrong_credentials_are_a_refusal_and_a_dead_server_is_unreachable() { + let dav = common::webdav().await; + let wrong = WebDav::new( + &format!("{}/dav/root", dav.url()), + common::USER, + "wrong", + common::http(), + ) + .unwrap(); + assert!(matches!( + wrong.list("versions").await.unwrap_err(), + StoreError::AuthFailed(_) + )); + let s3 = common::s3().await; + assert!(matches!( + s3_store(&s3, "AKIDSOMEONEELSE").get(V1).await.unwrap_err(), + StoreError::AuthFailed(_) + )); + dav.knobs + .down + .store(true, std::sync::atomic::Ordering::Relaxed); + let right = WebDav::new( + &format!("{}/dav/root", dav.url()), + common::USER, + common::PASS, + common::http(), + ) + .unwrap(); + assert!(matches!( + right.get(V1).await.unwrap_err(), + StoreError::Unreachable(_) + )); + let gone = WebDav::new( + &format!("{}/dav/root", common::nowhere()), + common::USER, + common::PASS, + common::http(), + ) + .unwrap(); + assert!(matches!( + gone.list("versions").await.unwrap_err(), + StoreError::Unreachable(_) + )); +} diff --git a/docs/config.md b/docs/config.md index 75a91421..5ac3653a 100644 --- a/docs/config.md +++ b/docs/config.md @@ -193,6 +193,7 @@ means. | `default_route` | string | — | The route for keys that do not name one. Unset: the route named `default`, or the built-in failover when there is none. | | `default_key` | string | — | The gateway key for clients that were not given a key of their own. Unset: the key named `default`, or the first key. It cannot be disabled. | | `plugins` | list of [`plugins[]`](#cfg-plugins) | `[]` | Script plugins, in the order they run. The app installs them; each one's code and settings are a file next to this one. | +| `sync` | object, [`sync`](#cfg-sync) | — | Multi-device sync. A local setting: configuration packs and other devices never carry it, and restoring a backup keeps the current one. Unset: sync is off. | ### `listen` @@ -1502,6 +1503,95 @@ export const manifest = { }; ``` +### `sync` + +Several devices (this Mac, a laptop, a server running a remote core) can keep +one configuration in step through a sync location you choose: a folder that a +cloud drive carries between devices, a WebDAV server, or S3-compatible +storage. Each change is merged object by object with what the other devices +did since the last sync; only when two devices change the same part of the +same object to different values is there a conflict, which the app shows on +every device until one of them resolves it. Turn sync on, join a location and +resolve conflicts in the app (Settings, Sync and backup) or with +`twcore sync enable` on a server; this section records the result. + +What syncs is the configuration a configuration pack carries: upstreams, +proxies, gateway keys, aliases, groups, routes, price sheets, security rules, +plugins and the general settings, keys included. `listen`, `retention` and +`sync` itself stay on this device, and so does a ChatGPT account's sign-in. + + + + +| Field | Type | Default | Description | +|---|---|---|---| +| `device_id` | string | **required** | Generated when sync is turned on. File names at the sync location show its first 8 characters. | +| `device_name` | string | **required** | Shown in the device list of every device that syncs here. | +| `encrypt` | bool | `true` | Encrypt what goes to the sync location with the sync passphrase. Every device at one location uses the same setting; change it in the app, which re-uploads the latest version. | +| `passphrase` | string | — | The sync passphrase; required when `encrypt` is on. Kept here in the clear, like the other keys in this file. | +| `exclude` | list of strings | `[]` | Upstreams, by name, that are not synced: they are not uploaded and another device's upstream of the same name does not replace them. Routes, aliases and groups that refer to them are not synced either. | +| `target` | object, [`sync.target`](#cfg-sync-target) | **required** | The sync location. The keyring, `versions/` and `devices/` live directly in it. | + + + + + +| Field | Type | Default | Description | +|---|---|---|---| +| `type` | `folder` \| `webdav` \| `s3` | **required** | Which kind of location; the other fields depend on it, see the three tables below. | + + + + + +| Field | Type | Default | Description | +|---|---|---|---| +| `type` | string | **required** | `folder` | +| `path` | string | **required** | An absolute path on the machine core runs on: a folder that iCloud Drive, Dropbox, OneDrive, Syncthing or a NAS share carries between devices. | +| `proxy` | string | `direct` | Not used for a folder. | + + + + + +| Field | Type | Default | Description | +|---|---|---|---| +| `type` | string | **required** | `webdav` | +| `url` | string | **required** | The WebDAV address of the folder. `https://`, with the certificate checked; `http://` only for this machine and the local network. | +| `user` | string | `''` | User name. | +| `pass` | string | `''` | Password, or the app password the service issues for WebDAV. | +| `proxy` | string | `direct` | Where requests to the sync location go out: `direct`, `system` (the system proxy), or the name of a proxy under `proxies`. | + + + + + +| Field | Type | Default | Description | +|---|---|---|---| +| `type` | string | **required** | `s3` | +| `endpoint` | string | **required** | The S3 endpoint. Hosts ending in `amazonaws.com`, `aliyuncs.com` or `myqcloud.com` put the bucket in the host name; every other endpoint (Cloudflare R2, MinIO, an address) puts it in the path. `http://` only for this machine and the local network. | +| `region` | string | **required** | The signing region; `auto` for Cloudflare R2. | +| `bucket` | string | **required** | The bucket. Keep it private. | +| `prefix` | string | `''` | A folder inside the bucket. | +| `access_key_id` | string | **required** | Access key ID. | +| `secret_access_key` | string | **required** | Secret access key. | +| `proxy` | string | `direct` | Where requests to the sync location go out: `direct`, `system` (the system proxy), or the name of a proxy under `proxies`. | + + +```yaml +sync: + device_id: 3f9a2c1e-0b1d-4e2f-9a3b-5c6d7e8f9a0b + device_name: Mac mini + encrypt: true + passphrase: "correct horse battery staple" + exclude: [local ollama] + target: + type: webdav + url: https://dav.example.com/dav/ThinkWatch + user: name@example.com + pass: "app password" +``` + ## Environment variables | Variable | Effect | diff --git a/docs/config.zh-CN.md b/docs/config.zh-CN.md index e6ba2936..9a471081 100644 --- a/docs/config.zh-CN.md +++ b/docs/config.zh-CN.md @@ -118,6 +118,7 @@ twcore config set /listen/gateway/port 8790 --int | `default_route` | 字符串 | — | 未指定路由的密钥走哪条路由。不写：名为 `default` 的路由；没有这条路由时走内置的故障转移。 | | `default_key` | 字符串 | — | 没有专用密钥的客户端使用哪一把。不写：名为 `default` 的那把，没有则取第一把。这把密钥不能停用。 | | `plugins` | 对象列表，见 [`plugins[]`](#cfg-plugins) | `[]` | 脚本插件，按运行的顺序。由应用安装，每个插件的代码和设置是本文件旁边的一个文件。 | +| `sync` | 对象，见 [`sync`](#cfg-sync) | — | 多设备同步。属于本机设置：配置包和其他设备不会带走它，恢复备份时保留当前值。不写：不同步。 | ### `listen` @@ -1100,6 +1101,84 @@ export const manifest = { }; ``` +### `sync` + +多台设备（这台 Mac、一台笔记本、一台运行远程 core 的服务器）可以通过自选的同步位置保持同一份配置：由云盘在设备之间传送的文件夹、WebDAV 服务器或 S3 兼容存储。每次改动都按对象与其他设备自上次同步以来的改动合并；只有两台设备把同一对象的同一部分改成不同内容时才构成冲突，冲突在每台设备上显示，任一设备处理之后随之清除。开启同步、加入已有的同步和处理冲突在应用里进行（设置 → 同步与备份），服务器上用 `twcore sync enable`；本节记录的是结果。 + +同步的内容即配置包带的内容：上游、出站代理、网关密钥、别名、分组、路由、价目表、安全规则、插件和全局设置，包括密钥。`listen`、`retention` 和 `sync` 本身留在本机，ChatGPT 账号的登录也一样。 + + + + +| 字段 | 类型 | 默认值 | 说明 | +|---|---|---|---| +| `device_id` | 字符串 | **必填** | 开启同步时生成。同步位置上的文件名里出现的是它的前 8 个字符。 | +| `device_name` | 字符串 | **必填** | 在每台参与同步的设备的设备列表里显示的名字。 | +| `encrypt` | 布尔 | `true` | 以同步口令加密存到同步位置的内容。同一同步位置上的所有设备采用同一设置；在应用里更改，应用会重新上传最新版本。 | +| `passphrase` | 字符串 | — | 同步口令，`encrypt` 开着时必填。和本文件里的其他密钥一样以明文保存。 | +| `exclude` | 字符串列表 | `[]` | 不同步的上游，按名字：不上传，也不被其他设备的同名上游覆盖。引用它们的路由、别名、分组随之不同步。 | +| `target` | 对象，见 [`sync.target`](#cfg-sync-target) | **必填** | 同步位置。`keyring`、`versions/`、`devices/` 直接放在它里面。 | + + + + + +| 字段 | 类型 | 默认值 | 说明 | +|---|---|---|---| +| `type` | `folder` \| `webdav` \| `s3` | **必填** | 同步位置的种类；其余字段随之不同，见下面三张表。 | + + + + + +| 字段 | 类型 | 默认值 | 说明 | +|---|---|---|---| +| `type` | 字符串 | **必填** | `folder` | +| `path` | 字符串 | **必填** | core 所在机器上的绝对路径：由 iCloud Drive、Dropbox、OneDrive、Syncthing 或 NAS 共享在设备之间传送的文件夹。 | +| `proxy` | 字符串 | `direct` | 文件夹用不到。 | + + + + + +| 字段 | 类型 | 默认值 | 说明 | +|---|---|---|---| +| `type` | 字符串 | **必填** | `webdav` | +| `url` | 字符串 | **必填** | 文件夹的 WebDAV 地址。`https://`，校验证书；`http://` 只用于本机和局域网。 | +| `user` | 字符串 | `''` | 用户名。 | +| `pass` | 字符串 | `''` | 密码，或者服务为 WebDAV 签发的应用密码。 | +| `proxy` | 字符串 | `direct` | 访问同步位置的请求从哪里出去：`direct`（直连）、`system`（跟随系统代理），或者 `proxies` 里一个代理的名字。 | + + + + + +| 字段 | 类型 | 默认值 | 说明 | +|---|---|---|---| +| `type` | 字符串 | **必填** | `s3` | +| `endpoint` | 字符串 | **必填** | S3 端点。主机以 `amazonaws.com`、`aliyuncs.com`、`myqcloud.com` 结尾的，存储桶名放进域名；其余端点（Cloudflare R2、MinIO、IP 地址）放进路径。`http://` 只用于本机和局域网。 | +| `region` | 字符串 | **必填** | 签名用的区域；Cloudflare R2 填 `auto`。 | +| `bucket` | 字符串 | **必填** | 存储桶，建议设为私有。 | +| `prefix` | 字符串 | `''` | 存储桶里的一个前缀（文件夹）。 | +| `access_key_id` | 字符串 | **必填** | Access Key ID。 | +| `secret_access_key` | 字符串 | **必填** | Secret Access Key。 | +| `proxy` | 字符串 | `direct` | 访问同步位置的请求从哪里出去：`direct`（直连）、`system`（跟随系统代理），或者 `proxies` 里一个代理的名字。 | + + +```yaml +sync: + device_id: 3f9a2c1e-0b1d-4e2f-9a3b-5c6d7e8f9a0b + device_name: Mac mini + encrypt: true + passphrase: "correct horse battery staple" + exclude: [本机 Ollama] + target: + type: webdav + url: https://dav.example.com/dav/ThinkWatch + user: name@example.com + pass: "应用密码" +``` + ## 环境变量 | 变量 | 作用 | From db4af14f2b98c63f7bc728be5093ee6c09ae1ca2 Mon Sep 17 00:00:00 2001 From: fylorn <249551762+fylorn@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:50:51 +0800 Subject: [PATCH 05/13] tw-pack: read_strict refuses a plain pack when a key is given; scrypt cap 18 (#354) - read_strict(bytes, unseal): with a passphrase or an identity, a pack that is not encrypted is refused (pack.not_encrypted). For a sync location with encryption on, where accepting a plain version would let anyone who can write there change this device's configuration. - The largest scrypt work factor a pack may ask for is now the one packs are written with (18), like the sync keyring; 22 meant about 4 GiB. Co-authored-by: Claude Opus 5.5 --- crates/tw-api/msg-codes.txt | 1 + crates/tw-pack/src/lib.rs | 26 +++++++++++++++++++++++--- crates/tw-pack/src/tests.rs | 31 +++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/crates/tw-api/msg-codes.txt b/crates/tw-api/msg-codes.txt index 5dd966db..3f3a4c23 100644 --- a/crates/tw-api/msg-codes.txt +++ b/crates/tw-api/msg-codes.txt @@ -512,6 +512,7 @@ pack.need_key pack.need_passphrase pack.no_such_item pack.not_base64 +pack.not_encrypted pack.passphrase_short pack.schema_too_new pack.too_large diff --git a/crates/tw-pack/src/lib.rs b/crates/tw-pack/src/lib.rs index 3ede6064..f32736b5 100644 --- a/crates/tw-pack/src/lib.rs +++ b/crates/tw-pack/src/lib.rs @@ -305,6 +305,9 @@ pub enum PackError { MissingEntry { name: String }, #[error("{}", self.msg())] TooLarge { limit: usize }, + /// 要求加密的地方读到了一个没加密的包（[`read_strict`]） + #[error("{}", self.msg())] + NotEncrypted, } impl PackError { @@ -347,6 +350,10 @@ impl PackError { "pack.missing_entry", name = name => "This configuration pack has no {name}." ), + PackError::NotEncrypted => msg!( + "pack.not_encrypted" => + "This configuration pack is not encrypted, though it has to be, so it was not opened." + ), PackError::TooLarge { limit } => msg!( "pack.too_large", limit = limit / (1024 * 1024) => "This configuration pack is larger than {limit} MiB." @@ -452,8 +459,9 @@ fn work_factor() -> u8 { } } -/// 解开时最多接受多大的工作量。比写的时候大一些：将来调高了，旧版本照样读得了新包 -const MAX_WORK_FACTOR: u8 = 22; +/// 解开时最多接受多大的工作量：**和写的时候一样**（同步位置的 keyring 也是这个上限）。一个 +/// 不可信的包能指定工作量，放宽了就是让它一打开就占掉几个 GB 内存（`2^22` 约 4 GiB） +const MAX_WORK_FACTOR: u8 = 18; /// **只给测试用**：把口令加密的工作量调低，测试不用每次等一秒。0 是不改 #[doc(hidden)] @@ -466,7 +474,19 @@ pub fn is_encrypted(bytes: &[u8]) -> bool { bytes.starts_with(b"age-encryption.org/") } -/// 读一个配置包。 +/// 同 [`read`]，**加密得和给的钥匙对得上**：给了口令或密钥（[`Unseal::Passphrase`] / +/// [`Unseal::Identity`]）时，没加密的包不收（[`PackError::NotEncrypted`]）。同步位置开着加密 +/// 时用它：一个照收的明文版本，等于对那个位置有写权限的谁都能把配置写进这台设备。不给 +/// （[`Unseal::None`]）时和 [`read`] 一样 +pub fn read_strict(bytes: &[u8], unseal: Unseal) -> Result { + if !matches!(unseal, Unseal::None) && !is_encrypted(bytes) { + return Err(PackError::NotEncrypted); + } + read(bytes, unseal) +} + +/// 读一个配置包。**给了钥匙也照收明文的包**：导入用户自己挑的文件时，加没加密都行；要求加密 +/// 的地方用 [`read_strict`] pub fn read(bytes: &[u8], unseal: Unseal) -> Result { if bytes.len() > MAX_BYTES { return Err(PackError::TooLarge { limit: MAX_BYTES }); diff --git a/crates/tw-pack/src/tests.rs b/crates/tw-pack/src/tests.rs index d4e9e03b..1a7a78ed 100644 --- a/crates/tw-pack/src/tests.rs +++ b/crates/tw-pack/src/tests.rs @@ -332,3 +332,34 @@ fn origins_keep_their_words() { } assert_eq!(PackOrigin::from_slug("nope"), PackOrigin::Other); } + +/// 要求加密的地方：给了钥匙就不收明文；加密的照常解；没给钥匙时和 `read` 一样 +#[test] +fn a_strict_read_refuses_a_plain_pack_when_a_key_is_given() { + fast(); + let pack = sample(); + let plain = write(&pack, Seal::None).unwrap(); + let id = age::x25519::Identity::generate(); + let sealed = write(&pack, Seal::Recipient(&id.to_public())).unwrap(); + assert_eq!( + read_strict(&plain, Unseal::Identity(&id)).unwrap_err(), + PackError::NotEncrypted + ); + assert_eq!( + read_strict(&plain, Unseal::Passphrase("whatever it is")).unwrap_err(), + PackError::NotEncrypted + ); + assert_eq!(PackError::NotEncrypted.msg().code, "pack.not_encrypted"); + assert_eq!( + read_strict(&sealed, Unseal::Identity(&id)) + .unwrap() + .config_yaml, + pack.config_yaml + ); + assert_eq!( + read_strict(&plain, Unseal::None).unwrap().config_yaml, + pack.config_yaml + ); + // `read` 照收：导入的是用户自己挑的文件 + assert!(read(&plain, Unseal::Identity(&id)).is_ok()); +} From 5d07d27cc837b7adffc67efba15412a73bede496 Mon Sep 17 00:00:00 2001 From: fylorn <249551762+fylorn@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:13:11 +0800 Subject: [PATCH 06/13] Packs: what a preview passes, the write does; secretless packs keep the keys here (#356) - A pack without secrets is compared and overwritten with the secrets already here (upstream keys, header values, AWS keys, OAuth client secrets, proxy passwords, gateway keys): an item that differs only in its secrets is the same, an overwrite no longer blanks the local key, and only a new gateway key is regenerated. - A gateway key imported under another name gets a new value instead of duplicating the one it was copied from. - Restore plans until nothing is missing: an item that stays (a pinned key, an unselected route) keeps what it refers to, shown in the preview as kept for `referenced`; restoring chosen items brings along what they need that is no longer here. An OAuth upstream whose token endpoint changed comes back with the tokens of that version, or disabled when it has none, instead of failing. - A whole restore in safe mode takes listen, the control key, retention and sync from the current text, or from the latest local backup when the text is not YAML, so a sync version or an exported pack can be restored. - Pinned keys are kept per source (PUT /pinned-keys { source, names }), the union is what counts, and a source not heard from in 30 days expires. - Import bodies may carry a full 16 MiB pack in base64; other endpoints keep the default limit. Passphrase decryption and encryption run on a blocking thread. - When the gateway cannot take a configuration that is already written, the plugin files stay as written instead of being rolled back. At start, plugin files that match neither the approved hash nor its copy are put back from a backup. The 64 KiB limit on client preferences applies to imports and restores too. Co-authored-by: Claude Opus 5.5 --- bin/twcore/src/pack_cli.rs | 6 +- crates/tw-api/src/lib.rs | 5 +- crates/tw-api/src/pack.rs | 11 +- crates/tw-control/src/backups.rs | 100 +++++++--- crates/tw-control/src/config.rs | 28 ++- crates/tw-control/tests/backups.rs | 163 +++++++++++++++++ crates/tw-pack/src/backup.rs | 105 +++++++++++ crates/tw-pack/src/disk.rs | 70 ++++++- crates/tw-pack/src/export.rs | 74 ++++++++ crates/tw-pack/src/import.rs | 26 ++- crates/tw-pack/src/restore.rs | 275 +++++++++++++++++++--------- crates/tw-pack/src/restore/tests.rs | 2 +- crates/tw-pack/tests/regressions.rs | 260 ++++++++++++++++++++++++++ 13 files changed, 989 insertions(+), 136 deletions(-) create mode 100644 crates/tw-pack/tests/regressions.rs diff --git a/bin/twcore/src/pack_cli.rs b/bin/twcore/src/pack_cli.rs index 8287bcf8..f02b860e 100644 --- a/bin/twcore/src/pack_cli.rs +++ b/bin/twcore/src/pack_cli.rs @@ -381,7 +381,10 @@ pub fn restore(path: &Path, id: &str, dry_run: bool) -> Result<()> { let pinned = tw_pack::disk::read_pinned(path); let after = if tw_pack::model::snapshot(&local).is_err() { println!("the configuration here does not load; the whole version comes back"); - tw_pack::restore::whole(&backup, &cur.text) + let last = tw_pack::backup::list(path) + .ok() + .and_then(|all| all.last().and_then(|b| tw_pack::backup::open(b).ok())); + tw_pack::restore::whole(&backup, &cur.text, last.as_ref()) } else { let rows = tw_pack::restore::preview(&local, &backup, &pinned, true).map_err(model)?; if rows.is_empty() { @@ -396,6 +399,7 @@ pub fn restore(path: &Path, id: &str, dry_run: bool) -> Result<()> { }; let kept = match r.kept { Some(Kept::KeyInUse) => " (kept: a client set up on this machine uses it)", + Some(Kept::Referenced) => " (kept: something that stays refers to it)", None => "", }; println!(" {change} {}{kept}", describe(&r.item)); diff --git a/crates/tw-api/src/lib.rs b/crates/tw-api/src/lib.rs index d545dc17..24d27632 100644 --- a/crates/tw-api/src/lib.rs +++ b/crates/tw-api/src/lib.rs @@ -971,7 +971,10 @@ pub const MSG_CODES: &str = include_str!("../msg-codes.txt"); /// 新增、相同、冲突、错误）和 `POST /config/import`（[`ImportRequest`]：逐项导入、覆盖、改名 /// 导入或跳过）。Lite 的通用偏好由 core 存：`GET` / `PUT /client-prefs`（[`ClientPrefs`]， /// 不解析，最大 64 KiB），变了发 [`Event::ClientPrefsChanged`]。`PUT /pinned-keys` -/// （[`PinnedKeys`]）报告接管的客户端在用哪几把密钥。导入覆盖、恢复改了或删了网关密钥时发 +/// （[`PinnedKeys`]：`source` 是这个 Lite 的标识，几个 Lite 各报各的、core 取并集，30 天没更新 +/// 的来源不算）报告接管的客户端在用哪几把密钥。恢复时留下的项目还引用着的不删 +/// （[`KeptReason`] 的 `referenced`），所选项目引用的、本机已没有的一起恢复；不带密钥的包导入时 +/// 比较和覆盖都沿用本机的密钥。导入覆盖、恢复改了或删了网关密钥时发 /// [`Event::GatewayKeysChanged`]。[`ConfigOrigin`] 删了 `rollback`，多了 `sync`、`import`、 /// `restore`、`repair`（一键修复以前记成 `ui`）。备份相关的端点在安全模式下照样可用。配置包的 /// 码是 `pack.*`，导入的是 `import.*`，备份与恢复的是 `backup.*`。停用着的上游可以缺凭据 diff --git a/crates/tw-api/src/pack.rs b/crates/tw-api/src/pack.rs index 689c940a..ab10f724 100644 --- a/crates/tw-api/src/pack.rs +++ b/crates/tw-api/src/pack.rs @@ -344,6 +344,9 @@ slug_enum! { pub enum KeptReason { /// 本机接管的客户端正在用这把网关密钥：不删 KeyInUse = "key_in_use", + /// 恢复之后留下的项目还引用着它（留着的密钥绑的路由、没选的路由用的上游……）：删了配置 + /// 就不成立，所以留着 + Referenced = "referenced", } } @@ -380,9 +383,15 @@ pub struct ClientPrefs { /// Lite 的通用偏好最大多少字节（写成 JSON 之后） pub const CLIENT_PREFS_MAX: usize = 64 * 1024; -/// `PUT /pinned-keys`：本机接管的客户端正在用的网关密钥。恢复、导入、同步不删它们。 +/// `PUT /pinned-keys`：这一个 Lite 接管的客户端正在用的网关密钥。恢复、导入、同步不删它们。 +/// +/// **按来源记**：几个 Lite 连着同一个 core（远程模式）时各报各的，core 取并集；一个来源报的 +/// 换掉它自己上次报的。30 天没更新的来源不再算 #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[cfg_attr(feature = "ts", derive(ts_rs::TS))] pub struct PinnedKeys { + /// 这个 Lite 的标识，各实例固定的一个随机串。不给是空串（当作同一个来源） + #[serde(default)] + pub source: String, pub names: Vec, } diff --git a/crates/tw-control/src/backups.rs b/crates/tw-control/src/backups.rs index c1fccdec..4286d3ad 100644 --- a/crates/tw-control/src/backups.rs +++ b/crates/tw-control/src/backups.rs @@ -36,12 +36,19 @@ use tw_types::{Msg, msg}; use crate::contract::RouterExt; use crate::{ApplyError, ControlState, Fail, apply_fail, fail, unreadable_config}; +/// 带着配置包内容的请求体最大多少字节：一个 [`tw_pack::MAX_BYTES`] 的包写成 base64，加上 +/// 请求里别的几个字段。**只给这两个端点放宽**，别的照 axum 的默认（2 MiB） +pub const PACK_BODY_MAX: usize = tw_pack::MAX_BYTES / 3 * 4 + 1024 * 1024; + pub fn router() -> axum::Router { + let packs = axum::Router::new() + .at(ep::ImportPreview, import_preview) + .at(ep::ImportConfig, import) + .layer(axum::extract::DefaultBodyLimit::max(PACK_BODY_MAX)); axum::Router::new() + .merge(packs) .at(ep::ConfigItems, items) .at(ep::ExportConfig, export) - .at(ep::ImportPreview, import_preview) - .at(ep::ImportConfig, import) .at(ep::Backups, backups) .at(ep::BackupFile, backup_file) .at(ep::RestorePreview, restore_preview) @@ -256,21 +263,37 @@ fn passphrase(p: Option<&str>) -> Result, Fail> { } } -fn open_pack(data_b64: &str, pass: Option<&str>) -> Result { +/// 解开一个配置包。**在阻塞线程上**：口令解密要算 scrypt（零点几秒、上百 MB 内存），不能 +/// 占着处理请求的线程 +async fn open_pack(data_b64: &str, pass: Option<&str>) -> Result { let bytes = decode(data_b64)?; - let unseal = match pass { - Some(p) => Unseal::Passphrase(p), - None => Unseal::None, - }; - tw_pack::read(&bytes, unseal).map_err(bad_pack) -} - -fn file_of(pack: &Pack, pass: Option<&str>) -> Result<(String, String), Fail> { - let seal = match pass { - Some(p) => Seal::Passphrase(p), - None => Seal::None, - }; - let bytes = tw_pack::write(pack, seal).map_err(bad_pack)?; + let pass = pass.map(str::to_string); + tokio::task::spawn_blocking(move || { + let unseal = match &pass { + Some(p) => Unseal::Passphrase(p), + None => Unseal::None, + }; + tw_pack::read(&bytes, unseal) + }) + .await + .map_err(crate::internal)? + .map_err(bad_pack) +} + +/// 写成文件。口令加密同样在阻塞线程上 +async fn file_of(pack: &Pack, pass: Option<&str>) -> Result<(String, String), Fail> { + let owned = pack.clone(); + let pass = pass.map(str::to_string); + let bytes = tokio::task::spawn_blocking(move || { + let seal = match &pass { + Some(p) => Seal::Passphrase(p), + None => Seal::None, + }; + tw_pack::write(&owned, seal) + }) + .await + .map_err(crate::internal)? + .map_err(bad_pack)?; Ok(( tw_pack::export::file_name(pack.manifest.created_at_ms), base64::engine::general_purpose::STANDARD.encode(bytes), @@ -291,6 +314,16 @@ fn fix_view(f: tw_config::repair::Fix) -> tw_api::ConfigFix { } } +fn prefs_too_large() -> Fail { + fail( + StatusCode::PAYLOAD_TOO_LARGE, + msg!( + "backup.prefs_too_large", limit = tw_api::CLIENT_PREFS_MAX / 1024 => + "The client preferences are larger than {limit} KiB." + ), + ) +} + // ─────────────────────────────────────────────────────────── 此刻的样子 /// 磁盘上此刻的配置（原文和版本）连同它的插件、通用偏好 @@ -316,6 +349,13 @@ async fn land( changed_keys: Vec, ) -> Result { let path = s.config_path().to_path_buf(); + // 偏好的上限导入、恢复也照样查：一份带着超大偏好的包不该绕过 `PUT /client-prefs` 的那道关 + if core_keeps_prefs() + && let Some(p) = &after.client_prefs + && p.to_string().len() > tw_api::CLIENT_PREFS_MAX + { + return Err(prefs_too_large()); + } let _edit = s.gateway.plugins.edits.lock().await; let undo = tw_pack::disk::write_plugins(&path, &before.plugins, &after.plugins).map_err(|e| { @@ -336,6 +376,9 @@ async fn land( .await; let version = match written { Ok(v) => v, + // **配置已经写下去了**、只是网关换不进来：插件文件和偏好跟着磁盘上的配置，不还原 —— + // 还原的话 config.yaml 里批准的哈希就对不上磁盘上的插件文件了 + Err(e @ ApplyError::Build(_)) => return Err(apply_fail(e)), Err(e) => { undo.restore(); if prefs_changed { @@ -421,7 +464,7 @@ async fn export( }, ) .map_err(|e| model_fail(e, None))?; - let (file_name, data_b64) = file_of(&out.pack, pass)?; + let (file_name, data_b64) = file_of(&out.pack, pass).await?; Ok(Json(tw_api::ExportedFile { file_name, data_b64, @@ -483,7 +526,7 @@ async fn import_preview( State(s): State, Json(req): Json, ) -> Result, Fail> { - let raw = open_pack(&req.data_b64, req.passphrase.as_deref())?; + let raw = open_pack(&req.data_b64, req.passphrase.as_deref()).await?; let (incoming, repairs) = tw_pack::import::prepare(raw.clone()).map_err(|e| import_fail(e, &raw))?; let (cur, local) = current(&s, PackOrigin::Import)?; @@ -502,7 +545,7 @@ async fn import( State(s): State, Json(req): Json, ) -> Result, Fail> { - let raw = open_pack(&req.data_b64, req.passphrase.as_deref())?; + let raw = open_pack(&req.data_b64, req.passphrase.as_deref()).await?; // 预览时修过的那几处，这里照样修一遍（同一份字节，修出来一样） let (incoming, _) = tw_pack::import::prepare(raw.clone()).map_err(|e| import_fail(e, &raw))?; let (cur, local) = current(&s, PackOrigin::Import)?; @@ -660,7 +703,7 @@ async fn backup_file( .map_err(|e| model_fail(e, Some(&pack)))?; let mut exported = out.pack; exported.manifest.created_at_ms = pack.manifest.created_at_ms; - let (file_name, data_b64) = file_of(&exported, pass)?; + let (file_name, data_b64) = file_of(&exported, pass).await?; Ok(Json(tw_api::ExportedFile { file_name, data_b64, @@ -680,6 +723,7 @@ fn restore_row(r: tw_pack::restore::Row) -> tw_api::RestoreRow { }, kept: r.kept.map(|k| match k { Kept::KeyInUse => tw_api::KeptReason::KeyInUse, + Kept::Referenced => tw_api::KeptReason::Referenced, }), } } @@ -744,7 +788,11 @@ async fn restore( .map(|xs| xs.iter().map(item_id).collect()); let (after, changed) = if model::snapshot(&local).is_err() { // 安全模式：现在这份读不进来，整份换回去 - let mut whole = tw_pack::restore::whole(&backup, &cur.text); + // 本机那几节取不到时（原文坏到读不成 YAML）用最近一个本机备份里的 + let last = tw_pack::backup::list(s.config_path()) + .ok() + .and_then(|all| all.last().and_then(|b| tw_pack::backup::open(b).ok())); + let mut whole = tw_pack::restore::whole(&backup, &cur.text, last.as_ref()); if !core_keeps_prefs() { whole.client_prefs = local.client_prefs.clone(); } @@ -780,13 +828,7 @@ async fn put_prefs( if let Some(p) = &req.prefs && p.to_string().len() > tw_api::CLIENT_PREFS_MAX { - return Err(fail( - StatusCode::PAYLOAD_TOO_LARGE, - msg!( - "backup.prefs_too_large", limit = tw_api::CLIENT_PREFS_MAX / 1024 => - "The client preferences are larger than {limit} KiB." - ), - )); + return Err(prefs_too_large()); } let path = s.config_path().to_path_buf(); if tw_pack::disk::read_prefs(&path) == req.prefs { @@ -813,7 +855,7 @@ async fn pin_keys( ) -> Result, Fail> { let names: BTreeSet = req.names.into_iter().collect(); s.cfg - .set_pinned_keys(names) + .set_pinned_keys(&req.source, names) .map_err(|e| io_fail(&tw_pack::disk::pinned_path(s.config_path()), e))?; Ok(Json(())) } diff --git a/crates/tw-control/src/config.rs b/crates/tw-control/src/config.rs index c8e756bc..9d988b59 100644 --- a/crates/tw-control/src/config.rs +++ b/crates/tw-control/src/config.rs @@ -28,7 +28,7 @@ pub struct ConfigManager { /// 每换入一份配置响一次（[`Self::applied`]）。默认插件那一路等着它 applied: tokio::sync::Notify, /// 固定密钥：本机接管的客户端在用的网关密钥，桌面端报上来（`PUT /pinned-keys`） - pinned: std::sync::Mutex>, + pinned: std::sync::Mutex>, /// 同步位置上的备份版本。没开同步是 None remote: std::sync::RwLock>>, /// 这份配置的同步服务，第一次用到时建（见 `crate::sync::service`） @@ -119,7 +119,7 @@ impl ConfigManager { gateway.set_config_dir(crate::plugins::dir_of(&path)); // 配置包、备份按「代码」「数据」分开比较插件，要插件那一套来拆 crate::backups::install_codec(); - let pinned = tw_pack::disk::read_pinned(&path); + let pinned = tw_pack::disk::read_pinned_sources(&path); Self { path, gateway, @@ -372,6 +372,14 @@ impl ConfigManager { /// 启动时：磁盘上此刻的样子和最新的备份版本不一样就留一版（第一次运行、升级之后、core /// 没在跑时命令行改过）。来源记成外部改动 —— 说不清是谁改的 pub fn record_startup(&self) { + // 写插件文件和写配置之间断过电：插件文件先从备份里找回来 + let healed = tw_pack::backup::heal_plugins(&self.path); + if !healed.is_empty() { + tracing::warn!( + plugins = ?healed, + "plugin files that did not match the configuration were put back from a backup" + ); + } if let Ok(cur) = self.current() { self.backup(tw_pack::backup::record( &self.path, @@ -381,19 +389,25 @@ impl ConfigManager { } } - /// 本机接管的客户端在用的网关密钥（固定密钥）。恢复、导入、同步不删它们 + /// 本机接管的客户端在用的网关密钥（固定密钥）：各个来源报上来的并集，30 天没更新的来源 + /// 不算。恢复、导入、同步不删它们 pub fn pinned_keys(&self) -> std::collections::BTreeSet { - self.pinned.lock().map(|g| g.clone()).unwrap_or_default() + self.pinned + .lock() + .map(|g| tw_pack::disk::pinned_union(&g, now_ms())) + .unwrap_or_default() } - /// 换掉固定密钥，存进数据目录（`pinned-keys.json`），重启之后还在 + /// 一个来源（一个 Lite）报上来它的固定密钥：换掉它上次报的，存进数据目录 + /// （`pinned-keys.json`），重启之后还在 pub fn set_pinned_keys( &self, + source: &str, names: std::collections::BTreeSet, ) -> std::io::Result<()> { - tw_pack::disk::write_pinned(&self.path, &names)?; + let all = tw_pack::disk::write_pinned(&self.path, source, &names)?; if let Ok(mut g) = self.pinned.lock() { - *g = names; + *g = all; } Ok(()) } diff --git a/crates/tw-control/tests/backups.rs b/crates/tw-control/tests/backups.rs index 69b8e6d3..3d016a43 100644 --- a/crates/tw-control/tests/backups.rs +++ b/crates/tw-control/tests/backups.rs @@ -625,3 +625,166 @@ async fn an_older_pack_is_previewed_repaired_and_a_newer_one_is_too_new() { assert_eq!(st, StatusCode::BAD_REQUEST, "{v}"); assert_eq!(v["code"], "pack.too_new"); } + +/// 导入的请求体放宽到装得下一个最大的包；别的端点照旧 +#[tokio::test] +async fn a_large_pack_reaches_the_import_and_other_bodies_keep_the_default_limit() { + let b = bed(&mine()); + // 3 MiB 的 base64：超过默认的 2 MiB，远不到配置包的上限 + let big = "A".repeat(3 * 1024 * 1024); + let (st, v) = call( + &b.app, + "POST", + "/config/import/preview", + serde_json::json!({ "data_b64": big }), + ) + .await; + assert_ne!(st, StatusCode::PAYLOAD_TOO_LARGE, "{v}"); + assert_eq!(v["code"], "pack.corrupt", "{v}"); + let (st, _) = call( + &b.app, + "PUT", + "/client-prefs", + serde_json::json!({ "prefs": { "big": big } }), + ) + .await; + assert_eq!(st, StatusCode::PAYLOAD_TOO_LARGE); +} + +/// 固定密钥按来源记：两个 Lite 各报各的，取并集；一个来源再报一次换掉它自己的 +#[tokio::test] +async fn pinned_keys_are_kept_per_source() { + let b = bed(&mine()); + for (source, names) in [("lite-a", vec!["default"]), ("lite-b", vec!["claude-code"])] { + let (st, v) = call( + &b.app, + "PUT", + "/pinned-keys", + serde_json::json!({ "source": source, "names": names }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{v}"); + } + let all: Vec = b.state.cfg.pinned_keys().into_iter().collect(); + assert_eq!(all, ["claude-code", "default"]); + call( + &b.app, + "PUT", + "/pinned-keys", + serde_json::json!({ "source": "lite-a", "names": [] }), + ) + .await; + let all: Vec = b.state.cfg.pinned_keys().into_iter().collect(); + assert_eq!(all, ["claude-code"]); +} + +/// 导入的包里带着超大的通用偏好：和 `PUT /client-prefs` 一样挡下 +#[tokio::test] +async fn oversized_preferences_in_a_pack_are_refused() { + let dst = bed(&mine()); + let mut m = tw_pack::Manifest::new(tw_pack::PackKind::Full, tw_pack::PackOrigin::Export); + m.secrets = true; + let p = tw_pack::Pack { + manifest: m, + config_yaml: "version: 1\n".into(), + plugins: Default::default(), + client_prefs: Some(serde_json::json!({ "big": "x".repeat(tw_api::CLIENT_PREFS_MAX + 1) })), + }; + let data = base64::engine::general_purpose::STANDARD + .encode(tw_pack::write(&p, tw_pack::Seal::None).unwrap()); + let (_, preview) = call( + &dst.app, + "POST", + "/config/import/preview", + serde_json::json!({ "data_b64": data }), + ) + .await; + let (st, v) = call( + &dst.app, + "POST", + "/config/import", + serde_json::json!({ + "data_b64": data, + "base_version": preview["base_version"], + "decisions": [{ "item": { "kind": "client_prefs", "name": "" }, "action": "import" }], + }), + ) + .await; + assert_eq!(st, StatusCode::PAYLOAD_TOO_LARGE, "{v}"); + assert_eq!(v["code"], "backup.prefs_too_large"); + assert!(tw_pack::disk::read_prefs(&dst.path()).is_none()); +} + +/// 同步位置上的版本：里面没有本机那几节 +struct OneRemote(tw_pack::Pack); + +impl tw_control::backups::RemoteBackups for OneRemote { + fn list( + &self, + ) -> tw_control::backups::BoxFuture<'_, Vec> { + let v = tw_control::backups::RemoteVersionInfo { + content_id: self.0.content_id(), + at_ms: 1, + device_name: "MacBook Air".into(), + core_version: tw_pack::CORE_VERSION.into(), + counts: self.0.manifest.counts, + }; + Box::pin(async move { vec![v] }) + } + fn fetch<'a>( + &'a self, + _content_id: &'a str, + ) -> tw_control::backups::BoxFuture<'a, Result> { + let p = self.0.clone(); + Box::pin(async move { Ok(p) }) + } +} + +/// 安全模式下整份恢复同步位置上的一个版本：钥匙、监听是现在这份原文里的，写得进去 +#[tokio::test] +async fn in_safe_mode_a_synced_version_comes_back_whole() { + let src = bed(&theirs()); + let file = export_all(&src, true).await; + let synced = tw_pack::read( + &base64::engine::general_purpose::STANDARD + .decode(file["data_b64"].as_str().unwrap()) + .unwrap(), + tw_pack::Unseal::None, + ) + .unwrap(); + assert!(!synced.config_yaml.contains("listen")); + let d = tempfile::tempdir().unwrap(); + let p = d.path().join("config.yaml"); + let broken = format!("{}not_a_field: 1\n", mine()); + std::fs::write(&p, &broken).unwrap(); + let (cfg, r) = tw_config::stand_in(&broken).unwrap(); + let gw = tw_gateway::AppState::new(cfg).unwrap(); + let bus = gw.bus.clone(); + let mgr = Arc::new(ConfigManager::new(p.clone(), gw.clone(), bus).standing_in(&r)); + mgr.set_remote_backups(Some(Arc::new(OneRemote(synced.clone())))); + let app = tw_control::router(state_for(p.clone(), mgr, gw)); + let (st, list) = call(&app, "GET", "/backups", serde_json::json!(null)).await; + assert_eq!(st, StatusCode::OK, "{list}"); + let id = list[0]["id"].as_str().unwrap().to_string(); + assert!(id.starts_with("sync-"), "{list}"); + let (st, preview) = call( + &app, + "POST", + "/backups/preview", + serde_json::json!({ "id": id }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{preview}"); + let (st, v) = call( + &app, + "POST", + "/backups/restore", + serde_json::json!({ "id": id, "base_version": preview["base_version"] }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{v}"); + let text = std::fs::read_to_string(&p).unwrap(); + let cfg = tw_config::try_parse(&text).unwrap_or_else(|e| panic!("{e}\n{text}")); + assert_eq!(cfg.listen.control.key.as_deref(), Some(KEY)); + assert!(cfg.providers.iter().any(|p| p.name == "relay")); +} diff --git a/crates/tw-pack/src/backup.rs b/crates/tw-pack/src/backup.rs index 0c6c7292..d95315ff 100644 --- a/crates/tw-pack/src/backup.rs +++ b/crates/tw-pack/src/backup.rs @@ -226,6 +226,66 @@ pub fn device_of(pack: &Pack) -> Option { .map(str::to_string) } +/// 启动时：插件文件和配置里批准的哈希都对不上的（脚本和底稿都不是那一份 —— 写插件文件和写 +/// 配置之间断了电），从最近的备份里找回哈希对得上的那一份写回去。返回修好的插件 id。 +/// +/// **底稿对得上的不动**：那是用户在磁盘上改了、还没批准的文件，不是断在半路 +pub fn heal_plugins(config_path: &Path) -> Vec { + let Ok(text) = std::fs::read_to_string(config_path) else { + return Vec::new(); + }; + let Ok(doc) = serde_yaml_ng::from_str::(&text) else { + return Vec::new(); + }; + let entries: Vec<(String, String)> = doc + .get(tw_config::plugins::DIR) + .and_then(serde_yaml_ng::Value::as_sequence) + .into_iter() + .flatten() + .filter_map(|p| { + let id = p.get("id")?.as_str()?.to_string(); + let sha = p.get("sha256")?.as_str()?.to_string(); + tw_config::plugins::valid_id(&id).then_some((id, sha)) + }) + .collect(); + let hash = crate::model::sha256_hex; + let mut healed = Vec::new(); + let mut backups: Option> = None; + for (id, sha) in entries { + let files = disk::read_plugin(config_path, &id); + let fine = files.as_ref().is_some_and(|f| { + hash(&f.script) == sha || f.approved.as_ref().is_some_and(|a| hash(a) == sha) + }); + if fine { + continue; + } + let packs = backups.get_or_insert_with(|| { + list(config_path) + .unwrap_or_default() + .iter() + .rev() + .filter_map(|b| open(b).ok()) + .collect() + }); + let found = packs.iter().find_map(|p| { + let f = p.plugins.get(&id)?; + [Some(&f.script), f.approved.as_ref()] + .into_iter() + .flatten() + .find(|b| hash(b) == sha) + .cloned() + }); + let Some(bytes) = found else { continue }; + let dir = disk::dir_of(config_path); + let ok = disk::write_private(&tw_config::plugins::file_path(&dir, &id), &bytes).is_ok() + && disk::write_private(&tw_config::plugins::approved_path(&dir, &id), &bytes).is_ok(); + if ok { + healed.push(id); + } + } + healed +} + /// 超出 [`KEEP`] 的老版本删掉，以前的修改记录（`*.yaml`）也删掉 fn prune(config_path: &Path) -> Result<(), BackupError> { let all = list(config_path)?; @@ -410,6 +470,51 @@ mod tests { assert_eq!(list(&p).unwrap().len(), 1, "不多出一版"); } + /// 写插件文件和写配置之间断了电：配置还是旧的，文件已经是新的。启动时从备份里找回旧的； + /// 用户自己改了、还没批准的文件（底稿对得上）不动 + #[test] + fn plugin_files_cut_off_halfway_are_put_back_from_a_backup() { + let (_d, p) = setup(); + let files = |s: &str, approved: Option<&str>| crate::PluginFiles { + script: s.as_bytes().to_vec(), + approved: approved.map(|a| a.as_bytes().to_vec()), + }; + let sha = crate::model::sha256_hex(b"v1"); + let text = format!( + "{}plugins:\n - id: a\n file: plugins/a.js\n sha256: {sha}\n - id: b\n file: plugins/b.js\n sha256: {sha}\n", + cfg(1) + ); + std::fs::write(&p, &text).unwrap(); + disk::write_plugins( + &p, + &Default::default(), + &[ + ("a".to_string(), files("v1", Some("v1"))), + ("b".to_string(), files("v1", Some("v1"))), + ] + .into_iter() + .collect(), + ) + .unwrap(); + record(&p, &text, Origin::Ui).unwrap().unwrap(); + // a：两份都换成了新的（断在半路）；b：用户改了脚本，底稿还是批准的那份 + disk::write_plugins( + &p, + &Default::default(), + &[ + ("a".to_string(), files("v2", Some("v2"))), + ("b".to_string(), files("edited", Some("v1"))), + ] + .into_iter() + .collect(), + ) + .unwrap(); + assert_eq!(heal_plugins(&p), ["a"]); + assert_eq!(disk::read_plugin(&p, "a").unwrap().script, b"v1"); + assert_eq!(disk::read_plugin(&p, "b").unwrap().script, b"edited"); + assert!(heal_plugins(&p).is_empty()); + } + #[test] fn a_rotation_leaves_no_version_and_does_not_count_as_a_change() { let (_d, p) = setup(); diff --git a/crates/tw-pack/src/disk.rs b/crates/tw-pack/src/disk.rs index 6dbc74a9..16340e15 100644 --- a/crates/tw-pack/src/disk.rs +++ b/crates/tw-pack/src/disk.rs @@ -52,23 +52,64 @@ pub fn write_prefs(config_path: &Path, prefs: Option<&serde_json::Value>) -> std } } -/// 存着的固定密钥。没有、读不懂都是空的 -pub fn read_pinned(config_path: &Path) -> BTreeSet { +/// 一个来源（一个连着这台 core 的 Lite）报上来的固定密钥多久没更新就不算了：30 天 +pub const PINNED_TTL_MS: u64 = 30 * 24 * 3600 * 1000; + +/// 一个来源报上来的固定密钥。 +#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct PinnedSource { + pub names: BTreeSet, + pub updated_at_ms: u64, +} + +/// 存着的固定密钥，**按来源**：几个 Lite 连着同一个 core（远程模式）时各报各的，取并集。 +/// 没有、读不懂都是空的 +pub fn read_pinned_sources(config_path: &Path) -> BTreeMap { #[derive(serde::Deserialize)] struct File { - names: BTreeSet, + sources: BTreeMap, } std::fs::read(pinned_path(config_path)) .ok() .and_then(|b| serde_json::from_slice::(&b).ok()) - .map(|f| f.names) + .map(|f| f.sources) .unwrap_or_default() } -pub fn write_pinned(config_path: &Path, names: &BTreeSet) -> std::io::Result<()> { - let bytes = serde_json::to_vec_pretty(&serde_json::json!({ "names": names })) +/// 各来源的并集，**过期的来源不算**（[`PINNED_TTL_MS`] 没更新：那个 Lite 多半已经不在了） +pub fn pinned_union(sources: &BTreeMap, now_ms: u64) -> BTreeSet { + sources + .values() + .filter(|s| now_ms.saturating_sub(s.updated_at_ms) <= PINNED_TTL_MS) + .flat_map(|s| s.names.iter().cloned()) + .collect() +} + +/// 存着的固定密钥：各个来源的并集，过期的不算 +pub fn read_pinned(config_path: &Path) -> BTreeSet { + pinned_union(&read_pinned_sources(config_path), crate::now_ms()) +} + +/// 一个来源报上来一组固定密钥：换掉它上次报的，过期的来源顺手删掉。返回存下的全部来源 +pub fn write_pinned( + config_path: &Path, + source: &str, + names: &BTreeSet, +) -> std::io::Result> { + let now = crate::now_ms(); + let mut sources = read_pinned_sources(config_path); + sources.retain(|_, s| now.saturating_sub(s.updated_at_ms) <= PINNED_TTL_MS); + sources.insert( + source.to_string(), + PinnedSource { + names: names.clone(), + updated_at_ms: now, + }, + ); + let bytes = serde_json::to_vec_pretty(&serde_json::json!({ "sources": sources })) .map_err(std::io::Error::other)?; - write_private(&pinned_path(config_path), &bytes) + write_private(&pinned_path(config_path), &bytes)?; + Ok(sources) } /// 配置原文里列着的插件 id，按顺序。读不成 YAML 是空的 @@ -277,9 +318,18 @@ mod tests { std::fs::write(prefs_path(&cfg), b"{not json").unwrap(); assert_eq!(read_prefs(&cfg), None); - let names: BTreeSet = ["codex".to_string()].into_iter().collect(); - write_pinned(&cfg, &names).unwrap(); - assert_eq!(read_pinned(&cfg), names); + // 两个 Lite 各报各的，取并集；一个来源再报一次是换掉它自己的那一组 + let a: BTreeSet = ["codex".to_string()].into_iter().collect(); + let b: BTreeSet = ["claude-code".to_string()].into_iter().collect(); + write_pinned(&cfg, "lite-a", &a).unwrap(); + write_pinned(&cfg, "lite-b", &b).unwrap(); + assert_eq!(read_pinned(&cfg), a.union(&b).cloned().collect()); + write_pinned(&cfg, "lite-a", &BTreeSet::new()).unwrap(); + assert_eq!(read_pinned(&cfg), b); + // 30 天没更新的来源不算 + let mut sources = read_pinned_sources(&cfg); + sources.get_mut("lite-b").unwrap().updated_at_ms = 1; + assert!(pinned_union(&sources, crate::now_ms()).is_empty()); } #[test] diff --git a/crates/tw-pack/src/export.rs b/crates/tw-pack/src/export.rs index 30d72f81..bc0b162b 100644 --- a/crates/tw-pack/src/export.rs +++ b/crates/tw-pack/src/export.rs @@ -163,6 +163,80 @@ pub fn strip_secrets(snap: &mut Snapshot) { } } +/// 不带密钥的包里的一项，**空着的密钥换成本机同一项里的**：上游的密钥、请求头的值（按名字）、 +/// AWS 访问密钥、OAuth 的客户端密钥，代理的密码，网关密钥的值。比较和覆盖都照这个样子：除了 +/// 密钥都一样的就是「相同」，覆盖不会把本机的密钥抹成空的 +pub fn fill_secrets(id: &ItemId, item: &mut model::Item, mine: &model::Item) { + let empty = |v: Option<&Json>| v.and_then(Json::as_str) == Some(""); + let take = |at: Option<&mut Json>, from: Option<&Json>| { + if let (Some(v), Some(f)) = (at, from) + && v.as_str() == Some("") + && f.as_str().is_some_and(|s| !s.is_empty()) + { + *v = f.clone(); + } + }; + match id.kind { + ItemKind::Provider => { + let (Some(conn), Some(local)) = + (item.units.get_mut(unit::CONN), mine.units.get(unit::CONN)) + else { + return; + }; + take(conn.get_mut("key"), local.get("key")); + if let Some(Json::Array(headers)) = conn.get_mut("headers") { + let theirs = local.get("headers").and_then(Json::as_array); + for h in headers { + if !empty(h.get(1)) { + continue; + } + let name = h + .get(0) + .and_then(Json::as_str) + .unwrap_or_default() + .to_string(); + let found = theirs.and_then(|hs| { + hs.iter().find(|x| { + x.get(0) + .and_then(Json::as_str) + .is_some_and(|n| n.eq_ignore_ascii_case(&name)) + }) + }); + take(h.get_mut(1), found.and_then(|x| x.get(1))); + } + } + for k in ["access_key_id", "secret_access_key", "session_token"] { + take( + conn.get_mut("aws").and_then(|a| a.get_mut(k)), + local.get("aws").and_then(|a| a.get(k)), + ); + } + take( + conn.get_mut("oauth") + .and_then(|o| o.get_mut("client_secret")), + local.get("oauth").and_then(|o| o.get("client_secret")), + ); + } + ItemKind::Proxy => { + let local = mine.units.get(unit::ALL).and_then(|a| a.get("auth")); + take( + item.units + .get_mut(unit::ALL) + .and_then(|a| a.get_mut("auth")) + .and_then(|a| a.get_mut("pass")), + local.and_then(|a| a.get("pass")), + ); + } + ItemKind::Key => { + take( + item.units.get_mut(unit::ALL).and_then(|a| a.get_mut("key")), + mine.units.get(unit::ALL).and_then(|a| a.get("key")), + ); + } + _ => {} + } +} + /// 一项里有没有被去掉的密钥（空字符串） pub fn missing_secret(id: &ItemId, item: &model::Item) -> bool { let empty = |v: Option<&Json>| v.and_then(Json::as_str) == Some(""); diff --git a/crates/tw-pack/src/import.rs b/crates/tw-pack/src/import.rs index 4621a094..b11daf90 100644 --- a/crates/tw-pack/src/import.rs +++ b/crates/tw-pack/src/import.rs @@ -182,6 +182,19 @@ fn skipped_ref(id: &ItemId, target: &ItemId) -> Msg { ) } +/// 不带密钥的包：本机也有的那几项，空着的密钥换成本机的（[`export::fill_secrets`]） +fn with_local_secrets(l: &Snapshot, mut i: Snapshot, incoming: &Pack) -> Snapshot { + if incoming.manifest.secrets { + return i; + } + for (id, item) in i.items.iter_mut() { + if let Some(mine) = l.items.get(id) { + export::fill_secrets(id, item, mine); + } + } + i +} + /// 全局设置和通用偏好：冲突只能覆盖或跳过 fn is_global(kind: ItemKind) -> bool { matches!(kind, ItemKind::Global | ItemKind::ClientPrefs) @@ -199,7 +212,7 @@ pub fn preview_with( codec: &dyn PluginCodec, ) -> Result, ImportError> { let l = model::snapshot_with(local, codec)?; - let i = model::snapshot_with(incoming, codec)?; + let i = with_local_secrets(&l, model::snapshot_with(incoming, codec)?, incoming); let both = union(&l, &i); let mut rows = Vec::new(); for (id, item) in &i.items { @@ -488,7 +501,7 @@ pub fn apply_with( codec: &dyn PluginCodec, ) -> Result { let l = model::snapshot_with(local, codec)?; - let mut i = model::snapshot_with(incoming, codec)?; + let mut i = with_local_secrets(&l, model::snapshot_with(incoming, codec)?, incoming); let rows = preview_with(local, incoming, ctx, codec)?; let errors: BTreeMap<&ItemId, &Msg> = rows .iter() @@ -552,7 +565,14 @@ pub fn apply_with( } Action::Import | Action::Overwrite | Action::Rename(_) => {} } - let item = prepared(&id, item, incoming, &l, ctx); + let mut item = prepared(&id, item, incoming, &l, ctx); + // 改名导入的网关密钥是另一把：值重新生成。照搬的话它和本机原来那把同值，配置不成立 + if id.kind == ItemKind::Key + && matches!(d.action, Action::Rename(_)) + && let Some(Json::Object(all)) = item.units.get_mut(unit::ALL) + { + all.insert("key".into(), Json::String((ctx.new_key)())); + } target.items.insert(id.clone(), item); taken.push(id); } diff --git a/crates/tw-pack/src/restore.rs b/crates/tw-pack/src/restore.rs index e53b7bb2..289756b6 100644 --- a/crates/tw-pack/src/restore.rs +++ b/crates/tw-pack/src/restore.rs @@ -29,6 +29,8 @@ pub enum Change { pub enum Kept { /// 接管的客户端在用这把网关密钥 KeyInUse, + /// 留下来的项目还引用着它（留着的密钥绑的路由、没选的路由用的上游……）：删了配置就不成立 + Referenced, } #[derive(Debug, Clone, PartialEq)] @@ -58,10 +60,11 @@ pub fn preview_with( ) -> Result, ModelError> { let l = model::snapshot_with(local, codec)?; let b = model::snapshot_with(backup, codec)?; - Ok(rows(&l, &b, pinned, with_prefs)) + Ok(plan(&l, &b, pinned, with_prefs, None).0) } -fn rows(l: &Snapshot, b: &Snapshot, pinned: &BTreeSet, with_prefs: bool) -> Vec { +/// 两份之间的差异，按项目 +fn diff(l: &Snapshot, b: &Snapshot, with_prefs: bool) -> Vec<(ItemId, Change)> { let (l, b) = (model::normalized(l), model::normalized(b)); let ids: BTreeSet<&ItemId> = l.items.keys().chain(b.items.keys()).collect(); ids.into_iter() @@ -73,29 +76,112 @@ fn rows(l: &Snapshot, b: &Snapshot, pinned: &BTreeSet, with_prefs: bool) (Some(x), Some(y)) if x.units != y.units => Change::Modify, _ => return None, }; - let kept = - (change == Change::Remove && id.kind == ItemKind::Key && pinned.contains(&id.name)) - .then_some(Kept::KeyInUse); - Some(Row { - item: id.clone(), - change, - kept, - }) + Some((id.clone(), change)) }) .collect() } +/// 这个引用缺了，配置就不成立吗。别名（模型名）、插件的范围缺了只是匹配不到什么 +fn hard(from: &ItemId, to: &ItemId) -> bool { + from.kind != ItemKind::Plugin + && to.kind != ItemKind::Alias + && !(to.kind == ItemKind::Route && to.name == tw_engine::DEFAULT_ROUTE) +} + +/// 照差异落成的目标，和预览里的那几行。 +/// +/// **预览能过的，写下去一定成**：恢复之后还有项目引用着缺了的东西时，缺的那一样 +/// +/// - 是这次要删的：留着（[`Kept::Referenced`]），比如留着的固定密钥绑的那条路由； +/// - 是那一版里有、没选上的：一起恢复，比如只选了一个已删的上游、它用的出站代理也删了。 +/// +/// 一直补到没有缺的为止 +fn plan( + l: &Snapshot, + b: &Snapshot, + pinned: &BTreeSet, + with_prefs: bool, + selected: Option<&BTreeSet>, +) -> (Vec, Snapshot) { + let changes = diff(l, b, with_prefs); + let mut chosen: BTreeSet = changes + .iter() + .map(|(id, _)| id.clone()) + .filter(|id| selected.is_none_or(|s| s.contains(id))) + .collect(); + let mut kept: std::collections::BTreeMap = changes + .iter() + .filter(|(id, c)| { + *c == Change::Remove && id.kind == ItemKind::Key && pinned.contains(&id.name) + }) + .map(|(id, _)| (id.clone(), Kept::KeyInUse)) + .collect(); + loop { + let mut target = l.clone(); + for (id, _) in changes.iter().filter(|(id, _)| chosen.contains(id)) { + if kept.contains_key(id) { + continue; + } + match b.items.get(id) { + Some(it) => { + target.items.insert(id.clone(), it.clone()); + } + None => { + target.items.remove(id); + } + } + } + target.relink(); + let mut more = false; + let missing: Vec = target + .items + .iter() + .flat_map(|(id, it)| { + it.refs + .iter() + .filter(|r| hard(id, r) && !target.items.contains_key(*r)) + .cloned() + .collect::>() + }) + .collect(); + for r in missing { + if l.items.contains_key(&r) { + // 这次要删的：留着 + if let std::collections::btree_map::Entry::Vacant(e) = kept.entry(r) { + e.insert(Kept::Referenced); + more = true; + } + } else if b.items.contains_key(&r) && chosen.insert(r) { + // 那一版里有、没选上的：一起恢复 + more = true; + } + } + if !more { + let rows = changes + .into_iter() + .filter(|(id, _)| chosen.contains(id) || kept.contains_key(id)) + .map(|(id, change)| Row { + kept: kept.get(&id).copied(), + item: id, + change, + }) + .collect(); + return (rows, target); + } + } +} + /// 恢复的结果。 #[derive(Debug, Clone)] pub struct Applied { pub pack: Pack, /// 值变了或者没了的网关密钥 pub changed_keys: Vec, - /// 因为固定密钥留着没删的 + /// 留着没删的：固定密钥，和留下来的项目还引用着的 pub kept: Vec, } -/// 恢复。`items` 是只恢复这几项，None 是全部 +/// 恢复。`items` 是只恢复这几项，None 是全部。所选项目引用的、本机已经没有的项目一起恢复 pub fn apply( local: &Pack, backup: &Pack, @@ -116,49 +202,33 @@ pub fn apply_with( ) -> Result { let l = model::snapshot_with(local, codec)?; let b = model::snapshot_with(backup, codec)?; - let picked: Option> = items.map(|xs| xs.iter().collect()); - let mut target = l.clone(); - let mut kept = Vec::new(); - // 那一版有、现在没有的 OAuth 上游：令牌不在快照里，落下去之后从那一版里取回来 - let mut tokens_from_backup: Vec = Vec::new(); - for row in rows(&l, &b, pinned, with_prefs) { - if picked.as_ref().is_some_and(|p| !p.contains(&row.item)) { + let picked: Option> = items.map(|xs| xs.iter().cloned().collect()); + let (rows, mut target) = plan(&l, &b, pinned, with_prefs, picked.as_ref()); + let kept = rows + .iter() + .filter(|r| r.kept.is_some()) + .map(|r| r.item.clone()) + .collect(); + // 那一版有、现在没有的 ChatGPT 账号上游：它的登录随删除吊销了，停用着回来 + for row in rows + .iter() + .filter(|r| r.change == Change::Add && r.kept.is_none()) + { + if row.item.kind != ItemKind::Provider { continue; } - if row.kept.is_some() { - kept.push(row.item.clone()); + let Some(it) = target.items.get_mut(&row.item) else { continue; - } - match b.items.get(&row.item) { - Some(it) => { - let mut it = it.clone(); - // 那一版有、现在没有的 ChatGPT 账号上游：它的登录随删除吊销了，停用着回来。 - // 别的 OAuth 上游：令牌从那一版里取回来（落下去之后补上） - let back = !l.items.contains_key(&row.item); - let p = (row.item.kind == ItemKind::Provider) - .then(|| model::provider_of(&row.item, &it).ok()) - .flatten(); - if back && let Some(p) = &p { - if model::is_chatgpt(p) { - if let Some(Json::Object(other)) = it.units.get_mut(unit::OTHER) { - other.insert("disabled".into(), Json::Bool(true)); - } - } else if p.oauth.is_some() { - tokens_from_backup.push(row.item.name.clone()); - } - } - target.items.insert(row.item.clone(), it); - } - None => { - target.items.remove(&row.item); - } + }; + if model::provider_of(&row.item, it).is_ok_and(|p| model::is_chatgpt(&p)) + && let Some(Json::Object(other)) = it.units.get_mut(unit::OTHER) + { + other.insert("disabled".into(), Json::Bool(true)); } } target.relink(); let mut pack = model::materialize_with(local, &target, codec)?; - if !tokens_from_backup.is_empty() { - pack.config_yaml = tokens_back(&pack.config_yaml, backup, &tokens_from_backup)?; - } + pack.config_yaml = settle_logins(&pack.config_yaml, backup)?; tw_config::try_parse(&pack.config_yaml).map_err(|r| ModelError::Config(Box::new(r)))?; Ok(Applied { changed_keys: import::changed_keys(local, &pack), @@ -167,43 +237,55 @@ pub fn apply_with( }) } -/// 把那一版里这几家 OAuth 上游的令牌写回 `text`。那一版里没有令牌（空的）的那家停用着回来 -fn tokens_back(text: &str, backup: &Pack, names: &[String]) -> Result { +/// 落下去之后没有令牌的 OAuth 上游（那一版有、现在没有的；后来换了令牌端点、本机的令牌 +/// 用不上的）：那一版里用的是同一个令牌端点、有令牌的，令牌从那一版里取回来；没有的停用着 +/// 回来，等填上刷新令牌 —— 不是恢复失败 +fn settle_logins(text: &str, backup: &Pack) -> Result { + let now = model::parse(text)?; let then = model::parse(&backup.config_yaml)?; let mut out = text.to_string(); - for name in names { - let Some(o) = then + for p in &now.providers { + let Some(o) = &p.oauth else { continue }; + if model::is_chatgpt(p) || p.disabled || !o.refresh.trim().is_empty() { + continue; + } + let from_then = then .providers .iter() - .find(|p| &p.name == name) - .and_then(|p| p.oauth.as_ref()) - else { - continue; - }; - if o.refresh.trim().is_empty() { - out = tw_config::edit::set( - &out, - &provider_field(&out, name, "disabled")?, - Some(&serde_yaml_ng::Value::Bool(true)), - )?; - continue; + .find(|x| x.name == p.name) + .and_then(|x| x.oauth.as_ref()) + .filter(|t| t.endpoint == o.endpoint && !t.refresh.trim().is_empty()); + match from_then { + Some(t) => out = put_tokens(&out, &p.name, t)?, + None => { + out = tw_config::edit::set( + &out, + &provider_field(&out, &p.name, "disabled")?, + Some(&serde_yaml_ng::Value::Bool(true)), + )?; + } } - let tokens = tw_config::RenewedTokens { - access: o.access.as_deref().unwrap_or_default(), - expires_at: o.expires_at.as_deref(), - refresh: Some(&o.refresh), - }; - out = tw_config::patch_oauth_tokens(&out, name, tokens).map_err(|e| ModelError::Shape { + } + Ok(out) +} + +/// 把一份 OAuth 的令牌写进 `text` 里上游 `name` 那一家 +fn put_tokens(text: &str, name: &str, o: &tw_config::OAuth) -> Result { + let tokens = tw_config::RenewedTokens { + access: o.access.as_deref().unwrap_or_default(), + expires_at: o.expires_at.as_deref(), + refresh: Some(&o.refresh), + }; + let mut out = + tw_config::patch_oauth_tokens(text, name, tokens).map_err(|e| ModelError::Shape { item: format!("provider `{name}`"), detail: e.to_string(), })?; - // 没有 access token 的：去掉写进去的空值，下一次用到时拿 refresh 换 - if o.access.is_none() { - let at = provider_field(&out, name, "oauth")?; - let mut p = at.clone(); - p.push(tw_yaml::Step::key("access")); - out = tw_config::edit::set(&out, &p, None)?; - } + // 没有 access token 的：去掉写进去的空值，下一次用到时拿 refresh 换 + if o.access.is_none() { + let mut p = provider_field(&out, name, "oauth")?; + p.push(tw_yaml::Step::key("access")); + out = tw_config::edit::set(&out, &p, None)?; } Ok(out) } @@ -224,15 +306,42 @@ fn provider_field(text: &str, name: &str, field: &str) -> Result Pack { +/// 配置读不进来（安全模式）时的恢复：整份换成那一版，**本机那几节（监听、控制面的钥匙、 +/// 保留期、同步设置）一律是现在的**：取现在这份原文里的（读得成 YAML 就取得到，哪怕它读不成 +/// 一份配置），取不到就取 `last_local`（最近一个本机备份）里的，再取不到才留着那一版自己的。 +/// 钥匙是现在这一把。插件和通用偏好照那一版。 +/// +/// 那一版可以是同步位置上的、导出的，里面本来就没有这几节 —— 不补上的话写回去连钥匙都没有 +pub fn whole(backup: &Pack, current_text: &str, last_local: Option<&Pack>) -> Pack { + use serde_yaml_ng::Value as Yaml; let real = tw_config::control_key::raw_in(current_text) .ok() .flatten() - .or_else(|| raw_key_line(current_text)); - let text = tw_config::control_key::unmask(&backup.config_yaml, real.as_deref()) - .unwrap_or_else(|_| backup.config_yaml.clone()); + .or_else(|| raw_key_line(current_text)) + .filter(|k| k.as_str() != tw_api::control::KEY_MASK); + let doc = |t: &str| { + serde_yaml_ng::from_str::(t) + .ok() + .filter(Yaml::is_mapping) + }; + let source = doc(current_text).or_else(|| last_local.and_then(|p| doc(&p.config_yaml))); + let mut text = backup.config_yaml.clone(); + // 两处都取不到（原文坏到读不成 YAML、也没有本机备份）：留着那一版自己的 + for section in source.iter().flat_map(|_| model::LOCAL_SECTIONS) { + let v = source.as_ref().and_then(|d| d.get(*section)).cloned(); + if let Ok(t) = tw_config::edit::set(&text, &[tw_yaml::Step::key(*section)], v.as_ref()) { + text = t; + } + } + if let Some(k) = real { + let at: Vec = tw_api::control::KEY_PATH + .iter() + .map(|k| tw_yaml::Step::key(*k)) + .collect(); + if let Ok(t) = tw_yaml::insert(&text, &at, &tw_yaml::Scalar::s(k)) { + text = t; + } + } Pack { config_yaml: text, ..backup.clone() diff --git a/crates/tw-pack/src/restore/tests.rs b/crates/tw-pack/src/restore/tests.rs index 409f9076..8e838879 100644 --- a/crates/tw-pack/src/restore/tests.rs +++ b/crates/tw-pack/src/restore/tests.rs @@ -98,7 +98,7 @@ fn restoring_a_few_items_leaves_the_rest_as_it_is() { #[test] fn in_safe_mode_the_whole_version_comes_back_with_todays_key() { let broken = format!("version: 1\nlisten:\n control:\n key: {OLD_KEY}\nproviders: [\n"); - let out = whole(&then(), &broken); + let out = whole(&then(), &broken, None); assert!(out.config_yaml.contains(OLD_KEY), "{}", out.config_yaml); assert!(!out.config_yaml.contains(tw_api::control::KEY_MASK)); assert!(out.config_yaml.contains("port: 8000")); diff --git a/crates/tw-pack/tests/regressions.rs b/crates/tw-pack/tests/regressions.rs new file mode 100644 index 00000000..2fdf0340 --- /dev/null +++ b/crates/tw-pack/tests/regressions.rs @@ -0,0 +1,260 @@ +//! 审查查出来的几条，各一个回归测试：预览能过的，写下去一定成；不带密钥的包不抹本机的密钥； +//! 安全模式下整份恢复一个不带本机设置的版本。 + +use std::collections::{BTreeMap, BTreeSet}; + +use tw_pack::import::{Action, Context, Decision, Flag, Status}; +use tw_pack::model::{ItemId, ItemKind}; +use tw_pack::restore::{Change, Kept}; +use tw_pack::{Manifest, Pack, PackKind, PackOrigin}; + +const KEY: &str = "c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00c0ffee00"; + +fn pack(text: &str) -> Pack { + Pack { + manifest: Manifest::new(PackKind::Full, PackOrigin::Ui), + config_yaml: text.to_string(), + plugins: BTreeMap::new(), + client_prefs: None, + } +} + +fn ctx() -> Context<'static> { + Context { + has_aws_profile: &|_| false, + new_key: &|| "tw-regenerated".to_string(), + with_prefs: true, + } +} + +fn local() -> Pack { + pack(&format!( + "version: 1\nlisten:\n control:\n key: {KEY}\n gateway:\n port: 9000\nclients:\n - name: default\n key: tw-local\nproxies:\n - name: px\n type: http\n addr: 10.0.0.1:8080\n auth: {{ user: u, pass: secret-pass }}\nproviders:\n - name: relay\n base_url: https://relay.example\n key: sk-old\n proxy: px\n headers:\n X-Token: tok-old\n" + )) +} + +fn roundtrip(p: &Pack) -> Pack { + tw_pack::read( + &tw_pack::write(p, tw_pack::Seal::None).unwrap(), + tw_pack::Unseal::None, + ) + .unwrap() +} + +fn export(p: &Pack, secrets: bool, origin: PackOrigin) -> Pack { + roundtrip( + &tw_pack::export::export( + p, + tw_pack::export::Options { + scope: None, + secrets, + client_prefs: None, + origin, + }, + ) + .unwrap() + .pack, + ) +} + +fn decide(item: &ItemId, action: Action) -> Decision { + Decision { + item: item.clone(), + action, + } +} + +/// 不带密钥的包：除了密钥都一样的是「相同」；覆盖时本机的密钥留着；网关密钥只给新加的那把 +/// 重新生成 +#[test] +fn a_pack_without_secrets_keeps_the_secrets_here() { + let l = local(); + let incoming = export(&l, false, PackOrigin::Export); + let rows = tw_pack::import::preview(&l, &incoming, &ctx()).unwrap(); + for r in &rows { + assert_eq!(r.status, Status::Same, "{} {:?}", r.item, r.differs); + assert!(r.flags.is_empty(), "{} {:?}", r.item, r.flags); + } + // 包里的上游改了地址、多了一家、多了一把密钥 + let mut changed = incoming.clone(); + changed.config_yaml = "version: 1\nclients:\n - name: default\n key: ''\n - name: other\n key: ''\nproxies:\n - name: px\n type: http\n addr: 10.0.0.1:8080\n auth: { user: u, pass: '' }\nproviders:\n - name: relay\n base_url: https://relay.example/v2\n key: ''\n proxy: px\n headers:\n X-Token: ''\n - name: fresh\n base_url: https://fresh.example\n key: ''\n".into(); + let rows = tw_pack::import::preview(&l, &changed, &ctx()).unwrap(); + let row = |kind, name: &str| { + rows.iter() + .find(|r| r.item == ItemId::new(kind, name)) + .unwrap_or_else(|| panic!("{name}: {rows:?}\n{}", changed.config_yaml)) + }; + assert_eq!(row(ItemKind::Provider, "relay").status, Status::Conflict); + assert_eq!(row(ItemKind::Provider, "relay").flags, Vec::::new()); + assert_eq!(row(ItemKind::Provider, "fresh").flags, [Flag::MissingKey]); + assert_eq!(row(ItemKind::Key, "default").status, Status::Same); + let out = tw_pack::import::apply( + &l, + &changed, + &[ + decide(&ItemId::new(ItemKind::Provider, "relay"), Action::Overwrite), + decide(&ItemId::new(ItemKind::Provider, "fresh"), Action::Import), + decide(&ItemId::new(ItemKind::Key, "other"), Action::Import), + decide(&ItemId::new(ItemKind::Proxy, "px"), Action::Overwrite), + ], + &ctx(), + ) + .unwrap(); + let cfg = tw_config::try_parse(&out.pack.config_yaml).unwrap(); + let relay = cfg.providers.iter().find(|p| p.name == "relay").unwrap(); + assert_eq!(relay.base_url, "https://relay.example/v2"); + assert_eq!(relay.key.as_ref().map(|k| k.raw()), Some("sk-old")); + assert_eq!(relay.headers.get("X-Token").unwrap().value.raw(), "tok-old"); + assert!(!relay.disabled); + let fresh = cfg.providers.iter().find(|p| p.name == "fresh").unwrap(); + assert!(fresh.disabled); + assert_eq!( + cfg.proxies[0].auth.as_ref().unwrap().pass.raw(), + "secret-pass" + ); + let key = |n: &str| { + cfg.clients + .iter() + .find(|c| c.name == n) + .unwrap() + .key + .clone() + }; + assert_eq!(key("default"), "tw-local"); + assert_eq!(key("other"), "tw-regenerated"); + assert!(out.changed_keys.is_empty(), "{:?}", out.changed_keys); +} + +/// 安全模式下整份恢复一个同步版本（里面没有本机那几节）：钥匙、监听取现在这份原文里的； +/// 原文坏到读不成 YAML 时取最近一个本机备份里的 +#[test] +fn a_whole_restore_brings_back_this_machines_sections() { + let l = local(); + let synced = export(&l, true, PackOrigin::Sync); + // 读得成 YAML、读不成一份配置（安全模式最常见的样子） + let broken = format!("{}not_a_field: 1\n", l.config_yaml); + assert!(tw_config::try_parse(&broken).is_err()); + let out = tw_pack::restore::whole(&synced, &broken, None); + let cfg = tw_config::try_parse(&out.config_yaml) + .unwrap_or_else(|e| panic!("{e}\n{}", out.config_yaml)); + assert_eq!(cfg.listen.gateway.port, 9000); + assert!(out.config_yaml.contains(KEY)); + // 读不成 YAML：本机那几节从最近的本机备份来（钥匙在那里是打码的，换成原文里的这一把） + let not_yaml = format!("version: 1\nlisten:\n control:\n key: {KEY}\nproviders: [\n"); + let mut last = l.clone(); + last.config_yaml = tw_config::control_key::mask(&l.config_yaml); + let out = tw_pack::restore::whole(&synced, ¬_yaml, Some(&last)); + let cfg = tw_config::try_parse(&out.config_yaml) + .unwrap_or_else(|e| panic!("{e}\n{}", out.config_yaml)); + assert_eq!(cfg.listen.gateway.port, 9000); + assert_eq!(cfg.listen.control.key.as_deref(), Some(KEY)); +} + +/// 改名导入一把和本机同名同值的网关密钥：副本的值重新生成 +#[test] +fn a_key_imported_under_another_name_gets_its_own_value() { + let l = pack(&format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n key: tw-a\n - name: codex\n key: tw-codex\n" + )); + let incoming = + pack("version: 1\nclients:\n - name: codex\n key: tw-codex\n max_concurrent: 2\n"); + let rows = tw_pack::import::preview(&l, &incoming, &ctx()).unwrap(); + let row = rows.iter().find(|r| r.item.name == "codex").unwrap(); + assert_eq!(row.status, Status::Conflict); + let out = tw_pack::import::apply( + &l, + &incoming, + &[decide( + &row.item, + Action::Rename(row.suggested_name.clone().unwrap()), + )], + &ctx(), + ) + .unwrap(); + let cfg = tw_config::try_parse(&out.pack.config_yaml).unwrap(); + let copy = cfg.clients.iter().find(|c| c.name == "codex (2)").unwrap(); + assert_eq!(copy.key, "tw-regenerated"); + assert_eq!(copy.max_concurrent, Some(2)); +} + +/// 恢复留着固定密钥，它绑的路由也留着；预览如实说 +#[test] +fn a_kept_key_keeps_the_route_it_is_bound_to() { + let now = pack(&format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n key: tw-a\n - name: codex\n key: tw-cx\n route: fast\nproviders:\n - name: kept\n base_url: https://kept.example\n key: sk-k\nroutes:\n - name: fast\n rules:\n - name: all\n to: kept\n" + )); + let then = pack(&format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n key: tw-a\nproviders:\n - name: kept\n base_url: https://kept.example\n key: sk-k\n" + )); + let pinned: BTreeSet = ["codex".to_string()].into(); + let rows = tw_pack::restore::preview(&now, &then, &pinned, true).unwrap(); + let kept: Vec<(String, Change, Option)> = rows + .iter() + .map(|r| (r.item.to_string(), r.change, r.kept)) + .collect(); + assert!( + kept.contains(&( + "route `fast`".into(), + Change::Remove, + Some(Kept::Referenced) + )), + "{kept:?}" + ); + assert!( + kept.contains(&("key `codex`".into(), Change::Remove, Some(Kept::KeyInUse))), + "{kept:?}" + ); + let out = tw_pack::restore::apply(&now, &then, None, &pinned, true).unwrap(); + let cfg = tw_config::try_parse(&out.pack.config_yaml).unwrap(); + assert!(cfg.routes.iter().any(|r| r.name == "fast")); + assert!(cfg.clients.iter().any(|c| c.name == "codex")); +} + +/// 只选了一个已删的上游，它用的出站代理也删了：一起恢复 +#[test] +fn restoring_one_item_brings_what_it_needs() { + let now = pack(&format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n key: tw-a\n" + )); + let then = pack(&format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n key: tw-a\nproxies:\n - name: px\n type: socks5h\n addr: 127.0.0.1:7890\nproviders:\n - name: gone\n base_url: https://gone.example\n key: sk-g\n proxy: px\n" + )); + let out = tw_pack::restore::apply( + &now, + &then, + Some(&[ItemId::new(ItemKind::Provider, "gone")]), + &BTreeSet::new(), + true, + ) + .unwrap(); + let cfg = tw_config::try_parse(&out.pack.config_yaml).unwrap(); + assert_eq!(cfg.providers[0].proxy, "px"); + assert_eq!(cfg.proxies[0].name, "px"); +} + +/// 恢复一个后来换了令牌端点的 OAuth 上游：本机的令牌用不上；那一版有令牌的取回来，没有的 +/// 停用着回来，都不是恢复失败 +#[test] +fn an_oauth_upstream_whose_endpoint_changed_comes_back() { + let oauth = |rt: &str, ep: &str| { + format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n key: tw-a\nproviders:\n - name: o\n base_url: https://corp.example\n oauth:\n refresh: '{rt}'\n endpoint: {ep}\n" + ) + }; + let now = pack(&oauth("rt-now", "https://t2.example/token")); + let then = pack(&oauth("rt-then", "https://t1.example/token")); + let out = tw_pack::restore::apply(&now, &then, None, &BTreeSet::new(), true).unwrap(); + let cfg = tw_config::try_parse(&out.pack.config_yaml).unwrap(); + let o = cfg.providers[0].oauth.as_ref().unwrap(); + assert_eq!(o.endpoint, "https://t1.example/token"); + assert_eq!(o.refresh, "rt-then"); + assert!(!cfg.providers[0].disabled); + // 那一版里也没有令牌：停用着回来 + let blank = pack( + &oauth("", "https://t1.example/token") + .replace(" oauth:", " disabled: true\n oauth:"), + ); + let out = tw_pack::restore::apply(&now, &blank, None, &BTreeSet::new(), true).unwrap(); + let cfg = tw_config::try_parse(&out.pack.config_yaml).unwrap(); + assert!(cfg.providers[0].disabled); +} From c772398a9d6e2f216c6e5102cf53217e4f917bdf Mon Sep 17 00:00:00 2001 From: fylorn <249551762+fylorn@users.noreply.github.com> Date: Mon, 19 Oct 2026 17:09:37 +0800 Subject: [PATCH 07/13] Sync: encryption cannot be downgraded, sign-ins stay local, nothing is swallowed (#357) * Sync: encryption cannot be downgraded, sign-ins stay local, nothing is swallowed Fixes from the review of the sync integration branch. Encryption - With encryption on (by setting, or a keyring at the location), versions and device records that are not encrypted are never read (tw_pack's read_strict); they are skipped and counted. - A missing keyring, an older "off" keyring put back, or one that stays unreadable is a state for a person (keyring_missing), never a reason to turn encryption off or drop the passphrase. Turning encryption off now rewrites the keyring as "off" (sealed with the passphrase, with a plain marker for devices that have none), and each keyring carries a growing generation so an old one is recognised. - Each device keeps the last keyring it opened; POST /sync/republish puts it back (or makes a new one) and uploads this device's configuration when the location has nothing readable. - A device whose own versions no longer open after a keyring race merges two-way, re-uploads with the keyring now there and deletes them. Merging - Upstreams signed in on another device (ChatGPT, OAuth tokens) stay disabled on a device without the sign-in; that disable is local state, never uploaded, in rounds and in both join modes. Signing in turns them on as synced. - The "nothing changed" shortcut is keyed on what the round started from plus what it did, so an edit or a version that lands mid-round is synced next round. Local content is hashed raw, so a sign-in counts. - The base is used only when it is an ancestor of the location's heads (versions record up to 200 ancestors); otherwise the merge is two-way. Pruning keeps the common ancestor of several heads, and earlier bases are kept locally to find one that was already pruned. - Global sections and client prefs reset to defaults merge field by field. - Invalid-merge fallback applies changes in reference order and repeats until nothing more fits. - Keeping both gateway keys gives the copy a new key and no client. Conflicts list their choices (options); "use the other side" is refused for invalid conflicts. The location - A file that is clearly broken is skipped at once; one that may still be arriving is waited for 30 minutes, then skipped. The count is in the view. - An empty read is checked over several rounds and ten minutes before the emptied state; nothing is turned off or deleted automatically. A WebDAV folder that is gone is unreachable, not empty. Purging deletes only sync files. - iCloud placeholders read as not yet arrived. Reads are size-limited while streaming. Leftover temporary files of this device are swept. A request on a pooled connection the server just closed is sent once more. - S3 prefixes may not contain .. or start with /; a plain http:// address cannot go through a proxy. API (still 50): sync states keyring_missing, emptied, misconfigured; SyncView.skipped; SyncConflict.options; POST /sync/republish; twcore sync republish. New codes: sync.keyring_missing, sync.keyring_rolled_back, sync.looks_empty, sync.location_empty (replaces sync.emptied), sync.take_not_allowed, sync.too_new_unknown, sync.store.too_large, sync.webdav.root_missing, sync.config.prefix, sync.config.plain_http_proxy. The fake S3 in the tests now verifies SigV4 signatures with its own implementation, checked against AWS's published examples. Co-Authored-By: Claude Opus 5.5 * Sync: a device with the passphrase ignores the plain "off" marker The marker in front of an "off" keyring is not authenticated. Only devices without a passphrase (never encrypted) go by it; a device that has the passphrase opens the keyring and goes by what it says, so a forged marker asks for the passphrase instead of turning encryption off. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- Cargo.lock | 1 + bin/twcore/src/main.rs | 4 + bin/twcore/src/sync_cmd.rs | 22 +- crates/tw-api/msg-codes.txt | 11 +- crates/tw-api/src/ep.rs | 4 + crates/tw-api/src/lib.rs | 5 +- crates/tw-api/src/sync.rs | 21 +- crates/tw-config/src/sync.rs | 78 +- crates/tw-config/tests/manual/schema.rs | 12 +- crates/tw-control/src/sync.rs | 83 +- crates/tw-control/tests/sync_devices.rs | 29 +- crates/tw-control/tests/sync_regressions.rs | 347 +++++ crates/tw-sync/Cargo.toml | 1 + crates/tw-sync/src/device.rs | 34 +- crates/tw-sync/src/engine.rs | 1548 ++++++++++++++----- crates/tw-sync/src/keyring.rs | 132 +- crates/tw-sync/src/layout.rs | 49 + crates/tw-sync/src/local.rs | 95 +- crates/tw-sync/src/merge.rs | 59 +- crates/tw-sync/src/record.rs | 33 + crates/tw-sync/src/store/folder.rs | 188 ++- crates/tw-sync/src/store/mod.rs | 88 +- crates/tw-sync/src/store/s3.rs | 18 +- crates/tw-sync/src/store/webdav.rs | 88 +- crates/tw-sync/tests/common/mod.rs | 220 ++- crates/tw-sync/tests/devices.rs | 63 +- crates/tw-sync/tests/regressions.rs | 1118 ++++++++++++++ crates/tw-sync/tests/stores.rs | 171 +- docs/config.md | 18 +- docs/config.zh-CN.md | 12 +- 30 files changed, 4023 insertions(+), 529 deletions(-) create mode 100644 crates/tw-control/tests/sync_regressions.rs create mode 100644 crates/tw-sync/tests/regressions.rs diff --git a/Cargo.lock b/Cargo.lock index 7b0055cb..f9224567 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3985,6 +3985,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "serde_yaml_ng", "sha2 0.11.0", "tempfile", "thiserror 2.0.20", diff --git a/bin/twcore/src/main.rs b/bin/twcore/src/main.rs index 1a2e251e..dc03ad21 100644 --- a/bin/twcore/src/main.rs +++ b/bin/twcore/src/main.rs @@ -176,6 +176,9 @@ enum SyncCmd { #[arg(long)] purge: bool, }, + /// After the sync location was emptied or lost its keyring: upload this device's + /// configuration again (and put back the keyring this device keeps) + Republish, } #[derive(Subcommand)] @@ -289,6 +292,7 @@ fn main() -> Result<()> { SyncCmd::Now => sync_cmd::now(&path), SyncCmd::Enable => sync_cmd::enable(&path), SyncCmd::Disable { purge } => sync_cmd::disable(&path, purge), + SyncCmd::Republish => sync_cmd::republish(&path), }, Command::Upgrade { check, diff --git a/bin/twcore/src/sync_cmd.rs b/bin/twcore/src/sync_cmd.rs index c52ff7b8..b9965132 100644 --- a/bin/twcore/src/sync_cmd.rs +++ b/bin/twcore/src/sync_cmd.rs @@ -102,6 +102,19 @@ fn print_view(v: &tw_api::SyncView) { if !v.excluded.is_empty() { println!("not synced: {}", v.excluded.join(", ")); } + if v.skipped > 0 { + println!( + "skipped: {} file(s) at the sync location could not be read", + v.skipped + ); + } + match v.state { + tw_api::SyncState::Emptied | tw_api::SyncState::KeyringMissing => println!( + "to go on: `twcore sync republish` uploads this device's configuration again, \ + `twcore sync disable` stops syncing" + ), + _ => {} + } for d in &v.devices { let mut notes = Vec::new(); if d.this { @@ -139,11 +152,18 @@ pub fn now(config: &Path) -> Result<()> { Ok(()) } +/// 同步位置空了、keyring 不见了之后：以本机配置重新上传（keyring 放回本机记着的那一份） +pub fn republish(config: &Path) -> Result<()> { + let v: tw_api::SyncView = send(config, "POST", "/sync/republish", None)?; + print_view(&v); + Ok(()) +} + pub fn disable(config: &Path, purge: bool) -> Result<()> { if purge && !yes( "Delete every version, device record and the keyring at the sync location? Other \ - devices stop syncing", + devices are told the location is empty", false, )? { diff --git a/crates/tw-api/msg-codes.txt b/crates/tw-api/msg-codes.txt index 3f3a4c23..8a1cda17 100644 --- a/crates/tw-api/msg-codes.txt +++ b/crates/tw-api/msg-codes.txt @@ -555,8 +555,9 @@ sync.config.device_name sync.config.folder_relative sync.config.multiline sync.config.passphrase_missing +sync.config.plain_http_proxy +sync.config.prefix sync.config.unknown_proxy -sync.emptied sync.folder.denied sync.folder.io sync.folder.missing @@ -564,11 +565,15 @@ sync.folder.task sync.http_client sync.join_undecided sync.keyring_corrupt +sync.keyring_missing +sync.keyring_rolled_back sync.keyring_too_new sync.local sync.local.io +sync.location_empty sync.location_taken sync.locked_new +sync.looks_empty sync.merge_invalid sync.not_arrived sync.not_encrypted @@ -586,12 +591,16 @@ sync.store.bad_path sync.store.denied sync.store.exists sync.store.server_error +sync.store.too_large sync.store.unexpected sync.store.unreachable +sync.take_not_allowed sync.too_new +sync.too_new_unknown sync.version_gone sync.webdav.bad_listing sync.webdav.bad_url +sync.webdav.root_missing sync.write_section t.auth test t.boom test diff --git a/crates/tw-api/src/ep.rs b/crates/tw-api/src/ep.rs index 138591a6..34735dc5 100644 --- a/crates/tw-api/src/ep.rs +++ b/crates/tw-api/src/ep.rs @@ -282,6 +282,10 @@ endpoints! { SyncUnlock: POST "/sync/unlock", api::SyncUnlock => api::SyncView; /// 停止同步；`purge` 时连同步位置上的数据一起删掉 SyncDisable: POST "/sync/disable", api::SyncDisable => (); + /// 同步位置空了（`emptied`）、keyring 不见了（`keyring_missing`）之后由人选的「重新上传本机 + /// 配置」：keyring 放回这台设备记着的那一份（没有就新建），没有读得出来的版本时以本机配置 + /// 上传一个，然后同步一轮 + SyncRepublish: POST "/sync/republish", () => api::SyncView; // ─────────────────────────────────────────────── 账号登录 StartChatgptLogin: POST "/chatgpt/login", api::ChatgptLoginStart => api::ChatgptLogin; diff --git a/crates/tw-api/src/lib.rs b/crates/tw-api/src/lib.rs index 24d27632..16bd6778 100644 --- a/crates/tw-api/src/lib.rs +++ b/crates/tw-api/src/lib.rs @@ -990,7 +990,10 @@ pub const MSG_CODES: &str = include_str!("../msg-codes.txt"); /// `PUT /sync/settings`（[`SyncSettingsSave`]）、`POST /sync/now`、`GET` / `POST /sync/conflicts` /// （[`SyncConflict`]、[`SyncResolve`]：采用本机、另一边或两者都保留）、`POST /sync/encryption` /// （[`SyncEncryption`]：开启、关闭、更改口令）、`POST /sync/unlock`（[`SyncUnlock`]）、 -/// `POST /sync/disable`（[`SyncDisable`]）。状态变了发 [`Event::SyncStatus`]；应用别的设备的 +/// `POST /sync/disable`（[`SyncDisable`]）、`POST /sync/republish`（同步位置空了、keyring 不见了 +/// 之后由人选的「重新上传本机配置」，→ [`SyncView`]）。[`SyncState`] 里要人处理的有 +/// `keyring_missing`、`emptied`、`misconfigured`，[`SyncView`] 的 `skipped` 是同步位置上跳过了的 +/// 文件数，[`SyncConflict`] 的 `options` 是能选的处理。状态变了发 [`Event::SyncStatus`]；应用别的设备的 /// 版本时配置的来源是 `sync`，改了网关密钥时发 `GatewayKeysChanged`。[`ProviderView`] 多了 /// `sync_excluded`（不同步的上游）和 `sync_url_changed`（地址经同步变更，改之前的地址）。 /// 同步的码是 `sync.*`。安全模式下不自动同步（`paused`） diff --git a/crates/tw-api/src/sync.rs b/crates/tw-api/src/sync.rs index 18800f47..c7dd837f 100644 --- a/crates/tw-api/src/sync.rs +++ b/crates/tw-api/src/sync.rs @@ -72,6 +72,17 @@ slug_enum! { TooNew = "too_new", /// 安全模式：不自动同步 Paused = "paused", + /// 同步位置上的 keyring 不见了、被换成了更早的、一直读不出来。**不会自己关掉加密**： + /// 由人选「重新上传本机配置」（`POST /sync/republish`：放回这台设备记着的 keyring）或者 + /// 停止同步。原因在 `error` + KeyringMissing = "keyring_missing", + /// 同步位置空了（连着几轮、过了一段时间都是空的）：其他设备删除了同步数据，或者它已经 + /// 不是原来那个位置。同步设置不动，由人选「重新上传本机配置」（`POST /sync/republish`） + /// 或者停止同步 + Emptied = "emptied", + /// 同步设置用不了（比如出站代理被删了、不加密的 `http://` 地址设了代理）：改设置。 + /// 原因在 `error` + Misconfigured = "misconfigured", } } @@ -122,6 +133,9 @@ pub struct SyncView { pub excluded: Vec, /// 随之不同步的项目：引用了不同步的上游的路由、别名、分组 pub excluded_cascade: Vec, + /// 同步位置上跳过了的文件：坏了的、不是这把密钥加密的、加密的位置上没加密的、等了 + /// 30 分钟还没传完的。不是 0 时界面提一句 + pub skipped: u32, } /// `POST /sync/test`、`POST /sync/enable/preview`：连上去看看。 @@ -256,9 +270,14 @@ pub struct SyncConflict { pub remote: SyncConflictSide, /// 两边不同的地方，逐个单位：点对象名时显示的差异 pub diff: Vec, - /// 能不能「两者都保留」：两边都还有这个对象，而且它有名字 + /// 能不能「两者都保留」：两边都还有这个对象，而且它有名字（和 `options` 里有没有 `both` + /// 是一回事） pub both_allowed: bool, pub reason: Option, + /// 这处冲突能选哪几种处理，界面只给这几个按钮：`local` 总在；`remote` 在合并结果过不了 + /// 校验（`invalid`）时没有；`both` 同 `both_allowed`。网关密钥的「两者都保留」新增的那一把 + /// 是一个新的值 + pub options: Vec, } slug_enum! { diff --git a/crates/tw-config/src/sync.rs b/crates/tw-config/src/sync.rs index 7a460ee4..4179739f 100644 --- a/crates/tw-config/src/sync.rs +++ b/crates/tw-config/src/sync.rs @@ -230,6 +230,12 @@ pub enum SyncError { BlankExclude, #[error("{}", self.msg())] MultiLine { field: &'static str }, + /// S3 前缀里有 `..`、`.` 这样的段，或者以 `/` 开头 + #[error("{}", self.msg())] + Prefix { prefix: String }, + /// `http://` 的内网地址经出站代理：同步位置的密码会明文交给代理 + #[error("{}", self.msg())] + PlainHttpProxy { proxy: String }, } impl SyncError { @@ -278,6 +284,16 @@ impl SyncError { "sync.config.multiline", field = field => "sync.{field} has to be on one line" ), + SyncError::Prefix { prefix } => msg!( + "sync.config.prefix", prefix = prefix => + "{prefix} is not a folder inside the bucket: no leading /, and no . or .. parts" + ), + SyncError::PlainHttpProxy { proxy } => msg!( + "sync.config.plain_http_proxy", proxy = proxy => + "an http:// sync location on the local network cannot go through proxy \ + `{proxy}`: its password would reach the proxy in the clear. Use direct, or an \ + https:// address" + ), } } } @@ -366,6 +382,11 @@ impl Target { if prefix.contains(['\n', '\r']) { return Err(SyncError::MultiLine { field: "prefix" }); } + if !prefix_ok(prefix) { + return Err(SyncError::Prefix { + prefix: prefix.clone(), + }); + } if !address_ok(endpoint) { return Err(SyncError::Address { field: "endpoint", @@ -383,9 +404,36 @@ impl Target { } } -/// 代理名对得上吗：`direct`、`system`，或者 `proxies` 里有的那一个 +/// S3 前缀：桶里的一个文件夹。不以 `/` 开头，没有 `.`、`..` 段和反斜杠（结尾的 `/` 无所谓） +fn prefix_ok(p: &str) -> bool { + let p = p.trim(); + !p.starts_with('/') + && !p.contains('\\') + && p.trim_end_matches('/') + .split('/') + .filter(|s| !p.is_empty() || !s.is_empty()) + .all(|seg| p.is_empty() || (!seg.is_empty() && seg != "." && seg != "..")) +} + +/// 地址是不是 `http://`（只给内网的那种） +fn plain_http(t: &Target) -> bool { + let url = match t { + Target::Webdav { url, .. } => url, + Target::S3 { endpoint, .. } => endpoint, + Target::Folder { .. } => return false, + }; + url.trim().to_ascii_lowercase().starts_with("http://") +} + +/// 代理名对得上吗：`direct`、`system`，或者 `proxies` 里有的那一个。`http://` 的内网地址 +/// 只能直连 pub fn check_proxy(target: &Target, proxies: &[crate::Proxy]) -> Result<(), SyncError> { let p = target.proxy(); + if p != DIRECT && plain_http(target) { + return Err(SyncError::PlainHttpProxy { + proxy: p.to_string(), + }); + } if p == DIRECT || p == crate::SYSTEM || proxies.iter().any(|x| x.name == p) { return Ok(()); } @@ -613,6 +661,34 @@ mod tests { assert_ne!(generate_device_id(), generate_device_id()); } + #[test] + fn an_s3_prefix_stays_inside_the_bucket() { + for ok in ["", "me", "team/tw", "team/tw/"] { + assert!(prefix_ok(ok), "{ok:?}"); + } + for bad in ["/abs", "a/../b", "..", "./x", "a//b", "a\\b"] { + assert!(!prefix_ok(bad), "{bad:?}"); + } + } + + #[test] + fn a_plain_http_location_never_goes_through_a_proxy() { + let mut t = webdav("http://192.168.1.2:5005/dav").target; + check_proxy(&t, &[]).unwrap(); + if let Target::Webdav { proxy, .. } = &mut t { + *proxy = "system".into(); + } + assert!(matches!( + check_proxy(&t, &[]), + Err(SyncError::PlainHttpProxy { .. }) + )); + let mut https = webdav("https://dav.example.com/").target; + if let Target::Webdav { proxy, .. } = &mut https { + *proxy = "system".into(); + } + check_proxy(&https, &[]).unwrap(); + } + #[test] fn folder_paths_have_to_be_absolute_on_either_platform() { for ok in [ diff --git a/crates/tw-config/tests/manual/schema.rs b/crates/tw-config/tests/manual/schema.rs index d885ec39..7d01198d 100644 --- a/crates/tw-config/tests/manual/schema.rs +++ b/crates/tw-config/tests/manual/schema.rs @@ -126,8 +126,8 @@ fn target_types() -> Vec<&'static str> { } const PROXY_DOC: T2 = t( - "Where requests to the sync location go out: `direct`, `system` (the system proxy), or the name of a proxy under `proxies`.", - "访问同步位置的请求从哪里出去：`direct`（直连）、`system`（跟随系统代理），或者 `proxies` 里一个代理的名字。", + "Where requests to the sync location go out: `direct`, `system` (the system proxy), or the name of a proxy under `proxies`. An `http://` address on this machine or the local network has to be `direct`.", + "访问同步位置的请求从哪里出去：`direct`（直连）、`system`（跟随系统代理），或者 `proxies` 里一个代理的名字。本机、内网的 `http://` 地址只能是 `direct`。", ); const RULE_ID: T2 = t("built-in rule id", "内置规则 id"); @@ -1921,8 +1921,8 @@ pub fn sections() -> Vec
{ Kind::Str, Def::Unset, t( - "The sync passphrase; required when `encrypt` is on. Kept here in the clear, like the other keys in this file.", - "同步口令，`encrypt` 开着时必填。和本文件里的其他密钥一样以明文保存。", + "The sync passphrase; required when `encrypt` is on, and kept after encryption is turned off: the keyring that says it is off is checked with it. Kept here in the clear, like the other keys in this file.", + "同步口令，`encrypt` 开着时必填；关闭加密之后也留着：同步位置上写着「已关闭」的 keyring 要用它核对。和本文件里的其他密钥一样以明文保存。", ), ), row( @@ -2066,8 +2066,8 @@ pub fn sections() -> Vec
{ Kind::Str, Def::Is("''"), t( - "A folder inside the bucket.", - "存储桶里的一个前缀（文件夹）。", + "A folder inside the bucket. No `..` parts, and not starting with `/`.", + "存储桶里的一个前缀（文件夹）。不能有 `..`，不以 `/` 开头。", ), ), row( diff --git a/crates/tw-control/src/sync.rs b/crates/tw-control/src/sync.rs index 41e27f91..3a6883b2 100644 --- a/crates/tw-control/src/sync.rs +++ b/crates/tw-control/src/sync.rs @@ -60,6 +60,7 @@ pub fn router() -> axum::Router { .at(ep::SyncEncryption, encryption) .at(ep::SyncUnlock, unlock) .at(ep::SyncDisable, disable) + .at(ep::SyncRepublish, republish) } fn now_ms() -> u64 { @@ -212,6 +213,9 @@ impl Host for CoreHost { fn http(cfg: &tw_config::Config, proxy: &str) -> Result { let mut b = reqwest::Client::builder() .connect_timeout(Duration::from_secs(10)) + // 一轮里的请求是接连发的；两轮之间的空闲连接不留：很多服务端几秒就关掉空闲连接， + // 留着的下一轮一发就断（发请求那一处另有一次重发） + .pool_idle_timeout(Duration::from_secs(4)) .redirect(reqwest::redirect::Policy::none()) .user_agent(tw_gateway::user_agent()); match proxy { @@ -444,8 +448,11 @@ fn round_fail(e: &RoundError) -> Fail { RoundError::Store(StoreError::AuthFailed(_)) => StatusCode::FORBIDDEN, RoundError::Store(_) => StatusCode::BAD_GATEWAY, RoundError::Locked(_) => StatusCode::BAD_REQUEST, - RoundError::TooNew { .. } | RoundError::Stale | RoundError::Emptied => StatusCode::CONFLICT, - RoundError::NotArrived => StatusCode::SERVICE_UNAVAILABLE, + RoundError::TooNew { .. } + | RoundError::Stale + | RoundError::Emptied + | RoundError::KeyringMissing(_) => StatusCode::CONFLICT, + RoundError::NotArrived | RoundError::LooksEmpty => StatusCode::SERVICE_UNAVAILABLE, RoundError::Local(_) => StatusCode::BAD_REQUEST, }; fail(code, e.msg()) @@ -605,6 +612,7 @@ impl Sync { conflicts: 0, excluded: Vec::new(), excluded_cascade: Vec::new(), + skipped: 0, }; let Some(mgr) = self.mgr() else { return off }; let cfg = mgr.gateway().config(); @@ -627,6 +635,11 @@ impl Sync { .map(|r| r.conflicts.len() as u32) .or(prev.as_ref().map(|p| p.conflicts)) .unwrap_or(0); + let skipped = report + .as_ref() + .map(|r| r.skipped) + .or(prev.as_ref().map(|p| p.skipped)) + .unwrap_or(0); let devices = match &report { Some(r) => r .devices @@ -680,6 +693,7 @@ impl Sync { conflicts, excluded: sync.exclude.clone(), excluded_cascade: cascade, + skipped, } } @@ -723,9 +737,10 @@ impl Sync { return Ok(v); } Err(m) => { + // 照设置建不起来（出站代理没了、地址不合规矩）：改设置，不是拒绝访问 let v = tw_api::SyncView { error: Some(m.clone()), - ..self.compute_view(None, Some(tw_api::SyncState::AuthFailed)) + ..self.compute_view(None, Some(tw_api::SyncState::Misconfigured)) }; self.publish(v.clone()); return Err(fail(StatusCode::BAD_REQUEST, m)); @@ -771,11 +786,13 @@ impl Sync { at_ms: now_ms(), }); } - // 别的设备关掉了加密：这台跟着改设置 - let follow = (encrypted != r.config.encrypt && !encrypted).then(|| { + // 别的设备关掉了加密（keyring 写着已关闭，口令解得开）、或者又开了（口令还是 + // 这台记着的那个）：这台跟着改设置。**口令留着**：靠它认出以后的 keyring + let follow = (encrypted != r.config.encrypt + && (!encrypted || r.config.passphrase.is_some())) + .then(|| { let mut s = r.config.clone(); - s.encrypt = false; - s.passphrase = None; + s.encrypt = encrypted; s }); Ok((report, follow)) @@ -795,27 +812,16 @@ impl Sync { drop(guard); self.view() } - Err(RoundError::Emptied) => { - // 别的设备停止同步并删除了同步数据：这台也停下，说一声 - r.engine.local().clear(); - drop(guard); - if let Some(mgr) = self.mgr() { - let _ = write_section(&mgr, None, Origin::Sync).await; - } - *self.run.lock().await = None; - self.register_backups(false); - tw_api::SyncView { - error: Some(RoundError::Emptied.msg()), - ..self.compute_view(None, Some(tw_api::SyncState::Off)) - } - } Err(e) => { + // 同步位置空了、keyring 不见了也一样：记下来、等人处理，**同步设置不动** self.record_failure(r, &e); let st = r.engine.local().state(); let state = match &e { RoundError::Store(StoreError::AuthFailed(_)) => tw_api::SyncState::AuthFailed, RoundError::Locked(_) => tw_api::SyncState::Locked, RoundError::TooNew { .. } => tw_api::SyncState::TooNew, + RoundError::Emptied => tw_api::SyncState::Emptied, + RoundError::KeyringMissing(_) => tw_api::SyncState::KeyringMissing, _ => { let since = st.failing_since_ms.unwrap_or_else(now_ms); if now_ms().saturating_sub(since) >= UNREACHABLE_AFTER_MS { @@ -904,6 +910,14 @@ impl Sync { tw_sync::record::Reason::KeyInUse => tw_api::SyncConflictReason::KeyInUse, tw_sync::record::Reason::Invalid => tw_api::SyncConflictReason::Invalid, }), + options: engine::options(c) + .into_iter() + .map(|t| match t { + Take::Local => tw_api::SyncTake::Local, + Take::Remote => tw_api::SyncTake::Remote, + Take::Both => tw_api::SyncTake::Both, + }) + .collect(), } }) .collect() @@ -1563,9 +1577,10 @@ async fn encryption( r.engine.enable_encryption(&p).await } tw_api::SyncEncryptionAction::Disable => { + // 口令留着：同步位置上的 keyring 写着「已关闭」，靠它认出来 let p = req.passphrase.clone().unwrap_or_default(); next.encrypt = false; - next.passphrase = None; + next.passphrase = Some(p.clone()).filter(|p| !p.is_empty()); r.engine.disable_encryption(&p).await } tw_api::SyncEncryptionAction::Change => { @@ -1602,10 +1617,8 @@ async fn unlock( .await .map_err(|e| round_fail(&e))?; let mut next = r.config.clone(); - if encrypted { - next.encrypt = true; - next.passphrase = Some(req.passphrase.clone()); - } + next.encrypt = encrypted; + next.passphrase = Some(req.passphrase.clone()); r.config = next.clone(); r.engine.set_settings(settings_of(&s.config(), &next)); drop(guard); @@ -1615,6 +1628,24 @@ async fn unlock( sync.round(&[]).await.map(Json) } +/// 同步位置空了、keyring 不见了之后，由人选的「重新上传本机配置」 +async fn republish(State(s): State) -> Result, Fail> { + let sync = service(&s); + let mut guard = sync.run.lock().await; + let r = match sync.ensure(&mut guard) { + Ok(Some(r)) => r, + Ok(None) => return Err(sync_off()), + Err(m) => return Err(fail(StatusCode::BAD_REQUEST, m)), + }; + let res = r.engine.republish().await; + drop(guard); + if let Err(e) = res { + return Err(round_fail(&e)); + } + sync.failures.store(0, Ordering::Relaxed); + sync.round(&[]).await.map(Json) +} + async fn disable( State(s): State, Json(req): Json, diff --git a/crates/tw-control/tests/sync_devices.rs b/crates/tw-control/tests/sync_devices.rs index 4ac190da..038dcf7e 100644 --- a/crates/tw-control/tests/sync_devices.rs +++ b/crates/tw-control/tests/sync_devices.rs @@ -432,11 +432,32 @@ async fn turning_sync_off_keeps_the_configuration_and_purging_empties_the_locati assert!(!a.config().providers.is_empty(), "本机保留当前配置"); assert!(!a.dir.path().join("sync").exists(), "本机的同步状态删掉了"); assert!(!loc.path().join("keyring").exists()); - // B 发现同步位置空了：停下，说一声 + // B：读到空的先不下结论；连着几轮、过了一段时间才是「同步位置空了」，同步设置不动，由人 + // 选重新上传或者停止 + for _ in 0..3 { + let v = b.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["error"]["code"], "sync.looks_empty", "{v}"); + assert_ne!(v["state"], "emptied"); + } + let local = tw_sync::local::Local::new(b.dir.path()); + let mut st = local.state(); + st.empty_since_ms = st.empty_since_ms.map(|t| t - 11 * 60 * 1000); + local.save_state(&st).unwrap(); let v = b.ok("POST", "/sync/now", Value::Null).await; - assert_eq!(v["state"], "off", "{v}"); - assert_eq!(v["error"]["code"], "sync.emptied"); - assert!(b.config().sync.is_none()); + assert_eq!(v["state"], "emptied", "{v}"); + assert_eq!(v["error"]["code"], "sync.location_empty"); + assert!(b.config().sync.is_some(), "同步设置不动"); + // 重新上传本机配置：keyring 放回 B 记着的那一份，版本以 B 的配置上传 + let v = b.ok("POST", "/sync/republish", Value::Null).await; + assert_eq!(v["state"], "idle", "{v}"); + assert!(v["error"].is_null(), "{v}"); + assert!(loc.path().join("keyring").exists()); + assert_eq!( + std::fs::read_dir(loc.path().join("versions")) + .unwrap() + .count(), + 1 + ); } /// 调度：本机改了配置，10 秒合并之后上传；另一台的文件夹监听一收到就合并 diff --git a/crates/tw-control/tests/sync_regressions.rs b/crates/tw-control/tests/sync_regressions.rs new file mode 100644 index 00000000..5473cf7f --- /dev/null +++ b/crates/tw-control/tests/sync_regressions.rs @@ -0,0 +1,347 @@ +//! 审查里找出来的同步问题在控制面这一层钉住：等登录的上游（加入的两种方式、登录之后）、要人 +//! 处理的状态（keyring 不见了）、冲突给出能选的处理、关闭加密之后口令留着。 + +use std::sync::Arc; + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use serde_json::{Value, json}; +use tower::ServiceExt; +use tw_control::{ConfigManager, ControlState}; + +const PASS: &str = "correct horse battery staple"; + +fn base(key: char) -> String { + format!( + "version: 1\nlisten:\n control:\n key: {}\nclients:\n - name: default\n key: tw-default-{key}\n", + key.to_string().repeat(64) + ) +} + +struct Dev { + /// 数据目录：活到测试结束 + _dir: tempfile::TempDir, + state: ControlState, + app: axum::Router, +} + +impl Dev { + fn new(key: char) -> Dev { + tw_sync::keyring::set_work_factor_for_tests(4); + let dir = tempfile::tempdir().unwrap(); + let p = dir.path().join("config.yaml"); + let text = base(key); + std::fs::write(&p, &text).unwrap(); + let cfg = tw_config::try_parse(&text).unwrap(); + let gw = tw_gateway::AppState::new(cfg).unwrap(); + gw.set_plugin_engine(Arc::new(tw_gateway::plugin::fake::FakeEngine)); + let bus = gw.bus.clone(); + let state = ControlState { + shutdown: Default::default(), + remote: Default::default(), + cfg: Arc::new(ConfigManager::new(p, gw.clone(), bus)), + gateway: gw, + store: None, + started: std::time::Instant::now(), + price_updater: Default::default(), + chatgpt: Default::default(), + zai: Default::default(), + }; + Dev { + app: tw_control::router(state.clone()), + state, + _dir: dir, + } + } + + async fn call(&self, method: &str, path: &str, body: Value) -> (StatusCode, Value) { + let r = self + .app + .clone() + .oneshot( + Request::builder() + .method(method) + .uri(path) + .header("content-type", "application/json") + .body(Body::from(if body.is_null() { + String::new() + } else { + body.to_string() + })) + .unwrap(), + ) + .await + .unwrap(); + let status = r.status(); + let bytes = axum::body::to_bytes(r.into_body(), usize::MAX) + .await + .unwrap(); + ( + status, + serde_json::from_slice(&bytes).unwrap_or(Value::Null), + ) + } + + async fn ok(&self, method: &str, path: &str, body: Value) -> Value { + let (s, v) = self.call(method, path, body).await; + assert!(s.is_success(), "{method} {path}: {s} {v}"); + v + } + + fn config(&self) -> tw_config::Config { + self.state.config().as_ref().clone() + } + + fn provider(&self, name: &str) -> Option { + self.config().providers.into_iter().find(|p| p.name == name) + } + + /// 用户在界面上改了配置（`PUT /config`） + async fn edit(&self, f: impl FnOnce(&str) -> String) { + let cur = self.ok("GET", "/config", Value::Null).await; + let next = f(cur["text"].as_str().unwrap()); + self.ok( + "PUT", + "/config", + json!({ "base_version": cur["version"], "text": next }), + ) + .await; + } + + async fn enable(&self, loc: &std::path::Path, encrypt: bool, name: &str, join: Value) { + let mut body = json!({ + "target": folder(loc), "encrypt": encrypt, "device_name": name, "join": "use_remote", + }); + if encrypt { + body["passphrase"] = json!(PASS); + } + if let Value::Object(extra) = join { + for (k, v) in extra { + body[k] = v; + } + } + self.ok("POST", "/sync/enable", body).await; + } +} + +fn folder(loc: &std::path::Path) -> Value { + json!({ "type": "folder", "path": loc.display().to_string(), "proxy": "direct" }) +} + +fn add_raw(text: &str, yaml: &str) -> String { + if text.contains("\nproviders:\n") { + text.replacen("\nproviders:\n", &format!("\nproviders:\n{yaml}"), 1) + } else { + format!("{text}providers:\n{yaml}") + } +} + +const CHATGPT: &str = " - name: chatgpt\n base_url: https://chatgpt.com/backend-api/codex\n oauth:\n refresh: rt-a\n endpoint: https://auth.openai.com/oauth/token\n headers:\n ChatGPT-Account-Id: acct-a\n"; + +/// 用 YAML 改配置里的一个上游 +fn with_provider(text: &str, name: &str, f: impl FnOnce(&mut serde_yaml_ng::Value)) -> String { + let mut cfg: serde_yaml_ng::Value = serde_yaml_ng::from_str(text).unwrap(); + let p = cfg["providers"] + .as_sequence_mut() + .unwrap() + .iter_mut() + .find(|p| p["name"] == name) + .unwrap(); + f(p); + serde_yaml_ng::to_string(&cfg).unwrap() +} + +/// 新安装加入一个有 ChatGPT 上游（A 上登录着）的同步位置：加入得了，上游在 B 上停用着，A 上 +/// 照旧启用 +#[tokio::test] +async fn a_fresh_device_joins_a_location_with_a_signed_in_upstream() { + let loc = tempfile::tempdir().unwrap(); + let a = Dev::new('a'); + a.edit(|t| add_raw(t, CHATGPT)).await; + a.enable(loc.path(), false, "A", Value::Null).await; + let b = Dev::new('b'); + let preview = b + .ok( + "POST", + "/sync/enable/preview", + json!({ "target": folder(loc.path()) }), + ) + .await; + assert_eq!(preview["initial"], true); + b.enable(loc.path(), false, "B", Value::Null).await; + let p = b.provider("chatgpt").expect("到了 B"); + assert!(p.disabled && p.oauth.is_none()); + let v = a.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["conflicts"], 0, "{v}"); + let p = a.provider("chatgpt").unwrap(); + assert!(!p.disabled, "A 上没被停用"); + assert_eq!(p.oauth.unwrap().refresh, "rt-a"); +} + +/// 已有配置的设备以「合并」加入：导入的规矩在 B 上停用了 ChatGPT 上游，这个停用不随加入 +/// 上传、不回流到 A。之后 B 上登录：照同步的样子启用 +#[tokio::test] +async fn merging_in_keeps_the_upstream_on_where_it_is_signed_in() { + let loc = tempfile::tempdir().unwrap(); + let a = Dev::new('a'); + a.edit(|t| add_raw(t, CHATGPT)).await; + a.enable(loc.path(), false, "A", Value::Null).await; + let b = Dev::new('b'); + b.edit(|t| { + add_raw( + t, + " - name: mine\n base_url: https://mine.example\n key: sk-m\n", + ) + }) + .await; + b.enable( + loc.path(), + false, + "B", + json!({ + "join": "merge", + "decisions": [ { "item": { "kind": "key", "name": "default" }, "action": "skip" } ], + }), + ) + .await; + assert!(b.provider("chatgpt").unwrap().disabled); + let v = a.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["conflicts"], 0, "{v}"); + let p = a.provider("chatgpt").unwrap(); + assert!(!p.disabled, "B 的合并加入没有停掉 A 上的"); + assert!(a.provider("mine").is_some()); + + // A 改并发：B 收到，还停用着 + a.edit(|t| with_provider(t, "chatgpt", |p| p["max_concurrent"] = 4.into())) + .await; + a.ok("POST", "/sync/now", Value::Null).await; + b.ok("POST", "/sync/now", Value::Null).await; + let p = b.provider("chatgpt").unwrap(); + assert_eq!(p.max_concurrent, Some(4)); + assert!(p.disabled); + + // B 上登录（写上自己的登录，停用还在） + b.edit(|t| { + with_provider(t, "chatgpt", |p| { + p["oauth"] = serde_yaml_ng::from_str( + "{ refresh: rt-b, endpoint: 'https://auth.openai.com/oauth/token' }", + ) + .unwrap(); + p["headers"] = serde_yaml_ng::from_str("{ ChatGPT-Account-Id: acct-b }").unwrap(); + }) + }) + .await; + let v = b.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["conflicts"], 0, "{v}"); + let p = b.provider("chatgpt").unwrap(); + assert!(!p.disabled, "登录之后照同步的样子启用"); + assert_eq!(p.oauth.unwrap().refresh, "rt-b"); + let v = a.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["conflicts"], 0, "{v}"); + let p = a.provider("chatgpt").unwrap(); + assert!(!p.disabled); + assert_eq!(p.oauth.unwrap().refresh, "rt-a", "A 的登录还是 A 的"); +} + +/// keyring 不见了：要人处理的状态（`keyring_missing`），同步设置和口令都不动；「重新上传」 +/// 放回这台记着的 keyring 之后照常 +#[tokio::test] +async fn a_missing_keyring_waits_for_a_person() { + let loc = tempfile::tempdir().unwrap(); + let a = Dev::new('a'); + a.enable(loc.path(), true, "A", Value::Null).await; + a.ok("POST", "/sync/now", Value::Null).await; + std::fs::remove_file(loc.path().join("keyring")).unwrap(); + let v = a.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["state"], "keyring_missing", "{v}"); + assert_eq!(v["error"]["code"], "sync.keyring_missing"); + let s = a.config().sync.unwrap(); + assert!(s.encrypt && s.passphrase.as_deref() == Some(PASS)); + let v = a.ok("POST", "/sync/republish", Value::Null).await; + assert_eq!(v["state"], "idle", "{v}"); + assert_eq!(v["encrypted"], true); + assert!(loc.path().join("keyring").exists()); +} + +/// 冲突带着能选的处理：网关密钥三种都能选，「两者都保留」那一份是一把新的密钥 +#[tokio::test] +async fn each_conflict_says_which_choices_it_has() { + let loc = tempfile::tempdir().unwrap(); + let (a, b) = (Dev::new('a'), Dev::new('b')); + a.edit(|t| { + t.replace( + "clients:\n", + "clients:\n - name: k2\n key: tw-k2-0001\n", + ) + }) + .await; + a.enable(loc.path(), false, "A", Value::Null).await; + b.enable(loc.path(), false, "B", Value::Null).await; + a.edit(|t| { + t.replace( + " key: tw-k2-0001\n", + " key: tw-k2-0001\n max_concurrent: 2\n", + ) + }) + .await; + b.edit(|t| { + t.replace( + " key: tw-k2-0001\n", + " key: tw-k2-0001\n max_concurrent: 3\n", + ) + }) + .await; + a.ok("POST", "/sync/now", Value::Null).await; + b.ok("POST", "/sync/now", Value::Null).await; + let list = b.ok("GET", "/sync/conflicts", Value::Null).await; + let c = list + .as_array() + .unwrap() + .iter() + .find(|c| c["item"]["name"] == "k2") + .unwrap_or_else(|| panic!("{list}")); + assert_eq!(c["options"], json!(["local", "remote", "both"])); + b.ok( + "POST", + "/sync/conflicts", + json!({ "decisions": [ { "id": c["id"], "take": "both" } ] }), + ) + .await; + let clients = b.config().clients; + let copy = clients.iter().find(|c| c.name.starts_with("k2 (")).unwrap(); + assert_ne!(copy.key, "tw-k2-0001"); +} + +/// 关闭加密之后两台设备的口令都留着（同步位置上写着「已关闭」的 keyring 靠它核对） +#[tokio::test] +async fn turning_encryption_off_keeps_the_passphrase() { + let loc = tempfile::tempdir().unwrap(); + let (a, b) = (Dev::new('a'), Dev::new('b')); + a.enable(loc.path(), true, "A", Value::Null).await; + b.enable(loc.path(), true, "B", Value::Null).await; + let v = a + .ok( + "POST", + "/sync/encryption", + json!({ "action": "disable", "passphrase": PASS }), + ) + .await; + assert_eq!(v["encrypted"], false, "{v}"); + let s = a.config().sync.unwrap(); + assert!(!s.encrypt && s.passphrase.as_deref() == Some(PASS)); + let v = b.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["encrypted"], false, "{v}"); + let s = b.config().sync.unwrap(); + assert!(!s.encrypt, "B 跟着改成不加密"); + assert_eq!(s.passphrase.as_deref(), Some(PASS), "口令留着"); + // 再开回来：B 用记着的口令跟着改回加密 + a.ok( + "POST", + "/sync/encryption", + json!({ "action": "enable", "passphrase": PASS }), + ) + .await; + let v = b.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["encrypted"], true, "{v}"); + assert!(b.config().sync.unwrap().encrypt); +} diff --git a/crates/tw-sync/Cargo.toml b/crates/tw-sync/Cargo.toml index 62dbc2de..085d9243 100644 --- a/crates/tw-sync/Cargo.toml +++ b/crates/tw-sync/Cargo.toml @@ -41,4 +41,5 @@ tempfile = "3" base64 = { workspace = true } # 两台设备的测试直接读写配置 tw-config = { workspace = true } +serde_yaml_ng = { workspace = true } tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time"] } diff --git a/crates/tw-sync/src/device.rs b/crates/tw-sync/src/device.rs index 86350912..ff31e9d6 100644 --- a/crates/tw-sync/src/device.rs +++ b/crates/tw-sync/src/device.rs @@ -6,6 +6,9 @@ use serde::{Deserialize, Serialize}; use crate::keyring::Keyring; +/// 一份设备记录最大多少字节：几百字节的 JSON，再大的不是我们写的 +pub const MAX_BYTES: usize = 64 * 1024; + /// 设备记录多久没更新就从同步位置上移除 pub const STALE_AFTER_MS: u64 = 180 * 24 * 3600 * 1000; @@ -35,8 +38,15 @@ impl DeviceRecord { } } - /// 读回来。解不开、读不懂的是 `None`（别的设备刚改了加密设置、还没传完、被人改坏了） - pub fn open(bytes: &[u8], keyring: Option<&Keyring>) -> Option { + /// 读回来。解不开、读不懂的是 `None`（别的设备刚改了加密设置、还没传完、被人改坏了）。 + /// `sealed_only`：同步位置加着密 —— **不加密的记录一律不认**，那是没有口令的人放进来的 + pub fn open(bytes: &[u8], keyring: Option<&Keyring>, sealed_only: bool) -> Option { + if bytes.len() > MAX_BYTES { + return None; + } + if sealed_only && !crate::keyring::is_age(bytes) { + return None; + } let json = if crate::keyring::is_age(bytes) { keyring?.decrypt(bytes)? } else { @@ -68,20 +78,28 @@ mod tests { #[test] fn a_record_reads_back_plain_or_sealed() { let r = rec(); - assert_eq!(DeviceRecord::open(&r.seal(None), None), Some(r.clone())); - let k = Keyring::generate(); + assert_eq!( + DeviceRecord::open(&r.seal(None), None, false), + Some(r.clone()) + ); + // 加着密的同步位置：明文的记录不认 + assert_eq!( + DeviceRecord::open(&r.seal(None), Some(&Keyring::generate(0)), true), + None + ); + let k = Keyring::generate(0); let sealed = r.seal(Some(&k)); assert!( !String::from_utf8_lossy(&sealed).contains("张三"), "加密时名字不能是明文" ); - assert_eq!(DeviceRecord::open(&sealed, Some(&k)), Some(r.clone())); - assert_eq!(DeviceRecord::open(&sealed, None), None); + assert_eq!(DeviceRecord::open(&sealed, Some(&k), true), Some(r.clone())); + assert_eq!(DeviceRecord::open(&sealed, None, false), None); assert_eq!( - DeviceRecord::open(&sealed, Some(&Keyring::generate())), + DeviceRecord::open(&sealed, Some(&Keyring::generate(0)), true), None ); - assert_eq!(DeviceRecord::open(b"{not json", None), None); + assert_eq!(DeviceRecord::open(b"{not json", None, false), None); } #[test] diff --git a/crates/tw-sync/src/engine.rs b/crates/tw-sync/src/engine.rs index d2828f86..e92f7870 100644 --- a/crates/tw-sync/src/engine.rs +++ b/crates/tw-sync/src/engine.rs @@ -2,13 +2,20 @@ //! //! # 一轮里做什么 //! -//! 1. 列出 `versions/`，读出每个版本（加密的用版本密钥解开）。读不出来的当作尚未到达：最新 -//! 那一层有没到的就等下一轮，不拿一半去合并。 -//! 2. 找出所有的**头**（没有别的版本依据它的版本）。不止一个时（两台设备同时上传、云盘晚到 +//! 1. 看 keyring 定这一轮加不加密（[`Engine::security`]）。**加密不可降级**：本机设置为加密、 +//! 或者同步位置上有开着的 keyring 时，不加密的版本和设备记录一律不认；keyring 不见了、被换成 +//! 旧的不当作「关掉了加密」，要人处理（[`RoundError::KeyringMissing`]）。只有用口令解得开、 +//! 写着已关闭的 keyring 才算关了加密。 +//! 2. 列出 `versions/`，读出每个版本（加密的用版本密钥解开）。读不出来的分两种：**坏了**的（不是 +//! 配置包、不是这把密钥加密的、加密的位置上没加密）马上跳过；**可能没传完**的当作尚未到达， +//! 最新那一层有没到的就等下一轮，等了 30 分钟还读不出来也跳过。跳过的计数报给界面。 +//! 3. 找出所有的**头**（没有别的版本依据它的版本）。不止一个时（两台设备同时上传、云盘晚到 //! 的版本）先以它们共同的上一版本为基准两两合并，得到「同步位置的最新」。 -//! 3. 以本机基准（上次同步的那一版）为基准，和本机当前配置三方合并（[`crate::merge`]）。 -//! 冲突照常往下走：本机留本机的，上传的留同步位置的，记进冲突记录。 -//! 4. 本机的结果和现在不一样就应用（经控制面的写入流程，来源记为同步）；上传的结果和同步 +//! 4. 以本机基准（上次同步的那一版）为基准，和本机当前配置三方合并（[`crate::merge`]）。基准 +//! **得是同步位置最新的祖先**（版本记着自己的祖先，见 [`Meta::history`]）；不是的话（同步位置 +//! 被别的设备重新上传过、keyring 换过）按两方比较。冲突照常往下走：本机留本机的，上传的留 +//! 同步位置的，记进冲突记录。 +//! 5. 本机的结果和现在不一样就应用（经控制面的写入流程，来源记为同步）；上传的结果和同步 //! 位置上的不一样（或者合并了几个头、冲突记录变了）就上传一个新版本，然后删掉更早的。 //! //! # 冲突记录随版本走 @@ -16,6 +23,14 @@ //! 一处冲突在被处理之前，**两边各留各的**：参与的每台设备用自己的内容，同步版本里是先上传 //! 的那一边（[`Conflict::shared`]）。任一设备处理之后，冲突从记录里去掉、结果上传，别的设备 //! 看到记录没了，就照同步版本改成处理之后的样子。 +//! +//! # 等登录的上游 +//! +//! ChatGPT 账号的登录、OAuth 的令牌属于各自的设备，不随同步走。一台设备上登录着的上游到了 +//! 另一台没登录的设备上，**在那台停用着**（不然过不了校验）。这个停用是那台设备自己的事 +//! （[`State::held_login`]），和插件的停用一样：合并之前换回同步的样子，落地时再停用，上传的 +//! 里面永远是同步的样子 —— 不会因为一台没登录的设备把别的设备上用着的上游停掉。那台登录之后 +//! 照同步的样子启用。 use std::collections::{BTreeMap, BTreeSet}; use std::future::Future; @@ -29,17 +44,24 @@ use tw_pack::{Pack, PackError, PackKind, PackOrigin, Seal, Unseal}; use tw_types::{Msg, msg}; use crate::device::DeviceRecord; -use crate::keyring::{Keyring, KeyringError}; +use crate::keyring::{self, Keyring, KeyringError}; use crate::layout::{self, DEVICES, KEYRING, Listed, VERSIONS, VersionName}; use crate::local::{Local, State, UrlChange}; use crate::merge::{self, Objects}; use crate::order; -use crate::record::{self, Conflict, Meta, Reason, Side}; +use crate::record::{self, Conflict, HISTORY, Meta, Reason, Side}; use crate::store::{Store, StoreError}; /// 设备记录多久没写就写一次（最后同步时间），即便这一轮什么都没变 const DEVICE_RECORD_EVERY_MS: u64 = 6 * 3600 * 1000; +/// 一个可能没传完的版本（或者 keyring）等多久：之后跳过，不再让它挡着 +pub const SKIP_AFTER_MS: u64 = 30 * 60 * 1000; + +/// 同步位置看上去空了：连着这么多轮、过了这么久、自己的设备记录也不在了，才算真的清空了 +const EMPTY_ROUNDS: u32 = 3; +const EMPTY_AFTER_MS: u64 = 10 * 60 * 1000; + /// 本机这一边：配置从哪儿读、往哪儿写。控制面实现它（`tw-control` 的 `sync`），测试里换成 /// 两个目录里的两份配置 pub trait Host: Send + Sync { @@ -111,11 +133,17 @@ pub enum RoundError { Store(StoreError), /// 同步口令不对（别的设备改了口令），或者同步位置开了加密而这台设备还没有口令 Locked(Msg), + /// keyring 不见了、被换成了更早的那一份、一直读不出来：**要人处理**（放回这台设备记着的那 + /// 一份，或者停止同步）。不会因此关掉加密，也不删本机的口令 + KeyringMissing(Msg), /// 同步位置的最新版本由较新的 Core 写，本机读不了：暂停 TooNew { core_version: String }, /// 最新的版本还没传完（云盘还在下载）：下一轮再来，不算失败 NotArrived, - /// 同步位置被清空了（别的设备停止同步并删除了同步数据） + /// 同步位置看上去空了（网络盘没挂上时挂载点是个空目录）：再看几轮才下结论 + LooksEmpty, + /// 同步位置确实空了（别的设备停止同步并删除了同步数据，或者换成了别的位置）：**要人处理** + /// （重新上传本机配置，或者停止同步）。本机的同步设置不动 Emptied, /// 本机的配置读不了、写不进去 Local(Msg), @@ -133,7 +161,13 @@ impl RoundError { pub fn msg(&self) -> Msg { match self { RoundError::Store(e) => e.msg(), - RoundError::Locked(m) | RoundError::Local(m) => m.clone(), + RoundError::Locked(m) | RoundError::Local(m) | RoundError::KeyringMissing(m) => { + m.clone() + } + RoundError::TooNew { core_version } if core_version.is_empty() => msg!( + "sync.too_new_unknown" => + "another device uses a newer version. Update ThinkWatch Lite to keep syncing" + ), RoundError::TooNew { core_version } => msg!( "sync.too_new", version = core_version => "another device uses a newer version (Core {version}). Update ThinkWatch Lite to \ @@ -143,10 +177,15 @@ impl RoundError { "sync.not_arrived" => "the latest version has not finished arriving at the sync location" ), + RoundError::LooksEmpty => msg!( + "sync.looks_empty" => + "the sync location looks empty. It is checked a few more times before anything \ + is done" + ), RoundError::Emptied => msg!( - "sync.emptied" => - "the sync location is empty: another device stopped syncing and deleted the \ - sync data" + "sync.location_empty" => + "the sync location is empty: its sync data was deleted, or it is not the \ + location it was. Upload this device's configuration to it again, or stop syncing" ), RoundError::Stale => msg!( "sync.stale" => "the configuration changed during the sync; it will run again" @@ -155,6 +194,21 @@ impl RoundError { } } +fn locked_new() -> RoundError { + RoundError::Locked(msg!( + "sync.locked_new" => + "the sync location is now encrypted. Enter the sync passphrase to keep syncing" + )) +} + +fn keyring_missing() -> RoundError { + RoundError::KeyringMissing(msg!( + "sync.keyring_missing" => + "the keyring is missing from the sync location, so nothing was synced. Put back the \ + copy this device keeps, or stop syncing" + )) +} + /// 一轮做了什么 #[derive(Debug, Clone, Default)] pub struct Report { @@ -165,10 +219,14 @@ pub struct Report { /// 还没处理的冲突 pub conflicts: Vec, pub devices: Vec, - /// 同步位置此刻加没加密。和设置不一样时（别的设备关了加密）调用方改设置 + /// 同步位置此刻加没加密。和设置不一样时（别的设备开了、或者经口令认证关了加密）调用方 + /// 改设置 pub encrypted: bool, /// 本机应用之后值变了、或者被删掉的网关密钥 pub keys_changed: Vec, + /// 同步位置上跳过了的文件：坏了的、不是这把密钥加密的、加密的位置上没加密的、等了太久 + /// 还没传完的 + pub skipped: u32, } /// 读出来的一个版本 @@ -181,10 +239,40 @@ pub struct Version { pub content: String, } +/// 读过的版本（缓存）。可能没传完的不缓存，下一轮再读 #[derive(Clone)] enum Fetched { Ok(Arc), TooNew(String), + /// 读不出来、也不会自己好 + Broken, +} + +enum Got { + Ok(Arc), + TooNew(String), + Broken, + /// 可能还没传完 + Pending, + /// 列出来之后又没了 + Gone, +} + +/// 同步位置上的版本，读过一遍之后 +#[derive(Default)] +struct Scan { + /// 参与这一轮的：读出来的、版本过新的、还在等的 + names: Vec, + ok: Vec>, + too_new: Vec<(VersionName, String)>, + /// 可能没传完、还在等的 + pending: Vec, + /// 跳过了的 + skipped: Vec, + /// 跳过了的里面这台设备自己写的（keyring 换过之后解不开了）：重新上传之后删掉 + own_broken: Vec, + /// 同步位置上全部的版本名 + all: Vec, } /// 同步位置的最新（几个头合并之后） @@ -213,17 +301,33 @@ pub struct Existing { pub devices: Vec, } +/// 上一轮结束时同步位置上参与的版本、本机的内容、加没加密：都没变就什么都不用做 +#[derive(Debug, Clone, PartialEq, Eq)] +struct Quiet { + listing: Vec, + content: String, + sealed: bool, +} + pub struct Engine { pub store: S, pub host: H, local: Local, settings: Settings, - /// 解开了的版本密钥，和 keyring 文件的哈希（文件变了要重新解） - keyring: Option<(Keyring, [u8; 32])>, + /// 解开了的版本密钥（开着或关着的） + keyring: Option, + /// 这一份 keyring 文件原样的字节：同步成功之后存进本机（[`Local::save_keyring`]） + keyring_raw: Option>, + /// 同步位置开着加密：只认加密的版本和设备记录 + sealed: bool, + /// 上面这几项是照哪一份 keyring 定的（文件的哈希，没有 keyring 是 `Some(None)`）。没定过是 + /// `None` + mode_of: Option>, cache: BTreeMap, - /// 上一轮结束时同步位置上的名字和本机的内容：都没变就什么都不用做 - quiet: Option<(Vec, String)>, + quiet: Option, last_report: Report, + /// 这台设备以前留下的临时文件清过了 + swept: bool, } fn now_ms() -> u64 { @@ -433,6 +537,19 @@ pub struct Decision { pub new_name: Option, } +/// 一处冲突能怎么处理：本机的总能选；另一边的、两者都保留看冲突（[`Conflict::remote_allowed`]、 +/// [`Conflict::both_allowed`]） +pub fn options(c: &Conflict) -> Vec { + let mut out = vec![Take::Local]; + if c.remote_allowed() { + out.push(Take::Remote); + } + if c.both_allowed() { + out.push(Take::Both); + } + out +} + /// 「两者都保留」那一份的名字。自定义安全规则的名字带着分项（`content/项目代号`），只换规则名 fn both_name(id: &ItemId, given: Option<&str>, device_name: &str) -> String { let (prefix, plain) = match (id.kind, id.name.split_once('/')) { @@ -470,6 +587,11 @@ pub fn excluded(names: &BTreeSet, snaps: &[&Snapshot]) -> BTreeSet bool { + matches!(id.kind, ItemKind::Global | ItemKind::ClientPrefs) +} + /// 一个上游的地址（`conn.base_url`） fn base_url(it: &Item) -> Option<&str> { it.units.get(unit::CONN)?.get("base_url")?.as_str() @@ -480,6 +602,145 @@ fn key_value(it: &Item) -> Option<&Value> { it.units.get(unit::ALL)?.get("key") } +// ─────────────────────────────────────────────── 等登录的上游 + +/// 本机配置里的上游（带着这台设备自己的登录） +fn local_providers(p: &Pack) -> BTreeMap { + tw_config::try_parse(&p.config_yaml) + .map(|c| { + c.providers + .into_iter() + .map(|p| (p.name.clone(), p)) + .collect() + }) + .unwrap_or_default() +} + +/// 这个上游落到这台设备上没有登录可用：ChatGPT 账号这台没登录，OAuth 这台没有同一个令牌端点 +/// 的刷新令牌（落地时令牌取本机的，见 `tw_pack::model`） +fn needs_login(id: &ItemId, it: &Item, here: &BTreeMap) -> bool { + if id.kind != ItemKind::Provider { + return false; + } + let Ok(p) = model::provider_of(id, it) else { + return false; + }; + let mine = here.get(&id.name); + if model::is_chatgpt(&p) { + return !mine.is_some_and(|m| model::is_chatgpt(m) && m.oauth.is_some()); + } + match &p.oauth { + Some(o) => !mine + .and_then(|m| m.oauth.as_ref()) + .is_some_and(|m| m.endpoint == o.endpoint && !m.refresh.trim().is_empty()), + None => false, + } +} + +fn disabled(it: &Item) -> bool { + it.units + .get(unit::OTHER) + .and_then(|o| o.get("disabled")) + .and_then(Value::as_bool) + .unwrap_or(false) +} + +/// 停用一个上游 +fn force_disabled(it: &mut Item) { + let other = it + .units + .entry(unit::OTHER.to_string()) + .or_insert_with(|| Value::Object(Default::default())); + if let Value::Object(o) = other { + o.insert("disabled".into(), Value::Bool(true)); + } +} + +/// 一个上游的停用照 `from` 的样子 +fn copy_disabled(it: &mut Item, from: &Item) { + let want = from + .units + .get(unit::OTHER) + .and_then(|o| o.get("disabled")) + .cloned(); + let other = it + .units + .entry(unit::OTHER.to_string()) + .or_insert_with(|| Value::Object(Default::default())); + if let Value::Object(o) = other { + match want { + Some(v) => o.insert("disabled".into(), v), + None => o.remove("disabled"), + }; + } +} + +/// 落地之前：没登录、在这台停用着的上游停用掉，记下哪些是这台自己停用的（同步的样子是启用的） +fn hold_logins( + target: &mut Snapshot, + here: &BTreeMap, +) -> BTreeSet { + let mut held = BTreeSet::new(); + let ids: Vec = target + .of_kind(ItemKind::Provider) + .filter(|(id, it)| needs_login(id, it, here) && !disabled(it)) + .map(|(id, _)| id.clone()) + .collect(); + for id in ids { + if let Some(it) = target.items.get_mut(&id) { + force_disabled(it); + } + held.insert(id.name); + } + held +} + +// ─────────────────────────────────────────────── 版本的谱系 + +/// 版本标识 → 它记着的上一版本和更早的祖先 +type Index = BTreeMap, Vec)>; + +fn index_of<'a>(vs: impl Iterator) -> Index { + vs.map(|v| { + ( + v.name.id(), + (v.meta.parents.clone(), v.meta.history.clone()), + ) + }) + .collect() +} + +/// `start` 自己和它的全部祖先（按谱系能找到的） +fn ancestors(index: &Index, start: &str) -> BTreeSet { + let mut out = BTreeSet::new(); + let mut todo = vec![start.to_string()]; + while let Some(id) = todo.pop() { + if !out.insert(id.clone()) { + continue; + } + if let Some((ps, hs)) = index.get(&id) { + todo.extend(ps.iter().cloned()); + todo.extend(hs.iter().cloned()); + } + } + out +} + +/// 依据 `parents` 的新版本要记的更早的祖先：父版本们的祖先，最近的 [`HISTORY`] 个 +fn history_of(parents: &[Arc]) -> Vec { + let direct: BTreeSet = parents.iter().map(|p| p.name.id()).collect(); + let mut all: BTreeSet = parents + .iter() + .flat_map(|p| p.meta.history.iter().chain(&p.meta.parents)) + .filter(|id| !direct.contains(*id)) + .filter_map(|id| VersionName::parse(id)) + .collect(); + while all.len() > HISTORY { + all.pop_first(); + } + all.into_iter().map(|n| n.id()).collect() +} + impl Engine { /// `data_dir` 是配置文件所在的目录：本机的同步状态放在它底下的 `sync/` pub fn new(store: S, host: H, data_dir: &Path, settings: Settings) -> Self { @@ -489,9 +750,13 @@ impl Engine { local: Local::new(data_dir), settings, keyring: None, + keyring_raw: None, + sealed: false, + mode_of: None, cache: BTreeMap::new(), quiet: None, last_report: Report::default(), + swept: false, } } @@ -501,6 +766,7 @@ impl Engine { pub fn set_settings(&mut self, s: Settings) { if s.passphrase != self.settings.passphrase || s.encrypt != self.settings.encrypt { + self.mode_of = None; self.keyring = None; self.cache.clear(); } @@ -526,32 +792,73 @@ impl Engine { self.quiet = None; } - // ─────────────────────────────────────────────── 读 + // ─────────────────────────────────────────────── 加不加密 + + /// 照 keyring 定下来：开着、关着、没有。`of` 是 keyring 文件的哈希 + fn set_mode( + &mut self, + of: Option<[u8; 32]>, + keyring: Option, + raw: Option>, + ) -> bool { + if self.mode_of != Some(of) { + self.cache.clear(); + self.quiet = None; + } + self.mode_of = Some(of); + self.sealed = keyring.as_ref().is_some_and(|k| !k.off()); + self.keyring = keyring; + self.keyring_raw = raw; + self.sealed + } - /// 拿到版本密钥。没加密是 `Ok(None)` - async fn keyring(&mut self) -> Result, RoundError> { + /// 这一轮加不加密：keyring 在、开着是 `true`（之后只认加密的版本和设备记录）。 + /// + /// - 没有 keyring：本机设置为加密、或者以前见过 keyring 时是 [`RoundError::KeyringMissing`] + /// —— **不当作关掉了加密**； + /// - keyring 解得开：开着就加密；写着已关闭（有口令的人关的）就不加密，代号比见过的小 + /// （有人放回了一份旧的）是 [`RoundError::KeyringMissing`]； + /// - 本来就不加密、**没有口令**的设备认明文的「已关闭」标记（见 [`crate::keyring`]）；有口令 + /// 的设备只认解开之后写着的 + async fn security(&mut self, st: &mut State) -> Result { let Some(bytes) = self.store.get(KEYRING).await? else { - self.keyring = None; - return Ok(None); + st.unreadable.remove(KEYRING); + if self.settings.encrypt || st.keyring_gen > 0 { + return Err(keyring_missing()); + } + return Ok(self.set_mode(None, None, None)); }; let hash: [u8; 32] = Sha256::digest(&bytes).into(); - if self.keyring.as_ref().is_some_and(|(_, h)| *h == hash) { - return Ok(self.keyring.as_ref().map(|(k, _)| k)); + if self.mode_of == Some(Some(hash)) { + return Ok(self.sealed); } let Some(pass) = self.settings.passphrase.clone().filter(|p| !p.is_empty()) else { - return Err(RoundError::Locked(msg!( - "sync.locked_new" => - "the sync location is now encrypted. Enter the sync passphrase to keep syncing" - ))); + // 没有口令、本来就不加密的设备认明文的「已关闭」标记。**有口令的设备不认它**，只认 + // 解开之后写着的：标记谁都加得上 + if keyring::marked_off(&bytes) && !self.settings.encrypt { + return Ok(self.set_mode(Some(hash), None, None)); + } + return Err(locked_new()); }; - let k = tokio::task::spawn_blocking(move || Keyring::open(&bytes, &pass)) - .await - .map_err(local_err)?; - match k { + let opened = { + let b = bytes.clone(); + tokio::task::spawn_blocking(move || Keyring::open(&b, &pass)) + .await + .map_err(local_err)? + }; + match opened { Ok(k) => { - self.cache.clear(); - self.keyring = Some((k, hash)); - Ok(self.keyring.as_ref().map(|(k, _)| k)) + st.unreadable.remove(KEYRING); + if k.off() && k.generation() < st.keyring_gen { + return Err(RoundError::KeyringMissing(msg!( + "sync.keyring_rolled_back" => + "the keyring at the sync location is older than the one this device \ + saw last, so nothing was synced. Put back the copy this device keeps, \ + or stop syncing" + ))); + } + st.keyring_gen = st.keyring_gen.max(k.generation()); + Ok(self.set_mode(Some(hash), Some(k), Some(bytes))) } Err(KeyringError::WrongPassphrase) => Err(RoundError::Locked(msg!( "sync.passphrase_changed" => @@ -560,25 +867,57 @@ impl Engine { Err(KeyringError::TooNew) => Err(RoundError::TooNew { core_version: String::new(), }), - Err(KeyringError::Corrupt) => Err(RoundError::NotArrived), + Err(KeyringError::Corrupt) => { + let since = *st + .unreadable + .entry(KEYRING.to_string()) + .or_insert_with(now_ms); + if now_ms().saturating_sub(since) >= SKIP_AFTER_MS { + Err(RoundError::KeyringMissing(KeyringError::Corrupt.msg())) + } else { + Err(RoundError::NotArrived) + } + } } } - /// 读一个版本。读不出来（没传完、坏了、不是这把密钥加密的）是 `None` - async fn fetch(&mut self, name: &VersionName) -> Result, RoundError> { + /// 解开了的 keyring 写设备记录、版本时用不用：加密开着才用 + fn sealing(&self) -> Option<&Keyring> { + self.keyring.as_ref().filter(|_| self.sealed) + } + + // ─────────────────────────────────────────────── 读 + + /// 读一个版本 + async fn fetch(&mut self, name: &VersionName) -> Result { if let Some(f) = self.cache.get(name) { - return Ok(Some(f.clone())); + return Ok(match f.clone() { + Fetched::Ok(v) => Got::Ok(v), + Fetched::TooNew(c) => Got::TooNew(c), + Fetched::Broken => Got::Broken, + }); } let Some(bytes) = self.store.get(&name.path()).await? else { - return Ok(None); + return Ok(Got::Gone); }; - let identity = self.keyring.as_ref().map(|(k, _)| k.identity().clone()); + // 空的：云盘的占位文件、正在写 + if bytes.is_empty() { + return Ok(Got::Pending); + } + // 认得出是配置包（ZIP）或者加密过的：读不出来多半是没传完 + let shaped = keyring::is_age(&bytes) || bytes.starts_with(b"PK\x03\x04"); + let identity = self.keyring.as_ref().map(|k| k.identity().clone()); + let strict = self.sealed; let decoded = tokio::task::spawn_blocking(move || { let unseal = match &identity { Some(id) => Unseal::Identity(id), None => Unseal::None, }; - let pack = tw_pack::read(&bytes, unseal)?; + let pack = if strict { + tw_pack::read_strict(&bytes, unseal)? + } else { + tw_pack::read(&bytes, unseal)? + }; let snap = model::snapshot(&pack); Ok::<_, PackError>((pack, snap)) }) @@ -586,47 +925,54 @@ impl Engine { .map_err(local_err)?; let fetched = match decoded { Err(PackError::TooNew { core_version, .. }) => Fetched::TooNew(core_version), - Err(_) => return Ok(None), + // 加密的版本，而 keyring 还没到 + Err(PackError::NeedKey) => return Ok(Got::Pending), + Err(PackError::Corrupt { .. }) if shaped => return Ok(Got::Pending), + Err(_) => Fetched::Broken, Ok((pack, Err(e))) if e.is_too_new() => Fetched::TooNew(pack.manifest.core_version), - Ok((_, Err(_))) => return Ok(None), + Ok((_, Err(_))) => Fetched::Broken, Ok((pack, Ok(snap))) => { - let Some(meta) = pack + let meta = pack .manifest .sync .clone() - .and_then(|v| serde_json::from_value::(v).ok()) - else { - return Ok(None); - }; - // 名字和内容对不上（被人挪过、改过名），或者不加密的版本校验和对不上：当作坏了 - if meta.seq != name.seq || layout::device_tag(&meta.device) != name.device { - return Ok(None); - } - if !pack.manifest.encrypted - && meta.sha256.as_deref() != Some(content_sha256(&pack).as_str()) - { - return Ok(None); + .and_then(|v| serde_json::from_value::(v).ok()); + match meta { + // 名字和内容对不上（被人挪过、改过名），或者不加密的版本校验和对不上：坏了 + Some(meta) + if meta.seq == name.seq + && layout::device_tag(&meta.device) == name.device + && (pack.manifest.encrypted + || meta.sha256.as_deref() + == Some(content_sha256(&pack).as_str())) => + { + let content = pack.content_id(); + Fetched::Ok(Arc::new(Version { + name: name.clone(), + pack, + meta, + snap, + content, + })) + } + _ => Fetched::Broken, } - let content = pack.content_id(); - Fetched::Ok(Arc::new(Version { - name: name.clone(), - pack, - meta, - snap, - content, - })) } }; self.cache.insert(name.clone(), fetched.clone()); - Ok(Some(fetched)) + Ok(match fetched { + Fetched::Ok(v) => Got::Ok(v), + Fetched::TooNew(c) => Got::TooNew(c), + Fetched::Broken => Got::Broken, + }) } - fn base(&self) -> Option { - let bytes = self.local.base()?; - let pack = tw_pack::read(&bytes, Unseal::None).ok()?; + /// 这一份基准（不加密的配置包）读成版本 + fn version_of(bytes: &[u8], id: &str) -> Option { + let pack = tw_pack::read(bytes, Unseal::None).ok()?; let meta: Meta = serde_json::from_value(pack.manifest.sync.clone()?).ok()?; let snap = model::snapshot(&pack).ok()?; - let name = VersionName::parse(&self.local.state().base?)?; + let name = VersionName::parse(id)?; let content = pack.content_id(); Some(Version { name, @@ -637,17 +983,12 @@ impl Engine { }) } - /// 同步位置上的版本：读得出来的、读不出来的名字 - async fn versions( - &mut self, - ) -> Result< - ( - Vec, - Vec>, - Vec<(VersionName, String)>, - ), - RoundError, - > { + fn base(&self) -> Option { + Self::version_of(&self.local.base()?, &self.local.state().base?) + } + + /// 同步位置上的版本名 + async fn listing(&self) -> Result, RoundError> { let mut names: Vec = self .store .list(VERSIONS) @@ -656,30 +997,97 @@ impl Engine { .filter_map(|e| VersionName::parse(&e.name)) .collect(); names.sort(); - self.cache.retain(|n, _| names.contains(n)); - let mut ok = Vec::new(); - let mut too_new = Vec::new(); - for n in &names { + names.dedup(); + Ok(names) + } + + /// 读一遍同步位置上的版本，分出读得出来的、过新的、还在等的、跳过的 + async fn scan(&mut self, all: Vec, st: &mut State) -> Result { + self.cache.retain(|n, _| all.contains(n)); + let me = layout::device_tag(&self.settings.device_id); + let now = now_ms(); + let mut sc = Scan::default(); + for n in &all { match self.fetch(n).await? { - Some(Fetched::Ok(v)) => ok.push(v), - Some(Fetched::TooNew(cv)) => too_new.push((n.clone(), cv)), - None => {} + Got::Ok(v) => { + st.unreadable.remove(&n.id()); + sc.names.push(n.clone()); + sc.ok.push(v); + } + Got::TooNew(cv) => { + st.unreadable.remove(&n.id()); + sc.names.push(n.clone()); + sc.too_new.push((n.clone(), cv)); + } + Got::Broken => { + st.unreadable.remove(&n.id()); + sc.skipped.push(n.clone()); + if n.device == me { + sc.own_broken.push(n.clone()); + } + } + Got::Pending => { + let since = *st.unreadable.entry(n.id()).or_insert(now); + if now.saturating_sub(since) >= SKIP_AFTER_MS { + sc.skipped.push(n.clone()); + } else { + sc.names.push(n.clone()); + sc.pending.push(n.clone()); + } + } + Got::Gone => {} } } - Ok((names, ok, too_new)) + st.unreadable + .retain(|k, _| k == KEYRING || all.iter().any(|n| &n.id() == k)); + sc.all = all; + Ok(sc) } - /// 同步位置上的设备记录。半年没更新的顺手删掉 + /// 头：没有别的版本依据它的版本。最新的是较新的 Core 写的是 [`RoundError::TooNew`]，最新那 + /// 一层还没到齐是 [`RoundError::NotArrived`] + fn heads(sc: &Scan) -> Result>, RoundError> { + let referenced: BTreeSet = sc + .ok + .iter() + .flat_map(|v| v.meta.parents.iter().cloned()) + .collect(); + if let Some((_, cv)) = sc + .too_new + .iter() + .find(|(n, _)| !referenced.contains(&n.id())) + { + return Err(RoundError::TooNew { + core_version: cv.clone(), + }); + } + let top_ok = sc.ok.iter().map(|v| v.name.seq).max(); + let top_all = sc.names.iter().map(|n| n.seq).max(); + if top_ok.is_none() || top_ok < top_all { + return Err(RoundError::NotArrived); + } + Ok(sc + .ok + .iter() + .filter(|v| !referenced.contains(&v.name.id())) + .cloned() + .collect()) + } + + /// 同步位置上的设备记录。半年没更新的顺手删掉。加密着时不加密的记录不认 async fn devices(&mut self) -> Result, RoundError> { - let kr = self.keyring.as_ref().map(|(k, _)| k); + let kr = self.keyring.as_ref(); let now = now_ms(); let mut out = Vec::new(); for e in self.store.list(DEVICES).await? { + if !layout::is_device_file(&e.name) { + continue; + } let path = format!("{DEVICES}/{}", e.name); let Some(bytes) = self.store.get(&path).await? else { continue; }; - let Some(rec) = DeviceRecord::open(&bytes, kr) else { + let Some(rec) = DeviceRecord::open(&bytes, kr, self.sealed) else { continue; }; if rec.is_stale(now) && rec.id != self.settings.device_id { @@ -692,7 +1100,7 @@ impl Engine { Ok(out) } - async fn write_device(&mut self, paused: bool) -> Result<(), RoundError> { + async fn write_device(&mut self, st: &mut State, paused: bool) -> Result<(), RoundError> { let rec = DeviceRecord { id: self.settings.device_id.clone(), name: self.settings.device_name.clone(), @@ -701,22 +1109,47 @@ impl Engine { paused_update: paused, remote_core: self.settings.remote_core, }; - let kr = self.keyring.as_ref().map(|(k, _)| k); + let bytes = rec.seal(self.sealing()); self.store - .put( - &layout::device_path(&self.settings.device_id), - &rec.seal(kr), - ) + .put(&layout::device_path(&self.settings.device_id), &bytes) .await?; - let mut st = self.local.state(); st.device_record_ms = rec.last_seen_ms; st.device_record_paused = paused; - self.local.save_state(&st).map_err(RoundError::Local)?; + Ok(()) + } + + /// 存下本机的同步状态，连同这一轮解开的 keyring 的原样（留着放回去用） + fn commit(&self, st: &State) -> Result<(), RoundError> { + self.local.save_state(st).map_err(RoundError::Local)?; + if let Some(raw) = &self.keyring_raw { + self.local.save_keyring(raw).map_err(RoundError::Local)?; + } Ok(()) } // ─────────────────────────────────────────────── 合并几个头 + /// 同步位置上早删了的版本：本机之前的基准里找，本机备份里按内容找 + fn elsewhere(&self, id: &str, content: Option<&String>) -> Option { + if let Some(v) = self + .local + .base_by_id(id) + .and_then(|b| Self::version_of(&b, id)) + { + return Some(v); + } + let content = content?; + let pack = self.host.backup(content)?; + let snap = model::snapshot(&pack).ok()?; + Some(Version { + name: VersionName::parse(id)?, + meta: Meta::default(), + content: content.clone(), + snap, + pack, + }) + } + /// `a`、`b` 共同的上一版本 fn common_ancestor( &self, @@ -725,35 +1158,18 @@ impl Engine { all: &[Arc], base: Option<&Version>, ) -> Option { - // 版本标识 → (上一版本, 它们的内容标识) - let mut parents: BTreeMap, Vec)> = BTreeMap::new(); - for v in all.iter().map(|v| v.as_ref()).chain(base) { - parents.insert( - v.name.id(), - (v.meta.parents.clone(), v.meta.parent_content.clone()), - ); - } + let index = index_of(all.iter().map(|v| v.as_ref()).chain(base)); let mut content_of: BTreeMap = BTreeMap::new(); - for (ps, cs) in parents.values() { - for (p, c) in ps.iter().zip(cs) { + for v in all.iter().map(|v| v.as_ref()).chain(base) { + for (p, c) in v.meta.parents.iter().zip(&v.meta.parent_content) { content_of.insert(p.clone(), c.clone()); } } - let ancestors = |start: &str| -> BTreeSet { - let mut out = BTreeSet::new(); - let mut todo = vec![start.to_string()]; - while let Some(id) = todo.pop() { - if !out.insert(id.clone()) { - continue; - } - if let Some((ps, _)) = parents.get(&id) { - todo.extend(ps.iter().cloned()); - } - } - out - }; - let left: BTreeSet = a.iter().flat_map(|v| ancestors(&v.name.id())).collect(); - let right = ancestors(&b.name.id()); + let left: BTreeSet = a + .iter() + .flat_map(|v| ancestors(&index, &v.name.id())) + .collect(); + let right = ancestors(&index, &b.name.id()); let mut common: Vec = left .intersection(&right) .filter_map(|id| VersionName::parse(id)) @@ -766,16 +1182,7 @@ impl Engine { if let Some(b) = base.filter(|b| b.name.id() == id) { return Some(b.clone()); } - let content = content_of.get(&id)?; - let pack = self.host.backup(content)?; - let snap = model::snapshot(&pack).ok()?; - Some(Version { - name: VersionName::parse(&id)?, - meta: Meta::default(), - content: content.clone(), - snap, - pack, - }) + self.elsewhere(&id, content_of.get(&id)) } /// 几个头合并成一份 @@ -818,6 +1225,7 @@ impl Engine { remote: &ro, hold_items: &hold_items, hold_units: &hold_units, + fieldwise: &fieldwise, }); let mut snap = rebuild(&out.shared, &[&h.snap, &acc.snap]); for (o, k) in ORDERS { @@ -903,6 +1311,16 @@ impl Engine { // ─────────────────────────────────────────────── 写 + /// 新版本的序号：比参与的版本都大，和同步位置上已有的名字不重 + fn next_seq(&self, sc_names: &[VersionName], all: &[VersionName]) -> u64 { + let mut seq = sc_names.iter().map(|n| n.seq).max().unwrap_or(0) + 1; + let tag = layout::device_tag(&self.settings.device_id); + while all.iter().any(|n| n.seq == seq && n.device == tag) { + seq += 1; + } + seq + } + /// 上传一个新版本：内容是 `shared`，依据 `parents` async fn publish( &mut self, @@ -927,16 +1345,17 @@ impl Engine { device_name: self.settings.device_name.clone(), parents: parents.iter().map(|p| p.name.id()).collect(), parent_content: parents.iter().map(|p| p.content.clone()).collect(), + history: history_of(parents), sha256: None, conflicts, }; - let encrypted = self.keyring.is_some(); + let encrypted = self.sealed; if !encrypted { meta.sha256 = Some(content_sha256(&pack)); } pack.manifest.sync = Some(serde_json::to_value(&meta).expect("plain data serializes")); let bytes = { - let recipient = self.keyring.as_ref().map(|(k, _)| k.recipient()); + let recipient = self.sealing().map(|k| k.recipient()); let seal = match &recipient { Some(r) => Seal::Recipient(r), None => Seal::None, @@ -965,33 +1384,52 @@ impl Engine { fn save_base(&self, v: &Version, st: &mut State) -> Result<(), RoundError> { let bytes = tw_pack::write(&v.pack, Seal::None).map_err(|e| RoundError::Local(e.msg()))?; - self.local.save_base(&bytes).map_err(RoundError::Local)?; + self.local + .save_base(&v.name.id(), &bytes) + .map_err(RoundError::Local)?; st.base = Some(v.name.id()); Ok(()) } - async fn prune(&mut self, names: &[VersionName], ok: &[Arc]) { + /// 上传之后删掉更早的版本（[`layout::to_delete`]），连同比留下的都早的、跳过了的文件。删了 + /// 哪些 + async fn prune( + &mut self, + names: &[VersionName], + ok: &[Arc], + skipped: &[VersionName], + ) -> BTreeSet { + let ids = |v: &[String]| -> Vec { + v.iter().filter_map(|p| VersionName::parse(p)).collect() + }; let listed: Vec = names .iter() - .map(|n| Listed { - name: n.clone(), - parents: ok - .iter() - .find(|v| &v.name == n) - .map(|v| { - v.meta - .parents - .iter() - .filter_map(|p| VersionName::parse(p)) - .collect() - }) - .unwrap_or_default(), + .map(|n| { + let meta = ok.iter().find(|v| &v.name == n).map(|v| &v.meta); + Listed { + name: n.clone(), + parents: meta.map(|m| ids(&m.parents)).unwrap_or_default(), + history: meta.map(|m| ids(&m.history)).unwrap_or_default(), + } }) .collect(); - for n in layout::to_delete(&listed) { - let _ = self.store.delete(&n.path()).await; - self.cache.remove(&n); + let del: BTreeSet = layout::to_delete(&listed).into_iter().collect(); + let oldest_kept = names + .iter() + .filter(|n| !del.contains(*n)) + .map(|n| n.seq) + .min(); + let mut gone = BTreeSet::new(); + let below = skipped + .iter() + .filter(|n| oldest_kept.is_some_and(|min| n.seq < min)); + for n in del.iter().chain(below) { + if self.store.delete(&n.path()).await.is_ok() { + self.cache.remove(n); + gone.insert(n.clone()); + } } + gone } // ─────────────────────────────────────────────── 一轮 @@ -1007,92 +1445,147 @@ impl Engine { if !decisions.is_empty() { self.quiet = None; } - match self.round_inner(decisions).await { - Ok(r) => { - self.last_report = r.clone(); - Ok(r) - } - Err(e) => Err(e), - } - } - - async fn round_inner(&mut self, decisions: &[Decision]) -> Result { let mut st = self.local.state(); if st.place != self.settings.place && !st.place.is_empty() { // 换了同步位置：原来的基准是那个位置上的，作废 self.local.clear(); - st = State { - place: self.settings.place.clone(), - ..Default::default() - }; - self.local.save_state(&st).map_err(RoundError::Local)?; + st = State::default(); self.cache.clear(); + self.mode_of = None; } st.place.clone_from(&self.settings.place); - let base = self.base(); - let kr = self.keyring().await?.is_some(); - let (names, ok, too_new) = self.versions().await?; - if names.is_empty() { - if base.is_some() { - return Err(RoundError::Emptied); - } + if !self.swept { + self.store + .sweep(&layout::device_tag(&self.settings.device_id)) + .await; + self.swept = true; + } + let r = self.round_inner(decisions, &mut st).await; + // 成没成都存：读不出来的文件从什么时候起、见过的 keyring 代号、看上去空了几轮 + self.commit(&st)?; + if let Ok(r) = &r { + self.last_report = r.clone(); + } + r + } + + /// 同步位置上没有可用的版本 + async fn on_empty(&mut self, st: &mut State, has_base: bool) -> Result { + if !has_base { // 开启同步时上传第一个版本没成（中途断了）：这一轮补上 - self.publish_first().await?; - return self.finish_quiet(Vec::new(), kr).await; + self.security(st).await?; + let all = self.listing().await?; + let seq = self.next_seq(&[], &all); + self.publish_first(seq, st).await?; + st.empty_rounds = 0; + st.empty_since_ms = None; + return Ok(Report { + encrypted: self.sealed, + devices: self.devices().await?, + ..Default::default() + }); + } + let now = now_ms(); + st.empty_rounds = st.empty_rounds.saturating_add(1); + let since = *st.empty_since_ms.get_or_insert(now); + let mine = self + .store + .get(&layout::device_path(&self.settings.device_id)) + .await?; + if st.empty_rounds >= EMPTY_ROUNDS + && now.saturating_sub(since) >= EMPTY_AFTER_MS + && mine.is_none() + { + return Err(RoundError::Emptied); } + Err(RoundError::LooksEmpty) + } + + async fn round_inner( + &mut self, + decisions: &[Decision], + st: &mut State, + ) -> Result { + let base = self.base(); + let listed = self.listing().await?; + if listed.is_empty() { + return self.on_empty(st, base.is_some()).await; + } + let kr = self.security(st).await?; + let sc = self.scan(listed, st).await?; + if sc.names.is_empty() { + return self.on_empty(st, base.is_some()).await; + } + st.empty_rounds = 0; + st.empty_since_ms = None; + let skipped = sc.skipped.len() as u32; let cur = self.host.current().map_err(RoundError::Local)?; - let listing: Vec = names.iter().map(|n| n.id()).collect(); - let local_content = cur.pack.content_id(); - if self.quiet.as_ref() == Some(&(listing.clone(), local_content.clone())) { + // 本机的内容按原文算（含这台设备自己的登录：登录了要放开等登录的上游） + let quiet = Quiet { + listing: sc.names.iter().map(|n| n.id()).collect(), + content: content_sha256(&cur.pack), + sealed: kr, + }; + if sc.pending.is_empty() && sc.own_broken.is_empty() && self.quiet.as_ref() == Some(&quiet) + { // 什么都没变：冲突还是上一轮的。照样记一次同步成功，设备记录隔一阵写一次，设备 // 列表重读（别的设备的记录会变） st.last_synced_ms = Some(now_ms()); st.failing_since_ms = None; st.error = None; - self.local.save_state(&st).map_err(RoundError::Local)?; if now_ms().saturating_sub(st.device_record_ms) > DEVICE_RECORD_EVERY_MS { - self.write_device(false).await?; + self.write_device(st, false).await?; } - let r = Report { + return Ok(Report { encrypted: kr, conflicts: self.last_report.conflicts.clone(), devices: self.devices().await?, + skipped, ..Default::default() - }; - return Ok(r); - } - // 头：没有别的版本依据它的版本 - let referenced: BTreeSet = ok - .iter() - .flat_map(|v| v.meta.parents.iter().cloned()) - .collect(); - if let Some((_, cv)) = too_new.iter().find(|(n, _)| !referenced.contains(&n.id())) { - if !st.device_record_paused { - self.write_device(true).await?; - } - return Err(RoundError::TooNew { - core_version: cv.clone(), }); } - let top_ok = ok.iter().map(|v| v.name.seq).max(); - let top_all = names.iter().map(|n| n.seq).max(); - if top_ok.is_none() || top_ok < top_all { - return Err(RoundError::NotArrived); - } - let heads: Vec> = ok - .iter() - .filter(|v| !referenced.contains(&v.name.id())) - .cloned() - .collect(); - // 同步位置没有 keyring、版本却是加密的：keyring 还没传到 - if !kr && heads.iter().any(|h| h.pack.manifest.encrypted) { - return Err(RoundError::NotArrived); - } - let remote = self.fold(&heads, &ok, base.as_ref()); + let heads = match Self::heads(&sc) { + Ok(h) => h, + Err(e @ RoundError::TooNew { .. }) => { + if !st.device_record_paused { + self.write_device(st, true).await?; + } + return Err(e); + } + Err(e) => return Err(e), + }; + // 基准得是同步位置最新的祖先：不是的话（同步位置被重新上传过、keyring 换过）按两方比较 + let index = index_of(sc.ok.iter().map(|v| v.as_ref()).chain(base.as_ref())); + let base = base.filter(|b| { + heads + .iter() + .any(|h| ancestors(&index, &h.name.id()).contains(&b.name.id())) + }); + let remote = self.fold(&heads, &sc.ok, base.as_ref()); let local_snap = model::snapshot(&cur.pack).map_err(|e| RoundError::Local(e.msg()))?; + let here = local_providers(&cur.pack); let me = self.settings.device_id.clone(); let mut local = local_snap.clone(); + // 等登录的上游：这台的停用不算本机的改动，换回同步的样子再合并（登录了的这一轮也换回 + // 去，落地时就照同步的样子启用） + let neutral: BTreeSet = local + .of_kind(ItemKind::Provider) + .filter(|(id, it)| needs_login(id, it, &here)) + .map(|(id, _)| id.name.clone()) + .chain(st.held_login.iter().cloned()) + .collect(); + for name in &neutral { + let id = ItemId::new(ItemKind::Provider, name.clone()); + let synced = base + .as_ref() + .and_then(|b| b.snap.items.get(&id)) + .or(remote.snap.items.get(&id)) + .cloned(); + if let (Some(it), Some(s)) = (local.items.get_mut(&id), synced) { + copy_disabled(it, &s); + } + } // 别的设备处理掉了的冲突：照同步版本改 let open_ids: BTreeSet<&str> = remote.conflicts.iter().map(|c| c.id.as_str()).collect(); if let Some(b) = &base { @@ -1134,6 +1627,13 @@ impl Engine { let Some(pos) = open.iter().position(|c| c.id == d.id) else { continue; }; + if d.take == Take::Remote && !open[pos].remote_allowed() { + return Err(RoundError::Local(msg!( + "sync.take_not_allowed" => + "using the other side is not possible for this conflict: the result would \ + not pass validation" + ))); + } let c = open.remove(pos); let local_units = local.items.get(&c.item).map(|it| c.pick(&it.units)); let (_, theirs) = c.mine(&me, local_units.as_ref()); @@ -1176,6 +1676,14 @@ impl Engine { for (u, v) in theirs.units.iter().flatten() { copy.units.insert(u.clone(), v.clone()); } + // 网关密钥的另一份是一把新的密钥：同一个值出现两次，客户端用的是哪一把 + // 就说不清了。它也不是为哪个客户端生成的 + if c.item.kind == ItemKind::Key + && let Some(Value::Object(all)) = copy.units.get_mut(unit::ALL) + { + all.insert("key".into(), Value::String(tw_config::generate_key())); + all.remove("client"); + } local.items.insert(id, copy); local.relink(); } @@ -1214,6 +1722,7 @@ impl Engine { remote: &ro, hold_items: &hold_items, hold_units: &hold_units, + fieldwise: &fieldwise, }); let mut target = rebuild(&out.local, &[&local, &remote.snap]); let mut shared = rebuild(&out.shared, &[&remote.snap, &local]); @@ -1294,10 +1803,15 @@ impl Engine { } target.relink(); shared.relink(); + // 没登录的上游在这台停用着 + st.held_login = hold_logins(&mut target, &here); + let mut start = local.clone(); + hold_logins(&mut start, &here); // 合并的结果要过配置校验：过不了的那几项留本机的，作为冲突列出 self.check_target( &cur.pack, + &start, &local, &mut target, &mut shared, @@ -1363,8 +1877,10 @@ impl Engine { let mut report = Report { encrypted: kr, + skipped, ..Default::default() }; + let mut content_after = quiet.content.clone(); // 应用 if model::normalized(&target) != model::normalized(&local_snap) { report.keys_changed = local_snap @@ -1376,6 +1892,7 @@ impl Engine { .collect(); let pack = model::materialize(&cur.pack, &target).map_err(|e| RoundError::Local(e.msg()))?; + content_after = content_sha256(&pack); let from: Vec = { let mut v: Vec = remote .heads @@ -1396,28 +1913,36 @@ impl Engine { } // 上传 let need_upload = heads.len() > 1 + || !sc.own_broken.is_empty() || model::normalized(&shared) != model::normalized(&remote.snap) || conflicts != remote.conflicts; + let mut listing: BTreeSet = sc.names.iter().cloned().collect(); let next_base = if need_upload { - let seq = names.iter().map(|n| n.seq).max().unwrap_or(0) + 1; + let seq = self.next_seq(&sc.names, &sc.all); let v = self .publish(&shared, seq, &heads, conflicts.clone()) .await?; report.uploaded = Some(v.name.id()); - let mut all: Vec = names.clone(); - all.push(v.name.clone()); - let mut ok_all = ok.clone(); + listing.insert(v.name.clone()); + let all: Vec = listing.iter().cloned().collect(); + let mut ok_all = sc.ok.clone(); ok_all.push(Arc::new(v.clone())); - self.prune(&all, &ok_all).await; + let gone = self.prune(&all, &ok_all, &sc.skipped).await; + // 这台设备自己写的、换了 keyring 之后解不开的：内容已经在新版本里了 + for n in &sc.own_broken { + if !gone.contains(n) && self.store.delete(&n.path()).await.is_ok() { + self.cache.remove(n); + } + } + listing.retain(|n| !gone.contains(n)); v } else { (*heads[0]).clone() }; - self.save_base(&next_base, &mut st)?; + self.save_base(&next_base, st)?; st.last_synced_ms = Some(now_ms()); st.failing_since_ms = None; st.error = None; - self.local.save_state(&st).map_err(RoundError::Local)?; report.conflicts = conflicts; let changed = report.uploaded.is_some() || !report.applied_from.is_empty(); @@ -1425,49 +1950,30 @@ impl Engine { || st.device_record_paused || now_ms().saturating_sub(st.device_record_ms) > DEVICE_RECORD_EVERY_MS { - self.write_device(false).await?; + self.write_device(st, false).await?; } report.devices = self.devices().await?; - // 这一轮之后同步位置上的样子 - let after: Vec = self - .store - .list(VERSIONS) - .await? - .into_iter() - .filter_map(|e| VersionName::parse(&e.name).map(|n| n.id())) - .collect(); - let local_after = self - .host - .current() - .map(|c| c.pack.content_id()) - .unwrap_or_default(); - self.quiet = Some((after, local_after)); + // 下一轮的捷径照这一轮**开始时**看到的算，加上这一轮自己做的：这一轮中途别的设备传上 + // 来的、本机被改的，下一轮都看得出来 + self.quiet = sc.pending.is_empty().then(|| Quiet { + listing: listing.iter().map(|n| n.id()).collect(), + content: content_after, + sealed: kr, + }); Ok(report) } - async fn finish_quiet( - &mut self, - conflicts: Vec, - encrypted: bool, - ) -> Result { - let devices = self.devices().await?; - let r = Report { - conflicts, - devices, - encrypted, - ..Default::default() - }; - Ok(r) - } - - /// 合并出来的本机结果过一遍配置校验。过不了时一项一项试：从本机现在的样子出发，把变了的 - /// 项目一个个换过去，换了就过不了的那一项留本机的，记成冲突（`invalid`），上传的里面也留 - /// 本机的 —— 不然别的设备收到的也是一份过不了校验的配置 + /// 合并出来的本机结果过一遍配置校验。过不了时一项一项试：从本机现在的样子（`start`）出发， + /// 把变了的项目一个个换过去，换了就过不了的那一项留本机的，记成冲突（`invalid`），上传的 + /// 里面也留本机的（`mine`：这台停用着的上游是同步的样子）—— 不然别的设备收到的也是一份 + /// 过不了校验的配置。**先试别的、后试上游**：一个新上游用着另一边新加的出站代理，出站代理 + /// 先换过去，上游才过得了 #[allow(clippy::too_many_arguments)] fn check_target( &self, local_pack: &Pack, - local: &Snapshot, + start: &Snapshot, + mine: &Snapshot, target: &mut Snapshot, shared: &mut Snapshot, remote: &Remote, @@ -1483,16 +1989,33 @@ impl Engine { if valid(target) { return Ok(()); } - let changed: Vec = local + let mut changed: Vec = start .items .keys() .chain(target.items.keys()) - .filter(|id| !id.kind.is_order() && local.items.get(*id) != target.items.get(*id)) + .filter(|id| !id.kind.is_order() && start.items.get(*id) != target.items.get(*id)) .cloned() .collect::>() .into_iter() .collect(); - let mut t = local.clone(); + // 新增和修改按引用的先后（被引用的先到位），删除倒过来（引用它的先走）。一遍下来还有 + // 过不了的，再来一遍：前面换过去的可能正是它缺的 + let rank = |id: &ItemId| -> u8 { + let r = match id.kind { + ItemKind::Provider => 1, + ItemKind::Alias | ItemKind::Group => 2, + ItemKind::Route => 3, + ItemKind::Key | ItemKind::Global => 4, + _ => 0, + }; + if target.items.contains_key(id) { + r + } else { + 10 - r + } + }; + changed.sort_by_key(|id| rank(id)); + let mut t = start.clone(); let fix_orders = |s: &mut Snapshot, from: &Snapshot| { for (o, k) in ORDERS { let names = names_of(s, *k); @@ -1510,23 +2033,35 @@ impl Engine { } s.relink(); }; - for id in changed { - let mut next = t.clone(); - match target.items.get(&id) { - Some(it) => next.items.insert(id.clone(), it.clone()), - None => next.items.remove(&id), - }; - fix_orders(&mut next, target); - if valid(&next) { - t = next; - continue; + let mut left = changed; + loop { + let before = left.len(); + let mut still = Vec::new(); + for id in left { + let mut next = t.clone(); + match target.items.get(&id) { + Some(it) => next.items.insert(id.clone(), it.clone()), + None => next.items.remove(&id), + }; + fix_orders(&mut next, target); + if valid(&next) { + t = next; + } else { + still.push(id); + } } + left = still; + if left.is_empty() || left.len() == before { + break; + } + } + for id in left { // 这一项留本机的 - match local.items.get(&id) { + match mine.items.get(&id) { Some(it) => shared.items.insert(id.clone(), it.clone()), None => shared.items.remove(&id), }; - let units: Vec = local + let units: Vec = mine .items .get(&id) .or(target.items.get(&id)) @@ -1538,7 +2073,7 @@ impl Engine { item: id.clone(), units, shared: side_remote(remote.snap.items.get(&id).map(|i| i.units.clone())), - other: side_me(local.items.get(&id).map(|i| i.units.clone())), + other: side_me(mine.items.get(&id).map(|i| i.units.clone())), reason: Some(Reason::Invalid), }); } @@ -1590,13 +2125,20 @@ impl Engine { } _ => None, }; - let encrypted = keyring.is_some(); + let encrypted = match (&keyring, &opened) { + (None, _) => false, + (Some(_), Some(k)) => !k.off(), + (Some(b), None) => !keyring::marked_off(b), + }; let unlocked = !encrypted || opened.is_some(); let mut devices = Vec::new(); if unlocked { for e in store.list(DEVICES).await.unwrap_or_default() { + if !layout::is_device_file(&e.name) { + continue; + } if let Ok(Some(b)) = store.get(&format!("{DEVICES}/{}", e.name)).await - && let Some(rec) = DeviceRecord::open(&b, opened.as_ref()) + && let Some(rec) = DeviceRecord::open(&b, opened.as_ref(), encrypted) { devices.push(rec); } @@ -1613,9 +2155,24 @@ impl Engine { } } - /// 同步位置空着时开启：加密的先建 keyring（**仅当不存在时**：两台设备同时开启时后写的 - /// 那一台拿到 `Ok(false)`，转去加入），然后以本机配置上传第一个版本 + /// 同步位置空着时开启：加密的先建 keyring（**仅当不存在时**），然后以本机配置上传第一个 + /// 版本。已经有了版本（两台设备同时开启、后写的那一台）是 `Ok(false)`，转去加入。 + /// + /// 只有 keyring、没有版本（之前开启加密时第一个版本没传上去）时，口令解得开就接着用它 pub async fn create(&mut self) -> Result { + self.local.clear(); + let mut st = State { + place: self.settings.place.clone(), + ..Default::default() + }; + let r = self.create_inner(&mut st).await; + if matches!(r, Ok(true)) { + self.commit(&st)?; + } + r + } + + async fn create_inner(&mut self, st: &mut State) -> Result { if self.settings.encrypt { let pass = self .settings @@ -1625,40 +2182,47 @@ impl Engine { .ok_or_else(|| { RoundError::Local(tw_config::sync::SyncError::PassphraseMissing.msg()) })?; - let k = Keyring::generate(); - let sealed = tokio::task::spawn_blocking({ + let (sealed, k) = tokio::task::spawn_blocking({ let pass = pass.clone(); - move || (k.seal(&pass), k) + move || { + let k = Keyring::generate(0); + (k.seal(&pass), k) + } }) .await .map_err(local_err)?; - let (sealed, k) = sealed; - if !self.store.create_exclusive(KEYRING, &sealed).await? { - return Ok(false); + if self.store.create_exclusive(KEYRING, &sealed).await? { + st.keyring_gen = k.generation(); + let hash: [u8; 32] = Sha256::digest(&sealed).into(); + self.set_mode(Some(hash), Some(k), Some(sealed)); + } else { + // 已经有一份：别的设备的（转去加入），或者上次开启到一半留下的 + self.security(st).await?; + if !self.sealed { + return Ok(false); + } } - let hash: [u8; 32] = Sha256::digest(&sealed).into(); - self.keyring = Some((k, hash)); } else if self.store.get(KEYRING).await?.is_some() { return Ok(false); + } else { + self.set_mode(None, None, None); } - if !self.store.list(VERSIONS).await?.is_empty() { + if !self.listing().await?.is_empty() { return Ok(false); } - self.local.clear(); - let st = State { - place: self.settings.place.clone(), - ..Default::default() - }; - self.local.save_state(&st).map_err(RoundError::Local)?; - self.publish_first().await?; + self.publish_first(1, st).await?; self.quiet = None; Ok(true) } - /// 同步位置的最新（几个头合并之后），写成一个配置包：加入时给导入预览比对。不含本机设置 + /// 同步位置的最新（几个头合并之后），写成一个配置包：加入时给导入预览比对。不含本机设置。 + /// 同步位置上还没有版本（只有 keyring）时是一份空的配置 pub async fn latest(&mut self) -> Result { - let remote = self.latest_remote().await?.0; - self.pack_of(&remote.snap) + let mut st = self.local.state(); + match self.latest_remote(&mut st).await? { + Some((remote, _)) => self.pack_of(&remote.snap), + None => self.pack_of(&Snapshot::default()), + } } fn pack_of(&self, snap: &Snapshot) -> Result { @@ -1671,42 +2235,64 @@ impl Engine { model::materialize(&empty, snap).map_err(|e| RoundError::Local(e.msg())) } + /// 同步位置的最新。没有可用的版本是 `None` async fn latest_remote( &mut self, - ) -> Result<(Remote, Vec, Vec>), RoundError> { - self.keyring().await?; - let (names, ok, too_new) = self.versions().await?; - let referenced: BTreeSet = ok - .iter() - .flat_map(|v| v.meta.parents.iter().cloned()) - .collect(); - if let Some((_, cv)) = too_new.iter().find(|(n, _)| !referenced.contains(&n.id())) { - return Err(RoundError::TooNew { - core_version: cv.clone(), - }); - } - let heads: Vec> = ok - .iter() - .filter(|v| !referenced.contains(&v.name.id())) - .cloned() - .collect(); - let top_ok = ok.iter().map(|v| v.name.seq).max(); - if heads.is_empty() || top_ok < names.iter().map(|n| n.seq).max() { - return Err(RoundError::NotArrived); + st: &mut State, + ) -> Result, RoundError> { + self.security(st).await?; + let listed = self.listing().await?; + let sc = self.scan(listed, st).await?; + if sc.names.is_empty() { + return Ok(None); } + let heads = Self::heads(&sc)?; let base = self.base(); - Ok((self.fold(&heads, &ok, base.as_ref()), names, ok)) + let remote = self.fold(&heads, &sc.ok, base.as_ref()); + Ok(Some((remote, sc))) } /// 加入已有的同步。`merged` 是本机按导入的规则合并之后的完整配置包（「合并」）；`None` 是 /// 用同步数据替换本机配置（新安装、或者选了「使用同步数据」）—— 那时本机已接管的客户端 /// 正在用的网关密钥并入同步数据，不同步的上游留在本机。 /// - /// 本机先照结果改，再上传一个依据同步位置最新版本的新版本，作为这台设备的基准 + /// 本机先照结果改，再上传一个依据同步位置最新版本的新版本，作为这台设备的基准。两种方式 + /// 下，别的设备上登录着、这台没登录的上游都在这台停用着，**上传的是同步的样子**（启用着）。 + /// 同步位置上只有 keyring、还没有版本时，以本机配置上传第一个版本 pub async fn join(&mut self, merged: Option) -> Result { - let (remote, names, _) = self.latest_remote().await?; + self.local.clear(); + let mut st = State { + place: self.settings.place.clone(), + ..Default::default() + }; + let r = self.join_inner(merged, &mut st).await; + if r.is_ok() { + self.commit(&st)?; + } + r + } + + async fn join_inner( + &mut self, + merged: Option, + st: &mut State, + ) -> Result { + let Some((remote, sc)) = self.latest_remote(st).await? else { + let all = self.listing().await?; + let seq = self.next_seq(&[], &all); + self.publish_first(seq, st).await?; + let report = Report { + encrypted: self.sealed, + devices: self.devices().await?, + ..Default::default() + }; + self.quiet = None; + self.last_report = report.clone(); + return Ok(report); + }; let cur = self.host.current().map_err(RoundError::Local)?; let local = model::snapshot(&cur.pack).map_err(|e| RoundError::Local(e.msg()))?; + let here = local_providers(&cur.pack); let ex = excluded(&self.settings.exclude, &[&local, &remote.snap]); let mut target = match &merged { Some(p) => model::snapshot(p).map_err(|e| RoundError::Local(e.msg()))?, @@ -1734,6 +2320,17 @@ impl Engine { None => shared.items.remove(id), }; } + // 没登录的上游：导入的规矩在本机停用了它，上传的照同步数据里的样子 + let waiting: Vec = shared + .of_kind(ItemKind::Provider) + .filter(|(id, it)| !ex.contains(*id) && needs_login(id, it, &here)) + .map(|(id, _)| id.clone()) + .collect(); + for id in &waiting { + if let (Some(it), Some(r)) = (shared.items.get_mut(id), remote.snap.items.get(id)) { + copy_disabled(it, r); + } + } for (o, k) in ORDERS { let m = order::merge( order_list(Some(&remote.snap), *o).as_deref(), @@ -1750,9 +2347,15 @@ impl Engine { } target.relink(); shared.relink(); + // 本机这一份：没登录的停用着；同步的样子是启用着的，记成等登录 + hold_logins(&mut target, &here); + st.held_login = waiting + .iter() + .filter(|id| shared.items.get(*id).is_some_and(|it| !disabled(it))) + .map(|id| id.name.clone()) + .collect(); // 不加密：经同步新来、或者脚本变了的插件在本机停用着，要在本机启用（同步一轮里是同一条） - let mut held = BTreeMap::new(); - if self.keyring.is_none() { + if !self.sealed { let ids: Vec = target .of_kind(ItemKind::Plugin) .map(|(id, _)| id.clone()) @@ -1774,12 +2377,14 @@ impl Engine { t.units .insert(unit::ENABLED.to_string(), Value::Bool(false)); } - held.insert(id.name.clone(), script.unwrap_or(Value::Null)); + st.held_plugins + .insert(id.name.clone(), script.unwrap_or(Value::Null)); } } } let mut report = Report { - encrypted: self.keyring.is_some(), + encrypted: self.sealed, + skipped: sc.skipped.len() as u32, ..Default::default() }; if model::normalized(&target) != model::normalized(&local) { @@ -1809,22 +2414,16 @@ impl Engine { } report.applied_from = vec![remote.device_name.clone()]; } - let seq = names.iter().map(|n| n.seq).max().unwrap_or(0) + 1; + let seq = self.next_seq(&sc.names, &sc.all); let v = self .publish(&shared, seq, &remote.heads, remote.conflicts.clone()) .await?; report.uploaded = Some(v.name.id()); - let mut st = State { - place: self.settings.place.clone(), - held_plugins: held, - ..Default::default() - }; - self.save_base(&v, &mut st)?; + self.save_base(&v, st)?; st.last_synced_ms = Some(now_ms()); st.failing_since_ms = None; st.error = None; - self.local.save_state(&st).map_err(RoundError::Local)?; - self.write_device(false).await?; + self.write_device(st, false).await?; report.devices = self.devices().await?; report.conflicts = remote.conflicts; self.quiet = None; @@ -1834,93 +2433,176 @@ impl Engine { // ─────────────────────────────────────────────── 加密 - /// 之后开启加密：建 keyring，以它重新上传最新版本，删掉未加密的版本。其他设备下次同步时 - /// 要输入口令 + /// 之后开启加密：建 keyring（同步位置上有一份已关闭的就换掉它），以它重新上传最新版本， + /// 删掉未加密的版本和设备记录。其他设备下次同步时要输入口令 pub async fn enable_encryption(&mut self, passphrase: &str) -> Result<(), RoundError> { - let (remote, names, _) = self.latest_remote().await?; - if self.keyring.is_some() { - return Ok(()); + let mut st = self.local.state(); + let r = self.enable_inner(passphrase, &mut st).await; + self.commit(&st)?; + r + } + + async fn enable_inner(&mut self, passphrase: &str, st: &mut State) -> Result<(), RoundError> { + let existing = self.store.get(KEYRING).await?; + let mut after = st.keyring_gen; + if let Some(b) = &existing { + // 给的口令、记着的口令，哪个解得开（第一个是给的那个） + let tries: Vec = [ + Some(passphrase.to_string()), + self.settings.passphrase.clone(), + ] + .into_iter() + .flatten() + .collect(); + let b2 = b.clone(); + let opened = tokio::task::spawn_blocking(move || { + tries + .iter() + .enumerate() + .find_map(|(i, p)| Keyring::open(&b2, p).ok().map(|k| (i == 0, k))) + }) + .await + .map_err(local_err)?; + match opened { + // 已经开着：同一个口令就当作开好了 + Some((true, k)) if !k.off() => { + self.settings.encrypt = true; + self.settings.passphrase = Some(passphrase.to_string()); + self.mode_of = None; + return Ok(()); + } + Some((_, k)) if !k.off() => return Err(locked_new()), + Some((_, k)) => after = after.max(k.generation()), + None if keyring::marked_off(b) => {} + None => return Err(locked_new()), + } } - let k = Keyring::generate(); + let remote = self.latest_remote(st).await?; + let all = self.listing().await?; + let k = Keyring::generate(after); let sealed = k.seal(passphrase); - if !self.store.create_exclusive(KEYRING, &sealed).await? { - return Err(RoundError::Locked(msg!( - "sync.locked_new" => - "the sync location is now encrypted. Enter the sync passphrase to keep syncing" - ))); + if existing.is_some() { + self.store.put(KEYRING, &sealed).await?; + } else if !self.store.create_exclusive(KEYRING, &sealed).await? { + return Err(locked_new()); } + st.keyring_gen = k.generation(); let hash: [u8; 32] = Sha256::digest(&sealed).into(); - self.keyring = Some((k, hash)); self.settings.encrypt = true; self.settings.passphrase = Some(passphrase.to_string()); - self.reupload(remote, &names).await + self.set_mode(Some(hash), Some(k), Some(sealed)); + self.reupload(remote, &all, st).await?; + // 不加密的设备记录：加密的位置上不认它们，各设备下次同步时加密着重写 + for e in self.store.list(DEVICES).await? { + if !layout::is_device_file(&e.name) { + continue; + } + let path = format!("{DEVICES}/{}", e.name); + if let Some(b) = self.store.get(&path).await? + && !keyring::is_age(&b) + { + let _ = self.store.delete(&path).await; + } + } + Ok(()) } - /// 关闭加密：口令要对。重新上传不加密的最新版本，删掉 keyring 和加密过的版本 + /// 关闭加密：口令要对。keyring 改写成「已关闭」（**不删**：别的设备靠它认出这是有口令的人 + /// 关的），重新上传不加密的最新版本，删掉加密过的版本 pub async fn disable_encryption(&mut self, passphrase: &str) -> Result<(), RoundError> { + let mut st = self.local.state(); + let r = self.disable_inner(passphrase, &mut st).await; + self.commit(&st)?; + r + } + + async fn disable_inner(&mut self, passphrase: &str, st: &mut State) -> Result<(), RoundError> { let Some(bytes) = self.store.get(KEYRING).await? else { - self.keyring = None; - return Ok(()); + return Err(keyring_missing()); }; - let hash: [u8; 32] = Sha256::digest(&bytes).into(); let pass = passphrase.to_string(); let opened = tokio::task::spawn_blocking(move || Keyring::open(&bytes, &pass)) .await .map_err(local_err)? .map_err(|e| RoundError::Locked(e.msg()))?; + if opened.off() { + self.settings.encrypt = false; + self.mode_of = None; + return Ok(()); + } self.settings.passphrase = Some(passphrase.to_string()); - self.keyring = Some((opened, hash)); - let (remote, names, _) = self.latest_remote().await?; - // 之后写的一律不加密 - self.keyring = None; + self.settings.encrypt = true; + self.mode_of = None; + let remote = self.latest_remote(st).await?; + let all = self.listing().await?; + let off = opened.next(true); + let sealed = off.seal(passphrase); + self.store.put(KEYRING, &sealed).await?; + st.keyring_gen = st.keyring_gen.max(off.generation()); + let hash: [u8; 32] = Sha256::digest(&sealed).into(); self.settings.encrypt = false; - self.reupload(remote, &names).await?; - self.store.delete(KEYRING).await?; - Ok(()) + self.set_mode(Some(hash), Some(off), Some(sealed)); + self.reupload(remote, &all, st).await } /// 以现在的加密设置重新上传最新版本，删掉别的版本（它们是另一种加密设置下写的） - async fn reupload(&mut self, remote: Remote, names: &[VersionName]) -> Result<(), RoundError> { - let seq = names.iter().map(|n| n.seq).max().unwrap_or(0) + 1; - let v = self - .publish(&remote.snap, seq, &remote.heads, remote.conflicts.clone()) - .await?; + async fn reupload( + &mut self, + remote: Option<(Remote, Scan)>, + names: &[VersionName], + st: &mut State, + ) -> Result<(), RoundError> { + let v = match remote { + Some((remote, sc)) => { + let seq = self.next_seq(&sc.names, names); + self.publish(&remote.snap, seq, &remote.heads, remote.conflicts.clone()) + .await? + } + None => { + let seq = self.next_seq(&[], names); + return self.publish_first(seq, st).await.map(|_| ()); + } + }; for n in names { let _ = self.store.delete(&n.path()).await; self.cache.remove(n); } - let mut st = self.local.state(); - self.save_base(&v, &mut st)?; + self.save_base(&v, st)?; st.last_synced_ms = Some(now_ms()); - self.local.save_state(&st).map_err(RoundError::Local)?; - self.write_device(false).await?; + self.write_device(st, false).await?; self.quiet = None; Ok(()) } /// 更改同步口令：只重新加密 keyring，已有的版本不变。其他设备下次同步时要输入新口令 pub async fn change_passphrase(&mut self, old: &str, new: &str) -> Result<(), RoundError> { + let mut st = self.local.state(); let Some(bytes) = self.store.get(KEYRING).await? else { return Err(RoundError::Local(msg!( "sync.not_encrypted" => "the sync location is not encrypted" ))); }; let (old, new2) = (old.to_string(), new.to_string()); - let sealed = tokio::task::spawn_blocking(move || { - Keyring::open(&bytes, &old).map(|k| (k.seal(&new2), k)) + let (sealed, k) = tokio::task::spawn_blocking(move || { + Keyring::open(&bytes, &old).map(|k| { + let off = k.off(); + let k = k.next(off); + (k.seal(&new2), k) + }) }) .await .map_err(local_err)? .map_err(|e| RoundError::Locked(e.msg()))?; - let (sealed, k) = sealed; self.store.put(KEYRING, &sealed).await?; + st.keyring_gen = st.keyring_gen.max(k.generation()); let hash: [u8; 32] = Sha256::digest(&sealed).into(); - self.keyring = Some((k, hash)); self.settings.passphrase = Some(new.to_string()); - Ok(()) + self.set_mode(Some(hash), Some(k), Some(sealed)); + self.commit(&st) } - /// 口令对不对得上同步位置的 keyring（别的设备改了口令之后重新输入）。没加密是 `Ok(false)` + /// 口令对不对得上同步位置的 keyring（别的设备改了口令、开了加密之后重新输入）。加密开着是 + /// `Ok(true)`；没有 keyring、或者是已关闭的是 `Ok(false)` pub async fn unlock(&mut self, passphrase: &str) -> Result { let Some(bytes) = self.store.get(KEYRING).await? else { return Ok(false); @@ -1930,21 +2612,109 @@ impl Engine { .await .map_err(local_err)?; match k { - Ok(_) => { + Ok(k) => { self.settings.passphrase = Some(passphrase.to_string()); - self.settings.encrypt = true; + self.settings.encrypt = !k.off(); + self.mode_of = None; self.keyring = None; self.quiet = None; - Ok(true) + Ok(!k.off()) } Err(e) => Err(RoundError::Locked(e.msg())), } } + // ─────────────────────────────────────────────── 要人处理的状态 + + /// 同步位置清空了（[`RoundError::Emptied`]）、keyring 不见了或者被换成旧的 + /// （[`RoundError::KeyringMissing`]）之后，由人选的「重新上传本机配置」： + /// + /// - 加密着时 keyring 放回去：这台设备记着的那一份（口令解得开的话），没有就新建一把 —— + /// 之前的版本这时谁也解不开了，各设备跳过它们； + /// - 同步位置上没有读得出来的版本时，以本机配置上传一个新版本； + /// - 然后照常同步一轮 + pub async fn republish(&mut self) -> Result { + let mut st = self.local.state(); + let r = self.republish_inner(&mut st).await; + self.commit(&st)?; + r?; + self.quiet = None; + self.round().await + } + + async fn republish_inner(&mut self, st: &mut State) -> Result<(), RoundError> { + let pass = self.settings.passphrase.clone().filter(|p| !p.is_empty()); + let open = |b: Vec| { + let pass = pass.clone(); + async move { + let Some(p) = pass else { + return Err(KeyringError::WrongPassphrase); + }; + tokio::task::spawn_blocking(move || Keyring::open(&b, &p)) + .await + .unwrap_or(Err(KeyringError::Corrupt)) + } + }; + let seen = st.keyring_gen; + let usable = |k: &Keyring| !(k.off() && k.generation() < seen); + let here = match self.store.get(KEYRING).await? { + Some(b) => Some(open(b).await), + None => None, + }; + let fine = matches!(&here, Some(Ok(k)) if usable(k)); + if !fine && (self.settings.encrypt || st.keyring_gen > 0) { + match &here { + // 口令不对的那一份不是丢了：别的设备改了口令，要输入新口令 + Some(Err(KeyringError::WrongPassphrase)) => { + return Err(RoundError::Locked(KeyringError::WrongPassphrase.msg())); + } + Some(Err(KeyringError::TooNew)) => { + return Err(RoundError::TooNew { + core_version: String::new(), + }); + } + _ => {} + } + let copy = match self.local.keyring() { + Some(b) => match open(b.clone()).await { + Ok(k) if usable(&k) => Some(b), + _ => None, + }, + None => None, + }; + match (copy, &pass) { + (Some(b), _) => self.store.put(KEYRING, &b).await?, + (None, Some(p)) if self.settings.encrypt => { + let p = p.clone(); + let k = Keyring::generate(seen); + st.keyring_gen = k.generation(); + let sealed = tokio::task::spawn_blocking(move || k.seal(&p)) + .await + .map_err(local_err)?; + self.store.put(KEYRING, &sealed).await?; + } + // 不加密、也没有记着的 keyring：当作从没加密过 + _ => st.keyring_gen = 0, + } + } + self.mode_of = None; + self.security(st).await?; + st.empty_rounds = 0; + st.empty_since_ms = None; + let all = self.listing().await?; + let sc = self.scan(all.clone(), st).await?; + if sc.ok.is_empty() { + let seq = self.next_seq(&[], &all); + self.publish_first(seq, st).await?; + } + Ok(()) + } + // ─────────────────────────────────────────────── 停止 /// 停止同步：从同步位置移除本设备的设备记录；`purge` 时连全部版本、设备记录和 keyring 一起 - /// 删掉（其他设备随后发现同步位置空了，停止同步）。本机的同步状态目录删掉 + /// 删掉（其他设备随后发现同步位置空了，由人决定怎么办）。**只删认得出的同步文件**：用户 + /// 自己放进 `versions/`、`devices/` 的不动。本机的同步状态目录删掉 pub async fn stop(&mut self, purge: bool) -> Result<(), RoundError> { let mine = self .store @@ -1952,10 +2722,14 @@ impl Engine { .await; if purge { for e in self.store.list(VERSIONS).await? { - self.store.delete(&format!("{VERSIONS}/{}", e.name)).await?; + if VersionName::parse(&e.name).is_some() { + self.store.delete(&format!("{VERSIONS}/{}", e.name)).await?; + } } for e in self.store.list(DEVICES).await? { - self.store.delete(&format!("{DEVICES}/{}", e.name)).await?; + if layout::is_device_file(&e.name) { + self.store.delete(&format!("{DEVICES}/{}", e.name)).await?; + } } self.store.delete(KEYRING).await?; } else { @@ -1964,6 +2738,7 @@ impl Engine { self.local.clear(); self.cache.clear(); self.quiet = None; + self.mode_of = None; Ok(()) } @@ -1971,9 +2746,10 @@ impl Engine { /// 同步位置上读得出来的版本（最新与上一个）：给备份列表 pub async fn remote_versions(&mut self) -> Result>, RoundError> { - self.keyring().await?; - let (_, ok, _) = self.versions().await?; - Ok(ok) + let mut st = self.local.state(); + self.security(&mut st).await?; + let listed = self.listing().await?; + Ok(self.scan(listed, &mut st).await?.ok) } /// 同步位置上内容标识是这个的版本 @@ -1986,8 +2762,8 @@ impl Engine { .map(|v| v.pack.clone())) } - /// 同步位置空着：以本机配置上传第一个版本（加密时先建 keyring） - async fn publish_first(&mut self) -> Result { + /// 以本机配置上传一个不依据别的版本的版本（同步位置空着时） + async fn publish_first(&mut self, seq: u64, st: &mut State) -> Result { let cur = self.host.current().map_err(RoundError::Local)?; let snap = model::snapshot(&cur.pack).map_err(|e| RoundError::Local(e.msg()))?; let mut shared = snap.clone(); @@ -2003,13 +2779,13 @@ impl Engine { set_order(&mut shared, *o, list); } shared.relink(); - let v = self.publish(&shared, 1, &[], Vec::new()).await?; - let mut st = self.local.state(); + let v = self.publish(&shared, seq, &[], Vec::new()).await?; st.place.clone_from(&self.settings.place); - self.save_base(&v, &mut st)?; + self.save_base(&v, st)?; st.last_synced_ms = Some(now_ms()); - self.local.save_state(&st).map_err(RoundError::Local)?; - self.write_device(false).await?; + st.failing_since_ms = None; + st.error = None; + self.write_device(st, false).await?; Ok(v) } } diff --git a/crates/tw-sync/src/keyring.rs b/crates/tw-sync/src/keyring.rs index b17f5d78..aa0eb0b5 100644 --- a/crates/tw-sync/src/keyring.rs +++ b/crates/tw-sync/src/keyring.rs @@ -7,6 +7,16 @@ //! - 没有口令的一方解不开 keyring，也就既读不了版本、也写不出别的设备认的版本（age 的认证 //! 加密，见 `tw_pack`）。 //! +//! **关闭加密也写在 keyring 里**（[`Keyring::off`]）：keyring 以口令加密、带认证，没有口令的人 +//! 造不出一个「已关闭」的 keyring。于是同步位置上的 keyring 被删掉、被换成别的，设备都不会把 +//! 这当成「关掉了加密」：只有用口令解得开、写着已关闭的 keyring 才算。 +//! +//! 每写一次 keyring 带一个更大的**代号**（[`Keyring::generation`]，取写的时刻，至少比上一个 +//! 大 1）。设备记着见过的最大代号：把一个旧的「已关闭」keyring 放回来（它当初是真的）也不算。 +//! +//! 已关闭的 keyring 文件开头另有一行明文的标记（[`OFF_MARK`]），给**没有口令**的设备看：它们 +//! 本来就不加密，看到标记就照常同步；有口令的设备不看标记，只认解开之后写着的。 +//! //! scrypt 的强度是**写死的**（[`WORK_FACTOR`]），不按这台机器的速度现定：一台快的 Mac 按自己 //! 定的强度加密，慢的服务器可能解不开（age 拒绝超过本机上限的强度）。 @@ -25,6 +35,9 @@ const MAX_WORK_FACTOR: u8 = WORK_FACTOR + 1; static WORK: AtomicU8 = AtomicU8::new(WORK_FACTOR); +/// 已关闭的 keyring 文件的第一行（明文，见模块说明） +pub const OFF_MARK: &[u8] = b"thinkwatch-sync: encryption off\n"; + /// **只给测试用**：把 scrypt 的强度调低，一条测试不必每次都算零点几秒。解的时候照样认 pub fn set_work_factor_for_tests(log_n: u8) { WORK.store(log_n.clamp(1, WORK_FACTOR), Ordering::Relaxed); @@ -37,17 +50,27 @@ struct Plain { v: u32, /// `AGE-SECRET-KEY-1…` identity: String, + /// 加密已关闭：版本和设备记录不再加密。keyring 留着，好让各设备认得出这是有口令的人关的 + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + off: bool, + /// 代号：每写一次就更大 + #[serde(default)] + generation: u64, } /// 一把解开了的版本密钥。**Debug 不打印它** pub struct Keyring { identity: age::x25519::Identity, + off: bool, + generation: u64, } impl std::fmt::Debug for Keyring { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Keyring") .field("recipient", &self.identity.to_public().to_string()) + .field("off", &self.off) + .field("generation", &self.generation) .finish() } } @@ -85,10 +108,31 @@ impl KeyringError { } impl Keyring { - /// 新的一把 - pub fn generate() -> Self { + /// 新的一把。`after` 是见过的最大代号：新的这把比它大 + pub fn generate(after: u64) -> Self { Self { identity: age::x25519::Identity::generate(), + off: false, + generation: next_generation(after), + } + } + + /// 加密已经关了（有口令的人关的） + pub fn off(&self) -> bool { + self.off + } + + pub fn generation(&self) -> u64 { + self.generation + } + + /// 同一把密钥再写一次（关加密、开回来、换口令）：开着还是关着，代号更大 + pub fn next(self, off: bool) -> Self { + let generation = next_generation(self.generation); + Self { + off, + generation, + ..self } } @@ -97,15 +141,28 @@ impl Keyring { let plain = Plain { v: 1, identity: self.identity.to_string().expose_secret().to_string(), + off: self.off, + generation: self.generation, }; let json = serde_json::to_vec(&plain).expect("plain data serializes"); let mut r = age::scrypt::Recipient::new(SecretString::from(passphrase.to_string())); r.set_work_factor(WORK.load(Ordering::Relaxed)); - age::encrypt(&r, &json).expect("encrypting to a passphrase does not fail") + let sealed = age::encrypt(&r, &json).expect("encrypting to a passphrase does not fail"); + if self.off { + [OFF_MARK, &sealed].concat() + } else { + sealed + } } - /// 用口令解开 `keyring` 文件 + /// 用口令解开 `keyring` 文件。开头的标记和里面写的对不上（有人加上或去掉了标记）算坏了 pub fn open(bytes: &[u8], passphrase: &str) -> Result { + let marked = marked_off(bytes); + let bytes = if marked { + &bytes[OFF_MARK.len()..] + } else { + bytes + }; let mut id = age::scrypt::Identity::new(SecretString::from(passphrase.to_string())); id.set_max_work_factor(MAX_WORK_FACTOR); let json = age::decrypt(&id, bytes).map_err(|e| match e { @@ -119,11 +176,18 @@ impl Keyring { if plain.v > 1 { return Err(KeyringError::TooNew); } + if plain.off != marked { + return Err(KeyringError::Corrupt); + } let identity = plain .identity .parse::() .map_err(|_| KeyringError::Corrupt)?; - Ok(Self { identity }) + Ok(Self { + identity, + off: plain.off, + generation: plain.generation, + }) } /// 加密用的公钥：版本、设备记录都加密给它 @@ -157,6 +221,20 @@ pub fn is_age(bytes: &[u8]) -> bool { bytes.starts_with(b"age-encryption.org/") } +/// `keyring` 文件开头有「已关闭」的标记。**没有认证**：只给没有口令的设备用（见模块说明） +pub fn marked_off(bytes: &[u8]) -> bool { + bytes.starts_with(OFF_MARK) +} + +/// 下一个代号：此刻的毫秒数，至少比 `after` 大 1（时钟慢的设备照样往上走） +fn next_generation(after: u64) -> u64 { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0); + now.max(after.saturating_add(1)) +} + #[cfg(test)] mod tests { use super::*; @@ -164,7 +242,7 @@ mod tests { #[test] fn a_keyring_opens_with_its_passphrase_and_only_with_it() { set_work_factor_for_tests(4); - let k = Keyring::generate(); + let k = Keyring::generate(0); let sealed = k.seal("correct horse battery staple"); assert!(is_age(&sealed)); let back = Keyring::open(&sealed, "correct horse battery staple").unwrap(); @@ -182,7 +260,7 @@ mod tests { #[test] fn changing_the_passphrase_keeps_the_same_key() { set_work_factor_for_tests(4); - let k = Keyring::generate(); + let k = Keyring::generate(0); let sealed = k.seal("old passphrase 1"); let opened = Keyring::open(&sealed, "old passphrase 1").unwrap(); let resealed = opened.seal("new passphrase 2"); @@ -190,7 +268,41 @@ mod tests { assert!(again.same_as(&k), "改口令只重新加密 keyring，密钥本身不变"); let data = k.encrypt(b"device record"); assert_eq!(again.decrypt(&data).unwrap(), b"device record"); - assert!(Keyring::generate().decrypt(&data).is_none()); + assert!(Keyring::generate(0).decrypt(&data).is_none()); + } + + #[test] + fn turning_encryption_off_is_written_in_the_keyring_itself() { + set_work_factor_for_tests(4); + let k = Keyring::generate(0); + let first = k.generation(); + assert!(!k.off()); + let sealed = k.next(true).seal("pass phrase 12"); + assert!(marked_off(&sealed), "没有口令的设备看得到标记"); + let back = Keyring::open(&sealed, "pass phrase 12").unwrap(); + assert!(back.off(), "关了就是关了"); + assert!(back.generation() > first, "每写一次代号更大"); + // 没有口令的人改不了它：口令不对就解不开，也就看不到「关了」 + assert!(Keyring::open(&sealed, "someone else!!").is_err()); + // 标记被去掉、或者加到一个开着的 keyring 上：和里面写的对不上，算坏了 + assert!(matches!( + Keyring::open(&sealed[OFF_MARK.len()..], "pass phrase 12"), + Err(KeyringError::Corrupt) + )); + let on_sealed = back.next(false).seal("pass phrase 12"); + assert!(!marked_off(&on_sealed)); + assert!(matches!( + Keyring::open(&[OFF_MARK, &on_sealed].concat(), "pass phrase 12"), + Err(KeyringError::Corrupt) + )); + let on = Keyring::open(&on_sealed, "pass phrase 12").unwrap(); + assert!(!on.off()); + } + + #[test] + fn a_new_keyring_always_comes_after_the_ones_seen() { + let far = u64::MAX / 2; + assert!(Keyring::generate(far).generation() > far, "时钟慢也往上走"); } #[test] @@ -198,7 +310,7 @@ mod tests { set_work_factor_for_tests(4); // 按强度 4 加密，再把文件头里写的强度改成 30：age 在算 scrypt 之前就按上限拒绝， // 测试不用真的去算一遍 2^30 - let sealed = Keyring::generate().seal("p"); + let sealed = Keyring::generate(0).seal("p"); let text = String::from_utf8_lossy(&sealed).to_string(); let line = text .lines() @@ -217,7 +329,7 @@ mod tests { #[test] fn debug_shows_only_the_public_half() { - let k = Keyring::generate(); + let k = Keyring::generate(0); let shown = format!("{k:?}"); assert!(shown.contains("age1"), "{shown}"); assert!(!shown.contains("AGE-SECRET-KEY"), "{shown}"); diff --git a/crates/tw-sync/src/layout.rs b/crates/tw-sync/src/layout.rs index 6f01ac26..e4a7a2d7 100644 --- a/crates/tw-sync/src/layout.rs +++ b/crates/tw-sync/src/layout.rs @@ -94,11 +94,21 @@ pub fn device_path(device_id: &str) -> String { format!("{DEVICES}/{}", device_tag(device_id)) } +/// `devices/` 里的这个名字是不是一份设备记录（设备标识的前 8 位）。别的是用户自己放的 +pub fn is_device_file(name: &str) -> bool { + name.len() == tw_config::sync::DEVICE_ID_PREFIX + && name + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + /// 同步位置上的一个版本：名字，以及它说自己依据的上一版本（读出来之后才知道） #[derive(Debug, Clone, PartialEq, Eq)] pub struct Listed { pub name: VersionName, pub parents: Vec, + /// 更早的祖先（版本自己记着的） + pub history: Vec, } /// 上传新版本之后要删掉哪些。 @@ -123,6 +133,27 @@ pub fn to_delete(listed: &[Listed]) -> Vec { for l in &heads { keep.extend(l.parents.iter().cloned()); } + // 不止一个头：它们共同的祖先也留着，连同从它到每个头之间的版本 —— 下一台设备合并它们要 + // 用它当基准，删了就只能两方比较 + if heads.len() > 1 { + let lineage = |l: &Listed| -> BTreeSet { + l.history + .iter() + .chain(&l.parents) + .cloned() + .chain([l.name.clone()]) + .collect() + }; + let sets: Vec> = heads.iter().map(|l| lineage(l)).collect(); + let common = sets.iter().skip(1).fold(sets[0].clone(), |acc, s| { + acc.intersection(s).cloned().collect() + }); + if let Some(ca) = common.iter().max() { + for s in &sets { + keep.extend(s.iter().filter(|n| n.seq >= ca.seq).cloned()); + } + } + } // 不知道上一版本是谁（读不出来）时，至少留住序号紧挨着的那一层 if listed .iter() @@ -158,6 +189,7 @@ mod tests { Listed { name: v(seq, dev), parents: parents.iter().map(|(s, d)| v(*s, d)).collect(), + history: Vec::new(), } } @@ -231,6 +263,23 @@ mod tests { assert!(to_delete(&listed).is_empty(), "{:?}", to_delete(&listed)); } + /// 两个头的共同祖先比它们的上一版本还早：它和中间的版本都留着 + #[test] + fn the_common_ancestor_of_two_heads_is_kept() { + let mut late = l(3, "bbbbbbbb", &[(2, "aaaaaaaa")]); + late.history = vec![v(1, "aaaaaaaa")]; + let mut top = l(4, "aaaaaaaa", &[(3, "aaaaaaaa")]); + top.history = vec![v(1, "aaaaaaaa"), v(2, "aaaaaaaa")]; + let listed = [ + l(1, "aaaaaaaa", &[]), + l(2, "aaaaaaaa", &[(1, "aaaaaaaa")]), + l(3, "aaaaaaaa", &[(2, "aaaaaaaa")]), + top, + late, + ]; + assert_eq!(to_delete(&listed), [v(1, "aaaaaaaa")]); + } + #[test] fn a_version_that_has_not_arrived_still_keeps_the_layer_below() { // 3 读不出来，不知道它依据谁：2 也还是头，2 和它依据的 1 都留着 diff --git a/crates/tw-sync/src/local.rs b/crates/tw-sync/src/local.rs index 4d8eac11..904d5c09 100644 --- a/crates/tw-sync/src/local.rs +++ b/crates/tw-sync/src/local.rs @@ -3,12 +3,17 @@ //! ```text //! sync/ //! base.twconfig 上次同步的那一版（合并基准），不加密，不计入本机保留的 10 个版本 -//! state.json 上次同步的时间、失败从什么时候开始、因同步停用的插件、地址经同步变更的上游 +//! bases/ 之前的几个基准（最近 8 个，按版本标识）：同步位置上早删了的共同上一版本 +//! 从这里找 +//! keyring 同步位置上的 keyring 最近一次解得开的那一份（以口令加密，原样）：同步位置 +//! 上的丢了、被换成旧的时，由人决定放回去 +//! state.json 上次同步的时间、失败从什么时候开始、因同步停用的插件和上游、地址经同步 +//! 变更的上游、读不出来的文件 //! ``` //! //! 同步位置上的版本被删掉（设备离线期间）不影响合并：基准在这里。停止同步时整个目录删掉。 -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::io::Write; use std::path::{Path, PathBuf}; @@ -18,7 +23,11 @@ use tw_types::{Msg, msg}; pub const DIR: &str = "sync"; const BASE: &str = "base.twconfig"; +const BASES: &str = "bases"; +const KEYRING: &str = "keyring"; const STATE: &str = "state.json"; +/// `bases/` 里留几个 +const KEEP_BASES: usize = 8; /// `state.json` #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] @@ -42,6 +51,18 @@ pub struct State { pub device_record_ms: u64, /// 上次写设备记录时是不是暂停着（版本过新） pub device_record_paused: bool, + /// 等这台设备登录的上游（名字）：别的设备上登录着、这台没有，于是在这台停用着。停用是 + /// 这台自己的事，不上传；登录之后照同步的样子启用 + #[serde(skip_serializing_if = "BTreeSet::is_empty")] + pub held_login: BTreeSet, + /// 见过的 keyring 的最大代号（[`crate::keyring::Keyring::generation`]） + pub keyring_gen: u64, + /// 同步位置上读不出来的文件：名字 → 第一次读不出来的时刻。读出来了就去掉 + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + pub unreadable: BTreeMap, + /// 同步位置看上去是空的：从什么时候起、连着几轮 + pub empty_since_ms: Option, + pub empty_rounds: u32, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -142,8 +163,45 @@ impl Local { std::fs::read(self.dir.join(BASE)).ok() } - pub fn save_base(&self, bytes: &[u8]) -> Result<(), Msg> { - write_private(&self.dir.join(BASE), bytes) + /// 换基准。`id` 是它的版本标识：同时留一份在 `bases/`，只留最近的几个 + pub fn save_base(&self, id: &str, bytes: &[u8]) -> Result<(), Msg> { + write_private(&self.dir.join(BASE), bytes)?; + let dir = self.dir.join(BASES); + write_private(&dir.join(format!("{id}.twconfig")), bytes)?; + let mut all: Vec<(std::time::SystemTime, PathBuf)> = std::fs::read_dir(&dir) + .map(|rd| { + rd.filter_map(|e| e.ok()) + .filter(|e| e.file_name().to_string_lossy().ends_with(".twconfig")) + .filter_map(|e| Some((e.metadata().ok()?.modified().ok()?, e.path()))) + .collect() + }) + .unwrap_or_default(); + all.sort(); + let extra = all.len().saturating_sub(KEEP_BASES); + for (_, p) in all.into_iter().take(extra) { + let _ = std::fs::remove_file(p); + } + Ok(()) + } + + /// 之前的基准里版本标识是 `id` 的那一个 + pub fn base_by_id(&self, id: &str) -> Option> { + if id.contains(['/', '\\']) || id.starts_with('.') { + return None; + } + std::fs::read(self.dir.join(BASES).join(format!("{id}.twconfig"))).ok() + } + + /// 记着的 keyring（同步位置上最近一次解得开的那一份） + pub fn keyring(&self) -> Option> { + std::fs::read(self.dir.join(KEYRING)).ok() + } + + pub fn save_keyring(&self, bytes: &[u8]) -> Result<(), Msg> { + if self.keyring().as_deref() == Some(bytes) { + return Ok(()); + } + write_private(&self.dir.join(KEYRING), bytes) } /// 停止同步：整个目录删掉 @@ -169,10 +227,14 @@ mod tests { ..Default::default() }; l.save_state(&s).unwrap(); - l.save_base(b"PK").unwrap(); + l.save_base("0000000001-aaaaaaaa", b"PK").unwrap(); let again = Local::new(d.path()); assert_eq!(again.state(), s); assert_eq!(again.base().as_deref(), Some(&b"PK"[..])); + assert_eq!( + again.base_by_id("0000000001-aaaaaaaa").as_deref(), + Some(&b"PK"[..]) + ); #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; @@ -187,6 +249,29 @@ mod tests { assert!(!d.path().join("sync").exists()); } + #[test] + fn only_the_latest_few_bases_are_kept() { + let d = tempfile::tempdir().unwrap(); + let l = Local::new(d.path()); + for i in 1..=12u64 { + l.save_base(&format!("{i:010}-aaaaaaaa"), format!("v{i}").as_bytes()) + .unwrap(); + // 修改时间要分得出先后 + std::thread::sleep(std::time::Duration::from_millis(5)); + } + assert!(l.base_by_id("0000000001-aaaaaaaa").is_none()); + assert!(l.base_by_id("0000000004-aaaaaaaa").is_none()); + assert_eq!( + l.base_by_id("0000000005-aaaaaaaa").as_deref(), + Some(&b"v5"[..]) + ); + assert_eq!( + l.base_by_id("0000000012-aaaaaaaa").as_deref(), + Some(&b"v12"[..]) + ); + assert!(l.base_by_id("../state.json").is_none()); + } + #[test] fn the_place_ignores_credentials() { use tw_config::sync::Target; diff --git a/crates/tw-sync/src/merge.rs b/crates/tw-sync/src/merge.rs index 0c215a59..f35e34d4 100644 --- a/crates/tw-sync/src/merge.rs +++ b/crates/tw-sync/src/merge.rs @@ -63,6 +63,10 @@ pub struct Input<'a, K> { pub hold_items: &'a BTreeSet, /// 这几个单位各留各的：还没处理的冲突 pub hold_units: &'a BTreeSet<(K, String)>, + /// 逐字段合并、全是默认值时整个对象不在的那几种（全局设置的一节、Lite 的通用偏好）： + /// **不在就是每个字段都是默认值**，不是被删了。一边把一节改回默认、另一边改了这一节的 + /// 另一个字段，各管各的字段，不是「删了对改了」 + pub fieldwise: &'a dyn Fn(&K) -> bool, } /// 合并的结果 @@ -93,13 +97,22 @@ pub fn merge(i: Input<'_, K>) -> Outcome { .chain(i.local.keys()) .chain(i.remote.keys()) .collect(); + let none = Units::new(); for k in keys { - let (b, l, r) = (base.get(k), i.local.get(k), i.remote.get(k)); + let (mut b, mut l, mut r) = (base.get(k), i.local.get(k), i.remote.get(k)); + let fieldwise = (i.fieldwise)(k); + if fieldwise { + // 没有基准时也照样：不在就是全是默认值 + b = b.or(Some(&none)); + l = l.or(Some(&none)); + r = r.or(Some(&none)); + } let mut put = |local: Option<&Units>, shared: Option<&Units>| { - if let Some(v) = local { + // 逐字段的那几种合并出来一个字段都没有：全是默认值，不写 + if let Some(v) = local.filter(|v| !(fieldwise && v.is_empty())) { out.local.insert(k.clone(), v.clone()); } - if let Some(v) = shared { + if let Some(v) = shared.filter(|v| !(fieldwise && v.is_empty())) { out.shared.insert(k.clone(), v.clone()); } }; @@ -249,6 +262,7 @@ mod tests { remote: &r, hold_items: &BTreeSet::new(), hold_units: &BTreeSet::new(), + fieldwise: &|_| false, }) } @@ -533,6 +547,7 @@ mod tests { remote: &remote, hold_items: &hold_items, hold_units: &hold_units, + fieldwise: &|_| false, }); assert!(o.conflicts.is_empty(), "{:?}", o.conflicts); assert_eq!(o.local["ollama"], obj(&[("conn", json!("local"))])); @@ -553,11 +568,49 @@ mod tests { remote: &Objects::new(), hold_items: &hold_items, hold_units: &BTreeSet::new(), + fieldwise: &|_| false, }); assert!(o.shared.is_empty()); assert_eq!(o.local.len(), 1); } + /// 全局设置一节：一边改回了默认（整节不在了），另一边改了这一节的另一个字段 —— 各管各的 + /// 字段，不是删了对改了 + #[test] + fn a_field_level_object_reset_to_defaults_is_not_a_delete() { + let base = objs(&[("failover", Some(obj(&[("pause_secs", json!(100))])))]); + let local = objs(&[( + "failover", + Some(obj(&[ + ("pause_secs", json!(100)), + ("failures_to_pause", json!(7)), + ])), + )]); + let remote = Objects::new(); + let o = merge(Input { + base: Some(&base), + local: &local, + remote: &remote, + hold_items: &BTreeSet::new(), + hold_units: &BTreeSet::new(), + fieldwise: &|k: &String| k == "failover", + }); + assert!(o.conflicts.is_empty(), "{:?}", o.conflicts); + let want = obj(&[("failures_to_pause", json!(7))]); + assert_eq!(o.local["failover"], want); + assert_eq!(o.shared["failover"], want); + // 两边都改回默认：整节不在 + let o = merge(Input { + base: Some(&base), + local: &Objects::new(), + remote: &Objects::new(), + hold_items: &BTreeSet::new(), + hold_units: &BTreeSet::new(), + fieldwise: &|_: &String| true, + }); + assert!(o.local.is_empty() && o.shared.is_empty() && o.conflicts.is_empty()); + } + #[test] fn whether_there_is_anything_to_upload() { let base = objs(&[("a", Some(obj(&[("conn", json!(1)), ("models", json!(1))])))]); diff --git a/crates/tw-sync/src/record.rs b/crates/tw-sync/src/record.rs index dc5d3c78..6b8ea3e4 100644 --- a/crates/tw-sync/src/record.rs +++ b/crates/tw-sync/src/record.rs @@ -25,6 +25,10 @@ pub struct Meta { /// 位置上已经删了时，按它去本机备份里找基准 #[serde(default)] pub parent_content: Vec, + /// 更早的祖先（版本标识，老的在前，最多 [`HISTORY`] 个）：基准在同步位置上早删了时，靠它 + /// 判断基准是不是这个版本的祖先 + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub history: Vec, /// 内容的 SHA-256（只在不加密时写）：发现传输中损坏。加密时 age 的认证加密已经管了 #[serde(default, skip_serializing_if = "Option::is_none")] pub sha256: Option, @@ -33,6 +37,21 @@ pub struct Meta { pub conflicts: Vec, } +/// 一个版本记多少个祖先 +pub const HISTORY: usize = 200; + +impl Meta { + /// 这个版本自己和它记得的全部祖先 + pub fn lineage(&self, own_id: &str) -> BTreeSet { + self.history + .iter() + .chain(&self.parents) + .cloned() + .chain([own_id.to_string()]) + .collect() + } +} + /// 一处冲突：同一个对象（的几个单位）在两台设备上被改成了不同的内容。 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct Conflict { @@ -82,6 +101,11 @@ impl Conflict { !self.whole_item() && self.item.kind.is_named() && self.reason.is_none() } + /// 「使用另一边」能不能选。合并结果过不了校验的（`invalid`）不能：选了还是过不了 + pub fn remote_allowed(&self) -> bool { + self.reason != Some(Reason::Invalid) + } + /// 这台设备看这处冲突时，哪一边是「本机」：自己写的那一边；不是参与的设备时，是和本机 /// 现在内容一样的那一边，都不一样就是同步版本里的那一边 pub fn mine<'a>( @@ -215,6 +239,7 @@ mod tests { device_name: "Mac mini".into(), parents: vec!["0000000006-aaaaaaaa".into()], parent_content: vec!["abc".into()], + history: vec!["0000000005-aaaaaaaa".into()], sha256: None, conflicts: vec![c(Some(json!({"base_url": "x"})), None, ItemKind::Key)], }; @@ -222,5 +247,13 @@ mod tests { assert_eq!(v["conflicts"][0]["item"]["kind"], "key"); let back: Meta = serde_json::from_value(v).unwrap(); assert_eq!(back, m); + assert_eq!( + m.lineage("0000000007-aaaaaaaa"), + BTreeSet::from([ + "0000000005-aaaaaaaa".to_string(), + "0000000006-aaaaaaaa".to_string(), + "0000000007-aaaaaaaa".to_string() + ]) + ); } } diff --git a/crates/tw-sync/src/store/folder.rs b/crates/tw-sync/src/store/folder.rs index a8fca449..a6ae4891 100644 --- a/crates/tw-sync/src/store/folder.rs +++ b/crates/tw-sync/src/store/folder.rs @@ -5,14 +5,34 @@ //! 版本。临时名以点开头，列目录时跳过；云盘客户端多数也不传点开头的临时文件。 //! //! 云盘里还没下载到本机的文件（iCloud 的 `.名字.icloud` 占位文件、Dropbox 的在线文件）列目录 -//! 时要么不出现、要么读不出来 —— 调用方把读不出来的版本当成「尚未到达」，到了再合并。 +//! 时照真名列出、读出来是空的（iCloud）或者读不出来 —— 调用方把读不出来的版本当成「尚未到达」， +//! 到了再合并。 use std::io::Write; use std::path::{Path, PathBuf}; use tw_types::msg; -use super::{Entry, Store, StoreError, check_path, split, temp_name}; +use super::{ + Entry, MAX_FILE, Store, StoreError, TEMP_PREFIX, check_path, own_temp, split, temp_name, + too_large, +}; + +/// iCloud Drive 还没下载到本机的文件，在目录里是一个 `.名字.icloud` 的占位文件 +const ICLOUD: &str = ".icloud"; + +/// 读一个文件，**边读边数**：超过上限就不读了 +fn read_limited(p: &Path) -> std::io::Result, ()>> { + use std::io::Read; + let f = std::fs::File::open(p)?; + let mut out = Vec::new(); + f.take(MAX_FILE as u64 + 1).read_to_end(&mut out)?; + Ok(if out.len() > MAX_FILE { + Err(()) + } else { + Ok(out) + }) +} pub struct Folder { root: PathBuf, @@ -144,7 +164,19 @@ impl Store for Folder { let mut out = Vec::new(); for e in rd.flatten() { let name = e.file_name().to_string_lossy().to_string(); - // 临时文件、云盘的占位文件、系统的 .DS_Store + // iCloud 还没下载的文件：照它的真名列出来，读的时候读不到 —— 当作尚未到达 + if let Some(real) = name + .strip_prefix('.') + .and_then(|n| n.strip_suffix(ICLOUD)) + .filter(|n| !n.starts_with('.') && !d.join(n).exists()) + { + out.push(Entry { + name: real.to_string(), + size: 0, + }); + continue; + } + // 临时文件、系统的 .DS_Store if name.starts_with('.') { continue; } @@ -166,13 +198,22 @@ impl Store for Folder { async fn get(&self, path: &str) -> Result>, StoreError> { let root = self.root.clone(); let p = self.at(path)?; - blocking(move || match std::fs::read(&p) { - Ok(b) => Ok(Some(b)), + let shown = path.to_string(); + blocking(move || match read_limited(&p) { + Ok(Ok(b)) => Ok(Some(b)), + Ok(Err(())) => Err(too_large(&shown, MAX_FILE)), Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - if root.is_dir() { - Ok(None) + if !root.is_dir() { + return Err(io(&root, e)); + } + // iCloud 还没下载的：有，只是还没到 —— 读成空的（调用方当作尚未到达） + let placeholder = p + .file_name() + .map(|n| p.with_file_name(format!(".{}{ICLOUD}", n.to_string_lossy()))); + if placeholder.is_some_and(|ph| ph.is_file()) { + Ok(Some(Vec::new())) } else { - Err(io(&root, e)) + Ok(None) } } Err(e) => Err(io(&p, e)), @@ -252,6 +293,41 @@ impl Store for Folder { }) .await } + + /// 这台设备写到一半断了留下的临时文件：一小时以前的删掉（还在写的那一个不动） + async fn sweep(&self, tag: &str) { + let root = self.root.clone(); + let tag = tag.to_string(); + let _ = blocking(move || { + let hour = std::time::Duration::from_secs(3600); + for d in [ + root.clone(), + root.join(crate::layout::VERSIONS), + root.join(crate::layout::DEVICES), + ] { + let Ok(rd) = std::fs::read_dir(&d) else { + continue; + }; + for e in rd.flatten() { + let name = e.file_name().to_string_lossy().to_string(); + if !name.starts_with(TEMP_PREFIX) || !own_temp(&name, &tag) { + continue; + } + let old = e + .metadata() + .and_then(|m| m.modified()) + .ok() + .and_then(|t| t.elapsed().ok()) + .is_some_and(|age| age > hour); + if old { + let _ = std::fs::remove_file(e.path()); + } + } + } + Ok(()) + }) + .await; + } } /// 盯着同步文件夹：新版本、新的 keyring 一到就发一个信号（见 [`tw_watch::watch`]）。盯的是 @@ -336,15 +412,27 @@ mod tests { b"placeholder", ) .unwrap(); + std::fs::write(d.path().join("versions/.DS_Store"), b"x").unwrap(); std::fs::create_dir(d.path().join("versions/sub")).unwrap(); let got = f.list("versions").await.unwrap(); assert_eq!( got, - [Entry { - name: "0000000002-bbbbbbbb.twconfig".into(), - size: 1 - }] + [ + Entry { + name: "0000000002-bbbbbbbb.twconfig".into(), + size: 1 + }, + Entry { + name: "0000000003-cccccccc.twconfig".into(), + size: 0 + } + ] ); + std::fs::remove_file( + d.path() + .join("versions/.0000000003-cccccccc.twconfig.icloud"), + ) + .unwrap(); f.delete("versions/0000000002-bbbbbbbb.twconfig") .await .unwrap(); @@ -354,6 +442,82 @@ mod tests { assert!(f.list("versions").await.unwrap().is_empty()); } + /// iCloud 还没下载的版本：列得出名字，读不到 —— 当作尚未到达，不是没有 + #[tokio::test] + async fn an_icloud_placeholder_is_a_version_that_has_not_arrived() { + let (d, f) = bed(); + std::fs::create_dir(d.path().join("versions")).unwrap(); + std::fs::write( + d.path() + .join("versions/.0000000003-cccccccc.twconfig.icloud"), + b"placeholder", + ) + .unwrap(); + let got = f.list("versions").await.unwrap(); + assert_eq!(got.len(), 1); + assert_eq!(got[0].name, "0000000003-cccccccc.twconfig"); + assert_eq!( + f.get("versions/0000000003-cccccccc.twconfig") + .await + .unwrap(), + Some(Vec::new()), + "读成空的：有，还没到" + ); + assert_eq!( + f.get("versions/0000000004-cccccccc.twconfig") + .await + .unwrap(), + None + ); + } + + #[tokio::test] + async fn an_oversized_file_is_not_read() { + let (d, f) = bed(); + std::fs::create_dir(d.path().join("versions")).unwrap(); + let big = vec![0u8; MAX_FILE + 1]; + std::fs::write(d.path().join("versions/0000000001-aaaaaaaa.twconfig"), big).unwrap(); + assert!( + f.get("versions/0000000001-aaaaaaaa.twconfig") + .await + .is_err() + ); + } + + /// 写到一半断了留下的临时文件：自己的、一小时以前的删掉，别人的和刚写的不动 + #[tokio::test] + async fn leftover_temporary_files_of_this_device_are_swept() { + let (d, f) = bed(); + std::fs::create_dir(d.path().join("versions")).unwrap(); + let mine = d + .path() + .join("versions") + .join(temp_name("0000000002-aaaaaaaa.twconfig")); + let theirs = d + .path() + .join("versions") + .join(temp_name("0000000002-bbbbbbbb.twconfig")); + let fresh = d + .path() + .join("versions") + .join(temp_name("0000000003-aaaaaaaa.twconfig")); + for p in [&mine, &theirs, &fresh] { + std::fs::write(p, b"x").unwrap(); + } + let old = std::time::SystemTime::now() - std::time::Duration::from_secs(7200); + for p in [&mine, &theirs] { + std::fs::File::options() + .write(true) + .open(p) + .unwrap() + .set_modified(old) + .unwrap(); + } + f.sweep("aaaaaaaa").await; + assert!(!mine.exists()); + assert!(theirs.exists() && fresh.exists()); + } + #[tokio::test] async fn a_missing_root_is_unreachable_and_never_created() { let d = tempfile::tempdir().unwrap(); diff --git a/crates/tw-sync/src/store/mod.rs b/crates/tw-sync/src/store/mod.rs index cd56a9c7..4e03c4a1 100644 --- a/crates/tw-sync/src/store/mod.rs +++ b/crates/tw-sync/src/store/mod.rs @@ -88,6 +88,46 @@ pub trait Store: Send + Sync { path: &str, bytes: &[u8], ) -> impl Future> + Send; + /// 清掉这台设备以前留下的临时文件（写到一半断了的）。`tag` 是设备标识的前 8 位。没有临时 + /// 文件可言的存储什么都不做 + fn sweep(&self, tag: &str) -> impl Future + Send { + let _ = tag; + async {} + } +} + +/// 从同步位置读一个文件最多多少字节：配置包的上限，加上加密和余量。**边读边数**，超过就 +/// 不读了 —— 对同步位置有写权限的人能放一个很大的文件进来 +pub const MAX_FILE: usize = tw_pack::MAX_BYTES + 1024 * 1024; + +/// 一个目录列表（PROPFIND、ListObjectsV2）的答复最多多少字节 +pub const MAX_LISTING: usize = 8 * 1024 * 1024; + +/// 太大了 +pub(crate) fn too_large(what: &str, max: usize) -> StoreError { + StoreError::Other(msg!( + "sync.store.too_large", what = what, limit = max / (1024 * 1024) => + "{what} at the sync location is larger than {limit} MiB, so it was not read" + )) +} + +/// 读一个 HTTP 答复的正文，**边读边数**：超过 `max` 就停 +pub(crate) async fn read_limited( + mut r: reqwest::Response, + max: usize, + what: &str, +) -> Result, StoreError> { + if r.content_length().is_some_and(|n| n as usize > max) { + return Err(too_large(what, max)); + } + let mut out = Vec::new(); + while let Some(chunk) = r.chunk().await.map_err(|e| http_error(&e))? { + if out.len() + chunk.len() > max { + return Err(too_large(what, max)); + } + out.extend_from_slice(&chunk); + } + Ok(out) } /// 按配置选出来的那一种。 @@ -170,12 +210,31 @@ impl Store for AnyStore { async fn create_exclusive(&self, path: &str, bytes: &[u8]) -> Result { each!(self, s => s.create_exclusive(path, bytes).await) } + async fn sweep(&self, tag: &str) { + each!(self, s => s.sweep(tag).await) + } } -/// 临时文件名：点开头（列目录时跳过），带随机串（两个进程不撞） +/// 临时文件名：点开头（列目录时跳过），带随机串（两个进程不撞），以最终的名字结尾（版本和 +/// 设备记录的名字里有设备标识，清理时认得出哪些是自己的） pub(crate) fn temp_name(final_name: &str) -> String { let r: u64 = rand::random(); - format!(".tmp-{r:016x}-{final_name}") + format!("{TEMP_PREFIX}{r:016x}-{final_name}") +} + +pub(crate) const TEMP_PREFIX: &str = ".tmp-"; + +/// 一个临时文件是不是这台设备留下的：最终的名字是它的版本、它的设备记录，或者 keyring +pub(crate) fn own_temp(name: &str, tag: &str) -> bool { + let Some(rest) = name.strip_prefix(TEMP_PREFIX) else { + return false; + }; + let Some((_, final_name)) = rest.split_once('-') else { + return false; + }; + final_name == tag + || final_name == crate::layout::KEYRING + || crate::layout::VersionName::parse(final_name).is_some_and(|v| v.device == tag) } /// 一个路径拆成目录和文件名。只有一层目录 @@ -229,6 +288,21 @@ pub(crate) fn http_status(status: reqwest::StatusCode, what: &str) -> StoreError } /// 请求没发出去或者没收到答复 +/// 发一个请求。**连接池里那条空闲连接刚被服务端关掉**（Apache 默认空闲 5 秒就关，发出去 +/// 就收到「连接在答复之前关了」）时换一条连接再发一次：请求根本没到服务端那里。超时、连不上 +/// 不重发 +pub(crate) async fn send(r: reqwest::RequestBuilder) -> Result { + let again = r.try_clone(); + match r.send().await { + Ok(resp) => Ok(resp), + Err(e) if e.is_request() && !e.is_timeout() && !e.is_connect() => match again { + Some(r) => r.send().await.map_err(|e| http_error(&e)), + None => Err(http_error(&e)), + }, + Err(e) => Err(http_error(&e)), + } +} + pub(crate) fn http_error(e: &reqwest::Error) -> StoreError { let mut detail = e.to_string(); let mut cur = std::error::Error::source(e); @@ -269,6 +343,16 @@ mod tests { } } + #[test] + fn a_temporary_file_is_recognised_as_this_devices() { + let t = temp_name("0000000003-aaaaaaaa.twconfig"); + assert!(own_temp(&t, "aaaaaaaa")); + assert!(!own_temp(&t, "bbbbbbbb")); + assert!(own_temp(&temp_name("aaaaaaaa"), "aaaaaaaa")); + assert!(own_temp(&temp_name("keyring"), "aaaaaaaa")); + assert!(!own_temp("notes.txt", "aaaaaaaa")); + } + #[test] fn statuses_are_sorted_by_what_to_do_next() { use reqwest::StatusCode as S; diff --git a/crates/tw-sync/src/store/s3.rs b/crates/tw-sync/src/store/s3.rs index a823ee1f..c23db4f8 100644 --- a/crates/tw-sync/src/store/s3.rs +++ b/crates/tw-sync/src/store/s3.rs @@ -17,7 +17,9 @@ use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, utf8_percent_encode}; use reqwest::Method; use tw_types::msg; -use super::{Entry, Store, StoreError, check_path, http_error, http_status}; +use super::{ + Entry, MAX_FILE, MAX_LISTING, Store, StoreError, check_path, http_status, read_limited, +}; const TIMEOUT: Duration = Duration::from_secs(120); @@ -179,12 +181,15 @@ impl S3 { if !body.is_empty() { r = r.body(body.to_vec()); } - r.send().await.map_err(|e| http_error(&e)) + super::send(r).await } /// S3 错误答复里的 `` async fn error_code(r: reqwest::Response) -> String { - let body = r.text().await.unwrap_or_default(); + let body = read_limited(r, 64 * 1024, "the error") + .await + .map(|b| String::from_utf8_lossy(&b).to_string()) + .unwrap_or_default(); super::xml::parse(&body, "Error") .and_then(|(rs, top)| { rs.first() @@ -280,7 +285,8 @@ impl Store for S3 { if !r.status().is_success() { return Err(http_status(r.status(), "ListObjectsV2")); } - let body = r.text().await.map_err(|e| http_error(&e))?; + let body = read_limited(r, MAX_LISTING, "the listing").await?; + let body = String::from_utf8_lossy(&body).to_string(); let Some((records, top)) = super::xml::parse(&body, "Contents") else { return Err(StoreError::Other(msg!( "sync.s3.bad_listing" => @@ -318,9 +324,7 @@ impl Store for S3 { let r = self.send(Method::GET, &url, &[], b"").await?; match r.status().as_u16() { 404 => Ok(None), - s if (200..300).contains(&s) => { - Ok(Some(r.bytes().await.map_err(|e| http_error(&e))?.to_vec())) - } + s if (200..300).contains(&s) => Ok(Some(read_limited(r, MAX_FILE, path).await?)), _ => Err(http_status(r.status(), "GET")), } } diff --git a/crates/tw-sync/src/store/webdav.rs b/crates/tw-sync/src/store/webdav.rs index 3b6a85fa..809f47f2 100644 --- a/crates/tw-sync/src/store/webdav.rs +++ b/crates/tw-sync/src/store/webdav.rs @@ -13,7 +13,10 @@ use std::time::Duration; use reqwest::{Method, StatusCode}; use tw_types::msg; -use super::{Entry, Store, StoreError, check_path, http_error, http_status, split, temp_name}; +use super::{ + Entry, MAX_FILE, MAX_LISTING, Store, StoreError, TEMP_PREFIX, check_path, http_status, + own_temp, read_limited, split, temp_name, +}; /// 一个请求最多等多久。版本最大 16 MiB，慢的网盘也够了 const TIMEOUT: Duration = Duration::from_secs(120); @@ -95,7 +98,7 @@ impl WebDav { } async fn send(&self, r: reqwest::RequestBuilder) -> Result { - r.send().await.map_err(|e| http_error(&e)) + super::send(r).await } /// 建目录。已经有了（405）不算错；上一级没有（409）先建上一级（同步位置的根） @@ -201,8 +204,9 @@ fn last_segment(href: &str) -> String { const PROPFIND: &str = r#" "#; -impl Store for WebDav { - async fn list(&self, dir: &str) -> Result, StoreError> { +impl WebDav { + /// 列一个目录，**连点开头的一起**（清临时文件要看到它们）。目录不在是 `Ok(None)` + async fn list_raw(&self, dir: &str) -> Result>, StoreError> { let url = self.dir(dir)?; let own = url.path().trim_end_matches('/').to_string(); let r = self @@ -215,10 +219,11 @@ impl Store for WebDav { .await?; match r.status().as_u16() { 207 | 200 => {} - 404 => return Ok(Vec::new()), + 404 => return Ok(None), _ => return Err(http_status(r.status(), "PROPFIND")), } - let body = r.text().await.map_err(|e| http_error(&e))?; + let body = read_limited(r, MAX_LISTING, "the directory listing").await?; + let body = String::from_utf8_lossy(&body); let Some((records, _)) = super::xml::parse(&body, "response") else { return Err(StoreError::Other(msg!( "sync.webdav.bad_listing" => @@ -236,7 +241,7 @@ impl Store for WebDav { continue; } let name = last_segment(href); - if name.is_empty() || name.starts_with('.') { + if name.is_empty() { continue; } out.push(Entry { @@ -248,16 +253,77 @@ impl Store for WebDav { }); } out.sort_by(|a, b| a.name.cmp(&b.name)); - Ok(out) + Ok(Some(out)) + } + + async fn list_raw_root(&self) -> Result>, StoreError> { + self.list_raw("").await + } + + /// 同步位置的根在不在。**目录列不出来（404）不等于是空的**：根本身不在时那是地址写错了、 + /// 或者服务器上的文件夹被挪走了，不能当成同步数据被清空了 + async fn root_exists(&self) -> Result { + let r = self + .send( + self.req(method("PROPFIND"), self.base.clone()) + .header("depth", "0") + .header("content-type", "application/xml; charset=utf-8") + .body(PROPFIND), + ) + .await?; + match r.status().as_u16() { + 207 | 200 => Ok(true), + 404 => Ok(false), + _ => Err(http_status(r.status(), "PROPFIND")), + } + } +} + +impl Store for WebDav { + async fn list(&self, dir: &str) -> Result, StoreError> { + match self.list_raw(dir).await? { + Some(all) => Ok(all + .into_iter() + .filter(|e| !e.name.starts_with('.')) + .collect()), + None if self.root_exists().await? => Ok(Vec::new()), + None => Err(StoreError::Unreachable(msg!( + "sync.webdav.root_missing", url = self.base.as_str() => + "the WebDAV folder {url} is not there" + ))), + } + } + + /// 这台设备写到一半断了留下的临时文件（`PUT` 了、没来得及 `MOVE`）。只在开始同步时清一次， + /// 那时这台设备没有在写的 + async fn sweep(&self, tag: &str) { + for dir in ["", crate::layout::VERSIONS, crate::layout::DEVICES] { + let listed = if dir.is_empty() { + self.list_raw_root().await + } else { + self.list_raw(dir).await + }; + let Ok(Some(all)) = listed else { continue }; + for e in all { + if e.name.starts_with(TEMP_PREFIX) && own_temp(&e.name, tag) { + let path = if dir.is_empty() { + e.name.clone() + } else { + format!("{dir}/{}", e.name) + }; + if let Ok(u) = self.file(&path) { + let _ = self.send(self.req(Method::DELETE, u)).await; + } + } + } + } } async fn get(&self, path: &str) -> Result>, StoreError> { let r = self.send(self.req(Method::GET, self.file(path)?)).await?; match r.status().as_u16() { 404 | 410 => Ok(None), - s if (200..300).contains(&s) => { - Ok(Some(r.bytes().await.map_err(|e| http_error(&e))?.to_vec())) - } + s if (200..300).contains(&s) => Ok(Some(read_limited(r, MAX_FILE, path).await?)), _ => Err(http_status(r.status(), "GET")), } } diff --git a/crates/tw-sync/tests/common/mod.rs b/crates/tw-sync/tests/common/mod.rs index 8f758d12..60086f52 100644 --- a/crates/tw-sync/tests/common/mod.rs +++ b/crates/tw-sync/tests/common/mod.rs @@ -3,6 +3,9 @@ //! **只实现同步用到的那几样**，但按规矩答：`PUT` 认 `If-None-Match: *`、`MOVE` 认 //! `Overwrite: F`、`PROPFIND` 只列一层、S3 的列目录认 `prefix` 和 `delimiter`。凭据不对回 401 / //! 403。可以让它「坏掉」：回 503，或者不认 `MOVE`、`If-None-Match`（有的服务器就是这样）。 +//! +//! 假 S3 **自己核对 SigV4 签名**（[`sigv4`]，照 AWS 的规范另写一份，不借被测的那一份；用 AWS +//! 文档里的示例核对过它自己，见 `stores.rs`）：签名算错了照样回 403。 #![allow(dead_code)] @@ -13,6 +16,7 @@ use std::sync::{Arc, Mutex}; use axum::body::Body; use axum::http::{HeaderMap, Request, Response, StatusCode}; +use sha2::{Digest, Sha256}; pub const USER: &str = "tester"; pub const PASS: &str = "app-password"; @@ -102,17 +106,16 @@ pub async fn webdav() -> Fake { if header(req.headers(), "authorization") != Some(want.as_str()) { return status(401); } - let Some(rel) = req.uri().path().strip_prefix("/dav/root/") else { - // 根本身 - if req.uri().path().trim_end_matches('/') == "/dav/root" { - return match req.method().as_str() { - "MKCOL" => status(405), - "PROPFIND" => status(207), - _ => status(405), - }; - } - return status(404); + let path = req.uri().path().to_string(); + let rel = match path.strip_prefix("/dav/root/") { + Some(r) => r, + // 根本身：用户选的那个文件夹，在 + None if path == "/dav/root" => "", + None => return status(404), }; + if rel.is_empty() && req.method().as_str() == "MKCOL" { + return status(405); + } let rel = percent_encoding::percent_decode_str(rel) .decode_utf8_lossy() .to_string(); @@ -128,26 +131,36 @@ pub async fn webdav() -> Fake { } "PROPFIND" => { let dir = rel.trim_end_matches('/').to_string(); - if !dirs.contains(&dir) { + if !dir.is_empty() && !dirs.contains(&dir) { return status(404); } + let at = |p: &str| { + if p.is_empty() { + "/dav/root/".to_string() + } else { + format!("/dav/root/{p}/") + } + }; let mut xml = String::from( r#""#, ); xml.push_str(&format!( - "/dav/root/{dir}/\ - " + "{}\ + ", + at(&dir) )); - for (name, bytes) in files.iter() { - if let Some((parent, leaf)) = name.rsplit_once('/') - && parent == dir - { - xml.push_str(&format!( - "/dav/root/{parent}/{leaf}\ - {}\ - HTTP/1.1 200 OK", - bytes.len() - )); + if header(req.headers(), "depth") != Some("0") { + for (name, bytes) in files.iter() { + let (parent, leaf) = name.rsplit_once('/').unwrap_or(("", name)); + if parent == dir { + xml.push_str(&format!( + "{}{leaf}\ + {}\ + HTTP/1.1 200 OK", + at(parent), + bytes.len() + )); + } } } xml.push_str(""); @@ -223,15 +236,22 @@ pub async fn s3() -> Fake { return status(503); } let auth = header(req.headers(), "authorization").unwrap_or_default(); - if !auth.starts_with(&format!("AWS4-HMAC-SHA256 Credential={AK}/")) - || !auth.contains("/s3/aws4_request") - || header(req.headers(), "x-amz-content-sha256").is_none() - { + let signed = sigv4( + req.method().as_str(), + req.uri().path(), + req.uri().query().unwrap_or_default(), + req.headers(), + &body, + AK, + SK, + ); + if !auth.starts_with(&format!("AWS4-HMAC-SHA256 Credential={AK}/")) || signed.is_err() { return Response::builder() .status(403) - .body(Body::from( - "SignatureDoesNotMatch", - )) + .body(Body::from(format!( + "SignatureDoesNotMatch{}", + signed.err().unwrap_or_default() + ))) .unwrap(); } let path = req.uri().path().to_string(); @@ -304,6 +324,146 @@ pub async fn s3() -> Fake { Fake { addr, files, knobs } } +// ─────────────────────────────────────────────── SigV4（服务端核对） + +/// HMAC-SHA256（RFC 2104），照定义写 +fn hmac(key: &[u8], msg: &[u8]) -> [u8; 32] { + let mut k = [0u8; 64]; + if key.len() > 64 { + let d: [u8; 32] = Sha256::digest(key).into(); + k[..32].copy_from_slice(&d); + } else { + k[..key.len()].copy_from_slice(key); + } + let mut inner = Sha256::new(); + inner.update(k.map(|b| b ^ 0x36)); + inner.update(msg); + let inner: [u8; 32] = inner.finalize().into(); + let mut outer = Sha256::new(); + outer.update(k.map(|b| b ^ 0x5c)); + outer.update(inner); + outer.finalize().into() +} + +fn hex(b: &[u8]) -> String { + b.iter().map(|x| format!("{x:02x}")).collect() +} + +/// AWS 的 URI 编码：只有 `A-Z a-z 0-9 - _ . ~` 不编码，别的一律 `%XX`（大写） +fn aws_encode(s: &str, keep_slash: bool) -> String { + let mut out = String::new(); + for b in s.bytes() { + match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + out.push(b as char) + } + b'/' if keep_slash => out.push('/'), + _ => out.push_str(&format!("%{b:02X}")), + } + } + out +} + +fn decode(s: &str) -> String { + percent_encoding::percent_decode_str(s) + .decode_utf8_lossy() + .to_string() +} + +/// 照 AWS SigV4 的规范从收到的请求重算签名，和 `Authorization` 里的比。不对是 `Err(原因)` +pub fn sigv4( + method: &str, + path: &str, + query: &str, + headers: &HeaderMap, + body: &[u8], + access_key: &str, + secret: &str, +) -> Result<(), String> { + let auth = header(headers, "authorization").ok_or("no Authorization")?; + let rest = auth + .strip_prefix("AWS4-HMAC-SHA256 ") + .ok_or("not AWS4-HMAC-SHA256")?; + let mut fields: BTreeMap<&str, &str> = BTreeMap::new(); + for part in rest.split(',') { + let (k, v) = part.trim().split_once('=').ok_or("bad Authorization")?; + fields.insert(k, v); + } + let cred = fields.get("Credential").ok_or("no Credential")?; + let signed = fields.get("SignedHeaders").ok_or("no SignedHeaders")?; + let got = fields.get("Signature").ok_or("no Signature")?; + let mut scope_parts = cred.splitn(2, '/'); + if scope_parts.next() != Some(access_key) { + return Err("unknown access key".into()); + } + let scope = scope_parts.next().ok_or("no scope")?; + let s: Vec<&str> = scope.split('/').collect(); + if s.len() != 4 || s[2] != "s3" || s[3] != "aws4_request" { + return Err(format!("bad scope {scope}")); + } + let amz_date = header(headers, "x-amz-date").ok_or("no x-amz-date")?; + if !amz_date.starts_with(s[0]) { + return Err("x-amz-date is not the scope's date".into()); + } + let payload = header(headers, "x-amz-content-sha256").ok_or("no x-amz-content-sha256")?; + if payload != "UNSIGNED-PAYLOAD" && payload != hex(&Sha256::digest(body)) { + return Err("x-amz-content-sha256 does not match the body".into()); + } + let names: Vec<&str> = signed.split(';').collect(); + if !names.contains(&"host") { + return Err("host is not signed".into()); + } + let mut canon_headers = String::new(); + for n in &names { + let v = headers + .get_all(*n) + .iter() + .filter_map(|v| v.to_str().ok()) + .map(|v| v.split_whitespace().collect::>().join(" ")) + .collect::>() + .join(","); + if v.is_empty() && headers.get(*n).is_none() { + return Err(format!("signed header {n} is not there")); + } + canon_headers.push_str(&format!("{n}:{v}\n")); + } + let uri: String = path + .split('/') + .map(|seg| aws_encode(&decode(seg), false)) + .collect::>() + .join("/"); + let mut pairs: Vec<(String, String)> = query + .split('&') + .filter(|p| !p.is_empty()) + .map(|p| { + let (k, v) = p.split_once('=').unwrap_or((p, "")); + (aws_encode(&decode(k), false), aws_encode(&decode(v), false)) + }) + .collect(); + pairs.sort(); + let canon_query = pairs + .iter() + .map(|(k, v)| format!("{k}={v}")) + .collect::>() + .join("&"); + let canonical = format!("{method}\n{uri}\n{canon_query}\n{canon_headers}\n{signed}\n{payload}"); + let to_sign = format!( + "AWS4-HMAC-SHA256\n{amz_date}\n{scope}\n{}", + hex(&Sha256::digest(canonical.as_bytes())) + ); + let k_date = hmac(format!("AWS4{secret}").as_bytes(), s[0].as_bytes()); + let k_region = hmac(&k_date, s[1].as_bytes()); + let k_service = hmac(&k_region, b"s3"); + let k_signing = hmac(&k_service, b"aws4_request"); + let want = hex(&hmac(&k_signing, to_sign.as_bytes())); + if want != *got { + return Err(format!( + "signature does not match; canonical request:\n{canonical}" + )); + } + Ok(()) +} + /// 一个没人听的地址：连上去就被拒绝 pub fn nowhere() -> String { let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); diff --git a/crates/tw-sync/tests/devices.rs b/crates/tw-sync/tests/devices.rs index add592cf..415c718c 100644 --- a/crates/tw-sync/tests/devices.rs +++ b/crates/tw-sync/tests/devices.rs @@ -514,13 +514,41 @@ async fn encryption_can_be_turned_on_off_and_its_passphrase_changed() { assert!(t.eb.unlock("a brand new passphrase").await.unwrap()); t.eb.round().await.unwrap(); - // A 关闭加密：B 跟着变成不加密 + // A 关闭加密：keyring 留着、改写成「已关闭」（有口令的人关的），版本不再加密；B 跟着变成 + // 不加密 t.ea.disable_encryption("a brand new passphrase") .await .unwrap(); - assert!(!t.loc.path().join("keyring").exists()); + let kr = std::fs::read(t.loc.path().join("keyring")).unwrap(); + assert!(tw_sync::keyring::marked_off(&kr)); + assert!( + tw_sync::keyring::Keyring::open(&kr, "a brand new passphrase") + .unwrap() + .off() + ); + for v in versions(t.loc.path()) { + let bytes = std::fs::read(t.loc.path().join("versions").join(&v)).unwrap(); + assert!(!tw_pack::is_encrypted(&bytes), "加密过的版本删了：{v}"); + } let r = t.eb.round().await.unwrap(); assert!(!r.encrypted, "同步位置不加密了，B 照着改设置"); + t.b.edit(|s| s.replace("sk-after-encryption", "sk-after-off")); + assert!(t.eb.round().await.unwrap().uploaded.is_some()); + t.ea.round().await.unwrap(); + assert_eq!( + t.a.provider("relay").unwrap().key.unwrap().raw(), + "sk-after-off" + ); + + // 再开回来：B 用同一个口令接着同步 + t.ea.enable_encryption("a brand new passphrase") + .await + .unwrap(); + assert!(!tw_sync::keyring::marked_off( + &std::fs::read(t.loc.path().join("keyring")).unwrap() + )); + let r = t.eb.round().await.unwrap(); + assert!(r.encrypted, "B 记着口令，解得开新的 keyring"); } /// 两台设备同时开启加密同步：keyring 只建得成一份，后写的那台转去加入 @@ -659,19 +687,48 @@ async fn a_key_in_use_here_is_not_deleted_by_another_device() { ); } -/// 另一台设备停止同步并删除了同步数据：这台设备发现同步位置空了 +/// 另一台设备停止同步并删除了同步数据：这台设备几轮、一段时间之后确认同步位置空了，由人 +/// 决定（重新上传本机配置），同步设置和口令都不动 #[tokio::test] async fn a_purged_location_is_noticed() { let mut t = bed(true).await; t.ea.stop(true).await.unwrap(); assert!(versions(t.loc.path()).is_empty()); assert!(!t.loc.path().join("keyring").exists()); + for _ in 0..3 { + let r = t.eb.round().await; + assert!( + matches!(r, Err(RoundError::LooksEmpty)), + "一轮读到空的不下结论：{:?}", + r.map(|r| r.uploaded) + ); + } + // 过了十分钟（改本机记着的时刻） + let mut st = t.eb.local().state(); + st.empty_since_ms = st.empty_since_ms.map(|t| t - 11 * 60 * 1000); + t.eb.local().save_state(&st).unwrap(); let r = t.eb.round().await; assert!( matches!(r, Err(RoundError::Emptied)), "{:?}", r.map(|r| r.uploaded) ); + assert!(t.eb.local().base().is_some(), "本机的同步状态不删"); + // 重新上传本机配置：keyring 放回这台记着的那一份，版本以本机配置上传 + t.b.edit(|s| add_provider(s, "kept", "https://kept.example", "sk-kept")); + let r = t.eb.republish().await.unwrap(); + assert!(r.encrypted); + assert!(t.loc.path().join("keyring").exists()); + assert_eq!(versions(t.loc.path()).len(), 1); + // A 重新加入，拿到 B 的配置 + let mut ea = Engine::new( + folder(t.loc.path()), + t.a.clone(), + t.a.dir.path(), + settings(A_ID, "Mac mini", true, t.loc.path()), + ); + ea.join(None).await.unwrap(); + assert!(t.a.provider("kept").is_some()); } /// 同一套流程走 WebDAV 和 S3（进程内的假服务） diff --git a/crates/tw-sync/tests/regressions.rs b/crates/tw-sync/tests/regressions.rs new file mode 100644 index 00000000..08440fb7 --- /dev/null +++ b/crates/tw-sync/tests/regressions.rs @@ -0,0 +1,1118 @@ +//! 审查里找出来的同步问题，一条一条钉住：加密不可降级、等登录的上游、捷径不吞改动、基准不是 +//! 祖先、晚到的头、一个坏文件挡住所有设备、同步位置读起来是空的、清除只删同步文件。 +//! +//! 设备这一边和 `devices.rs` 一样是最简单的那种（配置就是数据目录里的 `config.yaml`），多一个 +//! 钩子：一轮的中途（读完本机配置之后、上传之前）跑一次，模拟用户此刻改配置、别的设备此刻上传。 + +use std::collections::BTreeSet; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; + +use sha2::Digest; +use tw_pack::{Pack, PackKind, PackOrigin, Seal}; +use tw_sync::engine::{ + ApplyError, Current, Decision, Engine, Host, RoundError, SKIP_AFTER_MS, Settings, Take, +}; +use tw_sync::keyring::Keyring; +use tw_sync::store::{AnyStore, Folder}; +use tw_sync::{Entry, Store, StoreError}; +use tw_types::Msg; + +const PASS: &str = "correct horse battery staple"; + +fn base_config(c: char) -> String { + format!( + "version: 1\nlisten:\n control:\n key: {}\nclients:\n - name: default\n key: tw-default-key-0001\n", + c.to_string().repeat(64) + ) +} + +type Hook = Arc>>>; + +#[derive(Clone)] +struct Dev { + dir: Arc, + hook: Hook, +} + +impl Dev { + fn new(key: char) -> Self { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("config.yaml"), base_config(key)).unwrap(); + Self { + dir: Arc::new(dir), + hook: Default::default(), + } + } + fn path(&self) -> PathBuf { + self.dir.path().join("config.yaml") + } + fn text(&self) -> String { + std::fs::read_to_string(self.path()).unwrap() + } + fn config(&self) -> tw_config::Config { + tw_config::try_parse(&self.text()).unwrap_or_else(|e| panic!("{e}\n{}", self.text())) + } + fn edit(&self, f: impl FnOnce(&str) -> String) { + let next = f(&self.text()); + tw_config::try_parse(&next).unwrap_or_else(|e| panic!("测试写坏了配置：{e}\n{next}")); + std::fs::write(self.path(), next).unwrap(); + } + fn provider(&self, name: &str) -> Option { + self.config().providers.into_iter().find(|p| p.name == name) + } + /// 一轮的中途跑一次 + fn during_next_round(&self, f: impl FnOnce() + Send + 'static) { + *self.hook.lock().unwrap() = Some(Box::new(f)); + } +} + +impl Host for Dev { + fn current(&self) -> Result { + let text = self.text(); + Ok(Current { + pack: Pack { + manifest: tw_pack::Manifest::new(PackKind::Full, PackOrigin::Ui), + config_yaml: text.clone(), + plugins: Default::default(), + client_prefs: None, + }, + version: tw_config::version_of(&text), + changed_ms: 1, + }) + } + async fn apply(&self, pack: Pack, base_version: &str, _from: &str) -> Result<(), ApplyError> { + if tw_config::version_of(&self.text()) != base_version { + return Err(ApplyError::Stale); + } + tw_config::try_parse(&pack.config_yaml).map_err(|r| ApplyError::Rejected(r.msg()))?; + std::fs::write(self.path(), &pack.config_yaml).unwrap(); + Ok(()) + } + fn pinned_keys(&self) -> BTreeSet { + if let Some(f) = self.hook.lock().unwrap().take() { + f(); + } + BTreeSet::new() + } + fn backup(&self, _content_id: &str) -> Option { + None + } +} + +fn settings(id: &str, name: &str, encrypt: bool, place: &Path) -> Settings { + Settings { + device_id: id.into(), + device_name: name.into(), + encrypt, + passphrase: encrypt.then(|| PASS.to_string()), + exclude: BTreeSet::new(), + place: format!("folder:{}", place.display()), + remote_core: false, + } +} + +const A_ID: &str = "aaaaaaaa-0000-4000-8000-000000000001"; +const B_ID: &str = "bbbbbbbb-0000-4000-8000-000000000002"; + +struct Bed { + loc: tempfile::TempDir, + a: Dev, + b: Dev, + ea: Engine, + eb: Engine, +} + +fn folder(loc: &Path) -> AnyStore { + AnyStore::Folder(Folder::new(loc)) +} + +async fn bed(encrypt: bool) -> Bed { + tw_sync::keyring::set_work_factor_for_tests(4); + let loc = tempfile::tempdir().unwrap(); + let (a, b) = (Dev::new('a'), Dev::new('b')); + let mut ea = Engine::new( + folder(loc.path()), + a.clone(), + a.dir.path(), + settings(A_ID, "Mac mini", encrypt, loc.path()), + ); + let mut eb = Engine::new( + folder(loc.path()), + b.clone(), + b.dir.path(), + settings(B_ID, "MacBook Air", encrypt, loc.path()), + ); + assert!(ea.create().await.unwrap()); + eb.join(None).await.unwrap(); + Bed { loc, a, b, ea, eb } +} + +fn add_provider_raw(text: &str, yaml: &str) -> String { + if text.contains("\nproviders:\n") { + text.replacen("\nproviders:\n", &format!("\nproviders:\n{yaml}"), 1) + } else { + format!("{text}providers:\n{yaml}") + } +} + +fn add_provider(text: &str, name: &str, url: &str, key: &str) -> String { + add_provider_raw( + text, + &format!(" - name: {name}\n base_url: {url}\n key: {key}\n"), + ) +} + +fn versions(loc: &Path) -> Vec { + let mut v: Vec = std::fs::read_dir(loc.join("versions")) + .map(|rd| { + rd.filter_map(|e| e.ok()) + .map(|e| e.file_name().to_string_lossy().to_string()) + .filter(|n| !n.starts_with('.')) + .collect() + }) + .unwrap_or_default(); + v.sort(); + v +} + +fn read_version(loc: &Path, id: &str) -> Vec { + std::fs::read(loc.join("versions").join(format!("{id}.twconfig"))).unwrap() +} + +/// 没有口令的人往同步位置里放一个不加密的版本（manifest.sync 照格式写、带 SHA-256），依据 +/// 最新的那一个 +fn forge_plain_version(loc: &Path, config_yaml: String) -> String { + let head = versions(loc) + .last() + .unwrap() + .trim_end_matches(".twconfig") + .to_string(); + let seq: u64 = head[..10].parse().unwrap(); + let mut pack = Pack { + manifest: tw_pack::Manifest::new(PackKind::Full, PackOrigin::Sync), + config_yaml, + plugins: Default::default(), + client_prefs: None, + }; + let sha: [u8; 32] = sha2::Sha256::digest(pack.config_yaml.as_bytes()).into(); + let sha: String = sha.iter().map(|b| format!("{b:02x}")).collect(); + let meta = tw_sync::record::Meta { + seq: seq + 1, + device: "eeeeeeee-0000-4000-8000-00000000000e".into(), + device_name: "Eve".into(), + parents: vec![head], + sha256: Some(sha), + ..Default::default() + }; + pack.manifest.sync = Some(serde_json::to_value(&meta).unwrap()); + let bytes = tw_pack::write(&pack, Seal::None).unwrap(); + let name = format!("{:010}-eeeeeeee.twconfig", seq + 1); + std::fs::write(loc.join("versions").join(&name), bytes).unwrap(); + name +} + +/// 把本机记着的「从什么时候起」往前拨，模拟时间过去了 +fn age_state(e: &Engine, by_ms: u64) { + let mut st = e.local().state(); + for t in st.unreadable.values_mut() { + *t = t.saturating_sub(by_ms); + } + st.empty_since_ms = st.empty_since_ms.map(|t| t.saturating_sub(by_ms)); + e.local().save_state(&st).unwrap(); +} + +// ─────────────────────────────────────────────── 加密不可降级 + +/// 加密的同步位置：没有口令的人放进来一个不加密的版本 —— 不读、不合并，计入跳过 +#[tokio::test] +async fn a_plaintext_version_in_an_encrypted_location_is_never_read() { + let mut t = bed(true).await; + t.a.edit(|s| add_provider(s, "relay", "https://relay.example", "sk-secret-relay")); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + assert!(t.b.provider("relay").is_some()); + let evil = add_provider(&t.a.text(), "evil", "https://evil.example", "sk-attacker"); + forge_plain_version(t.loc.path(), evil); + let r = t.eb.round().await.unwrap(); + assert!(t.b.provider("evil").is_none(), "不加密的版本被合并了"); + assert!(r.encrypted); + assert_eq!(r.skipped, 1, "跳过了，界面看得到"); +} + +/// keyring 被删、再放进一个不加密的版本：不降级、不上传明文、口令不删，要人处理。放回这台 +/// 设备记着的 keyring 之后照常同步，伪造的那个版本照样不读 +#[tokio::test] +async fn a_deleted_keyring_does_not_turn_encryption_off() { + let mut t = bed(true).await; + t.a.edit(|s| add_provider(s, "relay", "https://relay.example", "sk-secret-relay")); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + std::fs::remove_file(t.loc.path().join("keyring")).unwrap(); + forge_plain_version(t.loc.path(), t.a.text()); + t.b.edit(|s| add_provider(s, "bnew", "https://bnew.example", "sk-b-secret")); + let before = versions(t.loc.path()); + let r = t.eb.round().await; + assert!( + matches!(r, Err(RoundError::KeyringMissing(_))), + "{:?}", + r.map(|r| r.uploaded) + ); + assert_eq!(versions(t.loc.path()), before, "什么都没上传"); + assert!(t.eb.settings().encrypt && t.eb.settings().passphrase.is_some()); + // 再来一轮还是一样：不会过一会儿自己转成不加密 + assert!(matches!( + t.eb.round().await, + Err(RoundError::KeyringMissing(_)) + )); + + // 由人选「重新上传」：keyring 放回 B 记着的那一份 + let r = t.eb.republish().await.unwrap(); + assert!(r.encrypted); + assert!(t.loc.path().join("keyring").exists()); + let up = r.uploaded.expect("B 的改动这时上传"); + assert!(tw_pack::is_encrypted(&read_version(t.loc.path(), &up))); + assert!(t.b.provider("bnew").is_some()); + // A 用的还是同一把密钥，照常同步 + t.ea.round().await.unwrap(); + assert!(t.a.provider("bnew").is_some()); + assert!(t.a.provider("evil").is_none()); +} + +/// 关过一次加密、又开回来之后，有人把当年那份「已关闭」的 keyring 放回来（它当初是真的， +/// 口令也解得开）：代号比见过的小，不认 +#[tokio::test] +async fn an_old_keyring_that_says_off_is_not_believed() { + let mut t = bed(true).await; + t.ea.disable_encryption(PASS).await.unwrap(); + let off = std::fs::read(t.loc.path().join("keyring")).unwrap(); + assert!(tw_sync::keyring::marked_off(&off)); + let r = t.eb.round().await.unwrap(); + assert!(!r.encrypted, "有口令的人关的：认"); + t.ea.enable_encryption(PASS).await.unwrap(); + assert!(t.eb.round().await.unwrap().encrypted); + + std::fs::write(t.loc.path().join("keyring"), &off).unwrap(); + t.b.edit(|s| add_provider(s, "secret", "https://s.example", "sk-not-for-plaintext")); + let before = versions(t.loc.path()); + let r = t.eb.round().await; + assert!( + matches!(&r, Err(RoundError::KeyringMissing(m)) if m.code == "sync.keyring_rolled_back"), + "{:?}", + r.map(|r| r.encrypted) + ); + assert_eq!( + versions(t.loc.path()), + before, + "没有按旧的 keyring 上传明文" + ); + // 放回 B 记着的那一份（较新的） + let r = t.eb.republish().await.unwrap(); + assert!(r.encrypted); + for v in versions(t.loc.path()) { + let bytes = std::fs::read(t.loc.path().join("versions").join(&v)).unwrap(); + assert!(!String::from_utf8_lossy(&bytes).contains("sk-not-for-plaintext")); + } +} + +/// 关了加密之后有人放进来一个只有明文标记的假 keyring（解不开）：记着口令的设备不认标记， +/// 要输入口令；没有口令的设备（本来就不加密）照常不加密同步 +#[tokio::test] +async fn a_fake_off_marker_does_not_fool_a_device_with_the_passphrase() { + let mut t = bed(true).await; + t.ea.disable_encryption(PASS).await.unwrap(); + assert!(!t.eb.round().await.unwrap().encrypted); + let mut fake = tw_sync::keyring::OFF_MARK.to_vec(); + fake.extend(Keyring::generate(0).seal("someone else's phrase")); + std::fs::write(t.loc.path().join("keyring"), fake).unwrap(); + let r = t.eb.round().await; + assert!( + matches!(r, Err(RoundError::Locked(_))), + "{:?}", + r.map(|r| r.encrypted) + ); + // 一台从没有过口令的设备照常 + let c = Dev::new('c'); + let mut s = settings( + "cccccccc-0000-4000-8000-000000000003", + "Mini PC", + false, + t.loc.path(), + ); + s.passphrase = None; + let mut ec = Engine::new(folder(t.loc.path()), c.clone(), c.dir.path(), s); + assert!(!ec.join(None).await.unwrap().encrypted); +} + +/// 加密的同步位置上，不加密的设备记录不列出来；之后开启加密时删掉不加密的设备记录 +#[tokio::test] +async fn plaintext_device_records_do_not_count_once_encrypted() { + let mut t = bed(false).await; + let fake = serde_json::json!({ + "id": "cccccccc-0000-4000-8000-00000000000c", "name": "Fake", "core_version": "0.77.0", + "last_seen_ms": 4102444800000u64, "paused_update": false, "remote_core": false + }); + std::fs::write( + t.loc.path().join("devices/cccccccc"), + serde_json::to_vec(&fake).unwrap(), + ) + .unwrap(); + let names: Vec = + t.ea.round() + .await + .unwrap() + .devices + .into_iter() + .map(|d| d.name) + .collect(); + assert!( + names.contains(&"Fake".to_string()), + "不加密时是一份普通的记录" + ); + t.ea.enable_encryption(PASS).await.unwrap(); + assert!( + !t.loc.path().join("devices/cccccccc").exists(), + "开启加密时删掉不加密的设备记录" + ); + std::fs::write( + t.loc.path().join("devices/cccccccc"), + serde_json::to_vec(&fake).unwrap(), + ) + .unwrap(); + t.a.edit(|s| add_provider(s, "x", "https://x.example", "sk-x")); + let names: Vec = + t.ea.round() + .await + .unwrap() + .devices + .into_iter() + .map(|d| d.name) + .collect(); + assert!(!names.contains(&"Fake".to_string()), "{names:?}"); + assert!(names.contains(&"Mac mini".to_string())); +} + +// ─────────────────────────────────────────────── 捷径不吞改动 + +/// 一轮中途用户改了配置：下一轮照样上传 +#[tokio::test] +async fn an_edit_made_during_a_round_is_uploaded_next_round() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider(s, "one", "https://one.example", "sk-1")); + let a = t.a.clone(); + t.a.during_next_round(move || { + a.edit(|s| add_provider(s, "two", "https://two.example", "sk-2")); + }); + assert!(t.ea.round().await.unwrap().uploaded.is_some()); + assert!(t.a.provider("two").is_some()); + assert!( + t.ea.round().await.unwrap().uploaded.is_some(), + "中途的改动这一轮上传" + ); + t.eb.round().await.unwrap(); + assert!(t.b.provider("two").is_some()); +} + +/// 一轮中途别的设备传了新版本：下一轮照样合并 +#[tokio::test] +async fn a_version_arriving_during_a_round_is_merged_next_round() { + let mut t = bed(false).await; + t.b.edit(|s| add_provider(s, "fromb", "https://b.example", "sk-b")); + let up_b = t.eb.round().await.unwrap().uploaded.unwrap(); + let parked = tempfile::tempdir().unwrap(); + let file = format!("{up_b}.twconfig"); + let in_loc = t.loc.path().join("versions").join(&file); + let out = parked.path().join(&file); + std::fs::rename(&in_loc, &out).unwrap(); + t.a.edit(|s| add_provider(s, "froma", "https://a.example", "sk-a")); + t.a.during_next_round(move || std::fs::rename(&out, &in_loc).unwrap()); + t.ea.round().await.unwrap(); + assert!(t.a.provider("fromb").is_none(), "这一轮开始时还没到"); + t.ea.round().await.unwrap(); + assert!(t.a.provider("fromb").is_some()); + t.eb.round().await.unwrap(); + assert!(t.b.provider("froma").is_some()); +} + +// ─────────────────────────────────────────────── 等登录的上游 + +const CHATGPT: &str = " - name: chatgpt\n base_url: https://chatgpt.com/backend-api/codex\n oauth:\n refresh: rt-a\n endpoint: https://auth.openai.com/oauth/token\n headers:\n ChatGPT-Account-Id: acct-a\n"; +const CORP: &str = " - name: corp\n base_url: https://corp.example\n oauth:\n refresh: rt-corp-a\n endpoint: https://auth.example/token\n"; + +/// A 上登录着的 ChatGPT 账号和 OAuth 上游到了没登录的 B：在 B 上停用着，没有冲突；A 上照旧 +/// 启用、登录还在。B 登录之后照同步的样子启用 +#[tokio::test] +async fn an_upstream_signed_in_elsewhere_waits_for_a_sign_in_here() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider_raw(&add_provider_raw(s, CHATGPT), CORP)); + t.ea.round().await.unwrap(); + let r = t.eb.round().await.unwrap(); + assert!(r.conflicts.is_empty(), "{:?}", r.conflicts); + for n in ["chatgpt", "corp"] { + let p = t.b.provider(n).unwrap_or_else(|| panic!("{n} 没到 B")); + assert!(p.disabled, "{n} 在 B 上停用着"); + } + assert!(t.b.provider("chatgpt").unwrap().oauth.is_none()); + assert_eq!( + t.eb.state().held_login, + BTreeSet::from(["chatgpt".to_string(), "corp".to_string()]) + ); + // 停用没有传回 A，也不会来回上传 + let listing = versions(t.loc.path()); + let ra = t.ea.round().await.unwrap(); + assert!(ra.uploaded.is_none() && ra.applied_from.is_empty()); + assert!(t.eb.round().await.unwrap().uploaded.is_none()); + assert_eq!(versions(t.loc.path()), listing); + let a = t.a.provider("chatgpt").unwrap(); + assert!(!a.disabled); + assert_eq!(a.oauth.unwrap().refresh, "rt-a"); + + // A 改了并发：B 收到，还停用着 + t.a.edit(|s| { + s.replace( + " base_url: https://corp.example\n", + " base_url: https://corp.example\n max_concurrent: 3\n", + ) + }); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + let corp = t.b.provider("corp").unwrap(); + assert_eq!(corp.max_concurrent, Some(3)); + assert!(corp.disabled); + + // B 上登录 corp（填上刷新令牌）：照同步的样子启用，A 不受影响 + t.b.edit(|s| { + let mut cfg: serde_yaml_ng::Value = serde_yaml_ng::from_str(s).unwrap(); + for p in cfg["providers"].as_sequence_mut().unwrap() { + if p["name"] == "corp" { + p["oauth"]["refresh"] = "rt-corp-b".into(); + } + } + serde_yaml_ng::to_string(&cfg).unwrap() + }); + let r = t.eb.round().await.unwrap(); + assert!(r.uploaded.is_none(), "登录是这台的事，不上传"); + let corp = t.b.provider("corp").unwrap(); + assert!(!corp.disabled, "登录之后启用"); + assert_eq!(corp.oauth.unwrap().refresh, "rt-corp-b"); + assert_eq!( + t.eb.state().held_login, + BTreeSet::from(["chatgpt".to_string()]) + ); + t.ea.round().await.unwrap(); + assert_eq!( + t.a.provider("corp").unwrap().oauth.unwrap().refresh, + "rt-corp-a" + ); +} + +/// B 上自己改了等登录上游的别的设置：照常上传，停用不跟着走 +#[tokio::test] +async fn editing_a_waiting_upstream_here_does_not_turn_it_off_elsewhere() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider_raw(s, CHATGPT)); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + assert!(t.b.provider("chatgpt").unwrap().disabled); + t.b.edit(|s| { + s.replace( + " base_url: https://chatgpt.com/backend-api/codex\n", + " base_url: https://chatgpt.com/backend-api/codex\n max_concurrent: 2\n", + ) + }); + assert!(t.eb.round().await.unwrap().uploaded.is_some()); + t.ea.round().await.unwrap(); + let a = t.a.provider("chatgpt").unwrap(); + assert_eq!(a.max_concurrent, Some(2)); + assert!(!a.disabled, "B 上的停用没传过来"); + assert!(a.oauth.is_some()); +} + +/// 两台都登录了同名的 ChatGPT 上游（各自的账号）：A 改并发，B 收到后登录还是 B 自己的 +#[tokio::test] +async fn two_devices_signed_in_to_one_chatgpt_upstream_keep_their_own_login() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider_raw(s, CHATGPT)); + t.b.edit(|s| { + add_provider_raw( + s, + &CHATGPT.replace("rt-a", "rt-b").replace("acct-a", "acct-b"), + ) + }); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + t.a.edit(|s| { + s.replace( + " base_url: https://chatgpt.com/backend-api/codex\n", + " base_url: https://chatgpt.com/backend-api/codex\n max_concurrent: 4\n", + ) + }); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + let p = t.b.provider("chatgpt").unwrap(); + assert_eq!(p.max_concurrent, Some(4)); + assert!(!p.disabled); + assert_eq!(p.oauth.unwrap().refresh, "rt-b"); +} + +// ─────────────────────────────────────────────── 冲突 + +/// 网关密钥的冲突选「两者都保留」：另一份是一把新的密钥，不是同一个值，也不归哪个客户端 +#[tokio::test] +async fn keeping_both_gateway_keys_makes_the_copy_a_new_key() { + let mut t = bed(false).await; + t.a.edit(|s| { + s.replace( + "clients:\n", + "clients:\n - name: k2\n key: tw-k2-value-0001\n client: codex\n", + ) + }); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + t.a.edit(|s| { + s.replace( + " client: codex\n", + " client: codex\n max_concurrent: 2\n", + ) + }); + t.b.edit(|s| { + s.replace( + " client: codex\n", + " client: codex\n max_concurrent: 3\n", + ) + }); + t.ea.round().await.unwrap(); + let rb = t.eb.round().await.unwrap(); + let c = rb.conflicts.iter().find(|c| c.item.name == "k2").unwrap(); + assert_eq!( + tw_sync::engine::options(c), + [Take::Local, Take::Remote, Take::Both] + ); + let r = + t.eb.round_with(&[Decision { + id: c.id.clone(), + take: Take::Both, + new_name: None, + }]) + .await + .unwrap(); + assert!(r.conflicts.is_empty()); + let clients = t.b.config().clients; + let orig = clients.iter().find(|c| c.name == "k2").unwrap(); + let copy = clients + .iter() + .find(|c| c.name.starts_with("k2 (")) + .unwrap_or_else(|| panic!("{clients:?}")); + assert_eq!(orig.key, "tw-k2-value-0001"); + assert_ne!(copy.key, orig.key, "另一份是一把新的密钥"); + assert!(copy.key.starts_with("tw-")); + assert_eq!(copy.max_concurrent, Some(2), "另一份是另一边的设置"); + assert!(copy.client.is_none(), "新的那把不归哪个客户端"); +} + +/// 合并结果过不了校验的冲突（`invalid`）只能选本机的：选另一边是一个明确的错误 +#[tokio::test] +async fn an_invalid_conflict_only_offers_this_side() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider(s, "old", "https://old.example", "sk-old")); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + t.a.edit(|s| { + s.replace( + " - name: old\n base_url: https://old.example\n key: sk-old\n", + "", + ) + .replace("providers:\n", "") + }); + t.b.edit(|s| { + format!("{s}routes:\n - name: viaold\n rules:\n - name: r\n to: old\n") + }); + t.eb.round().await.unwrap(); + let ra = t.ea.round().await.unwrap(); + let c = ra + .conflicts + .iter() + .find(|c| c.reason == Some(tw_sync::record::Reason::Invalid)) + .unwrap_or_else(|| panic!("{:?}", ra.conflicts)); + assert_eq!(tw_sync::engine::options(c), [Take::Local]); + let r = + t.ea.round_with(&[Decision { + id: c.id.clone(), + take: Take::Remote, + new_name: None, + }]) + .await; + assert!( + matches!(&r, Err(RoundError::Local(m)) if m.code == "sync.take_not_allowed"), + "{:?}", + r.map(|r| r.conflicts.len()) + ); + let r = + t.ea.round_with(&[Decision { + id: c.id.clone(), + take: Take::Local, + new_name: None, + }]) + .await + .unwrap(); + assert!(r.conflicts.is_empty()); +} + +/// 全局设置：一边把一节改回默认（这一项从快照里消失），另一边改了这一节的另一个字段 —— +/// 各管各的字段，不是冲突 +#[tokio::test] +async fn a_global_section_reset_to_defaults_merges_field_by_field() { + let mut t = bed(false).await; + t.a.edit(|s| format!("{s}failover:\n pause_secs: 100\n")); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + assert_eq!(t.b.config().failover.pause_secs, 100); + let default_pause = tw_config::Config::default().failover.pause_secs; + t.a.edit(|s| s.replace("failover:\n pause_secs: 100\n", "")); + t.b.edit(|s| { + s.replace( + "failover:\n pause_secs: 100\n", + "failover:\n pause_secs: 100\n failures_to_pause: 7\n", + ) + }); + t.ea.round().await.unwrap(); + let rb = t.eb.round().await.unwrap(); + assert!(rb.conflicts.is_empty(), "{:?}", rb.conflicts); + let f = t.b.config().failover; + assert_eq!((f.pause_secs, f.failures_to_pause), (default_pause, 7)); + t.ea.round().await.unwrap(); + let f = t.a.config().failover; + assert_eq!((f.pause_secs, f.failures_to_pause), (default_pause, 7)); +} + +/// 合并结果过不了校验时逐项回退：另一边新增的「上游 + 它用的出站代理」，出站代理先到位，上游 +/// 不算过不了校验 +#[tokio::test] +async fn a_new_upstream_with_its_new_proxy_is_not_taken_for_invalid() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider(s, "old", "https://old.example", "sk-old")); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + t.a.edit(|s| { + s.replace( + " - name: old\n base_url: https://old.example\n key: sk-old\n", + "", + ) + .replace("providers:\n", "") + }); + t.b.edit(|s| { + let s = add_provider_raw( + s, + " - name: q\n base_url: https://q.example\n key: sk-q\n proxy: hk\n", + ); + format!( + "{s}proxies:\n - name: hk\n type: socks5h\n addr: 127.0.0.1:7890\nroutes:\n - name: viaold\n rules:\n - name: r\n to: old\n" + ) + }); + t.eb.round().await.unwrap(); + let ra = t.ea.round().await.unwrap(); + assert!( + !ra.conflicts + .iter() + .any(|c| c.item.name == "q" || c.item.name == "hk"), + "{:?}", + ra.conflicts + ); + assert!(t.a.provider("q").is_some()); +} + +// ─────────────────────────────────────────────── 谱系 + +/// 云盘晚到：B 依据 v3 传了一个版本，A 那边还没看到时又传了两个、删掉了 v3。B 的版本到了 +/// 之后，两个头共同的上一版本只在 A 之前的基准里：照样三方合并，两边改同一个上游的不同单位 +/// 不是冲突 +#[tokio::test] +async fn a_late_head_merges_three_way_after_its_parent_was_pruned() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider(s, "relay", "https://relay.example", "sk-r")); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + let hidden: Arc>> = Default::default(); + let mut ea = Engine::new( + Lagging { + inner: folder(t.loc.path()), + hidden: hidden.clone(), + }, + t.a.clone(), + t.a.dir.path(), + settings(A_ID, "Mac mini", false, t.loc.path()), + ); + t.b.edit(|s| s.replace(" key: sk-r\n", " key: sk-r\n max_concurrent: 3\n")); + let up_b = t.eb.round().await.unwrap().uploaded.unwrap(); + hidden.lock().unwrap().insert(format!("{up_b}.twconfig")); + t.a.edit(|s| s.replace(" key: sk-r\n", " key: sk-r\n models: [m1]\n")); + ea.round().await.unwrap(); + t.a.edit(|s| s.replace(" models: [m1]\n", " models: [m1, m2]\n")); + ea.round().await.unwrap(); + hidden.lock().unwrap().clear(); + let r = ea.round().await.unwrap(); + assert!(r.conflicts.is_empty(), "{:?}", r.conflicts); + let p = t.a.provider("relay").unwrap(); + assert_eq!(p.max_concurrent, Some(3)); +} + +/// 不止一个头时，删旧版本留着它们共同的上一版本 +#[tokio::test] +async fn pruning_keeps_the_common_parent_of_two_heads() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider(s, "relay", "https://relay.example", "sk-r")); + t.ea.round().await.unwrap(); + t.eb.round().await.unwrap(); + let parent = versions(t.loc.path()).last().unwrap().clone(); + // 两台同时上传：两个头，依据同一个版本 + t.a.edit(|s| s.replace(" key: sk-r\n", " key: sk-r\n models: [m1]\n")); + t.b.edit(|s| s.replace(" key: sk-r\n", " key: sk-r\n max_concurrent: 3\n")); + let hidden: Arc>> = Default::default(); + let mut eb = Engine::new( + Lagging { + inner: folder(t.loc.path()), + hidden: hidden.clone(), + }, + t.b.clone(), + t.b.dir.path(), + settings(B_ID, "MacBook Air", false, t.loc.path()), + ); + let up_a = t.ea.round().await.unwrap().uploaded.unwrap(); + hidden.lock().unwrap().insert(format!("{up_a}.twconfig")); + eb.round().await.unwrap(); + hidden.lock().unwrap().clear(); + assert!(versions(t.loc.path()).contains(&parent)); + // A 合并两个头之后：共同的上一版本还在，A 的合并没有冲突 + t.a.edit(|s| s.replace(" models: [m1]\n", " models: [m1, m2]\n")); + let r = t.ea.round().await.unwrap(); + assert!(r.conflicts.is_empty(), "{:?}", r.conflicts); + assert_eq!(t.a.provider("relay").unwrap().max_concurrent, Some(3)); +} + +/// 云盘上两台几乎同时开启加密同步：B 那一刻还没收到 A 的 keyring 和第一个版本，于是新建， +/// 覆盖了 A 的 keyring。A 自己的版本这时解不开了：A 照两方比较合并、以新的 keyring 重新上传， +/// 删掉自己那个解不开的版本 —— 两台的配置都在 +#[tokio::test] +async fn a_keyring_race_on_a_cloud_folder_loses_nothing() { + tw_sync::keyring::set_work_factor_for_tests(4); + let loc = tempfile::tempdir().unwrap(); + let (a, b) = (Dev::new('a'), Dev::new('b')); + a.edit(|s| add_provider(s, "one", "https://one.example", "sk-1")); + b.edit(|s| add_provider(s, "two", "https://two.example", "sk-2")); + let hidden: Arc>> = Default::default(); + let mut ea = Engine::new( + folder(loc.path()), + a.clone(), + a.dir.path(), + settings(A_ID, "Mac mini", true, loc.path()), + ); + let mut eb = Engine::new( + Racy { + inner: folder(loc.path()), + hidden: hidden.clone(), + }, + b.clone(), + b.dir.path(), + settings(B_ID, "MacBook Air", true, loc.path()), + ); + assert!(ea.create().await.unwrap()); + hidden.lock().unwrap().extend([ + "keyring".to_string(), + "0000000001-aaaaaaaa.twconfig".to_string(), + ]); + assert!(eb.create().await.unwrap(), "B 没看到 A 的，自己新建"); + hidden.lock().unwrap().clear(); + let r = ea.round().await.unwrap(); + assert!(r.uploaded.is_some()); + assert!(a.provider("one").is_some(), "A 自己的上游还在"); + assert!(a.provider("two").is_some()); + assert!( + !versions(loc.path()).contains(&"0000000001-aaaaaaaa.twconfig".to_string()), + "解不开的那个自己的版本删了" + ); + eb.round().await.unwrap(); + assert!(b.provider("one").is_some()); + assert!(b.provider("two").is_some()); +} + +/// 两台几乎同时开启不加密的同步：两个不依据别的版本的第一个版本，按两方比较合并 +#[tokio::test] +async fn two_first_versions_are_merged_two_way() { + let loc = tempfile::tempdir().unwrap(); + let (a, b) = (Dev::new('a'), Dev::new('b')); + a.edit(|s| add_provider(s, "one", "https://one.example", "sk-1")); + b.edit(|s| add_provider(s, "two", "https://two.example", "sk-2")); + let hidden: Arc>> = Default::default(); + let mut ea = Engine::new( + folder(loc.path()), + a.clone(), + a.dir.path(), + settings(A_ID, "Mac mini", false, loc.path()), + ); + let mut eb = Engine::new( + Lagging { + inner: folder(loc.path()), + hidden: hidden.clone(), + }, + b.clone(), + b.dir.path(), + settings(B_ID, "MacBook Air", false, loc.path()), + ); + assert!(ea.create().await.unwrap()); + hidden + .lock() + .unwrap() + .insert("0000000001-aaaaaaaa.twconfig".to_string()); + assert!(eb.create().await.unwrap()); + hidden.lock().unwrap().clear(); + let r = ea.round().await.unwrap(); + assert!(r.conflicts.is_empty(), "{:?}", r.conflicts); + assert!(a.provider("two").is_some()); + eb.round().await.unwrap(); + assert!(b.provider("one").is_some()); +} + +/// 开启加密时 keyring 建好了、第一个版本没传上去：开启（口令对）接着传，加入也一样 +#[tokio::test] +async fn a_location_with_only_a_keyring_can_be_turned_on() { + tw_sync::keyring::set_work_factor_for_tests(4); + for join in [false, true] { + let loc = tempfile::tempdir().unwrap(); + let a = Dev::new('a'); + a.edit(|s| add_provider(s, "one", "https://one.example", "sk-1")); + folder(loc.path()) + .create_exclusive("keyring", &Keyring::generate(0).seal(PASS)) + .await + .unwrap(); + let mut ea = Engine::new( + folder(loc.path()), + a.clone(), + a.dir.path(), + settings(A_ID, "Mac mini", true, loc.path()), + ); + if join { + assert!(ea.join(None).await.unwrap().encrypted); + } else { + assert!(ea.create().await.unwrap()); + } + assert_eq!(versions(loc.path()).len(), 1); + let (b, bdir) = (Dev::new('b'), tempfile::tempdir().unwrap()); + let mut eb = Engine::new( + folder(loc.path()), + b.clone(), + bdir.path(), + settings(B_ID, "MacBook Air", true, loc.path()), + ); + eb.join(None).await.unwrap(); + assert!(b.provider("one").is_some()); + } +} + +// ─────────────────────────────────────────────── 读不出来的文件 + +/// 最新一层有一个坏文件（不是配置包）：马上跳过，计数；本机改动照常上传，别的设备照常收到 +#[tokio::test] +async fn a_broken_file_on_top_does_not_stop_sync() { + let mut t = bed(true).await; + std::fs::write( + t.loc.path().join("versions/9999999999-cccccccc.twconfig"), + b"garbage", + ) + .unwrap(); + t.a.edit(|s| add_provider(s, "one", "https://one.example", "sk-1")); + let r = t.ea.round().await.unwrap(); + assert!(r.uploaded.is_some()); + assert_eq!(r.skipped, 1); + let r = t.eb.round().await.unwrap(); + assert_eq!(r.skipped, 1); + assert!(t.b.provider("one").is_some()); +} + +/// 最新一层有一个看上去没传完的版本：等着（不合并一半）；等了 30 分钟还读不出来就跳过 +#[tokio::test] +async fn a_version_that_never_finishes_arriving_is_skipped_after_a_while() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider(s, "one", "https://one.example", "sk-1")); + let up = t.ea.round().await.unwrap().uploaded.unwrap(); + // B 那边：最新的那个只到了一半 + let bytes = read_version(t.loc.path(), &up); + let parked = tempfile::tempdir().unwrap(); + std::fs::write(parked.path().join("whole"), &bytes).unwrap(); + let seq: u64 = up[..10].parse().unwrap(); + let half = format!("{:010}-cccccccc.twconfig", seq + 1); + std::fs::write( + t.loc.path().join("versions").join(&half), + &bytes[..bytes.len() / 2], + ) + .unwrap(); + assert!(matches!(t.eb.round().await, Err(RoundError::NotArrived))); + assert!(t.b.provider("one").is_none(), "不拿一半去合并"); + age_state(&t.eb, SKIP_AFTER_MS + 1000); + let r = t.eb.round().await.unwrap(); + assert_eq!(r.skipped, 1); + assert!(t.b.provider("one").is_some()); +} + +/// iCloud 还没下载的最新版本（只有占位文件）：当作尚未到达 +#[tokio::test] +async fn an_icloud_placeholder_on_top_is_waited_for() { + let mut t = bed(false).await; + t.a.edit(|s| add_provider(s, "one", "https://one.example", "sk-1")); + let up = t.ea.round().await.unwrap().uploaded.unwrap(); + let file = format!("{up}.twconfig"); + let real = t.loc.path().join("versions").join(&file); + let parked = tempfile::tempdir().unwrap(); + std::fs::rename(&real, parked.path().join(&file)).unwrap(); + std::fs::write( + t.loc + .path() + .join("versions") + .join(format!(".{file}.icloud")), + b"placeholder", + ) + .unwrap(); + assert!(matches!(t.eb.round().await, Err(RoundError::NotArrived))); + std::fs::rename(parked.path().join(&file), &real).unwrap(); + t.eb.round().await.unwrap(); + assert!(t.b.provider("one").is_some()); +} + +// ─────────────────────────────────────────────── 空的同步位置 + +/// 文件夹位置的根目录还在、里面暂时是空的（网络盘没挂上时挂载点是个空目录）：不当作清空， +/// 挂回来之后照常同步 +#[tokio::test] +async fn an_empty_mount_point_is_not_taken_for_a_purge() { + let mut t = bed(false).await; + let parked = tempfile::tempdir().unwrap(); + for name in ["versions", "devices"] { + std::fs::rename(t.loc.path().join(name), parked.path().join(name)).unwrap(); + } + for _ in 0..5 { + assert!(matches!(t.ea.round().await, Err(RoundError::LooksEmpty))); + } + // 过了很久也一样：自己的设备记录不在只是没挂上的一种样子，结论要人下 + age_state(&t.ea, 60 * 60 * 1000); + assert!(matches!(t.ea.round().await, Err(RoundError::Emptied))); + assert!(t.ea.local().base().is_some(), "本机的同步状态不删"); + for name in ["versions", "devices"] { + std::fs::rename(parked.path().join(name), t.loc.path().join(name)).unwrap(); + } + t.a.edit(|s| add_provider(s, "back", "https://back.example", "sk-b")); + assert!(t.ea.round().await.unwrap().uploaded.is_some()); + t.eb.round().await.unwrap(); + assert!(t.b.provider("back").is_some()); +} + +/// 停止同步并删除同步数据：只删认得出的同步文件，用户自己放进去的不动 +#[tokio::test] +async fn purging_leaves_files_that_are_not_sync_data() { + let mut t = bed(false).await; + std::fs::write(t.loc.path().join("devices/notes.txt"), b"mine").unwrap(); + std::fs::write(t.loc.path().join("versions/readme.md"), b"mine").unwrap(); + t.ea.stop(true).await.unwrap(); + assert!(versions(t.loc.path()).iter().all(|n| n == "readme.md")); + assert!(t.loc.path().join("devices/notes.txt").exists()); + assert!(t.loc.path().join("versions/readme.md").exists()); + assert!(!t.loc.path().join("devices/bbbbbbbb").exists()); +} + +// ─────────────────────────────────────────────── 测试用的存储 + +/// 云盘晚到：藏起来的文件列不出、读不到 +struct Lagging { + inner: AnyStore, + hidden: Arc>>, +} + +impl Lagging { + fn hides(&self, path: &str) -> bool { + self.hidden + .lock() + .unwrap() + .iter() + .any(|h| path.ends_with(h.as_str())) + } +} + +impl Store for Lagging { + async fn list(&self, dir: &str) -> Result, StoreError> { + let hidden = self.hidden.lock().unwrap().clone(); + Ok(self + .inner + .list(dir) + .await? + .into_iter() + .filter(|e| !hidden.contains(&e.name)) + .collect()) + } + async fn get(&self, path: &str) -> Result>, StoreError> { + if self.hides(path) { + return Ok(None); + } + self.inner.get(path).await + } + async fn put_new(&self, path: &str, bytes: &[u8]) -> Result<(), StoreError> { + self.inner.put_new(path, bytes).await + } + async fn put(&self, path: &str, bytes: &[u8]) -> Result<(), StoreError> { + self.inner.put(path, bytes).await + } + async fn delete(&self, path: &str) -> Result<(), StoreError> { + self.inner.delete(path).await + } + async fn create_exclusive(&self, path: &str, bytes: &[u8]) -> Result { + self.inner.create_exclusive(path, bytes).await + } +} + +/// 云盘上的「仅当不存在时创建」管不了别的设备：藏起来的文件当作不存在，照写不误 +struct Racy { + inner: AnyStore, + hidden: Arc>>, +} + +impl Racy { + fn hides(&self, path: &str) -> bool { + self.hidden + .lock() + .unwrap() + .iter() + .any(|h| path.ends_with(h.as_str())) + } +} + +impl Store for Racy { + async fn list(&self, dir: &str) -> Result, StoreError> { + let hidden = self.hidden.lock().unwrap().clone(); + Ok(self + .inner + .list(dir) + .await? + .into_iter() + .filter(|e| !hidden.contains(&e.name)) + .collect()) + } + async fn get(&self, path: &str) -> Result>, StoreError> { + if self.hides(path) { + return Ok(None); + } + self.inner.get(path).await + } + async fn put_new(&self, path: &str, bytes: &[u8]) -> Result<(), StoreError> { + self.inner.put_new(path, bytes).await + } + async fn put(&self, path: &str, bytes: &[u8]) -> Result<(), StoreError> { + self.inner.put(path, bytes).await + } + async fn delete(&self, path: &str) -> Result<(), StoreError> { + self.inner.delete(path).await + } + async fn create_exclusive(&self, path: &str, bytes: &[u8]) -> Result { + if self.hides(path) { + self.inner.put(path, bytes).await?; + return Ok(true); + } + self.inner.create_exclusive(path, bytes).await + } +} diff --git a/crates/tw-sync/tests/stores.rs b/crates/tw-sync/tests/stores.rs index a26be7a4..cd86f9f9 100644 --- a/crates/tw-sync/tests/stores.rs +++ b/crates/tw-sync/tests/stores.rs @@ -117,6 +117,10 @@ async fn webdav_without_move_or_conditional_put_still_never_overwrites_a_version } fn s3_store(fake: &common::Fake, ak: &str) -> S3 { + s3_store_with(fake, ak, common::SK) +} + +fn s3_store_with(fake: &common::Fake, ak: &str, sk: &str) -> S3 { S3::new( store::s3::Config { endpoint: fake.url(), @@ -124,13 +128,119 @@ fn s3_store(fake: &common::Fake, ak: &str) -> S3 { bucket: "tw-bucket".into(), prefix: "team/tw".into(), access_key_id: ak.into(), - secret_access_key: common::SK.into(), + secret_access_key: sk.into(), }, common::http(), ) .unwrap() } +/// 假 S3 核对签名的那一份按 AWS 文档里的两个示例对得上（GET Object、ListObjects 带查询参数）： +/// 它核对得了，`S3` 的签名在它那里过得去才算数 +#[test] +fn the_fake_s3_checks_signatures_the_way_aws_does() { + use axum::http::{HeaderMap, HeaderValue}; + let secret = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; + let empty = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + let req = |auth: &str, range: bool| { + let mut h = HeaderMap::new(); + h.insert( + "host", + HeaderValue::from_static("examplebucket.s3.amazonaws.com"), + ); + if range { + h.insert("range", HeaderValue::from_static("bytes=0-9")); + } + h.insert("x-amz-content-sha256", HeaderValue::from_static(empty)); + h.insert("x-amz-date", HeaderValue::from_static("20130524T000000Z")); + h.insert("authorization", HeaderValue::from_str(auth).unwrap()); + h + }; + let get = req( + "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,\ + SignedHeaders=host;range;x-amz-content-sha256;x-amz-date,\ + Signature=f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41", + true, + ); + common::sigv4( + "GET", + "/test.txt", + "", + &get, + b"", + "AKIAIOSFODNN7EXAMPLE", + secret, + ) + .unwrap(); + let list = req( + "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,\ + SignedHeaders=host;x-amz-content-sha256;x-amz-date,\ + Signature=34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7", + false, + ); + common::sigv4( + "GET", + "/", + "max-keys=2&prefix=J", + &list, + b"", + "AKIAIOSFODNN7EXAMPLE", + secret, + ) + .unwrap(); + // 改一点就不认 + assert!( + common::sigv4( + "GET", + "/", + "max-keys=3&prefix=J", + &list, + b"", + "AKIAIOSFODNN7EXAMPLE", + secret + ) + .is_err() + ); + assert!( + common::sigv4( + "GET", + "/test.txt", + "", + &get, + b"", + "AKIAIOSFODNN7EXAMPLE", + "other" + ) + .is_err() + ); + assert!( + common::sigv4( + "GET", + "/test.txt", + "", + &get, + b"body", + "AKIAIOSFODNN7EXAMPLE", + secret + ) + .is_err() + ); +} + +/// 密钥 ID 对、密钥不对：签名对不上，拒绝访问 +#[tokio::test] +async fn s3_with_the_wrong_secret_is_refused() { + let fake = common::s3().await; + assert!(matches!( + s3_store_with(&fake, common::AK, "not-the-secret") + .list("versions") + .await + .unwrap_err(), + StoreError::AuthFailed(_) + )); + assert!(fake.names().is_empty()); +} + #[tokio::test] async fn s3_answers_like_the_others() { let fake = common::s3().await; @@ -196,3 +306,62 @@ async fn wrong_credentials_are_a_refusal_and_a_dead_server_is_unreachable() { StoreError::Unreachable(_) )); } + +/// 服务端关掉了连接池里那条空闲连接（Apache 空闲 5 秒就关）：发出去的请求收到「连接在答复 +/// 之前关了」，换一条连接再发一次就好。这里的服务端每条连接只答第一个请求，第二个请求一到 +/// 就关连接 —— 没有重发的话第二次读一定失败 +#[tokio::test] +async fn a_pooled_connection_the_server_closed_is_retried_once() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = l.local_addr().unwrap(); + let conns = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let n = conns.clone(); + tokio::spawn(async move { + loop { + let Ok((mut sock, _)) = l.accept().await else { + return; + }; + n.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + tokio::spawn(async move { + let mut buf = Vec::new(); + let mut served = 0; + loop { + let mut chunk = [0u8; 4096]; + let Ok(got) = sock.read(&mut chunk).await else { + return; + }; + if got == 0 { + return; + } + buf.extend_from_slice(&chunk[..got]); + while let Some(end) = buf.windows(4).position(|w| w == b"\r\n\r\n") { + buf.drain(..end + 4); + if served == 1 { + // 第二个请求：一个字也不答就关 + return; + } + served += 1; + let _ = sock + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\nhello") + .await; + } + } + }); + } + }); + let s = WebDav::new( + &format!("http://{addr}/dav/root"), + common::USER, + common::PASS, + common::http(), + ) + .unwrap(); + for _ in 0..3 { + assert_eq!(s.get(V1).await.unwrap().as_deref(), Some(&b"hello"[..])); + } + assert!( + conns.load(std::sync::atomic::Ordering::Relaxed) >= 3, + "每次都换了一条连接" + ); +} diff --git a/docs/config.md b/docs/config.md index 5ac3653a..63b3b118 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1518,7 +1518,15 @@ resolve conflicts in the app (Settings, Sync and backup) or with What syncs is the configuration a configuration pack carries: upstreams, proxies, gateway keys, aliases, groups, routes, price sheets, security rules, plugins and the general settings, keys included. `listen`, `retention` and -`sync` itself stay on this device, and so does a ChatGPT account's sign-in. +`sync` itself stay on this device, and so do a ChatGPT account's sign-in and +OAuth tokens. An upstream signed in on another device stays turned off on this +one until it is signed in here; that does not turn it off anywhere else. + +With encryption on, versions that are not encrypted are never read. A keyring +that went missing, or an older one put back, does not turn encryption off: the +app asks whether to put back the copy this device keeps. A location that reads +as empty is checked a few more times before the app says so, and turning sync +off is still up to you. @@ -1528,7 +1536,7 @@ plugins and the general settings, keys included. `listen`, `retention` and | `device_id` | string | **required** | Generated when sync is turned on. File names at the sync location show its first 8 characters. | | `device_name` | string | **required** | Shown in the device list of every device that syncs here. | | `encrypt` | bool | `true` | Encrypt what goes to the sync location with the sync passphrase. Every device at one location uses the same setting; change it in the app, which re-uploads the latest version. | -| `passphrase` | string | — | The sync passphrase; required when `encrypt` is on. Kept here in the clear, like the other keys in this file. | +| `passphrase` | string | — | The sync passphrase; required when `encrypt` is on, and kept after encryption is turned off: the keyring that says it is off is checked with it. Kept here in the clear, like the other keys in this file. | | `exclude` | list of strings | `[]` | Upstreams, by name, that are not synced: they are not uploaded and another device's upstream of the same name does not replace them. Routes, aliases and groups that refer to them are not synced either. | | `target` | object, [`sync.target`](#cfg-sync-target) | **required** | The sync location. The keyring, `versions/` and `devices/` live directly in it. | @@ -1560,7 +1568,7 @@ plugins and the general settings, keys included. `listen`, `retention` and | `url` | string | **required** | The WebDAV address of the folder. `https://`, with the certificate checked; `http://` only for this machine and the local network. | | `user` | string | `''` | User name. | | `pass` | string | `''` | Password, or the app password the service issues for WebDAV. | -| `proxy` | string | `direct` | Where requests to the sync location go out: `direct`, `system` (the system proxy), or the name of a proxy under `proxies`. | +| `proxy` | string | `direct` | Where requests to the sync location go out: `direct`, `system` (the system proxy), or the name of a proxy under `proxies`. An `http://` address on this machine or the local network has to be `direct`. | @@ -1572,10 +1580,10 @@ plugins and the general settings, keys included. `listen`, `retention` and | `endpoint` | string | **required** | The S3 endpoint. Hosts ending in `amazonaws.com`, `aliyuncs.com` or `myqcloud.com` put the bucket in the host name; every other endpoint (Cloudflare R2, MinIO, an address) puts it in the path. `http://` only for this machine and the local network. | | `region` | string | **required** | The signing region; `auto` for Cloudflare R2. | | `bucket` | string | **required** | The bucket. Keep it private. | -| `prefix` | string | `''` | A folder inside the bucket. | +| `prefix` | string | `''` | A folder inside the bucket. No `..` parts, and not starting with `/`. | | `access_key_id` | string | **required** | Access key ID. | | `secret_access_key` | string | **required** | Secret access key. | -| `proxy` | string | `direct` | Where requests to the sync location go out: `direct`, `system` (the system proxy), or the name of a proxy under `proxies`. | +| `proxy` | string | `direct` | Where requests to the sync location go out: `direct`, `system` (the system proxy), or the name of a proxy under `proxies`. An `http://` address on this machine or the local network has to be `direct`. | ```yaml diff --git a/docs/config.zh-CN.md b/docs/config.zh-CN.md index 9a471081..930fa9c0 100644 --- a/docs/config.zh-CN.md +++ b/docs/config.zh-CN.md @@ -1105,7 +1105,9 @@ export const manifest = { 多台设备（这台 Mac、一台笔记本、一台运行远程 core 的服务器）可以通过自选的同步位置保持同一份配置：由云盘在设备之间传送的文件夹、WebDAV 服务器或 S3 兼容存储。每次改动都按对象与其他设备自上次同步以来的改动合并；只有两台设备把同一对象的同一部分改成不同内容时才构成冲突，冲突在每台设备上显示，任一设备处理之后随之清除。开启同步、加入已有的同步和处理冲突在应用里进行（设置 → 同步与备份），服务器上用 `twcore sync enable`；本节记录的是结果。 -同步的内容即配置包带的内容：上游、出站代理、网关密钥、别名、分组、路由、价目表、安全规则、插件和全局设置，包括密钥。`listen`、`retention` 和 `sync` 本身留在本机，ChatGPT 账号的登录也一样。 +同步的内容即配置包带的内容：上游、出站代理、网关密钥、别名、分组、路由、价目表、安全规则、插件和全局设置，包括密钥。`listen`、`retention` 和 `sync` 本身留在本机，ChatGPT 账号的登录、OAuth 的令牌也一样。别的设备上登录着的上游在这台设备上停用着，在这台登录之后才启用；这不会让它在别的设备上停用。 + +开着加密时，不加密的版本一概不读。keyring 不见了、被换成更早的那一份，都不会关掉加密：应用会问要不要放回这台设备记着的那一份。同步位置读起来是空的时，要再看几次才提示，停不停止同步由你决定。 @@ -1115,7 +1117,7 @@ export const manifest = { | `device_id` | 字符串 | **必填** | 开启同步时生成。同步位置上的文件名里出现的是它的前 8 个字符。 | | `device_name` | 字符串 | **必填** | 在每台参与同步的设备的设备列表里显示的名字。 | | `encrypt` | 布尔 | `true` | 以同步口令加密存到同步位置的内容。同一同步位置上的所有设备采用同一设置；在应用里更改，应用会重新上传最新版本。 | -| `passphrase` | 字符串 | — | 同步口令，`encrypt` 开着时必填。和本文件里的其他密钥一样以明文保存。 | +| `passphrase` | 字符串 | — | 同步口令，`encrypt` 开着时必填；关闭加密之后也留着：同步位置上写着「已关闭」的 keyring 要用它核对。和本文件里的其他密钥一样以明文保存。 | | `exclude` | 字符串列表 | `[]` | 不同步的上游，按名字：不上传，也不被其他设备的同名上游覆盖。引用它们的路由、别名、分组随之不同步。 | | `target` | 对象，见 [`sync.target`](#cfg-sync-target) | **必填** | 同步位置。`keyring`、`versions/`、`devices/` 直接放在它里面。 | @@ -1147,7 +1149,7 @@ export const manifest = { | `url` | 字符串 | **必填** | 文件夹的 WebDAV 地址。`https://`，校验证书；`http://` 只用于本机和局域网。 | | `user` | 字符串 | `''` | 用户名。 | | `pass` | 字符串 | `''` | 密码，或者服务为 WebDAV 签发的应用密码。 | -| `proxy` | 字符串 | `direct` | 访问同步位置的请求从哪里出去：`direct`（直连）、`system`（跟随系统代理），或者 `proxies` 里一个代理的名字。 | +| `proxy` | 字符串 | `direct` | 访问同步位置的请求从哪里出去：`direct`（直连）、`system`（跟随系统代理），或者 `proxies` 里一个代理的名字。本机、内网的 `http://` 地址只能是 `direct`。 | @@ -1159,10 +1161,10 @@ export const manifest = { | `endpoint` | 字符串 | **必填** | S3 端点。主机以 `amazonaws.com`、`aliyuncs.com`、`myqcloud.com` 结尾的，存储桶名放进域名；其余端点（Cloudflare R2、MinIO、IP 地址）放进路径。`http://` 只用于本机和局域网。 | | `region` | 字符串 | **必填** | 签名用的区域；Cloudflare R2 填 `auto`。 | | `bucket` | 字符串 | **必填** | 存储桶，建议设为私有。 | -| `prefix` | 字符串 | `''` | 存储桶里的一个前缀（文件夹）。 | +| `prefix` | 字符串 | `''` | 存储桶里的一个前缀（文件夹）。不能有 `..`，不以 `/` 开头。 | | `access_key_id` | 字符串 | **必填** | Access Key ID。 | | `secret_access_key` | 字符串 | **必填** | Secret Access Key。 | -| `proxy` | 字符串 | `direct` | 访问同步位置的请求从哪里出去：`direct`（直连）、`system`（跟随系统代理），或者 `proxies` 里一个代理的名字。 | +| `proxy` | 字符串 | `direct` | 访问同步位置的请求从哪里出去：`direct`（直连）、`system`（跟随系统代理），或者 `proxies` 里一个代理的名字。本机、内网的 `http://` 地址只能是 `direct`。 | ```yaml From 6d232904349235b82ab761207a3e1493af80280d Mon Sep 17 00:00:00 2001 From: fylorn <249551762+fylorn@users.noreply.github.com> Date: Mon, 19 Oct 2026 18:22:42 +0800 Subject: [PATCH 08/13] Sync: joining adopts the sync data's default key; CLI asks per item; iCloud placeholders download (#358) Default gateway key on join - Every installation generates a default gateway key. When the key that default_key points to has the same name on both sides and a different value, joining is not a conflict: the joining device takes the sync data's value, in both join modes (a pinned default key is no longer kept over it on use_remote), and GatewayKeysChanged follows so taken-over clients are updated. Decisions given for that item are ignored. - POST /sync/enable/preview lists it in the new SyncJoinPreview.auto ({ item, action: overwrite, reason: default_key }); the row itself stays in rows. auto is filled for initial configurations too. twcore sync enable - Merging asks about each item that differs; the default is "use the sync data", "keep this device's version" says it replaces the other devices' one, and "keep both" is offered where a new name is suggested. Before, pressing Enter skipped every conflict, which pushed this device's values (its default key among them) onto the other devices. - The device name defaults to the computer name: ComputerName (then LocalHostName) on macOS, the host name on Linux, COMPUTERNAME on Windows. iCloud placeholders (macOS 13 and earlier) - Reading the real name of a legacy .name.icloud placeholder does not download it. The folder location now asks iCloud to download it through NSFileManager startDownloadingUbiquitousItemAtURL:error:, called from the system osascript (JavaScript for Automation) with the path as an argument, at most every 5 minutes per file. - A placeholder (an empty read) is "arriving", not "unreadable": it does not count toward the 30 minutes after which unreadable files are skipped, and is skipped only after a day so it cannot hold uploads back forever. The keyring gets the same treatment. Co-authored-by: Claude Opus 5.5 --- bin/twcore/src/sync_cmd.rs | 160 +++++++++++++++++--- crates/tw-api/src/lib.rs | 3 +- crates/tw-api/src/sync.rs | 25 ++- crates/tw-control/src/sync.rs | 62 ++++++-- crates/tw-control/tests/sync_devices.rs | 10 +- crates/tw-control/tests/sync_regressions.rs | 104 +++++++++++++ crates/tw-sync/src/engine.rs | 74 ++++++++- crates/tw-sync/src/local.rs | 4 + crates/tw-sync/src/store/folder.rs | 83 +++++++++- crates/tw-sync/tests/regressions.rs | 35 ++++- docs/config.md | 5 + docs/config.zh-CN.md | 2 + 12 files changed, 520 insertions(+), 47 deletions(-) diff --git a/bin/twcore/src/sync_cmd.rs b/bin/twcore/src/sync_cmd.rs index b9965132..a38cb948 100644 --- a/bin/twcore/src/sync_cmd.rs +++ b/bin/twcore/src/sync_cmd.rs @@ -179,19 +179,94 @@ pub fn disable(config: &Path, purge: bool) -> Result<()> { Ok(()) } +/// 一段机器名能不能当设备名：去掉首尾空白，不能有控制字符，太长的截短 +fn usable_name(raw: &str) -> Option { + let n = raw.trim(); + if n.is_empty() || n.chars().any(char::is_control) { + return None; + } + Some(n.chars().take(tw_config::sync::DEVICE_NAME_MAX).collect()) +} + +/// 跑一个系统自带的命令，取它输出的第一行 +#[cfg(unix)] +fn command_line(program: &str, args: &[&str]) -> Option { + let out = std::process::Command::new(program) + .args(args) + .stdin(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .output() + .ok()?; + if !out.status.success() { + return None; + } + usable_name(String::from_utf8_lossy(&out.stdout).lines().next()?) +} + +/// 这台计算机的名称，当设备名称的缺省值：macOS 是系统设置里的「电脑名称」（ComputerName， +/// 取不到用 LocalHostName），Linux 是主机名，Windows 是计算机名 fn host_name() -> String { - for var in ["COMPUTERNAME", "HOSTNAME"] { - if let Ok(v) = std::env::var(var) - && !v.trim().is_empty() - { - return v.trim().to_string(); + #[cfg(target_os = "macos")] + let found = command_line("/usr/sbin/scutil", &["--get", "ComputerName"]) + .or_else(|| command_line("/usr/sbin/scutil", &["--get", "LocalHostName"])) + .or_else(|| command_line("/bin/hostname", &[])); + #[cfg(windows)] + let found = std::env::var("COMPUTERNAME") + .ok() + .and_then(|v| usable_name(&v)); + #[cfg(all(unix, not(target_os = "macos")))] + let found = ["/etc/hostname", "/proc/sys/kernel/hostname"] + .iter() + .find_map(|p| { + std::fs::read_to_string(p) + .ok() + .and_then(|s| usable_name(&s)) + }) + .or_else(|| command_line("hostname", &[])) + .or_else(|| std::env::var("HOSTNAME").ok().and_then(|v| usable_name(&v))); + #[cfg(not(any(unix, windows)))] + let found: Option = None; + found.unwrap_or_else(|| "server".into()) +} + +/// 合并加入时，同步数据里和本机不一样的一项怎么办。**缺省是采用同步数据**：留本机的那一份 +/// 会随这次加入上传，把别的设备上的那一项也改成本机的 —— 那要明确选 +fn decide(row: &tw_api::ImportRow) -> Result { + let differs = if row.differs.is_empty() { + String::new() + } else { + format!(" ({})", row.differs.join(", ")) + }; + let what = match row.item.name.as_str() { + "" => row.item.kind.slug().to_string(), + name => format!("{} `{name}`", row.item.kind.slug()), + }; + eprintln!("{what}{differs} differs between this device and the sync data:"); + eprintln!(" u use the sync data (default)"); + eprintln!( + " k keep this device's version; it is uploaded and replaces the other devices' one" + ); + let rename = row.suggested_name.clone().filter(|_| !row.global); + if let Some(n) = &rename { + eprintln!(" r keep both: the sync data's one is added as {n}"); + } + loop { + let a = ask("Choice", Some("u"))?; + let action = match a.to_ascii_lowercase().as_str() { + "u" => "overwrite", + "k" => "skip", + "r" if rename.is_some() => "rename", + _ => { + eprintln!("type u, k{}", if rename.is_some() { " or r" } else { "" }); + continue; + } + }; + let mut d = serde_json::json!({ "item": row.item, "action": action }); + if action == "rename" { + d["new_name"] = rename.clone().into(); } + return Ok(d); } - std::fs::read_to_string("/etc/hostname") - .map(|s| s.trim().to_string()) - .ok() - .filter(|s| !s.is_empty()) - .unwrap_or_else(|| "server".into()) } pub fn enable(config: &Path) -> Result<()> { @@ -275,11 +350,23 @@ pub fn enable(config: &Path) -> Result<()> { "/sync/enable/preview", Some(serde_json::json!({ "target": target, "passphrase": body.get("passphrase") })), )?; + for a in &preview.auto { + match a.reason { + tw_api::SyncJoinAutoReason::DefaultKey => eprintln!( + "The default gateway key ({}) takes the sync data's value; clients set up \ + on this device are updated to it", + a.item.name + ), + } + } if !preview.initial { + let auto: Vec<&tw_api::ItemRef> = preview.auto.iter().map(|a| &a.item).collect(); let conflicts: Vec<&tw_api::ImportRow> = preview .rows .iter() - .filter(|r| r.status == tw_api::ImportStatus::Conflict) + .filter(|r| { + r.status == tw_api::ImportStatus::Conflict && !auto.contains(&&r.item) + }) .collect(); let new = preview .rows @@ -292,16 +379,17 @@ pub fn enable(config: &Path) -> Result<()> { conflicts.len() ); let replace = yes( - "Replace this device's configuration with the sync data (no keeps this \ - device's version of every item that differs)", + "Replace this device's configuration with the sync data (no merges them; \ + each item that differs is asked)", false, )?; if !replace { body["join"] = "merge".into(); - body["decisions"] = conflicts - .iter() - .map(|r| serde_json::json!({ "item": r.item, "action": "skip" })) - .collect(); + let mut decisions = Vec::new(); + for r in &conflicts { + decisions.push(decide(r)?); + } + body["decisions"] = decisions.into(); } } } @@ -311,3 +399,41 @@ pub fn enable(config: &Path) -> Result<()> { print_view(&v); Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_machine_name_is_trimmed_and_cut_to_a_device_name() { + assert_eq!(usable_name(" Mac mini \n").as_deref(), Some("Mac mini")); + assert_eq!(usable_name(" "), None); + assert_eq!(usable_name("a\u{7}b"), None); + let long = "名".repeat(100); + assert_eq!( + usable_name(&long).unwrap().chars().count(), + tw_config::sync::DEVICE_NAME_MAX + ); + } + + /// macOS 上缺省是「电脑名称」（ComputerName），不是 `server` + #[cfg(target_os = "macos")] + #[test] + fn on_macos_the_default_device_name_is_the_computer_name() { + let want = command_line("/usr/sbin/scutil", &["--get", "ComputerName"]) + .or_else(|| command_line("/usr/sbin/scutil", &["--get", "LocalHostName"])); + let got = host_name(); + assert_ne!(got, "server"); + if let Some(w) = want { + assert_eq!(got, w); + } + } + + /// 别的 Unix 上是主机名 + #[cfg(all(unix, not(target_os = "macos")))] + #[test] + fn elsewhere_the_default_device_name_is_the_host_name() { + let got = host_name(); + assert!(!got.is_empty()); + } +} diff --git a/crates/tw-api/src/lib.rs b/crates/tw-api/src/lib.rs index 16bd6778..cef7b79d 100644 --- a/crates/tw-api/src/lib.rs +++ b/crates/tw-api/src/lib.rs @@ -993,7 +993,8 @@ pub const MSG_CODES: &str = include_str!("../msg-codes.txt"); /// `POST /sync/disable`（[`SyncDisable`]）、`POST /sync/republish`（同步位置空了、keyring 不见了 /// 之后由人选的「重新上传本机配置」，→ [`SyncView`]）。[`SyncState`] 里要人处理的有 /// `keyring_missing`、`emptied`、`misconfigured`，[`SyncView`] 的 `skipped` 是同步位置上跳过了的 -/// 文件数，[`SyncConflict`] 的 `options` 是能选的处理。状态变了发 [`Event::SyncStatus`]；应用别的设备的 +/// 文件数，[`SyncConflict`] 的 `options` 是能选的处理，[`SyncJoinPreview`] 的 `auto` 是加入时 +/// 不用选的项（两边的默认网关密钥同名、值不同时采用同步数据的值）。状态变了发 [`Event::SyncStatus`]；应用别的设备的 /// 版本时配置的来源是 `sync`，改了网关密钥时发 `GatewayKeysChanged`。[`ProviderView`] 多了 /// `sync_excluded`（不同步的上游）和 `sync_url_changed`（地址经同步变更，改之前的地址）。 /// 同步的码是 `sync.*`。安全模式下不自动同步（`paused`） diff --git a/crates/tw-api/src/sync.rs b/crates/tw-api/src/sync.rs index c7dd837f..1e95e604 100644 --- a/crates/tw-api/src/sync.rs +++ b/crates/tw-api/src/sync.rs @@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize}; -use crate::{ImportDecision, ImportRow, ItemRef, Msg}; +use crate::{ImportAction, ImportDecision, ImportRow, ItemRef, Msg}; fn direct() -> String { "direct".to_string() @@ -179,6 +179,29 @@ pub struct SyncJoinPreview { pub initial: bool, /// 同步数据里的每个项目和本机比（[`ImportRow`]）。`initial` 时是空的 pub rows: Vec, + /// core 替人定了、不用选的那几项（两种加入方式都照它办，`decisions` 里给了也不看）。它们 + /// 在 `rows` 里照常列出，界面那一行显示成定下的处理和原因，不当作冲突 + pub auto: Vec, +} + +/// 加入时不用选的一项（[`SyncJoinPreview::auto`]） +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SyncJoinAuto { + pub item: ItemRef, + /// 怎么处理。现在只有 `overwrite`：采用同步数据 + pub action: ImportAction, + pub reason: SyncJoinAutoReason, +} + +slug_enum! { + /// 为什么不用选。 + pub enum SyncJoinAutoReason { + /// 两边的默认网关密钥（`default_key` 指着的那一把）同名、值不同：每个安装都会自动生成 + /// 一把，加入的设备采用同步数据的值。本机已接管的客户端随之自动更新 + /// （`GatewayKeysChanged`） + DefaultKey = "default_key", + } } slug_enum! { diff --git a/crates/tw-control/src/sync.rs b/crates/tw-control/src/sync.rs index 3a6883b2..e0db8a45 100644 --- a/crates/tw-control/src/sync.rs +++ b/crates/tw-control/src/sync.rs @@ -1158,9 +1158,10 @@ fn is_initial(snap: &Snapshot) -> bool { /// 加入时「合并」：和导入同一套规矩（`tw_pack::import`），本机打底，同步数据逐项并进来。 /// 新增的默认导入，冲突的必须选（覆盖、改名导入、跳过），没选是 `sync.join_undecided`。 /// -/// **两处和导入不一样**：导入进来的插件按导入的规矩是停用的，加入同步时照同步数据里的开关 +/// **三处和导入不一样**：导入进来的插件按导入的规矩是停用的，加入同步时照同步数据里的开关 /// （各设备一致，见方案 6.2；不加密时同步一轮里另有停用的规矩）；导入预览里只有本机有的 -/// 项目不出现，这里也一样 —— 它们会随这次加入上传 +/// 项目不出现，这里也一样 —— 它们会随这次加入上传；两边的默认网关密钥同名、值不同时不用选， +/// 采用同步数据的值（[`engine::adopted_default_key`]，给了别的选择也不看） fn merge_join( local: &Pack, remote: &Pack, @@ -1172,8 +1173,18 @@ fn merge_join( .iter() .map(|d| (crate::backups::item_id(&d.item), d)) .collect(); + let adopt = model::snapshot(local) + .ok() + .and_then(|l| adopted(&l, remote)); let mut out = Vec::new(); for r in &rows { + if adopt.as_ref() == Some(&r.item) { + out.push(tw_pack::import::Decision { + item: r.item.clone(), + action: tw_pack::import::Action::Overwrite, + }); + continue; + } let action = match decided.get(&r.item) { Some(d) => match d.action { tw_api::ImportAction::Import => tw_pack::import::Action::Import, @@ -1340,17 +1351,13 @@ async fn join_preview( let (target, store) = config_target(&s, &req.target)?; let cur = current_pack(&s.cfg).map_err(|m| fail(StatusCode::INTERNAL_SERVER_ERROR, m))?; let local = model::snapshot(&cur.pack).map_err(|e| fail(StatusCode::BAD_REQUEST, e.msg()))?; - if is_initial(&local) { - return Ok(Json(tw_api::SyncJoinPreview { - initial: true, - rows: Vec::new(), - })); - } + let initial = is_initial(&local); let probe = Eng::probe(&store, req.passphrase.as_deref()).await; if probe.existing.is_none() { return Ok(Json(tw_api::SyncJoinPreview { - initial: false, + initial, rows: Vec::new(), + auto: Vec::new(), })); } let encrypted = probe.existing.as_ref().is_some_and(|e| e.encrypted); @@ -1360,16 +1367,49 @@ async fn join_preview( store, &draft(target.clone(), encrypted, req.passphrase.clone(), "preview"), ); - let remote = e.latest().await.map_err(|e| round_fail(&e))?; + let remote = match e.latest().await { + Ok(r) => r, + // 初始配置不出现合并这一步：读不到同步数据（口令还没给）也照样答 + Err(_) if initial => { + return Ok(Json(tw_api::SyncJoinPreview { + initial, + rows: Vec::new(), + auto: Vec::new(), + })); + } + Err(e) => return Err(round_fail(&e)), + }; + let auto = adopted(&local, &remote) + .map(|id| tw_api::SyncJoinAuto { + item: item_ref(&id), + action: tw_api::ImportAction::Overwrite, + reason: tw_api::SyncJoinAutoReason::DefaultKey, + }) + .into_iter() + .collect(); + if initial { + return Ok(Json(tw_api::SyncJoinPreview { + initial, + rows: Vec::new(), + auto, + })); + } // 和导入预览同一套规矩 let rows = tw_pack::import::preview(&cur.pack, &remote, &crate::backups::import_ctx()) .map_err(|e| import_fail(&e))?; Ok(Json(tw_api::SyncJoinPreview { - initial: false, + initial, rows: rows.into_iter().map(crate::backups::row_view).collect(), + auto, })) } +/// 加入时不用选、采用同步数据的那一把默认网关密钥（[`engine::adopted_default_key`]） +fn adopted(local: &Snapshot, remote: &Pack) -> Option { + let remote = model::snapshot(remote).ok()?; + engine::adopted_default_key(local, &remote) +} + async fn enable( State(s): State, Json(req): Json, diff --git a/crates/tw-control/tests/sync_devices.rs b/crates/tw-control/tests/sync_devices.rs index 038dcf7e..de7bdb85 100644 --- a/crates/tw-control/tests/sync_devices.rs +++ b/crates/tw-control/tests/sync_devices.rs @@ -355,8 +355,6 @@ async fn joining_with_a_configuration_merges_by_the_import_rules() { "target": folder(loc.path()), "encrypt": false, "device_name": "B", "join": "merge", "decisions": [ { "item": { "kind": "provider", "name": "shared" }, "action": "rename", "new_name": "shared (A)" }, - // 两边各自生成的默认密钥值不同：留本机的 - { "item": { "kind": "key", "name": "default" }, "action": "skip" }, ], }), ) @@ -369,11 +367,9 @@ async fn joining_with_a_configuration_merges_by_the_import_rules() { assert_eq!(names(&b), ["mine", "shared", "shared (A)"]); a.ok("POST", "/sync/now", Value::Null).await; assert_eq!(names(&a), ["mine", "shared", "shared (A)"], "{}", a.text()); - assert_eq!( - a.config().clients[0].key, - "tw-default-b", - "选了留本机的那一把，A 跟着换" - ); + // 两边各自生成的默认密钥值不同：不用选，B 采用同步数据的值，A 的不变 + assert_eq!(b.config().clients[0].key, "tw-default-a"); + assert_eq!(a.config().clients[0].key, "tw-default-a"); } #[tokio::test] diff --git a/crates/tw-control/tests/sync_regressions.rs b/crates/tw-control/tests/sync_regressions.rs index 5473cf7f..7e8ca29e 100644 --- a/crates/tw-control/tests/sync_regressions.rs +++ b/crates/tw-control/tests/sync_regressions.rs @@ -345,3 +345,107 @@ async fn turning_encryption_off_keeps_the_passphrase() { assert_eq!(v["encrypted"], true, "{v}"); assert!(b.config().sync.unwrap().encrypt); } + +/// 收下 B 上的 `GatewayKeysChanged` 里的名字 +fn keys_changed(rx: &mut tokio::sync::broadcast::Receiver) -> Vec { + let mut out = Vec::new(); + while let Ok(ev) = rx.try_recv() { + if let tw_api::Event::GatewayKeysChanged { names, .. } = ev { + out.extend(names); + } + } + out +} + +/// 已有配置的设备以「合并」加入：两边的默认网关密钥同名、值不同 —— 不当作冲突（不用选，选了 +/// 「跳过」也不看），采用同步数据的值；A 上的那一把不变；B 发 `GatewayKeysChanged` +#[tokio::test] +async fn merging_in_adopts_the_default_key_of_the_sync_data() { + let loc = tempfile::tempdir().unwrap(); + let a = Dev::new('a'); + a.enable(loc.path(), false, "A", Value::Null).await; + let b = Dev::new('b'); + b.edit(|t| { + add_raw( + t, + " - name: mine\n base_url: https://mine.example\n key: sk-m\n", + ) + }) + .await; + b.ok("PUT", "/pinned-keys", json!({ "names": ["default"] })) + .await; + let preview = b + .ok( + "POST", + "/sync/enable/preview", + json!({ "target": folder(loc.path()) }), + ) + .await; + assert_eq!(preview["initial"], false); + assert_eq!( + preview["auto"], + json!([{ "item": { "kind": "key", "name": "default" }, "action": "overwrite", "reason": "default_key" }]), + "{preview}" + ); + assert!( + preview["rows"] + .as_array() + .unwrap() + .iter() + .any(|r| r["item"]["name"] == "default"), + "那一行照常列出：{preview}" + ); + let mut events = b.state.bus().subscribe(); + // 命令行以前的缺省：给默认密钥选「跳过」—— 不看 + b.enable( + loc.path(), + false, + "B", + json!({ + "join": "merge", + "decisions": [ { "item": { "kind": "key", "name": "default" }, "action": "skip" } ], + }), + ) + .await; + let key = |d: &Dev| { + d.config() + .clients + .into_iter() + .find(|c| c.name == "default") + .unwrap() + .key + }; + assert_eq!(key(&b), "tw-default-a", "B 采用同步数据的值"); + assert_eq!(keys_changed(&mut events), ["default"]); + let v = a.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["conflicts"], 0, "{v}"); + assert_eq!(key(&a), "tw-default-a", "A 的默认密钥没被改掉"); + assert!(a.provider("mine").is_some()); +} + +/// 新安装（初始配置）上已经接管了客户端：用同步数据加入时，默认密钥也采用同步数据的值，不把 +/// 本机那一把传上去改掉别的设备的 +#[tokio::test] +async fn a_fresh_device_with_clients_set_up_adopts_the_default_key() { + let loc = tempfile::tempdir().unwrap(); + let a = Dev::new('a'); + a.enable(loc.path(), false, "A", Value::Null).await; + let b = Dev::new('b'); + b.ok("PUT", "/pinned-keys", json!({ "names": ["default"] })) + .await; + let preview = b + .ok( + "POST", + "/sync/enable/preview", + json!({ "target": folder(loc.path()) }), + ) + .await; + assert_eq!(preview["initial"], true); + assert_eq!(preview["auto"][0]["reason"], "default_key", "{preview}"); + let mut events = b.state.bus().subscribe(); + b.enable(loc.path(), false, "B", Value::Null).await; + assert_eq!(b.config().clients[0].key, "tw-default-a"); + assert_eq!(keys_changed(&mut events), ["default"]); + a.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(a.config().clients[0].key, "tw-default-a"); +} diff --git a/crates/tw-sync/src/engine.rs b/crates/tw-sync/src/engine.rs index e92f7870..6ca71619 100644 --- a/crates/tw-sync/src/engine.rs +++ b/crates/tw-sync/src/engine.rs @@ -8,7 +8,8 @@ //! 写着已关闭的 keyring 才算关了加密。 //! 2. 列出 `versions/`，读出每个版本（加密的用版本密钥解开）。读不出来的分两种：**坏了**的（不是 //! 配置包、不是这把密钥加密的、加密的位置上没加密）马上跳过；**可能没传完**的当作尚未到达， -//! 最新那一层有没到的就等下一轮，等了 30 分钟还读不出来也跳过。跳过的计数报给界面。 +//! 最新那一层有没到的就等下一轮，等了 30 分钟还读不出来也跳过。云盘还没下载到本机的（iCloud +//! 的占位文件，文件夹位置请求下载、读成空的）不算读不出来，等一天。跳过的计数报给界面。 //! 3. 找出所有的**头**（没有别的版本依据它的版本）。不止一个时（两台设备同时上传、云盘晚到 //! 的版本）先以它们共同的上一版本为基准两两合并，得到「同步位置的最新」。 //! 4. 以本机基准（上次同步的那一版）为基准，和本机当前配置三方合并（[`crate::merge`]）。基准 @@ -58,6 +59,11 @@ const DEVICE_RECORD_EVERY_MS: u64 = 6 * 3600 * 1000; /// 一个可能没传完的版本（或者 keyring）等多久：之后跳过，不再让它挡着 pub const SKIP_AFTER_MS: u64 = 30 * 60 * 1000; +/// 云盘还没下载到本机的文件（iCloud 的占位文件，读出来是空的）等多久。**不算「读不出来」**： +/// 它是好的，只是还没到（文件夹位置已经请求了下载），所以等得比读不出来的久。一天还没到才 +/// 跳过，免得一直挡着这台设备上传 +pub const ARRIVING_SKIP_AFTER_MS: u64 = 24 * 3600 * 1000; + /// 同步位置看上去空了：连着这么多轮、过了这么久、自己的设备记录也不在了，才算真的清空了 const EMPTY_ROUNDS: u32 = 3; const EMPTY_AFTER_MS: u64 = 10 * 60 * 1000; @@ -254,6 +260,8 @@ enum Got { Broken, /// 可能还没传完 Pending, + /// 还没下载到本机（读出来是空的：云盘的占位文件） + Arriving, /// 列出来之后又没了 Gone, } @@ -602,6 +610,30 @@ fn key_value(it: &Item) -> Option<&Value> { it.units.get(unit::ALL)?.get("key") } +/// 一份配置的默认网关密钥（`default_key` 指着的那一把，不写就是叫 `default` 的那把）的名字 +fn default_key_name(s: &Snapshot) -> String { + s.items + .get(&ItemId::new(ItemKind::Global, model::global::DEFAULT_KEY)) + .and_then(|it| it.units.get(model::global::DEFAULT_KEY)) + .and_then(Value::as_str) + .unwrap_or(tw_config::DEFAULT_KEY) + .to_string() +} + +/// 加入时两边的默认网关密钥同名、值不同：**不当作冲突，采用同步数据的值**。每个安装都会 +/// 自动生成一把默认密钥，新加入的设备向已经在同步的那一方看齐 —— 反过来的话，加入的这一台 +/// 会把别的设备上的默认密钥改掉。本机已接管的客户端随之更新（`GatewayKeysChanged`）。是这样 +/// 时给出那一把的标识 +pub fn adopted_default_key(local: &Snapshot, remote: &Snapshot) -> Option { + let name = default_key_name(local); + if name != default_key_name(remote) { + return None; + } + let id = ItemId::new(ItemKind::Key, name); + let (l, r) = (local.items.get(&id)?, remote.items.get(&id)?); + (key_value(l) != key_value(r)).then_some(id) +} + // ─────────────────────────────────────────────── 等登录的上游 /// 本机配置里的上游（带着这台设备自己的登录） @@ -828,6 +860,18 @@ impl Engine { } return Ok(self.set_mode(None, None, None)); }; + // 还没下载到本机（云盘的占位文件）：等它到，不算读不出来 + if bytes.is_empty() { + let since = *st + .arriving + .entry(KEYRING.to_string()) + .or_insert_with(now_ms); + if now_ms().saturating_sub(since) >= ARRIVING_SKIP_AFTER_MS { + return Err(RoundError::KeyringMissing(KeyringError::Corrupt.msg())); + } + return Err(RoundError::NotArrived); + } + st.arriving.remove(KEYRING); let hash: [u8; 32] = Sha256::digest(&bytes).into(); if self.mode_of == Some(Some(hash)) { return Ok(self.sealed); @@ -902,7 +946,7 @@ impl Engine { }; // 空的：云盘的占位文件、正在写 if bytes.is_empty() { - return Ok(Got::Pending); + return Ok(Got::Arriving); } // 认得出是配置包（ZIP）或者加密过的：读不出来多半是没传完 let shaped = keyring::is_age(&bytes) || bytes.starts_with(b"PK\x03\x04"); @@ -1011,22 +1055,26 @@ impl Engine { match self.fetch(n).await? { Got::Ok(v) => { st.unreadable.remove(&n.id()); + st.arriving.remove(&n.id()); sc.names.push(n.clone()); sc.ok.push(v); } Got::TooNew(cv) => { st.unreadable.remove(&n.id()); + st.arriving.remove(&n.id()); sc.names.push(n.clone()); sc.too_new.push((n.clone(), cv)); } Got::Broken => { st.unreadable.remove(&n.id()); + st.arriving.remove(&n.id()); sc.skipped.push(n.clone()); if n.device == me { sc.own_broken.push(n.clone()); } } Got::Pending => { + st.arriving.remove(&n.id()); let since = *st.unreadable.entry(n.id()).or_insert(now); if now.saturating_sub(since) >= SKIP_AFTER_MS { sc.skipped.push(n.clone()); @@ -1035,11 +1083,22 @@ impl Engine { sc.pending.push(n.clone()); } } + Got::Arriving => { + st.unreadable.remove(&n.id()); + let since = *st.arriving.entry(n.id()).or_insert(now); + if now.saturating_sub(since) >= ARRIVING_SKIP_AFTER_MS { + sc.skipped.push(n.clone()); + } else { + sc.names.push(n.clone()); + sc.pending.push(n.clone()); + } + } Got::Gone => {} } } - st.unreadable - .retain(|k, _| k == KEYRING || all.iter().any(|n| &n.id() == k)); + let listed = |k: &String| k == KEYRING || all.iter().any(|n| &n.id() == k); + st.unreadable.retain(|k, _| listed(k)); + st.arriving.retain(|k, _| listed(k)); sc.all = all; Ok(sc) } @@ -2254,7 +2313,8 @@ impl Engine { /// 加入已有的同步。`merged` 是本机按导入的规则合并之后的完整配置包（「合并」）；`None` 是 /// 用同步数据替换本机配置（新安装、或者选了「使用同步数据」）—— 那时本机已接管的客户端 - /// 正在用的网关密钥并入同步数据，不同步的上游留在本机。 + /// 正在用的网关密钥并入同步数据（默认密钥除外：采用同步数据的值，见 + /// [`adopted_default_key`]），不同步的上游留在本机。 /// /// 本机先照结果改，再上传一个依据同步位置最新版本的新版本，作为这台设备的基准。两种方式 /// 下，别的设备上登录着、这台没登录的上游都在这台停用着，**上传的是同步的样子**（启用着）。 @@ -2297,10 +2357,12 @@ impl Engine { let mut target = match &merged { Some(p) => model::snapshot(p).map_err(|e| RoundError::Local(e.msg()))?, None => { + // 已接管的客户端在用的密钥并入同步数据；默认密钥例外，采用同步数据的值 let mut t = remote.snap.clone(); let pinned = self.host.pinned_keys(); + let adopted = adopted_default_key(&local, &remote.snap); for (id, it) in local.of_kind(ItemKind::Key) { - if pinned.contains(&id.name) { + if pinned.contains(&id.name) && adopted.as_ref() != Some(id) { t.items.insert(id.clone(), it.clone()); } } diff --git a/crates/tw-sync/src/local.rs b/crates/tw-sync/src/local.rs index 904d5c09..d1584626 100644 --- a/crates/tw-sync/src/local.rs +++ b/crates/tw-sync/src/local.rs @@ -60,6 +60,10 @@ pub struct State { /// 同步位置上读不出来的文件：名字 → 第一次读不出来的时刻。读出来了就去掉 #[serde(skip_serializing_if = "BTreeMap::is_empty")] pub unreadable: BTreeMap, + /// 同步位置上还没下载到本机的文件（云盘的占位文件）：名字 → 第一次看到的时刻。和读不出来 + /// 的分开算（等得更久） + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + pub arriving: BTreeMap, /// 同步位置看上去是空的：从什么时候起、连着几轮 pub empty_since_ms: Option, pub empty_rounds: u32, diff --git a/crates/tw-sync/src/store/folder.rs b/crates/tw-sync/src/store/folder.rs index a6ae4891..418bdcfc 100644 --- a/crates/tw-sync/src/store/folder.rs +++ b/crates/tw-sync/src/store/folder.rs @@ -18,9 +18,70 @@ use super::{ too_large, }; -/// iCloud Drive 还没下载到本机的文件，在目录里是一个 `.名字.icloud` 的占位文件 +/// iCloud Drive 还没下载到本机的文件，在目录里是一个 `.名字.icloud` 的占位文件（macOS 13 及 +/// 以前；之后的 iCloud Drive 文件照真名列出，读的时候系统自己去下载） const ICLOUD: &str = ".icloud"; +/// 同一个占位文件多久才再请求一次下载 +#[cfg(target_os = "macos")] +const DOWNLOAD_AGAIN_AFTER: std::time::Duration = std::time::Duration::from_secs(5 * 60); + +/// 请求下载的脚本（JavaScript for Automation）：路径是第一个参数，答 `true` / `false` +#[cfg(target_os = "macos")] +const DOWNLOAD_SCRIPT: &str = "function run(argv) { ObjC.import('Foundation'); \ + return String($.NSFileManager.defaultManager.startDownloadingUbiquitousItemAtURLError(\ + $.NSURL.fileURLWithPath(argv[0]), null)); }"; + +/// 让 iCloud 把一个还没下载的文件下载下来。**旧式占位文件读真名不会触发下载**，要明确请求： +/// 苹果的接口是 `-[NSFileManager startDownloadingUbiquitousItemAtURL:error:]`（macOS 10.7 起都有； +/// `brctl download` 在新系统上已经没有了）。经系统自带的 `osascript` 以 JavaScript for +/// Automation 调它，core 里不写原生调用；路径作为参数传进去，不拼进脚本。只请求、不等下载完： +/// 下载完了真名的文件就出现了，文件夹的监听随之再来一轮。别的系统什么都不做 +fn start_download(real: &Path) { + #[cfg(target_os = "macos")] + { + use std::collections::HashMap; + use std::sync::{Mutex, OnceLock}; + use std::time::Instant; + static ASKED: OnceLock>> = OnceLock::new(); + let asked = ASKED.get_or_init(Default::default); + if let Ok(mut m) = asked.lock() { + if m.get(real) + .is_some_and(|t| t.elapsed() < DOWNLOAD_AGAIN_AFTER) + { + return; + } + m.retain(|_, t| t.elapsed() < DOWNLOAD_AGAIN_AFTER); + m.insert(real.to_path_buf(), Instant::now()); + } + let child = std::process::Command::new("/usr/bin/osascript") + .args(["-l", "JavaScript", "-e", DOWNLOAD_SCRIPT]) + .arg(real) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn(); + match child { + // 不等它：另起一个线程收尸，最多等 10 秒 + Ok(mut c) => { + std::thread::spawn(move || { + for _ in 0..100 { + if !matches!(c.try_wait(), Ok(None)) { + return; + } + std::thread::sleep(std::time::Duration::from_millis(100)); + } + let _ = c.kill(); + let _ = c.wait(); + }); + } + Err(e) => tracing::debug!("iCloud could not be asked to download a file: {e}"), + } + } + #[cfg(not(target_os = "macos"))] + let _ = real; +} + /// 读一个文件，**边读边数**：超过上限就不读了 fn read_limited(p: &Path) -> std::io::Result, ()>> { use std::io::Read; @@ -206,11 +267,12 @@ impl Store for Folder { if !root.is_dir() { return Err(io(&root, e)); } - // iCloud 还没下载的：有，只是还没到 —— 读成空的（调用方当作尚未到达） + // iCloud 还没下载的：有，只是还没到 —— 请求下载，读成空的（调用方当作尚未到达） let placeholder = p .file_name() .map(|n| p.with_file_name(format!(".{}{ICLOUD}", n.to_string_lossy()))); if placeholder.is_some_and(|ph| ph.is_file()) { + start_download(&p); Ok(Some(Vec::new())) } else { Ok(None) @@ -471,6 +533,23 @@ mod tests { ); } + /// 请求下载的那段脚本在这台 Mac 上跑得通：对一个不在 iCloud 里的文件，接口答 `false`（不是 + /// 脚本出错、不是 osascript 崩了） + #[cfg(target_os = "macos")] + #[test] + fn the_download_request_script_runs() { + let d = tempfile::tempdir().unwrap(); + let f = d.path().join("x"); + std::fs::write(&f, b"x").unwrap(); + let out = std::process::Command::new("/usr/bin/osascript") + .args(["-l", "JavaScript", "-e", DOWNLOAD_SCRIPT]) + .arg(&f) + .output() + .unwrap(); + assert!(out.status.success(), "{out:?}"); + assert_eq!(String::from_utf8_lossy(&out.stdout).trim(), "false"); + } + #[tokio::test] async fn an_oversized_file_is_not_read() { let (d, f) = bed(); diff --git a/crates/tw-sync/tests/regressions.rs b/crates/tw-sync/tests/regressions.rs index 08440fb7..992b5bdf 100644 --- a/crates/tw-sync/tests/regressions.rs +++ b/crates/tw-sync/tests/regressions.rs @@ -215,7 +215,7 @@ fn forge_plain_version(loc: &Path, config_yaml: String) -> String { /// 把本机记着的「从什么时候起」往前拨，模拟时间过去了 fn age_state(e: &Engine, by_ms: u64) { let mut st = e.local().state(); - for t in st.unreadable.values_mut() { + for t in st.unreadable.values_mut().chain(st.arriving.values_mut()) { *t = t.saturating_sub(by_ms); } st.empty_since_ms = st.empty_since_ms.map(|t| t.saturating_sub(by_ms)); @@ -953,7 +953,8 @@ async fn a_version_that_never_finishes_arriving_is_skipped_after_a_while() { assert!(t.b.provider("one").is_some()); } -/// iCloud 还没下载的最新版本（只有占位文件）：当作尚未到达 +/// iCloud 还没下载的最新版本（只有旧式占位文件）：当作尚未到达，**不算读不出来**——过了 +/// 30 分钟照样等（文件夹位置请求了下载）；一天还没到才跳过 #[tokio::test] async fn an_icloud_placeholder_on_top_is_waited_for() { let mut t = bed(false).await; @@ -972,9 +973,39 @@ async fn an_icloud_placeholder_on_top_is_waited_for() { ) .unwrap(); assert!(matches!(t.eb.round().await, Err(RoundError::NotArrived))); + age_state(&t.eb, SKIP_AFTER_MS + 1000); + assert!( + matches!(t.eb.round().await, Err(RoundError::NotArrived)), + "占位文件不按读不出来的 30 分钟跳过" + ); std::fs::rename(parked.path().join(&file), &real).unwrap(); t.eb.round().await.unwrap(); assert!(t.b.provider("one").is_some()); + assert!(t.eb.state().arriving.is_empty()); +} + +/// 一直没下载下来的占位文件：一天之后跳过，不再挡着这台设备上传 +#[tokio::test] +async fn an_icloud_placeholder_that_never_downloads_is_skipped_after_a_day() { + let mut t = bed(false).await; + let up = t.ea.round().await.unwrap().uploaded; + assert!(up.is_none()); + let head = versions(t.loc.path()).last().unwrap().clone(); + let seq: u64 = head[..10].parse().unwrap(); + std::fs::write( + t.loc + .path() + .join("versions") + .join(format!(".{:010}-cccccccc.twconfig.icloud", seq + 1)), + b"placeholder", + ) + .unwrap(); + t.b.edit(|s| add_provider(s, "fromb", "https://b.example", "sk-b")); + assert!(matches!(t.eb.round().await, Err(RoundError::NotArrived))); + age_state(&t.eb, tw_sync::engine::ARRIVING_SKIP_AFTER_MS + 1000); + let r = t.eb.round().await.unwrap(); + assert_eq!(r.skipped, 1); + assert!(r.uploaded.is_some()); } // ─────────────────────────────────────────────── 空的同步位置 diff --git a/docs/config.md b/docs/config.md index 63b3b118..32abe538 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1522,6 +1522,11 @@ plugins and the general settings, keys included. `listen`, `retention` and OAuth tokens. An upstream signed in on another device stays turned off on this one until it is signed in here; that does not turn it off anywhere else. +A device that joins with a configuration of its own merges it item by item; +items that differ are chosen one by one, except the default gateway key: every +installation makes one, so the joining device takes the sync data's value and +the clients set up on it are updated. + With encryption on, versions that are not encrypted are never read. A keyring that went missing, or an older one put back, does not turn encryption off: the app asks whether to put back the copy this device keeps. A location that reads diff --git a/docs/config.zh-CN.md b/docs/config.zh-CN.md index 930fa9c0..b9b7ee4b 100644 --- a/docs/config.zh-CN.md +++ b/docs/config.zh-CN.md @@ -1107,6 +1107,8 @@ export const manifest = { 同步的内容即配置包带的内容：上游、出站代理、网关密钥、别名、分组、路由、价目表、安全规则、插件和全局设置，包括密钥。`listen`、`retention` 和 `sync` 本身留在本机，ChatGPT 账号的登录、OAuth 的令牌也一样。别的设备上登录着的上游在这台设备上停用着，在这台登录之后才启用；这不会让它在别的设备上停用。 +已有配置的设备加入时逐项合并，不一样的项目逐项选择；默认网关密钥例外：每个安装都会生成一把，加入的设备采用同步数据的值，本机已接管的客户端随之更新。 + 开着加密时，不加密的版本一概不读。keyring 不见了、被换成更早的那一份，都不会关掉加密：应用会问要不要放回这台设备记着的那一份。同步位置读起来是空的时，要再看几次才提示，停不停止同步由你决定。 From 6867b1ed58e68b32b446fdc05a764bff2a2db9e3 Mon Sep 17 00:00:00 2001 From: fylorn <249551762+fylorn@users.noreply.github.com> Date: Mon, 19 Oct 2026 19:26:06 +0800 Subject: [PATCH 09/13] Packs: secrets stay with their destination; restore keeps what this machine needs (#359) Import - A pack without secrets reuses the keys here only when overwriting an item whose destination is unchanged: same scheme, host and port, AWS region, OAuth token endpoint, and proxy type and address. Otherwise the secret stays blank, the row is flagged missing_key and the upstream comes in disabled. An item imported under another name never takes the keys here. - A proxy whose password is blank is flagged missing_key too. Restore - A route that names a group which is gone resolves to that group: kept when it is being removed, brought back when only the route is chosen. - Proxies used by this machine's own sections (the sync location's proxy) are kept; a whole restore copies the proxy over when the version lacks it. - Restoring a version from the sync location leaves the upstreams this machine does not sync, and what follows them, out of the comparison. - POST /backups/preview takes the chosen items and plans exactly what POST /backups/restore writes; the restore now answers RestoreWritten { version, rows } with the rows it actually applied. - Text that does not read as YAML is no longer kept as a version, and a whole restore takes this machine's sections from the newest readable one. Other - Pinned keys no longer expire; a source replaces its own set, under one lock, with limits on sources, names and lengths (backup.pinned.*). The older file shapes read as the blank source. - Plugin files put back at startup are reloaded by the gateway, and only when neither the script nor its approved copy matches the approved hash. - The 64 KiB preference limit applies to the preferences being written, in every writer; a large one already stored does not block a restore. - A passphrase-encrypted pack that asks for more work than allowed reports pack.too_much_work instead of pack.corrupt. Co-authored-by: Claude Opus 5.5 --- bin/twcore/src/pack_cli.rs | 4 +- crates/tw-api/msg-codes.txt | 4 + crates/tw-api/src/ep.rs | 2 +- crates/tw-api/src/lib.rs | 15 +- crates/tw-api/src/pack.rs | 18 +- crates/tw-control/src/backups.rs | 130 ++++++++----- crates/tw-control/src/config.rs | 33 +++- crates/tw-control/tests/backups.rs | 280 ++++++++++++++++++++++++++++ crates/tw-pack/src/backup.rs | 67 ++++++- crates/tw-pack/src/disk.rs | 190 ++++++++++++++++--- crates/tw-pack/src/export.rs | 96 +++++++++- crates/tw-pack/src/import.rs | 15 +- crates/tw-pack/src/lib.rs | 13 ++ crates/tw-pack/src/restore.rs | 226 ++++++++++++++++++++-- crates/tw-pack/src/restore/tests.rs | 26 ++- crates/tw-pack/src/tests.rs | 33 ++++ crates/tw-pack/tests/regressions.rs | 255 +++++++++++++++++++++++++ docs/config.md | 7 +- docs/config.zh-CN.md | 2 +- 19 files changed, 1293 insertions(+), 123 deletions(-) diff --git a/bin/twcore/src/pack_cli.rs b/bin/twcore/src/pack_cli.rs index f02b860e..be68ffc8 100644 --- a/bin/twcore/src/pack_cli.rs +++ b/bin/twcore/src/pack_cli.rs @@ -381,9 +381,7 @@ pub fn restore(path: &Path, id: &str, dry_run: bool) -> Result<()> { let pinned = tw_pack::disk::read_pinned(path); let after = if tw_pack::model::snapshot(&local).is_err() { println!("the configuration here does not load; the whole version comes back"); - let last = tw_pack::backup::list(path) - .ok() - .and_then(|all| all.last().and_then(|b| tw_pack::backup::open(b).ok())); + let last = tw_pack::backup::last_readable(path, &cur.text); tw_pack::restore::whole(&backup, &cur.text, last.as_ref()) } else { let rows = tw_pack::restore::preview(&local, &backup, &pinned, true).map_err(model)?; diff --git a/crates/tw-api/msg-codes.txt b/crates/tw-api/msg-codes.txt index 8a1cda17..95d160d9 100644 --- a/crates/tw-api/msg-codes.txt +++ b/crates/tw-api/msg-codes.txt @@ -5,6 +5,9 @@ # test only produced by tests; never reaches a UI backup.io backup.not_found +backup.pinned.name_too_long +backup.pinned.source_too_long +backup.pinned.too_many backup.prefs_too_large backup.write_failed config.alias_blank_model @@ -516,6 +519,7 @@ pack.not_encrypted pack.passphrase_short pack.schema_too_new pack.too_large +pack.too_much_work pack.too_new pack.verify_failed pack.wrong_key diff --git a/crates/tw-api/src/ep.rs b/crates/tw-api/src/ep.rs index 34735dc5..fa991be1 100644 --- a/crates/tw-api/src/ep.rs +++ b/crates/tw-api/src/ep.rs @@ -53,7 +53,7 @@ endpoints! { /// 恢复到一个版本会改什么。**不写任何东西** RestorePreview: POST "/backups/preview", api::RestorePreviewRequest => api::RestorePreview; /// 恢复到一个版本（全部，或者选的几项）。恢复本身也留一个版本 - Restore: POST "/backups/restore", api::RestoreRequest => api::ConfigWritten; + Restore: POST "/backups/restore", api::RestoreRequest => api::RestoreWritten; /// Lite 的通用偏好 GetClientPrefs: GET "/client-prefs", () => api::ClientPrefs; PutClientPrefs: PUT "/client-prefs", api::ClientPrefs => (); diff --git a/crates/tw-api/src/lib.rs b/crates/tw-api/src/lib.rs index cef7b79d..762163e5 100644 --- a/crates/tw-api/src/lib.rs +++ b/crates/tw-api/src/lib.rs @@ -964,17 +964,20 @@ pub const MSG_CODES: &str = include_str!("../msg-codes.txt"); /// `POST /config/rollback`（`ConfigVersion`、`RollbackRequest`）删了，换成 `GET /backups` /// （[`BackupEntry`]：本机最近 10 个版本，开着同步时连同同步位置上的，同一个版本一行）、 /// `POST /backups/file`（另存为配置包）、`POST /backups/preview`（[`RestorePreview`]：按项目 -/// 列出差异）、`POST /backups/restore`（可以只恢复几项，本机设置和 ChatGPT 的登录留着，接管的 -/// 客户端在用的密钥不删）。一版是一个完整的配置包，连同插件和 Lite 的通用偏好。新端点 +/// 列出差异；给了 `items` 是只恢复这几项时会动什么）、`POST /backups/restore`（可以只恢复几项， +/// 本机设置和 ChatGPT 的登录留着，接管的客户端在用的密钥不删；→ [`RestoreWritten`]，`rows` 是 +/// 实际动了的项目，和同样条件下的预览一样）。一版是一个完整的配置包，连同插件和 Lite 的通用偏好。新端点 /// `GET /config/items`（[`ItemNode`]）、`POST /config/export`（[`ExportRequest`] → /// [`ExportedFile`]）、`POST /config/import/preview`（[`ImportSource`] → [`ImportPreview`]：逐项 /// 新增、相同、冲突、错误）和 `POST /config/import`（[`ImportRequest`]：逐项导入、覆盖、改名 /// 导入或跳过）。Lite 的通用偏好由 core 存：`GET` / `PUT /client-prefs`（[`ClientPrefs`]， /// 不解析，最大 64 KiB），变了发 [`Event::ClientPrefsChanged`]。`PUT /pinned-keys` -/// （[`PinnedKeys`]：`source` 是这个 Lite 的标识，几个 Lite 各报各的、core 取并集，30 天没更新 -/// 的来源不算）报告接管的客户端在用哪几把密钥。恢复时留下的项目还引用着的不删 -/// （[`KeptReason`] 的 `referenced`），所选项目引用的、本机已没有的一起恢复；不带密钥的包导入时 -/// 比较和覆盖都沿用本机的密钥。导入覆盖、恢复改了或删了网关密钥时发 +/// （[`PinnedKeys`]：`source` 是这个 Lite 的标识，几个 Lite 各报各的、core 取并集，来源不过期， +/// 超出上限是 `backup.pinned.*`）报告接管的客户端在用哪几把密钥。恢复时留下的项目还引用着的、 +/// 同步设置走的出站代理不删（[`KeptReason`] 的 `referenced`），所选项目引用的、本机已没有的一起 +/// 恢复；恢复同步位置上的版本时本机不同步的上游不列成删除。不带密钥的包导入时比较和覆盖都沿用 +/// 本机的密钥，**只在目的地不变时**（协议、主机、端口、AWS 区域、OAuth 令牌端点、代理地址 +/// 一样）；改名导入的不沿用，缺着的标 `missing_key`（代理的密码也算）。导入覆盖、恢复改了或删了网关密钥时发 /// [`Event::GatewayKeysChanged`]。[`ConfigOrigin`] 删了 `rollback`，多了 `sync`、`import`、 /// `restore`、`repair`（一键修复以前记成 `ui`）。备份相关的端点在安全模式下照样可用。配置包的 /// 码是 `pack.*`，导入的是 `import.*`，备份与恢复的是 `backup.*`。停用着的上游可以缺凭据 diff --git a/crates/tw-api/src/pack.rs b/crates/tw-api/src/pack.rs index ab10f724..a86c9119 100644 --- a/crates/tw-api/src/pack.rs +++ b/crates/tw-api/src/pack.rs @@ -311,6 +311,10 @@ pub struct BackupFileRequest { #[cfg_attr(feature = "ts", derive(ts_rs::TS))] pub struct RestorePreviewRequest { pub id: String, + /// 只恢复这几项时会动什么（连同一起恢复的、留着的）。不给是全部。和 + /// [`RestoreRequest::items`] 一样，预览和写入走同一个计划 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub items: Option>, } /// 恢复会改什么。**什么都不写**。 @@ -371,6 +375,17 @@ pub struct RestoreRequest { pub items: Option>, } +/// `POST /backups/restore` 的结果。 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct RestoreWritten { + /// 写下去之后的配置版本 + pub version: String, + /// 实际动了的项目：所选的，连同一起恢复的（`kept` 为空）和留着没改的（`kept` 写明原因）。 + /// 和同样条件下的 [`RestorePreview::rows`] 一样。安全模式下整份换回时每一项都是新增 + pub rows: Vec, +} + /// Lite 的通用偏好（语言、外观、菜单栏、通知、检查更新），**core 不解析**，最大 64 KiB。 #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[cfg_attr(feature = "ts", derive(ts_rs::TS))] @@ -386,7 +401,8 @@ pub const CLIENT_PREFS_MAX: usize = 64 * 1024; /// `PUT /pinned-keys`：这一个 Lite 接管的客户端正在用的网关密钥。恢复、导入、同步不删它们。 /// /// **按来源记**：几个 Lite 连着同一个 core（远程模式）时各报各的，core 取并集；一个来源报的 -/// 换掉它自己上次报的。30 天没更新的来源不再算 +/// 换掉它自己上次报的。来源不过期（Lite 每次连上 core 都重报一次）；最多记 64 个来源，多出来 +/// 的去掉最久没报的。来源最长 128 字节，一次最多 1024 把、每个名字最长 256 字节，超出是 400 #[derive(Debug, Clone, Default, Serialize, Deserialize)] #[cfg_attr(feature = "ts", derive(ts_rs::TS))] pub struct PinnedKeys { diff --git a/crates/tw-control/src/backups.rs b/crates/tw-control/src/backups.rs index 4286d3ad..6aa48518 100644 --- a/crates/tw-control/src/backups.rs +++ b/crates/tw-control/src/backups.rs @@ -349,10 +349,12 @@ async fn land( changed_keys: Vec, ) -> Result { let path = s.config_path().to_path_buf(); - // 偏好的上限导入、恢复也照样查：一份带着超大偏好的包不该绕过 `PUT /client-prefs` 的那道关 - if core_keeps_prefs() + // 偏好的上限导入、恢复也照样查：一份带着超大偏好的包不该绕过 `PUT /client-prefs` 的那道关。 + // **只查要写下去的新偏好**：已经存着的一份超大的（上限之前存的）不该让每一次导入、恢复都失败 + let prefs_changed = core_keeps_prefs() && after.client_prefs != before.client_prefs; + if prefs_changed && let Some(p) = &after.client_prefs - && p.to_string().len() > tw_api::CLIENT_PREFS_MAX + && tw_pack::disk::prefs_too_large(p) { return Err(prefs_too_large()); } @@ -364,7 +366,6 @@ async fn land( e, ) })?; - let prefs_changed = core_keeps_prefs() && after.client_prefs != before.client_prefs; if prefs_changed && let Err(e) = tw_pack::disk::write_prefs(&path, after.client_prefs.as_ref()) { undo.restore(); @@ -742,23 +743,10 @@ async fn restore_preview( None => 0, }; // 配置读不进来：整份换回那一版，每一项都是「新增」 - let rows = match tw_pack::restore::preview( - &local, - &backup, - &s.cfg.pinned_keys(), - core_keeps_prefs(), - ) { + let opts = restore_options(&s, &req.id, req.items.as_deref(), &local, &backup); + let rows = match tw_pack::restore::preview_opts(&local, &backup, &opts) { Ok(rows) => rows.into_iter().map(restore_row).collect(), - Err(_) if model::snapshot(&local).is_err() => model::snapshot(&backup) - .map_err(|e| model_fail(e, Some(&backup)))? - .items - .keys() - .map(|id| tw_api::RestoreRow { - item: item_ref(id), - change: tw_api::RestoreChange::Add, - kept: None, - }) - .collect(), + Err(_) if model::snapshot(&local).is_err() => whole_rows(&backup)?, Err(e) => return Err(model_fail(e, Some(&backup))), }; Ok(Json(tw_api::RestorePreview { @@ -769,10 +757,54 @@ async fn restore_preview( })) } +/// 安全模式下整份换回：那一版的每一项都是「新增」 +fn whole_rows(backup: &Pack) -> Result, Fail> { + Ok(model::snapshot(backup) + .map_err(|e| model_fail(e, Some(backup)))? + .items + .keys() + .map(|id| tw_api::RestoreRow { + item: item_ref(id), + change: tw_api::RestoreChange::Add, + kept: None, + }) + .collect()) +} + +/// 恢复的条件：预览和写入用同一份。恢复同步位置上的版本时，本机不同步的上游和随之不同步的 +/// 项目不参与比较（规格 7.2）：那一版里本来就没有它们 +fn restore_options( + s: &ControlState, + id: &str, + items: Option<&[tw_api::ItemRef]>, + local: &Pack, + backup: &Pack, +) -> tw_pack::restore::Options { + let skip = if id.starts_with(SYNC_ID) { + let names: BTreeSet = tw_config::try_parse(&local.config_yaml) + .ok() + .and_then(|c| c.sync) + .map(|sync| sync.exclude.into_iter().collect()) + .unwrap_or_default(); + match (model::snapshot(local), model::snapshot(backup)) { + (Ok(l), Ok(b)) if !names.is_empty() => tw_sync::engine::excluded(&names, &[&l, &b]), + _ => BTreeSet::new(), + } + } else { + BTreeSet::new() + }; + tw_pack::restore::Options { + pinned: s.cfg.pinned_keys(), + with_prefs: core_keeps_prefs(), + items: items.map(|xs| xs.iter().map(item_id).collect()), + skip, + } +} + async fn restore( State(s): State, Json(req): Json, -) -> Result, Fail> { +) -> Result, Fail> { let raw = backup_pack(&s, &req.id).await?; let (backup, _) = tw_pack::import::prepare(raw.clone()).map_err(|e| import_fail(e, &raw))?; let (cur, local) = current(&s, PackOrigin::Restore)?; @@ -782,35 +814,26 @@ async fn restore( current: cur.version(), })); } - let picked: Option> = req - .items - .as_ref() - .map(|xs| xs.iter().map(item_id).collect()); - let (after, changed) = if model::snapshot(&local).is_err() { + let (after, changed, rows) = if model::snapshot(&local).is_err() { // 安全模式：现在这份读不进来，整份换回去 - // 本机那几节取不到时（原文坏到读不成 YAML）用最近一个本机备份里的 - let last = tw_pack::backup::list(s.config_path()) - .ok() - .and_then(|all| all.last().and_then(|b| tw_pack::backup::open(b).ok())); + // 本机那几节取不到时（原文坏到读不成 YAML）用最近一个读得成 YAML 的本机备份里的 + let last = tw_pack::backup::last_readable(s.config_path(), &cur.text); let mut whole = tw_pack::restore::whole(&backup, &cur.text, last.as_ref()); if !core_keeps_prefs() { whole.client_prefs = local.client_prefs.clone(); } - (whole, Vec::new()) + let rows = whole_rows(&backup)?; + (whole, Vec::new(), rows) } else { - let applied = tw_pack::restore::apply( - &local, - &backup, - picked.as_deref(), - &s.cfg.pinned_keys(), - core_keeps_prefs(), - ) - .map_err(|e| model_fail(e, Some(&backup)))?; - (applied.pack, applied.changed_keys) + let opts = restore_options(&s, &req.id, req.items.as_deref(), &local, &backup); + let applied = tw_pack::restore::apply_opts(&local, &backup, &opts) + .map_err(|e| model_fail(e, Some(&backup)))?; + let rows = applied.rows.into_iter().map(restore_row).collect(); + (applied.pack, applied.changed_keys, rows) }; crate::plugins::guard_raw_write(&s, &cur.text, &after.config_yaml).await?; let version = land(&s, &local, &after, &cur.version(), Origin::Restore, changed).await?; - Ok(Json(tw_api::ConfigWritten { version })) + Ok(Json(tw_api::RestoreWritten { version, rows })) } // ─────────────────────────────────────────────────────────── 通用偏好、固定密钥 @@ -826,7 +849,7 @@ async fn put_prefs( Json(req): Json, ) -> Result, Fail> { if let Some(p) = &req.prefs - && p.to_string().len() > tw_api::CLIENT_PREFS_MAX + && tw_pack::disk::prefs_too_large(p) { return Err(prefs_too_large()); } @@ -853,10 +876,31 @@ async fn pin_keys( State(s): State, Json(req): Json, ) -> Result, Fail> { + use crate::config::PinnedError; + use tw_pack::disk::{PINNED_NAME_MAX, PINNED_NAMES_MAX, PINNED_SOURCE_MAX, PinnedTooLarge}; let names: BTreeSet = req.names.into_iter().collect(); s.cfg .set_pinned_keys(&req.source, names) - .map_err(|e| io_fail(&tw_pack::disk::pinned_path(s.config_path()), e))?; + .map_err(|e| match e { + PinnedError::Io(e) => io_fail(&tw_pack::disk::pinned_path(s.config_path()), e), + PinnedError::TooLarge(t) => fail( + StatusCode::BAD_REQUEST, + match t { + PinnedTooLarge::Source => msg!( + "backup.pinned.source_too_long", max = PINNED_SOURCE_MAX => + "The source of the pinned keys is longer than {max} bytes." + ), + PinnedTooLarge::Names => msg!( + "backup.pinned.too_many", max = PINNED_NAMES_MAX => + "More than {max} pinned keys were reported at once." + ), + PinnedTooLarge::Name => msg!( + "backup.pinned.name_too_long", max = PINNED_NAME_MAX => + "A pinned key name is longer than {max} bytes." + ), + }, + ), + })?; Ok(Json(())) } diff --git a/crates/tw-control/src/config.rs b/crates/tw-control/src/config.rs index 9d988b59..d3a5b9c0 100644 --- a/crates/tw-control/src/config.rs +++ b/crates/tw-control/src/config.rs @@ -379,6 +379,8 @@ impl ConfigManager { plugins = ?healed, "plugin files that did not match the configuration were put back from a backup" ); + // 网关已经照找回来之前的文件读过插件（对不上的停用了）：照找回来的重读一遍 + self.gateway.reload_plugins(); } if let Ok(cur) = self.current() { self.backup(tw_pack::backup::record( @@ -389,26 +391,31 @@ impl ConfigManager { } } - /// 本机接管的客户端在用的网关密钥（固定密钥）：各个来源报上来的并集，30 天没更新的来源 - /// 不算。恢复、导入、同步不删它们 + /// 本机接管的客户端在用的网关密钥（固定密钥）：各个来源报上来的并集。恢复、导入、同步 + /// 不删它们 pub fn pinned_keys(&self) -> std::collections::BTreeSet { self.pinned .lock() - .map(|g| tw_pack::disk::pinned_union(&g, now_ms())) + .map(|g| tw_pack::disk::pinned_union(&g)) .unwrap_or_default() } /// 一个来源（一个 Lite）报上来它的固定密钥：换掉它上次报的，存进数据目录 - /// （`pinned-keys.json`），重启之后还在 + /// （`pinned-keys.json`），重启之后还在。**读、改、写都在锁里**：两个 Lite 同时报， + /// 谁也不会盖掉谁的 pub fn set_pinned_keys( &self, source: &str, names: std::collections::BTreeSet, - ) -> std::io::Result<()> { - let all = tw_pack::disk::write_pinned(&self.path, source, &names)?; - if let Ok(mut g) = self.pinned.lock() { - *g = all; - } + ) -> Result<(), PinnedError> { + let mut g = self + .pinned + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let next = tw_pack::disk::with_pinned(&g, source, &names, now_ms()) + .map_err(PinnedError::TooLarge)?; + tw_pack::disk::write_pinned_sources(&self.path, &next).map_err(PinnedError::Io)?; + *g = next; Ok(()) } @@ -641,6 +648,14 @@ fn bad_path(e: tw_yaml::PatchError) -> ApplyError { ApplyError::BadPath(e.msg()) } +/// 存固定密钥没成。 +#[derive(Debug)] +pub enum PinnedError { + /// 报上来的超出了上限（[`tw_pack::disk::PINNED_SOURCE_MAX`] 这几个） + TooLarge(tw_pack::disk::PinnedTooLarge), + Io(std::io::Error), +} + #[cfg(test)] mod patch_seq_tests { use super::*; diff --git a/crates/tw-control/tests/backups.rs b/crates/tw-control/tests/backups.rs index 3d016a43..837da99e 100644 --- a/crates/tw-control/tests/backups.rs +++ b/crates/tw-control/tests/backups.rs @@ -788,3 +788,283 @@ async fn in_safe_mode_a_synced_version_comes_back_whole() { assert_eq!(cfg.listen.control.key.as_deref(), Some(KEY)); assert!(cfg.providers.iter().any(|p| p.name == "relay")); } + +fn rows_of(v: &serde_json::Value) -> Vec { + v["rows"] + .as_array() + .unwrap() + .iter() + .map(|r| { + format!( + "{} {}/{}{}", + r["change"].as_str().unwrap(), + r["item"]["kind"].as_str().unwrap(), + r["item"]["name"].as_str().unwrap(), + r["kept"] + .as_str() + .map(|k| format!(" ({k})")) + .unwrap_or_default() + ) + }) + .collect() +} + +/// 只恢复几项：预览接收所选的项目，和写入走同一个计划；写入的结果说实际动了哪几项 +#[tokio::test] +async fn a_partial_restore_writes_what_its_preview_listed() { + let with = format!( + "{}proxies:\n - name: px\n type: http\n addr: 10.0.0.1:8080\n", + mine() + ) + .replace( + "routes:", + " - name: gone\n base_url: https://gone.example\n key: sk-g\n proxy: px\nroutes:", + ); + tw_config::try_parse(&with).unwrap(); + let b = bed(&with); + b.state.cfg.record_startup(); + b.state + .cfg + .write(&mine(), None, tw_config::history::Origin::Ui) + .await + .unwrap(); + let (_, list) = call(&b.app, "GET", "/backups", serde_json::json!(null)).await; + let oldest = list.as_array().unwrap().last().unwrap()["id"].clone(); + let items = serde_json::json!([{ "kind": "provider", "name": "gone" }]); + let (st, preview) = call( + &b.app, + "POST", + "/backups/preview", + serde_json::json!({ "id": oldest, "items": items }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{preview}"); + assert_eq!(rows_of(&preview), ["add provider/gone", "add proxy/px"]); + let (st, v) = call( + &b.app, + "POST", + "/backups/restore", + serde_json::json!({ "id": oldest, "base_version": preview["base_version"], "items": items }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{v}"); + assert_eq!(rows_of(&v), rows_of(&preview)); + assert_eq!(v["version"], serde_json::json!(b.version())); + let cfg = tw_config::try_parse(&b.file()).unwrap(); + assert_eq!(cfg.proxies.len(), 1); +} + +/// 恢复同步位置上的版本：本机不同步的上游（和指向它的路由）那一版里本来就没有，不列成删除 +#[tokio::test] +async fn restoring_a_synced_version_leaves_what_this_machine_does_not_sync() { + let d = tempfile::tempdir().unwrap(); + let here = format!( + "{}sync:\n device_id: 3f9a2c1e0b1d4e2f\n device_name: srv\n encrypt: false\n exclude: [ollama]\n target:\n type: folder\n path: {}\n", + mine().replace( + "routes:", + " - name: ollama\n base_url: http://127.0.0.1:11434\nroutes:\n - name: local\n rules:\n - name: all\n to: ollama\n" + ), + d.path().join("sync").display() + ); + tw_config::try_parse(&here).unwrap_or_else(|e| panic!("{e}\n{here}")); + // 同步位置上的那一版：没有 ollama 和 local，官方的地址不一样 + let synced = { + let mut m = tw_pack::Manifest::new(tw_pack::PackKind::Full, tw_pack::PackOrigin::Sync); + m.secrets = true; + tw_pack::Pack { + manifest: m, + config_yaml: mine() + .replace("https://api.anthropic.com\n", "https://synced.example\n") + .replace(&format!("listen:\n control:\n key: {KEY}\n"), ""), + plugins: Default::default(), + client_prefs: None, + } + }; + let b = bed(&here); + b.state + .cfg + .set_remote_backups(Some(Arc::new(OneRemote(synced)))); + let (_, list) = call(&b.app, "GET", "/backups", serde_json::json!(null)).await; + let id = list + .as_array() + .unwrap() + .iter() + .find(|e| e["id"].as_str().unwrap().starts_with("sync-")) + .unwrap()["id"] + .clone(); + let (st, preview) = call( + &b.app, + "POST", + "/backups/preview", + serde_json::json!({ "id": id }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{preview}"); + assert_eq!(rows_of(&preview), ["modify provider/官方"]); + let (st, v) = call( + &b.app, + "POST", + "/backups/restore", + serde_json::json!({ "id": id, "base_version": preview["base_version"] }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{v}"); + let cfg = tw_config::try_parse(&b.file()).unwrap(); + let names: Vec<&str> = cfg.providers.iter().map(|p| p.name.as_str()).collect(); + assert_eq!(names, ["官方", "ollama"]); + assert_eq!(cfg.providers[0].base_url, "https://synced.example"); + assert!(cfg.routes.iter().any(|r| r.name == "local")); +} + +/// 固定密钥：来源不过期，有上限；早先的文件照样算；同时报的几个来源谁也不盖掉谁 +#[tokio::test] +async fn pinned_keys_have_limits_and_an_older_file_still_counts() { + let d = tempfile::tempdir().unwrap(); + let p = d.path().join("config.yaml"); + std::fs::write(&p, mine()).unwrap(); + std::fs::write( + tw_pack::disk::pinned_path(&p), + br#"{"names":["claude-code"]}"#, + ) + .unwrap(); + let gw = tw_gateway::AppState::new(tw_config::try_parse(&mine()).unwrap()).unwrap(); + let bus = gw.bus.clone(); + let mgr = Arc::new(ConfigManager::new(p.clone(), gw.clone(), bus)); + assert!(mgr.pinned_keys().contains("claude-code")); + let app = tw_control::router(state_for(p.clone(), mgr.clone(), gw)); + for (body, code) in [ + ( + serde_json::json!({ "source": "x".repeat(129), "names": ["default"] }), + "backup.pinned.source_too_long", + ), + ( + serde_json::json!({ "source": "a", "names": ["n".repeat(257)] }), + "backup.pinned.name_too_long", + ), + ( + serde_json::json!({ "source": "a", "names": (0..1025).map(|i| i.to_string()).collect::>() }), + "backup.pinned.too_many", + ), + ] { + let (st, v) = call(&app, "PUT", "/pinned-keys", body).await; + assert_eq!(st, StatusCode::BAD_REQUEST, "{v}"); + assert_eq!(v["code"], code); + } + let tasks: Vec<_> = (0..16) + .map(|i| { + let app = app.clone(); + tokio::spawn(async move { + call( + &app, + "PUT", + "/pinned-keys", + serde_json::json!({ "source": format!("lite-{i}"), "names": [format!("k{i}")] }), + ) + .await + .0 + }) + }) + .collect(); + for t in tasks { + assert_eq!(t.await.unwrap(), StatusCode::OK); + } + let on_disk = tw_pack::disk::read_pinned(&p); + for i in 0..16 { + assert!(on_disk.contains(&format!("k{i}")), "{on_disk:?}"); + } + assert!(on_disk.contains("claude-code")); + assert_eq!(on_disk, mgr.pinned_keys()); +} + +/// 已经存着一份超大的通用偏好（上限之前存的）：恢复、导入照样能做，只有要写下去的新偏好才查 +#[tokio::test] +async fn a_large_preference_already_here_does_not_block_a_restore() { + let b = bed(&mine()); + let big = serde_json::json!({ "big": "x".repeat(tw_api::CLIENT_PREFS_MAX + 1) }); + std::fs::write( + tw_pack::disk::prefs_path(&b.path()), + serde_json::to_vec(&big).unwrap(), + ) + .unwrap(); + b.state.cfg.record_startup(); + b.state + .cfg + .write( + &mine().replace("https://api.anthropic.com\n", "https://changed.example\n"), + None, + tw_config::history::Origin::Ui, + ) + .await + .unwrap(); + let (_, list) = call(&b.app, "GET", "/backups", serde_json::json!(null)).await; + let oldest = list.as_array().unwrap().last().unwrap()["id"].clone(); + let (_, preview) = call( + &b.app, + "POST", + "/backups/preview", + serde_json::json!({ "id": oldest }), + ) + .await; + let (st, v) = call( + &b.app, + "POST", + "/backups/restore", + serde_json::json!({ "id": oldest, "base_version": preview["base_version"] }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{v}"); + assert_eq!(tw_pack::disk::read_prefs(&b.path()), Some(big)); +} + +/// 启动时从备份里找回插件文件（写插件文件和写配置之间断过电）：网关照找回来的重读，插件 +/// 照常跑。读不成 YAML 的原文不记成版本 +#[tokio::test] +async fn a_plugin_put_back_at_startup_runs_again() { + use tw_gateway::plugin::fake::{FakeEngine, source}; + let v1 = source( + serde_json::json!({"name": "A", "api": 1, "permissions": ["system"]}), + &["onRequest"], + ); + let v2 = source( + serde_json::json!({"name": "A2", "api": 1, "permissions": ["system"]}), + &["onRequest"], + ); + let sha = tw_gateway::plugin::load::sha256_hex(v1.as_bytes()); + let text = format!( + "{}plugins:\n - id: a\n file: plugins/a.js\n sha256: {sha}\n", + mine() + ); + let d = tempfile::tempdir().unwrap(); + let p = d.path().join("config.yaml"); + std::fs::write(&p, &text).unwrap(); + let files = |s: &str| tw_pack::PluginFiles { + script: s.as_bytes().to_vec(), + approved: Some(s.as_bytes().to_vec()), + }; + let one = |s: &str| std::collections::BTreeMap::from([("a".to_string(), files(s))]); + tw_pack::disk::write_plugins(&p, &Default::default(), &one(&v1)).unwrap(); + tw_pack::backup::record(&p, &text, tw_config::history::Origin::Ui).unwrap(); + // 断在半路：两份文件都已经是新的，配置还是旧的 + tw_pack::disk::write_plugins(&p, &one(&v1), &one(&v2)).unwrap(); + let gw = tw_gateway::AppState::new(tw_config::try_parse(&text).unwrap()).unwrap(); + gw.set_plugin_engine(Arc::new(FakeEngine)); + let bus = gw.bus.clone(); + let mgr = Arc::new(ConfigManager::new(p.clone(), gw.clone(), bus)); + let app = tw_control::router(state_for(p.clone(), mgr.clone(), gw)); + let status = || async { + let (_, v) = call(&app, "GET", "/plugins", serde_json::json!(null)).await; + v[0]["status"].clone() + }; + assert_eq!(status().await, serde_json::json!({"kind": "changed"})); + mgr.record_startup(); + assert_eq!(status().await, serde_json::json!({"kind": "ok"})); + assert_eq!( + std::fs::read_to_string(tw_config::plugins::file_path(d.path(), "a")).unwrap(), + v1 + ); + + let before = tw_pack::backup::list(&p).unwrap().len(); + std::fs::write(&p, "listen: [\n :::\n").unwrap(); + mgr.record_startup(); + assert_eq!(tw_pack::backup::list(&p).unwrap().len(), before); +} diff --git a/crates/tw-pack/src/backup.rs b/crates/tw-pack/src/backup.rs index d95315ff..a4e292ff 100644 --- a/crates/tw-pack/src/backup.rs +++ b/crates/tw-pack/src/backup.rs @@ -133,7 +133,7 @@ pub fn record( text: &str, origin: Origin, ) -> Result, BackupError> { - if origin == Origin::Rotation { + if origin == Origin::Rotation || !readable(text) { return Ok(None); } let pack = disk::current(config_path, text, PackOrigin::from(origin)); @@ -157,7 +157,7 @@ pub fn record_before( text: &str, origin: Origin, ) -> Result, BackupError> { - if origin == Origin::Rotation { + if origin == Origin::Rotation || !readable(text) { return Ok(None); } let all = list(config_path)?; @@ -172,6 +172,23 @@ pub fn record_before( save(config_path, &pack, Origin::External, all.last()).map(Some) } +/// 读得成 YAML（一个映射）吗。**读不成的原文不记成版本**（安全模式下启动时、改坏了的文件）： +/// 恢复不了它，它还会挤掉一个恢复得了的版本，整份恢复取本机那几节时也取不到东西 +fn readable(text: &str) -> bool { + serde_yaml_ng::from_str::(text).is_ok_and(|v| v.is_mapping()) +} + +/// 从新往旧第一个读得成 YAML、和现在这份原文不一样的本机版本：配置坏到读不成 YAML 时，整份 +/// 恢复从这里取本机那几节（[`crate::restore::whole`]） +pub fn last_readable(config_path: &Path, current_text: &str) -> Option { + let now = text_fingerprint(&tw_config::control_key::mask(current_text)); + list(config_path).ok()?.iter().rev().find_map(|b| { + open(b) + .ok() + .filter(|p| readable(&p.config_yaml) && text_fingerprint(&p.config_yaml) != now) + }) +} + fn masked(pack: &Pack) -> Pack { Pack { config_yaml: tw_config::control_key::mask(&pack.config_yaml), @@ -252,11 +269,16 @@ pub fn heal_plugins(config_path: &Path) -> Vec { let mut healed = Vec::new(); let mut backups: Option> = None; for (id, sha) in entries { - let files = disk::read_plugin(config_path, &id); - let fine = files.as_ref().is_some_and(|f| { - hash(&f.script) == sha || f.approved.as_ref().is_some_and(|a| hash(a) == sha) - }); - if fine { + // **只在脚本和底稿都对不上批准的哈希时**才找回来（写插件文件和写配置之间断过电）。 + // 底稿对得上、脚本对不上是用户改了脚本还没批准：不动它，网关照常把它停用、等批准。 + // 没有底稿的说不清是哪一种，也不动；脚本文件没了的找回来，盖不掉谁的改动 + let heal = match disk::read_plugin(config_path, &id) { + None => true, + Some(f) => { + hash(&f.script) != sha && f.approved.as_ref().is_some_and(|a| hash(a) != sha) + } + }; + if !heal { continue; } let packs = backups.get_or_insert_with(|| { @@ -481,7 +503,7 @@ mod tests { }; let sha = crate::model::sha256_hex(b"v1"); let text = format!( - "{}plugins:\n - id: a\n file: plugins/a.js\n sha256: {sha}\n - id: b\n file: plugins/b.js\n sha256: {sha}\n", + "{}plugins:\n - id: a\n file: plugins/a.js\n sha256: {sha}\n - id: b\n file: plugins/b.js\n sha256: {sha}\n - id: c\n file: plugins/c.js\n sha256: {sha}\n", cfg(1) ); std::fs::write(&p, &text).unwrap(); @@ -491,19 +513,22 @@ mod tests { &[ ("a".to_string(), files("v1", Some("v1"))), ("b".to_string(), files("v1", Some("v1"))), + ("c".to_string(), files("v1", Some("v1"))), ] .into_iter() .collect(), ) .unwrap(); record(&p, &text, Origin::Ui).unwrap().unwrap(); - // a：两份都换成了新的（断在半路）；b：用户改了脚本，底稿还是批准的那份 + // a：两份都换成了新的（断在半路）；b：用户改了脚本，底稿还是批准的那份；c：用户改了 + // 脚本，底稿没了（说不清是哪一种，不动） disk::write_plugins( &p, &Default::default(), &[ ("a".to_string(), files("v2", Some("v2"))), ("b".to_string(), files("edited", Some("v1"))), + ("c".to_string(), files("edited", None)), ] .into_iter() .collect(), @@ -512,9 +537,33 @@ mod tests { assert_eq!(heal_plugins(&p), ["a"]); assert_eq!(disk::read_plugin(&p, "a").unwrap().script, b"v1"); assert_eq!(disk::read_plugin(&p, "b").unwrap().script, b"edited"); + assert_eq!(disk::read_plugin(&p, "c").unwrap().script, b"edited"); assert!(heal_plugins(&p).is_empty()); } + /// 读不成 YAML 的原文不记成版本；整份恢复取本机那几节时从新往旧找第一个读得成的 + #[test] + fn unreadable_text_is_no_version_and_the_last_readable_one_is_found() { + let (_d, p) = setup(); + record(&p, &cfg(1), Origin::Ui).unwrap().unwrap(); + record(&p, &cfg(2), Origin::Ui).unwrap().unwrap(); + let garbage = "listen: [\n :::\n"; + assert!(serde_yaml_ng::from_str::(garbage).is_err()); + assert!(record(&p, garbage, Origin::External).unwrap().is_none()); + assert!(record_before(&p, garbage, Origin::Ui).unwrap().is_none()); + assert!( + record(&p, "just a string\n", Origin::External) + .unwrap() + .is_none() + ); + assert_eq!(list(&p).unwrap().len(), 2); + let last = last_readable(&p, garbage).unwrap(); + assert!(last.config_yaml.contains("port: 2"), "{}", last.config_yaml); + // 和现在这份一样的不算（它要是读得成，本机那几节本来就从现在这份取） + let last = last_readable(&p, &cfg(2)).unwrap(); + assert!(last.config_yaml.contains("port: 1"), "{}", last.config_yaml); + } + #[test] fn a_rotation_leaves_no_version_and_does_not_count_as_a_change() { let (_d, p) = setup(); diff --git a/crates/tw-pack/src/disk.rs b/crates/tw-pack/src/disk.rs index 16340e15..c7052385 100644 --- a/crates/tw-pack/src/disk.rs +++ b/crates/tw-pack/src/disk.rs @@ -38,9 +38,23 @@ pub fn read_prefs(config_path: &Path) -> Option { } /// 存下通用偏好。None 是删掉 +/// 这份通用偏好超出了上限吗（[`tw_api::CLIENT_PREFS_MAX`]，写成紧凑的 JSON 之后） +pub fn prefs_too_large(prefs: &serde_json::Value) -> bool { + prefs.to_string().len() > tw_api::CLIENT_PREFS_MAX +} + +/// 存下通用偏好，None 是删掉。**超出上限的不写**（`InvalidInput`）：界面、导入、恢复、同步、 +/// 命令行写偏好都走这里，谁也绕不过去 pub fn write_prefs(config_path: &Path, prefs: Option<&serde_json::Value>) -> std::io::Result<()> { let path = prefs_path(config_path); match prefs { + Some(p) if prefs_too_large(p) => Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!( + "the client preferences are larger than {} KiB", + tw_api::CLIENT_PREFS_MAX / 1024 + ), + )), Some(p) => { let bytes = serde_json::to_vec_pretty(p).map_err(std::io::Error::other)?; write_private(&path, &bytes) @@ -52,63 +66,141 @@ pub fn write_prefs(config_path: &Path, prefs: Option<&serde_json::Value>) -> std } } -/// 一个来源（一个连着这台 core 的 Lite）报上来的固定密钥多久没更新就不算了：30 天 -pub const PINNED_TTL_MS: u64 = 30 * 24 * 3600 * 1000; +/// 最多记几个来源。多出来的去掉最久没报过的那一个：**来源不过期**（Lite 每次连上 core 都 +/// 重报一次，一个来源再报是换掉它自己那一组），这个上限挡的是没完没了的新来源 +pub const PINNED_SOURCES_MAX: usize = 64; +/// 来源标识最长几个字节 +pub const PINNED_SOURCE_MAX: usize = 128; +/// 一个来源最多报几把密钥 +pub const PINNED_NAMES_MAX: usize = 1024; +/// 一把密钥的名字最长几个字节 +pub const PINNED_NAME_MAX: usize = 256; /// 一个来源报上来的固定密钥。 #[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct PinnedSource { pub names: BTreeSet, + /// 最近一次报上来的时候。来源多过 [`PINNED_SOURCES_MAX`] 时去掉最久没报的 + #[serde(default)] pub updated_at_ms: u64, } /// 存着的固定密钥，**按来源**：几个 Lite 连着同一个 core（远程模式）时各报各的，取并集。 -/// 没有、读不懂都是空的 +/// 没有、读不懂都是空的。早先的写法（一个名字数组，或者 `{"names": [...]}`）读作来源空串 pub fn read_pinned_sources(config_path: &Path) -> BTreeMap { #[derive(serde::Deserialize)] - struct File { - sources: BTreeMap, + #[serde(untagged)] + enum File { + Sources { + sources: BTreeMap, + }, + Names { + names: BTreeSet, + }, + Plain(BTreeSet), } - std::fs::read(pinned_path(config_path)) + let Some(file) = std::fs::read(pinned_path(config_path)) .ok() .and_then(|b| serde_json::from_slice::(&b).ok()) - .map(|f| f.sources) - .unwrap_or_default() + else { + return BTreeMap::new(); + }; + match file { + File::Sources { sources } => sources, + File::Names { names } | File::Plain(names) => BTreeMap::from([( + String::new(), + PinnedSource { + names, + updated_at_ms: 0, + }, + )]), + } } -/// 各来源的并集，**过期的来源不算**（[`PINNED_TTL_MS`] 没更新：那个 Lite 多半已经不在了） -pub fn pinned_union(sources: &BTreeMap, now_ms: u64) -> BTreeSet { +/// 各来源的并集 +pub fn pinned_union(sources: &BTreeMap) -> BTreeSet { sources .values() - .filter(|s| now_ms.saturating_sub(s.updated_at_ms) <= PINNED_TTL_MS) .flat_map(|s| s.names.iter().cloned()) .collect() } -/// 存着的固定密钥：各个来源的并集，过期的不算 +/// 存着的固定密钥：各个来源的并集 pub fn read_pinned(config_path: &Path) -> BTreeSet { - pinned_union(&read_pinned_sources(config_path), crate::now_ms()) + pinned_union(&read_pinned_sources(config_path)) } -/// 一个来源报上来一组固定密钥：换掉它上次报的，过期的来源顺手删掉。返回存下的全部来源 -pub fn write_pinned( - config_path: &Path, +/// 报上来的一组超出了上限 +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PinnedTooLarge { + Source, + Names, + Name, +} + +/// 一个来源报上来一组固定密钥：换掉它上次报的。来源多过上限时去掉最久没报过的。**不写盘** +pub fn with_pinned( + sources: &BTreeMap, source: &str, names: &BTreeSet, -) -> std::io::Result> { - let now = crate::now_ms(); - let mut sources = read_pinned_sources(config_path); - sources.retain(|_, s| now.saturating_sub(s.updated_at_ms) <= PINNED_TTL_MS); - sources.insert( + now_ms: u64, +) -> Result, PinnedTooLarge> { + if source.len() > PINNED_SOURCE_MAX { + return Err(PinnedTooLarge::Source); + } + if names.len() > PINNED_NAMES_MAX { + return Err(PinnedTooLarge::Names); + } + if names.iter().any(|n| n.len() > PINNED_NAME_MAX) { + return Err(PinnedTooLarge::Name); + } + let mut out = sources.clone(); + out.insert( source.to_string(), PinnedSource { names: names.clone(), - updated_at_ms: now, + updated_at_ms: now_ms, }, ); + while out.len() > PINNED_SOURCES_MAX { + let Some(oldest) = out + .iter() + .filter(|(k, _)| k.as_str() != source) + .min_by_key(|(_, s)| s.updated_at_ms) + .map(|(k, _)| k.clone()) + else { + break; + }; + out.remove(&oldest); + } + Ok(out) +} + +/// 存下全部来源 +pub fn write_pinned_sources( + config_path: &Path, + sources: &BTreeMap, +) -> std::io::Result<()> { let bytes = serde_json::to_vec_pretty(&serde_json::json!({ "sources": sources })) .map_err(std::io::Error::other)?; - write_private(&pinned_path(config_path), &bytes)?; + write_private(&pinned_path(config_path), &bytes) +} + +/// 一个来源报上来一组固定密钥，读文件、换掉、写回去。**只给单独一个进程用**（命令行、测试）； +/// 跑着的 core 在锁里改它记着的那一份（`ConfigManager::set_pinned_keys`） +pub fn write_pinned( + config_path: &Path, + source: &str, + names: &BTreeSet, +) -> std::io::Result> { + let sources = with_pinned( + &read_pinned_sources(config_path), + source, + names, + crate::now_ms(), + ) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, format!("{e:?}")))?; + write_pinned_sources(config_path, &sources)?; Ok(sources) } @@ -315,6 +407,11 @@ mod tests { write_prefs(&cfg, None).unwrap(); assert_eq!(read_prefs(&cfg), None); write_prefs(&cfg, None).unwrap(); + // 超出上限的不写 + let big = serde_json::json!({ "big": "x".repeat(tw_api::CLIENT_PREFS_MAX) }); + let e = write_prefs(&cfg, Some(&big)).unwrap_err(); + assert_eq!(e.kind(), std::io::ErrorKind::InvalidInput); + assert_eq!(read_prefs(&cfg), None); std::fs::write(prefs_path(&cfg), b"{not json").unwrap(); assert_eq!(read_prefs(&cfg), None); @@ -326,10 +423,53 @@ mod tests { assert_eq!(read_pinned(&cfg), a.union(&b).cloned().collect()); write_pinned(&cfg, "lite-a", &BTreeSet::new()).unwrap(); assert_eq!(read_pinned(&cfg), b); - // 30 天没更新的来源不算 + // 来源不过期：很久以前报的照样算 let mut sources = read_pinned_sources(&cfg); sources.get_mut("lite-b").unwrap().updated_at_ms = 1; - assert!(pinned_union(&sources, crate::now_ms()).is_empty()); + assert_eq!(pinned_union(&sources), b); + } + + /// 早先的写法读作来源空串，不读成空的 + #[test] + fn an_older_pinned_file_reads_as_the_blank_source() { + let d = tempfile::tempdir().unwrap(); + let cfg = d.path().join("config.yaml"); + let codex: BTreeSet = ["codex".to_string()].into(); + for old in [&br#"{"names":["codex"]}"#[..], br#"["codex"]"#] { + std::fs::write(pinned_path(&cfg), old).unwrap(); + assert_eq!(read_pinned(&cfg), codex); + assert_eq!(read_pinned_sources(&cfg)[""].names, codex); + } + // 再报一次（有来源的）：早先那一组还在，并集 + let b: BTreeSet = ["claude-code".to_string()].into(); + write_pinned(&cfg, "lite-a", &b).unwrap(); + assert_eq!(read_pinned(&cfg), codex.union(&b).cloned().collect()); + } + + #[test] + fn pinned_sources_have_limits() { + let none = BTreeMap::new(); + let one: BTreeSet = ["k".to_string()].into(); + let long = "x".repeat(PINNED_SOURCE_MAX + 1); + assert_eq!( + with_pinned(&none, &long, &one, 1), + Err(PinnedTooLarge::Source) + ); + let many: BTreeSet = (0..=PINNED_NAMES_MAX).map(|i| i.to_string()).collect(); + assert_eq!( + with_pinned(&none, "a", &many, 1), + Err(PinnedTooLarge::Names) + ); + let wide: BTreeSet = ["n".repeat(PINNED_NAME_MAX + 1)].into(); + assert_eq!(with_pinned(&none, "a", &wide, 1), Err(PinnedTooLarge::Name)); + // 来源多过上限：去掉最久没报过的那一个 + let mut all = BTreeMap::new(); + for i in 0..PINNED_SOURCES_MAX { + all = with_pinned(&all, &format!("s{i}"), &one, 10 + i as u64).unwrap(); + } + let all = with_pinned(&all, "new", &one, 1000).unwrap(); + assert_eq!(all.len(), PINNED_SOURCES_MAX); + assert!(!all.contains_key("s0") && all.contains_key("s1") && all.contains_key("new")); } #[test] diff --git a/crates/tw-pack/src/export.rs b/crates/tw-pack/src/export.rs index bc0b162b..fffad476 100644 --- a/crates/tw-pack/src/export.rs +++ b/crates/tw-pack/src/export.rs @@ -165,8 +165,15 @@ pub fn strip_secrets(snap: &mut Snapshot) { /// 不带密钥的包里的一项，**空着的密钥换成本机同一项里的**：上游的密钥、请求头的值（按名字）、 /// AWS 访问密钥、OAuth 的客户端密钥，代理的密码，网关密钥的值。比较和覆盖都照这个样子：除了 -/// 密钥都一样的就是「相同」，覆盖不会把本机的密钥抹成空的 +/// 密钥都一样的就是「相同」，覆盖不会把本机的密钥抹成空的。 +/// +/// **只在目的地不变时**（[`same_destination`]）：包里的上游换了主机、端口、AWS 区域或 OAuth +/// 令牌端点，代理换了地址，本机的密钥就不跟过去 —— 否则一份不带密钥的包改个地址，覆盖之后 +/// 本机的密钥就发给了包里写的那个地方 pub fn fill_secrets(id: &ItemId, item: &mut model::Item, mine: &model::Item) { + if !same_destination(id, item, mine) { + return; + } let empty = |v: Option<&Json>| v.and_then(Json::as_str) == Some(""); let take = |at: Option<&mut Json>, from: Option<&Json>| { if let (Some(v), Some(f)) = (at, from) @@ -237,6 +244,67 @@ pub fn fill_secrets(id: &ItemId, item: &mut model::Item, mine: &model::Item) { } } +/// 两份同名的项目，密钥是发去同一个地方的吗：上游的协议、主机、端口一样，AWS 区域一样， +/// OAuth 的令牌端点一样；代理的类型和地址一样。网关密钥是发给这台网关自己的，总是 +pub fn same_destination(id: &ItemId, theirs: &model::Item, mine: &model::Item) -> bool { + let at = |it: &model::Item, u: &str, path: &[&str]| -> Option { + let mut cur = it.units.get(u)?; + for p in path { + cur = cur.get(*p)?; + } + Some(cur.clone()) + }; + match id.kind { + ItemKind::Provider => { + let url = |it: &model::Item| { + at(it, unit::CONN, &["base_url"]).and_then(|v| v.as_str().and_then(destination)) + }; + url(theirs).is_some() + && url(theirs) == url(mine) + && at(theirs, unit::CONN, &["aws", "region"]) + == at(mine, unit::CONN, &["aws", "region"]) + && at(theirs, unit::CONN, &["oauth", "endpoint"]) + == at(mine, unit::CONN, &["oauth", "endpoint"]) + } + ItemKind::Proxy => { + let addr = |it: &model::Item| { + at(it, unit::ALL, &["addr"]) + .and_then(|v| v.as_str().map(|a| a.trim().to_ascii_lowercase())) + }; + addr(theirs).is_some() + && addr(theirs) == addr(mine) + && at(theirs, unit::ALL, &["type"]) == at(mine, unit::ALL, &["type"]) + } + _ => true, + } +} + +/// 一个地址发去哪儿：协议、主机（小写）、端口（没写时按协议的缺省）。读不懂是 None +pub fn destination(url: &str) -> Option<(String, String, u16)> { + let (scheme, rest) = url.trim().split_once("://")?; + let scheme = scheme.to_ascii_lowercase(); + let authority = rest.split(['/', '?', '#']).next()?; + let hostport = authority.rsplit('@').next()?; + let (host, port) = if let Some(v6) = hostport.strip_prefix('[') { + let (h, after) = v6.split_once(']')?; + (h.to_string(), after.strip_prefix(':')) + } else { + match hostport.rsplit_once(':') { + Some((h, p)) => (h.to_string(), Some(p)), + None => (hostport.to_string(), None), + } + }; + let port = match port { + Some(p) => p.parse().ok()?, + None => match scheme.as_str() { + "https" => 443, + "http" => 80, + _ => return None, + }, + }; + (!host.is_empty()).then(|| (scheme, host.to_ascii_lowercase(), port)) +} + /// 一项里有没有被去掉的密钥（空字符串） pub fn missing_secret(id: &ItemId, item: &model::Item) -> bool { let empty = |v: Option<&Json>| v.and_then(Json::as_str) == Some(""); @@ -260,6 +328,12 @@ pub fn missing_secret(id: &ItemId, item: &model::Item) -> bool { .is_some_and(|o| empty(o.get("refresh")) || empty(o.get("client_secret"))) } ItemKind::Key => empty(item.units.get(unit::ALL).and_then(|a| a.get("key"))), + ItemKind::Proxy => empty( + item.units + .get(unit::ALL) + .and_then(|a| a.get("auth")) + .and_then(|a| a.get("pass")), + ), _ => false, } } @@ -288,6 +362,26 @@ fn civil(days: u64) -> (i64, u32, u32) { mod tests { use super::*; + #[test] + fn a_destination_is_scheme_host_and_port() { + let d = |u| destination(u); + assert_eq!( + d("https://Relay.example/v1"), + Some(("https".into(), "relay.example".into(), 443)) + ); + assert_eq!( + d("https://relay.example:443/x"), + d("https://relay.example/y") + ); + assert_ne!(d("https://relay.example:8443"), d("https://relay.example")); + assert_ne!(d("http://relay.example"), d("https://relay.example")); + assert_eq!( + d("http://user:pw@[::1]:8080/p"), + Some(("http".into(), "::1".into(), 8080)) + ); + assert_eq!(d("not a url"), None); + } + #[test] fn the_file_name_carries_the_date() { assert_eq!(file_name(0), "ThinkWatch-1970-01-01.twconfig"); diff --git a/crates/tw-pack/src/import.rs b/crates/tw-pack/src/import.rs index b11daf90..d61b6ea8 100644 --- a/crates/tw-pack/src/import.rs +++ b/crates/tw-pack/src/import.rs @@ -501,7 +501,9 @@ pub fn apply_with( codec: &dyn PluginCodec, ) -> Result { let l = model::snapshot_with(local, codec)?; - let mut i = with_local_secrets(&l, model::snapshot_with(incoming, codec)?, incoming); + // **不在这里沿用本机的密钥**：改名导入的是另一个项目，不该带走本机那一项的密钥；覆盖时 + // 逐项沿用（目的地不变才沿用，见 `export::fill_secrets`） + let mut i = model::snapshot_with(incoming, codec)?; let rows = preview_with(local, incoming, ctx, codec)?; let errors: BTreeMap<&ItemId, &Msg> = rows .iter() @@ -548,12 +550,21 @@ pub fn apply_with( { return Err(ImportError::Item(d.item.clone(), Box::new((*e).clone()))); } - let Some(item) = i.items.get(&id) else { + let Some(raw) = i.items.get(&id) else { if d.action == Action::Skip { continue; } return Err(ImportError::NotInPack(d.item.clone())); }; + // 不带密钥的包：覆盖本机同名的那一项时沿用本机的密钥（目的地不变才沿用） + let mut item = raw.clone(); + if !incoming.manifest.secrets + && !matches!(d.action, Action::Rename(_)) + && let Some(mine) = l.items.get(&id) + { + export::fill_secrets(&id, &mut item, mine); + } + let item = &item; match &d.action { Action::Skip => continue, Action::Import if l.items.contains_key(&id) && !is_global(id.kind) => { diff --git a/crates/tw-pack/src/lib.rs b/crates/tw-pack/src/lib.rs index f32736b5..746c8be0 100644 --- a/crates/tw-pack/src/lib.rs +++ b/crates/tw-pack/src/lib.rs @@ -308,6 +308,10 @@ pub enum PackError { /// 要求加密的地方读到了一个没加密的包（[`read_strict`]） #[error("{}", self.msg())] NotEncrypted, + /// 口令加密的工作量超出了这一版肯解的上限（[`MAX_WORK_FACTOR`]）。不是包坏了：别的工具 + /// 用更高的工作量加密的 + #[error("{}", self.msg())] + TooMuchWork { required: u8, max: u8 }, } impl PackError { @@ -358,6 +362,11 @@ impl PackError { "pack.too_large", limit = limit / (1024 * 1024) => "This configuration pack is larger than {limit} MiB." ), + PackError::TooMuchWork { required, max } => msg!( + "pack.too_much_work", required = required, max = max => + "The passphrase encryption of this configuration pack takes more work than this \ + version allows (work factor {required}, at most {max}), so it was not opened." + ), } } } @@ -524,6 +533,10 @@ fn decrypt(bytes: &[u8], unseal: Unseal) -> Result, PackError> { age::DecryptError::DecryptionFailed | age::DecryptError::NoMatchingKeys => { PackError::WrongPassphrase } + age::DecryptError::ExcessiveWork { required, .. } => PackError::TooMuchWork { + required, + max: MAX_WORK_FACTOR, + }, other => corrupt(other), })? } else { diff --git a/crates/tw-pack/src/restore.rs b/crates/tw-pack/src/restore.rs index 289756b6..281002f6 100644 --- a/crates/tw-pack/src/restore.rs +++ b/crates/tw-pack/src/restore.rs @@ -40,6 +40,20 @@ pub struct Row { pub kept: Option, } +/// 恢复的条件。 +#[derive(Debug, Clone, Default)] +pub struct Options { + /// 固定密钥的名字：接管的客户端在用，不删 + pub pinned: BTreeSet, + /// 通用偏好算不算。经远程连接恢复时不算：偏好在 Lite 那边 + pub with_prefs: bool, + /// 只恢复这几项（连同它们要的，见 [`plan`]），None 是全部 + pub items: Option>, + /// 不参与比较、保持现在样子的项目：恢复同步位置上的版本时，本机不同步的上游和随之不同步 + /// 的项目（规格 7.2）—— 那一版里本来就没有它们，不是删了 + pub skip: BTreeSet, +} + /// 那一版和现在的差异。`pinned` 是固定密钥的名字。`with_prefs` 为 false 时通用偏好不算 /// （经远程连接恢复：偏好在 Lite 那边） pub fn preview( @@ -48,19 +62,130 @@ pub fn preview( pinned: &BTreeSet, with_prefs: bool, ) -> Result, ModelError> { - preview_with(local, backup, pinned, with_prefs, model::codec()) + preview_opts( + local, + backup, + &Options { + pinned: pinned.clone(), + with_prefs, + ..Default::default() + }, + ) +} + +/// 照这些条件恢复会动哪几项。**和 [`apply_opts`] 是同一个计划**：只选了几项时，预览列出的 +/// 就是写下去的（连同一起恢复的、留着的） +pub fn preview_opts(local: &Pack, backup: &Pack, opts: &Options) -> Result, ModelError> { + preview_with(local, backup, opts, model::codec()) } pub fn preview_with( local: &Pack, backup: &Pack, - pinned: &BTreeSet, - with_prefs: bool, + opts: &Options, codec: &dyn PluginCodec, ) -> Result, ModelError> { let l = model::snapshot_with(local, codec)?; let b = model::snapshot_with(backup, codec)?; - Ok(plan(&l, &b, pinned, with_prefs, None).0) + Ok(plan(&l, &b, &local_refs(local), opts).0) +} + +/// 本机那几节（[`model::LOCAL_SECTIONS`]）引用的项目：同步设置走的出站代理。恢复留着本机那 +/// 几节，它们引用的就得留着 —— 那一版里没有这个代理的话，删了它同步设置就不成立 +fn local_refs(local: &Pack) -> BTreeSet { + local_refs_in(&local.config_yaml) +} + +fn local_refs_in(text: &str) -> BTreeSet { + fn walk(v: &serde_yaml_ng::Value, out: &mut BTreeSet) { + match v { + serde_yaml_ng::Value::Mapping(m) => { + for (k, v) in m { + if k.as_str() == Some("proxy") + && let Some(name) = v.as_str() + && name != tw_config::DIRECT + && name != tw_config::SYSTEM + { + out.insert(ItemId::new(ItemKind::Proxy, name)); + } + walk(v, out); + } + } + serde_yaml_ng::Value::Sequence(xs) => xs.iter().for_each(|x| walk(x, out)), + _ => {} + } + } + let mut out = BTreeSet::new(); + if let Ok(doc) = serde_yaml_ng::from_str::(text) { + for section in model::LOCAL_SECTIONS { + if let Some(v) = doc.get(*section) { + walk(v, &mut out); + } + } + } + out +} + +/// 那一版，`skip` 里的项目换成现在的样子：现在有的照现在的，现在没有的不在；排列顺序里它们 +/// 照现在的位置 +fn overlay(l: &Snapshot, b: &Snapshot, skip: &BTreeSet) -> Snapshot { + if skip.is_empty() { + return b.clone(); + } + let mut out = b.clone(); + for id in skip { + match l.items.get(id) { + Some(it) => { + out.items.insert(id.clone(), it.clone()); + } + None => { + out.items.remove(id); + } + } + } + for kind in [ItemKind::Provider, ItemKind::Route, ItemKind::Plugin] { + let order = ItemId::new(kind.order().expect("has an order"), ""); + let skipped = |n: &String| skip.contains(&ItemId::new(kind, n.clone())); + let here = names_of(l.items.get(&order)); + let mut list: Vec = names_of(out.items.get(&order)) + .into_iter() + .filter(|n| !skipped(n)) + .collect(); + // 跳过的那几个插回现在的位置：紧跟在现在排在它前面、那一版里也有的那一个后面 + for (i, n) in here.iter().enumerate().filter(|(_, n)| skipped(n)) { + let at = here[..i] + .iter() + .rev() + .find_map(|p| list.iter().position(|x| x == p)) + .map_or(0, |p| p + 1); + list.insert(at, n.clone()); + } + if list.is_empty() && !out.items.contains_key(&order) { + continue; + } + out.items.insert( + order, + model::Item { + units: std::collections::BTreeMap::from([( + unit::ALL.to_string(), + Json::Array(list.into_iter().map(Json::String).collect()), + )]), + ..Default::default() + }, + ); + } + out.relink(); + out +} + +fn names_of(it: Option<&model::Item>) -> Vec { + match it.and_then(|it| it.units.get(unit::ALL)) { + Some(Json::Array(a)) => a + .iter() + .filter_map(|n| n.as_str().map(str::to_string)) + .collect(), + _ => Vec::new(), + } } /// 两份之间的差异，按项目 @@ -95,24 +220,26 @@ fn hard(from: &ItemId, to: &ItemId) -> bool { /// - 是这次要删的：留着（[`Kept::Referenced`]），比如留着的固定密钥绑的那条路由； /// - 是那一版里有、没选上的：一起恢复，比如只选了一个已删的上游、它用的出站代理也删了。 /// -/// 一直补到没有缺的为止 +/// 一直补到没有缺的为止。路由的去处写的是一个名字：上游里找不到时按同名的策略组找（组删了 +/// 之后它读成一个不存在的上游）。`local` 是本机那几节引用的项目（[`local_refs`]），和别的 +/// 引用一样总得留着 fn plan( l: &Snapshot, b: &Snapshot, - pinned: &BTreeSet, - with_prefs: bool, - selected: Option<&BTreeSet>, + local: &BTreeSet, + opts: &Options, ) -> (Vec, Snapshot) { - let changes = diff(l, b, with_prefs); + let b = &overlay(l, b, &opts.skip); + let changes = diff(l, b, opts.with_prefs); let mut chosen: BTreeSet = changes .iter() .map(|(id, _)| id.clone()) - .filter(|id| selected.is_none_or(|s| s.contains(id))) + .filter(|id| opts.items.as_ref().is_none_or(|s| s.contains(id))) .collect(); let mut kept: std::collections::BTreeMap = changes .iter() .filter(|(id, c)| { - *c == Change::Remove && id.kind == ItemKind::Key && pinned.contains(&id.name) + *c == Change::Remove && id.kind == ItemKind::Key && opts.pinned.contains(&id.name) }) .map(|(id, _)| (id.clone(), Kept::KeyInUse)) .collect(); @@ -143,8 +270,28 @@ fn plan( .cloned() .collect::>() }) + .chain( + local + .iter() + .filter(|r| !target.items.contains_key(*r)) + .cloned(), + ) .collect(); for r in missing { + // 指向一个名字的：上游里没有就是同名的策略组 + let group = ItemId::new(ItemKind::Group, r.name.clone()); + let r = if r.kind == ItemKind::Provider + && !l.items.contains_key(&r) + && !b.items.contains_key(&r) + && (l.items.contains_key(&group) || b.items.contains_key(&group)) + { + group + } else { + r + }; + if target.items.contains_key(&r) { + continue; + } if l.items.contains_key(&r) { // 这次要删的：留着 if let std::collections::btree_map::Entry::Vacant(e) = kept.entry(r) { @@ -179,6 +326,8 @@ pub struct Applied { pub changed_keys: Vec, /// 留着没删的：固定密钥，和留下来的项目还引用着的 pub kept: Vec, + /// 实际动了的那几项（连同一起恢复的、留着的），和同样条件下的预览一样 + pub rows: Vec, } /// 恢复。`items` 是只恢复这几项，None 是全部。所选项目引用的、本机已经没有的项目一起恢复 @@ -189,21 +338,31 @@ pub fn apply( pinned: &BTreeSet, with_prefs: bool, ) -> Result { - apply_with(local, backup, items, pinned, with_prefs, model::codec()) + apply_opts( + local, + backup, + &Options { + pinned: pinned.clone(), + with_prefs, + items: items.map(|xs| xs.iter().cloned().collect()), + ..Default::default() + }, + ) +} + +pub fn apply_opts(local: &Pack, backup: &Pack, opts: &Options) -> Result { + apply_with(local, backup, opts, model::codec()) } pub fn apply_with( local: &Pack, backup: &Pack, - items: Option<&[ItemId]>, - pinned: &BTreeSet, - with_prefs: bool, + opts: &Options, codec: &dyn PluginCodec, ) -> Result { let l = model::snapshot_with(local, codec)?; let b = model::snapshot_with(backup, codec)?; - let picked: Option> = items.map(|xs| xs.iter().cloned().collect()); - let (rows, mut target) = plan(&l, &b, pinned, with_prefs, picked.as_ref()); + let (rows, mut target) = plan(&l, &b, &local_refs(local), opts); let kept = rows .iter() .filter(|r| r.kept.is_some()) @@ -234,6 +393,7 @@ pub fn apply_with( changed_keys: import::changed_keys(local, &pack), pack, kept, + rows, }) } @@ -333,6 +493,38 @@ pub fn whole(backup: &Pack, current_text: &str, last_local: Option<&Pack>) -> Pa text = t; } } + // 本机那几节引用的出站代理（同步设置走的）那一版里没有：从取本机那几节的地方一起取来 + if let Some(src) = &source { + let named = |d: &Yaml, name: &str| -> Option { + d.get("proxies")? + .as_sequence()? + .iter() + .find(|p| p.get("name").and_then(Yaml::as_str) == Some(name)) + .cloned() + }; + for r in local_refs_in(&text) { + let Some(now) = doc(&text) else { break }; + if named(&now, &r.name).is_some() { + continue; + } + let Some(px) = named(src, &r.name) else { + continue; + }; + let mut list = now + .get("proxies") + .and_then(Yaml::as_sequence) + .cloned() + .unwrap_or_default(); + list.push(px); + if let Ok(t) = tw_config::edit::set( + &text, + &[tw_yaml::Step::key("proxies")], + Some(&Yaml::Sequence(list)), + ) { + text = t; + } + } + } if let Some(k) = real { let at: Vec = tw_api::control::KEY_PATH .iter() diff --git a/crates/tw-pack/src/restore/tests.rs b/crates/tw-pack/src/restore/tests.rs index 8e838879..aaa60de5 100644 --- a/crates/tw-pack/src/restore/tests.rs +++ b/crates/tw-pack/src/restore/tests.rs @@ -32,13 +32,22 @@ fn then() -> Pack { )) } +fn opts(items: Option<&[ItemId]>) -> Options { + Options { + pinned: pinned(), + with_prefs: true, + items: items.map(|xs| xs.iter().cloned().collect()), + ..Default::default() + } +} + fn pinned() -> BTreeSet { ["claude-code".to_string()].into_iter().collect() } #[test] fn the_preview_lists_what_differs_by_item() { - let rows = preview_with(&now(), &then(), &pinned(), true, &WholeFile).unwrap(); + let rows = preview_with(&now(), &then(), &opts(None), &WholeFile).unwrap(); let got: Vec<(String, Change, Option)> = rows .iter() .map(|r| (r.item.to_string(), r.change, r.kept)) @@ -62,7 +71,7 @@ fn the_preview_lists_what_differs_by_item() { #[test] fn restoring_everything_keeps_local_settings_logins_and_keys_in_use() { - let out = apply_with(&now(), &then(), None, &pinned(), true, &WholeFile).unwrap(); + let out = apply_with(&now(), &then(), &opts(None), &WholeFile).unwrap(); let cfg = tw_config::try_parse(&out.pack.config_yaml) .unwrap_or_else(|e| panic!("{e}\n{}", out.pack.config_yaml)); assert_eq!(cfg.listen.gateway.port, 9000, "本机设置留着现在的"); @@ -87,7 +96,7 @@ fn restoring_everything_keeps_local_settings_logins_and_keys_in_use() { #[test] fn restoring_a_few_items_leaves_the_rest_as_it_is() { let pick = [ItemId::new(ItemKind::Provider, "deleted")]; - let out = apply_with(&now(), &then(), Some(&pick), &pinned(), true, &WholeFile).unwrap(); + let out = apply_with(&now(), &then(), &opts(Some(&pick)), &WholeFile).unwrap(); let cfg = tw_config::try_parse(&out.pack.config_yaml).unwrap(); let names: Vec<&str> = cfg.providers.iter().map(|p| p.name.as_str()).collect(); assert_eq!(names, ["kept", "chatgpt", "deleted"]); @@ -122,7 +131,16 @@ fn oauth_tokens_come_from_now_or_from_the_version_it_comes_back_from() { oauth("gone", "rt-gone"), " - name: blank\n base_url: https://blank.example\n disabled: true\n oauth:\n refresh: ''\n endpoint: https://auth.example/token\n", )); - let out = apply_with(&now, &then, None, &BTreeSet::new(), true, &WholeFile).unwrap(); + let out = apply_with( + &now, + &then, + &Options { + with_prefs: true, + ..Default::default() + }, + &WholeFile, + ) + .unwrap(); let cfg = tw_config::try_parse(&out.pack.config_yaml) .unwrap_or_else(|e| panic!("{e}\n{}", out.pack.config_yaml)); let p = |n: &str| cfg.providers.iter().find(|p| p.name == n).unwrap(); diff --git a/crates/tw-pack/src/tests.rs b/crates/tw-pack/src/tests.rs index 1a7a78ed..d08c05d5 100644 --- a/crates/tw-pack/src/tests.rs +++ b/crates/tw-pack/src/tests.rs @@ -292,6 +292,14 @@ fn every_error_speaks_with_its_own_code() { "pack.missing_entry", ), (PackError::TooLarge { limit: MAX_BYTES }, "pack.too_large"), + (PackError::NotEncrypted, "pack.not_encrypted"), + ( + PackError::TooMuchWork { + required: 22, + max: 18, + }, + "pack.too_much_work", + ), ]; for (e, code) in all { assert_eq!(e.msg().code, code); @@ -363,3 +371,28 @@ fn a_strict_read_refuses_a_plain_pack_when_a_key_is_given() { // `read` 照收：导入的是用户自己挑的文件 assert!(read(&plain, Unseal::Identity(&id)).is_ok()); } + +/// 口令加密的工作量超出上限：说的是这件事，不是「包坏了」。**解之前就拒**，不会先占掉内存 +#[test] +fn a_pack_that_takes_too_much_work_says_so() { + fast(); + let bytes = write(&sample(), Seal::Passphrase("pw")).unwrap(); + // 头里的 `-> scrypt 10` 改成 22：头的校验在解出文件密钥之后，工作量在那之前就查 + let head = bytes.iter().position(|b| *b == b'\n').unwrap() + 1; + let line_end = head + bytes[head..].iter().position(|b| *b == b'\n').unwrap(); + let line = std::str::from_utf8(&bytes[head..line_end]).unwrap(); + assert!( + line.starts_with("-> scrypt ") && line.ends_with(" 10"), + "{line}" + ); + let mut patched = bytes[..line_end - 2].to_vec(); + patched.extend_from_slice(b"22"); + patched.extend_from_slice(&bytes[line_end..]); + assert_eq!( + read(&patched, Unseal::Passphrase("pw")).unwrap_err(), + PackError::TooMuchWork { + required: 22, + max: MAX_WORK_FACTOR + } + ); +} diff --git a/crates/tw-pack/tests/regressions.rs b/crates/tw-pack/tests/regressions.rs index 2fdf0340..44544fe3 100644 --- a/crates/tw-pack/tests/regressions.rs +++ b/crates/tw-pack/tests/regressions.rs @@ -258,3 +258,258 @@ fn an_oauth_upstream_whose_endpoint_changed_comes_back() { let cfg = tw_config::try_parse(&out.pack.config_yaml).unwrap(); assert!(cfg.providers[0].disabled); } + +/// 不带密钥的包把上游指去了别的地方（换主机、换端口、换协议），或者改名导入：本机的密钥 +/// **不跟过去** —— 留空、标「缺少密钥」、停用。代理换了地址也一样。只换路径照样沿用（见上一条） +#[test] +fn a_pack_that_moves_an_upstream_does_not_take_the_key_along() { + let l = local(); + let relay = ItemId::new(ItemKind::Provider, "relay"); + let px = ItemId::new(ItemKind::Proxy, "px"); + let with = |base_url: &str, addr: &str| { + let mut p = export(&l, false, PackOrigin::Export); + p.config_yaml = format!( + "version: 1\nproxies:\n - name: px\n type: http\n addr: {addr}\n auth: {{ user: u, pass: '' }}\nproviders:\n - name: relay\n base_url: {base_url}\n key: ''\n proxy: px\n headers:\n X-Token: ''\n" + ); + p + }; + for base_url in [ + "https://evil.example", + "https://relay.example:8443", + "http://relay.example", + ] { + let incoming = with(base_url, "10.0.0.1:8080"); + let rows = tw_pack::import::preview(&l, &incoming, &ctx()).unwrap(); + let row = rows.iter().find(|r| r.item == relay).unwrap(); + assert_eq!(row.status, Status::Conflict, "{base_url}"); + assert_eq!(row.flags, [Flag::MissingKey], "{base_url}"); + let out = + tw_pack::import::apply(&l, &incoming, &[decide(&relay, Action::Overwrite)], &ctx()) + .unwrap(); + let cfg = tw_config::try_parse(&out.pack.config_yaml).unwrap(); + let p = cfg.providers.iter().find(|p| p.name == "relay").unwrap(); + assert_eq!(p.base_url, base_url); + assert_eq!(p.key.as_ref().map(|k| k.raw()), Some(""), "{base_url}"); + assert_eq!( + p.headers.get("X-Token").unwrap().value.raw(), + "", + "{base_url}" + ); + assert!(p.disabled, "{base_url}"); + } + + // 改名导入是另一个项目：地址没变也不带走本机那一家的密钥 + let incoming = with("https://relay.example", "10.0.0.1:8080"); + let out = tw_pack::import::apply( + &l, + &incoming, + &[decide(&relay, Action::Rename("relay (2)".into()))], + &ctx(), + ) + .unwrap(); + let cfg = tw_config::try_parse(&out.pack.config_yaml).unwrap(); + let p = |n: &str| cfg.providers.iter().find(|p| p.name == n).unwrap(); + assert_eq!(p("relay (2)").key.as_ref().map(|k| k.raw()), Some("")); + assert!(p("relay (2)").disabled); + assert_eq!(p("relay").key.as_ref().map(|k| k.raw()), Some("sk-old")); + + // 代理换了地址：密码不跟过去 + let incoming = with("https://relay.example", "203.0.113.9:3128"); + let rows = tw_pack::import::preview(&l, &incoming, &ctx()).unwrap(); + let row = rows.iter().find(|r| r.item == px).unwrap(); + assert_eq!(row.status, Status::Conflict); + assert_eq!(row.flags, [Flag::MissingKey]); + let out = + tw_pack::import::apply(&l, &incoming, &[decide(&px, Action::Overwrite)], &ctx()).unwrap(); + let cfg = tw_config::try_parse(&out.pack.config_yaml).unwrap(); + assert_eq!(cfg.proxies[0].auth.as_ref().unwrap().pass.raw(), ""); +} + +/// AWS 区域、OAuth 令牌端点换了也是换了目的地 +#[test] +fn a_moved_aws_region_or_oauth_endpoint_does_not_take_the_secret_along() { + let l = pack(&format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n key: tw-local\nproviders:\n - name: br\n base_url: https://bedrock.corp.example\n protocol: bedrock\n aws:\n access_key_id: AKIAOLD\n secret_access_key: old-secret\n region: us-east-1\n - name: oa\n base_url: https://api.example\n oauth:\n refresh: r-old\n endpoint: https://auth.example/token\n client_id: cid\n client_secret: cs-old\n" + )); + let incoming = { + let mut p = export(&l, false, PackOrigin::Export); + p.config_yaml = "version: 1\nproviders:\n - name: br\n base_url: https://bedrock.corp.example\n protocol: bedrock\n aws:\n access_key_id: ''\n secret_access_key: ''\n region: eu-west-1\n - name: oa\n base_url: https://api.example\n oauth:\n refresh: ''\n endpoint: https://other.example/token\n client_id: cid\n client_secret: ''\n".into(); + p + }; + let br = ItemId::new(ItemKind::Provider, "br"); + let oa = ItemId::new(ItemKind::Provider, "oa"); + let out = tw_pack::import::apply( + &l, + &incoming, + &[ + decide(&br, Action::Overwrite), + decide(&oa, Action::Overwrite), + ], + &ctx(), + ) + .unwrap(); + let cfg = tw_config::try_parse(&out.pack.config_yaml) + .unwrap_or_else(|e| panic!("{e}\n{}", out.pack.config_yaml)); + let p = |n: &str| cfg.providers.iter().find(|p| p.name == n).unwrap(); + let aws = p("br").aws.as_ref().unwrap(); + assert_eq!(aws.secret_access_key.as_ref().map(|k| k.raw()), Some("")); + assert!(p("br").disabled); + let oauth = p("oa").oauth.as_ref().unwrap(); + assert_eq!(oauth.client_secret.as_deref(), Some("")); + assert!(p("oa").disabled); +} + +/// 留着的固定密钥绑的路由指向一个策略组，那一版里路由和组都没有：路由和组一起留着，恢复照样 +/// 成。只选这条已删路由时，组一起恢复 +#[test] +fn a_kept_route_that_points_at_a_group_keeps_the_group() { + let now = pack(&format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n key: tw-a\n - name: codex\n key: tw-cx\n route: fast\nproviders:\n - name: p1\n base_url: https://p1.example\n key: sk-1\ngroups:\n - name: g\n type: fallback\n providers: [p1]\nroutes:\n - name: fast\n rules:\n - name: all\n to: g\n" + )); + let then = pack(&format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n key: tw-a\nproviders:\n - name: p1\n base_url: https://p1.example\n key: sk-1\n" + )); + let pinned: BTreeSet = ["codex".to_string()].into(); + let rows = tw_pack::restore::preview(&now, &then, &pinned, true).unwrap(); + let kept = |kind, name: &str| { + rows.iter() + .find(|r| r.item == ItemId::new(kind, name)) + .and_then(|r| r.kept) + }; + assert_eq!(kept(ItemKind::Key, "codex"), Some(Kept::KeyInUse)); + assert_eq!(kept(ItemKind::Route, "fast"), Some(Kept::Referenced)); + assert_eq!(kept(ItemKind::Group, "g"), Some(Kept::Referenced)); + let out = tw_pack::restore::apply(&now, &then, None, &pinned, true).unwrap(); + let cfg = tw_config::try_parse(&out.pack.config_yaml).unwrap(); + assert_eq!(cfg.groups.len(), 1); + + // 反过来：现在没有路由和组，只选那一版里那条指向组的路由 + let now2 = then.clone(); + let then2 = now.clone(); + let pick = [ItemId::new(ItemKind::Route, "fast")]; + let out = tw_pack::restore::apply(&now2, &then2, Some(&pick), &BTreeSet::new(), true).unwrap(); + let cfg = tw_config::try_parse(&out.pack.config_yaml) + .unwrap_or_else(|e| panic!("{e}\n{}", out.pack.config_yaml)); + assert_eq!( + cfg.groups + .iter() + .map(|g| g.name.as_str()) + .collect::>(), + ["g"] + ); + assert!( + out.rows + .iter() + .any(|r| r.item == ItemId::new(ItemKind::Group, "g") && r.change == Change::Add) + ); +} + +const SYNC: &str = "sync:\n device_id: 3f9a2c1e-0b1d-4e2f-9a3b-5c6d7e8f9a0b\n device_name: srv\n encrypt: false\n target:\n type: webdav\n url: https://dav.example.com/tw\n user: u\n pass: p\n proxy: corp\n"; + +/// 同步设置走的出站代理那一版里没有：恢复留着它（本机那几节引用的），整份恢复从现在这份 +/// 原文里一起取来 +#[test] +fn the_proxy_the_sync_settings_use_stays() { + let now = pack(&format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n key: tw-a\nproxies:\n - name: corp\n type: socks5h\n addr: 127.0.0.1:7890\n{SYNC}" + )); + tw_config::try_parse(&now.config_yaml).unwrap(); + let then = pack(&format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n key: tw-a\n" + )); + let rows = tw_pack::restore::preview(&now, &then, &BTreeSet::new(), true).unwrap(); + let corp = ItemId::new(ItemKind::Proxy, "corp"); + assert_eq!( + rows.iter().find(|r| r.item == corp).and_then(|r| r.kept), + Some(Kept::Referenced) + ); + let out = tw_pack::restore::apply(&now, &then, None, &BTreeSet::new(), true).unwrap(); + let cfg = tw_config::try_parse(&out.pack.config_yaml).unwrap(); + assert_eq!(cfg.proxies.len(), 1); + assert_eq!(out.kept, [corp]); + + let synced = export(&then, true, PackOrigin::Sync); + let w = tw_pack::restore::whole( + &synced, + &format!("{}not_a_field: 1\n", now.config_yaml), + None, + ); + let cfg = + tw_config::try_parse(&w.config_yaml).unwrap_or_else(|e| panic!("{e}\n{}", w.config_yaml)); + assert_eq!(cfg.proxies[0].addr, "127.0.0.1:7890"); + assert_eq!(cfg.sync.unwrap().target.proxy(), "corp"); +} + +/// 恢复同步位置上的版本：本机不同步的上游（和随之不同步的路由）那一版里本来就没有，不列成 +/// 删除，排列顺序里也照现在的位置 +#[test] +fn restoring_a_synced_version_leaves_what_this_machine_does_not_sync() { + let now = pack(&format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n key: tw-a\nproviders:\n - name: a\n base_url: https://a.example\n key: sk-a\n - name: ollama\n base_url: http://127.0.0.1:11434\n - name: b\n base_url: https://b.example\n key: sk-b\nroutes:\n - name: local\n rules:\n - name: all\n to: ollama\n" + )); + tw_config::try_parse(&now.config_yaml).unwrap(); + // 那一版：同步位置上的，没有 ollama 和指向它的路由；b 在 a 前面，a 的密钥不一样 + let then = pack(&format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n key: tw-a\nproviders:\n - name: b\n base_url: https://b.example\n key: sk-b\n - name: a\n base_url: https://a.example\n key: sk-a2\n" + )); + let l = tw_pack::model::snapshot(&now).unwrap(); + let b = tw_pack::model::snapshot(&then).unwrap(); + let skip = { + // 和同步那一路的算法一样：不同步的上游，连同直接或间接引用它们的 + let mut out: BTreeSet = [ItemId::new(ItemKind::Provider, "ollama")].into(); + for s in [&l, &b] { + for (id, it) in &s.items { + if !id.kind.is_order() && it.refs.iter().any(|r| out.contains(r)) { + out.insert(id.clone()); + } + } + } + out + }; + assert!(skip.contains(&ItemId::new(ItemKind::Route, "local"))); + let opts = tw_pack::restore::Options { + with_prefs: true, + skip, + ..Default::default() + }; + let rows = tw_pack::restore::preview_opts(&now, &then, &opts).unwrap(); + assert!( + rows.iter().all(|r| r.change != Change::Remove), + "{:?}", + rows.iter() + .map(|r| (r.item.to_string(), r.change)) + .collect::>() + ); + let out = tw_pack::restore::apply_opts(&now, &then, &opts).unwrap(); + assert_eq!(out.rows, rows); + let cfg = tw_config::try_parse(&out.pack.config_yaml).unwrap(); + let names: Vec<&str> = cfg.providers.iter().map(|p| p.name.as_str()).collect(); + assert_eq!(names, ["b", "a", "ollama"]); + assert_eq!(cfg.routes.len(), 1); + let a = cfg.providers.iter().find(|p| p.name == "a").unwrap(); + assert_eq!(a.key.as_ref().map(|k| k.raw()), Some("sk-a2")); +} + +/// 只选几项时，预览和写入走同一个计划：预览列出的就是写下去的 +#[test] +fn a_partial_preview_is_what_the_restore_writes() { + let now = pack(&format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n key: tw-a\nproviders:\n - name: p1\n base_url: https://p1.example\n key: sk-1\n" + )); + let then = pack(&format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n key: tw-a\nproxies:\n - name: px\n type: http\n addr: 10.0.0.1:8080\nproviders:\n - name: p1\n base_url: https://p1.example\n key: sk-1\n - name: gone\n base_url: https://gone.example\n key: sk-g\n proxy: px\n" + )); + let opts = tw_pack::restore::Options { + with_prefs: true, + items: Some([ItemId::new(ItemKind::Provider, "gone")].into()), + ..Default::default() + }; + let rows = tw_pack::restore::preview_opts(&now, &then, &opts).unwrap(); + let ids: BTreeSet = rows.iter().map(|r| r.item.to_string()).collect(); + assert!( + ids.contains(&ItemId::new(ItemKind::Proxy, "px").to_string()), + "{ids:?}" + ); + let out = tw_pack::restore::apply_opts(&now, &then, &opts).unwrap(); + assert_eq!(out.rows, rows); +} diff --git a/docs/config.md b/docs/config.md index 32abe538..071833a5 100644 --- a/docs/config.md +++ b/docs/config.md @@ -142,7 +142,12 @@ clients set up on this machine use. An import adds what is not here and asks, item by item, what to do with what differs (`--on-conflict` answers for all). A pack carries no settings of this machine and no ChatGPT sign-in; without `--no-secrets` it holds keys in plain text unless `--encrypt` is given (the -passphrase is asked for, or read from `TWCORE_PASSPHRASE`). +passphrase is asked for, or read from `TWCORE_PASSPHRASE`). Overwriting an +item here with one from a pack without keys keeps this machine's keys only +while the item still points at the same place (scheme, host and port; AWS +region; OAuth token endpoint; proxy address); otherwise, and for an item +imported under another name, the upstream comes in disabled until its key +is entered. These commands work on the files directly, so they work when core is not running, which is when a restore is most needed; when the file does not diff --git a/docs/config.zh-CN.md b/docs/config.zh-CN.md index b9b7ee4b..aa16e5ac 100644 --- a/docs/config.zh-CN.md +++ b/docs/config.zh-CN.md @@ -81,7 +81,7 @@ twcore import all.twconfig --dry-run # 导入会做什么 twcore config set /listen/gateway/port 8790 --int ``` -恢复时保留本机设置（监听、控制面密钥、保留期限、同步设置）、ChatGPT 的登录，以及本机已接管的客户端正在使用的网关密钥。导入时加入本机没有的项目，与本机不同的项目逐项选择处理方式（`--on-conflict` 一次指定全部）。配置包不含本机设置与 ChatGPT 登录；未指定 `--no-secrets` 时其中以明文保存密钥，除非指定 `--encrypt`（口令在终端输入，或取自环境变量 `TWCORE_PASSPHRASE`）。 +恢复时保留本机设置（监听、控制面密钥、保留期限、同步设置）、ChatGPT 的登录，以及本机已接管的客户端正在使用的网关密钥。导入时加入本机没有的项目，与本机不同的项目逐项选择处理方式（`--on-conflict` 一次指定全部）。配置包不含本机设置与 ChatGPT 登录；未指定 `--no-secrets` 时其中以明文保存密钥，除非指定 `--encrypt`（口令在终端输入，或取自环境变量 `TWCORE_PASSPHRASE`）。以不含密钥的配置包覆盖本机项目时，只有目的地不变（协议、主机与端口，AWS 区域，OAuth 令牌端点，代理地址）才沿用本机的密钥；目的地变了的、改名导入的上游以停用状态导入，填入密钥后再启用。 这些命令直接操作文件，因此 core 没有运行时也能使用，而那往往正是最需要恢复的时候；配置完全无法读取时，恢复把整个版本写回，控制面密钥保持现在的值。正在运行的 core 会像对待其他保存一样接收这些改动。 From 0a06fe77ec557ad70348349412f24bd8bc519e2a Mon Sep 17 00:00:00 2001 From: fylorn <249551762+fylorn@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:23:03 +0800 Subject: [PATCH 10/13] Sync: oversized preferences are left out instead of failing every round; CLI explains keep and keep-both (#360) Preferences larger than the limit (they cannot be written here) used to fail the whole apply, every round, until the state read as unreachable. Now: - When the sync data's general preferences are too large, they are held: this device keeps its own, the uploaded versions keep the sync data's (this device's are never pushed over them), and everything else syncs. The same applies when the merged preferences come out too large, and on join. - SyncView.notices carries sync.prefs_too_large. It is not an error and does not count toward unreachable. - Once the sync data's preferences fit again and this device's did not change meanwhile, they land as synced instead of looking like a local edit. - The control plane's apply skips oversized preferences rather than failing. twcore sync enable: "keep this device's version" and "keep both" now say that this device's version is uploaded with the join and replaces the item of that name on the other devices. Co-authored-by: Claude Opus 5.5 --- Cargo.lock | 1 + bin/twcore/src/sync_cmd.rs | 14 ++- crates/tw-api/msg-codes.txt | 2 + crates/tw-api/src/lib.rs | 3 +- crates/tw-api/src/sync.rs | 3 + crates/tw-control/src/sync.rs | 17 +++- crates/tw-control/tests/sync_regressions.rs | 61 +++++++++++++ crates/tw-sync/Cargo.toml | 2 + crates/tw-sync/src/engine.rs | 95 ++++++++++++++++++++- crates/tw-sync/src/local.rs | 4 + crates/tw-sync/tests/regressions.rs | 75 ++++++++++++++-- 11 files changed, 265 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f9224567..35d51c94 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3991,6 +3991,7 @@ dependencies = [ "thiserror 2.0.20", "tokio", "tracing", + "tw-api", "tw-bedrock", "tw-config", "tw-pack", diff --git a/bin/twcore/src/sync_cmd.rs b/bin/twcore/src/sync_cmd.rs index a38cb948..81741e8e 100644 --- a/bin/twcore/src/sync_cmd.rs +++ b/bin/twcore/src/sync_cmd.rs @@ -93,6 +93,9 @@ fn print_view(v: &tw_api::SyncView) { if let Some(e) = &v.error { println!("problem: {}", e.text); } + for n in &v.notices { + println!("note: {}", n.text); + } if v.conflicts > 0 { println!( "conflicts: {} (resolve them in ThinkWatch Lite, Settings → Sync and backup)", @@ -242,13 +245,18 @@ fn decide(row: &tw_api::ImportRow) -> Result { name => format!("{} `{name}`", row.item.kind.slug()), }; eprintln!("{what}{differs} differs between this device and the sync data:"); - eprintln!(" u use the sync data (default)"); + eprintln!(" u use the sync data (default): this device takes the other devices' version"); eprintln!( - " k keep this device's version; it is uploaded and replaces the other devices' one" + " k keep this device's version: it is uploaded with this join and replaces the \ + other devices' version" ); let rename = row.suggested_name.clone().filter(|_| !row.global); if let Some(n) = &rename { - eprintln!(" r keep both: the sync data's one is added as {n}"); + eprintln!( + " r keep both: the sync data's version is added here as {n}; this device's \ + version keeps the name and is uploaded with this join, replacing the item of \ + that name on the other devices (they get {n} too)" + ); } loop { let a = ask("Choice", Some("u"))?; diff --git a/crates/tw-api/msg-codes.txt b/crates/tw-api/msg-codes.txt index 95d160d9..1b7a7fb4 100644 --- a/crates/tw-api/msg-codes.txt +++ b/crates/tw-api/msg-codes.txt @@ -586,6 +586,7 @@ sync.passphrase_changed sync.passphrase_short sync.passphrase_wrong sync.place_changed +sync.prefs_too_large sync.proxy_unusable sync.s3.bad_listing sync.s3.sign @@ -611,6 +612,7 @@ t.boom test t.broke test t.limited test t.plugin_threw test +t.prefs test t.upstream test t.x test yaml.anchor_or_alias diff --git a/crates/tw-api/src/lib.rs b/crates/tw-api/src/lib.rs index 762163e5..b59e6d4e 100644 --- a/crates/tw-api/src/lib.rs +++ b/crates/tw-api/src/lib.rs @@ -996,7 +996,8 @@ pub const MSG_CODES: &str = include_str!("../msg-codes.txt"); /// `POST /sync/disable`（[`SyncDisable`]）、`POST /sync/republish`（同步位置空了、keyring 不见了 /// 之后由人选的「重新上传本机配置」，→ [`SyncView`]）。[`SyncState`] 里要人处理的有 /// `keyring_missing`、`emptied`、`misconfigured`，[`SyncView`] 的 `skipped` 是同步位置上跳过了的 -/// 文件数，[`SyncConflict`] 的 `options` 是能选的处理，[`SyncJoinPreview`] 的 `auto` 是加入时 +/// 文件数，`notices` 是同步成了但没照办的地方（`sync.prefs_too_large`），[`SyncConflict`] 的 +/// `options` 是能选的处理，[`SyncJoinPreview`] 的 `auto` 是加入时 /// 不用选的项（两边的默认网关密钥同名、值不同时采用同步数据的值）。状态变了发 [`Event::SyncStatus`]；应用别的设备的 /// 版本时配置的来源是 `sync`，改了网关密钥时发 `GatewayKeysChanged`。[`ProviderView`] 多了 /// `sync_excluded`（不同步的上游）和 `sync_url_changed`（地址经同步变更，改之前的地址）。 diff --git a/crates/tw-api/src/sync.rs b/crates/tw-api/src/sync.rs index 1e95e604..82f6ae0d 100644 --- a/crates/tw-api/src/sync.rs +++ b/crates/tw-api/src/sync.rs @@ -136,6 +136,9 @@ pub struct SyncView { /// 同步位置上跳过了的文件：坏了的、不是这把密钥加密的、加密的位置上没加密的、等了 /// 30 分钟还没传完的。不是 0 时界面提一句 pub skipped: u32, + /// 同步成了，但有没照办的地方（上一轮的）：界面提一句，**不是失败**，不计入连不上。现在 + /// 只有 `sync.prefs_too_large`：同步数据里的通用偏好超出上限，这台留着自己的，其余照常 + pub notices: Vec, } /// `POST /sync/test`、`POST /sync/enable/preview`：连上去看看。 diff --git a/crates/tw-control/src/sync.rs b/crates/tw-control/src/sync.rs index e0db8a45..ef29ecc7 100644 --- a/crates/tw-control/src/sync.rs +++ b/crates/tw-control/src/sync.rs @@ -132,8 +132,14 @@ async fn apply_pack( e, ) })?; - // 远程连接下偏好在 Lite 那边；同步这一路跑在 core 自己的任务里，不是远程连接 - let prefs_changed = after.client_prefs.is_some() && after.client_prefs != before.client_prefs; + // 远程连接下偏好在 Lite 那边；同步这一路跑在 core 自己的任务里，不是远程连接。太大的偏好 + // 不写（引擎已经不会给，也报了 `sync.prefs_too_large`）：其余照常，不让整轮失败 + let prefs_changed = after.client_prefs.is_some() + && after.client_prefs != before.client_prefs + && !after + .client_prefs + .as_ref() + .is_some_and(tw_pack::disk::prefs_too_large); if prefs_changed && let Err(e) = tw_pack::disk::write_prefs(&path, after.client_prefs.as_ref()) { undo.restore(); @@ -613,6 +619,7 @@ impl Sync { excluded: Vec::new(), excluded_cascade: Vec::new(), skipped: 0, + notices: Vec::new(), }; let Some(mgr) = self.mgr() else { return off }; let cfg = mgr.gateway().config(); @@ -640,6 +647,11 @@ impl Sync { .map(|r| r.skipped) .or(prev.as_ref().map(|p| p.skipped)) .unwrap_or(0); + let notices = report + .as_ref() + .map(|r| r.notices.clone()) + .or(prev.as_ref().map(|p| p.notices.clone())) + .unwrap_or_default(); let devices = match &report { Some(r) => r .devices @@ -694,6 +706,7 @@ impl Sync { excluded: sync.exclude.clone(), excluded_cascade: cascade, skipped, + notices, } } diff --git a/crates/tw-control/tests/sync_regressions.rs b/crates/tw-control/tests/sync_regressions.rs index 7e8ca29e..74490dc4 100644 --- a/crates/tw-control/tests/sync_regressions.rs +++ b/crates/tw-control/tests/sync_regressions.rs @@ -449,3 +449,64 @@ async fn a_fresh_device_with_clients_set_up_adopts_the_default_key() { a.ok("POST", "/sync/now", Value::Null).await; assert_eq!(a.config().clients[0].key, "tw-default-a"); } + +/// 同步位置上来了一份超出上限的通用偏好（别处写进来的）：这一轮不失败，偏好不应用、其余照常， +/// 视图里如实说（`notices`），不是错误、也不计入连不上 +#[tokio::test] +async fn oversized_preferences_in_the_sync_data_do_not_fail_the_round() { + let loc = tempfile::tempdir().unwrap(); + let (a, b) = (Dev::new('a'), Dev::new('b')); + a.enable(loc.path(), false, "A", Value::Null).await; + b.enable(loc.path(), false, "B", Value::Null).await; + // 依据最新的那个版本，写一个带着很大偏好的版本进去 + let mut names: Vec = std::fs::read_dir(loc.path().join("versions")) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().to_string()) + .collect(); + names.sort(); + let head = names + .last() + .unwrap() + .trim_end_matches(".twconfig") + .to_string(); + let seq: u64 = head[..10].parse().unwrap(); + let text = add_raw( + &tw_pack::read( + &std::fs::read(loc.path().join("versions").join(format!("{head}.twconfig"))).unwrap(), + tw_pack::Unseal::None, + ) + .unwrap() + .config_yaml, + " - name: elsewhere\n base_url: https://e.example\n key: sk-e\n", + ); + let mut pack = tw_pack::Pack { + manifest: tw_pack::Manifest::new(tw_pack::PackKind::Full, tw_pack::PackOrigin::Sync), + config_yaml: text, + plugins: Default::default(), + client_prefs: Some(json!({ "notes": "x".repeat(70 * 1024) })), + }; + let meta = tw_sync::record::Meta { + seq: seq + 1, + device: "eeeeeeee-0000-4000-8000-00000000000e".into(), + device_name: "Elsewhere".into(), + parents: vec![head], + sha256: Some(tw_sync::engine::content_sha256(&pack)), + ..Default::default() + }; + pack.manifest.sync = Some(serde_json::to_value(&meta).unwrap()); + std::fs::write( + loc.path() + .join("versions") + .join(format!("{:010}-eeeeeeee.twconfig", seq + 1)), + tw_pack::write(&pack, tw_pack::Seal::None).unwrap(), + ) + .unwrap(); + for _ in 0..2 { + let v = b.ok("POST", "/sync/now", Value::Null).await; + assert_eq!(v["state"], "idle", "{v}"); + assert!(v["error"].is_null(), "{v}"); + assert!(v["failing_since_ms"].is_null(), "{v}"); + assert_eq!(v["notices"][0]["code"], "sync.prefs_too_large", "{v}"); + } + assert!(b.provider("elsewhere").is_some(), "其余照常应用"); +} diff --git a/crates/tw-sync/Cargo.toml b/crates/tw-sync/Cargo.toml index 085d9243..065e06bc 100644 --- a/crates/tw-sync/Cargo.toml +++ b/crates/tw-sync/Cargo.toml @@ -14,6 +14,8 @@ description = "Multi-device configuration sync: folder, WebDAV and S3 locations, # `sync` 一节的形状、地址规则 tw-config = { workspace = true } tw-types = { workspace = true } +# 通用偏好的上限 +tw-api = { workspace = true } # 配置包的格式与加密、把一份配置拆成项目与合并单位 tw-pack = { workspace = true } # S3 的 SigV4 和 Bedrock 用的是同一份 diff --git a/crates/tw-sync/src/engine.rs b/crates/tw-sync/src/engine.rs index 6ca71619..1122870c 100644 --- a/crates/tw-sync/src/engine.rs +++ b/crates/tw-sync/src/engine.rs @@ -233,6 +233,18 @@ pub struct Report { /// 同步位置上跳过了的文件：坏了的、不是这把密钥加密的、加密的位置上没加密的、等了太久 /// 还没传完的 pub skipped: u32, + /// 同步成了，但有没照办的地方，要告诉人（现在只有：通用偏好太大，没应用， + /// [`prefs_too_large_msg`]） + pub notices: Vec, +} + +/// 同步来的通用偏好超出上限、没有应用（其余照常） +pub fn prefs_too_large_msg() -> Msg { + msg!( + "sync.prefs_too_large", limit = tw_api::CLIENT_PREFS_MAX / 1024 => + "the general preferences in the sync data are larger than {limit} KiB, so this device \ + keeps its own; everything else was synced" + ) } /// 读出来的一个版本 @@ -353,7 +365,7 @@ fn local_err(e: impl std::fmt::Display) -> RoundError { } /// 内容的 SHA-256：配置原文、插件、通用偏好（不加密的版本带着它，发现传输损坏） -fn content_sha256(p: &Pack) -> String { +pub fn content_sha256(p: &Pack) -> String { let mut h = Sha256::new(); h.update(p.config_yaml.as_bytes()); for (id, f) in &p.plugins { @@ -634,6 +646,49 @@ pub fn adopted_default_key(local: &Snapshot, remote: &Snapshot) -> Option ItemId { + ItemId::new(ItemKind::ClientPrefs, "") +} + +/// 这一份里的通用偏好写不进本机：拼回 JSON 之后超出上限（[`tw_pack::disk::prefs_too_large`]， +/// 写偏好的地方都按它拒收） +fn prefs_oversized(s: &Snapshot) -> bool { + let Some(it) = s.items.get(&prefs_id()) else { + return false; + }; + let only = Snapshot { + items: BTreeMap::from([(prefs_id(), it.clone())]), + }; + let empty = Pack { + manifest: tw_pack::Manifest::new(PackKind::Full, PackOrigin::Sync), + config_yaml: format!("version: {}\n", tw_config::SCHEMA_VERSION), + plugins: Default::default(), + client_prefs: None, + }; + model::materialize(&empty, &only) + .ok() + .and_then(|p| p.client_prefs) + .is_some_and(|p| tw_pack::disk::prefs_too_large(&p)) +} + +/// 本机通用偏好此刻的指纹（[`State::held_prefs`]） +fn prefs_print(s: &Snapshot) -> String { + let units = s.items.get(&prefs_id()).map(|it| &it.units); + let json = serde_json::to_string(&units).unwrap_or_default(); + let out: [u8; 32] = Sha256::digest(json.as_bytes()).into(); + out.iter().map(|b| format!("{b:02x}")).collect() +} + +/// 一个项目照 `from` 的样子（`from` 里没有就去掉） +fn put_item(dst: &mut Snapshot, from: &Snapshot, id: &ItemId) { + match from.items.get(id) { + Some(it) => dst.items.insert(id.clone(), it.clone()), + None => dst.items.remove(id), + }; +} + // ─────────────────────────────────────────────── 等登录的上游 /// 本机配置里的上游（带着这台设备自己的登录） @@ -1598,6 +1653,7 @@ impl Engine { return Ok(Report { encrypted: kr, conflicts: self.last_report.conflicts.clone(), + notices: self.last_report.notices.clone(), devices: self.devices().await?, skipped, ..Default::default() @@ -1645,6 +1701,16 @@ impl Engine { copy_disabled(it, &s); } } + // 同步位置上的偏好太大时这台留着自己的（见下）。之后同步位置上的变回能写的：这台的偏好 + // 一直没动过的话，不算本机改了（换回同步的样子再合并），同步位置上的照常落地 + let hold_prefs = prefs_oversized(&remote.snap); + if let Some(print) = st.held_prefs.take() + && !hold_prefs + && prefs_print(&local) == print + { + let synced = base.as_ref().map(|b| &b.snap).unwrap_or(&remote.snap); + put_item(&mut local, synced, &prefs_id()); + } // 别的设备处理掉了的冲突：照同步版本改 let open_ids: BTreeSet<&str> = remote.conflicts.iter().map(|c| c.id.as_str()).collect(); if let Some(b) = &base { @@ -1761,6 +1827,14 @@ impl Engine { for (o, _) in ORDERS { hold_items.insert(ItemId::new(*o, "")); } + // 同步位置上的通用偏好太大、写不进本机：两边各留各的 —— 本机不应用，上传的也不拿本机 + // 的去换掉别的设备的。其余照常 + let mut notices = Vec::new(); + if hold_prefs { + hold_items.insert(prefs_id()); + notices.push(prefs_too_large_msg()); + st.held_prefs = Some(prefs_print(&local_snap)); + } // 不加密时经同步停用的插件：本机启用了就放开 st.held_plugins.retain(|id, _| { let item = local.items.get(&ItemId::new(ItemKind::Plugin, id.clone())); @@ -1860,6 +1934,15 @@ impl Engine { }); } } + // 两边各自不大、合起来太大：本机留本机的，上传的留同步位置的 + if prefs_oversized(&target) { + put_item(&mut target, &local_snap, &prefs_id()); + put_item(&mut shared, &remote.snap, &prefs_id()); + if notices.is_empty() { + notices.push(prefs_too_large_msg()); + } + st.held_prefs = Some(prefs_print(&local_snap)); + } target.relink(); shared.relink(); // 没登录的上游在这台停用着 @@ -1937,6 +2020,7 @@ impl Engine { let mut report = Report { encrypted: kr, skipped, + notices, ..Default::default() }; let mut content_after = quiet.content.clone(); @@ -2407,6 +2491,14 @@ impl Engine { set_order(&mut target, *o, m.local); set_order(&mut shared, *o, m.shared); } + // 通用偏好太大、写不进本机：本机留本机的，上传的照同步数据 + let mut notices = Vec::new(); + if prefs_oversized(&target) { + put_item(&mut target, &local, &prefs_id()); + put_item(&mut shared, &remote.snap, &prefs_id()); + notices.push(prefs_too_large_msg()); + st.held_prefs = Some(prefs_print(&local)); + } target.relink(); shared.relink(); // 本机这一份：没登录的停用着；同步的样子是启用着的，记成等登录 @@ -2447,6 +2539,7 @@ impl Engine { let mut report = Report { encrypted: self.sealed, skipped: sc.skipped.len() as u32, + notices, ..Default::default() }; if model::normalized(&target) != model::normalized(&local) { diff --git a/crates/tw-sync/src/local.rs b/crates/tw-sync/src/local.rs index d1584626..955d1d60 100644 --- a/crates/tw-sync/src/local.rs +++ b/crates/tw-sync/src/local.rs @@ -64,6 +64,10 @@ pub struct State { /// 的分开算（等得更久） #[serde(skip_serializing_if = "BTreeMap::is_empty")] pub arriving: BTreeMap, + /// 同步位置上的通用偏好太大、这台留着自己的：留下时本机偏好的指纹。之后同步位置上的变回 + /// 能写的、本机的又没改过，就照同步的样子落地（本机的不算改动） + #[serde(skip_serializing_if = "Option::is_none")] + pub held_prefs: Option, /// 同步位置看上去是空的：从什么时候起、连着几轮 pub empty_since_ms: Option, pub empty_rounds: u32, diff --git a/crates/tw-sync/tests/regressions.rs b/crates/tw-sync/tests/regressions.rs index 992b5bdf..7b9bb4a7 100644 --- a/crates/tw-sync/tests/regressions.rs +++ b/crates/tw-sync/tests/regressions.rs @@ -8,7 +8,6 @@ use std::collections::BTreeSet; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; -use sha2::Digest; use tw_pack::{Pack, PackKind, PackOrigin, Seal}; use tw_sync::engine::{ ApplyError, Current, Decision, Engine, Host, RoundError, SKIP_AFTER_MS, Settings, Take, @@ -47,6 +46,11 @@ impl Dev { fn path(&self) -> PathBuf { self.dir.path().join("config.yaml") } + /// 通用偏好（和控制面一样存在配置旁边） + fn prefs(&self) -> Option { + let b = std::fs::read(self.dir.path().join("prefs.json")).ok()?; + serde_json::from_slice(&b).ok() + } fn text(&self) -> String { std::fs::read_to_string(self.path()).unwrap() } @@ -75,7 +79,7 @@ impl Host for Dev { manifest: tw_pack::Manifest::new(PackKind::Full, PackOrigin::Ui), config_yaml: text.clone(), plugins: Default::default(), - client_prefs: None, + client_prefs: self.prefs(), }, version: tw_config::version_of(&text), changed_ms: 1, @@ -86,7 +90,24 @@ impl Host for Dev { return Err(ApplyError::Stale); } tw_config::try_parse(&pack.config_yaml).map_err(|r| ApplyError::Rejected(r.msg()))?; + // 和控制面一样：写不进的偏好（超出上限）整个应用就失败 + if pack + .client_prefs + .as_ref() + .is_some_and(tw_pack::disk::prefs_too_large) + { + return Err(ApplyError::Rejected(tw_types::msg!( + "t.prefs" => "preferences too large" + ))); + } std::fs::write(self.path(), &pack.config_yaml).unwrap(); + let prefs = self.dir.path().join("prefs.json"); + match &pack.client_prefs { + Some(p) => std::fs::write(prefs, serde_json::to_vec(p).unwrap()).unwrap(), + None => { + let _ = std::fs::remove_file(prefs); + } + } Ok(()) } fn pinned_keys(&self) -> BTreeSet { @@ -183,6 +204,15 @@ fn read_version(loc: &Path, id: &str) -> Vec { /// 没有口令的人往同步位置里放一个不加密的版本（manifest.sync 照格式写、带 SHA-256），依据 /// 最新的那一个 fn forge_plain_version(loc: &Path, config_yaml: String) -> String { + forge_plain_version_with(loc, config_yaml, None) +} + +/// 同上，带着通用偏好 +fn forge_plain_version_with( + loc: &Path, + config_yaml: String, + client_prefs: Option, +) -> String { let head = versions(loc) .last() .unwrap() @@ -193,10 +223,9 @@ fn forge_plain_version(loc: &Path, config_yaml: String) -> String { manifest: tw_pack::Manifest::new(PackKind::Full, PackOrigin::Sync), config_yaml, plugins: Default::default(), - client_prefs: None, + client_prefs, }; - let sha: [u8; 32] = sha2::Sha256::digest(pack.config_yaml.as_bytes()).into(); - let sha: String = sha.iter().map(|b| format!("{b:02x}")).collect(); + let sha = tw_sync::engine::content_sha256(&pack); let meta = tw_sync::record::Meta { seq: seq + 1, device: "eeeeeeee-0000-4000-8000-00000000000e".into(), @@ -1008,6 +1037,42 @@ async fn an_icloud_placeholder_that_never_downloads_is_skipped_after_a_day() { assert!(r.uploaded.is_some()); } +// ─────────────────────────────────────────────── 太大的通用偏好 + +/// 同步位置上的通用偏好超出上限（别处写进来的，这台写不进）：偏好不应用、其余照常，报 +/// `sync.prefs_too_large`，**不算失败**；之后上传的版本也不拿本机的偏好去换掉同步位置上的 +#[tokio::test] +async fn oversized_preferences_are_left_out_and_the_rest_applies() { + let mut t = bed(false).await; + let big = serde_json::json!({ "notes": "x".repeat(70 * 1024) }); + assert!(tw_pack::disk::prefs_too_large(&big)); + let cfg = add_provider(&t.a.text(), "elsewhere", "https://e.example", "sk-e"); + forge_plain_version_with(t.loc.path(), cfg, Some(big.clone())); + let r = t.eb.round().await.unwrap(); + let codes: Vec<&str> = r.notices.iter().map(|m| m.code.as_str()).collect(); + assert_eq!(codes, ["sync.prefs_too_large"]); + assert!(t.b.provider("elsewhere").is_some(), "其余照常应用"); + assert!(r.uploaded.is_none(), "不拿本机的偏好去换"); + // 下一轮（走捷径）还说着 + let r = t.eb.round().await.unwrap(); + assert_eq!(r.notices.len(), 1); + // B 改了别的：上传的版本里偏好还是同步位置上的那一份 + t.b.edit(|s| add_provider(s, "bnew", "https://bnew.example", "sk-bn")); + let up = t.eb.round().await.unwrap().uploaded.unwrap(); + let p = tw_pack::read(&read_version(t.loc.path(), &up), tw_pack::Unseal::None).unwrap(); + assert_eq!(p.client_prefs, Some(big)); + assert_eq!(t.b.prefs(), None, "B 的偏好没动"); + // 同步位置上的偏好变回不大的：照常落地（B 的偏好一直没动过，不算 B 改了），提示没了 + let small = serde_json::json!({ "notes": "short" }); + forge_plain_version_with(t.loc.path(), t.b.text(), Some(small.clone())); + let r = t.eb.round().await.unwrap(); + assert!(r.notices.is_empty(), "{:?}", r.notices); + assert!(r.conflicts.is_empty(), "{:?}", r.conflicts); + assert_eq!(t.b.prefs(), Some(small.clone())); + t.ea.round().await.unwrap(); + assert_eq!(t.a.prefs(), Some(small)); +} + // ─────────────────────────────────────────────── 空的同步位置 /// 文件夹位置的根目录还在、里面暂时是空的（网络盘没挂上时挂载点是个空目录）：不当作清空， From 1314b1a90ba93910f3fec3c77f73e24c5905bfbf Mon Sep 17 00:00:00 2001 From: fylorn <249551762+fylorn@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:48:27 +0800 Subject: [PATCH 11/13] Packs: compare destinations the way the gateway reads them; say what a rename brings (#361) - destination() parses with the url crate, the parser reqwest uses: a backslash separates the path as it does when the request goes out, and hosts are normalized (case, punycode, IPv6/IPv4 forms) with default ports filled in. A trailing dot stays a different name. - Upstream addresses with a backslash or a user name/password before the host are refused (config.base_url_backslash, config.base_url_userinfo). - The outbound proxy an upstream goes through is part of its destination: direct, system, or the proxy's type and address. A different route keeps this machine's keys out. - ImportRow.rename_flags tells what an item brings when imported under another name (it never takes the keys or sign-ins here); the CLI says so. - A whole restore of a version from the sync location brings back the upstreams this machine does not sync, and what follows them, from the configuration here or the newest readable version. Co-authored-by: Claude Opus 5.5 --- Cargo.lock | 1 + bin/twcore/src/pack_cli.rs | 46 ++++++- crates/tw-api/msg-codes.txt | 2 + crates/tw-api/src/lib.rs | 6 +- crates/tw-api/src/pack.rs | 4 + crates/tw-config/src/validate.rs | 76 ++++++++++++ crates/tw-config/tests/manual/schema.rs | 4 +- crates/tw-control/src/backups.rs | 22 ++-- crates/tw-control/tests/backups.rs | 28 +++++ crates/tw-pack/Cargo.toml | 2 + crates/tw-pack/src/export.rs | 151 ++++++++++++++--------- crates/tw-pack/src/import.rs | 26 +++- crates/tw-pack/src/restore.rs | 113 +++++++++++++++++ crates/tw-pack/tests/regressions.rs | 156 ++++++++++++++++++++++++ docs/config.md | 8 +- docs/config.zh-CN.md | 4 +- 16 files changed, 567 insertions(+), 82 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 35d51c94..64b3fc25 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3914,6 +3914,7 @@ dependencies = [ "tw-secret", "tw-types", "tw-yaml", + "url", "zip", ] diff --git a/bin/twcore/src/pack_cli.rs b/bin/twcore/src/pack_cli.rs index be68ffc8..4e80aa26 100644 --- a/bin/twcore/src/pack_cli.rs +++ b/bin/twcore/src/pack_cli.rs @@ -287,11 +287,19 @@ pub fn import( continue; } Status::New => { - println!(" new {what}"); + println!(" new {what}{}", flag_note(&r.item, &r.flags)); Action::Import } Status::Conflict => { - println!(" differs {what} ({})", r.differs.join(", ")); + println!( + " differs {what} ({}){}", + r.differs.join(", "), + flag_note(&r.item, &r.flags) + ); + let renamed = flag_note(&r.item, &r.rename_flags); + if !r.global && !renamed.is_empty() { + println!(" under another name{renamed}"); + } let choice = match on_conflict { Some(c) => c, None if dry_run => continue, @@ -311,7 +319,10 @@ pub fn import( }; match (choice, &r.suggested_name) { (OnConflict::Overwrite, _) => Action::Overwrite, - (OnConflict::Rename, Some(n)) => Action::Rename(n.clone()), + (OnConflict::Rename, Some(n)) => { + println!(" → as `{n}`{renamed}"); + Action::Rename(n.clone()) + } _ => Action::Skip, } } @@ -336,6 +347,35 @@ pub fn import( Ok(()) } +/// 一行导入带的标记说成一句：缺密钥、要登录的上游停用着进来，空的网关密钥换一个新值 +fn flag_note(id: &ItemId, flags: &[tw_pack::import::Flag]) -> String { + use tw_pack::import::Flag; + let mut parts: Vec<&str> = Vec::new(); + if flags.contains(&Flag::MissingKey) { + parts.push(match id.kind { + ItemKind::Key => "a new key value is generated", + ItemKind::Proxy => "its password is missing", + _ => "its key is missing", + }); + } + if flags.contains(&Flag::NeedsLogin) { + parts.push("it needs signing in"); + } + if id.kind == ItemKind::Provider + && (flags.contains(&Flag::MissingKey) || flags.contains(&Flag::NeedsLogin)) + { + parts.push("it comes in disabled"); + } + if flags.contains(&Flag::PluginDisabled) { + parts.push("it comes in turned off"); + } + if parts.is_empty() { + String::new() + } else { + format!(": {}", parts.join("; ")) + } +} + // ─────────────────────────────────────────────────────────── backup pub fn list(path: &Path) -> Result<()> { diff --git a/crates/tw-api/msg-codes.txt b/crates/tw-api/msg-codes.txt index 1b7a7fb4..bbd8d2d5 100644 --- a/crates/tw-api/msg-codes.txt +++ b/crates/tw-api/msg-codes.txt @@ -26,6 +26,8 @@ config.aws_profile.unsupported config.bad_allow_from config.bad_base_url config.bad_remote_allow_from +config.base_url_backslash +config.base_url_userinfo config.blank_models_only config.control_key_invalid config.control_key_missing diff --git a/crates/tw-api/src/lib.rs b/crates/tw-api/src/lib.rs index b59e6d4e..f3b459ab 100644 --- a/crates/tw-api/src/lib.rs +++ b/crates/tw-api/src/lib.rs @@ -976,8 +976,10 @@ pub const MSG_CODES: &str = include_str!("../msg-codes.txt"); /// 超出上限是 `backup.pinned.*`）报告接管的客户端在用哪几把密钥。恢复时留下的项目还引用着的、 /// 同步设置走的出站代理不删（[`KeptReason`] 的 `referenced`），所选项目引用的、本机已没有的一起 /// 恢复；恢复同步位置上的版本时本机不同步的上游不列成删除。不带密钥的包导入时比较和覆盖都沿用 -/// 本机的密钥，**只在目的地不变时**（协议、主机、端口、AWS 区域、OAuth 令牌端点、代理地址 -/// 一样）；改名导入的不沿用，缺着的标 `missing_key`（代理的密码也算）。导入覆盖、恢复改了或删了网关密钥时发 +/// 本机的密钥，**只在目的地不变时**（协议、主机、端口按 URL 库规范化后一样，AWS 区域、OAuth +/// 令牌端点、所走的出站代理、代理地址一样）；改名导入的不沿用，缺着的标 `missing_key`（代理的 +/// 密码也算），[`ImportRow`] 的 `rename_flags` 说改名导入时会带的标记。上游地址里有反斜杠、 +/// 主机前写了用户名或密码的不收（`config.base_url_backslash`、`config.base_url_userinfo`）。导入覆盖、恢复改了或删了网关密钥时发 /// [`Event::GatewayKeysChanged`]。[`ConfigOrigin`] 删了 `rollback`，多了 `sync`、`import`、 /// `restore`、`repair`（一键修复以前记成 `ui`）。备份相关的端点在安全模式下照样可用。配置包的 /// 码是 `pack.*`，导入的是 `import.*`，备份与恢复的是 `backup.*`。停用着的上游可以缺凭据 diff --git a/crates/tw-api/src/pack.rs b/crates/tw-api/src/pack.rs index a86c9119..c3e181b3 100644 --- a/crates/tw-api/src/pack.rs +++ b/crates/tw-api/src/pack.rs @@ -226,6 +226,10 @@ pub struct ImportRow { pub differs: Vec, /// 改名导入时的缺省名字（`公司中转 (2)`）。冲突的、有名字的项目才有 pub suggested_name: Option, + /// 改名导入时这一项会带的标记（`flags` 说的是导入、覆盖）。改名导入的是另一个项目，不沿用 + /// 本机的密钥和登录：不带密钥的包里，改名导入的上游缺着密钥、停用着进来（`missing_key`）， + /// ChatGPT 账号要重新登录（`needs_login`）。有 `suggested_name` 的行才有，别的行是空的 + pub rename_flags: Vec, pub error: Option, /// 它引用的、包里也有的项目。**被跳过的项目被谁引用着，谁就导入不了**（除非引用的那一项 /// 是改名导入）：界面按它把那几行标成错误 diff --git a/crates/tw-config/src/validate.rs b/crates/tw-config/src/validate.rs index 5c53b263..c8117b48 100644 --- a/crates/tw-config/src/validate.rs +++ b/crates/tw-config/src/validate.rs @@ -28,6 +28,14 @@ pub enum ValidationError { DuplicateClientKey(String, String, String), #[error("{}", self.msg())] BadBaseUrl { name: String, url: String }, + /// 地址里有反斜杠：URL 库把它当成路径的分隔（`https://a\\@b/` 连的是 a），人读起来 + /// 像是 b —— 两种读法对不上的地址一律不收 + #[error("{}", self.msg())] + BaseUrlBackslash { name: String, url: String }, + /// 地址的主机前面写了用户名或密码（`https://user:pass@host`）：凭据写在 `key` 或 + /// `headers` 里。消息里不带地址：那一段就是密码 + #[error("{}", self.msg())] + BaseUrlUserinfo { name: String }, #[error("{}", self.msg())] EmptyKey { name: String }, #[error("{}", self.msg())] @@ -208,6 +216,16 @@ impl ValidationError { "config.bad_base_url", upstream = name, url = url => "the endpoint of upstream `{upstream}` is neither http nor https: {url}" ), + BaseUrlBackslash { name, url } => msg!( + "config.base_url_backslash", upstream = name, url = url => + "the endpoint of upstream `{upstream}` contains a backslash: {url}. Write the \ + address with forward slashes only" + ), + BaseUrlUserinfo { name } => msg!( + "config.base_url_userinfo", upstream = name => + "the endpoint of upstream `{upstream}` has a user name or password before the \ + host. Put the credential in `key` or `headers` instead" + ), EmptyKey { name } => msg!( "config.empty_key", key = name => "the value of gateway key `{key}` is empty" @@ -485,6 +503,19 @@ pub fn validate(cfg: &Config) -> Result<(), ValidationError> { url: p.base_url.clone(), }); } + if p.base_url.contains('\\') { + return Err(ValidationError::BaseUrlBackslash { + name: p.name.clone(), + url: p.base_url.clone(), + }); + } + if let Ok(u) = url::Url::parse(p.base_url.trim()) + && (!u.username().is_empty() || u.password().is_some()) + { + return Err(ValidationError::BaseUrlUserinfo { + name: p.name.clone(), + }); + } p.check_credential() .map_err(|source| ValidationError::Credential { name: p.name.clone(), @@ -1222,6 +1253,46 @@ mod tests { )); } + /// 两种读法对不上的地址不收：反斜杠（URL 库当成路径分隔，人读成主机前的一段）、主机前 + /// 的用户名和密码 + #[test] + fn a_base_url_that_reads_two_ways_is_refused() { + for url in [ + "https://evil.example\\@relay.example/v1", + "https://relay.example\\v1", + ] { + let e = validate(&cfg(vec![c("d", "tw-1")], vec![p("r", url)])).unwrap_err(); + assert!( + matches!(e, ValidationError::BaseUrlBackslash { .. }), + "{url}: {e}" + ); + assert_eq!(e.msg().code, "config.base_url_backslash"); + } + for url in [ + "https://user:pass@relay.example/v1", + "https://user@relay.example", + "http://:secret@10.0.0.1:3000", + ] { + let e = validate(&cfg(vec![c("d", "tw-1")], vec![p("r", url)])).unwrap_err(); + assert!( + matches!(e, ValidationError::BaseUrlUserinfo { .. }), + "{url}: {e}" + ); + assert!( + !e.to_string().contains("user:pass") && !e.to_string().contains("secret"), + "{e}" + ); + } + // `@` 在路径里没关系 + assert!( + validate(&cfg( + vec![c("d", "tw-1")], + vec![p("r", "https://relay.example/v1/@team")] + )) + .is_ok() + ); + } + #[test] fn names_that_start_with_two_underscores_are_reserved() { // 「全部上游」在配置里叫 `__all__`。一个同名的上游或策略组会让 @@ -1863,6 +1934,11 @@ mod msg_codes { name: "a".into(), url: "ftp://x".into(), }, + BaseUrlBackslash { + name: "a".into(), + url: "https://a\\@b".into(), + }, + BaseUrlUserinfo { name: "a".into() }, EmptyKey { name: "k".into() }, ZeroConcurrency { name: "k".into() }, ProviderConcurrency { diff --git a/crates/tw-config/tests/manual/schema.rs b/crates/tw-config/tests/manual/schema.rs index 7d01198d..bd25378c 100644 --- a/crates/tw-config/tests/manual/schema.rs +++ b/crates/tw-config/tests/manual/schema.rs @@ -587,8 +587,8 @@ pub fn sections() -> Vec
{ Kind::Str, Def::Required, t( - "Endpoint, `http://` or `https://`, up to the version segment where the provider documents one (`https://api.anthropic.com`, `https://api.openai.com/v1`). For Bedrock, the region's runtime endpoint: `https://bedrock-runtime..amazonaws.com`.", - "接口地址，`http://` 或 `https://`，按服务商文档写到版本段为止（`https://api.anthropic.com`、`https://api.openai.com/v1`）。Bedrock 写所在区域的推理地址：`https://bedrock-runtime.<区域>.amazonaws.com`。", + "Endpoint, `http://` or `https://`, up to the version segment where the provider documents one (`https://api.anthropic.com`, `https://api.openai.com/v1`). For Bedrock, the region's runtime endpoint: `https://bedrock-runtime..amazonaws.com`. No backslashes, and no user name or password before the host (credentials go in `key` or `headers`).", + "接口地址，`http://` 或 `https://`，按服务商文档写到版本段为止（`https://api.anthropic.com`、`https://api.openai.com/v1`）。Bedrock 写所在区域的推理地址：`https://bedrock-runtime.<区域>.amazonaws.com`。地址中不能有反斜杠，主机前不能写用户名或密码（凭据写在 `key` 或 `headers` 中）。", ), ), row( diff --git a/crates/tw-control/src/backups.rs b/crates/tw-control/src/backups.rs index 6aa48518..715e9698 100644 --- a/crates/tw-control/src/backups.rs +++ b/crates/tw-control/src/backups.rs @@ -496,8 +496,17 @@ fn import_fail(e: tw_pack::import::ImportError, pack: &Pack) -> Fail { } } +fn flag_view(f: tw_pack::import::Flag) -> tw_api::ImportFlag { + use tw_pack::import::Flag; + match f { + Flag::MissingKey => tw_api::ImportFlag::MissingKey, + Flag::NeedsLogin => tw_api::ImportFlag::NeedsLogin, + Flag::PluginDisabled => tw_api::ImportFlag::PluginDisabled, + } +} + pub(crate) fn row_view(r: tw_pack::import::Row) -> tw_api::ImportRow { - use tw_pack::import::{Flag, Status}; + use tw_pack::import::Status; tw_api::ImportRow { item: item_ref(&r.item), status: match r.status { @@ -507,17 +516,10 @@ pub(crate) fn row_view(r: tw_pack::import::Row) -> tw_api::ImportRow { Status::Error => tw_api::ImportStatus::Error, }, global: r.global, - flags: r - .flags - .into_iter() - .map(|f| match f { - Flag::MissingKey => tw_api::ImportFlag::MissingKey, - Flag::NeedsLogin => tw_api::ImportFlag::NeedsLogin, - Flag::PluginDisabled => tw_api::ImportFlag::PluginDisabled, - }) - .collect(), + flags: r.flags.into_iter().map(flag_view).collect(), differs: r.differs, suggested_name: r.suggested_name, + rename_flags: r.rename_flags.into_iter().map(flag_view).collect(), error: r.error, refs: r.refs.iter().map(item_ref).collect(), } diff --git a/crates/tw-control/tests/backups.rs b/crates/tw-control/tests/backups.rs index 837da99e..39f480f4 100644 --- a/crates/tw-control/tests/backups.rs +++ b/crates/tw-control/tests/backups.rs @@ -1068,3 +1068,31 @@ async fn a_plugin_put_back_at_startup_runs_again() { mgr.record_startup(); assert_eq!(tw_pack::backup::list(&p).unwrap().len(), before); } + +/// 导入预览的每一行带着改名导入时的标记：不带密钥的包里，改名导入的上游缺着密钥进来 +#[tokio::test] +async fn an_import_row_says_what_a_rename_would_bring() { + let src = bed(&theirs()); + let file = export_all(&src, false).await; + let dst = bed(&mine()); + let (st, preview) = call( + &dst.app, + "POST", + "/config/import/preview", + serde_json::json!({ "data_b64": file["data_b64"] }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{preview}"); + let rows = preview["rows"].as_array().unwrap(); + let official = rows + .iter() + .find(|r| r["item"]["kind"] == "provider" && r["item"]["name"] == "官方") + .unwrap(); + assert_eq!(official["status"], "conflict", "{official}"); + // 只换了路径：覆盖沿用本机的密钥，改名导入不沿用 + assert_eq!(official["flags"], serde_json::json!([])); + assert_eq!(official["rename_flags"], serde_json::json!(["missing_key"])); + let relay = rows.iter().find(|r| r["item"]["name"] == "relay").unwrap(); + assert_eq!(relay["status"], "new"); + assert_eq!(relay["rename_flags"], serde_json::json!([])); +} diff --git a/crates/tw-pack/Cargo.toml b/crates/tw-pack/Cargo.toml index ba3b604b..3e649fe4 100644 --- a/crates/tw-pack/Cargo.toml +++ b/crates/tw-pack/Cargo.toml @@ -29,6 +29,8 @@ blake3 = { workspace = true } sha2 = { workspace = true } age = { workspace = true } zip = { workspace = true } +# 比较上游地址发去哪儿：和 reqwest 同一个解析器，两边对同一个地址的读法一样 +url = { workspace = true } [dev-dependencies] tempfile = "3" diff --git a/crates/tw-pack/src/export.rs b/crates/tw-pack/src/export.rs index fffad476..0f973cf7 100644 --- a/crates/tw-pack/src/export.rs +++ b/crates/tw-pack/src/export.rs @@ -167,11 +167,11 @@ pub fn strip_secrets(snap: &mut Snapshot) { /// AWS 访问密钥、OAuth 的客户端密钥，代理的密码，网关密钥的值。比较和覆盖都照这个样子：除了 /// 密钥都一样的就是「相同」，覆盖不会把本机的密钥抹成空的。 /// -/// **只在目的地不变时**（[`same_destination`]）：包里的上游换了主机、端口、AWS 区域或 OAuth -/// 令牌端点，代理换了地址，本机的密钥就不跟过去 —— 否则一份不带密钥的包改个地址，覆盖之后 -/// 本机的密钥就发给了包里写的那个地方 -pub fn fill_secrets(id: &ItemId, item: &mut model::Item, mine: &model::Item) { - if !same_destination(id, item, mine) { +/// **只在目的地不变时**（[`same_destination`]）：包里的上游换了主机、端口、AWS 区域、OAuth +/// 令牌端点或者走的出站代理，代理换了地址，本机的密钥就不跟过去 —— 否则一份不带密钥的包改个 +/// 地址，覆盖之后本机的密钥就发给了包里写的那个地方 +pub fn fill_secrets(id: &ItemId, item: &mut model::Item, mine: &model::Item, sides: Sides<'_>) { + if !same_destination(id, item, mine, sides) { return; } let empty = |v: Option<&Json>| v.and_then(Json::as_str) == Some(""); @@ -244,16 +244,24 @@ pub fn fill_secrets(id: &ItemId, item: &mut model::Item, mine: &model::Item) { } } -/// 两份同名的项目，密钥是发去同一个地方的吗：上游的协议、主机、端口一样，AWS 区域一样， -/// OAuth 的令牌端点一样；代理的类型和地址一样。网关密钥是发给这台网关自己的，总是 -pub fn same_destination(id: &ItemId, theirs: &model::Item, mine: &model::Item) -> bool { - let at = |it: &model::Item, u: &str, path: &[&str]| -> Option { - let mut cur = it.units.get(u)?; - for p in path { - cur = cur.get(*p)?; - } - Some(cur.clone()) - }; +/// 比较目的地时两边的全部项目：上游走的出站代理按名字在这里找。 +#[derive(Debug, Clone, Copy)] +pub struct Sides<'a> { + /// 包里的（导入时可能已经改过名）。包里没有的代理按本机的找：导入之后用的就是本机那个 + pub theirs: &'a Snapshot, + /// 本机的 + pub mine: &'a Snapshot, +} + +/// 两份同名的项目，密钥是发去同一个地方的吗：上游的协议、主机、端口一样（[`destination`]）， +/// AWS 区域一样，OAuth 的令牌端点一样，走的出站代理一样（直连、系统代理，或者类型和地址一样 +/// 的代理）；代理的类型和地址一样。网关密钥是发给这台网关自己的，总是 +pub fn same_destination( + id: &ItemId, + theirs: &model::Item, + mine: &model::Item, + sides: Sides<'_>, +) -> bool { match id.kind { ItemKind::Provider => { let url = |it: &model::Item| { @@ -265,44 +273,57 @@ pub fn same_destination(id: &ItemId, theirs: &model::Item, mine: &model::Item) - == at(mine, unit::CONN, &["aws", "region"]) && at(theirs, unit::CONN, &["oauth", "endpoint"]) == at(mine, unit::CONN, &["oauth", "endpoint"]) + && route_of(theirs, &[sides.theirs, sides.mine]).is_some() + && route_of(theirs, &[sides.theirs, sides.mine]) == route_of(mine, &[sides.mine]) } - ItemKind::Proxy => { - let addr = |it: &model::Item| { - at(it, unit::ALL, &["addr"]) - .and_then(|v| v.as_str().map(|a| a.trim().to_ascii_lowercase())) - }; - addr(theirs).is_some() - && addr(theirs) == addr(mine) - && at(theirs, unit::ALL, &["type"]) == at(mine, unit::ALL, &["type"]) - } + ItemKind::Proxy => proxy_at(theirs).is_some() && proxy_at(theirs) == proxy_at(mine), _ => true, } } -/// 一个地址发去哪儿：协议、主机（小写）、端口（没写时按协议的缺省）。读不懂是 None +fn at(it: &model::Item, u: &str, path: &[&str]) -> Option { + let mut cur = it.units.get(u)?; + for p in path { + cur = cur.get(*p)?; + } + Some(cur.clone()) +} + +/// 一个代理连去哪儿：类型和地址（地址去掉空白、小写） +fn proxy_at(it: &model::Item) -> Option<(String, String)> { + let kind = at(it, unit::ALL, &["type"])?.as_str()?.to_string(); + let addr = at(it, unit::ALL, &["addr"])? + .as_str()? + .trim() + .to_ascii_lowercase(); + Some((kind, addr)) +} + +/// 一个上游发出去走哪条路：直连、系统代理，或者一个出站代理的类型和地址（按名字依次在 +/// `snaps` 里找）。找不到那个代理是 None +fn route_of(it: &model::Item, snaps: &[&Snapshot]) -> Option<(String, String)> { + let name = at(it, unit::CONN, &["proxy"]) + .and_then(|v| v.as_str().map(str::to_string)) + .unwrap_or_else(|| tw_config::DIRECT.to_string()); + if name == tw_config::DIRECT || name == tw_config::SYSTEM { + return Some((name, String::new())); + } + let id = ItemId::new(ItemKind::Proxy, name); + snaps + .iter() + .find_map(|s| s.items.get(&id)) + .and_then(proxy_at) +} + +/// 一个地址发去哪儿：协议、主机、端口（没写时按协议的缺省）。**和 reqwest 用同一个解析器** +/// （`url`，WHATWG 的规矩）：主机是规范化之后的（小写、国际化域名转成 punycode、IPv6 和 IPv4 +/// 写成标准形式），反斜杠和网关发请求时一样当成路径的分隔。末尾的点不去掉：单个名字加不加点 +/// 解析到的可能不是同一台。读不懂是 None pub fn destination(url: &str) -> Option<(String, String, u16)> { - let (scheme, rest) = url.trim().split_once("://")?; - let scheme = scheme.to_ascii_lowercase(); - let authority = rest.split(['/', '?', '#']).next()?; - let hostport = authority.rsplit('@').next()?; - let (host, port) = if let Some(v6) = hostport.strip_prefix('[') { - let (h, after) = v6.split_once(']')?; - (h.to_string(), after.strip_prefix(':')) - } else { - match hostport.rsplit_once(':') { - Some((h, p)) => (h.to_string(), Some(p)), - None => (hostport.to_string(), None), - } - }; - let port = match port { - Some(p) => p.parse().ok()?, - None => match scheme.as_str() { - "https" => 443, - "http" => 80, - _ => return None, - }, - }; - (!host.is_empty()).then(|| (scheme, host.to_ascii_lowercase(), port)) + let u = url::Url::parse(url.trim()).ok()?; + let host = u.host_str()?.to_string(); + let port = u.port_or_known_default()?; + Some((u.scheme().to_string(), host, port)) } /// 一项里有没有被去掉的密钥（空字符串） @@ -362,26 +383,44 @@ fn civil(days: u64) -> (i64, u32, u32) { mod tests { use super::*; + /// 和 reqwest 读法一样：反斜杠是路径的分隔，主机规范化，缺省端口补上 #[test] fn a_destination_is_scheme_host_and_port() { let d = |u| destination(u); + let relay = Some(("https".to_string(), "relay.example".to_string(), 443)); + assert_eq!(d("https://Relay.EXAMPLE/v1"), relay); + assert_eq!(d("https://relay.example:443/x"), relay); + assert_eq!(d(" https://relay.example "), relay); + assert_ne!(d("https://relay.example:8443"), relay); + assert_ne!(d("http://relay.example"), relay); + assert_eq!(d("http://relay.example:80"), d("http://relay.example")); + // 反斜杠：连的是前面那一段，不是 @ 后面的 assert_eq!( - d("https://Relay.example/v1"), - Some(("https".into(), "relay.example".into(), 443)) + d("https://evil.example\\@relay.example/v1"), + Some(("https".into(), "evil.example".into(), 443)) ); + // 主机前的用户名和密码不算进主机 + assert_eq!(d("https://user:pw@relay.example/v1"), relay); + // 末尾的点是另一个名字 + assert_ne!(d("https://relay.example./v1"), relay); + // 国际化域名：写成 Unicode 和 punycode 是同一个 assert_eq!( - d("https://relay.example:443/x"), - d("https://relay.example/y") + d("https://bücher.example"), + d("https://xn--bcher-kva.example") ); - assert_ne!(d("https://relay.example:8443"), d("https://relay.example")); - assert_ne!(d("http://relay.example"), d("https://relay.example")); assert_eq!( - d("http://user:pw@[::1]:8080/p"), - Some(("http".into(), "::1".into(), 8080)) + d("https://BÜCHER.example").unwrap().1, + "xn--bcher-kva.example" ); + // IPv6、IPv4 写成标准形式 + assert_eq!( + d("http://[0:0::1]:8080/p"), + Some(("http".into(), "[::1]".into(), 8080)) + ); + assert_eq!(d("http://0x7f.1/"), d("http://127.0.0.1")); assert_eq!(d("not a url"), None); + assert_eq!(d("https://"), None); } - #[test] fn the_file_name_carries_the_date() { assert_eq!(file_name(0), "ThinkWatch-1970-01-01.twconfig"); diff --git a/crates/tw-pack/src/import.rs b/crates/tw-pack/src/import.rs index d61b6ea8..2ced8fb3 100644 --- a/crates/tw-pack/src/import.rs +++ b/crates/tw-pack/src/import.rs @@ -55,6 +55,9 @@ pub struct Row { /// 不同的合并单位 pub differs: Vec, pub suggested_name: Option, + /// 改名导入时这一项带的标记（`flags` 说的是导入、覆盖）：改名导入的是另一个项目，**不沿用 + /// 本机的密钥和登录**，不带密钥的包里缺着的就缺着、上游停用。有 `suggested_name` 的行才有 + pub rename_flags: Vec, pub error: Option, /// 它引用的、包里也有的项目 pub refs: Vec, @@ -187,9 +190,14 @@ fn with_local_secrets(l: &Snapshot, mut i: Snapshot, incoming: &Pack) -> Snapsho if incoming.manifest.secrets { return i; } + let theirs = i.clone(); + let sides = export::Sides { + theirs: &theirs, + mine: l, + }; for (id, item) in i.items.iter_mut() { if let Some(mine) = l.items.get(id) { - export::fill_secrets(id, item, mine); + export::fill_secrets(id, item, mine, sides); } } i @@ -212,7 +220,8 @@ pub fn preview_with( codec: &dyn PluginCodec, ) -> Result, ImportError> { let l = model::snapshot_with(local, codec)?; - let i = with_local_secrets(&l, model::snapshot_with(incoming, codec)?, incoming); + let raw = model::snapshot_with(incoming, codec)?; + let i = with_local_secrets(&l, raw.clone(), incoming); let both = union(&l, &i); let mut rows = Vec::new(); for (id, item) in &i.items { @@ -227,6 +236,7 @@ pub fn preview_with( flags: flags(id, item, incoming, &l), differs: Vec::new(), suggested_name: None, + rename_flags: Vec::new(), error: None, refs: both.items[id] .refs @@ -243,6 +253,8 @@ pub fn preview_with( row.differs = differing(&mine.units, &item.units); if !global { row.suggested_name = Some(free_name(&both, id.kind, &suggest_base(id))); + // 改名导入时本机没有同名的那一项可沿用：照包里原样（没填回本机的密钥）算 + row.rename_flags = flags(id, &raw.items[id], incoming, &Snapshot::default()); } } } @@ -562,7 +574,15 @@ pub fn apply_with( && !matches!(d.action, Action::Rename(_)) && let Some(mine) = l.items.get(&id) { - export::fill_secrets(&id, &mut item, mine); + export::fill_secrets( + &id, + &mut item, + mine, + export::Sides { + theirs: &i, + mine: &l, + }, + ); } let item = &item; match &d.action { diff --git a/crates/tw-pack/src/restore.rs b/crates/tw-pack/src/restore.rs index 281002f6..288876c0 100644 --- a/crates/tw-pack/src/restore.rs +++ b/crates/tw-pack/src/restore.rs @@ -486,6 +486,18 @@ pub fn whole(backup: &Pack, current_text: &str, last_local: Option<&Pack>) -> Pa }; let source = doc(current_text).or_else(|| last_local.and_then(|p| doc(&p.config_yaml))); let mut text = backup.config_yaml.clone(); + // 那一版是同步位置上的（没有本机那几节）：本机不同步的上游本来就不在里面 + let from_sync = backup.manifest.origin == crate::PackOrigin::Sync + || doc(&backup.config_yaml) + .is_some_and(|d| model::LOCAL_SECTIONS.iter().all(|k| d.get(*k).is_none())); + if from_sync { + let current = Pack { + config_yaml: current_text.to_string(), + ..backup.clone() + }; + let from: Vec<&Pack> = std::iter::once(¤t).chain(last_local).collect(); + text = bring_not_synced(&text, &from); + } // 两处都取不到（原文坏到读不成 YAML、也没有本机备份）：留着那一版自己的 for section in source.iter().flat_map(|_| model::LOCAL_SECTIONS) { let v = source.as_ref().and_then(|d| d.get(*section)).cloned(); @@ -540,6 +552,107 @@ pub fn whole(backup: &Pack, current_text: &str, last_local: Option<&Pack>) -> Pa } } +/// 本机不同步的上游（`sync.exclude`），连同直接或间接引用它们的项目（路由、策略组、别名、 +/// 网关密钥）。和同步那一路的算法一样 +fn not_synced(names: &BTreeSet, snap: &Snapshot) -> BTreeSet { + let mut out: BTreeSet = names + .iter() + .map(|n| ItemId::new(ItemKind::Provider, n.clone())) + .collect(); + loop { + let before = out.len(); + for (id, it) in &snap.items { + if !id.kind.is_order() && it.refs.iter().any(|r| out.contains(r)) { + out.insert(id.clone()); + } + } + if out.len() == before { + return out; + } + } +} + +/// 整份恢复同步位置上的一个版本：本机不同步的上游和随之不同步的项目，那一版里没有的从 `from` +/// 里第一份读得成一份配置的（现在的原文，或者最近一个读得成的本机版本）原样取来，连同它们的 +/// 登录，接在各段的末尾。插件不取：它的文件和批准不在配置里 +fn bring_not_synced(text: &str, from: &[&Pack]) -> String { + use tw_config::edit::{self, Section}; + const CLIENTS: Section = Section { + path: &["clients"], + what: "gateway key", + key: "name", + }; + let Some((src, snap, raw)) = from.iter().find_map(|p| { + Some(( + tw_config::try_parse(&p.config_yaml).ok()?, + model::snapshot(p).ok()?, + edit::parse(&p.config_yaml).ok()?, + )) + }) else { + return text.to_string(); + }; + let names: BTreeSet = src + .sync + .as_ref() + .map(|s| s.exclude.iter().cloned().collect()) + .unwrap_or_default(); + if names.is_empty() { + return text.to_string(); + } + let mut out = text.to_string(); + // 先上游，再引用它们的 + let mut ids: Vec = not_synced(&names, &snap) + .into_iter() + .filter(|id| snap.items.contains_key(id)) + .collect(); + ids.sort_by_key(|id| match id.kind { + ItemKind::Provider => 0, + ItemKind::Group => 1, + ItemKind::Route => 2, + ItemKind::Alias => 3, + _ => 4, + }); + for id in ids { + let Ok(now) = edit::parse(&out) else { break }; + let section = match id.kind { + ItemKind::Provider => edit::PROVIDERS, + ItemKind::Group => edit::GROUPS, + ItemKind::Route => edit::ROUTES, + ItemKind::Key => CLIENTS, + ItemKind::Alias => { + let there = now + .get("aliases") + .and_then(|a| a.get(id.name.as_str())) + .is_some(); + if !there + && let Some(a) = src.aliases.find(&id.name) + && let Ok(t) = edit::upsert_alias(&out, None, a) + { + out = t; + } + continue; + } + _ => continue, + }; + if section.index_of(&now, &id.name).is_some() { + continue; + } + let entry = section.index_of(&raw, &id.name).and_then(|i| { + let mut cur = &raw; + for k in section.path { + cur = cur.get(*k)?; + } + cur.get(i)?.as_mapping().cloned() + }); + if let Some(m) = entry + && let Ok(t) = edit::upsert(&out, section, None, &m) + { + out = t; + } + } + out +} + /// YAML 坏到按路径找不到时，退一步在原文里找 `key: <64 个十六进制字符>` 那一行 fn raw_key_line(text: &str) -> Option { text.lines().find_map(|l| { diff --git a/crates/tw-pack/tests/regressions.rs b/crates/tw-pack/tests/regressions.rs index 44544fe3..f8e20166 100644 --- a/crates/tw-pack/tests/regressions.rs +++ b/crates/tw-pack/tests/regressions.rs @@ -513,3 +513,159 @@ fn a_partial_preview_is_what_the_restore_writes() { let out = tw_pack::restore::apply_opts(&now, &then, &opts).unwrap(); assert_eq!(out.rows, rows); } + +/// 一个两种读法对不上的地址：`https://evil.example\@relay.example/v1`，人读成 relay.example， +/// 网关用的 URL 库连的是 evil.example。比较目的地照网关的读法；配置本来就不收这样的地址 +#[test] +fn an_address_that_reads_two_ways_does_not_take_the_key_along() { + let tricky = "https://evil.example\\@relay.example/v1"; + assert_eq!( + tw_pack::export::destination(tricky).map(|d| d.1), + Some("evil.example".to_string()) + ); + let l = local(); + let relay = ItemId::new(ItemKind::Provider, "relay"); + for (url, code) in [ + (tricky, "config.base_url_backslash"), + ("https://u:p@relay.example/v1", "config.base_url_userinfo"), + ] { + let mut incoming = export(&l, false, PackOrigin::Export); + incoming.config_yaml = format!( + "version: 1\nproviders:\n - name: relay\n base_url: '{url}'\n key: ''\n proxy: px\n" + ); + let rows = tw_pack::import::preview(&l, &incoming, &ctx()).unwrap(); + let row = rows.iter().find(|r| r.item == relay).unwrap(); + assert_eq!(row.status, Status::Error, "{url}"); + assert_eq!(row.error.as_ref().unwrap().code, code, "{url}"); + assert!( + tw_pack::import::apply(&l, &incoming, &[decide(&relay, Action::Overwrite)], &ctx()) + .is_err() + ); + } +} + +/// 地址不变、换了出站代理：走的是另一条路，本机的密钥不跟过去。走的代理类型和地址一样 +/// （哪怕换了个名字）照样沿用 +#[test] +fn a_new_outbound_proxy_does_not_take_the_key_along() { + let here = pack(&format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n key: tw-local\nproxies:\n - name: px\n type: http\n addr: 10.0.0.1:8080\nproviders:\n - name: lan\n base_url: http://192.168.1.10:3000/v1\n key: sk-lan-secret\n proxy: px\n" + )); + let lan = ItemId::new(ItemKind::Provider, "lan"); + let with = |proxies: &str, proxy: &str| { + let mut p = export(&here, false, PackOrigin::Export); + p.config_yaml = format!( + "version: 1\n{proxies}providers:\n - name: lan\n base_url: http://192.168.1.10:3000/v1\n key: ''\n proxy: {proxy}\n" + ); + p + }; + let key_after = |incoming: &Pack| { + let rows = tw_pack::import::preview(&here, incoming, &ctx()).unwrap(); + let decisions: Vec = rows + .iter() + .map(|r| { + decide( + &r.item, + if r.status == Status::New { + Action::Import + } else { + Action::Overwrite + }, + ) + }) + .collect(); + let out = tw_pack::import::apply(&here, incoming, &decisions, &ctx()).unwrap(); + let cfg = tw_config::try_parse(&out.pack.config_yaml).unwrap(); + let p = cfg + .providers + .iter() + .find(|p| p.name == "lan") + .unwrap() + .clone(); + let flags = rows.iter().find(|r| r.item == lan).unwrap().flags.clone(); + (p.key.map(|k| k.raw().to_string()), p.disabled, flags) + }; + let gone = (Some(String::new()), true, vec![Flag::MissingKey]); + // 包里新带一个代理 + let evil = "proxies:\n - name: evilpx\n type: http\n addr: proxy.evil.example:8080\n"; + assert_eq!(key_after(&with(evil, "evilpx")), gone); + // 同名的代理换了地址 + let moved = "proxies:\n - name: px\n type: http\n addr: proxy.evil.example:8080\n"; + assert_eq!(key_after(&with(moved, "px")), gone); + // 改成直连、系统代理 + assert_eq!(key_after(&with("", "direct")), gone); + assert_eq!(key_after(&with("", "system")), gone); + // 同一个代理（包里没带它，用的是本机的）；换了名字、类型和地址一样的 + let kept = (Some("sk-lan-secret".to_string()), false, Vec::::new()); + assert_eq!(key_after(&with("", "px")), kept); + let renamed = "proxies:\n - name: px2\n type: http\n addr: 10.0.0.1:8080\n"; + assert_eq!(key_after(&with(renamed, "px2")), kept); +} + +/// 改名导入：预览如实说改名之后缺密钥、停用（`rename_flags`），写下去也是这样 +#[test] +fn the_preview_says_what_a_rename_brings() { + let l = local(); + let mut incoming = export(&l, false, PackOrigin::Export); + incoming.config_yaml = "version: 1\nproxies:\n - name: px\n type: http\n addr: 10.0.0.1:8080\n auth: { user: u, pass: '' }\nproviders:\n - name: relay\n base_url: https://relay.example\n key: ''\n proxy: px\n headers:\n X-Token: ''\n max_concurrent: 2\n".into(); + let rows = tw_pack::import::preview(&l, &incoming, &ctx()).unwrap(); + let row = |kind, name: &str| { + rows.iter() + .find(|r| r.item == ItemId::new(kind, name)) + .unwrap() + }; + let relay = row(ItemKind::Provider, "relay"); + assert_eq!(relay.status, Status::Conflict); + assert!(relay.flags.is_empty(), "{:?}", relay.flags); + assert_eq!(relay.rename_flags, [Flag::MissingKey]); + // 相同的那一项没有改名可选 + let px = row(ItemKind::Proxy, "px"); + assert_eq!(px.status, Status::Same); + assert!(px.rename_flags.is_empty()); + let out = tw_pack::import::apply( + &l, + &incoming, + &[decide(&relay.item, Action::Rename("relay (2)".into()))], + &ctx(), + ) + .unwrap(); + let cfg = tw_config::try_parse(&out.pack.config_yaml).unwrap(); + let p = cfg + .providers + .iter() + .find(|p| p.name == "relay (2)") + .unwrap(); + assert_eq!(p.key.as_ref().map(|k| k.raw()), Some("")); + assert!(p.disabled); +} + +/// 安全模式下整份恢复同步位置上的版本：本机不同步的上游（和指向它的路由、绑着那条路由的 +/// 密钥）那一版里本来就没有，从最近一个读得成的本机版本里取回来 +#[test] +fn a_whole_restore_of_a_synced_version_keeps_what_this_machine_does_not_sync() { + let here = pack(&format!( + "version: 1\nlisten:\n control:\n key: {KEY}\nclients:\n - name: default\n key: tw-a\n - name: local-cli\n key: tw-l\n route: local\nproxies:\n - name: corp\n type: socks5h\n addr: 127.0.0.1:7890\nproviders:\n - name: a\n base_url: https://a.example\n key: sk-a\n - name: ollama\n base_url: http://127.0.0.1:11434\nroutes:\n - name: local\n rules:\n - name: all\n to: ollama\n{SYNC} exclude: [ollama]\n" + )); + tw_config::try_parse(&here.config_yaml).unwrap_or_else(|e| panic!("{e}\n{}", here.config_yaml)); + let mut synced = pack( + "version: 1\nclients:\n - name: default\n key: tw-a\nproviders:\n - name: a\n base_url: https://a2.example\n key: sk-a\n", + ); + synced.manifest.origin = PackOrigin::Sync; + // 现在这份读不成 YAML：本机那几节和不同步的上游都从最近一个本机版本里取 + let not_yaml = "listen: [\n :::\n"; + let out = tw_pack::restore::whole(&synced, not_yaml, Some(&here)); + let cfg = tw_config::try_parse(&out.config_yaml) + .unwrap_or_else(|e| panic!("{e}\n{}", out.config_yaml)); + let names: Vec<&str> = cfg.providers.iter().map(|p| p.name.as_str()).collect(); + assert_eq!(names, ["a", "ollama"]); + assert_eq!(cfg.providers[0].base_url, "https://a2.example"); + assert!(cfg.routes.iter().any(|r| r.name == "local")); + assert!(cfg.clients.iter().any(|c| c.name == "local-cli")); + assert_eq!(cfg.sync.unwrap().exclude, ["ollama"]); + // 读得成 YAML、读不成一份配置：一样 + let broken = format!("{}not_a_field: 1\n", here.config_yaml); + let out = tw_pack::restore::whole(&synced, &broken, Some(&here)); + let cfg = tw_config::try_parse(&out.config_yaml) + .unwrap_or_else(|e| panic!("{e}\n{}", out.config_yaml)); + assert!(cfg.providers.iter().any(|p| p.name == "ollama")); +} diff --git a/docs/config.md b/docs/config.md index 071833a5..f0788e96 100644 --- a/docs/config.md +++ b/docs/config.md @@ -145,9 +145,9 @@ A pack carries no settings of this machine and no ChatGPT sign-in; without passphrase is asked for, or read from `TWCORE_PASSPHRASE`). Overwriting an item here with one from a pack without keys keeps this machine's keys only while the item still points at the same place (scheme, host and port; AWS -region; OAuth token endpoint; proxy address); otherwise, and for an item -imported under another name, the upstream comes in disabled until its key -is entered. +region; OAuth token endpoint; the outbound proxy it goes through; a proxy's +address); otherwise, and for an item imported under another name, the +upstream comes in disabled until its key is entered. These commands work on the files directly, so they work when core is not running, which is when a restore is most needed; when the file does not @@ -410,7 +410,7 @@ Upstreams: the APIs requests are forwarded to. | Field | Type | Default | Description | |---|---|---|---| | `name` | string | **required** | Name of the upstream; unique, and not the name of a group. Names starting with `__` are reserved. | -| `base_url` | string | **required** | Endpoint, `http://` or `https://`, up to the version segment where the provider documents one (`https://api.anthropic.com`, `https://api.openai.com/v1`). For Bedrock, the region's runtime endpoint: `https://bedrock-runtime..amazonaws.com`. | +| `base_url` | string | **required** | Endpoint, `http://` or `https://`, up to the version segment where the provider documents one (`https://api.anthropic.com`, `https://api.openai.com/v1`). For Bedrock, the region's runtime endpoint: `https://bedrock-runtime..amazonaws.com`. No backslashes, and no user name or password before the host (credentials go in `key` or `headers`). | | `key` | string, `${VAR}` allowed | — | API key. It goes in the header the protocol expects: `x-api-key` (Anthropic), `Authorization: Bearer` (OpenAI, and a Bedrock API key), `x-goog-api-key` (Gemini). Leave it out for upstreams without a key, or when the credential is written in `headers`. Cannot be combined with `oauth` or `aws`. | | `headers` | map of header name → value | `{}` | Additional request headers, in the order written; values may use `${VAR}`, and `{{access_token}}` where `oauth` is set. At most 32. Headers HTTP or the gateway manages (`host`, `content-length`, `connection`, …) cannot be set. | | `oauth` | object, [`providers[].oauth`](#cfg-providers-oauth) | — | OAuth credential: an access token obtained from a refresh token. Instead of `key`. | diff --git a/docs/config.zh-CN.md b/docs/config.zh-CN.md index aa16e5ac..751d2ed5 100644 --- a/docs/config.zh-CN.md +++ b/docs/config.zh-CN.md @@ -81,7 +81,7 @@ twcore import all.twconfig --dry-run # 导入会做什么 twcore config set /listen/gateway/port 8790 --int ``` -恢复时保留本机设置（监听、控制面密钥、保留期限、同步设置）、ChatGPT 的登录，以及本机已接管的客户端正在使用的网关密钥。导入时加入本机没有的项目，与本机不同的项目逐项选择处理方式（`--on-conflict` 一次指定全部）。配置包不含本机设置与 ChatGPT 登录；未指定 `--no-secrets` 时其中以明文保存密钥，除非指定 `--encrypt`（口令在终端输入，或取自环境变量 `TWCORE_PASSPHRASE`）。以不含密钥的配置包覆盖本机项目时，只有目的地不变（协议、主机与端口，AWS 区域，OAuth 令牌端点，代理地址）才沿用本机的密钥；目的地变了的、改名导入的上游以停用状态导入，填入密钥后再启用。 +恢复时保留本机设置（监听、控制面密钥、保留期限、同步设置）、ChatGPT 的登录，以及本机已接管的客户端正在使用的网关密钥。导入时加入本机没有的项目，与本机不同的项目逐项选择处理方式（`--on-conflict` 一次指定全部）。配置包不含本机设置与 ChatGPT 登录；未指定 `--no-secrets` 时其中以明文保存密钥，除非指定 `--encrypt`（口令在终端输入，或取自环境变量 `TWCORE_PASSPHRASE`）。以不含密钥的配置包覆盖本机项目时，只有目的地不变（协议、主机与端口，AWS 区域，OAuth 令牌端点，所走的出站代理，代理的地址）才沿用本机的密钥；目的地变了的、改名导入的上游以停用状态导入，填入密钥后再启用。 这些命令直接操作文件，因此 core 没有运行时也能使用，而那往往正是最需要恢复的时候；配置完全无法读取时，恢复把整个版本写回，控制面密钥保持现在的值。正在运行的 core 会像对待其他保存一样接收这些改动。 @@ -294,7 +294,7 @@ clients: | 字段 | 类型 | 默认值 | 说明 | |---|---|---|---| | `name` | 字符串 | **必填** | 上游的名字，不能重复，也不能和策略组同名。以 `__` 开头的名字保留给内置项。 | -| `base_url` | 字符串 | **必填** | 接口地址，`http://` 或 `https://`，按服务商文档写到版本段为止（`https://api.anthropic.com`、`https://api.openai.com/v1`）。Bedrock 写所在区域的推理地址：`https://bedrock-runtime.<区域>.amazonaws.com`。 | +| `base_url` | 字符串 | **必填** | 接口地址，`http://` 或 `https://`，按服务商文档写到版本段为止（`https://api.anthropic.com`、`https://api.openai.com/v1`）。Bedrock 写所在区域的推理地址：`https://bedrock-runtime.<区域>.amazonaws.com`。地址中不能有反斜杠，主机前不能写用户名或密码（凭据写在 `key` 或 `headers` 中）。 | | `key` | 字符串，可写 `${VAR}` | — | API 密钥，放进协议规定的请求头：`x-api-key`（Anthropic）、`Authorization: Bearer`（OpenAI，以及 Bedrock API Key）、`x-goog-api-key`（Gemini）。上游不需要密钥、或凭据写在 `headers` 里时不写。不能和 `oauth`、`aws` 同时写。 | | `headers` | 请求头名 → 值的映射 | `{}` | 额外的请求头，按书写顺序发送；值可以用 `${VAR}`，配置了 `oauth` 时可以用 `{{access_token}}`。最多 32 个。HTTP 或网关管理的请求头（`host`、`content-length`、`connection` 等）不能设置。 | | `oauth` | 对象，见 [`providers[].oauth`](#cfg-providers-oauth) | — | OAuth 凭据：用 refresh token 换取 access token。与 `key` 二选一。 | From f929f33616f1931be350740b81df8f8750a03f66 Mon Sep 17 00:00:00 2001 From: fylorn <249551762+fylorn@users.noreply.github.com> Date: Mon, 19 Oct 2026 20:56:53 +0800 Subject: [PATCH 12/13] Cargo.lock: tw-pack and tw-sync follow the workspace version Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3eb26381..cb4b4a43 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3896,7 +3896,7 @@ dependencies = [ [[package]] name = "tw-pack" -version = "0.76.0" +version = "0.77.0" dependencies = [ "age", "blake3", @@ -3975,7 +3975,7 @@ dependencies = [ [[package]] name = "tw-sync" -version = "0.76.0" +version = "0.77.0" dependencies = [ "age", "axum", From bcc90042d6edc63df4370074a62e4ce854d7334a Mon Sep 17 00:00:00 2001 From: fylorn <249551762+fylorn@users.noreply.github.com> Date: Mon, 19 Oct 2026 21:08:14 +0800 Subject: [PATCH 13/13] v0.78.0 Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 42 ++++++++++++++++++++--------------------- Cargo.toml | 2 +- release-notes/0.78.0.md | 30 +++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 22 deletions(-) create mode 100644 release-notes/0.78.0.md diff --git a/Cargo.lock b/Cargo.lock index cb4b4a43..88ddbb21 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3670,7 +3670,7 @@ dependencies = [ [[package]] name = "tw-api" -version = "0.77.0" +version = "0.78.0" dependencies = [ "serde", "serde_json", @@ -3681,7 +3681,7 @@ dependencies = [ [[package]] name = "tw-bedrock" -version = "0.77.0" +version = "0.78.0" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -3702,7 +3702,7 @@ dependencies = [ [[package]] name = "tw-breaker" -version = "0.77.0" +version = "0.78.0" dependencies = [ "serde", "serde_json", @@ -3710,7 +3710,7 @@ dependencies = [ [[package]] name = "tw-config" -version = "0.77.0" +version = "0.78.0" dependencies = [ "blake3", "libc", @@ -3736,7 +3736,7 @@ dependencies = [ [[package]] name = "tw-control" -version = "0.77.0" +version = "0.78.0" dependencies = [ "axum", "base64 0.22.1", @@ -3781,7 +3781,7 @@ dependencies = [ [[package]] name = "tw-dialect" -version = "0.77.0" +version = "0.78.0" dependencies = [ "memchr", "serde", @@ -3790,7 +3790,7 @@ dependencies = [ [[package]] name = "tw-engine" -version = "0.77.0" +version = "0.78.0" dependencies = [ "chrono", "serde", @@ -3804,7 +3804,7 @@ dependencies = [ [[package]] name = "tw-gateway" -version = "0.77.0" +version = "0.78.0" dependencies = [ "arc-swap", "async-stream", @@ -3855,7 +3855,7 @@ dependencies = [ [[package]] name = "tw-guard" -version = "0.77.0" +version = "0.78.0" dependencies = [ "aho-corasick", "base64 0.22.1", @@ -3873,7 +3873,7 @@ dependencies = [ [[package]] name = "tw-link" -version = "0.77.0" +version = "0.78.0" dependencies = [ "serde", "serde_json", @@ -3887,7 +3887,7 @@ dependencies = [ [[package]] name = "tw-observe" -version = "0.77.0" +version = "0.78.0" dependencies = [ "tokio", "tracing", @@ -3896,7 +3896,7 @@ dependencies = [ [[package]] name = "tw-pack" -version = "0.77.0" +version = "0.78.0" dependencies = [ "age", "blake3", @@ -3920,7 +3920,7 @@ dependencies = [ [[package]] name = "tw-plugin" -version = "0.77.0" +version = "0.78.0" dependencies = [ "libc", "rand 0.10.2", @@ -3935,7 +3935,7 @@ dependencies = [ [[package]] name = "tw-pricing" -version = "0.77.0" +version = "0.78.0" dependencies = [ "arc-swap", "flate2", @@ -3948,14 +3948,14 @@ dependencies = [ [[package]] name = "tw-secret" -version = "0.77.0" +version = "0.78.0" dependencies = [ "thiserror 2.0.20", ] [[package]] name = "tw-store" -version = "0.77.0" +version = "0.78.0" dependencies = [ "blake3", "bytes", @@ -3975,7 +3975,7 @@ dependencies = [ [[package]] name = "tw-sync" -version = "0.77.0" +version = "0.78.0" dependencies = [ "age", "axum", @@ -4003,7 +4003,7 @@ dependencies = [ [[package]] name = "tw-types" -version = "0.77.0" +version = "0.78.0" dependencies = [ "serde", "serde_json", @@ -4012,7 +4012,7 @@ dependencies = [ [[package]] name = "tw-watch" -version = "0.77.0" +version = "0.78.0" dependencies = [ "libc", "notify", @@ -4023,7 +4023,7 @@ dependencies = [ [[package]] name = "tw-yaml" -version = "0.77.0" +version = "0.78.0" dependencies = [ "saphyr-parser", "serde_yaml_ng", @@ -4033,7 +4033,7 @@ dependencies = [ [[package]] name = "twcore" -version = "0.77.0" +version = "0.78.0" dependencies = [ "anyhow", "axum", diff --git a/Cargo.toml b/Cargo.toml index 0af5a769..05cd9195 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,7 +59,7 @@ categories = ["network-programming", "web-programming::http-server"] # 二进制走 CalVer，crate 走 SemVer —— 两者是两回事，卖给不同的人。 # 这里是 crate 的版本。 -version = "0.77.0" +version = "0.78.0" [workspace.dependencies] # ── 内部 crate ─────────────────────────────────────────── diff --git a/release-notes/0.78.0.md b/release-notes/0.78.0.md new file mode 100644 index 00000000..3fb456ff --- /dev/null +++ b/release-notes/0.78.0.md @@ -0,0 +1,30 @@ +This release adds configuration packs, import and export, local backups, and sync between devices. + +**Upgrade notes** + +- The control-plane protocol version is now 50. Use the ThinkWatch Lite release that ships with 0.78.0; earlier Lite releases cannot connect. +- The request store's schema is unchanged; nothing is cleared. +- Local backups replace the change history. The old history files (`history/*.yaml`) are removed when the first backup is kept. `GET /config/history` and `POST /config/rollback` are gone; use `/backups`. +- Upstream addresses that contain a backslash, or a user name or password before the host, are now rejected. A configuration that has one starts in safe mode; move the credential into the key or a header. +- `control.config_not_repairable` is replaced by `control.config_unrepairable`. +- For crates that embed `tw_bedrock`: `sign_service` is new and signs requests for any AWS service. + +**Configuration packs.** A pack is one `.twconfig` file that holds the configuration, the plugin scripts and the app's general preferences. Exports, shared upstreams, backups and sync versions all use it. A pack can be encrypted with a passphrase. + +**Export and import.** +- Export the whole configuration or chosen items; items they refer to come along. Secrets are included by default and can be left out. +- Import adds what is new. An item with the same name and different content is a conflict; each one is overwritten, imported under a new name, or skipped. Global settings are taken as they are when the device never changed them. +- A pack without secrets keeps this device's secret for an item only when the destination is the same: scheme, host, port, AWS region, OAuth token endpoint and outbound proxy. Otherwise the item arrives disabled and marked as missing its key. +- Imported plugins arrive disabled. ChatGPT logins and rotating OAuth refresh tokens never leave the device. +- Packs from older releases are repaired the way safe mode repairs a configuration, and the preview lists each fix. Packs from newer releases ask for an update. +- CLI: `twcore export`, `twcore import`. + +**Backups.** Every configuration change keeps a version with the plugins included, and the last 10 are kept. A restore can bring back the whole version or chosen items, and the preview and the write follow the same plan. The device's own settings (listening, control key, retention, sync) and current logins stay as they are, and a restore can itself be undone. Backups can be restored from safe mode. CLI: `twcore backup list | restore`. + +**Sync between devices.** +- A folder (iCloud Drive, Dropbox, OneDrive, Syncthing, a network share), a WebDAV server or S3-compatible storage keeps one configuration for several devices, remote cores included. +- Versions are only ever added. Changes merge item by item against the last version both sides share; only the same part of the same item changed in two ways is a conflict. A conflict travels with the versions and is resolved once, on any device, by taking either side or keeping both. +- Encryption with a sync passphrase is chosen when sync is turned on and is on by default. An encrypted location never accepts plain versions and never turns itself off. +- Upstreams can be kept off sync, together with the routes, aliases and groups that refer to them. Gateway keys used by clients this device has taken over are never removed by sync or a restore. +- The sync location keeps the latest and the previous version as an off-site backup. +- CLI: `twcore sync status | now | enable | disable | republish`.