Sync and backup: config packs, import and export, local backups, multi-device sync

Cargo.toml

@@ -28,6 +28,11 @@ members = [
     # crate、企业版用的第一层都不能沾上，tw-plugin 的 tests/boundary.rs 守着
     "crates/tw-plugin",
     "crates/tw-control",
+    # 配置包（.twconfig）：导出、导入、本机备份与同步共用的格式，以及把一份配置拆成
+    # 项目与合并单位的模型
+    "crates/tw-pack",
+    # 多设备同步：同步位置的存储（文件夹、WebDAV、S3）、keyring、三方合并
+    "crates/tw-sync",
     # 控制通道的握手与加密。桌面端按 tag 依赖它，所以只依赖契约层和 tw-yaml
     "crates/tw-link",
     # ── bin ──────────────────────────────────────────────
@@ -54,7 +59,7 @@ categories = ["network-programming", "web-programming::http-server"]
 
 # 二进制走 CalVer，crate 走 SemVer —— 两者是两回事，卖给不同的人。
 # 这里是 crate 的版本。
-version = "0.77.0"
+version = "0.78.0"
 
 [workspace.dependencies]
 # ── 内部 crate ───────────────────────────────────────────
@@ -76,6 +81,8 @@ tw-dialect  = { path = "crates/tw-dialect" }
 tw-bedrock  = { path = "crates/tw-bedrock" }
 tw-link     = { path = "crates/tw-link" }
 tw-plugin   = { path = "crates/tw-plugin" }
+tw-pack     = { path = "crates/tw-pack" }
+tw-sync     = { path = "crates/tw-sync" }
 
 # ── 版本与企业版 workspace 对齐，便于反向依赖时不打架 ────
 tokio = { version = "1", features = ["rt-multi-thread", "macros", "net", "time", "sync", "io-util", "signal"] }
@@ -142,6 +149,10 @@ aws-credential-types = "1"
 aws-smithy-eventstream = "0.60"
 aws-smithy-types = "1"
 percent-encoding = "2"
+# 地址解析（同步位置的 `http://` 只给内网）。reqwest 本来就依赖它
+url = "2"
+# WebDAV 的 PROPFIND、S3 的 ListObjectsV2 回的是 XML
+quick-xml = "0.42"
 
 thiserror = "2"
 anyhow = "1"
@@ -161,6 +172,11 @@ rand = "0.10"
 zstd = { version = "0.13", default-features = false }
 
 clap = { version = "4", features = ["derive"] }
+# 配置包的口令加密（scrypt）与同步版本的密钥加密（X25519）。默认特性是空的：不要
+# SSH 密钥、插件、命令行那几样
+age = { version = "0.11", default-features = false }
+# 配置包本身是 ZIP。只要 deflate（走已经在依赖树里的 flate2），不要别的压缩算法和加密
+zip = { version = "9", default-features = false, features = ["deflate-flate2"] }
 # 控制面契约导出成 TypeScript（tw-api 的 `ts` feature）。**不开任何 serde 以外的
 # 类型实现**：契约里只有 std 和 serde 的类型
 ts-rs = { version = "12", default-features = false, features = ["serde-compat"] }

README.md

@@ -156,6 +156,7 @@ a time, and cannot stop core, take the diagnostic bundle or change
 | `tw-observe` | Event bus |
 | `tw-gateway` | Data plane: the life of a request |
 | `tw-plugin` | Script-plugin sandbox: QuickJS compiled to WebAssembly, run by Wasmtime |
+| `tw-pack` | Configuration packs (`.twconfig`): format, encryption, local backups, and the item model import and sync compare on |
 | `tw-control` | Control-plane server |
 
 ThinkWatch Enterprise depends only on the first four, which depend only on one

README.zh-CN.md

@@ -93,6 +93,7 @@ twcore control-key --rotate     # 更换密钥；用旧密钥建立的连接随
 | `tw-observe` | 事件总线 |
 | `tw-gateway` | 数据面：一个请求的完整生命周期 |
 | `tw-plugin` | 脚本插件的沙箱：编成 WebAssembly 的 QuickJS，由 Wasmtime 运行 |
+| `tw-pack` | 配置包（`.twconfig`）：格式、加密、本机备份，以及导入与同步共同比较的项目模型 |
 | `tw-control` | 控制面服务 |
 
 ThinkWatch 企业版只依赖前四个 crate，这四个 crate 也只相互依赖；CI 会针对它们的每一次改动检查企业版能否编译。ThinkWatch Lite 把 `tw-api`、`tw-types`、`tw-yaml`、`tw-guard`、`tw-watch` 和 `tw-link` 固定在某个 Release 的 tag 上，并打包同一 Release 的 `twcore`。接管 AI 客户端和扫描其配置在 Lite 中完成，作用于应用所在的机器；`twcore` 只为每个客户端签发专用的网关密钥。二进制的源码位于 `bin/twcore`。只有 `tw-gateway` 和 `twcore` 可以依赖 `tw-plugin`——它是唯一一个构建时除了 Rust 还需要别的工具的 crate（见下文），因此用这些 crate 构建 Lite 或企业版时都不需要。

bin/twcore/Cargo.toml

@@ -16,13 +16,17 @@ categories.workspace = true
 [dependencies]
 tw-config = { workspace = true }
 tw-control = { workspace = true }
+# `twcore export` / `import` / `backup`：配置包和本机备份
+tw-pack = { workspace = true }
 tw-gateway = { workspace = true }
 tw-yaml = { workspace = true }
 tw-api = { workspace = true }
 tw-observe = { workspace = true }
 tw-store = { workspace = true }
 tw-pricing = { workspace = true }
 chrono = { workspace = true }
+# `twcore sync` 和控制面说 JSON
+serde_json = { workspace = true }
 tw-secret = { workspace = true }
 anyhow = { workspace = true }
 clap = { workspace = true }

bin/twcore/src/call.rs

@@ -18,6 +18,27 @@ pub fn run(
     data: Option<String>,
     out: Option<PathBuf>,
 ) -> Result<()> {
+    let (status, body) = request(config, method, endpoint, data)?;
+    match out {
+        Some(file) => {
+            std::fs::write(&file, &body).with_context(|| format!("writing {}", file.display()))?;
+            println!("{status}");
+        }
+        None => {
+            use std::io::Write;
+            std::io::stdout().write_all(&body)?;
+        }
+    }
+    Ok(())
+}
+
+/// 发一个请求，拿回状态码和响应体。`twcore sync` 这类命令也走它
+pub fn request(
+    config: &Path,
+    method: &str,
+    endpoint: &str,
+    data: Option<String>,
+) -> Result<(u16, Bytes)> {
     let dir = config
         .parent()
         .map(PathBuf::from)
@@ -36,7 +57,7 @@ pub fn run(
     let rt = tokio::runtime::Builder::new_current_thread()
         .enable_all()
         .build()?;
-    let (status, body) = rt.block_on(async move {
+    rt.block_on(async move {
         match Address::in_dir(&dir) {
             #[cfg(unix)]
             Address::Socket(p) => {
@@ -59,18 +80,7 @@ pub fn run(
                 send(s, &key, method, &endpoint, data).await
             }
         }
-    })?;
-    match out {
-        Some(file) => {
-            std::fs::write(&file, &body).with_context(|| format!("writing {}", file.display()))?;
-            println!("{status}");
-        }
-        None => {
-            use std::io::Write;
-            std::io::stdout().write_all(&body)?;
-        }
-    }
-    Ok(())
+    })
 }
 
 async fn send<S>(

bin/twcore/src/main.rs

@@ -11,7 +11,9 @@ use clap::{Parser, Subcommand};
 
 mod call;
 mod lockfile;
+mod pack_cli;
 mod proc;
+mod sync_cmd;
 mod upgrade;
 
 use lockfile::{LockFile, LockOutcome};
@@ -103,6 +105,47 @@ enum Command {
         #[arg(short, long)]
         out: Option<PathBuf>,
     },
+    /// Export the configuration as a configuration pack (.twconfig)
+    Export {
+        /// Export only these upstreams (comma-separated), with what they refer to
+        #[arg(long, value_delimiter = ',')]
+        providers: Vec<String>,
+        /// Leave the keys out: upstream keys, header values, AWS keys, proxy passwords and
+        /// gateway keys are left empty
+        #[arg(long)]
+        no_secrets: bool,
+        /// Encrypt the file with a passphrase, asked for or taken from TWCORE_PASSPHRASE
+        #[arg(long)]
+        encrypt: bool,
+        /// The file to write
+        #[arg(short, long)]
+        out: PathBuf,
+    },
+    /// Import a configuration pack: what is not here is added, and what differs is chosen item
+    /// by item
+    Import {
+        /// The configuration pack (.twconfig)
+        file: PathBuf,
+        /// Show what would happen and change nothing
+        #[arg(long)]
+        dry_run: bool,
+        /// What to do with every item that differs from the one here; asked item by item
+        /// without it
+        #[arg(long, value_enum)]
+        on_conflict: Option<pack_cli::OnConflict>,
+    },
+    /// The last versions of the configuration, and restoring one
+    Backup {
+        #[command(subcommand)]
+        what: BackupCmd,
+    },
+    /// Show or change multi-device sync. Talks to the running core
+    //
+    // 给没有界面的服务器（远程 core）用：和界面走同一套控制面端点
+    Sync {
+        #[command(subcommand)]
+        what: SyncCmd,
+    },
     /// Replace this twcore with another release from GitHub; the configuration is left alone
     //
     // 只给单独装的 twcore 用（服务器上）。桌面应用包里的那份由应用更新，见 upgrade.rs
@@ -119,6 +162,25 @@ enum Command {
     },
 }
 
+#[derive(Subcommand)]
+enum SyncCmd {
+    /// Print the sync state, the location and the devices
+    Status,
+    /// Sync once now
+    Now,
+    /// Turn sync on; asks for the location and the passphrase
+    Enable,
+    /// Turn sync off; this device keeps its configuration
+    Disable {
+        /// Also delete every version, device record and the keyring at the sync location
+        #[arg(long)]
+        purge: bool,
+    },
+    /// After the sync location was emptied or lost its keyring: upload this device's
+    /// configuration again (and put back the keyring this device keeps)
+    Republish,
+}
+
 #[derive(Subcommand)]
 enum RemoteCmd {
     /// Print whether the remote control port is open and where to connect
@@ -140,6 +202,21 @@ enum RemoteCmd {
     Disable,
 }
 
+#[derive(Subcommand)]
+enum BackupCmd {
+    /// List the versions, newest first
+    List,
+    /// Go back to a version. Settings that belong to this machine, ChatGPT sign-ins and the
+    /// gateway keys clients set up here use are kept
+    Restore {
+        /// The id from `twcore backup list`
+        id: String,
+        /// Show what would change and change nothing
+        #[arg(long)]
+        dry_run: bool,
+    },
+}
+
 #[derive(Subcommand)]
 enum ConfigCmd {
     /// Print the configuration as it is, with its version
@@ -159,10 +236,6 @@ enum ConfigCmd {
         #[arg(long, conflicts_with_all = ["int", "bool_value"])]
         null: bool,
     },
-    /// List the versions in the history
-    History,
-    /// Roll back to a version
-    Rollback { version: String },
 }
 
 fn main() -> Result<()> {
@@ -182,6 +255,8 @@ fn main() -> Result<()> {
 
     let cli = Cli::parse();
     let path = cli.config.clone().unwrap_or_else(tw_config::default_path);
+    // 配置包、备份按「代码」「数据」分开比较插件：拆法要插件那一套
+    tw_control::backups::install_codec();
 
     match cli.command {
         Command::Init { force } => cmd_init(&path, force),
@@ -191,12 +266,34 @@ fn main() -> Result<()> {
         Command::Config { what } => cmd_config(&path, what),
         Command::ControlKey { rotate } => cmd_control_key(&path, rotate),
         Command::Remote { what } => cmd_remote(&path, what.unwrap_or(RemoteCmd::Show)),
+        Command::Export {
+            providers,
+            no_secrets,
+            encrypt,
+            out,
+        } => pack_cli::export(&path, providers, no_secrets, encrypt, out),
+        Command::Import {
+            file,
+            dry_run,
+            on_conflict,
+        } => pack_cli::import(&path, &file, dry_run, on_conflict),
+        Command::Backup { what } => match what {
+            BackupCmd::List => pack_cli::list(&path),
+            BackupCmd::Restore { id, dry_run } => pack_cli::restore(&path, &id, dry_run),
+        },
         Command::Call {
             path: endpoint,
             method,
             data,
             out,
         } => call::run(&path, &endpoint, &method, data, out),
+        Command::Sync { what } => match what {
+            SyncCmd::Status => sync_cmd::status(&path),
+            SyncCmd::Now => sync_cmd::now(&path),
+            SyncCmd::Enable => sync_cmd::enable(&path),
+            SyncCmd::Disable { purge } => sync_cmd::disable(&path, purge),
+            SyncCmd::Republish => sync_cmd::republish(&path),
+        },
         Command::Upgrade {
             check,
             restart,
@@ -212,7 +309,7 @@ fn main() -> Result<()> {
 /// 改配置。
 ///
 /// **不连控制面，直接操作文件。**理由是这个命令必须在 core 没跑的时候
-/// 也能用 —— 「配置写坏了导致 core 起不来」正是最需要 `config rollback`
+/// 也能用 —— 「配置写坏了导致 core 起不来」正是最需要 `backup restore`
 /// 的时刻，而那时控制面根本不存在。
 ///
 /// 代价是 core 正在跑时，改动要等它的文件监听发现（几百毫秒）。那条路
@@ -225,40 +322,6 @@ fn cmd_config(path: &Path, what: ConfigCmd) -> Result<()> {
             eprintln!("── {} · {}", c.path.display(), c.version());
             Ok(())
         }
-        ConfigCmd::History => {
-            let all = tw_config::history::list(path)?;
-            if all.is_empty() {
-                println!("No versions yet; the first change to the configuration makes one.");
-                return Ok(());
-            }
-            let now = tw_config::store::read(path).map(|c| c.version()).ok();
-            // 新的在前 —— 要找的几乎总是最近那几版
-            for v in all.iter().rev() {
-                let mark = if Some(&v.version) == now.as_ref() {
-                    "← current"
-                } else {
-                    "      "
-                };
-                println!(
-                    "{mark}  {}  {:<10}  {} bytes  {}",
-                    v.version,
-                    v.origin.label(),
-                    v.bytes,
-                    fmt_time(v.at_ms)
-                );
-            }
-            println!();
-            println!(
-                "To roll back: twcore config rollback <version> (the first few characters are enough)"
-            );
-            Ok(())
-        }
-        ConfigCmd::Rollback { version } => {
-            let text = tw_config::history::rollback(path, &version)?;
-            println!("rolled back to {}", tw_config::store::version_of(&text));
-            eprintln!("(a running core picks this up within a second)");
-            Ok(())
-        }
         ConfigCmd::Set {
             path: pointer,
             value,
@@ -293,10 +356,11 @@ fn cmd_config(path: &Path, what: ConfigCmd) -> Result<()> {
             let next = tw_yaml::set(&cur.text, &steps, &scalar)?;
             // **先校验再写。**写完才发现读不回来，那份坏配置已经在盘上了。
             tw_config::try_parse(&next).map_err(|r| anyhow::anyhow!("{r}"))?;
-            // 改之前那一版进历史，这样这条命令也能被 rollback 撤销
-            let _ = tw_config::history::snapshot(path, &cur.text, tw_config::history::Origin::Cli);
+            // 前后各留一个备份版本，这样这条命令也能被 `twcore backup restore` 撤销
+            let _ =
+                tw_pack::backup::record_before(path, &cur.text, tw_config::history::Origin::Cli);
             tw_config::store::write_if_unchanged(path, &cur.fingerprint, &next)?;
-            let _ = tw_config::history::snapshot(path, &next, tw_config::history::Origin::Cli);
+            let _ = tw_pack::backup::record(path, &next, tw_config::history::Origin::Cli);
             println!(
                 "changed; the new version is {}",
                 tw_config::store::version_of(&next)
@@ -884,6 +948,9 @@ fn cmd_serve(path: &Path, port: Option<u16>, safe: bool, parent: Option<u32>) ->
             Some(r) => manager.standing_in(r),
             None => manager,
         });
+        // 磁盘上此刻的样子和最新的备份版本不一样就留一版：第一次运行、升级之后、core 没在跑时
+        // 命令行改过。配置读不进来时也留：一键修复之前的那一份在备份里
+        manager.record_startup();
         // 默认插件（随 core 发的那几个）：没给过的装上（停用着），没动过的换成新版。
         // 启动时在控制面起来之前走一遍，界面第一次取插件就看得到它们；之后每换入一份
         // 配置再走一遍。**不挡启动**：哪个没办成只记一行、说一声。安全模式不走 ——
@@ -931,6 +998,8 @@ fn cmd_serve(path: &Path, port: Option<u16>, safe: bool, parent: Option<u32>) ->
         // 这是一种（另一种是上面的默认插件）—— 理由是服务器换发新 refresh
         // token 的那一刻旧的就作废了，不写回等于让配置文件从那一秒起就是坏的。
         tw_control::rotation::spawn(control.clone());
+        // 多设备同步：启动时一轮，之后按本机改动、定时、文件夹变化、唤醒。安全模式下不自动同步
+        let sync = tw_control::sync::start(&control, safe);
         // 定期刷新默认价目表（`pricing.auto_update`，默认开）
         tw_control::pricing::spawn(control.clone());
 
@@ -999,6 +1068,8 @@ fn cmd_serve(path: &Path, port: Option<u16>, safe: bool, parent: Option<u32>) ->
             }
             _ = shutdown_requested(&shutdown) => {
                 tracing::info!("got a shutdown signal");
+                // 还没上传的本机改动，退出前上传（等不了太久：没传成的下次启动时传）
+                sync.flush().await;
                 Ok(())
             }
         }