Packs: repair older packs, plugin scope refs, OAuth tokens stay local

bin/twcore/src/pack_cli.rs

@@ -169,6 +169,22 @@ fn describe(id: &ItemId) -> String {
     }
 }
 
+/// 较旧版本写的、读不进来的字段在内存里修过的那几处
+fn print_repairs(fixes: &[tw_config::repair::Fix]) {
+    if fixes.is_empty() {
+        return;
+    }
+    println!("written by an older version; repaired before reading:");
+    for f in fixes {
+        let what = match f.kind {
+            tw_config::repair::FixKind::UnknownValue => "back to the default",
+            tw_config::repair::FixKind::UnknownField => "removed",
+        };
+        let line = f.line.map(|l| format!(" (line {l})")).unwrap_or_default();
+        println!("  {}{line}: {what}", f.field);
+    }
+}
+
 // ─────────────────────────────────────────────────────────── export
 
 pub fn export(
@@ -248,6 +264,8 @@ pub fn import(
 ) -> Result<()> {
     use tw_pack::import::{Action, Decision, Status};
     let incoming = read_pack(file)?;
+    let (incoming, repairs) = tw_pack::import::prepare(incoming).map_err(|e| anyhow!("{e}"))?;
+    print_repairs(&repairs);
     let (cur, local) = current(path, PackOrigin::Import)?;
     let ctx = tw_pack::import::Context {
         has_aws_profile: &|p| tw_config::aws_profile::credentials(p).is_ok(),
@@ -357,6 +375,8 @@ pub fn restore(path: &Path, id: &str, dry_run: bool) -> Result<()> {
     use tw_pack::restore::{Change, Kept};
     let b = tw_pack::backup::find(path, id).map_err(|e| anyhow!("{e}"))?;
     let backup = tw_pack::backup::open(&b).map_err(|e| anyhow!("{e}"))?;
+    let (backup, repairs) = tw_pack::import::prepare(backup).map_err(|e| anyhow!("{e}"))?;
+    print_repairs(&repairs);
     let (cur, local) = current(path, PackOrigin::Restore)?;
     let pinned = tw_pack::disk::read_pinned(path);
     let after = if tw_pack::model::snapshot(&local).is_err() {

crates/tw-api/msg-codes.txt

@@ -141,8 +141,8 @@ control.chatgpt_login.expired
 control.chatgpt_login.no_code
 control.chatgpt_login.page_error
 control.client_unknown
-control.config_not_repairable
 control.config_stale
+control.config_unrepairable
 control.control_key_locked
 control.default_key_cannot_delete
 control.default_key_cannot_disable

crates/tw-api/src/lib.rs

@@ -972,8 +972,11 @@ pub const MSG_CODES: &str = include_str!("../msg-codes.txt");
 /// [`Event::GatewayKeysChanged`]。[`ConfigOrigin`] 删了 `rollback`，多了 `sync`、`import`、
 /// `restore`、`repair`（一键修复以前记成 `ui`）。备份相关的端点在安全模式下照样可用。配置包的
 /// 码是 `pack.*`，导入的是 `import.*`，备份与恢复的是 `backup.*`。停用着的上游可以缺凭据
-/// （密钥、OAuth、AWS 访问密钥空着，ChatGPT 账号没登录）：导入时就是这样进来的。照 49 写的
-/// 界面调不到删掉的那两个端点。
+/// （密钥、OAuth、AWS 访问密钥空着，ChatGPT 账号没登录）：导入时就是这样进来的。较旧版本
+/// 导出的包、升级之前留的版本里读不进来的字段按一键修复的规矩在内存里修好，[`ImportPreview`]
+/// 和 [`RestorePreview`] 的 `repairs` 列出修了哪几处。OAuth 上游换发来的令牌（刷新令牌会轮换）
+/// 和 ChatGPT 的登录一样留在本机，不随导出带走、导入覆盖时留着本机的。一键修复修不了时的码换成
+/// `control.config_unrepairable`（句子改说从备份恢复）。照 49 写的界面调不到删掉的那两个端点。
 pub const CONTROL_API_VERSION: u32 = 50;
 
 /// `/live` 的速度看最近多久（见 [`LiveView::tokens_per_sec`]），毫秒

crates/tw-api/src/pack.rs

@@ -180,6 +180,9 @@ pub struct ImportPreview {
     #[serde(default, skip_serializing_if = "Option::is_none")]
     #[cfg_attr(feature = "ts", ts(type = "unknown"))]
     pub client_prefs: Option<serde_json::Value>,
+    /// 较旧版本导出的包里读不进来的字段，按一键修复的规矩（不认识的字段删掉、取值不合法的
+    /// 回到默认值）**在内存里修过了**：预览和导入都按修好的算，包本身不变。没修过是空的
+    pub repairs: Vec<crate::ConfigFix>,
 }
 
 slug_enum! {
@@ -320,6 +323,9 @@ pub struct RestorePreview {
     pub rows: Vec<RestoreRow>,
     /// 开着同步时，恢复的结果会同步到几台别的设备。没开同步是 0
     pub sync_devices: u32,
+    /// 那一版里这一版 Core 读不进来的字段（升级之前留的版本），按一键修复的规矩在内存里修过了。
+    /// 没修过是空的
+    pub repairs: Vec<crate::ConfigFix>,
 }
 
 slug_enum! {

crates/tw-control/src/backups.rs

@@ -191,8 +191,12 @@ fn bad_pack(e: PackError) -> Fail {
 
 /// 配置（本机的、包里的）拆不开。**由更新的版本写的**说成「太新」
 fn model_fail(e: model::ModelError, pack: Option<&Pack>) -> Fail {
+    let newer = |p: &Pack| {
+        matches!(e, model::ModelError::TooNew { .. })
+            || (e.is_too_new() && p.manifest.from_newer_core())
+    };
     if let Some(p) = pack
-        && e.is_too_new()
+        && newer(p)
     {
         return bad_pack(PackError::TooNew {
             format: p.manifest.format,
@@ -273,6 +277,20 @@ fn file_of(pack: &Pack, pass: Option<&str>) -> Result<(String, String), Fail> {
     ))
 }
 
+fn fix_view(f: tw_config::repair::Fix) -> tw_api::ConfigFix {
+    use tw_config::repair::FixKind;
+    tw_api::ConfigFix {
+        kind: match f.kind {
+            FixKind::UnknownValue => tw_api::ConfigFixKind::UnknownValue,
+            FixKind::UnknownField => tw_api::ConfigFixKind::UnknownField,
+        },
+        field: f.field,
+        line: f.line,
+        value: f.value,
+        now: f.now,
+    }
+}
+
 // ─────────────────────────────────────────────────────────── 此刻的样子
 
 /// 磁盘上此刻的配置（原文和版本）连同它的插件、通用偏好
@@ -430,6 +448,7 @@ fn import_fail(e: tw_pack::import::ImportError, pack: &Pack) -> Fail {
         E::Exists(_) | E::NameTaken(..) => fail(StatusCode::CONFLICT, e.msg()),
         E::NotInPack(_) => fail(StatusCode::NOT_FOUND, e.msg()),
         E::Item(..) => fail(StatusCode::UNPROCESSABLE_ENTITY, e.msg()),
+        E::TooNew(p) => bad_pack(p),
     }
 }
 
@@ -464,7 +483,9 @@ async fn import_preview(
     State(s): State<ControlState>,
     Json(req): Json<tw_api::ImportSource>,
 ) -> Result<Json<tw_api::ImportPreview>, Fail> {
-    let incoming = open_pack(&req.data_b64, req.passphrase.as_deref())?;
+    let raw = open_pack(&req.data_b64, req.passphrase.as_deref())?;
+    let (incoming, repairs) =
+        tw_pack::import::prepare(raw.clone()).map_err(|e| import_fail(e, &raw))?;
     let (cur, local) = current(&s, PackOrigin::Import)?;
     let rows = tw_pack::import::preview(&local, &incoming, &import_ctx())
         .map_err(|e| import_fail(e, &incoming))?;
@@ -473,14 +494,17 @@ async fn import_preview(
         base_version: cur.version(),
         rows: rows.into_iter().map(row_view).collect(),
         client_prefs: incoming.client_prefs.clone(),
+        repairs: repairs.into_iter().map(fix_view).collect(),
     }))
 }
 
 async fn import(
     State(s): State<ControlState>,
     Json(req): Json<tw_api::ImportRequest>,
 ) -> Result<Json<tw_api::ConfigWritten>, Fail> {
-    let incoming = open_pack(&req.data_b64, req.passphrase.as_deref())?;
+    let raw = open_pack(&req.data_b64, req.passphrase.as_deref())?;
+    // 预览时修过的那几处，这里照样修一遍（同一份字节，修出来一样）
+    let (incoming, _) = tw_pack::import::prepare(raw.clone()).map_err(|e| import_fail(e, &raw))?;
     let (cur, local) = current(&s, PackOrigin::Import)?;
     if cur.version() != req.base_version {
         return Err(apply_fail(ApplyError::Stale {
@@ -663,7 +687,10 @@ async fn restore_preview(
     State(s): State<ControlState>,
     Json(req): Json<tw_api::RestorePreviewRequest>,
 ) -> Result<Json<tw_api::RestorePreview>, Fail> {
-    let backup = backup_pack(&s, &req.id).await?;
+    let raw = backup_pack(&s, &req.id).await?;
+    // 升级之前留的版本可能有这一版不认识的写法：和导入一样在内存里修好
+    let (backup, repairs) =
+        tw_pack::import::prepare(raw.clone()).map_err(|e| import_fail(e, &raw))?;
     let (cur, local) = current(&s, PackOrigin::Restore)?;
     let sync_devices = match s.cfg.remote_backups() {
         Some(r) => r.other_devices().await,
@@ -693,14 +720,16 @@ async fn restore_preview(
         base_version: cur.version(),
         rows,
         sync_devices,
+        repairs: repairs.into_iter().map(fix_view).collect(),
     }))
 }
 
 async fn restore(
     State(s): State<ControlState>,
     Json(req): Json<tw_api::RestoreRequest>,
 ) -> Result<Json<tw_api::ConfigWritten>, Fail> {
-    let backup = backup_pack(&s, &req.id).await?;
+    let raw = backup_pack(&s, &req.id).await?;
+    let (backup, _) = tw_pack::import::prepare(raw.clone()).map_err(|e| import_fail(e, &raw))?;
     let (cur, local) = current(&s, PackOrigin::Restore)?;
     if cur.version() != req.base_version {
         return Err(apply_fail(ApplyError::Stale {

crates/tw-control/src/lib.rs

@@ -1549,9 +1549,9 @@ async fn repair_config(
         return Err(fail(
             StatusCode::BAD_REQUEST,
             msg!(
-                "control.config_not_repairable" =>
+                "control.config_unrepairable" =>
                 "The configuration has no error that can be repaired automatically. Edit the \
-                 configuration file, or roll back to an earlier version"
+                 configuration file, or restore an earlier version from the backups"
             ),
         ));
     };

crates/tw-control/tests/backups.rs

@@ -564,3 +564,64 @@ async fn in_safe_mode_a_backup_comes_back_whole() {
     let text = std::fs::read_to_string(&p).unwrap();
     assert_eq!(text, mine());
 }
+
+/// 较旧版本导出的包：读不进来的字段在内存里修好，预览说修了哪几处；更新的版本写的说太新
+#[tokio::test]
+async fn an_older_pack_is_previewed_repaired_and_a_newer_one_is_too_new() {
+    let dst = bed(&mine());
+    let pack_b64 = |core: &str| {
+        let mut m = tw_pack::Manifest::new(tw_pack::PackKind::Full, tw_pack::PackOrigin::Export);
+        m.core_version = core.to_string();
+        let p = tw_pack::Pack {
+            manifest: m,
+            config_yaml: "version: 1\nproviders:\n  - name: old\n    base_url: https://old.example\n    retired_field: 1\nclient_probes:\n  titling: passthrough\n".into(),
+            plugins: Default::default(),
+            client_prefs: None,
+        };
+        base64::engine::general_purpose::STANDARD
+            .encode(tw_pack::write(&p, tw_pack::Seal::None).unwrap())
+    };
+    let old = pack_b64("0.1.0");
+    let (st, preview) = call(
+        &dst.app,
+        "POST",
+        "/config/import/preview",
+        serde_json::json!({ "data_b64": old }),
+    )
+    .await;
+    assert_eq!(st, StatusCode::OK, "{preview}");
+    let fields: Vec<&str> = preview["repairs"]
+        .as_array()
+        .unwrap()
+        .iter()
+        .map(|f| f["field"].as_str().unwrap())
+        .collect();
+    assert_eq!(
+        fields,
+        ["providers[0].retired_field", "client_probes.titling"]
+    );
+    assert_eq!(preview["repairs"][1]["kind"], "unknown_value");
+    let (st, v) = call(
+        &dst.app,
+        "POST",
+        "/config/import",
+        serde_json::json!({
+            "data_b64": old,
+            "base_version": preview["base_version"],
+            "decisions": [{ "item": { "kind": "provider", "name": "old" }, "action": "import" }],
+        }),
+    )
+    .await;
+    assert_eq!(st, StatusCode::OK, "{v}");
+    assert!(dst.file().contains("https://old.example"));
+    assert!(!dst.file().contains("retired_field"));
+    let (st, v) = call(
+        &dst.app,
+        "POST",
+        "/config/import/preview",
+        serde_json::json!({ "data_b64": pack_b64("999.0.0") }),
+    )
+    .await;
+    assert_eq!(st, StatusCode::BAD_REQUEST, "{v}");
+    assert_eq!(v["code"], "pack.too_new");
+}

crates/tw-control/tests/safe_mode.rs

@@ -165,7 +165,7 @@ async fn an_error_that_is_not_fixable_offers_nothing() {
     )
     .await;
     assert_eq!(st, StatusCode::BAD_REQUEST, "{e}");
-    assert_eq!(e["code"], "control.config_not_repairable", "{e}");
+    assert_eq!(e["code"], "control.config_unrepairable", "{e}");
 }
 
 #[tokio::test]

crates/tw-pack/Cargo.toml

@@ -18,6 +18,8 @@ tw-engine = { workspace = true }
 tw-pricing = { workspace = true }
 tw-types = { workspace = true }
 tw-yaml = { workspace = true }
+# 修导入的旧配置包时，原值给人看之前脱敏
+tw-secret = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 serde_yaml_ng = { workspace = true }

crates/tw-pack/src/export.rs

@@ -29,7 +29,7 @@ pub struct Exported {
     pub pack: Pack,
     /// 包里有哪些项目（随附的也在），顺序项目不算
     pub included: Vec<ItemId>,
-    /// 登录没有随包带走的上游（ChatGPT 账号）
+    /// 登录没有随包带走的上游：ChatGPT 账号，和令牌留在本机的 OAuth 上游
     pub omitted_logins: Vec<String>,
 }
 
@@ -81,7 +81,7 @@ pub fn export_with(
     let omitted_logins = cfg
         .providers
         .iter()
-        .filter(|p| picked.contains(p.name.as_str()) && model::is_chatgpt(p))
+        .filter(|p| picked.contains(p.name.as_str()) && model::holds_login(p))
         .map(|p| p.name.clone())
         .collect();
     let kind = if opts.scope.is_some() {