From 308080b26dc3f4b6355373572f47fb3debd8860c Mon Sep 17 00:00:00 2001 From: Yao Jingxi <23722032@bjtu.edu.cn> Date: Sat, 24 Oct 2026 21:10:17 +0800 Subject: [PATCH 1/8] =?UTF-8?q?feat(macos):=20Agent=20=E5=AF=B9=E8=AF=9D?= =?UTF-8?q?=E6=98=BE=E7=A4=BA=E5=B7=A5=E5=85=B7=E5=9B=BE=E7=89=87=E7=BC=A9?= =?UTF-8?q?=E7=95=A5=E5=9B=BE=E3=80=81=E7=82=B9=E5=87=BB=E6=94=BE=E5=A4=A7?= =?UTF-8?q?=EF=BC=8C=E5=B9=B6=E5=B0=86=E5=90=8E=E5=8F=B0=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E9=80=9A=E7=9F=A5=E6=98=BE=E7=A4=BA=E4=B8=BA=E7=8A=B6=E6=80=81?= =?UTF-8?q?=E8=A1=8C=20(#1462)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...ranscript-images-and-task-notifications.md | 64 +++++ macos/Resources/en.lproj/Localizable.strings | 12 + .../zh-Hans.lproj/Localizable.strings | 12 + .../Lithe/Views/Agent/AgentImageViews.swift | 269 ++++++++++++++++++ .../Views/Agent/AgentToolGroupView.swift | 12 +- .../Views/Agent/AgentTranscriptView.swift | 104 +++++-- .../Application/AgentConnectionModel.swift | 20 +- .../Application/AgentConversationState.swift | 2 + .../Application/AgentImage.swift | 129 +++++++++ .../AgentSessionConfiguration.swift | 26 +- macos/Tests/LitheTests/AgentImageTests.swift | 100 +++++++ shared/i18n/locales/en-US.json | 6 + shared/i18n/locales/zh-CN.json | 6 + shared/i18n/messages.json | 73 +++++ .../agent-tool-activity-timeline.json | 11 +- 15 files changed, 811 insertions(+), 35 deletions(-) create mode 100644 .agents/notes/implemented/feature/2026-10-24-agent-transcript-images-and-task-notifications.md create mode 100644 macos/Sources/Lithe/Views/Agent/AgentImageViews.swift create mode 100644 macos/Sources/LitheAgentConversationModule/Application/AgentImage.swift create mode 100644 macos/Tests/LitheTests/AgentImageTests.swift diff --git a/.agents/notes/implemented/feature/2026-10-24-agent-transcript-images-and-task-notifications.md b/.agents/notes/implemented/feature/2026-10-24-agent-transcript-images-and-task-notifications.md new file mode 100644 index 0000000000..0d8cbc182d --- /dev/null +++ b/.agents/notes/implemented/feature/2026-10-24-agent-transcript-images-and-task-notifications.md @@ -0,0 +1,64 @@ +# Agent 笔记：对话中的图片缩略图、放大查看与后台任务通知 + +状态：已实现 + +## 先说结论 + +工具读取或截取图片时，对应的工具行（以及整组工具的标题行）右侧显示缩略图： +最多三张，其余用 `+N` 表示，悬停出大图预览，点击进入统一的放大查看器。用户消息里附带的图片也用同一套缩略图和查看器。 +Claude Code 发出的 `` 不再原样显示为用户气泡，而是一行状态文字。 +图片数据只在内存中以受限大小保存，原图由视图层按需异步解码。 + +## 问题 + +ACP（Agent Client Protocol，Agent 与客户端之间的协议）的工具结果可以携带 `image` 内容块， +但原先解析时只保留了类型名，图片本身被丢弃，用户看不到 Agent 读到的截图。 +用户发送的图片附件只以 `📎 路径` 文字出现，无法预览。Claude Code 的后台任务完成通知是协议标记， +却被当成用户输入显示。 + +## 决策 + +- `AgentToolDetails.images` 与 `AgentConversationMessage.images` 保存 `AgentImage`。内联图片解码后 + 不超过 10 MB、每条消息最多 24 张，超限直接丢弃，不进入会话状态。 + `AgentImage` 按内容指纹或路径判等，转录行的相等比较不会去比较像素。 +- 只提供文件路径的 `read` 工具调用（例如读取 `.png`）直接显示该文件。其他类型的工具不会因为位置是图片就当作“查看”。 +- 一次更新如果没有图片，不会清掉之前已收到的图片，避免最终状态更新把缩略图抹掉。 +- 缩略图在视图层用 ImageIO 在后台线程按需生成并放入 `NSCache`；历史回放只保存数据，不解码。 + 悬停预览复用 `workbenchHoverCard`，放大查看器是转录区域上的覆盖层，状态放在 + `AgentImageViewerState`，行只持有引用，打开或关闭不会重绘全部行。 +- `` 只有在整条用户消息完全由通知块组成时才解析，用户自己输入的相同文字不会被吞掉。 + 这是协议标记的呈现，不按 Agent 名称在宿主里分支。 + +### 不要这样做 + +- 不要把图片 `Data` 放进 `Equatable` 的逐字节比较，也不要在主线程解码原图。 +- 不要为放大查看另起一套弹窗样式或使用 `.popover`；悬停预览与查看器共用本 Note 中的入口。 + +## 考虑过的备选方案 + +### 把图片写到缓存目录再引用路径 + +好处是内存占用更小。但应用层不能直接做文件系统写入，需要新增平台存储适配；在单张和总量都有上限的前提下， +先在内存中保留，需要时再评估。 + +### 保留“复制”按钮并在工具行内渲染 + +issue #1462 指出的多余复制按钮来自不属于最终回复的过程文字。#1464 已把这类文字收进默认折叠的过程区， +空白回复也不再生成行，因此本次没有再改动复制按钮的显示规则。 + +## 后果 + +- 收益：Agent 看过的图片可以直接在对话里确认；用户附件可点击放大。 +- 代价：长会话里的内联图片占用内存，最坏情况每条消息 24 × 10 MB。 +- 需要重新评估：出现明显内存压力，或需要 Windows 端对齐时。 + +## 验证 + +- `./scripts/test-macos.sh --filter AgentImageTests` +- `./scripts/verify-agent-notes.sh` + +## 适用范围 + +- `macos/Sources/LitheAgentConversationModule/Application/AgentImage.swift` +- `macos/Sources/Lithe/Views/Agent/AgentImageViews.swift` +- `macos/Sources/Lithe/Views/Agent/AgentToolGroupView.swift` diff --git a/macos/Resources/en.lproj/Localizable.strings b/macos/Resources/en.lproj/Localizable.strings index 9260f8d240..4b714aa017 100644 --- a/macos/Resources/en.lproj/Localizable.strings +++ b/macos/Resources/en.lproj/Localizable.strings @@ -373,6 +373,8 @@ "Background Services" = "Background Services"; "Background services" = "Background services"; "Background services started in Lithe are still running" = "Background services started in Lithe are still running"; +"Background task ended" = "Background task ended"; +"Background task finished" = "Background task finished"; "Balanced reasoning" = "Balanced reasoning"; "Base" = "Base"; "Base %@ → Target %@" = "Base %@ → Target %@"; @@ -1180,6 +1182,7 @@ "End on next launch" = "End on next launch"; "End without asking" = "End without asking"; "English" = "English"; +"Enlarge image" = "Enlarge image"; "Enter API key" = "Enter API key"; "Enter a ChangeList name (1–100 characters)." = "Enter a ChangeList name (1–100 characters)."; "Enter a branch name" = "Enter a branch name"; @@ -1627,6 +1630,7 @@ "Invalid shared instruction request." = "Invalid shared instruction request."; "Invert" = "Invert"; "It can read files, run commands and edit code after asking for permission." = "It can read files, run commands and edit code after asking for permission."; +"Item %lld of %lld" = "Item %lld of %lld"; "Its details and actions will appear here." = "Its details and actions will appear here."; "JDK Home" = "JDK Home"; "JDK home" = "JDK home"; @@ -2014,6 +2018,7 @@ "Next console expression" = "Next console expression"; "Next diff match" = "Next diff match"; "Next difference" = "Next difference"; +"Next image" = "Next image"; "Next match" = "Next match"; "Next match (Return)" = "Next match (Return)"; "Next run" = "Next run"; @@ -2399,6 +2404,7 @@ "Previous console expression" = "Previous console expression"; "Previous diff match" = "Previous diff match"; "Previous difference" = "Previous difference"; +"Previous image" = "Previous image"; "Previous match" = "Previous match"; "Previous match (Shift+Return)" = "Previous match (Shift+Return)"; "Previous session" = "Previous session"; @@ -4175,6 +4181,8 @@ "native.backgroundServices" = "Background Services"; "native.backgroundServices.variant2" = "Background services"; "native.backgroundServicesStartedInLitheAreStillRunning" = "Background services started in Lithe are still running"; +"native.backgroundTaskEnded" = "Background task ended"; +"native.backgroundTaskFinished" = "Background task finished"; "native.balancedReasoning" = "Balanced reasoning"; "native.base" = "Base"; "native.base.variant2" = "Base: %1$@"; @@ -4995,6 +5003,7 @@ "native.endThisTerminal" = "End This Terminal"; "native.endWithoutAsking" = "End without asking"; "native.english" = "English"; +"native.enlargeImage" = "Enlarge image"; "native.enterABranchName" = "Enter a branch name"; "native.enterABranchName.variant2" = "Enter a branch name."; "native.enterABranchNameBeforePublishing" = "Enter a branch name before publishing."; @@ -5551,6 +5560,7 @@ "native.isNotInstalledLitheInstallsItWithYourNpmIntoItsApplicationData" = "%1$@ is not installed. Lithe installs it with your npm into its application data."; "native.isVersionOrNewerIsRequired" = "%1$@ is version %2$@; %3$@ or newer is required."; "native.itCanReadFilesRunCommandsAndEditCodeAfterAskingForPermission" = "It can read files, run commands and edit code after asking for permission."; +"native.itemLldOfLld" = "Item %1$lld of %2$lld"; "native.itsDetailsAndActionsWillAppearHere" = "Its details and actions will appear here."; "native.japaneseCharacters" = "Japanese characters"; "native.javaApplication" = "Java Application"; @@ -6003,6 +6013,7 @@ "native.nextDiffMatch" = "Next diff match"; "native.nextDifference" = "Next difference"; "native.nextFile" = "Next File"; +"native.nextImage" = "Next image"; "native.nextMatch" = "Next Match"; "native.nextMatch.variant2" = "Next match"; "native.nextMatchReturn" = "Next match (Return)"; @@ -6391,6 +6402,7 @@ "native.previousDiffMatch" = "Previous diff match"; "native.previousDifference" = "Previous difference"; "native.previousFile" = "Previous File"; +"native.previousImage" = "Previous image"; "native.previousMatch" = "Previous Match"; "native.previousMatch.variant2" = "Previous match"; "native.previousMatchShiftReturn" = "Previous match (Shift+Return)"; diff --git a/macos/Resources/zh-Hans.lproj/Localizable.strings b/macos/Resources/zh-Hans.lproj/Localizable.strings index da19ae2041..775413b6f3 100644 --- a/macos/Resources/zh-Hans.lproj/Localizable.strings +++ b/macos/Resources/zh-Hans.lproj/Localizable.strings @@ -373,6 +373,8 @@ "Background Services" = "后台服务"; "Background services" = "后台服务"; "Background services started in Lithe are still running" = "在 Lithe 中启动的后台服务仍在运行"; +"Background task ended" = "后台任务已结束"; +"Background task finished" = "后台任务已完成"; "Balanced reasoning" = "平衡思考深度与响应速度"; "Base" = "目标"; "Base %@ → Target %@" = "基准 %@ → 目标 %@"; @@ -1180,6 +1182,7 @@ "End on next launch" = "下次启动时结束"; "End without asking" = "直接结束"; "English" = "英文"; +"Enlarge image" = "放大查看图片"; "Enter API key" = "填写 API Key"; "Enter a ChangeList name (1–100 characters)." = "请输入更改列表名称（1–100 个字符）。"; "Enter a branch name" = "请输入分支名称"; @@ -1627,6 +1630,7 @@ "Invalid shared instruction request." = "共享指令请求无效。"; "Invert" = "反选"; "It can read files, run commands and edit code after asking for permission." = "它可以在征得你的许可后读取文件、运行命令和修改代码。"; +"Item %lld of %lld" = "第 %lld 项，共 %lld 项"; "Its details and actions will appear here." = "工作树详情和可用操作会显示在这里。"; "JDK Home" = "JDK 主目录"; "JDK home" = "JDK 主目录"; @@ -2014,6 +2018,7 @@ "Next console expression" = "下一条表达式"; "Next diff match" = "下一个 Diff 匹配项"; "Next difference" = "下一个差异"; +"Next image" = "下一张图片"; "Next match" = "下一个匹配"; "Next match (Return)" = "下一个匹配项（Return）"; "Next run" = "下次运行"; @@ -2399,6 +2404,7 @@ "Previous console expression" = "上一条表达式"; "Previous diff match" = "上一个 Diff 匹配项"; "Previous difference" = "上一个差异"; +"Previous image" = "上一张图片"; "Previous match" = "上一个匹配"; "Previous match (Shift+Return)" = "上一个匹配项（Shift+Return）"; "Previous session" = "上一次会话"; @@ -4175,6 +4181,8 @@ "native.backgroundServices" = "后台服务"; "native.backgroundServices.variant2" = "后台服务"; "native.backgroundServicesStartedInLitheAreStillRunning" = "在 Lithe 中启动的后台服务仍在运行"; +"native.backgroundTaskEnded" = "后台任务已结束"; +"native.backgroundTaskFinished" = "后台任务已完成"; "native.balancedReasoning" = "平衡思考深度与响应速度"; "native.base" = "目标"; "native.base.variant2" = "基点：%1$@"; @@ -4995,6 +5003,7 @@ "native.endThisTerminal" = "结束此终端"; "native.endWithoutAsking" = "直接结束"; "native.english" = "英文"; +"native.enlargeImage" = "放大查看图片"; "native.enterABranchName" = "请输入分支名称"; "native.enterABranchName.variant2" = "请输入分支名称。"; "native.enterABranchNameBeforePublishing" = "请输入分支名称后再发布。"; @@ -5551,6 +5560,7 @@ "native.isNotInstalledLitheInstallsItWithYourNpmIntoItsApplicationData" = "%1$@ 尚未安装。Lithe 会用你的 npm 把它安装到应用数据目录。"; "native.isVersionOrNewerIsRequired" = "%1$@ 的版本是 %2$@，需要 %3$@ 或更高版本。"; "native.itCanReadFilesRunCommandsAndEditCodeAfterAskingForPermission" = "它可以在征得你的许可后读取文件、运行命令和修改代码。"; +"native.itemLldOfLld" = "第 %1$lld 项，共 %2$lld 项"; "native.itsDetailsAndActionsWillAppearHere" = "工作树详情和可用操作会显示在这里。"; "native.japaneseCharacters" = "日文字符"; "native.javaApplication" = "Java 应用"; @@ -6003,6 +6013,7 @@ "native.nextDiffMatch" = "下一个 Diff 匹配项"; "native.nextDifference" = "下一个差异"; "native.nextFile" = "下一个文件"; +"native.nextImage" = "下一张图片"; "native.nextMatch" = "下一个匹配项"; "native.nextMatch.variant2" = "下一个匹配"; "native.nextMatchReturn" = "下一个匹配项（Return）"; @@ -6391,6 +6402,7 @@ "native.previousDiffMatch" = "上一个 Diff 匹配项"; "native.previousDifference" = "上一个差异"; "native.previousFile" = "上一个文件"; +"native.previousImage" = "上一张图片"; "native.previousMatch" = "上一个匹配项"; "native.previousMatch.variant2" = "上一个匹配"; "native.previousMatchShiftReturn" = "上一个匹配项（Shift+Return）"; diff --git a/macos/Sources/Lithe/Views/Agent/AgentImageViews.swift b/macos/Sources/Lithe/Views/Agent/AgentImageViews.swift new file mode 100644 index 0000000000..7f41fc2073 --- /dev/null +++ b/macos/Sources/Lithe/Views/Agent/AgentImageViews.swift @@ -0,0 +1,269 @@ +import AppKit +import ImageIO +import SwiftUI +import LitheAgentConversationModule + +/// Images of one tool group or message, opened at `index` in the enlarged viewer. +struct AgentImageGallery: Identifiable, Equatable { + let id = UUID() + let images: [AgentImage] + var index: Int +} + +/// Owns the open viewer. Rows hold only this reference, so opening or closing +/// the viewer never re-evaluates the transcript rows themselves. +@MainActor +final class AgentImageViewerState: ObservableObject { + @Published var gallery: AgentImageGallery? + + func open(_ images: [AgentImage], at index: Int) { + guard images.indices.contains(index) else { return } + gallery = AgentImageGallery(images: images, index: index) + } +} + +private struct AgentImageViewerStateKey: EnvironmentKey { + static let defaultValue: AgentImageViewerState? = nil +} + +extension EnvironmentValues { + var agentImageViewer: AgentImageViewerState? { + get { self[AgentImageViewerStateKey.self] } + set { self[AgentImageViewerStateKey.self] = newValue } + } +} + +/// Decodes bounded renditions off the main thread. Inline pictures can be several +/// megabytes, so rows ask for a thumbnail-sized decode and share it through a cache. +enum AgentImageLoader { + private static let cache = NSCache() + + static func cachedImage(for image: AgentImage, maxPixel: Int) -> NSImage? { + cache.object(forKey: key(image, maxPixel)) + } + + static func image(for image: AgentImage, maxPixel: Int) async -> NSImage? { + if let cached = cachedImage(for: image, maxPixel: maxPixel) { return cached } + let source = image.source + let rendition = await Task.detached(priority: .utility) { () -> CGImage? in + let imageSource: CGImageSource? + switch source { + case .data(let data): imageSource = CGImageSourceCreateWithData(data as CFData, nil) + case .file(let path): imageSource = CGImageSourceCreateWithURL(URL(fileURLWithPath: path) as CFURL, nil) + } + guard let imageSource else { return nil } + return CGImageSourceCreateThumbnailAtIndex(imageSource, 0, [ + kCGImageSourceCreateThumbnailFromImageAlways: true, + kCGImageSourceCreateThumbnailWithTransform: true, + kCGImageSourceThumbnailMaxPixelSize: maxPixel, + ] as CFDictionary) + }.value + guard let rendition else { return nil } + let result = NSImage(cgImage: rendition, size: NSSize(width: rendition.width, height: rendition.height)) + cache.setObject(result, forKey: key(image, maxPixel)) + return result + } + + private static func key(_ image: AgentImage, _ maxPixel: Int) -> NSString { "\(image.id)@\(maxPixel)" as NSString } +} + +/// A picture that fills its frame, or fits inside it, once decoded. +struct AgentImageThumbnail: View { + let image: AgentImage + var maxPixel = 160 + var fills = true + @State private var rendition: NSImage? + + var body: some View { + ZStack { + if let rendition { + Image(nsImage: rendition).resizable().aspectRatio(contentMode: fills ? .fill : .fit) + } else { + Rectangle().fill(LitheTheme.raised) + } + } + .task(id: image.id) { + rendition = AgentImageLoader.cachedImage(for: image, maxPixel: maxPixel) + if rendition == nil { rendition = await AgentImageLoader.image(for: image, maxPixel: maxPixel) } + } + } +} + +/// Newest pictures beside a tool row or group, `+N` for the rest. Hovering previews +/// one; clicking opens the whole set in the viewer. +struct AgentImageStrip: View { + @Environment(\.agentImageViewer) private var viewer + let images: [AgentImage] + static let visibleLimit = 3 + static let thumbnailSize = CGSize(width: 36, height: 26) + + var body: some View { + let first = max(images.count - Self.visibleLimit, 0) + HStack(spacing: 6) { + if first > 0 { + Text(verbatim: "+\(first)") + .font(LitheTheme.uiFont(size: 12.5)) + .foregroundStyle(LitheTheme.secondaryText) + } + ForEach(Array(images.enumerated()).suffix(Self.visibleLimit), id: \.element.id) { index, image in + AgentImageThumbnail(image: image) + .frame(width: Self.thumbnailSize.width, height: Self.thumbnailSize.height) + .clipShape(RoundedRectangle(cornerRadius: 5)) + .overlay(RoundedRectangle(cornerRadius: 5).stroke(LitheTheme.panelBorder, lineWidth: 1)) + .contentShape(Rectangle()) + .lithePointer() + .onTapGesture { viewer?.open(images, at: index) } + .workbenchHoverCard(placement: .above, maximumWidth: AgentImagePreviewCard.width) { + AgentImagePreviewCard(image: image, position: index + 1, count: images.count) + } + .accessibilityElement() + .accessibilityAddTraits(.isButton) + .accessibilityLabel(Text(verbatim: image.title ?? "Image")) + .accessibilityHint(Text("Enlarge image")) + } + } + .fixedSize() + .accessibilityIdentifier("agent-image-strip") + } +} + +/// Non-interactive hover card with a larger rendition and its caption. +private struct AgentImagePreviewCard: View { + @Environment(\.locale) private var locale + let image: AgentImage + let position: Int + let count: Int + static let width: CGFloat = 340 + + var body: some View { + VStack(alignment: .leading, spacing: 0) { + AgentImageThumbnail(image: image, maxPixel: 960, fills: false) + .frame(maxWidth: Self.width, maxHeight: 220) + .frame(minHeight: 60) + HStack(spacing: 8) { + Text(verbatim: image.title ?? "") + .lineLimit(1).truncationMode(.middle) + Spacer(minLength: 8) + if count > 1 { + Text(String(format: AgentLocalization.string("Item %lld of %lld", locale: locale), position, count)) + .foregroundStyle(LitheTheme.secondaryText) + } + } + .font(LitheTheme.uiFont(size: 12.5)) + .foregroundStyle(LitheTheme.HoverTooltip.foreground) + .padding(.horizontal, 12) + .padding(.vertical, 9) + } + .frame(width: Self.width) + .litheHoverTooltipSurface() + } +} + +/// Attachments of a user message, larger than tool thumbnails. +struct AgentUserImages: View { + @Environment(\.agentImageViewer) private var viewer + let images: [AgentImage] + + var body: some View { + HStack(spacing: 6) { + ForEach(Array(images.enumerated()), id: \.element.id) { index, image in + AgentImageThumbnail(image: image, maxPixel: 320) + .frame(width: 84, height: 64) + .clipShape(RoundedRectangle(cornerRadius: 7)) + .overlay(RoundedRectangle(cornerRadius: 7).stroke(LitheTheme.panelBorder, lineWidth: 1)) + .contentShape(Rectangle()) + .lithePointer() + .onTapGesture { viewer?.open(images, at: index) } + .accessibilityElement() + .accessibilityAddTraits(.isButton) + .accessibilityLabel(Text(verbatim: image.title ?? "Image")) + .accessibilityHint(Text("Enlarge image")) + } + } + .accessibilityIdentifier("agent-user-images") + } +} + +/// Full-pane viewer: dimmed backdrop, image fitted to the pane, arrows and Esc. +struct AgentImageViewer: View { + @Environment(\.locale) private var locale + @ObservedObject var state: AgentImageViewerState + let gallery: AgentImageGallery + + var body: some View { + let image = gallery.images[gallery.index] + ZStack { + Color.black.opacity(0.62) + .contentShape(Rectangle()) + .onTapGesture { state.gallery = nil } + VStack(spacing: 10) { + AgentImageThumbnail(image: image, maxPixel: 2400, fills: false) + .frame(maxWidth: .infinity, maxHeight: .infinity) + .shadow(color: .black.opacity(0.35), radius: 12) + .allowsHitTesting(false) + HStack(spacing: 8) { + Text(verbatim: image.title ?? "") + .lineLimit(1).truncationMode(.middle) + if gallery.images.count > 1 { + Text(String(format: AgentLocalization.string("Item %lld of %lld", locale: locale), + gallery.index + 1, gallery.images.count)) + .foregroundStyle(.white.opacity(0.7)) + } + } + .font(LitheTheme.uiFont(size: 12.5)) + .foregroundStyle(.white) + } + .padding(.horizontal, 56) + .padding(.vertical, 40) + HStack { + if gallery.images.count > 1 { + step(-1, systemImage: "chevron.left", key: .leftArrow, label: "Previous image") + } + Spacer() + if gallery.images.count > 1 { + step(1, systemImage: "chevron.right", key: .rightArrow, label: "Next image") + } + } + .padding(.horizontal, 12) + VStack { + HStack { + Spacer() + Button { state.gallery = nil } label: { + Image(systemName: "xmark").font(LitheTheme.uiFont(size: 13, weight: .semibold)) + .frame(width: 28, height: 28) + .background(.white.opacity(0.16), in: Circle()) + .contentShape(Circle()) + } + .buttonStyle(.litheNoPress) + .foregroundStyle(.white) + .keyboardShortcut(.cancelAction) + .lithePointer() + .help("Close") + .accessibilityLabel("Close") + } + Spacer() + } + .padding(12) + } + .accessibilityElement(children: .contain) + .accessibilityIdentifier("agent-image-viewer") + } + + private func step(_ delta: Int, systemImage: String, key: KeyEquivalent, label: LocalizedStringKey) -> some View { + Button { + let count = gallery.images.count + state.gallery?.index = (gallery.index + delta + count) % count + } label: { + Image(systemName: systemImage).font(LitheTheme.uiFont(size: 14, weight: .semibold)) + .frame(width: 32, height: 32) + .background(.white.opacity(0.16), in: Circle()) + .contentShape(Circle()) + } + .buttonStyle(.litheNoPress) + .foregroundStyle(.white) + .keyboardShortcut(key, modifiers: []) + .lithePointer() + .help(label) + .accessibilityLabel(label) + } +} diff --git a/macos/Sources/Lithe/Views/Agent/AgentToolGroupView.swift b/macos/Sources/Lithe/Views/Agent/AgentToolGroupView.swift index d1e13a9fa8..3a5829857f 100644 --- a/macos/Sources/Lithe/Views/Agent/AgentToolGroupView.swift +++ b/macos/Sources/Lithe/Views/Agent/AgentToolGroupView.swift @@ -73,6 +73,7 @@ struct AgentToolGroupView: View { @State private var showsAllRows = false @State private var expandedToolID: String? + private var groupImages: [AgentImage] { messages.flatMap(\.toolDetails.images) } private var failedCount: Int { messages.filter { $0.toolStatus == .failed }.count } private var isRunning: Bool { messages.contains { $0.toolStatus == .inProgress || $0.toolStatus == .pending } } private var hasExpandedDetails: Bool { @@ -107,7 +108,15 @@ struct AgentToolGroupView: View { private var header: some View { let summary = AgentToolGroupPresentation.summary(messages, locale: locale) - return Button { expanded.toggle() } label: { + return HStack(spacing: 8) { + headerButton(summary: summary) + Spacer(minLength: 8) + if !groupImages.isEmpty { AgentImageStrip(images: groupImages) } + } + } + + private func headerButton(summary: String) -> some View { + Button { expanded.toggle() } label: { HStack(spacing: 5) { if isRunning { ProgressView().controlSize(.mini) } Text(verbatim: summary).lineLimit(1) @@ -172,6 +181,7 @@ struct AgentToolGroupView: View { .lineLimit(1) .truncationMode(.middle) .frame(maxWidth: .infinity, alignment: .leading) + if !message.toolDetails.images.isEmpty { AgentImageStrip(images: message.toolDetails.images) } status(message.toolStatus) if hasDetails { Image(systemName: "chevron.right") diff --git a/macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift b/macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift index da48729b17..d6f9ae8a4c 100644 --- a/macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift +++ b/macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift @@ -163,6 +163,7 @@ struct AgentTranscriptView: View { var isEditorLayout = false @State private var navigation = AgentTranscriptNavigationState() @State private var showsAgentPicker = false + @StateObject private var imageViewer = AgentImageViewerState() // A filtered-out row can be destroyed. Keep its preference in the owning // transcript view, isolated by session and message, until the actual data goes away. @State private var thoughtExpansions: [String: [String: AgentThoughtExpansion]] = [:] @@ -309,6 +310,8 @@ struct AgentTranscriptView: View { .id(sessionID) } } + .environment(\.agentImageViewer, imageViewer) + .overlay { AgentImageViewerOverlay(state: imageViewer) } .onChange(of: feature.openSessionIDs) { sessionIDs in thoughtExpansions = thoughtExpansions.filter { sessionIDs.contains($0.key) } windowLimits = windowLimits.filter { sessionIDs.contains($0.key) } @@ -332,6 +335,17 @@ struct AgentTranscriptView: View { } } +/// Only this overlay observes the viewer, so opening an image leaves the rows untouched. +private struct AgentImageViewerOverlay: View { + @ObservedObject var state: AgentImageViewerState + + var body: some View { + if let gallery = state.gallery { + AgentImageViewer(state: state, gallery: gallery) + } + } +} + /// Transcript rows as a value of the conversation data. The owning view is /// re-evaluated for every published model change (quota, sessions, statistics); /// comparing these inputs keeps scrolling from re-diffing every row each time. @@ -529,33 +543,16 @@ private struct AgentMessageRow: View { var body: some View { switch message.role { case .user: - HStack(alignment: .bottom, spacing: 4) { - Spacer(minLength: 40) - AgentCopyButton(text: message.text, help: "Copy message") - .opacity(isHovered ? 1 : 0) - .allowsHitTesting(isHovered) - VStack(alignment: .trailing, spacing: 4) { - Text(message.text) - if let delivery = message.delivery { - Group { - switch delivery { - case .sending: Text("Sending…") - case .received: Text("Agent received") - case .failed(let reason): Text(AgentLocalization.message(reason, locale: locale)).foregroundStyle(LitheTheme.error) - } - } - .font(LitheTheme.uiFont(size: 10)) - .foregroundStyle(AgentPanelStyle.secondary) + if let notifications = AgentTaskNotification.parse(message.text) { + VStack(alignment: .leading, spacing: 4) { + ForEach(Array(notifications.enumerated()), id: \.offset) { _, notification in + AgentTaskNotificationRow(notification: notification) } } - .font(LitheTheme.uiFont(size: isEditorLayout ? 14 : 13)) - .textSelection(.enabled) - .padding(.horizontal, isEditorLayout ? 14 : 12) - .padding(.vertical, isEditorLayout ? 10 : 8) - .background(isEditorLayout ? AnyShapeStyle(AgentPanelStyle.context) : AnyShapeStyle(LitheTheme.accent.opacity(0.18)), - in: RoundedRectangle(cornerRadius: isEditorLayout ? 16 : 10)) + .frame(maxWidth: .infinity, alignment: .leading) + } else { + userBubble } - .onHover { isHovered = $0 } case .agent: VStack(alignment: .leading, spacing: 2) { AgentMarkdownMessage(text: message.text) @@ -575,4 +572,63 @@ private struct AgentMessageRow: View { AgentToolGroupView(messages: [message], searchText: "", onOpenFile: onOpenFile) } } + + /// Attached image files and replayed pictures show as thumbnails that open + /// the viewer; the text keeps the user's words and non-image attachments. + private var userBubble: some View { + let content = AgentUserMessageContent.parse(message.text) + let images = message.images + content.images + return HStack(alignment: .bottom, spacing: 4) { + Spacer(minLength: 40) + AgentCopyButton(text: content.text, help: "Copy message") + .opacity(isHovered ? 1 : 0) + .allowsHitTesting(isHovered) + VStack(alignment: .trailing, spacing: 6) { + if !images.isEmpty { AgentUserImages(images: images) } + if !content.text.isEmpty { Text(content.text) } + if let delivery = message.delivery { + Group { + switch delivery { + case .sending: Text("Sending…") + case .received: Text("Agent received") + case .failed(let reason): Text(AgentLocalization.message(reason, locale: locale)).foregroundStyle(LitheTheme.error) + } + } + .font(LitheTheme.uiFont(size: 10)) + .foregroundStyle(AgentPanelStyle.secondary) + } + } + .font(LitheTheme.uiFont(size: isEditorLayout ? 14 : 13)) + .textSelection(.enabled) + .padding(.horizontal, isEditorLayout ? 14 : 12) + .padding(.vertical, isEditorLayout ? 10 : 8) + .background(isEditorLayout ? AnyShapeStyle(AgentPanelStyle.context) : AnyShapeStyle(LitheTheme.accent.opacity(0.18)), + in: RoundedRectangle(cornerRadius: isEditorLayout ? 16 : 10)) + } + .onHover { isHovered = $0 } + } +} + +/// A Claude Code background-task report, shown as a quiet status line instead of the raw markup. +private struct AgentTaskNotificationRow: View { + @Environment(\.locale) private var locale + let notification: AgentTaskNotification + + private var failed: Bool { ["failed", "error", "killed", "stopped"].contains(notification.status?.lowercased() ?? "") } + + var body: some View { + HStack(spacing: 6) { + Image(systemName: failed ? "exclamationmark.circle" : "checkmark.circle") + .font(LitheTheme.uiFont(size: 11)) + .foregroundStyle(failed ? LitheTheme.error : LitheTheme.tertiaryText) + Text(verbatim: notification.summary ?? AgentLocalization.string( + failed ? "Background task ended" : "Background task finished", locale: locale)) + .lineLimit(2) + } + .font(LitheTheme.uiFont(size: 12.5)) + .foregroundStyle(LitheTheme.secondaryText) + .help(notification.outputFile ?? "") + .textSelection(.enabled) + .accessibilityIdentifier("agent-task-notification") + } } diff --git a/macos/Sources/LitheAgentConversationModule/Application/AgentConnectionModel.swift b/macos/Sources/LitheAgentConversationModule/Application/AgentConnectionModel.swift index ed19e78f7d..cd08878720 100644 --- a/macos/Sources/LitheAgentConversationModule/Application/AgentConnectionModel.swift +++ b/macos/Sources/LitheAgentConversationModule/Application/AgentConnectionModel.swift @@ -978,9 +978,10 @@ public final class AgentConnectionModel: ObservableObject { // Live input is already shown locally, including pending steering. // Replay still rebuilds the upstream-owned transcript while loading. guard conversations[sessionID]?.isResponding != true else { return } - guard let text = Self.text(of: update) else { return } + let image = (update["content"] as? [String: Any]).flatMap(AgentImage.parse(block:)) + guard let text = Self.text(of: update) ?? (image == nil ? nil : "") else { return } flushPendingText() - append(text, role: .user, to: sessionID, subagentSessionID: subagentSessionID) + append(text, role: .user, to: sessionID, subagentSessionID: subagentSessionID, image: image) case "tool_call", "tool_call_update": guard let toolCallID = update["toolCallId"] as? String else { return } flushPendingText() @@ -1207,14 +1208,20 @@ public final class AgentConnectionModel: ObservableObject { replayToolIndex[sessionID] = nil } - private func append(_ text: String, role: AgentConversationMessage.Role, to sessionID: String, subagentSessionID: String? = nil) { + private func append(_ text: String, role: AgentConversationMessage.Role, to sessionID: String, + subagentSessionID: String? = nil, image: AgentImage? = nil) { + func attach(_ image: AgentImage?, to message: inout AgentConversationMessage) { + if let image, message.images.count < AgentImage.maximumPerMessage { message.images.append(image) } + } if conversations[sessionID]?.isLoading == true { if let last = replayMessages[sessionID]?.indices.last, replayMessages[sessionID]?[last].role == role, replayMessages[sessionID]?[last].subagentSessionID == subagentSessionID { replayMessages[sessionID]?[last].text += text + attach(image, to: &replayMessages[sessionID]![last]) } else { var message = AgentConversationMessage(role: role, text: text) message.subagentSessionID = subagentSessionID + attach(image, to: &message) replayMessages[sessionID, default: []].append(message) } return @@ -1223,9 +1230,12 @@ public final class AgentConnectionModel: ObservableObject { if let last = conversation.messages.indices.last, conversation.messages[last].role == role, conversation.messages[last].subagentSessionID == subagentSessionID { conversation.messages[last].text += text + attach(image, to: &conversation.messages[last]) } else { - conversation.messages.append(AgentConversationMessage(role: role, text: text)) - conversation.messages[conversation.messages.count - 1].subagentSessionID = subagentSessionID + var message = AgentConversationMessage(role: role, text: text) + message.subagentSessionID = subagentSessionID + attach(image, to: &message) + conversation.messages.append(message) } conversations[sessionID] = conversation } diff --git a/macos/Sources/LitheAgentConversationModule/Application/AgentConversationState.swift b/macos/Sources/LitheAgentConversationModule/Application/AgentConversationState.swift index fd745f835e..e7526c8a35 100644 --- a/macos/Sources/LitheAgentConversationModule/Application/AgentConversationState.swift +++ b/macos/Sources/LitheAgentConversationModule/Application/AgentConversationState.swift @@ -33,6 +33,8 @@ public struct AgentConversationMessage: Identifiable, Equatable, Sendable { public var delivery: Delivery? public var toolStatus: ToolStatus? public var toolDetails = AgentToolDetails() + /// Pictures carried by a replayed user message; live attachments are parsed from its text. + public var images: [AgentImage] = [] /// Child output is retained with its owner, separate from the parent's reply. public var subagentSessionID: String? diff --git a/macos/Sources/LitheAgentConversationModule/Application/AgentImage.swift b/macos/Sources/LitheAgentConversationModule/Application/AgentImage.swift new file mode 100644 index 0000000000..110392058f --- /dev/null +++ b/macos/Sources/LitheAgentConversationModule/Application/AgentImage.swift @@ -0,0 +1,129 @@ +import Foundation + +/// An image the Agent produced or the user attached, shown beside the tool call +/// or message that carries it. Inline bytes stay bounded; file references are +/// decoded lazily by the view layer so replaying a long history never decodes pixels. +public struct AgentImage: Identifiable, Equatable, Sendable { + public enum Source: Sendable { + case data(Data) + /// Absolute path of an image on disk, such as a screenshot the Agent read. + case file(String) + } + + public let id: String + public var title: String? + public let source: Source + + /// Larger payloads are dropped: they would sit in the transcript state for the whole session. + public static let maximumBytes = 10_000_000 + /// One tool result rarely returns more; the cap keeps a runaway tool from filling memory. + public static let maximumPerMessage = 24 + public static let fileExtensions: Set = ["png", "jpg", "jpeg", "gif", "heic", "tif", "tiff", "webp", "bmp"] + + public init(data: Data, title: String? = nil) { + self.id = "data:\(data.count):\(Self.fingerprint(data))" + self.title = title + self.source = .data(data) + } + + public init(filePath: String, title: String? = nil) { + self.id = "file:\(filePath)" + self.title = title ?? (filePath as NSString).lastPathComponent + self.source = .file(filePath) + } + + // Identity is the content fingerprint or path; comparing megabytes of pixels + // on every transcript diff would make scrolling pay for each image. + public static func == (lhs: Self, rhs: Self) -> Bool { lhs.id == rhs.id && lhs.title == rhs.title } + + public static func isImageFile(_ path: String) -> Bool { + fileExtensions.contains((path as NSString).pathExtension.lowercased()) + } + + /// Reads an ACP `image` content block: base64 `data` with `mimeType`, or a local `file:` `uri`. + static func parse(block: [String: Any]) -> Self? { + guard block["type"] as? String == "image" else { return nil } + let uri = block["uri"] as? String + let uriTitle = uri.flatMap { URL(string: $0)?.lastPathComponent }.flatMap { $0.isEmpty ? nil : $0 } + if let encoded = block["data"] as? String, !encoded.isEmpty { + if let mime = block["mimeType"] as? String, !mime.lowercased().hasPrefix("image/") { return nil } + // Base64 inflates by a third; refuse before allocating the decoded copy. + guard encoded.utf8.count <= maximumBytes / 3 * 4, + let data = Data(base64Encoded: encoded, options: .ignoreUnknownCharacters), !data.isEmpty, + data.count <= maximumBytes else { return nil } + return Self(data: data, title: uriTitle) + } + if let uri, let url = URL(string: uri), url.isFileURL, isImageFile(url.path) { + return Self(filePath: url.path) + } + return nil + } + + /// FNV-1a over the bytes: stable within a run and cheap next to base64 decoding. + private static func fingerprint(_ data: Data) -> String { + var hash: UInt64 = 0xcbf2_9ce4_8422_2325 + for byte in data { hash = (hash ^ UInt64(byte)) &* 0x100_0000_01b3 } + return String(hash, radix: 16) + } +} + +/// A user message's text split from the image attachments it lists. +public struct AgentUserMessageContent: Equatable, Sendable { + public var text: String + public var images: [AgentImage] + + private static let attachmentMarker = "📎 " + + /// Composer prompts list attachments as `📎 /path` lines. Image files become + /// thumbnails; other attachments and the user's own words stay as text. + public static func parse(_ message: String) -> Self { + guard message.contains(attachmentMarker) else { return Self(text: message, images: []) } + var images: [AgentImage] = [] + var kept: [Substring] = [] + for line in message.split(separator: "\n", omittingEmptySubsequences: false) { + if line.hasPrefix(attachmentMarker) { + let path = String(line.dropFirst(attachmentMarker.count)) + if path.hasPrefix("/"), AgentImage.isImageFile(path), images.count < AgentImage.maximumPerMessage { + images.append(AgentImage(filePath: path)) + continue + } + } + kept.append(line) + } + guard !images.isEmpty else { return Self(text: message, images: []) } + let text = kept.joined(separator: "\n").trimmingCharacters(in: .whitespacesAndNewlines) + return Self(text: text, images: images) + } +} + +/// Claude Code reports finished background work as a `` block +/// inside a user message. It is protocol markup, not something the user typed. +public struct AgentTaskNotification: Equatable, Sendable { + public var taskID: String? + public var status: String? + public var summary: String? + public var outputFile: String? + + /// Nil unless the whole message consists of one or more notification blocks. + public static func parse(_ message: String) -> [Self]? { + let open = "", close = "" + var rest = message.trimmingCharacters(in: .whitespacesAndNewlines) + guard rest.hasPrefix(open) else { return nil } + var result: [Self] = [] + while !rest.isEmpty { + guard rest.hasPrefix(open), let end = rest.range(of: close) else { return nil } + let body = String(rest[rest.index(rest.startIndex, offsetBy: open.count).. String? { + guard let start = body.range(of: "<\(name)>"), + let end = body.range(of: "", range: start.upperBound..= 0 ? $0 : nil } : nil, deletions: validStats ? (stats?["removed"] as? Int).flatMap { $0 >= 0 ? $0 : nil } : nil) } + let reported = content.prefix(100).compactMap { item -> AgentImage? in + guard item["type"] as? String == "content", let block = item["content"] as? [String: Any] else { return nil } + return AgentImage.parse(block: block) + } + // An update without pictures must not erase ones an earlier update delivered. + if !reported.isEmpty { images = Array(reported.prefix(AgentImage.maximumPerMessage)) } self.content = content.prefix(100).compactMap { item in switch item["type"] as? String { case "content": guard let block = item["content"] as? [String: Any] else { return nil } + if block["type"] as? String == "image" { return nil } if let text = block["text"] as? String { return Content(title: "Output", text: Self.bounded(text)) } @@ -128,6 +137,7 @@ public struct AgentToolDetails: Equatable, Sendable { } } } + titleImages() // The pinned Claude adapter forwards the SDK's public FileWriteOutput // in tool metadata. Prefer its complete text over newline-less hunks; // the explicit create type distinguishes a new file from an insertion. @@ -149,6 +159,20 @@ public struct AgentToolDetails: Equatable, Sendable { } } + /// Reading an image file often reports only the path; show that file itself. + /// Untitled pictures borrow the file name or the tool's own caption. + private mutating func titleImages() { + if images.isEmpty, kind == "read" { + images = locations.lazy.map(\.path).filter(AgentImage.isImageFile).prefix(AgentImage.maximumPerMessage) + .map { AgentImage(filePath: $0) } + } + guard images.contains(where: { $0.title == nil }) else { return } + let caption = locations.first.map { ($0.path as NSString).lastPathComponent } + ?? content.first { $0.title == "Output" }?.text.split(whereSeparator: \.isNewline).first.map { String($0.prefix(80)) } + guard let caption else { return } + for index in images.indices where images[index].title == nil { images[index].title = caption } + } + private static func display(_ value: Any) -> String? { if value is NSNull { return nil } if let text = value as? String { return bounded(text) } diff --git a/macos/Tests/LitheTests/AgentImageTests.swift b/macos/Tests/LitheTests/AgentImageTests.swift new file mode 100644 index 0000000000..2b640b9529 --- /dev/null +++ b/macos/Tests/LitheTests/AgentImageTests.swift @@ -0,0 +1,100 @@ +import Foundation +import Testing +@testable import Lithe +@testable import LitheAgentConversationModule + +@Suite("Agent images and task notifications") +struct AgentImageTests { + private static let pixel = Data([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) + + private func imageBlock(_ data: Data = pixel, mime: String = "image/png") -> [String: Any] { + ["type": "content", "content": ["type": "image", "data": data.base64EncodedString(), "mimeType": mime]] + } + + @Test + func toolResultImageBecomesAThumbnailSourceInsteadOfAPlaceholderRow() { + var details = AgentToolDetails() + details.merge(["kind": "read", "locations": [["path": "/tmp/box-light.png"]], + "content": [imageBlock()]]) + #expect(details.images.count == 1) + #expect(details.images[0].title == "box-light.png") + #expect(details.content.isEmpty, "An image must not also surface as an 'image' text row") + #expect(!details.isEmpty) + } + + @Test + func laterUpdateWithoutPicturesKeepsEarlierOnes() { + var details = AgentToolDetails() + details.merge(["content": [imageBlock()]]) + details.merge(["content": [["type": "content", "content": ["type": "text", "text": "done"]]]]) + #expect(details.images.count == 1) + } + + @Test + func readingAnImageFileWithoutInlineDataShowsTheFile() { + var details = AgentToolDetails() + details.merge(["kind": "read", "locations": [["path": "/tmp/shot.PNG"], ["path": "/tmp/notes.txt"]]]) + #expect(details.images.map(\.title) == ["shot.PNG"]) + var edit = AgentToolDetails() + edit.merge(["kind": "edit", "locations": [["path": "/tmp/shot.png"]]]) + #expect(edit.images.isEmpty, "Only a read implies viewing the file") + } + + @Test + func invalidOrOversizedImageBlocksAreRejected() { + #expect(AgentImage.parse(block: ["type": "image", "data": "!!!", "mimeType": "image/png"]) == nil) + #expect(AgentImage.parse(block: ["type": "image", "data": Self.pixel.base64EncodedString(), "mimeType": "text/plain"]) == nil) + let huge = String(repeating: "A", count: AgentImage.maximumBytes / 3 * 4 + 4) + #expect(AgentImage.parse(block: ["type": "image", "data": huge, "mimeType": "image/png"]) == nil) + } + + @Test + func identicalBytesShareAnIdentity() { + #expect(AgentImage(data: Self.pixel).id == AgentImage(data: Self.pixel).id) + #expect(AgentImage(data: Self.pixel).id != AgentImage(data: Data([1, 2, 3])).id) + } + + @Test + func composerAttachmentLinesBecomeImagesAndOtherLinesStayText() { + let content = AgentUserMessageContent.parse("看看这个\n\n📎 /tmp/Pasted image.png\n\n📎 /tmp/report.pdf") + #expect(content.images.map(\.title) == ["Pasted image.png"]) + #expect(content.text == "看看这个\n\n\n📎 /tmp/report.pdf") + let plain = AgentUserMessageContent.parse("no attachments") + #expect(plain == AgentUserMessageContent(text: "no attachments", images: [])) + } + + @Test + func taskNotificationIsParsedFromTheWholeMessageOnly() throws { + let markup = """ + + abc + completed + Agent "Explore" finished + /tmp/abc.output + + """ + let parsed = try #require(AgentTaskNotification.parse(markup + "\n" + markup)) + #expect(parsed.count == 2) + #expect(parsed[0].summary == "Agent \"Explore\" finished") + #expect(parsed[0].status == "completed") + #expect(AgentTaskNotification.parse("Please explain tags") == nil) + #expect(AgentTaskNotification.parse(markup + " and some typed words") == nil) + } + + @Test + func blankAgentRepliesAreDroppedAndKeepToolRunsTogether() { + let messages = [ + AgentConversationMessage(id: "u", role: .user, text: "go"), + AgentConversationMessage(id: "t1", role: .tool, text: "Read", toolStatus: .completed), + AgentConversationMessage(id: "blank", role: .agent, text: "\n "), + AgentConversationMessage(id: "t2", role: .tool, text: "Read", toolStatus: .completed), + ] + let items = AgentTranscriptItem.grouped(messages) + #expect(items.count == 2) + if case .processGroup(let process) = items[1] { + #expect(AgentTranscriptItem.processContents(process).count == 1) + } else { + Issue.record("Expected one process group after the user message") + } + } +} diff --git a/shared/i18n/locales/en-US.json b/shared/i18n/locales/en-US.json index 7a86dc66d4..e3888ff472 100644 --- a/shared/i18n/locales/en-US.json +++ b/shared/i18n/locales/en-US.json @@ -8739,6 +8739,8 @@ "native.authentication": "Authentication", "native.auto": "Auto", "native.automaticTriesYourSSHAgentAndKeysFirstThenAsksForAPasswordLitheAsksInADialogWheneverTheSer": "Automatic tries your SSH agent and keys first, then asks for a password. Lithe asks in a dialog whenever the server needs an answer, such as a password, a passphrase, or a verification code.", + "native.backgroundTaskEnded": "Background task ended", + "native.backgroundTaskFinished": "Background task finished", "native.beforeAToolRuns": "Before a tool runs", "native.browse": "Browse…", "native.buildSystemX797f5735e5": "Build system", @@ -8789,6 +8791,7 @@ "native.editHook": "Edit Hook", "native.emptyProject": "Empty Project", "native.enableHook": "Enable Hook", + "native.enlargeImage": "Enlarge image", "native.enterAHost": "Enter a host.", "native.enterAValue": "Enter a value.", "native.enterAnAbsolutePathThatStartsWith": "Enter an absolute path that starts with /.", @@ -8830,6 +8833,7 @@ "native.importDataSources": "Import Data Sources…", "native.invalidHookFileGuards": "Invalid Hook file guards.", "native.isNeededForThisProjectButIsDisabled": "{arg1} is needed for this project but {arg2} is disabled.", + "native.itemLldOfLld": "Item {arg1} of {arg2}", "native.keepAliveInterval": "Keep-alive interval", "native.kernel": "Kernel", "native.latest": "Latest", @@ -8858,6 +8862,7 @@ "native.newHook": "New Hook", "native.newProjectX6f8d176462": "New Project", "native.newSSHConnection": "New SSH Connection", + "native.nextImage": "Next image", "native.noAccessibleDatabases": "No accessible databases", "native.noColumnsToDisplay": "No columns to display", "native.noHooksYet": "No Hooks yet.", @@ -8883,6 +8888,7 @@ "native.pressEnterOrClickOutsideTheListToApply": "Press Enter or click outside the list to apply", "native.previous30Days": "Previous 30 days", "native.previous7Days": "Previous 7 days", + "native.previousImage": "Previous image", "native.privateKey": "Private Key", "native.privateKeyX477bf990a2": "Private key", "native.privateNetwork": "Private network", diff --git a/shared/i18n/locales/zh-CN.json b/shared/i18n/locales/zh-CN.json index 089cc96cf2..b89b4196f3 100644 --- a/shared/i18n/locales/zh-CN.json +++ b/shared/i18n/locales/zh-CN.json @@ -8739,6 +8739,8 @@ "native.authentication": "认证方式", "native.auto": "自动", "native.automaticTriesYourSSHAgentAndKeysFirstThenAsksForAPasswordLitheAsksInADialogWheneverTheSer": "“自动”会先尝试 SSH Agent 和密钥，再询问密码。服务器需要回答时（例如密码、私钥口令或验证码），Lithe 会弹窗询问。", + "native.backgroundTaskEnded": "后台任务已结束", + "native.backgroundTaskFinished": "后台任务已完成", "native.beforeAToolRuns": "工具运行前", "native.browse": "浏览…", "native.buildSystemX797f5735e5": "构建系统", @@ -8789,6 +8791,7 @@ "native.editHook": "编辑 Hook", "native.emptyProject": "空项目", "native.enableHook": "启用 Hook", + "native.enlargeImage": "放大查看图片", "native.enterAHost": "请填写主机。", "native.enterAValue": "请输入内容。", "native.enterAnAbsolutePathThatStartsWith": "请输入以 / 开头的绝对路径。", @@ -8830,6 +8833,7 @@ "native.importDataSources": "导入数据源…", "native.invalidHookFileGuards": "Hook 文件校验信息无效。", "native.isNeededForThisProjectButIsDisabled": "此项目需要 {arg1} 支持，但 {arg2} 已停用。", + "native.itemLldOfLld": "第 {arg1} 项，共 {arg2} 项", "native.keepAliveInterval": "保活间隔", "native.kernel": "内核", "native.latest": "最新", @@ -8858,6 +8862,7 @@ "native.newHook": "新 Hook", "native.newProjectX6f8d176462": "新建项目", "native.newSSHConnection": "新建 SSH 连接", + "native.nextImage": "下一张图片", "native.noAccessibleDatabases": "没有可访问的数据库", "native.noColumnsToDisplay": "没有可显示的列", "native.noHooksYet": "还没有 Hook。", @@ -8883,6 +8888,7 @@ "native.pressEnterOrClickOutsideTheListToApply": "按 Enter 或点击列表外应用", "native.previous30Days": "过去 30 天", "native.previous7Days": "过去 7 天", + "native.previousImage": "上一张图片", "native.privateKey": "私钥", "native.privateKeyX477bf990a2": "私钥", "native.privateNetwork": "内网地址", diff --git a/shared/i18n/messages.json b/shared/i18n/messages.json index 5f6d1e684f..7e351dd6d2 100644 --- a/shared/i18n/messages.json +++ b/shared/i18n/messages.json @@ -80697,6 +80697,28 @@ "arguments": [] } }, + { + "id": "native.backgroundTaskEnded", + "description": "Native UI: Background task ended", + "platforms": [ + "macos" + ], + "macos": { + "key": "Background task ended", + "arguments": [] + } + }, + { + "id": "native.backgroundTaskFinished", + "description": "Native UI: Background task finished", + "platforms": [ + "macos" + ], + "macos": { + "key": "Background task finished", + "arguments": [] + } + }, { "id": "native.beforeAToolRuns", "description": "Native UI: Before a tool runs", @@ -81306,6 +81328,17 @@ "arguments": [] } }, + { + "id": "native.enlargeImage", + "description": "Native UI: Enlarge image", + "platforms": [ + "macos" + ], + "macos": { + "key": "Enlarge image", + "arguments": [] + } + }, { "id": "native.enterAHost", "description": "Native UI: Enter a host.", @@ -81804,6 +81837,24 @@ "arg2": "string" } }, + { + "id": "native.itemLldOfLld", + "description": "Native UI: Item %lld of %lld", + "platforms": [ + "macos" + ], + "macos": { + "key": "Item %lld of %lld", + "arguments": [ + "arg1", + "arg2" + ] + }, + "parameters": { + "arg1": "integer", + "arg2": "integer" + } + }, { "id": "native.keepAliveInterval", "description": "Native UI: Keep-alive interval", @@ -82144,6 +82195,17 @@ "arguments": [] } }, + { + "id": "native.nextImage", + "description": "Native UI: Next image", + "platforms": [ + "macos" + ], + "macos": { + "key": "Next image", + "arguments": [] + } + }, { "id": "native.noAccessibleDatabases", "description": "Native UI: No accessible databases", @@ -82427,6 +82489,17 @@ "arguments": [] } }, + { + "id": "native.previousImage", + "description": "Native UI: Previous image", + "platforms": [ + "macos" + ], + "macos": { + "key": "Previous image", + "arguments": [] + } + }, { "id": "native.privateKey", "description": "Native UI: Private Key", diff --git a/shared/platform-feature-matrix/features/agent-tool-activity-timeline.json b/shared/platform-feature-matrix/features/agent-tool-activity-timeline.json index c1be7f4517..96794b8fcb 100644 --- a/shared/platform-feature-matrix/features/agent-tool-activity-timeline.json +++ b/shared/platform-feature-matrix/features/agent-tool-activity-timeline.json @@ -2,14 +2,17 @@ "id": "agent-tool-activity-timeline", "area": "AI", "group": "Agent 对话", - "capability": "每轮工具调用与思考、过程说明放进同一个默认折叠的过程区；展开后相邻工具保留有序时间线、状态、输入输出和文件位置，最终回复单独展示", + "capability": "每轮工具调用与思考、过程说明放进同一个默认折叠的过程区；展开后相邻工具保留有序时间线、状态、输入输出和文件位置，最终回复单独展示；工具读取或截取的图片以缩略图显示在工具行和工具组标题旁（最多三张、其余 +N），悬停预览、点击放大，用户消息附带的图片同样可放大；Claude Code 后台任务通知显示为状态行（macOS）", "macos": { "evidence": [ "macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift", "macos/Sources/Lithe/Views/Agent/AgentToolGroupView.swift", "macos/Tests/LitheTests/AgentConversationPresentationTests.swift", "macos/Sources/Lithe/Views/Agent/AgentProcessGroupView.swift", - "macos/Tests/LitheTests/AgentThoughtTranscriptTests.swift" + "macos/Tests/LitheTests/AgentThoughtTranscriptTests.swift", + "macos/Sources/LitheAgentConversationModule/Application/AgentImage.swift", + "macos/Sources/Lithe/Views/Agent/AgentImageViews.swift", + "macos/Tests/LitheTests/AgentImageTests.swift" ], "implementationStatus": "implemented", "verificationStatus": "pending" @@ -30,6 +33,6 @@ "verificationStatus": "pending" }, "owner": "Agent", - "verification": "macOS 与 Windows：构造思考→过程说明→读文件→思考→执行命令→最终回复，确认每轮只有一个过程入口且默认收起，流式也不自动展开；收起时过程文字与工具行不占高度，最终回复始终可读。展开按原顺序呈现，只有相邻工具合并，状态与证据可继续查看。纯空白思考、过程说明不产生空行，没有思考和工具的直接回答不产生入口；新用户消息、已记录的轮次结束分别切断分组，工具或思考结束而没有最终回复时不伪造回复。搜索过程文字及工具输入、输出、文件路径，匹配组临时展开且不能在搜索期间收起，清空后恢复手动选择；切换和关闭会话验证偏好隔离与释放。深浅主题、窄宽面板、历史恢复和实时流均验证；历史导出、文件审阅、权限卡片、轮次统计和回复复制继续可用。单元测试与组件回放不替代真实 Agent 和 Windows Tauri 运行验收，未执行部分保持 pending。", - "notes": "2026-10-24：改为每轮一个默认折叠的过程区，包含思考、工具和中间说明；只改变呈现，不改写历史或 ACP 协议。ACP 无最终回复角色，按最后一条思考或工具的位置保留末尾正文；后续活动到达时才将先前正文移入过程。" + "verification": "macOS 与 Windows：构造思考→过程说明→读文件→思考→执行命令→最终回复，确认每轮只有一个过程入口且默认收起，流式也不自动展开；收起时过程文字与工具行不占高度，最终回复始终可读。展开按原顺序呈现，只有相邻工具合并，状态与证据可继续查看。纯空白思考、过程说明不产生空行，没有思考和工具的直接回答不产生入口；新用户消息、已记录的轮次结束分别切断分组，工具或思考结束而没有最终回复时不伪造回复。搜索过程文字及工具输入、输出、文件路径，匹配组临时展开且不能在搜索期间收起，清空后恢复手动选择；切换和关闭会话验证偏好隔离与释放。深浅主题、窄宽面板、历史恢复和实时流均验证；历史导出、文件审阅、权限卡片、轮次统计和回复复制继续可用。单元测试与组件回放不替代真实 Agent 和 Windows Tauri 运行验收，未执行部分保持 pending。 macOS 图片：让 Agent 读取多张图片，确认缩略图最多三张并显示 +N，悬停出现带标题和序号的预览，点击进入放大查看并可用方向键切换、Esc 关闭；用户粘贴图片后气泡显示缩略图且可放大；触发后台任务后不再出现原始 task-notification 标记。Windows 对应能力尚未实现。", + "notes": "2026-10-24：改为每轮一个默认折叠的过程区，包含思考、工具和中间说明；只改变呈现，不改写历史或 ACP 协议。ACP 无最终回复角色，按最后一条思考或工具的位置保留末尾正文；后续活动到达时才将先前正文移入过程。 2026-10-24：macOS 增加工具图片缩略图、悬停预览、放大查看器和后台任务通知状态行（#1462），Windows 待对齐。" } From 5682603c9a5e030ea48e2c82ddd7566ec404ad60 Mon Sep 17 00:00:00 2001 From: Yao Jingxi <23722032@bjtu.edu.cn> Date: Sat, 24 Oct 2026 22:33:57 +0800 Subject: [PATCH 2/8] =?UTF-8?q?feat(macos):=20=E5=9B=BE=E7=89=87=E6=94=BE?= =?UTF-8?q?=E5=A4=A7=E6=94=AF=E6=8C=81=E7=BC=A9=E6=94=BE=EF=BC=8C=E6=8A=98?= =?UTF-8?q?=E5=8F=A0=E8=BF=87=E7=A8=8B=E6=98=BE=E7=A4=BA=E7=BC=A9=E7=95=A5?= =?UTF-8?q?=E5=9B=BE=EF=BC=8C=E9=80=82=E9=85=8D=E6=9B=B4=E5=A4=9A=20Claude?= =?UTF-8?q?=20Code=20=E7=89=B9=E6=AE=8A=E6=B6=88=E6=81=AF=E6=A0=BC?= =?UTF-8?q?=E5=BC=8F=20(#1462)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...ranscript-images-and-task-notifications.md | 11 ++ macos/Resources/en.lproj/Localizable.strings | 6 + .../zh-Hans.lproj/Localizable.strings | 6 + .../Lithe/Views/Agent/AgentImageViews.swift | 66 ++++++- .../Views/Agent/AgentProcessGroupView.swift | 47 +++-- .../Views/Agent/AgentTranscriptView.swift | 124 ++++++++++-- .../Application/AgentImage.swift | 187 ++++++++++++++++-- macos/Tests/LitheTests/AgentImageTests.swift | 72 ++++++- shared/i18n/locales/en-US.json | 3 + shared/i18n/locales/zh-CN.json | 3 + shared/i18n/messages.json | 38 ++++ .../agent-tool-activity-timeline.json | 4 +- 12 files changed, 519 insertions(+), 48 deletions(-) diff --git a/.agents/notes/implemented/feature/2026-10-24-agent-transcript-images-and-task-notifications.md b/.agents/notes/implemented/feature/2026-10-24-agent-transcript-images-and-task-notifications.md index 0d8cbc182d..6ad2fd12ec 100644 --- a/.agents/notes/implemented/feature/2026-10-24-agent-transcript-images-and-task-notifications.md +++ b/.agents/notes/implemented/feature/2026-10-24-agent-transcript-images-and-task-notifications.md @@ -26,6 +26,17 @@ ACP（Agent Client Protocol，Agent 与客户端之间的协议）的工具结 - 缩略图在视图层用 ImageIO 在后台线程按需生成并放入 `NSCache`；历史回放只保存数据，不解码。 悬停预览复用 `workbenchHoverCard`，放大查看器是转录区域上的覆盖层，状态放在 `AgentImageViewerState`，行只持有引用，打开或关闭不会重绘全部行。 +- 放大查看器仍是转录区域上的覆盖层，在其上补充捏合缩放（相对适应窗口 25%–800%）、拖动、双击、 + 百分比和 ⌘+/⌘−/⌘0；侧栏很窄时可放大查看。曾试过改用独立窗口复用编辑器的 `MediaViewerView`， + 因为偏离了既有交互而放弃。 +- 折叠的“思考过程”标题行也显示图片缩略图，否则默认折叠时用户看不到工具返回的图片。 +- 用户消息里的 `[@名字](file:///路径)`、裸 `file://` 地址、`📎 路径` 和 Claude Code 的 + `[Image: source: …]` 指向图片文件时显示为缩略图；`[Image: original …]` 尺寸说明丢弃。 +- Claude Code 写进用户消息的特殊格式由 `AgentClaudeCodeMarkup` 识别：斜杠命令、本地命令输出、 + Shell 转义、中断提示、压缩续接摘要、技能展开和宿主上下文显示为状态行或折叠块； + ``、`` 等只给模型看的块在任何位置都不显示。 + 只有整条消息都是这些标记时才整体替换，用户自己输入的相似文字不会被吞掉。 + 格式清单来自对本机真实会话的标记名称统计，新增格式在这里补充。 - `` 只有在整条用户消息完全由通知块组成时才解析，用户自己输入的相同文字不会被吞掉。 这是协议标记的呈现，不按 Agent 名称在宿主里分支。 diff --git a/macos/Resources/en.lproj/Localizable.strings b/macos/Resources/en.lproj/Localizable.strings index 4b714aa017..1b9400b8b2 100644 --- a/macos/Resources/en.lproj/Localizable.strings +++ b/macos/Resources/en.lproj/Localizable.strings @@ -806,6 +806,7 @@ "Conventional Commits" = "Conventional Commits"; "Conversation" = "Conversation"; "Conversation actions" = "Conversation actions"; +"Conversation continued from a summary" = "Conversation continued from a summary"; "Conversation history" = "Conversation history"; "Conversation history is unavailable" = "Conversation history is unavailable"; "Conversation tabs" = "Conversation tabs"; @@ -1342,6 +1343,7 @@ "Find references to the selected symbol" = "Find references to the selected symbol"; "Finished" = "Finished"; "First %lld locations" = "First %lld locations"; +"Fit to Window" = "Fit to Window"; "Fix the failing checks below before installing." = "Fix the failing checks below before installing."; "Fixup" = "Fixup"; "Folder" = "Folder"; @@ -3093,6 +3095,7 @@ "Skill files" = "Skill files"; "Skill imports cannot contain symbolic links or special files." = "Skill imports cannot contain symbolic links or special files."; "Skill instructions" = "Skill instructions"; +"Skill loaded: %@" = "Skill loaded: %@"; "Skill name (lowercase-with-hyphens)" = "Skill name (lowercase-with-hyphens)"; "Skill project" = "Skill project"; "Skill saved. Refresh or reconnect the Agent if it does not appear in the conversation." = "Skill saved. Refresh or reconnect the Agent if it does not appear in the conversation."; @@ -4618,6 +4621,7 @@ "native.conventionalCommits" = "Conventional Commits"; "native.conversation" = "Conversation"; "native.conversationActions" = "Conversation actions"; +"native.conversationContinuedFromASummary" = "Conversation continued from a summary"; "native.conversationHistory" = "Conversation history"; "native.conversationHistoryIsUnavailable" = "Conversation history is unavailable"; "native.conversationTabs" = "Conversation tabs"; @@ -5167,6 +5171,7 @@ "native.finished" = "Finished"; "native.finishedWithoutCreating" = "%1$@ finished without creating %2$@."; "native.firstLldLocations" = "First %1$lld locations"; +"native.fitToWindow" = "Fit to Window"; "native.fixTheFailingChecksBelowBeforeInstalling" = "Fix the failing checks below before installing."; "native.fixup" = "Fixup"; "native.folder" = "Folder"; @@ -7084,6 +7089,7 @@ "native.skillFiles" = "Skill files"; "native.skillImportsCannotContainSymbolicLinksOrSpecialFiles" = "Skill imports cannot contain symbolic links or special files."; "native.skillInstructions" = "Skill instructions"; +"native.skillLoaded" = "Skill loaded: %1$@"; "native.skillMdExceedsThe128KibLimit" = "SKILL.md exceeds the 128 KiB limit."; "native.skillMdFrontmatterIsNotClosed" = "SKILL.md frontmatter is not closed."; "native.skillMdHasInvalidYamlFrontmatter" = "SKILL.md has invalid YAML frontmatter."; diff --git a/macos/Resources/zh-Hans.lproj/Localizable.strings b/macos/Resources/zh-Hans.lproj/Localizable.strings index 775413b6f3..dcdc5661ab 100644 --- a/macos/Resources/zh-Hans.lproj/Localizable.strings +++ b/macos/Resources/zh-Hans.lproj/Localizable.strings @@ -806,6 +806,7 @@ "Conventional Commits" = "Conventional Commits 规范"; "Conversation" = "讨论"; "Conversation actions" = "会话操作"; +"Conversation continued from a summary" = "对话已从摘要继续"; "Conversation history" = "对话历史"; "Conversation history is unavailable" = "当前无法读取对话历史"; "Conversation tabs" = "会话标签"; @@ -1342,6 +1343,7 @@ "Find references to the selected symbol" = "查找所选符号的引用"; "Finished" = "已完成"; "First %lld locations" = "前 %lld 个位置"; +"Fit to Window" = "适应窗口"; "Fix the failing checks below before installing." = "请先解决下方未通过的检测项，再进行安装。"; "Fixup" = "合并并丢弃消息"; "Folder" = "分组"; @@ -3093,6 +3095,7 @@ "Skill files" = "技能文件"; "Skill imports cannot contain symbolic links or special files." = "导入的技能不能包含符号链接或特殊文件。"; "Skill instructions" = "技能正文"; +"Skill loaded: %@" = "已加载技能：%@"; "Skill name (lowercase-with-hyphens)" = "技能名称（小写字母与连字符）"; "Skill project" = "当前项目"; "Skill saved. Refresh or reconnect the Agent if it does not appear in the conversation." = "技能已保存。如果对话中未显示，请刷新或重新连接 Agent。"; @@ -4618,6 +4621,7 @@ "native.conventionalCommits" = "Conventional Commits 规范"; "native.conversation" = "讨论"; "native.conversationActions" = "会话操作"; +"native.conversationContinuedFromASummary" = "对话已从摘要继续"; "native.conversationHistory" = "对话历史"; "native.conversationHistoryIsUnavailable" = "当前无法读取对话历史"; "native.conversationTabs" = "会话标签"; @@ -5167,6 +5171,7 @@ "native.finished" = "已完成"; "native.finishedWithoutCreating" = "%1$@ 已结束，但没有生成 %2$@。"; "native.firstLldLocations" = "前 %1$lld 个位置"; +"native.fitToWindow" = "适应窗口"; "native.fixTheFailingChecksBelowBeforeInstalling" = "请先解决下方未通过的检测项，再进行安装。"; "native.fixup" = "合并并丢弃消息"; "native.folder" = "分组"; @@ -7084,6 +7089,7 @@ "native.skillFiles" = "技能文件"; "native.skillImportsCannotContainSymbolicLinksOrSpecialFiles" = "导入的技能不能包含符号链接或特殊文件。"; "native.skillInstructions" = "技能正文"; +"native.skillLoaded" = "已加载技能：%1$@"; "native.skillMdExceedsThe128KibLimit" = "SKILL.md 超过了 128 KiB 的大小限制。"; "native.skillMdFrontmatterIsNotClosed" = "SKILL.md 的文件头缺少结束标记。"; "native.skillMdHasInvalidYamlFrontmatter" = "SKILL.md 的 YAML 文件头格式不正确。"; diff --git a/macos/Sources/Lithe/Views/Agent/AgentImageViews.swift b/macos/Sources/Lithe/Views/Agent/AgentImageViews.swift index 7f41fc2073..d7a35f1bfb 100644 --- a/macos/Sources/Lithe/Views/Agent/AgentImageViews.swift +++ b/macos/Sources/Lithe/Views/Agent/AgentImageViews.swift @@ -189,6 +189,12 @@ struct AgentImageViewer: View { @Environment(\.locale) private var locale @ObservedObject var state: AgentImageViewerState let gallery: AgentImageGallery + /// Relative to the fitted size, so 100% always shows the whole picture. + @State private var zoom: CGFloat = 1 + @State private var pinchBase: CGFloat = 1 + @State private var pan: CGSize = .zero + @State private var panBase: CGSize = .zero + private static let zoomRange: ClosedRange = 0.25...8 var body: some View { let image = gallery.images[gallery.index] @@ -198,9 +204,21 @@ struct AgentImageViewer: View { .onTapGesture { state.gallery = nil } VStack(spacing: 10) { AgentImageThumbnail(image: image, maxPixel: 2400, fills: false) + .scaleEffect(zoom) + .offset(pan) .frame(maxWidth: .infinity, maxHeight: .infinity) + .clipped() .shadow(color: .black.opacity(0.35), radius: 12) - .allowsHitTesting(false) + .contentShape(Rectangle()) + .gesture(MagnificationGesture() + .onChanged { setZoom(pinchBase * $0) } + .onEnded { _ in pinchBase = zoom }) + .simultaneousGesture(DragGesture() + .onChanged { if zoom > 1 { pan = CGSize(width: panBase.width + $0.translation.width, + height: panBase.height + $0.translation.height) } } + .onEnded { _ in panBase = pan }) + .onTapGesture(count: 2) { withAnimation(.none) { zoom > 1 ? resetZoom() : setZoom(2) } } + .onChange(of: gallery.index) { _ in resetZoom() } HStack(spacing: 8) { Text(verbatim: image.title ?? "") .lineLimit(1).truncationMode(.middle) @@ -212,6 +230,7 @@ struct AgentImageViewer: View { } .font(LitheTheme.uiFont(size: 12.5)) .foregroundStyle(.white) + zoomControls } .padding(.horizontal, 56) .padding(.vertical, 40) @@ -249,6 +268,51 @@ struct AgentImageViewer: View { .accessibilityIdentifier("agent-image-viewer") } + private var zoomControls: some View { + HStack(spacing: 6) { + zoomButton("minus", help: "Zoom Out", key: "-") { setZoom(zoom / 1.25) } + Button { resetZoom() } label: { + Text(verbatim: "\(Int((zoom * 100).rounded()))%") + .font(LitheTheme.uiFont(size: 12, weight: .medium)).monospacedDigit() + .frame(minWidth: 48, minHeight: 24) + .background(.white.opacity(0.16), in: Capsule()) + .contentShape(Capsule()) + } + .buttonStyle(.litheNoPress) + .foregroundStyle(.white) + .keyboardShortcut("0", modifiers: .command) + .lithePointer() + .help("Fit to Window") + zoomButton("plus", help: "Zoom In", key: "=") { setZoom(zoom * 1.25) } + } + } + + private func zoomButton(_ systemImage: String, help: LocalizedStringKey, key: KeyEquivalent, + action: @escaping () -> Void) -> some View { + Button(action: action) { + Image(systemName: systemImage).font(LitheTheme.uiFont(size: 11, weight: .semibold)) + .frame(width: 24, height: 24) + .background(.white.opacity(0.16), in: Circle()) + .contentShape(Circle()) + } + .buttonStyle(.litheNoPress) + .foregroundStyle(.white) + .keyboardShortcut(key, modifiers: .command) + .lithePointer() + .help(help) + .accessibilityLabel(help) + } + + private func setZoom(_ value: CGFloat) { + zoom = min(max(value, Self.zoomRange.lowerBound), Self.zoomRange.upperBound) + pinchBase = zoom + if zoom <= 1 { pan = .zero; panBase = .zero } + } + + private func resetZoom() { + zoom = 1; pinchBase = 1; pan = .zero; panBase = .zero + } + private func step(_ delta: Int, systemImage: String, key: KeyEquivalent, label: LocalizedStringKey) -> some View { Button { let count = gallery.images.count diff --git a/macos/Sources/Lithe/Views/Agent/AgentProcessGroupView.swift b/macos/Sources/Lithe/Views/Agent/AgentProcessGroupView.swift index a8fbaeec30..c847c6d86c 100644 --- a/macos/Sources/Lithe/Views/Agent/AgentProcessGroupView.swift +++ b/macos/Sources/Lithe/Views/Agent/AgentProcessGroupView.swift @@ -15,26 +15,39 @@ struct AgentProcessGroupView: View { expansion.isExpanded(isStreaming: isStreaming, isSearching: !searchText.isEmpty) } + private var headerButton: some View { + Button { expansion.toggle(isStreaming: isStreaming, isSearching: !searchText.isEmpty) } label: { + HStack(spacing: 5) { + if isStreaming { ProgressView().controlSize(.mini) } + Text("Thinking process").lineLimit(1) + Image(systemName: "chevron.down") + .font(LitheTheme.uiFont(size: 9, weight: .semibold)) + .rotationEffect(.degrees(isExpanded ? 0 : -90)) + } + .font(LitheTheme.uiFont(size: 12.5)) + .foregroundStyle(LitheTheme.secondaryText) + .padding(.vertical, 2) + .contentShape(Rectangle()) + } + .buttonStyle(.litheNoPress) + .lithePointer() + .help(isExpanded ? "Hide thinking" : "Show thinking") + .accessibilityValue(isExpanded ? AgentLocalization.string("Expanded", locale: locale) : AgentLocalization.string("Collapsed", locale: locale)) + .accessibilityIdentifier("agent-process-group-header") + } + + private var images: [AgentImage] { + messages.flatMap(\.toolDetails.images) + } + var body: some View { VStack(alignment: .leading, spacing: 8) { - Button { expansion.toggle(isStreaming: isStreaming, isSearching: !searchText.isEmpty) } label: { - HStack(spacing: 5) { - if isStreaming { ProgressView().controlSize(.mini) } - Text("Thinking process").lineLimit(1) - Image(systemName: "chevron.down") - .font(LitheTheme.uiFont(size: 9, weight: .semibold)) - .rotationEffect(.degrees(isExpanded ? 0 : -90)) - } - .font(LitheTheme.uiFont(size: 12.5)) - .foregroundStyle(LitheTheme.secondaryText) - .padding(.vertical, 2) - .contentShape(Rectangle()) + HStack(spacing: 8) { + headerButton + Spacer(minLength: 8) + // The process is collapsed by default; pictures the tools returned stay visible beside it; expanded, each tool group shows its own. + if !isExpanded && !images.isEmpty { AgentImageStrip(images: images) } } - .buttonStyle(.litheNoPress) - .lithePointer() - .help(isExpanded ? "Hide thinking" : "Show thinking") - .accessibilityValue(isExpanded ? AgentLocalization.string("Expanded", locale: locale) : AgentLocalization.string("Collapsed", locale: locale)) - .accessibilityIdentifier("agent-process-group-header") if isExpanded { VStack(alignment: .leading, spacing: 8) { diff --git a/macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift b/macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift index d6f9ae8a4c..acbac4690a 100644 --- a/macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift +++ b/macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift @@ -75,6 +75,10 @@ enum AgentTranscriptItem: Identifiable { pending.removeAll(keepingCapacity: true) } for message in messages { + if message.role == .user, AgentClaudeCodeMarkup.parse(message.text)?.isEmpty == true { + // Model-only reminders and caveats are not conversation. + continue + } if message.role == .user { flush() items.append(.message(message)) @@ -543,10 +547,10 @@ private struct AgentMessageRow: View { var body: some View { switch message.role { case .user: - if let notifications = AgentTaskNotification.parse(message.text) { + if let entries = AgentClaudeCodeMarkup.parse(message.text) { VStack(alignment: .leading, spacing: 4) { - ForEach(Array(notifications.enumerated()), id: \.offset) { _, notification in - AgentTaskNotificationRow(notification: notification) + ForEach(Array(entries.enumerated()), id: \.offset) { _, entry in + AgentClaudeCodeEntryRow(entry: entry) } } .frame(maxWidth: .infinity, alignment: .leading) @@ -576,7 +580,7 @@ private struct AgentMessageRow: View { /// Attached image files and replayed pictures show as thumbnails that open /// the viewer; the text keeps the user's words and non-image attachments. private var userBubble: some View { - let content = AgentUserMessageContent.parse(message.text) + let content = AgentUserMessageContent.parse(AgentClaudeCodeMarkup.removingModelOnlyBlocks(message.text)) let images = message.images + content.images return HStack(alignment: .bottom, spacing: 4) { Spacer(minLength: 40) @@ -609,26 +613,114 @@ private struct AgentMessageRow: View { } } -/// A Claude Code background-task report, shown as a quiet status line instead of the raw markup. -private struct AgentTaskNotificationRow: View { +/// Claude Code's own markup in a user message, shown as a quiet status line or output block. +private struct AgentClaudeCodeEntryRow: View { @Environment(\.locale) private var locale - let notification: AgentTaskNotification - - private var failed: Bool { ["failed", "error", "killed", "stopped"].contains(notification.status?.lowercased() ?? "") } + let entry: AgentClaudeCodeEntry var body: some View { + switch entry { + case .taskNotification(let notification): + let failed = ["failed", "error", "killed", "stopped"].contains(notification.status?.lowercased() ?? "") + line(systemImage: failed ? "exclamationmark.circle" : "checkmark.circle", isError: failed, + text: notification.summary ?? AgentLocalization.string( + failed ? "Background task ended" : "Background task finished", locale: locale)) + .help(notification.outputFile ?? "") + .accessibilityIdentifier("agent-task-notification") + case .command(let name, let arguments): + line(systemImage: "chevron.right.square", isError: false, + text: [name, arguments].compactMap { $0 }.joined(separator: " "), monospaced: true) + .accessibilityIdentifier("agent-command-entry") + case .shell(let command): + line(systemImage: "terminal", isError: false, text: command, monospaced: true) + .accessibilityIdentifier("agent-shell-entry") + case .output(let text, let isError): + Text(verbatim: text) + .font(LitheTheme.uiFont(size: 12, design: .monospaced)) + .foregroundStyle(isError ? LitheTheme.error : LitheTheme.secondaryText) + .lineLimit(12) + .textSelection(.enabled) + .padding(.horizontal, 10) + .padding(.vertical, 6) + .frame(maxWidth: .infinity, alignment: .leading) + .background(LitheTheme.raised.opacity(0.5), in: RoundedRectangle(cornerRadius: 8)) + .accessibilityIdentifier("agent-command-output") + case .interrupted(let text): + line(systemImage: "pause.circle", isError: false, text: text) + .accessibilityIdentifier("agent-interrupted-entry") + case .summary(let text, let tools): + AgentCollapsedBlock(title: AgentLocalization.string("Conversation continued from a summary", locale: locale), + systemImage: "text.append", detail: tools, text: text) + .accessibilityIdentifier("agent-summary-entry") + case .skill(let name, let body): + AgentCollapsedBlock(title: String(format: AgentLocalization.string("Skill loaded: %@", locale: locale), name), + systemImage: "sparkles", detail: nil, text: body) + .accessibilityIdentifier("agent-skill-entry") + case .context(let name, let body): + AgentCollapsedBlock(title: String(format: AgentLocalization.string("Context: %@", locale: locale), name), + systemImage: "tray.full", detail: nil, text: body) + .accessibilityIdentifier("agent-context-entry") + } + } + + private func line(systemImage: String, isError: Bool, text: String, monospaced: Bool = false) -> some View { HStack(spacing: 6) { - Image(systemName: failed ? "exclamationmark.circle" : "checkmark.circle") + Image(systemName: systemImage) .font(LitheTheme.uiFont(size: 11)) - .foregroundStyle(failed ? LitheTheme.error : LitheTheme.tertiaryText) - Text(verbatim: notification.summary ?? AgentLocalization.string( - failed ? "Background task ended" : "Background task finished", locale: locale)) + .foregroundStyle(isError ? LitheTheme.error : LitheTheme.tertiaryText) + Text(verbatim: text) + .font(monospaced ? LitheTheme.uiFont(size: 12, design: .monospaced) : LitheTheme.uiFont(size: 12.5)) .lineLimit(2) } - .font(LitheTheme.uiFont(size: 12.5)) .foregroundStyle(LitheTheme.secondaryText) - .help(notification.outputFile ?? "") .textSelection(.enabled) - .accessibilityIdentifier("agent-task-notification") + } +} + +/// Long Claude Code-owned text (recap, skill instructions, host context) behind a one-line header. +private struct AgentCollapsedBlock: View { + let title: String + let systemImage: String + let detail: String? + let text: String + @State private var expanded = false + + var body: some View { + VStack(alignment: .leading, spacing: 6) { + Button { expanded.toggle() } label: { + HStack(spacing: 6) { + Image(systemName: systemImage) + .font(LitheTheme.uiFont(size: 11)) + .foregroundStyle(LitheTheme.tertiaryText) + Text(verbatim: title).lineLimit(1) + Image(systemName: "chevron.down") + .font(LitheTheme.uiFont(size: 9, weight: .semibold)) + .rotationEffect(.degrees(expanded ? 0 : -90)) + } + .font(LitheTheme.uiFont(size: 12.5)) + .foregroundStyle(LitheTheme.secondaryText) + .contentShape(Rectangle()) + } + .buttonStyle(.litheNoPress) + .lithePointer() + .accessibilityValue(expanded ? Text("Expanded") : Text("Collapsed")) + if expanded { + VStack(alignment: .leading, spacing: 6) { + if let detail { + Text(verbatim: detail) + .font(LitheTheme.uiFont(size: 11, design: .monospaced)) + .foregroundStyle(LitheTheme.tertiaryText) + } + AgentMarkdownMessage(text: text, size: 12) + .equatable() + .foregroundStyle(LitheTheme.secondaryText) + } + .padding(.horizontal, 10) + .padding(.vertical, 8) + .frame(maxWidth: .infinity, alignment: .leading) + .background(LitheTheme.raised.opacity(0.5), in: RoundedRectangle(cornerRadius: 8)) + .textSelection(.enabled) + } + } } } diff --git a/macos/Sources/LitheAgentConversationModule/Application/AgentImage.swift b/macos/Sources/LitheAgentConversationModule/Application/AgentImage.swift index 110392058f..93c7a6f9f2 100644 --- a/macos/Sources/LitheAgentConversationModule/Application/AgentImage.swift +++ b/macos/Sources/LitheAgentConversationModule/Application/AgentImage.swift @@ -74,25 +74,68 @@ public struct AgentUserMessageContent: Equatable, Sendable { private static let attachmentMarker = "📎 " - /// Composer prompts list attachments as `📎 /path` lines. Image files become - /// thumbnails; other attachments and the user's own words stay as text. + /// `[@name](file:///path.png)` as Claude Code replays a referenced file, or a bare `file://` URL. + private static let fileLink = try! NSRegularExpression( + pattern: #"(?:\[[^\]\n]*\]\((file://[^)\n]+)\)|(file://[^\s)\]]+))"#) + + /// Claude Code's image placeholders: `[Image: source: /path]` names the file, + /// `[Image: original 3840x2486, displayed at ...]` only helps the model map coordinates. + private static let imagePlaceholder = try! NSRegularExpression( + pattern: #"\[Image: (?:source: ([^\]\n]+)|original [^\]\n]*)\]"#) + + /// Image attachments become thumbnails; other attachments and the user's own + /// words stay as text. Two shapes occur: the composer's `📎 /path` lines, and + /// file links (`[@name](file:///path)`, bare `file://` URLs) that an agent + /// replays for a referenced file, which would otherwise show as raw addresses. public static func parse(_ message: String) -> Self { - guard message.contains(attachmentMarker) else { return Self(text: message, images: []) } + guard message.contains(attachmentMarker) || message.contains("file://") || message.contains("[Image: ") else { + return Self(text: message, images: []) + } var images: [AgentImage] = [] + var seen = Set() + func add(_ path: String) -> Bool { + guard path.hasPrefix("/"), AgentImage.isImageFile(path), images.count < AgentImage.maximumPerMessage else { return false } + if seen.insert(path).inserted { images.append(AgentImage(filePath: path)) } + return true + } + func path(ofFileURL raw: String) -> String? { + let trimmed = raw.trimmingCharacters(in: .whitespaces) + if let url = URL(string: trimmed), url.isFileURL { return url.path } + // An unencoded space makes the address an invalid URL; decode it by hand. + return trimmed.hasPrefix("file://") ? String(trimmed.dropFirst("file://".count)).removingPercentEncoding : nil + } + // Links first, replacing from the end so earlier ranges stay valid. + var working = message + let ns = working as NSString + for match in fileLink.matches(in: working, range: NSRange(location: 0, length: ns.length)).reversed() { + let raw = ns.substring(with: match.range(at: match.range(at: 1).location != NSNotFound ? 1 : 2)) + guard let filePath = path(ofFileURL: raw), add(filePath), + let range = Range(match.range, in: working) else { continue } + working.replaceSubrange(range, with: "") + } + let placeholderText = working as NSString + for match in imagePlaceholder.matches(in: working, range: NSRange(location: 0, length: placeholderText.length)).reversed() { + let source = match.range(at: 1) + if source.location != NSNotFound, !add(placeholderText.substring(with: source)) { continue } + if let range = Range(match.range, in: working) { working.replaceSubrange(range, with: "") } + } var kept: [Substring] = [] - for line in message.split(separator: "\n", omittingEmptySubsequences: false) { + for line in working.split(separator: "\n", omittingEmptySubsequences: false) { if line.hasPrefix(attachmentMarker) { - let path = String(line.dropFirst(attachmentMarker.count)) - if path.hasPrefix("/"), AgentImage.isImageFile(path), images.count < AgentImage.maximumPerMessage { - images.append(AgentImage(filePath: path)) - continue - } + let value = String(line.dropFirst(attachmentMarker.count)) + if add(value) { continue } } kept.append(line) } guard !images.isEmpty else { return Self(text: message, images: []) } - let text = kept.joined(separator: "\n").trimmingCharacters(in: .whitespacesAndNewlines) - return Self(text: text, images: images) + // Pre-reversal order: the regex pass walked backwards, so restore reading order. + let ordered = images.sorted { lhs, rhs in + (message.range(of: lhs.title ?? "")?.lowerBound ?? message.endIndex) < (message.range(of: rhs.title ?? "")?.lowerBound ?? message.endIndex) + } + let text = kept.joined(separator: "\n").replacingOccurrences(of: "[ \\t]+\\n", with: "\n", options: .regularExpression) + .replacingOccurrences(of: "\n{3,}", with: "\n\n", options: .regularExpression) + .trimmingCharacters(in: .whitespacesAndNewlines) + return Self(text: text, images: ordered) } } @@ -127,3 +170,125 @@ public struct AgentTaskNotification: Equatable, Sendable { return value.isEmpty ? nil : value } } + +/// One visible item of a user message that Claude Code filled with its own markup. +public enum AgentClaudeCodeEntry: Equatable, Sendable { + case taskNotification(AgentTaskNotification) + /// A slash command the user ran, such as `/clear`, with its arguments. + case command(name: String, arguments: String?) + /// Output of a local command or shell escape; `isError` for stderr. + case output(String, isError: Bool) + /// A shell escape typed with `!`. + case shell(String) + /// `[Request interrupted by user]` and its variants. + case interrupted(String) + /// The recap Claude Code inserts when a conversation outgrows its context. + case summary(String, tools: String?) + /// A skill's instructions, expanded into the conversation when the skill is invoked. + case skill(name: String, body: String) + /// Context injected by a host application, kept collapsed. + case context(name: String, body: String) +} + +/// Claude Code stores slash commands, local command output, shell escapes, +/// compaction summaries and reminders for the model as user messages, some wrapped +/// in pseudo-XML. Those are not what the user typed, so they are rendered as quiet +/// status lines or collapsed blocks, or hidden when they only instruct the model. +public enum AgentClaudeCodeMarkup { + /// Wrappers that exist only for the model; they produce no entry. + private static let hiddenTags = ["local-command-caveat", "system-reminder", "user-prompt-submit-hook"] + /// Wrappers whose whole body is shown collapsed, with the tag as its name. + private static let contextTags: Set = ["lithe_workspace_context", "side-sessions-event", "create-pr-command"] + private static let knownTags: Set = contextTags.union([ + "task-notification", "command-name", "command-message", "command-args", + "local-command-stdout", "local-command-stderr", "bash-input", "bash-stdout", "bash-stderr", + ]) + private static let summaryPrefix = "This session is being continued from a previous conversation" + private static let skillPrefix = "Base directory for this skill:" + private static let hiddenBlock = try! NSRegularExpression( + pattern: "<(" + hiddenTags.joined(separator: "|") + ")>[\\s\\S]*?") + + /// Removes model-only blocks wherever they sit, so reminders appended to a + /// typed message never show up as part of it. + public static func removingModelOnlyBlocks(_ message: String) -> String { + guard hiddenTags.contains(where: { message.contains("<\($0)>") }) else { return message } + let range = NSRange(location: 0, length: (message as NSString).length) + return hiddenBlock.stringByReplacingMatches(in: message, range: range, withTemplate: "") + .trimmingCharacters(in: .whitespacesAndNewlines) + } + + /// Nil unless the whole message is Claude Code markup. An empty result means + /// the message is entirely model-only and should not be shown. + public static func parse(_ message: String) -> [AgentClaudeCodeEntry]? { + let stripped = removingModelOnlyBlocks(message) + let trimmed = stripped.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed.isEmpty { return stripped == message ? nil : [] } + if trimmed.hasPrefix("[Request interrupted by user"), trimmed.hasSuffix("]"), !trimmed.contains("\n") { + return [.interrupted(String(trimmed.dropFirst().dropLast()))] + } + if trimmed.hasPrefix(summaryPrefix) { return [summary(trimmed)] } + if trimmed.hasPrefix(skillPrefix) { + let first = trimmed.split(separator: "\n", maxSplits: 1).first.map(String.init) ?? trimmed + let directory = first.dropFirst(skillPrefix.count).trimmingCharacters(in: .whitespaces) + return [.skill(name: (directory as NSString).lastPathComponent, body: trimmed)] + } + var rest = Substring(trimmed) + var entries: [AgentClaudeCodeEntry] = [] + var commandName: String? + var commandArguments: String? + func flushCommand() { + if let name = commandName { entries.append(.command(name: name, arguments: commandArguments)) } + commandName = nil + commandArguments = nil + } + while !rest.isEmpty { + guard rest.hasPrefix("<"), let close = rest.firstIndex(of: ">") else { return nil } + let tag = String(rest[rest.index(after: rest.startIndex)..") else { return nil } + let body = String(rest[rest.index(after: close)..\(body)")?.first { + entries.append(.taskNotification(parsed)) + } + case "command-name": + flushCommand() + if !body.isEmpty { commandName = body } + case "command-args": + if !body.isEmpty { commandArguments = body } + case "command-message": + break + case "local-command-stdout", "bash-stdout": + flushCommand() + if !body.isEmpty { entries.append(.output(body, isError: false)) } + case "local-command-stderr", "bash-stderr": + flushCommand() + if !body.isEmpty { entries.append(.output(body, isError: true)) } + case "bash-input": + flushCommand() + if !body.isEmpty { entries.append(.shell(body)) } + default: + flushCommand() + if contextTags.contains(tag), !body.isEmpty { entries.append(.context(name: tag, body: body)) } + } + rest = rest[end.upperBound...] + while let first = rest.first, first.isWhitespace { rest = rest.dropFirst() } + } + flushCommand() + return entries + } + + /// The recap text without its `` header line. + private static func summary(_ text: String) -> AgentClaudeCodeEntry { + let open = "", close = "" + guard let start = text.range(of: open), let end = text.range(of: close, range: start.upperBound../clear + clear + now + """)) + #expect(command == [.command(name: "/clear", arguments: "now")]) + #expect(AgentClaudeCodeMarkup.parse("done") == [.output("done", isError: false)]) + #expect(AgentClaudeCodeMarkup.parse("lsaboom") + == [.shell("ls"), .output("a", isError: false), .output("boom", isError: true)]) + #expect(AgentClaudeCodeMarkup.parse("[Request interrupted by user for tool use]") == [.interrupted("Request interrupted by user for tool use")]) + } + + @Test + func modelOnlyMarkupIsHiddenAndTypedTextIsNeverTreatedAsMarkup() { + #expect(AgentClaudeCodeMarkup.parse("Caveat: ignore") == []) + #expect(AgentClaudeCodeMarkup.parse("x\n") == []) + #expect(AgentClaudeCodeMarkup.parse("what does mean?") == nil) + #expect(AgentClaudeCodeMarkup.parse("x") == nil) + #expect(AgentClaudeCodeMarkup.parse("ls trailing words") == nil) + let hidden = [AgentConversationMessage(id: "u", role: .user, text: "x")] + #expect(AgentTranscriptItem.grouped(hidden).isEmpty) + } + + @Test + func compactionSummarySkillAndHostContextAreCollapsedEntries() throws { + let recap = """ + This session is being continued from a previous conversation that ran out of context. + Bash, Edit, Read + Summary: + 1. Primary Request + """ + let summary = try #require(AgentClaudeCodeMarkup.parse(recap)) + #expect(summary == [.summary("This session is being continued from a previous conversation that ran out of context.\n\nSummary:\n1. Primary Request", tools: "Bash, Edit, Read")]) + let skill = try #require(AgentClaudeCodeMarkup.parse("Base directory for this skill: /Users/x/.claude/skills/deploy\n\n# Deploy")) + #expect(skill == [.skill(name: "deploy", body: "Base directory for this skill: /Users/x/.claude/skills/deploy\n\n# Deploy")]) + let context = try #require(AgentClaudeCodeMarkup.parse("cwd")) + #expect(context == [.context(name: "lithe_workspace_context", body: "cwd")]) + } + + @Test + func remindersAppendedToTypedTextAreRemovedWithoutTouchingTheText() { + let message = "继续\nbe brief" + #expect(AgentClaudeCodeMarkup.removingModelOnlyBlocks(message) == "继续") + #expect(AgentClaudeCodeMarkup.parse(message) == nil, "Typed text stays an ordinary message") + #expect(AgentClaudeCodeMarkup.removingModelOnlyBlocks("plain") == "plain") + } + + @Test + func claudeCodeImagePlaceholdersBecomeThumbnailsAndDropTheSizeNote() { + let content = AgentUserMessageContent.parse(""" + [Image: source: /private/tmp/claude/images/1.png] + [Image: original 3840x2486, displayed at 2000x1295. Multiply coordinates by 1.92 to map to original image.] + 看一下这个 + """) + #expect(content.images.map(\.title) == ["1.png"]) + #expect(content.text == "看一下这个") + #expect(AgentUserMessageContent.parse("[Image #1] 看这个").images.isEmpty) + } } diff --git a/shared/i18n/locales/en-US.json b/shared/i18n/locales/en-US.json index e3888ff472..539687fab6 100644 --- a/shared/i18n/locales/en-US.json +++ b/shared/i18n/locales/en-US.json @@ -8772,6 +8772,7 @@ "native.connectionTestResult": "Connection test result", "native.connectionTimeout": "Connection timeout", "native.conversationActions": "Conversation actions", + "native.conversationContinuedFromASummary": "Conversation continued from a summary", "native.couldNotOpenTheRemoteFolder": "Could not open the remote folder: {arg1}", "native.couldNotSaveTheConnections": "Could not save the connections: {arg1}", "native.createGitRepository": "Create Git repository", @@ -8803,6 +8804,7 @@ "native.filePatterns": "File patterns", "native.findInMarketplace": "Find in Marketplace", "native.finishedWithoutCreating": "{arg1} finished without creating {arg2}.", + "native.fitToWindow": "Fit to Window", "native.folder": "Folder", "native.folderNameIsRequired": "Folder name is required.", "native.forExampleNotify": "For example, notify", @@ -8931,6 +8933,7 @@ "native.shellCommands": "Shell commands", "native.showLldEarlierItems": "Show {arg1} earlier items", "native.showMore": "Show more", + "native.skillLoaded": "Skill loaded: {arg1}", "native.templateEngines": "Template Engines", "native.testConnection": "Test Connection", "native.testTheConnectionToSeeTheHostSSystemResourcesAndAddress": "Test the connection to see the host's system, resources, and address.", diff --git a/shared/i18n/locales/zh-CN.json b/shared/i18n/locales/zh-CN.json index b89b4196f3..52e6300186 100644 --- a/shared/i18n/locales/zh-CN.json +++ b/shared/i18n/locales/zh-CN.json @@ -8772,6 +8772,7 @@ "native.connectionTestResult": "连接测试结果", "native.connectionTimeout": "连接超时", "native.conversationActions": "会话操作", + "native.conversationContinuedFromASummary": "对话已从摘要继续", "native.couldNotOpenTheRemoteFolder": "无法打开远程文件夹：{arg1}", "native.couldNotSaveTheConnections": "无法保存连接：{arg1}", "native.createGitRepository": "创建 Git 仓库", @@ -8803,6 +8804,7 @@ "native.filePatterns": "文件模式", "native.findInMarketplace": "在插件市场中查找", "native.finishedWithoutCreating": "{arg1} 已结束，但没有生成 {arg2}。", + "native.fitToWindow": "适应窗口", "native.folder": "分组", "native.folderNameIsRequired": "请输入文件夹名称。", "native.forExampleNotify": "例如 notify", @@ -8931,6 +8933,7 @@ "native.shellCommands": "Shell 命令", "native.showLldEarlierItems": "显示更早的 {arg1} 项", "native.showMore": "显示更多", + "native.skillLoaded": "已加载技能：{arg1}", "native.templateEngines": "模板引擎", "native.testConnection": "测试连接", "native.testTheConnectionToSeeTheHostSSystemResourcesAndAddress": "测试连接后，这里会显示主机的系统、资源和地址。", diff --git a/shared/i18n/messages.json b/shared/i18n/messages.json index 7e351dd6d2..085292a1d5 100644 --- a/shared/i18n/messages.json +++ b/shared/i18n/messages.json @@ -81076,6 +81076,17 @@ "arguments": [] } }, + { + "id": "native.conversationContinuedFromASummary", + "description": "Native UI: Conversation continued from a summary", + "platforms": [ + "macos" + ], + "macos": { + "key": "Conversation continued from a summary", + "arguments": [] + } + }, { "id": "native.couldNotOpenTheRemoteFolder", "description": "Native UI: Could not open the remote folder: %@", @@ -81485,6 +81496,17 @@ "arg2": "string" } }, + { + "id": "native.fitToWindow", + "description": "Native UI: Fit to Window", + "platforms": [ + "macos" + ], + "macos": { + "key": "Fit to Window", + "arguments": [] + } + }, { "id": "native.folder", "description": "Native UI: Folder", @@ -82994,6 +83016,22 @@ "arguments": [] } }, + { + "id": "native.skillLoaded", + "description": "Native UI: Skill loaded: %@", + "platforms": [ + "macos" + ], + "macos": { + "key": "Skill loaded: %@", + "arguments": [ + "arg1" + ] + }, + "parameters": { + "arg1": "string" + } + }, { "id": "native.templateEngines", "description": "Native UI: Template Engines", diff --git a/shared/platform-feature-matrix/features/agent-tool-activity-timeline.json b/shared/platform-feature-matrix/features/agent-tool-activity-timeline.json index 96794b8fcb..87cdac2ae2 100644 --- a/shared/platform-feature-matrix/features/agent-tool-activity-timeline.json +++ b/shared/platform-feature-matrix/features/agent-tool-activity-timeline.json @@ -33,6 +33,6 @@ "verificationStatus": "pending" }, "owner": "Agent", - "verification": "macOS 与 Windows：构造思考→过程说明→读文件→思考→执行命令→最终回复，确认每轮只有一个过程入口且默认收起，流式也不自动展开；收起时过程文字与工具行不占高度，最终回复始终可读。展开按原顺序呈现，只有相邻工具合并，状态与证据可继续查看。纯空白思考、过程说明不产生空行，没有思考和工具的直接回答不产生入口；新用户消息、已记录的轮次结束分别切断分组，工具或思考结束而没有最终回复时不伪造回复。搜索过程文字及工具输入、输出、文件路径，匹配组临时展开且不能在搜索期间收起，清空后恢复手动选择；切换和关闭会话验证偏好隔离与释放。深浅主题、窄宽面板、历史恢复和实时流均验证；历史导出、文件审阅、权限卡片、轮次统计和回复复制继续可用。单元测试与组件回放不替代真实 Agent 和 Windows Tauri 运行验收，未执行部分保持 pending。 macOS 图片：让 Agent 读取多张图片，确认缩略图最多三张并显示 +N，悬停出现带标题和序号的预览，点击进入放大查看并可用方向键切换、Esc 关闭；用户粘贴图片后气泡显示缩略图且可放大；触发后台任务后不再出现原始 task-notification 标记。Windows 对应能力尚未实现。", - "notes": "2026-10-24：改为每轮一个默认折叠的过程区，包含思考、工具和中间说明；只改变呈现，不改写历史或 ACP 协议。ACP 无最终回复角色，按最后一条思考或工具的位置保留末尾正文；后续活动到达时才将先前正文移入过程。 2026-10-24：macOS 增加工具图片缩略图、悬停预览、放大查看器和后台任务通知状态行（#1462），Windows 待对齐。" + "verification": "macOS 与 Windows：构造思考→过程说明→读文件→思考→执行命令→最终回复，确认每轮只有一个过程入口且默认收起，流式也不自动展开；收起时过程文字与工具行不占高度，最终回复始终可读。展开按原顺序呈现，只有相邻工具合并，状态与证据可继续查看。纯空白思考、过程说明不产生空行，没有思考和工具的直接回答不产生入口；新用户消息、已记录的轮次结束分别切断分组，工具或思考结束而没有最终回复时不伪造回复。搜索过程文字及工具输入、输出、文件路径，匹配组临时展开且不能在搜索期间收起，清空后恢复手动选择；切换和关闭会话验证偏好隔离与释放。深浅主题、窄宽面板、历史恢复和实时流均验证；历史导出、文件审阅、权限卡片、轮次统计和回复复制继续可用。单元测试与组件回放不替代真实 Agent 和 Windows Tauri 运行验收，未执行部分保持 pending。 macOS 图片：让 Agent 读取多张图片，确认缩略图最多三张并显示 +N，悬停出现带标题和序号的预览，点击进入放大查看并可用方向键切换、Esc 关闭；用户粘贴图片后气泡显示缩略图且可放大；触发后台任务后不再出现原始 task-notification 标记。Windows 对应能力尚未实现。 点击缩略图后的大图支持双指缩放、拖动、百分比与 ⌘+/⌘−/⌘0；用户消息里的 file:// 链接与 [Image: source:] 占位符显示为缩略图；Claude Code 的斜杠命令、本地命令输出、Shell 转义、压缩续接摘要、技能展开和宿主上下文显示为状态行或折叠块，只给模型看的提醒不显示。", + "notes": "2026-10-24：改为每轮一个默认折叠的过程区，包含思考、工具和中间说明；只改变呈现，不改写历史或 ACP 协议。ACP 无最终回复角色，按最后一条思考或工具的位置保留末尾正文；后续活动到达时才将先前正文移入过程。 2026-10-24：macOS 增加工具图片缩略图、悬停预览、放大查看器和后台任务通知状态行（#1462），Windows 待对齐。 2026-10-24（续）：折叠的思考过程标题行也显示图片缩略图；放大查看器增加缩放；识别更多 Claude Code 特殊消息格式。" } From b6f859eeaeb3ea8dda5c9cebea0db8b08028f857 Mon Sep 17 00:00:00 2001 From: Yao Jingxi <23722032@bjtu.edu.cn> Date: Sat, 24 Oct 2026 22:38:55 +0800 Subject: [PATCH 3/8] =?UTF-8?q?fix(macos):=20=E5=AF=BC=E8=88=AA=E6=A0=8F?= =?UTF-8?q?=E4=B8=8D=E5=86=8D=E6=8A=8A=20Claude=20Code=20=E7=89=B9?= =?UTF-8?q?=E6=AE=8A=E6=B6=88=E6=81=AF=E5=BD=93=E4=BD=9C=E6=8F=90=E9=97=AE?= =?UTF-8?q?=EF=BC=8C=E5=9B=BE=E7=89=87=E6=9F=A5=E7=9C=8B=E5=99=A8=E4=B8=8D?= =?UTF-8?q?=E5=86=8D=E8=AE=A9=E8=BD=AC=E5=BD=95=E9=87=8D=E7=BB=98=20(#1462?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Views/Agent/AgentTranscriptNavigation.swift | 3 ++- .../Lithe/Views/Agent/AgentTranscriptView.swift | 11 +++++++---- macos/Tests/LitheTests/AgentImageTests.swift | 13 +++++++++++++ 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/macos/Sources/Lithe/Views/Agent/AgentTranscriptNavigation.swift b/macos/Sources/Lithe/Views/Agent/AgentTranscriptNavigation.swift index 0f5772b21e..cb0548cb9f 100644 --- a/macos/Sources/Lithe/Views/Agent/AgentTranscriptNavigation.swift +++ b/macos/Sources/Lithe/Views/Agent/AgentTranscriptNavigation.swift @@ -16,7 +16,8 @@ struct AgentTranscriptNavigationContent { init(messages: [AgentConversationMessage]) { for message in messages { - if message.role == .user { + // Claude Code's own markup (commands, notifications, recaps) is not a prompt the user wrote. + if message.role == .user, AgentClaudeCodeMarkup.parse(message.text) == nil { entries.append(Entry(id: message.id, prompt: String(message.text.prefix(320)))) } guard let turnID = entries.last?.id else { continue } diff --git a/macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift b/macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift index acbac4690a..73c20c1d24 100644 --- a/macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift +++ b/macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift @@ -167,7 +167,8 @@ struct AgentTranscriptView: View { var isEditorLayout = false @State private var navigation = AgentTranscriptNavigationState() @State private var showsAgentPicker = false - @StateObject private var imageViewer = AgentImageViewerState() + // A reference the rows share; only the overlay observes it, so opening an image does not re-evaluate this view. + @State private var imageViewer = AgentImageViewerState() // A filtered-out row can be destroyed. Keep its preference in the owning // transcript view, isolated by session and message, until the actual data goes away. @State private var thoughtExpansions: [String: [String: AgentThoughtExpansion]] = [:] @@ -584,9 +585,11 @@ private struct AgentMessageRow: View { let images = message.images + content.images return HStack(alignment: .bottom, spacing: 4) { Spacer(minLength: 40) - AgentCopyButton(text: content.text, help: "Copy message") - .opacity(isHovered ? 1 : 0) - .allowsHitTesting(isHovered) + if !content.text.isEmpty { + AgentCopyButton(text: content.text, help: "Copy message") + .opacity(isHovered ? 1 : 0) + .allowsHitTesting(isHovered) + } VStack(alignment: .trailing, spacing: 6) { if !images.isEmpty { AgentUserImages(images: images) } if !content.text.isEmpty { Text(content.text) } diff --git a/macos/Tests/LitheTests/AgentImageTests.swift b/macos/Tests/LitheTests/AgentImageTests.swift index 80b589bbd4..e361c5312b 100644 --- a/macos/Tests/LitheTests/AgentImageTests.swift +++ b/macos/Tests/LitheTests/AgentImageTests.swift @@ -167,4 +167,17 @@ struct AgentImageTests { #expect(content.text == "看一下这个") #expect(AgentUserMessageContent.parse("[Image #1] 看这个").images.isEmpty) } + + @Test + func claudeCodeMarkupMessagesDoNotBecomeNavigationPrompts() { + let messages = [ + AgentConversationMessage(id: "caveat", role: .user, text: "x"), + AgentConversationMessage(id: "real", role: .user, text: "hello"), + AgentConversationMessage(id: "note", role: .user, text: "completed"), + AgentConversationMessage(id: "reply", role: .agent, text: "hi"), + ] + let content = AgentTranscriptNavigationContent(messages: messages) + #expect(content.entries.map(\.id) == ["real"]) + #expect(content.turnByMessageID["reply"] == "real") + } } From 645172372eb3d7836b66e90a2f890e2449d051a1 Mon Sep 17 00:00:00 2001 From: Yao Jingxi <23722032@bjtu.edu.cn> Date: Sat, 24 Oct 2026 22:54:48 +0800 Subject: [PATCH 4/8] =?UTF-8?q?fix(macos):=20=E4=BF=AE=E5=A4=8D=20Agent=20?= =?UTF-8?q?=E5=9B=BE=E7=89=87=E6=98=BE=E7=A4=BA=E8=AF=84=E5=AE=A1=E9=97=AE?= =?UTF-8?q?=E9=A2=98=EF=BC=9Aid=20=E5=8E=BB=E9=87=8D=E3=80=81=E5=93=88?= =?UTF-8?q?=E5=B8=8C=E3=80=81=E7=84=A6=E7=82=B9=E3=80=81=E7=BC=93=E5=AD=98?= =?UTF-8?q?=E4=B8=8E=E8=A7=A3=E6=9E=90=E8=BE=B9=E7=95=8C=20(#1472)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 图片 id 去重，避免 ForEach 重复 id - 指纹改用 SHA-256，避免主线程逐字节循环 - 用户消息解析结果按消息缓存 - 查看器 ←/→ 仅在持有焦点时翻页，拖动限制在画面边缘内 - 只移除消息首尾的模型专用块，不吞用户中间引用的标签 - 图片按原文位置排序，超出上限的引用显示 +N - read 只对存在的文件生成缩略图，解码失败显示占位 - 缓存限制成本，键含修改时间，解码任务随行取消 - 内联数据须能被 ImageIO 识别，否则退回文字条目 --- ...ranscript-images-and-task-notifications.md | 17 ++- .../Lithe/Views/Agent/AgentImageViews.swift | 108 ++++++++++++-- .../Views/Agent/AgentProcessGroupView.swift | 2 +- .../Views/Agent/AgentToolGroupView.swift | 2 +- .../Views/Agent/AgentTranscriptView.swift | 38 ++++- .../Application/AgentConnectionModel.swift | 3 +- .../Application/AgentImage.swift | 140 ++++++++++++------ .../AgentSessionConfiguration.swift | 21 ++- macos/Tests/LitheTests/AgentImageTests.swift | 55 ++++++- 9 files changed, 299 insertions(+), 87 deletions(-) diff --git a/.agents/notes/implemented/feature/2026-10-24-agent-transcript-images-and-task-notifications.md b/.agents/notes/implemented/feature/2026-10-24-agent-transcript-images-and-task-notifications.md index 6ad2fd12ec..0e87ad5881 100644 --- a/.agents/notes/implemented/feature/2026-10-24-agent-transcript-images-and-task-notifications.md +++ b/.agents/notes/implemented/feature/2026-10-24-agent-transcript-images-and-task-notifications.md @@ -20,22 +20,25 @@ ACP（Agent Client Protocol，Agent 与客户端之间的协议）的工具结 - `AgentToolDetails.images` 与 `AgentConversationMessage.images` 保存 `AgentImage`。内联图片解码后 不超过 10 MB、每条消息最多 24 张，超限直接丢弃，不进入会话状态。 - `AgentImage` 按内容指纹或路径判等，转录行的相等比较不会去比较像素。 -- 只提供文件路径的 `read` 工具调用（例如读取 `.png`）直接显示该文件。其他类型的工具不会因为位置是图片就当作“查看”。 + `AgentImage` 按内容指纹（SHA-256，硬件加速）或路径判等，转录行的相等比较不会去比较像素。 + 同一条消息、同一组工具里 id 相同的图片用 `uniquedByID` 去重，`ForEach` 和查看器翻页都要求 id 唯一。 + 内联数据必须能被 ImageIO 识别，否则不生成缩略图，工具输出里对应条目退回文字显示。 +- 只提供文件路径的 `read` 工具调用（例如读取 `.png`）直接显示该文件，且文件必须存在。其他类型的工具不会因为位置是图片就当作“查看”。 - 一次更新如果没有图片，不会清掉之前已收到的图片，避免最终状态更新把缩略图抹掉。 -- 缩略图在视图层用 ImageIO 在后台线程按需生成并放入 `NSCache`；历史回放只保存数据，不解码。 +- 缩略图在视图层用 ImageIO 在后台线程按需生成并放入 `NSCache`（限制总像素成本和数量，文件缓存键含修改时间，行滚出后解码任务随之取消，解码失败显示占位图标）；历史回放只保存数据，不解码。 悬停预览复用 `workbenchHoverCard`，放大查看器是转录区域上的覆盖层，状态放在 `AgentImageViewerState`，行只持有引用，打开或关闭不会重绘全部行。 - 放大查看器仍是转录区域上的覆盖层，在其上补充捏合缩放（相对适应窗口 25%–800%）、拖动、双击、 - 百分比和 ⌘+/⌘−/⌘0；侧栏很窄时可放大查看。曾试过改用独立窗口复用编辑器的 `MediaViewerView`， + 百分比和 ⌘+/⌘−/⌘0，拖动范围限制在画面边缘内；←/→ 翻页只在查看器持有焦点时生效，不抢输入框的光标键；侧栏很窄时可放大查看。曾试过改用独立窗口复用编辑器的 `MediaViewerView`， 因为偏离了既有交互而放弃。 - 折叠的“思考过程”标题行也显示图片缩略图，否则默认折叠时用户看不到工具返回的图片。 - 用户消息里的 `[@名字](file:///路径)`、裸 `file://` 地址、`📎 路径` 和 Claude Code 的 - `[Image: source: …]` 指向图片文件时显示为缩略图；`[Image: original …]` 尺寸说明丢弃。 + `[Image: source: …]` 指向图片文件时显示为缩略图；`[Image: original …]` 尺寸说明丢弃。缩略图按引用在原文中的位置排序；超过 24 张的引用从文字中移除并显示 `+N`。 + 用户消息的解析结果按消息缓存，悬停和流式刷新不会重复扫描全文。 - Claude Code 写进用户消息的特殊格式由 `AgentClaudeCodeMarkup` 识别：斜杠命令、本地命令输出、 Shell 转义、中断提示、压缩续接摘要、技能展开和宿主上下文显示为状态行或折叠块； - ``、`` 等只给模型看的块在任何位置都不显示。 - 只有整条消息都是这些标记时才整体替换，用户自己输入的相似文字不会被吞掉。 + ``、`` 等只给模型看的块只在消息开头或结尾时移除（Claude Code 就是加在那里）， + 夹在用户文字中间的同名标签原样保留。只有整条消息都是这些标记时才整体替换，用户自己输入的相似文字不会被吞掉。 格式清单来自对本机真实会话的标记名称统计，新增格式在这里补充。 - `` 只有在整条用户消息完全由通知块组成时才解析，用户自己输入的相同文字不会被吞掉。 这是协议标记的呈现，不按 Agent 名称在宿主里分支。 diff --git a/macos/Sources/Lithe/Views/Agent/AgentImageViews.swift b/macos/Sources/Lithe/Views/Agent/AgentImageViews.swift index d7a35f1bfb..7ec1eee4a9 100644 --- a/macos/Sources/Lithe/Views/Agent/AgentImageViews.swift +++ b/macos/Sources/Lithe/Views/Agent/AgentImageViews.swift @@ -36,35 +36,54 @@ extension EnvironmentValues { /// Decodes bounded renditions off the main thread. Inline pictures can be several /// megabytes, so rows ask for a thumbnail-sized decode and share it through a cache. enum AgentImageLoader { - private static let cache = NSCache() + private static let cache: NSCache = { + let cache = NSCache() + // Large renditions are tens of megabytes decoded, so bound by pixels, not just count. + cache.totalCostLimit = 192 * 1024 * 1024 + cache.countLimit = 400 + return cache + }() static func cachedImage(for image: AgentImage, maxPixel: Int) -> NSImage? { cache.object(forKey: key(image, maxPixel)) } + /// Nil when the picture cannot be decoded, or when the calling task was cancelled. static func image(for image: AgentImage, maxPixel: Int) async -> NSImage? { if let cached = cachedImage(for: image, maxPixel: maxPixel) { return cached } let source = image.source - let rendition = await Task.detached(priority: .utility) { () -> CGImage? in + let cacheKey = key(image, maxPixel) + let decode = Task.detached(priority: .utility) { () -> CGImage? in + if Task.isCancelled { return nil } let imageSource: CGImageSource? switch source { case .data(let data): imageSource = CGImageSourceCreateWithData(data as CFData, nil) case .file(let path): imageSource = CGImageSourceCreateWithURL(URL(fileURLWithPath: path) as CFURL, nil) } - guard let imageSource else { return nil } + guard let imageSource, !Task.isCancelled else { return nil } return CGImageSourceCreateThumbnailAtIndex(imageSource, 0, [ kCGImageSourceCreateThumbnailFromImageAlways: true, kCGImageSourceCreateThumbnailWithTransform: true, kCGImageSourceThumbnailMaxPixelSize: maxPixel, ] as CFDictionary) - }.value + } + // A row scrolled away cancels its task; stop the decode instead of finishing unseen work. + let rendition = await withTaskCancellationHandler { await decode.value } onCancel: { decode.cancel() } guard let rendition else { return nil } let result = NSImage(cgImage: rendition, size: NSSize(width: rendition.width, height: rendition.height)) - cache.setObject(result, forKey: key(image, maxPixel)) + cache.setObject(result, forKey: cacheKey, cost: rendition.width * rendition.height * 4) return result } - private static func key(_ image: AgentImage, _ maxPixel: Int) -> NSString { "\(image.id)@\(maxPixel)" as NSString } + /// Files carry their modification time, so a screenshot rewritten at the same path is decoded again. + private static func key(_ image: AgentImage, _ maxPixel: Int) -> NSString { + var stamp = "" + if case .file(let path) = image.source, + let date = try? URL(fileURLWithPath: path).resourceValues(forKeys: [.contentModificationDateKey]).contentModificationDate { + stamp = "@\(date.timeIntervalSinceReferenceDate)" + } + return "\(image.id)\(stamp)@\(maxPixel)" as NSString + } } /// A picture that fills its frame, or fits inside it, once decoded. @@ -73,6 +92,7 @@ struct AgentImageThumbnail: View { var maxPixel = 160 var fills = true @State private var rendition: NSImage? + @State private var failed = false var body: some View { ZStack { @@ -80,11 +100,20 @@ struct AgentImageThumbnail: View { Image(nsImage: rendition).resizable().aspectRatio(contentMode: fills ? .fill : .fit) } else { Rectangle().fill(LitheTheme.raised) + if failed { + Image(systemName: "photo").foregroundStyle(LitheTheme.secondaryText).imageScale(.small) + } } } .task(id: image.id) { + failed = false rendition = AgentImageLoader.cachedImage(for: image, maxPixel: maxPixel) - if rendition == nil { rendition = await AgentImageLoader.image(for: image, maxPixel: maxPixel) } + if rendition == nil { + let decoded = await AgentImageLoader.image(for: image, maxPixel: maxPixel) + if Task.isCancelled { return } + rendition = decoded + failed = decoded == nil + } } } } @@ -163,6 +192,8 @@ private struct AgentImagePreviewCard: View { struct AgentUserImages: View { @Environment(\.agentImageViewer) private var viewer let images: [AgentImage] + /// References past the per-message cap, summarised as `+N` like the tool strip. + var hiddenCount = 0 var body: some View { HStack(spacing: 6) { @@ -179,6 +210,11 @@ struct AgentUserImages: View { .accessibilityLabel(Text(verbatim: image.title ?? "Image")) .accessibilityHint(Text("Enlarge image")) } + if hiddenCount > 0 { + Text(verbatim: "+\(hiddenCount)") + .font(LitheTheme.uiFont(size: 12.5)) + .foregroundStyle(LitheTheme.secondaryText) + } } .accessibilityIdentifier("agent-user-images") } @@ -194,6 +230,9 @@ struct AgentImageViewer: View { @State private var pinchBase: CGFloat = 1 @State private var pan: CGSize = .zero @State private var panBase: CGSize = .zero + @State private var viewport: CGSize = .zero + /// Arrow keys page only while the viewer holds focus, so a caret in the composer keeps them. + @FocusState private var hasFocus: Bool private static let zoomRange: ClosedRange = 0.25...8 var body: some View { @@ -207,6 +246,10 @@ struct AgentImageViewer: View { .scaleEffect(zoom) .offset(pan) .frame(maxWidth: .infinity, maxHeight: .infinity) + .background(GeometryReader { proxy in + Color.clear.onAppear { viewport = proxy.size } + .onChange(of: proxy.size) { viewport = $0 } + }) .clipped() .shadow(color: .black.opacity(0.35), radius: 12) .contentShape(Rectangle()) @@ -214,8 +257,8 @@ struct AgentImageViewer: View { .onChanged { setZoom(pinchBase * $0) } .onEnded { _ in pinchBase = zoom }) .simultaneousGesture(DragGesture() - .onChanged { if zoom > 1 { pan = CGSize(width: panBase.width + $0.translation.width, - height: panBase.height + $0.translation.height) } } + .onChanged { if zoom > 1 { pan = clamped(CGSize(width: panBase.width + $0.translation.width, + height: panBase.height + $0.translation.height)) } } .onEnded { _ in panBase = pan }) .onTapGesture(count: 2) { withAnimation(.none) { zoom > 1 ? resetZoom() : setZoom(2) } } .onChange(of: gallery.index) { _ in resetZoom() } @@ -236,11 +279,11 @@ struct AgentImageViewer: View { .padding(.vertical, 40) HStack { if gallery.images.count > 1 { - step(-1, systemImage: "chevron.left", key: .leftArrow, label: "Previous image") + step(-1, systemImage: "chevron.left", label: "Previous image") } Spacer() if gallery.images.count > 1 { - step(1, systemImage: "chevron.right", key: .rightArrow, label: "Next image") + step(1, systemImage: "chevron.right", label: "Next image") } } .padding(.horizontal, 12) @@ -264,10 +307,23 @@ struct AgentImageViewer: View { } .padding(12) } + .modifier(AgentImagePagingKeys(hasFocus: $hasFocus, canPage: gallery.images.count > 1, page: { page($0) })) .accessibilityElement(children: .contain) .accessibilityIdentifier("agent-image-viewer") } + /// Keeps at least the picture's edge in view: the scaled picture may overhang the pane by half its extra size. + private func clamped(_ offset: CGSize) -> CGSize { + let limitX = viewport.width * (zoom - 1) / 2, limitY = viewport.height * (zoom - 1) / 2 + return CGSize(width: min(max(offset.width, -limitX), limitX), height: min(max(offset.height, -limitY), limitY)) + } + + private func page(_ delta: Int) { + let count = gallery.images.count + guard count > 1 else { return } + state.gallery?.index = (gallery.index + delta + count) % count + } + private var zoomControls: some View { HStack(spacing: 6) { zoomButton("minus", help: "Zoom Out", key: "-") { setZoom(zoom / 1.25) } @@ -306,17 +362,16 @@ struct AgentImageViewer: View { private func setZoom(_ value: CGFloat) { zoom = min(max(value, Self.zoomRange.lowerBound), Self.zoomRange.upperBound) pinchBase = zoom - if zoom <= 1 { pan = .zero; panBase = .zero } + if zoom <= 1 { pan = .zero; panBase = .zero } else { pan = clamped(pan); panBase = pan } } private func resetZoom() { zoom = 1; pinchBase = 1; pan = .zero; panBase = .zero } - private func step(_ delta: Int, systemImage: String, key: KeyEquivalent, label: LocalizedStringKey) -> some View { + private func step(_ delta: Int, systemImage: String, label: LocalizedStringKey) -> some View { Button { - let count = gallery.images.count - state.gallery?.index = (gallery.index + delta + count) % count + page(delta) } label: { Image(systemName: systemImage).font(LitheTheme.uiFont(size: 14, weight: .semibold)) .frame(width: 32, height: 32) @@ -325,9 +380,30 @@ struct AgentImageViewer: View { } .buttonStyle(.litheNoPress) .foregroundStyle(.white) - .keyboardShortcut(key, modifiers: []) .lithePointer() .help(label) .accessibilityLabel(label) } } + +/// ←/→ page the viewer only while it holds focus, so the composer keeps its caret keys. +/// Key handling needs macOS 14; macOS 13 pages with the on-screen arrows. +private struct AgentImagePagingKeys: ViewModifier { + var hasFocus: FocusState.Binding + let canPage: Bool + let page: (Int) -> Void + + func body(content: Content) -> some View { + if #available(macOS 14.0, *) { + content + .focusable() + .focusEffectDisabled() + .focused(hasFocus) + .onAppear { hasFocus.wrappedValue = true } + .onKeyPress(.leftArrow) { canPage ? { page(-1); return .handled }() : .ignored } + .onKeyPress(.rightArrow) { canPage ? { page(1); return .handled }() : .ignored } + } else { + content + } + } +} diff --git a/macos/Sources/Lithe/Views/Agent/AgentProcessGroupView.swift b/macos/Sources/Lithe/Views/Agent/AgentProcessGroupView.swift index c847c6d86c..e62e0c6c6f 100644 --- a/macos/Sources/Lithe/Views/Agent/AgentProcessGroupView.swift +++ b/macos/Sources/Lithe/Views/Agent/AgentProcessGroupView.swift @@ -37,7 +37,7 @@ struct AgentProcessGroupView: View { } private var images: [AgentImage] { - messages.flatMap(\.toolDetails.images) + messages.flatMap(\.toolDetails.images).uniquedByID } var body: some View { diff --git a/macos/Sources/Lithe/Views/Agent/AgentToolGroupView.swift b/macos/Sources/Lithe/Views/Agent/AgentToolGroupView.swift index 3a5829857f..0285c7b04b 100644 --- a/macos/Sources/Lithe/Views/Agent/AgentToolGroupView.swift +++ b/macos/Sources/Lithe/Views/Agent/AgentToolGroupView.swift @@ -73,7 +73,7 @@ struct AgentToolGroupView: View { @State private var showsAllRows = false @State private var expandedToolID: String? - private var groupImages: [AgentImage] { messages.flatMap(\.toolDetails.images) } + private var groupImages: [AgentImage] { messages.flatMap(\.toolDetails.images).uniquedByID } private var failedCount: Int { messages.filter { $0.toolStatus == .failed }.count } private var isRunning: Bool { messages.contains { $0.toolStatus == .inProgress || $0.toolStatus == .pending } } private var hasExpandedDetails: Bool { diff --git a/macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift b/macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift index 73c20c1d24..73518d4be8 100644 --- a/macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift +++ b/macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift @@ -75,7 +75,7 @@ enum AgentTranscriptItem: Identifiable { pending.removeAll(keepingCapacity: true) } for message in messages { - if message.role == .user, AgentClaudeCodeMarkup.parse(message.text)?.isEmpty == true { + if message.role == .user, case .markup(let entries) = AgentUserMessagePresentation.of(message), entries.isEmpty { // Model-only reminders and caveats are not conversation. continue } @@ -548,7 +548,7 @@ private struct AgentMessageRow: View { var body: some View { switch message.role { case .user: - if let entries = AgentClaudeCodeMarkup.parse(message.text) { + if case .markup(let entries) = AgentUserMessagePresentation.of(message) { VStack(alignment: .leading, spacing: 4) { ForEach(Array(entries.enumerated()), id: \.offset) { _, entry in AgentClaudeCodeEntryRow(entry: entry) @@ -581,8 +581,9 @@ private struct AgentMessageRow: View { /// Attached image files and replayed pictures show as thumbnails that open /// the viewer; the text keeps the user's words and non-image attachments. private var userBubble: some View { - let content = AgentUserMessageContent.parse(AgentClaudeCodeMarkup.removingModelOnlyBlocks(message.text)) - let images = message.images + content.images + var content = AgentUserMessageContent.parse("") + if case .bubble(let parsed) = AgentUserMessagePresentation.of(message) { content = parsed } + let images = (message.images + content.images).uniquedByID return HStack(alignment: .bottom, spacing: 4) { Spacer(minLength: 40) if !content.text.isEmpty { @@ -591,7 +592,7 @@ private struct AgentMessageRow: View { .allowsHitTesting(isHovered) } VStack(alignment: .trailing, spacing: 6) { - if !images.isEmpty { AgentUserImages(images: images) } + if !images.isEmpty { AgentUserImages(images: images, hiddenCount: content.hiddenImageCount) } if !content.text.isEmpty { Text(content.text) } if let delivery = message.delivery { Group { @@ -616,6 +617,33 @@ private struct AgentMessageRow: View { } } +/// How a user message is drawn, parsed once. The regex scans touch the whole text, +/// and rows re-evaluate on hover and while the transcript streams, so the result is cached. +private enum AgentUserMessagePresentation { + case markup([AgentClaudeCodeEntry]) + case bubble(AgentUserMessageContent) + + private final class Box { let value: AgentUserMessagePresentation; init(_ value: AgentUserMessagePresentation) { self.value = value } } + private static let cache: NSCache = { + let cache = NSCache() + cache.countLimit = 500 + return cache + }() + + static func of(_ message: AgentConversationMessage) -> Self { + let key = "\(message.id)#\(message.text.utf8.count)" as NSString + if let cached = cache.object(forKey: key) { return cached.value } + let value: Self + if let entries = AgentClaudeCodeMarkup.parse(message.text) { + value = .markup(entries) + } else { + value = .bubble(AgentUserMessageContent.parse(AgentClaudeCodeMarkup.removingModelOnlyBlocks(message.text))) + } + cache.setObject(Box(value), forKey: key) + return value + } +} + /// Claude Code's own markup in a user message, shown as a quiet status line or output block. private struct AgentClaudeCodeEntryRow: View { @Environment(\.locale) private var locale diff --git a/macos/Sources/LitheAgentConversationModule/Application/AgentConnectionModel.swift b/macos/Sources/LitheAgentConversationModule/Application/AgentConnectionModel.swift index 0ae2a5cb11..4f58fa7ef3 100644 --- a/macos/Sources/LitheAgentConversationModule/Application/AgentConnectionModel.swift +++ b/macos/Sources/LitheAgentConversationModule/Application/AgentConnectionModel.swift @@ -1223,7 +1223,8 @@ public final class AgentConnectionModel: ObservableObject { private func append(_ text: String, role: AgentConversationMessage.Role, to sessionID: String, subagentSessionID: String? = nil, image: AgentImage? = nil) { func attach(_ image: AgentImage?, to message: inout AgentConversationMessage) { - if let image, message.images.count < AgentImage.maximumPerMessage { message.images.append(image) } + if let image, message.images.count < AgentImage.maximumPerMessage, + !message.images.contains(where: { $0.id == image.id }) { message.images.append(image) } } if conversations[sessionID]?.isLoading == true { if let last = replayMessages[sessionID]?.indices.last, replayMessages[sessionID]?[last].role == role, diff --git a/macos/Sources/LitheAgentConversationModule/Application/AgentImage.swift b/macos/Sources/LitheAgentConversationModule/Application/AgentImage.swift index 93c7a6f9f2..4309dcbcec 100644 --- a/macos/Sources/LitheAgentConversationModule/Application/AgentImage.swift +++ b/macos/Sources/LitheAgentConversationModule/Application/AgentImage.swift @@ -1,4 +1,6 @@ +import CryptoKit import Foundation +import ImageIO /// An image the Agent produced or the user attached, shown beside the tool call /// or message that carries it. Inline bytes stay bounded; file references are @@ -46,11 +48,12 @@ public struct AgentImage: Identifiable, Equatable, Sendable { let uri = block["uri"] as? String let uriTitle = uri.flatMap { URL(string: $0)?.lastPathComponent }.flatMap { $0.isEmpty ? nil : $0 } if let encoded = block["data"] as? String, !encoded.isEmpty { - if let mime = block["mimeType"] as? String, !mime.lowercased().hasPrefix("image/") { return nil } + let mime = (block["mimeType"] as? String)?.lowercased() + if let mime, !mime.hasPrefix("image/") { return nil } // Base64 inflates by a third; refuse before allocating the decoded copy. guard encoded.utf8.count <= maximumBytes / 3 * 4, let data = Data(base64Encoded: encoded, options: .ignoreUnknownCharacters), !data.isEmpty, - data.count <= maximumBytes else { return nil } + data.count <= maximumBytes, isDecodable(data) else { return nil } return Self(data: data, title: uriTitle) } if let uri, let url = URL(string: uri), url.isFileURL, isImageFile(url.path) { @@ -59,11 +62,26 @@ public struct AgentImage: Identifiable, Equatable, Sendable { return nil } - /// FNV-1a over the bytes: stable within a run and cheap next to base64 decoding. + /// Reads only the container header, so a payload that no decoder recognises + /// never becomes a thumbnail that stays blank. + private static func isDecodable(_ data: Data) -> Bool { + guard let source = CGImageSourceCreateWithData(data as CFData, nil) else { return false } + return CGImageSourceGetCount(source) > 0 + } + + /// SHA-256 is hardware accelerated, so even a 10 MB payload hashes in a few + /// milliseconds on the main actor; a byte loop over `Data` is far slower. private static func fingerprint(_ data: Data) -> String { - var hash: UInt64 = 0xcbf2_9ce4_8422_2325 - for byte in data { hash = (hash ^ UInt64(byte)) &* 0x100_0000_01b3 } - return String(hash, radix: 16) + SHA256.hash(data: data).prefix(12).map { String(format: "%02x", $0) }.joined() + } +} + +extension Sequence where Element == AgentImage { + /// Keeps the first of each id. `ForEach` and the viewer's paging need unique ids, + /// and the same screenshot read twice would otherwise appear twice. + public var uniquedByID: [AgentImage] { + var seen = Set() + return filter { seen.insert($0.id).inserted } } } @@ -71,6 +89,8 @@ public struct AgentImage: Identifiable, Equatable, Sendable { public struct AgentUserMessageContent: Equatable, Sendable { public var text: String public var images: [AgentImage] + /// Image references beyond `AgentImage.maximumPerMessage`: removed from the text, not shown. + public var hiddenImageCount = 0 private static let attachmentMarker = "📎 " @@ -83,6 +103,16 @@ public struct AgentUserMessageContent: Equatable, Sendable { private static let imagePlaceholder = try! NSRegularExpression( pattern: #"\[Image: (?:source: ([^\]\n]+)|original [^\]\n]*)\]"#) + /// The composer's `📎 /path` line, with its line break. + private static let attachmentLine = try! NSRegularExpression(pattern: "^📎 ([^\\n]*)(?:\\n|$)", options: .anchorsMatchLines) + + /// One image reference found in the message: where it sits and the file it names. + /// `path` is nil for a placeholder that carries no file; it is removed without a thumbnail. + private struct Reference { + var range: NSRange + var path: String? + } + /// Image attachments become thumbnails; other attachments and the user's own /// words stay as text. Two shapes occur: the composer's `📎 /path` lines, and /// file links (`[@name](file:///path)`, bare `file://` URLs) that an agent @@ -91,51 +121,60 @@ public struct AgentUserMessageContent: Equatable, Sendable { guard message.contains(attachmentMarker) || message.contains("file://") || message.contains("[Image: ") else { return Self(text: message, images: []) } - var images: [AgentImage] = [] - var seen = Set() - func add(_ path: String) -> Bool { - guard path.hasPrefix("/"), AgentImage.isImageFile(path), images.count < AgentImage.maximumPerMessage else { return false } - if seen.insert(path).inserted { images.append(AgentImage(filePath: path)) } - return true - } + let ns = message as NSString + let whole = NSRange(location: 0, length: ns.length) + func isImagePath(_ path: String) -> Bool { path.hasPrefix("/") && AgentImage.isImageFile(path) } func path(ofFileURL raw: String) -> String? { let trimmed = raw.trimmingCharacters(in: .whitespaces) if let url = URL(string: trimmed), url.isFileURL { return url.path } // An unencoded space makes the address an invalid URL; decode it by hand. return trimmed.hasPrefix("file://") ? String(trimmed.dropFirst("file://".count)).removingPercentEncoding : nil } - // Links first, replacing from the end so earlier ranges stay valid. - var working = message - let ns = working as NSString - for match in fileLink.matches(in: working, range: NSRange(location: 0, length: ns.length)).reversed() { + + // Every candidate is located in the original text, so thumbnails follow reading order. + var found: [Reference] = [] + for match in fileLink.matches(in: message, range: whole) { let raw = ns.substring(with: match.range(at: match.range(at: 1).location != NSNotFound ? 1 : 2)) - guard let filePath = path(ofFileURL: raw), add(filePath), - let range = Range(match.range, in: working) else { continue } - working.replaceSubrange(range, with: "") + if let filePath = path(ofFileURL: raw), isImagePath(filePath) { found.append(Reference(range: match.range, path: filePath)) } } - let placeholderText = working as NSString - for match in imagePlaceholder.matches(in: working, range: NSRange(location: 0, length: placeholderText.length)).reversed() { + for match in imagePlaceholder.matches(in: message, range: whole) { let source = match.range(at: 1) - if source.location != NSNotFound, !add(placeholderText.substring(with: source)) { continue } - if let range = Range(match.range, in: working) { working.replaceSubrange(range, with: "") } + if source.location == NSNotFound { + found.append(Reference(range: match.range, path: nil)) + } else if isImagePath(ns.substring(with: source)) { + found.append(Reference(range: match.range, path: ns.substring(with: source))) + } + } + for match in attachmentLine.matches(in: message, range: whole) { + let value = ns.substring(with: match.range(at: 1)) + if isImagePath(value) { found.append(Reference(range: match.range, path: value)) } } - var kept: [Substring] = [] - for line in working.split(separator: "\n", omittingEmptySubsequences: false) { - if line.hasPrefix(attachmentMarker) { - let value = String(line.dropFirst(attachmentMarker.count)) - if add(value) { continue } + found.sort { $0.range.location < $1.range.location } + + var images: [AgentImage] = [] + var seen = Set() + var hidden = 0 + var removed: [NSRange] = [] + for reference in found { + // An earlier reference already claimed this text (a link inside a 📎 line). + if let last = removed.last, NSMaxRange(last) > reference.range.location { continue } + if let path = reference.path { + if seen.insert((path as NSString).standardizingPath).inserted { + if images.count < AgentImage.maximumPerMessage { images.append(AgentImage(filePath: path)) } else { hidden += 1 } + } } - kept.append(line) + removed.append(reference.range) } - guard !images.isEmpty else { return Self(text: message, images: []) } - // Pre-reversal order: the regex pass walked backwards, so restore reading order. - let ordered = images.sorted { lhs, rhs in - (message.range(of: lhs.title ?? "")?.lowerBound ?? message.endIndex) < (message.range(of: rhs.title ?? "")?.lowerBound ?? message.endIndex) + guard !removed.isEmpty else { return Self(text: message, images: []) } + var working = message + for range in removed.reversed() { + if let swift = Range(range, in: working) { working.replaceSubrange(swift, with: "") } } - let text = kept.joined(separator: "\n").replacingOccurrences(of: "[ \\t]+\\n", with: "\n", options: .regularExpression) + guard !images.isEmpty || hidden > 0 else { return Self(text: message, images: []) } + let text = working.replacingOccurrences(of: "[ \\t]+\\n", with: "\n", options: .regularExpression) .replacingOccurrences(of: "\n{3,}", with: "\n\n", options: .regularExpression) .trimmingCharacters(in: .whitespacesAndNewlines) - return Self(text: text, images: ordered) + return Self(text: text, images: images, hiddenImageCount: hidden) } } @@ -205,18 +244,31 @@ public enum AgentClaudeCodeMarkup { ]) private static let summaryPrefix = "This session is being continued from a previous conversation" private static let skillPrefix = "Base directory for this skill:" - private static let hiddenBlock = try! NSRegularExpression( - pattern: "<(" + hiddenTags.joined(separator: "|") + ")>[\\s\\S]*?") - - /// Removes model-only blocks wherever they sit, so reminders appended to a - /// typed message never show up as part of it. + /// Removes model-only blocks that open or close the message, which is where + /// Claude Code adds them. A block the user quoted in the middle of their own + /// words stays, so what they typed is never swallowed. public static func removingModelOnlyBlocks(_ message: String) -> String { guard hiddenTags.contains(where: { message.contains("<\($0)>") }) else { return message } - let range = NSRange(location: 0, length: (message as NSString).length) - return hiddenBlock.stringByReplacingMatches(in: message, range: range, withTemplate: "") - .trimmingCharacters(in: .whitespacesAndNewlines) + var text = message.trimmingCharacters(in: .whitespacesAndNewlines) + var removedAny = true + while removedAny { + removedAny = false + for tag in hiddenTags { + let open = "<\(tag)>", close = "" + if text.hasPrefix(open), let end = text.range(of: close) { + text = String(text[end.upperBound...]).trimmingCharacters(in: .whitespacesAndNewlines) + removedAny = true + } else if text.hasSuffix(close), let start = text.range(of: open, options: .backwards), + !text[start.upperBound...].dropLast(close.count).contains(close) { + text = String(text[.. [AgentClaudeCodeEntry]? { diff --git a/macos/Sources/LitheAgentConversationModule/Application/AgentSessionConfiguration.swift b/macos/Sources/LitheAgentConversationModule/Application/AgentSessionConfiguration.swift index b4fe11acdc..c82e55cd90 100644 --- a/macos/Sources/LitheAgentConversationModule/Application/AgentSessionConfiguration.swift +++ b/macos/Sources/LitheAgentConversationModule/Application/AgentSessionConfiguration.swift @@ -110,17 +110,21 @@ public struct AgentToolDetails: Equatable, Sendable { additions: validStats ? (stats?["added"] as? Int).flatMap { $0 >= 0 ? $0 : nil } : nil, deletions: validStats ? (stats?["removed"] as? Int).flatMap { $0 >= 0 ? $0 : nil } : nil) } - let reported = content.prefix(100).compactMap { item -> AgentImage? in - guard item["type"] as? String == "content", let block = item["content"] as? [String: Any] else { return nil } - return AgentImage.parse(block: block) - } + // Parse each picture once; an `image` block no decoder accepts falls back to a text entry. + var parsedIndices = Set() + let reported = content.prefix(100).enumerated().compactMap { index, item -> AgentImage? in + guard item["type"] as? String == "content", let block = item["content"] as? [String: Any], + let image = AgentImage.parse(block: block) else { return nil } + parsedIndices.insert(index) + return image + }.uniquedByID // An update without pictures must not erase ones an earlier update delivered. if !reported.isEmpty { images = Array(reported.prefix(AgentImage.maximumPerMessage)) } - self.content = content.prefix(100).compactMap { item in + self.content = content.prefix(100).enumerated().compactMap { index, item in switch item["type"] as? String { case "content": guard let block = item["content"] as? [String: Any] else { return nil } - if block["type"] as? String == "image" { return nil } + if parsedIndices.contains(index) { return nil } if let text = block["text"] as? String { return Content(title: "Output", text: Self.bounded(text)) } @@ -163,8 +167,9 @@ public struct AgentToolDetails: Equatable, Sendable { /// Untitled pictures borrow the file name or the tool's own caption. private mutating func titleImages() { if images.isEmpty, kind == "read" { - images = locations.lazy.map(\.path).filter(AgentImage.isImageFile).prefix(AgentImage.maximumPerMessage) - .map { AgentImage(filePath: $0) } + // Only files that exist: a failed or stale read must not leave a blank thumbnail. + images = locations.lazy.map(\.path).filter { AgentImage.isImageFile($0) && FileManager.default.fileExists(atPath: $0) } + .map { AgentImage(filePath: $0) }.uniquedByID.prefix(AgentImage.maximumPerMessage).map { $0 } } guard images.contains(where: { $0.title == nil }) else { return } let caption = locations.first.map { ($0.path as NSString).lastPathComponent } diff --git a/macos/Tests/LitheTests/AgentImageTests.swift b/macos/Tests/LitheTests/AgentImageTests.swift index e361c5312b..156c618f4f 100644 --- a/macos/Tests/LitheTests/AgentImageTests.swift +++ b/macos/Tests/LitheTests/AgentImageTests.swift @@ -5,7 +5,8 @@ import Testing @Suite("Agent images and task notifications") struct AgentImageTests { - private static let pixel = Data([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]) + /// A real 1x1 PNG: inline pictures must be recognised by ImageIO to become thumbnails. + private static let pixel = Data(base64Encoded: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==")! private func imageBlock(_ data: Data = pixel, mime: String = "image/png") -> [String: Any] { ["type": "content", "content": ["type": "image", "data": data.base64EncodedString(), "mimeType": mime]] @@ -32,11 +33,16 @@ struct AgentImageTests { @Test func readingAnImageFileWithoutInlineDataShowsTheFile() { + let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + let shot = directory.appendingPathComponent("shot.PNG").path + try? Self.pixel.write(to: URL(fileURLWithPath: shot)) var details = AgentToolDetails() - details.merge(["kind": "read", "locations": [["path": "/tmp/shot.PNG"], ["path": "/tmp/notes.txt"]]]) - #expect(details.images.map(\.title) == ["shot.PNG"]) + details.merge(["kind": "read", "locations": [["path": shot], ["path": "/tmp/notes.txt"], ["path": "/tmp/missing-\(UUID()).png"]]]) + #expect(details.images.map(\.title) == ["shot.PNG"], "Missing files get no blank thumbnail") var edit = AgentToolDetails() - edit.merge(["kind": "edit", "locations": [["path": "/tmp/shot.png"]]]) + edit.merge(["kind": "edit", "locations": [["path": shot]]]) #expect(edit.images.isEmpty, "Only a read implies viewing the file") } @@ -156,6 +162,47 @@ struct AgentImageTests { #expect(AgentClaudeCodeMarkup.removingModelOnlyBlocks("plain") == "plain") } + @Test + func quotedReminderInsideTypedTextIsKept() { + let message = "比较一下 x 这个标签" + #expect(AgentClaudeCodeMarkup.removingModelOnlyBlocks(message) == message) + let both = "a\n你好\nb" + #expect(AgentClaudeCodeMarkup.removingModelOnlyBlocks(both) == "你好") + } + + @Test + func imageReferencesKeepReadingOrderEvenWithEncodedOrRepeatedNames() { + let content = AgentUserMessageContent.parse( + "a [@my shot.png](file:///x/my%20shot.png)\n📎 /y/a.png\n[Image: source: /z/a.png]\n[@dup](file:///x/my%20shot.png)") + #expect(content.images.map(\.id) == ["file:/x/my shot.png", "file:/y/a.png", "file:/z/a.png"]) + #expect(content.text == "a") + } + + @Test + func referencesBeyondTheCapAreCountedInsteadOfLeftAsRawText() { + let lines = (0..<(AgentImage.maximumPerMessage + 3)).map { "📎 /tmp/\($0).png" }.joined(separator: "\n") + let content = AgentUserMessageContent.parse("hi\n" + lines) + #expect(content.images.count == AgentImage.maximumPerMessage) + #expect(content.hiddenImageCount == 3) + #expect(content.text == "hi") + } + + @Test + func identicalPicturesShareAnIdAndCollapseToOne() { + let first = AgentImage(data: Data(repeating: 7, count: 64)) + let same = AgentImage(data: Data(repeating: 7, count: 64)) + let other = AgentImage(data: Data(repeating: 8, count: 64)) + #expect(first.id == same.id && first.id != other.id) + #expect([first, same, other].uniquedByID.count == 2) + } + + @Test + func inlineDataThatNoDecoderRecognisesIsRejected() { + let junk = Data(repeating: 1, count: 32).base64EncodedString() + #expect(AgentImage.parse(block: ["type": "image", "data": junk]) == nil) + #expect(AgentImage.parse(block: ["type": "image", "data": junk, "mimeType": "image/png"]) == nil) + } + @Test func claudeCodeImagePlaceholdersBecomeThumbnailsAndDropTheSizeNote() { let content = AgentUserMessageContent.parse(""" From 9f2ff811a248c38d00c3f5a51e89e1279f13623e Mon Sep 17 00:00:00 2001 From: Yao Jingxi <23722032@bjtu.edu.cn> Date: Sat, 24 Oct 2026 23:06:02 +0800 Subject: [PATCH 5/8] =?UTF-8?q?feat(macos):=20=E5=90=8E=E5=8F=B0=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E9=80=9A=E7=9F=A5=E5=B9=B6=E5=85=A5=E5=90=8E=E7=BB=AD?= =?UTF-8?q?=E8=BF=87=E7=A8=8B=E5=8C=BA=EF=BC=8C=E6=81=A2=E5=A4=8D=20Claude?= =?UTF-8?q?=20=E4=BC=9A=E8=AF=9D=E6=97=B6=E8=A1=A5=E5=85=A8=E5=8E=8B?= =?UTF-8?q?=E7=BC=A9=E5=89=8D=E7=9A=84=E5=AE=8C=E6=95=B4=E5=8E=86=E5=8F=B2?= =?UTF-8?q?=20(#1472)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 成功的后台任务通知不再单独成行，进入随后一轮的折叠“思考过程”；失败、被终止或后面没有过程的通知仍显示 - Claude 的英文 shell 任务摘要按界面语言显示（后台命令已完成：…） - Rust 宿主在 session/load 前只读取会话文件中最后一个压缩点之前的记录，转成回放更新，压缩前的用户消息、回复、思考和工具调用都能查看 - 读取范围：只读 projects/*/<会话 id>.jsonl，按所选账号目录解析，大小和条数有上限 - 新增翻译放在 shared/i18n，更新功能矩阵与 Agent Note Co-Authored-By: Claude Sonnet 5.5 --- ...ranscript-images-and-task-notifications.md | 27 ++ macos/Resources/en.lproj/Localizable.strings | 4 + .../zh-Hans.lproj/Localizable.strings | 4 + .../Views/Agent/AgentProcessGroupView.swift | 7 + .../Views/Agent/AgentTranscriptView.swift | 55 ++- .../Application/AgentImage.swift | 5 + macos/Tests/LitheTests/AgentImageTests.swift | 48 ++ .../src/claude_earlier_history.rs | 432 ++++++++++++++++++ .../src/claude_subscription.rs | 6 + .../src/codex_history/tests.rs | 1 + rust/lithe-agent-host/src/lib.rs | 25 + rust/lithe-agent-host/src/subscription.rs | 1 + rust/lithe-agent-host/src/tests.rs | 1 + shared/i18n/locales/en-US.json | 2 + shared/i18n/locales/zh-CN.json | 2 + shared/i18n/messages.json | 34 ++ .../agent-tool-activity-timeline.json | 6 +- 17 files changed, 651 insertions(+), 9 deletions(-) create mode 100644 rust/lithe-agent-host/src/claude_earlier_history.rs diff --git a/.agents/notes/implemented/feature/2026-10-24-agent-transcript-images-and-task-notifications.md b/.agents/notes/implemented/feature/2026-10-24-agent-transcript-images-and-task-notifications.md index 0e87ad5881..c74a09b3b8 100644 --- a/.agents/notes/implemented/feature/2026-10-24-agent-transcript-images-and-task-notifications.md +++ b/.agents/notes/implemented/feature/2026-10-24-agent-transcript-images-and-task-notifications.md @@ -43,6 +43,31 @@ ACP（Agent Client Protocol，Agent 与客户端之间的协议）的工具结 - `` 只有在整条用户消息完全由通知块组成时才解析，用户自己输入的相同文字不会被吞掉。 这是协议标记的呈现，不按 Agent 名称在宿主里分支。 +### 后台任务通知并入后续过程 + +后台命令的完成通知会唤醒 Agent 再跑一轮。连续几次后台命令会让对话里反复出现“通知行 + 思考过程”。 +现在成功的通知（`AgentTranscriptItem.isQuietNotification`）不单独成行，而是进入随后一轮的折叠过程区，展开后按原顺序显示。 +失败、被终止的通知，以及后面没有任何思考或工具的通知，仍作为状态行保留，避免用户错过。 +Claude Code 为 shell 任务生成的英文摘要（`Background command "…" completed (exit code 0)`）按界面语言显示。 + +### 恢复会话时补全压缩前的历史 + +Claude Agent SDK 的 `getSessionMessages` 只返回当前 parent 链；压缩边界（`compact_boundary`）没有 parent， +因此适配器回放从最后一个边界开始，更早的记录虽然还在 `<配置目录>/projects/*/<会话 id>.jsonl`，却不会到达客户端， +对话里看起来像“前文丢了”。Rust 宿主（`claude_earlier_history.rs`）在 `session/load` 之前只读取该文件中最后一个边界之前的行， +转成与回放一致的 ACP 更新（用户、回复、思考、工具调用和结果），先于适配器的回放发出， +所以每次压缩的摘要也会在原位置显示为“对话已从摘要继续”。 + +这是对“宿主不读取转录文件”约定的有限例外，边界如下： + +- 只读，只打开 `projects/*/<会话 id>.jsonl`；账号目录里的登录信息和其他文件不触碰；会话 id 只接受字母、数字、`-`、`_`。 +- 配置目录按所选 Claude 账号解析（账号目录、`CLAUDE_CONFIG_DIR`、`~/.claude`），不写入任何文件，不缓存。 +- 超过 128 MiB 的记录文件不读取；最多发出 30000 条更新；工具输入输出每段截断到 4000 字符；子代理侧链和 `isMeta` 行跳过，图片不回放。 +- 文件格式由 Claude Code 拥有。解析失败或格式变化时只是不显示更早的历史，不影响适配器自身的回放。 +- Codex 的历史仍由其官方 App Server 提供，不受此例外影响。 + +不要这样做：不要把前文合并进适配器已回放的片段，也不要改写或重新压缩 Claude 自己的记录文件。 + ### 不要这样做 - 不要把图片 `Data` 放进 `Equatable` 的逐字节比较，也不要在主线程解码原图。 @@ -69,6 +94,7 @@ issue #1462 指出的多余复制按钮来自不属于最终回复的过程文 ## 验证 - `./scripts/test-macos.sh --filter AgentImageTests` +- `cargo test -p lithe-agent-host claude_earlier_history` - `./scripts/verify-agent-notes.sh` ## 适用范围 @@ -76,3 +102,4 @@ issue #1462 指出的多余复制按钮来自不属于最终回复的过程文 - `macos/Sources/LitheAgentConversationModule/Application/AgentImage.swift` - `macos/Sources/Lithe/Views/Agent/AgentImageViews.swift` - `macos/Sources/Lithe/Views/Agent/AgentToolGroupView.swift` +- `rust/lithe-agent-host/src/claude_earlier_history.rs` diff --git a/macos/Resources/en.lproj/Localizable.strings b/macos/Resources/en.lproj/Localizable.strings index 437546734b..1fd605b172 100644 --- a/macos/Resources/en.lproj/Localizable.strings +++ b/macos/Resources/en.lproj/Localizable.strings @@ -374,6 +374,8 @@ "Back to all references" = "Back to all references"; "Back to the conversation" = "Back to the conversation"; "Background Services" = "Background Services"; +"Background command ended (exit code %@): %@" = "Background command ended (exit code %@): %@"; +"Background command finished: %@" = "Background command finished: %@"; "Background services" = "Background services"; "Background services started in Lithe are still running" = "Background services started in Lithe are still running"; "Background task ended" = "Background task ended"; @@ -4216,6 +4218,8 @@ "native.backToAllReferences" = "Back to all references"; "native.backToTheConversation" = "Back to the conversation"; "native.backUpDatabaseAsSQL" = "Back Up Database as SQL…"; +"native.backgroundCommandEndedExitCodeArg1Arg2" = "Background command ended (exit code %1$@): %2$@"; +"native.backgroundCommandFinished" = "Background command finished: %1$@"; "native.backgroundServices" = "Background Services"; "native.backgroundServices.variant2" = "Background services"; "native.backgroundServicesStartedInLitheAreStillRunning" = "Background services started in Lithe are still running"; diff --git a/macos/Resources/zh-Hans.lproj/Localizable.strings b/macos/Resources/zh-Hans.lproj/Localizable.strings index ae258bacb9..18910c0567 100644 --- a/macos/Resources/zh-Hans.lproj/Localizable.strings +++ b/macos/Resources/zh-Hans.lproj/Localizable.strings @@ -374,6 +374,8 @@ "Back to all references" = "返回所有引用"; "Back to the conversation" = "返回对话"; "Background Services" = "后台服务"; +"Background command ended (exit code %@): %@" = "后台命令已结束（退出码 %@）：%@"; +"Background command finished: %@" = "后台命令已完成：%@"; "Background services" = "后台服务"; "Background services started in Lithe are still running" = "在 Lithe 中启动的后台服务仍在运行"; "Background task ended" = "后台任务已结束"; @@ -4216,6 +4218,8 @@ "native.backToAllReferences" = "返回所有引用"; "native.backToTheConversation" = "返回对话"; "native.backUpDatabaseAsSQL" = "将数据库备份为 SQL…"; +"native.backgroundCommandEndedExitCodeArg1Arg2" = "后台命令已结束（退出码 %1$@）：%2$@"; +"native.backgroundCommandFinished" = "后台命令已完成：%1$@"; "native.backgroundServices" = "后台服务"; "native.backgroundServices.variant2" = "后台服务"; "native.backgroundServicesStartedInLitheAreStillRunning" = "在 Lithe 中启动的后台服务仍在运行"; diff --git a/macos/Sources/Lithe/Views/Agent/AgentProcessGroupView.swift b/macos/Sources/Lithe/Views/Agent/AgentProcessGroupView.swift index e62e0c6c6f..9e20c682a5 100644 --- a/macos/Sources/Lithe/Views/Agent/AgentProcessGroupView.swift +++ b/macos/Sources/Lithe/Views/Agent/AgentProcessGroupView.swift @@ -53,6 +53,13 @@ struct AgentProcessGroupView: View { VStack(alignment: .leading, spacing: 8) { ForEach(AgentTranscriptItem.processContents(messages).filter { $0.matches(searchText) }) { item in switch item { + case .message(let message) where message.role == .user: + // Background-task notices that woke the Agent for this turn. + VStack(alignment: .leading, spacing: 4) { + ForEach(Array((AgentClaudeCodeMarkup.parse(message.text) ?? []).enumerated()), id: \.offset) { _, entry in + AgentClaudeCodeEntryRow(entry: entry) + } + } case .message(let message): AgentMarkdownMessage(text: message.text.trimmingCharacters(in: .whitespacesAndNewlines), size: 12) .equatable() diff --git a/macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift b/macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift index 73518d4be8..72da53a6c2 100644 --- a/macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift +++ b/macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift @@ -67,7 +67,13 @@ enum AgentTranscriptItem: Identifiable { // ACP has no final/commentary role. Only text after the last thought // or tool is presented as the reply; earlier prose is process narration. if let lastActivity = pending.lastIndex(where: { $0.role != .agent }) { - items.append(.processGroup(Array(pending[...lastActivity]))) + let process = Array(pending[...lastActivity]) + // Notices that triggered no thinking or tool use have no process to join. + if process.allSatisfy(Self.isQuietNotification) { + items += process.map(Self.message) + } else { + items.append(.processGroup(process)) + } items += pending.dropFirst(lastActivity + 1).map(Self.message) } else { items += pending.map(Self.message) @@ -79,7 +85,12 @@ enum AgentTranscriptItem: Identifiable { // Model-only reminders and caveats are not conversation. continue } - if message.role == .user { + if Self.isQuietNotification(message) { + // A finished background task wakes the Agent for a follow-up turn. + // The notice joins that turn's collapsed process instead of + // interrupting the transcript between two replies. + pending.append(message) + } else if message.role == .user { flush() items.append(.message(message)) } else if message.role == .tool || !message.text.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { @@ -95,6 +106,16 @@ enum AgentTranscriptItem: Identifiable { return items } + /// A successful background-task notice: protocol markup that needs no attention. + /// Failures stay on the transcript where the user can see them. + static func isQuietNotification(_ message: AgentConversationMessage) -> Bool { + guard message.role == .user, let entries = AgentClaudeCodeMarkup.parse(message.text), !entries.isEmpty else { return false } + return entries.allSatisfy { + if case .taskNotification(let notification) = $0 { return !notification.isFailure } + return false + } + } + /// Within an expanded process, keep prose in order and combine adjacent tools. static func processContents(_ messages: [AgentConversationMessage]) -> [Self] { var items: [Self] = [] @@ -644,18 +665,40 @@ private enum AgentUserMessagePresentation { } } +extension AgentTaskNotification { + private static let shellSummary = try! NSRegularExpression( + pattern: "^Background command \"(.+)\" (?:completed|failed) \\(exit code (-?\\d+)\\)$") + + /// One line for the transcript. Claude Code words shell-task summaries in + /// English (`Background command "…" completed (exit code 0)`); those are + /// shown in the interface language, and any other summary is kept verbatim. + func displayText(locale: Locale) -> String { + guard let summary else { + return AgentLocalization.string(isFailure ? "Background task ended" : "Background task finished", locale: locale) + } + guard let match = Self.shellSummary.firstMatch(in: summary, range: NSRange(summary.startIndex..., in: summary)), + let name = Range(match.range(at: 1), in: summary), let code = Range(match.range(at: 2), in: summary) else { + return summary + } + if summary[code] == "0" { + return String(format: AgentLocalization.string("Background command finished: %@", locale: locale), String(summary[name])) + } + return String(format: AgentLocalization.string("Background command ended (exit code %@): %@", locale: locale), + String(summary[code]), String(summary[name])) + } +} + /// Claude Code's own markup in a user message, shown as a quiet status line or output block. -private struct AgentClaudeCodeEntryRow: View { +struct AgentClaudeCodeEntryRow: View { @Environment(\.locale) private var locale let entry: AgentClaudeCodeEntry var body: some View { switch entry { case .taskNotification(let notification): - let failed = ["failed", "error", "killed", "stopped"].contains(notification.status?.lowercased() ?? "") + let failed = notification.isFailure line(systemImage: failed ? "exclamationmark.circle" : "checkmark.circle", isError: failed, - text: notification.summary ?? AgentLocalization.string( - failed ? "Background task ended" : "Background task finished", locale: locale)) + text: notification.displayText(locale: locale)) .help(notification.outputFile ?? "") .accessibilityIdentifier("agent-task-notification") case .command(let name, let arguments): diff --git a/macos/Sources/LitheAgentConversationModule/Application/AgentImage.swift b/macos/Sources/LitheAgentConversationModule/Application/AgentImage.swift index 4309dcbcec..e0c9159819 100644 --- a/macos/Sources/LitheAgentConversationModule/Application/AgentImage.swift +++ b/macos/Sources/LitheAgentConversationModule/Application/AgentImage.swift @@ -186,6 +186,11 @@ public struct AgentTaskNotification: Equatable, Sendable { public var summary: String? public var outputFile: String? + /// Statuses that mean the task did not finish normally. + public var isFailure: Bool { + ["failed", "error", "killed", "stopped"].contains(status?.lowercased() ?? "") + } + /// Nil unless the whole message consists of one or more notification blocks. public static func parse(_ message: String) -> [Self]? { let open = "", close = "" diff --git a/macos/Tests/LitheTests/AgentImageTests.swift b/macos/Tests/LitheTests/AgentImageTests.swift index 156c618f4f..eaadb0c766 100644 --- a/macos/Tests/LitheTests/AgentImageTests.swift +++ b/macos/Tests/LitheTests/AgentImageTests.swift @@ -227,4 +227,52 @@ struct AgentImageTests { #expect(content.entries.map(\.id) == ["real"]) #expect(content.turnByMessageID["reply"] == "real") } + + private func notice(_ id: String, status: String = "completed", summary: String) -> AgentConversationMessage { + AgentConversationMessage(id: id, role: .user, text: "\(status)\(summary)") + } + + @Test + func finishedBackgroundTasksJoinTheFollowUpProcessInsteadOfInterruptingTheTranscript() { + let messages = [ + AgentConversationMessage(id: "u", role: .user, text: "go"), + AgentConversationMessage(id: "reply", role: .agent, text: "running"), + notice("n1", summary: "Background command \"Build\" completed (exit code 0)"), + AgentConversationMessage(id: "th", role: .thought, text: "checking"), + notice("n2", summary: "Background command \"Test\" completed (exit code 0)"), + AgentConversationMessage(id: "th2", role: .thought, text: "done"), + AgentConversationMessage(id: "final", role: .agent, text: "All green"), + ] + let items = AgentTranscriptItem.grouped(messages) + #expect(!items.contains { if case .message(let m) = $0 { m.id == "n1" || m.id == "n2" } else { false } }) + let processes = items.compactMap { item -> [AgentConversationMessage]? in + if case .processGroup(let process) = item { process } else { nil } + } + #expect(processes.flatMap { $0.map(\.id) }.contains("n1")) + #expect(processes.flatMap { $0.map(\.id) }.contains("n2")) + } + + @Test + func failedOrIsolatedBackgroundNoticesStayVisible() { + let failed = [ + AgentConversationMessage(id: "u", role: .user, text: "go"), + notice("bad", status: "failed", summary: "Background command \"Build\" failed (exit code 1)"), + AgentConversationMessage(id: "th", role: .thought, text: "hm"), + ] + #expect(AgentTranscriptItem.grouped(failed).contains { if case .message(let m) = $0 { m.id == "bad" } else { false } }) + // With no thinking or tool use after it, a successful notice is still shown, not an empty process. + let lone = [AgentConversationMessage(id: "u", role: .user, text: "go"), notice("ok", summary: "Task finished")] + #expect(AgentTranscriptItem.grouped(lone).contains { if case .message(let m) = $0 { m.id == "ok" } else { false } }) + } + + @Test + func shellTaskSummariesAreRewordedAndOtherSummariesKeptVerbatim() throws { + let locale = Locale(identifier: "en") + let ok = try #require(AgentTaskNotification.parse(notice("a", summary: "Background command \"Run tests\" completed (exit code 0)").text)?.first) + #expect(ok.displayText(locale: locale) == "Background command finished: Run tests") + let bad = try #require(AgentTaskNotification.parse(notice("b", status: "failed", summary: "Background command \"Run tests\" completed (exit code 2)").text)?.first) + #expect(bad.displayText(locale: locale) == "Background command ended (exit code 2): Run tests") + let other = try #require(AgentTaskNotification.parse(notice("c", summary: "Agent \"Explore\" finished").text)?.first) + #expect(other.displayText(locale: locale) == "Agent \"Explore\" finished") + } } diff --git a/rust/lithe-agent-host/src/claude_earlier_history.rs b/rust/lithe-agent-host/src/claude_earlier_history.rs new file mode 100644 index 0000000000..c9479b6328 --- /dev/null +++ b/rust/lithe-agent-host/src/claude_earlier_history.rs @@ -0,0 +1,432 @@ +//! Transcript segments that Claude Code compaction hides from `session/load`. +//! +//! The Claude Agent SDK replays only the active parent chain. A compaction +//! boundary has no parent, so everything before the latest boundary is on disk +//! but never reaches the client. This module reads those earlier segments from +//! the account's project transcript, read-only and bounded, and renders them as +//! the same ACP updates a replay would carry. The adapter's own replay then +//! continues from the latest boundary, so nothing is shown twice. +//! +//! Only `/projects/*/.jsonl` is opened. Credentials and +//! the rest of the account directory stay untouched, and nothing is written. + +use std::fs::File; +use std::io::{BufRead, BufReader}; +use std::path::{Path, PathBuf}; + +use serde_json::{json, Value}; + +/// Larger transcripts are skipped rather than read: the recap alone is shown. +const MAX_TRANSCRIPT_BYTES: u64 = 128 * 1024 * 1024; +/// Upper bound on emitted updates, keeping the replay event queue bounded. +const MAX_UPDATES: usize = 30_000; +/// Per-field text limit, in characters, for tool output and tool input values. +const TEXT_LIMIT: usize = 4_000; + +/// Projects directory of the account the adapter runs under: the explicit +/// account directory, then `CLAUDE_CONFIG_DIR`, then `~/.claude`. +pub(crate) fn projects_directory(config_dir: Option<&Path>) -> Option { + let base = match config_dir { + Some(directory) => directory.to_path_buf(), + None => match std::env::var_os("CLAUDE_CONFIG_DIR").filter(|v| !v.is_empty()) { + Some(directory) => PathBuf::from(directory), + None => PathBuf::from(std::env::var_os(if cfg!(windows) { + "USERPROFILE" + } else { + "HOME" + })?) + .join(".claude"), + }, + }; + base.is_absolute().then(|| base.join("projects")) +} + +/// Updates for every transcript row before the latest compaction boundary, in +/// order. Empty when the session was never compacted or the file is unusable. +pub(crate) fn earlier_updates(projects: &Path, cwd: &Path, session_id: &str) -> Vec { + // The id becomes a file name, so anything but a plain identifier is refused. + if session_id.is_empty() + || !session_id + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') + { + return Vec::new(); + } + let Some(path) = locate(projects, cwd, session_id) else { + return Vec::new(); + }; + let Some(boundary) = last_boundary_line(&path) else { + return Vec::new(); + }; + let Ok(file) = File::open(&path) else { + return Vec::new(); + }; + let mut converter = Converter::default(); + for line in BufReader::new(file).lines().take(boundary) { + let Ok(line) = line else { break }; + if let Ok(row) = serde_json::from_str::(&line) { + converter.row(&row); + } + if converter.updates.len() >= MAX_UPDATES { + break; + } + } + converter.updates +} + +/// Claude names a project directory after its workspace with every character +/// outside `[A-Za-z0-9]` replaced by `-`; other directories are searched when +/// that name does not match, for example after Claude shortens a long path. +fn locate(projects: &Path, cwd: &Path, session_id: &str) -> Option { + let name = format!("{session_id}.jsonl"); + let encoded: String = cwd + .to_string_lossy() + .chars() + .map(|c| if c.is_ascii_alphanumeric() { c } else { '-' }) + .collect(); + let direct = projects.join(encoded).join(&name); + let found = if direct.is_file() { + Some(direct) + } else { + std::fs::read_dir(projects) + .ok()? + .filter_map(Result::ok) + .map(|entry| entry.path().join(&name)) + .find(|candidate| candidate.is_file()) + }?; + (found.metadata().ok()?.len() <= MAX_TRANSCRIPT_BYTES).then_some(found) +} + +/// Zero-based line of the latest `compact_boundary` record. +fn last_boundary_line(path: &Path) -> Option { + let mut latest = None; + for (index, line) in BufReader::new(File::open(path).ok()?).lines().enumerate() { + let Ok(line) = line else { break }; + // Cheap substring test first; most rows are never parsed. + if !line.contains("compact_boundary") { + continue; + } + if let Ok(row) = serde_json::from_str::(&line) { + if row["type"] == "system" && row["subtype"] == "compact_boundary" { + latest = Some(index); + } + } + } + latest +} + +/// Which kind of text chunk was emitted last, so a new chunk of the same kind +/// starts a new paragraph instead of fusing with the previous message. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Chunk { + User, + Agent, + Thought, +} + +#[derive(Default)] +struct Converter { + updates: Vec, + last: Option, +} + +impl Converter { + fn row(&mut self, row: &Value) { + // Subagent sidechains and meta rows (hooks, reminders) are not conversation. + if row["isSidechain"] == true || row["isMeta"] == true { + return; + } + let role = match row["type"].as_str() { + Some("user") => Chunk::User, + Some("assistant") => Chunk::Agent, + _ => return, + }; + // API error stand-ins are synthesized by the CLI, not model output. + if row["message"]["model"] == "" { + return; + } + match &row["message"]["content"] { + Value::String(text) if role == Chunk::User => self.text(Chunk::User, text), + Value::Array(blocks) => { + for block in blocks { + self.block(role, block); + } + } + _ => {} + } + } + + fn block(&mut self, role: Chunk, block: &Value) { + match (role, block["type"].as_str()) { + (Chunk::User, Some("text")) => { + self.text(Chunk::User, block["text"].as_str().unwrap_or("")) + } + (Chunk::Agent, Some("text")) => { + self.text(Chunk::Agent, block["text"].as_str().unwrap_or("")) + } + (Chunk::Agent, Some("thinking")) => { + self.text(Chunk::Thought, block["thinking"].as_str().unwrap_or("")) + } + (Chunk::Agent, Some("tool_use")) => self.tool_call(block), + (Chunk::User, Some("tool_result")) => self.tool_result(block), + _ => {} + } + } + + fn text(&mut self, kind: Chunk, text: &str) { + if text.trim().is_empty() { + return; + } + let name = match kind { + Chunk::User => "user_message_chunk", + Chunk::Agent => "agent_message_chunk", + Chunk::Thought => "agent_thought_chunk", + }; + let text = if self.last == Some(kind) { + format!("\n\n{text}") + } else { + text.to_owned() + }; + self.last = Some(kind); + self.updates + .push(json!({"sessionUpdate": name, "content": {"type": "text", "text": text}})); + } + + fn tool_call(&mut self, block: &Value) { + let Some(id) = block["id"].as_str() else { + return; + }; + let name = block["name"].as_str().unwrap_or("Tool"); + let input = &block["input"]; + self.last = None; + self.updates.push(json!({ + "sessionUpdate": "tool_call", + "toolCallId": id, + "title": title(name, input), + "kind": kind(name), + "status": "completed", + "rawInput": bounded(input), + })); + } + + fn tool_result(&mut self, block: &Value) { + let Some(id) = block["tool_use_id"].as_str() else { + return; + }; + let text = match &block["content"] { + Value::String(text) => text.clone(), + Value::Array(parts) => parts + .iter() + .filter_map(|part| part["text"].as_str()) + .collect::>() + .join("\n"), + _ => String::new(), + }; + self.last = None; + let mut update = json!({ + "sessionUpdate": "tool_call_update", + "toolCallId": id, + "status": if block["is_error"] == true { "failed" } else { "completed" }, + }); + if !text.trim().is_empty() { + update["content"] = + json!([{"type": "content", "content": {"type": "text", "text": truncate(&text)}}]); + } + self.updates.push(update); + } +} + +/// ACP tool kind of a Claude Code tool, as the adapter classifies them. +fn kind(name: &str) -> &'static str { + match name { + "Bash" | "BashOutput" | "KillShell" => "execute", + "Read" | "NotebookRead" => "read", + "Edit" | "MultiEdit" | "Write" | "NotebookEdit" => "edit", + "Grep" | "Glob" => "search", + "WebFetch" | "WebSearch" => "fetch", + "Task" | "Agent" | "TodoWrite" => "think", + _ => "other", + } +} + +/// Row title in the adapter's style: the command or target, not the tool name alone. +fn title(name: &str, input: &Value) -> String { + let field = |key: &str| input[key].as_str().filter(|v| !v.is_empty()); + let detail = match name { + "Bash" => field("description").or_else(|| field("command")), + "Read" | "Edit" | "MultiEdit" | "Write" | "NotebookEdit" => { + field("file_path").or_else(|| field("notebook_path")) + } + "Grep" | "Glob" => field("pattern"), + "WebFetch" => field("url"), + "WebSearch" => field("query"), + "Task" | "Agent" => field("description"), + _ => None, + }; + match detail { + Some(detail) => { + let first = detail.lines().next().unwrap_or(detail); + truncate_to(&format!("{name} {first}"), 200) + } + None => name.to_owned(), + } +} + +/// Copy of `value` whose strings are cut to [`TEXT_LIMIT`], so a whole-file +/// `Write` input does not travel through the replay queue. +fn bounded(value: &Value) -> Value { + match value { + Value::String(text) => Value::String(truncate(text)), + Value::Array(items) => Value::Array(items.iter().map(bounded).collect()), + Value::Object(map) => { + Value::Object(map.iter().map(|(k, v)| (k.clone(), bounded(v))).collect()) + } + other => other.clone(), + } +} + +fn truncate(text: &str) -> String { + truncate_to(text, TEXT_LIMIT) +} + +fn truncate_to(text: &str, limit: usize) -> String { + match text.char_indices().nth(limit) { + Some((index, _)) => format!("{}…", &text[..index]), + None => text.to_owned(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + /// Writes `rows` as a transcript under a fresh projects directory and + /// returns that directory together with the workspace it belongs to. + fn transcript(rows: &[Value]) -> (tempdir::Dir, PathBuf) { + let dir = tempdir::Dir::new(); + let cwd = PathBuf::from("/work/my.app"); + let project = dir.path().join("-work-my-app"); + std::fs::create_dir_all(&project).unwrap(); + let mut file = File::create(project.join("abc-1.jsonl")).unwrap(); + for row in rows { + writeln!(file, "{row}").unwrap(); + } + (dir, cwd) + } + + fn user(text: &str) -> Value { + json!({"type": "user", "message": {"role": "user", "content": text}}) + } + + fn boundary() -> Value { + json!({"type": "system", "subtype": "compact_boundary", "parentUuid": null}) + } + + #[test] + fn replays_only_what_precedes_the_latest_boundary() { + let (dir, cwd) = transcript(&[ + user("first question"), + json!({"type": "assistant", "message": {"content": [ + {"type": "thinking", "thinking": "hmm"}, + {"type": "text", "text": "first answer"}, + {"type": "tool_use", "id": "t1", "name": "Bash", "input": {"command": "ls", "description": "List files"}}, + ]}}), + json!({"type": "user", "message": {"content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": "a\nb"}, + ]}}), + boundary(), + user("This session is being continued from a previous conversation"), + user("second question"), + boundary(), + user("after the last boundary"), + ]); + let updates = earlier_updates(dir.path(), &cwd, "abc-1"); + let kinds: Vec<_> = updates + .iter() + .map(|u| u["sessionUpdate"].as_str().unwrap()) + .collect(); + assert_eq!( + kinds, + [ + "user_message_chunk", + "agent_thought_chunk", + "agent_message_chunk", + "tool_call", + "tool_call_update", + "user_message_chunk", + "user_message_chunk", + ] + ); + assert_eq!(updates[3]["title"], "Bash List files"); + assert_eq!(updates[3]["kind"], "execute"); + assert_eq!(updates[4]["content"][0]["content"]["text"], "a\nb"); + // Two user rows in a row must not fuse into one bubble. + assert!(updates[6]["content"]["text"] + .as_str() + .unwrap() + .starts_with("\n\n")); + } + + #[test] + fn an_uncompacted_or_unsafe_session_yields_nothing() { + let (dir, cwd) = transcript(&[user("hi")]); + assert!(earlier_updates(dir.path(), &cwd, "abc-1").is_empty()); + assert!(earlier_updates(dir.path(), &cwd, "../abc-1").is_empty()); + assert!(earlier_updates(dir.path(), &cwd, "missing").is_empty()); + } + + #[test] + fn sidechains_meta_rows_and_failed_results_are_classified() { + let (dir, cwd) = transcript(&[ + json!({"type": "user", "isMeta": true, "message": {"content": "reminder"}}), + json!({"type": "assistant", "isSidechain": true, "message": {"content": [{"type": "text", "text": "child"}]}}), + json!({"type": "assistant", "message": {"content": [{"type": "tool_use", "id": "t", "name": "Read", "input": {"file_path": "/a/b.rs"}}]}}), + json!({"type": "user", "message": {"content": [{"type": "tool_result", "tool_use_id": "t", "is_error": true, "content": [{"type": "text", "text": "denied"}]}]}}), + boundary(), + ]); + let updates = earlier_updates(dir.path(), &cwd, "abc-1"); + assert_eq!(updates.len(), 2); + assert_eq!(updates[0]["title"], "Read /a/b.rs"); + assert_eq!(updates[1]["status"], "failed"); + } + + #[test] + fn long_text_is_cut_on_a_character_boundary() { + let long = "é".repeat(TEXT_LIMIT + 10); + let cut = truncate(&long); + assert_eq!(cut.chars().count(), TEXT_LIMIT + 1); + assert!(cut.ends_with('…')); + } + + /// Minimal self-cleaning temporary directory; the crate has no tempfile dependency. + mod tempdir { + use std::path::{Path, PathBuf}; + + pub struct Dir(PathBuf); + + impl Dir { + pub fn new() -> Self { + use std::sync::atomic::{AtomicU32, Ordering}; + static NEXT: AtomicU32 = AtomicU32::new(0); + let path = std::env::temp_dir().join(format!( + "lithe-earlier-history-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir_all(&path).unwrap(); + Self(path) + } + + pub fn path(&self) -> &Path { + &self.0 + } + } + + impl Drop for Dir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + } +} diff --git a/rust/lithe-agent-host/src/claude_subscription.rs b/rust/lithe-agent-host/src/claude_subscription.rs index d8ab61f1de..fd77a16dbf 100644 --- a/rust/lithe-agent-host/src/claude_subscription.rs +++ b/rust/lithe-agent-host/src/claude_subscription.rs @@ -70,6 +70,11 @@ pub(super) fn resolve( directory.to_string_lossy().into_owned(), )); } + let claude_projects = crate::claude_earlier_history::projects_directory( + env.iter() + .find(|(name, _)| name == CONFIG_DIR_ENVIRONMENT) + .map(|(_, value)| std::path::Path::new(value)), + ); Ok(ResolvedLaunch { command: install::installed_command(&data, agent), args: launch.args, @@ -80,6 +85,7 @@ pub(super) fn resolve( subscription_cli: None, claude_subscription: true, codex_home: None, + claude_projects, }) } diff --git a/rust/lithe-agent-host/src/codex_history/tests.rs b/rust/lithe-agent-host/src/codex_history/tests.rs index 794bd2df62..44ebc9a6d1 100644 --- a/rust/lithe-agent-host/src/codex_history/tests.rs +++ b/rust/lithe-agent-host/src/codex_history/tests.rs @@ -15,6 +15,7 @@ fn launch(env: Vec<(String, String)>, subscription_cli: Option) -> Reso subscription_cli, claude_subscription: false, codex_home: None, + claude_projects: None, } } diff --git a/rust/lithe-agent-host/src/lib.rs b/rust/lithe-agent-host/src/lib.rs index f4e4b44ab6..4418f5dbf0 100644 --- a/rust/lithe-agent-host/src/lib.rs +++ b/rust/lithe-agent-host/src/lib.rs @@ -7,6 +7,7 @@ //! See `.agents/notes/implemented/architecture/2026-09-25-shared-acp-agent-conversation.md`. pub mod catalog; +mod claude_earlier_history; mod claude_subscription; pub mod cli_update; mod codex_context; @@ -229,6 +230,9 @@ struct ResolvedLaunch { claude_subscription: bool, /// Account directory the Codex quota probe must share with the adapter. codex_home: Option, + /// Claude transcript root of the adapter's account, for history that + /// compaction hides from `session/load`; None for every other agent. + claude_projects: Option, } #[cfg(test)] @@ -311,6 +315,7 @@ fn resolve_with( subscription_cli: None, claude_subscription: false, codex_home: None, + claude_projects: None, }); }; let agent = catalog::find(&agent_id).ok_or_else(|| format!("Unknown agent `{agent_id}`"))?; @@ -369,6 +374,9 @@ fn resolve_with( subscription_cli: None, claude_subscription: false, codex_home: None, + claude_projects: (agent.id == "claude-acp") + .then(|| claude_earlier_history::projects_directory(None)) + .flatten(), }) } @@ -1144,6 +1152,7 @@ async fn run_agent( // Account credentials belong to the CLI in both subscription modes. let subscription = launch.subscription_cli.is_some() || launch.claude_subscription; let claude_subscription = launch.claude_subscription; + let claude_projects = launch.claude_projects.clone(); let secret = launch.secret.clone(); let transport = ByteStreams::new(stdin.compat_write(), stdout.compat()); let result = run_connection( @@ -1153,6 +1162,7 @@ async fn run_agent( launch.subscription_cli, launch.codex_home, claude_subscription, + claude_projects, history, context_recovery, observe_subagents, @@ -1303,6 +1313,7 @@ async fn run_connection( subscription_cli: Option, codex_home: Option, claude_subscription: bool, + claude_projects: Option, history: Option, context_recovery: Option>, observe_subagents: bool, @@ -1666,7 +1677,21 @@ where let session_meta = restored_meta.clone(); let confirm_model = session_meta.is_some(); let capture = context_recovery.as_ref().and_then(|recovery| recovery.capture(&session_id)); + let earlier = claude_projects.clone(); tasks.spawn(async move { + // Compaction hides earlier segments from the adapter's replay; + // they precede it, so the transcript reads in order. + if let Some(projects) = earlier { + let (workspace, id) = (cwd.clone(), session_id.clone()); + let updates = tokio::task::spawn_blocking(move || { + claude_earlier_history::earlier_updates(&projects, &workspace, &id) + }) + .await + .unwrap_or_default(); + for update in updates { + emit(AgentEvent::Update { session_id: session_id.clone(), update }); + } + } let mut request = LoadSessionRequest::new(session_id.clone(), cwd); request.meta = session_meta; let result = request_with_timeout( diff --git a/rust/lithe-agent-host/src/subscription.rs b/rust/lithe-agent-host/src/subscription.rs index d4fd984af6..0c9a450b5e 100644 --- a/rust/lithe-agent-host/src/subscription.rs +++ b/rust/lithe-agent-host/src/subscription.rs @@ -196,6 +196,7 @@ pub(super) fn resolve( subscription_cli: Some(executable), claude_subscription: false, codex_home, + claude_projects: None, }) } diff --git a/rust/lithe-agent-host/src/tests.rs b/rust/lithe-agent-host/src/tests.rs index 3d663f4084..d3354a7335 100644 --- a/rust/lithe-agent-host/src/tests.rs +++ b/rust/lithe-agent-host/src/tests.rs @@ -648,6 +648,7 @@ impl Harness { false, None, None, + None, observe_subagents, receiver, TurnActivity { diff --git a/shared/i18n/locales/en-US.json b/shared/i18n/locales/en-US.json index 314588b33f..a5ba4c354a 100644 --- a/shared/i18n/locales/en-US.json +++ b/shared/i18n/locales/en-US.json @@ -3785,6 +3785,8 @@ "native.backToAllReferences": "Back to all references", "native.backToTheConversation": "Back to the conversation", "native.backUpDatabaseAsSQL": "Back Up Database as SQL…", + "native.backgroundCommandEndedExitCodeArg1Arg2": "Background command ended (exit code {arg1}): {arg2}", + "native.backgroundCommandFinished": "Background command finished: {arg1}", "native.backgroundServices": "Background Services", "native.backgroundServices.variant2": "Background services", "native.backgroundServicesStartedInLitheAreStillRunning": "Background services started in Lithe are still running", diff --git a/shared/i18n/locales/zh-CN.json b/shared/i18n/locales/zh-CN.json index dfb33f91e9..6a756464b7 100644 --- a/shared/i18n/locales/zh-CN.json +++ b/shared/i18n/locales/zh-CN.json @@ -3785,6 +3785,8 @@ "native.backToAllReferences": "返回所有引用", "native.backToTheConversation": "返回对话", "native.backUpDatabaseAsSQL": "将数据库备份为 SQL…", + "native.backgroundCommandEndedExitCodeArg1Arg2": "后台命令已结束（退出码 {arg1}）：{arg2}", + "native.backgroundCommandFinished": "后台命令已完成：{arg1}", "native.backgroundServices": "后台服务", "native.backgroundServices.variant2": "后台服务", "native.backgroundServicesStartedInLitheAreStillRunning": "在 Lithe 中启动的后台服务仍在运行", diff --git a/shared/i18n/messages.json b/shared/i18n/messages.json index f5f0b54b45..088754bbe5 100644 --- a/shared/i18n/messages.json +++ b/shared/i18n/messages.json @@ -29745,6 +29745,40 @@ "arguments": [] } }, + { + "id": "native.backgroundCommandEndedExitCodeArg1Arg2", + "description": "Native UI: Background command ended (exit code %@): %@", + "platforms": [ + "macos" + ], + "macos": { + "key": "Background command ended (exit code %@): %@", + "arguments": [ + "arg1", + "arg2" + ] + }, + "parameters": { + "arg1": "string", + "arg2": "string" + } + }, + { + "id": "native.backgroundCommandFinished", + "description": "Native UI: Background command finished: %@", + "platforms": [ + "macos" + ], + "macos": { + "key": "Background command finished: %@", + "arguments": [ + "arg1" + ] + }, + "parameters": { + "arg1": "string" + } + }, { "id": "native.backgroundServices", "description": "Native UI: Background Services", diff --git a/shared/platform-feature-matrix/features/agent-tool-activity-timeline.json b/shared/platform-feature-matrix/features/agent-tool-activity-timeline.json index 87cdac2ae2..45e545cc7b 100644 --- a/shared/platform-feature-matrix/features/agent-tool-activity-timeline.json +++ b/shared/platform-feature-matrix/features/agent-tool-activity-timeline.json @@ -2,7 +2,7 @@ "id": "agent-tool-activity-timeline", "area": "AI", "group": "Agent 对话", - "capability": "每轮工具调用与思考、过程说明放进同一个默认折叠的过程区；展开后相邻工具保留有序时间线、状态、输入输出和文件位置，最终回复单独展示；工具读取或截取的图片以缩略图显示在工具行和工具组标题旁（最多三张、其余 +N），悬停预览、点击放大，用户消息附带的图片同样可放大；Claude Code 后台任务通知显示为状态行（macOS）", + "capability": "每轮工具调用与思考、过程说明放进同一个默认折叠的过程区；展开后相邻工具保留有序时间线、状态、输入输出和文件位置，最终回复单独展示；工具读取或截取的图片以缩略图显示在工具行和工具组标题旁（最多三张、其余 +N），悬停预览、点击放大，用户消息附带的图片同样可放大；Claude Code 后台任务通知：成功的并入随后一轮的折叠过程区，失败的与无后续过程的保持可见；恢复 Claude 会话时压缩前的完整记录也会显示（macOS）", "macos": { "evidence": [ "macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift", @@ -33,6 +33,6 @@ "verificationStatus": "pending" }, "owner": "Agent", - "verification": "macOS 与 Windows：构造思考→过程说明→读文件→思考→执行命令→最终回复，确认每轮只有一个过程入口且默认收起，流式也不自动展开；收起时过程文字与工具行不占高度，最终回复始终可读。展开按原顺序呈现，只有相邻工具合并，状态与证据可继续查看。纯空白思考、过程说明不产生空行，没有思考和工具的直接回答不产生入口；新用户消息、已记录的轮次结束分别切断分组，工具或思考结束而没有最终回复时不伪造回复。搜索过程文字及工具输入、输出、文件路径，匹配组临时展开且不能在搜索期间收起，清空后恢复手动选择；切换和关闭会话验证偏好隔离与释放。深浅主题、窄宽面板、历史恢复和实时流均验证；历史导出、文件审阅、权限卡片、轮次统计和回复复制继续可用。单元测试与组件回放不替代真实 Agent 和 Windows Tauri 运行验收，未执行部分保持 pending。 macOS 图片：让 Agent 读取多张图片，确认缩略图最多三张并显示 +N，悬停出现带标题和序号的预览，点击进入放大查看并可用方向键切换、Esc 关闭；用户粘贴图片后气泡显示缩略图且可放大；触发后台任务后不再出现原始 task-notification 标记。Windows 对应能力尚未实现。 点击缩略图后的大图支持双指缩放、拖动、百分比与 ⌘+/⌘−/⌘0；用户消息里的 file:// 链接与 [Image: source:] 占位符显示为缩略图；Claude Code 的斜杠命令、本地命令输出、Shell 转义、压缩续接摘要、技能展开和宿主上下文显示为状态行或折叠块，只给模型看的提醒不显示。", - "notes": "2026-10-24：改为每轮一个默认折叠的过程区，包含思考、工具和中间说明；只改变呈现，不改写历史或 ACP 协议。ACP 无最终回复角色，按最后一条思考或工具的位置保留末尾正文；后续活动到达时才将先前正文移入过程。 2026-10-24：macOS 增加工具图片缩略图、悬停预览、放大查看器和后台任务通知状态行（#1462），Windows 待对齐。 2026-10-24（续）：折叠的思考过程标题行也显示图片缩略图；放大查看器增加缩放；识别更多 Claude Code 特殊消息格式。" + "verification": "macOS 与 Windows：构造思考→过程说明→读文件→思考→执行命令→最终回复，确认每轮只有一个过程入口且默认收起，流式也不自动展开；收起时过程文字与工具行不占高度，最终回复始终可读。展开按原顺序呈现，只有相邻工具合并，状态与证据可继续查看。纯空白思考、过程说明不产生空行，没有思考和工具的直接回答不产生入口；新用户消息、已记录的轮次结束分别切断分组，工具或思考结束而没有最终回复时不伪造回复。搜索过程文字及工具输入、输出、文件路径，匹配组临时展开且不能在搜索期间收起，清空后恢复手动选择；切换和关闭会话验证偏好隔离与释放。深浅主题、窄宽面板、历史恢复和实时流均验证；历史导出、文件审阅、权限卡片、轮次统计和回复复制继续可用。单元测试与组件回放不替代真实 Agent 和 Windows Tauri 运行验收，未执行部分保持 pending。 macOS 图片：让 Agent 读取多张图片，确认缩略图最多三张并显示 +N，悬停出现带标题和序号的预览，点击进入放大查看并可用方向键切换、Esc 关闭；用户粘贴图片后气泡显示缩略图且可放大；触发后台任务后不再出现原始 task-notification 标记。Windows 对应能力尚未实现。 macOS 后台任务与压缩历史：让 Agent 连续启动多个后台命令，确认成功的通知不再单独成行，而是在对应一轮的“思考过程”展开后以中文状态行出现；失败的通知仍在对话中可见。恢复一个经历过多次压缩的 Claude 会话，确认压缩分界线之前的用户消息、回复、思考和工具调用都能滚动查看，且每次压缩处有“对话已从摘要继续”标记。 点击缩略图后的大图支持双指缩放、拖动、百分比与 ⌘+/⌘−/⌘0；用户消息里的 file:// 链接与 [Image: source:] 占位符显示为缩略图；Claude Code 的斜杠命令、本地命令输出、Shell 转义、压缩续接摘要、技能展开和宿主上下文显示为状态行或折叠块，只给模型看的提醒不显示。", + "notes": "2026-10-24：改为每轮一个默认折叠的过程区，包含思考、工具和中间说明；只改变呈现，不改写历史或 ACP 协议。ACP 无最终回复角色，按最后一条思考或工具的位置保留末尾正文；后续活动到达时才将先前正文移入过程。 2026-10-24：macOS 增加工具图片缩略图、悬停预览、放大查看器和后台任务通知状态行（#1462），Windows 待对齐。 2026-10-24（续）：折叠的思考过程标题行也显示图片缩略图；放大查看器增加缩放；识别更多 Claude Code 特殊消息格式。 2026-10-24（再续）：成功的后台任务通知并入后续过程区；Rust 宿主在 session/load 时只读取最后一个压缩点之前的会话记录，补全 Claude 适配器不回放的历史。" } From 54cfb7348fe6e00e70cb3ff535bcd94d35c6dc5c Mon Sep 17 00:00:00 2001 From: Yao Jingxi <23722032@bjtu.edu.cn> Date: Sat, 24 Oct 2026 23:10:56 +0800 Subject: [PATCH 6/8] =?UTF-8?q?perf(macos):=20=E5=90=8E=E5=8F=B0=E9=80=9A?= =?UTF-8?q?=E7=9F=A5=E5=88=A4=E6=96=AD=E5=A4=8D=E7=94=A8=E7=94=A8=E6=88=B7?= =?UTF-8?q?=E6=B6=88=E6=81=AF=E8=A7=A3=E6=9E=90=E7=BC=93=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 5.5 --- macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift b/macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift index 72da53a6c2..be1100c2a5 100644 --- a/macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift +++ b/macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift @@ -109,7 +109,7 @@ enum AgentTranscriptItem: Identifiable { /// A successful background-task notice: protocol markup that needs no attention. /// Failures stay on the transcript where the user can see them. static func isQuietNotification(_ message: AgentConversationMessage) -> Bool { - guard message.role == .user, let entries = AgentClaudeCodeMarkup.parse(message.text), !entries.isEmpty else { return false } + guard message.role == .user, case .markup(let entries) = AgentUserMessagePresentation.of(message), !entries.isEmpty else { return false } return entries.allSatisfy { if case .taskNotification(let notification) = $0 { return !notification.isFailure } return false From a5a7b7f0c5df4d8bf07fe3c24a7d238fa6df82ef Mon Sep 17 00:00:00 2001 From: Yao Jingxi <23722032@bjtu.edu.cn> Date: Sat, 24 Oct 2026 23:31:17 +0800 Subject: [PATCH 7/8] =?UTF-8?q?fix(macos):=20=E5=9B=9E=E6=94=BE=E6=97=B6?= =?UTF-8?q?=E7=9B=B8=E9=82=BB=E7=9A=84=E7=94=A8=E6=88=B7=E6=B6=88=E6=81=AF?= =?UTF-8?q?=E4=B8=8D=E5=86=8D=E5=90=88=E5=B9=B6=E6=88=90=E4=B8=80=E6=9D=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 连续的用户消息（例如后台通知后紧跟用户输入）在回放时被拼成一个气泡，导致通知标记无法识别而显示原文。 按上游 messageId 区分用户消息；压缩前的历史也带上记录 uuid。 Co-Authored-By: Claude Sonnet 5.5 --- .../Application/AgentConnectionModel.swift | 19 +++++++++++--- .../Tests/LitheTests/AgentHistoryTests.swift | 25 +++++++++++++++++++ .../src/claude_earlier_history.rs | 23 +++++++++++------ 3 files changed, 56 insertions(+), 11 deletions(-) diff --git a/macos/Sources/LitheAgentConversationModule/Application/AgentConnectionModel.swift b/macos/Sources/LitheAgentConversationModule/Application/AgentConnectionModel.swift index 4f58fa7ef3..18ce4efcfb 100644 --- a/macos/Sources/LitheAgentConversationModule/Application/AgentConnectionModel.swift +++ b/macos/Sources/LitheAgentConversationModule/Application/AgentConnectionModel.swift @@ -124,6 +124,9 @@ public final class AgentConnectionModel: ObservableObject { /// square of the history. The replay lands in `conversations` once, when it ends. private var replayMessages: [String: [AgentConversationMessage]] = [:] private var replayToolIndex: [String: [String: Int]] = [:] + /// Upstream id of the user message a replay last appended to; consecutive user + /// messages are separate entries even though ACP delivers them as chunks. + private var replayUserMessageID: [String: String] = [:] /// Locally prepared sessions without a submitted prompt or upstream history evidence. /// Codex does not persist their rollout until the first prompt, so they cannot be resumed. private var unpromptedSessionIDs: Set = [] @@ -993,7 +996,8 @@ public final class AgentConnectionModel: ObservableObject { let image = (update["content"] as? [String: Any]).flatMap(AgentImage.parse(block:)) guard let text = Self.text(of: update) ?? (image == nil ? nil : "") else { return } flushPendingText() - append(text, role: .user, to: sessionID, subagentSessionID: subagentSessionID, image: image) + append(text, role: .user, to: sessionID, subagentSessionID: subagentSessionID, image: image, + messageID: update["messageId"] as? String) case "tool_call", "tool_call_update": guard let toolCallID = update["toolCallId"] as? String else { return } flushPendingText() @@ -1212,22 +1216,31 @@ public final class AgentConnectionModel: ObservableObject { private func commitReplay(_ sessionID: String) { guard let messages = replayMessages.removeValue(forKey: sessionID) else { return } replayToolIndex[sessionID] = nil + replayUserMessageID[sessionID] = nil conversations[sessionID, default: AgentConversation()].messages = messages } private func discardReplay(_ sessionID: String) { replayMessages[sessionID] = nil replayToolIndex[sessionID] = nil + replayUserMessageID[sessionID] = nil } private func append(_ text: String, role: AgentConversationMessage.Role, to sessionID: String, - subagentSessionID: String? = nil, image: AgentImage? = nil) { + subagentSessionID: String? = nil, image: AgentImage? = nil, messageID: String? = nil) { func attach(_ image: AgentImage?, to message: inout AgentConversationMessage) { if let image, message.images.count < AgentImage.maximumPerMessage, !message.images.contains(where: { $0.id == image.id }) { message.images.append(image) } } if conversations[sessionID]?.isLoading == true { - if let last = replayMessages[sessionID]?.indices.last, replayMessages[sessionID]?[last].role == role, + // Two user messages in a row (a prompt after a background notice, say) must + // not fuse into one bubble, which would also stop the notice being recognised. + var startsNewMessage = false + if role == .user, let messageID { + startsNewMessage = replayUserMessageID[sessionID].map { $0 != messageID } ?? false + replayUserMessageID[sessionID] = messageID + } + if !startsNewMessage, let last = replayMessages[sessionID]?.indices.last, replayMessages[sessionID]?[last].role == role, replayMessages[sessionID]?[last].subagentSessionID == subagentSessionID { replayMessages[sessionID]?[last].text += text attach(image, to: &replayMessages[sessionID]![last]) diff --git a/macos/Tests/LitheTests/AgentHistoryTests.swift b/macos/Tests/LitheTests/AgentHistoryTests.swift index ea293d4573..0adb4e38ab 100644 --- a/macos/Tests/LitheTests/AgentHistoryTests.swift +++ b/macos/Tests/LitheTests/AgentHistoryTests.swift @@ -694,6 +694,31 @@ struct AgentHistoryTests { return try #require(events[name] as? [String: Any]) } + @Test + func consecutiveReplayedUserMessagesStaySeparateEntries() async throws { + try await HistoryFixture().run { fixture in + let feature = fixture.feature + fixture.transport.connection.onCommand = { [weak feature] command in + guard command["kind"] as? String == "loadSession", let feature else { return } + @MainActor func user(_ text: String, id: String) { + feature.receive(HistoryFixture.json(["kind": "update", "sessionId": "session-a", + "update": ["sessionUpdate": "user_message_chunk", "messageId": id, + "content": ["type": "text", "text": text]]])) + } + user("completed", id: "one") + user("Please continue", id: "two") + // Chunks of one message without a new id still belong together. + user(" now", id: "two") + feature.receive(HistoryFixture.json(["kind": "sessionLoaded", "sessionId": "session-a", + "token": command["token"] as Any])) + } + feature.selectSession("session-a") + #expect(feature.conversations["session-a"]?.messages.map(\.text) == [ + "completed", "Please continue now", + ]) + } + } + @Test func batchExportReplaysUnopenedHistoryWithoutChangingSelection() async throws { try await HistoryFixture().run { fixture in diff --git a/rust/lithe-agent-host/src/claude_earlier_history.rs b/rust/lithe-agent-host/src/claude_earlier_history.rs index c9479b6328..331ed6d2f5 100644 --- a/rust/lithe-agent-host/src/claude_earlier_history.rs +++ b/rust/lithe-agent-host/src/claude_earlier_history.rs @@ -128,6 +128,9 @@ enum Chunk { struct Converter { updates: Vec, last: Option, + /// `uuid` of the row being converted; stamps user chunks so the client can + /// tell consecutive user messages apart, as the adapter's own replay does. + row_id: Option, } impl Converter { @@ -136,6 +139,7 @@ impl Converter { if row["isSidechain"] == true || row["isMeta"] == true { return; } + self.row_id = row["uuid"].as_str().map(str::to_owned); let role = match row["type"].as_str() { Some("user") => Chunk::User, Some("assistant") => Chunk::Agent, @@ -182,14 +186,19 @@ impl Converter { Chunk::Agent => "agent_message_chunk", Chunk::Thought => "agent_thought_chunk", }; - let text = if self.last == Some(kind) { + // Same-kind chunks of one reply are fused by the client, so a new + // reply needs a break; user messages are told apart by message id. + let text = if kind != Chunk::User && self.last == Some(kind) { format!("\n\n{text}") } else { text.to_owned() }; self.last = Some(kind); - self.updates - .push(json!({"sessionUpdate": name, "content": {"type": "text", "text": text}})); + let mut update = json!({"sessionUpdate": name, "content": {"type": "text", "text": text}}); + if let (Chunk::User, Some(id)) = (kind, &self.row_id) { + update["messageId"] = json!(id); + } + self.updates.push(update); } fn tool_call(&mut self, block: &Value) { @@ -316,7 +325,7 @@ mod tests { } fn user(text: &str) -> Value { - json!({"type": "user", "message": {"role": "user", "content": text}}) + json!({"type": "user", "uuid": text, "message": {"role": "user", "content": text}}) } fn boundary() -> Value { @@ -362,10 +371,8 @@ mod tests { assert_eq!(updates[3]["kind"], "execute"); assert_eq!(updates[4]["content"][0]["content"]["text"], "a\nb"); // Two user rows in a row must not fuse into one bubble. - assert!(updates[6]["content"]["text"] - .as_str() - .unwrap() - .starts_with("\n\n")); + assert_ne!(updates[5]["messageId"], updates[6]["messageId"]); + assert!(updates[5]["messageId"].is_string()); } #[test] From 00f503dcae036f42d783d24b2b030c1bb15b3d01 Mon Sep 17 00:00:00 2001 From: Yao Jingxi <23722032@bjtu.edu.cn> Date: Sat, 24 Oct 2026 23:55:20 +0800 Subject: [PATCH 8/8] =?UTF-8?q?fix(agent):=20=E6=81=A2=E5=A4=8D=20Claude?= =?UTF-8?q?=20Desktop=20=E5=BC=80=E5=A7=8B=E7=9A=84=E4=BC=9A=E8=AF=9D?= =?UTF-8?q?=E6=97=B6=E6=98=BE=E7=A4=BA=E5=8F=AF=E8=AF=BB=E5=8E=86=E5=8F=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 宿主把 mcp__ccd_turn__wrap_up 改写成 Agent 回复，吞掉其后续 tool_call_update - 宿主把 mcp__ccd_turn__update_plan（整表/增量）改写成完整 ACP plan 更新 - 压缩前文回放补上对话中途排队的用户消息（queued_command），工具调用带上 name - 历史回放时计划快照作为清单显示在折叠过程之间，导出为任务列表，不计入消息数 - 折叠的过程标题附工具摘要（如运行了 N 条命令） --- ...ranscript-images-and-task-notifications.md | 23 ++ .../Lithe/Views/Agent/AgentHistoryView.swift | 2 +- .../Views/Agent/AgentProcessGroupView.swift | 11 + .../Agent/AgentSessionGuidanceViews.swift | 7 +- .../Views/Agent/AgentTranscriptView.swift | 13 +- .../Application/AgentConnectionModel.swift | 9 + .../Application/AgentConversationState.swift | 2 + .../AgentHistoryFeatureModel.swift | 11 +- macos/Tests/LitheTests/AgentImageTests.swift | 17 ++ rust/lithe-agent-host/src/claude_desktop.rs | 222 ++++++++++++++++++ .../src/claude_earlier_history.rs | 28 ++- rust/lithe-agent-host/src/lib.rs | 12 + 12 files changed, 348 insertions(+), 9 deletions(-) create mode 100644 rust/lithe-agent-host/src/claude_desktop.rs diff --git a/.agents/notes/implemented/feature/2026-10-24-agent-transcript-images-and-task-notifications.md b/.agents/notes/implemented/feature/2026-10-24-agent-transcript-images-and-task-notifications.md index c74a09b3b8..bdda2e0350 100644 --- a/.agents/notes/implemented/feature/2026-10-24-agent-transcript-images-and-task-notifications.md +++ b/.agents/notes/implemented/feature/2026-10-24-agent-transcript-images-and-task-notifications.md @@ -68,6 +68,28 @@ Claude Agent SDK 的 `getSessionMessages` 只返回当前 parent 链；压缩边 不要这样做：不要把前文合并进适配器已回放的片段，也不要改写或重新压缩 Claude 自己的记录文件。 +任务进行中排队发送的提示（含后台任务通知）在记录文件里只存成 `attachment` 行（`queued_command`）， +没有对应的用户行，所以前文转换把它当作用户消息回放，并用该行的 `uuid` 区分相邻消息。 + +### 兼容 Claude Desktop 开始的会话 + +Claude Desktop 给 Claude Code 挂了自己的 `ccd_turn` MCP 服务，模型在那里几乎不直接输出文字， +而是用 `mcp__ccd_turn__wrap_up` 工具交出每轮的总结（`recap`）和提问（`ask`），由 Desktop 渲染成卡片。 +在 Lithe 里恢复这类会话时，整段对话只剩折叠的工具调用，看起来像历史没有回放。 + +宿主（`claude_desktop.rs`）把 `wrap_up` 的工具调用改写成一条 Agent 回复（总结与提问各一段）， +并吞掉该调用后续的 `tool_call_update`；建议回复按钮不显示，因为用户点过的那条会作为下一条用户消息出现。 +改写同时作用于适配器的回放和上面的前文转换，Windows 共用同一宿主。Lithe 自己不提供 `ccd_turn` 工具， +所以只影响从 Desktop 带过来的历史。 + +`update_plan` 有整表（`items`）和增量（`done`/`start`/`advance`）两种写法。宿主按会话记住当前列表， +把每次调用改写成一条完整的 ACP `plan` 更新；增量调用之前没有整表时原样保留为工具调用。 +Swift 在加载历史期间把每份计划快照记成一条不带文字的 Agent 消息，转录里在两段折叠过程之间显示成清单， +导出 Markdown 时写成任务列表，不计入消息数。折叠的过程标题后面附上工具摘要（如“运行了 3 条命令”）， +没有文字回复的段落也能看出做了什么。 + +不要这样做：不要在 Swift 或 Windows 前端按工具名再做一份改写，也不要读取或模拟 Desktop 的 MCP 服务。 + ### 不要这样做 - 不要把图片 `Data` 放进 `Equatable` 的逐字节比较，也不要在主线程解码原图。 @@ -95,6 +117,7 @@ issue #1462 指出的多余复制按钮来自不属于最终回复的过程文 - `./scripts/test-macos.sh --filter AgentImageTests` - `cargo test -p lithe-agent-host claude_earlier_history` +- `cargo test -p lithe-agent-host claude_desktop` - `./scripts/verify-agent-notes.sh` ## 适用范围 diff --git a/macos/Sources/Lithe/Views/Agent/AgentHistoryView.swift b/macos/Sources/Lithe/Views/Agent/AgentHistoryView.swift index 53029102a6..9eae1185a5 100644 --- a/macos/Sources/Lithe/Views/Agent/AgentHistoryView.swift +++ b/macos/Sources/Lithe/Views/Agent/AgentHistoryView.swift @@ -210,7 +210,7 @@ enum AgentHistoryPresentation { var hasTranscript = false for message in conversation.messages where message.subagentSessionID == nil || message.role == .tool { hasTranscript = true - if message.role == .user || message.role == .agent { count += 1 } + if message.role == .user || (message.role == .agent && message.plan == nil) { count += 1 } } return conversation.isAttached || hasTranscript ? count : nil } diff --git a/macos/Sources/Lithe/Views/Agent/AgentProcessGroupView.swift b/macos/Sources/Lithe/Views/Agent/AgentProcessGroupView.swift index 9e20c682a5..815c278492 100644 --- a/macos/Sources/Lithe/Views/Agent/AgentProcessGroupView.swift +++ b/macos/Sources/Lithe/Views/Agent/AgentProcessGroupView.swift @@ -23,6 +23,13 @@ struct AgentProcessGroupView: View { Image(systemName: "chevron.down") .font(LitheTheme.uiFont(size: 9, weight: .semibold)) .rotationEffect(.degrees(isExpanded ? 0 : -90)) + // Like Claude Desktop's "Ran 3 commands": a collapsed process still says what it did. + if !isExpanded && !toolSummary.isEmpty { + Text(verbatim: toolSummary) + .lineLimit(1) + .truncationMode(.tail) + .foregroundStyle(LitheTheme.tertiaryText) + } } .font(LitheTheme.uiFont(size: 12.5)) .foregroundStyle(LitheTheme.secondaryText) @@ -36,6 +43,10 @@ struct AgentProcessGroupView: View { .accessibilityIdentifier("agent-process-group-header") } + private var toolSummary: String { + AgentToolGroupPresentation.summary(messages.filter { $0.role == .tool }, locale: locale) + } + private var images: [AgentImage] { messages.flatMap(\.toolDetails.images).uniquedByID } diff --git a/macos/Sources/Lithe/Views/Agent/AgentSessionGuidanceViews.swift b/macos/Sources/Lithe/Views/Agent/AgentSessionGuidanceViews.swift index 8cafdb5c51..0c84d1b98c 100644 --- a/macos/Sources/Lithe/Views/Agent/AgentSessionGuidanceViews.swift +++ b/macos/Sources/Lithe/Views/Agent/AgentSessionGuidanceViews.swift @@ -103,12 +103,15 @@ struct AgentPlanView: View { let plan: AgentPlan let isResponding: Bool let embedded: Bool + /// Pinned plans align with the composer; transcript entries align with messages. + let horizontalInset: CGFloat @State private var expanded = false - init(plan: AgentPlan, isResponding: Bool, expanded: Bool = false, embedded: Bool = false) { + init(plan: AgentPlan, isResponding: Bool, expanded: Bool = false, embedded: Bool = false, horizontalInset: CGFloat = 18) { self.plan = plan self.isResponding = isResponding self.embedded = embedded + self.horizontalInset = horizontalInset _expanded = State(initialValue: expanded) } @@ -162,7 +165,7 @@ struct AgentPlanView: View { .frame(maxWidth: .infinity, alignment: .leading) .background(embedded ? Color.clear : AgentPanelStyle.header, in: RoundedRectangle(cornerRadius: 5)) .overlay(RoundedRectangle(cornerRadius: 5).stroke(embedded ? Color.clear : AgentPanelStyle.border, lineWidth: 1)) - .padding(.horizontal, embedded ? 0 : 18) + .padding(.horizontal, embedded ? 0 : horizontalInset) .padding(.bottom, embedded ? 0 : 4) } diff --git a/macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift b/macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift index be1100c2a5..bb8aa498fd 100644 --- a/macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift +++ b/macos/Sources/Lithe/Views/Agent/AgentTranscriptView.swift @@ -38,7 +38,9 @@ enum AgentTranscriptItem: Identifiable { func matches(_ searchText: String) -> Bool { guard !searchText.isEmpty else { return true } switch self { - case .message(let message): return message.text.localizedStandardContains(searchText) + case .message(let message): + return message.text.localizedStandardContains(searchText) + || message.plan?.entries.contains { $0.content.localizedStandardContains(searchText) } == true case .processGroup(let messages): return messages.contains { $0.role == .tool ? Self.toolMatches($0, searchText) : $0.text.localizedStandardContains(searchText) } @@ -85,7 +87,12 @@ enum AgentTranscriptItem: Identifiable { // Model-only reminders and caveats are not conversation. continue } - if Self.isQuietNotification(message) { + if message.plan != nil { + // A replayed plan snapshot is what the Agent reported at that point; + // it stays on the transcript like Claude Desktop's checklist. + flush() + items.append(.message(message)) + } else if Self.isQuietNotification(message) { // A finished background task wakes the Agent for a follow-up turn. // The notice joins that turn's collapsed process instead of // interrupting the transcript between two replies. @@ -413,6 +420,8 @@ private struct AgentTranscriptRows: View, Equatable { ForEach(transcript) { item in Group { switch item { + case .message(let message) where message.plan != nil: + AgentPlanView(plan: message.plan!, isResponding: false, horizontalInset: 0) case .message(let message): AgentMessageRow( message: message, diff --git a/macos/Sources/LitheAgentConversationModule/Application/AgentConnectionModel.swift b/macos/Sources/LitheAgentConversationModule/Application/AgentConnectionModel.swift index 18ce4efcfb..583b984e51 100644 --- a/macos/Sources/LitheAgentConversationModule/Application/AgentConnectionModel.swift +++ b/macos/Sources/LitheAgentConversationModule/Application/AgentConnectionModel.swift @@ -967,6 +967,15 @@ public final class AgentConnectionModel: ObservableObject { case "plan": guard let plan = AgentPlan.parse(update) else { return } conversations[sessionID, default: AgentConversation()].plan = plan.entries.isEmpty ? nil : plan + // Live plans are pinned above the composer. A replay instead keeps every + // snapshot in order, so a history whose Agent wrote little prose (Claude + // Desktop reports progress this way) still shows what each turn did. + if conversations[sessionID]?.isLoading == true, subagentSessionID == nil, !plan.entries.isEmpty { + flushPendingText() + var message = AgentConversationMessage(role: .agent, text: "") + message.plan = plan + replayMessages[sessionID, default: []].append(message) + } // A valid plan proves recovery without implying streamed reasoning. if conversations[sessionID]?.responsePhase == .retrying { markResponseProgress(.waiting, in: sessionID) diff --git a/macos/Sources/LitheAgentConversationModule/Application/AgentConversationState.swift b/macos/Sources/LitheAgentConversationModule/Application/AgentConversationState.swift index e7526c8a35..9424b1a538 100644 --- a/macos/Sources/LitheAgentConversationModule/Application/AgentConversationState.swift +++ b/macos/Sources/LitheAgentConversationModule/Application/AgentConversationState.swift @@ -37,6 +37,8 @@ public struct AgentConversationMessage: Identifiable, Equatable, Sendable { public var images: [AgentImage] = [] /// Child output is retained with its owner, separate from the parent's reply. public var subagentSessionID: String? + /// Plan snapshot recorded in a replayed history; such an Agent entry has no text. + public var plan: AgentPlan? public init(id: String = UUID().uuidString, role: Role, text: String, toolStatus: ToolStatus? = nil) { self.id = id diff --git a/macos/Sources/LitheAgentConversationModule/Application/AgentHistoryFeatureModel.swift b/macos/Sources/LitheAgentConversationModule/Application/AgentHistoryFeatureModel.swift index 3d37172386..f36060bc73 100644 --- a/macos/Sources/LitheAgentConversationModule/Application/AgentHistoryFeatureModel.swift +++ b/macos/Sources/LitheAgentConversationModule/Application/AgentHistoryFeatureModel.swift @@ -158,9 +158,14 @@ public struct AgentHistoryDocument: Sendable { case .tool: role = String(localized: "Tool") } // Reasoning is exported as a quote so it stays distinct from the reply. - let body = message.role == .thought - ? message.text.components(separatedBy: .newlines).map { "> " + $0 }.joined(separator: "\n") - : message.text + // A replayed plan snapshot is exported as a task list. + let body = if let plan = message.plan { + plan.entries.map { "- [\($0.status == .completed ? "x" : " ")] \($0.content)" }.joined(separator: "\n") + } else if message.role == .thought { + message.text.components(separatedBy: .newlines).map { "> " + $0 }.joined(separator: "\n") + } else { + message.text + } var content = "## \(role)\n\n\(body)" if let status = message.toolStatus { content += "\n\nStatus: \(status.rawValue)" } for detail in [message.toolDetails.input, message.toolDetails.output].compactMap({ $0 }) { diff --git a/macos/Tests/LitheTests/AgentImageTests.swift b/macos/Tests/LitheTests/AgentImageTests.swift index eaadb0c766..5297c923d9 100644 --- a/macos/Tests/LitheTests/AgentImageTests.swift +++ b/macos/Tests/LitheTests/AgentImageTests.swift @@ -265,6 +265,23 @@ struct AgentImageTests { #expect(AgentTranscriptItem.grouped(lone).contains { if case .message(let m) = $0 { m.id == "ok" } else { false } }) } + @Test + func replayedPlanSnapshotsStayVisibleBetweenCollapsedProcesses() { + var plan = AgentConversationMessage(id: "plan", role: .agent, text: "") + plan.plan = AgentPlan(entries: [.init(content: "Build", status: .inProgress)]) + let messages = [ + AgentConversationMessage(id: "u", role: .user, text: "go"), + AgentConversationMessage(id: "t1", role: .tool, text: "Bash", toolStatus: .completed), + plan, + AgentConversationMessage(id: "t2", role: .tool, text: "Bash", toolStatus: .completed), + ] + let items = AgentTranscriptItem.grouped(messages) + // A turn of tool calls only (Claude Desktop history) must not collapse its plan away. + #expect(items.map(\.id) == ["u", "process:t1", "plan", "process:t2"]) + #expect(items.last?.matches("Build") == false) + #expect(items[2].matches("Build")) + } + @Test func shellTaskSummariesAreRewordedAndOtherSummariesKeptVerbatim() throws { let locale = Locale(identifier: "en") diff --git a/rust/lithe-agent-host/src/claude_desktop.rs b/rust/lithe-agent-host/src/claude_desktop.rs new file mode 100644 index 0000000000..3999820f2b --- /dev/null +++ b/rust/lithe-agent-host/src/claude_desktop.rs @@ -0,0 +1,222 @@ +//! Readable replay of sessions started in Claude Desktop. +//! +//! Claude Desktop runs Claude Code with its own `ccd_turn` MCP server. The +//! model there rarely writes prose: it reports progress through `update_plan` +//! and ends a turn with a `wrap_up` recap, which Desktop renders as a checklist +//! and a card. Resumed in Lithe, both would be anonymous rows in the collapsed +//! process area and the transcript would look empty. This presenter turns them +//! into what Desktop shows: an ACP `plan` update and an agent message. It +//! applies to the adapter's replay and to the earlier history read from disk +//! alike; Lithe never offers the `ccd_turn` tools, so live turns are unaffected. + +use std::collections::{HashMap, HashSet}; + +use serde_json::{json, Value}; + +/// Tool Claude Desktop's model uses to end a turn with a recap and a question. +const WRAP_UP_TOOL: &str = "mcp__ccd_turn__wrap_up"; +/// Tool Claude Desktop's model uses to set or advance its step checklist. +const UPDATE_PLAN_TOOL: &str = "mcp__ccd_turn__update_plan"; + +/// One checklist step; `status` is an ACP plan status +/// (`pending`, `in_progress` or `completed`). +#[derive(Clone)] +struct Step { + content: String, + status: &'static str, +} + +#[derive(Default)] +pub(crate) struct Presenter { + /// Tool call ids already shown another way; their follow-up + /// `tool_call_update`s (status, the tool's acknowledgement) are dropped. + absorbed: HashSet, + /// Latest checklist per session. `update_plan` may only advance or tick + /// steps of the list set earlier, so each call is applied to this state. + plans: HashMap>, +} + +impl Presenter { + /// `update` as the client should see it, or None when it is absorbed. + pub(crate) fn present(&mut self, session_id: &str, update: Value) -> Option { + let Some(id) = update["toolCallId"].as_str().map(str::to_owned) else { + return Some(update); + }; + let shown = match update["sessionUpdate"].as_str() { + Some("tool_call") if update["name"] == WRAP_UP_TOOL => { + wrap_up_text(&update["rawInput"]).map(|text| { + json!({ + "sessionUpdate": "agent_message_chunk", + "content": {"type": "text", "text": text}, + }) + }) + } + Some("tool_call") if update["name"] == UPDATE_PLAN_TOOL => { + let steps = self.plans.entry(session_id.to_owned()).or_default(); + apply_plan(steps, &update["rawInput"]).then(|| plan_update(steps)) + } + Some("tool_call_update") if self.absorbed.contains(&id) => return None, + _ => None, + }; + match shown { + Some(shown) => { + self.absorbed.insert(id); + Some(shown) + } + None => Some(update), + } + } +} + +/// The recap followed by the question, as separate paragraphs. The suggested +/// replies are buttons in Desktop; the prompt the user chose follows in the +/// transcript, so they are not repeated here. +fn wrap_up_text(input: &Value) -> Option { + let text = ["recap", "ask"] + .iter() + .filter_map(|key| input[key].as_str().map(str::trim).filter(|v| !v.is_empty())) + .collect::>() + .join("\n\n"); + (!text.is_empty()).then_some(text) +} + +/// Applies one `update_plan` input and reports whether it changed anything. +/// Inputs either replace the list (`items`), tick steps by one-based index +/// (`done`) and start one (`start`), or `advance`: finish the running step +/// and start the next pending one. +fn apply_plan(steps: &mut Vec, input: &Value) -> bool { + if let Some(items) = input["items"].as_array() { + *steps = items + .iter() + .filter_map(|item| { + Some(Step { + content: item["content"].as_str()?.to_owned(), + status: status(item["status"].as_str()), + }) + }) + .collect(); + return !steps.is_empty(); + } + if steps.is_empty() { + return false; + } + let mut changed = false; + let mut set = |index: usize, value: &'static str| { + if let Some(step) = index.checked_sub(1).and_then(|i| steps.get_mut(i)) { + step.status = value; + changed = true; + } + }; + for index in input["done"].as_array().into_iter().flatten() { + set(index.as_u64().unwrap_or(0) as usize, "completed"); + } + if let Some(index) = input["start"].as_u64() { + set(index as usize, "in_progress"); + } + if input["advance"] == true { + let running = steps.iter().position(|s| s.status == "in_progress"); + if let Some(index) = running { + steps[index].status = "completed"; + } + let from = running.map_or(0, |index| index + 1); + if let Some(next) = steps[from..].iter_mut().find(|s| s.status == "pending") { + next.status = "in_progress"; + } + changed = true; + } + changed +} + +fn status(value: Option<&str>) -> &'static str { + match value { + Some("completed") => "completed", + Some("in_progress") => "in_progress", + _ => "pending", + } +} + +fn plan_update(steps: &[Step]) -> Value { + let entries: Vec = steps + .iter() + .map(|step| json!({"content": step.content, "status": step.status, "priority": "medium"})) + .collect(); + json!({"sessionUpdate": "plan", "entries": entries}) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn call(name: &str, id: &str, input: Value) -> Value { + json!({"sessionUpdate": "tool_call", "toolCallId": id, "name": name, + "title": name, "status": "pending", "rawInput": input}) + } + + fn done(id: &str) -> Value { + json!({"sessionUpdate": "tool_call_update", "toolCallId": id, "status": "completed"}) + } + + fn statuses(update: &Value) -> Vec<&str> { + update["entries"] + .as_array() + .unwrap() + .iter() + .map(|e| e["status"].as_str().unwrap()) + .collect() + } + + #[test] + fn a_wrap_up_becomes_the_turn_answer_and_its_updates_are_absorbed() { + let mut presenter = Presenter::default(); + let input = json!({"recap": "PR opened.", "ask": "Mark it ready?", + "actions": [{"label": "Yes", "message": "yes"}]}); + let shown = presenter + .present("s", call(WRAP_UP_TOOL, "w", input)) + .unwrap(); + assert_eq!(shown["sessionUpdate"], "agent_message_chunk"); + assert_eq!(shown["content"]["text"], "PR opened.\n\nMark it ready?"); + assert!(presenter.present("s", done("w")).is_none()); + } + + #[test] + fn plan_calls_become_plan_updates_that_track_incremental_progress() { + let mut presenter = Presenter::default(); + let items = json!({"items": [ + {"content": "a", "status": "in_progress"}, + {"content": "b", "status": "pending"}, + {"content": "c", "status": "pending"}, + ]}); + let set = presenter + .present("s", call(UPDATE_PLAN_TOOL, "p1", items)) + .unwrap(); + assert_eq!(set["sessionUpdate"], "plan"); + assert_eq!(statuses(&set), ["in_progress", "pending", "pending"]); + assert!(presenter.present("s", done("p1")).is_none()); + + let advanced = presenter + .present("s", call(UPDATE_PLAN_TOOL, "p2", json!({"advance": true}))) + .unwrap(); + assert_eq!(statuses(&advanced), ["completed", "in_progress", "pending"]); + + let ticked = presenter + .present( + "s", + call(UPDATE_PLAN_TOOL, "p3", json!({"done": [2], "start": 3})), + ) + .unwrap(); + assert_eq!(statuses(&ticked), ["completed", "completed", "in_progress"]); + } + + #[test] + fn other_tools_empty_calls_and_unknown_sessions_pass_through() { + let mut presenter = Presenter::default(); + let empty = call(WRAP_UP_TOOL, "w", json!({})); + assert_eq!(presenter.present("s", empty.clone()), Some(empty)); + assert_eq!(presenter.present("s", done("w")), Some(done("w"))); + // An increment without a list to apply it to stays a plain tool call. + let orphan = call(UPDATE_PLAN_TOOL, "p", json!({"advance": true})); + assert_eq!(presenter.present("t", orphan.clone()), Some(orphan)); + let text = json!({"sessionUpdate": "agent_message_chunk", "content": {"type": "text", "text": "hi"}}); + assert_eq!(presenter.present("s", text.clone()), Some(text)); + } +} diff --git a/rust/lithe-agent-host/src/claude_earlier_history.rs b/rust/lithe-agent-host/src/claude_earlier_history.rs index 331ed6d2f5..0ac46fc635 100644 --- a/rust/lithe-agent-host/src/claude_earlier_history.rs +++ b/rust/lithe-agent-host/src/claude_earlier_history.rs @@ -143,13 +143,22 @@ impl Converter { let role = match row["type"].as_str() { Some("user") => Chunk::User, Some("assistant") => Chunk::Agent, + // A prompt sent while a turn was running is stored only as this + // attachment, never as a user row, so it is the message itself. + Some("attachment") if row["attachment"]["type"] == "queued_command" => { + return self.content(Chunk::User, &row["attachment"]["prompt"]); + } _ => return, }; // API error stand-ins are synthesized by the CLI, not model output. if row["message"]["model"] == "" { return; } - match &row["message"]["content"] { + self.content(role, &row["message"]["content"]); + } + + fn content(&mut self, role: Chunk, content: &Value) { + match content { Value::String(text) if role == Chunk::User => self.text(Chunk::User, text), Value::Array(blocks) => { for block in blocks { @@ -211,6 +220,7 @@ impl Converter { self.updates.push(json!({ "sessionUpdate": "tool_call", "toolCallId": id, + "name": name, "title": title(name, input), "kind": kind(name), "status": "completed", @@ -398,6 +408,22 @@ mod tests { assert_eq!(updates[1]["status"], "failed"); } + #[test] + fn prompts_queued_during_a_turn_and_tool_names_are_kept() { + let (dir, cwd) = transcript(&[ + json!({"type": "assistant", "message": {"content": [{"type": "tool_use", "id": "w", "name": "mcp__ccd_turn__wrap_up", "input": {"recap": "done"}}]}}), + json!({"type": "attachment", "uuid": "q1", "attachment": {"type": "queued_command", "prompt": "go faster"}}), + json!({"type": "attachment", "uuid": "r1", "attachment": {"type": "total_tokens_reminder"}}), + boundary(), + ]); + let updates = earlier_updates(dir.path(), &cwd, "abc-1"); + assert_eq!(updates.len(), 2); + assert_eq!(updates[0]["name"], "mcp__ccd_turn__wrap_up"); + assert_eq!(updates[1]["sessionUpdate"], "user_message_chunk"); + assert_eq!(updates[1]["content"]["text"], "go faster"); + assert_eq!(updates[1]["messageId"], "q1"); + } + #[test] fn long_text_is_cut_on_a_character_boundary() { let long = "é".repeat(TEXT_LIMIT + 10); diff --git a/rust/lithe-agent-host/src/lib.rs b/rust/lithe-agent-host/src/lib.rs index 4418f5dbf0..35a8619620 100644 --- a/rust/lithe-agent-host/src/lib.rs +++ b/rust/lithe-agent-host/src/lib.rs @@ -7,6 +7,7 @@ //! See `.agents/notes/implemented/architecture/2026-09-25-shared-acp-agent-conversation.md`. pub mod catalog; +mod claude_desktop; mod claude_earlier_history; mod claude_subscription; pub mod cli_update; @@ -1333,6 +1334,8 @@ where let session_meta = session_routing::metadata(gateway.as_ref())?; let workspace = Arc::new(workspace_context::WorkspaceContext::new(&cwd)?); let update_workspace = workspace.clone(); + let desktop = Arc::new(Mutex::new(claude_desktop::Presenter::default())); + let update_desktop = desktop.clone(); let restored_meta = session_routing::restored_metadata(gateway.as_ref())?; let (auth_tx, mut auth_rx) = tokio::sync::watch::channel(None::); let updates = emit.clone(); @@ -1377,6 +1380,9 @@ where let Some(update) = update_workspace.present_update(update) else { return Ok(()); }; + let Some(update) = update_desktop.lock().map_err(|_| internal("Replay presentation is unavailable"))?.present(&session_id, update) else { + return Ok(()); + }; if update["sessionUpdate"] == "usage_update" { if let Some(recovery) = &context_updates { recovery.prefer_upstream_report(&session_id); } } @@ -1678,6 +1684,7 @@ where let confirm_model = session_meta.is_some(); let capture = context_recovery.as_ref().and_then(|recovery| recovery.capture(&session_id)); let earlier = claude_projects.clone(); + let desktop = desktop.clone(); tasks.spawn(async move { // Compaction hides earlier segments from the adapter's replay; // they precede it, so the transcript reads in order. @@ -1688,6 +1695,11 @@ where }) .await .unwrap_or_default(); + // Collected first: the guard must not live across `emit`. + let updates: Vec<_> = match desktop.lock() { + Ok(mut desktop) => updates.into_iter().filter_map(|u| desktop.present(&session_id, u)).collect(), + Err(_) => updates, + }; for update in updates { emit(AgentEvent::Update { session_id: session_id.clone(), update }); }