feat: 搜索框语音输入（macOS 离线听写）；以声搜声按实测不上

.gitignore

@@ -44,6 +44,9 @@ coverage/
 # LiteRT-LM 原生库（scripts/download-models.mjs litert-runtime 下载）
 resources/litert
 
+# 语音转写辅助程序（scripts/build-speech-helper.mjs 编译产物，源码在 src/native/mac）
+resources/speech
+
 # Native modules cache
 .node-gyp/
 

CHANGELOG.md

@@ -24,6 +24,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   links). Available three ways: desktop Local API `/mcp`, server edition `/mcp`
   (enable with `VIXEL_MCP_TOKEN`), and `vixel mcp` on stdio for Claude Desktop /
   Cursor / Claude Code without any network service (setup snippet in the README).
+- **Voice input in the search bar (macOS 26+).** Click the microphone (or press it again to
+  stop), speak a query, and it is transcribed on-device into the search box — editable like
+  any typed text — then the normal text search runs. Transcription uses the system's offline
+  dictation model (SpeechAnalyzer) via a tiny helper process: nothing is recorded to disk
+  beyond a temp file that is deleted right away, and no audio ever leaves the Mac (the
+  browser Web Speech API, which uploads audio, is deliberately not used — so the web edition
+  has no microphone). Measured on 20 synthetic short queries: Chinese CER 6.4% (errors are
+  homophones), English WER 0%; voice and typed search agree on the top result 36/40 times
+  on the demo libraries. If the speech model for your language is not downloaded yet, Vixel
+  points you to System Settings → Keyboard → Dictation instead of downloading anything itself.
+  Windows is not covered yet (no offline engine met the quality bar).
+
+### Decided against (for now)
+- **Search by a sound clip.** Measured on the demo footage: audio queries against the joint
+  frames+audio segment vectors rank the right video first only 3/12 times (the picture
+  dominates the vector), and even pure audio-to-audio vectors cannot be thresholded — an
+  unrelated pink-noise clip scores higher than many true matches (0.947 vs 0.917). Without a
+  reliable "nothing really matches" gate the feature would return confidently wrong results,
+  so it stays out; the calibration script (`scripts/audio-search-eval.cjs`) documents the
+  numbers for re-evaluation with future models.
 
 ## [0.5.1] - 2026-10-15
 

build/entitlements.mac.plist

@@ -12,5 +12,8 @@
     <true/>
     <key>com.apple.security.files.downloads.read-write</key>
     <true/>
+    <!-- 语音搜索：麦克风（加固运行时下没有它 getUserMedia 直接失败；录音只在本机转写） -->
+    <key>com.apple.security.device.audio-input</key>
+    <true/>
 </dict>
 </plist>

electron-builder.yml

@@ -75,6 +75,11 @@ mac:
     NSDownloadsFolderUsageDescription: Vixel 只读访问你添加的文件夹，在本机为其中的照片和视频建立索引，不会修改或上传任何文件。
     NSRemovableVolumesUsageDescription: Vixel 只读访问移动硬盘上你添加的文件夹，在本机建立索引，不会修改或上传任何文件。
     NSNetworkVolumesUsageDescription: Vixel 只读访问网络磁盘上你添加的文件夹，在本机建立索引，不会修改或上传任何文件。
+    NSMicrophoneUsageDescription: Vixel 使用麦克风接收语音搜索；录音只在本机转写成文字，不会保存、不会上传。
+  # 语音转写辅助程序（scripts/build-speech-helper.mjs 编译；没编译出来也不影响打包，语音输入按不可用处理）
+  extraResources:
+    - from: resources/speech
+      to: speech
   # 公证：默认 false（dev / 无证书机器）。CI release 用 CLI override 打开：
   #   electron-builder --mac --c.mac.notarize=true
   # 配合 APPLE_ID / APPLE_APP_SPECIFIC_PASSWORD / APPLE_TEAM_ID 环境变量

package.json

@@ -14,11 +14,11 @@
   },
   "homepage": "https://github.com/RGB-loop/Vixel#readme",
   "scripts": {
-    "predev": "node scripts/dev-bundle.mjs",
+    "predev": "node scripts/dev-bundle.mjs && node scripts/build-speech-helper.mjs",
     "dev": "electron-vite dev",
     "build": "electron-vite build",
     "preview": "electron-vite preview",
-    "prepackage": "npm run build:web",
+    "prepackage": "npm run build:web && node scripts/build-speech-helper.mjs",
     "package": "electron-builder",
     "postinstall": "electron-builder install-app-deps",
     "typecheck": "tsc --noEmit -p tsconfig.node.json && tsc --noEmit -p tsconfig.web.json",

scripts/audio-search-eval.cjs

@@ -0,0 +1,185 @@
+// 以声搜声（sound-to-sound）可行性标定：外部音频查询（另截的 8s 音轨 / 合成音 / 语音）
+// 对库里向量的余弦 / z 分布。两种比对：
+//   eval  = 查询 ↔ 音视频片段向量（8 帧 + 音轨联合编码，库里现有的索引）
+//   audio = 查询 ↔ 纯音频向量（把有音轨的素材按 0-32s 音轨 audio-only 编码，模拟音频文件库）
+//
+// 结论（2026-10-18，EmbeddingGemma 2 · LiteRT，素材库 22 条 / 32 段 + 演示库拷贝）：两路都过不了相关性门槛，功能未上。
+//   eval：正例目标 z 0.20–2.71 / 余弦 0.705–0.813，top-1 仅 3/12；负例（合成音 / 跨库）z 最高 3.34、
+//        余弦最高 0.783 —— 画面主导联合向量，声音信号被淹没，正负例完全不可分。
+//   audio：top-1 8/12；正例最低余弦 0.917 < 非目标最高 0.959（粉噪对瀑布 0.947、440Hz 对野马 0.856）——
+//        纯音频空间过于紧致，连"同一段录音的另一截"都标不出可用门槛。
+//   因此"没有明显匹配就直说"的门槛不存在，按产品原则不上这个功能（见 PR 说明）。
+//
+// 用法（先 npm run build:cli；模型推理较慢，每种模式约 2-3 分钟）：
+//   ELECTRON_RUN_AS_NODE=1 npx electron scripts/audio-search-eval.cjs gen     # 生成查询音频到 /tmp/vx-as-q
+//   ELECTRON_RUN_AS_NODE=1 npx electron scripts/audio-search-eval.cjs eval    # 查询 ↔ 片段向量
+//   ELECTRON_RUN_AS_NODE=1 npx electron scripts/audio-search-eval.cjs audio   # 查询 ↔ 纯音频向量
+// 库（拷贝，不动原件）：/tmp/vx-voice-footage（22 条素材、32 段）、/tmp/vx-voice-data（演示库 34 图 + 3 段）
+const { join } = require('path')
+const { mkdirSync, writeFileSync, readdirSync, existsSync } = require('fs')
+const { execFileSync } = require('child_process')
+const root = join(__dirname, '..')
+const { initDatabase } = require(join(root, 'dist/cli/core/db'))
+const { extractAudioTrack } = require(join(root, 'dist/cli/core/audio/extract'))
+const { initEmbeddingServicePath, getEmbeddingService } = require(join(root, 'dist/cli/core/embedding'))
+
+const QDIR = '/tmp/vx-as-q'
+const LIBS = { footage: '/tmp/vx-voice-footage/library.db', demo: '/tmp/vx-voice-data/library.db' }
+const FOOTAGE = join(process.env.HOME, 'workspace/dancer/vixel-demo/footage')
+
+// 有音轨的素材（37% 时长处另截 8s，不与索引的 32s 片段网格对齐）
+const POSITIVE_CLIPS = [
+  'rain-sonoma.mp4', 'waterfall-delaneys.mp4', 'waterfall-watagataki.mp4', 'seagull-chicks.mp4',
+  'train-zagreb.mp4', 'fireworks-kamakura.mp4', 'rocket-falcon9.mp4', 'cat-curious.mp4',
+  'cooking-alioli.mp4', 'horse-wagons.mp4', 'birds-flying.mp4', 'wild-horses.mp4',
+]
+// 合成负例：和任何库内容都无关的声音（+ say 合成的中英文语音，模拟误把语音当声音查询）
+const SYNTH_NEGATIVES = [
+  ['tone-440.wav', (ffmpeg, out) => execFileSync(ffmpeg, ['-y', '-f', 'lavfi', '-i', 'sine=frequency=440:duration=8', '-ar', '16000', '-ac', '1', out], { stdio: 'ignore' })],
+  ['noise-pink.wav', (ffmpeg, out) => execFileSync(ffmpeg, ['-y', '-f', 'lavfi', '-i', 'anoisesrc=color=pink:duration=8', '-ar', '16000', '-ac', '1', out], { stdio: 'ignore' })],
+  ['speech-zh.wav', (_f, out) => execFileSync('say', ['-v', 'Tingting', '--data-format=LEI16@16000', '-o', out, '请把今天会议的录音整理成文字稿'], { stdio: 'ignore' })],
+  ['speech-en.wav', (_f, out) => execFileSync('say', ['-v', 'Samantha', '--data-format=LEI16@16000', '-o', out, 'please transcribe the recording of the meeting'], { stdio: 'ignore' })],
+]
+// 跨库负例：footage 的查询拿去搜演示库（3 段：无主题重合）
+const CROSS_NEGATIVES = ['clip-rain-sonoma.wav', 'clip-seagull-chicks.wav', 'clip-train-zagreb.wav',
+  'clip-fireworks-kamakura.wav', 'clip-rocket-falcon9.wav', 'clip-cooking-alioli.wav']
+
+function dot(a, b) { let s = 0; for (let i = 0; i < a.length; i++) s += a[i] * b[i]; return s }
+function stats(sims) {
+  const mean = sims.reduce((a, b) => a + b, 0) / sims.length
+  const sd = Math.sqrt(sims.reduce((a, b) => a + (b - mean) ** 2, 0) / sims.length) || 1e-6
+  return { mean, sd }
+}
+const toF32 = (buf) => { const b = Buffer.from(buf); return new Float32Array(b.buffer, b.byteOffset, b.length / 4) }
+
+function loadLib(dbPath) {
+  // 拷贝库可能是老 schema（缺新列）：读写打开让迁移补齐 —— 只动 /tmp 下的拷贝，从不碰原库
+  const inst = initDatabase(dbPath)
+  const images = inst.raw.prepare(`SELECT v.embedding e FROM image_vec_map m JOIN image_vecs v ON v.rowid = m.rowid`).all()
+    .map((r) => toF32(r.e))
+  const segments = inst.raw.prepare(`
+    SELECT s.video_id vid, s.start_ms s0, vd.file_name name, v.embedding e
+    FROM video_segment_vec_map m JOIN video_segment_vecs v ON v.rowid = m.rowid
+    JOIN video_segments s ON s.id = m.segment_id JOIN videos vd ON vd.id = s.video_id`).all()
+    .map((r) => ({ name: r.name, start: r.s0, v: toF32(r.e) }))
+  const videos = inst.raw.prepare(`SELECT file_name n, duration_ms d FROM videos WHERE deleted_at IS NULL`).all()
+  inst.close()
+  return { images, segments, videos }
+}
+
+function gen() {
+  mkdirSync(QDIR, { recursive: true })
+  const ffmpeg = require(join(root, 'node_modules/ffmpeg-static'))
+  const { videos } = loadLib(LIBS.footage)
+  for (const name of POSITIVE_CLIPS) {
+    const v = videos.find((x) => x.n === name)
+    const t = ((v?.d ?? 20000) * 0.37) / 1000
+    const out = join(QDIR, `clip-${name.replace(/\.\w+$/, '')}.wav`)
+    execFileSync(ffmpeg, ['-y', '-ss', t.toFixed(2), '-i', join(FOOTAGE, name), '-t', '8', '-vn', '-ac', '1', '-ar', '16000', out], { stdio: 'ignore' })
+    console.log('clip', out, `@${t.toFixed(1)}s`)
+  }
+  for (const [name, make] of SYNTH_NEGATIVES) {
+    make(ffmpeg, join(QDIR, name))
+    console.log('synth', name)
+  }
+  return Promise.resolve()
+}
+
+let svc
+async function encodeWav(path) {
+  if (!svc) {
+    initEmbeddingServicePath('/tmp/vx-as-userdata', join(root, 'resources/models'), join(root, 'resources/litert/darwin-arm64'))
+    svc = getEmbeddingService()
+    await svc.init()
+  }
+  const samples = await extractAudioTrack(path, { durationSec: 30 })
+  return svc.encode({ type: 'audio', samples })
+}
+
+/** 与 SearchEngine.baselineSample 同款：片段样本 < 20 时并入图片样本（同一嵌入空间） */
+function segStats(q, lib) {
+  const sample = lib.segments.map((x) => x.v)
+  const merged = sample.length >= 20 ? sample : [...sample, ...lib.images]
+  return stats(merged.map((v) => dot(v, q)))
+}
+
+function scoreAgainst(q, lib) {
+  const st = segStats(q, lib)
+  return lib.segments.map((x) => ({ name: x.name, start: x.start, cos: dot(x.v, q) }))
+    .map((x) => ({ ...x, z: (x.cos - st.mean) / st.sd })).sort((a, b) => b.z - a.z)
+}
+
+const fmt = (x) => x ? `${x.name}@${x.start / 1000}s cos ${x.cos.toFixed(3)} z ${x.z.toFixed(2)}` : '—'
+
+async function evalAll() {
+  const libs = { footage: loadLib(LIBS.footage), demo: loadLib(LIBS.demo) }
+  const rows = []
+  for (const qf of readdirSync(QDIR).filter((f) => f.startsWith('clip-'))) {
+    const q = await encodeWav(join(QDIR, qf))
+    const seg = scoreAgainst(q, libs.footage)
+    const tgt = qf.replace(/^clip-/, '').replace(/\.wav$/, '')
+    const isT = (x) => x.name.startsWith(tgt)
+    rows.push({ q: qf, kind: 'pos', target: seg.find(isT), best: seg[0], other: seg.find((x) => !isT(x)) })
+    console.log(`[pos] ${qf}`)
+    console.log(`   目标: ${fmt(seg.find(isT))} | 他片: ${fmt(seg.find((x) => !isT(x)))} | top: ${seg[0]?.name}`)
+  }
+  for (const [qf] of SYNTH_NEGATIVES) {
+    const q = await encodeWav(join(QDIR, qf))
+    const seg = scoreAgainst(q, libs.footage)
+    rows.push({ q: qf, kind: 'neg-synth', best: seg[0] })
+    console.log(`[neg-synth] ${qf} → footage: ${fmt(seg[0])}`)
+  }
+  for (const qf of CROSS_NEGATIVES) {
+    if (!existsSync(join(QDIR, qf))) continue
+    const q = await encodeWav(join(QDIR, qf))
+    const seg = scoreAgainst(q, libs.demo)
+    rows.push({ q: qf, kind: 'neg-cross', best: seg[0] })
+    console.log(`[neg-cross] ${qf} → demo: ${fmt(seg[0])}`)
+  }
+  writeFileSync(join(QDIR, 'eval.json'), JSON.stringify(rows, null, 1))
+  const pos = rows.filter((r) => r.kind === 'pos' && r.target)
+  const neg = rows.filter((r) => r.kind !== 'pos' && r.best)
+  const top1 = pos.filter((r) => r.best && r.target && r.best.name === r.target.name).length
+  console.log(`\n== 正例目标 z ${Math.min(...pos.map((r) => r.target.z)).toFixed(2)}–${Math.max(...pos.map((r) => r.target.z)).toFixed(2)}` +
+    ` 余弦 ${Math.min(...pos.map((r) => r.target.cos)).toFixed(3)}–${Math.max(...pos.map((r) => r.target.cos)).toFixed(3)}（top-1 ${top1}/${pos.length}）`)
+  console.log(`== 负例最高 z ${Math.max(...neg.map((r) => r.best.z)).toFixed(2)} 余弦最高 ${Math.max(...neg.map((r) => r.best.cos)).toFixed(3)}`)
+}
+
+const mode = process.argv[2] || 'eval'
+;(mode === 'gen' ? gen() : mode === 'audio' ? evalAudioOnly() : evalAll())
+  .then(() => process.exit(0))
+  .catch((e) => { console.error(e); process.exit(1) })
+
+/** audio 模式：把有音轨的素材按 0-32s 音轨 audio-only 编码当作"音频文件库"，同一批查询打分 */
+async function evalAudioOnly() {
+  const clips = [...POSITIVE_CLIPS, 'rocket-jason3.mp4', 'earth-night-lights.mp4']
+  const lib = []
+  for (const c of clips) {
+    const samples = await extractAudioTrack(join(FOOTAGE, c), { durationSec: 32 })
+    if (samples.length === 0) { console.log('skip silent', c); continue }
+    if (!svc) await encodeWav(join(QDIR, 'tone-440.wav')) // 初始化服务（结果不用）
+    lib.push({ name: c, v: await svc.encode({ type: 'audio', samples }) })
+  }
+  console.log('lib:', lib.length, 'audio-only vectors')
+  let top1 = 0, posN = 0, posMinCos = 1, nonTargetMaxCos = 0
+  for (const qf of readdirSync(QDIR).filter((f) => f.endsWith('.wav'))) {
+    const q = await encodeWav(join(QDIR, qf))
+    const st = stats(lib.map((x) => dot(x.v, q)))
+    const scored = lib.map((x) => ({ name: x.name, cos: dot(x.v, q) }))
+      .map((x) => ({ ...x, z: (x.cos - st.mean) / st.sd })).sort((a, b) => b.cos - a.cos)
+    const tgt = qf.replace(/^clip-/, '').replace(/\.wav$/, '')
+    const isT = (x) => x.name.startsWith(tgt)
+    if (qf.startsWith('clip-')) {
+      posN++
+      const t = scored.find(isT); const other = scored.find((x) => !isT(x))
+      if (scored[0] && isT(scored[0])) top1++
+      posMinCos = Math.min(posMinCos, t.cos)
+      nonTargetMaxCos = Math.max(nonTargetMaxCos, other.cos)
+      console.log(`[pos] ${qf}: 目标 cos ${t.cos.toFixed(3)} z ${t.z.toFixed(2)} | 次名 ${other.name} cos ${other.cos.toFixed(3)} z ${other.z.toFixed(2)}`)
+    } else {
+      nonTargetMaxCos = Math.max(nonTargetMaxCos, scored[0].cos)
+      console.log(`[neg] ${qf}: top ${scored[0].name} cos ${scored[0].cos.toFixed(3)} z ${scored[0].z.toFixed(2)}`)
+    }
+  }
+  console.log(`\ntop-1 ${top1}/${posN} | 正例最低余弦 ${posMinCos.toFixed(3)} | 非目标最高余弦 ${nonTargetMaxCos.toFixed(3)}`)
+}

scripts/build-speech-helper.mjs

@@ -0,0 +1,29 @@
+// 编译语音转写辅助程序（macOS 专用，见 src/native/mac/vixel-speech.swift）。
+// predev / prepackage 时跑：源码没变就跳过（mtime 比较），非 macOS / 没有 swiftc 时静默无事。
+// 产物 resources/speech/vixel-speech 不进 git（.gitignore），打包经 electron-builder.yml
+// 的 mac.extraResources 落到 <resources>/speech/。
+import { execFileSync } from 'node:child_process'
+import { existsSync, mkdirSync, statSync } from 'node:fs'
+import { join, dirname } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const root = dirname(dirname(fileURLToPath(import.meta.url)))
+const src = join(root, 'src/native/mac/vixel-speech.swift')
+const outDir = join(root, 'resources/speech')
+const out = join(outDir, 'vixel-speech')
+
+if (process.platform !== 'darwin') process.exit(0)
+try {
+  execFileSync('xcrun', ['-f', 'swiftc'], { stdio: 'ignore' })
+} catch {
+  console.warn('[speech-helper] swiftc not found — voice input will be unavailable in this build')
+  process.exit(0)
+}
+if (existsSync(out) && statSync(out).mtimeMs >= statSync(src).mtimeMs) process.exit(0)
+
+mkdirSync(outDir, { recursive: true })
+// 经 xcrun 调用（带上 SDKROOT；直接跑 toolchain 里的 swiftc 找不到标准库）。
+// 部署目标 = macOS 26（SpeechAnalyzer 的最低系统）；更老的系统上二进制加载不了 →
+// 主进程把 spawn 失败归为"系统不支持"（见 src/main/voice.ts）。
+execFileSync('xcrun', ['swiftc', '-O', '-target', `${process.arch === 'arm64' ? 'arm64' : 'x86_64'}-apple-macos26.0`, src, '-o', out], { stdio: 'inherit' })
+console.log('[speech-helper] built', out)

src/main/index.ts

@@ -28,6 +28,7 @@ import { registerTagIpc } from './tags'
 import { registerCollectIpc } from './collect'
 import { registerAutoTags } from './autotags'
 import { registerPdfRenderer, disposePdfRenderer, releasePdfWindowIfLast, PDF_ASSET_SCHEME } from './pdfRender'
+import { voiceStatus, voiceTranscribe, requestMicAccess } from './voice'
 import { timed, watchEventLoop, perfSnapshot, PROFILE } from '../core/perf'
 import { setInferenceTransport } from '../core/inference/transport'
 import { initDeepLinks, activateDeepLinks } from './deeplink'
@@ -485,6 +486,11 @@ function registerIpcHandlers(): void {
   // 以图搜图：拖入 / 粘贴的查询图只在内存和临时文件里处理，不进资料库（见 core/imageSearch.ts）
   handle(IPC_CHANNELS.SEARCH_BY_IMAGE, async (_event, input: ImageQueryInput, limit?: number, options?: MediaFilters) => api.searchByImage(input, limit, options))
 
+  // 语音输入（macOS 系统级离线听写；录音在渲染进程，转写在 vixel-speech 辅助进程，见 ./voice.ts）
+  handle(IPC_CHANNELS.VOICE_STATUS, async () => voiceStatus())
+  handle(IPC_CHANNELS.VOICE_REQUEST_MIC, async () => requestMicAccess())
+  handle(IPC_CHANNELS.VOICE_TRANSCRIBE, async (_event, wav: Uint8Array) => voiceTranscribe(wav))
+
   // 文件夹选择对话框是桌面专属；增删文件夹在服务层
   handle(IPC_CHANNELS.SELECT_FOLDER, async () => {
     const result = await dialog.showOpenDialog(BrowserWindow.getFocusedWindow() ?? mainWindow!, {