From 2d75e7ca3caf2328e33d57a130220a32ae3fc45a Mon Sep 17 00:00:00 2001 From: RGB-loop <91311486+RGB-loop@users.noreply.github.com> Date: Sun, 18 Oct 2026 01:04:24 +0800 Subject: [PATCH 1/3] =?UTF-8?q?eval(sound):=20sound-to-sound=20=E7=9B=B8?= =?UTF-8?q?=E5=85=B3=E6=80=A7=E6=A0=87=E5=AE=9A=20=E2=80=94=E2=80=94=20?= =?UTF-8?q?=E4=B8=A4=E8=B7=AF=E9=83=BD=E8=BF=87=E4=B8=8D=E4=BA=86=E9=97=A8?= =?UTF-8?q?=E6=A7=9B=EF=BC=8C=E4=BB=A5=E5=A3=B0=E6=90=9C=E5=A3=B0=E4=B8=8D?= =?UTF-8?q?=E4=B8=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 查询 ↔ 片段向量（8 帧 + 音轨联合编码）：top-1 3/12，正例 z 0.20–2.71 vs 负例 z 最高 3.34， 画面主导联合向量，声音被淹没；查询 ↔ 纯音频向量：top-1 8/12，正例最低余弦 0.917 < 非目标最高 0.959（粉噪对瀑布 0.947）。两路都标不出'没有明显匹配就直说'的门槛， 按产品原则不上（脚本留作证据与复测入口）。 --- scripts/audio-search-eval.cjs | 185 ++++++++++++++++++++++++++++++++++ 1 file changed, 185 insertions(+) create mode 100644 scripts/audio-search-eval.cjs diff --git a/scripts/audio-search-eval.cjs b/scripts/audio-search-eval.cjs new file mode 100644 index 0000000..cc01716 --- /dev/null +++ b/scripts/audio-search-eval.cjs @@ -0,0 +1,185 @@ +// 以声搜声（sound-to-sound）可行性标定：外部音频查询（另截的 8s 音轨 / 合成音 / 语音） +// 对库里向量的余弦 / z 分布。两种比对： +// eval = 查询 ↔ 音视频片段向量（8 帧 + 音轨联合编码，库里现有的索引） +// audio = 查询 ↔ 纯音频向量（把有音轨的素材按 0-32s 音轨 audio-only 编码，模拟音频文件库） +// +// 结论（2026-10-18，EmbeddingGemma 2 · LiteRT，素材库 22 条 / 32 段 + 演示库拷贝）：两路都过不了相关性门槛，功能未上。 +// eval：正例目标 z 0.20–2.71 / 余弦 0.705–0.813，top-1 仅 3/12；负例（合成音 / 跨库）z 最高 3.34、 +// 余弦最高 0.783 —— 画面主导联合向量，声音信号被淹没，正负例完全不可分。 +// audio：top-1 8/12；正例最低余弦 0.917 < 非目标最高 0.959（粉噪对瀑布 0.947、440Hz 对野马 0.856）—— +// 纯音频空间过于紧致，连"同一段录音的另一截"都标不出可用门槛。 +// 因此"没有明显匹配就直说"的门槛不存在，按产品原则不上这个功能（见 PR 说明）。 +// +// 用法（先 npm run build:cli；模型推理较慢，每种模式约 2-3 分钟）： +// ELECTRON_RUN_AS_NODE=1 npx electron scripts/audio-search-eval.cjs gen # 生成查询音频到 /tmp/vx-as-q +// ELECTRON_RUN_AS_NODE=1 npx electron scripts/audio-search-eval.cjs eval # 查询 ↔ 片段向量 +// ELECTRON_RUN_AS_NODE=1 npx electron scripts/audio-search-eval.cjs audio # 查询 ↔ 纯音频向量 +// 库（拷贝，不动原件）：/tmp/vx-voice-footage（22 条素材、32 段）、/tmp/vx-voice-data（演示库 34 图 + 3 段） +const { join } = require('path') +const { mkdirSync, writeFileSync, readdirSync, existsSync } = require('fs') +const { execFileSync } = require('child_process') +const root = join(__dirname, '..') +const { initDatabase } = require(join(root, 'dist/cli/core/db')) +const { extractAudioTrack } = require(join(root, 'dist/cli/core/audio/extract')) +const { initEmbeddingServicePath, getEmbeddingService } = require(join(root, 'dist/cli/core/embedding')) + +const QDIR = '/tmp/vx-as-q' +const LIBS = { footage: '/tmp/vx-voice-footage/library.db', demo: '/tmp/vx-voice-data/library.db' } +const FOOTAGE = join(process.env.HOME, 'workspace/dancer/vixel-demo/footage') + +// 有音轨的素材（37% 时长处另截 8s，不与索引的 32s 片段网格对齐） +const POSITIVE_CLIPS = [ + 'rain-sonoma.mp4', 'waterfall-delaneys.mp4', 'waterfall-watagataki.mp4', 'seagull-chicks.mp4', + 'train-zagreb.mp4', 'fireworks-kamakura.mp4', 'rocket-falcon9.mp4', 'cat-curious.mp4', + 'cooking-alioli.mp4', 'horse-wagons.mp4', 'birds-flying.mp4', 'wild-horses.mp4', +] +// 合成负例：和任何库内容都无关的声音（+ say 合成的中英文语音，模拟误把语音当声音查询） +const SYNTH_NEGATIVES = [ + ['tone-440.wav', (ffmpeg, out) => execFileSync(ffmpeg, ['-y', '-f', 'lavfi', '-i', 'sine=frequency=440:duration=8', '-ar', '16000', '-ac', '1', out], { stdio: 'ignore' })], + ['noise-pink.wav', (ffmpeg, out) => execFileSync(ffmpeg, ['-y', '-f', 'lavfi', '-i', 'anoisesrc=color=pink:duration=8', '-ar', '16000', '-ac', '1', out], { stdio: 'ignore' })], + ['speech-zh.wav', (_f, out) => execFileSync('say', ['-v', 'Tingting', '--data-format=LEI16@16000', '-o', out, '请把今天会议的录音整理成文字稿'], { stdio: 'ignore' })], + ['speech-en.wav', (_f, out) => execFileSync('say', ['-v', 'Samantha', '--data-format=LEI16@16000', '-o', out, 'please transcribe the recording of the meeting'], { stdio: 'ignore' })], +] +// 跨库负例：footage 的查询拿去搜演示库（3 段：无主题重合） +const CROSS_NEGATIVES = ['clip-rain-sonoma.wav', 'clip-seagull-chicks.wav', 'clip-train-zagreb.wav', + 'clip-fireworks-kamakura.wav', 'clip-rocket-falcon9.wav', 'clip-cooking-alioli.wav'] + +function dot(a, b) { let s = 0; for (let i = 0; i < a.length; i++) s += a[i] * b[i]; return s } +function stats(sims) { + const mean = sims.reduce((a, b) => a + b, 0) / sims.length + const sd = Math.sqrt(sims.reduce((a, b) => a + (b - mean) ** 2, 0) / sims.length) || 1e-6 + return { mean, sd } +} +const toF32 = (buf) => { const b = Buffer.from(buf); return new Float32Array(b.buffer, b.byteOffset, b.length / 4) } + +function loadLib(dbPath) { + // 拷贝库可能是老 schema（缺新列）：读写打开让迁移补齐 —— 只动 /tmp 下的拷贝，从不碰原库 + const inst = initDatabase(dbPath) + const images = inst.raw.prepare(`SELECT v.embedding e FROM image_vec_map m JOIN image_vecs v ON v.rowid = m.rowid`).all() + .map((r) => toF32(r.e)) + const segments = inst.raw.prepare(` + SELECT s.video_id vid, s.start_ms s0, vd.file_name name, v.embedding e + FROM video_segment_vec_map m JOIN video_segment_vecs v ON v.rowid = m.rowid + JOIN video_segments s ON s.id = m.segment_id JOIN videos vd ON vd.id = s.video_id`).all() + .map((r) => ({ name: r.name, start: r.s0, v: toF32(r.e) })) + const videos = inst.raw.prepare(`SELECT file_name n, duration_ms d FROM videos WHERE deleted_at IS NULL`).all() + inst.close() + return { images, segments, videos } +} + +function gen() { + mkdirSync(QDIR, { recursive: true }) + const ffmpeg = require(join(root, 'node_modules/ffmpeg-static')) + const { videos } = loadLib(LIBS.footage) + for (const name of POSITIVE_CLIPS) { + const v = videos.find((x) => x.n === name) + const t = ((v?.d ?? 20000) * 0.37) / 1000 + const out = join(QDIR, `clip-${name.replace(/\.\w+$/, '')}.wav`) + execFileSync(ffmpeg, ['-y', '-ss', t.toFixed(2), '-i', join(FOOTAGE, name), '-t', '8', '-vn', '-ac', '1', '-ar', '16000', out], { stdio: 'ignore' }) + console.log('clip', out, `@${t.toFixed(1)}s`) + } + for (const [name, make] of SYNTH_NEGATIVES) { + make(ffmpeg, join(QDIR, name)) + console.log('synth', name) + } + return Promise.resolve() +} + +let svc +async function encodeWav(path) { + if (!svc) { + initEmbeddingServicePath('/tmp/vx-as-userdata', join(root, 'resources/models'), join(root, 'resources/litert/darwin-arm64')) + svc = getEmbeddingService() + await svc.init() + } + const samples = await extractAudioTrack(path, { durationSec: 30 }) + return svc.encode({ type: 'audio', samples }) +} + +/** 与 SearchEngine.baselineSample 同款：片段样本 < 20 时并入图片样本（同一嵌入空间） */ +function segStats(q, lib) { + const sample = lib.segments.map((x) => x.v) + const merged = sample.length >= 20 ? sample : [...sample, ...lib.images] + return stats(merged.map((v) => dot(v, q))) +} + +function scoreAgainst(q, lib) { + const st = segStats(q, lib) + return lib.segments.map((x) => ({ name: x.name, start: x.start, cos: dot(x.v, q) })) + .map((x) => ({ ...x, z: (x.cos - st.mean) / st.sd })).sort((a, b) => b.z - a.z) +} + +const fmt = (x) => x ? `${x.name}@${x.start / 1000}s cos ${x.cos.toFixed(3)} z ${x.z.toFixed(2)}` : '—' + +async function evalAll() { + const libs = { footage: loadLib(LIBS.footage), demo: loadLib(LIBS.demo) } + const rows = [] + for (const qf of readdirSync(QDIR).filter((f) => f.startsWith('clip-'))) { + const q = await encodeWav(join(QDIR, qf)) + const seg = scoreAgainst(q, libs.footage) + const tgt = qf.replace(/^clip-/, '').replace(/\.wav$/, '') + const isT = (x) => x.name.startsWith(tgt) + rows.push({ q: qf, kind: 'pos', target: seg.find(isT), best: seg[0], other: seg.find((x) => !isT(x)) }) + console.log(`[pos] ${qf}`) + console.log(` 目标: ${fmt(seg.find(isT))} | 他片: ${fmt(seg.find((x) => !isT(x)))} | top: ${seg[0]?.name}`) + } + for (const [qf] of SYNTH_NEGATIVES) { + const q = await encodeWav(join(QDIR, qf)) + const seg = scoreAgainst(q, libs.footage) + rows.push({ q: qf, kind: 'neg-synth', best: seg[0] }) + console.log(`[neg-synth] ${qf} → footage: ${fmt(seg[0])}`) + } + for (const qf of CROSS_NEGATIVES) { + if (!existsSync(join(QDIR, qf))) continue + const q = await encodeWav(join(QDIR, qf)) + const seg = scoreAgainst(q, libs.demo) + rows.push({ q: qf, kind: 'neg-cross', best: seg[0] }) + console.log(`[neg-cross] ${qf} → demo: ${fmt(seg[0])}`) + } + writeFileSync(join(QDIR, 'eval.json'), JSON.stringify(rows, null, 1)) + const pos = rows.filter((r) => r.kind === 'pos' && r.target) + const neg = rows.filter((r) => r.kind !== 'pos' && r.best) + const top1 = pos.filter((r) => r.best && r.target && r.best.name === r.target.name).length + console.log(`\n== 正例目标 z ${Math.min(...pos.map((r) => r.target.z)).toFixed(2)}–${Math.max(...pos.map((r) => r.target.z)).toFixed(2)}` + + ` 余弦 ${Math.min(...pos.map((r) => r.target.cos)).toFixed(3)}–${Math.max(...pos.map((r) => r.target.cos)).toFixed(3)}（top-1 ${top1}/${pos.length}）`) + console.log(`== 负例最高 z ${Math.max(...neg.map((r) => r.best.z)).toFixed(2)} 余弦最高 ${Math.max(...neg.map((r) => r.best.cos)).toFixed(3)}`) +} + +const mode = process.argv[2] || 'eval' +;(mode === 'gen' ? gen() : mode === 'audio' ? evalAudioOnly() : evalAll()) + .then(() => process.exit(0)) + .catch((e) => { console.error(e); process.exit(1) }) + +/** audio 模式：把有音轨的素材按 0-32s 音轨 audio-only 编码当作"音频文件库"，同一批查询打分 */ +async function evalAudioOnly() { + const clips = [...POSITIVE_CLIPS, 'rocket-jason3.mp4', 'earth-night-lights.mp4'] + const lib = [] + for (const c of clips) { + const samples = await extractAudioTrack(join(FOOTAGE, c), { durationSec: 32 }) + if (samples.length === 0) { console.log('skip silent', c); continue } + if (!svc) await encodeWav(join(QDIR, 'tone-440.wav')) // 初始化服务（结果不用） + lib.push({ name: c, v: await svc.encode({ type: 'audio', samples }) }) + } + console.log('lib:', lib.length, 'audio-only vectors') + let top1 = 0, posN = 0, posMinCos = 1, nonTargetMaxCos = 0 + for (const qf of readdirSync(QDIR).filter((f) => f.endsWith('.wav'))) { + const q = await encodeWav(join(QDIR, qf)) + const st = stats(lib.map((x) => dot(x.v, q))) + const scored = lib.map((x) => ({ name: x.name, cos: dot(x.v, q) })) + .map((x) => ({ ...x, z: (x.cos - st.mean) / st.sd })).sort((a, b) => b.cos - a.cos) + const tgt = qf.replace(/^clip-/, '').replace(/\.wav$/, '') + const isT = (x) => x.name.startsWith(tgt) + if (qf.startsWith('clip-')) { + posN++ + const t = scored.find(isT); const other = scored.find((x) => !isT(x)) + if (scored[0] && isT(scored[0])) top1++ + posMinCos = Math.min(posMinCos, t.cos) + nonTargetMaxCos = Math.max(nonTargetMaxCos, other.cos) + console.log(`[pos] ${qf}: 目标 cos ${t.cos.toFixed(3)} z ${t.z.toFixed(2)} | 次名 ${other.name} cos ${other.cos.toFixed(3)} z ${other.z.toFixed(2)}`) + } else { + nonTargetMaxCos = Math.max(nonTargetMaxCos, scored[0].cos) + console.log(`[neg] ${qf}: top ${scored[0].name} cos ${scored[0].cos.toFixed(3)} z ${scored[0].z.toFixed(2)}`) + } + } + console.log(`\ntop-1 ${top1}/${posN} | 正例最低余弦 ${posMinCos.toFixed(3)} | 非目标最高余弦 ${nonTargetMaxCos.toFixed(3)}`) +} From 41a69182b02534c9f8bcfb4b29698da685f4225f Mon Sep 17 00:00:00 2001 From: RGB-loop <91311486+RGB-loop@users.noreply.github.com> Date: Sun, 18 Oct 2026 01:13:32 +0800 Subject: [PATCH 2/3] =?UTF-8?q?feat(voice):=20=E6=90=9C=E7=B4=A2=E6=A1=86?= =?UTF-8?q?=E8=AF=AD=E9=9F=B3=E8=BE=93=E5=85=A5=EF=BC=88macOS=20=E6=A1=8C?= =?UTF-8?q?=E9=9D=A2=E7=89=88=EF=BC=8C=E7=B3=BB=E7=BB=9F=E7=BA=A7=E7=A6=BB?= =?UTF-8?q?=E7=BA=BF=E5=90=AC=E5=86=99=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 语音 → 文字 → 正常文字搜索（直接拿语音向量搜索实测不可靠：中文失败、门槛不可分）。 转写走 macOS 26+ 的 SpeechAnalyzer（vixel-speech 辅助程序，swiftc 构建产物不进 git）： 模型是系统组件、全程本机运行、文件输入无需语音识别授权；模型未下载时不代下载， 界面指引去系统设置。实测合成查询 zh CER 6.4%（3/47 同音字错）、en WER 0%，单条 < 0.5 s。 引擎取舍：Embed2ASR（EG2 音频塔 + GRU-CTC reader，Apache-2.0）需要 pooling 前的 hidden states，LiteRT-LM C API 只给池化向量，跑不了；其公布中文 CER 15.6% 也不达标。 Windows / 网页版暂不提供（无达标离线引擎；Web Speech API 上传音频，违背离线承诺）。 渲染进程 getUserMedia 录音（≤15 s，电平表，Esc 取消）→ 16 kHz WAV 经 IPC 到主进程。 打包：mac extraResources + NSMicrophoneUsageDescription + audio-input entitlement。 --- .gitignore | 3 + build/entitlements.mac.plist | 3 + electron-builder.yml | 5 + package.json | 4 +- scripts/build-speech-helper.mjs | 29 +++++ src/main/index.ts | 6 + src/main/voice.ts | 89 +++++++++++++ src/native/mac/vixel-speech.swift | 72 ++++++++++ src/preload/index.ts | 11 +- src/renderer/src/components/SearchBar.tsx | 68 ++++++++++ src/renderer/src/lib/pcm.test.ts | 45 +++++++ src/renderer/src/lib/pcm.ts | 46 +++++++ src/renderer/src/lib/voiceCapture.ts | 152 ++++++++++++++++++++++ src/server/apiSurface.ts | 3 + src/shared/i18n/index.ts | 3 +- src/shared/i18n/voice.ts | 25 ++++ src/shared/types.ts | 22 ++++ src/web/api.ts | 8 ++ 18 files changed, 590 insertions(+), 4 deletions(-) create mode 100644 scripts/build-speech-helper.mjs create mode 100644 src/main/voice.ts create mode 100644 src/native/mac/vixel-speech.swift create mode 100644 src/renderer/src/lib/pcm.test.ts create mode 100644 src/renderer/src/lib/pcm.ts create mode 100644 src/renderer/src/lib/voiceCapture.ts create mode 100644 src/shared/i18n/voice.ts diff --git a/.gitignore b/.gitignore index c408a29..3e2d8a5 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,9 @@ coverage/ # LiteRT-LM 原生库（scripts/download-models.mjs litert-runtime 下载） resources/litert +# 语音转写辅助程序（scripts/build-speech-helper.mjs 编译产物，源码在 src/native/mac） +resources/speech + # Native modules cache .node-gyp/ diff --git a/build/entitlements.mac.plist b/build/entitlements.mac.plist index 84f85a3..a8b3593 100644 --- a/build/entitlements.mac.plist +++ b/build/entitlements.mac.plist @@ -12,5 +12,8 @@ com.apple.security.files.downloads.read-write + + com.apple.security.device.audio-input + diff --git a/electron-builder.yml b/electron-builder.yml index 74d8a04..49cfb8c 100644 --- a/electron-builder.yml +++ b/electron-builder.yml @@ -75,6 +75,11 @@ mac: NSDownloadsFolderUsageDescription: Vixel 只读访问你添加的文件夹，在本机为其中的照片和视频建立索引，不会修改或上传任何文件。 NSRemovableVolumesUsageDescription: Vixel 只读访问移动硬盘上你添加的文件夹，在本机建立索引，不会修改或上传任何文件。 NSNetworkVolumesUsageDescription: Vixel 只读访问网络磁盘上你添加的文件夹，在本机建立索引，不会修改或上传任何文件。 + NSMicrophoneUsageDescription: Vixel 使用麦克风接收语音搜索；录音只在本机转写成文字，不会保存、不会上传。 + # 语音转写辅助程序（scripts/build-speech-helper.mjs 编译；没编译出来也不影响打包，语音输入按不可用处理） + extraResources: + - from: resources/speech + to: speech # 公证：默认 false（dev / 无证书机器）。CI release 用 CLI override 打开： # electron-builder --mac --c.mac.notarize=true # 配合 APPLE_ID / APPLE_APP_SPECIFIC_PASSWORD / APPLE_TEAM_ID 环境变量 diff --git a/package.json b/package.json index 9bb170f..50c5e72 100644 --- a/package.json +++ b/package.json @@ -14,11 +14,11 @@ }, "homepage": "https://github.com/RGB-loop/Vixel#readme", "scripts": { - "predev": "node scripts/dev-bundle.mjs", + "predev": "node scripts/dev-bundle.mjs && node scripts/build-speech-helper.mjs", "dev": "electron-vite dev", "build": "electron-vite build", "preview": "electron-vite preview", - "prepackage": "npm run build:web", + "prepackage": "npm run build:web && node scripts/build-speech-helper.mjs", "package": "electron-builder", "postinstall": "electron-builder install-app-deps", "typecheck": "tsc --noEmit -p tsconfig.node.json && tsc --noEmit -p tsconfig.web.json", diff --git a/scripts/build-speech-helper.mjs b/scripts/build-speech-helper.mjs new file mode 100644 index 0000000..a694b6b --- /dev/null +++ b/scripts/build-speech-helper.mjs @@ -0,0 +1,29 @@ +// 编译语音转写辅助程序（macOS 专用，见 src/native/mac/vixel-speech.swift）。 +// predev / prepackage 时跑：源码没变就跳过（mtime 比较），非 macOS / 没有 swiftc 时静默无事。 +// 产物 resources/speech/vixel-speech 不进 git（.gitignore），打包经 electron-builder.yml +// 的 mac.extraResources 落到 /speech/。 +import { execFileSync } from 'node:child_process' +import { existsSync, mkdirSync, statSync } from 'node:fs' +import { join, dirname } from 'node:path' +import { fileURLToPath } from 'node:url' + +const root = dirname(dirname(fileURLToPath(import.meta.url))) +const src = join(root, 'src/native/mac/vixel-speech.swift') +const outDir = join(root, 'resources/speech') +const out = join(outDir, 'vixel-speech') + +if (process.platform !== 'darwin') process.exit(0) +try { + execFileSync('xcrun', ['-f', 'swiftc'], { stdio: 'ignore' }) +} catch { + console.warn('[speech-helper] swiftc not found — voice input will be unavailable in this build') + process.exit(0) +} +if (existsSync(out) && statSync(out).mtimeMs >= statSync(src).mtimeMs) process.exit(0) + +mkdirSync(outDir, { recursive: true }) +// 经 xcrun 调用（带上 SDKROOT；直接跑 toolchain 里的 swiftc 找不到标准库）。 +// 部署目标 = macOS 26（SpeechAnalyzer 的最低系统）；更老的系统上二进制加载不了 → +// 主进程把 spawn 失败归为"系统不支持"（见 src/main/voice.ts）。 +execFileSync('xcrun', ['swiftc', '-O', '-target', `${process.arch === 'arm64' ? 'arm64' : 'x86_64'}-apple-macos26.0`, src, '-o', out], { stdio: 'inherit' }) +console.log('[speech-helper] built', out) diff --git a/src/main/index.ts b/src/main/index.ts index 2eb35cf..a71e38c 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -28,6 +28,7 @@ import { registerTagIpc } from './tags' import { registerCollectIpc } from './collect' import { registerAutoTags } from './autotags' import { registerPdfRenderer, disposePdfRenderer, releasePdfWindowIfLast, PDF_ASSET_SCHEME } from './pdfRender' +import { voiceStatus, voiceTranscribe, requestMicAccess } from './voice' import { timed, watchEventLoop, perfSnapshot, PROFILE } from '../core/perf' import { setInferenceTransport } from '../core/inference/transport' import { initDeepLinks, activateDeepLinks } from './deeplink' @@ -485,6 +486,11 @@ function registerIpcHandlers(): void { // 以图搜图：拖入 / 粘贴的查询图只在内存和临时文件里处理，不进资料库（见 core/imageSearch.ts） handle(IPC_CHANNELS.SEARCH_BY_IMAGE, async (_event, input: ImageQueryInput, limit?: number, options?: MediaFilters) => api.searchByImage(input, limit, options)) + // 语音输入（macOS 系统级离线听写；录音在渲染进程，转写在 vixel-speech 辅助进程，见 ./voice.ts） + handle(IPC_CHANNELS.VOICE_STATUS, async () => voiceStatus()) + handle(IPC_CHANNELS.VOICE_REQUEST_MIC, async () => requestMicAccess()) + handle(IPC_CHANNELS.VOICE_TRANSCRIBE, async (_event, wav: Uint8Array) => voiceTranscribe(wav)) + // 文件夹选择对话框是桌面专属；增删文件夹在服务层 handle(IPC_CHANNELS.SELECT_FOLDER, async () => { const result = await dialog.showOpenDialog(BrowserWindow.getFocusedWindow() ?? mainWindow!, { diff --git a/src/main/voice.ts b/src/main/voice.ts new file mode 100644 index 0000000..dbadcb7 --- /dev/null +++ b/src/main/voice.ts @@ -0,0 +1,89 @@ +/** + * 语音搜索输入（macOS 桌面版）：渲染进程录音（getUserMedia → 16 kHz mono WAV），字节经 IPC + * 送到这里，写进临时文件后交给 vixel-speech 辅助程序（SpeechAnalyzer，系统级离线听写）转写。 + * 语音永远走 语音 → 文字 → 正常文字搜索：直接拿语音向量去搜实测不可靠 + * （top-1 4/6 vs 文字 5/6，中文失败、门槛余弦差只有 0.035，无法分辨"没有匹配"）。 + * + * 为什么用系统听写而不是自带 ASR 模型：候选过的 Embed2ASR（EmbeddingGemma 2 音频塔 + 小 reader， + * Apache-2.0）需要音频塔 pooling 之前的 hidden states（1536 维 / 25 fps），LiteRT-LM 的 C API + * 只给池化后的 768 维向量，我们的推理栈跑不了；上游只有 torch + transformers、无 ONNX 导出， + * 公布的中文 CER 15.6% 也到不了检索可用线。系统听写实测（20 条合成短查询 + 噪声样本，2026-10-18） + * zh CER 6.4% / en WER 0%，单条转写 < 0.5 s。 + * + * 隐私：模型是 macOS 的系统组件（与系统听写共享），全程本机运行；文件输入不需要语音识别授权 + * （SpeechAnalyzer 的设计），录音文件用完即删。语音模型未下载时**不代下载**（应用零出站请求）， + * 界面指引用户去 系统设置 → 键盘 → 听写。Windows / 网页版暂不提供：离线引擎未达质量线（见 PR 说明）。 + */ +import { execFile } from 'child_process' +import { mkdtemp, rm, writeFile } from 'fs/promises' +import { existsSync } from 'fs' +import { tmpdir } from 'os' +import { join } from 'path' +import { systemPreferences } from 'electron' +import { locale } from '../shared/i18n' +import type { VoiceStatus, VoiceTranscribeResult } from '../shared/types' + +/** 界面语言 → 听写语言。混说英文词的中文查询 zh-CN 听写也认得，反之亦然的能力弱一些 */ +const speechLocale = (): string => (locale === 'zh-CN' ? 'zh-CN' : 'en-US') + +/** 打包后在 /speech，开发时是 scripts/build-speech-helper.mjs 的产物目录 */ +function helperPath(): string { + const prod = join(process.resourcesPath ?? '', 'speech', 'vixel-speech') + if (process.resourcesPath && existsSync(prod)) return prod + return join(process.cwd(), 'resources', 'speech', 'vixel-speech') +} + +function runHelper(args: string[], timeout: number): Promise<{ code: number; stdout: string }> { + return new Promise((resolve, reject) => { + execFile(helperPath(), args, { timeout }, (err, stdout) => { + if (err && typeof (err as NodeJS.ErrnoException).code !== 'number') return reject(err) // spawn 失败 / 超时 + resolve({ code: err ? ((err as NodeJS.ErrnoException).code as unknown as number) : 0, stdout: String(stdout) }) + }) + }) +} + +export async function voiceStatus(): Promise { + if (process.platform !== 'darwin') return { available: false, reason: 'platform' } + if (!existsSync(helperPath())) return { available: false, reason: 'helper' } + try { + const { code, stdout } = await runHelper(['check', speechLocale()], 10_000) + if (code !== 0) return { available: false, reason: 'platform' } // exit 4 = 系统太旧 / 语言不支持 + const state = stdout.trim() + if (state === 'installed') return { available: true } + if (state === 'downloadable') return { available: false, reason: 'asset' } + return { available: false, reason: 'platform' } + } catch { + // spawn 不起来（更老的 macOS 加载不了部署目标 26 的二进制）与超时同判 + return { available: false, reason: 'platform' } + } +} + +/** 第一次点麦克风时请求系统麦克风权限（弹一次系统窗；拒绝过的直接 false，由界面指引去系统设置） */ +export async function requestMicAccess(): Promise { + if (process.platform !== 'darwin') return false + try { + return await systemPreferences.askForMediaAccess('microphone') + } catch { + return false + } +} + +export async function voiceTranscribe(wav: Uint8Array): Promise { + if (process.platform !== 'darwin' || !existsSync(helperPath())) return { error: 'unavailable' } + if (!wav || wav.byteLength === 0) return { error: 'failed' } + const dir = await mkdtemp(join(tmpdir(), 'vixel-voice-')) + try { + const p = join(dir, 'q.wav') + await writeFile(p, Buffer.isBuffer(wav) ? wav : Buffer.from(wav)) + // 转写实测 < 0.5 s；30 s 是模型冷加载的余量 + const { code, stdout } = await runHelper(['transcribe', speechLocale(), p], 30_000) + if (code === 3) return { error: 'asset-missing' } + if (code !== 0) return { error: 'failed' } + return { text: stdout.trim() } + } catch (err) { + console.warn('[voice] transcribe failed:', err) + return { error: 'failed' } + } finally { + await rm(dir, { recursive: true, force: true }).catch(() => {}) + } +} diff --git a/src/native/mac/vixel-speech.swift b/src/native/mac/vixel-speech.swift new file mode 100644 index 0000000..e3a8ac6 --- /dev/null +++ b/src/native/mac/vixel-speech.swift @@ -0,0 +1,72 @@ +// vixel-speech —— macOS 系统级离线听写辅助程序（语音搜索输入用，见 src/main/voice.ts）。 +// +// 用 macOS 26+ 的 SpeechAnalyzer / SpeechTranscriber 做"文件 → 文字"：模型是系统组件、 +// 全程在本机运行（与系统听写共享语音模型资产），文件输入不需要语音识别授权弹窗 +// （麦克风权限由 Electron 侧 getUserMedia 走，与本程序无关）。不自动下载语音模型 —— +// 应用承诺零出站请求，模型缺失时按 asset-missing 退出，由界面指引用户去系统设置下载。 +// +// 用法： +// vixel-speech check → stdout: installed | downloadable | unsupported +// vixel-speech transcribe → stdout: 识别出的文字（可为空 = 没听到语音） +// 退出码：0 成功 · 2 失败 · 3 语音模型未安装 · 4 系统太旧 / 语言不支持 +// +// 构建：scripts/build-speech-helper.mjs（swiftc，产物 resources/speech/vixel-speech，不进 git） +import Foundation +import Speech +import AVFoundation + +func die(_ code: Int32, _ msg: String) -> Never { + FileHandle.standardError.write(("vixel-speech: " + msg + "\n").data(using: .utf8)!) + exit(code) +} + +let args = CommandLine.arguments +guard args.count >= 3 else { die(2, "usage: vixel-speech check | transcribe ") } +let command = args[1] +let localeId = args[2] + +guard #available(macOS 26, *) else { die(4, "SpeechAnalyzer needs macOS 26+") } + +@available(macOS 26, *) +func run() async { + let locale = Locale(identifier: localeId) + let sameLocale = { (l: Locale) in l.identifier(.bcp47) == locale.identifier(.bcp47) } + let supported = await SpeechTranscriber.supportedLocales.contains(where: sameLocale) + let installed = await SpeechTranscriber.installedLocales.contains(where: sameLocale) + + if command == "check" { + print(!supported ? "unsupported" : installed ? "installed" : "downloadable") + exit(0) + } + guard command == "transcribe", args.count >= 4 else { die(2, "usage: vixel-speech transcribe ") } + if !supported { die(4, "locale \(localeId) not supported") } + if !installed { die(3, "speech model for \(localeId) not installed") } + + do { + let transcriber = SpeechTranscriber(locale: locale, transcriptionOptions: [], reportingOptions: [], attributeOptions: []) + let analyzer = SpeechAnalyzer(modules: [transcriber]) + let file = try AVAudioFile(forReading: URL(fileURLWithPath: args[3])) + // 结果流和分析并发收集：finalizeAndFinish 之后流才会结束 + async let text: String = { + var out = "" + for try await r in transcriber.results where r.isFinal { + out += String(r.text.characters) + } + return out + }() + if let last = try await analyzer.analyzeSequence(from: file) { + try await analyzer.finalizeAndFinish(through: last) + } else { + await analyzer.cancelAndFinishNow() + } + print(try await text.trimmingCharacters(in: .whitespacesAndNewlines)) + exit(0) + } catch { + die(2, "transcribe failed: \(error.localizedDescription)") + } +} + +let task = Task { await run() } +// run() 内部必定 exit；这里只是给异步任务留运行机会（60s 是兜底，主进程侧另有超时） +RunLoop.main.run(until: Date(timeIntervalSinceNow: 60)) +die(2, "timeout") diff --git a/src/preload/index.ts b/src/preload/index.ts index b9bdf93..fefc4d7 100644 --- a/src/preload/index.ts +++ b/src/preload/index.ts @@ -1,5 +1,5 @@ import { contextBridge, ipcRenderer, webUtils } from 'electron' -import { IPC_CHANNELS, type MediaFilters, type SmartAlbum, type SmartAlbumMenuAction, type SearchResult, type WatchedFolder, type PhotoDetail, type PhotoLocation, type IndexProgress, type ModelStatus, type EmbeddingConfig, type Person, type FaceRecord, type BackupStatus, type MediaKind, type TaskOverview, type MediaDetail, type LibraryCounts, type MenuCommand, type ItemMenuAction, type ThemeMode, type PersonSuggestion, type PersonFace, type ImageQueryInput, type ImageSearchResponse, type SkippedFolder } from '../shared/types' +import { IPC_CHANNELS, type MediaFilters, type SmartAlbum, type SmartAlbumMenuAction, type SearchResult, type WatchedFolder, type PhotoDetail, type PhotoLocation, type IndexProgress, type ModelStatus, type EmbeddingConfig, type Person, type FaceRecord, type BackupStatus, type MediaKind, type TaskOverview, type MediaDetail, type LibraryCounts, type MenuCommand, type ItemMenuAction, type ThemeMode, type PersonSuggestion, type PersonFace, type ImageQueryInput, type ImageSearchResponse, type SkippedFolder, type VoiceStatus, type VoiceTranscribeResult } from '../shared/types' import type { Locale, LocaleSetting } from '../shared/i18n' import { CLEANUP_CHANNELS, type DupScanResult, type DupScanProgress, type HiddenItem, type TrashResult } from '../shared/cleanup' import { COLLECT_CHANNELS, type CollectResult, type CollectServerStatus, type InboxInfo, type ItemSource } from '../shared/collect' @@ -27,6 +27,15 @@ const api = { getPathForFile: (file: File): string => { try { return webUtils.getPathForFile(file) } catch { return '' } }, + + // 语音搜索输入（macOS 系统级离线听写；其它平台 status.available 恒为 false，界面隐藏麦克风） + voice: { + status: (): Promise => ipcRenderer.invoke(IPC_CHANNELS.VOICE_STATUS), + /** 第一次录音前请求系统麦克风权限（弹一次系统窗；被拒过返回 false） */ + requestMic: (): Promise => ipcRenderer.invoke(IPC_CHANNELS.VOICE_REQUEST_MIC), + /** wav：渲染进程录好的 16 kHz mono 16-bit WAV 字节；录音只在本机转写，用完即删 */ + transcribe: (wav: Uint8Array): Promise => ipcRenderer.invoke(IPC_CHANNELS.VOICE_TRANSCRIBE, wav), + }, getPhotosWithGPS: (): Promise => { return ipcRenderer.invoke(IPC_CHANNELS.GET_PHOTOS_WITH_GPS) }, diff --git a/src/renderer/src/components/SearchBar.tsx b/src/renderer/src/components/SearchBar.tsx index e545da2..2c544d0 100644 --- a/src/renderer/src/components/SearchBar.tsx +++ b/src/renderer/src/components/SearchBar.tsx @@ -1,5 +1,6 @@ import { useState, useCallback, useRef, useEffect } from 'react' import { t } from '../../../shared/i18n' +import { useVoiceCapture } from '../lib/voiceCapture' interface SearchBarProps { /** 当前生效的查询；外部（侧边栏最近 / 已保存）改它时输入框跟着变 */ @@ -24,6 +25,15 @@ export function SearchBar({ value, onSearch, isSearching, resultCount, saved, sa const inputRef = useRef(null) const debounceRef = useRef() + // 语音输入（macOS 桌面版；其它平台 status.available 为 false，不渲染麦克风）： + // 语音 → 文字进搜索框（可编辑），随即跑一次正常文字搜索 + const voice = useVoiceCapture((text) => { + if (debounceRef.current) clearTimeout(debounceRef.current) + setQuery(text) + onSearch(text) + inputRef.current?.focus() + }) + const debouncedSearch = useCallback( (value: string) => { if (debounceRef.current) { @@ -85,6 +95,14 @@ export function SearchBar({ value, onSearch, isSearching, resultCount, saved, sa } }, []) + // 语音提示自动消失（点击提示本身立即关） + const { notice: voiceNotice, clearNotice: clearVoiceNotice } = voice + useEffect(() => { + if (!voiceNotice) return + const timer = setTimeout(clearVoiceNotice, 10_000) + return () => clearTimeout(timer) + }, [voiceNotice, clearVoiceNotice]) + // 搜索结果计数提示（以图搜图时没有文字也显示） const showResultHint = (query.trim() || image) && !isSearching && resultCount !== undefined @@ -144,6 +162,45 @@ export function SearchBar({ value, onSearch, isSearching, resultCount, saved, sa />
+ {/* 语音输入：仅在平台支持时出现（见 useVoiceCapture）；模型未下载时保留按钮、点击给指引 */} + {voice.status && (voice.status.available || voice.status.reason === 'asset') && ( + <> + {voice.state === 'recording' && ( + + + {/* 电平表：RMS 放大后限 0–1（语音一般只到 0.2 左右） */} + + + + + )} + + + )} {/* 上传查询图（网页版的以图搜图入口；桌面用拖入 / 粘贴 / 右键） */} {onPickImage && !image && (
+ + {/* 语音输入的错误 / 指引（权限、模型未下载等）：点击关闭，10s 自动消失（见上面的 useEffect） */} + {voice.notice && ( +
+ {voice.notice} +
+ )} ) } diff --git a/src/renderer/src/lib/pcm.test.ts b/src/renderer/src/lib/pcm.test.ts new file mode 100644 index 0000000..2a91857 --- /dev/null +++ b/src/renderer/src/lib/pcm.test.ts @@ -0,0 +1,45 @@ +import { describe, it, expect } from 'vitest' +import { resampleTo16k, encodeWavPcm16, rmsLevel } from './pcm' + +describe('resampleTo16k', () => { + it('16 kHz 输入原样拼接', () => { + const out = resampleTo16k([new Float32Array([0.1, 0.2]), new Float32Array([0.3])], 16000) + expect([...out].map((x) => x.toFixed(1))).toEqual(['0.1', '0.2', '0.3']) + }) + + it('48 kHz → 16 kHz：长度按 1/3，值取线性插值', () => { + const src = new Float32Array(48000).map((_, i) => Math.sin((i / 48000) * 2 * Math.PI * 440)) + const out = resampleTo16k([src], 48000) + expect(out.length).toBe(16000) + // 相位应与原信号一致：第 4000 个 16k 采样 = 0.25s 处，sin(2π·440·0.25) = 0 + expect(Math.abs(out[4000])).toBeLessThan(0.01) + }) + + it('空输入 → 空输出', () => { + expect(resampleTo16k([], 48000).length).toBe(0) + }) +}) + +describe('encodeWavPcm16', () => { + it('头部字段与数据长度正确，采样饱和截断', () => { + const wav = encodeWavPcm16(new Float32Array([0, 1, -1, 2]), 16000) + expect(wav.length).toBe(44 + 8) + const dv = new DataView(wav.buffer) + expect(String.fromCharCode(...wav.subarray(0, 4))).toBe('RIFF') + expect(String.fromCharCode(...wav.subarray(8, 12))).toBe('WAVE') + expect(dv.getUint32(24, true)).toBe(16000) // 采样率 + expect(dv.getUint16(22, true)).toBe(1) // 单声道 + expect(dv.getInt16(44, true)).toBe(0) + expect(dv.getInt16(46, true)).toBe(32767) + expect(dv.getInt16(48, true)).toBe(-32767) + expect(dv.getInt16(50, true)).toBe(32767) // 超界饱和 + }) +}) + +describe('rmsLevel', () => { + it('静音为 0，满幅方波为 1', () => { + expect(rmsLevel(new Float32Array(100))).toBe(0) + expect(rmsLevel(new Float32Array(100).fill(1))).toBeCloseTo(1) + expect(rmsLevel(new Float32Array(0))).toBe(0) + }) +}) diff --git a/src/renderer/src/lib/pcm.ts b/src/renderer/src/lib/pcm.ts new file mode 100644 index 0000000..f52ea2b --- /dev/null +++ b/src/renderer/src/lib/pcm.ts @@ -0,0 +1,46 @@ +/** + * 语音录音的纯转换：浏览器 AudioContext 采到的 Float32 PCM → 16 kHz mono 16-bit WAV 字节。 + * 不碰 DOM / 音频 API，方便单测（vitest，node 环境）；录音编排在 lib/voiceCapture.ts。 + */ + +/** 线性插值重采样（语音转写用，不追求音质；fromRate === 16000 时原样拼接） */ +export function resampleTo16k(chunks: Float32Array[], fromRate: number): Float32Array { + let total = 0 + for (const c of chunks) total += c.length + const joined = new Float32Array(total) + let off = 0 + for (const c of chunks) { joined.set(c, off); off += c.length } + if (fromRate === 16000 || joined.length === 0) return joined + const ratio = fromRate / 16000 + const out = new Float32Array(Math.floor(joined.length / ratio)) + for (let i = 0; i < out.length; i++) { + const pos = i * ratio + const i0 = Math.floor(pos) + const i1 = Math.min(i0 + 1, joined.length - 1) + out[i] = joined[i0] + (joined[i1] - joined[i0]) * (pos - i0) + } + return out +} + +/** mono Float32 [-1,1] → 16-bit PCM WAV（与 litertProvider.toWav 同格式，这里要在浏览器里跑） */ +export function encodeWavPcm16(samples: Float32Array, sampleRate = 16000): Uint8Array { + const out = new Uint8Array(44 + samples.length * 2) + const dv = new DataView(out.buffer) + const ascii = (off: number, s: string): void => { for (let i = 0; i < s.length; i++) out[off + i] = s.charCodeAt(i) } + ascii(0, 'RIFF'); dv.setUint32(4, 36 + samples.length * 2, true); ascii(8, 'WAVE') + ascii(12, 'fmt '); dv.setUint32(16, 16, true); dv.setUint16(20, 1, true); dv.setUint16(22, 1, true) + dv.setUint32(24, sampleRate, true); dv.setUint32(28, sampleRate * 2, true); dv.setUint16(32, 2, true); dv.setUint16(34, 16, true) + ascii(36, 'data'); dv.setUint32(40, samples.length * 2, true) + for (let i = 0; i < samples.length; i++) { + dv.setInt16(44 + i * 2, Math.round(Math.max(-1, Math.min(1, samples[i])) * 32767), true) + } + return out +} + +/** 一段采样的 RMS（电平表用；0–1 左右的量级） */ +export function rmsLevel(chunk: Float32Array): number { + if (chunk.length === 0) return 0 + let sum = 0 + for (let i = 0; i < chunk.length; i++) sum += chunk[i] * chunk[i] + return Math.sqrt(sum / chunk.length) +} diff --git a/src/renderer/src/lib/voiceCapture.ts b/src/renderer/src/lib/voiceCapture.ts new file mode 100644 index 0000000..746994f --- /dev/null +++ b/src/renderer/src/lib/voiceCapture.ts @@ -0,0 +1,152 @@ +/** + * 语音搜索输入的录音编排（SearchBar 的麦克风按钮用）。 + * + * 流程：getUserMedia → AudioContext + ScriptProcessor 采 Float32 PCM（顺带算电平）→ + * 停止后重采样到 16 kHz、编成 WAV（lib/pcm.ts，纯函数）→ window.api.voice.transcribe → + * 文字回调（交给正常的文字搜索）。≤ 15 s 自动停止；Esc 取消（丢弃录音）。 + * + * 隐私：音频只进内存和主进程的临时文件，转写用 macOS 系统离线听写（见 src/main/voice.ts）， + * 不碰浏览器的 Web Speech API（Chrome 会把音频送到 Google）。 + */ +import { useCallback, useEffect, useRef, useState } from 'react' +import { t } from '../../../shared/i18n' +import type { VoiceStatus } from '../../../shared/types' +import { resampleTo16k, encodeWavPcm16, rmsLevel } from './pcm' + +/** 最长录音时长：搜索查询都是短句，15 s 足够，也限住 IPC 字节量（15 s × 16 kHz × 2 B ≈ 470 KB） */ +const MAX_RECORD_MS = 15_000 + +export type VoiceUiState = 'idle' | 'recording' | 'transcribing' + +interface Session { + stream: MediaStream + ctx: AudioContext + source: MediaStreamAudioSourceNode + proc: ScriptProcessorNode + chunks: Float32Array[] + sampleRate: number + timer: ReturnType +} + +export interface VoiceCapture { + /** null = 状态还没取到 / 平台没有语音输入 → 不渲染麦克风 */ + status: VoiceStatus | null + state: VoiceUiState + /** 录音电平（RMS，0–1 量级），电平表用 */ + level: number + /** 错误 / 指引文案（已本地化）；点击或下一次操作时清掉 */ + notice: string | null + clearNotice: () => void + /** idle → 录音；recording → 停止并转写 */ + toggle: () => void +} + +export function useVoiceCapture(onText: (text: string) => void): VoiceCapture { + const [status, setStatus] = useState(null) + const [state, setState] = useState('idle') + const [level, setLevel] = useState(0) + const [notice, setNotice] = useState(null) + const session = useRef(null) + const onTextRef = useRef(onText) + onTextRef.current = onText + + useEffect(() => { + let mounted = true + window.api.voice.status().then((s) => { if (mounted) setStatus(s) }).catch(() => {}) + return () => { mounted = false } + }, []) + + const teardown = useCallback(() => { + const s = session.current + session.current = null + if (!s) return + clearTimeout(s.timer) + s.proc.onaudioprocess = null + try { s.source.disconnect(); s.proc.disconnect() } catch { /* 已断开 */ } + s.stream.getTracks().forEach((track) => track.stop()) + void s.ctx.close().catch(() => {}) + }, []) + + const cancel = useCallback(() => { + teardown() + setState('idle') + setLevel(0) + }, [teardown]) + + /** 停止录音 → 16 kHz WAV → 主进程转写 → 文字回调 */ + const finish = useCallback(async () => { + const s = session.current + if (!s) return + const { chunks, sampleRate } = s + teardown() + setState('transcribing') + try { + const wav = encodeWavPcm16(resampleTo16k(chunks, sampleRate)) + const r = await window.api.voice.transcribe(wav) + if (r.error === 'asset-missing') setNotice(t.voice.assetMissing) + else if (r.error) setNotice(t.voice.failed) + else if (!r.text) setNotice(t.voice.noSpeech) + else onTextRef.current(r.text) + } catch { + setNotice(t.voice.failed) + } finally { + setState('idle') + setLevel(0) + } + }, [teardown]) + + const start = useCallback(async () => { + setNotice(null) + // 模型未下载：按钮保留（可发现性），点击给指引而不是静默没反应 + if (status?.reason === 'asset') { setNotice(t.voice.assetMissing); return } + if (!status?.available) return + const allowed = await window.api.voice.requestMic().catch(() => false) + if (!allowed) { setNotice(t.voice.permissionDenied); return } + let stream: MediaStream + try { + stream = await navigator.mediaDevices.getUserMedia({ audio: true }) + } catch { + setNotice(t.voice.permissionDenied) + return + } + const ctx = new AudioContext() + const source = ctx.createMediaStreamSource(stream) + // ScriptProcessor 要接到 destination 才会被驱动；不往 outputBuffer 写就是静音，不会外放 + const proc = ctx.createScriptProcessor(4096, 1, 1) + const chunks: Float32Array[] = [] + proc.onaudioprocess = (e): void => { + const data = e.inputBuffer.getChannelData(0) + chunks.push(new Float32Array(data)) + setLevel(rmsLevel(data)) + } + source.connect(proc) + proc.connect(ctx.destination) + const timer = setTimeout(() => { void finish() }, MAX_RECORD_MS) + session.current = { stream, ctx, source, proc, chunks, sampleRate: ctx.sampleRate, timer } + setState('recording') + }, [status, finish]) + + const toggle = useCallback(() => { + if (state === 'recording') void finish() + else if (state === 'idle') void start() + }, [state, start, finish]) + + // 录音中按 Esc 取消（捕获阶段拦住，不让它顺带清掉查询 / 关掉浮层） + useEffect(() => { + if (state !== 'recording') return + const onKey = (e: KeyboardEvent): void => { + if (e.key !== 'Escape') return + e.preventDefault() + e.stopPropagation() + cancel() + } + window.addEventListener('keydown', onKey, true) + return () => window.removeEventListener('keydown', onKey, true) + }, [state, cancel]) + + // 卸载时收拾干净（关麦克风指示灯） + useEffect(() => teardown, [teardown]) + + const clearNotice = useCallback(() => setNotice(null), []) + return { status, state, level, notice, clearNotice, toggle } +} diff --git a/src/server/apiSurface.ts b/src/server/apiSurface.ts index cdd0c7b..f7536b9 100644 --- a/src/server/apiSurface.ts +++ b/src/server/apiSurface.ts @@ -84,6 +84,9 @@ export const IMPLEMENTED_METHODS = [ */ export const DESKTOP_ONLY_METHODS = [ 'getPathForFile', // webUtils.getPathForFile（渲染进程里的本地 File 对象） + 'voice.status', // 语音搜索输入：macOS 系统级离线听写（服务器 / 网页没有离线引擎，见 src/main/voice.ts） + 'voice.requestMic', + 'voice.transcribe', 'selectFolder', // 原生目录选择对话框 'openSettings', // 设置是独立窗口；网页版是路由 'showItemMenu', // 原生右键菜单 diff --git a/src/shared/i18n/index.ts b/src/shared/i18n/index.ts index e31862d..b526959 100644 --- a/src/shared/i18n/index.ts +++ b/src/shared/i18n/index.ts @@ -35,8 +35,9 @@ import design from './design' import quickpanel from './quickpanel' import localapi from './localapi' import web from './web' +import voice from './voice' -const DICTS = { common, shell, settings, menu, main, core, grid, inspector, detail, people, map, tasks, model, organize, albums, cleanup, tags, autotags, imagesearch, collect, design, quickpanel, localapi, web } +const DICTS = { common, shell, settings, menu, main, core, grid, inspector, detail, people, map, tasks, model, organize, albums, cleanup, tags, autotags, imagesearch, collect, design, quickpanel, localapi, web, voice } export type Locale = 'zh-CN' | 'en' export type LocaleSetting = 'system' | Locale diff --git a/src/shared/i18n/voice.ts b/src/shared/i18n/voice.ts new file mode 100644 index 0000000..4dc940b --- /dev/null +++ b/src/shared/i18n/voice.ts @@ -0,0 +1,25 @@ +/** 语音搜索输入（搜索框麦克风，macOS 桌面版；SearchBar / lib/voiceCapture.ts） */ +const zh = { + micTitle: '语音搜索', + // 录音中：按钮 title + 输入框占位 + stopTitle: '停止并转写（Esc 取消）', + listening: '正在听…', + transcribing: '正在转写…', + permissionDenied: '没有麦克风权限：到 系统设置 → 隐私与安全性 → 麦克风 里允许 Vixel 后重试。', + assetMissing: '这个语言的系统语音模型还没下载：到 系统设置 → 键盘 → 听写 启用后即可用（转写全程在本机进行）。', + noSpeech: '没有听到语音，再试一次。', + failed: '转写失败，再试一次。', +} + +const en: typeof zh = { + micTitle: 'Voice search', + stopTitle: 'Stop and transcribe (Esc to cancel)', + listening: 'Listening…', + transcribing: 'Transcribing…', + permissionDenied: 'No microphone access: allow Vixel under System Settings → Privacy & Security → Microphone, then try again.', + assetMissing: 'The system speech model for this language is not downloaded yet: enable it under System Settings → Keyboard → Dictation (transcription runs entirely on this Mac).', + noSpeech: 'No speech detected — try again.', + failed: 'Transcription failed — try again.', +} + +export default { zh, en } diff --git a/src/shared/types.ts b/src/shared/types.ts index 8dc57da..7e1cd35 100644 --- a/src/shared/types.ts +++ b/src/shared/types.ts @@ -192,6 +192,23 @@ export interface ImageSearchResponse { error?: 'decode-failed' | 'model-not-ready' } +/** + * 语音输入（搜索框麦克风，macOS 桌面版）：能否离线转写。说"不可用"的原因： + * platform = 不是 macOS 26+（系统离线听写 API 不存在） + * helper = 转写辅助程序缺失（打包 / 构建问题） + * asset = 这个语言的系统语音模型还没下载（系统设置 → 键盘 → 听写） + */ +export interface VoiceStatus { + available: boolean + reason?: 'platform' | 'helper' | 'asset' +} + +export interface VoiceTranscribeResult { + text?: string + /** asset-missing 单列：界面要给"去系统设置下载语音模型"的指引，而不是笼统的失败 */ + error?: 'unavailable' | 'asset-missing' | 'failed' +} + export interface CurrentTask { taskType: TaskType kind: MediaKind @@ -373,6 +390,11 @@ export const IPC_CHANNELS = { // 以图搜图（拖入 / 粘贴 / 右键"以此图搜索"） SEARCH_BY_IMAGE: 'search-by-image', + // 语音输入（macOS：系统级离线听写，见 src/main/voice.ts） + VOICE_STATUS: 'voice-status', + VOICE_REQUEST_MIC: 'voice-request-mic', + VOICE_TRANSCRIBE: 'voice-transcribe', + // 地图 GET_PHOTOS_WITH_GPS: 'get-photos-with-gps', diff --git a/src/web/api.ts b/src/web/api.ts index 4373a68..14ab3b3 100644 --- a/src/web/api.ts +++ b/src/web/api.ts @@ -146,6 +146,14 @@ export async function createWebApi(): Promise { return Promise.reject(new Error('searchByImage by local path is not available in the web edition')) }, getPathForFile: (): string => '', + + // 语音输入：网页版没有离线转写引擎（浏览器 Web Speech API 会把音频送云端，违背离线承诺， + // 不用；服务器侧也还没有达标的离线引擎）。status 恒不可用 → 界面不显示麦克风 + voice: { + status: async (): Promise => ({ available: false, reason: 'platform' }), + requestMic: async (): Promise => false, + transcribe: desktopOnly('voice.transcribe') as () => Promise, + }, getPhotosWithGPS: () => post('getPhotosWithGPS'), // 文件夹管理（网页版只读展示：addFolder/removeFolder 服务器有实现，但 UI 不提供入口） From 451c36646f78c773ffe3fd752308a6e19877b445 Mon Sep 17 00:00:00 2001 From: RGB-loop <91311486+RGB-loop@users.noreply.github.com> Date: Sun, 18 Oct 2026 01:21:41 +0800 Subject: [PATCH 3/3] =?UTF-8?q?docs(changelog):=20=E8=AF=AD=E9=9F=B3?= =?UTF-8?q?=E8=BE=93=E5=85=A5=E4=B8=8A=E7=BA=BF=E8=AF=B4=E6=98=8E=E4=B8=8E?= =?UTF-8?q?=E4=BB=A5=E5=A3=B0=E6=90=9C=E5=A3=B0=E7=9A=84=E5=8F=96=E8=88=8D?= =?UTF-8?q?=E8=AE=B0=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fbb3989..edb6de7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 links). Available three ways: desktop Local API `/mcp`, server edition `/mcp` (enable with `VIXEL_MCP_TOKEN`), and `vixel mcp` on stdio for Claude Desktop / Cursor / Claude Code without any network service (setup snippet in the README). +- **Voice input in the search bar (macOS 26+).** Click the microphone (or press it again to + stop), speak a query, and it is transcribed on-device into the search box — editable like + any typed text — then the normal text search runs. Transcription uses the system's offline + dictation model (SpeechAnalyzer) via a tiny helper process: nothing is recorded to disk + beyond a temp file that is deleted right away, and no audio ever leaves the Mac (the + browser Web Speech API, which uploads audio, is deliberately not used — so the web edition + has no microphone). Measured on 20 synthetic short queries: Chinese CER 6.4% (errors are + homophones), English WER 0%; voice and typed search agree on the top result 36/40 times + on the demo libraries. If the speech model for your language is not downloaded yet, Vixel + points you to System Settings → Keyboard → Dictation instead of downloading anything itself. + Windows is not covered yet (no offline engine met the quality bar). + +### Decided against (for now) +- **Search by a sound clip.** Measured on the demo footage: audio queries against the joint + frames+audio segment vectors rank the right video first only 3/12 times (the picture + dominates the vector), and even pure audio-to-audio vectors cannot be thresholded — an + unrelated pink-noise clip scores higher than many true matches (0.947 vs 0.917). Without a + reliable "nothing really matches" gate the feature would return confidently wrong results, + so it stays out; the calibration script (`scripts/audio-search-eval.cjs`) documents the + numbers for re-evaluation with future models. ## [0.5.1] - 2026-10-15