From ed4285d209678e78255f8656ab7e4a8f948b0176 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Fri, 23 Oct 2026 09:30:42 +0800 Subject: [PATCH 1/2] Measure received and decoded interviewer audio Transcript order and a Speaking pill could pass the browser smoke while the candidate received no audio packets or decoded only silence. Probe the public LiveKit receiver stats for the native track attached as Jim, and separately measure its decoded RMS during Speaking. Keep unavailable observations distinct from observed silence and retain the failed measurements in the capture before refusing the run. Bound receiver waits, sampling windows and subscription counts. Keep counter baselines within one receiver and Speaking interval, discard late results from replaced tracks, and require healthy evidence after resubscription. A generation whose track ends within its first second is superseded rather than judged on a cut-short window, while a longer one keeps its failure. A slow AudioContext resume does not hold up or end sampling. Do not infer physical audibility or word delivery from these counters. Install through CDP so the probe respects the real page CSP without requiring inline scripts or disabling the policy. Validate with controlled counter, silence, reset, timeout, cancellation and subscription tests, plus a Chromium WebRTC tone loopback under the restrictive CSP using the vendored SDK. Removing the late-track and Speaking-gap guards makes their regressions fail. The isolated full gate and final focused checks pass; live LiveKit/Gemini acceptance still needs an operator-selected isolated configuration and budget. --- docs/latency-observability.md | 34 ++ scripts/browser-check.cjs | 26 +- scripts/media-probe.cjs | 328 ++++++++++++++++ tests/browser/media-probe.test.js | 620 ++++++++++++++++++++++++++++++ 4 files changed, 1006 insertions(+), 2 deletions(-) create mode 100644 scripts/media-probe.cjs create mode 100644 tests/browser/media-probe.test.js diff --git a/docs/latency-observability.md b/docs/latency-observability.md index 2fac5228c..d3a57fe02 100644 --- a/docs/latency-observability.md +++ b/docs/latency-observability.md @@ -169,6 +169,40 @@ out, and input transport before the agent on the way in, are outside this record. A browser-perceived figure needs a matched audio measurement in the browser. +## Browser audio smoke evidence + +The credentialed `voice` and `barge` browser-check flows also capture +`jimAudio`: received byte/packet deltas, peak decoded RMS, playback-blocked +state and a bounded result per observed audio subscription. The harness uses +LiveKit's public track-specific `getRTCStatsReport()` and an analyser on the +same native track attached as Jim. It records no device labels or audio +samples. + +Sampling waits up to 120 seconds for Speaking, then samples at 100 ms +intervals for at most 10 seconds, stopping early once both counters grow and +RMS exceeds 0.001. Counter resets and receiver changes start a new baseline; +different track generations never share deltas. At most 64 subscriptions are +sampled; exceeding that limit is an unavailable observation, not a healthy +verdict. + +`unavailable` means there were insufficient counter pairs or decoded samples, +no attached Jim track, no Speaking interval, cancellation, a sampling error or +a subscription limit. `transport-silent` means bytes or packets, either one, +did not grow across the observed counter pairs, and `decoded-silent` means both +grew but decoded RMS stayed at or below the floor. These are separate failures. +The capture retains failed observations, and each resubscription that keeps its +track through its sampling window must pass independently. A generation whose +track is replaced, unsubscribed or disconnected before Speaking or within its +first second of it, before both counters grew and RMS exceeded the floor, is +marked `superseded`, since losing the track cut its window short and a fresh +analyser first reads near zero; it cannot fail an otherwise healthy +replacement, and the last generation must pass. A generation that sampled +longer is judged on its evidence even if its track then ends. Unmeasured +counters and RMS are `null`, not observed zeroes. This measures transport and +decoded signal during Speaking, not output-device audibility, physical level or +per-word delivery. The RMS floor is a smoke-check threshold, not a calibrated +speech detector. Local loopback tests do not replace the isolated provider run. + ## Validation The credential-free gate covers the trace lifecycle, the waterfall and the diff --git a/scripts/browser-check.cjs b/scripts/browser-check.cjs index b68cf62e1..b74072547 100644 --- a/scripts/browser-check.cjs +++ b/scripts/browser-check.cjs @@ -10,6 +10,7 @@ const { chromium } = require(process.env.PLAYWRIGHT_PATH); const assert = require("node:assert/strict"); const fs = require("fs"); +const { mediaProbeSource, assertJimAudio } = require("./media-probe.cjs"); const { installBargeReplay } = require("./barge-replay.cjs"); const { FloorCapture, @@ -793,6 +794,8 @@ async function isolateRustAgent( try { const page = await browser.newPage(); let bargeReplay = null; + let jimAudio = null; + let probingAudio = false; if (process.env.BROWSER_CHECK_FLOW === "barge") { await page.addInitScript( installBargeReplay, @@ -2310,6 +2313,10 @@ PatternLexicon.prototype.search = function(word) { } await page.goto(interviewUrl(PROBLEM_IDS.twoSum)); + probingAudio = credentialed && ["voice", "barge"].includes(flow); + if (probingAudio) { + await page.evaluate(mediaProbeSource()); + } await clearMediaGate(page); if (credentialed && !rustAgentIdentity) { roomName = await page @@ -2478,6 +2485,14 @@ PatternLexicon.prototype.search = function(word) { ); } } + if (probingAudio) { + jimAudio = await page.evaluate(() => + globalThis.__finishJimAudioProbe(), + ); + if (process.env.BROWSER_CHECK_CAPTURE) + writeIncompleteCapture({ jimAudio, bargeReplay }); + assertJimAudio(jimAudio); + } if (credentialed && flow === "barge") { if (!floorLogPath) throw new Error( @@ -2543,6 +2558,7 @@ PatternLexicon.prototype.search = function(word) { ? readFloorRecords(floorLogPath) : [], bargeReplay, + jimAudio, latencyLog: (agentOutput.join("") + "\n" + readServerLog()) .split("\n") .filter((line) => line.includes("codetrial latency ")) @@ -2557,10 +2573,16 @@ PatternLexicon.prototype.search = function(word) { })(), agentFailure ?? never, ]).catch(async (error) => { - if (process.env.BROWSER_CHECK_CAPTURE && floorLogPath) + if (probingAudio && !jimAudio) { + jimAudio = await page + .evaluate(() => globalThis.__finishJimAudioProbe(true)) + .catch(() => null); + } + if (process.env.BROWSER_CHECK_CAPTURE && (floorLogPath || probingAudio)) writeIncompleteCapture({ - floorRecords: readFloorRecords(floorLogPath), + floorRecords: floorLogPath ? readFloorRecords(floorLogPath) : [], bargeReplay, + jimAudio, }); console.error( redact( diff --git a/scripts/media-probe.cjs b/scripts/media-probe.cjs new file mode 100644 index 000000000..7de590842 --- /dev/null +++ b/scripts/media-probe.cjs @@ -0,0 +1,328 @@ +// The sampler is also injected into Chromium. Keep it independent of Node and +// LiveKit internals: getRTCStatsReport is the public, track-specific receiver API. +function createMediaSampler({ + now = () => performance.now(), + delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), + makeContext = () => + new (globalThis.AudioContext || globalThis.webkitAudioContext)(), + makeStream = (track) => new MediaStream([track]), + timeoutMs = 10000, + speakingWaitMs = timeoutMs, + intervalMs = 100, + rmsFloor = 0.001, + minimumWindowMs = 1000, +} = {}) { + return async function sample( + track, + generation, + isCurrent, + isSpeaking, + isStopped = () => false, + ) { + const began = now(); + const result = { + generation, + durationMs: 0, + samples: 0, + statsSamples: 0, + bytesReceived: 0, + packetsReceived: 0, + peakRms: null, + status: "unavailable", + reason: null, + }; + let context, source, analyser; + let windowBegan = null; + let previous = null; + let transportPairs = 0; + // Both counters grow only with a counter pair, and peakRms is set only by + // a decoded sample, so these three carry the whole proof. + const proven = () => + result.bytesReceived > 0 && + result.packetsReceived > 0 && + result.peakRms > rmsFloor; + const bounded = async (promise) => { + let timer; + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error("timeout")), + Math.max( + 1, + Math.min(500, timeoutMs - (now() - (windowBegan ?? began))), + ), + ); + }), + ]); + } finally { + clearTimeout(timer); + } + }; + try { + while ( + now() - (windowBegan ?? began) < + (windowBegan === null ? speakingWaitMs : timeoutMs) && + isCurrent() && + !isStopped() + ) { + if (!isSpeaking()) { + previous = null; + await delay(intervalMs); + continue; + } + windowBegan ??= now(); + result.samples++; + if (!context) { + try { + context = makeContext(); + // Not awaited: a slow resume must neither hold up receiver samples + // nor end decoded ones. The analyser is read once the context runs. + Promise.resolve() + .then(() => context.resume()) + .catch(() => {}); + source = context.createMediaStreamSource( + makeStream(track.mediaStreamTrack), + ); + analyser = context.createAnalyser(); + analyser.fftSize = 1024; + source.connect(analyser); + } catch {} + } + let stats; + try { + stats = await bounded( + Promise.resolve().then(() => track.getRTCStatsReport?.()), + ); + } catch {} + if (!isCurrent() || isStopped()) break; + if (!isSpeaking()) { + previous = null; + continue; + } + const inbound = []; + stats?.forEach((entry) => { + if ( + entry.type === "inbound-rtp" && + (entry.kind === "audio" || entry.mediaType === "audio") && + Number.isFinite(entry.bytesReceived) && + entry.bytesReceived >= 0 && + Number.isFinite(entry.packetsReceived) && + entry.packetsReceived >= 0 + ) + inbound.push(entry); + }); + // A receiver should expose one inbound audio stream. Ambiguous reports + // cannot prove which stream reached this track. + const current = inbound.length === 1 ? inbound[0] : null; + if (current) { + result.statsSamples++; + if ( + previous && + previous.id === current.id && + current.bytesReceived >= previous.bytesReceived && + current.packetsReceived >= previous.packetsReceived + ) { + result.bytesReceived += + current.bytesReceived - previous.bytesReceived; + result.packetsReceived += + current.packetsReceived - previous.packetsReceived; + transportPairs++; + } + previous = { + id: current.id, + bytesReceived: current.bytesReceived, + packetsReceived: current.packetsReceived, + }; + } else { + previous = null; + } + if (analyser && context.state === "running") { + const values = new Float32Array(analyser.fftSize); + analyser.getFloatTimeDomainData(values); + const rms = Math.sqrt( + values.reduce((sum, value) => sum + value * value, 0) / + values.length, + ); + if (!Number.isFinite(rms)) throw new Error("invalid decoded samples"); + result.peakRms = Math.max(result.peakRms ?? 0, rms); + } + if (proven()) break; + await delay(intervalMs); + } + if (isStopped()) { + result.reason = "sampling-cancelled"; + } else if ( + !isCurrent() && + !proven() && + (windowBegan === null || now() - windowBegan < minimumWindowMs) + ) { + // Losing the track early cuts the window short, and a fresh analyser + // reads near zero at first. A replacement is judged on its own; with + // none, the summary fails on this generation being the last. A longer + // window is judged on its evidence, so a reconnect cannot hide it. + result.status = "superseded"; + result.reason = transportPairs + ? "track-ended-before-signal" + : "track-ended-before-counter-pair"; + } else if (result.samples === 0) { + result.reason = "no-speaking-samples"; + } else if (!transportPairs) { + result.reason = "receiver-stats-unavailable"; + } else if (result.peakRms === null) { + result.reason = "decoded-signal-unavailable"; + } else if (!result.bytesReceived || !result.packetsReceived) { + result.status = "transport-silent"; + result.reason = "audio-packets-did-not-increase"; + } else if (result.peakRms <= rmsFloor) { + result.status = "decoded-silent"; + result.reason = "packets-arrived-without-decoded-signal"; + } else { + result.status = "ok"; + } + } catch { + result.reason = "sampling-failed"; + } finally { + try { + source?.disconnect(); + } catch {} + try { + analyser?.disconnect(); + } catch {} + if (context) + await bounded(Promise.resolve().then(() => context.close())).catch( + () => {}, + ); + result.durationMs = Math.max(0, now() - began); + result.counterPairs = transportPairs; + if (!transportPairs) { + result.bytesReceived = null; + result.packetsReceived = null; + } + } + return result; + }; +} + +// Installed after navigation, while the preflight still gates room creation. +// Observe public Room events and sample only the track the page attaches as Jim. +function installMediaProbe() { + const sdk = globalThis.LivekitClient; + const OriginalRoom = sdk.Room; + const sample = globalThis.__createMediaSampler({ speakingWaitMs: 120000 }); + let generation = 0; + let active = null; + let stopped = false; + let accepting = true; + let overflow = false; + const jobs = []; + // Every way a track stops being Jim's moves the generation, which is what + // ends its sampler. + const retire = () => { + active = null; + generation++; + }; + const speaking = () => + document.querySelector("#agent-state")?.textContent.includes("Speaking"); + sdk.Room = class extends OriginalRoom { + constructor(...args) { + super(...args); + this.on(sdk.RoomEvent.TrackSubscribed, (track) => { + if (!accepting || track.kind !== "audio") return; + // App listeners attach Jim after this listener. A microtask sees that + // choice without guessing agent identity or retaining participant data. + queueMicrotask(() => { + if (!accepting) return; + const element = document.querySelector("#jim-audio"); + if ( + !element?.srcObject + ?.getAudioTracks() + .includes(track.mediaStreamTrack) + ) + return; + if (jobs.length >= 64) { + overflow = true; + retire(); + return; + } + active = track; + const mine = ++generation; + jobs.push( + sample( + track, + mine, + () => mine === generation, + speaking, + () => stopped, + ), + ); + }); + }); + this.on(sdk.RoomEvent.TrackUnsubscribed, (track) => { + if (active === track) retire(); + }); + this.on(sdk.RoomEvent.Disconnected, retire); + } + }; + globalThis.__finishJimAudioProbe = async (cancel = false) => { + accepting = false; + if (cancel) stopped = true; + sdk.Room = OriginalRoom; + const generations = await Promise.all(jobs); + if (overflow) + generations.push({ + generation, + status: "unavailable", + reason: "subscription-observation-limit", + bytesReceived: null, + packetsReceived: null, + peakRms: null, + }); + return globalThis.__summarizeJimAudio( + generations, + document.querySelector("#audio-playback-notice")?.hidden === false, + ); + }; +} + +function summarizeJimAudio(generations, playbackBlocked = null) { + const last = generations.at(-1); + const failure = + generations.find((entry) => !["ok", "superseded"].includes(entry.status)) ?? + (last?.status === "ok" ? null : last); + const measured = (field) => + generations.map((entry) => entry[field]).filter(Number.isFinite); + const sum = (field) => + measured(field).reduce((total, value) => (total ?? 0) + value, null); + const peaks = measured("peakRms"); + return { + status: failure?.status ?? (last ? "ok" : "unavailable"), + reason: failure?.reason ?? (last ? null : "no-attached-agent-track"), + failedGeneration: failure?.generation ?? null, + playbackBlocked, + bytesReceived: sum("bytesReceived"), + packetsReceived: sum("packetsReceived"), + peakRms: peaks.length ? Math.max(...peaks) : null, + generations, + }; +} + +function mediaProbeSource() { + return `globalThis.__createMediaSampler = ${createMediaSampler.toString()}; globalThis.__summarizeJimAudio = ${summarizeJimAudio.toString()}; (${installMediaProbe.toString()})();`; +} + +function assertJimAudio(result) { + const verdict = summarizeJimAudio(result?.generations ?? []); + if (verdict.status !== "ok") + throw new Error( + `Jim audio ${verdict.status}: ${verdict.reason} (generation ${verdict.failedGeneration ?? "none"})`, + ); +} +module.exports = { + createMediaSampler, + installMediaProbe, + summarizeJimAudio, + mediaProbeSource, + assertJimAudio, +}; diff --git a/tests/browser/media-probe.test.js b/tests/browser/media-probe.test.js new file mode 100644 index 000000000..a79ebb713 --- /dev/null +++ b/tests/browser/media-probe.test.js @@ -0,0 +1,620 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + createMediaSampler, + assertJimAudio, + installMediaProbe, + summarizeJimAudio, + mediaProbeSource, +} from "../../scripts/media-probe.cjs"; + +function fixture({ + counts = [0, 10, 20], + rms = 0.1, + stats = true, + running = true, + contextError = false, + timeoutMs = 40, + speakingWaitMs = timeoutMs, + minimumWindowMs, + onDelay = () => {}, +} = {}) { + let clock = 0; + let index = 0; + let current = true; + let speaking = true; + let cleaned = 0; + let closed = 0; + const context = { + state: running ? "running" : "suspended", + resume: async () => {}, + close: async () => { + closed++; + }, + createMediaStreamSource: () => ({ + connect: () => {}, + disconnect: () => { + cleaned++; + }, + }), + createAnalyser: () => ({ + fftSize: 0, + getFloatTimeDomainData: (values) => values.fill(rms), + disconnect: () => { + cleaned++; + }, + }), + }; + const track = { + mediaStreamTrack: {}, + getRTCStatsReport: async () => { + if (!stats) return; + const value = counts[Math.min(index++, counts.length - 1)]; + return new Map([ + [ + "inbound", + { + type: "inbound-rtp", + kind: "audio", + id: "inbound", + bytesReceived: value, + packetsReceived: value, + }, + ], + ]); + }, + }; + const sampler = createMediaSampler({ + now: () => clock, + delay: async (ms) => { + clock += ms; + onDelay(); + }, + makeContext: () => { + if (contextError) throw new Error("unavailable"); + return context; + }, + makeStream: (input) => { + assert.equal(input, track.mediaStreamTrack); + return {}; + }, + timeoutMs, + speakingWaitMs, + minimumWindowMs, + intervalMs: 10, + }); + return { + track, + sampler, + context, + run: () => + sampler( + track, + 7, + () => current, + () => speaking, + ), + changeTrack: () => { + current = false; + }, + quiet: () => { + speaking = false; + }, + speak: () => { + speaking = true; + }, + cleaned: () => cleaned, + closed: () => closed, + }; +} + +test("track receiver counters and decoded signal establish distinct evidence", async () => { + const f = fixture(); + const result = await f.run(); + assert.equal(result.status, "ok"); + assert.equal(result.generation, 7); + assert.equal(result.bytesReceived, 10); + assert.equal(result.packetsReceived, 10); + assert.ok(result.peakRms > 0.01); + assert.equal(result.durationMs, 10); + assert.equal(f.cleaned(), 2); + assert.equal(f.closed(), 1); + assert.doesNotThrow(() => assertJimAudio({ generations: [result] })); +}); +for (const [options, status] of [ + [{ counts: [9, 9, 9] }, "transport-silent"], + [{ rms: 0 }, "decoded-silent"], + [{ stats: false }, "unavailable"], + [{ running: false }, "unavailable"], + [{ contextError: true }, "unavailable"], +]) { + test(`controlled ${JSON.stringify(options)} reports ${status}`, async () => { + const f = fixture(options); + const result = await f.run(); + assert.equal(result.status, status); + assert.throws( + () => assertJimAudio({ generations: [result] }), + new RegExp(status), + ); + if (!options.contextError) assert.equal(f.closed(), 1); + }); +} + +test("counter resets do not create a positive delta", async () => { + const f = fixture({ counts: [100, 1, 1, 1] }); + const result = await f.run(); + assert.equal(result.status, "transport-silent"); + assert.equal(result.bytesReceived, 0); +}); + +test("a replaced track cannot contribute late receiver observations", async () => { + const f = fixture(); + f.track.getRTCStatsReport = async () => { + f.changeTrack(); + return new Map([ + [ + "live", + { + type: "inbound-rtp", + kind: "audio", + id: "live", + bytesReceived: 100, + packetsReceived: 10, + }, + ], + ]); + }; + const result = await f.run(); + assert.equal(result.status, "superseded"); + assert.equal(result.statsSamples, 0); + assert.equal(f.closed(), 1); +}); + +test("a replacement after one counter pair supersedes a quiet generation instead of failing it", async () => { + let delays = 0; + const f = fixture({ + rms: 0, + timeoutMs: 100, + onDelay: () => { + if (++delays === 2) f.changeTrack(); + }, + }); + const result = await f.run(); + assert.equal(result.counterPairs, 1); + assert.equal(result.status, "superseded"); + assert.equal(result.reason, "track-ended-before-signal"); + assert.equal(f.closed(), 1); +}); + +test("a slow resume neither delays receiver samples nor ends decoded sampling", async () => { + let delays = 0; + let running; + const f = fixture({ + counts: [0, 10, 20, 30, 40, 50], + timeoutMs: 100, + onDelay: () => { + if (++delays === 3) running(); + }, + }); + f.context.state = "suspended"; + f.context.resume = () => + new Promise((resolve) => { + running = () => { + f.context.state = "running"; + resolve(); + }; + }); + const result = await f.run(); + assert.equal(result.status, "ok"); + assert.ok(result.statsSamples >= 4); + assert.equal(f.closed(), 1); +}); + +test("a generation that sampled a long window keeps its failure when its track ends", async () => { + let delays = 0; + const f = fixture({ + counts: [9], + timeoutMs: 1000, + minimumWindowMs: 100, + onDelay: () => { + if (++delays === 20) f.changeTrack(); + }, + }); + const result = await f.run(); + assert.equal(result.status, "transport-silent"); +}); + +test("no Speaking interval produces no signal claim or audio context", async () => { + const f = fixture(); + f.quiet(); + const result = await f.run(); + assert.equal(result.status, "unavailable"); + assert.equal(result.samples, 0); + assert.equal(result.reason, "no-speaking-samples"); + assert.equal(result.peakRms, null); + assert.equal(f.closed(), 0); +}); + +test("receiver timeout is bounded and releases its context", async () => { + const f = fixture({ timeoutMs: 10 }); + f.track.getRTCStatsReport = () => new Promise(() => {}); + const result = await f.run(); + assert.equal(result.status, "unavailable"); + assert.equal(f.closed(), 1); +}); + +test("a good old generation cannot hide a silent resubscription", () => { + assert.throws( + () => + assertJimAudio({ + generations: [ + { status: "ok" }, + { generation: 2, status: "transport-silent", reason: "flat" }, + ], + }), + /generation 2/, + ); + assert.throws( + () => assertJimAudio({ generations: [] }), + /no-attached-agent-track/, + ); +}); + +test("receiver identity resets and ambiguous streams cannot fabricate packet deltas", async () => { + const f = fixture(); + let calls = 0; + f.track.getRTCStatsReport = async () => + new Map([ + [ + "inbound", + { + type: "inbound-rtp", + kind: "audio", + id: `stream-${calls++}`, + bytesReceived: calls * 100, + packetsReceived: calls, + }, + ], + ]); + assert.equal((await f.run()).status, "unavailable"); + const g = fixture(); + g.track.getRTCStatsReport = async () => + new Map( + [1, 2].map((id) => [ + id, + { + type: "inbound-rtp", + kind: "audio", + id, + bytesReceived: 100, + packetsReceived: 10, + }, + ]), + ); + assert.equal((await g.run()).status, "unavailable"); +}); + +test("setup, analyser failure and refused cleanup remain bounded unavailable observations", async () => { + const f = fixture(); + f.context.createMediaStreamSource = () => { + throw new Error("setup"); + }; + assert.equal((await f.run()).status, "unavailable"); + assert.equal(f.closed(), 1); + const g = fixture(); + g.context.createAnalyser = () => ({ + disconnect: () => {}, + getFloatTimeDomainData: () => { + throw new Error("read"); + }, + }); + assert.equal((await g.run()).status, "unavailable"); + assert.equal(g.closed(), 1); + const h = fixture(); + h.context.close = async () => { + throw new Error("already closed"); + }; + assert.equal((await h.run()).status, "ok"); +}); + +test("public Room subscription hooks follow only Jim's attached track and keep generation outcomes", async (t) => { + const { EventEmitter } = await import("node:events"); + const names = [ + "LivekitClient", + "__createMediaSampler", + "__finishJimAudioProbe", + "__summarizeJimAudio", + "document", + ]; + const previous = names.map((name) => [ + name, + Object.getOwnPropertyDescriptor(globalThis, name), + ]); + t.after(() => { + for (const [name, descriptor] of previous) { + if (descriptor) Object.defineProperty(globalThis, name, descriptor); + else delete globalThis[name]; + } + }); + const events = { + TrackSubscribed: "sub", + TrackUnsubscribed: "unsub", + Disconnected: "disconnect", + }; + globalThis.LivekitClient = { Room: EventEmitter, RoomEvent: events }; + let attached; + globalThis.document = { + querySelector: (selector) => + selector === "#jim-audio" + ? { srcObject: { getAudioTracks: () => [attached] } } + : { textContent: "Speaking" }, + }; + const seen = []; + globalThis.__createMediaSampler = + () => async (track, generation, isCurrent, isSpeaking) => { + seen.push({ track, generation, isCurrent, isSpeaking }); + return { + generation, + status: "ok", + bytesReceived: 10, + packetsReceived: 1, + peakRms: 0.1, + }; + }; + globalThis.__summarizeJimAudio = summarizeJimAudio; + installMediaProbe(); + const room = new globalThis.LivekitClient.Room(); + const first = { kind: "audio", mediaStreamTrack: {} }; + const unrelated = { kind: "audio", mediaStreamTrack: {} }; + const replacement = { kind: "audio", mediaStreamTrack: {} }; + attached = first.mediaStreamTrack; + room.emit("sub", unrelated); + room.emit("sub", first); + await new Promise((resolve) => queueMicrotask(resolve)); + assert.equal(seen.length, 1); + assert.equal(seen[0].isCurrent(), true); + assert.equal(seen[0].isSpeaking(), true); + attached = replacement.mediaStreamTrack; + room.emit("sub", replacement); + await new Promise((resolve) => queueMicrotask(resolve)); + room.emit("unsub", first); + assert.equal(seen[0].isCurrent(), false); + assert.equal(seen[1].isCurrent(), true); + room.emit("disconnect"); + assert.equal(seen[1].isCurrent(), false); + const result = await globalThis.__finishJimAudioProbe(); + assert.equal(result.generations.length, 2); + assert.equal(result.bytesReceived, 20); + assert.equal(globalThis.LivekitClient.Room, EventEmitter); + installMediaProbe(); + const noisyRoom = new globalThis.LivekitClient.Room(); + for (let index = 0; index < 65; index++) { + const track = { kind: "audio", mediaStreamTrack: {} }; + attached = track.mediaStreamTrack; + noisyRoom.emit("sub", track); + await new Promise((resolve) => queueMicrotask(resolve)); + } + const overflow = await globalThis.__finishJimAudioProbe(); + assert.equal(overflow.generations.length, 65); + assert.equal(overflow.status, "unavailable"); + assert.equal( + overflow.generations.at(-1).reason, + "subscription-observation-limit", + ); + assert.equal(overflow.generations.at(-1).bytesReceived, null); + assert.equal(overflow.generations.at(-1).packetsReceived, null); +}); + +test("Chromium measures real received WebRTC packets and decoded tone without provider credentials", async (t) => { + const { launchChromium, root } = await import("./source.js"); + const { join } = await import("node:path"); + const { readFileSync } = await import("node:fs"); + const browser = await launchChromium({ + args: ["--autoplay-policy=no-user-gesture-required"], + }); + if (!browser) return t.skip("playwright chromium unavailable"); + t.after(() => browser.close()); + const page = await browser.newPage(); + await page.route("http://localhost:31999/**", async (route) => { + const script = route.request().url().endsWith("/vendor/livekit-client.js"); + await route.fulfill({ + status: 200, + headers: { + "content-type": script ? "text/javascript" : "text/html", + "content-security-policy": + "default-src 'self'; script-src 'self' blob: 'wasm-unsafe-eval'", + }, + body: script + ? readFileSync(join(root, "web/vendor/livekit-client.js"), "utf8") + : '', + }); + }); + await page.goto("http://localhost:31999/"); + await page.evaluate(mediaProbeSource()); + const sdkProbe = await page.evaluate(async () => { + const room = new globalThis.LivekitClient.Room(); + const result = await globalThis.__finishJimAudioProbe(); + await room.disconnect(); + return result; + }); + assert.equal(sdkProbe.status, "unavailable"); + assert.equal(sdkProbe.generations.length, 0); + const result = await page.evaluate(async () => { + const sender = new RTCPeerConnection(); + const receiver = new RTCPeerConnection(); + const context = new AudioContext(); + const oscillator = context.createOscillator(); + const destination = context.createMediaStreamDestination(); + oscillator.connect(destination); + oscillator.start(); + sender.onicecandidate = (event) => { + if (event.candidate) void receiver.addIceCandidate(event.candidate); + }; + receiver.onicecandidate = (event) => { + if (event.candidate) void sender.addIceCandidate(event.candidate); + }; + // Bounded, so a loopback that never negotiates fails here and still + // reaches the cleanup below instead of hanging the suite. + const received = new Promise((resolve, reject) => { + receiver.ontrack = resolve; + setTimeout(() => reject(new Error("loopback track timeout")), 10000); + }); + sender.addTrack(destination.stream.getAudioTracks()[0], destination.stream); + try { + await sender.setLocalDescription(await sender.createOffer()); + await receiver.setRemoteDescription(sender.localDescription); + await receiver.setLocalDescription(await receiver.createAnswer()); + await sender.setRemoteDescription(receiver.localDescription); + const { track, receiver: inbound } = await received; + const audio = document.createElement("audio"); + audio.srcObject = new MediaStream([track]); + document.body.append(audio); + await audio.play(); + return await globalThis.__createMediaSampler()( + { + mediaStreamTrack: track, + getRTCStatsReport: () => inbound.getStats(), + }, + 1, + () => true, + () => true, + ); + } finally { + oscillator.stop(); + destination.stream.getTracks().forEach((track) => track.stop()); + sender.close(); + receiver.close(); + await context.close(); + } + }); + assert.equal(result.status, "ok", JSON.stringify(result)); + assert.ok(result.bytesReceived > 0); + assert.ok(result.packetsReceived > 0); + assert.ok(result.peakRms > 0.001); +}); + +test("non-Speaking gaps cannot bridge counter deltas", async () => { + let ticks = 0; + const f = fixture({ + counts: [0, 100, 100, 100], + onDelay: () => { + ticks++; + if (ticks === 1) f.quiet(); + if (ticks === 2) f.speak(); + }, + }); + const result = await f.run(); + assert.equal(result.status, "transport-silent"); + assert.equal(result.bytesReceived, 0); +}); + +test("Speaking lost during a receiver await discards the sample and its baseline", async () => { + let calls = 0; + const f = fixture({ + onDelay: () => { + if (calls === 2) f.speak(); + }, + }); + f.track.getRTCStatsReport = async () => { + calls++; + if (calls === 2) f.quiet(); + return new Map([ + [ + "inbound", + { + type: "inbound-rtp", + kind: "audio", + id: "inbound", + bytesReceived: calls === 1 ? 0 : 100, + packetsReceived: calls === 1 ? 0 : 10, + }, + ], + ]); + }; + const result = await f.run(); + assert.equal(result.status, "transport-silent"); + assert.equal(result.bytesReceived, 0); +}); + +test("cancel discards a pending stats observation and closes the context", async () => { + const f = fixture(); + let stopped = false; + f.track.getRTCStatsReport = async () => { + stopped = true; + return new Map([ + [ + "inbound", + { + type: "inbound-rtp", + kind: "audio", + id: "inbound", + bytesReceived: 100, + packetsReceived: 10, + }, + ], + ]); + }; + const result = await f.sampler( + f.track, + 1, + () => true, + () => true, + () => stopped, + ); + assert.equal(result.statsSamples, 0); + assert.equal(result.status, "unavailable"); + assert.equal(result.reason, "sampling-cancelled"); + assert.equal(f.closed(), 1); +}); + +test("waiting for Speaking does not spend the sampling window", async () => { + let ticks = 0; + const f = fixture({ + timeoutMs: 20, + speakingWaitMs: 60, + onDelay: () => { + if (++ticks === 3) f.speak(); + }, + }); + f.quiet(); + const result = await f.run(); + assert.equal(result.status, "ok"); + assert.equal(result.durationMs, 40); +}); + +test("unknown aggregate observations remain null and superseded empty generations require a healthy successor", () => { + const absent = summarizeJimAudio([ + { + generation: 1, + status: "unavailable", + reason: "stats", + bytesReceived: null, + packetsReceived: null, + peakRms: null, + }, + ]); + assert.equal(absent.bytesReceived, null); + assert.equal(absent.packetsReceived, null); + assert.equal(absent.peakRms, null); + assert.equal(absent.reason, "stats"); + const superseded = { + generation: 1, + status: "superseded", + reason: "replaced", + bytesReceived: null, + packetsReceived: null, + peakRms: null, + }; + const healthy = { + generation: 2, + status: "ok", + bytesReceived: 10, + packetsReceived: 1, + peakRms: 0.1, + }; + assert.equal(summarizeJimAudio([superseded, healthy]).status, "ok"); + assert.throws( + () => assertJimAudio({ generations: [superseded] }), + /superseded/, + ); +}); From 5301df5a0e86efa014cd52e57ed23cdf75f9d4ea Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Fri, 23 Oct 2026 09:36:19 +0800 Subject: [PATCH 2/2] Keep Jim's volume and mute across playback changes Candidates could route Jim to a speaker but could not attenuate or mute him locally. Remember a validated positive volume and keep mute within the current page, so reload restores the last positive level. Expose a labelled slider, a fixed-name mute toggle, a reset and a persistent muted indicator. A single step up from the muted slider unmutes at the saved level instead of saving 1%. Keep mute available when volume setters are unsupported, moving focus before hiding the unusable slider. Apply preferences whenever Jim's audio attaches or returns after a subscription change. Restore unsolicited SDK volume changes, including an unmute without a playback-status event. Reapply synchronously during Resume audio and after its promises settle so a stalled or stale startAudio cannot leave playback unmuted while the UI claims otherwise. Remove the element listener when its attachment goes away. Expanded audio controls and playback warnings could cover report-wait actions on a short phone screen. Bound and scroll the compact stage, and reserve its measured height in the wait instead of fixed padding. Keep recovery reachable by keyboard inside the clipped stage. Preference, readback, attachment, recovery and DOM checks pass, as do Chromium keyboard, reload and phone-layout checks. Removing either SDK unmute restoration or synchronous recovery reapply makes its regression fail. The full gate covers this and the audio probe together. Physical audible-level and remote Meet-capture observations remain operator acceptance work; the interface identifies that possible effect, and the Meet manual check now resets the volume before recording. --- docs/meet-tab-audio-manual-check.md | 19 +- tests/browser/audio-playback.test.js | 86 ++++++++- tests/browser/interview-layout.test.js | 9 + tests/browser/jim-volume-ui.test.js | 154 +++++++++++++++ tests/browser/jim-volume.test.js | 252 +++++++++++++++++++++++++ tests/browser/meet-tab-audio.test.js | 8 + web/audio-output.js | 89 +++++++++ web/audio-playback.js | 18 +- web/interview.html | 45 +++++ web/interview.js | 44 ++++- web/styles.css | 64 ++++++- 11 files changed, 765 insertions(+), 23 deletions(-) create mode 100644 tests/browser/jim-volume-ui.test.js create mode 100644 tests/browser/jim-volume.test.js diff --git a/docs/meet-tab-audio-manual-check.md b/docs/meet-tab-audio-manual-check.md index 2f4930e89..86f0ae7f6 100644 --- a/docs/meet-tab-audio-manual-check.md +++ b/docs/meet-tab-audio-manual-check.md @@ -37,20 +37,25 @@ re-vendored, or before relying on the mode for a real interview. confirm Jim moves to it. This step is what catches a broken device list or a preference that fails to persist, so do not skip it when Jim is already audible. -3. In Meet, click Present now. -4. Choose "A tab". -5. Select the CodeTrial tab. -6. Enable "Share tab audio" before clicking Share. -7. Confirm Meet's microphone is still the candidate microphone, not "None" and +3. Open "Audio controls" on the stage. Jim's volume and mute set the level of + the tab audio Meet captures, and the volume is saved across sessions, so a + low level left from an earlier run reaches the interviewer as a quiet Jim. + Press Reset volume, and confirm the slider reads 100% and "Mute Jim" is not + pressed. +4. In Meet, click Present now. +5. Choose "A tab". +6. Select the CodeTrial tab. +7. Enable "Share tab audio" before clicking Share. +8. Confirm Meet's microphone is still the candidate microphone, not "None" and not a virtual device. -8. Record 10 seconds in Meet while Jim is speaking and the candidate says one +9. Record 10 seconds in Meet while Jim is speaking and the candidate says one sentence. ## Pass criteria All five must hold: -- The recording contains Jim's voice. +- The recording contains Jim's voice at the level set in step 3. - The recording contains the CodeTrial tab video. - The recording contains the candidate's voice. - Meet's selected microphone is still the candidate's physical microphone at diff --git a/tests/browser/audio-playback.test.js b/tests/browser/audio-playback.test.js index e7b8bfa84..1d2c1c65e 100644 --- a/tests/browser/audio-playback.test.js +++ b/tests/browser/audio-playback.test.js @@ -20,7 +20,7 @@ function deferred() { return { promise, resolve, reject }; } -function fixture() { +function fixture({ reapply = () => {} } = {}) { const notice = { hidden: true }; const document = { activeElement: null }; const button = new EventTarget(); @@ -38,6 +38,7 @@ function fixture() { button, focusTarget, announcement, + reapply, }); playback.setRoom(room, "playback"); return { @@ -386,3 +387,86 @@ test("report recovery keeps playback tracked; only the rendered report ends it", assert.equal(f.notice.hidden, true); assert.equal(f.room.listenerCount("playback"), 0); }); + +test("playback recovery restores session mute after LiveKit unmutes its elements", async () => { + const { createJimVolume } = await import("../../web/audio-output.js"); + let refused = true; + const element = { + volume: 1, + muted: false, + play: async () => { + if (refused) throw blocked(); + }, + }; + const volume = createJimVolume({ element: () => element, storage: null }); + volume.setVolume(25); + volume.toggleMute(); + const f = fixture({ reapply: () => volume.apply() }); + await f.playback.play(element); + f.room.startAudio = async () => { + element.muted = false; + }; + refused = false; + f.click(); + await tick(); + assert.equal(f.notice.hidden, true); + assert.equal(element.muted, true); + assert.equal(element.volume, 0.25); + element.muted = false; + f.status(true); + assert.equal(element.muted, true); +}); + +test("a late old-room recovery reapplies preferences to the currently attached element", async () => { + const { createJimVolume } = await import("../../web/audio-output.js"); + let current = { ...denied(), volume: 1, muted: false }; + const volume = createJimVolume({ element: () => current, storage: null }); + volume.toggleMute(); + const f = fixture({ reapply: () => volume.apply() }); + await f.playback.play(current); + const held = deferred(); + f.room.startAudio = () => + held.promise.then(() => { + current.muted = false; + }); + f.click(); + // The interview reconnects: a new room and a new element arrive while the + // old room's recovery is still pending. + const next = new EventEmitter(); + next.canPlaybackAudio = true; + next.startAudio = async () => {}; + f.playback.setRoom(next, "playback"); + current = { ...playing(), volume: 1, muted: false }; + await f.playback.play(current); + held.resolve(); + await tick(); + assert.equal(current.muted, true); +}); + +test("synchronous startAudio unmute is corrected before a stalled resume can play", async () => { + let muted = true; + const playedMuted = []; + const element = { + play: async () => { + playedMuted.push(muted); + throw blocked(); + }, + }; + const f = fixture({ + reapply: () => { + muted = true; + }, + }); + await f.playback.play(element); + const held = deferred(); + f.room.startAudio = () => { + muted = false; + return held.promise; + }; + playedMuted.length = 0; + f.click(); + assert.equal(muted, true); + assert.deepEqual(playedMuted, [true]); + held.resolve(); + await tick(); +}); diff --git a/tests/browser/interview-layout.test.js b/tests/browser/interview-layout.test.js index 05b7a30c4..a375afea7 100644 --- a/tests/browser/interview-layout.test.js +++ b/tests/browser/interview-layout.test.js @@ -335,6 +335,15 @@ test("on a phone the self view does not grow the card over the wait", async (t) document.querySelector("#captions-text").textContent = "Thanks for walking me through that. ".repeat(30); }); + await page.waitForFunction( + () => + parseFloat( + getComputedStyle(document.querySelector("#ending-overlay")) + .paddingBottom, + ) >= + document.querySelector("#jim-stage").getBoundingClientRect().height + + 47, + ); const under = await page.evaluate(() => { const card = document.querySelector("#jim-stage").getBoundingClientRect(); return [...document.querySelectorAll("#ending-overlay button")] diff --git a/tests/browser/jim-volume-ui.test.js b/tests/browser/jim-volume-ui.test.js new file mode 100644 index 000000000..dea5153c1 --- /dev/null +++ b/tests/browser/jim-volume-ui.test.js @@ -0,0 +1,154 @@ +import { before, after, test } from "node:test"; +import assert from "node:assert/strict"; +import { + launchChromium, + startStaticServer, + DEFAULT_RUNTIME_CONFIG, + closeSetupCheck, +} from "./source.js"; + +let browser, server, base; +before(async () => { + browser = await launchChromium(); + if (browser) + ({ server, base } = await startStaticServer({ + runtimeConfig: DEFAULT_RUNTIME_CONFIG, + })); +}); +after(async () => { + await browser?.close(); + await new Promise((resolve) => (server ? server.close(resolve) : resolve())); +}); +async function open(t, width = 1280, height = 720) { + const page = await browser.newPage({ viewport: { width, height } }); + t.after(() => page.close()); + await page.goto(`${base}/interview.html?problem=chargeback-pair-match`); + await openControls(page); + return page; +} + +async function openControls(page) { + await page.waitForFunction( + () => document.querySelector("#candidate-case-input").placeholder !== "", + ); + await closeSetupCheck(page); + await page.locator(".jim-volume-controls summary").click(); +} + +test("volume controls preserve positive level across reload and expose session mute to keyboard users", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await open(t); + const range = page.getByRole("slider", { name: /Jim volume/ }); + await range.evaluate((input) => { + input.value = "27"; + input.dispatchEvent(new Event("input", { bubbles: true })); + }); + await page.waitForFunction( + () => document.querySelector("#jim-volume-value").textContent === "27%", + ); + await range.focus(); + await range.press("Home"); + await page.waitForFunction( + () => !document.querySelector("#jim-muted-indicator").hidden, + ); + assert.equal( + await page.locator("#jim-mute").getAttribute("aria-pressed"), + "true", + ); + assert.equal( + await page.evaluate(() => localStorage.getItem("codetrial:jimVolume")), + "27", + ); + await range.press("ArrowRight"); + await page.waitForFunction( + () => document.querySelector("#jim-muted-indicator").hidden, + ); + assert.equal(await range.inputValue(), "27"); + assert.equal( + await page.evaluate(() => localStorage.getItem("codetrial:jimVolume")), + "27", + ); + await range.press("Home"); + await page.reload(); + await openControls(page); + assert.equal(await range.inputValue(), "27"); + assert.equal(await page.locator("#jim-muted-indicator").isVisible(), false); + await page.locator("#jim-mute").click(); + await page.locator("#jim-volume-reset").click(); + await page.waitForFunction( + () => document.querySelector("#jim-volume-value").textContent === "100%", + ); + assert.equal( + await page.locator("#jim-mute").getAttribute("aria-pressed"), + "false", + ); +}); + +test("expanded audio controls, mute and playback recovery leave ending actions clear on a phone", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await open(t, 390, 480); + await page.locator("#jim-mute").click(); + await page.evaluate(() => { + document.querySelector("#ending-overlay").hidden = false; + document.querySelector("#ending-withdraw-consent").hidden = false; + document.querySelector("#audio-playback-notice").hidden = false; + document.querySelector("#captions-text").textContent = + "Thanks for walking through the solution. ".repeat(25); + }); + await page.waitForFunction( + () => + parseFloat( + getComputedStyle(document.querySelector("#ending-overlay")) + .paddingBottom, + ) >= + document.querySelector("#jim-stage").getBoundingClientRect().height + 47, + ); + await page.locator("#ending-overlay").evaluate((overlay) => { + overlay.scrollTop = overlay.scrollHeight; + }); + const overlap = await page.evaluate(() => { + const stage = document.querySelector("#jim-stage").getBoundingClientRect(); + return [...document.querySelectorAll("#ending-overlay button")] + .filter((button) => button.offsetParent) + .filter((button) => { + const box = button.getBoundingClientRect(); + return box.bottom > stage.top && box.top < stage.bottom; + }) + .map((button) => button.id); + }); + assert.deepEqual(overlap, []); + assert.equal(await page.locator("#jim-mute").isVisible(), true); + const actions = await page.evaluate(() => { + const stage = document.querySelector("#jim-stage").getBoundingClientRect(); + return [...document.querySelectorAll("#ending-overlay button")] + .filter((button) => button.offsetParent) + .map((button) => { + const box = button.getBoundingClientRect(); + return { + id: button.id, + reachable: + box.top >= 0 && + box.bottom <= stage.top && + box.bottom <= innerHeight, + }; + }); + }); + assert.ok(actions.length > 0); + assert.ok( + actions.every((action) => action.reachable), + JSON.stringify(actions), + ); + await page.locator("#resume-audio").focus(); + assert.equal( + await page.locator("#resume-audio").evaluate((button) => { + const box = button.getBoundingClientRect(); + return button.contains( + document.elementFromPoint( + box.x + box.width / 2, + box.y + box.height / 2, + ), + ); + }), + true, + ); +}); diff --git a/tests/browser/jim-volume.test.js b/tests/browser/jim-volume.test.js new file mode 100644 index 000000000..5fbfcc849 --- /dev/null +++ b/tests/browser/jim-volume.test.js @@ -0,0 +1,252 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createJimVolume, JIM_VOLUME_KEY } from "../../web/audio-output.js"; +import { loadInterview, memoryStorage } from "./source.js"; + +function fixture(stored = null, audio = { volume: 1, muted: false }) { + const storage = memoryStorage(); + if (stored !== null) storage.setItem(JIM_VOLUME_KEY, stored); + let current = audio; + const volume = createJimVolume({ storage, element: () => current }); + return { + volume, + storage, + audio, + replace: (next) => { + current = next; + volume.apply(); + }, + }; +} + +for (const stored of [ + null, + "", + "0", + "101", + "-1", + "NaN", + "1.5", + " 50 ", + "1e2", +]) { + test(`invalid stored volume ${JSON.stringify(stored)} defaults to 100`, () => { + const f = fixture(stored); + f.volume.apply(); + assert.equal(f.volume.level, 100); + assert.equal(f.audio.volume, 1); + assert.equal(f.audio.muted, false); + }); +} + +test("positive volume survives reload but zero and mute do not", () => { + const f = fixture("42"); + f.volume.apply(); + assert.equal(f.audio.volume, 0.42); + f.volume.setVolume(23); + f.volume.setVolume(0); + assert.equal(f.volume.muted, true); + assert.equal(f.audio.muted, true); + assert.equal(f.storage.getItem(JIM_VOLUME_KEY), "23"); + const reloaded = createJimVolume({ storage: f.storage }); + assert.equal(reloaded.level, 23); + assert.equal(reloaded.muted, false); + f.volume.toggleMute(); + assert.equal(f.audio.muted, false); + assert.equal(f.audio.volume, 0.23); + f.volume.toggleMute(); + f.volume.reset(); + assert.equal(f.audio.volume, 1); + assert.equal(f.audio.muted, false); + assert.equal(f.storage.getItem(JIM_VOLUME_KEY), "100"); +}); + +test("a single step up from a mute unmutes at the saved level, and a chosen value is kept", () => { + const f = fixture("27"); + f.volume.setVolume(0); + f.volume.setVolume(1); + assert.equal(f.volume.muted, false); + assert.equal(f.volume.level, 27); + assert.equal(f.audio.volume, 0.27); + assert.equal(f.storage.getItem(JIM_VOLUME_KEY), "27"); + f.volume.setVolume(40); + assert.equal(f.volume.level, 40); + assert.equal(f.storage.getItem(JIM_VOLUME_KEY), "40"); + f.volume.setVolume(0); + f.volume.setVolume(60); + assert.equal(f.volume.muted, false); + assert.equal(f.volume.level, 60); + assert.equal(f.storage.getItem(JIM_VOLUME_KEY), "60"); +}); + +test("newly attached elements inherit attenuation and session mute", () => { + const f = fixture("30"); + f.volume.toggleMute(); + const next = { volume: 1, muted: false }; + f.replace(next); + assert.equal(next.volume, 0.3); + assert.equal(next.muted, true); +}); + +for (const throws of [false, true]) { + test(`unsupported volume ${throws ? "throwing" : "no-op"} setter preserves mute`, () => { + const audio = { muted: false }; + Object.defineProperty(audio, "volume", { + get: () => 1, + set: () => { + if (throws) throw new Error("unsupported"); + }, + }); + const f = fixture("30", audio); + f.volume.apply(); + assert.equal(f.volume.supported, false); + f.volume.toggleMute(); + assert.equal(audio.muted, true); + f.volume.reset(); + assert.equal(audio.muted, false); + assert.equal(f.volume.supported, false); + }); +} + +test("invalid settings are ignored and blocked storage does not break playback preferences", () => { + const f = fixture(); + for (const value of [-1, 101, NaN, 1.5, "50"]) f.volume.setVolume(value); + assert.equal(f.volume.level, 100); + const volume = createJimVolume({ + storage: { + getItem: () => { + throw new Error("blocked"); + }, + setItem: () => { + throw new Error("blocked"); + }, + }, + }); + volume.setVolume(33); + assert.equal(volume.level, 33); + volume.reset(); + assert.equal(volume.level, 100); +}); + +test("interview attach, replacement and drop/resubscribe reapply volume and mute", () => { + const own = { + jimAudio: null, + jimTrack: null, + isCurrentAgent: () => true, + isAvatarAnalyserTrack: () => false, + audioPlayback: { play: () => Promise.resolve(), forget: () => {} }, + document: { + createElement: () => ({ + volume: 1, + muted: false, + isConnected: false, + remove() { + this.isConnected = false; + }, + }), + body: { + appendChild: (element) => { + element.isConnected = true; + }, + }, + }, + }; + own.jimVolume = createJimVolume({ + element: () => own.jimAudio, + storage: memoryStorage(), + }); + const { playRemoteAudio: play, dropRemoteAudio: drop } = loadInterview( + ["playRemoteAudio", "dropRemoteAudio"], + own, + { stubMissing: true }, + ); + const track = () => ({ + kind: "audio", + elements: [], + attach(element) { + this.elements.push(element); + element.muted = false; + }, + detach(only) { + const removed = this.elements.filter( + (element) => !only || element === only, + ); + this.elements = this.elements.filter( + (element) => !removed.includes(element), + ); + return removed; + }, + }); + own.jimVolume.setVolume(35); + own.jimVolume.toggleMute(); + const first = track(); + play(first, {}); + assert.equal(own.jimAudio.volume, 0.35); + assert.equal(own.jimAudio.muted, true); + const original = own.jimAudio; + const replacement = track(); + play(replacement, {}); + assert.equal(own.jimAudio, original); + assert.equal(own.jimAudio.muted, true); + drop(replacement); + assert.equal(own.jimAudio, null); + play(track(), {}); + assert.notEqual(own.jimAudio, original); + assert.equal(own.jimAudio.volume, 0.35); + assert.equal(own.jimAudio.muted, true); +}); + +test("unsupported setter UI hides its slider, keeps mute and moves focus before hiding", () => { + const nodes = { + jimVolumeRange: { + value: "", + setAttribute(name, value) { + this[name] = value; + }, + }, + jimVolumeRow: { hidden: false }, + jimVolumeValue: { textContent: "" }, + jimVolumeUnsupported: { hidden: true }, + jimMutedIndicator: { hidden: true }, + jimMute: { + setAttribute(name, value) { + this[name] = value; + }, + focus() { + own.document.activeElement = this; + }, + }, + }; + const own = { + nodes, + jimVolume: { supported: false, muted: true, level: 30 }, + document: { activeElement: nodes.jimVolumeRange }, + }; + loadInterview(["renderJimVolume"], own).renderJimVolume(); + assert.equal(nodes.jimVolumeRow.hidden, true); + assert.equal(nodes.jimVolumeUnsupported.hidden, false); + assert.equal(nodes.jimMutedIndicator.hidden, false); + assert.equal(nodes.jimMute["aria-pressed"], "true"); + assert.equal(own.document.activeElement, nodes.jimMute); +}); + +test("unsolicited LiveKit unmute and volume changes restore preferences without a room status event", () => { + const audio = Object.assign(new EventTarget(), { volume: 1, muted: false }); + const f = fixture("32", audio); + f.volume.toggleMute(); + audio.muted = false; + audio.volume = 1; + audio.dispatchEvent(new Event("volumechange")); + assert.equal(audio.muted, true); + assert.equal(audio.volume, 0.32); + const next = Object.assign(new EventTarget(), { volume: 1, muted: false }); + f.replace(next); + audio.muted = false; + audio.dispatchEvent(new Event("volumechange")); + assert.equal(audio.muted, false); + assert.equal(next.muted, true); + f.replace(null); + next.muted = false; + next.dispatchEvent(new Event("volumechange")); + assert.equal(next.muted, false); +}); diff --git a/tests/browser/meet-tab-audio.test.js b/tests/browser/meet-tab-audio.test.js index 8b06f0d85..d5933e733 100644 --- a/tests/browser/meet-tab-audio.test.js +++ b/tests/browser/meet-tab-audio.test.js @@ -37,6 +37,14 @@ const steps = [ "Enable Share tab audio before clicking Share.", ]; +// docs/meet-tab-audio-manual-check.md has the operator reset Jim's volume by +// these names before recording. A rename here strands that step. +test("audio controls keep the labels the Meet manual check names", () => { + assert.match(page, /Audio controls<\/summary>/); + assert.match(page, />\s*Reset volume\s*\s*Mute Jim\s* { for (const step of steps) { assert.ok( diff --git a/web/audio-output.js b/web/audio-output.js index 8142dfd25..d8e792eab 100644 --- a/web/audio-output.js +++ b/web/audio-output.js @@ -144,3 +144,92 @@ export function writeStored(key, value, storage = storageArea("localStorage")) { // Private browsing refuses writes; a preference is not worth failing on. } } + +export const JIM_VOLUME_KEY = "codetrial:jimVolume"; + +const sameVolume = (a, b) => Math.abs(a - b) < 0.001; + +/// Some browsers ignore or reject a volume write; read it back to tell. +function setsVolume(audio, value) { + try { + audio.volume = value; + return sameVolume(audio.volume, value); + } catch { + return false; + } +} + +/// Zero is a session mute, never a persisted level: reloading must restore the +/// last positive setting instead of starting a new interviewer silently. +export function createJimVolume({ + storage = storageArea("localStorage"), + element = () => null, + onChange = () => {}, +} = {}) { + const stored = readStored(JIM_VOLUME_KEY, storage); + let level = /^(?:[1-9][0-9]?|100)$/.test(stored ?? "") ? Number(stored) : 100; + let muted = false; + let supported = true; + let tested = null; + const restore = () => { + const audio = element(); + if ( + audio && + (audio.muted !== muted || + (supported && !sameVolume(audio.volume, level / 100))) + ) + apply(); + }; + function apply() { + const audio = element(); + const fresh = tested !== audio; + if (fresh) { + tested?.removeEventListener?.("volumechange", restore); + tested = audio; + } + if (audio) { + // At 100%, a no-op setter and a working setter read identically. Probe + // another level synchronously, then restore the requested attenuation. + if (fresh) supported = setsVolume(audio, 0.5); + if (supported) supported = setsVolume(audio, level / 100); + audio.muted = muted; + if (fresh) audio.addEventListener?.("volumechange", restore); + } + onChange(); + } + function setVolume(value) { + if (!Number.isInteger(value) || value < 0 || value > 100) return; + // A muted slider sits at 0, so a single step up reads as 1%. Unmute at + // the saved level rather than saving that step; a value the candidate + // pointed at or jumped to directly is taken as chosen. + if (muted && value === 1) { + muted = false; + apply(); + return; + } + muted = value === 0; + if (value > 0) { + level = value; + writeStored(JIM_VOLUME_KEY, String(level), storage); + } + apply(); + } + return { + get level() { + return level; + }, + get muted() { + return muted; + }, + get supported() { + return supported; + }, + apply, + setVolume, + toggleMute() { + muted = !muted; + apply(); + }, + reset: () => setVolume(100), + }; +} diff --git a/web/audio-playback.js b/web/audio-playback.js index f40c54062..249690cc3 100644 --- a/web/audio-playback.js +++ b/web/audio-playback.js @@ -22,6 +22,7 @@ export function createAudioPlayback({ button, focusTarget, announcement, + reapply = () => {}, }) { let room = null; let event = null; @@ -49,7 +50,10 @@ export function createAudioPlayback({ } function status() { - if (element) observe({ type: "status", allowed: room?.canPlaybackAudio }); + if (element) { + reapply(); + observe({ type: "status", allowed: room?.canPlaybackAudio }); + } } /// Drops the current source silently: a source that went away was not @@ -82,6 +86,7 @@ export function createAudioPlayback({ revision += 1; element = nextElement; pending = false; + reapply(); render(); if (room.canPlaybackAudio === false) observe({ type: "status", allowed: false }); @@ -112,11 +117,14 @@ export function createAudioPlayback({ // Both calls start inside the click gesture. Awaiting startAudio first // can spend transient activation before the element's play request. The // executor runs synchronously, and turns a throw into a rejection. + // startAudio can unmute the element synchronously, so reapply runs between + // the two: the play request must never start with Jim's mute undone. const start = (call) => new Promise((resolve) => resolve(call())); - void Promise.allSettled([ - start(() => room.startAudio()), - start(() => element.play()), - ]).then(([, played]) => { + const unlocked = start(() => room.startAudio()); + reapply(); + const attempts = [unlocked, start(() => element.play())]; + void Promise.allSettled(attempts).then(([, played]) => { + reapply(); if (current !== revision) return; pending = false; observe( diff --git a/web/interview.html b/web/interview.html index 825931bd0..d15a88c05 100644 --- a/web/interview.html +++ b/web/interview.html @@ -155,6 +155,10 @@

Loading interview...

+

+ Jim's local volume and mute may also change what Meet + participants hear. Check their audio after adjusting it. +

Once you share the tab, Google Meet controls who receives this interview. Recordings or copies made in Meet, or by anyone in @@ -771,6 +775,47 @@

+
+ Audio controls +
+ + 100% + +
+ +
+ + +
+
+