diff --git a/docs/latency-observability.md b/docs/latency-observability.md index 2fac5228c..d3a57fe02 100644 --- a/docs/latency-observability.md +++ b/docs/latency-observability.md @@ -169,6 +169,40 @@ out, and input transport before the agent on the way in, are outside this record. A browser-perceived figure needs a matched audio measurement in the browser. +## Browser audio smoke evidence + +The credentialed `voice` and `barge` browser-check flows also capture +`jimAudio`: received byte/packet deltas, peak decoded RMS, playback-blocked +state and a bounded result per observed audio subscription. The harness uses +LiveKit's public track-specific `getRTCStatsReport()` and an analyser on the +same native track attached as Jim. It records no device labels or audio +samples. + +Sampling waits up to 120 seconds for Speaking, then samples at 100 ms +intervals for at most 10 seconds, stopping early once both counters grow and +RMS exceeds 0.001. Counter resets and receiver changes start a new baseline; +different track generations never share deltas. At most 64 subscriptions are +sampled; exceeding that limit is an unavailable observation, not a healthy +verdict. + +`unavailable` means there were insufficient counter pairs or decoded samples, +no attached Jim track, no Speaking interval, cancellation, a sampling error or +a subscription limit. `transport-silent` means bytes or packets, either one, +did not grow across the observed counter pairs, and `decoded-silent` means both +grew but decoded RMS stayed at or below the floor. These are separate failures. +The capture retains failed observations, and each resubscription that keeps its +track through its sampling window must pass independently. A generation whose +track is replaced, unsubscribed or disconnected before Speaking or within its +first second of it, before both counters grew and RMS exceeded the floor, is +marked `superseded`, since losing the track cut its window short and a fresh +analyser first reads near zero; it cannot fail an otherwise healthy +replacement, and the last generation must pass. A generation that sampled +longer is judged on its evidence even if its track then ends. Unmeasured +counters and RMS are `null`, not observed zeroes. This measures transport and +decoded signal during Speaking, not output-device audibility, physical level or +per-word delivery. The RMS floor is a smoke-check threshold, not a calibrated +speech detector. Local loopback tests do not replace the isolated provider run. + ## Validation The credential-free gate covers the trace lifecycle, the waterfall and the diff --git a/docs/meet-tab-audio-manual-check.md b/docs/meet-tab-audio-manual-check.md index 2f4930e89..86f0ae7f6 100644 --- a/docs/meet-tab-audio-manual-check.md +++ b/docs/meet-tab-audio-manual-check.md @@ -37,20 +37,25 @@ re-vendored, or before relying on the mode for a real interview. confirm Jim moves to it. This step is what catches a broken device list or a preference that fails to persist, so do not skip it when Jim is already audible. -3. In Meet, click Present now. -4. Choose "A tab". -5. Select the CodeTrial tab. -6. Enable "Share tab audio" before clicking Share. -7. Confirm Meet's microphone is still the candidate microphone, not "None" and +3. Open "Audio controls" on the stage. Jim's volume and mute set the level of + the tab audio Meet captures, and the volume is saved across sessions, so a + low level left from an earlier run reaches the interviewer as a quiet Jim. + Press Reset volume, and confirm the slider reads 100% and "Mute Jim" is not + pressed. +4. In Meet, click Present now. +5. Choose "A tab". +6. Select the CodeTrial tab. +7. Enable "Share tab audio" before clicking Share. +8. Confirm Meet's microphone is still the candidate microphone, not "None" and not a virtual device. -8. Record 10 seconds in Meet while Jim is speaking and the candidate says one +9. Record 10 seconds in Meet while Jim is speaking and the candidate says one sentence. ## Pass criteria All five must hold: -- The recording contains Jim's voice. +- The recording contains Jim's voice at the level set in step 3. - The recording contains the CodeTrial tab video. - The recording contains the candidate's voice. - Meet's selected microphone is still the candidate's physical microphone at diff --git a/scripts/browser-check.cjs b/scripts/browser-check.cjs index b68cf62e1..b74072547 100644 --- a/scripts/browser-check.cjs +++ b/scripts/browser-check.cjs @@ -10,6 +10,7 @@ const { chromium } = require(process.env.PLAYWRIGHT_PATH); const assert = require("node:assert/strict"); const fs = require("fs"); +const { mediaProbeSource, assertJimAudio } = require("./media-probe.cjs"); const { installBargeReplay } = require("./barge-replay.cjs"); const { FloorCapture, @@ -793,6 +794,8 @@ async function isolateRustAgent( try { const page = await browser.newPage(); let bargeReplay = null; + let jimAudio = null; + let probingAudio = false; if (process.env.BROWSER_CHECK_FLOW === "barge") { await page.addInitScript( installBargeReplay, @@ -2310,6 +2313,10 @@ PatternLexicon.prototype.search = function(word) { } await page.goto(interviewUrl(PROBLEM_IDS.twoSum)); + probingAudio = credentialed && ["voice", "barge"].includes(flow); + if (probingAudio) { + await page.evaluate(mediaProbeSource()); + } await clearMediaGate(page); if (credentialed && !rustAgentIdentity) { roomName = await page @@ -2478,6 +2485,14 @@ PatternLexicon.prototype.search = function(word) { ); } } + if (probingAudio) { + jimAudio = await page.evaluate(() => + globalThis.__finishJimAudioProbe(), + ); + if (process.env.BROWSER_CHECK_CAPTURE) + writeIncompleteCapture({ jimAudio, bargeReplay }); + assertJimAudio(jimAudio); + } if (credentialed && flow === "barge") { if (!floorLogPath) throw new Error( @@ -2543,6 +2558,7 @@ PatternLexicon.prototype.search = function(word) { ? readFloorRecords(floorLogPath) : [], bargeReplay, + jimAudio, latencyLog: (agentOutput.join("") + "\n" + readServerLog()) .split("\n") .filter((line) => line.includes("codetrial latency ")) @@ -2557,10 +2573,16 @@ PatternLexicon.prototype.search = function(word) { })(), agentFailure ?? never, ]).catch(async (error) => { - if (process.env.BROWSER_CHECK_CAPTURE && floorLogPath) + if (probingAudio && !jimAudio) { + jimAudio = await page + .evaluate(() => globalThis.__finishJimAudioProbe(true)) + .catch(() => null); + } + if (process.env.BROWSER_CHECK_CAPTURE && (floorLogPath || probingAudio)) writeIncompleteCapture({ - floorRecords: readFloorRecords(floorLogPath), + floorRecords: floorLogPath ? readFloorRecords(floorLogPath) : [], bargeReplay, + jimAudio, }); console.error( redact( diff --git a/scripts/media-probe.cjs b/scripts/media-probe.cjs new file mode 100644 index 000000000..7de590842 --- /dev/null +++ b/scripts/media-probe.cjs @@ -0,0 +1,328 @@ +// The sampler is also injected into Chromium. Keep it independent of Node and +// LiveKit internals: getRTCStatsReport is the public, track-specific receiver API. +function createMediaSampler({ + now = () => performance.now(), + delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), + makeContext = () => + new (globalThis.AudioContext || globalThis.webkitAudioContext)(), + makeStream = (track) => new MediaStream([track]), + timeoutMs = 10000, + speakingWaitMs = timeoutMs, + intervalMs = 100, + rmsFloor = 0.001, + minimumWindowMs = 1000, +} = {}) { + return async function sample( + track, + generation, + isCurrent, + isSpeaking, + isStopped = () => false, + ) { + const began = now(); + const result = { + generation, + durationMs: 0, + samples: 0, + statsSamples: 0, + bytesReceived: 0, + packetsReceived: 0, + peakRms: null, + status: "unavailable", + reason: null, + }; + let context, source, analyser; + let windowBegan = null; + let previous = null; + let transportPairs = 0; + // Both counters grow only with a counter pair, and peakRms is set only by + // a decoded sample, so these three carry the whole proof. + const proven = () => + result.bytesReceived > 0 && + result.packetsReceived > 0 && + result.peakRms > rmsFloor; + const bounded = async (promise) => { + let timer; + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error("timeout")), + Math.max( + 1, + Math.min(500, timeoutMs - (now() - (windowBegan ?? began))), + ), + ); + }), + ]); + } finally { + clearTimeout(timer); + } + }; + try { + while ( + now() - (windowBegan ?? began) < + (windowBegan === null ? speakingWaitMs : timeoutMs) && + isCurrent() && + !isStopped() + ) { + if (!isSpeaking()) { + previous = null; + await delay(intervalMs); + continue; + } + windowBegan ??= now(); + result.samples++; + if (!context) { + try { + context = makeContext(); + // Not awaited: a slow resume must neither hold up receiver samples + // nor end decoded ones. The analyser is read once the context runs. + Promise.resolve() + .then(() => context.resume()) + .catch(() => {}); + source = context.createMediaStreamSource( + makeStream(track.mediaStreamTrack), + ); + analyser = context.createAnalyser(); + analyser.fftSize = 1024; + source.connect(analyser); + } catch {} + } + let stats; + try { + stats = await bounded( + Promise.resolve().then(() => track.getRTCStatsReport?.()), + ); + } catch {} + if (!isCurrent() || isStopped()) break; + if (!isSpeaking()) { + previous = null; + continue; + } + const inbound = []; + stats?.forEach((entry) => { + if ( + entry.type === "inbound-rtp" && + (entry.kind === "audio" || entry.mediaType === "audio") && + Number.isFinite(entry.bytesReceived) && + entry.bytesReceived >= 0 && + Number.isFinite(entry.packetsReceived) && + entry.packetsReceived >= 0 + ) + inbound.push(entry); + }); + // A receiver should expose one inbound audio stream. Ambiguous reports + // cannot prove which stream reached this track. + const current = inbound.length === 1 ? inbound[0] : null; + if (current) { + result.statsSamples++; + if ( + previous && + previous.id === current.id && + current.bytesReceived >= previous.bytesReceived && + current.packetsReceived >= previous.packetsReceived + ) { + result.bytesReceived += + current.bytesReceived - previous.bytesReceived; + result.packetsReceived += + current.packetsReceived - previous.packetsReceived; + transportPairs++; + } + previous = { + id: current.id, + bytesReceived: current.bytesReceived, + packetsReceived: current.packetsReceived, + }; + } else { + previous = null; + } + if (analyser && context.state === "running") { + const values = new Float32Array(analyser.fftSize); + analyser.getFloatTimeDomainData(values); + const rms = Math.sqrt( + values.reduce((sum, value) => sum + value * value, 0) / + values.length, + ); + if (!Number.isFinite(rms)) throw new Error("invalid decoded samples"); + result.peakRms = Math.max(result.peakRms ?? 0, rms); + } + if (proven()) break; + await delay(intervalMs); + } + if (isStopped()) { + result.reason = "sampling-cancelled"; + } else if ( + !isCurrent() && + !proven() && + (windowBegan === null || now() - windowBegan < minimumWindowMs) + ) { + // Losing the track early cuts the window short, and a fresh analyser + // reads near zero at first. A replacement is judged on its own; with + // none, the summary fails on this generation being the last. A longer + // window is judged on its evidence, so a reconnect cannot hide it. + result.status = "superseded"; + result.reason = transportPairs + ? "track-ended-before-signal" + : "track-ended-before-counter-pair"; + } else if (result.samples === 0) { + result.reason = "no-speaking-samples"; + } else if (!transportPairs) { + result.reason = "receiver-stats-unavailable"; + } else if (result.peakRms === null) { + result.reason = "decoded-signal-unavailable"; + } else if (!result.bytesReceived || !result.packetsReceived) { + result.status = "transport-silent"; + result.reason = "audio-packets-did-not-increase"; + } else if (result.peakRms <= rmsFloor) { + result.status = "decoded-silent"; + result.reason = "packets-arrived-without-decoded-signal"; + } else { + result.status = "ok"; + } + } catch { + result.reason = "sampling-failed"; + } finally { + try { + source?.disconnect(); + } catch {} + try { + analyser?.disconnect(); + } catch {} + if (context) + await bounded(Promise.resolve().then(() => context.close())).catch( + () => {}, + ); + result.durationMs = Math.max(0, now() - began); + result.counterPairs = transportPairs; + if (!transportPairs) { + result.bytesReceived = null; + result.packetsReceived = null; + } + } + return result; + }; +} + +// Installed after navigation, while the preflight still gates room creation. +// Observe public Room events and sample only the track the page attaches as Jim. +function installMediaProbe() { + const sdk = globalThis.LivekitClient; + const OriginalRoom = sdk.Room; + const sample = globalThis.__createMediaSampler({ speakingWaitMs: 120000 }); + let generation = 0; + let active = null; + let stopped = false; + let accepting = true; + let overflow = false; + const jobs = []; + // Every way a track stops being Jim's moves the generation, which is what + // ends its sampler. + const retire = () => { + active = null; + generation++; + }; + const speaking = () => + document.querySelector("#agent-state")?.textContent.includes("Speaking"); + sdk.Room = class extends OriginalRoom { + constructor(...args) { + super(...args); + this.on(sdk.RoomEvent.TrackSubscribed, (track) => { + if (!accepting || track.kind !== "audio") return; + // App listeners attach Jim after this listener. A microtask sees that + // choice without guessing agent identity or retaining participant data. + queueMicrotask(() => { + if (!accepting) return; + const element = document.querySelector("#jim-audio"); + if ( + !element?.srcObject + ?.getAudioTracks() + .includes(track.mediaStreamTrack) + ) + return; + if (jobs.length >= 64) { + overflow = true; + retire(); + return; + } + active = track; + const mine = ++generation; + jobs.push( + sample( + track, + mine, + () => mine === generation, + speaking, + () => stopped, + ), + ); + }); + }); + this.on(sdk.RoomEvent.TrackUnsubscribed, (track) => { + if (active === track) retire(); + }); + this.on(sdk.RoomEvent.Disconnected, retire); + } + }; + globalThis.__finishJimAudioProbe = async (cancel = false) => { + accepting = false; + if (cancel) stopped = true; + sdk.Room = OriginalRoom; + const generations = await Promise.all(jobs); + if (overflow) + generations.push({ + generation, + status: "unavailable", + reason: "subscription-observation-limit", + bytesReceived: null, + packetsReceived: null, + peakRms: null, + }); + return globalThis.__summarizeJimAudio( + generations, + document.querySelector("#audio-playback-notice")?.hidden === false, + ); + }; +} + +function summarizeJimAudio(generations, playbackBlocked = null) { + const last = generations.at(-1); + const failure = + generations.find((entry) => !["ok", "superseded"].includes(entry.status)) ?? + (last?.status === "ok" ? null : last); + const measured = (field) => + generations.map((entry) => entry[field]).filter(Number.isFinite); + const sum = (field) => + measured(field).reduce((total, value) => (total ?? 0) + value, null); + const peaks = measured("peakRms"); + return { + status: failure?.status ?? (last ? "ok" : "unavailable"), + reason: failure?.reason ?? (last ? null : "no-attached-agent-track"), + failedGeneration: failure?.generation ?? null, + playbackBlocked, + bytesReceived: sum("bytesReceived"), + packetsReceived: sum("packetsReceived"), + peakRms: peaks.length ? Math.max(...peaks) : null, + generations, + }; +} + +function mediaProbeSource() { + return `globalThis.__createMediaSampler = ${createMediaSampler.toString()}; globalThis.__summarizeJimAudio = ${summarizeJimAudio.toString()}; (${installMediaProbe.toString()})();`; +} + +function assertJimAudio(result) { + const verdict = summarizeJimAudio(result?.generations ?? []); + if (verdict.status !== "ok") + throw new Error( + `Jim audio ${verdict.status}: ${verdict.reason} (generation ${verdict.failedGeneration ?? "none"})`, + ); +} +module.exports = { + createMediaSampler, + installMediaProbe, + summarizeJimAudio, + mediaProbeSource, + assertJimAudio, +}; diff --git a/tests/browser/audio-playback.test.js b/tests/browser/audio-playback.test.js index e7b8bfa84..1d2c1c65e 100644 --- a/tests/browser/audio-playback.test.js +++ b/tests/browser/audio-playback.test.js @@ -20,7 +20,7 @@ function deferred() { return { promise, resolve, reject }; } -function fixture() { +function fixture({ reapply = () => {} } = {}) { const notice = { hidden: true }; const document = { activeElement: null }; const button = new EventTarget(); @@ -38,6 +38,7 @@ function fixture() { button, focusTarget, announcement, + reapply, }); playback.setRoom(room, "playback"); return { @@ -386,3 +387,86 @@ test("report recovery keeps playback tracked; only the rendered report ends it", assert.equal(f.notice.hidden, true); assert.equal(f.room.listenerCount("playback"), 0); }); + +test("playback recovery restores session mute after LiveKit unmutes its elements", async () => { + const { createJimVolume } = await import("../../web/audio-output.js"); + let refused = true; + const element = { + volume: 1, + muted: false, + play: async () => { + if (refused) throw blocked(); + }, + }; + const volume = createJimVolume({ element: () => element, storage: null }); + volume.setVolume(25); + volume.toggleMute(); + const f = fixture({ reapply: () => volume.apply() }); + await f.playback.play(element); + f.room.startAudio = async () => { + element.muted = false; + }; + refused = false; + f.click(); + await tick(); + assert.equal(f.notice.hidden, true); + assert.equal(element.muted, true); + assert.equal(element.volume, 0.25); + element.muted = false; + f.status(true); + assert.equal(element.muted, true); +}); + +test("a late old-room recovery reapplies preferences to the currently attached element", async () => { + const { createJimVolume } = await import("../../web/audio-output.js"); + let current = { ...denied(), volume: 1, muted: false }; + const volume = createJimVolume({ element: () => current, storage: null }); + volume.toggleMute(); + const f = fixture({ reapply: () => volume.apply() }); + await f.playback.play(current); + const held = deferred(); + f.room.startAudio = () => + held.promise.then(() => { + current.muted = false; + }); + f.click(); + // The interview reconnects: a new room and a new element arrive while the + // old room's recovery is still pending. + const next = new EventEmitter(); + next.canPlaybackAudio = true; + next.startAudio = async () => {}; + f.playback.setRoom(next, "playback"); + current = { ...playing(), volume: 1, muted: false }; + await f.playback.play(current); + held.resolve(); + await tick(); + assert.equal(current.muted, true); +}); + +test("synchronous startAudio unmute is corrected before a stalled resume can play", async () => { + let muted = true; + const playedMuted = []; + const element = { + play: async () => { + playedMuted.push(muted); + throw blocked(); + }, + }; + const f = fixture({ + reapply: () => { + muted = true; + }, + }); + await f.playback.play(element); + const held = deferred(); + f.room.startAudio = () => { + muted = false; + return held.promise; + }; + playedMuted.length = 0; + f.click(); + assert.equal(muted, true); + assert.deepEqual(playedMuted, [true]); + held.resolve(); + await tick(); +}); diff --git a/tests/browser/interview-layout.test.js b/tests/browser/interview-layout.test.js index 05b7a30c4..a375afea7 100644 --- a/tests/browser/interview-layout.test.js +++ b/tests/browser/interview-layout.test.js @@ -335,6 +335,15 @@ test("on a phone the self view does not grow the card over the wait", async (t) document.querySelector("#captions-text").textContent = "Thanks for walking me through that. ".repeat(30); }); + await page.waitForFunction( + () => + parseFloat( + getComputedStyle(document.querySelector("#ending-overlay")) + .paddingBottom, + ) >= + document.querySelector("#jim-stage").getBoundingClientRect().height + + 47, + ); const under = await page.evaluate(() => { const card = document.querySelector("#jim-stage").getBoundingClientRect(); return [...document.querySelectorAll("#ending-overlay button")] diff --git a/tests/browser/jim-volume-ui.test.js b/tests/browser/jim-volume-ui.test.js new file mode 100644 index 000000000..dea5153c1 --- /dev/null +++ b/tests/browser/jim-volume-ui.test.js @@ -0,0 +1,154 @@ +import { before, after, test } from "node:test"; +import assert from "node:assert/strict"; +import { + launchChromium, + startStaticServer, + DEFAULT_RUNTIME_CONFIG, + closeSetupCheck, +} from "./source.js"; + +let browser, server, base; +before(async () => { + browser = await launchChromium(); + if (browser) + ({ server, base } = await startStaticServer({ + runtimeConfig: DEFAULT_RUNTIME_CONFIG, + })); +}); +after(async () => { + await browser?.close(); + await new Promise((resolve) => (server ? server.close(resolve) : resolve())); +}); +async function open(t, width = 1280, height = 720) { + const page = await browser.newPage({ viewport: { width, height } }); + t.after(() => page.close()); + await page.goto(`${base}/interview.html?problem=chargeback-pair-match`); + await openControls(page); + return page; +} + +async function openControls(page) { + await page.waitForFunction( + () => document.querySelector("#candidate-case-input").placeholder !== "", + ); + await closeSetupCheck(page); + await page.locator(".jim-volume-controls summary").click(); +} + +test("volume controls preserve positive level across reload and expose session mute to keyboard users", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await open(t); + const range = page.getByRole("slider", { name: /Jim volume/ }); + await range.evaluate((input) => { + input.value = "27"; + input.dispatchEvent(new Event("input", { bubbles: true })); + }); + await page.waitForFunction( + () => document.querySelector("#jim-volume-value").textContent === "27%", + ); + await range.focus(); + await range.press("Home"); + await page.waitForFunction( + () => !document.querySelector("#jim-muted-indicator").hidden, + ); + assert.equal( + await page.locator("#jim-mute").getAttribute("aria-pressed"), + "true", + ); + assert.equal( + await page.evaluate(() => localStorage.getItem("codetrial:jimVolume")), + "27", + ); + await range.press("ArrowRight"); + await page.waitForFunction( + () => document.querySelector("#jim-muted-indicator").hidden, + ); + assert.equal(await range.inputValue(), "27"); + assert.equal( + await page.evaluate(() => localStorage.getItem("codetrial:jimVolume")), + "27", + ); + await range.press("Home"); + await page.reload(); + await openControls(page); + assert.equal(await range.inputValue(), "27"); + assert.equal(await page.locator("#jim-muted-indicator").isVisible(), false); + await page.locator("#jim-mute").click(); + await page.locator("#jim-volume-reset").click(); + await page.waitForFunction( + () => document.querySelector("#jim-volume-value").textContent === "100%", + ); + assert.equal( + await page.locator("#jim-mute").getAttribute("aria-pressed"), + "false", + ); +}); + +test("expanded audio controls, mute and playback recovery leave ending actions clear on a phone", async (t) => { + if (!browser) return t.skip("playwright chromium unavailable"); + const page = await open(t, 390, 480); + await page.locator("#jim-mute").click(); + await page.evaluate(() => { + document.querySelector("#ending-overlay").hidden = false; + document.querySelector("#ending-withdraw-consent").hidden = false; + document.querySelector("#audio-playback-notice").hidden = false; + document.querySelector("#captions-text").textContent = + "Thanks for walking through the solution. ".repeat(25); + }); + await page.waitForFunction( + () => + parseFloat( + getComputedStyle(document.querySelector("#ending-overlay")) + .paddingBottom, + ) >= + document.querySelector("#jim-stage").getBoundingClientRect().height + 47, + ); + await page.locator("#ending-overlay").evaluate((overlay) => { + overlay.scrollTop = overlay.scrollHeight; + }); + const overlap = await page.evaluate(() => { + const stage = document.querySelector("#jim-stage").getBoundingClientRect(); + return [...document.querySelectorAll("#ending-overlay button")] + .filter((button) => button.offsetParent) + .filter((button) => { + const box = button.getBoundingClientRect(); + return box.bottom > stage.top && box.top < stage.bottom; + }) + .map((button) => button.id); + }); + assert.deepEqual(overlap, []); + assert.equal(await page.locator("#jim-mute").isVisible(), true); + const actions = await page.evaluate(() => { + const stage = document.querySelector("#jim-stage").getBoundingClientRect(); + return [...document.querySelectorAll("#ending-overlay button")] + .filter((button) => button.offsetParent) + .map((button) => { + const box = button.getBoundingClientRect(); + return { + id: button.id, + reachable: + box.top >= 0 && + box.bottom <= stage.top && + box.bottom <= innerHeight, + }; + }); + }); + assert.ok(actions.length > 0); + assert.ok( + actions.every((action) => action.reachable), + JSON.stringify(actions), + ); + await page.locator("#resume-audio").focus(); + assert.equal( + await page.locator("#resume-audio").evaluate((button) => { + const box = button.getBoundingClientRect(); + return button.contains( + document.elementFromPoint( + box.x + box.width / 2, + box.y + box.height / 2, + ), + ); + }), + true, + ); +}); diff --git a/tests/browser/jim-volume.test.js b/tests/browser/jim-volume.test.js new file mode 100644 index 000000000..5fbfcc849 --- /dev/null +++ b/tests/browser/jim-volume.test.js @@ -0,0 +1,252 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createJimVolume, JIM_VOLUME_KEY } from "../../web/audio-output.js"; +import { loadInterview, memoryStorage } from "./source.js"; + +function fixture(stored = null, audio = { volume: 1, muted: false }) { + const storage = memoryStorage(); + if (stored !== null) storage.setItem(JIM_VOLUME_KEY, stored); + let current = audio; + const volume = createJimVolume({ storage, element: () => current }); + return { + volume, + storage, + audio, + replace: (next) => { + current = next; + volume.apply(); + }, + }; +} + +for (const stored of [ + null, + "", + "0", + "101", + "-1", + "NaN", + "1.5", + " 50 ", + "1e2", +]) { + test(`invalid stored volume ${JSON.stringify(stored)} defaults to 100`, () => { + const f = fixture(stored); + f.volume.apply(); + assert.equal(f.volume.level, 100); + assert.equal(f.audio.volume, 1); + assert.equal(f.audio.muted, false); + }); +} + +test("positive volume survives reload but zero and mute do not", () => { + const f = fixture("42"); + f.volume.apply(); + assert.equal(f.audio.volume, 0.42); + f.volume.setVolume(23); + f.volume.setVolume(0); + assert.equal(f.volume.muted, true); + assert.equal(f.audio.muted, true); + assert.equal(f.storage.getItem(JIM_VOLUME_KEY), "23"); + const reloaded = createJimVolume({ storage: f.storage }); + assert.equal(reloaded.level, 23); + assert.equal(reloaded.muted, false); + f.volume.toggleMute(); + assert.equal(f.audio.muted, false); + assert.equal(f.audio.volume, 0.23); + f.volume.toggleMute(); + f.volume.reset(); + assert.equal(f.audio.volume, 1); + assert.equal(f.audio.muted, false); + assert.equal(f.storage.getItem(JIM_VOLUME_KEY), "100"); +}); + +test("a single step up from a mute unmutes at the saved level, and a chosen value is kept", () => { + const f = fixture("27"); + f.volume.setVolume(0); + f.volume.setVolume(1); + assert.equal(f.volume.muted, false); + assert.equal(f.volume.level, 27); + assert.equal(f.audio.volume, 0.27); + assert.equal(f.storage.getItem(JIM_VOLUME_KEY), "27"); + f.volume.setVolume(40); + assert.equal(f.volume.level, 40); + assert.equal(f.storage.getItem(JIM_VOLUME_KEY), "40"); + f.volume.setVolume(0); + f.volume.setVolume(60); + assert.equal(f.volume.muted, false); + assert.equal(f.volume.level, 60); + assert.equal(f.storage.getItem(JIM_VOLUME_KEY), "60"); +}); + +test("newly attached elements inherit attenuation and session mute", () => { + const f = fixture("30"); + f.volume.toggleMute(); + const next = { volume: 1, muted: false }; + f.replace(next); + assert.equal(next.volume, 0.3); + assert.equal(next.muted, true); +}); + +for (const throws of [false, true]) { + test(`unsupported volume ${throws ? "throwing" : "no-op"} setter preserves mute`, () => { + const audio = { muted: false }; + Object.defineProperty(audio, "volume", { + get: () => 1, + set: () => { + if (throws) throw new Error("unsupported"); + }, + }); + const f = fixture("30", audio); + f.volume.apply(); + assert.equal(f.volume.supported, false); + f.volume.toggleMute(); + assert.equal(audio.muted, true); + f.volume.reset(); + assert.equal(audio.muted, false); + assert.equal(f.volume.supported, false); + }); +} + +test("invalid settings are ignored and blocked storage does not break playback preferences", () => { + const f = fixture(); + for (const value of [-1, 101, NaN, 1.5, "50"]) f.volume.setVolume(value); + assert.equal(f.volume.level, 100); + const volume = createJimVolume({ + storage: { + getItem: () => { + throw new Error("blocked"); + }, + setItem: () => { + throw new Error("blocked"); + }, + }, + }); + volume.setVolume(33); + assert.equal(volume.level, 33); + volume.reset(); + assert.equal(volume.level, 100); +}); + +test("interview attach, replacement and drop/resubscribe reapply volume and mute", () => { + const own = { + jimAudio: null, + jimTrack: null, + isCurrentAgent: () => true, + isAvatarAnalyserTrack: () => false, + audioPlayback: { play: () => Promise.resolve(), forget: () => {} }, + document: { + createElement: () => ({ + volume: 1, + muted: false, + isConnected: false, + remove() { + this.isConnected = false; + }, + }), + body: { + appendChild: (element) => { + element.isConnected = true; + }, + }, + }, + }; + own.jimVolume = createJimVolume({ + element: () => own.jimAudio, + storage: memoryStorage(), + }); + const { playRemoteAudio: play, dropRemoteAudio: drop } = loadInterview( + ["playRemoteAudio", "dropRemoteAudio"], + own, + { stubMissing: true }, + ); + const track = () => ({ + kind: "audio", + elements: [], + attach(element) { + this.elements.push(element); + element.muted = false; + }, + detach(only) { + const removed = this.elements.filter( + (element) => !only || element === only, + ); + this.elements = this.elements.filter( + (element) => !removed.includes(element), + ); + return removed; + }, + }); + own.jimVolume.setVolume(35); + own.jimVolume.toggleMute(); + const first = track(); + play(first, {}); + assert.equal(own.jimAudio.volume, 0.35); + assert.equal(own.jimAudio.muted, true); + const original = own.jimAudio; + const replacement = track(); + play(replacement, {}); + assert.equal(own.jimAudio, original); + assert.equal(own.jimAudio.muted, true); + drop(replacement); + assert.equal(own.jimAudio, null); + play(track(), {}); + assert.notEqual(own.jimAudio, original); + assert.equal(own.jimAudio.volume, 0.35); + assert.equal(own.jimAudio.muted, true); +}); + +test("unsupported setter UI hides its slider, keeps mute and moves focus before hiding", () => { + const nodes = { + jimVolumeRange: { + value: "", + setAttribute(name, value) { + this[name] = value; + }, + }, + jimVolumeRow: { hidden: false }, + jimVolumeValue: { textContent: "" }, + jimVolumeUnsupported: { hidden: true }, + jimMutedIndicator: { hidden: true }, + jimMute: { + setAttribute(name, value) { + this[name] = value; + }, + focus() { + own.document.activeElement = this; + }, + }, + }; + const own = { + nodes, + jimVolume: { supported: false, muted: true, level: 30 }, + document: { activeElement: nodes.jimVolumeRange }, + }; + loadInterview(["renderJimVolume"], own).renderJimVolume(); + assert.equal(nodes.jimVolumeRow.hidden, true); + assert.equal(nodes.jimVolumeUnsupported.hidden, false); + assert.equal(nodes.jimMutedIndicator.hidden, false); + assert.equal(nodes.jimMute["aria-pressed"], "true"); + assert.equal(own.document.activeElement, nodes.jimMute); +}); + +test("unsolicited LiveKit unmute and volume changes restore preferences without a room status event", () => { + const audio = Object.assign(new EventTarget(), { volume: 1, muted: false }); + const f = fixture("32", audio); + f.volume.toggleMute(); + audio.muted = false; + audio.volume = 1; + audio.dispatchEvent(new Event("volumechange")); + assert.equal(audio.muted, true); + assert.equal(audio.volume, 0.32); + const next = Object.assign(new EventTarget(), { volume: 1, muted: false }); + f.replace(next); + audio.muted = false; + audio.dispatchEvent(new Event("volumechange")); + assert.equal(audio.muted, false); + assert.equal(next.muted, true); + f.replace(null); + next.muted = false; + next.dispatchEvent(new Event("volumechange")); + assert.equal(next.muted, false); +}); diff --git a/tests/browser/media-probe.test.js b/tests/browser/media-probe.test.js new file mode 100644 index 000000000..a79ebb713 --- /dev/null +++ b/tests/browser/media-probe.test.js @@ -0,0 +1,620 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + createMediaSampler, + assertJimAudio, + installMediaProbe, + summarizeJimAudio, + mediaProbeSource, +} from "../../scripts/media-probe.cjs"; + +function fixture({ + counts = [0, 10, 20], + rms = 0.1, + stats = true, + running = true, + contextError = false, + timeoutMs = 40, + speakingWaitMs = timeoutMs, + minimumWindowMs, + onDelay = () => {}, +} = {}) { + let clock = 0; + let index = 0; + let current = true; + let speaking = true; + let cleaned = 0; + let closed = 0; + const context = { + state: running ? "running" : "suspended", + resume: async () => {}, + close: async () => { + closed++; + }, + createMediaStreamSource: () => ({ + connect: () => {}, + disconnect: () => { + cleaned++; + }, + }), + createAnalyser: () => ({ + fftSize: 0, + getFloatTimeDomainData: (values) => values.fill(rms), + disconnect: () => { + cleaned++; + }, + }), + }; + const track = { + mediaStreamTrack: {}, + getRTCStatsReport: async () => { + if (!stats) return; + const value = counts[Math.min(index++, counts.length - 1)]; + return new Map([ + [ + "inbound", + { + type: "inbound-rtp", + kind: "audio", + id: "inbound", + bytesReceived: value, + packetsReceived: value, + }, + ], + ]); + }, + }; + const sampler = createMediaSampler({ + now: () => clock, + delay: async (ms) => { + clock += ms; + onDelay(); + }, + makeContext: () => { + if (contextError) throw new Error("unavailable"); + return context; + }, + makeStream: (input) => { + assert.equal(input, track.mediaStreamTrack); + return {}; + }, + timeoutMs, + speakingWaitMs, + minimumWindowMs, + intervalMs: 10, + }); + return { + track, + sampler, + context, + run: () => + sampler( + track, + 7, + () => current, + () => speaking, + ), + changeTrack: () => { + current = false; + }, + quiet: () => { + speaking = false; + }, + speak: () => { + speaking = true; + }, + cleaned: () => cleaned, + closed: () => closed, + }; +} + +test("track receiver counters and decoded signal establish distinct evidence", async () => { + const f = fixture(); + const result = await f.run(); + assert.equal(result.status, "ok"); + assert.equal(result.generation, 7); + assert.equal(result.bytesReceived, 10); + assert.equal(result.packetsReceived, 10); + assert.ok(result.peakRms > 0.01); + assert.equal(result.durationMs, 10); + assert.equal(f.cleaned(), 2); + assert.equal(f.closed(), 1); + assert.doesNotThrow(() => assertJimAudio({ generations: [result] })); +}); +for (const [options, status] of [ + [{ counts: [9, 9, 9] }, "transport-silent"], + [{ rms: 0 }, "decoded-silent"], + [{ stats: false }, "unavailable"], + [{ running: false }, "unavailable"], + [{ contextError: true }, "unavailable"], +]) { + test(`controlled ${JSON.stringify(options)} reports ${status}`, async () => { + const f = fixture(options); + const result = await f.run(); + assert.equal(result.status, status); + assert.throws( + () => assertJimAudio({ generations: [result] }), + new RegExp(status), + ); + if (!options.contextError) assert.equal(f.closed(), 1); + }); +} + +test("counter resets do not create a positive delta", async () => { + const f = fixture({ counts: [100, 1, 1, 1] }); + const result = await f.run(); + assert.equal(result.status, "transport-silent"); + assert.equal(result.bytesReceived, 0); +}); + +test("a replaced track cannot contribute late receiver observations", async () => { + const f = fixture(); + f.track.getRTCStatsReport = async () => { + f.changeTrack(); + return new Map([ + [ + "live", + { + type: "inbound-rtp", + kind: "audio", + id: "live", + bytesReceived: 100, + packetsReceived: 10, + }, + ], + ]); + }; + const result = await f.run(); + assert.equal(result.status, "superseded"); + assert.equal(result.statsSamples, 0); + assert.equal(f.closed(), 1); +}); + +test("a replacement after one counter pair supersedes a quiet generation instead of failing it", async () => { + let delays = 0; + const f = fixture({ + rms: 0, + timeoutMs: 100, + onDelay: () => { + if (++delays === 2) f.changeTrack(); + }, + }); + const result = await f.run(); + assert.equal(result.counterPairs, 1); + assert.equal(result.status, "superseded"); + assert.equal(result.reason, "track-ended-before-signal"); + assert.equal(f.closed(), 1); +}); + +test("a slow resume neither delays receiver samples nor ends decoded sampling", async () => { + let delays = 0; + let running; + const f = fixture({ + counts: [0, 10, 20, 30, 40, 50], + timeoutMs: 100, + onDelay: () => { + if (++delays === 3) running(); + }, + }); + f.context.state = "suspended"; + f.context.resume = () => + new Promise((resolve) => { + running = () => { + f.context.state = "running"; + resolve(); + }; + }); + const result = await f.run(); + assert.equal(result.status, "ok"); + assert.ok(result.statsSamples >= 4); + assert.equal(f.closed(), 1); +}); + +test("a generation that sampled a long window keeps its failure when its track ends", async () => { + let delays = 0; + const f = fixture({ + counts: [9], + timeoutMs: 1000, + minimumWindowMs: 100, + onDelay: () => { + if (++delays === 20) f.changeTrack(); + }, + }); + const result = await f.run(); + assert.equal(result.status, "transport-silent"); +}); + +test("no Speaking interval produces no signal claim or audio context", async () => { + const f = fixture(); + f.quiet(); + const result = await f.run(); + assert.equal(result.status, "unavailable"); + assert.equal(result.samples, 0); + assert.equal(result.reason, "no-speaking-samples"); + assert.equal(result.peakRms, null); + assert.equal(f.closed(), 0); +}); + +test("receiver timeout is bounded and releases its context", async () => { + const f = fixture({ timeoutMs: 10 }); + f.track.getRTCStatsReport = () => new Promise(() => {}); + const result = await f.run(); + assert.equal(result.status, "unavailable"); + assert.equal(f.closed(), 1); +}); + +test("a good old generation cannot hide a silent resubscription", () => { + assert.throws( + () => + assertJimAudio({ + generations: [ + { status: "ok" }, + { generation: 2, status: "transport-silent", reason: "flat" }, + ], + }), + /generation 2/, + ); + assert.throws( + () => assertJimAudio({ generations: [] }), + /no-attached-agent-track/, + ); +}); + +test("receiver identity resets and ambiguous streams cannot fabricate packet deltas", async () => { + const f = fixture(); + let calls = 0; + f.track.getRTCStatsReport = async () => + new Map([ + [ + "inbound", + { + type: "inbound-rtp", + kind: "audio", + id: `stream-${calls++}`, + bytesReceived: calls * 100, + packetsReceived: calls, + }, + ], + ]); + assert.equal((await f.run()).status, "unavailable"); + const g = fixture(); + g.track.getRTCStatsReport = async () => + new Map( + [1, 2].map((id) => [ + id, + { + type: "inbound-rtp", + kind: "audio", + id, + bytesReceived: 100, + packetsReceived: 10, + }, + ]), + ); + assert.equal((await g.run()).status, "unavailable"); +}); + +test("setup, analyser failure and refused cleanup remain bounded unavailable observations", async () => { + const f = fixture(); + f.context.createMediaStreamSource = () => { + throw new Error("setup"); + }; + assert.equal((await f.run()).status, "unavailable"); + assert.equal(f.closed(), 1); + const g = fixture(); + g.context.createAnalyser = () => ({ + disconnect: () => {}, + getFloatTimeDomainData: () => { + throw new Error("read"); + }, + }); + assert.equal((await g.run()).status, "unavailable"); + assert.equal(g.closed(), 1); + const h = fixture(); + h.context.close = async () => { + throw new Error("already closed"); + }; + assert.equal((await h.run()).status, "ok"); +}); + +test("public Room subscription hooks follow only Jim's attached track and keep generation outcomes", async (t) => { + const { EventEmitter } = await import("node:events"); + const names = [ + "LivekitClient", + "__createMediaSampler", + "__finishJimAudioProbe", + "__summarizeJimAudio", + "document", + ]; + const previous = names.map((name) => [ + name, + Object.getOwnPropertyDescriptor(globalThis, name), + ]); + t.after(() => { + for (const [name, descriptor] of previous) { + if (descriptor) Object.defineProperty(globalThis, name, descriptor); + else delete globalThis[name]; + } + }); + const events = { + TrackSubscribed: "sub", + TrackUnsubscribed: "unsub", + Disconnected: "disconnect", + }; + globalThis.LivekitClient = { Room: EventEmitter, RoomEvent: events }; + let attached; + globalThis.document = { + querySelector: (selector) => + selector === "#jim-audio" + ? { srcObject: { getAudioTracks: () => [attached] } } + : { textContent: "Speaking" }, + }; + const seen = []; + globalThis.__createMediaSampler = + () => async (track, generation, isCurrent, isSpeaking) => { + seen.push({ track, generation, isCurrent, isSpeaking }); + return { + generation, + status: "ok", + bytesReceived: 10, + packetsReceived: 1, + peakRms: 0.1, + }; + }; + globalThis.__summarizeJimAudio = summarizeJimAudio; + installMediaProbe(); + const room = new globalThis.LivekitClient.Room(); + const first = { kind: "audio", mediaStreamTrack: {} }; + const unrelated = { kind: "audio", mediaStreamTrack: {} }; + const replacement = { kind: "audio", mediaStreamTrack: {} }; + attached = first.mediaStreamTrack; + room.emit("sub", unrelated); + room.emit("sub", first); + await new Promise((resolve) => queueMicrotask(resolve)); + assert.equal(seen.length, 1); + assert.equal(seen[0].isCurrent(), true); + assert.equal(seen[0].isSpeaking(), true); + attached = replacement.mediaStreamTrack; + room.emit("sub", replacement); + await new Promise((resolve) => queueMicrotask(resolve)); + room.emit("unsub", first); + assert.equal(seen[0].isCurrent(), false); + assert.equal(seen[1].isCurrent(), true); + room.emit("disconnect"); + assert.equal(seen[1].isCurrent(), false); + const result = await globalThis.__finishJimAudioProbe(); + assert.equal(result.generations.length, 2); + assert.equal(result.bytesReceived, 20); + assert.equal(globalThis.LivekitClient.Room, EventEmitter); + installMediaProbe(); + const noisyRoom = new globalThis.LivekitClient.Room(); + for (let index = 0; index < 65; index++) { + const track = { kind: "audio", mediaStreamTrack: {} }; + attached = track.mediaStreamTrack; + noisyRoom.emit("sub", track); + await new Promise((resolve) => queueMicrotask(resolve)); + } + const overflow = await globalThis.__finishJimAudioProbe(); + assert.equal(overflow.generations.length, 65); + assert.equal(overflow.status, "unavailable"); + assert.equal( + overflow.generations.at(-1).reason, + "subscription-observation-limit", + ); + assert.equal(overflow.generations.at(-1).bytesReceived, null); + assert.equal(overflow.generations.at(-1).packetsReceived, null); +}); + +test("Chromium measures real received WebRTC packets and decoded tone without provider credentials", async (t) => { + const { launchChromium, root } = await import("./source.js"); + const { join } = await import("node:path"); + const { readFileSync } = await import("node:fs"); + const browser = await launchChromium({ + args: ["--autoplay-policy=no-user-gesture-required"], + }); + if (!browser) return t.skip("playwright chromium unavailable"); + t.after(() => browser.close()); + const page = await browser.newPage(); + await page.route("http://localhost:31999/**", async (route) => { + const script = route.request().url().endsWith("/vendor/livekit-client.js"); + await route.fulfill({ + status: 200, + headers: { + "content-type": script ? "text/javascript" : "text/html", + "content-security-policy": + "default-src 'self'; script-src 'self' blob: 'wasm-unsafe-eval'", + }, + body: script + ? readFileSync(join(root, "web/vendor/livekit-client.js"), "utf8") + : '', + }); + }); + await page.goto("http://localhost:31999/"); + await page.evaluate(mediaProbeSource()); + const sdkProbe = await page.evaluate(async () => { + const room = new globalThis.LivekitClient.Room(); + const result = await globalThis.__finishJimAudioProbe(); + await room.disconnect(); + return result; + }); + assert.equal(sdkProbe.status, "unavailable"); + assert.equal(sdkProbe.generations.length, 0); + const result = await page.evaluate(async () => { + const sender = new RTCPeerConnection(); + const receiver = new RTCPeerConnection(); + const context = new AudioContext(); + const oscillator = context.createOscillator(); + const destination = context.createMediaStreamDestination(); + oscillator.connect(destination); + oscillator.start(); + sender.onicecandidate = (event) => { + if (event.candidate) void receiver.addIceCandidate(event.candidate); + }; + receiver.onicecandidate = (event) => { + if (event.candidate) void sender.addIceCandidate(event.candidate); + }; + // Bounded, so a loopback that never negotiates fails here and still + // reaches the cleanup below instead of hanging the suite. + const received = new Promise((resolve, reject) => { + receiver.ontrack = resolve; + setTimeout(() => reject(new Error("loopback track timeout")), 10000); + }); + sender.addTrack(destination.stream.getAudioTracks()[0], destination.stream); + try { + await sender.setLocalDescription(await sender.createOffer()); + await receiver.setRemoteDescription(sender.localDescription); + await receiver.setLocalDescription(await receiver.createAnswer()); + await sender.setRemoteDescription(receiver.localDescription); + const { track, receiver: inbound } = await received; + const audio = document.createElement("audio"); + audio.srcObject = new MediaStream([track]); + document.body.append(audio); + await audio.play(); + return await globalThis.__createMediaSampler()( + { + mediaStreamTrack: track, + getRTCStatsReport: () => inbound.getStats(), + }, + 1, + () => true, + () => true, + ); + } finally { + oscillator.stop(); + destination.stream.getTracks().forEach((track) => track.stop()); + sender.close(); + receiver.close(); + await context.close(); + } + }); + assert.equal(result.status, "ok", JSON.stringify(result)); + assert.ok(result.bytesReceived > 0); + assert.ok(result.packetsReceived > 0); + assert.ok(result.peakRms > 0.001); +}); + +test("non-Speaking gaps cannot bridge counter deltas", async () => { + let ticks = 0; + const f = fixture({ + counts: [0, 100, 100, 100], + onDelay: () => { + ticks++; + if (ticks === 1) f.quiet(); + if (ticks === 2) f.speak(); + }, + }); + const result = await f.run(); + assert.equal(result.status, "transport-silent"); + assert.equal(result.bytesReceived, 0); +}); + +test("Speaking lost during a receiver await discards the sample and its baseline", async () => { + let calls = 0; + const f = fixture({ + onDelay: () => { + if (calls === 2) f.speak(); + }, + }); + f.track.getRTCStatsReport = async () => { + calls++; + if (calls === 2) f.quiet(); + return new Map([ + [ + "inbound", + { + type: "inbound-rtp", + kind: "audio", + id: "inbound", + bytesReceived: calls === 1 ? 0 : 100, + packetsReceived: calls === 1 ? 0 : 10, + }, + ], + ]); + }; + const result = await f.run(); + assert.equal(result.status, "transport-silent"); + assert.equal(result.bytesReceived, 0); +}); + +test("cancel discards a pending stats observation and closes the context", async () => { + const f = fixture(); + let stopped = false; + f.track.getRTCStatsReport = async () => { + stopped = true; + return new Map([ + [ + "inbound", + { + type: "inbound-rtp", + kind: "audio", + id: "inbound", + bytesReceived: 100, + packetsReceived: 10, + }, + ], + ]); + }; + const result = await f.sampler( + f.track, + 1, + () => true, + () => true, + () => stopped, + ); + assert.equal(result.statsSamples, 0); + assert.equal(result.status, "unavailable"); + assert.equal(result.reason, "sampling-cancelled"); + assert.equal(f.closed(), 1); +}); + +test("waiting for Speaking does not spend the sampling window", async () => { + let ticks = 0; + const f = fixture({ + timeoutMs: 20, + speakingWaitMs: 60, + onDelay: () => { + if (++ticks === 3) f.speak(); + }, + }); + f.quiet(); + const result = await f.run(); + assert.equal(result.status, "ok"); + assert.equal(result.durationMs, 40); +}); + +test("unknown aggregate observations remain null and superseded empty generations require a healthy successor", () => { + const absent = summarizeJimAudio([ + { + generation: 1, + status: "unavailable", + reason: "stats", + bytesReceived: null, + packetsReceived: null, + peakRms: null, + }, + ]); + assert.equal(absent.bytesReceived, null); + assert.equal(absent.packetsReceived, null); + assert.equal(absent.peakRms, null); + assert.equal(absent.reason, "stats"); + const superseded = { + generation: 1, + status: "superseded", + reason: "replaced", + bytesReceived: null, + packetsReceived: null, + peakRms: null, + }; + const healthy = { + generation: 2, + status: "ok", + bytesReceived: 10, + packetsReceived: 1, + peakRms: 0.1, + }; + assert.equal(summarizeJimAudio([superseded, healthy]).status, "ok"); + assert.throws( + () => assertJimAudio({ generations: [superseded] }), + /superseded/, + ); +}); diff --git a/tests/browser/meet-tab-audio.test.js b/tests/browser/meet-tab-audio.test.js index 8b06f0d85..d5933e733 100644 --- a/tests/browser/meet-tab-audio.test.js +++ b/tests/browser/meet-tab-audio.test.js @@ -37,6 +37,14 @@ const steps = [ "Enable Share tab audio before clicking Share.", ]; +// docs/meet-tab-audio-manual-check.md has the operator reset Jim's volume by +// these names before recording. A rename here strands that step. +test("audio controls keep the labels the Meet manual check names", () => { + assert.match(page, /Audio controls<\/summary>/); + assert.match(page, />\s*Reset volume\s*\s*Mute Jim\s* { for (const step of steps) { assert.ok( diff --git a/web/audio-output.js b/web/audio-output.js index 8142dfd25..d8e792eab 100644 --- a/web/audio-output.js +++ b/web/audio-output.js @@ -144,3 +144,92 @@ export function writeStored(key, value, storage = storageArea("localStorage")) { // Private browsing refuses writes; a preference is not worth failing on. } } + +export const JIM_VOLUME_KEY = "codetrial:jimVolume"; + +const sameVolume = (a, b) => Math.abs(a - b) < 0.001; + +/// Some browsers ignore or reject a volume write; read it back to tell. +function setsVolume(audio, value) { + try { + audio.volume = value; + return sameVolume(audio.volume, value); + } catch { + return false; + } +} + +/// Zero is a session mute, never a persisted level: reloading must restore the +/// last positive setting instead of starting a new interviewer silently. +export function createJimVolume({ + storage = storageArea("localStorage"), + element = () => null, + onChange = () => {}, +} = {}) { + const stored = readStored(JIM_VOLUME_KEY, storage); + let level = /^(?:[1-9][0-9]?|100)$/.test(stored ?? "") ? Number(stored) : 100; + let muted = false; + let supported = true; + let tested = null; + const restore = () => { + const audio = element(); + if ( + audio && + (audio.muted !== muted || + (supported && !sameVolume(audio.volume, level / 100))) + ) + apply(); + }; + function apply() { + const audio = element(); + const fresh = tested !== audio; + if (fresh) { + tested?.removeEventListener?.("volumechange", restore); + tested = audio; + } + if (audio) { + // At 100%, a no-op setter and a working setter read identically. Probe + // another level synchronously, then restore the requested attenuation. + if (fresh) supported = setsVolume(audio, 0.5); + if (supported) supported = setsVolume(audio, level / 100); + audio.muted = muted; + if (fresh) audio.addEventListener?.("volumechange", restore); + } + onChange(); + } + function setVolume(value) { + if (!Number.isInteger(value) || value < 0 || value > 100) return; + // A muted slider sits at 0, so a single step up reads as 1%. Unmute at + // the saved level rather than saving that step; a value the candidate + // pointed at or jumped to directly is taken as chosen. + if (muted && value === 1) { + muted = false; + apply(); + return; + } + muted = value === 0; + if (value > 0) { + level = value; + writeStored(JIM_VOLUME_KEY, String(level), storage); + } + apply(); + } + return { + get level() { + return level; + }, + get muted() { + return muted; + }, + get supported() { + return supported; + }, + apply, + setVolume, + toggleMute() { + muted = !muted; + apply(); + }, + reset: () => setVolume(100), + }; +} diff --git a/web/audio-playback.js b/web/audio-playback.js index f40c54062..249690cc3 100644 --- a/web/audio-playback.js +++ b/web/audio-playback.js @@ -22,6 +22,7 @@ export function createAudioPlayback({ button, focusTarget, announcement, + reapply = () => {}, }) { let room = null; let event = null; @@ -49,7 +50,10 @@ export function createAudioPlayback({ } function status() { - if (element) observe({ type: "status", allowed: room?.canPlaybackAudio }); + if (element) { + reapply(); + observe({ type: "status", allowed: room?.canPlaybackAudio }); + } } /// Drops the current source silently: a source that went away was not @@ -82,6 +86,7 @@ export function createAudioPlayback({ revision += 1; element = nextElement; pending = false; + reapply(); render(); if (room.canPlaybackAudio === false) observe({ type: "status", allowed: false }); @@ -112,11 +117,14 @@ export function createAudioPlayback({ // Both calls start inside the click gesture. Awaiting startAudio first // can spend transient activation before the element's play request. The // executor runs synchronously, and turns a throw into a rejection. + // startAudio can unmute the element synchronously, so reapply runs between + // the two: the play request must never start with Jim's mute undone. const start = (call) => new Promise((resolve) => resolve(call())); - void Promise.allSettled([ - start(() => room.startAudio()), - start(() => element.play()), - ]).then(([, played]) => { + const unlocked = start(() => room.startAudio()); + reapply(); + const attempts = [unlocked, start(() => element.play())]; + void Promise.allSettled(attempts).then(([, played]) => { + reapply(); if (current !== revision) return; pending = false; observe( diff --git a/web/interview.html b/web/interview.html index 825931bd0..d15a88c05 100644 --- a/web/interview.html +++ b/web/interview.html @@ -155,6 +155,10 @@

Loading interview...

+

+ Jim's local volume and mute may also change what Meet + participants hear. Check their audio after adjusting it. +

Once you share the tab, Google Meet controls who receives this interview. Recordings or copies made in Meet, or by anyone in @@ -771,6 +775,47 @@

+
+ Audio controls +
+ + 100% + +
+ +
+ + +
+
+