From aa4985d95a3cd71045d287e1b9a1c5adf6af666f Mon Sep 17 00:00:00 2001 From: achlyschang Date: Sat, 24 Oct 2026 14:58:59 +0800 Subject: [PATCH] Warn about Gemini billing before an interview An interview runs on Gemini Live, which bills the server's Google AI project on the whole context of the session, and nothing told the operator or the candidate so before it started. The Setup page and the lobby now say who pays and what reaches the model: voice and video, voice, or voice and board images at a whiteboard. /api/session reports candidateVideo only when this process hosts the interviewer and null otherwise, since the web half of a split deployment cannot see the agent's settings, and the lobby shows the notice only when it knows. Close #161 --- docs/provider-cost-and-degradation.md | 16 +++ src/dispatch.rs | 4 + src/web/auth.rs | 9 ++ src/web/mod.rs | 11 ++ src/web/setup.rs | 1 + tests/browser/lobby.test.js | 195 ++++++++++++++++++++++++++ tests/browser/setup-page.test.js | 5 + tests/unit/dispatch.rs | 22 +++ tests/web/accounts.rs | 82 ++++++++++- web/app.js | 44 +++++- web/index.html | 9 +- web/styles.css | 7 + 12 files changed, 398 insertions(+), 7 deletions(-) diff --git a/docs/provider-cost-and-degradation.md b/docs/provider-cost-and-degradation.md index 3dd8dbcd..f5c5829a 100644 --- a/docs/provider-cost-and-degradation.md +++ b/docs/provider-cost-and-degradation.md @@ -325,6 +325,22 @@ says that no scores or verdict were created. Browser fallback may summarize local test progress only for a session that never reached an interviewer, and never presents canned feedback as an agent evaluation. +Before an interview on a server that hosts its own interviewer, the lobby says +above Start that Gemini Live usage is billed to this server's Google AI project +on the whole context of the session, and names what reaches it: voice and video +for a coding interview with candidate video enabled, voice for one without, and +voice and board images at a whiteboard, where the camera is off but every board +stays in the context as a realtime image (see +[What a whiteboard interview adds](#what-a-whiteboard-interview-adds)). A coding +interview with example drawing checked also names the drawing, sent whenever +Jim reads it: each `read_board` call sends the current example JPEG, which then +stays in the context like any other image. The +Setup page says the same beside the Google API key. `/api/session` reports +`candidateVideo` from the local agent config and `null` on the web half of a +split deployment, whose agent's settings it cannot see; the lobby shows no +notice while it is `null`, including before the answer and when the request +fails. + ## Operating it Watch `codetrial dispatch_refused ... reason=at_capacity` and diff --git a/src/dispatch.rs b/src/dispatch.rs index 75f90337..d39e3fb6 100644 --- a/src/dispatch.rs +++ b/src/dispatch.rs @@ -45,6 +45,10 @@ impl RoomDispatcher for LocalDispatcher { true } + fn candidate_video(&self) -> Option { + Some(self.config.gemini_candidate_video_enabled) + } + fn ensure_agent(&self, room_name: &str, provider: &Provider) -> Result<(), DispatchRefusal> { self.ensure_agent_with_kind(room_name, provider, false, None) } diff --git a/src/web/auth.rs b/src/web/auth.rs index 39d9e89b..46850621 100644 --- a/src/web/auth.rs +++ b/src/web/auth.rs @@ -316,6 +316,13 @@ pub(crate) async fn session_handler( // because the duration row is live before anyone signs in. let max_duration_min = super::token::duration_ceiling(state.config.recording.as_ref().map(|it| it.max_minutes)); + + // Null on the web half of a split deployment, where the lobby keeps its + // billing notice hidden rather than describe an agent it cannot see. + let candidate_video = state + .dispatcher + .as_ref() + .and_then(|dispatcher| dispatcher.candidate_video()); match current_user(state.accounts.as_ref(), request.headers()).await { Ok(Some(user)) => json_response( StatusCode::OK, @@ -325,6 +332,7 @@ pub(crate) async fn session_handler( "loginRequired": login_required, "guestModeAvailable": state.guest_mode_available(), "maxDurationMin": max_duration_min, + "candidateVideo": candidate_video, "user": { "login": user.login, "avatarUrl": user.avatar_url @@ -338,6 +346,7 @@ pub(crate) async fn session_handler( "loginRequired": login_required, "guestModeAvailable": state.guest_mode_available(), "maxDurationMin": max_duration_min, + "candidateVideo": candidate_video, }), ), Err(_) => json_response( diff --git a/src/web/mod.rs b/src/web/mod.rs index 46692526..827423ba 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -619,6 +619,17 @@ pub trait RoomDispatcher: Send + Sync + 'static { false } + /// Whether the interviewer this dispatcher starts sends candidate video to + /// Gemini, so the lobby can say what a coding interview is billed on. + /// + /// `None` unless the dispatcher runs the interviewer from a config it + /// holds. An agent that joins from another host reads its own environment, + /// and the web side guessing at it would put a promise in front of the + /// candidate that nothing here can keep. + fn candidate_video(&self) -> Option { + None + } + fn ensure_guest_agent( &self, _room_name: &str, diff --git a/src/web/setup.rs b/src/web/setup.rs index 58723b0d..1b3ef15b 100644 --- a/src/web/setup.rs +++ b/src/web/setup.rs @@ -293,6 +293,7 @@ const SETUP_PAGE: &str = r#"
Google AI Optional

Add a key if this server will host the interviewer.

+

Gemini Live usage is billed to this server's Google AI project, on the whole context of the session: retained audio and images cost again on every later turn.

diff --git a/tests/browser/lobby.test.js b/tests/browser/lobby.test.js index 89f7097e..0f3beb0a 100644 --- a/tests/browser/lobby.test.js +++ b/tests/browser/lobby.test.js @@ -7772,3 +7772,198 @@ lobbyTest( } }, ); + +/// Loads the lobby and waits until `/api/session` has been applied. The +/// status starts as "Checking account..." in the markup, so only the text the +/// answer writes says the notice has been decided; any non-empty text would +/// pass before the response arrived. +async function lobbyAfterSession(page, status) { + const answered = page.waitForResponse( + (response) => new URL(response.url()).pathname === "/api/session", + ); + await page.goto(`${base}/`, { waitUntil: "domcontentloaded" }); + await answered; + await page.waitForFunction( + (expected) => + document.querySelector("#account-status").textContent === expected, + status, + ); +} + +const BILLED_TO_SERVER = + "Gemini Live usage is billed to this server's Google AI project, on the whole context of the session."; + +lobbyTest( + "the billing notice names voice and video for a coding interview with candidate video", + async (page) => { + session = { + signedIn: true, + user: { login: "candidate" }, + candidateVideo: true, + }; + await lobbyAfterSession(page, "Signed in as candidate"); + const note = page.locator("#cost-note"); + assert.equal(await note.isVisible(), true); + assert.equal( + await note.textContent(), + `This interview sends your voice and video to Gemini Live. ${BILLED_TO_SERVER}`, + ); + assert.equal( + await page.getAttribute("#start", "aria-describedby"), + "cost-note", + ); + }, +); + +lobbyTest( + "the billing notice names voice alone for a coding interview without candidate video", + async (page) => { + session = { signedIn: false, candidateVideo: false }; + await lobbyAfterSession(page, "Signed out"); + const note = page.locator("#cost-note"); + assert.equal(await note.isVisible(), true); + assert.equal( + await note.textContent(), + `This interview sends your voice to Gemini Live. ${BILLED_TO_SERVER}`, + ); + }, +); + +for (const candidateVideo of [true, false]) { + lobbyTest( + `the billing notice names board images at a whiteboard (candidate video ${candidateVideo})`, + async (page) => { + session = { + signedIn: true, + user: { login: "candidate" }, + candidateVideo, + }; + await lobbyAfterSession(page, "Signed in as candidate"); + const note = page.locator("#cost-note"); + const coding = `This interview sends ${ + candidateVideo ? "your voice and video" : "your voice" + } to Gemini Live. ${BILLED_TO_SERVER}`; + assert.equal(await note.textContent(), coding); + + await page.locator('[data-mode="whiteboard"]').click(); + assert.equal(await note.isVisible(), true); + assert.equal( + await note.textContent(), + `This interview sends your voice and board images to Gemini Live. ${BILLED_TO_SERVER}`, + ); + + await page.locator('[data-mode="coding"]').click(); + assert.equal(await note.textContent(), coding); + }, + ); +} + +// A web-only server answers null, an older one leaves the field out, and a +// failed request means offline practice: none of them can say who is billed. +for (const [name, setup, status] of [ + [ + "is null", + () => (session = { signedIn: false, candidateVideo: null }), + "Signed out", + ], + [ + "is missing", + () => (session = { signedIn: true, user: { login: "candidate" } }), + "Signed in as candidate", + ], + ["request fails", () => failing.add("/api/session"), "Signed out"], +]) { + lobbyTest( + `the billing notice stays hidden when candidateVideo ${name}`, + async (page) => { + setup(); + await lobbyAfterSession(page, status); + assert.equal(await page.locator("#cost-note").isVisible(), false); + await page.locator('[data-mode="whiteboard"]').click(); + assert.equal(await page.locator("#cost-note").isVisible(), false); + }, + ); +} + +lobbyTest( + "a failed account refresh takes down the billing notice", + async (page) => { + session = { + signedIn: true, + user: { login: "candidate" }, + candidateVideo: true, + }; + await lobbyAfterSession(page, "Signed in as candidate"); + assert.equal(await page.locator("#cost-note").isVisible(), true); + + failing.add("/api/session"); + const answered = page.waitForResponse( + (response) => new URL(response.url()).pathname === "/api/session", + ); + await restore(page); + await answered; + await page.waitForFunction( + () => + document.querySelector("#account-status").textContent === "Signed out", + ); + assert.equal(await page.locator("#cost-note").isVisible(), false); + }, +); + +for (const candidateVideo of [true, false]) { + lobbyTest( + `the billing notice names the coding drawing while example drawing is on (candidate video ${candidateVideo})`, + async (page) => { + session = { + signedIn: true, + user: { login: "candidate" }, + candidateVideo, + }; + await lobbyAfterSession(page, "Signed in as candidate"); + const note = page.locator("#cost-note"); + const voice = candidateVideo ? "your voice and video" : "your voice"; + const plain = `This interview sends ${voice} to Gemini Live. ${BILLED_TO_SERVER}`; + const drawing = `This interview sends ${voice}, and your drawing whenever Jim reads it, to Gemini Live. ${BILLED_TO_SERVER}`; + assert.equal(await note.textContent(), plain); + + await page.locator("#enable-example-drawing").check(); + assert.equal(await note.textContent(), drawing); + + // The whiteboard's own wording wins while the checkbox is disabled, and + // the drawing comes back with coding because the box stayed checked. + await page.locator('[data-mode="whiteboard"]').click(); + assert.equal( + await note.textContent(), + `This interview sends your voice and board images to Gemini Live. ${BILLED_TO_SERVER}`, + ); + await page.locator('[data-mode="coding"]').click(); + assert.equal(await note.textContent(), drawing); + + await page.locator("#enable-example-drawing").uncheck(); + assert.equal(await note.textContent(), plain); + }, + ); +} + +lobbyTest( + "a restored example drawing setting is named in the billing notice", + async (page) => { + session = { + signedIn: true, + user: { login: "candidate" }, + candidateVideo: false, + }; + await page.addInitScript(() => + localStorage.setItem( + "codetrial.interviewSettings", + JSON.stringify({ exampleDrawing: true }), + ), + ); + await lobbyAfterSession(page, "Signed in as candidate"); + assert.equal(await page.isChecked("#enable-example-drawing"), true); + assert.equal( + await page.locator("#cost-note").textContent(), + `This interview sends your voice, and your drawing whenever Jim reads it, to Gemini Live. ${BILLED_TO_SERVER}`, + ); + }, +); diff --git a/tests/browser/setup-page.test.js b/tests/browser/setup-page.test.js index abbd0b26..f9408e51 100644 --- a/tests/browser/setup-page.test.js +++ b/tests/browser/setup-page.test.js @@ -103,6 +103,11 @@ test("setup shows distinct credential groups without narrow-screen overflow", as await page.locator("fieldset").last().innerText(), /Optional/i, ); + // Beside the key, so whoever enters it learns whose project pays. + assert.match( + await page.locator("fieldset").last().innerText(), + /Gemini Live usage is billed to this server's Google AI project, on the whole context of the session/, + ); assert.equal(await page.locator("input:required").count(), 3); assert.equal(await page.locator("#googleApiKey:required").count(), 0); assert.equal( diff --git a/tests/unit/dispatch.rs b/tests/unit/dispatch.rs index e2cc4e9e..57d9dce5 100644 --- a/tests/unit/dispatch.rs +++ b/tests/unit/dispatch.rs @@ -600,3 +600,25 @@ async fn local_dispatcher_enforces_guest_admission_capacity() { Err(DispatchRefusal::AtCapacity), ); } + +/// The lobby's billing notice names video only on this answer, so it has to +/// be the setting the local interviewer will run with, both ways. +#[tokio::test] +async fn candidate_video_reports_the_local_agent_setting() { + for (value, expected) in [("true", true), ("false", false)] { + let dispatcher = LocalDispatcher { + config: crate::config::load_from_pairs([ + ("LIVEKIT_URL", "wss://primary.example"), + ("LIVEKIT_API_KEY", "primary-key"), + ("LIVEKIT_API_SECRET", "primary-secret"), + ("GOOGLE_API_KEY", "primary-google"), + ("CODETRIAL_GEMINI_CANDIDATE_VIDEO_ENABLED", value), + ]) + .unwrap(), + runtime: tokio::runtime::Handle::current(), + state: Arc::default(), + max_concurrent: 1, + }; + assert_eq!(dispatcher.candidate_video(), Some(expected), "{value}"); + } +} diff --git a/tests/web/accounts.rs b/tests/web/accounts.rs index b08d04e4..a9197b20 100644 --- a/tests/web/accounts.rs +++ b/tests/web/accounts.rs @@ -129,7 +129,8 @@ async fn account_routes_record_interviewee_github_login() { assert_eq!( session.json::().await.unwrap(), json!({"signedIn": false, "loginRequired": true, - "guestModeAvailable": true, "maxDurationMin": MAX_DURATION_MIN}) + "guestModeAvailable": true, "maxDurationMin": MAX_DURATION_MIN, + "candidateVideo": null}) ); let signed_in = client @@ -168,6 +169,73 @@ async fn account_routes_record_interviewee_github_login() { remove_database(path).await; } +/// A dispatcher that answers `candidate_video` with whatever it is built with. +struct CandidateVideoDispatcher(Option); + +impl RoomDispatcher for CandidateVideoDispatcher { + fn ensure_agent( + &self, + _room_name: &str, + _provider: &codetrial::config::Provider, + ) -> Result<(), DispatchRefusal> { + Ok(()) + } + + fn candidate_video(&self) -> Option { + self.0 + } +} + +/// Both branches of `/api/session` carry the dispatcher's answer, each value of +/// it, and null with no dispatcher at all. Every other assertion here runs +/// against a dispatcher that does not know, so a handler that hardcoded the +/// field, or a signed-in branch that dropped it, would pass them. +#[tokio::test] +async fn session_reports_candidate_video_signed_out_and_signed_in() { + for (label, answer, expected) in [ + ("video-on", Some(Some(true)), json!(true)), + ("video-off", Some(Some(false)), json!(false)), + ("video-unknown", Some(None), Value::Null), + ("video-no-dispatcher", None, Value::Null), + ] { + let path = account_db_path(label); + initialize_account_database(&path).unwrap(); + let mut config = web_config(); + config.session_secret = Some("session-secret".to_string()); + config.db_path = Some(path.clone()); + let (base, server) = match answer { + Some(answer) => { + spawn_web_server_with_dispatcher( + config, + std::sync::Arc::new(CandidateVideoDispatcher(answer)), + ) + .await + } + None => spawn_web_server(config).await, + }; + let client = http_client(); + let session = |cookie: Option| { + let mut request = client.get(format!("{base}/api/session")); + if let Some(cookie) = cookie { + request = request.header("cookie", cookie); + } + async move { request.send().await.unwrap().json::().await.unwrap() } + }; + + let signed_out = session(None).await; + assert_eq!(signed_out["signedIn"], false, "{label}"); + assert_eq!(signed_out.get("candidateVideo"), Some(&expected), "{label}"); + + let cookie = record_login(&client, &base, "octo-cat").await; + let signed_in = session(Some(cookie)).await; + assert_eq!(signed_in["signedIn"], true, "{label}"); + assert_eq!(signed_in.get("candidateVideo"), Some(&expected), "{label}"); + + server.shutdown().await; + remove_database(path).await; + } +} + /// A typed handle is not proof of anything, so it must not be the key an /// account is found by. If it were, typing a name someone else used would hand /// over their interview history, and nothing stops anyone typing any name. @@ -251,7 +319,8 @@ async fn an_unmigratable_account_database_refuses_rather_than_disabling_login() assert_eq!( session, json!({"signedIn": false, "loginRequired": true, - "guestModeAvailable": false, "maxDurationMin": MAX_DURATION_MIN}), + "guestModeAvailable": false, "maxDurationMin": MAX_DURATION_MIN, + "candidateVideo": null}), "a database this binary cannot migrate still requires a login" ); @@ -292,7 +361,8 @@ async fn an_unopenable_account_database_refuses_rather_than_disabling_login() { assert_eq!( session, json!({"signedIn": false, "loginRequired": true, - "guestModeAvailable": false, "maxDurationMin": MAX_DURATION_MIN}), + "guestModeAvailable": false, "maxDurationMin": MAX_DURATION_MIN, + "candidateVideo": null}), "a broken database still requires a login; it cannot serve one" ); @@ -348,7 +418,8 @@ async fn an_unopenable_account_database_refuses_rather_than_disabling_login() { "signedIn": false, "loginRequired": false, "guestModeAvailable": true, - "maxDurationMin": MAX_DURATION_MIN + "maxDurationMin": MAX_DURATION_MIN, + "candidateVideo": null }), "nothing to sign in to, and the lobby has to be told so" ); @@ -867,7 +938,8 @@ async fn account_reports_are_scoped_to_the_signed_in_user() { assert_eq!( signed_out, json!({"signedIn": false, "loginRequired": true, - "guestModeAvailable": true, "maxDurationMin": MAX_DURATION_MIN}), + "guestModeAvailable": true, "maxDurationMin": MAX_DURATION_MIN, + "candidateVideo": null}), "accounts exist here, so the browser has to know to demand a sign-in" ); diff --git a/web/app.js b/web/app.js index 744bfa1b..045bbb36 100644 --- a/web/app.js +++ b/web/app.js @@ -101,6 +101,12 @@ let deletingReports = false; /// file is still in its temporal dead zone, and reading one there throws /// before the lobby has rendered anything at all. let durationCeiling = Infinity; +/// Whether the interviewer sends candidate video, from `/api/session`. Null +/// until it answers, when it fails, and when this server does not host the +/// interviewer itself; the billing notice above Start stays hidden while null. +/// Here for the same temporal dead zone: `setInterviewMode` reads it during +/// setup. +let candidateVideo = null; // The roll a recommendation is drawn with. It changes when the candidate asks // for something different and at no other time, so recommending twice for one @@ -151,6 +157,7 @@ const nodes = { groundingClear: document.querySelector("#grounding-clear"), groundingError: document.querySelector("#grounding-error"), durationNote: document.querySelector("#duration-note"), + costNote: document.querySelector("#cost-note"), problemSearch: document.querySelector("#problem-search"), problemSearchModes: document.querySelectorAll('[name="problem-search-mode"]'), problemTopic: document.querySelector("#problem-topic"), @@ -531,7 +538,10 @@ for (const button of document.querySelectorAll("[data-mode]")) { }); } -nodes.exampleDrawing.addEventListener("change", rememberSettings); +nodes.exampleDrawing.addEventListener("change", () => { + rememberSettings(); + applyCostNote(); +}); const start = document.querySelector("#start"); @@ -916,6 +926,11 @@ async function loadAccount(generation) { } accountHistory = false; applyDurationCeiling(session.maxDurationMin); + candidateVideo = + typeof session.candidateVideo === "boolean" + ? session.candidateVideo + : null; + applyCostNote(); if (session.signedIn) { loginResult = null; accountHistory = true; @@ -951,6 +966,11 @@ async function loadAccount(generation) { } catch { // A server that cannot answer about accounts is not one that will mint a // token either, but the editor still works offline, so do not lock the page. + // Nor will it bill one, so a notice from an earlier answer comes down. + if (generation === historyLoad) { + candidateVideo = null; + applyCostNote(); + } } if (generation !== historyLoad) return; if (loginResult) { @@ -1460,6 +1480,27 @@ function setInterviewMode(value) { ? "Whiteboard: no editor and no test runner. You explain by drawing, and Jim sees the board as you go." : ""; note.hidden = mode !== "whiteboard"; + applyCostNote(); +} + +/// Names who pays and what reaches Gemini Live in the chosen mode. A whiteboard +/// interview sends no camera, but every board it publishes stays in the +/// billed context as a realtime image, so it is not voice alone. A coding +/// interview with example drawing sends a board only when `read_board` asks for +/// it, which is why the drawing is named as conditional there. +function applyCostNote() { + const voice = candidateVideo ? "your voice and video" : "your voice"; + const media = + mode === "whiteboard" + ? "your voice and board images" + : nodes.exampleDrawing.checked + ? `${voice}, and your drawing whenever Jim reads it,` + : voice; + nodes.costNote.hidden = candidateVideo === null; + nodes.costNote.textContent = + candidateVideo === null + ? "" + : `This interview sends ${media} to Gemini Live. Gemini Live usage is billed to this server's Google AI project, on the whole context of the session.`; } function rememberSettings() { @@ -1485,6 +1526,7 @@ function restoreSettings() { } if (!settings || typeof settings !== "object") return; nodes.exampleDrawing.checked = settings.exampleDrawing === true; + applyCostNote(); if ( durations.some( (button) => Number(button.dataset.duration) === settings.durationMin, diff --git a/web/index.html b/web/index.html index 3cb718e3..edd8735f 100644 --- a/web/index.html +++ b/web/index.html @@ -1949,7 +1949,14 @@

  • A live interview with Jim, by voice
  • A scored report for every phase
    • - diff --git a/web/styles.css b/web/styles.css index 74652adc..8b1460ed 100644 --- a/web/styles.css +++ b/web/styles.css @@ -3714,6 +3714,13 @@ body:not([data-recording]) .replay-detail > :not(header) { line-height: 1.4; } +.cost-note { + margin: -0.5rem 0 1.25rem; + color: var(--sub); + font-size: 0.875rem; + line-height: 1.4; +} + /* The dot is painted, not a marker or generated content: the stylesheet draws no text, and tests/browser/replay-render.test.js holds it to that. */ .next-steps li {