diff --git a/docs/provider-cost-and-degradation.md b/docs/provider-cost-and-degradation.md index 3dd8dbcd..f5c5829a 100644 --- a/docs/provider-cost-and-degradation.md +++ b/docs/provider-cost-and-degradation.md @@ -325,6 +325,22 @@ says that no scores or verdict were created. Browser fallback may summarize local test progress only for a session that never reached an interviewer, and never presents canned feedback as an agent evaluation. +Before an interview on a server that hosts its own interviewer, the lobby says +above Start that Gemini Live usage is billed to this server's Google AI project +on the whole context of the session, and names what reaches it: voice and video +for a coding interview with candidate video enabled, voice for one without, and +voice and board images at a whiteboard, where the camera is off but every board +stays in the context as a realtime image (see +[What a whiteboard interview adds](#what-a-whiteboard-interview-adds)). A coding +interview with example drawing checked also names the drawing, sent whenever +Jim reads it: each `read_board` call sends the current example JPEG, which then +stays in the context like any other image. The +Setup page says the same beside the Google API key. `/api/session` reports +`candidateVideo` from the local agent config and `null` on the web half of a +split deployment, whose agent's settings it cannot see; the lobby shows no +notice while it is `null`, including before the answer and when the request +fails. + ## Operating it Watch `codetrial dispatch_refused ... reason=at_capacity` and diff --git a/src/dispatch.rs b/src/dispatch.rs index 75f90337..d39e3fb6 100644 --- a/src/dispatch.rs +++ b/src/dispatch.rs @@ -45,6 +45,10 @@ impl RoomDispatcher for LocalDispatcher { true } + fn candidate_video(&self) -> Option { + Some(self.config.gemini_candidate_video_enabled) + } + fn ensure_agent(&self, room_name: &str, provider: &Provider) -> Result<(), DispatchRefusal> { self.ensure_agent_with_kind(room_name, provider, false, None) } diff --git a/src/web/auth.rs b/src/web/auth.rs index 39d9e89b..46850621 100644 --- a/src/web/auth.rs +++ b/src/web/auth.rs @@ -316,6 +316,13 @@ pub(crate) async fn session_handler( // because the duration row is live before anyone signs in. let max_duration_min = super::token::duration_ceiling(state.config.recording.as_ref().map(|it| it.max_minutes)); + + // Null on the web half of a split deployment, where the lobby keeps its + // billing notice hidden rather than describe an agent it cannot see. + let candidate_video = state + .dispatcher + .as_ref() + .and_then(|dispatcher| dispatcher.candidate_video()); match current_user(state.accounts.as_ref(), request.headers()).await { Ok(Some(user)) => json_response( StatusCode::OK, @@ -325,6 +332,7 @@ pub(crate) async fn session_handler( "loginRequired": login_required, "guestModeAvailable": state.guest_mode_available(), "maxDurationMin": max_duration_min, + "candidateVideo": candidate_video, "user": { "login": user.login, "avatarUrl": user.avatar_url @@ -338,6 +346,7 @@ pub(crate) async fn session_handler( "loginRequired": login_required, "guestModeAvailable": state.guest_mode_available(), "maxDurationMin": max_duration_min, + "candidateVideo": candidate_video, }), ), Err(_) => json_response( diff --git a/src/web/mod.rs b/src/web/mod.rs index 46692526..827423ba 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -619,6 +619,17 @@ pub trait RoomDispatcher: Send + Sync + 'static { false } + /// Whether the interviewer this dispatcher starts sends candidate video to + /// Gemini, so the lobby can say what a coding interview is billed on. + /// + /// `None` unless the dispatcher runs the interviewer from a config it + /// holds. An agent that joins from another host reads its own environment, + /// and the web side guessing at it would put a promise in front of the + /// candidate that nothing here can keep. + fn candidate_video(&self) -> Option { + None + } + fn ensure_guest_agent( &self, _room_name: &str, diff --git a/src/web/setup.rs b/src/web/setup.rs index 58723b0d..1b3ef15b 100644 --- a/src/web/setup.rs +++ b/src/web/setup.rs @@ -293,6 +293,7 @@ const SETUP_PAGE: &str = r#"
Google AI Optional

Add a key if this server will host the interviewer.

+

Gemini Live usage is billed to this server's Google AI project, on the whole context of the session: retained audio and images cost again on every later turn.

diff --git a/tests/browser/lobby.test.js b/tests/browser/lobby.test.js index 89f7097e..0f3beb0a 100644 --- a/tests/browser/lobby.test.js +++ b/tests/browser/lobby.test.js @@ -7772,3 +7772,198 @@ lobbyTest( } }, ); + +/// Loads the lobby and waits until `/api/session` has been applied. The +/// status starts as "Checking account..." in the markup, so only the text the +/// answer writes says the notice has been decided; any non-empty text would +/// pass before the response arrived. +async function lobbyAfterSession(page, status) { + const answered = page.waitForResponse( + (response) => new URL(response.url()).pathname === "/api/session", + ); + await page.goto(`${base}/`, { waitUntil: "domcontentloaded" }); + await answered; + await page.waitForFunction( + (expected) => + document.querySelector("#account-status").textContent === expected, + status, + ); +} + +const BILLED_TO_SERVER = + "Gemini Live usage is billed to this server's Google AI project, on the whole context of the session."; + +lobbyTest( + "the billing notice names voice and video for a coding interview with candidate video", + async (page) => { + session = { + signedIn: true, + user: { login: "candidate" }, + candidateVideo: true, + }; + await lobbyAfterSession(page, "Signed in as candidate"); + const note = page.locator("#cost-note"); + assert.equal(await note.isVisible(), true); + assert.equal( + await note.textContent(), + `This interview sends your voice and video to Gemini Live. ${BILLED_TO_SERVER}`, + ); + assert.equal( + await page.getAttribute("#start", "aria-describedby"), + "cost-note", + ); + }, +); + +lobbyTest( + "the billing notice names voice alone for a coding interview without candidate video", + async (page) => { + session = { signedIn: false, candidateVideo: false }; + await lobbyAfterSession(page, "Signed out"); + const note = page.locator("#cost-note"); + assert.equal(await note.isVisible(), true); + assert.equal( + await note.textContent(), + `This interview sends your voice to Gemini Live. ${BILLED_TO_SERVER}`, + ); + }, +); + +for (const candidateVideo of [true, false]) { + lobbyTest( + `the billing notice names board images at a whiteboard (candidate video ${candidateVideo})`, + async (page) => { + session = { + signedIn: true, + user: { login: "candidate" }, + candidateVideo, + }; + await lobbyAfterSession(page, "Signed in as candidate"); + const note = page.locator("#cost-note"); + const coding = `This interview sends ${ + candidateVideo ? "your voice and video" : "your voice" + } to Gemini Live. ${BILLED_TO_SERVER}`; + assert.equal(await note.textContent(), coding); + + await page.locator('[data-mode="whiteboard"]').click(); + assert.equal(await note.isVisible(), true); + assert.equal( + await note.textContent(), + `This interview sends your voice and board images to Gemini Live. ${BILLED_TO_SERVER}`, + ); + + await page.locator('[data-mode="coding"]').click(); + assert.equal(await note.textContent(), coding); + }, + ); +} + +// A web-only server answers null, an older one leaves the field out, and a +// failed request means offline practice: none of them can say who is billed. +for (const [name, setup, status] of [ + [ + "is null", + () => (session = { signedIn: false, candidateVideo: null }), + "Signed out", + ], + [ + "is missing", + () => (session = { signedIn: true, user: { login: "candidate" } }), + "Signed in as candidate", + ], + ["request fails", () => failing.add("/api/session"), "Signed out"], +]) { + lobbyTest( + `the billing notice stays hidden when candidateVideo ${name}`, + async (page) => { + setup(); + await lobbyAfterSession(page, status); + assert.equal(await page.locator("#cost-note").isVisible(), false); + await page.locator('[data-mode="whiteboard"]').click(); + assert.equal(await page.locator("#cost-note").isVisible(), false); + }, + ); +} + +lobbyTest( + "a failed account refresh takes down the billing notice", + async (page) => { + session = { + signedIn: true, + user: { login: "candidate" }, + candidateVideo: true, + }; + await lobbyAfterSession(page, "Signed in as candidate"); + assert.equal(await page.locator("#cost-note").isVisible(), true); + + failing.add("/api/session"); + const answered = page.waitForResponse( + (response) => new URL(response.url()).pathname === "/api/session", + ); + await restore(page); + await answered; + await page.waitForFunction( + () => + document.querySelector("#account-status").textContent === "Signed out", + ); + assert.equal(await page.locator("#cost-note").isVisible(), false); + }, +); + +for (const candidateVideo of [true, false]) { + lobbyTest( + `the billing notice names the coding drawing while example drawing is on (candidate video ${candidateVideo})`, + async (page) => { + session = { + signedIn: true, + user: { login: "candidate" }, + candidateVideo, + }; + await lobbyAfterSession(page, "Signed in as candidate"); + const note = page.locator("#cost-note"); + const voice = candidateVideo ? "your voice and video" : "your voice"; + const plain = `This interview sends ${voice} to Gemini Live. ${BILLED_TO_SERVER}`; + const drawing = `This interview sends ${voice}, and your drawing whenever Jim reads it, to Gemini Live. ${BILLED_TO_SERVER}`; + assert.equal(await note.textContent(), plain); + + await page.locator("#enable-example-drawing").check(); + assert.equal(await note.textContent(), drawing); + + // The whiteboard's own wording wins while the checkbox is disabled, and + // the drawing comes back with coding because the box stayed checked. + await page.locator('[data-mode="whiteboard"]').click(); + assert.equal( + await note.textContent(), + `This interview sends your voice and board images to Gemini Live. ${BILLED_TO_SERVER}`, + ); + await page.locator('[data-mode="coding"]').click(); + assert.equal(await note.textContent(), drawing); + + await page.locator("#enable-example-drawing").uncheck(); + assert.equal(await note.textContent(), plain); + }, + ); +} + +lobbyTest( + "a restored example drawing setting is named in the billing notice", + async (page) => { + session = { + signedIn: true, + user: { login: "candidate" }, + candidateVideo: false, + }; + await page.addInitScript(() => + localStorage.setItem( + "codetrial.interviewSettings", + JSON.stringify({ exampleDrawing: true }), + ), + ); + await lobbyAfterSession(page, "Signed in as candidate"); + assert.equal(await page.isChecked("#enable-example-drawing"), true); + assert.equal( + await page.locator("#cost-note").textContent(), + `This interview sends your voice, and your drawing whenever Jim reads it, to Gemini Live. ${BILLED_TO_SERVER}`, + ); + }, +); diff --git a/tests/browser/setup-page.test.js b/tests/browser/setup-page.test.js index abbd0b26..f9408e51 100644 --- a/tests/browser/setup-page.test.js +++ b/tests/browser/setup-page.test.js @@ -103,6 +103,11 @@ test("setup shows distinct credential groups without narrow-screen overflow", as await page.locator("fieldset").last().innerText(), /Optional/i, ); + // Beside the key, so whoever enters it learns whose project pays. + assert.match( + await page.locator("fieldset").last().innerText(), + /Gemini Live usage is billed to this server's Google AI project, on the whole context of the session/, + ); assert.equal(await page.locator("input:required").count(), 3); assert.equal(await page.locator("#googleApiKey:required").count(), 0); assert.equal( diff --git a/tests/unit/dispatch.rs b/tests/unit/dispatch.rs index e2cc4e9e..57d9dce5 100644 --- a/tests/unit/dispatch.rs +++ b/tests/unit/dispatch.rs @@ -600,3 +600,25 @@ async fn local_dispatcher_enforces_guest_admission_capacity() { Err(DispatchRefusal::AtCapacity), ); } + +/// The lobby's billing notice names video only on this answer, so it has to +/// be the setting the local interviewer will run with, both ways. +#[tokio::test] +async fn candidate_video_reports_the_local_agent_setting() { + for (value, expected) in [("true", true), ("false", false)] { + let dispatcher = LocalDispatcher { + config: crate::config::load_from_pairs([ + ("LIVEKIT_URL", "wss://primary.example"), + ("LIVEKIT_API_KEY", "primary-key"), + ("LIVEKIT_API_SECRET", "primary-secret"), + ("GOOGLE_API_KEY", "primary-google"), + ("CODETRIAL_GEMINI_CANDIDATE_VIDEO_ENABLED", value), + ]) + .unwrap(), + runtime: tokio::runtime::Handle::current(), + state: Arc::default(), + max_concurrent: 1, + }; + assert_eq!(dispatcher.candidate_video(), Some(expected), "{value}"); + } +} diff --git a/tests/web/accounts.rs b/tests/web/accounts.rs index b08d04e4..a9197b20 100644 --- a/tests/web/accounts.rs +++ b/tests/web/accounts.rs @@ -129,7 +129,8 @@ async fn account_routes_record_interviewee_github_login() { assert_eq!( session.json::().await.unwrap(), json!({"signedIn": false, "loginRequired": true, - "guestModeAvailable": true, "maxDurationMin": MAX_DURATION_MIN}) + "guestModeAvailable": true, "maxDurationMin": MAX_DURATION_MIN, + "candidateVideo": null}) ); let signed_in = client @@ -168,6 +169,73 @@ async fn account_routes_record_interviewee_github_login() { remove_database(path).await; } +/// A dispatcher that answers `candidate_video` with whatever it is built with. +struct CandidateVideoDispatcher(Option); + +impl RoomDispatcher for CandidateVideoDispatcher { + fn ensure_agent( + &self, + _room_name: &str, + _provider: &codetrial::config::Provider, + ) -> Result<(), DispatchRefusal> { + Ok(()) + } + + fn candidate_video(&self) -> Option { + self.0 + } +} + +/// Both branches of `/api/session` carry the dispatcher's answer, each value of +/// it, and null with no dispatcher at all. Every other assertion here runs +/// against a dispatcher that does not know, so a handler that hardcoded the +/// field, or a signed-in branch that dropped it, would pass them. +#[tokio::test] +async fn session_reports_candidate_video_signed_out_and_signed_in() { + for (label, answer, expected) in [ + ("video-on", Some(Some(true)), json!(true)), + ("video-off", Some(Some(false)), json!(false)), + ("video-unknown", Some(None), Value::Null), + ("video-no-dispatcher", None, Value::Null), + ] { + let path = account_db_path(label); + initialize_account_database(&path).unwrap(); + let mut config = web_config(); + config.session_secret = Some("session-secret".to_string()); + config.db_path = Some(path.clone()); + let (base, server) = match answer { + Some(answer) => { + spawn_web_server_with_dispatcher( + config, + std::sync::Arc::new(CandidateVideoDispatcher(answer)), + ) + .await + } + None => spawn_web_server(config).await, + }; + let client = http_client(); + let session = |cookie: Option| { + let mut request = client.get(format!("{base}/api/session")); + if let Some(cookie) = cookie { + request = request.header("cookie", cookie); + } + async move { request.send().await.unwrap().json::().await.unwrap() } + }; + + let signed_out = session(None).await; + assert_eq!(signed_out["signedIn"], false, "{label}"); + assert_eq!(signed_out.get("candidateVideo"), Some(&expected), "{label}"); + + let cookie = record_login(&client, &base, "octo-cat").await; + let signed_in = session(Some(cookie)).await; + assert_eq!(signed_in["signedIn"], true, "{label}"); + assert_eq!(signed_in.get("candidateVideo"), Some(&expected), "{label}"); + + server.shutdown().await; + remove_database(path).await; + } +} + /// A typed handle is not proof of anything, so it must not be the key an /// account is found by. If it were, typing a name someone else used would hand /// over their interview history, and nothing stops anyone typing any name. @@ -251,7 +319,8 @@ async fn an_unmigratable_account_database_refuses_rather_than_disabling_login() assert_eq!( session, json!({"signedIn": false, "loginRequired": true, - "guestModeAvailable": false, "maxDurationMin": MAX_DURATION_MIN}), + "guestModeAvailable": false, "maxDurationMin": MAX_DURATION_MIN, + "candidateVideo": null}), "a database this binary cannot migrate still requires a login" ); @@ -292,7 +361,8 @@ async fn an_unopenable_account_database_refuses_rather_than_disabling_login() { assert_eq!( session, json!({"signedIn": false, "loginRequired": true, - "guestModeAvailable": false, "maxDurationMin": MAX_DURATION_MIN}), + "guestModeAvailable": false, "maxDurationMin": MAX_DURATION_MIN, + "candidateVideo": null}), "a broken database still requires a login; it cannot serve one" ); @@ -348,7 +418,8 @@ async fn an_unopenable_account_database_refuses_rather_than_disabling_login() { "signedIn": false, "loginRequired": false, "guestModeAvailable": true, - "maxDurationMin": MAX_DURATION_MIN + "maxDurationMin": MAX_DURATION_MIN, + "candidateVideo": null }), "nothing to sign in to, and the lobby has to be told so" ); @@ -867,7 +938,8 @@ async fn account_reports_are_scoped_to_the_signed_in_user() { assert_eq!( signed_out, json!({"signedIn": false, "loginRequired": true, - "guestModeAvailable": true, "maxDurationMin": MAX_DURATION_MIN}), + "guestModeAvailable": true, "maxDurationMin": MAX_DURATION_MIN, + "candidateVideo": null}), "accounts exist here, so the browser has to know to demand a sign-in" ); diff --git a/web/app.js b/web/app.js index 744bfa1b..045bbb36 100644 --- a/web/app.js +++ b/web/app.js @@ -101,6 +101,12 @@ let deletingReports = false; /// file is still in its temporal dead zone, and reading one there throws /// before the lobby has rendered anything at all. let durationCeiling = Infinity; +/// Whether the interviewer sends candidate video, from `/api/session`. Null +/// until it answers, when it fails, and when this server does not host the +/// interviewer itself; the billing notice above Start stays hidden while null. +/// Here for the same temporal dead zone: `setInterviewMode` reads it during +/// setup. +let candidateVideo = null; // The roll a recommendation is drawn with. It changes when the candidate asks // for something different and at no other time, so recommending twice for one @@ -151,6 +157,7 @@ const nodes = { groundingClear: document.querySelector("#grounding-clear"), groundingError: document.querySelector("#grounding-error"), durationNote: document.querySelector("#duration-note"), + costNote: document.querySelector("#cost-note"), problemSearch: document.querySelector("#problem-search"), problemSearchModes: document.querySelectorAll('[name="problem-search-mode"]'), problemTopic: document.querySelector("#problem-topic"), @@ -531,7 +538,10 @@ for (const button of document.querySelectorAll("[data-mode]")) { }); } -nodes.exampleDrawing.addEventListener("change", rememberSettings); +nodes.exampleDrawing.addEventListener("change", () => { + rememberSettings(); + applyCostNote(); +}); const start = document.querySelector("#start"); @@ -916,6 +926,11 @@ async function loadAccount(generation) { } accountHistory = false; applyDurationCeiling(session.maxDurationMin); + candidateVideo = + typeof session.candidateVideo === "boolean" + ? session.candidateVideo + : null; + applyCostNote(); if (session.signedIn) { loginResult = null; accountHistory = true; @@ -951,6 +966,11 @@ async function loadAccount(generation) { } catch { // A server that cannot answer about accounts is not one that will mint a // token either, but the editor still works offline, so do not lock the page. + // Nor will it bill one, so a notice from an earlier answer comes down. + if (generation === historyLoad) { + candidateVideo = null; + applyCostNote(); + } } if (generation !== historyLoad) return; if (loginResult) { @@ -1460,6 +1480,27 @@ function setInterviewMode(value) { ? "Whiteboard: no editor and no test runner. You explain by drawing, and Jim sees the board as you go." : ""; note.hidden = mode !== "whiteboard"; + applyCostNote(); +} + +/// Names who pays and what reaches Gemini Live in the chosen mode. A whiteboard +/// interview sends no camera, but every board it publishes stays in the +/// billed context as a realtime image, so it is not voice alone. A coding +/// interview with example drawing sends a board only when `read_board` asks for +/// it, which is why the drawing is named as conditional there. +function applyCostNote() { + const voice = candidateVideo ? "your voice and video" : "your voice"; + const media = + mode === "whiteboard" + ? "your voice and board images" + : nodes.exampleDrawing.checked + ? `${voice}, and your drawing whenever Jim reads it,` + : voice; + nodes.costNote.hidden = candidateVideo === null; + nodes.costNote.textContent = + candidateVideo === null + ? "" + : `This interview sends ${media} to Gemini Live. Gemini Live usage is billed to this server's Google AI project, on the whole context of the session.`; } function rememberSettings() { @@ -1485,6 +1526,7 @@ function restoreSettings() { } if (!settings || typeof settings !== "object") return; nodes.exampleDrawing.checked = settings.exampleDrawing === true; + applyCostNote(); if ( durations.some( (button) => Number(button.dataset.duration) === settings.durationMin, diff --git a/web/index.html b/web/index.html index 3cb718e3..edd8735f 100644 --- a/web/index.html +++ b/web/index.html @@ -1949,7 +1949,14 @@

  • A live interview with Jim, by voice
  • A scored report for every phase
    • - diff --git a/web/styles.css b/web/styles.css index 74652adc..8b1460ed 100644 --- a/web/styles.css +++ b/web/styles.css @@ -3714,6 +3714,13 @@ body:not([data-recording]) .replay-detail > :not(header) { line-height: 1.4; } +.cost-note { + margin: -0.5rem 0 1.25rem; + color: var(--sub); + font-size: 0.875rem; + line-height: 1.4; +} + /* The dot is painted, not a marker or generated content: the stylesheet draws no text, and tests/browser/replay-render.test.js holds it to that. */ .next-steps li {