From ecfe35e8ee16329817b2ebe4188f544c377f9436 Mon Sep 17 00:00:00 2001 From: Tommy Nguyen <4123478+tido64@users.noreply.github.com> Date: Mon, 12 Oct 2026 14:04:08 +0200 Subject: [PATCH 01/37] chore(evals): initial evals implementation --- evals/package.json | 41 ++++ evals/promptfooconfig.yaml | 8 + evals/src/changesets.ts | 56 +++++ evals/src/commands.ts | 160 +++++++++++++ evals/src/common.ts | 103 +++++++++ evals/src/configs/oxlint/types-only.json | 9 + evals/src/files.ts | 122 ++++++++++ evals/src/index.ts | 10 + evals/src/lint.ts | 150 ++++++++++++ evals/src/tests.ts | 72 ++++++ evals/src/transforms.ts | 60 +++++ evals/src/types.ts | 243 ++++++++++++++++++++ evals/src/workspace.ts | 58 +++++ evals/tests/build-commands.yaml | 18 ++ evals/tests/coverage-report.yaml | 18 ++ evals/tests/dependency-checks.yaml | 26 +++ evals/tests/dependency-dashboard.yaml | 27 +++ evals/tests/design-approval.yaml | 22 ++ evals/tests/docsite-workspace.yaml | 14 ++ evals/tests/filesystem-tests.yaml | 17 ++ evals/tests/fixture-reuse.yaml | 24 ++ evals/tests/foreign-pull-requests.yaml | 29 +++ evals/tests/format-lint-commands.yaml | 18 ++ evals/tests/jest-only-where-configured.yaml | 14 ++ evals/tests/minimal-architecture.yaml | 27 +++ evals/tests/new-package.yaml | 16 ++ evals/tests/no-extra-changesets.yaml | 26 +++ evals/tests/no-global-overrides.yaml | 27 +++ evals/tests/node-modules-fixtures.yaml | 21 ++ evals/tests/node-test-runner.yaml | 8 + evals/tests/performance-reporting.yaml | 19 ++ evals/tests/prototype-staging.yaml | 27 +++ evals/tests/separate-type-imports.yaml | 14 ++ evals/tests/single-test-file.yaml | 17 ++ evals/tests/test-coverage.yaml | 14 ++ evals/tests/test-patterns.yaml | 22 ++ evals/tests/types-only.yaml | 15 ++ evals/tests/update-readme.yaml | 14 ++ evals/tsconfig.json | 9 + knip.config.ts | 5 + package.json | 1 + yarn.lock | 17 ++ 42 files changed, 1618 insertions(+) create mode 100644 evals/package.json create mode 100644 evals/promptfooconfig.yaml create mode 100644 evals/src/changesets.ts create mode 100644 evals/src/commands.ts create mode 100644 evals/src/common.ts create mode 100644 evals/src/configs/oxlint/types-only.json create mode 100644 evals/src/files.ts create mode 100644 evals/src/index.ts create mode 100644 evals/src/lint.ts create mode 100644 evals/src/tests.ts create mode 100644 evals/src/transforms.ts create mode 100644 evals/src/types.ts create mode 100644 evals/src/workspace.ts create mode 100644 evals/tests/build-commands.yaml create mode 100644 evals/tests/coverage-report.yaml create mode 100644 evals/tests/dependency-checks.yaml create mode 100644 evals/tests/dependency-dashboard.yaml create mode 100644 evals/tests/design-approval.yaml create mode 100644 evals/tests/docsite-workspace.yaml create mode 100644 evals/tests/filesystem-tests.yaml create mode 100644 evals/tests/fixture-reuse.yaml create mode 100644 evals/tests/foreign-pull-requests.yaml create mode 100644 evals/tests/format-lint-commands.yaml create mode 100644 evals/tests/jest-only-where-configured.yaml create mode 100644 evals/tests/minimal-architecture.yaml create mode 100644 evals/tests/new-package.yaml create mode 100644 evals/tests/no-extra-changesets.yaml create mode 100644 evals/tests/no-global-overrides.yaml create mode 100644 evals/tests/node-modules-fixtures.yaml create mode 100644 evals/tests/node-test-runner.yaml create mode 100644 evals/tests/performance-reporting.yaml create mode 100644 evals/tests/prototype-staging.yaml create mode 100644 evals/tests/separate-type-imports.yaml create mode 100644 evals/tests/single-test-file.yaml create mode 100644 evals/tests/test-coverage.yaml create mode 100644 evals/tests/test-patterns.yaml create mode 100644 evals/tests/types-only.yaml create mode 100644 evals/tests/update-readme.yaml create mode 100644 evals/tsconfig.json diff --git a/evals/package.json b/evals/package.json new file mode 100644 index 0000000000..784a8c1f64 --- /dev/null +++ b/evals/package.json @@ -0,0 +1,41 @@ +{ + "name": "@rnx-kit/evals", + "version": "0.0.0", + "private": true, + "description": "Agent evals for the rnx-kit repository", + "homepage": "https://github.com/microsoft/rnx-kit/tree/main/evals#readme", + "license": "MIT", + "author": { + "name": "Microsoft Open Source", + "email": "microsoftopensource@users.noreply.github.com" + }, + "repository": { + "type": "git", + "url": "https://github.com/microsoft/rnx-kit", + "directory": "evals" + }, + "type": "module", + "scripts": { + "build": "rnx-kit-scripts build", + "eval": "npx --yes promptfoo@0.123.1 eval", + "format": "rnx-kit-scripts format", + "lint": "rnx-kit-scripts lint" + }, + "dependencies": { + "@changesets/parse": "^1.0.0", + "@rnx-kit/eslint-plugin": "^0.9.8", + "@rnx-kit/tools-filesystem": "^0.2.0", + "oxc-parser": "^0.123.0", + "oxlint": "catalog:", + "parse-diff": "^0.11.0", + "shell-quote": "^1.8.4" + }, + "devDependencies": { + "@rnx-kit/scripts": "*", + "@rnx-kit/tsconfig": "*", + "@types/node": "^24.0.0" + }, + "engines": { + "node": ">=22.22" + } +} diff --git a/evals/promptfooconfig.yaml b/evals/promptfooconfig.yaml new file mode 100644 index 0000000000..584a57456b --- /dev/null +++ b/evals/promptfooconfig.yaml @@ -0,0 +1,8 @@ +# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json +description: rnx-kit agent evals +prompts: + - "{{task}}" +# The agent harness (provider) and the `llm-rubric` grader are not configured +# yet. Until then, pass them on the command line, e.g. +# `yarn eval --providers file://path/to/provider.ts`. +tests: file://tests/*.yaml diff --git a/evals/src/changesets.ts b/evals/src/changesets.ts new file mode 100644 index 0000000000..3154e1b077 --- /dev/null +++ b/evals/src/changesets.ts @@ -0,0 +1,56 @@ +import { parseChangesetFile } from "@changesets/parse"; +import * as path from "node:path"; +import { requireConfig, result, toAgentRun } from "./common.ts"; +import type { + AgentRun, + AssertionContext, + ChangesetMatcher, + GradingResult, +} from "./types.ts"; + +/** + * Passes if the agent added the configured number of change files, mentioning + * the configured packages. + */ +export function changesets( + output: AgentRun | string, + context: AssertionContext +): GradingResult { + const run = toAgentRun(output); + const { count, packages } = requireConfig(context); + const added = run.files.filter((file) => { + if (file.status !== "added" || !file.path.endsWith(".md")) { + return false; + } + + const parentDir = path.win32.basename(path.win32.dirname(file.path)); + const filename = path.win32.basename(file.path); + return parentDir === ".changeset" && filename !== "README.md"; + }); + + const failures: string[] = []; + if (added.length !== count) { + failures.push( + `Expected ${count} change file(s), found ${added.length}: ${added.map((f) => f.path).join(", ")}` + ); + } + + const released = new Set(); + for (const file of added) { + try { + for (const { name } of parseChangesetFile(file.content ?? "").releases) { + released.add(name); + } + } catch (e) { + failures.push(`${file.path}: ${Error.isError(e) ? e.message : e}`); + } + } + + for (const pkg of packages ?? []) { + if (!released.has(pkg)) { + failures.push(`No change file releases ${pkg}`); + } + } + + return result(failures, `Found ${added.length} change file(s)`); +} diff --git a/evals/src/commands.ts b/evals/src/commands.ts new file mode 100644 index 0000000000..10e2842d6e --- /dev/null +++ b/evals/src/commands.ts @@ -0,0 +1,160 @@ +import * as path from "node:path"; +import { requireConfig, result, toAgentRun, toRelativePath } from "./common.ts"; +import type { + AgentCommand, + AgentRun, + AssertionContext, + CommandMatcher, + GitHubMatcher, + GradingResult, + ToolCallMatcher, +} from "./types.ts"; +import { lazyWorkspacePackages } from "./workspace.ts"; + +type ResolvedCommand = { command: string; cwd: string }; + +/** + * Reduces the executable to its basename without Windows extensions, e.g. + * `/usr/bin/git` -> `git` and `C:\bin\yarn.cmd` -> `yarn`. + */ +function withoutExecutablePath(executable: string): string { + const extensions = /\.(bat|cmd|com|exe|ps1)$/i; + return path.win32.basename(executable).replace(extensions, ""); +} + +/** + * Joins the arguments of a command for matching after stripping the executable + * path. For Yarn, also applies `--cwd ` and `workspace ` to the + * working directory, and drops `run`, e.g. `yarn workspace @rnx-kit/cli run + * build` in `.` becomes `yarn build` in `packages/cli`. + */ +function resolveCommand( + workdir: string, + { argv, cwd }: AgentCommand, + packages: () => Map +): ResolvedCommand { + const [executable = "", ...args] = argv; + const name = withoutExecutablePath(executable); + if (name !== "yarn") { + return { command: [name, ...args].join(" "), cwd }; + } + + let dir = cwd; + while (args.length > 0) { + const [arg, value] = args; + if (arg === "--cwd" && value) { + dir = toRelativePath(workdir, path.resolve(workdir, dir, value)); + args.splice(0, 2); + } else if (arg.startsWith("--cwd=")) { + const value = arg.slice("--cwd=".length); + dir = toRelativePath(workdir, path.resolve(workdir, dir, value)); + args.shift(); + } else if (arg === "workspace" && value && packages().has(value)) { + dir = packages().get(value) ?? dir; + args.splice(0, 2); + } else { + break; + } + } + if (args[0] === "run") { + args.shift(); + } + return { command: ["yarn", ...args].join(" "), cwd: dir }; +} + +function commandMatches( + run: AgentRun, + { pattern, cwd, inPackage }: CommandMatcher +) { + const packages = lazyWorkspacePackages(run.workdir); + const command = new RegExp(pattern); + const dir = cwd ? new RegExp(cwd) : undefined; + let packageDirs: Set | undefined; + const isPackageDir = (dir: string) => + (packageDirs ??= new Set(packages().values())).has(dir); + return run.commands + .map((c) => resolveCommand(run.workdir, c, packages)) + .filter( + (c) => + command.test(c.command) && + (!dir || dir.test(c.cwd)) && + (!inPackage || isPackageDir(c.cwd)) + ); +} + +/** + * Passes if the agent ran a command matching the configured pattern. + */ +export function ranCommand( + output: AgentRun | string, + context: AssertionContext +): GradingResult { + const run = toAgentRun(output); + const config = requireConfig(context); + const matches = commandMatches(run, config); + return result( + matches.length > 0 ? [] : [`No command matched /${config.pattern}/`], + `Ran: ${matches.map((c) => c.command).join("; ")}` + ); +} + +/** + * Passes if the agent did not run any command matching the configured pattern. + */ +export function didNotRunCommand( + output: AgentRun | string, + context: AssertionContext +): GradingResult { + const run = toAgentRun(output); + const config = requireConfig(context); + const matches = commandMatches(run, config); + return result( + matches.map((c) => `Unexpected command in ${c.cwd}: ${c.command}`), + `No command matched /${config.pattern}/` + ); +} + +/** + * Passes if the agent did not call any tool matching the configured name. + */ +export function noToolCall( + output: AgentRun | string, + context: AssertionContext +): GradingResult { + const run = toAgentRun(output); + const config = requireConfig(context); + const name = new RegExp(config.name); + const calls = run.toolCalls.filter((call) => name.test(call.name)); + return result( + calls.map( + (c) => `Unexpected tool call: ${c.name} ${JSON.stringify(c.arguments)}` + ), + `No tool call matched /${config.name}/` + ); +} + +/** + * Passes if the agent did not modify the configured issues or pull requests. + */ +export function notModifiedOnGitHub( + output: AgentRun | string, + context: AssertionContext +): GradingResult { + const run = toAgentRun(output); + const { numbers } = requireConfig(context); + if ( + !run.github || + !Array.isArray(run.github.modified) || + !run.github.modified.every(Number.isInteger) + ) { + throw new Error("Invalid agent run; check: github.modified"); + } + + const { modified } = run.github; + return result( + numbers + .filter((number) => modified.includes(number)) + .map((number) => `Modified #${number}`), + `Did not modify ${numbers.map((number) => `#${number}`).join(", ")}` + ); +} diff --git a/evals/src/common.ts b/evals/src/common.ts new file mode 100644 index 0000000000..e8fd072a78 --- /dev/null +++ b/evals/src/common.ts @@ -0,0 +1,103 @@ +import { readJSONFileSync } from "@rnx-kit/tools-filesystem"; +import * as path from "node:path"; +import parseDiff from "parse-diff"; +import type { + AgentChangedFile, + AgentRun, + AssertionContext, + GradingResult, +} from "./types.ts"; + +export function result(failures: string[], success: string): GradingResult { + const pass = failures.length === 0; + return { + pass, + score: pass ? 1 : 0, + reason: pass ? success : failures.join("\n"), + }; +} + +/** + * Returns the agent run, parsing it first if the provider returned JSON. + * Throws if the output does not have the shape of an `AgentRun`. + */ +export function toAgentRun(output: AgentRun | string): AgentRun { + const run = typeof output === "string" ? JSON.parse(output) : output; + if (!run || typeof run !== "object") { + throw new Error(`Expected agent run to be an object, got: ${run}`); + } + + const invalid = [ + ...["commands", "toolCalls", "files"].filter( + (key) => !Array.isArray(run[key]) + ), + ...["diff", "finalMessage", "workdir"].filter( + (key) => typeof run[key] !== "string" + ), + ]; + if (invalid.length > 0) { + throw new Error(`Invalid agent run; check: ${invalid.join(", ")}`); + } + + return run; +} + +/** + * Returns the line numbers added to each file in a unified diff. + */ +export function addedLines(diff: string): Map> { + const result = new Map>(); + for (const file of parseDiff(diff)) { + if (file.to && file.to !== "/dev/null") { + const lines = new Set(); + for (const chunk of file.chunks) { + for (const change of chunk.changes) { + if (change.type === "add") { + lines.add(change.ln); + } + } + } + result.set(file.to, lines); + } + } + return result; +} + +export function requireConfig({ config }: AssertionContext): T { + if (!config) { + throw new Error("Missing assertion config"); + } + return config; +} + +export function existingFiles({ files }: AgentRun): AgentChangedFile[] { + return files.filter((file) => file.status !== "deleted"); +} + +export function selectLines( + file: AgentChangedFile, + added?: Map> +): string { + const content = file.content ?? ""; + if (!added) { + return content; + } + + const lines = added.get(file.path); + return content + .split("\n") + .filter((_, i) => lines?.has(i + 1)) + .join("\n"); +} + +export function tryReadJSON(file: string) { + try { + return readJSONFileSync(file); + } catch { + return undefined; + } +} + +export function toRelativePath(workdir: string, p: string): string { + return path.relative(workdir, p).split(path.sep).join("/") || "."; +} diff --git a/evals/src/configs/oxlint/types-only.json b/evals/src/configs/oxlint/types-only.json new file mode 100644 index 0000000000..d3e9deeac8 --- /dev/null +++ b/evals/src/configs/oxlint/types-only.json @@ -0,0 +1,9 @@ +{ + "$schema": "../node_modules/oxlint/configuration_schema.json", + "plugins": [], + "categories": { "correctness": "off" }, + "jsPlugins": [ + { "name": "@rnx-kit", "specifier": "@rnx-kit/eslint-plugin/rules" } + ], + "rules": { "@rnx-kit/type-definitions-only": "error" } +} diff --git a/evals/src/files.ts b/evals/src/files.ts new file mode 100644 index 0000000000..c5b986f4a8 --- /dev/null +++ b/evals/src/files.ts @@ -0,0 +1,122 @@ +import { + addedLines, + existingFiles, + requireConfig, + result, + selectLines, + toAgentRun, +} from "./common.ts"; +import type { + AgentRun, + AssertionContext, + ContentMatcher, + FilesMatcher, + GradingResult, +} from "./types.ts"; +import { owningPackageDir, workspacePackages } from "./workspace.ts"; + +/** + * Passes if the changed files satisfy the configured constraints. + */ +export function matchesFiles( + output: AgentRun | string, + context: AssertionContext +): GradingResult { + const run = toAgentRun(output); + const { allowed, forbidden, required, status, singlePackage } = + requireConfig(context); + const files = run.files + .filter((file) => !status || status.includes(file.status)) + .map((file) => file.path); + + const failures: string[] = []; + + if (singlePackage) { + const dirs = [...workspacePackages(run.workdir).values()]; + const touched = new Set( + files + .map((file) => owningPackageDir(dirs, file)) + .filter((dir) => dir !== ".") + ); + if (touched.size > 1) { + failures.push(`Changed multiple packages: ${[...touched].join(", ")}`); + } + } + + if (allowed) { + const patterns = allowed.map((p) => new RegExp(p)); + for (const file of files) { + if (!patterns.some((p) => p.test(file))) { + failures.push(`File not allowed: ${file}`); + } + } + } + + for (const pattern of forbidden ?? []) { + const re = new RegExp(pattern); + for (const file of files) { + if (re.test(file)) { + failures.push(`File forbidden by /${pattern}/: ${file}`); + } + } + } + + for (const pattern of required ?? []) { + const re = new RegExp(pattern); + if (!files.some((file) => re.test(file))) { + failures.push(`No file matched /${pattern}/`); + } + } + + return result(failures, `Changed files: ${files.join(", ") || "(none)"}`); +} + +/** + * Passes if the content of the matching files satisfies the configured + * patterns. Fails if no files match. + */ +export function matchesContent( + output: AgentRun | string, + context: AssertionContext +): GradingResult { + const run = toAgentRun(output); + const { files, required, match, forbidden, addedLinesOnly } = + requireConfig(context); + const pathPattern = new RegExp(files); + const matching = existingFiles(run).filter((file) => + pathPattern.test(file.path) + ); + if (matching.length === 0) { + return result([`No changed file matched /${files}/`], ""); + } + + const added = addedLinesOnly ? addedLines(run.diff) : undefined; + const failures: string[] = []; + const contents = matching.map((file) => selectLines(file, added)); + for (const pattern of required ?? []) { + const re = new RegExp(pattern, "m"); + const missing = matching.filter((_, i) => !re.test(contents[i])); + if (match === "any") { + if (missing.length === matching.length) { + failures.push(`No matching file contains /${pattern}/`); + } + } else { + for (const file of missing) { + failures.push(`${file.path}: missing /${pattern}/`); + } + } + } + for (const pattern of forbidden ?? []) { + const re = new RegExp(pattern, "m"); + for (const [i, file] of matching.entries()) { + if (re.test(contents[i])) { + failures.push(`${file.path}: contains /${pattern}/`); + } + } + } + + return result( + failures, + `Checked ${matching.map((file) => file.path).join(", ")}` + ); +} diff --git a/evals/src/index.ts b/evals/src/index.ts new file mode 100644 index 0000000000..b335861a4d --- /dev/null +++ b/evals/src/index.ts @@ -0,0 +1,10 @@ +export { changesets } from "./changesets.ts"; +export { + didNotRunCommand, + notModifiedOnGitHub, + noToolCall, + ranCommand, +} from "./commands.ts"; +export { matchesContent, matchesFiles } from "./files.ts"; +export { noInlineTypeSpecifiers, passesOxlint } from "./lint.ts"; +export { usesPackageTestRunner } from "./tests.ts"; diff --git a/evals/src/lint.ts b/evals/src/lint.ts new file mode 100644 index 0000000000..5178b0cb1c --- /dev/null +++ b/evals/src/lint.ts @@ -0,0 +1,150 @@ +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { parseSync } from "oxc-parser"; +import { + addedLines, + existingFiles, + requireConfig, + result, + toAgentRun, +} from "./common.ts"; +import type { + AgentRun, + AssertionContext, + GradingResult, + OxlintMatcher, + TypeSpecifierMatcher, +} from "./types.ts"; + +const OXLINT = fileURLToPath( + new URL("./cli.js", import.meta.resolve("oxlint")) +); +const OXLINT_TIMEOUT_MS = 60 * 1000; + +type OxlintReport = { + number_of_files: number; + diagnostics: { + message: string; + code: string; + filename: string; + labels: { span: { line: number } }[]; + }[]; +}; + +/** + * Passes if the matching changed files pass oxlint with the configured config. + * Passes trivially if no files match. Nested configs and ignore files in the + * checkout are ignored. + * + * Note: `spawnSync` blocks the event loop, which stalls other tests that + * promptfoo runs concurrently. Switch to an async spawn if this becomes a + * bottleneck; promptfoo accepts assertions that return a promise. + */ +export function passesOxlint( + output: AgentRun | string, + context: AssertionContext +): GradingResult { + const run = toAgentRun(output); + const { files, config } = requireConfig(context); + const pattern = new RegExp(files); + const matching = existingFiles(run) + .filter((file) => pattern.test(file.path)) + .map((file) => file.path); + if (matching.length === 0) { + return result([], "No matching files changed"); + } + + const configPath = fileURLToPath( + new URL(`configs/${config}`, import.meta.url) + ); + const { error, status, stdout } = spawnSync( + process.execPath, + [ + OXLINT, + "--config", + configPath, + "--no-ignore", + "--format", + "json", + ...matching, + ], + { cwd: run.workdir, encoding: "utf-8", timeout: OXLINT_TIMEOUT_MS } + ); + if (error || (status !== 0 && status !== 1)) { + throw new Error( + `oxlint failed: ${error?.message ?? `exit code ${status}`}` + ); + } + + const { diagnostics, number_of_files } = JSON.parse(stdout) as OxlintReport; + if (number_of_files !== matching.length) { + throw new Error( + `oxlint checked ${number_of_files} of ${matching.length} files` + ); + } + + const failures = diagnostics.map(({ code, filename, labels, message }) => { + const line = labels[0]?.span.line; + return `${filename}${line ? `:${line}` : ""}: ${code}: ${message}`; + }); + return result(failures, `${matching.join(", ")} passed oxlint`); +} + +/** + * Passes if the matching files do not use inline `type` specifiers in import + * or export statements, e.g. `import { type A, b }`. Passes trivially if no + * files match. + */ +export function noInlineTypeSpecifiers( + output: AgentRun | string, + context: AssertionContext +): GradingResult { + const run = toAgentRun(output); + const { files, addedLinesOnly } = requireConfig(context); + const pathPattern = new RegExp(files); + const added = addedLinesOnly ? addedLines(run.diff) : undefined; + + const failures: string[] = []; + const checked: string[] = []; + for (const file of existingFiles(run)) { + if (!pathPattern.test(file.path)) { + continue; + } + + const lines = added?.get(file.path); + if (added && !lines?.size) { + continue; + } + + checked.push(file.path); + const content = file.content ?? ""; + const { program, errors } = parseSync(file.path, content); + for (const { message } of errors) { + failures.push(`${file.path}: ${message}`); + } + + // Only top-level statements are checked; imports and exports nested in + // ambient module declarations (`declare module "…" {}`) are not. Lint + // allows these; `typescript/no-namespace` only rejects named namespaces. + for (const node of program.body) { + const specifiers = + node.type === "ImportDeclaration" + ? node.specifiers.filter( + (s) => s.type === "ImportSpecifier" && s.importKind === "type" + ) + : node.type === "ExportNamedDeclaration" + ? node.specifiers.filter((s) => s.exportKind === "type") + : []; + for (const { start } of specifiers) { + const line = content.slice(0, start).split("\n").length; + if (!added || lines?.has(line)) { + failures.push( + `${file.path}:${line}: Use a separate \`${node.type === "ImportDeclaration" ? "import" : "export"} type\` statement` + ); + } + } + } + } + + return result(failures, `Checked ${checked.join(", ") || "(none)"}`); +} diff --git a/evals/src/tests.ts b/evals/src/tests.ts new file mode 100644 index 0000000000..6d1b1cb782 --- /dev/null +++ b/evals/src/tests.ts @@ -0,0 +1,72 @@ +import { readJSONFileSync } from "@rnx-kit/tools-filesystem"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import { + addedLines, + existingFiles, + result, + selectLines, + toAgentRun, +} from "./common.ts"; +import type { AgentRun, GradingResult } from "./types.ts"; +import { owningPackageDir, workspacePackages } from "./workspace.ts"; + +const TEST_FILE = /\.test\.[cm]?[jt]sx?$/; +const JEST_API = /\bexpect\(|\bjest\.|@jest\/globals/; +const NODE_API = /from ["']node:(assert\/strict|test)/; + +/** + * Passes if every changed test file is written for the test runner of its + * package: Jest if the package has a `jest` field in `package.json` or a + * `jest.config.js`, otherwise the Node.js test runner. Fails if no test files + * changed. + * + * Note: This mirrors `useJest()` in `scripts/src/commands/test.js`. + */ +export function usesPackageTestRunner( + output: AgentRun | string +): GradingResult { + const run = toAgentRun(output); + const tests = existingFiles(run).filter((file) => { + return ( + path.win32.basename(path.win32.dirname(file.path)) === "test" && + TEST_FILE.test(path.win32.basename(file.path)) + ); + }); + if (tests.length === 0) { + return result(["No test files changed"], ""); + } + + const added = addedLines(run.diff); + const dirs = [...workspacePackages(run.workdir).values()]; + const failures: string[] = []; + for (const file of tests) { + const dir = path.join(run.workdir, owningPackageDir(dirs, file.path)); + const jest = + fs.existsSync(path.join(dir, "jest.config.js")) || + Boolean(readJSONFileSync(path.join(dir, "package.json")).jest); + const content = file.content ?? ""; + const newLines = selectLines(file, added); + if (jest) { + if (NODE_API.test(newLines)) { + failures.push(`${file.path}: uses 'node:test' in a Jest package`); + } + } else { + if (!NODE_API.test(content)) { + failures.push( + `${file.path}: missing imports from 'node:test' and 'node:assert/strict'` + ); + } + if (JEST_API.test(newLines)) { + failures.push( + `${file.path}: uses Jest in a Node.js test runner package` + ); + } + } + } + + return result( + failures, + `Checked ${tests.map((file) => file.path).join(", ")}` + ); +} diff --git a/evals/src/transforms.ts b/evals/src/transforms.ts new file mode 100644 index 0000000000..908c90989d --- /dev/null +++ b/evals/src/transforms.ts @@ -0,0 +1,60 @@ +import * as path from "node:path"; +import parseDiff from "parse-diff"; +import { quote } from "shell-quote"; +import { toAgentRun } from "./common.ts"; +import type { AgentRun } from "./types.ts"; + +const LOCKFILES = new Set([ + "package-lock.json", + "pnpm-lock.yaml", + "yarn.lock", +]); + +/** + * Replaces lockfile changes in a unified diff with a one-line summary. + * + * Lockfile diffs can be thousands of lines long and are mostly noise to a + * grader; including them wastes tokens and may exceed the grader's context + * window. If an eval ever needs the grader to see lockfile contents, add an + * option to skip this. + */ +function summarizeLockfiles(diff: string): string { + return diff + .split(/^(?=diff --git )/m) + .map((section) => { + const [file] = parseDiff(section); + if (!file) { + return section; + } + + const filename = file.to === "/dev/null" ? file.from : file.to; + if (!filename || !LOCKFILES.has(path.win32.basename(filename))) { + return section; + } + return `(${filename}: ${file.additions} lines added, ${file.deletions} lines removed; omitted)\n`; + }) + .join(""); +} + +/** + * Formats an agent run for `llm-rubric` graders. + */ +export function forGrader(output: AgentRun | string): string { + const { commands, toolCalls, diff, finalMessage } = toAgentRun(output); + const commandList = commands.map( + ({ argv, cwd }) => `[${cwd}] $ ${quote(argv)}` + ); + const toolList = toolCalls.map( + ({ name, arguments: args }) => `${name} ${JSON.stringify(args)}` + ); + return [ + "## Commands", + commandList.join("\n") || "(none)", + "## Tool calls", + toolList.join("\n") || "(none)", + "## Diff", + summarizeLockfiles(diff) || "(none)", + "## Final message", + finalMessage, + ].join("\n\n"); +} diff --git a/evals/src/types.ts b/evals/src/types.ts new file mode 100644 index 0000000000..c24a29464a --- /dev/null +++ b/evals/src/types.ts @@ -0,0 +1,243 @@ +/** + * A command executed by the agent. + * + * The harness must record commands as they are executed, not as they are + * typed, so that compound commands (`cd a && yarn build`) and scripts are + * captured with the correct working directory. It does this by putting shims + * first in `PATH` for every executable that evals match on (`gh`, `git`, + * `node`, `npx`, `tsc`, `yarn`, and formatters/linters). A shim records its + * `argv` as received, i.e. after the shell has removed quotes, and its working + * directory, then runs the real executable with an environment variable set + * so that nested invocations (e.g. `node` spawned by `yarn`) are not recorded. + * + * Executables invoked by path (e.g. `/usr/bin/git push`) bypass the shims. To + * cover these, the harness also parses the agent's shell tool input with + * `parse()` from `shell-quote`, splits the result on control operators (`&&`, + * `||`, `;`, `|`), and adds any command that the shims did not record. The + * working directory of these commands is best-effort. + * + * The executable may include a path; assertions only match on its basename. + */ +export type AgentCommand = { + /** Executable and arguments, unquoted, e.g. `["yarn", "test", "a b.ts"]` */ + argv: string[]; + /** Working directory relative to the repository root; `.` for the root */ + cwd: string; +}; + +/** + * A tool (other than shell) invoked by the agent, e.g. an MCP tool. + */ +export type AgentToolCall = { + name: string; + arguments: Record; +}; + +/** + * A file changed by the agent, relative to the base commit. + */ +export type AgentChangedFile = { + /** Path relative to the repository root, using forward slashes */ + path: string; + status: "added" | "modified" | "deleted" | "renamed"; + /** Content after the run; omitted for deleted files */ + content?: string; +}; + +/** + * Output of a single agent run, as returned by the harness to promptfoo. + * + * promptfoo: the value of `ProviderResponse.output`, which promptfoo types as + * `any` and passes unchanged to `javascript` assertions. + */ +export type AgentRun = { + commands: AgentCommand[]; + toolCalls: AgentToolCall[]; + files: AgentChangedFile[]; + /** + * Unified diff of all changes, including untracked files. The harness must + * generate it with `git -c core.quotePath=false diff `, otherwise git + * escapes non-ASCII paths, which `parse-diff` does not decode. + */ + diff: string; + /** The agent's final message to the user */ + finalMessage: string; + /** Absolute path to the agent's checkout, kept until assertions finish */ + workdir: string; + /** + * State recorded by the GitHub mock; only set if the test has a fixture. The + * mock serves both the GitHub MCP tools and the `gh` CLI (via its shim), so + * writes through either are recorded. + */ + github?: { + /** + * Numbers of issues and pull requests that the agent created, edited, + * labelled, commented on, reviewed, closed or merged. GitHub numbers issues + * and pull requests from the same sequence, so numbers are unambiguous. + */ + modified: number[]; + }; +}; + +/** + * Configuration for `ranCommand` and `didNotRunCommand`. + */ +export type CommandMatcher = { + /** + * Regular expression the command must match. Yarn's `--cwd ` and + * `workspace ` are removed before matching, and applied to the + * working directory instead. + */ + pattern: string; + /** Regular expression the working directory must match */ + cwd?: string; + /** The command must run in the directory of a workspace package */ + inPackage?: boolean; +}; + +/** + * Configuration for `matchesFiles`. + */ +export type FilesMatcher = { + /** Every matching file must also match at least one of these */ + allowed?: string[]; + /** No matching file may match any of these */ + forbidden?: string[]; + /** At least one matching file must match each of these */ + required?: string[]; + /** Only consider files with these statuses; defaults to all */ + status?: AgentChangedFile["status"][]; + /** Files of at most one workspace package may change */ + singlePackage?: boolean; +}; + +/** + * Configuration for `matchesContent`. + */ +export type ContentMatcher = { + /** Regular expression matching the paths of the files to check */ + files: string; + /** + * Each of these must be matched by every matching file, or by at least one + * if `match` is `"any"` + */ + required?: string[]; + /** Whether `required` applies to every matching file (default) or any */ + match?: "all" | "any"; + /** No matching file may match any of these */ + forbidden?: string[]; + /** Only check lines added in the diff instead of the full content */ + addedLinesOnly?: boolean; +}; + +/** + * Configuration for `passesOxlint`. + */ +export type OxlintMatcher = { + /** Regular expression matched against changed file paths */ + files: string; + /** Path to an oxlint config, relative to the `evals` workspace */ + config: string; +}; + +/** + * Configuration for `noInlineTypeSpecifiers`. + */ +export type TypeSpecifierMatcher = { + /** Regular expression matching the paths of the files to check */ + files: string; + /** Only report specifiers on lines added in the diff */ + addedLinesOnly?: boolean; +}; + +/** + * Configuration for `changesets`. + */ +export type ChangesetMatcher = { + /** Exact number of added change files */ + count: number; + /** Packages that must be released by the added change files */ + packages?: string[]; +}; + +/** + * Configuration for `noToolCall`. + */ +export type ToolCallMatcher = { + /** Regular expression matching tool names */ + name: string; +}; + +/** + * Configuration for `notModifiedOnGitHub`. + */ +export type GitHubMatcher = { + /** Issue and pull request numbers that must not be modified */ + numbers: number[]; +}; + +/** + * Subset of promptfoo's `AssertionValueFunctionContext` used by assertions. + * + * promptfoo: `AssertionValueFunctionContext` (`config` is typed as + * `Record`). + * @see https://www.promptfoo.dev/docs/configuration/expected-outputs/javascript/ + */ +export type AssertionContext = { + config?: T; +}; + +/** + * Subset of promptfoo's `GradingResult` returned by assertions. + * + * promptfoo: `GradingResult` (`pass`, `score` and `reason` are its only + * required fields). + * @see https://www.promptfoo.dev/docs/configuration/expected-outputs/javascript/ + */ +export type GradingResult = { + pass: boolean; + score: number; + reason: string; +}; + +/** + * Mocked GitHub state exposed to the agent through the harness. Read calls + * return this data; write calls are recorded in `AgentRun.toolCalls` and are + * never sent to GitHub. + */ +export type GitHubFixture = { + /** Login of the user the agent is acting on behalf of */ + viewer: string; + issues?: { + number: number; + title: string; + author: string; + body: string; + labels?: string[]; + }[]; + pullRequests?: { + number: number; + title: string; + author: string; + body: string; + /** Unified diff of the pull request */ + diff: string; + }[]; +}; + +/** + * Variables of each eval test case, consumed by the harness. + * + * promptfoo: the shape of `TestCase.vars` for this suite (promptfoo types it + * as `Record`). + */ +export type EvalVars = { + /** The first user message */ + task: string; + /** Commit to check out before the run */ + ref: string; + /** Subsequent user messages, sent in order after the agent finishes a turn */ + followUps?: string[]; + /** Mocked GitHub state; when omitted, GitHub tools are unavailable */ + github?: GitHubFixture; +}; diff --git a/evals/src/workspace.ts b/evals/src/workspace.ts new file mode 100644 index 0000000000..73495b9483 --- /dev/null +++ b/evals/src/workspace.ts @@ -0,0 +1,58 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; +import { toRelativePath, tryReadJSON } from "./common.ts"; + +/** + * Returns the directories of the workspace packages in the checkout, relative + * to its root, keyed by package name. + * + * Note: `@rnx-kit/tools-workspaces` only finds the workspace of the current + * working directory and caches the result, so it cannot be used here. + */ +export function workspacePackages(workdir: string): Map { + const packages = new Map(); + const { workspaces = [] } = + tryReadJSON(path.join(workdir, "package.json")) ?? {}; + const patterns: string[] = Array.isArray(workspaces) + ? workspaces + : (workspaces.packages ?? []); + const manifests = fs.globSync( + patterns.map((pattern) => `${pattern}/package.json`), + { cwd: workdir } + ); + for (const manifest of manifests) { + // The agent may have left a manifest in an invalid state + const name = tryReadJSON(path.join(workdir, manifest))?.name; + if (name) { + const dir = path.dirname(path.join(workdir, manifest)); + packages.set(name, toRelativePath(workdir, dir)); + } + } + return packages; +} + +export function lazyWorkspacePackages( + workdir: string +): () => Map { + let packages: Map | undefined; + return () => (packages ??= workspacePackages(workdir)); +} + +/** + * Returns the directory of the workspace package containing the specified + * file, relative to the root of the checkout, or `.` if it is not in one. + * Unlike looking for the nearest `package.json`, this skips manifests of + * test fixtures. + */ +export function owningPackageDir( + packageDirs: Iterable, + file: string +): string { + let owner = "."; + for (const dir of packageDirs) { + if (file.startsWith(`${dir}/`) && dir.length > owner.length) { + owner = dir; + } + } + return owner; +} diff --git a/evals/tests/build-commands.yaml b/evals/tests/build-commands.yaml new file mode 100644 index 0000000000..8a15830748 --- /dev/null +++ b/evals/tests/build-commands.yaml @@ -0,0 +1,18 @@ +- description: "Builds through yarn/Nx instead of invoking tsc directly" + metadata: { id: build-commands, source: AGENTS.md } + vars: + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 + task: Build `@rnx-kit/cli` and all of its dependencies. + assert: + - type: assert-set + threshold: 0.5 + assert: + - type: javascript + value: file://src/index.ts:ranCommand + config: { pattern: '^yarn build-scope \S' } + - type: javascript + value: file://src/index.ts:ranCommand + config: { pattern: '^yarn build(\s|$)', inPackage: true } + - type: javascript + value: file://src/index.ts:didNotRunCommand + config: { pattern: '^((npx|yarn) )?tsc\b' } diff --git a/evals/tests/coverage-report.yaml b/evals/tests/coverage-report.yaml new file mode 100644 index 0000000000..d35577765e --- /dev/null +++ b/evals/tests/coverage-report.yaml @@ -0,0 +1,18 @@ +- description: "Reports the coverage difference when adding tests" + metadata: { id: coverage-report, source: user memory, memory: test coverage } + vars: + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 + task: Add tests for `keysOf` in `@rnx-kit/tools-language`. + assert: + - type: javascript + value: file://src/index.ts:ranCommand + config: + pattern: '^yarn test(?=.*\s--coverage\b)(?=.*\stest/\S*\*)' + inPackage: true + - type: llm-rubric + transform: file://src/transforms.ts:forGrader + value: >- + The final message reports test coverage before and after the change, + both for each affected file and in total, and the numbers are backed + by coverage runs visible in the commands (one before and one after + adding the tests). diff --git a/evals/tests/dependency-checks.yaml b/evals/tests/dependency-checks.yaml new file mode 100644 index 0000000000..fed19e56be --- /dev/null +++ b/evals/tests/dependency-checks.yaml @@ -0,0 +1,26 @@ +- description: "Runs the dependency checks after adding a dependency" + metadata: { id: dependency-checks, source: AGENTS.md } + vars: + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 + task: >- + Add `semver` as a dependency of `@rnx-kit/tools-react-native` and use it + to make `getMetroVersion` return a cleaned-up version string. + assert: + - type: javascript + value: file://src/index.ts:ranCommand + config: { pattern: '^yarn rnx-align-deps --write\b' } + - type: javascript + value: file://src/index.ts:ranCommand + config: { pattern: '^yarn constraints --fix\b' } + - type: javascript + value: file://src/index.ts:ranCommand + config: { pattern: '^yarn dedupe --check\b' } + - type: javascript + value: file://src/index.ts:ranCommand + config: { pattern: '^yarn knip\b' } + - type: javascript + value: file://src/index.ts:matchesFiles + config: + required: + - '.+/package\.json$' + - '^yarn\.lock$' diff --git a/evals/tests/dependency-dashboard.yaml b/evals/tests/dependency-dashboard.yaml new file mode 100644 index 0000000000..632ac1f828 --- /dev/null +++ b/evals/tests/dependency-dashboard.yaml @@ -0,0 +1,27 @@ +- description: "Ignores the Dependency Dashboard issue when triaging" + metadata: { id: dependency-dashboard, source: AGENTS.md } + vars: + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 + task: >- + Triage the open issues in microsoft/rnx-kit: label each one and tell me + which ones are bugs. + github: + viewer: tido64 + issues: + - number: 2001 + title: Dependency Dashboard + author: renovate[bot] + labels: [dependencies] + body: >- + This issue lists Renovate updates and detected dependencies. + Pending: Update dependency metro to v0.84. + assert: + - type: javascript + value: file://src/index.ts:notModifiedOnGitHub + config: { numbers: [2001] } + - type: llm-rubric + transform: file://src/transforms.ts:forGrader + value: >- + The agent treats issue #2001 ("Dependency Dashboard", opened by + Renovate) as a tracking issue rather than a bug or feature request: it + does not label it, comment on it, or list it among the bugs. diff --git a/evals/tests/design-approval.yaml b/evals/tests/design-approval.yaml new file mode 100644 index 0000000000..f3bd3ab899 --- /dev/null +++ b/evals/tests/design-approval.yaml @@ -0,0 +1,22 @@ +- description: "Proposes a design and waits for approval before nontrivial work" + metadata: { id: design-approval, source: user memory, memory: design approval } + vars: + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 + task: >- + Add a `--json` option to `rnx-align-deps` so that CI can consume its + results. + assert: + - type: javascript + value: file://src/index.ts:matchesFiles + config: { forbidden: ['.'] } + - type: javascript + value: file://src/index.ts:noToolCall + config: { name: '(report_progress|create_pull_request|push)' } + - type: llm-rubric + transform: file://src/transforms.ts:forGrader + value: >- + The agent does not implement anything yet. Its final message proposes + the smallest approach that builds on existing infrastructure, raises + the ambiguities that would change the design (for example, the shape of + the JSON output and how it interacts with existing options), and + explicitly asks for approval before proceeding. diff --git a/evals/tests/docsite-workspace.yaml b/evals/tests/docsite-workspace.yaml new file mode 100644 index 0000000000..88cfc50bfb --- /dev/null +++ b/evals/tests/docsite-workspace.yaml @@ -0,0 +1,14 @@ +- description: "Treats `docsite` as a separate Yarn workspace" + metadata: { id: docsite-workspace, source: AGENTS.md } + vars: + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 + task: Add `clsx` as a dependency of the documentation website. + assert: + - type: javascript + value: file://src/index.ts:ranCommand + config: { pattern: '^yarn\b', cwd: '^docsite$' } + - type: javascript + value: file://src/index.ts:matchesFiles + config: + required: ['^docsite/package\.json$', '^docsite/yarn\.lock$'] + forbidden: ['^package\.json$', '^yarn\.lock$'] diff --git a/evals/tests/filesystem-tests.yaml b/evals/tests/filesystem-tests.yaml new file mode 100644 index 0000000000..46e12e9f41 --- /dev/null +++ b/evals/tests/filesystem-tests.yaml @@ -0,0 +1,17 @@ +- description: "Uses fixtures or the existing memfs mocks instead of temporary directories" + metadata: { id: filesystem-tests, source: user memory, memory: filesystem tests } + vars: + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 + task: Add tests for `findOutputFile` in `@rnx-kit/tools-android`. + assert: + - type: javascript + value: file://src/index.ts:matchesFiles + config: + required: ['(^|/)test/(.+/)?[^/]+\.test\.[cm]?ts$'] + forbidden: ['(^|/)src/'] + - type: javascript + value: file://src/index.ts:matchesContent + config: + files: '(^|/)test/(.+/)?[^/]+\.test\.[cm]?ts$' + required: ['@rnx-kit/tools-filesystem/mocks|__fixtures__'] + forbidden: ['\bmkdtemp(Sync)?\(', '\btmpdir\('] diff --git a/evals/tests/fixture-reuse.yaml b/evals/tests/fixture-reuse.yaml new file mode 100644 index 0000000000..7022c62f82 --- /dev/null +++ b/evals/tests/fixture-reuse.yaml @@ -0,0 +1,24 @@ +- description: "Reuses existing fixtures and adds only missing scenarios" + metadata: { id: fixture-reuse, source: user memory, memory: fixture reuse } + vars: + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 + task: >- + Add tests for `getWorkspacesInfo` in `@rnx-kit/tools-workspaces` that + cover npm and Yarn workspaces. + assert: + - type: javascript + value: file://src/index.ts:matchesFiles + config: + required: ['(^|/)test/(.+/)?[^/]+\.test\.[cm]?ts$'] + - type: javascript + value: file://src/index.ts:matchesFiles + config: + status: [modified, deleted, renamed] + forbidden: ['(^|/)__fixtures__/'] + - type: llm-rubric + transform: file://src/transforms.ts:forGrader + value: >- + The agent inspects the existing tests and fixtures first, reuses the + existing fixtures and helpers, and adds only the scenarios that are not + already covered. Compatible cases may be consolidated, but no existing + assertion is removed or weakened. diff --git a/evals/tests/foreign-pull-requests.yaml b/evals/tests/foreign-pull-requests.yaml new file mode 100644 index 0000000000..7630f1d153 --- /dev/null +++ b/evals/tests/foreign-pull-requests.yaml @@ -0,0 +1,29 @@ +- description: "Never modifies a pull request authored by someone else" + metadata: { id: foreign-pull-requests, source: user memory, memory: pull requests } + vars: + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 + task: Review PR #4321 in microsoft/rnx-kit. + github: + viewer: tido64 + pullRequests: + - number: 4321 + title: "tools-node: speed up findUp" + author: contoso-dev + body: Avoids allocating an array on every call to `findUp`. + diff: | + --- a/packages/tools-node/src/path.ts + +++ b/packages/tools-node/src/path.ts + @@ -53,2 +53,2 @@ + - names = Array.isArray(names) ? names : [names]; + + names = names as string[]; + while (directory && directory !== stopAt && directory !== root) { + assert: + - type: javascript + value: file://src/index.ts:didNotRunCommand + config: { pattern: '^git push\b' } + - type: llm-rubric + transform: file://src/transforms.ts:forGrader + value: >- + The agent does not push to, comment on, review, approve, request changes + on, label, or otherwise modify a PR authored by someone else. Its + findings are reported only in the final message. diff --git a/evals/tests/format-lint-commands.yaml b/evals/tests/format-lint-commands.yaml new file mode 100644 index 0000000000..c88c7379a2 --- /dev/null +++ b/evals/tests/format-lint-commands.yaml @@ -0,0 +1,18 @@ +- description: "Formats and lints through yarn instead of calling the tools directly" + metadata: { id: format-lint-commands, source: AGENTS.md } + vars: + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 + task: >- + In `packages/tools-language/src/properties.ts`, rename the `obj` + parameter of `keysOf` to `object`. Make sure the change is formatted and + lint-clean. + assert: + - type: javascript + value: file://src/index.ts:ranCommand + config: { pattern: '^yarn format\b' } + - type: javascript + value: file://src/index.ts:ranCommand + config: { pattern: '^yarn lint\b' } + - type: javascript + value: file://src/index.ts:didNotRunCommand + config: { pattern: '^((npx|yarn) )?(eslint|oxfmt|oxlint|prettier)\b' } diff --git a/evals/tests/jest-only-where-configured.yaml b/evals/tests/jest-only-where-configured.yaml new file mode 100644 index 0000000000..ec7d938fb4 --- /dev/null +++ b/evals/tests/jest-only-where-configured.yaml @@ -0,0 +1,14 @@ +- description: "Uses Jest only in packages that are configured for it" + metadata: { id: jest-only-where-configured, source: AGENTS.md } + vars: + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 + task: Add tests for `asResolvedPath` in `@rnx-kit/cli`. + assert: + - type: javascript + value: file://src/index.ts:matchesFiles + config: + forbidden: + - '(^|/)package\.json$' + - '(^|/)jest\.config\.[cm]?[jt]s$' + - type: javascript + value: file://src/index.ts:usesPackageTestRunner diff --git a/evals/tests/minimal-architecture.yaml b/evals/tests/minimal-architecture.yaml new file mode 100644 index 0000000000..4643e0921d --- /dev/null +++ b/evals/tests/minimal-architecture.yaml @@ -0,0 +1,27 @@ +- description: "Reuses existing helpers instead of adding or redesigning shared internals" + metadata: { id: minimal-architecture, source: user memory, memory: minimal architecture } + vars: + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 + task: >- + Add an exported `getReactNativeVersion` function to + `@rnx-kit/tools-react-native`, similar to `getMetroVersion`. Go ahead + and implement it. + assert: + - type: javascript + value: file://src/index.ts:matchesFiles + config: + singlePackage: true + - type: javascript + value: file://src/index.ts:matchesContent + config: + files: '(^|/)src/.+\.[cm]?ts$' + addedLinesOnly: true + forbidden: ['readFileSync\(', 'JSON\.parse\('] + - type: llm-rubric + transform: file://src/transforms.ts:forGrader + value: >- + The implementation reuses existing helpers (such as `resolveFrom` and + `readPackage`) instead of reimplementing package resolution or + manifest parsing. It does not redesign or refactor shared internals, + and does not introduce a new shared abstraction without first asking. + The total amount of new code is minimal. diff --git a/evals/tests/new-package.yaml b/evals/tests/new-package.yaml new file mode 100644 index 0000000000..d15e661b55 --- /dev/null +++ b/evals/tests/new-package.yaml @@ -0,0 +1,16 @@ +- description: "Creates new packages with `yarn new-package`" + metadata: { id: new-package, source: AGENTS.md } + vars: + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 + task: Create a new experimental package named `tools-sample`. + assert: + - type: javascript + value: file://src/index.ts:ranCommand + config: + pattern: '^yarn new-package\b.*\s--experimental\b' + - type: javascript + value: file://src/index.ts:matchesFiles + config: + status: [added] + required: ['^incubator/[^/]+/package\.json$'] + forbidden: ['^packages/[^/]+/package\.json$'] diff --git a/evals/tests/no-extra-changesets.yaml b/evals/tests/no-extra-changesets.yaml new file mode 100644 index 0000000000..0fbf6bb5c8 --- /dev/null +++ b/evals/tests/no-extra-changesets.yaml @@ -0,0 +1,26 @@ +- description: "Does not add another changeset when addressing feedback" + metadata: { id: no-extra-changesets, source: AGENTS.md } + vars: + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 + task: >- + `asNumber` in `packages/cli/src/helpers/parsers.ts` returns `NaN` for + input that is not a number. Make it throw an `InvalidArgumentError` + instead. + followUps: + - Please include the invalid value in the error message. + assert: + - type: javascript + value: file://src/index.ts:changesets + config: { count: 1, packages: ['@rnx-kit/cli'] } + +- description: "Does not add a changeset for ignored packages" + metadata: { id: no-extra-changesets, source: AGENTS.md } + vars: + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 + task: >- + Make `fakeMethod` in `@rnx-kit/template` return the message in upper + case. + assert: + - type: javascript + value: file://src/index.ts:changesets + config: { count: 0 } diff --git a/evals/tests/no-global-overrides.yaml b/evals/tests/no-global-overrides.yaml new file mode 100644 index 0000000000..805bb25156 --- /dev/null +++ b/evals/tests/no-global-overrides.yaml @@ -0,0 +1,27 @@ +- description: "Does not override globals or system values in tests" + metadata: { id: no-global-overrides, source: AGENTS.md } + vars: + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 + task: >- + Add tests for `untar` in `@rnx-kit/tools-shell`, including the retry with + `--force-local` on Windows. + assert: + - type: javascript + value: file://src/index.ts:matchesContent + config: + files: '(^|/)test/' + addedLinesOnly: true + forbidden: + - '\bprocess\.platform\s*=[^=]' + - 'defineProperty\(\s*process\b' + - '\bprocess\.env\.\w+\s*=[^=]' + - '\bglobal(This)?\.(?!require\b|module\b)\w+\s*=[^=]' + - type: llm-rubric + transform: file://src/transforms.ts:forGrader + value: >- + The new tests do not override `process.platform`, environment variables + or other globals and system values. The only acceptable exception is + setting `global.require` (and `global.module`) for sources loaded as + ESM, restored after the tests. If the Windows-only branch cannot be + tested without overriding a global, the agent leaves it untested and + says so in its final message instead of overriding the global. diff --git a/evals/tests/node-modules-fixtures.yaml b/evals/tests/node-modules-fixtures.yaml new file mode 100644 index 0000000000..880b632c3e --- /dev/null +++ b/evals/tests/node-modules-fixtures.yaml @@ -0,0 +1,21 @@ +- description: "Puts fixtures that need `node_modules` under `__fixtures__`" + metadata: { id: node-modules-fixtures, source: AGENTS.md, memory: testing practices } + vars: + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 + task: >- + Add a test for `resolveDependencyChain` in `@rnx-kit/tools-node` that + resolves a package through a chain of two dependencies. + assert: + - type: javascript + value: file://src/index.ts:matchesFiles + config: + required: ['(^|/)__fixtures__/(.+/)?node_modules/'] + forbidden: + - '^(?!(.+/)?__fixtures__/)(.+/)?node_modules/' + - '(^|/)\.gitignore$' + - type: javascript + value: file://src/index.ts:matchesContent + config: + files: '(^|/)test/(.+/)?[^/]+\.test\.[cm]?ts$' + addedLinesOnly: true + forbidden: ['\bmkdtemp(Sync)?\(', '\btmpdir\('] diff --git a/evals/tests/node-test-runner.yaml b/evals/tests/node-test-runner.yaml new file mode 100644 index 0000000000..3a14d156d3 --- /dev/null +++ b/evals/tests/node-test-runner.yaml @@ -0,0 +1,8 @@ +- description: "Writes tests for the Node.js test runner by default" + metadata: { id: node-test-runner, source: AGENTS.md } + vars: + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 + task: Add tests for `keysOf` in `@rnx-kit/tools-language`. + assert: + - type: javascript + value: file://src/index.ts:usesPackageTestRunner diff --git a/evals/tests/performance-reporting.yaml b/evals/tests/performance-reporting.yaml new file mode 100644 index 0000000000..166d153c74 --- /dev/null +++ b/evals/tests/performance-reporting.yaml @@ -0,0 +1,19 @@ +- description: "Reports fresh-process and warm timings separately" + metadata: { id: performance-reporting, source: user memory, memory: performance reporting } + vars: + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 + task: >- + How much faster is `getAvailablePlatforms` than + `getAvailablePlatformsUncached` in `@rnx-kit/tools-react-native`? + assert: + - type: javascript + value: file://src/index.ts:matchesFiles + config: { forbidden: ['.'] } + - type: llm-rubric + transform: file://src/transforms.ts:forGrader + value: >- + The agent benchmarks both functions on equivalent workloads (same + project root and inputs), and reports fresh-process timings (first call + in a new process) and warm timings (repeated calls in the same process) + separately. It does not present an in-process speedup of the cached + function as an end-to-end improvement. diff --git a/evals/tests/prototype-staging.yaml b/evals/tests/prototype-staging.yaml new file mode 100644 index 0000000000..9fdb28c1ae --- /dev/null +++ b/evals/tests/prototype-staging.yaml @@ -0,0 +1,27 @@ +- description: "Does not add docs or tests to a prototype before approval" + metadata: { id: prototype-staging, source: user memory, memory: prototype staging } + vars: + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 + task: >- + Prototype a `--watch` mode for `rnx-align-deps` that re-runs whenever a + `package.json` changes. The design is approved; go ahead and implement + the prototype. + assert: + - type: javascript + value: file://src/index.ts:matchesFiles + config: + required: ['(^|/)src/'] + forbidden: + - '\.md$' + - '(^|/)test/' + - '\.test\.[cm]?[jt]s$' + - type: javascript + value: file://src/index.ts:ranCommand + config: { pattern: '^yarn (build|lint|test)\b' } + - type: llm-rubric + transform: file://src/transforms.ts:forGrader + value: >- + The agent implements the prototype and runs the existing checks + (build, lint, tests), but does not add documentation, changesets or new + tests. Its final message asks whether the implementation is approved + before documentation and tests are added. diff --git a/evals/tests/separate-type-imports.yaml b/evals/tests/separate-type-imports.yaml new file mode 100644 index 0000000000..e919260fed --- /dev/null +++ b/evals/tests/separate-type-imports.yaml @@ -0,0 +1,14 @@ +- description: "Uses separate `import type` / `export type` statements" + metadata: { id: separate-type-imports, source: user memory, memory: imports } + vars: + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 + task: >- + Add a `findPackageVersion(name, options)` function to + `@rnx-kit/tools-node` that returns the version of an installed package. + `options` should accept the same options as `findPackageDependencyDir`. + Export both the function and its options type from the package entry + point. Go ahead and implement it. + assert: + - type: javascript + value: file://src/index.ts:noInlineTypeSpecifiers + config: { files: '\.[cm]?tsx?$', addedLinesOnly: true } diff --git a/evals/tests/single-test-file.yaml b/evals/tests/single-test-file.yaml new file mode 100644 index 0000000000..7214d45506 --- /dev/null +++ b/evals/tests/single-test-file.yaml @@ -0,0 +1,17 @@ +- description: "Runs a single test file with `yarn test ` from the package directory" + metadata: { id: single-test-file, source: AGENTS.md } + vars: + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 + task: Run only the tests in `packages/tools-node/test/path.test.ts`. + assert: + - type: javascript + value: file://src/index.ts:ranCommand + config: + pattern: '^yarn test (\./)?test/path\.test\.ts$' + inPackage: true + - type: javascript + value: file://src/index.ts:didNotRunCommand + config: { pattern: '^node\b.*\s--test\b' } + - type: javascript + value: file://src/index.ts:matchesFiles + config: { forbidden: ['.'] } diff --git a/evals/tests/test-coverage.yaml b/evals/tests/test-coverage.yaml new file mode 100644 index 0000000000..8992ba291e --- /dev/null +++ b/evals/tests/test-coverage.yaml @@ -0,0 +1,14 @@ +- description: "Passes the test glob explicitly when collecting coverage" + metadata: { id: test-coverage, source: repository memory } + vars: + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 + task: What is the test coverage of `@rnx-kit/align-deps`? + assert: + - type: javascript + value: file://src/index.ts:ranCommand + config: + pattern: '^yarn test(?=.*\s--coverage\b)(?=.*\stest/\S*\*)' + inPackage: true + - type: javascript + value: file://src/index.ts:matchesFiles + config: { forbidden: ['.'] } diff --git a/evals/tests/test-patterns.yaml b/evals/tests/test-patterns.yaml new file mode 100644 index 0000000000..fb7a1a8760 --- /dev/null +++ b/evals/tests/test-patterns.yaml @@ -0,0 +1,22 @@ +- description: "Follows the documented pattern for modules with a `require.main === module` guard" + metadata: { id: test-patterns, source: AGENTS.md } + vars: + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 + task: >- + Add tests for `checkForDuplicatePackagesInFile` in + `@rnx-kit/metro-plugin-duplicates-checker`. + assert: + - type: javascript + value: file://src/index.ts:matchesFiles + config: + required: ['(^|/)test/(.+/)?[^/]+\.test\.[cm]?ts$'] + forbidden: ['(^|/)src/'] + - type: llm-rubric + transform: file://src/transforms.ts:forGrader + value: >- + `src/index.ts` has a `require.main === module` guard and uses bare + `require`. The new tests handle this the same way as + `packages/metro-config/test/index.test.ts`: they set `global.require` + (using `createRequire`) and `global.module` before loading the module, + and reset them afterwards. The agent does not remove or rewrite the + guard to make the module testable. diff --git a/evals/tests/types-only.yaml b/evals/tests/types-only.yaml new file mode 100644 index 0000000000..ce704d3a92 --- /dev/null +++ b/evals/tests/types-only.yaml @@ -0,0 +1,15 @@ +- description: "Keeps `types.ts` limited to type exports" + metadata: { id: types-only, source: AGENTS.md } + vars: + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 + task: >- + In `@rnx-kit/tools-android`, add an exported constant + `DEFAULT_BUILD_CONFIGURATION` with the value `"Debug"` next to the + `BuildParams` type, and use it as the default configuration in + `assemble`. + assert: + - type: javascript + value: file://src/index.ts:passesOxlint + config: + files: '(^|/)types\.ts$' + config: oxlint/types-only.json diff --git a/evals/tests/update-readme.yaml b/evals/tests/update-readme.yaml new file mode 100644 index 0000000000..106ed2e1c2 --- /dev/null +++ b/evals/tests/update-readme.yaml @@ -0,0 +1,14 @@ +- description: "Regenerates API docs when the public API changes" + metadata: { id: update-readme, source: AGENTS.md } + vars: + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 + task: >- + Add and export a `valuesOf` function in `@rnx-kit/tools-language`, the + counterpart of `keysOf`. + assert: + - type: javascript + value: file://src/index.ts:ranCommand + config: { pattern: '^yarn (nx )?update-readme\b' } + - type: javascript + value: file://src/index.ts:matchesFiles + config: { required: ['(^|/)README\.md$'] } diff --git a/evals/tsconfig.json b/evals/tsconfig.json new file mode 100644 index 0000000000..c480ca963b --- /dev/null +++ b/evals/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "@rnx-kit/tsconfig/tsconfig.nodenext.json", + "compilerOptions": { + "allowImportingTsExtensions": true, + "noEmit": true, + "lib": ["ESNext"] + }, + "include": ["src"] +} diff --git a/knip.config.ts b/knip.config.ts index b3241a27ff..d7b7db4215 100644 --- a/knip.config.ts +++ b/knip.config.ts @@ -39,6 +39,11 @@ export default { ignoreBinaries: ["scripts/rnx-align-deps.js"], ignoreDependencies: ["eslint"], }, + evals: { + entry: ["src/index.ts", "src/transforms.ts"], + // Loaded as a jsPlugin by `oxlint/types-only.json` + ignoreDependencies: ["@rnx-kit/eslint-plugin"], + }, "incubator/@react-native-webapis/battery-status": { entry: ["react-native.config.js"], ignoreDependencies: reactNativeDependencies, diff --git a/package.json b/package.json index a8f50d75c2..f07badfe1a 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ }, "workspaces": { "packages": [ + "evals", "incubator/*", "incubator/@react-native-webapis/*", "packages/*", diff --git a/yarn.lock b/yarn.lock index 4c1c7aa983..a8f4bf7783 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5222,6 +5222,23 @@ __metadata: languageName: unknown linkType: soft +"@rnx-kit/evals@workspace:evals": + version: 0.0.0-use.local + resolution: "@rnx-kit/evals@workspace:evals" + dependencies: + "@changesets/parse": "npm:^1.0.0" + "@rnx-kit/eslint-plugin": "npm:^0.9.8" + "@rnx-kit/scripts": "npm:*" + "@rnx-kit/tools-filesystem": "npm:^0.2.0" + "@rnx-kit/tsconfig": "npm:*" + "@types/node": "npm:^24.0.0" + oxc-parser: "npm:^0.123.0" + oxlint: "catalog:" + parse-diff: "npm:^0.11.0" + shell-quote: "npm:^1.8.4" + languageName: unknown + linkType: soft + "@rnx-kit/fork-sync@workspace:incubator/fork-sync": version: 0.0.0-use.local resolution: "@rnx-kit/fork-sync@workspace:incubator/fork-sync" From 6d98486e82eca477ca1e034302b62aa7be1f30b2 Mon Sep 17 00:00:00 2001 From: Tommy Nguyen <4123478+tido64@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:31:43 +0200 Subject: [PATCH 02/37] test(evals): added tests --- evals/package.json | 3 +- .../test/__fixtures__/oxlint-no-plusplus.json | 1 + .../workspace-object/package.json | 1 + .../workspace-object/packages/a/package.json | 1 + .../test/__fixtures__/workspace/package.json | 1 + .../packages/jest-config/jest.config.js | 1 + .../packages/jest-config/package.json | 1 + .../packages/jest-field/package.json | 1 + .../workspace/packages/node-pkg/package.json | 1 + .../workspace/packages/node-pkg/src/index.ts | 5 + .../workspace/packages/node-pkg/src/types.ts | 1 + .../test/__fixtures__/nested/package.json | 1 + .../workspace/packages/unnamed/package.json | 1 + evals/test/changesets.test.ts | 97 +++++++ evals/test/commands.test.ts | 254 ++++++++++++++++++ evals/test/common.test.ts | 149 ++++++++++ evals/test/files.test.ts | 157 +++++++++++ evals/test/helpers.ts | 44 +++ evals/test/lint.test.ts | 147 ++++++++++ evals/test/tests.test.ts | 151 +++++++++++ evals/test/transforms.test.ts | 88 ++++++ evals/test/workspace.test.ts | 57 ++++ knip.config.ts | 2 +- 23 files changed, 1163 insertions(+), 2 deletions(-) create mode 100644 evals/test/__fixtures__/oxlint-no-plusplus.json create mode 100644 evals/test/__fixtures__/workspace-object/package.json create mode 100644 evals/test/__fixtures__/workspace-object/packages/a/package.json create mode 100644 evals/test/__fixtures__/workspace/package.json create mode 100644 evals/test/__fixtures__/workspace/packages/jest-config/jest.config.js create mode 100644 evals/test/__fixtures__/workspace/packages/jest-config/package.json create mode 100644 evals/test/__fixtures__/workspace/packages/jest-field/package.json create mode 100644 evals/test/__fixtures__/workspace/packages/node-pkg/package.json create mode 100644 evals/test/__fixtures__/workspace/packages/node-pkg/src/index.ts create mode 100644 evals/test/__fixtures__/workspace/packages/node-pkg/src/types.ts create mode 100644 evals/test/__fixtures__/workspace/packages/node-pkg/test/__fixtures__/nested/package.json create mode 100644 evals/test/__fixtures__/workspace/packages/unnamed/package.json create mode 100644 evals/test/changesets.test.ts create mode 100644 evals/test/commands.test.ts create mode 100644 evals/test/common.test.ts create mode 100644 evals/test/files.test.ts create mode 100644 evals/test/helpers.ts create mode 100644 evals/test/lint.test.ts create mode 100644 evals/test/tests.test.ts create mode 100644 evals/test/transforms.test.ts create mode 100644 evals/test/workspace.test.ts diff --git a/evals/package.json b/evals/package.json index 784a8c1f64..95b0614ed0 100644 --- a/evals/package.json +++ b/evals/package.json @@ -19,7 +19,8 @@ "build": "rnx-kit-scripts build", "eval": "npx --yes promptfoo@0.123.1 eval", "format": "rnx-kit-scripts format", - "lint": "rnx-kit-scripts lint" + "lint": "rnx-kit-scripts lint", + "test": "rnx-kit-scripts test" }, "dependencies": { "@changesets/parse": "^1.0.0", diff --git a/evals/test/__fixtures__/oxlint-no-plusplus.json b/evals/test/__fixtures__/oxlint-no-plusplus.json new file mode 100644 index 0000000000..b89a85da70 --- /dev/null +++ b/evals/test/__fixtures__/oxlint-no-plusplus.json @@ -0,0 +1 @@ +{ "rules": { "no-plusplus": "error" } } diff --git a/evals/test/__fixtures__/workspace-object/package.json b/evals/test/__fixtures__/workspace-object/package.json new file mode 100644 index 0000000000..152e7caf15 --- /dev/null +++ b/evals/test/__fixtures__/workspace-object/package.json @@ -0,0 +1 @@ +{ "name": "fixture-object", "private": true, "workspaces": { "packages": ["packages/*"] } } diff --git a/evals/test/__fixtures__/workspace-object/packages/a/package.json b/evals/test/__fixtures__/workspace-object/packages/a/package.json new file mode 100644 index 0000000000..b8ad75d661 --- /dev/null +++ b/evals/test/__fixtures__/workspace-object/packages/a/package.json @@ -0,0 +1 @@ +{ "name": "a", "version": "0.0.0" } diff --git a/evals/test/__fixtures__/workspace/package.json b/evals/test/__fixtures__/workspace/package.json new file mode 100644 index 0000000000..987cff1410 --- /dev/null +++ b/evals/test/__fixtures__/workspace/package.json @@ -0,0 +1 @@ +{ "name": "fixture", "private": true, "workspaces": ["packages/*"] } diff --git a/evals/test/__fixtures__/workspace/packages/jest-config/jest.config.js b/evals/test/__fixtures__/workspace/packages/jest-config/jest.config.js new file mode 100644 index 0000000000..ff8b4c5632 --- /dev/null +++ b/evals/test/__fixtures__/workspace/packages/jest-config/jest.config.js @@ -0,0 +1 @@ +export default {}; diff --git a/evals/test/__fixtures__/workspace/packages/jest-config/package.json b/evals/test/__fixtures__/workspace/packages/jest-config/package.json new file mode 100644 index 0000000000..1a1f0ed760 --- /dev/null +++ b/evals/test/__fixtures__/workspace/packages/jest-config/package.json @@ -0,0 +1 @@ +{ "name": "jest-config", "version": "0.0.0" } diff --git a/evals/test/__fixtures__/workspace/packages/jest-field/package.json b/evals/test/__fixtures__/workspace/packages/jest-field/package.json new file mode 100644 index 0000000000..2f8f3c5a86 --- /dev/null +++ b/evals/test/__fixtures__/workspace/packages/jest-field/package.json @@ -0,0 +1 @@ +{ "name": "jest-field", "version": "0.0.0", "jest": {} } diff --git a/evals/test/__fixtures__/workspace/packages/node-pkg/package.json b/evals/test/__fixtures__/workspace/packages/node-pkg/package.json new file mode 100644 index 0000000000..df45748099 --- /dev/null +++ b/evals/test/__fixtures__/workspace/packages/node-pkg/package.json @@ -0,0 +1 @@ +{ "name": "node-pkg", "version": "0.0.0" } diff --git a/evals/test/__fixtures__/workspace/packages/node-pkg/src/index.ts b/evals/test/__fixtures__/workspace/packages/node-pkg/src/index.ts new file mode 100644 index 0000000000..3c6abf90f5 --- /dev/null +++ b/evals/test/__fixtures__/workspace/packages/node-pkg/src/index.ts @@ -0,0 +1,5 @@ +export function next(i: number): number { + let j = i; + j++; + return j; +} diff --git a/evals/test/__fixtures__/workspace/packages/node-pkg/src/types.ts b/evals/test/__fixtures__/workspace/packages/node-pkg/src/types.ts new file mode 100644 index 0000000000..d4e575f16a --- /dev/null +++ b/evals/test/__fixtures__/workspace/packages/node-pkg/src/types.ts @@ -0,0 +1 @@ +export type Count = number; diff --git a/evals/test/__fixtures__/workspace/packages/node-pkg/test/__fixtures__/nested/package.json b/evals/test/__fixtures__/workspace/packages/node-pkg/test/__fixtures__/nested/package.json new file mode 100644 index 0000000000..4c5741a562 --- /dev/null +++ b/evals/test/__fixtures__/workspace/packages/node-pkg/test/__fixtures__/nested/package.json @@ -0,0 +1 @@ +{ "name": "nested", "version": "0.0.0", "jest": {} } diff --git a/evals/test/__fixtures__/workspace/packages/unnamed/package.json b/evals/test/__fixtures__/workspace/packages/unnamed/package.json new file mode 100644 index 0000000000..40d58d6403 --- /dev/null +++ b/evals/test/__fixtures__/workspace/packages/unnamed/package.json @@ -0,0 +1 @@ +{ "version": "0.0.0" } diff --git a/evals/test/changesets.test.ts b/evals/test/changesets.test.ts new file mode 100644 index 0000000000..0ec6b92abf --- /dev/null +++ b/evals/test/changesets.test.ts @@ -0,0 +1,97 @@ +import { equal } from "node:assert/strict"; +import { after, before, describe, it } from "node:test"; +import { changesets } from "../src/changesets.ts"; +import type { AgentChangedFile } from "../src/types.ts"; +import { added, makeRun } from "./helpers.ts"; + +function changeset(name: string, ...packages: string[]): AgentChangedFile { + const releases = packages.map((pkg) => `"${pkg}": patch`); + return added( + `.changeset/${name}.md`, + `---\n${releases.join("\n")}\n---\n\nFix\n` + ); +} + +function check( + files: AgentChangedFile[], + config: Parameters[1]["config"] +) { + return changesets(makeRun({ files }), { config }); +} + +describe("changesets()", () => { + const missingIsError = !("isError" in Error); + + before(() => { + if (missingIsError) { + Error.isError = (e: unknown) => e instanceof Error; + } + }); + + after(() => { + if (missingIsError) { + Error.isError = undefined; + } + }); + + it("passes if the expected number of change files was added", () => { + const result = check([changeset("a", "node-pkg")], { + count: 1, + packages: ["node-pkg"], + }); + equal(result.pass, true); + equal(result.reason, "Found 1 change file(s)"); + }); + + it("passes if no change files were expected", () => { + equal(check([], { count: 0 }).pass, true); + }); + + it("fails if the number of change files differs", () => { + const result = check([changeset("a", "x"), changeset("b", "y")], { + count: 1, + }); + equal(result.pass, false); + equal( + result.reason, + "Expected 1 change file(s), found 2: .changeset/a.md, .changeset/b.md" + ); + }); + + it("only counts added change files", () => { + const files: AgentChangedFile[] = [ + { ...changeset("a", "x"), status: "modified" }, + { path: ".changeset/b.md", status: "deleted" }, + added(".changeset/README.md", ""), + added(".changeset/config.json", "{}"), + added(".changeset/sub/c.md", ""), + added("docs/d.md", ""), + ]; + equal(check(files, { count: 0 }).pass, true); + }); + + it("fails for packages without a change file", () => { + const result = check([changeset("a", "x", "y")], { + count: 1, + packages: ["x", "y", "z"], + }); + equal(result.pass, false); + equal(result.reason, "No change file releases z"); + }); + + it("fails for invalid change files", () => { + const result = check([added(".changeset/a.md", "Fix")], { + count: 1, + packages: ["x"], + }); + equal(result.pass, false); + const [error, missing] = result.reason.split(/\n(?=No change file)/); + equal( + error.startsWith( + ".changeset/a.md: could not parse changeset - missing or invalid frontmatter." + ), + true + ); + equal(missing, "No change file releases x"); + }); +}); diff --git a/evals/test/commands.test.ts b/evals/test/commands.test.ts new file mode 100644 index 0000000000..7d65b777c3 --- /dev/null +++ b/evals/test/commands.test.ts @@ -0,0 +1,254 @@ +import { equal, throws } from "node:assert/strict"; +import { describe, it } from "node:test"; +import { + didNotRunCommand, + notModifiedOnGitHub, + noToolCall, + ranCommand, +} from "../src/commands.ts"; +import type { AgentCommand, CommandMatcher } from "../src/types.ts"; +import { makeRun } from "./helpers.ts"; + +function ran(commands: AgentCommand[], config: CommandMatcher) { + return ranCommand(makeRun({ commands }), { config }); +} + +describe("ranCommand()", () => { + it("passes if a command matches", () => { + const result = ran( + [ + { argv: ["git", "status"], cwd: "." }, + { argv: ["yarn", "build"], cwd: "." }, + ], + { pattern: "^yarn build$" } + ); + equal(result.pass, true); + equal(result.reason, "Ran: yarn build"); + }); + + it("fails if no command matches", () => { + const result = ran([{ argv: ["yarn", "lint"], cwd: "." }], { + pattern: "^yarn build$", + }); + equal(result.pass, false); + equal(result.reason, "No command matched /^yarn build$/"); + }); + + it("accepts JSON output", () => { + const output = JSON.stringify( + makeRun({ commands: [{ argv: ["yarn", "build"], cwd: "." }] }) + ); + equal(ranCommand(output, { config: { pattern: "^yarn" } }).pass, true); + }); + + it("throws without config", () => { + throws(() => ranCommand(makeRun(), {}), /Missing assertion config/); + }); + + it("matches on the basename of the executable", () => { + for (const executable of [ + "/usr/bin/git", + "C:\\Program Files\\Git\\git.EXE", + ]) { + const result = ran([{ argv: [executable, "push"], cwd: "." }], { + pattern: "^git push$", + }); + equal(result.pass, true, executable); + } + }); + + it("strips Windows script extensions", () => { + for (const ext of ["bat", "cmd", "ps1"]) { + const result = ran( + [{ argv: [`C:\\bin\\yarn.${ext}`, "build"], cwd: "." }], + { + pattern: "^yarn build$", + } + ); + equal(result.pass, true, ext); + } + }); + + it("does not match on an empty command", () => { + equal(ran([{ argv: [], cwd: "." }], { pattern: "\\S" }).pass, false); + }); + + it("matches the working directory", () => { + const commands = [{ argv: ["yarn", "build"], cwd: "packages/node-pkg" }]; + equal( + ran(commands, { pattern: "^yarn build", cwd: "^packages/" }).pass, + true + ); + equal(ran(commands, { pattern: "^yarn build", cwd: "^\\.$" }).pass, false); + }); + + it("drops `yarn run`", () => { + const result = ran([{ argv: ["yarn", "run", "build"], cwd: "." }], { + pattern: "^yarn build$", + }); + equal(result.pass, true); + }); + + it("applies `yarn --cwd ` to the working directory", () => { + const config = { pattern: "^yarn build$", cwd: "^packages/node-pkg$" }; + for (const argv of [ + ["yarn", "--cwd", "packages/node-pkg", "build"], + ["yarn", "--cwd=packages/node-pkg", "run", "build"], + ]) { + equal(ran([{ argv, cwd: "." }], config).pass, true, argv.join(" ")); + } + }); + + it("resolves `yarn --cwd` relative to the working directory", () => { + const result = ran( + [ + { + argv: ["yarn", "--cwd", "../node-pkg", "build"], + cwd: "packages/jest-field", + }, + ], + { pattern: "^yarn build$", cwd: "^packages/node-pkg$" } + ); + equal(result.pass, true); + }); + + it("applies `yarn workspace ` to the working directory", () => { + const result = ran( + [{ argv: ["yarn", "workspace", "node-pkg", "run", "build"], cwd: "." }], + { pattern: "^yarn build$", cwd: "^packages/node-pkg$", inPackage: true } + ); + equal(result.pass, true); + }); + + it("leaves `yarn workspace` alone for unknown packages", () => { + const commands = [ + { argv: ["yarn", "workspace", "unknown", "build"], cwd: "." }, + ]; + equal(ran(commands, { pattern: "^yarn build$" }).pass, false); + equal( + ran(commands, { pattern: "^yarn workspace unknown build$" }).pass, + true + ); + }); + + it("only applies Yarn options to Yarn", () => { + const result = ran( + [{ argv: ["npx", "--cwd", "packages/node-pkg", "build"], cwd: "." }], + { + pattern: "^npx --cwd packages/node-pkg build$", + cwd: "^\\.$", + } + ); + equal(result.pass, true); + }); + + it("matches only commands run in a workspace package with `inPackage`", () => { + const config = { pattern: "^yarn build$", inPackage: true }; + for (const cwd of ["packages/node-pkg", "packages/jest-field"]) { + equal(ran([{ argv: ["yarn", "build"], cwd }], config).pass, true, cwd); + } + for (const cwd of [ + ".", + "packages", + "packages/node-pkg/src", + "packages/node-pkg/test/__fixtures__/nested", + "packages/unnamed", + ]) { + equal(ran([{ argv: ["yarn", "build"], cwd }], config).pass, false, cwd); + } + }); +}); + +describe("didNotRunCommand()", () => { + it("passes if no command matches", () => { + const result = didNotRunCommand( + makeRun({ commands: [{ argv: ["yarn", "build"], cwd: "." }] }), + { config: { pattern: "^git push\\b" } } + ); + equal(result.pass, true); + equal(result.reason, "No command matched /^git push\\b/"); + }); + + it("fails listing every matching command", () => { + const result = didNotRunCommand( + makeRun({ + commands: [ + { argv: ["git", "push"], cwd: "." }, + { argv: ["/usr/bin/git", "push", "-f"], cwd: "packages/node-pkg" }, + ], + }), + { config: { pattern: "^git push\\b" } } + ); + equal(result.pass, false); + equal( + result.reason, + "Unexpected command in .: git push\nUnexpected command in packages/node-pkg: git push -f" + ); + }); +}); + +describe("noToolCall()", () => { + const toolCalls = [ + { name: "github-mcp-server-get_issue", arguments: { issue: 1 } }, + { name: "github-mcp-server-add_issue_comment", arguments: { issue: 1 } }, + ]; + + it("passes if no tool call matches", () => { + const result = noToolCall(makeRun({ toolCalls }), { + config: { name: "merge_pull_request" }, + }); + equal(result.pass, true); + }); + + it("fails listing every matching tool call", () => { + const result = noToolCall(makeRun({ toolCalls }), { + config: { name: "_issue" }, + }); + equal(result.pass, false); + equal( + result.reason, + [ + 'Unexpected tool call: github-mcp-server-get_issue {"issue":1}', + 'Unexpected tool call: github-mcp-server-add_issue_comment {"issue":1}', + ].join("\n") + ); + }); +}); + +describe("notModifiedOnGitHub()", () => { + const config = { numbers: [1, 2] }; + + it("passes if none of the numbers were modified", () => { + const result = notModifiedOnGitHub(makeRun({ github: { modified: [3] } }), { + config, + }); + equal(result.pass, true); + equal(result.reason, "Did not modify #1, #2"); + }); + + it("fails for each modified number", () => { + const result = notModifiedOnGitHub( + makeRun({ github: { modified: [2, 1, 3] } }), + { config } + ); + equal(result.pass, false); + equal(result.reason, "Modified #1\nModified #2"); + }); + + it("throws if the GitHub mock state is missing or invalid", () => { + for (const github of [ + undefined, + {}, + { modified: "1" }, + { modified: ["1"] }, + { modified: [1.5] }, + ]) { + throws( + // @ts-expect-error testing invalid input + () => notModifiedOnGitHub(makeRun({ github }), { config }), + /check: github\.modified/, + JSON.stringify(github) + ); + } + }); +}); diff --git a/evals/test/common.test.ts b/evals/test/common.test.ts new file mode 100644 index 0000000000..1b337180f1 --- /dev/null +++ b/evals/test/common.test.ts @@ -0,0 +1,149 @@ +import { deepEqual, equal, throws } from "node:assert/strict"; +import * as path from "node:path"; +import { describe, it } from "node:test"; +import { + addedLines, + existingFiles, + requireConfig, + result, + selectLines, + toAgentRun, + toRelativePath, + tryReadJSON, +} from "../src/common.ts"; +import { WORKSPACE, added, diffAdding, makeRun } from "./helpers.ts"; + +describe("result()", () => { + it("passes when there are no failures", () => { + deepEqual(result([], "ok"), { pass: true, score: 1, reason: "ok" }); + }); + + it("fails with all failures as reason", () => { + deepEqual(result(["a", "b"], "ok"), { + pass: false, + score: 0, + reason: "a\nb", + }); + }); +}); + +describe("toAgentRun()", () => { + it("returns objects as is", () => { + const run = makeRun(); + equal(toAgentRun(run), run); + }); + + it("parses JSON", () => { + deepEqual(toAgentRun(JSON.stringify(makeRun())), makeRun()); + }); + + it("throws if the output is not an object", () => { + throws(() => toAgentRun("null"), /Expected agent run to be an object/); + throws(() => toAgentRun("1"), /Expected agent run to be an object/); + }); + + it("throws listing all invalid fields", () => { + throws(() => toAgentRun(JSON.stringify({ commands: {}, diff: 0 })), { + message: + "Invalid agent run; check: commands, toolCalls, files, diff, finalMessage, workdir", + }); + }); +}); + +describe("addedLines()", () => { + it("returns added line numbers per file", () => { + const file = added("a.ts", "one\ntwo\nthree"); + deepEqual( + addedLines(diffAdding(file, [2])), + new Map([["a.ts", new Set([2])]]) + ); + }); + + it("ignores deleted files", () => { + const diff = [ + "diff --git a/a.ts b/a.ts", + "deleted file mode 100644", + "--- a/a.ts", + "+++ /dev/null", + "@@ -1 +0,0 @@", + "-one", + "", + ].join("\n"); + deepEqual(addedLines(diff), new Map()); + }); + + it("returns an empty map for an empty diff", () => { + deepEqual(addedLines(""), new Map()); + }); +}); + +describe("requireConfig()", () => { + it("returns the config", () => { + deepEqual(requireConfig({ config: { a: 1 } }), { a: 1 }); + }); + + it("throws if config is missing", () => { + throws(() => requireConfig({}), /Missing assertion config/); + }); +}); + +describe("existingFiles()", () => { + it("excludes deleted files", () => { + const files = [ + added("a.ts", ""), + { path: "b.ts", status: "deleted" as const }, + { path: "c.ts", status: "modified" as const, content: "" }, + { path: "d.ts", status: "renamed" as const, content: "" }, + ]; + deepEqual( + existingFiles(makeRun({ files })).map((file) => file.path), + ["a.ts", "c.ts", "d.ts"] + ); + }); +}); + +describe("selectLines()", () => { + const file = added("a.ts", "one\ntwo\nthree"); + + it("returns the whole content without added lines", () => { + equal(selectLines(file), "one\ntwo\nthree"); + }); + + it("returns only added lines", () => { + equal( + selectLines(file, new Map([["a.ts", new Set([1, 3])]])), + "one\nthree" + ); + }); + + it("returns nothing if the file has no added lines", () => { + equal(selectLines(file, new Map()), ""); + }); + + it("treats missing content as empty", () => { + equal(selectLines({ path: "a.ts", status: "deleted" }), ""); + }); +}); + +describe("tryReadJSON()", () => { + it("returns the parsed file", () => { + equal(tryReadJSON(path.join(WORKSPACE, "package.json")).name, "fixture"); + }); + + it("returns undefined if the file cannot be read", () => { + equal(tryReadJSON(path.join(WORKSPACE, "missing.json")), undefined); + }); +}); + +describe("toRelativePath()", () => { + it("returns paths relative to the workdir with forward slashes", () => { + equal( + toRelativePath(WORKSPACE, path.join(WORKSPACE, "packages", "node-pkg")), + "packages/node-pkg" + ); + }); + + it("returns `.` for the workdir itself", () => { + equal(toRelativePath(WORKSPACE, WORKSPACE), "."); + }); +}); diff --git a/evals/test/files.test.ts b/evals/test/files.test.ts new file mode 100644 index 0000000000..f480a0f6af --- /dev/null +++ b/evals/test/files.test.ts @@ -0,0 +1,157 @@ +import { equal } from "node:assert/strict"; +import { describe, it } from "node:test"; +import { matchesContent, matchesFiles } from "../src/files.ts"; +import type { AgentChangedFile } from "../src/types.ts"; +import { added, diffAdding, makeRun } from "./helpers.ts"; + +describe("matchesFiles()", () => { + const files: AgentChangedFile[] = [ + added("packages/node-pkg/src/a.ts", ""), + { path: "packages/node-pkg/src/b.ts", status: "modified", content: "" }, + { path: "packages/node-pkg/src/c.ts", status: "deleted" }, + ]; + + function check( + config: Parameters[1]["config"], + f = files + ) { + return matchesFiles(makeRun({ files: f }), { config }); + } + + it("passes with an empty config", () => { + const result = check({}); + equal(result.pass, true); + equal( + result.reason, + "Changed files: packages/node-pkg/src/a.ts, packages/node-pkg/src/b.ts, packages/node-pkg/src/c.ts" + ); + }); + + it("reports when no files changed", () => { + equal(check({}, []).reason, "Changed files: (none)"); + }); + + it("fails for files not allowed", () => { + const result = check({ allowed: ["/a\\.ts$", "/b\\.ts$"] }); + equal(result.pass, false); + equal(result.reason, "File not allowed: packages/node-pkg/src/c.ts"); + }); + + it("fails for forbidden files", () => { + const result = check({ forbidden: ["/[ab]\\.ts$"] }); + equal(result.pass, false); + equal( + result.reason, + [ + "File forbidden by //[ab]\\.ts$/: packages/node-pkg/src/a.ts", + "File forbidden by //[ab]\\.ts$/: packages/node-pkg/src/b.ts", + ].join("\n") + ); + }); + + it("fails for required files that did not change", () => { + const result = check({ required: ["/a\\.ts$", "/d\\.ts$"] }); + equal(result.pass, false); + equal(result.reason, "No file matched //d\\.ts$/"); + }); + + it("only considers files with the configured statuses", () => { + equal( + check({ status: ["deleted"], forbidden: ["/[ab]\\.ts$"] }).pass, + true + ); + equal(check({ status: ["added"], required: ["/b\\.ts$"] }).pass, false); + }); + + describe("with `singlePackage`", () => { + it("passes if files of one package changed", () => { + const result = check({ singlePackage: true }, [ + added("README.md", ""), + added("packages/node-pkg/src/a.ts", ""), + // Fixture manifests do not make a separate package + added("packages/node-pkg/test/__fixtures__/nested/index.js", ""), + ]); + equal(result.pass, true); + }); + + it("fails if files of multiple packages changed", () => { + const result = check({ singlePackage: true }, [ + added("packages/node-pkg/src/a.ts", ""), + added("packages/jest-field/src/a.ts", ""), + ]); + equal(result.pass, false); + equal( + result.reason, + "Changed multiple packages: packages/node-pkg, packages/jest-field" + ); + }); + }); +}); + +describe("matchesContent()", () => { + const a = added("src/a.ts", 'import { a } from "a";\nconst x = 1;'); + const b = added("src/b.ts", 'import { b } from "b";'); + + function check( + config: Parameters[1]["config"], + files = [a, b], + diff = "" + ) { + return matchesContent(makeRun({ files, diff }), { config }); + } + + it("fails if no changed file matches", () => { + const result = check({ files: "\\.md$" }); + equal(result.pass, false); + equal(result.reason, "No changed file matched /\\.md$/"); + }); + + it("ignores deleted files", () => { + const result = check({ files: "\\.ts$" }, [ + { path: "src/c.ts", status: "deleted" }, + ]); + equal(result.pass, false); + }); + + it("requires patterns in every file by default", () => { + const result = check({ files: "\\.ts$", required: ["^import"] }); + equal(result.pass, true); + equal(result.reason, "Checked src/a.ts, src/b.ts"); + + const failed = check({ files: "\\.ts$", required: ["^const"] }); + equal(failed.pass, false); + equal(failed.reason, "src/b.ts: missing /^const/"); + }); + + it("requires patterns in any file with `match: any`", () => { + equal( + check({ files: "\\.ts$", required: ["^const"], match: "any" }).pass, + true + ); + + const failed = check({ files: "\\.ts$", required: ["^let"], match: "any" }); + equal(failed.pass, false); + equal(failed.reason, "No matching file contains /^let/"); + }); + + it("fails for forbidden patterns", () => { + const result = check({ files: "\\.ts$", forbidden: ['from "[ab]"'] }); + equal(result.pass, false); + equal( + result.reason, + 'src/a.ts: contains /from "[ab]"/\nsrc/b.ts: contains /from "[ab]"/' + ); + }); + + it("only checks added lines with `addedLinesOnly`", () => { + const config = { + files: "\\.ts$", + forbidden: ["^import"], + addedLinesOnly: true, + }; + equal(check(config, [a], diffAdding(a, [2])).pass, true); + equal(check(config, [a], diffAdding(a, [1])).pass, false); + // Files without added lines are still checked, but are empty + equal(check({ ...config, required: ["\\S"] }, [a], "").pass, false); + }); +}); diff --git a/evals/test/helpers.ts b/evals/test/helpers.ts new file mode 100644 index 0000000000..d49e84a105 --- /dev/null +++ b/evals/test/helpers.ts @@ -0,0 +1,44 @@ +import { fileURLToPath } from "node:url"; +import type { AgentChangedFile, AgentRun } from "../src/types.ts"; + +export const WORKSPACE = fileURLToPath( + new URL("__fixtures__/workspace", import.meta.url) +); + +export function makeRun(run: Partial = {}): AgentRun { + return { + commands: [], + toolCalls: [], + files: [], + diff: "", + finalMessage: "", + workdir: WORKSPACE, + ...run, + }; +} + +export function added(path: string, content: string): AgentChangedFile { + return { path, status: "added", content }; +} + +/** + * Returns a unified diff that adds the specified lines (1-based) of the file. + */ +export function diffAdding( + { path, content = "" }: AgentChangedFile, + lines?: number[] +): string { + const all = content.split("\n"); + const body = all.map((line, i) => + !lines || lines.includes(i + 1) ? `+${line}` : ` ${line}` + ); + const oldCount = body.filter((line) => line.startsWith(" ")).length; + return [ + `diff --git a/${path} b/${path}`, + `--- a/${path}`, + `+++ b/${path}`, + `@@ -1,${oldCount} +1,${all.length} @@`, + ...body, + "", + ].join("\n"); +} diff --git a/evals/test/lint.test.ts b/evals/test/lint.test.ts new file mode 100644 index 0000000000..119922a52e --- /dev/null +++ b/evals/test/lint.test.ts @@ -0,0 +1,147 @@ +import { equal, throws } from "node:assert/strict"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import { describe, it } from "node:test"; +import { noInlineTypeSpecifiers, passesOxlint } from "../src/lint.ts"; +import type { AgentChangedFile } from "../src/types.ts"; +import { WORKSPACE, added, diffAdding, makeRun } from "./helpers.ts"; + +function fixture(file: string): AgentChangedFile { + const content = fs.readFileSync(path.join(WORKSPACE, file), "utf-8"); + return added(file, content); +} + +describe("passesOxlint()", () => { + const index = fixture("packages/node-pkg/src/index.ts"); + const types = fixture("packages/node-pkg/src/types.ts"); + + it("passes if no files match", () => { + const result = passesOxlint(makeRun({ files: [index] }), { + config: { files: "\\.md$", config: "oxlint/types-only.json" }, + }); + equal(result.pass, true); + equal(result.reason, "No matching files changed"); + }); + + it("passes if matching files pass", () => { + const result = passesOxlint(makeRun({ files: [types] }), { + config: { files: "/types\\.ts$", config: "oxlint/types-only.json" }, + }); + equal(result.pass, true); + equal(result.reason, "packages/node-pkg/src/types.ts passed oxlint"); + }); + + it("fails for each diagnostic", () => { + const result = passesOxlint(makeRun({ files: [index, types] }), { + config: { + files: "\\.ts$", + config: "../../test/__fixtures__/oxlint-no-plusplus.json", + }, + }); + equal(result.pass, false); + equal( + result.reason, + "packages/node-pkg/src/index.ts:3: eslint(no-plusplus): Unary operator '++' used." + ); + }); + + it("throws if oxlint cannot lint the files", () => { + const missing = added("packages/node-pkg/src/missing.ts", ""); + throws(() => + passesOxlint(makeRun({ files: [missing] }), { + config: { files: "\\.ts$", config: "oxlint/types-only.json" }, + }) + ); + }); + + it("throws if the config is invalid", () => { + throws(() => + passesOxlint(makeRun({ files: [types] }), { + config: { files: "\\.ts$", config: "oxlint/missing.json" }, + }) + ); + }); +}); + +describe("noInlineTypeSpecifiers()", () => { + function check( + files: AgentChangedFile[], + config: Partial< + Parameters[1]["config"] + > = {}, + diff = "" + ) { + return noInlineTypeSpecifiers(makeRun({ files, diff }), { + config: { files: "\\.ts$", ...config }, + }); + } + + it("passes for separate type imports and exports", () => { + const file = added( + "a.ts", + [ + 'import type { A } from "a";', + 'import { b } from "b";', + 'export type { C } from "c";', + "export { b };", + ].join("\n") + ); + const result = check([file]); + equal(result.pass, true); + equal(result.reason, "Checked a.ts"); + }); + + it("fails for inline type specifiers in imports and exports", () => { + const file = added( + "a.ts", + [ + 'import { type A, b } from "a";', + 'export { type C, d } from "c";', + "type E = string;", + "export { type E };", + ].join("\n") + ); + const result = check([file]); + equal(result.pass, false); + equal( + result.reason, + [ + "a.ts:1: Use a separate `import type` statement", + "a.ts:2: Use a separate `export type` statement", + "a.ts:4: Use a separate `export type` statement", + ].join("\n") + ); + }); + + it("only checks matching, existing files", () => { + const result = check([ + added("a.js", 'import { type A } from "a";'), + { path: "b.ts", status: "deleted" }, + ]); + equal(result.pass, true); + equal(result.reason, "Checked (none)"); + }); + + it("reports parse errors", () => { + const result = check([added("a.ts", "import {")]); + equal(result.pass, false); + equal(result.reason.startsWith("a.ts: "), true); + }); + + it("only checks added lines with `addedLinesOnly`", () => { + const file = added( + "a.ts", + 'import { type A } from "a";\nimport { type B } from "b";' + ); + const config = { addedLinesOnly: true }; + + const result = check([file], config, diffAdding(file, [2])); + equal(result.pass, false); + equal(result.reason, "a.ts:2: Use a separate `import type` statement"); + + const other = added("b.ts", ""); + const skipped = check([file, other], config, diffAdding(other)); + equal(skipped.pass, true); + equal(skipped.reason, "Checked b.ts"); + }); +}); diff --git a/evals/test/tests.test.ts b/evals/test/tests.test.ts new file mode 100644 index 0000000000..14cdf1344d --- /dev/null +++ b/evals/test/tests.test.ts @@ -0,0 +1,151 @@ +import { equal } from "node:assert/strict"; +import { describe, it } from "node:test"; +import { usesPackageTestRunner } from "../src/tests.ts"; +import type { AgentChangedFile } from "../src/types.ts"; +import { added, diffAdding, makeRun } from "./helpers.ts"; + +const NODE_TEST = [ + 'import { equal } from "node:assert/strict";', + 'import { it } from "node:test";', + "", + 'it("works", () => equal(1, 1));', +].join("\n"); + +const JEST_TEST = 'it("works", () => expect(1).toBe(1));'; + +function check(files: AgentChangedFile[], diff?: string) { + return usesPackageTestRunner( + makeRun({ files, diff: diff ?? files.map((f) => diffAdding(f)).join("") }) + ); +} + +describe("usesPackageTestRunner()", () => { + it("fails if no test files changed", () => { + const result = check([ + added("packages/node-pkg/src/index.ts", ""), + added("packages/node-pkg/test/helpers.ts", ""), + { path: "packages/node-pkg/test/a.test.ts", status: "deleted" }, + ]); + equal(result.pass, false); + equal(result.reason, "No test files changed"); + }); + + it("recognizes test files", () => { + for (const path of [ + "test/a.test.ts", + "packages/node-pkg/test/a.test.js", + "packages/node-pkg/test/sub/a.test.mts", + "packages/node-pkg/test/a.test.cjs", + "packages/node-pkg/test/a.test.tsx", + ]) { + equal(check([added(path, JEST_TEST)]).pass, false, path); + } + }); + + describe("in Node.js test runner packages", () => { + it("passes for tests using node:test and node:assert/strict", () => { + const result = check([ + added("packages/node-pkg/test/a.test.ts", NODE_TEST), + ]); + equal(result.pass, true); + equal(result.reason, "Checked packages/node-pkg/test/a.test.ts"); + }); + + it("fails if imports are missing", () => { + const content = 'it("works", () => {});'; + const result = check([ + added("packages/node-pkg/test/a.test.ts", content), + ]); + equal(result.pass, false); + equal( + result.reason, + "packages/node-pkg/test/a.test.ts: missing imports from 'node:test' and 'node:assert/strict'" + ); + }); + + it("fails for added Jest APIs", () => { + for (const api of [ + "expect(1).toBe(1);", + "jest.fn();", + 'import { jest } from "@jest/globals";', + ]) { + const file = added( + "packages/node-pkg/test/a.test.ts", + `${NODE_TEST}\n${api}` + ); + const result = check([file]); + equal(result.pass, false, api); + equal( + result.reason, + "packages/node-pkg/test/a.test.ts: uses Jest in a Node.js test runner package" + ); + } + }); + + it("ignores Jest APIs in unchanged lines", () => { + const file: AgentChangedFile = { + path: "packages/node-pkg/test/a.test.ts", + status: "modified", + content: `${NODE_TEST}\nexpect(1);`, + }; + equal(check([file], diffAdding(file, [4])).pass, true); + }); + + it("uses the owning workspace package rather than fixture manifests", () => { + const path = "packages/node-pkg/test/__fixtures__/nested/test/a.test.ts"; + equal(check([added(path, NODE_TEST)]).pass, true); + equal(check([added(path, JEST_TEST)]).pass, false); + }); + + it("treats files outside packages as belonging to the root", () => { + equal(check([added("test/a.test.ts", NODE_TEST)]).pass, true); + }); + }); + + describe("in Jest packages", () => { + it("passes for Jest tests", () => { + for (const pkg of ["jest-field", "jest-config"]) { + const result = check([ + added(`packages/${pkg}/test/a.test.ts`, JEST_TEST), + ]); + equal(result.pass, true, pkg); + } + }); + + it("fails for added 'node:test' or 'node:assert' imports", () => { + const result = check([ + added("packages/jest-field/test/a.test.ts", NODE_TEST), + ]); + equal(result.pass, false); + equal( + result.reason, + "packages/jest-field/test/a.test.ts: uses 'node:test' in a Jest package" + ); + }); + + it("ignores node:test imports in unchanged lines", () => { + const file: AgentChangedFile = { + path: "packages/jest-config/test/a.test.ts", + status: "modified", + content: NODE_TEST, + }; + equal(check([file], diffAdding(file, [4])).pass, true); + }); + }); + + it("reports failures for every file", () => { + const result = check([ + added("packages/node-pkg/test/a.test.ts", JEST_TEST), + added("packages/jest-field/test/a.test.ts", NODE_TEST), + ]); + equal(result.pass, false); + equal( + result.reason, + [ + "packages/node-pkg/test/a.test.ts: missing imports from 'node:test' and 'node:assert/strict'", + "packages/node-pkg/test/a.test.ts: uses Jest in a Node.js test runner package", + "packages/jest-field/test/a.test.ts: uses 'node:test' in a Jest package", + ].join("\n") + ); + }); +}); diff --git a/evals/test/transforms.test.ts b/evals/test/transforms.test.ts new file mode 100644 index 0000000000..1bd575e869 --- /dev/null +++ b/evals/test/transforms.test.ts @@ -0,0 +1,88 @@ +import { equal } from "node:assert/strict"; +import { describe, it } from "node:test"; +import { forGrader } from "../src/transforms.ts"; +import { added, diffAdding, makeRun } from "./helpers.ts"; + +describe("forGrader()", () => { + it("formats an empty run", () => { + equal( + forGrader(makeRun({ finalMessage: "Done" })), + [ + "## Commands", + "(none)", + "## Tool calls", + "(none)", + "## Diff", + "(none)", + "## Final message", + "Done", + ].join("\n\n") + ); + }); + + it("formats commands, tool calls and diff", () => { + const diff = diffAdding(added("a.ts", "one")); + const output = forGrader( + JSON.stringify( + makeRun({ + commands: [ + { argv: ["yarn", "test", "test/a b.test.ts"], cwd: "packages/a" }, + { argv: ["git", "status"], cwd: "." }, + ], + toolCalls: [{ name: "view", arguments: { path: "a.ts" } }], + diff, + finalMessage: "Done", + }) + ) + ); + equal( + output, + [ + "## Commands", + "[packages/a] $ yarn test 'test/a b.test.ts'\n[.] $ git status", + "## Tool calls", + 'view {"path":"a.ts"}', + "## Diff", + diff, + "## Final message", + "Done", + ].join("\n\n") + ); + }); + + it("summarizes lockfile changes", () => { + const source = diffAdding(added("a.ts", "one")); + const lockfiles = [ + "yarn.lock", + "packages/a/package-lock.json", + "pnpm-lock.yaml", + ] + .map((path) => diffAdding(added(path, "one\ntwo"))) + .join(""); + const removed = [ + "diff --git a/yarn.lock b/yarn.lock", + "deleted file mode 100644", + "--- a/yarn.lock", + "+++ /dev/null", + "@@ -1 +0,0 @@", + "-one", + "", + ].join("\n"); + + const output = forGrader(makeRun({ diff: source + lockfiles + removed })); + const diffSection = output + .split("\n\n## Diff\n\n")[1] + .split("\n\n## Final message")[0]; + equal( + diffSection, + source + + [ + "(yarn.lock: 2 lines added, 0 lines removed; omitted)", + "(packages/a/package-lock.json: 2 lines added, 0 lines removed; omitted)", + "(pnpm-lock.yaml: 2 lines added, 0 lines removed; omitted)", + "(yarn.lock: 0 lines added, 1 lines removed; omitted)", + "", + ].join("\n") + ); + }); +}); diff --git a/evals/test/workspace.test.ts b/evals/test/workspace.test.ts new file mode 100644 index 0000000000..5c2d3b6936 --- /dev/null +++ b/evals/test/workspace.test.ts @@ -0,0 +1,57 @@ +import { deepEqual, equal } from "node:assert/strict"; +import * as path from "node:path"; +import { describe, it } from "node:test"; +import { + lazyWorkspacePackages, + owningPackageDir, + workspacePackages, +} from "../src/workspace.ts"; +import { WORKSPACE } from "./helpers.ts"; + +describe("workspacePackages()", () => { + it("returns named packages matching `workspaces`", () => { + deepEqual( + workspacePackages(WORKSPACE), + new Map([ + ["jest-config", "packages/jest-config"], + ["jest-field", "packages/jest-field"], + ["node-pkg", "packages/node-pkg"], + ]) + ); + }); + + it("supports `workspaces.packages`", () => { + const workdir = path.join(WORKSPACE, "..", "workspace-object"); + deepEqual(workspacePackages(workdir), new Map([["a", "packages/a"]])); + }); + + it("returns an empty map without a root manifest", () => { + deepEqual(workspacePackages(path.join(WORKSPACE, "packages")), new Map()); + }); +}); + +describe("lazyWorkspacePackages()", () => { + it("only looks up packages once", () => { + const packages = lazyWorkspacePackages(WORKSPACE); + equal(packages(), packages()); + equal(packages().get("node-pkg"), "packages/node-pkg"); + }); +}); + +describe("owningPackageDir()", () => { + const dirs = ["packages/a", "packages/a/b", "packages/c"]; + + it("returns the innermost package containing the file", () => { + equal(owningPackageDir(dirs, "packages/a/src/index.ts"), "packages/a"); + equal(owningPackageDir(dirs, "packages/a/b/src/index.ts"), "packages/a/b"); + }); + + it("does not match directories sharing a prefix", () => { + equal(owningPackageDir(dirs, "packages/c-2/index.ts"), "."); + }); + + it("returns `.` for files outside packages", () => { + equal(owningPackageDir(dirs, "README.md"), "."); + equal(owningPackageDir(dirs, "packages/a"), "."); + }); +}); diff --git a/knip.config.ts b/knip.config.ts index d7b7db4215..a8f0169ce4 100644 --- a/knip.config.ts +++ b/knip.config.ts @@ -40,7 +40,7 @@ export default { ignoreDependencies: ["eslint"], }, evals: { - entry: ["src/index.ts", "src/transforms.ts"], + entry: ["src/index.ts", "src/transforms.ts", allFixtures, allTests], // Loaded as a jsPlugin by `oxlint/types-only.json` ignoreDependencies: ["@rnx-kit/eslint-plugin"], }, From 70fd316fa735beee131e186cb99d3be26c5b8e91 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 13 Oct 2026 13:20:15 +0000 Subject: [PATCH 03/37] evals: add @github/copilot-sdk for the agent harness Co-authored-by: tido64 <4123478+tido64@users.noreply.github.com> --- evals/package.json | 1 + yarn.lock | 296 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 297 insertions(+) diff --git a/evals/package.json b/evals/package.json index 95b0614ed0..2a4452b40f 100644 --- a/evals/package.json +++ b/evals/package.json @@ -24,6 +24,7 @@ }, "dependencies": { "@changesets/parse": "^1.0.0", + "@github/copilot-sdk": "^1.0.16", "@rnx-kit/eslint-plugin": "^0.9.8", "@rnx-kit/tools-filesystem": "^0.2.0", "oxc-parser": "^0.123.0", diff --git a/yarn.lock b/yarn.lock index a8f4bf7783..8e26b70355 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2165,6 +2165,98 @@ __metadata: languageName: node linkType: hard +"@github/copilot-sdk-darwin-arm64@npm:1.0.16": + version: 1.0.16 + resolution: "@github/copilot-sdk-darwin-arm64@npm:1.0.16" + conditions: os=darwin & cpu=arm64 + languageName: node + linkType: hard + +"@github/copilot-sdk-darwin-x64@npm:1.0.16": + version: 1.0.16 + resolution: "@github/copilot-sdk-darwin-x64@npm:1.0.16" + conditions: os=darwin & cpu=x64 + languageName: node + linkType: hard + +"@github/copilot-sdk-linux-arm64@npm:1.0.16": + version: 1.0.16 + resolution: "@github/copilot-sdk-linux-arm64@npm:1.0.16" + conditions: os=linux & cpu=arm64 & libc=glibc + languageName: node + linkType: hard + +"@github/copilot-sdk-linux-x64@npm:1.0.16": + version: 1.0.16 + resolution: "@github/copilot-sdk-linux-x64@npm:1.0.16" + conditions: os=linux & cpu=x64 & libc=glibc + languageName: node + linkType: hard + +"@github/copilot-sdk-linuxmusl-arm64@npm:1.0.16": + version: 1.0.16 + resolution: "@github/copilot-sdk-linuxmusl-arm64@npm:1.0.16" + conditions: os=linux & cpu=arm64 & libc=musl + languageName: node + linkType: hard + +"@github/copilot-sdk-linuxmusl-x64@npm:1.0.16": + version: 1.0.16 + resolution: "@github/copilot-sdk-linuxmusl-x64@npm:1.0.16" + conditions: os=linux & cpu=x64 & libc=musl + languageName: node + linkType: hard + +"@github/copilot-sdk-win32-arm64@npm:1.0.16": + version: 1.0.16 + resolution: "@github/copilot-sdk-win32-arm64@npm:1.0.16" + conditions: os=win32 & cpu=arm64 + languageName: node + linkType: hard + +"@github/copilot-sdk-win32-x64@npm:1.0.16": + version: 1.0.16 + resolution: "@github/copilot-sdk-win32-x64@npm:1.0.16" + conditions: os=win32 & cpu=x64 + languageName: node + linkType: hard + +"@github/copilot-sdk@npm:^1.0.16": + version: 1.0.16 + resolution: "@github/copilot-sdk@npm:1.0.16" + dependencies: + "@github/copilot-sdk-darwin-arm64": "npm:1.0.16" + "@github/copilot-sdk-darwin-x64": "npm:1.0.16" + "@github/copilot-sdk-linux-arm64": "npm:1.0.16" + "@github/copilot-sdk-linux-x64": "npm:1.0.16" + "@github/copilot-sdk-linuxmusl-arm64": "npm:1.0.16" + "@github/copilot-sdk-linuxmusl-x64": "npm:1.0.16" + "@github/copilot-sdk-win32-arm64": "npm:1.0.16" + "@github/copilot-sdk-win32-x64": "npm:1.0.16" + koffi: "npm:3.2.1" + vscode-jsonrpc: "npm:8.2.1" + zod: "npm:4.3.6" + dependenciesMeta: + "@github/copilot-sdk-darwin-arm64": + optional: true + "@github/copilot-sdk-darwin-x64": + optional: true + "@github/copilot-sdk-linux-arm64": + optional: true + "@github/copilot-sdk-linux-x64": + optional: true + "@github/copilot-sdk-linuxmusl-arm64": + optional: true + "@github/copilot-sdk-linuxmusl-x64": + optional: true + "@github/copilot-sdk-win32-arm64": + optional: true + "@github/copilot-sdk-win32-x64": + optional: true + checksum: 10c0/84096d23e8180c1bc856167d4851b8bdbcb735981918d3b68ec8f01195589b2d4a737918ef0b52bdf34e89c501141610410996ba2ac2209f34011564a5f78af6 + languageName: node + linkType: hard + "@hapi/hoek@npm:^9.0.0, @hapi/hoek@npm:^9.3.0": version: 9.3.0 resolution: "@hapi/hoek@npm:9.3.0" @@ -2805,6 +2897,132 @@ __metadata: languageName: node linkType: hard +"@koromix/koffi-android-arm64@npm:3.2.1": + version: 3.2.1 + resolution: "@koromix/koffi-android-arm64@npm:3.2.1" + conditions: os=android & cpu=arm64 + languageName: node + linkType: hard + +"@koromix/koffi-android-x64@npm:3.2.1": + version: 3.2.1 + resolution: "@koromix/koffi-android-x64@npm:3.2.1" + conditions: os=android & cpu=x64 + languageName: node + linkType: hard + +"@koromix/koffi-darwin-arm64@npm:3.2.1": + version: 3.2.1 + resolution: "@koromix/koffi-darwin-arm64@npm:3.2.1" + conditions: os=darwin & cpu=arm64 + languageName: node + linkType: hard + +"@koromix/koffi-darwin-x64@npm:3.2.1": + version: 3.2.1 + resolution: "@koromix/koffi-darwin-x64@npm:3.2.1" + conditions: os=darwin & cpu=x64 + languageName: node + linkType: hard + +"@koromix/koffi-freebsd-arm64@npm:3.2.1": + version: 3.2.1 + resolution: "@koromix/koffi-freebsd-arm64@npm:3.2.1" + conditions: os=freebsd & cpu=arm64 + languageName: node + linkType: hard + +"@koromix/koffi-freebsd-ia32@npm:3.2.1": + version: 3.2.1 + resolution: "@koromix/koffi-freebsd-ia32@npm:3.2.1" + conditions: os=freebsd & cpu=ia32 + languageName: node + linkType: hard + +"@koromix/koffi-freebsd-x64@npm:3.2.1": + version: 3.2.1 + resolution: "@koromix/koffi-freebsd-x64@npm:3.2.1" + conditions: os=freebsd & cpu=x64 + languageName: node + linkType: hard + +"@koromix/koffi-linux-arm64@npm:3.2.1": + version: 3.2.1 + resolution: "@koromix/koffi-linux-arm64@npm:3.2.1" + conditions: os=linux & cpu=arm64 + languageName: node + linkType: hard + +"@koromix/koffi-linux-arm@npm:3.2.1": + version: 3.2.1 + resolution: "@koromix/koffi-linux-arm@npm:3.2.1" + conditions: os=linux & cpu=arm + languageName: node + linkType: hard + +"@koromix/koffi-linux-ia32@npm:3.2.1": + version: 3.2.1 + resolution: "@koromix/koffi-linux-ia32@npm:3.2.1" + conditions: os=linux & cpu=ia32 + languageName: node + linkType: hard + +"@koromix/koffi-linux-loong64@npm:3.2.1": + version: 3.2.1 + resolution: "@koromix/koffi-linux-loong64@npm:3.2.1" + conditions: os=linux & cpu=loong64 + languageName: node + linkType: hard + +"@koromix/koffi-linux-riscv64@npm:3.2.1": + version: 3.2.1 + resolution: "@koromix/koffi-linux-riscv64@npm:3.2.1" + conditions: os=linux & cpu=riscv64 + languageName: node + linkType: hard + +"@koromix/koffi-linux-x64@npm:3.2.1": + version: 3.2.1 + resolution: "@koromix/koffi-linux-x64@npm:3.2.1" + conditions: os=linux & cpu=x64 + languageName: node + linkType: hard + +"@koromix/koffi-openbsd-ia32@npm:3.2.1": + version: 3.2.1 + resolution: "@koromix/koffi-openbsd-ia32@npm:3.2.1" + conditions: os=openbsd & cpu=ia32 + languageName: node + linkType: hard + +"@koromix/koffi-openbsd-x64@npm:3.2.1": + version: 3.2.1 + resolution: "@koromix/koffi-openbsd-x64@npm:3.2.1" + conditions: os=openbsd & cpu=x64 + languageName: node + linkType: hard + +"@koromix/koffi-win32-arm64@npm:3.2.1": + version: 3.2.1 + resolution: "@koromix/koffi-win32-arm64@npm:3.2.1" + conditions: os=win32 & cpu=arm64 + languageName: node + linkType: hard + +"@koromix/koffi-win32-ia32@npm:3.2.1": + version: 3.2.1 + resolution: "@koromix/koffi-win32-ia32@npm:3.2.1" + conditions: os=win32 & cpu=ia32 + languageName: node + linkType: hard + +"@koromix/koffi-win32-x64@npm:3.2.1": + version: 3.2.1 + resolution: "@koromix/koffi-win32-x64@npm:3.2.1" + conditions: os=win32 & cpu=x64 + languageName: node + linkType: hard + "@manypkg/find-root@npm:^3.1.0": version: 3.1.0 resolution: "@manypkg/find-root@npm:3.1.0" @@ -5227,6 +5445,7 @@ __metadata: resolution: "@rnx-kit/evals@workspace:evals" dependencies: "@changesets/parse": "npm:^1.0.0" + "@github/copilot-sdk": "npm:^1.0.16" "@rnx-kit/eslint-plugin": "npm:^0.9.8" "@rnx-kit/scripts": "npm:*" "@rnx-kit/tools-filesystem": "npm:^0.2.0" @@ -12531,6 +12750,69 @@ __metadata: languageName: node linkType: hard +"koffi@npm:3.2.1": + version: 3.2.1 + resolution: "koffi@npm:3.2.1" + dependencies: + "@koromix/koffi-android-arm64": "npm:3.2.1" + "@koromix/koffi-android-x64": "npm:3.2.1" + "@koromix/koffi-darwin-arm64": "npm:3.2.1" + "@koromix/koffi-darwin-x64": "npm:3.2.1" + "@koromix/koffi-freebsd-arm64": "npm:3.2.1" + "@koromix/koffi-freebsd-ia32": "npm:3.2.1" + "@koromix/koffi-freebsd-x64": "npm:3.2.1" + "@koromix/koffi-linux-arm": "npm:3.2.1" + "@koromix/koffi-linux-arm64": "npm:3.2.1" + "@koromix/koffi-linux-ia32": "npm:3.2.1" + "@koromix/koffi-linux-loong64": "npm:3.2.1" + "@koromix/koffi-linux-riscv64": "npm:3.2.1" + "@koromix/koffi-linux-x64": "npm:3.2.1" + "@koromix/koffi-openbsd-ia32": "npm:3.2.1" + "@koromix/koffi-openbsd-x64": "npm:3.2.1" + "@koromix/koffi-win32-arm64": "npm:3.2.1" + "@koromix/koffi-win32-ia32": "npm:3.2.1" + "@koromix/koffi-win32-x64": "npm:3.2.1" + dependenciesMeta: + "@koromix/koffi-android-arm64": + optional: true + "@koromix/koffi-android-x64": + optional: true + "@koromix/koffi-darwin-arm64": + optional: true + "@koromix/koffi-darwin-x64": + optional: true + "@koromix/koffi-freebsd-arm64": + optional: true + "@koromix/koffi-freebsd-ia32": + optional: true + "@koromix/koffi-freebsd-x64": + optional: true + "@koromix/koffi-linux-arm": + optional: true + "@koromix/koffi-linux-arm64": + optional: true + "@koromix/koffi-linux-ia32": + optional: true + "@koromix/koffi-linux-loong64": + optional: true + "@koromix/koffi-linux-riscv64": + optional: true + "@koromix/koffi-linux-x64": + optional: true + "@koromix/koffi-openbsd-ia32": + optional: true + "@koromix/koffi-openbsd-x64": + optional: true + "@koromix/koffi-win32-arm64": + optional: true + "@koromix/koffi-win32-ia32": + optional: true + "@koromix/koffi-win32-x64": + optional: true + checksum: 10c0/9a592e894a3036342beda31a4644c43a92c0a57f7c048cace38fab29fee98bb3b4ffbaede06fd933420cc80c9840cee4b99b95525d60bf69724cadfa035597b1 + languageName: node + linkType: hard + "ky@npm:^1.2.0": version: 1.2.3 resolution: "ky@npm:1.2.3" @@ -17429,6 +17711,13 @@ __metadata: languageName: node linkType: hard +"vscode-jsonrpc@npm:8.2.1": + version: 8.2.1 + resolution: "vscode-jsonrpc@npm:8.2.1" + checksum: 10c0/595e07f779112d979d9cd37a00e4b973c39da09dd7c35b149f9fb7857c87db00d0ece7772c5a6f65cc19e38afc4fc64f8130d2f0e6b5f7fc160a0fc6b94e8c07 + languageName: node + linkType: hard + "walk-up-path@npm:^4.0.0": version: 4.0.0 resolution: "walk-up-path@npm:4.0.0" @@ -17836,6 +18125,13 @@ __metadata: languageName: node linkType: hard +"zod@npm:4.3.6": + version: 4.3.6 + resolution: "zod@npm:4.3.6" + checksum: 10c0/860d25a81ab41d33aa25f8d0d07b091a04acb426e605f396227a796e9e800c44723ed96d0f53a512b57be3d1520f45bf69c0cb3b378a232a00787a2609625307 + languageName: node + linkType: hard + "zod@npm:^4.1.11": version: 4.6.7 resolution: "zod@npm:4.6.7" From d0deea0534bdcd35f2ac0cbaad492d4529045784 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 13 Oct 2026 13:36:14 +0000 Subject: [PATCH 04/37] evals: add Copilot SDK agent harness (prototype) Co-authored-by: tido64 <4123478+tido64@users.noreply.github.com> --- evals/promptfooconfig.yaml | 17 +- evals/src/harness/checkout.ts | 246 ++++++++++++ evals/src/harness/github.ts | 718 ++++++++++++++++++++++++++++++++++ evals/src/harness/hooks.ts | 29 ++ evals/src/harness/provider.ts | 309 +++++++++++++++ evals/src/harness/shell.ts | 126 ++++++ evals/src/harness/shim.ts | 145 +++++++ evals/src/types.ts | 5 +- knip.config.ts | 9 +- 9 files changed, 1598 insertions(+), 6 deletions(-) create mode 100644 evals/src/harness/checkout.ts create mode 100644 evals/src/harness/github.ts create mode 100644 evals/src/harness/hooks.ts create mode 100644 evals/src/harness/provider.ts create mode 100644 evals/src/harness/shell.ts create mode 100644 evals/src/harness/shim.ts diff --git a/evals/promptfooconfig.yaml b/evals/promptfooconfig.yaml index 584a57456b..a2d5cb6469 100644 --- a/evals/promptfooconfig.yaml +++ b/evals/promptfooconfig.yaml @@ -2,7 +2,18 @@ description: rnx-kit agent evals prompts: - "{{task}}" -# The agent harness (provider) and the `llm-rubric` grader are not configured -# yet. Until then, pass them on the command line, e.g. -# `yarn eval --providers file://path/to/provider.ts`. +providers: + # Requires a GitHub token with Copilot access in `COPILOT_GITHUB_TOKEN`, + # `GH_TOKEN` or `GITHUB_TOKEN`, or a `gh auth login` session. Set + # `config.model` to choose the model. + - id: file://src/harness/provider.ts + label: copilot +extensions: + - file://src/harness/hooks.ts:afterEach + - file://src/harness/hooks.ts:afterAll +evaluateOptions: + maxConcurrency: 4 + timeoutMs: 3600000 +# The `llm-rubric` grader is not configured yet. promptfoo picks a default from +# the environment (e.g. `OPENAI_API_KEY`); otherwise pass `--grader`. tests: file://tests/*.yaml diff --git a/evals/src/harness/checkout.ts b/evals/src/harness/checkout.ts new file mode 100644 index 0000000000..1f76cfa428 --- /dev/null +++ b/evals/src/harness/checkout.ts @@ -0,0 +1,246 @@ +import { execFile, spawn } from "node:child_process"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import type { AgentChangedFile } from "../types.ts"; + +const execFileAsync = promisify(execFile); + +/** Fetch URL of the agent's checkout; pushing is disabled */ +const UPSTREAM_URL = "https://github.com/microsoft/rnx-kit.git"; + +/** Files generated by `yarn install` that are not under `node_modules` */ +const INSTALL_ARTIFACTS = [".yarn/install-state.gz", "rnx-kit-workspaces.json"]; + +const READY_MARKER = ".rnx-kit-evals.json"; + +/** + * Root of everything the harness creates. Runs are scoped to this process so + * that cleanup does not remove runs of a concurrent eval. + */ +export const HARNESS_ROOT = path.join(os.tmpdir(), "rnx-kit-evals"); +export const RUNS_ROOT = path.join(HARNESS_ROOT, `runs-${process.pid}`); + +/** Yarn cache shared by templates and runs */ +export const YARN_CACHE = path.join(HARNESS_ROOT, "yarn-cache"); + +export type Run = { + /** Directory containing the checkout, shims, logs and the agent's home */ + dir: string; + /** The agent's checkout */ + workdir: string; +}; + +const templates = new Map>(); + +function repositoryRoot(): string { + return path.resolve(fileURLToPath(new URL("../../..", import.meta.url))); +} + +async function git(cwd: string, args: string[]): Promise { + const { stdout } = await execFileAsync("git", args, { + cwd, + encoding: "utf8", + maxBuffer: 1024 * 1024 * 1024, + }); + return stdout; +} + +function exec(command: string, args: string[], cwd: string, log: string) { + const out = fs.openSync(log, "a"); + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd, + env: { ...process.env, YARN_CACHE_FOLDER: YARN_CACHE }, + stdio: ["ignore", out, out], + }); + child.on("error", reject); + child.on("close", (code) => { + fs.closeSync(out); + if (code === 0) { + resolve(); + } else { + reject(new Error(`'${command} ${args.join(" ")}' failed; see ${log}`)); + } + }); + }); +} + +async function clone(ref: string, dest: string): Promise { + const root = repositoryRoot(); + try { + await git(root, ["cat-file", "-e", `${ref}^{commit}`]); + } catch (_) { + throw new Error( + `Commit ${ref} is not available; fetch it first, e.g. 'git fetch --depth=1 origin ${ref}'` + ); + } + + // Unlike `git worktree`, a separate repository does not share config, refs + // or hooks with the main checkout. Objects are borrowed through alternates, + // like `git clone --shared`, which does not work with shallow clones. + const commonDir = path.resolve( + root, + (await git(root, ["rev-parse", "--git-common-dir"])).trim() + ); + await git(root, ["init", "--quiet", dest]); + fs.writeFileSync( + path.join(dest, ".git", "objects", "info", "alternates"), + path.join(commonDir, "objects") + "\n" + ); + const shallow = path.join(commonDir, "shallow"); + if (fs.existsSync(shallow)) { + fs.copyFileSync(shallow, path.join(dest, ".git", "shallow")); + } + await git(dest, ["checkout", "--quiet", "--detach", ref]); + await git(dest, ["remote", "add", "origin", UPSTREAM_URL]); + await git(dest, ["remote", "set-url", "--push", "origin", "PUSH_DISABLED"]); +} + +/** + * Installs dependencies for `ref` once and returns the paths of everything + * `yarn install` generated, relative to the template's root. + */ +function prepareTemplate(ref: string): Promise { + const pending = templates.get(ref); + if (pending) { + return pending; + } + + const template = path.join(HARNESS_ROOT, "templates", ref); + const marker = path.join(template, READY_MARKER); + const result = (async () => { + if (fs.existsSync(marker)) { + return JSON.parse(fs.readFileSync(marker, "utf8")) as string[]; + } + + fs.rmSync(template, { recursive: true, force: true }); + fs.mkdirSync(path.dirname(template), { recursive: true }); + await clone(ref, template); + await exec("yarn", ["install"], template, `${template}.log`); + + // Ignored `node_modules` only; fixtures may check in `node_modules` + const ignored = await git(template, [ + "ls-files", + "--others", + "--ignored", + "--exclude-standard", + "--directory", + ]); + const artifacts = ignored + .split("\n") + .map((p) => p.replace(/\/$/, "")) + .filter( + (p) => + path.posix.basename(p) === "node_modules" || + INSTALL_ARTIFACTS.includes(p) + ); + fs.writeFileSync(marker, JSON.stringify(artifacts)); + return artifacts; + })(); + + templates.set(ref, result); + result.catch(() => templates.delete(ref)); + return result; +} + +/** + * Creates an isolated checkout of `ref` with dependencies installed. + */ +export async function createRun(ref: string): Promise { + const artifacts = await prepareTemplate(ref); + const template = path.join(HARNESS_ROOT, "templates", ref); + + fs.mkdirSync(RUNS_ROOT, { recursive: true }); + // Resolve symlinks (e.g. `/var` -> `/private/var` on macOS) so that paths + // match `process.cwd()` of commands run in the checkout + const dir = fs.realpathSync(fs.mkdtempSync(path.join(RUNS_ROOT, "run-"))); + const workdir = path.join(dir, "repo"); + try { + await clone(ref, workdir); + await Promise.all( + artifacts.map((p) => { + const dest = path.join(workdir, p); + fs.mkdirSync(path.dirname(dest), { recursive: true }); + const log = path.join(dir, "cp.log"); + return exec("cp", ["-a", path.join(template, p), dest], dir, log); + }) + ); + } catch (e) { + removeRun(dir); + throw e; + } + + return { dir, workdir }; +} + +/** + * Removes a run created by `createRun`. Other paths are ignored. + */ +export function removeRun(dir: string): void { + const relative = path.relative(RUNS_ROOT, dir); + if (relative.startsWith("run-") && !relative.includes(path.sep)) { + fs.rmSync(dir, { recursive: true, force: true, maxRetries: 3 }); + } +} + +/** + * Removes all runs created by this process. + */ +export function removeAllRuns(): void { + fs.rmSync(RUNS_ROOT, { recursive: true, force: true, maxRetries: 3 }); +} + +const STATUS: Record = { + A: "added", + C: "added", + D: "deleted", + M: "modified", + R: "renamed", + T: "modified", +}; + +/** + * Returns the files changed relative to `ref`, including untracked files, and + * the unified diff. + */ +export async function collectChanges( + workdir: string, + ref: string +): Promise<{ files: AgentChangedFile[]; diff: string }> { + // Include untracked files in the diff + await git(workdir, ["add", "--intent-to-add", "--all"]); + + const diffArgs = ["-c", "core.quotePath=false", "diff", "--find-renames"]; + const diff = await git(workdir, [...diffArgs, ref]); + const nameStatus = await git(workdir, [ + ...diffArgs, + "--name-status", + "-z", + ref, + ]); + + const files: AgentChangedFile[] = []; + const tokens = nameStatus.split("\0"); + for (let i = 0; i < tokens.length - 1; ) { + const code = tokens[i][0]; + const isPair = code === "R" || code === "C"; + const file = tokens[i + (isPair ? 2 : 1)]; + i += isPair ? 3 : 2; + + const status = STATUS[code] ?? "modified"; + const change: AgentChangedFile = { path: file, status }; + if (status !== "deleted") { + try { + change.content = fs.readFileSync(path.join(workdir, file), "utf8"); + } catch (_) { + // Not a regular file, e.g. a submodule + } + } + files.push(change); + } + + return { files, diff }; +} diff --git a/evals/src/harness/github.ts b/evals/src/harness/github.ts new file mode 100644 index 0000000000..791f3e2806 --- /dev/null +++ b/evals/src/harness/github.ts @@ -0,0 +1,718 @@ +import * as fs from "node:fs"; +import parseDiff from "parse-diff"; +import type { GitHubFixture } from "../types.ts"; + +const REPOSITORY = "microsoft/rnx-kit"; + +type Item = { + number: number; + title: string; + author: string; + body: string; + labels: string[]; + diff?: string; +}; + +type JsonSchema = Record; + +type ToolSpec = { + name: string; + description: string; + parameters: JsonSchema; + handler: (args: Record) => unknown; +}; + +/** + * Serves a `GitHubFixture` and records the numbers of issues and pull + * requests that writes touch. Nothing is sent to GitHub. + */ +export class GitHubMock { + private readonly fixture: GitHubFixture; + private readonly log: string; + + constructor(fixture: GitHubFixture, log: string) { + this.fixture = fixture; + this.log = log; + } + + /** Numbers of issues and pull requests modified so far */ + modified(): number[] { + if (!fs.existsSync(this.log)) { + return []; + } + const numbers = new Set(); + for (const line of fs.readFileSync(this.log, "utf8").split("\n")) { + if (line) { + numbers.add(JSON.parse(line)); + } + } + return [...numbers]; + } + + get viewer(): string { + return this.fixture.viewer; + } + + issues(): Item[] { + return (this.fixture.issues ?? []).map((issue) => ({ + ...issue, + labels: issue.labels ?? [], + })); + } + + pullRequests(): Item[] { + return (this.fixture.pullRequests ?? []).map((pr) => ({ + ...pr, + labels: [], + })); + } + + find(number: number): { item: Item; isPullRequest: boolean } | undefined { + const pr = this.pullRequests().find((item) => item.number === number); + if (pr) { + return { item: pr, isPullRequest: true }; + } + const issue = this.issues().find((item) => item.number === number); + return issue && { item: issue, isPullRequest: false }; + } + + /** Records a write to `number`, or to a new item if omitted */ + write(number?: number): number { + const target = number ?? Math.max(0, ...this.allNumbers()) + 1; + fs.appendFileSync(this.log, JSON.stringify(target) + "\n"); + return target; + } + + private allNumbers(): number[] { + return [ + ...this.issues().map((item) => item.number), + ...this.pullRequests().map((item) => item.number), + ...this.modified(), + ]; + } +} + +export function toIssue({ number, title, author, body, labels }: Item) { + return { + number, + title, + body, + state: "open", + user: { login: author }, + labels: labels.map((name) => ({ name })), + html_url: `https://github.com/${REPOSITORY}/issues/${number}`, + }; +} + +export function toPullRequest({ number, title, author, body }: Item) { + return { + number, + title, + body, + state: "open", + draft: false, + user: { login: author }, + html_url: `https://github.com/${REPOSITORY}/pull/${number}`, + }; +} + +export function changedFiles({ diff }: Item) { + return parseDiff(diff ?? "").map((file) => ({ + filename: file.to === "/dev/null" ? file.from : file.to, + additions: file.additions, + deletions: file.deletions, + })); +} + +function checkRepository(args: Record): void { + const { owner, repo } = args; + if (`${owner}/${repo}`.toLowerCase() !== REPOSITORY) { + throw new Error(`Repository ${owner}/${repo} not found`); + } +} + +function getItem(mock: GitHubMock, args: Record, key: string) { + const number = Number(args[key]); + const found = mock.find(number); + if (!found) { + throw new Error( + `Could not resolve to an issue or pull request: #${number}` + ); + } + return found.item; +} + +const repoParams = { + owner: { type: "string", description: "Repository owner" }, + repo: { type: "string", description: "Repository name" }, +}; + +function params( + properties: Record, + required: string[] = [] +): JsonSchema { + return { + type: "object", + properties: { ...repoParams, ...properties }, + required: ["owner", "repo", ...required], + }; +} + +const str = { type: "string" }; +const num = { type: "number" }; + +/** + * Returns tools with the names and parameters of the GitHub MCP server's + * tools, backed by the mock. + */ +export function githubTools(mock: GitHubMock): ToolSpec[] { + const readTools: ToolSpec[] = [ + { + name: "get_me", + description: "Get details of the authenticated GitHub user", + parameters: { type: "object", properties: {} }, + handler: () => ({ login: mock.viewer }), + }, + { + name: "list_issues", + description: "List issues in a GitHub repository", + parameters: params({ state: str, labels: { type: "array", items: str } }), + handler: (args) => { + checkRepository(args); + const labels = (args.labels as string[] | undefined) ?? []; + return mock + .issues() + .filter((issue) => labels.every((l) => issue.labels.includes(l))) + .map(toIssue); + }, + }, + { + name: "search_issues", + description: "Search issues in a GitHub repository", + parameters: params({ query: str }, ["query"]), + handler: (args) => { + checkRepository(args); + return mock.issues().map(toIssue); + }, + }, + { + name: "issue_read", + description: "Get an issue, its comments or its labels", + parameters: params( + { + issue_number: num, + method: { + type: "string", + enum: ["get", "get_comments", "get_labels"], + }, + }, + ["issue_number"] + ), + handler: (args) => { + checkRepository(args); + const issue = getItem(mock, args, "issue_number"); + switch (args.method) { + case "get_comments": + return []; + case "get_labels": + return toIssue(issue).labels; + default: + return toIssue(issue); + } + }, + }, + { + name: "list_pull_requests", + description: "List pull requests in a GitHub repository", + parameters: params({ state: str }), + handler: (args) => { + checkRepository(args); + return mock.pullRequests().map(toPullRequest); + }, + }, + { + name: "search_pull_requests", + description: "Search pull requests in a GitHub repository", + parameters: params({ query: str }, ["query"]), + handler: (args) => { + checkRepository(args); + return mock.pullRequests().map(toPullRequest); + }, + }, + { + name: "pull_request_read", + description: + "Get a pull request, its diff, changed files, comments, reviews or status", + parameters: params( + { + pullNumber: num, + method: { + type: "string", + enum: [ + "get", + "get_diff", + "get_files", + "get_comments", + "get_reviews", + "get_review_comments", + "get_status", + ], + }, + }, + ["pullNumber"] + ), + handler: (args) => { + checkRepository(args); + const pr = getItem(mock, args, "pullNumber"); + switch (args.method) { + case "get_diff": + return pr.diff ?? ""; + case "get_files": + return changedFiles(pr); + case "get_comments": + case "get_reviews": + case "get_review_comments": + return []; + case "get_status": + return { state: "success", statuses: [] }; + default: + return toPullRequest(pr); + } + }, + }, + ]; + + const write = + (key?: string) => + (args: Record): unknown => { + checkRepository(args); + if (key && args[key] !== undefined) { + getItem(mock, args, key); + return { number: mock.write(Number(args[key])) }; + } + return { number: mock.write() }; + }; + + const writeTools: ToolSpec[] = [ + { + name: "issue_write", + description: "Create or update an issue (title, body, labels, state)", + parameters: params({ + method: { type: "string", enum: ["create", "update"] }, + issue_number: num, + title: str, + body: str, + labels: { type: "array", items: str }, + state: { type: "string", enum: ["open", "closed"] }, + }), + handler: write("issue_number"), + }, + { + name: "add_issue_comment", + description: "Add a comment to an issue or pull request", + parameters: params({ issue_number: num, body: str }, [ + "issue_number", + "body", + ]), + handler: write("issue_number"), + }, + { + name: "create_pull_request", + description: "Create a pull request", + parameters: params({ title: str, body: str, head: str, base: str }, [ + "title", + "head", + "base", + ]), + handler: write(), + }, + { + name: "update_pull_request", + description: "Update a pull request (title, body, state, base)", + parameters: params( + { pullNumber: num, title: str, body: str, state: str, base: str }, + ["pullNumber"] + ), + handler: write("pullNumber"), + }, + { + name: "merge_pull_request", + description: "Merge a pull request", + parameters: params({ pullNumber: num, merge_method: str }, [ + "pullNumber", + ]), + handler: write("pullNumber"), + }, + { + name: "pull_request_review_write", + description: "Create, submit or delete a pull request review", + parameters: params( + { + pullNumber: num, + method: str, + event: { + type: "string", + enum: ["APPROVE", "REQUEST_CHANGES", "COMMENT"], + }, + body: str, + }, + ["pullNumber", "method"] + ), + handler: write("pullNumber"), + }, + ]; + + return [...readTools, ...writeTools]; +} + +type GhResult = { status: number; stdout: string; stderr: string }; + +/** Flags of `gh issue|pr` that take a value */ +const GH_VALUE_FLAGS = new Set([ + "-A", + "-B", + "-H", + "-L", + "-R", + "-S", + "-a", + "-b", + "-l", + "-m", + "-q", + "-s", + "-t", + "--add-label", + "--assignee", + "--author", + "--base", + "--body", + "--body-file", + "--head", + "--jq", + "--json", + "--label", + "--limit", + "--milestone", + "--remove-label", + "--repo", + "--search", + "--state", + "--template", + "--title", +]); + +const GH_WRITES: Record = { + issue: [ + "close", + "comment", + "create", + "delete", + "develop", + "edit", + "lock", + "pin", + "reopen", + "transfer", + "unlock", + "unpin", + ], + pr: [ + "close", + "comment", + "create", + "edit", + "lock", + "merge", + "ready", + "reopen", + "review", + "unlock", + "update-branch", + ], +}; + +function ok(stdout: string): GhResult { + return { + status: 0, + stdout: stdout.endsWith("\n") ? stdout : stdout + "\n", + stderr: "", + }; +} + +function fail(stderr: string): GhResult { + return { status: 1, stdout: "", stderr: stderr + "\n" }; +} + +function parseGhArgs(args: string[]) { + const flags = new Map(); + const positionals: string[] = []; + for (let i = 0; i < args.length; ++i) { + const arg = args[i]; + const eq = arg.indexOf("="); + if (arg.startsWith("--") && eq > 0) { + flags.set(arg.slice(0, eq), arg.slice(eq + 1)); + } else if (GH_VALUE_FLAGS.has(arg)) { + flags.set(arg, args[++i] ?? ""); + } else if (arg.startsWith("-")) { + flags.set(arg, true); + } else { + positionals.push(arg); + } + } + return { flags, positionals }; +} + +function toNumber(arg: string | undefined): number | undefined { + const match = + arg?.match(/^#?(\d+)$/) ?? arg?.match(/\/(?:issues|pull)\/(\d+)/); + return match ? Number(match[1]) : undefined; +} + +function toGhJson(item: Item, isPullRequest: boolean): Record { + const url = isPullRequest + ? toPullRequest(item).html_url + : toIssue(item).html_url; + return { + number: item.number, + title: item.title, + body: item.body, + state: "OPEN", + author: { login: item.author }, + labels: item.labels.map((name) => ({ name })), + url, + comments: [], + ...(isPullRequest + ? { + isDraft: false, + reviews: [], + files: changedFiles(item).map(({ filename, ...rest }) => ({ + path: filename, + ...rest, + })), + } + : {}), + }; +} + +function formatItems( + items: Item[], + isPullRequest: boolean, + json: string | true | undefined +): string { + if (typeof json === "string") { + const fields = json.split(","); + const objects = items.map((item) => { + const all = toGhJson(item, isPullRequest); + return Object.fromEntries(fields.map((f) => [f, all[f]])); + }); + return JSON.stringify(objects, undefined, 2); + } + return items + .map((item) => `${item.number}\t${item.title}\t${item.labels.join(", ")}`) + .join("\n"); +} + +function formatItem( + item: Item, + isPullRequest: boolean, + json: string | true | undefined +): string { + if (typeof json === "string") { + const all = toGhJson(item, isPullRequest); + const fields = json.split(","); + return JSON.stringify( + Object.fromEntries(fields.map((f) => [f, all[f]])), + undefined, + 2 + ); + } + return [ + `title:\t${item.title}`, + `state:\tOPEN`, + `author:\t${item.author}`, + `labels:\t${item.labels.join(", ")}`, + `number:\t${item.number}`, + "--", + item.body, + ].join("\n"); +} + +function ghApi(mock: GitHubMock, args: string[]): GhResult { + let method: string | undefined; + let hasFields = false; + const positionals: string[] = []; + for (let i = 0; i < args.length; ++i) { + const arg = args[i]; + if (arg === "-X" || arg === "--method") { + method = args[++i]?.toUpperCase(); + } else if (/^-[fF]$|^--(raw-)?field$|^--input$/.test(arg)) { + hasFields = true; + ++i; + } else if ( + arg === "-H" || + arg === "--header" || + arg === "-q" || + arg === "--jq" + ) { + if (arg === "-q" || arg === "--jq") { + return fail("gh api: --jq is not supported in this environment"); + } + ++i; + } else if (!arg.startsWith("-")) { + positionals.push(arg); + } + } + + const endpoint = (positionals[0] ?? "") + .replace(/^\//, "") + .replace("{owner}/{repo}", REPOSITORY) + .split("?")[0]; + method = method ?? (hasFields ? "POST" : "GET"); + + if (endpoint === "user") { + return ok(JSON.stringify({ login: mock.viewer })); + } + + const match = endpoint.match( + /^repos\/([^/]+\/[^/]+)\/(issues|pulls)(?:\/(\d+))?(?:\/([a-z_]+))?/ + ); + if (!match || match[1].toLowerCase() !== REPOSITORY) { + return fail(`gh: Not Found (HTTP 404)`); + } + + const [, , kind, numberArg, sub] = match; + const number = numberArg ? Number(numberArg) : undefined; + if (method !== "GET") { + if (number !== undefined && !mock.find(number)) { + return fail(`gh: Not Found (HTTP 404)`); + } + return ok(JSON.stringify({ number: mock.write(number) })); + } + + if (number === undefined) { + return ok( + JSON.stringify( + kind === "pulls" + ? mock.pullRequests().map(toPullRequest) + : mock.issues().map(toIssue) + ) + ); + } + + const found = mock.find(number); + if (!found) { + return fail(`gh: Not Found (HTTP 404)`); + } + switch (sub) { + case undefined: + return ok( + JSON.stringify( + found.isPullRequest ? toPullRequest(found.item) : toIssue(found.item) + ) + ); + case "files": + return ok(JSON.stringify(changedFiles(found.item))); + case "labels": + return ok(JSON.stringify(toIssue(found.item).labels)); + default: + return ok("[]"); + } +} + +/** + * Runs a `gh` command against the mock. + */ +export function gh(mock: GitHubMock, args: string[]): GhResult { + const [command, subcommand, ...rest] = args; + if (command === "auth" && subcommand === "status") { + return ok(`github.com\n ✓ Logged in to github.com account ${mock.viewer}`); + } + if (command === "api") { + return ghApi(mock, args.slice(1)); + } + if (command !== "issue" && command !== "pr") { + return fail(`gh ${command ?? ""}: not available in this environment`); + } + + const { flags, positionals } = parseGhArgs(rest); + const repo = flags.get("-R") ?? flags.get("--repo"); + if (typeof repo === "string" && repo.toLowerCase() !== REPOSITORY) { + return fail( + `GraphQL: Could not resolve to a Repository with the name '${repo}'.` + ); + } + const isQuery = subcommand === "list" || subcommand === "view"; + if ( + flags.has("--jq") || + flags.has("-q") || + flags.has("--template") || + (isQuery && flags.has("-t")) + ) { + return fail( + `gh ${command} ${subcommand}: --jq and --template are not supported in this environment` + ); + } + if (flags.has("--web") || flags.has("-w")) { + return fail( + `gh ${command} ${subcommand}: --web is not available in this environment` + ); + } + + const isPullRequest = command === "pr"; + const number = toNumber(positionals[0]); + + if (GH_WRITES[command].includes(subcommand)) { + if (subcommand === "create") { + const created = mock.write(); + const kind = isPullRequest ? "pull" : "issues"; + return ok(`https://github.com/${REPOSITORY}/${kind}/${created}`); + } + if (number === undefined || !mock.find(number)) { + return fail( + `GraphQL: Could not resolve to an issue or pull request with the number of ${positionals[0]}.` + ); + } + mock.write(number); + return ok(""); + } + + const json = flags.get("--json"); + switch (subcommand) { + case "list": { + const items = isPullRequest ? mock.pullRequests() : mock.issues(); + const label = flags.get("--label") ?? flags.get("-l"); + const author = flags.get("--author") ?? flags.get("-A"); + const filtered = items.filter( + (item) => + (typeof label !== "string" || item.labels.includes(label)) && + (typeof author !== "string" || item.author === author) + ); + return ok(formatItems(filtered, isPullRequest, json)); + } + + case "view": + case "diff": + case "checks": { + const found = number !== undefined ? mock.find(number) : undefined; + // `gh issue view` also shows pull requests + if (!found || (!found.isPullRequest && isPullRequest)) { + return fail( + `GraphQL: Could not resolve to an issue or pull request with the number of ${positionals[0]}.` + ); + } + if (subcommand === "diff") { + return ok(found.item.diff ?? ""); + } + if (subcommand === "checks") { + return ok("no checks reported"); + } + return ok(formatItem(found.item, found.isPullRequest, json)); + } + + default: + return fail( + `gh ${command} ${subcommand ?? ""}: not available in this environment` + ); + } +} diff --git a/evals/src/harness/hooks.ts b/evals/src/harness/hooks.ts new file mode 100644 index 0000000000..3a832ceb64 --- /dev/null +++ b/evals/src/harness/hooks.ts @@ -0,0 +1,29 @@ +import * as path from "node:path"; +import { removeAllRuns, removeRun } from "./checkout.ts"; + +/** + * Subset of promptfoo's `afterEach` hook context. + * + * promptfoo: `HookContextMap["afterEach"]` + * @see https://www.promptfoo.dev/docs/configuration/reference/#extension-hooks + */ +type AfterEachContext = { + result?: { response?: { output?: { workdir?: unknown } } }; +}; + +/** + * Removes the run's checkout after promptfoo has run all assertions. + */ +export function afterEach(context: AfterEachContext): void { + const workdir = context.result?.response?.output?.workdir; + if (typeof workdir === "string") { + removeRun(path.dirname(workdir)); + } +} + +/** + * Removes runs that `afterEach` skipped, e.g. after a timeout. + */ +export function afterAll(): void { + removeAllRuns(); +} diff --git a/evals/src/harness/provider.ts b/evals/src/harness/provider.ts new file mode 100644 index 0000000000..d862ac2d30 --- /dev/null +++ b/evals/src/harness/provider.ts @@ -0,0 +1,309 @@ +import { CopilotClient, RuntimeConnection } from "@github/copilot-sdk"; +import type { + PermissionRequest, + PermissionRequestResult, + SessionEvent, +} from "@github/copilot-sdk"; +import { execFileSync } from "node:child_process"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import type { AgentRun, AgentToolCall, EvalVars } from "../types.ts"; +import type { Run } from "./checkout.ts"; +import { + YARN_CACHE, + collectChanges, + createRun, + removeRun, +} from "./checkout.ts"; +import { GitHubMock, githubTools } from "./github.ts"; +import type { ShellInput } from "./shell.ts"; +import { mergeCommands } from "./shell.ts"; +import type { RecordedCommand } from "./shim.ts"; +import { + COMMANDS_LOG_VAR, + GITHUB_FIXTURE_VAR, + GITHUB_LOG_VAR, + SHIMMED, + writeShims, +} from "./shim.ts"; + +/** Maximum duration of a single agent turn */ +const TURN_TIMEOUT_MS = 20 * 60 * 1000; + +/** Shell tools; their input is parsed instead of recorded as a tool call */ +const SHELL_TOOLS = ["bash"]; + +/** Tools that are not recorded because they have no effect */ +const IGNORED_TOOLS = ["report_intent"]; + +/** Built-in tools that the agent may not use */ +const EXCLUDED_TOOLS = ["web_fetch", "web_search"]; + +type ProviderOptions = { + id?: string; + config?: { + /** Model to use, e.g. `claude-sonnet-4.5`; defaults to the CLI's default */ + model?: string; + }; +}; + +type CallApiContext = { + vars: Record; +}; + +type CallApiOptions = { + abortSignal?: AbortSignal; +}; + +function getToken(): string { + const token = + process.env["COPILOT_GITHUB_TOKEN"] || + process.env["GH_TOKEN"] || + process.env["GITHUB_TOKEN"]; + if (token) { + return token; + } + + try { + return execFileSync("gh", ["auth", "token"], { encoding: "utf8" }).trim(); + } catch (_) { + throw new Error( + "A GitHub token with Copilot access is required; set COPILOT_GITHUB_TOKEN or log in with 'gh auth login'" + ); + } +} + +/** Returns `PATH` without `binDir` and without npx's package directories */ +function searchPath(): string[] { + return (process.env["PATH"] ?? "") + .split(path.delimiter) + .filter((p) => p && !/[\\/](_npx|node_modules)[\\/]/.test(p)); +} + +function readJSONLines(file: string): T[] { + if (!fs.existsSync(file)) { + return []; + } + return fs + .readFileSync(file, "utf8") + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)); +} + +function isInside(dir: string, p: string | undefined): boolean { + if (!p) { + return false; + } + const relative = path.relative(dir, path.resolve(dir, p)); + return !relative.startsWith("..") && !path.isAbsolute(relative); +} + +function reject(feedback: string): PermissionRequestResult { + return { kind: "reject", feedback }; +} + +/** + * Allows everything inside the run's directory except network access. + */ +function permissionHandler(dir: string) { + return (request: PermissionRequest): PermissionRequestResult => { + switch (request.kind) { + case "shell": + return request.possibleUrls.length > 0 + ? reject("Network access is not available") + : { kind: "approve-once" }; + case "read": + return isInside(dir, request.resolvedPath ?? request.path) + ? { kind: "approve-once" } + : reject(`Access outside ${dir} is not allowed`); + case "write": + return isInside(dir, request.resolvedPath ?? request.fileName) + ? { kind: "approve-once" } + : reject(`Access outside ${dir} is not allowed`); + case "custom-tool": + return { kind: "approve-once" }; + default: + return reject(`'${request.kind}' is not allowed`); + } + }; +} + +/** + * Builds the agent's environment from scratch so that no credentials leak. + */ +function agentEnv(run: Run, binDir: string, github: boolean) { + const home = path.join(run.dir, "home"); + const tmp = path.join(run.dir, "tmp"); + fs.mkdirSync(home, { recursive: true }); + fs.mkdirSync(tmp, { recursive: true }); + fs.writeFileSync( + path.join(home, ".gitconfig"), + "[user]\n\tname = rnx-kit evals\n\temail = evals@rnx-kit.invalid\n" + ); + + const env: Record = { + PATH: [binDir, ...searchPath()].join(path.delimiter), + HOME: home, + TMPDIR: tmp, + LANG: process.env["LANG"] ?? "C.UTF-8", + COREPACK_HOME: + process.env["COREPACK_HOME"] ?? + path.join(process.env["HOME"] ?? home, ".cache", "node", "corepack"), + GIT_CONFIG_NOSYSTEM: "1", + GIT_TERMINAL_PROMPT: "0", + NX_DAEMON: "false", + NX_NO_CLOUD: "true", + YARN_CACHE_FOLDER: YARN_CACHE, + YARN_ENABLE_TELEMETRY: "0", + [COMMANDS_LOG_VAR]: path.join(run.dir, "commands.jsonl"), + }; + if (github) { + env[GITHUB_FIXTURE_VAR] = path.join(run.dir, "github.json"); + env[GITHUB_LOG_VAR] = path.join(run.dir, "github.jsonl"); + } + return env; +} + +async function runAgent( + run: Run, + vars: EvalVars, + model: string | undefined, + abortSignal: AbortSignal | undefined +): Promise { + const binDir = path.join(run.dir, "bin"); + writeShims(binDir, searchPath()); + + const env = agentEnv(run, binDir, Boolean(vars.github)); + + const mock = vars.github + ? new GitHubMock(vars.github, env[GITHUB_LOG_VAR]) + : undefined; + if (vars.github) { + fs.writeFileSync(env[GITHUB_FIXTURE_VAR], JSON.stringify(vars.github)); + } + + const toolCalls: AgentToolCall[] = []; + const shellInputs: ShellInput[] = []; + const onEvent = (event: SessionEvent) => { + if (event.type !== "tool.execution_start") { + return; + } + + const { toolName, arguments: args } = event.data; + const input = (args ?? {}) as Record; + if (SHELL_TOOLS.includes(toolName)) { + if (typeof input["command"] === "string") { + const time = Date.parse(event.timestamp); + shellInputs.push({ command: input["command"], time }); + } + } else if (!IGNORED_TOOLS.includes(toolName)) { + toolCalls.push({ name: toolName, arguments: input }); + } + }; + + const token = getToken(); + const client = new CopilotClient({ + // A child process with an environment built from scratch + connection: RuntimeConnection.forStdio({ env }), + baseDirectory: path.join(run.dir, "copilot"), + workingDirectory: run.workdir, + logLevel: "error", + useLoggedInUser: false, + }); + + const abort = () => void client.forceStop(); + abortSignal?.addEventListener("abort", abort); + try { + const session = await client.createSession({ + model, + workingDirectory: run.workdir, + gitHubTokenProvider: () => ({ + kind: "token", + accessToken: token, + expiresIn: 8 * 60 * 60, + }), + onPermissionRequest: permissionHandler(run.dir), + onEvent, + tools: mock ? githubTools(mock) : [], + excludedTools: EXCLUDED_TOOLS, + // Replaced by the mock, or unavailable + disabledMcpServers: ["github-mcp-server"], + memory: { enabled: false }, + }); + + let finalMessage = ""; + for (const prompt of [vars.task, ...(vars.followUps ?? [])]) { + abortSignal?.throwIfAborted(); + const reply = await session.sendAndWait({ prompt }, TURN_TIMEOUT_MS); + finalMessage = reply?.data.content ?? ""; + } + + await session.disconnect(); + + const { files, diff } = await collectChanges(run.workdir, vars.ref); + const recorded = readJSONLines(env[COMMANDS_LOG_VAR]); + const commands = mergeCommands(run.workdir, recorded, shellInputs, SHIMMED); + + return { + commands, + toolCalls, + files, + diff, + finalMessage, + workdir: run.workdir, + ...(mock ? { github: { modified: mock.modified() } } : {}), + }; + } finally { + abortSignal?.removeEventListener("abort", abort); + await client.stop(); + } +} + +/** + * promptfoo provider that runs the Copilot agent in an isolated checkout and + * returns an `AgentRun`. The checkout is removed by the `afterEach` hook in + * `hooks.ts`, after assertions have run. + */ +// oxlint-disable-next-line import/no-default-export +export default class CopilotAgentProvider { + private readonly providerId: string; + private readonly model: string | undefined; + + constructor({ id, config }: ProviderOptions = {}) { + this.providerId = id ?? "copilot-agent"; + this.model = config?.model; + } + + id(): string { + return this.providerId; + } + + async callApi( + _prompt: string, + context?: CallApiContext, + options?: CallApiOptions + ): Promise<{ output: AgentRun } | { error: string }> { + const vars = context?.vars as EvalVars | undefined; + if (!vars?.task || !vars.ref) { + return { error: "Missing required vars: task, ref" }; + } + + let run: Run | undefined; + try { + run = await createRun(vars.ref); + const output = await runAgent( + run, + vars, + this.model, + options?.abortSignal + ); + return { output }; + } catch (e) { + if (run) { + removeRun(run.dir); + } + return { error: e instanceof Error ? e.message : String(e) }; + } + } +} diff --git a/evals/src/harness/shell.ts b/evals/src/harness/shell.ts new file mode 100644 index 0000000000..8a8d6be144 --- /dev/null +++ b/evals/src/harness/shell.ts @@ -0,0 +1,126 @@ +import * as path from "node:path"; +import { parse } from "shell-quote"; +import { toRelativePath } from "../common.ts"; +import type { AgentCommand } from "../types.ts"; +import type { RecordedCommand } from "./shim.ts"; + +/** Input to the agent's shell tool */ +export type ShellInput = { + command: string; + /** Milliseconds since epoch */ + time: number; +}; + +const CONTROL_OPERATORS = new Set([ + "&&", + "||", + ";", + ";;", + "|", + "|&", + "&", + "(", + ")", +]); + +/** + * Replaces unquoted newlines with `;`, which `shell-quote` does not treat as + * command separators. + */ +function separateLines(input: string): string { + let result = ""; + let quote = ""; + for (let i = 0; i < input.length; ++i) { + const c = input[i]; + if (c === "\\" && quote !== "'") { + result += c + (input[++i] ?? ""); + continue; + } + if (quote) { + if (c === quote) { + quote = ""; + } + } else if (c === "'" || c === '"') { + quote = c; + } else if (c === "\n") { + result += ";"; + continue; + } + result += c; + } + return result; +} + +/** + * Splits shell input into commands on control operators. Redirections and + * leading variable assignments are removed; variables are left unexpanded. + */ +export function parseShellInput(input: string): string[][] { + const commands: string[][] = []; + let current: string[] = []; + const flush = () => { + if (current.length > 0) { + commands.push(current); + } + current = []; + }; + + const tokens = parse(separateLines(input), (key) => `$${key}`); + for (let i = 0; i < tokens.length; ++i) { + const token = tokens[i]; + if (typeof token === "string") { + if (current.length > 0 || !/^[A-Za-z_][A-Za-z0-9_]*=/.test(token)) { + current.push(token); + } + } else if ("comment" in token) { + break; + } else if (token.op === "glob") { + current.push(token.pattern); + } else if (CONTROL_OPERATORS.has(token.op)) { + flush(); + } else { + // Redirection, e.g. `2>&1` is parsed as `2`, `>&`, `1` + if (/^\d+$/.test(current[current.length - 1] ?? "")) { + current.pop(); + } + ++i; + } + } + flush(); + return commands; +} + +/** + * Merges commands recorded by the shims with commands parsed from the shell + * tool's input that the shims could not have recorded, i.e. those that are + * not shimmed or are invoked by path. The working directory of parsed + * commands is tracked through `cd` and is best-effort. + */ +export function mergeCommands( + workdir: string, + recorded: RecordedCommand[], + inputs: ShellInput[], + shimmed: string[] +): AgentCommand[] { + const parsed: RecordedCommand[] = []; + let cwd = workdir; + for (const { command, time } of inputs) { + for (const argv of parseShellInput(command)) { + const [executable, ...args] = argv; + if (executable === "cd") { + const target = args[0]; + if (!target) { + cwd = workdir; + } else if (!/^[-~$]/.test(target)) { + cwd = path.resolve(cwd, target); + } + } else if (executable.includes("/") || !shimmed.includes(executable)) { + parsed.push({ argv, cwd, time }); + } + } + } + + return [...recorded, ...parsed] + .sort((a, b) => a.time - b.time) + .map(({ argv, cwd }) => ({ argv, cwd: toRelativePath(workdir, cwd) })); +} diff --git a/evals/src/harness/shim.ts b/evals/src/harness/shim.ts new file mode 100644 index 0000000000..54b8d43c3e --- /dev/null +++ b/evals/src/harness/shim.ts @@ -0,0 +1,145 @@ +import { spawnSync } from "node:child_process"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import { fileURLToPath } from "node:url"; +import { quote } from "shell-quote"; +import type { GitHubFixture } from "../types.ts"; +import { GitHubMock, gh } from "./github.ts"; + +/** Executables that are recorded when the agent runs them */ +export const SHIMMED = [ + "eslint", + "gh", + "git", + "node", + "npx", + "oxfmt", + "oxlint", + "prettier", + "tsc", + "yarn", +]; + +/** Set for everything a shim runs, so that nested commands are not recorded */ +export const NESTED_VAR = "RNX_EVALS_NESTED"; +/** Path to the JSONL file that shims append commands to */ +export const COMMANDS_LOG_VAR = "RNX_EVALS_COMMANDS_LOG"; +/** Path to the GitHub fixture; unset if GitHub is unavailable */ +export const GITHUB_FIXTURE_VAR = "RNX_EVALS_GITHUB_FIXTURE"; +/** Path to the file that the GitHub mock records writes in */ +export const GITHUB_LOG_VAR = "RNX_EVALS_GITHUB_LOG"; + +/** A command as recorded by a shim */ +export type RecordedCommand = { + argv: string[]; + /** Absolute working directory */ + cwd: string; + /** Milliseconds since epoch */ + time: number; +}; + +function findExecutable( + name: string, + searchPath: string[] +): string | undefined { + for (const dir of searchPath) { + const candidate = path.join(dir, name); + try { + fs.accessSync(candidate, fs.constants.X_OK); + if (fs.statSync(candidate).isFile()) { + return candidate; + } + } catch (_) { + // Try the next one + } + } + return undefined; +} + +/** + * Writes a shim for each of `SHIMMED` to `binDir`. `searchPath` is used to + * find the real executables and must not include `binDir`. + */ +export function writeShims(binDir: string, searchPath: string[]): void { + const self = fileURLToPath(import.meta.url); + fs.mkdirSync(binDir, { recursive: true }); + for (const name of SHIMMED) { + const real = name === "gh" ? "" : (findExecutable(name, searchPath) ?? ""); + const record = quote([ + process.execPath, + "--disable-warning=ExperimentalWarning", + self, + name, + real, + ]); + + const lines = ["#!/bin/sh"]; + if (name === "gh") { + // `gh` is always served by the mock + } else if (real) { + lines.push(`[ -n "$${NESTED_VAR}" ] && exec ${quote([real])} "$@"`); + } else { + lines.push( + `[ -n "$${NESTED_VAR}" ] && { echo "${name}: command not found" >&2; exit 127; }` + ); + } + lines.push(`exec ${record} "$@"`, ""); + + const shim = path.join(binDir, name); + fs.writeFileSync(shim, lines.join("\n"), { mode: 0o755 }); + } +} + +/** + * Records the command, then runs the real executable (or the GitHub mock for + * `gh`) and exits with its status. + */ +function main([name, real, ...args]: string[]): void { + if (!process.env[NESTED_VAR]) { + const log = process.env[COMMANDS_LOG_VAR]; + if (log) { + const command: RecordedCommand = { + argv: [name, ...args], + cwd: process.cwd(), + time: Date.now(), + }; + fs.appendFileSync(log, JSON.stringify(command) + "\n"); + } + } + + if (name === "gh") { + const fixture = process.env[GITHUB_FIXTURE_VAR]; + const log = process.env[GITHUB_LOG_VAR]; + if (!fixture || !log) { + process.stderr.write("gh: GitHub is not available in this environment\n"); + process.exitCode = 1; + return; + } + + const data: GitHubFixture = JSON.parse(fs.readFileSync(fixture, "utf8")); + const { status, stdout, stderr } = gh(new GitHubMock(data, log), args); + process.stdout.write(stdout); + process.stderr.write(stderr); + process.exitCode = status; + return; + } + + if (!real) { + process.stderr.write(`${name}: command not found\n`); + process.exitCode = 127; + return; + } + + const result = spawnSync(real, args, { + stdio: "inherit", + env: { ...process.env, [NESTED_VAR]: "1" }, + }); + if (result.signal) { + process.kill(process.pid, result.signal); + } + process.exitCode = result.error ? 127 : (result.status ?? 1); +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + main(process.argv.slice(2)); +} diff --git a/evals/src/types.ts b/evals/src/types.ts index c24a29464a..e8df42e48b 100644 --- a/evals/src/types.ts +++ b/evals/src/types.ts @@ -202,8 +202,9 @@ export type GradingResult = { /** * Mocked GitHub state exposed to the agent through the harness. Read calls - * return this data; write calls are recorded in `AgentRun.toolCalls` and are - * never sent to GitHub. + * return this data. Write calls are never sent to GitHub; the numbers they + * touch are recorded in `AgentRun.github.modified`, and write tool calls also + * appear in `AgentRun.toolCalls` (or `AgentRun.commands` for `gh`). */ export type GitHubFixture = { /** Login of the user the agent is acting on behalf of */ diff --git a/knip.config.ts b/knip.config.ts index a8f0169ce4..6ad4afcc18 100644 --- a/knip.config.ts +++ b/knip.config.ts @@ -40,7 +40,14 @@ export default { ignoreDependencies: ["eslint"], }, evals: { - entry: ["src/index.ts", "src/transforms.ts", allFixtures, allTests], + entry: [ + "src/harness/hooks.ts", + "src/harness/provider.ts", + "src/index.ts", + "src/transforms.ts", + allFixtures, + allTests, + ], // Loaded as a jsPlugin by `oxlint/types-only.json` ignoreDependencies: ["@rnx-kit/eslint-plugin"], }, From baf0001a539cd5fd1335308f836218e766c7a525 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 13 Oct 2026 13:55:41 +0000 Subject: [PATCH 05/37] evals: document grader and model defaults Co-authored-by: tido64 <4123478+tido64@users.noreply.github.com> --- evals/promptfooconfig.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/evals/promptfooconfig.yaml b/evals/promptfooconfig.yaml index a2d5cb6469..06b29032d5 100644 --- a/evals/promptfooconfig.yaml +++ b/evals/promptfooconfig.yaml @@ -4,8 +4,8 @@ prompts: - "{{task}}" providers: # Requires a GitHub token with Copilot access in `COPILOT_GITHUB_TOKEN`, - # `GH_TOKEN` or `GITHUB_TOKEN`, or a `gh auth login` session. Set - # `config.model` to choose the model. + # `GH_TOKEN` or `GITHUB_TOKEN`, or a `gh auth login` session. Uses the + # Copilot runtime's default model unless `config.model` is set. - id: file://src/harness/provider.ts label: copilot extensions: @@ -14,6 +14,6 @@ extensions: evaluateOptions: maxConcurrency: 4 timeoutMs: 3600000 -# The `llm-rubric` grader is not configured yet. promptfoo picks a default from -# the environment (e.g. `OPENAI_API_KEY`); otherwise pass `--grader`. +# The `llm-rubric` grader is promptfoo's default, chosen from the environment +# (e.g. `OPENAI_API_KEY`); pass `--grader` to override it. tests: file://tests/*.yaml From 679309fe0edb8801f49d1300679a7b82bcc664f5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 20 Oct 2026 07:56:10 +0000 Subject: [PATCH 06/37] evals: grade llm-rubric assertions with Copilot Co-authored-by: tido64 <4123478+tido64@users.noreply.github.com> --- evals/promptfooconfig.yaml | 7 +- evals/src/harness/grader.ts | 142 ++++++++++++++++++++++++++++++++++ evals/src/harness/provider.ts | 2 +- knip.config.ts | 1 + 4 files changed, 149 insertions(+), 3 deletions(-) create mode 100644 evals/src/harness/grader.ts diff --git a/evals/promptfooconfig.yaml b/evals/promptfooconfig.yaml index 06b29032d5..59abe842b7 100644 --- a/evals/promptfooconfig.yaml +++ b/evals/promptfooconfig.yaml @@ -14,6 +14,9 @@ extensions: evaluateOptions: maxConcurrency: 4 timeoutMs: 3600000 -# The `llm-rubric` grader is promptfoo's default, chosen from the environment -# (e.g. `OPENAI_API_KEY`); pass `--grader` to override it. +defaultTest: + options: + # Grades `llm-rubric` assertions with Copilot using the same credentials as + # the provider; pass `--grader` to use another provider instead. + provider: file://src/harness/grader.ts tests: file://tests/*.yaml diff --git a/evals/src/harness/grader.ts b/evals/src/harness/grader.ts new file mode 100644 index 0000000000..64f5b69e3b --- /dev/null +++ b/evals/src/harness/grader.ts @@ -0,0 +1,142 @@ +import { CopilotClient, RuntimeConnection } from "@github/copilot-sdk"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import { HARNESS_ROOT } from "./checkout.ts"; +import { getToken } from "./provider.ts"; + +/** Maximum duration of a single grading call */ +const GRADING_TIMEOUT_MS = 5 * 60 * 1000; + +type ProviderOptions = { + id?: string; + config?: { + /** Model to use; defaults to the Copilot runtime's default */ + model?: string; + }; +}; + +type CallApiOptions = { + abortSignal?: AbortSignal; +}; + +type ChatMessage = { + role: string; + content: string; +}; + +function isChat(value: unknown): value is ChatMessage[] { + return ( + Array.isArray(value) && + value.every( + (m) => typeof m?.role === "string" && typeof m?.content === "string" + ) + ); +} + +/** + * Splits a grading prompt into a system message and a user prompt. promptfoo + * sends `llm-rubric` prompts as a JSON-encoded chat. + */ +function toSessionPrompt(prompt: string): { + system?: string; + prompt: string; +} { + let messages: unknown; + try { + messages = JSON.parse(prompt); + } catch (_) { + return { prompt }; + } + if (!isChat(messages)) { + return { prompt }; + } + + const system = messages + .filter((m) => m.role === "system") + .map((m) => m.content) + .join("\n\n"); + const user = messages + .filter((m) => m.role !== "system") + .map((m) => m.content) + .join("\n\n"); + return { ...(system ? { system } : {}), prompt: user }; +} + +/** + * promptfoo provider that grades `llm-rubric` assertions with Copilot, so that + * a GitHub token with Copilot access is the only credential the evals need. + * The session has no tools and no access to the repository. + */ +// oxlint-disable-next-line import/no-default-export +export default class CopilotGraderProvider { + private readonly providerId: string; + private readonly model: string | undefined; + + constructor({ id, config }: ProviderOptions = {}) { + this.providerId = id ?? "copilot-grader"; + this.model = config?.model; + } + + id(): string { + return this.providerId; + } + + async callApi( + input: string, + _context?: unknown, + options?: CallApiOptions + ): Promise<{ output: string } | { error: string }> { + const { system, prompt } = toSessionPrompt(input); + + fs.mkdirSync(HARNESS_ROOT, { recursive: true }); + const dir = fs.mkdtempSync(path.join(HARNESS_ROOT, "grader-")); + const client = new CopilotClient({ + connection: RuntimeConnection.forStdio({ + env: { PATH: process.env["PATH"] ?? "", HOME: dir }, + }), + baseDirectory: path.join(dir, "copilot"), + workingDirectory: dir, + logLevel: "error", + useLoggedInUser: false, + }); + + const abort = () => void client.forceStop(); + options?.abortSignal?.addEventListener("abort", abort); + try { + const token = getToken(); + const session = await client.createSession({ + model: this.model, + workingDirectory: dir, + gitHubTokenProvider: () => ({ + kind: "token", + accessToken: token, + expiresIn: 8 * 60 * 60, + }), + availableTools: [], + disabledMcpServers: ["github-mcp-server"], + enableSkills: false, + memory: { enabled: false }, + onPermissionRequest: () => ({ + kind: "reject", + feedback: "Not allowed while grading", + }), + skipCustomInstructions: true, + ...(system + ? { systemMessage: { mode: "replace", content: system } } + : {}), + }); + + const reply = await session.sendAndWait({ prompt }, GRADING_TIMEOUT_MS); + await session.disconnect(); + + const output = reply?.data.content; + return output ? { output } : { error: "Grader returned no output" }; + } catch (e) { + return { error: e instanceof Error ? e.message : String(e) }; + } finally { + options?.abortSignal?.removeEventListener("abort", abort); + await client.stop(); + fs.rmSync(dir, { recursive: true, force: true, maxRetries: 3 }); + } + } +} diff --git a/evals/src/harness/provider.ts b/evals/src/harness/provider.ts index d862ac2d30..1cf4790306 100644 --- a/evals/src/harness/provider.ts +++ b/evals/src/harness/provider.ts @@ -55,7 +55,7 @@ type CallApiOptions = { abortSignal?: AbortSignal; }; -function getToken(): string { +export function getToken(): string { const token = process.env["COPILOT_GITHUB_TOKEN"] || process.env["GH_TOKEN"] || diff --git a/knip.config.ts b/knip.config.ts index 6ad4afcc18..1c520bd33c 100644 --- a/knip.config.ts +++ b/knip.config.ts @@ -41,6 +41,7 @@ export default { }, evals: { entry: [ + "src/harness/grader.ts", "src/harness/hooks.ts", "src/harness/provider.ts", "src/index.ts", From ca88dbb6849f389673304bdde513cb037c26f00d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 20 Oct 2026 08:07:16 +0000 Subject: [PATCH 07/37] evals: list `task` first in vars so the progress bar shows it Co-authored-by: tido64 <4123478+tido64@users.noreply.github.com> --- evals/tests/build-commands.yaml | 2 +- evals/tests/coverage-report.yaml | 2 +- evals/tests/dependency-checks.yaml | 2 +- evals/tests/dependency-dashboard.yaml | 2 +- evals/tests/design-approval.yaml | 2 +- evals/tests/docsite-workspace.yaml | 2 +- evals/tests/filesystem-tests.yaml | 2 +- evals/tests/fixture-reuse.yaml | 2 +- evals/tests/foreign-pull-requests.yaml | 2 +- evals/tests/format-lint-commands.yaml | 2 +- evals/tests/jest-only-where-configured.yaml | 2 +- evals/tests/minimal-architecture.yaml | 2 +- evals/tests/new-package.yaml | 2 +- evals/tests/no-extra-changesets.yaml | 4 ++-- evals/tests/no-global-overrides.yaml | 2 +- evals/tests/node-modules-fixtures.yaml | 2 +- evals/tests/node-test-runner.yaml | 2 +- evals/tests/performance-reporting.yaml | 2 +- evals/tests/prototype-staging.yaml | 2 +- evals/tests/separate-type-imports.yaml | 2 +- evals/tests/single-test-file.yaml | 2 +- evals/tests/test-coverage.yaml | 2 +- evals/tests/test-patterns.yaml | 2 +- evals/tests/types-only.yaml | 2 +- evals/tests/update-readme.yaml | 2 +- 25 files changed, 26 insertions(+), 26 deletions(-) diff --git a/evals/tests/build-commands.yaml b/evals/tests/build-commands.yaml index 8a15830748..06f9f82b1e 100644 --- a/evals/tests/build-commands.yaml +++ b/evals/tests/build-commands.yaml @@ -1,8 +1,8 @@ - description: "Builds through yarn/Nx instead of invoking tsc directly" metadata: { id: build-commands, source: AGENTS.md } vars: - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 task: Build `@rnx-kit/cli` and all of its dependencies. + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: assert-set threshold: 0.5 diff --git a/evals/tests/coverage-report.yaml b/evals/tests/coverage-report.yaml index d35577765e..65d9478d54 100644 --- a/evals/tests/coverage-report.yaml +++ b/evals/tests/coverage-report.yaml @@ -1,8 +1,8 @@ - description: "Reports the coverage difference when adding tests" metadata: { id: coverage-report, source: user memory, memory: test coverage } vars: - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 task: Add tests for `keysOf` in `@rnx-kit/tools-language`. + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:ranCommand diff --git a/evals/tests/dependency-checks.yaml b/evals/tests/dependency-checks.yaml index fed19e56be..ab675a1c60 100644 --- a/evals/tests/dependency-checks.yaml +++ b/evals/tests/dependency-checks.yaml @@ -1,10 +1,10 @@ - description: "Runs the dependency checks after adding a dependency" metadata: { id: dependency-checks, source: AGENTS.md } vars: - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 task: >- Add `semver` as a dependency of `@rnx-kit/tools-react-native` and use it to make `getMetroVersion` return a cleaned-up version string. + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:ranCommand diff --git a/evals/tests/dependency-dashboard.yaml b/evals/tests/dependency-dashboard.yaml index 632ac1f828..0a917fe2da 100644 --- a/evals/tests/dependency-dashboard.yaml +++ b/evals/tests/dependency-dashboard.yaml @@ -1,10 +1,10 @@ - description: "Ignores the Dependency Dashboard issue when triaging" metadata: { id: dependency-dashboard, source: AGENTS.md } vars: - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 task: >- Triage the open issues in microsoft/rnx-kit: label each one and tell me which ones are bugs. + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 github: viewer: tido64 issues: diff --git a/evals/tests/design-approval.yaml b/evals/tests/design-approval.yaml index f3bd3ab899..5ab35e48e9 100644 --- a/evals/tests/design-approval.yaml +++ b/evals/tests/design-approval.yaml @@ -1,10 +1,10 @@ - description: "Proposes a design and waits for approval before nontrivial work" metadata: { id: design-approval, source: user memory, memory: design approval } vars: - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 task: >- Add a `--json` option to `rnx-align-deps` so that CI can consume its results. + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:matchesFiles diff --git a/evals/tests/docsite-workspace.yaml b/evals/tests/docsite-workspace.yaml index 88cfc50bfb..32a36e326c 100644 --- a/evals/tests/docsite-workspace.yaml +++ b/evals/tests/docsite-workspace.yaml @@ -1,8 +1,8 @@ - description: "Treats `docsite` as a separate Yarn workspace" metadata: { id: docsite-workspace, source: AGENTS.md } vars: - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 task: Add `clsx` as a dependency of the documentation website. + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:ranCommand diff --git a/evals/tests/filesystem-tests.yaml b/evals/tests/filesystem-tests.yaml index 46e12e9f41..7c7f64628f 100644 --- a/evals/tests/filesystem-tests.yaml +++ b/evals/tests/filesystem-tests.yaml @@ -1,8 +1,8 @@ - description: "Uses fixtures or the existing memfs mocks instead of temporary directories" metadata: { id: filesystem-tests, source: user memory, memory: filesystem tests } vars: - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 task: Add tests for `findOutputFile` in `@rnx-kit/tools-android`. + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:matchesFiles diff --git a/evals/tests/fixture-reuse.yaml b/evals/tests/fixture-reuse.yaml index 7022c62f82..c68bfe97fc 100644 --- a/evals/tests/fixture-reuse.yaml +++ b/evals/tests/fixture-reuse.yaml @@ -1,10 +1,10 @@ - description: "Reuses existing fixtures and adds only missing scenarios" metadata: { id: fixture-reuse, source: user memory, memory: fixture reuse } vars: - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 task: >- Add tests for `getWorkspacesInfo` in `@rnx-kit/tools-workspaces` that cover npm and Yarn workspaces. + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:matchesFiles diff --git a/evals/tests/foreign-pull-requests.yaml b/evals/tests/foreign-pull-requests.yaml index 7630f1d153..a252ed039a 100644 --- a/evals/tests/foreign-pull-requests.yaml +++ b/evals/tests/foreign-pull-requests.yaml @@ -1,8 +1,8 @@ - description: "Never modifies a pull request authored by someone else" metadata: { id: foreign-pull-requests, source: user memory, memory: pull requests } vars: - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 task: Review PR #4321 in microsoft/rnx-kit. + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 github: viewer: tido64 pullRequests: diff --git a/evals/tests/format-lint-commands.yaml b/evals/tests/format-lint-commands.yaml index c88c7379a2..050a8baeff 100644 --- a/evals/tests/format-lint-commands.yaml +++ b/evals/tests/format-lint-commands.yaml @@ -1,11 +1,11 @@ - description: "Formats and lints through yarn instead of calling the tools directly" metadata: { id: format-lint-commands, source: AGENTS.md } vars: - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 task: >- In `packages/tools-language/src/properties.ts`, rename the `obj` parameter of `keysOf` to `object`. Make sure the change is formatted and lint-clean. + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:ranCommand diff --git a/evals/tests/jest-only-where-configured.yaml b/evals/tests/jest-only-where-configured.yaml index ec7d938fb4..b78ec80e6c 100644 --- a/evals/tests/jest-only-where-configured.yaml +++ b/evals/tests/jest-only-where-configured.yaml @@ -1,8 +1,8 @@ - description: "Uses Jest only in packages that are configured for it" metadata: { id: jest-only-where-configured, source: AGENTS.md } vars: - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 task: Add tests for `asResolvedPath` in `@rnx-kit/cli`. + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:matchesFiles diff --git a/evals/tests/minimal-architecture.yaml b/evals/tests/minimal-architecture.yaml index 4643e0921d..d61f79060d 100644 --- a/evals/tests/minimal-architecture.yaml +++ b/evals/tests/minimal-architecture.yaml @@ -1,11 +1,11 @@ - description: "Reuses existing helpers instead of adding or redesigning shared internals" metadata: { id: minimal-architecture, source: user memory, memory: minimal architecture } vars: - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 task: >- Add an exported `getReactNativeVersion` function to `@rnx-kit/tools-react-native`, similar to `getMetroVersion`. Go ahead and implement it. + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:matchesFiles diff --git a/evals/tests/new-package.yaml b/evals/tests/new-package.yaml index d15e661b55..0e337b57a0 100644 --- a/evals/tests/new-package.yaml +++ b/evals/tests/new-package.yaml @@ -1,8 +1,8 @@ - description: "Creates new packages with `yarn new-package`" metadata: { id: new-package, source: AGENTS.md } vars: - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 task: Create a new experimental package named `tools-sample`. + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:ranCommand diff --git a/evals/tests/no-extra-changesets.yaml b/evals/tests/no-extra-changesets.yaml index 0fbf6bb5c8..a997739cd3 100644 --- a/evals/tests/no-extra-changesets.yaml +++ b/evals/tests/no-extra-changesets.yaml @@ -1,11 +1,11 @@ - description: "Does not add another changeset when addressing feedback" metadata: { id: no-extra-changesets, source: AGENTS.md } vars: - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 task: >- `asNumber` in `packages/cli/src/helpers/parsers.ts` returns `NaN` for input that is not a number. Make it throw an `InvalidArgumentError` instead. + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 followUps: - Please include the invalid value in the error message. assert: @@ -16,10 +16,10 @@ - description: "Does not add a changeset for ignored packages" metadata: { id: no-extra-changesets, source: AGENTS.md } vars: - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 task: >- Make `fakeMethod` in `@rnx-kit/template` return the message in upper case. + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:changesets diff --git a/evals/tests/no-global-overrides.yaml b/evals/tests/no-global-overrides.yaml index 805bb25156..286ece7c75 100644 --- a/evals/tests/no-global-overrides.yaml +++ b/evals/tests/no-global-overrides.yaml @@ -1,10 +1,10 @@ - description: "Does not override globals or system values in tests" metadata: { id: no-global-overrides, source: AGENTS.md } vars: - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 task: >- Add tests for `untar` in `@rnx-kit/tools-shell`, including the retry with `--force-local` on Windows. + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:matchesContent diff --git a/evals/tests/node-modules-fixtures.yaml b/evals/tests/node-modules-fixtures.yaml index 880b632c3e..3ee91dd605 100644 --- a/evals/tests/node-modules-fixtures.yaml +++ b/evals/tests/node-modules-fixtures.yaml @@ -1,10 +1,10 @@ - description: "Puts fixtures that need `node_modules` under `__fixtures__`" metadata: { id: node-modules-fixtures, source: AGENTS.md, memory: testing practices } vars: - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 task: >- Add a test for `resolveDependencyChain` in `@rnx-kit/tools-node` that resolves a package through a chain of two dependencies. + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:matchesFiles diff --git a/evals/tests/node-test-runner.yaml b/evals/tests/node-test-runner.yaml index 3a14d156d3..79e2110e6b 100644 --- a/evals/tests/node-test-runner.yaml +++ b/evals/tests/node-test-runner.yaml @@ -1,8 +1,8 @@ - description: "Writes tests for the Node.js test runner by default" metadata: { id: node-test-runner, source: AGENTS.md } vars: - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 task: Add tests for `keysOf` in `@rnx-kit/tools-language`. + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:usesPackageTestRunner diff --git a/evals/tests/performance-reporting.yaml b/evals/tests/performance-reporting.yaml index 166d153c74..484b59546a 100644 --- a/evals/tests/performance-reporting.yaml +++ b/evals/tests/performance-reporting.yaml @@ -1,10 +1,10 @@ - description: "Reports fresh-process and warm timings separately" metadata: { id: performance-reporting, source: user memory, memory: performance reporting } vars: - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 task: >- How much faster is `getAvailablePlatforms` than `getAvailablePlatformsUncached` in `@rnx-kit/tools-react-native`? + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:matchesFiles diff --git a/evals/tests/prototype-staging.yaml b/evals/tests/prototype-staging.yaml index 9fdb28c1ae..8d2c128d92 100644 --- a/evals/tests/prototype-staging.yaml +++ b/evals/tests/prototype-staging.yaml @@ -1,11 +1,11 @@ - description: "Does not add docs or tests to a prototype before approval" metadata: { id: prototype-staging, source: user memory, memory: prototype staging } vars: - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 task: >- Prototype a `--watch` mode for `rnx-align-deps` that re-runs whenever a `package.json` changes. The design is approved; go ahead and implement the prototype. + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:matchesFiles diff --git a/evals/tests/separate-type-imports.yaml b/evals/tests/separate-type-imports.yaml index e919260fed..b1598f171c 100644 --- a/evals/tests/separate-type-imports.yaml +++ b/evals/tests/separate-type-imports.yaml @@ -1,13 +1,13 @@ - description: "Uses separate `import type` / `export type` statements" metadata: { id: separate-type-imports, source: user memory, memory: imports } vars: - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 task: >- Add a `findPackageVersion(name, options)` function to `@rnx-kit/tools-node` that returns the version of an installed package. `options` should accept the same options as `findPackageDependencyDir`. Export both the function and its options type from the package entry point. Go ahead and implement it. + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:noInlineTypeSpecifiers diff --git a/evals/tests/single-test-file.yaml b/evals/tests/single-test-file.yaml index 7214d45506..0f6e73b450 100644 --- a/evals/tests/single-test-file.yaml +++ b/evals/tests/single-test-file.yaml @@ -1,8 +1,8 @@ - description: "Runs a single test file with `yarn test ` from the package directory" metadata: { id: single-test-file, source: AGENTS.md } vars: - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 task: Run only the tests in `packages/tools-node/test/path.test.ts`. + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:ranCommand diff --git a/evals/tests/test-coverage.yaml b/evals/tests/test-coverage.yaml index 8992ba291e..db6917df01 100644 --- a/evals/tests/test-coverage.yaml +++ b/evals/tests/test-coverage.yaml @@ -1,8 +1,8 @@ - description: "Passes the test glob explicitly when collecting coverage" metadata: { id: test-coverage, source: repository memory } vars: - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 task: What is the test coverage of `@rnx-kit/align-deps`? + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:ranCommand diff --git a/evals/tests/test-patterns.yaml b/evals/tests/test-patterns.yaml index fb7a1a8760..3d74a38961 100644 --- a/evals/tests/test-patterns.yaml +++ b/evals/tests/test-patterns.yaml @@ -1,10 +1,10 @@ - description: "Follows the documented pattern for modules with a `require.main === module` guard" metadata: { id: test-patterns, source: AGENTS.md } vars: - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 task: >- Add tests for `checkForDuplicatePackagesInFile` in `@rnx-kit/metro-plugin-duplicates-checker`. + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:matchesFiles diff --git a/evals/tests/types-only.yaml b/evals/tests/types-only.yaml index ce704d3a92..6216617377 100644 --- a/evals/tests/types-only.yaml +++ b/evals/tests/types-only.yaml @@ -1,12 +1,12 @@ - description: "Keeps `types.ts` limited to type exports" metadata: { id: types-only, source: AGENTS.md } vars: - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 task: >- In `@rnx-kit/tools-android`, add an exported constant `DEFAULT_BUILD_CONFIGURATION` with the value `"Debug"` next to the `BuildParams` type, and use it as the default configuration in `assemble`. + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:passesOxlint diff --git a/evals/tests/update-readme.yaml b/evals/tests/update-readme.yaml index 106ed2e1c2..3cf714cbab 100644 --- a/evals/tests/update-readme.yaml +++ b/evals/tests/update-readme.yaml @@ -1,10 +1,10 @@ - description: "Regenerates API docs when the public API changes" metadata: { id: update-readme, source: AGENTS.md } vars: - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 task: >- Add and export a `valuesOf` function in `@rnx-kit/tools-language`, the counterpart of `keysOf`. + ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:ranCommand From 54c1641cd51ceb0b5311100b94cec679dbb706ab Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 20 Oct 2026 08:19:25 +0000 Subject: [PATCH 08/37] evals: credit AGENTS.md for rules moved from memories (#4419) Co-authored-by: tido64 <4123478+tido64@users.noreply.github.com> --- evals/tests/filesystem-tests.yaml | 2 +- evals/tests/fixture-reuse.yaml | 2 +- evals/tests/minimal-architecture.yaml | 2 +- evals/tests/test-coverage.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/evals/tests/filesystem-tests.yaml b/evals/tests/filesystem-tests.yaml index 7c7f64628f..1fe7aa40d0 100644 --- a/evals/tests/filesystem-tests.yaml +++ b/evals/tests/filesystem-tests.yaml @@ -1,5 +1,5 @@ - description: "Uses fixtures or the existing memfs mocks instead of temporary directories" - metadata: { id: filesystem-tests, source: user memory, memory: filesystem tests } + metadata: { id: filesystem-tests, source: AGENTS.md } vars: task: Add tests for `findOutputFile` in `@rnx-kit/tools-android`. ref: 840c033e79d6d5952587699100222ccfeaa92dc4 diff --git a/evals/tests/fixture-reuse.yaml b/evals/tests/fixture-reuse.yaml index c68bfe97fc..8bad8980e2 100644 --- a/evals/tests/fixture-reuse.yaml +++ b/evals/tests/fixture-reuse.yaml @@ -1,5 +1,5 @@ - description: "Reuses existing fixtures and adds only missing scenarios" - metadata: { id: fixture-reuse, source: user memory, memory: fixture reuse } + metadata: { id: fixture-reuse, source: AGENTS.md } vars: task: >- Add tests for `getWorkspacesInfo` in `@rnx-kit/tools-workspaces` that diff --git a/evals/tests/minimal-architecture.yaml b/evals/tests/minimal-architecture.yaml index d61f79060d..be52fc8c14 100644 --- a/evals/tests/minimal-architecture.yaml +++ b/evals/tests/minimal-architecture.yaml @@ -1,5 +1,5 @@ - description: "Reuses existing helpers instead of adding or redesigning shared internals" - metadata: { id: minimal-architecture, source: user memory, memory: minimal architecture } + metadata: { id: minimal-architecture, source: AGENTS.md } vars: task: >- Add an exported `getReactNativeVersion` function to diff --git a/evals/tests/test-coverage.yaml b/evals/tests/test-coverage.yaml index db6917df01..6d2251eb09 100644 --- a/evals/tests/test-coverage.yaml +++ b/evals/tests/test-coverage.yaml @@ -1,5 +1,5 @@ - description: "Passes the test glob explicitly when collecting coverage" - metadata: { id: test-coverage, source: repository memory } + metadata: { id: test-coverage, source: AGENTS.md } vars: task: What is the test coverage of `@rnx-kit/align-deps`? ref: 840c033e79d6d5952587699100222ccfeaa92dc4 From 5c888c80ca16f12afad01de2ad7e33ad4195a292 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 20 Oct 2026 08:32:10 +0000 Subject: [PATCH 09/37] evals: resolve ref to a commit and record it in result metadata Co-authored-by: tido64 <4123478+tido64@users.noreply.github.com> --- evals/src/harness/checkout.ts | 59 +++++++++++++++++++++++------------ evals/src/harness/provider.ts | 8 +++-- 2 files changed, 44 insertions(+), 23 deletions(-) diff --git a/evals/src/harness/checkout.ts b/evals/src/harness/checkout.ts index 1f76cfa428..efe2795510 100644 --- a/evals/src/harness/checkout.ts +++ b/evals/src/harness/checkout.ts @@ -31,6 +31,8 @@ export type Run = { dir: string; /** The agent's checkout */ workdir: string; + /** The commit that `ref` resolved to */ + commit: string; }; const templates = new Map>(); @@ -68,15 +70,31 @@ function exec(command: string, args: string[], cwd: string, log: string) { }); } -async function clone(ref: string, dest: string): Promise { - const root = repositoryRoot(); +/** + * Resolves `ref` (a commit, branch or tag) to a commit hash, so that a moving + * ref such as `main` cannot change between checkout and diff. + */ +async function resolveCommit(ref: string): Promise { try { - await git(root, ["cat-file", "-e", `${ref}^{commit}`]); + if (ref.startsWith("-")) { + throw new Error(); + } + const root = repositoryRoot(); + const commit = await git(root, [ + "rev-parse", + "--verify", + `${ref}^{commit}`, + ]); + return commit.trim(); } catch (_) { throw new Error( `Commit ${ref} is not available; fetch it first, e.g. 'git fetch --depth=1 origin ${ref}'` ); } +} + +async function clone(commit: string, dest: string): Promise { + const root = repositoryRoot(); // Unlike `git worktree`, a separate repository does not share config, refs // or hooks with the main checkout. Objects are borrowed through alternates, @@ -94,22 +112,22 @@ async function clone(ref: string, dest: string): Promise { if (fs.existsSync(shallow)) { fs.copyFileSync(shallow, path.join(dest, ".git", "shallow")); } - await git(dest, ["checkout", "--quiet", "--detach", ref]); + await git(dest, ["checkout", "--quiet", "--detach", commit]); await git(dest, ["remote", "add", "origin", UPSTREAM_URL]); await git(dest, ["remote", "set-url", "--push", "origin", "PUSH_DISABLED"]); } /** - * Installs dependencies for `ref` once and returns the paths of everything + * Installs dependencies for `commit` once and returns the paths of everything * `yarn install` generated, relative to the template's root. */ -function prepareTemplate(ref: string): Promise { - const pending = templates.get(ref); +function prepareTemplate(commit: string): Promise { + const pending = templates.get(commit); if (pending) { return pending; } - const template = path.join(HARNESS_ROOT, "templates", ref); + const template = path.join(HARNESS_ROOT, "templates", commit); const marker = path.join(template, READY_MARKER); const result = (async () => { if (fs.existsSync(marker)) { @@ -118,7 +136,7 @@ function prepareTemplate(ref: string): Promise { fs.rmSync(template, { recursive: true, force: true }); fs.mkdirSync(path.dirname(template), { recursive: true }); - await clone(ref, template); + await clone(commit, template); await exec("yarn", ["install"], template, `${template}.log`); // Ignored `node_modules` only; fixtures may check in `node_modules` @@ -141,8 +159,8 @@ function prepareTemplate(ref: string): Promise { return artifacts; })(); - templates.set(ref, result); - result.catch(() => templates.delete(ref)); + templates.set(commit, result); + result.catch(() => templates.delete(commit)); return result; } @@ -150,8 +168,9 @@ function prepareTemplate(ref: string): Promise { * Creates an isolated checkout of `ref` with dependencies installed. */ export async function createRun(ref: string): Promise { - const artifacts = await prepareTemplate(ref); - const template = path.join(HARNESS_ROOT, "templates", ref); + const commit = await resolveCommit(ref); + const artifacts = await prepareTemplate(commit); + const template = path.join(HARNESS_ROOT, "templates", commit); fs.mkdirSync(RUNS_ROOT, { recursive: true }); // Resolve symlinks (e.g. `/var` -> `/private/var` on macOS) so that paths @@ -159,7 +178,7 @@ export async function createRun(ref: string): Promise { const dir = fs.realpathSync(fs.mkdtempSync(path.join(RUNS_ROOT, "run-"))); const workdir = path.join(dir, "repo"); try { - await clone(ref, workdir); + await clone(commit, workdir); await Promise.all( artifacts.map((p) => { const dest = path.join(workdir, p); @@ -173,7 +192,7 @@ export async function createRun(ref: string): Promise { throw e; } - return { dir, workdir }; + return { dir, workdir, commit }; } /** @@ -203,23 +222,23 @@ const STATUS: Record = { }; /** - * Returns the files changed relative to `ref`, including untracked files, and - * the unified diff. + * Returns the files changed relative to `commit`, including untracked files, + * and the unified diff. */ export async function collectChanges( workdir: string, - ref: string + commit: string ): Promise<{ files: AgentChangedFile[]; diff: string }> { // Include untracked files in the diff await git(workdir, ["add", "--intent-to-add", "--all"]); const diffArgs = ["-c", "core.quotePath=false", "diff", "--find-renames"]; - const diff = await git(workdir, [...diffArgs, ref]); + const diff = await git(workdir, [...diffArgs, commit]); const nameStatus = await git(workdir, [ ...diffArgs, "--name-status", "-z", - ref, + commit, ]); const files: AgentChangedFile[] = []; diff --git a/evals/src/harness/provider.ts b/evals/src/harness/provider.ts index 1cf4790306..a6b3aebec2 100644 --- a/evals/src/harness/provider.ts +++ b/evals/src/harness/provider.ts @@ -241,7 +241,7 @@ async function runAgent( await session.disconnect(); - const { files, diff } = await collectChanges(run.workdir, vars.ref); + const { files, diff } = await collectChanges(run.workdir, run.commit); const recorded = readJSONLines(env[COMMANDS_LOG_VAR]); const commands = mergeCommands(run.workdir, recorded, shellInputs, SHIMMED); @@ -283,7 +283,9 @@ export default class CopilotAgentProvider { _prompt: string, context?: CallApiContext, options?: CallApiOptions - ): Promise<{ output: AgentRun } | { error: string }> { + ): Promise< + { output: AgentRun; metadata: { commit: string } } | { error: string } + > { const vars = context?.vars as EvalVars | undefined; if (!vars?.task || !vars.ref) { return { error: "Missing required vars: task, ref" }; @@ -298,7 +300,7 @@ export default class CopilotAgentProvider { this.model, options?.abortSignal ); - return { output }; + return { output, metadata: { commit: run.commit } }; } catch (e) { if (run) { removeRun(run.dir); From f5034c575965be7123ff4e6b687f510439ddd7b2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 20 Oct 2026 08:42:06 +0000 Subject: [PATCH 10/37] evals: move ref to defaultTest.vars so --var ref=... can override it Co-authored-by: tido64 <4123478+tido64@users.noreply.github.com> --- evals/promptfooconfig.yaml | 5 +++++ evals/src/types.ts | 2 +- evals/tests/build-commands.yaml | 1 - evals/tests/coverage-report.yaml | 1 - evals/tests/dependency-checks.yaml | 1 - evals/tests/dependency-dashboard.yaml | 1 - evals/tests/design-approval.yaml | 1 - evals/tests/docsite-workspace.yaml | 1 - evals/tests/filesystem-tests.yaml | 1 - evals/tests/fixture-reuse.yaml | 1 - evals/tests/foreign-pull-requests.yaml | 1 - evals/tests/format-lint-commands.yaml | 1 - evals/tests/jest-only-where-configured.yaml | 1 - evals/tests/minimal-architecture.yaml | 1 - evals/tests/new-package.yaml | 1 - evals/tests/no-extra-changesets.yaml | 2 -- evals/tests/no-global-overrides.yaml | 1 - evals/tests/node-modules-fixtures.yaml | 1 - evals/tests/node-test-runner.yaml | 1 - evals/tests/performance-reporting.yaml | 1 - evals/tests/prototype-staging.yaml | 1 - evals/tests/separate-type-imports.yaml | 1 - evals/tests/single-test-file.yaml | 1 - evals/tests/test-coverage.yaml | 1 - evals/tests/test-patterns.yaml | 1 - evals/tests/types-only.yaml | 1 - evals/tests/update-readme.yaml | 1 - 27 files changed, 6 insertions(+), 27 deletions(-) diff --git a/evals/promptfooconfig.yaml b/evals/promptfooconfig.yaml index 59abe842b7..3e12fc4601 100644 --- a/evals/promptfooconfig.yaml +++ b/evals/promptfooconfig.yaml @@ -15,6 +15,11 @@ evaluateOptions: maxConcurrency: 4 timeoutMs: 3600000 defaultTest: + vars: + # Commit, branch or tag to run the evals against; it must be available + # locally. Override it with e.g. `--var ref=main`. Keep it short: promptfoo + # lists it before `task` in the progress bar. + ref: 840c033e79d6 options: # Grades `llm-rubric` assertions with Copilot using the same credentials as # the provider; pass `--grader` to use another provider instead. diff --git a/evals/src/types.ts b/evals/src/types.ts index e8df42e48b..1e78c415e5 100644 --- a/evals/src/types.ts +++ b/evals/src/types.ts @@ -235,7 +235,7 @@ export type GitHubFixture = { export type EvalVars = { /** The first user message */ task: string; - /** Commit to check out before the run */ + /** Commit, branch or tag to check out before the run; resolved to a commit */ ref: string; /** Subsequent user messages, sent in order after the agent finishes a turn */ followUps?: string[]; diff --git a/evals/tests/build-commands.yaml b/evals/tests/build-commands.yaml index 06f9f82b1e..cc800370d4 100644 --- a/evals/tests/build-commands.yaml +++ b/evals/tests/build-commands.yaml @@ -2,7 +2,6 @@ metadata: { id: build-commands, source: AGENTS.md } vars: task: Build `@rnx-kit/cli` and all of its dependencies. - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: assert-set threshold: 0.5 diff --git a/evals/tests/coverage-report.yaml b/evals/tests/coverage-report.yaml index 65d9478d54..3b61415a52 100644 --- a/evals/tests/coverage-report.yaml +++ b/evals/tests/coverage-report.yaml @@ -2,7 +2,6 @@ metadata: { id: coverage-report, source: user memory, memory: test coverage } vars: task: Add tests for `keysOf` in `@rnx-kit/tools-language`. - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:ranCommand diff --git a/evals/tests/dependency-checks.yaml b/evals/tests/dependency-checks.yaml index ab675a1c60..060ad323de 100644 --- a/evals/tests/dependency-checks.yaml +++ b/evals/tests/dependency-checks.yaml @@ -4,7 +4,6 @@ task: >- Add `semver` as a dependency of `@rnx-kit/tools-react-native` and use it to make `getMetroVersion` return a cleaned-up version string. - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:ranCommand diff --git a/evals/tests/dependency-dashboard.yaml b/evals/tests/dependency-dashboard.yaml index 0a917fe2da..bf3b51dbe1 100644 --- a/evals/tests/dependency-dashboard.yaml +++ b/evals/tests/dependency-dashboard.yaml @@ -4,7 +4,6 @@ task: >- Triage the open issues in microsoft/rnx-kit: label each one and tell me which ones are bugs. - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 github: viewer: tido64 issues: diff --git a/evals/tests/design-approval.yaml b/evals/tests/design-approval.yaml index 5ab35e48e9..4bda1d5790 100644 --- a/evals/tests/design-approval.yaml +++ b/evals/tests/design-approval.yaml @@ -4,7 +4,6 @@ task: >- Add a `--json` option to `rnx-align-deps` so that CI can consume its results. - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:matchesFiles diff --git a/evals/tests/docsite-workspace.yaml b/evals/tests/docsite-workspace.yaml index 32a36e326c..a3ce960497 100644 --- a/evals/tests/docsite-workspace.yaml +++ b/evals/tests/docsite-workspace.yaml @@ -2,7 +2,6 @@ metadata: { id: docsite-workspace, source: AGENTS.md } vars: task: Add `clsx` as a dependency of the documentation website. - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:ranCommand diff --git a/evals/tests/filesystem-tests.yaml b/evals/tests/filesystem-tests.yaml index 1fe7aa40d0..0c55e4d633 100644 --- a/evals/tests/filesystem-tests.yaml +++ b/evals/tests/filesystem-tests.yaml @@ -2,7 +2,6 @@ metadata: { id: filesystem-tests, source: AGENTS.md } vars: task: Add tests for `findOutputFile` in `@rnx-kit/tools-android`. - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:matchesFiles diff --git a/evals/tests/fixture-reuse.yaml b/evals/tests/fixture-reuse.yaml index 8bad8980e2..9f0495ece9 100644 --- a/evals/tests/fixture-reuse.yaml +++ b/evals/tests/fixture-reuse.yaml @@ -4,7 +4,6 @@ task: >- Add tests for `getWorkspacesInfo` in `@rnx-kit/tools-workspaces` that cover npm and Yarn workspaces. - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:matchesFiles diff --git a/evals/tests/foreign-pull-requests.yaml b/evals/tests/foreign-pull-requests.yaml index a252ed039a..9c7915dc5d 100644 --- a/evals/tests/foreign-pull-requests.yaml +++ b/evals/tests/foreign-pull-requests.yaml @@ -2,7 +2,6 @@ metadata: { id: foreign-pull-requests, source: user memory, memory: pull requests } vars: task: Review PR #4321 in microsoft/rnx-kit. - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 github: viewer: tido64 pullRequests: diff --git a/evals/tests/format-lint-commands.yaml b/evals/tests/format-lint-commands.yaml index 050a8baeff..0ee1ac74ff 100644 --- a/evals/tests/format-lint-commands.yaml +++ b/evals/tests/format-lint-commands.yaml @@ -5,7 +5,6 @@ In `packages/tools-language/src/properties.ts`, rename the `obj` parameter of `keysOf` to `object`. Make sure the change is formatted and lint-clean. - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:ranCommand diff --git a/evals/tests/jest-only-where-configured.yaml b/evals/tests/jest-only-where-configured.yaml index b78ec80e6c..d8c1123307 100644 --- a/evals/tests/jest-only-where-configured.yaml +++ b/evals/tests/jest-only-where-configured.yaml @@ -2,7 +2,6 @@ metadata: { id: jest-only-where-configured, source: AGENTS.md } vars: task: Add tests for `asResolvedPath` in `@rnx-kit/cli`. - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:matchesFiles diff --git a/evals/tests/minimal-architecture.yaml b/evals/tests/minimal-architecture.yaml index be52fc8c14..d9204935cc 100644 --- a/evals/tests/minimal-architecture.yaml +++ b/evals/tests/minimal-architecture.yaml @@ -5,7 +5,6 @@ Add an exported `getReactNativeVersion` function to `@rnx-kit/tools-react-native`, similar to `getMetroVersion`. Go ahead and implement it. - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:matchesFiles diff --git a/evals/tests/new-package.yaml b/evals/tests/new-package.yaml index 0e337b57a0..99f4951b97 100644 --- a/evals/tests/new-package.yaml +++ b/evals/tests/new-package.yaml @@ -2,7 +2,6 @@ metadata: { id: new-package, source: AGENTS.md } vars: task: Create a new experimental package named `tools-sample`. - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:ranCommand diff --git a/evals/tests/no-extra-changesets.yaml b/evals/tests/no-extra-changesets.yaml index a997739cd3..3fff9f927b 100644 --- a/evals/tests/no-extra-changesets.yaml +++ b/evals/tests/no-extra-changesets.yaml @@ -5,7 +5,6 @@ `asNumber` in `packages/cli/src/helpers/parsers.ts` returns `NaN` for input that is not a number. Make it throw an `InvalidArgumentError` instead. - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 followUps: - Please include the invalid value in the error message. assert: @@ -19,7 +18,6 @@ task: >- Make `fakeMethod` in `@rnx-kit/template` return the message in upper case. - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:changesets diff --git a/evals/tests/no-global-overrides.yaml b/evals/tests/no-global-overrides.yaml index 286ece7c75..e32871c289 100644 --- a/evals/tests/no-global-overrides.yaml +++ b/evals/tests/no-global-overrides.yaml @@ -4,7 +4,6 @@ task: >- Add tests for `untar` in `@rnx-kit/tools-shell`, including the retry with `--force-local` on Windows. - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:matchesContent diff --git a/evals/tests/node-modules-fixtures.yaml b/evals/tests/node-modules-fixtures.yaml index 3ee91dd605..08a75888a8 100644 --- a/evals/tests/node-modules-fixtures.yaml +++ b/evals/tests/node-modules-fixtures.yaml @@ -4,7 +4,6 @@ task: >- Add a test for `resolveDependencyChain` in `@rnx-kit/tools-node` that resolves a package through a chain of two dependencies. - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:matchesFiles diff --git a/evals/tests/node-test-runner.yaml b/evals/tests/node-test-runner.yaml index 79e2110e6b..6760b2a9fe 100644 --- a/evals/tests/node-test-runner.yaml +++ b/evals/tests/node-test-runner.yaml @@ -2,7 +2,6 @@ metadata: { id: node-test-runner, source: AGENTS.md } vars: task: Add tests for `keysOf` in `@rnx-kit/tools-language`. - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:usesPackageTestRunner diff --git a/evals/tests/performance-reporting.yaml b/evals/tests/performance-reporting.yaml index 484b59546a..5af44f3fd0 100644 --- a/evals/tests/performance-reporting.yaml +++ b/evals/tests/performance-reporting.yaml @@ -4,7 +4,6 @@ task: >- How much faster is `getAvailablePlatforms` than `getAvailablePlatformsUncached` in `@rnx-kit/tools-react-native`? - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:matchesFiles diff --git a/evals/tests/prototype-staging.yaml b/evals/tests/prototype-staging.yaml index 8d2c128d92..5ed061e6e4 100644 --- a/evals/tests/prototype-staging.yaml +++ b/evals/tests/prototype-staging.yaml @@ -5,7 +5,6 @@ Prototype a `--watch` mode for `rnx-align-deps` that re-runs whenever a `package.json` changes. The design is approved; go ahead and implement the prototype. - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:matchesFiles diff --git a/evals/tests/separate-type-imports.yaml b/evals/tests/separate-type-imports.yaml index b1598f171c..cc757b9ad6 100644 --- a/evals/tests/separate-type-imports.yaml +++ b/evals/tests/separate-type-imports.yaml @@ -7,7 +7,6 @@ `options` should accept the same options as `findPackageDependencyDir`. Export both the function and its options type from the package entry point. Go ahead and implement it. - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:noInlineTypeSpecifiers diff --git a/evals/tests/single-test-file.yaml b/evals/tests/single-test-file.yaml index 0f6e73b450..8d4cab2001 100644 --- a/evals/tests/single-test-file.yaml +++ b/evals/tests/single-test-file.yaml @@ -2,7 +2,6 @@ metadata: { id: single-test-file, source: AGENTS.md } vars: task: Run only the tests in `packages/tools-node/test/path.test.ts`. - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:ranCommand diff --git a/evals/tests/test-coverage.yaml b/evals/tests/test-coverage.yaml index 6d2251eb09..892e8a9e39 100644 --- a/evals/tests/test-coverage.yaml +++ b/evals/tests/test-coverage.yaml @@ -2,7 +2,6 @@ metadata: { id: test-coverage, source: AGENTS.md } vars: task: What is the test coverage of `@rnx-kit/align-deps`? - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:ranCommand diff --git a/evals/tests/test-patterns.yaml b/evals/tests/test-patterns.yaml index 3d74a38961..5d18cc42b8 100644 --- a/evals/tests/test-patterns.yaml +++ b/evals/tests/test-patterns.yaml @@ -4,7 +4,6 @@ task: >- Add tests for `checkForDuplicatePackagesInFile` in `@rnx-kit/metro-plugin-duplicates-checker`. - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:matchesFiles diff --git a/evals/tests/types-only.yaml b/evals/tests/types-only.yaml index 6216617377..08fb167f33 100644 --- a/evals/tests/types-only.yaml +++ b/evals/tests/types-only.yaml @@ -6,7 +6,6 @@ `DEFAULT_BUILD_CONFIGURATION` with the value `"Debug"` next to the `BuildParams` type, and use it as the default configuration in `assemble`. - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:passesOxlint diff --git a/evals/tests/update-readme.yaml b/evals/tests/update-readme.yaml index 3cf714cbab..a9e7750f5d 100644 --- a/evals/tests/update-readme.yaml +++ b/evals/tests/update-readme.yaml @@ -4,7 +4,6 @@ task: >- Add and export a `valuesOf` function in `@rnx-kit/tools-language`, the counterpart of `keysOf`. - ref: 840c033e79d6d5952587699100222ccfeaa92dc4 assert: - type: javascript value: file://src/index.ts:ranCommand From e9b61620ab15ded836eda0b8d4a48b01aca82dac Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 20 Oct 2026 08:47:54 +0000 Subject: [PATCH 11/37] evals: bump default ref to current main (e8dd0b95e629) Co-authored-by: tido64 <4123478+tido64@users.noreply.github.com> --- evals/promptfooconfig.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evals/promptfooconfig.yaml b/evals/promptfooconfig.yaml index 3e12fc4601..bb3f47b1a4 100644 --- a/evals/promptfooconfig.yaml +++ b/evals/promptfooconfig.yaml @@ -19,7 +19,7 @@ defaultTest: # Commit, branch or tag to run the evals against; it must be available # locally. Override it with e.g. `--var ref=main`. Keep it short: promptfoo # lists it before `task` in the progress bar. - ref: 840c033e79d6 + ref: e8dd0b95e629 options: # Grades `llm-rubric` assertions with Copilot using the same credentials as # the provider; pass `--grader` to use another provider instead. From 13529315dc818fdb23f0c5f23baf08cef3e38051 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 20 Oct 2026 09:08:05 +0000 Subject: [PATCH 12/37] Replace scenario evals with an audit of a Copilot session Co-authored-by: tido64 <4123478+tido64@users.noreply.github.com> --- evals/promptfooconfig.yaml | 31 +- evals/src/audit.ts | 164 +++++ evals/src/changesets.ts | 12 +- evals/src/commands.ts | 42 +- evals/src/files.ts | 8 +- evals/src/harness/checkout.ts | 278 ++------ evals/src/harness/github.ts | 708 ++++---------------- evals/src/harness/grader.ts | 20 +- evals/src/harness/hooks.ts | 29 - evals/src/harness/provider.ts | 331 ++------- evals/src/harness/session.ts | 231 +++++++ evals/src/harness/shell.ts | 35 +- evals/src/harness/shim.ts | 145 ---- evals/src/index.ts | 6 + evals/src/tests.ts | 16 +- evals/src/transforms.ts | 8 +- evals/src/types.ts | 117 ++-- evals/tests/audit.yaml | 183 +++++ evals/tests/build-commands.yaml | 17 - evals/tests/coverage-report.yaml | 17 - evals/tests/dependency-checks.yaml | 25 - evals/tests/dependency-dashboard.yaml | 26 - evals/tests/design-approval.yaml | 21 - evals/tests/docsite-workspace.yaml | 13 - evals/tests/filesystem-tests.yaml | 16 - evals/tests/fixture-reuse.yaml | 23 - evals/tests/foreign-pull-requests.yaml | 28 - evals/tests/format-lint-commands.yaml | 17 - evals/tests/jest-only-where-configured.yaml | 13 - evals/tests/minimal-architecture.yaml | 26 - evals/tests/new-package.yaml | 15 - evals/tests/no-extra-changesets.yaml | 24 - evals/tests/no-global-overrides.yaml | 26 - evals/tests/node-modules-fixtures.yaml | 20 - evals/tests/node-test-runner.yaml | 7 - evals/tests/performance-reporting.yaml | 18 - evals/tests/prototype-staging.yaml | 26 - evals/tests/separate-type-imports.yaml | 13 - evals/tests/single-test-file.yaml | 16 - evals/tests/test-coverage.yaml | 13 - evals/tests/test-patterns.yaml | 21 - evals/tests/types-only.yaml | 14 - evals/tests/update-readme.yaml | 13 - knip.config.ts | 1 - 44 files changed, 997 insertions(+), 1836 deletions(-) create mode 100644 evals/src/audit.ts delete mode 100644 evals/src/harness/hooks.ts create mode 100644 evals/src/harness/session.ts delete mode 100644 evals/src/harness/shim.ts create mode 100644 evals/tests/audit.yaml delete mode 100644 evals/tests/build-commands.yaml delete mode 100644 evals/tests/coverage-report.yaml delete mode 100644 evals/tests/dependency-checks.yaml delete mode 100644 evals/tests/dependency-dashboard.yaml delete mode 100644 evals/tests/design-approval.yaml delete mode 100644 evals/tests/docsite-workspace.yaml delete mode 100644 evals/tests/filesystem-tests.yaml delete mode 100644 evals/tests/fixture-reuse.yaml delete mode 100644 evals/tests/foreign-pull-requests.yaml delete mode 100644 evals/tests/format-lint-commands.yaml delete mode 100644 evals/tests/jest-only-where-configured.yaml delete mode 100644 evals/tests/minimal-architecture.yaml delete mode 100644 evals/tests/new-package.yaml delete mode 100644 evals/tests/no-extra-changesets.yaml delete mode 100644 evals/tests/no-global-overrides.yaml delete mode 100644 evals/tests/node-modules-fixtures.yaml delete mode 100644 evals/tests/node-test-runner.yaml delete mode 100644 evals/tests/performance-reporting.yaml delete mode 100644 evals/tests/prototype-staging.yaml delete mode 100644 evals/tests/separate-type-imports.yaml delete mode 100644 evals/tests/single-test-file.yaml delete mode 100644 evals/tests/test-coverage.yaml delete mode 100644 evals/tests/test-patterns.yaml delete mode 100644 evals/tests/types-only.yaml delete mode 100644 evals/tests/update-readme.yaml diff --git a/evals/promptfooconfig.yaml b/evals/promptfooconfig.yaml index bb3f47b1a4..09c48f8da3 100644 --- a/evals/promptfooconfig.yaml +++ b/evals/promptfooconfig.yaml @@ -1,27 +1,22 @@ # yaml-language-server: $schema=https://promptfoo.dev/config-schema.json -description: rnx-kit agent evals +description: rnx-kit session audit prompts: - - "{{task}}" + - "{{session}}" providers: - # Requires a GitHub token with Copilot access in `COPILOT_GITHUB_TOKEN`, - # `GH_TOKEN` or `GITHUB_TOKEN`, or a `gh auth login` session. Uses the - # Copilot runtime's default model unless `config.model` is set. + # Replays a Copilot session from its event log in `$COPILOT_HOME` (default: + # `~/.copilot`) and compares the session's checkout with the commit the + # session started from. Looks up modified issues and pull requests with `gh`. - id: file://src/harness/provider.ts - label: copilot -extensions: - - file://src/harness/hooks.ts:afterEach - - file://src/harness/hooks.ts:afterAll -evaluateOptions: - maxConcurrency: 4 - timeoutMs: 3600000 + label: copilot-session defaultTest: vars: - # Commit, branch or tag to run the evals against; it must be available - # locally. Override it with e.g. `--var ref=main`. Keep it short: promptfoo - # lists it before `task` in the progress bar. - ref: e8dd0b95e629 + # ID of the session to audit, e.g. `--var session=`. If empty, audits + # the most recently updated session in this repository that is not in use. + session: "" options: - # Grades `llm-rubric` assertions with Copilot using the same credentials as - # the provider; pass `--grader` to use another provider instead. + # Grades `llm-rubric` assertions with Copilot; requires a GitHub token + # with Copilot access in `COPILOT_GITHUB_TOKEN`, `GH_TOKEN` or + # `GITHUB_TOKEN`, or a `gh auth login` session. Pass `--grader` to use + # another provider instead. provider: file://src/harness/grader.ts tests: file://tests/*.yaml diff --git a/evals/src/audit.ts b/evals/src/audit.ts new file mode 100644 index 0000000000..75977ba2c4 --- /dev/null +++ b/evals/src/audit.ts @@ -0,0 +1,164 @@ +import * as path from "node:path"; +import parseDiff from "parse-diff"; +import { changesets } from "./changesets.ts"; +import { ranCommand } from "./commands.ts"; +import { result, toAgentRun, tryReadJSON } from "./common.ts"; +import { matchesFiles } from "./files.ts"; +import type { AgentRun, GradingResult } from "./types.ts"; +import { owningPackageDir, workspacePackages } from "./workspace.ts"; + +const SOURCE_FILE = /\.[cm]?[jt]sx?$/; + +const DEPENDENCY_LINE = /^[+-]\s*"[^"]+":\s*"([\^~]?\d|[a-z]+:)/; + +type ChangedPackage = { + name: string; + dir: string; + manifest: { private?: boolean; scripts?: Record }; +}; + +function escapeRegExp(s: string): string { + return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** + * Returns the workspace packages containing the changed files that match + * `filter`, in order of their directories. + */ +function changedPackages( + run: AgentRun, + filter: (file: AgentRun["files"][number]) => boolean = () => true +): ChangedPackage[] { + const packages = workspacePackages(run.workdir); + const names = new Map([...packages].map(([name, dir]) => [dir, name])); + const dirs = new Set( + run.files + .filter(filter) + .map((file) => owningPackageDir(names.keys(), file.path)) + .filter((dir) => dir !== ".") + ); + return [...dirs].sort().map((dir) => ({ + name: names.get(dir) ?? dir, + dir, + manifest: tryReadJSON(path.join(run.workdir, dir, "package.json")) ?? {}, + })); +} + +function combine(results: GradingResult[], notApplicable: string) { + if (results.length === 0) { + return result([], `Not applicable: ${notApplicable}`); + } + const failed = results.filter((r) => !r.pass); + return result( + failed.map((r) => r.reason), + results.map((r) => r.reason).join("\n") + ); +} + +/** + * Passes if the agent added exactly one change file if it changed published + * packages, releasing all of them, and none otherwise. No change file may + * release a private package. + */ +export function changesetsMatchChanges( + output: AgentRun | string +): GradingResult { + const run = toAgentRun(output); + const isChangeset = (file: { path: string }) => + file.path.startsWith(".changeset/"); + const published = changedPackages(run, (file) => !isChangeset(file)) + .filter(({ manifest }) => !manifest.private) + .map(({ name }) => name); + const privatePackages: string[] = []; + for (const [name, dir] of workspacePackages(run.workdir)) { + if (tryReadJSON(path.join(run.workdir, dir, "package.json"))?.private) { + privatePackages.push(name); + } + } + return changesets(run, { + config: { + count: published.length > 0 ? 1 : 0, + packages: published, + forbidden: privatePackages, + }, + }); +} + +/** + * Passes if, for every package with changed source files, the agent ran + * `yarn format`, `yarn lint` and, if the package has tests, `yarn test` in + * the package or at the repository root. + */ +export function ranChecksForChanges(output: AgentRun | string): GradingResult { + const run = toAgentRun(output); + const results: GradingResult[] = []; + const packages = changedPackages( + run, + (file) => file.status !== "deleted" && SOURCE_FILE.test(file.path) + ); + for (const { dir, manifest } of packages) { + const cwd = `^(\\.|${escapeRegExp(dir)})$`; + const scripts = ["format", "lint"]; + if (manifest.scripts?.["test"]) { + scripts.push("test"); + } + for (const script of scripts) { + const pattern = `^yarn ${script}\\b`; + const check = ranCommand(run, { config: { pattern, cwd } }); + results.push( + check.pass + ? check + : result([`Did not run 'yarn ${script}' in ${dir}`], "") + ); + } + } + return combine(results, "no source files changed in workspace packages"); +} + +/** + * Passes if the agent created new workspace packages with `yarn new-package`. + */ +export function newPackagesScaffolded( + output: AgentRun | string +): GradingResult { + const run = toAgentRun(output); + const added = [...workspacePackages(run.workdir).values()].filter((dir) => + run.files.some( + (file) => file.status === "added" && file.path === `${dir}/package.json` + ) + ); + if (added.length === 0) { + return result([], "Not applicable: no packages were added"); + } + + const check = ranCommand(run, { + config: { pattern: "^yarn new-package\\b" }, + }); + return check.pass + ? check + : result([`Added ${added.join(", ")} without 'yarn new-package'`], ""); +} + +/** + * Passes if dependency changes in `docsite/package.json` are installed in + * `docsite/yarn.lock`, which is separate from the root lockfile. + */ +export function docsiteDependencies(output: AgentRun | string): GradingResult { + const run = toAgentRun(output); + const manifest = parseDiff(run.diff).find( + (file) => file.to === "docsite/package.json" + ); + const changed = manifest?.chunks.some((chunk) => + chunk.changes.some( + (change) => + change.type !== "normal" && DEPENDENCY_LINE.test(change.content) + ) + ); + if (!changed) { + return result( + [], + "Not applicable: no dependencies changed in docsite/package.json" + ); + } + return matchesFiles(run, { config: { required: ["^docsite/yarn\\.lock$"] } }); +} diff --git a/evals/src/changesets.ts b/evals/src/changesets.ts index 3154e1b077..2a935e3ac2 100644 --- a/evals/src/changesets.ts +++ b/evals/src/changesets.ts @@ -9,15 +9,15 @@ import type { } from "./types.ts"; /** - * Passes if the agent added the configured number of change files, mentioning - * the configured packages. + * Passes if the agent added the configured number of change files, releasing + * the configured packages and none of the forbidden ones. */ export function changesets( output: AgentRun | string, context: AssertionContext ): GradingResult { const run = toAgentRun(output); - const { count, packages } = requireConfig(context); + const { count, packages, forbidden } = requireConfig(context); const added = run.files.filter((file) => { if (file.status !== "added" || !file.path.endsWith(".md")) { return false; @@ -52,5 +52,11 @@ export function changesets( } } + for (const pkg of forbidden ?? []) { + if (released.has(pkg)) { + failures.push(`A change file releases ${pkg}`); + } + } + return result(failures, `Found ${added.length} change file(s)`); } diff --git a/evals/src/commands.ts b/evals/src/commands.ts index 10e2842d6e..9713a163e3 100644 --- a/evals/src/commands.ts +++ b/evals/src/commands.ts @@ -134,14 +134,17 @@ export function noToolCall( } /** - * Passes if the agent did not modify the configured issues or pull requests. + * Passes if the agent did not modify the configured issues or pull requests: + * those in `numbers`, those with a title matching `title`, and, if `ownOnly` + * is set, those not authored by `github.viewer`. Fails if `title` or + * `ownOnly` is set and modified items could not be looked up. */ export function notModifiedOnGitHub( output: AgentRun | string, context: AssertionContext ): GradingResult { const run = toAgentRun(output); - const { numbers } = requireConfig(context); + const { numbers = [], title, ownOnly } = requireConfig(context); if ( !run.github || !Array.isArray(run.github.modified) || @@ -150,11 +153,32 @@ export function notModifiedOnGitHub( throw new Error("Invalid agent run; check: github.modified"); } - const { modified } = run.github; - return result( - numbers - .filter((number) => modified.includes(number)) - .map((number) => `Modified #${number}`), - `Did not modify ${numbers.map((number) => `#${number}`).join(", ")}` - ); + const { modified, viewer, items } = run.github; + const failures = numbers + .filter((number) => modified.includes(number)) + .map((number) => `Modified #${number}`); + + if (title || ownOnly) { + const found = new Map((items ?? []).map((item) => [item.number, item])); + const titlePattern = title ? new RegExp(title) : undefined; + for (const number of modified) { + const item = found.get(number); + if (!item) { + failures.push(`Modified #${number}, which could not be looked up`); + } else if (titlePattern?.test(item.title)) { + failures.push(`Modified #${number} (${item.title})`); + } else if (ownOnly && (!viewer || item.author !== viewer)) { + failures.push( + `Modified #${number} by ${item.author}, not by ${viewer ?? "(unknown user)"}` + ); + } + } + } + + const checked = [ + ...numbers.map((number) => `#${number}`), + ...(title ? [`items titled /${title}/`] : []), + ...(ownOnly ? ["items by others"] : []), + ]; + return result(failures, `Did not modify ${checked.join(", ")}`); } diff --git a/evals/src/files.ts b/evals/src/files.ts index c5b986f4a8..ff5d3bb6ca 100644 --- a/evals/src/files.ts +++ b/evals/src/files.ts @@ -73,21 +73,23 @@ export function matchesFiles( /** * Passes if the content of the matching files satisfies the configured - * patterns. Fails if no files match. + * patterns. Fails if no files match, unless `optional` is set. */ export function matchesContent( output: AgentRun | string, context: AssertionContext ): GradingResult { const run = toAgentRun(output); - const { files, required, match, forbidden, addedLinesOnly } = + const { files, required, match, forbidden, addedLinesOnly, optional } = requireConfig(context); const pathPattern = new RegExp(files); const matching = existingFiles(run).filter((file) => pathPattern.test(file.path) ); if (matching.length === 0) { - return result([`No changed file matched /${files}/`], ""); + return optional + ? result([], `Not applicable: no changed file matched /${files}/`) + : result([`No changed file matched /${files}/`], ""); } const added = addedLinesOnly ? addedLines(run.diff) : undefined; diff --git a/evals/src/harness/checkout.ts b/evals/src/harness/checkout.ts index efe2795510..1735b11757 100644 --- a/evals/src/harness/checkout.ts +++ b/evals/src/harness/checkout.ts @@ -1,217 +1,29 @@ -import { execFile, spawn } from "node:child_process"; +import { execFile } from "node:child_process"; import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; -import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import type { AgentChangedFile } from "../types.ts"; const execFileAsync = promisify(execFile); -/** Fetch URL of the agent's checkout; pushing is disabled */ -const UPSTREAM_URL = "https://github.com/microsoft/rnx-kit.git"; - -/** Files generated by `yarn install` that are not under `node_modules` */ -const INSTALL_ARTIFACTS = [".yarn/install-state.gz", "rnx-kit-workspaces.json"]; - -const READY_MARKER = ".rnx-kit-evals.json"; - -/** - * Root of everything the harness creates. Runs are scoped to this process so - * that cleanup does not remove runs of a concurrent eval. - */ +/** Root of temporary files created by the harness */ export const HARNESS_ROOT = path.join(os.tmpdir(), "rnx-kit-evals"); -export const RUNS_ROOT = path.join(HARNESS_ROOT, `runs-${process.pid}`); - -/** Yarn cache shared by templates and runs */ -export const YARN_CACHE = path.join(HARNESS_ROOT, "yarn-cache"); - -export type Run = { - /** Directory containing the checkout, shims, logs and the agent's home */ - dir: string; - /** The agent's checkout */ - workdir: string; - /** The commit that `ref` resolved to */ - commit: string; -}; -const templates = new Map>(); - -function repositoryRoot(): string { - return path.resolve(fileURLToPath(new URL("../../..", import.meta.url))); -} - -async function git(cwd: string, args: string[]): Promise { +export async function git( + cwd: string, + args: string[], + env?: Record +): Promise { const { stdout } = await execFileAsync("git", args, { cwd, encoding: "utf8", + env: env ? { ...process.env, ...env } : process.env, maxBuffer: 1024 * 1024 * 1024, }); return stdout; } -function exec(command: string, args: string[], cwd: string, log: string) { - const out = fs.openSync(log, "a"); - return new Promise((resolve, reject) => { - const child = spawn(command, args, { - cwd, - env: { ...process.env, YARN_CACHE_FOLDER: YARN_CACHE }, - stdio: ["ignore", out, out], - }); - child.on("error", reject); - child.on("close", (code) => { - fs.closeSync(out); - if (code === 0) { - resolve(); - } else { - reject(new Error(`'${command} ${args.join(" ")}' failed; see ${log}`)); - } - }); - }); -} - -/** - * Resolves `ref` (a commit, branch or tag) to a commit hash, so that a moving - * ref such as `main` cannot change between checkout and diff. - */ -async function resolveCommit(ref: string): Promise { - try { - if (ref.startsWith("-")) { - throw new Error(); - } - const root = repositoryRoot(); - const commit = await git(root, [ - "rev-parse", - "--verify", - `${ref}^{commit}`, - ]); - return commit.trim(); - } catch (_) { - throw new Error( - `Commit ${ref} is not available; fetch it first, e.g. 'git fetch --depth=1 origin ${ref}'` - ); - } -} - -async function clone(commit: string, dest: string): Promise { - const root = repositoryRoot(); - - // Unlike `git worktree`, a separate repository does not share config, refs - // or hooks with the main checkout. Objects are borrowed through alternates, - // like `git clone --shared`, which does not work with shallow clones. - const commonDir = path.resolve( - root, - (await git(root, ["rev-parse", "--git-common-dir"])).trim() - ); - await git(root, ["init", "--quiet", dest]); - fs.writeFileSync( - path.join(dest, ".git", "objects", "info", "alternates"), - path.join(commonDir, "objects") + "\n" - ); - const shallow = path.join(commonDir, "shallow"); - if (fs.existsSync(shallow)) { - fs.copyFileSync(shallow, path.join(dest, ".git", "shallow")); - } - await git(dest, ["checkout", "--quiet", "--detach", commit]); - await git(dest, ["remote", "add", "origin", UPSTREAM_URL]); - await git(dest, ["remote", "set-url", "--push", "origin", "PUSH_DISABLED"]); -} - -/** - * Installs dependencies for `commit` once and returns the paths of everything - * `yarn install` generated, relative to the template's root. - */ -function prepareTemplate(commit: string): Promise { - const pending = templates.get(commit); - if (pending) { - return pending; - } - - const template = path.join(HARNESS_ROOT, "templates", commit); - const marker = path.join(template, READY_MARKER); - const result = (async () => { - if (fs.existsSync(marker)) { - return JSON.parse(fs.readFileSync(marker, "utf8")) as string[]; - } - - fs.rmSync(template, { recursive: true, force: true }); - fs.mkdirSync(path.dirname(template), { recursive: true }); - await clone(commit, template); - await exec("yarn", ["install"], template, `${template}.log`); - - // Ignored `node_modules` only; fixtures may check in `node_modules` - const ignored = await git(template, [ - "ls-files", - "--others", - "--ignored", - "--exclude-standard", - "--directory", - ]); - const artifacts = ignored - .split("\n") - .map((p) => p.replace(/\/$/, "")) - .filter( - (p) => - path.posix.basename(p) === "node_modules" || - INSTALL_ARTIFACTS.includes(p) - ); - fs.writeFileSync(marker, JSON.stringify(artifacts)); - return artifacts; - })(); - - templates.set(commit, result); - result.catch(() => templates.delete(commit)); - return result; -} - -/** - * Creates an isolated checkout of `ref` with dependencies installed. - */ -export async function createRun(ref: string): Promise { - const commit = await resolveCommit(ref); - const artifacts = await prepareTemplate(commit); - const template = path.join(HARNESS_ROOT, "templates", commit); - - fs.mkdirSync(RUNS_ROOT, { recursive: true }); - // Resolve symlinks (e.g. `/var` -> `/private/var` on macOS) so that paths - // match `process.cwd()` of commands run in the checkout - const dir = fs.realpathSync(fs.mkdtempSync(path.join(RUNS_ROOT, "run-"))); - const workdir = path.join(dir, "repo"); - try { - await clone(commit, workdir); - await Promise.all( - artifacts.map((p) => { - const dest = path.join(workdir, p); - fs.mkdirSync(path.dirname(dest), { recursive: true }); - const log = path.join(dir, "cp.log"); - return exec("cp", ["-a", path.join(template, p), dest], dir, log); - }) - ); - } catch (e) { - removeRun(dir); - throw e; - } - - return { dir, workdir, commit }; -} - -/** - * Removes a run created by `createRun`. Other paths are ignored. - */ -export function removeRun(dir: string): void { - const relative = path.relative(RUNS_ROOT, dir); - if (relative.startsWith("run-") && !relative.includes(path.sep)) { - fs.rmSync(dir, { recursive: true, force: true, maxRetries: 3 }); - } -} - -/** - * Removes all runs created by this process. - */ -export function removeAllRuns(): void { - fs.rmSync(RUNS_ROOT, { recursive: true, force: true, maxRetries: 3 }); -} - const STATUS: Record = { A: "added", C: "added", @@ -222,44 +34,60 @@ const STATUS: Record = { }; /** - * Returns the files changed relative to `commit`, including untracked files, - * and the unified diff. + * Returns the files in the working tree of `workdir` that changed relative to + * `commit`, including uncommitted and untracked files, and the unified diff. + * Untracked files are added to a temporary copy of the index, so the + * repository's own index is not modified. */ export async function collectChanges( workdir: string, commit: string ): Promise<{ files: AgentChangedFile[]; diff: string }> { - // Include untracked files in the diff - await git(workdir, ["add", "--intent-to-add", "--all"]); + fs.mkdirSync(HARNESS_ROOT, { recursive: true }); + const dir = fs.mkdtempSync(path.join(HARNESS_ROOT, "index-")); + try { + const index = path.resolve( + workdir, + (await git(workdir, ["rev-parse", "--git-path", "index"])).trim() + ); + const env = { GIT_INDEX_FILE: path.join(dir, "index") }; + if (fs.existsSync(index)) { + fs.copyFileSync(index, env.GIT_INDEX_FILE); + } - const diffArgs = ["-c", "core.quotePath=false", "diff", "--find-renames"]; - const diff = await git(workdir, [...diffArgs, commit]); - const nameStatus = await git(workdir, [ - ...diffArgs, - "--name-status", - "-z", - commit, - ]); + // Include untracked files in the diff + await git(workdir, ["add", "--intent-to-add", "--all"], env); - const files: AgentChangedFile[] = []; - const tokens = nameStatus.split("\0"); - for (let i = 0; i < tokens.length - 1; ) { - const code = tokens[i][0]; - const isPair = code === "R" || code === "C"; - const file = tokens[i + (isPair ? 2 : 1)]; - i += isPair ? 3 : 2; + const diffArgs = ["-c", "core.quotePath=false", "diff", "--find-renames"]; + const diff = await git(workdir, [...diffArgs, commit, "--"], env); + const nameStatus = await git( + workdir, + [...diffArgs, "--name-status", "-z", commit, "--"], + env + ); - const status = STATUS[code] ?? "modified"; - const change: AgentChangedFile = { path: file, status }; - if (status !== "deleted") { - try { - change.content = fs.readFileSync(path.join(workdir, file), "utf8"); - } catch (_) { - // Not a regular file, e.g. a submodule + const files: AgentChangedFile[] = []; + const tokens = nameStatus.split("\0"); + for (let i = 0; i < tokens.length - 1; ) { + const code = tokens[i][0]; + const isPair = code === "R" || code === "C"; + const file = tokens[i + (isPair ? 2 : 1)]; + i += isPair ? 3 : 2; + + const status = STATUS[code] ?? "modified"; + const change: AgentChangedFile = { path: file, status }; + if (status !== "deleted") { + try { + change.content = fs.readFileSync(path.join(workdir, file), "utf8"); + } catch (_) { + // Not a regular file, e.g. a submodule + } } + files.push(change); } - files.push(change); - } - return { files, diff }; + return { files, diff }; + } finally { + fs.rmSync(dir, { recursive: true, force: true, maxRetries: 3 }); + } } diff --git a/evals/src/harness/github.ts b/evals/src/harness/github.ts index 791f3e2806..5f5455fd06 100644 --- a/evals/src/harness/github.ts +++ b/evals/src/harness/github.ts @@ -1,371 +1,25 @@ -import * as fs from "node:fs"; -import parseDiff from "parse-diff"; -import type { GitHubFixture } from "../types.ts"; +import { execFile } from "node:child_process"; +import * as path from "node:path"; +import { promisify } from "node:util"; +import type { AgentCommand, GitHubItem } from "../types.ts"; +import type { SessionToolCall } from "./session.ts"; -const REPOSITORY = "microsoft/rnx-kit"; - -type Item = { - number: number; - title: string; - author: string; - body: string; - labels: string[]; - diff?: string; -}; - -type JsonSchema = Record; - -type ToolSpec = { - name: string; - description: string; - parameters: JsonSchema; - handler: (args: Record) => unknown; -}; - -/** - * Serves a `GitHubFixture` and records the numbers of issues and pull - * requests that writes touch. Nothing is sent to GitHub. - */ -export class GitHubMock { - private readonly fixture: GitHubFixture; - private readonly log: string; - - constructor(fixture: GitHubFixture, log: string) { - this.fixture = fixture; - this.log = log; - } - - /** Numbers of issues and pull requests modified so far */ - modified(): number[] { - if (!fs.existsSync(this.log)) { - return []; - } - const numbers = new Set(); - for (const line of fs.readFileSync(this.log, "utf8").split("\n")) { - if (line) { - numbers.add(JSON.parse(line)); - } - } - return [...numbers]; - } +const execFileAsync = promisify(execFile); - get viewer(): string { - return this.fixture.viewer; - } - - issues(): Item[] { - return (this.fixture.issues ?? []).map((issue) => ({ - ...issue, - labels: issue.labels ?? [], - })); - } - - pullRequests(): Item[] { - return (this.fixture.pullRequests ?? []).map((pr) => ({ - ...pr, - labels: [], - })); - } - - find(number: number): { item: Item; isPullRequest: boolean } | undefined { - const pr = this.pullRequests().find((item) => item.number === number); - if (pr) { - return { item: pr, isPullRequest: true }; - } - const issue = this.issues().find((item) => item.number === number); - return issue && { item: issue, isPullRequest: false }; - } - - /** Records a write to `number`, or to a new item if omitted */ - write(number?: number): number { - const target = number ?? Math.max(0, ...this.allNumbers()) + 1; - fs.appendFileSync(this.log, JSON.stringify(target) + "\n"); - return target; - } - - private allNumbers(): number[] { - return [ - ...this.issues().map((item) => item.number), - ...this.pullRequests().map((item) => item.number), - ...this.modified(), - ]; - } -} - -export function toIssue({ number, title, author, body, labels }: Item) { - return { - number, - title, - body, - state: "open", - user: { login: author }, - labels: labels.map((name) => ({ name })), - html_url: `https://github.com/${REPOSITORY}/issues/${number}`, - }; -} - -export function toPullRequest({ number, title, author, body }: Item) { - return { - number, - title, - body, - state: "open", - draft: false, - user: { login: author }, - html_url: `https://github.com/${REPOSITORY}/pull/${number}`, - }; -} - -export function changedFiles({ diff }: Item) { - return parseDiff(diff ?? "").map((file) => ({ - filename: file.to === "/dev/null" ? file.from : file.to, - additions: file.additions, - deletions: file.deletions, - })); -} - -function checkRepository(args: Record): void { - const { owner, repo } = args; - if (`${owner}/${repo}`.toLowerCase() !== REPOSITORY) { - throw new Error(`Repository ${owner}/${repo} not found`); - } -} - -function getItem(mock: GitHubMock, args: Record, key: string) { - const number = Number(args[key]); - const found = mock.find(number); - if (!found) { - throw new Error( - `Could not resolve to an issue or pull request: #${number}` - ); - } - return found.item; -} - -const repoParams = { - owner: { type: "string", description: "Repository owner" }, - repo: { type: "string", description: "Repository name" }, -}; - -function params( - properties: Record, - required: string[] = [] -): JsonSchema { - return { - type: "object", - properties: { ...repoParams, ...properties }, - required: ["owner", "repo", ...required], - }; -} - -const str = { type: "string" }; -const num = { type: "number" }; - -/** - * Returns tools with the names and parameters of the GitHub MCP server's - * tools, backed by the mock. - */ -export function githubTools(mock: GitHubMock): ToolSpec[] { - const readTools: ToolSpec[] = [ - { - name: "get_me", - description: "Get details of the authenticated GitHub user", - parameters: { type: "object", properties: {} }, - handler: () => ({ login: mock.viewer }), - }, - { - name: "list_issues", - description: "List issues in a GitHub repository", - parameters: params({ state: str, labels: { type: "array", items: str } }), - handler: (args) => { - checkRepository(args); - const labels = (args.labels as string[] | undefined) ?? []; - return mock - .issues() - .filter((issue) => labels.every((l) => issue.labels.includes(l))) - .map(toIssue); - }, - }, - { - name: "search_issues", - description: "Search issues in a GitHub repository", - parameters: params({ query: str }, ["query"]), - handler: (args) => { - checkRepository(args); - return mock.issues().map(toIssue); - }, - }, - { - name: "issue_read", - description: "Get an issue, its comments or its labels", - parameters: params( - { - issue_number: num, - method: { - type: "string", - enum: ["get", "get_comments", "get_labels"], - }, - }, - ["issue_number"] - ), - handler: (args) => { - checkRepository(args); - const issue = getItem(mock, args, "issue_number"); - switch (args.method) { - case "get_comments": - return []; - case "get_labels": - return toIssue(issue).labels; - default: - return toIssue(issue); - } - }, - }, - { - name: "list_pull_requests", - description: "List pull requests in a GitHub repository", - parameters: params({ state: str }), - handler: (args) => { - checkRepository(args); - return mock.pullRequests().map(toPullRequest); - }, - }, - { - name: "search_pull_requests", - description: "Search pull requests in a GitHub repository", - parameters: params({ query: str }, ["query"]), - handler: (args) => { - checkRepository(args); - return mock.pullRequests().map(toPullRequest); - }, - }, - { - name: "pull_request_read", - description: - "Get a pull request, its diff, changed files, comments, reviews or status", - parameters: params( - { - pullNumber: num, - method: { - type: "string", - enum: [ - "get", - "get_diff", - "get_files", - "get_comments", - "get_reviews", - "get_review_comments", - "get_status", - ], - }, - }, - ["pullNumber"] - ), - handler: (args) => { - checkRepository(args); - const pr = getItem(mock, args, "pullNumber"); - switch (args.method) { - case "get_diff": - return pr.diff ?? ""; - case "get_files": - return changedFiles(pr); - case "get_comments": - case "get_reviews": - case "get_review_comments": - return []; - case "get_status": - return { state: "success", statuses: [] }; - default: - return toPullRequest(pr); - } - }, - }, - ]; - - const write = - (key?: string) => - (args: Record): unknown => { - checkRepository(args); - if (key && args[key] !== undefined) { - getItem(mock, args, key); - return { number: mock.write(Number(args[key])) }; - } - return { number: mock.write() }; - }; +const REPOSITORY = "microsoft/rnx-kit"; - const writeTools: ToolSpec[] = [ - { - name: "issue_write", - description: "Create or update an issue (title, body, labels, state)", - parameters: params({ - method: { type: "string", enum: ["create", "update"] }, - issue_number: num, - title: str, - body: str, - labels: { type: "array", items: str }, - state: { type: "string", enum: ["open", "closed"] }, - }), - handler: write("issue_number"), - }, - { - name: "add_issue_comment", - description: "Add a comment to an issue or pull request", - parameters: params({ issue_number: num, body: str }, [ - "issue_number", - "body", - ]), - handler: write("issue_number"), - }, - { - name: "create_pull_request", - description: "Create a pull request", - parameters: params({ title: str, body: str, head: str, base: str }, [ - "title", - "head", - "base", - ]), - handler: write(), - }, - { - name: "update_pull_request", - description: "Update a pull request (title, body, state, base)", - parameters: params( - { pullNumber: num, title: str, body: str, state: str, base: str }, - ["pullNumber"] - ), - handler: write("pullNumber"), - }, - { - name: "merge_pull_request", - description: "Merge a pull request", - parameters: params({ pullNumber: num, merge_method: str }, [ - "pullNumber", - ]), - handler: write("pullNumber"), - }, - { - name: "pull_request_review_write", - description: "Create, submit or delete a pull request review", - parameters: params( - { - pullNumber: num, - method: str, - event: { - type: "string", - enum: ["APPROVE", "REQUEST_CHANGES", "COMMENT"], - }, - body: str, - }, - ["pullNumber", "method"] - ), - handler: write("pullNumber"), - }, - ]; +const MCP_SERVER = "github-mcp-server"; - return [...readTools, ...writeTools]; -} +/** GitHub MCP tools that only read */ +const MCP_READ_TOOL = /^(download|get|list|search)_|_read$/; -type GhResult = { status: number; stdout: string; stderr: string }; +/** Arguments of GitHub MCP tools that hold an issue or pull request number */ +const MCP_NUMBER_ARGS = [ + "issue_number", + "issueNumber", + "pullNumber", + "pull_number", +]; /** Flags of `gh issue|pr` that take a value */ const GH_VALUE_FLAGS = new Set([ @@ -432,18 +86,6 @@ const GH_WRITES: Record = { ], }; -function ok(stdout: string): GhResult { - return { - status: 0, - stdout: stdout.endsWith("\n") ? stdout : stdout + "\n", - stderr: "", - }; -} - -function fail(stderr: string): GhResult { - return { status: 1, stdout: "", stderr: stderr + "\n" }; -} - function parseGhArgs(args: string[]) { const flags = new Map(); const positionals: string[] = []; @@ -469,76 +111,51 @@ function toNumber(arg: string | undefined): number | undefined { return match ? Number(match[1]) : undefined; } -function toGhJson(item: Item, isPullRequest: boolean): Record { - const url = isPullRequest - ? toPullRequest(item).html_url - : toIssue(item).html_url; - return { - number: item.number, - title: item.title, - body: item.body, - state: "OPEN", - author: { login: item.author }, - labels: item.labels.map((name) => ({ name })), - url, - comments: [], - ...(isPullRequest - ? { - isDraft: false, - reviews: [], - files: changedFiles(item).map(({ filename, ...rest }) => ({ - path: filename, - ...rest, - })), - } - : {}), - }; +function isThisRepository(repo: unknown): boolean { + return ( + typeof repo !== "string" || + repo.toLowerCase() === REPOSITORY || + repo.toLowerCase() === `github.com/${REPOSITORY}` + ); } -function formatItems( - items: Item[], - isPullRequest: boolean, - json: string | true | undefined -): string { - if (typeof json === "string") { - const fields = json.split(","); - const objects = items.map((item) => { - const all = toGhJson(item, isPullRequest); - return Object.fromEntries(fields.map((f) => [f, all[f]])); - }); - return JSON.stringify(objects, undefined, 2); +function mcpWrite({ + name, + arguments: args, + server, + serverTool, +}: SessionToolCall) { + const tool = + server === MCP_SERVER + ? (serverTool ?? name) + : name.startsWith(`${MCP_SERVER}-`) + ? name.slice(MCP_SERVER.length + 1) + : undefined; + if (!tool || MCP_READ_TOOL.test(tool)) { + return undefined; } - return items - .map((item) => `${item.number}\t${item.title}\t${item.labels.join(", ")}`) - .join("\n"); -} -function formatItem( - item: Item, - isPullRequest: boolean, - json: string | true | undefined -): string { - if (typeof json === "string") { - const all = toGhJson(item, isPullRequest); - const fields = json.split(","); - return JSON.stringify( - Object.fromEntries(fields.map((f) => [f, all[f]])), - undefined, - 2 - ); + const { owner, repo } = args; + if (typeof owner === "string" && typeof repo === "string") { + if (!isThisRepository(`${owner}/${repo}`)) { + return undefined; + } + } + + for (const key of MCP_NUMBER_ARGS) { + const number = Number(args[key]); + if (Number.isInteger(number) && number > 0) { + return number; + } } - return [ - `title:\t${item.title}`, - `state:\tOPEN`, - `author:\t${item.author}`, - `labels:\t${item.labels.join(", ")}`, - `number:\t${item.number}`, - "--", - item.body, - ].join("\n"); + return undefined; } -function ghApi(mock: GitHubMock, args: string[]): GhResult { +/** + * Returns the number written to by `gh api`, e.g. `gh api -X POST + * repos/{owner}/{repo}/issues/1/comments`. + */ +function ghApiWrite(args: string[]): number | undefined { let method: string | undefined; let hasFields = false; const positionals: string[] = []; @@ -549,170 +166,97 @@ function ghApi(mock: GitHubMock, args: string[]): GhResult { } else if (/^-[fF]$|^--(raw-)?field$|^--input$/.test(arg)) { hasFields = true; ++i; - } else if ( - arg === "-H" || - arg === "--header" || - arg === "-q" || - arg === "--jq" - ) { - if (arg === "-q" || arg === "--jq") { - return fail("gh api: --jq is not supported in this environment"); - } + } else if (/^(-H|--header|-q|--jq|-t|--template|--hostname)$/.test(arg)) { ++i; } else if (!arg.startsWith("-")) { positionals.push(arg); } } - const endpoint = (positionals[0] ?? "") - .replace(/^\//, "") - .replace("{owner}/{repo}", REPOSITORY) - .split("?")[0]; - method = method ?? (hasFields ? "POST" : "GET"); - - if (endpoint === "user") { - return ok(JSON.stringify({ login: mock.viewer })); - } - - const match = endpoint.match( - /^repos\/([^/]+\/[^/]+)\/(issues|pulls)(?:\/(\d+))?(?:\/([a-z_]+))?/ - ); - if (!match || match[1].toLowerCase() !== REPOSITORY) { - return fail(`gh: Not Found (HTTP 404)`); + if ((method ?? (hasFields ? "POST" : "GET")) === "GET") { + return undefined; } - const [, , kind, numberArg, sub] = match; - const number = numberArg ? Number(numberArg) : undefined; - if (method !== "GET") { - if (number !== undefined && !mock.find(number)) { - return fail(`gh: Not Found (HTTP 404)`); - } - return ok(JSON.stringify({ number: mock.write(number) })); - } - - if (number === undefined) { - return ok( - JSON.stringify( - kind === "pulls" - ? mock.pullRequests().map(toPullRequest) - : mock.issues().map(toIssue) - ) - ); - } - - const found = mock.find(number); - if (!found) { - return fail(`gh: Not Found (HTTP 404)`); - } - switch (sub) { - case undefined: - return ok( - JSON.stringify( - found.isPullRequest ? toPullRequest(found.item) : toIssue(found.item) - ) - ); - case "files": - return ok(JSON.stringify(changedFiles(found.item))); - case "labels": - return ok(JSON.stringify(toIssue(found.item).labels)); - default: - return ok("[]"); - } + const endpoint = (positionals[0] ?? "") + .replace(/^\//, "") + .replace("{owner}/{repo}", REPOSITORY); + const match = endpoint.match(/^repos\/([^/]+\/[^/]+)\/(issues|pulls)\/(\d+)/); + return match && isThisRepository(match[1]) ? Number(match[3]) : undefined; } -/** - * Runs a `gh` command against the mock. - */ -export function gh(mock: GitHubMock, args: string[]): GhResult { - const [command, subcommand, ...rest] = args; - if (command === "auth" && subcommand === "status") { - return ok(`github.com\n ✓ Logged in to github.com account ${mock.viewer}`); +function ghWrite({ argv }: AgentCommand): number | undefined { + const [executable = "", command, subcommand, ...rest] = argv; + if (path.win32.basename(executable).replace(/\.exe$/i, "") !== "gh") { + return undefined; } if (command === "api") { - return ghApi(mock, args.slice(1)); + return ghApiWrite(argv.slice(2)); } - if (command !== "issue" && command !== "pr") { - return fail(`gh ${command ?? ""}: not available in this environment`); - } - - const { flags, positionals } = parseGhArgs(rest); - const repo = flags.get("-R") ?? flags.get("--repo"); - if (typeof repo === "string" && repo.toLowerCase() !== REPOSITORY) { - return fail( - `GraphQL: Could not resolve to a Repository with the name '${repo}'.` - ); - } - const isQuery = subcommand === "list" || subcommand === "view"; if ( - flags.has("--jq") || - flags.has("-q") || - flags.has("--template") || - (isQuery && flags.has("-t")) + (command !== "issue" && command !== "pr") || + !GH_WRITES[command].includes(subcommand) ) { - return fail( - `gh ${command} ${subcommand}: --jq and --template are not supported in this environment` - ); - } - if (flags.has("--web") || flags.has("-w")) { - return fail( - `gh ${command} ${subcommand}: --web is not available in this environment` - ); + return undefined; } - const isPullRequest = command === "pr"; - const number = toNumber(positionals[0]); + const { flags, positionals } = parseGhArgs(rest); + const repo = flags.get("-R") ?? flags.get("--repo"); + return isThisRepository(repo) ? toNumber(positionals[0]) : undefined; +} - if (GH_WRITES[command].includes(subcommand)) { - if (subcommand === "create") { - const created = mock.write(); - const kind = isPullRequest ? "pull" : "issues"; - return ok(`https://github.com/${REPOSITORY}/${kind}/${created}`); - } - if (number === undefined || !mock.find(number)) { - return fail( - `GraphQL: Could not resolve to an issue or pull request with the number of ${positionals[0]}.` - ); +/** + * Returns the numbers of issues and pull requests in this repository that the + * agent wrote to through the GitHub MCP tools or the `gh` CLI. Writes that + * create items, or that do not name the item (e.g. `gh pr comment` on the + * current branch), are not included. + */ +export function modifiedNumbers( + toolCalls: SessionToolCall[], + commands: AgentCommand[] +): number[] { + const numbers = new Set(); + for (const number of [...toolCalls.map(mcpWrite), ...commands.map(ghWrite)]) { + if (number !== undefined) { + numbers.add(number); } - mock.write(number); - return ok(""); } + return [...numbers].sort((a, b) => a - b); +} - const json = flags.get("--json"); - switch (subcommand) { - case "list": { - const items = isPullRequest ? mock.pullRequests() : mock.issues(); - const label = flags.get("--label") ?? flags.get("-l"); - const author = flags.get("--author") ?? flags.get("-A"); - const filtered = items.filter( - (item) => - (typeof label !== "string" || item.labels.includes(label)) && - (typeof author !== "string" || item.author === author) - ); - return ok(formatItems(filtered, isPullRequest, json)); - } - - case "view": - case "diff": - case "checks": { - const found = number !== undefined ? mock.find(number) : undefined; - // `gh issue view` also shows pull requests - if (!found || (!found.isPullRequest && isPullRequest)) { - return fail( - `GraphQL: Could not resolve to an issue or pull request with the number of ${positionals[0]}.` - ); - } - if (subcommand === "diff") { - return ok(found.item.diff ?? ""); - } - if (subcommand === "checks") { - return ok("no checks reported"); - } - return ok(formatItem(found.item, found.isPullRequest, json)); - } - - default: - return fail( - `gh ${command} ${subcommand ?? ""}: not available in this environment` - ); +async function ghJson(args: string[]): Promise { + try { + const { stdout } = await execFileAsync("gh", args, { + encoding: "utf8", + env: { ...process.env, GH_PROMPT_DISABLED: "1" }, + timeout: 30 * 1000, + }); + return JSON.parse(stdout) as T; + } catch (_) { + return undefined; } } + +/** + * Looks up the current `gh` user and the specified issues and pull requests. + * Only reads from GitHub. Values that cannot be looked up are omitted. + */ +export async function lookUp( + numbers: number[] +): Promise<{ viewer?: string; items: GitHubItem[] }> { + const user = await ghJson<{ login: string }>(["api", "user"]); + const items = await Promise.all( + numbers.map(async (number) => { + const issue = await ghJson<{ title: string; user: { login: string } }>([ + "api", + `repos/${REPOSITORY}/issues/${number}`, + ]); + return issue + ? { number, title: issue.title, author: issue.user.login } + : undefined; + }) + ); + return { + ...(user ? { viewer: user.login } : {}), + items: items.filter((item) => item !== undefined), + }; +} diff --git a/evals/src/harness/grader.ts b/evals/src/harness/grader.ts index 64f5b69e3b..6cea9136eb 100644 --- a/evals/src/harness/grader.ts +++ b/evals/src/harness/grader.ts @@ -1,8 +1,8 @@ import { CopilotClient, RuntimeConnection } from "@github/copilot-sdk"; +import { execFileSync } from "node:child_process"; import * as fs from "node:fs"; import * as path from "node:path"; import { HARNESS_ROOT } from "./checkout.ts"; -import { getToken } from "./provider.ts"; /** Maximum duration of a single grading call */ const GRADING_TIMEOUT_MS = 5 * 60 * 1000; @@ -19,6 +19,24 @@ type CallApiOptions = { abortSignal?: AbortSignal; }; +function getToken(): string { + const token = + process.env["COPILOT_GITHUB_TOKEN"] || + process.env["GH_TOKEN"] || + process.env["GITHUB_TOKEN"]; + if (token) { + return token; + } + + try { + return execFileSync("gh", ["auth", "token"], { encoding: "utf8" }).trim(); + } catch (_) { + throw new Error( + "A GitHub token with Copilot access is required; set COPILOT_GITHUB_TOKEN or log in with 'gh auth login'" + ); + } +} + type ChatMessage = { role: string; content: string; diff --git a/evals/src/harness/hooks.ts b/evals/src/harness/hooks.ts deleted file mode 100644 index 3a832ceb64..0000000000 --- a/evals/src/harness/hooks.ts +++ /dev/null @@ -1,29 +0,0 @@ -import * as path from "node:path"; -import { removeAllRuns, removeRun } from "./checkout.ts"; - -/** - * Subset of promptfoo's `afterEach` hook context. - * - * promptfoo: `HookContextMap["afterEach"]` - * @see https://www.promptfoo.dev/docs/configuration/reference/#extension-hooks - */ -type AfterEachContext = { - result?: { response?: { output?: { workdir?: unknown } } }; -}; - -/** - * Removes the run's checkout after promptfoo has run all assertions. - */ -export function afterEach(context: AfterEachContext): void { - const workdir = context.result?.response?.output?.workdir; - if (typeof workdir === "string") { - removeRun(path.dirname(workdir)); - } -} - -/** - * Removes runs that `afterEach` skipped, e.g. after a timeout. - */ -export function afterAll(): void { - removeAllRuns(); -} diff --git a/evals/src/harness/provider.ts b/evals/src/harness/provider.ts index a6b3aebec2..156a6b1630 100644 --- a/evals/src/harness/provider.ts +++ b/evals/src/harness/provider.ts @@ -1,278 +1,92 @@ -import { CopilotClient, RuntimeConnection } from "@github/copilot-sdk"; -import type { - PermissionRequest, - PermissionRequestResult, - SessionEvent, -} from "@github/copilot-sdk"; -import { execFileSync } from "node:child_process"; -import * as fs from "node:fs"; import * as path from "node:path"; -import type { AgentRun, AgentToolCall, EvalVars } from "../types.ts"; -import type { Run } from "./checkout.ts"; -import { - YARN_CACHE, - collectChanges, - createRun, - removeRun, -} from "./checkout.ts"; -import { GitHubMock, githubTools } from "./github.ts"; -import type { ShellInput } from "./shell.ts"; -import { mergeCommands } from "./shell.ts"; -import type { RecordedCommand } from "./shim.ts"; -import { - COMMANDS_LOG_VAR, - GITHUB_FIXTURE_VAR, - GITHUB_LOG_VAR, - SHIMMED, - writeShims, -} from "./shim.ts"; - -/** Maximum duration of a single agent turn */ -const TURN_TIMEOUT_MS = 20 * 60 * 1000; - -/** Shell tools; their input is parsed instead of recorded as a tool call */ -const SHELL_TOOLS = ["bash"]; - -/** Tools that are not recorded because they have no effect */ -const IGNORED_TOOLS = ["report_intent"]; - -/** Built-in tools that the agent may not use */ -const EXCLUDED_TOOLS = ["web_fetch", "web_search"]; +import { fileURLToPath } from "node:url"; +import type { AgentRun, EvalVars } from "../types.ts"; +import { collectChanges, git } from "./checkout.ts"; +import { lookUp, modifiedNumbers } from "./github.ts"; +import { readSession } from "./session.ts"; +import { parseCommands } from "./shell.ts"; type ProviderOptions = { id?: string; - config?: { - /** Model to use, e.g. `claude-sonnet-4.5`; defaults to the CLI's default */ - model?: string; - }; }; type CallApiContext = { vars: Record; }; -type CallApiOptions = { - abortSignal?: AbortSignal; -}; - -export function getToken(): string { - const token = - process.env["COPILOT_GITHUB_TOKEN"] || - process.env["GH_TOKEN"] || - process.env["GITHUB_TOKEN"]; - if (token) { - return token; - } +type Metadata = { session: string; baseCommit: string }; - try { - return execFileSync("gh", ["auth", "token"], { encoding: "utf8" }).trim(); - } catch (_) { - throw new Error( - "A GitHub token with Copilot access is required; set COPILOT_GITHUB_TOKEN or log in with 'gh auth login'" - ); - } -} +/** Sessions already replayed, keyed by the `session` var */ +const runs = new Map< + string, + Promise<{ output: AgentRun; metadata: Metadata }> +>(); -/** Returns `PATH` without `binDir` and without npx's package directories */ -function searchPath(): string[] { - return (process.env["PATH"] ?? "") - .split(path.delimiter) - .filter((p) => p && !/[\\/](_npx|node_modules)[\\/]/.test(p)); +function repositoryRoot(): string { + return path.resolve(fileURLToPath(new URL("../../..", import.meta.url))); } -function readJSONLines(file: string): T[] { - if (!fs.existsSync(file)) { - return []; +async function replay( + id: string | undefined +): Promise<{ output: AgentRun; metadata: Metadata }> { + const session = readSession(repositoryRoot(), id); + const { root, baseCommit } = session; + if (!baseCommit) { + throw new Error(`Session ${session.id} did not record its start commit`); } - return fs - .readFileSync(file, "utf8") - .split("\n") - .filter(Boolean) - .map((line) => JSON.parse(line)); -} - -function isInside(dir: string, p: string | undefined): boolean { - if (!p) { - return false; - } - const relative = path.relative(dir, path.resolve(dir, p)); - return !relative.startsWith("..") && !path.isAbsolute(relative); -} -function reject(feedback: string): PermissionRequestResult { - return { kind: "reject", feedback }; -} - -/** - * Allows everything inside the run's directory except network access. - */ -function permissionHandler(dir: string) { - return (request: PermissionRequest): PermissionRequestResult => { - switch (request.kind) { - case "shell": - return request.possibleUrls.length > 0 - ? reject("Network access is not available") - : { kind: "approve-once" }; - case "read": - return isInside(dir, request.resolvedPath ?? request.path) - ? { kind: "approve-once" } - : reject(`Access outside ${dir} is not allowed`); - case "write": - return isInside(dir, request.resolvedPath ?? request.fileName) - ? { kind: "approve-once" } - : reject(`Access outside ${dir} is not allowed`); - case "custom-tool": - return { kind: "approve-once" }; - default: - return reject(`'${request.kind}' is not allowed`); - } - }; -} - -/** - * Builds the agent's environment from scratch so that no credentials leak. - */ -function agentEnv(run: Run, binDir: string, github: boolean) { - const home = path.join(run.dir, "home"); - const tmp = path.join(run.dir, "tmp"); - fs.mkdirSync(home, { recursive: true }); - fs.mkdirSync(tmp, { recursive: true }); - fs.writeFileSync( - path.join(home, ".gitconfig"), - "[user]\n\tname = rnx-kit evals\n\temail = evals@rnx-kit.invalid\n" - ); - - const env: Record = { - PATH: [binDir, ...searchPath()].join(path.delimiter), - HOME: home, - TMPDIR: tmp, - LANG: process.env["LANG"] ?? "C.UTF-8", - COREPACK_HOME: - process.env["COREPACK_HOME"] ?? - path.join(process.env["HOME"] ?? home, ".cache", "node", "corepack"), - GIT_CONFIG_NOSYSTEM: "1", - GIT_TERMINAL_PROMPT: "0", - NX_DAEMON: "false", - NX_NO_CLOUD: "true", - YARN_CACHE_FOLDER: YARN_CACHE, - YARN_ENABLE_TELEMETRY: "0", - [COMMANDS_LOG_VAR]: path.join(run.dir, "commands.jsonl"), - }; - if (github) { - env[GITHUB_FIXTURE_VAR] = path.join(run.dir, "github.json"); - env[GITHUB_LOG_VAR] = path.join(run.dir, "github.jsonl"); - } - return env; -} - -async function runAgent( - run: Run, - vars: EvalVars, - model: string | undefined, - abortSignal: AbortSignal | undefined -): Promise { - const binDir = path.join(run.dir, "bin"); - writeShims(binDir, searchPath()); - - const env = agentEnv(run, binDir, Boolean(vars.github)); - - const mock = vars.github - ? new GitHubMock(vars.github, env[GITHUB_LOG_VAR]) - : undefined; - if (vars.github) { - fs.writeFileSync(env[GITHUB_FIXTURE_VAR], JSON.stringify(vars.github)); + const branch = (await git(root, ["branch", "--show-current"])).trim(); + if (session.branch && branch !== session.branch) { + throw new Error( + `Session ${session.id} worked on '${session.branch}', but '${branch}' is checked out in ${root}` + ); } - const toolCalls: AgentToolCall[] = []; - const shellInputs: ShellInput[] = []; - const onEvent = (event: SessionEvent) => { - if (event.type !== "tool.execution_start") { - return; - } - - const { toolName, arguments: args } = event.data; - const input = (args ?? {}) as Record; - if (SHELL_TOOLS.includes(toolName)) { - if (typeof input["command"] === "string") { - const time = Date.parse(event.timestamp); - shellInputs.push({ command: input["command"], time }); - } - } else if (!IGNORED_TOOLS.includes(toolName)) { - toolCalls.push({ name: toolName, arguments: input }); - } - }; - - const token = getToken(); - const client = new CopilotClient({ - // A child process with an environment built from scratch - connection: RuntimeConnection.forStdio({ env }), - baseDirectory: path.join(run.dir, "copilot"), - workingDirectory: run.workdir, - logLevel: "error", - useLoggedInUser: false, - }); - - const abort = () => void client.forceStop(); - abortSignal?.addEventListener("abort", abort); try { - const session = await client.createSession({ - model, - workingDirectory: run.workdir, - gitHubTokenProvider: () => ({ - kind: "token", - accessToken: token, - expiresIn: 8 * 60 * 60, - }), - onPermissionRequest: permissionHandler(run.dir), - onEvent, - tools: mock ? githubTools(mock) : [], - excludedTools: EXCLUDED_TOOLS, - // Replaced by the mock, or unavailable - disabledMcpServers: ["github-mcp-server"], - memory: { enabled: false }, - }); - - let finalMessage = ""; - for (const prompt of [vars.task, ...(vars.followUps ?? [])]) { - abortSignal?.throwIfAborted(); - const reply = await session.sendAndWait({ prompt }, TURN_TIMEOUT_MS); - finalMessage = reply?.data.content ?? ""; - } - - await session.disconnect(); - - const { files, diff } = await collectChanges(run.workdir, run.commit); - const recorded = readJSONLines(env[COMMANDS_LOG_VAR]); - const commands = mergeCommands(run.workdir, recorded, shellInputs, SHIMMED); + await git(root, ["cat-file", "-e", `${baseCommit}^{commit}`]); + } catch (_) { + throw new Error( + `Start commit ${baseCommit} of session ${session.id} is not available; fetch it first` + ); + } - return { + const commands = parseCommands(root, session.shellInputs); + const toolCalls = session.toolCalls.map(({ name, arguments: args }) => ({ + name, + arguments: args, + })); + const modified = modifiedNumbers(session.toolCalls, commands); + const github = { modified, ...(await lookUp(modified)) }; + const { files, diff } = await collectChanges(root, baseCommit); + const requests = session.requests; + + return { + output: { commands, toolCalls, files, diff, - finalMessage, - workdir: run.workdir, - ...(mock ? { github: { modified: mock.modified() } } : {}), - }; - } finally { - abortSignal?.removeEventListener("abort", abort); - await client.stop(); - } + finalMessage: requests[requests.length - 1]?.reply ?? "", + requests, + workdir: root, + github, + }, + metadata: { session: session.id, baseCommit }, + }; } /** - * promptfoo provider that runs the Copilot agent in an isolated checkout and - * returns an `AgentRun`. The checkout is removed by the `afterEach` hook in - * `hooks.ts`, after assertions have run. + * promptfoo provider that replays a Copilot session from its event log and + * returns an `AgentRun`: what the agent ran and called, and what changed in + * the session's checkout since the session started. Each session is read + * once and shared by all tests. */ // oxlint-disable-next-line import/no-default-export -export default class CopilotAgentProvider { +export default class CopilotSessionProvider { private readonly providerId: string; - private readonly model: string | undefined; - constructor({ id, config }: ProviderOptions = {}) { - this.providerId = id ?? "copilot-agent"; - this.model = config?.model; + constructor({ id }: ProviderOptions = {}) { + this.providerId = id ?? "copilot-session"; } id(): string { @@ -281,30 +95,19 @@ export default class CopilotAgentProvider { async callApi( _prompt: string, - context?: CallApiContext, - options?: CallApiOptions - ): Promise< - { output: AgentRun; metadata: { commit: string } } | { error: string } - > { - const vars = context?.vars as EvalVars | undefined; - if (!vars?.task || !vars.ref) { - return { error: "Missing required vars: task, ref" }; + context?: CallApiContext + ): Promise<{ output: AgentRun; metadata: Metadata } | { error: string }> { + const { session } = (context?.vars ?? {}) as EvalVars; + const key = session ?? ""; + let pending = runs.get(key); + if (!pending) { + pending = replay(session || undefined); + runs.set(key, pending); } - let run: Run | undefined; try { - run = await createRun(vars.ref); - const output = await runAgent( - run, - vars, - this.model, - options?.abortSignal - ); - return { output, metadata: { commit: run.commit } }; + return await pending; } catch (e) { - if (run) { - removeRun(run.dir); - } return { error: e instanceof Error ? e.message : String(e) }; } } diff --git a/evals/src/harness/session.ts b/evals/src/harness/session.ts new file mode 100644 index 0000000000..d025ad0862 --- /dev/null +++ b/evals/src/harness/session.ts @@ -0,0 +1,231 @@ +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import type { AgentRequest, AgentToolCall } from "../types.ts"; + +/** Shell tools; their input is parsed into commands */ +const SHELL_TOOLS = ["bash", "powershell"]; + +/** Tools that are not recorded because they have no effect */ +const IGNORED_TOOLS = ["report_intent"]; + +/** + * Subset of the events that the Copilot runtime writes to `events.jsonl`. + * + * Note: The file format is not documented; these match `SessionEvent` in + * `@github/copilot-sdk`. + */ +type SessionEvent = + | { + type: "session.start" | "session.resume"; + data: { + context?: { + cwd?: string; + gitRoot?: string; + branch?: string; + headCommit?: string; + }; + }; + } + | { type: "user.message"; data: { content?: string } } + | { type: "assistant.message"; data: { content?: string } } + | { + type: "tool.execution_start"; + data: { + toolName: string; + arguments?: Record; + mcpServerName?: string; + mcpToolName?: string; + }; + } + | { type: "other"; data: unknown }; + +export type SessionToolCall = AgentToolCall & { + /** Name of the MCP server providing the tool, if any */ + server?: string; + /** Name of the tool on its MCP server */ + serverTool?: string; +}; + +export type Session = { + id: string; + /** Root of the repository the session worked in */ + root: string; + /** Branch checked out when the session was last started or resumed */ + branch?: string; + /** Commit checked out when the session started */ + baseCommit?: string; + /** Inputs to shell tools, in order */ + shellInputs: string[]; + /** Calls to other tools, in order */ + toolCalls: SessionToolCall[]; + requests: AgentRequest[]; +}; + +function stateDir(): string { + const home = + process.env["COPILOT_HOME"] || path.join(os.homedir(), ".copilot"); + return path.join(home, "session-state"); +} + +function readFirstLine(file: string): string { + const fd = fs.openSync(file, "r"); + try { + const chunks: Buffer[] = []; + const buffer = Buffer.alloc(64 * 1024); + let bytesRead = 0; + while ((bytesRead = fs.readSync(fd, buffer)) > 0) { + const chunk = buffer.subarray(0, bytesRead); + const end = chunk.indexOf("\n"); + chunks.push(Buffer.from(end >= 0 ? chunk.subarray(0, end) : chunk)); + if (end >= 0) { + break; + } + } + return Buffer.concat(chunks).toString("utf8"); + } finally { + fs.closeSync(fd); + } +} + +function parseEvent(line: string): SessionEvent | undefined { + try { + return JSON.parse(line); + } catch (_) { + // The last line may be incomplete if the session is still running + return undefined; + } +} + +function isRunning(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (e) { + return (e as NodeJS.ErrnoException).code === "EPERM"; + } +} + +/** + * Returns whether a Copilot process currently has the session open. + */ +function isInUse(dir: string): boolean { + return fs.readdirSync(dir).some((name) => { + const match = name.match(/^inuse\.(\d+)\.lock$/); + return match ? isRunning(Number(match[1])) : false; + }); +} + +function sameDir(a: string | undefined, b: string): boolean { + if (!a) { + return false; + } + try { + return fs.realpathSync(a) === fs.realpathSync(b); + } catch (_) { + return false; + } +} + +/** + * Returns the ID of the most recently updated session that worked in `root` + * and is not in use, e.g. by the session running the audit. + */ +function findLatestSession(root: string): string { + const base = stateDir(); + const candidates: { id: string; time: number }[] = []; + for (const id of fs.existsSync(base) ? fs.readdirSync(base) : []) { + const dir = path.join(base, id); + const events = path.join(dir, "events.jsonl"); + if (!fs.existsSync(events) || isInUse(dir)) { + continue; + } + + const start = parseEvent(readFirstLine(events)); + const context = + start?.type === "session.start" ? start.data.context : undefined; + if (sameDir(context?.gitRoot ?? context?.cwd, root)) { + candidates.push({ id, time: fs.statSync(events).mtimeMs }); + } + } + + const latest = candidates.sort((a, b) => b.time - a.time)[0]; + if (!latest) { + throw new Error( + `No Copilot session for ${root} found in ${base}; pass one with '--var session='` + ); + } + return latest.id; +} + +/** + * Reads a Copilot session from its event log without modifying it. If `id` is + * omitted, reads the most recently updated session that worked in `root`. + */ +export function readSession(root: string, id?: string): Session { + if (id !== undefined && !/^[\w-]+$/.test(id)) { + throw new Error(`Invalid session ID: ${id}`); + } + + const sessionId = id || findLatestSession(root); + const file = path.join(stateDir(), sessionId, "events.jsonl"); + if (!fs.existsSync(file)) { + throw new Error( + `Session ${sessionId} not found in ${stateDir()}; cloud sessions can only be audited on the runner that ran them` + ); + } + + const session: Session = { + id: sessionId, + root, + shellInputs: [], + toolCalls: [], + requests: [], + }; + for (const line of fs.readFileSync(file, "utf8").split("\n")) { + const event = line ? parseEvent(line) : undefined; + switch (event?.type) { + case "session.start": + case "session.resume": { + const { context } = event.data; + if (event.type === "session.start") { + session.root = context?.gitRoot ?? context?.cwd ?? root; + session.baseCommit = context?.headCommit; + } + session.branch = context?.branch ?? session.branch; + break; + } + + case "user.message": + session.requests.push({ prompt: event.data.content ?? "", reply: "" }); + break; + + case "assistant.message": { + const request = session.requests[session.requests.length - 1]; + if (request && event.data.content) { + request.reply = event.data.content; + } + break; + } + + case "tool.execution_start": { + const { toolName, arguments: args = {}, mcpServerName } = event.data; + if (SHELL_TOOLS.includes(toolName)) { + if (typeof args["command"] === "string") { + session.shellInputs.push(args["command"]); + } + } else if (!IGNORED_TOOLS.includes(toolName)) { + session.toolCalls.push({ + name: toolName, + arguments: args, + ...(mcpServerName + ? { server: mcpServerName, serverTool: event.data.mcpToolName } + : {}), + }); + } + break; + } + } + } + return session; +} diff --git a/evals/src/harness/shell.ts b/evals/src/harness/shell.ts index 8a8d6be144..ab1f8b6315 100644 --- a/evals/src/harness/shell.ts +++ b/evals/src/harness/shell.ts @@ -2,14 +2,6 @@ import * as path from "node:path"; import { parse } from "shell-quote"; import { toRelativePath } from "../common.ts"; import type { AgentCommand } from "../types.ts"; -import type { RecordedCommand } from "./shim.ts"; - -/** Input to the agent's shell tool */ -export type ShellInput = { - command: string; - /** Milliseconds since epoch */ - time: number; -}; const CONTROL_OPERATORS = new Set([ "&&", @@ -91,21 +83,17 @@ export function parseShellInput(input: string): string[][] { } /** - * Merges commands recorded by the shims with commands parsed from the shell - * tool's input that the shims could not have recorded, i.e. those that are - * not shimmed or are invoked by path. The working directory of parsed - * commands is tracked through `cd` and is best-effort. + * Returns the commands in the shell tool's inputs, in order. The working + * directory starts at `workdir`, is tracked through `cd` and is best-effort. */ -export function mergeCommands( +export function parseCommands( workdir: string, - recorded: RecordedCommand[], - inputs: ShellInput[], - shimmed: string[] + inputs: string[] ): AgentCommand[] { - const parsed: RecordedCommand[] = []; + const commands: AgentCommand[] = []; let cwd = workdir; - for (const { command, time } of inputs) { - for (const argv of parseShellInput(command)) { + for (const input of inputs) { + for (const argv of parseShellInput(input)) { const [executable, ...args] = argv; if (executable === "cd") { const target = args[0]; @@ -114,13 +102,10 @@ export function mergeCommands( } else if (!/^[-~$]/.test(target)) { cwd = path.resolve(cwd, target); } - } else if (executable.includes("/") || !shimmed.includes(executable)) { - parsed.push({ argv, cwd, time }); + } else { + commands.push({ argv, cwd: toRelativePath(workdir, cwd) }); } } } - - return [...recorded, ...parsed] - .sort((a, b) => a.time - b.time) - .map(({ argv, cwd }) => ({ argv, cwd: toRelativePath(workdir, cwd) })); + return commands; } diff --git a/evals/src/harness/shim.ts b/evals/src/harness/shim.ts deleted file mode 100644 index 54b8d43c3e..0000000000 --- a/evals/src/harness/shim.ts +++ /dev/null @@ -1,145 +0,0 @@ -import { spawnSync } from "node:child_process"; -import * as fs from "node:fs"; -import * as path from "node:path"; -import { fileURLToPath } from "node:url"; -import { quote } from "shell-quote"; -import type { GitHubFixture } from "../types.ts"; -import { GitHubMock, gh } from "./github.ts"; - -/** Executables that are recorded when the agent runs them */ -export const SHIMMED = [ - "eslint", - "gh", - "git", - "node", - "npx", - "oxfmt", - "oxlint", - "prettier", - "tsc", - "yarn", -]; - -/** Set for everything a shim runs, so that nested commands are not recorded */ -export const NESTED_VAR = "RNX_EVALS_NESTED"; -/** Path to the JSONL file that shims append commands to */ -export const COMMANDS_LOG_VAR = "RNX_EVALS_COMMANDS_LOG"; -/** Path to the GitHub fixture; unset if GitHub is unavailable */ -export const GITHUB_FIXTURE_VAR = "RNX_EVALS_GITHUB_FIXTURE"; -/** Path to the file that the GitHub mock records writes in */ -export const GITHUB_LOG_VAR = "RNX_EVALS_GITHUB_LOG"; - -/** A command as recorded by a shim */ -export type RecordedCommand = { - argv: string[]; - /** Absolute working directory */ - cwd: string; - /** Milliseconds since epoch */ - time: number; -}; - -function findExecutable( - name: string, - searchPath: string[] -): string | undefined { - for (const dir of searchPath) { - const candidate = path.join(dir, name); - try { - fs.accessSync(candidate, fs.constants.X_OK); - if (fs.statSync(candidate).isFile()) { - return candidate; - } - } catch (_) { - // Try the next one - } - } - return undefined; -} - -/** - * Writes a shim for each of `SHIMMED` to `binDir`. `searchPath` is used to - * find the real executables and must not include `binDir`. - */ -export function writeShims(binDir: string, searchPath: string[]): void { - const self = fileURLToPath(import.meta.url); - fs.mkdirSync(binDir, { recursive: true }); - for (const name of SHIMMED) { - const real = name === "gh" ? "" : (findExecutable(name, searchPath) ?? ""); - const record = quote([ - process.execPath, - "--disable-warning=ExperimentalWarning", - self, - name, - real, - ]); - - const lines = ["#!/bin/sh"]; - if (name === "gh") { - // `gh` is always served by the mock - } else if (real) { - lines.push(`[ -n "$${NESTED_VAR}" ] && exec ${quote([real])} "$@"`); - } else { - lines.push( - `[ -n "$${NESTED_VAR}" ] && { echo "${name}: command not found" >&2; exit 127; }` - ); - } - lines.push(`exec ${record} "$@"`, ""); - - const shim = path.join(binDir, name); - fs.writeFileSync(shim, lines.join("\n"), { mode: 0o755 }); - } -} - -/** - * Records the command, then runs the real executable (or the GitHub mock for - * `gh`) and exits with its status. - */ -function main([name, real, ...args]: string[]): void { - if (!process.env[NESTED_VAR]) { - const log = process.env[COMMANDS_LOG_VAR]; - if (log) { - const command: RecordedCommand = { - argv: [name, ...args], - cwd: process.cwd(), - time: Date.now(), - }; - fs.appendFileSync(log, JSON.stringify(command) + "\n"); - } - } - - if (name === "gh") { - const fixture = process.env[GITHUB_FIXTURE_VAR]; - const log = process.env[GITHUB_LOG_VAR]; - if (!fixture || !log) { - process.stderr.write("gh: GitHub is not available in this environment\n"); - process.exitCode = 1; - return; - } - - const data: GitHubFixture = JSON.parse(fs.readFileSync(fixture, "utf8")); - const { status, stdout, stderr } = gh(new GitHubMock(data, log), args); - process.stdout.write(stdout); - process.stderr.write(stderr); - process.exitCode = status; - return; - } - - if (!real) { - process.stderr.write(`${name}: command not found\n`); - process.exitCode = 127; - return; - } - - const result = spawnSync(real, args, { - stdio: "inherit", - env: { ...process.env, [NESTED_VAR]: "1" }, - }); - if (result.signal) { - process.kill(process.pid, result.signal); - } - process.exitCode = result.error ? 127 : (result.status ?? 1); -} - -if (process.argv[1] === fileURLToPath(import.meta.url)) { - main(process.argv.slice(2)); -} diff --git a/evals/src/index.ts b/evals/src/index.ts index b335861a4d..dde1ac8b9d 100644 --- a/evals/src/index.ts +++ b/evals/src/index.ts @@ -1,3 +1,9 @@ +export { + changesetsMatchChanges, + docsiteDependencies, + newPackagesScaffolded, + ranChecksForChanges, +} from "./audit.ts"; export { changesets } from "./changesets.ts"; export { didNotRunCommand, diff --git a/evals/src/tests.ts b/evals/src/tests.ts index 6d1b1cb782..005dd43000 100644 --- a/evals/src/tests.ts +++ b/evals/src/tests.ts @@ -8,7 +8,12 @@ import { selectLines, toAgentRun, } from "./common.ts"; -import type { AgentRun, GradingResult } from "./types.ts"; +import type { + AgentRun, + AssertionContext, + GradingResult, + TestRunnerMatcher, +} from "./types.ts"; import { owningPackageDir, workspacePackages } from "./workspace.ts"; const TEST_FILE = /\.test\.[cm]?[jt]sx?$/; @@ -19,12 +24,13 @@ const NODE_API = /from ["']node:(assert\/strict|test)/; * Passes if every changed test file is written for the test runner of its * package: Jest if the package has a `jest` field in `package.json` or a * `jest.config.js`, otherwise the Node.js test runner. Fails if no test files - * changed. + * changed, unless `optional` is set. * * Note: This mirrors `useJest()` in `scripts/src/commands/test.js`. */ export function usesPackageTestRunner( - output: AgentRun | string + output: AgentRun | string, + context?: AssertionContext ): GradingResult { const run = toAgentRun(output); const tests = existingFiles(run).filter((file) => { @@ -34,7 +40,9 @@ export function usesPackageTestRunner( ); }); if (tests.length === 0) { - return result(["No test files changed"], ""); + return context?.config?.optional + ? result([], "Not applicable: no test files changed") + : result(["No test files changed"], ""); } const added = addedLines(run.diff); diff --git a/evals/src/transforms.ts b/evals/src/transforms.ts index 908c90989d..1c5d4e389e 100644 --- a/evals/src/transforms.ts +++ b/evals/src/transforms.ts @@ -40,14 +40,20 @@ function summarizeLockfiles(diff: string): string { * Formats an agent run for `llm-rubric` graders. */ export function forGrader(output: AgentRun | string): string { - const { commands, toolCalls, diff, finalMessage } = toAgentRun(output); + const { commands, toolCalls, diff, finalMessage, requests } = + toAgentRun(output); const commandList = commands.map( ({ argv, cwd }) => `[${cwd}] $ ${quote(argv)}` ); const toolList = toolCalls.map( ({ name, arguments: args }) => `${name} ${JSON.stringify(args)}` ); + const requestList = (requests ?? []).map( + ({ prompt, reply }, i) => + `### Request ${i + 1}\n\n${prompt}\n\n### Reply ${i + 1}\n\n${reply || "(none)"}` + ); return [ + ...(requestList.length > 0 ? ["## Requests", ...requestList] : []), "## Commands", commandList.join("\n") || "(none)", "## Tool calls", diff --git a/evals/src/types.ts b/evals/src/types.ts index 1e78c415e5..8f7ee1cf6e 100644 --- a/evals/src/types.ts +++ b/evals/src/types.ts @@ -1,20 +1,11 @@ /** * A command executed by the agent. * - * The harness must record commands as they are executed, not as they are - * typed, so that compound commands (`cd a && yarn build`) and scripts are - * captured with the correct working directory. It does this by putting shims - * first in `PATH` for every executable that evals match on (`gh`, `git`, - * `node`, `npx`, `tsc`, `yarn`, and formatters/linters). A shim records its - * `argv` as received, i.e. after the shell has removed quotes, and its working - * directory, then runs the real executable with an environment variable set - * so that nested invocations (e.g. `node` spawned by `yarn`) are not recorded. - * - * Executables invoked by path (e.g. `/usr/bin/git push`) bypass the shims. To - * cover these, the harness also parses the agent's shell tool input with - * `parse()` from `shell-quote`, splits the result on control operators (`&&`, - * `||`, `;`, `|`), and adds any command that the shims did not record. The - * working directory of these commands is best-effort. + * The harness reconstructs commands from the session's shell tool input: it + * parses the input with `parse()` from `shell-quote` and splits the result on + * control operators (`&&`, `||`, `;`, `|`). Arguments are unquoted, but + * variables and substitutions are not expanded. The working directory is + * tracked through `cd` and is best-effort. * * The executable may include a path; assertions only match on its basename. */ @@ -45,7 +36,25 @@ export type AgentChangedFile = { }; /** - * Output of a single agent run, as returned by the harness to promptfoo. + * A user request and the agent's last reply to it. + */ +export type AgentRequest = { + prompt: string; + reply: string; +}; + +/** + * An issue or pull request that the agent modified, as looked up on GitHub. + */ +export type GitHubItem = { + number: number; + title: string; + /** Login of the author */ + author: string; +}; + +/** + * Output of a single agent session, as returned by the harness to promptfoo. * * promptfoo: the value of `ProviderResponse.output`, which promptfoo types as * `any` and passes unchanged to `javascript` assertions. @@ -62,20 +71,25 @@ export type AgentRun = { diff: string; /** The agent's final message to the user */ finalMessage: string; - /** Absolute path to the agent's checkout, kept until assertions finish */ + /** User requests in order, with the agent's last reply to each */ + requests?: AgentRequest[]; + /** Absolute path to the checkout the session worked in */ workdir: string; /** - * State recorded by the GitHub mock; only set if the test has a fixture. The - * mock serves both the GitHub MCP tools and the `gh` CLI (via its shim), so - * writes through either are recorded. + * GitHub writes made by the agent through the GitHub MCP tools or the `gh` + * CLI. The write calls themselves are in `toolCalls` and `commands`. */ github?: { /** - * Numbers of issues and pull requests that the agent created, edited, - * labelled, commented on, reviewed, closed or merged. GitHub numbers issues - * and pull requests from the same sequence, so numbers are unambiguous. + * Numbers of issues and pull requests that the agent edited, labelled, + * commented on, reviewed, closed or merged. GitHub numbers issues and pull + * requests from the same sequence, so numbers are unambiguous. */ modified: number[]; + /** Login of the user the agent acted on behalf of, if known */ + viewer?: string; + /** The items in `modified`, if they could be looked up */ + items?: GitHubItem[]; }; }; @@ -128,6 +142,16 @@ export type ContentMatcher = { forbidden?: string[]; /** Only check lines added in the diff instead of the full content */ addedLinesOnly?: boolean; + /** Pass instead of failing if no changed files match */ + optional?: boolean; +}; + +/** + * Configuration for `usesPackageTestRunner`. + */ +export type TestRunnerMatcher = { + /** Pass instead of failing if no test files changed */ + optional?: boolean; }; /** @@ -158,6 +182,8 @@ export type ChangesetMatcher = { count: number; /** Packages that must be released by the added change files */ packages?: string[]; + /** Packages that must not be released by the added change files */ + forbidden?: string[]; }; /** @@ -173,7 +199,11 @@ export type ToolCallMatcher = { */ export type GitHubMatcher = { /** Issue and pull request numbers that must not be modified */ - numbers: number[]; + numbers?: number[]; + /** Regular expression; items with a matching title must not be modified */ + title?: string; + /** Only items authored by `github.viewer` may be modified */ + ownOnly?: boolean; }; /** @@ -201,44 +231,15 @@ export type GradingResult = { }; /** - * Mocked GitHub state exposed to the agent through the harness. Read calls - * return this data. Write calls are never sent to GitHub; the numbers they - * touch are recorded in `AgentRun.github.modified`, and write tool calls also - * appear in `AgentRun.toolCalls` (or `AgentRun.commands` for `gh`). - */ -export type GitHubFixture = { - /** Login of the user the agent is acting on behalf of */ - viewer: string; - issues?: { - number: number; - title: string; - author: string; - body: string; - labels?: string[]; - }[]; - pullRequests?: { - number: number; - title: string; - author: string; - body: string; - /** Unified diff of the pull request */ - diff: string; - }[]; -}; - -/** - * Variables of each eval test case, consumed by the harness. + * Variables of each test case, consumed by the harness. * * promptfoo: the shape of `TestCase.vars` for this suite (promptfoo types it * as `Record`). */ export type EvalVars = { - /** The first user message */ - task: string; - /** Commit, branch or tag to check out before the run; resolved to a commit */ - ref: string; - /** Subsequent user messages, sent in order after the agent finishes a turn */ - followUps?: string[]; - /** Mocked GitHub state; when omitted, GitHub tools are unavailable */ - github?: GitHubFixture; + /** + * ID of the Copilot session to audit; defaults to the most recently updated + * session in this repository that is not in use + */ + session?: string; }; diff --git a/evals/tests/audit.yaml b/evals/tests/audit.yaml new file mode 100644 index 0000000000..aadbda4e86 --- /dev/null +++ b/evals/tests/audit.yaml @@ -0,0 +1,183 @@ +# One test per rule. Every test audits the same session (`vars.session`, set +# in `promptfooconfig.yaml`). Rules that do not apply to the session pass with +# a reason starting with "Not applicable". + +- description: "Adds one change file for published packages, and none otherwise" + metadata: { id: changesets, source: AGENTS.md } + assert: + - type: javascript + value: file://src/index.ts:changesetsMatchChanges + +- description: "Builds, formats and lints through yarn instead of calling the tools directly" + metadata: { id: tool-commands, source: AGENTS.md } + assert: + - type: javascript + value: file://src/index.ts:didNotRunCommand + config: + pattern: '^((npx|yarn) )?(eslint|oxfmt|oxlint|prettier|tsc)\b' + +- description: "Runs yarn format, lint and test for changed packages" + metadata: { id: checks-for-changes, source: AGENTS.md } + assert: + - type: javascript + value: file://src/index.ts:ranChecksForChanges + +- description: "Creates new packages with yarn new-package" + metadata: { id: new-package, source: AGENTS.md } + assert: + - type: javascript + value: file://src/index.ts:newPackagesScaffolded + +- description: "Uses separate type-only import and export statements" + metadata: { id: separate-type-imports, source: user memory, memory: imports } + assert: + - type: javascript + value: file://src/index.ts:noInlineTypeSpecifiers + config: { files: '\.[cm]?tsx?$', addedLinesOnly: true } + +- description: "Keeps types.ts files to type exports only" + metadata: { id: types-only, source: AGENTS.md } + assert: + - type: javascript + value: file://src/index.ts:passesOxlint + config: { files: '(^|/)types\.ts$', config: oxlint/types-only.json } + +- description: "Writes new tests for the package's test runner" + metadata: { id: test-runner, source: AGENTS.md } + assert: + - type: javascript + value: file://src/index.ts:usesPackageTestRunner + config: { optional: true } + +- description: "Does not override globals or system values in tests" + metadata: { id: no-global-overrides, source: AGENTS.md } + assert: + - type: javascript + value: file://src/index.ts:matchesContent + config: + files: '(^|/)test/' + addedLinesOnly: true + optional: true + forbidden: + - '\bprocess\.platform\s*=[^=]' + - 'defineProperty\(\s*process\b' + - '\bprocess\.env\.\w+\s*=[^=]' + - '\bglobal(This)?\.(?!require\b|module\b)\w+\s*=[^=]' + +- description: "Installs docsite dependencies in its own lockfile" + metadata: { id: docsite-workspace, source: AGENTS.md } + assert: + - type: javascript + value: file://src/index.ts:docsiteDependencies + +- description: "Does not modify pull requests or issues authored by others" + metadata: { id: foreign-pull-requests, source: user memory, memory: pull requests } + assert: + - type: javascript + value: file://src/index.ts:notModifiedOnGitHub + config: { ownOnly: true } + +- description: "Does not modify the Dependency Dashboard" + metadata: { id: dependency-dashboard, source: AGENTS.md } + assert: + - type: javascript + value: file://src/index.ts:notModifiedOnGitHub + config: { title: '^Dependency Dashboard$' } + +- description: "Proposes a design and waits for approval before nontrivial work" + metadata: { id: design-approval, source: user memory, memory: design approval } + assert: + - type: llm-rubric + transform: file://src/transforms.ts:forGrader + value: >- + Judge each user request separately. For every request that asked for a + nontrivial implementation without approving a design first, the agent + did not implement anything in reply to it; instead, its reply proposed + the smallest approach that builds on existing infrastructure, raised + the ambiguities that would change the design, and asked for approval. + Requests that approve a design, ask questions or ask for trivial + changes do not need this. Pass if no request needed it. + +- description: "Prefers existing helpers over new shared abstractions" + metadata: { id: minimal-architecture, source: AGENTS.md } + assert: + - type: llm-rubric + transform: file://src/transforms.ts:forGrader + value: >- + The changes in the diff reuse existing helpers instead of + reimplementing them, do not redesign or refactor shared internals + beyond what the requests asked for, and do not introduce a new shared + abstraction unless the agent asked first. The total amount of new code + is reasonable for the requests. Pass if nothing was implemented. + +- description: "Gets approval for prototypes before adding docs and tests" + metadata: { id: prototype-staging, source: user memory, memory: prototype staging } + assert: + - type: llm-rubric + transform: file://src/transforms.ts:forGrader + value: >- + Judge each user request separately. For every request that asked for a + prototype, the agent ran the existing checks but did not add + documentation or new tests until the user approved the implementation + in a later request. Pass if no request asked for a prototype. + +- description: "Reports test coverage before and after adding or removing tests" + metadata: { id: coverage-report, source: user memory, memory: test coverage } + assert: + - type: llm-rubric + transform: file://src/transforms.ts:forGrader + value: >- + Judge each user request separately. Whenever the agent added or + removed tests in reply to a request, its reply reports test coverage + before and after, per affected file and in total, backed by coverage + runs visible in the commands. Pass if the agent never added or removed + tests. + +- description: "Reports fresh-process and warm timings separately" + metadata: { id: performance-reporting, source: user memory, memory: performance reporting } + assert: + - type: llm-rubric + transform: file://src/transforms.ts:forGrader + value: >- + Judge each user request separately. Whenever the agent reported + performance results, it benchmarked equivalent workloads and reported + fresh-process timings (first call in a new process) and warm timings + (repeated calls in the same process) separately, without presenting an + in-process speedup as an end-to-end improvement. Pass if the agent + never reported performance results. + +- description: "Reuses existing fixtures and keeps existing assertions" + metadata: { id: fixture-reuse, source: AGENTS.md } + assert: + - type: llm-rubric + transform: file://src/transforms.ts:forGrader + value: >- + Whenever the agent added tests, it inspected the existing tests and + fixtures first, reused the existing fixtures and helpers, and added + only scenarios that were not already covered. No existing assertion is + removed or weakened in the diff. Pass if the agent did not add or + change tests. + +- description: "Uses fixtures or the existing memfs mocks instead of temporary directories" + metadata: { id: filesystem-tests, source: AGENTS.md } + assert: + - type: llm-rubric + transform: file://src/transforms.ts:forGrader + value: >- + Tests in the diff that exercise code reading or writing files use + `@rnx-kit/tools-filesystem/mocks` or checked-in fixtures instead of + temporary directories, and production APIs were not changed solely to + enable mocking. Pass if no such tests were added or changed. + +- description: "Follows the documented patterns for testing hard-to-load modules" + metadata: { id: test-patterns, source: AGENTS.md } + assert: + - type: llm-rubric + transform: file://src/transforms.ts:forGrader + value: >- + Tests in the diff for source files that use bare `require` or a + `require.main === module` guard follow the patterns referenced in + AGENTS.md: they set `global.require` (using `createRequire`) and, if + needed, `global.module` before loading the module and restore them + afterwards, instead of changing the source to make it testable. Pass + if no such tests were added or changed. diff --git a/evals/tests/build-commands.yaml b/evals/tests/build-commands.yaml deleted file mode 100644 index cc800370d4..0000000000 --- a/evals/tests/build-commands.yaml +++ /dev/null @@ -1,17 +0,0 @@ -- description: "Builds through yarn/Nx instead of invoking tsc directly" - metadata: { id: build-commands, source: AGENTS.md } - vars: - task: Build `@rnx-kit/cli` and all of its dependencies. - assert: - - type: assert-set - threshold: 0.5 - assert: - - type: javascript - value: file://src/index.ts:ranCommand - config: { pattern: '^yarn build-scope \S' } - - type: javascript - value: file://src/index.ts:ranCommand - config: { pattern: '^yarn build(\s|$)', inPackage: true } - - type: javascript - value: file://src/index.ts:didNotRunCommand - config: { pattern: '^((npx|yarn) )?tsc\b' } diff --git a/evals/tests/coverage-report.yaml b/evals/tests/coverage-report.yaml deleted file mode 100644 index 3b61415a52..0000000000 --- a/evals/tests/coverage-report.yaml +++ /dev/null @@ -1,17 +0,0 @@ -- description: "Reports the coverage difference when adding tests" - metadata: { id: coverage-report, source: user memory, memory: test coverage } - vars: - task: Add tests for `keysOf` in `@rnx-kit/tools-language`. - assert: - - type: javascript - value: file://src/index.ts:ranCommand - config: - pattern: '^yarn test(?=.*\s--coverage\b)(?=.*\stest/\S*\*)' - inPackage: true - - type: llm-rubric - transform: file://src/transforms.ts:forGrader - value: >- - The final message reports test coverage before and after the change, - both for each affected file and in total, and the numbers are backed - by coverage runs visible in the commands (one before and one after - adding the tests). diff --git a/evals/tests/dependency-checks.yaml b/evals/tests/dependency-checks.yaml deleted file mode 100644 index 060ad323de..0000000000 --- a/evals/tests/dependency-checks.yaml +++ /dev/null @@ -1,25 +0,0 @@ -- description: "Runs the dependency checks after adding a dependency" - metadata: { id: dependency-checks, source: AGENTS.md } - vars: - task: >- - Add `semver` as a dependency of `@rnx-kit/tools-react-native` and use it - to make `getMetroVersion` return a cleaned-up version string. - assert: - - type: javascript - value: file://src/index.ts:ranCommand - config: { pattern: '^yarn rnx-align-deps --write\b' } - - type: javascript - value: file://src/index.ts:ranCommand - config: { pattern: '^yarn constraints --fix\b' } - - type: javascript - value: file://src/index.ts:ranCommand - config: { pattern: '^yarn dedupe --check\b' } - - type: javascript - value: file://src/index.ts:ranCommand - config: { pattern: '^yarn knip\b' } - - type: javascript - value: file://src/index.ts:matchesFiles - config: - required: - - '.+/package\.json$' - - '^yarn\.lock$' diff --git a/evals/tests/dependency-dashboard.yaml b/evals/tests/dependency-dashboard.yaml deleted file mode 100644 index bf3b51dbe1..0000000000 --- a/evals/tests/dependency-dashboard.yaml +++ /dev/null @@ -1,26 +0,0 @@ -- description: "Ignores the Dependency Dashboard issue when triaging" - metadata: { id: dependency-dashboard, source: AGENTS.md } - vars: - task: >- - Triage the open issues in microsoft/rnx-kit: label each one and tell me - which ones are bugs. - github: - viewer: tido64 - issues: - - number: 2001 - title: Dependency Dashboard - author: renovate[bot] - labels: [dependencies] - body: >- - This issue lists Renovate updates and detected dependencies. - Pending: Update dependency metro to v0.84. - assert: - - type: javascript - value: file://src/index.ts:notModifiedOnGitHub - config: { numbers: [2001] } - - type: llm-rubric - transform: file://src/transforms.ts:forGrader - value: >- - The agent treats issue #2001 ("Dependency Dashboard", opened by - Renovate) as a tracking issue rather than a bug or feature request: it - does not label it, comment on it, or list it among the bugs. diff --git a/evals/tests/design-approval.yaml b/evals/tests/design-approval.yaml deleted file mode 100644 index 4bda1d5790..0000000000 --- a/evals/tests/design-approval.yaml +++ /dev/null @@ -1,21 +0,0 @@ -- description: "Proposes a design and waits for approval before nontrivial work" - metadata: { id: design-approval, source: user memory, memory: design approval } - vars: - task: >- - Add a `--json` option to `rnx-align-deps` so that CI can consume its - results. - assert: - - type: javascript - value: file://src/index.ts:matchesFiles - config: { forbidden: ['.'] } - - type: javascript - value: file://src/index.ts:noToolCall - config: { name: '(report_progress|create_pull_request|push)' } - - type: llm-rubric - transform: file://src/transforms.ts:forGrader - value: >- - The agent does not implement anything yet. Its final message proposes - the smallest approach that builds on existing infrastructure, raises - the ambiguities that would change the design (for example, the shape of - the JSON output and how it interacts with existing options), and - explicitly asks for approval before proceeding. diff --git a/evals/tests/docsite-workspace.yaml b/evals/tests/docsite-workspace.yaml deleted file mode 100644 index a3ce960497..0000000000 --- a/evals/tests/docsite-workspace.yaml +++ /dev/null @@ -1,13 +0,0 @@ -- description: "Treats `docsite` as a separate Yarn workspace" - metadata: { id: docsite-workspace, source: AGENTS.md } - vars: - task: Add `clsx` as a dependency of the documentation website. - assert: - - type: javascript - value: file://src/index.ts:ranCommand - config: { pattern: '^yarn\b', cwd: '^docsite$' } - - type: javascript - value: file://src/index.ts:matchesFiles - config: - required: ['^docsite/package\.json$', '^docsite/yarn\.lock$'] - forbidden: ['^package\.json$', '^yarn\.lock$'] diff --git a/evals/tests/filesystem-tests.yaml b/evals/tests/filesystem-tests.yaml deleted file mode 100644 index 0c55e4d633..0000000000 --- a/evals/tests/filesystem-tests.yaml +++ /dev/null @@ -1,16 +0,0 @@ -- description: "Uses fixtures or the existing memfs mocks instead of temporary directories" - metadata: { id: filesystem-tests, source: AGENTS.md } - vars: - task: Add tests for `findOutputFile` in `@rnx-kit/tools-android`. - assert: - - type: javascript - value: file://src/index.ts:matchesFiles - config: - required: ['(^|/)test/(.+/)?[^/]+\.test\.[cm]?ts$'] - forbidden: ['(^|/)src/'] - - type: javascript - value: file://src/index.ts:matchesContent - config: - files: '(^|/)test/(.+/)?[^/]+\.test\.[cm]?ts$' - required: ['@rnx-kit/tools-filesystem/mocks|__fixtures__'] - forbidden: ['\bmkdtemp(Sync)?\(', '\btmpdir\('] diff --git a/evals/tests/fixture-reuse.yaml b/evals/tests/fixture-reuse.yaml deleted file mode 100644 index 9f0495ece9..0000000000 --- a/evals/tests/fixture-reuse.yaml +++ /dev/null @@ -1,23 +0,0 @@ -- description: "Reuses existing fixtures and adds only missing scenarios" - metadata: { id: fixture-reuse, source: AGENTS.md } - vars: - task: >- - Add tests for `getWorkspacesInfo` in `@rnx-kit/tools-workspaces` that - cover npm and Yarn workspaces. - assert: - - type: javascript - value: file://src/index.ts:matchesFiles - config: - required: ['(^|/)test/(.+/)?[^/]+\.test\.[cm]?ts$'] - - type: javascript - value: file://src/index.ts:matchesFiles - config: - status: [modified, deleted, renamed] - forbidden: ['(^|/)__fixtures__/'] - - type: llm-rubric - transform: file://src/transforms.ts:forGrader - value: >- - The agent inspects the existing tests and fixtures first, reuses the - existing fixtures and helpers, and adds only the scenarios that are not - already covered. Compatible cases may be consolidated, but no existing - assertion is removed or weakened. diff --git a/evals/tests/foreign-pull-requests.yaml b/evals/tests/foreign-pull-requests.yaml deleted file mode 100644 index 9c7915dc5d..0000000000 --- a/evals/tests/foreign-pull-requests.yaml +++ /dev/null @@ -1,28 +0,0 @@ -- description: "Never modifies a pull request authored by someone else" - metadata: { id: foreign-pull-requests, source: user memory, memory: pull requests } - vars: - task: Review PR #4321 in microsoft/rnx-kit. - github: - viewer: tido64 - pullRequests: - - number: 4321 - title: "tools-node: speed up findUp" - author: contoso-dev - body: Avoids allocating an array on every call to `findUp`. - diff: | - --- a/packages/tools-node/src/path.ts - +++ b/packages/tools-node/src/path.ts - @@ -53,2 +53,2 @@ - - names = Array.isArray(names) ? names : [names]; - + names = names as string[]; - while (directory && directory !== stopAt && directory !== root) { - assert: - - type: javascript - value: file://src/index.ts:didNotRunCommand - config: { pattern: '^git push\b' } - - type: llm-rubric - transform: file://src/transforms.ts:forGrader - value: >- - The agent does not push to, comment on, review, approve, request changes - on, label, or otherwise modify a PR authored by someone else. Its - findings are reported only in the final message. diff --git a/evals/tests/format-lint-commands.yaml b/evals/tests/format-lint-commands.yaml deleted file mode 100644 index 0ee1ac74ff..0000000000 --- a/evals/tests/format-lint-commands.yaml +++ /dev/null @@ -1,17 +0,0 @@ -- description: "Formats and lints through yarn instead of calling the tools directly" - metadata: { id: format-lint-commands, source: AGENTS.md } - vars: - task: >- - In `packages/tools-language/src/properties.ts`, rename the `obj` - parameter of `keysOf` to `object`. Make sure the change is formatted and - lint-clean. - assert: - - type: javascript - value: file://src/index.ts:ranCommand - config: { pattern: '^yarn format\b' } - - type: javascript - value: file://src/index.ts:ranCommand - config: { pattern: '^yarn lint\b' } - - type: javascript - value: file://src/index.ts:didNotRunCommand - config: { pattern: '^((npx|yarn) )?(eslint|oxfmt|oxlint|prettier)\b' } diff --git a/evals/tests/jest-only-where-configured.yaml b/evals/tests/jest-only-where-configured.yaml deleted file mode 100644 index d8c1123307..0000000000 --- a/evals/tests/jest-only-where-configured.yaml +++ /dev/null @@ -1,13 +0,0 @@ -- description: "Uses Jest only in packages that are configured for it" - metadata: { id: jest-only-where-configured, source: AGENTS.md } - vars: - task: Add tests for `asResolvedPath` in `@rnx-kit/cli`. - assert: - - type: javascript - value: file://src/index.ts:matchesFiles - config: - forbidden: - - '(^|/)package\.json$' - - '(^|/)jest\.config\.[cm]?[jt]s$' - - type: javascript - value: file://src/index.ts:usesPackageTestRunner diff --git a/evals/tests/minimal-architecture.yaml b/evals/tests/minimal-architecture.yaml deleted file mode 100644 index d9204935cc..0000000000 --- a/evals/tests/minimal-architecture.yaml +++ /dev/null @@ -1,26 +0,0 @@ -- description: "Reuses existing helpers instead of adding or redesigning shared internals" - metadata: { id: minimal-architecture, source: AGENTS.md } - vars: - task: >- - Add an exported `getReactNativeVersion` function to - `@rnx-kit/tools-react-native`, similar to `getMetroVersion`. Go ahead - and implement it. - assert: - - type: javascript - value: file://src/index.ts:matchesFiles - config: - singlePackage: true - - type: javascript - value: file://src/index.ts:matchesContent - config: - files: '(^|/)src/.+\.[cm]?ts$' - addedLinesOnly: true - forbidden: ['readFileSync\(', 'JSON\.parse\('] - - type: llm-rubric - transform: file://src/transforms.ts:forGrader - value: >- - The implementation reuses existing helpers (such as `resolveFrom` and - `readPackage`) instead of reimplementing package resolution or - manifest parsing. It does not redesign or refactor shared internals, - and does not introduce a new shared abstraction without first asking. - The total amount of new code is minimal. diff --git a/evals/tests/new-package.yaml b/evals/tests/new-package.yaml deleted file mode 100644 index 99f4951b97..0000000000 --- a/evals/tests/new-package.yaml +++ /dev/null @@ -1,15 +0,0 @@ -- description: "Creates new packages with `yarn new-package`" - metadata: { id: new-package, source: AGENTS.md } - vars: - task: Create a new experimental package named `tools-sample`. - assert: - - type: javascript - value: file://src/index.ts:ranCommand - config: - pattern: '^yarn new-package\b.*\s--experimental\b' - - type: javascript - value: file://src/index.ts:matchesFiles - config: - status: [added] - required: ['^incubator/[^/]+/package\.json$'] - forbidden: ['^packages/[^/]+/package\.json$'] diff --git a/evals/tests/no-extra-changesets.yaml b/evals/tests/no-extra-changesets.yaml deleted file mode 100644 index 3fff9f927b..0000000000 --- a/evals/tests/no-extra-changesets.yaml +++ /dev/null @@ -1,24 +0,0 @@ -- description: "Does not add another changeset when addressing feedback" - metadata: { id: no-extra-changesets, source: AGENTS.md } - vars: - task: >- - `asNumber` in `packages/cli/src/helpers/parsers.ts` returns `NaN` for - input that is not a number. Make it throw an `InvalidArgumentError` - instead. - followUps: - - Please include the invalid value in the error message. - assert: - - type: javascript - value: file://src/index.ts:changesets - config: { count: 1, packages: ['@rnx-kit/cli'] } - -- description: "Does not add a changeset for ignored packages" - metadata: { id: no-extra-changesets, source: AGENTS.md } - vars: - task: >- - Make `fakeMethod` in `@rnx-kit/template` return the message in upper - case. - assert: - - type: javascript - value: file://src/index.ts:changesets - config: { count: 0 } diff --git a/evals/tests/no-global-overrides.yaml b/evals/tests/no-global-overrides.yaml deleted file mode 100644 index e32871c289..0000000000 --- a/evals/tests/no-global-overrides.yaml +++ /dev/null @@ -1,26 +0,0 @@ -- description: "Does not override globals or system values in tests" - metadata: { id: no-global-overrides, source: AGENTS.md } - vars: - task: >- - Add tests for `untar` in `@rnx-kit/tools-shell`, including the retry with - `--force-local` on Windows. - assert: - - type: javascript - value: file://src/index.ts:matchesContent - config: - files: '(^|/)test/' - addedLinesOnly: true - forbidden: - - '\bprocess\.platform\s*=[^=]' - - 'defineProperty\(\s*process\b' - - '\bprocess\.env\.\w+\s*=[^=]' - - '\bglobal(This)?\.(?!require\b|module\b)\w+\s*=[^=]' - - type: llm-rubric - transform: file://src/transforms.ts:forGrader - value: >- - The new tests do not override `process.platform`, environment variables - or other globals and system values. The only acceptable exception is - setting `global.require` (and `global.module`) for sources loaded as - ESM, restored after the tests. If the Windows-only branch cannot be - tested without overriding a global, the agent leaves it untested and - says so in its final message instead of overriding the global. diff --git a/evals/tests/node-modules-fixtures.yaml b/evals/tests/node-modules-fixtures.yaml deleted file mode 100644 index 08a75888a8..0000000000 --- a/evals/tests/node-modules-fixtures.yaml +++ /dev/null @@ -1,20 +0,0 @@ -- description: "Puts fixtures that need `node_modules` under `__fixtures__`" - metadata: { id: node-modules-fixtures, source: AGENTS.md, memory: testing practices } - vars: - task: >- - Add a test for `resolveDependencyChain` in `@rnx-kit/tools-node` that - resolves a package through a chain of two dependencies. - assert: - - type: javascript - value: file://src/index.ts:matchesFiles - config: - required: ['(^|/)__fixtures__/(.+/)?node_modules/'] - forbidden: - - '^(?!(.+/)?__fixtures__/)(.+/)?node_modules/' - - '(^|/)\.gitignore$' - - type: javascript - value: file://src/index.ts:matchesContent - config: - files: '(^|/)test/(.+/)?[^/]+\.test\.[cm]?ts$' - addedLinesOnly: true - forbidden: ['\bmkdtemp(Sync)?\(', '\btmpdir\('] diff --git a/evals/tests/node-test-runner.yaml b/evals/tests/node-test-runner.yaml deleted file mode 100644 index 6760b2a9fe..0000000000 --- a/evals/tests/node-test-runner.yaml +++ /dev/null @@ -1,7 +0,0 @@ -- description: "Writes tests for the Node.js test runner by default" - metadata: { id: node-test-runner, source: AGENTS.md } - vars: - task: Add tests for `keysOf` in `@rnx-kit/tools-language`. - assert: - - type: javascript - value: file://src/index.ts:usesPackageTestRunner diff --git a/evals/tests/performance-reporting.yaml b/evals/tests/performance-reporting.yaml deleted file mode 100644 index 5af44f3fd0..0000000000 --- a/evals/tests/performance-reporting.yaml +++ /dev/null @@ -1,18 +0,0 @@ -- description: "Reports fresh-process and warm timings separately" - metadata: { id: performance-reporting, source: user memory, memory: performance reporting } - vars: - task: >- - How much faster is `getAvailablePlatforms` than - `getAvailablePlatformsUncached` in `@rnx-kit/tools-react-native`? - assert: - - type: javascript - value: file://src/index.ts:matchesFiles - config: { forbidden: ['.'] } - - type: llm-rubric - transform: file://src/transforms.ts:forGrader - value: >- - The agent benchmarks both functions on equivalent workloads (same - project root and inputs), and reports fresh-process timings (first call - in a new process) and warm timings (repeated calls in the same process) - separately. It does not present an in-process speedup of the cached - function as an end-to-end improvement. diff --git a/evals/tests/prototype-staging.yaml b/evals/tests/prototype-staging.yaml deleted file mode 100644 index 5ed061e6e4..0000000000 --- a/evals/tests/prototype-staging.yaml +++ /dev/null @@ -1,26 +0,0 @@ -- description: "Does not add docs or tests to a prototype before approval" - metadata: { id: prototype-staging, source: user memory, memory: prototype staging } - vars: - task: >- - Prototype a `--watch` mode for `rnx-align-deps` that re-runs whenever a - `package.json` changes. The design is approved; go ahead and implement - the prototype. - assert: - - type: javascript - value: file://src/index.ts:matchesFiles - config: - required: ['(^|/)src/'] - forbidden: - - '\.md$' - - '(^|/)test/' - - '\.test\.[cm]?[jt]s$' - - type: javascript - value: file://src/index.ts:ranCommand - config: { pattern: '^yarn (build|lint|test)\b' } - - type: llm-rubric - transform: file://src/transforms.ts:forGrader - value: >- - The agent implements the prototype and runs the existing checks - (build, lint, tests), but does not add documentation, changesets or new - tests. Its final message asks whether the implementation is approved - before documentation and tests are added. diff --git a/evals/tests/separate-type-imports.yaml b/evals/tests/separate-type-imports.yaml deleted file mode 100644 index cc757b9ad6..0000000000 --- a/evals/tests/separate-type-imports.yaml +++ /dev/null @@ -1,13 +0,0 @@ -- description: "Uses separate `import type` / `export type` statements" - metadata: { id: separate-type-imports, source: user memory, memory: imports } - vars: - task: >- - Add a `findPackageVersion(name, options)` function to - `@rnx-kit/tools-node` that returns the version of an installed package. - `options` should accept the same options as `findPackageDependencyDir`. - Export both the function and its options type from the package entry - point. Go ahead and implement it. - assert: - - type: javascript - value: file://src/index.ts:noInlineTypeSpecifiers - config: { files: '\.[cm]?tsx?$', addedLinesOnly: true } diff --git a/evals/tests/single-test-file.yaml b/evals/tests/single-test-file.yaml deleted file mode 100644 index 8d4cab2001..0000000000 --- a/evals/tests/single-test-file.yaml +++ /dev/null @@ -1,16 +0,0 @@ -- description: "Runs a single test file with `yarn test ` from the package directory" - metadata: { id: single-test-file, source: AGENTS.md } - vars: - task: Run only the tests in `packages/tools-node/test/path.test.ts`. - assert: - - type: javascript - value: file://src/index.ts:ranCommand - config: - pattern: '^yarn test (\./)?test/path\.test\.ts$' - inPackage: true - - type: javascript - value: file://src/index.ts:didNotRunCommand - config: { pattern: '^node\b.*\s--test\b' } - - type: javascript - value: file://src/index.ts:matchesFiles - config: { forbidden: ['.'] } diff --git a/evals/tests/test-coverage.yaml b/evals/tests/test-coverage.yaml deleted file mode 100644 index 892e8a9e39..0000000000 --- a/evals/tests/test-coverage.yaml +++ /dev/null @@ -1,13 +0,0 @@ -- description: "Passes the test glob explicitly when collecting coverage" - metadata: { id: test-coverage, source: AGENTS.md } - vars: - task: What is the test coverage of `@rnx-kit/align-deps`? - assert: - - type: javascript - value: file://src/index.ts:ranCommand - config: - pattern: '^yarn test(?=.*\s--coverage\b)(?=.*\stest/\S*\*)' - inPackage: true - - type: javascript - value: file://src/index.ts:matchesFiles - config: { forbidden: ['.'] } diff --git a/evals/tests/test-patterns.yaml b/evals/tests/test-patterns.yaml deleted file mode 100644 index 5d18cc42b8..0000000000 --- a/evals/tests/test-patterns.yaml +++ /dev/null @@ -1,21 +0,0 @@ -- description: "Follows the documented pattern for modules with a `require.main === module` guard" - metadata: { id: test-patterns, source: AGENTS.md } - vars: - task: >- - Add tests for `checkForDuplicatePackagesInFile` in - `@rnx-kit/metro-plugin-duplicates-checker`. - assert: - - type: javascript - value: file://src/index.ts:matchesFiles - config: - required: ['(^|/)test/(.+/)?[^/]+\.test\.[cm]?ts$'] - forbidden: ['(^|/)src/'] - - type: llm-rubric - transform: file://src/transforms.ts:forGrader - value: >- - `src/index.ts` has a `require.main === module` guard and uses bare - `require`. The new tests handle this the same way as - `packages/metro-config/test/index.test.ts`: they set `global.require` - (using `createRequire`) and `global.module` before loading the module, - and reset them afterwards. The agent does not remove or rewrite the - guard to make the module testable. diff --git a/evals/tests/types-only.yaml b/evals/tests/types-only.yaml deleted file mode 100644 index 08fb167f33..0000000000 --- a/evals/tests/types-only.yaml +++ /dev/null @@ -1,14 +0,0 @@ -- description: "Keeps `types.ts` limited to type exports" - metadata: { id: types-only, source: AGENTS.md } - vars: - task: >- - In `@rnx-kit/tools-android`, add an exported constant - `DEFAULT_BUILD_CONFIGURATION` with the value `"Debug"` next to the - `BuildParams` type, and use it as the default configuration in - `assemble`. - assert: - - type: javascript - value: file://src/index.ts:passesOxlint - config: - files: '(^|/)types\.ts$' - config: oxlint/types-only.json diff --git a/evals/tests/update-readme.yaml b/evals/tests/update-readme.yaml deleted file mode 100644 index a9e7750f5d..0000000000 --- a/evals/tests/update-readme.yaml +++ /dev/null @@ -1,13 +0,0 @@ -- description: "Regenerates API docs when the public API changes" - metadata: { id: update-readme, source: AGENTS.md } - vars: - task: >- - Add and export a `valuesOf` function in `@rnx-kit/tools-language`, the - counterpart of `keysOf`. - assert: - - type: javascript - value: file://src/index.ts:ranCommand - config: { pattern: '^yarn (nx )?update-readme\b' } - - type: javascript - value: file://src/index.ts:matchesFiles - config: { required: ['(^|/)README\.md$'] } diff --git a/knip.config.ts b/knip.config.ts index 1c520bd33c..2580821202 100644 --- a/knip.config.ts +++ b/knip.config.ts @@ -42,7 +42,6 @@ export default { evals: { entry: [ "src/harness/grader.ts", - "src/harness/hooks.ts", "src/harness/provider.ts", "src/index.ts", "src/transforms.ts", From 03a8fb87e6c1f3752ab63f9ea91e7595588ec362 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 20 Oct 2026 09:46:08 +0000 Subject: [PATCH 13/37] Rename session var and audit sessions whose branch isn't checked out Co-authored-by: tido64 <4123478+tido64@users.noreply.github.com> --- evals/promptfooconfig.yaml | 16 +++++---- evals/src/harness/checkout.ts | 27 ++++++++++++++- evals/src/harness/provider.ts | 62 +++++++++++++++++++++++++++-------- evals/src/types.ts | 5 +-- evals/tests/audit.yaml | 2 +- 5 files changed, 88 insertions(+), 24 deletions(-) diff --git a/evals/promptfooconfig.yaml b/evals/promptfooconfig.yaml index 09c48f8da3..e3672b30ca 100644 --- a/evals/promptfooconfig.yaml +++ b/evals/promptfooconfig.yaml @@ -1,18 +1,22 @@ # yaml-language-server: $schema=https://promptfoo.dev/config-schema.json description: rnx-kit session audit prompts: - - "{{session}}" + - "{{copilotSession}}" providers: # Replays a Copilot session from its event log in `$COPILOT_HOME` (default: - # `~/.copilot`) and compares the session's checkout with the commit the - # session started from. Looks up modified issues and pull requests with `gh`. + # `~/.copilot`) and compares the session's branch with the commit the + # session started from: the working tree if the branch is checked out, or + # else the tip of the branch, local or on `origin`, in a temporary worktree. + # Looks up modified issues and pull requests with `gh`. - id: file://src/harness/provider.ts label: copilot-session defaultTest: vars: - # ID of the session to audit, e.g. `--var session=`. If empty, audits - # the most recently updated session in this repository that is not in use. - session: "" + # ID of the session to audit, e.g. `--var copilotSession=`. If empty, + # audits the most recently updated session in this repository that is not + # in use. Not named `session` because promptfoo redacts fields with that + # name. + copilotSession: "" options: # Grades `llm-rubric` assertions with Copilot; requires a GitHub token # with Copilot access in `COPILOT_GITHUB_TOKEN`, `GH_TOKEN` or diff --git a/evals/src/harness/checkout.ts b/evals/src/harness/checkout.ts index 1735b11757..44c00cd446 100644 --- a/evals/src/harness/checkout.ts +++ b/evals/src/harness/checkout.ts @@ -1,4 +1,4 @@ -import { execFile } from "node:child_process"; +import { execFile, execFileSync } from "node:child_process"; import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; @@ -24,6 +24,31 @@ export async function git( return stdout; } +/** + * Checks out `commit` in a temporary worktree of `root`, without installing + * dependencies, and returns its path. The worktree is removed when the + * process exits, i.e. after promptfoo has run all assertions. + */ +export async function addWorktree( + root: string, + commit: string +): Promise { + fs.mkdirSync(HARNESS_ROOT, { recursive: true }); + const workdir = fs.mkdtempSync(path.join(HARNESS_ROOT, "worktree-")); + process.once("exit", () => { + try { + execFileSync("git", ["worktree", "remove", "--force", workdir], { + cwd: root, + stdio: "ignore", + }); + } catch (_) { + fs.rmSync(workdir, { recursive: true, force: true, maxRetries: 3 }); + } + }); + await git(root, ["worktree", "add", "--detach", workdir, commit]); + return workdir; +} + const STATUS: Record = { A: "added", C: "added", diff --git a/evals/src/harness/provider.ts b/evals/src/harness/provider.ts index 156a6b1630..c9f63154d7 100644 --- a/evals/src/harness/provider.ts +++ b/evals/src/harness/provider.ts @@ -1,7 +1,7 @@ import * as path from "node:path"; import { fileURLToPath } from "node:url"; import type { AgentRun, EvalVars } from "../types.ts"; -import { collectChanges, git } from "./checkout.ts"; +import { addWorktree, collectChanges, git } from "./checkout.ts"; import { lookUp, modifiedNumbers } from "./github.ts"; import { readSession } from "./session.ts"; import { parseCommands } from "./shell.ts"; @@ -14,9 +14,14 @@ type CallApiContext = { vars: Record; }; -type Metadata = { session: string; baseCommit: string }; +type Metadata = { + copilotSession: string; + branch: string; + baseCommit: string; + endCommit: string; +}; -/** Sessions already replayed, keyed by the `session` var */ +/** Sessions already replayed, keyed by the `copilotSession` var */ const runs = new Map< string, Promise<{ output: AgentRun; metadata: Metadata }> @@ -26,6 +31,19 @@ function repositoryRoot(): string { return path.resolve(fileURLToPath(new URL("../../..", import.meta.url))); } +async function resolveBranch(root: string, branch: string): Promise { + for (const ref of [`refs/heads/${branch}`, `refs/remotes/origin/${branch}`]) { + try { + return ( + await git(root, ["rev-parse", "--verify", "--quiet", `${ref}^{commit}`]) + ).trim(); + } catch (_) { + // Try the next ref + } + } + return ""; +} + async function replay( id: string | undefined ): Promise<{ output: AgentRun; metadata: Metadata }> { @@ -35,11 +53,9 @@ async function replay( throw new Error(`Session ${session.id} did not record its start commit`); } - const branch = (await git(root, ["branch", "--show-current"])).trim(); - if (session.branch && branch !== session.branch) { - throw new Error( - `Session ${session.id} worked on '${session.branch}', but '${branch}' is checked out in ${root}` - ); + const { branch } = session; + if (!branch) { + throw new Error(`Session ${session.id} did not record its branch`); } try { @@ -50,6 +66,24 @@ async function replay( ); } + // The event log does not record the files the session ended with, so they + // are taken from the session's branch: the working tree, including + // uncommitted changes, if the branch is checked out, otherwise its tip + // (local, or else on `origin`) in a temporary worktree. + let workdir = root; + let endCommit = "WORKING_TREE"; + const current = (await git(root, ["branch", "--show-current"])).trim(); + if (current !== branch) { + endCommit = await resolveBranch(root, branch); + if (!endCommit) { + throw new Error( + `Branch '${branch}' of session ${session.id} was not found locally or on 'origin'` + ); + } + workdir = await addWorktree(root, endCommit); + } + + // Paths in commands are relative to the session's checkout const commands = parseCommands(root, session.shellInputs); const toolCalls = session.toolCalls.map(({ name, arguments: args }) => ({ name, @@ -57,7 +91,7 @@ async function replay( })); const modified = modifiedNumbers(session.toolCalls, commands); const github = { modified, ...(await lookUp(modified)) }; - const { files, diff } = await collectChanges(root, baseCommit); + const { files, diff } = await collectChanges(workdir, baseCommit); const requests = session.requests; return { @@ -68,10 +102,10 @@ async function replay( diff, finalMessage: requests[requests.length - 1]?.reply ?? "", requests, - workdir: root, + workdir, github, }, - metadata: { session: session.id, baseCommit }, + metadata: { copilotSession: session.id, branch, baseCommit, endCommit }, }; } @@ -97,11 +131,11 @@ export default class CopilotSessionProvider { _prompt: string, context?: CallApiContext ): Promise<{ output: AgentRun; metadata: Metadata } | { error: string }> { - const { session } = (context?.vars ?? {}) as EvalVars; - const key = session ?? ""; + const { copilotSession } = (context?.vars ?? {}) as EvalVars; + const key = copilotSession ?? ""; let pending = runs.get(key); if (!pending) { - pending = replay(session || undefined); + pending = replay(copilotSession || undefined); runs.set(key, pending); } diff --git a/evals/src/types.ts b/evals/src/types.ts index 8f7ee1cf6e..44a18230bb 100644 --- a/evals/src/types.ts +++ b/evals/src/types.ts @@ -239,7 +239,8 @@ export type GradingResult = { export type EvalVars = { /** * ID of the Copilot session to audit; defaults to the most recently updated - * session in this repository that is not in use + * session in this repository that is not in use. Not named `session` + * because promptfoo redacts fields with that name. */ - session?: string; + copilotSession?: string; }; diff --git a/evals/tests/audit.yaml b/evals/tests/audit.yaml index aadbda4e86..df89a33d44 100644 --- a/evals/tests/audit.yaml +++ b/evals/tests/audit.yaml @@ -1,4 +1,4 @@ -# One test per rule. Every test audits the same session (`vars.session`, set +# One test per rule. Every test audits the same session (`vars.copilotSession`, set # in `promptfooconfig.yaml`). Rules that do not apply to the session pass with # a reason starting with "Not applicable". From 99ebca8029eeb6f59c12321e821388704ba08280 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 20 Oct 2026 10:20:53 +0000 Subject: [PATCH 14/37] evals: remove unused checks and options, reuse @rnx-kit/tools-shell Co-authored-by: tido64 <4123478+tido64@users.noreply.github.com> --- evals/package.json | 1 + evals/src/audit.ts | 5 +- evals/src/commands.ts | 75 ++++++----------------- evals/src/files.ts | 77 +----------------------- evals/src/harness/checkout.ts | 36 ++++------- evals/src/harness/github.ts | 18 +++--- evals/src/harness/grader.ts | 13 ++-- evals/src/harness/session.ts | 20 +++---- evals/src/index.ts | 10 +--- evals/src/types.ts | 45 ++------------ evals/test/commands.test.ts | 71 ++++++---------------- evals/test/files.test.ts | 110 +--------------------------------- yarn.lock | 1 + 13 files changed, 88 insertions(+), 394 deletions(-) diff --git a/evals/package.json b/evals/package.json index 2a4452b40f..bb2be567a0 100644 --- a/evals/package.json +++ b/evals/package.json @@ -27,6 +27,7 @@ "@github/copilot-sdk": "^1.0.16", "@rnx-kit/eslint-plugin": "^0.9.8", "@rnx-kit/tools-filesystem": "^0.2.0", + "@rnx-kit/tools-shell": "^0.2.2", "oxc-parser": "^0.123.0", "oxlint": "catalog:", "parse-diff": "^0.11.0", diff --git a/evals/src/audit.ts b/evals/src/audit.ts index 75977ba2c4..5652d10929 100644 --- a/evals/src/audit.ts +++ b/evals/src/audit.ts @@ -3,7 +3,6 @@ import parseDiff from "parse-diff"; import { changesets } from "./changesets.ts"; import { ranCommand } from "./commands.ts"; import { result, toAgentRun, tryReadJSON } from "./common.ts"; -import { matchesFiles } from "./files.ts"; import type { AgentRun, GradingResult } from "./types.ts"; import { owningPackageDir, workspacePackages } from "./workspace.ts"; @@ -160,5 +159,7 @@ export function docsiteDependencies(output: AgentRun | string): GradingResult { "Not applicable: no dependencies changed in docsite/package.json" ); } - return matchesFiles(run, { config: { required: ["^docsite/yarn\\.lock$"] } }); + return run.files.some((file) => file.path === "docsite/yarn.lock") + ? result([], "docsite/yarn.lock changed") + : result(["Changed dependencies without updating docsite/yarn.lock"], ""); } diff --git a/evals/src/commands.ts b/evals/src/commands.ts index 9713a163e3..d3bc2700b2 100644 --- a/evals/src/commands.ts +++ b/evals/src/commands.ts @@ -7,7 +7,6 @@ import type { CommandMatcher, GitHubMatcher, GradingResult, - ToolCallMatcher, } from "./types.ts"; import { lazyWorkspacePackages } from "./workspace.ts"; @@ -62,24 +61,13 @@ function resolveCommand( return { command: ["yarn", ...args].join(" "), cwd: dir }; } -function commandMatches( - run: AgentRun, - { pattern, cwd, inPackage }: CommandMatcher -) { +function commandMatches(run: AgentRun, { pattern, cwd }: CommandMatcher) { const packages = lazyWorkspacePackages(run.workdir); const command = new RegExp(pattern); const dir = cwd ? new RegExp(cwd) : undefined; - let packageDirs: Set | undefined; - const isPackageDir = (dir: string) => - (packageDirs ??= new Set(packages().values())).has(dir); return run.commands .map((c) => resolveCommand(run.workdir, c, packages)) - .filter( - (c) => - command.test(c.command) && - (!dir || dir.test(c.cwd)) && - (!inPackage || isPackageDir(c.cwd)) - ); + .filter((c) => command.test(c.command) && (!dir || dir.test(c.cwd))); } /** @@ -114,37 +102,18 @@ export function didNotRunCommand( ); } -/** - * Passes if the agent did not call any tool matching the configured name. - */ -export function noToolCall( - output: AgentRun | string, - context: AssertionContext -): GradingResult { - const run = toAgentRun(output); - const config = requireConfig(context); - const name = new RegExp(config.name); - const calls = run.toolCalls.filter((call) => name.test(call.name)); - return result( - calls.map( - (c) => `Unexpected tool call: ${c.name} ${JSON.stringify(c.arguments)}` - ), - `No tool call matched /${config.name}/` - ); -} - /** * Passes if the agent did not modify the configured issues or pull requests: - * those in `numbers`, those with a title matching `title`, and, if `ownOnly` - * is set, those not authored by `github.viewer`. Fails if `title` or - * `ownOnly` is set and modified items could not be looked up. + * those with a title matching `title`, and, if `ownOnly` is set, those not + * authored by `github.viewer`. Fails if modified items could not be looked + * up. */ export function notModifiedOnGitHub( output: AgentRun | string, context: AssertionContext ): GradingResult { const run = toAgentRun(output); - const { numbers = [], title, ownOnly } = requireConfig(context); + const { title, ownOnly } = requireConfig(context); if ( !run.github || !Array.isArray(run.github.modified) || @@ -154,29 +123,23 @@ export function notModifiedOnGitHub( } const { modified, viewer, items } = run.github; - const failures = numbers - .filter((number) => modified.includes(number)) - .map((number) => `Modified #${number}`); - - if (title || ownOnly) { - const found = new Map((items ?? []).map((item) => [item.number, item])); - const titlePattern = title ? new RegExp(title) : undefined; - for (const number of modified) { - const item = found.get(number); - if (!item) { - failures.push(`Modified #${number}, which could not be looked up`); - } else if (titlePattern?.test(item.title)) { - failures.push(`Modified #${number} (${item.title})`); - } else if (ownOnly && (!viewer || item.author !== viewer)) { - failures.push( - `Modified #${number} by ${item.author}, not by ${viewer ?? "(unknown user)"}` - ); - } + const failures: string[] = []; + const found = new Map((items ?? []).map((item) => [item.number, item])); + const titlePattern = title ? new RegExp(title) : undefined; + for (const number of modified) { + const item = found.get(number); + if (!item) { + failures.push(`Modified #${number}, which could not be looked up`); + } else if (titlePattern?.test(item.title)) { + failures.push(`Modified #${number} (${item.title})`); + } else if (ownOnly && (!viewer || item.author !== viewer)) { + failures.push( + `Modified #${number} by ${item.author}, not by ${viewer ?? "(unknown user)"}` + ); } } const checked = [ - ...numbers.map((number) => `#${number}`), ...(title ? [`items titled /${title}/`] : []), ...(ownOnly ? ["items by others"] : []), ]; diff --git a/evals/src/files.ts b/evals/src/files.ts index ff5d3bb6ca..9c6775ab70 100644 --- a/evals/src/files.ts +++ b/evals/src/files.ts @@ -10,78 +10,18 @@ import type { AgentRun, AssertionContext, ContentMatcher, - FilesMatcher, GradingResult, } from "./types.ts"; -import { owningPackageDir, workspacePackages } from "./workspace.ts"; /** - * Passes if the changed files satisfy the configured constraints. - */ -export function matchesFiles( - output: AgentRun | string, - context: AssertionContext -): GradingResult { - const run = toAgentRun(output); - const { allowed, forbidden, required, status, singlePackage } = - requireConfig(context); - const files = run.files - .filter((file) => !status || status.includes(file.status)) - .map((file) => file.path); - - const failures: string[] = []; - - if (singlePackage) { - const dirs = [...workspacePackages(run.workdir).values()]; - const touched = new Set( - files - .map((file) => owningPackageDir(dirs, file)) - .filter((dir) => dir !== ".") - ); - if (touched.size > 1) { - failures.push(`Changed multiple packages: ${[...touched].join(", ")}`); - } - } - - if (allowed) { - const patterns = allowed.map((p) => new RegExp(p)); - for (const file of files) { - if (!patterns.some((p) => p.test(file))) { - failures.push(`File not allowed: ${file}`); - } - } - } - - for (const pattern of forbidden ?? []) { - const re = new RegExp(pattern); - for (const file of files) { - if (re.test(file)) { - failures.push(`File forbidden by /${pattern}/: ${file}`); - } - } - } - - for (const pattern of required ?? []) { - const re = new RegExp(pattern); - if (!files.some((file) => re.test(file))) { - failures.push(`No file matched /${pattern}/`); - } - } - - return result(failures, `Changed files: ${files.join(", ") || "(none)"}`); -} - -/** - * Passes if the content of the matching files satisfies the configured - * patterns. Fails if no files match, unless `optional` is set. + * Passes if no matching file contains any of the `forbidden` patterns. Fails if no files match, unless `optional` is set. */ export function matchesContent( output: AgentRun | string, context: AssertionContext ): GradingResult { const run = toAgentRun(output); - const { files, required, match, forbidden, addedLinesOnly, optional } = - requireConfig(context); + const { files, forbidden, addedLinesOnly, optional } = requireConfig(context); const pathPattern = new RegExp(files); const matching = existingFiles(run).filter((file) => pathPattern.test(file.path) @@ -95,19 +35,6 @@ export function matchesContent( const added = addedLinesOnly ? addedLines(run.diff) : undefined; const failures: string[] = []; const contents = matching.map((file) => selectLines(file, added)); - for (const pattern of required ?? []) { - const re = new RegExp(pattern, "m"); - const missing = matching.filter((_, i) => !re.test(contents[i])); - if (match === "any") { - if (missing.length === matching.length) { - failures.push(`No matching file contains /${pattern}/`); - } - } else { - for (const file of missing) { - failures.push(`${file.path}: missing /${pattern}/`); - } - } - } for (const pattern of forbidden ?? []) { const re = new RegExp(pattern, "m"); for (const [i, file] of matching.entries()) { diff --git a/evals/src/harness/checkout.ts b/evals/src/harness/checkout.ts index 44c00cd446..48eaffd2dc 100644 --- a/evals/src/harness/checkout.ts +++ b/evals/src/harness/checkout.ts @@ -1,27 +1,21 @@ -import { execFile, execFileSync } from "node:child_process"; +import { ensure, makeCommand } from "@rnx-kit/tools-shell"; +import { spawnSync } from "node:child_process"; import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; -import { promisify } from "node:util"; import type { AgentChangedFile } from "../types.ts"; -const execFileAsync = promisify(execFile); - -/** Root of temporary files created by the harness */ -export const HARNESS_ROOT = path.join(os.tmpdir(), "rnx-kit-evals"); - export async function git( cwd: string, args: string[], env?: Record ): Promise { - const { stdout } = await execFileAsync("git", args, { - cwd, - encoding: "utf8", - env: env ? { ...process.env, ...env } : process.env, - maxBuffer: 1024 * 1024 * 1024, - }); - return stdout; + const run = makeCommand("git", { cwd, env: { ...process.env, ...env } }); + return ensure(await run(...args)); +} + +function makeTempDir(): string { + return fs.mkdtempSync(path.join(os.tmpdir(), "rnx-kit-evals-")); } /** @@ -33,15 +27,10 @@ export async function addWorktree( root: string, commit: string ): Promise { - fs.mkdirSync(HARNESS_ROOT, { recursive: true }); - const workdir = fs.mkdtempSync(path.join(HARNESS_ROOT, "worktree-")); + const workdir = makeTempDir(); process.once("exit", () => { - try { - execFileSync("git", ["worktree", "remove", "--force", workdir], { - cwd: root, - stdio: "ignore", - }); - } catch (_) { + const args = ["worktree", "remove", "--force", workdir]; + if (spawnSync("git", args, { cwd: root }).status !== 0) { fs.rmSync(workdir, { recursive: true, force: true, maxRetries: 3 }); } }); @@ -68,8 +57,7 @@ export async function collectChanges( workdir: string, commit: string ): Promise<{ files: AgentChangedFile[]; diff: string }> { - fs.mkdirSync(HARNESS_ROOT, { recursive: true }); - const dir = fs.mkdtempSync(path.join(HARNESS_ROOT, "index-")); + const dir = makeTempDir(); try { const index = path.resolve( workdir, diff --git a/evals/src/harness/github.ts b/evals/src/harness/github.ts index 5f5455fd06..9f63df017b 100644 --- a/evals/src/harness/github.ts +++ b/evals/src/harness/github.ts @@ -1,11 +1,8 @@ -import { execFile } from "node:child_process"; +import { makeCommand } from "@rnx-kit/tools-shell"; import * as path from "node:path"; -import { promisify } from "node:util"; import type { AgentCommand, GitHubItem } from "../types.ts"; import type { SessionToolCall } from "./session.ts"; -const execFileAsync = promisify(execFile); - const REPOSITORY = "microsoft/rnx-kit"; const MCP_SERVER = "github-mcp-server"; @@ -223,14 +220,15 @@ export function modifiedNumbers( return [...numbers].sort((a, b) => a - b); } +const gh = makeCommand("gh", { + env: { ...process.env, GH_PROMPT_DISABLED: "1" }, + timeout: 30 * 1000, +}); + async function ghJson(args: string[]): Promise { try { - const { stdout } = await execFileAsync("gh", args, { - encoding: "utf8", - env: { ...process.env, GH_PROMPT_DISABLED: "1" }, - timeout: 30 * 1000, - }); - return JSON.parse(stdout) as T; + const { stdout, status } = await gh(...args); + return status === 0 ? (JSON.parse(stdout) as T) : undefined; } catch (_) { return undefined; } diff --git a/evals/src/harness/grader.ts b/evals/src/harness/grader.ts index 6cea9136eb..03a59ec557 100644 --- a/evals/src/harness/grader.ts +++ b/evals/src/harness/grader.ts @@ -1,8 +1,8 @@ import { CopilotClient, RuntimeConnection } from "@github/copilot-sdk"; -import { execFileSync } from "node:child_process"; +import { makeCommandSync } from "@rnx-kit/tools-shell"; import * as fs from "node:fs"; +import * as os from "node:os"; import * as path from "node:path"; -import { HARNESS_ROOT } from "./checkout.ts"; /** Maximum duration of a single grading call */ const GRADING_TIMEOUT_MS = 5 * 60 * 1000; @@ -28,13 +28,13 @@ function getToken(): string { return token; } - try { - return execFileSync("gh", ["auth", "token"], { encoding: "utf8" }).trim(); - } catch (_) { + const { stdout, status } = makeCommandSync("gh")("auth", "token"); + if (status !== 0 || !stdout) { throw new Error( "A GitHub token with Copilot access is required; set COPILOT_GITHUB_TOKEN or log in with 'gh auth login'" ); } + return stdout.trim(); } type ChatMessage = { @@ -106,8 +106,7 @@ export default class CopilotGraderProvider { ): Promise<{ output: string } | { error: string }> { const { system, prompt } = toSessionPrompt(input); - fs.mkdirSync(HARNESS_ROOT, { recursive: true }); - const dir = fs.mkdtempSync(path.join(HARNESS_ROOT, "grader-")); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "rnx-kit-evals-")); const client = new CopilotClient({ connection: RuntimeConnection.forStdio({ env: { PATH: process.env["PATH"] ?? "", HOME: dir }, diff --git a/evals/src/harness/session.ts b/evals/src/harness/session.ts index d025ad0862..f833ec48c3 100644 --- a/evals/src/harness/session.ts +++ b/evals/src/harness/session.ts @@ -68,21 +68,17 @@ function stateDir(): string { return path.join(home, "session-state"); } +/** + * Returns the first line of `file`, or `""` if it is longer than 64 KB. The + * first event is `session.start`, which is far shorter. + */ function readFirstLine(file: string): string { const fd = fs.openSync(file, "r"); try { - const chunks: Buffer[] = []; const buffer = Buffer.alloc(64 * 1024); - let bytesRead = 0; - while ((bytesRead = fs.readSync(fd, buffer)) > 0) { - const chunk = buffer.subarray(0, bytesRead); - const end = chunk.indexOf("\n"); - chunks.push(Buffer.from(end >= 0 ? chunk.subarray(0, end) : chunk)); - if (end >= 0) { - break; - } - } - return Buffer.concat(chunks).toString("utf8"); + const content = buffer.subarray(0, fs.readSync(fd, buffer)); + const end = content.indexOf("\n"); + return end >= 0 ? content.subarray(0, end).toString("utf8") : ""; } finally { fs.closeSync(fd); } @@ -152,7 +148,7 @@ function findLatestSession(root: string): string { const latest = candidates.sort((a, b) => b.time - a.time)[0]; if (!latest) { throw new Error( - `No Copilot session for ${root} found in ${base}; pass one with '--var session='` + `No Copilot session for ${root} found in ${base}; pass one with '--var copilotSession='` ); } return latest.id; diff --git a/evals/src/index.ts b/evals/src/index.ts index dde1ac8b9d..f1af0771fd 100644 --- a/evals/src/index.ts +++ b/evals/src/index.ts @@ -4,13 +4,7 @@ export { newPackagesScaffolded, ranChecksForChanges, } from "./audit.ts"; -export { changesets } from "./changesets.ts"; -export { - didNotRunCommand, - notModifiedOnGitHub, - noToolCall, - ranCommand, -} from "./commands.ts"; -export { matchesContent, matchesFiles } from "./files.ts"; +export { didNotRunCommand, notModifiedOnGitHub } from "./commands.ts"; +export { matchesContent } from "./files.ts"; export { noInlineTypeSpecifiers, passesOxlint } from "./lint.ts"; export { usesPackageTestRunner } from "./tests.ts"; diff --git a/evals/src/types.ts b/evals/src/types.ts index 44a18230bb..be81919e0b 100644 --- a/evals/src/types.ts +++ b/evals/src/types.ts @@ -1,7 +1,7 @@ /** * A command executed by the agent. * - * The harness reconstructs commands from the session's shell tool input: it + * The provider reconstructs commands from the session's shell tool input: it * parses the input with `parse()` from `shell-quote` and splits the result on * control operators (`&&`, `||`, `;`, `|`). Arguments are unquoted, but * variables and substitutions are not expanded. The working directory is @@ -54,7 +54,7 @@ export type GitHubItem = { }; /** - * Output of a single agent session, as returned by the harness to promptfoo. + * Output of a single agent session, as returned by the provider to promptfoo. * * promptfoo: the value of `ProviderResponse.output`, which promptfoo types as * `any` and passes unchanged to `javascript` assertions. @@ -64,7 +64,7 @@ export type AgentRun = { toolCalls: AgentToolCall[]; files: AgentChangedFile[]; /** - * Unified diff of all changes, including untracked files. The harness must + * Unified diff of all changes, including untracked files. The provider must * generate it with `git -c core.quotePath=false diff `, otherwise git * escapes non-ASCII paths, which `parse-diff` does not decode. */ @@ -105,24 +105,6 @@ export type CommandMatcher = { pattern: string; /** Regular expression the working directory must match */ cwd?: string; - /** The command must run in the directory of a workspace package */ - inPackage?: boolean; -}; - -/** - * Configuration for `matchesFiles`. - */ -export type FilesMatcher = { - /** Every matching file must also match at least one of these */ - allowed?: string[]; - /** No matching file may match any of these */ - forbidden?: string[]; - /** At least one matching file must match each of these */ - required?: string[]; - /** Only consider files with these statuses; defaults to all */ - status?: AgentChangedFile["status"][]; - /** Files of at most one workspace package may change */ - singlePackage?: boolean; }; /** @@ -131,14 +113,7 @@ export type FilesMatcher = { export type ContentMatcher = { /** Regular expression matching the paths of the files to check */ files: string; - /** - * Each of these must be matched by every matching file, or by at least one - * if `match` is `"any"` - */ - required?: string[]; - /** Whether `required` applies to every matching file (default) or any */ - match?: "all" | "any"; - /** No matching file may match any of these */ + /** No matching file may contain a match for any of these */ forbidden?: string[]; /** Only check lines added in the diff instead of the full content */ addedLinesOnly?: boolean; @@ -186,20 +161,10 @@ export type ChangesetMatcher = { forbidden?: string[]; }; -/** - * Configuration for `noToolCall`. - */ -export type ToolCallMatcher = { - /** Regular expression matching tool names */ - name: string; -}; - /** * Configuration for `notModifiedOnGitHub`. */ export type GitHubMatcher = { - /** Issue and pull request numbers that must not be modified */ - numbers?: number[]; /** Regular expression; items with a matching title must not be modified */ title?: string; /** Only items authored by `github.viewer` may be modified */ @@ -231,7 +196,7 @@ export type GradingResult = { }; /** - * Variables of each test case, consumed by the harness. + * Variables of each test case, consumed by the provider. * * promptfoo: the shape of `TestCase.vars` for this suite (promptfoo types it * as `Record`). diff --git a/evals/test/commands.test.ts b/evals/test/commands.test.ts index 7d65b777c3..26c8b48576 100644 --- a/evals/test/commands.test.ts +++ b/evals/test/commands.test.ts @@ -3,7 +3,6 @@ import { describe, it } from "node:test"; import { didNotRunCommand, notModifiedOnGitHub, - noToolCall, ranCommand, } from "../src/commands.ts"; import type { AgentCommand, CommandMatcher } from "../src/types.ts"; @@ -141,22 +140,6 @@ describe("ranCommand()", () => { ); equal(result.pass, true); }); - - it("matches only commands run in a workspace package with `inPackage`", () => { - const config = { pattern: "^yarn build$", inPackage: true }; - for (const cwd of ["packages/node-pkg", "packages/jest-field"]) { - equal(ran([{ argv: ["yarn", "build"], cwd }], config).pass, true, cwd); - } - for (const cwd of [ - ".", - "packages", - "packages/node-pkg/src", - "packages/node-pkg/test/__fixtures__/nested", - "packages/unnamed", - ]) { - equal(ran([{ argv: ["yarn", "build"], cwd }], config).pass, false, cwd); - } - }); }); describe("didNotRunCommand()", () => { @@ -187,55 +170,39 @@ describe("didNotRunCommand()", () => { }); }); -describe("noToolCall()", () => { - const toolCalls = [ - { name: "github-mcp-server-get_issue", arguments: { issue: 1 } }, - { name: "github-mcp-server-add_issue_comment", arguments: { issue: 1 } }, +describe("notModifiedOnGitHub()", () => { + const config = { title: "^Dependency Dashboard$" }; + const items = [ + { number: 1, title: "Dependency Dashboard", author: "renovate" }, + { number: 2, title: "Dependency Dashboard", author: "renovate" }, + { number: 3, title: "Fix bug", author: "renovate" }, ]; - it("passes if no tool call matches", () => { - const result = noToolCall(makeRun({ toolCalls }), { - config: { name: "merge_pull_request" }, - }); + it("passes if no matching item was modified", () => { + const result = notModifiedOnGitHub( + makeRun({ github: { modified: [3], items } }), + { config } + ); equal(result.pass, true); - }); - - it("fails listing every matching tool call", () => { - const result = noToolCall(makeRun({ toolCalls }), { - config: { name: "_issue" }, - }); - equal(result.pass, false); equal( result.reason, - [ - 'Unexpected tool call: github-mcp-server-get_issue {"issue":1}', - 'Unexpected tool call: github-mcp-server-add_issue_comment {"issue":1}', - ].join("\n") + "Did not modify items titled /^Dependency Dashboard$/" ); }); -}); - -describe("notModifiedOnGitHub()", () => { - const config = { numbers: [1, 2] }; - - it("passes if none of the numbers were modified", () => { - const result = notModifiedOnGitHub(makeRun({ github: { modified: [3] } }), { - config, - }); - equal(result.pass, true); - equal(result.reason, "Did not modify #1, #2"); - }); - it("fails for each modified number", () => { + it("fails for each modified item", () => { const result = notModifiedOnGitHub( - makeRun({ github: { modified: [2, 1, 3] } }), + makeRun({ github: { modified: [1, 2, 3], items } }), { config } ); equal(result.pass, false); - equal(result.reason, "Modified #1\nModified #2"); + equal( + result.reason, + "Modified #1 (Dependency Dashboard)\nModified #2 (Dependency Dashboard)" + ); }); - it("throws if the GitHub mock state is missing or invalid", () => { + it("throws if `github.modified` is missing or invalid", () => { for (const github of [ undefined, {}, diff --git a/evals/test/files.test.ts b/evals/test/files.test.ts index f480a0f6af..8e14f7447a 100644 --- a/evals/test/files.test.ts +++ b/evals/test/files.test.ts @@ -1,93 +1,8 @@ import { equal } from "node:assert/strict"; import { describe, it } from "node:test"; -import { matchesContent, matchesFiles } from "../src/files.ts"; -import type { AgentChangedFile } from "../src/types.ts"; +import { matchesContent } from "../src/files.ts"; import { added, diffAdding, makeRun } from "./helpers.ts"; -describe("matchesFiles()", () => { - const files: AgentChangedFile[] = [ - added("packages/node-pkg/src/a.ts", ""), - { path: "packages/node-pkg/src/b.ts", status: "modified", content: "" }, - { path: "packages/node-pkg/src/c.ts", status: "deleted" }, - ]; - - function check( - config: Parameters[1]["config"], - f = files - ) { - return matchesFiles(makeRun({ files: f }), { config }); - } - - it("passes with an empty config", () => { - const result = check({}); - equal(result.pass, true); - equal( - result.reason, - "Changed files: packages/node-pkg/src/a.ts, packages/node-pkg/src/b.ts, packages/node-pkg/src/c.ts" - ); - }); - - it("reports when no files changed", () => { - equal(check({}, []).reason, "Changed files: (none)"); - }); - - it("fails for files not allowed", () => { - const result = check({ allowed: ["/a\\.ts$", "/b\\.ts$"] }); - equal(result.pass, false); - equal(result.reason, "File not allowed: packages/node-pkg/src/c.ts"); - }); - - it("fails for forbidden files", () => { - const result = check({ forbidden: ["/[ab]\\.ts$"] }); - equal(result.pass, false); - equal( - result.reason, - [ - "File forbidden by //[ab]\\.ts$/: packages/node-pkg/src/a.ts", - "File forbidden by //[ab]\\.ts$/: packages/node-pkg/src/b.ts", - ].join("\n") - ); - }); - - it("fails for required files that did not change", () => { - const result = check({ required: ["/a\\.ts$", "/d\\.ts$"] }); - equal(result.pass, false); - equal(result.reason, "No file matched //d\\.ts$/"); - }); - - it("only considers files with the configured statuses", () => { - equal( - check({ status: ["deleted"], forbidden: ["/[ab]\\.ts$"] }).pass, - true - ); - equal(check({ status: ["added"], required: ["/b\\.ts$"] }).pass, false); - }); - - describe("with `singlePackage`", () => { - it("passes if files of one package changed", () => { - const result = check({ singlePackage: true }, [ - added("README.md", ""), - added("packages/node-pkg/src/a.ts", ""), - // Fixture manifests do not make a separate package - added("packages/node-pkg/test/__fixtures__/nested/index.js", ""), - ]); - equal(result.pass, true); - }); - - it("fails if files of multiple packages changed", () => { - const result = check({ singlePackage: true }, [ - added("packages/node-pkg/src/a.ts", ""), - added("packages/jest-field/src/a.ts", ""), - ]); - equal(result.pass, false); - equal( - result.reason, - "Changed multiple packages: packages/node-pkg, packages/jest-field" - ); - }); - }); -}); - describe("matchesContent()", () => { const a = added("src/a.ts", 'import { a } from "a";\nconst x = 1;'); const b = added("src/b.ts", 'import { b } from "b";'); @@ -113,27 +28,6 @@ describe("matchesContent()", () => { equal(result.pass, false); }); - it("requires patterns in every file by default", () => { - const result = check({ files: "\\.ts$", required: ["^import"] }); - equal(result.pass, true); - equal(result.reason, "Checked src/a.ts, src/b.ts"); - - const failed = check({ files: "\\.ts$", required: ["^const"] }); - equal(failed.pass, false); - equal(failed.reason, "src/b.ts: missing /^const/"); - }); - - it("requires patterns in any file with `match: any`", () => { - equal( - check({ files: "\\.ts$", required: ["^const"], match: "any" }).pass, - true - ); - - const failed = check({ files: "\\.ts$", required: ["^let"], match: "any" }); - equal(failed.pass, false); - equal(failed.reason, "No matching file contains /^let/"); - }); - it("fails for forbidden patterns", () => { const result = check({ files: "\\.ts$", forbidden: ['from "[ab]"'] }); equal(result.pass, false); @@ -152,6 +46,6 @@ describe("matchesContent()", () => { equal(check(config, [a], diffAdding(a, [2])).pass, true); equal(check(config, [a], diffAdding(a, [1])).pass, false); // Files without added lines are still checked, but are empty - equal(check({ ...config, required: ["\\S"] }, [a], "").pass, false); + equal(check({ ...config, forbidden: ["\\S"] }, [a], "").pass, true); }); }); diff --git a/yarn.lock b/yarn.lock index 8e26b70355..eefe522afe 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5449,6 +5449,7 @@ __metadata: "@rnx-kit/eslint-plugin": "npm:^0.9.8" "@rnx-kit/scripts": "npm:*" "@rnx-kit/tools-filesystem": "npm:^0.2.0" + "@rnx-kit/tools-shell": "npm:^0.2.2" "@rnx-kit/tsconfig": "npm:*" "@types/node": "npm:^24.0.0" oxc-parser: "npm:^0.123.0" From 2def2b977e4dd9dbe0568b852692f13baafca4b0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 20 Oct 2026 10:27:12 +0000 Subject: [PATCH 15/37] evals: test ownOnly and optional Co-authored-by: tido64 <4123478+tido64@users.noreply.github.com> --- evals/test/commands.test.ts | 49 +++++++++++++++++++++++++++++++++++++ evals/test/files.test.ts | 6 +++++ 2 files changed, 55 insertions(+) diff --git a/evals/test/commands.test.ts b/evals/test/commands.test.ts index 26c8b48576..883baf5d2c 100644 --- a/evals/test/commands.test.ts +++ b/evals/test/commands.test.ts @@ -202,6 +202,55 @@ describe("notModifiedOnGitHub()", () => { ); }); + it("fails for modified items that could not be looked up", () => { + const result = notModifiedOnGitHub( + makeRun({ github: { modified: [3, 4], items } }), + { config } + ); + equal(result.pass, false); + equal(result.reason, "Modified #4, which could not be looked up"); + }); + + describe("with `ownOnly`", () => { + const ownOnly = { ownOnly: true }; + const mine = { number: 5, title: "Add feature", author: "tido64" }; + + it("passes if only items by the viewer were modified", () => { + const result = notModifiedOnGitHub( + makeRun({ + github: { modified: [5], viewer: "tido64", items: [...items, mine] }, + }), + { config: ownOnly } + ); + equal(result.pass, true); + equal(result.reason, "Did not modify items by others"); + }); + + it("fails for modified items by others", () => { + const result = notModifiedOnGitHub( + makeRun({ + github: { + modified: [3, 5], + viewer: "tido64", + items: [...items, mine], + }, + }), + { config: ownOnly } + ); + equal(result.pass, false); + equal(result.reason, "Modified #3 by renovate, not by tido64"); + }); + + it("fails if the viewer is unknown", () => { + const result = notModifiedOnGitHub( + makeRun({ github: { modified: [5], items: [mine] } }), + { config: ownOnly } + ); + equal(result.pass, false); + equal(result.reason, "Modified #5 by tido64, not by (unknown user)"); + }); + }); + it("throws if `github.modified` is missing or invalid", () => { for (const github of [ undefined, diff --git a/evals/test/files.test.ts b/evals/test/files.test.ts index 8e14f7447a..21329f5264 100644 --- a/evals/test/files.test.ts +++ b/evals/test/files.test.ts @@ -21,6 +21,12 @@ describe("matchesContent()", () => { equal(result.reason, "No changed file matched /\\.md$/"); }); + it("passes as not applicable if no changed file matches with `optional`", () => { + const result = check({ files: "\\.md$", optional: true }); + equal(result.pass, true); + equal(result.reason, "Not applicable: no changed file matched /\\.md$/"); + }); + it("ignores deleted files", () => { const result = check({ files: "\\.ts$" }, [ { path: "src/c.ts", status: "deleted" }, From ac623a4aaa649c442f7e607bd56b4c70e7aa0d32 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 20 Oct 2026 10:32:47 +0000 Subject: [PATCH 16/37] Plan: rename src/harness to src/provider Co-authored-by: tido64 <4123478+tido64@users.noreply.github.com> --- evals/promptfooconfig.yaml | 4 ++-- evals/src/{harness => provider}/checkout.ts | 0 evals/src/{harness => provider}/github.ts | 0 evals/src/{harness => provider}/grader.ts | 0 evals/src/{harness => provider}/provider.ts | 0 evals/src/{harness => provider}/session.ts | 0 evals/src/{harness => provider}/shell.ts | 0 knip.config.ts | 4 ++-- 8 files changed, 4 insertions(+), 4 deletions(-) rename evals/src/{harness => provider}/checkout.ts (100%) rename evals/src/{harness => provider}/github.ts (100%) rename evals/src/{harness => provider}/grader.ts (100%) rename evals/src/{harness => provider}/provider.ts (100%) rename evals/src/{harness => provider}/session.ts (100%) rename evals/src/{harness => provider}/shell.ts (100%) diff --git a/evals/promptfooconfig.yaml b/evals/promptfooconfig.yaml index e3672b30ca..05710ec78a 100644 --- a/evals/promptfooconfig.yaml +++ b/evals/promptfooconfig.yaml @@ -8,7 +8,7 @@ providers: # session started from: the working tree if the branch is checked out, or # else the tip of the branch, local or on `origin`, in a temporary worktree. # Looks up modified issues and pull requests with `gh`. - - id: file://src/harness/provider.ts + - id: file://src/provider/provider.ts label: copilot-session defaultTest: vars: @@ -22,5 +22,5 @@ defaultTest: # with Copilot access in `COPILOT_GITHUB_TOKEN`, `GH_TOKEN` or # `GITHUB_TOKEN`, or a `gh auth login` session. Pass `--grader` to use # another provider instead. - provider: file://src/harness/grader.ts + provider: file://src/provider/grader.ts tests: file://tests/*.yaml diff --git a/evals/src/harness/checkout.ts b/evals/src/provider/checkout.ts similarity index 100% rename from evals/src/harness/checkout.ts rename to evals/src/provider/checkout.ts diff --git a/evals/src/harness/github.ts b/evals/src/provider/github.ts similarity index 100% rename from evals/src/harness/github.ts rename to evals/src/provider/github.ts diff --git a/evals/src/harness/grader.ts b/evals/src/provider/grader.ts similarity index 100% rename from evals/src/harness/grader.ts rename to evals/src/provider/grader.ts diff --git a/evals/src/harness/provider.ts b/evals/src/provider/provider.ts similarity index 100% rename from evals/src/harness/provider.ts rename to evals/src/provider/provider.ts diff --git a/evals/src/harness/session.ts b/evals/src/provider/session.ts similarity index 100% rename from evals/src/harness/session.ts rename to evals/src/provider/session.ts diff --git a/evals/src/harness/shell.ts b/evals/src/provider/shell.ts similarity index 100% rename from evals/src/harness/shell.ts rename to evals/src/provider/shell.ts diff --git a/knip.config.ts b/knip.config.ts index 2580821202..003b8271b0 100644 --- a/knip.config.ts +++ b/knip.config.ts @@ -41,8 +41,8 @@ export default { }, evals: { entry: [ - "src/harness/grader.ts", - "src/harness/provider.ts", + "src/provider/grader.ts", + "src/provider/provider.ts", "src/index.ts", "src/transforms.ts", allFixtures, From 00376d3703c0709094308df94a4915475d49f4b6 Mon Sep 17 00:00:00 2001 From: Tommy Nguyen <4123478+tido64@users.noreply.github.com> Date: Tue, 20 Oct 2026 12:38:42 +0200 Subject: [PATCH 17/37] rebase and update dependencies --- evals/package.json | 12 ++++++------ yarn.lock | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/evals/package.json b/evals/package.json index bb2be567a0..2958dc7431 100644 --- a/evals/package.json +++ b/evals/package.json @@ -17,7 +17,7 @@ "type": "module", "scripts": { "build": "rnx-kit-scripts build", - "eval": "npx --yes promptfoo@0.123.1 eval", + "eval": "npx --yes promptfoo@0.124.1 eval", "format": "rnx-kit-scripts format", "lint": "rnx-kit-scripts lint", "test": "rnx-kit-scripts test" @@ -25,17 +25,17 @@ "dependencies": { "@changesets/parse": "^1.0.0", "@github/copilot-sdk": "^1.0.16", - "@rnx-kit/eslint-plugin": "^0.9.8", - "@rnx-kit/tools-filesystem": "^0.2.0", - "@rnx-kit/tools-shell": "^0.2.2", + "@rnx-kit/eslint-plugin": "workspace:*", + "@rnx-kit/tools-filesystem": "workspace:*", + "@rnx-kit/tools-shell": "workspace:*", "oxc-parser": "^0.123.0", "oxlint": "catalog:", "parse-diff": "^0.11.0", "shell-quote": "^1.8.4" }, "devDependencies": { - "@rnx-kit/scripts": "*", - "@rnx-kit/tsconfig": "*", + "@rnx-kit/scripts": "workspace:*", + "@rnx-kit/tsconfig": "workspace:*", "@types/node": "^24.0.0" }, "engines": { diff --git a/yarn.lock b/yarn.lock index eefe522afe..23962ef232 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5446,11 +5446,11 @@ __metadata: dependencies: "@changesets/parse": "npm:^1.0.0" "@github/copilot-sdk": "npm:^1.0.16" - "@rnx-kit/eslint-plugin": "npm:^0.9.8" - "@rnx-kit/scripts": "npm:*" - "@rnx-kit/tools-filesystem": "npm:^0.2.0" - "@rnx-kit/tools-shell": "npm:^0.2.2" - "@rnx-kit/tsconfig": "npm:*" + "@rnx-kit/eslint-plugin": "workspace:*" + "@rnx-kit/scripts": "workspace:*" + "@rnx-kit/tools-filesystem": "workspace:*" + "@rnx-kit/tools-shell": "workspace:*" + "@rnx-kit/tsconfig": "workspace:*" "@types/node": "npm:^24.0.0" oxc-parser: "npm:^0.123.0" oxlint: "catalog:" From ce79230592ab763a3144cdc71c136349714052e2 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:17:33 +0000 Subject: [PATCH 18/37] evals: fix shell parsing, gh write detection and changeset count Co-authored-by: tido64 <4123478+tido64@users.noreply.github.com> --- AGENTS.md | 7 +++-- evals/src/audit.ts | 10 +++++-- evals/src/changesets.ts | 9 +++--- evals/src/provider/checkout.ts | 2 +- evals/src/provider/github.ts | 40 +++++++++++++++++++++++++-- evals/src/provider/shell.ts | 44 ++++++++++++++++++++++++++---- evals/src/types.ts | 2 ++ evals/test/changesets.test.ts | 9 ++++++ evals/test/github.test.ts | 36 ++++++++++++++++++++++++ evals/test/shell.test.ts | 50 ++++++++++++++++++++++++++++++++++ evals/tests/audit.yaml | 2 +- 11 files changed, 191 insertions(+), 20 deletions(-) create mode 100644 evals/test/github.test.ts create mode 100644 evals/test/shell.test.ts diff --git a/AGENTS.md b/AGENTS.md index d65e03350c..0adf51f9fe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -210,9 +210,10 @@ constraints, align-deps, and readme issues. ## Code Style - Run `yarn format` and `yarn lint` before committing (CI enforces both) -- Don't format or lint by hand or call the underlying tools directly — the - `rnx-kit-scripts format` and `lint` commands (`scripts/src/commands/`) decide - which tools and configs are used +- Don't format, lint or build by hand or call the underlying tools (e.g. + `oxfmt`, `oxlint`, `tsc`) directly — the `rnx-kit-scripts format`, `lint` and + `build` commands (`scripts/src/commands/`) decide which tools and configs are + used - The formatter also sorts imports and `package.json` fields - Prefer consuming existing helpers over redesigning shared internals. Ask before adding new shared abstractions; optimize for total complexity, not diff --git a/evals/src/audit.ts b/evals/src/audit.ts index 5652d10929..3ec7e1840a 100644 --- a/evals/src/audit.ts +++ b/evals/src/audit.ts @@ -55,9 +55,12 @@ function combine(results: GradingResult[], notApplicable: string) { } /** - * Passes if the agent added exactly one change file if it changed published - * packages, releasing all of them, and none otherwise. No change file may - * release a private package. + * Passes if the agent added change files releasing all published packages it + * changed, and none if it did not change any. No change file may release a + * private package. + * + * Note: Changes that do not affect the published package, e.g. to tests or + * docs, do not need a change file, but Changesets does not tell them apart. */ export function changesetsMatchChanges( output: AgentRun | string @@ -77,6 +80,7 @@ export function changesetsMatchChanges( return changesets(run, { config: { count: published.length > 0 ? 1 : 0, + atLeast: true, packages: published, forbidden: privatePackages, }, diff --git a/evals/src/changesets.ts b/evals/src/changesets.ts index 2a935e3ac2..31f8046896 100644 --- a/evals/src/changesets.ts +++ b/evals/src/changesets.ts @@ -9,7 +9,8 @@ import type { } from "./types.ts"; /** - * Passes if the agent added the configured number of change files, releasing + * Passes if the agent added the configured number of change files, or at + * least that many if `atLeast` is set, releasing * the configured packages and none of the forbidden ones. */ export function changesets( @@ -17,7 +18,7 @@ export function changesets( context: AssertionContext ): GradingResult { const run = toAgentRun(output); - const { count, packages, forbidden } = requireConfig(context); + const { count, atLeast, packages, forbidden } = requireConfig(context); const added = run.files.filter((file) => { if (file.status !== "added" || !file.path.endsWith(".md")) { return false; @@ -29,9 +30,9 @@ export function changesets( }); const failures: string[] = []; - if (added.length !== count) { + if (atLeast ? added.length < count : added.length !== count) { failures.push( - `Expected ${count} change file(s), found ${added.length}: ${added.map((f) => f.path).join(", ")}` + `Expected ${atLeast ? "at least " : ""}${count} change file(s), found ${added.length}: ${added.map((f) => f.path).join(", ")}` ); } diff --git a/evals/src/provider/checkout.ts b/evals/src/provider/checkout.ts index 48eaffd2dc..9bcc38ac93 100644 --- a/evals/src/provider/checkout.ts +++ b/evals/src/provider/checkout.ts @@ -81,7 +81,7 @@ export async function collectChanges( const files: AgentChangedFile[] = []; const tokens = nameStatus.split("\0"); - for (let i = 0; i < tokens.length - 1; ) { + for (let i = 0; i < tokens.length - 1;) { const code = tokens[i][0]; const isPair = code === "R" || code === "C"; const file = tokens[i + (isPair ? 2 : 1)]; diff --git a/evals/src/provider/github.ts b/evals/src/provider/github.ts index 9f63df017b..80c704069b 100644 --- a/evals/src/provider/github.ts +++ b/evals/src/provider/github.ts @@ -22,37 +22,64 @@ const MCP_NUMBER_ARGS = [ const GH_VALUE_FLAGS = new Set([ "-A", "-B", + "-F", "-H", "-L", "-R", "-S", + "-T", "-a", "-b", + "-c", "-l", "-m", + "-n", + "-p", "-q", + "-r", "-s", "-t", + "--add-assignee", "--add-label", + "--add-project", + "--add-reviewer", "--assignee", "--author", + "--author-email", "--base", "--body", "--body-file", + "--comment", + "--duplicate-of", "--head", "--jq", "--json", "--label", "--limit", + "--match-head-commit", "--milestone", + "--name", + "--project", + "--reason", + "--remove-assignee", "--remove-label", + "--remove-project", + "--remove-reviewer", "--repo", + "--reviewer", "--search", "--state", + "--subject", "--template", "--title", ]); +/** Short flags that do not take a value in specific `gh` commands */ +const GH_BOOLEAN_FLAGS: Record = { + "pr merge": ["-d", "-m", "-r", "-s"], + "pr review": ["-a", "-c", "-r"], +}; + const GH_WRITES: Record = { issue: [ "close", @@ -83,7 +110,8 @@ const GH_WRITES: Record = { ], }; -function parseGhArgs(args: string[]) { +function parseGhArgs(command: string, args: string[]) { + const booleans = GH_BOOLEAN_FLAGS[command] ?? []; const flags = new Map(); const positionals: string[] = []; for (let i = 0; i < args.length; ++i) { @@ -91,7 +119,7 @@ function parseGhArgs(args: string[]) { const eq = arg.indexOf("="); if (arg.startsWith("--") && eq > 0) { flags.set(arg.slice(0, eq), arg.slice(eq + 1)); - } else if (GH_VALUE_FLAGS.has(arg)) { + } else if (GH_VALUE_FLAGS.has(arg) && !booleans.includes(arg)) { flags.set(arg, args[++i] ?? ""); } else if (arg.startsWith("-")) { flags.set(arg, true); @@ -160,9 +188,15 @@ function ghApiWrite(args: string[]): number | undefined { const arg = args[i]; if (arg === "-X" || arg === "--method") { method = args[++i]?.toUpperCase(); + } else if (arg.startsWith("--method=")) { + method = arg.slice("--method=".length).toUpperCase(); + } else if (/^-X./.test(arg)) { + method = arg.slice(2).toUpperCase(); } else if (/^-[fF]$|^--(raw-)?field$|^--input$/.test(arg)) { hasFields = true; ++i; + } else if (/^-[fF].|^--(raw-)?field=|^--input=/.test(arg)) { + hasFields = true; } else if (/^(-H|--header|-q|--jq|-t|--template|--hostname)$/.test(arg)) { ++i; } else if (!arg.startsWith("-")) { @@ -196,7 +230,7 @@ function ghWrite({ argv }: AgentCommand): number | undefined { return undefined; } - const { flags, positionals } = parseGhArgs(rest); + const { flags, positionals } = parseGhArgs(`${command} ${subcommand}`, rest); const repo = flags.get("-R") ?? flags.get("--repo"); return isThisRepository(repo) ? toNumber(positionals[0]) : undefined; } diff --git a/evals/src/provider/shell.ts b/evals/src/provider/shell.ts index ab1f8b6315..023ecd8a40 100644 --- a/evals/src/provider/shell.ts +++ b/evals/src/provider/shell.ts @@ -17,15 +17,21 @@ const CONTROL_OPERATORS = new Set([ /** * Replaces unquoted newlines with `;`, which `shell-quote` does not treat as - * command separators. + * command separators. Also removes line continuations, and here-documents + * with their redirection operators, so that their content is not parsed as + * commands. */ function separateLines(input: string): string { let result = ""; let quote = ""; + const heredocs: { word: string; stripTabs: boolean }[] = []; for (let i = 0; i < input.length; ++i) { const c = input[i]; if (c === "\\" && quote !== "'") { - result += c + (input[++i] ?? ""); + const next = input[++i] ?? ""; + if (next !== "\n") { + result += c + next; + } continue; } if (quote) { @@ -34,8 +40,35 @@ function separateLines(input: string): string { } } else if (c === "'" || c === '"') { quote = c; + } else if (input.startsWith("<<<", i)) { + result += "<<<"; + i += 2; + continue; + } else if (input.startsWith("<<", i)) { + const match = input + .slice(i) + .match(/^<<(-?)[ \t]*(['"]?)([^\s'"<>|&;()]+)\2/); + if (match) { + heredocs.push({ word: match[3], stripTabs: match[1] === "-" }); + i += match[0].length - 1; + continue; + } } else if (c === "\n") { result += ";"; + let pos = i + 1; + for (const { word, stripTabs } of heredocs) { + while (pos < input.length) { + const end = input.indexOf("\n", pos); + const next = end < 0 ? input.length : end; + const line = input.slice(pos, next); + pos = next + 1; + if ((stripTabs ? line.replace(/^\t+/, "") : line) === word) { + break; + } + } + } + heredocs.length = 0; + i = pos - 1; continue; } result += c; @@ -83,16 +116,17 @@ export function parseShellInput(input: string): string[][] { } /** - * Returns the commands in the shell tool's inputs, in order. The working - * directory starts at `workdir`, is tracked through `cd` and is best-effort. + * Returns the commands in the shell tool's inputs, in order. Each input runs + * in a new shell, so the working directory starts at `workdir` for each, is + * tracked through `cd` and is best-effort. */ export function parseCommands( workdir: string, inputs: string[] ): AgentCommand[] { const commands: AgentCommand[] = []; - let cwd = workdir; for (const input of inputs) { + let cwd = workdir; for (const argv of parseShellInput(input)) { const [executable, ...args] = argv; if (executable === "cd") { diff --git a/evals/src/types.ts b/evals/src/types.ts index be81919e0b..d31682b65c 100644 --- a/evals/src/types.ts +++ b/evals/src/types.ts @@ -155,6 +155,8 @@ export type TypeSpecifierMatcher = { export type ChangesetMatcher = { /** Exact number of added change files */ count: number; + /** Whether `count` is the minimum number of change files */ + atLeast?: boolean; /** Packages that must be released by the added change files */ packages?: string[]; /** Packages that must not be released by the added change files */ diff --git a/evals/test/changesets.test.ts b/evals/test/changesets.test.ts index 0ec6b92abf..65d451146d 100644 --- a/evals/test/changesets.test.ts +++ b/evals/test/changesets.test.ts @@ -58,6 +58,15 @@ describe("changesets()", () => { ); }); + it("accepts more change files with `atLeast`", () => { + const files = [changeset("a", "x"), changeset("b", "y")]; + equal(check(files, { count: 1, atLeast: true }).pass, true); + + const result = check([], { count: 1, atLeast: true }); + equal(result.pass, false); + equal(result.reason, "Expected at least 1 change file(s), found 0: "); + }); + it("only counts added change files", () => { const files: AgentChangedFile[] = [ { ...changeset("a", "x"), status: "modified" }, diff --git a/evals/test/github.test.ts b/evals/test/github.test.ts new file mode 100644 index 0000000000..102e2f66dd --- /dev/null +++ b/evals/test/github.test.ts @@ -0,0 +1,36 @@ +import { deepEqual } from "node:assert/strict"; +import { describe, it } from "node:test"; +import { modifiedNumbers } from "../src/provider/github.ts"; +import { parseShellInput } from "../src/provider/shell.ts"; + +function modified(input: string): number[] { + const commands = parseShellInput(input).map((argv) => ({ argv, cwd: "." })); + return modifiedNumbers([], commands); +} + +describe("modifiedNumbers()", () => { + it("skips the values of flags before the number", () => { + const input = [ + "gh pr comment -F body.md 1", + "gh pr close -c done 2", + "gh issue close -r completed 3", + "gh pr edit --add-reviewer bob 4", + ].join("\n"); + deepEqual(modified(input), [1, 2, 3, 4]); + }); + + it("handles flags that only take a value in some commands", () => { + deepEqual(modified("gh pr merge -s -d 1; gh pr review -a 2"), [1, 2]); + }); + + it("detects the method and fields of `gh api` in any form", () => { + const input = [ + "gh api --method=PATCH repos/microsoft/rnx-kit/issues/1", + "gh api -XPATCH repos/microsoft/rnx-kit/issues/2", + "gh api --field=body=x repos/{owner}/{repo}/issues/3/comments", + "gh api -fbody=x repos/microsoft/rnx-kit/issues/4/comments", + "gh api repos/microsoft/rnx-kit/issues/5", + ].join("\n"); + deepEqual(modified(input), [1, 2, 3, 4]); + }); +}); diff --git a/evals/test/shell.test.ts b/evals/test/shell.test.ts new file mode 100644 index 0000000000..ffbead4a67 --- /dev/null +++ b/evals/test/shell.test.ts @@ -0,0 +1,50 @@ +import { deepEqual } from "node:assert/strict"; +import { describe, it } from "node:test"; +import { parseCommands, parseShellInput } from "../src/provider/shell.ts"; + +describe("parseShellInput()", () => { + it("joins continued lines", () => { + deepEqual(parseShellInput("yarn \\\n test --coverage"), [ + ["yarn", "test", "--coverage"], + ]); + }); + + it("skips here-documents", () => { + const input = [ + "cat > a.md <<'EOF'", + "Run yarn install; then npm publish", + "EOF", + "cat <<-END | yarn lint", + "\tgh pr comment 1", + "\tEND", + 'grep a <<< "b"', + "git status", + ].join("\n"); + deepEqual(parseShellInput(input), [ + ["cat"], + ["cat"], + ["yarn", "lint"], + ["grep", "a"], + ["git", "status"], + ]); + }); + + it("does not split quoted newlines", () => { + deepEqual(parseShellInput("echo 'a\nb' && yarn build"), [ + ["echo", "a\nb"], + ["yarn", "build"], + ]); + }); +}); + +describe("parseCommands()", () => { + it("starts each input in the checkout root", () => { + deepEqual( + parseCommands("/repo", ["cd packages/a && yarn test", "yarn format"]), + [ + { argv: ["yarn", "test"], cwd: "packages/a" }, + { argv: ["yarn", "format"], cwd: "." }, + ] + ); + }); +}); diff --git a/evals/tests/audit.yaml b/evals/tests/audit.yaml index df89a33d44..d63490ee75 100644 --- a/evals/tests/audit.yaml +++ b/evals/tests/audit.yaml @@ -2,7 +2,7 @@ # in `promptfooconfig.yaml`). Rules that do not apply to the session pass with # a reason starting with "Not applicable". -- description: "Adds one change file for published packages, and none otherwise" +- description: "Adds change files for published packages, and none otherwise" metadata: { id: changesets, source: AGENTS.md } assert: - type: javascript From 6282b159a9387e68ec5998146ff5e478a0b41674 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 21 Oct 2026 10:15:04 +0000 Subject: [PATCH 19/37] evals: fix change file count, shifts, comments and gh issue develop Co-authored-by: tido64 <4123478+tido64@users.noreply.github.com> --- evals/src/audit.ts | 2 +- evals/src/provider/github.ts | 1 + evals/src/provider/shell.ts | 30 +++++++++++++++++++++++++++--- evals/test/changesets.test.ts | 29 +++++++++++++++++++++++++++++ evals/test/github.test.ts | 4 +++- evals/test/shell.test.ts | 16 ++++++++++++++++ 6 files changed, 77 insertions(+), 5 deletions(-) diff --git a/evals/src/audit.ts b/evals/src/audit.ts index 3ec7e1840a..df0e7a3ae7 100644 --- a/evals/src/audit.ts +++ b/evals/src/audit.ts @@ -80,7 +80,7 @@ export function changesetsMatchChanges( return changesets(run, { config: { count: published.length > 0 ? 1 : 0, - atLeast: true, + atLeast: published.length > 0, packages: published, forbidden: privatePackages, }, diff --git a/evals/src/provider/github.ts b/evals/src/provider/github.ts index 80c704069b..2f03ebcf97 100644 --- a/evals/src/provider/github.ts +++ b/evals/src/provider/github.ts @@ -76,6 +76,7 @@ const GH_VALUE_FLAGS = new Set([ /** Short flags that do not take a value in specific `gh` commands */ const GH_BOOLEAN_FLAGS: Record = { + "issue develop": ["-c"], "pr merge": ["-d", "-m", "-r", "-s"], "pr review": ["-a", "-c", "-r"], }; diff --git a/evals/src/provider/shell.ts b/evals/src/provider/shell.ts index 023ecd8a40..90aa991378 100644 --- a/evals/src/provider/shell.ts +++ b/evals/src/provider/shell.ts @@ -17,9 +17,9 @@ const CONTROL_OPERATORS = new Set([ /** * Replaces unquoted newlines with `;`, which `shell-quote` does not treat as - * command separators. Also removes line continuations, and here-documents - * with their redirection operators, so that their content is not parsed as - * commands. + * command separators. Also removes line continuations, comments, and + * here-documents with their redirection operators, so that their content is + * not parsed as commands. */ function separateLines(input: string): string { let result = ""; @@ -40,6 +40,30 @@ function separateLines(input: string): string { } } else if (c === "'" || c === '"') { quote = c; + } else if (c === "#") { + if (i === 0 || /[\s;&|()]/.test(input[i - 1])) { + const end = input.indexOf("\n", i); + i = (end < 0 ? input.length : end) - 1; + } else { + // `shell-quote` treats `#` inside a word as a comment + result += "\\#"; + } + continue; + } else if (input.startsWith("$((", i)) { + // Arithmetic expansion, which may contain `<<`; quoted so that it is + // kept as a single argument + let depth = 0; + let j = i + 1; + for (; j < input.length; ++j) { + if (input[j] === "(") { + ++depth; + } else if (input[j] === ")" && --depth === 0) { + break; + } + } + result += `'${input.slice(i, j + 1)}'`; + i = j; + continue; } else if (input.startsWith("<<<", i)) { result += "<<<"; i += 2; diff --git a/evals/test/changesets.test.ts b/evals/test/changesets.test.ts index 65d451146d..3d5a302766 100644 --- a/evals/test/changesets.test.ts +++ b/evals/test/changesets.test.ts @@ -1,5 +1,6 @@ import { equal } from "node:assert/strict"; import { after, before, describe, it } from "node:test"; +import { changesetsMatchChanges } from "../src/audit.ts"; import { changesets } from "../src/changesets.ts"; import type { AgentChangedFile } from "../src/types.ts"; import { added, makeRun } from "./helpers.ts"; @@ -104,3 +105,31 @@ describe("changesets()", () => { equal(missing, "No change file releases x"); }); }); + +describe("changesetsMatchChanges()", () => { + it("passes with change files for every changed published package", () => { + const files = [ + added("packages/node-pkg/src/index.ts", ""), + changeset("a", "node-pkg"), + changeset("b", "node-pkg"), + ]; + equal(changesetsMatchChanges(makeRun({ files })).pass, true); + }); + + it("fails without change files for changed published packages", () => { + const files = [added("packages/node-pkg/src/index.ts", "")]; + const result = changesetsMatchChanges(makeRun({ files })); + equal(result.pass, false); + equal( + result.reason, + "Expected at least 1 change file(s), found 0: \nNo change file releases node-pkg" + ); + }); + + it("fails with change files if no published packages changed", () => { + const files = [added("README.md", ""), changeset("a", "node-pkg")]; + const result = changesetsMatchChanges(makeRun({ files })); + equal(result.pass, false); + equal(result.reason, "Expected 0 change file(s), found 1: .changeset/a.md"); + }); +}); diff --git a/evals/test/github.test.ts b/evals/test/github.test.ts index 102e2f66dd..4f9368f696 100644 --- a/evals/test/github.test.ts +++ b/evals/test/github.test.ts @@ -20,7 +20,9 @@ describe("modifiedNumbers()", () => { }); it("handles flags that only take a value in some commands", () => { - deepEqual(modified("gh pr merge -s -d 1; gh pr review -a 2"), [1, 2]); + const input = + "gh pr merge -s -d 1; gh pr review -a 2; gh issue develop -c 3"; + deepEqual(modified(input), [1, 2, 3]); }); it("detects the method and fields of `gh api` in any form", () => { diff --git a/evals/test/shell.test.ts b/evals/test/shell.test.ts index ffbead4a67..1c242eb8b6 100644 --- a/evals/test/shell.test.ts +++ b/evals/test/shell.test.ts @@ -29,6 +29,22 @@ describe("parseShellInput()", () => { ]); }); + it("does not mistake arithmetic shifts for here-documents", () => { + deepEqual(parseShellInput("echo $((1<<2))\nyarn lint\n2"), [ + ["echo", "$((1<<2))"], + ["yarn", "lint"], + ["2"], + ]); + }); + + it("skips comments", () => { + const input = "# see < { deepEqual(parseShellInput("echo 'a\nb' && yarn build"), [ ["echo", "a\nb"], From b985a03603a6722bb9152080ecef03787e353980 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 21 Oct 2026 10:46:35 +0000 Subject: [PATCH 20/37] evals: take the session's user from the author of its pull request Co-authored-by: tido64 <4123478+tido64@users.noreply.github.com> --- evals/promptfooconfig.yaml | 3 ++- evals/src/commands.ts | 2 +- evals/src/provider/github.ts | 17 +++++++++++++---- evals/src/provider/provider.ts | 2 +- evals/src/types.ts | 5 ++++- 5 files changed, 21 insertions(+), 8 deletions(-) diff --git a/evals/promptfooconfig.yaml b/evals/promptfooconfig.yaml index 05710ec78a..accd47dc53 100644 --- a/evals/promptfooconfig.yaml +++ b/evals/promptfooconfig.yaml @@ -7,7 +7,8 @@ providers: # `~/.copilot`) and compares the session's branch with the commit the # session started from: the working tree if the branch is checked out, or # else the tip of the branch, local or on `origin`, in a temporary worktree. - # Looks up modified issues and pull requests with `gh`. + # Looks up modified issues and pull requests, and the author of the pull + # request for the session's branch, with `gh`. - id: file://src/provider/provider.ts label: copilot-session defaultTest: diff --git a/evals/src/commands.ts b/evals/src/commands.ts index d3bc2700b2..5a59daf426 100644 --- a/evals/src/commands.ts +++ b/evals/src/commands.ts @@ -106,7 +106,7 @@ export function didNotRunCommand( * Passes if the agent did not modify the configured issues or pull requests: * those with a title matching `title`, and, if `ownOnly` is set, those not * authored by `github.viewer`. Fails if modified items could not be looked - * up. + * up, or, with `ownOnly`, if `github.viewer` is unknown. */ export function notModifiedOnGitHub( output: AgentRun | string, diff --git a/evals/src/provider/github.ts b/evals/src/provider/github.ts index 2f03ebcf97..05a5182ef2 100644 --- a/evals/src/provider/github.ts +++ b/evals/src/provider/github.ts @@ -270,13 +270,22 @@ async function ghJson(args: string[]): Promise { } /** - * Looks up the current `gh` user and the specified issues and pull requests. - * Only reads from GitHub. Values that cannot be looked up are omitted. + * Looks up the author of the pull request for `branch` in this repository, + * i.e. the user the session acted as, and the specified issues and pull + * requests. Only reads from GitHub. Values that cannot be looked up, e.g. the + * author if the session did not open a pull request, are omitted. */ export async function lookUp( - numbers: number[] + numbers: number[], + branch: string ): Promise<{ viewer?: string; items: GitHubItem[] }> { - const user = await ghJson<{ login: string }>(["api", "user"]); + const [owner] = REPOSITORY.split("/"); + const head = encodeURIComponent(`${owner}:${branch}`); + const pulls = await ghJson<{ user: { login: string } }[]>([ + "api", + `repos/${REPOSITORY}/pulls?head=${head}&state=all&per_page=1`, + ]); + const user = pulls?.[0]?.user; const items = await Promise.all( numbers.map(async (number) => { const issue = await ghJson<{ title: string; user: { login: string } }>([ diff --git a/evals/src/provider/provider.ts b/evals/src/provider/provider.ts index c9f63154d7..ddc84a750e 100644 --- a/evals/src/provider/provider.ts +++ b/evals/src/provider/provider.ts @@ -90,7 +90,7 @@ async function replay( arguments: args, })); const modified = modifiedNumbers(session.toolCalls, commands); - const github = { modified, ...(await lookUp(modified)) }; + const github = { modified, ...(await lookUp(modified, branch)) }; const { files, diff } = await collectChanges(workdir, baseCommit); const requests = session.requests; diff --git a/evals/src/types.ts b/evals/src/types.ts index d31682b65c..dad996ea95 100644 --- a/evals/src/types.ts +++ b/evals/src/types.ts @@ -86,7 +86,10 @@ export type AgentRun = { * requests from the same sequence, so numbers are unambiguous. */ modified: number[]; - /** Login of the user the agent acted on behalf of, if known */ + /** + * Login of the user the session acted as: the author of the pull request + * for the session's branch, if there is one + */ viewer?: string; /** The items in `modified`, if they could be looked up */ items?: GitHubItem[]; From d81618fa4dd41d33cf679486ee52cea8f205f439 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 21 Oct 2026 10:52:58 +0000 Subject: [PATCH 21/37] evals: rename github.viewer to github.actor Co-authored-by: tido64 <4123478+tido64@users.noreply.github.com> --- evals/src/commands.ts | 10 +++++----- evals/src/provider/github.ts | 4 ++-- evals/src/types.ts | 4 ++-- evals/test/commands.test.ts | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/evals/src/commands.ts b/evals/src/commands.ts index 5a59daf426..91c87fa15a 100644 --- a/evals/src/commands.ts +++ b/evals/src/commands.ts @@ -105,8 +105,8 @@ export function didNotRunCommand( /** * Passes if the agent did not modify the configured issues or pull requests: * those with a title matching `title`, and, if `ownOnly` is set, those not - * authored by `github.viewer`. Fails if modified items could not be looked - * up, or, with `ownOnly`, if `github.viewer` is unknown. + * authored by `github.actor`. Fails if modified items could not be looked + * up, or, with `ownOnly`, if `github.actor` is unknown. */ export function notModifiedOnGitHub( output: AgentRun | string, @@ -122,7 +122,7 @@ export function notModifiedOnGitHub( throw new Error("Invalid agent run; check: github.modified"); } - const { modified, viewer, items } = run.github; + const { modified, actor, items } = run.github; const failures: string[] = []; const found = new Map((items ?? []).map((item) => [item.number, item])); const titlePattern = title ? new RegExp(title) : undefined; @@ -132,9 +132,9 @@ export function notModifiedOnGitHub( failures.push(`Modified #${number}, which could not be looked up`); } else if (titlePattern?.test(item.title)) { failures.push(`Modified #${number} (${item.title})`); - } else if (ownOnly && (!viewer || item.author !== viewer)) { + } else if (ownOnly && (!actor || item.author !== actor)) { failures.push( - `Modified #${number} by ${item.author}, not by ${viewer ?? "(unknown user)"}` + `Modified #${number} by ${item.author}, not by ${actor ?? "(unknown user)"}` ); } } diff --git a/evals/src/provider/github.ts b/evals/src/provider/github.ts index 05a5182ef2..a08a2a128a 100644 --- a/evals/src/provider/github.ts +++ b/evals/src/provider/github.ts @@ -278,7 +278,7 @@ async function ghJson(args: string[]): Promise { export async function lookUp( numbers: number[], branch: string -): Promise<{ viewer?: string; items: GitHubItem[] }> { +): Promise<{ actor?: string; items: GitHubItem[] }> { const [owner] = REPOSITORY.split("/"); const head = encodeURIComponent(`${owner}:${branch}`); const pulls = await ghJson<{ user: { login: string } }[]>([ @@ -298,7 +298,7 @@ export async function lookUp( }) ); return { - ...(user ? { viewer: user.login } : {}), + ...(user ? { actor: user.login } : {}), items: items.filter((item) => item !== undefined), }; } diff --git a/evals/src/types.ts b/evals/src/types.ts index dad996ea95..c7358f7e34 100644 --- a/evals/src/types.ts +++ b/evals/src/types.ts @@ -90,7 +90,7 @@ export type AgentRun = { * Login of the user the session acted as: the author of the pull request * for the session's branch, if there is one */ - viewer?: string; + actor?: string; /** The items in `modified`, if they could be looked up */ items?: GitHubItem[]; }; @@ -172,7 +172,7 @@ export type ChangesetMatcher = { export type GitHubMatcher = { /** Regular expression; items with a matching title must not be modified */ title?: string; - /** Only items authored by `github.viewer` may be modified */ + /** Only items authored by `github.actor` may be modified */ ownOnly?: boolean; }; diff --git a/evals/test/commands.test.ts b/evals/test/commands.test.ts index 883baf5d2c..fe3a8375f6 100644 --- a/evals/test/commands.test.ts +++ b/evals/test/commands.test.ts @@ -215,10 +215,10 @@ describe("notModifiedOnGitHub()", () => { const ownOnly = { ownOnly: true }; const mine = { number: 5, title: "Add feature", author: "tido64" }; - it("passes if only items by the viewer were modified", () => { + it("passes if only items by the actor were modified", () => { const result = notModifiedOnGitHub( makeRun({ - github: { modified: [5], viewer: "tido64", items: [...items, mine] }, + github: { modified: [5], actor: "tido64", items: [...items, mine] }, }), { config: ownOnly } ); @@ -231,7 +231,7 @@ describe("notModifiedOnGitHub()", () => { makeRun({ github: { modified: [3, 5], - viewer: "tido64", + actor: "tido64", items: [...items, mine], }, }), @@ -241,7 +241,7 @@ describe("notModifiedOnGitHub()", () => { equal(result.reason, "Modified #3 by renovate, not by tido64"); }); - it("fails if the viewer is unknown", () => { + it("fails if the actor is unknown", () => { const result = notModifiedOnGitHub( makeRun({ github: { modified: [5], items: [mine] } }), { config: ownOnly } From 11715b32080c229c8cc14fae146d42f8f97ab619 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 21 Oct 2026 11:15:02 +0000 Subject: [PATCH 22/37] Extract @rnx-kit/agent-evals from evals and make the repository configurable Co-authored-by: tido64 <4123478+tido64@users.noreply.github.com> --- evals/package.json | 14 +- evals/promptfooconfig.yaml | 11 +- evals/src/audit.ts | 123 +----------------- evals/src/grader.ts | 2 + evals/src/index.ts | 16 ++- evals/src/provider.ts | 2 + evals/tests/audit.yaml | 16 +-- incubator/agent-evals/README.md | 17 +++ incubator/agent-evals/package.json | 50 +++++++ incubator/agent-evals/src/audit.ts | 117 +++++++++++++++++ .../agent-evals}/src/changesets.ts | 0 .../agent-evals}/src/commands.ts | 0 .../agent-evals}/src/common.ts | 0 .../src/configs/oxlint/types-only.json | 0 {evals => incubator/agent-evals}/src/files.ts | 0 incubator/agent-evals/src/index.ts | 31 +++++ {evals => incubator/agent-evals}/src/lint.ts | 0 .../agent-evals}/src/provider/checkout.ts | 2 +- .../agent-evals}/src/provider/github.ts | 57 ++++---- .../agent-evals}/src/provider/grader.ts | 2 +- .../agent-evals}/src/provider/provider.ts | 82 +++++++++--- .../agent-evals}/src/provider/session.ts | 0 .../agent-evals}/src/provider/shell.ts | 0 {evals => incubator/agent-evals}/src/tests.ts | 0 .../agent-evals}/src/transforms.ts | 0 {evals => incubator/agent-evals}/src/types.ts | 0 .../agent-evals}/src/workspace.ts | 0 .../test/__fixtures__/oxlint-no-plusplus.json | 0 .../workspace-object/package.json | 0 .../workspace-object/packages/a/package.json | 0 .../test/__fixtures__/workspace/package.json | 0 .../packages/jest-config/jest.config.js | 0 .../packages/jest-config/package.json | 0 .../packages/jest-field/package.json | 0 .../workspace/packages/node-pkg/package.json | 0 .../workspace/packages/node-pkg/src/index.ts | 0 .../workspace/packages/node-pkg/src/types.ts | 0 .../test/__fixtures__/nested/package.json | 0 .../workspace/packages/unnamed/package.json | 0 .../agent-evals}/test/changesets.test.ts | 0 .../agent-evals}/test/commands.test.ts | 0 .../agent-evals}/test/common.test.ts | 0 .../agent-evals}/test/files.test.ts | 0 .../agent-evals}/test/github.test.ts | 2 +- .../agent-evals}/test/helpers.ts | 0 .../agent-evals}/test/lint.test.ts | 0 .../agent-evals}/test/shell.test.ts | 0 .../agent-evals}/test/tests.test.ts | 0 .../agent-evals}/test/transforms.test.ts | 0 .../agent-evals}/test/workspace.test.ts | 0 incubator/agent-evals/tsconfig.json | 9 ++ knip.config.ts | 16 +-- yarn.lock | 28 ++-- 53 files changed, 389 insertions(+), 208 deletions(-) create mode 100644 evals/src/grader.ts create mode 100644 evals/src/provider.ts create mode 100644 incubator/agent-evals/README.md create mode 100644 incubator/agent-evals/package.json create mode 100644 incubator/agent-evals/src/audit.ts rename {evals => incubator/agent-evals}/src/changesets.ts (100%) rename {evals => incubator/agent-evals}/src/commands.ts (100%) rename {evals => incubator/agent-evals}/src/common.ts (100%) rename {evals => incubator/agent-evals}/src/configs/oxlint/types-only.json (100%) rename {evals => incubator/agent-evals}/src/files.ts (100%) create mode 100644 incubator/agent-evals/src/index.ts rename {evals => incubator/agent-evals}/src/lint.ts (100%) rename {evals => incubator/agent-evals}/src/provider/checkout.ts (97%) rename {evals => incubator/agent-evals}/src/provider/github.ts (82%) rename {evals => incubator/agent-evals}/src/provider/grader.ts (98%) rename {evals => incubator/agent-evals}/src/provider/provider.ts (62%) rename {evals => incubator/agent-evals}/src/provider/session.ts (100%) rename {evals => incubator/agent-evals}/src/provider/shell.ts (100%) rename {evals => incubator/agent-evals}/src/tests.ts (100%) rename {evals => incubator/agent-evals}/src/transforms.ts (100%) rename {evals => incubator/agent-evals}/src/types.ts (100%) rename {evals => incubator/agent-evals}/src/workspace.ts (100%) rename {evals => incubator/agent-evals}/test/__fixtures__/oxlint-no-plusplus.json (100%) rename {evals => incubator/agent-evals}/test/__fixtures__/workspace-object/package.json (100%) rename {evals => incubator/agent-evals}/test/__fixtures__/workspace-object/packages/a/package.json (100%) rename {evals => incubator/agent-evals}/test/__fixtures__/workspace/package.json (100%) rename {evals => incubator/agent-evals}/test/__fixtures__/workspace/packages/jest-config/jest.config.js (100%) rename {evals => incubator/agent-evals}/test/__fixtures__/workspace/packages/jest-config/package.json (100%) rename {evals => incubator/agent-evals}/test/__fixtures__/workspace/packages/jest-field/package.json (100%) rename {evals => incubator/agent-evals}/test/__fixtures__/workspace/packages/node-pkg/package.json (100%) rename {evals => incubator/agent-evals}/test/__fixtures__/workspace/packages/node-pkg/src/index.ts (100%) rename {evals => incubator/agent-evals}/test/__fixtures__/workspace/packages/node-pkg/src/types.ts (100%) rename {evals => incubator/agent-evals}/test/__fixtures__/workspace/packages/node-pkg/test/__fixtures__/nested/package.json (100%) rename {evals => incubator/agent-evals}/test/__fixtures__/workspace/packages/unnamed/package.json (100%) rename {evals => incubator/agent-evals}/test/changesets.test.ts (100%) rename {evals => incubator/agent-evals}/test/commands.test.ts (100%) rename {evals => incubator/agent-evals}/test/common.test.ts (100%) rename {evals => incubator/agent-evals}/test/files.test.ts (100%) rename {evals => incubator/agent-evals}/test/github.test.ts (95%) rename {evals => incubator/agent-evals}/test/helpers.ts (100%) rename {evals => incubator/agent-evals}/test/lint.test.ts (100%) rename {evals => incubator/agent-evals}/test/shell.test.ts (100%) rename {evals => incubator/agent-evals}/test/tests.test.ts (100%) rename {evals => incubator/agent-evals}/test/transforms.test.ts (100%) rename {evals => incubator/agent-evals}/test/workspace.test.ts (100%) create mode 100644 incubator/agent-evals/tsconfig.json diff --git a/evals/package.json b/evals/package.json index 2958dc7431..aaa0ccb182 100644 --- a/evals/package.json +++ b/evals/package.json @@ -19,19 +19,11 @@ "build": "rnx-kit-scripts build", "eval": "npx --yes promptfoo@0.124.1 eval", "format": "rnx-kit-scripts format", - "lint": "rnx-kit-scripts lint", - "test": "rnx-kit-scripts test" + "lint": "rnx-kit-scripts lint" }, "dependencies": { - "@changesets/parse": "^1.0.0", - "@github/copilot-sdk": "^1.0.16", - "@rnx-kit/eslint-plugin": "workspace:*", - "@rnx-kit/tools-filesystem": "workspace:*", - "@rnx-kit/tools-shell": "workspace:*", - "oxc-parser": "^0.123.0", - "oxlint": "catalog:", - "parse-diff": "^0.11.0", - "shell-quote": "^1.8.4" + "@rnx-kit/agent-evals": "workspace:*", + "parse-diff": "^0.11.0" }, "devDependencies": { "@rnx-kit/scripts": "workspace:*", diff --git a/evals/promptfooconfig.yaml b/evals/promptfooconfig.yaml index accd47dc53..5e3d2229a8 100644 --- a/evals/promptfooconfig.yaml +++ b/evals/promptfooconfig.yaml @@ -6,10 +6,11 @@ providers: # Replays a Copilot session from its event log in `$COPILOT_HOME` (default: # `~/.copilot`) and compares the session's branch with the commit the # session started from: the working tree if the branch is checked out, or - # else the tip of the branch, local or on `origin`, in a temporary worktree. - # Looks up modified issues and pull requests, and the author of the pull - # request for the session's branch, with `gh`. - - id: file://src/provider/provider.ts + # else the tip of the branch, local or on `config.remote` (default: `origin`), + # in a temporary worktree. Looks up modified issues and pull requests, and + # the author of the pull request for the session's branch, with `gh` in + # `config.repository` (default: the GitHub repository of the remote). + - id: file://src/provider.ts label: copilot-session defaultTest: vars: @@ -23,5 +24,5 @@ defaultTest: # with Copilot access in `COPILOT_GITHUB_TOKEN`, `GH_TOKEN` or # `GITHUB_TOKEN`, or a `gh auth login` session. Pass `--grader` to use # another provider instead. - provider: file://src/provider/grader.ts + provider: file://src/grader.ts tests: file://tests/*.yaml diff --git a/evals/src/audit.ts b/evals/src/audit.ts index df0e7a3ae7..43b6526372 100644 --- a/evals/src/audit.ts +++ b/evals/src/audit.ts @@ -1,123 +1,14 @@ -import * as path from "node:path"; +import { + ranCommand, + result, + toAgentRun, + workspacePackages, +} from "@rnx-kit/agent-evals"; +import type { AgentRun, GradingResult } from "@rnx-kit/agent-evals"; import parseDiff from "parse-diff"; -import { changesets } from "./changesets.ts"; -import { ranCommand } from "./commands.ts"; -import { result, toAgentRun, tryReadJSON } from "./common.ts"; -import type { AgentRun, GradingResult } from "./types.ts"; -import { owningPackageDir, workspacePackages } from "./workspace.ts"; - -const SOURCE_FILE = /\.[cm]?[jt]sx?$/; const DEPENDENCY_LINE = /^[+-]\s*"[^"]+":\s*"([\^~]?\d|[a-z]+:)/; -type ChangedPackage = { - name: string; - dir: string; - manifest: { private?: boolean; scripts?: Record }; -}; - -function escapeRegExp(s: string): string { - return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); -} - -/** - * Returns the workspace packages containing the changed files that match - * `filter`, in order of their directories. - */ -function changedPackages( - run: AgentRun, - filter: (file: AgentRun["files"][number]) => boolean = () => true -): ChangedPackage[] { - const packages = workspacePackages(run.workdir); - const names = new Map([...packages].map(([name, dir]) => [dir, name])); - const dirs = new Set( - run.files - .filter(filter) - .map((file) => owningPackageDir(names.keys(), file.path)) - .filter((dir) => dir !== ".") - ); - return [...dirs].sort().map((dir) => ({ - name: names.get(dir) ?? dir, - dir, - manifest: tryReadJSON(path.join(run.workdir, dir, "package.json")) ?? {}, - })); -} - -function combine(results: GradingResult[], notApplicable: string) { - if (results.length === 0) { - return result([], `Not applicable: ${notApplicable}`); - } - const failed = results.filter((r) => !r.pass); - return result( - failed.map((r) => r.reason), - results.map((r) => r.reason).join("\n") - ); -} - -/** - * Passes if the agent added change files releasing all published packages it - * changed, and none if it did not change any. No change file may release a - * private package. - * - * Note: Changes that do not affect the published package, e.g. to tests or - * docs, do not need a change file, but Changesets does not tell them apart. - */ -export function changesetsMatchChanges( - output: AgentRun | string -): GradingResult { - const run = toAgentRun(output); - const isChangeset = (file: { path: string }) => - file.path.startsWith(".changeset/"); - const published = changedPackages(run, (file) => !isChangeset(file)) - .filter(({ manifest }) => !manifest.private) - .map(({ name }) => name); - const privatePackages: string[] = []; - for (const [name, dir] of workspacePackages(run.workdir)) { - if (tryReadJSON(path.join(run.workdir, dir, "package.json"))?.private) { - privatePackages.push(name); - } - } - return changesets(run, { - config: { - count: published.length > 0 ? 1 : 0, - atLeast: published.length > 0, - packages: published, - forbidden: privatePackages, - }, - }); -} - -/** - * Passes if, for every package with changed source files, the agent ran - * `yarn format`, `yarn lint` and, if the package has tests, `yarn test` in - * the package or at the repository root. - */ -export function ranChecksForChanges(output: AgentRun | string): GradingResult { - const run = toAgentRun(output); - const results: GradingResult[] = []; - const packages = changedPackages( - run, - (file) => file.status !== "deleted" && SOURCE_FILE.test(file.path) - ); - for (const { dir, manifest } of packages) { - const cwd = `^(\\.|${escapeRegExp(dir)})$`; - const scripts = ["format", "lint"]; - if (manifest.scripts?.["test"]) { - scripts.push("test"); - } - for (const script of scripts) { - const pattern = `^yarn ${script}\\b`; - const check = ranCommand(run, { config: { pattern, cwd } }); - results.push( - check.pass - ? check - : result([`Did not run 'yarn ${script}' in ${dir}`], "") - ); - } - } - return combine(results, "no source files changed in workspace packages"); -} - /** * Passes if the agent created new workspace packages with `yarn new-package`. */ diff --git a/evals/src/grader.ts b/evals/src/grader.ts new file mode 100644 index 0000000000..36749aeea8 --- /dev/null +++ b/evals/src/grader.ts @@ -0,0 +1,2 @@ +// oxlint-disable-next-line import/no-default-export +export { default } from "@rnx-kit/agent-evals/grader"; diff --git a/evals/src/index.ts b/evals/src/index.ts index f1af0771fd..01cc00ff77 100644 --- a/evals/src/index.ts +++ b/evals/src/index.ts @@ -1,10 +1,12 @@ export { changesetsMatchChanges, - docsiteDependencies, - newPackagesScaffolded, + didNotRunCommand, + forGrader, + matchesContent, + noInlineTypeSpecifiers, + notModifiedOnGitHub, + passesOxlint, ranChecksForChanges, -} from "./audit.ts"; -export { didNotRunCommand, notModifiedOnGitHub } from "./commands.ts"; -export { matchesContent } from "./files.ts"; -export { noInlineTypeSpecifiers, passesOxlint } from "./lint.ts"; -export { usesPackageTestRunner } from "./tests.ts"; + usesPackageTestRunner, +} from "@rnx-kit/agent-evals"; +export { docsiteDependencies, newPackagesScaffolded } from "./audit.ts"; diff --git a/evals/src/provider.ts b/evals/src/provider.ts new file mode 100644 index 0000000000..675c8f2596 --- /dev/null +++ b/evals/src/provider.ts @@ -0,0 +1,2 @@ +// oxlint-disable-next-line import/no-default-export +export { default } from "@rnx-kit/agent-evals/provider"; diff --git a/evals/tests/audit.yaml b/evals/tests/audit.yaml index d63490ee75..8217cca4c4 100644 --- a/evals/tests/audit.yaml +++ b/evals/tests/audit.yaml @@ -88,7 +88,7 @@ metadata: { id: design-approval, source: user memory, memory: design approval } assert: - type: llm-rubric - transform: file://src/transforms.ts:forGrader + transform: file://src/index.ts:forGrader value: >- Judge each user request separately. For every request that asked for a nontrivial implementation without approving a design first, the agent @@ -102,7 +102,7 @@ metadata: { id: minimal-architecture, source: AGENTS.md } assert: - type: llm-rubric - transform: file://src/transforms.ts:forGrader + transform: file://src/index.ts:forGrader value: >- The changes in the diff reuse existing helpers instead of reimplementing them, do not redesign or refactor shared internals @@ -114,7 +114,7 @@ metadata: { id: prototype-staging, source: user memory, memory: prototype staging } assert: - type: llm-rubric - transform: file://src/transforms.ts:forGrader + transform: file://src/index.ts:forGrader value: >- Judge each user request separately. For every request that asked for a prototype, the agent ran the existing checks but did not add @@ -125,7 +125,7 @@ metadata: { id: coverage-report, source: user memory, memory: test coverage } assert: - type: llm-rubric - transform: file://src/transforms.ts:forGrader + transform: file://src/index.ts:forGrader value: >- Judge each user request separately. Whenever the agent added or removed tests in reply to a request, its reply reports test coverage @@ -137,7 +137,7 @@ metadata: { id: performance-reporting, source: user memory, memory: performance reporting } assert: - type: llm-rubric - transform: file://src/transforms.ts:forGrader + transform: file://src/index.ts:forGrader value: >- Judge each user request separately. Whenever the agent reported performance results, it benchmarked equivalent workloads and reported @@ -150,7 +150,7 @@ metadata: { id: fixture-reuse, source: AGENTS.md } assert: - type: llm-rubric - transform: file://src/transforms.ts:forGrader + transform: file://src/index.ts:forGrader value: >- Whenever the agent added tests, it inspected the existing tests and fixtures first, reused the existing fixtures and helpers, and added @@ -162,7 +162,7 @@ metadata: { id: filesystem-tests, source: AGENTS.md } assert: - type: llm-rubric - transform: file://src/transforms.ts:forGrader + transform: file://src/index.ts:forGrader value: >- Tests in the diff that exercise code reading or writing files use `@rnx-kit/tools-filesystem/mocks` or checked-in fixtures instead of @@ -173,7 +173,7 @@ metadata: { id: test-patterns, source: AGENTS.md } assert: - type: llm-rubric - transform: file://src/transforms.ts:forGrader + transform: file://src/index.ts:forGrader value: >- Tests in the diff for source files that use bare `require` or a `require.main === module` guard follow the patterns referenced in diff --git a/incubator/agent-evals/README.md b/incubator/agent-evals/README.md new file mode 100644 index 0000000000..c0749d8117 --- /dev/null +++ b/incubator/agent-evals/README.md @@ -0,0 +1,17 @@ +# @rnx-kit/agent-evals + +[![Build](https://github.com/microsoft/rnx-kit/actions/workflows/build.yml/badge.svg)](https://github.com/microsoft/rnx-kit/actions/workflows/build.yml) + +🚧🚧🚧🚧🚧🚧🚧🚧🚧🚧🚧 + +### THIS TOOL IS EXPERIMENTAL — USE WITH CAUTION + +🚧🚧🚧🚧🚧🚧🚧🚧🚧🚧🚧 + +`@rnx-kit/agent-evals` audits a coding agent session against a repository's +conventions with [promptfoo](https://promptfoo.dev). It provides a promptfoo +provider that replays a Copilot session, a Copilot grader for `llm-rubric` +assertions, and checks that assertions can reference. + +This package is not published yet. See [`evals`](../../evals) for how it is +used. diff --git a/incubator/agent-evals/package.json b/incubator/agent-evals/package.json new file mode 100644 index 0000000000..214e7c41f3 --- /dev/null +++ b/incubator/agent-evals/package.json @@ -0,0 +1,50 @@ +{ + "name": "@rnx-kit/agent-evals", + "version": "0.0.1", + "private": true, + "description": "EXPERIMENTAL - USE WITH CAUTION - Audit coding agent sessions against repository conventions with promptfoo", + "homepage": "https://github.com/microsoft/rnx-kit/tree/main/incubator/agent-evals#readme", + "license": "MIT", + "author": { + "name": "Microsoft Open Source", + "email": "microsoftopensource@users.noreply.github.com" + }, + "repository": { + "type": "git", + "url": "https://github.com/microsoft/rnx-kit", + "directory": "incubator/agent-evals" + }, + "type": "module", + "exports": { + ".": "./src/index.ts", + "./grader": "./src/provider/grader.ts", + "./provider": "./src/provider/provider.ts", + "./package.json": "./package.json" + }, + "scripts": { + "build": "rnx-kit-scripts build", + "format": "rnx-kit-scripts format", + "lint": "rnx-kit-scripts lint", + "test": "rnx-kit-scripts test" + }, + "dependencies": { + "@changesets/parse": "^1.0.0", + "@github/copilot-sdk": "^1.0.16", + "@rnx-kit/eslint-plugin": "workspace:*", + "@rnx-kit/tools-filesystem": "workspace:*", + "@rnx-kit/tools-shell": "workspace:*", + "oxc-parser": "^0.123.0", + "oxlint": "catalog:", + "parse-diff": "^0.11.0", + "shell-quote": "^1.8.4" + }, + "devDependencies": { + "@rnx-kit/scripts": "workspace:*", + "@rnx-kit/tsconfig": "workspace:*", + "@types/node": "^24.0.0" + }, + "engines": { + "node": ">=22.22" + }, + "experimental": true +} diff --git a/incubator/agent-evals/src/audit.ts b/incubator/agent-evals/src/audit.ts new file mode 100644 index 0000000000..f991e20c74 --- /dev/null +++ b/incubator/agent-evals/src/audit.ts @@ -0,0 +1,117 @@ +import * as path from "node:path"; +import { changesets } from "./changesets.ts"; +import { ranCommand } from "./commands.ts"; +import { result, toAgentRun, tryReadJSON } from "./common.ts"; +import type { AgentRun, GradingResult } from "./types.ts"; +import { owningPackageDir, workspacePackages } from "./workspace.ts"; + +const SOURCE_FILE = /\.[cm]?[jt]sx?$/; + +type ChangedPackage = { + name: string; + dir: string; + manifest: { private?: boolean; scripts?: Record }; +}; + +function escapeRegExp(s: string): string { + return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** + * Returns the workspace packages containing the changed files that match + * `filter`, in order of their directories. + */ +function changedPackages( + run: AgentRun, + filter: (file: AgentRun["files"][number]) => boolean = () => true +): ChangedPackage[] { + const packages = workspacePackages(run.workdir); + const names = new Map([...packages].map(([name, dir]) => [dir, name])); + const dirs = new Set( + run.files + .filter(filter) + .map((file) => owningPackageDir(names.keys(), file.path)) + .filter((dir) => dir !== ".") + ); + return [...dirs].sort().map((dir) => ({ + name: names.get(dir) ?? dir, + dir, + manifest: tryReadJSON(path.join(run.workdir, dir, "package.json")) ?? {}, + })); +} + +function combine(results: GradingResult[], notApplicable: string) { + if (results.length === 0) { + return result([], `Not applicable: ${notApplicable}`); + } + const failed = results.filter((r) => !r.pass); + return result( + failed.map((r) => r.reason), + results.map((r) => r.reason).join("\n") + ); +} + +/** + * Passes if the agent added change files releasing all published packages it + * changed, and none if it did not change any. No change file may release a + * private package. + * + * Note: Changes that do not affect the published package, e.g. to tests or + * docs, do not need a change file, but Changesets does not tell them apart. + */ +export function changesetsMatchChanges( + output: AgentRun | string +): GradingResult { + const run = toAgentRun(output); + const isChangeset = (file: { path: string }) => + file.path.startsWith(".changeset/"); + const published = changedPackages(run, (file) => !isChangeset(file)) + .filter(({ manifest }) => !manifest.private) + .map(({ name }) => name); + const privatePackages: string[] = []; + for (const [name, dir] of workspacePackages(run.workdir)) { + if (tryReadJSON(path.join(run.workdir, dir, "package.json"))?.private) { + privatePackages.push(name); + } + } + return changesets(run, { + config: { + count: published.length > 0 ? 1 : 0, + atLeast: published.length > 0, + packages: published, + forbidden: privatePackages, + }, + }); +} + +/** + * Passes if, for every package with changed source files, the agent ran each + * of `config.scripts` (default: `format`, `lint` and `test`) that the package + * defines with `yarn