Creating agent harness for rnx-kit eval tests

.changeset/happy-cougars-taste.md

@@ -0,0 +1,2 @@
+---
+---

.yarnrc.yml

@@ -39,6 +39,7 @@ catalog:
   oxc-resolver: ^11.0.0
   oxfmt: ^0.72.0
   oxlint: ^1.87.0
+  parse-diff: ^0.11.0
   qrcode: ^1.5.0
   type-fest: ^4.0.0
   typescript: ^7.0.2

AGENTS.md

@@ -142,8 +142,8 @@ One change file per feature/fix. No need for multiple entries when addressing PR
 feedback. Releases are automated via CI — on merge to `main`, the changesets
 action creates a release PR or publishes to npm.
 
-Ignored packages (no changesets needed): `@rnx-kit/ignore`, `@rnx-kit/template`,
-test apps.
+Don't add change files for private packages (`"private": true`), e.g.
+`@rnx-kit/template` and test apps.
 
 ## Key Packages
 
@@ -210,9 +210,10 @@ constraints, align-deps, and readme issues.
 ## Code Style
 
 - Run `yarn format` and `yarn lint` before committing (CI enforces both)
-- Don't format or lint by hand or call the underlying tools directly — the
-  `rnx-kit-scripts format` and `lint` commands (`scripts/src/commands/`) decide
-  which tools and configs are used
+- Don't format, lint or build by hand or call the underlying tools (e.g.
+  `oxfmt`, `oxlint`, `tsc`) directly — the `rnx-kit-scripts format`, `lint` and
+  `build` commands (`scripts/src/commands/`) decide which tools and configs are
+  used
 - The formatter also sorts imports and `package.json` fields
 - Prefer consuming existing helpers over redesigning shared internals. Ask
   before adding new shared abstractions; optimize for total complexity, not

evals/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "@rnx-kit/evals",
+  "version": "0.0.0",
+  "private": true,
+  "description": "Agent evals for the rnx-kit repository",
+  "homepage": "https://github.com/microsoft/rnx-kit/tree/main/evals#readme",
+  "license": "MIT",
+  "author": {
+    "name": "Microsoft Open Source",
+    "email": "microsoftopensource@users.noreply.github.com"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/microsoft/rnx-kit",
+    "directory": "evals"
+  },
+  "type": "module",
+  "scripts": {
+    "build": "rnx-kit-scripts build",
+    "eval": "npx --yes promptfoo@0.124.1 eval",
+    "format": "rnx-kit-scripts format",
+    "lint": "rnx-kit-scripts lint",
+    "promptfoo": "npx --yes promptfoo@0.124.1",
+    "test": "rnx-kit-scripts test"
+  },
+  "dependencies": {
+    "@rnx-kit/agent-evals": "workspace:*",
+    "@rnx-kit/eslint-plugin": "workspace:*",
+    "@rnx-kit/tools-filesystem": "workspace:*",
+    "oxc-parser": "catalog:",
+    "oxlint": "catalog:",
+    "parse-diff": "catalog:"
+  },
+  "devDependencies": {
+    "@rnx-kit/scripts": "workspace:*",
+    "@rnx-kit/tsconfig": "workspace:*",
+    "@types/node": "^24.0.0"
+  },
+  "engines": {
+    "node": ">=24.11"
+  }
+}

evals/promptfooconfig.yaml

@@ -0,0 +1,35 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: rnx-kit session audit
+prompts:
+  - "{{copilotSession}}"
+providers:
+  # Replays a Copilot session from its event log in `$COPILOT_HOME` (default:
+  # `~/.copilot`) and compares the session's branch with the commit the
+  # session started from: the working tree if the branch is checked out, or
+  # else the tip of the branch, local or on `config.remote` (default: `origin`),
+  # in a temporary worktree. Looks up modified issues and pull requests, and
+  # the author of the pull request for the session's branch, with `gh` in
+  # `config.repository` (default: the GitHub repository of the remote).
+  - id: file://src/provider.ts
+    label: copilot-session
+defaultTest:
+  vars:
+    # ID of the session to audit, e.g. `--var copilotSession=<id>`. If empty,
+    # audits the most recently updated session in this repository that is not
+    # in use. Not named `session` because promptfoo redacts fields with that
+    # name.
+    copilotSession: ""
+  options:
+    # Grades `llm-rubric` assertions with Copilot; requires a GitHub token
+    # with Copilot access in `COPILOT_GITHUB_TOKEN`, `GH_TOKEN` or
+    # `GITHUB_TOKEN`, or a `gh auth login` session. Pass `--grader` to use
+    # another provider instead.
+    provider: file://src/grader.ts
+tests: file://tests/*.yaml
+extensions:
+  # Prints each test with the reasons it failed, instead of promptfoo's table.
+  # Pass `--table` to show the table as well, or run `yarn view` to browse the
+  # results.
+  - file://src/report.ts:afterAll
+commandLineOptions:
+  table: false

evals/src/audit.ts

@@ -0,0 +1,98 @@
+import {
+  changedPackages,
+  changesets,
+  ranCommand,
+  result,
+  toAgentRun,
+  tryReadJSON,
+  workspacePackages,
+} from "@rnx-kit/agent-evals";
+import type { AgentRun, GradingResult } from "@rnx-kit/agent-evals";
+import * as path from "node:path";
+import parseDiff from "parse-diff";
+
+const DEPENDENCY_LINE = /^[+-]\s*"[^"]+":\s*"([\^~]?\d|[a-z]+:)/;
+
+/**
+ * Passes if the agent created new workspace packages with `yarn new-package`.
+ */
+export function newPackagesScaffolded(
+  output: AgentRun | string
+): GradingResult {
+  const run = toAgentRun(output);
+  const added = Array.from(workspacePackages(run.workdir).values()).filter(
+    (dir) =>
+      run.files.some(
+        (file) => file.status === "added" && file.path === `${dir}/package.json`
+      )
+  );
+  if (added.length === 0) {
+    return result([], "Not applicable: no packages were added");
+  }
+
+  const check = ranCommand(run, {
+    config: { pattern: "^yarn new-package\\b" },
+  });
+  return check.pass
+    ? check
+    : result([`Added ${added.join(", ")} without 'yarn new-package'`], "");
+}
+
+/**
+ * Passes if dependency changes in `docsite/package.json` are installed in
+ * `docsite/yarn.lock`, which is separate from the root lockfile.
+ */
+export function docsiteDependencies(output: AgentRun | string): GradingResult {
+  const run = toAgentRun(output);
+  const manifest = parseDiff(run.diff).find(
+    (file) => file.to === "docsite/package.json"
+  );
+  const changed = manifest?.chunks.some((chunk) =>
+    chunk.changes.some(
+      (change) =>
+        change.type !== "normal" && DEPENDENCY_LINE.test(change.content)
+    )
+  );
+  if (!changed) {
+    return result(
+      [],
+      "Not applicable: no dependencies changed in docsite/package.json"
+    );
+  }
+  return run.files.some((file) => file.path === "docsite/yarn.lock")
+    ? result([], "docsite/yarn.lock changed")
+    : result(["Changed dependencies without updating docsite/yarn.lock"], "");
+}
+
+/**
+ * Passes if the agent added change files releasing all published packages it
+ * changed, and none if it did not change any. No change file may release a
+ * private package.
+ *
+ * Note: Changes that do not affect the published package, e.g. to tests or
+ * docs, do not need a change file, but Changesets does not tell them apart.
+ */
+export function changesetsMatchChanges(
+  output: AgentRun | string
+): GradingResult {
+  const run = toAgentRun(output);
+  const isChangeset = (file: { path: string }) =>
+    file.path.startsWith(".changeset/");
+  const published = changedPackages(run, (file) => !isChangeset(file))
+    .filter(({ manifest }) => !manifest.private)
+    .map(({ name }) => name);
+  const privatePackages: string[] = [];
+  for (const [name, dir] of workspacePackages(run.workdir)) {
+    if (tryReadJSON(path.join(run.workdir, dir, "package.json"))?.private) {
+      privatePackages.push(name);
+    }
+  }
+  return changesets(run, {
+    config: {
+      count: published.length > 0 ? 1 : 0,
+      atLeast: published.length > 0,
+      packages: published,
+      forbidden: privatePackages,
+    },
+  });
+}

evals/src/configs/oxlint/types-only.json

@@ -0,0 +1,9 @@
+{
+  "$schema": "../../../node_modules/oxlint/configuration_schema.json",
+  "plugins": [],
+  "categories": { "correctness": "off" },
+  "jsPlugins": [
+    { "name": "@rnx-kit", "specifier": "@rnx-kit/eslint-plugin/rules" }
+  ],
+  "rules": { "@rnx-kit/type-definitions-only": "error" }
+}

evals/src/grader.ts

@@ -0,0 +1,2 @@
+// oxlint-disable-next-line import/no-default-export
+export { default } from "@rnx-kit/agent-evals/grader";

evals/src/index.ts

@@ -0,0 +1,15 @@
+export {
+  didNotRunCommand,
+  forGrader,
+  matchesContent,
+  notModifiedOnGitHub,
+  ranChecksForChanges,
+} from "@rnx-kit/agent-evals";
+export {
+  changesetsMatchChanges,
+  docsiteDependencies,
+  newPackagesScaffolded,
+} from "./audit.ts";
+export { noInlineTypeSpecifiers } from "./lint.ts";
+export { passesOxlint } from "./oxlint.ts";
+export { usesPackageTestRunner } from "./tests.ts";

evals/src/lint.ts

@@ -0,0 +1,85 @@
+import {
+  addedLines,
+  existingFiles,
+  requireConfig,
+  result,
+  toAgentRun,
+} from "@rnx-kit/agent-evals";
+import type {
+  AgentRun,
+  AssertionContext,
+  GradingResult,
+} from "@rnx-kit/agent-evals";
+import { parseSync } from "oxc-parser";
+
+/**
+ * Configuration for `noInlineTypeSpecifiers`.
+ */
+export type TypeSpecifierMatcher = {
+  /** Regular expression matching the paths of the files to check */
+  files: string;
+};
+
+/**
+ * Passes if the matching files do not use inline `type` specifiers in import
+ * or export statements, e.g. `import { type A, b }`, on lines added in the
+ * diff. Passes trivially if no files match.
+ */
+export function noInlineTypeSpecifiers(
+  output: AgentRun | string,
+  context: AssertionContext<TypeSpecifierMatcher>
+): GradingResult {
+  const run = toAgentRun(output);
+  const pathPattern = new RegExp(requireConfig(context).files);
+  const added = addedLines(run.diff);
+
+  const failures: string[] = [];
+  const checked: string[] = [];
+  for (const file of existingFiles(run)) {
+    if (!pathPattern.test(file.path)) {
+      continue;
+    }
+
+    const lines = added.get(file.path);
+    if (!lines?.size) {
+      continue;
+    }
+
+    checked.push(file.path);
+    const content = file.content ?? "";
+    const { program, errors } = parseSync(file.path, content);
+    for (const { message } of errors) {
+      failures.push(`${file.path}: ${message}`);
+    }
+
+    // Only top-level statements are checked; imports and exports nested in
+    // ambient module declarations (`declare module "…" {}`) are not. Lint
+    // allows these; `typescript/no-namespace` only rejects named namespaces.
+    for (const node of program.body) {
+      let keyword: string;
+      let specifiers: { start: number }[];
+      if (node.type === "ImportDeclaration") {
+        keyword = "import";
+        specifiers = node.specifiers.filter(
+          (s) => s.type === "ImportSpecifier" && s.importKind === "type"
+        );
+      } else if (node.type === "ExportNamedDeclaration") {
+        keyword = "export";
+        specifiers = node.specifiers.filter((s) => s.exportKind === "type");
+      } else {
+        continue;
+      }
+
+      for (const { start } of specifiers) {
+        const line = content.substring(0, start).split("\n").length;
+        if (lines.has(line)) {
+          failures.push(
+            `${file.path}:${line}: Use a separate \`${keyword} type\` statement`
+          );
+        }
+      }
+    }
+  }
+
+  return result(failures, `Checked ${checked.join(", ") || "(none)"}`);
+}