diff --git a/.github/workflows/pipboy.yml b/.github/workflows/pipboy.yml index 274f988f67..9d8ba65a9a 100644 --- a/.github/workflows/pipboy.yml +++ b/.github/workflows/pipboy.yml @@ -26,9 +26,12 @@ jobs: contents: read outputs: base: ${{ steps.changes.outputs.base }} + base_history: ${{ steps.changes.outputs.base_history }} footprint: ${{ steps.changes.outputs.footprint }} tool: ${{ steps.changes.outputs.tool }} rust: ${{ steps.changes.outputs.rust }} + native: ${{ steps.changes.outputs.native }} + native_matrix: ${{ steps.changes.outputs.native_matrix }} steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: @@ -38,16 +41,40 @@ jobs: env: EVENT: ${{ github.event_name }} PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + PUSH_BEFORE: ${{ github.event.before }} + LABELS: ${{ toJSON(github.event.pull_request.labels.*.name) }} + # The sidecar binaries the publish workflow ships, on the same runners. + # `linux` builds on any Rust change; the rest build on platform-specific + # changes or the `ci: native-all` label, which is read on the next push. + NATIVE_ALL: >- + [ + {"platform":"linux-x64-gnu","target":"x86_64-unknown-linux-gnu","runner":"ubuntu-latest","dockerfile":"linux-gnu","clang":"","tier":"linux"}, + {"platform":"linux-arm64-gnu","target":"aarch64-unknown-linux-gnu","runner":"ubuntu-24.04-arm","dockerfile":"linux-gnu","clang":"","tier":"all"}, + {"platform":"darwin-x64","target":"x86_64-apple-darwin","runner":"ubuntu-latest","dockerfile":"darwin","clang":"x86_64-apple-darwin20.4","tier":"all"}, + {"platform":"darwin-arm64","target":"aarch64-apple-darwin","runner":"ubuntu-latest","dockerfile":"darwin","clang":"aarch64-apple-darwin20.4","tier":"all"} + ] run: | + RUST='(\.rs$|(^|/)Cargo\.(toml|lock)$|^deny\.toml$|^rust-toolchain\.toml$|^docker/build/|^scripts/pipboy/|^\.github/workflows/pipboy\.yml$)' + PLATFORM_SENSITIVE='((^|/)Cargo\.(toml|lock)$|(^|/)build\.rs$|^docker/build/|^rust-toolchain\.toml$)' + matrix() { jq -c --arg tier "$1" 'map(select($tier == "all" or .tier == $tier))' <<< "$NATIVE_ALL"; } + if [ "$EVENT" != "pull_request" ]; then + # Pushes to main record baselines. Natives build only when Rust changed, + # and then for every platform. Hygiene stores no baseline, so it skips main. echo "footprint=true" >> "$GITHUB_OUTPUT" echo "tool=true" >> "$GITHUB_OUTPUT" - # Hygiene stores no baseline, so it skips main. echo "rust=false" >> "$GITHUB_OUTPUT" + native=none + if ! files="$(git diff --name-only "$PUSH_BEFORE" HEAD 2>/dev/null)" || grep -qE "$RUST" <<< "$files"; then + native=all + fi + echo "native=$native" >> "$GITHUB_OUTPUT" + echo "native_matrix=$(matrix "$native")" >> "$GITHUB_OUTPUT" exit 0 fi base="$(git merge-base "$PR_BASE_SHA" HEAD)" echo "base=$base" >> "$GITHUB_OUTPUT" + echo "base_history=$(git rev-list --first-parent -n 200 "$base" | paste -sd, -)" >> "$GITHUB_OUTPUT" files="$(git diff --name-only "$base" HEAD)" footprint=false if grep -qE '^(packages/|pnpm-(lock|workspace)\.yaml|scripts/pipboy/|scripts/publish/|\.github/workflows/pipboy\.yml)' <<< "$files"; then @@ -60,10 +87,19 @@ jobs: fi echo "tool=$tool" >> "$GITHUB_OUTPUT" rust=false - if grep -qE '(\.rs$|(^|/)Cargo\.(toml|lock)$|^deny\.toml$|^rust-toolchain\.toml$|^scripts/pipboy/|^\.github/workflows/pipboy\.yml$)' <<< "$files"; then + native=none + if grep -qE "$RUST" <<< "$files"; then rust=true + native=linux + if grep -qE "$PLATFORM_SENSITIVE" <<< "$files" \ + || git diff -U0 "$base" HEAD -- '*.rs' | grep -qE '^\+.*(cfg\(target_|target_os|target_arch)' \ + || jq -e 'index("ci: native-all")' <<< "$LABELS" > /dev/null; then + native=all + fi fi echo "rust=$rust" >> "$GITHUB_OUTPUT" + echo "native=$native" >> "$GITHUB_OUTPUT" + echo "native_matrix=$(matrix "$native")" >> "$GITHUB_OUTPUT" self-test: name: self-test @@ -210,15 +246,72 @@ jobs: if-no-files-found: ignore retention-days: 30 + native: + name: native · ${{ matrix.platform }} + needs: changes + if: needs.changes.outputs.native != 'none' + runs-on: ${{ matrix.runner }} + timeout-minutes: 120 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.changes.outputs.native_matrix) }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + # The publish workflow's Dockerfile and build arguments in release mode, so + # the size is what ships. Toolchain versions are read from publish.yaml. + - name: Build with the publish Dockerfile + env: + PLATFORM: ${{ matrix.platform }} + TARGET: ${{ matrix.target }} + DOCKERFILE: ${{ matrix.dockerfile }} + CLANG: ${{ matrix.clang }} + run: | + publish_env() { sed -n "s/^ $1: \"\(.*\)\"$/\1/p" .github/workflows/publish.yaml; } + args=(--build-arg "TARGET=$TARGET" --build-arg BUILD_PROFILE=release --build-arg "CACHE_PLATFORM=$PLATFORM" + --build-arg "RUST_TOOLCHAIN=$(publish_env RUST_TOOLCHAIN)") + if [ "$DOCKERFILE" = darwin ]; then + args+=(--build-arg "CLANG=$CLANG") + else + args+=(--build-arg "LINUX_GNU_LLVM_VERSION=$(publish_env LINUX_GNU_LLVM_VERSION)" + --build-arg "LINUX_GNU_SYSROOT_TAG=$(publish_env LINUX_GNU_SYSROOT_TAG)") + fi + docker buildx build -f "docker/build/$DOCKERFILE.Dockerfile" "${args[@]}" -t pipboy-native --load . + cid="$(docker create pipboy-native)" + docker cp "$cid:/artifacts/agentos-native-sidecar" "$RUNNER_TEMP/agentos-native-sidecar" + docker rm "$cid" + - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 + # No pnpm cache: this job runs fork code and doesn't need one. + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: 24 + - run: pnpm install --frozen-lockfile --filter 'pipboy...' + - name: Measure + env: + PLATFORM: ${{ matrix.platform }} + run: | + mkdir -p "$RUNNER_TEMP/reports" + pnpm --silent --filter pipboy pipboy native --target agentos-native-sidecar --platform "$PLATFORM" \ + --artifact "$RUNNER_TEMP/agentos-native-sidecar" --out "$RUNNER_TEMP/reports/native-agentos-native-sidecar-$PLATFORM.json" + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: report-native-agentos-native-sidecar-${{ matrix.platform }} + path: ${{ runner.temp }}/reports/native-agentos-native-sidecar-${{ matrix.platform }}.json + retention-days: 90 + report: name: report - needs: [changes, footprint, hygiene] + needs: [changes, footprint, hygiene, native] # Not `always()`: a run cancelled by a newer push must not post its partial results. if: ${{ !cancelled() }} runs-on: ubuntu-latest timeout-minutes: 10 permissions: - actions: read # job timings and report artifacts + actions: read # job timings, report artifacts, and native baselines from main pull-requests: write # the one Pip-Boy comment steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 @@ -239,12 +332,16 @@ jobs: - name: Write the comment and run summary env: GITHUB_TOKEN: ${{ github.token }} + GH_TOKEN: ${{ github.token }} # Empty for pushes and fork PRs: they get the run summary only. PR_NUMBER: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.number || '' }} HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + BASE_HISTORY: ${{ needs.changes.outputs.base_history }} # Each check job's result, so a failed job is reported as failed, not skipped. JOB_RESULTS: ${{ toJSON(needs) }} - run: pnpm --silent --filter pipboy pipboy report --reports "$RUNNER_TEMP/reports" + run: | + pnpm --silent --filter pipboy pipboy report --reports "$RUNNER_TEMP/reports" \ + --base-history "$BASE_HISTORY" zizmor: name: zizmor diff --git a/scripts/pipboy/src/cli.ts b/scripts/pipboy/src/cli.ts index a045f43854..81cbb06402 100644 --- a/scripts/pipboy/src/cli.ts +++ b/scripts/pipboy/src/cli.ts @@ -1,3 +1,4 @@ +import { tmpdir } from "node:os"; import { join } from "node:path"; import { NodeHttpClient, @@ -11,26 +12,35 @@ import { FileSystem, Layer, Option, + Redacted, Schema, } from "effect"; import { Command, Flag } from "effect/cli"; +import { HttpClient, HttpClientRequest } from "effect/http"; import { compareFootprint } from "./compare/footprint.ts"; import { compareHygiene } from "./compare/hygiene.ts"; +import { compareNative } from "./compare/native.ts"; import { REPO_ROOT } from "./config.ts"; +import { run } from "./exec.ts"; import { inspectEntrypoints } from "./footprint/inspect.ts"; import { pack, readPackManifest } from "./footprint/pack.ts"; import * as Registry from "./footprint/registry.ts"; import { resolveSide } from "./footprint/resolve.ts"; import * as GitHubApi from "./github/api.ts"; -import { fetchBaseline } from "./github/baseline.ts"; +import { fetchBaseline, fetchNearestBaselines } from "./github/baseline.ts"; import { upsertComment } from "./github/comment.ts"; import { readTiming, renderTiming } from "./github/timing.ts"; import { collectHygiene } from "./hygiene/collect.ts"; +import { measureNative } from "./native/measure.ts"; import { renderFootprint } from "./render/footprint.ts"; import { renderHygiene } from "./render/hygiene.ts"; +import { renderNative } from "./render/native.ts"; import { JobResults, renderCheckSection } from "./render/status.ts"; +import { renderTraceHelp } from "./render/trace.ts"; import { FootprintReport, type FootprintSide } from "./schema/footprint.ts"; import { HygieneReport } from "./schema/hygiene.ts"; +import { NativeReport } from "./schema/native.ts"; +import { commitTrace } from "./trace/commit.ts"; export class ChecksFailed extends Schema.TaggedError()( "ChecksFailed", @@ -173,6 +183,32 @@ const hygieneCommand = Command.make( ), ); +const nativeCommand = Command.make( + "native", + { + target: Flag.String("target").pipe( + Flag.withDescription("agentos-native-sidecar"), + ), + platform: Flag.String("platform"), + artifact: Flag.String("artifact").pipe( + Flag.withDescription("Path to the built binary"), + ), + out: Flag.String("out"), + }, + Effect.fn("Cli.native")(function* ({ target, platform, artifact, out }) { + const fs = yield* FileSystem.FileSystem; + const report = yield* measureNative(target, platform, artifact); + yield* fs.writeFileString(out, JSON.stringify(report)); + yield* Console.log( + `${target} ${platform}: ${report.artifact.bytes} bytes, ${Object.keys(report.artifact.crates).length} crates`, + ); + }, Effect.provide(Registry.layer)), +).pipe( + Command.withDescription( + "Measure one published binary built by the publish Dockerfiles", + ), +); + const baselineCommand = Command.make( "baseline", { @@ -201,83 +237,207 @@ const reportCommand = Command.make( reports: Flag.String("reports").pipe( Flag.withDescription("Directory holding the checks' JSON reports"), ), + baseHistory: Flag.String("base-history").pipe( + Flag.withDefault(""), + Flag.withDescription( + "Comma-separated merge base ancestry, newest first, for native baselines", + ), + ), }, - Effect.fn("Cli.report")(function* ({ reports }) { - const fs = yield* FileSystem.FileSystem; - const github = yield* GitHubApi.GitHub; - const env = yield* Config.all({ - // Empty for pushes and fork PRs, which get the run summary instead of a comment. - prNumber: Config.String("PR_NUMBER").pipe(Config.withDefault("")), - headSha: Config.String("HEAD_SHA"), - summary: Config.option(Config.String("GITHUB_STEP_SUMMARY")), - jobResults: Config.String("JOB_RESULTS").pipe(Config.withDefault("{}")), - }); - const results = yield* Schema.decodeUnknownEffect( - Schema.fromJsonString(JobResults), - )(env.jobResults); - const section = ( - heading: string, - job: string, - skipReason: string, - rendered: string | undefined, - ) => - renderCheckSection( - { heading, job, skipReason, rendered }, - results, - github.runUrl, + Effect.fn("Cli.report")( + function* ({ reports, baseHistory }) { + const fs = yield* FileSystem.FileSystem; + const currentRun = yield* GitHubApi.CurrentRun; + const env = yield* Config.all({ + // Empty for pushes and fork PRs, which get the run summary instead of a comment. + prNumber: Config.String("PR_NUMBER").pipe(Config.withDefault("")), + headSha: Config.String("HEAD_SHA"), + summary: Config.option(Config.String("GITHUB_STEP_SUMMARY")), + jobResults: Config.String("JOB_RESULTS").pipe(Config.withDefault("{}")), + }); + const results = yield* Schema.decodeUnknownEffect( + Schema.fromJsonString(JobResults), + )(env.jobResults); + const section = ( + heading: string, + job: string, + skipReason: string, + rendered: string | undefined, + ) => + renderCheckSection( + { heading, job, skipReason, rendered }, + results, + currentRun.url, + ); + + const sections: Array = [ + `## 📟 Pip-Boy · \`${env.headSha.slice(0, 7)}\``, + ]; + const footprintPath = join(reports, "footprint.json"); + sections.push( + section( + "### 📦 Packages", + "footprint", + "this PR doesn't change any published package.", + (yield* fs.exists(footprintPath)) + ? renderFootprint( + compareFootprint( + yield* readReport(footprintPath, FootprintReport), + ), + ) + : undefined, + ), + ); + const hygienePath = join(reports, "hygiene.json"); + sections.push( + section( + "### 🧹 Rust dependencies", + "hygiene", + "this PR doesn't change any Rust code or dependencies.", + (yield* fs.exists(hygienePath)) + ? renderHygiene( + compareHygiene(yield* readReport(hygienePath, HygieneReport)), + ) + : undefined, + ), ); - const sections: Array = [ - `## 📟 Pip-Boy · \`${env.headSha.slice(0, 7)}\``, - ]; - const footprintPath = join(reports, "footprint.json"); - sections.push( - section( - "### 📦 Packages", - "footprint", - "this PR doesn't change any published package.", - (yield* fs.exists(footprintPath)) - ? renderFootprint( - compareFootprint( - yield* readReport(footprintPath, FootprintReport), + // No report artifacts at all (every check skipped) means no reports directory. + const reportFiles = (yield* fs.exists(reports)) + ? yield* fs.readDirectory(reports) + : []; + const nativeFiles = reportFiles + .filter((f) => /^native-.+\.json$/.test(f)) + .sort(); + let nativeRendered: string | undefined; + if (nativeFiles.length > 0) { + const history = baseHistory + .split(",") + .filter((sha) => /^[0-9a-f]{40}$/.test(sha)); + const baseDir = yield* fs.makeTempDirectoryScoped({ + prefix: "pipboy-native-base-", + }); + const pr = yield* Effect.forEach(nativeFiles, (file) => + readReport(join(reports, file), NativeReport), + ); + const baseline = yield* fetchNearestBaselines( + history, + nativeFiles.map((file) => `report-${file.replace(/\.json$/, "")}`), + baseDir, + ); + const base = Option.isSome(baseline) + ? yield* Effect.forEach(baseline.value.found, (artifact) => + readReport( + join(baseDir, `${artifact.replace(/^report-/, "")}.json`), + NativeReport, ), ) - : undefined, - ), - ); - const hygienePath = join(reports, "hygiene.json"); - sections.push( - section( - "### 🧹 Rust dependencies", - "hygiene", - "this PR doesn't change any Rust code or dependencies.", - (yield* fs.exists(hygienePath)) - ? renderHygiene( - compareHygiene(yield* readReport(hygienePath, HygieneReport)), - ) - : undefined, + : []; + nativeRendered = renderNative( + compareNative(pr, base), + Option.map(baseline, ({ sha }) => ({ + sha, + isMergeBase: sha === history[0], + })), + ); + } + sections.push( + section( + "### 🦀 Binaries", + "native", + "this PR doesn't change any Rust code.", + nativeRendered, + ), + ); + + const timing = yield* readTiming(); + sections.push(renderTiming(timing), renderTraceHelp(env.headSha)); + const body = sections.join("\n\n"); + + if (Option.isSome(env.summary)) { + yield* fs.writeFileString(env.summary.value, `${body}\n`, { + flag: "a", + }); + } + if (env.prNumber !== "") { + const result = yield* upsertComment( + Number(env.prNumber), + env.headSha, + fitComment(body, currentRun.url), + ); + yield* Console.log(`comment ${result}`); + } + }, + Effect.scoped, + Effect.provide(GitHubApi.layer), + ), +).pipe( + Command.withDescription( + "Write the PR comment and run summary from the checks' reports", + ), +); + +export class InvalidSha extends Schema.TaggedError()("InvalidSha", { + sha: Schema.String, +}) { + override get message() { + return `--sha must be a full 40-character commit SHA, got \`${this.sha}\``; + } +} + +const traceCommand = Command.make( + "trace", + { + sha: Flag.String("sha").pipe( + Flag.withDescription("Commit to trace (full SHA)"), + ), + repo: Flag.String("repo").pipe(Flag.withDefault("rivet-dev/agentos")), + out: Flag.String("out").pipe( + Flag.optional, + Flag.withDescription("Write the OTLP/JSON trace to this file"), + ), + otlp: Flag.String("otlp").pipe( + Flag.optional, + Flag.withDescription( + "Send the trace to this OTLP/HTTP endpoint, such as motel at http://127.0.0.1:27686", ), + ), + }, + Effect.fn("Cli.trace")(function* ({ sha, repo, out, otlp }) { + if (!/^[0-9a-f]{40}$/.test(sha)) return yield* new InvalidSha({ sha }); + const fs = yield* FileSystem.FileSystem; + // Locally, fall back to the GitHub CLI's login so no token needs exporting. + const token = + Option.getOrUndefined( + yield* Config.option(Config.Redacted("GITHUB_TOKEN")), + ) ?? Redacted.make((yield* run("gh", ["auth", "token"])).trim()); + const { runs, document } = yield* Effect.scoped(commitTrace(sha)).pipe( + Effect.provide(GitHubApi.layerFor(repo, token)), ); + const spans = document.resourceSpans[0]?.scopeSpans[0]?.spans.length ?? 0; - const timing = yield* readTiming(); - sections.push(renderTiming(timing)); - const body = sections.join("\n\n"); - - if (Option.isSome(env.summary)) { - yield* fs.writeFileString(env.summary.value, `${body}\n`, { flag: "a" }); - } - if (env.prNumber !== "") { - const result = yield* upsertComment( - Number(env.prNumber), - env.headSha, - body, + // The default stays out of the checkout, where version control would pick it up. + const path = Option.getOrElse(out, () => + join(tmpdir(), `pipboy-trace-${sha.slice(0, 7)}.json`), + ); + yield* fs.writeFileString(path, JSON.stringify(document)); + yield* Console.log(`${runs} workflow runs, ${spans} spans → ${path}`); + if (Option.isSome(otlp)) { + const http = (yield* HttpClient.HttpClient).pipe( + HttpClient.filterStatusOk, + ); + yield* HttpClientRequest.post( + `${otlp.value.replace(/\/$/, "")}/v1/traces`, + ).pipe( + HttpClientRequest.bodyJson(document), + Effect.flatMap(http.execute), ); - yield* Console.log(`comment ${result}`); + yield* Console.log(`sent to ${otlp.value}`); } - }, Effect.provide(GitHubApi.layer)), + }), ).pipe( Command.withDescription( - "Write the PR comment and run summary from the checks' reports", + "Build one trace of the CI that ran on a commit: CI, Pip-Boy, and releases", ), ); @@ -306,6 +466,13 @@ const readReport = Effect.fn("Cli.readReport")(function* < ); }); +/** GitHub rejects comments over 65,536 characters; the run summary always has the full report. */ +const MAX_COMMENT = 60_000; +function fitComment(body: string, runUrl: string): string { + if (body.length <= MAX_COMMENT) return body; + return `${body.slice(0, MAX_COMMENT)}\n\n…truncated. The [run summary](${runUrl}) has the full report.`; +} + /** Appends `name=value` to the step's outputs. A no-op outside GitHub Actions. */ const setOutput = Effect.fn("Cli.setOutput")(function* ( name: string, @@ -322,8 +489,10 @@ const cli = Command.make("pipboy").pipe( packCommand, footprintCommand, hygieneCommand, + nativeCommand, baselineCommand, reportCommand, + traceCommand, ]), ); diff --git a/scripts/pipboy/src/compare/native.ts b/scripts/pipboy/src/compare/native.ts new file mode 100644 index 0000000000..8f46770801 --- /dev/null +++ b/scripts/pipboy/src/compare/native.ts @@ -0,0 +1,60 @@ +import { THRESHOLDS } from "../config.ts"; +import type { NativeReport } from "../schema/native.ts"; +import { level, type Severity } from "./footprint.ts"; + +export interface NativeRow { + readonly target: string; + readonly platform: string; + readonly base?: number; + readonly pr: number; + readonly latest?: number; + readonly severity: Severity; + /** Crates whose symbol bytes changed most, largest change first. */ + readonly crates: ReadonlyArray<{ + readonly name: string; + readonly base: number; + readonly pr: number; + }>; +} + +/** Pairs each PR artifact with its baseline from main, if one was found. */ +export function compareNative( + pr: ReadonlyArray, + base: ReadonlyArray, +): ReadonlyArray { + return pr + .map((report) => { + const a = report.artifact; + const b = base.find( + (r) => + r.artifact.target === a.target && r.artifact.platform === a.platform, + )?.artifact; + const names = new Set([ + ...Object.keys(a.crates), + ...Object.keys(b?.crates ?? {}), + ]); + const crates = [...names] + .map((name) => ({ + name, + base: b?.crates[name] ?? 0, + pr: a.crates[name] ?? 0, + })) + .filter((c) => c.base !== c.pr) + .sort((x, y) => Math.abs(y.pr - y.base) - Math.abs(x.pr - x.base)) + .slice(0, 10); + return { + target: a.target, + platform: a.platform, + ...(b ? { base: b.bytes } : {}), + pr: a.bytes, + ...(report.latestBytes !== undefined + ? { latest: report.latestBytes } + : {}), + severity: level(b?.bytes, a.bytes, THRESHOLDS.binary), + crates: b ? crates : [], + }; + }) + .sort((x, y) => + `${x.target}${x.platform}`.localeCompare(`${y.target}${y.platform}`), + ); +} diff --git a/scripts/pipboy/src/config.ts b/scripts/pipboy/src/config.ts index 136ac6d326..9aa72c37a1 100644 --- a/scripts/pipboy/src/config.ts +++ b/scripts/pipboy/src/config.ts @@ -85,6 +85,10 @@ export const THRESHOLDS = { warn: { pct: 5, abs: 5 * 1024 }, alert: { pct: 20, abs: 25 * 1024 }, }, + binary: { + warn: { pct: 3, abs: 1024 ** 2 }, + alert: { pct: 10, abs: 5 * 1024 ** 2 }, + }, } satisfies Record; /** Synthetic version for packed workspace packages, so npm can never pick a registry copy instead. */ diff --git a/scripts/pipboy/src/github/api.ts b/scripts/pipboy/src/github/api.ts index 7d33fdf080..a1aa59ff93 100644 --- a/scripts/pipboy/src/github/api.ts +++ b/scripts/pipboy/src/github/api.ts @@ -7,25 +7,8 @@ import { HttpClientResponse, } from "effect/http"; -/** GitHub Actions context, read from the standard runner environment. */ -export const ActionsEnv = Config.all({ - token: Config.Redacted("GITHUB_TOKEN"), - repository: Config.String("GITHUB_REPOSITORY"), - runId: Config.String("GITHUB_RUN_ID"), - runAttempt: Config.String("GITHUB_RUN_ATTEMPT").pipe(Config.withDefault("1")), - apiUrl: Config.String("GITHUB_API_URL").pipe( - Config.withDefault("https://api.github.com"), - ), - serverUrl: Config.String("GITHUB_SERVER_URL").pipe( - Config.withDefault("https://github.com"), - ), -}); - export interface Interface { readonly repository: string; - readonly runId: string; - readonly runAttempt: string; - readonly runUrl: string; readonly get: ( path: string, schema: S, @@ -48,39 +31,67 @@ export class GitHub extends Context.Service()( "pipboy/GitHub", ) {} -export const layer = Layer.effect( - GitHub, - Effect.gen(function* () { - const env = yield* ActionsEnv; - const http = (yield* HttpClient.HttpClient).pipe( - HttpClient.mapRequest((request) => - request.pipe( - HttpClientRequest.prependUrl(env.apiUrl), - HttpClientRequest.bearerToken(Redacted.value(env.token)), - HttpClientRequest.setHeader("accept", "application/vnd.github+json"), - HttpClientRequest.setHeader("x-github-api-version", "2022-11-28"), - // GitHub rejects API requests without a User-Agent. - HttpClientRequest.setHeader("user-agent", "rivet-pipboy"), - ), - ), - HttpClient.filterStatusOk, - HttpClient.retryTransient({ times: 4 }), - ); - return GitHub.of({ - repository: env.repository, - runId: env.runId, - runAttempt: env.runAttempt, - runUrl: `${env.serverUrl}/${env.repository}/actions/runs/${env.runId}`, - get: (path, schema) => - http - .get(path) - .pipe(Effect.flatMap(HttpClientResponse.schemaBodyJson(schema))), - send: (method, path, body) => - HttpClientRequest.make(method)(path).pipe( - HttpClientRequest.bodyJson(body), - Effect.flatMap(http.execute), - Effect.asVoid, +/** The current GitHub Actions run, read from the standard runner environment. */ +export const CurrentRun = Config.all({ + repository: Config.String("GITHUB_REPOSITORY"), + runId: Config.String("GITHUB_RUN_ID"), + runAttempt: Config.String("GITHUB_RUN_ATTEMPT").pipe(Config.withDefault("1")), + serverUrl: Config.String("GITHUB_SERVER_URL").pipe( + Config.withDefault("https://github.com"), + ), +}).pipe( + Config.map((run) => ({ + ...run, + url: `${run.serverUrl}/${run.repository}/actions/runs/${run.runId}`, + })), +); + +/** A GitHub API client for `repository`, authenticated with `token` or `GITHUB_TOKEN`. */ +export const layerFor = ( + repository: string, + explicitToken?: Redacted.Redacted, +) => + Layer.effect( + GitHub, + Effect.gen(function* () { + const token = explicitToken ?? (yield* Config.Redacted("GITHUB_TOKEN")); + const apiUrl = yield* Config.String("GITHUB_API_URL").pipe( + Config.withDefault("https://api.github.com"), + ); + const http = (yield* HttpClient.HttpClient).pipe( + HttpClient.mapRequest((request) => + request.pipe( + HttpClientRequest.prependUrl(apiUrl), + HttpClientRequest.bearerToken(Redacted.value(token)), + HttpClientRequest.setHeader( + "accept", + "application/vnd.github+json", + ), + HttpClientRequest.setHeader("x-github-api-version", "2022-11-28"), + // GitHub rejects API requests without a User-Agent. + HttpClientRequest.setHeader("user-agent", "rivet-pipboy"), + ), ), - }); - }), + HttpClient.filterStatusOk, + HttpClient.retryTransient({ times: 4 }), + ); + return GitHub.of({ + repository, + get: (path, schema) => + http + .get(path) + .pipe(Effect.flatMap(HttpClientResponse.schemaBodyJson(schema))), + send: (method, path, body) => + HttpClientRequest.make(method)(path).pipe( + HttpClientRequest.bodyJson(body), + Effect.flatMap(http.execute), + Effect.asVoid, + ), + }); + }), + ); + +/** A client for the repository of the current Actions run. */ +export const layer = Layer.unwrap( + Config.String("GITHUB_REPOSITORY").pipe(Effect.map(layerFor)), ); diff --git a/scripts/pipboy/src/github/baseline.ts b/scripts/pipboy/src/github/baseline.ts index 01e7864339..4fae49324f 100644 --- a/scripts/pipboy/src/github/baseline.ts +++ b/scripts/pipboy/src/github/baseline.ts @@ -23,55 +23,121 @@ const Artifacts = Schema.Struct({ ), }); +type Run = Schema.Schema.Type["workflow_runs"][number]; + /** - * Downloads `artifact` from this workflow's `main` push run for `sha`. Only - * pushes to `main` in this repository qualify, so a PR run can never supply - * a baseline. Returns none when no such run or artifact exists. + * This workflow's completed push runs on `main` in this repository, newest + * first. Nothing else is trusted as a baseline: a PR run can never supply one. + * Empty until this workflow exists on main, when the API answers 404. */ +const listMainRuns = Effect.fn("Baseline.listMainRuns")(function* ( + filter: string, + pages: number, +) { + const github = yield* GitHub; + const runs: Array = []; + for (let page = 1; page <= pages; page++) { + const { workflow_runs } = yield* github + .get( + `/repos/${github.repository}/actions/workflows/pipboy.yml/runs?event=push&branch=main&status=completed${filter}&per_page=100&page=${page}`, + Runs, + ) + .pipe( + Effect.catchIf( + (error) => + error._tag === "HttpClientError" && error.response?.status === 404, + () => Effect.succeed({ workflow_runs: [] }), + ), + ); + runs.push( + ...workflow_runs.filter( + (r) => + r.event === "push" && + r.head_branch === "main" && + r.path === WORKFLOW_PATH && + r.head_repository.full_name === github.repository, + ), + ); + if (workflow_runs.length < 100) break; + } + return runs; +}); + +const artifactNames = Effect.fn("Baseline.artifactNames")(function* ( + runId: number, +) { + const github = yield* GitHub; + const { artifacts } = yield* github.get( + `/repos/${github.repository}/actions/runs/${runId}/artifacts?per_page=100`, + Artifacts, + ); + return new Set(artifacts.filter((a) => !a.expired).map((a) => a.name)); +}); + +const download = Effect.fn("Baseline.download")(function* ( + runId: number, + artifact: string, + outDir: string, +) { + const github = yield* GitHub; + yield* run("gh", [ + "run", + "download", + String(runId), + "--repo", + github.repository, + "--name", + artifact, + "--dir", + outDir, + ]); +}); + +/** Downloads `artifact` from main's run for exactly `sha`. Returns the run id, or none. */ export const fetchBaseline = Effect.fn("Baseline.fetch")(function* ( sha: string, artifact: string, outDir: string, ) { - const github = yield* GitHub; - // 404 until this workflow exists on main, so the first rollout has no baseline. - const runs = yield* github - .get( - `/repos/${github.repository}/actions/workflows/pipboy.yml/runs?event=push&branch=main&head_sha=${sha}&per_page=20`, - Runs, - ) - .pipe( - Effect.catchIf( - (error) => - error._tag === "HttpClientError" && error.response?.status === 404, - () => Effect.succeed({ workflow_runs: [] }), - ), - ); - for (const run_ of runs.workflow_runs) { - const trusted = - run_.event === "push" && - run_.head_branch === "main" && - run_.head_sha === sha && - run_.path === WORKFLOW_PATH && - run_.head_repository.full_name === github.repository; - if (!trusted) continue; - const { artifacts } = yield* github.get( - `/repos/${github.repository}/actions/runs/${run_.id}/artifacts?name=${encodeURIComponent(artifact)}`, - Artifacts, - ); - if (!artifacts.some((a) => a.name === artifact && !a.expired)) continue; - yield* run("gh", [ - "run", - "download", - String(run_.id), - "--repo", - github.repository, - "--name", - artifact, - "--dir", - outDir, - ]); - return Option.some(run_.id); + for (const r of (yield* listMainRuns(`&head_sha=${sha}`, 1)).filter( + (r) => r.head_sha === sha, + )) { + if (!(yield* artifactNames(r.id)).has(artifact)) continue; + yield* download(r.id, artifact, outDir); + return Option.some(r.id); } return Option.none(); }); + +/** + * Downloads `artifacts` from the newest commit in `history` (newest first) + * whose main run produced any of them. Main produces some artifacts only for + * some commits, such as native builds only when Rust changed, so the result + * names the commit the baseline came from. + */ +export const fetchNearestBaselines = Effect.fn("Baseline.fetchNearest")( + function* ( + history: ReadonlyArray, + artifacts: ReadonlyArray, + outDir: string, + ) { + if (history.length === 0 || artifacts.length === 0) + return Option.none<{ sha: string; found: ReadonlyArray }>(); + const bySha = new Map>(); + for (const r of yield* listMainRuns("", 3)) + bySha.set(r.head_sha, [...(bySha.get(r.head_sha) ?? []), r]); + for (const sha of history) { + for (const r of bySha.get(sha) ?? []) { + const available = yield* artifactNames(r.id); + const found = artifacts.filter((a) => available.has(a)); + if (found.length === 0) continue; + yield* Effect.forEach(found, (a) => download(r.id, a, outDir), { + concurrency: 4, + discard: true, + }); + return Option.some({ sha, found }); + } + } + return Option.none<{ sha: string; found: ReadonlyArray }>(); + }, +); diff --git a/scripts/pipboy/src/github/timing.ts b/scripts/pipboy/src/github/timing.ts index 07f53670ff..68363bf803 100644 --- a/scripts/pipboy/src/github/timing.ts +++ b/scripts/pipboy/src/github/timing.ts @@ -1,6 +1,7 @@ import { Effect, Schema } from "effect"; import { code } from "../render/format.ts"; -import { GitHub } from "./api.ts"; +import { stepLabel, toMillis } from "../trace/spans.ts"; +import { CurrentRun, GitHub } from "./api.ts"; const Step = Schema.Struct({ name: Schema.String, @@ -14,6 +15,7 @@ const Job = Schema.Struct({ id: Schema.Number, name: Schema.String, conclusion: Schema.NullOr(Schema.String), + created_at: Schema.optionalKey(Schema.String), started_at: Schema.NullOr(Schema.String), completed_at: Schema.NullOr(Schema.String), html_url: Schema.optionalKey(Schema.String), @@ -29,35 +31,33 @@ export interface Timing { readonly now: number; } -/** Job and step timings of the current run, including still-running jobs. */ -export const readTiming = Effect.fn("Timing.read")(function* () { +/** Every job of one run attempt, including still-running jobs. */ +export const readJobs = Effect.fn("Timing.readJobs")(function* ( + runId: number | string, + attempt: number | string, +) { const github = yield* GitHub; - const { jobs } = yield* github.get( - `/repos/${github.repository}/actions/runs/${github.runId}/attempts/${github.runAttempt}/jobs?per_page=100`, - Jobs, - ); + const jobs: Array = []; + for (let page = 1; ; page++) { + const batch = yield* github.get( + `/repos/${github.repository}/actions/runs/${runId}/attempts/${attempt}/jobs?per_page=100&page=${page}`, + Jobs, + ); + jobs.push(...batch.jobs); + if (batch.jobs.length < 100) return jobs; + } +}); + +/** Job and step timings of the current run. */ +export const readTiming = Effect.fn("Timing.read")(function* () { + const run = yield* CurrentRun; + const jobs = yield* readJobs(run.runId, run.runAttempt); return { jobs: jobs.filter((job) => job.started_at !== null), now: Date.now(), } satisfies Timing; }); -/** - * Short step names: `Run actions/checkout@` becomes `checkout`. Rendered - * as code, so GitHub never turns `owner/repo@sha` into a commit link or `@name` - * into a mention. - */ -export function stepLabel(name: string): string { - const action = name.match( - /^(Post )?Run [\w.-]+\/([\w.-]+)(?:\/[\w./-]+)?@\S+$/, - ); - if (action) return `${action[1] ? "post " : ""}${action[2]}`; - return name.split("\n")[0] ?? name; -} - -const ms = (iso: string | null, fallback: number) => - iso === null ? fallback : Date.parse(iso); - function duration(totalMs: number): string { const s = Math.max(0, Math.round(totalMs / 1000)); return s >= 60 @@ -74,11 +74,13 @@ export function renderTiming(timing: Timing): string { "|---|---:|---|", ]; for (const job of timing.jobs) { - const start = ms(job.started_at, timing.now); + const start = toMillis(job.started_at, timing.now); const slowest = [...(job.steps ?? [])] .map((s) => ({ name: s.name, - ms: ms(s.completed_at, timing.now) - ms(s.started_at, timing.now), + ms: + toMillis(s.completed_at, timing.now) - + toMillis(s.started_at, timing.now), })) .sort((a, b) => b.ms - a.ms)[0]; const name = @@ -96,7 +98,7 @@ export function renderTiming(timing: Timing): string { ? "⊘ " : ""; lines.push( - `| ${icon}${name} | ${duration(ms(job.completed_at, timing.now) - start)} | ${slowest ? `${code(stepLabel(slowest.name))} · ${duration(slowest.ms)}` : "—"} |`, + `| ${icon}${name} | ${duration(toMillis(job.completed_at, timing.now) - start)} | ${slowest ? `${code(stepLabel(slowest.name))} · ${duration(slowest.ms)}` : "—"} |`, ); } return lines.join("\n"); diff --git a/scripts/pipboy/src/native/measure.ts b/scripts/pipboy/src/native/measure.ts new file mode 100644 index 0000000000..ce4b186f42 --- /dev/null +++ b/scripts/pipboy/src/native/measure.ts @@ -0,0 +1,65 @@ +import { Effect, FileSystem, Option } from "effect"; +import { capture } from "../exec.ts"; +import { Registry } from "../footprint/registry.ts"; +import type { NativeReport } from "../schema/native.ts"; + +/** The npm package that ships each build artifact. */ +const NPM_PACKAGE: Record string> = { + "agentos-native-sidecar": (platform) => `@rivet-dev/agentos-runtime-sidecar-${platform}`, +}; + +/** + * Measures one published binary: its size, and symbol bytes per crate. Crates + * come from demangled symbol names (`crate::module::item`), so the split is + * approximate; symbols without a Rust path count as `[c/other]`. + */ +export const measureNative = Effect.fn("Native.measure")(function* ( + target: string, + platform: string, + path: string, +) { + const fs = yield* FileSystem.FileSystem; + const bytes = Number((yield* fs.stat(path)).size); + const crates = yield* crateSizes(path); + + const latestBytes = yield* latestPublishedSize( + NPM_PACKAGE[target]?.(platform), + ); + return { + schema: 1, + check: "native", + artifact: { target, platform, bytes, crates }, + ...(latestBytes !== undefined ? { latestBytes } : {}), + } satisfies NativeReport; +}); + +/** Unpacked size of the npm package's `latest` version, if it's published. */ +const latestPublishedSize = Effect.fn("Native.latestPublishedSize")(function* ( + name: string | undefined, +) { + if (name === undefined) return undefined; + const registry = yield* Registry; + const version = yield* registry.latestVersion(name); + return Option.isSome(version) + ? yield* registry.packageSize(name, version.value) + : undefined; +}); + +// Stripped binaries, and Mach-O or PE binaries the runner's GNU nm can't read, report no crates. +const crateSizes = (path: string) => + capture("nm", ["--demangle", "--print-size", "--size-sort", path]).pipe( + Effect.map(({ exitCode, stdout }) => { + const crates: Record = {}; + if (exitCode !== 0) return crates; + for (const line of stdout.split("\n")) { + // `
`, sizes in hex. + const match = line.match(/^[0-9a-f]+ ([0-9a-f]+) \S (.+)$/i); + if (!match) continue; + const crate = + match[2]?.match(/([A-Za-z_][A-Za-z0-9_]*)::/)?.[1] ?? "[c/other]"; + crates[crate] = + (crates[crate] ?? 0) + Number.parseInt(match[1] ?? "0", 16); + } + return crates; + }), + ); diff --git a/scripts/pipboy/src/render/native.ts b/scripts/pipboy/src/render/native.ts new file mode 100644 index 0000000000..8c2dc48b06 --- /dev/null +++ b/scripts/pipboy/src/render/native.ts @@ -0,0 +1,51 @@ +import { Option } from "effect"; +import type { NativeRow } from "../compare/native.ts"; +import { bytes, change, code, ICON } from "./format.ts"; + +/** `baseline` is the main commit the base sizes came from, if any. */ +export function renderNative( + rows: ReadonlyArray, + baseline: Option.Option<{ + readonly sha: string; + readonly isMergeBase: boolean; + }>, +): string { + const source = Option.match(baseline, { + onNone: () => "No native build on main to compare against yet.", + onSome: ({ sha, isMergeBase }) => + isMergeBase + ? `Base: \`main@${sha.slice(0, 7)}\`, the merge base.` + : `Base: \`main@${sha.slice(0, 7)}\`, the newest main commit with native builds before the merge base. Size changes merged to main since then show up here too.`, + }); + const out = [ + "### 🦀 Binaries · publish Dockerfiles, `--release`", + "", + source, + "", + "| artifact | npm `@latest` | base | this PR | Δ vs base |", + "|---|---:|---:|---:|---|", + ]; + for (const row of rows) { + out.push( + `| ${ICON[row.severity]}${code(`${row.target} ${row.platform}`)} | ${row.latest === undefined ? "—" : bytes(row.latest)} | ${row.base === undefined ? "—" : bytes(row.base)} | ${bytes(row.pr)} | ${row.base === undefined ? "—" : change(row.base, row.pr)} |`, + ); + } + for (const row of rows.filter( + (r) => r.crates.length > 0 && r.severity !== "ok", + )) { + out.push( + "", + `
${code(`${row.target} ${row.platform}`)} · crates that changed most (from symbol sizes, approximate)`, + "", + "| crate | base | this PR | Δ |", + "|---|---:|---:|---|", + ...row.crates.map( + (c) => + `| ${code(c.name)} | ${bytes(c.base)} | ${bytes(c.pr)} | ${change(c.base, c.pr)} |`, + ), + "", + "
", + ); + } + return out.join("\n"); +} diff --git a/scripts/pipboy/src/render/trace.ts b/scripts/pipboy/src/render/trace.ts new file mode 100644 index 0000000000..e6e5ba3bfc --- /dev/null +++ b/scripts/pipboy/src/render/trace.ts @@ -0,0 +1,19 @@ +/** Collapsed comment section: how a maintainer or agent pulls the full CI trace for `sha`. */ +export function renderTraceHelp(sha: string): string { + return [ + "
🔭 Full CI trace for this commit", + "", + `One OpenTelemetry trace of CI and Pip-Boy for \`${sha.slice(0, 7)}\`: every job and step, with the sizes above attached. Run after CI finishes.`, + "", + "```bash", + "pnpm install --frozen-lockfile --filter 'pipboy...'", + `pnpm --filter pipboy pipboy trace --sha ${sha}`, + "```", + "", + "Add `--otlp ` to send it to any OTLP/HTTP collector.", + "", + "**Agent instruction:** From a `rivet-dev/agentos` checkout, run the commands above, then read the trace file the last command prints. List the slowest jobs and steps and explain where CI time went.", + "", + "
", + ].join("\n"); +} diff --git a/scripts/pipboy/src/schema/native.ts b/scripts/pipboy/src/schema/native.ts new file mode 100644 index 0000000000..f25d4fa76b --- /dev/null +++ b/scripts/pipboy/src/schema/native.ts @@ -0,0 +1,22 @@ +import { Schema } from "effect"; + +export const NativeArtifact = Schema.Struct({ + /** `agentos-native-sidecar`. */ + target: Schema.String, + platform: Schema.String, + /** Size of the file we publish. */ + bytes: Schema.Number, + /** Symbol bytes by crate, from the binary's symbol table. Empty when the binary has no symbols. */ + crates: Schema.Record(Schema.String, Schema.Number), +}); +export interface NativeArtifact + extends Schema.Schema.Type {} + +export const NativeReport = Schema.Struct({ + schema: Schema.Literal(1), + check: Schema.Literal("native"), + artifact: NativeArtifact, + /** Unpacked size of the published npm platform package, for reference. */ + latestBytes: Schema.optionalKey(Schema.Number), +}); +export interface NativeReport extends Schema.Schema.Type {} diff --git a/scripts/pipboy/src/trace/commit.ts b/scripts/pipboy/src/trace/commit.ts new file mode 100644 index 0000000000..24b8c2c3d4 --- /dev/null +++ b/scripts/pipboy/src/trace/commit.ts @@ -0,0 +1,248 @@ +import { join } from "node:path"; +import { Effect, FileSystem, Schema } from "effect"; +import { compareFootprint } from "../compare/footprint.ts"; +import { compareHygiene } from "../compare/hygiene.ts"; +import { run } from "../exec.ts"; +import { GitHub } from "../github/api.ts"; +import { WORKFLOW_PATH } from "../github/baseline.ts"; +import { readJobs } from "../github/timing.ts"; +import { FootprintReport } from "../schema/footprint.ts"; +import { HygieneReport } from "../schema/hygiene.ts"; +import { NativeReport } from "../schema/native.ts"; +import { + type Attribute, + attr, + jobSpans, + otlpDocument, + runSpanId, + span, + toMillis, + traceIdFor, +} from "./spans.ts"; + +const WorkflowRun = Schema.Struct({ + id: Schema.Number, + name: Schema.NullOr(Schema.String), + path: Schema.String, + event: Schema.String, + status: Schema.NullOr(Schema.String), + conclusion: Schema.NullOr(Schema.String), + run_attempt: Schema.Number, + created_at: Schema.String, + run_started_at: Schema.optionalKey(Schema.NullOr(Schema.String)), + updated_at: Schema.String, + html_url: Schema.String, +}); +type WorkflowRun = Schema.Schema.Type; + +const Runs = Schema.Struct({ + total_count: Schema.Number, + workflow_runs: Schema.Array(WorkflowRun), +}); +const Artifacts = Schema.Struct({ + artifacts: Schema.Array( + Schema.Struct({ name: Schema.String, expired: Schema.Boolean }), + ), +}); + +/** + * The workflows that build and test the code, by file. Automation such as docs + * sync is left out of the trace. + */ +const TRACED_WORKFLOWS = new Set([ + ".github/workflows/ci.yml", + WORKFLOW_PATH, + ".github/workflows/publish.yaml", + ".github/workflows/bench.yml", +]); + +/** Every workflow run GitHub started for `sha`, across all workflows and events. */ +const listRuns = Effect.fn("CommitTrace.listRuns")(function* (sha: string) { + const github = yield* GitHub; + const runs: Array = []; + for (let page = 1; ; page++) { + const batch = yield* github.get( + `/repos/${github.repository}/actions/runs?head_sha=${sha}&per_page=100&page=${page}`, + Runs, + ); + runs.push(...batch.workflow_runs); + if (batch.workflow_runs.length < 100 || runs.length >= batch.total_count) + return runs; + } +}); + +/** + * Span attributes from a Pip-Boy run's reports, keyed by job name, so a + * job's span carries the numbers it measured. + */ +const reportAttributes = Effect.fn("CommitTrace.reportAttributes")(function* ( + runId: number, +) { + const github = yield* GitHub; + const fs = yield* FileSystem.FileSystem; + const byJob = new Map>(); + const add = (job: string, attributes: ReadonlyArray) => + byJob.set(job, [...(byJob.get(job) ?? []), ...attributes]); + + const { artifacts } = yield* github.get( + `/repos/${github.repository}/actions/runs/${runId}/artifacts?per_page=100`, + Artifacts, + ); + const reports = artifacts.filter( + (a) => a.name.startsWith("report-") && !a.expired, + ); + if (reports.length === 0) return byJob; + const dir = yield* fs.makeTempDirectoryScoped({ + prefix: "pipboy-trace-", + }); + yield* run("gh", [ + "run", + "download", + String(runId), + "--repo", + github.repository, + "--dir", + dir, + ...reports.flatMap((a) => ["--name", a.name]), + ]); + + const read = (path: string, schema: S) => + fs + .readFileString(path) + .pipe( + Effect.flatMap( + Schema.decodeUnknownEffect(Schema.fromJsonString(schema)), + ), + ); + for (const artifact of reports) { + const artifactDir = reports.length === 1 ? dir : join(dir, artifact.name); + for (const file of yield* fs.readDirectory(artifactDir)) { + const path = join(artifactDir, file); + if (file === "footprint.json") { + const comparison = compareFootprint(yield* read(path, FootprintReport)); + for (const row of comparison.rows) { + const pr = row.install[0]?.pr; + if (!pr) continue; + add("footprint", [ + attr(`rivet.footprint.install_bytes.${row.name}`, pr.bytes), + attr(`rivet.footprint.packages.${row.name}`, pr.count), + ]); + } + } else if (file === "hygiene.json") { + for (const row of compareHygiene(yield* read(path, HygieneReport)) + .rows) { + add("hygiene", [ + attr(`rivet.hygiene.${row.check}.new`, row.added.length), + attr(`rivet.hygiene.${row.check}.existing`, row.existing), + ]); + } + } else if (/^native-.+\.json$/.test(file)) { + const { artifact: a } = yield* read(path, NativeReport); + add(`native · ${a.target} · ${a.platform}`, [ + attr("rivet.native.bytes", a.bytes), + ]); + } + } + } + return byJob; +}); + +export class NoRunsForCommit extends Schema.TaggedError()( + "NoRunsForCommit", + { + sha: Schema.String, + repository: Schema.String, + }, +) { + override get message() { + return `no workflow runs found for ${this.sha} in ${this.repository}`; + } +} + +/** + * One trace for the CI that ran on a commit: commit, then each traced workflow + * run, its jobs, and their steps. Pip-Boy jobs also carry the sizes and + * findings they measured. + */ +export const commitTrace = Effect.fn("CommitTrace.build")(function* ( + sha: string, +) { + const github = yield* GitHub; + const now = Date.now(); + const traceId = traceIdFor(github.repository, sha); + // A push can start the same workflow twice for one commit; the newer run cancels the older. + // Only the newest run of each traced workflow is kept. + const newest = new Map(); + for (const r of yield* listRuns(sha)) { + if (!TRACED_WORKFLOWS.has(r.path)) continue; + const kept = newest.get(r.path); + if (kept === undefined || r.id > kept.id) newest.set(r.path, r); + } + const runs = [...newest.values()]; + if (runs.length === 0) + return yield* new NoRunsForCommit({ sha, repository: github.repository }); + + const perRun = yield* Effect.forEach( + runs, + (r) => + Effect.all({ + jobs: readJobs(r.id, r.run_attempt), + attributes: + r.path === WORKFLOW_PATH + ? reportAttributes(r.id) + : Effect.succeed(new Map>()), + }).pipe(Effect.map((details) => ({ run: r, ...details }))), + { concurrency: 8 }, + ); + + const rootId = runSpanId(`commit:${sha}`); + const runStart = (r: WorkflowRun) => + toMillis(r.run_started_at, toMillis(r.created_at, now)); + const runEnd = (r: WorkflowRun) => + r.status === "completed" ? toMillis(r.updated_at, now) : now; + const spans = [ + span({ + traceId, + id: rootId, + name: `commit ${sha.slice(0, 7)}`, + startMs: Math.min(...runs.map(runStart)), + endMs: Math.max(...runs.map(runEnd)), + failed: runs.some((r) => r.conclusion === "failure"), + attributes: [ + attr("vcs.ref.head.revision", sha), + attr("vcs.repository.name", github.repository), + attr("cicd.pipeline.run.count", runs.length), + ], + }), + ]; + for (const { run: r, jobs, attributes } of perRun) { + const id = runSpanId(r.id); + spans.push( + span({ + traceId, + id, + parent: rootId, + name: r.name ?? r.path, + startMs: runStart(r), + endMs: runEnd(r), + failed: r.conclusion === "failure", + attributes: [ + attr("cicd.pipeline.name", r.name ?? r.path), + attr("cicd.pipeline.run.id", r.id), + attr("cicd.pipeline.run.url.full", r.html_url), + attr("cicd.pipeline.result", r.conclusion ?? r.status ?? "unknown"), + attr("cicd.pipeline.trigger.event", r.event), + attr("cicd.pipeline.run.attempt", r.run_attempt), + ], + }), + ...jobSpans( + traceId, + id, + jobs, + now, + (job) => attributes.get(job.name) ?? [], + ), + ); + } + return { runs: runs.length, document: otlpDocument(spans) }; +}); diff --git a/scripts/pipboy/src/trace/spans.ts b/scripts/pipboy/src/trace/spans.ts new file mode 100644 index 0000000000..b57ee5bf12 --- /dev/null +++ b/scripts/pipboy/src/trace/spans.ts @@ -0,0 +1,163 @@ +import { createHash } from "node:crypto"; + +export interface Step { + readonly name: string; + readonly number: number; + readonly conclusion: string | null; + readonly started_at: string | null; + readonly completed_at: string | null; +} + +export interface Job { + readonly id: number; + readonly name: string; + readonly conclusion: string | null; + readonly created_at?: string; + readonly started_at: string | null; + readonly completed_at: string | null; + readonly steps?: ReadonlyArray; +} + +export interface Attribute { + readonly key: string; + readonly value: + | { readonly stringValue: string } + | { readonly intValue: string }; +} + +export interface Span { + readonly traceId: string; + readonly spanId: string; + readonly parentSpanId?: string; + readonly name: string; + readonly kind: 1; + readonly startTimeUnixNano: string; + readonly endTimeUnixNano: string; + readonly attributes: ReadonlyArray; + readonly status: { readonly code: 1 | 2 }; +} + +const hex = (input: string, length: number) => + createHash("sha256").update(input).digest("hex").slice(0, length); + +/** One trace per commit: every workflow run for the commit lands in it, from any tool. */ +export const traceIdFor = (repository: string, sha: string) => + hex(`${repository}:${sha}`, 32); +export const runSpanId = (runId: number | string) => hex(`run:${runId}`, 16); + +export const attr = (key: string, value: string | number): Attribute => + typeof value === "number" + ? { key, value: { intValue: String(Math.round(value)) } } + : { key, value: { stringValue: value } }; + +/** + * Short step names: `Run actions/checkout@` becomes `checkout`. Rendered + * as code in comments, so GitHub never turns `owner/repo@sha` into a commit + * link or `@name` into a mention. + */ +export function stepLabel(name: string): string { + const action = name.match( + /^(Post )?Run [\w.-]+\/([\w.-]+)(?:\/[\w./-]+)?@\S+$/, + ); + if (action) return `${action[1] ? "post " : ""}${action[2]}`; + return name.split("\n")[0] ?? name; +} + +export const toMillis = (iso: string | null | undefined, fallback: number) => + iso === null || iso === undefined ? fallback : Date.parse(iso); + +export function span(fields: { + traceId: string; + id: string; + parent?: string; + name: string; + startMs: number; + endMs: number; + failed?: boolean; + attributes?: ReadonlyArray; +}): Span { + const nanos = (ms: number) => `${BigInt(Math.round(ms)) * 1_000_000n}`; + return { + traceId: fields.traceId, + spanId: fields.id, + ...(fields.parent ? { parentSpanId: fields.parent } : {}), + name: fields.name, + kind: 1, + startTimeUnixNano: nanos(fields.startMs), + endTimeUnixNano: nanos(Math.max(fields.startMs, fields.endMs)), + attributes: fields.attributes ?? [], + status: { code: fields.failed ? 2 : 1 }, + }; +} + +/** Job and step spans under `parent`. Running jobs and steps end at `now`. */ +export function jobSpans( + traceId: string, + parent: string, + jobs: ReadonlyArray, + now: number, + extraAttributes: (job: Job) => ReadonlyArray = () => [], +): Array { + const spans: Array = []; + for (const job of jobs) { + if (job.started_at === null) continue; + const jobId = hex(`job:${job.id}`, 16); + const start = toMillis(job.started_at, now); + spans.push( + span({ + traceId, + id: jobId, + parent, + name: job.name, + startMs: start, + endMs: toMillis(job.completed_at, now), + failed: job.conclusion === "failure", + attributes: [ + attr("cicd.pipeline.task.name", job.name), + attr("cicd.pipeline.result", job.conclusion ?? "running"), + // Time spent waiting for a runner before the job started. + ...(job.created_at + ? [ + attr( + "cicd.pipeline.task.queued_ms", + start - toMillis(job.created_at, start), + ), + ] + : []), + ...extraAttributes(job), + ], + }), + ); + for (const step of job.steps ?? []) { + if (step.started_at === null) continue; + spans.push( + span({ + traceId, + id: hex(`step:${job.id}:${step.number}`, 16), + parent: jobId, + name: stepLabel(step.name), + startMs: toMillis(step.started_at, now), + endMs: toMillis(step.completed_at, now), + failed: step.conclusion === "failure", + attributes: [ + attr("cicd.pipeline.task.step.name", step.name), + attr("cicd.pipeline.result", step.conclusion ?? "running"), + ], + }), + ); + } + } + return spans; +} + +/** An OTLP/JSON trace export, as accepted by `POST /v1/traces`. */ +export function otlpDocument(spans: ReadonlyArray) { + return { + resourceSpans: [ + { + resource: { attributes: [attr("service.name", "rivet-ci")] }, + scopeSpans: [{ scope: { name: "pipboy" }, spans }], + }, + ], + }; +}