diff --git a/.ai/skills/hack-repo-verify/SKILL.md b/.ai/skills/hack-repo-verify/SKILL.md index ed6106405..0faa9c9ff 100644 --- a/.ai/skills/hack-repo-verify/SKILL.md +++ b/.ai/skills/hack-repo-verify/SKILL.md @@ -98,7 +98,7 @@ untracked candidate additions. Keep private artifacts excluded from the index. | Native authored command slices | `tests/native-authored-command.test.ts` (unsupported matrix), `tests/native-authored-command-dispatch.test.ts`, `tests/native-authored-project-{start,run,observe,logs,exec,recovery}.test.ts` and the [command contract](../../../docs/reference/native-authored-commands.md); the dispatch "source CLI" cases need the prepared compiler (`bun run build:config-compiler` or absolute `HACK_CONFIG_COMPILER_BINARY`) and fail without it; `native-authored-project-start` and `native-authored-project-exec` are nondeterministic on a local macOS host, so a single local pass or failure is not proof; live acceptance runs host-serialized on one prepared macOS provider slot | | Native VM/routing/reclamation | [candidate guide](../../../docs/guides/native-candidate.md), relevant acceptance gate and isolated host fixture; Linux checks cannot qualify macOS effects | | Resource/performance claims | [performance skill](../hack-repo-performance/SKILL.md), matched workload and measured boundaries | -| Bun runtime crashes | Read-only history: `bun run ci:bun-crash-census -- --since --until ` (failed `test` job logs; `--scan-passing` for all; since the suite moved to `test shard i/4` jobs, pass `--job "test shard i/4"` once per shard for newer runs). Matched reproduction: the gated [experiment](../../../.github/workflows/bun-crash-experiment.yml), which spends hosted macOS minutes and needs an explicit go-ahead; never loop on a shared host. Its default `focused` mode loops `tests/native-https-owner.test.ts` (where both original segfaults happened); `full` needs a budget of at least 40 minutes because one full suite takes 30-35. A run never starts unless its estimate fits the remaining budget, so read `summary.json` for the verdict and per-mode crash, failure and skipped-for-budget counts. Both share [the classifier](../../../scripts/lib/bun-crash-classify.ts) | +| Bun runtime crashes | Read-only history: `bun run ci:bun-crash-census -- --since --until ` (failed test-suite job logs: every `test shard i/n` job, or the `test` job of a run from before the split, since `test` is now the logless fan-in; `--scan-passing` for all; `--job NAME` selects by exact name or a `*` pattern such as `"test shard *"`; a `warning:` line means no job matched or a selected job ran no Bun, so its zero is not evidence). Matched reproduction: the gated [experiment](../../../.github/workflows/bun-crash-experiment.yml), which spends hosted macOS minutes and needs an explicit go-ahead; never loop on a shared host. Its default `focused` mode loops `tests/native-https-owner.test.ts` (where both original segfaults happened); `full` needs a budget of at least 40 minutes because one full suite takes 30-35. A run never starts unless its estimate fits the remaining budget, so read `summary.json` for the verdict and per-mode crash, failure and skipped-for-budget counts. Both share [the classifier](../../../scripts/lib/bun-crash-classify.ts) | Read [CI](../../../.github/workflows/ci.yml) for current hosted gates. A local pass does not mean CI ran. diff --git a/docs/reference/shared-host-lifecycle.md b/docs/reference/shared-host-lifecycle.md index 0255d8138..587a8a172 100644 --- a/docs/reference/shared-host-lifecycle.md +++ b/docs/reference/shared-host-lifecycle.md @@ -226,9 +226,10 @@ Progress events are also closed: `lock-waiting`, `lock-acquired`, `claim-stale`, `bun test tests/shared-host-lifecycle.test.ts` runs portable local fixtures. Each fixture is a Bun process on a loopback port with a private HOME. No Docker, real -HOME, keychain or privileged effect is involved. The file runs in the hosted macOS -`test` job and is registered in the hosted Linux `linux-process-lifetime` job. A -Linux pass is a receipt only once that hosted job has run it. +HOME, keychain or privileged effect is involved. The file runs in one of the hosted +macOS `test shard i/4` jobs (the required `test` check passes only when every shard +passes) and is registered in the hosted Linux `linux-process-lifetime` job. A Linux +pass is a receipt only once that hosted job has run it. | Scenario | Asserted behavior | | --- | --- | diff --git a/docs/security/env-secrets-threat-model.md b/docs/security/env-secrets-threat-model.md index 7cff5ae79..a077f7de1 100644 --- a/docs/security/env-secrets-threat-model.md +++ b/docs/security/env-secrets-threat-model.md @@ -96,7 +96,7 @@ refer to `hack-dance/hack`. | A key copy ships in build output or a static site. | `hack doctor` reports `env-key-public-path` for copies under `public/`, `static/`, `dist/`, `build/`, `out/`, `www/`, `_site/`, `.next/` and `.output/` (#309). | | A weak, short or repetitive key. | `env-key-wrong-length` and `env-key-low-entropy` (#309). Generated keys are 32 random bytes in base64url. | | Materialized `.hack/.env` or local overrides are committed. | Hack owns a committed `.hack/.gitignore` block, and `hack doctor` flags tracked generated files (see [ignore rules](../env.md#ignore-rules-committed-hackgitignore)). | -| A secret leaks into a public artifact through a path nobody listed. | `tests/env-secret-sentinel-scan.test.ts` runs in the CI `test` job. It gives a primary checkout and a linked worktree random synthetic secrets through `env add --secret --value-stdin` (shared and `--local`), runs materialize, exec, up (Compose generation), env apply, logs, list, explain and doctor, then commits everything with `git add -A`. It requires zero occurrences of any value or the project key, raw or hex/base64/base64url encoded, in CLI output, recorded child argv, tracked files, every Git object, HACK_HOME, the private HOME and its caches, and generated Compose and runtime files. Plaintext may exist only in `.hack/.env`, and key text only in key files; each must be git-ignored, untracked and mode 0600. `.hack/.internal/compose.env.override.yml` must hold `${HACK_ENV_...}` references and no value, and must also be private and ignored. Values may reach a child's environment only for the exec'd command and `docker compose`. Positive controls prove delivery and that the scanner finds planted encodings in files and Git objects. Docker is a recording shim; no engine, keychain or installed Hack is reached. | +| A secret leaks into a public artifact through a path nobody listed. | `tests/env-secret-sentinel-scan.test.ts` runs in one of the hosted CI `test shard i/4` jobs; the required `test` check passes only when every shard passes. It gives a primary checkout and a linked worktree random synthetic secrets through `env add --secret --value-stdin` (shared and `--local`), runs materialize, exec, up (Compose generation), env apply, logs, list, explain and doctor, then commits everything with `git add -A`. It requires zero occurrences of any value or the project key, raw or hex/base64/base64url encoded, in CLI output, recorded child argv, tracked files, every Git object, HACK_HOME, the private HOME and its caches, and generated Compose and runtime files. Plaintext may exist only in `.hack/.env`, and key text only in key files; each must be git-ignored, untracked and mode 0600. `.hack/.internal/compose.env.override.yml` must hold `${HACK_ENV_...}` references and no value, and must also be private and ignored. Values may reach a child's environment only for the exec'd command and `docker compose`. Positive controls prove delivery and that the scanner finds planted encodings in files and Git objects. Docker is a recording shim; no engine, keychain or installed Hack is reached. | | A dependency vulnerability ships in Hack. | `bun install --frozen-lockfile`, and the read-only `scripts/dependabot-inventory.ts` lists open alerts (#309). Hack does not sandbox project dependencies that run under `hack host exec`; they receive the injected env by design. | ### Agent and MCP exfiltration diff --git a/scripts/ci-bun-crash-census.ts b/scripts/ci-bun-crash-census.ts index c6b853687..f04cce77c 100644 --- a/scripts/ci-bun-crash-census.ts +++ b/scripts/ci-bun-crash-census.ts @@ -3,10 +3,19 @@ * Read-only census of Bun crashes in hosted CI history. * * For a `--since`/`--until` window it lists every run of the CI workflow, reads - * the `test` job of every run attempt, downloads only failed-job logs (all + * the test-suite jobs of every run attempt, downloads only failed-job logs (all * completed logs with `--scan-passing`) into an owned temporary directory, and * classifies each with the shared classifier used by `scripts/bun-crash-loop.ts`. * + * By default the test-suite jobs are every `test shard i/n` job of a run. Since + * HACK-1381 the `test` job is only the required fan-in: it runs no Bun, so its + * log holds nothing to classify. A run with no shard job predates the split, so + * its `test` job ran the suite and is read instead. `--job NAME` selects jobs by + * exact name, or by a pattern where `*` matches any text + * (`--job "test shard *"`). The report warns when no job matched, or when a + * selected job shows no Bun setup in its log or workflow block, because a zero + * crash count from such a job is not evidence. + * * It uses the caller's existing `gh` authentication and never handles a token. * It is strictly read-only: no reruns, comments, labels or uploads. It prints * counts, closed tokens, versions and numeric run/job ids only, never log bodies. @@ -62,6 +71,18 @@ export interface JobRecord { readonly reason: BunRunReason | null; } +/** + * A closed warning. Each one means the census could not see the Bun output a + * zero crash count needs, so the totals are not evidence by themselves. + */ +export type CensusWarning = + | { readonly kind: "no_jobs_matched" } + | { + readonly kind: "no_bun_setup"; + readonly jobs: number; + readonly selected: number; + }; + export interface CrashingJob { readonly runId: number; readonly attempt: number; @@ -85,7 +106,15 @@ export interface VersionTotals { const ISO_INSTANT = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$/; const REPOSITORY = /^[A-Za-z0-9_.-]{1,100}\/[A-Za-z0-9_.-]{1,100}$/; const WORKFLOW_FILE = /^[A-Za-z0-9_.-]{1,100}\.ya?ml$/; -const JOB_NAME = /^[A-Za-z0-9_. -]{1,100}$/; +const JOB_NAME = /^[A-Za-z0-9_. /*-]{1,100}$/; +/** A shard job of the hosted test suite (`.github/workflows/ci.yml`, HACK-1381). */ +const TEST_SHARD_JOB = /^test shard \d+\/\d+$/; +/** Before the suite was sharded this job ran it; now it is the logless fan-in. */ +const LEGACY_TEST_JOB = "test"; +const WORKFLOW_JOB_KEY = /^ {2}([A-Za-z0-9_-]+):\s*$/; +const WORKFLOW_JOB_NAME = /^ {4}name:\s*(.+?)\s*$/; +const WORKFLOW_EXPRESSION = /\$\{\{.*?\}\}/; +const WORKFLOW_BUN_VERSION = /^\s+bun-version:\s*["']?([^"'\s]+)["']?\s*$/; const SHA = /^[0-9a-f]{40}$/; /** The runs endpoint returns at most 1000 results for one `created` query. */ const MAX_WINDOW_RUNS = 1000; @@ -107,24 +136,76 @@ const positiveInteger = (value: unknown): number | null => ? value : null; +const escapeRegExp = (text: string): string => + text.replace(/[.*+?^${}()|[\]\\/]/g, "\\$&"); + +/** Whether a job display name matches a `--job` value (`*` matches any text). */ +export function jobNameMatches(pattern: string, name: string): boolean { + return new RegExp( + `^${pattern.split("*").map(escapeRegExp).join(".*")}$` + ).test(name); +} + /** - * The `bun-version` given to the named job in a workflow file, or null. The - * job block runs from ` :` to the next two-space-indented key. + * Whether a workflow job with key `key` and an optional `name:` template has + * the display name `name`. A `${{ ... }}` expression in the template matches + * any non-empty text, so `test shard ${{ matrix.shard }}/4` names + * `test shard 2/4`. + */ +function workflowJobNamed(opts: { + readonly key: string; + readonly template: string | null; + readonly name: string; +}): boolean { + if (opts.template === null) { + return opts.key === opts.name; + } + const template = opts.template.replace(/^(["'])(.*)\1$/, "$2"); + return new RegExp( + `^${template.split(WORKFLOW_EXPRESSION).map(escapeRegExp).join(".+")}$` + ).test(opts.name); +} + +/** + * The `bun-version` given to the job with display name `job` in a workflow + * file, or null. A job block runs from its two-space-indented key under + * `jobs:` to the next one. Its display name is its `name:` template, or its + * key when it has none, so `test shard 2/4` finds the `test-shard` matrix job. + * A job with no `bun-version`, such as the `test` fan-in, returns null. */ export function workflowJobBunVersion( workflow: string, job: string ): string | null { const lines = workflow.split(/\r?\n/); - const start = lines.findIndex((line) => line.trimEnd() === ` ${job}:`); - if (start === -1) { + const jobsAt = lines.findIndex((line) => line.trimEnd() === "jobs:"); + if (jobsAt === -1) { return null; } - for (const line of lines.slice(start + 1)) { - if (/^ {2}\S/.test(line)) { - return null; + const blocks: { readonly key: string; readonly lines: string[] }[] = []; + for (const line of lines.slice(jobsAt + 1)) { + if (/^\S/.test(line)) { + break; } - const match = /^\s+bun-version:\s*["']?([^"'\s]+)["']?\s*$/.exec(line); + const key = WORKFLOW_JOB_KEY.exec(line)?.[1]; + if (key) { + blocks.push({ key, lines: [] }); + } else { + blocks.at(-1)?.lines.push(line); + } + } + const block = blocks.find((candidate) => + workflowJobNamed({ + key: candidate.key, + template: + candidate.lines + .map((line) => WORKFLOW_JOB_NAME.exec(line)?.[1]) + .find((value) => value !== undefined) ?? null, + name: job, + }) + ); + for (const line of block?.lines ?? []) { + const match = WORKFLOW_BUN_VERSION.exec(line); if (match) { return closedBunVersion(match[1]); } @@ -132,6 +213,38 @@ export function workflowJobBunVersion( return null; } +/** + * Which job names of one run the census reads. With no `--job`, the run's + * `test shard i/n` jobs, or its `test` job when it has no shard job (a run + * from before the split, whose `test` job ran the suite). + */ +export function suiteJobSelector(opts: { + readonly job: string | null; + readonly runJobNames: readonly string[]; +}): (name: string) => boolean { + const { job } = opts; + if (job !== null) { + return (name) => jobNameMatches(job, name); + } + if (opts.runJobNames.some((name) => TEST_SHARD_JOB.test(name))) { + return (name) => TEST_SHARD_JOB.test(name); + } + return (name) => name === LEGACY_TEST_JOB; +} + +/** The warnings for a finished census; see {@link CensusWarning}. */ +export function censusWarnings(records: readonly JobRecord[]): CensusWarning[] { + if (records.length === 0) { + return [{ kind: "no_jobs_matched" }]; + } + const withoutBun = records.filter( + (entry) => entry.versionSource === "unknown" + ).length; + return withoutBun === 0 + ? [] + : [{ kind: "no_bun_setup", jobs: withoutBun, selected: records.length }]; +} + export function emptyTotals(bunVersion: string): VersionTotals { return { bunVersion, @@ -200,7 +313,8 @@ export function aggregateCensus( export interface CensusOptions { readonly repository: string; readonly workflow: string; - readonly job: string; + /** Null reads the test-suite jobs; see {@link suiteJobSelector}. */ + readonly job: string | null; readonly since: string; readonly until: string; readonly scanPassing: boolean; @@ -211,6 +325,7 @@ export interface CensusResult { readonly runs: number; readonly records: readonly JobRecord[]; readonly totals: readonly VersionTotals[]; + readonly warnings: readonly CensusWarning[]; } async function mapConcurrent( @@ -335,11 +450,15 @@ export async function runCensus( } const workflowVersions = new Map>(); - const workflowVersion = (sha: string | null): Promise => { + const workflowVersion = ( + sha: string | null, + job: string + ): Promise => { if (sha === null) { return Promise.resolve(null); } - const cached = workflowVersions.get(sha); + const cacheKey = `${sha}\n${job}`; + const cached = workflowVersions.get(cacheKey); if (cached) { return cached; } @@ -358,11 +477,11 @@ export async function runCensus( } return workflowJobBunVersion( Buffer.from(value.content, "base64").toString("utf8"), - options.job + job ); }) .catch(() => null); - workflowVersions.set(sha, pending); + workflowVersions.set(cacheKey, pending); return pending; }; @@ -372,16 +491,24 @@ export async function runCensus( async (run) => { const seen = new Set(); const records: JobRecord[] = []; - const lines = await gh.lines( - `repos/${repo}/actions/runs/${run.id}/jobs?filter=all&per_page=100`, - ".jobs[] | {id, name, run_attempt, status, conclusion}" - ); - for (const line of lines) { - const job = parseLine(line); + const jobs = ( + await gh.lines( + `repos/${repo}/actions/runs/${run.id}/jobs?filter=all&per_page=100`, + ".jobs[] | {id, name, run_attempt, status, conclusion}" + ) + ).map(parseLine); + const selected = suiteJobSelector({ + job: options.job, + runJobNames: jobs.flatMap((job) => + typeof job.name === "string" ? [job.name] : [] + ), + }); + for (const job of jobs) { const jobId = positiveInteger(job.id); const attempt = positiveInteger(job.run_attempt); if ( - job.name !== options.job || + typeof job.name !== "string" || + !selected(job.name) || job.status !== "completed" || job.conclusion === "skipped" || jobId === null || @@ -399,7 +526,7 @@ export async function runCensus( : null; const fromWorkflow = scanned?.bunVersion ? null - : await workflowVersion(run.headSha); + : await workflowVersion(run.headSha, job.name); records.push({ runId: run.id, attempt, @@ -414,9 +541,26 @@ export async function runCensus( } ); const records = perRun.flat(); - return { runs: runs.length, records, totals: aggregateCensus(records) }; + return { + runs: runs.length, + records, + totals: aggregateCensus(records), + warnings: censusWarnings(records), + }; +} + +/** One closed, value-free line for a warning. */ +export function renderCensusWarning(warning: CensusWarning): string { + if (warning.kind === "no_jobs_matched") { + return "warning: no completed job matched the job selector, so zero crashes is not evidence; check --job against the workflow's job names"; + } + return `warning: ${warning.jobs} of ${warning.selected} selected jobs show no Bun setup in their log or workflow, so their crash count is not evidence; the test job is a fan-in that runs no Bun since HACK-1381, so omit --job to read the test shard jobs`; } +/** The report's `job=` value: the quoted selector, or `default`. */ +const jobLabel = (job: string | null): string => + job === null ? "default" : JSON.stringify(job); + /** Closed, value-free text report. */ export function renderCensus(opts: { readonly options: CensusOptions; @@ -424,7 +568,7 @@ export function renderCensus(opts: { }): string { const { options, result } = opts; const lines = [ - `Bun crash census repo=${options.repository} workflow=${options.workflow} job=${options.job} since=${options.since} until=${options.until} runs=${result.runs} jobs=${result.records.length} scan_passing=${options.scanPassing}`, + `Bun crash census repo=${options.repository} workflow=${options.workflow} job=${jobLabel(options.job)} since=${options.since} until=${options.until} runs=${result.runs} jobs=${result.records.length} scan_passing=${options.scanPassing}`, ]; for (const totals of result.totals) { const sources = Object.entries(totals.versionSources) @@ -439,6 +583,7 @@ export function renderCensus(opts: { ); } } + lines.push(...result.warnings.map(renderCensusWarning)); return `${lines.join("\n")}\n`; } @@ -495,12 +640,12 @@ export function parseCensusArguments(argv: readonly string[]): CensusOptions & { } const repository = values.get("repo") ?? "hack-dance/hack"; const workflow = values.get("workflow") ?? "ci.yml"; - const job = values.get("job") ?? "test"; + const job = values.get("job") ?? null; if ( !( REPOSITORY.test(repository) && WORKFLOW_FILE.test(workflow) && - JOB_NAME.test(job) + (job === null || JOB_NAME.test(job)) ) ) { throw new CensusError("Invalid argument"); @@ -583,9 +728,12 @@ async function main(): Promise { const result = await runCensus(options, ghCliReader, workDir); process.stdout.write( json - ? `${JSON.stringify({ options, runs: result.runs, totals: result.totals }, null, 2)}\n` + ? `${JSON.stringify({ options, runs: result.runs, totals: result.totals, warnings: result.warnings }, null, 2)}\n` : renderCensus({ options, result }) ); + for (const warning of result.warnings) { + process.stderr.write(`${renderCensusWarning(warning)}\n`); + } } finally { process.off("SIGINT", interrupt); process.off("SIGTERM", interrupt); diff --git a/tests/ci-bun-crash-census.test.ts b/tests/ci-bun-crash-census.test.ts index ca037363d..acf751556 100644 --- a/tests/ci-bun-crash-census.test.ts +++ b/tests/ci-bun-crash-census.test.ts @@ -1,15 +1,17 @@ import { afterEach, describe, expect, test } from "bun:test"; -import { mkdtemp, readdir, rm } from "node:fs/promises"; +import { mkdtemp, readdir, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { join, resolve } from "node:path"; import { aggregateCensus, type CensusOptions, type GhReader, type JobRecord, + jobNameMatches, parseCensusArguments, renderCensus, runCensus, + suiteJobSelector, workflowJobBunVersion, } from "../scripts/ci-bun-crash-census.ts"; import { @@ -407,12 +409,26 @@ test("census arguments are validated and closed", () => { ).toEqual({ repository: "hack-dance/hack", workflow: "ci.yml", - job: "test", + job: null, since: "2026-10-01T17:35:00Z", until: "2026-10-11T23:43:30Z", scanPassing: true, json: false, }); + const window = [ + "--since", + "2026-10-01T17:35:00Z", + "--until", + "2026-10-11T23:43:30Z", + ]; + // The verify skill documents `--job "test shard i/4"`; a `/` or `*` used to be refused. + expect(parseCensusArguments([...window, "--job", "test shard 1/4"]).job).toBe( + "test shard 1/4" + ); + expect(parseCensusArguments([...window, "--job=test shard *"]).job).toBe( + "test shard *" + ); + expect(parseCensusArguments([...window, "--job", "test"]).job).toBe("test"); for (const argv of [ [], ["--since", "2026-10-01", "--until", "2026-10-11T23:43:30Z"], @@ -440,7 +456,238 @@ test("census arguments are validated and closed", () => { "--json", "--json", ], + [...window, "--job", "test;shard"], + [...window, "--job", "t".repeat(101)], ]) { expect(() => parseCensusArguments(argv)).toThrow(); } }); + +describe("sharded test suite (HACK-1424)", () => { + const SHARDED_WORKFLOW = [ + "name: CI", + "on:", + " push:", + " pull_request:", + "jobs:", + " test-shard:", + " name: test shard ${{ matrix.shard }}/4", + " steps:", + " - uses: oven-sh/setup-bun@v1", + " with:", + ' bun-version: "1.4.2"', + " test:", + " needs: test-shard", + " if: ${{ always() }}", + " steps:", + " - name: Require every test shard to succeed", + " run: exit 0", + " quoted:", + ' name: "quoted ${{ matrix.os }}"', + " steps:", + " - with:", + " bun-version: 1.3.9", + ].join("\n"); + + test("a shard job's Bun version comes from its name template; the fan-in has none", () => { + expect(workflowJobBunVersion(SHARDED_WORKFLOW, "test shard 3/4")).toBe( + "1.4.2" + ); + expect(workflowJobBunVersion(SHARDED_WORKFLOW, "test")).toBeNull(); + expect(workflowJobBunVersion(SHARDED_WORKFLOW, "test shard /4")).toBeNull(); + expect(workflowJobBunVersion(SHARDED_WORKFLOW, "test-shard")).toBeNull(); + expect(workflowJobBunVersion(SHARDED_WORKFLOW, "quoted macos")).toBe( + "1.3.9" + ); + // `on:` keys are not jobs. + expect(workflowJobBunVersion(SHARDED_WORKFLOW, "push")).toBeNull(); + }); + + test("the real CI workflow gives every shard the pinned Bun and the test fan-in none", async () => { + const workflow = await readFile( + resolve(import.meta.dir, "../.github/workflows/ci.yml"), + "utf8" + ); + const manifest: unknown = JSON.parse( + await readFile(resolve(import.meta.dir, "../package.json"), "utf8") + ); + const pin = + typeof manifest === "object" && + manifest !== null && + "packageManager" in manifest && + typeof manifest.packageManager === "string" + ? manifest.packageManager.replace(/^bun@/, "") + : null; + expect(pin).toMatch(/^\d+\.\d+\.\d+$/); + for (const shard of [1, 2, 3, 4]) { + expect(workflowJobBunVersion(workflow, `test shard ${shard}/4`)).toBe( + pin + ); + } + expect(workflowJobBunVersion(workflow, "test")).toBeNull(); + }); + + test("job selectors: default reads shards, or the legacy test job when a run has none", () => { + const sharded = suiteJobSelector({ + job: null, + runJobNames: ["test shard 1/4", "test shard 2/4", "test", "docker-e2e"], + }); + expect( + ["test shard 1/4", "test shard 2/4", "test", "docker-e2e shard 1/4"].map( + sharded + ) + ).toEqual([true, true, false, false]); + const legacy = suiteJobSelector({ + job: null, + runJobNames: ["test", "runtime-images"], + }); + expect(["test", "runtime-images"].map(legacy)).toEqual([true, false]); + expect(jobNameMatches("test shard *", "test shard 4/4")).toBe(true); + expect(jobNameMatches("test shard *", "test")).toBe(false); + expect(jobNameMatches("test shard 1/4", "test shard 1/4")).toBe(true); + expect(jobNameMatches("test shard 1/4", "test shard 1/40")).toBe(false); + expect(jobNameMatches("test.shard", "testXshard")).toBe(false); + }); + + const FAN_IN_FAILURE_LOG = [ + stamp('##[group]Run if [ "$SHARD_RESULT" != "success" ]; then'), + stamp("test shards did not all succeed (result: failure)"), + stamp("##[error]Process completed with exit code 1."), + ].join("\n"); + const SHA_SHARDED = "c".repeat(40); + + function shardedGh() { + const downloads: string[] = []; + const logs: Record = { + 31: CRASH_LOG, + 33: FAN_IN_FAILURE_LOG, + 41: TEST_FAILURE_LOG, + }; + const job = (opts: { + readonly id: number; + readonly name: string; + readonly conclusion: string; + readonly attempt?: number; + }): string => + JSON.stringify({ + id: opts.id, + name: opts.name, + run_attempt: opts.attempt ?? 1, + status: "completed", + conclusion: opts.conclusion, + }); + const reader: GhReader = { + json: async (path) => { + if (path.includes("/runs?created=")) { + return { total_count: 2 }; + } + if (path.endsWith(`?ref=${SHA_SHARDED}`)) { + return { + content: Buffer.from(SHARDED_WORKFLOW).toString("base64"), + }; + } + if (path.endsWith(`?ref=${SHA_OLD}`)) { + return { content: workflowFor("1.3.9") }; + } + throw new Error("unexpected path"); + }, + lines: async (path) => { + if (path.includes("/runs?created=")) { + return [ + JSON.stringify({ id: 300, head_sha: SHA_SHARDED }), + JSON.stringify({ id: 400, head_sha: SHA_OLD }), + ]; + } + if (path.includes("/runs/300/jobs?filter=all")) { + return [ + job({ id: 31, name: "test shard 1/4", conclusion: "failure" }), + job({ id: 32, name: "test shard 2/4", conclusion: "success" }), + job({ id: 33, name: "test", conclusion: "failure" }), + job({ + id: 34, + name: "docker-e2e shard 1/4", + conclusion: "failure", + }), + ]; + } + if (path.includes("/runs/400/jobs?filter=all")) { + return [ + job({ id: 41, name: "test", conclusion: "failure" }), + job({ id: 42, name: "test", conclusion: "success", attempt: 2 }), + ]; + } + throw new Error("unexpected path"); + }, + download: async (path, destination) => { + const jobId = Number(/\/jobs\/(\d+)\/logs$/.exec(path)?.[1]); + downloads.push(path); + await Bun.write(destination, logs[jobId] ?? ""); + }, + }; + return { reader, downloads }; + } + + test("with no --job the census reads shard jobs, skips the fan-in, and still reads pre-split runs", async () => { + const dir = await workDir(); + const gh = shardedGh(); + const defaults: CensusOptions = { ...options, job: null }; + const result = await runCensus(defaults, gh.reader, dir); + expect(gh.downloads).toEqual([ + "repos/hack-dance/hack/actions/jobs/31/logs", + "repos/hack-dance/hack/actions/jobs/41/logs", + ]); + expect(await readdir(dir)).toEqual([]); + expect( + result.records.map((entry) => [ + entry.jobId, + entry.bunVersion, + entry.versionSource, + ]) + ).toEqual([ + [31, "1.3.9", "log"], + [32, "1.4.2", "workflow"], + [41, "1.4.2", "log"], + [42, "1.3.9", "workflow"], + ]); + expect(result.warnings).toEqual([]); + const text = renderCensus({ options: defaults, result }); + expect(text).toContain(" job=default "); + expect(text).toContain(" crash run=300 attempt=1 job=31 reason=banner"); + expect(text).not.toContain("warning:"); + }); + + test("an explicit --job test on sharded runs warns that the fan-in has no Bun output", async () => { + const dir = await workDir(); + const gh = shardedGh(); + const fanIn: CensusOptions = { ...options, job: "test" }; + const result = await runCensus(fanIn, gh.reader, dir); + expect(result.records.map((entry) => entry.jobId)).toEqual([33, 41, 42]); + expect(result.records[0]).toMatchObject({ + bunVersion: "unknown", + versionSource: "unknown", + scan: "failure", + }); + expect(result.warnings).toEqual([ + { kind: "no_bun_setup", jobs: 1, selected: 3 }, + ]); + const text = renderCensus({ options: fanIn, result }); + expect(text).toContain(' job="test" '); + expect(text).toContain( + "warning: 1 of 3 selected jobs show no Bun setup in their log or workflow" + ); + expect(text).not.toContain("SHARD_RESULT"); + }); + + test("a selector that matches no job warns instead of reporting a silent zero", async () => { + const dir = await workDir(); + const gh = shardedGh(); + const none: CensusOptions = { ...options, job: "test shard 9/9" }; + const result = await runCensus(none, gh.reader, dir); + expect(result.records).toEqual([]); + expect(gh.downloads).toEqual([]); + expect(result.warnings).toEqual([{ kind: "no_jobs_matched" }]); + expect(renderCensus({ options: none, result })).toContain( + "warning: no completed job matched the job selector" + ); + }); +});