From 7dc0359a75275e4d32dbfdcc75db4bf7ae296a4c Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Sat, 24 Oct 2026 13:56:37 -0400 Subject: [PATCH 1/2] test: keep the write guard, pid handshakes and store cases reliable under load (HACK-1439) - checkout write guard: bound each git call (CHECKOUT_GIT_BUDGET_MS, killed and refused closed at the bound) and register the end-of-run afterAll with CHECKOUT_GUARD_HOOK_BUDGET_MS instead of Bun's 5 s default, which killed git mid-status and failed runs whose cases all passed. Guarded fixture keep-alive cases get a named budget too. - native-runtime-client: one bounded wait-for-complete-content helper backs waitForPid and waitForFileText; the cancellation case waits for a complete pid before it aborts (its fixture now publishes a torn handshake every run), and no case waits for existence and then reads. - private-env-file and native-compose-storage-witness-xattr-owner: a file-scoped named default budget replaces Bun's 5 s default and the fixed 30_000 literals; captured carriers outlive their case budget. Co-Authored-By: Claude Opus 5.5 --- tests/checkout-write-guard.test.ts | 96 +++++++++++++++++- tests/helpers/checkout-write-guard.ts | 49 +++++++-- ...ompose-storage-witness-xattr-owner.test.ts | 75 ++++++++------ tests/native-runtime-client.test.ts | 99 ++++++++++--------- tests/private-env-file.test.ts | 22 ++++- 5 files changed, 252 insertions(+), 89 deletions(-) diff --git a/tests/checkout-write-guard.test.ts b/tests/checkout-write-guard.test.ts index 6c91c7b4c..da162d093 100644 --- a/tests/checkout-write-guard.test.ts +++ b/tests/checkout-write-guard.test.ts @@ -12,6 +12,7 @@ import { join, resolve } from "node:path"; import { ClosedCliError } from "../src/lib/cli-result.ts"; import { assertNoCheckoutWrites, + CHECKOUT_GIT_BUDGET_MS, checkoutStatusEntries, newStatusEntries, } from "./helpers/checkout-write-guard.ts"; @@ -34,6 +35,14 @@ const HELPERS = resolve(import.meta.dir, "helpers"); const CLI_RESULT = resolve(import.meta.dir, "..", "src/lib/cli-result.ts"); /** Each case starts a fresh `bun` child; a cold start under suite load is slow. */ const CHILD_TEST_TIMEOUT_MS = 60_000; +/** + * Bun timeout of a guarded fixture's keep-alive case. That case only has to + * outlive the timed-out case's late call, but a late `spawnSync` blocks the + * fixture's event loop while its cold `bun` child starts, which under host + * load passed Bun's 5 s default (HACK-1439). It stays inside the outer case's + * budget. + */ +const KEEP_ALIVE_CASE_TIMEOUT_MS = 30_000; let root = ""; @@ -54,10 +63,14 @@ const fixture = (): string => async function run(opts: { readonly argv: readonly string[]; readonly cwd: string; + readonly env?: Readonly>; }): Promise<{ readonly exitCode: number; readonly output: string }> { const child = Bun.spawn([...opts.argv], { cwd: opts.cwd, - env: privateCliEnv({ root: fixture(), env: { NO_COLOR: "1" } }), + env: privateCliEnv({ + root: fixture(), + env: { NO_COLOR: "1", ...opts.env }, + }), stdin: "ignore", stdout: "pipe", stderr: "pipe", @@ -276,6 +289,29 @@ test( CHILD_TEST_TIMEOUT_MS ); +test( + "the guard kills and refuses a git status that reaches its bound instead of waiting on it", + async () => { + const checkout = await gitCheckout("hanging-status"); + const bin = join(fixture(), "hanging-bin"); + await mkdir(bin); + const hanging = join(bin, "git"); + await writeFile(hanging, "#!/bin/sh\nexec sleep 30\n", { mode: 0o700 }); + const started = performance.now(); + expectClosedThrow( + () => + checkoutStatusEntries( + { git: hanging, env: gitRunner().env, budgetMs: 200 }, + checkout + ), + "checkout write guard: git status did not finish within 200 ms" + ); + // Killed at its bound, not left to finish its 30 s sleep. + expect(performance.now() - started).toBeLessThan(20_000); + }, + CHILD_TEST_TIMEOUT_MS +); + /** * A bun-test project in a fresh git checkout, preloading the guard exactly as * the repository's bunfig.toml does. @@ -328,7 +364,7 @@ test( }); test("keeps the run alive past the late call", async () => { await Bun.sleep(1000); - }); + }, ${KEEP_ALIVE_CASE_TIMEOUT_MS}); `, }); const result = await run({ @@ -372,6 +408,60 @@ test( CHILD_TEST_TIMEOUT_MS ); +/** Longer than Bun's 5 s default hook timeout, far inside the status bound. */ +const SLOW_STATUS_SECONDS = 6; + +test( + "an end-of-run git status slower than Bun's default hook timeout still passes a clean run", + async () => { + // HACK-1439: under host load the end-of-run status passed 5 s, Bun's + // default hook timeout killed git mid-read ("killed 1 dangling process"), + // and the guard refused a run whose cases all passed. A git wrapper makes + // only the end-of-run status slow: the case arms it after the baseline. + expect(SLOW_STATUS_SECONDS * 1000).toBeLessThan(CHECKOUT_GIT_BUDGET_MS); + const realGit = gitRunner().git; + const bin = join(fixture(), "slow-bin"); + await mkdir(bin); + const armed = join(fixture(), "slow-status-armed"); + const slept = join(fixture(), "slow-status-slept"); + await writeFile( + join(bin, "git"), + [ + "#!/bin/sh", + `if [ "$1" = status ] && [ -e ${JSON.stringify(armed)} ]; then`, + ` sleep ${SLOW_STATUS_SECONDS}`, + ` : > ${JSON.stringify(slept)}`, + "fi", + `exec ${JSON.stringify(realGit)} "$@"`, + "", + ].join("\n"), + { mode: 0o700 } + ); + const checkout = await guardedProject({ + name: "slow-status", + testSource: ` + import { expect, test } from "bun:test"; + import { writeFileSync } from "node:fs"; + test("arms the slow end-of-run status", () => { + writeFileSync(${JSON.stringify(armed)}, ""); + expect(true).toBe(true); + }); + `, + }); + const result = await run({ + argv: [process.execPath, "test"], + cwd: checkout, + env: { PATH: `${bin}:${process.env.PATH ?? ""}` }, + }); + expect(result.output).not.toContain("checkout write guard"); + expect(result.output).not.toContain("dangling process"); + expect(result.exitCode).toBe(0); + // The hook waited out the slow status rather than skipping it. + expect(await Bun.file(slept).exists()).toBe(true); + }, + CHILD_TEST_TIMEOUT_MS +); + test( "a late spawn with a reset cwd that rewrites a tracked file fails the run, named", async () => { @@ -401,7 +491,7 @@ test( }); test("keeps the run alive past the late spawn", async () => { await Bun.sleep(1000); - }); + }, ${KEEP_ALIVE_CASE_TIMEOUT_MS}); `, }); const result = await run({ diff --git a/tests/helpers/checkout-write-guard.ts b/tests/helpers/checkout-write-guard.ts index 08b876eb0..a8756b97f 100644 --- a/tests/helpers/checkout-write-guard.ts +++ b/tests/helpers/checkout-write-guard.ts @@ -19,32 +19,61 @@ import { ClosedCliError } from "../../src/lib/cli-result.ts"; /** At most this many paths are named in a failure. */ const MAX_NAMED = 50; +/** + * Wall-clock bound on one guard `git` call. A read-only `git status` of this + * checkout takes well under a second on an idle host, but its cost is file + * system work that grows with host load: at a load of 56-118 on an 18-CPU + * host it passed 5 s (HACK-1439). The bound is a hang guard, not a speed + * assertion; a call that reaches it is killed and refused, closed. + */ +export const CHECKOUT_GIT_BUDGET_MS = 60_000; + +/** + * Bun timeout of the end-of-run `afterAll` hook. It outlasts the status + * call's own bound, so a slow status ends as the guard's named refusal rather + * than Bun's 5 s default hook timeout killing `git` mid-read, which failed + * runs whose cases all passed (HACK-1439). + */ +export const CHECKOUT_GUARD_HOOK_BUDGET_MS = CHECKOUT_GIT_BUDGET_MS + 30_000; + type GitRunner = { readonly git: string; readonly env: Readonly>; + /** Bound on each call; `CHECKOUT_GIT_BUDGET_MS` when omitted. */ + readonly budgetMs?: number; }; function runGit( runner: GitRunner, cwd: string, args: readonly string[] -): { readonly exitCode: number; readonly stdout: string } { +): { + readonly exitCode: number; + readonly timedOut: boolean; + readonly stdout: string; +} { const result = Bun.spawnSync([runner.git, ...args], { cwd, env: runner.env, stdin: "ignore", stdout: "pipe", stderr: "ignore", + timeout: runner.budgetMs ?? CHECKOUT_GIT_BUDGET_MS, + killSignal: "SIGKILL", }); - return { exitCode: result.exitCode, stdout: result.stdout.toString() }; + return { + exitCode: result.exitCode, + timedOut: result.exitedDueToTimeout === true, + stdout: result.stdout.toString(), + }; } /** * The `git status` entries of `checkout`: every untracked, non-ignored file * and every changed tracked file, each as its two-letter status and its * checkout-relative path (`?? new.txt`, ` M package.json`). A failed - * `git status` is refused, closed: the guard never treats an unreadable - * checkout as clean. + * `git status`, or one that reaches its bound, is refused, closed: the guard + * never treats an unreadable checkout as clean. */ export function checkoutStatusEntries( runner: GitRunner, @@ -56,6 +85,11 @@ export function checkoutStatusEntries( "-z", "--untracked-files=all", ]); + if (status.timedOut) { + throw new ClosedCliError( + `checkout write guard: git status did not finish within ${runner.budgetMs ?? CHECKOUT_GIT_BUDGET_MS} ms; the checkout could not be checked for files written by tests.` + ); + } if (status.exitCode !== 0) { throw new ClosedCliError( "checkout write guard: git status failed; the checkout could not be checked for files written by tests." @@ -110,7 +144,8 @@ export function assertNoCheckoutWrites(opts: { /** * Records the baseline for the git checkout that contains `start` and - * registers the end-of-run check through `afterAll`. + * registers the end-of-run check through `afterAll`, with the named + * `CHECKOUT_GUARD_HOOK_BUDGET_MS` timeout instead of Bun's 5 s default. * * When `start` is not in a git work tree (an export, or a container mount of a * linked worktree) there is nothing to compare, so the guard stays inactive @@ -119,7 +154,7 @@ export function assertNoCheckoutWrites(opts: { */ export function installCheckoutWriteGuard(opts: { readonly start: string; - readonly afterAll: (hook: () => void) => void; + readonly afterAll: (hook: () => void, timeoutMs: number) => void; readonly env?: Readonly>; }): void { const source = opts.env ?? process.env; @@ -158,5 +193,5 @@ export function installCheckoutWriteGuard(opts: { checkoutStatusEntries(runner, checkout) ), }); - }); + }, CHECKOUT_GUARD_HOOK_BUDGET_MS); } diff --git a/tests/native-compose-storage-witness-xattr-owner.test.ts b/tests/native-compose-storage-witness-xattr-owner.test.ts index 125e9db6e..2bf1903a7 100644 --- a/tests/native-compose-storage-witness-xattr-owner.test.ts +++ b/tests/native-compose-storage-witness-xattr-owner.test.ts @@ -1,4 +1,11 @@ -import { afterEach, beforeEach, expect, spyOn, test } from "bun:test"; +import { + afterEach, + beforeEach, + expect, + setDefaultTimeout, + spyOn, + test, +} from "bun:test"; import { createHash } from "node:crypto"; import * as fs from "node:fs/promises"; import { @@ -89,14 +96,22 @@ const artifact: NativeComposeStorageXattrArtifact = { * every run. That takes about 5-7 s on hosted CI, but 5-60 s or more on a * host whose load is several times its CPU count, so this budget is a hang * guard, not a speed assertion (HACK-1422). + * + * Nearly every case here opens a store and publishes at least one generation, + * so this is the file's default: no case keeps Bun's 5 s default or a fixed + * 30 s literal, which 17 or more cases passed at 3-4x the CPU count in load while + * their assertions held (HACK-1439). `setDefaultTimeout` is scoped to this + * file, so it changes no other file's budget. */ const STORE_CASE_BUDGET_MS = 180_000; +setDefaultTimeout(STORE_CASE_BUDGET_MS); /** - * Wall-clock lifetime of the fake carrier. Every carrier is captured inside a - * case, so its deadline is always later than that case's Bun budget: a slow - * host fails the case as a clean timeout, never by exhausting the carrier - * deadline, which refuses with the storage witness error and would change the - * asserted outcome (HACK-1422). + * Wall-clock lifetime of a carrier a case captures for real use. Every such + * carrier is captured inside a case, so its deadline is always later than that + * case's Bun budget: a slow host fails the case as a clean timeout, never by + * exhausting the carrier deadline, which refuses with the storage witness + * error and would change the asserted outcome (HACK-1422). The one case that + * mocks `Date.now` keeps its own deadline on its own clock. */ const FAKE_CARRIER_LIFETIME_MS = 2 * STORE_CASE_BUDGET_MS; const roots: string[] = []; @@ -807,7 +822,7 @@ test.each([ expect(transport.calls).not.toContain("seed"); expect(transport.calls).not.toContain("provision"); expect((await store.loadCurrent()).pending).toBeNull(); -}, 30_000); +}); test("warm enrollment proves in full when the material binding moved since the effect-entry proof", async () => { const { store, transport } = await rememberedActive(); @@ -870,7 +885,7 @@ test("warm enrollment proves in full when the material binding moved since the e } expect(transport.calls).toContain("workload"); expect(transport.calls).not.toContain("seed"); -}, 30_000); +}); test("an adjacent verification never reuses a proof across a receipt publication or a later lease", async () => { const { store, transport, generation, reference } = await rememberedActive(); @@ -935,7 +950,7 @@ test("an adjacent verification never reuses a proof across a receipt publication ).rejects.toThrow(witnessRefusal); expect(proofs()).toBe(before + 1); }); -}, 30_000); +}); test.each( (["prepublication", "effect-entry"] as const).flatMap((phase) => @@ -998,7 +1013,7 @@ test.each( phase === "prepublication" ); expect(transport.calls).not.toContain("seed"); -}, 30_000); +}); test("warm closing proof observes xattr drift during the preceding receipt await", async () => { const { store, generation, transport } = await rememberedActive(); @@ -1053,7 +1068,7 @@ test("warm closing proof observes xattr drift during the preceding receipt await expect(changed).toBe(true); expect(effects).toBe(0); expect((await store.loadCurrent()).pending).toBeNull(); -}, 30_000); +}); test.each([ "incarnation", @@ -1124,7 +1139,7 @@ test.each([ expect(changed).toBe(true); expect(effects).toBe(0); expect(transport.calls.filter((call) => call === "verify")).toHaveLength(1); -}, 30_000); +}); test("a newly observed retained birth uses the original leading proof and writing path", async () => { const { store, generation, transport } = await active(); @@ -1168,7 +1183,7 @@ test("a newly observed retained birth uses the original leading proof and writin }); }); expect(transport.calls).not.toContain("seed"); -}, 30_000); +}); test.each([ "run", @@ -1216,7 +1231,7 @@ test.each([ }, }); }); -}, 30_000); +}); test.each([ "arbitrary", @@ -1266,7 +1281,7 @@ test.each([ ); expect(effects).toBe(0); expect((await store.loadCurrent()).pending).toBeNull(); -}, 30_000); +}); test.each([ "up", @@ -1352,7 +1367,7 @@ test.each([ expect(transport.calls).not.toContain("seed"); expect(transport.calls).toContain("verify"); expect((await store.loadCurrent()).pending).toBeNull(); -}, 30_000); +}); test.each([ "up", @@ -1448,7 +1463,7 @@ test.each([ expect(verified).toBeGreaterThanOrEqual(3); expect(spawned).toBe(false); expect(transport.calls).toContain("seed"); -}, 30_000); +}); test("cold run reaches the real pre-spawn guard with exactly eleven helper calls", async () => { const store = await fixture(); @@ -1531,7 +1546,7 @@ test("cold run reaches the real pre-spawn guard with exactly eleven helper calls expect(transport.calls.filter((call) => call === "provision")).toHaveLength( 1 ); -}, 30_000); +}); test("original cold run storage authority cannot grant general effects, adoption or another generation", async () => { const store = await fixture(); @@ -1683,7 +1698,7 @@ test.each([ transport.calls.indexOf("workload") ); expect((await store.loadCurrent()).pending).toBeNull(); -}, 30_000); +}); test("cold run enrollment authority expires when its original effect returns before seed", async () => { const store = await fixture(); @@ -2173,7 +2188,7 @@ test("fresh readonly proofs tolerate observed UID change and exact owned running ); expect(transport.calls).not.toContain("provision"); expect(transport.calls).not.toContain("seed"); -}, 30_000); +}); test.each( (["up", "run"] as const).flatMap((operation) => @@ -2571,7 +2586,7 @@ test("captured artifacts/ports cannot be substituted and cancelled lifetime stop artifact: { ...artifact }, ports: { ...transport.ports }, signal: transport.controller.signal, - deadline: Date.now() + 60_000, + deadline: Date.now() + FAKE_CARRIER_LIFETIME_MS, }; const carrier = captureNativeComposeStorageXattrCarrier(mutable); mutable.artifact.helperHash = "f".repeat(64); @@ -2637,7 +2652,7 @@ test("prepare captures original carrier, selection and admission callback before const poison = captureNativeComposeStorageXattrCarrier({ artifact, signal: transport.controller.signal, - deadline: Date.now() + 60_000, + deadline: Date.now() + FAKE_CARRIER_LIFETIME_MS, ports: { inspect: async () => { substituted++; @@ -2960,7 +2975,7 @@ for (const evidence of ["issued", "copied"] as const) { } finally { commandReaderCases--; } - }, 30_000); + }); } test.each([ @@ -3023,7 +3038,7 @@ test.each([ expect(after.storageWitnessesPending).toBe(true); expect(after.pending?.token).toBe(before.pending?.token); expect(after.pending?.operation).toBe("up"); -}, 30_000); +}); test.each([ "unrecorded", @@ -3103,7 +3118,7 @@ test.each([ expect(observed).toBe(0); expect(await readFile(owned.journal, "utf8")).toBe(changed); expect(owned.transport.calls).toEqual([]); -}, 30_000); +}); test("readonly observation rejects a journal replacement across its await", async () => { const owned = await interruptedReadonlyVerification(); @@ -3135,7 +3150,7 @@ test("readonly observation rejects a journal replacement across its await", asyn expect((await lstat(owned.journal)).ino).not.toBe(owned.info.ino); expect((await owned.store.loadCurrent()).storageWitnessesPending).toBe(true); expect(owned.transport.calls).toEqual([]); -}, 30_000); +}); test("ordinary inspection authority cannot enter saved carrier recovery observation", async () => { const owned = await active(); @@ -3170,7 +3185,7 @@ test("ordinary inspection authority cannot enter saved carrier recovery observat }); }); expect(observed).toBe(0); -}, 30_000); +}); function adjacentVerifier(opts: { readonly transport: ReturnType; @@ -3211,7 +3226,7 @@ test("adjacent storage proof is reused only within its own lease", async () => { }); expect(transport.calls).not.toContain("seed"); expect(transport.calls).not.toContain("provision"); -}, 30_000); +}); test("adjacent storage proof still runs every host check and never replaces the next fresh proof", async () => { const { store, transport, generation, reference } = await active(); @@ -3243,7 +3258,7 @@ test("adjacent storage proof still runs every host check and never replaces the await expect(verify(mutation, true)).rejects.toThrow("values omitted"); }); expect(transport.calls).not.toContain("seed"); -}, 30_000); +}); test("adjacent storage proof is not reused across a changed material binding", async () => { const { store, transport, generation, reference } = await active(); @@ -3278,4 +3293,4 @@ test("adjacent storage proof is not reused across a changed material binding", a await verify(mutation, true); expect(helperOperations(transport)).toHaveLength(finished + 3); }); -}, 30_000); +}); diff --git a/tests/native-runtime-client.test.ts b/tests/native-runtime-client.test.ts index ec85d0d50..a89d944c6 100644 --- a/tests/native-runtime-client.test.ts +++ b/tests/native-runtime-client.test.ts @@ -83,73 +83,71 @@ function test( bunTest(name, body, timeoutMs); } -/** Wait for a fixture to write `path`; replaces fixed windows before acting. */ -async function waitForFile( - path: string, - budgetMs = HANDSHAKE_BUDGET_MS -): Promise { +/** + * Wait, bounded, until a fixture file at `path` holds complete content, and + * return what `complete` makes of it. A fixture's writeFileSync creates the + * file before it writes the bytes, so a wait for existence alone can return + * while the file is still "" (HACK-1422, HACK-1439). Every handshake here is + * read through this helper; no case waits for existence and then reads. + * `complete` returns undefined for content that is not complete yet. + */ +async function waitForFileContent(opts: { + readonly path: string; + readonly complete: (text: string) => T | undefined; + readonly what: string; + readonly budgetMs?: number; +}): Promise { + const budgetMs = opts.budgetMs ?? HANDSHAKE_BUDGET_MS; const deadline = performance.now() + budgetMs; - while (!(await Bun.file(path).exists())) { + for (;;) { + const file = Bun.file(opts.path); + if (await file.exists()) { + const value = opts.complete(await file.text()); + if (value !== undefined) { + return value; + } + } if (performance.now() >= deadline) { throw new Error( - `Fixture handshake did not appear within ${budgetMs} ms.` + `Fixture ${opts.what} did not complete within ${budgetMs} ms.` ); } await Bun.sleep(10); } } -/** - * Wait until a fixture's completion file holds exactly `expected`. A - * fixture's writeFileSync creates the file before it writes the bytes, so an - * existence wait can return while the file is still "" (HACK-1422). - */ +/** Wait until a fixture's completion file holds exactly `expected`. */ async function waitForFileText( path: string, expected: string, budgetMs = HANDSHAKE_BUDGET_MS ): Promise { - const deadline = performance.now() + budgetMs; - for (;;) { - const file = Bun.file(path); - if ((await file.exists()) && (await file.text()) === expected) { - return; - } - if (performance.now() >= deadline) { - throw new Error( - `Fixture completion did not write its text within ${budgetMs} ms.` - ); - } - await Bun.sleep(10); - } + await waitForFileContent({ + path, + budgetMs, + what: "completion text", + complete: (text) => (text === expected ? true : undefined), + }); } /** - * Wait until a fixture's pid handshake holds a complete pid. A fixture's - * writeFileSync creates the file before it writes the digits, so a read that - * only waited for existence can see "": Number("") is 0, and kill(0, 0) - * always succeeds, so an absence wait on it would never end. + * Wait until a fixture's pid handshake holds a complete pid. Read early, the + * file can hold "": Number("") is 0, and kill(0, 0) always succeeds, so an + * absence wait on it would never end. */ async function waitForPid( path: string, budgetMs = HANDSHAKE_BUDGET_MS ): Promise { - const deadline = performance.now() + budgetMs; - for (;;) { - const file = Bun.file(path); - if (await file.exists()) { - const pid = Number(await file.text()); - if (Number.isSafeInteger(pid) && pid > 1) { - return pid; - } - } - if (performance.now() >= deadline) { - throw new Error( - `Fixture pid handshake did not complete within ${budgetMs} ms.` - ); - } - await Bun.sleep(10); - } + return await waitForFileContent({ + path, + budgetMs, + what: "pid handshake", + complete: (text) => { + const pid = Number(text); + return Number.isSafeInteger(pid) && pid > 1 ? pid : undefined; + }, + }); } function processPresent(pid: number): boolean { @@ -518,7 +516,7 @@ ${mode === "abort" ? "await Bun.sleep(60_000);" : `(await import('node:fs')).wri try { let endedAt: number; if (mode === "abort") { - await waitForFile(holding); + await waitForPid(holding); endedAt = Date.now(); controller.abort(); await expect(request).rejects.toThrow("canceled"); @@ -734,7 +732,7 @@ process.exit(existsSync(${JSON.stringify(writerPid)})?23:3); } finally { await Bun.write(exitObserved, "observed"); await request.catch(() => undefined); - completed = await waitForFile(finished).then( + completed = await waitForFileText(finished, "finished").then( () => true, () => false ); @@ -821,12 +819,17 @@ test( const runtime = await fixture(); const marker = join(runtime.home, "attempts"); const pidFile = join(runtime.home, "pid"); + // The pid is published as a torn handshake (an empty file first), so a + // cancel that only waited for the file to exist would kill the child + // before its digits and leave "" behind on every run (HACK-1439). await Bun.write( runtime.binary, `#!${process.execPath} import { appendFileSync, writeFileSync } from "node:fs"; ${mode === "timeout" ? "await Bun.sleep(60_000);" : ""} appendFileSync(${JSON.stringify(marker)}, "attempt\\n"); +writeFileSync(${JSON.stringify(pidFile)}, ""); +await Bun.sleep(${TORN_HANDSHAKE_GAP_MS}); writeFileSync(${JSON.stringify(pidFile)}, String(process.pid)); await Bun.sleep(60_000); ` @@ -866,7 +869,7 @@ await Bun.sleep(60_000); }); } if (mode === "running") { - await waitForFile(pidFile); + await waitForPid(pidFile); controller.abort(); } await expect(attempt).rejects.toThrow( diff --git a/tests/private-env-file.test.ts b/tests/private-env-file.test.ts index a1063d2bd..2eea6ed72 100644 --- a/tests/private-env-file.test.ts +++ b/tests/private-env-file.test.ts @@ -1,4 +1,11 @@ -import { afterEach, beforeEach, expect, spyOn, test } from "bun:test"; +import { + afterEach, + beforeEach, + expect, + setDefaultTimeout, + spyOn, + test, +} from "bun:test"; import { randomUUID } from "node:crypto"; import * as privateFiles from "node:fs/promises"; import { @@ -39,6 +46,19 @@ const SENTINEL_KEY = "SYNTHETIC_PRIVATE_ENV_KEY"; const SENTINEL_VALUE = "synthetic-private-env-value-never-in-errors"; const OUTSIDE_TEXT = "OUTSIDE_SENTINEL=synthetic-outside-file-never-read\n"; +/** + * Bun timeout for every case and hook here. The cases wait on no timer: their + * cost is a fixed amount of env store work (the layer lock, a no-follow read, + * a staged write and an atomic rename per write, and for `materializeProjectEnv` + * the whole layered config resolution). That takes under 1 s at a load of + * 3x the CPU count, but the `materializeProjectEnv` cases passed Bun's 5 s + * default on a more loaded host while their assertions held (HACK-1439). This + * is a hang guard, not a speed assertion. `setDefaultTimeout` is scoped to + * this file, so it changes no other file's budget. + */ +const ENV_STORE_CASE_BUDGET_MS = 60_000; +setDefaultTimeout(ENV_STORE_CASE_BUDGET_MS); + let root: string | null = null; let previousHome: string | undefined; let previousGlobalConfigPath: string | undefined; From 3e3faa17af8a083f060efc5f1706b90fdfacac89 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Sat, 24 Oct 2026 19:34:55 -0400 Subject: [PATCH 2/2] test(native): size the xattr store case budget for sustained 8x host load (HACK-1439) Four store cases (warm no-op up/restart, warm enrollment, adjacent verification) passed the 180 s budget in every run of a wave at a sustained load average of 110-190 on an 18-CPU host, while their assertions held; the slowest took 63 s at a load of about 90. Raise STORE_CASE_BUDGET_MS to 600 s: it is a hang guard for fixed store work, not a speed assertion. The fake carrier lifetime stays twice the budget. Co-Authored-By: Claude Opus 5.5 --- ...ve-compose-storage-witness-xattr-owner.test.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/native-compose-storage-witness-xattr-owner.test.ts b/tests/native-compose-storage-witness-xattr-owner.test.ts index 2bf1903a7..ab7dd636e 100644 --- a/tests/native-compose-storage-witness-xattr-owner.test.ts +++ b/tests/native-compose-storage-witness-xattr-owner.test.ts @@ -93,17 +93,18 @@ const artifact: NativeComposeStorageXattrArtifact = { * fake carrier answers at once. Their cost is a fixed amount of durable store * file work, mostly system time: the second `up` of the same-birth case alone * makes about 19k lstat, 13k realpath and 4.5k open calls, the same count on - * every run. That takes about 5-7 s on hosted CI, but 5-60 s or more on a - * host whose load is several times its CPU count, so this budget is a hang - * guard, not a speed assertion (HACK-1422). + * every run. That takes about 5-7 s on hosted CI, but the slowest case took + * 63 s at a load average of about 90 on an 18-CPU host, and four cases passed + * 180 s at a sustained load average of 110-190 (HACK-1422, HACK-1439). The + * budget is a hang guard, not a speed assertion. * * Nearly every case here opens a store and publishes at least one generation, * so this is the file's default: no case keeps Bun's 5 s default or a fixed - * 30 s literal, which 17 or more cases passed at 3-4x the CPU count in load while - * their assertions held (HACK-1439). `setDefaultTimeout` is scoped to this - * file, so it changes no other file's budget. + * 30 s literal, which 17 or more cases passed under load while their + * assertions held (HACK-1439). `setDefaultTimeout` is scoped to this file, so + * it changes no other file's budget. */ -const STORE_CASE_BUDGET_MS = 180_000; +const STORE_CASE_BUDGET_MS = 600_000; setDefaultTimeout(STORE_CASE_BUDGET_MS); /** * Wall-clock lifetime of a carrier a case captures for real use. Every such