diff --git a/.ai/skills/hack-repo-verify/SKILL.md b/.ai/skills/hack-repo-verify/SKILL.md index 2c83dba6..ed610640 100644 --- a/.ai/skills/hack-repo-verify/SKILL.md +++ b/.ai/skills/hack-repo-verify/SKILL.md @@ -98,10 +98,27 @@ untracked candidate additions. Keep private artifacts excluded from the index. | Native authored command slices | `tests/native-authored-command.test.ts` (unsupported matrix), `tests/native-authored-command-dispatch.test.ts`, `tests/native-authored-project-{start,run,observe,logs,exec,recovery}.test.ts` and the [command contract](../../../docs/reference/native-authored-commands.md); the dispatch "source CLI" cases need the prepared compiler (`bun run build:config-compiler` or absolute `HACK_CONFIG_COMPILER_BINARY`) and fail without it; `native-authored-project-start` and `native-authored-project-exec` are nondeterministic on a local macOS host, so a single local pass or failure is not proof; live acceptance runs host-serialized on one prepared macOS provider slot | | Native VM/routing/reclamation | [candidate guide](../../../docs/guides/native-candidate.md), relevant acceptance gate and isolated host fixture; Linux checks cannot qualify macOS effects | | Resource/performance claims | [performance skill](../hack-repo-performance/SKILL.md), matched workload and measured boundaries | -| Bun runtime crashes | Read-only history: `bun run ci:bun-crash-census -- --since --until ` (failed `test` job logs; `--scan-passing` for all). Matched reproduction: the gated [experiment](../../../.github/workflows/bun-crash-experiment.yml), which spends hosted macOS minutes and needs an explicit go-ahead; never loop on a shared host. Its default `focused` mode loops `tests/native-https-owner.test.ts` (where both original segfaults happened); `full` needs a budget of at least 40 minutes because one full suite takes 30-35. A run never starts unless its estimate fits the remaining budget, so read `summary.json` for the verdict and per-mode crash, failure and skipped-for-budget counts. Both share [the classifier](../../../scripts/lib/bun-crash-classify.ts) | +| Bun runtime crashes | Read-only history: `bun run ci:bun-crash-census -- --since --until ` (failed `test` job logs; `--scan-passing` for all; since the suite moved to `test shard i/4` jobs, pass `--job "test shard i/4"` once per shard for newer runs). Matched reproduction: the gated [experiment](../../../.github/workflows/bun-crash-experiment.yml), which spends hosted macOS minutes and needs an explicit go-ahead; never loop on a shared host. Its default `focused` mode loops `tests/native-https-owner.test.ts` (where both original segfaults happened); `full` needs a budget of at least 40 minutes because one full suite takes 30-35. A run never starts unless its estimate fits the remaining budget, so read `summary.json` for the verdict and per-mode crash, failure and skipped-for-budget counts. Both share [the classifier](../../../scripts/lib/bun-crash-classify.ts) | Read [CI](../../../.github/workflows/ci.yml) for current hosted gates. A local pass -does not mean CI ran. Existing unsupported remote/gateway/native-app checks are +does not mean CI ran. + +Hosted CI runs the `bun run test` file set on four `test shard i/4` jobs; +the required `test` check is their fan-in and passes only when every shard did. +[`scripts/test-shard-plan.ts`](../../../scripts/test-shard-plan.ts) splits the files +by the durations in `scripts/test-shard-durations.json`. A file with no recorded +duration is planned at 5 s. Each shard runs each of its files in its own +`bun test ./` process and prints a start line and a result line per file. +It stops a file after 12 min, and stops the shard at 30 min or on cancellation. +When it stops early it names the running file, the files not started and the +slowest files. Each shard first checks that the four shards cover the files +`git ls-files` lists exactly once. To reproduce one shard locally, run +`bun scripts/test-shard.ts plan --shard=i/4` or +`bun scripts/test-shard.ts run --shard=i/4`. To check the split after adding +or moving tests, run `bun scripts/test-shard.ts verify --count=4`. To refresh the +durations, download the `test-shard-durations-` artifacts and run +`bun scripts/test-shard.ts update-durations --source= `. A per-file +process does not share module state with other files, unlike one `bun test` run. Existing unsupported remote/gateway/native-app checks are not default local-product requirements; use them only for explicitly scoped work. ## Verify the verifier diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d8a2d828..1ad31b59 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -361,9 +361,21 @@ jobs: exit 1 fi - test: + test-shard: + # One deterministic slice of the `bun run test` file set per runner + # (scripts/test-shard-plan.ts, HACK-1381). The serial job took 37-42 min and + # reached the 60 min limit on slow runners; each file now runs in its own + # `bun test` process with a start/result line, so a hang names its file. + name: test shard ${{ matrix.shard }}/4 + strategy: + # Let every shard finish so one failure still reports the other files. + fail-fast: false + matrix: + shard: [1, 2, 3, 4] runs-on: blacksmith-6vcpu-macos-15 - timeout-minutes: 60 + # A shard plans about 10.5 min of tests. The runner's own 30 min deadline + # stops first and names the running file; this bounds a hung runner. + timeout-minutes: 40 steps: - name: Checkout uses: actions/checkout@v4 @@ -373,22 +385,61 @@ jobs: bun-version: "1.4.2" - name: Install dependencies run: bun install --frozen-lockfile + # The one-off checks and builds run once, on shard 1. - name: Typecheck (Turbo) + if: matrix.shard == 1 run: bun run turbo:typecheck - name: Privacy check + if: matrix.shard == 1 run: bun run privacy:check - name: Quality checks (Turbo) + if: matrix.shard == 1 run: bun run turbo:check - uses: dtolnay/rust-toolchain@1.97.1 - name: Prepare matching compiler for private file owner tests run: bun run build:config-compiler - - name: Tests (Turbo) - run: bun run turbo:test + # Runs check:bun-version and test:preflight as `bun run test` does, then + # verifies that the four shards cover the full test file set exactly once. + - name: Tests (shard) + env: + HACK_TEST_SHARD: ${{ matrix.shard }}/4 + run: bun scripts/test-shard.ts run --shard="$HACK_TEST_SHARD" --durations-out="$RUNNER_TEMP/test-shard-durations.json" + - name: Preserve measured file durations + # Per-file seconds only; refresh scripts/test-shard-durations.json with + # `bun scripts/test-shard.ts update-durations`. + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-shard-durations-${{ matrix.shard }} + path: ${{ runner.temp }}/test-shard-durations.json + if-no-files-found: ignore + retention-days: 14 - name: Build CLI + if: matrix.shard == 1 run: bun run build - name: Build release smoke (no tests) + if: matrix.shard == 1 run: bun run build:release --skip-tests --no-clean --out=dist/release-ci + test: + # The required `test` check (repository ruleset) for `next` and `main`. It + # runs even when a shard fails or is cancelled and passes only when every + # shard succeeded, so a failed shard can never surface as a skipped check. + needs: test-shard + if: ${{ always() }} + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: {} + steps: + - name: Require every test shard to succeed + env: + SHARD_RESULT: ${{ needs.test-shard.result }} + run: | + if [ "$SHARD_RESULT" != "success" ]; then + echo "test shards did not all succeed (result: $SHARD_RESULT)" + exit 1 + fi + linux-process-lifetime: runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 15 diff --git a/scripts/prerelease-plan.ts b/scripts/prerelease-plan.ts index 1fbebbb7..46636c2a 100644 --- a/scripts/prerelease-plan.ts +++ b/scripts/prerelease-plan.ts @@ -223,7 +223,7 @@ export function requireHumanApproval({ } } -/** Jobs that must each succeed on the exact-head CI run; `docker-e2e` is the shard fan-in. */ +/** Jobs that must each succeed on the exact-head CI run; `docker-e2e` and `test` are shard fan-ins. */ export const CI_JOBS = [ "Runtime state models", "Candidate core (ubuntu-latest)", @@ -235,6 +235,10 @@ export const CI_JOBS = [ "docker-e2e shard 3/4", "docker-e2e shard 4/4", "docker-e2e", + "test shard 1/4", + "test shard 2/4", + "test shard 3/4", + "test shard 4/4", "test", "linux-process-lifetime", ] as const; diff --git a/scripts/test-shard-durations.json b/scripts/test-shard-durations.json new file mode 100644 index 00000000..36fc920f --- /dev/null +++ b/scripts/test-shard-durations.json @@ -0,0 +1,575 @@ +{ + "schema": "hack.test-shard-durations.v1", + "source": "Sum of per-test times from the serial hosted test job (blacksmith-6vcpu-macos-15, Bun 1.4.2), CI run 39873410893 job 120153061379 on next 4e0bfa55; excludes per-file load and hook time", + "seconds": { + "packages/db/tests/smoke.test.ts": 0, + "tests/aes-gcm-v1-context-binding.test.ts": 0.3, + "tests/agent-client-capabilities.test.ts": 0, + "tests/agent-instruction-source.test.ts": 0, + "tests/agent-onboard-command.test.ts": 0, + "tests/agent-plugin-generation.test.ts": 3.5, + "tests/branch-hosts.test.ts": 0, + "tests/branches-effective-branch.test.ts": 0.6, + "tests/branches.test.ts": 0, + "tests/bun-crash-loop.test.ts": 0.2, + "tests/bun-version-guard.test.ts": 0, + "tests/caddy-ca-export.test.ts": 0, + "tests/caddy-ca-identity.test.ts": 0, + "tests/checkout-write-guard.test.ts": 2.4, + "tests/ci-bun-crash-census.test.ts": 0, + "tests/cli-closed-error-output.test.ts": 0, + "tests/cli-command.test.ts": 0, + "tests/cli-help.test.ts": 0, + "tests/cli-passthrough-argv.test.ts": 2, + "tests/cli-per-command-options.test.ts": 0.1, + "tests/cli-reference-docs.test.ts": 0, + "tests/cli-renderer-loading.test.ts": 2.8, + "tests/cli-result.test.ts": 0, + "tests/cli-spawn-env-isolation.test.ts": 0.5, + "tests/clipboard.test.ts": 0, + "tests/closed-refusal-registry.test.ts": 0, + "tests/cloudflare-commands.test.ts": 0, + "tests/codex-slim-maintenance.test.ts": 0.1, + "tests/compose-env-at-rest-cli.test.ts": 10.1, + "tests/compose-env-at-rest.test.ts": 0.1, + "tests/compose-env-reference-least-privilege.test.ts": 0, + "tests/compose-env-shared-references.test.ts": 0, + "tests/compose-env-spawn-size-cli.test.ts": 0.7, + "tests/compose-env-staleness.test.ts": 0.1, + "tests/compose-managed-env.test.ts": 0, + "tests/compose-startup-budget.test.ts": 0, + "tests/compose-startup-phases.test.ts": 3.7, + "tests/compose.test.ts": 0, + "tests/config-command.test.ts": 0, + "tests/config-compiler-build.test.ts": 0, + "tests/config-paths.test.ts": 0, + "tests/config.test.ts": 0, + "tests/control-plane-config.test.ts": 0, + "tests/control-plane-extensions.test.ts": 0, + "tests/coredns-config.test.ts": 0, + "tests/crash-capture.test.ts": 0, + "tests/daemon-autostart-bounded.test.ts": 0, + "tests/daemon-client.test.ts": 0, + "tests/daemon-command.test.ts": 0.2, + "tests/daemon-env.test.ts": 0.2, + "tests/daemon-orphan.test.ts": 0, + "tests/daemon-paths.test.ts": 0, + "tests/daemon-sessions.test.ts": 0, + "tests/daemon-status.test.ts": 0, + "tests/dependabot-inventory.test.ts": 0, + "tests/dependency-cache-protocol-override.test.ts": 0, + "tests/dependency-cache-protocol.test.ts": 0, + "tests/dependency-cache.test.ts": 0.1, + "tests/dependency-startup-progress.test.ts": 1, + "tests/dev-wrapper.test.ts": 0, + "tests/discovery.test.ts": 0, + "tests/dispatch-e2e.test.ts": 0, + "tests/display.test.ts": 0, + "tests/disposable-cache-volumes.test.ts": 0, + "tests/docker-context.test.ts": 0, + "tests/docker-events.test.ts": 0, + "tests/docker-logs.test.ts": 0, + "tests/doctor-adopted-project.test.ts": 8.9, + "tests/doctor-browser-network.test.ts": 0.2, + "tests/doctor-command.test.ts": 0.5, + "tests/doctor-domain-migration.test.ts": 1.1, + "tests/doctor-env-key.test.ts": 0.7, + "tests/doctor-fix-adopted-project.test.ts": 9.7, + "tests/doctor-fix-noninteractive.test.ts": 46.2, + "tests/doctor-generated-files.test.ts": 0.3, + "tests/doctor-host-trust.test.ts": 0, + "tests/doctor-json.test.ts": 0, + "tests/doctor-local-domain.test.ts": 0, + "tests/doctor-utils.test.ts": 0, + "tests/doctor-worktree.test.ts": 0.2, + "tests/e2e-capabilities.test.ts": 0, + "tests/e2e-harness-stream.test.ts": 0.2, + "tests/e2e-launchd-guard.test.ts": 0, + "tests/e2e-portable-bootstrap-controls.test.ts": 0, + "tests/e2e-scenario-selection.test.ts": 0, + "tests/e2e-shard.test.ts": 0.4, + "tests/e2e/exec-stale-env-output.test.ts": 0, + "tests/e2e/pattern-fixture.test.ts": 0, + "tests/e2e/run-dependency-cache.test.ts": 3.5, + "tests/env-backend-command.test.ts": 4.7, + "tests/env-command-matrix.test.ts": 0, + "tests/env-command-modern.test.ts": 0, + "tests/env-exec-command.test.ts": 0.2, + "tests/env-get.test.ts": 1.3, + "tests/env-hydrate-input.test.ts": 0, + "tests/env-key-argument-redaction-cli.test.ts": 29, + "tests/env-key-rotation-model.test.ts": 0, + "tests/env-secret-sentinel-scan.test.ts": 0.2, + "tests/env-selection-parity.test.ts": 0.2, + "tests/env-value-stdin-cli.test.ts": 2.8, + "tests/env-value-stdin.test.ts": 0, + "tests/env.test.ts": 0, + "tests/experimental-gating.test.ts": 0, + "tests/focused-skills.test.ts": 0, + "tests/fzf.test.ts": 0, + "tests/gateway-client.test.ts": 0, + "tests/gateway-e2e.test.ts": 0, + "tests/gateway-tokens.test.ts": 0, + "tests/git-info-exclude.test.ts": 0.1, + "tests/git-registration-metadata.test.ts": 0.8, + "tests/global-command.macos.test.ts": 0.1, + "tests/global-command.test.ts": 0, + "tests/global-dns-authorizer-wiring.test.ts": 0, + "tests/global-service-hosts.test.ts": 0, + "tests/guards.test.ts": 0, + "tests/gum-end-of-options.test.ts": 0, + "tests/gum.test.ts": 0, + "tests/hack-env.test.ts": 0, + "tests/hack-gitignore.test.ts": 0.2, + "tests/hack-init-skill.test.ts": 0, + "tests/host-command-observation.test.ts": 0, + "tests/host-exec-lifetime.test.ts": 6.6, + "tests/host-exec-tty.test.ts": 5.4, + "tests/init-agent-integrations.test.ts": 0, + "tests/init-assistant.test.ts": 0.1, + "tests/init-discovery-validation.test.ts": 0, + "tests/init-with-command.test.ts": 0.1, + "tests/install-detection.test.ts": 0, + "tests/integration-ownership.test.ts": 0, + "tests/integration-sync.test.ts": 0, + "tests/interactivity.test.ts": 0, + "tests/internal-extra-hosts-worktree.test.ts": 1, + "tests/internal-tls-environment.test.ts": 0, + "tests/launchd-isolation.test.ts": 0.6, + "tests/launchd.test.ts": 0, + "tests/legacy-agent-artifacts.test.ts": 0, + "tests/lifecycle-git-status-clean.test.ts": 10.7, + "tests/lifecycle-json.test.ts": 0.2, + "tests/lifecycle-process-delivery.test.ts": 0, + "tests/linux-trust-bundle.test.ts": 0, + "tests/local-network-hint-call-sites.test.ts": 1.7, + "tests/local-network-hint.test.ts": 0.3, + "tests/log-backend.test.ts": 0, + "tests/log-group.test.ts": 0, + "tests/log-json.test.ts": 0, + "tests/log-pipe.test.ts": 0, + "tests/log-stream.test.ts": 0, + "tests/logs-loki-empty-fallback-cli.test.ts": 0.7, + "tests/logs-utils.test.ts": 0, + "tests/loki-short-lived-logs.test.ts": 0, + "tests/mac-ca-trust.test.ts": 0, + "tests/macos-ghostty-source.test.ts": 0, + "tests/mcp-artifact-info.test.ts": 0, + "tests/mcp-bundle-activation.test.ts": 0.3, + "tests/mcp-bundle-client.test.ts": 0, + "tests/mcp-bundle.test.ts": 5.2, + "tests/mcp-codex-removal.test.ts": 0, + "tests/mcp-command-admission.test.ts": 0, + "tests/mcp-entrypoint.test.ts": 0.6, + "tests/mcp-inspect-tools.test.ts": 0.2, + "tests/mcp-install.test.ts": 0, + "tests/mcp-lifetime.test.ts": 20.8, + "tests/mcp-logs-contract.test.ts": 4.6, + "tests/mcp-managed-startup.test.ts": 0, + "tests/mcp-native-adapter.test.ts": 0, + "tests/mcp-native-candidate.test.ts": 0, + "tests/mcp-native-owner.test.ts": 0, + "tests/mcp-output-budget.test.ts": 0, + "tests/mcp-prompts.test.ts": 0, + "tests/mcp-publication-readiness.test.ts": 1.5, + "tests/mcp-repeat-install.test.ts": 2.8, + "tests/mcp-session-context.test.ts": 0.3, + "tests/mcp-socket-backend.test.ts": 3.9, + "tests/mcp-startup-contract.test.ts": 0.3, + "tests/mcp-startup-supervision.test.ts": 0, + "tests/mcp.test.ts": 0.1, + "tests/missing-project-error-code.test.ts": 1.3, + "tests/multi-node-harness.test.ts": 0, + "tests/mutagen-binary.test.ts": 0, + "tests/mutagen-sync.test.ts": 0, + "tests/native-acquisition-plan-transport.test.ts": 0.7, + "tests/native-acquisition-project.test.ts": 0.3, + "tests/native-admin-authorization.test.ts": 0, + "tests/native-authored-command-dispatch.test.ts": 4.1, + "tests/native-authored-command.test.ts": 0, + "tests/native-authored-graph-protocol.test.ts": 0, + "tests/native-authored-hook-stop.test.ts": 0.1, + "tests/native-authored-host-processes.test.ts": 3, + "tests/native-authored-live-stop.test.ts": 1.4, + "tests/native-authored-mutation-lease.test.ts": 0, + "tests/native-authored-process-census.test.ts": 0.1, + "tests/native-authored-process-incarnation.test.ts": 0.1, + "tests/native-authored-project-exec.test.ts": 1.8, + "tests/native-authored-project-logs.test.ts": 4.2, + "tests/native-authored-project-observe.test.ts": 2.5, + "tests/native-authored-project-oneoff.test.ts": 1.2, + "tests/native-authored-project-process.test.ts": 3, + "tests/native-authored-project-recovery.test.ts": 8.2, + "tests/native-authored-project-restart.test.ts": 0, + "tests/native-authored-project-run.test.ts": 1.2, + "tests/native-authored-project-start.test.ts": 31, + "tests/native-authored-recovery-protocol.test.ts": 0, + "tests/native-authored-runtime-metadata.test.ts": 0, + "tests/native-authored-storage-tool.test.ts": 0, + "tests/native-aws-environment.test.ts": 0, + "tests/native-ca-doctor-cli.test.ts": 0.7, + "tests/native-ca-doctor.test.ts": 0, + "tests/native-ca-live-rotation.test.ts": 0, + "tests/native-ca-rotation-wire.test.ts": 0.2, + "tests/native-candidate-authored-artifact.test.ts": 0.1, + "tests/native-candidate-launcher.test.ts": 0.1, + "tests/native-compose-adoption-binding.test.ts": 18.8, + "tests/native-compose-adoption-branch-fixture.test.ts": 0, + "tests/native-compose-adoption-branch.test.ts": 0, + "tests/native-compose-adoption-build-diagnostics.test.ts": 0, + "tests/native-compose-adoption-build-evidence.test.ts": 0, + "tests/native-compose-adoption-build-fixture.test.ts": 3.4, + "tests/native-compose-adoption-build-format.test.ts": 0, + "tests/native-compose-adoption-build-generation.test.ts": 23.6, + "tests/native-compose-adoption-build-recovery-transport.test.ts": 0.3, + "tests/native-compose-adoption-build-source.test.ts": 0.6, + "tests/native-compose-adoption-checkout.test.ts": 1, + "tests/native-compose-adoption-compiler.test.ts": 0.1, + "tests/native-compose-adoption-dependency-fixture.test.ts": 0, + "tests/native-compose-adoption-dependency-staged-read.test.ts": 0.1, + "tests/native-compose-adoption-diagnostics.test.ts": 0, + "tests/native-compose-adoption-env-inputs.test.ts": 5.8, + "tests/native-compose-adoption-execution.test.ts": 0.5, + "tests/native-compose-adoption-file-generation.test.ts": 45.7, + "tests/native-compose-adoption-files.test.ts": 0.1, + "tests/native-compose-adoption-generation.test.ts": 317.2, + "tests/native-compose-adoption-job-diagnostics.test.ts": 0, + "tests/native-compose-adoption-job-inputs.test.ts": 0.1, + "tests/native-compose-adoption-job-worktrees.test.ts": 6.8, + "tests/native-compose-adoption-jobs.test.ts": 0.6, + "tests/native-compose-adoption-local-fixture.test.ts": 7.8, + "tests/native-compose-adoption-local.test.ts": 2.6, + "tests/native-compose-adoption-managed-fixture.test.ts": 10.4, + "tests/native-compose-adoption-mount-format.test.ts": 0, + "tests/native-compose-adoption-plan.test.ts": 0, + "tests/native-compose-adoption-probe.test.ts": 0.6, + "tests/native-compose-adoption-projection.test.ts": 2, + "tests/native-compose-adoption-publication-diagnostics.test.ts": 0, + "tests/native-compose-adoption-routing-execution.test.ts": 0.2, + "tests/native-compose-adoption-routing-fixture.test.ts": 0, + "tests/native-compose-adoption-routing-generation.test.ts": 128.9, + "tests/native-compose-adoption-routing-receipt.test.ts": 0, + "tests/native-compose-adoption-routing-resolution.test.ts": 0, + "tests/native-compose-adoption-secret-permissions.test.ts": 0.1, + "tests/native-compose-adoption-source-bind-fixture.test.ts": 1.6, + "tests/native-compose-adoption-source-bind-generation.test.ts": 37.9, + "tests/native-compose-adoption-source-bind.test.ts": 0.4, + "tests/native-compose-adoption-worktrees-fixture.test.ts": 0.1, + "tests/native-compose-after-command.test.ts": 15.9, + "tests/native-compose-backend-features-cli.test.ts": 2.7, + "tests/native-compose-backend-features.test.ts": 0, + "tests/native-compose-before-command.test.ts": 2.5, + "tests/native-compose-build-command.test.ts": 9.1, + "tests/native-compose-build.test.ts": 0.4, + "tests/native-compose-completion.test.ts": 0, + "tests/native-compose-down-command.test.ts": 55.1, + "tests/native-compose-down-hooks.test.ts": 0, + "tests/native-compose-effect-diagnostics.test.ts": 0, + "tests/native-compose-engine-identity.test.ts": 0.3, + "tests/native-compose-file-cold-home.test.ts": 1.3, + "tests/native-compose-file-command.test.ts": 67, + "tests/native-compose-file-inventory.test.ts": 0, + "tests/native-compose-file-owner.test.ts": 20, + "tests/native-compose-file-permissions.test.ts": 0, + "tests/native-compose-file-sources.test.ts": 5, + "tests/native-compose-fixture-volume.test.ts": 0, + "tests/native-compose-fixture.test.ts": 0, + "tests/native-compose-generation.test.ts": 12.1, + "tests/native-compose-host-hooks.test.ts": 0.2, + "tests/native-compose-ingress.test.ts": 0.3, + "tests/native-compose-inputs.test.ts": 2.4, + "tests/native-compose-material-authority.test.ts": 0.3, + "tests/native-compose-network-command.test.ts": 1.9, + "tests/native-compose-networks.test.ts": 0, + "tests/native-compose-open.test.ts": 7.8, + "tests/native-compose-ownership.test.ts": 11.1, + "tests/native-compose-phase-summary.test.ts": 0, + "tests/native-compose-phase-trace.test.ts": 3, + "tests/native-compose-proxy-probe.test.ts": 15.9, + "tests/native-compose-proxy-routes.test.ts": 0, + "tests/native-compose-readiness-bounds.test.ts": 2.3, + "tests/native-compose-renderer-config.test.ts": 0, + "tests/native-compose-renderer.test.ts": 0, + "tests/native-compose-retained-storage-command.test.ts": 128.6, + "tests/native-compose-retained-storage.test.ts": 0, + "tests/native-compose-route-claims.test.ts": 9.2, + "tests/native-compose-route-inventory.test.ts": 0.5, + "tests/native-compose-route-owner.test.ts": 13, + "tests/native-compose-routed-run-command.test.ts": 16.2, + "tests/native-compose-routing-command.test.ts": 7.7, + "tests/native-compose-routing.test.ts": 0, + "tests/native-compose-run-projection.test.ts": 1.4, + "tests/native-compose-runtime-metadata-command.test.ts": 2.6, + "tests/native-compose-saved-command.test.ts": 8.5, + "tests/native-compose-saved-exec-storage.test.ts": 0, + "tests/native-compose-selection.test.ts": 0, + "tests/native-compose-start-storage-exclusion.test.ts": 23.4, + "tests/native-compose-storage-activation-command.test.ts": 327.6, + "tests/native-compose-storage-carrier-journal-completion.test.ts": 0.1, + "tests/native-compose-storage-command-executable.test.ts": 0, + "tests/native-compose-storage-command-reader.test.ts": 0.6, + "tests/native-compose-storage-command-record.test.ts": 17.1, + "tests/native-compose-storage-fence-cost.test.ts": 125.3, + "tests/native-compose-storage-witness-codec.test.ts": 0, + "tests/native-compose-storage-witness-command.test.ts": 4.9, + "tests/native-compose-storage-witness-dependency.test.ts": 0, + "tests/native-compose-storage-witness-docker-io.test.ts": 0, + "tests/native-compose-storage-witness-docker.test.ts": 1.4, + "tests/native-compose-storage-witness-xattr-owner.test.ts": 199.7, + "tests/native-compose-storage-witness-xattr.test.ts": 0, + "tests/native-compose-storage-witness.test.ts": 6.7, + "tests/native-compose-vm-effect-diagnostics.test.ts": 0, + "tests/native-compose-vm-file-client-selection.test.ts": 0, + "tests/native-compose-vm-file-client.test.ts": 1.8, + "tests/native-compose-vm-file-finalization.test.ts": 15.1, + "tests/native-compose-vm-file-image-format.test.ts": 0, + "tests/native-compose-vm-file-isolation.test.ts": 0, + "tests/native-compose-vm-file-owner.test.ts": 52.9, + "tests/native-compose-vm-file-prepending.test.ts": 10.5, + "tests/native-compose-vm-file-protocol.test.ts": 0.2, + "tests/native-compose-wait-ready.test.ts": 0.5, + "tests/native-compose-warm-up-command.test.ts": 33, + "tests/native-config-build.test.ts": 0, + "tests/native-config-command.test.ts": 1.5, + "tests/native-config-compiler.test.ts": 1.2, + "tests/native-config-dto.test.ts": 0, + "tests/native-config-files-evidence.test.ts": 0, + "tests/native-config-import-build.test.ts": 0.6, + "tests/native-config-import-files.test.ts": 0, + "tests/native-config-import-jobs.test.ts": 0, + "tests/native-config-import-plan.test.ts": 0, + "tests/native-config-import-preview.test.ts": 0.7, + "tests/native-config-import-readiness.test.ts": 0, + "tests/native-config-import-routing.test.ts": 0, + "tests/native-config-import-storage.test.ts": 0, + "tests/native-config-process-policy.test.ts": 0, + "tests/native-config-protected-files-fixture.test.ts": 1.7, + "tests/native-config-protected-mount-format.test.ts": 0, + "tests/native-config-protected-source.test.ts": 0, + "tests/native-config-validate-image-admission.test.ts": 1.2, + "tests/native-cpu-accounting.test.ts": 2.3, + "tests/native-cpu-diagnostics.test.ts": 0, + "tests/native-cpu-report-fd.test.ts": 0.1, + "tests/native-domain-dns-activate.test.ts": 0, + "tests/native-domain-dns-authorization.test.ts": 0, + "tests/native-domain-dns-deactivate.test.ts": 0, + "tests/native-domain-dns-host.test.ts": 3.3, + "tests/native-domain-dns-launchd.test.ts": 0.3, + "tests/native-domain-dns-plan.test.ts": 0, + "tests/native-domain-dns-snapshot.test.ts": 0, + "tests/native-endpoint-plan-transport.test.ts": 1.3, + "tests/native-endpoint-project.test.ts": 0.3, + "tests/native-engine-tripwire.test.ts": 0, + "tests/native-env-metadata.test.ts": 1.3, + "tests/native-env-plan-transport.test.ts": 1.7, + "tests/native-env-values.test.ts": 2.6, + "tests/native-exec-environment.test.ts": 0.1, + "tests/native-exec-stale-env.test.ts": 0.1, + "tests/native-file-plan-protocol.test.ts": 0.1, + "tests/native-hack-local-route-cli.test.ts": 0.9, + "tests/native-hack-local-route.test.ts": 0.2, + "tests/native-host-hook-runner.test.ts": 2.4, + "tests/native-host-lifecycle-contract.test.ts": 0, + "tests/native-host-plan-transport.test.ts": 1.3, + "tests/native-https-empty-owner-recovery.test.ts": 0.3, + "tests/native-https-owner-protocol.test.ts": 0.1, + "tests/native-https-owner.test.ts": 16.6, + "tests/native-https-port.test.ts": 0, + "tests/native-https-release-failure.test.ts": 1, + "tests/native-https-startup-failure.test.ts": 0, + "tests/native-network-plan-transport.test.ts": 0.9, + "tests/native-network-project.test.ts": 0.3, + "tests/native-networks-fixture.test.ts": 4, + "tests/native-plan-cli-wiring.test.ts": 0, + "tests/native-plan-graph-contract.test.ts": 0, + "tests/native-prepared-base.test.ts": 0, + "tests/native-private-lock-recovery.test.ts": 0.6, + "tests/native-process-census-diagnostics.test.ts": 0, + "tests/native-process-plan-transport.test.ts": 0.9, + "tests/native-process-policy-initial-trace.test.ts": 2.1, + "tests/native-process-policy-replay-reason.test.ts": 0, + "tests/native-process-policy-startup-diagnostic.test.ts": 0, + "tests/native-process-project.test.ts": 0.4, + "tests/native-project-adaptation.test.ts": 0, + "tests/native-project-branch-cli.test.ts": 0.9, + "tests/native-project-branch.test.ts": 0.1, + "tests/native-project-cache.test.ts": 0, + "tests/native-project-cleanup-command.test.ts": 0.6, + "tests/native-project-dependencies.test.ts": 0.5, + "tests/native-project-dependency-refresh.test.ts": 0.1, + "tests/native-project-down.test.ts": 1.5, + "tests/native-project-exec.test.ts": 0.4, + "tests/native-project-finalization.test.ts": 0.1, + "tests/native-project-https.test.ts": 0.8, + "tests/native-project-input.test.ts": 0.1, + "tests/native-project-inputs-acquisition-race.test.ts": 0, + "tests/native-project-inputs.test.ts": 1.9, + "tests/native-project-inspect.test.ts": 0.1, + "tests/native-project-interrupted-cleanup.test.ts": 0, + "tests/native-project-lifecycle.test.ts": 0.2, + "tests/native-project-mapping-command.test.ts": 0, + "tests/native-project-network.test.ts": 0, + "tests/native-project-observe.test.ts": 1.1, + "tests/native-project-one-off.test.ts": 0.1, + "tests/native-project-process.test.ts": 6.8, + "tests/native-project-recovery.test.ts": 0, + "tests/native-project-restart.test.ts": 0, + "tests/native-project-restore.test.ts": 0, + "tests/native-project-retained-images.test.ts": 0, + "tests/native-project-retained-startup.test.ts": 0, + "tests/native-project-review.test.ts": 0.1, + "tests/native-project-routing.test.ts": 0, + "tests/native-project-run-filesystem.test.ts": 0.1, + "tests/native-project-run.test.ts": 0.1, + "tests/native-project-source-preflight.test.ts": 0, + "tests/native-project-start.test.ts": 0.8, + "tests/native-refusal-cli.test.ts": 3.6, + "tests/native-retained-jobs-model.test.ts": 0, + "tests/native-routed-down-hooks-fixture.test.ts": 0.2, + "tests/native-routed-run-fixture.test.ts": 0.1, + "tests/native-routing-fixture.test.ts": 0, + "tests/native-routing-input-race.test.ts": 0, + "tests/native-routing-inputs.test.ts": 0.6, + "tests/native-routing-plan-transport.test.ts": 3.7, + "tests/native-routing-project.test.ts": 1.7, + "tests/native-run-selection-cli.test.ts": 1.9, + "tests/native-runtime-client.test.ts": 1.6, + "tests/native-runtime-metadata.test.ts": 0, + "tests/native-runtime-not-running-cli.test.ts": 1.5, + "tests/native-runtime-preflight.test.ts": 0, + "tests/native-selection-refusals.test.ts": 0, + "tests/native-up-env-fingerprint-wiring.test.ts": 0, + "tests/native-up-runtime-metadata-selection.test.ts": 0, + "tests/native-vm-file-acceptance.test.ts": 0.2, + "tests/native-workspace-cache.test.ts": 0, + "tests/node-attach-instructions.test.ts": 0, + "tests/node-pair-command.test.ts": 0, + "tests/node-pairings-registry.test.ts": 0, + "tests/node-provider-railway.test.ts": 0, + "tests/node-status-auth-cache.test.ts": 0, + "tests/node-workspace-bootstrap.test.ts": 0.3, + "tests/node-workspace-command.test.ts": 0, + "tests/node-workspace-map.test.ts": 0, + "tests/nodes-registry.test.ts": 0, + "tests/onboarding-entry-parity.test.ts": 1.5, + "tests/onboarding-handoff.test.ts": 0, + "tests/onboarding-prompt.test.ts": 0, + "tests/open-host.test.ts": 0, + "tests/output-hygiene.test.ts": 0, + "tests/pattern-catalogue.test.ts": 0, + "tests/pattern-examples.test.ts": 0.1, + "tests/plugin-migration.test.ts": 0.1, + "tests/policy-audit.test.ts": 0, + "tests/policy-risk.test.ts": 0, + "tests/portable-bootstrap-env-refusal.test.ts": 1, + "tests/portable-bootstrap-smoke.test.ts": 1.8, + "tests/prerelease-install.test.ts": 3.7, + "tests/prerelease-plan.test.ts": 1.1, + "tests/prerequisites-matrix.test.ts": 0, + "tests/private-env-file.test.ts": 0.1, + "tests/project-config-schema.test.ts": 0, + "tests/project-config.test.ts": 0, + "tests/project-domain-command.test.ts": 0.1, + "tests/project-domain-migration.test.ts": 0.3, + "tests/project-domain-plan.test.ts": 0, + "tests/project-down-worktree-safety.test.ts": 0.5, + "tests/project-env-config.test.ts": 1.5, + "tests/project-env-hydrate.test.ts": 2, + "tests/project-env-local-inheritance.test.ts": 1.1, + "tests/project-env-metadata.test.ts": 0.2, + "tests/project-env-stable-ciphertext.test.ts": 1.3, + "tests/project-env-worktree-key.test.ts": 2.1, + "tests/project-exec-command.test.ts": 0.2, + "tests/project-execution-routing.test.ts": 0, + "tests/project-init-root.test.ts": 0, + "tests/project-input-selection.test.ts": 0, + "tests/project-lifecycle-env.test.ts": 0, + "tests/project-lifecycle-hygiene.test.ts": 0, + "tests/project-lifecycle-processes.test.ts": 0, + "tests/project-lifecycle-sessions.test.ts": 0, + "tests/project-lifecycle-singleton.test.ts": 0, + "tests/project-meta.test.ts": 0, + "tests/project-name-command.test.ts": 7.2, + "tests/project-name-registry.test.ts": 0.2, + "tests/project-owner-command.test.ts": 0, + "tests/project-restart-command.test.ts": 0, + "tests/project-run-command.test.ts": 0.2, + "tests/project-runtime-hygiene.test.ts": 0, + "tests/project-startup-state.test.ts": 0, + "tests/project-target-argument-redaction-cli.test.ts": 8.6, + "tests/project-target-inspect.test.ts": 7.5, + "tests/project-task-catalog.test.ts": 6.6, + "tests/project-up-command.test.ts": 0, + "tests/project-up-startup-state.test.ts": 3, + "tests/project-views.test.ts": 0, + "tests/projects-command.test.ts": 0, + "tests/projects-prune.test.ts": 0, + "tests/projects-registry-admission.test.ts": 2.6, + "tests/projects-registry-identity.test.ts": 0.5, + "tests/projects-registry-lock.test.ts": 2.8, + "tests/projects-registry-optional-lock.test.ts": 0.1, + "tests/projects-registry-worktrees.test.ts": 0.2, + "tests/projects-registry.test.ts": 0.3, + "tests/provider-route-resolver.test.ts": 0, + "tests/qr.test.ts": 0, + "tests/refresh-scheduler.test.ts": 0.1, + "tests/registry-credential-preflight.test.ts": 0, + "tests/registry-touch-worktrees.test.ts": 0.5, + "tests/release-boundary.test.ts": 0, + "tests/release-config.test.ts": 0.2, + "tests/release-download-installer.test.ts": 0.1, + "tests/remote-caddy-routes.test.ts": 0, + "tests/run-child-exit-status.test.ts": 1.6, + "tests/run-exec-branch-default.test.ts": 0.3, + "tests/runtime-backend.test.ts": 0, + "tests/runtime-build-output.test.ts": 0, + "tests/runtime-cache.test.ts": 0, + "tests/runtime-guidance.test.ts": 0, + "tests/runtime-host-metadata.test.ts": 0, + "tests/runtime-projects.test.ts": 0, + "tests/runtime-routing-config.test.ts": 0, + "tests/runtime-turbo-inputs.test.ts": 0.9, + "tests/secret-key-provisioning-convergence.test.ts": 0.7, + "tests/secret-store-file-hygiene.test.ts": 1, + "tests/secret-store.test.ts": 0.2, + "tests/self-update.test.ts": 0, + "tests/session-command.test.ts": 0, + "tests/session-env-command.test.ts": 0, + "tests/session-help.test.ts": 0, + "tests/session-names.test.ts": 0, + "tests/session-utils.test.ts": 0, + "tests/setup-migrate-command.test.ts": 1, + "tests/setup-sync.test.ts": 0, + "tests/setup.test.ts": 0, + "tests/shared-agent-skill.test.ts": 0, + "tests/shared-host-lifecycle.test.ts": 15.7, + "tests/shell-abort.test.ts": 13.2, + "tests/shell-bounded-stream.test.ts": 0, + "tests/shell-observation.test.ts": 1.6, + "tests/shell-output.test.ts": 0.2, + "tests/shell-private-io.test.ts": 0, + "tests/shell-timeout.test.ts": 0.2, + "tests/skill-references.test.ts": 0, + "tests/spawn-env-limit.test.ts": 0, + "tests/ssh-command.test.ts": 0, + "tests/supervisor-commands.test.ts": 0, + "tests/supervisor-job-store.test.ts": 0, + "tests/supervisor-runner.test.ts": 0, + "tests/supervisor-service.test.ts": 0.1, + "tests/tailscale-oauth-commands.test.ts": 0, + "tests/templates.test.ts": 0, + "tests/test-config-compiler.test.ts": 2, + "tests/time.test.ts": 0, + "tests/tla-result.test.ts": 0, + "tests/tla-runtime-models.test.ts": 0, + "tests/tmux-backend.test.ts": 0, + "tests/unix-socket-publish.test.ts": 0, + "tests/update-command.test.ts": 0, + "tests/update-homebrew-tap.test.ts": 0, + "tests/usage-details.test.ts": 0, + "tests/worktree-git-inspection.test.ts": 0, + "tests/worktree-runtime-target.test.ts": 0, + "tests/zellij-backend.test.ts": 0, + "tests/zig-aarch64-musl-linker.test.ts": 0 + } +} diff --git a/scripts/test-shard-plan.ts b/scripts/test-shard-plan.ts new file mode 100644 index 00000000..a2596890 --- /dev/null +++ b/scripts/test-shard-plan.ts @@ -0,0 +1,522 @@ +/** + * Deterministic sharding of the hosted `bun test` suite (HACK-1381). + * + * The serial hosted `test` job ran every test file in one `bun test` process + * under Turbo and took 37-42 minutes, so a slow runner reached GitHub's + * 60-minute job limit, and Turbo's buffered output could not name the file + * that was running when the job was cancelled. CI now splits the same files + * across parallel `test shard i/n` jobs (`scripts/test-shard.ts`). + * + * The partition is a pure function of the discovered file paths, the shard + * count and the checked-in duration table, so every runner of one commit + * computes the same plan without coordinating. Balancing is longest + * processing time first: files are taken from the longest recorded duration + * down (path order breaks ties) and each goes to the currently lightest shard + * (lowest index breaks ties). A file without a recorded duration gets + * {@link DEFAULT_FILE_SECONDS}, so a new file is always planned; a stale + * duration costs wall time, never coverage. + * + * Coverage is checked, closed, on every shard run: the union of all shards + * must equal the full test-file set that `bun run test` (Turbo) runs, which is + * enumerated independently through `git ls-files`, and no file may appear in + * two shards. + */ +import { statSync } from "node:fs"; +import { relative, resolve, sep } from "node:path"; +import { ClosedCliError } from "../src/lib/cli-result.ts"; +import { isRecord } from "../src/lib/guards.ts"; + +/** + * One `bun test` invocation of the Turbo `test` task: the files under `root` + * run with `cwd` as the working directory, so that directory's `bunfig.toml` + * preloads and its tsconfig path aliases apply exactly as under Turbo. + * Both paths are repository-relative. + */ +export type TestSuite = { + readonly id: string; + readonly cwd: string; + readonly root: string; +}; + +/** + * The suites `bun run test` runs through Turbo. `packages/cli`'s `test` runs + * `bun test` at the repository root, where `bunfig.toml` sets `root = + * "./tests"`; `packages/db`'s `test` runs `bun test` in its own directory. + */ +export const TEST_SUITES: readonly TestSuite[] = [ + { id: "cli", cwd: ".", root: "tests" }, + { id: "db", cwd: "packages/db", root: "packages/db" }, +]; + +/** Workspaces whose `test` script Turbo runs; each maps to a suite above. */ +export const TEST_SCRIPT_WORKSPACES: readonly string[] = [ + "packages/cli", + "packages/db", +]; + +/** The root `bunfig.toml` test root that the `cli` suite mirrors. */ +export const ROOT_BUNFIG_TEST_ROOT = "./tests"; + +/** Upper bound on `n` in `--shard=i/n`. */ +export const MAX_TEST_SHARD_COUNT = 32; + +/** Planned duration of a file with no recorded duration yet. */ +export const DEFAULT_FILE_SECONDS = 5; + +/** Planned cost of starting one `bun test` process with the suite preload. */ +export const PER_FILE_START_SECONDS = 0.3; + +/** Repository-relative path of the checked-in duration table. */ +export const DURATION_TABLE_PATH = "scripts/test-shard-durations.json"; + +/** Schema tag of the duration table and of each shard's measured output. */ +export const DURATION_TABLE_SCHEMA = "hack.test-shard-durations.v1"; + +/** Bun's test-file names: `*.test.*`, `*_test.*`, `*.spec.*`, `*_spec.*`. */ +const TEST_FILE_NAME = + /(?:\.test|_test|\.spec|_spec)\.(?:jsx?|tsx?|[cm][jt]s)$/; + +const SHARD_PATTERN = /^([1-9][0-9]*)\/([1-9][0-9]*)$/; + +/** Paths named in one coverage failure before the rest are counted. */ +const MAX_NAMED_PATHS = 20; + +export type ShardSelection = { + /** One-based shard index, `1 <= index <= count`. */ + readonly index: number; + readonly count: number; +}; + +/** Parse `i/n`; null for anything outside `1 <= i <= n <= 32`. */ +export function parseShardSelection(value: string): ShardSelection | null { + const match = SHARD_PATTERN.exec(value); + if (!match) { + return null; + } + const index = Number(match[1]); + const count = Number(match[2]); + if (count > MAX_TEST_SHARD_COUNT || index > count) { + return null; + } + return { index, count }; +} + +/** + * Whether Bun's test discovery would run `path` (repository-relative, `/` + * separated): a test-file name outside `node_modules` and hidden directories. + */ +export function isTestFilePath(path: string): boolean { + const segments = path.split("/"); + const name = segments.at(-1) ?? ""; + if (!TEST_FILE_NAME.test(name)) { + return false; + } + return segments.every( + (segment) => + segment.length > 0 && + segment !== "node_modules" && + !segment.startsWith(".") + ); +} + +function within(opts: { readonly root: string; readonly path: string }) { + return opts.root === "." || opts.path.startsWith(`${opts.root}/`); +} + +/** The suite that runs `file`, refused closed when none does. */ +export function suiteForFile( + file: string, + suites: readonly TestSuite[] = TEST_SUITES +): TestSuite { + const owners = suites.filter((suite) => + within({ root: suite.root, path: file }) + ); + if (owners.length !== 1) { + throw new ClosedCliError( + `test shard: ${file} must belong to exactly one test suite, found ${owners.length}.` + ); + } + return owners[0] as TestSuite; +} + +/** + * Every test file Bun would discover under the suite roots, found on disk, + * as sorted repository-relative paths. This is the set the shards run. + */ +export function discoverTestFiles(opts: { + readonly repoRoot: string; + readonly suites?: readonly TestSuite[]; +}): string[] { + const suites = opts.suites ?? TEST_SUITES; + const found = new Set(); + const glob = new Bun.Glob("**/*"); + for (const suite of suites) { + const base = resolve(opts.repoRoot, suite.root); + if (!isDirectory(base)) { + throw new ClosedCliError( + `test shard: the ${suite.id} suite root ${suite.root} is not a directory.` + ); + } + for (const entry of glob.scanSync({ cwd: base, onlyFiles: true })) { + const path = relative(opts.repoRoot, resolve(base, entry)) + .split(sep) + .join("/"); + if (isTestFilePath(path)) { + found.add(path); + } + } + } + return [...found].sort(); +} + +/** + * The same test-file set enumerated independently through Git: tracked files + * plus untracked files that are not ignored, under the suite roots. Refused, + * closed, when Git cannot list the checkout. + */ +export function listGitTestFiles(opts: { + readonly repoRoot: string; + readonly suites?: readonly TestSuite[]; +}): string[] { + const suites = opts.suites ?? TEST_SUITES; + const git = Bun.which("git"); + if (!git) { + throw new ClosedCliError( + "test shard: git is not on PATH, so the full test-file set cannot be enumerated." + ); + } + const result = Bun.spawnSync( + [ + git, + "ls-files", + "-z", + "--cached", + "--others", + "--exclude-standard", + "--", + ...suites.map((suite) => suite.root), + ], + { + cwd: opts.repoRoot, + env: { ...process.env, GIT_OPTIONAL_LOCKS: "0" }, + stdin: "ignore", + stdout: "pipe", + stderr: "ignore", + } + ); + if (result.exitCode !== 0) { + throw new ClosedCliError( + "test shard: git ls-files failed, so the full test-file set cannot be enumerated." + ); + } + const files = new Set(); + for (const path of result.stdout.toString().split("\0")) { + // A tracked file deleted in the work tree is still listed; Bun cannot run it. + if ( + path.length > 0 && + isTestFilePath(path) && + isFile(resolve(opts.repoRoot, path)) + ) { + files.add(path); + } + } + return [...files].sort(); +} + +function isFile(path: string): boolean { + try { + return statSync(path).isFile(); + } catch { + return false; + } +} + +function isDirectory(path: string): boolean { + try { + return statSync(path).isDirectory(); + } catch { + return false; + } +} + +/** + * Parse a duration table, closed: `{ "schema": DURATION_TABLE_SCHEMA, + * "seconds": { "": = 0> } }`. + */ +export function parseDurationTable(text: string): Record { + let value: unknown; + try { + value = JSON.parse(text); + } catch { + throw new ClosedCliError("test shard: the duration table is not JSON."); + } + if ( + !( + isRecord(value) && + value.schema === DURATION_TABLE_SCHEMA && + isRecord(value.seconds) + ) + ) { + throw new ClosedCliError( + `test shard: the duration table must have schema ${DURATION_TABLE_SCHEMA} and a seconds object.` + ); + } + const seconds: Record = {}; + for (const [path, entry] of Object.entries(value.seconds)) { + if ( + !( + typeof entry === "number" && + Number.isFinite(entry) && + entry >= 0 && + isTestFilePath(path) + ) + ) { + throw new ClosedCliError( + "test shard: every duration table entry must be a test-file path with finite non-negative seconds." + ); + } + seconds[path] = entry; + } + return seconds; +} + +/** + * Planned seconds for `file`: its recorded duration (else the default) plus + * the per-file process start. Without that addend, files recorded at zero + * seconds would not add load and would all land on one shard. + */ +export function plannedSeconds(opts: { + readonly file: string; + readonly durations: Readonly>; +}): number { + const recorded = Object.hasOwn(opts.durations, opts.file) + ? (opts.durations[opts.file] as number) + : DEFAULT_FILE_SECONDS; + return recorded + PER_FILE_START_SECONDS; +} + +/** + * Split `files` into `count` disjoint shards whose union is `files`, each in + * path order. Duplicate paths are refused because they would run twice. + */ +export function partitionTestFiles(opts: { + readonly files: readonly string[]; + readonly count: number; + readonly durations: Readonly>; +}): string[][] { + if ( + !Number.isSafeInteger(opts.count) || + opts.count < 1 || + opts.count > MAX_TEST_SHARD_COUNT + ) { + throw new ClosedCliError( + `test shard: the shard count must be an integer from 1 to ${MAX_TEST_SHARD_COUNT}.` + ); + } + if (new Set(opts.files).size !== opts.files.length) { + throw new ClosedCliError("test shard: test file paths must be unique."); + } + const longestFirst = [...opts.files].sort((left, right) => { + const byDuration = + plannedSeconds({ file: right, durations: opts.durations }) - + plannedSeconds({ file: left, durations: opts.durations }); + if (byDuration !== 0) { + return byDuration; + } + return left < right ? -1 : 1; + }); + const loads: number[] = Array.from({ length: opts.count }, () => 0); + const shards: string[][] = Array.from({ length: opts.count }, () => []); + for (const file of longestFirst) { + let lightest = 0; + for (let shard = 1; shard < opts.count; shard += 1) { + if ((loads[shard] as number) < (loads[lightest] as number)) { + lightest = shard; + } + } + loads[lightest] = + (loads[lightest] as number) + + plannedSeconds({ file, durations: opts.durations }); + (shards[lightest] as string[]).push(file); + } + return shards.map((shard) => shard.sort()); +} + +export type CoverageReport = { + /** Files in the full set that no shard runs. */ + readonly missing: readonly string[]; + /** Files that more than one shard (or one shard twice) runs. */ + readonly duplicated: readonly string[]; + /** Files a shard runs that are not in the full set. */ + readonly unexpected: readonly string[]; +}; + +/** Compare the shards' union with the full set; every list empty means exact. */ +export function shardCoverage(opts: { + readonly expected: readonly string[]; + readonly shards: readonly (readonly string[])[]; +}): CoverageReport { + const expected = new Set(opts.expected); + const seen = new Map(); + for (const shard of opts.shards) { + for (const file of shard) { + seen.set(file, (seen.get(file) ?? 0) + 1); + } + } + return { + missing: [...expected].filter((file) => !seen.has(file)).sort(), + duplicated: [...seen] + .filter(([, count]) => count > 1) + .map(([file]) => file) + .sort(), + unexpected: [...seen.keys()].filter((file) => !expected.has(file)).sort(), + }; +} + +function namedPaths(label: string, paths: readonly string[]): string[] { + if (paths.length === 0) { + return []; + } + const lines = [`${label} (${paths.length}):`]; + for (const path of paths.slice(0, MAX_NAMED_PATHS)) { + lines.push(` ${path}`); + } + if (paths.length > MAX_NAMED_PATHS) { + lines.push(` ... and ${paths.length - MAX_NAMED_PATHS} more`); + } + return lines; +} + +/** Refuses, closed, unless the shards cover `expected` exactly once each. */ +export function assertShardCoverage(opts: { + readonly expected: readonly string[]; + readonly shards: readonly (readonly string[])[]; +}): void { + const report = shardCoverage(opts); + const lines = [ + ...namedPaths("not run by any shard", report.missing), + ...namedPaths("run by more than one shard", report.duplicated), + ...namedPaths( + "run by a shard but not in the full test set", + report.unexpected + ), + ]; + if (lines.length > 0) { + throw new ClosedCliError( + [ + `test shard: the ${opts.shards.length} shards do not cover the full test set exactly once.`, + ...lines, + ].join("\n") + ); + } +} + +/** + * Refuses, closed, when Turbo's `test` task would run a suite the shards do + * not know: a workspace `test` script outside {@link TEST_SCRIPT_WORKSPACES}, + * a known one that disappeared, or a changed root `bunfig.toml` test root. + */ +export async function assertSuiteInputs(opts: { + readonly repoRoot: string; +}): Promise { + const rootPackage: unknown = await Bun.file( + resolve(opts.repoRoot, "package.json") + ).json(); + const workspaces = + isRecord(rootPackage) && Array.isArray(rootPackage.workspaces) + ? rootPackage.workspaces.filter( + (entry): entry is string => typeof entry === "string" + ) + : []; + const withTest: string[] = []; + for (const pattern of workspaces) { + const glob = new Bun.Glob(`${pattern}/package.json`); + for (const manifest of glob.scanSync({ + cwd: opts.repoRoot, + onlyFiles: true, + })) { + const parsed: unknown = await Bun.file( + resolve(opts.repoRoot, manifest) + ).json(); + if ( + isRecord(parsed) && + isRecord(parsed.scripts) && + typeof parsed.scripts.test === "string" + ) { + withTest.push( + manifest + .split(sep) + .join("/") + .replace(/\/package\.json$/, "") + ); + } + } + } + const actual = [...new Set(withTest)].sort(); + const known = [...TEST_SCRIPT_WORKSPACES].sort(); + if (JSON.stringify(actual) !== JSON.stringify(known)) { + throw new ClosedCliError( + `test shard: Turbo runs test scripts in [${actual.join(", ")}] but the shard suites cover [${known.join(", ")}]; update TEST_SUITES in scripts/test-shard-plan.ts.` + ); + } + const bunfig: unknown = Bun.TOML.parse( + await Bun.file(resolve(opts.repoRoot, "bunfig.toml")).text() + ); + const testRoot = + isRecord(bunfig) && isRecord(bunfig.test) ? bunfig.test.root : undefined; + if (testRoot !== ROOT_BUNFIG_TEST_ROOT) { + throw new ClosedCliError( + `test shard: bunfig.toml [test].root must be ${ROOT_BUNFIG_TEST_ROOT} to match the cli suite root.` + ); + } +} + +export type ShardPlan = { + /** The full test-file set, from disk; equal to the Git enumeration. */ + readonly files: readonly string[]; + readonly shards: readonly (readonly string[])[]; + readonly durations: Readonly>; +}; + +/** + * Discover, partition and verify the plan for `count` shards. Refuses, + * closed, when disk and Git disagree on the test-file set or when the shards + * do not cover it exactly once. + */ +export async function buildVerifiedPlan(opts: { + readonly repoRoot: string; + readonly count: number; + readonly suites?: readonly TestSuite[]; + readonly durationTablePath?: string; + /** Test seam: replaces the partition, to prove the coverage check fails. */ + readonly partition?: typeof partitionTestFiles; +}): Promise { + const suites = opts.suites ?? TEST_SUITES; + if (!opts.suites) { + await assertSuiteInputs({ repoRoot: opts.repoRoot }); + } + const durations = parseDurationTable( + await Bun.file( + resolve(opts.repoRoot, opts.durationTablePath ?? DURATION_TABLE_PATH) + ).text() + ); + const files = discoverTestFiles({ repoRoot: opts.repoRoot, suites }); + const tracked = listGitTestFiles({ repoRoot: opts.repoRoot, suites }); + const drift = shardCoverage({ expected: tracked, shards: [files] }); + if (drift.missing.length > 0 || drift.unexpected.length > 0) { + throw new ClosedCliError( + [ + "test shard: the test files found on disk and listed by git differ.", + ...namedPaths("listed by git but not found on disk", drift.missing), + ...namedPaths("found on disk but not listed by git", drift.unexpected), + ].join("\n") + ); + } + const shards = (opts.partition ?? partitionTestFiles)({ + files, + count: opts.count, + durations, + }); + assertShardCoverage({ expected: tracked, shards }); + return { files, shards, durations }; +} diff --git a/scripts/test-shard.ts b/scripts/test-shard.ts new file mode 100644 index 00000000..fc009c33 --- /dev/null +++ b/scripts/test-shard.ts @@ -0,0 +1,694 @@ +#!/usr/bin/env bun +/** + * Hosted test shard runner (HACK-1381). See scripts/test-shard-plan.ts for the + * partition and its coverage check. + * + * bun scripts/test-shard.ts verify --count=4 + * bun scripts/test-shard.ts plan --shard=2/4 + * bun scripts/test-shard.ts run --shard=2/4 [--durations-out=FILE] + * [--file-timeout-seconds=720] [--deadline-seconds=1800] + * bun scripts/test-shard.ts update-durations --source=TEXT FILE... + * + * `run` first checks the Bun pin and the compiler preflight exactly as + * `bun run test` does, verifies the plan, then runs each of its files in its + * own `bun test ./` process from the file's suite directory, so the + * suite's `bunfig.toml` preload applies. Each file prints a start line, Bun's + * own output, and a result line with its duration and the shard's elapsed + * time; a file running longer than two minutes prints a heartbeat. A file + * that outlives `--file-timeout-seconds` is stopped and reported, and when + * the shard reaches `--deadline-seconds`, or receives SIGINT/SIGTERM (a + * GitHub cancellation or job timeout), it names the running file, the files + * not started and the slowest files before it exits. The run fails when any + * file fails, times out or does not report exactly one file run. + */ +import { appendFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { ClosedCliError } from "../src/lib/cli-result.ts"; +import { + buildVerifiedPlan, + DURATION_TABLE_PATH, + DURATION_TABLE_SCHEMA, + discoverTestFiles, + parseDurationTable, + parseShardSelection, + plannedSeconds, + type ShardSelection, + suiteForFile, + TEST_SUITES, + type TestSuite, +} from "./test-shard-plan.ts"; + +const REPO_ROOT = resolve(import.meta.dir, ".."); + +/** Default per-file limit: about twice the slowest measured hosted file. */ +export const DEFAULT_FILE_TIMEOUT_SECONDS = 720; + +/** Default shard limit, below the job's `timeout-minutes` so the report prints. */ +export const DEFAULT_DEADLINE_SECONDS = 1800; + +/** A running file prints a heartbeat this often. */ +export const HEARTBEAT_SECONDS = 120; + +/** Files named in the slowest-files report. */ +const SLOWEST_COUNT = 10; + +/** Bytes of a file's stderr kept to read Bun's summary line. */ +const STDERR_TAIL_BYTES = 16_384; + +/** How long to keep reading a file's stderr after its Bun process exits. */ +const STDERR_DRAIN_MS = 2000; + +const BUN_SUMMARY = /Ran (\d+) tests? across (\d+) files?\./g; + +export type FileOutcome = + | "pass" + | "fail" + | "file-count" + | "timeout" + | "deadline" + | "interrupted"; + +export type FileResult = { + readonly file: string; + readonly outcome: FileOutcome; + readonly seconds: number; + /** Bun's exit code; null when the process was stopped. */ + readonly exitCode: number | null; + /** Files Bun reported running; null when no summary was printed. */ + readonly filesRan: number | null; +}; + +/** What one file's process returned; `stopped` means the runner stopped it. */ +export type FileRun = { + readonly exitCode: number | null; + readonly filesRan: number | null; + readonly stopped: boolean; +}; + +/** Runs one file; must resolve promptly after `signal` aborts. */ +export type RunFile = (opts: { + readonly file: string; + readonly signal: AbortSignal; +}) => Promise; + +export type ShardState = { + readonly label: string; + readonly files: readonly string[]; + readonly results: FileResult[]; + current: { readonly file: string; readonly startedAt: number } | null; + readonly startedAt: number; +}; + +/** `0.4s` below a minute, `12m05s` from a minute up. */ +export function formatDuration(seconds: number): string { + const clamped = Math.max(0, seconds); + if (clamped < 59.95) { + return `${clamped.toFixed(1)}s`; + } + const whole = Math.round(clamped); + const rest = whole % 60; + return `${Math.floor(whole / 60)}m${String(rest).padStart(2, "0")}s`; +} + +/** The number of files Bun's last summary line reports, or null. */ +export function bunFilesRan(output: string): number | null { + let last: number | null = null; + for (const match of output.matchAll(BUN_SUMMARY)) { + last = Number(match[2]); + } + return last; +} + +/** Classify one file's run. Exactly one file must have run for a pass. */ +export function classifyFileRun(opts: { + readonly run: FileRun; + readonly stopReason: "timeout" | "deadline" | "interrupted" | null; +}): FileOutcome { + if (opts.stopReason) { + return opts.stopReason; + } + if (opts.run.exitCode !== 0) { + return "fail"; + } + return opts.run.filesRan === 1 ? "pass" : "file-count"; +} + +/** Slowest finished files, longest first (path order breaks ties). */ +export function slowestFiles( + results: readonly FileResult[], + count = SLOWEST_COUNT +): FileResult[] { + return [...results] + .sort( + (left, right) => + right.seconds - left.seconds || (left.file < right.file ? -1 : 1) + ) + .slice(0, count); +} + +/** + * The report printed when the shard stops early: the running file, the files + * never started and the slowest finished files. + */ +export function stopReport(opts: { + readonly state: ShardState; + readonly reason: string; + readonly now: number; +}): string[] { + const { state } = opts; + const lines = [ + `${state.label} stopped (${opts.reason}) after ${formatDuration((opts.now - state.startedAt) / 1000)}; ${state.results.length}/${state.files.length} files finished.`, + ]; + if (state.current) { + lines.push( + `${state.label} running file: ${state.current.file} (${formatDuration((opts.now - state.current.startedAt) / 1000)})` + ); + } + const done = new Set(state.results.map((result) => result.file)); + const notStarted = state.files.filter( + (file) => !done.has(file) && file !== state.current?.file + ); + if (notStarted.length > 0) { + lines.push(`${state.label} not started (${notStarted.length}):`); + for (const file of notStarted) { + lines.push(` ${file}`); + } + } + lines.push(...slowestReport(state)); + return lines; +} + +function slowestReport(state: ShardState): string[] { + const slowest = slowestFiles(state.results); + if (slowest.length === 0) { + return []; + } + return [ + `${state.label} slowest files:`, + ...slowest.map( + (result) => + ` ${formatDuration(result.seconds).padStart(7)} ${result.outcome.padEnd(11)} ${result.file}` + ), + ]; +} + +/** The closing summary; failures first, then the slowest files. */ +export function summaryReport(state: ShardState, now: number): string[] { + const failed = state.results.filter((result) => result.outcome !== "pass"); + const lines = [ + `${state.label} finished ${state.results.length}/${state.files.length} files in ${formatDuration((now - state.startedAt) / 1000)}: ${state.results.length - failed.length} passed, ${failed.length} failed.`, + ]; + if (failed.length > 0) { + lines.push(`${state.label} failed files:`); + for (const result of failed) { + lines.push( + ` ${result.outcome.padEnd(11)} ${result.file} (exit ${result.exitCode ?? "none"}, files reported ${result.filesRan ?? "none"}, ${formatDuration(result.seconds)})` + ); + } + } + lines.push(...slowestReport(state)); + return lines; +} + +/** + * Run `state.files` one at a time through `runFile`, recording each result. + * Stops a file at `fileTimeoutMs`, and the shard at `deadlineAt` or when + * `interrupt` aborts; files after a stop are left unstarted. + */ +export async function runShardFiles(opts: { + readonly state: ShardState; + readonly runFile: RunFile; + readonly fileTimeoutMs: number; + readonly deadlineAt: number; + readonly heartbeatMs: number; + readonly interrupt: AbortSignal; + readonly now: () => number; + readonly log: (line: string) => void; +}): Promise<"complete" | "deadline" | "interrupted"> { + const { state } = opts; + for (const [position, file] of state.files.entries()) { + if (opts.interrupt.aborted) { + return "interrupted"; + } + if (opts.now() >= opts.deadlineAt) { + return "deadline"; + } + const startedAt = opts.now(); + state.current = { file, startedAt }; + opts.log( + `${state.label} (${position + 1}/${state.files.length}) start ${file}` + ); + const controller = new AbortController(); + let stopReason: "timeout" | "deadline" | "interrupted" | null = null; + const stop = (reason: "timeout" | "deadline" | "interrupted") => { + if (!stopReason) { + stopReason = reason; + controller.abort(); + } + }; + const fileTimer = setTimeout( + () => stop("timeout"), + Math.max(0, Math.min(opts.fileTimeoutMs, opts.deadlineAt - startedAt)) + ); + const deadlineFirst = opts.deadlineAt - startedAt < opts.fileTimeoutMs; + const onInterrupt = () => stop("interrupted"); + opts.interrupt.addEventListener("abort", onInterrupt); + const heartbeat = setInterval(() => { + opts.log( + `${state.label} still running ${file} (${formatDuration((opts.now() - startedAt) / 1000)})` + ); + }, opts.heartbeatMs); + let run: FileRun; + try { + run = await opts.runFile({ file, signal: controller.signal }); + } finally { + clearTimeout(fileTimer); + clearInterval(heartbeat); + opts.interrupt.removeEventListener("abort", onInterrupt); + } + if (stopReason === "timeout" && deadlineFirst) { + stopReason = "deadline"; + } + const result: FileResult = { + file, + outcome: classifyFileRun({ run, stopReason }), + seconds: (opts.now() - startedAt) / 1000, + exitCode: run.exitCode, + filesRan: run.filesRan, + }; + state.results.push(result); + state.current = null; + opts.log( + `${state.label} (${position + 1}/${state.files.length}) ${result.outcome.toUpperCase()} ${file} in ${formatDuration(result.seconds)}; shard elapsed ${formatDuration((opts.now() - state.startedAt) / 1000)}` + ); + if (result.outcome === "deadline" || result.outcome === "interrupted") { + return result.outcome; + } + } + return "complete"; +} + +/** Serialize a duration table: sorted paths, seconds to one decimal. */ +export function formatDurationTable(opts: { + readonly source: string; + readonly seconds: Readonly>; +}): string { + const seconds: Record = {}; + for (const file of Object.keys(opts.seconds).sort()) { + seconds[file] = Math.round((opts.seconds[file] as number) * 10) / 10; + } + return `${JSON.stringify({ schema: DURATION_TABLE_SCHEMA, source: opts.source, seconds }, null, 2)}\n`; +} + +/** + * Measured seconds per finished file, in the duration-table format. Stopped + * files are left out: their time is a limit, not a measurement. + */ +export function measuredDurations(opts: { + readonly label: string; + readonly results: readonly FileResult[]; +}): string { + const seconds: Record = {}; + for (const result of opts.results) { + if (result.outcome === "pass" || result.outcome === "fail") { + seconds[result.file] = result.seconds; + } + } + return formatDurationTable({ source: opts.label, seconds }); +} + +/** + * Merge measured shard outputs over `table`, keeping only files that still + * exist. Later inputs win for a file measured twice. + */ +export function mergeDurationTables(opts: { + readonly source: string; + readonly table: Readonly>; + readonly measured: readonly Readonly>[]; + readonly files: readonly string[]; +}): string { + const present = new Set(opts.files); + const merged: Record = {}; + for (const source of [opts.table, ...opts.measured]) { + for (const [file, seconds] of Object.entries(source)) { + if (present.has(file)) { + merged[file] = seconds; + } + } + } + return formatDurationTable({ source: opts.source, seconds: merged }); +} + +/** + * Runs `bun test ./` from the file's suite directory with the caller's + * environment, forwarding its output unless `quiet`. Stopping sends SIGTERM, + * then SIGKILL after ten seconds. + */ +export function bunTestFile(opts: { + readonly repoRoot: string; + readonly suites?: readonly TestSuite[]; + readonly quiet?: boolean; +}): RunFile { + return async ({ file, signal }) => { + const suite = suiteForFile(file, opts.suites ?? TEST_SUITES); + const cwd = resolve(opts.repoRoot, suite.cwd); + const relativePath = + suite.cwd === "." ? file : file.slice(suite.cwd.length + 1); + const child = Bun.spawn([process.execPath, "test", `./${relativePath}`], { + cwd, + env: process.env, + stdin: "ignore", + stdout: opts.quiet ? "ignore" : "inherit", + stderr: "pipe", + }); + let killTimer: ReturnType | null = null; + const onAbort = () => { + child.kill("SIGTERM"); + killTimer = setTimeout(() => child.kill("SIGKILL"), 10_000); + }; + if (signal.aborted) { + onAbort(); + } else { + signal.addEventListener("abort", onAbort, { once: true }); + } + let tail = ""; + const decoder = new TextDecoder(); + const reader = child.stderr.getReader(); + const forward = (async () => { + for (;;) { + const { done, value } = await reader.read(); + if (done) { + return; + } + if (!opts.quiet) { + process.stderr.write(value); + } + tail = (tail + decoder.decode(value, { stream: true })).slice( + -STDERR_TAIL_BYTES + ); + } + })(); + const exitCode = await child.exited; + // A process a test left running can hold the pipe open after Bun exits; + // take what it has written so far rather than wait for its end. + await Promise.race([forward, Bun.sleep(STDERR_DRAIN_MS)]); + await reader.cancel().catch(() => undefined); + signal.removeEventListener("abort", onAbort); + if (killTimer) { + clearTimeout(killTimer); + } + return { + exitCode: signal.aborted ? null : exitCode, + filesRan: bunFilesRan(tail), + stopped: signal.aborted, + }; + }; +} + +function runPackageScript(script: string): void { + const result = Bun.spawnSync([process.execPath, "run", script], { + cwd: REPO_ROOT, + env: process.env, + stdin: "ignore", + stdout: "inherit", + stderr: "inherit", + }); + if (result.exitCode !== 0) { + throw new ClosedCliError( + `test shard: bun run ${script} failed; the shard has not started.` + ); + } +} + +function githubError(message: string): void { + if (process.env.GITHUB_ACTIONS === "true") { + process.stdout.write(`::error title=test shard::${message}\n`); + } +} + +function writeStepSummary(state: ShardState, now: number): void { + const path = process.env.GITHUB_STEP_SUMMARY; + if (!path) { + return; + } + const failed = state.results.filter((result) => result.outcome !== "pass"); + const lines = [ + `### ${state.label}`, + "", + `${state.results.length}/${state.files.length} files finished in ${formatDuration((now - state.startedAt) / 1000)}; ${failed.length} failed.`, + "", + "| Seconds | Outcome | File |", + "| ---: | --- | --- |", + ...[...failed, ...slowestFiles(state.results)] + .filter( + (result, index, all) => + all.findIndex((other) => other.file === result.file) === index + ) + .map( + (result) => + `| ${result.seconds.toFixed(1)} | ${result.outcome} | \`${result.file}\` |` + ), + "", + ]; + try { + appendFileSync(path, `${lines.join("\n")}\n`); + } catch { + process.stderr.write(`${state.label} could not write the step summary.\n`); + } +} + +function optionValue( + args: readonly string[], + name: string +): string | undefined { + const prefix = `--${name}=`; + const found = args.filter((arg) => arg.startsWith(prefix)); + if (found.length > 1) { + throw new ClosedCliError(`test shard: --${name} was given more than once.`); + } + return found[0]?.slice(prefix.length); +} + +function positiveSeconds(value: string | undefined, fallback: number): number { + if (value === undefined) { + return fallback; + } + if (!/^[1-9][0-9]{0,5}$/.test(value)) { + throw new ClosedCliError( + "test shard: a seconds option must be a positive whole number." + ); + } + return Number(value); +} + +function requireSelection(args: readonly string[]): ShardSelection { + const selection = parseShardSelection(optionValue(args, "shard") ?? ""); + if (!selection) { + throw new ClosedCliError( + "test shard: --shard=i/n with 1 <= i <= n <= 32 is required." + ); + } + return selection; +} + +async function commandVerify(args: readonly string[]): Promise { + const count = Number(optionValue(args, "count") ?? ""); + const plan = await buildVerifiedPlan({ repoRoot: REPO_ROOT, count }); + console.log( + `test shard plan: ${plan.files.length} files in ${count} shards cover the full test set exactly once.` + ); + for (const [index, shard] of plan.shards.entries()) { + const seconds = shard.reduce( + (total, file) => + total + plannedSeconds({ file, durations: plan.durations }), + 0 + ); + const unknown = shard.filter( + (file) => !Object.hasOwn(plan.durations, file) + ).length; + console.log( + ` shard ${index + 1}/${count}: ${shard.length} files, planned ${formatDuration(seconds)}${unknown > 0 ? ` (${unknown} without a recorded duration)` : ""}` + ); + } + return 0; +} + +async function commandPlan(args: readonly string[]): Promise { + const selection = requireSelection(args); + const plan = await buildVerifiedPlan({ + repoRoot: REPO_ROOT, + count: selection.count, + }); + for (const file of plan.shards[selection.index - 1] ?? []) { + console.log(file); + } + return 0; +} + +async function commandRun(args: readonly string[]): Promise { + const selection = requireSelection(args); + const fileTimeoutSeconds = positiveSeconds( + optionValue(args, "file-timeout-seconds"), + DEFAULT_FILE_TIMEOUT_SECONDS + ); + const deadlineSeconds = positiveSeconds( + optionValue(args, "deadline-seconds"), + DEFAULT_DEADLINE_SECONDS + ); + const durationsOut = optionValue(args, "durations-out"); + const label = `[test shard ${selection.index}/${selection.count}]`; + const startedAt = Date.now(); + runPackageScript("check:bun-version"); + runPackageScript("test:preflight"); + const plan = await buildVerifiedPlan({ + repoRoot: REPO_ROOT, + count: selection.count, + }); + const files = plan.shards[selection.index - 1] ?? []; + const state: ShardState = { + label, + files, + results: [], + current: null, + startedAt, + }; + const planned = files.reduce( + (total, file) => + total + plannedSeconds({ file, durations: plan.durations }), + 0 + ); + console.log( + `${label} ${files.length} of ${plan.files.length} files, planned ${formatDuration(planned)}; file timeout ${formatDuration(fileTimeoutSeconds)}, shard deadline ${formatDuration(deadlineSeconds)}.` + ); + const interrupt = new AbortController(); + let signalName: string | null = null; + const onSignal = (name: "SIGINT" | "SIGTERM") => { + if (!signalName) { + signalName = name; + for (const line of stopReport({ + state, + reason: `received ${name}`, + now: Date.now(), + })) { + console.log(line); + } + interrupt.abort(); + } + }; + process.on("SIGINT", () => onSignal("SIGINT")); + process.on("SIGTERM", () => onSignal("SIGTERM")); + const ended = await runShardFiles({ + state, + runFile: bunTestFile({ repoRoot: REPO_ROOT }), + fileTimeoutMs: fileTimeoutSeconds * 1000, + deadlineAt: startedAt + deadlineSeconds * 1000, + heartbeatMs: HEARTBEAT_SECONDS * 1000, + interrupt: interrupt.signal, + now: Date.now, + log: (line) => console.log(line), + }); + const now = Date.now(); + if (ended === "deadline") { + for (const line of stopReport({ + state, + reason: `reached the ${formatDuration(deadlineSeconds)} shard deadline`, + now, + })) { + console.log(line); + } + } + for (const line of summaryReport(state, now)) { + console.log(line); + } + for (const result of state.results) { + if (result.outcome !== "pass") { + githubError( + `${result.file}: ${result.outcome} after ${formatDuration(result.seconds)}` + ); + } + } + if (ended !== "complete") { + githubError( + `${label} stopped (${ended}) with ${files.length - state.results.length} files not finished` + ); + } + writeStepSummary(state, now); + if (durationsOut) { + await Bun.write( + durationsOut, + measuredDurations({ label, results: state.results }) + ); + } + const failed = + ended !== "complete" || + state.results.length !== files.length || + state.results.some((result) => result.outcome !== "pass"); + if (signalName === "SIGINT") { + return 130; + } + if (signalName === "SIGTERM") { + return 143; + } + return failed ? 1 : 0; +} + +async function commandUpdateDurations( + args: readonly string[] +): Promise { + const source = optionValue(args, "source"); + const inputs = args.filter((arg) => !arg.startsWith("--")); + if (!source || inputs.length === 0) { + throw new ClosedCliError( + "test shard: update-durations needs --source=TEXT (where the numbers were measured) and one or more measured shard files." + ); + } + const tablePath = resolve(REPO_ROOT, DURATION_TABLE_PATH); + const table = parseDurationTable(await Bun.file(tablePath).text()); + const measured: Record[] = []; + for (const path of inputs) { + measured.push(parseDurationTable(await Bun.file(path).text())); + } + await Bun.write( + tablePath, + mergeDurationTables({ + source, + table, + measured, + files: discoverTestFiles({ repoRoot: REPO_ROOT }), + }) + ); + console.log( + `test shard: updated ${DURATION_TABLE_PATH} from ${inputs.length} file(s).` + ); + return 0; +} + +export async function main(argv: readonly string[]): Promise { + const [command, ...args] = argv; + try { + switch (command) { + case "verify": + return await commandVerify(args); + case "plan": + return await commandPlan(args); + case "run": + return await commandRun(args); + case "update-durations": + return await commandUpdateDurations(args); + default: + throw new ClosedCliError( + "test shard: usage: bun scripts/test-shard.ts verify --count=n | plan --shard=i/n | run --shard=i/n | update-durations --source=TEXT FILE..." + ); + } + } catch (error) { + if (error instanceof ClosedCliError) { + console.error(error.message); + return 2; + } + throw error; + } +} + +if (import.meta.main) { + process.exit(await main(process.argv.slice(2))); +} diff --git a/tests/e2e/README.md b/tests/e2e/README.md index eb0c6e5f..bc3030b7 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -23,8 +23,10 @@ failed (nothing ran) or the launchd canary saw the real hackd plist change. ## CI tiering -Only the `test` job (typecheck, lint, unit tests, build) is a required status -check on `next` and `main`. `docker-e2e`, `Config compiler`, `Candidate core`, +Only the `test` job is a required status check on `next` and `main`. It is the +fan-in of four `test shard i/4` jobs, which run the unit test files in parallel. +Shard 1 also runs typecheck, lint and the build. `test` passes only when every +shard succeeded (`scripts/test-shard-plan.ts`). `docker-e2e`, `Config compiler`, `Candidate core`, `Runtime state models`, `secret-scan`, `runtime-images` and `linux-process-lifetime` run on every push and pull request but are optional checks: a red result is a signal to triage, not a merge block. "Registered in diff --git a/tests/prerelease-plan.test.ts b/tests/prerelease-plan.test.ts index db952f0a..a5bf5a45 100644 --- a/tests/prerelease-plan.test.ts +++ b/tests/prerelease-plan.test.ts @@ -86,6 +86,10 @@ const jobNames = [ "docker-e2e shard 3/4", "docker-e2e shard 4/4", "docker-e2e", + "test shard 1/4", + "test shard 2/4", + "test shard 3/4", + "test shard 4/4", "test", "linux-process-lifetime", ]; diff --git a/tests/project-exec-command.test.ts b/tests/project-exec-command.test.ts index b76b4e61..ab2f0fab 100644 --- a/tests/project-exec-command.test.ts +++ b/tests/project-exec-command.test.ts @@ -23,6 +23,7 @@ const psJsonByComposeProject = new Map(); const psEnvs: Array | undefined> = []; const tempDirs = new Set(); const originalHackHome = process.env.HACK_HOME; +const originalLogger = process.env.HACK_LOGGER; const runtimeBackendMock = await registerScopedModuleMock({ importerPath: import.meta.path, @@ -68,6 +69,10 @@ const { execCommand, resolveModernComposeEnvOverrides } = await import( ); beforeAll(() => { + // Pin the logger backend: without it the logger picks gum or clack from the + // host, and clack writes diagnostics to stdout, which the stdout assertions + // below would see. Do not rely on another test file leaking this setting. + process.env.HACK_LOGGER = "console"; runtimeBackendMock.activate(); }); @@ -88,6 +93,11 @@ afterEach(async () => { afterAll(() => { runtimeBackendMock.deactivate(); + if (originalLogger === undefined) { + Reflect.deleteProperty(process.env, "HACK_LOGGER"); + } else { + process.env.HACK_LOGGER = originalLogger; + } }); test("exec reaches the running service while registry registration is locked", async () => { diff --git a/tests/test-shard.test.ts b/tests/test-shard.test.ts new file mode 100644 index 00000000..67c67c28 --- /dev/null +++ b/tests/test-shard.test.ts @@ -0,0 +1,859 @@ +import { afterAll, describe, expect, test } from "bun:test"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { CI_JOBS } from "../scripts/prerelease-plan.ts"; +import { + bunFilesRan, + bunTestFile, + classifyFileRun, + type FileResult, + formatDuration, + formatDurationTable, + measuredDurations, + mergeDurationTables, + type RunFile, + runShardFiles, + type ShardState, + stopReport, + summaryReport, +} from "../scripts/test-shard.ts"; +import { + assertShardCoverage, + assertSuiteInputs, + buildVerifiedPlan, + DEFAULT_FILE_SECONDS, + DURATION_TABLE_PATH, + isTestFilePath, + listGitTestFiles, + MAX_TEST_SHARD_COUNT, + PER_FILE_START_SECONDS, + parseDurationTable, + parseShardSelection, + partitionTestFiles, + plannedSeconds, + shardCoverage, + suiteForFile, + type TestSuite, +} from "../scripts/test-shard-plan.ts"; + +const repoRoot = resolve(import.meta.dir, ".."); +const owned: string[] = []; + +afterAll(async () => { + await Promise.all( + owned.map((path) => rm(path, { recursive: true, force: true })) + ); +}); + +const FIXTURE_SUITES: readonly TestSuite[] = [ + { id: "cli", cwd: ".", root: "tests" }, + { id: "db", cwd: "packages/db", root: "packages/db" }, +]; + +/** A throwaway Git work tree with test files; nothing is committed. */ +async function fixtureRepo(opts: { + readonly files: readonly string[]; + readonly gitignore?: string; + readonly durations?: Record; +}): Promise { + const root = await mkdtemp(join(tmpdir(), "hack-test-shard-")); + owned.push(root); + const init = Bun.spawnSync(["git", "init", "--quiet", root], { + stdin: "ignore", + stdout: "ignore", + stderr: "ignore", + }); + expect(init.exitCode).toBe(0); + for (const suite of FIXTURE_SUITES) { + await mkdir(join(root, suite.root), { recursive: true }); + } + for (const file of opts.files) { + await mkdir(dirname(join(root, file)), { recursive: true }); + await writeFile(join(root, file), ""); + } + if (opts.gitignore) { + await writeFile(join(root, ".gitignore"), opts.gitignore); + } + await mkdir(join(root, "scripts"), { recursive: true }); + await writeFile( + join(root, DURATION_TABLE_PATH), + formatDurationTable({ source: "fixture", seconds: opts.durations ?? {} }) + ); + return root; +} + +function expectExactPartition(opts: { + readonly shards: readonly (readonly string[])[]; + readonly files: readonly string[]; +}): void { + const flattened = opts.shards.flat(); + expect(new Set(flattened).size).toBe(flattened.length); + expect([...flattened].sort()).toEqual([...opts.files].sort()); +} + +function load(opts: { + readonly shard: readonly string[]; + readonly durations: Record; +}): number { + return opts.shard.reduce( + (total, file) => + total + plannedSeconds({ file, durations: opts.durations }), + 0 + ); +} + +describe("test shard selection", () => { + test("parses only 1 <= i <= n <= 32", () => { + expect(parseShardSelection("1/4")).toEqual({ index: 1, count: 4 }); + expect(parseShardSelection("4/4")).toEqual({ index: 4, count: 4 }); + expect(parseShardSelection(`1/${MAX_TEST_SHARD_COUNT}`)).toEqual({ + index: 1, + count: MAX_TEST_SHARD_COUNT, + }); + for (const invalid of [ + "", + "1", + "0/4", + "5/4", + "1/0", + "01/4", + "-1/4", + "1/33", + "1/4 ", + "a/b", + ]) { + expect(parseShardSelection(invalid)).toBeNull(); + } + }); + + test("recognizes Bun test-file names outside node_modules and hidden directories", () => { + for (const path of [ + "tests/a.test.ts", + "tests/sub/b_test.tsx", + "tests/c.spec.js", + "tests/d_spec.mjs", + "packages/db/tests/smoke.test.ts", + ]) { + expect(isTestFilePath(path)).toBe(true); + } + for (const path of [ + "tests/helpers/env.ts", + "tests/a.test.ts.snap", + "tests/fixtures/data.json", + "packages/db/node_modules/zod/a.test.ts", + "tests/.cache/a.test.ts", + ]) { + expect(isTestFilePath(path)).toBe(false); + } + }); + + test("assigns each file to exactly one suite", () => { + expect(suiteForFile("tests/a.test.ts", FIXTURE_SUITES).id).toBe("cli"); + expect( + suiteForFile("packages/db/tests/smoke.test.ts", FIXTURE_SUITES).id + ).toBe("db"); + expect(() => suiteForFile("src/a.test.ts", FIXTURE_SUITES)).toThrow( + "must belong to exactly one test suite, found 0" + ); + }); +}); + +describe("test shard partition", () => { + const files = Array.from( + { length: 40 }, + (_, i) => `tests/f${String(i).padStart(2, "0")}.test.ts` + ); + const durations: Record = { + "tests/f00.test.ts": 300, + "tests/f01.test.ts": 290, + "tests/f02.test.ts": 120, + "tests/f03.test.ts": 110, + }; + + test("splits files into disjoint shards whose union is the full set, deterministically", () => { + for (let count = 1; count <= 6; count += 1) { + const shards = partitionTestFiles({ files, count, durations }); + expect(shards).toHaveLength(count); + expectExactPartition({ shards, files }); + expect( + partitionTestFiles({ files: [...files].reverse(), count, durations }) + ).toEqual(shards); + for (const shard of shards) { + expect([...shard]).toEqual([...shard].sort()); + } + } + }); + + test("puts the longest files on separate shards", () => { + const shards = partitionTestFiles({ files, count: 4, durations }); + for (const long of [ + "tests/f00.test.ts", + "tests/f01.test.ts", + "tests/f02.test.ts", + "tests/f03.test.ts", + ]) { + expect(shards.filter((shard) => shard.includes(long))).toHaveLength(1); + } + expect( + shards.filter( + (shard) => + shard.includes("tests/f00.test.ts") || + shard.includes("tests/f01.test.ts") + ) + ).toHaveLength(2); + }); + + test("spreads files recorded at zero seconds instead of stacking them on one shard", () => { + const zero: Record = {}; + for (const file of files) { + zero[file] = 0; + } + const shards = partitionTestFiles({ files, count: 4, durations: zero }); + expect(shards.map((shard) => shard.length)).toEqual([10, 10, 10, 10]); + }); + + test("plans a file without a recorded duration at the default", () => { + expect(plannedSeconds({ file: "tests/new.test.ts", durations })).toBe( + DEFAULT_FILE_SECONDS + PER_FILE_START_SECONDS + ); + expect(plannedSeconds({ file: "tests/f00.test.ts", durations })).toBe( + 300 + PER_FILE_START_SECONDS + ); + const withNew = [...files, "tests/new.test.ts"]; + expectExactPartition({ + shards: partitionTestFiles({ files: withNew, count: 3, durations }), + files: withNew, + }); + }); + + test("balances the planned load within the longest file", () => { + const shards = partitionTestFiles({ files, count: 3, durations }); + const loads = shards.map((shard) => load({ shard, durations })); + expect(Math.max(...loads) - Math.min(...loads)).toBeLessThanOrEqual( + 300 + PER_FILE_START_SECONDS + ); + }); + + test("refuses duplicate paths and out-of-range counts", () => { + expect(() => + partitionTestFiles({ + files: ["tests/a.test.ts", "tests/a.test.ts"], + count: 2, + durations, + }) + ).toThrow("must be unique"); + for (const count of [0, 33, 1.5, Number.NaN]) { + expect(() => partitionTestFiles({ files, count, durations })).toThrow( + "shard count must be an integer" + ); + } + }); +}); + +describe("test shard coverage check", () => { + const files = ["tests/a.test.ts", "tests/b.test.ts", "tests/c.test.ts"]; + + test("accepts an exact cover", () => { + expect(() => + assertShardCoverage({ + expected: files, + shards: [["tests/a.test.ts"], ["tests/b.test.ts", "tests/c.test.ts"]], + }) + ).not.toThrow(); + }); + + test("mutation: a dropped file fails and is named", () => { + const shards = [["tests/a.test.ts"], ["tests/c.test.ts"]]; + expect(shardCoverage({ expected: files, shards }).missing).toEqual([ + "tests/b.test.ts", + ]); + expect(() => assertShardCoverage({ expected: files, shards })).toThrow( + "not run by any shard (1):\n tests/b.test.ts" + ); + }); + + test("mutation: a file in two shards fails and is named", () => { + const shards = [ + ["tests/a.test.ts", "tests/b.test.ts"], + ["tests/b.test.ts", "tests/c.test.ts"], + ]; + expect(() => assertShardCoverage({ expected: files, shards })).toThrow( + "run by more than one shard (1):\n tests/b.test.ts" + ); + }); + + test("mutation: a file outside the full set fails and is named", () => { + const shards = [ + ["tests/a.test.ts", "tests/b.test.ts"], + ["tests/c.test.ts", "tests/z.test.ts"], + ]; + expect(() => assertShardCoverage({ expected: files, shards })).toThrow( + "run by a shard but not in the full test set (1):\n tests/z.test.ts" + ); + }); + + test("a verified plan covers a fixture checkout's git test files exactly once", async () => { + const root = await fixtureRepo({ + files: [ + "tests/a.test.ts", + "tests/sub/b.test.ts", + "tests/helper.ts", + "packages/db/tests/smoke.test.ts", + "packages/db/node_modules/dep/x.test.ts", + ], + gitignore: "node_modules/\n", + durations: { "tests/a.test.ts": 10 }, + }); + const plan = await buildVerifiedPlan({ + repoRoot: root, + count: 2, + suites: FIXTURE_SUITES, + }); + expect(plan.files).toEqual([ + "packages/db/tests/smoke.test.ts", + "tests/a.test.ts", + "tests/sub/b.test.ts", + ]); + expect( + listGitTestFiles({ repoRoot: root, suites: FIXTURE_SUITES }) + ).toEqual([...plan.files]); + expectExactPartition({ shards: plan.shards, files: plan.files }); + }); + + test("mutation: a partition that drops a file fails the verified plan", async () => { + const root = await fixtureRepo({ + files: ["tests/a.test.ts", "tests/b.test.ts", "tests/c.test.ts"], + }); + await expect( + buildVerifiedPlan({ + repoRoot: root, + count: 2, + suites: FIXTURE_SUITES, + partition: (opts) => + partitionTestFiles({ ...opts, files: opts.files.slice(0, -1) }), + }) + ).rejects.toThrow("not run by any shard (1):\n tests/c.test.ts"); + }); + + test("mutation: a test file that git does not list fails the verified plan", async () => { + const root = await fixtureRepo({ + files: ["tests/a.test.ts", "tests/ignored/b.test.ts"], + gitignore: "tests/ignored/\n", + }); + await expect( + buildVerifiedPlan({ repoRoot: root, count: 1, suites: FIXTURE_SUITES }) + ).rejects.toThrow( + "found on disk but not listed by git (1):\n tests/ignored/b.test.ts" + ); + }); + + test("the current checkout's four CI shards cover every git test file exactly once", async () => { + const tracked = listGitTestFiles({ repoRoot }); + expect(tracked).toContain("tests/test-shard.test.ts"); + expect(tracked).toContain("packages/db/tests/smoke.test.ts"); + const plan = await buildVerifiedPlan({ repoRoot, count: 4 }); + expect(plan.files).toEqual(tracked); + expectExactPartition({ shards: plan.shards, files: tracked }); + for (const shard of plan.shards) { + expect(shard.length).toBeGreaterThan(0); + } + }); + + test("refuses a missing suite root instead of planning it as empty", async () => { + const root = await fixtureRepo({ files: ["tests/a.test.ts"] }); + await rm(join(root, "packages", "db"), { recursive: true }); + await expect( + buildVerifiedPlan({ repoRoot: root, count: 1, suites: FIXTURE_SUITES }) + ).rejects.toThrow("the db suite root packages/db is not a directory"); + }); + + test("refuses a workspace test script the shard suites do not cover", async () => { + const root = await fixtureRepo({ files: [] }); + await writeFile(join(root, "bunfig.toml"), '[test]\nroot = "./tests"\n'); + await writeFile( + join(root, "package.json"), + JSON.stringify({ workspaces: ["packages/*"] }) + ); + for (const name of ["cli", "db"]) { + await mkdir(join(root, "packages", name), { recursive: true }); + await writeFile( + join(root, "packages", name, "package.json"), + JSON.stringify({ scripts: { test: "bun test" } }) + ); + } + await expect( + assertSuiteInputs({ repoRoot: root }) + ).resolves.toBeUndefined(); + await mkdir(join(root, "packages", "extra"), { recursive: true }); + await writeFile( + join(root, "packages", "extra", "package.json"), + JSON.stringify({ scripts: { test: "bun test" } }) + ); + await expect(assertSuiteInputs({ repoRoot: root })).rejects.toThrow( + "Turbo runs test scripts in [packages/cli, packages/db, packages/extra]" + ); + await rm(join(root, "packages", "extra"), { recursive: true }); + await writeFile(join(root, "bunfig.toml"), '[test]\nroot = "./spec"\n'); + await expect(assertSuiteInputs({ repoRoot: root })).rejects.toThrow( + "[test].root must be ./tests" + ); + }); +}); + +describe("test shard duration table", () => { + test("round-trips and refuses malformed tables", () => { + const text = formatDurationTable({ + source: "fixture", + seconds: { "tests/b.test.ts": 1.26, "tests/a.test.ts": 3 }, + }); + expect(parseDurationTable(text)).toEqual({ + "tests/a.test.ts": 3, + "tests/b.test.ts": 1.3, + }); + for (const bad of [ + "not json", + "{}", + JSON.stringify({ schema: "other", seconds: {} }), + JSON.stringify({ + schema: "hack.test-shard-durations.v1", + seconds: { "tests/a.test.ts": -1 }, + }), + JSON.stringify({ + schema: "hack.test-shard-durations.v1", + seconds: { "tests/a.ts": 1 }, + }), + ]) { + expect(() => parseDurationTable(bad)).toThrow("test shard:"); + } + }); + + test("the checked-in table parses and names only existing test files", async () => { + const table = parseDurationTable( + await Bun.file(resolve(repoRoot, DURATION_TABLE_PATH)).text() + ); + const tracked = new Set(listGitTestFiles({ repoRoot })); + expect(Object.keys(table).filter((file) => !tracked.has(file))).toEqual([]); + }); + + test("merging keeps later measurements and drops deleted files", () => { + const merged = parseDurationTable( + mergeDurationTables({ + source: "fixture", + table: { "tests/a.test.ts": 1, "tests/gone.test.ts": 9 }, + measured: [{ "tests/a.test.ts": 2 }, { "tests/b.test.ts": 4 }], + files: ["tests/a.test.ts", "tests/b.test.ts"], + }) + ); + expect(merged).toEqual({ "tests/a.test.ts": 2, "tests/b.test.ts": 4 }); + }); + + test("measured output keeps finished files only", () => { + const results: FileResult[] = [ + { + file: "tests/a.test.ts", + outcome: "pass", + seconds: 2.04, + exitCode: 0, + filesRan: 1, + }, + { + file: "tests/b.test.ts", + outcome: "fail", + seconds: 1, + exitCode: 1, + filesRan: 1, + }, + { + file: "tests/c.test.ts", + outcome: "timeout", + seconds: 720, + exitCode: null, + filesRan: null, + }, + ]; + expect( + parseDurationTable(measuredDurations({ label: "fixture", results })) + ).toEqual({ + "tests/a.test.ts": 2, + "tests/b.test.ts": 1, + }); + }); +}); + +function shardState(files: readonly string[]): ShardState { + return { + label: "[test shard 1/1]", + files, + results: [], + current: null, + startedAt: Date.now(), + }; +} + +const passRun: RunFile = async () => ({ + exitCode: 0, + filesRan: 1, + stopped: false, +}); + +function hangingRun(hang: ReadonlySet): RunFile { + return ({ file, signal }) => { + if (!hang.has(file)) { + return passRun({ file, signal }); + } + return new Promise((done) => { + signal.addEventListener( + "abort", + () => done({ exitCode: null, filesRan: null, stopped: true }), + { once: true } + ); + }); + }; +} + +async function runFixture(opts: { + readonly files: readonly string[]; + readonly runFile: RunFile; + readonly fileTimeoutMs?: number; + readonly deadlineMs?: number; + readonly interrupt?: AbortSignal; +}) { + const state = shardState(opts.files); + const lines: string[] = []; + const ended = await runShardFiles({ + state, + runFile: opts.runFile, + fileTimeoutMs: opts.fileTimeoutMs ?? 10_000, + deadlineAt: state.startedAt + (opts.deadlineMs ?? 60_000), + heartbeatMs: 10_000, + interrupt: opts.interrupt ?? new AbortController().signal, + now: Date.now, + log: (line) => lines.push(line), + }); + return { state, lines, ended }; +} + +describe("test shard runner", () => { + const files = ["tests/a.test.ts", "tests/b.test.ts", "tests/c.test.ts"]; + + test("prints a start and a result line naming each file", async () => { + const { state, lines, ended } = await runFixture({ + files, + runFile: passRun, + }); + expect(ended).toBe("complete"); + expect(state.results.map((result) => result.outcome)).toEqual([ + "pass", + "pass", + "pass", + ]); + expect(lines[0]).toBe("[test shard 1/1] (1/3) start tests/a.test.ts"); + expect(lines[1]).toStartWith( + "[test shard 1/1] (1/3) PASS tests/a.test.ts in " + ); + expect(lines).toHaveLength(6); + }); + + test("a file that outlives its timeout is stopped, named and the shard continues", async () => { + const { state, ended } = await runFixture({ + files, + runFile: hangingRun(new Set(["tests/b.test.ts"])), + fileTimeoutMs: 50, + }); + expect(ended).toBe("complete"); + expect( + state.results.map((result) => [result.file, result.outcome]) + ).toEqual([ + ["tests/a.test.ts", "pass"], + ["tests/b.test.ts", "timeout"], + ["tests/c.test.ts", "pass"], + ]); + expect(summaryReport(state, Date.now()).join("\n")).toContain( + "timeout tests/b.test.ts (exit none, files reported none" + ); + }); + + test("the shard deadline stops the running file and leaves the rest unstarted", async () => { + const { state, ended } = await runFixture({ + files, + runFile: hangingRun(new Set(["tests/b.test.ts"])), + deadlineMs: 100, + }); + expect(ended).toBe("deadline"); + expect( + state.results.map((result) => [result.file, result.outcome]) + ).toEqual([ + ["tests/a.test.ts", "pass"], + ["tests/b.test.ts", "deadline"], + ]); + }); + + test("an interrupt stops the running file", async () => { + const interrupt = new AbortController(); + setTimeout(() => interrupt.abort(), 50); + const { state, ended } = await runFixture({ + files, + runFile: hangingRun(new Set(["tests/a.test.ts"])), + interrupt: interrupt.signal, + }); + expect(ended).toBe("interrupted"); + expect(state.results.map((result) => result.outcome)).toEqual([ + "interrupted", + ]); + }); + + test("the stop report names the running file, the unstarted files and the slowest files", () => { + const state = shardState(files); + state.results.push({ + file: "tests/a.test.ts", + outcome: "pass", + seconds: 42, + exitCode: 0, + filesRan: 1, + }); + state.current = { file: "tests/b.test.ts", startedAt: state.startedAt }; + const report = stopReport({ + state, + reason: "received SIGTERM", + now: state.startedAt + 125_000, + }).join("\n"); + expect(report).toContain( + "[test shard 1/1] stopped (received SIGTERM) after 2m05s; 1/3 files finished." + ); + expect(report).toContain( + "[test shard 1/1] running file: tests/b.test.ts (2m05s)" + ); + expect(report).toContain( + "[test shard 1/1] not started (1):\n tests/c.test.ts" + ); + expect(report).toContain( + "[test shard 1/1] slowest files:\n 42.0s pass tests/a.test.ts" + ); + }); + + test("a pass needs exit 0 and exactly one file reported by Bun", () => { + expect( + classifyFileRun({ + run: { exitCode: 0, filesRan: 1, stopped: false }, + stopReason: null, + }) + ).toBe("pass"); + expect( + classifyFileRun({ + run: { exitCode: 1, filesRan: 1, stopped: false }, + stopReason: null, + }) + ).toBe("fail"); + for (const filesRan of [0, 2, null]) { + expect( + classifyFileRun({ + run: { exitCode: 0, filesRan, stopped: false }, + stopReason: null, + }) + ).toBe("file-count"); + } + expect( + classifyFileRun({ + run: { exitCode: null, filesRan: null, stopped: true }, + stopReason: "timeout", + }) + ).toBe("timeout"); + }); + + test("reads the file count from Bun's last summary line", () => { + expect( + bunFilesRan(" 5 pass\nRan 5 tests across 1 file. [142.00ms]\n") + ).toBe(1); + expect( + bunFilesRan("Ran 1 test across 1 file.\nRan 7 tests across 2 files. [1s]") + ).toBe(2); + expect(bunFilesRan("error: something failed\n")).toBeNull(); + }); + + test("formats durations", () => { + expect(formatDuration(0.04)).toBe("0.0s"); + expect(formatDuration(12.34)).toBe("12.3s"); + expect(formatDuration(59.96)).toBe("1m00s"); + expect(formatDuration(725)).toBe("12m05s"); + }); +}); + +type Workflow = { + readonly jobs: Record< + string, + { + readonly name?: string; + readonly needs?: string | readonly string[]; + readonly if?: string; + readonly "runs-on"?: string; + readonly "timeout-minutes"?: number; + readonly strategy?: { + readonly "fail-fast"?: boolean; + readonly matrix?: { readonly shard?: readonly number[] }; + }; + readonly steps: readonly { + readonly name?: string; + readonly if?: string; + readonly run?: string; + readonly uses?: string; + readonly env?: Record; + }[]; + } + >; +}; + +async function ciWorkflow(): Promise { + return Bun.YAML.parse( + await Bun.file(resolve(repoRoot, ".github/workflows/ci.yml")).text() + ) as Workflow; +} + +describe("test shard per-file bun test process", () => { + async function bunFixture(): Promise { + const root = await mkdtemp(join(tmpdir(), "hack-test-shard-bun-")); + owned.push(root); + const files: Record = { + "tests/pass.test.ts": + 'import { expect, test } from "bun:test";\ntest("passes", () => expect(1).toBe(1));\n', + "tests/fail.test.ts": + 'import { expect, test } from "bun:test";\ntest("fails", () => expect(1).toBe(2));\n', + "tests/hang.test.ts": + 'import { test } from "bun:test";\ntest("hangs", () => new Promise(() => undefined), 60_000);\n', + // A background writer that keeps the stderr pipe open after Bun exits. + "tests/writer.test.ts": + 'import { test } from "bun:test";\ntest("leaves a writer", () => { Bun.spawn(["sh", "-c", "sleep 6; echo late >&2"], { stdio: ["ignore", "ignore", "inherit"] }).unref(); });\n', + "packages/db/tests/cwd.test.ts": + 'import { expect, test } from "bun:test";\ntest("runs in the suite directory", () => expect(process.cwd().endsWith("/packages/db")).toBe(true));\n', + }; + for (const [path, text] of Object.entries(files)) { + await mkdir(dirname(join(root, path)), { recursive: true }); + await writeFile(join(root, path), text); + } + return root; + } + + test("reports exit status and exactly one file from the suite directory", async () => { + const root = await bunFixture(); + const run = bunTestFile({ + repoRoot: root, + suites: FIXTURE_SUITES, + quiet: true, + }); + const signal = new AbortController().signal; + expect(await run({ file: "tests/pass.test.ts", signal })).toEqual({ + exitCode: 0, + filesRan: 1, + stopped: false, + }); + expect( + await run({ file: "packages/db/tests/cwd.test.ts", signal }) + ).toEqual({ + exitCode: 0, + filesRan: 1, + stopped: false, + }); + const failed = await run({ file: "tests/fail.test.ts", signal }); + expect(failed.exitCode).not.toBe(0); + expect(failed.stopped).toBe(false); + }, 30_000); + + test("stops a hung file when its signal aborts", async () => { + const root = await bunFixture(); + const controller = new AbortController(); + setTimeout(() => controller.abort(), 1000); + const startedAt = Date.now(); + const run = await bunTestFile({ + repoRoot: root, + suites: FIXTURE_SUITES, + quiet: true, + })({ + file: "tests/hang.test.ts", + signal: controller.signal, + }); + expect(run).toEqual({ exitCode: null, filesRan: null, stopped: true }); + expect(Date.now() - startedAt).toBeLessThan(8000); + }, 30_000); + + test("does not wait for a leftover process that holds stderr open", async () => { + const root = await bunFixture(); + const startedAt = Date.now(); + const run = await bunTestFile({ + repoRoot: root, + suites: FIXTURE_SUITES, + quiet: true, + })({ + file: "tests/writer.test.ts", + signal: new AbortController().signal, + }); + expect(run.exitCode).toBe(0); + expect(Date.now() - startedAt).toBeLessThan(5500); + }, 30_000); +}); + +describe("hosted test CI fan-in", () => { + test("the shard matrix covers 1..n, keeps the suite setup and runs its own slice", async () => { + const shardJob = (await ciWorkflow()).jobs["test-shard"]; + const matrix = shardJob?.strategy?.matrix?.shard ?? []; + const count = matrix.length; + expect(count).toBeGreaterThanOrEqual(3); + expect(matrix).toEqual(Array.from({ length: count }, (_, i) => i + 1)); + expect(shardJob?.strategy?.["fail-fast"]).toBe(false); + expect(shardJob?.name).toBe(`test shard \${{ matrix.shard }}/${count}`); + expect(shardJob?.["runs-on"]).toBe("blacksmith-6vcpu-macos-15"); + expect(shardJob?.["timeout-minutes"]).toBeLessThan(60); + const steps = shardJob?.steps ?? []; + const run = steps.find((step) => step.name === "Tests (shard)"); + expect(run?.env?.HACK_TEST_SHARD).toBe(`\${{ matrix.shard }}/${count}`); + expect(run?.run).toContain( + 'bun scripts/test-shard.ts run --shard="$HACK_TEST_SHARD"' + ); + const commands = steps.map((step) => step.run ?? step.uses ?? ""); + expect(commands.indexOf("bun run build:config-compiler")).toBeLessThan( + steps.findIndex((step) => step.name === "Tests (shard)") + ); + for (const command of [ + "bun install --frozen-lockfile", + "bun run turbo:typecheck", + "bun run privacy:check", + "bun run turbo:check", + "bun run build", + ]) { + expect(commands).toContain(command); + } + const plan = await buildVerifiedPlan({ repoRoot, count }); + expect(plan.shards).toHaveLength(count); + }); + + test("test passes only when every shard succeeded and never skips", async () => { + const workflow = await ciWorkflow(); + const fanIn = workflow.jobs.test; + expect(fanIn?.name ?? "test").toBe("test"); + expect(fanIn?.needs).toBe("test-shard"); + expect(fanIn?.if).toBe("${{ always() }}"); + const gate = fanIn?.steps.find( + (step) => step.name === "Require every test shard to succeed" + ); + expect(gate?.env?.SHARD_RESULT).toBe("${{ needs.test-shard.result }}"); + expect(gate?.run).toContain('if [ "$SHARD_RESULT" != "success" ]; then'); + expect(gate?.run).toContain("exit 1"); + const named = Object.entries(workflow.jobs).filter( + ([key, job]) => (job.name ?? key) === "test" + ); + expect(named.map(([key]) => key)).toEqual(["test"]); + }); + + test("the prerelease gate requires the fan-in and every shard job by name", async () => { + const count = + (await ciWorkflow()).jobs["test-shard"]?.strategy?.matrix?.shard + ?.length ?? 0; + const required: readonly string[] = CI_JOBS; + expect(required).toContain("test"); + for (let shard = 1; shard <= count; shard += 1) { + expect(required).toContain(`test shard ${shard}/${count}`); + } + expect( + required.filter((name) => name.startsWith("test shard ")) + ).toHaveLength(count); + }); +});