fix(ci): shard the hosted test suite so a slow runner cannot hit the 60 min limit

.ai/skills/hack-repo-verify/SKILL.md

@@ -98,10 +98,27 @@ untracked candidate additions. Keep private artifacts excluded from the index.
 | Native authored command slices | `tests/native-authored-command.test.ts` (unsupported matrix), `tests/native-authored-command-dispatch.test.ts`, `tests/native-authored-project-{start,run,observe,logs,exec,recovery}.test.ts` and the [command contract](../../../docs/reference/native-authored-commands.md); the dispatch "source CLI" cases need the prepared compiler (`bun run build:config-compiler` or absolute `HACK_CONFIG_COMPILER_BINARY`) and fail without it; `native-authored-project-start` and `native-authored-project-exec` are nondeterministic on a local macOS host, so a single local pass or failure is not proof; live acceptance runs host-serialized on one prepared macOS provider slot |
 | Native VM/routing/reclamation | [candidate guide](../../../docs/guides/native-candidate.md), relevant acceptance gate and isolated host fixture; Linux checks cannot qualify macOS effects |
 | Resource/performance claims | [performance skill](../hack-repo-performance/SKILL.md), matched workload and measured boundaries |
-| Bun runtime crashes | Read-only history: `bun run ci:bun-crash-census -- --since <UTC> --until <UTC>` (failed `test` job logs; `--scan-passing` for all). Matched reproduction: the gated [experiment](../../../.github/workflows/bun-crash-experiment.yml), which spends hosted macOS minutes and needs an explicit go-ahead; never loop on a shared host. Its default `focused` mode loops `tests/native-https-owner.test.ts` (where both original segfaults happened); `full` needs a budget of at least 40 minutes because one full suite takes 30-35. A run never starts unless its estimate fits the remaining budget, so read `summary.json` for the verdict and per-mode crash, failure and skipped-for-budget counts. Both share [the classifier](../../../scripts/lib/bun-crash-classify.ts) |
+| Bun runtime crashes | Read-only history: `bun run ci:bun-crash-census -- --since <UTC> --until <UTC>` (failed `test` job logs; `--scan-passing` for all; since the suite moved to `test shard i/4` jobs, pass `--job "test shard i/4"` once per shard for newer runs). Matched reproduction: the gated [experiment](../../../.github/workflows/bun-crash-experiment.yml), which spends hosted macOS minutes and needs an explicit go-ahead; never loop on a shared host. Its default `focused` mode loops `tests/native-https-owner.test.ts` (where both original segfaults happened); `full` needs a budget of at least 40 minutes because one full suite takes 30-35. A run never starts unless its estimate fits the remaining budget, so read `summary.json` for the verdict and per-mode crash, failure and skipped-for-budget counts. Both share [the classifier](../../../scripts/lib/bun-crash-classify.ts) |
 
 Read [CI](../../../.github/workflows/ci.yml) for current hosted gates. A local pass
-does not mean CI ran. Existing unsupported remote/gateway/native-app checks are
+does not mean CI ran.
+
+Hosted CI runs the `bun run test` file set on four `test shard i/4` jobs;
+the required `test` check is their fan-in and passes only when every shard did.
+[`scripts/test-shard-plan.ts`](../../../scripts/test-shard-plan.ts) splits the files
+by the durations in `scripts/test-shard-durations.json`. A file with no recorded
+duration is planned at 5 s. Each shard runs each of its files in its own
+`bun test ./<file>` process and prints a start line and a result line per file.
+It stops a file after 12 min, and stops the shard at 30 min or on cancellation.
+When it stops early it names the running file, the files not started and the
+slowest files. Each shard first checks that the four shards cover the files
+`git ls-files` lists exactly once. To reproduce one shard locally, run
+`bun scripts/test-shard.ts plan --shard=i/4` or
+`bun scripts/test-shard.ts run --shard=i/4`. To check the split after adding
+or moving tests, run `bun scripts/test-shard.ts verify --count=4`. To refresh the
+durations, download the `test-shard-durations-<i>` artifacts and run
+`bun scripts/test-shard.ts update-durations --source=<run> <files>`. A per-file
+process does not share module state with other files, unlike one `bun test` run. Existing unsupported remote/gateway/native-app checks are
 not default local-product requirements; use them only for explicitly scoped work.
 
 ## Verify the verifier

.github/workflows/ci.yml

@@ -361,9 +361,21 @@ jobs:
             exit 1
           fi
 
-  test:
+  test-shard:
+    # One deterministic slice of the `bun run test` file set per runner
+    # (scripts/test-shard-plan.ts, HACK-1381). The serial job took 37-42 min and
+    # reached the 60 min limit on slow runners; each file now runs in its own
+    # `bun test` process with a start/result line, so a hang names its file.
+    name: test shard ${{ matrix.shard }}/4
+    strategy:
+      # Let every shard finish so one failure still reports the other files.
+      fail-fast: false
+      matrix:
+        shard: [1, 2, 3, 4]
     runs-on: blacksmith-6vcpu-macos-15
-    timeout-minutes: 60
+    # A shard plans about 10.5 min of tests. The runner's own 30 min deadline
+    # stops first and names the running file; this bounds a hung runner.
+    timeout-minutes: 40
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -373,22 +385,61 @@ jobs:
           bun-version: "1.4.2"
       - name: Install dependencies
         run: bun install --frozen-lockfile
+      # The one-off checks and builds run once, on shard 1.
       - name: Typecheck (Turbo)
+        if: matrix.shard == 1
         run: bun run turbo:typecheck
       - name: Privacy check
+        if: matrix.shard == 1
         run: bun run privacy:check
       - name: Quality checks (Turbo)
+        if: matrix.shard == 1
         run: bun run turbo:check
       - uses: dtolnay/rust-toolchain@1.97.1
       - name: Prepare matching compiler for private file owner tests
         run: bun run build:config-compiler
-      - name: Tests (Turbo)
-        run: bun run turbo:test
+      # Runs check:bun-version and test:preflight as `bun run test` does, then
+      # verifies that the four shards cover the full test file set exactly once.
+      - name: Tests (shard)
+        env:
+          HACK_TEST_SHARD: ${{ matrix.shard }}/4
+        run: bun scripts/test-shard.ts run --shard="$HACK_TEST_SHARD" --durations-out="$RUNNER_TEMP/test-shard-durations.json"
+      - name: Preserve measured file durations
+        # Per-file seconds only; refresh scripts/test-shard-durations.json with
+        # `bun scripts/test-shard.ts update-durations`.
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-shard-durations-${{ matrix.shard }}
+          path: ${{ runner.temp }}/test-shard-durations.json
+          if-no-files-found: ignore
+          retention-days: 14
       - name: Build CLI
+        if: matrix.shard == 1
         run: bun run build
       - name: Build release smoke (no tests)
+        if: matrix.shard == 1
         run: bun run build:release --skip-tests --no-clean --out=dist/release-ci
 
+  test:
+    # The required `test` check (repository ruleset) for `next` and `main`. It
+    # runs even when a shard fails or is cancelled and passes only when every
+    # shard succeeded, so a failed shard can never surface as a skipped check.
+    needs: test-shard
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions: {}
+    steps:
+      - name: Require every test shard to succeed
+        env:
+          SHARD_RESULT: ${{ needs.test-shard.result }}
+        run: |
+          if [ "$SHARD_RESULT" != "success" ]; then
+            echo "test shards did not all succeed (result: $SHARD_RESULT)"
+            exit 1
+          fi
+
   linux-process-lifetime:
     runs-on: blacksmith-4vcpu-ubuntu-2404
     timeout-minutes: 15

scripts/prerelease-plan.ts

@@ -223,7 +223,7 @@ export function requireHumanApproval({
   }
 }
 
-/** Jobs that must each succeed on the exact-head CI run; `docker-e2e` is the shard fan-in. */
+/** Jobs that must each succeed on the exact-head CI run; `docker-e2e` and `test` are shard fan-ins. */
 export const CI_JOBS = [
   "Runtime state models",
   "Candidate core (ubuntu-latest)",
@@ -235,6 +235,10 @@ export const CI_JOBS = [
   "docker-e2e shard 3/4",
   "docker-e2e shard 4/4",
   "docker-e2e",
+  "test shard 1/4",
+  "test shard 2/4",
+  "test shard 3/4",
+  "test shard 4/4",
   "test",
   "linux-process-lifetime",
 ] as const;