From d9c05a82e30008be7e732fcd52cce412d9d98dd1 Mon Sep 17 00:00:00 2001 From: wan9chi Date: Mon, 5 Oct 2026 12:43:32 +0800 Subject: [PATCH] test(cache): authenticate remote cache e2e uploads with a GitHub Actions stand-in The public cache service only accepts uploads with a GitHub Actions OIDC token for a push to the registered repository's main branch, whose audience is the endpoint. The test backend accepted any upload, so the e2e cases never sent a token and never exercised the client's OIDC support. The backend process now stands in for GitHub's token service, and `remote-cache-server run --github-actions` runs a command as if in a GitHub Actions job with `id-token: write`. The request token picks the workflow run that the token is for, a push to `main` by default. Like the service, the backend checks the token's signature, time bounds, audience, and claims, and answers other stores with `401 Invalid credentials` or `403 Write not permitted`. Every step that uploads now runs with `--github-actions`. The only snapshot changes are those command lines. Co-Authored-By: Claude Opus 5.5 --- .../fixtures/remote_cache/snapshots.toml | 17 +++- .../remote_cache/snapshots/corrupt_archive.md | 2 +- .../snapshots/ctrl_c_during_upload.md | 2 +- .../remote_cache/snapshots/fallback.md | 2 +- .../snapshots/fast_fail_during_upload.md | 2 +- .../snapshots/hide_pending_uploads.md | 2 +- .../snapshots/local_and_remote_hits.md | 2 +- .../remote_cache/snapshots/pending_uploads.md | 2 +- .../fixtures/remote_cache/snapshots/read.md | 2 +- .../remote_cache/snapshots/read_write.md | 4 +- .../remote_cache/snapshots/restore.md | 4 +- .../remote_cache/snapshots/restore_failure.md | 2 +- packages/tools/README.md | 13 ++- packages/tools/src/remote-cache/backend.ts | 78 +++++++++++++++++- packages/tools/src/remote-cache/cli.ts | 18 +++-- packages/tools/src/remote-cache/daemon.ts | 32 +++++++- packages/tools/src/remote-cache/github.ts | 79 +++++++++++++++++++ packages/tools/src/remote-cache/state.ts | 2 + 18 files changed, 238 insertions(+), 27 deletions(-) create mode 100644 packages/tools/src/remote-cache/github.ts diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots.toml b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots.toml index 5adb4521a..f05c9f7bb 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots.toml +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots.toml @@ -1,7 +1,9 @@ # Cases that use the remote cache backend are ignored because it runs on # Node.js. Windows is skipped because the PTY launcher cannot execute pnpm # command shims. Each of these cases starts its own backend first, runs -# commands against it with `remote-cache-server run`, and stops it last. +# commands against it with `remote-cache-server run`, and stops it last. The +# backend only accepts uploads with a GitHub Actions token, so steps that +# upload run with `--github-actions`, as a job of a push to the main branch. # Whether an upload to the backend is still running when the tasks finish # depends on timing, so steps that upload hide the message about pending # uploads. The cases that test it stall the uploads instead, with @@ -18,6 +20,7 @@ steps = [ { argv = [ "remote-cache-server", "run", + "--github-actions", "vt", "run", "build", @@ -34,6 +37,7 @@ steps = [ { argv = [ "remote-cache-server", "run", + "--github-actions", "vt", "run", "build", @@ -61,6 +65,7 @@ steps = [ { argv = [ "remote-cache-server", "run", + "--github-actions", "vt", "run", "build", @@ -105,6 +110,7 @@ steps = [ { argv = [ "remote-cache-server", "run", + "--github-actions", "vt", "run", "build", @@ -132,6 +138,7 @@ steps = [ { argv = [ "remote-cache-server", "run", + "--github-actions", "vt", "run", "build", @@ -189,6 +196,7 @@ steps = [ { argv = [ "remote-cache-server", "run", + "--github-actions", "vt", "run", "build", @@ -242,6 +250,7 @@ steps = [ { argv = [ "remote-cache-server", "run", + "--github-actions", "vt", "run", "build", @@ -292,6 +301,7 @@ steps = [ { argv = [ "remote-cache-server", "run", + "--github-actions", "vt", "run", "build", @@ -353,6 +363,7 @@ steps = [ { argv = [ "remote-cache-server", "run", + "--github-actions", "vt", "run", "build", @@ -433,6 +444,7 @@ steps = [ { argv = [ "remote-cache-server", "run", + "--github-actions", "vtt", "stalled-remote-cache", "--stall", @@ -655,6 +667,7 @@ steps = [ { argv = [ "remote-cache-server", "run", + "--github-actions", "vtt", "stalled-remote-cache", "--stall", @@ -699,6 +712,7 @@ steps = [ { argv = [ "remote-cache-server", "run", + "--github-actions", "vtt", "stalled-remote-cache", "--stall", @@ -733,6 +747,7 @@ steps = [ { argv = [ "remote-cache-server", "run", + "--github-actions", "vtt", "stalled-remote-cache", "--stall", diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/corrupt_archive.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/corrupt_archive.md index fda21925f..2ee9ceaf5 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/corrupt_archive.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/corrupt_archive.md @@ -5,7 +5,7 @@ ``` ``` -## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run vt run build` +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run --github-actions vt run build` ``` $ vtt write-file dist/output.txt built diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/ctrl_c_during_upload.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/ctrl_c_during_upload.md index 7b58cfb0a..ac45358e7 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/ctrl_c_during_upload.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/ctrl_c_during_upload.md @@ -5,7 +5,7 @@ ``` ``` -## `VP_REMOTE_CACHE=read-write remote-cache-server run vtt stalled-remote-cache --stall /store vt run build` +## `VP_REMOTE_CACHE=read-write remote-cache-server run --github-actions vtt stalled-remote-cache --stall /store vt run build` The proxy forwards the fetch to the backend, which has no entry, but never forwards the upload. Ctrl-C cancels it while vt run waits. diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/fallback.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/fallback.md index 46d51b4d5..c74d23543 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/fallback.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/fallback.md @@ -5,7 +5,7 @@ ``` ``` -## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run vt run build` +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run --github-actions vt run build` ``` $ vtt write-file dist/output.txt built diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/fast_fail_during_upload.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/fast_fail_during_upload.md index a9ae14f86..ea5940f0e 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/fast_fail_during_upload.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/fast_fail_during_upload.md @@ -5,7 +5,7 @@ ``` ``` -## `VP_REMOTE_CACHE=read-write remote-cache-server run vtt stalled-remote-cache --stall /store vt run fail-after-build` +## `VP_REMOTE_CACHE=read-write remote-cache-server run --github-actions vtt stalled-remote-cache --stall /store vt run fail-after-build` The proxy never forwards the upload. fail-after-build exits after build finishes, which doesn't cancel build's upload, so vt run waits for it until Ctrl-C. diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/hide_pending_uploads.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/hide_pending_uploads.md index 77394fc6c..cd6ac44ec 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/hide_pending_uploads.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/hide_pending_uploads.md @@ -5,7 +5,7 @@ ``` ``` -## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run vtt stalled-remote-cache --stall /store vt run build` +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run --github-actions vtt stalled-remote-cache --stall /store vt run build` The proxy never forwards the upload. vt run waits for it until Ctrl-C, without the message about pending uploads. diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/local_and_remote_hits.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/local_and_remote_hits.md index a514d151c..a483a3b8a 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/local_and_remote_hits.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/local_and_remote_hits.md @@ -5,7 +5,7 @@ ``` ``` -## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run vt run build` +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run --github-actions vt run build` ``` $ vtt write-file dist/output.txt built diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/pending_uploads.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/pending_uploads.md index fe68b6a0c..4ab9fed0d 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/pending_uploads.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/pending_uploads.md @@ -5,7 +5,7 @@ ``` ``` -## `VP_REMOTE_CACHE=read-write remote-cache-server run vtt stalled-remote-cache --stall /store vt run all` +## `VP_REMOTE_CACHE=read-write remote-cache-server run --github-actions vtt stalled-remote-cache --stall /store vt run all` The proxy never forwards the uploads. check doesn't wait for build's upload, so both are still running when check finishes, and vt run waits for them until Ctrl-C. diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/read.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/read.md index dd843e4e2..9d5a33314 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/read.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/read.md @@ -5,7 +5,7 @@ ``` ``` -## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run vt run build` +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run --github-actions vt run build` ``` $ vtt write-file dist/output.txt built diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/read_write.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/read_write.md index 1f8cf0286..d1d560b15 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/read_write.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/read_write.md @@ -5,7 +5,7 @@ ``` ``` -## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run vt run build` +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run --github-actions vt run build` The fetch finds no entry. The new execution is uploaded with one store request, and vt run waits for it after the task finishes. @@ -16,7 +16,7 @@ $ vtt write-file dist/output.txt built [remote-cache] POST /store 200 ``` -## `VP_REMOTE_CACHE=read-write remote-cache-server run vt run build` +## `VP_REMOTE_CACHE=read-write remote-cache-server run --github-actions vt run build` A local hit makes no requests. diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/restore.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/restore.md index 04a6e1983..aabd036b3 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/restore.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/restore.md @@ -5,7 +5,7 @@ ``` ``` -## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run vt run build` +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run --github-actions vt run build` ``` $ vtt write-file dist/output.txt built @@ -24,7 +24,7 @@ $ vtt write-file dist/output.txt built ``` ``` -## `VP_REMOTE_CACHE=read-write remote-cache-server run vt run build` +## `VP_REMOTE_CACHE=read-write remote-cache-server run --github-actions vt run build` A remote hit downloads the output archive. Hits never upload. diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/restore_failure.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/restore_failure.md index b15d0e160..4f8aaf0f3 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/restore_failure.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/restore_failure.md @@ -5,7 +5,7 @@ ``` ``` -## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run vt run build` +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run --github-actions vt run build` ``` $ vtt write-file dist/output.txt built diff --git a/packages/tools/README.md b/packages/tools/README.md index 0c5c8bac5..92f9b59c9 100644 --- a/packages/tools/README.md +++ b/packages/tools/README.md @@ -6,14 +6,14 @@ Run `pnpm install` at the repository root to install `remote-cache-server` into ```sh remote-cache-server start -VP_REMOTE_CACHE=read-write remote-cache-server run vt run build +VP_REMOTE_CACHE=read-write remote-cache-server run --github-actions vt run build remote-cache-server stop ``` An E2E case starts its own backend in its first step and stops it in its last, so the backend keeps its state for the whole case. Each subcommand works in the current directory, the case's directory: - `remote-cache-server start` starts the backend in the background on free loopback ports and returns once it's ready. The backend runs in its own session and doesn't use the terminal, so the step can finish and Ctrl-C in later steps doesn't reach it. It writes its endpoint, `http://127.0.0.1:/projects/test`, to `remote-cache/server.json`, and its output to `remote-cache/server.log`. -- `remote-cache-server run COMMAND [ARGS...]` runs the command with `VP_REMOTE_CACHE_URL` set to the endpoint. The fixed base path gives the endpoint a namespace path. The command inherits stdio and handles Ctrl-C, which `run` ignores. `run` exits with the command's exit code. +- `remote-cache-server run [--github-actions] COMMAND [ARGS...]` runs the command with `VP_REMOTE_CACHE_URL` set to the endpoint. The fixed base path gives the endpoint a namespace path. With `--github-actions`, the command runs as if in a GitHub Actions job with `id-token: write`: `ACTIONS_ID_TOKEN_REQUEST_URL` points to the backend's stand-in for GitHub's token service, and `ACTIONS_ID_TOKEN_REQUEST_TOKEN` defaults to `main-push`. The command inherits stdio and handles Ctrl-C, which `run` ignores. `run` exits with the command's exit code. - `remote-cache-server corrupt-blob NUMBER` overwrites a stored blob, numbered as in the request lines, with other bytes. - `remote-cache-server stop` stops the backend. It fails if the backend answered a request with a 5xx status or couldn't be reached. @@ -28,9 +28,16 @@ The endpoint is a tap in front of the backend. It forwards every request and res [remote-cache] GET /blob/1 200 ``` +Like the public cache service, the backend only accepts a store with a GitHub Actions token for a push to the main branch of its registered repository, whose audience is the endpoint. It answers other stores as the service does: `401` with `Invalid credentials` for a missing or invalid token, and `403` with `Write not permitted` for a token that the write policy doesn't allow. The stand-in signs tokens with a key that the backend trusts in place of GitHub's, for the workflow run that the request token stands for: + +| Request token | Workflow run | +| -------------- | ----------------------------------------- | +| `main-push` | A push to `main` of `owner/repository` | +| `pull-request` | A pull request against `owner/repository` | + The backend keeps its state in `remote-cache/`. `state.json` holds the entries and associations, with keys and values hex-encoded. Each blob is a file in `remote-cache/blobs/` named by its blob ID, a random UUID. -The backend implements `POST /fetch`, `POST /store`, and `GET /blob/{blob_id}` from the [remote cache server API](https://github.com/voidzero-dev/vite-task/pull/713). A fetch that matches neither key gets a `404` with the plain-text body `Not found`. Keys, values, and blobs are opaque bytes without length limits. There is no authentication. +The backend implements `POST /fetch`, `POST /store`, and `GET /blob/{blob_id}` from the [remote cache server API](https://github.com/voidzero-dev/vite-task/pull/713). A fetch that matches neither key gets a `404` with the plain-text body `Not found`. Keys, values, and blobs are opaque bytes without length limits. Writes need a token as described above. Run `pnpm --filter vite-task-tools check` for type checking. Run `cargo test -p vt_bin --test e2e_snapshots -- remote_cache --ignored` for the snapshots that use the backend. diff --git a/packages/tools/src/remote-cache/backend.ts b/packages/tools/src/remote-cache/backend.ts index 559d2236d..44c5d7479 100644 --- a/packages/tools/src/remote-cache/backend.ts +++ b/packages/tools/src/remote-cache/backend.ts @@ -1,12 +1,13 @@ import { Busboy } from '@fastify/busboy'; import { decode } from 'cbor2/decoder'; import { encode } from 'cbor2/encoder'; -import { randomUUID } from 'node:crypto'; +import { randomUUID, verify } from 'node:crypto'; import { once } from 'node:events'; import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'; import type { AddressInfo } from 'node:net'; import { join } from 'node:path'; +import { issuer, type repository, type TrustedKey } from './github.ts'; interface Entry { value: string; @@ -91,6 +92,51 @@ async function readParts(body: Buffer, contentType: string): Promise | undefined { + try { + const value: unknown = JSON.parse(Buffer.from(part, 'base64url').toString()); + return typeof value === 'object' && value !== null + ? (value as Record) + : undefined; + } catch { + return undefined; + } +} + +/** + * The claims of `token`, if it's an RS256 JSON Web Token that `key` signed for + * GitHub's issuer and it's valid now, with the time bounds the service uses. + */ +function verifiedClaims(token: string, key: TrustedKey): Record | undefined { + const [header, payload, signature] = token.split('.') as [string, string, string]; + const protectedHeader = decodePart(header); + if (protectedHeader?.['alg'] !== 'RS256' || protectedHeader['kid'] !== key.kid) return undefined; + const signed = Buffer.from(`${header}.${payload}`); + if (!verify('sha256', signed, key.publicKey, Buffer.from(signature, 'base64url'))) + return undefined; + const claims = decodePart(payload); + const [exp, nbf, iat] = [claims?.['exp'], claims?.['nbf'], claims?.['iat']]; + const now = Date.now() / 1000; + if ( + claims?.['iss'] !== issuer || + !Number.isSafeInteger(exp) || + !Number.isSafeInteger(nbf) || + !Number.isSafeInteger(iat) + ) { + return undefined; + } + const [expires, notBefore, issued] = [exp as number, nbf as number, iat as number]; + const valid = + expires > now && + notBefore <= now + 30 && + issued <= now + 30 && + issued >= now - 930 && + notBefore <= expires && + issued < expires && + expires - issued <= 900; + return valid ? claims : undefined; +} + function cbor(response: ServerResponse, value: unknown): void { response.writeHead(200, { 'content-type': 'application/cbor' }); response.end(encode(value)); @@ -110,13 +156,25 @@ export interface Backend { * `directory`: entries and associations in `state.json`, and each blob in * `blobs/` under its ID, a random UUID. Keys, values, and blobs remain opaque * bytes. A fetch that matches neither key gets a plain-text 404. + * + * Like the public cache service, the backend only accepts a store with a + * GitHub Actions token whose audience is `endpoint`, for a push to the main + * branch of `registered`. It trusts tokens signed with `key` in place of + * GitHub's. It answers other stores as the service does: 401 for a missing or + * invalid token, and 403 for a token that the write policy doesn't allow. */ export async function startBackend({ basePath, directory, + endpoint, + key, + registered, }: { basePath: string; directory: string; + endpoint: string; + key: TrustedKey; + registered: typeof repository; }): Promise { const stateFile = join(directory, 'state.json'); const blobDirectory = join(directory, 'blobs'); @@ -124,6 +182,23 @@ export async function startBackend({ ? JSON.parse(readFileSync(stateFile, 'utf8')) : { entries: {}, associations: {} }; const entries = new Map(Object.entries(state.entries)); + + function authorize(authorization: string | undefined): void { + const token = /^Bearer ([\w-]+\.[\w-]+\.[\w-]+)$/i.exec(authorization ?? '')?.[1]; + const claims = token === undefined ? undefined : verifiedClaims(token, key); + if (claims === undefined) throw new RequestError(401, 'Invalid credentials'); + if ( + claims['aud'] !== endpoint || + claims['repository_id'] !== registered.id || + claims['repository_owner_id'] !== registered.ownerId || + claims['repository_visibility'] !== 'public' || + claims['ref'] !== registered.branch || + claims['ref_type'] !== 'branch' || + claims['event_name'] !== 'push' + ) { + throw new RequestError(403, 'Write not permitted'); + } + } const associations = new Map(Object.entries(state.associations)); async function handle( @@ -142,6 +217,7 @@ export async function startBackend({ throw new RequestError(404, 'Route not found'); } + if (path === `${basePath}/store`) authorize(request.headers.authorization); const contentType = request.headers['content-type'] ?? ''; const body = await readBody(request); if (path === `${basePath}/fetch`) { diff --git a/packages/tools/src/remote-cache/cli.ts b/packages/tools/src/remote-cache/cli.ts index 89c2a6b1b..2014bd71e 100755 --- a/packages/tools/src/remote-cache/cli.ts +++ b/packages/tools/src/remote-cache/cli.ts @@ -6,10 +6,11 @@ import { request } from 'node:http'; import { tmpdir } from 'node:os'; import { resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { defaultRequestToken } from './github.ts'; import { logFile, serverFile, stateDirectory, type ServerInfo } from './state.ts'; const usage = - 'Usage: remote-cache-server start | run COMMAND [ARGS...] | corrupt-blob NUMBER | stop'; + 'Usage: remote-cache-server start | run [--github-actions] COMMAND [ARGS...] | corrupt-blob NUMBER | stop'; const directory = resolve(stateDirectory); function fail(message: string): never { @@ -73,15 +74,20 @@ async function start(): Promise { daemon.unref(); } -async function run([command, ...args]: string[]): Promise { +async function run(argv: string[]): Promise { + const githubActions = argv[0] === '--github-actions'; + const [command, ...args] = githubActions ? argv.slice(1) : argv; if (command === undefined) fail(usage); const info = readInfo(); + const env: NodeJS.ProcessEnv = { ...process.env, VP_REMOTE_CACHE_URL: info.url }; + if (githubActions) { + // As in a GitHub Actions job with `id-token: write`. + env['ACTIONS_ID_TOKEN_REQUEST_URL'] = info.tokenRequestUrl; + env['ACTIONS_ID_TOKEN_REQUEST_TOKEN'] ??= defaultRequestToken; + } // Ctrl-C is left to the command. process.on('SIGINT', () => {}); - const child = spawn(command, args, { - stdio: 'inherit', - env: { ...process.env, VP_REMOTE_CACHE_URL: info.url }, - }); + const child = spawn(command, args, { stdio: 'inherit', env }); const [code] = (await once(child, 'exit')) as [number | null]; printRequests(await control(info, '/take')); process.exitCode = code; diff --git a/packages/tools/src/remote-cache/daemon.ts b/packages/tools/src/remote-cache/daemon.ts index 1a95e579e..2530e104d 100644 --- a/packages/tools/src/remote-cache/daemon.ts +++ b/packages/tools/src/remote-cache/daemon.ts @@ -4,6 +4,7 @@ import { existsSync, rmSync, writeFileSync } from 'node:fs'; import { createServer, request as forward, type IncomingHttpHeaders, type Server } from 'node:http'; import type { AddressInfo } from 'node:net'; import { startBackend } from './backend.ts'; +import { createSigningKey, repository } from './github.ts'; import { basePath, serverFile, type ServerInfo } from './state.ts'; // The process that `remote-cache-server start` leaves running for one e2e case. @@ -11,7 +12,8 @@ import { basePath, serverFile, type ServerInfo } from './state.ts'; // and responses unchanged and records a line for each response. Blob IDs in // the lines become numbers in upload order, so they're the same on every run. // A control server hands the lines to `remote-cache-server run` and `stop`, -// and changes blobs for `remote-cache-server corrupt-blob`. +// changes blobs for `remote-cache-server corrupt-blob`, and stands in for +// GitHub Actions' token service. const directory = process.argv[2]!; /** Stop after this long without requests, e.g. when a case timed out before its stop step. */ @@ -25,7 +27,7 @@ const hopByHop = new Set([ 'upgrade', ]); -const backend = await startBackend({ basePath, directory }); +const signingKey = createSigningKey(); /** Request lines that no `run` or `stop` has taken yet. */ const requests: string[] = []; /** Problems on the backend's side, which make `stop` fail. */ @@ -107,6 +109,19 @@ const control = createServer((request, response) => { response.writeHead(200, { 'content-type': 'application/json' }); response.end(JSON.stringify(value)); }; + const url = new URL(request.url ?? '/', 'http://localhost'); + if (request.method === 'GET' && url.pathname === '/token') { + // The request token stands for the workflow run, as in GitHub Actions. + const requestToken = /^Bearer (.+)$/.exec(request.headers.authorization ?? '')?.[1]; + const audience = url.searchParams.get('audience'); + const value = + requestToken === undefined || audience === null + ? undefined + : signingKey.issue(requestToken, audience); + if (value === undefined) response.writeHead(401).end(); + else reply({ count: 1, value }); + return; + } const corrupt = /^\/corrupt-blob\/(\d+)$/.exec(request.url ?? '')?.[1]; if (request.method === 'POST' && request.url === '/take') { reply(requests.splice(0)); @@ -149,10 +164,21 @@ async function stop(): Promise { process.exit(0); } +const controlOrigin = `http://127.0.0.1:${await listen(control)}`; const info: ServerInfo = { url: `http://127.0.0.1:${await listen(tap)}${basePath}`, - control: `http://127.0.0.1:${await listen(control)}`, + control: controlOrigin, + tokenRequestUrl: `${controlOrigin}/token?api-version=2.0`, }; +// Upload tokens carry the endpoint as their audience, so the backend learns it +// before the first request. +const backend = await startBackend({ + basePath, + directory, + endpoint: info.url, + key: signingKey, + registered: repository, +}); writeFileSync(serverFile(directory), `${JSON.stringify(info, null, 2)}\n`); process.on('SIGTERM', () => void stop()); setInterval(() => { diff --git a/packages/tools/src/remote-cache/github.ts b/packages/tools/src/remote-cache/github.ts new file mode 100644 index 000000000..41d08307c --- /dev/null +++ b/packages/tools/src/remote-cache/github.ts @@ -0,0 +1,79 @@ +import { generateKeyPairSync, sign, type KeyObject, type webcrypto } from 'node:crypto'; + +// A stand-in for GitHub Actions' token service, which issues the OIDC tokens +// that authorize uploads. Its key signs tokens for GitHub's issuer, and the +// backend trusts that key in place of GitHub's. + +export const issuer = 'https://token.actions.githubusercontent.com'; + +/** The repository registered with the backend, whose pushes to `main` may upload. */ +export const repository = { + name: 'owner/repository', + id: '1', + ownerId: '1', + branch: 'refs/heads/main', +}; + +/** + * The workflow run that each request token stands for. A step chooses one with + * `ACTIONS_ID_TOKEN_REQUEST_TOKEN`; the default is a push to `main`. + */ +const runs: Record = { + 'main-push': { event_name: 'push', ref: repository.branch }, + 'pull-request': { event_name: 'pull_request', ref: 'refs/pull/1/merge' }, +}; + +export const defaultRequestToken = 'main-push'; + +/** The key that the backend trusts for upload tokens. */ +export interface TrustedKey { + /** The key ID in each token's header. */ + kid: string; + publicKey: KeyObject; + /** The public key as an entry of GitHub's key set. */ + jwk: webcrypto.JsonWebKey & { kid: string }; +} + +export interface SigningKey extends TrustedKey { + /** + * The token that GitHub would issue for `audience` to the run that + * `requestToken` stands for, or none for an unknown request token. + */ + issue(requestToken: string, audience: string): string | undefined; +} + +function encode(value: unknown): string { + return Buffer.from(JSON.stringify(value)).toString('base64url'); +} + +export function createSigningKey(): SigningKey { + const { publicKey, privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + const kid = 'test'; + return { + kid, + publicKey, + jwk: { ...publicKey.export({ format: 'jwk' }), kid, alg: 'RS256', use: 'sig' }, + issue(requestToken, audience) { + const run = runs[requestToken]; + if (run === undefined) return undefined; + const now = Math.floor(Date.now() / 1000); + const claims = { + iss: issuer, + aud: audience, + sub: `repo:${repository.name}:ref:${run.ref}`, + iat: now, + nbf: now, + exp: now + 300, + repository: repository.name, + repository_id: repository.id, + repository_owner_id: repository.ownerId, + repository_visibility: 'public', + ref: run.ref, + ref_type: 'branch', + event_name: run.event_name, + }; + const unsigned = `${encode({ alg: 'RS256', typ: 'JWT', kid })}.${encode(claims)}`; + return `${unsigned}.${sign('sha256', Buffer.from(unsigned), privateKey).toString('base64url')}`; + }, + }; +} diff --git a/packages/tools/src/remote-cache/state.ts b/packages/tools/src/remote-cache/state.ts index 1312aeda9..a4e1b67e2 100644 --- a/packages/tools/src/remote-cache/state.ts +++ b/packages/tools/src/remote-cache/state.ts @@ -12,6 +12,8 @@ export interface ServerInfo { url: string; /** The origin of the control server, e.g. `http://127.0.0.1:1235`. */ control: string; + /** Where to request GitHub Actions OIDC tokens, as `ACTIONS_ID_TOKEN_REQUEST_URL`. */ + tokenRequestUrl: string; } export function serverFile(directory: string): string {