test(cache): number remote cache blobs and corrupt them through the backend

crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots.toml

@@ -306,10 +306,9 @@ steps = [
     ],
   ] },
   { argv = [
-    "vtt",
-    "write-file",
-    "remote-cache/blobs/1",
-    "corrupt",
+    "remote-cache-server",
+    "corrupt-blob",
+    "1",
   ], comment = "Overwrite the stored archive." },
   [
     "vt",

crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/corrupt_archive.md

@@ -14,7 +14,7 @@ $ vtt write-file dist/output.txt built
 [remote-cache] POST /store 200
 ```
 
-## `vtt write-file remote-cache/blobs/1 corrupt`
+## `remote-cache-server corrupt-blob 1`
 
 Overwrite the stored archive.
 

packages/tools/README.md

@@ -14,11 +14,12 @@ An E2E case starts its own backend in its first step and stops it in its last, s
 
 - `remote-cache-server start` starts the backend in the background on free loopback ports and returns once it's ready. The backend runs in its own session and doesn't use the terminal, so the step can finish and Ctrl-C in later steps doesn't reach it. It writes its endpoint, `http://127.0.0.1:<port>/projects/test`, to `remote-cache/server.json`, and its output to `remote-cache/server.log`.
 - `remote-cache-server run COMMAND [ARGS...]` runs the command with `VP_REMOTE_CACHE_URL` set to the endpoint. The fixed base path gives the endpoint a namespace path. The command inherits stdio and handles Ctrl-C, which `run` ignores. `run` exits with the command's exit code.
+- `remote-cache-server corrupt-blob NUMBER` overwrites a stored blob, numbered as in the request lines, with other bytes.
 - `remote-cache-server stop` stops the backend. It fails if the backend answered a request with a 5xx status or couldn't be reached.
 
 A backend whose case directory disappears stops by itself, as does one that has had no requests for ten minutes, e.g. because its case timed out before the stop step.
 
-The endpoint is a tap in front of the backend. It forwards every request and response unchanged and records a line for each response, in the order of the responses. Each line has the method, the path below the base path, and the status. Successful fetch responses add their kind. After the command exits, `run` prints the lines for the requests it caused to stderr, and `stop` prints any that are left:
+The endpoint is a tap in front of the backend. It forwards every request and response unchanged and records a line for each response, in the order of the responses. Each line has the method, the path below the base path, and the status. Successful fetch responses add their kind. Blob IDs are random, so blob paths show the blob's number in upload order instead, keeping snapshots deterministic. After the command exits, `run` prints the lines for the requests it caused to stderr, and `stop` prints any that are left:
 
 ```text
 [remote-cache] POST /fetch 404
@@ -27,7 +28,7 @@ The endpoint is a tap in front of the backend. It forwards every request and res
 [remote-cache] GET /blob/1 200
 ```
 
-The backend keeps its state in `remote-cache/`. `state.json` holds the entries, associations, and next blob ID, with keys and values hex-encoded. Each blob is a file in `remote-cache/blobs/` named by its blob ID. Blob IDs are sequential strings, keeping snapshots deterministic.
+The backend keeps its state in `remote-cache/`. `state.json` holds the entries and associations, with keys and values hex-encoded. Each blob is a file in `remote-cache/blobs/` named by its blob ID, a random UUID.
 
 The backend implements `POST /fetch`, `POST /store`, and `GET /blob/{blob_id}` from the [remote cache server API](https://github.com/voidzero-dev/vite-task/pull/713). A fetch that matches neither key gets a `404` with the plain-text body `Not found`. Keys, values, and blobs are opaque bytes without length limits. There is no authentication.
 

packages/tools/src/remote-cache/backend.ts

@@ -1,6 +1,7 @@
 import { Busboy } from '@fastify/busboy';
 import { decode } from 'cbor2/decoder';
 import { encode } from 'cbor2/encoder';
+import { randomUUID } from 'node:crypto';
 import { once } from 'node:events';
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
 import { createServer, type IncomingMessage, type ServerResponse } from 'node:http';
@@ -14,7 +15,6 @@ interface Entry {
 
 /** The contents of `state.json`. Keys and values are hex-encoded. */
 interface State {
-  next_blob_id: number;
   entries: Record<string, Entry>;
   associations: Record<string, string>;
 }
@@ -100,14 +100,16 @@ function cbor(response: ServerResponse, value: unknown): void {
 export interface Backend {
   /** Where the backend listens, e.g. `http://127.0.0.1:1234`, without a path. */
   origin: string;
+  /** Replace the contents of the blob with ID `id`. */
+  writeBlob(id: string, contents: Uint8Array): Promise<void>;
   close(): Promise<void>;
 }
 
 /**
  * Start a test backend on a free loopback port that keeps its state in
  * `directory`: entries and associations in `state.json`, and each blob in
- * `blobs/` under its ID. Keys, values, and blobs remain opaque bytes. A fetch
- * that matches neither key gets a plain-text 404.
+ * `blobs/` under its ID, a random UUID. Keys, values, and blobs remain opaque
+ * bytes. A fetch that matches neither key gets a plain-text 404.
  */
 export async function startBackend({
   basePath,
@@ -120,10 +122,9 @@ export async function startBackend({
   const blobDirectory = join(directory, 'blobs');
   const state: State = existsSync(stateFile)
     ? JSON.parse(readFileSync(stateFile, 'utf8'))
-    : { next_blob_id: 1, entries: {}, associations: {} };
+    : { entries: {}, associations: {} };
   const entries = new Map(Object.entries(state.entries));
   const associations = new Map(Object.entries(state.associations));
-  let nextBlobId = state.next_blob_id;
 
   async function handle(
     request: IncomingMessage,
@@ -182,12 +183,11 @@ export async function startBackend({
     const key = toHex(fields.get('key')!);
     const blob = parts.get('blob');
     mkdirSync(blobDirectory, { recursive: true });
-    const blobId = blob === undefined ? null : String(nextBlobId++);
+    const blobId = blob === undefined ? null : randomUUID();
     if (blobId !== null) writeFileSync(join(blobDirectory, blobId), blob!);
     entries.set(key, { value: toHex(fields.get('value')!), blob_id: blobId });
     associations.set(toHex(fields.get('secondary_key')!), key);
     const saved: State = {
-      next_blob_id: nextBlobId,
       entries: Object.fromEntries(entries),
       associations: Object.fromEntries(associations),
     };
@@ -212,6 +212,7 @@ export async function startBackend({
   const { port } = server.address() as AddressInfo;
   return {
     origin: `http://127.0.0.1:${port}`,
+    writeBlob: async (id, contents) => writeFileSync(join(blobDirectory, id), contents),
     close: () =>
       new Promise((resolve) => {
         server.close(() => resolve());

packages/tools/src/remote-cache/cli.ts

@@ -8,7 +8,7 @@ import { resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { logFile, serverFile, stateDirectory, type ServerInfo } from './state.ts';
 
-const usage = 'Usage: remote-cache-server start | run COMMAND [ARGS...] | stop';
+const usage = 'Usage: remote-cache-server start | run COMMAND [ARGS...] | corrupt-blob NUMBER | stop';
 const directory = resolve(stateDirectory);
 
 function fail(message: string): never {
@@ -23,12 +23,19 @@ function readInfo(): ServerInfo {
   return JSON.parse(readFileSync(serverFile(directory), 'utf8')) as ServerInfo;
 }
 
-/** POST to the backend's control server and return the JSON response. */
-function control<T>(info: ServerInfo, path: string): Promise<T> {
+/**
+ * POST to the backend's control server and return the JSON response. A 404
+ * fails with `notFound`.
+ */
+function control<T>(info: ServerInfo, path: string, notFound = 'unknown request'): Promise<T> {
   return new Promise((resolve) => {
     const unreachable = (error: Error) =>
       fail(`the backend is unreachable (${error.message}). See ${logFile(stateDirectory)}.`);
     const call = request(`${info.control}${path}`, { method: 'POST', agent: false }, (response) => {
+      if (response.statusCode === 404) fail(notFound);
+      if (response.statusCode !== 200) {
+        fail(`the backend failed (${response.statusCode}). See ${logFile(stateDirectory)}.`);
+      }
       const chunks: Buffer[] = [];
       response.on('data', (chunk: Buffer) => chunks.push(chunk));
       response.on('end', () => resolve(JSON.parse(Buffer.concat(chunks).toString()) as T));
@@ -79,6 +86,11 @@ async function run([command, ...args]: string[]): Promise<void> {
   process.exitCode = code;
 }
 
+async function corruptBlob([number]: string[]): Promise<void> {
+  if (number === undefined || !/^[1-9]\d*$/.test(number)) fail(usage);
+  await control(readInfo(), `/corrupt-blob/${number}`, `no blob ${number} has been stored.`);
+}
+
 async function stop(): Promise<void> {
   const { requests, anomalies } = await control<{ requests: string[]; anomalies: string[] }>(
     readInfo(),
@@ -92,5 +104,6 @@ async function stop(): Promise<void> {
 const [subcommand, ...args] = process.argv.slice(2);
 if (subcommand === 'start') await start();
 else if (subcommand === 'run') await run(args);
+else if (subcommand === 'corrupt-blob') await corruptBlob(args);
 else if (subcommand === 'stop') await stop();
 else fail(usage);

packages/tools/src/remote-cache/daemon.ts

@@ -13,8 +13,10 @@ import { basePath, serverFile, type ServerInfo } from './state.ts';
 
 // The process that `remote-cache-server start` leaves running for one e2e case.
 // It serves the backend at the endpoint through a tap that forwards requests
-// and responses unchanged and records a line for each response. A control
-// server hands those lines to `remote-cache-server run` and `stop`.
+// and responses unchanged and records a line for each response. Blob IDs in
+// the lines become numbers in upload order, so they're the same on every run.
+// A control server hands the lines to `remote-cache-server run` and `stop`,
+// and changes blobs for `remote-cache-server corrupt-blob`.
 
 const directory = process.argv[2]!;
 /** Stop after this long without requests, e.g. when a case timed out before its stop step. */
@@ -27,31 +29,38 @@ const backend = await startBackend({ basePath, directory });
 const requests: string[] = [];
 /** Problems on the backend's side, which make `stop` fail. */
 const anomalies: string[] = [];
+/** The ID of each stored blob, in upload order. Blob number `n` is at `n - 1`. */
+const blobs: string[] = [];
 let lastUse = Date.now();
 let stopping = false;
 
 function endToEnd(headers: IncomingHttpHeaders): IncomingHttpHeaders {
   return Object.fromEntries(Object.entries(headers).filter(([name]) => !hopByHop.has(name)));
 }
 
-/** The kind of a successful fetch response. */
-function fetchKind(body: Buffer): string | undefined {
+/** The fields of a CBOR map. */
+function cborFields(body: Buffer): Record<string, unknown> {
   try {
     const value: unknown = decode(body);
-    if (typeof value === 'object' && value !== null && 'kind' in value) {
-      return typeof value.kind === 'string' ? value.kind : undefined;
-    }
+    if (typeof value === 'object' && value !== null) return value as Record<string, unknown>;
   } catch {
     // Not CBOR. vp reports the malformed response itself.
   }
-  return undefined;
+  return {};
+}
+
+/** `route` with a stored blob's ID replaced by its number. */
+function numbered(route: string): string {
+  const id = /^\/blob\/(.+)$/.exec(route)?.[1];
+  const index = id === undefined ? -1 : blobs.indexOf(id);
+  return index === -1 ? route : `/blob/${index + 1}`;
 }
 
 const tap = createServer((request, response) => {
   lastUse = Date.now();
   const path = new URL(request.url ?? '/', 'http://localhost').pathname;
   const route = path.startsWith(basePath) ? path.slice(basePath.length) : path;
-  const label = `${request.method} ${route}`;
+  const label = `${request.method} ${numbered(route)}`;
   let clientGone = false;
   const upstream = forward(
     `${backend.origin}${request.url}`,
@@ -60,13 +69,17 @@ const tap = createServer((request, response) => {
       const status = reply.statusCode!;
       if (status >= 500) anomalies.push(`${label} got ${status} from the backend`);
       response.writeHead(status, endToEnd(reply.headers));
+      // Fetch and store responses are small CBOR maps with the kind or blob ID.
       const chunks: Buffer[] = [];
-      if (route === '/fetch' && status === 200) {
+      if ((route === '/fetch' || route === '/store') && status === 200) {
         reply.on('data', (chunk: Buffer) => chunks.push(chunk));
       }
       reply.pipe(response);
       response.on('finish', () => {
-        const kind = chunks.length > 0 ? fetchKind(Buffer.concat(chunks)) : undefined;
+        const fields = chunks.length > 0 ? cborFields(Buffer.concat(chunks)) : {};
+        const blobId = fields['blob_id'];
+        if (route === '/store' && typeof blobId === 'string') blobs.push(blobId);
+        const kind = route === '/fetch' ? fields['kind'] : undefined;
         requests.push([label, status, kind].filter((part) => part !== undefined).join(' '));
       });
     },
@@ -93,8 +106,22 @@ const control = createServer((request, response) => {
     response.writeHead(200, { 'content-type': 'application/json' });
     response.end(JSON.stringify(value));
   };
+  const corrupt = /^\/corrupt-blob\/(\d+)$/.exec(request.url ?? '')?.[1];
   if (request.method === 'POST' && request.url === '/take') {
     reply(requests.splice(0));
+  } else if (request.method === 'POST' && corrupt !== undefined) {
+    const id = blobs[Number(corrupt) - 1];
+    if (id === undefined) {
+      response.writeHead(404).end();
+      return;
+    }
+    backend.writeBlob(id, Buffer.from('corrupt')).then(
+      () => reply(null),
+      (error: unknown) => {
+        console.error(error);
+        response.writeHead(500).end();
+      },
+    );
   } else if (request.method === 'POST' && request.url === '/stop') {
     response.on('finish', () => void stop());
     reply({ requests: requests.splice(0), anomalies });