format: stamp info, resources and program objects with their schema and specification version

.prettierignore

@@ -8,6 +8,7 @@ package-lock.json
 
 # Auto-generated files
 packages/format/src/schemas/yamls.ts
+packages/format/src/version.ts
 packages/bugc/src/examples/generated.ts
 
 # Solidity fixtures are compiler inputs; this repo has no Solidity Prettier parser.

CHANGELOG.md

@@ -53,6 +53,30 @@ The sections do not signal obligations; the prefixes do.
 
 ## Unreleased
 
+### Added
+
+- A stamp in an `ethdebug` field on **ethdebug/format/info**,
+  **ethdebug/format/info/resources** and **ethdebug/format/program** names
+  the schema the object conforms to and the specification version that
+  defines it. The new **ethdebug/format/data/stamp** schema defines the
+  stamp. Info documents and resources objects must carry the stamp
+  ([#305]).
+  - Schemas: **ethdebug/format/data/stamp**,
+    **ethdebug/format/info**, **ethdebug/format/info/resources**,
+    **ethdebug/format/program**
+  - Producers: required: **ethdebug/format/info** and
+    **ethdebug/format/info/resources** list `ethdebug` in `required`, so
+    an info document or resources object must carry
+    `ethdebug: { schema, version }`. A program emitted outside an info
+    document **should** carry the stamp; a program inside an info
+    document **should not**. All objects of one compilation must name the
+    same `version`.
+  - Consumers: required: a consumer that validates data against the
+    schemas must use this version's schemas. **ethdebug/format/program**
+    and **ethdebug/format/info** are closed objects
+    (`unevaluatedProperties: false`), so the previous version's schemas
+    reject the `ethdebug` key.
+
 ### Changed
 
 - The `offset` of a segment counts bytes from the most significant byte of the
@@ -632,4 +656,5 @@ First published version of the specification.
 [#285]: https://github.com/ethdebug/format/pull/285
 [#286]: https://github.com/ethdebug/format/pull/286
 [#303]: https://github.com/ethdebug/format/pull/303
+[#305]: https://github.com/ethdebug/format/pull/305
 [#309]: https://github.com/ethdebug/format/pull/309

RELEASING.md

@@ -130,6 +130,21 @@ guards that run in CI live in `bin/check-tarballs.ts` and
    yarn tsx bin/version.ts [keyword] [--all]
    ```
 
+   When `@ethdebug/format` moves, the `Publish` commit also rewrites
+   the specification version in the schema examples, so `schemas/` may
+   appear in that commit beside the manifests; the dry run prints the
+   count of version literals it rewrites.
+
+   A `schemas/: no example names the specification version` problem
+   means every `ethdebug` block was removed from the examples; a
+   `names X, expected Y` problem means one drifted from the version
+   `@ethdebug/format` carries. Edit the example and re-run.
+
+   After the bump, run `yarn build` before running
+   `yarn test packages/format` again: the generated
+   `src/version.ts` still names the old version until the build
+   regenerates it. CI does this step in the publish workflow.
+
    The script never pushes. If it fails after it started writing, it
    prints the undo commands for the stage it reached. The dry run of
    step 3 reports the same guards and findings as this run, but it

bin/release/schema-versions.test.ts

@@ -0,0 +1,97 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+import { describe, expect, it } from "vitest";
+
+import {
+  checkVersions,
+  readSchemas,
+  setVersions,
+  versionSites,
+} from "./schema-versions.js";
+
+const root = fileURLToPath(new URL("../..", import.meta.url));
+const read = (path: string) => readFileSync(join(root, path), "utf8");
+
+const info = read("schemas/info.schema.yaml");
+const program = read("schemas/program.schema.yaml");
+const stamp = read("schemas/data/stamp.schema.yaml");
+const hex = read("schemas/data/hex.schema.yaml");
+
+const lineAt = (text: string, index: number) =>
+  text.slice(0, index).split("\n").length;
+
+describe("versionSites", () => {
+  it("finds the nested ethdebug block and ignores compiler versions", () => {
+    expect(info).toMatch(/^\s+version: 0\.2\.3/m);
+    const sites = versionSites(info);
+    expect(sites).toHaveLength(1);
+    expect(sites[0].path).toMatch(/^examples\/\d+\/ethdebug$/);
+  });
+
+  it("finds an example that is itself a stamp", () => {
+    expect(versionSites(stamp).map((s) => s.path)).toEqual(["examples/0"]);
+  });
+
+  it("ignores the ethdebug property that declares the field", () => {
+    expect(program).toMatch(/^properties:\n(.*\n)*? {2}ethdebug:/m);
+    expect(versionSites(program).map((s) => s.path)).toEqual([
+      "examples/0/ethdebug",
+    ]);
+  });
+
+  it("reports the line and the range of the quoted scalar", () => {
+    const [site] = versionSites(program);
+    const [start, end] = site.range;
+    expect(program.slice(start, end)).toBe(`"${site.version}"`);
+    expect(site.line).toBe(lineAt(program, start));
+  });
+});
+
+describe("setVersions", () => {
+  it("sets the site and leaves everything else byte for byte", () => {
+    const [start, end] = versionSites(program)[0].range;
+    const after = setVersions(program, "0.1.0-draft.1");
+    expect(after.slice(0, start)).toBe(program.slice(0, start));
+    expect(after.slice(start, after.length - (program.length - end))).toBe(
+      '"0.1.0-draft.1"',
+    );
+    expect(after.endsWith(program.slice(end))).toBe(true);
+  });
+
+  it("returns the text unchanged when there is no site", () => {
+    expect(versionSites(hex)).toEqual([]);
+    expect(setVersions(hex, "0.2.0")).toBe(hex);
+  });
+});
+
+describe("checkVersions", () => {
+  const path = "schemas/program.schema.yaml";
+  const sites = versionSites(program);
+  const file = { path, text: program, sites };
+
+  it("names the file, the line and both versions when one differs", () => {
+    const [problem] = checkVersions(file, "0.9.9");
+    expect(problem).toContain(`${path}:${sites[0].line}`);
+    expect(problem).toContain(sites[0].version);
+    expect(problem).toContain("0.9.9");
+  });
+});
+
+describe("the schemas in this repository", () => {
+  const packageVersion = JSON.parse(read("packages/format/package.json"))
+    .version as string;
+
+  it("carry at least one version site", () => {
+    const sites = readSchemas(root).flatMap((file) => file.sites);
+    expect(sites.length).toBeGreaterThan(0);
+  });
+
+  it("name the version @ethdebug/format carries", () => {
+    const problems = readSchemas(root).flatMap((file) =>
+      checkVersions(file, packageVersion),
+    );
+    expect(problems).toEqual([]);
+  });
+});

bin/release/schema-versions.ts

@@ -0,0 +1,116 @@
+import { readdirSync, readFileSync } from "node:fs";
+import { join, relative } from "node:path";
+
+import { isMap, isScalar, isSeq, parseDocument } from "yaml";
+
+// a `version` scalar that a release rewrites: the version field of a
+// stamp sitting inside a schema's top-level `examples`
+export interface Site {
+  // a path like "examples/0/ethdebug", for messages
+  path: string;
+  line: number;
+  version: string;
+  // byte range of the scalar token, quotes included
+  range: [number, number];
+}
+
+export interface SchemaFile {
+  // relative to the repository root
+  path: string;
+  text: string;
+  sites: Site[];
+}
+
+const schemaNamePrefix = "ethdebug/format/";
+
+// a stamp is any mapping with a `schema` value that starts with
+// `ethdebug/format/` and a `version` beside it.
+function collect(node: unknown, path: string, text: string, out: Site[]) {
+  if (isMap(node)) {
+    const schema = node.get("schema");
+    const version = node.get("version", true);
+    if (
+      typeof schema === "string" &&
+      schema.startsWith(schemaNamePrefix) &&
+      isScalar(version) &&
+      typeof version.value === "string" &&
+      version.range
+    ) {
+      out.push({
+        path,
+        line: text.slice(0, version.range[0]).split("\n").length,
+        version: version.value,
+        range: [version.range[0], version.range[1]],
+      });
+    }
+    for (const item of node.items) {
+      const key = String(
+        (isScalar(item.key) ? item.key.value : undefined) ?? "?",
+      );
+      collect(item.value, `${path}/${key}`, text, out);
+    }
+  } else if (isSeq(node)) {
+    node.items.forEach((item, index) =>
+      collect(item, `${path}/${index}`, text, out),
+    );
+  }
+}
+
+export function versionSites(text: string): Site[] {
+  const examples = parseDocument(text).get("examples", true);
+  const out: Site[] = [];
+  if (examples) {
+    collect(examples, "examples", text, out);
+  }
+  return out.sort((a, b) => a.range[0] - b.range[0]);
+}
+
+// splices each site into the original bytes, so comments, spacing and
+// the long block descriptions survive exactly.
+export function setVersions(text: string, to: string): string {
+  let result = "";
+  let cursor = 0;
+  for (const site of versionSites(text)) {
+    const [start, end] = site.range;
+    const original = text.slice(start, end);
+    const quote = original.startsWith('"')
+      ? '"'
+      : original.startsWith("'")
+        ? "'"
+        : "";
+    result += text.slice(cursor, start) + `${quote}${to}${quote}`;
+    cursor = end;
+  }
+  return result + text.slice(cursor);
+}
+
+// every site must already name the version the package carries; one
+// that does not means a schema example drifted from the release
+export function checkVersions(file: SchemaFile, expected: string): string[] {
+  return file.sites
+    .filter((site) => site.version !== expected)
+    .map(
+      (site) =>
+        `${file.path}:${site.line}: ${site.path} names ` +
+        `${site.version}, expected ${expected}`,
+    );
+}
+
+function schemaPaths(dir: string): string[] {
+  return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
+    const path = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      return schemaPaths(path);
+    }
+    return entry.name.endsWith(".schema.yaml") ? [path] : [];
+  });
+}
+
+export function readSchemas(root: string): SchemaFile[] {
+  return schemaPaths(join(root, "schemas"))
+    .sort()
+    .map((path) => {
+      const text = readFileSync(path, "utf8");
+      return { path: relative(root, path), text, sites: versionSites(text) };
+    });
+}

bin/version.test.ts

@@ -608,7 +608,7 @@ describe("undoAdvice", () => {
 
   it("restores the manifests when nothing was committed or tagged", () => {
     expect(undoAdvice([], false)).toBe(
-      "undo: git checkout HEAD -- packages/*/package.json",
+      "undo: git checkout HEAD -- packages/*/package.json schemas/",
     );
   });
 });

bin/version.ts

@@ -5,6 +5,12 @@ import { fileURLToPath, pathToFileURL } from "node:url";
 
 import semver from "semver";
 
+import {
+  checkVersions,
+  readSchemas,
+  setVersions,
+} from "./release/schema-versions.js";
+
 // the schemas ship inside this package, so its version is the version
 // of the specification
 export const specPackage = "@ethdebug/format";
@@ -636,7 +642,7 @@ export function undoAdvice(created: string[], committed: boolean): string {
   if (tags.length > 0) {
     return `undo: ${tags}`;
   }
-  return "undo: git checkout HEAD -- packages/*/package.json";
+  return "undo: git checkout HEAD -- packages/*/package.json schemas/";
 }
 
 function report(plan: Move[]): void {
@@ -681,6 +687,32 @@ export function main(argv: string[]): number {
     all,
   });
   const problems = planProblems(plan, manifests, keyword);
+  // the schemas ship inside @ethdebug/format, so their examples name
+  // the version it moves to; when it stays put they are left alone
+  const specMove = plan.find((move) => move.name === specPackage);
+  let schemas: { path: string; text: string }[] = [];
+  let siteCount = 0;
+  if (specMove !== undefined) {
+    const files = readSchemas(root);
+    siteCount = files.flatMap((file) => file.sites).length;
+    if (siteCount === 0) {
+      problems.push(
+        "schemas/: no example names the specification version; " +
+          "the release would rewrite nothing",
+      );
+    }
+    problems.push(
+      ...files.flatMap((file) => checkVersions(file, specMove.from)),
+    );
+    schemas = files
+      .map((file) => ({
+        path: file.path,
+        text: setVersions(file.text, specMove.to),
+        before: file.text,
+      }))
+      .filter((file) => file.text !== file.before)
+      .map(({ path, text }) => ({ path, text }));
+  }
   if (problems.length > 0) {
     for (const problem of problems) {
       console.error(problem);
@@ -700,6 +732,10 @@ export function main(argv: string[]): number {
   }
   console.log(`${keyword}: ${plan.length} workspace(s) move`);
   report(plan);
+  if (specMove !== undefined) {
+    const literals = `${siteCount} version literals`;
+    console.log(`  schemas: ${literals} -> ${specMove.to}`);
+  }
 
   const changelogs = changelogProblems(
     requiredChangelogs(plan, manifests, root).map(({ path, version }) => ({
@@ -729,9 +765,15 @@ export function main(argv: string[]): number {
   // every release
   const headBefore = git(root, ["rev-parse", "HEAD"]);
   let written: string[] = [];
+  let schemaCount = 0;
   const created: string[] = [];
   try {
     written = writeManifests(root, manifests, plan);
+    for (const { path, text } of schemas) {
+      writeFileSync(join(root, path), text);
+      written.push(path);
+    }
+    schemaCount = schemas.length;
     commitAndTag(root, written, plan, created);
   } catch (error) {
     const message = error instanceof Error ? error.message : String(error);
@@ -742,7 +784,12 @@ export function main(argv: string[]): number {
   }
   console.log(`tagged: ${created.join(", ")}`);
   if (written.length > 0) {
-    console.log(`committed Publish with ${written.length} manifest(s)`);
+    const schemaNote =
+      schemaCount > 0 ? ` and ${schemaCount} schema file(s)` : "";
+    console.log(
+      `committed Publish with ${written.length - schemaCount} ` +
+        `manifest(s)${schemaNote}`,
+    );
     console.log("next: git push --atomic origin main --follow-tags");
     return 0;
   }

eslint.config.js

@@ -72,6 +72,7 @@ export default tseslint.config(
       "**/*.config.js",
       "**/*.config.ts",
       "packages/format/src/schemas/yamls.ts",
+      "packages/format/src/version.ts",
       "packages/bugc/src/examples/generated.ts",
       "packages/web/.docusaurus/",
       "packages/web/build/",