From 3d987853646bbbb46e4586b2733674bfd0542c6b Mon Sep 17 00:00:00 2001 From: Titus Date: Sat, 17 Oct 2026 01:56:13 +0200 Subject: [PATCH 1/5] onlyworlds.okf: the OKF v0.2 exporter, ported from schema-dist okf-exporter 8d0f92f (Keel #191) export_okf(folder, out_dir) and `python -m onlyworlds.okf FOLDER OUT-DIR`. Same behaviour as walk/okf_export.py, every review fix carried (every splitlines() break quoted, unreadable and dot-prefixed files noted and skipped, world.json time_range_current fallback and x_*/snapshot_* carry, api dropped, undeclared keys and unknown kinds noted, wrong-type and null links noted and left out, heading lines escaped, generated.by explained). On the example world the bundle is byte-identical to the branch's run on the same pinned dist (dist.23). The four conditions: dependencies = [] stays and PyYAML is the new `okf` extra (import onlyworlds loads no yaml; a test asserts it). The folder is read only through read_folder: skipped files, ignored dirs, duplicates and in_both_layouts become notes or the duplicate-id error. The schema comes from _schema.py: the generator now emits BASE_FIELDS, WORLD_FIELDS, SECTIONS, LINK_TARGETS, GENERIC_LINKS and EXTENSION_PREFIXES from the vendored walk (decode_schema) and lint_folder, cross-checked against FIELD_KINDS and GENERIC_PAIRS; known keys are push.NOT_SENT. The 24 tests ported to tests/test_okf.py on the vendored Hyperion, plus two for the conditions; the six mutants each fail at least one test. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 + README.md | 22 +- codegen/generate_schema.py | 130 +++++++- pyproject.toml | 5 +- src/onlyworlds/_schema.py | 416 ++++++++++++++++++++++++- src/onlyworlds/okf.py | 616 +++++++++++++++++++++++++++++++++++++ tests/test_okf.py | 550 +++++++++++++++++++++++++++++++++ uv.lock | 7 + 8 files changed, 1747 insertions(+), 5 deletions(-) create mode 100644 src/onlyworlds/okf.py create mode 100644 tests/test_okf.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 696f50c..85e3bbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ The package version runs on its own semver; the schema it is generated from is `SCHEMA_DIST_TAG`. +## 0.6.0 (unreleased) + +- **Export a world folder to Open Knowledge Format (OKF) v0.2** (`onlyworlds.okf`, Keel #191): `export_okf(folder, out_dir)` or `python -m onlyworlds.okf FOLDER OUT-DIR` writes one markdown concept per element (fields in the frontmatter, text under headings, links as bundle paths and as markdown links), the World as `world.md`, and an index per type. The same folder gives the same bytes. Nothing is left out in silence: a file the reader skipped, a key that is not a field of its type, or a link that is not an id or points at the wrong type is a note in the returned `OkfReport` (stderr from the command line). Ported from schema-dist's `walk/okf_export.py`, it reads through `read_folder` and the generated schema tables, and gives the same bundle on the example world. +- **PyYAML only for OKF**: the export needs the new `okf` extra (`pip install "onlyworlds[okf]"`). `import onlyworlds` still imports nothing outside the standard library. +- **The schema module grows** (same pin, `v0.32.0-dist.23`): `BASE_FIELDS`, `WORLD_FIELDS`, `SECTIONS` (each type's fields under their schema groups, in order), `LINK_TARGETS`, `GENERIC_LINKS` and `EXTENSION_PREFIXES`, generated from the vendored walk like the rest and checked against `FIELD_KINDS` and `GENERIC_PAIRS` at generation. + ## 0.5.1 (2026-10-17) - **The reader skips a dot-prefixed file in a type directory**, a tool's litter rather than an element, and lists it in `Folder.skipped`, even when it holds a valid body. Before, it was read like any file, and one that sorted first could stand in for the real element (Skeld's clarification of the folder spec, 2026-10-17). diff --git a/README.md b/README.md index 928c672..fe1ea59 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # onlyworlds (Python) -OnlyWorlds is an open standard for worldbuilding data: 22 element types (characters, locations, events, laws and the rest) that any tool can read and write. This is a Python client for its API, and a reader and writer for OnlyWorlds world folders (a world as a folder of JSON files). Python 3.12+, no runtime dependencies. +OnlyWorlds is an open standard for worldbuilding data: 22 element types (characters, locations, events, laws and the rest) that any tool can read and write. This is a Python client for its API, and a reader and writer for OnlyWorlds world folders (a world as a folder of JSON files). Python 3.12+, no runtime dependencies (the optional OKF export needs PyYAML). ## Install @@ -71,6 +71,24 @@ result = push("moppetopia-before", "moppetopia", api_key="ow_w_…", pin="ow_s_ `push` sends only the fields that changed, retries what the server asks it to retry, checks that what came back is what it sent, and logs every write, so running it again skips what already landed. `dry_run=True` shows the plan without sending it. +## Export to OKF + +A world folder can be written as a bundle in [Open Knowledge Format](https://github.com/GoogleCloudPlatform/open-knowledge-format) (OKF) v0.2: one markdown file per element, with its fields in the frontmatter, its text under headings and its links as markdown links, plus an index per type. This needs PyYAML, which the rest of the package does not: + +```bash +pip install "onlyworlds[okf]" +python -m onlyworlds.okf moppetopia moppetopia-okf +``` + +```python +from onlyworlds.okf import export_okf + +report = export_okf("moppetopia", "moppetopia-okf") +print(report.elements, "elements,", len(report.notes), "notes") +``` + +The output directory must not exist yet. Anything the export leaves out (a file that isn't an element, a key that isn't a field, a link to the wrong type) is listed in `report.notes`; the command prints these on stderr. The same folder always gives the same bytes. + ## Errors A failed call raises `ApiError` with the API's `code`, `param` (the field that failed) and `doc_url`, and flags for the cases you handle differently: `is_validation_error`, `is_auth_error`, `is_not_author`, `is_owner_only`, `is_id_conflict`, `is_resync_required`, `is_busy`, `is_precondition_failed`. @@ -107,6 +125,8 @@ Three groups of tests run only when asked: - `OW_OPENAPI=https://www.onlyworlds.com/api/v2/openapi.json` (or a saved copy): checks the client's routes, parameters and fields against the API's own OpenAPI document. - `OW_LIVE=1`: read-only calls against the demo world. +`OKF_REFERENCE_SRC=` (the `src/` of an open-knowledge-format checkout) also runs OKF's own parser and viewer over an exported bundle. + ## License diff --git a/codegen/generate_schema.py b/codegen/generate_schema.py index 5dd554b..2b15c80 100644 --- a/codegen/generate_schema.py +++ b/codegen/generate_schema.py @@ -126,6 +126,70 @@ def generic_pairs(walk: Any, types: list[str]) -> dict[str, list[tuple[str, str] return out +def decoded_schema(walk: Any) -> dict[str, Any]: + """The walk's decode_schema (the shape schema.json ships), with the same notes policy as field_kinds.""" + notes: list[str] = [] + decoded: dict[str, Any] = walk.decode_schema(DIST / "schema", note=notes.append) + unexpected = [n for n in notes if not n.startswith(EXPECTED_NOTES)] + if unexpected: + raise SystemExit(f"the walk raised notes this generator does not know: {unexpected}") + return decoded + + +def layout_tables( + decoded: dict[str, Any], + types: list[str], + kinds: dict[str, dict[str, str]], + pairs: dict[str, list[tuple[str, str]]], +) -> dict[str, Any]: + """What a reader of whole elements needs beyond FIELD_KINDS: the base and World field names in + order, each type's fields under their YAML groups, each link's target, each generic link's two + keys. Cross-checked against FIELD_KINDS and GENERIC_PAIRS, so the tables cannot drift apart.""" + sections: dict[str, list[tuple[str, list[str]]]] = {} + targets: dict[str, dict[str, str]] = {} + generic: dict[str, dict[str, tuple[str, str]]] = {} + for t in types: + groups: list[tuple[str, list[str]]] = [] + for f in decoded["types"][t]: + if not groups or groups[-1][0] != f["section"]: + groups.append((f["section"], [])) + groups[-1][1].append(f["name"]) + if f["kind"] in ("single", "multi"): + if not f.get("target"): + raise SystemExit(f"{t}.{f['name']}: a link with no target from the walk") + targets.setdefault(t, {})[f["name"]] = f["target"] + elif f["kind"] == "generic": + generic.setdefault(t, {})[f["name"]] = (f["type_key"], f["id_key"]) + if f["kind"] != "generic" and f["name"] not in kinds[t]: + raise SystemExit(f"{t}.{f['name']}: in the walk's sections but not in FIELD_KINDS") + if len({s for s, _ in groups}) != len(groups): + raise SystemExit(f"{t}: a section split in two by the walk's order") + sections[t] = groups + if sorted(generic.get(t, {}).values()) != sorted(pairs.get(t, [])): + raise SystemExit( + f"{t}: the walk's generic keys {generic.get(t)} disagree with GENERIC_PAIRS {pairs.get(t)}" + ) + return { + "base": [f["name"] for f in decoded["base"]], + "world": [f["name"] for f in decoded["world"]], + "sections": sections, + "targets": targets, + "generic": generic, + } + + +def extension_prefixes() -> tuple[str, ...]: + """The extension prefixes as the dist's own folder lint declares them (ruling extension-passthrough).""" + spec = importlib.util.spec_from_file_location("lint_folder", DIST / "walk" / "lint_folder.py") + assert spec is not None and spec.loader is not None + lint = importlib.util.module_from_spec(spec) + spec.loader.exec_module(lint) + prefixes = tuple(lint.EXTENSION_PREFIXES) + if not prefixes or not all(isinstance(p, str) and p.endswith("_") for p in prefixes): + raise SystemExit(f"lint_folder.EXTENSION_PREFIXES is not a tuple of prefixes: {prefixes!r}") + return prefixes + + def element_types(walk: Any) -> list[str]: from_walk: list[str] = list(walk.ELEMENT_TYPES) from_files = sorted(p.stem for p in (DIST / "schema").glob("*.yaml") if p.stem not in NON_ELEMENT_SCHEMA_FILES) @@ -139,13 +203,31 @@ def render( types: list[str], kinds: dict[str, dict[str, str]], pairs: dict[str, list[tuple[str, str]]], + layout: dict[str, Any], + prefixes: tuple[str, ...], ) -> str: + def strs(names: list[str]) -> str: + return "(" + ", ".join(f'"{n}"' for n in names) + ("," if len(names) == 1 else "") + ")" + + def wrapped(head: str, names: list[str], tail: str) -> list[str]: + """``head(names)tail`` on one line, or the names packed onto lines within ruff's 120.""" + if len(head + strs(names) + tail) <= 120: + return [head + strs(names) + tail] + indent = " " * (len(head) - len(head.lstrip())) + out, cur = [head + "("], "" + for piece in (f'"{n}",' for n in names): + if cur and len(indent) + 4 + len(cur) + 1 + len(piece) > 120: + out.append(indent + " " + cur) + cur = "" + cur = f"{cur} {piece}" if cur else piece + return [*out, indent + " " + cur, indent + ")" + tail] + lines = [ "# GENERATED by codegen/generate_schema.py from the vendored schema distribution. Do not hand-edit.", "# Regenerate: python codegen/generate_schema.py (CI/test: --check)", f"# Source: {pin['repo']} @ {pin['tag']} ({pin['commit']})", "", - '"""Schema provenance and the 22 element types, generated from the pinned schema distribution."""', + '"""Schema provenance, the 22 element types and their fields, from the pinned schema distribution."""', "", f'SCHEMA_DIST_TAG = "{pin["tag"]}"', f'SCHEMA_VERSION = "{pin["canonical_version"]}"', @@ -173,6 +255,48 @@ def render( *[f' "{t}": ({", ".join(f'("{a}", "{b}")' for a, b in pairs[t])},),' for t in types if t in pairs], "}", "", + "# Every element's shared fields (base_properties.yaml), in schema order, as data spells them.", + *wrapped("BASE_FIELDS: tuple[str, ...] = ", layout["base"], ""), + "", + "# The World's fields (world.yaml), in schema order.", + *wrapped("WORLD_FIELDS: tuple[str, ...] = ", layout["world"], ""), + "", + "# type -> ((section, (field, ...)), ...): each type's own fields in schema order, under the YAML", + "# group that holds them. A generic link is listed by its own name (pin's `element`); GENERIC_LINKS", + "# names the two keys that hold it in data.", + "SECTIONS: dict[str, tuple[tuple[str, tuple[str, ...]], ...]] = {", + ] + for t in types: + lines.append(f' "{t}": (') + for s, names in layout["sections"][t]: + lines.extend(wrapped(f' ("{s}", ', names, "),")) + lines.append(" ),") + lines += [ + "}", + "", + "# type -> {link field: the type it links to}, for every single and multi link.", + "LINK_TARGETS: dict[str, dict[str, str]] = {", + ] + for t in types: + if t in layout["targets"]: + lines.append(f' "{t}": {{') + lines.extend(f' "{name}": "{target}",' for name, target in layout["targets"][t].items()) + lines.append(" },") + lines += [ + "}", + "", + "# type -> {generic link field: (type key, id key)}: the walk's names for the two keys that hold it.", + "GENERIC_LINKS: dict[str, dict[str, tuple[str, str]]] = {", + *[ + f' "{t}": {{{", ".join(f'"{n}": ("{a}", "{b}")' for n, (a, b) in layout["generic"][t].items())}}},' + for t in types + if t in layout["generic"] + ], + "}", + "", + "# Prefixes of the extension keys a tool may add to any element (dist walk/lint_folder.py).", + f"EXTENSION_PREFIXES: tuple[str, ...] = {strs(list(prefixes))}", + "", ] return "\n".join(lines) @@ -184,7 +308,9 @@ def main() -> int: pin = verify_dist() walk = load_walk() types = element_types(walk) - text = render(pin, types, field_kinds(walk, types), generic_pairs(walk, types)) + kinds, pairs = field_kinds(walk, types), generic_pairs(walk, types) + layout = layout_tables(decoded_schema(walk), types, kinds, pairs) + text = render(pin, types, kinds, pairs, layout, extension_prefixes()) if args.check: current = OUT.read_bytes().decode("utf-8") if OUT.is_file() else "" if current != text: diff --git a/pyproject.toml b/pyproject.toml index 31b38c8..d2323d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,12 +11,15 @@ classifiers = [ "Typing :: Typed", ] +[project.optional-dependencies] +okf = ["PyYAML>=6"] # onlyworlds.okf, the OKF bundle export; the core imports no YAML + [dependency-groups] dev = [ "pytest>=8", "ruff>=0.6", "mypy>=1.11", - "pyyaml>=6", # build-time only: codegen imports the vendored schema walk + "pyyaml>=6", # codegen imports the vendored schema walk; tests/test_okf.py needs it too "types-PyYAML", ] diff --git a/src/onlyworlds/_schema.py b/src/onlyworlds/_schema.py index 75e717f..308c80c 100644 --- a/src/onlyworlds/_schema.py +++ b/src/onlyworlds/_schema.py @@ -2,7 +2,7 @@ # Regenerate: python codegen/generate_schema.py (CI/test: --check) # Source: https://github.com/OnlyWorlds/schema-dist @ v0.32.0-dist.23 (a76a2a9ea4bbc8f26d8abd430dc8b1db7382902e) -"""Schema provenance and the 22 element types, generated from the pinned schema distribution.""" +"""Schema provenance, the 22 element types and their fields, from the pinned schema distribution.""" SCHEMA_DIST_TAG = "v0.32.0-dist.23" SCHEMA_VERSION = "00.32.00" @@ -554,3 +554,417 @@ GENERIC_PAIRS: dict[str, tuple[tuple[str, str], ...]] = { "pin": (("element_type", "element_id"),), } + +# Every element's shared fields (base_properties.yaml), in schema order, as data spells them. +BASE_FIELDS: tuple[str, ...] = ("id", "name", "description", "supertype", "subtype", "image_url", "world") + +# The World's fields (world.yaml), in schema order. +WORLD_FIELDS: tuple[str, ...] = ( + "id", "name", "description", "image_url", "created_at", "updated_at", "public_read", "time_format_equivalents", + "time_format_names", "time_basic_unit", "time_range_min", "time_range_max", "time_current", "length_unit", + "mass_unit", "distance_unit", +) + +# type -> ((section, (field, ...)), ...): each type's own fields in schema order, under the YAML +# group that holds them. A generic link is listed by its own name (pin's `element`); GENERIC_LINKS +# names the two keys that hold it in data. +SECTIONS: dict[str, tuple[tuple[str, tuple[str, ...]], ...]] = { + "ability": ( + ("Mechanics", ("activation", "duration", "potency", "range", "effects", "challenges", "talents", "requisites")), + ("World", ("prevalence", "tradition", "source", "locus", "instruments", "systems")), + ), + "collective": ( + ("Formation", ("composition", "count", "formation_date", "operator", "equipment")), + ("Dynamics", ("activity", "disposition", "state", "abilities", "symbolism")), + ("World", ("species", "characters", "creatures", "phenomena")), + ), + "character": ( + ("Constitution", ("physicality", "mentality", "height", "weight", "species", "traits", "abilities")), + ("Origins", ("background", "motivations", "birth_date", "birthplace", "languages")), + ("World", ("reputation", "location", "objects", "institutions")), + ("Personality", ("charisma", "coercion", "competence", "compassion", "creativity", "courage")), + ("Social", ("family", "friends", "rivals")), + ("TTRPG", ("level", "hit_points", "STR", "DEX", "CON", "INT", "WIS", "CHA")), + ), + "construct": ( + ("Nature", ("rationale", "history", "status", "reach", "start_date", "end_date", "founder", "custodian")), + ("Involves", ( + "characters", "objects", "locations", "species", "creatures", "institutions", "traits", "collectives", + "zones", "abilities", "phenomena", "languages", "families", "relations", "titles", "constructs", "events", + "narratives", + )), + ), + "creature": ( + ("Biology", ("appearance", "weight", "height", "species")), + ("Behavior", ("habits", "demeanor", "traits", "abilities", "languages")), + ("World", ("status", "birth_date", "location", "zone")), + ("TTRPG", ("challenge_rating", "hit_points", "armor_class", "speed", "actions")), + ), + "event": ( + ("Nature", ("history", "challenges", "consequences", "start_date", "end_date", "triggers")), + ("Involves", ( + "characters", "objects", "locations", "species", "creatures", "institutions", "traits", "collectives", + "zones", "abilities", "phenomena", "languages", "families", "relations", "titles", "constructs", + )), + ), + "family": ( + ("Identity", ("spirit", "history", "traditions", "traits", "abilities", "languages", "ancestors")), + ("World", ("reputation", "estates", "governs", "heirlooms", "creatures")), + ), + "institution": ( + ("Foundation", ("doctrine", "founding_date", "parent_institution")), + ("Claims", ("zones", "objects", "creatures")), + ("World", ("status", "allies", "adversaries", "constructs")), + ), + "language": ( + ("Structure", ("phonology", "grammar", "lexicon", "writing", "classification")), + ("World", ("status", "spread", "dialects")), + ), + "law": ( + ("Code", ("declaration", "purpose", "date", "parent_law", "penalties")), + ("World", ("author", "locations", "zones", "prohibitions", "adjudicators", "enforcers")), + ), + "location": ( + ("Setting", ("form", "function", "founding_date", "parent_location", "populations")), + ("Politics", ( + "political_climate", "primary_power", "governing_title", "secondary_powers", "zone", "rival", "partner", + )), + ("World", ("customs", "founders", "cults", "delicacies")), + ("Production", ("extraction_methods", "extraction_goods", "industry_methods", "industry_goods")), + ("Commerce", ("infrastructure", "extraction_markets", "industry_markets", "currencies")), + ("Construction", ("architecture", "buildings", "building_methods")), + ("Defense", ("defensibility", "elevation", "fighters", "defensive_objects")), + ), + "map": ( + ("Details", ("background_color", "hierarchy", "width", "height", "depth", "parent_map", "location")), + ), + "marker": ( + ("Details", ("map", "zone", "x", "y", "z", "order")), + ), + "narrative": ( + ("Context", ( + "story", "consequences", "start_date", "end_date", "order", "parent_narrative", "protagonist", "antagonist", + "narrator", "conservator", + )), + ("Involves", ( + "events", "characters", "objects", "locations", "species", "creatures", "institutions", "traits", + "collectives", "zones", "abilities", "phenomena", "languages", "families", "relations", "titles", + "constructs", "laws", + )), + ), + "object": ( + ("Form", ("aesthetics", "weight", "amount", "parent_object", "materials", "technology")), + ("Function", ("utility", "effects", "abilities", "consumes")), + ("World", ("origins", "location", "language", "affinities")), + ), + "phenomenon": ( + ("Mechanics", ("expression", "effects", "duration", "catalysts", "empowerments")), + ("World", ("mythology", "system", "triggers", "wielders", "environments")), + ), + "pin": ( + ("Details", ("map", "element", "x", "y", "z")), + ), + "relation": ( + ("Nature", ("background", "start_date", "end_date", "intensity", "actor", "events")), + ("Involves", ( + "characters", "objects", "locations", "species", "creatures", "institutions", "traits", "collectives", + "zones", "abilities", "phenomena", "languages", "families", "titles", "constructs", "narratives", + )), + ), + "species": ( + ("Biology", ("appearance", "life_span", "weight", "nourishment", "reproduction", "adaptations")), + ("Psychology", ("instincts", "sociality", "temperament", "communication", "aggression", "traits")), + ("World", ("role", "parent_species", "locations", "zones", "affinities")), + ), + "title": ( + ("Mandate", ( + "authority", "eligibility", "grant_date", "revoke_date", "issuer", "body", "superior_title", "holders", + "symbols", + )), + ("World", ( + "status", "history", "characters", "institutions", "families", "zones", "locations", "objects", + "constructs", "laws", "collectives", "creatures", "phenomena", "species", "languages", + )), + ), + "trait": ( + ("Qualitative", ( + "social_effects", "physical_effects", "functional_effects", "personality_effects", "behaviour_effects", + )), + ("Quantitative", ("charisma", "coercion", "competence", "compassion", "creativity", "courage")), + ("World", ("significance", "anti_trait", "empowered_abilities")), + ), + "zone": ( + ("Scope", ("role", "start_date", "end_date", "phenomena", "linked_zones")), + ("World", ("context", "populations", "titles", "principles")), + ), +} + +# type -> {link field: the type it links to}, for every single and multi link. +LINK_TARGETS: dict[str, dict[str, str]] = { + "ability": { + "effects": "phenomenon", + "talents": "trait", + "requisites": "construct", + "tradition": "construct", + "source": "phenomenon", + "locus": "location", + "instruments": "object", + "systems": "construct", + }, + "collective": { + "operator": "institution", + "equipment": "object", + "abilities": "ability", + "symbolism": "construct", + "species": "species", + "characters": "character", + "creatures": "creature", + "phenomena": "phenomenon", + }, + "character": { + "species": "species", + "traits": "trait", + "abilities": "ability", + "birthplace": "location", + "languages": "language", + "location": "location", + "objects": "object", + "institutions": "institution", + "family": "family", + "friends": "character", + "rivals": "character", + }, + "construct": { + "founder": "character", + "custodian": "institution", + "characters": "character", + "objects": "object", + "locations": "location", + "species": "species", + "creatures": "creature", + "institutions": "institution", + "traits": "trait", + "collectives": "collective", + "zones": "zone", + "abilities": "ability", + "phenomena": "phenomenon", + "languages": "language", + "families": "family", + "relations": "relation", + "titles": "title", + "constructs": "construct", + "events": "event", + "narratives": "narrative", + }, + "creature": { + "species": "species", + "traits": "trait", + "abilities": "ability", + "languages": "language", + "location": "location", + "zone": "zone", + "actions": "ability", + }, + "event": { + "triggers": "event", + "characters": "character", + "objects": "object", + "locations": "location", + "species": "species", + "creatures": "creature", + "institutions": "institution", + "traits": "trait", + "collectives": "collective", + "zones": "zone", + "abilities": "ability", + "phenomena": "phenomenon", + "languages": "language", + "families": "family", + "relations": "relation", + "titles": "title", + "constructs": "construct", + }, + "family": { + "traditions": "construct", + "traits": "trait", + "abilities": "ability", + "languages": "language", + "ancestors": "character", + "estates": "location", + "governs": "institution", + "heirlooms": "object", + "creatures": "creature", + }, + "institution": { + "parent_institution": "institution", + "zones": "zone", + "objects": "object", + "creatures": "creature", + "allies": "institution", + "adversaries": "institution", + "constructs": "construct", + }, + "language": { + "classification": "construct", + "spread": "location", + "dialects": "language", + }, + "law": { + "parent_law": "law", + "penalties": "construct", + "author": "institution", + "locations": "location", + "zones": "zone", + "prohibitions": "construct", + "adjudicators": "title", + "enforcers": "title", + }, + "location": { + "parent_location": "location", + "populations": "collective", + "primary_power": "institution", + "governing_title": "title", + "secondary_powers": "institution", + "zone": "zone", + "rival": "location", + "partner": "location", + "founders": "character", + "cults": "construct", + "delicacies": "species", + "extraction_methods": "construct", + "extraction_goods": "construct", + "industry_methods": "construct", + "industry_goods": "construct", + "extraction_markets": "location", + "industry_markets": "location", + "currencies": "construct", + "buildings": "object", + "building_methods": "construct", + "fighters": "construct", + "defensive_objects": "object", + }, + "map": { + "parent_map": "map", + "location": "location", + }, + "marker": { + "map": "map", + "zone": "zone", + }, + "narrative": { + "parent_narrative": "narrative", + "protagonist": "character", + "antagonist": "character", + "narrator": "character", + "conservator": "institution", + "events": "event", + "characters": "character", + "objects": "object", + "locations": "location", + "species": "species", + "creatures": "creature", + "institutions": "institution", + "traits": "trait", + "collectives": "collective", + "zones": "zone", + "abilities": "ability", + "phenomena": "phenomenon", + "languages": "language", + "families": "family", + "relations": "relation", + "titles": "title", + "constructs": "construct", + "laws": "law", + }, + "object": { + "parent_object": "object", + "materials": "construct", + "technology": "construct", + "effects": "phenomenon", + "abilities": "ability", + "consumes": "construct", + "location": "location", + "language": "language", + "affinities": "trait", + }, + "phenomenon": { + "catalysts": "object", + "empowerments": "ability", + "system": "phenomenon", + "triggers": "construct", + "wielders": "character", + "environments": "location", + }, + "pin": { + "map": "map", + }, + "relation": { + "actor": "character", + "events": "event", + "characters": "character", + "objects": "object", + "locations": "location", + "species": "species", + "creatures": "creature", + "institutions": "institution", + "traits": "trait", + "collectives": "collective", + "zones": "zone", + "abilities": "ability", + "phenomena": "phenomenon", + "languages": "language", + "families": "family", + "titles": "title", + "constructs": "construct", + "narratives": "narrative", + }, + "species": { + "nourishment": "species", + "reproduction": "construct", + "adaptations": "ability", + "traits": "trait", + "parent_species": "species", + "locations": "location", + "zones": "zone", + "affinities": "phenomenon", + }, + "title": { + "issuer": "institution", + "body": "institution", + "superior_title": "title", + "holders": "character", + "symbols": "object", + "characters": "character", + "institutions": "institution", + "families": "family", + "zones": "zone", + "locations": "location", + "objects": "object", + "constructs": "construct", + "laws": "law", + "collectives": "collective", + "creatures": "creature", + "phenomena": "phenomenon", + "species": "species", + "languages": "language", + }, + "trait": { + "anti_trait": "trait", + "empowered_abilities": "ability", + }, + "zone": { + "phenomena": "phenomenon", + "linked_zones": "zone", + "populations": "collective", + "titles": "title", + "principles": "construct", + }, +} + +# type -> {generic link field: (type key, id key)}: the walk's names for the two keys that hold it. +GENERIC_LINKS: dict[str, dict[str, tuple[str, str]]] = { + "pin": {"element": ("element_type", "element_id")}, +} + +# Prefixes of the extension keys a tool may add to any element (dist walk/lint_folder.py). +EXTENSION_PREFIXES: tuple[str, ...] = ("x_", "atlas_", "shadow_") diff --git a/src/onlyworlds/okf.py b/src/onlyworlds/okf.py new file mode 100644 index 0000000..ad77a88 --- /dev/null +++ b/src/onlyworlds/okf.py @@ -0,0 +1,616 @@ +"""Export a world folder to an Open Knowledge Format bundle. + + python -m onlyworlds.okf PATH/TO/WORLD-FOLDER OUT-DIR + +or ``export_okf(folder, out_dir)``. Needs PyYAML: ``pip install "onlyworlds[okf]"``. + +Writes OUT-DIR (which must not exist yet) as a bundle in Google Cloud's Open +Knowledge Format (OKF) v0.2, as specified at one pinned commit: + + repo: https://github.com/GoogleCloudPlatform/open-knowledge-format + commit: ad30107c31c06aec8a7d5636e0d1058118604e6f (SPEC.md "Version 0.2", + 2026-08-21, the ISO 8601 timestamp edit) + +The bundle: + + index.md root index; frontmatter `okf_version: "0.2"` + world.md the World as a concept (`type: OnlyWorlds + World`), carrying the bundle metadata + /index.md one per element type present + /--.md one concept per element + +Each element concept: + - `type` is the OnlyWorlds type, qualified as OKF's own examples qualify + theirs (`BigQuery Table`): `OnlyWorlds Character`. `title` is the element's + name, `description` its description. Never `verified`: an export confirms + nothing. + - `generated.by` is this exporter, a machine actor, and `generated.at` the + element's `updated_at`. OKF §7 asks for a `human:` actor on hand-authored + content; the words are the world's authors', but the bundle names no + author (the folder does not say who wrote what) and these files are + generated, so the machine actor is the true one. Consumers therefore read + the bundle as unverified (§5.3). + - scalar fields sit in the frontmatter under their OnlyWorlds names: `id`, + `supertype`, `subtype`, `image_url` and every integer field. A field + whose name OKF defines (§4.1, §5, §10: `status` and the like) would be + written as `ow_`; no frontmatter field needs it today. + - every link field is in the frontmatter, keyed by its OnlyWorlds field + name, as bundle-absolute paths (`location: /location/barovia--1a2b3c4d.md`, + a list for a multi link; `element` for a pin's generic link), so the kind + of each link survives as a key although OKF links are untyped (§6.1). + A link to an id the folder does not hold is written `urn:uuid:`. + - the body holds the type's own text fields under `#
` / + `## ` headings in schema order, and every link again as a markdown + link in a bullet (`- **location**: [Barovia](../location/....md)`), so + viewers that draw the link graph from bodies draw it. Body links are + relative: the reference viewer at the pinned commit skips `/` links. A + text line markdown would read as a heading (`#`, or a `===`/`---` + underline) is escaped with a backslash, so text cannot make a section. + - unset values (the empty string, null, an empty list) are left out. + Extension keys (`x_`, `atlas_`, `shadow_`) are carried as they are; the + server and folder keys the package never pushes (``push.NOT_SENT``) are + not, except `updated_at` as `generated.at`. + +world.md carries the World's schema fields (`time_current` read from +`time_range_current` when only that spelling is present, folder format §5), +its extension keys, `owner_character` (as a link), `format_version`, +`writable` and every `snapshot_*` key. The `api` block is dropped: it is a +credential and a sync switch, not world content (folder format §4.1, the +portable copy). + +Nothing is dropped in silence. Each of these is a note (on stderr from the +command line, in ``OkfReport.notes`` from Python): a file the folder reader +skipped (not JSON, not an object, no `id`), a dot-prefixed file (tool-private: +folder format §4.1 says skip it, leave it, log it), a directory that is not a +type directory, an id in both `elements/` and the legacy `spatial/` (the +`elements/` copy is served, folder format v0.3.8), a key that is not a field +of its type, a field kind this exporter does not know, and a link left out +(not an id, a null in a multi link, or an element of the wrong type). A link +to an id the folder does not hold is listed in ``OkfReport.unresolved``. + +The folder is read by ``onlyworlds.folder.read_folder``, the package's one +reader; the schema comes from the package's generated tables (``_schema``), +so this module reads no schema YAML. + +The output is deterministic: the same folder and the same package give a +byte-identical bundle (UTF-8, LF, no export timestamp). From the command +line: exit 0 written, 1 when world.json cannot be read or one id is in two +files of one layout, 2 on bad arguments (a missing folder, an existing +OUT-DIR). +""" + +from __future__ import annotations + +import argparse +import io +import os +import re +import sys +import unicodedata +from collections.abc import Callable, Mapping +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Any + +try: + import yaml +except ImportError as exc: # pragma: no cover - exercised only without the extra + raise ImportError('onlyworlds.okf needs PyYAML: pip install "onlyworlds[okf]"') from exc + +from ._schema import ( + BASE_FIELDS, + ELEMENT_TYPES, + EXTENSION_PREFIXES, + FIELD_KINDS, + GENERIC_LINKS, + LINK_TARGETS, + SCHEMA_DIST_SERIAL, + SCHEMA_VERSION, + SECTIONS, + WORLD_FIELDS, +) +from .folder import WIRE_TO_DISK_WORLD_KEYS, WORLD_FILE, Folder, read_folder +from .push import NOT_SENT + +__all__ = ["EXPORTER", "OKF_COMMIT", "OKF_VERSION", "FolderError", "OkfReport", "build_bundle", "export_okf", "main"] + +EXPORTER = "onlyworlds-okf-export/0.1" +OKF_VERSION = "0.2" +OKF_REPO = "https://github.com/GoogleCloudPlatform/open-knowledge-format" +OKF_COMMIT = "ad30107c31c06aec8a7d5636e0d1058118604e6f" + +# Frontmatter keys OKF v0.2 gives a meaning (SPEC §4.1, §5, §10, §12). An +# OnlyWorlds field of the same name is written as `ow_`. +OKF_KEYS = frozenset( + { + "type", "title", "description", "resource", "tags", "sources", "usage_window", "generated", "verified", + "status", "stale_after", "runtime", "parameters", "computation", "executor", "attester", "okf_version", + } +) # fmt: skip +# world.json keys the folder format defines beside the schema's (§4.2, §5). +WORLD_FOLDER_KEYS = ("format_version", "writable") +WORLD_DROPPED = ("api", *WIRE_TO_DISK_WORLD_KEYS) +# Every break str.splitlines() honours: the reference parser splits the +# frontmatter on these, so a string holding one is written double-quoted. +LINE_BREAKS = "\n\r\x0b\x0c\x1c\x1d\x1e\x85\u2028\u2029" + +Note = Callable[[str], None] + + +def _type_name(slug: str) -> str: + """The OKF `type` of a slug (or the World), qualified: `OnlyWorlds Character`.""" + return f"OnlyWorlds {slug.capitalize()}" + + +class FolderError(ValueError): + """The folder cannot be exported: world.json unreadable, or one id in two files of one layout.""" + + +@dataclass +class OkfReport: + """What an export did. ``notes`` are what it left out or chose, one line each, in order.""" + + elements: int = 0 + types: int = 0 + files: int = 0 + links: int = 0 + #: ``: `` -> `` for each link to an id the folder does not hold + unresolved: list[str] = field(default_factory=list) + notes: list[str] = field(default_factory=list) + out: Path | None = None + + +# --- YAML: block style, line-break-bearing strings double-quoted on one line +# (the breaks escaped), so no frontmatter line can read as the closing `---`. + + +class _Dumper(yaml.SafeDumper): + pass + + +def _str_rep(dumper: yaml.SafeDumper, value: str) -> yaml.ScalarNode: + style = '"' if any(c in value for c in LINE_BREAKS) else None + return dumper.represent_scalar("tag:yaml.org,2002:str", value, style=style) + + +_Dumper.add_representer(str, _str_rep) + + +def _frontmatter(data: dict[str, Any]) -> str: + return str( + yaml.dump(data, Dumper=_Dumper, sort_keys=False, allow_unicode=True, default_flow_style=False, width=1 << 30) + ) + + +# --- reading the folder: read_folder's report, as notes + + +def _rel(folder: Folder, path: Path) -> str: + return path.relative_to(folder.root).as_posix() + + +def _elements(folder: Folder, note: Note) -> list[tuple[str, dict[str, Any]]]: + """(type slug, body) for every exportable element, elements/ before spatial/, type order, path order. + What read_folder skipped or set aside is noted here; a duplicate id in one layout is an error.""" + for s in folder.skipped: + if s.path.parent == folder.root and s.path.name == WORLD_FILE: + continue + rel = _rel(folder, s.path) + note( + f"{rel}: dot-prefixed, tool-private; skipped" + if s.path.name.startswith(".") + else f"{rel}: {s.reason}; skipped" + ) + for d in folder.ignored_dirs: + note(f"{_rel(folder, d)}/: not a type directory; skipped") + for dup in folder.duplicates: + if dup.ignored.name.startswith("."): + note(f"{_rel(folder, dup.ignored)}: dot-prefixed, tool-private; skipped") + continue + raise FolderError(f"{_rel(folder, dup.ignored)}: duplicate id {dup.id} (also {_rel(folder, dup.kept)})") + for eid in folder.in_both_layouts: + legacy, served = folder.legacy[eid].path, folder.elements[eid].path + note(f"{_rel(folder, legacy)}: id {eid} also at {_rel(folder, served)}; served elements/") + + order = {slug: i for i, slug in enumerate(ELEMENT_TYPES)} + found = sorted( + folder.all_elements().values(), + key=lambda e: (e.layout != "elements", order.get(e.path.parent.name.lower(), len(order)), e.path), + ) + out: list[tuple[str, dict[str, Any]]] = [] + for el in found: + rel = _rel(folder, el.path) + if el.path.name.startswith("."): + note(f"{rel}: dot-prefixed, tool-private; skipped") + elif el.type not in ELEMENT_TYPES: + note(f"{rel}: `type` {el.type!r} is not one of the 22 types; skipped") + else: + out.append((el.type, el.body)) + return out + + +# --- names and links + + +def _slugify(name: str) -> str: + """The bundle's own file names (not the folder's: §3.2 filenames are presentation, and so are these).""" + text = unicodedata.normalize("NFKD", name).encode("ascii", "ignore").decode() + text = re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-") + return text[:60].rstrip("-") or "element" + + +def assign_paths(elements: list[tuple[str, dict[str, Any]]]) -> dict[str, str]: + """id -> bundle path (`character/the-consul--8b711e25.md`). Names are the slug plus the id's + last 8 hex digits; a name two elements of one type would share falls back to the full id for + both, so no order decides it.""" + stems: dict[str, tuple[str, str]] = {} + for slug, body in elements: + hexid = re.sub(r"[^0-9a-f]", "", body["id"].lower()) or "0" + stems[body["id"]] = (slug, f"{_slugify(str(body.get('name') or ''))}--{hexid[-8:]}") + counts: dict[tuple[str, str], int] = {} + for key in stems.values(): + counts[key] = counts.get(key, 0) + 1 + paths = {} + for eid, (slug, stem) in stems.items(): + if counts[(slug, stem)] > 1: + stem = stem.rsplit("--", 1)[0] + "--" + re.sub(r"[^0-9a-f]", "", eid.lower()) + paths[eid] = f"{slug}/{stem}.md" + return paths + + +def _link_text(name: str) -> str: + text = " ".join(str(name).split()) or "(unnamed)" + return text.replace("\\", "\\\\").replace("[", "\\[").replace("]", "\\]") + + +def _one_line(text: str) -> str: + return " ".join(str(text).split()) + + +def _heading(name: str) -> str: + words = name.replace("_", " ") + return words[:1].upper() + words[1:] + + +_FAKE_HEADING = re.compile(r" {0,3}#| {0,3}(=+|-+)[ \t]*$") + + +def escape_text(text: str) -> str: + """A text field as body markdown: LF, outer blank lines trimmed, and a backslash before any + line markdown would read as a heading (`# x`, or a `===`/`---` line that would underline the + line above).""" + out = [] + for line in str(text).replace("\r\n", "\n").replace("\r", "\n").strip("\n").split("\n"): + if _FAKE_HEADING.match(line): + i = len(line) - len(line.lstrip(" ")) + line = line[:i] + "\\" + line[i:] + out.append(line) + return "\n".join(out) + + +def _timestamp(value: Any) -> str | None: + """An ISO 8601 datetime with an explicit offset (SPEC §5), else None.""" + if not isinstance(value, str): + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + return value if parsed.tzinfo is not None else None + + +def _unset(value: Any) -> bool: + return value is None or value == "" or value == [] + + +def _fm_key(name: str) -> str: + return f"ow_{name}" if name in OKF_KEYS else name + + +def _generated(body: Mapping[str, Any]) -> dict[str, str]: + gen = {"by": EXPORTER} + at = _timestamp(body.get("updated_at")) + if at: + gen["at"] = at + return gen + + +class _Links: + """Resolves one file's links: a bundle path and a body link per id, or a note and nothing for + an id that cannot be exported.""" + + def __init__(self, here: str, paths: Mapping[str, str], names: Mapping[str, str], report: OkfReport, note: Note): + self.here, self.paths, self.names = here, paths, names + self.report, self.note = report, note + + def one(self, name: str, eid: Any, target: str | None) -> tuple[str, str] | None: + if not isinstance(eid, str) or not eid: + self.note(f"{self.here}: `{name}` holds {eid!r}, not an id; left out") + return None + if eid not in self.paths: + self.report.unresolved.append(f"{self.here}: `{name}` -> {eid}") + return f"urn:uuid:{eid}", f"`{eid}` (not in this bundle)" + actual = self.paths[eid].split("/", 1)[0] + if target and actual != target: + self.note(f"{self.here}: `{name}` -> a {actual}, expected {target}; left out") + return None + self.report.links += 1 + rel = "../" + self.paths[eid] if "/" in self.here else self.paths[eid] + return "/" + self.paths[eid], f"[{_link_text(self.names[eid])}]({rel})" + + +# --- one element + + +def element_doc( + slug: str, + body: Mapping[str, Any], + paths: Mapping[str, str], + names: Mapping[str, str], + report: OkfReport, + note: Note, + *, + kinds: Mapping[str, Mapping[str, str]] = FIELD_KINDS, + sections: Mapping[str, tuple[tuple[str, tuple[str, ...]], ...]] = SECTIONS, +) -> str: + """One element as an OKF concept. ``kinds`` and ``sections`` default to the generated schema.""" + here = paths[body["id"]] + links = _Links(here, paths, names, report, note) + fm: dict[str, Any] = {"type": _type_name(slug)} + if not _unset(body.get("name")): + fm["title"] = str(body["name"]) + if not _unset(body.get("description")): + fm["description"] = str(body["description"]) + fm["generated"] = _generated(body) + known = set(NOT_SENT) + for name in BASE_FIELDS: + known.add(name) + if name in ("name", "description", "world"): + continue + if not _unset(body.get(name)): + fm[_fm_key(name)] = body[name] + + blocks: list[tuple[str, str]] = [] # (kind, text): kind "text" or "link" + current: str | None = None + + def open_section(name: str) -> None: + nonlocal current + if current != name: + current = name + blocks.append(("text", f"# {name}")) + + def add_links(section: str, name: str, found: list[tuple[str, str]], many: bool) -> None: + if not found: + return + fm[_fm_key(name)] = [p for p, _ in found] if many else found[0][0] + open_section(section) + blocks.append(("link", f"- **{name}**: " + ", ".join(s for _, s in found))) + + generic = GENERIC_LINKS.get(slug, {}) + targets = LINK_TARGETS.get(slug, {}) + for section, field_names in sections[slug]: + for name in field_names: + if name in generic: + type_key, id_key = generic[name] + known.update((type_key, id_key)) + eid, target = body.get(id_key), body.get(type_key) + if _unset(eid) and _unset(target): + continue + if not isinstance(target, str) or target not in ELEMENT_TYPES: + note(f"{here}: `{name}` names type {target!r}, not a type slug; left out") + continue + hit = links.one(name, eid, target) + add_links(section, name, [hit] if hit else [], many=False) + continue + known.add(name) + value = body.get(name) + if _unset(value): + continue + kind = kinds[slug].get(name) + if kind == "text": + open_section(section) + blocks.append(("text", f"## {_heading(name)}\n\n{escape_text(str(value))}")) + elif kind == "integer": + fm[_fm_key(name)] = value + elif kind == "single_link": + hit = links.one(name, value, targets.get(name)) + add_links(section, name, [hit] if hit else [], many=False) + elif kind == "multi_link": + if not isinstance(value, list): + note(f"{here}: `{name}` is not a list; left out") + continue + hits = [links.one(name, v, targets.get(name)) for v in value] + add_links(section, name, [h for h in hits if h], many=True) + else: + note(f"{here}: `{name}` has kind `{kind}`, which this exporter does not know; left out") + + for key, value in body.items(): + if key.startswith(EXTENSION_PREFIXES): + fm[key] = value + elif key not in known: + note(f"{here}: `{key}` is not a field of {slug}; left out") + + out: list[str] = [] + for i, (kind, text) in enumerate(blocks): + if i and not (kind == "link" and blocks[i - 1][0] == "link"): + out.append("\n") + out.append(text + "\n") + return f"---\n{_frontmatter(fm)}---\n\n" + "".join(out) + + +# --- the world and the indexes + + +def world_doc( + world: Mapping[str, Any], + time_current: Any, + count: int, + paths: Mapping[str, str], + names: Mapping[str, str], + report: OkfReport, + note: Note, +) -> str: + """world.md. ``time_current`` is the folder's (``Folder.time_current``: either spelling, §5).""" + links = _Links("world.md", paths, names, report, note) + world = dict(world) + if time_current is not None: + world["time_current"] = time_current + fm: dict[str, Any] = {"type": _type_name("world")} + if not _unset(world.get("name")): + fm["title"] = str(world["name"]) + if not _unset(world.get("description")): + fm["description"] = str(world["description"]) + fm["generated"] = _generated(world) + known = {*WORLD_DROPPED, "owner_character"} + for name in WORLD_FIELDS: + known.add(name) + if name in ("name", "description"): + continue + if not _unset(world.get(name)): + fm[_fm_key(name)] = world[name] + if not _unset(world.get("owner_character")): + hit = links.one("owner_character", world["owner_character"], "character") + if hit: + fm["owner_character"] = hit[0] + for key, value in world.items(): + if key in WORLD_FOLDER_KEYS or key.startswith(("snapshot_", *EXTENSION_PREFIXES)): + fm[_fm_key(key)] = value + elif key not in known: + note(f"world.json: `{key}` is not a World field; left out") + fm["onlyworlds_export"] = { + "exporter": EXPORTER, + "schema": SCHEMA_VERSION, + "schema_dist_serial": SCHEMA_DIST_SERIAL, + "okf_version": OKF_VERSION, + "okf_spec": f"{OKF_REPO}/blob/{OKF_COMMIT}/SPEC.md", + "elements": count, + } + return f"---\n{_frontmatter(fm)}---\n" + + +def index_entry(title: str, link: str, desc: str) -> str: + suffix = f" - {_one_line(desc)}" if _one_line(desc) else "" + return f"* [{_link_text(title)}]({link}){suffix}" + + +def build_bundle(folder: Folder | str | os.PathLike[str], note: Note | None = None) -> tuple[dict[str, str], OkfReport]: + """Every bundle file as {bundle path: text}, and the report. Writes nothing. + + ``note`` is called with each note as it is made (they are also in ``report.notes``). + Raises ``FolderError`` when world.json cannot be read or one id is in two files of one layout. + """ + report = OkfReport() + + def noted(msg: str) -> None: + report.notes.append(msg) + if note is not None: + note(msg) + + if not isinstance(folder, Folder): + folder = read_folder(folder) + if folder.world is None: + why = [s.reason for s in folder.skipped if s.path.parent == folder.root and s.path.name == WORLD_FILE] + raise FolderError(why[0] if why else f"{folder.root}: no {WORLD_FILE}") + elements = _elements(folder, noted) + paths = assign_paths(elements) + names = {body["id"]: str(body.get("name") or "") for _, body in elements} + report.elements = len(elements) + files: dict[str, str] = {} + by_type: dict[str, list[tuple[str, str, str]]] = {} + for slug, body in elements: + path = paths[body["id"]] + files[path] = element_doc(slug, body, paths, names, report, noted) + by_type.setdefault(slug, []).append( + (names[body["id"]], path.split("/", 1)[1], str(body.get("description") or "")) + ) + + files["world.md"] = world_doc(folder.world, folder.time_current, len(elements), paths, names, report, noted) + root = [ + f'---\nokf_version: "{OKF_VERSION}"\n---\n', + "# World\n", + index_entry( + str(folder.world.get("name") or "World"), + "world.md", + f"The world. OnlyWorlds schema {SCHEMA_VERSION}; OKF {OKF_VERSION} at commit {OKF_COMMIT[:7]}.", + ), + "", + "# Element types\n", + ] + for slug in ELEMENT_TYPES: + if slug not in by_type: + continue + entries = sorted(by_type[slug], key=lambda e: (e[0].casefold(), e[1])) + n = len(entries) + root.append(index_entry(_type_name(slug), f"{slug}/index.md", f"{n} element{'s' if n != 1 else ''}")) + lines = [f"# {_type_name(slug)}", ""] + lines += [index_entry(t, link, d) for t, link, d in entries] + files[f"{slug}/index.md"] = "\n".join(lines) + "\n" + files["index.md"] = "\n".join(root) + "\n" + report.types = len(by_type) + report.files = len(files) + return files, report + + +def write_bundle(files: Mapping[str, str], out: str | os.PathLike[str]) -> None: + """Write ``files`` under ``out``, which must not exist (``FileExistsError``).""" + out = Path(out) + out.mkdir(parents=True, exist_ok=False) + for rel in sorted(files): + path = out / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(files[rel].encode("utf-8")) + + +def export_okf( + folder: str | os.PathLike[str], out_dir: str | os.PathLike[str], *, note: Note | None = None +) -> OkfReport: + """Export the world folder at ``folder`` as an OKF bundle at ``out_dir`` (which must not exist). + + Returns the report: counts, the unresolved links and the notes (``note`` is also called with + each as it is made). Raises ``FileExistsError`` if ``out_dir`` exists, ``FolderError`` if the + folder cannot be exported. Nothing is written unless the whole bundle was built. + """ + out = Path(out_dir) + if out.exists(): + raise FileExistsError(f"{out}: already exists; give a new directory") + files, report = build_bundle(folder, note) + write_bundle(files, out) + report.out = out + return report + + +def main(argv: list[str] | None = None) -> int: + for stream in (sys.stdout, sys.stderr): + if isinstance(stream, io.TextIOWrapper): + stream.reconfigure(encoding="utf-8") + ap = argparse.ArgumentParser(prog="python -m onlyworlds.okf", description=(__doc__ or "").splitlines()[0]) + ap.add_argument("folder", type=Path, help="a world folder (holds world.json and elements/)") + ap.add_argument("out", type=Path, help="the bundle directory to create; must not exist") + args = ap.parse_args(argv) + if not (args.folder / WORLD_FILE).is_file(): + print(f"{args.folder}: not a world folder (no world.json)", file=sys.stderr) + return 2 + if args.out.exists(): + print(f"{args.out}: already exists; give a new directory", file=sys.stderr) + return 2 + + def note(msg: str) -> None: + print(f"note: {msg}", file=sys.stderr) + + try: + report = export_okf(args.folder, args.out, note=note) + except FolderError as e: + print(f"{e}", file=sys.stderr) + return 1 + for line in report.unresolved: + print(f"unresolved link: {line}", file=sys.stderr) + print( + f"exported {report.elements} elements in {report.types} types to {args.out}: " + f"{report.files} files, {report.links} links, {len(report.unresolved)} unresolved" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_okf.py b/tests/test_okf.py new file mode 100644 index 0000000..fc46332 --- /dev/null +++ b/tests/test_okf.py @@ -0,0 +1,550 @@ +"""OKF export (onlyworlds.okf) on the example world the vendored dist ships (codegen/schema-dist/example/hyperion). + +Ported from schema-dist's walk/test_okf_export.py (branch okf-exporter, 8d0f92f): the same 24 tests, +on the package's reader and generated schema tables. Needs PyYAML (the ``okf`` extra; the dev group has +it); without it the module skips. OKF's reference parser and viewer +(github.com/GoogleCloudPlatform/open-knowledge-format at the commit okf.py pins) run too when importable: +put a checkout's ``src/`` on OKF_REFERENCE_SRC; otherwise those checks skip and say so. +""" + +from __future__ import annotations + +import importlib.util +import io +import json +import os +import re +import shutil +import subprocess +import sys +from collections.abc import Callable, Iterator +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path +from typing import Any + +import pytest + +yaml = pytest.importorskip("yaml", reason='onlyworlds.okf needs PyYAML: pip install "onlyworlds[okf]"') + +from onlyworlds import okf # noqa: E402 +from onlyworlds._schema import ( # noqa: E402 + EXTENSION_PREFIXES, + FIELD_KINDS, + GENERIC_LINKS, + SCHEMA_VERSION, + SECTIONS, +) +from onlyworlds.push import NOT_SENT # noqa: E402 + +ROOT = Path(__file__).resolve().parents[1] +HYPERION = ROOT / "codegen" / "schema-dist" / "example" / "hyperion" +BODY_LINK = re.compile(r"\]\(([^)\s]+\.md)\)") + +if os.environ.get("OKF_REFERENCE_SRC"): + sys.path.insert(0, os.environ["OKF_REFERENCE_SRC"]) +try: + from reference_agent.bundle.document import OKFDocument # type: ignore[import-not-found] + from reference_agent.viewer import generate_visualization # type: ignore[import-not-found] + + REFERENCE = None +except ImportError as e: + OKFDocument = generate_visualization = None + REFERENCE = ( + f"OKF reference parser not importable ({e}); set OKF_REFERENCE_SRC " + f"to the src/ of an open-knowledge-format checkout at {okf.OKF_COMMIT[:7]}" + ) + + +def run(src: Path, out: Path) -> tuple[int, str]: + err = io.StringIO() + with redirect_stdout(io.StringIO()), redirect_stderr(err): + code = okf.main([str(src), str(out)]) + return code, err.getvalue() + + +def tree(root: Path) -> dict[str, bytes]: + return {p.relative_to(root).as_posix(): p.read_bytes() for p in sorted(root.rglob("*")) if p.is_file()} + + +def copy_world(dst: Path) -> Path: + shutil.copytree(HYPERION, dst) + return dst + + +def split(text: str) -> tuple[dict[str, Any], str]: + """Frontmatter and body, found the way the reference parser finds them: + str.splitlines(), so every line break Python knows ends a line.""" + lines = text.splitlines() + assert lines and lines[0] == "---", "no frontmatter" + end = lines.index("---", 1) + return yaml.safe_load("\n".join(lines[1:end])) or {}, "\n".join(lines[end + 1 :]) + + +def heading(name: str) -> str: + words = name.replace("_", " ") + return words[:1].upper() + words[1:] + + +def escape_text(text: str) -> str: + """The body's text rule: a line markdown would read as a heading is escaped.""" + out = [] + for line in text.replace("\r\n", "\n").replace("\r", "\n").strip("\n").split("\n"): + if re.match(r" {0,3}#", line) or re.fullmatch(r" {0,3}(=+|-+)[ \t]*", line): + i = len(line) - len(line.lstrip(" ")) + line = line[:i] + "\\" + line[i:] + out.append(line) + return "\n".join(out) + + +def body_texts(body: str) -> dict[str, str]: + found = re.finditer(r"^## ([^\n]+)\n\n(.*?)(?=\n\n#{1,2} |\n\n- \*\*|\Z)", body, re.S | re.M) + return {m.group(1): m.group(2).rstrip("\n") for m in found} + + +def body_links(body: str) -> dict[str, list[str]]: + found = re.finditer(r"^- \*\*(\w+)\*\*: (.*)$", body, re.M) + return {m.group(1): BODY_LINK.findall(m.group(2)) for m in found} + + +def type_fields(slug: str) -> list[str]: + return [name for _, names in SECTIONS[slug] for name in names] + + +def link_kind(slug: str, name: str) -> str | None: + if name in GENERIC_LINKS.get(slug, {}): + return "generic" + kind = FIELD_KINDS[slug].get(name) + return kind if kind in ("single_link", "multi_link") else None + + +def link_ids(src: dict[str, Any], slug: str, name: str) -> list[Any]: + kind = link_kind(slug, name) + if kind == "single_link": + return [src[name]] if src.get(name) else [] + if kind == "multi_link": + return list(src.get(name) or []) + if kind == "generic": + id_key = GENERIC_LINKS[slug][name][1] + return [src[id_key]] if src.get(id_key) else [] + return [] + + +class Bundle: + def __init__(self, out: Path) -> None: + self.out = out + self.code, self.err = run(HYPERION, out) + assert self.code == 0, self.err + self.files = tree(out) + self.sources: dict[str, dict[str, Any]] = { + s["id"]: s + for s in ( + json.loads(p.read_text(encoding="utf-8")) for p in sorted((HYPERION / "elements").rglob("*.json")) + ) + } + self.docs = { + rel: split(data.decode("utf-8")) + for rel, data in self.files.items() + if rel.endswith(".md") and not rel.endswith("index.md") and rel != "world.md" + } + self.by_id = {fm["id"]: rel for rel, (fm, _) in self.docs.items()} + + def target_id(self, path: str, base: str | None = None) -> str: + """The id inside the file a link names (bundle-absolute, or relative to base).""" + rel = path[1:] if base is None else (Path(base).parent / path).as_posix() + rel = os.path.normpath(rel).replace("\\", "/") + assert rel in self.docs, f"link {path} from {base}: no such concept" + return str(self.docs[rel][0]["id"]) + + +@pytest.fixture(scope="module") +def bundle(tmp_path_factory: pytest.TempPathFactory) -> Bundle: + return Bundle(tmp_path_factory.mktemp("okf") / "bundle") + + +# --- Hyperion's bundle + + +def test_one_file_per_element(bundle: Bundle) -> None: + assert len(bundle.docs) == len(bundle.sources) + assert set(bundle.by_id) == set(bundle.sources) + + +def test_type_title_description(bundle: Bundle) -> None: + for rel, (fm, _) in bundle.docs.items(): + src = bundle.sources[fm["id"]] + assert fm["type"] == "OnlyWorlds " + src["type"].capitalize(), rel + assert fm["title"] == src["name"], rel + assert fm.get("description", "") == (src.get("description") or ""), rel + assert fm["generated"] == {"by": okf.EXPORTER, "at": src["updated_at"]} + assert "verified" not in fm, rel + + +def test_scalars_and_extensions_match_source(bundle: Bundle) -> None: + """Base strings, every integer (0 included) and every extension key.""" + checked = 0 + for rel, (fm, _) in bundle.docs.items(): + src = bundle.sources[fm["id"]] + for name in ("supertype", "subtype", "image_url"): + assert fm.get(name, "") == (src.get(name) or ""), f"{rel}: {name}" + for name in type_fields(src["type"]): + if FIELD_KINDS[src["type"]].get(name) == "integer": + assert fm.get(name) == src.get(name), f"{rel}: {name}" + checked += src.get(name) == 0 + for key, value in src.items(): + if key.startswith(EXTENSION_PREFIXES): + assert fm.get(key) == value, f"{rel}: {key}" + checked += 1 + assert checked > 1, "the fixture holds a 0 and an extension key" + + +def test_text_fields_match_source(bundle: Bundle) -> None: + checked = 0 + for rel, (fm, body) in bundle.docs.items(): + src = bundle.sources[fm["id"]] + expected = { + heading(name): escape_text(src[name]) + for name in type_fields(src["type"]) + if FIELD_KINDS[src["type"]].get(name) == "text" and src.get(name) + } + assert body_texts(body) == expected, rel + checked += len(expected) + assert checked > 0 + + +def test_every_link_names_the_right_element(bundle: Bundle) -> None: + """Frontmatter paths and body links, field by field, each target read by + the id in its file: single, multi and generic, none missing, none extra.""" + checked = 0 + for rel, (fm, body) in bundle.docs.items(): + src = bundle.sources[fm["id"]] + bullets = body_links(body) + for name in type_fields(src["type"]): + kind = link_kind(src["type"], name) + if kind is None: + continue + want = link_ids(src, src["type"], name) + if not want: + assert name not in fm, rel + assert name not in bullets, rel + continue + value = fm.get(name) + assert value is not None, f"{rel}: {name} missing" + paths = value if kind == "multi_link" else [value] + assert [bundle.target_id(p) for p in paths] == want, f"{rel}: {name}" + assert [bundle.target_id(p, rel) for p in bullets.get(name, [])] == want, f"{rel}: body {name}" + checked += len(want) + assert checked == 53 + + +def test_indexes(bundle: Bundle) -> None: + root = bundle.files["index.md"].decode("utf-8") + assert root.startswith('---\nokf_version: "0.2"\n---\n') + assert "* [OnlyWorlds Character](character/index.md) - 3 elements" in root + assert bundle.files["character/index.md"].startswith(b"# OnlyWorlds Character\n") + listed = [] + for rel, data in bundle.files.items(): + if rel.endswith("/index.md"): + assert not data.startswith(b"---"), f"{rel}: frontmatter in an index" + for link in BODY_LINK.findall(data.decode("utf-8")): + listed.append(bundle.target_id(link, rel)) + assert sorted(listed) == sorted(bundle.sources) + + +def test_world_metadata(bundle: Bundle) -> None: + fm, _ = split(bundle.files["world.md"].decode("utf-8")) + meta = fm["onlyworlds_export"] + assert fm["type"] == "OnlyWorlds World" + assert fm["title"] == "Hyperion" + assert fm["time_current"] == 0 + assert meta["schema"] == SCHEMA_VERSION + assert okf.OKF_COMMIT in meta["okf_spec"] + assert meta["elements"] == len(bundle.sources) + + +def test_rerun_is_byte_identical(bundle: Bundle, tmp_path: Path) -> None: + code, _ = run(HYPERION, tmp_path / "bundle") + assert code == 0 + assert tree(tmp_path / "bundle") == bundle.files + + +def test_lf_utf8(bundle: Bundle) -> None: + for rel, data in bundle.files.items(): + assert b"\r" not in data, rel + data.decode("utf-8") + + +def test_refuses_existing_out_dir(tmp_path: Path) -> None: + assert run(HYPERION, tmp_path)[0] == 2 + assert list(tmp_path.iterdir()) == [] + + +@pytest.mark.skipif(REFERENCE is not None, reason=REFERENCE or "") +def test_reference_parser_and_viewer(bundle: Bundle, tmp_path: Path) -> None: + """OKF's own code at the pinned commit: every concept parses and has a + type, indexes carry no frontmatter but the root's okf_version, and the + viewer draws one edge per distinct linked pair.""" + for rel, data in bundle.files.items(): + doc = OKFDocument.parse(data.decode("utf-8")) + if rel.endswith("index.md"): + allowed = {"okf_version"} if rel == "index.md" else set() + assert set(doc.frontmatter) <= allowed, rel + else: + doc.validate() + pairs = { + (s["id"], t) + for s in bundle.sources.values() + for name in type_fields(s["type"]) + for t in link_ids(s, s["type"], name) + if t != s["id"] + } + stats = generate_visualization(bundle.out, tmp_path / "viz.html") + assert stats["concepts"] == len(bundle.sources) + 1 + assert stats["edges"] == len(pairs) + + +# --- the legacy layout + + +def test_legacy_spatial_folder_exports_the_same_bundle(tmp_path: Path) -> None: + """Map, pin, zone and marker under the legacy spatial/, one zone also in + elements/ under another name: the bundle is Hyperion's, byte for byte, + with the elements/ copy served.""" + src = copy_world(tmp_path / "world") + (src / "spatial").mkdir() + for slug in ("map", "pin", "zone", "marker"): + shutil.move(str(src / "elements" / slug), str(src / "spatial" / slug)) + zone = next((src / "spatial" / "zone").glob("*.json")) + (src / "elements" / "zone").mkdir() + shutil.copy(zone, src / "elements" / "zone" / zone.name) + old = json.loads(zone.read_text(encoding="utf-8")) + zone.write_text(json.dumps(dict(old, name="Swamps (stale copy)")), encoding="utf-8") + code, err = run(src, tmp_path / "bundle") + assert code == 0, err + assert "served elements/" in err + assert run(HYPERION, tmp_path / "plain")[0] == 0 + assert tree(tmp_path / "bundle") == tree(tmp_path / "plain") + + +# --- hard cases + + +def test_awkward_cases(tmp_path: Path) -> None: + """A frontmatter-breaking description, a name with brackets, two elements of one type + sharing a name and id tail, and a link to an element the folder does not hold.""" + src = copy_world(tmp_path / "world") + chars = src / "elements" / "character" + consul_path = next(chars.glob("the-consul--*.json")) + consul = json.loads(consul_path.read_text(encoding="utf-8")) + consul["name"] = "The [Consul]" + consul["description"] = "line one\n---\nline: three" + consul["friends"] = ["00000000-0000-4000-8000-000000000001"] + consul_path.write_text(json.dumps(consul), encoding="utf-8") + twin = dict(consul, id="11111111-1111-4111-8111-1111" + consul["id"][-8:]) + twin["friends"] = [] + (chars / "twin.json").write_text(json.dumps(twin), encoding="utf-8") + + out = tmp_path / "bundle" + code, err = run(src, out) + assert code == 0 + assert "unresolved link" in err + twins = sorted(p.name for p in (out / "character").glob("the-consul--*.md")) + assert twins == sorted(f"the-consul--{i.replace('-', '')}.md" for i in (consul["id"], twin["id"])) + docs = [split((out / "character" / n).read_text(encoding="utf-8"))[0] for n in twins] + assert docs[0]["description"] == "line one\n---\nline: three" + assert docs[0]["title"] == "The [Consul]" + index = (out / "character" / "index.md").read_text(encoding="utf-8") + assert "* [The \\[Consul\\]](" in index + dangling = next(d for d in docs if "friends" in d) + assert dangling["friends"] == ["urn:uuid:00000000-0000-4000-8000-000000000001"] + + +# --- Kael's review of the branch (h29 #6371), one test per finding + + +class World: + def __init__(self, root: Path) -> None: + self.src = copy_world(root / "world") + self.out = root / "bundle" + + def edit(self, rel_glob: str, change: Callable[[dict[str, Any]], Any]) -> dict[str, Any]: + path = next(self.src.glob(rel_glob)) + body: dict[str, Any] = json.loads(path.read_text(encoding="utf-8")) + change(body) + path.write_text(json.dumps(body), encoding="utf-8") + return body + + def doc(self, rel_glob: str) -> tuple[dict[str, Any], str]: + return split(next(self.out.glob(rel_glob)).read_text(encoding="utf-8")) + + def run(self) -> tuple[int, str]: + return run(self.src, self.out) + + +@pytest.fixture +def world(tmp_path: Path) -> Iterator[World]: + yield World(tmp_path) + + +def test_2_2_every_line_break_is_quoted(world: World) -> None: + """U+2028, U+2029, NEL and the other breaks str.splitlines() knows.""" + text = "one\u2028---\u2028two\u2029---\x85---\x0b---\x0c---\x1c---\x1d---\x1e---" + world.edit("elements/character/the-consul--*.json", lambda b: b.update(description=text, name="A ---")) + code, err = world.run() + assert code == 0, err + fm, _ = world.doc("character/a--*.md") + assert (fm["description"], fm["title"]) == (text, "A ---") + if OKFDocument: + doc = OKFDocument.parse(next(world.out.glob("character/a--*.md")).read_text(encoding="utf-8")) + assert doc.frontmatter["description"] == text + + +def test_2_3_unreadable_files_are_noted_and_skipped(world: World) -> None: + chars = world.src / "elements" / "character" + (chars / ".atlas-cache.json").write_text("[]", encoding="utf-8") + (chars / "no-id.json").write_text('{"name": "Nobody"}', encoding="utf-8") + (chars / "broken.json").write_text("{not json", encoding="utf-8") + (chars / "list.json").write_text("[1, 2]", encoding="utf-8") + code, err = world.run() + assert code == 0, err + for name in (".atlas-cache.json", "no-id.json", "broken.json", "list.json"): + assert f"elements/character/{name}" in err + assert len(list((world.out / "character").glob("*--*.md"))) == 3 + + +def test_2_4_world_keys_carried(world: World) -> None: + world_path = world.src / "world.json" + data = json.loads(world_path.read_text(encoding="utf-8")) + del data["time_current"] + consul = next((world.src / "elements" / "character").glob("the-consul--*.json")) + consul_id = json.loads(consul.read_text(encoding="utf-8"))["id"] + data.update( + time_range_current=7, + x_calendar={"months": 12}, + atlas_style="dark", + owner_character=consul_id, + snapshot_label="ch05", + snapshot_counts={"character": 3}, + writable=False, + api={"world_id": "w", "api_key": "secret"}, + ) + world_path.write_text(json.dumps(data), encoding="utf-8") + code, err = world.run() + assert code == 0, err + fm, _ = split((world.out / "world.md").read_text(encoding="utf-8")) + assert fm["time_current"] == 7 + assert "time_range_current" not in fm + for key in ("x_calendar", "atlas_style", "snapshot_label", "snapshot_counts", "writable", "format_version"): + assert fm[key] == data[key], key + assert fm["owner_character"] == "/character/" + consul.stem + ".md" + assert "api" not in fm + assert "secret" not in (world.out / "world.md").read_text(encoding="utf-8") + + +def test_2_4_time_current_preferred(world: World) -> None: + world_path = world.src / "world.json" + data = json.loads(world_path.read_text(encoding="utf-8")) + data.update(time_current=3, time_range_current=9) + world_path.write_text(json.dumps(data), encoding="utf-8") + assert world.run()[0] == 0 + assert split((world.out / "world.md").read_text(encoding="utf-8"))[0]["time_current"] == 3 + + +def test_2_5_undeclared_keys_are_noted(world: World) -> None: + world.edit( + "elements/character/the-consul--*.json", + lambda b: b.update(birthplace_old="somewhere", local_updated_at="2026-01-01T00:00:00Z"), + ) + code, err = world.run() + assert code == 0, err + assert "`birthplace_old`" in err + assert "local_updated_at" not in err # a folder key: push.NOT_SENT + fm, _ = world.doc("character/the-consul--*.md") + assert "birthplace_old" not in fm + + +def test_2_5_unknown_kind_is_noted() -> None: + kinds = {**FIELD_KINDS, "character": {**FIELD_KINDS["character"], "aura": "float"}} + sections = {**SECTIONS, "character": (*SECTIONS["character"], ("Constitution", ("aura",)))} + notes: list[str] = [] + consul = next((HYPERION / "elements" / "character").glob("the-consul--*.json")) + body = dict(json.loads(consul.read_text(encoding="utf-8")), aura=0.5) + paths = okf.assign_paths([("character", body)]) + okf.element_doc( + "character", body, paths, {body["id"]: "x"}, okf.OkfReport(), notes.append, kinds=kinds, sections=sections + ) + assert any("aura" in n and "float" in n for n in notes), notes + + +def test_nit_wrong_type_and_null_links_are_noted_and_left_out(world: World) -> None: + chars = world.src / "elements" / "character" + steinway = json.loads(next(chars.glob("mr-steinway--*.json")).read_text(encoding="utf-8")) + world.edit( + "elements/character/the-consul--*.json", + lambda b: b.update(location=steinway["id"], objects=[None, *b["objects"]]), + ) + world.edit("elements/pin/*.json", lambda b: b.update(element_type="location")) + code, err = world.run() + assert code == 0, err + fm, body = world.doc("character/the-consul--*.md") + assert "location" not in fm + assert "**location**" not in body + assert len(fm["objects"]) == 2 + assert "None" not in body + json.dumps(fm) + pin, pin_body = world.doc("pin/*--*.md") + assert "element" not in pin + assert "**element**" not in pin_body + assert "`location`" in err + assert "`objects`" in err + assert "`element`" in err + + +def test_nit_text_cannot_make_a_section(world: World) -> None: + world.edit( + "elements/character/the-consul--*.json", + lambda b: b.update(background="# Fake section\n ## Fake too\nplain\n===\n---"), + ) + assert world.run()[0] == 0 + _, body = world.doc("character/the-consul--*.md") + assert "\\# Fake section\n \\## Fake too\nplain\n\\===\n\\---" in body + assert "\n# Fake" not in body + + +def test_nit_hyperion_run_is_quiet(world: World) -> None: + assert run(HYPERION, world.out) == (0, "") + + +def test_nit_only_stderr_redirected(world: World) -> None: + out = io.TextIOWrapper(io.BytesIO(), encoding="utf-8") + with redirect_stdout(out), redirect_stderr(io.StringIO()): + code = okf.main([str(HYPERION), str(world.out)]) + assert code == 0 + + +def test_nit_generated_by_is_explained() -> None: + assert "human:" in (okf.__doc__ or "") + + +# --- the package's conditions + + +def test_core_import_does_not_load_yaml() -> None: + """`import onlyworlds` stays dependency-free: PyYAML is only the okf extra's.""" + r = subprocess.run( + [sys.executable, "-c", "import sys, onlyworlds; assert 'yaml' not in sys.modules, 'yaml imported'"], + capture_output=True, + text=True, + ) + assert r.returncode == 0, r.stderr + + +def test_known_keys_cover_the_dist_lint() -> None: + """The server and folder keys the exporter passes in silence (push.NOT_SENT) include every + key the pinned dist's folder lint passes, and the extension prefixes are the lint's.""" + spec = importlib.util.spec_from_file_location( + "lint_folder", ROOT / "codegen" / "schema-dist" / "walk" / "lint_folder.py" + ) + assert spec is not None and spec.loader is not None + lint = importlib.util.module_from_spec(spec) + spec.loader.exec_module(lint) + assert set(lint.BODY_EXTRA_KEYS) | set(lint.FOLDER_KEYS) <= NOT_SENT + assert tuple(lint.EXTENSION_PREFIXES) == EXTENSION_PREFIXES diff --git a/uv.lock b/uv.lock index fd9d7f3..7a4e4de 100644 --- a/uv.lock +++ b/uv.lock @@ -256,6 +256,11 @@ wheels = [ name = "onlyworlds" source = { editable = "." } +[package.optional-dependencies] +okf = [ + { name = "pyyaml" }, +] + [package.dev-dependencies] dev = [ { name = "mypy" }, @@ -266,6 +271,8 @@ dev = [ ] [package.metadata] +requires-dist = [{ name = "pyyaml", marker = "extra == 'okf'", specifier = ">=6" }] +provides-extras = ["okf"] [package.metadata.requires-dev] dev = [ From b125cb252c9ed213922fac7d2698aea52ae59aec Mon Sep 17 00:00:00 2001 From: Titus Date: Sat, 17 Oct 2026 02:06:13 +0200 Subject: [PATCH 2/5] okf: Skeld's rulings (h29 #6392): the bundle names the package (onlyworlds-py/); dotfiles and a null time_current are the reader's now (0.5.1), so the exporter's own dotfile handling goes Co-Authored-By: Claude Opus 5.5 --- src/onlyworlds/okf.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/src/onlyworlds/okf.py b/src/onlyworlds/okf.py index ad77a88..ff00365 100644 --- a/src/onlyworlds/okf.py +++ b/src/onlyworlds/okf.py @@ -60,8 +60,8 @@ Nothing is dropped in silence. Each of these is a note (on stderr from the command line, in ``OkfReport.notes`` from Python): a file the folder reader -skipped (not JSON, not an object, no `id`), a dot-prefixed file (tool-private: -folder format §4.1 says skip it, leave it, log it), a directory that is not a +skipped (not JSON, not an object, no `id`, a dot-prefixed file: a tool's litter, +not an element, as the folder spec clarifies), a directory that is not a type directory, an id in both `elements/` and the legacy `spatial/` (the `elements/` copy is served, folder format v0.3.8), a key that is not a field of its type, a field kind this exporter does not know, and a link left out @@ -110,12 +110,13 @@ SECTIONS, WORLD_FIELDS, ) +from ._version import __version__ from .folder import WIRE_TO_DISK_WORLD_KEYS, WORLD_FILE, Folder, read_folder from .push import NOT_SENT __all__ = ["EXPORTER", "OKF_COMMIT", "OKF_VERSION", "FolderError", "OkfReport", "build_bundle", "export_okf", "main"] -EXPORTER = "onlyworlds-okf-export/0.1" +EXPORTER = f"onlyworlds-py/{__version__}" OKF_VERSION = "0.2" OKF_REPO = "https://github.com/GoogleCloudPlatform/open-knowledge-format" OKF_COMMIT = "ad30107c31c06aec8a7d5636e0d1058118604e6f" @@ -196,18 +197,10 @@ def _elements(folder: Folder, note: Note) -> list[tuple[str, dict[str, Any]]]: for s in folder.skipped: if s.path.parent == folder.root and s.path.name == WORLD_FILE: continue - rel = _rel(folder, s.path) - note( - f"{rel}: dot-prefixed, tool-private; skipped" - if s.path.name.startswith(".") - else f"{rel}: {s.reason}; skipped" - ) + note(f"{_rel(folder, s.path)}: {s.reason}; skipped") for d in folder.ignored_dirs: note(f"{_rel(folder, d)}/: not a type directory; skipped") for dup in folder.duplicates: - if dup.ignored.name.startswith("."): - note(f"{_rel(folder, dup.ignored)}: dot-prefixed, tool-private; skipped") - continue raise FolderError(f"{_rel(folder, dup.ignored)}: duplicate id {dup.id} (also {_rel(folder, dup.kept)})") for eid in folder.in_both_layouts: legacy, served = folder.legacy[eid].path, folder.elements[eid].path @@ -221,9 +214,7 @@ def _elements(folder: Folder, note: Note) -> list[tuple[str, dict[str, Any]]]: out: list[tuple[str, dict[str, Any]]] = [] for el in found: rel = _rel(folder, el.path) - if el.path.name.startswith("."): - note(f"{rel}: dot-prefixed, tool-private; skipped") - elif el.type not in ELEMENT_TYPES: + if el.type not in ELEMENT_TYPES: note(f"{rel}: `type` {el.type!r} is not one of the 22 types; skipped") else: out.append((el.type, el.body)) From 9c8648bb0584fae0e948bc70b44b84e6d0665028 Mon Sep 17 00:00:00 2001 From: Titus Date: Sat, 17 Oct 2026 10:55:18 +0200 Subject: [PATCH 3/5] okf: a text line that already begins with a backslash gets one more, so `# x` and `\# x` no longer export the same (found by reading a bundle back, OKF sweep #360); a test watched failing with the old pattern Co-Authored-By: Claude Opus 5.5 --- src/onlyworlds/okf.py | 8 +++++--- tests/test_okf.py | 13 ++++++++++++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/onlyworlds/okf.py b/src/onlyworlds/okf.py index ff00365..ea58ffc 100644 --- a/src/onlyworlds/okf.py +++ b/src/onlyworlds/okf.py @@ -45,7 +45,9 @@ viewers that draw the link graph from bodies draw it. Body links are relative: the reference viewer at the pinned commit skips `/` links. A text line markdown would read as a heading (`#`, or a `===`/`---` - underline) is escaped with a backslash, so text cannot make a section. + underline) is escaped with a backslash, so text cannot make a section; + so is a line that already begins with a backslash, so a reader removes + one backslash from each such line and gets the text back. - unset values (the empty string, null, an empty list) are left out. Extension keys (`x_`, `atlas_`, `shadow_`) are carried as they are; the server and folder keys the package never pushes (``push.NOT_SENT``) are @@ -264,13 +266,13 @@ def _heading(name: str) -> str: return words[:1].upper() + words[1:] -_FAKE_HEADING = re.compile(r" {0,3}#| {0,3}(=+|-+)[ \t]*$") +_FAKE_HEADING = re.compile(r" {0,3}[#\\]| {0,3}(=+|-+)[ \t]*$") def escape_text(text: str) -> str: """A text field as body markdown: LF, outer blank lines trimmed, and a backslash before any line markdown would read as a heading (`# x`, or a `===`/`---` line that would underline the - line above).""" + line above) or that already begins with one (else `# x` and `\\# x` would read the same).""" out = [] for line in str(text).replace("\r\n", "\n").replace("\r", "\n").strip("\n").split("\n"): if _FAKE_HEADING.match(line): diff --git a/tests/test_okf.py b/tests/test_okf.py index fc46332..f92a7c5 100644 --- a/tests/test_okf.py +++ b/tests/test_okf.py @@ -89,7 +89,7 @@ def escape_text(text: str) -> str: """The body's text rule: a line markdown would read as a heading is escaped.""" out = [] for line in text.replace("\r\n", "\n").replace("\r", "\n").strip("\n").split("\n"): - if re.match(r" {0,3}#", line) or re.fullmatch(r" {0,3}(=+|-+)[ \t]*", line): + if re.match(r" {0,3}[#\\]", line) or re.fullmatch(r" {0,3}(=+|-+)[ \t]*", line): i = len(line) - len(line.lstrip(" ")) line = line[:i] + "\\" + line[i:] out.append(line) @@ -509,6 +509,17 @@ def test_nit_text_cannot_make_a_section(world: World) -> None: assert "\n# Fake" not in body +def test_nit_an_escaped_line_reads_back(world: World) -> None: + """A line that already begins with a backslash gets one more, so `# x` and `\\# x` stay apart.""" + world.edit( + "elements/character/the-consul--*.json", + lambda b: b.update(background="# x\n\\# x\n \\ y\n \\ z"), + ) + assert world.run()[0] == 0 + _, body = world.doc("character/the-consul--*.md") + assert "\\# x\n\\\\# x\n \\\\ y\n \\ z" in body + + def test_nit_hyperion_run_is_quiet(world: World) -> None: assert run(HYPERION, world.out) == (0, "") From 201891d2f74bf6309e30dd25bfe84085d89bba74 Mon Sep 17 00:00:00 2001 From: Titus Date: Sat, 17 Oct 2026 11:35:51 +0200 Subject: [PATCH 4/5] =?UTF-8?q?okf:=20no=20generated=20actor=20(OKF=20?= =?UTF-8?q?=C2=A77=20asks=20human:=20for=20hand-written=20words;=20a=20fol?= =?UTF-8?q?der=20does=20not=20say=20who=20wrote=20them),=20in-prose=20ow:/?= =?UTF-8?q?/=20references=20become=20links=20into=20the=20bundle=20(an=20a?= =?UTF-8?q?bsent=20id=20stays=20and=20is=20listed=20as=20unresolved);=20CH?= =?UTF-8?q?ANGELOG=20and=20README=20say=20how=20the=20port=20now=20differs?= =?UTF-8?q?=20from=20the=20branch=20(Skeld's=20gap=20read=20for=20#360,=20?= =?UTF-8?q?h58=20#6516).=20Four=20mutants,=20each=20failing=20a=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- README.md | 2 +- src/onlyworlds/okf.py | 62 ++++++++++++++++++++++--------------------- tests/test_okf.py | 39 ++++++++++++++++++++++++--- 4 files changed, 70 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85e3bbb..b9bc1cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ The package version runs on its own semver; the schema it is generated from is ` ## 0.6.0 (unreleased) -- **Export a world folder to Open Knowledge Format (OKF) v0.2** (`onlyworlds.okf`, Keel #191): `export_okf(folder, out_dir)` or `python -m onlyworlds.okf FOLDER OUT-DIR` writes one markdown concept per element (fields in the frontmatter, text under headings, links as bundle paths and as markdown links), the World as `world.md`, and an index per type. The same folder gives the same bytes. Nothing is left out in silence: a file the reader skipped, a key that is not a field of its type, or a link that is not an id or points at the wrong type is a note in the returned `OkfReport` (stderr from the command line). Ported from schema-dist's `walk/okf_export.py`, it reads through `read_folder` and the generated schema tables, and gives the same bundle on the example world. +- **Export a world folder to Open Knowledge Format (OKF) v0.2** (`onlyworlds.okf`, Keel #191): `export_okf(folder, out_dir)` or `python -m onlyworlds.okf FOLDER OUT-DIR` writes one markdown concept per element (fields in the frontmatter, text under headings, links as bundle paths and as markdown links), the World as `world.md`, and an index per type. The same folder gives the same bytes. Nothing is left out in silence: a file the reader skipped, a key that is not a field of its type, or a link that is not an id or points at the wrong type is a note in the returned `OkfReport` (stderr from the command line). Ported from schema-dist's `walk/okf_export.py`, it reads through `read_folder` and the generated schema tables. It differs from the branch on purpose: no `generated` actor (OKF §7 asks for a `human:` actor on hand-written words, and a folder does not say who wrote them), in-prose `ow://` references become links into the bundle, and a text line that already begins with a backslash gets one more, so every escaped line reads back. - **PyYAML only for OKF**: the export needs the new `okf` extra (`pip install "onlyworlds[okf]"`). `import onlyworlds` still imports nothing outside the standard library. - **The schema module grows** (same pin, `v0.32.0-dist.23`): `BASE_FIELDS`, `WORLD_FIELDS`, `SECTIONS` (each type's fields under their schema groups, in order), `LINK_TARGETS`, `GENERIC_LINKS` and `EXTENSION_PREFIXES`, generated from the vendored walk like the rest and checked against `FIELD_KINDS` and `GENERIC_PAIRS` at generation. diff --git a/README.md b/README.md index fe1ea59..06d181d 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ result = push("moppetopia-before", "moppetopia", api_key="ow_w_…", pin="ow_s_ ## Export to OKF -A world folder can be written as a bundle in [Open Knowledge Format](https://github.com/GoogleCloudPlatform/open-knowledge-format) (OKF) v0.2: one markdown file per element, with its fields in the frontmatter, its text under headings and its links as markdown links, plus an index per type. This needs PyYAML, which the rest of the package does not: +A world folder can be written as a bundle in [Open Knowledge Format](https://github.com/GoogleCloudPlatform/open-knowledge-format) (OKF) v0.2: one markdown file per element, with its fields in the frontmatter, its text under headings and its links (in-prose `ow://` references included) as markdown links, plus an index per type. This needs PyYAML, which the rest of the package does not: ```bash pip install "onlyworlds[okf]" diff --git a/src/onlyworlds/okf.py b/src/onlyworlds/okf.py index ea58ffc..2cdb5f4 100644 --- a/src/onlyworlds/okf.py +++ b/src/onlyworlds/okf.py @@ -24,12 +24,12 @@ theirs (`BigQuery Table`): `OnlyWorlds Character`. `title` is the element's name, `description` its description. Never `verified`: an export confirms nothing. - - `generated.by` is this exporter, a machine actor, and `generated.at` the - element's `updated_at`. OKF §7 asks for a `human:` actor on hand-authored - content; the words are the world's authors', but the bundle names no - author (the folder does not say who wrote what) and these files are - generated, so the machine actor is the true one. Consumers therefore read - the bundle as unverified (§5.3). + - no `generated` either. OKF §7 says producers MUST use a `human:` actor + for hand-authored content, and the words are the world's authors'; but + the folder does not say who wrote what, so a `human:` actor would be + invented and a machine actor would be false. With no trust frontmatter a + concept is unverified and still conformant (§5.3, §11). The exporter is + named once, in world.md's `onlyworlds_export` block. - scalar fields sit in the frontmatter under their OnlyWorlds names: `id`, `supertype`, `subtype`, `image_url` and every integer field. A field whose name OKF defines (§4.1, §5, §10: `status` and the like) would be @@ -51,7 +51,12 @@ - unset values (the empty string, null, an empty list) are left out. Extension keys (`x_`, `atlas_`, `shadow_`) are carried as they are; the server and folder keys the package never pushes (``push.NOT_SENT``) are - not, except `updated_at` as `generated.at`. + not, `updated_at` included. + - an in-prose element reference in a text field, `[Label](ow:///)` + (folder format, in-prose references), becomes a relative link to that + element's concept, so a markdown viewer can follow it; one to an id the + folder does not hold stays as written and is listed in + ``OkfReport.unresolved``. world.md carries the World's schema fields (`time_current` read from `time_range_current` when only that spelling is present, folder format §5), @@ -91,7 +96,6 @@ import unicodedata from collections.abc import Callable, Mapping from dataclasses import dataclass, field -from datetime import datetime from pathlib import Path from typing import Any @@ -282,17 +286,6 @@ def escape_text(text: str) -> str: return "\n".join(out) -def _timestamp(value: Any) -> str | None: - """An ISO 8601 datetime with an explicit offset (SPEC §5), else None.""" - if not isinstance(value, str): - return None - try: - parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) - except ValueError: - return None - return value if parsed.tzinfo is not None else None - - def _unset(value: Any) -> bool: return value is None or value == "" or value == [] @@ -301,12 +294,8 @@ def _fm_key(name: str) -> str: return f"ow_{name}" if name in OKF_KEYS else name -def _generated(body: Mapping[str, Any]) -> dict[str, str]: - gen = {"by": EXPORTER} - at = _timestamp(body.get("updated_at")) - if at: - gen["at"] = at - return gen +# An in-prose reference as a markdown link target (atlas `src/core/ow-links.ts`, OW_LINK_HREF_RE). +_OW_LINK = re.compile(r"\]\(ow://([A-Za-z_]+)/([\w-]+)\)") class _Links: @@ -329,8 +318,23 @@ def one(self, name: str, eid: Any, target: str | None) -> tuple[str, str] | None self.note(f"{self.here}: `{name}` -> a {actual}, expected {target}; left out") return None self.report.links += 1 - rel = "../" + self.paths[eid] if "/" in self.here else self.paths[eid] - return "/" + self.paths[eid], f"[{_link_text(self.names[eid])}]({rel})" + return "/" + self.paths[eid], f"[{_link_text(self.names[eid])}]({self._rel(eid)})" + + def _rel(self, eid: str) -> str: + return "../" + self.paths[eid] if "/" in self.here else self.paths[eid] + + def prose(self, name: str, text: str) -> str: + """A text field's `](ow:///)` link targets as relative bundle paths; one to an + id the folder does not hold stays as written and is listed as unresolved.""" + + def swap(m: re.Match[str]) -> str: + eid = m.group(2) + if eid not in self.paths: + self.report.unresolved.append(f"{self.here}: `{name}` (prose) -> {eid}") + return m.group(0) + return f"]({self._rel(eid)})" + + return _OW_LINK.sub(swap, text) # --- one element @@ -355,7 +359,6 @@ def element_doc( fm["title"] = str(body["name"]) if not _unset(body.get("description")): fm["description"] = str(body["description"]) - fm["generated"] = _generated(body) known = set(NOT_SENT) for name in BASE_FIELDS: known.add(name) @@ -403,7 +406,7 @@ def add_links(section: str, name: str, found: list[tuple[str, str]], many: bool) kind = kinds[slug].get(name) if kind == "text": open_section(section) - blocks.append(("text", f"## {_heading(name)}\n\n{escape_text(str(value))}")) + blocks.append(("text", f"## {_heading(name)}\n\n{links.prose(name, escape_text(str(value)))}")) elif kind == "integer": fm[_fm_key(name)] = value elif kind == "single_link": @@ -454,7 +457,6 @@ def world_doc( fm["title"] = str(world["name"]) if not _unset(world.get("description")): fm["description"] = str(world["description"]) - fm["generated"] = _generated(world) known = {*WORLD_DROPPED, "owner_character"} for name in WORLD_FIELDS: known.add(name) diff --git a/tests/test_okf.py b/tests/test_okf.py index f92a7c5..47d0fd7 100644 --- a/tests/test_okf.py +++ b/tests/test_okf.py @@ -175,7 +175,7 @@ def test_type_title_description(bundle: Bundle) -> None: assert fm["type"] == "OnlyWorlds " + src["type"].capitalize(), rel assert fm["title"] == src["name"], rel assert fm.get("description", "") == (src.get("description") or ""), rel - assert fm["generated"] == {"by": okf.EXPORTER, "at": src["updated_at"]} + assert "generated" not in fm, rel # no actor: the folder does not say who wrote what (OKF §7) assert "verified" not in fm, rel @@ -199,10 +199,15 @@ def test_scalars_and_extensions_match_source(bundle: Bundle) -> None: def test_text_fields_match_source(bundle: Bundle) -> None: checked = 0 + path_of = {fm["id"]: rel for rel, (fm, _) in bundle.docs.items() if "id" in fm} + + def prose(text: str) -> str: # an in-prose ow:// target becomes the relative path of its concept + return re.sub(r"\]\(ow://[A-Za-z_]+/([\w-]+)\)", lambda m: f"](../{path_of[m.group(1)]})", text) + for rel, (fm, body) in bundle.docs.items(): src = bundle.sources[fm["id"]] expected = { - heading(name): escape_text(src[name]) + heading(name): prose(escape_text(src[name])) for name in type_fields(src["type"]) if FIELD_KINDS[src["type"]].get(name) == "text" and src.get(name) } @@ -531,10 +536,38 @@ def test_nit_only_stderr_redirected(world: World) -> None: assert code == 0 -def test_nit_generated_by_is_explained() -> None: +def test_in_prose_ow_links_point_into_the_bundle(bundle: Bundle) -> None: + """Hyperion's one-sentence narrative links three elements in its story as ow://: each becomes a + relative link to that element's concept, and no ow:// target is left in any body.""" + rel, (_, body) = next((r, d) for r, d in bundle.docs.items() if r.startswith("narrative/a-world-in-a-sentence--")) + targets = re.findall(r"\]\(([^)]+)\)", body) + for t in ("../title/", "../institution/", "../character/"): + hit = [x for x in targets if x.startswith(t)] + assert hit and all((bundle.out / "narrative" / x).resolve().is_file() for x in hit), (rel, t) + assert not any("](ow://" in b for _, b in bundle.docs.values()) + + +def test_nit_an_ow_link_to_an_absent_id_stays_and_is_listed(world: World) -> None: + ghost = "00000000-0000-7000-8000-000000000000" + world.edit( + "elements/character/the-consul--*.json", + lambda b: b.update(background=f"See [nobody](ow://character/{ghost})."), + ) + assert world.run()[0] == 0 + _, body = world.doc("character/the-consul--*.md") + assert f"[nobody](ow://character/{ghost})" in body + + +def test_nit_no_actor_is_explained() -> None: assert "human:" in (okf.__doc__ or "") +def test_world_names_no_actor_and_the_exporter_once(bundle: Bundle) -> None: + fm, _ = split(bundle.files["world.md"].decode("utf-8")) + assert "generated" not in fm + assert fm["onlyworlds_export"]["exporter"] == okf.EXPORTER + + # --- the package's conditions From a315d60edf928ee37f660f601603cb953fd8947d Mon Sep 17 00:00:00 2001 From: Titus Date: Sat, 17 Oct 2026 12:37:10 +0200 Subject: [PATCH 5/5] 0.6.0: the OKF exporter (onlyworlds.okf, Keel #191, the #360 sweep's door); a test fails when a schema bump puts a frontmatter field on a key OKF defines (watched failing with supertype added to the set); the README's line is the sweep's: a valid OKF 0.2 bundle at one pinned commit Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- README.md | 4 ++-- src/onlyworlds/_version.py | 2 +- tests/test_okf.py | 21 +++++++++++++++++++++ 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9bc1cf..bd93e03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ The package version runs on its own semver; the schema it is generated from is `SCHEMA_DIST_TAG`. -## 0.6.0 (unreleased) +## 0.6.0 (2026-10-17) - **Export a world folder to Open Knowledge Format (OKF) v0.2** (`onlyworlds.okf`, Keel #191): `export_okf(folder, out_dir)` or `python -m onlyworlds.okf FOLDER OUT-DIR` writes one markdown concept per element (fields in the frontmatter, text under headings, links as bundle paths and as markdown links), the World as `world.md`, and an index per type. The same folder gives the same bytes. Nothing is left out in silence: a file the reader skipped, a key that is not a field of its type, or a link that is not an id or points at the wrong type is a note in the returned `OkfReport` (stderr from the command line). Ported from schema-dist's `walk/okf_export.py`, it reads through `read_folder` and the generated schema tables. It differs from the branch on purpose: no `generated` actor (OKF §7 asks for a `human:` actor on hand-written words, and a folder does not say who wrote them), in-prose `ow://` references become links into the bundle, and a text line that already begins with a backslash gets one more, so every escaped line reads back. - **PyYAML only for OKF**: the export needs the new `okf` extra (`pip install "onlyworlds[okf]"`). `import onlyworlds` still imports nothing outside the standard library. diff --git a/README.md b/README.md index 06d181d..6aa269b 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ result = push("moppetopia-before", "moppetopia", api_key="ow_w_…", pin="ow_s_ ## Export to OKF -A world folder can be written as a bundle in [Open Knowledge Format](https://github.com/GoogleCloudPlatform/open-knowledge-format) (OKF) v0.2: one markdown file per element, with its fields in the frontmatter, its text under headings and its links (in-prose `ow://` references included) as markdown links, plus an index per type. This needs PyYAML, which the rest of the package does not: +A world folder can be exported as a valid [Open Knowledge Format](https://github.com/GoogleCloudPlatform/open-knowledge-format) (OKF) 0.2 bundle, as the spec stands at one pinned commit (`onlyworlds.okf.OKF_COMMIT`): one markdown file per element, with its fields in the frontmatter, its text under headings and its links (in-prose `ow://` references included) as markdown links, plus an index per type. This needs PyYAML, which the rest of the package does not: ```bash pip install "onlyworlds[okf]" @@ -87,7 +87,7 @@ report = export_okf("moppetopia", "moppetopia-okf") print(report.elements, "elements,", len(report.notes), "notes") ``` -The output directory must not exist yet. Anything the export leaves out (a file that isn't an element, a key that isn't a field, a link to the wrong type) is listed in `report.notes`; the command prints these on stderr. The same folder always gives the same bytes. +The output directory must not exist yet. Anything the export leaves out (a file that isn't an element, a key that isn't a field, a link to the wrong type) is listed in `report.notes`; the command prints these on stderr. The same folder always gives the same bytes. What a bundle never carries: the server's bookkeeping (`change_seq`, `created_at`, who wrote what) and the world's key, so a bundle is a copy to read elsewhere, not a folder to sync. ## Errors diff --git a/src/onlyworlds/_version.py b/src/onlyworlds/_version.py index dd9b22c..906d362 100644 --- a/src/onlyworlds/_version.py +++ b/src/onlyworlds/_version.py @@ -1 +1 @@ -__version__ = "0.5.1" +__version__ = "0.6.0" diff --git a/tests/test_okf.py b/tests/test_okf.py index 47d0fd7..769dbf9 100644 --- a/tests/test_okf.py +++ b/tests/test_okf.py @@ -28,11 +28,13 @@ from onlyworlds import okf # noqa: E402 from onlyworlds._schema import ( # noqa: E402 + BASE_FIELDS, EXTENSION_PREFIXES, FIELD_KINDS, GENERIC_LINKS, SCHEMA_VERSION, SECTIONS, + WORLD_FIELDS, ) from onlyworlds.push import NOT_SENT # noqa: E402 @@ -558,6 +560,25 @@ def test_nit_an_ow_link_to_an_absent_id_stays_and_is_listed(world: World) -> Non assert f"[nobody](ow://character/{ghost})" in body +# Frontmatter keys an OnlyWorlds field takes as `ow_`, because OKF gives that name a meaning. +# None today; a schema bump that adds one fails the test below until it is added here and the +# CHANGELOG says how that field is written (#360: decided, not found by a reader). +OKF_RENAMED: frozenset[str] = frozenset() + + +def test_no_frontmatter_field_takes_an_okf_key_unannounced() -> None: + """Every field the exporter writes into frontmatter (not text: that goes in the body) against + the keys OKF defines.""" + frontmatter = ( + {n for kinds in FIELD_KINDS.values() for n, kind in kinds.items() if kind != "text"} + | (set(BASE_FIELDS) - {"name", "description", "world"}) + | (set(WORLD_FIELDS) - {"name", "description"}) + | {n for links in GENERIC_LINKS.values() for n in links} + ) + assert len(frontmatter) > 100, "the schema tables read as empty" + assert frontmatter & okf.OKF_KEYS == OKF_RENAMED + + def test_nit_no_actor_is_explained() -> None: assert "human:" in (okf.__doc__ or "")