From 857864123a706470cbaa0954f44abec8c82a3eed Mon Sep 17 00:00:00 2001 From: Tom Main Date: Mon, 12 Oct 2026 15:40:42 -0400 Subject: [PATCH 1/3] Freeze real table holdouts and preserve selected-page asset identity --- .github/workflows/ci.yml | 1 + CHANGELOG.md | 7 + README.md | 6 +- benchmark/eval/README.md | 6 + benchmark/eval/table_topology_real/README.md | 70 +++ benchmark/eval/table_topology_real/cases.json | 132 ++++ .../eval/table_topology_real/freeze.json | 28 + benchmark/eval/table_topology_real/prepare.py | 77 +++ .../eval/table_topology_real/sources.json | 35 ++ .../eval/table_topology_real/test_prepare.py | 89 +++ .../eval/table_topology_real/test_verify.py | 575 ++++++++++++++++++ benchmark/eval/table_topology_real/verify.py | 504 +++++++++++++++ docs/output-schema.md | 9 +- integrations/javascript/integration.test.mjs | 4 +- .../javascript/vector-routing.test.mjs | 4 +- src/integration_test.zig | 18 +- src/main.zig | 4 +- src/specialist_protocol.zig | 2 +- src/visual_assets.zig | 107 +++- 19 files changed, 1657 insertions(+), 21 deletions(-) create mode 100644 benchmark/eval/table_topology_real/README.md create mode 100644 benchmark/eval/table_topology_real/cases.json create mode 100644 benchmark/eval/table_topology_real/freeze.json create mode 100644 benchmark/eval/table_topology_real/prepare.py create mode 100644 benchmark/eval/table_topology_real/sources.json create mode 100644 benchmark/eval/table_topology_real/test_prepare.py create mode 100644 benchmark/eval/table_topology_real/test_verify.py create mode 100644 benchmark/eval/table_topology_real/verify.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1528011..cc54859 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,6 +140,7 @@ jobs: python3 -m unittest discover -s benchmark/eval -p 'test_table_compare.py' python3 -m unittest discover -s benchmark/eval -p 'test_bounded_zig.py' python3 -m unittest discover -s benchmark/eval/table_topology -p 'test_*.py' + python3 -m unittest discover -s benchmark/eval/table_topology_real -p 'test_*.py' python3 -m unittest discover -s benchmark/eval/reading_order_next -p 'test_*.py' python3 benchmark/eval/table_topology/generate.py --check python3 benchmark/eval/reading_order_next/generate.py --check diff --git a/CHANGELOG.md b/CHANGELOG.md index d10679b..d6752cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ ### Added +- A source-bound real-document table pack freezes selected merged headers and + continuation controls across separate development and holdout issuers. Local + original PDFs stay outside Git; failed qualifications remain evidence. + - Frozen development and holdout suites check table topology, exact cell source ownership, and emitted reading order independently of graph diagnostics. - A project-local build monitor bounds observed compiler-cache allocation and @@ -20,6 +24,9 @@ ### Fixed +- Schema 0.19.2 keeps batch debug assets bound to original document pages when + extraction selects a window starting after page one. + - Schema 0.19.1 keeps reconstructed table evidence at cell granularity, assigns shared-rule text once, and retains prose surrounding a promoted table. - Legacy native ordering resets columns at corroborated section headings. diff --git a/README.md b/README.md index 4d754c1..d7bc071 100644 --- a/README.md +++ b/README.md @@ -670,11 +670,13 @@ and ruling lines, invokes OCR only for scanned routes, then reconciles native, OCR, table, formula, and form spans with typed provenance. The versioned JSON, artifact JSONL, and streaming JSONL schema is currently -`0.19.1` in local development, with parser build identity `0.4.0-dev` after the +`0.19.2` in local development, with parser build identity `0.4.0-dev` after the `v0.3.0` release. Schema `0.17.0` landed in [PR #6](https://github.com/Tom-R-Main/pdf-parser/pull/6) at `9cac608`, with all six CI checks green. Schemas `0.18.0` and `0.19.0` landed in [PR #7](https://github.com/Tom-R-Main/pdf-parser/pull/7) at `478d73a`, with all -six PR CI checks green. Schema `0.19.1` is a local working-tree continuation. +six PR CI checks green. Schema `0.19.1` landed in +[PR #8](https://github.com/Tom-R-Main/pdf-parser/pull/8) at `1483f41`, with all +six PR CI checks green. Schema `0.19.2` is a local working-tree continuation. The `0.19` vector-routing continuation's 23 outlined-text and negative-control fixtures, 13 real CLI/SDK tests and both JSONL readers are qualified in the [implementation evidence](docs/parser-host-implementation-plan.md#bounded-vector-routing-019-locally-checked). diff --git a/benchmark/eval/README.md b/benchmark/eval/README.md index c041aca..7852bf8 100644 --- a/benchmark/eval/README.md +++ b/benchmark/eval/README.md @@ -1,5 +1,11 @@ # Evaluation Harness +The [real-document table pack](table_topology_real/README.md) freezes selected +merged headers and positive/negative continuation checks from three complete +government originals. It preserves failed qualifications and uses local, +hash-verified inputs without CI downloads. Selected-cell checks are not a +complete-table topology accuracy measure. + New frozen [table topology controls](table_topology/README.md) and [reading-order holdouts](reading_order_next/README.md) assess source ownership, physical cell spans and emitted order separately. The earlier reading-order diff --git a/benchmark/eval/table_topology_real/README.md b/benchmark/eval/table_topology_real/README.md new file mode 100644 index 0000000..2e331a4 --- /dev/null +++ b/benchmark/eval/table_topology_real/README.md @@ -0,0 +1,70 @@ +# Real-document table regions and continuation + +This pack checks selected rows, literal merged header cells, and continuation +links in three complete government publications. It complements the frozen +synthetic topology pack; it does not measure complete-table topology accuracy. + +The development source is an NCUA financial report. Treasury cash/debt tables +and Census construction tables are holdouts. Issuers and template families are +disjoint across the split. Source cards bind complete original PDFs by SHA256, +physical page count, source URL, and the reference checkout commit. Rights +metadata is retained from those cards; no agency endorses this evaluation. + +Independent Poppler text, word geometry, and source renders established truth +before parser assessment. Rendered rules establish Census header spans of two +and three columns, with a two-row stub. The Treasury debt-limit table includes +an integrated caption, two vertically merged headers, and three opening-balance +periods. Approximate raw PDF regions select tables; they do not qualify exact +cell-edge geometry. `freeze.json` binds source cards, selected truth, selectors, +and the blind evidence inventory. + +Both public JSONL formats are assessed independently. Missing or ambiguous +tables fail. Negative continuation checks require both tables to exist; +positive links must be reciprocal with a shared logical identity and consecutive +part indices. Row checks preserve currency markers, zeros, dashes, parentheses, +and `NA`, with whitespace normalization only. Treasury prefix checks select the +deposit/issues panel without combining its entity with an adjacent withdrawal +or redemption panel. Header checks require literal placement and spans. Context +notes describe source units and measurement bases; they are not assertions of a +public unit field. + +Prepare existing local originals without downloading or overwriting mismatches: + +```sh +python3 benchmark/eval/table_topology_real/prepare.py \ + --source-root /path/to/originals +python3 -m unittest discover -s benchmark/eval/table_topology_real -p 'test_*.py' +``` + +The default prepared input directory is ignored +`benchmark/eval/raw_cache/table_topology_real`. Complete source files remain +outside Git. Assess development first, then the frozen holdouts: + +```sh +python3 benchmark/eval/table_topology_real/verify.py \ + --cli zig-out/bin/pdf-parser --split dev \ + --output benchmark/eval/outputs/real-table-dev +python3 benchmark/eval/table_topology_real/verify.py \ + --cli zig-out/bin/pdf-parser --split holdout \ + --output benchmark/eval/outputs/real-table-holdout +``` + +Output directories must be new. Each run preserves commands, raw output, +stderr, failures, source/binary/verifier hashes, and a report. A failed +qualification exits nonzero; a failed baseline remains useful evidence. +CI tests the scorer and frozen metadata without fetching the source PDFs. + +Only one explicit positive-continuation source/template is represented. The +Treasury Table IIIA/IIIB pair and the Census annualized/unadjusted pair are +negative controls despite repeated headers or amounts. Distinct NCUA statements +are negatives too. Do not infer cross-template positive-continuation accuracy. + +Default local Treasury renders failed because unembedded Helvetica substituted +incorrectly. Blind review used an isolated Fontconfig file and existing Arial +fonts, with its cache under the ignored evidence directory; failed renders were +preserved. NCUA footer rendering is excluded from truth. + +Retain observed failures. Do not tune selectors, truth, thresholds, or parser +behavior around these holdouts. Correct objective harness defects with +regressions and preserve the original report. Future parser changes need separate +development evidence and fresh holdouts before broader qualification claims. diff --git a/benchmark/eval/table_topology_real/cases.json b/benchmark/eval/table_topology_real/cases.json new file mode 100644 index 0000000..f6cef0f --- /dev/null +++ b/benchmark/eval/table_topology_real/cases.json @@ -0,0 +1,132 @@ +[ + { + "id": "ncua-statements-dev", "source_id": "e013", "split": "dev", "pages": [3, 5], + "tables": [ + { + "id": "balance", "page_index": 2, "bbox": [51, 139, 557, 650], + "context": "Balance sheets, dollars in thousands", + "rows": [ + {"tokens": "Fund Balance with Treasury $ 28,485 $ 26,245"}, + {"tokens": "Accounts Payable - Due to the NCUA Operating Fund $ - $ 19"}, + {"tokens": "TOTAL ASSETS $ 20,553,523 $ 20,483,729"} + ] + }, + { + "id": "net-cost", "page_index": 3, "bbox": [52, 272, 557, 658], + "context": "Net cost; July, year-to-date July 2023, year-to-date July 2022; thousands", + "rows": [ + {"tokens": "Reserve Expense (Reduction) - 20,002 7,506"}, + {"tokens": "AME Receivable Bad Debt Expense - (21,778) (51,864)"} + ] + }, + { + "id": "net-position", "page_index": 4, "bbox": [52, 207, 559, 645], + "context": "Changes in net position; July 2023 and July 2022; thousands", + "rows": [ + {"tokens": "Beginning Balances $ 3,258,998 $ 4,780,200"}, + {"tokens": "Beginning Balances 16,916,436 15,783,657"}, + {"tokens": "NET POSITION $ 20,346,828 $ 20,311,501", "occurrences": 2} + ] + } + ], + "relations": [ + {"from": "balance", "to": "net-cost", "continues": false}, + {"from": "net-cost", "to": "net-position", "continues": false} + ] + }, + { + "id": "treasury-continuation-holdout", "source_id": "e012", "split": "holdout", "pages": [1, 3], + "tables": [ + { + "id": "cash-1", "page_index": 0, "bbox": [36.8, 31.5, 581.1, 589.5], "column_count": 8, + "context": "Table II: independent deposit and withdrawal panels, millions", + "rows": [ + {"tokens": "Dept of Agriculture (USDA) - misc 30 273 1,904", "prefix": true}, + {"tokens": "USDA - Federal Crop Insurance Corp Fund 0 0 33", "prefix": true}, + {"tokens": "Legislative Branch - misc 0 3 40", "prefix": true} + ] + }, + { + "id": "cash-2", "page_index": 1, "bbox": [36.8, 341, 581.1, 735.7], "column_count": 8, + "context": "Table II - cont.; blank deposit detail region beside continuing withdrawals", + "rows": [ + {"tokens": "Legislative Branch - Library of Congress 0 38 146", "prefix": true}, + {"tokens": "Unclassified - Deposits 10 217 2,892", "prefix": true}, + {"tokens": "Total Deposits 8,170 408,982 2,621,282", "prefix": true}, + {"tokens": "Treasury General Account Total Deposits $ 224,427 $ 2,155,471 $ 17,629,416", "prefix": true} + ] + }, + { + "id": "debt-face-value", "page_index": 1, "bbox": [36.8, 131.5, 581.1, 332], "column_count": 8, + "context": "Table IIIA public debt transactions, face value, millions", + "rows": [ + {"tokens": "Government Account Series 570,559 7,051,129 67,726,222", "prefix": true}, + {"tokens": "Total Issues $ 788,818 $ 8,823,334 $ 82,899,841", "prefix": true} + ] + }, + { + "id": "debt-cash-basis", "page_index": 2, "bbox": [36.8, 570.5, 581.1, 735.7], "column_count": 8, + "context": "Table IIIB adjustment to cash basis; repeated amounts are not continuation", + "rows": [ + {"tokens": "Public Debt Issues (Table IIIA) $ 788,818 $ 8,823,334 $ 82,899,841", "prefix": true}, + {"tokens": "Government Account Transactions (-) 570,559 7,051,129 67,726,222", "prefix": true} + ] + }, + { + "id": "debt-limit-groups", "page_index": 2, "bbox": [36.8, 408.5, 581.1, 560.5], "column_count": 5, + "context": "Table IIIC: integrated caption; closing balance and opening balance periods, millions", + "rows": [ + {"tokens": "Debt Held by the Public $ 28,900,195 $ 28,924,758 $ 28,914,411 $ 28,307,312"}, + {"tokens": "Intragovernmental Holdings 7,315,899 7,290,501 7,304,513 7,157,362"} + ], + "headers": [ + {"text": "TABLE IIIC - Debt Subject to Limit", "row": 0, "column": 0, "rowspan": 1, "colspan": 5}, + {"text": "Balance Transactions", "row": 1, "column": 0, "rowspan": 2, "colspan": 1}, + {"text": "Closing balance today", "row": 1, "column": 1, "rowspan": 2, "colspan": 1}, + {"text": "Opening balance", "row": 1, "column": 2, "rowspan": 1, "colspan": 3}, + {"text": "Today", "row": 2, "column": 2, "rowspan": 1, "colspan": 1}, + {"text": "This month", "row": 2, "column": 3, "rowspan": 1, "colspan": 1}, + {"text": "Fiscal year", "row": 2, "column": 4, "rowspan": 1, "colspan": 1} + ] + } + ], + "relations": [ + {"from": "cash-1", "to": "cash-2", "continues": true}, + {"from": "debt-face-value", "to": "debt-cash-basis", "continues": false} + ] + }, + { + "id": "construction-groups-holdout", "source_id": "e019", "split": "holdout", "pages": [3, 5], + "tables": [ + { + "id": "annualized", "page_index": 2, "bbox": [40.5, 133.5, 544, 732], "column_count": 9, + "context": "Table 1 seasonally adjusted annual rate, millions", + "rows": [{"tokens": "Total Construction 2,196,060 2,206,880 2,194,337 2,191,059 2,184,796 2,135,771 -0.5 2.8"}] + }, + { + "id": "unadjusted", "page_index": 3, "bbox": [40.5, 138.5, 559, 732], "column_count": 10, + "context": "Table 2 not seasonally adjusted monthly values, millions", + "rows": [{"tokens": "Total Construction 172,187 157,055 156,483 164,364 178,806 167,395 485,726 472,183 2.9"}] + }, + { + "id": "variation", "page_index": 4, "bbox": [40.5, 110.5, 558, 729.7], "column_count": 6, + "context": "Table 3 percent; NA means Not applicable, not zero or blank", + "rows": [ + {"tokens": "Total Construction 0.7 0.5 0.6 0.5 0.7"}, + {"tokens": "Communication NA NA NA NA NA", "occurrences": 2} + ], + "headers": [ + {"text": "Type of Construction", "row": 0, "column": 0, "rowspan": 2, "colspan": 1}, + {"text": "Coefficient of variation", "row": 0, "column": 1, "rowspan": 1, "colspan": 2}, + {"text": "Standard error", "row": 0, "column": 3, "rowspan": 1, "colspan": 3}, + {"text": "Monthly estimate", "row": 1, "column": 1, "rowspan": 1, "colspan": 1}, + {"text": "Year-to-date estimate", "row": 1, "column": 2, "rowspan": 1, "colspan": 1}, + {"text": "Year-to-date change", "row": 1, "column": 3, "rowspan": 1, "colspan": 1}, + {"text": "Month-to-month change", "row": 1, "column": 4, "rowspan": 1, "colspan": 1}, + {"text": "Month-to-month change from prev. year", "row": 1, "column": 5, "rowspan": 1, "colspan": 1} + ] + } + ], + "relations": [{"from": "annualized", "to": "unadjusted", "continues": false}] + } +] diff --git a/benchmark/eval/table_topology_real/freeze.json b/benchmark/eval/table_topology_real/freeze.json new file mode 100644 index 0000000..731d6ea --- /dev/null +++ b/benchmark/eval/table_topology_real/freeze.json @@ -0,0 +1,28 @@ +{ + "schema_version": 1, + "frozen_at": "2026-10-12T19:13:32.616875+00:00", + "parser_outputs_observed": false, + "annotation_method": "Independent Poppler layout text, word geometry and visually reviewed source renders; government originals unchanged. Approximate raw PDF regions; literal selected cell/header facts only.", + "files_sha256": { + "sources.json": "f037e47aeeddc5def583ccd6678703ed6471668333c126cdba3ce614d1a73fc5", + "cases.json": "803d98a043407faaa3fa647de9fc4b435fc8efe730c2de17d6f0054f5f111147" + }, + "controls": { + "min_region_coverage": 0.7, + "min_table_coverage": 0.7 + }, + "blind_evidence_sha256": { + "blind-annotation-suggestions.json": "db11fd1700c3f7ca0cea913cc6868b8f28395b91450e7a91e5b959996d2ffd3f", + "blind-annotation-supplement-table-iiic.json": "11f2cfe5b33ee4613f54ca14c5461815e2816a2d9ca6f32cec295897fe8ad318", + "source-identity-audit.json": "a33bfdc5f61a36235c2a1943a2a127606d5c5930b50e6c5602300662aa33914b", + "source-review-evidence-inventory.json": "6fb3745ec30f3ede4f7ccd86e122cfebadbf30ce2aef4798b4a3c59f56d72a72" + }, + "split_policy": "Issuer and template disjoint: NCUA development; Treasury and Census holdout. No parser tuning after holdout assessment.", + "limits": [ + "Partial selected-cell and region truth; no complete-table topology metric.", + "One positive-continuation source/template; no cross-template positive generalization.", + "No exact cell-edge geometry accuracy claim.", + "Default Treasury renders failed due to unembedded Helvetica substitution; source-review includes readable isolated existing-Arial renders and failed originals." + ], + "stop_rule": "Retain failing baseline; do not modify truth, selectors, thresholds or kernel to fit observed holdouts. Correct objective harness defects with regressions and preserved initial evidence; use separate development data for future parser changes." +} diff --git a/benchmark/eval/table_topology_real/prepare.py b/benchmark/eval/table_topology_real/prepare.py new file mode 100644 index 0000000..c7d4dad --- /dev/null +++ b/benchmark/eval/table_topology_real/prepare.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Copy hash-verified local originals into the ignored evaluation source cache.""" +import argparse +import json +import os +from pathlib import Path +import tempfile + +from verify import DEFAULT_SOURCE_ROOT, ROOT, basename, load_corpus, sha + + +def prepare_sources(source_root, output, sources): + source_root, output = Path(source_root), Path(output) + if source_root.resolve() == output.resolve(): + raise ValueError("source and destination must differ") + if not isinstance(sources, list) or not sources: + raise ValueError("empty source inventory") + names = set() + # Check the complete inventory before copying anything. + for source in sources: + name = source.get("filename") + if not basename(name) or name in names: + raise ValueError("invalid or duplicate source basename") + names.add(name) + original = source_root / name + if original.is_symlink() or not original.is_file() or sha(original) != source["sha256"]: + raise ValueError("wrong local source hash: " + name) + destination = output / name + if destination.is_symlink() or (destination.exists() and (not destination.is_file() or sha(destination) != source["sha256"])): + raise ValueError("refusing to replace destination: " + name) + output.mkdir(parents=True, exist_ok=True) + result = [] + for source in sources: + name = source["filename"] + original, destination = source_root / name, output / name + reused = destination.exists() + if not reused: + temporary = None + try: + with tempfile.NamedTemporaryFile(prefix=".prepare-", dir=output, delete=False) as handle: + temporary = Path(handle.name) + with original.open("rb") as input_file: + for chunk in iter(lambda: input_file.read(1024 * 1024), b""): + handle.write(chunk) + handle.flush() + os.fsync(handle.fileno()) + if sha(temporary) != source["sha256"] or sha(original) != source["sha256"]: + raise ValueError("source changed while preparing: " + name) + # Atomic no-clobber publication; never replace an existing file. + try: + os.link(temporary, destination) + except FileExistsError: + if destination.is_symlink() or not destination.is_file() or sha(destination) != source["sha256"]: + raise ValueError("destination appeared with wrong hash: " + name) + reused = True + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + if sha(original) != source["sha256"] or sha(destination) != source["sha256"]: + raise ValueError("post-copy source/destination hash mismatch: " + name) + result.append(dict(id=source["id"], filename=name, sha256=source["sha256"], reused=reused)) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-root", type=Path, required=True) + parser.add_argument("--output", type=Path, default=DEFAULT_SOURCE_ROOT) + args = parser.parse_args() + sources, _, _ = load_corpus(ROOT) + result = prepare_sources(args.source_root, args.output, sources) + print(json.dumps(dict(status="prepared", output=str(args.output), sources=result), indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/eval/table_topology_real/sources.json b/benchmark/eval/table_topology_real/sources.json new file mode 100644 index 0000000..ce92018 --- /dev/null +++ b/benchmark/eval/table_topology_real/sources.json @@ -0,0 +1,35 @@ +[ + { + "id": "e013", "filename": "e013.pdf", "page_count": 7, "byte_count": 246631, + "title": "Share Insurance Fund Preliminary and Unaudited Financial Highlights, July 31, 2023", + "issuer": "National Credit Union Administration", "publication_date": "2023-09-05", + "template_family": "ncua-fund-monthly-statements", + "source_url": "https://ncua.gov/files/publications/share-insurance-fund-financial-highlights-2023-july.pdf", + "sha256": "8c6a1d354fd41a5f6aeb972af9d7a7758932094325024f2869d509bf3502b2b1", + "rights_metadata": "public-domain-us-government", + "rights_basis_from_source_card": "U.S. government-authored official publication under 17 U.S.C.105; official marks preserved solely for source identification, without endorsement.", + "reference_commit": "c7abe276a970605feb9cb8b27513365b6c10726e" + }, + { + "id": "e012", "filename": "e012.pdf", "page_count": 4, "byte_count": 83677, + "title": "Daily Treasury Statement, March 25, 2025", + "issuer": "Bureau of the Fiscal Service", "publication_date": "2025-03-25", + "template_family": "daily-treasury-statement", + "source_url": "https://fiscaldata.treasury.gov/static-data/published-reports/dts/DailyTreasuryStatement_20250325.pdf", + "sha256": "0daea9cbb63f571d1f9513d2bb295943351dabc965301cad83890bb2db185b04", + "rights_metadata": "public-domain-us-government", + "rights_basis_from_source_card": "U.S. government-authored official publication under 17 U.S.C.105; official marks preserved solely for source identification, without endorsement.", + "reference_commit": "c7abe276a970605feb9cb8b27513365b6c10726e" + }, + { + "id": "e019", "filename": "e019.pdf", "page_count": 5, "byte_count": 331158, + "title": "Monthly Construction Spending, March 2025", + "issuer": "U.S. Census Bureau", "publication_date": "2025-05-01", + "template_family": "census-construction-news", + "source_url": "https://www.census.gov/construction/c30/pdf/pr202503.pdf", + "sha256": "465e6711c81fa675175a548f721b6adfc479ec946acc21e6d80ae266e7b81bd5", + "rights_metadata": "public-domain-us-government", + "rights_basis_from_source_card": "U.S. government-authored official publication under 17 U.S.C.105; official marks preserved solely for source identification, without endorsement.", + "reference_commit": "c7abe276a970605feb9cb8b27513365b6c10726e" + } +] diff --git a/benchmark/eval/table_topology_real/test_prepare.py b/benchmark/eval/table_topology_real/test_prepare.py new file mode 100644 index 0000000..9c6de79 --- /dev/null +++ b/benchmark/eval/table_topology_real/test_prepare.py @@ -0,0 +1,89 @@ +"""Local source preparation must preserve originals and mismatched destinations.""" +import hashlib +from pathlib import Path +import sys +import tempfile +import unittest +from unittest import mock + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import prepare + + +class PrepareTests(unittest.TestCase): + def fixture(self, directory): + root = Path(directory) + source = root / "originals" + source.mkdir() + output = root / "cache" + contents = b"original PDF bytes for a local preparation test" + (source / "sample.pdf").write_bytes(contents) + cards = [dict(id="sample", filename="sample.pdf", sha256=hashlib.sha256(contents).hexdigest())] + return source, output, cards, contents + + def test_verified_copy_then_identical_reuse(self): + with tempfile.TemporaryDirectory() as directory: + source, output, cards, contents = self.fixture(directory) + result = prepare.prepare_sources(source, output, cards) + self.assertFalse(result[0]["reused"]) + self.assertEqual((output / "sample.pdf").read_bytes(), contents) + inode = (output / "sample.pdf").stat().st_ino + self.assertTrue(prepare.prepare_sources(source, output, cards)[0]["reused"]) + self.assertEqual((output / "sample.pdf").stat().st_ino, inode) + self.assertEqual((source / "sample.pdf").read_bytes(), contents) + self.assertEqual(sorted(p.name for p in output.iterdir()), ["sample.pdf"]) + + def test_wrong_source_hash_does_not_create_destination(self): + with tempfile.TemporaryDirectory() as directory: + source, output, cards, _ = self.fixture(directory) + cards[0]["sha256"] = "a" * 64 + with self.assertRaisesRegex(ValueError, "source hash"): + prepare.prepare_sources(source, output, cards) + self.assertFalse(output.exists()) + + def test_wrong_destination_is_preserved(self): + with tempfile.TemporaryDirectory() as directory: + source, output, cards, _ = self.fixture(directory) + output.mkdir() + (output / "sample.pdf").write_bytes(b"unique existing evidence") + with self.assertRaisesRegex(ValueError, "refusing to replace"): + prepare.prepare_sources(source, output, cards) + self.assertEqual((output / "sample.pdf").read_bytes(), b"unique existing evidence") + + def test_complete_inventory_validated_before_first_copy(self): + with tempfile.TemporaryDirectory() as directory: + source, output, cards, _ = self.fixture(directory) + cards.append(dict(id="missing", filename="missing.pdf", sha256="a" * 64)) + with self.assertRaises(ValueError): + prepare.prepare_sources(source, output, cards) + self.assertFalse(output.exists()) + + def test_bad_basename_same_root_and_symlink_rejected(self): + with tempfile.TemporaryDirectory() as directory: + source, output, cards, _ = self.fixture(directory) + with self.assertRaisesRegex(ValueError, "must differ"): + prepare.prepare_sources(source, source, cards) + bad = [dict(cards[0], filename="../sample.pdf")] + with self.assertRaisesRegex(ValueError, "basename"): + prepare.prepare_sources(source, output, bad) + (source / "alias.pdf").symlink_to(source / "sample.pdf") + with self.assertRaisesRegex(ValueError, "source hash"): + prepare.prepare_sources(source, output, [dict(cards[0], filename="alias.pdf")]) + + def test_atomic_publication_does_not_clobber_racing_destination(self): + with tempfile.TemporaryDirectory() as directory: + source, output, cards, _ = self.fixture(directory) + + def race(temporary, destination): + destination.write_bytes(b"racing unique contents") + raise FileExistsError() + + with mock.patch.object(prepare.os, "link", side_effect=race): + with self.assertRaisesRegex(ValueError, "appeared with wrong hash"): + prepare.prepare_sources(source, output, cards) + self.assertEqual((output / "sample.pdf").read_bytes(), b"racing unique contents") + self.assertEqual(sorted(p.name for p in output.iterdir()), ["sample.pdf"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmark/eval/table_topology_real/test_verify.py b/benchmark/eval/table_topology_real/test_verify.py new file mode 100644 index 0000000..3ea667e --- /dev/null +++ b/benchmark/eval/table_topology_real/test_verify.py @@ -0,0 +1,575 @@ +"""Regression tests for strict partial-region qualification, independent of PDFs.""" +import copy +import json +import os +from pathlib import Path +import signal +import subprocess +import sys +import tempfile +import time +import unittest +from unittest import mock + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import verify + + +def envelope(kind, page=None): + record = dict(record_type=kind, schema_name=kind, schema_version="0.19.1", source_id="case", document_id="doc", + provenance=dict(source_id="case", input_sha256="a" * 64, document_id="doc", page_index=page, span_ids=[])) + if page is not None: + record["page_index"] = page + return record + + +def fixture(positive=False): + source = dict(id="source", filename="source.pdf", sha256="a" * 64, page_count=5, issuer="issuer", template_family="template") + case = dict(id="case", source_id="source", split="dev", pages=[3, 4], tables=[ + dict(id="left", page_index=2, bbox=[0, 0, 100, 100], column_count=2, + rows=[dict(tokens="Label 0"), dict(tokens="Dash -"), dict(tokens="Missing NA")]), + dict(id="right", page_index=3, bbox=[0, 0, 100, 100], column_count=2, + rows=[dict(tokens="Next 17")])], relations=[dict(**{"from": "left", "to": "right", "continues": positive})]) + manifest = envelope("document_manifest") + manifest.update(page_count=5, input_sha256=source["sha256"]) + records = [manifest] + for ti, (page, pairs) in enumerate(((2, [("Label", "0"), ("Dash", "-"), ("Missing", "NA"), ("Blank", "")]), + (3, [("Next", "17")]))): + table = envelope("table", page) + table.update(table_id=f"table-{ti}", logical_table_id="logical" if positive else f"logical-{ti}", table_part_index=ti if positive else 0, + continued_from_table_id="table-0" if positive and ti else None, + continued_to_table_id="table-1" if positive and not ti else None, + column_count=2, bbox=dict(x0=0, y0=0, x1=100, y1=100), rows=[], source_span_ids=[]) + for ri, pair in enumerate(pairs): + row = dict(row_index=ri, cells=[]) + for ci, text in enumerate(pair): + sid = f"span-{ti}-{ri}-{ci}" + cell = envelope("table_cell", page) + cell.update(cell_id=f"cell-{ti}-{ri}-{ci}", row=ri, column=ci, rowspan=1, colspan=1, text=text, + source_span_ids=[sid] if text else [], reconciled_block_ids=[]) + cell["provenance"]["span_ids"] = cell["source_span_ids"][:] + row["cells"].append(cell) + if text: + span = envelope("span", page) + span.update(span_id=sid, text=text) + records.append(span) + table["source_span_ids"].append(sid) + table["rows"].append(row) + records.append(table) + return records, case, source + + +def stream(records, case): + output = [records[0]] + for page in range(case["pages"][0] - 1, case["pages"][1]): + start = envelope("page_started", page) + start["status"] = "running" + output.append(start) + output.extend(r for r in records[1:] if r.get("page_index") == page) + end = envelope("page_finished", page) + end["status"] = "completed" + output.append(end) + end = envelope("document_finished") + end["status"] = "completed" + output.append(end) + return output + + +class ScoreTests(unittest.TestCase): + def assert_failed(self, records, case, source, contains, fmt="artifact-jsonl"): + result = verify.score(records, case, source, fmt) + self.assertEqual(result["status"], "failed", result) + self.assertTrue(any(contains in failure for failure in result["failures"]), result["failures"]) + + def test_valid_negative_and_reciprocal_positive_in_both_formats(self): + for positive in (False, True): + records, case, source = fixture(positive) + for fmt, output in (("artifact-jsonl", records), ("stream-jsonl", stream(records, case))): + result = verify.score(output, case, source, fmt) + self.assertEqual(result["status"], "verified", result["failures"]) + + def test_missing_tables_cannot_pass_negative_relation(self): + records, case, source = fixture() + records = [r for r in records if r["record_type"] != "table"] + self.assert_failed(records, case, source, "both selected tables present") + + def test_one_missing_table_cannot_pass_negative_relation(self): + records, case, source = fixture() + records = [r for r in records if r.get("table_id") != "table-1"] + self.assert_failed(records, case, source, "both selected tables present") + + def test_one_way_positive_and_nonconsecutive_parts_fail(self): + records, case, source = fixture(True) + tables = [r for r in records if r["record_type"] == "table"] + tables[1]["continued_from_table_id"] = None + self.assert_failed(records, case, source, "reciprocal continuation") + tables[1]["continued_from_table_id"] = "table-0" + tables[1]["table_part_index"] = 3 + self.assert_failed(records, case, source, "consecutive parts") + + def test_false_continuation_and_logical_aliases_fail(self): + records, case, source = fixture(True) + case["relations"][0]["continues"] = False + self.assert_failed(records, case, source, "separate logical identity") + self.assert_failed(records, case, source, "no cross continuation links") + + def test_page_local_span_ids_must_be_global_and_links_on_same_page(self): + records, case, source = fixture() + spans = [r for r in records if r["record_type"] == "span"] + spans[-1]["span_id"] = spans[0]["span_id"] + self.assert_failed(records, case, source, "unique global span identity") + records, case, source = fixture() + spans = [r for r in records if r["record_type"] == "span"] + spans[0]["page_index"] = 3 + spans[0]["provenance"]["page_index"] = 3 + self.assert_failed(records, case, source, "nondangling page-local span") + + def test_blank_zero_dash_and_na_remain_distinct(self): + for index, replacement in ((0, ""), (1, "0"), (2, "0")): + records, case, source = fixture() + table = next(r for r in records if r["record_type"] == "table") + table["rows"][index]["cells"][1]["text"] = replacement + self.assert_failed(records, case, source, "literal selected row") + records, case, source = fixture() + table = next(r for r in records if r["record_type"] == "table") + blank = table["rows"][3]["cells"][1] + blank["source_span_ids"] = table["rows"][0]["cells"][1]["source_span_ids"][:] + blank["provenance"]["span_ids"] = blank["source_span_ids"][:] + self.assert_failed(records, case, source, "blank owns no text") + + def test_blank_cell_cannot_claim_reconciled_block(self): + records, case, source = fixture() + table = next(r for r in records if r["record_type"] == "table") + table["rows"][3]["cells"][1]["reconciled_block_ids"] = ["block-1"] + self.assert_failed(records, case, source, "blank owns no blocks") + + def test_whitespace_only_normalization_and_token_boundaries(self): + records, case, source = fixture() + case["tables"][0]["rows"][0]["tokens"] = " Label\t0\n" + self.assertEqual(verify.score(records, case, source)["status"], "verified") + case["tables"][0]["rows"][0]["tokens"] = "Label 0.0" + self.assert_failed(records, case, source, "literal selected row") + + def test_prefix_does_not_fuse_independent_panel_rows(self): + records, case, source = fixture() + case["tables"][0]["rows"] = [dict(tokens="Label", prefix=True)] + self.assertEqual(verify.score(records, case, source)["status"], "verified") + case["tables"][0]["rows"][0]["tokens"] = "Label 0 Dash -" + self.assert_failed(records, case, source, "literal selected row") + + def test_physical_merged_two_and_three_child_headers(self): + records, case, source = fixture() + table = next(r for r in records if r["record_type"] == "table") + table["column_count"] = 5 + case["tables"][0]["column_count"] = 5 + header_row = dict(row_index=0, cells=[]) + for ci, (text, column, width) in enumerate((("Coefficient of variation", 0, 2), ("Standard error", 2, 3))): + sid = f"header-span-{ci}" + span = envelope("span", 2) + span.update(span_id=sid, text=text) + records.insert(1, span) + cell = envelope("table_cell", 2) + cell.update(cell_id=f"header-{ci}", row=0, column=column, rowspan=1, colspan=width, text=text, source_span_ids=[sid]) + cell["provenance"]["span_ids"] = [sid] + header_row["cells"].append(cell) + table["source_span_ids"].append(sid) + for row in table["rows"]: + row["row_index"] += 1 + for cell in row["cells"]: + cell["row"] += 1 + table["rows"].insert(0, header_row) + case["tables"][0]["headers"] = [dict(text=c["text"], row=0, column=c["column"], rowspan=1, colspan=c["colspan"]) for c in header_row["cells"]] + self.assertEqual(verify.score(records, case, source)["status"], "verified") + header_row["cells"][0]["colspan"] = 1 + self.assert_failed(records, case, source, "physical merged header") + header_row["cells"][0]["colspan"] = 3 + self.assert_failed(records, case, source, "bounded physical topology") + + def test_frozen_candidate_coverage_is_bidirectional_and_ambiguity_fails(self): + records, case, source = fixture() + table = next(r for r in records if r["record_type"] == "table") + table["bbox"] = [0, 0, 1000, 1000] + self.assert_failed(records, case, source, "exactly one frozen-region candidate") + table["bbox"] = [0, 0, 20, 20] + self.assert_failed(records, case, source, "exactly one frozen-region candidate") + table["bbox"] = [0, 0, 100, 100] + duplicate = copy.deepcopy(table) + duplicate["table_id"] = "other" + records.append(duplicate) + self.assert_failed(records, case, source, "exactly one frozen-region candidate") + + def test_original_page_hash_and_source_identity(self): + records, case, source = fixture() + records[0]["page_count"] = 2 + records[0]["input_sha256"] = "wrong" + records[-1]["source_id"] = "other" + self.assert_failed(records, case, source, "original document page count") + self.assert_failed(records, case, source, "whole-file input SHA256") + self.assert_failed(records, case, source, "source identity") + + def test_selected_pages_keep_original_stream_indices(self): + records, case, source = fixture() + output = stream(records, case) + for record in output: + if "page_index" in record: + record["page_index"] -= 2 + record["provenance"]["page_index"] -= 2 + self.assert_failed(output, case, source, "original selected page", "stream-jsonl") + + def test_lifecycle_missing_duplicate_reordered_and_after_end_fail(self): + records, case, source = fixture() + output = stream(records, case) + broken = [r for r in output if r["record_type"] != "document_finished"] + self.assert_failed(broken, case, source, "complete selected-page stream", "stream-jsonl") + broken = output + [output[-1]] + self.assert_failed(broken, case, source, "before document completion", "stream-jsonl") + broken = copy.deepcopy(output) + finish = next(r for r in broken if r["record_type"] == "page_finished") + finish["page_index"] = 3 + self.assert_failed(broken, case, source, "page finish", "stream-jsonl") + self.assert_failed(output, case, source, "batch lifecycle") + + def test_boolean_numeric_slots_and_nonfinite_geometry_fail(self): + for field in ("column_count", "table_part_index"): + records, case, source = fixture() + table = next(r for r in records if r["record_type"] == "table") + table[field] = True + self.assertEqual(verify.score(records, case, source)["status"], "failed") + for field in ("row", "column", "rowspan", "colspan"): + records, case, source = fixture() + table = next(r for r in records if r["record_type"] == "table") + table["rows"][0]["cells"][0][field] = True + self.assert_failed(records, case, source, "numeric physical slots") + records, case, source = fixture() + table = next(r for r in records if r["record_type"] == "table") + table["bbox"]["x0"] = float("nan") + self.assert_failed(records, case, source, "table bounds") + + def test_actual_public_bbox_objects_and_malformed_boxes(self): + records, case, source = fixture() + self.assertEqual(verify.score(records, case, source)["status"], "verified") + table = next(r for r in records if r["record_type"] == "table") + del table["bbox"]["x1"] + self.assert_failed(records, case, source, "table bounds") + table["bbox"]["x1"] = 100 + table["bbox"]["x0"] = False + self.assert_failed(records, case, source, "table bounds") + + def test_malformed_blank_placeholder_cannot_hide_under_merged_cell(self): + records, case, source = fixture() + table = next(r for r in records if r["record_type"] == "table") + merged, placeholder = table["rows"][0]["cells"] + merged["colspan"] = 2 + placeholder.update(text="", source_span_ids=[], colspan=5) + placeholder["provenance"]["span_ids"] = [] + self.assert_failed(records, case, source, "physical slots within table") + + def test_no_ocr_command_uses_original_pdf_and_selected_range(self): + _, case, _ = fixture() + command = verify.extraction_command("parser", "original.pdf", case, "stream-jsonl") + self.assertIn("--no-ocr", command) + self.assertNotIn("--disable-ocr", command) + self.assertEqual(command[command.index("--pages") + 1], "3-4") + self.assertEqual(command[command.index("--source-id") + 1], "case") + + +class CorpusTests(unittest.TestCase): + def make_corpus(self, root): + _, case, source = fixture() + (root / "sources.json").write_text(json.dumps([source])) + (root / "cases.json").write_text(json.dumps([case])) + freeze = dict(files_sha256={name: verify.sha(root / name) for name in ("sources.json", "cases.json")}, + controls=dict(min_region_coverage=0.7, min_table_coverage=0.7)) + (root / "freeze.json").write_text(json.dumps(freeze)) + return freeze + + def test_current_real_pool_loads_with_frozen_sources_and_cases(self): + sources, cases, freeze = verify.load_corpus() + self.assertEqual(len(sources), 3) + self.assertEqual(len(cases), 3) + self.assertEqual(sum(len(case["tables"]) for case in cases), 11) + self.assertTrue({"dev", "holdout"} == {case["split"] for case in cases}) + self.assertEqual(freeze["controls"], dict(min_region_coverage=0.7, min_table_coverage=0.7)) + + def main_fixture(self, directory): + parent = Path(directory) + root = parent / "corpus" + root.mkdir() + self.make_corpus(root) + (parent / "table_compare.py").write_text("# comparator identity test\n") + originals = parent / "originals" + originals.mkdir() + (originals / "source.pdf").write_bytes(b"unchanged local original") + output = parent / "evidence" + args = ["verify.py", "--cli", sys.executable, "--split", "dev", "--source-root", str(originals), "--output", str(output)] + return root, originals, output, args + + def test_wrong_input_hash_retains_operational_failure_receipt(self): + with tempfile.TemporaryDirectory() as directory: + root, _, output, args = self.main_fixture(directory) + corpus = verify.load_corpus(root) + with mock.patch.object(verify, "ROOT", root), mock.patch.object(verify, "load_corpus", return_value=corpus), mock.patch.object(sys, "argv", args): + self.assertEqual(verify.main(), 1) + report = json.loads((output / "report.json").read_text()) + self.assertEqual(report["status"], "failed") + self.assertIn("wrong whole-file input hash", report["failures"][0]) + self.assertTrue((output / "identity.json").is_file()) + self.assertEqual(report["total"], 0) + + def test_existing_evidence_directory_is_never_overwritten(self): + with tempfile.TemporaryDirectory() as directory: + _, _, output, args = self.main_fixture(directory) + output.mkdir() + (output / "report.json").write_text("original evidence") + with mock.patch.object(sys, "argv", args), self.assertRaises(FileExistsError): + verify.main() + self.assertEqual((output / "report.json").read_text(), "original evidence") + + def test_input_mutation_stops_after_first_format_and_retains_output(self): + with tempfile.TemporaryDirectory() as directory: + root, originals, output, args = self.main_fixture(directory) + sources, cases, freeze = verify.load_corpus(root) + digest = verify.sha(originals / "source.pdf") + sources[0]["sha256"] = digest + records, _, _ = fixture() + records[0]["input_sha256"] = digest + for record in records: + record["provenance"]["input_sha256"] = digest + for row in record.get("rows", []): + for cell in row["cells"]: + cell["provenance"]["input_sha256"] = digest + + def mutate(command, stdout_path, stderr_path, timeout): + stdout_path.write_text("\n".join(json.dumps(r) for r in records) + "\n") + stderr_path.write_text("") + (originals / "source.pdf").write_bytes(b"changed during extraction") + return dict(exit_code=0, failure=None, stdout_bytes=stdout_path.stat().st_size) + + with mock.patch.object(verify, "ROOT", root), mock.patch.object(verify, "load_corpus", return_value=(sources, cases, freeze)), mock.patch.object(verify, "run_command", side_effect=mutate) as runner, mock.patch.object(sys, "argv", args): + self.assertEqual(verify.main(), 1) + self.assertEqual(runner.call_count, 1) + report = json.loads((output / "report.json").read_text()) + self.assertEqual(report["total"], 1) + self.assertIn("source changed during verification", report["results"][0]["failures"]) + self.assertTrue((output / "case-artifact-jsonl.jsonl").is_file()) + self.assertFalse((output / "case-stream-jsonl.jsonl").exists()) + + def test_interrupt_retains_completed_results_active_command_and_raw_files(self): + with tempfile.TemporaryDirectory() as directory: + root, originals, output, args = self.main_fixture(directory) + sources, cases, freeze = verify.load_corpus(root) + digest = verify.sha(originals / "source.pdf") + sources[0]["sha256"] = digest + records, _, _ = fixture() + records[0]["input_sha256"] = digest + for record in records: + record["provenance"]["input_sha256"] = digest + for row in record.get("rows", []): + for cell in row["cells"]: + cell["provenance"]["input_sha256"] = digest + calls = [] + + def interrupt_second(command, stdout_path, stderr_path, timeout): + calls.append(command) + stderr_path.write_text("") + if len(calls) == 2: + stdout_path.write_text("partial interrupted output\n") + raise KeyboardInterrupt() + stdout_path.write_text("\n".join(json.dumps(r) for r in records) + "\n") + return dict(exit_code=0, failure=None, stdout_bytes=stdout_path.stat().st_size) + + with mock.patch.object(verify, "ROOT", root), mock.patch.object(verify, "load_corpus", return_value=(sources, cases, freeze)), mock.patch.object(verify, "run_command", side_effect=interrupt_second), mock.patch.object(sys, "argv", args): + self.assertEqual(verify.main(), 130) + report = json.loads((output / "report.json").read_text()) + self.assertTrue(report["interrupted"]) + self.assertEqual(report["status"], "failed") + self.assertEqual(report["total"], 1) + self.assertEqual(report["verified"], 1) + self.assertEqual(report["active_execution"]["format"], "stream-jsonl") + self.assertEqual((output / "case-stream-jsonl.jsonl").read_text(), "partial interrupted output\n") + self.assertTrue((output / "case-artifact-jsonl.report.json").is_file()) + + def test_frozen_truth_and_selector_drift_rejected(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + freeze = self.make_corpus(root) + self.assertEqual(len(verify.load_corpus(root)[1]), 1) + (root / "cases.json").write_text("[]") + with self.assertRaisesRegex(ValueError, "corpus drift"): + verify.load_corpus(root) + freeze = self.make_corpus(root) + freeze["controls"]["min_region_coverage"] = 0.5 + (root / "freeze.json").write_text(json.dumps(freeze)) + with self.assertRaisesRegex(ValueError, "selector"): + verify.load_corpus(root) + + def test_binding_paths_cannot_escape_corpus(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + freeze = self.make_corpus(root) + freeze["files_sha256"]["../elsewhere"] = "a" * 64 + (root / "freeze.json").write_text(json.dumps(freeze)) + with self.assertRaisesRegex(ValueError, "corpus drift"): + verify.load_corpus(root) + + def test_empty_bool_pages_and_cross_split_family_rejected(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.make_corpus(root) + source, cases, freeze = verify.load_corpus(root) + second = copy.deepcopy(cases[0]) + second.update(id="holdout", split="holdout") + (root / "cases.json").write_text(json.dumps(cases + [second])) + freeze["files_sha256"]["cases.json"] = verify.sha(root / "cases.json") + (root / "freeze.json").write_text(json.dumps(freeze)) + with self.assertRaisesRegex(ValueError, "crosses frozen splits"): + verify.load_corpus(root) + cases[0]["pages"] = [True, 4] + (root / "cases.json").write_text(json.dumps(cases)) + freeze["files_sha256"]["cases.json"] = verify.sha(root / "cases.json") + (root / "freeze.json").write_text(json.dumps(freeze)) + with self.assertRaisesRegex(ValueError, "page selection"): + verify.load_corpus(root) + + def test_issuer_and_template_are_independently_disjoint(self): + for field in ("issuer", "template_family"): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self.make_corpus(root) + sources, cases, freeze = verify.load_corpus(root) + second_source = dict(sources[0], id="second", filename="second.pdf") + other = "template_family" if field == "issuer" else "issuer" + second_source[other] = "different" + second_case = copy.deepcopy(cases[0]) + second_case.update(id="holdout", source_id="second", split="holdout") + (root / "sources.json").write_text(json.dumps(sources + [second_source])) + (root / "cases.json").write_text(json.dumps(cases + [second_case])) + freeze["files_sha256"] = {name: verify.sha(root / name) for name in ("sources.json", "cases.json")} + (root / "freeze.json").write_text(json.dumps(freeze)) + with self.assertRaisesRegex(ValueError, "crosses frozen splits"): + verify.load_corpus(root) + + def test_bounded_command_retains_timeout_output(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + result = verify.run_command([sys.executable, "-u", "-c", "import time; print('kept'); time.sleep(5)"], root / "out", root / "err", 0.1) + self.assertEqual(result["failure"], "CLI timeout") + self.assertIn("kept", (root / "out").read_text()) + + def test_bounded_command_rejects_output_budget_even_after_exit(self): + with tempfile.TemporaryDirectory() as directory, mock.patch.object(verify, "MAX_OUTPUT_BYTES", 1024): + root = Path(directory) + result = verify.run_command([sys.executable, "-c", "print('x'*2048)"], root / "out", root / "err", 1) + self.assertEqual(result["failure"], "CLI output budget exceeded") + self.assertGreater((root / "out").stat().st_size, 1024) + + +class OwnedGroupTests(unittest.TestCase): + def writer_command(self, root, normal_exit=False): + ready = root / "descendant.pid" + child = ("import os, signal, time\nfrom pathlib import Path\n" + "signal.signal(signal.SIGTERM, signal.SIG_IGN)\n" + f"Path({str(ready)!r}).write_text(str(os.getpid()))\n" + "while True:\n os.write(1, b'retained evidence\\n')\n time.sleep(0.01)\n") + leader = ("import subprocess, sys, time\nfrom pathlib import Path\n" + f"subprocess.Popen([sys.executable, '-u', '-c', {child!r}])\n" + f"while not Path({str(ready)!r}).exists(): time.sleep(0.005)\n" + + ("sys.exit(0)\n" if normal_exit else "time.sleep(10)\n")) + return [sys.executable, "-u", "-c", leader], ready + + def assert_receipt_stable_and_child_stopped(self, stdout, ready): + self.assertTrue(ready.is_file(), "descendant must start before this regression is exercised") + before = stdout.stat().st_size + time.sleep(0.15) + self.assertEqual(stdout.stat().st_size, before, "owned descendant kept writing after run_command returned") + state = subprocess.run(["ps", "-p", ready.read_text(), "-o", "stat="], capture_output=True, text=True).stdout.strip() + self.assertTrue(not state or state.startswith("Z"), f"owned descendant is still active: {state}") + + def cleanup_if_active(self, ready): + # Clean up only a live child created by this regression if it failed. + if ready.is_file(): + child_pid = int(ready.read_text()) + state = subprocess.run(["ps", "-p", str(child_pid), "-o", "stat="], capture_output=True, text=True).stdout.strip() + if state and not state.startswith("Z"): + try: + os.kill(child_pid, signal.SIGKILL) + except ProcessLookupError: + pass + + def test_timeout_finalizes_term_ignoring_descendant_after_leader_exits(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + command, ready = self.writer_command(root) + try: + result = verify.run_command(command, root / "stdout", root / "stderr", 0.3) + self.assertEqual(result["failure"], "CLI timeout") + self.assert_receipt_stable_and_child_stopped(root / "stdout", ready) + finally: + self.cleanup_if_active(ready) + + def test_normal_leader_exit_finalizes_term_ignoring_descendant(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + command, ready = self.writer_command(root, normal_exit=True) + try: + result = verify.run_command(command, root / "stdout", root / "stderr", 2) + self.assertEqual(result["exit_code"], 0) + self.assertIsNone(result["failure"]) + self.assert_receipt_stable_and_child_stopped(root / "stdout", ready) + finally: + self.cleanup_if_active(ready) + + def test_output_breach_finalizes_term_ignoring_descendant(self): + with tempfile.TemporaryDirectory() as directory, mock.patch.object(verify, "MAX_OUTPUT_BYTES", 64): + root = Path(directory) + command, ready = self.writer_command(root) + try: + result = verify.run_command(command, root / "stdout", root / "stderr", 2) + self.assertEqual(result["failure"], "CLI output budget exceeded") + self.assert_receipt_stable_and_child_stopped(root / "stdout", ready) + finally: + self.cleanup_if_active(ready) + + def test_monitor_exception_and_interrupt_finalize_owned_group(self): + real_sleep = time.sleep + for exception in (OSError, KeyboardInterrupt): + with self.subTest(exception=exception.__name__), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + command, ready = self.writer_command(root) + raised = False + + def fail_after_child_ready(seconds): + nonlocal raised + if ready.exists() and not raised: + raised = True + raise exception("injected monitor interruption") + real_sleep(seconds) + + try: + with mock.patch.object(verify.time, "sleep", side_effect=fail_after_child_ready), self.assertRaises(exception): + verify.run_command(command, root / "stdout", root / "stderr", 2) + self.assert_receipt_stable_and_child_stopped(root / "stdout", ready) + finally: + self.cleanup_if_active(ready) + + def test_interrupt_during_term_grace_still_sends_kill(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + command, ready = self.writer_command(root, normal_exit=True) + real_sleep = time.sleep + + def interrupt_grace(seconds): + if seconds == 0.2: + raise KeyboardInterrupt("injected grace interruption") + real_sleep(seconds) + + try: + with mock.patch.object(verify.time, "sleep", side_effect=interrupt_grace), self.assertRaises(KeyboardInterrupt): + verify.run_command(command, root / "stdout", root / "stderr", 2) + self.assert_receipt_stable_and_child_stopped(root / "stdout", ready) + finally: + self.cleanup_if_active(ready) + + +if __name__ == "__main__": + unittest.main() diff --git a/benchmark/eval/table_topology_real/verify.py b/benchmark/eval/table_topology_real/verify.py new file mode 100644 index 0000000..aa71cfd --- /dev/null +++ b/benchmark/eval/table_topology_real/verify.py @@ -0,0 +1,504 @@ +#!/usr/bin/env python3 +"""Qualify frozen real-document regions, selected rows, and continuation links.""" +import argparse +import hashlib +import json +import math +import os +from pathlib import Path +import signal +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parent +REPO = ROOT.parents[2] +DEFAULT_SOURCE_ROOT = ROOT.parent / "raw_cache" / "table_topology_real" +MAX_OUTPUT_BYTES = 128 * 1024 * 1024 +sys.path.insert(0, str(ROOT.parent)) +from table_compare import table_topology + + +def sha(path): + digest = hashlib.sha256() + with Path(path).open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def integer(value, minimum=0, maximum=1000): + return type(value) is int and minimum <= value <= maximum + + +def tokens(value): + if not isinstance(value, str): + raise ValueError("text must be a string") + return value.split() + + +def coordinates(value): + """Truth uses lists; public JSON boxes use x0/y0/x1/y1 objects.""" + if isinstance(value, dict) and all(key in value for key in ("x0", "y0", "x1", "y1")): + value = [value[key] for key in ("x0", "y0", "x1", "y1")] + if (isinstance(value, list) and len(value) == 4 + and all(type(v) in (int, float) and math.isfinite(v) for v in value) + and value[2] > value[0] and value[3] > value[1]): + return value + return None + + +def bbox(value): + return coordinates(value) is not None + + +def coverage(region, candidate): + region, candidate = coordinates(region), coordinates(candidate) + if region is None or candidate is None: + return (0.0, 0.0) + area = max(0, min(region[2], candidate[2]) - max(region[0], candidate[0])) * max( + 0, min(region[3], candidate[3]) - max(region[1], candidate[1])) + return (area / ((region[2] - region[0]) * (region[3] - region[1])), + area / ((candidate[2] - candidate[0]) * (candidate[3] - candidate[1]))) + + +def basename(value): + return isinstance(value, str) and value not in ("", ".", "..") and Path(value).name == value and "/" not in value and "\\" not in value + + +def load_corpus(root=ROOT): + root = Path(root) + freeze = json.loads((root / "freeze.json").read_text()) + bindings = freeze["files_sha256"] + if not isinstance(bindings, dict) or not {"sources.json", "cases.json"} <= bindings.keys(): + raise ValueError("freeze must bind sources.json and cases.json") + for name, digest in bindings.items(): + if not basename(name) or sha(root / name) != digest: + raise ValueError("frozen corpus drift: " + str(name)) + controls = freeze["controls"] + for key in ("min_region_coverage", "min_table_coverage"): + if type(controls.get(key)) not in (int, float) or controls[key] != 0.7: + raise ValueError("unsupported frozen selector: " + key) + sources = json.loads((root / "sources.json").read_text()) + cases = json.loads((root / "cases.json").read_text()) + if not isinstance(sources, list) or not 0 < len(sources) <= 3 or not isinstance(cases, list) or not 0 < len(cases) <= 20: + raise ValueError("empty or malformed corpus") + source_ids = set() + for source in sources: + if not isinstance(source, dict) or not basename(source.get("id")) or source["id"] in source_ids: + raise ValueError("duplicate or invalid source id") + source_ids.add(source["id"]) + if not basename(source.get("filename")) or not integer(source.get("page_count"), 1, 7): + raise ValueError("invalid source filename or page count") + if any(not isinstance(source.get(key), str) or not source[key] for key in ("issuer", "template_family")): + raise ValueError("missing issuer/template identity") + digest = source.get("sha256") + if not isinstance(digest, str) or len(digest) != 64 or any(c not in "0123456789abcdef" for c in digest): + raise ValueError("invalid source SHA256") + by_source = {source["id"]: source for source in sources} + case_ids = set() + families = {"dev": set(), "holdout": set()} + issuers = {"dev": set(), "holdout": set()} + for case in cases: + if not isinstance(case, dict) or not basename(case.get("id")) or case["id"] in case_ids: + raise ValueError("duplicate or invalid case id") + case_ids.add(case["id"]) + source = by_source.get(case.get("source_id")) + if source is None or case.get("split") not in families: + raise ValueError("unknown source or split") + families[case["split"]].add(source["template_family"]) + issuers[case["split"]].add(source["issuer"]) + pages = case.get("pages") + if (not isinstance(pages, list) or len(pages) != 2 + or not all(integer(p, 1, source["page_count"]) for p in pages) + or pages[0] > pages[1]): + raise ValueError("invalid original page selection") + tables = case.get("tables") + if not isinstance(tables, list) or not tables or len(tables) > 100: + raise ValueError("invalid selected tables") + keys = set() + for table in tables: + if not isinstance(table, dict) or not basename(table.get("id")) or table["id"] in keys: + raise ValueError("duplicate or invalid table key") + keys.add(table["id"]) + if not integer(table.get("page_index"), pages[0] - 1, pages[1] - 1) or not bbox(table.get("bbox")): + raise ValueError("invalid table page or frozen region") + if "column_count" in table and not integer(table["column_count"], 1): + raise ValueError("invalid truth column count") + for row in table.get("rows", []): + if (not isinstance(row, dict) or not tokens(row.get("tokens")) + or type(row.get("prefix", False)) is not bool + or not integer(row.get("occurrences", 1), 1)): + raise ValueError("invalid row assertion") + for header in table.get("headers", []): + if (not isinstance(header, dict) or not tokens(header.get("text")) + or not integer(header.get("row")) or not integer(header.get("column")) + or not integer(header.get("rowspan"), 1) or not integer(header.get("colspan"), 1)): + raise ValueError("invalid physical header assertion") + for relation in case.get("relations", []): + if (not isinstance(relation, dict) or relation.get("from") not in keys + or relation.get("to") not in keys or relation["from"] == relation["to"] + or type(relation.get("continues")) is not bool): + raise ValueError("invalid continuation assertion") + if families["dev"] & families["holdout"] or issuers["dev"] & issuers["holdout"]: + raise ValueError("issuer/template family crosses frozen splits") + return sources, cases, freeze + + +def extraction_command(binary, pdf, case, output_format): + first, last = case["pages"] + return [str(binary), "extract-adaptive", "--no-ocr", "--input", str(pdf), + "--pages", f"{first}-{last}", "--source-id", case["id"], "--format", output_format] + + +def score(records, case, source, output_format="artifact-jsonl", controls=None): + controls = controls or {"min_region_coverage": 0.7, "min_table_coverage": 0.7} + checks = [] + + def check(name, passed, **details): + checks.append(dict(check=name, passed=bool(passed), **details)) + + if not isinstance(records, list) or not all(isinstance(r, dict) for r in records): + check("JSONL records are objects", False) + return finish(checks, {}) + selected_pages = list(range(case["pages"][0] - 1, case["pages"][1])) + manifests = [r for r in records if r.get("record_type") == "document_manifest"] + check("manifest first and unique", bool(records) and records[0].get("record_type") == "document_manifest" and len(manifests) == 1) + manifest = manifests[0] if manifests else {} + check("original document page count", integer(manifest.get("page_count"), 1) and manifest.get("page_count") == source["page_count"]) + check("whole-file input SHA256", manifest.get("input_sha256") == source["sha256"]) + check("caller source identity", manifest.get("source_id") == case["id"]) + document_id = manifest.get("document_id") + check("document identity present", isinstance(document_id, str) and bool(document_id)) + active = None + completed = [] + ended = False + for index, record in enumerate(records): + kind = record.get("record_type") + label = f"record {index} {kind}" + check(label + " schema envelope", isinstance(record.get("schema_name"), str) and bool(record["schema_name"]) + and isinstance(record.get("schema_version"), str) and bool(record["schema_version"]) and isinstance(kind, str)) + provenance = record.get("provenance") + check(label + " source identity", record.get("source_id") == case["id"] and record.get("document_id") == document_id + and isinstance(provenance, dict) and provenance.get("source_id") == case["id"] + and provenance.get("input_sha256") == source["sha256"] and provenance.get("document_id") == document_id) + check(label + " no errors", not record.get("errors") and record.get("status") not in ("failed", "partial", "cancelled")) + page = record.get("page_index") + if page is not None: + check(label + " original selected page", integer(page) and page in selected_pages) + check(label + " provenance page", isinstance(provenance, dict) and integer(provenance.get("page_index")) and provenance["page_index"] == page) + if output_format == "stream-jsonl": + check(label + " before document completion", not ended) + if kind == "page_started": + expected = selected_pages[len(completed)] if len(completed) < len(selected_pages) else None + check(label + " page start", active is None and page == expected and record.get("status") == "running") + active = page + elif kind == "page_finished": + check(label + " page finish", active is not None and page == active and record.get("status") == "completed") + completed.append(page) + active = None + elif kind == "document_finished": + check(label + " document finish", active is None and completed == selected_pages and record.get("status") == "completed") + ended = True + elif page is not None: + check(label + " active page artifact", active is not None and page == active) + else: + check(label + " batch lifecycle", kind not in ("page_started", "page_finished", "document_finished")) + if output_format == "stream-jsonl": + check("complete selected-page stream", ended and active is None and completed == selected_pages) + spans = {} + tables = [] + table_ids = set() + cell_ids = set() + owned = set() + for record in records: + if record.get("record_type") == "span": + sid = record.get("span_id") + check("span original selected page and literal text", integer(record.get("page_index")) + and record["page_index"] in selected_pages and isinstance(record.get("text"), str), span_id=sid) + valid = isinstance(sid, str) and bool(sid) and sid not in spans + check("unique global span identity", valid, span_id=sid) + if valid: + spans[sid] = record + for table in (r for r in records if r.get("record_type") == "table"): + tid = table.get("table_id") + valid_id = isinstance(tid, str) and bool(tid) and tid not in table_ids + check("unique global table identity", valid_id, table_id=tid) + if valid_id: + table_ids.add(tid) + tables.append(table) + check(f"{tid} table bounds", bbox(table.get("bbox"))) + check(f"{tid} numeric column extent", integer(table.get("column_count"), 1)) + check(f"{tid} logical identity and part", isinstance(table.get("logical_table_id"), str) and bool(table["logical_table_id"]) + and integer(table.get("table_part_index"))) + for field in ("continued_from_table_id", "continued_to_table_id"): + check(f"{tid} {field} type", field in table and (table[field] is None or (isinstance(table[field], str) and bool(table[field])))) + rows = table.get("rows") + well_formed = isinstance(rows, list) and bool(rows) and len(rows) <= 1000 + check(f"{tid} bounded rows", well_formed) + if not well_formed: + continue + safe_topology = True + for ri, row in enumerate(rows): + cells = row.get("cells") if isinstance(row, dict) else None + valid_row = isinstance(row, dict) and integer(row.get("row_index")) and row["row_index"] == ri and isinstance(cells, list) and len(cells) <= 1000 + check(f"{tid} row {ri} physical extent", valid_row) + if not valid_row: + safe_topology = False + continue + for ci, cell in enumerate(cells): + label = f"{tid} cell {ri}/{ci}" + valid_cell = isinstance(cell, dict) and isinstance(cell.get("text"), str) + check(label + " literal text", valid_cell) + if not valid_cell: + safe_topology = False + continue + check(label + " schema envelope", cell.get("schema_name") == "table_cell" and cell.get("record_type") == "table_cell" + and isinstance(cell.get("schema_version"), str) and bool(cell["schema_version"])) + dimensions = all(integer(cell.get(k), 1 if k in ("rowspan", "colspan") else 0) for k in ("row", "column", "rowspan", "colspan")) + check(label + " numeric physical slots", dimensions and cell.get("row") == ri) + extents = (dimensions and integer(table.get("column_count"), 1) + and cell["row"] + cell["rowspan"] <= len(rows) + and cell["column"] + cell["colspan"] <= table["column_count"]) + check(label + " physical slots within table", extents) + safe_topology &= extents + cid = cell.get("cell_id") + valid_id = isinstance(cid, str) and bool(cid) and cid not in cell_ids + check(label + " unique cell identity", valid_id) + if valid_id: + cell_ids.add(cid) + prov = cell.get("provenance") + check(label + " page and source identity", integer(cell.get("page_index")) and cell.get("page_index") == table.get("page_index") + and cell.get("source_id") == case["id"] and isinstance(prov, dict) + and prov.get("source_id") == case["id"] and prov.get("input_sha256") == source["sha256"] + and prov.get("document_id") == document_id and integer(prov.get("page_index")) and prov["page_index"] == table.get("page_index")) + ids = cell.get("source_span_ids") + valid_links = isinstance(ids, list) and all(isinstance(s, str) and bool(s) for s in ids) and len(set(ids)) == len(ids) + check(label + " span link shape", valid_links) + if not valid_links: + continue + check(label + " provenance span links", isinstance(prov, dict) and prov.get("span_ids") == ids) + if not cell["text"].strip(): + check(label + " blank owns no text", not ids) + check(label + " blank owns no blocks", cell.get("reconciled_block_ids") == []) + else: + check(label + " nonblank has source text", bool(ids)) + source_text = [] + for sid in ids: + span = spans.get(sid) + same_page = span is not None and span.get("page_index") == table.get("page_index") and span.get("source_id") == case["id"] and isinstance(span.get("text"), str) + check(label + " nondangling page-local span", same_page, span_id=sid) + check(label + " exclusive span ownership", sid not in owned, span_id=sid) + owned.add(sid) + if same_page: + source_text.append(span["text"]) + check(label + " literal source text agreement", " ".join(source_text).split() == cell["text"].split()) + check(f"{tid} bounded physical topology", safe_topology and table_topology(table) is not None) + table_links = table.get("source_span_ids") + check(f"{tid} table source links", isinstance(table_links, list) and all(isinstance(sid, str) and sid in spans + and spans[sid].get("page_index") == table.get("page_index") for sid in table_links)) + selected = {} + for truth in case["tables"]: + candidates = [] + for table in tables: + overlap_region, overlap_table = coverage(truth["bbox"], table.get("bbox")) + if (table.get("page_index") == truth["page_index"] and overlap_region >= controls["min_region_coverage"] + and overlap_table >= controls["min_table_coverage"]): + candidates.append(table) + key = truth["id"] + check(key + " exactly one frozen-region candidate", len(candidates) == 1, candidate_ids=[t.get("table_id") for t in candidates]) + if len(candidates) != 1: + continue + table = candidates[0] + selected[key] = table + if "column_count" in truth: + check(key + " selected column extent", type(table.get("column_count")) is int and table["column_count"] == truth["column_count"]) + valid_rows = table.get("rows", []) + valid_rows = valid_rows if isinstance(valid_rows, list) else [] + row_tokens = [] + cells = [] + for row in valid_rows: + row_cells = row.get("cells", []) if isinstance(row, dict) else [] + if not isinstance(row_cells, list) or not all(isinstance(c, dict) and isinstance(c.get("text"), str) and integer(c.get("column")) for c in row_cells): + continue + row_cells = sorted(row_cells, key=lambda c: c["column"]) + cells.extend(row_cells) + row_tokens.append(" ".join(c["text"] for c in row_cells).split()) + for assertion in truth.get("rows", []): + expected = tokens(assertion["tokens"]) + matches = sum((actual[:len(expected)] == expected if assertion.get("prefix", False) else actual == expected) for actual in row_tokens) + check(key + " literal selected row", matches == assertion.get("occurrences", 1), tokens=assertion["tokens"], matches=matches) + for header in truth.get("headers", []): + matches = [c for c in cells if c["text"].split() == tokens(header["text"]) + and all(type(c.get(k)) is int and c[k] == header[k] for k in ("row", "column", "rowspan", "colspan"))] + check(key + " physical merged header", len(matches) == 1, header=header, matches=len(matches)) + for relation in case.get("relations", []): + label = f"relation {relation['from']} -> {relation['to']}" + left, right = selected.get(relation["from"]), selected.get(relation["to"]) + present = left is not None and right is not None + check(label + " both selected tables present", present) + if not present: + continue + distinct = left.get("table_id") != right.get("table_id") + check(label + " distinct physical tables", distinct) + if relation["continues"]: + check(label + " reciprocal continuation", distinct and left.get("continued_to_table_id") == right.get("table_id") + and right.get("continued_from_table_id") == left.get("table_id")) + check(label + " logical identity and consecutive parts", isinstance(left.get("logical_table_id"), str) + and bool(left["logical_table_id"]) and left.get("logical_table_id") == right.get("logical_table_id") + and integer(left.get("table_part_index")) and integer(right.get("table_part_index")) + and right["table_part_index"] == left["table_part_index"] + 1) + else: + check(label + " separate logical identity", distinct and isinstance(left.get("logical_table_id"), str) + and isinstance(right.get("logical_table_id"), str) and left["logical_table_id"] != right["logical_table_id"]) + check(label + " no cross continuation links", all(left.get(field) != right.get("table_id") and right.get(field) != left.get("table_id") + for field in ("continued_from_table_id", "continued_to_table_id"))) + return finish(checks, {key: table.get("table_id") for key, table in selected.items()}) + + +def finish(checks, selected): + failures = [c["check"] for c in checks if not c["passed"]] + return dict(status="failed" if failures else "verified", failures=failures, checks=checks, selected_tables=selected, + qualification_scope="frozen regions, selected literal rows and headers, continuation links; not full-table accuracy") + + +def signal_owned_group(process, signum): + """Signal only the group created by this invocation's new session.""" + try: + os.killpg(process.pid, signum) + except ProcessLookupError: + pass + except PermissionError: + # Darwin can deny signalling a group containing only unreaped zombies. + # An active member's denial must remain an operational failure. + states = subprocess.run(["ps", "-ax", "-o", "pgid=,stat="], capture_output=True, text=True, check=True).stdout.splitlines() + for row in states: + fields = row.split() + if len(fields) == 2 and fields[0] == str(process.pid) and not fields[1].startswith("Z"): + raise + + +def finalize_owned_group(process, grace_seconds=0.2): + """Kill remaining descendants before reaping the identity-bearing leader.""" + try: + signal_owned_group(process, signal.SIGTERM) + time.sleep(grace_seconds) + finally: + # Always send KILL, even when TERM already ended the leader, or when + # an exception/interrupt arrived during the grace period. + try: + signal_owned_group(process, signal.SIGKILL) + finally: + process.wait(timeout=5) + + +def run_command(command, stdout_path, stderr_path, timeout): + started = time.monotonic() + reason = None + if not all(hasattr(os, name) for name in ("waitid", "WNOWAIT", "WEXITED", "WNOHANG", "P_PID")): + raise OSError("owner-safe process-group finalization requires waitid/WNOWAIT") + with stdout_path.open("xb") as stdout, stderr_path.open("xb") as stderr: + process = subprocess.Popen(command, stdout=stdout, stderr=stderr, start_new_session=True) + try: + while True: + if time.monotonic() - started >= timeout: + reason = "CLI timeout" + elif stdout_path.stat().st_size + stderr_path.stat().st_size > MAX_OUTPUT_BYTES: + reason = "CLI output budget exceeded" + if reason: + break + # Observe completion without reaping. Retaining the leader + # reserves its PID/PGID until group finalization has finished. + if os.waitid(os.P_PID, process.pid, os.WEXITED | os.WNOHANG | os.WNOWAIT) is not None: + break + time.sleep(0.05) + finally: + finalize_owned_group(process) + if stdout_path.stat().st_size + stderr_path.stat().st_size > MAX_OUTPUT_BYTES: + reason = "CLI output budget exceeded" + return dict(exit_code=process.returncode, failure=reason, owned_process_group=process.pid, + group_finalization="TERM, bounded grace, KILL, leader reaped last", elapsed_seconds=time.monotonic() - started, + stdout_bytes=stdout_path.stat().st_size, stderr_bytes=stderr_path.stat().st_size, + output_budget_bytes=MAX_OUTPUT_BYTES, output_budget_enforcement="sampled, 50 ms; overshoot possible") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--cli", type=Path, required=True) + parser.add_argument("--split", choices=("dev", "holdout", "all"), required=True) + parser.add_argument("--source-root", type=Path, default=DEFAULT_SOURCE_ROOT) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--timeout", type=float, default=60) + args = parser.parse_args() + if not math.isfinite(args.timeout) or not 0 < args.timeout <= 600: + parser.error("--timeout must be in (0, 600]") + args.output.mkdir(parents=True, exist_ok=False) + report = dict(status="failed", results=[], failures=[], qualification_scope="partial selected-region truth, not full-table accuracy") + try: + sources, cases, freeze = load_corpus() + binary = args.cli.resolve(strict=True) + identity = dict(cli=str(binary), cli_sha256=sha(binary), verifier_sha256=sha(Path(__file__)), + comparator_sha256=sha(ROOT.parent / "table_compare.py"), freeze_sha256=sha(ROOT / "freeze.json"), + head=subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=REPO, text=True).strip(), + split=args.split, controls=freeze["controls"], requested_ocr_mode="--no-ocr") + report["identity"] = identity + (args.output / "identity.json").write_text(json.dumps(identity, indent=2) + "\n") + by_source = {s["id"]: s for s in sources} + selected_cases = [c for c in cases if args.split == "all" or c["split"] == args.split] + if not selected_cases: + raise ValueError("selected split has no cases") + for case in selected_cases: + source = by_source[case["source_id"]] + pdf = args.source_root / source["filename"] + before = sha(pdf) + if before != source["sha256"]: + raise ValueError("wrong whole-file input hash: " + source["id"]) + for fmt in ("artifact-jsonl", "stream-jsonl"): + name = case["id"] + "-" + fmt + command = extraction_command(binary, pdf, case, fmt) + (args.output / (name + ".command.json")).write_text(json.dumps(command, indent=2) + "\n") + stdout_path = args.output / (name + ".jsonl") + report["active_execution"] = dict(id=case["id"], format=fmt, command=command, + stdout=str(stdout_path), stderr=str(args.output / (name + ".stderr"))) + execution = run_command(command, stdout_path, args.output / (name + ".stderr"), args.timeout) + result = dict(id=case["id"], format=fmt, command=command, execution=execution) + try: + if execution["stdout_bytes"] > MAX_OUTPUT_BYTES: + raise ValueError("output too large to score") + records = [json.loads(line) for line in stdout_path.read_text().splitlines() if line.strip()] + result.update(score(records, case, source, fmt, freeze["controls"])) + except (ValueError, KeyError, TypeError, UnicodeError) as error: + result.update(status="failed", failures=["invalid public output: " + str(error)], checks=[]) + after = sha(pdf) + result["source_sha256_before"] = before + result["source_sha256_after"] = after + if execution["failure"] or execution["exit_code"] != 0 or after != before: + result["status"] = "failed" + result["failures"].extend(x for x in (execution["failure"], "CLI nonzero exit" if execution["exit_code"] != 0 else None, + "source changed during verification" if after != before else None) if x) + report["results"].append(result) + (args.output / (name + ".report.json")).write_text(json.dumps(result, indent=2) + "\n") + report.pop("active_execution", None) + if after != before: + raise ValueError("stop: source changed") + load_corpus() + for path, digest in ((binary, identity["cli_sha256"]), (Path(__file__), identity["verifier_sha256"]), + (ROOT.parent / "table_compare.py", identity["comparator_sha256"]), (ROOT / "freeze.json", identity["freeze_sha256"])): + if sha(path) != digest: + raise ValueError("verification identity changed: " + str(path)) + report["status"] = "verified" if all(r["status"] == "verified" for r in report["results"]) else "failed" + except KeyboardInterrupt: + report["status"] = "failed" + report["interrupted"] = True + report["failures"].append("verification interrupted; completed results and raw evidence retained") + except (OSError, ValueError, KeyError, TypeError, subprocess.SubprocessError) as error: + report["failures"].append("operational failure: " + str(error)) + report["verified"] = sum(r["status"] == "verified" for r in report["results"]) + report["total"] = len(report["results"]) + (args.output / "report.json").write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps({k: report[k] for k in ("status", "verified", "total", "failures")})) + return 130 if report.get("interrupted") else int(report["status"] != "verified") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/output-schema.md b/docs/output-schema.md index e6a6193..7ed4018 100644 --- a/docs/output-schema.md +++ b/docs/output-schema.md @@ -6,10 +6,10 @@ should depend on the records documented here. ## Version -Current development schema version: `0.19.1` (local working-tree continuation). +Current development schema version: `0.19.2` (local working-tree continuation). Current parser build identity: `0.4.0-dev`. The `v0.3.0` release emitted schema -`0.11.0`; the development suffix keeps post-release schema `0.19.1` artifacts +`0.11.0`; the development suffix keeps post-release schema `0.19.2` artifacts distinguishable until the next release is tagged. The project is still pre-`1.0.0`, so incompatible schema changes may happen. @@ -34,6 +34,11 @@ JSON output is a `document_manifest` object with arrays of typed records. but records are produced page by page so host applications can persist partial artifacts and enqueue embeddings before the document finishes. +Schema `0.19.2` fixes batch debug-asset page identity for a selected page window. +Assets retain the original zero-based document page indices, matching their +source content and the streaming format. No fields, types or documented meanings +change. + Schema `0.19.1` fixes table evidence granularity and shared-rule assignment. Reconstructed table spans retain individual nonempty cell text and bounds, rather than synthesizing whole-row spans whose centers can falsely identify diff --git a/integrations/javascript/integration.test.mjs b/integrations/javascript/integration.test.mjs index 55e8ebe..72d6a6b 100644 --- a/integrations/javascript/integration.test.mjs +++ b/integrations/javascript/integration.test.mjs @@ -80,7 +80,7 @@ test('actual CLI -> validated rich text -> overlapping chunks -> page/span locat skip: available ? false : 'Build the parser with zig build to run the real CLI integration', }, async () => { const document = await extractPdfDocument(fixturePdf(), { binaryPath }); - assert.equal(document.manifest.schema_version, '0.19.1'); + assert.equal(document.manifest.schema_version, '0.19.2'); assert.equal(document.spans.length, 4); assert.equal(document.blocks.length, 4); assert.equal(document.sourceText, Array(4).fill('Repeated evidence.').join('\n\n')); @@ -141,7 +141,7 @@ test('actual multi-page stream preserves global ownership with page-local locato const receipt = await extractPdfPages(fixturePdf(), { binaryPath, onPage: async page => { pages.push(page); } }); assert.equal(receipt.complete, true); assert.equal(receipt.pageCount, 2); - assert.equal(receipt.schemaVersion, '0.19.1'); + assert.equal(receipt.schemaVersion, '0.19.2'); assert.deepEqual(pages.map(page => page.pageIndex), [0, 1]); assert.equal(new Set(pages.flatMap(page => page.spans.map(span => span.span_id))).size, 4); for (const page of pages) { diff --git a/integrations/javascript/vector-routing.test.mjs b/integrations/javascript/vector-routing.test.mjs index 1714afd..bc2e466 100644 --- a/integrations/javascript/vector-routing.test.mjs +++ b/integrations/javascript/vector-routing.test.mjs @@ -62,7 +62,7 @@ async function cliRecords(t, name, flags = [], format = 'artifact-jsonl') { encoding: 'utf8', timeout: 45000, maxBuffer: 10 * 1024 * 1024, }); const records = raw.trim().split('\n').map(JSON.parse); - assert.equal(records[0].schema_version, '0.19.1'); + assert.equal(records[0].schema_version, '0.19.2'); const document = format === 'artifact-jsonl' ? parseArtifactJsonl(raw, { inputSha256: createHash('sha256').update(bytes).digest('hex') }) : null; return { bytes, raw, records, document }; } @@ -194,7 +194,7 @@ test('deterministic regional vector OCR binds routes attempts source text chunks const pages = []; const receipt = await extractPdfPages(bytes, { binaryPath: fake.binaryPath, enableOcr: true, onPage: page => pages.push(page) }); assert.equal(receipt.complete, true); - assert.equal(receipt.schemaVersion, '0.19.1'); + assert.equal(receipt.schemaVersion, '0.19.2'); assertNative(pages[0], 'mixed-native-outlines'); assertLocated(pages[0], 'VECTOR EVIDENCE'); const cropPaths = (await readFile(fake.log, 'utf8')).trim().split('\n'); diff --git a/src/integration_test.zig b/src/integration_test.zig index b38b2b6..a337141 100644 --- a/src/integration_test.zig +++ b/src/integration_test.zig @@ -728,7 +728,7 @@ test "versioned schema renders native document manifest spans blocks chunks and const parsed = try std.json.parseFromSlice(std.json.Value, allocator, json, .{}); defer parsed.deinit(); try std.testing.expectEqualStrings("document_manifest", parsed.value.object.get("schema_name").?.string); - try std.testing.expectEqualStrings("0.19.1", parsed.value.object.get("schema_version").?.string); + try std.testing.expectEqualStrings("0.19.2", parsed.value.object.get("schema_version").?.string); try std.testing.expectEqualStrings("0.4.0-dev", parsed.value.object.get("parser_version").?.string); try std.testing.expectEqualStrings("document_manifest", parsed.value.object.get("record_type").?.string); try std.testing.expectEqualStrings("external-clean-native", parsed.value.object.get("source_id").?.string); @@ -774,7 +774,7 @@ test "versioned schema renders native document manifest spans blocks chunks and const debug_assets = parsed.value.object.get("debug_assets").?.array.items; try std.testing.expect(debug_assets.len > 0); try expectProvenanceObject(debug_assets[0]); - try std.testing.expectEqualStrings("0.19.1", debug_assets[0].object.get("schema_version").?.string); + try std.testing.expectEqualStrings("0.19.2", debug_assets[0].object.get("schema_version").?.string); try std.testing.expect(debug_assets[0].object.get("asset_kind") != null); try std.testing.expect(debug_assets[0].object.get("path") != null); try std.testing.expectEqual(.null, debug_assets[0].object.get("path").?); @@ -785,7 +785,7 @@ test "versioned schema renders native document manifest spans blocks chunks and try std.testing.expectEqualStrings("debug", debug_assets[0].object.get("provenance").?.object.get("source_kind").?.string); try std.testing.expect(std.mem.indexOf(u8, json, "\"schema_name\":\"document_manifest\"") != null); - try std.testing.expect(std.mem.indexOf(u8, json, "\"schema_version\":\"0.19.1\"") != null); + try std.testing.expect(std.mem.indexOf(u8, json, "\"schema_version\":\"0.19.2\"") != null); try std.testing.expect(std.mem.indexOf(u8, json, "\"record_type\":\"span\"") != null); try std.testing.expect(std.mem.indexOf(u8, json, "\"record_type\":\"block\"") != null); try std.testing.expect(std.mem.indexOf(u8, json, "\"record_type\":\"rag_chunk\"") != null); @@ -909,7 +909,7 @@ test "versioned artifact jsonl starts with manifest then typed records" { try std.testing.expect(manifest.value.object.get("output_artifacts") != null); try std.testing.expect(manifest.value.object.get("extraction_counts") != null); try std.testing.expect(manifest.value.object.get("capability_coverage") != null); - try std.testing.expect(std.mem.indexOf(u8, jsonl[first_newline + 1 ..], "\"schema_version\":\"0.19.1\"") != null); + try std.testing.expect(std.mem.indexOf(u8, jsonl[first_newline + 1 ..], "\"schema_version\":\"0.19.2\"") != null); try std.testing.expect(std.mem.indexOf(u8, jsonl[first_newline + 1 ..], "\"record_type\":\"span\"") != null); try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"record_type\":\"route_trace\"") != null); try expectJsonlLinesParse(allocator, jsonl); @@ -941,7 +941,7 @@ test "specialist protocol emits batch request records for routed table formula r const requests = parsed.value.object.get("specialist_requests").?.array.items; try std.testing.expect(requests.len > 0); try expectProvenanceObject(requests[0]); - try std.testing.expectEqualStrings("0.19.1", requests[0].object.get("schema_version").?.string); + try std.testing.expectEqualStrings("0.19.2", requests[0].object.get("schema_version").?.string); try std.testing.expectEqualStrings("specialist_request", requests[0].object.get("record_type").?.string); try std.testing.expectEqualStrings("external-specialist", requests[0].object.get("source_id").?.string); try std.testing.expect(requests[0].object.get("requested_kind") != null); @@ -1035,7 +1035,7 @@ test "configured formula specialist produces owned linked lifecycle records" { \\request_id=$(printf '%s' "$request" | sed -n 's/.*"request_id":"\([^"]*\)".*/\1/p') \\crop=$(printf '%s' "$request" | sed -n 's/.*"crop_image_path":"\([^"]*\)".*/\1/p') \\test -n "$crop" && test -f "$crop" || exit 7 - \\printf '{"schema_version":"0.19.1","record_type":"specialist_response","request_id":"%s","specialist_id":"fixture-formula","specialist_kind":"formula","status":"completed","formulas":[{"text":"x^2+y^2=z^2","format":"latex","confidence":0.95}]}\n' "$request_id" + \\printf '{"schema_version":"0.19.2","record_type":"specialist_response","request_id":"%s","specialist_id":"fixture-formula","specialist_kind":"formula","status":"completed","formulas":[{"text":"x^2+y^2=z^2","format":"latex","confidence":0.95}]}\n' "$request_id" \\ ); runtime.closeFile(script); @@ -1111,7 +1111,7 @@ test "configured formula specialist produces owned linked lifecycle records" { const formulas = response.object.get("formulas").?.array.items; try std.testing.expectEqual(@as(usize, 1), formulas.len); try std.testing.expectEqualStrings("formula", formulas[0].object.get("schema_name").?.string); - try std.testing.expectEqualStrings("0.19.1", formulas[0].object.get("schema_version").?.string); + try std.testing.expectEqualStrings("0.19.2", formulas[0].object.get("schema_version").?.string); try std.testing.expectEqualStrings("formula", formulas[0].object.get("record_type").?.string); try std.testing.expectEqualStrings("source-formula", formulas[0].object.get("source_id").?.string); try expectProvenanceObject(formulas[0]); @@ -1512,7 +1512,7 @@ test "versioned schema exposes financial table cell span metadata" { const tables = parsed.value.object.get("tables").?.array.items; try std.testing.expect(tables.len > 0); try expectProvenanceObject(tables[0]); - try std.testing.expectEqualStrings("0.19.1", tables[0].object.get("schema_version").?.string); + try std.testing.expectEqualStrings("0.19.2", tables[0].object.get("schema_version").?.string); try std.testing.expect(tables[0].object.get("logical_table_id") != null); try std.testing.expect(tables[0].object.get("table_part_index") != null); try std.testing.expect(tables[0].object.get("continued_from_table_id") != null); @@ -1522,7 +1522,7 @@ test "versioned schema exposes financial table cell span metadata" { const cells = tables[0].object.get("rows").?.array.items[0].object.get("cells").?.array.items; try std.testing.expect(cells.len > 0); try std.testing.expectEqualStrings("table_cell", cells[0].object.get("schema_name").?.string); - try std.testing.expectEqualStrings("0.19.1", cells[0].object.get("schema_version").?.string); + try std.testing.expectEqualStrings("0.19.2", cells[0].object.get("schema_version").?.string); try std.testing.expect(cells[0].object.get("cell_id") != null); try std.testing.expectEqualStrings("external-merged-cells", cells[0].object.get("source_id").?.string); try std.testing.expect(cells[0].object.get("raw_text") != null); diff --git a/src/main.zig b/src/main.zig index 3b29412..d86001a 100644 --- a/src/main.zig +++ b/src/main.zig @@ -2072,7 +2072,7 @@ test "adaptive CLI emits specialist request JSONL sidecars" { \\request_id=$(printf '%s' "$request" | sed -n 's/.*"request_id":"\([^"]*\)".*/\1/p') \\crop=$(printf '%s' "$request" | sed -n 's/.*"crop_image_path":"\([^"]*\)".*/\1/p') \\test -n "$crop" && test -f "$crop" || exit 7 - \\printf '{"schema_version":"0.19.1","record_type":"specialist_response","request_id":"%s","specialist_id":"cli-formula","specialist_kind":"formula","status":"completed","formulas":[{"text":"x^2+y^2=z^2","format":"latex","confidence":0.95}]}\n' "$request_id" + \\printf '{"schema_version":"0.19.2","record_type":"specialist_response","request_id":"%s","specialist_id":"cli-formula","specialist_kind":"formula","status":"completed","formulas":[{"text":"x^2+y^2=z^2","format":"latex","confidence":0.95}]}\n' "$request_id" \\ ); runtime.closeFile(formula_file); @@ -2146,7 +2146,7 @@ test "adaptive CLI emits specialist request JSONL sidecars" { const requests = try runtime.readFileAllocAlignedCwd(allocator, requests_path, .fromByteUnits(1)); defer allocator.free(requests); try std.testing.expect(std.mem.indexOf(u8, requests, "\"record_type\":\"specialist_request\"") != null); - try std.testing.expect(std.mem.indexOf(u8, requests, "\"schema_version\":\"0.19.1\"") != null); + try std.testing.expect(std.mem.indexOf(u8, requests, "\"schema_version\":\"0.19.2\"") != null); try std.testing.expect(std.mem.indexOf(u8, requests, "\"source_id\":\"external-specialist-cli\"") != null); try std.testing.expect(std.mem.indexOf(u8, requests, "\"requested_kind\":\"") != null); try std.testing.expect(std.mem.indexOf(u8, requests, "\"requested_outputs\"") != null); diff --git a/src/specialist_protocol.zig b/src/specialist_protocol.zig index 3098af7..68c9930 100644 --- a/src/specialist_protocol.zig +++ b/src/specialist_protocol.zig @@ -12,7 +12,7 @@ const reconcile = @import("reconcile.zig"); const runtime = @import("runtime.zig"); const page_geometry = @import("page_geometry.zig"); -pub const schema_version = "0.19.1"; +pub const schema_version = "0.19.2"; pub const SpecialistKind = enum { ocr, diff --git a/src/visual_assets.zig b/src/visual_assets.zig index 36a07ac..d5389fa 100644 --- a/src/visual_assets.zig +++ b/src/visual_assets.zig @@ -98,7 +98,11 @@ pub fn collectBatch( } for (0..pageCount(result)) |page_usize| { - const page_index: u32 = @intCast(page_usize); + // Routes retain original PDF page indices, including selected ranges. + const page_index: u32 = if (result.page_routes.len > 0) + result.page_routes[page_usize].page_index + else + @intCast(page_usize); for (page_asset_specs) |spec| { var record = try pageRecord(allocator, page_index, spec); errdefer record.deinit(allocator); @@ -664,3 +668,104 @@ test "visual asset kind names are stable" { try std.testing.expectEqualStrings("route_trace_json", assetKindName(.route_trace_json)); try std.testing.expectEqualStrings("glyph_trace_jsonl", assetKindName(.glyph_trace_jsonl)); } + +test "batch assets preserve original selected pages and materialize matching source evidence" { + const adaptive = @import("adaptive.zig"); + const allocator = std.testing.allocator; + var spans = [_]reconcile.ReconciledSpan{ + .{ + .span = layout.TextSpan.init(.{ .page_index = 2, .bbox = .{ .x0 = 31, .y0 = 47, .x1 = 91, .y1 = 59 }, .text = "Third source page" }), + .source_mask = 1, + .chosen_source = .native_pdf, + .confidence = 1, + }, + .{ + .span = layout.TextSpan.init(.{ .page_index = 6, .bbox = .{ .x0 = 111, .y0 = 127, .x1 = 171, .y1 = 139 }, .text = "Seventh source page" }), + .source_mask = 1, + .chosen_source = .native_pdf, + .confidence = 1, + }, + }; + var native_third = [_]layout.TextSpan{spans[0].span}; + var native_seventh = [_]layout.TextSpan{spans[1].span}; + var native_pages = [_][]layout.TextSpan{ &native_third, &native_seventh }; + var routes = [_]adaptive.PageRoute{ + .{ .page_index = 2, .bbox = .{ .x0 = 10, .y0 = 20, .x1 = 610, .y1 = 820 }, .span_count = 1, .image_count = 0, .char_count = 17, .signals = .{}, .route = .{ .native_fast_path = true }, .reason_mask = 0 }, + .{ .page_index = 6, .bbox = .{ .x0 = 100, .y0 = 120, .x1 = 700, .y1 = 920 }, .span_count = 1, .image_count = 0, .char_count = 19, .signals = .{}, .route = .{ .native_fast_path = true }, .reason_mask = 0 }, + }; + var result = adaptive.Result{ + .allocator = allocator, + .reconciled = .{ .allocator = allocator, .spans = &spans, .blocks = &.{}, .chunks = &.{} }, + .native_pages = &native_pages, + .layout_blocks = &.{}, + .page_routes = &routes, + .region_routes = &.{}, + .trace_records = &.{}, + .ocr_attempts = &.{}, + .formula_attempts = &.{}, + .formula_artifacts = &.{}, + .form_fields = &.{}, + .tables = &.{}, + .reading_order_pages = &.{}, + }; + + var dir_buf: [96]u8 = undefined; + const dir = try std.fmt.bufPrint(&dir_buf, "pdf-parser-selected-page-assets-{x}-{x}", .{ std.testing.random_seed, runtime.nanoTimestamp() }); + // Do not delete a preexisting directory: only this test's new assets are owned. + try std.testing.expect(!runtime.dirExists(dir)); + defer runtime.deleteTreeCwd(dir); + const records = try collectBatch(allocator, &result, dir, true); + defer deinitRecords(allocator, records); + try std.testing.expectEqual(assetCount(&result, true), records.len); + try std.testing.expectEqual(@as(usize, 16), records.len); + + for (records[document_asset_specs.len..], 0..) |record, index| { + const expected_page: u32 = if (index < page_asset_specs.len) 2 else 6; + try std.testing.expectEqual(expected_page, record.page_index.?); + try std.testing.expect(record.sha256 != null); + try std.testing.expect(record.byte_length != null); + const filename_prefix = if (expected_page == 2) "page-0003." else "page-0007."; + try std.testing.expect(std.mem.startsWith(u8, std.fs.path.basename(record.path.?), filename_prefix)); + const id_prefix = if (expected_page == 2) "page-2-" else "page-6-"; + try std.testing.expect(std.mem.startsWith(u8, record.debug_asset_id, id_prefix)); + if (!std.mem.eql(u8, record.asset_kind, "page_overlay_svg") and !std.mem.eql(u8, record.asset_kind, "glyph_trace_jsonl")) continue; + const bytes = try runtime.readFileAllocAlignedCwd(allocator, record.path.?, .fromByteUnits(1)); + defer allocator.free(bytes); + try std.testing.expectEqual(bytes.len, record.byte_length.?); + const hash = try sha256Hex(allocator, bytes); + defer allocator.free(hash); + try std.testing.expectEqualStrings(hash, record.sha256.?); + if (std.mem.eql(u8, record.asset_kind, "page_overlay_svg")) { + const present_text = if (expected_page == 2) "Third source page" else "Seventh source page"; + const absent_text = if (expected_page == 2) "Seventh source page" else "Third source page"; + try std.testing.expect(std.mem.indexOf(u8, bytes, present_text) != null); + try std.testing.expect(std.mem.indexOf(u8, bytes, absent_text) == null); + const source_box = if (expected_page == 2) "viewBox=\"10.00 20.00 600.00 800.00\"" else "viewBox=\"100.00 120.00 600.00 800.00\""; + const source_coordinate = if (expected_page == 2) "x=\"31.00\" y=\"47.00\"" else "x=\"111.00\" y=\"127.00\""; + try std.testing.expect(std.mem.indexOf(u8, bytes, source_box) != null); + try std.testing.expect(std.mem.indexOf(u8, bytes, source_coordinate) != null); + } else { + const present_page = if (expected_page == 2) "\"page_index\":2," else "\"page_index\":6,"; + const absent_page = if (expected_page == 2) "\"page_index\":6," else "\"page_index\":2,"; + try std.testing.expect(std.mem.indexOf(u8, bytes, present_page) != null); + try std.testing.expect(std.mem.indexOf(u8, bytes, absent_page) == null); + } + } + + // Full-document routes keep their existing indices and asset count. + routes[0].page_index = 0; + routes[1].page_index = 1; + const full_records = try collectBatch(allocator, &result, null, true); + defer deinitRecords(allocator, full_records); + try std.testing.expectEqual(@as(usize, 16), full_records.len); + try std.testing.expectEqual(@as(?u32, 0), full_records[4].page_index); + try std.testing.expectEqual(@as(?u32, 1), full_records[10].page_index); + + // With no routes, retain the established contiguous extent fallback. + result.page_routes = &.{}; + const fallback_records = try collectBatch(allocator, &result, null, true); + defer deinitRecords(allocator, fallback_records); + try std.testing.expectEqual(@as(usize, 46), fallback_records.len); + try std.testing.expectEqual(@as(?u32, 0), fallback_records[4].page_index); + try std.testing.expectEqual(@as(?u32, 6), fallback_records[40].page_index); +} From f8f12f120bfb587cd05ea80e5270a465da1cae22 Mon Sep 17 00:00:00 2001 From: Tom Main Date: Tue, 13 Oct 2026 14:41:45 -0400 Subject: [PATCH 2/3] Require object-shaped public table bounds in real-document gate --- benchmark/eval/table_topology_real/test_verify.py | 13 ++++++++++--- benchmark/eval/table_topology_real/verify.py | 8 +++++++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/benchmark/eval/table_topology_real/test_verify.py b/benchmark/eval/table_topology_real/test_verify.py index 3ea667e..eb349e4 100644 --- a/benchmark/eval/table_topology_real/test_verify.py +++ b/benchmark/eval/table_topology_real/test_verify.py @@ -188,11 +188,11 @@ def test_physical_merged_two_and_three_child_headers(self): def test_frozen_candidate_coverage_is_bidirectional_and_ambiguity_fails(self): records, case, source = fixture() table = next(r for r in records if r["record_type"] == "table") - table["bbox"] = [0, 0, 1000, 1000] + table["bbox"] = dict(x0=0, y0=0, x1=1000, y1=1000) self.assert_failed(records, case, source, "exactly one frozen-region candidate") - table["bbox"] = [0, 0, 20, 20] + table["bbox"] = dict(x0=0, y0=0, x1=20, y1=20) self.assert_failed(records, case, source, "exactly one frozen-region candidate") - table["bbox"] = [0, 0, 100, 100] + table["bbox"] = dict(x0=0, y0=0, x1=100, y1=100) duplicate = copy.deepcopy(table) duplicate["table_id"] = "other" records.append(duplicate) @@ -255,6 +255,13 @@ def test_actual_public_bbox_objects_and_malformed_boxes(self): table["bbox"]["x0"] = False self.assert_failed(records, case, source, "table bounds") + def test_array_shaped_public_bbox_fails_in_both_formats(self): + for fmt in ("artifact-jsonl", "stream-jsonl"): + records, case, source = fixture() + table = next(r for r in records if r["record_type"] == "table") + table["bbox"] = [table["bbox"][key] for key in ("x0", "y0", "x1", "y1")] + self.assert_failed(records, case, source, "table bounds", fmt) + def test_malformed_blank_placeholder_cannot_hide_under_merged_cell(self): records, case, source = fixture() table = next(r for r in records if r["record_type"] == "table") diff --git a/benchmark/eval/table_topology_real/verify.py b/benchmark/eval/table_topology_real/verify.py index aa71cfd..47f6c23 100644 --- a/benchmark/eval/table_topology_real/verify.py +++ b/benchmark/eval/table_topology_real/verify.py @@ -52,6 +52,10 @@ def bbox(value): return coordinates(value) is not None +def public_bbox(value): + return isinstance(value, dict) and bbox(value) + + def coverage(region, candidate): region, candidate = coordinates(region), coordinates(candidate) if region is None or candidate is None: @@ -227,7 +231,7 @@ def check(name, passed, **details): if valid_id: table_ids.add(tid) tables.append(table) - check(f"{tid} table bounds", bbox(table.get("bbox"))) + check(f"{tid} table bounds", public_bbox(table.get("bbox"))) check(f"{tid} numeric column extent", integer(table.get("column_count"), 1)) check(f"{tid} logical identity and part", isinstance(table.get("logical_table_id"), str) and bool(table["logical_table_id"]) and integer(table.get("table_part_index"))) @@ -301,6 +305,8 @@ def check(name, passed, **details): for truth in case["tables"]: candidates = [] for table in tables: + if not public_bbox(table.get("bbox")): + continue overlap_region, overlap_table = coverage(truth["bbox"], table.get("bbox")) if (table.get("page_index") == truth["page_index"] and overlap_region >= controls["min_region_coverage"] and overlap_table >= controls["min_table_coverage"]): From 92e34940e688fd10f950ff1d1976bf7dc2b508fe Mon Sep 17 00:00:00 2001 From: Tom Main Date: Tue, 13 Oct 2026 14:42:44 -0400 Subject: [PATCH 3/3] Exercise valid streaming lifecycle in bbox regression --- benchmark/eval/table_topology_real/test_verify.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/benchmark/eval/table_topology_real/test_verify.py b/benchmark/eval/table_topology_real/test_verify.py index eb349e4..d37b88b 100644 --- a/benchmark/eval/table_topology_real/test_verify.py +++ b/benchmark/eval/table_topology_real/test_verify.py @@ -260,6 +260,8 @@ def test_array_shaped_public_bbox_fails_in_both_formats(self): records, case, source = fixture() table = next(r for r in records if r["record_type"] == "table") table["bbox"] = [table["bbox"][key] for key in ("x0", "y0", "x1", "y1")] + if fmt == "stream-jsonl": + records = stream(records, case) self.assert_failed(records, case, source, "table bounds", fmt) def test_malformed_blank_placeholder_cannot_hide_under_merged_cell(self):