diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f1ec759..a6082e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,6 +135,13 @@ jobs: - name: Build shared library run: zig build shared --summary all + - uses: actions/setup-node@v6 + with: + node-version: "22" + + - name: Test public JavaScript adapters against current CLI + run: node --test integrations/javascript/*.test.mjs + python: name: Python ${{ matrix.python-version }} bindings and wheel runs-on: ubuntu-latest diff --git a/README.md b/README.md index 34e5fa0..5ce7b1a 100644 --- a/README.md +++ b/README.md @@ -463,7 +463,7 @@ never emits the password. Use three integration modes, in this order: -1. **CLI subprocess**: recommended default for workers, Siftable ingestion, and +1. **CLI subprocess**: recommended default for workers, research applications ingestion, and Cloud Run jobs. It is easiest to isolate, supervise, retry, and deploy: ```bash @@ -503,7 +503,7 @@ Use three integration modes, in this order: Python exposes the same surface as `zpdf.extract_adaptive(..., password="known-password")` or `password_file=".pdf-password"`. Node wrappers should prefer Node-API over direct V8 bindings for ABI stability, - while Siftable can continue using the subprocess path first. + while research applications can continue using the subprocess path first. 3. **HTTP server**: useful for later long-running batch corpora or internal services. It is a stateless wrapper around the same adapter: @@ -557,7 +557,7 @@ OCR/table/form/formula extraction counts, output artifact hashes or stream hash slots, warnings/errors, and capability coverage. That makes it suitable as the stable run record for general pipelines, not just this CLI. -For Siftable-style ingestion, `stream-jsonl` maps naturally to durable +For research applications-style ingestion, `stream-jsonl` maps naturally to durable processing records: `document_manifest` as a manifest artifact, `page_started`/`page_finished`/`document_finished` as status artifacts, `span`/`block`/`table` as extracted text artifacts, `route_trace` and @@ -669,13 +669,24 @@ and ruling lines, invokes OCR only for scanned routes, then reconciles native, OCR, table, formula, and form spans with typed provenance. The versioned JSON, artifact JSONL, and streaming JSONL schema is currently -`0.13.0`, with parser build identity `0.4.0-dev` after the `v0.3.0` release. +`0.15.0`, with parser build identity `0.4.0-dev` after the `v0.3.0` release. Every emitted record carries a `provenance` envelope with document and source identity, input hash context, artifact id, page/bbox, source kind, confidence, related span/block/chunk ids, route trace ids, and route reasons. This makes parser outputs usable as reviewable evidence in host pipelines without removing the older top-level compatibility fields. +Schema `0.14.0` also provides hash-bound text views, block-to-span source maps, +and chunk-to-block maps with explicit synthetic whitespace. The +[neutral JavaScript adapters](docs/host-adapters.md) expose rich extraction, +UTF-16 locators, overlap-aware chunk origins and separate prompt projections. +Schema `0.15.0` adds reconciled span/table/cell ownership and page/region recovery +assessments. Streaming supports output and adaptive page allocation budgets; +the JavaScript page adapter validates lifecycle completion and applies +backpressure. Visual completeness remains unverified. +Host adoption remains explicit; accepted schema families and parser/compiler +pins must be updated together. + ## Comparison | Feature | pdf-parser | pdfium | MuPDF | pdfplumber | diff --git a/benchmark/eval/host_locator_probe.cjs b/benchmark/eval/host_locator_probe.cjs new file mode 100644 index 0000000..f6b7abd --- /dev/null +++ b/benchmark/eval/host_locator_probe.cjs @@ -0,0 +1,58 @@ +#!/usr/bin/env node +/** Research reduction of the host's actual normalization/offset functions. + * Loads only those functions; never imports host configuration, DB, or providers. + * Requires the host's already-installed TypeScript compiler API. + */ +const fs = require('node:fs'); +const path = require('node:path'); +const crypto = require('node:crypto'); +const vm = require('node:vm'); +const { createRequire } = require('node:module'); + +const [hostRoot, chunkingPath, indexingPath] = process.argv.slice(2); +if (!hostRoot || !chunkingPath || !indexingPath) { + throw new Error('Usage: node host_locator_probe.cjs CONSUMER_ROOT CHUNKING_SOURCE INDEXING_SOURCE'); +} +const hostRequire = createRequire(path.join(hostRoot, 'package.json')); +const ts = hostRequire('typescript'); +const sources = { + chunking: chunkingPath, + indexing: indexingPath, +}; +const contents = Object.fromEntries(Object.entries(sources).map(([key, relative]) => + [key, fs.readFileSync(path.join(hostRoot, relative), 'utf8')])); + +function extractFunction(source, first, next) { + const start = source.indexOf(`function ${first}(`); + const end = source.indexOf(`function ${next}(`, start); + if (start < 0 || end < 0) throw new Error(`Source changed: ${first} boundaries missing`); + return source.slice(start, end); +} + +const selected = extractFunction(contents.chunking, 'normaliseWhitespace', 'splitSentences') + + '\n' + extractFunction(contents.indexing, 'findOffset', 'metadataString'); +const compiled = ts.transpileModule(selected, { + compilerOptions: { target: ts.ScriptTarget.ES2022 }, +}).outputText; +const context = vm.createContext({}); +vm.runInContext(compiled, context, { timeout: 1000 }); + +const cases = ['A B C', 'A B C', 'The office\n\napproved the budget.', 'A πŸ˜€ B']; +const results = cases.map((raw) => { + const chunk = context.normaliseWhitespace(raw); + const offsets = context.findOffset(raw, chunk, 0); + const rawSlice = raw.slice(offsets.start, offsets.end); + const normalizedRawSlice = context.normaliseWhitespace(rawSlice); + return { raw, chunk, offsets, rawSlice, normalizedRawSlice, + sourceRangeCoversChunk: normalizedRawSlice === chunk }; +}); +console.log(JSON.stringify({ + evidenceBoundary: 'isolated actual host functions; no DB, UI, or ingestion run', + hostRoot, + typescriptVersion: ts.version, + sources: Object.entries(sources).map(([key, relative]) => ({ + path: path.join(hostRoot, relative), + sha256: crypto.createHash('sha256').update(contents[key]).digest('hex'), + })), + results, +}, null, 2)); diff --git a/benchmark/eval/reference_review_probes.py b/benchmark/eval/reference_review_probes.py new file mode 100644 index 0000000..874e823 --- /dev/null +++ b/benchmark/eval/reference_review_probes.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Read-only parser comparisons against local reference fixtures. + +Generated reductions and JSON evidence stay in ignored eval output paths. +This is a research harness, not a quality gate or a reference-model benchmark. +""" + +import argparse +import hashlib +import json +import subprocess +from pathlib import Path + + +REPO = Path(__file__).resolve().parents[2] +DEFAULT_REFERENCES = Path("/Volumes/External SSD/referencerepos") + + +def simple_pdf(content: str) -> bytes: + stream = content.encode("ascii") + objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] " + b"/Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>", + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + b"<< /Length " + str(len(stream)).encode() + b" >>\nstream\n" + + stream + b"\nendstream", + ] + data = bytearray(b"%PDF-1.4\n") + offsets = [0] + for index, obj in enumerate(objects, 1): + offsets.append(len(data)) + data.extend(f"{index} 0 obj\n".encode() + obj + b"\nendobj\n") + xref = len(data) + data.extend(f"xref\n0 {len(objects) + 1}\n0000000000 65535 f \n".encode()) + for offset in offsets[1:]: + data.extend(f"{offset:010d} 00000 n \n".encode()) + data.extend( + f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\n" + f"startxref\n{xref}\n%%EOF\n".encode() + ) + return bytes(data) + + +def run(args: list[str]) -> dict: + result = subprocess.run(args, capture_output=True, text=True, timeout=30) + return {"command": args, "exit_code": result.returncode, + "stdout": result.stdout, "stderr": result.stderr} + + +def probe(executable: Path, pdf: Path, label: str, disable_ocr: bool = True) -> dict: + native = run([str(executable), "extract", str(pdf)]) + adaptive_args = [str(executable), "extract-adaptive", "--input", str(pdf)] + if disable_ocr: + adaptive_args.append("--no-ocr") + adaptive = run(adaptive_args + ["--format", "artifact-jsonl"]) + records = [json.loads(line) for line in adaptive["stdout"].splitlines() + if line.strip()] if adaptive["exit_code"] == 0 else [] + spans = [r for r in records if r.get("record_type") == "span"] + chunks = [r for r in records if r.get("record_type") == "rag_chunk"] + routes = [r for r in records if r.get("record_type") == "route_trace" + and r.get("trace_kind") in {"page_route", "region_route"}] + return { + "label": label, "pdf_path": str(pdf), + "input_sha256": hashlib.sha256(pdf.read_bytes()).hexdigest(), + "native_exit_code": native["exit_code"], + "native_text": native["stdout"], "native_stderr": native["stderr"], + "adaptive_exit_code": adaptive["exit_code"], + "adaptive_stderr": adaptive["stderr"], + "spans": [{k: r.get(k) for k in ("text", "bbox", "mcid", "source", + "confidence", "duplicate_count")} for r in spans], + "chunks": [{k: r.get(k) for k in ("content", "text", "confidence")} + for r in chunks], + "routes": [{k: r.get(k) for k in ("trace_kind", "route", "reasons", + "signals", "bbox")} for r in routes], + "commands": [native["command"], adaptive["command"]], + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--references", type=Path, default=DEFAULT_REFERENCES) + parser.add_argument("--executable", type=Path, default=REPO / "zig-out/bin/pdf-parser") + parser.add_argument("--output-dir", type=Path, + default=REPO / "benchmark/eval/outputs/reference-review-2026-10-09") + args = parser.parse_args() + args.output_dir.mkdir(parents=True, exist_ok=True) + reductions = { + "same-mcid-distinct-lines": "/Span << /MCID 0 >> BDC BT /F1 12 Tf " + "20 250 Td (FIRST) Tj 0 -24 Td (SECOND) Tj ET EMC", + "distinct-lines-control": "BT /F1 12 Tf 20 250 Td (FIRST) Tj " + "0 -24 Td (SECOND) Tj ET", + "rotation-45-word-gap": "BT /F1 12 Tf " + ".70710678 .70710678 -.70710678 .70710678 100 50 Tm " + "(A) Tj 8.004 0 Td (B) Tj 11.340 0 Td (C) Tj 8.664 0 Td (D) Tj ET", + "horizontal-word-gap-control": "BT /F1 12 Tf 1 0 0 1 100 50 Tm " + "(A) Tj 8.004 0 Td (B) Tj 11.340 0 Td (C) Tj 8.664 0 Td (D) Tj ET", + "ordinary-font-invisible-text": "BT /F1 12 Tf 20 250 Td " + "(VISIBLE) Tj 0 -24 Td 3 Tr (INVISIBLE) Tj ET", + "clipped-out-text": "q 0 0 10 10 re W n BT /F1 12 Tf " + "100 200 Td (OUTSIDE-CLIP) Tj ET Q", + "unique-page-title": "BT /F1 12 Tf 20 280 Td (Unique Page Title) Tj " + "0 -45 Td (First body line with enough words) Tj " + "0 -24 Td (Second body line with enough words) Tj " + "0 -24 Td (Third body line with enough words) Tj " + "0 -24 Td (Fourth body line with enough words) Tj ET", + } + evidence = [] + for name, content in reductions.items(): + path = args.output_dir / f"{name}.pdf" + path.write_bytes(simple_pdf(content)) + evidence.append(probe(args.executable, path, name)) + fixture_dir = args.references / "docling/tests/data/pdf" + for name in ("text_with_vector_glyphs.pdf", "text_with_vector_rule.pdf", + "invisible_text_layer.pdf", "vector_glyphs_in_figure.pdf"): + path = fixture_dir / name + if path.exists(): + evidence.append(probe(args.executable, path, f"docling/{name}")) + if name in {"text_with_vector_glyphs.pdf", "invisible_text_layer.pdf"}: + evidence.append(probe(args.executable, path, + f"docling/{name}/ocr-enabled", disable_ocr=False)) + output = args.output_dir / "probes.json" + output.write_text(json.dumps(evidence, indent=2, ensure_ascii=False) + "\n") + for entry in evidence: + print(json.dumps({"label": entry["label"], + "exit_code": entry["adaptive_exit_code"], + "native_text": entry["native_text"].strip(), + "adaptive_spans": [s["text"] for s in entry["spans"]], + "mcids": [s["mcid"] for s in entry["spans"]], + "duplicate_counts": [s["duplicate_count"] for s in entry["spans"]], + "page_routes": [{"route": r["route"], "reasons": r["reasons"], + "hidden_ocr": (r["signals"] or {}).get("hidden_ocr")} + for r in entry["routes"] if r["trace_kind"] == "page_route"]}, + ensure_ascii=False)) + print(f"Evidence: {output}") + + +if __name__ == "__main__": + main() diff --git a/docs/host-adapters.md b/docs/host-adapters.md new file mode 100644 index 0000000..a185eea --- /dev/null +++ b/docs/host-adapters.md @@ -0,0 +1,213 @@ +# Neutral JavaScript host adapters + +The dependency-free modules under `integrations/javascript` make phase 1 +available to both host projects. This change writes only to pdf-parser. It does +not wire a host route, update a deployed parser pin, change stored documents, +or re-embed a corpus. + +Use Node 22 or newer. Import the modules directly from a vendored parser checkout +or an explicit local package path; nothing is downloaded or published here. +The `.d.mts` declarations provide the TypeScript boundary. The host continues +to own executable discovery, authorization, storage, acceptance and deployment. + +```js +import { + composeChunksWithOrigins, + extractPdfDocument, + locateSourceRange, + projectPromptText, +} from './integrations/javascript/index.mjs'; + +const document = await extractPdfDocument(pdfBytes, { + binaryPath: approvedParserBinary, + timeoutMs: 45_000, + maxOutputBytes: 25 * 1024 * 1024, +}); +const chunks = composeChunksWithOrigins({ + text: document.sourceText, + maxTokens: 260, + overlapTokens: 40, + strategy: 'hybrid', +}); +const indexedChunks = chunks.map(chunk => ({ + ...chunk, + locator: locateSourceRange(document, chunk.startOffset, chunk.endOffset, { + sourceTextSha256: chunk.sourceTextSha256, + }), +})); +const prompt = projectPromptText(document, 50_000); +``` + +## Overlap-aware composition + +`composeChunksWithOrigins({text, maxTokens, overlapTokens, strategy, metadata})` +returns chunk bodies with `startOffset`, `endOffset`, `offsetUnit`, the exact +untrimmed input's SHA-256, normalization policy and metadata lineage. + +Origins are carried through structural windows, trimming, whitespace collapse +and semantic sentence ranges. Overlap can start inside the preceding chunk; +there is no forward end cursor or search for the chunk body. Repeated paragraphs +therefore retain their actual positions. Half-open UTF-16 boundaries never split +surrogate pairs. Empty text returns no chunks; malformed Unicode and invalid +budgets fail explicitly. + +The token budget is an approximation of four UTF-16 units per token, not a model +tokenizer. Defaults are 256/32 and hybrid; supply host settings explicitly. +Unlike the current host chunker, zero overlap stays zero and small windows have +no 200-character floor. Oversized overlap is clamped below the window and always +advances. Semantic punctuation handling also differs. This is an explicit new +chunker version, not a promise of identical chunk bodies or embedding identities. + +For retrieval indexing, use these origins instead of rediscovering offsets in +research indexing. Existing UTF-16 offset columns can hold these ranges; record +the source text hash/view identity and chunker version in host metadata. PDF +page/span evidence comes from the rich document locator; ordinary text sources +can use the origin helper without PDF geometry. Keep existing RLS and scope +transactions. Rechunk/backfill and embedding decisions require their own bounded +host rollout; this library performs none. + +## Rich extraction and compatible projections + +`extractPdfDocument(bytes, options)` invokes the host-selected binary's +`extract-adaptive --format artifact-jsonl`. It snapshots input bytes before +asynchronous I/O, supplies the input SHA-256 as source identity, uses a private +0600 temporary PDF and removes scratch files on completion/failure. Default +limits are 45 seconds and 25 MiB. OCR is disabled by default; +`enableOcr:true` uses already-configured local OCR tools. Cancellation, timeout, +oversized output and process failures remain explicit. + +The result retains the manifest, raw JSONL, all protocol records, spans, blocks +and parser chunks. Its `sourceText` is exact block text in artifact order joined +by synthetic `\n\n` separators, with its own SHA-256 and UTF-16 source map. +Parser byte offsets are translated at valid UTF-8 boundaries; byte positions +are never saved as JavaScript character offsets. + +`extractPdfText` provides the string-shaped projection of that rich API. Its +adaptive whitespace/order may differ from the existing native `extract` lane. +An editorial application can add the rich result alongside its existing extractor and migrate +reference, chat and transcript callers individually; retain its current binary +resolver, failure-response policy, edition scoping and quote-safety checks. + +Store the authoritative rich text/map before building prompts. +`projectPromptText(document, maxUtf16Units)` returns a bounded text projection, +truncation flag and source range/hash, without changing document text or offsets. +A stored-document size limit is a separate decision; do not claim a truncated +projection is the entire source. + +## Locator and protocol acceptance + +`locateSourceRange` validates the exact source-text hash and UTF-16 range, then +returns intersecting span/block/page evidence and synthetic gaps. Span bboxes +remain PDF-space boxes, not precise character quads or viewer pixels. Apply a +page-specific rotation/crop/display transform in the host. Synthetic whitespace +has null span geometry. Bind stored locators to `inputSha256`, source identity, +text hash/view and the record ids; do not resolve them against another run by +phrase alone. + +`parseArtifactJsonl(rawJsonl, {inputSha256, expectedSourceId})` validates manifest-first batch +ordering, schema `0.14.x`/`0.15.x`, input/provenance identity, text hashes, range boundaries, +full source-map coverage, referenced artifact sets and page relationships. It +rejects truncated/malformed streams, forged synthetic text and mismatched source +views. This is a batch reader; use `extractPdfPages` for lifecycle streams. + +`source_id` must be present and contain a nonempty string or null. Supply +`expectedSourceId` when the caller selected an external source identity; the +extraction adapter always binds it to the input hash supplied to the CLI. +For `0.14.x` only, chunks can carry an internal label when manifest/provenance +source identity is null. The reader retains that label as metadata; it cannot +override manifest, input hash or provenance identity. Current `0.15.x` records +must agree with the manifest, including null when no source id was supplied. + +Specialist records retain their legacy shared provenance labels; uniqueness is +checked with their actual request/attempt/response/result ids. Some current table +provenance block references describe pre-reconciliation indices. The adapter +preserves those raw values and reports missing references as +`unresolvedProvenanceLinks`, rather than using them for source-text locators. +Core span/block/chunk maps remain strict. In schema `0.15.x`, `document.tables` +exposes table/cell `reconciled_block_ids` checked against owning source spans. +Span `reconciled_block_id` and ownership provenance must agree with block maps; +cells must share their table's page. Use these authoritative fields for links. + +`PdfExtractionError.code` distinguishes `unavailable`, `timeout`, `output_limit`, +`memory_limit`, `aborted`, `crashed`, `document`, `protocol`, `callback` and `cleanup`. A cleanup failure +attached to an earlier error does not replace the original diagnosis. The host +maps these outcomes to its own user-facing messages and retry policy. + +On macOS/Linux, timeout, cancellation and output limits close output pipes and +terminate descendants from leaves upward, allowing each live parent to reap its +children before terminating the private process group. This uses the local `ps` +inventory with a one-second cleanup grace period. An unavailable inventory or a +parent unable to reap attaches `cleanupError` to the original extraction error; +cleanup is then unqualified. Descendants that deliberately create a separate +session are outside that group. Other platforms kill the direct child and close +its pipes; process-tree cleanup is not qualified there. + +The result deliberately reports `pageCoverage:'unverified'`. A valid process, +manifest or source map does not establish that every page/region was extracted. +Schema `0.15.x` adds `document.pageOutcomes` and page/region route `coverage`. +These distinguish retained text, empty unverified output, required/failed +recovery and recovered regions. The reader rejects out-of-document pages and +contradictory recovery states. Preserve unresolved assessments in intake and +retrieval policy. Paint/clipping evidence and regional OCR crops remain pending; +these assessments do not establish quote verification or publishing authority. + +## Incremental pages and resource budgets + +```js +import { extractPdfPages } from './integrations/javascript/index.mjs'; + +const receipt = await extractPdfPages(pdfBytes, { + binaryPath, + maxOutputBytes: 25 * 1024 * 1024, + maxPageBytes: 4 * 1024 * 1024, + maxNativePageMemoryBytes: 8 * 1024 * 1024, + onPage: async (page) => { + await persistProvisionalPage(page); + }, +}); +await acceptCompletedStream(receipt); +``` + +`onPage` is required and awaited serially, providing stdout backpressure. Each +page has document-global artifact ids/page indices, but page-local source-text +offsets and hash, `sourceTextScope:'page'` and `streamComplete:false`. Bind each +locator to that page projection. Callbacks are provisional; only a returned +receipt with `complete:true`, validated lifecycle/counts and exit zero qualifies +stream completion. A later failure leaves previously delivered pages incomplete. + +Defaults are 45 seconds, 25 MiB output, 4 MiB retained page JSONL, 1 MiB per +record, 100,000 records and 10,000 pages. Override with `timeoutMs`, +`maxOutputBytes`, `maxPageBytes`, `maxRecordBytes`, `maxRecords`, `maxPages` and +`signal`. Limits also cover callback waits. The adapter retains the current +page plus a bounded identity set; decoded objects have overhead beyond their +serialized size. Caller-retained pages and input bytes are outside these limits. + +Optional `maxNativePageMemoryBytes` forwards the CLI's adaptive page allocation +budget. It excludes document caches/mappings, allocator metadata and child RSS. +The native CLI also exposes `--max-output-bytes` for aggregate serialization. +Neither limit is a whole-process memory cap. Native budget failures are surfaced +as `memory_limit`/`output_limit`, without a complete receipt. + +## Adoption and verification + +Readers limited to `0.11.x`/`0.13.x` must add +`0.15.x` with contract tests before changing their parser pin. The adapter retains +`0.14.x` reader support. Applications must pair +their adopted parser revision with the selected Zig 0.17/compiler/archive identities +and run its baseline CPU/runtime smoke lane. Formula children must echo the new +request schema version. Neither host is changed by this work. + +Run the local adapter regressions and actual CLI composition test: + +```sh +zig build --summary all +node --test integrations/javascript/*.test.mjs +zig build test --summary all +``` + +The real CLI test generates a two-page PDF with repeated same-MCID lines, then +checks rich extraction -> overlapping chunk origins -> page/span locator slices. +It reports a skip when no built executable exists; that skip does not qualify +the integration. Unit tests cover Unicode byte/unit translation, repeated +phrases, zero/large overlap, forged mappings, process limits, cancellation and +scratch cleanup. No host database, browser, deployment or paid model is exercised. diff --git a/docs/host-usecase-reference-review-2026-10-10.md b/docs/host-usecase-reference-review-2026-10-10.md new file mode 100644 index 0000000..c0f303e --- /dev/null +++ b/docs/host-usecase-reference-review-2026-10-10.md @@ -0,0 +1,174 @@ +# Parser improvements for Editorial consumer and Research consumer + +Local source review, 2026-10-10. Extends the [initial parser review](/Users/thomasmain/projects/pdf-parser/docs/local-reference-review-2026-10-09.md) with actual host workflows and newly added reference repositories. + +**The highest-value shared capability is traceable extraction: every quotation, retrieved passage, and table value should lead back to a specific source run, page, and text/geometry range.** Fixing text loss remains first. After that, prioritize source mapping and host adoption of existing artifacts before adding more model complexity. + +## What the hosts actually consume + +### Editorial consumer + +`extractPdfText` (`server/lib/pdf-extract.ts:250`) executes `pdf-parser extract` and returns a string, with a 45-second timeout and 25 MiB output limit. The surrounding code distinguishes process crashes, parser unavailability, resource limits, and document errors. Preserve those distinctions. + +Verified workflows include: + +- Chat attachments and general file extraction (`server/claude.ts:1261`): PDFs become text supplied as assistant context. +- Reference-document upload (`server/routes.ts:3676`): extracted text is persisted and sent into asynchronous chunk/embedding ingestion. +- Source-reference intake (`server/routes.ts:1184`): returns up to 50,000 characters and explicitly reports truncation. +- PDF transcript import (`server/routes.ts:6666`): feeds the existing transcript/editor pipeline. +- Document quote extraction (`server/domains/quotes/ingest.ts:453`): quote candidates carry character offsets against stored document text; offset mismatches are retained as review metadata. This downstream boundary is relevant to richer PDF locators, but not every PDF upload necessarily enters it. +- Quote-safety evaluation (`server/domains/quotes/safety.ts:62`): unverified/needs-review quotations remain a publishing gate. Better extraction must not automatically confer verification. + +These paths chiefly need trustworthy readable text, explicit missing-page/region status, and source locators. Adaptive table or geometry improvements will not reach these callers until a host adapter consumes the richer output. Keep Newsroom/Civic and publication/team scopes in the host. + +The build bootstrap (`scripts/ensure-pdf-parser.mjs:20`) currently pairs Zig 0.16.0 with parser commit `836ab436...`, checks the source archive hash, and builds for a baseline CPU. That pair is internally consistent. The parser project now selects Zig 0.17.0; updating Editorial consumer should update the accepted parser revision, compiler/archive identities, and runtime smoke evidence together. Changing only the compiler pin would not deliver the new parser behavior. + +### Research consumer + +`convertWithPdfParser` (`src/services/pdfConversionService.ts:547`) already consumes adaptive artifact JSONL. The protocol reader (`src/services/pdfParserProtocol.ts:228`) checks manifest/schema/source identity, input hash when supplied, per-record provenance, and specialist relationships. Supported schema families are currently `0.11.x` and `0.13.x`. + +The conversion adapter (`src/services/pdfConversionService.ts:205`) retains the raw protocol and maps tables. The document-intelligence workflow (`src/services/documentIntelligenceExtractionService.ts:855`) persists the parser ledger and specialist artifacts. These are substantial integration assets, not missing infrastructure. + +However, retrieval chunks flattened attachment text (`src/services/researchSourceIndexService.ts:683`), rather than directly consuming parser `rag_chunk`/block records. Page and section locators are copied from attachment-wide metadata. The reader already renders page/section locators (`web/src/uix/sourceReaderBlock.tsx:48`), so improving their accuracy has an existing product surface. + +The table summary preserves raw/normalized text, numeric value, confidence, and source span IDs. It does not map all of the parser's cell bbox/role/rowspan/colspan or logical-table continuation fields. Those richer fields remain in the saved ledger; the opportunity is making them usable for evidence and analysis. + +Conversion accepts nonempty Markdown without process/specialist errors (`src/services/pdfConversionService.ts:414`); rejection triggers the existing explicit Gemini fallback. A successful native conversion reports `imageCount: 0`, and the image extraction/upload path follows fallback. Thus native success does not by itself establish complete figure/image incorporation. + +## A freshly reproduced host locator defect + +Research consumer's `findOffset` (`src/services/researchSourceIndexService.ts:461`) searches whitespace-normalized text when exact matching fails, then returns normalized positions as raw-text positions. Its hybrid chunker normalizes whitespace (`src/utils/chunking.ts:40`), so the fallback is reachable for ordinary paragraph and table whitespace. + +[The new probe](/Users/thomasmain/projects/pdf-parser/benchmark/eval/host_locator_probe.cjs) isolates and executes those actual functions using the host's installed TypeScript compiler API. It does not import configuration, database, or providers. + +| Source text | Chunk text | Returned raw slice | Coverage | +| --- | --- | --- | --- | +| `A B C` | `A B C` | `A B C` | Exact control passes | +| `A B C` | `A B C` | `A B` | Omits `C` | +| Two newlines after `The office` | Single-space version | Omits the terminal period | Wrong raw end offset | +| `A πŸ˜€ B` | `A πŸ˜€ B` | `A πŸ˜€ ` | Omits `B` | + +The evidence compares normalized raw slices with normalized chunks; expected whitespace differences alone are not counted as failures. [Results and source hashes](/Users/thomasmain/projects/pdf-parser/benchmark/eval/outputs/host-reference-review-2026-10-10/locator-probe.json) preserve the exact tested functions. This proves the function-level mapping defect, not a live database/UI failure. + +## Ranked improvements by use case + +| Priority | Shared improvement | Editorial consumer benefit | Research consumer benefit | Owning boundary | +| --- | --- | --- | --- | --- | +| P0 | Prevent legitimate same-MCID text from being deduplicated | Protect future adaptive adoption and tagged-document imports | Protect current adaptive ingestion from losing source lines | Reconciler | +| P1 | Explicit text views and reverse source maps | Verify quotes against the correct decoded source and highlight the PDF | Reliable citations, chunk offsets, source inspection | Parser contract + host mapping | +| P1 | Consume artifact geometry/chunks instead of flattening first | Structured document references alongside readable context | Page/section/table-aware retrieval using retained evidence | Host adapters; small parser linkage improvements | +| P1 | Per-page/region quality and bounded regional recovery | Scanned meeting packets, releases, PDF transcripts | Mixed exhibits, legal/financial documents, reduced whole-document fallback | Adaptive routing + host acceptance policy | +| P1/P2 | Exact table topology, numeric text, and continuation links | Budgets and financial source material without column scrambling | Investigative row/cell evidence across pages | Existing table grid + optional specialist + host mapping | +| P2 | Figure/crop references with captions and transforms | Explain where an upload contains unread diagrams/labels | Inspect source figures without treating native text as full visual coverage | Artifact/assets + host reader | +| P1 operational | Bounded output, cancellation, reproducible parser identity | Reliable uploads on the deployed host | Large-corpus ingestion without unbounded buffering or duplicate fallback work | Host process runner + native resource limits | + +### 1. Fix loss and improve structural identity + +The [unconditional MCID duplicate shortcut](/Users/thomasmain/projects/pdf-parser/src/reconcile.zig:749) still exists in the current source. The original reduction showed two distinct native lines collapsing to one adaptive span. The immediate fix should require equivalent content and placement rather than semantic group equality. + +PDF Oxide adds a stronger follow-on reference: [`McidScope`]() distinguishes page, Form XObject, and tiling-pattern namespaces. Its [structure traversal]() carries those scoped references into ordered content. Borrow that distinction between structural grouping and extraction identity. Repeated drawing instances also need placement/instance evidence; `(scope, MCID)` alone is still not duplicate proof. + +Both hosts need the text preserved before any quote, retrieval, or table feature can be trusted. Editorial consumer's present plain-text lane is not the adaptive reconciliation reduction, so do not report it as an observed Editorial consumer failure. + +### 2. Reuse the parser's source mapping for quotation and retrieval locators + +The parser already has [`SearchTextView`, transform flags, and reverse mapping runs](/Users/thomasmain/projects/pdf-parser/src/text_search.zig:1). The [glyph search source](/Users/thomasmain/projects/pdf-parser/src/root.zig:2452) attaches byte ranges to glyph geometry. Extend/reuse this machinery instead of creating a second normalization engine. + +New reference evidence: + +- MuPDF makes [ligature/whitespace preservation and dehyphenation explicit options](), and stores [character quads and paint flags](). +- pdfplumber's [`TextMap`]() maps output characters to source characters; synthetic layout whitespace can have no source character. Its [word extraction can retain constituent characters](). +- pypdfium2 explicitly [translates between PDFium character and text indices](), because inserted/excluded characters make them unequal. That implementation is a useful warning against assuming one string index space. + +Proposed neutral contract: named text-view identity/hash, offset unit, transformation records, reverse ranges, page/span references, and optional precise geometry. Preserve decoded source text separately from readable/retrieval-normalized text and model suggestions. β€œDecoded source” is not a claim of byte-for-byte correspondence to the original PDF encoding. + +Bind locators to the input hash, parser/config identity, saved ledger hash, and artifact IDs. Do not promise order-based IDs survive changed versions or routes. Translate Zig UTF-8 byte ranges into JavaScript UTF-16 ranges deliberately. Specify page index base, crop box, rotation, PDF-to-render transform, and coordinate units before highlighting. + +Editorial consumer should keep its existing string API as a compatibility wrapper around a richer extraction result, and bind quote offsets to the exact persisted text view. Research consumer should fix `findOffset` with a reverse map or explicit unknown locator, then attach parser block/span evidence to retrieval chunks. Neither host should certify a quotation solely because OCR/model output is fluent. + +Acceptance: round-trip locators through whitespace collapse, ligatures, dehyphenation, repeated identical quotes, non-BMP characters, ActualText, rotated pages, and OCR substitution. A transformed view must map to all relevant source ranges or admit ambiguity. + +### 3. Adopt existing artifacts before enlarging the schema + +The parser already emits [RAG chunk page ranges and source block IDs](/Users/thomasmain/projects/pdf-parser/src/schema.zig:1607), plus detailed table and provenance records. Its chunk `source_span_ids` are currently empty; derive span evidence through linked blocks or add tested range mappings rather than implying those arrays are complete. + +First teach Research consumer's chunk adapter to use saved parser blocks/chunks as inputs, respecting the host's retrieval budget and overlap needs. Parser chunk sizes may not fit those budgets, so subdivision still needs source-range mapping. Preserve section ancestry, table rows/cell ownership, and page-local locators through embedding and reader presentation. + +For Editorial consumer, add an opt-in richer PDF extraction result with text, page outcomes, locators, and structured tables. Use it at the shared extractor boundary so chat, reference ingestion, and transcript import do not each reinvent parsing. Keep full extracted evidence separate from the 50,000-character prompt/intake projection and its existing truncation flag. + +Marker's [JSON renderer]() combines block IDs, hierarchy, polygon/bbox, children, and images. Its [block metadata]() tracks previous text and model usage. Borrow the artifact organization; do not overwrite original evidence with generated HTML or edited text. + +New fields need the parser's schema policy and coordinated Research consumer reader support. A new `0.14` family would currently be rejected. Editorial consumer's bootstrap must also advance its accepted parser pin. Keep all product-specific persistence and permission rules outside Zig. + +### 4. Make recovery regional, bounded, and accountable + +The initial review's visibility and vector-OCR gaps are particularly relevant to scanned civic packets, mixed PDFs, chart labels, and exhibits. A readable native page can still contain an unread scanned region. + +Marker [reconstructs digital tables first, then OCRs unresolved crops](). Its [grid judge]() scores multiple candidate grids. PDF Oxide's [routing function]() returns actual source/reason and distinguishes OCR unavailable from attempted OCR that failed or produced nothing. The latter is especially useful operationally; its page-level merge heuristics should not be copied as geometry-faithful deduplication. + +MinerU's [PDF pipeline]() separates classification/native work from model lifecycle, and uses [bounded page windows](). Its native foundation and significant table/render implementations delegate to DocVortex, which is not a separate checkout here. This review establishes the boundary/lifecycle pattern, not the behavior of that missing dependency. + +Use deterministic page/region signals to retain reliable native evidence and request only unresolved regions. Record attempts, budgets, actual source, transforms, and unresolved coverage. Existing lifecycle/specialist records already provide much of the scaffolding; add missing acceptance/coverage information rather than inventing a parallel protocol. + +Research consumer could eventually recover selected regions before invoking its existing whole-document Gemini fallback. That is a proposed acceptance-policy change, not permission to accept incomplete evidence. Editorial consumer could route local OCR through the richer extractor while retaining its established generative provider boundary. No new vendor/model is selected here. + +Acceptance: exact page enumeration; distinguish verified blank, no selectable text, OCR unavailable, low-quality recognition, and clipped/nonpainting text. An unresolved page must be visible to both the reader and retrieval policy. Never turn process completion into a coverage claim. + +### 5. Preserve table facts and make geometry inspectable + +For budget/earnings source material and investigations, the useful target is a traceable fact such as `(row label, period/header, raw value, units/footnote, page, cell)`. Markdown appearance or a plausible aggregate alone is insufficient. + +pdfplumber offers a concrete deterministic comparison: [snap/join edges](), then [intersections, cells, and table groups](), with [debug overlays](). The parser already has native table grids and overlays; use these references to design harder broken-rule/borderless controls and inspect disagreements, not to add a second table kernel. + +Map existing cell geometry, roles, spans, raw numbers, continuation IDs, and source spans into Research consumer's usable artifacts. Use explicit block/table linkage after reconciliation so the host no longer needs to infer replacements from pre-reconciliation indices. Preserve compatibility while adding that relationship. + +Keep numeric parse hints subordinate to raw strings. Parenthesized negatives, blanks versus zero, scaled units, footnotes, and repeated headers deserve exact assertions. Table candidates can be evaluated with topology metrics plus exact cell/number/source-coverage metrics; do not infer confidence calibration from Marker judge constants. + +Figure crops and captions are another useful evidence surface. Emit/reference materialized source crops with hash, page geometry, transform, and owner IDs, allowing the host to show chart/table/signature regions. Separate visual descriptions from extracted source text. Use existing render/debug adapters before considering a model for interpretation. + +### 6. Bound the host and native work together + +Research consumer's process runner (`src/services/pdfConversionService.ts:331`) caps stderr but accumulates stdout without a byte limit. Editorial consumer already caps text output. For large artifact streams, add a host output budget or incremental JSONL persistence, cancellation, and deterministic final acceptance. Persisting useful partial artifacts should not mark a canceled run complete. + +Ghostty's [limited allocator]() explicitly limits each allocation request, not total live memory. Borrow that precision; a parser budget needs separate limits for decoded stream size, aggregate retained memory, and host serialized output. MinerU's windowing is a separate control on work retained at once. Do not confuse any of these with a process-wide guarantee. + +TigerBeetle's [seeded fault simulator]() makes failures reproducible. Apply the principle to parser/host fault tests: malformed JSONL halfway through a page, output limits, process death, interrupted rasterization, retries, and storage failures, with replayable seeds and cleanup assertions. There is no reason to import its database architecture. + +Valgrind's [custom allocation annotations]() help tools see allocation boundaries. Keep the parser's existing diagnostics as a separate validation lane; a reference checkout or counter returning zero is not itself proof of a full Memcheck run. No new memory/performance qualification was run in this review. + +## Reference identities and adoption limits + +| Newly inventoried reference | Local HEAD | Primary lesson | +| --- | --- | --- | +| MinerU | `31e4e3d0a54e5a82d169bdd81b2f3eddb652bb27` | Native/model separation, windows, cleanup | +| Marker | `e7c67f1d239ea6a805cbf4ed6c6b2056d435e22d` | Block hierarchy, native table judge, regional fallback | +| MuPDF | `200426aee5b4c19fb60a3d0787f0343524ac218f` | Explicit text transforms, character quads, paint evidence | +| PDF Oxide | `ad49c4cb3638dc950a29350ef882161372fde473` | Scoped MCIDs and honest fallback reasons | +| pdfplumber | `4c64b92d5caccd71c645e98e0fabb0c4dba7ff45` | Output-to-character maps, table construction/debugging | +| pypdfium2 | `062c481370902ada05f9b6635a8381755a898eef` | Index translation and optional render/geometry oracle | +| Ghostty | `1da3ac43bef626ec1155c366c1960cb0f4f74d1f` | Precise allocator limits and diagnostics | +| TigerBeetle | `c95d7a53a3d044741409b03b2f334a8ed033fa3c` | Reproducible fault injection | +| Valgrind | `0ff27aab0831786c7f419e73a2b5217cbbcc792d` | Allocation visibility for memory tooling | + +For actual dependency adoption, distinguish source and model terms: [pdfplumber is MIT](), [PDF Oxide declares MIT/Apache-2.0](), and [pypdfium2/PDFium distributions have component attribution requirements](). [MuPDF supplies AGPL/commercial terms](); [Marker separates code from model-weight terms](); [MinerU adds online-service attribution and commercial thresholds](). This is local license inventory, not an adoption decision or legal clearance. Source patterns can guide an independently implemented design without selecting any of these as a production dependency. + +## Delivery and acceptance sequence + +1. **Correctness foundation:** fix MCID data loss, fix the host locator reduction, and define text-view/offset/coordinate semantics. Keep parser and host changes independently reviewable. +2. **Use existing evidence:** consume parser block/chunk/table references in Research consumer and introduce Editorial consumer's compatible richer extraction result. Preserve original decoded evidence through prompt projections and embeddings. +3. **Quality and resource controls:** expose page/region acceptance, cap/incrementally persist artifact output, and evaluate regional OCR on mixed documents before changing fallback policy. +4. **Richer document semantics:** test table linkage/continuations, figure crops/captions, and section ancestry; maintain the previous reading-order experiment's frozen stop rule. + +Build a small source-grounded acceptance matrix from public/redacted newsroom releases, transcripts, civic packets, legal/financial reports, mixed scans, and rotated/Unicode documents. These are recommended fixture classes; no private production documents were read here. Score each host's actual lane separately: quote slice/highlight correctness; page and section locator correctness; exact numbers/row relationships; unresolved-coverage reporting; cancellation/output limits; and deterministic replay. Keep native, external readable fallback, OCR, and generated interpretation distinguishable. + +## Evidence boundary + +Inspected parser HEAD `1ca410286d0ee80b645aa58f437a53c35829f898` with selected Zig 0.17.0, Editorial consumer HEAD `b00afca769d940b6a625bb47131d5072a00d225d`, and Research consumer HEAD `5ef8719b66f6a363891013bdb99e609f380bf4ab`. Research consumer had 36 tracked working-tree changes during inspection, so findings describe the local checkout rather than an assertion about that commit or deployed behavior. The locator probe records exact file hashes. The new reference checkouts had no tracked changes. + +Fresh execution was limited to the isolated four-case host locator probe and harness syntax checks. The initial parser defects/2,886-test baseline belong to the prior dated review; the retained MCID shortcut was rechecked in current source. No host runtime, database, paid inference, specialist model, full suite, or performance benchmark ran in this extension. No production code, host files, reference checkouts, compiler pins, or deployments were changed. Added research files and ignored evidence only. + +Reproduce the locator evidence with the already-installed host compiler API: + +```sh +node benchmark/eval/host_locator_probe.cjs /path/to/consumer path/to/chunking.ts path/to/indexing.ts +``` diff --git a/docs/local-reference-review-2026-10-09.md b/docs/local-reference-review-2026-10-09.md new file mode 100644 index 0000000..df843cb --- /dev/null +++ b/docs/local-reference-review-2026-10-09.md @@ -0,0 +1,162 @@ +# Local reference review: improving extraction fidelity + +Reviewed 2026-10-09 against `/Volumes/External SSD/referencerepos`. + +Follow-up: [Editorial consumer and Research consumer use cases plus newly added local references](/Users/thomasmain/projects/pdf-parser/docs/host-usecase-reference-review-2026-10-10.md), reviewed 2026-10-10 against the current Zig 0.17 checkout. The build/test evidence below remains the original dated Zig 0.16 baseline. + +The strongest next step is to preserve more of the PDF's evidence through extraction and reconciliation. The parser already has a substantial kernel, glyph geometry, native layout, table grids, adaptive routing, provenance, and public artifact boundaries. Preserve that architecture. The local references suggest specific improvements at those boundaries, rather than replacing the kernel or introducing a general model dependency. + +**First fix the confirmed marked-content data loss. Then improve visibility, outlined-text routing, and line/word boundaries.** Reading-order, table specialists, and semantic heading hierarchy are worthwhile follow-on experiments, with explicit acceptance gates. + +## Scope and source identities + +I inventoried the local reference directory and focused on its document-processing implementations. Docling and Docling Parse provided the closest extraction and layout comparisons; PaddleOCR provided specialist contracts and table evaluation; LibreOffice provided an independent graphics-state implementation; Docjev provided host completeness and corpus practices. Other checkouts in the directory largely concern unrelated applications, agents, infrastructure, and UI. + +| Checkout | Local HEAD | Role in this review | +| --- | --- | --- | +| pdf-parser | `836ab4360b7c31d72f519ab2be82d7268c5f4852` | Current implementation and fresh probes | +| docling | `d0f55469c56d38d93ed47049b8c9d17f6785e94b` | Visibility, selective OCR, layout, tables, quality | +| docling-parse | `e0e678ce182a8a9b1b143e50251b51048ad767e4` | Rotated cell contraction, spacing, clipping | +| PaddleOCR | `dab3fe35379033fdcb2d0e9572fac0b36c9a9ebf` | Table matching/evaluation and preprocessing interfaces | +| libreoffice | `1acf2756db642e214921f35b7ab2c69980c195bc` | Graphics-state clipping and rendering mode | +| docjev | `c7abe276a970605feb9cb8b27513365b6c10726e` | Page completeness and real-document corpus design | + +Reference source was inspected locally. No reference checkout was modified and no specialist models were installed or run. The parser was built with Zig 0.16.0. This review adds a research harness and this report; it does not change parser behavior. + +## Fresh evidence + +[The probe harness](/Users/thomasmain/projects/pdf-parser/benchmark/eval/reference_review_probes.py) creates seven small PDF reductions and reads four existing Docling fixtures in place. Two fixtures also run with OCR enabled, for thirteen cases total. All native/adaptive commands exited successfully. Success did not imply complete text. + +| Case | Observed result | Interpretation | +| --- | --- | --- | +| Two lines inside one `/MCID 0` group | Native: `FIRST\nSECOND`; adaptive spans: only `FIRST`, duplicate count 1 | Confirmed adaptive data loss | +| Same lines without marked-content ID | Both lines survive | Control isolates the reconciliation problem | +| 45-degree positioned `AB CD` | Native: `AB\nCD`; adaptive: two spans `AB`, `CD` | Visual line/word relationship is lost | +| Horizontal version of the same positions | `AB C D` | Extra intra-word space; orientation is not the only boundary issue | +| Ordinary Helvetica with `3 Tr` | Invisible text emitted like ordinary native text; `hidden_ocr = 0` | Visibility is not captured by the font-name heuristic | +| Text wholly outside a rectangular clip | `OUTSIDE-CLIP` emitted like ordinary native text | No public distinction between readable source text and painted text | +| Docling mixed native/path-drawn glyph fixture, OCR enabled | Only `Native`, `Boxed`; no OCR route for the outlined glyphs | Routing lacks a vector-content signal | +| Native text above a vector rule | Native text retained | Useful control against indiscriminate vector-triggered OCR | +| Unique title above body text | Title retained in adaptive public spans | Does not support a blanket claim that public extraction drops page titles | + +[Raw evidence](/Users/thomasmain/projects/pdf-parser/benchmark/eval/outputs/reference-review-2026-10-09/probes.json) records input hashes, commands, stderr, text, spans, routes, and duplicate counts. These are focused reductions, not a representative accuracy benchmark. Reference expectations are grounded in local fixture definitions/tests; the reference pipelines themselves were not run. + +## 1. Stop treating MCID equality as duplicate identity + +**Priority: immediate; relatively small implementation.** + +[`findDuplicate`](/Users/thomasmain/projects/pdf-parser/src/reconcile.zig:745) returns a match whenever two spans on the same page have equal MCIDs, before comparing text or geometry. `mergeDuplicate` then frees one span. A marked-content group can contain several distinct lines. Our reduction demonstrates loss of the second line even though native extraction recovered it. + +Docling Parse's [cell deduplication]() requires text, font, and corresponding quad geometry. The transferable principle is to prove duplication from equivalent content and placement; a structural grouping ID is insufficient. + +Remove the unconditional MCID shortcut and retain the existing text/overlap checks as the initial fix. Longer term, distinguish page/content-stream or Form XObject scope and extraction segment identity from semantic group identity. Preserve provenance from equivalent native/OCR candidates without collapsing legitimate siblings. + +Acceptance: both lines survive; repeated equivalent spans still deduplicate; identical text at different positions survives; reused MCIDs across Form XObjects do not collide. Add a derived or real tagged-document fixture after the reduction. No public schema change is needed for the narrow fix. + +## 2. Carry text visibility through the native/adaptive boundary + +**Priority: high; medium scope, staged implementation.** + +[`Tr` is ignored by the interpreter](/Users/thomasmain/projects/pdf-parser/src/interpreter.zig:249), and the main extraction path does not retain rendering-mode/clip visibility on text spans. [`hidden_ocr`](/Users/thomasmain/projects/pdf-parser/src/complexity.zig:214) is inferred from font names. Invisible text in ordinary Helvetica therefore looks like visible native text to routing. + +Docling [separates visible and invisible PDF cells]() before OCR merging. Docling Parse [tracks clipped bounds and visibility](). LibreOffice independently [transforms and intersects clip paths](). + +Start with rendering modes and graphics-state save/restore, then conservative clipping and opacity evidence. Use explicit visible, nonpainting, clipped-out, and uncertain classifications internally. Retain raw embedded text: invisible text can be useful OCR or accessibility content. Make visible-reading selection and raw-source retention separate policies with recorded reasons. + +Do not equate every invisible span with OCR, or pretend bbox clipping proves exact visibility for arbitrary paths. Avoid Docling's blanket overlap suppression where it could discard adjacent legitimate text. Compare candidate text and geometry before suppressing embedded or fresh OCR. + +Acceptance: ordinary-font invisible text is distinguishable; visible text never loses priority solely because of font names; clipping respects transforms and `q/Q`; partially clipped text retains uncertainty; Type 3 text and text clipping modes have explicit controls. Public visibility/provenance fields require the documented schema/version update; current adaptive schema is 0.13.0. + +## 3. Route unexplained vector glyphs to OCR + +**Priority: high; depends partly on visibility evidence.** + +[`PageInput`](/Users/thomasmain/projects/pdf-parser/src/complexity.zig:20) has native spans, images, and structure evidence, but no general vector-shape regions. Ruling-line extraction is useful for tables; it does not establish coverage of outlined text. On Docling's local outlined-glyph fixture, enabling OCR still leaves the path-drawn letters absent. + +Docling's [PDF-aware OCR rectangles]() consider visible text, bitmaps, and connected shapes together. They exclude thin rules and shapes explained by native text, and judge shape coverage within the candidate region. This avoids treating a tiny caption as an explanation for all chart labels. + +Expose bounded vector summaries from the kernel: geometry, paint kind, connected-component bounds, clipping, and simple rule/highlight classification. Let the adaptive layer identify unexplained visual regions and request regional OCR through the existing external adapter. Keep raw path storage optional and bounded. + +Acceptance: outlined text beside native text and inside charts gets a recorded OCR request; rules, native text highlights, and ordinary filled boxes do not force OCR. Include merged fills, edge-touching glyphs, and dense decorative drawings. The route must be testable with a deterministic fake OCR adapter before measuring a real OCR engine's accuracy. + +## 4. Use one placement-aware line/word boundary model + +**Priority: high; medium scope, higher regression risk.** + +The parser already projects glyph positions into local writing axes. However, [`sameOrientation`](/Users/thomasmain/projects/pdf-parser/src/native_layout.zig:419) uses an absolute dot product, accepting opposite directions, while [spacing uses a fixed `0.105 * em` threshold](/Users/thomasmain/projects/pdf-parser/src/native_layout.zig:644). Other layout flatteners have separate spacing rules. Our rotated and horizontal reductions show inconsistent boundaries. + +Docling Parse's [line continuation]() uses signed writing-axis agreement, facing-edge overlap, and bounded cursor jumps scaled by placement advances. Its [word-gap inference]() uses line-local normalized gaps, preserving explicit whitespace and limiting inferred thresholds. The [rotated contraction tests]() explicitly expect words `AB`, `CD` and line `AB CD`. + +Reuse the existing glyph boundary/provenance records as the authoritative decision. Partition writing direction first; distinguish line membership from word separation; infer local gaps only when enough evidence exists. Carry those boundaries through span flattening instead of introducing another downstream global threshold. + +Acceptance: 45/90/180-degree, vertical, RTL, superscript/subscript, tiny-font, explicit-space, and malformed-width controls. Preserve existing column/gutter safeguards. Evaluate on a new real/derived set before changing defaults; do not tune to one four-letter reduction. + +## 5. Add separator and relationship evidence to the hierarchy + +**Priority: medium; a new gated experiment.** + +The parser already has hierarchy-native reading-order diagnostics. Fresh runs resolve all **96 anchors across 12 real fixtures**. Mean diagnostic precedence F1 is **0.981482 development / 0.943020 holdout**, compared with legacy **0.787322 / 0.791060**. These results do not authorize promotion: the [frozen audit](/Users/thomasmain/projects/pdf-parser/benchmark/eval/reading_order_real/representation-audit.json) records failed required-edge, ambiguity-preservation, and valid-projection gates. The fresh holdout again has only **5/6 valid projections**. Preserve the frozen stop rule and use new development/holdout documents for further experiments. + +Docling [constructs filtered page separators]() from thin filled/stroked shapes, rejects rules inside graphics or crossing content, and requires content on both sides. Its [ordering stage]() preserves header/footer elements. Its [container assembly]() maintains parent/child ownership and provenance. + +Feed validated separator evidence into the existing hierarchy rather than replacing it with a flat sort. Add caption/footnote ownership and spanning-region constraints. Preserve furniture as classified nodes, with explicit exclusion policies and cross-page recurrence evidence. The old flat-block audit shows header-removal representation failures, but the fresh public-title control passes; these are different surfaces. + +Acceptance: required edges, forbidden paths, ambiguity preservation, valid projections, source coverage, determinism, and ReleaseFast overhead. Keep diagnostics separate from applying a new order. + +## 6. Separate table topology prediction from text extraction + +**Priority: medium; optional specialist capability.** + +The native table grid already supports rich cell spans, roles, continuations, and provenance. Build on it. Table protocol records exist, but formula execution is the implemented specialist path today; a protocol family alone does not establish a working table adapter. + +Docling's [table model]() predicts topology, then [matches existing PDF/OCR text cells]() through a spatial index. PaddleOCR's [table matcher]() provides containment, overlap, and unmatched-cell fallbacks. Its [TEDS implementation]() evaluates structure and optionally content. + +Implement an opt-in specialist that returns topology/cell geometry and attaches existing text with source IDs. Validate bounds, spans, overlaps, assignment uniqueness, and reading order. Abstain on ambiguous assignments rather than blindly copying nearest-cell fallback. Docling's overlap threshold is also not a complete ownership policy: one text cell can match multiple predicted cells. + +Compute actual topology metrics in the host evaluation harness. [`eval_runner` currently passes through `options.teds`](/Users/thomasmain/projects/pdf-parser/src/eval_runner.zig:413); the presence of a TEDS field does not mean it is measured. Keep existing exact numeric, row/column/span, and source-coverage metrics alongside TEDS or GriTS. + +## 7. Make completeness and evaluation lanes explicit + +**Priority: high for quality infrastructure; independent of model work.** + +Docjev's [page validator]() checks exact page sequence/count and verifies visually blank pages before accepting empty extraction. Docling computes [page parse quality from the weakest tenth of cells]() and [document parse quality from the weakest tenth of pages](). These mechanisms expose local failures that average confidence can hide. + +Add coverage accounting across raw spans, reconciliation, selection, and public emission. Classify each page as completed, verified blank, needs review, or failed, and distinguish text completeness from graphics-only content. Retain reasons and unresolved routes. Extend lifecycle receipts compatibly, with an opt-in strict mode and the required schema update. A page count alone would not catch the MCID loss; post-reconciliation source accounting is essential. + +Also make benchmark lanes match the boundary they claim to measure. The CLI [can supply verified `pdftotext` readable fallback](/Users/thomasmain/projects/pdf-parser/src/main.zig:1167), whereas [adaptive evaluation calls the library directly](/Users/thomasmain/projects/pdf-parser/src/eval_runner.zig:482). Report native, augmented CLI, and specialist lanes separately, including executable/config identities. The fresh graph score run contains poor text metrics on some fixtures (first fixture token F1 0.125090); graph success therefore is not evidence of general text quality. Those text results require a separate lane/ground-truth investigation. + +Broaden regional render-oracle controls: expected visible and clipped regions need meaningful coverage floors. Whole-page averages can hide small missing labels. Docling Parse's clipping tests illustrate local-region checks. + +Docjev's [real-small corpus card]() describes 40 authentic PDFs/116 source pages with explicit issuer/template dependence. It is useful for sourcing additional local inputs, not extraction ground truth. Add extraction annotations and split by template/issuer where generalization is claimed; do not treat its classification labels as text/table truth. Keep raw PDFs in ignored caches and respect provenance/redistribution limits. + +## 8. Improve semantic hierarchy; measure preprocessing and concurrency later + +**Priority: medium for hierarchy; lower until measured for the others.** + +Docling's [heading inference]() prefers matched bookmarks, then numbering, then style. The parser already parses outlines and font-based headings. Connect those signals into block/chunk section ancestry, preserving source evidence and abstaining on ambiguous matches. Avoid treating dates as numbered headings. Richer public chunk/block fields need a schema update. + +PaddleOCR exposes [document orientation, unwarping, text-line orientation, and ruled/borderless table options](). Much of PP-StructureV3 delegates to PaddleX, so this local wrapper does not prove the underlying algorithms' behavior. Consider bounded preprocessing retries through specialist contracts only after scan failures are measured. Preserve reverse coordinate transforms; nonlinear unwarping needs a mapping, not merely a rotation flag. + +Docling's [bounded threaded queue]() is a useful eventual host-stage pattern: backpressure, close semantics, and per-run failure handling. The parser already releases page results in streaming mode. Instrument page/object-cache lifetimes and measure ReleaseFast lanes before adding concurrency. Keep deterministic output ordering and allocator ownership clear. No performance claim follows from this Debug review. + +## Recommended delivery sequence + +1. Fix MCID reconciliation with focused regression controls. Add post-reconciliation source coverage assertions to the research/eval boundary. +2. Implement visibility metadata and vector summaries internally; validate routing without a model, then evaluate regional OCR. Version public fields deliberately. +3. Consolidate line/word boundaries behind diagnostics and new real/derived controls before changing defaults. +4. Start a fresh separator/relationship experiment and an independent optional table-topology adapter. Preserve the existing reading-order holdout stop rule. +5. Improve semantic chunks and consider preprocessing/concurrency only with measured failure or performance evidence. + +## Validation and reproduction + +Fresh checks: **build 11/11 steps passed; tests 60/60 steps and 2,886/2,886 tests passed**. Those tests establish the current suite's baseline, not absence of the newly reproduced defects. No parser code changed. + +```sh +zig build --cache-dir .zig-cache --global-cache-dir .zig-cache/global --summary all +zig build test --cache-dir .zig-cache --global-cache-dir .zig-cache/global --summary all +python3 benchmark/eval/reference_review_probes.py +zig-out/bin/pdf-parser-eval --adaptive --disable-ocr --reading-order-mode diagnostic --reading-graph-audit --manifest benchmark/eval/reading_order_real/manifest.tsv --output benchmark/eval/outputs/reference-review-2026-10-09/reading-order-audit.jsonl +zig-out/bin/pdf-parser-eval --adaptive --disable-ocr --reading-order-mode diagnostic --manifest benchmark/eval/reading_order_real/manifest.tsv --output benchmark/eval/outputs/reference-review-2026-10-09/reading-order-scores.jsonl +``` + +Generated reductions and JSONL evidence remain under ignored eval outputs. The harness reads local reference PDFs in place and does not copy them into the tracked corpus. Evidence is narrow correctness/graph evidence; no head-to-head model benchmark, broad ranking, or speedup has been established. diff --git a/docs/output-schema.md b/docs/output-schema.md index a8077fe..857fe46 100644 --- a/docs/output-schema.md +++ b/docs/output-schema.md @@ -6,10 +6,10 @@ should depend on the records documented here. ## Version -Current schema version: `0.13.0` +Current schema version: `0.15.0` Current parser build identity: `0.4.0-dev`. The `v0.3.0` release emitted schema -`0.11.0`; the development suffix keeps post-release schema `0.13.0` artifacts +`0.11.0`; the development suffix keeps post-release schema `0.15.0` artifacts distinguishable until the next release is tagged. The project is still pre-`1.0.0`, so incompatible schema changes may happen. @@ -34,6 +34,23 @@ JSON output is a `document_manifest` object with arrays of typed records. but records are produced page by page so host applications can persist partial artifacts and enqueue embeddings before the document finishes. +Schema `0.15.0` adds authoritative reconciled block references to spans, tables +and cells, and page/region recovery assessments to route traces. Span provenance +block references now resolve to actual reconciled owners, including later pages +in streaming output. Numeric compatibility fields remain unchanged. Readers +must explicitly accept `0.15.x` before adopting this revision. +When the caller omits source identity, chunk `source_id` is now null, matching +the manifest and provenance. Schema `0.14.x` chunks used an internal default +label in that field; readers may retain it as compatibility metadata while +binding identity through the manifest and provenance. + +Schema `0.14.0` adds identity text views and source maps to spans, blocks and +retrieval chunks. Chunk span references now resolve to owning reconciled spans; +streamed mappings use document-global ids. These are additive fields with no +change to existing text, bbox, page-index or compatibility fields. A host that +restricts accepted schema families must explicitly add `0.14.x` before adopting +this parser revision. Formula children must echo the request's schema version. + Schema `0.13.0` closes the first formula-specialist loop for file-backed, unrotated pages. Routed formula regions are padded, clamped to the page, mapped from PDF coordinates to raster coordinates, and rendered as deterministic PNG @@ -267,6 +284,110 @@ protocol consumers. Chunk record with chunk id, source id, content, block range, page range, bbox, source mask, confidence, and source block/span references. +### Text-view and source-map contract (0.14) + +`span`, `block`, and `rag_chunk` include `text_view`: + +```json +{ + "text_view_id": "block-2:text", + "kind": "reconciled_block", + "offset_unit": "utf8_byte", + "range_convention": "half_open", + "normalization": "none", + "text_sha256": "SHA256 of the exact UTF-8 text field" +} +``` + +Kinds are `reconciled_span`, `reconciled_block`, and `rag_chunk`; the latter +hashes `content`. IDs are artifact-local, so bind a locator to document/source +identity, input SHA-256, the text-view id and text SHA-256. Never resolve a +locator against another extraction run solely because an id or phrase matches. + +Blocks include ordered `source_map` runs covering their entire `text`: + +```json +[ + {"text_start_byte":0,"text_end_byte":5,"source_span_id":"span-4","source_start_byte":0,"source_end_byte":5}, + {"text_start_byte":5,"text_end_byte":6,"source_span_id":null,"source_start_byte":null,"source_end_byte":null}, + {"text_start_byte":6,"text_end_byte":11,"source_span_id":"span-5","source_start_byte":0,"source_end_byte":5} +] +``` + +Ranges are half-open UTF-8 bytes, must end on Unicode scalar boundaries, and +map byte-for-byte to the referenced exact text. Parser-inserted spaces/newlines +have null source ids and ranges: they have no invented glyph evidence. Origin +ranges are retained during block construction, including repeated spans; no +text search is used to infer them. A span bbox covers the whole span, not the +individual character subset selected by a mapped range. + +Chunks use the same run fields with `source_block_id` instead of +`source_span_id`; their `\n\n` block separators are synthetic null runs. +`source_span_ids` and `provenance.span_ids` enumerate the spans belonging to +their source blocks. Resolve chunk -> block -> span maps to recover evidence. +IDs in batch and streaming maps are document-global. Byte ranges are local to +their own exact text view, not offsets in the input PDF file or in Markdown. + +The manifest's `text_contract` declares the byte/range units, zero-based page +indices and geometry: `pdf_user_space`, points, `span_bbox` highlight precision, +and `display_transform:null`. Existing bbox semantics are unchanged. Neither +these boxes nor a multi-page union bbox are screen pixels. A viewer must apply +its own page rotation/crop/display transform; the contract does not supply one. +The origin can be native, OCR, specialist, or fallback text as its span's +provenance declares; these mappings do not prove source-image visibility, +complete page coverage, quotation verification, or absence of extraction loss. + +The neutral [JavaScript adapters](host-adapters.md) explicitly translate these +UTF-8 byte boundaries to JavaScript UTF-16 units. Their document text projection +joins exact block texts with two newlines, has its own hash and source map, and +remains separate from prompt truncation and whitespace-normalized chunks. Rich +search normalization in `src/text_search.zig` retains its existing reverse map; +these identity views add no second native normalization engine. + +### Reconciled ownership and recovery assessments (0.15) + +Each span has `reconciled_block_id`, a document-global block id or null if no +reconciled block owns it. Its `provenance.block_ids` agrees with that owner. +The older numeric `block_id` retains its layout-index meaning. + +Tables and nested cells have `reconciled_block_ids`. These list the reconciled +owners of their `source_span_ids` on the same page. Source spans are selected +by the existing span-center-inside-bbox rule; a bbox intersection alone does +not establish ownership. A spanning block can therefore belong to multiple +cells. Legacy table `block_index`, `block_count` and `provenance.block_ids` +retain pre-reconciliation indices and must not be used as authoritative links. +These additions establish source linkage, not table topology accuracy. + +Page and region route traces include `coverage` with `status`, `completeness`, +`native_span_count`, `recognized_span_count`, `requested_recovery`, +`recovery_state` and `unresolved_region_count`. Status is `text_available`, +`empty_unverified`, `recovery_required`, `recovery_failed` or `recovered`. +Recovery state is `not_required`, `not_invoked`, `completed`, `empty`, +`unavailable`, `failed`, `timeout` or `invalid_output`. + +`completeness` is always `unverified`. Empty output does not prove a blank +page; retained native text does not prove that image regions were recognized. +Selected OCR/formula attempts count as recovery only when retained recognized +spans intersect the assessed region. Successful OCR cannot hide unresolved +formula or disjoint image regions. Lifecycle `completed` means extraction +finished, not that the source has complete visual coverage. Paint/clipping +evidence and bounded regional OCR crops remain future work. + +### Streaming resource budgets + +`extract-adaptive --format stream-jsonl` accepts positive integer +`--max-output-bytes N` and `--max-page-memory-bytes N`. The first caps aggregate +serialized output; the second caps live requested allocation bytes in the +adaptive page allocator. Document caches/mappings, allocator overhead and +subprocess RSS are outside the page allocation budget. Batch formats and the +specialist-request sidecar prepass reject these flags. + +Budget failures return nonzero (`OutputLimitExceeded` or +`MemoryBudgetExceeded`). Partial records/pages may already have been emitted. +The entire `document_finished` record is reserved before writing, so an output +cap cannot leave a truncated success record. Accept a complete stream only +after valid lifecycle/count checks and successful process exit. + ### `debug_asset` Reference to visual review and coordinate artifacts. Debug assets are always @@ -341,14 +462,14 @@ When `--debug-assets-dir` is provided, page-scoped `debug_asset` records are emitted before that page's `page_finished` event. Document-scoped debug assets are emitted near the end of the stream before `document_finished`. -## Siftable Artifact Mapping +## Host Artifact Mapping `source_id` is intentionally neutral: callers can set it to a database id, object-store key, URL, CMS id, processing-run source id, or any other external identifier. The parser does not interpret it. `stream-jsonl` is designed to map cleanly onto host application artifact -pipelines, including Siftable-style `processing_runs` and `stage_artifacts`: +pipelines, including durable processing runs and stage artifacts: - `document_manifest` -> `manifest` or run summary, including input hash, parser version, page count, encryption/corruption flags, route totals, diff --git a/docs/parser-host-implementation-plan.md b/docs/parser-host-implementation-plan.md new file mode 100644 index 0000000..dcfed56 --- /dev/null +++ b/docs/parser-host-implementation-plan.md @@ -0,0 +1,171 @@ +# Parser, Research consumer, and Editorial consumer implementation plan + +Status: phases 0/1 implemented and locally verified; reconciled table links, bounded streaming and recovery assessments implemented in pdf-parser, host adoption pending, 2026-10-10. All current writes are restricted to pdf-parser: planned host edits below remain adoption instructions. Based on the [local reference review](local-reference-review-2026-10-09.md) and [host use-case review](host-usecase-reference-review-2026-10-10.md). Paint/clipping evidence, regional OCR crops, richer hierarchy/figures and host rollout remain planned. + +## Delivery order and ownership + +| Phase | pdf-parser owns | Research consumer owns | Editorial consumer owns | Dependency / exit gate | +| --- | --- | --- | --- | --- | +| 0: protect evidence | Remove MCID-only duplicate identity; span preservation and duplicate-provenance regressions | Translate normalized chunk matches back to raw UTF-16 source offsets; service persistence regressions | No immediate code change; current plain-text lane does not use adaptive reconciliation | Independent patches; red-to-green tests and fresh parser build | +| 1: source locator contract | Reuse search text views and reverse maps; populate chunk span references; document text and geometry units | Preserve source text identity; carry chunk origin ranges through composition; consume parser chunk/block references | Introduce a rich extraction result behind the existing string API; preserve source text separately from truncated prompt projections | Versioned artifact contract, Unicode/rotation and overlapping-chunk fixtures | +| 2: host adoption and work bounds | Deterministic run identity and stream lifecycle; exercise cancellation/resource-limit fixtures | Retain table geometry/continuation fields; bound process stdout and cancellation; reader links to exact evidence | Update parser revision/compiler/archive pins together; route reference/transcript consumers through rich results | Both host readers pass contract fixtures and process failure tests | +| 3: coverage and regional recovery | Per-page/region quality states; paint/clipping evidence; bounded regional OCR via existing specialist protocol | Persist unresolved coverage; apply explicit acceptance policy before fallback/retrieval | Surface unresolved pages in source intake and quote review | Mixed scan/native controls; no success-as-completeness inference | +| 4: richer document evidence | Explicit reconciled table/block linkage, continuation tests, section ancestry and figure/crop references | Map row/cell/figure locators into document intelligence and source reader | Add table/source inspection for civic packets and financial references | Exact values, topology, crop identity and source coverage holdouts | + +Implement phase 0 first. Agree on phase 1 contracts before changing either host's extraction defaults. Phase 2 output limits can proceed independently of new schema fields; coverage and richer evidence depend on the source locator contract. Model/vendor selection is a separate decision. + +## Phase 0: two bounded correctness fixes + +**Parser files:** `src/reconcile.zig`, `src/testpdf.zig`, `src/integration_test.zig`, plus ignored CLI reduction output from `benchmark/eval/reference_review_probes.py`. + +- An MCID groups content; it does not prove two spans are duplicates. Require the existing normalized-text equality and same-page geometry overlap for all duplicates. +- Regress distinct text with a shared MCID even at identical placement, repeated text at different placements, and genuine native/OCR duplicates with retained source masks/counts. +- Run the narrow MCID tests before and after the patch, the full Zig test/build gates, and the actual two-line tagged-PDF CLI reduction. Keep public schema 0.13.0 unchanged. + +**Research consumer files:** `src/services/researchSourceIndexService.ts` and its existing test file. + +- `startOffset`/`endOffset` remain half-open JavaScript UTF-16 offsets into the original persisted source text. Normalize only for matching, reverse-map boundaries through whitespace runs, and advance the cursor in raw source coordinates. +- Assert the values passed into chunk persistence using the real hybrid chunker for ordinary cases: exact text, repeated spaces, paragraph breaks, surrogate pairs, leading/trailing whitespace, tabs/CRLF/nonbreaking spaces. Controlled chunk output covers repeated occurrences and unavailable matches. +- Unmatched text retains null offsets. Preserve RLS, schema, chunk hashes/identities, embedding semantics, and forward cursor policy. Overlapping chunk origin tracking is phase 1; do not fabricate positions when that policy cannot resolve a chunk. +- Existing incorrect persisted offsets are not repaired automatically. Explicit rechunking updates matching chunk identities through the existing upsert; a separately scoped repair must validate source hashes and avoid triggering unintended embedding work. +- Run the service Jest lane and installed compiler's typecheck. Report Nx checkout collisions separately if they prevent Nx orchestration; do not modify unrelated checkouts to run this fix. + +Acceptance: both defect tests demonstrably fail on the original implementation and pass after the patch; true duplicates still merge; stored ranges cover the complete normalized chunk. These are local code/CLI/service-mock checks, not a database or deployed UI acceptance claim. + +## Phase 1: preserve text identity and exact source locations + +**Parser:** extend `src/text_search.zig`, `src/schema.zig`, `src/reconcile.zig`, `src/specialist_protocol.zig`, and `docs/output-schema.md`. Reuse the current reverse map rather than building another normalization engine. Populate `rag_chunk.source_span_ids` from owning blocks and spans. Specify source/input hash, decoded/view text identity, normalization flags, UTF-8 byte ranges, page indexing, and PDF-point geometry with rotation/crop transforms. Scope structural MCIDs by page/content stream or Form instance when structural identity is exposed; never restore MCID-only deduplication. + +**Research consumer:** update `researchSourceIndexService.ts`, `utils/chunking.ts`, `services/pdfParserProtocol.ts`, `services/pdfConversionService.ts`, and `services/documentIntelligenceExtractionService.ts`. Carry origin ranges from structural/semantic chunk composition, including overlap, instead of rediscovering repeated passages by text search. Convert UTF-8 boundaries to UTF-16 explicitly at the host boundary; reject invalid/out-of-range identities. Prefer parser chunk/block locators for PDF attachments, retaining the text-only chunk path for other sources. + +**Editorial consumer:** extend `server/lib/pdf-extract.ts` with a rich extraction API returning text, source/run identity, page/block/span locators and coverage. Keep `extractPdfText` as a compatibility projection. Update PDF consumers in `server/claude.ts`, `server/routes.ts`, and `server/domains/quotes/ingest.ts` to retain source evidence. Prompt limits and stored-document limits must be separate projections with explicit truncation; validate quote offsets against the exact retained text view. Parser evidence does not automatically verify a quotation. + +Gate: a shared small public/redacted fixture set covers ligatures, repeated phrases, collapsed whitespace, Unicode beyond BMP, overlapping chunks, rotated/cropped pages, and synthetic whitespace with no glyph evidence. Each returned range round-trips to the declared text view and source geometry; wrong input/text-view identities fail explicitly. Additive public fields require the documented schema increment and reader compatibility tests before host activation. + +Phase 1 delivery is documented in [host adapters](host-adapters.md) and the +[0.14 text/source-map contract](output-schema.md). Shared modules carry origins +through overlapping composition, validate rich extraction, convert byte ranges +to UTF-16 and link chunks to page/span evidence. The native contract preserves +exact identity text rather than introducing another normalization engine. +Span geometry is explicit; viewer transforms and structural MCID namespaces +are not newly exposed. Host database/UI adoption, deployments and backfills +remain pending; neither host checkout was edited in phase 1. + +## Phase 2: adopt existing artifacts and bound execution + +**Research consumer:** map existing table cell bbox, role, row/column spans, source spans and logical continuation IDs in `pdfConversionService.ts`; retain raw values alongside parsed hints. Add output byte budgets to its subprocess runner, then incremental JSONL persistence where warranted. Cancellation/timeout/oversized stdout must terminate and clean up the process and prevent a complete-run receipt. Retain current explicit fallback behavior until coverage policy is separately qualified. Test malformed/truncated JSONL, child exit, huge stdout, late output after cancellation and idempotent retries. + +**Editorial consumer:** change `scripts/ensure-pdf-parser.mjs` only with a tested parser revision + Zig 0.17 + archive identities as a pair. Preserve baseline CPU and runtime smoke checks. Expand `server/test/pdf-extract.test.ts`, reference intake tests and transcript/quote tests for both old string and new rich result consumers. Keep the current timeout/output bound and failure distinctions. Network archive fetching and deployment are separate authorized steps, not part of the current local patch. + +**Parser:** extend stream lifecycle and resource fixtures in `src/stream.zig`, native limit owners, `src/main.zig` and integration tests only where a missing limit is demonstrated. Distinguish decoded-stream, retained-memory and serialized-output budgets. Record parser build identity, options, specialist versions and terminal state in a deterministic receipt. + +Gate: no unbounded host accumulation; no partial/canceled run accepted as complete; both hosts retain stable input/artifact identity and can reproduce a locator. Run actual host tests and neutral parser JSONL fixtures before switching defaults. Roll back host adoption through the compatibility projection/accepted reader family; retain evidence from unsuccessful runs as incomplete. + +## Phase 3: coverage and regional recovery + +**Parser:** propagate text rendering mode, clipping/paint evidence and page completeness through native spans, layout, adaptive routing and schema. Distinguish verified blank, no selectable text, OCR unavailable/failed, low recognition quality, and unresolved region. Route bounded crops through existing OCR/specialist adapters outside the Zig kernel. Do not interpret outlined lettering as a verified blank page. + +**Hosts:** persist those states in Research consumer's parser ledger/reader and Editorial consumer's extraction result/intake review. Decide when to accept partial text, withhold retrieval, request review or invoke existing fallback. Keep native text, OCR and generated interpretations separately attributable. + +Gate: deterministic fixtures for blank pages, outlined glyphs, invisible OCR, clipping, scans mixed with native text, rotation, unavailable rasterizer/OCR, and partial specialist failure. Assert exact page enumeration and unresolved-region visibility. Use separate new held-out documents; do not retune the earlier failed reading-order holdout or promote its diagnostic graph. + +## Phase 4: tables, hierarchy and figures + +**Parser:** explicit post-reconciliation block/table relationships; tested cross-page continuation and section ancestry; source crop/figure records with hash, owner, bbox and transform. Extend existing table grid/debug tooling with broken-rule and borderless reductions informed by local pdfplumber patterns. Keep raw financial strings authoritative over numeric hints. + +**Research consumer:** expose cell/header/page relationships through document intelligence and `web/src/uix/sourceReaderBlock.tsx`; retain row labels, periods, units and footnotes. Show figures even when native text conversion succeeds. + +**Editorial consumer:** add source/table inspection within edition-scoped reference workflows, carrying existing quote-safety and publishing gates through richer evidence. + +Gate: exact assertions for parentheses negatives, blank versus zero, scale/units, footnotes, merged cells and repeated headers; inspect crop identity and geometry round-trips. Measure topology and source coverage separately from readable Markdown. Evaluate interpretation models only after these deterministic controls. + +## Validation and rollout discipline + +- Parser: selected Zig 0.17; narrow regressions, `zig fmt --check`, `zig build test --summary all`, `zig build --summary all`; adaptive corpus eval when behavior touches routing/layout/schema. Fast-build benchmark lanes qualify any later performance claim. +- Research consumer: installed package Jest lanes plus authoritative TypeScript 7 compiler; Nx targets where its project graph resolves. Protocol/process tests before any changed fallback or ingestion default; reader tests before locator UI claims. +- Editorial consumer: existing Vitest PDF/reference/quote/transcript lanes plus typecheck; paired toolchain bootstrap and deployed runtime/browser checks are later gates. +- Track each phase's source tests, runtime artifacts, deployed revision and product acceptance separately. Preserve unrelated dirty files; no raw private/large PDFs in Git. No deployment, paid inference, archive fetch or bulk backfill is included in phase 0. + +## Phase 0 execution evidence + +- Red: the original reconciler failed both preservation tests (expected two spans, got one), while the true duplicate control passed. The original host service failed six of the first seven added cases; its exact-text control passed. +- Green: all three focused MCID unit tests passed. A generated PDF integration test now verifies both lines, MCID preservation, Markdown order, artifact JSONL/provenance and retrieval chunk coverage. +- Parser gates: Zig 0.17.0; tracked Zig files pass formatting; `zig build test --summary all` passes all 62 steps after the final integration regression (2,101 tests executed in that run, other test steps cached). `zig build --summary all` passes all 12 steps. +- Host gates: research indexing, structural chunking and hybrid chunking Jest lanes pass all 33 tests across three suites; package typecheck passes with the authoritative TypeScript 7.0.2 compiler. Whitespace variants select source occurrences in order; an unavailable match retains null offsets without rewinding the raw cursor. +- Actual CLI evidence: [MCID reduction](../benchmark/eval/outputs/phase-0-fixes-2026-10-10/mcid-probe.json) retains `FIRST` and `SECOND` in native text, adaptive spans and chunks, with zero false duplicates. [Host function probe](../benchmark/eval/outputs/phase-0-fixes-2026-10-10/locator-probe.json) now covers the full source slice in all four original cases. These local evidence files are ignored. +- Nx orchestration was attempted but its graph failed on pre-existing duplicate project definitions in `.context` checkouts. Used the installed package Jest/typecheck scripts; no workspace cleanup or install was needed. +- Public schema and host database schema are unchanged. Existing persisted locators are untouched. Editorial consumer is planned only; no production host runtime/database/UI test, deployment, backfill, paid inference or performance qualification was performed. + +## Phase 1 execution evidence + +- All phase 1 writes are in pdf-parser. Host code was read to define adoption requirements; neither Editorial consumer nor Research consumer was edited. +- Schema 0.14.0 adds exact-text view hashes and half-open UTF-8 source maps. Construction records span byte origins, including repeated Unicode text. Chunk source-span ids and streaming map ids resolve to the emitted document-global records; synthetic separators have no span geometry. +- New neutral Node modules expose overlap-aware origins, validated rich extraction, separate prompt projection and hash-bound source-range lookup. Typed declarations pass the installed TypeScript 7 strict no-emit check; no dependency installation was needed. +- Node checks pass 9 chunk-origin tests, 57 extraction/protocol/process tests, and 2 actual CLI integration tests. Both real integration tests executed rather than skipped: generated repeated same-MCID lines across two pages, and checked-in complex financial artifacts. +- Zig formatting and `zig build test --summary all` pass all 62 steps (2,183 tests executed in the last full run, remaining steps cached). The final build passes all 12 steps, including shared-library artifacts. +- The adaptive corpus evaluation completed with local Tesseract/Poppler and the checked-in manifest. This is extraction evidence, not a comparative performance claim or a new page-coverage qualification. +- Actual CLI probes also consumed forms, rotated native text and a mixed native/scan fixture with OCR disabled. Their rich results keep page coverage unverified. Financial fixtures retain legacy specialist labels and report unresolved table/block provenance separately; those legacy references do not contribute to source locators. +- No host ingestion, database migration, corpus backfill, runtime switch, deployment or publishing occurred. Host wiring and explicit 0.14 reader acceptance remain adoption work outside this authorized write scope. + +## Review and remediation + +Observed on the current working tree: independent review reproduced weak source +identity validation and unbounded waits for wrapper descendants. Both now have +passing regressions; POSIX termination owns the process group and closes pipes, +and extraction validates the exact source id it supplied. A native allocation +failure regression also exposed detached block/chunk buffers; error cleanup now +passes all injected allocation failures. Explicit consumer names and repository +paths were removed from the added code and documentation. + +Observed checks: 73 JavaScript tests passed, both real CLI integrations executed, +strict declaration typecheck passed, Zig formatting passed, and the full Zig +test/build targets passed (62/62 and 12/12 steps). A source/binary-hash-bound +[verification receipt](../benchmark/eval/outputs/phase-1-review-2026-10-10/receipt.json) +retains the final public adapter/CLI readback. Windows process-tree cleanup and +consumer runtime adoption remain unverified. + +## Parser capability continuation + +Schema 0.15.0 implements three parser-side units from the later phases: + +- Authoritative reconciled span/table/cell owners use source-span membership, + including document-global streaming ids. Numeric layout compatibility fields + remain available; the reader validates exact ownership and same-page scope. +- Native streaming caps aggregate output and adaptive page allocations. A new + incremental page adapter applies page/record/count limits and serial callback + backpressure. Budget, cancellation, process and callback failures cannot + produce an accepted complete receipt. +- Page/region assessments report retained text and unresolved recovery. Selected + OCR/formula output must intersect the assessed region; failed or disabled + recovery remains visible alongside native text. Completeness stays unverified. + +Regressions exercise forged ownership, contradictory coverage, later-page source +locators, actual native allocation exhaustion, mixed native/scan extraction and +stream failure paths. The previous phase receipts describe historical hashes; +the continuation receipt records the current source/binary and gate results. + +Observed checks: all 128 JavaScript tests pass without skips, strict declaration +typechecking and Zig formatting pass, and native test/build targets pass all +62/62 and 12/12 steps. All 36 checked-in corpus documents pass both public batch +and streaming readers with OCR disabled. The adaptive evaluation also completes +all 36 cases with local OCR enabled. A mixed native/scan readback changes from +`recovery_required` to `recovered` when OCR is enabled, while keeping visual +completeness unverified. These runs qualify the exercised contracts, not +universal extraction accuracy or performance. + +Independent read-only review found forged span ownership, cross-page cells and +out-of-scope/contradictory coverage acceptance. All are rejected after remediation +and covered by permanent regressions. The hash-bound +[continuation receipt](../benchmark/eval/outputs/parser-capabilities-2026-10-10/receipt.json) +retains the gates, corpus results, actual OCR outcome and limitations. + +Next parser work is paint/clipping provenance and bounded regional OCR crops, +with outlined-lettering, invisible-text, clipped-text and mixed-region controls. +Host activation remains a separate adoption step with reader/persistence tests. +Table topology, cross-page continuation, hierarchy and figures need their own +holdouts; exact reconciled links alone do not qualify those capabilities. + +Inferred next priority from the financial fixture's three unresolved legacy +table/block references: implement exact reconciled table linkage with additive +fields, then bound streaming/retained memory, then expose page/region coverage. +These are recommendations; no subsequent phase was started by this review. diff --git a/integrations/javascript/chunk-origins.d.mts b/integrations/javascript/chunk-origins.d.mts new file mode 100644 index 0000000..5ceefa3 --- /dev/null +++ b/integrations/javascript/chunk-origins.d.mts @@ -0,0 +1,24 @@ +export type ChunkStrategy = 'legacy' | 'hybrid' | 'light'; +export type ChunkMetadata = { lineage?: string[]; [key: string]: unknown }; +export type ComposeChunksWithOriginsArgs = { + text: string; + strategy?: ChunkStrategy; + /** Four UTF-16 code units per token approximation; default 256. */ + maxTokens?: number; + /** Default 32; clamped below the structural window to guarantee progress. */ + overlapTokens?: number; + metadata?: ChunkMetadata; +}; +export type ChunkWithOrigin = { + index: number; + body: string; + /** Half-open offsets into the original, untrimmed input string. */ + startOffset: number; + endOffset: number; + offsetUnit: 'utf16_code_unit'; + /** SHA-256 of the original input, encoded as UTF-8. */ + sourceTextSha256: string; + normalization: 'collapse_whitespace_trim'; + metadata: ChunkMetadata & { lineage: string[] }; +}; +export function composeChunksWithOrigins(args: ComposeChunksWithOriginsArgs): ChunkWithOrigin[]; diff --git a/integrations/javascript/chunk-origins.mjs b/integrations/javascript/chunk-origins.mjs new file mode 100644 index 0000000..920b923 --- /dev/null +++ b/integrations/javascript/chunk-origins.mjs @@ -0,0 +1,160 @@ +import { createHash } from 'node:crypto'; + +const WHITESPACE = /\s/u; + +function safeBoundary(text, offset, direction) { + if (offset > 0 && offset < text.length && + /[\uD800-\uDBFF]/u.test(text[offset - 1]) && /[\uDC00-\uDFFF]/u.test(text[offset])) { + return offset + direction; + } + return offset; +} + +function trimRange(text, start, end) { + while (start < end && WHITESPACE.test(text[start])) start += 1; + while (end > start && WHITESPACE.test(text[end - 1])) end -= 1; + return { start, end }; +} + +// Runs keep normalization reversible without allocating one map entry per character. +function normalizeRange(text, start, end) { + const pieces = []; + const runs = []; + let length = 0; + const pattern = /\s+|\S+/gu; + const input = text.slice(start, end); + for (const match of input.matchAll(pattern)) { + const sourceStart = start + match.index; + const sourceEnd = sourceStart + match[0].length; + const whitespace = WHITESPACE.test(match[0][0]); + const body = whitespace ? ' ' : match[0]; + pieces.push(body); + runs.push({ start: length, end: length + body.length, sourceStart, sourceEnd, whitespace }); + length += body.length; + } + return { body: pieces.join(''), runs }; +} + +function sourceBoundary(runs, offset, isEnd) { + let low = 0; + let high = runs.length; + while (low < high) { + const middle = (low + high) >>> 1; + const beforeBoundary = isEnd ? runs[middle].end < offset : runs[middle].end <= offset; + if (beforeBoundary) low = middle + 1; + else high = middle; + } + const run = runs[low]; + if (!run) return runs.at(-1).sourceEnd; + if (run.whitespace) return isEnd ? run.sourceEnd : run.sourceStart; + return run.sourceStart + offset - run.start; +} + +function structuralRanges(text, start, end, maxChars, overlapChars) { + const ranges = []; + while (start < end) { + let stop = safeBoundary(text, Math.min(start + maxChars, end), -1); + if (stop < end) { + const threshold = start + overlapChars; + const paragraph = text.lastIndexOf('\n\n', stop); + const sentence = text.lastIndexOf('. ', stop - 1); + const word = text.lastIndexOf(' ', stop); + if (paragraph > threshold && paragraph <= stop) stop = paragraph; + else if (sentence > threshold) stop = sentence + 1; + else if (word > threshold) stop = word; + } + const trimmed = trimRange(text, start, stop); + if (trimmed.start < trimmed.end) ranges.push(trimmed); + if (stop === end) break; + start = safeBoundary(text, Math.max(stop - overlapChars, start + 1), 1); + while (start < end && WHITESPACE.test(text[start])) start += 1; + } + return ranges; +} + +function semanticRanges(body, maxTokens, overlapTokens) { + const sentences = []; + let start = 0; + for (const match of body.matchAll(/[.!?]+(?=\s|$)/gu)) { + const end = match.index + match[0].length; + const range = trimRange(body, start, end); + if (range.start < range.end) sentences.push(range); + start = end; + } + const last = trimRange(body, start, body.length); + if (last.start < last.end) sentences.push(last); + const estimate = ({ start, end }) => Math.ceil((end - start) / 4); + const ranges = []; + let first = 0; + while (first < sentences.length) { + let next = first; + let tokens = 0; + while (next < sentences.length && (next === first || tokens + estimate(sentences[next]) <= maxTokens)) { + tokens += estimate(sentences[next]); + next += 1; + } + ranges.push({ start: sentences[first].start, end: sentences[next - 1].end }); + if (next === sentences.length) break; + let retained = 0; + let newFirst = next; + while (overlapTokens > 0 && newFirst > first + 1 && retained < overlapTokens) { + newFirst -= 1; + retained += estimate(sentences[newFirst]); + } + first = newFirst; + } + return ranges; +} + +/** + * Compose chunks while retaining original UTF-16 ranges through every transform. + * Budgets use an explicit four UTF-16 units/token approximation, not a tokenizer. + * Structural overlap is clamped below the window size to guarantee advancement. + * The exact untrimmed input is the source of offsets and the SHA-256 (UTF-8 encoded). + */ +export function composeChunksWithOrigins({ + text, + strategy = 'hybrid', + maxTokens = 256, + overlapTokens = 32, + metadata = {}, +}) { + if (typeof text !== 'string') throw new TypeError('text must be a string'); + if (!text.isWellFormed()) throw new TypeError('text must contain well-formed Unicode'); + if (!['legacy', 'hybrid', 'light'].includes(strategy)) throw new TypeError('unsupported chunk strategy'); + if (!Number.isSafeInteger(maxTokens) || maxTokens < 1 || maxTokens > Math.floor(Number.MAX_SAFE_INTEGER / 4)) { + throw new RangeError('maxTokens must be a positive safe integer within the character budget'); + } + if (!Number.isSafeInteger(overlapTokens) || overlapTokens < 0) throw new RangeError('overlapTokens must be a nonnegative safe integer'); + if (!metadata || typeof metadata !== 'object' || Array.isArray(metadata)) throw new TypeError('metadata must be an object'); + if (metadata.lineage !== undefined && (!Array.isArray(metadata.lineage) || !metadata.lineage.every((entry) => typeof entry === 'string'))) { + throw new TypeError('metadata.lineage must be an array of strings'); + } + const inputRange = trimRange(text, 0, text.length); + if (inputRange.start === inputRange.end) return []; + const sourceTextSha256 = createHash('sha256').update(text, 'utf8').digest('hex'); + const maxChars = maxTokens * 4; + const overlapChars = Math.min(overlapTokens, maxTokens) * 4; + const structural = structuralRanges(text, inputRange.start, inputRange.end, maxChars, Math.min(overlapChars, maxChars - 1)); + const result = []; + structural.forEach((range, structuralIndex) => { + const view = normalizeRange(text, range.start, range.end); + const segments = strategy === 'hybrid' + ? semanticRanges(view.body, maxTokens, overlapTokens) + : [{ start: 0, end: view.body.length }]; + segments.forEach((segment, segmentIndex) => { + const index = result.length; + result.push({ + index, + body: view.body.slice(segment.start, segment.end), + startOffset: sourceBoundary(view.runs, segment.start, false), + endOffset: sourceBoundary(view.runs, segment.end, true), + offsetUnit: 'utf16_code_unit', + sourceTextSha256, + normalization: 'collapse_whitespace_trim', + metadata: { ...metadata, lineage: [...(metadata.lineage ?? []), `structural:${structuralIndex}`, `segment:${segmentIndex}`, `chunk:${index}`] }, + }); + }); + }); + return result; +} diff --git a/integrations/javascript/chunk-origins.test.mjs b/integrations/javascript/chunk-origins.test.mjs new file mode 100644 index 0000000..1d929f9 --- /dev/null +++ b/integrations/javascript/chunk-origins.test.mjs @@ -0,0 +1,105 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import test from 'node:test'; +import { composeChunksWithOrigins } from './chunk-origins.mjs'; + +const normalize = (text) => text.replace(/\s+/gu, ' ').trim(); +function assertOrigins(text, chunks) { + const hash = createHash('sha256').update(text, 'utf8').digest('hex'); + for (const [index, chunk] of chunks.entries()) { + assert.equal(chunk.index, index); + assert.equal(normalize(text.slice(chunk.startOffset, chunk.endOffset)), chunk.body); + assert.equal(chunk.sourceTextSha256, hash); + assert.equal(chunk.offsetUnit, 'utf16_code_unit'); + assert.ok(chunk.startOffset >= 0 && chunk.endOffset <= text.length && chunk.endOffset > chunk.startOffset); + assert.ok(!(/[\uD800-\uDBFF]/u.test(text[chunk.startOffset - 1] ?? '') && /[\uDC00-\uDFFF]/u.test(text[chunk.startOffset] ?? ''))); + assert.ok(!(/[\uD800-\uDBFF]/u.test(text[chunk.endOffset - 1] ?? '') && /[\uDC00-\uDFFF]/u.test(text[chunk.endOffset] ?? ''))); + } +} + +test('retains original leading whitespace, collapsed runs, and untrimmed source identity', () => { + const text = ' \tA πŸ˜€\n\nB C\r\n'; + const chunks = composeChunksWithOrigins({ text }); + assert.equal(chunks.length, 1); + assert.equal(chunks[0].startOffset, 3); + assert.equal(chunks[0].body, 'A πŸ˜€ B C'); + assertOrigins(text, chunks); +}); + +test('overlapping origins stay correct when an end cursor cannot locate the next chunk', () => { + const text = 'one two three four five six seven eight nine ten eleven twelve'; + const chunks = composeChunksWithOrigins({ text, maxTokens: 6, overlapTokens: 2, strategy: 'light' }); + assert.ok(chunks.length > 2); + assert.ok(chunks.some((chunk, index) => index > 0 && chunk.startOffset < chunks[index - 1].endOffset)); + assert.ok(chunks.some((chunk, index) => index > 0 && text.indexOf(chunk.body, chunks[index - 1].endOffset) === -1)); + assertOrigins(text, chunks); +}); + +test('repeated paragraphs keep their actual distinct origins', () => { + const text = ' Repeat this phrase.\n\nRepeat this phrase.\n\nRepeat this phrase.'; + const chunks = composeChunksWithOrigins({ text, maxTokens: 5, overlapTokens: 0, strategy: 'legacy' }); + assert.deepEqual(chunks.map((chunk) => chunk.body), Array(3).fill('Repeat this phrase.')); + assert.deepEqual(chunks.map((chunk) => chunk.startOffset), [2, 23, 44]); + assertOrigins(text, chunks); +}); + +test('zero overlap does not acquire a minimum overlap window', () => { + const text = 'alpha beta gamma delta epsilon zeta eta theta'; + const chunks = composeChunksWithOrigins({ text, maxTokens: 4, overlapTokens: 0 }); + for (let index = 1; index < chunks.length; index += 1) assert.ok(chunks[index].startOffset >= chunks[index - 1].endOffset); + assertOrigins(text, chunks); +}); + +test('astral Unicode remains intact at forced split and overlap boundaries', () => { + const text = 'aπŸ˜€πŸ˜€πŸ˜€πŸ˜€πŸ˜€πŸ˜€πŸ˜€πŸ˜€πŸ˜€πŸ˜€z'; + for (const overlapTokens of [0, 1, 2, 50]) { + const chunks = composeChunksWithOrigins({ text, maxTokens: 2, overlapTokens }); + assert.ok(chunks.length < text.length); + assert.equal(chunks.at(-1).endOffset, text.length); + assertOrigins(text, chunks); + } +}); + +test('oversized sentences are forcibly sliced and oversized overlap terminates deterministically', () => { + const text = 'abcdefghijklmnopqrstuvwx'.repeat(8) + '.'; + for (const overlapTokens of [0, 2, 4, 999]) { + const args = { text, maxTokens: 4, overlapTokens, strategy: 'hybrid' }; + const chunks = composeChunksWithOrigins(args); + assert.ok(chunks.length <= text.length); + assert.ok(chunks.every((chunk) => chunk.body.length <= 16)); + assert.equal(chunks.at(-1).endOffset, text.length); + assert.deepEqual(chunks, composeChunksWithOrigins(args)); + assertOrigins(text, chunks); + } +}); + +test('semantic sentence ranges preserve punctuation and do not search for repeated content', () => { + const text = ' Hi. Hi. Hi. Hi. Hi. Hi. !!! '; + const metadata = { lineage: ['attachment:1'], source: 'file' }; + const chunks = composeChunksWithOrigins({ text, maxTokens: 5, overlapTokens: 1, metadata }); + assert.ok(chunks.length > 1); + assertOrigins(text, chunks); + assert.equal(chunks.at(-1).body.endsWith('!!!'), true); + assert.deepEqual(metadata.lineage, ['attachment:1']); + assert.ok(chunks.every((chunk) => chunk.metadata.lineage[0] === 'attachment:1')); +}); + +test('hybrid sentence overlap retains source ranges inside one structural window', () => { + const text = 'A. B. C. D. E. F. G. H. I. J.'; + const chunks = composeChunksWithOrigins({ text, maxTokens: 4, overlapTokens: 1 }); + assertOrigins(text, chunks); + const siblings = chunks.filter((chunk) => chunk.metadata.lineage.includes('structural:0')); + assert.equal(siblings.length, 2); + assert.equal(siblings[0].body, 'A. B. C. D.'); + assert.equal(siblings[1].body, 'D. E.'); + assert.ok(siblings[1].startOffset < siblings[0].endOffset); +}); + +test('empty input and invalid budgets have explicit outcomes', () => { + for (const text of ['', ' \n\t\r ']) assert.deepEqual(composeChunksWithOrigins({ text }), []); + for (const maxTokens of [0, -1, Infinity, NaN, 1.5]) assert.throws(() => composeChunksWithOrigins({ text: 'x', maxTokens }), RangeError); + for (const overlapTokens of [-1, Infinity, 1.5]) assert.throws(() => composeChunksWithOrigins({ text: 'x', overlapTokens }), RangeError); + assert.throws(() => composeChunksWithOrigins({ text: 'x', strategy: 'unknown' }), TypeError); + assert.throws(() => composeChunksWithOrigins({ text: null }), TypeError); + assert.throws(() => composeChunksWithOrigins({ text: '\ud800' }), TypeError); +}); diff --git a/integrations/javascript/extract.d.mts b/integrations/javascript/extract.d.mts new file mode 100644 index 0000000..a0bef6b --- /dev/null +++ b/integrations/javascript/extract.d.mts @@ -0,0 +1,178 @@ +export type ExtractionErrorCode = 'protocol' | 'unavailable' | 'timeout' | 'output_limit' | 'memory_limit' | 'aborted' | 'crashed' | 'document' | 'cleanup' | 'callback'; +export class PdfExtractionError extends Error { + readonly code: ExtractionErrorCode; + cleanupError?: PdfExtractionError; + constructor(code: ExtractionErrorCode, message: string, options?: ErrorOptions); +} +export interface ExtractionOptions { + /** Host-owned executable selection. No environment or default path discovery. */ + binaryPath: string; + timeoutMs?: number; + maxOutputBytes?: number; + signal?: AbortSignal; + /** Defaults to false. Requires host-installed OCR tools when enabled. */ + enableOcr?: boolean; +} +export interface TextView { + text_view_id: string; + kind: 'reconciled_span' | 'reconciled_block' | 'rag_chunk'; + offset_unit: 'utf8_byte'; + range_convention: 'half_open'; + normalization: 'none'; + text_sha256: string; +} +export interface Provenance { + document_id: string; + source_id: string | null; + input_sha256: string; + /** Specialist compatibility ids name a class. Their record-owned request_id, + * attempt_id, response_id or specialist_result_id provides unique identity. */ + artifact_id: string; + span_ids?: string[]; + block_ids?: string[]; + chunk_ids?: string[]; + [key: string]: unknown; +} +export interface ArtifactRecord { + schema_name: string; + schema_version: string; + record_type: string; + document_id: string; + source_id: string | null; + provenance: Provenance; + [key: string]: unknown; +} +export interface ManifestRecord extends ArtifactRecord { + record_type: 'document_manifest'; + page_count: number; + input_sha256: string; + artifact_counts: { spans: number; blocks: number; rag_chunks: number; [key: string]: unknown }; + text_contract: { offset_unit: 'utf8_byte'; range_convention: 'half_open'; page_index_base: 0; + geometry: { coordinate_space: 'pdf_user_space'; unit: 'point'; highlight_precision: 'span_bbox'; display_transform: null } }; +} +/** Unchanged PDF-space bbox; these are not browser or screen coordinates. */ +export interface PdfBBox { x0: number; y0: number; x1: number; y1: number } +export interface SpanRecord extends ArtifactRecord { + record_type: 'span'; + span_id: string; + reconciled_block_id?: string | null; + page_index: number; + bbox: PdfBBox; + text: string; + text_view: TextView; +} +export interface ByteSourceSegment { + text_start_byte: number; + text_end_byte: number; + source_start_byte: number | null; + source_end_byte: number | null; +} +export interface BlockRecord extends ArtifactRecord { + record_type: 'block'; + block_id: string; + page_index: number; + bbox: PdfBBox; + text: string; + text_view: TextView; + source_map: (ByteSourceSegment & { source_span_id: string | null })[]; +} +export interface ChunkRecord extends ArtifactRecord { + record_type: 'rag_chunk'; + chunk_id: string; + content: string; + page_start: number; + page_end: number; + source_span_ids: string[]; + source_block_ids: string[]; + text_view: TextView; + source_map: (ByteSourceSegment & { source_block_id: string | null })[]; +} +export interface SourceTextSegment { + /** Half-open UTF-16 positions in RichPdfDocument.sourceText. */ + start: number; + end: number; + sourceBlockId: string | null; + sourceSpanId: string | null; + pageIndex: number | null; + bbox: PdfBBox | null; + /** Half-open UTF-16 positions in the referenced span; null for synthetic text. */ + sourceStart: number | null; + sourceEnd: number | null; +} +export interface RichPdfDocument { + rawJsonl: string; + records: ArtifactRecord[]; + manifest: ManifestRecord; + inputSha256: string; + spans: SpanRecord[]; + blocks: BlockRecord[]; + chunks: ChunkRecord[]; + tables: TableRecord[]; + pageOutcomes: PageOutcome[]; + /** Exact block texts in artifact order, joined by synthetic two-newline separators. */ + sourceText: string; + sourceTextSha256: string; + sourceMap: SourceTextSegment[]; + /** Legacy table block refs can predate reconciliation. These refs are + * unverified metadata and never contribute to source-text locators. */ + unresolvedProvenanceLinks: { recordType: 'table'; artifactId: string; field: 'block_ids'; id: string }[]; + offsetUnit: 'utf16_code_unit'; + rangeConvention: 'half_open'; + normalization: 'none'; + /** A valid map proves text identity, not full document extraction. */ + pageCoverage: 'unverified'; +} +export interface TableCellRecord { + cell_id: string; + page_index: number; + text: string; + raw_text: string; + row: number; + column: number; + rowspan: number; + colspan: number; + role: string; + bbox: PdfBBox; + source_span_ids: string[]; + reconciled_block_ids?: string[]; + [key: string]: unknown; +} +export interface TableRecord extends ArtifactRecord { + table_id: string; + page_index: number; + source_span_ids: string[]; + /** Authoritative 0.15 links; legacy layout/provenance ids remain separate. */ + reconciled_block_ids?: string[]; + rows: { cells: TableCellRecord[]; [key: string]: unknown }[]; +} +export interface PageOutcome { + pageIndex: number; + status?: 'text_available' | 'empty_unverified' | 'recovery_required' | 'recovery_failed' | 'recovered'; + completeness?: 'unverified'; + recovery_state?: 'not_required' | 'not_invoked' | 'completed' | 'empty' | 'unavailable' | 'failed' | 'timeout' | 'invalid_output'; + requested_recovery?: boolean; + native_span_count?: number; + recognized_span_count?: number; + unresolved_region_count?: number; +} +export function utf8BoundaryMap(text: string): Map; +export function parseArtifactJsonl(rawJsonl: string, options: { inputSha256: string; expectedSourceId?: string | null }): RichPdfDocument; +export function extractPdfDocument(buffer: Uint8Array, options: ExtractionOptions): Promise; +/** Uses the adaptive block projection; native extract whitespace can differ. */ +export function extractPdfText(buffer: Uint8Array, options: ExtractionOptions): Promise; +export function locateSourceRange(document: RichPdfDocument, startOffset: number, endOffset: number, options: { sourceTextSha256: string }): { + startOffset: number; + endOffset: number; + sourceTextSha256: string; + offsetUnit: 'utf16_code_unit'; + rangeConvention: 'half_open'; + segments: SourceTextSegment[]; +}; +export function projectPromptText(document: RichPdfDocument, maxUtf16Units: number): { + text: string; + truncated: boolean; + sourceTextSha256: string; + sourceStart: 0; + sourceEnd: number; + offsetUnit: 'utf16_code_unit'; +}; diff --git a/integrations/javascript/extract.mjs b/integrations/javascript/extract.mjs new file mode 100644 index 0000000..8c005ca --- /dev/null +++ b/integrations/javascript/extract.mjs @@ -0,0 +1,389 @@ +import { spawn } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { terminateSubprocess } from './subprocess.mjs'; + +export class PdfExtractionError extends Error { + constructor(code, message, options) { + super(message, options); + this.name = 'PdfExtractionError'; + this.code = code; + } +} + +const fail = (message) => { throw new PdfExtractionError('protocol', message); }; +const hash = (value) => createHash('sha256').update(value).digest('hex'); +const integer = (value) => Number.isSafeInteger(value) && value >= 0; +const object = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); + +/** Converts UTF-8 byte boundaries to JS UTF-16 positions without rounding. */ +export function utf8BoundaryMap(text) { + if (typeof text !== 'string' || !text.isWellFormed()) fail('Text is not well-formed Unicode'); + const positions = new Map([[0, 0]]); + let byte = 0; + let unit = 0; + for (const character of text) { + byte += Buffer.byteLength(character); + unit += character.length; + positions.set(byte, unit); + } + return positions; +} + +function range(mapping, start, end, label) { + if (!integer(start) || !integer(end) || end < start || !mapping.has(start) || !mapping.has(end)) { + fail(`Invalid UTF-8 range for ${label}`); + } + return [mapping.get(start), mapping.get(end)]; +} + +const textOf = (record) => record.record_type === 'rag_chunk' ? record.content : record.text; +const idOf = (record) => record.span_id ?? record.block_id ?? record.chunk_id; + +function checkTextView(record, expectedKind) { + const text = textOf(record); + const boundaries = utf8BoundaryMap(text); + const view = record.text_view; + if (!object(view) || view.text_view_id !== `${idOf(record)}:text` || view.kind !== expectedKind || + view.offset_unit !== 'utf8_byte' || view.range_convention !== 'half_open' || + view.normalization !== 'none' || view.text_sha256 !== hash(text)) fail('Invalid text_view identity or hash'); + return boundaries; +} + +function checkSourceMap(record, sources, sourceField, boundaries) { + const text = textOf(record); + if (!Array.isArray(record.source_map)) fail('Missing source_map'); + let cursor = 0; + const referenced = new Set(); + for (const segment of record.source_map) { + if (!object(segment) || segment.text_start_byte !== cursor || segment.text_end_byte <= cursor) fail('Source map is not ordered full coverage'); + const [start, end] = range(boundaries, segment.text_start_byte, segment.text_end_byte, idOf(record)); + const sourceId = segment[sourceField]; + if (sourceId === null) { + if (segment.source_start_byte !== null || segment.source_end_byte !== null) fail('Synthetic map has source offsets'); + if (!/^\s*$/u.test(text.slice(start, end))) fail('Synthetic map contains non-separator text'); + } else { + const source = sources.get(sourceId); + if (!source) fail(`Unresolved ${sourceField}`); + if (record.record_type === 'block' && source.record.page_index !== record.page_index) fail('Block source span page mismatch'); + if (record.record_type === 'rag_chunk' && (source.record.page_index < record.page_start || source.record.page_index > record.page_end)) fail('Chunk source block page mismatch'); + const [sourceStart, sourceEnd] = range(source.boundaries, segment.source_start_byte, segment.source_end_byte, sourceId); + if (text.slice(start, end) !== textOf(source.record).slice(sourceStart, sourceEnd)) fail('Source map slice mismatch'); + referenced.add(sourceId); + } + cursor = segment.text_end_byte; + } + if (cursor !== Buffer.byteLength(text)) fail('Source map does not cover text'); + return referenced; +} + +function checkIdentitySet(ids, expected, label) { + if (!Array.isArray(ids) || ids.length !== expected.size || new Set(ids).size !== ids.length || ids.some((id) => !expected.has(id))) fail(`Inconsistent ${label}`); +} + +/** Validates batch JSONL before projecting any host-facing source positions. */ +export function parseArtifactJsonl(rawJsonl, { inputSha256, expectedSourceId } = {}) { + if (typeof rawJsonl !== 'string' || !/^[a-f0-9]{64}$/.test(inputSha256 ?? '')) fail('An input SHA-256 is required'); + const lines = rawJsonl.split('\n'); + if (lines.at(-1) === '') lines.pop(); + const records = lines.map((line) => { + try { return JSON.parse(line); } catch { fail('Malformed artifact JSONL'); } + }); + const manifest = records[0]; + if (!object(manifest) || manifest.record_type !== 'document_manifest') fail('Expected manifest first'); + if (!/^0\.(14|15)\.\d+$/.test(manifest.schema_version ?? '') || manifest.schema_name !== 'document_manifest' || + typeof manifest.document_id !== 'string' || !manifest.document_id || !integer(manifest.page_count) || + manifest.input_sha256 !== inputSha256) fail('Unsupported schema or manifest input identity'); + if (!(manifest.source_id === null || (typeof manifest.source_id === 'string' && manifest.source_id.length > 0)) || + (expectedSourceId !== undefined && manifest.source_id !== expectedSourceId)) fail('Invalid or mismatched source identity'); + const contract = manifest.text_contract; + if (!object(contract) || contract.offset_unit !== 'utf8_byte' || contract.range_convention !== 'half_open' || + contract.page_index_base !== 0 || contract.geometry?.coordinate_space !== 'pdf_user_space' || + contract.geometry.unit !== 'point' || contract.geometry.highlight_precision !== 'span_bbox' || + contract.geometry.display_transform !== null) fail('Unsupported manifest text contract'); + const artifactIds = new Set(); + const specialistIdFields = { specialist_request: 'request_id', specialist_attempt: 'attempt_id', + specialist_response: 'response_id', specialist_result: 'specialist_result_id' }; + for (const record of records) { + const provenance = record?.provenance; + // Specialist compatibility provenance ids identify a class, not a record. + // Their document-global record-owned ids are the unique identity instead. + const idField = specialistIdFields[record?.record_type]; + const identity = idField ? record[idField] : provenance?.artifact_id; + // 0.14 chunks carried a default internal label when the caller supplied no + // source identity. Only provenance binds those legacy chunks to the source. + const legacyChunkSource = manifest.schema_version.startsWith('0.14.') && manifest.source_id === null && + record?.record_type === 'rag_chunk' && typeof record.source_id === 'string' && record.source_id.length > 0; + if (!object(record) || typeof record.record_type !== 'string' || record.schema_name !== record.record_type || record.schema_version !== manifest.schema_version || + record.document_id !== manifest.document_id || (!legacyChunkSource && record.source_id !== manifest.source_id) || + !object(provenance) || provenance.document_id !== manifest.document_id || + provenance.source_id !== manifest.source_id || provenance.input_sha256 !== inputSha256 || + typeof provenance.artifact_id !== 'string' || !provenance.artifact_id || + typeof identity !== 'string' || !identity || artifactIds.has(identity)) { + fail('Inconsistent record provenance or duplicate artifact id'); + } + artifactIds.add(identity); + } + if (records.slice(1).some((record) => record.record_type === 'document_manifest')) fail('Duplicate manifest'); + const spans = records.filter((record) => record.record_type === 'span'); + const blocks = records.filter((record) => record.record_type === 'block'); + const chunks = records.filter((record) => record.record_type === 'rag_chunk'); + const index = (items, kind, prefix) => new Map(items.map((record) => { + const id = idOf(record); + if (typeof id !== 'string' || !new RegExp(`^${prefix}-\\d+$`).test(id) || record.provenance.artifact_id !== id) fail('Invalid artifact identity'); + if (kind !== 'rag_chunk' && (!integer(record.page_index) || record.page_index >= manifest.page_count)) fail('Invalid page index'); + if (kind !== 'rag_chunk') { + const box = record.bbox; + if (!object(box) || !['x0', 'y0', 'x1', 'y1'].every((key) => Number.isFinite(box[key])) || + box.x0 > box.x1 || box.y0 > box.y1) fail('Invalid PDF-space bbox'); + } + return [id, { record, boundaries: checkTextView(record, kind) }]; + })); + const spanIndex = index(spans, 'reconciled_span', 'span'); + const blockIndex = index(blocks, 'reconciled_block', 'block'); + const chunkIndex = index(chunks, 'rag_chunk', 'chunk'); + const unresolvedProvenanceLinks = []; + for (const [field, items] of [['spans', spans], ['blocks', blocks], ['rag_chunks', chunks]]) { + if (manifest.artifact_counts?.[field] !== items.length) fail(`Manifest ${field} count mismatch`); + } + for (const [field, type] of [['tables', 'table'], ['form_fields', 'form_field'], ['route_traces', 'route_trace'], + ['specialist_requests', 'specialist_request'], ['specialist_attempts', 'specialist_attempt'], + ['specialist_responses', 'specialist_response'], ['specialist_results', 'specialist_result'], ['debug_assets', 'debug_asset']]) { + if (field in manifest.artifact_counts && manifest.artifact_counts[field] !== records.filter((record) => record.record_type === type).length) fail(`Manifest ${field} count mismatch`); + } + for (const record of records) { + for (const [field, sourceIndex] of [['span_ids', spanIndex], ['block_ids', blockIndex], ['chunk_ids', chunkIndex]]) { + const ids = record.provenance[field]; + if (ids === undefined) continue; + if (!Array.isArray(ids) || ids.some((id) => typeof id !== 'string')) fail(`Invalid provenance ${field}`); + for (const id of ids) { + if (sourceIndex.has(id)) continue; + // Legacy table block refs precede reconciliation and can name blocks + // absent from public artifacts. Retain and diagnose, never use in maps. + if (record.record_type === 'table' && field === 'block_ids') { + unresolvedProvenanceLinks.push({ recordType: record.record_type, artifactId: record.provenance.artifact_id, field, id }); + } else fail(`Unresolved provenance ${field}`); + } + } + } + for (const block of blocks) { + const ids = checkSourceMap(block, spanIndex, 'source_span_id', blockIndex.get(block.block_id).boundaries); + checkIdentitySet(block.provenance.span_ids, ids, 'block span provenance'); + checkIdentitySet(block.provenance.block_ids, new Set([block.block_id]), 'block identity provenance'); + } + for (const chunk of chunks) { + if (!integer(chunk.page_start) || !integer(chunk.page_end) || chunk.page_end < chunk.page_start || chunk.page_end >= manifest.page_count) fail('Invalid chunk pages'); + const blockIds = checkSourceMap(chunk, blockIndex, 'source_block_id', chunkIndex.get(chunk.chunk_id).boundaries); + const spanIds = new Set([...blockIds].flatMap((id) => blockIndex.get(id).record.provenance.span_ids)); + checkIdentitySet(chunk.source_block_ids, blockIds, 'chunk source_block_ids'); + checkIdentitySet(chunk.source_span_ids, spanIds, 'chunk source_span_ids'); + checkIdentitySet(chunk.provenance.block_ids, blockIds, 'chunk block provenance'); + checkIdentitySet(chunk.provenance.span_ids, spanIds, 'chunk span provenance'); + checkIdentitySet(chunk.provenance.chunk_ids, new Set([chunk.chunk_id]), 'chunk identity provenance'); + } + const tables = records.filter(record => record.record_type === 'table'); + if (manifest.schema_version.startsWith('0.15.')) { + const owners = new Map(); + for (const block of blocks) for (const id of block.provenance.span_ids) { + if (!owners.has(id)) owners.set(id, new Set()); + owners.get(id).add(block.block_id); + } + for (const span of spans) { + const expected = owners.get(span.span_id) ?? new Set(); + if (expected.size > 1 || span.reconciled_block_id !== ([...expected][0] ?? null)) fail('Invalid reconciled span ownership'); + checkIdentitySet(span.provenance.block_ids, expected, 'span block provenance'); + checkIdentitySet(span.provenance.span_ids, new Set([span.span_id]), 'span identity provenance'); + } + const validateLinks = record => { + if (!object(record) || !Array.isArray(record.source_span_ids) || !integer(record.page_index) || record.page_index >= manifest.page_count) fail('Invalid table source evidence'); + const expected = new Set(); + for (const id of record.source_span_ids) { + const span = spanIndex.get(id)?.record; + if (!span || span.page_index !== record.page_index) fail('Unresolved or cross-page table span'); + for (const owner of owners.get(id) ?? []) expected.add(owner); + } + checkIdentitySet(record.reconciled_block_ids, expected, 'reconciled table block linkage'); + }; + for (const table of tables) { + validateLinks(table); + if (!Array.isArray(table.rows)) fail('Invalid table rows'); + for (const row of table.rows) { + if (!object(row) || !Array.isArray(row.cells)) fail('Invalid table cells'); + for (const cell of row.cells) { + validateLinks(cell); + if (cell.page_index !== table.page_index) fail('Cell page does not match its table'); + } + } + } + } + const pageOutcomes = records.filter(record => record.record_type === 'route_trace' && record.trace_kind === 'page_route') + .map(record => ({ pageIndex: record.page_index, ...record.coverage })); + if (manifest.schema_version.startsWith('0.15.')) { + const statuses = new Set(['text_available', 'empty_unverified', 'recovery_required', 'recovery_failed', 'recovered']); + const recoveryStates = new Set(['not_required', 'not_invoked', 'completed', 'empty', 'unavailable', 'failed', 'timeout', 'invalid_output']); + const failedStates = new Set(['unavailable', 'failed', 'timeout', 'invalid_output']); + const pageRoutes = new Set(); + for (const record of records.filter(record => record.record_type === 'route_trace' && ['page_route', 'region_route'].includes(record.trace_kind))) { + const assessment = record.coverage; + if (!integer(record.page_index) || record.page_index >= manifest.page_count || + !object(assessment) || !statuses.has(assessment.status) || assessment.completeness !== 'unverified' || + !recoveryStates.has(assessment.recovery_state) || typeof assessment.requested_recovery !== 'boolean' || + !['native_span_count', 'recognized_span_count', 'unresolved_region_count'].every(key => integer(assessment[key]))) fail('Invalid coverage assessment'); + if (record.trace_kind === 'page_route') { + if (pageRoutes.has(record.page_index)) fail('Duplicate page coverage assessment'); + pageRoutes.add(record.page_index); + } + if (!assessment.requested_recovery) { + if (!['text_available', 'empty_unverified'].includes(assessment.status) || assessment.recovery_state !== 'not_required' || assessment.unresolved_region_count !== 0) fail('Inconsistent coverage assessment'); + if (assessment.status === 'empty_unverified' && (assessment.native_span_count || assessment.recognized_span_count)) fail('Empty coverage has retained text'); + } else if (assessment.recovery_state === 'not_required' || !['recovery_required', 'recovery_failed', 'recovered'].includes(assessment.status) || + (assessment.status === 'recovery_failed') !== failedStates.has(assessment.recovery_state) || + (assessment.status === 'recovered' && (assessment.recovery_state !== 'completed' || assessment.recognized_span_count === 0 || assessment.unresolved_region_count !== 0))) fail('Inconsistent recovery assessment'); + } + } + const sourceMap = []; + const texts = []; + let position = 0; + for (const block of blocks) { + if (texts.length) { + sourceMap.push({ start: position, end: position + 2, sourceBlockId: null, sourceSpanId: null, pageIndex: null, bbox: null, sourceStart: null, sourceEnd: null }); + position += 2; + } + const boundaries = blockIndex.get(block.block_id).boundaries; + for (const segment of block.source_map) { + const [start, end] = range(boundaries, segment.text_start_byte, segment.text_end_byte, block.block_id); + const span = spanIndex.get(segment.source_span_id); + const spanRange = span ? range(span.boundaries, segment.source_start_byte, segment.source_end_byte, segment.source_span_id) : [null, null]; + sourceMap.push({ start: position + start, end: position + end, sourceBlockId: block.block_id, + sourceSpanId: segment.source_span_id, pageIndex: block.page_index, + bbox: span?.record.bbox ?? null, sourceStart: spanRange[0], sourceEnd: spanRange[1] }); + } + texts.push(block.text); + position += block.text.length; + } + const sourceText = texts.join('\n\n'); + return { rawJsonl, records, manifest, inputSha256, spans, blocks, chunks, tables, pageOutcomes, sourceText, + sourceTextSha256: hash(sourceText), sourceMap, unresolvedProvenanceLinks, offsetUnit: 'utf16_code_unit', + rangeConvention: 'half_open', normalization: 'none', pageCoverage: 'unverified' }; +} + +/** Prompt projection does not modify the authoritative source text or locators. */ +export function projectPromptText(document, maxUtf16Units) { + if (!integer(maxUtf16Units)) throw new RangeError('maxUtf16Units must be a nonnegative safe integer'); + let end = Math.min(document.sourceText.length, maxUtf16Units); + if (end > 0 && end < document.sourceText.length && /[\uD800-\uDBFF]/.test(document.sourceText[end - 1]) && /[\uDC00-\uDFFF]/.test(document.sourceText[end])) end--; + return { text: document.sourceText.slice(0, end), truncated: end < document.sourceText.length, sourceTextSha256: document.sourceTextSha256, sourceStart: 0, sourceEnd: end, offsetUnit: 'utf16_code_unit' }; +} + +/** Joins half-open source-text positions to span geometry, including explicit gaps. */ +export function locateSourceRange(document, startOffset, endOffset, { sourceTextSha256 } = {}) { + if (sourceTextSha256 !== document.sourceTextSha256 || hash(document.sourceText) !== sourceTextSha256) fail('Source text hash mismatch'); + const boundary = offset => offset <= document.sourceText.length && !(offset > 0 && offset < document.sourceText.length && + /[\uD800-\uDBFF]/.test(document.sourceText[offset - 1]) && /[\uDC00-\uDFFF]/.test(document.sourceText[offset])); + if (!integer(startOffset) || !integer(endOffset) || endOffset < startOffset || + !boundary(startOffset) || !boundary(endOffset)) fail('Invalid UTF-16 source range'); + const segments = []; + for (const segment of document.sourceMap) { + const start = Math.max(startOffset, segment.start); + const end = Math.min(endOffset, segment.end); + if (end <= start) continue; + segments.push({ ...segment, start, end, + sourceStart: segment.sourceStart === null ? null : segment.sourceStart + start - segment.start, + sourceEnd: segment.sourceStart === null ? null : segment.sourceStart + end - segment.start }); + } + return { startOffset, endOffset, sourceTextSha256, offsetUnit: 'utf16_code_unit', rangeConvention: 'half_open', segments }; +} + +function runParser(binaryPath, args, { timeoutMs, maxOutputBytes, signal }) { + return new Promise((resolve, reject) => { + let child; + // A private POSIX process group includes wrapper/OCR descendants. Killing + // only the wrapper can leave workers alive and their output pipes open. + const grouped = process.platform !== 'win32'; + try { child = spawn(binaryPath, args, { stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true, detached: grouped }); } + catch (cause) { reject(new PdfExtractionError('unavailable', 'Parser could not be started', { cause })); return; } + const output = []; + let bytes = 0; + let imposedFailure; + let termination = Promise.resolve(); + const stop = (code, message) => { + if (imposedFailure) return; + imposedFailure = new PdfExtractionError(code, message); + termination = terminateSubprocess(child, grouped).catch(cause => { + imposedFailure.cleanupError = new PdfExtractionError('cleanup', 'Process-tree cleanup could not be qualified', { cause }); + }); + // Do not wait for an inherited pipe to be closed by a descendant. Other + // platforms kill the direct child but do not promise process-tree cleanup. + child.stdout.destroy(); + child.stderr.destroy(); + }; + const timer = setTimeout(() => stop('timeout', `Parser exceeded ${timeoutMs}ms`), timeoutMs); + const onAbort = () => stop('aborted', 'Extraction was aborted'); + signal?.addEventListener('abort', onAbort, { once: true }); + if (signal?.aborted) onAbort(); + child.stdout.on('data', (data) => { + bytes += data.length; + if (bytes > maxOutputBytes) stop('output_limit', `Parser exceeded ${maxOutputBytes} output bytes`); + else if (!imposedFailure) output.push(data); + }); + // Drain diagnostics without retaining document contents or scratch paths. + child.stderr.resume(); + child.once('error', (cause) => { imposedFailure ??= new PdfExtractionError('unavailable', 'Parser could not be executed', { cause }); }); + child.once('close', async (code, terminationSignal) => { + clearTimeout(timer); + signal?.removeEventListener('abort', onAbort); + await termination; + if (imposedFailure) reject(imposedFailure); + else if (terminationSignal) reject(new PdfExtractionError('crashed', `Parser terminated by ${terminationSignal}`)); + else if (code !== 0) reject(new PdfExtractionError('document', `Parser returned exit ${code}`)); + else { + const buffer = Buffer.concat(output); + try { resolve(new TextDecoder('utf-8', { fatal: true }).decode(buffer)); } + catch (cause) { reject(new PdfExtractionError('protocol', 'Parser output is not valid UTF-8', { cause })); } + } + }); + }); +} + +/** Dependency-free, bounded local CLI adapter. The host chooses its binary. */ +export async function extractPdfDocument(buffer, options) { + if (!(buffer instanceof Uint8Array)) throw new TypeError('PDF input must be bytes'); + if (!options || typeof options.binaryPath !== 'string' || !options.binaryPath) throw new TypeError('binaryPath is required'); + const timeoutMs = options.timeoutMs ?? 45_000; + const maxOutputBytes = options.maxOutputBytes ?? 25 * 1024 * 1024; + if (!integer(timeoutMs) || timeoutMs < 1 || timeoutMs > 2_147_483_647 || !integer(maxOutputBytes) || maxOutputBytes < 1) throw new RangeError('Invalid extraction limits'); + if (options.signal?.aborted) throw new PdfExtractionError('aborted', 'Extraction was aborted'); + // Own a snapshot before awaiting I/O: a caller can reuse or mutate its buffer. + const inputBytes = Buffer.from(buffer); + const inputSha256 = hash(inputBytes); + let tempDir; + let failure; + try { + tempDir = await mkdtemp(join(tmpdir(), 'pdf-parser-extract-')); + const inputPath = join(tempDir, 'input.pdf'); + await writeFile(inputPath, inputBytes, { mode: 0o600 }); + const args = ['extract-adaptive', '--input', inputPath, '--source-id', inputSha256, '--format', 'artifact-jsonl']; + if (!options.enableOcr) args.push('--no-ocr'); + const rawJsonl = await runParser(options.binaryPath, args, { timeoutMs, maxOutputBytes, signal: options.signal }); + return parseArtifactJsonl(rawJsonl, { inputSha256, expectedSourceId: inputSha256 }); + } catch (error) { + failure = error instanceof PdfExtractionError ? error : new PdfExtractionError('unavailable', 'Local extraction I/O failed', { cause: error }); + throw failure; + } finally { + if (tempDir) { + try { await rm(tempDir, { recursive: true, force: true }); } + catch (cause) { + const cleanupError = new PdfExtractionError('cleanup', 'Could not remove extraction scratch files', { cause }); + if (failure) failure.cleanupError = cleanupError; + else throw cleanupError; + } + } + } +} + +/** Compatibility-shaped string API; uses adaptive block text, not native extract. */ +export async function extractPdfText(buffer, options) { + return (await extractPdfDocument(buffer, options)).sourceText; +} diff --git a/integrations/javascript/extract.test.mjs b/integrations/javascript/extract.test.mjs new file mode 100644 index 0000000..9a2dfa7 --- /dev/null +++ b/integrations/javascript/extract.test.mjs @@ -0,0 +1,358 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { mkdtemp, writeFile, rm, readFile, access } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import fs from 'node:fs'; +import { syncBuiltinESMExports } from 'node:module'; +import { extractPdfDocument, extractPdfText, parseArtifactJsonl, projectPromptText, locateSourceRange, utf8BoundaryMap } from './extract.mjs'; + +const bytes = Buffer.from('%PDF-fake'); +const sha = (value) => createHash('sha256').update(value).digest('hex'); +const inputSha256 = sha(bytes); +const bbox = { x0: 10, y0: 20, x1: 60, y1: 30 }; +function fixture() { + const common = (type, id, refs = {}) => ({ schema_name: type, schema_version: '0.14.0', record_type: type, + document_id: 'doc', source_id: inputSha256, + provenance: { document_id: 'doc', source_id: inputSha256, input_sha256: inputSha256, artifact_id: id, + span_ids: [], block_ids: [], chunk_ids: [], ...refs } }); + const view = (id, kind, text) => ({ text_view_id: `${id}:text`, kind, offset_unit: 'utf8_byte', range_convention: 'half_open', normalization: 'none', text_sha256: sha(text) }); + const texts = ['A πŸ˜€ B', 'Next']; + const spans = texts.map((text, i) => ({ ...common('span', `span-${i}`, { span_ids: [`span-${i}`], block_ids: [`block-${i}`] }), span_id: `span-${i}`, text, bbox, page_index: 0, + text_view: view(`span-${i}`, 'reconciled_span', text) })); + const blocks = texts.map((text, i) => ({ ...common('block', `block-${i}`, { span_ids: [`span-${i}`], block_ids: [`block-${i}`] }), block_id: `block-${i}`, text, bbox, page_index: 0, + text_view: view(`block-${i}`, 'reconciled_block', text), source_map: [{ text_start_byte: 0, text_end_byte: Buffer.byteLength(text), source_span_id: `span-${i}`, source_start_byte: 0, source_end_byte: Buffer.byteLength(text) }] })); + const content = texts.join('\n\n'); + const firstLength = Buffer.byteLength(texts[0]); + const chunk = { ...common('rag_chunk', 'chunk-0', { span_ids: ['span-0', 'span-1'], block_ids: ['block-0', 'block-1'], chunk_ids: ['chunk-0'] }), chunk_id: 'chunk-0', content, + page_start: 0, page_end: 0, source_span_ids: ['span-0', 'span-1'], source_block_ids: ['block-0', 'block-1'], + text_view: view('chunk-0', 'rag_chunk', content), source_map: [ + { text_start_byte: 0, text_end_byte: firstLength, source_block_id: 'block-0', source_start_byte: 0, source_end_byte: firstLength }, + { text_start_byte: firstLength, text_end_byte: firstLength + 2, source_block_id: null, source_start_byte: null, source_end_byte: null }, + { text_start_byte: firstLength + 2, text_end_byte: Buffer.byteLength(content), source_block_id: 'block-1', source_start_byte: 0, source_end_byte: Buffer.byteLength(texts[1]) }, + ] }; + return [{ ...common('document_manifest', 'document_manifest'), input_sha256: inputSha256, page_count: 1, + text_contract: { offset_unit: 'utf8_byte', range_convention: 'half_open', page_index_base: 0, + geometry: { coordinate_space: 'pdf_user_space', unit: 'point', highlight_precision: 'span_bbox', display_transform: null } }, + artifact_counts: { spans: 2, blocks: 2, rag_chunks: 1 } }, ...spans, ...blocks, chunk]; +} +const serialize = (records) => records.map((record) => JSON.stringify(record)).join('\n') + '\n'; +const parse = (records) => parseArtifactJsonl(serialize(records), { inputSha256 }); + +function currentFixture() { + const records = fixture(); + for (const record of records) record.schema_version = '0.15.0'; + for (const span of records.filter(record => record.record_type === 'span')) span.reconciled_block_id = span.provenance.block_ids[0]; + records[0].artifact_counts.tables = 1; + records[0].artifact_counts.route_traces = 1; + const envelope = (type, id) => ({ ...records[0], schema_name: type, record_type: type, + provenance: { ...records[0].provenance, artifact_id: id } }); + records.push({ ...envelope('table', 'table-0'), page_index: 0, source_span_ids: ['span-0'], + reconciled_block_ids: ['block-0'], rows: [{ cells: [{ page_index: 0, source_span_ids: ['span-0'], reconciled_block_ids: ['block-0'] }] }] }); + records.push({ ...envelope('route_trace', 'route-page-0'), trace_kind: 'page_route', page_index: 0, + coverage: { status: 'text_available', completeness: 'unverified', native_span_count: 2, + recognized_span_count: 0, requested_recovery: false, recovery_state: 'not_required', unresolved_region_count: 0 } }); + return records; +} + +test('current schema exposes authoritative table ownership and unverified page outcomes', () => { + const document = parse(currentFixture()); + assert.deepEqual(document.tables[0].reconciled_block_ids, ['block-0']); + assert.equal(document.pageOutcomes[0].status, 'text_available'); + assert.equal(document.pageOutcomes[0].completeness, 'unverified'); +}); + +test('legacy chunk label remains metadata when caller source identity is null', () => { + const records = fixture(); + for (const record of records) { record.source_id = null; record.provenance.source_id = null; } + records[5].source_id = 'document'; + const document = parseArtifactJsonl(serialize(records), { inputSha256, expectedSourceId: null }); + assert.equal(document.manifest.source_id, null); + assert.equal(document.chunks[0].source_id, 'document'); + records[0].source_id = inputSha256; + assert.throws(() => parse(records), { code: 'protocol' }); +}); + +test('malformed table rows remain protocol errors through the subprocess adapter', async (t) => { + const records = currentFixture(); + records[6].rows = [null]; + const { binaryPath } = await fakeParser(t, `process.stdout.write(${JSON.stringify(serialize(records))});`); + await assert.rejects(extractPdfDocument(bytes, { binaryPath }), { code: 'protocol' }); +}); + +for (const [name, mutate] of [ + ['unknown span owner', r => { r[1].reconciled_block_id = 'block-999'; }], + ['wrong existing span owner', r => { r[1].reconciled_block_id = 'block-1'; }], + ['null owned span', r => { r[1].reconciled_block_id = null; }], + ['missing span owner', r => { delete r[1].reconciled_block_id; }], + ['wrong span ownership provenance', r => { r[1].provenance.block_ids = ['block-1']; }], + ['wrong span self provenance', r => { r[1].provenance.span_ids = ['span-1']; }], + ['wrong table owner', r => { r[6].reconciled_block_ids = ['block-1']; }], + ['missing table owner', r => { delete r[6].reconciled_block_ids; }], + ['malformed table row', r => { r[6].rows = [null]; }], + ['wrong cell owner', r => { r[6].rows[0].cells[0].reconciled_block_ids = ['block-1']; }], + ['cross-page cell sources', r => { r[0].page_count = 2; r[6].rows[0].cells[0].page_index = 1; }], + ['invented complete coverage', r => { r[7].coverage.completeness = 'complete'; }], + ['missing coverage', r => { delete r[7].coverage; }], + ['unknown recovery state', r => { r[7].coverage.recovery_state = 'successful'; }], + ['invalid coverage count', r => { r[7].coverage.unresolved_region_count = -1; }], + ['coverage outside document', r => { r[7].page_index = 999; }], + ['contradictory recovery', r => { Object.assign(r[7].coverage, { status: 'recovered', recovery_state: 'failed' }); }], + ['unrecognized successful recovery', r => { Object.assign(r[7].coverage, { status: 'recovered', requested_recovery: true, recovery_state: 'completed' }); }], + ['successful recovery with unresolved regions', r => { Object.assign(r[7].coverage, { status: 'recovered', requested_recovery: true, recovery_state: 'completed', recognized_span_count: 1, unresolved_region_count: 1 }); }], + ['cell on another valid page', r => { + r[0].page_count = 2; + r[2].page_index = 1; + r[4].page_index = 1; + r[5].page_end = 1; + Object.assign(r[6].rows[0].cells[0], { page_index: 1, source_span_ids: ['span-1'], reconciled_block_ids: ['block-1'] }); + }], +]) test(`current schema rejects ${name}`, () => { + const records = currentFixture(); + mutate(records); + assert.throws(() => parse(records), { code: 'protocol' }); +}); + +test('rich projection preserves whitespace, Unicode, records and span geometry', () => { + const document = parse(fixture()); + assert.equal(document.sourceText, 'A πŸ˜€ B\n\nNext'); + assert.equal(document.sourceTextSha256, sha(document.sourceText)); + assert.equal(document.pageCoverage, 'unverified'); + assert.equal(document.spans.length, 2); + assert.equal(document.chunks[0].content, document.sourceText); + assert.equal(document.sourceMap[0].end, 7); + assert.deepEqual(document.sourceMap[1], { start: 7, end: 9, sourceBlockId: null, sourceSpanId: null, pageIndex: null, bbox: null, sourceStart: null, sourceEnd: null }); + assert.deepEqual(document.sourceMap[0].bbox, bbox); + assert.deepEqual([...utf8BoundaryMap('AπŸ˜€Γ©')], [[0, 0], [1, 1], [5, 3], [7, 4]]); +}); + +test('prompt projection cannot split emoji or mutate authoritative source offsets', () => { + const document = parse(fixture()); + const projection = projectPromptText(document, 3); + assert.equal(projection.text, 'A '); + assert.equal(projection.truncated, true); + assert.equal(document.sourceText, 'A πŸ˜€ B\n\nNext'); + assert.equal(projection.sourceEnd, 2); + assert.throws(() => projectPromptText(document, -1), RangeError); +}); + +test('range lookup clips UTF-16 span origins and preserves synthetic gaps', () => { + const document = parse(fixture()); + const options = { sourceTextSha256: document.sourceTextSha256 }; + const result = locateSourceRange(document, 2, 10, options); + assert.equal(result.segments[0].sourceStart, 2); + assert.equal(result.segments[0].sourceEnd, 7); + assert.equal(result.segments[1].bbox, null); + assert.equal(result.segments[2].sourceEnd, 1); + assert.throws(() => locateSourceRange(document, 3, 4, options), /UTF-16/); + assert.throws(() => locateSourceRange(document, 0, 1, { sourceTextSha256: 'a'.repeat(64) }), /hash/); + assert.throws(() => locateSourceRange(document, 0, 1), /hash/); + document.sourceText += '!'; + assert.throws(() => locateSourceRange(document, 0, 1, options), /hash/); +}); + +const corruptions = [ + ['schema family', (r) => { r[0].schema_version = '0.13.0'; }], + ['per-record version', (r) => { r[1].schema_version = '0.14.1'; }], + ['manifest coordinate contract', (r) => { r[0].text_contract.geometry.coordinate_space = 'screen'; }], + ['nonnumeric span geometry', (r) => { r[1].bbox = { x0: '10', y0: 20, x1: 60, y1: 30 }; }], + ['inverted span geometry', (r) => { r[1].bbox = { x0: 60, y0: 20, x1: 10, y1: 30 }; }], + ['missing span geometry', (r) => { delete r[1].bbox; }], + ['manifest offset contract', (r) => { r[0].text_contract.offset_unit = 'utf16_code_unit'; }], + ['declared route count', (r) => { r[0].artifact_counts.route_traces = 1; }], + ['schema name', (r) => { r[1].schema_name = 'block'; }], + ['input hash', (r) => { r[0].input_sha256 = 'f'.repeat(64); }], + ['provenance', (r) => { r[1].provenance.input_sha256 = 'f'.repeat(64); }], + ['source identity', (r) => { r[1].source_id = 'other'; }], + ['missing source identity', (r) => { for (const record of r) { delete record.source_id; delete record.provenance.source_id; } }], + ['invalid source identity type', (r) => { for (const record of r) { record.source_id = 42; record.provenance.source_id = 42; } }], + ['duplicate artifact', (r) => { r.push(r[1]); }], + ['duplicate manifest', (r) => { r.push({ ...r[0], provenance: { ...r[0].provenance, artifact_id: 'other' } }); }], + ['missing first manifest', (r) => { r.shift(); }], + ['truncated batch', (r) => { r.pop(); }], + ['text hash', (r) => { r[1].text += '!'; }], + ['normalization contract', (r) => { r[1].text_view.normalization = 'whitespace'; }], + ['UTF-8 boundary inside emoji', (r) => { r[3].source_map[0].source_end_byte = 3; }], + ['missing coverage', (r) => { r[3].source_map = []; }], + ['unordered coverage', (r) => { r[3].source_map[0].text_start_byte = 1; }], + ['unresolved span', (r) => { r[3].source_map[0].source_span_id = 'span-100'; }], + ['unresolved chunk source', (r) => { r[5].source_block_ids.push('block-100'); }], + ['unresolved provenance link', (r) => { r[3].provenance.span_ids.push('span-100'); }], + ['synthetic source offsets', (r) => { r[5].source_map[1].source_start_byte = 0; }], + ['synthetic forged text', (r) => { r[3].source_map[0].source_span_id = null; r[3].source_map[0].source_start_byte = null; r[3].source_map[0].source_end_byte = null; }], + ['cross-page span', (r) => { r[0].page_count = 2; r[1].page_index = 1; }], + ['cross-page block', (r) => { r[0].page_count = 2; r[1].page_index = 1; r[3].page_index = 1; }], + ['chunk source identity omission', (r) => { r[5].source_span_ids.pop(); }], + ['chunk source identity duplication', (r) => { r[5].source_block_ids.push('block-0'); }], + ['block provenance omission', (r) => { r[3].provenance.span_ids = []; }], + ['chunk provenance omission', (r) => { r[5].provenance.span_ids = []; }], + ['slice mismatch', (r) => { r[3].source_map[0].source_span_id = 'span-1'; }], + ['invalid page', (r) => { r[1].page_index = 2; }], +]; +for (const [name, change] of corruptions) test(`rejects ${name}`, () => { + const records = fixture(); + change(records); + assert.throws(() => parse(records), { code: 'protocol' }); +}); +test('rejects malformed JSON, empty input, blank lines and unpaired surrogates', () => { + for (const text of ['{', '', serialize(fixture()) + '\n']) assert.throws(() => parseArtifactJsonl(text, { inputSha256 }), { code: 'protocol' }); + assert.throws(() => utf8BoundaryMap('\ud800'), { code: 'protocol' }); +}); +for (const [type, idField, countField] of [ + ['specialist_request', 'request_id', 'specialist_requests'], + ['specialist_attempt', 'attempt_id', 'specialist_attempts'], + ['specialist_response', 'response_id', 'specialist_responses'], + ['specialist_result', 'specialist_result_id', 'specialist_results'], +]) { + const specialistFixture = () => { + const records = fixture(); + records[0].artifact_counts[countField] = 2; + for (let i = 0; i < 2; i++) records.push({ schema_name: type, schema_version: '0.14.0', record_type: type, + document_id: 'doc', source_id: inputSha256, [idField]: `${type}-${i}`, + provenance: { ...records[0].provenance, artifact_id: type } }); + return records; + }; + test(`${type} retains repeated compatibility ids with unique record ids`, () => { + assert.equal(parse(specialistFixture()).records.filter((record) => record.record_type === type).length, 2); + }); + test(`${type} rejects duplicated or missing actual record identity`, () => { + const records = specialistFixture(); + records.at(-1)[idField] = records.at(-2)[idField]; + assert.throws(() => parse(records), { code: 'protocol' }); + delete records.at(-1)[idField]; + assert.throws(() => parse(records), { code: 'protocol' }); + }); +} +test('legacy unresolved table block refs are diagnostic metadata, never locator inputs', () => { + const records = fixture(); + records[0].artifact_counts.tables = 1; + records.push({ schema_name: 'table', schema_version: '0.14.0', record_type: 'table', + document_id: 'doc', source_id: inputSha256, + provenance: { ...records[0].provenance, artifact_id: 'table-0', block_ids: ['block-0', 'block-99'] } }); + const document = parse(records); + assert.deepEqual(document.unresolvedProvenanceLinks, [{ recordType: 'table', artifactId: 'table-0', field: 'block_ids', id: 'block-99' }]); + assert.equal(document.records.at(-1).provenance.block_ids[1], 'block-99'); + assert.ok(locateSourceRange(document, 0, document.sourceText.length, { sourceTextSha256: document.sourceTextSha256 }).segments.every((segment) => segment.sourceBlockId !== 'block-99')); + records[3].provenance.block_ids.push('block-99'); + assert.throws(() => parse(records), { code: 'protocol' }); + records[3].provenance.block_ids.pop(); + records.at(-1).provenance.span_ids.push('span-99'); + assert.throws(() => parse(records), { code: 'protocol' }); +}); + +async function fakeParser(t, body) { + const dir = await mkdtemp(join(tmpdir(), 'pdf-parser-adapter-test-')); + t.after(() => rm(dir, { recursive: true, force: true })); + const binaryPath = join(dir, 'parser.mjs'); + await writeFile(binaryPath, `#!${process.execPath}\n${body}\n`, { mode: 0o700 }); + return { dir, binaryPath }; +} +test('actual subprocess uses private input, bounded adapter protocol and cleans scratch files', async (t) => { + const { dir, binaryPath } = await fakeParser(t, `import {writeFileSync,statSync} from 'node:fs'; + const input=process.argv[process.argv.indexOf('--input')+1]; + writeFileSync(new URL('./audit.json',import.meta.url),JSON.stringify({input,args:process.argv.slice(2),mode:statSync(input).mode&511})); + process.stdout.write(${JSON.stringify(serialize(fixture()))});`); + const document = await extractPdfDocument(bytes, { binaryPath }); + assert.equal(document.sourceText, 'A πŸ˜€ B\n\nNext'); + const audit = JSON.parse(await readFile(join(dir, 'audit.json'), 'utf8')); + assert.equal(audit.mode, 0o600); + assert.ok(audit.args.includes('--no-ocr')); + assert.ok(audit.args.includes(inputSha256)); + await assert.rejects(access(audit.input), { code: 'ENOENT' }); + assert.equal(await extractPdfText(bytes, { binaryPath }), document.sourceText); +}); + +test('validates the source identity selected by the caller and adapter', async (t) => { + const records = fixture(); + for (const record of records) { record.source_id = 'other'; record.provenance.source_id = 'other'; } + assert.throws(() => parseArtifactJsonl(serialize(records), { inputSha256, expectedSourceId: inputSha256 }), /source identity/); + const { binaryPath } = await fakeParser(t, `process.stdout.write(${JSON.stringify(serialize(records))});`); + await assert.rejects(extractPdfDocument(bytes, { binaryPath }), { code: 'protocol' }); + const nullable = fixture(); + for (const record of nullable) { record.source_id = null; record.provenance.source_id = null; } + assert.equal(parseArtifactJsonl(serialize(nullable), { inputSha256, expectedSourceId: null }).manifest.source_id, null); +}); + +for (const reason of ['timeout', 'aborted']) test(`bounds ${reason} and cleans scratch when a wrapper leaves inherited descendant pipes`, { + skip: process.platform === 'win32' ? 'POSIX process-group regression' : false, +}, async (t) => { + const { dir, binaryPath } = await fakeParser(t, `import { spawn } from 'node:child_process'; + import { writeFileSync } from 'node:fs'; + const input = process.argv[process.argv.indexOf('--input') + 1]; + const descendant = spawn(process.execPath, ['-e', 'setTimeout(() => {}, 3000)'], { stdio: 'inherit' }); + writeFileSync(new URL('./descendant.json', import.meta.url), JSON.stringify({ pid: descendant.pid, input })); + setInterval(() => {}, 1000);`); + const controller = new AbortController(); + const started = performance.now(); + const pending = extractPdfDocument(bytes, { binaryPath, timeoutMs: reason === 'timeout' ? 1500 : 5000, signal: controller.signal }); + // Observe descendant creation before cancellation; startup latency must not + // turn this regression into a test that kills the wrapper before it forks. + let audit; + while (!audit && performance.now() - started < 2000) { + try { audit = JSON.parse(await readFile(join(dir, 'descendant.json'), 'utf8')); } + catch (error) { if (error.code !== 'ENOENT') throw error; await new Promise(resolve => setTimeout(resolve, 20)); } + } + assert.ok(audit, 'wrapper never created the descendant'); + const terminationStarted = reason === 'timeout' ? started : performance.now(); + if (reason === 'aborted') controller.abort(); + await assert.rejects(pending, { code: reason }); + assert.ok(performance.now() - terminationStarted < (reason === 'timeout' ? 2300 : 850), 'termination waited for descendant-held pipes'); + await assert.rejects(access(audit.input), { code: 'ENOENT' }); + await new Promise(resolve => setTimeout(resolve, 50)); + assert.throws(() => process.kill(audit.pid, 0), { code: 'ESRCH' }); +}); +test('owns input bytes across asynchronous writes and supports explicit OCR opt-in', async (t) => { + const { binaryPath } = await fakeParser(t, `import {readFileSync} from 'node:fs'; + const input=process.argv[process.argv.indexOf('--input')+1]; + if(process.argv.includes('--no-ocr') || readFileSync(input).toString()!==${JSON.stringify(bytes.toString())}) process.exit(2); + process.stdout.write(${JSON.stringify(serialize(fixture()))});`); + const reusable = Uint8Array.from(bytes); + const pending = extractPdfDocument(reusable, { binaryPath, enableOcr: true }); + reusable.fill(0); + assert.equal((await pending).inputSha256, inputSha256); +}); +for (const [name, body, options, code] of [ + ['timeout', 'setInterval(()=>{},1000);', { timeoutMs: 100 }, 'timeout'], + ['output cap', 'process.stdout.write("x".repeat(10000));', { maxOutputBytes: 100 }, 'output_limit'], + ['signal crash', 'process.kill(process.pid,"SIGKILL");', {}, 'crashed'], + ['document failure', 'process.exit(2);', {}, 'document'], + ['malformed output', 'process.stdout.write("{bad");', {}, 'protocol'], + ['invalid UTF-8', 'process.stdout.write(Buffer.from([255]));', {}, 'protocol'], +]) test(`subprocess classifies ${name}`, async (t) => { + const { binaryPath } = await fakeParser(t, body); + await assert.rejects(extractPdfDocument(bytes, { binaryPath, ...options }), { code }); +}); +test('unavailable binary, limits and cancellation remain distinct', async (t) => { + const { binaryPath } = await fakeParser(t, 'setInterval(()=>{},1000);'); + await assert.rejects(extractPdfDocument(bytes, { binaryPath: `${binaryPath}-missing` }), { code: 'unavailable' }); + await assert.rejects(extractPdfDocument(bytes, { binaryPath, maxOutputBytes: 0 }), RangeError); + await assert.rejects(extractPdfDocument(bytes, {}), TypeError); + await assert.rejects(extractPdfDocument(bytes, { binaryPath, signal: AbortSignal.abort() }), { code: 'aborted' }); + const controller = new AbortController(); + const pending = extractPdfDocument(bytes, { binaryPath, signal: controller.signal }); + setTimeout(() => controller.abort(), 30); + await assert.rejects(pending, { code: 'aborted' }); +}); +test('cleanup failure is reported, preserving an earlier extraction failure', async (t) => { + const success = await fakeParser(t, `process.stdout.write(${JSON.stringify(serialize(fixture()))});`); + const malformed = await fakeParser(t, 'process.stdout.write("bad");'); + const originalRm = fs.promises.rm; + const scratch = []; + fs.promises.rm = async (path, options) => { + if (String(path).includes('pdf-parser-extract-')) { + scratch.push(path); + throw Object.assign(new Error('Injected cleanup failure'), { code: 'EACCES' }); + } + return originalRm(path, options); + }; + syncBuiltinESMExports(); + try { + await assert.rejects(extractPdfDocument(bytes, { binaryPath: success.binaryPath }), { code: 'cleanup' }); + await assert.rejects(extractPdfDocument(bytes, { binaryPath: malformed.binaryPath }), (error) => error.code === 'protocol' && error.cleanupError.code === 'cleanup'); + } finally { + fs.promises.rm = originalRm; + syncBuiltinESMExports(); + await Promise.all(scratch.map((path) => originalRm(path, { recursive: true, force: true }))); + } +}); diff --git a/integrations/javascript/index.d.mts b/integrations/javascript/index.d.mts new file mode 100644 index 0000000..85d6aee --- /dev/null +++ b/integrations/javascript/index.d.mts @@ -0,0 +1,3 @@ +export * from './chunk-origins.mjs'; +export * from './stream.mjs'; +export * from './extract.mjs'; diff --git a/integrations/javascript/index.mjs b/integrations/javascript/index.mjs new file mode 100644 index 0000000..536c184 --- /dev/null +++ b/integrations/javascript/index.mjs @@ -0,0 +1,3 @@ +export { composeChunksWithOrigins } from './chunk-origins.mjs'; +export { extractPdfPages } from './stream.mjs'; +export { extractPdfDocument, extractPdfText, parseArtifactJsonl, projectPromptText, locateSourceRange, utf8BoundaryMap, PdfExtractionError } from './extract.mjs'; diff --git a/integrations/javascript/integration.test.mjs b/integrations/javascript/integration.test.mjs new file mode 100644 index 0000000..ba3f95f --- /dev/null +++ b/integrations/javascript/integration.test.mjs @@ -0,0 +1,161 @@ +import assert from 'node:assert/strict'; +import { access, readFile, mkdtemp, writeFile, rm } from 'node:fs/promises'; +import { constants } from 'node:fs'; +import { createHash } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import test from 'node:test'; +import { composeChunksWithOrigins, extractPdfDocument, extractPdfPages, parseArtifactJsonl, locateSourceRange, projectPromptText } from './index.mjs'; + +const binaryPath = fileURLToPath(new URL('../../zig-out/bin/pdf-parser', import.meta.url)); +let available = true; +try { await access(binaryPath, constants.X_OK); } catch { available = false; } + +test('actual CLI without caller source identity keeps every artifact source null', { skip: !available }, async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'pdf-parser-source-contract-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const inputPath = join(directory, 'input.pdf'); + const bytes = fixturePdf(); + await writeFile(inputPath, bytes, { mode: 0o600 }); + const rawJsonl = execFileSync(binaryPath, ['extract-adaptive', '--input', inputPath, '--format', 'artifact-jsonl', '--no-ocr'], { encoding: 'utf8', timeout: 5000 }); + const document = parseArtifactJsonl(rawJsonl, { inputSha256: createHash('sha256').update(bytes).digest('hex'), expectedSourceId: null }); + assert.ok(document.chunks.length > 0); + for (const record of document.records) { + assert.equal(record.source_id, null); + assert.equal(record.provenance.source_id, null); + } +}); + +// Small generated, redistribution-safe PDF. Each marked group contains repeated +// lines; a second page checks that origin identity includes the actual page. +function fixturePdf() { + const content = '/Span << /MCID 7 >> BDC BT /F1 12 Tf 20 250 Td (Repeated evidence.) Tj 0 -24 Td (Repeated evidence.) Tj ET EMC'; + const objects = [ + '<< /Type /Catalog /Pages 2 0 R >>', + '<< /Type /Pages /Kids [4 0 R 6 0 R] /Count 2 >>', + '<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>', + '<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Resources << /Font << /F1 3 0 R >> >> /Contents 5 0 R >>', + `<< /Length ${Buffer.byteLength(content)} >>\nstream\n${content}\nendstream`, + '<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Resources << /Font << /F1 3 0 R >> >> /Contents 7 0 R >>', + `<< /Length ${Buffer.byteLength(content)} >>\nstream\n${content}\nendstream`, + ]; + let pdf = '%PDF-1.4\n'; + const offsets = [0]; + objects.forEach((object, index) => { + offsets.push(Buffer.byteLength(pdf)); + pdf += `${index + 1} 0 obj\n${object}\nendobj\n`; + }); + const xref = Buffer.byteLength(pdf); + pdf += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n`; + for (const offset of offsets.slice(1)) pdf += `${String(offset).padStart(10, '0')} 00000 n \n`; + pdf += `trailer\n<< /Size ${objects.length + 1} /Root 1 0 R >>\nstartxref\n${xref}\n%%EOF\n`; + return Buffer.from(pdf, 'ascii'); +} + +test('actual CLI -> validated rich text -> overlapping chunks -> page/span locators', { + skip: available ? false : 'Build the parser with zig build to run the real CLI integration', +}, async () => { + const document = await extractPdfDocument(fixturePdf(), { binaryPath }); + assert.equal(document.manifest.schema_version, '0.15.0'); + assert.equal(document.spans.length, 4); + assert.equal(document.blocks.length, 4); + assert.equal(document.sourceText, Array(4).fill('Repeated evidence.').join('\n\n')); + assert.deepEqual(document.spans.map((span) => span.page_index), [0, 0, 1, 1]); + const chunks = composeChunksWithOrigins({ text: document.sourceText, maxTokens: 8, overlapTokens: 2 }); + assert.ok(chunks.length > 1); + assert.ok(chunks.some((chunk, index) => index > 0 && chunk.startOffset < chunks[index - 1].endOffset)); + const spanById = new Map(document.spans.map((span) => [span.span_id, span])); + const pages = new Set(); + for (const chunk of chunks) { + assert.equal(document.sourceText.slice(chunk.startOffset, chunk.endOffset).replace(/\s+/gu, ' ').trim(), chunk.body); + const locator = locateSourceRange(document, chunk.startOffset, chunk.endOffset, { sourceTextSha256: chunk.sourceTextSha256 }); + assert.ok(locator.segments.length > 0); + for (const segment of locator.segments) { + if (segment.sourceSpanId === null) { assert.equal(segment.bbox, null); continue; } + const span = spanById.get(segment.sourceSpanId); + assert.equal(document.sourceText.slice(segment.start, segment.end), span.text.slice(segment.sourceStart, segment.sourceEnd)); + assert.equal(segment.pageIndex, span.page_index); + pages.add(segment.pageIndex); + } + } + assert.deepEqual([...pages].sort(), [0, 1]); + const projection = projectPromptText(document, 10); + assert.equal(projection.truncated, true); + assert.equal(document.sourceText.length, 78); + assert.equal(document.pageCoverage, 'unverified'); +}); + +test('actual financial artifacts retain specialist identities and diagnose legacy table links', { + skip: available ? false : 'Build the parser with zig build to run the real CLI integration', +}, async () => { + const pdf = await readFile(new URL('../../benchmark/eval/corpus/financial_tables/complex-financials.pdf', import.meta.url)); + const document = await extractPdfDocument(pdf, { binaryPath }); + assert.ok(document.records.some((record) => record.record_type === 'table')); + const owners = new Map(document.spans.map(span => [span.span_id, span.reconciled_block_id])); + for (const table of document.tables) { + for (const record of [table, ...table.rows.flatMap(row => row.cells)]) { + assert.deepEqual(new Set(record.reconciled_block_ids), new Set(record.source_span_ids.map(id => owners.get(id)).filter(id => id !== null))); + for (const id of record.reconciled_block_ids) assert.equal(document.blocks.find(block => block.block_id === id).page_index, record.page_index); + } + } + const requests = document.records.filter((record) => record.record_type === 'specialist_request'); + assert.ok(requests.length > 1); + assert.equal(new Set(requests.map((record) => record.request_id)).size, requests.length); + for (const diagnostic of document.unresolvedProvenanceLinks) { + assert.equal(diagnostic.recordType, 'table'); + assert.equal(diagnostic.field, 'block_ids'); + assert.ok(!document.sourceMap.some((segment) => segment.sourceBlockId === diagnostic.id)); + } + for (const chunk of composeChunksWithOrigins({ text: document.sourceText, maxTokens: 20, overlapTokens: 5 })) { + const locator = locateSourceRange(document, chunk.startOffset, chunk.endOffset, { sourceTextSha256: chunk.sourceTextSha256 }); + assert.ok(locator.segments.some((segment) => segment.sourceSpanId !== null)); + } +}); + +test('actual multi-page stream preserves global ownership with page-local locators', { skip: !available }, async () => { + const pages = []; + const receipt = await extractPdfPages(fixturePdf(), { binaryPath, onPage: async page => { pages.push(page); } }); + assert.equal(receipt.complete, true); + assert.equal(receipt.pageCount, 2); + assert.equal(receipt.schemaVersion, '0.15.0'); + assert.deepEqual(pages.map(page => page.pageIndex), [0, 1]); + assert.equal(new Set(pages.flatMap(page => page.spans.map(span => span.span_id))).size, 4); + for (const page of pages) { + assert.equal(page.sourceTextScope, 'page'); + assert.equal(page.streamComplete, false); + assert.equal(page.sourceMap[0].start, 0); + assert.equal(page.pageOutcomes[0].status, 'text_available'); + assert.equal(page.pageOutcomes[0].completeness, 'unverified'); + const locator = locateSourceRange(page, 0, page.sourceText.length, { sourceTextSha256: page.sourceTextSha256 }); + for (const segment of locator.segments.filter(segment => segment.sourceSpanId !== null)) { + const span = page.spans.find(span => span.span_id === segment.sourceSpanId); + assert.equal(span.reconciled_block_id, segment.sourceBlockId); + assert.deepEqual(span.provenance.block_ids, [segment.sourceBlockId]); + assert.equal(segment.pageIndex, page.pageIndex); + } + } +}); + +test('actual native page allocation limit fails without delivering a page or receipt', { skip: !available }, async () => { + let pages = 0; + await assert.rejects(extractPdfPages(fixturePdf(), { binaryPath, maxNativePageMemoryBytes: 1, + onPage: () => { pages++; } }), { code: 'memory_limit' }); + assert.equal(pages, 0); + assert.equal((await extractPdfPages(fixturePdf(), { binaryPath, maxNativePageMemoryBytes: 8 * 1024 * 1024, + onPage: () => {} })).complete, true); +}); + +test('actual mixed scan keeps native evidence and exposes disabled recovery', { skip: !available }, async () => { + const pdf = await readFile(new URL('../../benchmark/eval/corpus/scanned_typewritten/mixed-native-scan.pdf', import.meta.url)); + const document = await extractPdfDocument(pdf, { binaryPath }); + assert.ok(document.sourceText.includes('Native cover text')); + assert.equal(document.pageOutcomes.length, document.manifest.page_count); + const outcome = document.pageOutcomes[0]; + assert.equal(outcome.status, 'recovery_required'); + assert.equal(outcome.requested_recovery, true); + assert.equal(outcome.recovery_state, 'not_invoked'); + assert.equal(outcome.completeness, 'unverified'); + assert.ok(outcome.native_span_count > 0); +}); diff --git a/integrations/javascript/stream.d.mts b/integrations/javascript/stream.d.mts new file mode 100644 index 0000000..befd2d4 --- /dev/null +++ b/integrations/javascript/stream.d.mts @@ -0,0 +1,34 @@ +import type { RichPdfDocument } from './extract.mjs'; +/** Global artifact IDs and page index; source positions are local to this page. + * Delivered pages remain provisional until the extraction receipt succeeds. */ +export interface RichPdfPage extends RichPdfDocument { + pageIndex: number; + sourceTextScope: 'page'; + streamComplete: false; +} +export interface StreamExtractionOptions { + binaryPath: string; + onPage: (page: RichPdfPage) => void | Promise; + timeoutMs?: number; + maxOutputBytes?: number; + maxPageBytes?: number; + maxRecordBytes?: number; + maxRecords?: number; + maxPages?: number; + /** Live allocations through the adaptive page allocator; not process RSS. */ + maxNativePageMemoryBytes?: number; + signal?: AbortSignal; + enableOcr?: boolean; +} +export interface StreamExtractionReceipt { + complete: true; + inputSha256: string; + schemaVersion: string; + pageCount: number; + records: number; + outputBytes: number; + artifactCounts: Record; +} +/** Retains one page and a bounded identity set; caller-retained pages and parser + * process RSS are outside these byte budgets. No receipt on partial failure. */ +export function extractPdfPages(buffer: Uint8Array, options: StreamExtractionOptions): Promise; diff --git a/integrations/javascript/stream.mjs b/integrations/javascript/stream.mjs new file mode 100644 index 0000000..859a42a --- /dev/null +++ b/integrations/javascript/stream.mjs @@ -0,0 +1,228 @@ +import { spawn } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { parseArtifactJsonl, PdfExtractionError } from './extract.mjs'; +import { terminateSubprocess } from './subprocess.mjs'; + +const integer = (value) => Number.isSafeInteger(value) && value >= 0; +const object = (value) => value !== null && typeof value === 'object' && !Array.isArray(value); +const countFields = { span: 'spans', block: 'blocks', table: 'tables', route_trace: 'route_traces', + specialist_request: 'specialist_requests', specialist_attempt: 'specialist_attempts', + specialist_response: 'specialist_responses', specialist_result: 'specialist_results', + rag_chunk: 'rag_chunks', debug_asset: 'debug_assets' }; +const emptyCounts = () => Object.fromEntries(Object.values(countFields).map((key) => [key, 0])); +const routeKeys = ['native_pages', 'page_routes', 'region_routes', 'ocr_regions', 'table_regions', 'formula_regions']; +const fail = (message) => { throw new PdfExtractionError('protocol', message); }; +const checkCounts = (actual, expected) => { + if (!object(actual) || Object.keys(expected).some((key) => actual[key] !== expected[key])) fail('Stream artifact count mismatch'); +}; + +async function consume(binaryPath, args, options, inputSha256) { + const grouped = process.platform !== 'win32'; + const child = spawn(binaryPath, args, { stdio: ['ignore', 'pipe', 'pipe'], detached: grouped, windowsHide: true }); + let failure; + let termination = Promise.resolve(); + let rejectFailure; + const failed = new Promise((_, reject) => { rejectFailure = reject; }); + failed.catch(() => {}); + const stop = (error) => { + if (failure) return; + failure = error; + termination = terminateSubprocess(child, grouped).catch(cause => { + failure.cleanupError = new PdfExtractionError('cleanup', 'Process-tree cleanup could not be qualified', { cause }); + }); + child.stdout.destroy(); + child.stderr.destroy(); + rejectFailure(error); + }; + const closed = new Promise((resolve) => { + child.once('error', (cause) => stop(new PdfExtractionError('unavailable', 'Parser could not be executed', { cause }))); + child.once('close', (code, signal) => resolve({ code, signal })); + }); + let nativeFailure; + let diagnosticTail = ''; + child.stderr.on('data', bytes => { + diagnosticTail = (diagnosticTail + bytes.toString('utf8')).slice(-256); + if (diagnosticTail.includes('MemoryBudgetExceeded')) nativeFailure = new PdfExtractionError('memory_limit', 'Native adaptive page allocation budget exceeded'); + else if (diagnosticTail.includes('OutputLimitExceeded')) nativeFailure ??= new PdfExtractionError('output_limit', 'Native serialized output budget exceeded'); + }); + const timer = setTimeout(() => stop(new PdfExtractionError('timeout', 'Streaming extraction exceeded its deadline')), options.timeoutMs); + const abort = () => stop(new PdfExtractionError('aborted', 'Extraction was aborted')); + options.signal?.addEventListener('abort', abort, { once: true }); + if (options.signal?.aborted) abort(); + const totals = emptyCounts(); + const routeTotals = Object.fromEntries(routeKeys.map((key) => [key, 0])); + const identities = new Set(); + let manifest; + let currentPage = null; + let pageCount = 0; + let pageBytes = 0; + let pageLines = []; + let pageCounts = emptyCounts(); + let records = 0; + let outputBytes = 0; + let finished = false; + const decoder = new TextDecoder('utf-8', { fatal: true }); + const line = async (buffer) => { + if (++records > options.maxRecords) throw new PdfExtractionError('output_limit', 'Stream exceeded its record budget'); + let raw; + let record; + try { raw = decoder.decode(buffer); record = JSON.parse(raw); } + catch (cause) { throw new PdfExtractionError('protocol', 'Malformed UTF-8 JSONL record', { cause }); } + if (finished) fail('Record follows document completion'); + const type = record?.record_type; + if (!object(record) || record.event_index !== records - 1 || record.event_type !== type || record.schema_name !== type) fail('Invalid stream event sequence'); + const firstRecord = !manifest; + if (firstRecord) { + if (type !== 'document_manifest' || record.streaming !== true || record.sequence_scope !== 'document' || + !/^0\.(14|15)\.\d+$/.test(record.schema_version ?? '') || !integer(record.page_count) || + typeof record.document_id !== 'string' || !record.document_id || record.input_sha256 !== inputSha256 || record.source_id !== inputSha256) fail('Invalid stream manifest'); + if (record.page_count > options.maxPages) throw new PdfExtractionError('output_limit', 'Stream exceeded its page-count budget'); + if (!Array.isArray(record.errors) || record.errors.length) fail('Stream manifest reports document errors'); + manifest = record; + // Also validate an empty document: there may be no page on which the + // richer batch validator can otherwise check the text contract. + parseArtifactJsonl(JSON.stringify({ ...manifest, artifact_counts: emptyCounts() }) + '\n', { inputSha256, expectedSourceId: inputSha256 }); + } + const provenance = record.provenance; + const specialField = { specialist_request: 'request_id', specialist_attempt: 'attempt_id', + specialist_response: 'response_id', specialist_result: 'specialist_result_id' }[type]; + const id = specialField ? record[specialField] : provenance?.artifact_id; + if (record.schema_version !== manifest.schema_version || record.document_id !== manifest.document_id || record.source_id !== inputSha256 || + !object(provenance) || provenance.document_id !== manifest.document_id || provenance.source_id !== inputSha256 || provenance.input_sha256 !== inputSha256 || + typeof provenance.artifact_id !== 'string' || !provenance.artifact_id || typeof id !== 'string' || !id || identities.has(id)) fail('Invalid stream identity'); + identities.add(id); + if (type === 'document_manifest') { + if (!firstRecord) fail('Duplicate stream manifest'); + return; + } + if (type === 'page_started') { + if (currentPage !== null || record.page_index !== pageCount || pageCount >= manifest.page_count || record.sequence_scope !== 'page' || record.status !== 'running') fail('Invalid page start'); + currentPage = record.page_index; + pageBytes = buffer.length + 1; + pageLines = []; + pageCounts = emptyCounts(); + } else if (type === 'page_finished') { + if (currentPage === null || record.page_index !== currentPage || record.sequence_scope !== 'page' || record.status !== 'completed') fail('Invalid page completion'); + checkCounts(record.artifact_counts, pageCounts); + if (!object(record.route_counts) || routeKeys.some((key) => !integer(record.route_counts[key]))) fail('Invalid page route counts'); + for (const key of routeKeys) { + routeTotals[key] += record.route_counts[key]; + if (!integer(routeTotals[key])) fail('Route count overflow'); + } + pageBytes += buffer.length + 1; + if (pageBytes > options.maxPageBytes) throw new PdfExtractionError('output_limit', 'Stream exceeded its page budget'); + const batchManifest = { ...manifest, streaming: false, artifact_counts: pageCounts }; + const document = parseArtifactJsonl([JSON.stringify(batchManifest), ...pageLines].join('\n') + '\n', { inputSha256, expectedSourceId: inputSha256 }); + // This projection has page-local offsets even though artifact IDs and page + // indices remain document-global. Delivered pages are provisional until + // the final receipt; callbacks must not treat delivery as completion. + const page = { ...document, pageIndex: currentPage, sourceTextScope: 'page', streamComplete: false }; + pageLines = []; + try { await Promise.race([Promise.resolve().then(() => options.onPage(page)), failed]); } + catch (cause) { + if (failure) throw failure; + throw new PdfExtractionError('callback', 'Page callback failed', { cause }); + } + pageCount++; + currentPage = null; + } else if (type === 'document_finished') { + if (currentPage !== null || pageCount !== manifest.page_count || record.sequence_scope !== 'document' || record.status !== 'completed' || + !Array.isArray(record.errors) || record.errors.length) fail('Invalid document completion'); + checkCounts(record.artifact_counts, totals); + checkCounts(record.route_counts, routeTotals); + finished = true; + } else { + const field = countFields[type]; + if (!field) fail('Unsupported streaming artifact'); + if (currentPage !== null) { + if (record.sequence_scope !== 'page' || (type === 'rag_chunk' ? record.page_start !== currentPage || record.page_end !== currentPage : record.page_index !== currentPage)) fail('Artifact page mismatch'); + pageBytes += buffer.length + 1; + if (pageBytes > options.maxPageBytes) throw new PdfExtractionError('output_limit', 'Stream exceeded its page budget'); + pageLines.push(raw); + pageCounts[field]++; + } else if (type !== 'debug_asset' || record.sequence_scope !== 'document' || + ['span_ids', 'block_ids', 'chunk_ids'].some((key) => provenance[key]?.length)) fail('Artifact outside page lifecycle'); + totals[field]++; + } + if (pageBytes > options.maxPageBytes) throw new PdfExtractionError('output_limit', 'Stream exceeded its page budget'); + }; + try { + let pending = Buffer.alloc(0); + for await (const chunk of child.stdout) { + if (failure) throw failure; + outputBytes += chunk.length; + if (outputBytes > options.maxOutputBytes) throw new PdfExtractionError('output_limit', 'Stream exceeded its aggregate output budget'); + let start = 0; + while (start < chunk.length) { + const newline = chunk.indexOf(10, start); + const end = newline < 0 ? chunk.length : newline; + if (pending.length + end - start > options.maxRecordBytes) throw new PdfExtractionError('output_limit', 'Stream exceeded its record byte budget'); + const part = chunk.subarray(start, end); + if (newline < 0) { pending = Buffer.concat([pending, part]); break; } + const recordBytes = pending.length ? Buffer.concat([pending, part]) : part; + pending = Buffer.alloc(0); + await line(recordBytes); + start = newline + 1; + } + } + if (failure) throw failure; + const result = await Promise.race([closed, failed]); + if (result.signal) throw new PdfExtractionError('crashed', `Parser terminated by ${result.signal}`); + if (result.code !== 0) throw nativeFailure ?? new PdfExtractionError('document', `Parser returned exit ${result.code}`); + if (pending.length) fail('Truncated JSONL record'); + if (!finished) fail('Stream ended before document completion'); + return { complete: true, inputSha256, schemaVersion: manifest.schema_version, pageCount, records, outputBytes, artifactCounts: totals }; + } catch (error) { + stop(error instanceof PdfExtractionError ? error : new PdfExtractionError('protocol', 'Streaming extraction failed', { cause: error })); + await closed; + await termination; + throw failure; + } finally { + clearTimeout(timer); + options.signal?.removeEventListener('abort', abort); + } +} + +/** Incremental local CLI adapter. Memory budgets cover retained JSONL and IDs, + * not the parser's RSS or data the caller retains inside its callback. */ +export async function extractPdfPages(buffer, options) { + if (!(buffer instanceof Uint8Array)) throw new TypeError('PDF input must be bytes'); + if (!options || typeof options.binaryPath !== 'string' || !options.binaryPath || typeof options.onPage !== 'function') throw new TypeError('binaryPath and onPage are required'); + const defaults = { timeoutMs: 45_000, maxOutputBytes: 25 * 1024 * 1024, maxPageBytes: 4 * 1024 * 1024, + maxRecordBytes: 1024 * 1024, maxRecords: 100_000, maxPages: 10_000 }; + const limits = { ...options }; + for (const [key, defaultValue] of Object.entries(defaults)) { + limits[key] ??= defaultValue; + if (!integer(limits[key]) || limits[key] < 1 || (key === 'timeoutMs' && limits[key] > 2_147_483_647)) throw new RangeError('Invalid streaming limits'); + } + if (options.signal?.aborted) throw new PdfExtractionError('aborted', 'Extraction was aborted'); + if (options.maxNativePageMemoryBytes !== undefined && (!integer(options.maxNativePageMemoryBytes) || options.maxNativePageMemoryBytes < 1)) throw new RangeError('Invalid native page allocation budget'); + const input = Buffer.from(buffer); + const inputSha256 = createHash('sha256').update(input).digest('hex'); + let directory; + let failure; + try { + directory = await mkdtemp(join(tmpdir(), 'pdf-parser-stream-')); + const path = join(directory, 'input.pdf'); + await writeFile(path, input, { mode: 0o600 }); + const args = ['extract-adaptive', '--input', path, '--source-id', inputSha256, '--format', 'stream-jsonl']; + if (options.maxNativePageMemoryBytes !== undefined) args.push('--max-page-memory-bytes', String(options.maxNativePageMemoryBytes)); + if (!options.enableOcr) args.push('--no-ocr'); + return await consume(options.binaryPath, args, limits, inputSha256); + } catch (error) { + failure = error instanceof PdfExtractionError ? error : new PdfExtractionError('unavailable', 'Local streaming I/O failed', { cause: error }); + throw failure; + } finally { + if (directory) { + try { await rm(directory, { recursive: true, force: true }); } + catch (cause) { + const cleanup = new PdfExtractionError('cleanup', 'Could not remove streaming scratch files', { cause }); + if (failure) failure.cleanupError = cleanup; + else throw cleanup; + } + } + } +} diff --git a/integrations/javascript/stream.test.mjs b/integrations/javascript/stream.test.mjs new file mode 100644 index 0000000..2df9c07 --- /dev/null +++ b/integrations/javascript/stream.test.mjs @@ -0,0 +1,201 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { mkdtemp, writeFile, readFile, rm, access } from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { setTimeout as delay } from 'node:timers/promises'; +import { extractPdfPages } from './stream.mjs'; + +const input = Buffer.from('%PDF-stream-test'); +const hash = (value) => createHash('sha256').update(value).digest('hex'); +const inputSha256 = hash(input); +const bbox = { x0: 1, y0: 2, x1: 9, y1: 10 }; +const countKeys = ['spans', 'blocks', 'tables', 'route_traces', 'specialist_requests', 'specialist_attempts', 'specialist_responses', 'specialist_results', 'rag_chunks', 'debug_assets']; +const counts = (value = 0) => Object.fromEntries(countKeys.map((key) => [key, ['spans', 'blocks'].includes(key) ? value : 0])); +const routes = (pages) => ({ native_pages: pages, page_routes: pages, region_routes: 0, ocr_regions: 0, table_regions: 0, formula_regions: 0 }); +function fixture() { + const records = []; + const record = (type, id, fields = {}, refs = {}) => { + records.push({ schema_name: type, schema_version: '0.14.0', record_type: type, event_type: type, + event_index: records.length, sequence_scope: type.startsWith('document_') ? 'document' : 'page', + document_id: 'doc', source_id: inputSha256, + provenance: { document_id: 'doc', source_id: inputSha256, input_sha256: inputSha256, artifact_id: id, + span_ids: [], block_ids: [], chunk_ids: [], ...refs }, ...fields }); + }; + record('document_manifest', 'document_manifest', { streaming: true, input_sha256: inputSha256, page_count: 2, + errors: [], text_contract: { offset_unit: 'utf8_byte', range_convention: 'half_open', page_index_base: 0, + geometry: { coordinate_space: 'pdf_user_space', unit: 'point', highlight_precision: 'span_bbox', display_transform: null } } }); + for (let p = 0; p < 2; p++) { + record('page_started', `page-started-${p}`, { page_index: p, status: 'running' }); + const text = `Page ${p} πŸ˜€`; + const view = (id, kind) => ({ text_view_id: `${id}:text`, kind, text_sha256: hash(text), offset_unit: 'utf8_byte', range_convention: 'half_open', normalization: 'none' }); + const refs = { span_ids: [`span-${p}`], block_ids: [`block-${p}`] }; + record('span', `span-${p}`, { span_id: `span-${p}`, page_index: p, text, bbox, text_view: view(`span-${p}`, 'reconciled_span') }, refs); + record('block', `block-${p}`, { block_id: `block-${p}`, page_index: p, text, bbox, text_view: view(`block-${p}`, 'reconciled_block'), + source_map: [{ text_start_byte: 0, text_end_byte: Buffer.byteLength(text), source_span_id: `span-${p}`, source_start_byte: 0, source_end_byte: Buffer.byteLength(text) }] }, refs); + record('page_finished', `page-finished-${p}`, { page_index: p, status: 'completed', artifact_counts: counts(1), route_counts: routes(1) }); + } + record('document_finished', 'document_finished', { status: 'completed', errors: [], artifact_counts: counts(2), route_counts: routes(2) }); + return records; +} +const jsonl = (records) => records.map((record) => JSON.stringify(record)).join('\n') + '\n'; +async function fake(t, script) { + const directory = await mkdtemp(join(tmpdir(), 'pdf-stream-test-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const path = join(directory, 'parser'); + const audit = join(directory, 'audit.json'); + await writeFile(path, `#!${process.execPath}\nconst fs=require('node:fs'); const path=process.argv[process.argv.indexOf('--input')+1]; fs.writeFileSync(${JSON.stringify(audit)},JSON.stringify({path,mode:fs.statSync(path).mode&511,input:fs.readFileSync(path).toString(),args:process.argv}));\n${script}\n`, { mode: 0o700 }); + return { path, audit, directory }; +} +async function run(t, records = fixture(), extra = {}, code = '') { + const driver = await fake(t, `process.stdout.write(${JSON.stringify(jsonl(records))}); ${code}`); + const pages = []; + const receipt = await extractPdfPages(input, { binaryPath: driver.path, onPage: (page) => pages.push(page), ...extra }); + return { ...driver, pages, receipt }; +} + +test('streams isolated page projections with global IDs and accepts only final receipt', async (t) => { + const result = await run(t); + assert.equal(result.receipt.complete, true); + assert.equal(result.receipt.pageCount, 2); + assert.equal(result.receipt.records, 10); + assert.equal(result.pages[1].sourceText, 'Page 1 πŸ˜€'); + assert.equal(result.pages[1].sourceMap[0].start, 0); + assert.equal(result.pages[1].sourceMap[0].sourceSpanId, 'span-1'); + assert.equal(result.pages[1].sourceTextScope, 'page'); + assert.equal(result.pages[1].streamComplete, false); + const audit = JSON.parse(await readFile(result.audit, 'utf8')); + assert.equal(audit.mode, 0o600); + assert.equal(audit.input, input.toString()); + assert.ok(audit.args.includes('stream-jsonl')); + assert.ok(audit.args.includes('--no-ocr')); + await assert.rejects(access(audit.path), { code: 'ENOENT' }); +}); + +test('forwards the native allocation budget and validates it before starting', async (t) => { + const result = await run(t, fixture(), { maxNativePageMemoryBytes: 4096 }); + const { args } = JSON.parse(await readFile(result.audit, 'utf8')); + assert.equal(args[args.indexOf('--max-page-memory-bytes') + 1], '4096'); + for (const value of [0, -1, 1.5, NaN, Number.MAX_SAFE_INTEGER + 1]) { + await assert.rejects(extractPdfPages(input, { binaryPath: result.path, onPage() {}, maxNativePageMemoryBytes: value }), RangeError); + } +}); + +for (const [diagnostic, code] of [['MemoryBudgetExceeded', 'memory_limit'], ['OutputLimitExceeded', 'output_limit']]) { + test(`classifies native ${code} without a terminal receipt`, async (t) => { + const driver = await fake(t, `process.stderr.write('error: ${diagnostic}'); process.exitCode=1;`); + await assert.rejects(extractPdfPages(input, { binaryPath: driver.path, onPage() {} }), { code }); + }); +} + +test('awaits callbacks serially even when both pages arrive in one write', async (t) => { + const driver = await fake(t, `process.stdout.write(${JSON.stringify(jsonl(fixture()))});`); + let release; + let entered; + const firstEntered = new Promise((resolve) => { entered = resolve; }); + const gate = new Promise((resolve) => { release = resolve; }); + const seen = []; + const extraction = extractPdfPages(input, { binaryPath: driver.path, onPage: async (page) => { + seen.push(page.pageIndex); + if (page.pageIndex === 0) { entered(); await gate; } + } }); + await firstEntered; + assert.deepEqual(seen, [0]); + release(); + assert.equal((await extraction).complete, true); + assert.deepEqual(seen, [0, 1]); +}); + +for (const [label, mutate] of [ + ['truncated lifecycle', (records) => records.pop()], + ['event index gap', (records) => { records[2].event_index++; }], + ['wrong source identity', (records) => { records[2].source_id = 'wrong'; }], + ['duplicate global artifact', (records) => { records[6].provenance.artifact_id = 'span-0'; }], + ['page count mismatch', (records) => { records[0].page_count = 3; }], + ['per-page counts mismatch', (records) => { records[4].artifact_counts.spans++; }], + ['terminal counts mismatch', (records) => { records.at(-1).artifact_counts.spans++; }], + ['terminal route counts mismatch', (records) => { records.at(-1).route_counts.native_pages++; }], + ['record after completion', (records) => { records.push({ ...records.at(-1), event_index: records.length }); }], + ['artifact page mismatch', (records) => { records[2].page_index = 1; }], + ['invalid text contract', (records) => { records[0].text_contract.offset_unit = 'utf16_code_unit'; }], + ['second manifest with distinct id', (records) => { records[1] = { ...records[0], event_index: 1, + provenance: { ...records[0].provenance, artifact_id: 'other-manifest' } }; }], +]) test(`rejects ${label}`, async (t) => { + const records = fixture(); + mutate(records); + await assert.rejects(run(t, records), { code: 'protocol' }); +}); + +for (const limit of [{ maxOutputBytes: 100 }, { maxPageBytes: 100 }, { maxRecordBytes: 100 }, { maxRecords: 3 }]) { + test(`enforces ${Object.keys(limit)[0]}`, async (t) => { + await assert.rejects(run(t, fixture(), limit), { code: 'output_limit' }); + }); +} +test('rejects page-count budget before callbacks', async (t) => { + await assert.rejects(run(t, fixture(), { maxPages: 1 }), { code: 'output_limit' }); +}); +test('fatal UTF-8 decoder rejects malformed bytes', async (t) => { + const driver = await fake(t, 'process.stdout.write(Buffer.from([255,10]));'); + await assert.rejects(extractPdfPages(input, { binaryPath: driver.path, onPage() {} }), { code: 'protocol' }); +}); +test('requires terminated JSONL lines', async (t) => { + const driver = await fake(t, `process.stdout.write(${JSON.stringify(jsonl(fixture()).slice(0, -1))});`); + await assert.rejects(extractPdfPages(input, { binaryPath: driver.path, onPage() {} }), { code: 'protocol' }); +}); +test('valid terminal event followed by nonzero exit never succeeds', async (t) => { + await assert.rejects(run(t, fixture(), {}, 'process.exitCode=2;'), { code: 'document' }); +}); +test('abort terminates an unresolved callback and cleans private input', async (t) => { + const controller = new AbortController(); + const driver = await fake(t, `process.stdout.write(${JSON.stringify(jsonl(fixture()))}); setInterval(()=>{},1000);`); + await assert.rejects(extractPdfPages(input, { binaryPath: driver.path, signal: controller.signal, + onPage() { controller.abort(); return new Promise(() => {}); } }), { code: 'aborted' }); + const audit = JSON.parse(await readFile(driver.audit, 'utf8')); + await assert.rejects(access(audit.path), { code: 'ENOENT' }); +}); +test('callback rejection terminates parser and retains original cause', async (t) => { + const expected = new Error('callback rejected'); + await assert.rejects(run(t, fixture(), { onPage() { throw expected; } }, 'setInterval(()=>{},1000);'), (error) => error.code === 'callback' && error.cause === expected); +}); +test('timeout includes callback backpressure', async (t) => { + await assert.rejects(run(t, fixture(), { timeoutMs: 200, onPage: () => new Promise(() => {}) }, 'setInterval(()=>{},1000);'), { code: 'timeout' }); +}); +test('validates limits and required callback before launching', async () => { + await assert.rejects(extractPdfPages(input, { binaryPath: 'unused' }), TypeError); + await assert.rejects(extractPdfPages(input, { binaryPath: 'unused', onPage() {}, maxRecords: 0 }), RangeError); + await assert.rejects(extractPdfPages(input, { binaryPath: 'unused', onPage() {}, signal: AbortSignal.abort() }), { code: 'aborted' }); +}); +test('input is snapshotted before asynchronous staging', async (t) => { + const callerBytes = Buffer.from(input); + const driver = await fake(t, `process.stdout.write(${JSON.stringify(jsonl(fixture()))});`); + const promise = extractPdfPages(callerBytes, { binaryPath: driver.path, onPage() {} }); + callerBytes.fill(0); + assert.equal((await promise).inputSha256, inputSha256); + const audit = JSON.parse(await readFile(driver.audit, 'utf8')); + assert.equal(audit.input, input.toString()); +}); +test('missing binary is unavailable and cleans staged input', async () => { + await assert.rejects(extractPdfPages(input, { binaryPath: '/does-not-exist/pdf-parser', onPage() {} }), { code: 'unavailable' }); +}); +test('timeout kills POSIX wrapper descendants', { skip: process.platform === 'win32' }, async (t) => { + const driver = await fake(t, `const child=require('node:child_process').spawn(process.execPath,['-e','setInterval(()=>{},1000)']); const auditPath=${JSON.stringify('AUDIT_PATH')}; fs.writeFileSync(auditPath,String(child.pid)); setInterval(()=>{},1000);`); + const pidPath = join(driver.directory, 'child.pid'); + const script = await readFile(driver.path, 'utf8'); + await writeFile(driver.path, script.replace(JSON.stringify('AUDIT_PATH'), JSON.stringify(pidPath))); + await assert.rejects(extractPdfPages(input, { binaryPath: driver.path, timeoutMs: 500, onPage() {} }), { code: 'timeout' }); + const pid = Number(await readFile(pidPath, 'utf8')); + t.after(() => { try { process.kill(pid, 'SIGKILL'); } catch {} }); + let alive = true; + for (let attempt = 0; attempt < 20; attempt++) { + try { + process.kill(pid, 0); + // Zombies still occupy PID slots and do not qualify as cleaned up. + } catch { alive = false; } + if (!alive) break; + await delay(50); + } + assert.equal(alive, false, 'owned descendant must be terminated'); + const audit = JSON.parse(await readFile(driver.audit, 'utf8')); + await assert.rejects(access(audit.path), { code: 'ENOENT' }); +}); diff --git a/integrations/javascript/subprocess.mjs b/integrations/javascript/subprocess.mjs new file mode 100644 index 0000000..04ab008 --- /dev/null +++ b/integrations/javascript/subprocess.mjs @@ -0,0 +1,34 @@ +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { setTimeout as delay } from 'node:timers/promises'; + +const inspect = promisify(execFile); +const kill = pid => { try { process.kill(pid, 'SIGKILL'); } catch (error) { if (error.code !== 'ESRCH') throw error; } }; + +/** Give each live parent a chance to reap killed children before killing it. + * A blocked parent or unavailable process inventory can leave cleanup uncertain; + * callers retain that gap on the original failure instead of claiming success. */ +export async function terminateSubprocess(child, grouped) { + if (!child.pid) return; + if (!grouped) { child.kill('SIGKILL'); return; } + const deadline = performance.now() + 1000; + let gap; + try { + while (performance.now() < deadline) { + const { stdout } = await inspect('ps', ['-eo', 'pid=,ppid=,pgid='], { timeout: 200, maxBuffer: 1024 * 1024 }); + const members = stdout.trim().split('\n').map(line => line.trim().split(/\s+/u).map(Number)) + .filter(([pid, parent, group]) => Number.isSafeInteger(pid) && pid > 1 && Number.isSafeInteger(parent) && group === child.pid && pid !== child.pid); + if (members.length === 0) break; + const parents = new Set(members.map(([, parent]) => parent)); + for (const [pid] of members) if (!parents.has(pid)) kill(pid); + // Do not promote an unreaped zombie to a terminated subtree. A fresh + // inventory must show removal before its parent becomes a leaf. + await delay(20); + } + if (performance.now() >= deadline) gap = new Error('Descendant reaping exceeded the cleanup grace period'); + } catch (cause) { gap = new Error('Descendant inventory or termination failed', { cause }); } + // Also catches descendants forked after the last inventory. Escaped sessions + // and parents unable to reap are outside this bounded cleanup qualification. + try { kill(-child.pid); } catch (cause) { gap ??= new Error('Process group termination failed', { cause }); child.kill('SIGKILL'); } + if (gap) throw gap; +} diff --git a/integrations/javascript/subprocess.test.mjs b/integrations/javascript/subprocess.test.mjs new file mode 100644 index 0000000..c7a31bc --- /dev/null +++ b/integrations/javascript/subprocess.test.mjs @@ -0,0 +1,34 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, writeFile, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; +import { extractPdfDocument, extractPdfPages } from './index.mjs'; + +for (const [name, extract] of [['batch', extractPdfDocument], ['stream', extractPdfPages]]) { + test(`${name} cancellation lets nested parents reap descendants before termination`, { skip: process.platform === 'win32' }, async t => { + const directory = await mkdtemp(join(tmpdir(), 'pdf-parser-reaping-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const pids = join(directory, 'pids.json'); + const reaped = join(directory, 'reaped.log'); + const middle = `const fs=require('node:fs'); const child=require('node:child_process').spawn(process.execPath,['-e','setInterval(()=>{},1000)']); + fs.writeFileSync(${JSON.stringify(pids)},JSON.stringify({middle:process.pid,leaf:child.pid})); + child.on('exit',()=>fs.appendFileSync(${JSON.stringify(reaped)},'leaf-reaped\\n')); setInterval(()=>{},1000);`; + const binaryPath = join(directory, 'parser'); + await writeFile(binaryPath, `#!${process.execPath}\nconst fs=require('node:fs'); + const child=require('node:child_process').spawn(process.execPath,['-e',${JSON.stringify(middle)}]); + child.on('exit',()=>fs.appendFileSync(${JSON.stringify(reaped)},'middle-reaped\\n')); setInterval(()=>{},1000);\n`, { mode: 0o700 }); + const controller = new AbortController(); + const operation = extract(Buffer.from('%PDF-test'), { binaryPath, timeoutMs: 5000, signal: controller.signal, onPage() {} }); + let audit; + for (let attempt = 0; attempt < 100 && !audit; attempt++) { + try { audit = JSON.parse(await readFile(pids, 'utf8')); } catch (error) { if (error.code !== 'ENOENT') throw error; await delay(20); } + } + assert.ok(audit, 'nested descendant must exist before cancellation'); + controller.abort(); + await assert.rejects(operation, { code: 'aborted' }); + assert.equal(await readFile(reaped, 'utf8'), 'leaf-reaped\nmiddle-reaped\n'); + for (const pid of Object.values(audit)) assert.throws(() => process.kill(pid, 0), { code: 'ESRCH' }); + }); +} diff --git a/src/adapter.zig b/src/adapter.zig index 357b5bf..28ec467 100644 --- a/src/adapter.zig +++ b/src/adapter.zig @@ -27,6 +27,8 @@ pub const AdaptiveAdapterOptions = struct { debug_assets_dir: ?[]const u8 = null, emit_specialist_requests_path: ?[]const u8 = null, specialist_config_path: ?[]const u8 = null, + max_output_bytes: ?usize = null, + max_page_memory_bytes: ?usize = null, }; pub const AdaptiveAdapterSummary = struct { @@ -42,6 +44,10 @@ pub fn extractAdaptive( writer: anytype, options: AdaptiveAdapterOptions, ) !AdaptiveAdapterSummary { + if (options.max_output_bytes != null or options.max_page_memory_bytes != null) { + if (options.format != .stream_jsonl or options.emit_specialist_requests_path != null) return error.InvalidResourceLimitOptions; + if (options.max_output_bytes == 0 or options.max_page_memory_bytes == 0) return error.InvalidResourceLimit; + } const input_sha256 = try schema.sha256Hex(allocator, document.data); defer allocator.free(input_sha256); @@ -102,6 +108,8 @@ pub fn extractAdaptive( .adaptive_options = adaptive_options, .schema_options = render_options, .include_debug_asset_refs = options.include_debug_asset_refs, + .max_output_bytes = options.max_output_bytes, + .max_page_memory_bytes = options.max_page_memory_bytes, }); return .{ .format = options.format, diff --git a/src/coverage.zig b/src/coverage.zig new file mode 100644 index 0000000..aeba54c --- /dev/null +++ b/src/coverage.zig @@ -0,0 +1,273 @@ +//! Evidence-based text availability. Completeness remains unverified: retained +//! spans and successful recognition do not prove that all visible marks were read. +const std = @import("std"); +const layout = @import("layout.zig"); +const reconcile = @import("reconcile.zig"); + +pub const Status = enum { text_available, empty_unverified, recovery_required, recovery_failed, recovered }; +pub const Completeness = enum { unverified }; +pub const RecoveryState = enum { not_required, not_invoked, completed, empty, unavailable, failed, timeout, invalid_output }; + +pub const Assessment = struct { + status: Status = .empty_unverified, + completeness: Completeness = .unverified, + native_span_count: usize = 0, + recognized_span_count: usize = 0, + requested_recovery: bool = false, + recovery_state: RecoveryState = .not_required, + unresolved_region_count: usize = 0, +}; + +pub fn assessRegion(result: anytype, route: anytype) Assessment { + var assessment = countSpans(result, route.page_index, route.bbox); + var unresolved = false; + if (route.route.needs_ocr) { + assessment.requested_recovery = true; + const state = ocrState(result, route.page_index); + assessment.recovery_state = combine(assessment.recovery_state, state); + unresolved = state != .completed or countRecognized(result, route.page_index, route.bbox, .fresh_ocr) == 0; + } + if (route.route.needs_formula_model) { + assessment.requested_recovery = true; + const state = formulaState(result, route.page_index, route.region_index); + assessment.recovery_state = combine(assessment.recovery_state, state); + unresolved = unresolved or state != .completed or countRecognized(result, route.page_index, route.bbox, .formula_model) == 0; + } + assessment.unresolved_region_count = @intFromBool(unresolved); + assessment.status = statusFor(assessment, unresolved); + return assessment; +} + +pub fn assessPage(result: anytype, page_index: u32) Assessment { + var assessment = countSpans(result, page_index, null); + var unresolved = false; + for (result.page_routes) |route| { + if (route.page_index != page_index or !route.route.needs_ocr) continue; + assessment.requested_recovery = true; + const state = ocrState(result, page_index); + assessment.recovery_state = combine(assessment.recovery_state, state); + unresolved = state != .completed or countRecognized(result, page_index, null, .fresh_ocr) == 0; + } + for (result.region_routes) |route| { + if (route.page_index != page_index) continue; + const region = assessRegion(result, route); + if (!region.requested_recovery) continue; + assessment.requested_recovery = true; + assessment.recovery_state = combine(assessment.recovery_state, region.recovery_state); + assessment.unresolved_region_count += region.unresolved_region_count; + unresolved = unresolved or region.unresolved_region_count > 0; + } + assessment.status = statusFor(assessment, unresolved); + return assessment; +} + +fn statusFor(assessment: Assessment, unresolved: bool) Status { + if (assessment.requested_recovery) { + if (unresolved) return switch (assessment.recovery_state) { + .unavailable, .failed, .timeout, .invalid_output => .recovery_failed, + else => .recovery_required, + }; + return .recovered; + } + if (assessment.status == .text_available) return .text_available; + return .empty_unverified; +} + +fn intersects(a: layout.BBox, b: layout.BBox) bool { + return @min(a.x1, b.x1) > @max(a.x0, b.x0) and @min(a.y1, b.y1) > @max(a.y0, b.y0); +} + +fn included(span: anytype, page_index: u32, bbox: ?layout.BBox) bool { + if (span.span.page_index != page_index or std.mem.trim(u8, span.span.text, " \t\r\n").len == 0) return false; + return if (bbox) |region| intersects(span.span.bbox, region) else true; +} + +fn countSpans(result: anytype, page_index: u32, bbox: ?layout.BBox) Assessment { + var assessment: Assessment = .{}; + for (result.reconciled.spans) |span| { + if (!included(span, page_index, bbox)) continue; + assessment.status = .text_available; + if (reconcile.hasSource(span.source_mask, .native_pdf) or reconcile.hasSource(span.source_mask, .embedded_ocr)) assessment.native_span_count += 1; + if (reconcile.hasSource(span.source_mask, .fresh_ocr) or reconcile.hasSource(span.source_mask, .formula_model)) assessment.recognized_span_count += 1; + } + return assessment; +} + +fn countRecognized(result: anytype, page_index: u32, bbox: ?layout.BBox, source: layout.SourceKind) usize { + var count: usize = 0; + for (result.reconciled.spans) |span| { + if (included(span, page_index, bbox) and reconcile.hasSource(span.source_mask, source)) count += 1; + } + return count; +} + +fn ocrState(result: anytype, page_index: u32) RecoveryState { + var state: RecoveryState = .not_invoked; + for (result.ocr_attempts) |attempt| { + if (attempt.page_index != page_index) continue; + state = switch (attempt.status) { + .not_invoked => .not_invoked, + .completed => .completed, + .empty => .empty, + .unavailable => .unavailable, + .failed => .failed, + .timeout => .timeout, + .invalid_output => .invalid_output, + }; + if (attempt.selected) return state; + } + // A nonselected completed attempt is insufficient to attribute output to it. + return if (state == .completed) .not_invoked else state; +} + +fn formulaState(result: anytype, page_index: u32, region_index: u32) RecoveryState { + var state: RecoveryState = .not_invoked; + for (result.formula_attempts) |attempt| { + if (attempt.page_index != page_index or attempt.region_index != region_index) continue; + state = switch (attempt.status) { + .completed => .completed, + .empty => .empty, + .unavailable => .unavailable, + .failed => .failed, + .timeout => .timeout, + .invalid_output => .invalid_output, + }; + } + return state; +} + +fn combine(a: RecoveryState, b: RecoveryState) RecoveryState { + return if (severity(b) > severity(a)) b else a; +} + +fn severity(state: RecoveryState) u8 { + return switch (state) { + .not_required => 0, + .completed => 1, + .not_invoked => 2, + .empty => 3, + .unavailable => 4, + .failed => 5, + .timeout => 6, + .invalid_output => 7, + }; +} + +const TestRoute = struct { + page_index: u32 = 0, + region_index: u32 = 0, + bbox: layout.BBox = .{ .x0 = 0, .y0 = 0, .x1 = 100, .y1 = 100 }, + route: struct { needs_ocr: bool = false, needs_formula_model: bool = false, needs_table_model: bool = false } = .{}, +}; +const TestOcrStatus = enum { not_invoked, completed, empty, unavailable, failed, timeout, invalid_output }; +const TestOcrAttempt = struct { page_index: u32 = 0, status: TestOcrStatus, selected: bool = false }; +const TestFormulaStatus = enum { completed, empty, unavailable, failed, timeout, invalid_output }; +const TestFormulaAttempt = struct { page_index: u32 = 0, region_index: u32 = 0, status: TestFormulaStatus }; +const TestResult = struct { + reconciled: struct { spans: []const reconcile.ReconciledSpan = &.{} } = .{}, + page_routes: []const TestRoute = &.{}, + region_routes: []const TestRoute = &.{}, + ocr_attempts: []const TestOcrAttempt = &.{}, + formula_attempts: []const TestFormulaAttempt = &.{}, +}; + +fn testSpan(source: layout.SourceKind, bbox: layout.BBox) reconcile.ReconciledSpan { + return .{ + .span = layout.TextSpan.init(.{ .text = "text", .bbox = bbox, .source = source }), + .source_mask = @as(reconcile.SourceMask, 1) << @as(u5, @intCast(@backingInt(source))), + .chosen_source = source, + .confidence = 1, + }; +} + +test "coverage never proves blankness or completeness from text presence" { + const empty = assessPage(TestResult{}, 0); + try std.testing.expectEqual(Status.empty_unverified, empty.status); + try std.testing.expectEqual(Completeness.unverified, empty.completeness); + const native = testSpan(.native_pdf, .{ .x1 = 10, .y1 = 10 }); + const text = assessPage(TestResult{ .reconciled = .{ .spans = &.{native} } }, 0); + try std.testing.expectEqual(Status.text_available, text.status); + try std.testing.expectEqual(Completeness.unverified, text.completeness); +} + +test "coverage distinguishes disabled unavailable empty and selected OCR recovery" { + const route: TestRoute = .{ .route = .{ .needs_ocr = true } }; + const recognized = testSpan(.fresh_ocr, .{ .x1 = 10, .y1 = 10 }); + for ([_]TestOcrStatus{ .not_invoked, .unavailable, .failed, .timeout, .invalid_output, .empty }) |state| { + const assessed = assessPage(TestResult{ .page_routes = &.{route}, .ocr_attempts = &.{.{ .status = state }} }, 0); + try std.testing.expectEqual(if (state == .empty or state == .not_invoked) Status.recovery_required else Status.recovery_failed, assessed.status); + try std.testing.expectEqualStrings(@tagName(state), @tagName(assessed.recovery_state)); + } + const recovered = assessPage(TestResult{ .page_routes = &.{route}, .reconciled = .{ .spans = &.{recognized} }, .ocr_attempts = &.{ .{ .status = .failed }, .{ .status = .completed, .selected = true } } }, 0); + try std.testing.expectEqual(Status.recovered, recovered.status); + try std.testing.expectEqual(RecoveryState.completed, recovered.recovery_state); + const unselected = assessPage(TestResult{ .page_routes = &.{route}, .reconciled = .{ .spans = &.{recognized} }, .ocr_attempts = &.{.{ .status = .completed }} }, 0); + try std.testing.expectEqual(Status.recovery_required, unselected.status); +} + +test "coverage page OCR does not recover a disjoint scan region" { + const native = testSpan(.native_pdf, .{ .x1 = 20, .y1 = 20 }); + const recognized = testSpan(.fresh_ocr, .{ .x1 = 20, .y1 = 20 }); + const scan: TestRoute = .{ .bbox = .{ .x0 = 50, .y0 = 50, .x1 = 100, .y1 = 100 }, .route = .{ .needs_ocr = true } }; + const result: TestResult = .{ .region_routes = &.{scan}, .reconciled = .{ .spans = &.{ native, recognized } }, .ocr_attempts = &.{.{ .status = .completed, .selected = true }} }; + try std.testing.expectEqual(Status.recovery_required, assessRegion(result, scan).status); + const page = assessPage(result, 0); + try std.testing.expectEqual(Status.recovery_required, page.status); + try std.testing.expectEqual(@as(usize, 1), page.unresolved_region_count); + try std.testing.expectEqual(@as(usize, 1), page.native_span_count); +} + +test "coverage formula recovery requires matching region output and table routes remain native" { + const formula: TestRoute = .{ .region_index = 3, .route = .{ .needs_formula_model = true } }; + const recognized = testSpan(.formula_model, .{ .x1 = 10, .y1 = 10 }); + const result: TestResult = .{ .region_routes = &.{formula}, .reconciled = .{ .spans = &.{recognized} }, .formula_attempts = &.{ .{ .region_index = 9, .status = .failed }, .{ .region_index = 3, .status = .completed } } }; + try std.testing.expectEqual(Status.recovered, assessRegion(result, formula).status); + try std.testing.expectEqual(Status.recovered, assessPage(result, 0).status); + const absent_output = assessRegion(TestResult{ .formula_attempts = result.formula_attempts }, formula); + try std.testing.expectEqual(Status.recovery_required, absent_output.status); + const native = testSpan(.native_pdf, .{ .x1 = 10, .y1 = 10 }); + const table: TestRoute = .{ .route = .{ .needs_table_model = true } }; + const table_assessment = assessRegion(TestResult{ .reconciled = .{ .spans = &.{native} } }, table); + try std.testing.expectEqual(Status.text_available, table_assessment.status); + try std.testing.expect(!table_assessment.requested_recovery); +} + +test "coverage ignores whitespace and other pages but recognizes retained merged provenance" { + var whitespace = testSpan(.native_pdf, .{ .x1 = 10, .y1 = 10 }); + whitespace.span.text = " \t\n"; + var other_page = testSpan(.native_pdf, .{ .x1 = 10, .y1 = 10 }); + other_page.span.page_index = 1; + try std.testing.expectEqual(Status.empty_unverified, assessPage(TestResult{ .reconciled = .{ .spans = &.{ whitespace, other_page } } }, 0).status); + var merged = testSpan(.native_pdf, .{ .x1 = 10, .y1 = 10 }); + merged.source_mask |= @as(reconcile.SourceMask, 1) << @as(u5, @intCast(@backingInt(layout.SourceKind.fresh_ocr))); + const route: TestRoute = .{ .route = .{ .needs_ocr = true } }; + const result: TestResult = .{ .page_routes = &.{route}, .reconciled = .{ .spans = &.{merged} }, .ocr_attempts = &.{.{ .status = .completed, .selected = true }} }; + const assessed = assessPage(result, 0); + try std.testing.expectEqual(Status.recovered, assessed.status); + try std.testing.expectEqual(@as(usize, 1), assessed.native_span_count); + try std.testing.expectEqual(@as(usize, 1), assessed.recognized_span_count); +} + +test "coverage successful OCR cannot hide failed formula recovery on a mixed page" { + const scan: TestRoute = .{ .region_index = 1, .route = .{ .needs_ocr = true } }; + const formula: TestRoute = .{ .region_index = 2, .route = .{ .needs_formula_model = true } }; + const recognized = testSpan(.fresh_ocr, .{ .x1 = 10, .y1 = 10 }); + const failed: TestResult = .{ + .region_routes = &.{ scan, formula }, + .reconciled = .{ .spans = &.{recognized} }, + .ocr_attempts = &.{.{ .status = .completed, .selected = true }}, + .formula_attempts = &.{.{ .region_index = 2, .status = .failed }}, + }; + const assessed = assessPage(failed, 0); + try std.testing.expectEqual(Status.recovery_failed, assessed.status); + try std.testing.expectEqual(RecoveryState.failed, assessed.recovery_state); + try std.testing.expectEqual(@as(usize, 1), assessed.unresolved_region_count); + const formula_span = testSpan(.formula_model, .{ .x1 = 10, .y1 = 10 }); + const retried: TestResult = .{ + .region_routes = failed.region_routes, + .reconciled = .{ .spans = &.{ recognized, formula_span } }, + .ocr_attempts = failed.ocr_attempts, + .formula_attempts = &.{ .{ .region_index = 2, .status = .failed }, .{ .region_index = 2, .status = .completed } }, + }; + try std.testing.expectEqual(Status.recovered, assessPage(retried, 0).status); +} diff --git a/src/integration_test.zig b/src/integration_test.zig index d329952..9880e80 100644 --- a/src/integration_test.zig +++ b/src/integration_test.zig @@ -57,6 +57,79 @@ fn expectProvenanceObject(value: std.json.Value) !void { try std.testing.expect(provenance.object.get("route_reasons") != null); } +fn expectSourceMapsResolve(allocator: std.mem.Allocator, jsonl: []const u8) !void { + var texts = std.StringHashMap([]u8).init(allocator); + defer { + var entries = texts.iterator(); + while (entries.next()) |entry| { + allocator.free(entry.key_ptr.*); + allocator.free(entry.value_ptr.*); + } + texts.deinit(); + } + var mapped_records: usize = 0; + var lines = std.mem.splitScalar(u8, jsonl, '\n'); + while (lines.next()) |line| { + if (line.len == 0) continue; + const parsed = try std.json.parseFromSlice(std.json.Value, allocator, line, .{}); + defer parsed.deinit(); + const object = parsed.value.object; + const kind = object.get("record_type").?.string; + const is_span = std.mem.eql(u8, kind, "span"); + const is_block = std.mem.eql(u8, kind, "block"); + const is_chunk = std.mem.eql(u8, kind, "rag_chunk"); + if (!is_span and !is_block and !is_chunk) continue; + const text = object.get(if (is_chunk) "content" else "text").?.string; + const id = object.get(if (is_span) "span_id" else if (is_block) "block_id" else "chunk_id").?.string; + const view = object.get("text_view").?.object; + try std.testing.expectEqualStrings("utf8_byte", view.get("offset_unit").?.string); + try std.testing.expectEqualStrings("half_open", view.get("range_convention").?.string); + var digest: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(text, &digest, .{}); + const hex = std.fmt.bytesToHex(digest, .lower); + try std.testing.expectEqualStrings(&hex, view.get("text_sha256").?.string); + if (!is_span) { + mapped_records += 1; + var cursor: usize = 0; + for (object.get("source_map").?.array.items) |run_value| { + const run = run_value.object; + const start: usize = @intCast(run.get("text_start_byte").?.integer); + const end: usize = @intCast(run.get("text_end_byte").?.integer); + try std.testing.expectEqual(cursor, start); + try std.testing.expect(start < end and end <= text.len); + try std.testing.expect(std.unicode.utf8ValidateSlice(text[start..end])); + const source_id = run.get(if (is_block) "source_span_id" else "source_block_id").?; + if (source_id == .null) { + try std.testing.expect(run.get("source_start_byte").? == .null); + try std.testing.expect(run.get("source_end_byte").? == .null); + for (text[start..end]) |byte| try std.testing.expect(std.ascii.isWhitespace(byte)); + } else { + const source = texts.get(source_id.string) orelse return error.UnresolvedSourceId; + const source_start: usize = @intCast(run.get("source_start_byte").?.integer); + const source_end: usize = @intCast(run.get("source_end_byte").?.integer); + try std.testing.expect(source_start < source_end and source_end <= source.len); + try std.testing.expectEqualStrings(source[source_start..source_end], text[start..end]); + } + cursor = end; + } + try std.testing.expectEqual(text.len, cursor); + if (is_chunk) { + const span_ids = object.get("source_span_ids").?.array.items; + try std.testing.expect(span_ids.len > 0); + try std.testing.expectEqual(span_ids.len, object.get("provenance").?.object.get("span_ids").?.array.items.len); + for (span_ids) |span_id| try std.testing.expect(texts.contains(span_id.string)); + } + } + try std.testing.expect(!texts.contains(id)); + const owned_id = try allocator.dupe(u8, id); + errdefer allocator.free(owned_id); + const owned_text = try allocator.dupe(u8, text); + errdefer allocator.free(owned_text); + try texts.put(owned_id, owned_text); + } + try std.testing.expect(mapped_records > 0); +} + test "parse minimal PDF" { const allocator = std.testing.allocator; @@ -200,6 +273,63 @@ test "adaptive extraction returns native spans routes traces and chunks" { try std.testing.expect(std.mem.indexOf(u8, markdown, "Adaptive Test") != null); } +test "adaptive extraction retains both lines within one MCID in spans and outputs" { + const allocator = std.testing.allocator; + const pdf_data = try testpdf.generateSharedMcidLinesPdf(allocator); + defer allocator.free(pdf_data); + + const doc = try zpdf.Document.openFromMemory(allocator, pdf_data, zpdf.ErrorConfig.permissive()); + defer doc.close(); + var result = try doc.extractAdaptive(allocator, .{ .enable_ocr = false }); + defer result.deinit(); + + try std.testing.expectEqual(@as(usize, 2), result.reconciled.spans.len); + try std.testing.expectEqualStrings("FIRST", result.reconciled.spans[0].span.text); + try std.testing.expectEqualStrings("SECOND", result.reconciled.spans[1].span.text); + for (result.reconciled.spans) |span| { + try std.testing.expectEqual(@as(?i32, 7), span.span.mcid); + try std.testing.expectEqual(@as(u32, 0), span.duplicate_count); + } + const markdown = try result.render(allocator, .markdown); + defer allocator.free(markdown); + try expectIndexBefore(markdown, "FIRST", "SECOND"); + const jsonl = try result.render(allocator, .artifact_jsonl); + defer allocator.free(jsonl); + try expectJsonlLinesParse(allocator, jsonl); + try expectSourceMapsResolve(allocator, jsonl); + try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"text\":\"FIRST\"") != null); + try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"text\":\"SECOND\"") != null); + try std.testing.expect(result.reconciled.chunks.len > 0); + for ([_][]const u8{ "FIRST", "SECOND" }) |needle| { + var present = false; + for (result.reconciled.chunks) |chunk| { + if (std.mem.indexOf(u8, chunk.content, needle) != null) present = true; + } + try std.testing.expect(present); + } +} + +test "streamed source maps resolve document-global span and block ids on later pages" { + const allocator = std.testing.allocator; + var threaded: std.Io.Threaded = .init(allocator, .{}); + defer threaded.deinit(); + runtime.setIo(threaded.io()); + const pdf_data = try testpdf.generateMultiPagePdf(allocator, &.{ "Repeated phrase", "Repeated phrase" }); + defer allocator.free(pdf_data); + const doc = try zpdf.Document.openFromMemory(allocator, pdf_data, zpdf.ErrorConfig.permissive()); + defer doc.close(); + var output: std.ArrayList(u8) = .empty; + defer output.deinit(allocator); + _ = try doc.extractAdaptiveStreaming(allocator, runtime.arrayListWriter(&output, allocator), .{ + .adaptive_options = .{ .enable_ocr = false }, + .schema_options = .{ .source_id = "mapping-fixture" }, + .include_debug_asset_refs = false, + }); + try expectSourceMapsResolve(allocator, output.items); + try std.testing.expect(std.mem.indexOf(u8, output.items, "\"source_span_id\":\"span-1\"") != null); + try std.testing.expect(std.mem.indexOf(u8, output.items, "\"source_block_id\":\"block-1\"") != null); +} + test "adaptive OCR route invokes rasterizer and Tesseract adapter into fresh OCR spans" { if (builtin.target.os.tag == .windows) return error.SkipZigTest; @@ -568,7 +698,7 @@ test "versioned schema renders native document manifest spans blocks chunks and const parsed = try std.json.parseFromSlice(std.json.Value, allocator, json, .{}); defer parsed.deinit(); try std.testing.expectEqualStrings("document_manifest", parsed.value.object.get("schema_name").?.string); - try std.testing.expectEqualStrings("0.13.0", parsed.value.object.get("schema_version").?.string); + try std.testing.expectEqualStrings("0.15.0", parsed.value.object.get("schema_version").?.string); try std.testing.expectEqualStrings("0.4.0-dev", parsed.value.object.get("parser_version").?.string); try std.testing.expectEqualStrings("document_manifest", parsed.value.object.get("record_type").?.string); try std.testing.expectEqualStrings("external-clean-native", parsed.value.object.get("source_id").?.string); @@ -614,7 +744,7 @@ test "versioned schema renders native document manifest spans blocks chunks and const debug_assets = parsed.value.object.get("debug_assets").?.array.items; try std.testing.expect(debug_assets.len > 0); try expectProvenanceObject(debug_assets[0]); - try std.testing.expectEqualStrings("0.13.0", debug_assets[0].object.get("schema_version").?.string); + try std.testing.expectEqualStrings("0.15.0", debug_assets[0].object.get("schema_version").?.string); try std.testing.expect(debug_assets[0].object.get("asset_kind") != null); try std.testing.expect(debug_assets[0].object.get("path") != null); try std.testing.expectEqual(.null, debug_assets[0].object.get("path").?); @@ -625,7 +755,7 @@ test "versioned schema renders native document manifest spans blocks chunks and try std.testing.expectEqualStrings("debug", debug_assets[0].object.get("provenance").?.object.get("source_kind").?.string); try std.testing.expect(std.mem.indexOf(u8, json, "\"schema_name\":\"document_manifest\"") != null); - try std.testing.expect(std.mem.indexOf(u8, json, "\"schema_version\":\"0.13.0\"") != null); + try std.testing.expect(std.mem.indexOf(u8, json, "\"schema_version\":\"0.15.0\"") != null); try std.testing.expect(std.mem.indexOf(u8, json, "\"record_type\":\"span\"") != null); try std.testing.expect(std.mem.indexOf(u8, json, "\"record_type\":\"block\"") != null); try std.testing.expect(std.mem.indexOf(u8, json, "\"record_type\":\"rag_chunk\"") != null); @@ -749,7 +879,7 @@ test "versioned artifact jsonl starts with manifest then typed records" { try std.testing.expect(manifest.value.object.get("output_artifacts") != null); try std.testing.expect(manifest.value.object.get("extraction_counts") != null); try std.testing.expect(manifest.value.object.get("capability_coverage") != null); - try std.testing.expect(std.mem.indexOf(u8, jsonl[first_newline + 1 ..], "\"schema_version\":\"0.13.0\"") != null); + try std.testing.expect(std.mem.indexOf(u8, jsonl[first_newline + 1 ..], "\"schema_version\":\"0.15.0\"") != null); try std.testing.expect(std.mem.indexOf(u8, jsonl[first_newline + 1 ..], "\"record_type\":\"span\"") != null); try std.testing.expect(std.mem.indexOf(u8, jsonl, "\"record_type\":\"route_trace\"") != null); try expectJsonlLinesParse(allocator, jsonl); @@ -781,7 +911,7 @@ test "specialist protocol emits batch request records for routed table formula r const requests = parsed.value.object.get("specialist_requests").?.array.items; try std.testing.expect(requests.len > 0); try expectProvenanceObject(requests[0]); - try std.testing.expectEqualStrings("0.13.0", requests[0].object.get("schema_version").?.string); + try std.testing.expectEqualStrings("0.15.0", requests[0].object.get("schema_version").?.string); try std.testing.expectEqualStrings("specialist_request", requests[0].object.get("record_type").?.string); try std.testing.expectEqualStrings("external-specialist", requests[0].object.get("source_id").?.string); try std.testing.expect(requests[0].object.get("requested_kind") != null); @@ -860,7 +990,7 @@ test "configured formula specialist produces owned linked lifecycle records" { \\request_id=$(printf '%s' "$request" | sed -n 's/.*"request_id":"\([^"]*\)".*/\1/p') \\crop=$(printf '%s' "$request" | sed -n 's/.*"crop_image_path":"\([^"]*\)".*/\1/p') \\test -n "$crop" && test -f "$crop" || exit 7 - \\printf '{"schema_version":"0.13.0","record_type":"specialist_response","request_id":"%s","specialist_id":"fixture-formula","specialist_kind":"formula","status":"completed","formulas":[{"text":"x^2+y^2=z^2","format":"latex","confidence":0.95}]}\n' "$request_id" + \\printf '{"schema_version":"0.15.0","record_type":"specialist_response","request_id":"%s","specialist_id":"fixture-formula","specialist_kind":"formula","status":"completed","formulas":[{"text":"x^2+y^2=z^2","format":"latex","confidence":0.95}]}\n' "$request_id" \\ ); runtime.closeFile(script); @@ -936,7 +1066,7 @@ test "configured formula specialist produces owned linked lifecycle records" { const formulas = response.object.get("formulas").?.array.items; try std.testing.expectEqual(@as(usize, 1), formulas.len); try std.testing.expectEqualStrings("formula", formulas[0].object.get("schema_name").?.string); - try std.testing.expectEqualStrings("0.13.0", formulas[0].object.get("schema_version").?.string); + try std.testing.expectEqualStrings("0.15.0", formulas[0].object.get("schema_version").?.string); try std.testing.expectEqualStrings("formula", formulas[0].object.get("record_type").?.string); try std.testing.expectEqualStrings("source-formula", formulas[0].object.get("source_id").?.string); try expectProvenanceObject(formulas[0]); @@ -1336,7 +1466,7 @@ test "versioned schema exposes financial table cell span metadata" { const tables = parsed.value.object.get("tables").?.array.items; try std.testing.expect(tables.len > 0); try expectProvenanceObject(tables[0]); - try std.testing.expectEqualStrings("0.13.0", tables[0].object.get("schema_version").?.string); + try std.testing.expectEqualStrings("0.15.0", tables[0].object.get("schema_version").?.string); try std.testing.expect(tables[0].object.get("logical_table_id") != null); try std.testing.expect(tables[0].object.get("table_part_index") != null); try std.testing.expect(tables[0].object.get("continued_from_table_id") != null); @@ -1346,7 +1476,7 @@ test "versioned schema exposes financial table cell span metadata" { const cells = tables[0].object.get("rows").?.array.items[0].object.get("cells").?.array.items; try std.testing.expect(cells.len > 0); try std.testing.expectEqualStrings("table_cell", cells[0].object.get("schema_name").?.string); - try std.testing.expectEqualStrings("0.13.0", cells[0].object.get("schema_version").?.string); + try std.testing.expectEqualStrings("0.15.0", cells[0].object.get("schema_version").?.string); try std.testing.expect(cells[0].object.get("cell_id") != null); try std.testing.expectEqualStrings("external-merged-cells", cells[0].object.get("source_id").?.string); try std.testing.expect(cells[0].object.get("raw_text") != null); diff --git a/src/main.zig b/src/main.zig index 8705c21..e4616d5 100644 --- a/src/main.zig +++ b/src/main.zig @@ -108,6 +108,8 @@ fn printUsage() !void { \\ Write visual review sidecar assets for adaptive outputs \\ --emit-specialist-requests FILE \\ Write specialist request JSONL without invoking specialists + \\ --max-output-bytes N Stream serialized-byte cap (positive integer) + \\ --max-page-memory-bytes N Stream live adaptive-page allocation cap \\ --specialist-config FILE \\ Specialist executable config for future local adapters \\ --trace Emit adaptive route trace JSON @@ -1033,6 +1035,8 @@ fn runExtractAdaptive(allocator: std.mem.Allocator, args: []const []const u8) !v var ocr_config = zpdf.OcrConfig{}; var enable_ocr = true; var allow_partial = false; + var max_output_bytes: ?usize = null; + var max_page_memory_bytes: ?usize = null; var i: usize = 0; while (i < args.len) : (i += 1) { @@ -1049,6 +1053,11 @@ fn runExtractAdaptive(allocator: std.mem.Allocator, args: []const []const u8) !v } else if (std.mem.eql(u8, arg, "--source-id")) { i += 1; if (i < args.len) source_id = args[i]; + } else if (std.mem.eql(u8, arg, "--max-output-bytes") or std.mem.eql(u8, arg, "--max-page-memory-bytes")) { + i += 1; + if (i >= args.len) return error.InvalidArguments; + const limit = try parsePositiveResourceLimit(args[i]); + if (std.mem.eql(u8, arg, "--max-output-bytes")) max_output_bytes = limit else max_page_memory_bytes = limit; } else if (std.mem.eql(u8, arg, "--password")) { i += 1; if (i < args.len) password = args[i]; @@ -1131,6 +1140,13 @@ fn runExtractAdaptive(allocator: std.mem.Allocator, args: []const []const u8) !v } } + if (max_output_bytes != null or max_page_memory_bytes != null) { + if (adapter_format != .stream_jsonl or specialist_requests_file != null) { + std.debug.print("Resource limits require stream-jsonl without a specialist-request sidecar.\n", .{}); + return error.InvalidArguments; + } + } + const path = input_file orelse { std.debug.print("Error: extract-adaptive requires --input \n", .{}); return error.InvalidArguments; @@ -1187,12 +1203,15 @@ fn runExtractAdaptive(allocator: std.mem.Allocator, args: []const []const u8) !v if (output_handle) |h| { var file_writer = runtime.fileWriter(h, &write_buf); const writer = &file_writer.interface; + defer writer.flush() catch {}; const summary = zpdf.adapter.extractAdaptive(allocator, doc, writer, .{ .source_id = source_id, .format = adapter_format, .debug_assets_dir = debug_assets_dir, .emit_specialist_requests_path = specialist_requests_file, .specialist_config_path = specialist_config_file, + .max_output_bytes = max_output_bytes, + .max_page_memory_bytes = max_page_memory_bytes, .adaptive_options = .{ .page_start = window.start, .page_end = window.end, @@ -1212,12 +1231,15 @@ fn runExtractAdaptive(allocator: std.mem.Allocator, args: []const []const u8) !v } else { var stdout_writer = runtime.stdoutWriter(&write_buf); const writer = &stdout_writer.interface; + defer writer.flush() catch {}; const summary = zpdf.adapter.extractAdaptive(allocator, doc, writer, .{ .source_id = source_id, .format = adapter_format, .debug_assets_dir = debug_assets_dir, .emit_specialist_requests_path = specialist_requests_file, .specialist_config_path = specialist_config_file, + .max_output_bytes = max_output_bytes, + .max_page_memory_bytes = max_page_memory_bytes, .adaptive_options = .{ .page_start = window.start, .page_end = window.end, @@ -1237,6 +1259,60 @@ fn runExtractAdaptive(allocator: std.mem.Allocator, args: []const []const u8) !v } } +fn parsePositiveResourceLimit(value: []const u8) !usize { + if (value.len == 0) return error.InvalidArguments; + for (value) |byte| if (byte < '0' or byte > '9') return error.InvalidArguments; + const limit = std.fmt.parseInt(usize, value, 10) catch return error.InvalidArguments; + if (limit == 0) return error.InvalidArguments; + return limit; +} + +test "stream resource arguments reject invalid limits and non-stream formats" { + for ([_][]const u8{ "", "0", "-1", "+1", "1.5", "no", "1_000", "184467440737095516160" }) |value| { + try std.testing.expectError(error.InvalidArguments, parsePositiveResourceLimit(value)); + } + try std.testing.expectEqual(@as(usize, 1024), try parsePositiveResourceLimit("1024")); + for ([_][]const u8{ "--max-output-bytes", "--max-page-memory-bytes" }) |flag| { + try std.testing.expectError(error.InvalidArguments, runExtractAdaptive(std.testing.allocator, &.{flag})); + try std.testing.expectError(error.InvalidArguments, runExtractAdaptive(std.testing.allocator, &.{ flag, "0" })); + try std.testing.expectError(error.InvalidArguments, runExtractAdaptive(std.testing.allocator, &.{ "--input", "unused.pdf", flag, "1024" })); + try std.testing.expectError(error.InvalidArguments, runExtractAdaptive(std.testing.allocator, &.{ "--input", "unused.pdf", "--format", "stream-jsonl", "--emit-specialist-requests", "unused.jsonl", flag, "1024" })); + } +} + +test "stream CLI budget failures preserve bounded prefix without completion" { + const allocator = std.testing.allocator; + var threaded: std.Io.Threaded = .init(allocator, .{}); + defer threaded.deinit(); + runtime.setIo(threaded.io()); + const pdf_data = try @import("testpdf.zig").generateCleanNativePdf(allocator); + defer allocator.free(pdf_data); + var input_buf: [96]u8 = undefined; + const input_path = try std.fmt.bufPrint(&input_buf, "pdf-parser-cli-budget-{x}.pdf", .{std.testing.random_seed}); + defer runtime.deleteFileCwd(input_path); + const file = try runtime.createFileCwd(input_path); + try runtime.writeAllFile(file, pdf_data); + runtime.closeFile(file); + var output_buf: [96]u8 = undefined; + const output_path = try std.fmt.bufPrint(&output_buf, "pdf-parser-cli-budget-{x}.jsonl", .{std.testing.random_seed}); + defer runtime.deleteFileCwd(output_path); + for ([_]struct { flag: []const u8, value: []const u8, failure: anyerror }{ + .{ .flag = "--max-output-bytes", .value = "1600", .failure = error.OutputLimitExceeded }, + .{ .flag = "--max-page-memory-bytes", .value = "1", .failure = error.MemoryBudgetExceeded }, + }) |case| { + try std.testing.expectError(case.failure, runExtractAdaptive(allocator, &.{ "--input", input_path, "--output", output_path, "--format", "stream-jsonl", "--no-ocr", case.flag, case.value })); + const output = try runtime.readFileAllocAlignedCwd(allocator, output_path, .fromByteUnits(1)); + defer allocator.free(output); + try std.testing.expect(output.len > 0); + if (std.mem.eql(u8, case.flag, "--max-output-bytes")) try std.testing.expect(output.len <= 1600); + try std.testing.expect(std.mem.indexOf(u8, output, "document_finished") == null); + } + try runExtractAdaptive(allocator, &.{ "--input", input_path, "--output", output_path, "--format", "stream-jsonl", "--no-ocr", "--max-output-bytes", "1000000", "--max-page-memory-bytes", "1000000" }); + const output = try runtime.readFileAllocAlignedCwd(allocator, output_path, .fromByteUnits(1)); + defer allocator.free(output); + try std.testing.expect(std.mem.indexOf(u8, output, "document_finished") != null); +} + fn runInspect(allocator: std.mem.Allocator, args: []const []const u8) !void { if (args.len == 0) { std.debug.print("Usage: pdf-parser inspect complexity|extraction|structure [--format json] \n", .{}); @@ -1964,7 +2040,7 @@ test "adaptive CLI emits specialist request JSONL sidecars" { \\request_id=$(printf '%s' "$request" | sed -n 's/.*"request_id":"\([^"]*\)".*/\1/p') \\crop=$(printf '%s' "$request" | sed -n 's/.*"crop_image_path":"\([^"]*\)".*/\1/p') \\test -n "$crop" && test -f "$crop" || exit 7 - \\printf '{"schema_version":"0.13.0","record_type":"specialist_response","request_id":"%s","specialist_id":"cli-formula","specialist_kind":"formula","status":"completed","formulas":[{"text":"x^2+y^2=z^2","format":"latex","confidence":0.95}]}\n' "$request_id" + \\printf '{"schema_version":"0.15.0","record_type":"specialist_response","request_id":"%s","specialist_id":"cli-formula","specialist_kind":"formula","status":"completed","formulas":[{"text":"x^2+y^2=z^2","format":"latex","confidence":0.95}]}\n' "$request_id" \\ ); runtime.closeFile(formula_file); @@ -2038,7 +2114,7 @@ test "adaptive CLI emits specialist request JSONL sidecars" { const requests = try runtime.readFileAllocAlignedCwd(allocator, requests_path, .fromByteUnits(1)); defer allocator.free(requests); try std.testing.expect(std.mem.indexOf(u8, requests, "\"record_type\":\"specialist_request\"") != null); - try std.testing.expect(std.mem.indexOf(u8, requests, "\"schema_version\":\"0.13.0\"") != null); + try std.testing.expect(std.mem.indexOf(u8, requests, "\"schema_version\":\"0.15.0\"") != null); try std.testing.expect(std.mem.indexOf(u8, requests, "\"source_id\":\"external-specialist-cli\"") != null); try std.testing.expect(std.mem.indexOf(u8, requests, "\"requested_kind\":\"") != null); try std.testing.expect(std.mem.indexOf(u8, requests, "\"requested_outputs\"") != null); diff --git a/src/reconcile.zig b/src/reconcile.zig index 2376bc2..e532e3a 100644 --- a/src/reconcile.zig +++ b/src/reconcile.zig @@ -37,6 +37,9 @@ pub const ReconciledSpan = struct { duplicate_count: u32 = 0, chosen_source: SourceKind, confidence: f32, + /// Half-open UTF-8 byte range in the owning reconciled block's exact text. + block_text_start: usize = 0, + block_text_end: usize = 0, }; pub const ReconciledBlock = struct { @@ -446,7 +449,7 @@ pub fn renderDebugSvg(allocator: std.mem.Allocator, doc: *const ReconciledDocume fn buildBlocks( allocator: std.mem.Allocator, - spans: []const ReconciledSpan, + spans: []ReconciledSpan, options: ReconcileOptions, ) ![]ReconciledBlock { var blocks: std.ArrayList(ReconciledBlock) = .empty; @@ -468,8 +471,8 @@ fn buildBlocks( var confidence_sum: f32 = 0; var confidence_count: u32 = 0; - for (spans, 0..) |span, index| { - if (index > start_index and shouldStartBlock(spans[index - 1], span, options)) { + for (spans, 0..) |*span, index| { + if (index > start_index and shouldStartBlock(spans[index - 1], span.*, options)) { try finishBlock( allocator, &blocks, @@ -505,7 +508,9 @@ fn buildBlocks( try text.append(allocator, '\n'); } } + span.block_text_start = text.items.len; try text.appendSlice(allocator, span.span.text); + span.block_text_end = text.items.len; confidence_sum += span.confidence; confidence_count += 1; } @@ -541,6 +546,7 @@ fn finishBlock( confidence_count: u32, ) !void { const owned_text = try text.toOwnedSlice(allocator); + errdefer allocator.free(owned_text); text.* = .empty; if (owned_text.len == 0) { allocator.free(owned_text); @@ -660,10 +666,13 @@ fn finishChunk( confidence_count: u32, ) !void { const content = try text.toOwnedSlice(allocator); + errdefer allocator.free(content); text.* = .empty; + const owned_source_id = try allocator.dupe(u8, source_id); + errdefer allocator.free(owned_source_id); try chunks.append(allocator, .{ - .source_id = try allocator.dupe(u8, source_id), + .source_id = owned_source_id, .chunk_index = @intCast(chunks.items.len), .block_start = @intCast(block_start), .block_count = @intCast(block_count), @@ -746,9 +755,8 @@ fn findDuplicate(spans: []const ReconciledSpan, duplicate_index: *const Duplicat const same_page_indices = duplicate_index.indicesForPage(candidate.span.page_index) orelse return null; for (same_page_indices) |index| { const existing = spans[index]; - if (existing.span.mcid != null and candidate.span.mcid != null and existing.span.mcid.? == candidate.span.mcid.?) { - return index; - } + // MCIDs identify marked-content groups, which may contain multiple + // distinct spans. Duplicate identity still requires text and placement. if (!std.mem.eql(u8, normalizedText(existing.span.text), normalizedText(candidate.span.text))) continue; if (intersectionOverMinArea(existing.span.bbox, candidate.span.bbox) >= overlap_threshold) return index; } @@ -1142,6 +1150,95 @@ test "reconciler duplicate index keeps page identity" { try std.testing.expect(untouched_page1); } +test "reconciler preserves distinct text sharing an MCID even at the same placement" { + var first = testSpan("FIRST", .native_pdf, 10, 700, 50, 712); + var second = testSpan("SECOND", .native_pdf, 10, 700, 50, 712); + first.mcid = 7; + second.mcid = 7; + + var doc = try reconcile(std.testing.allocator, &.{ + .{ .source = .native_pdf, .spans = &.{ first, second } }, + }, .{ .preserve_input_order = true }); + defer doc.deinit(); + + try std.testing.expectEqual(@as(usize, 2), doc.spans.len); + try std.testing.expectEqualStrings("FIRST", doc.spans[0].span.text); + try std.testing.expectEqualStrings("SECOND", doc.spans[1].span.text); +} + +test "reconciler preserves repeated text sharing an MCID at distinct placements" { + var first = testSpan("Total", .native_pdf, 10, 700, 50, 712); + var second = testSpan("Total", .native_pdf, 10, 650, 50, 662); + first.mcid = 7; + second.mcid = 7; + + var doc = try reconcile(std.testing.allocator, &.{ + .{ .source = .native_pdf, .spans = &.{ first, second } }, + }, .{}); + defer doc.deinit(); + + try std.testing.expectEqual(@as(usize, 2), doc.spans.len); + for (doc.spans) |span| { + try std.testing.expectEqualStrings("Total", span.span.text); + try std.testing.expectEqual(@as(u32, 0), span.duplicate_count); + } +} + +test "reconciler still merges equivalent overlapping MCID spans with provenance" { + var native = testSpan("Total", .native_pdf, 10, 700, 50, 712); + var ocr = testSpan(" Total ", .fresh_ocr, 10.5, 700, 50.5, 712); + native.mcid = 7; + ocr.mcid = 7; + + var doc = try reconcile(std.testing.allocator, &.{ + .{ .source = .native_pdf, .spans = &.{native} }, + .{ .source = .fresh_ocr, .spans = &.{ocr} }, + }, .{}); + defer doc.deinit(); + + try std.testing.expectEqual(@as(usize, 1), doc.spans.len); + try std.testing.expectEqualStrings("Total", doc.spans[0].span.text); + try std.testing.expectEqual(@as(u8, 2), doc.spans[0].source_count); + try std.testing.expect(hasSource(doc.spans[0].source_mask, .native_pdf)); + try std.testing.expect(hasSource(doc.spans[0].source_mask, .fresh_ocr)); + try std.testing.expectEqual(@as(u32, 1), doc.spans[0].duplicate_count); +} + +test "reconciler records byte origins during construction of repeated Unicode spans" { + var doc = try reconcile(std.testing.allocator, &.{ + .{ .spans = &.{ + testSpan("AπŸ˜€", .native_pdf, 10, 700, 40, 712), + testSpan("AπŸ˜€", .native_pdf, 45, 700, 75, 712), + } }, + }, .{}); + defer doc.deinit(); + + try std.testing.expectEqual(@as(usize, 1), doc.blocks.len); + try std.testing.expectEqualStrings("AπŸ˜€ AπŸ˜€", doc.blocks[0].text); + try std.testing.expectEqual(@as(usize, 0), doc.spans[0].block_text_start); + try std.testing.expectEqual(@as(usize, 5), doc.spans[0].block_text_end); + try std.testing.expectEqual(@as(usize, 6), doc.spans[1].block_text_start); + try std.testing.expectEqual(@as(usize, 11), doc.spans[1].block_text_end); + for (doc.spans) |span| { + try std.testing.expectEqualStrings(span.span.text, doc.blocks[0].text[span.block_text_start..span.block_text_end]); + } +} + +fn reconcileAllocationProbe(allocator: std.mem.Allocator) !void { + var deterministic = std.testing.FailingAllocator.init(allocator, .{ .resize_fail_index = 0 }); + var doc = try reconcile(deterministic.allocator(), &.{ + .{ .spans = &.{ + testSpan("FIRST", .native_pdf, 10, 700, 50, 712), + testSpan("SECOND", .native_pdf, 10, 650, 60, 662), + } }, + }, .{ .max_chunk_bytes = 8 }); + defer doc.deinit(); +} + +test "reconciler construction cleans up every allocation failure" { + try std.testing.checkAllAllocationFailures(std.testing.allocator, reconcileAllocationProbe, .{}); +} + test "reconciler emits specialist blocks and rag jsonl chunks" { const table = [_]TextSpan{ testSpan("Year Revenue\n2025 42", .table_model, 20, 650, 180, 690), diff --git a/src/resource_limits.zig b/src/resource_limits.zig new file mode 100644 index 0000000..ed92f6c --- /dev/null +++ b/src/resource_limits.zig @@ -0,0 +1,189 @@ +//! Explicit budgets for serialized output and live allocator-owned bytes. +const std = @import("std"); + +pub fn LimitedWriter(comptime Writer: type) type { + return struct { + writer: Writer, + max_bytes: ?usize, + written: usize = 0, + + pub fn ensureCapacity(self: *const @This(), count: usize) !void { + if (self.max_bytes) |limit| { + if (count > limit - self.written) return error.OutputLimitExceeded; + } + if (count > std.math.maxInt(usize) - self.written) return error.OutputLimitExceeded; + } + + pub fn writeAll(self: *@This(), bytes: []const u8) !void { + try self.ensureCapacity(bytes.len); + try self.writer.writeAll(bytes); + self.written += bytes.len; + } + + pub fn writeByte(self: *@This(), byte: u8) !void { + try self.ensureCapacity(1); + try self.writer.writeByte(byte); + self.written += 1; + } + + pub fn print(self: *@This(), comptime fmt: []const u8, args: anytype) !void { + const count = std.fmt.count(fmt, args); + try self.ensureCapacity(count); + try self.writer.print(fmt, args); + self.written += count; + } + }; +} + +pub const CountingWriter = struct { + written: usize = 0, + + pub fn writeAll(self: *@This(), bytes: []const u8) !void { + self.written = std.math.add(usize, self.written, bytes.len) catch return error.OutputLimitExceeded; + } + + pub fn writeByte(self: *@This(), _: u8) !void { + self.written = std.math.add(usize, self.written, 1) catch return error.OutputLimitExceeded; + } + + pub fn print(self: *@This(), comptime fmt: []const u8, args: anytype) !void { + self.written = std.math.add(usize, self.written, std.fmt.count(fmt, args)) catch return error.OutputLimitExceeded; + } +}; + +/// Must remain at a stable address until every borrowed allocation is freed. +/// Counts requested live bytes, including simultaneous old/new realloc buffers; +/// excludes parent allocator metadata, document caches, and subprocess memory. +pub const BudgetAllocator = struct { + parent: std.mem.Allocator, + max_bytes: usize, + live_bytes: std.atomic.Value(usize) = .init(0), + limit_hit: std.atomic.Value(bool) = .init(false), + + pub fn allocator(self: *BudgetAllocator) std.mem.Allocator { + return .{ .ptr = self, .vtable = &.{ .alloc = alloc, .resize = resize, .remap = remap, .free = free } }; + } + + fn reserve(self: *BudgetAllocator, count: usize) bool { + var live = self.live_bytes.load(.monotonic); + while (true) { + if (count > self.max_bytes - live) { + self.limit_hit.store(true, .monotonic); + return false; + } + live = self.live_bytes.cmpxchgWeak(live, live + count, .monotonic, .monotonic) orelse return true; + } + } + + fn release(self: *BudgetAllocator, count: usize) void { + const previous = self.live_bytes.fetchSub(count, .monotonic); + std.debug.assert(previous >= count); + } + + fn alloc(ctx: *anyopaque, len: usize, alignment: std.mem.Alignment, ra: usize) ?[*]u8 { + const self: *BudgetAllocator = @ptrCast(@alignCast(ctx)); + if (!self.reserve(len)) return null; + return self.parent.rawAlloc(len, alignment, ra) orelse { + self.release(len); + return null; + }; + } + + fn resize(ctx: *anyopaque, memory: []u8, alignment: std.mem.Alignment, new_len: usize, ra: usize) bool { + const self: *BudgetAllocator = @ptrCast(@alignCast(ctx)); + const growth = new_len -| memory.len; + if (!self.reserve(growth)) return false; + if (!self.parent.rawResize(memory, alignment, new_len, ra)) { + self.release(growth); + return false; + } + self.release(memory.len -| new_len); + return true; + } + + fn remap(ctx: *anyopaque, memory: []u8, alignment: std.mem.Alignment, new_len: usize, ra: usize) ?[*]u8 { + const self: *BudgetAllocator = @ptrCast(@alignCast(ctx)); + const growth = new_len -| memory.len; + if (!self.reserve(growth)) return null; + const ptr = self.parent.rawRemap(memory, alignment, new_len, ra) orelse { + self.release(growth); + return null; + }; + self.release(memory.len -| new_len); + return ptr; + } + + fn free(ctx: *anyopaque, memory: []u8, alignment: std.mem.Alignment, ra: usize) void { + const self: *BudgetAllocator = @ptrCast(@alignCast(ctx)); + self.parent.rawFree(memory, alignment, ra); + self.release(memory.len); + } +}; + +test "limited writer rejects every operation before overrun" { + const runtime = @import("runtime.zig"); + var bytes: std.ArrayList(u8) = .empty; + defer bytes.deinit(std.testing.allocator); + var writer = LimitedWriter(runtime.ArrayListWriter){ .writer = runtime.arrayListWriter(&bytes, std.testing.allocator), .max_bytes = 5 }; + try writer.writeAll("ab"); + try writer.print("{d}", .{@as(u8, 12)}); + try std.testing.expectError(error.OutputLimitExceeded, writer.writeAll("cd")); + try std.testing.expectError(error.OutputLimitExceeded, writer.print("{d}", .{@as(u8, 12)})); + try writer.writeByte('!'); + try std.testing.expectError(error.OutputLimitExceeded, writer.writeByte('!')); + try std.testing.expectEqualStrings("ab12!", bytes.items); +} + +test "budget allocator accounts allocation resize remap and free" { + var storage: [64]u8 = undefined; + var fixed = std.heap.FixedBufferAllocator.init(&storage); + var budget = BudgetAllocator{ .parent = fixed.allocator(), .max_bytes = 16 }; + const allocator = budget.allocator(); + var bytes = try allocator.alloc(u8, 8); + try std.testing.expectEqual(@as(usize, 8), budget.live_bytes.load(.monotonic)); + try std.testing.expect(allocator.resize(bytes, 12)); + bytes = bytes.ptr[0..12]; + try std.testing.expectEqual(@as(usize, 12), budget.live_bytes.load(.monotonic)); + try std.testing.expect(!allocator.resize(bytes, 17)); + try std.testing.expect(allocator.remap(bytes, 17) == null); + try std.testing.expectEqual(@as(usize, 12), budget.live_bytes.load(.monotonic)); + bytes = allocator.remap(bytes, 4).?; + try std.testing.expectEqual(@as(usize, 4), budget.live_bytes.load(.monotonic)); + allocator.free(bytes); + try std.testing.expectEqual(@as(usize, 0), budget.live_bytes.load(.monotonic)); + try std.testing.expect(budget.limit_hit.load(.monotonic)); +} + +test "budget allocator rolls back underlying allocation resize and remap failures" { + var failing = std.testing.FailingAllocator.init(std.testing.allocator, .{ .resize_fail_index = 0 }); + var budget = BudgetAllocator{ .parent = failing.allocator(), .max_bytes = 64 }; + const allocator = budget.allocator(); + const bytes = try allocator.alloc(u8, 8); + failing.fail_index = failing.alloc_index; + try std.testing.expectError(error.OutOfMemory, allocator.alloc(u8, 8)); + try std.testing.expect(!allocator.resize(bytes, 16)); + try std.testing.expect(allocator.remap(bytes, 16) == null); + try std.testing.expectEqual(@as(usize, 8), budget.live_bytes.load(.monotonic)); + try std.testing.expect(!budget.limit_hit.load(.monotonic)); + allocator.free(bytes); + try std.testing.expectEqual(@as(usize, 0), budget.live_bytes.load(.monotonic)); +} + +test "budget allocator concurrent reservations cannot overcommit" { + var budget = BudgetAllocator{ .parent = std.testing.allocator, .max_bytes = 8 }; + var attempted = std.atomic.Value(usize).init(0); + var admitted = std.atomic.Value(usize).init(0); + const worker = try std.Thread.spawn(.{}, reservationWorker, .{ &budget, &attempted, &admitted }); + reservationWorker(&budget, &attempted, &admitted); + worker.join(); + try std.testing.expectEqual(@as(usize, 1), admitted.load(.monotonic)); + try std.testing.expectEqual(@as(usize, 0), budget.live_bytes.load(.monotonic)); +} + +fn reservationWorker(budget: *BudgetAllocator, attempted: *std.atomic.Value(usize), admitted: *std.atomic.Value(usize)) void { + const accepted = budget.reserve(8); + if (accepted) _ = admitted.fetchAdd(1, .monotonic); + _ = attempted.fetchAdd(1, .monotonic); + while (attempted.load(.monotonic) < 2) std.atomic.spinLoopHint(); + if (accepted) budget.release(8); +} diff --git a/src/schema.zig b/src/schema.zig index 7ee48e1..52897c1 100644 --- a/src/schema.zig +++ b/src/schema.zig @@ -10,6 +10,7 @@ const layout = @import("layout.zig"); const runtime = @import("runtime.zig"); const specialist_protocol = @import("specialist_protocol.zig"); const visual_assets = @import("visual_assets.zig"); +const coverage = @import("coverage.zig"); pub const schema_version = specialist_protocol.schema_version; pub const parser_version = "0.4.0-dev"; @@ -347,12 +348,12 @@ pub fn renderComplexityJson(allocator: std.mem.Allocator, result: anytype, docum try writer.writeAll("\",\"pages\":["); for (result.page_routes, 0..) |route, index| { if (index > 0) try writer.writeByte(','); - try writePageRouteRecord(writer, document_id, null, null, route, @intCast(index), null); + try writePageRouteRecord(writer, document_id, null, null, route, @intCast(index), null, coverage.assessPage(result, route.page_index)); } try writer.writeAll("],\"regions\":["); for (result.region_routes, 0..) |route, index| { if (index > 0) try writer.writeByte(','); - try writeRegionRouteRecord(writer, document_id, null, null, route, @intCast(result.page_routes.len + index), null); + try writeRegionRouteRecord(writer, document_id, null, null, route, @intCast(result.page_routes.len + index), null, coverage.assessRegion(result, route)); } try writer.writeAll("]}"); return output.toOwnedSlice(allocator); @@ -376,12 +377,12 @@ pub fn renderTraceJsonWithOptions(allocator: std.mem.Allocator, result: anytype, try writer.writeAll(",\"pages\":["); for (result.page_routes, 0..) |route, index| { if (index > 0) try writer.writeByte(','); - try writePageRouteRecord(writer, options.document_id, options.source_id, options.input_sha256, route, @intCast(index), null); + try writePageRouteRecord(writer, options.document_id, options.source_id, options.input_sha256, route, @intCast(index), null, coverage.assessPage(result, route.page_index)); } try writer.writeAll("],\"regions\":["); for (result.region_routes, 0..) |route, index| { if (index > 0) try writer.writeByte(','); - try writeRegionRouteRecord(writer, options.document_id, options.source_id, options.input_sha256, route, @intCast(result.page_routes.len + index), null); + try writeRegionRouteRecord(writer, options.document_id, options.source_id, options.input_sha256, route, @intCast(result.page_routes.len + index), null, coverage.assessRegion(result, route)); } try writer.writeAll("],\"trace\":["); for (result.trace_records, 0..) |record, index| { @@ -412,6 +413,7 @@ fn writeDocumentManifestOpen(writer: anytype, result: anytype, options: RenderOp try writeJsonEscaped(writer, options.document_id); try writer.writeAll("\",\"source_id\":"); try writeOptionalString(writer, options.source_id); + try writeTextContract(writer); try writer.writeAll(",\"parser_version\":\""); try writer.writeAll(parser_version); try writer.writeAll("\",\"page_count\":"); @@ -955,6 +957,7 @@ pub fn writeStreamManifestRecord(writer: anytype, options: RenderOptions, event_ }); try writeDocumentId(writer, options.document_id); try writeSourceId(writer, options.source_id); + try writeTextContract(writer); try writer.writeAll(",\"parser_version\":\""); try writer.writeAll(parser_version); try writer.writeAll("\",\"streaming\":true,\"page_count\":"); @@ -1171,10 +1174,16 @@ fn writeSpanRecord(writer: anytype, result: anytype, document_id: []const u8, so try writeFontJson(writer, span.span.font, span.span.font_size); try writer.writeAll(",\"block_id\":"); try writeOptionalU32(writer, span.span.block_id); + try writer.writeAll(",\"reconciled_block_id\":"); + const owning_block_id = owningBlockId(result, span_index - offsets.span_base, offsets.block_base); + if (owning_block_id) |block_id| { + try writer.print("\"block-{d}\"", .{block_id}); + } else try writer.writeAll("null"); try writer.writeAll(",\"line_id\":"); try writeOptionalU32(writer, span.span.line_id); try writer.writeAll(",\"mcid\":"); try writeOptionalI32(writer, span.span.mcid); + try writeTextView(writer, "span", span_index, "reconciled_span", span.span.text); try writeProvenancePrefix(writer, .{ .document_id = document_id, .source_id = source_id, @@ -1188,7 +1197,7 @@ fn writeSpanRecord(writer: anytype, result: anytype, document_id: []const u8, so }); try writeSingleId(writer, "span", span_index); try writer.writeAll(",\"block_ids\":"); - if (span.span.block_id) |block_id| { + if (owning_block_id) |block_id| { try writeSingleId(writer, "block", block_id); } else { try writer.writeAll("[]"); @@ -1199,6 +1208,13 @@ fn writeSpanRecord(writer: anytype, result: anytype, document_id: []const u8, so try writer.writeByte('}'); } +fn owningBlockId(result: anytype, local_span_index: u32, block_base: u32) ?u32 { + for (result.reconciled.blocks) |block| { + if (local_span_index >= block.span_start and local_span_index - block.span_start < block.span_count) return block_base + block.id; + } + return null; +} + fn writeBlockRecord(writer: anytype, result: anytype, document_id: []const u8, source_id: ?[]const u8, input_sha256: ?[]const u8, block: anytype, block_base: u32, offsets: RecordOffsets, stream_meta: ?StreamRecordMeta) !void { try writeRecordHeader(writer, "block"); if (stream_meta) |meta| try writeStreamMeta(writer, meta); @@ -1224,6 +1240,8 @@ fn writeBlockRecord(writer: anytype, result: anytype, document_id: []const u8, s }); try writer.writeAll(",\"candidate_kind\":"); try writeCandidateKind(writer, block.kind); + try writeTextView(writer, "block", block_base + block.id, "reconciled_block", block.text); + try writeBlockSourceMap(writer, result, block, offsets); try writeProvenancePrefix(writer, .{ .document_id = document_id, .source_id = source_id, @@ -1235,7 +1253,7 @@ fn writeBlockRecord(writer: anytype, result: anytype, document_id: []const u8, s .source_kind = provenanceSourceKindFromMask(block.source_mask), .confidence = block.confidence, }); - try writeIdRange(writer, "span", block.span_start, block.span_count); + try writeIdRange(writer, "span", offsets.span_base + block.span_start, block.span_count); try writer.writeAll(",\"block_ids\":"); try writeSingleId(writer, "block", block_base + block.id); try writer.writeAll(",\"chunk_ids\":[]"); @@ -1265,6 +1283,8 @@ fn writeTableRecord(allocator: std.mem.Allocator, writer: anytype, document_id: try writeBBoxJson(writer, table.bounds.bbox); try writer.writeAll(",\"source_span_ids\":"); try writeSpanIdsInBBox(writer, result, table.page_index, table.bounds.bbox, offsets); + try writer.writeAll(",\"reconciled_block_ids\":"); + try writeOwningBlockIds(writer, result, table.page_index, table.bounds.bbox, offsets); try writer.writeAll(",\"rows\":["); for (table.rows, 0..) |row, row_index| { if (row_index > 0) try writer.writeByte(','); @@ -1344,6 +1364,8 @@ fn writeCellJson( try writeBBoxJson(writer, cell.bounds.bbox); try writer.writeAll(",\"source_span_ids\":"); try writeSpanIdArray(writer, cell_span_ids.items); + try writer.writeAll(",\"reconciled_block_ids\":"); + try writeOwningBlockIds(writer, result, cell.bounds.page_index, cell.bounds.bbox, offsets); try writeProvenancePrefix(writer, .{ .document_id = document_id, .source_id = source_id, @@ -1423,7 +1445,7 @@ fn writeRouteTraceRecords(writer: anytype, document_id: []const u8, source_id: ? if (jsonl) try writer.writeByte('\n') else try writer.writeByte(','); } const stream_meta = nextRouteStreamMeta(stream_event_index, route.page_index); - try writePageRouteRecord(writer, document_id, source_id, input_sha256, route, offsets.route_base + @as(u32, @intCast(local_index)), stream_meta); + try writePageRouteRecord(writer, document_id, source_id, input_sha256, route, offsets.route_base + @as(u32, @intCast(local_index)), stream_meta, coverage.assessPage(result, route.page_index)); wrote = true; } const region_base = offsets.route_base + @as(u32, @intCast(result.page_routes.len)); @@ -1432,7 +1454,7 @@ fn writeRouteTraceRecords(writer: anytype, document_id: []const u8, source_id: ? if (jsonl) try writer.writeByte('\n') else try writer.writeByte(','); } const stream_meta = nextRouteStreamMeta(stream_event_index, route.page_index); - try writeRegionRouteRecord(writer, document_id, source_id, input_sha256, route, region_base + @as(u32, @intCast(local_index)), stream_meta); + try writeRegionRouteRecord(writer, document_id, source_id, input_sha256, route, region_base + @as(u32, @intCast(local_index)), stream_meta, coverage.assessRegion(result, route)); wrote = true; } const trace_base = region_base + @as(u32, @intCast(result.region_routes.len)); @@ -1447,7 +1469,7 @@ fn writeRouteTraceRecords(writer: anytype, document_id: []const u8, source_id: ? if (jsonl and wrote) try writer.writeByte('\n'); } -fn writePageRouteRecord(writer: anytype, document_id: []const u8, source_id: ?[]const u8, input_sha256: ?[]const u8, route: anytype, route_index: u32, stream_meta: ?StreamRecordMeta) !void { +fn writePageRouteRecord(writer: anytype, document_id: []const u8, source_id: ?[]const u8, input_sha256: ?[]const u8, route: anytype, route_index: u32, stream_meta: ?StreamRecordMeta, assessment: coverage.Assessment) !void { try writeRecordHeader(writer, "route_trace"); if (stream_meta) |meta| try writeStreamMeta(writer, meta); try writeDocumentId(writer, document_id); @@ -1459,6 +1481,7 @@ fn writePageRouteRecord(writer: anytype, document_id: []const u8, source_id: ?[] } try writer.print(",\"trace_kind\":\"page_route\",\"page_index\":{},\"region_index\":null,\"stage\":\"route_decision\"", .{route.page_index}); try writeRouteFields(writer, route.route, route.reason_mask, route.span_count, 0, route.image_count, route.char_count, route.signals, route.bbox); + try writeCoverage(writer, assessment); try specialist_protocol.writeRouteTraceSpecialistFieldsWithReason(writer, route.route, route.reason_mask, route.page_index, null); try writeProvenancePrefix(writer, .{ .document_id = document_id, @@ -1482,7 +1505,7 @@ fn writePageRouteRecord(writer: anytype, document_id: []const u8, source_id: ?[] try writer.writeByte('}'); } -fn writeRegionRouteRecord(writer: anytype, document_id: []const u8, source_id: ?[]const u8, input_sha256: ?[]const u8, route: anytype, route_index: u32, stream_meta: ?StreamRecordMeta) !void { +fn writeRegionRouteRecord(writer: anytype, document_id: []const u8, source_id: ?[]const u8, input_sha256: ?[]const u8, route: anytype, route_index: u32, stream_meta: ?StreamRecordMeta, assessment: coverage.Assessment) !void { try writeRecordHeader(writer, "route_trace"); if (stream_meta) |meta| try writeStreamMeta(writer, meta); try writeDocumentId(writer, document_id); @@ -1506,6 +1529,7 @@ fn writeRegionRouteRecord(writer: anytype, document_id: []const u8, source_id: ? try writeRouteFields(writer, route.route, route.reason_mask, route.span_count, 0, route.image_count, route.char_count, route.signals, route.bbox); try writer.writeAll(",\"specialist\":"); try writeSpecialistJson(writer, route); + try writeCoverage(writer, assessment); try specialist_protocol.writeRouteTraceSpecialistFieldsWithReason(writer, route.route, route.reason_mask, route.page_index, route.region_index); try writeProvenancePrefix(writer, .{ .document_id = document_id, @@ -1608,9 +1632,9 @@ fn writeRagChunkRecord(writer: anytype, result: anytype, document_id: []const u8 try writeRecordHeader(writer, "rag_chunk"); if (stream_meta) |meta| try writeStreamMeta(writer, meta); try writeDocumentId(writer, document_id); - try writer.print(",\"chunk_id\":\"chunk-{d}\",\"chunk_index\":{},\"source_id\":\"", .{ chunk_base + chunk.chunk_index, chunk_base + chunk.chunk_index }); - try writeJsonEscaped(writer, source_id orelse chunk.source_id); - try writer.writeAll("\",\"content\":\""); + try writer.print(",\"chunk_id\":\"chunk-{d}\",\"chunk_index\":{}", .{ chunk_base + chunk.chunk_index, chunk_base + chunk.chunk_index }); + try writeSourceId(writer, source_id); + try writer.writeAll(",\"content\":\""); try writeJsonEscaped(writer, chunk.content); try writer.print("\",\"block_start\":{},\"block_count\":{},\"page_start\":{},\"page_end\":{}", .{ chunk.block_start, @@ -1627,7 +1651,10 @@ fn writeRagChunkRecord(writer: anytype, result: anytype, document_id: []const u8 if (offset > 0) try writer.writeByte(','); try writer.print("\"block-{d}\"", .{block_base + chunk.block_start + @as(u32, @intCast(offset))}); } - try writer.writeAll("],\"source_span_ids\":[]"); + try writer.writeAll("],\"source_span_ids\":"); + try writeChunkSpanIds(writer, result, chunk, offsets); + try writeTextView(writer, "chunk", chunk_base + chunk.chunk_index, "rag_chunk", chunk.content); + try writeChunkSourceMap(writer, result, chunk, block_base); try writeProvenancePrefix(writer, .{ .document_id = document_id, .source_id = source_id, @@ -1639,7 +1666,7 @@ fn writeRagChunkRecord(writer: anytype, result: anytype, document_id: []const u8 .source_kind = provenanceSourceKindFromMask(chunk.source_mask), .confidence = chunk.confidence, }); - try writer.writeAll("[]"); + try writeChunkSpanIds(writer, result, chunk, offsets); try writer.writeAll(",\"block_ids\":"); try writeIdRange(writer, "block", block_base + chunk.block_start, chunk.block_count); try writer.writeAll(",\"chunk_ids\":"); @@ -1649,6 +1676,76 @@ fn writeRagChunkRecord(writer: anytype, result: anytype, document_id: []const u8 try writer.writeByte('}'); } +fn writeTextView(writer: anytype, prefix: []const u8, index: u32, kind: []const u8, text: []const u8) !void { + var digest: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(text, &digest, .{}); + const hex = std.fmt.bytesToHex(digest, .lower); + try writer.print(",\"text_view\":{{\"text_view_id\":\"{s}-{d}:text\",\"kind\":\"{s}\",\"offset_unit\":\"utf8_byte\",\"range_convention\":\"half_open\",\"normalization\":\"none\",\"text_sha256\":\"{s}\"}}", .{ prefix, index, kind, hex }); +} + +fn writeTextContract(writer: anytype) !void { + try writer.writeAll(",\"text_contract\":{\"offset_unit\":\"utf8_byte\",\"range_convention\":\"half_open\",\"page_index_base\":0,\"geometry\":{\"coordinate_space\":\"pdf_user_space\",\"unit\":\"point\",\"highlight_precision\":\"span_bbox\",\"display_transform\":null}}"); +} + +fn writeSourceMapRun(writer: anytype, start: usize, end: usize, source_kind: []const u8, source_index: ?u32, source_length: usize) !void { + try writer.print("{{\"text_start_byte\":{},\"text_end_byte\":{},\"source_{s}_id\":", .{ start, end, source_kind }); + if (source_index) |index| { + try writer.print("\"{s}-{d}\",\"source_start_byte\":0,\"source_end_byte\":{}}}", .{ source_kind, index, source_length }); + } else { + try writer.writeAll("null,\"source_start_byte\":null,\"source_end_byte\":null}"); + } +} + +fn writeBlockSourceMap(writer: anytype, result: anytype, block: anytype, offsets: RecordOffsets) !void { + try writer.writeAll(",\"source_map\":["); + var cursor: usize = 0; + var first = true; + const start: usize = block.span_start; + for (result.reconciled.spans[start .. start + block.span_count], start..) |span, index| { + if (span.block_text_start > cursor) { + if (!first) try writer.writeByte(','); + try writeSourceMapRun(writer, cursor, span.block_text_start, "span", null, 0); + first = false; + } + if (!first) try writer.writeByte(','); + try writeSourceMapRun(writer, span.block_text_start, span.block_text_end, "span", offsets.span_base + @as(u32, @intCast(index)), span.span.text.len); + first = false; + cursor = span.block_text_end; + } + try writer.writeByte(']'); +} + +fn writeChunkSourceMap(writer: anytype, result: anytype, chunk: anytype, block_base: u32) !void { + try writer.writeAll(",\"source_map\":["); + var cursor: usize = 0; + const start: usize = chunk.block_start; + for (result.reconciled.blocks[start .. start + chunk.block_count], start..) |block, index| { + if (index > start) { + try writer.writeByte(','); + try writeSourceMapRun(writer, cursor, cursor + 2, "block", null, 0); + cursor += 2; + try writer.writeByte(','); + } + try writeSourceMapRun(writer, cursor, cursor + block.text.len, "block", block_base + @as(u32, @intCast(index)), block.text.len); + cursor += block.text.len; + } + try writer.writeByte(']'); +} + +fn writeChunkSpanIds(writer: anytype, result: anytype, chunk: anytype, offsets: RecordOffsets) !void { + try writer.writeByte('['); + var first = true; + const start: usize = chunk.block_start; + for (result.reconciled.blocks[start .. start + chunk.block_count]) |block| { + for (0..block.span_count) |index| { + if (!first) try writer.writeByte(','); + try writer.print("\"span-{d}\"", .{offsets.span_base + block.span_start + @as(u32, @intCast(index))}); + first = false; + } + } + try writer.writeByte(']'); +} + pub fn writeDebugAssetRecords(writer: anytype, document_id: []const u8, source_id: ?[]const u8, input_sha256: ?[]const u8, records: []const visual_assets.AssetRecord, jsonl: bool, stream_event_index: ?*u64) !void { for (records, 0..) |asset, index| { if (index > 0) { @@ -1860,6 +1957,38 @@ fn writeSpanIdsInBBox(writer: anytype, result: anytype, page_index: u32, bbox: l try writer.writeByte(']'); } +// Match the exact span selection used by source_span_ids, then follow its +// post-reconciliation block ownership. Layout block indices are a different +// namespace and remain only in compatibility fields. +fn writeOwningBlockIds(writer: anytype, result: anytype, page_index: u32, bbox: layout.BBox, offsets: RecordOffsets) !void { + try writer.writeByte('['); + var first = true; + for (result.reconciled.blocks) |block| { + if (block.page_index != page_index) continue; + const start: usize = block.span_start; + var matched = false; + for (result.reconciled.spans[start .. start + block.span_count]) |span| { + if (span.span.page_index == page_index and centerInside(span.span.bbox, bbox)) { + matched = true; + break; + } + } + if (!matched) continue; + if (!first) try writer.writeByte(','); + try writer.print("\"block-{d}\"", .{offsets.block_base + block.id}); + first = false; + } + try writer.writeByte(']'); +} + +fn writeCoverage(writer: anytype, assessment: coverage.Assessment) !void { + try writer.print(",\"coverage\":{{\"status\":\"{s}\",\"completeness\":\"{s}\",\"native_span_count\":{},\"recognized_span_count\":{},\"requested_recovery\":{},\"recovery_state\":\"{s}\",\"unresolved_region_count\":{}}}", .{ + @tagName(assessment.status), @tagName(assessment.completeness), assessment.native_span_count, + assessment.recognized_span_count, assessment.requested_recovery, @tagName(assessment.recovery_state), + assessment.unresolved_region_count, + }); +} + fn collectCellSpanIds(allocator: std.mem.Allocator, span_ids: *std.ArrayList(u32), result: anytype, cell: layout.TableCell, offsets: RecordOffsets) !void { span_ids.clearRetainingCapacity(); if (offsets.span_lookup) |lookup| { diff --git a/src/specialist_protocol.zig b/src/specialist_protocol.zig index 5e2146f..d271a77 100644 --- a/src/specialist_protocol.zig +++ b/src/specialist_protocol.zig @@ -10,7 +10,7 @@ const complexity = @import("complexity.zig"); const layout = @import("layout.zig"); const runtime = @import("runtime.zig"); -pub const schema_version = "0.13.0"; +pub const schema_version = "0.15.0"; pub const SpecialistKind = enum { ocr, diff --git a/src/stream.zig b/src/stream.zig index b9311ed..8f92c02 100644 --- a/src/stream.zig +++ b/src/stream.zig @@ -11,6 +11,7 @@ const runtime = @import("runtime.zig"); const schema = @import("schema.zig"); const specialist_protocol = @import("specialist_protocol.zig"); const visual_assets = @import("visual_assets.zig"); +const resource_limits = @import("resource_limits.zig"); pub const StreamingEventType = enum { document_manifest, @@ -33,6 +34,8 @@ pub const StreamingOptions = struct { adaptive_options: adaptive.ExtractOptions = .{}, schema_options: schema.RenderOptions = .{}, include_debug_asset_refs: bool = true, + max_output_bytes: ?usize = null, + max_page_memory_bytes: ?usize = null, }; pub const StreamingSummary = struct { @@ -47,9 +50,15 @@ pub const StreamingSummary = struct { pub fn extractAdaptiveStreaming( allocator: std.mem.Allocator, document: anytype, - writer: anytype, + output_writer: anytype, options: StreamingOptions, ) !StreamingSummary { + if (options.max_output_bytes == 0 or options.max_page_memory_bytes == 0) return error.InvalidResourceLimit; + var limited_writer = resource_limits.LimitedWriter(@TypeOf(output_writer)){ + .writer = output_writer, + .max_bytes = options.max_output_bytes, + }; + const writer = &limited_writer; const page_start = options.adaptive_options.page_start orelse 0; const page_end = options.adaptive_options.page_end orelse document.pages.items.len; if (page_start > page_end or page_end > document.pages.items.len) return error.InvalidPageRange; @@ -97,13 +106,24 @@ pub fn extractAdaptiveStreaming( page_options.fallback_page_index = fallback_page_index; page_options.region_index_base = region_index_base; - var page_result = try adaptive.extractDocument(allocator, document, page_options); + var page_budget = resource_limits.BudgetAllocator{ + .parent = allocator, + .max_bytes = options.max_page_memory_bytes orelse std.math.maxInt(usize), + }; + const page_allocator = if (options.max_page_memory_bytes != null) page_budget.allocator() else allocator; + var page_result = adaptive.extractDocument(page_allocator, document, page_options) catch |err| { + if (page_budget.limit_hit.load(.monotonic)) return error.MemoryBudgetExceeded; + return err; + }; defer page_result.deinit(); + // Some native fallbacks deliberately tolerate allocation errors. A + // budget breach must still fail the stream rather than publish omission. + if (page_budget.limit_hit.load(.monotonic)) return error.MemoryBudgetExceeded; region_index_base += @intCast(page_result.region_routes.len); if (page_result.hasSpecialistFailures()) summary.specialist_failure_count += 1; - const page_counts = try writePageArtifacts( - allocator, + const page_counts = writePageArtifacts( + page_allocator, writer, manifest_options.document_id, manifest_options.source_id, @@ -114,7 +134,10 @@ pub fn extractAdaptiveStreaming( summary.artifact_counts, &table_context, &event_index, - ); + ) catch |err| { + if (page_budget.limit_hit.load(.monotonic)) return error.MemoryBudgetExceeded; + return err; + }; summary.artifact_counts.spans += page_counts.spans; summary.artifact_counts.blocks += page_counts.blocks; summary.artifact_counts.tables += page_counts.tables; @@ -152,6 +175,21 @@ pub fn extractAdaptiveStreaming( const elapsed_ns = runtime.nanoTimestamp() - started_ns; summary.elapsed_ms = if (elapsed_ns > 0) @intCast(@divTrunc(elapsed_ns, 1_000_000)) else 0; + // Reserve the whole terminal record before emitting any of it. A bounded + // failure can leave an incomplete artifact, but never a completion marker. + var terminal_counter = resource_limits.CountingWriter{}; + try schema.writeDocumentFinishedRecord( + &terminal_counter, + manifest_options.document_id, + manifest_options.source_id, + manifest_options.input_sha256, + summary.artifact_counts, + summary.route_counts, + summary.elapsed_ms, + event_index, + ); + try terminal_counter.writeByte('\n'); + try limited_writer.ensureCapacity(terminal_counter.written); try schema.writeDocumentFinishedRecord( writer, manifest_options.document_id, @@ -394,3 +432,54 @@ test "streaming event enum includes lifecycle and artifact records" { try std.testing.expectEqual(StreamingEventType.document_manifest, StreamingEventType.document_manifest); try std.testing.expectEqual(StreamingEventType.document_finished, StreamingEventType.document_finished); } + +test "stream output cap reserves terminal record atomically" { + const allocator = std.testing.allocator; + var threaded: std.Io.Threaded = .init(allocator, .{}); + defer threaded.deinit(); + runtime.setIo(threaded.io()); + const root = @import("root.zig"); + const pdf_data = try @import("testpdf.zig").generateCleanNativePdf(allocator); + defer allocator.free(pdf_data); + const doc = try root.Document.openFromMemory(allocator, pdf_data, root.ErrorConfig.default()); + defer doc.close(); + var output: std.ArrayList(u8) = .empty; + defer output.deinit(allocator); + const options = StreamingOptions{ .adaptive_options = .{ .page_start = 0, .page_end = 0, .enable_ocr = false }, .include_debug_asset_refs = false }; + _ = try extractAdaptiveStreaming(allocator, doc, runtime.arrayListWriter(&output, allocator), options); + const terminal_start = std.mem.lastIndexOfScalar(u8, output.items[0 .. output.items.len - 1], '\n').? + 1; + output.clearRetainingCapacity(); + var limited_options = options; + limited_options.max_output_bytes = terminal_start + 1; + try std.testing.expectError(error.OutputLimitExceeded, extractAdaptiveStreaming(allocator, doc, runtime.arrayListWriter(&output, allocator), limited_options)); + try std.testing.expectEqual(terminal_start, output.items.len); + try std.testing.expect(std.mem.indexOf(u8, output.items, "document_finished") == null); +} + +test "stream page allocation budget failures clean up partial extraction" { + const allocator = std.testing.allocator; + var threaded: std.Io.Threaded = .init(allocator, .{}); + defer threaded.deinit(); + runtime.setIo(threaded.io()); + const root = @import("root.zig"); + const pdf_data = try @import("testpdf.zig").generateCleanNativePdf(allocator); + defer allocator.free(pdf_data); + const doc = try root.Document.openFromMemory(allocator, pdf_data, root.ErrorConfig.default()); + defer doc.close(); + var output: std.ArrayList(u8) = .empty; + defer output.deinit(allocator); + var failed_count: usize = 0; + for ([_]usize{ 1, 32, 128, 512, 2048, 8192, 32768 }) |limit| { + output.clearRetainingCapacity(); + if (extractAdaptiveStreaming(allocator, doc, runtime.arrayListWriter(&output, allocator), .{ + .adaptive_options = .{ .enable_ocr = false }, + .include_debug_asset_refs = false, + .max_page_memory_bytes = limit, + })) |_| {} else |err| { + try std.testing.expectEqual(error.MemoryBudgetExceeded, err); + try std.testing.expect(std.mem.indexOf(u8, output.items, "document_finished") == null); + failed_count += 1; + } + } + try std.testing.expect(failed_count > 0); +} diff --git a/src/testpdf.zig b/src/testpdf.zig index 504af29..646771f 100644 --- a/src/testpdf.zig +++ b/src/testpdf.zig @@ -652,6 +652,19 @@ fn repeatedWidths(allocator: std.mem.Allocator, count: usize, width_value: []con return widths.toOwnedSlice(allocator); } +/// One marked-content group with two independently positioned text spans. +pub fn generateSharedMcidLinesPdf(allocator: std.mem.Allocator) ![]u8 { + const content = + "/Span << /MCID 7 >> BDC\n" ++ + "BT /F1 12 Tf 100 700 Td (FIRST) Tj 0 -24 Td (SECOND) Tj ET\n" ++ + "EMC\n"; + const font = + "5 0 obj\n" ++ + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>\n" ++ + "endobj\n"; + return generateSinglePageFontFixturePdf(allocator, "/F1 5 0 R", content, &.{font}); +} + /// Generate a fixture where /ActualText repairs intentionally bad visible text. pub fn generateActualTextRepairPdf(allocator: std.mem.Allocator) ![]u8 { const content =