Preserve PDF text origins and bound adaptive streaming

.github/workflows/ci.yml

@@ -135,6 +135,13 @@ jobs:
       - name: Build shared library
         run: zig build shared --summary all
 
+      - uses: actions/setup-node@v6
+        with:
+          node-version: "22"
+
+      - name: Test public JavaScript adapters against current CLI
+        run: node --test integrations/javascript/*.test.mjs
+
   python:
     name: Python ${{ matrix.python-version }} bindings and wheel
     runs-on: ubuntu-latest

README.md

@@ -463,7 +463,7 @@ never emits the password.
 
 Use three integration modes, in this order:
 
-1. **CLI subprocess**: recommended default for workers, Siftable ingestion, and
+1. **CLI subprocess**: recommended default for workers, research applications ingestion, and
    Cloud Run jobs. It is easiest to isolate, supervise, retry, and deploy:
 
    ```bash
@@ -503,7 +503,7 @@ Use three integration modes, in this order:
    Python exposes the same surface as `zpdf.extract_adaptive(...,
    password="known-password")` or `password_file=".pdf-password"`. Node
    wrappers should prefer Node-API over direct V8 bindings for ABI stability,
-   while Siftable can continue using the subprocess path first.
+   while research applications can continue using the subprocess path first.
 
 3. **HTTP server**: useful for later long-running batch corpora or internal
    services. It is a stateless wrapper around the same adapter:
@@ -557,7 +557,7 @@ OCR/table/form/formula extraction counts, output artifact hashes or stream hash
 slots, warnings/errors, and capability coverage. That makes it suitable as the
 stable run record for general pipelines, not just this CLI.
 
-For Siftable-style ingestion, `stream-jsonl` maps naturally to durable
+For research applications-style ingestion, `stream-jsonl` maps naturally to durable
 processing records: `document_manifest` as a manifest artifact,
 `page_started`/`page_finished`/`document_finished` as status artifacts,
 `span`/`block`/`table` as extracted text artifacts, `route_trace` and
@@ -669,13 +669,24 @@ and ruling lines, invokes OCR only for scanned routes, then reconciles native,
 OCR, table, formula, and form spans with typed provenance.
 
 The versioned JSON, artifact JSONL, and streaming JSONL schema is currently
-`0.13.0`, with parser build identity `0.4.0-dev` after the `v0.3.0` release.
+`0.15.0`, with parser build identity `0.4.0-dev` after the `v0.3.0` release.
 Every emitted record carries a `provenance` envelope with document and
 source identity, input hash context, artifact id, page/bbox, source kind,
 confidence, related span/block/chunk ids, route trace ids, and route reasons.
 This makes parser outputs usable as reviewable evidence in host pipelines
 without removing the older top-level compatibility fields.
 
+Schema `0.14.0` also provides hash-bound text views, block-to-span source maps,
+and chunk-to-block maps with explicit synthetic whitespace. The
+[neutral JavaScript adapters](docs/host-adapters.md) expose rich extraction,
+UTF-16 locators, overlap-aware chunk origins and separate prompt projections.
+Schema `0.15.0` adds reconciled span/table/cell ownership and page/region recovery
+assessments. Streaming supports output and adaptive page allocation budgets;
+the JavaScript page adapter validates lifecycle completion and applies
+backpressure. Visual completeness remains unverified.
+Host adoption remains explicit; accepted schema families and parser/compiler
+pins must be updated together.
+
 ## Comparison
 
 | Feature | pdf-parser | pdfium | MuPDF | pdfplumber |

benchmark/eval/host_locator_probe.cjs

@@ -0,0 +1,58 @@
+#!/usr/bin/env node
+/** Research reduction of the host's actual normalization/offset functions.
+ * Loads only those functions; never imports host configuration, DB, or providers.
+ * Requires the host's already-installed TypeScript compiler API.
+ */
+const fs = require('node:fs');
+const path = require('node:path');
+const crypto = require('node:crypto');
+const vm = require('node:vm');
+const { createRequire } = require('node:module');
+
+const [hostRoot, chunkingPath, indexingPath] = process.argv.slice(2);
+if (!hostRoot || !chunkingPath || !indexingPath) {
+  throw new Error('Usage: node host_locator_probe.cjs CONSUMER_ROOT CHUNKING_SOURCE INDEXING_SOURCE');
+}
+const hostRequire = createRequire(path.join(hostRoot, 'package.json'));
+const ts = hostRequire('typescript');
+const sources = {
+  chunking: chunkingPath,
+  indexing: indexingPath,
+};
+const contents = Object.fromEntries(Object.entries(sources).map(([key, relative]) =>
+  [key, fs.readFileSync(path.join(hostRoot, relative), 'utf8')]));
+
+function extractFunction(source, first, next) {
+  const start = source.indexOf(`function ${first}(`);
+  const end = source.indexOf(`function ${next}(`, start);
+  if (start < 0 || end < 0) throw new Error(`Source changed: ${first} boundaries missing`);
+  return source.slice(start, end);
+}
+
+const selected = extractFunction(contents.chunking, 'normaliseWhitespace', 'splitSentences')
+  + '\n' + extractFunction(contents.indexing, 'findOffset', 'metadataString');
+const compiled = ts.transpileModule(selected, {
+  compilerOptions: { target: ts.ScriptTarget.ES2022 },
+}).outputText;
+const context = vm.createContext({});
+vm.runInContext(compiled, context, { timeout: 1000 });
+
+const cases = ['A B C', 'A   B C', 'The office\n\napproved the budget.', 'A 😀  B'];
+const results = cases.map((raw) => {
+  const chunk = context.normaliseWhitespace(raw);
+  const offsets = context.findOffset(raw, chunk, 0);
+  const rawSlice = raw.slice(offsets.start, offsets.end);
+  const normalizedRawSlice = context.normaliseWhitespace(rawSlice);
+  return { raw, chunk, offsets, rawSlice, normalizedRawSlice,
+    sourceRangeCoversChunk: normalizedRawSlice === chunk };
+});
+console.log(JSON.stringify({
+  evidenceBoundary: 'isolated actual host functions; no DB, UI, or ingestion run',
+  hostRoot,
+  typescriptVersion: ts.version,
+  sources: Object.entries(sources).map(([key, relative]) => ({
+    path: path.join(hostRoot, relative),
+    sha256: crypto.createHash('sha256').update(contents[key]).digest('hex'),
+  })),
+  results,
+}, null, 2));

benchmark/eval/reference_review_probes.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+"""Read-only parser comparisons against local reference fixtures.
+
+Generated reductions and JSON evidence stay in ignored eval output paths.
+This is a research harness, not a quality gate or a reference-model benchmark.
+"""
+
+import argparse
+import hashlib
+import json
+import subprocess
+from pathlib import Path
+
+
+REPO = Path(__file__).resolve().parents[2]
+DEFAULT_REFERENCES = Path("/Volumes/External SSD/referencerepos")
+
+
+def simple_pdf(content: str) -> bytes:
+    stream = content.encode("ascii")
+    objects = [
+        b"<< /Type /Catalog /Pages 2 0 R >>",
+        b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
+        b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] "
+        b"/Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>",
+        b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
+        b"<< /Length " + str(len(stream)).encode() + b" >>\nstream\n"
+        + stream + b"\nendstream",
+    ]
+    data = bytearray(b"%PDF-1.4\n")
+    offsets = [0]
+    for index, obj in enumerate(objects, 1):
+        offsets.append(len(data))
+        data.extend(f"{index} 0 obj\n".encode() + obj + b"\nendobj\n")
+    xref = len(data)
+    data.extend(f"xref\n0 {len(objects) + 1}\n0000000000 65535 f \n".encode())
+    for offset in offsets[1:]:
+        data.extend(f"{offset:010d} 00000 n \n".encode())
+    data.extend(
+        f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\n"
+        f"startxref\n{xref}\n%%EOF\n".encode()
+    )
+    return bytes(data)
+
+
+def run(args: list[str]) -> dict:
+    result = subprocess.run(args, capture_output=True, text=True, timeout=30)
+    return {"command": args, "exit_code": result.returncode,
+            "stdout": result.stdout, "stderr": result.stderr}
+
+
+def probe(executable: Path, pdf: Path, label: str, disable_ocr: bool = True) -> dict:
+    native = run([str(executable), "extract", str(pdf)])
+    adaptive_args = [str(executable), "extract-adaptive", "--input", str(pdf)]
+    if disable_ocr:
+        adaptive_args.append("--no-ocr")
+    adaptive = run(adaptive_args + ["--format", "artifact-jsonl"])
+    records = [json.loads(line) for line in adaptive["stdout"].splitlines()
+               if line.strip()] if adaptive["exit_code"] == 0 else []
+    spans = [r for r in records if r.get("record_type") == "span"]
+    chunks = [r for r in records if r.get("record_type") == "rag_chunk"]
+    routes = [r for r in records if r.get("record_type") == "route_trace"
+              and r.get("trace_kind") in {"page_route", "region_route"}]
+    return {
+        "label": label, "pdf_path": str(pdf),
+        "input_sha256": hashlib.sha256(pdf.read_bytes()).hexdigest(),
+        "native_exit_code": native["exit_code"],
+        "native_text": native["stdout"], "native_stderr": native["stderr"],
+        "adaptive_exit_code": adaptive["exit_code"],
+        "adaptive_stderr": adaptive["stderr"],
+        "spans": [{k: r.get(k) for k in ("text", "bbox", "mcid", "source",
+                   "confidence", "duplicate_count")} for r in spans],
+        "chunks": [{k: r.get(k) for k in ("content", "text", "confidence")}
+                   for r in chunks],
+        "routes": [{k: r.get(k) for k in ("trace_kind", "route", "reasons",
+                    "signals", "bbox")} for r in routes],
+        "commands": [native["command"], adaptive["command"]],
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--references", type=Path, default=DEFAULT_REFERENCES)
+    parser.add_argument("--executable", type=Path, default=REPO / "zig-out/bin/pdf-parser")
+    parser.add_argument("--output-dir", type=Path,
+                        default=REPO / "benchmark/eval/outputs/reference-review-2026-10-09")
+    args = parser.parse_args()
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+    reductions = {
+        "same-mcid-distinct-lines": "/Span << /MCID 0 >> BDC BT /F1 12 Tf "
+        "20 250 Td (FIRST) Tj 0 -24 Td (SECOND) Tj ET EMC",
+        "distinct-lines-control": "BT /F1 12 Tf 20 250 Td (FIRST) Tj "
+        "0 -24 Td (SECOND) Tj ET",
+        "rotation-45-word-gap": "BT /F1 12 Tf "
+        ".70710678 .70710678 -.70710678 .70710678 100 50 Tm "
+        "(A) Tj 8.004 0 Td (B) Tj 11.340 0 Td (C) Tj 8.664 0 Td (D) Tj ET",
+        "horizontal-word-gap-control": "BT /F1 12 Tf 1 0 0 1 100 50 Tm "
+        "(A) Tj 8.004 0 Td (B) Tj 11.340 0 Td (C) Tj 8.664 0 Td (D) Tj ET",
+        "ordinary-font-invisible-text": "BT /F1 12 Tf 20 250 Td "
+        "(VISIBLE) Tj 0 -24 Td 3 Tr (INVISIBLE) Tj ET",
+        "clipped-out-text": "q 0 0 10 10 re W n BT /F1 12 Tf "
+        "100 200 Td (OUTSIDE-CLIP) Tj ET Q",
+        "unique-page-title": "BT /F1 12 Tf 20 280 Td (Unique Page Title) Tj "
+        "0 -45 Td (First body line with enough words) Tj "
+        "0 -24 Td (Second body line with enough words) Tj "
+        "0 -24 Td (Third body line with enough words) Tj "
+        "0 -24 Td (Fourth body line with enough words) Tj ET",
+    }
+    evidence = []
+    for name, content in reductions.items():
+        path = args.output_dir / f"{name}.pdf"
+        path.write_bytes(simple_pdf(content))
+        evidence.append(probe(args.executable, path, name))
+    fixture_dir = args.references / "docling/tests/data/pdf"
+    for name in ("text_with_vector_glyphs.pdf", "text_with_vector_rule.pdf",
+                 "invisible_text_layer.pdf", "vector_glyphs_in_figure.pdf"):
+        path = fixture_dir / name
+        if path.exists():
+            evidence.append(probe(args.executable, path, f"docling/{name}"))
+            if name in {"text_with_vector_glyphs.pdf", "invisible_text_layer.pdf"}:
+                evidence.append(probe(args.executable, path,
+                                      f"docling/{name}/ocr-enabled", disable_ocr=False))
+    output = args.output_dir / "probes.json"
+    output.write_text(json.dumps(evidence, indent=2, ensure_ascii=False) + "\n")
+    for entry in evidence:
+        print(json.dumps({"label": entry["label"],
+                          "exit_code": entry["adaptive_exit_code"],
+                          "native_text": entry["native_text"].strip(),
+                          "adaptive_spans": [s["text"] for s in entry["spans"]],
+                          "mcids": [s["mcid"] for s in entry["spans"]],
+                          "duplicate_counts": [s["duplicate_count"] for s in entry["spans"]],
+                          "page_routes": [{"route": r["route"], "reasons": r["reasons"],
+                                           "hidden_ocr": (r["signals"] or {}).get("hidden_ocr")}
+                                          for r in entry["routes"] if r["trace_kind"] == "page_route"]},
+                         ensure_ascii=False))
+    print(f"Evidence: {output}")
+
+
+if __name__ == "__main__":
+    main()