From 9138eed9632e08bb2d860660410b969176fc2a0e Mon Sep 17 00:00:00 2001 From: sabith Date: Sun, 11 Oct 2026 20:41:32 -0700 Subject: [PATCH 1/2] feat: add opt-in sparse table record serializer --- .github/workflows/stackgen-serve.yml | 99 ----- docling/utils/table_records.py | 186 +++++++++ docs/concepts/serialization.md | 24 ++ integrations/stackgen_serve/Dockerfile | 11 - .../stackgen_serve/Dockerfile.dockerignore | 6 - integrations/stackgen_serve/README.md | 79 ---- .../stackgen_serve/build-constraints.txt | 1 - integrations/stackgen_serve/build.sh | 7 - integrations/stackgen_serve/pyproject.toml | 13 - .../src/stackgen_docling/__init__.py | 1 - .../src/stackgen_docling/__main__.py | 29 -- .../src/stackgen_docling/app.py | 26 -- .../src/stackgen_docling/install.py | 40 -- .../src/stackgen_docling/record_tables.py | 102 ----- integrations/stackgen_serve/test-image.sh | 29 -- .../stackgen_serve/tests/http_smoke.py | 268 ------------- .../stackgen_serve/tests/runtime_checks.py | 72 ---- .../stackgen_serve/tests/test_http_smoke.py | 52 --- .../stackgen_serve/tests/test_launch.py | 77 ---- .../tests/test_record_tables.py | 360 ------------------ .../tests/test_runtime_checks.py | 77 ---- .../stackgen_serve/tests/verify_install.py | 31 -- tests/test_table_records.py | 244 ++++++++++++ 23 files changed, 454 insertions(+), 1380 deletions(-) delete mode 100644 .github/workflows/stackgen-serve.yml create mode 100644 docling/utils/table_records.py delete mode 100644 integrations/stackgen_serve/Dockerfile delete mode 100644 integrations/stackgen_serve/Dockerfile.dockerignore delete mode 100644 integrations/stackgen_serve/README.md delete mode 100644 integrations/stackgen_serve/build-constraints.txt delete mode 100644 integrations/stackgen_serve/build.sh delete mode 100644 integrations/stackgen_serve/pyproject.toml delete mode 100644 integrations/stackgen_serve/src/stackgen_docling/__init__.py delete mode 100644 integrations/stackgen_serve/src/stackgen_docling/__main__.py delete mode 100644 integrations/stackgen_serve/src/stackgen_docling/app.py delete mode 100644 integrations/stackgen_serve/src/stackgen_docling/install.py delete mode 100644 integrations/stackgen_serve/src/stackgen_docling/record_tables.py delete mode 100644 integrations/stackgen_serve/test-image.sh delete mode 100644 integrations/stackgen_serve/tests/http_smoke.py delete mode 100644 integrations/stackgen_serve/tests/runtime_checks.py delete mode 100644 integrations/stackgen_serve/tests/test_http_smoke.py delete mode 100644 integrations/stackgen_serve/tests/test_launch.py delete mode 100644 integrations/stackgen_serve/tests/test_record_tables.py delete mode 100644 integrations/stackgen_serve/tests/test_runtime_checks.py delete mode 100644 integrations/stackgen_serve/tests/verify_install.py create mode 100644 tests/test_table_records.py diff --git a/.github/workflows/stackgen-serve.yml b/.github/workflows/stackgen-serve.yml deleted file mode 100644 index b929d8fd2c..0000000000 --- a/.github/workflows/stackgen-serve.yml +++ /dev/null @@ -1,99 +0,0 @@ -name: StackGen Serve image -on: - pull_request: - paths: - - 'docling/**' - - 'pyproject.toml' - - 'integrations/stackgen_serve/**' - - '.github/workflows/stackgen-serve.yml' - push: - branches: [main] - paths: - - 'docling/**' - - 'pyproject.toml' - - 'integrations/stackgen_serve/**' - - '.github/workflows/stackgen-serve.yml' - workflow_dispatch: - inputs: - publish: - description: Publish the image (main branch only) - type: boolean - default: false -permissions: - contents: read -jobs: - image-test: - strategy: - fail-fast: false - matrix: - runner: [ubuntu-24.04, ubuntu-24.04-arm] - runs-on: ${{ matrix.runner }} - timeout-minutes: 30 - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: astral-sh/setup-uv@v7 - - name: Python format and lint - run: | - uvx --from ruff==0.15.12 ruff format --check --config pyproject.toml integrations/stackgen_serve - uvx --from ruff==0.15.12 ruff check --config pyproject.toml integrations/stackgen_serve - - name: Build checkout wheels and local image - run: sh integrations/stackgen_serve/build.sh - - name: Verify fork, unit tests, HTTP default, workers and reload - run: sh integrations/stackgen_serve/test-image.sh - - publish: - needs: image-test - if: >- - github.repository == 'stackgenhq/docling' - && github.ref == 'refs/heads/main' - && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.publish)) - runs-on: ubuntu-24.04 - timeout-minutes: 30 - permissions: - contents: read - packages: write - concurrency: - group: stackgen-serve-publication - cancel-in-progress: false - env: - IMAGE: ghcr.io/stackgenhq/docling-serve - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - - uses: astral-sh/setup-uv@v7 - - name: Build wheels from the tested revision - run: | - uv build --wheel --build-constraints integrations/stackgen_serve/build-constraints.txt --out-dir integrations/stackgen_serve/dist - uv build --wheel --build-constraints integrations/stackgen_serve/build-constraints.txt integrations/stackgen_serve --out-dir integrations/stackgen_serve/dist - - uses: docker/setup-qemu-action@v3 - - uses: docker/setup-buildx-action@v3 - - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Reject an outdated main revision - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - latest=$(gh api "repos/$GITHUB_REPOSITORY/commits/main" --jq .sha) - test "$latest" = "$GITHUB_SHA" || { echo 'Main advanced; do not overwrite its image tag.' >&2; exit 1; } - - name: Publish multi-platform image - id: image - uses: docker/build-push-action@v6 - with: - context: . - file: integrations/stackgen_serve/Dockerfile - platforms: linux/amd64,linux/arm64 - network: none - push: true - tags: | - ${{ env.IMAGE }}:main - ${{ env.IMAGE }}:sha-${{ github.sha }} - labels: | - org.opencontainers.image.source=https://github.com/stackgenhq/docling - org.opencontainers.image.revision=${{ github.sha }} - - name: Report deployment pin - env: - DIGEST: ${{ steps.image.outputs.digest }} - run: | - printf 'Published image pin: `%s@%s`\n' "$IMAGE" "$DIGEST" >> "$GITHUB_STEP_SUMMARY" diff --git a/docling/utils/table_records.py b/docling/utils/table_records.py new file mode 100644 index 0000000000..e44a263746 --- /dev/null +++ b/docling/utils/table_records.py @@ -0,0 +1,186 @@ +# SPDX-FileCopyrightText: The Docling Contributors +# SPDX-License-Identifier: MIT +"""Render table cells as labeled records without building a padded grid.""" + +import json +from typing import Any + +from docling_core.transforms.serializer.base import ( + BaseDocSerializer, + BaseTableSerializer, + SerializationResult, +) +from docling_core.transforms.serializer.common import create_ser_result +from docling_core.transforms.serializer.markdown import MarkdownParams +from docling_core.types.doc import ( + CaptionPlacement, + CoordOrigin, + DoclingDocument, + NodeItem, + ProvenanceItem, + RichTableCell, + TableCell, + TableItem, + TextItem, +) + + +class RecordTableSerializer(BaseTableSerializer): + """Write one entry per source cell, including its position and span. + + Missing positions are omitted; explicit empty cells are not. The output is + text in a Markdown document, not a Markdown pipe table. Select this class + on a ``MarkdownDocSerializer`` instance to leave other exports unchanged. + """ + + def serialize( + self, + *, + item: TableItem, + doc_serializer: BaseDocSerializer, + doc: DoclingDocument, + **kwargs: Any, + ) -> SerializationResult: + if kwargs.get("_nested_in_table"): + visited = kwargs.get("visited") + if isinstance(visited, set): + _mark_visited(item, doc, visited) + return create_ser_result( + text=_nested_text( + item, doc, doc_serializer.get_excluded_refs(**kwargs) + ), + span_source=item, + ) + + params = MarkdownParams(**kwargs) + caption = doc_serializer.serialize_captions(item=item, **kwargs) + caption_after = _caption_after(item, doc, params) + parts: list[SerializationResult] = [] + if caption.text and not caption_after: + parts.append(caption) + + if item.self_ref not in doc_serializer.get_excluded_refs(**kwargs): + lines = [ + f"Table: {item.data.num_rows} rows, {item.data.num_cols} columns. " + "Only actual cells follow.\n" + ] + current_row = -1 + for cell in sorted( + item.data.table_cells, + key=lambda cell: (cell.start_row_offset_idx, cell.start_col_offset_idx), + ): + if cell.start_row_offset_idx != current_row: + current_row = cell.start_row_offset_idx + lines.append(f"Record {current_row + 1}\n") + if isinstance(cell, RichTableCell): + value = doc_serializer.serialize( + item=cell.ref.resolve(doc=doc), + **{**kwargs, "_nested_in_table": True, "in_table_cell": True}, + ).text + else: + value = cell.text + literal = json.dumps(value, ensure_ascii=False) + if params.escape_html: + for character, escaped in ( + ("<", r"\u003c"), + (">", r"\u003e"), + ("&", r"\u0026"), + ): + literal = literal.replace(character, escaped) + lines.append( + f"- Column {cell.start_col_offset_idx + 1}{_cell_label(cell)}: {literal}\n" + ) + parts.append( + create_ser_result(text="".join(lines).rstrip(), span_source=item) + ) + + if caption.text and caption_after: + parts.append(caption) + return create_ser_result( + text="\n\n".join(part.text for part in parts if part.text), + span_source=parts, + ) + + +def _cell_label(cell: TableCell) -> str: + labels = [] + if cell.column_header: + labels.append("column header") + if cell.row_header: + labels.append("row header") + if cell.row_span > 1: + labels.append(f"rows {cell.start_row_offset_idx + 1}-{cell.end_row_offset_idx}") + if cell.col_span > 1: + labels.append( + f"columns {cell.start_col_offset_idx + 1}-{cell.end_col_offset_idx}" + ) + return f" ({'; '.join(labels)})" if labels else "" + + +def _mark_visited(item: NodeItem, doc: DoclingDocument, visited: set[str]) -> None: + visited.add(item.self_ref) + for child in item.children: + _mark_visited(child.resolve(doc=doc), doc, visited) + + +def _nested_text(item: NodeItem, doc: DoclingDocument, excluded: set[str]) -> str: + if item.self_ref in excluded: + return "" + if isinstance(item, TableItem): + values = [] + for cell in sorted( + item.data.table_cells, + key=lambda cell: (cell.start_row_offset_idx, cell.start_col_offset_idx), + ): + value = ( + _nested_text(cell.ref.resolve(doc=doc), doc, excluded) + if isinstance(cell, RichTableCell) + else cell.text + ) + if value: + values.append(value) + return " ".join(values) + values = [item.text] if isinstance(item, TextItem) and item.text else [] + for child in item.children: + text = _nested_text(child.resolve(doc=doc), doc, excluded) + if text: + values.append(text) + return " ".join(values) + + +def _caption_after( + item: TableItem, doc: DoclingDocument, params: MarkdownParams +) -> bool: + if ( + params.caption_placement != CaptionPlacement.LAYOUT + or not item.prov + or not item.captions + ): + return False + table_prov = item.prov[0] + table_y = _center_y(table_prov, doc) + if table_y is None: + return False + for caption_ref in item.captions: + caption = caption_ref.resolve(doc=doc) + if not isinstance(caption, TextItem): + return False + caption_prov = next( + (prov for prov in caption.prov if prov.page_no == table_prov.page_no), + None, + ) + if caption_prov is None or (caption_y := _center_y(caption_prov, doc)) is None: + return False + if caption_y <= table_y: + return False + return True + + +def _center_y(prov: ProvenanceItem, doc: DoclingDocument) -> float | None: + bbox = prov.bbox + if bbox.coord_origin != CoordOrigin.TOPLEFT: + page = doc.pages.get(prov.page_no) + if page is None: + return None + bbox = bbox.to_top_left_origin(page_height=page.size.height) + return (bbox.t + bbox.b) / 2 diff --git a/docs/concepts/serialization.md b/docs/concepts/serialization.md index e7fa8970d8..3a817782ca 100644 --- a/docs/concepts/serialization.md +++ b/docs/concepts/serialization.md @@ -65,6 +65,30 @@ serializer for any format by subclassing `BaseTableSerializer` and passing your implementation when instantiating the document serializer. +## Record-oriented table text + +For wide, sparse tables, you can opt in to a record-oriented table serializer. +It writes each source cell once, with its one-based row and column position. +It does not build the padded `TableData.grid` during table rendering. + +```python +from docling_core.transforms.serializer.markdown import MarkdownDocSerializer +from docling.utils.table_records import RecordTableSerializer + +text = MarkdownDocSerializer( + doc=doc, + table_serializer=RecordTableSerializer(), +).serialize().text +``` + +This is text within a Markdown export, **not** a Markdown pipe table. Missing +positions are omitted, but explicit empty cells are present. Header flags and +merged ranges are written beside their source cells. `RichTableCell` content is +flattened to text; use `export_to_dict()` if you need the full document structure. +The default `export_to_markdown()` output does not change. Conversion and the +final output string still use memory, so this option is not a general memory +limit for large documents. + ## Examples For an example showcasing how to use serializers, see diff --git a/integrations/stackgen_serve/Dockerfile b/integrations/stackgen_serve/Dockerfile deleted file mode 100644 index d357629dfd..0000000000 --- a/integrations/stackgen_serve/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -# Build the wheels from this checkout before building with the repository root context. -# CPU execution only. Use the upstream CPU variant instead of CUDA dependencies. -FROM ghcr.io/docling-project/docling-serve-cpu:v1.36.0@sha256:225c8586e20d5d0fc6811a9e0e044fa602bcc4393f00389009bad42d6787b58f -COPY --chown=1001:0 integrations/stackgen_serve/dist/*.whl /tmp/fork-wheels/ -# The base has docling-slim, not the legacy overlapping docling distribution. -# Fail if that layout changes. Do not resolve or upgrade dependencies. -RUN python -c "from importlib.metadata import distributions; assert 'docling' not in {d.metadata['Name'] for d in distributions()}" \ - && python -m pip install --no-index --no-deps --force-reinstall /tmp/fork-wheels/*.whl \ - && rm -rf /tmp/fork-wheels -ENTRYPOINT ["/opt/app-root/bin/python", "-m", "stackgen_docling"] -CMD ["run"] diff --git a/integrations/stackgen_serve/Dockerfile.dockerignore b/integrations/stackgen_serve/Dockerfile.dockerignore deleted file mode 100644 index 459f7817c2..0000000000 --- a/integrations/stackgen_serve/Dockerfile.dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -# Only the two built wheels belong in the Docker build context. -** -!integrations/ -!integrations/stackgen_serve/ -!integrations/stackgen_serve/dist/ -!integrations/stackgen_serve/dist/*.whl diff --git a/integrations/stackgen_serve/README.md b/integrations/stackgen_serve/README.md deleted file mode 100644 index ad89af302b..0000000000 --- a/integrations/stackgen_serve/README.md +++ /dev/null @@ -1,79 +0,0 @@ -# StackGen record-table Serve integration - -## Benefit - -Markdown exports iterate actual table cells instead of constructing -`TableData.grid`. Sparse tables do not allocate a padded rectangle. Explicit -empty values, source coordinates, column/row header flags, merged ranges and -rich content remain available. Each table uses continuous `Record N` blocks, not headings. -The structured document and its JSON export remain unchanged. Parsing, sorting -and output still use memory; this is not protection against all OOM failures. - -## Image and startup - -This repository owns the exporter and Serve startup adapter. The image installs -both wheels built from this checkout. It uses the digest-pinned upstream -`docling-serve-cpu` base: Serve 1.36.0, Core 2.99.0 and Jobkit 3.8.1. It supports -CPU execution, not GPU/CUDA. Runtime installation is offline with `--no-deps`; -other base dependencies are not upgraded. Do not install an overlapping legacy -`docling` distribution alongside `docling-slim`. - -The launcher enables the exporter before Serve imports. An importable factory -also enables it in spawned workers and reload children. Importing the adapter -alone does not change SDK behavior. Explicit table serializers override its -default. The adapter uses private Core helpers for annotations and caption -placement; update the regression tests before changing the pinned runtime. - -## Local validation - -Run from the repository root with Python 3.11+, uv and Docker: - -```sh -sh integrations/stackgen_serve/build.sh -sh integrations/stackgen_serve/test-image.sh -uvx --from ruff==0.15.12 ruff check --config pyproject.toml integrations/stackgen_serve -uvx --from ruff==0.15.12 ruff format --check --config pyproject.toml integrations/stackgen_serve -make validate -``` - -Image tests mount only tests and a source-hash manifest, not application source. -They compare the installed converter with the checkout, exercise CSV and HTML -in default/workers/reload modes, and require `pip check` to pass. Containers -have no external network, a 3 GiB memory limit and two CPUs. Models are not loaded. -Process identity and OOM guards prevent health responses from hiding worker loss. -The CSV fixture includes blank lines and explicit empty fields. - -For a larger approved UTF-8 CSV, set `CSV` to its absolute path: - -```sh -docker run --rm --network=none --memory=4g --cpus=2 \ - --entrypoint /opt/app-root/bin/python \ - -v "$PWD/integrations/stackgen_serve/tests:/tests:ro" \ - -v "$CSV:/fixture/input.csv:ro" stackgen-docling:fork-review \ - /tests/http_smoke.py default --csv /fixture/input.csv --requests 1 --timeout 600 -``` - -Repeat with `workers` and `reload`. The test compares every field and record -label. One worker-mode request does not prove coverage of both workers or -concurrent upload capacity. Memory counters include the test client and startup; -they are not a server-only profile. Do not commit customer files or large fixtures. -These checks do not establish PDF model behavior or upload-to-search acceptance. - -## CI and publication - -`stackgen-serve.yml` tests native AMD64 and ARM64 images on pull requests and -relevant `main` pushes. After both test jobs pass, a `main` push publishes -`ghcr.io/stackgenhq/docling-serve:main` and `:sha-` as a multi-platform image. -Manual dispatch is test-only unless `publish` is selected on `main`. PRs cannot -publish. Publication rebuilds wheels from the tested revision with the same -pinned base and build constraints; it does not promote the local test image IDs. - -GHCR login uses `GITHUB_TOKEN` with package-write permission only in the publish -job. Organisation package policy and native runner access must permit this. -The workflow summary reports the manifest digest; deployments should use it. -Publishing does not deploy or update downstream deployment image references. - -Remote CI and publication have not been executed. Local checks cover native -ARM64 and emulated AMD64, not native AMD64 capacity. Vulnerability remediation -is deferred; these tests are not a security approval. Production registry access, -concurrent large uploads, and upload-through-search acceptance remain required. diff --git a/integrations/stackgen_serve/build-constraints.txt b/integrations/stackgen_serve/build-constraints.txt deleted file mode 100644 index 67251d57b0..0000000000 --- a/integrations/stackgen_serve/build-constraints.txt +++ /dev/null @@ -1 +0,0 @@ -hatchling==1.30.1 diff --git a/integrations/stackgen_serve/build.sh b/integrations/stackgen_serve/build.sh deleted file mode 100644 index 56ad80597c..0000000000 --- a/integrations/stackgen_serve/build.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh -set -eu -# Run from the repository root. Keep wheel inputs explicit and remove stale wheels. -rm -f integrations/stackgen_serve/dist/*.whl -uv build --wheel --build-constraints integrations/stackgen_serve/build-constraints.txt --out-dir integrations/stackgen_serve/dist -uv build --wheel --build-constraints integrations/stackgen_serve/build-constraints.txt integrations/stackgen_serve --out-dir integrations/stackgen_serve/dist -docker build --network=none -f integrations/stackgen_serve/Dockerfile -t stackgen-docling:fork-review . diff --git a/integrations/stackgen_serve/pyproject.toml b/integrations/stackgen_serve/pyproject.toml deleted file mode 100644 index a73544928f..0000000000 --- a/integrations/stackgen_serve/pyproject.toml +++ /dev/null @@ -1,13 +0,0 @@ -[build-system] -requires = ["hatchling==1.30.1"] -build-backend = "hatchling.build" - -[project] -name = "stackgen-docling" -version = "0.1.0" -description = "Opt-in record-table export and StackGen Docling Serve launcher" -requires-python = ">=3.10,<4" -dependencies = ["docling-core==2.99.0"] - -[tool.hatch.build.targets.wheel] -packages = ["src/stackgen_docling"] diff --git a/integrations/stackgen_serve/src/stackgen_docling/__init__.py b/integrations/stackgen_serve/src/stackgen_docling/__init__.py deleted file mode 100644 index a3c5e62fa2..0000000000 --- a/integrations/stackgen_serve/src/stackgen_docling/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Pinned Docling Serve integration; only table-to-Markdown rendering changes.""" diff --git a/integrations/stackgen_serve/src/stackgen_docling/__main__.py b/integrations/stackgen_serve/src/stackgen_docling/__main__.py deleted file mode 100644 index e3d2df6486..0000000000 --- a/integrations/stackgen_serve/src/stackgen_docling/__main__.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Preserve Serve's CLI while directing every server process to our factory.""" - -import importlib -from typing import Any -from unittest.mock import patch - -import uvicorn - -from stackgen_docling.install import install - - -class _ServerRunner: - """Redirect the pinned CLI's factory without patching the shared Uvicorn module.""" - - def run(self, *, app: str, **kwargs: Any) -> None: - """Keep upstream options, but install inside spawned workers/reload children.""" - uvicorn.run(app="stackgen_docling.app:create_app", **kwargs) - - -def main() -> None: - """Install for CLI work; delegate serving-process installation to our factory.""" - install() - upstream = importlib.import_module("docling_serve.__main__") - with patch.object(upstream, "uvicorn", _ServerRunner()): - upstream.app() - - -if __name__ == "__main__": - main() diff --git a/integrations/stackgen_serve/src/stackgen_docling/app.py b/integrations/stackgen_serve/src/stackgen_docling/app.py deleted file mode 100644 index 7cf192f8e5..0000000000 --- a/integrations/stackgen_serve/src/stackgen_docling/app.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Process-local factory used by both direct and spawned Uvicorn servers.""" - -import logging -import os -from typing import TYPE_CHECKING - -from stackgen_docling.install import install - -if TYPE_CHECKING: - from fastapi import FastAPI - - -def create_app() -> "FastAPI": - """Install before upstream imports in every worker, including reload children. - - Parent-process module replacements are not inherited by multiprocessing spawn. - Without this factory, workers silently restore upstream's dense-grid exporter. - """ - install() - from docling_serve.app import create_app as upstream_factory - - application = upstream_factory() - logging.getLogger(__name__).info( - "Record table factory installed pid=%s", os.getpid() - ) - return application diff --git a/integrations/stackgen_serve/src/stackgen_docling/install.py b/integrations/stackgen_serve/src/stackgen_docling/install.py deleted file mode 100644 index 1681d63492..0000000000 --- a/integrations/stackgen_serve/src/stackgen_docling/install.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Install the replacement before Serve imports its conversion/export machinery.""" - -import logging -from importlib.metadata import version - -from docling_core.transforms.serializer import markdown -from docling_core.transforms.serializer.base import BaseTableSerializer -from pydantic import Field - -from stackgen_docling.record_tables import RecordTableSerializer - - -class RecordMarkdownDocSerializer(markdown.MarkdownDocSerializer): - """Keep upstream document serialization; replace its default table strategy.""" - - table_serializer: BaseTableSerializer = Field(default_factory=RecordTableSerializer) - - -def install() -> None: - """Select record tables for Markdown exports, including callback character counts. - - Document.export_to_markdown imports this class at call time, so the original - signature/defaults/filtering stay intact. Install once before starting Serve. - An explicit table_serializer passed by a caller still overrides the default. - """ - expected = { - "docling-core": "2.99.0", - "docling-serve": "1.36.0", - "docling-jobkit": "3.8.1", - } - for distribution, pinned in expected.items(): - installed = version(distribution) - if installed != pinned: - raise RuntimeError( - f"Unsupported {distribution} {installed}; expected {pinned}" - ) - if markdown.MarkdownDocSerializer is RecordMarkdownDocSerializer: - return - markdown.MarkdownDocSerializer = RecordMarkdownDocSerializer - logging.getLogger(__name__).info("Record-oriented table exporter installed") diff --git a/integrations/stackgen_serve/src/stackgen_docling/record_tables.py b/integrations/stackgen_serve/src/stackgen_docling/record_tables.py deleted file mode 100644 index 7ff96ad270..0000000000 --- a/integrations/stackgen_serve/src/stackgen_docling/record_tables.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Render actual table cells without materializing Docling's padded grid.""" - -import json -from typing import Any - -from docling_core.transforms.serializer.base import ( - BaseDocSerializer, - BaseTableSerializer, - SerializationResult, -) -from docling_core.transforms.serializer.common import create_ser_result -from docling_core.transforms.serializer.markdown import ( - MarkdownParams, - _caption_goes_after, - _should_use_legacy_annotations, -) -from docling_core.types.doc import DoclingDocument, RichTableCell, TableCell, TableItem - - -class RecordTableSerializer(BaseTableSerializer): - """Export existing cells by row; absent positions never become empty objects. - - Parsed cells and output remain in memory. No parser/size-budget changes. - Span labels retain merged-cell coordinates without duplicating their values. - """ - - def serialize( - self, - *, - item: TableItem, - doc_serializer: BaseDocSerializer, - doc: DoclingDocument, - **kwargs: Any, - ) -> SerializationResult: - """Preserve upstream captions/options while replacing only table rendering.""" - params = MarkdownParams(**kwargs) - results = [] - caption = doc_serializer.serialize_captions(item=item, **kwargs) - caption_after = _caption_goes_after(item, doc, params, standard_after=False) - if caption.text and not caption_after: - results.append(caption) - if item.self_ref not in doc_serializer.get_excluded_refs(**kwargs): - if _should_use_legacy_annotations(params=params, item=item): - annotation = doc_serializer.serialize_annotations(item=item, **kwargs) - if annotation.text: - results.append(annotation) - parts = [ - f"Table: {item.data.num_rows} rows, {item.data.num_cols} columns. " - "Only actual cells follow.\n" - ] - current_row = None - for cell in sorted( - item.data.table_cells, - key=lambda c: ( - c.start_row_offset_idx, - c.start_col_offset_idx, - ), - ): - if cell.start_row_offset_idx != current_row: - current_row = cell.start_row_offset_idx - parts.append(f"Record {current_row + 1}\n") - if isinstance(cell, RichTableCell): - value = doc_serializer.serialize( - item=cell.ref.resolve(doc=doc), - **{**kwargs, "_nested_in_table": True, "in_table_cell": True}, - ).text - else: - value = cell.text - literal = json.dumps(value, ensure_ascii=False) - if params.escape_html: - for character, escaped in ( - ("<", r"\u003c"), - (">", r"\u003e"), - ("&", r"\u0026"), - ): - literal = literal.replace(character, escaped) - parts.append( - f"- Column {cell.start_col_offset_idx + 1}{self._label(cell)}: {literal}\n" - ) - results.append(create_ser_result(text="".join(parts), span_source=item)) - if caption.text and caption_after: - results.append(caption) - return create_ser_result( - text="\n\n".join(r.text for r in results), span_source=results - ) - - def _label(self, cell: TableCell) -> str: - """Keep header flags and merged ranges attached to their actual source cell.""" - labels = [] - if cell.column_header: - labels.append("column header") - if cell.row_header: - labels.append("row header") - if cell.row_span > 1: - labels.append( - f"rows {cell.start_row_offset_idx + 1}-{cell.end_row_offset_idx}" - ) - if cell.col_span > 1: - labels.append( - f"columns {cell.start_col_offset_idx + 1}-{cell.end_col_offset_idx}" - ) - return f" ({'; '.join(labels)})" if labels else "" diff --git a/integrations/stackgen_serve/test-image.sh b/integrations/stackgen_serve/test-image.sh deleted file mode 100644 index 4876c10019..0000000000 --- a/integrations/stackgen_serve/test-image.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh -set -eu -image=${1:-stackgen-docling:fork-review} -tmp=$(mktemp -d) -trap 'rm -rf "$tmp"' EXIT -python3 - "$tmp/manifest.json" <<'PY' -import hashlib -import json -import sys -import tomllib -from pathlib import Path -project = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8"))["project"] -files = {str(p): hashlib.sha256(p.read_bytes()).hexdigest() for p in Path("docling").rglob("*.py")} -manifest = {"distribution": project["name"], "version": project["version"], "files": files} -Path(sys.argv[1]).write_text(json.dumps(manifest), encoding="utf-8") -PY -# Only tests and a digest manifest are mounted. No adapter or SDK source. -docker run --rm --network=none --memory=3g --cpus=2 --entrypoint /opt/app-root/bin/python \ - -v "$PWD/integrations/stackgen_serve/tests:/tests:ro" -v "$tmp:/evidence:ro" \ - "$image" /tests/verify_install.py /evidence/manifest.json -docker run --rm --network=none --memory=3g --cpus=2 --entrypoint /opt/app-root/bin/python \ - -v "$PWD/integrations/stackgen_serve/tests:/tests:ro" "$image" \ - -m unittest discover -s /tests -p 'test_*.py' -for mode in default workers reload; do - docker run --rm --network=none --memory=3g --cpus=2 --entrypoint /opt/app-root/bin/python \ - -v "$PWD/integrations/stackgen_serve/tests:/tests:ro" "$image" /tests/http_smoke.py "$mode" -done -# Require dependency consistency; never filter pip check output. -docker run --rm --network=none --memory=3g --cpus=2 --entrypoint /opt/app-root/bin/python "$image" -m pip check diff --git a/integrations/stackgen_serve/tests/http_smoke.py b/integrations/stackgen_serve/tests/http_smoke.py deleted file mode 100644 index bd8dd4e40c..0000000000 --- a/integrations/stackgen_serve/tests/http_smoke.py +++ /dev/null @@ -1,268 +0,0 @@ -"""Real HTTP checks for single-process, spawned workers and reload startup.""" - -import argparse -import csv -import hashlib -import io -import json -import os -import signal -import subprocess -import sys -import tempfile -import time -import urllib.error -import urllib.request -from pathlib import Path - -from runtime_checks import RuntimeGuard, serving_pids - - -def wait_ready( - server: subprocess.Popen, log_path: Path, count: int, previous: set[str] -) -> set[str]: - """Require healthy new serving processes so a stale worker cannot mask reload.""" - deadline = time.monotonic() + 180 - while time.monotonic() < deadline: - if server.poll() is not None: - raise RuntimeError("Serve exited before health check") - pids = serving_pids(log_path) - previous - if len(pids) >= count: - try: - with urllib.request.urlopen("http://127.0.0.1:5001/health", timeout=3): - text = log_path.read_text() - if all( - f"Record table factory installed pid={pid}" in text - for pid in pids - ): - return pids - except (OSError, urllib.error.URLError): - pass - time.sleep(1) - raise TimeoutError("Serve processes/factory installation timed out") - - -def multipart_body( - boundary: str, *, filename: str, content_type: str, content: bytes -) -> bytes: - """Build each upload directly so file content cannot be mistaken for headers.""" - return ( - ( - f'--{boundary}\r\nContent-Disposition: form-data; name="to_formats"\r\n\r\nmd\r\n' - f'--{boundary}\r\nContent-Disposition: form-data; name="image_export_mode"\r\n' - "\r\nplaceholder\r\n" - f'--{boundary}\r\nContent-Disposition: form-data; name="files"; ' - f'filename="{filename}"\r\n' - f"Content-Type: {content_type}\r\n\r\n" - ).encode() - + content - + f"\r\n--{boundary}--\r\n".encode() - ) - - -def main() -> None: - """Verify record output in real server children, including a reload restart.""" - parser = argparse.ArgumentParser() - parser.add_argument( - "mode", choices=["default", "workers", "reload"], nargs="?", default="default" - ) - parser.add_argument("--csv", type=Path, help="Use an existing UTF-8 CSV fixture") - parser.add_argument("--requests", type=int, help="Requests per server generation") - parser.add_argument("--timeout", type=int, default=180) - args = parser.parse_args() - if args.requests is not None and args.requests < 1: - parser.error("--requests must be positive") - mode = args.mode - if args.csv is not None: - source_bytes = args.csv.read_bytes() - else: - rows = [ - [f"h{i}" for i in range(1881)], - [], # The backend drops blank lines, but not explicit empty-field rows. - ["alpha", "123", "", "line1\nline2 ü "], - ["", ""], - ] - source = io.StringIO(newline="") - csv.writer(source).writerows(rows) - source_bytes = source.getvalue().encode() - rows = [ - row - for row in csv.reader(io.StringIO(source_bytes.decode("utf-8-sig"), newline="")) - if row - ] - boundary = "docling-record-smoke" - body = multipart_body( - boundary, filename="records.csv", content_type="text/csv", content=source_bytes - ) - env = { - **os.environ, - "UVICORN_WORKERS": "2" if mode == "workers" else "1", - "UVICORN_RELOAD": "true" if mode == "reload" else "false", - "DOCLING_SERVE_LOG_LEVEL": "INFO", - } - # CSV and HTML need no model weights. Apply the same test budget in all modes. - env["DOCLING_SERVE_LOAD_MODELS_AT_BOOT"] = "false" - with tempfile.TemporaryDirectory() as directory: - log_path = Path(directory) / "serve.log" - watched = Path(directory) / "reload_probe.py" - watched.write_text("# initial reload probe\n") - with log_path.open("w+") as log: - guard = RuntimeGuard() - pids: set[str] = set() - server = subprocess.Popen( - [ - sys.executable, - "-m", - "stackgen_docling", - "run", - "--host", - "127.0.0.1", - "--port", - "5001", - ], - cwd=directory, - env=env, - start_new_session=True, - stdout=log, - stderr=subprocess.STDOUT, - ) - try: - expected = 2 if mode == "workers" else 1 - pids = wait_ready(server, log_path, expected, set()) - assert len(pids) == expected, "Unexpected startup worker replacement" - active = guard.capture(pids) - guard.check(log_path, active, pids) - generations = 2 if mode == "reload" else 1 - # New HTTP connections exercise worker dispatch; installation - # markers independently confirm the hook ran in every worker. - requests = ( - args.requests - if args.requests is not None - else (6 if mode == "workers" else 1) - ) - conversion_seconds = [] - for generation in range(generations): - if generation: - guard.check(log_path, active, pids) - watched.write_text("# changed reload probe\n") - new_pids = wait_ready(server, log_path, 1, pids) - assert len(new_pids) == 1, "Unexpected extra reload generation" - pids |= new_pids - active = guard.capture(new_pids) - guard.check(log_path, active, pids) - for _ in range(requests): - assert server.poll() is None, "Server supervisor exited" - guard.check(log_path, active, pids) - request = urllib.request.Request( - "http://127.0.0.1:5001/v1/convert/file", - data=body, - headers={ - "Content-Type": f"multipart/form-data; boundary={boundary}" - }, - ) - started = time.monotonic() - with urllib.request.urlopen( - request, timeout=args.timeout - ) as response: - assert response.status == 200 - markdown = json.load(response)["document"]["md_content"] - conversion_seconds.append(time.monotonic() - started) - values = [ - json.loads(line.split(": ", 1)[1]) - for line in markdown.splitlines() - if line.startswith("- Column ") - ] - assert values == [value for row in rows for value in row] - record_labels = [ - line - for line in markdown.splitlines() - if line.startswith("Record ") - ] - assert record_labels == [ - f"Record {i + 1}" for i in range(len(rows)) - ] - assert "## Record " not in markdown - assert not any( - not line.strip() for line in markdown.splitlines() - ) - guard.check(log_path, active, pids) - html_body = multipart_body( - boundary, - filename="records.html", - content_type="text/html", - content=( - b"

Native HTML

" - b"
bluegreen
" - ), - ) - request = urllib.request.Request( - "http://127.0.0.1:5001/v1/convert/file", - data=html_body, - headers={ - "Content-Type": f"multipart/form-data; boundary={boundary}" - }, - ) - with urllib.request.urlopen(request, timeout=180) as response: - html_markdown = json.load(response)["document"]["md_content"] - assert all( - token in html_markdown - for token in ("Native HTML", "blue", "green", "Record 1") - ) - with urllib.request.urlopen( - "http://127.0.0.1:5001/health", timeout=5 - ) as response: - assert response.status == 200 - assert server.poll() is None, "Server supervisor exited" - guard.check(log_path, active, pids) - report = { - "status": "passed", - "mode": mode, - "verified_fields": len(values), - "source_rows": len(rows), - "source_bytes": len(source_bytes), - "source_sha256": hashlib.sha256(source_bytes).hexdigest(), - "conversion_seconds": conversion_seconds, - "output_bytes": len(markdown.encode()), - "serving_pids": sorted(pids), - "active_pids": sorted(active), - "requests": requests * generations, - "health": 200, - "oom_events_before": guard.baseline, - } - except BaseException: - log.flush() - print(log_path.read_text(), file=sys.stderr) - raise - finally: - try: - os.killpg(server.pid, signal.SIGTERM) - except ProcessLookupError: - pass - try: - server.wait(timeout=20) - except subprocess.TimeoutExpired: - try: - os.killpg(server.pid, signal.SIGKILL) - except ProcessLookupError: - pass - server.wait(timeout=5) - # Intentional shutdown no longer requires live workers, but - # cannot hide a last-moment OOM or unplanned replacement. - try: - guard.check(log_path, {}, pids or serving_pids(log_path)) - except BaseException: - log.flush() - print(log_path.read_text(), file=sys.stderr) - raise - report["oom_events_after"] = guard.counters() - peak_path = Path("/sys/fs/cgroup/memory.peak") - report["memory_peak_bytes"] = ( - int(peak_path.read_text(encoding="utf-8")) - if peak_path.exists() - else None - ) - print(json.dumps(report)) - - -if __name__ == "__main__": - main() diff --git a/integrations/stackgen_serve/tests/runtime_checks.py b/integrations/stackgen_serve/tests/runtime_checks.py deleted file mode 100644 index e3fc5128ef..0000000000 --- a/integrations/stackgen_serve/tests/runtime_checks.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Linux-container checks that prevent healthy HTTP from hiding worker loss.""" - -import re -from pathlib import Path - - -def serving_pids(log_path: Path) -> set[str]: - """Read historical serving PIDs; liveness must be checked independently.""" - return set(re.findall(r"Started server process \[(\d+)\]", log_path.read_text())) - - -class RuntimeGuard: - """Require observable OOM counters and stable live workers throughout a smoke run.""" - - def __init__( - self, cgroup: Path = Path("/sys/fs/cgroup"), proc: Path = Path("/proc") - ) -> None: - self.proc = proc - self.counter_path = cgroup / "memory.events" - if not self.counter_path.exists(): - self.counter_path = cgroup / "memory/memory.oom_control" - self.baseline = self.counters() - - def counters(self) -> dict[str, int]: - """Read v2 events or v1 kill counters, failing closed if unavailable.""" - try: - counters = { - key: int(value) - for key, value in ( - line.split() for line in self.counter_path.read_text().splitlines() - ) - } - if "oom_kill" not in counters: - raise ValueError("oom_kill counter missing") - return { - key: value - for key, value in counters.items() - if key in {"oom", "oom_kill", "oom_group_kill"} - } - except (OSError, ValueError) as error: - raise RuntimeError( - f"Cannot verify OOM counters at {self.counter_path}" - ) from error - - def identity(self, pid: str) -> str: - """Identify a live, non-zombie process by PID and Linux starttime.""" - try: - fields = (self.proc / pid / "stat").read_text().rsplit(")", 1)[1].split() - if fields[0] in {"Z", "X", "x"}: - raise ValueError("dead/zombie process") - return fields[19] - except (OSError, ValueError, IndexError) as error: - raise RuntimeError(f"Serving worker {pid} is not alive") from error - - def capture(self, pids: set[str]) -> dict[str, str]: - """Capture the expected generation so PID reuse cannot mask worker loss.""" - return {pid: self.identity(pid) for pid in pids} - - def check(self, log_path: Path, active: dict[str, str], history: set[str]) -> None: - """Reject OOM events, unplanned generations and lost/reused active workers.""" - counters = self.counters() - if counters.keys() != self.baseline.keys() or any( - value != self.baseline[key] for key, value in counters.items() - ): - raise RuntimeError( - f"OOM counters changed: before={self.baseline}, after={counters}" - ) - if serving_pids(log_path) != history: - raise RuntimeError("Unexpected serving-worker restart") - for pid, starttime in active.items(): - if self.identity(pid) != starttime: - raise RuntimeError(f"Serving worker {pid} was replaced/reused") diff --git a/integrations/stackgen_serve/tests/test_http_smoke.py b/integrations/stackgen_serve/tests/test_http_smoke.py deleted file mode 100644 index 6fe1efa620..0000000000 --- a/integrations/stackgen_serve/tests/test_http_smoke.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Request-building regressions for the HTTP smoke checks.""" - -import unittest -from email import policy -from email.parser import BytesParser - -from http_smoke import multipart_body - - -class MultipartBodyTests(unittest.TestCase): - def test_payload_matching_form_settings_keeps_file_part(self) -> None: - """Keep file headers intact when CSV values also occur in request settings.""" - boundary = "docling-record-smoke" - html = b"

Native HTML

" - for filename, content_type, content in ( - ("records.csv", "text/csv", b"md"), - ("records.csv", "text/csv", b"md\r\n"), - ("records.csv", "text/csv", b"placeholder"), - ("records.html", "text/html", html), - ): - with self.subTest(filename=filename, content=content): - body = multipart_body( - boundary, - filename=filename, - content_type=content_type, - content=content, - ) - message = BytesParser(policy=policy.default).parsebytes( - ( - f"Content-Type: multipart/form-data; boundary={boundary}\r\n" - "MIME-Version: 1.0\r\n\r\n" - ).encode() - + body - ) - parts = list(message.iter_parts()) - fields = { - part.get_param("name", header="content-disposition"): part - for part in parts - } - self.assertEqual(len(parts), 3) - self.assertEqual( - set(fields), {"to_formats", "image_export_mode", "files"} - ) - self.assertEqual(fields["to_formats"].get_payload(decode=True), b"md") - self.assertEqual( - fields["image_export_mode"].get_payload(decode=True), - b"placeholder", - ) - file = fields["files"] - self.assertEqual(file.get_filename(), filename) - self.assertEqual(file.get_content_type(), content_type) - self.assertEqual(file.get_payload(decode=True), content) diff --git a/integrations/stackgen_serve/tests/test_launch.py b/integrations/stackgen_serve/tests/test_launch.py deleted file mode 100644 index 9cfc45586b..0000000000 --- a/integrations/stackgen_serve/tests/test_launch.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Protect process-local installation and upstream CLI configuration forwarding.""" - -import importlib -import unittest -from types import SimpleNamespace -from unittest.mock import patch - - -class LaunchTests(unittest.TestCase): - """Require an importable factory rather than parent-only state inheritance.""" - - def test_factory_installs_before_importing_upstream_application(self) -> None: - factory = importlib.import_module("stackgen_docling.app") - events = [] - application = object() - - def upstream_factory() -> object: - events.append("create") - return application - - original_import = __import__ - - def check_import(name: str, *args: object, **kwargs: object) -> object: - if name == "docling_serve.app": - self.assertEqual(events, ["install"]) - events.append("import") - return original_import(name, *args, **kwargs) - - with ( - patch.object( - factory, "install", side_effect=lambda: events.append("install") - ), - patch.dict( - "sys.modules", - {"docling_serve.app": SimpleNamespace(create_app=upstream_factory)}, - ), - patch("builtins.__import__", side_effect=check_import), - ): - self.assertIs(factory.create_app(), application) - self.assertEqual(events, ["install", "import", "create"]) - - def test_cli_redirects_only_factory_and_preserves_worker_reload_ssl_options( - self, - ) -> None: - launcher = importlib.import_module("stackgen_docling.__main__") - options = { - "app": "docling_serve.app:create_app", - "factory": True, - "workers": 2, - "reload": False, - "host": "127.0.0.1", - "port": 5123, - "root_path": "/convert", - "ssl_certfile": "certificate.pem", - "log_config": None, - } - upstream = SimpleNamespace(uvicorn=SimpleNamespace()) - - def cli() -> None: - upstream.uvicorn.run(**options) - - upstream.app = cli - original_runner = upstream.uvicorn - with ( - patch.object(launcher, "install"), - patch.object(launcher.uvicorn, "run") as run, - patch.dict("sys.modules", {"docling_serve.__main__": upstream}), - ): - launcher.main() - run.assert_called_once_with( - **{**options, "app": "stackgen_docling.app:create_app"} - ) - self.assertIs(upstream.uvicorn, original_runner) - - -if __name__ == "__main__": - unittest.main() diff --git a/integrations/stackgen_serve/tests/test_record_tables.py b/integrations/stackgen_serve/tests/test_record_tables.py deleted file mode 100644 index 84a06e2ff9..0000000000 --- a/integrations/stackgen_serve/tests/test_record_tables.py +++ /dev/null @@ -1,360 +0,0 @@ -"""Behavioral regression tests against the runtime-pinned Docling library.""" - -import json -import tempfile -import unittest -from pathlib import Path -from unittest.mock import patch - -from docling_core.transforms.serializer.markdown import ( - MarkdownDocSerializer as UpstreamDocSerializer, -) -from docling_core.types.doc import ( - DoclingDocument, - RefItem, - RichTableCell, - TableCell, - TableData, -) -from stackgen_docling.install import install - - -class RecordTableTests(unittest.TestCase): - """Protect actual cells and export options without allowing a dense grid.""" - - @classmethod - def setUpClass(cls) -> None: - install() - - def setUp(self) -> None: - self.grid = patch.object(TableData, "grid", property(self.forbid_grid)) - self.grid.start() - self.addCleanup(self.grid.stop) - - @staticmethod - def forbid_grid(_data: TableData) -> None: - raise AssertionError("Export must never construct TableData.grid") - - def test_actual_coordinates_empty_values_and_special_characters(self) -> None: - doc = DoclingDocument(name="actual cells") - cells = [ - TableCell( - text=text, - start_row_offset_idx=row, - end_row_offset_idx=row + 1, - start_col_offset_idx=col, - end_col_offset_idx=col + 1, - ) - for row, col, text in [ - (2, 7, 'line1\n"ü" |