diff --git a/apps/sim/lib/file-parsers/pdf-layout.test.ts b/apps/sim/lib/file-parsers/pdf-layout.test.ts new file mode 100644 index 00000000000..74c336dd5a3 --- /dev/null +++ b/apps/sim/lib/file-parsers/pdf-layout.test.ts @@ -0,0 +1,374 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { + type BufferedItem, + findInterleavedBands, + MAX_PDF_LAYOUT_ITEMS, + PdfPageCollector, +} from '@/lib/file-parsers/pdf-layout' +import { joinLines, type PdfItemGeometry } from '@/lib/file-parsers/pdf-lines' + +const HEIGHT = 8 +const CHAR_WIDTH = 4.5 + +function placed(str: string, x: number, y: number, height = HEIGHT): BufferedItem { + const geometry: PdfItemGeometry = { x, y, width: str.length * CHAR_WIDTH, height } + return { str, geometry, hasEOL: false } +} + +function readPage(items: readonly BufferedItem[]): string { + const collector = new PdfPageCollector() + for (const item of items) collector.add(item.str, item.geometry, item.hasEOL) + return joinLines(collector.finish()) +} + +const WEEKDAYS = ['Su', 'Mo', 'Tu', 'We', 'Th', 'Fr', 'Sa'] as const +const CELL = 16 +const ROW_PITCH = 12 + +interface Month { + name: string + firstWeekday: number + days: number + x: number +} + +/** A month's rows as text, the way a reader expects them. */ +function monthRows(month: Month): string[] { + const rows = [month.name, WEEKDAYS.join(' ')] + let week: string[] = [] + for (let day = 1; day <= month.days; day++) { + week.push(String(day)) + if ((month.firstWeekday + day) % 7 === 0 || day === month.days) { + rows.push(week.join(' ')) + week = [] + } + } + return rows +} + +/** Positioned items for one visual row of a month: its title, weekday header, or a week. */ +function monthRowItems(month: Month, row: number, y: number): BufferedItem[] { + if (row === 0) return [placed(month.name, month.x, y)] + if (row === 1) return WEEKDAYS.map((day, i) => placed(day, month.x + i * CELL, y)) + const items: BufferedItem[] = [] + for (let day = 1; day <= month.days; day++) { + const slot = month.firstWeekday + day - 1 + if (Math.floor(slot / 7) + 2 === row) { + items.push(placed(String(day), month.x + (slot % 7) * CELL, y)) + } + } + return items +} + +/** + * Three months side by side, drawn the way calendar producers draw them: one + * visual row across every month before the next row. + */ +function quarterCalendar(): { items: BufferedItem[]; months: Month[] } { + const months: Month[] = [ + { name: 'January 2026', firstWeekday: 4, days: 31, x: 40 }, + { name: 'February 2026', firstWeekday: 0, days: 28, x: 188 }, + { name: 'March 2026', firstWeekday: 0, days: 31, x: 336 }, + ] + const items: BufferedItem[] = [placed('2026 Calendar', 200, 740, 14)] + for (let row = 0; row < 8; row++) { + for (const month of months) items.push(...monthRowItems(month, row, 700 - row * ROW_PITCH)) + } + return { items, months } +} + +describe('PdfPageCollector', () => { + it('reads side-by-side calendar months one month at a time', () => { + const { items, months } = quarterCalendar() + + const text = readPage(items) + + const expected = ['2026 Calendar', ...months.map((month) => monthRows(month).join('\n'))] + expect(text).toBe(expected.join('\n\n')) + }) + + it('leaves calendar months the stream already draws one at a time', () => { + const { items } = quarterCalendar() + const monthByMonth = [...items].sort((a, b) => (a.geometry?.x ?? 0) - (b.geometry?.x ?? 0)) + + expect(findInterleavedBands(monthByMonth)).toEqual([]) + }) + + it('keeps a two-column table of wrapped prose cells row by row', () => { + const left = [ + 'Every request is reviewed on a fixed weekly', + 'schedule set by the team that owns the queue.', + 'Each review closes the request or sends it back.', + ] + const right = [ + 'Office hours open two days after each release', + 'and close two weeks before the next one ships.', + 'Freeze periods apply to every team member.', + ] + const items = left.flatMap((line, i) => [ + placed(line, 40, 700 - i * 12), + placed(right[i], 300, 700 - i * 12), + ]) + + expect(findInterleavedBands(items)).toEqual([]) + }) + + it('keeps a table with evenly spaced columns row by row', () => { + const rows = [ + ['Name', 'Opened', 'Closed'], + ['Ana Park', '1200', '300'], + ['Ben Ortiz', '800', '200'], + ] + const items = rows.flatMap((cells, i) => + cells.map((cell, column) => placed(cell, 40 + column * 120, 700 - i * 12)) + ) + + expect(findInterleavedBands(items)).toEqual([]) + expect(readPage(items)).toBe('Name Opened Closed\nAna Park 1200 300\nBen Ortiz 800 200') + }) + + it('keeps a label and value list row by row across a wide gutter', () => { + const rows = [ + ['Employee', 'Ana Park'], + ['Start date', '2024-03-01'], + ['Location', 'Springfield'], + ] + const items = rows.flatMap(([label, value], i) => [ + placed(label, 40, 700 - i * 12), + placed(value, 400, 700 - i * 12), + ]) + + expect(findInterleavedBands(items)).toEqual([]) + }) + + it('keeps a table whose wrapped cells leave its label column sparse', () => { + const items = [ + placed('Role', 40, 700), + placed('Responsibility', 200, 700), + placed('Author', 40, 688), + placed('Writes the change and opens a request for review with a', 200, 688), + placed('short summary of the risk involved.', 200, 676), + placed('Owner', 40, 664), + placed('Owns the document and reviews it once a year, then', 200, 664), + placed('approves every exception in writing.', 200, 652), + ] + + expect(findInterleavedBands(items)).toEqual([]) + }) + + it('leaves side-by-side blocks the stream already draws one at a time', () => { + const address = ['Jane Doe', 'Example Co', '1 Main Street', 'Phone: 555 0100'] + const items = [ + placed('TO:', 40, 700), + ...address.map((line, i) => placed(line, 40, 688 - i * 12)), + placed('SHIP TO:', 300, 700), + ...address.map((line, i) => placed(line, 300, 688 - i * 12)), + ] + + expect(findInterleavedBands(items)).toEqual([]) + expect(readPage(items)).toBe(`TO:\n${address.join('\n')}\n\nSHIP TO:\n${address.join('\n')}`) + }) + + it('reads a sparse legend column as its own block', () => { + const { items, months } = quarterCalendar() + const legend = ['Holiday', 'Office Closure', 'Deadline'] + items.push(...legend.map((label, i) => placed(label, 500, 688 - i * ROW_PITCH))) + + const text = readPage(items) + + const expected = [ + '2026 Calendar', + ...months.map((month) => monthRows(month).join('\n')), + legend.join('\n'), + ] + expect(text).toBe(expected.join('\n\n')) + }) + + it('keeps a table whose column groups differ in shape row by row', () => { + const rows = [ + ['QUANTITY', 'DESCRIPTION', 'UNIT PRICE', 'TOTAL'], + ['10', 'Standard widget, blue', '12.45', '124.50'], + ['25', 'Standard widget, red', '8.10', '202.50'], + ] + const columns = [40, 100, 360, 440] + const items = rows.flatMap((cells, i) => + cells.map((cell, column) => placed(cell, columns[column], 700 - i * 12)) + ) + + expect(findInterleavedBands(items)).toEqual([]) + }) + + it('matches a repeated header whose cells hold several words', () => { + const boxes = [ + { + x: 40, + rows: [ + ['Cash', 'Payables'], + ['Receivables', 'Accrued wages'], + ], + }, + { + x: 400, + rows: [ + ['Deposits', 'Loans'], + ['Securities', 'Borrowings'], + ], + }, + ] + const header = ['Current Assets', 'Current Liabilities'] + const items: BufferedItem[] = [] + for (let i = 0; i < 3; i++) { + for (const box of boxes) { + const cells = i === 0 ? header : box.rows[i - 1] + cells.forEach((cell, c) => items.push(placed(cell, box.x + c * 110, 700 - i * 12))) + } + } + + expect(findInterleavedBands(items)[0]?.blocks).toHaveLength(2) + }) + + it('separates compact small-type grid rows on a page of larger body text', () => { + const { items, months } = quarterCalendar() + const compact = items.map((item) => + item.geometry ? placed(item.str, item.geometry.x, 400 + (item.geometry.y - 700) / 3, 3) : item + ) + const body = Array.from({ length: 200 }, (_, i) => + placed(`body text line ${i}`, 40, 300 - i * 14, 12) + ) + + const bands = findInterleavedBands([...compact, ...body]) + + expect(bands).toHaveLength(1) + expect(bands[0].blocks.map((block) => block.length)).toEqual( + months.map((month) => monthRows(month).length) + ) + }) + + it('keeps a grid of values without repeated headers row by row', () => { + const words = [ + ['w0 = 603deb10', 'w1 = 15ca71be', 'w2 = 2b73aef0', 'w3 = 857d7781'], + ['w4 = 1f352c07', 'w5 = 3b6108d7', 'w6 = 2d9810a3', 'w7 = 0914dff4'], + ] + const items = words.flatMap((row, i) => + row.flatMap((word, column) => { + const [name, equals, value] = word.split(' ') + const x = 40 + column * 130 + return [ + placed(name, x, 700 - i * 12), + placed(equals, x + 16, 700 - i * 12), + placed(value, x + 28, 700 - i * 12), + ] + }) + ) + + expect(findInterleavedBands(items)).toEqual([]) + }) + + it('reads side-by-side account boxes that repeat a header one box at a time', () => { + const boxes = [ + { x: 40, rows: ['Business', 'Assets Liabilities', 'Buildings Loans'] }, + { x: 260, rows: ['Bank', 'Assets Liabilities', 'Securities Deposits'] }, + { x: 480, rows: ['Household', 'Assets Liabilities', 'Deposits Loans'] }, + ] + const items: BufferedItem[] = [] + for (let row = 0; row < 3; row++) { + for (const box of boxes) { + const words = box.rows[row].split(' ') + words.forEach((word, i) => items.push(placed(word, box.x + i * 50, 700 - row * 12))) + } + } + + expect(readPage(items)).toBe(boxes.map((box) => box.rows.join('\n')).join('\n\n')) + }) + + it('still splits calendar months when dense text elsewhere lowers the page pitch', () => { + const { items } = quarterCalendar() + for (let i = 0; i < 40; i++) { + items.push( + placed(`Footnote line ${i} with enough words to read as running text`, 40, 500 - i * 5, 3) + ) + } + + const bands = findInterleavedBands(items) + + expect(bands).toHaveLength(1) + expect(bands[0].blocks).toHaveLength(3) + }) + + it('matches a weekday header drawn as one text item', () => { + const { items } = quarterCalendar() + const merged: BufferedItem[] = [] + for (const item of items) { + if (!['Su', 'Mo', 'Tu', 'We', 'Th', 'Fr', 'Sa'].includes(item.str)) merged.push(item) + else if (item.str === 'Su' && item.geometry) { + merged.push(placed(WEEKDAYS.join(' '), item.geometry.x, item.geometry.y)) + } + } + + expect(findInterleavedBands(merged)[0]?.blocks).toHaveLength(3) + }) + + it('keeps column groups under a repeated one-item label row by row', () => { + const items: BufferedItem[] = [] + const rows = [ + ['Total Amount Due', 'Total Amount Due'], + ['100 12', '300 34'], + ['200 56', '400 78'], + ] + rows.forEach((groups, i) => { + groups.forEach((group, g) => { + const x = 40 + g * 260 + if (i === 0) items.push(placed(group, x, 700)) + else + group.split(' ').forEach((cell, c) => items.push(placed(cell, x + c * 60, 700 - i * 12))) + }) + }) + + expect(findInterleavedBands(items)).toEqual([]) + }) + + it('keeps column groups that share only a single-cell label row by row', () => { + const items: BufferedItem[] = [] + const rows = [ + ['Amount', 'Amount'], + ['100 12', '300 34'], + ['200 56', '400 78'], + ['500 90', '600 11'], + ] + rows.forEach((groups, i) => { + groups.forEach((group, g) => { + group + .split(' ') + .forEach((cell, c) => items.push(placed(cell, 40 + g * 260 + c * 60, 700 - i * 12))) + }) + }) + + expect(findInterleavedBands(items)).toEqual([]) + }) + + it('keeps stream order when any item has no usable geometry', () => { + const { items } = quarterCalendar() + items.push({ str: 'rotated caption', geometry: undefined, hasEOL: true }) + + expect(findInterleavedBands(items)).toEqual([]) + }) + + it('streams in pdf.js order once a page exceeds the layout item cap', () => { + const collector = new PdfPageCollector() + for (let i = 0; i <= MAX_PDF_LAYOUT_ITEMS; i++) { + collector.add( + 'x', + { x: (i % 2) * 300, y: 700 - Math.floor(i / 2), width: 5, height: 1 }, + false + ) + } + + const lines = collector.finish() + + expect(lines.some((line) => line.blockStart)).toBe(false) + expect(lines.length).toBeGreaterThan(0) + }) +}) diff --git a/apps/sim/lib/file-parsers/pdf-layout.ts b/apps/sim/lib/file-parsers/pdf-layout.ts new file mode 100644 index 00000000000..2b35beb2636 --- /dev/null +++ b/apps/sim/lib/file-parsers/pdf-layout.ts @@ -0,0 +1,483 @@ +/** + * Reads side-by-side blocks one at a time when pdf.js streams them interleaved. + * + * pdf.js streams text in content-stream (draw) order, which is the right + * reading order for almost every page. It fails when a page places independent + * grids next to each other — calendar months, account boxes — and the + * producer draws one row across every block before the next: the text comes + * out as `Su Mo … Sa Su Mo … Sa` then `1 2 3 1 2 3 4 …`, and no cell can be tied + * back to its block. This module finds bands of rows that a wide vertical + * gutter divides into grids cut from one template (same header row, same width + * and inner columns), and only when the stream crosses those gutters row after row does it + * emit each block top to bottom instead. Tables, prose, and everything else + * keep stream order. + */ + +import { type PdfItemGeometry, type PdfLine, PdfLineBuilder } from '@/lib/file-parsers/pdf-lines' + +/** + * Ceiling on buffered items per page. Layout analysis needs a whole page in + * hand; past this the page streams straight into the line builder in pdf.js + * order, so a pathological page costs no more than it did before. + */ +export const MAX_PDF_LAYOUT_ITEMS = 20_000 + +/** Ceiling on distinct x-intervals tracked per band; past it the page keeps stream order. */ +const MAX_BAND_INTERVALS = 256 + +/** Items whose baselines differ by at most this fraction of their height share a row. */ +const ROW_TOLERANCE_RATIO = 0.5 + +/** A gutter must be at least this many body heights wide. */ +const GUTTER_MIN_RATIO = 1.5 + +/** A gutter must be at least this many times wider than any column gap inside the blocks it divides. */ +const GUTTER_DOMINANCE_RATIO = 2 + +/** A vertical gap inside a block at least this many body heights wide makes it a grid. */ +const GRID_GAP_RATIO = 0.5 + +/** A block needs at least this many rows to stand on its own. */ +const MIN_BLOCK_ROWS = 2 + +/** + * A plain column — a legend or sidebar — may stand apart from the blocks when + * it fills at most this share of the band's rows. + */ +const SPARSE_BLOCK_ROW_FRACTION = 0.5 + +/** Blocks from one template differ in width by at most this fraction. */ +const CONGRUENT_WIDTH_RATIO = 0.1 + +/** A block's shared header row must sit within this many rows of its top. */ +const HEADER_SEARCH_ROWS = 3 + +const LETTER = /\p{L}/u + +/** A vertical step beyond this many row pitches ends a band, so stacked rows of blocks read in order. */ +const BAND_BREAK_PITCHES = 2 + +export interface BufferedItem { + str: string + geometry: PdfItemGeometry | undefined + hasEOL: boolean +} + +interface LayoutItem { + /** Position in the page's stream. */ + index: number + str: string + geometry: PdfItemGeometry +} + +interface LayoutRow { + y: number + items: LayoutItem[] +} + +/** A closed x-interval `[start, end]` in PDF user space. */ +type Interval = [number, number] + +/** A band whose blocks are read one at a time, spliced in where its stream span began. */ +export interface ReorderedBand { + /** Blocks left to right, each a list of rows top to bottom. */ + blocks: LayoutRow[][] + /** Stream index of the band's last item. */ + last: number +} + +/** + * Collects one page's text items and turns them into lines. Buffers up to + * `MAX_PDF_LAYOUT_ITEMS` so interleaved blocks can be detected; past that it + * replays the buffer and streams the rest in pdf.js order. + */ +export class PdfPageCollector { + private readonly builder = new PdfLineBuilder() + private buffer: BufferedItem[] | undefined = [] + + add(str: string, geometry: PdfItemGeometry | undefined, hasEOL: boolean): void { + if (this.buffer) { + this.buffer.push({ str, geometry, hasEOL }) + if (this.buffer.length <= MAX_PDF_LAYOUT_ITEMS) return + const buffered = this.buffer + this.buffer = undefined + for (const entry of buffered) this.stream(entry) + return + } + this.stream({ str, geometry, hasEOL }) + } + + finish(): PdfLine[] { + const buffered = this.buffer + this.buffer = undefined + if (buffered) this.replay(buffered, findInterleavedBands(buffered)) + return this.builder.finish() + } + + /** + * Streams the page, emitting each reordered band block by block at the point + * its first item was drawn. Whitespace and line-break items inside a band's + * stream span belong to the order being replaced, so they are skipped. + */ + private replay(entries: readonly BufferedItem[], bands: readonly ReorderedBand[]): void { + const bandAt = new Map() + for (const band of bands) { + for (const block of band.blocks) { + for (const row of block) for (const item of row.items) bandAt.set(item.index, band) + } + } + const emitted = new Set() + let open: ReorderedBand | undefined + for (const [index, entry] of entries.entries()) { + const band = bandAt.get(index) + if (band) { + if (!emitted.has(band)) { + emitted.add(band) + for (const block of band.blocks) this.emitBlock(block) + this.builder.startBlock() + } + open = band + continue + } + if (open && index > open.last) open = undefined + if (open && entry.str.trim().length === 0) continue + this.stream(entry) + } + } + + /** Feeds one item in stream order, deriving separators from geometry and pdf.js's `hasEOL`. */ + private stream({ str, geometry, hasEOL }: BufferedItem): void { + const separator = str.length > 0 ? this.builder.separatorBefore(str, geometry) : '' + if (separator === '\n') this.builder.endLine() + else if (separator.length > 0) this.builder.append(separator) + this.builder.append(str, geometry) + if (hasEOL) this.builder.endLine() + } + + /** Emits a block's rows as lines; the block's first line starts a new paragraph. */ + private emitBlock(rows: readonly LayoutRow[]): void { + this.builder.startBlock() + for (const row of rows) { + for (const item of row.items) { + /** Items of one row never split it: an overlapping item only needs a space. */ + if (this.builder.separatorBefore(item.str, item.geometry).length > 0) { + this.builder.append(' ') + } + this.builder.append(item.str, item.geometry) + } + this.builder.endLine() + } + } +} + +/** + * Bands of the page whose side-by-side blocks the stream interleaves, in page + * order. Empty when geometry is missing or unusable, or when every band is + * already streamed block by block — the caller then keeps pdf.js's order. + */ +export function findInterleavedBands(entries: readonly BufferedItem[]): ReorderedBand[] { + const items: LayoutItem[] = [] + for (const [index, entry] of entries.entries()) { + if (entry.str.trim().length === 0) continue + /** One unplaceable item means the page's geometry cannot be trusted as a whole. */ + if (!entry.geometry) return [] + items.push({ index, str: entry.str, geometry: entry.geometry }) + } + if (items.length === 0) return [] + + const bodyHeight = medianHeight(items) + if (bodyHeight <= 0) return [] + + const rows = groupRows(items, bodyHeight) + const minGutter = GUTTER_MIN_RATIO * bodyHeight + const maxStep = + BAND_BREAK_PITCHES * + rowPitch(rows.filter((row) => widestGap(rowIntervals(row, minGutter)) >= minGutter)) + const bands: ReorderedBand[] = [] + let start = 0 + while (start < rows.length) { + const band = growBand(rows, start, bodyHeight, maxStep) + if (band === undefined) return [] + const blocks = band.rows.length >= MIN_BLOCK_ROWS ? splitBand(band, bodyHeight) : undefined + if (blocks && isInterleaved(blocks)) bands.push(spanOf(blocks)) + start += band.rows.length + } + return bands +} + +interface Band { + rows: LayoutRow[] + /** Coalesced x-extent of the band's items, left to right. */ + intervals: Interval[] +} + +/** + * Extends a band from `start` while the rows' combined x-extent keeps at least + * one gutter-wide gap and no vertical step exceeds `maxStep`. Undefined when + * the band's interval count exceeds the cap. + */ +function growBand( + rows: readonly LayoutRow[], + start: number, + bodyHeight: number, + maxStep: number +): Band | undefined { + const minGutter = GUTTER_MIN_RATIO * bodyHeight + let intervals = rowIntervals(rows[start], minGutter) + let end = start + 1 + if (widestGap(intervals) < minGutter) return { rows: rows.slice(start, end), intervals } + while (end < rows.length) { + if (maxStep > 0 && rows[end - 1].y - rows[end].y > maxStep) break + const merged = mergeIntervals(intervals, rowIntervals(rows[end], minGutter), minGutter) + if (merged.length > MAX_BAND_INTERVALS) return undefined + if (widestGap(merged) < minGutter) break + intervals = merged + end++ + } + return { rows: rows.slice(start, end), intervals } +} + +/** + * Splits a band at its dominant gutters, or undefined unless the pieces are + * independent grids cut from one template: each repeats the same header row and + * has the same width and inner columns. A column of plain cells may stand + * apart only when it is sparse, like a legend; one that spans the band labels + * every row, so the band is a table and is left alone. + */ +function splitBand(band: Band, bodyHeight: number): LayoutRow[][] | undefined { + const minGutter = GUTTER_MIN_RATIO * bodyHeight + const { intervals } = band + const widest = widestGap(intervals) + if (widest < minGutter) return undefined + + /** Blocks are the runs of intervals between gaps at least half the widest one. */ + const threshold = Math.max(minGutter, widest / GUTTER_DOMINANCE_RATIO) + const groups: Interval[][] = [[intervals[0]]] + for (let i = 1; i < intervals.length; i++) { + if (intervals[i][0] - intervals[i - 1][1] >= threshold) groups.push([]) + groups[groups.length - 1].push(intervals[i]) + } + if (groups.length < 2) return undefined + + const gutterStarts = groups.slice(1).map((_, i) => groups[i][groups[i].length - 1][1]) + const narrowestGutter = Math.min( + ...groups.slice(1).map((group, i) => group[0][0] - gutterStarts[i]) + ) + + const blocks: LayoutRow[][] = groups.map(() => []) + for (const row of band.rows) { + const parts: LayoutItem[][] = groups.map(() => []) + for (const item of row.items) parts[blockIndex(item.geometry.x, gutterStarts)].push(item) + parts.forEach((items, i) => { + if (items.length > 0) blocks[i].push({ y: row.y, items }) + }) + } + + const minGridGap = GRID_GAP_RATIO * bodyHeight + const gridBlocks: LayoutRow[][] = [] + const grids: Interval[][] = [] + for (const rows of blocks) { + const grid = gridSignature(rows, minGridGap) + if (!grid) { + if (rows.length > SPARSE_BLOCK_ROW_FRACTION * band.rows.length) return undefined + continue + } + if (narrowestGutter < GUTTER_DOMINANCE_RATIO * widestGap(grid)) return undefined + gridBlocks.push(rows) + grids.push(grid) + } + if (gridBlocks.length < 2 || !sharesHeader(gridBlocks, grids, minGridGap)) return undefined + if (!grids.every((grid) => isCongruent(grids[0], grid, bodyHeight))) return undefined + return blocks +} + +/** + * A block's inner columns, or undefined when it is not a grid. Only rows with + * two or more cells count, so a title spanning the first columns — + * `January 2026` against `May 2026` — does not make two months look different. + */ +function gridSignature(rows: readonly LayoutRow[], minGridGap: number): Interval[] | undefined { + const gridRows = rows.filter((row) => rowIntervals(row, minGridGap).length >= 2) + if (gridRows.length < MIN_BLOCK_ROWS) return undefined + const intervals = blockIntervals(gridRows, minGridGap) + return widestGap(intervals) >= minGridGap ? intervals : undefined +} + +/** Same width and the same inner column starts, within tolerance. */ +function isCongruent( + reference: readonly Interval[], + grid: readonly Interval[], + bodyHeight: number +): boolean { + const referenceWidth = extentWidth(reference) + const tolerance = Math.max(CONGRUENT_WIDTH_RATIO * referenceWidth, bodyHeight) + if (Math.abs(extentWidth(grid) - referenceWidth) > tolerance) return false + if (grid.length !== reference.length) return false + const origin = grid[0][0] + const referenceOrigin = reference[0][0] + return grid.every( + (interval, i) => + Math.abs(interval[0] - origin - (reference[i][0] - referenceOrigin)) <= bodyHeight + ) +} + +function extentWidth(intervals: readonly Interval[]): number { + return intervals[intervals.length - 1][1] - intervals[0][0] +} + +/** + * Whether every grid repeats one header row near its top that names each of + * its columns — one cell per column (`Current Assets | Current Liabilities`) + * or, when drawn as a single text item, one word per column (`Su Mo Tu We Th + * Fr Sa`). Independent blocks cut from one template carry it; a table's column + * groups do not, and a lone or repeated group label never counts. + */ +function sharesHeader( + blocks: readonly LayoutRow[][], + grids: readonly Interval[][], + minGridGap: number +): boolean { + const headers = blocks.map((rows, i) => { + const columns = grids[i].length + const names = rows.slice(0, HEADER_SEARCH_ROWS).filter((row) => { + const text = rowText(row) + if (!LETTER.test(text)) return false + const cells = rowIntervals(row, minGridGap).length + return cells === columns || (cells === 1 && text.split(/\s+/).length === columns) + }) + return new Set(names.map(rowText)) + }) + const [first, ...rest] = headers + for (const text of first) if (rest.every((header) => header.has(text))) return true + return false +} + +function rowText(row: LayoutRow): string { + return row.items.map((item) => item.str.trim()).join(' ') +} + +/** + * Whether the stream crosses between blocks more often than one pass per + * block would — a producer that already draws block by block is left alone. + */ +function isInterleaved(blocks: readonly LayoutRow[][]): boolean { + const owner: Array<[number, number]> = [] + blocks.forEach((rows, block) => { + for (const row of rows) for (const item of row.items) owner.push([item.index, block]) + }) + owner.sort((a, b) => a[0] - b[0]) + let switches = 0 + for (let i = 1; i < owner.length; i++) if (owner[i][1] !== owner[i - 1][1]) switches++ + return switches > blocks.length +} + +function spanOf(blocks: LayoutRow[][]): ReorderedBand { + let last = Number.NEGATIVE_INFINITY + for (const rows of blocks) { + for (const row of rows) for (const item of row.items) last = Math.max(last, item.index) + } + return { blocks, last } +} + +/** Which block an x-position belongs to, given the left edges of the gutters. */ +function blockIndex(x: number, gutterStarts: readonly number[]): number { + let index = 0 + while (index < gutterStarts.length && x >= gutterStarts[index]) index++ + return index +} + +/** A block's x-extent across all its rows, with gaps narrower than `minGap` coalesced. */ +function blockIntervals(rows: readonly LayoutRow[], minGap: number): Interval[] { + const intervals: Interval[] = [] + for (const row of rows) { + for (const item of row.items) { + intervals.push([item.geometry.x, item.geometry.x + item.geometry.width]) + } + } + return mergeIntervals([], intervals, minGap) +} + +function widestGap(intervals: readonly Interval[]): number { + let widest = 0 + for (let i = 1; i < intervals.length; i++) { + widest = Math.max(widest, intervals[i][0] - intervals[i - 1][1]) + } + return widest +} + +/** Rows from top to bottom, each sorted left to right. */ +function groupRows(items: readonly LayoutItem[], bodyHeight: number): LayoutRow[] { + const sorted = [...items].sort( + (left, right) => right.geometry.y - left.geometry.y || left.geometry.x - right.geometry.x + ) + const rows: LayoutRow[] = [] + let current: LayoutRow | undefined + let rowHeight = 0 + for (const item of sorted) { + /** The row's own heights set the tolerance; the page's body height only fills in when none is known. */ + const height = Math.max(item.geometry.height, rowHeight) || bodyHeight + if (current && current.y - item.geometry.y <= ROW_TOLERANCE_RATIO * height) { + current.items.push(item) + rowHeight = Math.max(rowHeight, item.geometry.height) + continue + } + current = { y: item.geometry.y, items: [item] } + rowHeight = item.geometry.height + rows.push(current) + } + for (const row of rows) row.items.sort((left, right) => left.geometry.x - right.geometry.x) + return rows +} + +/** A row's x-extent, with intervals closer than `minGap` coalesced. */ +function rowIntervals(row: LayoutRow, minGap: number): Interval[] { + return mergeIntervals( + [], + row.items.map((item) => [item.geometry.x, item.geometry.x + item.geometry.width]), + minGap + ) +} + +/** + * Unions two interval lists, coalescing any pair separated by less than + * `minGap`; narrower gaps never matter to the caller, and dropping them keeps + * the list short. + */ +function mergeIntervals( + left: readonly Interval[], + right: readonly Interval[], + minGap: number +): Interval[] { + const all = [...left, ...right].sort((a, b) => a[0] - b[0]) + const merged: Interval[] = [] + for (const [start, end] of all) { + const last = merged[merged.length - 1] + if (last && start - last[1] < minGap) { + last[1] = Math.max(last[1], end) + continue + } + merged.push([start, end]) + } + return merged +} + +/** + * Median downward step between consecutive rows; 0 with fewer than two. Only + * rows that a gutter divides are passed in, so dense text elsewhere on the page + * cannot shrink the pitch a band is measured against. + */ +function rowPitch(rows: readonly LayoutRow[]): number { + const steps: number[] = [] + for (let i = 1; i < rows.length; i++) steps.push(rows[i - 1].y - rows[i].y) + if (steps.length === 0) return 0 + steps.sort((left, right) => left - right) + return steps[Math.floor((steps.length - 1) / 2)] +} + +/** Median positive item height; 0 when no item reports one. */ +function medianHeight(items: readonly LayoutItem[]): number { + const heights = items.map((item) => item.geometry.height).filter((height) => height > 0) + if (heights.length === 0) return 0 + heights.sort((left, right) => left - right) + return heights[Math.floor((heights.length - 1) / 2)] +} diff --git a/apps/sim/lib/file-parsers/pdf-lines.test.ts b/apps/sim/lib/file-parsers/pdf-lines.test.ts index 0c50a51ebe1..ff552bfc1df 100644 --- a/apps/sim/lib/file-parsers/pdf-lines.test.ts +++ b/apps/sim/lib/file-parsers/pdf-lines.test.ts @@ -192,6 +192,14 @@ describe('joinLines', () => { expect(joinLines(lines, { headingMarkers: false })).toBe('InfraStructure') }) + + it('keeps a soft hyphen from joining across a layout block boundary', () => { + const [first, second] = paragraph(['Infra\u00AD', 'Structure'], 700) + + expect(joinLines([first, { ...second, blockStart: true }], { headingMarkers: false })).toBe( + 'Infra\u00AD\n\nStructure' + ) + }) }) }) diff --git a/apps/sim/lib/file-parsers/pdf-lines.ts b/apps/sim/lib/file-parsers/pdf-lines.ts index 070765f9aa5..0e7594eff5c 100644 --- a/apps/sim/lib/file-parsers/pdf-lines.ts +++ b/apps/sim/lib/file-parsers/pdf-lines.ts @@ -34,6 +34,8 @@ export interface PdfLine { y?: number /** Height of the line's dominant item; 0 when unknown. */ height: number + /** First line of a layout block read out of stream order; always starts a new paragraph. */ + blockStart?: boolean } export type PdfLineSeparator = '' | ' ' | '\n' @@ -167,6 +169,7 @@ export class PdfLineBuilder { private prevEndX: number | undefined private prevY = 0 private lineHeight = 0 + private blockStart = false /** * Separator the geometry rules call for before `str`; '' at line start or @@ -211,7 +214,10 @@ export class PdfLineBuilder { } const text = this.parts.join('') if (text.trim().length > 0) { - this.lines.push({ text, y: this.lineY, height: this.dominantHeight }) + const line: PdfLine = { text, y: this.lineY, height: this.dominantHeight } + if (this.blockStart) line.blockStart = true + this.lines.push(line) + this.blockStart = false } this.parts = [] this.lineY = undefined @@ -222,6 +228,12 @@ export class PdfLineBuilder { this.lineHeight = 0 } + /** Closes the current line and marks the next non-blank line as the start of a layout block. */ + startBlock(): void { + this.endLine() + this.blockStart = true + } + finish(): PdfLine[] { if (this.lines.length >= MAX_PDF_LINES && this.parts.length > 0) { const last = this.lines[this.lines.length - 1] @@ -345,7 +357,7 @@ export function joinLines(lines: readonly PdfLine[], options: JoinLinesOptions = const line = lines[i] const separator = separatorBetween(lines, i, pitch, bodyHeight) const last = parts[parts.length - 1] - if (last.endsWith(SOFT_HYPHEN)) { + if (last.endsWith(SOFT_HYPHEN) && !line.blockStart) { parts[parts.length - 1] = last.slice(0, -1) + line.text continue } @@ -441,13 +453,14 @@ function separatorBetween( ): ' ' | '\n' | '\n\n' { const a = lines[index - 1] const b = lines[index] + if (b.blockStart) return '\n\n' if (a.y === undefined || b.y === undefined) return '\n' const dy = a.y - b.y const maxHeight = Math.max(a.height, b.height) if (maxHeight > 0 ? Math.abs(dy) < SAME_ROW_RATIO * maxHeight : dy === 0) return ' ' if (dy < 0) return isRowReturn(dy, pitch) ? ' ' : '\n\n' const next = lines[index + 1] - if (next?.y !== undefined && isRowReturn(b.y - next.y, pitch)) return ' ' + if (next?.y !== undefined && !next.blockStart && isRowReturn(b.y - next.y, pitch)) return ' ' if (maxHeight > 0 && Math.abs(a.height - b.height) > HEIGHT_CHANGE_RATIO * maxHeight) return '\n\n' const scale = bodyHeight > 0 ? Math.max(1, maxHeight / bodyHeight) : 1 diff --git a/apps/sim/lib/file-parsers/pdf-parser.test.ts b/apps/sim/lib/file-parsers/pdf-parser.test.ts index 36a31010acf..752a9f99c0e 100644 --- a/apps/sim/lib/file-parsers/pdf-parser.test.ts +++ b/apps/sim/lib/file-parsers/pdf-parser.test.ts @@ -2,6 +2,7 @@ * @vitest-environment node */ import { deflateSync } from 'zlib' +import { PDFDocument, StandardFonts } from 'pdf-lib' import { describe, expect, it } from 'vitest' import { MAX_PDF_TEXT_CHARS, PdfParser } from '@/lib/file-parsers/pdf-parser' import { openPdfDocument } from '@/lib/file-parsers/pdfjs-server' @@ -136,6 +137,52 @@ function parseBomb(): Promise { return bombParse } +const WEEKDAYS = ['Su', 'Mo', 'Tu', 'We', 'Th', 'Fr', 'Sa'] as const + +/** Day numbers of `month` (0-based) in 2026, one array per calendar week. */ +function calendarWeeks(month: number): string[][] { + const firstWeekday = new Date(Date.UTC(2026, month, 1)).getUTCDay() + const days = new Date(Date.UTC(2026, month + 1, 0)).getUTCDate() + const weeks: string[][] = [[]] + for (let day = 1; day <= days; day++) { + if (weeks[weeks.length - 1].length > 0 && (firstWeekday + day - 1) % 7 === 0) weeks.push([]) + weeks[weeks.length - 1].push(String(day)) + } + return weeks +} + +/** + * A quarter of 2026 laid out as three month grids side by side, drawn the way + * calendar producers draw them: each visual row across all three months before + * the next row. + */ +async function buildQuarterCalendarPdf(): Promise { + const doc = await PDFDocument.create() + const font = await doc.embedFont(StandardFonts.Helvetica) + const page = doc.addPage([612, 792]) + const size = 8 + const cell = 20 + const monthX = [40, 220, 400] + const names = ['January 2026', 'February 2026', 'March 2026'] + page.drawText('2026 Calendar', { x: 220, y: 740, size: 14, font }) + for (let row = 0; row < 8; row++) { + const y = 700 - row * 14 + for (const [month, x] of monthX.entries()) { + if (row === 0) page.drawText(names[month], { x, y, size, font }) + if (row === 1) { + WEEKDAYS.forEach((day, i) => page.drawText(day, { x: x + i * cell, y, size, font })) + } + if (row < 2) continue + const weeks = calendarWeeks(month) + const week = weeks[row - 2] + if (!week) continue + const offset = row === 2 ? 7 - week.length : 0 + week.forEach((day, i) => page.drawText(day, { x: x + (offset + i) * cell, y, size, font })) + } + } + return Buffer.from(await doc.save()) +} + describe('PdfParser', () => { it('preloads the server worker instead of relying on a runtime-relative worker path', async () => { const previousWorker: unknown = Reflect.get(globalThis, 'pdfjsWorker') @@ -229,4 +276,13 @@ describe('PdfParser', () => { code: 'encrypted_file', }) }) + + it('reads side-by-side calendar months one month at a time', async () => { + const result = await new PdfParser().parseBuffer(await buildQuarterCalendarPdf()) + + const months = ['January 2026', 'February 2026', 'March 2026'].map((name, month) => + [name, WEEKDAYS.join(' '), ...calendarWeeks(month).map((week) => week.join(' '))].join('\n') + ) + expect(result.content).toBe(['2026 Calendar', ...months].join('\n\n')) + }, 30_000) }) diff --git a/apps/sim/lib/file-parsers/pdf-parser.ts b/apps/sim/lib/file-parsers/pdf-parser.ts index 66f24009326..3ae6f213783 100644 --- a/apps/sim/lib/file-parsers/pdf-parser.ts +++ b/apps/sim/lib/file-parsers/pdf-parser.ts @@ -4,6 +4,7 @@ import { sleep } from '@sim/utils/helpers' import type { PDFDocumentProxy, PDFPageProxy } from 'pdfjs-dist/types/src/pdf' import { FileParserError } from '@/lib/file-parsers/errors' import { type PdfPageLines, suppressFurniture } from '@/lib/file-parsers/pdf-furniture' +import { PdfPageCollector } from '@/lib/file-parsers/pdf-layout' import { collectCompounds, collectWords, @@ -12,9 +13,7 @@ import { joinLines, normalizePdfWhitespace, PDF_HEADING_MARKERS_ENABLED, - type PdfItemGeometry, type PdfLine, - PdfLineBuilder, type PdfTextItem, readItemGeometry, } from '@/lib/file-parsers/pdf-lines' @@ -171,7 +170,7 @@ async function readPageWithinBudget( .streamTextContent() .getReader() as ReadableStreamDefaultReader - const builder = new PdfLineBuilder() + const collector = new PdfPageCollector() let remaining = budget let completed = false let dropped = false @@ -213,20 +212,17 @@ async function readPageWithinBudget( const str = item.str const hasEOL = item.hasEOL === true const geometry = readItemGeometry(item) - const separator = str.length > 0 ? builder.separatorBefore(str, geometry) : '' /** Only text and pdf.js's own line breaks count, exactly as before geometry separators existed. */ const cost = str.length + (hasEOL ? 1 : 0) if (cost > remaining) { - appendTruncated(builder, separator, str, geometry, remaining) + /** Keeps as much of `str` as the budget allows, mirroring the old `slice(0, remaining)`. */ + if (remaining > 0) collector.add(str.slice(0, remaining), geometry, false) remaining = 0 dropped = true break } - if (separator === '\n') builder.endLine() - else if (separator.length > 0) builder.append(separator) - builder.append(str, geometry) - if (hasEOL) builder.endLine() + collector.add(str, geometry, hasEOL) remaining -= cost } } @@ -239,21 +235,7 @@ async function readPageWithinBudget( } } - return { lines: builder.finish(), used: budget - remaining, completed, deadlineReached } -} - -/** Applies the free separator, then as much of `str` as `remaining` allows, mirroring the old `slice(0, remaining)`. */ -function appendTruncated( - builder: PdfLineBuilder, - separator: string, - str: string, - geometry: PdfItemGeometry | undefined, - remaining: number -): void { - if (remaining <= 0) return - if (separator === '\n') builder.endLine() - else if (separator.length > 0) builder.append(separator) - builder.append(str.slice(0, remaining), geometry) + return { lines: collector.finish(), used: budget - remaining, completed, deadlineReached } } /** Page height in user space, or undefined when the page cannot report a viewport. */