diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd7e7929e..f44c9deed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -176,7 +176,7 @@ jobs: cache: npm - run: npm ci - name: Test CI scope classifier - run: node --test scripts/configure-ci-apt.spec.mjs scripts/install-playwright-browser.spec.mjs scripts/commit-staged-via-api.spec.mjs scripts/ci-queue-evidence.spec.mjs scripts/ci-scope.spec.mjs scripts/cockpit-dependencies.spec.mjs scripts/cockpit-matrix.spec.mjs scripts/cockpit-frontend-matrix.spec.mjs scripts/cockpit-ports.spec.mjs scripts/cockpit-runtime-bridge-coverage.spec.mjs scripts/react-cockpit/ag-ui-client-tools-configuration.spec.mjs scripts/react-cockpit/ag-ui-client-tools-preparation.spec.mjs scripts/react-cockpit/runtimes-microsoft-agent-framework-configuration.spec.mjs scripts/react-cockpit/prepare-native-maf-fixture.spec.mjs scripts/react-cockpit/runtimes-aws-strands-configuration.spec.mjs scripts/react-cockpit/prepare-native-strands-fixture.spec.mjs scripts/react-cockpit/chat-theming-configuration.spec.mjs scripts/react-cockpit/consumer-preparation.spec.mjs scripts/website-angular-startup.spec.mjs scripts/website-agents-startup.spec.mjs + run: node --test scripts/configure-ci-apt.spec.mjs scripts/install-playwright-browser.spec.mjs scripts/commit-staged-via-api.spec.mjs scripts/ci-queue-evidence.spec.mjs scripts/ci-scope.spec.mjs scripts/cockpit-dependencies.spec.mjs scripts/cockpit-matrix.spec.mjs scripts/cockpit-frontend-matrix.spec.mjs scripts/cockpit-ports.spec.mjs scripts/cockpit-runtime-bridge-coverage.spec.mjs scripts/react-cockpit/ag-ui-client-tools-configuration.spec.mjs scripts/react-cockpit/runtimes-microsoft-agent-framework-configuration.spec.mjs scripts/react-cockpit/runtimes-aws-strands-configuration.spec.mjs scripts/react-cockpit/chat-theming-configuration.spec.mjs scripts/react-cockpit/consumer-preparation.spec.mjs scripts/website-angular-startup.spec.mjs scripts/website-agents-startup.spec.mjs - name: Test Debug static configuration without Python workers run: node --test --test-name-pattern='^debug ' scripts/react-cockpit/chat-debug-configuration.spec.mjs - name: Validate CI workflow guards @@ -328,7 +328,7 @@ jobs: - name: Verify production builds strip the devtools hook if: matrix.lane == 'packages' run: npx nx run chat:test-devtools-bundle - - name: Verify canonical A2UI native fixtures + - name: Verify canonical A2UI native graph if: matrix.lane == 'packages' timeout-minutes: 15 env: @@ -351,10 +351,9 @@ jobs: PYTHONDONTWRITEBYTECODE=1 LANGSMITH_TRACING=false LANGCHAIN_TRACING_V2=false \ "$A2UI_FIXTURE_ENV/bin/python" -E -B -m unittest discover -s cockpit/chat/a2ui/python/tests -p test_booking_form.py env -i HOME="$fixture_home" PATH="$fixture_path" LANG=C.UTF-8 \ - A2UI_FIXTURE_UV="$uv_path" A2UI_FIXTURE_PYTHON="$python_path" \ - A2UI_FIXTURE_ENV="$A2UI_FIXTURE_ENV" A2UI_FIXTURE_CACHE="$A2UI_FIXTURE_CACHE" \ + UV_CACHE_DIR="$A2UI_FIXTURE_CACHE" UV_PYTHON="$python_path" \ PYTHONDONTWRITEBYTECODE=1 LANGSMITH_TRACING=false LANGCHAIN_TRACING_V2=false \ - "$node_path" --test --test-concurrency=1 scripts/react-cockpit/chat-a2ui-fixture.spec.mjs scripts/react-cockpit/chat-a2ui-configuration.spec.mjs scripts/react-cockpit/chat-a2ui-host.spec.mjs scripts/react-cockpit/chat-a2ui-readiness.spec.mjs + "$node_path" --test --test-concurrency=1 --test-force-exit scripts/react-cockpit/chat-a2ui-configuration.spec.mjs scripts/react-cockpit/chat-a2ui-native-probe.spec.mjs - name: Configure bounded Ubuntu package downloads if: matrix.lane == 'packages' run: sudo "$(command -v node)" scripts/configure-ci-apt.mjs @@ -1112,11 +1111,6 @@ jobs: run: | npx nx test "${{ matrix.cap.angular }}" --reporters=default --skip-nx-cache npx nx lint "${{ matrix.cap.angular }}" --skip-nx-cache - - name: Verify canonical client tools fixture ownership - if: matrix.cap.angular == 'cockpit-ag-ui-client-tools-react' - run: | - npx nx run cockpit-ag-ui-client-tools-react:prepare-fixture --skip-nx-cache - node --test scripts/react-cockpit/ag-ui-client-tools-fixture.spec.mjs - name: Prepare native A2UI proof ownership if: matrix.cap.angular == 'cockpit-ag-ui-a2ui-react' run: | @@ -1136,29 +1130,12 @@ jobs: mkdirSync(retain, { recursive: true }); writeFileSync(join(retain, 'provenance.json'), JSON.stringify(provenance, null, 2) + '\n', { flag: 'wx' }); NODE - timeout 1200s node --test --test-concurrency=1 --test-timeout=120000 scripts/react-cockpit/ag-ui-a2ui-configuration.spec.mjs scripts/react-cockpit/ag-ui-a2ui-preparation.spec.mjs scripts/react-cockpit/ag-ui-a2ui-fixture.spec.mjs scripts/react-cockpit/ag-ui-a2ui-native-probe.spec.mjs - - name: Verify native MAF fixture ownership - if: matrix.cap.angular == 'cockpit-runtimes-microsoft-agent-framework-react' - run: | - npx nx run cockpit-runtimes-microsoft-agent-framework-react:prepare-fixture --skip-nx-cache - node --test --test-concurrency=1 scripts/react-cockpit/native-maf-fixture.spec.mjs fixtures/react-parity/native-maf-react/server.spec.mjs - python -m unittest discover -s fixtures/react-parity/native-maf-react -p test_model.py - - name: Verify native Strands fixture ownership - if: matrix.cap.angular == 'cockpit-runtimes-aws-strands-react' - run: | - npx nx run cockpit-runtimes-aws-strands-react:prepare-fixture --skip-nx-cache - node --test --test-concurrency=1 scripts/react-cockpit/native-strands-fixture.spec.mjs fixtures/react-parity/native-strands-react/server.spec.mjs - python -m unittest discover -s fixtures/react-parity/native-strands-react -p test_model.py - - name: Verify native Theming fixture ownership - if: matrix.cap.angular == 'cockpit-chat-theming-react' - run: | - npx nx run cockpit-chat-theming-python:test --skip-nx-cache - node --test --test-concurrency=1 scripts/react-cockpit/chat-theming-fixture.spec.mjs + timeout 1200s node --test --test-concurrency=1 --test-timeout=120000 scripts/react-cockpit/ag-ui-a2ui-configuration.spec.mjs scripts/react-cockpit/ag-ui-a2ui-preparation.spec.mjs scripts/react-cockpit/ag-ui-a2ui-native-probe.spec.mjs - name: Verify native Debug fixture and installed session ownership if: matrix.cap.angular == 'cockpit-chat-debug-react' run: | npx nx run cockpit-chat-debug-python:test --skip-nx-cache - node --test --test-concurrency=1 scripts/react-cockpit/chat-debug-configuration.spec.mjs scripts/react-cockpit/chat-debug-fixture.spec.mjs + node --test --test-concurrency=1 scripts/react-cockpit/chat-debug-configuration.spec.mjs npx nx build core --skip-nx-cache node scripts/react-cockpit/chat-debug-native-session-probe.mjs - name: nx e2e ${{ matrix.cap.angular }} @@ -1232,12 +1209,6 @@ jobs: uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 with: python-version: '3.12' - - name: Prepare frozen canonical client tools browser producer - if: matrix.suite == 'ag-ui' - run: node scripts/react-cockpit/prepare-ag-ui-client-tools-fixture.mjs - - name: Prepare frozen canonical A2UI browser producer - if: matrix.suite == 'ag-ui' - run: node scripts/react-cockpit/prepare-ag-ui-a2ui-fixture.mjs - name: Configure bounded Ubuntu package downloads run: sudo "$(command -v node)" scripts/configure-ci-apt.mjs - name: Install Chromium diff --git a/apps/website/agents-startup.ts b/apps/website/agents-startup.ts index 99e9e9397..2768af6cb 100644 --- a/apps/website/agents-startup.ts +++ b/apps/website/agents-startup.ts @@ -5,11 +5,11 @@ type WebServer = Exclude< unknown[] | undefined >; -/** Each original smoke must still succeed before its own runtime starts. */ +/** The five Deep Agents preview servers start as one pool with one readiness budget. */ export function poolAgentServers(servers: WebServer[]): WebServer[] { const matches = servers.flatMap((server, index) => { const match = - /^npx nx run cockpit-deep-agents-(planning|filesystem|subagents|skills|memory)-python:smoke && node scripts\/react-cockpit\/serve\.mjs deep-agents-\1 --no-parent$/.exec( + /^node scripts\/react-cockpit\/serve\.mjs deep-agents-(planning|filesystem|subagents|skills|memory) --no-parent --backend$/.exec( server.command ); return match ? [{ index, topic: match[1] }] : []; @@ -20,7 +20,7 @@ export function poolAgentServers(servers: WebServer[]): WebServer[] { matches.some((match, i) => match.index !== matches[0].index + i) ) { throw new Error( - 'Agents startup requires the original contiguous five unique smoke/server chains' + 'Agents startup requires the original contiguous five unique preview servers' ); } const first = matches[0].index; diff --git a/apps/website/content/docs/react/ag-ui/guides/client-tools.mdx b/apps/website/content/docs/react/ag-ui/guides/client-tools.mdx index b9f2e4dce..0c4bc1c0f 100644 --- a/apps/website/content/docs/react/ag-ui/guides/client-tools.mdx +++ b/apps/website/content/docs/react/ag-ui/guides/client-tools.mdx @@ -73,6 +73,6 @@ npx nx build cockpit-ag-ui-client-tools-react npx nx e2e cockpit-ag-ui-client-tools-react ~~~ -The e2e target depends on frozen canonical fixture preparation. For explicit preparation, run **npx nx run cockpit-ag-ui-client-tools-react:prepare-fixture**. The build installs the real private package artifacts with strict types. Deterministic offline browser checks use the actual canonical graph and locked AG-UI adapter with scripted model responses over HTTP/SSE. They verify local execution, physical continuations, and owned cleanup; they do not prove a live model-provider request. Docs, Code, and Run select this same source implementation. +The build installs the real private package artifacts with strict types. Deterministic offline browser checks run the actual graph behind the locked AG-UI adapter, with scripted model responses replayed by aimock, over HTTP/SSE. They verify local execution and physical continuations, and that the model receives each exact local result; they do not prove a live model-provider request. Docs, Code, and Run select this same source implementation. diff --git a/apps/website/e2e/dev-warmup-setup.ts b/apps/website/e2e/dev-warmup-setup.ts new file mode 100644 index 000000000..78e1a0154 --- /dev/null +++ b/apps/website/e2e/dev-warmup-setup.ts @@ -0,0 +1,43 @@ +import { chromium, type FullConfig } from '@playwright/test'; + +/** + * Playwright globalSetup for local runs against `next dev`, which compiles + * each route (and the client chunks it loads lazily) on first request. + * With several workers, the first tests all hit uncompiled routes at once; + * the compiles then compete for CPU and a hydration, frame or framework-switch + * check misses its window. Readiness of the dev server's port says nothing + * about that, so this visits the shared docs route once, in every workspace + * mode and both frameworks, before any worker starts. + */ +const pages = [ + '/', + '/docs/angular/deep-agents/capabilities/memory', + '/docs/react/deep-agents/capabilities/memory', + '/docs/react/deep-agents/capabilities/memory?mode=code', + '/docs/react/deep-agents/capabilities/memory?mode=run', +]; + +export default async function warmUp(config: FullConfig) { + const baseURL = config.projects[0]?.use.baseURL; + if (!baseURL) return; + const browser = await chromium.launch(); + try { + const page = await browser.newPage(); + for (const path of pages) { + await page.goto(new URL(path, baseURL).toString(), { + waitUntil: 'load', + timeout: 300_000, + }); + const shell = page.locator('[data-workspace-shell]'); + if (await shell.count()) + await shell.waitFor({ state: 'attached', timeout: 300_000 }); + await page + .locator('[data-workspace-shell][data-hydrated="true"]') + .or(page.locator('body:not(:has([data-workspace-shell]))')) + .first() + .waitFor({ timeout: 300_000 }); + } + } finally { + await browser.close(); + } +} diff --git a/apps/website/e2e/learning-workbench.spec.ts b/apps/website/e2e/learning-workbench.spec.ts index 7969c3fdb..ec897c30f 100644 --- a/apps/website/e2e/learning-workbench.spec.ts +++ b/apps/website/e2e/learning-workbench.spec.ts @@ -1,3 +1,6 @@ +// The preview server (`serve.mjs --backend`) runs the real graph under +// `langgraph dev` on aimock fixtures shared with the Angular example; controls +// come from scripts/react-cockpit/langgraph-proxy.mjs. import { test, expect, type Page, type FrameLocator } from '@playwright/test'; import { selectFramework } from './fixtures/framework-controls'; @@ -208,8 +211,12 @@ for (const framework of ['react', 'angular'] as const) { if (request.method() === 'POST' && path.endsWith('/runs/stream')) submittedRuns.push(path); }); - if (framework === 'react') + if (framework === 'react') { await request.post('http://127.0.0.1:4600/__reset'); + await request.post('http://127.0.0.1:4600/__ready', { timeout: 180000 }); + // "First" streams "Live partial" + " complete"; hold after the first chunk. + await request.post('http://127.0.0.1:4600/__hold/stream', { data: { afterText: 'Live partial' } }); + } const caseId = `learning-${framework}-${testInfo.retry}`; await request.delete(`http://127.0.0.1:4399/__requests/${caseId}`); const host = await openWorkbench(page, framework); @@ -417,6 +424,8 @@ test('React stop and failure remain distinct from application call settlement', request, }) => { await request.post('http://127.0.0.1:4600/__reset'); + await request.post('http://127.0.0.1:4600/__ready', { timeout: 180000 }); + await request.post('http://127.0.0.1:4600/__hold/stream', { data: { afterText: 'Live partial' } }); const host = await openWorkbench(page, 'react'); await host .getByRole('button', { name: 'Live conversation', exact: true }) @@ -458,6 +467,8 @@ test('framework changes dispose an active React stream and restore opt-in replay request, }) => { await request.post('http://127.0.0.1:4600/__reset'); + await request.post('http://127.0.0.1:4600/__ready', { timeout: 180000 }); + await request.post('http://127.0.0.1:4600/__hold/stream', { data: { afterText: 'Live partial' } }); const host = await openWorkbench(page, 'react'); await host .getByRole('button', { name: 'Live conversation', exact: true }) @@ -471,7 +482,7 @@ test('framework changes dispose an active React stream and restore opt-in replay async () => ( await (await request.get('http://127.0.0.1:4600/__lifetime')).json() - ).active + ).activeStream ) .toBe(true); await selectFramework(page, 'angular'); @@ -481,7 +492,7 @@ test('framework changes dispose an active React stream and restore opt-in replay async () => ( await (await request.get('http://127.0.0.1:4600/__lifetime')).json() - ).active + ).activeStream ) .toBe(false); }); diff --git a/apps/website/e2e/react-ag-ui-a2ui-preview.spec.ts b/apps/website/e2e/react-ag-ui-a2ui-preview.spec.ts index 8ecad4c63..758464d06 100644 --- a/apps/website/e2e/react-ag-ui-a2ui-preview.spec.ts +++ b/apps/website/e2e/react-ag-ui-a2ui-preview.spec.ts @@ -37,10 +37,10 @@ test.afterEach(async ({ page, request }, testInfo) => { }); expect(observed?.requests).toEqual([]); expect(observed?.errors).toEqual([]); - const lifetimeReceipt = await ( - await request.get('http://127.0.0.1:4647/__lifetime') - ).json(); - expect(lifetimeReceipt.current.requests).toEqual([]); + // The recording proxy saw no agent request (scripts/react-cockpit/ag-ui-proxy.mjs). + expect( + await (await request.get('http://127.0.0.1:4647/__requests')).json() + ).toEqual([]); expect((await request.post('http://127.0.0.1:4647/__reset')).ok()).toBe(true); }); diff --git a/apps/website/e2e/react-ag-ui-client-tools-preview.spec.ts b/apps/website/e2e/react-ag-ui-client-tools-preview.spec.ts index 4b3121568..1a7905647 100644 --- a/apps/website/e2e/react-ag-ui-client-tools-preview.spec.ts +++ b/apps/website/e2e/react-ag-ui-client-tools-preview.spec.ts @@ -10,10 +10,14 @@ interface Message { id: string; role: string; content: string; toolCallId?: stri interface RecordRow { body: { threadId: string; runId: string; messages: Message[]; tools: { name: string }[] }; events: { type: string; messages?: Message[] }[]; - observations: { socketAttempts: string[]; modelMessages: { type: string; id?: string; tool_call_id?: string; content: string }[][] }; + /** Every model call so far, from aimock's request journal (scripts/react-cockpit/ag-ui-proxy.mjs). */ + modelMessages: { role: string; tool_call_id?: string; content: string }[][]; } async function records(request: APIRequestContext): Promise { - return (await request.get(origin + '/__requests')).json(); + const wire = await (await request.get(origin + '/__requests')).json(); + const journal: { body: { messages?: RecordRow['modelMessages'][number] } }[] = await (await request.get(origin + '/__model-requests')).json(); + const modelMessages = journal.map(entry => entry.body.messages ?? []); + return wire.map((row: RecordRow) => ({ ...row, modelMessages })); } async function hydrated(page: Page, mode: string) { await expect(page.locator('[data-workspace-shell]')).toHaveAttribute('data-hydrated', 'true'); @@ -31,12 +35,11 @@ function native(wire: RecordRow[], count: number) { expect(new Set(wire.map(row => row.body.runId)).size).toBe(count); for (const row of wire) { expect(row.body.tools.map(tool => tool.name)).toEqual(names); - expect(row.observations.socketAttempts).toEqual([]); - const model = row.observations.modelMessages.at(-1) ?? []; const snapshot = row.events.filter(event => event.type === 'MESSAGES_SNAPSHOT').at(-1)?.messages ?? []; for (const result of row.body.messages.filter(message => message.role === 'tool')) { expect(row.body.messages.filter(message => message.toolCalls?.some(call => call.id === result.toolCallId))).toHaveLength(1); - expect(model.filter(message => message.type === 'tool' && message.id === result.id && message.tool_call_id === result.toolCallId && message.content === result.content)).toHaveLength(1); + // The model received the browser's exact result for that call. + expect(row.modelMessages.some(messages => messages.filter(message => message.role === 'tool' && message.tool_call_id === result.toolCallId && message.content === result.content).length === 1)).toBe(true); expect(snapshot.filter(message => message.role === 'tool' && message.id === result.id && message.content === result.content)).toHaveLength(1); } } @@ -44,21 +47,14 @@ function native(wire: RecordRow[], count: number) { test.describe('native React AG-UI Client Tools workspace', () => { test.describe.configure({ mode: 'serial' }); test.beforeEach(async ({ request }) => { - test.skip(Boolean(process.env['BASE_URL']), 'Canonical offline ownership is verified locally'); + test.skip(Boolean(process.env['BASE_URL']), 'The real AG-UI server on aimock is verified locally'); expect((await request.post(origin + '/__reset')).ok()).toBe(true); }); test.afterEach(async ({ page, request }, info) => { if (process.env['BASE_URL']) return; - // Both success and assertion failure dispose the iframe before closing its exact worker owner. - try { - await page.goto('about:blank'); - const wire = await records(request); - await writeFile(info.outputPath('native-wire.json'), JSON.stringify(wire, null, 2)); - } finally { - const stats = await (await request.post(origin + '/__finish')).json(); - await writeFile(info.outputPath('owned-fixture-receipt.json'), JSON.stringify(stats, null, 2)); - expect(Object.values(stats.closed.at(-1).closure).every(Boolean)).toBe(true); - } + await page.goto('about:blank'); + const wire = await records(request); + await writeFile(info.outputPath('native-wire.json'), JSON.stringify(wire, null, 2)); }); test('actual Docs, Code and Run retain native identity, source bytes and six inert presets', async ({ page, request }, info) => { await page.goto(reactRoute + '?topicProof=b4'); diff --git a/apps/website/e2e/react-ag-ui-interrupts-preview.spec.ts b/apps/website/e2e/react-ag-ui-interrupts-preview.spec.ts index 5a9808471..65352249d 100644 --- a/apps/website/e2e/react-ag-ui-interrupts-preview.spec.ts +++ b/apps/website/e2e/react-ag-ui-interrupts-preview.spec.ts @@ -71,9 +71,11 @@ for (const phase of ['paused', 'resume']) { await frame.getByRole('button', { name: 'Send', exact: true }).click(); await expect(frame.getByRole('status')).toHaveText('Awaiting your decision.'); if (phase === 'resume') { + // The resume run streams no text: hold it right after it starts + // (scripts/react-cockpit/ag-ui-proxy.mjs). + await request.post(origin + '/__hold/stream', { data: { afterType: 'RUN_STARTED' } }); await frame.getByRole('button', { name: 'Issue refund', exact: true }).click(); - await expect(frame.getByRole('region', { name: 'Conversation', exact: true })).toContainText('Live partial'); - expect((await (await request.get(origin + '/__lifetime')).json()).active).toBe(true); + await expect.poll(async () => (await (await request.get(origin + '/__lifetime')).json()).active).toBe(true); } const oldFrame = await page.locator('iframe').elementHandle(); await selectFramework(page, 'angular'); diff --git a/apps/website/e2e/react-ag-ui-json-render-preview.spec.ts b/apps/website/e2e/react-ag-ui-json-render-preview.spec.ts index 9c6d4df73..2fe910eb6 100644 --- a/apps/website/e2e/react-ag-ui-json-render-preview.spec.ts +++ b/apps/website/e2e/react-ag-ui-json-render-preview.spec.ts @@ -19,7 +19,7 @@ test('canonical JSON Render Run keeps a retained layout bound to current native }) => { test.skip( Boolean(process.env['BASE_URL']), - 'The compiled offline producer is verified before deployment' + 'The real AG-UI server is verified before deployment' ); const origin = 'http://127.0.0.1:4612'; await request.post(origin + '/__reset'); @@ -175,7 +175,8 @@ test('switching away disposes a held server-tool stream and returning stays lazy await page.goto('/docs/react/ag-ui/guides/json-render?mode=run'); const frame = page.frameLocator('iframe'); await expect(frame.getByRole('status', { name: 'Conversation status' })).toHaveText('Ready.'); - await request.post(origin + '/__hold/result'); + // Hold the real stream after the first tool result (scripts/react-cockpit/ag-ui-proxy.mjs). + await request.post(origin + '/__hold/stream', { data: { afterType: 'TOOL_CALL_RESULT' } }); await frame.getByLabel('Message', { exact: true }).fill('full'); const response = page.waitForResponse( (candidate) => diff --git a/apps/website/e2e/react-ag-ui-streaming-preview.spec.ts b/apps/website/e2e/react-ag-ui-streaming-preview.spec.ts index 38e3ca4e5..46d3a1f1c 100644 --- a/apps/website/e2e/react-ag-ui-streaming-preview.spec.ts +++ b/apps/website/e2e/react-ag-ui-streaming-preview.spec.ts @@ -78,6 +78,8 @@ test('switching away releases the held AG-UI Streaming stream and returning stay ); const origin = 'http://127.0.0.1:4609'; await request.post(origin + '/__reset'); + // Hold the real stream after its first answer chunk (scripts/react-cockpit/ag-ui-proxy.mjs). + await request.post(origin + '/__hold/stream'); await page.goto( '/docs/react/ag-ui/reference/event-mapping?mode=run' ); @@ -89,7 +91,7 @@ test('switching away releases the held AG-UI Streaming stream and returning stay await frame.getByRole('button', { name: 'Send', exact: true }).click(); await expect( frame.getByRole('region', { name: 'Conversation', exact: true }) - ).toContainText('Live partial'); + ).toContainText('Reply to'); const oldFrame = await page.locator('iframe').elementHandle(); await selectFramework(page, 'angular'); await expect.poll(() => oldFrame?.evaluate((element) => element.isConnected)).toBe(false); diff --git a/apps/website/e2e/react-ag-ui-subagents-preview.spec.ts b/apps/website/e2e/react-ag-ui-subagents-preview.spec.ts index 88be65c51..1a67bdf79 100644 --- a/apps/website/e2e/react-ag-ui-subagents-preview.spec.ts +++ b/apps/website/e2e/react-ag-ui-subagents-preview.spec.ts @@ -137,7 +137,10 @@ test('switching away disposes a held child stream and returning stays lazy', asy await page.goto('/docs/react/ag-ui/guides/subagents?mode=run'); const frame = page.frameLocator('iframe'); await expect(frame.getByRole('status')).toHaveText('Ready.'); - await request.post(origin + '/__hold/child-text'); + // Hold the real stream after the first specialist text (scripts/react-cockpit/ag-ui-proxy.mjs). + await request.post(origin + '/__hold/stream', { + data: { afterPattern: '"type": ?"TEXT_MESSAGE_CONTENT".*"subagentRunId"|"subagentRunId".*"type": ?"TEXT_MESSAGE_CONTENT"' }, + }); await frame.getByLabel('Message', { exact: true }).fill('sequential'); const response = page.waitForResponse( (candidate) => diff --git a/apps/website/e2e/react-ag-ui-tool-views-preview.spec.ts b/apps/website/e2e/react-ag-ui-tool-views-preview.spec.ts index c2d437f10..87e2f92a8 100644 --- a/apps/website/e2e/react-ag-ui-tool-views-preview.spec.ts +++ b/apps/website/e2e/react-ag-ui-tool-views-preview.spec.ts @@ -69,6 +69,8 @@ test('switching away disposes a held server-tool stream and returning stays lazy ); const origin = 'http://127.0.0.1:4611'; await request.post(origin + '/__reset'); + // Hold the real stream after the tool result (scripts/react-cockpit/ag-ui-proxy.mjs). + await request.post(origin + '/__hold/stream', { data: { afterType: 'TOOL_CALL_RESULT' } }); await page.goto('/docs/react/ag-ui/guides/tool-views?mode=run'); const frame = page.frameLocator('iframe'); await expect(frame.getByRole('status', { name: 'Conversation status' })).toHaveText('Ready.'); diff --git a/apps/website/e2e/react-client-tools-preview.spec.ts b/apps/website/e2e/react-client-tools-preview.spec.ts index 7f3999289..2987452e1 100644 --- a/apps/website/e2e/react-client-tools-preview.spec.ts +++ b/apps/website/e2e/react-client-tools-preview.spec.ts @@ -1,3 +1,6 @@ +// The preview server (`serve.mjs --backend`) runs the real graph under +// `langgraph dev` on aimock fixtures shared with the Angular example; controls +// come from scripts/react-cockpit/langgraph-proxy.mjs. import { documentationModes } from './fixtures/workspace-controls'; import { selectFramework, expectFrameworkControl } from './fixtures/framework-controls'; import { test, expect } from '@playwright/test'; @@ -27,6 +30,7 @@ test('public React client tools aligns the canonical Chat guide, source, runtime test('switching away disposes pending browser decisions and returning starts empty', async ({ page, request }) => { test.skip(Boolean(process.env['BASE_URL']), 'Local browser decision ownership is verified before deployment'); await request.post('http://127.0.0.1:4603/__reset'); + await request.post('http://127.0.0.1:4603/__ready', { timeout: 180000 }); await page.goto('/docs/react/chat/guides/client-tools?mode=run'); const frame = page.frameLocator('iframe'); await frame.getByLabel('Message', { exact: true }).fill('Bookings'); diff --git a/apps/website/e2e/react-debug-preview.spec.ts b/apps/website/e2e/react-debug-preview.spec.ts index ee0b79d4e..6231e648d 100644 --- a/apps/website/e2e/react-debug-preview.spec.ts +++ b/apps/website/e2e/react-debug-preview.spec.ts @@ -2,7 +2,6 @@ import { selectWorkspaceMode } from './fixtures/workspace-controls'; import { selectFramework } from './fixtures/framework-controls'; import { test, expect, type Page, type APIRequestContext } from '@playwright/test'; import { writeFile } from 'node:fs/promises'; -import { createHash } from 'node:crypto'; import { fromPageOrReactPreview } from './fixtures/react-preview-requests'; const origin = 'http://127.0.0.1:4645'; @@ -49,17 +48,11 @@ test.describe('native Debug Website', () => { await page.goto('about:blank'); expect((await request.post(origin + '/__release')).ok()).toBe(true); expect((await request.post(origin + '/__reset')).ok()).toBe(true); - await expect.poll(async () => (await read(request, '/__lifetime')).active).toEqual({ children: 0, streams: 0, reads: 0, handlers: 0, responses: 0 }); + // No proxied connection, held read or held stream outlives the test. + await expect.poll(async () => (await read(request, '/__lifetime')).openResponses).toBe(0); const lifetime = await read(request, '/__lifetime'); - for (const child of lifetime.children) { - expect(child.reaped).toBe(true); - for (const id of [child.pid, -child.pgid]) { - let code; - try { process.kill(id, 0); } catch (error) { code = (error as NodeJS.ErrnoException).code; } - expect(code).toBe('ESRCH'); - } - } - await writeFile(info.outputPath('observed-child-absence.json'), JSON.stringify(lifetime, null, 2)); + expect(lifetime).toMatchObject({ activeStream: false, activeHistory: false, activeState: false }); + await writeFile(info.outputPath('observed-lifetime.json'), JSON.stringify(lifetime, null, 2)); expect(observations.get(page)?.errors).toEqual([]); }); @@ -146,16 +139,16 @@ test.describe('native Debug Website', () => { const captures = await read(request, '/__captures'); expect(captures).toHaveLength(1); const capture = captures[0]; - expect(capture.proof).toMatchObject({ actualCompiledGraph: true, networkAttempts: 0, providerVariant: 'natural' }); - expect(capture.localCapturedReplay).toBe(true); - expect(capture.sseSha256).toBe(createHash('sha256').update(capture.sse).digest('hex')); - expect(capture.state.values.messages.filter((message: { type: string }) => message.type === 'ai')).toHaveLength(3); - const answer = capture.state.values.messages[1].content; - expect(capture.state.values.analysis).toEqual({ characters: [...answer].length, words: answer.split(' ').length }); + // The real server's stream and saved state (scripts/react-cockpit/langgraph-proxy.mjs). + const runId = /"run_id": ?"([^"]+)"/.exec(capture.sse)?.[1]; + expect(runId).toBeTruthy(); + expect(capture.values.messages.filter((message: { type: string }) => message.type === 'ai')).toHaveLength(3); + const answer = capture.values.messages[1].content; + expect(capture.values.analysis).toEqual({ characters: [...answer].length, words: answer.split(' ').length }); await frame.getByLabel('Message', { exact: true }).fill('Retained follow-up draft ✈ café'); await frame.getByRole('button', { name: 'Open debug panel', exact: true }).click(); await frame.getByRole('tab', { name: 'State', exact: true }).click(); - await expect.poll(async () => JSON.parse(await frame.locator('.tp-debug__value pre').first().textContent() ?? 'null')).toEqual({ analysis: capture.state.values.analysis }); + await expect.poll(async () => JSON.parse(await frame.locator('.tp-debug__value pre').first().textContent() ?? 'null')).toEqual({ analysis: capture.values.analysis }); const historyResponse = page.waitForResponse(response => frameURL.test(response.frame().url()) && response.url().endsWith('/history')); await frame.getByRole('button', { name: 'Refresh checkpoints' }).click(); const history = await (await historyResponse).json(); @@ -172,15 +165,15 @@ test.describe('native Debug Website', () => { await expect(region.locator('.tp-debug__value pre')).toHaveText('{}'); await region.getByText('Compare selected checkpoint with current state', { exact: true }).click(); await expect.poll(async () => JSON.parse(await region.getByLabel('Selected checkpoint to current state JSON', { exact: true }).textContent() ?? 'null')).toEqual([ - { kind: 'added', path: '/analysis', after: capture.state.values.analysis }, + { kind: 'added', path: '/analysis', after: capture.values.analysis }, ]); - await expect.poll(async () => JSON.parse(await frame.locator('.tp-debug__value pre').first().textContent() ?? 'null')).toEqual({ analysis: capture.state.values.analysis }); + await expect.poll(async () => JSON.parse(await frame.locator('.tp-debug__value pre').first().textContent() ?? 'null')).toEqual({ analysis: capture.values.analysis }); const ledger = await read(request, '/__requests'); - const thread = capture.state.checkpoint.thread_id; + const thread = capture.checkpoint.thread_id; expect(ledger.map((entry: { method: string; path: string }) => [entry.method, entry.path])).toEqual([ ['POST', '/api/threads'], ['POST', `/api/threads/${thread}/runs/stream`], - ['GET', `/api/threads/${thread}/runs/${capture.runId}`], + ['GET', `/api/threads/${thread}/runs/${runId}`], ['GET', `/api/threads/${thread}/state`], ['POST', `/api/threads/${thread}/state/checkpoint`], ['POST', `/api/threads/${thread}/history`], @@ -191,7 +184,7 @@ test.describe('native Debug Website', () => { expect(ledger[1].body.input.messages[0].content).toBe('Explain lift ✈ café'); expect(ledger[6].body).toMatchObject({ limit: 10 }); expect(ledger[7].body.checkpoint).toEqual(history[early].checkpoint); - await writeFile(info.outputPath('observed-selection.json'), JSON.stringify({ history, selected, current: capture.state, ledger }, null, 2)); + await writeFile(info.outputPath('observed-selection.json'), JSON.stringify({ history, selected, current: { values: capture.values, checkpoint: capture.checkpoint }, ledger }, null, 2)); await page.screenshot({ path: info.outputPath('run-inspection.png') }); await frame.getByRole('button', { name: 'Close debug panel', exact: true }).click(); for (const name of ['Code', 'Docs', 'Run'] as const) await mode(page, name); diff --git a/apps/website/e2e/react-deep-agents-planning-preview.spec.ts b/apps/website/e2e/react-deep-agents-planning-preview.spec.ts index 83c041435..50dc7f6e3 100644 --- a/apps/website/e2e/react-deep-agents-planning-preview.spec.ts +++ b/apps/website/e2e/react-deep-agents-planning-preview.spec.ts @@ -208,13 +208,19 @@ test('reload clears an unsent React draft without runtime requests', async ({ expect(observations.get(page)?.requests).toEqual(before); }); -test('the aviation suggestion sends its exact prompt', async ({ page }) => { +test('the aviation suggestion sends its exact prompt', async ({ page, request }) => { // Sending starts a model run; deployed targets (preview and production) // keep previews request-free, so the send is verified locally only. test.skip( Boolean(process.env['BASE_URL']), 'Welcome suggestion sends are verified against the local agents suite' ); + test.setTimeout(240_000); + // The preview's aimock + `langgraph dev` start on its first API request; + // wait them out here rather than inside the send's response timeout. + expect( + (await request.post('http://127.0.0.1:4628/__ready', { timeout: 180_000 })).ok() + ).toBe(true); await page.goto(reactRoute + '?mode=run', { waitUntil: 'domcontentloaded', }); diff --git a/apps/website/e2e/react-deep-agents-subagents-preview.spec.ts b/apps/website/e2e/react-deep-agents-subagents-preview.spec.ts index ca087061a..decb42465 100644 --- a/apps/website/e2e/react-deep-agents-subagents-preview.spec.ts +++ b/apps/website/e2e/react-deep-agents-subagents-preview.spec.ts @@ -222,13 +222,19 @@ test('reload clears an unsent React draft without runtime requests', async ({ expect(observations.get(page)?.requests).toEqual(before); }); -test('the aviation suggestion sends its exact prompt', async ({ page }) => { +test('the aviation suggestion sends its exact prompt', async ({ page, request }) => { // Sending starts a model run; deployed targets (preview and production) // keep previews request-free, so the send is verified locally only. test.skip( Boolean(process.env['BASE_URL']), 'Welcome suggestion sends are verified against the local agents suite' ); + test.setTimeout(240_000); + // The preview's aimock + `langgraph dev` start on its first API request; + // wait them out here rather than inside the send's response timeout. + expect( + (await request.post('http://127.0.0.1:4630/__ready', { timeout: 180_000 })).ok() + ).toBe(true); await page.goto(reactRoute + '?mode=run', { waitUntil: 'domcontentloaded', }); diff --git a/apps/website/e2e/react-deployment-runtime-preview.spec.ts b/apps/website/e2e/react-deployment-runtime-preview.spec.ts index 111edaee4..de4ebff14 100644 --- a/apps/website/e2e/react-deployment-runtime-preview.spec.ts +++ b/apps/website/e2e/react-deployment-runtime-preview.spec.ts @@ -1,3 +1,6 @@ +// The preview server (`serve.mjs --backend`) runs the real graph under +// `langgraph dev` on aimock fixtures shared with the Angular example; controls +// come from scripts/react-cockpit/langgraph-proxy.mjs. import { documentationModes } from './fixtures/workspace-controls'; import { selectFramework, expectFrameworkControl } from './fixtures/framework-controls'; import { test, expect } from '@playwright/test'; @@ -78,6 +81,9 @@ test('switching away releases the held Deployment Runtime stream and returning s ); const origin = 'http://127.0.0.1:4608'; await request.post(origin + '/__reset'); + await request.post(origin + '/__ready', { timeout: 180000 }); + // "Hold" streams "Reply to Hold" in chunks; hold after "Reply to". + await request.post(origin + '/__hold/stream', { data: { afterText: 'Reply to' } }); await page.goto('/docs/react/langgraph/guides/deployment?mode=run'); const frame = page.frameLocator('iframe'); await expect( @@ -87,7 +93,7 @@ test('switching away releases the held Deployment Runtime stream and returning s await frame.getByRole('button', { name: 'Send', exact: true }).click(); await expect( frame.getByRole('region', { name: 'Conversation', exact: true }) - ).toContainText('Live partial'); + ).toContainText('Reply to'); const oldFrame = await page.locator('iframe').elementHandle(); await selectFramework(page, 'angular'); await expect.poll(() => oldFrame?.evaluate((element) => element.isConnected)).toBe(false); @@ -96,7 +102,7 @@ test('switching away releases the held Deployment Runtime stream and returning s async () => ( await (await request.get(origin + '/__lifetime')).json() - ).active + ).activeStream ) .toBe(false); await request.post(origin + '/__release'); diff --git a/apps/website/e2e/react-durable-execution-preview.spec.ts b/apps/website/e2e/react-durable-execution-preview.spec.ts index 2fb265066..cd4b3240e 100644 --- a/apps/website/e2e/react-durable-execution-preview.spec.ts +++ b/apps/website/e2e/react-durable-execution-preview.spec.ts @@ -1,3 +1,6 @@ +// The preview server (`serve.mjs --backend`) runs the real graph under +// `langgraph dev` on aimock fixtures shared with the Angular example; controls +// come from scripts/react-cockpit/langgraph-proxy.mjs. import { documentationModes } from './fixtures/workspace-controls'; import { selectFramework, expectFrameworkControl } from './fixtures/framework-controls'; import { test, expect } from '@playwright/test'; @@ -65,6 +68,12 @@ test('switching away releases a held status read and returning starts empty', as ); const origin = 'http://127.0.0.1:4605'; await request.post(origin + '/__reset'); + await request.post(origin + '/__ready', { timeout: 180000 }); + // An unknown run whose connection ends after the first draft, with saved + // history inconclusive until its outcome is published. + await request.post(origin + '/__stream/edit', { data: { ops: [], anonymous: true } }); + await request.post(origin + '/__hold/stream', { data: { end: true } }); + await request.post(origin + '/__history/pending'); await page.goto( '/docs/react/langgraph/guides/durable-execution?mode=run' ); @@ -74,8 +83,35 @@ test('switching away releases a held status read and returning starts empty', as await expect( frame.getByRole('button', { name: 'Check status', exact: true }) ).toBeVisible(); - await request.post(origin + '/__publish-final'); - await request.post(origin + '/__hold-history'); + // The app reads saved history once on its own; let that read finish before + // the next one is held. + await expect + .poll(async () => + ((await (await request.get(origin + '/__requests')).json()) as { path: string }[]) + .filter((entry) => entry.path.endsWith('/history')).length, + { timeout: 20000 } + ) + .toBe(1); + const runs: { path: string; body: { input?: { messages: { id: string }[] } } }[] = + await (await request.get(origin + '/__requests')).json(); + const humanId = runs.find((entry) => entry.path.endsWith('/runs/stream'))! + .body.input!.messages[0].id; + await request.post(origin + '/__history/safe'); + await request.post(origin + '/__history-values', { + data: { + set: { + messages: [ + { type: 'human', id: humanId, content: 'Uncertain' }, + { type: 'ai', id: 'final-answer', content: 'Final answer' }, + ], + step: 'generate', + completed_turn_id: humanId, + completed_answer_id: 'final-answer', + }, + entry: { next: [], tasks: [] }, + }, + }); + await request.post(origin + '/__hold/history'); await frame .getByRole('button', { name: 'Check status', exact: true }) .click(); @@ -88,7 +124,7 @@ test('switching away releases a held status read and returning starts empty', as async () => ( await (await request.get(origin + '/__lifetime')).json() - ).active + ).activeHistory ) .toBe(false); await request.post(origin + '/__release'); diff --git a/apps/website/e2e/react-interrupts-preview.spec.ts b/apps/website/e2e/react-interrupts-preview.spec.ts index f4349740d..94baf60b4 100644 --- a/apps/website/e2e/react-interrupts-preview.spec.ts +++ b/apps/website/e2e/react-interrupts-preview.spec.ts @@ -1,3 +1,6 @@ +// The preview server (`serve.mjs --backend`) runs the real graph under +// `langgraph dev` on aimock fixtures shared with the Angular example; controls +// come from scripts/react-cockpit/langgraph-proxy.mjs. import { documentationModes } from './fixtures/workspace-controls'; import { selectFramework, expectFrameworkControl } from './fixtures/framework-controls'; import { test, expect } from '@playwright/test'; @@ -58,11 +61,14 @@ test('switching away releases a held React decision and returning starts fresh', 'Local held decision lifetime is verified before deployment' ); await request.post('http://127.0.0.1:4601/__reset'); - await request.post('http://127.0.0.1:4601/__hold-resume'); + await request.post('http://127.0.0.1:4601/__ready', { timeout: 180000 }); await page.goto('/docs/react/langgraph/guides/interrupts?mode=run'); const frame = page.frameLocator('iframe'); await frame.getByLabel('Message', { exact: true }).fill('Refund'); await frame.getByRole('button', { name: 'Send', exact: true }).click(); + await expect(frame.getByRole('button', { name: 'Issue refund' })).toBeVisible(); + // Hold the resume (the next stream) right after its first event. + await request.post('http://127.0.0.1:4601/__hold/stream', { data: { immediate: true } }); await frame.getByRole('button', { name: 'Issue refund' }).click(); await expect(frame.getByRole('status')).toHaveText('Sending decision…'); const oldFrame = await page.locator('iframe').elementHandle(); @@ -73,7 +79,7 @@ test('switching away releases a held React decision and returning starts fresh', async () => ( await (await request.get('http://127.0.0.1:4601/__lifetime')).json() - ).active + ).activeStream ) .toBe(false); await selectFramework(page, 'react'); diff --git a/apps/website/e2e/react-maf-preview.spec.ts b/apps/website/e2e/react-maf-preview.spec.ts index 28c27bd78..cf906a9e4 100644 --- a/apps/website/e2e/react-maf-preview.spec.ts +++ b/apps/website/e2e/react-maf-preview.spec.ts @@ -1,14 +1,18 @@ import { selectWorkspaceMode } from './fixtures/workspace-controls'; import { writeFile } from 'node:fs/promises'; -import { test, expect, type Page, type Locator } from '@playwright/test'; +import { test, expect, type Page, type Locator, type APIRequestContext } from '@playwright/test'; const origin = 'http://127.0.0.1:4642'; const approved = 'Expense recorded: $220.00 to Blue Finch Bistro (meals) — queued for reimbursement.'; const rejected = 'Error: Tool call invocation was rejected by user.'; const browserErrors = new WeakMap(); -function events(base64: string) { - return [...Buffer.from(base64, 'base64').toString().matchAll(/^data: (.+)$/gm)].map(match => JSON.parse(match[1])); -} +// The preview's backend: aimock replays cockpit/runtimes/microsoft-agent-framework/angular/e2e/fixtures +// into the example's real Agent Framework AG-UI server, behind the recording proxy +// (scripts/react-cockpit/ag-ui-proxy.mjs): /__requests lists each agent POST with the +// events the server streamed (`events`) and what the browser got (`sent`). +const runs = async (request: APIRequestContext) => (await request.get(origin + '/__requests')).json(); +const model = async (request: APIRequestContext) => (await request.get(origin + '/__model-requests')).json(); +const submitted = (run: { events: { type: string; toolCallId?: string }[] }) => run.events.filter(event => event.type === 'TOOL_CALL_RESULT' && event.toolCallId === 'c0_submit_expense'); async function mode(page: Page, name: 'Docs' | 'Code' | 'Run') { await selectWorkspaceMode(page, name); } @@ -23,26 +27,22 @@ async function fits(locator: Locator) { test.describe('native MAF Website', () => { test.describe.configure({ mode: 'serial' }); - test.skip(!!process.env['BASE_URL'], 'Owned actual-service fixture is local only.'); + test.skip(!!process.env['BASE_URL'], 'The aimock-backed preview is local only.'); test.beforeEach(async ({ page, request }) => { const errors: string[] = []; browserErrors.set(page, errors); page.on('pageerror', error => errors.push(error.message)); page.on('console', message => { if (message.type() === 'error') errors.push(message.text()); }); - expect((await request.post(origin + '/__reset', { data: { scenario: 'approval' } })).ok()).toBe(true); + expect((await request.post(origin + '/__reset')).ok()).toBe(true); }); test.afterEach(async ({ page, request }, info) => { await writeFile(info.outputPath('browser-errors.json'), JSON.stringify(browserErrors.get(page), null, 2)); await writeFile(info.outputPath('dom.html'), await page.content()); await page.screenshot({ path: info.outputPath('final-page.png') }); await page.goto('about:blank'); - // These receipts establish owned fixture closure after page exit. The - // connection integration spec separately proves synchronous native abort - // and revoked retained callbacks on unmount. - await writeFile(info.outputPath('actual-service-evidence.json'), JSON.stringify(await (await request.get(origin + '/__stats')).json(), null, 2)); - const receipt = await (await request.post(origin + '/__finish')).json(); - await writeFile(info.outputPath('owned-cleanup.json'), JSON.stringify(receipt, null, 2)); - for (const owner of receipt.closed) expect(Object.values(owner.closure).every(Boolean)).toBe(true); + // The connection integration spec proves synchronous native abort and + // revoked retained callbacks on unmount. + await writeFile(info.outputPath('actual-service-evidence.json'), JSON.stringify({ runs: await runs(request), model: await model(request) }, null, 2)); }); for (const width of [1280, 390]) for (const decision of ['Approve', 'Decline'] as const) { test(`Docs Code Run and actual native ${decision} at ${width}`, async ({ page, request }, info) => { @@ -85,21 +85,21 @@ test.describe('native MAF Website', () => { expect(await page.locator('iframe').evaluate((current, retained) => current === retained, iframe)).toBe(true); await expect(frame.getByRole('textbox', { name: 'Message', exact: true })).toHaveValue(draft); await expect(frame.getByRole('status')).toHaveText('Ready.'); - expect((await (await request.get(origin + '/__stats')).json()).current.requests).toHaveLength(0); + expect(await runs(request)).toHaveLength(0); await frame.getByRole('button', { name: 'Send', exact: true }).click(); await expect(frame.getByRole('status')).toHaveText('Awaiting your decision.'); for (const control of [frame.getByRole('button', { name: 'Approve', exact: true }), frame.getByRole('button', { name: 'Decline', exact: true }), frame.getByRole('textbox', { name: 'Message', exact: true }), frame.getByRole('status')]) await fits(control); await expect(frame.locator('.tp-approval-card')).toContainText('Blue Finch Bistro'); await fits(frame.locator('.tp-approval-card')); await page.screenshot({ path: info.outputPath('approval.png') }); - const before = (await (await request.get(origin + '/__stats')).json()).current; - expect(before.requests).toHaveLength(1); - const initial = before.requests[0]; - expect(initial.upstreamBase64).toBe(initial.sentBase64); - const pause = events(initial.upstreamBase64).find(event => event.type === 'RUN_FINISHED'); + const before = await runs(request); + expect(before).toHaveLength(1); + const initial = before[0]; + expect(initial.sent).toEqual(initial.events); + const pause = initial.events.find((event: { type: string }) => event.type === 'RUN_FINISHED'); expect(pause).toMatchObject({ runId: initial.body.runId, threadId: initial.body.threadId, outcome: { type: 'interrupt' } }); expect(pause.outcome.interrupts).toHaveLength(1); - expect(before.observed.entries).toBe(0); + expect(submitted(initial)).toHaveLength(0); await frame.getByRole('button', { name: decision, exact: true }).click(); await expect(frame.getByRole('status')).toHaveText(decision === 'Approve' ? 'Fictional expense approved.' : 'Expense declined.'); const literal = decision === 'Approve' ? approved : rejected; @@ -107,19 +107,19 @@ test.describe('native MAF Website', () => { await expect(result).toHaveText(literal); for (const control of [result, frame.getByRole('textbox', { name: 'Message', exact: true }), frame.getByRole('button', { name: 'Send', exact: true }), frame.getByRole('button', { name: 'New conversation', exact: true }), frame.getByRole('status')]) await fits(control); await page.screenshot({ path: info.outputPath('result.png') }); - const after = (await (await request.get(origin + '/__stats')).json()).current; - expect(after.requests).toHaveLength(2); - const resumed = after.requests[1]; + const after = await runs(request); + expect(after).toHaveLength(2); + const resumed = after[1]; expect(resumed.body.threadId).toBe(initial.body.threadId); expect(resumed.body.resume).toEqual([{ interruptId: pause.outcome.interrupts[0].id, status: 'resolved', payload: { approved: decision === 'Approve' } }]); expect(resumed.body.forwardedProps).not.toHaveProperty('command'); - expect(resumed.upstreamBase64).toBe(resumed.sentBase64); - expect(events(resumed.upstreamBase64).find(event => event.type === 'RUN_FINISHED')).toMatchObject({ runId: resumed.body.runId, threadId: resumed.body.threadId }); - expect(after.observed.entries).toBe(decision === 'Approve' ? 1 : 0); - expect(after.observed.model.httpSeen).toBe(3); - expect(after.observed.model.records[2].actualContinuationToolContent).toBe(literal); - expect(after.observed.profile.kind).toContain('no body replacement'); - for (const module of Object.values(after.observed.modules) as { sha256: string }[]) expect(module.sha256).toMatch(/^[a-f0-9]{64}$/); + expect(resumed.sent).toEqual(resumed.events); + expect(resumed.events.find((event: { type: string }) => event.type === 'RUN_FINISHED')).toMatchObject({ runId: resumed.body.runId, threadId: resumed.body.threadId }); + // The real submit_expense ran only when approved. + expect(submitted(resumed)).toHaveLength(decision === 'Approve' ? 1 : 0); + const calls = await model(request); + expect(calls).toHaveLength(3); + expect(calls[2].body.messages.filter((row: { role: string; tool_call_id?: string }) => row.role === 'tool' && row.tool_call_id === 'c0_submit_expense').map((row: { content: string }) => row.content)).toEqual([literal]); }); } }); diff --git a/apps/website/e2e/react-mastra-preview.spec.ts b/apps/website/e2e/react-mastra-preview.spec.ts index 7162f6d7c..eb465e7da 100644 --- a/apps/website/e2e/react-mastra-preview.spec.ts +++ b/apps/website/e2e/react-mastra-preview.spec.ts @@ -1,17 +1,18 @@ import { documentationModes } from './fixtures/workspace-controls'; import { writeFile } from 'node:fs/promises'; -import { test, expect } from '@playwright/test'; +import { test, expect, type APIRequestContext } from '@playwright/test'; const origin = 'http://127.0.0.1:4641'; +// The preview's backend: aimock replays cockpit/runtimes/mastra/angular/e2e/fixtures into the +// real deployments/ag-ui-mastra service, behind the recording proxy +// (scripts/react-cockpit/ag-ui-proxy.mjs), which lists each agent POST at /__requests. +const runs = async (request: APIRequestContext) => (await request.get(origin + '/__requests')).json(); test.describe('native Mastra Website', () => { - test.skip(!!process.env['BASE_URL'], 'Owned actual-service fixture is local only.'); - test.beforeEach(async ({ request }) => { await request.post(origin + '/__reset', { data: { mode: 'normal' } }); }); + test.skip(!!process.env['BASE_URL'], 'The aimock-backed preview is local only.'); + test.beforeEach(async ({ request }) => { await request.post(origin + '/__reset'); }); test.afterEach(async ({ page, request }, info) => { await page.goto('about:blank'); - await writeFile(info.outputPath('private-actual-service-evidence.json'), JSON.stringify(await (await request.get(origin + '/__stats')).json(), null, 2)); - const receipt = await (await request.get(origin + '/__finish')).json(); - await writeFile(info.outputPath('private-owned-cleanup.json'), JSON.stringify(receipt, null, 2)); - for (const owner of receipt.closed) expect(Object.values(owner.closure).every(Boolean)).toBe(true); + await writeFile(info.outputPath('private-actual-service-evidence.json'), JSON.stringify({ runs: await runs(request), model: await (await request.get(origin + '/__model-requests')).json() }, null, 2)); }); test('Docs Code and Run select the authored reservation implementation', async ({ page, request }, info) => { await page.goto('/docs/react/runtimes/mastra/overview'); @@ -36,14 +37,14 @@ test.describe('native Mastra Website', () => { const frame = page.frameLocator('iframe'); await expect(frame.getByRole('status')).toHaveText('Ready.'); await frame.getByRole('button', { name: 'Reserve the campsite', exact: true }).click(); - expect((await (await request.get(origin + '/__stats')).json()).current.requests).toHaveLength(0); + expect(await runs(request)).toHaveLength(0); await frame.getByRole('button', { name: 'Send', exact: true }).click(); await expect(frame.getByRole('button', { name: 'Approve', exact: true })).toBeEnabled(); await expect(frame.getByRole('region', { name: 'Reservation approval' })).toContainText('North Pines'); await frame.getByRole('button', { name: 'Approve', exact: true }).click(); await expect(frame.getByRole('status')).toHaveText('Fictional reservation approved.'); await expect(frame.getByRole('region', { name: 'Observed backend result' }).locator('pre')).toHaveText(JSON.stringify('Reserved North Pines for 2 night(s) — total $90. Confirmation TP-0288.')); - const wire = (await (await request.get(origin + '/__stats')).json()).current.requests; + const wire = await runs(request); expect(wire).toHaveLength(2); expect(wire[1].body.resume[0]).toMatchObject({ status: 'resolved', payload: { approved: true } }); expect(wire[1].body.forwardedProps).not.toHaveProperty('command'); diff --git a/apps/website/e2e/react-memory-preview.spec.ts b/apps/website/e2e/react-memory-preview.spec.ts index e2e23ec50..bfdfeec06 100644 --- a/apps/website/e2e/react-memory-preview.spec.ts +++ b/apps/website/e2e/react-memory-preview.spec.ts @@ -1,3 +1,6 @@ +// The preview server (`serve.mjs --backend`) runs the real graph under +// `langgraph dev` on aimock fixtures shared with the Angular example; controls +// come from scripts/react-cockpit/langgraph-proxy.mjs. import { documentationModes } from './fixtures/workspace-controls'; import { selectFramework, expectFrameworkControl } from './fixtures/framework-controls'; import { test, expect } from '@playwright/test'; @@ -62,7 +65,9 @@ test('switching away releases held React extraction and returning starts empty', 'Local extraction lifetime is verified before deployment' ); await request.post('http://127.0.0.1:4602/__reset'); - await request.post('http://127.0.0.1:4602/__hold-extraction'); + await request.post('http://127.0.0.1:4602/__ready', { timeout: 180000 }); + // Hold once the reply has streamed, before extraction saves facts. + await request.post('http://127.0.0.1:4602/__hold/stream', { data: { afterText: 'Memory reply complete.' } }); await page.goto('/docs/react/langgraph/guides/memory?mode=run'); const frame = page.frameLocator('iframe'); await frame.getByLabel('Message', { exact: true }).fill('Remember Mira'); @@ -81,7 +86,7 @@ test('switching away releases held React extraction and returning starts empty', async () => ( await (await request.get('http://127.0.0.1:4602/__lifetime')).json() - ).active + ).activeStream ) .toBe(false); await selectFramework(page, 'react'); diff --git a/apps/website/e2e/react-persistence-preview.spec.ts b/apps/website/e2e/react-persistence-preview.spec.ts index 89950fea9..d5b40d2c1 100644 --- a/apps/website/e2e/react-persistence-preview.spec.ts +++ b/apps/website/e2e/react-persistence-preview.spec.ts @@ -1,3 +1,6 @@ +// The preview server (`serve.mjs --backend`) runs the real graph under +// `langgraph dev` on aimock fixtures shared with the Angular example; controls +// come from scripts/react-cockpit/langgraph-proxy.mjs. import { documentationModes } from './fixtures/workspace-controls'; import { selectFramework, expectFrameworkControl } from './fixtures/framework-controls'; import { test, expect } from '@playwright/test'; @@ -65,6 +68,7 @@ test('switching away releases held React history and returning starts with an em 'Local history lifetime is verified before deployment' ); await request.post('http://127.0.0.1:4604/__reset'); + await request.post('http://127.0.0.1:4604/__ready', { timeout: 180000 }); await page.goto('/docs/react/langgraph/guides/persistence?mode=run'); const frame = page.frameLocator('iframe'); for (const text of [ @@ -79,7 +83,7 @@ test('switching away releases held React history and returning starts with an em await expect(frame.getByRole('status')).toHaveText('Ready.'); } } - await request.post('http://127.0.0.1:4604/__hold-history'); + await request.post('http://127.0.0.1:4604/__hold/history'); await frame .getByRole('button', { name: 'Conversation 1', exact: true }) .click(); @@ -94,7 +98,7 @@ test('switching away releases held React history and returning starts with an em async () => ( await (await request.get('http://127.0.0.1:4604/__lifetime')).json() - ).active + ).activeHistory ) .toBe(false); await request.post('http://127.0.0.1:4604/__release'); diff --git a/apps/website/e2e/react-strands-preview.spec.ts b/apps/website/e2e/react-strands-preview.spec.ts index f4cba8847..e64426cfd 100644 --- a/apps/website/e2e/react-strands-preview.spec.ts +++ b/apps/website/e2e/react-strands-preview.spec.ts @@ -1,14 +1,17 @@ import { selectWorkspaceMode } from './fixtures/workspace-controls'; import { writeFile } from 'node:fs/promises'; -import { test, expect, type Page, type Locator } from '@playwright/test'; +import { test, expect, type Page, type Locator, type APIRequestContext } from '@playwright/test'; const origin = 'http://127.0.0.1:4643'; const approved = 'Meeting booked for Tuesday 10:00: Q3 roadmap review'; const rejected = 'The human declined. Meeting NOT booked: Q3 roadmap review'; const browserErrors = new WeakMap(); -function events(base64: string) { - return [...Buffer.from(base64, 'base64').toString().matchAll(/^data: (.+)$/gm)].map(match => JSON.parse(match[1])); -} +// The preview's backend: aimock replays cockpit/runtimes/aws-strands/angular/e2e/fixtures +// into the example's real Strands AG-UI server, behind the recording proxy +// (scripts/react-cockpit/ag-ui-proxy.mjs): /__requests lists each agent POST with the +// events the server streamed (`events`) and what the browser got (`sent`). +const runs = async (request: APIRequestContext) => (await request.get(origin + '/__requests')).json(); +const model = async (request: APIRequestContext) => (await request.get(origin + '/__model-requests')).json(); async function mode(page: Page, name: 'Docs' | 'Code' | 'Run') { await selectWorkspaceMode(page, name); } @@ -23,27 +26,20 @@ async function fits(locator: Locator) { test.describe('native Strands Website', () => { test.describe.configure({ mode: 'serial' }); - test.skip(!!process.env['BASE_URL'], 'Owned actual-service fixture is local only.'); + test.skip(!!process.env['BASE_URL'], 'The aimock-backed preview is local only.'); test.beforeEach(async ({ page, request }) => { const errors: string[] = []; browserErrors.set(page, errors); page.on('pageerror', error => errors.push(error.message)); page.on('console', message => { if (message.type() === 'error') errors.push(message.text()); }); - expect((await request.post(origin + '/__reset', { data: { scenario: 'approval' } })).ok()).toBe(true); + expect((await request.post(origin + '/__reset')).ok()).toBe(true); }); test.afterEach(async ({ page, request }, info) => { - try { - await writeFile(info.outputPath('browser-errors.json'), JSON.stringify(browserErrors.get(page), null, 2)); - await writeFile(info.outputPath('dom.html'), await page.content()); - await page.screenshot({ path: info.outputPath('final-page.png') }); - await page.goto('about:blank'); - await writeFile(info.outputPath('actual-service-evidence.json'), JSON.stringify(await (await request.get(origin + '/__stats')).json(), null, 2)); - } finally { - const receipt = await (await request.post(origin + '/__finish')).json(); - await writeFile(info.outputPath('owned-cleanup.json'), JSON.stringify(receipt, null, 2)); - expect(receipt.closed.length).toBeGreaterThan(0); - for (const owner of receipt.closed) expect(Object.values(owner.closure).every(Boolean)).toBe(true); - } + await writeFile(info.outputPath('browser-errors.json'), JSON.stringify(browserErrors.get(page), null, 2)); + await writeFile(info.outputPath('dom.html'), await page.content()); + await page.screenshot({ path: info.outputPath('final-page.png') }); + await page.goto('about:blank'); + await writeFile(info.outputPath('actual-service-evidence.json'), JSON.stringify({ runs: await runs(request), model: await model(request) }, null, 2)); expect(browserErrors.get(page)).toEqual([]); }); for (const width of [1280, 390]) for (const decision of ['Approve', 'Decline'] as const) { @@ -85,21 +81,21 @@ test.describe('native Strands Website', () => { expect(await page.locator('iframe').evaluate((element, original) => element === original, iframe)).toBe(true); } await expect(frame.getByRole('textbox', { name: 'Message', exact: true })).toHaveValue(draft); - expect((await (await request.get(origin + '/__stats')).json()).current.requests).toHaveLength(0); + expect(await runs(request)).toHaveLength(0); await frame.getByRole('button', { name: 'Send', exact: true }).click(); await expect(frame.getByRole('status')).toHaveText('Awaiting your decision.'); for (const control of [frame.getByRole('button', { name: 'Approve', exact: true }), frame.getByRole('button', { name: 'Decline', exact: true }), frame.getByRole('textbox', { name: 'Message', exact: true }), frame.getByRole('status')]) await fits(control); await expect(frame.locator('.tp-approval-card')).toContainText('Q3 roadmap review'); await fits(frame.locator('.tp-approval-card')); await page.screenshot({ path: info.outputPath('approval.png') }); - const before = (await (await request.get(origin + '/__stats')).json()).current; - expect(before.requests).toHaveLength(1); - const initial = before.requests[0]; - expect(initial.upstreamBase64).toBe(initial.sentBase64); - const pause = events(initial.upstreamBase64).find(event => event.type === 'RUN_FINISHED'); + const before = await runs(request); + expect(before).toHaveLength(1); + const initial = before[0]; + expect(initial.sent).toEqual(initial.events); + const pause = initial.events.find((event: { type: string }) => event.type === 'RUN_FINISHED'); expect(pause).toMatchObject({ runId: initial.body.runId, threadId: initial.body.threadId, outcome: { type: 'interrupt' } }); expect(pause.outcome.interrupts).toHaveLength(1); - expect(before.observed.completedReturns).toHaveLength(0); + expect(initial.events.some((event: { type: string; content?: string }) => event.type === 'TOOL_CALL_RESULT' && /booked/i.test(event.content ?? ''))).toBe(false); expect(pause.outcome.interrupts[0]).toMatchObject({ reason: 'book_meeting', metadata: { reason: { topic: 'Q3 roadmap review', slot: 'Tuesday 10:00' } } }); await frame.getByRole('button', { name: decision, exact: true }).click(); await expect(frame.getByRole('status')).toHaveText(decision === 'Approve' ? 'Fictional meeting booked.' : 'Meeting declined.'); @@ -108,34 +104,31 @@ test.describe('native Strands Website', () => { await expect(result).toHaveText(literal); for (const control of [result, frame.getByRole('textbox', { name: 'Message', exact: true }), frame.getByRole('button', { name: 'Send', exact: true }), frame.getByRole('button', { name: 'New conversation', exact: true }), frame.getByRole('status')]) await fits(control); await page.screenshot({ path: info.outputPath('result.png') }); - const after = (await (await request.get(origin + '/__stats')).json()).current; - expect(after.requests).toHaveLength(2); - const resumed = after.requests[1]; + const after = await runs(request); + expect(after).toHaveLength(2); + const resumed = after[1]; expect(resumed.body.threadId).toBe(initial.body.threadId); expect(resumed.body.resume).toEqual([{ interruptId: pause.outcome.interrupts[0].id, status: 'resolved', payload: { approved: decision === 'Approve' } }]); expect(resumed.body.forwardedProps).not.toHaveProperty('command'); - expect(resumed.upstreamBase64).toBe(resumed.sentBase64); - expect(events(resumed.upstreamBase64).find(event => event.type === 'RUN_FINISHED')).toMatchObject({ runId: resumed.body.runId, threadId: resumed.body.threadId }); - expect(after.observed.completedReturns).toHaveLength(1); + expect(resumed.sent).toEqual(resumed.events); + expect(resumed.events.find((event: { type: string }) => event.type === 'RUN_FINISHED')).toMatchObject({ runId: resumed.body.runId, threadId: resumed.body.threadId }); expect(resumed.body.runId).not.toBe(initial.body.runId); - const canonical = events(resumed.upstreamBase64).filter(event => event.type === 'MESSAGES_SNAPSHOT').at(-1).messages; + const canonical = resumed.events.filter((event: { type: string }) => event.type === 'MESSAGES_SNAPSHOT').at(-1).messages; const bookingCall = canonical.flatMap((row: { toolCalls?: { id: string; function: { name: string } }[] }) => row.toolCalls ?? []).find((call: { function: { name: string } }) => call.function.name === 'book_meeting'); const results = canonical.filter((row: { role: string; toolCallId?: string }) => row.role === 'tool' && row.toolCallId === bookingCall.id); expect(results).toHaveLength(1); expect(results[0].content).toBe(JSON.stringify(literal)); await expect(frame.getByRole('region', { name: 'Last observed schedule' })).toContainText('Observed booking status: pending'); - expect(after.observed.model.httpSeen).toBe(3); - const toolResults = after.observed.model.records.at(-1).body.messages.filter((row: { role: string; tool_call_id?: string }) => row.role === 'tool' && row.tool_call_id === bookingCall.id); + const calls = await model(request); + expect(calls).toHaveLength(3); + const toolResults = calls.at(-1).body.messages.filter((row: { role: string; tool_call_id?: string }) => row.role === 'tool' && row.tool_call_id === bookingCall.id); expect(toolResults).toHaveLength(1); expect(toolResults[0].content).toBe(literal); - - for (const module of Object.values(after.observed.modules) as { sha256: string }[]) expect(module.sha256).toMatch(/^[a-f0-9]{64}$/); }); } for (const width of [1280, 390]) { test(`unconfirmed native result preserves revoked authority across modes at ${width}`, async ({ page, request }) => { test.setTimeout(90000); - expect((await request.post(origin + '/__reset', { data: { scenario: 'missing-terminal' } })).ok()).toBe(true); await page.setViewportSize({ width, height: width === 390 ? 844 : 900 }); await page.goto('/docs/react/runtimes/aws-strands/overview'); await expect(page.locator('[data-workspace-shell]')).toHaveAttribute('data-hydrated', 'true', { timeout: 30000 }); @@ -145,22 +138,25 @@ test.describe('native Strands Website', () => { await frame.getByRole('button', { name: 'Q3 roadmap review', exact: true }).click(); await frame.getByRole('button', { name: 'Send', exact: true }).click(); await expect(frame.getByRole('button', { name: 'Approve', exact: true })).toBeEnabled(); + // Injected data: the proxy drops the resumed run's real terminal on its way to the browser. + expect((await request.post(origin + '/__stream/edit', { data: { ops: [{ type: 'RUN_FINISHED', op: 'drop' }] } })).ok()).toBe(true); await frame.getByRole('button', { name: 'Approve', exact: true }).click(); await expect(frame.getByRole('status')).toHaveText('Meeting unconfirmed. Start a new conversation.'); await expect(frame.getByRole('region', { name: 'Observed backend result' })).toHaveCount(0); await expect(frame.getByRole('button', { name: 'Send', exact: true })).toBeDisabled(); await expect(frame.getByRole('button', { name: 'Approve', exact: true })).toHaveCount(0); - const evidence = (await (await request.get(origin + '/__stats')).json()).current; - expect(evidence.requests).toHaveLength(2); - expect(evidence.observed.completedReturns).toHaveLength(1); - const resumed = evidence.requests[1]; - expect(events(resumed.upstreamBase64).some(event => event.type === 'RUN_FINISHED' && event.runId === resumed.body.runId)).toBe(true); - expect(events(resumed.sentBase64).some(event => event.type === 'RUN_FINISHED' && event.runId === resumed.body.runId)).toBe(false); + const evidence = await runs(request); + expect(evidence).toHaveLength(2); + const resumed = evidence[1]; + // The real booking completed upstream; only the browser lost the terminal. + expect(resumed.events.some((event: { type: string; content?: string }) => event.type === 'TOOL_CALL_RESULT' && (event.content ?? '').includes(approved))).toBe(true); + expect(resumed.events.some((event: { type: string; runId?: string }) => event.type === 'RUN_FINISHED' && event.runId === resumed.body.runId)).toBe(true); + expect(resumed.sent.some((event: { type: string; runId?: string }) => event.type === 'RUN_FINISHED' && event.runId === resumed.body.runId)).toBe(false); await mode(page, 'Docs'); await expect(page.locator('iframe')).toHaveCount(1); await mode(page, 'Run'); await expect(page.frameLocator('iframe').getByRole('status')).toHaveText('Meeting unconfirmed. Start a new conversation.'); - expect((await (await request.get(origin + '/__stats')).json()).current.requests).toHaveLength(2); + expect(await runs(request)).toHaveLength(2); }); } }); diff --git a/apps/website/e2e/react-streaming-preview.spec.ts b/apps/website/e2e/react-streaming-preview.spec.ts index dc90a8391..24358f050 100644 --- a/apps/website/e2e/react-streaming-preview.spec.ts +++ b/apps/website/e2e/react-streaming-preview.spec.ts @@ -1,3 +1,6 @@ +// The preview server (`serve.mjs --backend`) runs the real graph under +// `langgraph dev` on aimock fixtures shared with the Angular example; controls +// come from scripts/react-cockpit/langgraph-proxy.mjs. import { documentationModes } from './fixtures/workspace-controls'; import { frameworkControl, selectFramework } from './fixtures/framework-controls'; import { test, expect } from '@playwright/test'; @@ -83,6 +86,9 @@ test('the installed React frame streams and releases work when the public select 'Local fixture stream lifetime is verified before deployment' ); await request.post('http://127.0.0.1:4600/__reset'); + await request.post('http://127.0.0.1:4600/__ready', { timeout: 180000 }); + // "First" streams "Live partial" + " complete"; hold after the first chunk. + await request.post('http://127.0.0.1:4600/__hold/stream', { data: { afterText: 'Live partial' } }); await page.goto('/docs/react/langgraph/guides/streaming?mode=run'); const frame = page.frameLocator('iframe'); await frame.getByLabel('Message', { exact: true }).fill('First'); @@ -99,7 +105,7 @@ test('the installed React frame streams and releases work when the public select async () => ( await (await request.get('http://127.0.0.1:4600/__lifetime')).json() - ).active + ).activeStream ) .toBe(false); await selectFramework(page, 'react'); diff --git a/apps/website/e2e/react-subgraphs-preview.spec.ts b/apps/website/e2e/react-subgraphs-preview.spec.ts index 4b728efec..393363b49 100644 --- a/apps/website/e2e/react-subgraphs-preview.spec.ts +++ b/apps/website/e2e/react-subgraphs-preview.spec.ts @@ -1,3 +1,6 @@ +// The preview server (`serve.mjs --backend`) runs the real graph under +// `langgraph dev` on aimock fixtures shared with the Angular example; controls +// come from scripts/react-cockpit/langgraph-proxy.mjs. import { documentationModes } from './fixtures/workspace-controls'; import { selectFramework, expectFrameworkControl } from './fixtures/framework-controls'; import { test, expect } from '@playwright/test'; @@ -65,7 +68,8 @@ test('switching away releases a held child and returning starts empty', async ({ ); const origin = 'http://127.0.0.1:4606'; await request.post(origin + '/__reset'); - await request.post(origin + '/__hold-child'); + await request.post(origin + '/__ready', { timeout: 180000 }); + await request.post(origin + '/__hold/stream', { data: { afterChildMessage: true } }); await page.goto('/docs/react/langgraph/guides/subgraphs?mode=run'); const frame = page.frameLocator('iframe'); await frame.getByLabel('Message', { exact: true }).fill('Research first'); @@ -84,7 +88,7 @@ test('switching away releases a held child and returning starts empty', async ({ async () => ( await (await request.get(origin + '/__lifetime')).json() - ).active + ).activeStream ) .toBe(false); await request.post(origin + '/__release'); diff --git a/apps/website/e2e/react-theming-preview.spec.ts b/apps/website/e2e/react-theming-preview.spec.ts index d34317e08..3d51ddeec 100644 --- a/apps/website/e2e/react-theming-preview.spec.ts +++ b/apps/website/e2e/react-theming-preview.spec.ts @@ -2,9 +2,11 @@ import { selectWorkspaceMode } from './fixtures/workspace-controls'; import { selectFramework } from './fixtures/framework-controls'; import { test, expect, type Page } from '@playwright/test'; import { writeFile } from 'node:fs/promises'; -import { createHash } from 'node:crypto'; import { fromPageOrReactPreview } from './fixtures/react-preview-requests'; +// The preview's server (`serve.mjs chat-theming --backend`) runs the real +// graph under `langgraph dev` on aimock fixtures shared with the Angular +// example, and records requests through scripts/react-cockpit/langgraph-proxy.mjs. const origin = 'http://127.0.0.1:4644'; const route = '/docs/angular/chat/guides/theming'; const reactRoute = '/docs/react/chat/guides/theming'; @@ -25,7 +27,7 @@ test.describe('native Theming Website', () => { test.describe.configure({ mode: 'serial' }); test.skip( !!process.env['BASE_URL'], - 'Compiled graph controls and deterministic providers are local only.' + 'The aimock-backed graph and its proxy controls are local only.' ); test.beforeEach(async ({ page, request }) => { const observed = { @@ -46,6 +48,10 @@ test.describe('native Theming Website', () => { }); }); expect((await request.post(origin + '/__reset')).ok()).toBe(true); + // The backend starts lazily; wait for it before measuring UI transitions. + expect( + (await request.post(origin + '/__ready', { timeout: 180000 })).ok() + ).toBe(true); }); test.afterEach(async ({ page, request }, info) => { await writeFile(info.outputPath('page.html'), await page.content()); @@ -53,37 +59,11 @@ test.describe('native Theming Website', () => { info.outputPath('browser-observations.json'), JSON.stringify(observations.get(page), null, 2) ); + // Every proxied browser connection closes and nothing stays held. await page.goto('about:blank'); await expect - .poll( - async () => - ( - await (await request.get(origin + '/__lifetime')).json() - ).active - ) - .toEqual({ children: 0, streams: 0, histories: 0, handlers: 0 }); - await expect - .poll( - async () => - ( - await (await request.get(origin + '/__lifetime')).json() - ).responseOpen - ) - .toBe(0); - const lifetime = await (await request.get(origin + '/__lifetime')).json(); - for (const child of lifetime.children) { - expect(child.reaped).toBe(true); - expect(() => process.kill(child.pid, 0)).toThrow(/ESRCH/); - expect(() => process.kill(-child.pgid, 0)).toThrow(/ESRCH/); - } - await writeFile( - info.outputPath('observed-child-absence.json'), - JSON.stringify( - { children: lifetime.children, observedPidAndPgidAbsence: 'ESRCH' }, - null, - 2 - ) - ); + .poll(async () => (await request.get(origin + '/__lifetime')).json()) + .toMatchObject({ activeStream: false, activeHistory: false, openResponses: 0 }); for (const path of ['/__requests', '/__captures', '/__lifetime']) await writeFile( info.outputPath(path.slice(3) + '.json'), @@ -216,13 +196,17 @@ test.describe('native Theming Website', () => { 'GET', 'POST', ]); - const captures = await (await request.get(origin + '/__captures')).json(); + const captures: { + sse: string; + savedMessages: { type: string; content: string }[]; + next: string[]; + titleBefore: string | null; + titleAfter: string | null; + }[] = await (await request.get(origin + '/__captures')).json(); expect(captures).toHaveLength(1); - const { threadId, runId } = captures[0]; - expect(wire[0].body).toMatchObject({ - thread_id: threadId, - if_exists: 'raise', - }); + const threadId = wire[0].body.thread_id; + const runId = wire[2].path.split('/').at(-1); + expect(wire[0].body).toMatchObject({ if_exists: 'raise' }); expect(wire.map((r: { path: string }) => r.path)).toEqual([ '/api/threads', `/api/threads/${threadId}/runs/stream`, @@ -233,32 +217,15 @@ test.describe('native Theming Website', () => { expect(wire[1].body.input.messages[0].content).toBe( 'Explain lift ✈ café' ); - expect(captures[0].sseSha256).toBe( - createHash('sha256').update(captures[0].sse).digest('hex') - ); - expect(captures[0].historyResponses).toHaveLength(1); - expect(captures[0].historyResponses[0]).toHaveLength(1); - expect(captures[0].historyResponses[0][0]).toMatchObject({ - values: { messages: captures[0].savedMessages }, - next: [], - checkpoint: { thread_id: threadId, checkpoint_ns: '' }, - }); - expect(captures[0].proof).toMatchObject({ - actualCompiledGraph: true, - networkConnectAttempts: 0, - titleMessageCallbacks: 0, - }); - expect(captures[0].titleUpdates).toHaveLength(1); - expect(captures[0].sse).not.toContain('Authored metadata title'); + // The real server saved one finished turn with its title. + expect(captures[0].next).toEqual([]); + expect(captures[0].titleBefore).toBeNull(); + expect(captures[0].titleAfter).toBe('Aviation snippet title'); expect( - captures[0].savedMessages.find( - (m: { type: string }) => m.type === 'human' - ).content + captures[0].savedMessages.find((m) => m.type === 'human')?.content ).toBe('Explain lift ✈ café'); expect( - captures[0].savedMessages.filter( - (m: { type: string }) => m.type === 'ai' - ) + captures[0].savedMessages.filter((m) => m.type === 'ai') ).toHaveLength(1); const before = [...(observations.get(page)?.requests ?? [])]; await frame diff --git a/apps/website/e2e/react-time-travel-preview.spec.ts b/apps/website/e2e/react-time-travel-preview.spec.ts index c4f0125ff..582b7a815 100644 --- a/apps/website/e2e/react-time-travel-preview.spec.ts +++ b/apps/website/e2e/react-time-travel-preview.spec.ts @@ -1,3 +1,6 @@ +// The preview server (`serve.mjs --backend`) runs the real graph under +// `langgraph dev` on aimock fixtures shared with the Angular example; controls +// come from scripts/react-cockpit/langgraph-proxy.mjs. import { documentationModes } from './fixtures/workspace-controls'; import { selectFramework, expectFrameworkControl } from './fixtures/framework-controls'; import { test, expect } from '@playwright/test'; @@ -65,6 +68,7 @@ test('switching away releases a held fork and returning starts with an empty che ); const origin = 'http://127.0.0.1:4607'; await request.post(origin + '/__reset'); + await request.post(origin + '/__ready', { timeout: 180000 }); await page.goto('/docs/react/langgraph/guides/time-travel?mode=run'); const frame = page.frameLocator('iframe'); for (const text of ['A', 'B']) { @@ -72,16 +76,29 @@ test('switching away releases a held fork and returning starts with an empty che await frame.getByRole('button', { name: 'Send', exact: true }).click(); await expect(frame.getByRole('status')).toHaveText('Response complete.'); } + // Real history saves input, start and done checkpoints per turn: select + // turn A's completed checkpoint from the page the app just loaded. + const loaded = page.waitForResponse((r) => r.url().endsWith('/history')); await frame .getByRole('button', { name: 'Refresh saved checkpoints' }) .click(); - await frame.getByRole('button', { name: 'Select checkpoint 3' }).click(); - await request.post(origin + '/__hold-stream'); + const history: { next: string[]; values: { messages: unknown[] } }[] = + await (await loaded).json(); + const a = history.findIndex( + (entry) => entry.next.length === 0 && entry.values.messages.length === 2 + ); + await frame + .getByRole('button', { name: `Select checkpoint ${a + 1}` }) + .click(); + await request.post(origin + '/__hold/stream'); await frame.getByLabel('Message', { exact: true }).fill('Held fork'); await frame.getByRole('button', { name: 'Fork selected checkpoint' }).click(); - await expect( - frame.getByRole('region', { name: 'Conversation', exact: true }) - ).toContainText('Transient chunk'); + await expect + .poll( + async () => + (await (await request.get(origin + '/__lifetime')).json()).activeStream + ) + .toBe(true); const oldFrame = await page.locator('iframe').elementHandle(); await selectFramework(page, 'angular'); await expect.poll(() => oldFrame?.evaluate((element) => element.isConnected)).toBe(false); @@ -90,7 +107,7 @@ test('switching away releases a held fork and returning starts with an empty che async () => ( await (await request.get(origin + '/__lifetime')).json() - ).active + ).activeStream ) .toBe(false); await request.post(origin + '/__release'); diff --git a/apps/website/playwright.config.ts b/apps/website/playwright.config.ts index 5f7658215..f37d77db4 100644 --- a/apps/website/playwright.config.ts +++ b/apps/website/playwright.config.ts @@ -6,6 +6,9 @@ type WebsitePlaywrightEnvironment = Readonly< Record >; +const oneOrMore = (files: string[]) => + files.length === 0 ? undefined : files.length === 1 ? files[0] : files; + export const createWebsitePlaywrightConfig = ( environment: WebsitePlaywrightEnvironment = process.env ) => { @@ -74,9 +77,16 @@ export const createWebsitePlaywrightConfig = ( ? '**/custom-runtime-bfcache.spec.ts' : undefined, testIgnore, - globalSetup: runtimeBypass - ? resolve(__dirname, 'e2e', 'runtime-bypass-setup.ts') - : undefined, + globalSetup: oneOrMore([ + ...(runtimeBypass + ? [resolve(__dirname, 'e2e', 'runtime-bypass-setup.ts')] + : []), + // `next dev` compiles routes on first request; compile the shared docs + // route once before parallel workers race for it. + ...(shouldStartLocalServer && !productionMode && !bfcacheRuntimeTest + ? [resolve(__dirname, 'e2e', 'dev-warmup-setup.ts')] + : []), + ]), fullyParallel: true, // Match the cockpit configs: 2 retries on CI to absorb transient Next.js // dev-server startup flake; 0 locally for fast feedback. @@ -166,9 +176,10 @@ export const createWebsitePlaywrightConfig = ( reuseExistingServer, }, ...(!productionMode && !bfcacheRuntimeTest ? [{ - command: 'node scripts/react-cockpit/prepare-ag-ui-a2ui-fixture.mjs --serve', + // Render-only preview: its aimock + AG-UI server never boot. + command: 'node scripts/react-cockpit/serve.mjs ag-ui-a2ui --no-parent --backend', cwd: '../..', url: 'http://127.0.0.1:4647', - reuseExistingServer: false, timeout: 330_000, + reuseExistingServer: false, timeout: 180_000, gracefulShutdown: { signal: 'SIGTERM' as const, timeout: 15000 }, }] : []), ...(!bfcacheRuntimeTest @@ -250,137 +261,140 @@ export const createWebsitePlaywrightConfig = ( // website:e2e builds every installed React example first, // outside the server-readiness budget. Direct local // Playwright runs require those build outputs too. - 'node scripts/react-cockpit/serve.mjs', + 'node scripts/react-cockpit/serve.mjs streaming --backend', + // Previews whose specs drive the backend wait on /__ready: + // its aimock + `langgraph dev` boot during startup, not + // mid-run beside other workers' page renders. Previews only + // rendered (no API traffic) never boot a backend. cwd: '../..', - url: 'http://127.0.0.1:4600', + url: 'http://127.0.0.1:4600/__ready', reuseExistingServer, timeout: 180_000, }, { command: - 'node scripts/react-cockpit/serve.mjs interrupts --no-parent', + 'node scripts/react-cockpit/serve.mjs interrupts --no-parent --backend', cwd: '../..', - url: 'http://127.0.0.1:4601', + url: 'http://127.0.0.1:4601/__ready', reuseExistingServer, timeout: 180_000, }, { command: - 'node scripts/react-cockpit/serve.mjs memory --no-parent', + 'node scripts/react-cockpit/serve.mjs memory --no-parent --backend', cwd: '../..', - url: 'http://127.0.0.1:4602', + url: 'http://127.0.0.1:4602/__ready', reuseExistingServer, timeout: 180_000, }, { command: - 'node scripts/react-cockpit/serve.mjs client-tools --no-parent', + 'node scripts/react-cockpit/serve.mjs client-tools --no-parent --backend', cwd: '../..', - url: 'http://127.0.0.1:4603', + url: 'http://127.0.0.1:4603/__ready', reuseExistingServer, timeout: 180_000, }, { command: - 'node scripts/react-cockpit/serve.mjs persistence --no-parent', + 'node scripts/react-cockpit/serve.mjs persistence --no-parent --backend', cwd: '../..', - url: 'http://127.0.0.1:4604', + url: 'http://127.0.0.1:4604/__ready', reuseExistingServer, timeout: 180_000, }, { command: - 'node scripts/react-cockpit/serve.mjs durable-execution --no-parent', + 'node scripts/react-cockpit/serve.mjs durable-execution --no-parent --backend', cwd: '../..', - url: 'http://127.0.0.1:4605', + url: 'http://127.0.0.1:4605/__ready', reuseExistingServer, timeout: 180_000, }, { command: - 'node scripts/react-cockpit/serve.mjs subgraphs --no-parent', + 'node scripts/react-cockpit/serve.mjs subgraphs --no-parent --backend', cwd: '../..', - url: 'http://127.0.0.1:4606', + url: 'http://127.0.0.1:4606/__ready', reuseExistingServer, timeout: 180_000, }, { command: - 'node scripts/react-cockpit/serve.mjs time-travel --no-parent', + 'node scripts/react-cockpit/serve.mjs time-travel --no-parent --backend', cwd: '../..', - url: 'http://127.0.0.1:4607', + url: 'http://127.0.0.1:4607/__ready', reuseExistingServer, timeout: 180_000, }, { command: - 'node scripts/react-cockpit/serve.mjs deployment-runtime --no-parent', + 'node scripts/react-cockpit/serve.mjs deployment-runtime --no-parent --backend', cwd: '../..', - url: 'http://127.0.0.1:4608', + url: 'http://127.0.0.1:4608/__ready', reuseExistingServer, timeout: 180_000, }, { - command: 'node scripts/react-cockpit/serve.mjs ag-ui-streaming --no-parent', + command: 'node scripts/react-cockpit/serve.mjs ag-ui-streaming --no-parent --backend', cwd: '../..', - url: 'http://127.0.0.1:4609', + url: 'http://127.0.0.1:4609/__ready', reuseExistingServer, timeout: 180_000, }, { - command: 'node scripts/react-cockpit/serve.mjs ag-ui-interrupts --no-parent', + command: 'node scripts/react-cockpit/serve.mjs ag-ui-interrupts --no-parent --backend', cwd: '../..', - url: 'http://127.0.0.1:4610', + url: 'http://127.0.0.1:4610/__ready', reuseExistingServer, timeout: 180_000, }, { - command: 'node scripts/react-cockpit/serve.mjs ag-ui-tool-views --no-parent', + command: 'node scripts/react-cockpit/serve.mjs ag-ui-tool-views --no-parent --backend', cwd: '../..', - url: 'http://127.0.0.1:4611', + url: 'http://127.0.0.1:4611/__ready', reuseExistingServer, timeout: 180_000, }, { - command: 'node scripts/react-cockpit/prepare-ag-ui-client-tools-fixture.mjs --serve', + command: 'node scripts/react-cockpit/serve.mjs ag-ui-client-tools --no-parent --backend', cwd: '../..', - url: 'http://127.0.0.1:4640', - // A reused listener cannot establish canonical worker ownership. + url: 'http://127.0.0.1:4640/__ready', reuseExistingServer: false, timeout: 180_000, }, { - command: 'node scripts/react-cockpit/serve.mjs runtimes-mastra --no-parent', + command: 'node scripts/react-cockpit/serve.mjs runtimes-mastra --no-parent --backend', cwd: '../..', - url: 'http://127.0.0.1:4641', + url: 'http://127.0.0.1:4641/__ready', reuseExistingServer: false, timeout: 180_000, }, { - command: 'node scripts/react-cockpit/serve.mjs runtimes-microsoft-agent-framework --no-parent', + command: 'node scripts/react-cockpit/serve.mjs runtimes-microsoft-agent-framework --no-parent --backend', cwd: '../..', - url: 'http://127.0.0.1:4642', + url: 'http://127.0.0.1:4642/__ready', reuseExistingServer: false, timeout: 180_000, }, { - command: 'node scripts/react-cockpit/serve.mjs runtimes-aws-strands --no-parent', + command: 'node scripts/react-cockpit/serve.mjs runtimes-aws-strands --no-parent --backend', cwd: '../..', - url: 'http://127.0.0.1:4643', + url: 'http://127.0.0.1:4643/__ready', reuseExistingServer: false, timeout: 180_000, }, { - command: 'npx nx run cockpit-ag-ui-json-render-python:smoke && node scripts/react-cockpit/serve.mjs ag-ui-json-render --no-parent', + command: 'node scripts/react-cockpit/serve.mjs ag-ui-json-render --no-parent --backend', cwd: '../..', - url: 'http://127.0.0.1:4612', + url: 'http://127.0.0.1:4612/__ready', reuseExistingServer, timeout: 180_000, }, { - command: 'npx nx run cockpit-ag-ui-subagents-python:smoke && node scripts/react-cockpit/serve.mjs ag-ui-subagents --no-parent', + command: 'node scripts/react-cockpit/serve.mjs ag-ui-subagents --no-parent --backend', cwd: '../..', - url: 'http://127.0.0.1:4613', + url: 'http://127.0.0.1:4613/__ready', reuseExistingServer, timeout: 180_000, }, @@ -434,23 +448,23 @@ export const createWebsitePlaywrightConfig = ( timeout: 180_000, }, { - command: 'node scripts/react-cockpit/serve.mjs chat-theming --no-parent', + command: 'node scripts/react-cockpit/serve.mjs chat-theming --no-parent --backend', cwd: '../..', - url: 'http://127.0.0.1:4644', + url: 'http://127.0.0.1:4644/__ready', reuseExistingServer: false, gracefulShutdown: { signal: 'SIGTERM' as const, timeout: 5000 }, timeout: 180_000, }, { - command: 'node scripts/react-cockpit/serve.mjs chat-debug --no-parent', + command: 'node scripts/react-cockpit/serve.mjs chat-debug --no-parent --backend', cwd: '../..', - url: 'http://127.0.0.1:4645', + url: 'http://127.0.0.1:4645/__ready', reuseExistingServer: false, gracefulShutdown: { signal: 'SIGTERM' as const, timeout: 5000 }, timeout: 180_000, }, { - command: 'node scripts/react-cockpit/serve.mjs chat-a2ui --no-parent', + command: 'node scripts/react-cockpit/serve.mjs chat-a2ui --no-parent --backend', cwd: '../..', url: 'http://127.0.0.1:4646', reuseExistingServer: false, @@ -479,42 +493,42 @@ export const createWebsitePlaywrightConfig = ( timeout: 180_000, }, { - command: 'npx nx run cockpit-deep-agents-planning-python:smoke && node scripts/react-cockpit/serve.mjs deep-agents-planning --no-parent', + command: 'node scripts/react-cockpit/serve.mjs deep-agents-planning --no-parent --backend', cwd: '../..', - url: 'http://127.0.0.1:4628', + url: 'http://127.0.0.1:4628/__ready', reuseExistingServer, timeout: 180_000, }, { - command: 'npx nx run cockpit-deep-agents-filesystem-python:smoke && node scripts/react-cockpit/serve.mjs deep-agents-filesystem --no-parent', + command: 'node scripts/react-cockpit/serve.mjs deep-agents-filesystem --no-parent --backend', cwd: '../..', url: 'http://127.0.0.1:4629', reuseExistingServer, timeout: 180_000, }, { - command: 'npx nx run cockpit-deep-agents-subagents-python:smoke && node scripts/react-cockpit/serve.mjs deep-agents-subagents --no-parent', + command: 'node scripts/react-cockpit/serve.mjs deep-agents-subagents --no-parent --backend', cwd: '../..', - url: 'http://127.0.0.1:4630', + url: 'http://127.0.0.1:4630/__ready', reuseExistingServer, timeout: 180_000, }, { - command: 'npx nx run cockpit-deep-agents-skills-python:smoke && node scripts/react-cockpit/serve.mjs deep-agents-skills --no-parent', + command: 'node scripts/react-cockpit/serve.mjs deep-agents-skills --no-parent --backend', cwd: '../..', url: 'http://127.0.0.1:4631', reuseExistingServer, timeout: 180_000, }, { - command: 'npx nx run cockpit-deep-agents-memory-python:smoke && node scripts/react-cockpit/serve.mjs deep-agents-memory --no-parent', + command: 'node scripts/react-cockpit/serve.mjs deep-agents-memory --no-parent --backend', cwd: '../..', url: 'http://127.0.0.1:4632', reuseExistingServer, timeout: 180_000, }, { - command: 'npx nx run cockpit-chat-generative-ui-python:smoke && node scripts/react-cockpit/serve.mjs chat-generative-ui --no-parent', + command: 'node scripts/react-cockpit/serve.mjs chat-generative-ui --no-parent --backend', cwd: '../..', url: 'http://127.0.0.1:4627', reuseExistingServer, @@ -550,7 +564,7 @@ export const createWebsitePlaywrightConfig = ( config.testMatch = '**/react-ag-ui-client-tools-preview.spec.ts'; if (Array.isArray(config.webServer)) { config.webServer = config.webServer.filter(server => - [localURL, 'http://localhost:4325', 'http://127.0.0.1:4640'].includes(server.url ?? '') + [localURL, 'http://localhost:4325', 'http://127.0.0.1:4640/__ready'].includes(server.url ?? '') ); } } @@ -558,7 +572,7 @@ export const createWebsitePlaywrightConfig = ( config.testMatch = '**/react-mastra-preview.spec.ts'; if (Array.isArray(config.webServer)) { config.webServer = config.webServer.filter(server => - [localURL, 'http://127.0.0.1:4641'].includes(server.url ?? '') + [localURL, 'http://127.0.0.1:4641/__ready'].includes(server.url ?? '') ); } } @@ -570,7 +584,7 @@ export const createWebsitePlaywrightConfig = ( config.outputDir = resolve(__dirname, '../../test-results/native-maf-website'); if (Array.isArray(config.webServer)) { config.webServer = config.webServer.filter(server => - [localURL, 'http://127.0.0.1:4642'].includes(server.url ?? '') + [localURL, 'http://127.0.0.1:4642/__ready'].includes(server.url ?? '') ); } } @@ -582,7 +596,7 @@ export const createWebsitePlaywrightConfig = ( config.outputDir = resolve(__dirname, '../../test-results/native-strands-website'); if (Array.isArray(config.webServer)) { config.webServer = config.webServer.filter(server => - [localURL, 'http://127.0.0.1:4643'].includes(server.url ?? '') + [localURL, 'http://127.0.0.1:4643/__ready'].includes(server.url ?? '') ); } } @@ -593,7 +607,7 @@ export const createWebsitePlaywrightConfig = ( config.retries = 0; config.outputDir = resolve(__dirname, '../../output/playwright/chat-theming-website'); if (Array.isArray(config.webServer)) { - config.webServer = config.webServer.filter(server => [localURL, 'http://127.0.0.1:4644'].includes(server.url ?? '')).map(server => ({ ...server, reuseExistingServer: false, gracefulShutdown: { signal: 'SIGTERM' as const, timeout: 5000 } })); + config.webServer = config.webServer.filter(server => [localURL, 'http://127.0.0.1:4644/__ready'].includes(server.url ?? '')).map(server => ({ ...server, reuseExistingServer: false, gracefulShutdown: { signal: 'SIGTERM' as const, timeout: 5000 } })); } } if (environment['WEBSITE_E2E_REACT_TOPIC'] === 'chat-debug') { @@ -604,7 +618,7 @@ export const createWebsitePlaywrightConfig = ( config.outputDir = resolve(__dirname, '../../output/playwright/chat-debug-website'); if (Array.isArray(config.webServer)) { config.webServer = config.webServer - .filter(server => [localURL, 'http://127.0.0.1:4645'].includes(server.url ?? '')) + .filter(server => [localURL, 'http://127.0.0.1:4645/__ready'].includes(server.url ?? '')) .map(server => ({ ...server, reuseExistingServer: false, gracefulShutdown: { signal: 'SIGTERM' as const, timeout: 5000 } })); } } diff --git a/apps/website/project.json b/apps/website/project.json index 9372127c0..07eddd3e7 100644 --- a/apps/website/project.json +++ b/apps/website/project.json @@ -256,14 +256,6 @@ "executor": "@nx/playwright:playwright", "cache": false, "dependsOn": [ - { - "target": "prepare-fixture", - "projects": [ - "cockpit-runtimes-mastra-react", - "cockpit-runtimes-microsoft-agent-framework-react", - "cockpit-runtimes-aws-strands-react" - ] - }, { "target": "build", "projects": [ @@ -327,7 +319,7 @@ }, { "target": "prepare-fixture", - "projects": ["cockpit-runtimes-mastra-react", "cockpit-runtimes-microsoft-agent-framework-react", "cockpit-runtimes-aws-strands-react", "cockpit-ag-ui-a2ui-react"] + "projects": ["cockpit-ag-ui-a2ui-react"] }, { "target": "build", diff --git a/apps/website/src/e2e-suites.spec.ts b/apps/website/src/e2e-suites.spec.ts index 1b9f597f0..ebd4e2851 100644 --- a/apps/website/src/e2e-suites.spec.ts +++ b/apps/website/src/e2e-suites.spec.ts @@ -185,10 +185,10 @@ describe('bounded Angular startup', () => { }); describe('bounded Agents startup', () => { - it('rejects missing, duplicate or interleaved smoke/server chains', () => { + it('rejects missing, duplicate or interleaved preview servers', () => { const config = createWebsiteSuitePlaywrightConfig('agents', { CI: 'true' }); const servers = Array.isArray(config.webServer) ? config.webServer : []; - const first = servers.findIndex(s => s.command.startsWith('npx nx run cockpit-deep-agents-')); + const first = servers.findIndex(s => s.command.startsWith('node scripts/react-cockpit/serve.mjs deep-agents-')); expect(first).toBeGreaterThan(0); expect(() => poolAgentServers(servers.filter((_, i) => i !== first))).toThrow(/five/); const duplicates = [...servers]; @@ -199,7 +199,7 @@ describe('bounded Agents startup', () => { expect(() => poolAgentServers(interleaved)).toThrow(/contiguous/); }); - it('preserves all five complete smoke/server commands, startup barriers and browser settings', () => { + it('preserves all five complete preview server commands, startup barriers and browser settings', () => { const environment = { CI: 'true', WEBSITE_E2E_DEPLOYED_OWNER: 'true' }; const original = createWebsiteSuitePlaywrightConfig('agents', environment); const pooled = createWebsiteSuitePlaywrightConfig('agents', { @@ -212,7 +212,7 @@ describe('bounded Agents startup', () => { expect(children).toHaveLength(5); expect(servers.flatMap(s => s === pool ? children : [s])).toEqual(original.webServer); expect(children.map((s: { url: string }) => new URL(s.url).port).sort()).toEqual(['4628', '4629', '4630', '4631', '4632']); - expect(children.every((s: { command: string; timeout: number }) => s.command.includes(':smoke && node ') && s.timeout === 180_000)).toBe(true); + expect(children.every((s: { command: string; timeout: number }) => s.command.endsWith(' --no-parent --backend') && s.timeout === 180_000)).toBe(true); expect(pool).toMatchObject({ cwd: '../..', timeout: 900_000, wait: { stdout: /Website Agents servers ready/ }, diff --git a/apps/website/src/playwright-config.spec.ts b/apps/website/src/playwright-config.spec.ts index c8669afeb..02acc502d 100644 --- a/apps/website/src/playwright-config.spec.ts +++ b/apps/website/src/playwright-config.spec.ts @@ -83,48 +83,50 @@ it('scopes Chat Theming Website to two owned servers with zero retries and focus expect(config.retries).toBe(0); expect(config.outputDir).toContain('chat-theming-website'); const servers = Array.isArray(config.webServer) ? config.webServer : []; - expect(servers.map(server => server.url)).toEqual(['http://127.0.0.1:4308', 'http://127.0.0.1:4644']); + expect(servers.map(server => server.url)).toEqual(['http://127.0.0.1:4308', 'http://127.0.0.1:4644/__ready']); expect(servers.every(server => server.reuseExistingServer === false)).toBe(true); - expect(servers[1]).toMatchObject({ command: 'node scripts/react-cockpit/serve.mjs chat-theming --no-parent', gracefulShutdown: { signal: 'SIGTERM', timeout: 5000 } }); + expect(servers[1]).toMatchObject({ command: 'node scripts/react-cockpit/serve.mjs chat-theming --no-parent --backend', gracefulShutdown: { signal: 'SIGTERM', timeout: 5000 } }); const project = JSON.parse(readFileSync(resolve(import.meta.dirname, '../project.json'), 'utf8')); expect(project.implicitDependencies).toContain('cockpit-chat-theming-react'); expect(project.targets.e2e.dependsOn.find((dependency: { target?: string }) => dependency.target === 'build').projects).toContain('cockpit-chat-theming-react'); expect(project.targets.e2e.dependsOn.find((dependency: { target?: string }) => dependency.target === 'test').projects).toContain('cockpit-chat-theming-python'); }); -it('scopes native MAF Website checks to the owned fixture after frozen preparation and build', () => { +it('scopes native MAF Website checks to the aimock-backed preview after its build', () => { const config = createWebsitePlaywrightConfig({ WEBSITE_E2E_REACT_TOPIC: 'runtimes-microsoft-agent-framework', WEBSITE_E2E_PORT: '4390', PLAYWRIGHT_REUSE_EXISTING_SERVER: 'true' }); expect(config.testMatch).toBe('**/react-maf-preview.spec.ts'); const servers = Array.isArray(config.webServer) ? config.webServer : []; - expect(servers.map(server => server.url)).toEqual(['http://127.0.0.1:4390', 'http://127.0.0.1:4642']); - expect(servers[1]).toMatchObject({ command: 'node scripts/react-cockpit/serve.mjs runtimes-microsoft-agent-framework --no-parent', reuseExistingServer: false }); + expect(servers.map(server => server.url)).toEqual(['http://127.0.0.1:4390', 'http://127.0.0.1:4642/__ready']); + expect(servers[1]).toMatchObject({ command: 'node scripts/react-cockpit/serve.mjs runtimes-microsoft-agent-framework --no-parent --backend', reuseExistingServer: false }); const project = JSON.parse(readFileSync(resolve(import.meta.dirname, '../project.json'), 'utf8')); const name = 'cockpit-runtimes-microsoft-agent-framework-react'; expect(project.implicitDependencies).toContain(name); - for (const target of ['build', 'prepare-fixture']) expect(project.targets.e2e.dependsOn.find((dependency: { target?: string }) => dependency.target === target).projects).toContain(name); + expect(project.targets.e2e.dependsOn.find((dependency: { target?: string }) => dependency.target === 'build').projects).toContain(name); + expect(project.targets.e2e.dependsOn.find((dependency: { target?: string }) => dependency.target === 'prepare-fixture').projects).not.toContain(name); }); -it('scopes native Strands Website checks to the owned fixture after frozen preparation and build', () => { +it('scopes native Strands Website checks to the aimock-backed preview after its build', () => { const config = createWebsitePlaywrightConfig({ WEBSITE_E2E_REACT_TOPIC: 'runtimes-aws-strands', WEBSITE_E2E_PORT: '4390', PLAYWRIGHT_REUSE_EXISTING_SERVER: 'true' }); expect(config.workers).toBe(1); expect(config.fullyParallel).toBe(false); expect(config.retries).toBe(0); expect(config.testMatch).toBe('**/react-strands-preview.spec.ts'); const servers = Array.isArray(config.webServer) ? config.webServer : []; - expect(servers.map(server => server.url)).toEqual(['http://127.0.0.1:4390', 'http://127.0.0.1:4643']); - expect(servers[1]).toMatchObject({ command: 'node scripts/react-cockpit/serve.mjs runtimes-aws-strands --no-parent', reuseExistingServer: false }); + expect(servers.map(server => server.url)).toEqual(['http://127.0.0.1:4390', 'http://127.0.0.1:4643/__ready']); + expect(servers[1]).toMatchObject({ command: 'node scripts/react-cockpit/serve.mjs runtimes-aws-strands --no-parent --backend', reuseExistingServer: false }); const project = JSON.parse(readFileSync(resolve(import.meta.dirname, '../project.json'), 'utf8')); const name = 'cockpit-runtimes-aws-strands-react'; expect(project.implicitDependencies).toContain(name); - for (const target of ['build', 'prepare-fixture']) expect(project.targets.e2e.dependsOn.find((dependency: { target?: string }) => dependency.target === target).projects).toContain(name); + expect(project.targets.e2e.dependsOn.find((dependency: { target?: string }) => dependency.target === 'build').projects).toContain(name); + expect(project.targets.e2e.dependsOn.find((dependency: { target?: string }) => dependency.target === 'prepare-fixture').projects).not.toContain(name); }); -it('scopes native Mastra Website checks to the owned actual service fixture', () => { +it('scopes native Mastra Website checks to the aimock-backed preview', () => { const config = createWebsitePlaywrightConfig({ WEBSITE_E2E_REACT_TOPIC: 'runtimes-mastra', WEBSITE_E2E_PORT: '4390' }); expect(config.testMatch).toBe('**/react-mastra-preview.spec.ts'); const servers = Array.isArray(config.webServer) ? config.webServer : []; - expect(servers.map(server => server.url)).toEqual(['http://127.0.0.1:4390', 'http://127.0.0.1:4641']); - expect(servers[1]).toMatchObject({ command: 'node scripts/react-cockpit/serve.mjs runtimes-mastra --no-parent', reuseExistingServer: false }); + expect(servers.map(server => server.url)).toEqual(['http://127.0.0.1:4390', 'http://127.0.0.1:4641/__ready']); + expect(servers[1]).toMatchObject({ command: 'node scripts/react-cockpit/serve.mjs runtimes-mastra --no-parent --backend', reuseExistingServer: false }); }); describe('Website Playwright configuration', () => { @@ -145,9 +147,9 @@ describe('Website Playwright configuration', () => { const config = createWebsitePlaywrightConfig({ CI: 'true' }); const servers = Array.isArray(config.webServer) ? config.webServer : []; const nativeServer = servers.find( - server => server.url === 'http://127.0.0.1:4600' + server => server.url === 'http://127.0.0.1:4600/__ready' ); - expect(nativeServer?.command).toBe('node scripts/react-cockpit/serve.mjs'); + expect(nativeServer?.command).toBe('node scripts/react-cockpit/serve.mjs streaming --backend'); expect(nativeServer?.timeout).toBe(180_000); expect(nativeServer?.reuseExistingServer).toBe(false); }); @@ -177,7 +179,7 @@ describe('Website Playwright configuration', () => { const config = createWebsitePlaywrightConfig({CI:'true'}); const servers = Array.isArray(config.webServer) ? config.webServer : []; expect(servers.find(server => server.url === 'http://localhost:4322')?.command).toBe('npx nx run cockpit-ag-ui-tool-views-angular:serve:cockpit --port 4322'); - expect(servers.find(server => server.url === 'http://127.0.0.1:4611')?.command).toBe('node scripts/react-cockpit/serve.mjs ag-ui-tool-views --no-parent'); + expect(servers.find(server => server.url === 'http://127.0.0.1:4611/__ready')?.command).toBe('node scripts/react-cockpit/serve.mjs ag-ui-tool-views --no-parent --backend'); const project = JSON.parse(readFileSync(resolve(import.meta.dirname,'../project.json'),'utf8')); expect(project.implicitDependencies).toContain('cockpit-ag-ui-tool-views-angular'); }); @@ -185,10 +187,12 @@ describe('Website Playwright configuration', () => { const config = createWebsitePlaywrightConfig({ CI: 'true' }); const servers = Array.isArray(config.webServer) ? config.webServer : []; const expectedUrls = getCockpitFrontends().filter(frontend => frontend.frontend === 'react').map(frontend => `http://127.0.0.1:${frontend.port}`).sort(); - const reactServers = servers.filter(server => expectedUrls.includes(server.url ?? '')); - expect(reactServers.map(server => server.url).sort()).toEqual(expectedUrls); + // Previews whose specs drive a backend wait on `/__ready`. + const origin = (url = '') => (URL.canParse(url) ? new URL(url).origin : url); + const reactServers = servers.filter(server => expectedUrls.includes(origin(server.url))); + expect(reactServers.map(server => origin(server.url)).sort()).toEqual(expectedUrls); expect(reactServers.every(server => server.reuseExistingServer === false)).toBe(true); - expect(reactServers.find(server => server.url === 'http://127.0.0.1:4610')?.command).toBe('node scripts/react-cockpit/serve.mjs ag-ui-interrupts --no-parent'); + expect(reactServers.find(server => server.url === 'http://127.0.0.1:4610/__ready')?.command).toBe('node scripts/react-cockpit/serve.mjs ag-ui-interrupts --no-parent --backend'); const project = JSON.parse(readFileSync(resolve(import.meta.dirname, '../project.json'), 'utf8')); const projects = project.targets.e2e.dependsOn.find( (dependency: { target?: string }) => dependency.target === 'build' @@ -215,12 +219,12 @@ describe('Website Playwright configuration', () => { expect(projects).toContain('cockpit-deep-agents-subagents-react'); expect(projects).toContain('cockpit-deep-agents-skills-react'); expect(projects).toContain('cockpit-deep-agents-memory-react'); - expect(reactServers.find(server => server.url === 'http://127.0.0.1:4632')?.command).toBe('npx nx run cockpit-deep-agents-memory-python:smoke && node scripts/react-cockpit/serve.mjs deep-agents-memory --no-parent'); - expect(reactServers.find(server => server.url === 'http://127.0.0.1:4631')?.command).toBe('npx nx run cockpit-deep-agents-skills-python:smoke && node scripts/react-cockpit/serve.mjs deep-agents-skills --no-parent'); - expect(reactServers.find(server => server.url === 'http://127.0.0.1:4630')?.command).toBe('npx nx run cockpit-deep-agents-subagents-python:smoke && node scripts/react-cockpit/serve.mjs deep-agents-subagents --no-parent'); - expect(reactServers.find(server => server.url === 'http://127.0.0.1:4629')?.command).toBe('npx nx run cockpit-deep-agents-filesystem-python:smoke && node scripts/react-cockpit/serve.mjs deep-agents-filesystem --no-parent'); - expect(reactServers.find(server => server.url === 'http://127.0.0.1:4628')?.command).toBe('npx nx run cockpit-deep-agents-planning-python:smoke && node scripts/react-cockpit/serve.mjs deep-agents-planning --no-parent'); - expect(reactServers.find(server => server.url === 'http://127.0.0.1:4627')?.command).toBe('npx nx run cockpit-chat-generative-ui-python:smoke && node scripts/react-cockpit/serve.mjs chat-generative-ui --no-parent'); + expect(reactServers.find(server => server.url === 'http://127.0.0.1:4632')?.command).toBe('node scripts/react-cockpit/serve.mjs deep-agents-memory --no-parent --backend'); + expect(reactServers.find(server => server.url === 'http://127.0.0.1:4631')?.command).toBe('node scripts/react-cockpit/serve.mjs deep-agents-skills --no-parent --backend'); + expect(reactServers.find(server => server.url === 'http://127.0.0.1:4630/__ready')?.command).toBe('node scripts/react-cockpit/serve.mjs deep-agents-subagents --no-parent --backend'); + expect(reactServers.find(server => server.url === 'http://127.0.0.1:4629')?.command).toBe('node scripts/react-cockpit/serve.mjs deep-agents-filesystem --no-parent --backend'); + expect(reactServers.find(server => server.url === 'http://127.0.0.1:4628/__ready')?.command).toBe('node scripts/react-cockpit/serve.mjs deep-agents-planning --no-parent --backend'); + expect(reactServers.find(server => server.url === 'http://127.0.0.1:4627')?.command).toBe('node scripts/react-cockpit/serve.mjs chat-generative-ui --no-parent --backend'); expect(reactServers.find(server => server.url === 'http://127.0.0.1:4626')?.command).toBe('npx nx run cockpit-chat-timeline-python:smoke && node scripts/react-cockpit/serve.mjs chat-timeline --no-parent'); expect(reactServers.find(server => server.url === 'http://127.0.0.1:4625')?.command).toBe('npx nx run cockpit-chat-threads-python:smoke && node scripts/react-cockpit/serve.mjs chat-threads --no-parent'); expect(projects).toContain('cockpit-chat-subagents-react'); @@ -233,8 +237,8 @@ describe('Website Playwright configuration', () => { expect(reactServers.find(server => server.url === 'http://127.0.0.1:4622')?.command).toBe('npx nx run cockpit-chat-interrupts-python:smoke && node scripts/react-cockpit/serve.mjs chat-interrupts --no-parent'); expect(reactServers.find(server => server.url === 'http://127.0.0.1:4619')?.command).toBe('node scripts/react-cockpit/serve.mjs render-computed-functions --no-parent'); expect(reactServers.find(server => server.url === 'http://127.0.0.1:4614')?.command).toBe('node scripts/react-cockpit/serve.mjs render-spec-rendering --no-parent'); - expect(reactServers.find(server => server.url === 'http://127.0.0.1:4613')?.command).toBe('npx nx run cockpit-ag-ui-subagents-python:smoke && node scripts/react-cockpit/serve.mjs ag-ui-subagents --no-parent'); - expect(reactServers.find(server=>server.url==='http://127.0.0.1:4612')?.command).toBe('npx nx run cockpit-ag-ui-json-render-python:smoke && node scripts/react-cockpit/serve.mjs ag-ui-json-render --no-parent'); + expect(reactServers.find(server => server.url === 'http://127.0.0.1:4613/__ready')?.command).toBe('node scripts/react-cockpit/serve.mjs ag-ui-subagents --no-parent --backend'); + expect(reactServers.find(server=>server.url==='http://127.0.0.1:4612/__ready')?.command).toBe('node scripts/react-cockpit/serve.mjs ag-ui-json-render --no-parent --backend'); }); it('sends the Vercel automation bypass only when CI supplies it', () => { const withoutSecret = createWebsitePlaywrightConfig({}); @@ -408,10 +412,10 @@ describe('Website Playwright configuration', () => { url: 'http://localhost:4300', }), expect.objectContaining({ - command: 'node scripts/react-cockpit/prepare-ag-ui-a2ui-fixture.mjs --serve', + command: 'node scripts/react-cockpit/serve.mjs ag-ui-a2ui --no-parent --backend', url: 'http://127.0.0.1:4647', reuseExistingServer: false, - timeout: 330_000, + timeout: 180_000, }), expect.objectContaining({ command: expect.stringContaining( @@ -454,62 +458,62 @@ describe('Website Playwright configuration', () => { url: 'http://127.0.0.1:4399/health', }), expect.objectContaining({ - command: 'node scripts/react-cockpit/serve.mjs', - url: 'http://127.0.0.1:4600', + command: 'node scripts/react-cockpit/serve.mjs streaming --backend', + url: 'http://127.0.0.1:4600/__ready', }), expect.objectContaining({ - command: 'node scripts/react-cockpit/serve.mjs interrupts --no-parent', - url: 'http://127.0.0.1:4601', + command: 'node scripts/react-cockpit/serve.mjs interrupts --no-parent --backend', + url: 'http://127.0.0.1:4601/__ready', }), expect.objectContaining({ - command: 'node scripts/react-cockpit/serve.mjs memory --no-parent', - url: 'http://127.0.0.1:4602', + command: 'node scripts/react-cockpit/serve.mjs memory --no-parent --backend', + url: 'http://127.0.0.1:4602/__ready', }), expect.objectContaining({ command: - 'node scripts/react-cockpit/serve.mjs client-tools --no-parent', - url: 'http://127.0.0.1:4603', + 'node scripts/react-cockpit/serve.mjs client-tools --no-parent --backend', + url: 'http://127.0.0.1:4603/__ready', }), expect.objectContaining({ - command: 'node scripts/react-cockpit/serve.mjs persistence --no-parent', - url: 'http://127.0.0.1:4604', + command: 'node scripts/react-cockpit/serve.mjs persistence --no-parent --backend', + url: 'http://127.0.0.1:4604/__ready', }), expect.objectContaining({ command: - 'node scripts/react-cockpit/serve.mjs durable-execution --no-parent', - url: 'http://127.0.0.1:4605', + 'node scripts/react-cockpit/serve.mjs durable-execution --no-parent --backend', + url: 'http://127.0.0.1:4605/__ready', }), expect.objectContaining({ - command: 'node scripts/react-cockpit/serve.mjs subgraphs --no-parent', - url: 'http://127.0.0.1:4606', + command: 'node scripts/react-cockpit/serve.mjs subgraphs --no-parent --backend', + url: 'http://127.0.0.1:4606/__ready', }), expect.objectContaining({ - command: 'node scripts/react-cockpit/serve.mjs time-travel --no-parent', - url: 'http://127.0.0.1:4607', + command: 'node scripts/react-cockpit/serve.mjs time-travel --no-parent --backend', + url: 'http://127.0.0.1:4607/__ready', }), expect.objectContaining({ command: - 'node scripts/react-cockpit/serve.mjs deployment-runtime --no-parent', - url: 'http://127.0.0.1:4608', + 'node scripts/react-cockpit/serve.mjs deployment-runtime --no-parent --backend', + url: 'http://127.0.0.1:4608/__ready', }), expect.objectContaining({ - command: 'node scripts/react-cockpit/serve.mjs ag-ui-streaming --no-parent', - url: 'http://127.0.0.1:4609', + command: 'node scripts/react-cockpit/serve.mjs ag-ui-streaming --no-parent --backend', + url: 'http://127.0.0.1:4609/__ready', }), expect.objectContaining({ - command: 'node scripts/react-cockpit/serve.mjs ag-ui-interrupts --no-parent', - url: 'http://127.0.0.1:4610', + command: 'node scripts/react-cockpit/serve.mjs ag-ui-interrupts --no-parent --backend', + url: 'http://127.0.0.1:4610/__ready', }), expect.objectContaining({ - command: 'node scripts/react-cockpit/serve.mjs ag-ui-tool-views --no-parent', - url: 'http://127.0.0.1:4611', + command: 'node scripts/react-cockpit/serve.mjs ag-ui-tool-views --no-parent --backend', + url: 'http://127.0.0.1:4611/__ready', }), - expect.objectContaining({command:'node scripts/react-cockpit/prepare-ag-ui-client-tools-fixture.mjs --serve',url:'http://127.0.0.1:4640'}), - expect.objectContaining({command:'node scripts/react-cockpit/serve.mjs runtimes-mastra --no-parent',url:'http://127.0.0.1:4641'}), - expect.objectContaining({command:'node scripts/react-cockpit/serve.mjs runtimes-microsoft-agent-framework --no-parent',url:'http://127.0.0.1:4642'}), - expect.objectContaining({command:'node scripts/react-cockpit/serve.mjs runtimes-aws-strands --no-parent',url:'http://127.0.0.1:4643'}), - expect.objectContaining({command:'npx nx run cockpit-ag-ui-json-render-python:smoke && node scripts/react-cockpit/serve.mjs ag-ui-json-render --no-parent',url:'http://127.0.0.1:4612'}), - expect.objectContaining({command:'npx nx run cockpit-ag-ui-subagents-python:smoke && node scripts/react-cockpit/serve.mjs ag-ui-subagents --no-parent',url:'http://127.0.0.1:4613'}), + expect.objectContaining({command:'node scripts/react-cockpit/serve.mjs ag-ui-client-tools --no-parent --backend',url:'http://127.0.0.1:4640/__ready'}), + expect.objectContaining({command:'node scripts/react-cockpit/serve.mjs runtimes-mastra --no-parent --backend',url:'http://127.0.0.1:4641/__ready'}), + expect.objectContaining({command:'node scripts/react-cockpit/serve.mjs runtimes-microsoft-agent-framework --no-parent --backend',url:'http://127.0.0.1:4642/__ready'}), + expect.objectContaining({command:'node scripts/react-cockpit/serve.mjs runtimes-aws-strands --no-parent --backend',url:'http://127.0.0.1:4643/__ready'}), + expect.objectContaining({command:'node scripts/react-cockpit/serve.mjs ag-ui-json-render --no-parent --backend',url:'http://127.0.0.1:4612/__ready'}), + expect.objectContaining({command:'node scripts/react-cockpit/serve.mjs ag-ui-subagents --no-parent --backend',url:'http://127.0.0.1:4613/__ready'}), expect.objectContaining({command:'node scripts/react-cockpit/serve.mjs render-spec-rendering --no-parent',url:'http://127.0.0.1:4614'}), expect.objectContaining({command:'node scripts/react-cockpit/serve.mjs render-state-management --no-parent',url:'http://127.0.0.1:4615'}), expect.objectContaining({command:'node scripts/react-cockpit/serve.mjs render-repeat-loops --no-parent',url:'http://127.0.0.1:4616'}), @@ -517,18 +521,18 @@ describe('Website Playwright configuration', () => { expect.objectContaining({command:'node scripts/react-cockpit/serve.mjs render-element-rendering --no-parent',url:'http://127.0.0.1:4618'}), expect.objectContaining({command:'node scripts/react-cockpit/serve.mjs render-computed-functions --no-parent',url:'http://127.0.0.1:4619'}), expect.objectContaining({command:'npx nx run cockpit-chat-messages-python:smoke && node scripts/react-cockpit/serve.mjs chat-messages --no-parent',url:'http://127.0.0.1:4620'}), - expect.objectContaining({command:'node scripts/react-cockpit/serve.mjs chat-theming --no-parent',url:'http://127.0.0.1:4644'}), - expect.objectContaining({command:'node scripts/react-cockpit/serve.mjs chat-debug --no-parent',url:'http://127.0.0.1:4645'}), - expect.objectContaining({command:'node scripts/react-cockpit/serve.mjs chat-a2ui --no-parent',url:'http://127.0.0.1:4646'}), + expect.objectContaining({command:'node scripts/react-cockpit/serve.mjs chat-theming --no-parent --backend',url:'http://127.0.0.1:4644/__ready'}), + expect.objectContaining({command:'node scripts/react-cockpit/serve.mjs chat-debug --no-parent --backend',url:'http://127.0.0.1:4645/__ready'}), + expect.objectContaining({command:'node scripts/react-cockpit/serve.mjs chat-a2ui --no-parent --backend',url:'http://127.0.0.1:4646'}), expect.objectContaining({command:'npx nx run cockpit-chat-input-python:smoke && node scripts/react-cockpit/serve.mjs chat-input --no-parent',url:'http://127.0.0.1:4621'}), expect.objectContaining({command:'npx nx run cockpit-chat-interrupts-python:smoke && node scripts/react-cockpit/serve.mjs chat-interrupts --no-parent',url:'http://127.0.0.1:4622'}), expect.objectContaining({command:'npx nx run cockpit-chat-tool-calls-python:smoke && node scripts/react-cockpit/serve.mjs chat-tool-calls --no-parent',url:'http://127.0.0.1:4623'}), - expect.objectContaining({command:'npx nx run cockpit-deep-agents-planning-python:smoke && node scripts/react-cockpit/serve.mjs deep-agents-planning --no-parent',url:'http://127.0.0.1:4628'}), - expect.objectContaining({command:'npx nx run cockpit-deep-agents-filesystem-python:smoke && node scripts/react-cockpit/serve.mjs deep-agents-filesystem --no-parent',url:'http://127.0.0.1:4629'}), - expect.objectContaining({command:'npx nx run cockpit-deep-agents-subagents-python:smoke && node scripts/react-cockpit/serve.mjs deep-agents-subagents --no-parent',url:'http://127.0.0.1:4630'}), - expect.objectContaining({command:'npx nx run cockpit-deep-agents-skills-python:smoke && node scripts/react-cockpit/serve.mjs deep-agents-skills --no-parent',url:'http://127.0.0.1:4631'}), - expect.objectContaining({command:'npx nx run cockpit-deep-agents-memory-python:smoke && node scripts/react-cockpit/serve.mjs deep-agents-memory --no-parent',url:'http://127.0.0.1:4632'}), - expect.objectContaining({command:'npx nx run cockpit-chat-generative-ui-python:smoke && node scripts/react-cockpit/serve.mjs chat-generative-ui --no-parent',url:'http://127.0.0.1:4627'}), + expect.objectContaining({command:'node scripts/react-cockpit/serve.mjs deep-agents-planning --no-parent --backend',url:'http://127.0.0.1:4628/__ready'}), + expect.objectContaining({command:'node scripts/react-cockpit/serve.mjs deep-agents-filesystem --no-parent --backend',url:'http://127.0.0.1:4629'}), + expect.objectContaining({command:'node scripts/react-cockpit/serve.mjs deep-agents-subagents --no-parent --backend',url:'http://127.0.0.1:4630/__ready'}), + expect.objectContaining({command:'node scripts/react-cockpit/serve.mjs deep-agents-skills --no-parent --backend',url:'http://127.0.0.1:4631'}), + expect.objectContaining({command:'node scripts/react-cockpit/serve.mjs deep-agents-memory --no-parent --backend',url:'http://127.0.0.1:4632'}), + expect.objectContaining({command:'node scripts/react-cockpit/serve.mjs chat-generative-ui --no-parent --backend',url:'http://127.0.0.1:4627'}), expect.objectContaining({command:'npx nx run cockpit-chat-timeline-python:smoke && node scripts/react-cockpit/serve.mjs chat-timeline --no-parent',url:'http://127.0.0.1:4626'}), expect.objectContaining({command:'npx nx run cockpit-chat-threads-python:smoke && node scripts/react-cockpit/serve.mjs chat-threads --no-parent',url:'http://127.0.0.1:4625'}), expect.objectContaining({command:'npx nx run cockpit-chat-subagents-python:smoke && node scripts/react-cockpit/serve.mjs chat-subagents --no-parent',url:'http://127.0.0.1:4624'}), @@ -575,11 +579,11 @@ describe('Website Playwright configuration', () => { }); }); -it('owns frozen AG-UI Client Tools preparation without reusing a foreign fixture', () => { +it('owns the AG-UI Client Tools preview and its aimock backend without reusing a foreign server', () => { const config = createWebsitePlaywrightConfig({ PLAYWRIGHT_REUSE_EXISTING_SERVER: 'true' }); const servers = Array.isArray(config.webServer) ? config.webServer : []; - expect(servers.find(server => server.url === 'http://127.0.0.1:4640')).toMatchObject({ - command: 'node scripts/react-cockpit/prepare-ag-ui-client-tools-fixture.mjs --serve', + expect(servers.find(server => server.url === 'http://127.0.0.1:4640/__ready')).toMatchObject({ + command: 'node scripts/react-cockpit/serve.mjs ag-ui-client-tools --no-parent --backend', timeout: 180_000, reuseExistingServer: false, }); const project = JSON.parse(readFileSync(resolve(import.meta.dirname, '../project.json'), 'utf8')); @@ -592,7 +596,7 @@ it('scopes native Chat A2UI checks to Website and the exact owned React server', const servers = Array.isArray(config.webServer) ? config.webServer : []; expect(servers.map(server => server.url)).toEqual(['http://127.0.0.1:4390', 'http://127.0.0.1:4646']); expect(servers.every(server => server.reuseExistingServer === false)).toBe(true); - expect(servers[1]?.command).toBe('node scripts/react-cockpit/serve.mjs chat-a2ui --no-parent'); + expect(servers[1]?.command).toBe('node scripts/react-cockpit/serve.mjs chat-a2ui --no-parent --backend'); expect(config.testMatch).toBe('**/react-chat-a2ui-preview.spec.ts'); expect(config.workers).toBe(1); expect(config.retries).toBe(0); @@ -603,7 +607,7 @@ it('scopes native AG-UI A2UI checks to Website and the prepared owned React serv const servers = Array.isArray(config.webServer) ? config.webServer : []; expect(servers.map(server => server.url)).toEqual(['http://127.0.0.1:4390', 'http://127.0.0.1:4647']); expect(servers.every(server => server.reuseExistingServer === false)).toBe(true); - expect(servers[1]?.command).toBe('node scripts/react-cockpit/prepare-ag-ui-a2ui-fixture.mjs --serve'); + expect(servers[1]?.command).toBe('node scripts/react-cockpit/serve.mjs ag-ui-a2ui --no-parent --backend'); expect(config.testMatch).toBe('**/react-ag-ui-a2ui-preview.spec.ts'); expect(config.workers).toBe(1); expect(config.retries).toBe(0); @@ -618,6 +622,6 @@ it('scopes native AG-UI A2UI checks to Website and the prepared owned React serv it('scopes native client-tools checks to the Website and exact two owned frontend servers', () => { const config = createWebsitePlaywrightConfig({ WEBSITE_E2E_REACT_TOPIC: 'ag-ui-client-tools', WEBSITE_E2E_PORT: '4390' }); const servers = Array.isArray(config.webServer) ? config.webServer : []; - expect(servers.map(server => server.url)).toEqual(['http://127.0.0.1:4390', 'http://localhost:4325', 'http://127.0.0.1:4640']); + expect(servers.map(server => server.url)).toEqual(['http://127.0.0.1:4390', 'http://localhost:4325', 'http://127.0.0.1:4640/__ready']); expect(config.testMatch).toBe('**/react-ag-ui-client-tools-preview.spec.ts'); }); diff --git a/cockpit/ag-ui/a2ui/angular/e2e/fixtures/a2ui.json b/cockpit/ag-ui/a2ui/angular/e2e/fixtures/a2ui.json index 84d191e97..e433bab9c 100644 --- a/cockpit/ag-ui/a2ui/angular/e2e/fixtures/a2ui.json +++ b/cockpit/ag-ui/a2ui/angular/e2e/fixtures/a2ui.json @@ -27,6 +27,52 @@ } ] } + }, + { + "match": { "systemMessage": "In 3-5 words, summarize" }, + "response": { "content": "Local booking title" } + }, + { + "match": { + "systemMessage": "find_routes() tool returned the following flights", + "userMessage": "SFO→SEA" + }, + "response": { + "toolCalls": [ + { + "name": "FlightResultsSpec", + "arguments": "{\"surface_id\": \"results\", \"data_model\": {}, \"components\": [{\"id\": \"root\", \"component\": \"Column\", \"children\": [\"card_0\"]}, {\"id\": \"card_0\", \"component\": \"Card\", \"child\": \"col_0\"}, {\"id\": \"col_0\", \"component\": \"Column\", \"children\": [\"title_0\", \"route_0\", \"time_0\", \"btn_0\"]}, {\"id\": \"title_0\", \"component\": \"Text\", \"text\": \"UA flight UA204\", \"variant\": \"h3\"}, {\"id\": \"route_0\", \"component\": \"Text\", \"text\": \"SFO → SEA • 120 min • Boeing 737\"}, {\"id\": \"time_0\", \"component\": \"Text\", \"text\": \"Depart 09:00 • Arrive 11:00 • Gate F11\"}, {\"id\": \"btn_0\", \"component\": \"Button\", \"child\": \"btn_label_0\", \"action\": {\"event\": {\"name\": \"flightSelect\", \"context\": {\"flightId\": \"UA204\"}}}}, {\"id\": \"btn_label_0\", \"component\": \"Text\", \"text\": \"Select\"}]}" + } + ] + } + }, + { + "match": { + "systemMessage": "from the flight results surface", + "userMessage": "flight UA204" + }, + "response": { + "toolCalls": [ + { + "name": "ConfirmationSpec", + "arguments": "{\"surface_id\": \"confirmation\", \"data_model\": {}, \"components\": [{\"id\": \"root\", \"component\": \"Column\", \"children\": [\"card\"]}, {\"id\": \"card\", \"component\": \"Card\", \"child\": \"card_col\"}, {\"id\": \"card_col\", \"component\": \"Column\", \"children\": [\"title\", \"route_text\", \"time_text\", \"gate_text\", \"party_text\", \"modify\"]}, {\"id\": \"title\", \"component\": \"Text\", \"text\": \"UA flight UA204\", \"variant\": \"h2\"}, {\"id\": \"route_text\", \"component\": \"Text\", \"text\": \"SFO → SEA • Boeing 737\"}, {\"id\": \"time_text\", \"component\": \"Text\", \"text\": \"09:00 → 11:00\"}, {\"id\": \"gate_text\", \"component\": \"Text\", \"text\": \"Gate F11\"}, {\"id\": \"party_text\", \"component\": \"Text\", \"text\": \"3 passengers • Business\"}, {\"id\": \"modify\", \"component\": \"Button\", \"child\": \"modify_label\", \"action\": {\"event\": {\"name\": \"modifySearch\", \"context\": {\"formId\": \"booking\"}}}}, {\"id\": \"modify_label\", \"component\": \"Text\", \"text\": \"Modify search\"}]}" + } + ] + } + }, + { + "match": { + "systemMessage": "booking-form designer", + "userMessage": "\"modifySearch\"" + }, + "response": { + "toolCalls": [ + { + "name": "BookingFormSpec", + "arguments": "{\"surface_id\": \"booking\", \"data_model\": {\"origin\": [\"SFO\"], \"dest\": [\"SEA\"], \"date\": \"2027-02-17\", \"passengers\": 3, \"fare_class\": [\"Business\"]}, \"components\": [{\"id\": \"root\", \"component\": \"Column\", \"children\": [\"card\"]}, {\"id\": \"card\", \"component\": \"Card\", \"child\": \"card_col\"}, {\"id\": \"card_col\", \"component\": \"Column\", \"children\": [\"title\", \"origin\", \"dest\", \"date\", \"passengers\", \"fare\", \"submit\"]}, {\"id\": \"title\", \"component\": \"Text\", \"text\": \"Book a flight\", \"variant\": \"h2\"}, {\"id\": \"origin\", \"component\": \"ChoicePicker\", \"label\": \"Origin\", \"options\": [{\"label\": \"LAX\", \"value\": \"LAX\"}, {\"label\": \"JFK\", \"value\": \"JFK\"}, {\"label\": \"SFO\", \"value\": \"SFO\"}, {\"label\": \"ORD\", \"value\": \"ORD\"}, {\"label\": \"BOS\", \"value\": \"BOS\"}, {\"label\": \"ATL\", \"value\": \"ATL\"}, {\"label\": \"DFW\", \"value\": \"DFW\"}, {\"label\": \"SEA\", \"value\": \"SEA\"}, {\"label\": \"MIA\", \"value\": \"MIA\"}, {\"label\": \"DEN\", \"value\": \"DEN\"}], \"value\": {\"path\": \"/origin\"}, \"variant\": \"mutuallyExclusive\"}, {\"id\": \"dest\", \"component\": \"ChoicePicker\", \"label\": \"Destination\", \"options\": [{\"label\": \"LAX\", \"value\": \"LAX\"}, {\"label\": \"JFK\", \"value\": \"JFK\"}, {\"label\": \"SFO\", \"value\": \"SFO\"}, {\"label\": \"ORD\", \"value\": \"ORD\"}, {\"label\": \"BOS\", \"value\": \"BOS\"}, {\"label\": \"ATL\", \"value\": \"ATL\"}, {\"label\": \"DFW\", \"value\": \"DFW\"}, {\"label\": \"SEA\", \"value\": \"SEA\"}, {\"label\": \"MIA\", \"value\": \"MIA\"}, {\"label\": \"DEN\", \"value\": \"DEN\"}], \"value\": {\"path\": \"/dest\"}, \"variant\": \"mutuallyExclusive\"}, {\"id\": \"fare\", \"component\": \"ChoicePicker\", \"label\": \"Fare class\", \"options\": [{\"label\": \"Economy\", \"value\": \"Economy\"}, {\"label\": \"Premium\", \"value\": \"Premium\"}, {\"label\": \"Business\", \"value\": \"Business\"}, {\"label\": \"First\", \"value\": \"First\"}], \"value\": {\"path\": \"/fare_class\"}, \"variant\": \"mutuallyExclusive\"}, {\"id\": \"date\", \"component\": \"DateTimeInput\", \"label\": \"Departure date (YYYY-MM-DD)\", \"value\": {\"path\": \"/date\"}, \"enableDate\": true}, {\"id\": \"passengers\", \"component\": \"TextField\", \"label\": \"Passengers\", \"value\": {\"path\": \"/passengers\"}, \"variant\": \"number\"}, {\"id\": \"submit\", \"component\": \"Button\", \"child\": \"submit_label\", \"variant\": \"primary\", \"action\": {\"event\": {\"name\": \"bookingSubmit\", \"context\": {\"formId\": \"booking\", \"origin\": {\"path\": \"/origin\"}, \"dest\": {\"path\": \"/dest\"}, \"date\": {\"path\": \"/date\"}, \"passengers\": {\"path\": \"/passengers\"}, \"fare_class\": {\"path\": \"/fare_class\"}}}}}, {\"id\": \"submit_label\", \"component\": \"Text\", \"text\": \"Search flights\"}]}" + } + ] + } } ] } diff --git a/cockpit/ag-ui/a2ui/react/README.md b/cockpit/ag-ui/a2ui/react/README.md index 0082227a4..8e5fb5f79 100644 --- a/cockpit/ag-ui/a2ui/react/README.md +++ b/cockpit/ag-ui/a2ui/react/README.md @@ -4,6 +4,6 @@ This private experimental frontend shares the existing AG-UI A2UI Python graph a Run `npx nx e2e cockpit-ag-ui-a2ui-react` for uncached frozen preparation, one isolated installed-application proof, and the browser suite. Existing Python 3.12 and uv must be available; explicit absolute executable overrides are `THREADPLANE_AGUI_A2UI_PYTHON` and `THREADPLANE_AGUI_A2UI_UV`. `THREADPLANE_AGUI_A2UI_APPLICATION_ROOT` retains all attempted proof consumers, bytes and receipts. Python, caches and HOME are owned outside that uploaded subtree. -The private loopback server buffers actual canonical encoded AG-UI bytes before browser delivery. Each browser conversation owns a separate canonical worker and checkpointer, limited to four real POSTs; generations admit at most eight owners and 32 real POSTs. Reset revokes authority before bounded cleanup, archives receipts and opens fresh routing only after verified closure. Hold/release and explicitly labeled fake negative transport controls exist only in the local proof server. They are excluded from the application bundle and public Code assets. +The browser suite runs against the example's real AG-UI server (`src/server.py`), with model replies replayed by aimock from the fixtures shared with the Angular example (`cockpit/ag-ui/a2ui/angular/e2e/fixtures`). A local recording proxy (`scripts/react-cockpit/ag-ui-proxy.mjs`) records each agent POST with the real events and can hold, release or fail the next run; those controls exist only in local tests and are excluded from the application bundle and public Code assets. -The browser flow exercises Request, edited Search, Select and server-restored Modify. Native idle state and the original successful run plus matching root completion/transcript are necessary before presentation admission. Stop, error and New synchronously quarantine stale work. Historical form values remain read-only. Positive canonical wire evidence and fake negative transport requests are labeled separately. Local deterministic graph proof does not establish live-provider behavior or full A2UI catalog support. +The browser flow exercises Request, edited Search, Select and server-restored Modify. Native idle state and the original successful run plus matching root completion/transcript are necessary before presentation admission. Stop, error and New synchronously quarantine stale work. Historical form values remain read-only. Injected transport failures are labeled in the specs. Replayed model output does not establish live-provider behavior or full A2UI catalog support. diff --git a/cockpit/ag-ui/a2ui/react/e2e/a2ui.spec.ts b/cockpit/ag-ui/a2ui/react/e2e/a2ui.spec.ts index cecc325c9..3f589f92d 100644 --- a/cockpit/ag-ui/a2ui/react/e2e/a2ui.spec.ts +++ b/cockpit/ag-ui/a2ui/react/e2e/a2ui.spec.ts @@ -6,61 +6,27 @@ import { } from '@playwright/test'; import { writeFile } from 'node:fs/promises'; -interface CanonicalPost { - kind: 'actual-canonical-post'; - threadId: string; - runId: string; - body: { messages: { content: string }[] }; - rawBody: string; - response: { rawWire: string }; -} -interface OwnerReceipt { - stats: { - ready: { pid: number }; - closure: { - worker: boolean; - listener: boolean; - sockets: boolean; - responses: boolean; - }; - requests: { - body: { protocolVersion: string }; - rawBody: string; - sentWire: string[]; - worker: { - events: unknown[]; - observations: { - requestCount: number; - deniedNetwork: unknown[]; - state: { values: { messages: unknown[] } }; - }; - }; - }[]; - }; -} -interface GenerationReceipt { - holds: number; +// Backend: aimock replays cockpit/ag-ui/a2ui/angular/e2e/fixtures (the +// react-parity A2UI model's outputs for this flow) into the example's real +// AG-UI server, whose real tools search flights and look one up. The +// recording proxy (scripts/react-cockpit/ag-ui-proxy.mjs) records each agent +// POST with the real events, and holds or fails the next run. +interface AgentPost { + path: string; + body: { threadId: string; runId: string; protocolVersion: string; messages: { content: string }[] }; + events: { type: string; threadId?: string; runId?: string }[]; + held: boolean; closed: boolean; - proxyResponsesClosed: boolean; - owners: OwnerReceipt[]; - requests: (CanonicalPost | { kind: 'fake-negative-transport' })[]; } -interface ManagerReceipt { - current: GenerationReceipt; - archived: GenerationReceipt[]; -} - const base = '/ag-ui/a2ui/react/'; const current = (page: Page) => page.getByRole('region', { name: 'Current flight preview', exact: true }); const earlier = (page: Page) => page.getByRole('region', { name: 'Earlier flight preview', exact: true }); -const lifetime = async (request: APIRequestContext): Promise => +const lifetime = async (request: APIRequestContext): Promise<{ active: boolean; openResponses: number }> => (await request.get('/__lifetime')).json(); -const posts = (receipt: ManagerReceipt) => - receipt.current.requests.filter( - (record): record is CanonicalPost => record.kind === 'actual-canonical-post' - ); +const posts = async (request: APIRequestContext): Promise => + (await request.get('/__requests')).json(); async function form(page: Page) { await page.getByRole('button', { name: 'Try a flight request' }).click(); await page @@ -93,37 +59,22 @@ test.beforeEach(async ({ page, request }) => { ).toBeVisible(); await expect(page.getByRole('status', { name: 'Conversation status' })).toHaveText('Ready.'); }); -test.afterEach(async ({ request }, info) => { - const receipt = await lifetime(request), - path = info.outputPath('canonical-manager-receipt.json'); - await writeFile(path, JSON.stringify(receipt, null, 2)); - await info.attach('canonical-manager-receipt', { - path, - contentType: 'application/json', - }); - // Archive and verify all canonical children before another browser case starts. - expect((await request.post('/__reset')).ok()).toBe(true); - const closed = await lifetime(request); - for (const generation of closed.archived) { - expect(generation.closed).toBe(true); - expect(generation.proxyResponsesClosed).toBe(true); - for (const owner of generation.owners) - expect(owner.stats.closure).toEqual({ - worker: true, - listener: true, - sockets: true, - responses: true, - }); - } +test.afterEach(async ({ page, request }, info) => { + const path = info.outputPath('agent-requests.json'); + await writeFile(path, JSON.stringify(await posts(request), null, 2)); + await info.attach('agent-requests', { path, contentType: 'application/json' }); + // No proxied agent stream outlives the case. + await page.goto('about:blank'); + await expect.poll(async () => (await lifetime(request)).openResponses).toBe(0); }); test('four actual native turns preserve edited Search, Select and server-restored Modify', async ({ page, request, }, info) => { - expect(posts(await lifetime(request))).toHaveLength(0); + expect(await posts(request)).toHaveLength(0); await page.getByRole('button', { name: 'Try a flight request' }).click(); - expect(posts(await lifetime(request))).toHaveLength(0); + expect(await posts(request)).toHaveLength(0); await page .getByRole('textbox', { name: 'Message', exact: true }) .press('Enter'); @@ -169,13 +120,10 @@ test('four actual native turns preserve edited Search, Select and server-restore value ); await expect(earlier(page)).toHaveCount(3); - const receipt = await lifetime(request), - requests = posts(receipt); + const requests = await posts(request); expect(requests).toHaveLength(4); - expect(receipt.current.owners).toHaveLength(1); - const owner = receipt.current.owners[0]; - expect(new Set(requests.map((record) => record.threadId)).size).toBe(1); - expect(new Set(requests.map((record) => record.runId)).size).toBe(4); + expect(new Set(requests.map((record) => record.body.threadId)).size).toBe(1); + expect(new Set(requests.map((record) => record.body.runId)).size).toBe(4); const action = (turn: number) => { const message = requests[turn].body.messages.at(-1); expect(message).toBeDefined(); @@ -199,19 +147,12 @@ test('four actual native turns preserve edited Search, Select and server-restore context: { flightId: 'UA204' }, }); expect(action(3).name).toBe('modifySearch'); - for (const [index, record] of owner.stats.requests.entries()) { - const observation = record.worker.observations; - expect(observation.requestCount).toBe(index + 1); - expect(observation.state.values.messages).toHaveLength((index + 1) * 2); - expect(observation.deniedNetwork).toEqual([]); + // Each turn is one real run that the server finished for that thread and run. + for (const record of requests) { expect(record.body.protocolVersion).toBe('1.0'); - expect(record.rawBody).toBe(requests[index].rawBody); - expect(record.sentWire.join('')).toBe(requests[index].response.rawWire); - expect(record.worker.events).toContainEqual({ - type: 'RUN_FINISHED', - threadId: requests[index].threadId, - runId: requests[index].runId, - }); + expect(record.events).toContainEqual( + expect.objectContaining({ type: 'RUN_FINISHED', threadId: record.body.threadId, runId: record.body.runId }) + ); } await page.setViewportSize({ width: 390, height: 844 }); expect( @@ -240,10 +181,11 @@ test('invalid drafts focus the field and repeated busy Search sends exactly one await expect(passengers).toHaveValue(value); await expect(passengers).toBeFocused(); await expect(passengers).toHaveAttribute('aria-invalid', 'true'); - expect(posts(await lifetime(request))).toHaveLength(1); + expect(await posts(request)).toHaveLength(1); } await edit(page); - await request.post('/__hold/stream'); + // A2UI runs stream structured tool calls, no text: hold right after the start. + await request.post('/__hold/stream', { data: { afterType: 'RUN_STARTED' } }); await current(page) .getByRole('button', { name: 'Search flights', exact: true }) .evaluate((button: HTMLButtonElement) => { @@ -251,9 +193,9 @@ test('invalid drafts focus the field and repeated busy Search sends exactly one button.click(); }); await expect - .poll(async () => (await lifetime(request)).current.holds) - .toBe(1); - expect(posts(await lifetime(request))).toHaveLength(2); + .poll(async () => (await lifetime(request)).active) + .toBe(true); + expect(await posts(request)).toHaveLength(2); await expect( page.getByRole('button', { name: 'Send', exact: true }) ).toBeDisabled(); @@ -264,26 +206,31 @@ test('invalid drafts focus the field and repeated busy Search sends exactly one await expect( current(page).getByRole('button', { name: 'Select', exact: true }) ).toBeVisible(); - expect(posts(await lifetime(request))).toHaveLength(2); + expect(await posts(request)).toHaveLength(2); }); -test('Stop, fake transport error, New and held old completion cannot regain authority', async ({ +test('Stop, failed transport, New and a held old run cannot regain authority', async ({ page, request, }) => { await form(page); await edit(page); - await request.post('/__hold/stream'); + // Hold every run right after it starts until /__hold/off. + await request.post('/__hold/stream', { data: { afterType: 'RUN_STARTED', persist: true } }); await current(page) .getByRole('button', { name: 'Search flights', exact: true }) .click(); await expect - .poll(async () => (await lifetime(request)).current.holds) - .toBe(1); + .poll(async () => (await lifetime(request)).active) + .toBe(true); await page.getByRole('button', { name: 'Stop', exact: true }).click(); await expect(page.getByRole('status', { name: 'Conversation status' })).toHaveText( 'Start a new conversation to continue.' ); + // Stop closes the held stream itself: nothing late can reach the page. + await expect + .poll(async () => (await lifetime(request)).active) + .toBe(false); await expect( earlier(page).getByLabel('Passengers', { exact: true }) ).toHaveValue('003'); @@ -295,21 +242,16 @@ test('Stop, fake transport error, New and held old completion cannot regain auth .click(); await request.post('/__hold/off'); await form(page); - // The new owner completes while the old encoded response is still held. - await expect - .poll(async () => (await lifetime(request)).current.holds) - .toBe(1); await request.post('/__release'); - const first = await lifetime(request); - expect(first.current.owners).toHaveLength(2); - expect( - new Set(first.current.owners.map((owner) => owner.stats.ready.pid)).size - ).toBe(2); await expect(earlier(page)).toHaveCount(0); + const threads = new Set((await posts(request)).map((record) => record.body.threadId)); + expect(threads.size).toBe(2); await page .getByRole('button', { name: 'New conversation', exact: true }) .click(); - await request.post('/__fail-transport'); + // The next agent POST answers 500 without reaching the server. + await request.post('/__fail-stream'); + const beforeFailure = (await posts(request)).length; await page.getByRole('button', { name: 'Try a flight request' }).click(); await page .getByRole('textbox', { name: 'Message', exact: true }) @@ -317,34 +259,33 @@ test('Stop, fake transport error, New and held old completion cannot regain auth await expect(page.getByRole('status', { name: 'Conversation status' })).toHaveText( 'Start a new conversation to continue.' ); - expect( - (await lifetime(request)).current.requests.filter( - (record) => record.kind === 'fake-negative-transport' - ) - ).toHaveLength(1); + expect(await posts(request)).toHaveLength(beforeFailure + 1); await page .getByRole('button', { name: 'New conversation', exact: true }) .click(); await form(page); - await request.post('/__hold/stream'); + await request.post('/__hold/stream', { data: { afterType: 'RUN_STARTED', persist: true } }); await edit(page); await current(page) .getByRole('button', { name: 'Search flights', exact: true }) .click(); await expect - .poll(async () => (await lifetime(request)).current.holds) - .toBe(1); + .poll(async () => (await lifetime(request)).active) + .toBe(true); await page .getByRole('button', { name: 'New conversation', exact: true }) .click(); + await expect + .poll(async () => (await lifetime(request)).active) + .toBe(false); await request.post('/__hold/off'); await form(page); await request.post('/__release'); - const before = posts(await lifetime(request)).length; + const before = (await posts(request)).length; await page.evaluate(() => window.dispatchEvent(new PageTransitionEvent('pagehide')) ); await request.post('/__release'); - expect(posts(await lifetime(request))).toHaveLength(before); + expect(await posts(request)).toHaveLength(before); await expect(page.locator('main')).toHaveCount(0); }); diff --git a/cockpit/ag-ui/a2ui/react/e2e/global-setup-impl.ts b/cockpit/ag-ui/a2ui/react/e2e/global-setup-impl.ts new file mode 100644 index 000000000..f988ecd6d --- /dev/null +++ b/cockpit/ag-ui/a2ui/react/e2e/global-setup-impl.ts @@ -0,0 +1,6 @@ +// eslint-disable-next-line @nx/enforce-module-boundaries -- Shared aimock setup for the React cockpit examples. +import { reactAimockAgUiGlobalSetup } from '../../../../shared/react-aimock-setup'; + +// aimock replays the Angular example's fixtures into the example's real AG-UI +// server; the static server (webServer) proxies the agent endpoint to it. +export default reactAimockAgUiGlobalSetup({ example: 'ag-ui/a2ui', port: 4647 }); diff --git a/cockpit/ag-ui/a2ui/react/e2e/playwright.config.ts b/cockpit/ag-ui/a2ui/react/e2e/playwright.config.ts index 7a11c474b..1e3faa5f2 100644 --- a/cockpit/ag-ui/a2ui/react/e2e/playwright.config.ts +++ b/cockpit/ag-ui/a2ui/react/e2e/playwright.config.ts @@ -8,14 +8,14 @@ export default defineConfig({ outputDir: '../../../../../output/playwright/ag-ui-a2ui-react', use: { baseURL: 'http://127.0.0.1:4647', trace: 'retain-on-failure' }, projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }], + // aimock + the real AG-UI server, shared with cockpit/ag-ui/a2ui/angular. + globalSetup: './global-setup-impl.ts', + globalTeardown: require.resolve('../../../../../libs/e2e-harness/src/global-teardown'), webServer: { - command: - 'node scripts/react-cockpit/prepare-ag-ui-a2ui-fixture.mjs --serve', + command: 'node scripts/react-cockpit/serve.mjs ag-ui-a2ui --no-parent', cwd: '../../../../..', url: 'http://127.0.0.1:4647/ag-ui/a2ui/react/', reuseExistingServer: false, - stdout: 'pipe', - timeout: 330000, - gracefulShutdown: { signal: 'SIGTERM', timeout: 15000 }, + timeout: 30000, }, }); diff --git a/cockpit/ag-ui/a2ui/react/project.json b/cockpit/ag-ui/a2ui/react/project.json index 4c30861ba..46c046740 100644 --- a/cockpit/ag-ui/a2ui/react/project.json +++ b/cockpit/ag-ui/a2ui/react/project.json @@ -3,13 +3,17 @@ "$schema": "../../../../node_modules/nx/schemas/project-schema.json", "sourceRoot": "cockpit/ag-ui/a2ui/react/src", "projectType": "application", - "implicitDependencies": ["cockpit-a2ui-react"], - "tags": ["scope:cockpit-examples", "scope:cockpit-e2e", "framework:react"], + "implicitDependencies": [ + "cockpit-a2ui-react" + ], + "tags": [ + "scope:cockpit-examples", + "scope:cockpit-e2e", + "framework:react" + ], "namedInputs": { "nativeA2uiFixture": [ "{workspaceRoot}/fixtures/react-parity/native-ag-ui-a2ui/**/*", - "{workspaceRoot}/scripts/react-cockpit/ag-ui-a2ui-fixture.mjs", - "{workspaceRoot}/scripts/react-cockpit/ag-ui-a2ui-fixture.spec.mjs", "{workspaceRoot}/scripts/react-cockpit/prepare-ag-ui-a2ui-fixture.mjs", "{workspaceRoot}/scripts/react-cockpit/ag-ui-a2ui-preparation.spec.mjs", "{workspaceRoot}/scripts/react-cockpit/ag-ui-a2ui-native-probe.mjs", @@ -27,7 +31,10 @@ "prepare-fixture": { "executor": "nx:run-commands", "cache": false, - "inputs": ["default", "nativeA2uiFixture"], + "inputs": [ + "default", + "nativeA2uiFixture" + ], "options": { "command": "node scripts/react-cockpit/prepare-ag-ui-a2ui-fixture.mjs" } @@ -35,9 +42,20 @@ "fixture-test": { "executor": "nx:run-commands", "cache": false, - "inputs": ["default", "nativeA2uiFixture"], + "inputs": [ + "default", + "nativeA2uiFixture" + ], "dependsOn": [ - { "target": "build", "projects": ["core", "content", "react", "a2ui"] } + { + "target": "build", + "projects": [ + "core", + "content", + "react", + "a2ui" + ] + } ], "options": { "command": "node scripts/react-cockpit/prepare-ag-ui-a2ui-fixture.mjs --verify-application" @@ -46,8 +64,15 @@ "e2e": { "executor": "@nx/playwright:playwright", "cache": false, - "inputs": ["default", "nativeA2uiFixture"], - "dependsOn": ["prepare-fixture", "build", "fixture-test"], + "inputs": [ + "default", + "reactAimockBackend", + "nativeA2uiFixture" + ], + "dependsOn": [ + "build", + "fixture-test" + ], "options": { "config": "cockpit/ag-ui/a2ui/react/e2e/playwright.config.ts", "skipInstall": true @@ -57,21 +82,40 @@ "executor": "@nx/vitest:test", "options": { "configFile": "cockpit/ag-ui/a2ui/react/vite.config.mts", - "reporters": ["default"] + "reporters": [ + "default" + ] } }, "build": { "executor": "nx:run-commands", "cache": false, "dependsOn": [ - { "target": "build", "projects": ["core", "content", "react", "a2ui"] }, - { "target": "react-native-a2ui-ag-ui", "projects": ["website-runtime-preparation"] } + { + "target": "build", + "projects": [ + "core", + "content", + "react", + "a2ui" + ] + }, + { + "target": "react-native-a2ui-ag-ui", + "projects": [ + "website-runtime-preparation" + ] + } + ], + "outputs": [ + "{workspaceRoot}/dist/cockpit/ag-ui/a2ui/react" ], - "outputs": ["{workspaceRoot}/dist/cockpit/ag-ui/a2ui/react"], "options": { "command": "node scripts/react-cockpit/build.mjs ag-ui-a2ui" } }, - "lint": { "executor": "@nx/eslint:lint" } + "lint": { + "executor": "@nx/eslint:lint" + } } } diff --git a/cockpit/ag-ui/client-tools/angular/e2e/fixtures/client-tools.json b/cockpit/ag-ui/client-tools/angular/e2e/fixtures/client-tools.json index ac7c85d7b..6273d3773 100644 --- a/cockpit/ag-ui/client-tools/angular/e2e/fixtures/client-tools.json +++ b/cockpit/ag-ui/client-tools/angular/e2e/fixtures/client-tools.json @@ -35,6 +35,201 @@ { "match": { "userMessage": "Plan a trip to Portland" }, "response": { "toolCalls": [ { "name": "get_weather", "arguments": { "location": "Portland" } }, { "name": "confirm_booking", "arguments": { "summary": "Fictional trip to Portland" } } ] } + }, + { + "match": { "toolCallId": "rounds-2" }, + "response": { "content": "Canonical local results received." } + }, + { + "match": { "toolCallId": "rounds-1" }, + "response": { + "toolCalls": [ + { + "name": "weather_card", + "arguments": "{\"location\": \"San Francisco\", \"temperatureF\": 68, \"conditions\": \"Sunny\", \"humidity\": 55, \"windMph\": 8}", + "id": "rounds-2" + } + ] + } + }, + { + "match": { + "userMessage": "Rounds", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "get_weather", + "arguments": "{\"location\": \"San Francisco\"}", + "id": "rounds-1" + } + ] + } + }, + { + "match": { + "userMessage": "Malformed", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "weather_card", + "arguments": "{\"location\": \"San Francisco\", \"temperatureF\": 68, \"conditions\": \"Sunny\", \"humidity\": 101, \"windMph\": 8}" + } + ] + } + }, + { + "match": { + "userMessage": "Unknown", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "unknown_weather", + "arguments": "{}" + } + ] + } + }, + { + "match": { + "userMessage": "Mixed", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "weather_snapshot", + "arguments": "{\"location\": \"San Francisco\", \"temperatureF\": 68, \"conditions\": \"Sunny\", \"humidity\": 55, \"windMph\": 8}" + }, + { + "name": "get_weather", + "arguments": "{\"location\": \"San Francisco\"}" + } + ] + } + }, + { + "match": { + "userMessage": "Bookings", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "confirm_booking", + "arguments": "{\"summary\": \"Fictional dinner A\"}" + }, + { + "name": "confirm_booking", + "arguments": "{\"summary\": \"Fictional dinner B\"}" + } + ] + } + }, + { + "match": { + "userMessage": "confirm_booking", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "confirm_booking", + "arguments": "{\"summary\": \"Fictional dinner booking\"}" + } + ] + } + }, + { + "match": { + "userMessage": "slow_status_check", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "slow_status_check", + "arguments": "{\"label\": \"demo check\"}" + } + ] + } + }, + { + "match": { + "userMessage": "Literal", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "weather_card", + "arguments": "{\"location\": \"\", \"temperatureF\": 68, \"conditions\": \"Sunny\", \"humidity\": 55, \"windMph\": 8}" + } + ] + } + }, + { + "match": { + "userMessage": "weather_snapshot", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "weather_snapshot", + "arguments": "{\"location\": \"San Francisco\", \"temperatureF\": 68, \"conditions\": \"Sunny\", \"humidity\": 55, \"windMph\": 8}" + } + ] + } + }, + { + "match": { + "userMessage": "weather_card", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "weather_card", + "arguments": "{\"location\": \"San Francisco\", \"temperatureF\": 68, \"conditions\": \"Sunny\", \"humidity\": 55, \"windMph\": 8}" + } + ] + }, + "chunkSize": 12 + }, + { + "match": { + "userMessage": "get_weather", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "get_weather", + "arguments": "{\"location\": \"San Francisco\"}" + } + ] + } + }, + { + "match": { + "hasToolResult": true, + "userMessage": "" + }, + "response": { "content": "Canonical local results received." } + }, + { + "match": { "userMessage": "Hello canonical" }, + "response": { "content": "Canonical plain answer: Hello canonical" } + }, + { + "match": { "userMessage": "" }, + "response": { "content": "Canonical plain answer." } } ] } diff --git a/cockpit/ag-ui/client-tools/react/README.md b/cockpit/ag-ui/client-tools/react/README.md index bef352eea..c608e18b6 100644 --- a/cockpit/ag-ui/client-tools/react/README.md +++ b/cockpit/ag-ui/client-tools/react/README.md @@ -58,6 +58,6 @@ npx nx build cockpit-ag-ui-client-tools-react npx nx e2e cockpit-ag-ui-client-tools-react ~~~ -The e2e target depends on frozen canonical fixture preparation. For explicit preparation, run **npx nx run cockpit-ag-ui-client-tools-react:prepare-fixture**. The build installs the real private package artifacts with strict types. Deterministic offline browser checks use the actual canonical graph and locked AG-UI adapter with scripted model responses over HTTP/SSE. They verify local execution, physical continuations, and owned cleanup; they do not prove a live model-provider request. Docs, Code, and Run select this same source implementation. +The build installs the real private package artifacts with strict types. Deterministic offline browser checks run the actual graph behind the locked AG-UI adapter, with scripted model responses replayed by aimock, over HTTP/SSE. They verify local execution and physical continuations, and that the model receives each exact local result; they do not prove a live model-provider request. Docs, Code, and Run select this same source implementation. Source: `cockpit/ag-ui/client-tools/react/project.json` diff --git a/cockpit/ag-ui/client-tools/react/e2e/client-tools.spec.ts b/cockpit/ag-ui/client-tools/react/e2e/client-tools.spec.ts index 062188e8b..4d501aa1a 100644 --- a/cockpit/ag-ui/client-tools/react/e2e/client-tools.spec.ts +++ b/cockpit/ag-ui/client-tools/react/e2e/client-tools.spec.ts @@ -1,17 +1,28 @@ -import { writeFile } from 'node:fs/promises'; import { test, expect, type Page, type APIRequestContext } from '@playwright/test'; +// Backend: aimock replays cockpit/ag-ui/client-tools/angular/e2e/fixtures (the +// cockpit model of the react-parity client-tools server, as fixtures) into +// the example's real AG-UI server; the browser executes the client tools. The +// recording proxy (scripts/react-cockpit/ag-ui-proxy.mjs) records each run's +// real events live, holds runs, and serves aimock's request journal: what the +// model actually received. interface WireMessage { id: string; role: string; content: string; toolCallId?: string; toolCalls?: {id: string}[] } -interface ModelMessage { id?: string; type: string; content: string; tool_call_id?: string } +interface ModelMessage { role: string; content: string; tool_call_id?: string } interface WireRecord { body: {threadId: string; runId: string; messages: WireMessage[]; tools: {name: string}[]}; events: {type: string; messages?: WireMessage[]; delta?: string}[]; - observations: {socketAttempts: string[]; graphIdentity: number; checkpointerIdentity: number; modelIdentity: number; modelMessages: ModelMessage[][]}; + /** Every model call so far, oldest first. */ + modelMessages: ModelMessage[][]; held: boolean; closed: boolean; } const base = '/ag-ui/client-tools/react/'; const names = ['get_weather', 'slow_status_check', 'weather_card', 'weather_snapshot', 'confirm_booking']; const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; -async function records(request: APIRequestContext) { return (await request.get('/__requests')).json() as Promise; } +async function records(request: APIRequestContext): Promise { + const wire = await (await request.get('/__requests')).json(); + const journal: { body: { messages?: ModelMessage[] } }[] = await (await request.get('/__model-requests')).json(); + const modelMessages = journal.map(entry => entry.body.messages ?? []); + return wire.map((row: WireRecord) => ({ ...row, modelMessages })); +} async function send(page: Page, text: string) { await page.getByLabel('Message', {exact:true}).fill(text); await page.getByRole('button', {name:'Send', exact:true}).click(); @@ -31,29 +42,18 @@ function causal(wire: WireRecord[], count: number) { expect(row.body.runId).toMatch(uuid); expect(row.body.threadId).toMatch(uuid); expect(row.body.tools.map((tool: {name: string}) => tool.name)).toEqual(names); - expect(row.observations.socketAttempts).toEqual([]); - expect(row.observations.graphIdentity).toBe(wire[0].observations.graphIdentity); - expect(row.observations.checkpointerIdentity).toBe(wire[0].observations.checkpointerIdentity); - expect(row.observations.modelIdentity).toBe(wire[0].observations.modelIdentity); - const model = row.observations.modelMessages.at(-1) ?? []; const snapshot = row.events.filter(event => event.type === 'MESSAGES_SNAPSHOT').at(-1)?.messages ?? []; for (const result of row.body.messages.filter(message => message.role === 'tool')) { const caller = row.body.messages.find(message => message.toolCalls?.some(call => call.id === result.toolCallId)); expect(caller).toBeDefined(); expect(row.body.messages.filter(message => message.id === result.id)).toHaveLength(1); - expect(model.filter(message => message.type === 'tool' && message.id === result.id && message.tool_call_id === result.toolCallId && message.content === result.content)).toHaveLength(1); + // The model received the browser's exact result for that call. + expect(row.modelMessages.some(messages => messages.filter(message => message.role === 'tool' && message.tool_call_id === result.toolCallId && message.content === result.content).length === 1)).toBe(true); expect(snapshot.filter(message => message.role === 'tool' && message.id === result.id && message.toolCallId === result.toolCallId && message.content === result.content)).toHaveLength(1); } } } test.beforeEach(async ({request}) => { expect((await request.post('/__reset')).ok()).toBe(true); }); -test.afterEach(async ({request}, info) => { - const stats = await (await request.post('/__finish')).json(); - expect(Object.values(stats.closed.at(-1).closure).every(Boolean)).toBe(true); - const receipt = info.outputPath('canonical-owned-fixture-receipt.json'); - await writeFile(receipt, JSON.stringify(stats, null, 2)); - await info.attach('canonical-owned-fixture-receipt', {path: receipt, contentType:'application/json'}); -}); test('mount issues no physical request and offers six example prompts', async ({page,request}) => { await page.goto(base); await expect(page.getByRole('group', {name:'Example prompts'}).getByRole('button')).toHaveCount(6); @@ -95,7 +95,7 @@ for (const tool of ['get_weather','weather_card']) { const wire = await records(request); causal(wire, 2); const result = wire[1].body.messages.filter((message:WireMessage) => message.role === 'tool'); expect(result).toHaveLength(1); - expect(wire[1].observations.modelMessages.at(-1)?.filter((message:ModelMessage) => message.type === 'tool')).toHaveLength(1); + expect(wire[1].modelMessages.at(-1)?.filter((message:ModelMessage) => message.role === 'tool')).toHaveLength(1); expect(wire[1].body.tools).toEqual(wire[0].body.tools); expect(result[0].content).toBe(tool === 'get_weather' ? JSON.stringify({location:'San Francisco',temperatureF:68,conditions:'Sunny',humidity:55,windMph:8}) : JSON.stringify({displayed:true})); await expect(page.locator('[data-native-call]')).toHaveCount(1); @@ -169,7 +169,9 @@ for (const text of ['Unknown','Malformed']) { } for (const action of ['release','Stop','page exit']) { test(`actual streamed arguments held before terminal: ${action} respects execution boundary`, async ({page,request}) => { - await request.post('/__reset',{data:{mode:'stream',holdTerminal:true}}); + // Hold every run just before its terminal event; weather_card's arguments + // stream in several chunks (fixture chunkSize). + await request.post('/__hold/stream',{data:{beforeType:'RUN_FINISHED',persist:true}}); await page.goto(base); await send(page,'Use weather_card'); await expect.poll(async()=> (await records(request))[0]?.held).toBe(true); await expect(page.locator('.weather-card, .tp-approval-card')).toHaveCount(0); @@ -197,7 +199,7 @@ test('multiple native groups keep distinct causal identities on retained history await expect(page.locator('[data-native-call]')).toHaveCount(3); const ids=wire[3].body.messages.flatMap((message:WireMessage)=>message.toolCalls?.map((call:{id:string})=>call.id)??[]); expect(new Set(ids).size).toBe(3); - expect(wire[3].observations.modelMessages.at(-1)?.filter((message:ModelMessage)=>message.type==='tool')).toHaveLength(3); + expect(wire[3].modelMessages.at(-1)?.filter((message:ModelMessage)=>message.role==='tool')).toHaveLength(3); }); test('mobile literal content and weather readings remain safe', async ({page,request}) => { await page.setViewportSize({width:390,height:844}); await page.goto(base); @@ -241,7 +243,9 @@ for(const action of ['Stop','page exit']) { }); } test('Stop during actual held continuation preserves complete sent-unconfirmed local evidence without replay',async({page,request})=>{ - await request.post('/__reset',{data:{holdTerminal:true,holdAfterRequest:1,holdReceipt:true}}); + // Hold the continuation (the second run) before the server acknowledges the + // result: right after it starts. + await request.post('/__hold/stream',{data:{afterType:'RUN_STARTED',persist:true,fromRequest:1}}); await page.goto(base); await send(page,'Use get_weather'); await expect.poll(async()=> (await records(request))[1]?.held).toBe(true); await expect(page.getByRole('region',{name:'Local tool evidence'})).toContainText('Local execution: complete'); diff --git a/cockpit/ag-ui/client-tools/react/e2e/global-setup-impl.ts b/cockpit/ag-ui/client-tools/react/e2e/global-setup-impl.ts new file mode 100644 index 000000000..a51593a02 --- /dev/null +++ b/cockpit/ag-ui/client-tools/react/e2e/global-setup-impl.ts @@ -0,0 +1,6 @@ +// eslint-disable-next-line @nx/enforce-module-boundaries -- Shared aimock setup for the React cockpit examples. +import { reactAimockAgUiGlobalSetup } from '../../../../shared/react-aimock-setup'; + +// aimock replays the Angular example's fixtures into the example's real AG-UI +// server; the static server (webServer) proxies the agent endpoint to it. +export default reactAimockAgUiGlobalSetup({ example: 'ag-ui/client-tools', port: 4640 }); diff --git a/cockpit/ag-ui/client-tools/react/e2e/playwright.config.ts b/cockpit/ag-ui/client-tools/react/e2e/playwright.config.ts index a15ad2115..bd0002689 100644 --- a/cockpit/ag-ui/client-tools/react/e2e/playwright.config.ts +++ b/cockpit/ag-ui/client-tools/react/e2e/playwright.config.ts @@ -3,9 +3,12 @@ export default defineConfig({ testDir: '.', outputDir: './test-results', workers: 1, fullyParallel: false, timeout: 30000, use: { baseURL: 'http://127.0.0.1:4640', trace: 'retain-on-failure' }, projects: [{name:'chromium', use: {...devices['Desktop Chrome']}}], + // aimock + the real AG-UI server, shared with cockpit/ag-ui/client-tools/angular. + globalSetup: './global-setup-impl.ts', + globalTeardown: require.resolve('../../../../../libs/e2e-harness/src/global-teardown'), webServer: { - command: 'node scripts/react-cockpit/prepare-ag-ui-client-tools-fixture.mjs --serve', + command: 'node scripts/react-cockpit/serve.mjs ag-ui-client-tools --no-parent', cwd: '../../../../..', - url: 'http://127.0.0.1:4640', reuseExistingServer: false, timeout: 180000, + url: 'http://127.0.0.1:4640', reuseExistingServer: false, timeout: 30000, }, }); diff --git a/cockpit/ag-ui/client-tools/react/project.json b/cockpit/ag-ui/client-tools/react/project.json index 85c507c82..3a3606abb 100644 --- a/cockpit/ag-ui/client-tools/react/project.json +++ b/cockpit/ag-ui/client-tools/react/project.json @@ -3,6 +3,10 @@ "$schema": "../../../../node_modules/nx/schemas/project-schema.json", "sourceRoot": "cockpit/ag-ui/client-tools/react/src", "projectType": "application", + "implicitDependencies": [ + "cockpit-ag-ui-client-tools-angular", + "cockpit-ag-ui-client-tools-python" + ], "tags": [ "scope:cockpit-examples", "scope:cockpit-e2e", @@ -30,7 +34,12 @@ "react" ] }, - { "target": "react-ag-ui", "projects": ["website-runtime-preparation"] } + { + "target": "react-ag-ui", + "projects": [ + "website-runtime-preparation" + ] + } ], "outputs": [ "{workspaceRoot}/dist/cockpit/ag-ui/client-tools/react" @@ -39,24 +48,26 @@ "command": "node scripts/react-cockpit/build.mjs ag-ui-client-tools" } }, - "prepare-fixture": { - "executor": "nx:run-commands", - "cache": false, - "options": { - "command": "node scripts/react-cockpit/prepare-ag-ui-client-tools-fixture.mjs" - } - }, "e2e": { "executor": "@nx/playwright:playwright", "cache": false, "dependsOn": [ - "prepare-fixture", - "build" + "build", + { + "target": "test", + "projects": [ + "cockpit-ag-ui-client-tools-python" + ] + } ], "options": { "config": "cockpit/ag-ui/client-tools/react/e2e/playwright.config.ts", "skipInstall": true - } + }, + "inputs": [ + "default", + "reactAimockBackend" + ] } } } diff --git a/cockpit/ag-ui/interrupts/angular/e2e/fixtures/interrupts.json b/cockpit/ag-ui/interrupts/angular/e2e/fixtures/interrupts.json index 7fcf25d0b..76fef3e3f 100644 --- a/cockpit/ag-ui/interrupts/angular/e2e/fixtures/interrupts.json +++ b/cockpit/ag-ui/interrupts/angular/e2e/fixtures/interrupts.json @@ -31,6 +31,60 @@ "response": { "content": "Understood — a $129.00 refund to cus_z19fp for a chargeback. Pausing for operator approval; no refund is issued until a human reviews and approves." } + }, + { + "match": { + "responseFormat": "json_schema", + "userMessage": "Negative amount" + }, + "response": { + "content": { + "customer_id": "cus_demo", + "amount": -1, + "reason": "Fictional damaged item" + } + } + }, + { + "match": { + "responseFormat": "json_schema", + "userMessage": "" + }, + "response": { + "content": { + "customer_id": "cus_demo", + "amount": 25, + "reason": "Fictional damaged item" + } + } + }, + { + "match": { + "systemMessage": "Refund Authorization Assistant", + "userMessage": "Hold" + }, + "response": { "content": "Draft ready." }, + "chunkSize": 3 + }, + { + "match": { + "systemMessage": "Refund Authorization Assistant", + "userMessage": "Model failure" + }, + "response": { + "error": { + "message": "PRIVATE model failure", + "type": "server_error" + }, + "status": 500 + } + }, + { + "match": { + "systemMessage": "Refund Authorization Assistant", + "userMessage": "" + }, + "response": { "content": "Draft ready." } } ] } diff --git a/cockpit/ag-ui/interrupts/react/e2e/global-setup-impl.ts b/cockpit/ag-ui/interrupts/react/e2e/global-setup-impl.ts new file mode 100644 index 000000000..eb7637b7f --- /dev/null +++ b/cockpit/ag-ui/interrupts/react/e2e/global-setup-impl.ts @@ -0,0 +1,6 @@ +// eslint-disable-next-line @nx/enforce-module-boundaries -- Shared aimock setup for the React cockpit examples. +import { reactAimockAgUiGlobalSetup } from '../../../../shared/react-aimock-setup'; + +// aimock replays the Angular example's fixtures into the example's real AG-UI +// server; the static server (webServer) proxies the agent endpoint to it. +export default reactAimockAgUiGlobalSetup({ example: 'ag-ui/interrupts', port: 4610 }); diff --git a/cockpit/ag-ui/interrupts/react/e2e/interrupts.spec.ts b/cockpit/ag-ui/interrupts/react/e2e/interrupts.spec.ts index b5d4ac6de..2061e57bd 100644 --- a/cockpit/ag-ui/interrupts/react/e2e/interrupts.spec.ts +++ b/cockpit/ag-ui/interrupts/react/e2e/interrupts.spec.ts @@ -1,7 +1,73 @@ -import { test, expect, type Page } from '@playwright/test'; +import { test, expect, type Page, type APIRequestContext } from '@playwright/test'; +// Backend: aimock replays cockpit/ag-ui/interrupts/angular/e2e/fixtures into +// the example's real AG-UI server, whose native endpoint (/agent/native) +// pauses with an AG-UI 1.0 interrupt outcome and resumes from `resume`. The +// recording proxy (scripts/react-cockpit/ag-ui-proxy.mjs) holds, fails or +// edits the next run. test.beforeEach(async ({ request }) => { await request.post('/__reset'); }); +const edit = (request: APIRequestContext, ops: object[]) => + request.post('/__stream/edit', { data: { ops } }); +const runError = [ + { type: '^(TEXT_MESSAGE_START|STEP_STARTED)$', op: 'insertBefore', event: { type: 'RUN_ERROR', message: 'PRIVATE model failure', code: 'TEST' } }, + { type: '^(TEXT_MESSAGE_START|STEP_STARTED)$', op: 'end' }, +]; +/** The approval the real graph pauses with, as the legacy notice carries it. */ +const legacyNotice = { + type: 'CUSTOM', + name: 'on_interrupt', + value: JSON.stringify([{ kind: 'refund_approval', amount: 25, customer_id: 'cus_demo', reason: 'Fictional damaged item' }]), +}; +/** + * Arms a draft-run fault. `Negative amount` is real (the extractor fixture + * returns -1); the rest are injected protocol shapes a correct server won't + * produce, for the client's fault handling. + */ +async function draftFault(request: APIRequestContext, mode: string) { + if (mode === 'HTTP error') await request.post('/__fail-stream'); + if (mode === 'Run error') await edit(request, runError); + if (mode === 'Legacy') + await edit(request, [ + { type: '^RUN_FINISHED$', op: 'insertBefore', event: legacyNotice }, + { type: '^RUN_FINISHED$', op: 'omit', keys: ['outcome'] }, + ]); + if (mode === 'Notice EOF') + await edit(request, [ + { type: '^RUN_FINISHED$', op: 'insertBefore', event: legacyNotice }, + { type: '^RUN_FINISHED$', op: 'end' }, + ]); + const outcome = { Multiple: 'multiple', Expired: 'expired', Malformed: 'malformed', 'Child interrupt': 'child' }[mode]; + if (outcome) await edit(request, [{ type: '^RUN_FINISHED$', op: 'outcome', mode: outcome }]); + if (mode === 'Child') + await edit(request, [ + { type: '^RUN_STARTED$', op: 'insertAfter', event: { type: 'SUBAGENT_STARTED', subagentRunId: 'child', name: 'worker' } }, + ]); + if (mode === 'Tool') await edit(request, [{ type: '^MESSAGES_SNAPSHOT$', op: 'snapshot', mode: 'tool-call' }]); + if (mode === 'Duplicate identity') + await edit(request, [{ type: '^MESSAGES_SNAPSHOT$', op: 'snapshot', mode: 'duplicate-identity' }]); + if (mode === 'Wrong terminal') await edit(request, [{ type: '^RUN_FINISHED$', op: 'patch', set: { runId: 'wrong-run' } }]); +} +/** Arms a resume-run fault (injected protocol shapes). */ +async function resumeFault(request: APIRequestContext, mode: string) { + if (mode === 'Resume error') await edit(request, runError); + if (mode === 'Resume interrupted') + await edit(request, [{ type: '^RUN_FINISHED$', op: 'patch', set: { outcome: { type: 'cancelled' } } }]); + if (mode === 'Resume wrong terminal') + await edit(request, [{ type: '^RUN_FINISHED$', op: 'patch', set: { runId: 'wrong-run' } }]); + const snapshot = { + 'Resume truncated prefix': 'answer-only', + 'Resume old state': 'sent', + 'Resume duplicate identity': 'duplicate-identity', + }[mode]; + if (snapshot) await edit(request, [{ type: '^MESSAGES_SNAPSHOT$', op: 'snapshot', mode: snapshot }]); + if (mode === 'Resume second pause') { + // The draft run's real pause, offered again after the decision. + const [draft] = await (await request.get('/__captures')).json(); + const finished = draft.events.find((event: { type: string }) => event.type === 'RUN_FINISHED'); + await edit(request, [{ type: '^RUN_FINISHED$', op: 'patch', set: { outcome: finished.outcome } }]); + } +} const conversation = (page: Page) => page.getByRole('region', { name: 'Conversation', exact: true }); async function send(page: Page, text: string) { @@ -28,6 +94,7 @@ for (const mode of [ request, }) => { await page.goto('/ag-ui/interrupts/react/'); + await draftFault(request, mode); await send(page, mode); await expect(page.getByRole('status')).toHaveText( 'Start a new conversation to continue.' @@ -69,6 +136,7 @@ for (const mode of [ await expect(page.getByRole('status')).toHaveText( 'Awaiting your decision.' ); + await resumeFault(request, mode); await page .getByRole('button', { name: 'Issue refund', exact: true }) .click(); @@ -93,19 +161,22 @@ for (const phase of ['draft', 'resume']) request, }) => { await page.goto('/ag-ui/interrupts/react/'); + // The draft streams in 3-character chunks: hold after the first. The + // resume run streams no text: hold right after it starts. + if (phase === 'draft') await request.post('/__hold/stream'); await send(page, phase === 'draft' ? 'Hold' : 'Resume hold'); if (phase === 'resume') { await expect(page.getByRole('status')).toHaveText( 'Awaiting your decision.' ); + await request.post('/__hold/stream', { data: { afterType: 'RUN_STARTED' } }); await page .getByRole('button', { name: 'Issue refund', exact: true }) .click(); - } - await expect(conversation(page)).toContainText('Live partial'); - expect((await (await request.get('/__lifetime')).json()).active).toBe( - true - ); + } else await expect(conversation(page)).toContainText('Dra'); + await expect + .poll(async () => (await (await request.get('/__lifetime')).json()).active) + .toBe(true); if (exit === 'Stop') { await page.getByRole('button', { name: 'Stop', exact: true }).click(); await expect(page.getByRole('status')).toHaveText( @@ -133,19 +204,24 @@ test('an incremental draft settles to the current pause and a held resume permit request, }) => { await page.goto('/ag-ui/interrupts/react/'); + await request.post('/__hold/stream'); await send(page, 'Hold'); - await expect(conversation(page)).toContainText('Live partial'); + await expect(conversation(page)).toContainText('Dra'); + await expect(conversation(page)).not.toContainText('Draft ready.'); await expect( page.getByRole('button', { name: 'Issue refund', exact: true }) ).toHaveCount(0); await request.post('/__release'); await expect(page.getByRole('status')).toHaveText('Awaiting your decision.'); - await expect(conversation(page)).not.toContainText('Live partial'); + await expect(conversation(page)).toContainText('Draft ready.'); await page.getByRole('button', { name: 'New conversation' }).click(); await send(page, 'Resume hold'); await expect(page.getByRole('status')).toHaveText('Awaiting your decision.'); + await request.post('/__hold/stream', { data: { afterType: 'RUN_STARTED' } }); await page.getByRole('button', { name: 'Issue refund', exact: true }).click(); - await expect(conversation(page)).toContainText('Live partial'); + await expect + .poll(async () => (await (await request.get('/__lifetime')).json()).active) + .toBe(true); await expect( page.getByRole('button', { name: 'Issue refund', exact: true }) ).toBeDisabled(); @@ -297,9 +373,12 @@ for (const action of ['Issue refund', 'Edit and approve', "Don't refund"]) { ' Fictional refund ', 'Draft ready.', ]); + // The real graph's interrupt id, as its draft run reported it. + const [draft] = await (await request.get('/__captures')).json(); + const pause = draft.events.find((event: { type: string }) => event.type === 'RUN_FINISHED').outcome.interrupts[0]; expect(before[1].body.resume).toEqual([ { - interruptId: 'interrupt-1', + interruptId: pause.id, status: 'resolved', payload: action === "Don't refund" diff --git a/cockpit/ag-ui/interrupts/react/e2e/playwright.config.ts b/cockpit/ag-ui/interrupts/react/e2e/playwright.config.ts index b881aa8fc..7958a7ed9 100644 --- a/cockpit/ag-ui/interrupts/react/e2e/playwright.config.ts +++ b/cockpit/ag-ui/interrupts/react/e2e/playwright.config.ts @@ -5,6 +5,9 @@ export default defineConfig({ fullyParallel: false, use: { baseURL: 'http://127.0.0.1:4610', trace: 'retain-on-failure' }, projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }], + // aimock + the real AG-UI server, shared with cockpit/ag-ui/interrupts/angular. + globalSetup: './global-setup-impl.ts', + globalTeardown: require.resolve('../../../../../libs/e2e-harness/src/global-teardown'), webServer: { command: 'node scripts/react-cockpit/serve.mjs ag-ui-interrupts', cwd: '../../../../..', diff --git a/cockpit/ag-ui/interrupts/react/project.json b/cockpit/ag-ui/interrupts/react/project.json index 8f27266cb..f516bf0eb 100644 --- a/cockpit/ag-ui/interrupts/react/project.json +++ b/cockpit/ag-ui/interrupts/react/project.json @@ -3,7 +3,15 @@ "$schema": "../../../../node_modules/nx/schemas/project-schema.json", "sourceRoot": "cockpit/ag-ui/interrupts/react/src", "projectType": "application", - "tags": ["scope:cockpit-examples", "scope:cockpit-e2e", "framework:react"], + "implicitDependencies": [ + "cockpit-ag-ui-interrupts-angular", + "cockpit-ag-ui-interrupts-python" + ], + "tags": [ + "scope:cockpit-examples", + "scope:cockpit-e2e", + "framework:react" + ], "targets": { "test": { "executor": "@nx/vitest:test", @@ -17,11 +25,22 @@ "dependsOn": [ { "target": "build", - "projects": ["core", "content", "react"] + "projects": [ + "core", + "content", + "react" + ] }, - { "target": "react-ag-ui", "projects": ["website-runtime-preparation"] } + { + "target": "react-ag-ui", + "projects": [ + "website-runtime-preparation" + ] + } + ], + "outputs": [ + "{workspaceRoot}/dist/cockpit/ag-ui/interrupts/react" ], - "outputs": ["{workspaceRoot}/dist/cockpit/ag-ui/interrupts/react"], "options": { "command": "node scripts/react-cockpit/build.mjs ag-ui-interrupts" } @@ -32,11 +51,17 @@ "e2e": { "executor": "@nx/playwright:playwright", "cache": false, - "dependsOn": ["build"], + "dependsOn": [ + "build" + ], "options": { "config": "cockpit/ag-ui/interrupts/react/e2e/playwright.config.ts", "skipInstall": true - } + }, + "inputs": [ + "default", + "reactAimockBackend" + ] } } } diff --git a/cockpit/ag-ui/json-render/angular/e2e/fixtures/json-render.json b/cockpit/ag-ui/json-render/angular/e2e/fixtures/json-render.json index 85d43fc47..33f7d50a9 100644 --- a/cockpit/ag-ui/json-render/angular/e2e/fixtures/json-render.json +++ b/cockpit/ag-ui/json-render/angular/e2e/fixtures/json-render.json @@ -42,6 +42,371 @@ } ] } + }, + { + "match": { "systemMessage": "In 3-5 words, summarize" }, + "response": { "content": "Airline operations dashboard" } + }, + { + "match": { "systemMessage": "Provide a brief (1-2 sentence) conversational summary" }, + "response": { "content": "The fictional dashboard is ready." } + }, + { + "match": { + "userMessage": "full", + "hasToolResult": true + }, + "response": { "content": "Dashboard prepared." } + }, + { + "match": { + "userMessage": "full", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "render_spec", + "arguments": "{\"root\": \"dashboard\", \"elements\": {\"dashboard\": {\"type\": \"dashboard_grid\", \"children\": [\"row\", \"table\"]}, \"row\": {\"type\": \"container\", \"props\": {\"direction\": \"row\"}, \"children\": [\"card\", \"trend\", \"airlines\"]}, \"card\": {\"type\": \"stat_card\", \"props\": {\"label\": \"On time\", \"value\": {\"$state\": \"/on_time/value\"}, \"delta\": {\"$state\": \"/on_time/delta\"}}}, \"trend\": {\"type\": \"line_chart\", \"props\": {\"title\": \"Trend\", \"data\": {\"$state\": \"/on_time_trend\"}, \"xKey\": \"month\", \"yKey\": \"on_time_pct\"}}, \"airlines\": {\"type\": \"bar_chart\", \"props\": {\"title\": \"Airlines\", \"data\": {\"$state\": \"/flights_by_airline\"}, \"labelKey\": \"airline\", \"valueKey\": \"count\"}}, \"table\": {\"type\": \"data_grid\", \"props\": {\"title\": \"Disruptions\", \"rows\": {\"$state\": \"/recent_disruptions\"}, \"columns\": [\"flight_number\", \"type\", \"minutes\", \"route\", \"date\"]}}}}" + }, + { + "name": "query_airline_kpis", + "arguments": "{}" + }, + { + "name": "query_on_time_trend", + "arguments": "{\"months\": 6}" + }, + { + "name": "query_flights_by_airline", + "arguments": "{\"airlines\": [\"United\", \"Delta\"]}" + }, + { + "name": "query_recent_disruptions", + "arguments": "{\"limit\": 5}" + } + ] + } + }, + { + "match": { + "userMessage": "prose", + "hasToolResult": true + }, + "response": { "content": "Dashboard prepared." } + }, + { + "match": { + "userMessage": "prose", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "render_spec", + "arguments": "{\"root\": \"dashboard\", \"elements\": {\"dashboard\": {\"type\": \"dashboard_grid\", \"children\": [\"row\", \"table\"]}, \"row\": {\"type\": \"container\", \"props\": {\"direction\": \"row\"}, \"children\": [\"card\", \"trend\", \"airlines\"]}, \"card\": {\"type\": \"stat_card\", \"props\": {\"label\": \"On time\", \"value\": {\"$state\": \"/on_time/value\"}, \"delta\": {\"$state\": \"/on_time/delta\"}}}, \"trend\": {\"type\": \"line_chart\", \"props\": {\"title\": \"Trend\", \"data\": {\"$state\": \"/on_time_trend\"}, \"xKey\": \"month\", \"yKey\": \"on_time_pct\"}}, \"airlines\": {\"type\": \"bar_chart\", \"props\": {\"title\": \"Airlines\", \"data\": {\"$state\": \"/flights_by_airline\"}, \"labelKey\": \"airline\", \"valueKey\": \"count\"}}, \"table\": {\"type\": \"data_grid\", \"props\": {\"title\": \"Disruptions\", \"rows\": {\"$state\": \"/recent_disruptions\"}, \"columns\": [\"flight_number\", \"type\", \"minutes\", \"route\", \"date\"]}}}}" + }, + { + "name": "query_airline_kpis", + "arguments": "{}" + }, + { + "name": "query_on_time_trend", + "arguments": "{\"months\": 6}" + }, + { + "name": "query_flights_by_airline", + "arguments": "{\"airlines\": [\"United\", \"Delta\"]}" + }, + { + "name": "query_recent_disruptions", + "arguments": "{\"limit\": 5}" + } + ], + "content": "Preparing a fictional dashboard." + } + }, + { + "match": { + "userMessage": "filter", + "hasToolResult": true + }, + "response": { "content": "Dashboard prepared." } + }, + { + "match": { + "userMessage": "filter", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "query_recent_disruptions", + "arguments": "{\"type\": \"cancelled\", \"limit\": 5}" + } + ] + } + }, + { + "match": { + "userMessage": "all-airlines", + "hasToolResult": true + }, + "response": { "content": "Dashboard prepared." } + }, + { + "match": { + "userMessage": "all-airlines", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "query_flights_by_airline", + "arguments": "{\"airlines\": []}" + } + ] + } + }, + { + "match": { + "userMessage": "structural", + "hasToolResult": true + }, + "response": { "content": "Dashboard prepared." } + }, + { + "match": { + "userMessage": "structural", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "render_spec", + "arguments": "{\"root\": \"new-card\", \"elements\": {\"new-card\": {\"type\": \"stat_card\", \"props\": {\"label\": \"Updated flights\", \"value\": {\"$state\": \"/flights_today/value\"}}}}}" + } + ] + } + }, + { + "match": { "userMessage": "zero" }, + "response": { "content": "Dashboard prepared." } + }, + { + "match": { "toolCallId": "rounds-2" }, + "response": { "content": "Dashboard prepared." } + }, + { + "match": { "toolCallId": "rounds-1" }, + "response": { + "toolCalls": [ + { + "name": "query_airline_kpis", + "arguments": "{}", + "id": "rounds-2" + } + ] + } + }, + { + "match": { + "userMessage": "rounds", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "query_airline_kpis", + "arguments": "{}", + "id": "rounds-1" + } + ] + } + }, + { + "match": { "toolCallId": "cap-1" }, + "response": { + "toolCalls": [ + { + "name": "query_airline_kpis", + "arguments": "{}", + "id": "cap-2" + } + ] + } + }, + { + "match": { "toolCallId": "cap-2" }, + "response": { + "toolCalls": [ + { + "name": "query_airline_kpis", + "arguments": "{}", + "id": "cap-3" + } + ] + } + }, + { + "match": { "toolCallId": "cap-3" }, + "response": { + "toolCalls": [ + { + "name": "query_airline_kpis", + "arguments": "{}", + "id": "cap-4" + } + ] + } + }, + { + "match": { "toolCallId": "cap-4" }, + "response": { + "toolCalls": [ + { + "name": "query_airline_kpis", + "arguments": "{}", + "id": "cap-5" + } + ] + } + }, + { + "match": { "toolCallId": "cap-5" }, + "response": { + "toolCalls": [ + { + "name": "query_airline_kpis", + "arguments": "{}", + "id": "cap-6" + } + ] + } + }, + { + "match": { "toolCallId": "cap-6" }, + "response": { + "toolCalls": [ + { + "name": "query_airline_kpis", + "arguments": "{}", + "id": "cap-7" + } + ] + } + }, + { + "match": { "toolCallId": "cap-7" }, + "response": { + "toolCalls": [ + { + "name": "query_airline_kpis", + "arguments": "{}", + "id": "cap-8" + } + ] + } + }, + { + "match": { "toolCallId": "cap-8" }, + "response": { + "toolCalls": [ + { + "name": "query_airline_kpis", + "arguments": "{}", + "id": "cap-9" + } + ] + } + }, + { + "match": { "toolCallId": "cap-9" }, + "response": { + "toolCalls": [ + { + "name": "query_airline_kpis", + "arguments": "{}", + "id": "cap-10" + } + ] + } + }, + { + "match": { "toolCallId": "cap-10" }, + "response": { + "toolCalls": [ + { + "name": "query_airline_kpis", + "arguments": "{}", + "id": "cap-11" + } + ] + } + }, + { + "match": { "toolCallId": "cap-11" }, + "response": { + "toolCalls": [ + { + "name": "query_airline_kpis", + "arguments": "{}", + "id": "cap-12" + } + ] + } + }, + { + "match": { + "userMessage": "cap", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "query_airline_kpis", + "arguments": "{}", + "id": "cap-1" + } + ] + } + }, + { + "match": { + "userMessage": "", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "render_spec", + "arguments": "{\"root\": \"dashboard\", \"elements\": {\"dashboard\": {\"type\": \"dashboard_grid\", \"children\": [\"row\", \"table\"]}, \"row\": {\"type\": \"container\", \"props\": {\"direction\": \"row\"}, \"children\": [\"card\", \"trend\", \"airlines\"]}, \"card\": {\"type\": \"stat_card\", \"props\": {\"label\": \"On time\", \"value\": {\"$state\": \"/on_time/value\"}, \"delta\": {\"$state\": \"/on_time/delta\"}}}, \"trend\": {\"type\": \"line_chart\", \"props\": {\"title\": \"Trend\", \"data\": {\"$state\": \"/on_time_trend\"}, \"xKey\": \"month\", \"yKey\": \"on_time_pct\"}}, \"airlines\": {\"type\": \"bar_chart\", \"props\": {\"title\": \"Airlines\", \"data\": {\"$state\": \"/flights_by_airline\"}, \"labelKey\": \"airline\", \"valueKey\": \"count\"}}, \"table\": {\"type\": \"data_grid\", \"props\": {\"title\": \"Disruptions\", \"rows\": {\"$state\": \"/recent_disruptions\"}, \"columns\": [\"flight_number\", \"type\", \"minutes\", \"route\", \"date\"]}}}}" + }, + { + "name": "query_airline_kpis", + "arguments": "{}" + }, + { + "name": "query_on_time_trend", + "arguments": "{\"months\": 6}" + }, + { + "name": "query_flights_by_airline", + "arguments": "{\"airlines\": [\"United\", \"Delta\"]}" + }, + { + "name": "query_recent_disruptions", + "arguments": "{\"limit\": 5}" + } + ] + } + }, + { + "match": { + "userMessage": "", + "hasToolResult": true + }, + "response": { "content": "Dashboard prepared." } } ] } diff --git a/cockpit/ag-ui/json-render/react/e2e/global-setup-impl.ts b/cockpit/ag-ui/json-render/react/e2e/global-setup-impl.ts new file mode 100644 index 000000000..cfb7820c0 --- /dev/null +++ b/cockpit/ag-ui/json-render/react/e2e/global-setup-impl.ts @@ -0,0 +1,6 @@ +// eslint-disable-next-line @nx/enforce-module-boundaries -- Shared aimock setup for the React cockpit examples. +import { reactAimockAgUiGlobalSetup } from '../../../../shared/react-aimock-setup'; + +// aimock replays the Angular example's fixtures into the example's real AG-UI +// server; the static server (webServer) proxies the agent endpoint to it. +export default reactAimockAgUiGlobalSetup({ example: 'ag-ui/json-render', port: 4612 }); diff --git a/cockpit/ag-ui/json-render/react/e2e/json-render.spec.ts b/cockpit/ag-ui/json-render/react/e2e/json-render.spec.ts index 046d9b047..9416881e2 100644 --- a/cockpit/ag-ui/json-render/react/e2e/json-render.spec.ts +++ b/cockpit/ag-ui/json-render/react/e2e/json-render.spec.ts @@ -1,4 +1,66 @@ -import { test, expect, type Page } from '@playwright/test'; +import { test, expect, type Page, type APIRequestContext } from '@playwright/test'; +// Backend: aimock replays cockpit/ag-ui/json-render/angular/e2e/fixtures (the +// scripted dashboard model of the Python tests) into the example's real AG-UI +// server. The recording proxy (scripts/react-cockpit/ag-ui-proxy.mjs) holds, +// fails or edits the next run; /__captures has what the server sent. +const edit = (request: APIRequestContext, ops: object[]) => + request.post('/__stream/edit', { data: { ops } }); +/** Where each held stage stops the real stream. */ +async function hold(request: APIRequestContext, stage: string) { + const afterType = { state: 'STATE_SNAPSHOT', result: 'TOOL_CALL_RESULT', final: 'MESSAGES_SNAPSHOT' }[stage]; + await request.post('/__hold/stream', { data: { afterType } }); +} +/** The final snapshot of each finished run, as the real server sent it. */ +async function finals(request: APIRequestContext) { + const captures: { events: { type: string; messages?: unknown[] }[] }[] = await ( + await request.get('/__captures') + ).json(); + return captures.map((capture) => capture.events.findLast((event) => event.type === 'MESSAGES_SNAPSHOT')!.messages!); +} +/** Arms a fault on the next run: injected protocol shapes a correct server won't produce. */ +async function fault(request: APIRequestContext, mode: string) { + const finished = (set: object) => edit(request, [{ type: '^RUN_FINISHED$', op: 'patch', set }]); + const snapshot = (name: string) => edit(request, [{ type: '^MESSAGES_SNAPSHOT$', op: 'snapshot', mode: name }]); + if (mode === 'http-error') await request.post('/__fail-stream'); + if (mode === 'run-error') + await edit(request, [ + { type: '^RUN_STARTED$', op: 'insertAfter', event: { type: 'RUN_ERROR', message: 'PRIVATE provider failure' } }, + { type: '^(?!RUN_STARTED$)', op: 'end' }, + ]); + if (mode === 'wrong-run') await finished({ runId: 'wrong-run' }); + if (mode === 'wrong-thread') await finished({ threadId: 'wrong-thread' }); + if (mode === 'eof') await edit(request, [{ type: '^RUN_FINISHED$', op: 'drop' }]); + if (mode === 'child') + await edit(request, [ + { type: '^RUN_STARTED$', op: 'insertAfter', event: { type: 'SUBAGENT_STARTED', subagentRunId: 'child', name: 'worker' } }, + ]); + if (mode === 'legacy') + await edit(request, [ + { type: '^RUN_STARTED$', op: 'insertAfter', event: { type: 'CUSTOM', name: 'on_interrupt', value: { note: 'legacy pause' } } }, + ]); + if (mode === 'notice-eof') + await edit(request, [ + { type: '^RUN_FINISHED$', op: 'insertBefore', event: { type: 'CUSTOM', name: 'unsupported', value: {} } }, + { type: '^RUN_FINISHED$', op: 'end' }, + ]); + if (mode === 'bad-state') + await edit(request, [ + { type: '^STATE_SNAPSHOT$', op: 'setPath', path: 'snapshot.on_time', value: { value: { invalid: true }, delta: null } }, + ]); + const snapshotModes: Record = { + 'truncated-prefix': 'from-human', + 'duplicate-result': 'duplicate-result', + 'orphan-result': 'orphan-result', + 'missing-result': 'missing-result', + 'wrong-result-id': 'wrong-result-id', + 'unknown-tool': 'unknown-tool', + }; + if (snapshotModes[mode]) await snapshot(snapshotModes[mode]); + if (mode === 'malformed-spec') + await edit(request, [ + { type: '^(TOOL_CALL_ARGS|MESSAGES_SNAPSHOT)$', op: 'replace', from: 'dashboard_grid', to: 'custom_html' }, + ]); +} const base = '/ag-ui/json-render/react/'; const rows = (page: Page) => page.locator('[data-dashboard-message]'); async function send(page: Page, text: string) { @@ -66,6 +128,7 @@ test('actual compiled graph and installed SDK retain all six views, canonical ba } const requests = await (await request.get('/__requests')).json(); expect(requests).toHaveLength(8); + const final = await finals(request); expect( new Set( requests.map( @@ -88,10 +151,10 @@ test('actual compiled graph and installed SDK retain all six views, canonical ba index ? nativeIds[index - 1] : [] ); if (index) { - expect(prefix).toHaveLength(requests[index - 1].final.length); + expect(prefix).toHaveLength(final[index - 1].length); for (const message of prefix) expect(message).toEqual( - requests[index - 1].final.find( + final[index - 1].find( (row: { id: string }) => row.id === message.id ) ); @@ -100,7 +163,7 @@ test('actual compiled graph and installed SDK retain all six views, canonical ba ); } const calls = new Set(); - for (const message of entry.final) { + for (const message of final[index] as { role: string; id: string; toolCallId?: string; toolCalls?: { id: string }[] }[]) { if (message.role === 'assistant') for (const call of message.toolCalls ?? []) calls.add(call.id); if (message.role === 'tool') { @@ -126,7 +189,7 @@ for (const stage of ['state', 'result', 'final']) request, }) => { await page.goto(base); - await request.post('/__hold/' + stage); + await hold(request, stage); await send(page, 'full'); await expect .poll( @@ -155,7 +218,7 @@ test('retained layout reads valid streamed shared-state updates while native com const chartData = page.locator('.dashboard-chart-data').first(); await chartData.locator('summary').click(); const originalChart = await chartData.elementHandle(); - await request.post('/__hold/final'); + await hold(request, 'final'); await send(page, 'filter'); await expect .poll(async () => (await (await request.get('/__lifetime')).json()).active) @@ -215,7 +278,7 @@ for (const mode of [ await page.goto(base); await send(page, 'full'); await expect(page.getByRole('status', { name: 'Conversation status' })).toHaveText('Response complete.'); - await request.post('/__mode/' + mode); + await fault(request, mode); await send(page, 'full'); await expect(page.getByRole('status', { name: 'Conversation status' })).toHaveText( 'Start a new conversation to continue.' @@ -235,7 +298,7 @@ for (const action of ['Stop', 'pagehide']) request, }) => { await page.goto(base); - await request.post('/__hold/result'); + await hold(request, 'result'); await send(page, 'full'); await expect .poll( @@ -351,9 +414,10 @@ test('the authored developer endpoint routes all native requests and stays out o .slice(0, -1) .map((message: { id: string }) => message.id) ).toEqual(priorIds); + const [first] = await finals(request); for (const message of wire[1].body.messages.slice(0, -1)) expect(message).toEqual( - wire[0].final.find((row: { id: string }) => row.id === message.id) + (first as { id: string }[]).find((row) => row.id === message.id) ); await expect(frame.locator('body')).not.toContainText('/developer-agent'); }); diff --git a/cockpit/ag-ui/json-render/react/e2e/playwright.config.ts b/cockpit/ag-ui/json-render/react/e2e/playwright.config.ts index 3c4d57fea..d5f5ecdfe 100644 --- a/cockpit/ag-ui/json-render/react/e2e/playwright.config.ts +++ b/cockpit/ag-ui/json-render/react/e2e/playwright.config.ts @@ -7,6 +7,9 @@ export default defineConfig({ expect: { timeout: 15000 }, use: { baseURL: 'http://127.0.0.1:4612', trace: 'retain-on-failure' }, projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }], + // aimock + the real AG-UI server, shared with cockpit/ag-ui/json-render/angular. + globalSetup: './global-setup-impl.ts', + globalTeardown: require.resolve('../../../../../libs/e2e-harness/src/global-teardown'), webServer: { command: 'node scripts/react-cockpit/serve.mjs ag-ui-json-render', cwd: '../../../../..', diff --git a/cockpit/ag-ui/json-render/react/project.json b/cockpit/ag-ui/json-render/react/project.json index 8b87101cc..be021b4e6 100644 --- a/cockpit/ag-ui/json-render/react/project.json +++ b/cockpit/ag-ui/json-render/react/project.json @@ -3,7 +3,15 @@ "$schema": "../../../../node_modules/nx/schemas/project-schema.json", "sourceRoot": "cockpit/ag-ui/json-render/react/src", "projectType": "application", - "tags": ["scope:cockpit-examples", "scope:cockpit-e2e", "framework:react"], + "implicitDependencies": [ + "cockpit-ag-ui-json-render-angular", + "cockpit-ag-ui-json-render-python" + ], + "tags": [ + "scope:cockpit-examples", + "scope:cockpit-e2e", + "framework:react" + ], "targets": { "test": { "executor": "@nx/vitest:test", @@ -15,26 +23,51 @@ "executor": "nx:run-commands", "cache": false, "dependsOn": [ - { "target": "build", "projects": ["core", "content", "react"] }, - { "target": "react-ag-ui", "projects": ["website-runtime-preparation"] } + { + "target": "build", + "projects": [ + "core", + "content", + "react" + ] + }, + { + "target": "react-ag-ui", + "projects": [ + "website-runtime-preparation" + ] + } + ], + "outputs": [ + "{workspaceRoot}/dist/cockpit/ag-ui/json-render/react" ], - "outputs": ["{workspaceRoot}/dist/cockpit/ag-ui/json-render/react"], "options": { "command": "node scripts/react-cockpit/build.mjs ag-ui-json-render" } }, - "lint": { "executor": "@nx/eslint:lint" }, + "lint": { + "executor": "@nx/eslint:lint" + }, "e2e": { "executor": "@nx/playwright:playwright", "cache": false, "dependsOn": [ "build", - { "target": "smoke", "projects": ["cockpit-ag-ui-json-render-python"] } + { + "target": "smoke", + "projects": [ + "cockpit-ag-ui-json-render-python" + ] + } ], "options": { "config": "cockpit/ag-ui/json-render/react/e2e/playwright.config.ts", "skipInstall": true - } + }, + "inputs": [ + "default", + "reactAimockBackend" + ] } } } diff --git a/cockpit/ag-ui/streaming/angular/e2e/fixtures/streaming.json b/cockpit/ag-ui/streaming/angular/e2e/fixtures/streaming.json index d54869ff9..c83afe81d 100644 --- a/cockpit/ag-ui/streaming/angular/e2e/fixtures/streaming.json +++ b/cockpit/ag-ui/streaming/angular/e2e/fixtures/streaming.json @@ -7,6 +7,41 @@ "response": { "content": "Angular signals are a reactive primitive (signal, computed, effect) that track dependencies to provide fine-grained reactivity and more efficient change detection. They let you update state synchronously via set()/update() and ensure only consumers that read an affected signal are re\u2011evaluated." } + }, + { + "match": { "userMessage": "Alpha" }, + "response": { "content": "Reply to Alpha" } + }, + { + "match": { "userMessage": "Beta" }, + "response": { "content": "Reply to Beta" } + }, + { + "match": { "userMessage": "Gamma" }, + "response": { "content": "Reply to Gamma" } + }, + { + "match": { "userMessage": "Recovered" }, + "response": { "content": "Reply to Recovered" } + }, + { + "match": { "userMessage": "Hold" }, + "response": { "content": "Reply to Hold, settled." }, + "chunkSize": 9 + }, + { + "match": { "userMessage": "Model failure" }, + "response": { + "error": { + "message": "PRIVATE model failure", + "type": "server_error" + }, + "status": 500 + } + }, + { + "match": { "userMessage": "" }, + "response": { "content": "Reply recorded." } } ] } diff --git a/cockpit/ag-ui/streaming/react/e2e/global-setup-impl.ts b/cockpit/ag-ui/streaming/react/e2e/global-setup-impl.ts new file mode 100644 index 000000000..9466c1959 --- /dev/null +++ b/cockpit/ag-ui/streaming/react/e2e/global-setup-impl.ts @@ -0,0 +1,6 @@ +// eslint-disable-next-line @nx/enforce-module-boundaries -- Shared aimock setup for the React cockpit examples. +import { reactAimockAgUiGlobalSetup } from '../../../../shared/react-aimock-setup'; + +// aimock replays the Angular example's fixtures into the example's real AG-UI +// server; the static server (webServer) proxies the agent endpoint to it. +export default reactAimockAgUiGlobalSetup({ example: 'ag-ui/streaming', port: 4609 }); diff --git a/cockpit/ag-ui/streaming/react/e2e/playwright.config.ts b/cockpit/ag-ui/streaming/react/e2e/playwright.config.ts index d31d2119d..c7d5c8a4e 100644 --- a/cockpit/ag-ui/streaming/react/e2e/playwright.config.ts +++ b/cockpit/ag-ui/streaming/react/e2e/playwright.config.ts @@ -5,6 +5,9 @@ export default defineConfig({ fullyParallel: false, use: { baseURL: 'http://127.0.0.1:4609', trace: 'retain-on-failure' }, projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }], + // aimock + the real AG-UI server, shared with cockpit/ag-ui/streaming/angular. + globalSetup: './global-setup-impl.ts', + globalTeardown: require.resolve('../../../../../libs/e2e-harness/src/global-teardown'), webServer: { command: 'node scripts/react-cockpit/serve.mjs ag-ui-streaming', cwd: '../../../../..', diff --git a/cockpit/ag-ui/streaming/react/e2e/streaming.spec.ts b/cockpit/ag-ui/streaming/react/e2e/streaming.spec.ts index 84742003a..35aadd7f0 100644 --- a/cockpit/ag-ui/streaming/react/e2e/streaming.spec.ts +++ b/cockpit/ag-ui/streaming/react/e2e/streaming.spec.ts @@ -1,7 +1,49 @@ -import { test, expect, type Page } from '@playwright/test'; +import { test, expect, type Page, type APIRequestContext } from '@playwright/test'; +// Backend: aimock replays cockpit/ag-ui/streaming/angular/e2e/fixtures into +// the example's real AG-UI server (ag-ui-langgraph). The recording proxy +// (scripts/react-cockpit/ag-ui-proxy.mjs) holds, fails or edits the next run. test.beforeEach(async ({ request }) => { await request.post('/__reset'); }); +/** + * Arms the protocol fault a scenario needs on the next run. All but + * `HTTP error` and `Run error` are injected data: stream shapes a correct + * server won't produce, for the client's fault handling. + */ +async function fault(request: APIRequestContext, text: string) { + const edit = (ops: object[]) => request.post('/__stream/edit', { data: { ops } }); + const finished = (set: object) => edit([{ type: '^RUN_FINISHED$', op: 'patch', set }]); + if (text === 'HTTP error') await request.post('/__fail-stream'); + // `Model failure` is real: its fixture fails the model call, and the server + // drops the stream with no terminal event. + if (text === 'Run error') + await edit([ + { type: '^TEXT_MESSAGE_START$', op: 'insertBefore', event: { type: 'RUN_ERROR', message: 'PRIVATE model failure', code: 'TEST' } }, + { type: '^TEXT_MESSAGE_START$', op: 'end' }, + ]); + if (text === 'Interrupt') + await finished({ outcome: { type: 'interrupt', interrupts: [{ id: 'pause', reason: 'approval' }] } }); + if (text === 'Cancelled') await finished({ outcome: { type: 'cancelled' } }); + if (text === 'Pending tools') + await finished({ outcome: { type: 'success', pendingToolCallIds: ['call'] } }); + if (text === 'Wrong terminal') await finished({ runId: 'different-run' }); + if (text === 'Notice EOF') + await edit([ + { type: '^MESSAGES_SNAPSHOT$', op: 'insertBefore', event: { type: 'CUSTOM', name: 'on_interrupt', value: { note: 'notice' } } }, + { type: '^MESSAGES_SNAPSHOT$', op: 'end' }, + ]); + if (text === 'Child') + await edit([ + { type: '^RUN_STARTED$', op: 'insertAfter', event: { type: 'SUBAGENT_STARTED', subagentRunId: 'child', name: 'worker' } }, + ]); + const snapshot = { + Tool: 'tool-call', + 'Duplicate identity': 'duplicate-identity', + 'Truncated prefix': 'truncated-prefix', + 'Old pair': 'old-pair', + }[text]; + if (snapshot) await edit([{ type: '^MESSAGES_SNAPSHOT$', op: 'snapshot', mode: snapshot }]); +} const conversation = (page: Page) => page.getByRole('region', { name: 'Conversation', exact: true }); async function send(page: Page, text: string) { @@ -36,13 +78,17 @@ test('installed AG-UI client sends full native history on one lazy UUID and New await expect(conversation(page).locator('li')).toHaveCount(4); const before: Wire[] = await (await request.get('/__requests')).json(); expect(before).toHaveLength(2); + // The client returns the server's last STATE_SNAPSHOT as the next run's state. + const [firstRun] = await (await request.get('/__captures')).json(); + const snapshot = firstRun.events.filter((e: { type: string }) => e.type === 'STATE_SNAPSHOT').at(-1).snapshot; + expect(before[0].body.state).toEqual({}); + expect(before[1].body.state).toEqual(snapshot); for (const entry of before) { expect(entry.path).toBe('/ag-ui/streaming/agent'); expect(entry.method).toBe('POST'); expect(entry.body.protocolVersion).toBe('1.0'); expect(entry.body.tools).toEqual([]); expect(entry.body.context).toEqual([]); - expect(entry.body.state).toEqual({}); expect(entry.body.forwardedProps).toEqual({}); } expect(before[0].body.messages.map((x) => [x.role, x.content])).toEqual([ @@ -73,21 +119,24 @@ test('native incremental partial settles to one canonical final row', async ({ request, }) => { await page.goto('/ag-ui/streaming/react/'); + // The answer streams in 9-character chunks; hold after the first one. + await request.post('/__hold/stream'); await send(page, 'Hold'); - await expect(conversation(page)).toContainText('Live partial'); + await expect(conversation(page)).toContainText('Reply to'); + await expect(conversation(page)).not.toContainText('settled'); await expect( page.getByRole('button', { name: 'New conversation' }) ).toBeDisabled(); await request.post('/__release'); await expect(page.getByRole('status')).toHaveText('Response complete.'); await expect(conversation(page).locator('li')).toHaveCount(2); - await expect(conversation(page)).toContainText('Reply to Hold'); - await expect(conversation(page)).not.toContainText('Live partial'); + await expect(conversation(page)).toContainText('Reply to Hold, settled.'); }); for (const text of [ 'HTTP error', 'Run error', + 'Model failure', 'Interrupt', 'Cancelled', 'Notice EOF', @@ -108,6 +157,7 @@ for (const text of [ await send(page, 'Alpha'); await expect(page.getByRole('status')).toHaveText('Response complete.'); } + await fault(request, text); await send(page, text); await expect(page.getByRole('status')).toHaveText( 'Start a new conversation to continue.' @@ -134,8 +184,9 @@ for (const exit of ['Stop', 'pagehide']) { request, }) => { await page.goto('/ag-ui/streaming/react/'); + await request.post('/__hold/stream'); await send(page, 'Hold'); - await expect(conversation(page)).toContainText('Live partial'); + await expect(conversation(page)).toContainText('Reply to'); expect((await (await request.get('/__lifetime')).json()).active).toBe(true); if (exit === 'Stop') { await page.getByRole('button', { name: 'Stop', exact: true }).click(); @@ -156,7 +207,7 @@ for (const exit of ['Stop', 'pagehide']) { await request.post('/__release'); expect(await (await request.get('/__requests')).json()).toHaveLength(1); if (exit === 'Stop') - await expect(conversation(page)).not.toContainText('Reply to Hold'); + await expect(conversation(page)).not.toContainText('settled'); else await expect(conversation(page)).toHaveCount(0); }); } diff --git a/cockpit/ag-ui/streaming/react/project.json b/cockpit/ag-ui/streaming/react/project.json index 2c5ec25b9..40a78e62a 100644 --- a/cockpit/ag-ui/streaming/react/project.json +++ b/cockpit/ag-ui/streaming/react/project.json @@ -3,6 +3,10 @@ "$schema": "../../../../node_modules/nx/schemas/project-schema.json", "sourceRoot": "cockpit/ag-ui/streaming/react/src", "projectType": "application", + "implicitDependencies": [ + "cockpit-ag-ui-streaming-angular", + "cockpit-ag-ui-streaming-python" + ], "tags": [ "scope:cockpit-examples", "scope:cockpit-e2e", @@ -27,7 +31,12 @@ "react" ] }, - { "target": "react-ag-ui", "projects": ["website-runtime-preparation"] } + { + "target": "react-ag-ui", + "projects": [ + "website-runtime-preparation" + ] + } ], "outputs": [ "{workspaceRoot}/dist/cockpit/ag-ui/streaming/react" @@ -48,7 +57,11 @@ "options": { "config": "cockpit/ag-ui/streaming/react/e2e/playwright.config.ts", "skipInstall": true - } + }, + "inputs": [ + "default", + "reactAimockBackend" + ] } } } diff --git a/cockpit/ag-ui/subagents/angular/e2e/fixtures/subagents.json b/cockpit/ag-ui/subagents/angular/e2e/fixtures/subagents.json index 418c0e01c..5ffb3e010 100644 --- a/cockpit/ag-ui/subagents/angular/e2e/fixtures/subagents.json +++ b/cockpit/ag-ui/subagents/angular/e2e/fixtures/subagents.json @@ -87,6 +87,298 @@ "response": { "content": "All set — I researched both airports, found flights, and drafted an itinerary. Recommended plan: book the balanced mid-morning nonstop from LAX to JFK, arrive about two hours early, and take the AirTrain into the city. Pack for mild spring weather in New York." } + }, + { + "match": { "systemMessage": "In 3-5 words, summarize" }, + "response": { "content": "Trip planning" } + }, + { + "match": { + "systemMessage": "Booking Agent", + "userMessage": "failure:" + }, + "response": { + "error": { + "message": "private-provider-marker-14", + "type": "server_error" + }, + "status": 500 + } + }, + { + "match": { + "systemMessage": "Research Agent", + "userMessage": "(round 2)" + }, + "response": { "content": "A fictional specialist observation. (research, round 2)" } + }, + { + "match": { "systemMessage": "Research Agent" }, + "response": { "content": "A fictional specialist observation. (research, round 1)" } + }, + { + "match": { + "systemMessage": "Booking Agent", + "userMessage": "(round 2)" + }, + "response": { "content": "A fictional specialist observation. (booking, round 2)" } + }, + { + "match": { "systemMessage": "Booking Agent" }, + "response": { "content": "A fictional specialist observation. (booking, round 1)" } + }, + { + "match": { + "systemMessage": "Itinerary Agent", + "userMessage": "(round 2)" + }, + "response": { "content": "A fictional specialist observation. (itinerary, round 2)" } + }, + { + "match": { "systemMessage": "Itinerary Agent" }, + "response": { "content": "A fictional specialist observation. (itinerary, round 1)" } + }, + { + "match": { + "systemMessage": "Trip Planner Orchestrator", + "userMessage": "direct" + }, + "response": { "content": "A fictional parent summary." } + }, + { + "match": { + "systemMessage": "Trip Planner Orchestrator", + "userMessage": "batch", + "hasToolResult": true + }, + "response": { "content": "A fictional parent summary." } + }, + { + "match": { + "systemMessage": "Trip Planner Orchestrator", + "userMessage": "batch", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "task", + "arguments": "{\"role\": \"research\", \"task_description\": \"batch: a fictional LAX to JFK trip.\"}" + }, + { + "name": "task", + "arguments": "{\"role\": \"booking\", \"task_description\": \"batch: a fictional LAX to JFK trip.\"}" + }, + { + "name": "task", + "arguments": "{\"role\": \"itinerary\", \"task_description\": \"batch: a fictional LAX to JFK trip.\"}" + } + ] + } + }, + { + "match": { + "systemMessage": "Trip Planner Orchestrator", + "userMessage": "sequential", + "toolResultContains": "(itinerary, round 1)" + }, + "response": { "content": "A fictional parent summary." } + }, + { + "match": { + "systemMessage": "Trip Planner Orchestrator", + "userMessage": "sequential", + "toolResultContains": "(booking, round 1)" + }, + "response": { + "toolCalls": [ + { + "name": "task", + "arguments": "{\"role\": \"itinerary\", \"task_description\": \"sequential: a fictional LAX to JFK trip.\"}" + } + ] + } + }, + { + "match": { + "systemMessage": "Trip Planner Orchestrator", + "userMessage": "sequential", + "toolResultContains": "(research, round 1)" + }, + "response": { + "toolCalls": [ + { + "name": "task", + "arguments": "{\"role\": \"booking\", \"task_description\": \"sequential: a fictional LAX to JFK trip.\"}" + } + ] + } + }, + { + "match": { + "systemMessage": "Trip Planner Orchestrator", + "userMessage": "sequential", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "task", + "arguments": "{\"role\": \"research\", \"task_description\": \"sequential: a fictional LAX to JFK trip.\"}" + } + ] + } + }, + { + "match": { + "systemMessage": "Trip Planner Orchestrator", + "userMessage": "failure", + "toolResultContains": "(itinerary, round 1)" + }, + "response": { "content": "A fictional parent summary." } + }, + { + "match": { + "systemMessage": "Trip Planner Orchestrator", + "userMessage": "failure", + "toolResultContains": "(booking, round 1)" + }, + "response": { + "toolCalls": [ + { + "name": "task", + "arguments": "{\"role\": \"itinerary\", \"task_description\": \"failure: a fictional LAX to JFK trip.\"}" + } + ] + } + }, + { + "match": { + "systemMessage": "Trip Planner Orchestrator", + "userMessage": "failure", + "toolResultContains": "(research, round 1)" + }, + "response": { + "toolCalls": [ + { + "name": "task", + "arguments": "{\"role\": \"booking\", \"task_description\": \"failure: a fictional LAX to JFK trip.\"}" + } + ] + } + }, + { + "match": { + "systemMessage": "Trip Planner Orchestrator", + "userMessage": "failure", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "task", + "arguments": "{\"role\": \"research\", \"task_description\": \"failure: a fictional LAX to JFK trip.\"}" + } + ] + } + }, + { + "match": { + "systemMessage": "Trip Planner Orchestrator", + "userMessage": "rounds", + "toolResultContains": "(itinerary, round 2)" + }, + "response": { "content": "A fictional parent summary." } + }, + { + "match": { + "systemMessage": "Trip Planner Orchestrator", + "userMessage": "rounds", + "toolResultContains": "(booking, round 2)" + }, + "response": { + "toolCalls": [ + { + "name": "task", + "arguments": "{\"role\": \"itinerary\", \"task_description\": \"rounds: a fictional LAX to JFK trip. (round 2)\"}" + } + ] + } + }, + { + "match": { + "systemMessage": "Trip Planner Orchestrator", + "userMessage": "rounds", + "toolResultContains": "(research, round 2)" + }, + "response": { + "toolCalls": [ + { + "name": "task", + "arguments": "{\"role\": \"booking\", \"task_description\": \"rounds: a fictional LAX to JFK trip. (round 2)\"}" + } + ] + } + }, + { + "match": { + "systemMessage": "Trip Planner Orchestrator", + "userMessage": "rounds", + "toolResultContains": "(itinerary, round 1)" + }, + "response": { + "toolCalls": [ + { + "name": "task", + "arguments": "{\"role\": \"research\", \"task_description\": \"rounds: a fictional LAX to JFK trip. (round 2)\"}" + } + ] + } + }, + { + "match": { + "systemMessage": "Trip Planner Orchestrator", + "userMessage": "rounds", + "toolResultContains": "(booking, round 1)" + }, + "response": { + "toolCalls": [ + { + "name": "task", + "arguments": "{\"role\": \"itinerary\", \"task_description\": \"rounds: a fictional LAX to JFK trip.\"}" + } + ] + } + }, + { + "match": { + "systemMessage": "Trip Planner Orchestrator", + "userMessage": "rounds", + "toolResultContains": "(research, round 1)" + }, + "response": { + "toolCalls": [ + { + "name": "task", + "arguments": "{\"role\": \"booking\", \"task_description\": \"rounds: a fictional LAX to JFK trip.\"}" + } + ] + } + }, + { + "match": { + "systemMessage": "Trip Planner Orchestrator", + "userMessage": "rounds", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "task", + "arguments": "{\"role\": \"research\", \"task_description\": \"rounds: a fictional LAX to JFK trip.\"}" + } + ] + } } ] } diff --git a/cockpit/ag-ui/subagents/react/e2e/global-setup-impl.ts b/cockpit/ag-ui/subagents/react/e2e/global-setup-impl.ts new file mode 100644 index 000000000..eeae8e7de --- /dev/null +++ b/cockpit/ag-ui/subagents/react/e2e/global-setup-impl.ts @@ -0,0 +1,6 @@ +// eslint-disable-next-line @nx/enforce-module-boundaries -- Shared aimock setup for the React cockpit examples. +import { reactAimockAgUiGlobalSetup } from '../../../../shared/react-aimock-setup'; + +// aimock replays the Angular example's fixtures into the example's real AG-UI +// server; the static server (webServer) proxies the agent endpoint to it. +export default reactAimockAgUiGlobalSetup({ example: 'ag-ui/subagents', port: 4613 }); diff --git a/cockpit/ag-ui/subagents/react/e2e/playwright.config.ts b/cockpit/ag-ui/subagents/react/e2e/playwright.config.ts index 319aabf13..017cec7bc 100644 --- a/cockpit/ag-ui/subagents/react/e2e/playwright.config.ts +++ b/cockpit/ag-ui/subagents/react/e2e/playwright.config.ts @@ -7,6 +7,9 @@ export default defineConfig({ expect: { timeout: 15000 }, use: { baseURL: 'http://127.0.0.1:4613', trace: 'retain-on-failure' }, projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }], + // aimock + the real AG-UI server, shared with cockpit/ag-ui/subagents/angular. + globalSetup: './global-setup-impl.ts', + globalTeardown: require.resolve('../../../../../libs/e2e-harness/src/global-teardown'), webServer: { command: 'node scripts/react-cockpit/serve.mjs ag-ui-subagents', cwd: '../../../../..', diff --git a/cockpit/ag-ui/subagents/react/e2e/subagents.spec.ts b/cockpit/ag-ui/subagents/react/e2e/subagents.spec.ts index 110287b89..4eda3af2f 100644 --- a/cockpit/ag-ui/subagents/react/e2e/subagents.spec.ts +++ b/cockpit/ag-ui/subagents/react/e2e/subagents.spec.ts @@ -4,6 +4,77 @@ import { type Page, type APIRequestContext, } from '@playwright/test'; +// Backend: aimock replays cockpit/ag-ui/subagents/angular/e2e/fixtures (the +// scripted orchestrator and specialists of the Python tests) into the +// example's real AG-UI server, whose SubagentEmittingAgent runs real child +// graphs. The recording proxy (scripts/react-cockpit/ag-ui-proxy.mjs) holds, +// fails or edits the next run; /__captures has what the server sent. +const edit = (request: APIRequestContext, ops: object[]) => + request.post('/__stream/edit', { data: { ops } }); +const childEvent = '"subagentRunId"'; +/** Where each held stage stops the real stream. */ +async function hold(request: APIRequestContext, stage: string) { + const data = { + 'child-text': { afterPattern: `"type": ?"TEXT_MESSAGE_CONTENT".*${childEvent}|${childEvent}.*"type": ?"TEXT_MESSAGE_CONTENT"` }, + 'child-finished': { afterType: 'SUBAGENT_FINISHED' }, + final: { afterType: 'MESSAGES_SNAPSHOT' }, + }[stage]; + await request.post('/__hold/stream', { data }); +} +/** + * Arms a scenario on the next run. `without-child-tokens` stands in for a + * nonstreaming specialist provider (no child text events, only the final + * result); the rest are injected protocol shapes a correct server won't + * produce. + */ +async function arm(request: APIRequestContext, name: string) { + const finished = (set: object) => edit(request, [{ type: '^RUN_FINISHED$', op: 'patch', set }]); + const snapshot = (snapshotMode: string) => edit(request, [{ type: '^MESSAGES_SNAPSHOT$', op: 'snapshot', mode: snapshotMode }]); + const firstChild = (set: object) => edit(request, [{ type: '^SUBAGENT_STARTED$', op: 'patch', once: true, set }]); + if (name === 'without-child-tokens') + await edit(request, [{ type: '^TEXT_MESSAGE_(START|CONTENT|END)$', dataPattern: childEvent, op: 'drop' }]); + if (name === 'http-error') await request.post('/__fail-stream'); + if (name === 'run-error') + await edit(request, [ + { type: '^RUN_STARTED$', op: 'insertAfter', event: { type: 'RUN_ERROR', message: 'PRIVATE provider failure' } }, + { type: '^(?!RUN_STARTED$)', op: 'end' }, + ]); + if (name === 'orphan-text') + await edit(request, [ + { type: '^RUN_STARTED$', op: 'insertAfter', event: { type: 'TEXT_MESSAGE_START', messageId: 'orphan-m1', role: 'assistant', subagentRunId: 'orphan' } }, + { type: '^RUN_STARTED$', op: 'insertAfter', event: { type: 'TEXT_MESSAGE_CONTENT', messageId: 'orphan-m1', delta: 'Unknown child', subagentRunId: 'orphan' } }, + { type: '^RUN_STARTED$', op: 'insertAfter', event: { type: 'TEXT_MESSAGE_END', messageId: 'orphan-m1', subagentRunId: 'orphan' } }, + ]); + if (name === 'wrong-run') await finished({ runId: 'wrong-run' }); + if (name === 'wrong-thread') await finished({ threadId: 'wrong-thread' }); + if (name === 'eof') await edit(request, [{ type: '^RUN_FINISHED$', op: 'drop' }]); + if (name === 'child') + await edit(request, [ + { type: '^RUN_STARTED$', op: 'insertAfter', event: { type: 'SUBAGENT_STARTED', subagentRunId: 'child', name: 'worker' } }, + ]); + if (name === 'legacy') + await edit(request, [ + { type: '^RUN_STARTED$', op: 'insertAfter', event: { type: 'CUSTOM', name: 'on_interrupt', value: { note: 'legacy pause' } } }, + ]); + if (name === 'notice-eof') + await edit(request, [ + { type: '^RUN_FINISHED$', op: 'insertBefore', event: { type: 'CUSTOM', name: 'unsupported', value: {} } }, + { type: '^RUN_FINISHED$', op: 'end' }, + ]); + const snapshotModes: Record = { + 'truncated-prefix': 'from-human', + 'duplicate-result': 'duplicate-result', + 'orphan-result': 'orphan-result', + 'missing-result': 'missing-result', + 'wrong-result-id': 'wrong-result-id', + 'unknown-tool': 'unknown-tool', + }; + if (snapshotModes[name]) await snapshot(snapshotModes[name]); + if (name === 'unfinished-child') await edit(request, [{ type: '^SUBAGENT_FINISHED$', op: 'drop', once: true }]); + if (name === 'wrong-parent') await firstChild({ parentToolCallId: 'foreign-call' }); + if (name === 'unknown-role') await firstChild({ name: 'unknown' }); + if (name === 'wrong-child-id') await firstChild({ subagentRunId: 'foreign-child' }); +} test('the authored developer endpoint routes all native requests and stays out of visible text', async ({ page, request, @@ -41,7 +112,7 @@ test('the authored developer endpoint routes all native requests and stays out o ) ); } - const wire = await (await request.get('/__requests')).json(); + const wire = await requests(request); expect(wire).toHaveLength(2); expect( wire.every( @@ -72,8 +143,16 @@ async function send(page: Page, text: string) { await page.getByRole('textbox', { name: 'Message', exact: true }).fill(text); await page.getByRole('button', { name: 'Send', exact: true }).click(); } +/** Agent requests, each with the final snapshot the real server sent for it. */ async function requests(request: APIRequestContext) { - return (await request.get('/__requests')).json(); + const wire = await (await request.get('/__requests')).json(); + const captures: { events: { type: string; messages?: unknown[] }[] }[] = await ( + await request.get('/__captures') + ).json(); + return wire.map((entry: object, index: number) => ({ + ...entry, + final: captures[index]?.events.findLast((event) => event.type === 'MESSAGES_SNAPSHOT')?.messages, + })); } async function nativeRows(page: Page) { return page @@ -158,7 +237,7 @@ test('real nonstreaming child providers retain final result fallback', async ({ page, request, }) => { - await request.post('/__mode/without-child-tokens'); + await arm(request, 'without-child-tokens'); await ready(page); await send(page, 'sequential'); await completed(page); @@ -172,7 +251,7 @@ for (const stage of ['child-text', 'child-finished', 'final']) page, request, }) => { - await request.post('/__hold/' + stage); + await hold(request, stage); await ready(page); await send(page, 'sequential'); await expect(page.locator('[data-child-id]').first()).toBeVisible(); @@ -211,7 +290,7 @@ for (const mode of [ 'orphan-text', ]) test(`${mode} requires New with no replay`, async ({ page, request }) => { - await request.post('/__mode/' + mode); + await arm(request, mode); await ready(page); await send(page, 'sequential'); await expect(page.getByRole('status')).toHaveText( @@ -244,7 +323,7 @@ for (const action of ['Stop', 'pagehide']) page, request, }) => { - await request.post('/__hold/child-text'); + await hold(request, 'child-text'); await ready(page); await send(page, 'sequential'); await expect(page.locator('[data-child-id]')).toHaveCount(1); @@ -275,7 +354,7 @@ for (const action of ['Stop', 'pagehide']) }); test('a truncated final snapshot cannot replace a previously confirmed canonical prefix', async ({ page, request }) => { await ready(page); await send(page, 'sequential'); await completed(page); - await request.post('/__mode/truncated-prefix'); await send(page, 'sequential'); + await arm(request, 'truncated-prefix'); await send(page, 'sequential'); await expect(page.getByRole('status')).toHaveText('Start a new conversation to continue.'); await expect(page.getByRole('button', { name: 'Send' })).toBeDisabled(); expect(await requests(request)).toHaveLength(2); diff --git a/cockpit/ag-ui/subagents/react/project.json b/cockpit/ag-ui/subagents/react/project.json index 14ce58dd3..5be6d5947 100644 --- a/cockpit/ag-ui/subagents/react/project.json +++ b/cockpit/ag-ui/subagents/react/project.json @@ -3,7 +3,15 @@ "$schema": "../../../../node_modules/nx/schemas/project-schema.json", "sourceRoot": "cockpit/ag-ui/subagents/react/src", "projectType": "application", - "tags": ["scope:cockpit-examples", "scope:cockpit-e2e", "framework:react"], + "implicitDependencies": [ + "cockpit-ag-ui-subagents-angular", + "cockpit-ag-ui-subagents-python" + ], + "tags": [ + "scope:cockpit-examples", + "scope:cockpit-e2e", + "framework:react" + ], "targets": { "test": { "executor": "@nx/vitest:test", @@ -15,26 +23,51 @@ "executor": "nx:run-commands", "cache": false, "dependsOn": [ - { "target": "build", "projects": ["core", "content", "react"] }, - { "target": "react-ag-ui", "projects": ["website-runtime-preparation"] } + { + "target": "build", + "projects": [ + "core", + "content", + "react" + ] + }, + { + "target": "react-ag-ui", + "projects": [ + "website-runtime-preparation" + ] + } + ], + "outputs": [ + "{workspaceRoot}/dist/cockpit/ag-ui/subagents/react" ], - "outputs": ["{workspaceRoot}/dist/cockpit/ag-ui/subagents/react"], "options": { "command": "node scripts/react-cockpit/build.mjs ag-ui-subagents" } }, - "lint": { "executor": "@nx/eslint:lint" }, + "lint": { + "executor": "@nx/eslint:lint" + }, "e2e": { "executor": "@nx/playwright:playwright", "cache": false, "dependsOn": [ "build", - { "target": "smoke", "projects": ["cockpit-ag-ui-subagents-python"] } + { + "target": "smoke", + "projects": [ + "cockpit-ag-ui-subagents-python" + ] + } ], "options": { "config": "cockpit/ag-ui/subagents/react/e2e/playwright.config.ts", "skipInstall": true - } + }, + "inputs": [ + "default", + "reactAimockBackend" + ] } } } diff --git a/cockpit/ag-ui/tool-views/angular/e2e/fixtures/tool-views.json b/cockpit/ag-ui/tool-views/angular/e2e/fixtures/tool-views.json index 2069a0412..7f0bebb4a 100644 --- a/cockpit/ag-ui/tool-views/angular/e2e/fixtures/tool-views.json +++ b/cockpit/ag-ui/tool-views/angular/e2e/fixtures/tool-views.json @@ -23,6 +23,532 @@ } ] } + }, + { + "match": { "userMessage": "Zero" }, + "response": { "content": "Reply to Zero" } + }, + { + "match": { + "userMessage": "One", + "hasToolResult": true + }, + "response": { "content": "Reply to One" } + }, + { + "match": { + "userMessage": "One", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "weather_card", + "arguments": { "location": "San Francisco" } + } + ] + } + }, + { + "match": { + "userMessage": "Hold args", + "hasToolResult": true + }, + "response": { "content": "Reply to Hold args" } + }, + { + "match": { + "userMessage": "Hold args", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "weather_card", + "arguments": { "location": "San Francisco" } + } + ] + } + }, + { + "match": { + "userMessage": "Hold call", + "hasToolResult": true + }, + "response": { "content": "Reply to Hold call" } + }, + { + "match": { + "userMessage": "Hold call", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "weather_card", + "arguments": { "location": "San Francisco" } + } + ] + } + }, + { + "match": { + "userMessage": "Hold result", + "hasToolResult": true + }, + "response": { "content": "Reply to Hold result" } + }, + { + "match": { + "userMessage": "Hold result", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "weather_card", + "arguments": { "location": "San Francisco" } + } + ] + } + }, + { + "match": { + "userMessage": "Hold final", + "hasToolResult": true + }, + "response": { "content": "Reply to Hold final" } + }, + { + "match": { + "userMessage": "Hold final", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "weather_card", + "arguments": { "location": "San Francisco" } + } + ] + } + }, + { + "match": { + "userMessage": "Unstable result identity", + "hasToolResult": true + }, + "response": { "content": "Reply to Unstable result identity" } + }, + { + "match": { + "userMessage": "Unstable result identity", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "weather_card", + "arguments": { "location": "San Francisco" } + } + ] + } + }, + { + "match": { + "userMessage": "HTTP error", + "hasToolResult": true + }, + "response": { "content": "Reply to HTTP error" } + }, + { + "match": { + "userMessage": "HTTP error", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "weather_card", + "arguments": { "location": "San Francisco" } + } + ] + } + }, + { + "match": { + "userMessage": "Run error", + "hasToolResult": true + }, + "response": { "content": "Reply to Run error" } + }, + { + "match": { + "userMessage": "Run error", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "weather_card", + "arguments": { "location": "San Francisco" } + } + ] + } + }, + { + "match": { + "userMessage": "Child", + "hasToolResult": true + }, + "response": { "content": "Reply to Child" } + }, + { + "match": { + "userMessage": "Child", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "weather_card", + "arguments": { "location": "San Francisco" } + } + ] + } + }, + { + "match": { + "userMessage": "Decision", + "hasToolResult": true + }, + "response": { "content": "Reply to Decision" } + }, + { + "match": { + "userMessage": "Decision", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "weather_card", + "arguments": { "location": "San Francisco" } + } + ] + } + }, + { + "match": { + "userMessage": "Pending tools", + "hasToolResult": true + }, + "response": { "content": "Reply to Pending tools" } + }, + { + "match": { + "userMessage": "Pending tools", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "weather_card", + "arguments": { "location": "San Francisco" } + } + ] + } + }, + { + "match": { + "userMessage": "Wrong terminal", + "hasToolResult": true + }, + "response": { "content": "Reply to Wrong terminal" } + }, + { + "match": { + "userMessage": "Wrong terminal", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "weather_card", + "arguments": { "location": "San Francisco" } + } + ] + } + }, + { + "match": { + "userMessage": "Truncated prefix", + "hasToolResult": true + }, + "response": { "content": "Reply to Truncated prefix" } + }, + { + "match": { + "userMessage": "Truncated prefix", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "weather_card", + "arguments": { "location": "San Francisco" } + } + ] + } + }, + { + "match": { + "userMessage": "Duplicate result", + "hasToolResult": true + }, + "response": { "content": "Reply to Duplicate result" } + }, + { + "match": { + "userMessage": "Duplicate result", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "weather_card", + "arguments": { "location": "San Francisco" } + } + ] + } + }, + { + "match": { + "userMessage": "Orphan result", + "hasToolResult": true + }, + "response": { "content": "Reply to Orphan result" } + }, + { + "match": { + "userMessage": "Orphan result", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "weather_card", + "arguments": { "location": "San Francisco" } + } + ] + } + }, + { + "match": { + "userMessage": "Malformed weather", + "hasToolResult": true + }, + "response": { "content": "Reply to Malformed weather" } + }, + { + "match": { + "userMessage": "Malformed weather", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "weather_card", + "arguments": { "location": "San Francisco" } + } + ] + } + }, + { + "match": { + "userMessage": "Legacy notice", + "hasToolResult": true + }, + "response": { "content": "Reply to Legacy notice" } + }, + { + "match": { + "userMessage": "Legacy notice", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "weather_card", + "arguments": { "location": "San Francisco" } + } + ] + } + }, + { + "match": { + "userMessage": "Notice EOF", + "hasToolResult": true + }, + "response": { "content": "Reply to Notice EOF" } + }, + { + "match": { + "userMessage": "Notice EOF", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "weather_card", + "arguments": { "location": "San Francisco" } + } + ] + } + }, + { + "match": { + "userMessage": "EOF", + "hasToolResult": true + }, + "response": { "content": "Reply to EOF" } + }, + { + "match": { + "userMessage": "EOF", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "weather_card", + "arguments": { "location": "San Francisco" } + } + ] + } + }, + { + "match": { + "userMessage": "Batch", + "hasToolResult": true + }, + "response": { "content": "Reply to Batch" } + }, + { + "match": { + "userMessage": "Batch", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "weather_card", + "arguments": { "location": "San Francisco" } + }, + { + "name": "weather_card", + "arguments": { "location": "Boston" } + } + ] + } + }, + { + "match": { "toolCallId": "rounds-second" }, + "response": { "content": "Reply to Rounds" } + }, + { + "match": { "toolCallId": "rounds-first" }, + "response": { + "toolCalls": [ + { + "name": "weather_card", + "arguments": { "location": "Boston" }, + "id": "rounds-second" + } + ] + } + }, + { + "match": { + "userMessage": "Rounds", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "weather_card", + "arguments": { "location": "San Francisco" }, + "id": "rounds-first" + } + ] + } + }, + { + "match": { + "userMessage": "Unknown tool", + "hasToolResult": true + }, + "response": { "content": "Reply to Unknown tool" } + }, + { + "match": { + "userMessage": "Unknown tool", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "unknown", + "arguments": { "location": "San Francisco" } + } + ] + } + }, + { + "match": { + "userMessage": "Malformed args", + "hasToolResult": true + }, + "response": { "content": "Reply to Malformed args" } + }, + { + "match": { + "userMessage": "Malformed args", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "weather_card", + "arguments": {} + } + ] + } + }, + { + "match": { + "userMessage": "Literal", + "hasToolResult": true + }, + "response": { "content": "" } + }, + { + "match": { + "userMessage": "Literal", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "weather_card", + "arguments": { "location": "" } + } + ] + } + }, + { + "match": { "userMessage": "" }, + "response": { "content": "Reply recorded." } } ] } diff --git a/cockpit/ag-ui/tool-views/react/e2e/global-setup-impl.ts b/cockpit/ag-ui/tool-views/react/e2e/global-setup-impl.ts new file mode 100644 index 000000000..c5e922c7b --- /dev/null +++ b/cockpit/ag-ui/tool-views/react/e2e/global-setup-impl.ts @@ -0,0 +1,6 @@ +// eslint-disable-next-line @nx/enforce-module-boundaries -- Shared aimock setup for the React cockpit examples. +import { reactAimockAgUiGlobalSetup } from '../../../../shared/react-aimock-setup'; + +// aimock replays the Angular example's fixtures into the example's real AG-UI +// server; the static server (webServer) proxies the agent endpoint to it. +export default reactAimockAgUiGlobalSetup({ example: 'ag-ui/tool-views', port: 4611 }); diff --git a/cockpit/ag-ui/tool-views/react/e2e/playwright.config.ts b/cockpit/ag-ui/tool-views/react/e2e/playwright.config.ts index 28d83addf..14f56d0b4 100644 --- a/cockpit/ag-ui/tool-views/react/e2e/playwright.config.ts +++ b/cockpit/ag-ui/tool-views/react/e2e/playwright.config.ts @@ -5,6 +5,9 @@ export default defineConfig({ fullyParallel: false, use: { baseURL: 'http://127.0.0.1:4611', trace: 'retain-on-failure' }, projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }], + // aimock + the real AG-UI server, shared with cockpit/ag-ui/tool-views/angular. + globalSetup: './global-setup-impl.ts', + globalTeardown: require.resolve('../../../../../libs/e2e-harness/src/global-teardown'), webServer: { command: 'node scripts/react-cockpit/serve.mjs ag-ui-tool-views', cwd: '../../../../..', diff --git a/cockpit/ag-ui/tool-views/react/e2e/tool-views.spec.ts b/cockpit/ag-ui/tool-views/react/e2e/tool-views.spec.ts index 126c8ac28..47bffb1b3 100644 --- a/cockpit/ag-ui/tool-views/react/e2e/tool-views.spec.ts +++ b/cockpit/ag-ui/tool-views/react/e2e/tool-views.spec.ts @@ -1,7 +1,67 @@ -import { test, expect, type Page } from '@playwright/test'; +import { test, expect, type Page, type APIRequestContext } from '@playwright/test'; +// Backend: aimock replays cockpit/ag-ui/tool-views/angular/e2e/fixtures into +// the example's real AG-UI server, whose graph calls the real weather tool. +// The recording proxy (scripts/react-cockpit/ag-ui-proxy.mjs) holds, fails or +// edits the next run. test.beforeEach(async ({ request }) => { await request.post('/__reset'); }); +const edit = (request: APIRequestContext, ops: object[]) => + request.post('/__stream/edit', { data: { ops } }); +/** Where each hold stops the real stream. */ +const holds: Record = { + 'Hold args': { afterType: 'TOOL_CALL_START' }, + 'Hold call': { afterType: 'TOOL_CALL_END' }, + 'Hold result': { afterType: 'TOOL_CALL_RESULT' }, + 'Hold final': { afterType: 'MESSAGES_SNAPSHOT' }, +}; +/** + * Arms a fault. `Unknown tool` and `Malformed args` are real (the model calls + * a tool that does not exist, or the weather tool without a location); the + * rest are injected protocol shapes a correct server won't produce. + */ +async function fault(request: APIRequestContext, mode: string) { + const finished = (set: object) => edit(request, [{ type: '^RUN_FINISHED$', op: 'patch', set }]); + const snapshot = (name: string) => edit(request, [{ type: '^MESSAGES_SNAPSHOT$', op: 'snapshot', mode: name }]); + if (mode === 'HTTP error') await request.post('/__fail-stream'); + if (mode === 'Run error') + await edit(request, [ + { type: '^TOOL_CALL_START$', op: 'insertBefore', event: { type: 'RUN_ERROR', message: 'PRIVATE model failure', code: 'TEST' } }, + { type: '^TOOL_CALL_START$', op: 'end' }, + ]); + if (mode === 'Child') + await edit(request, [ + { type: '^RUN_STARTED$', op: 'insertAfter', event: { type: 'SUBAGENT_STARTED', subagentRunId: 'child', name: 'worker' } }, + ]); + if (mode === 'Decision') await finished({ outcome: { type: 'interrupt', interrupts: [{ id: 'pause', reason: 'approval' }] } }); + if (mode === 'Pending tools') await finished({ outcome: { type: 'success', pendingToolCallIds: ['unobserved'] } }); + if (mode === 'Wrong terminal') await finished({ runId: 'different-run' }); + if (mode === 'Truncated prefix') await snapshot('drop-sent'); + if (mode === 'Duplicate result') await snapshot('duplicate-result'); + if (mode === 'Orphan result') await snapshot('orphan-result'); + // The backend saves every result under another identity from then on. + if (mode === 'Unstable result identity') + await request.post('/__stream/edit', { + data: { persist: true, ops: [{ type: '^MESSAGES_SNAPSHOT$', op: 'snapshot', mode: 'unstable-result' }] }, + }); + if (mode === 'Malformed weather') { + const content = JSON.stringify({ location: 'San Francisco', temperatureF: 'hot', conditions: 'Sunny', humidity: 55, windMph: 8 }); + await edit(request, [ + { type: '^TOOL_CALL_RESULT$', op: 'patch', set: { content } }, + { type: '^MESSAGES_SNAPSHOT$', op: 'snapshot', mode: 'result-content', content }, + ]); + } + if (mode === 'Legacy notice') + await edit(request, [ + { type: '^RUN_STARTED$', op: 'insertAfter', event: { type: 'CUSTOM', name: 'on_interrupt', value: { note: 'legacy pause' } } }, + ]); + if (mode === 'Notice EOF') + await edit(request, [ + { type: '^MESSAGES_SNAPSHOT$', op: 'insertBefore', event: { type: 'CUSTOM', name: 'unsupported', value: {} } }, + { type: '^MESSAGES_SNAPSHOT$', op: 'end' }, + ]); + if (mode === 'EOF') await edit(request, [{ type: '^RUN_FINISHED$', op: 'end' }]); +} const rows = (page: Page) => page.locator('[data-weather-message]'); async function send(page: Page, text: string) { await page.getByLabel('Message', { exact: true }).fill(text); @@ -88,6 +148,7 @@ for (const mode of ['Hold args', 'Hold call', 'Hold result', 'Hold final']) { request, }) => { await page.goto('/ag-ui/tool-views/react/'); + await request.post('/__hold/stream', { data: holds[mode] }); await send(page, mode); await expect .poll( @@ -121,6 +182,7 @@ for (const mode of ['Hold args', 'Hold call', 'Hold result', 'Hold final']) { request, }) => { await page.goto('/ag-ui/tool-views/react/'); + await request.post('/__hold/stream', { data: holds[mode] }); await send(page, mode); await expect .poll( @@ -170,6 +232,7 @@ for (const mode of [ await send(page, 'One'); await expect(page.getByRole('status', { name: 'Conversation status' })).toHaveText('Response complete.'); } + await fault(request, mode); await send(page, mode); await expect(page.getByRole('status', { name: 'Conversation status' })).toHaveText( 'Start a new conversation to continue.' @@ -194,6 +257,7 @@ test('a result saved under another identity keeps its card and its place in the // The stream names the result after its call; the final snapshot saves it // under another identifier, as graph backends do. await page.goto('/ag-ui/tool-views/react/'); + await fault(request, 'Unstable result identity'); await send(page, 'Unstable result identity'); await expect( page.getByRole('status', { name: 'Conversation status' }) diff --git a/cockpit/ag-ui/tool-views/react/project.json b/cockpit/ag-ui/tool-views/react/project.json index d06ebddd7..1bb700abd 100644 --- a/cockpit/ag-ui/tool-views/react/project.json +++ b/cockpit/ag-ui/tool-views/react/project.json @@ -3,7 +3,15 @@ "$schema": "../../../../node_modules/nx/schemas/project-schema.json", "sourceRoot": "cockpit/ag-ui/tool-views/react/src", "projectType": "application", - "tags": ["scope:cockpit-examples", "scope:cockpit-e2e", "framework:react"], + "implicitDependencies": [ + "cockpit-ag-ui-tool-views-angular", + "cockpit-ag-ui-tool-views-python" + ], + "tags": [ + "scope:cockpit-examples", + "scope:cockpit-e2e", + "framework:react" + ], "targets": { "test": { "executor": "@nx/vitest:test", @@ -17,11 +25,22 @@ "dependsOn": [ { "target": "build", - "projects": ["core", "content", "react"] + "projects": [ + "core", + "content", + "react" + ] }, - { "target": "react-ag-ui", "projects": ["website-runtime-preparation"] } + { + "target": "react-ag-ui", + "projects": [ + "website-runtime-preparation" + ] + } + ], + "outputs": [ + "{workspaceRoot}/dist/cockpit/ag-ui/tool-views/react" ], - "outputs": ["{workspaceRoot}/dist/cockpit/ag-ui/tool-views/react"], "options": { "command": "node scripts/react-cockpit/build.mjs ag-ui-tool-views" } @@ -32,11 +51,17 @@ "e2e": { "executor": "@nx/playwright:playwright", "cache": false, - "dependsOn": ["build"], + "dependsOn": [ + "build" + ], "options": { "config": "cockpit/ag-ui/tool-views/react/e2e/playwright.config.ts", "skipInstall": true - } + }, + "inputs": [ + "default", + "reactAimockBackend" + ] } } } diff --git a/cockpit/chat/a2ui/angular/e2e/fixtures/c-a2ui.json b/cockpit/chat/a2ui/angular/e2e/fixtures/c-a2ui.json index 84d191e97..0922c62f8 100644 --- a/cockpit/chat/a2ui/angular/e2e/fixtures/c-a2ui.json +++ b/cockpit/chat/a2ui/angular/e2e/fixtures/c-a2ui.json @@ -1,5 +1,63 @@ { "fixtures": [ + { + "match": { + "systemMessage": "In 3-5 words, summarize", + "userMessage": "[scenario:title-down]" + }, + "response": { + "error": { + "message": "Injected: title provider down", + "type": "server_error" + }, + "status": 500 + } + }, + { + "match": { + "systemMessage": "You are an aviation booking-form designer", + "userMessage": "[scenario:retry]", + "sequenceIndex": 0 + }, + "response": { + "toolCalls": [ + { + "name": "BookingFormSpec", + "arguments": "{\"surface_id\": \"booking\", \"data_model\": {\"origin\": [\"LAX\"], \"dest\": [\"JFK\"], \"date\": \"\", \"passengers\": 1, \"fare_class\": [\"Economy\"]}, \"components\": [{\"id\": \"root\", \"component\": \"Column\", \"children\": [\"card\"]}, {\"id\": \"card\", \"component\": \"Card\", \"child\": \"card_col\"}, {\"id\": \"card_col\", \"component\": \"Column\", \"children\": [\"title\", \"origin\", \"dest\", \"date\", \"passengers\", \"fare\", \"submit\"]}, {\"id\": \"title\", \"component\": \"Text\", \"text\": \"Book a flight\", \"variant\": \"h2\"}, {\"id\": \"origin\", \"component\": \"ChoicePicker\", \"label\": \"Origin\", \"options\": [{\"label\": \"LAX\", \"value\": \"LAX\"}, {\"label\": \"JFK\", \"value\": \"JFK\"}, {\"label\": \"SFO\", \"value\": \"SFO\"}, {\"label\": \"ORD\", \"value\": \"ORD\"}, {\"label\": \"BOS\", \"value\": \"BOS\"}, {\"label\": \"ATL\", \"value\": \"ATL\"}, {\"label\": \"DFW\", \"value\": \"DFW\"}, {\"label\": \"SEA\", \"value\": \"SEA\"}, {\"label\": \"MIA\", \"value\": \"MIA\"}, {\"label\": \"DEN\", \"value\": \"DEN\"}], \"value\": {\"path\": \"/origin\"}, \"variant\": \"mutuallyExclusive\"}, {\"id\": \"dest\", \"component\": \"ChoicePicker\", \"label\": \"Destination\", \"options\": [{\"label\": \"LAX\", \"value\": \"LAX\"}, {\"label\": \"JFK\", \"value\": \"JFK\"}, {\"label\": \"SFO\", \"value\": \"SFO\"}, {\"label\": \"ORD\", \"value\": \"ORD\"}, {\"label\": \"BOS\", \"value\": \"BOS\"}, {\"label\": \"ATL\", \"value\": \"ATL\"}, {\"label\": \"DFW\", \"value\": \"DFW\"}, {\"label\": \"SEA\", \"value\": \"SEA\"}, {\"label\": \"MIA\", \"value\": \"MIA\"}, {\"label\": \"DEN\", \"value\": \"DEN\"}], \"value\": {\"path\": \"/dest\"}, \"variant\": \"mutuallyExclusive\"}, {\"id\": \"date\", \"component\": \"DateTimeInput\", \"label\": \"Departure date (YYYY-MM-DD)\", \"value\": {\"path\": \"/date\"}, \"enableDate\": true}, {\"id\": \"passengers\", \"component\": \"TextField\", \"label\": \"Passengers\", \"value\": {\"path\": \"/passengers\"}, \"variant\": \"number\"}, {\"id\": \"fare\", \"component\": \"ChoicePicker\", \"label\": \"Fare class\", \"options\": [{\"label\": \"Economy\", \"value\": \"Economomy\"}, {\"label\": \"Premium\", \"value\": \"Premium\"}, {\"label\": \"Business\", \"value\": \"Business\"}, {\"label\": \"First\", \"value\": \"First\"}], \"value\": {\"path\": \"/fare_class\"}, \"variant\": \"mutuallyExclusive\"}, {\"id\": \"submit_label\", \"component\": \"Text\", \"text\": \"Search flights\"}, {\"id\": \"submit\", \"component\": \"Button\", \"child\": \"submit_label\", \"variant\": \"primary\", \"action\": {\"event\": {\"name\": \"bookingSubmit\", \"context\": {\"formId\": \"booking\", \"origin\": {\"path\": \"/origin\"}, \"dest\": {\"path\": \"/dest\"}, \"date\": {\"path\": \"/date\"}, \"passengers\": {\"path\": \"/passengers\"}, \"fare_class\": {\"path\": \"/fare_class\"}}}}}]}" + } + ] + } + }, + { + "match": { + "systemMessage": "You are an aviation booking-form designer", + "userMessage": "[scenario:retry]", + "sequenceIndex": 1 + }, + "response": { + "toolCalls": [ + { + "name": "BookingFormSpec", + "arguments": "{\"surface_id\":\"booking\",\"data_model\":{\"origin\":[\"LAX\"],\"dest\":[\"JFK\"],\"date\":\"\",\"passengers\":1,\"fare_class\":[\"Economy\"]},\"components\":[{\"id\":\"root\",\"component\":\"Column\",\"children\":[\"card\"]},{\"id\":\"card\",\"component\":\"Card\",\"child\":\"card_col\"},{\"id\":\"card_col\",\"component\":\"Column\",\"children\":[\"title\",\"origin\",\"dest\",\"date\",\"passengers\",\"fare\",\"submit\"]},{\"id\":\"title\",\"component\":\"Text\",\"text\":\"Book a flight\",\"variant\":\"h2\"},{\"id\":\"origin\",\"component\":\"ChoicePicker\",\"label\":\"Origin\",\"options\":[{\"label\":\"LAX\",\"value\":\"LAX\"},{\"label\":\"JFK\",\"value\":\"JFK\"},{\"label\":\"SFO\",\"value\":\"SFO\"},{\"label\":\"ORD\",\"value\":\"ORD\"},{\"label\":\"BOS\",\"value\":\"BOS\"},{\"label\":\"ATL\",\"value\":\"ATL\"},{\"label\":\"DFW\",\"value\":\"DFW\"},{\"label\":\"SEA\",\"value\":\"SEA\"},{\"label\":\"MIA\",\"value\":\"MIA\"},{\"label\":\"DEN\",\"value\":\"DEN\"}],\"value\":{\"path\":\"/origin\"},\"variant\":\"mutuallyExclusive\"},{\"id\":\"dest\",\"component\":\"ChoicePicker\",\"label\":\"Destination\",\"options\":[{\"label\":\"LAX\",\"value\":\"LAX\"},{\"label\":\"JFK\",\"value\":\"JFK\"},{\"label\":\"SFO\",\"value\":\"SFO\"},{\"label\":\"ORD\",\"value\":\"ORD\"},{\"label\":\"BOS\",\"value\":\"BOS\"},{\"label\":\"ATL\",\"value\":\"ATL\"},{\"label\":\"DFW\",\"value\":\"DFW\"},{\"label\":\"SEA\",\"value\":\"SEA\"},{\"label\":\"MIA\",\"value\":\"MIA\"},{\"label\":\"DEN\",\"value\":\"DEN\"}],\"value\":{\"path\":\"/dest\"},\"variant\":\"mutuallyExclusive\"},{\"id\":\"date\",\"component\":\"DateTimeInput\",\"label\":\"Departure date (YYYY-MM-DD)\",\"value\":{\"path\":\"/date\"},\"enableDate\":true},{\"id\":\"passengers\",\"component\":\"TextField\",\"label\":\"Passengers\",\"value\":{\"path\":\"/passengers\"},\"variant\":\"number\"},{\"id\":\"fare\",\"component\":\"ChoicePicker\",\"label\":\"Fare class\",\"options\":[{\"label\":\"Economy\",\"value\":\"Economy\"},{\"label\":\"Premium\",\"value\":\"Premium\"},{\"label\":\"Business\",\"value\":\"Business\"},{\"label\":\"First\",\"value\":\"First\"}],\"value\":{\"path\":\"/fare_class\"},\"variant\":\"mutuallyExclusive\"},{\"id\":\"submit_label\",\"component\":\"Text\",\"text\":\"Search flights\"},{\"id\":\"submit\",\"component\":\"Button\",\"child\":\"submit_label\",\"variant\":\"primary\",\"action\":{\"event\":{\"name\":\"bookingSubmit\",\"context\":{\"formId\":\"booking\",\"origin\":{\"path\":\"/origin\"},\"dest\":{\"path\":\"/dest\"},\"date\":{\"path\":\"/date\"},\"passengers\":{\"path\":\"/passengers\"},\"fare_class\":{\"path\":\"/fare_class\"}}}}}]}", + "id": "call_DiKu9KXnOKtbRcNBtDlXwTzx" + } + ] + } + }, + { + "match": { + "systemMessage": "You are an aviation booking-form designer", + "userMessage": "[scenario:invalid]" + }, + "response": { + "toolCalls": [ + { + "name": "BookingFormSpec", + "arguments": "{\"surface_id\": \"booking\", \"data_model\": {\"origin\": [\"LAX\"], \"dest\": [\"JFK\"], \"date\": \"\", \"passengers\": 1, \"fare_class\": [\"Economy\"]}, \"components\": [{\"id\": \"root\", \"component\": \"Column\", \"children\": [\"card\"]}, {\"id\": \"card\", \"component\": \"Card\", \"child\": \"card_col\"}, {\"id\": \"card_col\", \"component\": \"Column\", \"children\": [\"title\", \"origin\", \"dest\", \"date\", \"passengers\", \"fare\", \"submit\"]}, {\"id\": \"title\", \"component\": \"Text\", \"text\": \"Book a flight\", \"variant\": \"h2\"}, {\"id\": \"origin\", \"component\": \"ChoicePicker\", \"label\": \"Origin\", \"options\": [{\"label\": \"LAX\", \"value\": \"LAX\"}, {\"label\": \"JFK\", \"value\": \"JFK\"}, {\"label\": \"SFO\", \"value\": \"SFO\"}, {\"label\": \"ORD\", \"value\": \"ORD\"}, {\"label\": \"BOS\", \"value\": \"BOS\"}, {\"label\": \"ATL\", \"value\": \"ATL\"}, {\"label\": \"DFW\", \"value\": \"DFW\"}, {\"label\": \"SEA\", \"value\": \"SEA\"}, {\"label\": \"MIA\", \"value\": \"MIA\"}, {\"label\": \"DEN\", \"value\": \"DEN\"}], \"value\": {\"path\": \"/origin\"}, \"variant\": \"mutuallyExclusive\"}, {\"id\": \"dest\", \"component\": \"ChoicePicker\", \"label\": \"Destination\", \"options\": [{\"label\": \"LAX\", \"value\": \"LAX\"}, {\"label\": \"JFK\", \"value\": \"JFK\"}, {\"label\": \"SFO\", \"value\": \"SFO\"}, {\"label\": \"ORD\", \"value\": \"ORD\"}, {\"label\": \"BOS\", \"value\": \"BOS\"}, {\"label\": \"ATL\", \"value\": \"ATL\"}, {\"label\": \"DFW\", \"value\": \"DFW\"}, {\"label\": \"SEA\", \"value\": \"SEA\"}, {\"label\": \"MIA\", \"value\": \"MIA\"}, {\"label\": \"DEN\", \"value\": \"DEN\"}], \"value\": {\"path\": \"/dest\"}, \"variant\": \"mutuallyExclusive\"}, {\"id\": \"date\", \"component\": \"DateTimeInput\", \"label\": \"Departure date (YYYY-MM-DD)\", \"value\": {\"path\": \"/date\"}, \"enableDate\": true}, {\"id\": \"passengers\", \"component\": \"TextField\", \"label\": \"Passengers\", \"value\": {\"path\": \"/passengers\"}, \"variant\": \"number\"}, {\"id\": \"fare\", \"component\": \"ChoicePicker\", \"label\": \"Fare class\", \"options\": [{\"label\": \"Economy\", \"value\": \"Economomy\"}, {\"label\": \"Premium\", \"value\": \"Premium\"}, {\"label\": \"Business\", \"value\": \"Business\"}, {\"label\": \"First\", \"value\": \"First\"}], \"value\": {\"path\": \"/fare_class\"}, \"variant\": \"mutuallyExclusive\"}, {\"id\": \"submit_label\", \"component\": \"Text\", \"text\": \"Search flights\"}, {\"id\": \"submit\", \"component\": \"Button\", \"child\": \"submit_label\", \"variant\": \"primary\", \"action\": {\"event\": {\"name\": \"bookingSubmit\", \"context\": {\"formId\": \"booking\", \"origin\": {\"path\": \"/origin\"}, \"dest\": {\"path\": \"/dest\"}, \"date\": {\"path\": \"/date\"}, \"passengers\": {\"path\": \"/passengers\"}, \"fare_class\": {\"path\": \"/fare_class\"}}}}}]}" + } + ] + } + }, { "match": { "userMessage": "I want to fly LAX to JFK" @@ -27,6 +85,118 @@ } ] } + }, + { + "match": { + "systemMessage": "In 3-5 words, summarize" + }, + "response": { + "content": "Local booking title" + } + }, + { + "match": { + "systemMessage": "You just received a booking submission. The find_routes() tool returned the following flights:\n\n[\n {\n \"flight_number\": \"UA204\",\n \"airline\": \"UA\",\n \"from\": \"SFO\",\n \"to\": \"SEA\",\n \"depart_local\": \"09:00\",\n \"arrive_local\": \"11:00\",\n \"duration_min\": 120,\n \"status\": \"on_time\",\n \"gate\": \"F11\",\n \"aircraft\": \"Boeing 737\"\n }\n]\n\nForm data (for context): {\n \"formId\": \"booking\",\n \"origin\": \"SFO\",\n \"dest\": \"SEA\",\n \"date\": \"2026-11-03\",\n \"passengers\": 3,\n \"fare_class\": \"Business\"\n}\n\nEmit an A2UI v0.9 results surface using the FlightResultsSpec schema.\n\nA2UI format (CRITICAL): every component is FLAT \u2014 `{\"id\": \"...\", \"component\": \"\", }`. The component name is a plain string; props sit at the top level of the same object. Dynamic values are bare literals or `{\"path\": \"/ptr\"}`.\n\nAllowed component names: Column, Row, Card, Text, TextField, Button, Divider, List.\n\nPer-component shapes you'll need:\n Column / List: {\"children\": [\"id1\", \"id2\"]}\n Card: {\"child\": \"\"}\n Text: {\"text\": \"literal\", \"variant\": \"h2\"} (or h1/h3/body/caption)\n Button: {\"child\": \"\", \"variant\": \"primary\", \"action\": {\"event\": {\"name\": \"\", \"context\": {\"flightId\": \"\"}}}}\n Divider: {}\n\nSurface constraints:\n surface_id MUST be \"results\"\n data_model can be {}\n Exactly ONE component MUST have id \"root\": a Column whose children lists every flight Card id (or just [\"no_flights\"] when empty)\n\nBuild pattern (one per flight):\n card_ (Card, child=col_)\n col_ (Column, children = [title_, route_, time_, btn_])\n title_ (Text, text=\" flight \", variant=\"h3\")\n route_ (Text, text=\" \u2192 \u2022 min \u2022 \", variant=\"body\")\n time_ (Text, text=\"Depart \u2022 Arrive \u2022 Gate \", variant=\"caption\")\n btn_ (Button, child=btn_label_, variant=\"primary\",\n action={\"event\":{\"name\":\"flightSelect\",\"context\":{\"flightId\":\"\"}}})\n btn_label_ (Text, text=\"Select\")\n\nEmpty case: components = [\n {\"id\":\"root\", \"component\":\"Column\", \"children\":[\"no_flights\"]},\n {\"id\":\"no_flights\", \"component\":\"Card\", \"child\":\"empty_col\"},\n {\"id\":\"empty_col\", \"component\":\"Column\", \"children\":[\"empty_msg\",\"modify_btn\"]},\n {\"id\":\"empty_msg\", \"component\":\"Text\", \"text\":\"No flights found\", \"variant\":\"h3\"},\n {\"id\":\"modify_btn\", \"component\":\"Button\", \"child\":\"modify_label\", \"action\":{\"event\":{\"name\":\"modifySearch\",\"context\":{\"formId\":\"booking\"}}}},\n {\"id\":\"modify_label\", \"component\":\"Text\", \"text\":\"Modify search\"}\n]\n\nUse unique ids for every component." + }, + "response": { + "toolCalls": [ + { + "name": "FlightResultsSpec", + "arguments": "{\"surface_id\": \"results\", \"data_model\": {}, \"components\": [{\"id\": \"root\", \"component\": \"Column\", \"children\": [\"card_0\"]}, {\"id\": \"card_0\", \"component\": \"Card\", \"child\": \"col_0\"}, {\"id\": \"col_0\", \"component\": \"Column\", \"children\": [\"title_0\", \"route_0\", \"time_0\", \"btn_0\"]}, {\"id\": \"title_0\", \"component\": \"Text\", \"text\": \"UA flight UA204\", \"variant\": \"h3\"}, {\"id\": \"route_0\", \"component\": \"Text\", \"text\": \"SFO \u2192 SEA \u2022 120 min \u2022 Boeing 737\", \"variant\": \"body\"}, {\"id\": \"time_0\", \"component\": \"Text\", \"text\": \"Depart 09:00 \u2022 Arrive 11:00 \u2022 Gate F11\", \"variant\": \"caption\"}, {\"id\": \"btn_0\", \"component\": \"Button\", \"child\": \"btn_label_0\", \"variant\": \"primary\", \"action\": {\"event\": {\"name\": \"flightSelect\", \"context\": {\"flightId\": \"UA204\"}}}}, {\"id\": \"btn_label_0\", \"component\": \"Text\", \"text\": \"Select\"}]}" + } + ] + } + }, + { + "match": { + "systemMessage": "You just received a Select event from the flight results surface. The user picked a flight; here are its details from lookup_flight():\n\n{\n \"flight_number\": \"UA204\",\n \"airline\": \"UA\",\n \"from\": \"SFO\",\n \"to\": \"SEA\",\n \"depart_local\": \"09:00\",\n \"arrive_local\": \"11:00\",\n \"duration_min\": 120,\n \"status\": \"on_time\",\n \"gate\": \"F11\",\n \"aircraft\": \"Boeing 737\"\n}\n\nThe user's prior search context (party_text, derived from passengers + fare_class on the original booking submission): 3 passengers \u2022 Business\n\nThe selected flight number (from the Select button's action context, used in the fallback title if flight_json is null): UA204\n\nEmit an A2UI v0.9 confirmation surface using the ConfirmationSpec schema.\n\nA2UI format (CRITICAL): every component is FLAT \u2014 `{\"id\": \"...\", \"component\": \"\", }`. The component name is a plain string; props sit at the top level of the same object. Dynamic values are bare literals or `{\"path\": \"/ptr\"}`. Exactly ONE component MUST have id \"root\".\n\nAllowed component names: Column, Card, Text, Button, Divider.\n\nPer-component shapes you'll need:\n Column: {\"children\": [\"id1\",\"id2\"]}\n Card: {\"child\": \"\"}\n Text: {\"text\": \"literal\", \"variant\": \"h2\"} (h1/h2/h3/body/caption)\n Button: {\"child\": \"\", \"variant\": \"primary\", \"action\": {\"event\": {\"name\": \"\", \"context\": {\"formId\": \"booking\"}}}}\n Divider: {}\n\nSurface constraints:\n surface_id MUST be \"confirmation\"\n data_model = {}\n\nBuild this component tree exactly (use these ids):\n root (Column, children = [card])\n card (Card, child = card_col)\n card_col (Column, children = [title, route_text, time_text, gate_text, divider, party_text, modify])\n title (Text, \" flight \", variant=\"h2\")\n route_text (Text, \" \u2192 \u2022 min \u2022 \", variant=\"body\")\n time_text (Text, \"Depart \u2022 Arrive \", variant=\"caption\")\n gate_text (Text, \"Gate \", variant=\"caption\")\n divider (Divider, {})\n party_text (Text, \"3 passengers \u2022 Business\", variant=\"body\") \u2190 use the supplied string verbatim\n modify (Button, child=modify_label, variant=\"primary\",\n action={\"event\":{\"name\":\"modifySearch\",\"context\":{\"formId\":\"booking\"}}})\n modify_label (Text, \"Modify search\")\n\nIf flight_json is null, omit route_text, time_text, gate_text from card_col's children and set the title to \"Flight UA204 selected\" (or \"Booking selected\" if UA204 is empty). Always include party_text + modify." + }, + "response": { + "toolCalls": [ + { + "name": "ConfirmationSpec", + "arguments": "{\"surface_id\": \"confirmation\", \"data_model\": {}, \"components\": [{\"id\": \"root\", \"component\": \"Column\", \"children\": [\"card\"]}, {\"id\": \"card\", \"component\": \"Card\", \"child\": \"card_col\"}, {\"id\": \"card_col\", \"component\": \"Column\", \"children\": [\"title\", \"party_text\", \"modify\"]}, {\"id\": \"title\", \"component\": \"Text\", \"text\": \"UA flight UA204\", \"variant\": \"h2\"}, {\"id\": \"party_text\", \"component\": \"Text\", \"text\": \"3 passengers \u2022 Business\", \"variant\": \"body\"}, {\"id\": \"modify\", \"component\": \"Button\", \"child\": \"modify_label\", \"variant\": \"primary\", \"action\": {\"event\": {\"name\": \"modifySearch\", \"context\": {\"formId\": \"booking\"}}}}, {\"id\": \"modify_label\", \"component\": \"Text\", \"text\": \"Modify search\"}]}" + } + ] + } + }, + { + "match": { + "systemMessage": "You are an aviation booking-form designer. Emit an A2UI v0.9 booking form using the structured output schema.\n\nA2UI FORMAT (CRITICAL): each component is FLAT \u2014 `{\"id\": \"...\", \"component\": \"\", }`. The component name is a plain string and the props sit at the top level of the same object. ComponentName must be one of:\n Column, Row, Card, Text, TextField, ChoicePicker, DateTimeInput, CheckBox, Button, Divider, List, Image, Icon, Modal, Slider, Tabs\n\nDynamic values are bare literals (\"LAX\", 3, true) or data bindings `{\"path\": \"/json/pointer\"}`. Never wrap literals.\n\nPer-component shapes:\n Column / Row / List: {\"children\": [\"id1\", \"id2\"]}\n Card: {\"child\": \"\"} \u2190 single child only\n Button: {\"child\": \"\", \"variant\": \"primary\", \"action\": {\"event\": {\"name\": \"\", \"context\": {\"formId\": \"booking\"}}}}\n Text: {\"text\": \"literal string\", \"variant\": \"h2\"} (h1/h2/h3/h4/h5/caption/body)\n TextField: {\"label\": \"Field\", \"value\": {\"path\": \"/p\"}, \"variant\": \"shortText\"} (shortText/longText/number/obscured)\n DateTimeInput: {\"label\": \"Date\", \"value\": {\"path\": \"/p\"}, \"enableDate\": true}\n ChoicePicker: {\"label\": \"Origin\", \"options\": [{\"label\":\"LAX\",\"value\":\"LAX\"}], \"value\": {\"path\":\"/origin\"}, \"variant\": \"mutuallyExclusive\"}\n CheckBox: {\"label\": \"...\", \"value\": {\"path\":\"/p\"}}\n Divider: {}\n\nExactly ONE component MUST have id \"root\" \u2014 it is the root of the tree.\n\nRequired form composition for THIS task:\n surface_id MUST be \"booking\"\n data_model MUST be {\"origin\": [\"SFO\"], \"dest\": [\"SEA\"], \"date\": \"2026-11-03\", \"passengers\": 3, \"fare_class\": [\"Business\"]} \u2190 use these values verbatim as the field defaults\n\n Build this component tree:\n root (Column, children=[\"card\"])\n card (Card, child=card_col)\n card_col (Column, children=[title, origin, dest, date, passengers, fare, submit])\n title (Text, text=\"Book a flight\", variant=\"h2\")\n origin (ChoicePicker, label=\"Origin\", options=[{'label': 'LAX', 'value': 'LAX'}, {'label': 'JFK', 'value': 'JFK'}, {'label': 'SFO', 'value': 'SFO'}, {'label': 'ORD', 'value': 'ORD'}, {'label': 'BOS', 'value': 'BOS'}, {'label': 'ATL', 'value': 'ATL'}, {'label': 'DFW', 'value': 'DFW'}, {'label': 'SEA', 'value': 'SEA'}, {'label': 'MIA', 'value': 'MIA'}, {'label': 'DEN', 'value': 'DEN'}], value={\"path\":\"/origin\"}, variant=\"mutuallyExclusive\")\n dest (ChoicePicker, label=\"Destination\", options=[{'label': 'LAX', 'value': 'LAX'}, {'label': 'JFK', 'value': 'JFK'}, {'label': 'SFO', 'value': 'SFO'}, {'label': 'ORD', 'value': 'ORD'}, {'label': 'BOS', 'value': 'BOS'}, {'label': 'ATL', 'value': 'ATL'}, {'label': 'DFW', 'value': 'DFW'}, {'label': 'SEA', 'value': 'SEA'}, {'label': 'MIA', 'value': 'MIA'}, {'label': 'DEN', 'value': 'DEN'}], value={\"path\":\"/dest\"}, variant=\"mutuallyExclusive\")\n date (DateTimeInput, label=\"Departure date (YYYY-MM-DD)\", value={\"path\":\"/date\"}, enableDate=true)\n passengers (TextField, label=\"Passengers\", value={\"path\":\"/passengers\"}, variant=\"number\")\n fare (ChoicePicker, label=\"Fare class\", options=[{'label': 'Economy', 'value': 'Economy'}, {'label': 'Premium', 'value': 'Premium'}, {'label': 'Business', 'value': 'Business'}, {'label': 'First', 'value': 'First'}], value={\"path\":\"/fare_class\"}, variant=\"mutuallyExclusive\")\n submit (Button, child=submit_label, variant=\"primary\", action={\"event\":{\"name\":\"bookingSubmit\",\"context\":{\n \"formId\":\"booking\",\n \"origin\":{\"path\":\"/origin\"},\n \"dest\":{\"path\":\"/dest\"},\n \"date\":{\"path\":\"/date\"},\n \"passengers\":{\"path\":\"/passengers\"},\n \"fare_class\":{\"path\":\"/fare_class\"}\n }}})\n submit_label (Text, text=\"Search flights\")\n\nUse these exact ids." + }, + "response": { + "toolCalls": [ + { + "name": "BookingFormSpec", + "arguments": "{\"surface_id\": \"booking\", \"data_model\": {\"origin\": [\"SFO\"], \"dest\": [\"SEA\"], \"date\": \"2026-11-03\", \"passengers\": 3, \"fare_class\": [\"Business\"]}, \"components\": [{\"id\": \"root\", \"component\": \"Column\", \"children\": [\"card\"]}, {\"id\": \"card\", \"component\": \"Card\", \"child\": \"card_col\"}, {\"id\": \"card_col\", \"component\": \"Column\", \"children\": [\"title\", \"origin\", \"dest\", \"date\", \"passengers\", \"fare\", \"submit\"]}, {\"id\": \"title\", \"component\": \"Text\", \"text\": \"Book a flight\", \"variant\": \"h2\"}, {\"id\": \"origin\", \"component\": \"ChoicePicker\", \"label\": \"Origin\", \"options\": [{\"label\": \"LAX\", \"value\": \"LAX\"}, {\"label\": \"JFK\", \"value\": \"JFK\"}, {\"label\": \"SFO\", \"value\": \"SFO\"}, {\"label\": \"ORD\", \"value\": \"ORD\"}, {\"label\": \"BOS\", \"value\": \"BOS\"}, {\"label\": \"ATL\", \"value\": \"ATL\"}, {\"label\": \"DFW\", \"value\": \"DFW\"}, {\"label\": \"SEA\", \"value\": \"SEA\"}, {\"label\": \"MIA\", \"value\": \"MIA\"}, {\"label\": \"DEN\", \"value\": \"DEN\"}], \"value\": {\"path\": \"/origin\"}, \"variant\": \"mutuallyExclusive\"}, {\"id\": \"dest\", \"component\": \"ChoicePicker\", \"label\": \"Destination\", \"options\": [{\"label\": \"LAX\", \"value\": \"LAX\"}, {\"label\": \"JFK\", \"value\": \"JFK\"}, {\"label\": \"SFO\", \"value\": \"SFO\"}, {\"label\": \"ORD\", \"value\": \"ORD\"}, {\"label\": \"BOS\", \"value\": \"BOS\"}, {\"label\": \"ATL\", \"value\": \"ATL\"}, {\"label\": \"DFW\", \"value\": \"DFW\"}, {\"label\": \"SEA\", \"value\": \"SEA\"}, {\"label\": \"MIA\", \"value\": \"MIA\"}, {\"label\": \"DEN\", \"value\": \"DEN\"}], \"value\": {\"path\": \"/dest\"}, \"variant\": \"mutuallyExclusive\"}, {\"id\": \"fare\", \"component\": \"ChoicePicker\", \"label\": \"Fare class\", \"options\": [{\"label\": \"Economy\", \"value\": \"Economy\"}, {\"label\": \"Premium\", \"value\": \"Premium\"}, {\"label\": \"Business\", \"value\": \"Business\"}, {\"label\": \"First\", \"value\": \"First\"}], \"value\": {\"path\": \"/fare_class\"}, \"variant\": \"mutuallyExclusive\"}, {\"id\": \"date\", \"component\": \"DateTimeInput\", \"label\": \"Departure date (YYYY-MM-DD)\", \"value\": {\"path\": \"/date\"}, \"enableDate\": true}, {\"id\": \"passengers\", \"component\": \"TextField\", \"label\": \"Passengers\", \"value\": {\"path\": \"/passengers\"}, \"variant\": \"number\"}, {\"id\": \"submit\", \"component\": \"Button\", \"child\": \"submit_label\", \"variant\": \"primary\", \"action\": {\"event\": {\"name\": \"bookingSubmit\", \"context\": {\"formId\": \"booking\", \"origin\": {\"path\": \"/origin\"}, \"dest\": {\"path\": \"/dest\"}, \"date\": {\"path\": \"/date\"}, \"passengers\": {\"path\": \"/passengers\"}, \"fare_class\": {\"path\": \"/fare_class\"}}}}}, {\"id\": \"submit_label\", \"component\": \"Text\", \"text\": \"Search flights\"}]}" + } + ] + } + }, + { + "match": { + "systemMessage": "You just received a booking submission. The find_routes() tool returned the following flights:\n\n[\n {\n \"flight_number\": \"UA123\",\n \"airline\": \"UA\",\n \"from\": \"LAX\",\n \"to\": \"JFK\",\n \"depart_local\": \"08:00\",\n \"arrive_local\": \"16:30\",\n \"duration_min\": 330,\n \"status\": \"on_time\",\n \"gate\": \"B14\",\n \"aircraft\": \"Boeing 787\"\n }\n]\n\nForm data (for context): {\n \"formId\": \"booking\",\n \"origin\": \"LAX\",\n \"dest\": \"JFK\",\n \"date\": \"2026-11-03\",\n \"passengers\": 2,\n \"fare_class\": \"Business\"\n}\n\nEmit an A2UI v0.9 results surface using the FlightResultsSpec schema.\n\nA2UI format (CRITICAL): every component is FLAT \u2014 `{\"id\": \"...\", \"component\": \"\", }`. The component name is a plain string; props sit at the top level of the same object. Dynamic values are bare literals or `{\"path\": \"/ptr\"}`.\n\nAllowed component names: Column, Row, Card, Text, TextField, Button, Divider, List.\n\nPer-component shapes you'll need:\n Column / List: {\"children\": [\"id1\", \"id2\"]}\n Card: {\"child\": \"\"}\n Text: {\"text\": \"literal\", \"variant\": \"h2\"} (or h1/h3/body/caption)\n Button: {\"child\": \"\", \"variant\": \"primary\", \"action\": {\"event\": {\"name\": \"\", \"context\": {\"flightId\": \"\"}}}}\n Divider: {}\n\nSurface constraints:\n surface_id MUST be \"results\"\n data_model can be {}\n Exactly ONE component MUST have id \"root\": a Column whose children lists every flight Card id (or just [\"no_flights\"] when empty)\n\nBuild pattern (one per flight):\n card_ (Card, child=col_)\n col_ (Column, children = [title_, route_, time_, btn_])\n title_ (Text, text=\" flight \", variant=\"h3\")\n route_ (Text, text=\" \u2192 \u2022 min \u2022 \", variant=\"body\")\n time_ (Text, text=\"Depart \u2022 Arrive \u2022 Gate \", variant=\"caption\")\n btn_ (Button, child=btn_label_, variant=\"primary\",\n action={\"event\":{\"name\":\"flightSelect\",\"context\":{\"flightId\":\"\"}}})\n btn_label_ (Text, text=\"Select\")\n\nEmpty case: components = [\n {\"id\":\"root\", \"component\":\"Column\", \"children\":[\"no_flights\"]},\n {\"id\":\"no_flights\", \"component\":\"Card\", \"child\":\"empty_col\"},\n {\"id\":\"empty_col\", \"component\":\"Column\", \"children\":[\"empty_msg\",\"modify_btn\"]},\n {\"id\":\"empty_msg\", \"component\":\"Text\", \"text\":\"No flights found\", \"variant\":\"h3\"},\n {\"id\":\"modify_btn\", \"component\":\"Button\", \"child\":\"modify_label\", \"action\":{\"event\":{\"name\":\"modifySearch\",\"context\":{\"formId\":\"booking\"}}}},\n {\"id\":\"modify_label\", \"component\":\"Text\", \"text\":\"Modify search\"}\n]\n\nUse unique ids for every component." + }, + "response": { + "toolCalls": [ + { + "name": "FlightResultsSpec", + "arguments": "{\"surface_id\": \"results\", \"data_model\": {}, \"components\": [{\"id\": \"root\", \"component\": \"Column\", \"children\": [\"card_0\"]}, {\"id\": \"card_0\", \"component\": \"Card\", \"child\": \"col_0\"}, {\"id\": \"col_0\", \"component\": \"Column\", \"children\": [\"title_0\", \"route_0\", \"time_0\", \"btn_0\"]}, {\"id\": \"title_0\", \"component\": \"Text\", \"text\": \"UA flight UA123\", \"variant\": \"h3\"}, {\"id\": \"route_0\", \"component\": \"Text\", \"text\": \"LAX \u2192 JFK \u2022 330 min \u2022 Boeing 787\", \"variant\": \"body\"}, {\"id\": \"time_0\", \"component\": \"Text\", \"text\": \"Depart 08:00 \u2022 Arrive 16:30 \u2022 Gate B14\", \"variant\": \"caption\"}, {\"id\": \"btn_0\", \"component\": \"Button\", \"child\": \"btn_label_0\", \"variant\": \"primary\", \"action\": {\"event\": {\"name\": \"flightSelect\", \"context\": {\"flightId\": \"UA123\"}}}}, {\"id\": \"btn_label_0\", \"component\": \"Text\", \"text\": \"Select\"}]}" + } + ] + } + }, + { + "match": { + "systemMessage": "You just received a Select event from the flight results surface. The user picked a flight; here are its details from lookup_flight():\n\n{\n \"flight_number\": \"UA123\",\n \"airline\": \"UA\",\n \"from\": \"LAX\",\n \"to\": \"JFK\",\n \"depart_local\": \"08:00\",\n \"arrive_local\": \"16:30\",\n \"duration_min\": 330,\n \"status\": \"on_time\",\n \"gate\": \"B14\",\n \"aircraft\": \"Boeing 787\"\n}\n\nThe user's prior search context (party_text, derived from passengers + fare_class on the original booking submission): 2 passengers \u2022 Business\n\nThe selected flight number (from the Select button's action context, used in the fallback title if flight_json is null): UA123\n\nEmit an A2UI v0.9 confirmation surface using the ConfirmationSpec schema.\n\nA2UI format (CRITICAL): every component is FLAT \u2014 `{\"id\": \"...\", \"component\": \"\", }`. The component name is a plain string; props sit at the top level of the same object. Dynamic values are bare literals or `{\"path\": \"/ptr\"}`. Exactly ONE component MUST have id \"root\".\n\nAllowed component names: Column, Card, Text, Button, Divider.\n\nPer-component shapes you'll need:\n Column: {\"children\": [\"id1\",\"id2\"]}\n Card: {\"child\": \"\"}\n Text: {\"text\": \"literal\", \"variant\": \"h2\"} (h1/h2/h3/body/caption)\n Button: {\"child\": \"\", \"variant\": \"primary\", \"action\": {\"event\": {\"name\": \"\", \"context\": {\"formId\": \"booking\"}}}}\n Divider: {}\n\nSurface constraints:\n surface_id MUST be \"confirmation\"\n data_model = {}\n\nBuild this component tree exactly (use these ids):\n root (Column, children = [card])\n card (Card, child = card_col)\n card_col (Column, children = [title, route_text, time_text, gate_text, divider, party_text, modify])\n title (Text, \" flight \", variant=\"h2\")\n route_text (Text, \" \u2192 \u2022 min \u2022 \", variant=\"body\")\n time_text (Text, \"Depart \u2022 Arrive \", variant=\"caption\")\n gate_text (Text, \"Gate \", variant=\"caption\")\n divider (Divider, {})\n party_text (Text, \"2 passengers \u2022 Business\", variant=\"body\") \u2190 use the supplied string verbatim\n modify (Button, child=modify_label, variant=\"primary\",\n action={\"event\":{\"name\":\"modifySearch\",\"context\":{\"formId\":\"booking\"}}})\n modify_label (Text, \"Modify search\")\n\nIf flight_json is null, omit route_text, time_text, gate_text from card_col's children and set the title to \"Flight UA123 selected\" (or \"Booking selected\" if UA123 is empty). Always include party_text + modify." + }, + "response": { + "toolCalls": [ + { + "name": "ConfirmationSpec", + "arguments": "{\"surface_id\": \"confirmation\", \"data_model\": {}, \"components\": [{\"id\": \"root\", \"component\": \"Column\", \"children\": [\"card\"]}, {\"id\": \"card\", \"component\": \"Card\", \"child\": \"card_col\"}, {\"id\": \"card_col\", \"component\": \"Column\", \"children\": [\"title\", \"party_text\", \"modify\"]}, {\"id\": \"title\", \"component\": \"Text\", \"text\": \"UA flight UA123\", \"variant\": \"h2\"}, {\"id\": \"party_text\", \"component\": \"Text\", \"text\": \"2 passengers \u2022 Business\", \"variant\": \"body\"}, {\"id\": \"modify\", \"component\": \"Button\", \"child\": \"modify_label\", \"variant\": \"primary\", \"action\": {\"event\": {\"name\": \"modifySearch\", \"context\": {\"formId\": \"booking\"}}}}, {\"id\": \"modify_label\", \"component\": \"Text\", \"text\": \"Modify search\"}]}" + } + ] + } + }, + { + "match": { + "systemMessage": "You are an aviation booking-form designer. Emit an A2UI v0.9 booking form using the structured output schema.\n\nA2UI FORMAT (CRITICAL): each component is FLAT \u2014 `{\"id\": \"...\", \"component\": \"\", }`. The component name is a plain string and the props sit at the top level of the same object. ComponentName must be one of:\n Column, Row, Card, Text, TextField, ChoicePicker, DateTimeInput, CheckBox, Button, Divider, List, Image, Icon, Modal, Slider, Tabs\n\nDynamic values are bare literals (\"LAX\", 3, true) or data bindings `{\"path\": \"/json/pointer\"}`. Never wrap literals.\n\nPer-component shapes:\n Column / Row / List: {\"children\": [\"id1\", \"id2\"]}\n Card: {\"child\": \"\"} \u2190 single child only\n Button: {\"child\": \"\", \"variant\": \"primary\", \"action\": {\"event\": {\"name\": \"\", \"context\": {\"formId\": \"booking\"}}}}\n Text: {\"text\": \"literal string\", \"variant\": \"h2\"} (h1/h2/h3/h4/h5/caption/body)\n TextField: {\"label\": \"Field\", \"value\": {\"path\": \"/p\"}, \"variant\": \"shortText\"} (shortText/longText/number/obscured)\n DateTimeInput: {\"label\": \"Date\", \"value\": {\"path\": \"/p\"}, \"enableDate\": true}\n ChoicePicker: {\"label\": \"Origin\", \"options\": [{\"label\":\"LAX\",\"value\":\"LAX\"}], \"value\": {\"path\":\"/origin\"}, \"variant\": \"mutuallyExclusive\"}\n CheckBox: {\"label\": \"...\", \"value\": {\"path\":\"/p\"}}\n Divider: {}\n\nExactly ONE component MUST have id \"root\" \u2014 it is the root of the tree.\n\nRequired form composition for THIS task:\n surface_id MUST be \"booking\"\n data_model MUST be {\"origin\": [\"LAX\"], \"dest\": [\"JFK\"], \"date\": \"2026-11-03\", \"passengers\": 2, \"fare_class\": [\"Business\"]} \u2190 use these values verbatim as the field defaults\n\n Build this component tree:\n root (Column, children=[\"card\"])\n card (Card, child=card_col)\n card_col (Column, children=[title, origin, dest, date, passengers, fare, submit])\n title (Text, text=\"Book a flight\", variant=\"h2\")\n origin (ChoicePicker, label=\"Origin\", options=[{'label': 'LAX', 'value': 'LAX'}, {'label': 'JFK', 'value': 'JFK'}, {'label': 'SFO', 'value': 'SFO'}, {'label': 'ORD', 'value': 'ORD'}, {'label': 'BOS', 'value': 'BOS'}, {'label': 'ATL', 'value': 'ATL'}, {'label': 'DFW', 'value': 'DFW'}, {'label': 'SEA', 'value': 'SEA'}, {'label': 'MIA', 'value': 'MIA'}, {'label': 'DEN', 'value': 'DEN'}], value={\"path\":\"/origin\"}, variant=\"mutuallyExclusive\")\n dest (ChoicePicker, label=\"Destination\", options=[{'label': 'LAX', 'value': 'LAX'}, {'label': 'JFK', 'value': 'JFK'}, {'label': 'SFO', 'value': 'SFO'}, {'label': 'ORD', 'value': 'ORD'}, {'label': 'BOS', 'value': 'BOS'}, {'label': 'ATL', 'value': 'ATL'}, {'label': 'DFW', 'value': 'DFW'}, {'label': 'SEA', 'value': 'SEA'}, {'label': 'MIA', 'value': 'MIA'}, {'label': 'DEN', 'value': 'DEN'}], value={\"path\":\"/dest\"}, variant=\"mutuallyExclusive\")\n date (DateTimeInput, label=\"Departure date (YYYY-MM-DD)\", value={\"path\":\"/date\"}, enableDate=true)\n passengers (TextField, label=\"Passengers\", value={\"path\":\"/passengers\"}, variant=\"number\")\n fare (ChoicePicker, label=\"Fare class\", options=[{'label': 'Economy', 'value': 'Economy'}, {'label': 'Premium', 'value': 'Premium'}, {'label': 'Business', 'value': 'Business'}, {'label': 'First', 'value': 'First'}], value={\"path\":\"/fare_class\"}, variant=\"mutuallyExclusive\")\n submit (Button, child=submit_label, variant=\"primary\", action={\"event\":{\"name\":\"bookingSubmit\",\"context\":{\n \"formId\":\"booking\",\n \"origin\":{\"path\":\"/origin\"},\n \"dest\":{\"path\":\"/dest\"},\n \"date\":{\"path\":\"/date\"},\n \"passengers\":{\"path\":\"/passengers\"},\n \"fare_class\":{\"path\":\"/fare_class\"}\n }}})\n submit_label (Text, text=\"Search flights\")\n\nUse these exact ids." + }, + "response": { + "toolCalls": [ + { + "name": "BookingFormSpec", + "arguments": "{\"surface_id\": \"booking\", \"data_model\": {\"origin\": [\"LAX\"], \"dest\": [\"JFK\"], \"date\": \"2026-11-03\", \"passengers\": 2, \"fare_class\": [\"Business\"]}, \"components\": [{\"id\": \"root\", \"component\": \"Column\", \"children\": [\"card\"]}, {\"id\": \"card\", \"component\": \"Card\", \"child\": \"card_col\"}, {\"id\": \"card_col\", \"component\": \"Column\", \"children\": [\"title\", \"origin\", \"dest\", \"date\", \"passengers\", \"fare\", \"submit\"]}, {\"id\": \"title\", \"component\": \"Text\", \"text\": \"Book a flight\", \"variant\": \"h2\"}, {\"id\": \"origin\", \"component\": \"ChoicePicker\", \"label\": \"Origin\", \"options\": [{\"label\": \"LAX\", \"value\": \"LAX\"}, {\"label\": \"JFK\", \"value\": \"JFK\"}, {\"label\": \"SFO\", \"value\": \"SFO\"}, {\"label\": \"ORD\", \"value\": \"ORD\"}, {\"label\": \"BOS\", \"value\": \"BOS\"}, {\"label\": \"ATL\", \"value\": \"ATL\"}, {\"label\": \"DFW\", \"value\": \"DFW\"}, {\"label\": \"SEA\", \"value\": \"SEA\"}, {\"label\": \"MIA\", \"value\": \"MIA\"}, {\"label\": \"DEN\", \"value\": \"DEN\"}], \"value\": {\"path\": \"/origin\"}, \"variant\": \"mutuallyExclusive\"}, {\"id\": \"dest\", \"component\": \"ChoicePicker\", \"label\": \"Destination\", \"options\": [{\"label\": \"LAX\", \"value\": \"LAX\"}, {\"label\": \"JFK\", \"value\": \"JFK\"}, {\"label\": \"SFO\", \"value\": \"SFO\"}, {\"label\": \"ORD\", \"value\": \"ORD\"}, {\"label\": \"BOS\", \"value\": \"BOS\"}, {\"label\": \"ATL\", \"value\": \"ATL\"}, {\"label\": \"DFW\", \"value\": \"DFW\"}, {\"label\": \"SEA\", \"value\": \"SEA\"}, {\"label\": \"MIA\", \"value\": \"MIA\"}, {\"label\": \"DEN\", \"value\": \"DEN\"}], \"value\": {\"path\": \"/dest\"}, \"variant\": \"mutuallyExclusive\"}, {\"id\": \"fare\", \"component\": \"ChoicePicker\", \"label\": \"Fare class\", \"options\": [{\"label\": \"Economy\", \"value\": \"Economy\"}, {\"label\": \"Premium\", \"value\": \"Premium\"}, {\"label\": \"Business\", \"value\": \"Business\"}, {\"label\": \"First\", \"value\": \"First\"}], \"value\": {\"path\": \"/fare_class\"}, \"variant\": \"mutuallyExclusive\"}, {\"id\": \"date\", \"component\": \"DateTimeInput\", \"label\": \"Departure date (YYYY-MM-DD)\", \"value\": {\"path\": \"/date\"}, \"enableDate\": true}, {\"id\": \"passengers\", \"component\": \"TextField\", \"label\": \"Passengers\", \"value\": {\"path\": \"/passengers\"}, \"variant\": \"number\"}, {\"id\": \"submit\", \"component\": \"Button\", \"child\": \"submit_label\", \"variant\": \"primary\", \"action\": {\"event\": {\"name\": \"bookingSubmit\", \"context\": {\"formId\": \"booking\", \"origin\": {\"path\": \"/origin\"}, \"dest\": {\"path\": \"/dest\"}, \"date\": {\"path\": \"/date\"}, \"passengers\": {\"path\": \"/passengers\"}, \"fare_class\": {\"path\": \"/fare_class\"}}}}}, {\"id\": \"submit_label\", \"component\": \"Text\", \"text\": \"Search flights\"}]}" + } + ] + } + }, + { + "match": { + "systemMessage": "You just received a booking submission. The find_routes() tool returned the following flights:\n\n[\n {\n \"flight_number\": \"UA204\",\n \"airline\": \"UA\",\n \"from\": \"SFO\",\n \"to\": \"SEA\",\n \"depart_local\": \"09:00\",\n \"arrive_local\": \"11:00\",\n \"duration_min\": 120,\n \"status\": \"on_time\",\n \"gate\": \"F11\",\n \"aircraft\": \"Boeing 737\"\n }\n]\n\nForm data (for context): {\n \"formId\": \"booking\",\n \"origin\": \"SFO\",\n \"dest\": \"SEA\",\n \"date\": \"2027-02-17\",\n \"passengers\": 3,\n \"fare_class\": \"Business\"\n}\n\nEmit an A2UI v0.9 results surface using the FlightResultsSpec schema.\n\nA2UI format (CRITICAL): every component is FLAT \u2014 `{\"id\": \"...\", \"component\": \"\", }`. The component name is a plain string; props sit at the top level of the same object. Dynamic values are bare literals or `{\"path\": \"/ptr\"}`.\n\nAllowed component names: Column, Row, Card, Text, TextField, Button, Divider, List.\n\nPer-component shapes you'll need:\n Column / List: {\"children\": [\"id1\", \"id2\"]}\n Card: {\"child\": \"\"}\n Text: {\"text\": \"literal\", \"variant\": \"h2\"} (or h1/h3/body/caption)\n Button: {\"child\": \"\", \"variant\": \"primary\", \"action\": {\"event\": {\"name\": \"\", \"context\": {\"flightId\": \"\"}}}}\n Divider: {}\n\nSurface constraints:\n surface_id MUST be \"results\"\n data_model can be {}\n Exactly ONE component MUST have id \"root\": a Column whose children lists every flight Card id (or just [\"no_flights\"] when empty)\n\nBuild pattern (one per flight):\n card_ (Card, child=col_)\n col_ (Column, children = [title_, route_, time_, btn_])\n title_ (Text, text=\" flight \", variant=\"h3\")\n route_ (Text, text=\" \u2192 \u2022 min \u2022 \", variant=\"body\")\n time_ (Text, text=\"Depart \u2022 Arrive \u2022 Gate \", variant=\"caption\")\n btn_ (Button, child=btn_label_, variant=\"primary\",\n action={\"event\":{\"name\":\"flightSelect\",\"context\":{\"flightId\":\"\"}}})\n btn_label_ (Text, text=\"Select\")\n\nEmpty case: components = [\n {\"id\":\"root\", \"component\":\"Column\", \"children\":[\"no_flights\"]},\n {\"id\":\"no_flights\", \"component\":\"Card\", \"child\":\"empty_col\"},\n {\"id\":\"empty_col\", \"component\":\"Column\", \"children\":[\"empty_msg\",\"modify_btn\"]},\n {\"id\":\"empty_msg\", \"component\":\"Text\", \"text\":\"No flights found\", \"variant\":\"h3\"},\n {\"id\":\"modify_btn\", \"component\":\"Button\", \"child\":\"modify_label\", \"action\":{\"event\":{\"name\":\"modifySearch\",\"context\":{\"formId\":\"booking\"}}}},\n {\"id\":\"modify_label\", \"component\":\"Text\", \"text\":\"Modify search\"}\n]\n\nUse unique ids for every component." + }, + "response": { + "toolCalls": [ + { + "name": "FlightResultsSpec", + "arguments": "{\"surface_id\": \"results\", \"data_model\": {}, \"components\": [{\"id\": \"root\", \"component\": \"Column\", \"children\": [\"card_0\"]}, {\"id\": \"card_0\", \"component\": \"Card\", \"child\": \"col_0\"}, {\"id\": \"col_0\", \"component\": \"Column\", \"children\": [\"title_0\", \"route_0\", \"time_0\", \"btn_0\"]}, {\"id\": \"title_0\", \"component\": \"Text\", \"text\": \"UA flight UA204\", \"variant\": \"h3\"}, {\"id\": \"route_0\", \"component\": \"Text\", \"text\": \"SFO \u2192 SEA \u2022 120 min \u2022 Boeing 737\", \"variant\": \"body\"}, {\"id\": \"time_0\", \"component\": \"Text\", \"text\": \"Depart 09:00 \u2022 Arrive 11:00 \u2022 Gate F11\", \"variant\": \"caption\"}, {\"id\": \"btn_0\", \"component\": \"Button\", \"child\": \"btn_label_0\", \"variant\": \"primary\", \"action\": {\"event\": {\"name\": \"flightSelect\", \"context\": {\"flightId\": \"UA204\"}}}}, {\"id\": \"btn_label_0\", \"component\": \"Text\", \"text\": \"Select\"}]}" + } + ] + } + }, + { + "match": { + "systemMessage": "You are an aviation booking-form designer. Emit an A2UI v0.9 booking form using the structured output schema.\n\nA2UI FORMAT (CRITICAL): each component is FLAT \u2014 `{\"id\": \"...\", \"component\": \"\", }`. The component name is a plain string and the props sit at the top level of the same object. ComponentName must be one of:\n Column, Row, Card, Text, TextField, ChoicePicker, DateTimeInput, CheckBox, Button, Divider, List, Image, Icon, Modal, Slider, Tabs\n\nDynamic values are bare literals (\"LAX\", 3, true) or data bindings `{\"path\": \"/json/pointer\"}`. Never wrap literals.\n\nPer-component shapes:\n Column / Row / List: {\"children\": [\"id1\", \"id2\"]}\n Card: {\"child\": \"\"} \u2190 single child only\n Button: {\"child\": \"\", \"variant\": \"primary\", \"action\": {\"event\": {\"name\": \"\", \"context\": {\"formId\": \"booking\"}}}}\n Text: {\"text\": \"literal string\", \"variant\": \"h2\"} (h1/h2/h3/h4/h5/caption/body)\n TextField: {\"label\": \"Field\", \"value\": {\"path\": \"/p\"}, \"variant\": \"shortText\"} (shortText/longText/number/obscured)\n DateTimeInput: {\"label\": \"Date\", \"value\": {\"path\": \"/p\"}, \"enableDate\": true}\n ChoicePicker: {\"label\": \"Origin\", \"options\": [{\"label\":\"LAX\",\"value\":\"LAX\"}], \"value\": {\"path\":\"/origin\"}, \"variant\": \"mutuallyExclusive\"}\n CheckBox: {\"label\": \"...\", \"value\": {\"path\":\"/p\"}}\n Divider: {}\n\nExactly ONE component MUST have id \"root\" \u2014 it is the root of the tree.\n\nRequired form composition for THIS task:\n surface_id MUST be \"booking\"\n data_model MUST be {\"origin\": [\"SFO\"], \"dest\": [\"SEA\"], \"date\": \"2027-02-17\", \"passengers\": 3, \"fare_class\": [\"Business\"]} \u2190 use these values verbatim as the field defaults\n\n Build this component tree:\n root (Column, children=[\"card\"])\n card (Card, child=card_col)\n card_col (Column, children=[title, origin, dest, date, passengers, fare, submit])\n title (Text, text=\"Book a flight\", variant=\"h2\")\n origin (ChoicePicker, label=\"Origin\", options=[{'label': 'LAX', 'value': 'LAX'}, {'label': 'JFK', 'value': 'JFK'}, {'label': 'SFO', 'value': 'SFO'}, {'label': 'ORD', 'value': 'ORD'}, {'label': 'BOS', 'value': 'BOS'}, {'label': 'ATL', 'value': 'ATL'}, {'label': 'DFW', 'value': 'DFW'}, {'label': 'SEA', 'value': 'SEA'}, {'label': 'MIA', 'value': 'MIA'}, {'label': 'DEN', 'value': 'DEN'}], value={\"path\":\"/origin\"}, variant=\"mutuallyExclusive\")\n dest (ChoicePicker, label=\"Destination\", options=[{'label': 'LAX', 'value': 'LAX'}, {'label': 'JFK', 'value': 'JFK'}, {'label': 'SFO', 'value': 'SFO'}, {'label': 'ORD', 'value': 'ORD'}, {'label': 'BOS', 'value': 'BOS'}, {'label': 'ATL', 'value': 'ATL'}, {'label': 'DFW', 'value': 'DFW'}, {'label': 'SEA', 'value': 'SEA'}, {'label': 'MIA', 'value': 'MIA'}, {'label': 'DEN', 'value': 'DEN'}], value={\"path\":\"/dest\"}, variant=\"mutuallyExclusive\")\n date (DateTimeInput, label=\"Departure date (YYYY-MM-DD)\", value={\"path\":\"/date\"}, enableDate=true)\n passengers (TextField, label=\"Passengers\", value={\"path\":\"/passengers\"}, variant=\"number\")\n fare (ChoicePicker, label=\"Fare class\", options=[{'label': 'Economy', 'value': 'Economy'}, {'label': 'Premium', 'value': 'Premium'}, {'label': 'Business', 'value': 'Business'}, {'label': 'First', 'value': 'First'}], value={\"path\":\"/fare_class\"}, variant=\"mutuallyExclusive\")\n submit (Button, child=submit_label, variant=\"primary\", action={\"event\":{\"name\":\"bookingSubmit\",\"context\":{\n \"formId\":\"booking\",\n \"origin\":{\"path\":\"/origin\"},\n \"dest\":{\"path\":\"/dest\"},\n \"date\":{\"path\":\"/date\"},\n \"passengers\":{\"path\":\"/passengers\"},\n \"fare_class\":{\"path\":\"/fare_class\"}\n }}})\n submit_label (Text, text=\"Search flights\")\n\nUse these exact ids." + }, + "response": { + "toolCalls": [ + { + "name": "BookingFormSpec", + "arguments": "{\"surface_id\": \"booking\", \"data_model\": {\"origin\": [\"SFO\"], \"dest\": [\"SEA\"], \"date\": \"2027-02-17\", \"passengers\": 3, \"fare_class\": [\"Business\"]}, \"components\": [{\"id\": \"root\", \"component\": \"Column\", \"children\": [\"card\"]}, {\"id\": \"card\", \"component\": \"Card\", \"child\": \"card_col\"}, {\"id\": \"card_col\", \"component\": \"Column\", \"children\": [\"title\", \"origin\", \"dest\", \"date\", \"passengers\", \"fare\", \"submit\"]}, {\"id\": \"title\", \"component\": \"Text\", \"text\": \"Book a flight\", \"variant\": \"h2\"}, {\"id\": \"origin\", \"component\": \"ChoicePicker\", \"label\": \"Origin\", \"options\": [{\"label\": \"LAX\", \"value\": \"LAX\"}, {\"label\": \"JFK\", \"value\": \"JFK\"}, {\"label\": \"SFO\", \"value\": \"SFO\"}, {\"label\": \"ORD\", \"value\": \"ORD\"}, {\"label\": \"BOS\", \"value\": \"BOS\"}, {\"label\": \"ATL\", \"value\": \"ATL\"}, {\"label\": \"DFW\", \"value\": \"DFW\"}, {\"label\": \"SEA\", \"value\": \"SEA\"}, {\"label\": \"MIA\", \"value\": \"MIA\"}, {\"label\": \"DEN\", \"value\": \"DEN\"}], \"value\": {\"path\": \"/origin\"}, \"variant\": \"mutuallyExclusive\"}, {\"id\": \"dest\", \"component\": \"ChoicePicker\", \"label\": \"Destination\", \"options\": [{\"label\": \"LAX\", \"value\": \"LAX\"}, {\"label\": \"JFK\", \"value\": \"JFK\"}, {\"label\": \"SFO\", \"value\": \"SFO\"}, {\"label\": \"ORD\", \"value\": \"ORD\"}, {\"label\": \"BOS\", \"value\": \"BOS\"}, {\"label\": \"ATL\", \"value\": \"ATL\"}, {\"label\": \"DFW\", \"value\": \"DFW\"}, {\"label\": \"SEA\", \"value\": \"SEA\"}, {\"label\": \"MIA\", \"value\": \"MIA\"}, {\"label\": \"DEN\", \"value\": \"DEN\"}], \"value\": {\"path\": \"/dest\"}, \"variant\": \"mutuallyExclusive\"}, {\"id\": \"fare\", \"component\": \"ChoicePicker\", \"label\": \"Fare class\", \"options\": [{\"label\": \"Economy\", \"value\": \"Economy\"}, {\"label\": \"Premium\", \"value\": \"Premium\"}, {\"label\": \"Business\", \"value\": \"Business\"}, {\"label\": \"First\", \"value\": \"First\"}], \"value\": {\"path\": \"/fare_class\"}, \"variant\": \"mutuallyExclusive\"}, {\"id\": \"date\", \"component\": \"DateTimeInput\", \"label\": \"Departure date (YYYY-MM-DD)\", \"value\": {\"path\": \"/date\"}, \"enableDate\": true}, {\"id\": \"passengers\", \"component\": \"TextField\", \"label\": \"Passengers\", \"value\": {\"path\": \"/passengers\"}, \"variant\": \"number\"}, {\"id\": \"submit\", \"component\": \"Button\", \"child\": \"submit_label\", \"variant\": \"primary\", \"action\": {\"event\": {\"name\": \"bookingSubmit\", \"context\": {\"formId\": \"booking\", \"origin\": {\"path\": \"/origin\"}, \"dest\": {\"path\": \"/dest\"}, \"date\": {\"path\": \"/date\"}, \"passengers\": {\"path\": \"/passengers\"}, \"fare_class\": {\"path\": \"/fare_class\"}}}}}, {\"id\": \"submit_label\", \"component\": \"Text\", \"text\": \"Search flights\"}]}" + } + ] + } } ] } diff --git a/cockpit/chat/a2ui/react/README.md b/cockpit/chat/a2ui/react/README.md index f1754ba74..bd3c06aec 100644 --- a/cockpit/chat/a2ui/react/README.md +++ b/cockpit/chat/a2ui/react/README.md @@ -19,4 +19,4 @@ WEBSITE_E2E_REACT_TOPIC=chat-a2ui npx nx e2e website --grep 'native Chat A2UI' - The app e2e target builds the actual private packages and runs its installed terminal admission suite first. Website e2e retains its full registered React build prerequisites even when the topic selector narrows the running servers. Local serving uses `node scripts/react-cockpit/serve.mjs chat-a2ui --no-parent` on port 4646 after preparation. -Local verification runs the actual compiled canonical graph with deterministic providers, the real SDK and saved checkpoints. These fixture results establish local interoperability, not live-provider timing or durable deployment behavior. Production sources and public Code exclude fixture controls, testing helpers and the private probe. Runtime settings choose one configured shared backend; this example adds no service or public API. +Local verification runs the canonical graph in a real `langgraph dev`, with aimock replaying the model's recorded outputs (`cockpit/chat/a2ui/angular/e2e/fixtures/c-a2ui.json`), the real SDK and saved checkpoints. These results establish local interoperability, not live-provider timing or durable deployment behavior. Production sources and public Code exclude the local proxy controls, testing helpers and the private probe. Runtime settings choose one configured shared backend; this example adds no service or public API. diff --git a/cockpit/chat/a2ui/react/e2e/a2ui.spec.ts b/cockpit/chat/a2ui/react/e2e/a2ui.spec.ts index 8cb00b41f..ea0a75e27 100644 --- a/cockpit/chat/a2ui/react/e2e/a2ui.spec.ts +++ b/cockpit/chat/a2ui/react/e2e/a2ui.spec.ts @@ -4,11 +4,20 @@ import { writeFile } from 'node:fs/promises'; const base = '/chat/a2ui/react/'; const current = (page: Page) => page.getByRole('region', { name: 'Current flight preview', exact: true }); const earlier = (page: Page) => page.getByRole('region', { name: 'Earlier flight preview', exact: true }); -const lifetime = async (request: APIRequestContext) => (await request.get('/__lifetime')).json(); +// Backend: aimock replays cockpit/chat/a2ui/angular/e2e/fixtures (the A2UI +// model's outputs for these flows, keyed on each node's full system prompt) +// into a real `langgraph dev`, whose real tools find routes and look flights +// up. The recording proxy (scripts/react-cockpit/langgraph-proxy.mjs) records +// /api requests and holds the next run stream. +/** Every /api request so far, as the proxy recorded it. */ +const lifetime = async (request: APIRequestContext) => ({ requests: await (await request.get('/__requests')).json() }); +/** Whether the proxy holds a run stream right now. */ +const held = async (request: APIRequestContext) => (await (await request.get('/__lifetime')).json()).activeStream; +const hold = (request: APIRequestContext) => request.post('/__hold/stream', { data: { immediate: true } }); const streamRequests = (receipt: any) => receipt.requests.filter((r: any) => r.method === 'POST' && r.path.endsWith('/runs/stream')); async function save(request: APIRequestContext, info: TestInfo, extra: unknown = {}) { const path = info.outputPath('actual-graph-and-transport.json'); - await writeFile(path, JSON.stringify({ receipt: await lifetime(request), extra }, null, 2)); + await writeFile(path, JSON.stringify({ receipt: await lifetime(request), captures: await (await request.get('/__captures')).json(), extra }, null, 2)); await info.attach('actual-graph-and-transport.json', { path, contentType: 'application/json' }); } async function form(page: Page) { @@ -78,9 +87,10 @@ test('installed native application saves request, edited Search, Select and serv expect(search.action).toMatchObject({ name: 'bookingSubmit', surfaceId: 'booking', context: { formId: 'booking', origin: ['SFO'], dest: ['SEA'], date: '2027-02-17', passengers: 3, fare_class: ['Business'] } }); expect(JSON.parse(runs[2].body.input.messages[0].content).action).toMatchObject({ name: 'flightSelect', context: { flightId: 'UA204' } }); expect(JSON.parse(runs[3].body.input.messages[0].content).action.name).toBe('modifySearch'); - expect(receipt.runReads).toHaveLength(4); - expect(receipt.runReads.every((r: any) => r.run.status === 'success' && r.threadId === threadId)).toBe(true); - expect(new Set(receipt.runReads.map((r: any) => r.run.run_id)).size).toBe(4); + const runReads = receipt.requests.filter((r: any) => r.method === 'GET' && /\/runs\/[^/]+$/.test(r.path)); + expect(runReads).toHaveLength(4); + expect(runReads.every((r: any) => r.path.startsWith(`/api/threads/${threadId}/runs/`))).toBe(true); + expect(new Set(runReads.map((r: any) => r.path)).size).toBe(4); await expect.poll(() => exactReads.length).toBe(4); for (const [index, state] of exactReads.entries()) { expect(state.checkpoint).toMatchObject({ thread_id: threadId, checkpoint_ns: '' }); @@ -89,10 +99,10 @@ test('installed native application saves request, edited Search, Select and serv expect(state.values.messages).toHaveLength((index + 1) * 2); expect(state.values.messages.at(-2).id).toBe(runs[index].body.input.messages[0].id); } - expect(receipt.proofs.every((p: any) => p.actualCompiledGraph && p.networkConnectAttempts === 0 && p.sourceSha256 === '9a54abf5d03dfd7d285265fc4f3c37ca3b5a830dee319cf52093584907ea8462' && p.lockSha256 === '56247b7f8c8a4cd27f43238a3c99f37f251c8e3a850a94d1761d7ac472abf303')).toBe(true); - expect(JSON.stringify(receipt.proofs)).toContain('_find_routes_impl'); - expect(JSON.stringify(receipt.events)).toContain('generate_title'); - expect(JSON.stringify(receipt.events)).toContain('messages'); + // What the real server streamed: title and message callbacks included. + const streamed = JSON.stringify(await (await request.get('/__captures')).json()); + expect(streamed).toContain('generate_title'); + expect(streamed).toContain('event: messages'); await save(request, info, { exactReads }); }); @@ -116,9 +126,9 @@ test('invalid lexical drafts and direct user activation send zero actions; rapid await expect(current(page).getByText('Enter a real date as YYYY-MM-DD.')).toBeVisible(); expect(streamRequests(await lifetime(request))).toHaveLength(1); await edit(page); - await request.post('/__hold/stream'); + await hold(request); await search.evaluate((button: HTMLButtonElement) => { button.click(); button.click(); }); - await expect.poll(async () => (await lifetime(request)).heldCompleted).toBe(1); + await expect.poll(async () => held(request)).toBe(true); expect(streamRequests(await lifetime(request))).toHaveLength(2); await request.post('/__release'); await expect(current(page).getByRole('button', { name: 'Select', exact: true })).toBeVisible(); @@ -173,16 +183,16 @@ test('untrusted saved boundaries fail closed, retain prior cards, and late held for (const command of ['Stop', 'New conversation']) { await page.getByRole('button', { name: 'New conversation', exact: true }).click(); await form(page); await edit(page); - await request.post('/__hold/stream'); + await hold(request); await current(page).getByRole('button', { name: 'Search flights', exact: true }).click(); - await expect.poll(async () => (await lifetime(request)).heldCompleted).toBe(1); + await expect.poll(async () => held(request)).toBe(true); await page.getByRole('button', { name: command, exact: true }).click(); - await expect.poll(async () => (await lifetime(request)).heldStreams).toBe(0); + await expect.poll(async () => held(request)).toBe(false); await request.post('/__release'); await expect(current(page)).toHaveCount(0); await expect(page.getByRole('button', { name: 'Select', exact: true })).toHaveCount(0); } - await save(request, info, { failures, lifetimeMeaning: 'Actual graph completed; HTTP delivery was held. No server-side graph-cancellation claim.' }); + await save(request, info, { failures, lifetimeMeaning: 'The proxy held the real run stream after its first event; Stop or New closed it.' }); }); test('390px keyboard workflow remains readable and pagehide revokes held HTTP delivery and views', async ({ page, request }, info) => { @@ -207,16 +217,65 @@ test('390px keyboard workflow remains readable and pagehide revokes held HTTP de await expect(current(page).getByRole('button', { name: 'Modify search', exact: true })).toBeVisible(); await noOverflow(page); await page.screenshot({ path: info.outputPath('mobile-confirmation.png'), fullPage: true }); - await request.post('/__hold/stream'); + await hold(request); await current(page).getByRole('button', { name: 'Modify search', exact: true }).click(); - await expect.poll(async () => (await lifetime(request)).heldCompleted).toBe(1); + await expect.poll(async () => held(request)).toBe(true); await page.evaluate(() => window.dispatchEvent(new Event('pagehide'))); await expect(page.locator('#root')).toBeEmpty(); - await expect.poll(async () => (await lifetime(request)).heldStreams).toBe(0); + await expect.poll(async () => held(request)).toBe(false); await request.post('/__release'); await expect(page.locator('#root')).toBeEmpty(); - const receipt = await lifetime(request); - expect(receipt.streamCloses.some((closed: any) => closed.held && closed.destroyed)).toBe(true); - expect(streamRequests(receipt)).toHaveLength(4); - await save(request, info, { scope: 'Chromium desktop/mobile viewport, labels, Tab/Enter, synthetic pagehide; graph completed before held HTTP close.' }); + expect(streamRequests(await lifetime(request))).toHaveLength(4); + await save(request, info, { scope: 'Chromium desktop/mobile viewport, labels, Tab/Enter, synthetic pagehide closing a held run stream.' }); +}); + +// The real graph's own model-failure paths, reached through model output. +// Injected data: aimock fixtures tagged `[scenario:]` in +// cockpit/chat/a2ui/angular/e2e/fixtures/c-a2ui.json answer the title prompt +// with a provider error, or the booking-form designer with a form whose fare +// option is `Economomy` (outside the schema's literals). +async function graphTurn(request: APIRequestContext, content: string) { + const thread = await (await request.post('/api/threads', { data: {} })).json(); + const stream = await request.post(`/api/threads/${thread.thread_id}/runs/stream`, { + data: { assistant_id: 'c-a2ui', input: { messages: [{ type: 'human', content }] }, stream_mode: ['values', 'messages-tuple'] }, + }); + expect(stream.status()).toBe(200); + const sse = await stream.text(); + const runId = /"run_id":\s*"([^"]+)"/.exec(sse)?.[1]; + expect(runId).toBeTruthy(); + const run = await (await request.get(`/api/threads/${thread.thread_id}/runs/${runId}`)).json(); + const state = await (await request.get(`/api/threads/${thread.thread_id}/state`)).json(); + const captures = await (await request.get('/__captures')).json(); + return { sse, run, state, capture: captures.at(-1) }; +} +const formMessage = (state: any) => state.values.messages.at(-1).content as string; + +test('a failed title model call is swallowed: the form still saves and the thread keeps no title', async ({ request }) => { + const { sse, run, state, capture } = await graphTurn(request, 'I want to fly LAX to JFK [scenario:title-down]'); + expect(run.status).toBe('success'); + expect(state.next).toEqual([]); + expect(state.values.messages).toHaveLength(2); + expect(formMessage(state)).toMatch(/^---a2ui_JSON---\n/); + expect(formMessage(state)).not.toMatch(/Book a flight \(fallback\)/); + expect(sse).not.toContain('generate_title'); + expect(capture.titleAfter ?? null).toBeNull(); +}); + +test('one invalid structured form is retried and the corrected form saves', async ({ request }) => { + const { run, state } = await graphTurn(request, 'I want to fly LAX to JFK [scenario:retry]'); + expect(run.status).toBe('success'); + expect(state.values.messages).toHaveLength(2); + // Without the retry the invalid first answer would have ended in the fallback form. + expect(formMessage(state)).toMatch(/^---a2ui_JSON---\n/); + expect(formMessage(state)).not.toMatch(/Book a flight \(fallback\)/); + expect(formMessage(state)).not.toContain('Economomy'); +}); + +test('three invalid structured forms end in the graph\'s fallback form, seeded from the request', async ({ request }) => { + const { run, state } = await graphTurn(request, 'I want to fly LAX to JFK [scenario:invalid]'); + expect(run.status).toBe('success'); + expect(state.values.messages).toHaveLength(2); + expect(formMessage(state)).toMatch(/Book a flight \(fallback\)/); + const data = formMessage(state).split('\n').slice(1).filter(Boolean).map((line) => JSON.parse(line)).find((e) => e.updateDataModel)?.updateDataModel.value; + expect(data).toMatchObject({ origin: ['LAX'], dest: ['JFK'], passengers: 1, fare_class: ['Economy'] }); }); diff --git a/cockpit/chat/a2ui/react/e2e/global-setup-impl.ts b/cockpit/chat/a2ui/react/e2e/global-setup-impl.ts new file mode 100644 index 000000000..8405016e7 --- /dev/null +++ b/cockpit/chat/a2ui/react/e2e/global-setup-impl.ts @@ -0,0 +1,6 @@ +// eslint-disable-next-line @nx/enforce-module-boundaries -- Shared aimock setup for the React cockpit examples. +import { reactAimockGlobalSetup } from '../../../../shared/react-aimock-setup'; + +// aimock replays the Angular example's fixtures into a real `langgraph dev`; +// the static server (webServer) proxies /api to it. +export default reactAimockGlobalSetup({ example: 'chat/a2ui', port: 4646 }); diff --git a/cockpit/chat/a2ui/react/e2e/playwright.config.ts b/cockpit/chat/a2ui/react/e2e/playwright.config.ts index 211d97b03..b9cce7b30 100644 --- a/cockpit/chat/a2ui/react/e2e/playwright.config.ts +++ b/cockpit/chat/a2ui/react/e2e/playwright.config.ts @@ -8,6 +8,9 @@ export default defineConfig({ outputDir: '../../../../../output/playwright/chat-a2ui-react', use: { baseURL: 'http://127.0.0.1:4646', trace: 'retain-on-failure' }, projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }], + // aimock + real `langgraph dev`, shared with cockpit/chat/a2ui/angular. + globalSetup: './global-setup-impl.ts', + globalTeardown: require.resolve('../../../../../libs/e2e-harness/src/global-teardown'), webServer: { command: 'node scripts/react-cockpit/serve.mjs chat-a2ui --no-parent', cwd: '../../../../..', diff --git a/cockpit/chat/a2ui/react/fixtures/terminal-admission.spec.mjs b/cockpit/chat/a2ui/react/fixtures/terminal-admission.spec.mjs index baa685767..56379ff6b 100644 --- a/cockpit/chat/a2ui/react/fixtures/terminal-admission.spec.mjs +++ b/cockpit/chat/a2ui/react/fixtures/terminal-admission.spec.mjs @@ -6,7 +6,7 @@ test( 'installed actual application helpers admit four canonical native graph turns', { timeout: 180000 }, async () => { - // runNativeProbe explicitly prepares the existing frozen Python fixture configuration. + // The installed consumer runs against aimock + a real `langgraph dev`, behind the recording proxy. const report = await runNativeProbe({ mode: 'application-admission' }); assert.equal(report.mode, 'application-admission'); assert.equal(report.compiler.claim, 'actual application admission helpers'); @@ -38,6 +38,10 @@ test( assert.ok(report.receipt.cleanup.every((row) => row.closed)); assert.equal(report.closure.processAbsent, true); assert.equal(report.closure.groupAbsent, true); - assert.equal(report.receipt.fixture.worker.pythonAbsent, true); + // The real backend (aimock + `langgraph dev`) was the one the consumer + // drove, and its process tree was gone when the consumer closed it. + assert.equal(report.receipt.backend.kind, 'aimock-langgraph'); + assert.equal(report.receipt.backend.python, 'cockpit/chat/a2ui/python'); + assert.ok(report.receipt.cleanup.some((row) => row.name === 'backend' && row.closed)); } ); diff --git a/cockpit/chat/a2ui/react/project.json b/cockpit/chat/a2ui/react/project.json index b9a5c0629..3e13b4f40 100644 --- a/cockpit/chat/a2ui/react/project.json +++ b/cockpit/chat/a2ui/react/project.json @@ -18,6 +18,7 @@ "e2e": { "executor": "@nx/playwright:playwright", "cache": false, + "inputs": ["default", "reactAimockBackend"], "dependsOn": ["build", "fixture-test"], "options": { "config": "cockpit/chat/a2ui/react/e2e/playwright.config.ts", diff --git a/cockpit/chat/a2ui/react/src/connection.integration.spec.ts b/cockpit/chat/a2ui/react/src/connection.integration.spec.ts index 222018a83..254d8c842 100644 --- a/cockpit/chat/a2ui/react/src/connection.integration.spec.ts +++ b/cockpit/chat/a2ui/react/src/connection.integration.spec.ts @@ -1,63 +1,53 @@ -import { createServer } from 'node:http'; +import { createServer, type Server } from 'node:http'; import { once } from 'node:events'; -import { mkdtempSync, writeFileSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { resolve } from 'node:path'; import type { AddressInfo } from 'node:net'; -import { expect, it, onTestFinished } from 'vitest'; +import { afterAll, beforeAll, expect, it, onTestFinished } from 'vitest'; import { createConnectedApplication } from './connection'; import type { A2uiApplication } from './application'; // eslint-disable-next-line @nx/enforce-module-boundaries -- Borrow the moved private presentation type. import type { SurfaceView } from '../../../../../libs/cockpit-a2ui-react/src/surface'; +/* eslint-disable @nx/enforce-module-boundaries -- Local test backend tooling is never bundled. */ +// @ts-expect-error Local test-only JavaScript has no published declaration. +import { createAimockBackend } from '../../../../../scripts/react-cockpit/aimock-backend.mjs'; +// @ts-expect-error Local test-only JavaScript has no published declaration. +import { createLanggraphProxy } from '../../../../../scripts/react-cockpit/langgraph-proxy.mjs'; +// @ts-expect-error Local test-only JavaScript has no published declaration. +import { reactCockpitConfiguration } from '../../../../../scripts/react-cockpit/configuration.mjs'; +/* eslint-enable @nx/enforce-module-boundaries */ + +// The real graph runs under `langgraph dev` on aimock fixtures shared with the +// Angular example (cockpit/chat/a2ui/angular/e2e/fixtures); the in-process +// proxy records each /api request (scripts/react-cockpit/langgraph-proxy.mjs). +const { backend: config } = reactCockpitConfiguration('chat-a2ui'); +const backend = createAimockBackend({ root: resolve(__dirname, '../../../../..'), backend: config }); +beforeAll(() => backend.ensure(), 180000); +afterAll(() => backend.close()); +type Handler = (request: unknown, response: unknown, pathname: string) => Promise; async function fixture() { - const location = new URL( - '../../../../../scripts/react-cockpit/chat-a2ui-fixture.mjs', - import.meta.url - ).href; - const { createChatA2uiFixture, fixtureConfiguration } = await import( - /* @vite-ignore */ location - ); - // Existing fixture prepares one frozen isolated Python environment; no model credentials. - const evidence = mkdtempSync( - join(tmpdir(), 'threadplane-chat-a2ui-task3-sdk-') - ); - const handle = createChatA2uiFixture({ - configuration: fixtureConfiguration(), + const proxy: Handler = createLanggraphProxy({ + upstream: `http://127.0.0.1:${config.langgraphPort}`, + ready: () => backend.ensure(), }); - const server = createServer(async (req, res) => { - if ( - !(await handle( - req, - res, - new URL(req.url ?? '', 'http://localhost').pathname - )) - ) { + const server: Server = createServer(async (req, res) => { + if (!(await proxy(req, res, new URL(req.url ?? '', 'http://localhost').pathname))) { res.writeHead(404); res.end(); } }); server.listen(0, '127.0.0.1'); await once(server, 'listening'); - const app = createConnectedApplication({ - apiUrl: `http://127.0.0.1:${(server.address() as AddressInfo).port}/api`, - headers: {}, - }); + const base = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + const app = createConnectedApplication({ apiUrl: base + '/api', headers: {} }); onTestFinished(async () => { await app.dispose(); - await handle.close(); server.closeAllConnections(); await new Promise((r) => server.close(() => r())); - const receipt = handle.receipt(); - writeFileSync( - join(evidence, 'receipt.json'), - JSON.stringify(receipt, null, 2) + '\n', - { mode: 0o600 } - ); - console.log('Task 3 actual SDK evidence: ' + evidence); - expect(receipt.worker.reaped).toBe(true); - expect(receipt.worker.pythonAbsent).toBe(true); }); + const handle = { + receipt: async () => ({ requests: await (await fetch(base + '/__requests')).json() }), + }; return { app, handle }; } async function action(app: A2uiApplication, surface: SurfaceView, id: string) { @@ -75,7 +65,7 @@ async function action(app: A2uiApplication, surface: SurfaceView, id: string) { } it('actual default SDK session admits the compiled graph four-turn edited Search, Select and Modify flow', async () => { const f = await fixture(); - expect(f.handle.receipt().requests).toEqual([]); + expect((await f.handle.receipt()).requests).toEqual([]); const publications: ReturnType[] = []; f.app.subscribe(() => publications.push(f.app.getSnapshot())); expect(await f.app.submit('I want to fly LAX to JFK')).toBe(true); @@ -116,7 +106,7 @@ it('actual default SDK session admits the compiled graph four-turn edited Search expect(state.messages.length).toBe(state.surfaces.length * 2); if (state.busy) expect(state.activeSurface).toBeNull(); } - const receipt = f.handle.receipt(); + const receipt = await f.handle.receipt(); expect( receipt.requests.filter((r: { path: string }) => r.path === '/api/threads') ).toHaveLength(1); @@ -160,18 +150,10 @@ it('actual default SDK session admits the compiled graph four-turn edited Search r.body.checkpoint.thread_id === f.app.getSnapshot().threadId ) ).toBe(true); + // Each turn's run status was read once from the real server. expect( - receipt.runReads.map((r: { run: { status: string } }) => r.run.status) - ).toEqual(['success', 'success', 'success', 'success']); - expect( - receipt.proofs.every( - (p: { actualCompiledGraph: boolean; networkConnectAttempts: number }) => - p.actualCompiledGraph && p.networkConnectAttempts === 0 - ) - ).toBe(true); - expect( - receipt.proofs.some((p: { tools?: { name: string }[] }) => - p.tools?.some((t) => t.name === '_lookup_flight_impl') + receipt.requests.filter( + (r: { method: string; path: string }) => r.method === 'GET' && /\/runs\/[^/]+$/.test(r.path) ) - ).toBe(true); + ).toHaveLength(4); }, 60000); diff --git a/cockpit/chat/debug/angular/e2e/fixtures/c-debug.json b/cockpit/chat/debug/angular/e2e/fixtures/c-debug.json index 33d8cd90e..e95a2ff6f 100644 --- a/cockpit/chat/debug/angular/e2e/fixtures/c-debug.json +++ b/cockpit/chat/debug/angular/e2e/fixtures/c-debug.json @@ -377,6 +377,38 @@ "metadata": { "systemHash": "05dfe622" } + }, + { + "match": { "systemMessage": "Provide a brief one-sentence summary" }, + "response": { "content": "The traveler asked an aviation question and received a short explanation." } + }, + { + "match": { "systemMessage": "In 3-5 words, summarize" }, + "response": { "content": "Aviation debug question" } + }, + { + "match": { "userMessage": "Explain lift ✈ café" }, + "response": { "content": "Lift is the upward force a wing makes as air flows faster over its curved top than beneath it." } + }, + { + "match": { "userMessage": "Continue lift — aérodrome" }, + "response": { "content": "At a high aérodrome the thinner air means a wing needs more speed to make the same lift." } + }, + { + "match": { "userMessage": "first input 🛩️" }, + "response": { "content": "A first answer about small aircraft and their wings." } + }, + { + "match": { "userMessage": "second input" }, + "response": { "content": "A second answer that follows the first one." } + }, + { + "match": { "userMessage": "labelled generated Unicode extension" }, + "response": { "content": "Cleared for departure 🛩️ at the café gate." } + }, + { + "match": { "userMessage": "labelled omitted B callback projection" }, + "response": { "content": "An answer whose processing callback the transport omits." } } ] } \ No newline at end of file diff --git a/cockpit/chat/debug/react/e2e/debug.spec.ts b/cockpit/chat/debug/react/e2e/debug.spec.ts index 1334e78ed..8443c278a 100644 --- a/cockpit/chat/debug/react/e2e/debug.spec.ts +++ b/cockpit/chat/debug/react/e2e/debug.spec.ts @@ -101,24 +101,14 @@ test.afterEach(async ({ page, request }, info) => { await page.goto('about:blank'); await control(request, '/__release'); await control(request, '/__reset'); + // No proxied browser connection, held read or held stream outlives the test. await expect - .poll(async () => (await read(request, '/__lifetime')).active) - .toEqual({ children: 0, streams: 0, reads: 0, handlers: 0, responses: 0 }); + .poll(async () => (await read(request, '/__lifetime')).openResponses) + .toBe(0); const lifetime = await read(request, '/__lifetime'); - for (const child of lifetime.children) { - expect(child.reaped).toBe(true); - for (const id of [child.pid, -child.pgid]) { - let code; - try { - process.kill(id, 0); - } catch (error) { - code = (error as NodeJS.ErrnoException).code; - } - expect(code).toBe('ESRCH'); - } - } + expect(lifetime).toMatchObject({ activeStream: false, activeHistory: false, activeState: false }); await writeFile( - info.outputPath('observed-child-absence.json'), + info.outputPath('observed-lifetime.json'), JSON.stringify(lifetime, null, 2) ); await writeFile( @@ -142,18 +132,14 @@ for (const width of [1280, 390]) await confirmed(page); await expect(page.locator('article')).toHaveCount(4); const first = await page.locator('article').allTextContents(); + // The real server's saved state for the run (scripts/react-cockpit/langgraph-proxy.mjs). const captures = await read(request, '/__captures'); - const a = captures[0].state.values.messages[1].content; - expect(captures[0].state.values.analysis).toEqual({ + const a = captures[0].values.messages[1].content; + // The graph's `process` node counts Python code points. + expect(captures[0].values.analysis).toEqual({ characters: [...a].length, words: a.split(' ').length, }); - expect(captures[0].proof).toMatchObject({ - actualCompiledGraph: true, - networkAttempts: 0, - providerVariant: 'natural', - }); - expect(captures[0].localCapturedReplay).toBe(true); await expect( page.getByLabel('Observed analysis', { exact: true }) ).toContainText(`${[...a].length} characters`); @@ -206,7 +192,7 @@ for (const width of [1280, 390]) displayed ); expect(JSON.parse(displayed ?? '{}')).toEqual({ - analysis: captures[0].state.values.analysis, + analysis: captures[0].values.analysis, }); const history = await refresh(page); const afterRefresh = await wire(request); @@ -236,26 +222,26 @@ for (const width of [1280, 390]) .getByText('Compare selected checkpoint with current state', { exact: true }) .click(); await expect.poll(async () => JSON.parse(await selected.getByLabel('Selected checkpoint to current state JSON', { exact: true }).textContent() ?? 'null')).toEqual([ - { kind: 'added', path: '/analysis', after: captures[0].state.values.analysis }, + { kind: 'added', path: '/analysis', after: captures[0].values.analysis }, ]); const laterIndex = history.findIndex( (row) => row.values.analysis !== undefined && (row.values.messages?.length ?? 0) < - captures[0].state.values.messages.length && + captures[0].values.messages.length && row.checkpoint.checkpoint_id !== - captures[0].state.checkpoint.checkpoint_id + captures[0].checkpoint.checkpoint_id ); expect(laterIndex).toBeGreaterThanOrEqual(0); const observedLater = await inspect(page, laterIndex); expect(observedLater.checkpoint).toEqual(history[laterIndex].checkpoint); expect(observedLater.checkpoint.checkpoint_id).not.toBe( - captures[0].state.checkpoint.checkpoint_id + captures[0].checkpoint.checkpoint_id ); expect(observedLater.values.analysis).toEqual( - captures[0].state.values.analysis + captures[0].values.analysis ); - expect(observedLater.values).not.toEqual(captures[0].state.values); + expect(observedLater.values).not.toEqual(captures[0].values); await selected .getByText('Displayed checkpoint values', { exact: true }) .click(); @@ -417,8 +403,8 @@ for (const action of ['Stop', 'New conversation', 'unmount']) await control(request, '/__hold/history'); await page.getByRole('button', { name: 'Refresh checkpoints' }).click(); await expect - .poll(async () => (await read(request, '/__lifetime')).active.reads) - .toBe(1); + .poll(async () => (await read(request, '/__lifetime')).activeHistory) + .toBe(true); if (action === 'New conversation') await page .getByRole('button', { name: 'Close debug panel', exact: true }) @@ -426,7 +412,7 @@ for (const action of ['Stop', 'New conversation', 'unmount']) if (action === 'unmount') await page.goto('about:blank'); else await page.getByRole('button', { name: action, exact: true }).click(); await expect - .poll(async () => (await read(request, '/__lifetime')).active.reads) + .poll(async () => (await read(request, '/__lifetime')).openResponses) .toBe(0); await control(request, '/__release'); const ledger = await wire(request); @@ -595,14 +581,14 @@ for (const action of ['Stop', 'New conversation', 'unmount']) await ready(page); await send(page); await expect - .poll(async () => (await read(request, '/__lifetime')).active.streams) - .toBe(1); + .poll(async () => (await read(request, '/__lifetime')).activeStream) + .toBe(true); await expect(mainStatus(page)).toHaveText('Receiving response…'); await expect(page.locator('article')).toHaveCount(2); if (action === 'unmount') await page.goto('about:blank'); else await page.getByRole('button', { name: action, exact: true }).click(); await expect - .poll(async () => (await read(request, '/__lifetime')).active.streams) + .poll(async () => (await read(request, '/__lifetime')).openResponses) .toBe(0); await control(request, '/__release'); const ledger = await wire(request); @@ -615,7 +601,7 @@ for (const action of ['Stop', 'New conversation', 'unmount']) JSON.stringify( { label: - 'Captured actual graph SSE replay; compiled graph already completed before HTTP replay hold', + 'Real langgraph dev stream held by the recording proxy after the first answer chunk', action, ledger, }, diff --git a/cockpit/chat/debug/react/e2e/global-setup-impl.ts b/cockpit/chat/debug/react/e2e/global-setup-impl.ts new file mode 100644 index 000000000..5c03a3adf --- /dev/null +++ b/cockpit/chat/debug/react/e2e/global-setup-impl.ts @@ -0,0 +1,6 @@ +// eslint-disable-next-line @nx/enforce-module-boundaries -- Shared aimock setup for the React cockpit examples. +import { reactAimockGlobalSetup } from '../../../../shared/react-aimock-setup'; + +// aimock replays the Angular example's fixtures into a real `langgraph dev`; +// the static server (webServer) proxies /api to it. +export default reactAimockGlobalSetup({ example: 'chat/debug', port: 4645 }); diff --git a/cockpit/chat/debug/react/e2e/playwright.config.ts b/cockpit/chat/debug/react/e2e/playwright.config.ts index 353ff150c..c2142fd32 100644 --- a/cockpit/chat/debug/react/e2e/playwright.config.ts +++ b/cockpit/chat/debug/react/e2e/playwright.config.ts @@ -5,11 +5,14 @@ export default defineConfig({ retries: 0, fullyParallel: false, timeout: 45000, - // The first command includes cold startup within the owned worker's 10s bound. + // A real `langgraph dev` turn runs three nodes and two model calls. expect: { timeout: 15000 }, outputDir: '../../../../../output/playwright/chat-debug-react', use: { baseURL: 'http://127.0.0.1:4645', trace: 'retain-on-failure' }, projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }], + // aimock + real `langgraph dev`, shared with cockpit/chat/debug/angular. + globalSetup: './global-setup-impl.ts', + globalTeardown: require.resolve('../../../../../libs/e2e-harness/src/global-teardown'), webServer: { command: 'node scripts/react-cockpit/serve.mjs chat-debug --no-parent', cwd: '../../../../..', diff --git a/cockpit/chat/debug/react/project.json b/cockpit/chat/debug/react/project.json index b561d3749..398cc52f4 100644 --- a/cockpit/chat/debug/react/project.json +++ b/cockpit/chat/debug/react/project.json @@ -3,6 +3,10 @@ "$schema": "../../../../node_modules/nx/schemas/project-schema.json", "sourceRoot": "cockpit/chat/debug/react/src", "projectType": "application", + "implicitDependencies": [ + "cockpit-chat-debug-angular", + "cockpit-chat-debug-python" + ], "tags": [ "scope:cockpit-examples", "scope:cockpit-e2e", @@ -27,7 +31,12 @@ "react" ] }, - { "target": "react-langgraph", "projects": ["website-runtime-preparation"] } + { + "target": "react-langgraph", + "projects": [ + "website-runtime-preparation" + ] + } ], "outputs": [ "{workspaceRoot}/dist/cockpit/chat/debug/react" @@ -44,7 +53,8 @@ "cache": false, "inputs": [ "default", - "debugFixture" + "reactAimockBackend", + "debugProbe" ], "dependsOn": [ "build", @@ -62,22 +72,11 @@ } }, "namedInputs": { - "debugFixture": [ + "debugProbe": [ "{workspaceRoot}/scripts/react-cockpit/chat-debug-configuration.spec.mjs", - "{workspaceRoot}/scripts/react-cockpit/configuration.mjs", - "{workspaceRoot}/scripts/react-cockpit/serve.mjs", - "{workspaceRoot}/scripts/react-cockpit/chat-debug-fixture.mjs", - "{workspaceRoot}/scripts/react-cockpit/chat-debug-wire.py", - "{workspaceRoot}/scripts/react-cockpit/chat-debug-fixture.spec.mjs", "{workspaceRoot}/scripts/react-cockpit/chat-debug-native-session-probe.mjs", - "{workspaceRoot}/cockpit/chat/debug/python/src/graph.py", - "{workspaceRoot}/cockpit/chat/debug/python/tests/providers.py", - "{workspaceRoot}/cockpit/chat/debug/python/pyproject.toml", - "{workspaceRoot}/cockpit/chat/debug/python/uv.lock", "{workspaceRoot}/fixtures/react-parity/langgraph-candidate/chat-debug-native-session.mjs", - "{workspaceRoot}/cockpit/chat/debug/python/tests/test_streaming.py", - "{workspaceRoot}/cockpit/chat/debug/python/prompts/debug.md", - "{workspaceRoot}/cockpit/chat/debug/python/langgraph.json" + "{workspaceRoot}/cockpit/chat/debug/angular/e2e/fixtures/c-debug.json" ] } } diff --git a/cockpit/chat/generative-ui/angular/e2e/fixtures/c-generative-ui.json b/cockpit/chat/generative-ui/angular/e2e/fixtures/c-generative-ui.json index aa1de776b..371a8259c 100644 --- a/cockpit/chat/generative-ui/angular/e2e/fixtures/c-generative-ui.json +++ b/cockpit/chat/generative-ui/angular/e2e/fixtures/c-generative-ui.json @@ -27,6 +27,128 @@ } ] } + }, + { + "match": { "systemMessage": "In 3-5 words, summarize" }, + "response": { "content": "Local dashboard title" } + }, + { + "match": { + "systemMessage": "Answer the user's interpretive question", + "userMessage": "(interpreter down)" + }, + "response": { + "error": { + "message": "PRIVATE interpretation failure", + "type": "server_error" + }, + "status": 500 + } + }, + { + "match": { "systemMessage": "Answer the user's interpretive question" }, + "response": { "content": "On-time performance needs operational context to explain." } + }, + { + "match": { + "systemMessage": "Confirmed checkpoint:", + "userMessage": "(no decision)" + }, + "response": { "content": "I will just describe the dashboard." } + }, + { + "match": { + "systemMessage": "Confirmed checkpoint:", + "userMessage": "(invalid decision)" + }, + "response": { + "toolCalls": [ + { + "name": "plan_dashboard_operation", + "arguments": "{\"kind\": \"data_update\", \"calls\": [{\"name\": \"query_recent_disruptions\", \"args\": {\"type\": \"invented\"}}], \"interpretation\": \"\"}" + } + ] + } + }, + { + "match": { + "systemMessage": "Confirmed checkpoint:", + "userMessage": "(provider down)" + }, + "response": { + "error": { + "message": "PRIVATE decision failure", + "type": "server_error" + }, + "status": 500 + } + }, + { + "match": { + "systemMessage": "Confirmed checkpoint:", + "userMessage": "cancelled" + }, + "response": { + "toolCalls": [ + { + "name": "plan_dashboard_operation", + "arguments": "{\"kind\": \"data_update\", \"calls\": [{\"name\": \"query_recent_disruptions\", \"args\": {\"limit\": 5, \"type\": \"cancelled\"}}], \"interpretation\": \"\"}" + } + ] + } + }, + { + "match": { + "systemMessage": "Confirmed checkpoint:", + "userMessage": "Remove the" + }, + "response": { + "toolCalls": [ + { + "name": "plan_dashboard_operation", + "arguments": "{\"kind\": \"restructure\", \"calls\": [{\"name\": \"render_spec\", \"args\": {\"elements\": {\"root\": {\"type\": \"dashboard_grid\", \"children\": [\"stats_row\", \"charts_row\"]}, \"stats_row\": {\"type\": \"container\", \"props\": {\"direction\": \"row\"}, \"children\": [\"on_time_card\", \"flights_card\", \"delay_card\", \"load_card\"]}, \"on_time_card\": {\"type\": \"stat_card\", \"props\": {\"label\": \"On-time %\", \"value\": {\"$state\": \"/on_time/value\"}, \"delta\": {\"$state\": \"/on_time/delta\"}}}, \"flights_card\": {\"type\": \"stat_card\", \"props\": {\"label\": \"Flights Today\", \"value\": {\"$state\": \"/flights_today/value\"}, \"delta\": {\"$state\": \"/flights_today/delta\"}}}, \"delay_card\": {\"type\": \"stat_card\", \"props\": {\"label\": \"Avg Delay\", \"value\": {\"$state\": \"/avg_delay/value\"}, \"delta\": {\"$state\": \"/avg_delay/delta\"}}}, \"load_card\": {\"type\": \"stat_card\", \"props\": {\"label\": \"Load Factor\", \"value\": {\"$state\": \"/load_factor/value\"}, \"delta\": {\"$state\": \"/load_factor/delta\"}}}, \"charts_row\": {\"type\": \"container\", \"props\": {\"direction\": \"row\"}, \"children\": [\"trend_chart\", \"airline_chart\"]}, \"trend_chart\": {\"type\": \"line_chart\", \"props\": {\"title\": \"On-time % Trend\", \"data\": {\"$state\": \"/on_time_trend\"}, \"xKey\": \"month\", \"yKey\": \"on_time_pct\"}}, \"airline_chart\": {\"type\": \"bar_chart\", \"props\": {\"title\": \"Flights by Airline\", \"data\": {\"$state\": \"/flights_by_airline\"}, \"labelKey\": \"airline\", \"valueKey\": \"count\"}}}, \"root\": \"root\"}}], \"interpretation\": \"\"}" + } + ] + } + }, + { + "match": { + "systemMessage": "Confirmed checkpoint:", + "userMessage": "Why is on-time" + }, + "response": { + "toolCalls": [ + { + "name": "plan_dashboard_operation", + "arguments": "{\"kind\": \"interpret\", \"calls\": [], \"interpretation\": \"why is on-time performance changing?\"}" + } + ] + } + }, + { + "match": { + "systemMessage": "Confirmed checkpoint:", + "userMessage": "Show another dashboard" + }, + "response": { + "toolCalls": [ + { + "name": "plan_dashboard_operation", + "arguments": "{\"kind\": \"restructure\", \"calls\": [{\"name\": \"render_spec\", \"args\": {\"elements\": {\"root\": {\"type\": \"dashboard_grid\", \"children\": [\"stats_row\", \"charts_row\", \"table_section\"]}, \"stats_row\": {\"type\": \"container\", \"props\": {\"direction\": \"row\"}, \"children\": [\"on_time_card\", \"flights_card\", \"delay_card\", \"load_card\"]}, \"on_time_card\": {\"type\": \"stat_card\", \"props\": {\"label\": \"On-time %\", \"value\": {\"$state\": \"/on_time/value\"}, \"delta\": {\"$state\": \"/on_time/delta\"}}}, \"flights_card\": {\"type\": \"stat_card\", \"props\": {\"label\": \"Flights Today\", \"value\": {\"$state\": \"/flights_today/value\"}, \"delta\": {\"$state\": \"/flights_today/delta\"}}}, \"delay_card\": {\"type\": \"stat_card\", \"props\": {\"label\": \"Avg Delay\", \"value\": {\"$state\": \"/avg_delay/value\"}, \"delta\": {\"$state\": \"/avg_delay/delta\"}}}, \"load_card\": {\"type\": \"stat_card\", \"props\": {\"label\": \"Load Factor\", \"value\": {\"$state\": \"/load_factor/value\"}, \"delta\": {\"$state\": \"/load_factor/delta\"}}}, \"charts_row\": {\"type\": \"container\", \"props\": {\"direction\": \"row\"}, \"children\": [\"trend_chart\", \"airline_chart\"]}, \"trend_chart\": {\"type\": \"line_chart\", \"props\": {\"title\": \"On-time % Trend\", \"data\": {\"$state\": \"/on_time_trend\"}, \"xKey\": \"month\", \"yKey\": \"on_time_pct\"}}, \"airline_chart\": {\"type\": \"bar_chart\", \"props\": {\"title\": \"Flights by Airline\", \"data\": {\"$state\": \"/flights_by_airline\"}, \"labelKey\": \"airline\", \"valueKey\": \"count\"}}, \"table_section\": {\"type\": \"data_grid\", \"props\": {\"title\": \"Recent Disruptions\", \"rows\": {\"$state\": \"/recent_disruptions\"}, \"columns\": [\"flight_number\", \"type\", \"minutes\", \"route\", \"date\"]}}}, \"root\": \"root\"}}], \"interpretation\": \"\"}" + } + ] + } + }, + { + "match": { "systemMessage": "Confirmed checkpoint:" }, + "response": { + "toolCalls": [ + { + "name": "plan_dashboard_operation", + "arguments": "{\"kind\": \"create\", \"calls\": [{\"name\": \"render_spec\", \"args\": {\"elements\": {\"root\": {\"type\": \"dashboard_grid\", \"children\": [\"stats_row\", \"charts_row\", \"table_section\"]}, \"stats_row\": {\"type\": \"container\", \"props\": {\"direction\": \"row\"}, \"children\": [\"on_time_card\", \"flights_card\", \"delay_card\", \"load_card\"]}, \"on_time_card\": {\"type\": \"stat_card\", \"props\": {\"label\": \"On-time %\", \"value\": {\"$state\": \"/on_time/value\"}, \"delta\": {\"$state\": \"/on_time/delta\"}}}, \"flights_card\": {\"type\": \"stat_card\", \"props\": {\"label\": \"Flights Today\", \"value\": {\"$state\": \"/flights_today/value\"}, \"delta\": {\"$state\": \"/flights_today/delta\"}}}, \"delay_card\": {\"type\": \"stat_card\", \"props\": {\"label\": \"Avg Delay\", \"value\": {\"$state\": \"/avg_delay/value\"}, \"delta\": {\"$state\": \"/avg_delay/delta\"}}}, \"load_card\": {\"type\": \"stat_card\", \"props\": {\"label\": \"Load Factor\", \"value\": {\"$state\": \"/load_factor/value\"}, \"delta\": {\"$state\": \"/load_factor/delta\"}}}, \"charts_row\": {\"type\": \"container\", \"props\": {\"direction\": \"row\"}, \"children\": [\"trend_chart\", \"airline_chart\"]}, \"trend_chart\": {\"type\": \"line_chart\", \"props\": {\"title\": \"On-time % Trend\", \"data\": {\"$state\": \"/on_time_trend\"}, \"xKey\": \"month\", \"yKey\": \"on_time_pct\"}}, \"airline_chart\": {\"type\": \"bar_chart\", \"props\": {\"title\": \"Flights by Airline\", \"data\": {\"$state\": \"/flights_by_airline\"}, \"labelKey\": \"airline\", \"valueKey\": \"count\"}}, \"table_section\": {\"type\": \"data_grid\", \"props\": {\"title\": \"Recent Disruptions\", \"rows\": {\"$state\": \"/recent_disruptions\"}, \"columns\": [\"flight_number\", \"type\", \"minutes\", \"route\", \"date\"]}}}, \"root\": \"root\"}}, {\"name\": \"query_airline_kpis\", \"args\": {}}, {\"name\": \"query_on_time_trend\", \"args\": {}}, {\"name\": \"query_flights_by_airline\", \"args\": {}}, {\"name\": \"query_recent_disruptions\", \"args\": {}}], \"interpretation\": \"\"}" + } + ] + } } ] } diff --git a/cockpit/chat/generative-ui/react/e2e/generative-ui.spec.ts b/cockpit/chat/generative-ui/react/e2e/generative-ui.spec.ts index f5504caab..80e6eb987 100644 --- a/cockpit/chat/generative-ui/react/e2e/generative-ui.spec.ts +++ b/cockpit/chat/generative-ui/react/e2e/generative-ui.spec.ts @@ -4,19 +4,31 @@ import { type Page, type APIRequestContext, } from '@playwright/test'; -import { createHash } from 'node:crypto'; -import { readFileSync, writeFileSync } from 'node:fs'; -import { resolve } from 'node:path'; +import { writeFileSync } from 'node:fs'; +// Backend: aimock replays cockpit/chat/generative-ui/angular/e2e/fixtures (the +// decision, interpretation and title models, keyed on their system prompts) +// into a real `langgraph dev`, so admission, the real dashboard tools, +// assessment and publication all run in the graph. The recording proxy +// (scripts/react-cockpit/langgraph-proxy.mjs) holds, fails or rewrites reads. +// +// Faults inside the graph that no model output can reach are covered by the +// Python tests in cockpit/chat/generative-ui/python/tests/test_streaming.py, +// not here: a throwing tool or malformed tool data rolling back the whole +// batch (test_malformed_failed_missing_or_uncorrelated_results_roll_back_entire_batch, +// test_mixed_genuine_success_and_runtime_error_is_atomic), and prose already +// on the action parent (test_genuine_nonempty_parent_fallback_no_wrapper_or_data). +// Admission validates every query argument and binding before execution, and +// the parent it creates is always empty. const base = '/chat/generative-ui/react/'; const box = (page: Page) => page.getByRole('textbox', { name: 'Message', exact: true }); const boards = (page: Page) => page.locator('[data-dashboard-spec-owner]'); const requests = async (request: APIRequestContext) => (await request.get('/__requests')).json(); -const configure = async ( - request: APIRequestContext, - data: Record -) => expect((await request.post('/__configure', { data })).status()).toBe(200); +const control = async (request: APIRequestContext, path: string) => + expect((await request.post(path)).status()).toBe(200); +const lifetime = async (request: APIRequestContext) => + (await request.get('/__lifetime')).json(); async function send(page: Page, text: string) { await box(page).fill(text); await page.getByRole('button', { name: 'Send', exact: true }).click(); @@ -27,8 +39,6 @@ async function send(page: Page, text: string) { const errors = new WeakMap(); test.beforeEach(async ({ page, request }) => { expect((await request.post('/__reset')).status()).toBe(200); - // Prepare the real graph before UI assertions, without creating a conversation. - expect((await request.post('/__configure', { data: { failAgent: false } })).status()).toBe(200); const list: string[] = []; errors.set(page, list); page.on('pageerror', (e) => list.push(e.message)); @@ -38,39 +48,8 @@ test.beforeEach(async ({ page, request }) => { }); test.afterEach(async ({ page, request }) => { expect(errors.get(page)).toEqual([]); - const proofs = await (await request.get('/__graph-proof')).json(); - for (const proof of proofs) - expect(proof).toMatchObject({ - actualCompiledGraph: true, - networkConnectAttempts: 0, - titleMessageCallbacks: 0, - decisionMessageCallbacks: 0, - interpretationMessageCallbacks: 0, - productionSourceSha256: Object.fromEntries( - [ - 'src/graph.py', - 'src/dashboard_tools.py', - 'src/operations.py', - 'src/dashboard_contract.py', - 'prompts/generative-ui.md', - ].map((path) => [ - path, - createHash('sha256') - .update(readFileSync(resolve(__dirname, '../../python/' + path))) - .digest('hex'), - ]) - ), - sourceSha256: createHash('sha256') - .update(readFileSync(resolve(__dirname, '../../python/src/graph.py'))) - .digest('hex'), - lockSha256: createHash('sha256') - .update(readFileSync(resolve(__dirname, '../../python/uv.lock'))) - .digest('hex'), - }); await page.goto('about:blank'); - await expect - .poll(async () => (await (await request.get('/__lifetime')).json()).held) - .toBe(0); + await expect.poll(async () => (await lifetime(request)).openResponses).toBe(0); }); test('mount is inert; four confirmed turns retain ownership and share refreshed data', async ({ page, @@ -155,12 +134,9 @@ test('mount is inert; four confirmed turns retain ownership and share refreshed captured.filter((r: { path: string }) => r.path === '/api/threads') ).toHaveLength(1); writeFileSync( - testInfo.outputPath('graph-proof.json'), + testInfo.outputPath('wire.json'), JSON.stringify( - { - requests: captured, - proofs: await (await request.get('/__graph-proof')).json(), - }, + { requests: captured, captures: await (await request.get('/__captures')).json() }, null, 2 ) @@ -176,35 +152,6 @@ test('mount is inert; four confirmed turns retain ownership and share refreshed fullPage: true, }); }); -test('nonempty parent preserves prose with inspectable raw result and notice', async ({ - page, - request, -}) => { - await page.goto(base); - await send(page, 'prose render'); - await expect(boards(page)).toHaveCount(0); - await expect(page.getByText('Here is the layout.')).toBeVisible(); - await expect( - page.getByText( - 'This layout was not applied because the assistant already supplied prose. The tool result is available for inspection.' - ) - ).toBeVisible(); - await page.locator('details').filter({ has: page.getByText('Server tool result', { exact: true }) }).locator('summary').first().click(); - await expect( - page - .locator('details[open] > pre') - .filter({ hasText: 'dashboard_grid' }) - .first() - ).toBeVisible(); - expect(await requests(request)).toHaveLength(5); - const proofs = await (await request.get('/__graph-proof')).json(); - expect( - proofs.some( - (p: { op: string; fixtureControls: { prose_parent: boolean } }) => - p.op === 'submit' && p.fixtureControls.prose_parent - ) - ).toBe(true); -}); for (const decisionFault of ['missing', 'invalid']) test(`saved ${decisionFault} decision honestly leaves the prior board unchanged`, async ({ page, @@ -214,8 +161,8 @@ for (const decisionFault of ['missing', 'invalid']) await send(page, 'Show dashboard'); const old = await boards(page).innerText(); const owner = await boards(page).getAttribute('data-dashboard-spec-owner'); - await configure(request, { decisionFault }); - await send(page, 'Filter to cancelled flights'); + // Real model output that admission rejects: no decision, or an invented filter. + await send(page, `Filter to cancelled flights (${decisionFault === 'missing' ? 'no' : 'invalid'} decision)`); await expect(boards(page)).toHaveCount(1); expect(await boards(page).innerText()).toBe(old); expect(await boards(page).getAttribute('data-dashboard-spec-owner')).toBe( @@ -233,47 +180,6 @@ for (const decisionFault of ['missing', 'invalid']) ) ).toHaveLength(2); }); -for (const toolFault of ['failKpis', 'malformedTrend']) - test(`real ${toolFault} batch keeps successful peers inspectable but cannot replace the confirmed board`, async ({ - page, - request, - }) => { - await configure(request, { partialDashboard: true }); - await page.goto(base); - await send(page, 'Show dashboard'); - const old = await boards(page).innerText(); - const owner = await boards(page).getAttribute('data-dashboard-spec-owner'); - await configure(request, { partialDashboard: false, [toolFault]: true }); - await box(page).fill('Show another dashboard'); - await box(page).press('Enter'); - await expect(page.getByRole('alert')).toContainText( - 'could not be confirmed' - ); - await expect(boards(page)).toHaveCount(1); - expect(await boards(page).innerText()).toBe(old); - expect(await boards(page).getAttribute('data-dashboard-spec-owner')).toBe( - owner - ); - await expect(box(page)).toBeDisabled(); - await expect( - page.getByText('Start a new conversation to continue.', { exact: true }) - ).toBeVisible(); - await page.locator('details').filter({ has: page.getByText('Server tool result', { exact: true }) }).locator('summary').last().click(); - await expect( - page - .locator('details[open] > pre') - .filter({ hasText: 'on_time_pct' }) - .last() - ).toBeVisible(); - expect( - (await requests(request)).filter((r: { path: string }) => - r.path.endsWith('/runs/stream') - ) - ).toHaveLength(2); - await page.getByRole('button', { name: 'New conversation' }).click(); - await expect(boards(page)).toHaveCount(0); - await expect(box(page)).toBeEnabled(); - }); test('failed read-only interpretation retains the confirmed board and requires New', async ({ page, request, @@ -281,8 +187,8 @@ test('failed read-only interpretation retains the confirmed board and requires N await page.goto(base); await send(page, 'Show dashboard'); const old = await boards(page).innerText(); - await configure(request, { failInterpretation: true }); - await box(page).fill('Why is on-time low?'); + // The interpretation model fails for this prompt (fixture). + await box(page).fill('Why is on-time low? (interpreter down)'); await box(page).press('Enter'); await expect(page.getByRole('alert')).toBeVisible(); await expect(boards(page)).toHaveCount(1); @@ -301,14 +207,16 @@ test('pending update keeps the old board; double submit, Stop and New cannot pub await page.goto(base); await send(page, 'Show dashboard'); const old = await boards(page).innerText(); - await configure(request, { holdStream: true }); + // The decision is not streamed and a data update writes no model text: + // hold the run right after its first event. + expect((await request.post('/__hold/stream', { data: { immediate: true } })).status()).toBe(200); await box(page).fill('Filter to only the cancelled flights.'); await box(page).press('Enter'); await box(page).press('Enter'); await expect(box(page)).toHaveValue(''); await expect - .poll(async () => (await (await request.get('/__lifetime')).json()).held) - .toBe(1); + .poll(async () => (await lifetime(request)).activeStream) + .toBe(true); expect(await boards(page).innerText()).toBe(old); expect( (await requests(request)).filter((r: { path: string }) => @@ -323,17 +231,17 @@ test('pending update keeps the old board; double submit, Stop and New cannot pub await request.post('/__release'); await page.getByRole('button', { name: 'New conversation' }).click(); await expect(boards(page)).toHaveCount(0); - await configure(request, { holdStream: false, holdState: true }); + // The confirmation reads the exact checkpoint. + await control(request, '/__hold/state'); await box(page).fill('New dashboard'); await box(page).press('Enter'); await expect - .poll(async () => (await (await request.get('/__lifetime')).json()).held) - .toBe(1); + .poll(async () => (await lifetime(request)).activeState) + .toBe(true); await page.getByRole('button', { name: 'New conversation' }).click(); - await request.post('/__release'); + await request.post('/__release-state'); await expect(boards(page)).toHaveCount(0); await expect(box(page)).toBeEnabled(); - await configure(request, { holdState: false }); await send(page, 'Fresh dashboard'); await expect(boards(page)).toHaveCount(1); expect( @@ -355,7 +263,8 @@ for (const fault of [ await page.goto(base); await send(page, 'Show dashboard'); const old = await boards(page).innerText(); - await configure(request, { stateFault: fault }); + // The confirmation's checkpoint read is rewritten (injected data). + await control(request, '/__state/' + (fault === 'pending-tasks' ? 'unsafe' : fault)); await box(page).fill('Show another dashboard'); await box(page).press('Enter'); await expect(page.getByRole('alert')).toContainText( @@ -370,13 +279,12 @@ for (const fault of [ ) ).toHaveLength(2); }); -for (const options of [{ runStatus: 'error' }, { failAgent: true }]) - test(`run failure ${JSON.stringify( - options - )} leaves no provisional layout`, async ({ page, request }) => { +for (const failure of ['run status error', 'decision provider down']) + test(`run failure (${failure}) leaves no provisional layout`, async ({ page, request }) => { await page.goto(base); - await configure(request, options); - await box(page).fill('Show dashboard'); + // The run reports status error; or the decision model fails (fixture). + if (failure === 'run status error') await control(request, '/__run-status/error'); + await box(page).fill(failure === 'run status error' ? 'Show dashboard' : 'Show dashboard (provider down)'); await box(page).press('Enter'); await expect(page.getByRole('alert')).toBeVisible(); await expect(boards(page)).toHaveCount(0); diff --git a/cockpit/chat/generative-ui/react/e2e/global-setup-impl.ts b/cockpit/chat/generative-ui/react/e2e/global-setup-impl.ts new file mode 100644 index 000000000..9bd41ff6d --- /dev/null +++ b/cockpit/chat/generative-ui/react/e2e/global-setup-impl.ts @@ -0,0 +1,6 @@ +// eslint-disable-next-line @nx/enforce-module-boundaries -- Shared aimock setup for the React cockpit examples. +import { reactAimockGlobalSetup } from '../../../../shared/react-aimock-setup'; + +// aimock replays the Angular example's fixtures into a real `langgraph dev`; +// the static server (webServer) proxies /api to it. +export default reactAimockGlobalSetup({ example: 'chat/generative-ui', port: 4627 }); diff --git a/cockpit/chat/generative-ui/react/e2e/playwright.config.ts b/cockpit/chat/generative-ui/react/e2e/playwright.config.ts index 531cb9f5c..38f7b2e59 100644 --- a/cockpit/chat/generative-ui/react/e2e/playwright.config.ts +++ b/cockpit/chat/generative-ui/react/e2e/playwright.config.ts @@ -6,6 +6,9 @@ export default defineConfig({ timeout: 30000, use: { baseURL: 'http://127.0.0.1:4627', trace: 'retain-on-failure' }, projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }], + // aimock + real `langgraph dev`, shared with cockpit/chat/generative-ui/angular. + globalSetup: './global-setup-impl.ts', + globalTeardown: require.resolve('../../../../../libs/e2e-harness/src/global-teardown'), webServer: { command: 'node scripts/react-cockpit/serve.mjs chat-generative-ui --no-parent', diff --git a/cockpit/chat/generative-ui/react/project.json b/cockpit/chat/generative-ui/react/project.json index cec3d64fc..00e58a2ec 100644 --- a/cockpit/chat/generative-ui/react/project.json +++ b/cockpit/chat/generative-ui/react/project.json @@ -3,6 +3,7 @@ "$schema": "../../../../node_modules/nx/schemas/project-schema.json", "sourceRoot": "cockpit/chat/generative-ui/react/src", "projectType": "application", + "implicitDependencies": ["cockpit-chat-generative-ui-angular", "cockpit-chat-generative-ui-python"], "tags": ["scope:cockpit-examples", "scope:cockpit-e2e", "framework:react"], "targets": { "build": { @@ -32,24 +33,12 @@ "e2e": { "executor": "@nx/playwright:playwright", "cache": false, - "dependsOn": ["build", "fixture-test"], + "inputs": ["default", "reactAimockBackend"], + "dependsOn": ["build", { "target": "test", "projects": ["cockpit-chat-generative-ui-python"] }], "options": { "config": "cockpit/chat/generative-ui/react/e2e/playwright.config.ts", "skipInstall": true } - }, - "fixture-test": { - "executor": "nx:run-commands", - "cache": false, - "dependsOn": [ - { - "target": "test", - "projects": ["cockpit-chat-generative-ui-python"] - } - ], - "options": { - "command": "node --test scripts/react-cockpit/chat-generative-ui-fixture.spec.mjs" - } } } } diff --git a/cockpit/chat/generative-ui/react/src/connection.integration.spec.ts b/cockpit/chat/generative-ui/react/src/connection.integration.spec.ts index 444628199..4a431402b 100644 --- a/cockpit/chat/generative-ui/react/src/connection.integration.spec.ts +++ b/cockpit/chat/generative-ui/react/src/connection.integration.spec.ts @@ -1,25 +1,40 @@ -import { createServer } from 'node:http'; +import { createServer, type Server } from 'node:http'; import { once } from 'node:events'; import type { AddressInfo } from 'node:net'; -import { expect, it, vi, onTestFinished } from 'vitest'; +import { resolve } from 'node:path'; +import { afterAll, beforeAll, expect, it, vi, onTestFinished } from 'vitest'; import { createConnectedApplication } from './connection'; + +/* eslint-disable @nx/enforce-module-boundaries -- Local test backend tooling is never bundled. */ +// @ts-expect-error Local test-only JavaScript has no published declaration. +import { createAimockBackend } from '../../../../../scripts/react-cockpit/aimock-backend.mjs'; +// @ts-expect-error Local test-only JavaScript has no published declaration. +import { createLanggraphProxy } from '../../../../../scripts/react-cockpit/langgraph-proxy.mjs'; +// @ts-expect-error Local test-only JavaScript has no published declaration. +import { reactCockpitConfiguration } from '../../../../../scripts/react-cockpit/configuration.mjs'; +/* eslint-enable @nx/enforce-module-boundaries */ + +// The real graph runs under `langgraph dev` on aimock fixtures shared with the +// Angular example (cockpit/chat/generative-ui/angular/e2e/fixtures); the +// in-process proxy records requests and holds or fails the next call +// (scripts/react-cockpit/langgraph-proxy.mjs). Faults no model output can +// reach (a throwing tool, malformed tool data, prose on the action parent) +// are covered by cockpit/chat/generative-ui/python/tests/test_streaming.py: +// test_malformed_failed_missing_or_uncorrelated_results_roll_back_entire_batch, +// test_mixed_genuine_success_and_runtime_error_is_atomic and +// test_genuine_nonempty_parent_fallback_no_wrapper_or_data. +const { backend: config } = reactCockpitConfiguration('chat-generative-ui'); +const backend = createAimockBackend({ root: resolve(__dirname, '../../../../..'), backend: config }); +beforeAll(() => backend.ensure(), 180000); +afterAll(() => backend.close()); +type Handler = (request: unknown, response: unknown, pathname: string) => Promise; async function fixture() { - const location = new URL( - '../../../../../scripts/react-cockpit/chat-generative-ui-fixture.mjs', - import.meta.url - ).href; - const { createChatGenerativeUiFixture } = await import( - /* @vite-ignore */ location - ); - const handle = createChatGenerativeUiFixture(); - const server = createServer(async (req, res) => { - if ( - !(await handle( - req, - res, - new URL(req.url ?? '', 'http://localhost').pathname - )) - ) { + const handle: Handler = createLanggraphProxy({ + upstream: `http://127.0.0.1:${config.langgraphPort}`, + ready: () => backend.ensure(), + }); + const server: Server = createServer(async (req, res) => { + if (!(await handle(req, res, new URL(req.url ?? '', 'http://localhost').pathname))) { res.writeHead(404); res.end(); } @@ -33,18 +48,12 @@ async function fixture() { }); onTestFinished(async () => { await app.dispose(); - await handle.close(); server.closeAllConnections(); await new Promise((r) => server.close(() => r())); }); - const control = async (body: object) => { + const control = async (path: string, body?: object) => { expect( - ( - await fetch(base + '/__configure', { - method: 'POST', - body: JSON.stringify(body), - }) - ).status + (await fetch(base + path, { method: 'POST', ...(body ? { body: JSON.stringify(body) } : {}) })).status ).toBe(200); }; const requests = async () => @@ -93,24 +102,16 @@ it('actual default session confirms initial, cancelled filter, structural and pr requests.filter((r) => r.path.endsWith('/state/checkpoint')) ).toHaveLength(4); }, 60000); -it('actual nonempty-parent render keeps prose and raw result without mounting a new layout', async () => { - const f = await fixture(); - expect(await f.app.submit('Show dashboard')).toBe(true); - const prior = f.app.getSnapshot(); - expect(await f.app.submit('prose render')).toBe(true); - expect(f.app.getSnapshot().surfaces).toEqual(prior.surfaces); - expect(f.app.getSnapshot().notices).toHaveLength(1); - const last = f.app.getSnapshot().toolCalls.at(-1); - expect(last?.status === 'complete' && last.result).not.toBe('rendered'); -}, 60000); it.each(['missing', 'invalid'])( 'actual %s decision confirms an honest saved no-op with the prior dashboard', async (decisionFault) => { const f = await fixture(); expect(await f.app.submit('Show dashboard')).toBe(true); const prior = f.app.getSnapshot(); - await f.control({ decisionFault }); - expect(await f.app.submit('Filter to cancelled flights')).toBe(true); + // Real model output that admission rejects (fixtures). + expect( + await f.app.submit(`Filter to cancelled flights (${decisionFault === 'missing' ? 'no' : 'invalid'} decision)`) + ).toBe(true); const saved = f.app.getSnapshot(); expect(saved.dashboard).toEqual(prior.dashboard); expect(saved.surfaces).toEqual(prior.surfaces); @@ -125,47 +126,12 @@ it.each(['missing', 'invalid'])( }, 60000 ); -it.each(['failKpis', 'malformedTrend'])( - 'actual %s structural rollback stays unconfirmed with real successful peers', - async (toolFault) => { - const f = await fixture(); - await f.control({ partialDashboard: true }); - expect(await f.app.submit('Show dashboard')).toBe(true); - const prior = f.app.getSnapshot(); - expect(Object.keys(prior.dashboard).sort()).toEqual([ - 'flights_by_airline', - 'recent_disruptions', - ]); - await f.control({ partialDashboard: false, [toolFault]: true }); - expect(await f.app.submit('Show another dashboard')).toBe(false); - const failed = f.app.getSnapshot(); - expect(failed.dashboard).toBe(prior.dashboard); - expect(failed.surfaces).toBe(prior.surfaces); - expect(failed.canSubmit).toBe(false); - const render = failed.toolCalls - .filter((c) => c.name === 'render_spec') - .at(-1); - expect(render?.status === 'complete' && render.result).not.toBe('rendered'); - const peer = failed.toolCalls.find((c) => c.name === 'query_on_time_trend'); - expect(peer?.status).toBe('complete'); - const count = (await f.requests()).length; - expect(await f.app.submit('Try again')).toBe(false); - expect(await f.requests()).toHaveLength(count); - await f.app.newConversation(); - expect(f.app.getSnapshot()).toMatchObject({ - canSubmit: true, - surfaces: [], - dashboard: {}, - }); - }, - 60000 -); it('actual failed read-only interpretation retains the confirmed dashboard and requires New', async () => { const f = await fixture(); expect(await f.app.submit('Show dashboard')).toBe(true); const prior = f.app.getSnapshot(); - await f.control({ failInterpretation: true }); - expect(await f.app.submit('Why is on-time low?')).toBe(false); + // The interpretation model fails for this prompt (fixture). + expect(await f.app.submit('Why is on-time low? (interpreter down)')).toBe(false); expect(f.app.getSnapshot().dashboard).toBe(prior.dashboard); expect(f.app.getSnapshot().surfaces).toBe(prior.surfaces); expect(f.app.getSnapshot().canSubmit).toBe(false); @@ -177,13 +143,14 @@ it('actual pending task rejects confirmation and Stop preserves the confirmed bo const f = await fixture(); expect(await f.app.submit('Show dashboard')).toBe(true); const prior = f.app.getSnapshot(); - await f.control({ stateFault: 'pending-tasks' }); + await f.control('/__state/unsafe'); expect(await f.app.submit('Filter to cancelled flights')).toBe(false); expect(f.app.getSnapshot().surfaces).toBe(prior.surfaces); expect(f.app.getSnapshot().dashboard).toBe(prior.dashboard); expect(f.app.getSnapshot().canSubmit).toBe(false); await f.app.newConversation(); - await f.control({ stateFault: null, holdStream: true }); + await f.control('/__state/safe'); + await f.control('/__hold/stream', { immediate: true }); const pending = f.app.submit('Show dashboard'); await vi.waitFor(async () => expect( @@ -214,7 +181,8 @@ it.each([ const f = await fixture(); expect(await f.app.submit('Show dashboard')).toBe(true); const prior = f.app.getSnapshot(); - await f.control({ stateFault: fault }); + // The confirmation's checkpoint read is rewritten (injected data). + await f.control('/__state/' + fault); expect( await f.app.submit( fault === 'malformed-layout' @@ -235,7 +203,7 @@ it.each(['running', 'error'])( 'actual independent exact run status %s cannot grant confirmation', async (runStatus) => { const f = await fixture(); - await f.control({ runStatus }); + await f.control('/__run-status/' + runStatus); expect(await f.app.submit('Show dashboard')).toBe(false); expect(f.app.getSnapshot()).toMatchObject({ canSubmit: false, diff --git a/cockpit/chat/input/angular/e2e/fixtures/c-input.json b/cockpit/chat/input/angular/e2e/fixtures/c-input.json index 2f2a7c77c..53d4e9bc3 100644 --- a/cockpit/chat/input/angular/e2e/fixtures/c-input.json +++ b/cockpit/chat/input/angular/e2e/fixtures/c-input.json @@ -1,10 +1,24 @@ { "fixtures": [ + { + "match": { "systemMessage": "In 3-5 words, summarize" }, + "response": { "content": "Aviation snippet title" } + }, { "match": { "userMessage": "Hello" }, "response": { "content": "Hi! I'm the chat-input demo. Try Enter to send, Shift+Enter for newline, and watch the sidebar reflect the streaming status." } + }, + { + "match": { "userMessage": "Stream a TypeScript code fence" }, + "response": { "content": "Here is the snippet:\n\n```typescript\nconst answer = 42;\n```\n\nThe constant is available for later use." }, + "chunkSize": 3, + "latency": 25 + }, + { + "match": { "userMessage": "" }, + "response": { "content": "Saved answer for the chat-input demo." } } ] } diff --git a/cockpit/chat/input/react/e2e/global-setup-impl.ts b/cockpit/chat/input/react/e2e/global-setup-impl.ts new file mode 100644 index 000000000..6627775a1 --- /dev/null +++ b/cockpit/chat/input/react/e2e/global-setup-impl.ts @@ -0,0 +1,6 @@ +// eslint-disable-next-line @nx/enforce-module-boundaries -- Shared aimock setup for the React cockpit examples. +import { reactAimockGlobalSetup } from '../../../../shared/react-aimock-setup'; + +// aimock replays the Angular example's fixtures into a real `langgraph dev`; +// the static server (webServer) proxies /api to it. +export default reactAimockGlobalSetup({ example: 'chat/input', port: 4621 }); diff --git a/cockpit/chat/input/react/e2e/input.spec.ts b/cockpit/chat/input/react/e2e/input.spec.ts index f5b2c13dc..5abd6e88c 100644 --- a/cockpit/chat/input/react/e2e/input.spec.ts +++ b/cockpit/chat/input/react/e2e/input.spec.ts @@ -1,4 +1,9 @@ import { test, expect, type Page } from '@playwright/test'; +// Backend: aimock replays the Angular example's e2e fixtures (shared) into a +// real `langgraph dev`; the React static server proxies /api to it, recording +// requests and injecting holds and failures (scripts/react-cockpit/langgraph-proxy.mjs). +const FENCE = 'Stream a TypeScript code fence'; +const HELLO = 'Hello'; type Wire = { path: string; method: string; @@ -24,18 +29,18 @@ async function send(page: Page, text: string) { const conversation = (page: Page) => page.getByRole('region', { name: 'Conversation', exact: true }); -test('installed runtime renders compiled graph code fences, isolates titles and confirms each continued turn', async ({ +test('real graph renders code fences, isolates titles and confirms each continued turn', async ({ page, request, }) => { await page.goto('/chat/input/react/'); await expect(page.getByRole('status')).toHaveText('Ready.'); expect(await (await request.get('/__requests')).json()).toEqual([]); - await send(page, 'Explain the aviation example'); + await send(page, FENCE); await expect(conversation(page).locator('pre code')).toHaveText( 'const answer = 42;' ); - await expect(conversation(page)).not.toContainText('Authored metadata title'); + await expect(conversation(page)).not.toContainText('Aviation snippet title'); await send(page, 'Continue with the same conversation'); await expect(conversation(page).locator('article')).toHaveCount(4); const wire: Wire[] = await (await request.get('/__requests')).json(); @@ -49,14 +54,6 @@ test('installed runtime renders compiled graph code fences, isolates titles and ).toBe(true); expect(wire.filter((r) => r.path.endsWith('/history'))).toHaveLength(2); expect(wire.filter((r) => r.method === 'GET')).toHaveLength(2); - const proofs = await (await request.get('/__graph-proof')).json(); - expect(proofs).toHaveLength(2); - for (const proof of proofs) - expect(proof).toMatchObject({ - actualCompiledGraph: true, - networkConnectAttempts: 0, - titleMessageCallbacks: 0, - }); await page.getByRole('button', { name: 'New conversation' }).click(); await expect(page.getByRole('status')).toHaveText('Ready.'); await expect(conversation(page).locator('article')).toHaveCount(0); @@ -88,8 +85,9 @@ test('held response Stop fences the run and explicit reset remains local', async page, request, }) => { + await request.post('/__hold/stream'); await page.goto('/chat/input/react/'); - await submit(page, 'Hold'); + await submit(page, HELLO); await expect(page.getByRole('status')).toHaveText('Receiving response…'); await page.getByRole('button', { name: 'Stop', exact: true }).click(); await expect(page.getByRole('status')).toHaveText( @@ -102,12 +100,13 @@ test('held response Stop fences the run and explicit reset remains local', async await expect(page.getByRole('status')).toHaveText('Ready.'); expect(await (await request.get('/__requests')).json()).toEqual(before); }); -test('renders actual graph output inside an unfinished code fence before release', async ({ +test('renders streamed graph output inside an unfinished code fence before release', async ({ page, request, }) => { + await request.post('/__hold/stream', { data: { afterText: 'const answer' } }); await page.goto('/chat/input/react/'); - await submit(page, 'Hold fence'); + await submit(page, FENCE); await expect(page.getByRole('status')).toHaveText('Receiving response…'); await expect(conversation(page).locator('pre code')).toContainText( 'const answer' @@ -131,8 +130,9 @@ test('pagehide closes an active installed transport and a new page never replays page, request, }) => { + await request.post('/__hold/stream'); await page.goto('/chat/input/react/'); - await submit(page, 'Hold'); + await submit(page, HELLO); await expect(page.getByRole('status')).toHaveText('Receiving response…'); expect(await (await request.get('/__lifetime')).json()).toMatchObject({ activeStream: true, @@ -323,8 +323,9 @@ for (const phase of ['stream', 'history'] as const) { request, }) => { if (phase === 'history') await request.post('/__history/hold'); + else await request.post('/__hold/stream'); await page.goto('/chat/input/react/'); - await submit(page, phase === 'stream' ? 'Hold' : 'Confirm this turn'); + await submit(page, HELLO); await expect(page.getByRole('status')).toHaveText( phase === 'stream' ? 'Receiving response…' : 'Confirming saved response…' ); @@ -370,7 +371,8 @@ test('local disabling blocks Send but preserves Stop during an active response', const disabled = page.getByRole('checkbox', { name: 'Disable message input', }); - await input.fill('Hold'); + await request.post('/__hold/stream'); + await input.fill(HELLO); await disabled.check(); await expect(input).toBeDisabled(); await expect( diff --git a/cockpit/chat/input/react/e2e/playwright.config.ts b/cockpit/chat/input/react/e2e/playwright.config.ts index 42380cadd..a20d6c7fb 100644 --- a/cockpit/chat/input/react/e2e/playwright.config.ts +++ b/cockpit/chat/input/react/e2e/playwright.config.ts @@ -5,6 +5,9 @@ export default defineConfig({ fullyParallel: false, use: { baseURL: 'http://127.0.0.1:4621', trace: 'retain-on-failure' }, projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }], + // aimock + real `langgraph dev`, shared with cockpit/chat/input/angular. + globalSetup: './global-setup-impl.ts', + globalTeardown: require.resolve('../../../../../libs/e2e-harness/src/global-teardown'), webServer: { command: 'node scripts/react-cockpit/serve.mjs chat-input', cwd: '../../../../..', diff --git a/cockpit/chat/input/react/project.json b/cockpit/chat/input/react/project.json index 09c449287..a5a5e5027 100644 --- a/cockpit/chat/input/react/project.json +++ b/cockpit/chat/input/react/project.json @@ -3,6 +3,7 @@ "$schema": "../../../../node_modules/nx/schemas/project-schema.json", "sourceRoot": "cockpit/chat/input/react/src", "projectType": "application", + "implicitDependencies": ["cockpit-chat-input-angular", "cockpit-chat-input-python"], "tags": ["scope:cockpit-examples", "scope:cockpit-e2e", "framework:react"], "targets": { "build": { @@ -32,6 +33,7 @@ "e2e": { "executor": "@nx/playwright:playwright", "cache": false, + "inputs": ["default", "reactAimockBackend"], "dependsOn": [ "build", { diff --git a/cockpit/chat/interrupts/angular/e2e/fixtures/c-interrupts.json b/cockpit/chat/interrupts/angular/e2e/fixtures/c-interrupts.json index fd80b43cd..c10bfce28 100644 --- a/cockpit/chat/interrupts/angular/e2e/fixtures/c-interrupts.json +++ b/cockpit/chat/interrupts/angular/e2e/fixtures/c-interrupts.json @@ -1,5 +1,26 @@ { "fixtures": [ + { + "match": { + "systemMessage": "In 3-5 words, summarize", + "userMessage": "(title fails)" + }, + "response": { + "error": { + "message": "PRIVATE title model failure", + "type": "server_error" + }, + "status": 500 + } + }, + { + "match": { + "systemMessage": "In 3-5 words, summarize" + }, + "response": { + "content": "Aviation snippet title" + } + }, { "match": { "userMessage": "Book me on UA123.", @@ -71,6 +92,172 @@ "systemHash": "18e3aede", "toolsHash": "3644d4ff" } + }, + { + "match": { + "toolCallId": "ic-padded-0" + }, + "response": { + "content": "Your booking request is complete." + } + }, + { + "match": { + "toolCallId": "ic-aa404-0" + }, + "response": { + "content": "Your booking request is complete." + } + }, + { + "match": { + "toolCallId": "ic-ua123-0" + }, + "response": { + "content": "Your booking request is complete." + } + }, + { + "match": { + "toolCallId": "ic-read-0" + }, + "response": { + "toolCalls": [ + { + "name": "book_flight", + "arguments": "{\"flight_number\": \"UA123\"}", + "id": "ic-read-1" + } + ] + } + }, + { + "match": { + "toolCallId": "ic-read-1" + }, + "response": { + "content": "Your booking request is complete." + } + }, + { + "match": { + "toolCallId": "ic-unknown-0" + }, + "response": { + "content": "I could not find that flight, so nothing was booked." + } + }, + { + "match": { + "toolCallId": "ic-multi-1" + }, + "response": { + "content": "Your booking request is complete." + } + }, + { + "match": { + "userMessage": "Book padded flight.", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "book_flight", + "arguments": "{\"flight_number\": \" ua123 \"}", + "id": "ic-padded-0" + } + ] + } + }, + { + "match": { + "userMessage": "Book AA404.", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "book_flight", + "arguments": "{\"flight_number\": \"AA404\"}", + "id": "ic-aa404-0" + } + ] + } + }, + { + "match": { + "userMessage": "Book UA123.", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "book_flight", + "arguments": "{\"flight_number\": \"UA123\"}", + "id": "ic-ua123-0" + } + ] + } + }, + { + "match": { + "userMessage": "Read then book UA123.", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "lookup_flight", + "arguments": "{\"flight_number\": \"UA123\"}", + "id": "ic-read-0" + } + ] + } + }, + { + "match": { + "userMessage": "Book unknown flight.", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "book_flight", + "arguments": "{\"flight_number\": \"ZZ999\"}", + "id": "ic-unknown-0" + } + ] + } + }, + { + "match": { + "userMessage": "Book multiple flights.", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "book_flight", + "arguments": "{\"flight_number\": \"UA123\"}", + "id": "ic-multi-0" + }, + { + "name": "book_flight", + "arguments": "{\"flight_number\": \"AA404\"}", + "id": "ic-multi-1" + } + ] + } + }, + { + "match": { + "userMessage": "Explain aviation" + }, + "response": { + "content": "An aviation answer.\n\n```typescript\nconst approved = true;\n```" + }, + "chunkSize": 3 } ] } \ No newline at end of file diff --git a/cockpit/chat/interrupts/react/e2e/global-setup-impl.ts b/cockpit/chat/interrupts/react/e2e/global-setup-impl.ts new file mode 100644 index 000000000..8fb0e2ffa --- /dev/null +++ b/cockpit/chat/interrupts/react/e2e/global-setup-impl.ts @@ -0,0 +1,6 @@ +// eslint-disable-next-line @nx/enforce-module-boundaries -- Shared aimock setup for the React cockpit examples. +import { reactAimockGlobalSetup } from '../../../../shared/react-aimock-setup'; + +// aimock replays the Angular example's fixtures into a real `langgraph dev`; +// the static server (webServer) proxies /api to it. +export default reactAimockGlobalSetup({ example: 'chat/interrupts', port: 4622 }); diff --git a/cockpit/chat/interrupts/react/e2e/interrupts.spec.ts b/cockpit/chat/interrupts/react/e2e/interrupts.spec.ts index 88b2f759a..d2748fe46 100644 --- a/cockpit/chat/interrupts/react/e2e/interrupts.spec.ts +++ b/cockpit/chat/interrupts/react/e2e/interrupts.spec.ts @@ -1,4 +1,9 @@ import { test, expect, type Page } from '@playwright/test'; +// Backend: aimock replays cockpit/chat/interrupts/angular/e2e/fixtures (shared +// with the Angular example; React prompts key tool rounds on toolCallId) into a +// real `langgraph dev`, which raises the real book_flight interrupt. The React +// static server proxies /api to it, recording requests and injecting holds and +// failures (scripts/react-cockpit/langgraph-proxy.mjs). type Wire = { path: string; method: string; @@ -103,16 +108,8 @@ test('installed SDK confirms then cancels real saved pauses on one conversation' expect(wire.filter((item) => item.path.endsWith('/history'))).toHaveLength(4); expect(wire.filter((item) => item.method === 'GET')).toHaveLength(4); await expect(conversation(page)).not.toContainText( - 'Authored aviation conversation' + 'Aviation snippet title' ); - const proofs = await (await request.get('/__graph-proof')).json(); - expect(proofs).toHaveLength(4); - for (const proof of proofs) - expect(proof).toMatchObject({ - actualCompiledGraph: true, - networkConnectAttempts: 0, - titleMessageCallbacks: 0, - }); }); test('direct answers and causally completed read tools precede a saved approval', async ({ page, @@ -203,7 +200,7 @@ test('held resume admits one decision and Stop fences late completion', async ({ }) => { await page.goto(base); await pause(page); - await request.post('/__hold-resume'); + await request.post('/__hold/stream'); await page.getByRole('button', { name: 'Book flight', exact: true }).click(); await expect( page.getByRole('button', { name: 'Book flight', exact: true }) @@ -224,7 +221,7 @@ test('failed resume is never retried and New only resets local state', async ({ }) => { await page.goto(base); await pause(page); - await request.post('/__fail-resume'); + await request.post('/__fail-stream'); await page.getByRole('button', { name: "Don't book", exact: true }).click(); await blocked(page); const wire: Wire[] = await (await request.get('/__requests')).json(); @@ -259,12 +256,13 @@ test('metadata failure cannot corrupt the real graph wire or expose a title resp page, request, }) => { - await request.post('/__metadata-failure'); + // The title model call for this prompt fails (its own aimock error fixture); + // generate_title swallows it like any metadata failure. await page.goto(base); - await pause(page); + await pause(page, 'Book padded flight. (title fails)'); await page.getByRole('button', { name: "Don't book", exact: true }).click(); await saved(page); - await expect(conversation(page)).not.toContainText('Authored metadata'); + await expect(conversation(page)).not.toContainText('PRIVATE'); }); test('authored developer connection routes only to its configured target', async ({ page, @@ -343,7 +341,7 @@ test('Stop during held initial streaming cannot expose a late approval', async ( page, request, }) => { - await request.post('/__hold-stream'); + await request.post('/__hold/stream'); await page.goto(base); await submit(page, 'Book UA123.'); await expect diff --git a/cockpit/chat/interrupts/react/e2e/playwright.config.ts b/cockpit/chat/interrupts/react/e2e/playwright.config.ts index 19b92564d..e3e89579a 100644 --- a/cockpit/chat/interrupts/react/e2e/playwright.config.ts +++ b/cockpit/chat/interrupts/react/e2e/playwright.config.ts @@ -5,6 +5,9 @@ export default defineConfig({ fullyParallel: false, use: { baseURL: 'http://127.0.0.1:4622', trace: 'retain-on-failure' }, projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }], + // aimock + real `langgraph dev`, shared with cockpit/chat/interrupts/angular. + globalSetup: './global-setup-impl.ts', + globalTeardown: require.resolve('../../../../../libs/e2e-harness/src/global-teardown'), webServer: { command: 'node scripts/react-cockpit/serve.mjs chat-interrupts', cwd: '../../../../..', diff --git a/cockpit/chat/interrupts/react/project.json b/cockpit/chat/interrupts/react/project.json index c1ae47b7f..20d8ba9de 100644 --- a/cockpit/chat/interrupts/react/project.json +++ b/cockpit/chat/interrupts/react/project.json @@ -3,6 +3,7 @@ "$schema": "../../../../node_modules/nx/schemas/project-schema.json", "sourceRoot": "cockpit/chat/interrupts/react/src", "projectType": "application", + "implicitDependencies": ["cockpit-chat-interrupts-angular", "cockpit-chat-interrupts-python"], "tags": ["scope:cockpit-examples", "scope:cockpit-e2e", "framework:react"], "targets": { "build": { @@ -32,24 +33,12 @@ "e2e": { "executor": "@nx/playwright:playwright", "cache": false, - "dependsOn": ["build", "fixture-test"], + "inputs": ["default", "reactAimockBackend"], + "dependsOn": ["build", { "target": "test", "projects": ["cockpit-chat-interrupts-python"] }], "options": { "config": "cockpit/chat/interrupts/react/e2e/playwright.config.ts", "skipInstall": true } - }, - "fixture-test": { - "executor": "nx:run-commands", - "cache": false, - "dependsOn": [ - { - "target": "test", - "projects": ["cockpit-chat-interrupts-python"] - } - ], - "options": { - "command": "node --test scripts/react-cockpit/chat-interrupts-fixture.spec.mjs" - } } } } diff --git a/cockpit/chat/messages/react/e2e/global-setup-impl.ts b/cockpit/chat/messages/react/e2e/global-setup-impl.ts index 6e1d73a66..22867e9fb 100644 --- a/cockpit/chat/messages/react/e2e/global-setup-impl.ts +++ b/cockpit/chat/messages/react/e2e/global-setup-impl.ts @@ -1,15 +1,6 @@ -import { resolve } from 'node:path'; -import { createLanggraphGlobalSetup } from '@threadplane-internal/e2e-harness'; -// eslint-disable-next-line @nx/enforce-module-boundaries -- Closed React cockpit identity shared with the static server. -import { reactCockpitConfiguration } from '../../../../../scripts/react-cockpit/configuration.mjs'; +// eslint-disable-next-line @nx/enforce-module-boundaries -- Shared aimock setup for the React cockpit examples. +import { reactAimockGlobalSetup } from '../../../../shared/react-aimock-setup'; -const { project, backend } = reactCockpitConfiguration('chat-messages'); - -// aimock replays the Angular example's fixtures; `langgraph dev` runs the real -// graph against it. The static server (webServer) proxies /api here. -export default createLanggraphGlobalSetup({ - stateKey: project, - langgraphCwd: backend.python, - langgraphPort: backend.langgraphPort, - fixturesDir: resolve(__dirname, '../../../../..', backend.fixtures), -}); +// aimock replays the Angular example's fixtures into a real `langgraph dev`; +// the static server (webServer) proxies /api to it. +export default reactAimockGlobalSetup({ example: 'chat/messages', port: 4620 }); diff --git a/cockpit/chat/messages/react/project.json b/cockpit/chat/messages/react/project.json index 3137081b0..b0552e021 100644 --- a/cockpit/chat/messages/react/project.json +++ b/cockpit/chat/messages/react/project.json @@ -43,6 +43,7 @@ "e2e": { "executor": "@nx/playwright:playwright", "cache": false, + "inputs": ["default", "reactAimockBackend"], "dependsOn": [ "build", { diff --git a/cockpit/chat/subagents/angular/e2e/fixtures/c-subagents.json b/cockpit/chat/subagents/angular/e2e/fixtures/c-subagents.json index bf1abbdad..e4474a279 100644 --- a/cockpit/chat/subagents/angular/e2e/fixtures/c-subagents.json +++ b/cockpit/chat/subagents/angular/e2e/fixtures/c-subagents.json @@ -1,5 +1,349 @@ { "fixtures": [ + { + "match": { + "systemMessage": "In 3-5 words, summarize", + "userMessage": "(title fails)" + }, + "response": { + "error": { + "message": "PRIVATE title model failure", + "type": "server_error" + }, + "status": 500 + } + }, + { + "match": { + "systemMessage": "In 3-5 words, summarize" + }, + "response": { + "content": "Aviation snippet title" + } + }, + { + "match": { + "systemMessage": "You are a Research Agent", + "userMessage": "Research LAX to JFK for delegate sequential" + }, + "response": { + "content": "Research suggestion: Research LAX to JFK for delegate sequential." + } + }, + { + "match": { + "systemMessage": "You are a Booking Agent", + "userMessage": "Booking LAX to JFK for delegate sequential" + }, + "response": { + "content": "Booking suggestion: Booking LAX to JFK for delegate sequential." + } + }, + { + "match": { + "systemMessage": "You are an Itinerary Agent", + "userMessage": "Itinerary LAX to JFK for delegate sequential" + }, + "response": { + "content": "Itinerary suggestion: Itinerary LAX to JFK for delegate sequential." + } + }, + { + "match": { + "systemMessage": "You are a Research Agent", + "userMessage": "Research LAX to JFK for delegate parallel" + }, + "response": { + "content": "Research suggestion: Research LAX to JFK for delegate parallel." + } + }, + { + "match": { + "systemMessage": "You are a Booking Agent", + "userMessage": "Booking LAX to JFK for delegate parallel" + }, + "response": { + "content": "Booking suggestion: Booking LAX to JFK for delegate parallel." + } + }, + { + "match": { + "systemMessage": "You are an Itinerary Agent", + "userMessage": "Itinerary LAX to JFK for delegate parallel" + }, + "response": { + "content": "Itinerary suggestion: Itinerary LAX to JFK for delegate parallel." + } + }, + { + "match": { + "systemMessage": "You are a Research Agent", + "userMessage": "Research LAX to JFK for repeated task (1)" + }, + "response": { + "content": "Research suggestion: Research LAX to JFK for repeated task (1)." + } + }, + { + "match": { + "systemMessage": "You are a Research Agent", + "userMessage": "Research LAX to JFK for repeated task (2)" + }, + "response": { + "content": "Research suggestion: Research LAX to JFK for repeated task (2)." + } + }, + { + "match": { + "systemMessage": "You are a Research Agent", + "userMessage": "Research LAX to JFK for empty-child" + }, + "response": { + "content": "Research suggestion: Research LAX to JFK for empty-child." + } + }, + { + "match": { + "systemMessage": "You are a Booking Agent", + "userMessage": "Booking LAX to JFK for empty-child" + }, + "response": { + "content": " " + } + }, + { + "match": { + "systemMessage": "You are an Itinerary Agent", + "userMessage": "Itinerary LAX to JFK for empty-child" + }, + "response": { + "content": "Itinerary suggestion: Itinerary LAX to JFK for empty-child." + } + }, + { + "match": { + "systemMessage": "You are a Research Agent", + "userMessage": "Research LAX to JFK for fail-child" + }, + "response": { + "content": "Research suggestion: Research LAX to JFK for fail-child." + } + }, + { + "match": { + "systemMessage": "You are a Booking Agent", + "userMessage": "Booking LAX to JFK for fail-child" + }, + "response": { + "content": "Authored partial child output that never completes." + }, + "chunkSize": 8, + "truncateAfterChunks": 1 + }, + { + "match": { + "systemMessage": "You are an Itinerary Agent", + "userMessage": "Itinerary LAX to JFK for fail-child" + }, + "response": { + "content": "Itinerary suggestion: Itinerary LAX to JFK for fail-child." + } + }, + { + "match": { + "systemMessage": "You are a Research Agent", + "userMessage": "Research LAX to JFK for Literal long " + }, + "response": { + "content": "Research suggestion: Research LAX to JFK for Literal long ." + } + }, + { + "match": { + "systemMessage": "You are a Booking Agent", + "userMessage": "Booking LAX to JFK for Literal long " + }, + "response": { + "content": "Booking suggestion: Booking LAX to JFK for Literal long ." + } + }, + { + "match": { + "systemMessage": "You are an Itinerary Agent", + "userMessage": "Itinerary LAX to JFK for Literal long " + }, + "response": { + "content": "Itinerary suggestion: Itinerary LAX to JFK for Literal long ." + } + }, + { + "match": { + "toolCallId": "task-seq-0" + }, + "response": { + "toolCalls": [ + { + "name": "task", + "arguments": "{\"subagent_type\": \"booking\", \"task_description\": \"Booking LAX to JFK for delegate sequential\"}", + "id": "task-seq-1" + } + ] + } + }, + { + "match": { + "toolCallId": "task-seq-1" + }, + "response": { + "toolCalls": [ + { + "name": "task", + "arguments": "{\"subagent_type\": \"itinerary\", \"task_description\": \"Itinerary LAX to JFK for delegate sequential\"}", + "id": "task-seq-2" + } + ] + } + }, + { + "match": { + "toolCallId": "task-seq-2" + }, + "response": { + "content": "Here is your LAX to JFK trip plan." + } + }, + { + "match": { + "toolCallId": "task-par-2" + }, + "response": { + "content": "Here is your LAX to JFK trip plan." + } + }, + { + "match": { + "toolCallId": "task-rep-1" + }, + "response": { + "content": "Here is your LAX to JFK trip plan." + } + }, + { + "match": { + "toolCallId": "task-empty-0" + }, + "response": { + "toolCalls": [ + { + "name": "task", + "arguments": "{\"subagent_type\": \"booking\", \"task_description\": \"Booking LAX to JFK for empty-child\"}", + "id": "task-empty-1" + } + ] + } + }, + { + "match": { + "toolCallId": "task-empty-1" + }, + "response": { + "toolCalls": [ + { + "name": "task", + "arguments": "{\"subagent_type\": \"itinerary\", \"task_description\": \"Itinerary LAX to JFK for empty-child\"}", + "id": "task-empty-2" + } + ] + } + }, + { + "match": { + "toolCallId": "task-empty-2" + }, + "response": { + "content": "Here is your LAX to JFK trip plan." + } + }, + { + "match": { + "toolCallId": "task-fail-0" + }, + "response": { + "toolCalls": [ + { + "name": "task", + "arguments": "{\"subagent_type\": \"booking\", \"task_description\": \"Booking LAX to JFK for fail-child\"}", + "id": "task-fail-1" + } + ] + } + }, + { + "match": { + "toolCallId": "task-fail-1" + }, + "response": { + "toolCalls": [ + { + "name": "task", + "arguments": "{\"subagent_type\": \"itinerary\", \"task_description\": \"Itinerary LAX to JFK for fail-child\"}", + "id": "task-fail-2" + } + ] + } + }, + { + "match": { + "toolCallId": "task-fail-2" + }, + "response": { + "content": "Here is your LAX to JFK trip plan." + } + }, + { + "match": { + "toolCallId": "task-literal-0" + }, + "response": { + "toolCalls": [ + { + "name": "task", + "arguments": "{\"subagent_type\": \"booking\", \"task_description\": \"Booking LAX to JFK for Literal long \"}", + "id": "task-literal-1" + } + ] + } + }, + { + "match": { + "toolCallId": "task-literal-1" + }, + "response": { + "toolCalls": [ + { + "name": "task", + "arguments": "{\"subagent_type\": \"itinerary\", \"task_description\": \"Itinerary LAX to JFK for Literal long \"}", + "id": "task-literal-2" + } + ] + } + }, + { + "match": { + "toolCallId": "task-literal-2" + }, + "response": { + "content": "Here is your LAX to JFK trip plan." + } + }, + { + "match": { + "toolCallId": "task-malformed-0" + }, + "response": { + "content": "Here is your LAX to JFK trip plan." + } + }, { "match": { "userMessage": "Plan a trip from LAX to JFK", @@ -119,6 +463,134 @@ "systemHash": "03217c65", "toolsHash": "1ce7a06f" } + }, + { + "match": { + "userMessage": "delegate sequential", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "task", + "arguments": "{\"subagent_type\": \"research\", \"task_description\": \"Research LAX to JFK for delegate sequential\"}", + "id": "task-seq-0" + } + ] + } + }, + { + "match": { + "userMessage": "delegate parallel", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "task", + "arguments": "{\"subagent_type\": \"research\", \"task_description\": \"Research LAX to JFK for delegate parallel\"}", + "id": "task-par-0" + }, + { + "name": "task", + "arguments": "{\"subagent_type\": \"booking\", \"task_description\": \"Booking LAX to JFK for delegate parallel\"}", + "id": "task-par-1" + }, + { + "name": "task", + "arguments": "{\"subagent_type\": \"itinerary\", \"task_description\": \"Itinerary LAX to JFK for delegate parallel\"}", + "id": "task-par-2" + } + ] + } + }, + { + "match": { + "userMessage": "repeated task", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "task", + "arguments": "{\"subagent_type\": \"research\", \"task_description\": \"Research LAX to JFK for repeated task (1)\"}", + "id": "task-rep-0" + }, + { + "name": "task", + "arguments": "{\"subagent_type\": \"research\", \"task_description\": \"Research LAX to JFK for repeated task (2)\"}", + "id": "task-rep-1" + } + ] + } + }, + { + "match": { + "userMessage": "empty-child", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "task", + "arguments": "{\"subagent_type\": \"research\", \"task_description\": \"Research LAX to JFK for empty-child\"}", + "id": "task-empty-0" + } + ] + } + }, + { + "match": { + "userMessage": "fail-child", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "task", + "arguments": "{\"subagent_type\": \"research\", \"task_description\": \"Research LAX to JFK for fail-child\"}", + "id": "task-fail-0" + } + ] + } + }, + { + "match": { + "userMessage": "Literal long ", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "task", + "arguments": "{\"subagent_type\": \"research\", \"task_description\": \"Research LAX to JFK for Literal long \"}", + "id": "task-literal-0" + } + ] + } + }, + { + "match": { + "userMessage": "malformed task", + "hasToolResult": false + }, + "response": { + "toolCalls": [ + { + "name": "task", + "arguments": "{\"subagent_type\": \"research\", \"task_description\": 123}", + "id": "task-malformed-0" + } + ] + } + }, + { + "match": { + "userMessage": "direct answer" + }, + "response": { + "content": "A direct answer with no specialists." + } } ] -} \ No newline at end of file +} diff --git a/cockpit/chat/subagents/react/e2e/global-setup-impl.ts b/cockpit/chat/subagents/react/e2e/global-setup-impl.ts new file mode 100644 index 000000000..e5cc2ffb4 --- /dev/null +++ b/cockpit/chat/subagents/react/e2e/global-setup-impl.ts @@ -0,0 +1,6 @@ +// eslint-disable-next-line @nx/enforce-module-boundaries -- Shared aimock setup for the React cockpit examples. +import { reactAimockGlobalSetup } from '../../../../shared/react-aimock-setup'; + +// aimock replays the Angular example's fixtures into a real `langgraph dev`; +// the static server (webServer) proxies /api to it. +export default reactAimockGlobalSetup({ example: 'chat/subagents', port: 4624 }); diff --git a/cockpit/chat/subagents/react/e2e/playwright.config.ts b/cockpit/chat/subagents/react/e2e/playwright.config.ts index 547916ef9..2e441f380 100644 --- a/cockpit/chat/subagents/react/e2e/playwright.config.ts +++ b/cockpit/chat/subagents/react/e2e/playwright.config.ts @@ -5,6 +5,9 @@ export default defineConfig({ fullyParallel: false, use: { baseURL: 'http://127.0.0.1:4624', trace: 'retain-on-failure' }, projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }], + // aimock + real `langgraph dev`, shared with cockpit/chat/subagents/angular. + globalSetup: './global-setup-impl.ts', + globalTeardown: require.resolve('../../../../../libs/e2e-harness/src/global-teardown'), webServer: { command: 'node scripts/react-cockpit/serve.mjs chat-subagents', cwd: '../../../../..', diff --git a/cockpit/chat/subagents/react/e2e/subagents.spec.ts b/cockpit/chat/subagents/react/e2e/subagents.spec.ts index b5098800e..67487028b 100644 --- a/cockpit/chat/subagents/react/e2e/subagents.spec.ts +++ b/cockpit/chat/subagents/react/e2e/subagents.spec.ts @@ -1,13 +1,15 @@ import { test, expect, type Page } from '@playwright/test'; -import { createHash } from 'node:crypto'; -import { readFileSync } from 'node:fs'; -import { resolve } from 'node:path'; +// Backend: aimock replays cockpit/chat/subagents/angular/e2e/fixtures (shared +// with the Angular example; React prompts key parent rounds on toolCallId and +// child calls on their role prompt + task text) into a real `langgraph dev`. +// The React static server proxies /api to it, recording requests and injecting +// holds and failures (scripts/react-cockpit/langgraph-proxy.mjs). test('a held real tool call exposes complete arguments before its result', async ({ page, request, }) => { - expect((await request.post('/__hold-tool')).status()).toBe(200); + expect((await request.post('/__hold/stream', { data: { beforeToolResult: true } })).status()).toBe(200); await page.goto(base); await submit(page, 'delegate sequential'); // The transcript shows progress while the run is held, not just the footer status. @@ -32,7 +34,7 @@ test('worker failure is generic and never retried', async ({ page, request, }) => { - expect((await request.post('/__worker-failure')).status()).toBe(200); + expect((await request.post('/__fail-stream')).status()).toBe(200); await page.goto(base); await submit(page, 'delegate sequential'); await blocked(page); @@ -41,7 +43,7 @@ test('worker failure is generic and never retried', async ({ wire.filter((item) => item.path.endsWith('/runs/stream')) ).toHaveLength(1); expect(wire.filter((item) => item.path.endsWith('/history'))).toHaveLength(0); - await expect(page.locator('body')).not.toContainText('Graph worker'); + await expect(page.locator('body')).not.toContainText('PRIVATE'); }); test('long literal tool arguments and data remain text on mobile', async ({ page, @@ -93,12 +95,13 @@ test('metadata failure preserves the real tool stream and hides private diagnost page, request, }) => { - await request.post('/__metadata-failure'); + // The title model call for this prompt fails (its own aimock error fixture); + // generate_title swallows it like any metadata failure. await page.goto(base); - await submit(page, 'delegate sequential'); + await submit(page, 'delegate sequential (title fails)'); await saved(page); await expect(cards(page)).toHaveCount(3); - await expect(page.locator('body')).not.toContainText('Authored metadata'); + await expect(page.locator('body')).not.toContainText('PRIVATE'); }); test('authored developer connection routes only to its configured target', async ({ page, @@ -170,21 +173,6 @@ async function blocked(page: Page) { test.beforeEach(async ({ request }) => { await request.post('/__reset'); }); -test.afterEach(async ({ request }) => { - const proofs = await (await request.get('/__graph-proof')).json(); - for (const proof of proofs) - expect(proof).toMatchObject({ - actualCompiledGraph: true, - networkConnectAttempts: 0, - titleMessageCallbacks: 0, - sourceSha256: createHash('sha256') - .update(readFileSync(resolve(__dirname, '../../python/src/graph.py'))) - .digest('hex'), - lockSha256: createHash('sha256') - .update(readFileSync(resolve(__dirname, '../../python/uv.lock'))) - .digest('hex'), - }); -}); test('installed SDK confirms sequential and parallel specialists on one saved conversation', async ({ page, request, @@ -234,7 +222,7 @@ test('installed SDK confirms sequential and parallel specialists on one saved co expect(wire.filter((item) => item.path.endsWith('/history'))).toHaveLength(2); expect(wire.filter((item) => item.method === 'GET')).toHaveLength(2); await expect(conversation(page)).not.toContainText( - 'Authored trip conversation' + 'Aviation snippet title' ); expect(errors).toEqual([]); }); @@ -310,9 +298,11 @@ for (const hold of ['stream', 'child', 'history']) { page, request, }) => { - await request.post( - hold === 'history' ? '/__history/hold' : `/__hold-${hold}` - ); + if (hold === 'history') await request.post('/__history/hold'); + else + await request.post('/__hold/stream', { + data: hold === 'child' ? { afterChildMessage: true } : {}, + }); await page.goto(base); await submit(page, 'delegate sequential'); if (hold === 'history') @@ -390,7 +380,7 @@ test('held child text is local and Stop labels it without claiming specialist su page, request, }) => { - await request.post('/__hold-child'); + await request.post('/__hold/stream', { data: { afterChildMessage: true } }); await page.goto(base); await submit(page, 'delegate sequential'); const disclosure = page.locator('details.child-observations'); diff --git a/cockpit/chat/subagents/react/project.json b/cockpit/chat/subagents/react/project.json index 7c4a4896c..2fd2c41c5 100644 --- a/cockpit/chat/subagents/react/project.json +++ b/cockpit/chat/subagents/react/project.json @@ -3,6 +3,7 @@ "$schema": "../../../../node_modules/nx/schemas/project-schema.json", "sourceRoot": "cockpit/chat/subagents/react/src", "projectType": "application", + "implicitDependencies": ["cockpit-chat-subagents-angular", "cockpit-chat-subagents-python"], "tags": ["scope:cockpit-examples", "scope:cockpit-e2e", "framework:react"], "targets": { "build": { @@ -32,24 +33,12 @@ "e2e": { "executor": "@nx/playwright:playwright", "cache": false, - "dependsOn": ["build", "fixture-test"], + "inputs": ["default", "reactAimockBackend"], + "dependsOn": ["build", { "target": "test", "projects": ["cockpit-chat-subagents-python"] }], "options": { "config": "cockpit/chat/subagents/react/e2e/playwright.config.ts", "skipInstall": true } - }, - "fixture-test": { - "executor": "nx:run-commands", - "cache": false, - "dependsOn": [ - { - "target": "test", - "projects": ["cockpit-chat-subagents-python"] - } - ], - "options": { - "command": "node --test scripts/react-cockpit/chat-subagents-fixture.spec.mjs" - } } } } diff --git a/cockpit/chat/theming/angular/e2e/fixtures/c-theming.json b/cockpit/chat/theming/angular/e2e/fixtures/c-theming.json index 3da770b38..1fad62de5 100644 --- a/cockpit/chat/theming/angular/e2e/fixtures/c-theming.json +++ b/cockpit/chat/theming/angular/e2e/fixtures/c-theming.json @@ -1,10 +1,18 @@ { "fixtures": [ + { + "match": { "systemMessage": "In 3-5 words, summarize" }, + "response": { "content": "Aviation snippet title" } + }, { "match": { "userMessage": "Hello" }, "response": { "content": "Hi! I'm the chat-theming demo. Try the theme picker buttons in the sidebar to switch between dark, light, ocean, and forest schemes." } + }, + { + "match": { "userMessage": "" }, + "response": { "content": "Lift is the upward force on a wing.\n\n```typescript\nconst lift = 'up';\n```" } } ] } diff --git a/cockpit/chat/theming/react/e2e/global-setup-impl.ts b/cockpit/chat/theming/react/e2e/global-setup-impl.ts new file mode 100644 index 000000000..e4a820634 --- /dev/null +++ b/cockpit/chat/theming/react/e2e/global-setup-impl.ts @@ -0,0 +1,6 @@ +// eslint-disable-next-line @nx/enforce-module-boundaries -- Shared aimock setup for the React cockpit examples. +import { reactAimockGlobalSetup } from '../../../../shared/react-aimock-setup'; + +// aimock replays the Angular example's fixtures into a real `langgraph dev`; +// the static server (webServer) proxies /api to it. +export default reactAimockGlobalSetup({ example: 'chat/theming', port: 4644 }); diff --git a/cockpit/chat/theming/react/e2e/playwright.config.ts b/cockpit/chat/theming/react/e2e/playwright.config.ts index 1d755f756..30d53712c 100644 --- a/cockpit/chat/theming/react/e2e/playwright.config.ts +++ b/cockpit/chat/theming/react/e2e/playwright.config.ts @@ -8,6 +8,9 @@ export default defineConfig({ outputDir: '../../../../../output/playwright/chat-theming-react', use: { baseURL: 'http://127.0.0.1:4644', trace: 'retain-on-failure' }, projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }], + // aimock + real `langgraph dev`, shared with cockpit/chat/theming/angular. + globalSetup: './global-setup-impl.ts', + globalTeardown: require.resolve('../../../../../libs/e2e-harness/src/global-teardown'), webServer: { command: 'node scripts/react-cockpit/serve.mjs chat-theming --no-parent', cwd: '../../../../..', diff --git a/cockpit/chat/theming/react/e2e/theming.spec.ts b/cockpit/chat/theming/react/e2e/theming.spec.ts index 3d262494e..6094a9387 100644 --- a/cockpit/chat/theming/react/e2e/theming.spec.ts +++ b/cockpit/chat/theming/react/e2e/theming.spec.ts @@ -56,30 +56,25 @@ async function submit(page: Page, text: string) { test.beforeEach(async ({ request }) => { await control(request, '/__reset'); }); +/** + * Injected data, opted into only by the test that styles the Reasoning + * region: langchain_openai's ChatOpenAI does not extract `reasoning_content` + * from Chat Completions responses, so the real graph never surfaces reasoning + * (scripts/react-cockpit/langgraph-proxy.mjs, /__inject/reasoning). + */ +const injectReasoning = async (request: APIRequestContext) => + expect( + (await request.post('/__inject/reasoning', { + data: { text: 'Considering the aviation question.' }, + })).ok() + ).toBe(true); test.afterEach(async ({ page, request }, info) => { + // Every browser connection through the proxy closes and nothing stays held. + // (No per-run graph child processes exist any more: the graph is `langgraph dev`.) await page.goto('about:blank'); await expect - .poll(async () => (await (await request.get('/__lifetime')).json()).active) - .toEqual({ children: 0, streams: 0, histories: 0, handlers: 0 }); - await expect - .poll( - async () => (await (await request.get('/__lifetime')).json()).responseOpen - ) - .toBe(0); - const lifetime = await (await request.get('/__lifetime')).json(); - for (const child of lifetime.children) { - expect(child.reaped).toBe(true); - expect(() => process.kill(child.pid, 0)).toThrow(/ESRCH/); - expect(() => process.kill(-child.pgid, 0)).toThrow(/ESRCH/); - } - await writeFile( - info.outputPath('observed-child-absence.json'), - JSON.stringify( - { children: lifetime.children, observedPidAndPgidAbsence: 'ESRCH' }, - null, - 2 - ) - ); + .poll(async () => (await request.get('/__lifetime')).json()) + .toMatchObject({ activeStream: false, activeHistory: false, openResponses: 0 }); for (const path of ['/__requests', '/__captures', '/__lifetime']) { const body = JSON.stringify( await (await request.get(path)).json(), @@ -98,6 +93,7 @@ for (const width of [1280, 390]) { page, request, }, info) => { + await injectReasoning(request); await page.setViewportSize({ width, height: 900 }); await page.goto('/chat/theming/react/'); await expect(page.getByRole('status')).toHaveText('Ready.'); @@ -222,28 +218,38 @@ for (const width of [1280, 390]) { ); expect(runs.every((r) => r.body.assistant_id === 'c-theming')).toBe(true); expect(wire.filter((r) => r.path.endsWith('/history'))).toHaveLength(2); - const captures = await (await request.get('/__captures')).json(); + const captures: { + sse: string; + savedMessages: { type: string; content: string }[]; + titleBefore: string | null; + titleAfter: string | null; + }[] = await (await request.get('/__captures')).json(); expect(captures).toHaveLength(2); + // The title model's reply (its own aimock fixture) never streams as answer + // content. (It does appear in later runs' metadata: thread metadata.) + const streamedContent = (sse: string) => + sse + .split(/\r?\n\r?\n/) + .filter((event) => /^event: messages/m.test(event)) + .map((event) => JSON.parse(event.split(/\r?\n/).find((l) => l.startsWith('data:'))!.slice(5))[0].content) + .join(''); for (const c of captures) { - expect(c.proof).toMatchObject({ - actualCompiledGraph: true, - networkConnectAttempts: 0, - titleMessageCallbacks: 0, - }); - expect(c.sse).not.toContain('Authored metadata title'); + expect(streamedContent(c.sse)).toContain('upward force'); + expect(streamedContent(c.sse)).not.toContain('Aviation snippet title'); } expect( - captures[0].savedMessages.find( - (m: { type: string }) => m.type === 'human' - ).content + captures[0].savedMessages.find((m) => m.type === 'human')?.content ).toBe('Explain lift ✈ café'); expect( captures[1].savedMessages - .filter((m: { type: string }) => m.type === 'human') - .map((m: { content: string }) => m.content) + .filter((m) => m.type === 'human') + .map((m) => m.content) ).toEqual(['Explain lift ✈ café', 'Continue lift — aérodrome']); - expect(captures[0].titleUpdates).toHaveLength(1); - expect(captures[1].titleUpdates).toHaveLength(0); + // The title is written once, on the first turn, and kept on the second. + expect(captures[0].titleBefore).toBeNull(); + expect(captures[0].titleAfter).toBe('Aviation snippet title'); + expect(captures[1].titleBefore).toBe('Aviation snippet title'); + expect(captures[1].titleAfter).toBe('Aviation snippet title'); await page.getByRole('button', { name: 'New conversation' }).click(); await expect(page.getByRole('status')).toHaveText('Ready.'); await expect(page.locator('article')).toHaveCount(0); @@ -258,8 +264,9 @@ for (const width of [1280, 390]) { request, }) => { await page.setViewportSize({ width, height: 900 }); + await control(request, '/__hold/stream'); await page.goto('/chat/theming/react/'); - await submit(page, 'Hold'); + await submit(page, 'Hello'); await expect(page.getByRole('status')).toHaveText('Receiving response…'); await page .getByLabel('Message', { exact: true }) @@ -297,9 +304,9 @@ for (const phase of ['stream', 'history']) page, request, }) => { - if (phase === 'history') await control(request, '/__history/hold'); + await control(request, phase === 'history' ? '/__history/hold' : '/__hold/stream'); await page.goto('/chat/theming/react/'); - await submit(page, phase === 'stream' ? 'Hold' : 'Explain lift'); + await submit(page, 'Hello'); await expect(page.getByRole('status')).toHaveText( phase === 'stream' ? 'Receiving response…' : 'Confirming saved response…' ); @@ -321,8 +328,9 @@ test('pagehide closes the original stream and remount never replays it', async ( page, request, }) => { + await control(request, '/__hold/stream'); await page.goto('/chat/theming/react/'); - await submit(page, 'Hold'); + await submit(page, 'Hello'); await expect(page.getByRole('status')).toHaveText('Receiving response…'); await page.goto('about:blank'); await expect @@ -330,9 +338,9 @@ test('pagehide closes the original stream and remount never replays it', async ( async () => ( await (await request.get('/__lifetime')).json() - ).active.streams + ).activeStream ) - .toBe(0); + .toBe(false); const before = await requests(request); await page.goto('/chat/theming/react/'); await expect(page.getByRole('status')).toHaveText('Ready.'); diff --git a/cockpit/chat/theming/react/project.json b/cockpit/chat/theming/react/project.json index deaefb3d1..bcf5f399f 100644 --- a/cockpit/chat/theming/react/project.json +++ b/cockpit/chat/theming/react/project.json @@ -3,6 +3,7 @@ "$schema": "../../../../node_modules/nx/schemas/project-schema.json", "sourceRoot": "cockpit/chat/theming/react/src", "projectType": "application", + "implicitDependencies": ["cockpit-chat-theming-angular", "cockpit-chat-theming-python"], "tags": ["scope:cockpit-examples", "scope:cockpit-e2e", "framework:react"], "targets": { "build": { @@ -32,7 +33,7 @@ "e2e": { "executor": "@nx/playwright:playwright", "cache": false, - "inputs": ["default", "themingFixture"], + "inputs": ["default", "reactAimockBackend"], "dependsOn": [ "build", { @@ -45,12 +46,5 @@ "skipInstall": true } } - }, - "namedInputs": { - "themingFixture": [ - "{workspaceRoot}/scripts/react-cockpit/chat-theming-fixture.mjs", - "{workspaceRoot}/scripts/react-cockpit/chat-theming-wire.py", - "{workspaceRoot}/scripts/react-cockpit/chat-theming-fixture.spec.mjs" - ] } } diff --git a/cockpit/chat/threads/angular/e2e/fixtures/c-threads.json b/cockpit/chat/threads/angular/e2e/fixtures/c-threads.json index 8c430436e..cd1f14d18 100644 --- a/cockpit/chat/threads/angular/e2e/fixtures/c-threads.json +++ b/cockpit/chat/threads/angular/e2e/fixtures/c-threads.json @@ -1,10 +1,24 @@ { "fixtures": [ + { + "match": { "systemMessage": "In 3-5 words, summarize" }, + "response": { "content": "Aviation snippet title" } + }, { "match": { "userMessage": "Hello" }, "response": { "content": "Hi! I'm the chat-threads demo. Switch between threads using the sidebar list to see independent conversation contexts." } + }, + { + "match": { "userMessage": "fail-stream" }, + "response": { "content": "A partial answer that never finishes streaming." }, + "chunkSize": 3, + "truncateAfterChunks": 4 + }, + { + "match": { "userMessage": "" }, + "response": { "content": "A saved answer.\n\n```typescript\nconst answer = 42;\n```" } } ] } diff --git a/cockpit/chat/threads/react/e2e/global-setup-impl.ts b/cockpit/chat/threads/react/e2e/global-setup-impl.ts new file mode 100644 index 000000000..08cd4dc4c --- /dev/null +++ b/cockpit/chat/threads/react/e2e/global-setup-impl.ts @@ -0,0 +1,6 @@ +// eslint-disable-next-line @nx/enforce-module-boundaries -- Shared aimock setup for the React cockpit examples. +import { reactAimockGlobalSetup } from '../../../../shared/react-aimock-setup'; + +// aimock replays the Angular example's fixtures into a real `langgraph dev`; +// the static server (webServer) proxies /api to it. +export default reactAimockGlobalSetup({ example: 'chat/threads', port: 4625 }); diff --git a/cockpit/chat/threads/react/e2e/mobile.spec.ts b/cockpit/chat/threads/react/e2e/mobile.spec.ts index 22a8f9666..6e1b3bc48 100644 --- a/cockpit/chat/threads/react/e2e/mobile.spec.ts +++ b/cockpit/chat/threads/react/e2e/mobile.spec.ts @@ -4,7 +4,7 @@ test('mobile A/B/A preserves drafts and a usable transcript after viewport reduc page, request, }) => { - await request.post('/__reset'); + await request.post('/__purge'); await page.setViewportSize({ width: 375, height: 812 }); await page.goto('/chat/threads/react/'); const input = () => @@ -28,7 +28,8 @@ test('mobile A/B/A preserves drafts and a usable transcript after viewport reduc await input().press('Enter'); await expect(page.getByRole('status')).toHaveText('Response saved.'); await input().fill('B mobile draft'); - await picker.nth(0).click(); + // The server lists the newest-created conversation (B) first. + await picker.last().click(); await expect(page.getByRole('status')).toHaveText('Conversation loaded.'); await expect(input()).toHaveValue('A mobile draft'); await expect(transcript).toContainText('A first'); @@ -38,7 +39,7 @@ test('mobile A/B/A preserves drafts and a usable transcript after viewport reduc await expect(input()).toBeFocused(); await expect(input()).toHaveValue('A mobile draft'); expect((await transcript.boundingBox())?.height).toBeGreaterThan(80); - await picker.nth(1).click(); + await picker.first().click(); await expect(page.getByRole('status')).toHaveText('Conversation loaded.'); await expect(input()).toHaveValue('B mobile draft'); await expect(transcript).toContainText('B first'); diff --git a/cockpit/chat/threads/react/e2e/playwright.config.ts b/cockpit/chat/threads/react/e2e/playwright.config.ts index 75c17c75e..c8f85db1d 100644 --- a/cockpit/chat/threads/react/e2e/playwright.config.ts +++ b/cockpit/chat/threads/react/e2e/playwright.config.ts @@ -5,6 +5,9 @@ export default defineConfig({ fullyParallel: false, use: { baseURL: 'http://127.0.0.1:4625', trace: 'retain-on-failure' }, projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }], + // aimock + real `langgraph dev`, shared with cockpit/chat/threads/angular. + globalSetup: './global-setup-impl.ts', + globalTeardown: require.resolve('../../../../../libs/e2e-harness/src/global-teardown'), webServer: { command: 'node scripts/react-cockpit/serve.mjs chat-threads', cwd: '../../../../..', diff --git a/cockpit/chat/threads/react/e2e/threads.spec.ts b/cockpit/chat/threads/react/e2e/threads.spec.ts index 495caf639..571c68f5d 100644 --- a/cockpit/chat/threads/react/e2e/threads.spec.ts +++ b/cockpit/chat/threads/react/e2e/threads.spec.ts @@ -1,12 +1,15 @@ import { test, expect, + type Locator, type Page, type APIRequestContext, } from '@playwright/test'; -import { createHash } from 'node:crypto'; -import { readFileSync } from 'node:fs'; -import { resolve } from 'node:path'; +// Backend: aimock replays cockpit/chat/threads/angular/e2e/fixtures (shared +// with the Angular example) into a real `langgraph dev`, whose generate_title +// node writes the real thread title. The React static server proxies /api to +// it, recording requests and injecting holds, title and history rewrites, and +// failures (scripts/react-cockpit/langgraph-proxy.mjs). type Wire = { path: string; method: string; @@ -21,11 +24,14 @@ type Wire = { const base = '/chat/threads/react/'; const box = (page: Page) => page.getByLabel('Message', { exact: true }); // Conversation rows of the connected sidebar (not New chat or row menus). +// The server lists the most recently updated conversation first. const picker = (page: Page) => page .getByRole('navigation', { name: 'Conversations' }) .getByRole('list', { name: 'Recent' }) .locator('.tp-thread-list__button'); +// A row's title without its updated-at