From 310a6e851f57e974bdb89d6cbe94bcf7a9ba490e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E3=81=AF=E3=82=8B?= <165422770+ahmad-ajmal@users.noreply.github.com> Date: Wed, 4 Nov 2026 18:32:40 +0000 Subject: [PATCH] Dev (#521) * fix: installer setup flow * fix: installer flow * fix: satisfy staging Ruff formatting and lint checks --- .cargo/config.toml | 3 + .github/workflows/agent-app.yml | 10 +- .github/workflows/launcher.yml | 15 + .github/workflows/runtime-bootstrap.yml | 35 ++ agent-app/package-lock.json | 9 + agent-app/spec/pocketbase-checksums.json | 11 + agent-app/tools/package.json | 3 + agent-app/tools/src/commands/dev.ts | 6 +- agent-app/tools/src/commands/pb.test.ts | 150 +++++++ agent-app/tools/src/commands/pb.ts | 152 +++++-- agent-app/tools/src/commands/validate.ts | 8 +- agent-app/tools/src/lib/os-adapter.ts | 62 ++- app/agent_app/agent_view.py | 8 +- app/agent_app/manager.py | 4 +- app/agent_app/runner.py | 23 +- app/agent_app/test_a2app_native_auth.py | 7 +- app/agent_app/test_backups.py | 7 +- app/credentials/api_request.py | 267 +++++++++--- app/credentials/vault.py | 73 +++- app/data/action/credential_actions.py | 85 +++- app/downloads.py | 138 +++--- app/mini_browser/login.py | 15 +- app/mini_browser/ops.py | 8 +- app/mini_browser/ws.py | 73 +++- app/node_runtime.py | 359 +++++++--------- app/process_ledger.py | 51 ++- app/provision/__init__.py | 11 +- app/provision/deps.py | 98 ++++- app/provision/pipeline.py | 4 +- app/provision/proc.py | 2 + app/provision/runtimes.py | 192 +++------ app/provision/types.py | 9 +- app/runtime-manifest.json | 56 +++ app/runtime_manifest.py | 54 +++ app/ui_layer/adapters/browser_adapter.py | 6 +- app/ui_layer/events/event_bus.py | 12 +- craftbot.py | 24 +- craftos_integrations/README.md | 4 +- .../providers/supabase/client.py | 249 ++++++++--- .../providers/supabase/operations.py | 30 +- .../providers/supabase/provider.py | 17 +- install.py | 10 +- launcher/Cargo.lock | 190 ++++++++- launcher/Cargo.toml | 13 +- launcher/README.md | 35 +- launcher/src/download.rs | 91 +++- launcher/src/install.rs | 7 +- launcher/src/main.rs | 3 + launcher/src/paths.rs | 4 +- launcher/src/python.rs | 149 +++++-- run.py | 13 +- scripts/package_source.py | 50 ++- scripts/prefetch_runtimes.py | 18 +- scripts/smoke_supabase.py | 314 ++++++++++---- scripts/test_runtime_bootstrap.py | 133 ++++++ tests/install_bootstrap/test_bootstrap.py | 399 ++++++++++++++++++ .../test_claude_code_subscription.py | 4 +- .../test_listener_text_fidelity.py | 5 +- .../integrations/test_supabase_conformance.py | 106 +++-- tests/llm/golden/conftest.py | 15 +- tests/llm/test_reliability.py | 4 +- tests/test_app_readiness.py | 13 +- tests/test_credential_api_request.py | 387 +++++++++++------ tests/test_mini_browser_actions_api.py | 12 +- tests/test_mini_browser_agent_integration.py | 6 +- tests/test_mini_browser_fleet.py | 6 +- tests/test_mini_browser_ops.py | 11 +- tests/test_mini_browser_vault.py | 101 +++-- tests/test_mini_browser_ws.py | 78 ++-- tests/test_process_ledger.py | 47 +++ tests/test_question_dismissal.py | 12 +- tests/test_session_clear.py | 170 ++++++-- tests/test_settings_transactions.py | 4 +- 73 files changed, 3541 insertions(+), 1219 deletions(-) create mode 100644 .cargo/config.toml create mode 100644 .github/workflows/runtime-bootstrap.yml create mode 100644 agent-app/spec/pocketbase-checksums.json create mode 100644 agent-app/tools/src/commands/pb.test.ts create mode 100644 app/runtime-manifest.json create mode 100644 app/runtime_manifest.py create mode 100644 scripts/test_runtime_bootstrap.py create mode 100644 tests/install_bootstrap/test_bootstrap.py diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 00000000..a3d4dc1b --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,3 @@ +# The installer must start before the VC++ Redistributable is installed. +[target.'cfg(all(windows, target_env = "msvc"))'] +rustflags = ["-C", "target-feature=+crt-static"] diff --git a/.github/workflows/agent-app.yml b/.github/workflows/agent-app.yml index 03b2a2eb..fa0ea33f 100644 --- a/.github/workflows/agent-app.yml +++ b/.github/workflows/agent-app.yml @@ -43,7 +43,15 @@ jobs: key: pb-${{ runner.os }}-${{ hashFiles('agent-app/spec/pocketbase.version') }} - name: Install workspace - run: npm install + run: npm ci + + - name: PocketBase bootstrap regressions + run: node --test tools/src/commands/pb.test.ts + + - name: Cold PocketBase download + env: + AGENT_APP_PB_CACHE: ${{ runner.temp }}/craftbot-pb-cold + run: node tools/src/cli.ts pb fetch - name: Typecheck (kit + tools) run: npm run typecheck diff --git a/.github/workflows/launcher.yml b/.github/workflows/launcher.yml index cff2f4ab..28c832e0 100644 --- a/.github/workflows/launcher.yml +++ b/.github/workflows/launcher.yml @@ -8,10 +8,14 @@ on: push: paths: - "launcher/**" + - ".cargo/config.toml" + - "app/runtime-manifest.json" - ".github/workflows/launcher.yml" pull_request: paths: - "launcher/**" + - ".cargo/config.toml" + - "app/runtime-manifest.json" - ".github/workflows/launcher.yml" env: @@ -69,5 +73,16 @@ jobs: - name: Test run: cargo test --release + - name: Bootstrap pinned Python without a system interpreter + run: cargo test --release live_python_bootstrap -- --ignored + - name: Build run: cargo build --release + + - name: Start on minimal Linux without desktop or developer packages + if: matrix.os_label == 'linux' + run: | + . /etc/os-release + docker run --rm --network none \ + --volume "$PWD/target/release/CraftBotInstaller:/installer:ro" \ + "ubuntu:$VERSION_ID" /installer --headless status diff --git a/.github/workflows/runtime-bootstrap.yml b/.github/workflows/runtime-bootstrap.yml new file mode 100644 index 00000000..a37cc030 --- /dev/null +++ b/.github/workflows/runtime-bootstrap.yml @@ -0,0 +1,35 @@ +name: Runtime bootstrap + +on: + pull_request: + paths: + - 'app/runtime*' + - 'app/node_runtime.py' + - 'app/downloads.py' + - 'app/provision/**' + - 'agent-app/**' + - 'install.py' + - 'craftbot.py' + - 'run.py' + - 'tests/install_bootstrap/**' + - 'scripts/test_runtime_bootstrap.py' + - '.github/workflows/runtime-bootstrap.yml' + workflow_dispatch: + +jobs: + bootstrap: + strategy: + fail-fast: false + matrix: + os: [windows-latest, macos-latest, ubuntu-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 25 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Bootstrap regression tests (stdlib only) + run: python -m unittest discover -s tests/install_bootstrap -v + - name: Download runtimes and prepare Agent Apps with an empty PATH + run: python scripts/test_runtime_bootstrap.py diff --git a/agent-app/package-lock.json b/agent-app/package-lock.json index 3e2024f8..1dd7aa6f 100644 --- a/agent-app/package-lock.json +++ b/agent-app/package-lock.json @@ -3136,6 +3136,12 @@ } } }, + "node_modules/fflate": { + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.3.tgz", + "integrity": "sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==", + "license": "MIT" + }, "node_modules/file-entry-cache": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-8.0.0.tgz", @@ -4599,6 +4605,9 @@ "devDependencies": { "@types/node": "^24.0.0", "playwright": "^1.61.1" + }, + "dependencies": { + "fflate": "0.8.3" } } } diff --git a/agent-app/spec/pocketbase-checksums.json b/agent-app/spec/pocketbase-checksums.json new file mode 100644 index 00000000..942d310e --- /dev/null +++ b/agent-app/spec/pocketbase-checksums.json @@ -0,0 +1,11 @@ +{ + "pocketbase_0.39.7_darwin_amd64.zip": "32ee65426529c5fc48b7899431b9cf22e54c40a2f39f9fde92948bcabebaa933", + "pocketbase_0.39.7_darwin_arm64.zip": "f94b96eac1684d35cad5368334c156673fa21ae28df721e712e1cf3b8176aeb8", + "pocketbase_0.39.7_linux_amd64.zip": "0fe09a4e1a8f6e5b53d206c2e6b94a5812febcb43082d66d69bc8ba4d8e8429c", + "pocketbase_0.39.7_linux_arm64.zip": "ff6296e90bb5545592c5696e9d591fa4570fbdf31aaec58b51c70ce7d15f581d", + "pocketbase_0.39.7_linux_armv7.zip": "be8f85b0a6e5bf284e26ca6dca5d88e2ab5234d030274069020566f1b836941c", + "pocketbase_0.39.7_linux_ppc64le.zip": "a643da593e4c234bec4a91039fae4530003f9012137fab181f0d7f1799413301", + "pocketbase_0.39.7_linux_s390x.zip": "5f2dc7cf20d04b548f9f23e124670d97ab4d51ecff1215b391255c268050bed0", + "pocketbase_0.39.7_windows_amd64.zip": "0f32dfda39949cfbec38ea18fe99f1e9e49350ce1efe87537591b44dcee51517", + "pocketbase_0.39.7_windows_arm64.zip": "93ea40181210b1db7d7a885caf832cbd8945ab6ae1d43ed2c858bfc4390b4d26" +} diff --git a/agent-app/tools/package.json b/agent-app/tools/package.json index c2d4a0ce..b9d868b8 100644 --- a/agent-app/tools/package.json +++ b/agent-app/tools/package.json @@ -13,5 +13,8 @@ "devDependencies": { "@types/node": "^24.0.0", "playwright": "^1.61.1" + }, + "dependencies": { + "fflate": "0.8.3" } } diff --git a/agent-app/tools/src/commands/dev.ts b/agent-app/tools/src/commands/dev.ts index 34875cc2..a9f652b4 100644 --- a/agent-app/tools/src/commands/dev.ts +++ b/agent-app/tools/src/commands/dev.ts @@ -61,12 +61,10 @@ export async function run(args: string[]): Promise { ); children.push(pb); - // Windows: npm is npm.cmd, which Node refuses to spawn without a shell. - const isWin = process.platform === 'win32'; - const vite = spawn(isWin ? 'npm.cmd' : 'npm', ['run', 'dev'], { + const [node, npm] = osAdapter.npmCommand(); + const vite = spawn(node, [npm, 'run', 'dev'], { cwd: join(projectDir, 'frontend'), stdio: 'inherit', - shell: isWin, env: { ...process.env, VITE_PB_URL: `http://127.0.0.1:${pbPort}`, diff --git a/agent-app/tools/src/commands/pb.test.ts b/agent-app/tools/src/commands/pb.test.ts new file mode 100644 index 00000000..82ca8623 --- /dev/null +++ b/agent-app/tools/src/commands/pb.test.ts @@ -0,0 +1,150 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { test } from 'node:test'; +import { zipSync, strToU8 } from 'fflate'; +import { OSAdapter } from '../lib/os-adapter.ts'; +import { acquireDownloadLock, binaryReady, downloadArchive, ensurePbBinary } from './pb.ts'; + +test('npm is invoked as a JS entry point with the selected Node, including special paths', () => { + const root = mkdtempSync(join(tmpdir(), 'node & caf\u00e9-')); + try { + const node = join(root, 'node'); + writeFileSync(node, 'placeholder'); + const adapter = new OSAdapter(); + assert.throws(() => adapter.npmCommand(node), /Matching npm is missing/); + const directory = join(root, 'node_modules/npm/bin'); + mkdirSync(directory, { recursive: true }); + const cli = join(directory, 'npm-cli.js'); + writeFileSync(cli, 'placeholder'); + assert.deepEqual(adapter.npmCommand(node), [realpathSync(node), realpathSync(cli)]); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('npm discovery supports distro symlinks', { skip: process.platform === 'win32' }, () => { + const root = mkdtempSync(join(tmpdir(), 'node-distro-')); + try { + mkdirSync(join(root, 'bin')); + mkdirSync(join(root, 'share/npm/bin'), { recursive: true }); + const node = join(root, 'bin/node'); + const cli = join(root, 'share/npm/bin/npm-cli.js'); + writeFileSync(node, 'placeholder'); + writeFileSync(cli, 'placeholder'); + symlinkSync(cli, join(root, 'bin/npm')); + assert.deepEqual(new OSAdapter().npmCommand(node), [realpathSync(node), realpathSync(cli)]); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('extracts ZIP on each platform without tools, including Unicode and spaces', () => { + const root = mkdtempSync(join(tmpdir(), 'pb-zip-')); + const oldPath = process.env.PATH; + try { + const zip = join(root, 'input.zip'); + writeFileSync(zip, zipSync({ 'folder/hello.txt': strToU8('hello') })); + process.env.PATH = join(root, 'no-tools'); + for (const platform of ['win32', 'darwin', 'linux'] as const) { + const dest = join(root, platform, 'User Name & caf\u00e9 \u6587\u4ef6'); + new OSAdapter(platform, 'x64').extractZip(zip, dest); + assert.equal(readFileSync(join(dest, 'folder/hello.txt'), 'utf8'), 'hello'); + } + } finally { + process.env.PATH = oldPath; + rmSync(root, { recursive: true, force: true }); + } +}); + +test('rejects escaping ZIP entries before writing outside staging', () => { + const root = mkdtempSync(join(tmpdir(), 'pb-unsafe-')); + try { + for (const name of ['../outside', '..\\outside', '/absolute', 'C:/absolute']) { + const zip = join(root, 'input.zip'); + writeFileSync(zip, zipSync({ [name]: strToU8('bad') })); + assert.throws(() => new OSAdapter().extractZip(zip, join(root, 'out')), /Unsafe ZIP/); + assert.equal(existsSync(join(root, 'outside')), false); + } + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('rejects truncated ZIP data', () => { + const root = mkdtempSync(join(tmpdir(), 'pb-truncated-')); + try { + const zip = join(root, 'input.zip'); + writeFileSync(zip, zipSync({ hello: strToU8('hello') }).subarray(0, 10)); + assert.throws(() => new OSAdapter().extractZip(zip, join(root, 'out'))); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('unsupported architectures cannot silently select x64', () => { + assert.throws(() => new OSAdapter('linux', 'riscv64'), /Unsupported CPU/); + assert.throws(() => new OSAdapter('win32', 'ia32'), /Unsupported CPU/); +}); + +test('cached binaries must run and report the expected version', () => { + const root = mkdtempSync(join(tmpdir(), 'pb-binary-')); + try { + const broken = join(root, 'pocketbase.exe'); + writeFileSync(broken, ''); + assert.equal(binaryReady(broken, '0.39.7'), false); + assert.equal(binaryReady(process.execPath, process.version), true); + assert.equal(binaryReady(process.execPath, '0.39.7'), false); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('readiness check never fetches missing PocketBase', async () => { + const root = mkdtempSync(join(tmpdir(), 'pb-check-')); + const cache = process.env.AGENT_APP_PB_CACHE; + const fetch = globalThis.fetch; + try { + process.env.AGENT_APP_PB_CACHE = root; + globalThis.fetch = async () => { throw new Error('unexpected download'); }; + await assert.rejects(ensurePbBinary(false), /missing or damaged/); + } finally { + globalThis.fetch = fetch; + if (cache === undefined) delete process.env.AGENT_APP_PB_CACHE; + else process.env.AGENT_APP_PB_CACHE = cache; + rmSync(root, { recursive: true, force: true }); + } +}); + +test('download checks the pinned digest', async () => { + const root = mkdtempSync(join(tmpdir(), 'pb-download-')); + const fetch = globalThis.fetch; + try { + const data = Buffer.from('archive'); + globalThis.fetch = async () => new Response(data); + await downloadArchive('https://example.test/pb.zip', join(root, 'ok'), createHash('sha256').update(data).digest('hex')); + await assert.rejects(downloadArchive('https://example.test/pb.zip', join(root, 'bad'), '0'.repeat(64)), /SHA-256/); + } finally { globalThis.fetch = fetch; rmSync(root, { recursive: true, force: true }); } +}); + +test('simultaneous downloads serialize across lock owners', async () => { + const root = mkdtempSync(join(tmpdir(), 'pb-lock-')); + try { + const lock = join(root, '.lock'); + const release = await acquireDownloadLock(lock); + let entered = false; + const next = acquireDownloadLock(lock).then((unlock) => { entered = true; return unlock; }); + await new Promise((resolve) => setTimeout(resolve, 150)); + assert.equal(entered, false); + release(); + const unlock = await next; + assert.equal(entered, true); + unlock(); + assert.equal(existsSync(lock), false); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('empty lock left by an interrupted writer can be recovered', async () => { + const { utimesSync } = await import('node:fs'); + const root = mkdtempSync(join(tmpdir(), 'pb-stale-')); + try { + const lock = join(root, '.lock'); + writeFileSync(lock, ''); + utimesSync(lock, new Date(0), new Date(0)); + const release = await acquireDownloadLock(lock); + release(); + assert.equal(existsSync(lock), false); + } finally { rmSync(root, { recursive: true, force: true }); } +}); diff --git a/agent-app/tools/src/commands/pb.ts b/agent-app/tools/src/commands/pb.ts index c4557975..e7a8edde 100644 --- a/agent-app/tools/src/commands/pb.ts +++ b/agent-app/tools/src/commands/pb.ts @@ -1,56 +1,126 @@ -/** - * agent-app pb — manage the pinned PocketBase binary. - * - * agent-app pb fetch download + cache the pinned version for this OS/arch - * agent-app pb path print the cached binary path (fetches if missing) - * agent-app pb version print the pinned version - */ +/** Pinned PocketBase downloads shared by setup, repair and app imports. */ import { execFileSync } from 'node:child_process'; -import { chmodSync, createWriteStream, existsSync } from 'node:fs'; -import { unlink } from 'node:fs/promises'; +import { createHash } from 'node:crypto'; +import { chmodSync, closeSync, createReadStream, createWriteStream, existsSync, mkdirSync, mkdtempSync, openSync, readFileSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; -import { Readable } from 'node:stream'; +import { Readable, Transform } from 'node:stream'; import type { ReadableStream as WebReadableStream } from 'node:stream/web'; import { pipeline } from 'node:stream/promises'; +import { setTimeout as delay } from 'node:timers/promises'; import { log } from '../lib/log.ts'; import { osAdapter } from '../lib/os-adapter.ts'; -import { pinnedPbVersion } from '../lib/paths.ts'; +import { pinnedPbVersion, workspaceRoot } from '../lib/paths.ts'; const RELEASE_BASE = 'https://github.com/pocketbase/pocketbase/releases/download'; -export async function ensurePbBinary(): Promise { - const version = pinnedPbVersion(); - const cacheDir = osAdapter.pbCacheDir(version); - const binPath = join(cacheDir, osAdapter.pbBinaryName()); - - if (existsSync(binPath)) return binPath; +export function binaryReady(bin: string, version: string): boolean { + try { + if (!existsSync(bin) || statSync(bin).size === 0) return false; + const reported = execFileSync(bin, ['--version'], { encoding: 'utf8', timeout: 15000, windowsHide: true, stdio: 'pipe' }).trim(); + return reported.split(/\s+/).at(-1) === version; + } catch { return false; } +} - const asset = osAdapter.pbAssetName(version); - const url = `${RELEASE_BASE}/v${version}/${asset}`; - const zipPath = join(cacheDir, asset); +/** Exclusive across CLI processes. Dead owners are recoverable after a crash. */ +export async function acquireDownloadLock(path: string): Promise<() => void> { + const token = `${process.pid}:${Date.now()}:${Math.random()}`; + const deadline = Date.now() + 300000; + while (true) { + try { + const fd = openSync(path, 'wx'); + try { writeFileSync(fd, token); } finally { closeSync(fd); } + return () => { + try { if (readFileSync(path, 'utf8') === token) unlinkSync(path); } catch { /* already gone */ } + }; + } catch (err) { + if ((err as NodeJS.ErrnoException).code !== 'EEXIST') throw err; + } + try { + const owner = readFileSync(path, 'utf8'); + const pid = Number(owner.split(':')[0]); + let dead = false; + if (Number.isInteger(pid) && pid > 0) { + try { process.kill(pid, 0); } + catch (err) { dead = (err as NodeJS.ErrnoException).code === 'ESRCH'; } + } else { + // A process could die between creating the file and writing its PID. + dead = Date.now() - statSync(path).mtimeMs > 10000; + } + if (dead && readFileSync(path, 'utf8') === owner) { unlinkSync(path); continue; } + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') continue; + throw err; + } + if (Date.now() >= deadline) throw new Error('Another PocketBase download is still running. Retry when it finishes.'); + await delay(100); + } +} - log.step(`Downloading PocketBase v${version} (${asset})…`); - const res = await fetch(url, { redirect: 'follow' }); - if (!res.ok || res.body === null) { - throw new Error(`Download failed (${res.status}) — ${url}`); +export async function downloadArchive(url: string, dest: string, expectedHash: string): Promise { + for (let attempt = 0; attempt < 3; attempt++) { + const controller = new AbortController(); + let idle = setTimeout(() => controller.abort(), 120000); + try { + const res = await fetch(url, { redirect: 'follow', signal: controller.signal }); + if (!res.ok || res.body === null) { + throw new Error(`PocketBase download failed (HTTP ${res.status}): ${url}`); + } + // Bound inactivity rather than total transfer time: slow links can still finish. + const watchdog = new Transform({ + transform(chunk, _encoding, done) { + clearTimeout(idle); + idle = setTimeout(() => controller.abort(), 120000); + done(null, chunk); + }, + }); + await pipeline(Readable.fromWeb(res.body as unknown as WebReadableStream), watchdog, createWriteStream(dest)); + break; + } catch (err) { + if (attempt === 2 || /HTTP 4(?!08|29)/.test(String(err))) throw err; + log.step(`PocketBase download interrupted; retrying (${attempt + 2}/3)…`); + await delay(1000 * (attempt + 1)); + } finally { + clearTimeout(idle); + } } - await pipeline( - Readable.fromWeb(res.body as unknown as WebReadableStream), - createWriteStream(zipPath), - ); + const digest = createHash('sha256'); + for await (const chunk of createReadStream(dest)) digest.update(chunk as Buffer); + if (digest.digest('hex') !== expectedHash) throw new Error('PocketBase download failed SHA-256 verification; no executable was installed.'); +} - log.step('Extracting…'); - osAdapter.extractZip(zipPath, cacheDir); - await unlink(zipPath); +export async function ensurePbBinary(download = true): Promise { + const version = pinnedPbVersion(); + const cacheDir = osAdapter.pbCacheDir(version); + const binPath = join(cacheDir, osAdapter.pbBinaryName()); + if (binaryReady(binPath, version)) return binPath; + if (!download) throw new Error(`PocketBase ${version} is missing or damaged. Run setup or Repair.`); - if (!existsSync(binPath)) { - throw new Error(`Extraction did not produce ${binPath}`); + const release = await acquireDownloadLock(join(cacheDir, '.download.lock')); + let staging: string | undefined; + try { + if (binaryReady(binPath, version)) return binPath; + const asset = osAdapter.pbAssetName(version); + const hashes = JSON.parse(readFileSync(join(workspaceRoot(), 'spec', 'pocketbase-checksums.json'), 'utf8')) as Record; + const expected = hashes[asset]; + if (!expected || !/^[a-f0-9]{64}$/.test(expected)) throw new Error(`No approved PocketBase checksum for ${asset}`); + staging = mkdtempSync(join(cacheDir, '.download-')); + const zip = join(staging, asset); + log.step(`Downloading PocketBase v${version} (${asset})…`); + await downloadArchive(`${RELEASE_BASE}/v${version}/${asset}`, zip, expected); + const unpacked = join(staging, 'unpacked'); + mkdirSync(unpacked); + log.step('Extracting PocketBase…'); + osAdapter.extractZip(zip, unpacked); + const candidate = join(unpacked, osAdapter.pbBinaryName()); + if (osAdapter.platform !== 'win32' && existsSync(candidate)) chmodSync(candidate, 0o755); + if (!binaryReady(candidate, version)) throw new Error(`PocketBase ${version} failed its execution check on ${osAdapter.platform}/${osAdapter.arch}`); + renameSync(candidate, binPath); + log.ok(`Cached ${binPath} (version ${version})`); + return binPath; + } finally { + try { if (staging) rmSync(staging, { recursive: true, force: true }); } + finally { release(); } } - if (osAdapter.platform !== 'win32') chmodSync(binPath, 0o755); - - const reported = execFileSync(binPath, ['--version'], { encoding: 'utf8' }).trim(); - log.ok(`Cached ${binPath} (${reported})`); - return binPath; } export async function run(args: string[]): Promise { @@ -65,13 +135,17 @@ export async function run(args: string[]): Promise { log.raw(await ensurePbBinary()); return 0; } + case 'check': { + log.raw(await ensurePbBinary(false)); + return 0; + } case 'version': { log.raw(pinnedPbVersion()); return 0; } default: log.error(`Unknown subcommand: pb ${sub}`); - log.raw('Try: agent-app pb fetch | path | version'); + log.raw('Try: agent-app pb fetch | path | check | version'); return 1; } } diff --git a/agent-app/tools/src/commands/validate.ts b/agent-app/tools/src/commands/validate.ts index 31dad08e..5cb4908c 100644 --- a/agent-app/tools/src/commands/validate.ts +++ b/agent-app/tools/src/commands/validate.ts @@ -18,6 +18,7 @@ import { join } from 'node:path'; import { fileMatchesCanon, recordFileHash } from '../lib/hashes.ts'; import { log } from '../lib/log.ts'; import { ensurePbBinary } from './pb.ts'; +import { osAdapter } from '../lib/os-adapter.ts'; interface GateError { step: string; @@ -827,11 +828,10 @@ export async function run(args: string[]): Promise { const frontendDir = join(projectDir, 'frontend'); const errors: GateError[] = []; - // Windows: npm is npm.cmd, which Node refuses to spawn without a shell. - const isWin = process.platform === 'win32'; const npmRun = (script: string, extra: string[] = []): void => { - execFileSync(isWin ? 'npm.cmd' : 'npm', ['run', script, ...extra], { - cwd: frontendDir, stdio: 'pipe', encoding: 'utf8', shell: isWin, + const [node, npm] = osAdapter.npmCommand(); + execFileSync(node, [npm, 'run', script, ...extra], { + cwd: frontendDir, stdio: 'pipe', encoding: 'utf8', windowsHide: true, }); }; diff --git a/agent-app/tools/src/lib/os-adapter.ts b/agent-app/tools/src/lib/os-adapter.ts index e75ba56d..00068fa6 100644 --- a/agent-app/tools/src/lib/os-adapter.ts +++ b/agent-app/tools/src/lib/os-adapter.ts @@ -3,9 +3,10 @@ * Commands never branch on process.platform; they ask the adapter. */ import { execFileSync } from 'node:child_process'; -import { existsSync, mkdirSync } from 'node:fs'; +import { existsSync, lstatSync, mkdirSync, readFileSync, realpathSync, statSync, writeFileSync } from 'node:fs'; import { homedir } from 'node:os'; -import { join } from 'node:path'; +import { dirname, join, resolve, sep } from 'node:path'; +import { unzipSync } from 'fflate'; export type Platform = 'darwin' | 'linux' | 'win32'; @@ -18,6 +19,9 @@ export class OSAdapter { throw new Error(`Unsupported platform: ${platform}`); } this.platform = platform; + if (arch !== 'x64' && arch !== 'arm64') { + throw new Error(`Unsupported CPU architecture: ${arch}`); + } this.arch = arch === 'arm64' ? 'arm64' : 'amd64'; } @@ -33,7 +37,7 @@ export class OSAdapter { /** Central per-host PocketBase binary cache, versioned (spec B1). */ pbCacheDir(version: string): string { const base = process.env['AGENT_APP_PB_CACHE'] ?? join(this.cacheRoot(), 'pb'); - const dir = join(base, version); + const dir = join(base, version, `${this.platform}-${this.arch}`); mkdirSync(dir, { recursive: true }); return dir; } @@ -56,20 +60,50 @@ export class OSAdapter { return this.platform === 'win32' ? 'pocketbase.exe' : 'pocketbase'; } - /** - * Extract a zip without npm deps: bsdtar handles zip on macOS and Windows 10+; - * on Linux prefer unzip, fall back to bsdtar. - */ + /** Use the npm belonging to this Node; .cmd wrappers require a shell. */ + npmCommand(nodeExecutable: string = process.execPath): [string, string] { + const node = realpathSync(nodeExecutable); + const bin = dirname(node); + const candidates = [join(bin, 'node_modules/npm/bin/npm-cli.js'), join(bin, '../lib/node_modules/npm/bin/npm-cli.js')]; + // Distro packages may link /usr/bin/npm to /usr/share/nodejs/npm/bin/. + const sibling = join(bin, 'npm'); + if (existsSync(sibling)) candidates.push(join(dirname(realpathSync(sibling)), 'npm-cli.js')); + const cli = candidates + .find((candidate) => existsSync(candidate)); + if (!cli) throw new Error('Matching npm is missing. Run CraftBot Repair to restore Node/npm.'); + return [node, cli]; + } + + /** Extract internally: no PATH tools, shell, or platform-specific encodings. */ extractZip(zipPath: string, destDir: string): void { - if (this.platform === 'linux') { - try { - execFileSync('unzip', ['-o', zipPath, '-d', destDir], { stdio: 'pipe' }); - return; - } catch { - // fall through to tar + const limit = 128 * 1024 * 1024; + if (statSync(zipPath).size > limit) throw new Error('ZIP archive exceeds size limit'); + const root = resolve(destDir); + let total = 0; + const entries = unzipSync(readFileSync(zipPath), { + filter: (entry) => { + total += entry.originalSize; + const name = entry.name.replaceAll('\\', '/'); + if (total > limit) throw new Error('ZIP contents exceed size limit'); + if (name.startsWith('/') || name.includes(':') || name.split('/').includes('..')) { + throw new Error(`Unsafe ZIP entry: ${entry.name}`); + } + return true; + }, + }); + for (const [name, contents] of Object.entries(entries)) { + const output = resolve(root, name.replaceAll('\\', '/')); + if (output !== root && !output.startsWith(root + sep)) throw new Error(`Unsafe ZIP entry: ${name}`); + // Never follow an existing symlink while extracting. + for (let path = output; path !== root; path = dirname(path)) { + if (existsSync(path) && lstatSync(path).isSymbolicLink()) throw new Error(`Symlink in ZIP destination: ${path}`); + } + if (name.endsWith('/')) mkdirSync(output, { recursive: true }); + else { + mkdirSync(dirname(output), { recursive: true }); + writeFileSync(output, contents); } } - execFileSync('tar', ['-xf', zipPath, '-C', destDir], { stdio: 'pipe' }); } /** Terminate a process tree. */ diff --git a/app/agent_app/agent_view.py b/app/agent_app/agent_view.py index 7ea2becf..02c4c883 100644 --- a/app/agent_app/agent_view.py +++ b/app/agent_app/agent_view.py @@ -130,9 +130,13 @@ def operations_block(base_url: str, max_chars: int = 6000) -> Optional[str]: area, verb = name, name params = op.get("params") or {} required = [ - k for k, spec in params.items() if isinstance(spec, dict) and spec.get("required") + k + for k, spec in params.items() + if isinstance(spec, dict) and spec.get("required") ] - groups.setdefault(area, []).append((verb, required, bool(op.get("destructive")))) + groups.setdefault(area, []).append( + (verb, required, bool(op.get("destructive"))) + ) if not groups: return None diff --git a/app/agent_app/manager.py b/app/agent_app/manager.py index e33f1ea0..20179ced 100644 --- a/app/agent_app/manager.py +++ b/app/agent_app/manager.py @@ -4185,7 +4185,9 @@ async def _wait_for_health_check( """Read-only HTTP probes with a total deadline; never block the event loop.""" import aiohttp - async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=2)) as client: + async with aiohttp.ClientSession( + timeout=aiohttp.ClientTimeout(total=2) + ) as client: async def probe(): try: diff --git a/app/agent_app/runner.py b/app/agent_app/runner.py index 198a138e..07c32582 100644 --- a/app/agent_app/runner.py +++ b/app/agent_app/runner.py @@ -126,10 +126,9 @@ def ensure_available(self) -> None: ) raise AgentAppRunnerUnavailable( f"Node.js >= {MIN_NODE_MAJOR} is required to build Agent Apps " - f"and none was found{hint}. CraftBot never upgrades your " - "default Node: run `python install.py` to add a side-by-side " - "Node (nvm install 24 also works), or point the CRAFTBOT_NODE " - "env var at a >= 24 binary, then restart." + f"with working npm and none was found{hint}. Use Repair in " + "the installer to restore CraftBot's private Node/npm. For " + "a source checkout, run `python craftbot.py install`, then restart." ) major = node_runtime.major_of(rt.version) if major is not None and major < MIN_NODE_MAJOR: @@ -254,13 +253,16 @@ async def install(self, project_dir: Path) -> None: frontend = project_dir / "frontend" if (frontend / "node_modules").exists(): return - # Windows: bare "npm" is npm.cmd — CreateProcess only finds it via a - # full path, so never spawn the bare name. - npm = node_runtime.npm_cmd() or "npm" + # Invoke the selected Node's npm JS directly, without a Windows shell. + npm = node_runtime.npm_command() + if not npm: + raise AgentAppRunnerUnavailable( + "npm is unavailable. Run CraftBot Repair to restore the Node runtime." + ) code, out = await self._run( # --ignore-scripts: any npm package is allowed in a project, so # lifecycle scripts must never run (supply-chain guard, spec B7). - [npm, "install", "--no-audit", "--no-fund", "--ignore-scripts"], + npm + ["install", "--no-audit", "--no-fund", "--ignore-scripts"], timeout=INSTALL_TIMEOUT_S, cwd=frontend, ) @@ -413,7 +415,10 @@ async def ensure_superuser( # PocketBase's CLI exits 0 even when a command fails, so the exit # code alone proves nothing: the save is confirmed from the database # itself (the superuser row exists and its `updated` stamp moved). - saved = code == 0 and _superuser_stamp(db_path, email) not in (None, stamp_before) + saved = code == 0 and _superuser_stamp(db_path, email) not in ( + None, + stamp_before, + ) if not saved: # FAIL CLOSED. A serve without a superuser makes PocketBase pop # the user's SYSTEM BROWSER with a one-time /_/#/pbinstall/… diff --git a/app/agent_app/test_a2app_native_auth.py b/app/agent_app/test_a2app_native_auth.py index 4b708574..2f97f2d7 100644 --- a/app/agent_app/test_a2app_native_auth.py +++ b/app/agent_app/test_a2app_native_auth.py @@ -19,6 +19,7 @@ import json import os +import platform import shutil import socket import subprocess @@ -71,9 +72,13 @@ def _pinned_pb_binary() -> Path: root = Path(cache) elif os.name == "nt": root = Path(os.environ["LOCALAPPDATA"]) / "craftos-agent-app" / "pb" + elif sys.platform == "darwin": + root = Path.home() / "Library/Caches/craftos-agent-app/pb" else: root = Path.home() / ".cache" / "craftos-agent-app" / "pb" - return root / version / ("pocketbase.exe" if os.name == "nt" else "pocketbase") + arch = "arm64" if platform.machine().lower() in ("arm64", "aarch64") else "amd64" + exe = "pocketbase.exe" if os.name == "nt" else "pocketbase" + return root / version / f"{sys.platform}-{arch}" / exe class _NoRedirect(urllib.request.HTTPRedirectHandler): diff --git a/app/agent_app/test_backups.py b/app/agent_app/test_backups.py index 9b2e0dc2..fde822be 100644 --- a/app/agent_app/test_backups.py +++ b/app/agent_app/test_backups.py @@ -619,6 +619,8 @@ async def _flaky_relaunch(pid): # build on this machine). Skipped when absent — §1-§7 stay deterministic. import json as _json # noqa: E402 import os as _os # noqa: E402 +import platform as _platform # noqa: E402 +import sys as _sys # noqa: E402 import socket as _socket # noqa: E402 import subprocess as _sp # noqa: E402 import time as _time # noqa: E402 @@ -641,10 +643,13 @@ def _pinned_pb_binary() -> Path: root = Path(cache) elif _os.name == "nt": root = Path(_os.environ["LOCALAPPDATA"]) / "craftos-agent-app" / "pb" + elif _sys.platform == "darwin": + root = Path.home() / "Library/Caches/craftos-agent-app/pb" else: root = Path.home() / ".cache" / "craftos-agent-app" / "pb" exe = "pocketbase.exe" if _os.name == "nt" else "pocketbase" - return root / version / exe + arch = "arm64" if _platform.machine().lower() in ("arm64", "aarch64") else "amd64" + return root / version / f"{_sys.platform}-{arch}" / exe _pb_bin = _pinned_pb_binary() diff --git a/app/credentials/api_request.py b/app/credentials/api_request.py index d3a40658..e2488f10 100644 --- a/app/credentials/api_request.py +++ b/app/credentials/api_request.py @@ -23,16 +23,38 @@ _MAX_BODY_BYTES = 256 * 1024 _HEADER_NAME = re.compile(r"^[A-Za-z0-9!#$%&'*+.^_`|~-]{1,100}$") -_RESERVED_HEADERS = frozenset({ - "host", "connection", "content-length", "transfer-encoding", "trailer", - "upgrade", "proxy-authorization", "proxy-connection", "cookie", - "accept-encoding", "te", -}) +_RESERVED_HEADERS = frozenset( + { + "host", + "connection", + "content-length", + "transfer-encoding", + "trailer", + "upgrade", + "proxy-authorization", + "proxy-connection", + "cookie", + "accept-encoding", + "te", + } +) _METHODS = frozenset({"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"}) -_INPUTS = frozenset({ - "credential_id", "url", "method", "auth_type", "header_name", "headers", - "params", "json", "data", "timeout", "simulated_mode", "_session_id", -}) +_INPUTS = frozenset( + { + "credential_id", + "url", + "method", + "auth_type", + "header_name", + "headers", + "params", + "json", + "data", + "timeout", + "simulated_mode", + "_session_id", + } +) def _error(code: str, message: str) -> dict: @@ -43,28 +65,49 @@ def _list_keys(data: dict) -> dict: try: query = data.get("query", "") if not isinstance(query, str) or len(query) > 200: - return _error("CREDENTIAL_INVALID_INPUT", "Use a search query of at most 200 characters.") + return _error( + "CREDENTIAL_INVALID_INPUT", + "Use a search query of at most 200 characters.", + ) vault = get_vault() status = vault.status() if not status.get("ok"): - return _error("CREDENTIAL_VAULT_UNAVAILABLE", "Check the credential vault in Settings.") + return _error( + "CREDENTIAL_VAULT_UNAVAILABLE", + "Check the credential vault in Settings.", + ) query = query.strip().casefold() entries = [ - {key: entry.get(key, "other" if key == "group" else "") - for key in ("id", "label", "site", "group")} + { + key: entry.get(key, "other" if key == "group" else "") + for key in ("id", "label", "site", "group") + } for entry in vault.list_entries() if entry.get("kind") == "api_key" - and (not query or query in " ".join(str(entry.get(k, "")) for k in ("label", "site", "group")).casefold()) + and ( + not query + or query + in " ".join( + str(entry.get(k, "")) for k in ("label", "site", "group") + ).casefold() + ) ] return {"status": "success", "credentials": entries} except Exception: - return _error("CREDENTIAL_VAULT_UNAVAILABLE", "Saved API keys are unavailable. Check Settings > Credentials.") + return _error( + "CREDENTIAL_VAULT_UNAVAILABLE", + "Saved API keys are unavailable. Check Settings > Credentials.", + ) def _valid_header(name: object, value: object) -> bool: - return (isinstance(name, str) and bool(_HEADER_NAME.fullmatch(name)) - and isinstance(value, str) and len(value) <= 8192 - and not any(ord(c) < 32 or ord(c) == 127 for c in value)) + return ( + isinstance(name, str) + and bool(_HEADER_NAME.fullmatch(name)) + and isinstance(value, str) + and len(value) <= 8192 + and not any(ord(c) < 32 or ord(c) == 127 for c in value) + ) def _charset(content_type: str) -> str: @@ -92,15 +135,23 @@ def _redact(text: str, secret: Secret) -> str: value = secret.reveal() raw = value.encode("utf-8") variants = { - value, quote(value, safe=""), quote_plus(value, safe=""), - html.escape(value), html.escape(value, quote=False), + value, + quote(value, safe=""), + quote_plus(value, safe=""), + html.escape(value), + html.escape(value, quote=False), json.dumps(value, ensure_ascii=True)[1:-1], json.dumps(value, ensure_ascii=False)[1:-1], - base64.b64encode(raw).decode(), base64.urlsafe_b64encode(raw).decode(), - raw.hex(), raw.hex().upper(), + base64.b64encode(raw).decode(), + base64.urlsafe_b64encode(raw).decode(), + raw.hex(), + raw.hex().upper(), + } + variants |= { + re.sub(r"%[0-9A-F]{2}", lambda m: m[0].lower(), value) + for value in variants + if "%" in value } - variants |= {re.sub(r"%[0-9A-F]{2}", lambda m: m[0].lower(), value) - for value in variants if "%" in value} # Also catch JSON unicode escapes, escaped slashes and response object keys. try: decoded = json.loads(text) @@ -120,7 +171,11 @@ def clean(item): return {clean(key): clean(child) for key, child in item.items()} return item - return clean(json.dumps(clean(decoded), ensure_ascii=False)) if decoded is not None else clean(text) + return ( + clean(json.dumps(clean(decoded), ensure_ascii=False)) + if decoded is not None + else clean(text) + ) def _request(data: dict) -> dict: @@ -131,64 +186,155 @@ def _request(data: dict) -> dict: credential_id = data.get("credential_id") url = data.get("url") method = data.get("method", "GET") - if not isinstance(credential_id, str) or not credential_id or len(credential_id) > 100: - return _error("CREDENTIAL_INVALID_INPUT", "Choose an ID from credential_list_api_keys.") - if (not isinstance(url, str) or len(url) > 8192 or not url - or any(ord(c) <= 32 or ord(c) == 127 for c in url) or "\\" in url): - return _error("CREDENTIAL_INVALID_INPUT", "Use an absolute HTTPS URL without whitespace or backslashes.") + if ( + not isinstance(credential_id, str) + or not credential_id + or len(credential_id) > 100 + ): + return _error( + "CREDENTIAL_INVALID_INPUT", + "Choose an ID from credential_list_api_keys.", + ) + if ( + not isinstance(url, str) + or len(url) > 8192 + or not url + or any(ord(c) <= 32 or ord(c) == 127 for c in url) + or "\\" in url + ): + return _error( + "CREDENTIAL_INVALID_INPUT", + "Use an absolute HTTPS URL without whitespace or backslashes.", + ) parts = urlsplit(url) - if parts.scheme not in ("https", "http") or not parts.hostname or parts.username is not None or parts.password is not None or parts.fragment: - return _error("CREDENTIAL_INVALID_INPUT", "URLs cannot contain user information or fragments.") + if ( + parts.scheme not in ("https", "http") + or not parts.hostname + or parts.username is not None + or parts.password is not None + or parts.fragment + ): + return _error( + "CREDENTIAL_INVALID_INPUT", + "URLs cannot contain user information or fragments.", + ) if not isinstance(method, str) or method not in _METHODS: return _error("CREDENTIAL_INVALID_INPUT", "Choose a supported HTTP method.") auth_type = data.get("auth_type", "bearer") - header_name = data.get("header_name", "Authorization" if auth_type == "bearer" else "X-API-Key") - if auth_type not in ("bearer", "header") or not _valid_header(header_name, "") or header_name.lower() in _RESERVED_HEADERS: - return _error("CREDENTIAL_INVALID_INPUT", "Use bearer authentication or a valid API-key header.") + header_name = data.get( + "header_name", "Authorization" if auth_type == "bearer" else "X-API-Key" + ) + if ( + auth_type not in ("bearer", "header") + or not _valid_header(header_name, "") + or header_name.lower() in _RESERVED_HEADERS + ): + return _error( + "CREDENTIAL_INVALID_INPUT", + "Use bearer authentication or a valid API-key header.", + ) headers = data.get("headers", {}) - if not isinstance(headers, dict) or len(headers) > 50 or any( - not _valid_header(k, v) or k.lower() in _RESERVED_HEADERS | {"authorization", header_name.lower()} - for k, v in headers.items() + if ( + not isinstance(headers, dict) + or len(headers) > 50 + or any( + not _valid_header(k, v) + or k.lower() + in _RESERVED_HEADERS | {"authorization", header_name.lower()} + for k, v in headers.items() + ) ): - return _error("CREDENTIAL_INVALID_INPUT", "Invalid or conflicting request headers.") + return _error( + "CREDENTIAL_INVALID_INPUT", "Invalid or conflicting request headers." + ) params = data.get("params", {}) if not isinstance(params, dict) or len(params) > 100: - return _error("CREDENTIAL_INVALID_INPUT", "Query parameters must be an object of at most 100 fields.") + return _error( + "CREDENTIAL_INVALID_INPUT", + "Query parameters must be an object of at most 100 fields.", + ) if "json" in data and "data" in data: - return _error("CREDENTIAL_INVALID_INPUT", "Send either a JSON body or a text body.") + return _error( + "CREDENTIAL_INVALID_INPUT", "Send either a JSON body or a text body." + ) if "data" in data and not isinstance(data["data"], str): return _error("CREDENTIAL_INVALID_INPUT", "The raw body must be text.") - if len(json.dumps({k: data[k] for k in ("json", "data", "params") if k in data}).encode()) > _MAX_BODY_BYTES: - return _error("CREDENTIAL_INVALID_INPUT", "The request body or query is too large.") + if ( + len( + json.dumps( + {k: data[k] for k in ("json", "data", "params") if k in data} + ).encode() + ) + > _MAX_BODY_BYTES + ): + return _error( + "CREDENTIAL_INVALID_INPUT", "The request body or query is too large." + ) timeout = data.get("timeout", 30) - if isinstance(timeout, bool) or not isinstance(timeout, (int, float)) or not math.isfinite(timeout) or not 1 <= timeout <= 60: - return _error("CREDENTIAL_INVALID_INPUT", "Timeout must be between 1 and 60 seconds.") + if ( + isinstance(timeout, bool) + or not isinstance(timeout, (int, float)) + or not math.isfinite(timeout) + or not 1 <= timeout <= 60 + ): + return _error( + "CREDENTIAL_INVALID_INPUT", "Timeout must be between 1 and 60 seconds." + ) with requests.Session() as session: session.trust_env = False - prepared = session.prepare_request(requests.Request( - method, url, headers=headers, params=params, - json=data.get("json"), data=data.get("data"), - )) + prepared = session.prepare_request( + requests.Request( + method, + url, + headers=headers, + params=params, + json=data.get("json"), + data=data.get("data"), + ) + ) # Check the actual transport URL after requests has canonicalized it. secret = get_vault().api_key_for_url(credential_id, prepared.url) - prepared.headers[header_name] = ("Bearer " if auth_type == "bearer" else "") + secret.reveal() - with session.send(prepared, timeout=timeout, allow_redirects=False, verify=True, stream=True) as response: + prepared.headers[header_name] = ( + "Bearer " if auth_type == "bearer" else "" + ) + secret.reveal() + with session.send( + prepared, + timeout=timeout, + allow_redirects=False, + verify=True, + stream=True, + ) as response: # Real redirects only (a Location to follow): a 304 Not # Modified is an ordinary answer to a conditional request. if response.is_redirect: - return _error("CREDENTIAL_REDIRECT_REFUSED", "Redirect refused. Use a direct URL on the saved website.") | {"status_code": response.status_code} + return _error( + "CREDENTIAL_REDIRECT_REFUSED", + "Redirect refused. Use a direct URL on the saved website.", + ) | {"status_code": response.status_code} body = bytearray() for chunk in response.iter_content(chunk_size=16384): body.extend(chunk) if len(body) > _MAX_BODY_BYTES: # Never return a truncated value that could expose half a key. - return _error("CREDENTIAL_RESPONSE_TOO_LARGE", "Response exceeded 256 KiB; no body was returned.") + return _error( + "CREDENTIAL_RESPONSE_TOO_LARGE", + "Response exceeded 256 KiB; no body was returned.", + ) content_type_header = response.headers.get("Content-Type", "") content_type = content_type_header.split(";", 1)[0].lower() - if content_type and not (content_type.startswith("text/") or content_type.endswith(("json", "+json", "xml", "+xml"))): - return _error("CREDENTIAL_RESPONSE_TYPE", "Only text and JSON API responses are returned.") - text = _redact(bytes(body).decode(_charset(content_type_header), errors="replace"), secret) + if content_type and not ( + content_type.startswith("text/") + or content_type.endswith(("json", "+json", "xml", "+xml")) + ): + return _error( + "CREDENTIAL_RESPONSE_TYPE", + "Only text and JSON API responses are returned.", + ) + text = _redact( + bytes(body).decode(_charset(content_type_header), errors="replace"), + secret, + ) return { "status": "success" if response.status_code < 400 else "error", "status_code": response.status_code, @@ -197,9 +343,14 @@ def _request(data: dict) -> dict: } except VaultError as exc: # Vault details are fixed validation messages, never a key or request. - return _error("CREDENTIAL_UNAVAILABLE", exc.detail or "Check Settings > Credentials.") + return _error( + "CREDENTIAL_UNAVAILABLE", exc.detail or "Check Settings > Credentials." + ) except Exception: - return _error("CREDENTIAL_REQUEST_FAILED", "The authenticated request failed. Check the endpoint, connection and saved key.") + return _error( + "CREDENTIAL_REQUEST_FAILED", + "The authenticated request failed. Check the endpoint, connection and saved key.", + ) async def list_api_keys(data: dict) -> dict: diff --git a/app/credentials/vault.py b/app/credentials/vault.py index e12eef24..23cd97dc 100644 --- a/app/credentials/vault.py +++ b/app/credentials/vault.py @@ -693,7 +693,9 @@ def reveal_entry(self, entry_id: str) -> Secret: if entry is None: raise _invalid(_MSG_NOT_FOUND) if entry.get("kind", "login") not in ("api_key", "secure_note"): - raise _invalid("Login passwords are available only to browser autofill.") + raise _invalid( + "Login passwords are available only to browser autofill." + ) return Secret(entry["password"]) def api_key_for_url(self, entry_id: str, url: str) -> Secret: @@ -722,9 +724,15 @@ def api_key_for_url(self, entry_id: str, url: str) -> Secret: raise _invalid("Choose a saved API key.") saved = _saved_parts(entry["site"], keep_www=True) if saved is None: - raise _invalid("Add a website to this API key in Settings > Credentials first.") - if host != saved[0] or target_port != (saved[1] or (443 if scheme == "https" else 80)): - raise _invalid("The request must match the API key's saved website and port.") + raise _invalid( + "Add a website to this API key in Settings > Credentials first." + ) + if host != saved[0] or target_port != ( + saved[1] or (443 if scheme == "https" else 80) + ): + raise _invalid( + "The request must match the API key's saved website and port." + ) return Secret(entry["password"]) def delete_entry(self, entry_id: str) -> bool: @@ -988,8 +996,13 @@ def _clean_site(site: Any, kind: str) -> str: def _new_fields( - site: Any, username: Any, password: Secret, label: Any, - *, kind: str = "login", group: Optional[str] = None, + site: Any, + username: Any, + password: Secret, + label: Any, + *, + kind: str = "login", + group: Optional[str] = None, ) -> _Entry: kind = _clean_choice(kind, CREDENTIAL_KINDS, "type") clean_site = _clean_site(site, kind) @@ -1014,14 +1027,21 @@ def _new_fields( def _changed_fields( - site: Any, username: Any, password: Optional[Secret], label: Any, - *, kind: str = "login", group: Optional[str] = None, + site: Any, + username: Any, + password: Optional[Secret], + label: Any, + *, + kind: str = "login", + group: Optional[str] = None, ) -> _Entry: changes = _Entry() if site is not None: changes["site"] = _clean_site(site, kind) if username is not None: - changes["username"] = _clean_username(username) if kind == "login" or username else "" + changes["username"] = ( + _clean_username(username) if kind == "login" or username else "" + ) if password: # None or "" keeps the current password problem = _password_problem(password, kind) if problem: @@ -1050,7 +1070,9 @@ def _update_fields( current_kind = entry.get("kind", "login") if kind is not None and kind != current_kind: raise _invalid("The credential type cannot be changed. Add a new item instead.") - changes = _changed_fields(site, username, password, label, kind=current_kind, group=group) + changes = _changed_fields( + site, username, password, label, kind=current_kind, group=group + ) if current_kind != "login" and not changes.get("label", entry.get("label")): raise _invalid("Enter a name for this credential.") return _apply_update(entry_id, changes, entries) @@ -1084,9 +1106,9 @@ def _upsert(new: _Entry, entries: List[_Entry]) -> Tuple[Dict[str, Any], bool]: lastUsedAt=None, ) entries.append(entry) - for field in ("kind", "group"): - if field in new: - entry[field] = new[field] + for field_name in ("kind", "group"): + if field_name in new: + entry[field_name] = new[field_name] return _public(entry), True @@ -1101,9 +1123,12 @@ def _apply_update( username = changes.get("username", entry["username"]) key = (site, username.casefold()) for other in entries: - if (entry.get("kind", "login") == "login" - and other.get("kind", "login") == "login" and other is not entry - and (other["site"], other["username"].casefold()) == key): + if ( + entry.get("kind", "login") == "login" + and other.get("kind", "login") == "login" + and other is not entry + and (other["site"], other["username"].casefold()) == key + ): raise _invalid(_MSG_DUPLICATE) changed = False for name, value in changes.items(): @@ -1142,9 +1167,9 @@ def _public(entry: Dict[str, Any]) -> Dict[str, Any]: "updatedAt": entry.get("updatedAt") or None, "lastUsedAt": entry.get("lastUsedAt") or None, } - for field in ("kind", "group"): - if field in entry: - public[field] = entry[field] + for field_name in ("kind", "group"): + if field_name in entry: + public[field_name] = entry[field_name] return public @@ -1255,11 +1280,15 @@ def _parse_document(plaintext: Secret) -> List[_Entry]: kind = entry.get("kind", "login") if not isinstance(kind, str) or kind not in CREDENTIAL_KINDS: raise _Unreadable(_PROBLEM_DAMAGED) - if "group" in entry and (not isinstance(entry["group"], str) or entry["group"] not in CREDENTIAL_GROUPS): + if "group" in entry and ( + not isinstance(entry["group"], str) + or entry["group"] not in CREDENTIAL_GROUPS + ): raise _Unreadable(_PROBLEM_DAMAGED) for name in ("id", "site", "username", "password"): - if (not isinstance(entry.get(name), str) - or (not entry[name] and (kind == "login" or name in ("id", "password")))): + if not isinstance(entry.get(name), str) or ( + not entry[name] and (kind == "login" or name in ("id", "password")) + ): raise _Unreadable(_PROBLEM_DAMAGED) if not isinstance(entry.get("label"), str): entry["label"] = "" diff --git a/app/data/action/credential_actions.py b/app/data/action/credential_actions.py index 2584e5ad..dc6ab2f6 100644 --- a/app/data/action/credential_actions.py +++ b/app/data/action/credential_actions.py @@ -15,11 +15,17 @@ execution_mode="internal", action_sets=["core"], input_schema={ - "query": {"type": "string", "description": "Optional. Filter name, website or group (at most 200 characters)."}, + "query": { + "type": "string", + "description": "Optional. Filter name, website or group (at most 200 characters).", + }, }, output_schema={ "status": {"type": "string", "description": "success or error."}, - "credentials": {"type": "array", "description": "Metadata only: id, label, site and group."}, + "credentials": { + "type": "array", + "description": "Metadata only: id, label, site and group.", + }, "message": {"type": "string", "description": "Safe failure explanation."}, }, test_payload={"simulated_mode": True}, @@ -51,29 +57,76 @@ async def credential_list_api_keys(input_data: dict) -> dict: action_sets=["core"], parallelizable=False, input_schema={ - "credential_id": {"type": "string", "description": "Saved API-key ID from credential_list_api_keys; never the key value."}, - "url": {"type": "string", "description": "Absolute API URL on the key's saved website and port."}, - "method": {"type": "string", "enum": ["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"], "description": "Optional. HTTP method, default GET."}, - "auth_type": {"type": "string", "enum": ["bearer", "header"], "description": "Optional. bearer (default) sends Bearer plus key; header sends the raw key."}, - "header_name": {"type": "string", "description": "Optional. Header to receive the key; default Authorization for bearer, X-API-Key for header."}, - "headers": {"type": "object", "description": "Optional. Non-secret headers; cannot override authentication or transport headers."}, - "params": {"type": "object", "description": "Optional. Non-secret query parameters. The key is never inserted into the URL."}, - "json": {"type": "object", "description": "Optional. JSON request body; mutually exclusive with data."}, - "data": {"type": "string", "description": "Optional. Raw text request body; mutually exclusive with json."}, - "timeout": {"type": "number", "description": "Optional. Timeout in seconds, 1-60, default 30."}, + "credential_id": { + "type": "string", + "description": "Saved API-key ID from credential_list_api_keys; never the key value.", + }, + "url": { + "type": "string", + "description": "Absolute API URL on the key's saved website and port.", + }, + "method": { + "type": "string", + "enum": ["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"], + "description": "Optional. HTTP method, default GET.", + }, + "auth_type": { + "type": "string", + "enum": ["bearer", "header"], + "description": "Optional. bearer (default) sends Bearer plus key; header sends the raw key.", + }, + "header_name": { + "type": "string", + "description": "Optional. Header to receive the key; default Authorization for bearer, X-API-Key for header.", + }, + "headers": { + "type": "object", + "description": "Optional. Non-secret headers; cannot override authentication or transport headers.", + }, + "params": { + "type": "object", + "description": "Optional. Non-secret query parameters. The key is never inserted into the URL.", + }, + "json": { + "type": "object", + "description": "Optional. JSON request body; mutually exclusive with data.", + }, + "data": { + "type": "string", + "description": "Optional. Raw text request body; mutually exclusive with json.", + }, + "timeout": { + "type": "number", + "description": "Optional. Timeout in seconds, 1-60, default 30.", + }, }, output_schema={ "status": {"type": "string", "description": "success or error."}, - "status_code": {"type": "integer", "description": "HTTP status code when available."}, - "body": {"type": "string", "description": "Text/JSON response with reflected key values redacted, at most 256 KiB."}, + "status_code": { + "type": "integer", + "description": "HTTP status code when available.", + }, + "body": { + "type": "string", + "description": "Text/JSON response with reflected key values redacted, at most 256 KiB.", + }, "message": {"type": "string", "description": "Safe status or failure message."}, "error_code": {"type": "string", "description": "Failure code when available."}, }, - test_payload={"credential_id": "example", "url": "https://api.example.com/v1/items", "simulated_mode": True}, + test_payload={ + "credential_id": "example", + "url": "https://api.example.com/v1/items", + "simulated_mode": True, + }, ) async def credential_api_request(input_data: dict) -> dict: if input_data.get("simulated_mode"): - return {"status": "success", "status_code": 200, "body": '{"ok": true}', "message": "Simulated request; no key read or network access."} + return { + "status": "success", + "status_code": 200, + "body": '{"ok": true}', + "message": "Simulated request; no key read or network access.", + } from app.credentials.api_request import request_with_api_key return await request_with_api_key(input_data) diff --git a/app/downloads.py b/app/downloads.py index 8ed0ea89..5a487f2c 100644 --- a/app/downloads.py +++ b/app/downloads.py @@ -17,9 +17,12 @@ from __future__ import annotations import os +import hashlib import shutil import ssl +import tempfile import time +import urllib.error import urllib.request from typing import Callable, Optional @@ -87,31 +90,7 @@ def cache_name(url: str) -> str: return unquote(name) or "download.bin" -def find_cached(pattern: str) -> Optional[str]: - """A cached archive matching a glob, or None. - - Lets a caller use the cache WITHOUT first resolving a download URL. - That matters more than it sounds: resolving the Python runtime's URL - means fetching a large JSON release index, and on a slow or flaky link - that request is itself a failure point — observed as - `IncompleteRead(1277952 bytes read)` in Windows Sandbox. Having the file - already and still failing because the index could not be read is an - absurd way to lose an install. - - The archive names carry version and platform, so a glob like - `cpython-3.10.*-x86_64-pc-windows-msvc-install_only.tar.gz` identifies - the right file without asking anyone. - """ - import glob as _glob - - cache = cache_dir() - if not cache: - return None - matches = sorted(_glob.glob(os.path.join(cache, pattern))) - return matches[-1] if matches else None - - -def download( +def _download_once( url: str, dest: str, log: Optional[LogFn] = None, @@ -125,25 +104,11 @@ def download( tens of megabytes and a low-memory machine should not need twice that just to unpack them. - Uses CRAFTBOT_DOWNLOAD_CACHE when set — see cache_dir(). + Cache lookup and atomic publication are handled by download(). """ say: LogFn = log or (lambda _m: None) what = label or os.path.basename(dest) or url - cache = cache_dir() - cached = os.path.join(cache, cache_name(url)) if cache else None - - if cached and os.path.isfile(cached): - size = os.path.getsize(cached) - say(f" using cached {what} ({_human(size)})") - shutil.copyfile(cached, dest) - if progress_cb: - try: - progress_cb(size, size) - except Exception: - pass - return dest - req = urllib.request.Request(url, headers={"User-Agent": "CraftBot"}) with urllib.request.urlopen(req, timeout=timeout, context=ssl_context()) as resp: header = resp.getheader("Content-Length") @@ -181,17 +146,88 @@ def download( else: say(f" {_human(read)}") + if total is not None and read != total: + raise OSError(f"Download incomplete: {read} of {total} bytes") say(f" downloaded {_human(read)}") + return dest - # Populate the cache so the next install on this machine - or the next - # test run against a mapped cache - does not fetch it again. Skipped when - # the download already went straight into the cache (the prefetch script - # does that), because copying a file onto itself raises SameFileError and - # reads as a failure when nothing is wrong. - if cached and os.path.abspath(dest) != os.path.abspath(cached): - try: - shutil.copyfile(dest, cached) - except OSError as e: - say(f" (could not cache: {str(e)[:120]})") - return dest +def verify_checksum(path: str, expected: Optional[str]) -> bool: + if not os.path.isfile(path): + return False + if expected is None: + return os.path.getsize(path) > 0 + digest = hashlib.sha256() + with open(path, "rb") as source: + for chunk in iter(lambda: source.read(_CHUNK), b""): + digest.update(chunk) + return digest.hexdigest() == expected + + +def download( + url: str, + dest: str, + log: Optional[LogFn] = None, + label: str = "", + timeout: int = 600, + progress_cb: Optional[Callable[[int, Optional[int]], None]] = None, + expected_sha256: Optional[str] = None, + offline: bool = False, +) -> str: + """Publish only complete, verified downloads; bad caches never block repair.""" + say = log or (lambda _m: None) + dest = os.path.abspath(dest) + os.makedirs(os.path.dirname(dest), exist_ok=True) + cache = cache_dir() + cached = os.path.join(cache, cache_name(url)) if cache else None + fd, part = tempfile.mkstemp( + prefix=".download-", suffix=".part", dir=os.path.dirname(dest) + ) + os.close(fd) + try: + if cached and verify_checksum(cached, expected_sha256): + say(f" using verified cache: {label or cache_name(url)}") + shutil.copyfile(cached, part) + else: + if offline: + raise OSError( + f"Offline: no verified cached download for {cache_name(url)}" + ) + for attempt in range(3): + try: + _download_once( + url, part, say, label or cache_name(url), timeout, progress_cb + ) + if not verify_checksum(part, expected_sha256): + raise ValueError( + f"SHA-256 verification failed for {cache_name(url)}" + ) + break + except Exception as exc: + # A permanent HTTP error or bad checksum won't improve with retry. + permanent = isinstance(exc, ValueError) or ( + isinstance(exc, urllib.error.HTTPError) + and exc.code < 500 + and exc.code not in (408, 429) + ) + if permanent or attempt == 2: + raise + say(f" download interrupted; retrying ({attempt + 2}/3): {exc}") + time.sleep(attempt + 1) + os.replace(part, dest) + if cached and os.path.abspath(cached) != dest: + cache_part = None + try: + fd, cache_part = tempfile.mkstemp(prefix=".cache-", dir=cache) + os.close(fd) + shutil.copyfile(dest, cache_part) + os.replace(cache_part, cached) + except OSError as exc: + say(f" could not cache download: {exc}") + finally: + if cache_part and os.path.exists(cache_part): + os.unlink(cache_part) + return dest + finally: + if os.path.exists(part): + os.unlink(part) diff --git a/app/mini_browser/login.py b/app/mini_browser/login.py index 77176513..6f352929 100644 --- a/app/mini_browser/login.py +++ b/app/mini_browser/login.py @@ -479,7 +479,8 @@ def _check_origin(page: Any, frame: Any, origin: Tuple[str, str, int]) -> None: def _choose( - entries: List[Dict[str, Any]], username: Optional[str], + entries: List[Dict[str, Any]], + username: Optional[str], login_name: Optional[str] = None, ) -> Tuple[Optional[Dict[str, Any]], List[str]]: """The entry to use (vault order = best first) and the other usernames.""" @@ -803,8 +804,12 @@ def _without_fragment(url: str) -> str: async def autofill( - core: Any, tab: Any, *, username: Optional[str] = None, - login_name: Optional[str] = None, submit: bool = True + core: Any, + tab: Any, + *, + username: Optional[str] = None, + login_name: Optional[str] = None, + submit: bool = True, ) -> Dict[str, Any]: """Fill (and by default submit) the page's sign-in form from the vault. @@ -839,9 +844,7 @@ async def autofill( selector = f" for saved login name {login_name!r}" if login_name else "" if username: selector += f" for username {username!r}" - raise MiniBrowserError( - "MINI_BROWSER_NO_SAVED_LOGIN", site=f"{site}{selector}" - ) + raise MiniBrowserError("MINI_BROWSER_NO_SAVED_LOGIN", site=f"{site}{selector}") secret = _Hidden(str(entry.get("password") or "")) entry_id = str(entry.get("id") or "") account = str(entry.get("username") or "") diff --git a/app/mini_browser/ops.py b/app/mini_browser/ops.py index 045591bf..255c2445 100644 --- a/app/mini_browser/ops.py +++ b/app/mini_browser/ops.py @@ -2809,8 +2809,12 @@ async def screenshot(core: Any, tab: Any, *, full_page: bool = False) -> Dict[st async def login( - core: Any, tab: Any, *, username: Optional[str] = None, - login_name: Optional[str] = None, submit: bool = True + core: Any, + tab: Any, + *, + username: Optional[str] = None, + login_name: Optional[str] = None, + submit: bool = True, ) -> Dict[str, Any]: """Sign in with a saved login from the password vault.""" from app.mini_browser.login import autofill diff --git a/app/mini_browser/ws.py b/app/mini_browser/ws.py index 87b973dc..c4f03a09 100644 --- a/app/mini_browser/ws.py +++ b/app/mini_browser/ws.py @@ -620,8 +620,11 @@ def _public_entries(entries: Any) -> List[Dict[str, Any]]: if not isinstance(entries, (list, tuple)): return [] return [ - {key: _json_safe(entry.get(key)) for key in _ENTRY_FIELDS - if key not in ("kind", "group") or key in entry} + { + key: _json_safe(entry.get(key)) + for key in _ENTRY_FIELDS + if key not in ("kind", "group") or key in entry + } for entry in entries if isinstance(entry, dict) ] @@ -1649,20 +1652,39 @@ async def _handle_vault_list(self, ws: Any, data: Dict[str, Any]) -> None: ) async def _handle_vault_add(self, ws: Any, data: Dict[str, Any]) -> None: - kind = _vault_text(data, "kind", _MAX_VAULT_CHOICE_CHARS, required=False) or "login" - metadata = {key: _vault_text(data, key, _MAX_VAULT_CHOICE_CHARS, required=False) - for key in ("kind", "group") if key in data} - site = _vault_text(data, "site", _MAX_VAULT_SITE_CHARS, required=kind == "login") or "" - username = _vault_text( - data, "username", _MAX_VAULT_USERNAME_CHARS, required=kind == "login" - ) or "" + kind = ( + _vault_text(data, "kind", _MAX_VAULT_CHOICE_CHARS, required=False) + or "login" + ) + metadata = { + key: _vault_text(data, key, _MAX_VAULT_CHOICE_CHARS, required=False) + for key in ("kind", "group") + if key in data + } + site = ( + _vault_text(data, "site", _MAX_VAULT_SITE_CHARS, required=kind == "login") + or "" + ) + username = ( + _vault_text( + data, "username", _MAX_VAULT_USERNAME_CHARS, required=kind == "login" + ) + or "" + ) password = _vault_text( - data, "password", _MAX_VAULT_NOTE_CHARS if kind == "secure_note" else _MAX_VAULT_PASSWORD_CHARS, required=True + data, + "password", + _MAX_VAULT_NOTE_CHARS + if kind == "secure_note" + else _MAX_VAULT_PASSWORD_CHARS, + required=True, ) label = _vault_text(data, "label", _MAX_VAULT_LABEL_CHARS, required=False) or "" try: await _vault_call( - lambda vault: vault.add_entry(site, username, password, label, **metadata), + lambda vault: vault.add_entry( + site, username, password, label, **metadata + ), secrets=(password,), ) finally: @@ -1671,14 +1693,22 @@ async def _handle_vault_add(self, ws: Any, data: Dict[str, Any]) -> None: async def _handle_vault_update(self, ws: Any, data: Dict[str, Any]) -> None: entry_id = _short_id(data.get("id"), "id", required=True) - metadata = {key: _vault_text(data, key, _MAX_VAULT_CHOICE_CHARS, required=False) - for key in ("kind", "group") if key in data} + metadata = { + key: _vault_text(data, key, _MAX_VAULT_CHOICE_CHARS, required=False) + for key in ("kind", "group") + if key in data + } site = _vault_text(data, "site", _MAX_VAULT_SITE_CHARS, required=False) username = _vault_text( data, "username", _MAX_VAULT_USERNAME_CHARS, required=False ) password = _vault_text( - data, "password", _MAX_VAULT_NOTE_CHARS if data.get("kind") == "secure_note" else _MAX_VAULT_PASSWORD_CHARS, required=False + data, + "password", + _MAX_VAULT_NOTE_CHARS + if data.get("kind") == "secure_note" + else _MAX_VAULT_PASSWORD_CHARS, + required=False, ) label = _vault_text(data, "label", _MAX_VAULT_LABEL_CHARS, required=False) try: @@ -1703,12 +1733,17 @@ async def _handle_vault_reveal(self, ws: Any, data: Dict[str, Any]) -> None: secret = await _vault_call(lambda vault: vault.reveal_entry(entry_id)) try: # Explicit user request only: no broadcast, resource cache or logs. - self._adapter.send_only_to(ws, { - "type": "mini_browser_vault_value", - "data": {"id": entry_id, "token": token, "value": secret.reveal()}, - }) + self._adapter.send_only_to( + ws, + { + "type": "mini_browser_vault_value", + "data": {"id": entry_id, "token": token, "value": secret.reveal()}, + }, + ) except Exception: - logger.warning("[MINI_BROWSER] Could not deliver the requested credential value") + logger.warning( + "[MINI_BROWSER] Could not deliver the requested credential value" + ) async def _handle_vault_delete(self, ws: Any, data: Dict[str, Any]) -> None: entry_id = _short_id(data.get("id"), "id", required=True) diff --git a/app/node_runtime.py b/app/node_runtime.py index 8973f240..856acf33 100644 --- a/app/node_runtime.py +++ b/app/node_runtime.py @@ -8,20 +8,11 @@ TypeScript executed through Node's native type stripping — older majors throw ERR_UNKNOWN_FILE_EXTENSION). -Resolution order (cached per process): - 1. CRAFTBOT_NODE env var — explicit override (ignored with a warning when - the path doesn't exist). - 2. PATH node when its major >= MIN_NODE_MAJOR. - 3. Newest >= MIN_NODE_MAJOR among nvm/nvm-windows installs and the sidecar - install.py downloads into /runtime/node. - 4. An unprobeable PATH node, as the last resort — a broken probe must - never block a good Node, but must not shadow a working install either. - -The system default Node is NEVER upgraded, replaced, or shadowed outside -CraftBot's own subprocesses. When nothing >= MIN resolves, resolve() returns -None and callers fall back to plain PATH lookup — the frontend and bridge -run fine on Node 20; only Agent App hard-requires the resolved runtime -(runner.ensure_available raises the actionable message). +Resolution is cached per process. An explicit valid CRAFTBOT_NODE override +wins. Managed installs then use the release-pinned private Node/npm pair. +Source/Conda installs may use a compatible complete runtime from PATH or nvm. +Only CraftBot subprocess PATHs change; system installations are never modified. +A present Node without a functioning npm is not a complete runtime. Stdlib-only on purpose: install.py and run.py import this before any dependencies exist (app/__init__.py is empty). @@ -38,12 +29,6 @@ from pathlib import Path from typing import Optional -logger = logging.getLogger(__name__) - -MIN_NODE_MAJOR = 24 # keep the nodejs>=24 pin in environment.yml in sync - -REPO_ROOT = Path(__file__).resolve().parents[1] - # app.paths is the single answer to "where does state live" — in a dev # checkout that is the repo (so this is unchanged), and in an install it is # the per-user data dir. Before this, SIDECAR_DIR was derived from __file__ @@ -52,7 +37,15 @@ # reachable Node at all, and Living UI could not start. # app.paths is stdlib-only, like this module, so the import is safe at the # point install.py and run.py import us — before dependencies exist. -from app.paths import NODE_DIR as SIDECAR_DIR # noqa: E402 +from app.paths import is_managed_install +from app import runtime_manifest +from app.paths import NODE_DIR as SIDECAR_DIR + +logger = logging.getLogger(__name__) + +MIN_NODE_MAJOR = 24 # keep the nodejs>=24 pin in environment.yml in sync + +REPO_ROOT = Path(__file__).resolve().parents[1] _NODE_VER_RE = re.compile(r"v?(\d+)\.(\d+)\.(\d+)") @@ -166,13 +159,23 @@ def resolve(refresh: bool = False) -> Optional[NodeRuntime]: version=probe_version(override), source="override", ) - return _cached + if usable(_cached): + return _cached + _cached = None # A typo'd override must not become a raw FileNotFoundError at the # first spawn — warn and resolve normally instead. logger.warning( - f"[NODE] CRAFTBOT_NODE points at a missing file ({override}) — ignoring it" + f"[NODE] CRAFTBOT_NODE is missing or lacks a usable Node/npm pair ({override}) — ignoring it" ) + if is_managed_install(): + candidate = _pinned_exe() + if candidate.is_file(): + rt = _runtime(candidate) + if usable(rt): + _cached = rt + return _cached + path_node = shutil.which("node") path_version = probe_version(path_node) if path_node else None path_major = major_of(path_version) @@ -184,10 +187,14 @@ def resolve(refresh: bool = False) -> Optional[NodeRuntime]: version=path_version, source="path", ) - return _cached + if usable(_cached): + return _cached + _cached = None - discovered = discover_node() - if discovered: + # A broken higher-version nvm installation must not shadow our repaired copy. + for discovered in (discover_node(), str(_pinned_exe())): + if not discovered or not os.path.isfile(discovered): + continue _cached = NodeRuntime( node=discovered, npm=_sibling_npm(discovered), @@ -198,20 +205,9 @@ def resolve(refresh: bool = False) -> Optional[NodeRuntime]: logger.info( f"[NODE] PATH node unsuitable — using {discovered} for all components" ) - return _cached - - # Fail open on a broken probe, but only as the LAST resort: an - # unprobeable PATH node (hanging shim, corrupted binary) must never - # block a launch — yet must not shadow a working discovered install. - if path_node and path_major is None: - _cached = NodeRuntime( - node=path_node, - npm=_sibling_npm(path_node) or shutil.which("npm"), - bin_dir=os.path.dirname(path_node), - version=path_version, - source="path", - ) - return _cached + if usable(_cached): + return _cached + _cached = None return None @@ -232,6 +228,10 @@ def child_env(extra: Optional[dict] = None) -> dict: env = {**os.environ, **path_env()} if extra: env.update(extra) + if sys.platform != "win32" and not shutil.which("sh", path=env.get("PATH", "")): + # npm runs package scripts through an OS shell. A private Node PATH + # must still work without exposing system Git/tar or requiring setup. + env.setdefault("npm_config_script_shell", "/bin/sh") return env @@ -251,192 +251,141 @@ def npm_cmd() -> Optional[str]: def _platform_suffix(): - """(suffix, extension) of the Node archive for this machine. - - Windows gets the zip because tar would not preserve anything it needs; - unix gets a tarball because zip does not preserve the executable bit. - """ - import platform - - machine = platform.machine().lower() - arch = "arm64" if machine in ("arm64", "aarch64") else "x64" - if sys.platform == "win32": - return f"win-{arch}", "zip" - if sys.platform == "darwin": - return f"darwin-{arch}", "tar.gz" - return f"linux-{arch}", "tar.xz" + target = runtime_manifest.target("node") + return target, ( + "zip" + if target.startswith("win-") + else ("tar.gz" if target.startswith("darwin-") else "tar.xz") + ) def latest_download_url(log=None) -> Optional[str]: - """URL of the newest Node on the MIN_NODE_MAJOR line for this machine. - - Split out of download_sidecar so the archive can be pre-fetched into a - cache without installing it (scripts/prefetch_runtimes.py). - """ - import json - import ssl - import urllib.request - - say = log or (lambda _m: None) - + """Compatibility entry point for prefetch: the approved release, not latest.""" try: - import certifi - - ctx = ssl.create_default_context(cafile=certifi.where()) - except ImportError: - ctx = ssl.create_default_context() - - suffix, ext = _platform_suffix() - - try: - req = urllib.request.Request( - "https://nodejs.org/dist/index.json", headers={"User-Agent": "CraftBot"} - ) - index = json.loads(urllib.request.urlopen(req, timeout=60, context=ctx).read()) - except Exception as e: - say(f" Could not reach the Node index: {str(e)[:160]}") - return None - - ver = next( - ( - e["version"] - for e in index - if e.get("version", "").startswith(f"v{MIN_NODE_MAJOR}.") - ), - None, - ) - if not ver: - say(f" No v{MIN_NODE_MAJOR}.x release found in the Node index") + return runtime_manifest.asset("node")["url"] + except ValueError as exc: + (log or (lambda _: None))(str(exc)) return None - return f"https://nodejs.org/dist/{ver}/node-{ver}-{suffix}.{ext}" -def _extract_node(archive_src, ver, suffix, ext, say, url=None): - """Put a Node archive into SIDECAR_DIR and return its binary path. +def _pinned_exe(): + target = runtime_manifest.target("node") + root = SIDECAR_DIR / f"node-v{runtime_manifest.version('node')}-{target}" + return root / ("node.exe" if sys.platform == "win32" else "bin/node") - Shared by the cached and downloaded paths so extraction, layout and the - binary probe cannot drift between them. - """ - import tarfile - import zipfile - dest_root = str(SIDECAR_DIR) - os.makedirs(dest_root, exist_ok=True) - archive = os.path.join(dest_root, f"_download.{ext}") - try: - if archive_src: - say(f" Using cached {os.path.basename(archive_src)}") - shutil.copyfile(archive_src, archive) - else: - # Streamed with progress: the archive is 30-55MB, and a single - # silent blocking copy is indistinguishable from a hung install. - from app import downloads - - downloads.download(url, archive, log=say, label=f"Node {ver}") - say(" Extracting...") - if ext == "zip": - zipfile.ZipFile(archive).extractall(dest_root) - else: - # tar preserves the executable bit; zip does not, which is why - # Windows gets the zip and unix the tarball. - tarfile.open(archive, mode="r:*").extractall(dest_root) - except Exception as e: - say(f" Node setup failed: {str(e)[:200]}") +def npm_command(rt=None): + """Run npm's JS entry point with its own Node, without cmd.exe or shell quoting.""" + rt = rt or resolve() + if rt is None: return None - finally: - try: - os.remove(archive) - except OSError: - pass - - binary = os.path.join( - dest_root, - f"node-{ver}-{suffix}", - "node.exe" if sys.platform == "win32" else os.path.join("bin", "node"), - ) - return binary if os.path.isfile(binary) else None - - -def download_sidecar(log=None) -> Optional[str]: - """Download an official Node build into SIDECAR_DIR; return the binary. - - No PATH edits, nothing else touched — plain discovery picks it up on the - next resolve(refresh=True). Lives here rather than in install.py because - the installer needs it too: installer users never run install.py, so that - was the only route by which they could obtain Node, and they had none. - - Stdlib-only (certifi when importable, else the system trust store). - """ - import json - import ssl - import urllib.request + binary = Path(rt.node).resolve() + candidates = [ + binary.parent / "node_modules/npm/bin/npm-cli.js", + binary.parent.parent / "lib/node_modules/npm/bin/npm-cli.js", + ] + if rt.npm: + candidates.append(Path(rt.npm).resolve().parent / "npm-cli.js") + for entry in candidates: + if entry.is_file(): + return [str(binary), str(entry)] + return None - say = log or (lambda _m: None) +def usable(rt): + if rt is None or (major_of(rt.version) or 0) < MIN_NODE_MAJOR: + return False + cmd = npm_command(rt) + if not cmd: + return False try: - import certifi + kwargs = ( + {"creationflags": subprocess.CREATE_NO_WINDOW} + if sys.platform == "win32" + else {} + ) + result = subprocess.run( + cmd + ["--version"], capture_output=True, text=True, timeout=20, **kwargs + ) + return result.returncode == 0 and bool( + re.match(r"^\d+\.", result.stdout.strip()) + ) + except (OSError, subprocess.TimeoutExpired): + return False - ctx = ssl.create_default_context(cafile=certifi.where()) - except ImportError: - ctx = ssl.create_default_context() - suffix, ext = _platform_suffix() +def _runtime(node, source="sidecar"): + return NodeRuntime( + str(node), + _sibling_npm(str(node)), + str(Path(node).parent), + probe_version(str(node)), + source, + ) - # Consult the cache BEFORE the network. Resolving the version means - # fetching nodejs.org's release index, which on a slow or flaky link is - # its own failure point — and failing there while the archive is already - # on disk would be a pointless way to lose an install. - cached_archive = None - ver_from_cache = None - try: - from app import downloads - cached_archive = downloads.find_cached( - f"node-v{MIN_NODE_MAJOR}.*-{suffix}.{ext}" - ) - if cached_archive: - m = _NODE_VER_RE.search(os.path.basename(cached_archive)) - if m: - ver_from_cache = "v" + ".".join(m.groups()) - except Exception: - cached_archive = None - - if cached_archive and ver_from_cache: - ver = ver_from_cache - url = f"https://nodejs.org/dist/{ver}/node-{ver}-{suffix}.{ext}" - return _extract_node(cached_archive, ver, suffix, ext, say, url) +def download_sidecar(log=None, offline=False) -> Optional[str]: + import tarfile + import tempfile + import zipfile + from app import downloads + say = log or (lambda _: None) try: - # Newest release on the MIN_NODE_MAJOR line (the index is newest-first). - req = urllib.request.Request( - "https://nodejs.org/dist/index.json", headers={"User-Agent": "CraftBot"} - ) - index = json.loads(urllib.request.urlopen(req, timeout=60, context=ctx).read()) - ver = next( - ( - e["version"] - for e in index - if e.get("version", "").startswith(f"v{MIN_NODE_MAJOR}.") - ), - None, - ) - if not ver: - say(f" ⚠ No v{MIN_NODE_MAJOR}.x release found in the Node index") - return None - - url = f"https://nodejs.org/dist/{ver}/node-{ver}-{suffix}.{ext}" - return _extract_node(None, ver, suffix, ext, say, url) - except Exception as e: - say(f" ⚠ Sidecar Node download failed: {str(e)[:200]}") + asset = runtime_manifest.asset("node") + target, ext = _platform_suffix() + version = "v" + runtime_manifest.version("node") + SIDECAR_DIR.mkdir(parents=True, exist_ok=True) + installed = SIDECAR_DIR / f"node-{version}-{target}" + with tempfile.TemporaryDirectory(prefix=".node-", dir=SIDECAR_DIR) as staging: + stage = Path(staging) + archive = stage / ("download." + ext) + downloads.download( + asset["url"], + str(archive), + log=say, + label=f"Node {version}", + expected_sha256=asset["sha256"], + offline=offline, + ) + if ext == "zip": + with zipfile.ZipFile(archive) as zf: + zf.extractall(stage) + else: + with tarfile.open(archive) as tf: + tf.extractall(stage, filter="data") + candidate = stage / installed.name + exe = candidate / ("node.exe" if sys.platform == "win32" else "bin/node") + if not usable(_runtime(exe)): + raise RuntimeError("Downloaded Node/npm failed its execution check") + backup = SIDECAR_DIR / (installed.name + ".previous") + if backup.exists(): + shutil.rmtree(backup) + if installed.exists(): + installed.rename(backup) + try: + candidate.rename(installed) + except Exception: + if backup.exists(): + backup.rename(installed) + raise + if backup.exists(): + shutil.rmtree(backup, ignore_errors=True) + probe_version.cache_clear() + return str(_pinned_exe()) + except Exception as exc: + say(f"Node setup failed: {exc}") return None -def ensure_sidecar(log=None) -> Optional[NodeRuntime]: - """Return a usable Node >= MIN_NODE_MAJOR, downloading one if needed. - Idempotent: a no-op when a suitable Node already resolves.""" - rt = resolve() - if rt is not None: +def ensure_sidecar(log=None, offline=False) -> Optional[NodeRuntime]: + """Ensure a runnable Node/npm pair, repairing incomplete installations.""" + global _cached, _resolved + rt = resolve(refresh=True) + if usable(rt): return rt - if download_sidecar(log=log): - return resolve(refresh=True) + node = download_sidecar(log=log, offline=offline) + if node: + _cached, _resolved = _runtime(node), True + return _cached return None diff --git a/app/process_ledger.py b/app/process_ledger.py index 2ddf504f..1bcc4fcc 100644 --- a/app/process_ledger.py +++ b/app/process_ledger.py @@ -256,19 +256,54 @@ def craftbot_listeners(port: int) -> List[tuple]: return found -def kill_tree(pid: int, grace: float = 5.0) -> None: - """Stop a process and every descendant: SIGTERM, then SIGKILL whatever is - left after `grace` seconds (Windows has no graceful signal for a windowless - tree, so it is `taskkill /T /F` as before). Callers must have verified - ownership first — this does not check.""" - if os.name == "nt": - # taskkill /T walks the tree even when psutil is unavailable. - subprocess.run( +def _kill_windows_tree(pid: int, grace: float) -> None: + """Fall back to direct process APIs when taskkill cannot access WMI.""" + try: + result = subprocess.run( ["taskkill", "/T", "/F", "/PID", str(pid)], capture_output=True, timeout=15, creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), ) + if result.returncode == 0: + return + detail = result.stderr.decode("utf-8", errors="replace").strip() + except (OSError, subprocess.TimeoutExpired) as exc: + detail = str(exc) + if psutil is None: + raise RuntimeError(f"Cannot stop owned pid {pid}: taskkill failed ({detail})") + logger.warning(f"taskkill failed for owned pid {pid}; using process APIs: {detail}") + try: + root = psutil.Process(pid) + procs = root.children(recursive=True) + [root] + except psutil.NoSuchProcess: + return + for proc in procs: + try: + proc.terminate() + except psutil.NoSuchProcess: + pass + _, alive = psutil.wait_procs(procs, timeout=grace) + for proc in alive: + try: + proc.kill() + except psutil.NoSuchProcess: + pass + _, alive = psutil.wait_procs(alive, timeout=grace) + if alive: + raise RuntimeError( + f"Could not stop owned process tree {pid}; still running: " + + ", ".join(str(proc.pid) for proc in alive) + ) + + +def kill_tree(pid: int, grace: float = 5.0) -> None: + """Stop a process and every descendant: SIGTERM, then SIGKILL whatever is + left after `grace` seconds (Windows has no graceful signal for a windowless + tree, so use taskkill with a direct process API fallback). Callers must have verified + ownership first — this does not check.""" + if os.name == "nt": + _kill_windows_tree(pid, grace) return if psutil is None: import signal diff --git a/app/provision/__init__.py b/app/provision/__init__.py index 3a39b14e..57fe6f29 100644 --- a/app/provision/__init__.py +++ b/app/provision/__init__.py @@ -13,17 +13,20 @@ disk → enough free space to finish at all python → the interpreter everything else installs into + pip → the package installer compatible with the locks python-deps → the locked package set native → prove the embedding stack loads (not merely installed) smoke → prove what actions import is importable node → the one Node runtime + agent-app-tools → CLI dependencies, including the ZIP reader + pocketbase → the pinned executable used by Agent Apps frontend → browser UI's npm tree (default UI, so required) whatsapp → bridge's npm tree ripgrep → the bundled search binary grep_files runs, checked not fetched playwright → chromium for browser-automation actions -Node comes after Python deps because nothing before it needs Node, and a slow -optional download should not delay the failure of a required stage. +Node comes after Python deps because nothing before it needs Node. Node, +the Agent App tooling and PocketBase are required for a complete installation. """ from __future__ import annotations @@ -33,10 +36,12 @@ from app import paths from app.provision.deps import ( + AgentAppToolsStage, FrontendStage, PipStage, PlaywrightStage, PythonDepsStage, + PocketBaseStage, WhatsAppBridgeStage, ) from app.provision.pipeline import format_report, run @@ -78,6 +83,8 @@ def default_stages() -> List[Stage]: NativeRuntimeStage(), SmokeStage(), NodeStage(), + AgentAppToolsStage(), + PocketBaseStage(), FrontendStage(), WhatsAppBridgeStage(), RipgrepStage(), diff --git a/app/provision/deps.py b/app/provision/deps.py index 9341fe37..968006e3 100644 --- a/app/provision/deps.py +++ b/app/provision/deps.py @@ -168,9 +168,8 @@ class PipStage: it. Nothing in the repo chose between them. Two independent sources of that drift: - * the sidecar's pip is whatever python-build-standalone bundled on the - day it was downloaded, and the release tag is resolved from - /releases/latest rather than pinned; + * the sidecar's pip is whatever the pinned python-build-standalone + release bundled, which need not match our dependency lock tooling; * PythonStage may hand the install to an interpreter already on the machine (steps 2 and 3 of its resolution order), bringing that machine's pip with it. @@ -528,12 +527,13 @@ def apply(self, ctx: Context, log: LogFn) -> StageResult: class _NpmTreeStage: - """Shared logic for the two npm trees CraftBot ships.""" + """Shared logic for the npm trees CraftBot ships.""" name = "npm" description = "npm dependencies" optional = True rel_dir = "" + install_args = ("install",) #: A file that only exists once `npm install` has succeeded. sentinel = "node_modules" @@ -554,13 +554,13 @@ def apply(self, ctx: Context, log: LogFn) -> StageResult: return StageResult(Status.FAILED, "offline: cannot npm install") from app import node_runtime - npm = node_runtime.npm_cmd() + npm = node_runtime.npm_command() if not npm: return StageResult(Status.FAILED, "no npm (Node stage must run first)") d = self._dir(ctx) if not d.is_dir(): return StageResult(Status.SKIPPED, f"{self.rel_dir} not present") - cmd = [npm, "install", "--no-audit", "--no-fund"] + cmd = npm + list(self.install_args) + ["--no-audit", "--no-fund"] # A pre-warmed npm cache turns ~50 MB from the registry into local # reads. --prefer-offline rather than --offline: it uses the cache @@ -578,10 +578,96 @@ def apply(self, ctx: Context, log: LogFn) -> StageResult: # child_env() exists for exactly this and was going unused. res = proc.run(cmd, log, cwd=str(d), stream=True, env=node_runtime.child_env()) if res.returncode != 0: + log("\n".join(filter(None, [res.stdout, res.stderr]))) return StageResult(Status.FAILED, proc.failure_detail(res, "npm failed")) return self.check(ctx) +class AgentAppToolsStage(_NpmTreeStage): + """Install the CLI's own dependencies before it can fetch PocketBase.""" + + name = "agent-app-tools" + description = "Agent App tooling" + optional = False + rel_dir = "agent-app" + install_args = ("ci", "--ignore-scripts", "--include=dev") + + def check(self, ctx: Context) -> StageResult: + from app import node_runtime + + directory = self._dir(ctx) + if not (directory / "tools/src/cli.ts").is_file(): + return StageResult( + Status.FAILED, "Agent App tooling is missing from this installation" + ) + result = super().check(ctx) + if not result.ok: + return result + installed_at = (directory / "node_modules/.package-lock.json").stat().st_mtime + for workspace in ("kit", "tools"): + manifest = directory / workspace / "package.json" + if not manifest.is_file() or manifest.stat().st_mtime > installed_at: + return StageResult( + Status.MISSING, f"{workspace}/package.json changed or is missing" + ) + node = node_runtime.node_cmd() + if not node: + return StageResult(Status.MISSING, "Node is unavailable") + probe = proc.run( + [node, "--input-type=module", "-e", "await import('fflate')"], + cwd=str(directory / "tools"), + timeout=30, + echo=False, + env=node_runtime.child_env(), + ) + if probe.returncode != 0: + return StageResult( + Status.MISSING, proc.failure_detail(probe, "ZIP dependency is missing") + ) + return StageResult(Status.SATISFIED, "Agent App tooling ready") + + +class PocketBaseStage: + name = "pocketbase" + description = "PocketBase runtime" + optional = False + + def _run(self, ctx: Context, subcommand: str, log=None): + from app import node_runtime + + node = node_runtime.node_cmd() + if not node: + return StageResult(Status.MISSING, "Node is unavailable") + cli = Path(ctx.code_root) / "agent-app/tools/src/cli.ts" + result = proc.run( + [node, str(cli), "pb", subcommand], + log, + timeout=900, + env=node_runtime.child_env(), + ) + if result.returncode: + if log: + log("\n".join(filter(None, [result.stdout, result.stderr]))) + return StageResult( + Status.FAILED if subcommand == "fetch" else Status.MISSING, + proc.failure_detail(result, "PocketBase setup failed"), + ) + return StageResult( + Status.SATISFIED, (result.stdout or "").strip().splitlines()[-1] + ) + + def check(self, ctx: Context) -> StageResult: + return self._run(ctx, "check") + + def apply(self, ctx: Context, log: LogFn) -> StageResult: + if ctx.offline: + return StageResult( + Status.FAILED, + "Offline: PocketBase is not cached; retry setup with a connection", + ) + return self._run(ctx, "fetch", log) + + class FrontendStage(_NpmTreeStage): """The browser UI's npm tree — only needed to BUILD it. diff --git a/app/provision/pipeline.py b/app/provision/pipeline.py index df7b4cae..3b5f2100 100644 --- a/app/provision/pipeline.py +++ b/app/provision/pipeline.py @@ -25,7 +25,7 @@ def run( A required stage that fails stops the run: continuing past a failed dependency install only produces a second, more confusing error further - down. Optional stages (Node, Playwright) record the failure and continue — + down. Optional stages (Playwright, WhatsApp) record the failure and continue — core chat works without them, and taking the whole install down for a feature the user may never touch is worse than a warning. """ @@ -33,6 +33,8 @@ def run( report = PipelineReport() for stage in stages: + if not stage.optional: + report.required.add(stage.name) try: before = stage.check(ctx) except Exception as e: # a broken check must not abort the install diff --git a/app/provision/proc.py b/app/provision/proc.py index c4f54bc3..00bc112f 100644 --- a/app/provision/proc.py +++ b/app/provision/proc.py @@ -88,6 +88,8 @@ def run( cwd=cwd, capture_output=True, text=True, + encoding="utf-8", + errors="replace", timeout=timeout, env=env, **_no_window_kwargs(), diff --git a/app/provision/runtimes.py b/app/provision/runtimes.py index 458e79e7..7bb6c976 100644 --- a/app/provision/runtimes.py +++ b/app/provision/runtimes.py @@ -10,7 +10,6 @@ from __future__ import annotations import os -import platform import shutil import subprocess import sys @@ -19,44 +18,25 @@ from app.provision.types import Context, LogFn, StageResult, Status -# python-build-standalone: relocatable CPython for all three platforms, with -# pip. python.org's embeddable package is Windows-only and ships without pip, -# so it cannot serve the same role. -# -# The release tag and patch version are RESOLVED FROM THE API, never -# hardcoded. Hardcoding them was tried and produced a silent 404: the guessed -# tag, patch version and platform triple were all wrong, so the download -# failed and the install aborted at the first stage. The asset name embeds -# all three (cpython-3.10.21+20260825-x86_64-pc-windows-msvc-install_only), -# so any one being stale breaks it. Same approach as node_runtime, which -# reads nodejs.org/dist/index.json rather than pinning a version. -PBS_API = ( - "https://api.github.com/repos/astral-sh/python-build-standalone/releases/latest" -) - -# The version the locks are generated for. NOT a minimum: a lock is valid -# for exactly one (platform, python) pair, so accepting "3.10 or newer" would -# mean a user on 3.14 has no lock at all — which is precisely what happened on -# a machine whose conda env had drifted to 3.14.7 while environment.yml still -# said 3.10.19. Anything else gets the sidecar, which costs a download and -# buys the reproducibility the lock exists for. -TARGET_PYTHON = (3, 10) +from app import runtime_manifest + +# The dependency locks remain on this Python major/minor line. +TARGET_PYTHON = tuple(int(v) for v in runtime_manifest.version("python").split(".")[:2]) class NodeStage: name = "node" description = "Node.js runtime" - # Core chat works without Node; only Living UI hard-requires it. Failing - # the whole install because a Node download timed out would be worse than - # the feature being unavailable. - optional = True + optional = False def check(self, ctx: Context) -> StageResult: from app import node_runtime rt = node_runtime.resolve(refresh=True) - if rt is None: - return StageResult(Status.MISSING, "no Node >= 24 found") + if not node_runtime.usable(rt): + return StageResult( + Status.MISSING, "a working Node >= 24 and matching npm are required" + ) return StageResult( Status.SATISFIED, f"{rt.version or '?'} ({rt.source})", @@ -64,11 +44,9 @@ def check(self, ctx: Context) -> StageResult: ) def apply(self, ctx: Context, log: LogFn) -> StageResult: - if ctx.offline: - return StageResult(Status.FAILED, "offline: cannot download Node") from app import node_runtime - rt = node_runtime.ensure_sidecar(log=log) + rt = node_runtime.ensure_sidecar(log=log, offline=ctx.offline) if rt is None: return StageResult(Status.FAILED, "sidecar download failed") return StageResult( @@ -160,58 +138,20 @@ def _probe_python(exe: str) -> Optional[tuple]: def _pbs_triple() -> Optional[str]: - """The python-build-standalone platform triple for this machine.""" - machine = platform.machine().lower() - arch = "aarch64" if machine in ("arm64", "aarch64") else "x86_64" - if sys.platform == "win32": - # No win-arm64 build is published; arm64 Windows runs the x64 build - # under emulation, which is slower but works. - return "x86_64-pc-windows-msvc" - if sys.platform == "darwin": - return f"{arch}-apple-darwin" - return f"{arch}-unknown-linux-gnu" - - -def _pbs_download_url(log: LogFn) -> Optional[str]: - """Resolve the download URL for a TARGET_PYTHON build, or None. - - Asks the API for the newest release and picks the asset matching this - machine's triple. `install_only` is the variant that extracts to a - ready-to-run tree; the full builds carry object files and headers nobody - here needs. - """ - import json - import ssl - import urllib.request - - triple = _pbs_triple() - if not triple: - return None - try: - import certifi + return runtime_manifest.target("python") + except ValueError: + return None - ctx = ssl.create_default_context(cafile=certifi.where()) - except ImportError: - ctx = ssl.create_default_context() +def _pbs_download_url(log: LogFn) -> Optional[str]: + """Resolve locally, including offline; never depend on upstream 'latest'.""" try: - req = urllib.request.Request(PBS_API, headers={"User-Agent": "CraftBot"}) - data = json.loads(urllib.request.urlopen(req, timeout=60, context=ctx).read()) - except Exception as e: - log(f" could not reach the Python download index: {str(e)[:160]}") + return runtime_manifest.asset("python")["url"] + except ValueError as exc: + log(str(exc)) return None - prefix = f"cpython-{TARGET_PYTHON[0]}.{TARGET_PYTHON[1]}." - suffix = f"-{triple}-install_only.tar.gz" - for asset in data.get("assets", []): - name = asset.get("name") or "" - if name.startswith(prefix) and name.endswith(suffix): - return asset.get("browser_download_url") - - log(f" no {prefix}*{suffix} in release {data.get('tag_name')}") - return None - class PythonStage: """Ensure a real CPython on the locked version line exists. @@ -303,69 +243,53 @@ def check(self, ctx: Context) -> StageResult: ) def apply(self, ctx: Context, log: LogFn) -> StageResult: - if ctx.offline: - return StageResult(Status.FAILED, "offline: cannot download Python") - - import shutil as _shutil import tarfile - + import tempfile from app import downloads dest = Path(ctx.state_root) / "runtime" / "python" dest.mkdir(parents=True, exist_ok=True) - archive = dest / "_download.tar.gz" - - # Look in the cache BEFORE asking the network anything. Resolving the - # URL means fetching a large release index, which is itself a failure - # point on a slow link - and failing there while holding the very file - # we need would be perverse. - triple = _pbs_triple() - cached = ( - downloads.find_cached( - f"cpython-{TARGET_PYTHON[0]}.{TARGET_PYTHON[1]}.*-{triple}-install_only.tar.gz" - ) - if triple - else None - ) - - url = None - if not cached: - url = _pbs_download_url(log) - if not url: - return StageResult( - Status.FAILED, - f"no portable Python {TARGET_PYTHON[0]}.{TARGET_PYTHON[1]} " - f"available for {platform.machine()} on {sys.platform}", - ) - try: - if cached: - log(f" using cached {os.path.basename(cached)}") - _shutil.copyfile(cached, archive) - else: + asset = runtime_manifest.asset("python") + # Staging keeps failed extraction from leaving a seemingly ready runtime. + with tempfile.TemporaryDirectory(prefix=".python-", dir=dest) as staging: + stage = Path(staging) + archive = stage / "python.tar.gz" downloads.download( - url, + asset["url"], str(archive), log=log, - label=f"Python {TARGET_PYTHON[0]}.{TARGET_PYTHON[1]}", + label="Python " + runtime_manifest.version("python"), + expected_sha256=asset["sha256"], + offline=ctx.offline, + ) + log(" extracting Python...") + with tarfile.open(archive, mode="r:*") as tf: + tf.extractall(stage, filter="data") + relative_exe = ( + Path("python.exe") + if sys.platform == "win32" + else Path("bin/python3") ) - log(" extracting...") - # tar preserves the executable bit, which a zip would not — that - # matters on macOS/Linux where the extracted python must be +x. - with tarfile.open(archive, mode="r:*") as tf: - tf.extractall(dest) - except Exception as e: - return StageResult(Status.FAILED, f"download failed: {str(e)[:200]}") - finally: - try: - archive.unlink() - except OSError: - pass - - exe = self._sidecar_exe(ctx) - if not exe.is_file(): - return StageResult(Status.FAILED, f"extracted, but {exe} is missing") - ver = _probe_python(str(exe)) - if not ver or ver[:2] != TARGET_PYTHON: - return StageResult(Status.FAILED, f"sidecar will not run ({ver})") - return self._accept(ctx, str(exe), "sidecar", ver) + candidate = stage / "python" / relative_exe + if _probe_python(str(candidate)) != TARGET_PYTHON: + raise RuntimeError("Downloaded Python failed its execution check") + installed = dest / "python" + backup = dest / ".python-previous" + if backup.exists(): + shutil.rmtree(backup) + if installed.exists(): + installed.rename(backup) + try: + (stage / "python").rename(installed) + except Exception: + if backup.exists(): + backup.rename(installed) + raise + if backup.exists(): + shutil.rmtree(backup, ignore_errors=True) + return self._accept( + ctx, str(self._sidecar_exe(ctx)), "sidecar", TARGET_PYTHON + ) + except Exception as exc: + return StageResult(Status.FAILED, f"Python setup failed: {exc}") diff --git a/app/provision/types.py b/app/provision/types.py index c86be118..e50c155a 100644 --- a/app/provision/types.py +++ b/app/provision/types.py @@ -101,8 +101,7 @@ class Stage(Protocol): name: str #: Human-readable one-liner shown in progress output. description: str - #: When False, a FAILED result stops the pipeline. Node is not required - #: for core chat, so it is optional; Python dependencies are not. + #: When False, a failed result stops installation and prevents success. optional: bool def check(self, ctx: Context) -> StageResult: ... @@ -113,6 +112,11 @@ def apply(self, ctx: Context, log: LogFn) -> StageResult: ... @dataclass class PipelineReport: results: List[tuple] = field(default_factory=list) # (stage_name, StageResult) + required: set = field(default_factory=set) + + @property + def required_failures(self) -> List[tuple]: + return [(n, r) for n, r in self.failures if n in self.required] @property def ok(self) -> bool: @@ -125,6 +129,7 @@ def failures(self) -> List[tuple]: def to_dict(self) -> Dict[str, Any]: return { "ok": self.ok, + "required_failures": [name for name, _ in self.required_failures], "stages": [ { "name": name, diff --git a/app/runtime-manifest.json b/app/runtime-manifest.json new file mode 100644 index 00000000..47f77c99 --- /dev/null +++ b/app/runtime-manifest.json @@ -0,0 +1,56 @@ +{ + "python": { + "version": "3.10.22", + "assets": { + "x86_64-pc-windows-msvc": { + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20261009/cpython-3.10.22%2B20261009-x86_64-pc-windows-msvc-install_only.tar.gz", + "sha256": "cb023e83e42d8937ba77c7cf83641083c3b81e88e1226e1c694a347ae1924d54" + }, + "x86_64-apple-darwin": { + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20261009/cpython-3.10.22%2B20261009-x86_64-apple-darwin-install_only.tar.gz", + "sha256": "998d4159933724eba107bbb0e01ed920d267d59c815b256fc39ffe8658df1350" + }, + "aarch64-apple-darwin": { + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20261009/cpython-3.10.22%2B20261009-aarch64-apple-darwin-install_only.tar.gz", + "sha256": "02d1a66b5fe8676e6df5084f4e0b71779a4e6f1c46f192d3d3a12b7eb3a789ec" + }, + "x86_64-unknown-linux-gnu": { + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20261009/cpython-3.10.22%2B20261009-x86_64-unknown-linux-gnu-install_only.tar.gz", + "sha256": "1780de9ddda04e956d2a4a0ad09b7f5667d92f4f12cd0881ec939de5d542903f" + }, + "aarch64-unknown-linux-gnu": { + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20261009/cpython-3.10.22%2B20261009-aarch64-unknown-linux-gnu-install_only.tar.gz", + "sha256": "ad9fc572c8e67cae305de627433e3cb6d459cbe94711921d5b673766b5a20c72" + } + } + }, + "node": { + "version": "24.19.0", + "assets": { + "win-x64": { + "url": "https://nodejs.org/dist/v24.19.0/node-v24.19.0-win-x64.zip", + "sha256": "57f71ab3652e797d84acddc79c81cc9ff1c6ddb2a1974cdb83f00fee9bff4c73" + }, + "win-arm64": { + "url": "https://nodejs.org/dist/v24.19.0/node-v24.19.0-win-arm64.zip", + "sha256": "8502f4a50b458d4cc38ed8f2001556c2cd239d464920f74017926ccb1e1c157f" + }, + "darwin-x64": { + "url": "https://nodejs.org/dist/v24.19.0/node-v24.19.0-darwin-x64.tar.gz", + "sha256": "d1b5e999db158c62fe8f7267a4476b035d8bd93b1a605bac24a3f0dd166e3316" + }, + "darwin-arm64": { + "url": "https://nodejs.org/dist/v24.19.0/node-v24.19.0-darwin-arm64.tar.gz", + "sha256": "8294b7aa9b03997481c06babf1e8b270c859358f27da57a11509afe537ac381d" + }, + "linux-x64": { + "url": "https://nodejs.org/dist/v24.19.0/node-v24.19.0-linux-x64.tar.xz", + "sha256": "14b342e71204f811bde6153be8e04b62aef63c236fef92b55f9c83154b409647" + }, + "linux-arm64": { + "url": "https://nodejs.org/dist/v24.19.0/node-v24.19.0-linux-arm64.tar.xz", + "sha256": "01443c1e1a29e531ccad5a46fefa6df490d2189c49f7955904aecdbb0fe86fdc" + } + } + } +} diff --git a/app/runtime_manifest.py b/app/runtime_manifest.py new file mode 100644 index 00000000..2949c5f9 --- /dev/null +++ b/app/runtime_manifest.py @@ -0,0 +1,54 @@ +"""Release-owned runtime downloads. Stdlib-only: used before pip is available.""" + +import json +import platform +import sys +from pathlib import Path +from urllib.parse import urlparse + +MANIFEST_PATH = Path(__file__).with_name("runtime-manifest.json") + + +def manifest(): + return json.loads(MANIFEST_PATH.read_text(encoding="utf-8")) + + +def version(runtime): + return manifest()[runtime]["version"] + + +def target(runtime): + machine = platform.machine().lower() + if machine not in ("amd64", "x86_64", "arm64", "aarch64"): + raise ValueError(f"Unsupported CPU architecture: {machine}") + arm = machine in ("arm64", "aarch64") + if runtime == "python": + # The locked Windows Python is x64, including under ARM64 emulation. + if sys.platform == "win32": + return "x86_64-pc-windows-msvc" + arch = "aarch64" if arm else "x86_64" + suffix = {"darwin": "apple-darwin", "linux": "unknown-linux-gnu"} + else: + arch = "arm64" if arm else "x64" + suffix = {"win32": "win", "darwin": "darwin", "linux": "linux"} + if sys.platform not in suffix: + raise ValueError(f"Unsupported operating system: {sys.platform}") + return ( + f"{arch}-{suffix[sys.platform]}" + if runtime == "python" + else f"{suffix[sys.platform]}-{arch}" + ) + + +def asset(runtime, platform_target=None): + key = platform_target or target(runtime) + try: + item = manifest()[runtime]["assets"][key] + except KeyError as exc: + raise ValueError(f"No approved {runtime} download for {key}") from exc + digest = item.get("sha256", "") + if len(digest) != 64 or any(c not in "0123456789abcdef" for c in digest): + raise ValueError(f"Invalid {runtime} checksum for {key}") + if urlparse(item["url"]).scheme != "https": + raise ValueError("Runtime downloads must use HTTPS") + return item diff --git a/app/ui_layer/adapters/browser_adapter.py b/app/ui_layer/adapters/browser_adapter.py index dd649939..76a22781 100644 --- a/app/ui_layer/adapters/browser_adapter.py +++ b/app/ui_layer/adapters/browser_adapter.py @@ -4332,7 +4332,11 @@ async def _handle_question_response( pending_questions: list = [] if self._chat and message_id: question_text = next( - (m.content for m in self._chat._messages if m.message_id == message_id), + ( + m.content + for m in self._chat._messages + if m.message_id == message_id + ), "", ) storage = self._chat._storage diff --git a/app/ui_layer/events/event_bus.py b/app/ui_layer/events/event_bus.py index 8a597c0b..18cb7bb3 100644 --- a/app/ui_layer/events/event_bus.py +++ b/app/ui_layer/events/event_bus.py @@ -70,10 +70,14 @@ def _invoke(self, handler: EventHandler, event: UIEvent) -> None: async def drain(self, task_id: str) -> None: """Settle this session's already-emitted UI work before clearing it.""" current = asyncio.current_task() - while tasks := [task for task, sid in self._pending.items() - if sid == task_id and task is not current]: - await asyncio.gather(*(asyncio.shield(task) for task in tasks), - return_exceptions=True) + while tasks := [ + task + for task, sid in self._pending.items() + if sid == task_id and task is not current + ]: + await asyncio.gather( + *(asyncio.shield(task) for task in tasks), return_exceptions=True + ) def subscribe( self, diff --git a/craftbot.py b/craftbot.py index eb8c91bd..b46be638 100644 --- a/craftbot.py +++ b/craftbot.py @@ -552,6 +552,7 @@ def _stop_orphan_listeners(args: Optional[List[str]] = None) -> int: from app.process_ledger import kill_tree stopped = 0 + failures = [] for pid, description in orphans: print( f" {ORANGE}▸{RESET} {WHITE}STOPPING ORPHANED CRAFTBOT{RESET}" @@ -562,6 +563,11 @@ def _stop_orphan_listeners(args: Optional[List[str]] = None) -> int: stopped += 1 except Exception as e: print(f" {DIM}(could not stop pid {pid}: {e}){RESET}") + failures.append(str(pid)) + if failures: + raise RuntimeError( + "Could not stop orphaned CraftBot pids: " + ", ".join(failures) + ) return stopped @@ -818,14 +824,9 @@ def cmd_stop(quiet: bool = False, args: Optional[List[str]] = None) -> bool: def _kill_owned_pid(pid: int) -> None: """Terminate a pid already verified as ours, with its whole tree.""" if _PLATFORM == "win32": - try: - subprocess.run( - ["taskkill", "/PID", str(pid), "/F", "/T"], - capture_output=True, - timeout=15, - ) - except Exception as e: - print(f"Warning: taskkill failed — {e}") + from app.process_ledger import kill_tree + + kill_tree(pid) else: try: # Kill the entire process group so child processes also die @@ -1583,12 +1584,7 @@ def _full_install_frozen( if not report.ok: print() print(provision.format_report(report)) - required_failed = [ - name - for name, res in report.results - if not res.ok - and name in ("python", "python-deps", "native-runtime", "smoke", "frontend") - ] + required_failed = [name for name, _ in report.required_failures] if required_failed: raise RuntimeError( "Setup could not complete: " diff --git a/craftos_integrations/README.md b/craftos_integrations/README.md index b5357387..a76b6182 100644 --- a/craftos_integrations/README.md +++ b/craftos_integrations/README.md @@ -169,7 +169,9 @@ has `listen` enabled, and reconciles whenever accounts change. "messageId": "", "is_self_message": False, "raw": {...}, # full original platform event - "attachments": [...], # normalized non-text payloads ({kind, id, name, mime, size, url, extra}) + "attachments": [ + ... + ], # normalized non-text payloads ({kind, id, name, mime, size, url, extra}) "truncated": False, # True when the listener capped messageBody } ``` diff --git a/craftos_integrations/providers/supabase/client.py b/craftos_integrations/providers/supabase/client.py index 338f94c3..1fa99031 100644 --- a/craftos_integrations/providers/supabase/client.py +++ b/craftos_integrations/providers/supabase/client.py @@ -434,7 +434,9 @@ def _go() -> Result: # ── transport: project data plane ──────────────────────────────── - async def _keys_for(self, ref: str, *, refresh: bool = False) -> Dict[str, Optional[str]]: + async def _keys_for( + self, ref: str, *, refresh: bool = False + ) -> Dict[str, Optional[str]]: if not refresh and ref in self._project_keys: return self._project_keys[ref] result = await self._mgmt( @@ -447,7 +449,11 @@ async def _keys_for(self, ref: str, *, refresh: bool = False) -> Dict[str, Optio + " — the connected account needs the secrets:read permission " "on this project." ) - keys: Dict[str, Optional[str]] = {"secret": None, "publishable": None, "jwt": None} + keys: Dict[str, Optional[str]] = { + "secret": None, + "publishable": None, + "jwt": None, + } legacy_anon: Optional[str] = None for item in result.get("result") or []: kind = item.get("type") @@ -649,7 +655,9 @@ async def get_project(self, project_ref: Optional[str] = None) -> Result: async def list_regions( self, organization_slug: str, *, continent: Optional[str] = None ) -> Result: - params = _clean({"organization_slug": organization_slug, "continent": continent}) + params = _clean( + {"organization_slug": organization_slug, "continent": continent} + ) return await self._mgmt("GET", "projects/available-regions", params=params) async def create_project( @@ -685,11 +693,17 @@ async def create_project( result = await self._mgmt( "POST", "projects", json_body=body, expected=_EXPECT_WRITE, timeout=60.0 ) - if "error" not in result and generated and isinstance(result.get("result"), dict): + if ( + "error" not in result + and generated + and isinstance(result.get("result"), dict) + ): result["result"]["generated_db_password"] = password return result - async def update_project(self, name: str, *, project_ref: Optional[str] = None) -> Result: + async def update_project( + self, name: str, *, project_ref: Optional[str] = None + ) -> Result: self._guard_write() return await self._mgmt( "PATCH", @@ -1033,8 +1047,12 @@ async def select_rows( if count: headers["Prefer"] = "count=exact" result, response = await self._rest( - ref, "GET", f"rest/v1/{quote(table, safe='')}", - params=params, headers=headers, expected=(200, 206), + ref, + "GET", + f"rest/v1/{quote(table, safe='')}", + params=params, + headers=headers, + expected=(200, 206), ) if "error" in result: return result @@ -1075,8 +1093,12 @@ async def insert_rows( headers["Prefer"] = ",".join(prefer) params = [("on_conflict", on_conflict)] if on_conflict else None result, _ = await self._rest( - ref, "POST", f"rest/v1/{quote(table, safe='')}", - params=params, json_body=rows, headers=headers, + ref, + "POST", + f"rest/v1/{quote(table, safe='')}", + params=params, + json_body=rows, + headers=headers, ) return result @@ -1100,8 +1122,12 @@ async def update_rows( headers = self._profile_headers(schema, write=True) headers["Prefer"] = "return=representation" result, _ = await self._rest( - self._ref(project_ref), "PATCH", f"rest/v1/{quote(table, safe='')}", - params=self._filter_params(filters), json_body=values, headers=headers, + self._ref(project_ref), + "PATCH", + f"rest/v1/{quote(table, safe='')}", + params=self._filter_params(filters), + json_body=values, + headers=headers, ) return result @@ -1122,8 +1148,11 @@ async def delete_rows( headers = self._profile_headers(schema, write=True) headers["Prefer"] = "return=representation" result, _ = await self._rest( - self._ref(project_ref), "DELETE", f"rest/v1/{quote(table, safe='')}", - params=self._filter_params(filters), headers=headers, + self._ref(project_ref), + "DELETE", + f"rest/v1/{quote(table, safe='')}", + params=self._filter_params(filters), + headers=headers, ) return result @@ -1138,8 +1167,11 @@ async def call_rpc( # A database function can do anything, so treat it as a write. self._guard_write() result, _ = await self._rest( - self._ref(project_ref), "POST", f"rest/v1/rpc/{quote(function, safe='')}", - json_body=args or {}, headers=self._profile_headers(schema, write=True), + self._ref(project_ref), + "POST", + f"rest/v1/rpc/{quote(function, safe='')}", + json_body=args or {}, + headers=self._profile_headers(schema, write=True), ) return result @@ -1191,7 +1223,10 @@ async def update_bucket( # PUT replaces the bucket's settings — read first so unspecified # fields keep their current values. current, _ = await self._data( - ref, "GET", f"storage/v1/bucket/{quote(name, safe='')}", expected=_EXPECT_READ + ref, + "GET", + f"storage/v1/bucket/{quote(name, safe='')}", + expected=_EXPECT_READ, ) if "error" in current: return current @@ -1216,17 +1251,24 @@ async def update_bucket( ) return result - async def delete_bucket(self, name: str, *, project_ref: Optional[str] = None) -> Result: + async def delete_bucket( + self, name: str, *, project_ref: Optional[str] = None + ) -> Result: self._guard_write() result, _ = await self._data( - self._ref(project_ref), "DELETE", f"storage/v1/bucket/{quote(name, safe='')}" + self._ref(project_ref), + "DELETE", + f"storage/v1/bucket/{quote(name, safe='')}", ) return result - async def empty_bucket(self, name: str, *, project_ref: Optional[str] = None) -> Result: + async def empty_bucket( + self, name: str, *, project_ref: Optional[str] = None + ) -> Result: self._guard_write() result, _ = await self._data( - self._ref(project_ref), "POST", + self._ref(project_ref), + "POST", f"storage/v1/bucket/{quote(name, safe='')}/empty", ) return result @@ -1251,9 +1293,11 @@ async def list_files( } ) result, _ = await self._data( - self._ref(project_ref), "POST", + self._ref(project_ref), + "POST", f"storage/v1/object/list/{quote(bucket, safe='')}", - json_body=body, expected=_EXPECT_READ, + json_body=body, + expected=_EXPECT_READ, ) if "error" in result: return result @@ -1278,14 +1322,19 @@ async def upload_file( local = os.path.abspath(file_path) if not os.path.isfile(local): return {"error": f"File not found: {local}"} - mime = content_type or mimetypes.guess_type(local)[0] or "application/octet-stream" + mime = ( + content_type or mimetypes.guess_type(local)[0] or "application/octet-stream" + ) with open(local, "rb") as f: payload = f.read() headers = {"Content-Type": mime, "x-upsert": "true" if upsert else "false"} result, _ = await self._data( - self._ref(project_ref), "POST", + self._ref(project_ref), + "POST", f"storage/v1/object/{quote(bucket, safe='')}/{_object_path(path)}", - content=payload, headers=headers, timeout=300.0, + content=payload, + headers=headers, + timeout=300.0, ) if "error" not in result: result["result"] = { @@ -1293,7 +1342,11 @@ async def upload_file( "path": path.strip("/"), "size": len(payload), "content_type": mime, - **(result.get("result") if isinstance(result.get("result"), dict) else {}), + **( + result.get("result") + if isinstance(result.get("result"), dict) + else {} + ), } return result @@ -1306,9 +1359,12 @@ async def download_file( project_ref: Optional[str] = None, ) -> Result: result, response = await self._data( - self._ref(project_ref), "GET", + self._ref(project_ref), + "GET", f"storage/v1/object/{quote(bucket, safe='')}/{_object_path(path)}", - expected=_EXPECT_READ, timeout=300.0, raw=True, + expected=_EXPECT_READ, + timeout=300.0, + raw=True, ) if "error" in result or response is None: return result @@ -1336,7 +1392,8 @@ async def delete_files( if not paths: return {"error": "paths must list at least one object path."} result, _ = await self._data( - self._ref(project_ref), "DELETE", + self._ref(project_ref), + "DELETE", f"storage/v1/object/{quote(bucket, safe='')}", json_body={"prefixes": [p.strip("/") for p in paths]}, ) @@ -1398,7 +1455,8 @@ async def create_signed_url( ) -> Result: ref = self._ref(project_ref) result, _ = await self._data( - ref, "POST", + ref, + "POST", f"storage/v1/object/sign/{quote(bucket, safe='')}/{_object_path(path)}", json_body={"expiresIn": _clamp(expires_in, 1, 60 * 60 * 24 * 365, 3600)}, ) @@ -1426,7 +1484,9 @@ async def list_users( ) -> Result: per = _clamp(per_page, 1, 1000, 50) result, _ = await self._data( - self._ref(project_ref), "GET", "auth/v1/admin/users", + self._ref(project_ref), + "GET", + "auth/v1/admin/users", params={"page": max(int(page or 1), 1), "per_page": per}, expected=_EXPECT_READ, ) @@ -1442,9 +1502,13 @@ async def list_users( }, } - async def get_user(self, user_id: str, *, project_ref: Optional[str] = None) -> Result: + async def get_user( + self, user_id: str, *, project_ref: Optional[str] = None + ) -> Result: result, _ = await self._data( - self._ref(project_ref), "GET", f"auth/v1/admin/users/{quote(user_id, safe='')}", + self._ref(project_ref), + "GET", + f"auth/v1/admin/users/{quote(user_id, safe='')}", expected=_EXPECT_READ, ) return result @@ -1510,8 +1574,10 @@ async def update_user( if not body: return {"error": "Nothing to update — pass at least one field."} result, _ = await self._data( - self._ref(project_ref), "PUT", - f"auth/v1/admin/users/{quote(user_id, safe='')}", json_body=body, + self._ref(project_ref), + "PUT", + f"auth/v1/admin/users/{quote(user_id, safe='')}", + json_body=body, ) return result @@ -1524,7 +1590,8 @@ async def delete_user( ) -> Result: self._guard_write() result, _ = await self._data( - self._ref(project_ref), "DELETE", + self._ref(project_ref), + "DELETE", f"auth/v1/admin/users/{quote(user_id, safe='')}", json_body={"should_soft_delete": bool(soft_delete)}, ) @@ -1541,8 +1608,11 @@ async def invite_user( self._guard_write() params = {"redirect_to": redirect_to} if redirect_to else None result, _ = await self._data( - self._ref(project_ref), "POST", "auth/v1/invite", - params=params, json_body=_clean({"email": email, "data": data}), + self._ref(project_ref), + "POST", + "auth/v1/invite", + params=params, + json_body=_clean({"email": email, "data": data}), ) return result @@ -1560,8 +1630,14 @@ async def generate_auth_link( """Generate a signup / magiclink / recovery / invite / email-change link without sending an email.""" self._guard_write() - valid = ("signup", "magiclink", "recovery", "invite", "email_change_current", - "email_change_new") + valid = ( + "signup", + "magiclink", + "recovery", + "invite", + "email_change_current", + "email_change_new", + ) if link_type not in valid: return {"error": f"link_type must be one of {', '.join(valid)}"} params = {"redirect_to": redirect_to} if redirect_to else None @@ -1575,8 +1651,11 @@ async def generate_auth_link( } ) result, _ = await self._data( - self._ref(project_ref), "POST", "auth/v1/admin/generate_link", - params=params, json_body=body, + self._ref(project_ref), + "POST", + "auth/v1/admin/generate_link", + params=params, + json_body=body, ) return result @@ -1587,7 +1666,9 @@ async def generate_auth_link( async def list_functions(self, *, project_ref: Optional[str] = None) -> Result: return await self._mgmt("GET", f"projects/{self._ref(project_ref)}/functions") - async def get_function(self, slug: str, *, project_ref: Optional[str] = None) -> Result: + async def get_function( + self, slug: str, *, project_ref: Optional[str] = None + ) -> Result: return await self._mgmt( "GET", f"projects/{self._ref(project_ref)}/functions/{quote(slug, safe='')}" ) @@ -1613,8 +1694,10 @@ async def deploy_function( self._guard_write() sources: List[Tuple[str, bytes]] = [] if files: - sources = [(p.replace("\\", "/").lstrip("/"), c.encode("utf-8")) - for p, c in files.items()] + sources = [ + (p.replace("\\", "/").lstrip("/"), c.encode("utf-8")) + for p, c in files.items() + ] elif source_dir: root = os.path.abspath(source_dir) if not os.path.isdir(root): @@ -1665,8 +1748,10 @@ async def update_function( self._guard_write() body = _clean({"name": name, "verify_jwt": verify_jwt}) if not body: - return {"error": "Nothing to update — pass name and/or verify_jwt. " - "To change code, use deploy_supabase_function."} + return { + "error": "Nothing to update — pass name and/or verify_jwt. " + "To change code, use deploy_supabase_function." + } return await self._mgmt( "PATCH", f"projects/{self._ref(project_ref)}/functions/{quote(slug, safe='')}", @@ -1674,7 +1759,9 @@ async def update_function( expected=_EXPECT_WRITE, ) - async def delete_function(self, slug: str, *, project_ref: Optional[str] = None) -> Result: + async def delete_function( + self, slug: str, *, project_ref: Optional[str] = None + ) -> Result: self._guard_write() return await self._mgmt( "DELETE", @@ -1696,14 +1783,22 @@ async def invoke_function( verb = (method or "POST").upper() if verb not in ("GET", "POST", "PUT", "PATCH", "DELETE"): return {"error": f"Unsupported method {method!r}."} - extra = {k: v for k, v in (headers or {}).items() - if k.lower() not in ("apikey", "authorization")} + extra = { + k: v + for k, v in (headers or {}).items() + if k.lower() not in ("apikey", "authorization") + } # Functions with verify_jwt need a JWT; only legacy keys are JWTs. result, response = await self._data( - self._ref(project_ref), verb, f"functions/v1/{quote(slug, safe='')}", + self._ref(project_ref), + verb, + f"functions/v1/{quote(slug, safe='')}", json_body=body if verb != "GET" else None, - headers=extra, prefer_jwt=True, expected=range(200, 300), - timeout=150.0, raw=True, + headers=extra, + prefer_jwt=True, + expected=range(200, 300), + timeout=150.0, + raw=True, ) if response is None: return result @@ -1824,13 +1919,21 @@ async def update_branch( ) -> Result: self._guard_write() body = _clean( - {"branch_name": branch_name, "git_branch": git_branch, "persistent": persistent} + { + "branch_name": branch_name, + "git_branch": git_branch, + "persistent": persistent, + } ) if not body: - return {"error": "Nothing to update — pass branch_name, git_branch or persistent."} + return { + "error": "Nothing to update — pass branch_name, git_branch or persistent." + } return await self._mgmt( - "PATCH", f"branches/{quote(branch_id, safe='')}", - json_body=body, expected=_EXPECT_WRITE, + "PATCH", + f"branches/{quote(branch_id, safe='')}", + json_body=body, + expected=_EXPECT_WRITE, ) async def delete_branch(self, branch_id: str) -> Result: @@ -1844,9 +1947,11 @@ async def merge_branch( ) -> Result: self._guard_write() return await self._mgmt( - "POST", f"branches/{quote(branch_id, safe='')}/merge", + "POST", + f"branches/{quote(branch_id, safe='')}/merge", json_body=_clean({"migration_version": migration_version}), - expected=_EXPECT_WRITE, timeout=60.0, + expected=_EXPECT_WRITE, + timeout=60.0, ) async def reset_branch( @@ -1854,16 +1959,20 @@ async def reset_branch( ) -> Result: self._guard_write() return await self._mgmt( - "POST", f"branches/{quote(branch_id, safe='')}/reset", + "POST", + f"branches/{quote(branch_id, safe='')}/reset", json_body=_clean({"migration_version": migration_version}), - expected=_EXPECT_WRITE, timeout=60.0, + expected=_EXPECT_WRITE, + timeout=60.0, ) async def diff_branch( self, branch_id: str, *, schemas: Optional[List[str]] = None ) -> Result: params = {"included_schemas": ",".join(schemas)} if schemas else {} - return await self._mgmt_text(f"branches/{quote(branch_id, safe='')}/diff", params) + return await self._mgmt_text( + f"branches/{quote(branch_id, safe='')}/diff", params + ) # ================================================================= # Monitoring — logs, advisors, usage @@ -1912,7 +2021,9 @@ def _iso(ts: float) -> str: "iso_timestamp_end": _iso(end), } path = f"projects/{self._ref(project_ref)}/analytics/endpoints/logs" - result = _analytics_result(await self._mgmt("GET", path, params=params, timeout=60.0)) + result = _analytics_result( + await self._mgmt("GET", path, params=params, timeout=60.0) + ) # Supabase asks for a retry on transient backend failures. if "Backend error" in str(result.get("details") or ""): await asyncio.sleep(2.0) @@ -1928,14 +2039,20 @@ def _iso(ts: float) -> str: ) return result - async def get_security_advisors(self, *, project_ref: Optional[str] = None) -> Result: + async def get_security_advisors( + self, *, project_ref: Optional[str] = None + ) -> Result: return await self._mgmt( "GET", f"projects/{self._ref(project_ref)}/advisors/security", timeout=60.0 ) - async def get_performance_advisors(self, *, project_ref: Optional[str] = None) -> Result: + async def get_performance_advisors( + self, *, project_ref: Optional[str] = None + ) -> Result: return await self._mgmt( - "GET", f"projects/{self._ref(project_ref)}/advisors/performance", timeout=60.0 + "GET", + f"projects/{self._ref(project_ref)}/advisors/performance", + timeout=60.0, ) async def get_api_usage( diff --git a/craftos_integrations/providers/supabase/operations.py b/craftos_integrations/providers/supabase/operations.py index d12f2a2d..5ca5a276 100644 --- a/craftos_integrations/providers/supabase/operations.py +++ b/craftos_integrations/providers/supabase/operations.py @@ -122,9 +122,7 @@ def build_operations() -> List[Operation]: "takes), name, organization_slug, region and status." ), input_schema={ - "organization_slug": _s( - "Only projects in this organization.", "" - ), + "organization_slug": _s("Only projects in this organization.", ""), }, tags=(_PROJECTS, _UMBRELLA), ), @@ -165,9 +163,7 @@ def build_operations() -> List[Operation]: "Organization slug from list_supabase_organizations.", "acme-inc", ), - "db_pass": _s( - "Database password. Omit to generate a strong one.", "" - ), + "db_pass": _s("Database password. Omit to generate a strong one.", ""), "region": _s( "Region code (e.g. 'us-east-1', 'eu-central-1') or a " "smart group ('americas', 'emea', 'apac'). See " @@ -403,9 +399,7 @@ def build_operations() -> List[Operation]: "Call this before querying an unfamiliar database." ), input_schema={ - "schemas": _arr( - "Schemas to list. Default ['public'].", ["public"] - ), + "schemas": _arr("Schemas to list. Default ['public'].", ["public"]), "project_ref": _ref(), }, tags=(_DATABASE, _UMBRELLA), @@ -482,8 +476,9 @@ def build_operations() -> List[Operation]: "stored rollback statements." ), input_schema={ - "version": _s("Migration version from list_supabase_migrations.", - "20250312000000"), + "version": _s( + "Migration version from list_supabase_migrations.", "20250312000000" + ), "project_ref": _ref(), }, tags=(_MIGRATIONS,), @@ -587,8 +582,7 @@ def build_operations() -> List[Operation]: ), "upsert": _b("Merge into existing rows on conflict.", False), "on_conflict": _s( - "Comma-separated unique columns for upsert. Default: " - "primary key.", + "Comma-separated unique columns for upsert. Default: primary key.", "", ), "schema": _schema(), @@ -890,10 +884,12 @@ def build_operations() -> List[Operation]: "password": _s("Initial password.", ""), "email_confirm": _b("Mark the email as confirmed.", True), "phone_confirm": _b("Mark the phone as confirmed.", False), - "user_metadata": _obj("Profile data the user can edit.", - {"full_name": "Jane Doe"}), - "app_metadata": _obj("Server-controlled data (roles, plan).", - {"role": "admin"}), + "user_metadata": _obj( + "Profile data the user can edit.", {"full_name": "Jane Doe"} + ), + "app_metadata": _obj( + "Server-controlled data (roles, plan).", {"role": "admin"} + ), "project_ref": _ref(), }, parallelizable=False, diff --git a/craftos_integrations/providers/supabase/provider.py b/craftos_integrations/providers/supabase/provider.py index f562ceb5..7c062595 100644 --- a/craftos_integrations/providers/supabase/provider.py +++ b/craftos_integrations/providers/supabase/provider.py @@ -352,19 +352,28 @@ async def run_login(self) -> Tuple[Optional[str], Optional[Dict[str, Any]], str] result = await flow.run() access_token = result.get("access_token") or "" if not access_token: - return None, None, f"Supabase OAuth failed: {result.get('error', 'no token')}" + return ( + None, + None, + f"Supabase OAuth failed: {result.get('error', 'no token')}", + ) headers = {"Authorization": f"Bearer {access_token}"} orgs = http_request( "GET", f"{SUPABASE_MGMT}/organizations", headers=headers, expected=(200,) ) - org_list = [o for o in (orgs.get("result") or []) if isinstance(o, dict)] \ - if "error" not in orgs else [] + org_list = ( + [o for o in (orgs.get("result") or []) if isinstance(o, dict)] + if "error" not in orgs + else [] + ) org = org_list[0] if org_list else {} if len(org_list) > 1: logger.warning( "[SUPABASE] OAuth grant spans %d organizations; keying the " - "account by the first (%s).", len(org_list), org.get("slug") + "account by the first (%s).", + len(org_list), + org.get("slug"), ) # Best effort — some OAuth grants cannot read /profile. diff --git a/install.py b/install.py index 792d4357..676a4e16 100644 --- a/install.py +++ b/install.py @@ -1802,18 +1802,14 @@ def _check_mac_python() -> None: print(f"\n {RED}[x]{RESET} {WHITE}Setup incomplete.{RESET}\n") print(provision.format_report(_report)) print( - "\n Re-run `python install.py` to retry — every stage is idempotent," + "\n Use Repair in the installer, or re-run `python craftbot.py install`" + "\n for a source checkout. Setup can be retried safely," "\n so the parts that already succeeded are not redone." ) # Only a required stage is fatal. An optional one (Playwright, the # WhatsApp bridge) degrades a feature; taking the whole install down # for it would be worse than starting without it. - _required_failed = [ - name - for name, res in _report.results - if not res.ok - and name in ("python", "python-deps", "native-runtime", "smoke", "frontend") - ] + _required_failed = [name for name, _ in _report.required_failures] if _required_failed: sys.exit(1) diff --git a/launcher/Cargo.lock b/launcher/Cargo.lock index 7874c37f..9165faea 100644 --- a/launcher/Cargo.lock +++ b/launcher/Cargo.lock @@ -627,6 +627,15 @@ dependencies = [ "no_std_io2", ] +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "block2" version = "0.5.1" @@ -932,6 +941,16 @@ dependencies = [ "libc", ] +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -945,9 +964,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c07782be35f9e1140080c6b96f0d44b739e2278479f64e02fdab4e32dfd8b081" dependencies = [ "bitflags 1.3.2", - "core-foundation", + "core-foundation 0.9.4", "core-graphics-types", - "foreign-types", + "foreign-types 0.5.0", "libc", ] @@ -958,7 +977,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45390e6114f68f718cc7a830514a96f903cccd70d02a8f6d9f643ac4ba45afaf" dependencies = [ "bitflags 1.3.2", - "core-foundation", + "core-foundation 0.9.4", "libc", ] @@ -977,21 +996,34 @@ version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7704b5fdd17b18ae31c4c1da5a2e0305a2bf17b5249300a9ee9ed7b72114c636" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "craftbot-launcher" version = "0.1.0" dependencies = [ "dirs", "flate2", + "fontique", "fs4", "libc", + "native-tls", "open", "rfd", "serde", "serde_json", + "sha2", "slint", "slint-build", "tar", + "tempfile", "ureq", "windows-sys 0.59.0", "winresource", @@ -1053,6 +1085,16 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "ctor" version = "0.10.1" @@ -1119,6 +1161,16 @@ dependencies = [ "syn 3.0.4", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "dirs" version = "6.0.0" @@ -1516,6 +1568,15 @@ dependencies = [ "yeslogic-fontconfig-sys", ] +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared 0.1.1", +] + [[package]] name = "foreign-types" version = "0.5.0" @@ -1523,7 +1584,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" dependencies = [ "foreign-types-macros", - "foreign-types-shared", + "foreign-types-shared 0.3.1", ] [[package]] @@ -1537,6 +1598,12 @@ dependencies = [ "syn 3.0.4", ] +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "foreign-types-shared" version = "0.3.1" @@ -1685,6 +1752,16 @@ dependencies = [ "libc", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "gethostname" version = "1.1.0" @@ -2947,6 +3024,23 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe 0.2.2", + "openssl-sys", + "schannel", + "security-framework 3.7.0", + "security-framework-sys", + "tempfile", +] + [[package]] name = "natord" version = "1.0.9" @@ -3513,12 +3607,55 @@ dependencies = [ "libc", ] +[[package]] +name = "openssl" +version = "0.10.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4de8111e94a56adca7ebfe6c365b6d42f99deb69b74b6becacd4204e2f193aa2" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "foreign-types 0.3.2", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "openssl-probe" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +[[package]] +name = "openssl-probe" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e036b411754132fd6b74e94b74e9a13c5cf606fa93d1885c35860d3060820b1" + +[[package]] +name = "openssl-sys" +version = "0.9.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19c413d6ec9ebaf5e1c829d355cb3e9f783778d700f135be0921d52fce3c60ae" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -4318,11 +4455,11 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" dependencies = [ - "openssl-probe", + "openssl-probe 0.1.6", "rustls-pemfile", "rustls-pki-types", "schannel", - "security-framework", + "security-framework 2.11.1", ] [[package]] @@ -4434,7 +4571,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ "bitflags 2.13.1", - "core-foundation", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.13.1", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -4519,6 +4669,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "shlex" version = "1.3.0" @@ -5248,6 +5409,12 @@ dependencies = [ "serde", ] +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "udev" version = "0.9.3" @@ -5363,6 +5530,7 @@ checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" dependencies = [ "base64", "log", + "native-tls", "once_cell", "rustls", "rustls-native-certs", @@ -5448,6 +5616,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -6043,7 +6217,7 @@ dependencies = [ "calloop 0.13.0", "cfg_aliases", "concurrent-queue", - "core-foundation", + "core-foundation 0.9.4", "core-graphics", "cursor-icon", "dpi", diff --git a/launcher/Cargo.toml b/launcher/Cargo.toml index 31f4b6fb..28cc394b 100644 --- a/launcher/Cargo.toml +++ b/launcher/Cargo.toml @@ -30,11 +30,13 @@ slint = { version = "1.17", default-features = false, features = [ # HTTP, blocking. All downloads run on a worker thread, so a synchronous # client is the simplest correct choice. rustls: no OpenSSL to find or ship. -# native-certs: trust the OS certificate store rather than a bundled list, so -# a corporate TLS-inspecting proxy whose CA is installed on the machine works. +# native-certs: trust the OS certificate store. Windows uses Schannel below +# so a fresh machine can resolve certificate chains through Windows Update. ureq = { version = "2.12", default-features = false, features = ["tls", "native-certs", "json"] } serde = { version = "1", features = ["derive"] } serde_json = "1" +sha2 = "0.10" +tempfile = "3" # Archives. The source payload is a zip; python-build-standalone is .tar.gz. zip = { version = "2.4", default-features = false, features = ["deflate"] } @@ -48,6 +50,8 @@ dirs = "6" fs4 = "0.13" [target.'cfg(windows)'.dependencies] +native-tls = "0.2" +ureq = { version = "2.12", default-features = false, features = ["native-tls"] } windows-sys = { version = "0.59", features = [ "Win32_Foundation", "Win32_System_Threading", @@ -56,6 +60,11 @@ windows-sys = { version = "0.59", features = [ [target.'cfg(unix)'.dependencies] libc = "0.2" +[target.'cfg(target_os = "linux")'.dependencies] +# Font discovery is optional at startup, especially for headless installs on +# minimal Linux. Fontique gracefully falls back when fontconfig is absent. +fontique = { version = "0.10", default-features = false, features = ["fontconfig-dlopen"] } + [build-dependencies] slint-build = "1.17" diff --git a/launcher/README.md b/launcher/README.md index 4dd6209a..9f83951b 100644 --- a/launcher/README.md +++ b/launcher/README.md @@ -27,7 +27,8 @@ two things onto the disk that Python cannot fetch for itself: 1. **A Python interpreter.** It downloads [python-build-standalone](https://github.com/astral-sh/python-build-standalone) - 3.10 — the same build `app/provision/runtimes.py` uses — into the same + 3.10 — pinned with its checksum in `app/runtime-manifest.json`, embedded + in the launcher and shared with `app/provision/runtimes.py` — into the same location (`/runtime/python/python/`). When install.py's own `python` stage runs later, it finds this one and accepts it. 2. **The source payload.** `CraftBot-src.zip` from the GitHub release @@ -50,6 +51,24 @@ State is read the same way `craftbot.py` writes it in source mode: `craftbot.pid` beside `craftbot.py`, and `.agent-ready` under the user data directory for "running and actually serving" versus "still starting". +Node/npm, the Agent App tooling dependencies and PocketBase are required +provisioning stages. Setup only completes when those stages succeed. Node +is pinned in the same runtime manifest; PocketBase keeps its version pin +and per-asset checksums under `agent-app/spec/`. ZIP extraction does not +require Git, tar or unzip on the user's machine. + +Python remains on 3.10 to match the existing dependency locks. Its portable +build is pinned to the last verified release that includes this version; +migrating off this end-of-life Python line also requires regenerating and +testing the application locks. Runtime pins must be updated with their +official SHA-256 digests, not by resolving upstream `latest` at install time. + +Run `python -m unittest discover -s tests/install_bootstrap -v` for the +stdlib-only bootstrap regressions. `python scripts/test_runtime_bootstrap.py` +exercises fresh runtime downloads, app scaffolding/build/serving and reuse +in a temporary installation with an empty PATH. It does not install the +full Python application dependencies or register the CraftBot service. + ### Files the launcher owns | | Windows | macOS | Linux | @@ -73,6 +92,20 @@ Rust 1.85+ (`rustup` installs it). No system libraries are needed on Windows or macOS; on Linux the runtime uses the system's X11/Wayland and OpenGL via `dlopen`, with a software renderer as fallback. +Windows builds statically link the C runtime through the repository's +`.cargo/config.toml`. The executable must start on a fresh Windows before +Python provisioning installs the VC++ Redistributable for application wheels. +The launcher refuses to compile with a dynamic MSVC runtime. + +Windows downloads use Schannel so certificate chains resolve on a fresh +machine, including roots Windows retrieves on demand. Other platforms use +rustls with the OS trust store. The downloader honors proxy environment +variables and always verifies TLS and pinned runtime checksums. + +Linux font discovery uses optional `dlopen` loading for fontconfig. The +headless installer can start on a minimal OS without desktop font libraries; +opening the window still requires a working graphical session. + The CraftBot version is baked in at build time: `CRAFTBOT_VERSION=1.4.0 cargo build --release`, or a `VERSION` file at the repo root, or "latest" for a dev build (downloads the newest release). diff --git a/launcher/src/download.rs b/launcher/src/download.rs index a997661a..7d774056 100644 --- a/launcher/src/download.rs +++ b/launcher/src/download.rs @@ -2,6 +2,7 @@ //! already on the worker thread, and a blocking read loop is the simplest //! thing that reports bytes as they arrive. +use sha2::{Digest, Sha256}; use std::io::{Read, Write}; use std::path::Path; use std::time::Duration; @@ -10,33 +11,69 @@ const USER_AGENT: &str = concat!("CraftBotInstaller/", env!("CARGO_PKG_VERSION") pub type Progress<'a> = &'a mut dyn FnMut(u64, Option); -fn agent() -> ureq::Agent { - ureq::AgentBuilder::new() +fn agent() -> Result { + let builder = ureq::AgentBuilder::new() .user_agent(USER_AGENT) + .try_proxy_from_env(true) .timeout_connect(Duration::from_secs(30)) // Reads are bounded per call, not per download: a 100 MB file on a // slow link legitimately takes minutes. .timeout_read(Duration::from_secs(120)) - .redirects(10) - .build() + .redirects(10); + // Loading a snapshot of Windows' root store misses roots fetched on demand + // on clean PCs. Schannel performs the normal Windows chain validation and + // respects locally installed enterprise CAs without weakening verification. + #[cfg(windows)] + let builder = builder.tls_connector(std::sync::Arc::new( + native_tls::TlsConnector::new() + .map_err(|e| format!("cannot initialize Windows TLS: {e}"))?, + )); + Ok(builder.build()) } -/// Fetch a small JSON document (a GitHub release listing). -pub fn get_json(url: &str) -> Result { - let resp = agent() - .get(url) - .set("Accept", "application/vnd.github+json") - .call() - .map_err(|e| describe(url, e))?; - resp.into_json() - .map_err(|e| format!("{url}: bad JSON: {e}")) +/// Runtime pins carry a digest. Check it before anything can execute. +pub fn verified_to_file( + url: &str, + dest: &Path, + sha256: &str, + progress: Progress, +) -> Result<(), String> { + for attempt in 0..3 { + match to_file(url, dest, progress) { + Ok(()) => return verify_sha256(dest, sha256), + Err(e) if attempt == 2 => return Err(e), + Err(_) => std::thread::sleep(Duration::from_secs(attempt + 1)), + } + } + unreachable!() +} + +fn verify_sha256(path: &Path, expected: &str) -> Result<(), String> { + let mut source = std::fs::File::open(path).map_err(|e| e.to_string())?; + let mut digest = Sha256::new(); + let mut buf = [0; 256 * 1024]; + loop { + let count = source.read(&mut buf).map_err(|e| e.to_string())?; + if count == 0 { + break; + } + digest.update(&buf[..count]); + } + if format!("{:x}", digest.finalize()) != expected { + let _ = std::fs::remove_file(path); + return Err(format!( + "SHA-256 verification failed for {}", + path.display() + )); + } + Ok(()) } /// Download `url` to `dest`, writing to a `.part` file first so a partial /// download is never mistaken for a finished one. `progress` is called with /// (bytes so far, total if the server said). pub fn to_file(url: &str, dest: &Path, progress: Progress) -> Result<(), String> { - let resp = agent().get(url).call().map_err(|e| describe(url, e))?; + let resp = agent()?.get(url).call().map_err(|e| describe(url, e))?; let total = resp .header("Content-Length") .and_then(|v| v.trim().parse::().ok()); @@ -93,3 +130,29 @@ fn describe(url: &str, e: ureq::Error) -> String { pub fn mb(bytes: u64) -> String { format!("{:.0}", bytes as f64 / (1024.0 * 1024.0)) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn corrupted_download_is_rejected_and_removed() { + let dir = tempfile::tempdir().unwrap(); + let file = dir.path().join("runtime"); + std::fs::write(&file, b"truncated").unwrap(); + assert!(verify_sha256(&file, &"0".repeat(64)).is_err()); + assert!(!file.exists()); + } + + #[test] + fn complete_download_matches_digest() { + let dir = tempfile::tempdir().unwrap(); + let file = dir.path().join("runtime"); + std::fs::write(&file, b"abc").unwrap(); + assert!(verify_sha256( + &file, + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ) + .is_ok()); + } +} diff --git a/launcher/src/install.rs b/launcher/src/install.rs index 9f9ca3c0..835c49db 100644 --- a/launcher/src/install.rs +++ b/launcher/src/install.rs @@ -226,7 +226,12 @@ fn stop_if_running(tx: &Sender) -> Result<(), String> { } if let Some(rec) = snap.record { let _ = tx.send(Event::Status("Stopping the running CraftBot first…".into())); - let _ = run_craftbot(&rec.python, rec.dir(), &["stop"], tx)?; + let code = run_craftbot(&rec.python, rec.dir(), &["stop"], tx)?; + if code != 0 { + return Err(format!( + "Could not stop CraftBot before setup (exit {code}). See the log." + )); + } } Ok(()) } diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 23d2ec0c..b3fd0e69 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -17,6 +17,9 @@ #![cfg_attr(windows, windows_subsystem = "windows")] +#[cfg(all(windows, target_env = "msvc", not(target_feature = "crt-static")))] +compile_error!("The Windows installer needs a static C runtime. Build from the repository or launcher directory so .cargo/config.toml is loaded."); + mod craftbot; mod download; mod install; diff --git a/launcher/src/paths.rs b/launcher/src/paths.rs index 8e5e5ff6..a8168924 100644 --- a/launcher/src/paths.rs +++ b/launcher/src/paths.rs @@ -80,8 +80,8 @@ pub fn python_runtime_dir() -> PathBuf { } /// The sidecar interpreter, matching `PythonStage._sidecar_exe()`. -pub fn python_exe() -> PathBuf { - let root = python_runtime_dir().join("python"); +pub fn python_exe_in(runtime_dir: &Path) -> PathBuf { + let root = runtime_dir.join("python"); if cfg!(windows) { root.join("python.exe") } else { diff --git a/launcher/src/python.rs b/launcher/src/python.rs index 33511602..9e3d2143 100644 --- a/launcher/src/python.rs +++ b/launcher/src/python.rs @@ -8,9 +8,8 @@ //! module puts it. When install.py later runs its own `python` stage it //! finds this one already there and accepts it; nothing is downloaded twice. //! -//! The release tag, patch version and platform triple are resolved from the -//! GitHub API rather than hardcoded, for the reason recorded in runtimes.py: -//! a hardcoded asset name went stale and produced a silent 404. +//! Download pins are shared with Python provisioning and embedded at build +//! time: the launcher needs them before the source payload or Python exists. use crate::download; use crate::logger; @@ -21,15 +20,28 @@ use std::path::PathBuf; /// are generated for, not a minimum. pub const TARGET: (u32, u32) = (3, 10); -const PBS_API: &str = - "https://api.github.com/repos/astral-sh/python-build-standalone/releases/latest"; +const RUNTIMES: &str = include_str!("../../app/runtime-manifest.json"); + +#[derive(serde::Deserialize)] +struct Asset { + url: String, + sha256: String, +} /// The interpreter to run CraftBot with, downloading it if needed. pub fn ensure( say: &mut dyn FnMut(&str), progress: &mut dyn FnMut(u64, Option), ) -> Result { - let exe = paths::python_exe(); + ensure_in(&paths::python_runtime_dir(), say, progress) +} + +fn ensure_in( + dest_dir: &std::path::Path, + say: &mut dyn FnMut(&str), + progress: &mut dyn FnMut(u64, Option), +) -> Result { + let exe = paths::python_exe_in(dest_dir); if exe.is_file() { if let Some(v) = probe(&exe) { if v == TARGET { @@ -48,9 +60,6 @@ pub fn ensure( exe.display() )); } - // A wrong or broken sidecar is removed wholesale; a partial tree is - // worse than none. - let _ = std::fs::remove_dir_all(paths::python_runtime_dir().join("python")); } let triple = triple().ok_or_else(|| { @@ -60,25 +69,46 @@ pub fn ensure( ) })?; say(&format!("Downloading Python {}.{}…", TARGET.0, TARGET.1)); - let url = resolve_url(&triple)?; - logger::log(&format!("python: {url}")); + let asset = resolve_asset(&triple)?; + logger::log(&format!("python: {}", asset.url)); - let dest_dir = paths::python_runtime_dir(); - std::fs::create_dir_all(&dest_dir) + std::fs::create_dir_all(dest_dir) .map_err(|e| format!("cannot create {}: {e}", dest_dir.display()))?; - let archive = dest_dir.join("_download.tar.gz"); - download::to_file(&url, &archive, progress)?; + let stage = tempfile::tempdir_in(dest_dir).map_err(|e| e.to_string())?; + let archive = stage.path().join("_download.tar.gz"); + download::verified_to_file(&asset.url, &archive, &asset.sha256, progress)?; say("Unpacking Python…"); - let result = extract_tar_gz(&archive, &dest_dir); + let result = extract_tar_gz(&archive, stage.path()); let _ = std::fs::remove_file(&archive); result?; - if !exe.is_file() { - return Err(format!("Python unpacked, but {} is missing", exe.display())); + let candidate = paths::python_exe_in(stage.path()); + if !candidate.is_file() { + return Err(format!( + "Python unpacked, but {} is missing", + candidate.display() + )); } - match probe(&exe) { + match probe(&candidate) { Some(v) if v == TARGET => { + let installed = dest_dir.join("python"); + let backup = dest_dir.join(".python-previous"); + if backup.exists() { + std::fs::remove_dir_all(&backup).map_err(|e| e.to_string())?; + } + if installed.exists() { + std::fs::rename(&installed, &backup).map_err(|e| e.to_string())?; + } + if let Err(e) = std::fs::rename(stage.path().join("python"), &installed) { + if backup.exists() { + let _ = std::fs::rename(&backup, &installed); + } + return Err(format!("cannot finish Python installation: {e}")); + } + if backup.exists() { + let _ = std::fs::remove_dir_all(&backup); + } say(&format!("Python {}.{} ready", v.0, v.1)); Ok(exe) } @@ -115,8 +145,10 @@ pub fn probe(exe: &std::path::Path) -> Option<(u32, u32)> { fn triple() -> Option { let arch = if cfg!(target_arch = "aarch64") { "aarch64" - } else { + } else if cfg!(target_arch = "x86_64") { "x86_64" + } else { + return None; }; if cfg!(target_os = "windows") { Some("x86_64-pc-windows-msvc".to_string()) @@ -129,28 +161,17 @@ fn triple() -> Option { } } -fn resolve_url(triple: &str) -> Result { - let data = download::get_json(PBS_API) - .map_err(|e| format!("could not reach the Python download index: {e}"))?; - let prefix = format!("cpython-{}.{}.", TARGET.0, TARGET.1); - let suffix = format!("-{triple}-install_only.tar.gz"); - let assets = data - .get("assets") - .and_then(|a| a.as_array()) - .cloned() - .unwrap_or_default(); - for asset in &assets { - let name = asset.get("name").and_then(|n| n.as_str()).unwrap_or(""); - if name.starts_with(&prefix) && name.ends_with(&suffix) { - if let Some(url) = asset.get("browser_download_url").and_then(|u| u.as_str()) { - return Ok(url.to_string()); - } - } +fn resolve_asset(triple: &str) -> Result { + let data: serde_json::Value = serde_json::from_str(RUNTIMES).map_err(|e| e.to_string())?; + let asset: Asset = serde_json::from_value(data["python"]["assets"][triple].clone()) + .map_err(|_| format!("No approved Python download for {triple}"))?; + if !asset.url.starts_with("https://") + || asset.sha256.len() != 64 + || !asset.sha256.bytes().all(|c| c.is_ascii_hexdigit()) + { + return Err("Invalid Python runtime download manifest".into()); } - let tag = data.get("tag_name").and_then(|t| t.as_str()).unwrap_or("?"); - Err(format!( - "no {prefix}*{suffix} in python-build-standalone release {tag}" - )) + Ok(asset) } /// Extract a .tar.gz, preserving permissions — the executable bit on @@ -174,3 +195,49 @@ pub fn free_space(path: &std::path::Path) -> Option { } fs4::available_space(&probe).ok() } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + #[ignore = "downloads and executes the release-pinned Python"] + fn live_python_bootstrap() { + let temp = tempfile::tempdir().unwrap(); + let dest = temp.path().join("User Name & caf\u{e9}"); + let executable = ensure_in(&dest, &mut |_| {}, &mut |_, _| {}).unwrap(); + assert_eq!(probe(&executable), Some(TARGET)); + assert_eq!( + ensure_in(&dest, &mut |_| {}, &mut |_, _| panic!( + "ready runtime must be reused" + )) + .unwrap(), + executable + ); + } + + #[test] + fn manifest_resolves_all_platforms_without_network() { + for target in [ + "x86_64-pc-windows-msvc", + "x86_64-apple-darwin", + "aarch64-apple-darwin", + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-gnu", + ] { + let asset = resolve_asset(target).unwrap(); + assert!(asset.url.contains("/20261009/cpython-3.10.22%2B")); + assert_eq!(asset.sha256.len(), 64); + } + assert!(resolve_asset("unsupported").is_err()); + } + + #[test] + fn manifest_matches_dependency_lock_python() { + let data: serde_json::Value = serde_json::from_str(RUNTIMES).unwrap(); + assert!(data["python"]["version"] + .as_str() + .unwrap() + .starts_with(&format!("{}.{}.", TARGET.0, TARGET.1))); + } +} diff --git a/run.py b/run.py index c94a507f..c337feb1 100644 --- a/run.py +++ b/run.py @@ -1081,10 +1081,15 @@ def launch_agent_background( # skips re-resolution and both processes agree on the same binary. Not # in conda mode: there the env's own node (>= 24 via environment.yml, # first on PATH under conda run) is the intended runtime. - if not use_conda and "CRAFTBOT_NODE" not in agent_env: - _rt = node_runtime.resolve() - if _rt: - agent_env["CRAFTBOT_NODE"] = _rt.node + if not use_conda: + # MCP servers invoke bare npx/node too. Passing only CRAFTBOT_NODE + # helps our wrappers but leaves those subprocesses unable to start + # on a machine without a system Node installation. + agent_env.update(node_runtime.path_env()) + if "CRAFTBOT_NODE" not in agent_env: + _rt = node_runtime.resolve() + if _rt: + agent_env["CRAFTBOT_NODE"] = _rt.node # When running as a PyInstaller frozen binary, run main() in a thread # instead of spawning a subprocess (sys.executable is the binary itself) diff --git a/scripts/package_source.py b/scripts/package_source.py index 725843bd..8b840f43 100644 --- a/scripts/package_source.py +++ b/scripts/package_source.py @@ -59,9 +59,13 @@ EXCLUDE_SUFFIXES = (".md",) -# Kept despite the rules above — README is the one doc worth shipping, and -# the i18n README variants are not. -KEEP_EXACT = {"README.md", "LICENSE"} +# These Markdown files are runtime inputs, not just repository documentation: +# create.ts substitutes the blueprint, and the creator skill reads the guide. +AGENT_APP_RUNTIME_DOCS = { + "agent-app/blueprint/AGENT_APP.md", + "agent-app/docs/agent-guide.md", +} +KEEP_EXACT = {"README.md", "LICENSE"} | AGENT_APP_RUNTIME_DOCS def tracked_files() -> List[str]: @@ -131,20 +135,32 @@ def build(output: Path) -> int: # A payload without these is installable and then broken at runtime, in # ways that surface far from the cause. Fail the build instead. - required = ripgrep_runtime.bundled_files() + [ - "run.py", - "main.py", - "requirements.txt", - "app/paths.py", - "app/provision/__init__.py", - "app/ui_layer/browser/frontend/dist/index.html", - "craftos_integrations/providers/whatsapp_web/bridge.js", - # craftbot.py imports these at module scope; without them every - # command fails before parsing its arguments. - "installer/helpers.py", - "installer/metadata.py", - "installer/payload.py", - ] + required = ( + ripgrep_runtime.bundled_files() + + sorted(AGENT_APP_RUNTIME_DOCS) + + [ + "run.py", + "main.py", + "requirements.txt", + "app/paths.py", + "app/provision/__init__.py", + "app/runtime-manifest.json", + "app/runtime_manifest.py", + "agent-app/package.json", + "agent-app/package-lock.json", + "agent-app/tools/package.json", + "agent-app/tools/src/cli.ts", + "agent-app/spec/pocketbase.version", + "agent-app/spec/pocketbase-checksums.json", + "app/ui_layer/browser/frontend/dist/index.html", + "craftos_integrations/providers/whatsapp_web/bridge.js", + # craftbot.py imports these at module scope; without them every + # command fails before parsing its arguments. + "installer/helpers.py", + "installer/metadata.py", + "installer/payload.py", + ] + ) with zipfile.ZipFile(output) as zf: names = set(zf.namelist()) absent = [r for r in required if r not in names] diff --git a/scripts/prefetch_runtimes.py b/scripts/prefetch_runtimes.py index 62f4ff63..d780f20a 100644 --- a/scripts/prefetch_runtimes.py +++ b/scripts/prefetch_runtimes.py @@ -44,7 +44,7 @@ def _fetch_archives(out: Path) -> int: """Runtime archives: Python, Node, and (on Windows) the VC++ runtime.""" - from app import downloads, node_runtime + from app import downloads, node_runtime, runtime_manifest from app.provision import runtimes from app.provision.verify import _vcredist_url @@ -52,28 +52,32 @@ def _fetch_archives(out: Path) -> int: py_url = runtimes._pbs_download_url(print) if py_url: - targets.append(("Python runtime", py_url)) + targets.append( + ("Python runtime", py_url, runtime_manifest.asset("python")["sha256"]) + ) else: print("warning: could not resolve the Python runtime URL", file=sys.stderr) node_url = node_runtime.latest_download_url(log=print) if node_url: - targets.append(("Node.js", node_url)) + targets.append(("Node.js", node_url, runtime_manifest.asset("node")["sha256"])) else: print("warning: could not resolve the Node URL", file=sys.stderr) if sys.platform == "win32": - targets.append(("Visual C++ runtime", _vcredist_url())) + targets.append(("Visual C++ runtime", _vcredist_url(), None)) failures = 0 - for label, url in targets: + for label, url, checksum in targets: dest = out / downloads.cache_name(url) - if dest.is_file(): + if downloads.verify_checksum(str(dest), checksum): print(f"\n{label}: cached already ({dest.stat().st_size / 1048576:.1f} MB)") continue print(f"\n{label}:") try: - downloads.download(url, str(dest), log=print, label=label) + downloads.download( + url, str(dest), log=print, label=label, expected_sha256=checksum + ) except Exception as e: failures += 1 print(f" failed: {str(e)[:200]}", file=sys.stderr) diff --git a/scripts/smoke_supabase.py b/scripts/smoke_supabase.py index d1cd6181..b86e0023 100644 --- a/scripts/smoke_supabase.py +++ b/scripts/smoke_supabase.py @@ -168,8 +168,11 @@ async def run(args: argparse.Namespace) -> int: p.get("organization_slug") for p in (result_of(projects) or []) ] if slugs and slugs[0]: - await check(results, "list_organization_members", - client.list_organization_members(slugs[0])) + await check( + results, + "list_organization_members", + client.list_organization_members(slugs[0]), + ) await check(results, "list_regions", client.list_regions(slugs[0])) else: results.add(SKIP, "list_organization_members", "no organization visible") @@ -177,7 +180,8 @@ async def run(args: argparse.Namespace) -> int: ref = args.project_ref if not ref: active = [ - p for p in (result_of(projects) or []) + p + for p in (result_of(projects) or []) if str(p.get("status", "")).startswith("ACTIVE") ] ref = active[0]["ref"] if active else "" @@ -189,26 +193,42 @@ async def run(args: argparse.Namespace) -> int: # ── project reads ──────────────────────────────────────────────── print("\n--- project reads ---") await check(results, "get_project", client.get_project(ref)) - await check(results, "get_project_health", client.get_project_health(project_ref=ref)) + await check( + results, "get_project_health", client.get_project_health(project_ref=ref) + ) await check(results, "get_project_keys", client.get_project_keys(ref)) for service in ("auth", "postgrest", "realtime", "storage", "postgres"): await check( - results, f"get_service_config[{service}]", + results, + f"get_service_config[{service}]", client.get_service_config(service, project_ref=ref), ) - await check(results, "run_sql_readonly", client.run_sql_readonly( - "select $1::text as echo, now() as at", parameters=["hi"], project_ref=ref)) + await check( + results, + "run_sql_readonly", + client.run_sql_readonly( + "select $1::text as echo, now() as at", parameters=["hi"], project_ref=ref + ), + ) tables = await check(results, "list_tables", client.list_tables(project_ref=ref)) table_rows = result_of(tables) or [] if table_rows: first = table_rows[0] - await check(results, "describe_table", client.describe_table( - first["name"], schema=first["schema"], project_ref=ref)) + await check( + results, + "describe_table", + client.describe_table( + first["name"], schema=first["schema"], project_ref=ref + ), + ) else: results.add(SKIP, "describe_table", "no tables in public") await check(results, "list_extensions", client.list_extensions(project_ref=ref)) - await check(results, "generate_typescript_types", - client.generate_typescript_types(project_ref=ref)) + await check( + results, + "generate_typescript_types", + client.generate_typescript_types(project_ref=ref), + ) await check(results, "list_backups", client.list_backups(project_ref=ref)) await check(results, "list_migrations", client.list_migrations(project_ref=ref)) await check(results, "list_buckets", client.list_buckets(project_ref=ref)) @@ -217,11 +237,19 @@ async def run(args: argparse.Namespace) -> int: await check(results, "list_secrets", client.list_secrets(project_ref=ref)) await check(results, "list_branches", client.list_branches(project_ref=ref)) for source in ("api", "postgres", "auth"): - await check(results, f"get_logs[{source}]", - client.get_logs(source=source, limit=5, project_ref=ref)) - await check(results, "get_security_advisors", client.get_security_advisors(project_ref=ref)) - await check(results, "get_performance_advisors", - client.get_performance_advisors(project_ref=ref)) + await check( + results, + f"get_logs[{source}]", + client.get_logs(source=source, limit=5, project_ref=ref), + ) + await check( + results, "get_security_advisors", client.get_security_advisors(project_ref=ref) + ) + await check( + results, + "get_performance_advisors", + client.get_performance_advisors(project_ref=ref), + ) await check(results, "get_api_usage", client.get_api_usage(project_ref=ref)) if args.write: @@ -252,77 +280,156 @@ async def run_writes(client: Any, ref: str, results: Results) -> None: # ── migrations + rows ──────────────────────────────────────────── print("\n--- write: schema + rows ---") - migration = await check(results, "apply_migration", client.apply_migration( - f"create table public.{name} (id bigint generated always as identity " - f"primary key, label text not null, n int default 0);", - name=f"create_{name}", - rollback=f"drop table if exists public.{name};", - project_ref=ref, - )) + migration = await check( + results, + "apply_migration", + client.apply_migration( + f"create table public.{name} (id bigint generated always as identity " + f"primary key, label text not null, n int default 0);", + name=f"create_{name}", + rollback=f"drop table if exists public.{name};", + project_ref=ref, + ), + ) if migration is not None: - await check(results, "insert_rows", client.insert_rows( - name, [{"label": "a"}, {"label": "b"}], project_ref=ref)) - await check(results, "select_rows", client.select_rows( - name, order="id.asc", count=True, project_ref=ref)) - await check(results, "update_rows", client.update_rows( - name, {"n": 5}, {"label": "eq.a"}, project_ref=ref)) - await check(results, "delete_rows", client.delete_rows( - name, {"label": "eq.b"}, project_ref=ref)) - await check(results, "run_sql (write)", client.run_sql( - f"insert into public.{name} (label) values ($1) returning id", - parameters=["c"], project_ref=ref)) + await check( + results, + "insert_rows", + client.insert_rows(name, [{"label": "a"}, {"label": "b"}], project_ref=ref), + ) + await check( + results, + "select_rows", + client.select_rows(name, order="id.asc", count=True, project_ref=ref), + ) + await check( + results, + "update_rows", + client.update_rows(name, {"n": 5}, {"label": "eq.a"}, project_ref=ref), + ) + await check( + results, + "delete_rows", + client.delete_rows(name, {"label": "eq.b"}, project_ref=ref), + ) + await check( + results, + "run_sql (write)", + client.run_sql( + f"insert into public.{name} (label) values ($1) returning id", + parameters=["c"], + project_ref=ref, + ), + ) else: - for step in ("insert_rows", "select_rows", "update_rows", "delete_rows", - "run_sql (write)"): + for step in ( + "insert_rows", + "select_rows", + "update_rows", + "delete_rows", + "run_sql (write)", + ): results.add(SKIP, step, "table was not created") # ── storage ────────────────────────────────────────────────────── print("\n--- write: storage ---") bucket = name.replace("_", "-") - made = await check(results, "create_bucket", client.create_bucket(bucket, project_ref=ref)) + made = await check( + results, "create_bucket", client.create_bucket(bucket, project_ref=ref) + ) if made is not None: - await check(results, "update_bucket", client.update_bucket( - bucket, file_size_limit=1048576, project_ref=ref)) + await check( + results, + "update_bucket", + client.update_bucket(bucket, file_size_limit=1048576, project_ref=ref), + ) with tempfile.TemporaryDirectory() as tmp: src = os.path.join(tmp, "hello.txt") with open(src, "w", encoding="utf-8") as f: f.write("hello from craftbot") - await check(results, "upload_file", client.upload_file( - bucket, "dir/hello.txt", src, project_ref=ref)) - await check(results, "list_files", client.list_files( - bucket, prefix="dir", project_ref=ref)) + await check( + results, + "upload_file", + client.upload_file(bucket, "dir/hello.txt", src, project_ref=ref), + ) + await check( + results, + "list_files", + client.list_files(bucket, prefix="dir", project_ref=ref), + ) dst = os.path.join(tmp, "back.txt") - got = await check(results, "download_file", client.download_file( - bucket, "dir/hello.txt", dst, project_ref=ref)) + got = await check( + results, + "download_file", + client.download_file(bucket, "dir/hello.txt", dst, project_ref=ref), + ) if got is not None: with open(dst, encoding="utf-8") as f: same = f.read() == "hello from craftbot" - results.add(PASS if same else FAIL, "download round-trip", - "content matches" if same else "content differs") - await check(results, "create_signed_url", client.create_signed_url( - bucket, "dir/hello.txt", expires_in=60, project_ref=ref)) - await check(results, "copy_file", client.copy_file( - bucket, "dir/hello.txt", "dir/copy.txt", project_ref=ref)) - await check(results, "move_file", client.move_file( - bucket, "dir/copy.txt", "dir/moved.txt", project_ref=ref)) - await check(results, "delete_files", client.delete_files( - bucket, ["dir/moved.txt"], project_ref=ref)) - await check(results, "empty_bucket", client.empty_bucket(bucket, project_ref=ref)) - await check(results, "delete_bucket", client.delete_bucket(bucket, project_ref=ref)) + results.add( + PASS if same else FAIL, + "download round-trip", + "content matches" if same else "content differs", + ) + await check( + results, + "create_signed_url", + client.create_signed_url( + bucket, "dir/hello.txt", expires_in=60, project_ref=ref + ), + ) + await check( + results, + "copy_file", + client.copy_file(bucket, "dir/hello.txt", "dir/copy.txt", project_ref=ref), + ) + await check( + results, + "move_file", + client.move_file(bucket, "dir/copy.txt", "dir/moved.txt", project_ref=ref), + ) + await check( + results, + "delete_files", + client.delete_files(bucket, ["dir/moved.txt"], project_ref=ref), + ) + await check( + results, "empty_bucket", client.empty_bucket(bucket, project_ref=ref) + ) + await check( + results, "delete_bucket", client.delete_bucket(bucket, project_ref=ref) + ) # ── auth users ─────────────────────────────────────────────────── print("\n--- write: auth users ---") - user = await check(results, "create_user", client.create_user( - email=f"{name}@example.com", password=f"Smoke-{stamp}-pw!", - email_confirm=True, project_ref=ref)) + user = await check( + results, + "create_user", + client.create_user( + email=f"{name}@example.com", + password=f"Smoke-{stamp}-pw!", + email_confirm=True, + project_ref=ref, + ), + ) user_id = (result_of(user) or {}).get("id") if user else None if user_id: await check(results, "get_user", client.get_user(user_id, project_ref=ref)) - await check(results, "update_user", client.update_user( - user_id, user_metadata={"smoke": True}, project_ref=ref)) - await check(results, "generate_auth_link", client.generate_auth_link( - "magiclink", f"{name}@example.com", project_ref=ref)) - await check(results, "delete_user", client.delete_user(user_id, project_ref=ref)) + await check( + results, + "update_user", + client.update_user(user_id, user_metadata={"smoke": True}, project_ref=ref), + ) + await check( + results, + "generate_auth_link", + client.generate_auth_link( + "magiclink", f"{name}@example.com", project_ref=ref + ), + ) + await check( + results, "delete_user", client.delete_user(user_id, project_ref=ref) + ) else: for step in ("get_user", "update_user", "generate_auth_link", "delete_user"): results.add(SKIP, step, "user was not created") @@ -330,47 +437,74 @@ async def run_writes(client: Any, ref: str, results: Results) -> None: # ── secrets + edge function ────────────────────────────────────── print("\n--- write: secrets + edge function ---") secret = f"CRAFTBOT_SMOKE_{stamp}" - await check(results, "set_secrets", client.set_secrets({secret: "ok"}, project_ref=ref)) + await check( + results, "set_secrets", client.set_secrets({secret: "ok"}, project_ref=ref) + ) slug = f"craftbot-smoke-{stamp}" - deployed = await check(results, "deploy_function", client.deploy_function( - slug, - files={"index.ts": ( - "Deno.serve(async (req) => new Response(JSON.stringify({ " - f"secret: Deno.env.get('{secret}') }}), " - "{ headers: { 'Content-Type': 'application/json' } }))" - )}, - verify_jwt=False, - project_ref=ref, - )) + deployed = await check( + results, + "deploy_function", + client.deploy_function( + slug, + files={ + "index.ts": ( + "Deno.serve(async (req) => new Response(JSON.stringify({ " + f"secret: Deno.env.get('{secret}') }}), " + "{ headers: { 'Content-Type': 'application/json' } }))" + ) + }, + verify_jwt=False, + project_ref=ref, + ), + ) if deployed is not None: await check(results, "get_function", client.get_function(slug, project_ref=ref)) - await check(results, "update_function", client.update_function( - slug, name="CraftBot smoke", project_ref=ref)) + await check( + results, + "update_function", + client.update_function(slug, name="CraftBot smoke", project_ref=ref), + ) # Freshly deployed functions can take a few seconds to route. await asyncio.sleep(5) - await check(results, "invoke_function", client.invoke_function( - slug, body={}, project_ref=ref)) - await check(results, "delete_function", client.delete_function(slug, project_ref=ref)) - await check(results, "delete_secrets", client.delete_secrets([secret], project_ref=ref)) + await check( + results, + "invoke_function", + client.invoke_function(slug, body={}, project_ref=ref), + ) + await check( + results, "delete_function", client.delete_function(slug, project_ref=ref) + ) + await check( + results, "delete_secrets", client.delete_secrets([secret], project_ref=ref) + ) # ── cleanup: a reversing migration (rollback is branch-only) ───── print("\n--- write: cleanup ---") if migration is not None: - await check(results, "apply_migration (drop)", client.apply_migration( - f"drop table if exists public.{name};", - name=f"drop_{name}", - project_ref=ref, - )) + await check( + results, + "apply_migration (drop)", + client.apply_migration( + f"drop table if exists public.{name};", + name=f"drop_{name}", + project_ref=ref, + ), + ) def main() -> int: parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) - parser.add_argument("--token", default="", help="Supabase personal access token (sbp_…)") parser.add_argument( - "--connected", action="store_true", + "--token", default="", help="Supabase personal access token (sbp_…)" + ) + parser.add_argument( + "--connected", + action="store_true", help="use the Supabase account connected in CraftBot instead of --token", ) - parser.add_argument("--project-ref", default="", help="project to test (default: first active)") + parser.add_argument( + "--project-ref", default="", help="project to test (default: first active)" + ) parser.add_argument( "--write", action="store_true", diff --git a/scripts/test_runtime_bootstrap.py b/scripts/test_runtime_bootstrap.py new file mode 100644 index 00000000..60e0b06e --- /dev/null +++ b/scripts/test_runtime_bootstrap.py @@ -0,0 +1,133 @@ +"""Online smoke test of runtime setup in an isolated, tool-free installation. + +Copies only source (no node_modules/runtimes), downloads the pinned runtimes, +prepares Agent Apps, and verifies a second run reuses the installation. Does +not install the Python application packages or register/start the service. +""" + +import os +import argparse +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--download-cache", + type=Path, + help="Optional verified runtime archive cache for slow test hosts", + ) + args = parser.parse_args() + with tempfile.TemporaryDirectory(prefix="CraftBot setup caf\u00e9 ") as temporary: + root = Path(temporary) + app = root / "app" + app.mkdir() + for name in ( + "__init__.py", + "paths.py", + "downloads.py", + "node_runtime.py", + "runtime_manifest.py", + "runtime-manifest.json", + ): + shutil.copy2(ROOT / "app" / name, app / name) + shutil.copytree( + ROOT / "app/provision", + app / "provision", + ignore=shutil.ignore_patterns("__pycache__"), + ) + shutil.copytree( + ROOT / "agent-app", + root / "agent-app", + ignore=shutil.ignore_patterns( + "node_modules", "examples", "dist", "__pycache__" + ), + ) + (root / "agent-app/examples").mkdir() + (root / ".craftbot-managed").touch() + # Exercise the shipped code under a fresh source layout and user-data root. + worker = r""" +import os, subprocess, socket, time, urllib.request +from app import node_runtime +from app.provision import Context +from app.provision.pipeline import run +from app.provision.runtimes import PythonStage, NodeStage +from app.provision.deps import AgentAppToolsStage, PocketBaseStage +ctx = Context(os.getcwd(), os.environ['CRAFTBOT_HOME']) +stages = [PythonStage(), NodeStage(), AgentAppToolsStage(), PocketBaseStage()] +# Force bootstrap even when the test harness happens to be Python 3.10. +result = stages[0].apply(ctx, print) +assert result.ok, result +report = run(stages, ctx, print) +assert report.ok, report.to_dict() +node = node_runtime.node_cmd() +cli = os.path.join(ctx.code_root, 'agent-app/tools/src/cli.ts') +subprocess.run([node, cli, 'create', 'Bootstrap smoke', '--dir', os.environ['CRAFTBOT_TEST_APPS'], '--port', '18090'], env=node_runtime.child_env(), check=True) +project = os.path.join(os.environ['CRAFTBOT_TEST_APPS'], 'bootstrap-smoke') +subprocess.run(node_runtime.npm_command() + ['install', '--ignore-scripts', '--no-audit', '--no-fund'], + cwd=os.path.join(project, 'frontend'), env=node_runtime.child_env(), check=True) +subprocess.run([node, cli, 'validate', project], env=node_runtime.child_env(), check=True) +binary = subprocess.check_output([node, cli, 'pb', 'path'], env=node_runtime.child_env(), text=True).strip().splitlines()[-1] +with socket.socket() as reservation: + reservation.bind(('127.0.0.1', 0)) + port = reservation.getsockname()[1] +server = subprocess.Popen([binary, 'serve', f'--http=127.0.0.1:{port}', '--dir', os.path.join(project, 'pb/pb_data'), + '--hooksDir', os.path.join(project, 'pb/pb_hooks'), '--publicDir', os.path.join(project, 'pb/pb_public')], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) +try: + for attempt in range(60): + try: + with urllib.request.urlopen(f'http://127.0.0.1:{port}/api/health', timeout=2) as response: + assert response.status == 200 + with urllib.request.urlopen(f'http://127.0.0.1:{port}/', timeout=2) as response: + assert b' None: self._recorder = recorder - self.chat = SimpleNamespace( - completions=SimpleNamespace(create=self._create) - ) + self.chat = SimpleNamespace(completions=SimpleNamespace(create=self._create)) def _create(self, **kwargs): self._recorder.record("openai_compat", "chat.completions.create", kwargs) @@ -120,9 +118,7 @@ def converse(self, **kwargs): self._recorder.record("bedrock", "converse", kwargs) return { "output": { - "message": { - "content": [{"text": self._recorder.next_content()}] - } + "message": {"content": [{"text": self._recorder.next_content()}]} }, "usage": {"inputTokens": 100, "outputTokens": 10}, } @@ -276,9 +272,7 @@ def build_interface(monkeypatch, provider: str, model: str): # Phase 2; patch the module reference the live code actually uses. from agent_core.core.impl.llm.transports import chat_completions - monkeypatch.setattr( - chat_completions, "requests", make_fake_requests(recorder) - ) + monkeypatch.setattr(chat_completions, "requests", make_fake_requests(recorder)) else: # openai / deepseek / grok / openrouter / glm / fugu / minimax / moonshot ctx["client"] = FakeOpenAIClient(recorder) @@ -311,8 +305,7 @@ def assert_snapshot(name: str, data: Dict[str, Any]) -> None: if os.environ.get("GOLDEN_UPDATE") == "1" or not path.exists(): path.write_text( - json.dumps(canonical, indent=2, sort_keys=True, ensure_ascii=False) - + "\n", + json.dumps(canonical, indent=2, sort_keys=True, ensure_ascii=False) + "\n", encoding="utf-8", ) return diff --git a/tests/llm/test_reliability.py b/tests/llm/test_reliability.py index 36006e56..d0f6d868 100644 --- a/tests/llm/test_reliability.py +++ b/tests/llm/test_reliability.py @@ -122,9 +122,7 @@ def fallback_pair(monkeypatch): import app.config as app_config - monkeypatch.setattr( - app_config, "get_fallback_providers", lambda: ["anthropic"] - ) + monkeypatch.setattr(app_config, "get_fallback_providers", lambda: ["anthropic"]) monkeypatch.setattr(app_config, "get_api_key", lambda p: "k") # Inject the pre-built fallback interface (its factory patch would # otherwise have been torn down by build_interface's second call). diff --git a/tests/test_app_readiness.py b/tests/test_app_readiness.py index 685b68c3..381697da 100644 --- a/tests/test_app_readiness.py +++ b/tests/test_app_readiness.py @@ -67,10 +67,13 @@ async def probe(): def test_healthy_probe_slower_than_half_a_second_can_finish_within_deadline(): async def run(): manager = AgentAppManager.__new__(AgentAppManager) + async def probe(): - await asyncio.sleep(.6) + await asyncio.sleep(0.6) return True + assert await manager._probe_ready(probe, 1) + asyncio.run(run()) @@ -79,16 +82,20 @@ async def run(): manager = AgentAppManager.__new__(AgentAppManager) entered, cancelled = asyncio.Event(), asyncio.Event() status = [None] + async def probe(): entered.set() try: await asyncio.Event().wait() finally: cancelled.set() - task = asyncio.create_task(manager._probe_ready( - probe, 10, SimpleNamespace(poll=lambda: status[0]))) + + task = asyncio.create_task( + manager._probe_ready(probe, 10, SimpleNamespace(poll=lambda: status[0])) + ) await entered.wait() status[0] = 1 assert not await asyncio.wait_for(task, 1) assert cancelled.is_set() + asyncio.run(run()) diff --git a/tests/test_credential_api_request.py b/tests/test_credential_api_request.py index e3c05492..3859c666 100644 --- a/tests/test_credential_api_request.py +++ b/tests/test_credential_api_request.py @@ -16,14 +16,14 @@ from app.credentials import api_request as api from app.logger import logger -KEY = 'sk-test-only+api/key&2026=' -TEXT = 'Café crème' +KEY = "sk-test-only+api/key&2026=" +TEXT = "Café crème" @pytest.fixture def vault(tmp_path, monkeypatch): - vault = CredentialVault(base_dir=tmp_path / '.credentials') - monkeypatch.setattr(api, 'get_vault', lambda: vault) + vault = CredentialVault(base_dir=tmp_path / ".credentials") + monkeypatch.setattr(api, "get_vault", lambda: vault) return vault @@ -36,63 +36,90 @@ def log_message(self, *_args): pass def do_GET(self): - calls.append({'path': self.path, 'headers': dict(self.headers)}) - if self.path == '/redirect': + calls.append({"path": self.path, "headers": dict(self.headers)}) + if self.path == "/redirect": self.send_response(302) - self.send_header('Location', '/must-not-follow') + self.send_header("Location", "/must-not-follow") self.end_headers() return - if self.path == '/not-modified': + if self.path == "/not-modified": self.send_response(304) self.end_headers() return - if self.path in ('/text', '/latin1'): - latin1 = self.path == '/latin1' + if self.path in ("/text", "/latin1"): + latin1 = self.path == "/latin1" self.send_response(200) - self.send_header('Content-Type', 'text/plain; charset=ISO-8859-1' if latin1 else 'text/plain') + self.send_header( + "Content-Type", + "text/plain; charset=ISO-8859-1" if latin1 else "text/plain", + ) self.end_headers() - self.wfile.write(TEXT.encode('latin-1' if latin1 else 'utf-8')) + self.wfile.write(TEXT.encode("latin-1" if latin1 else "utf-8")) return - if self.path == '/large': - body = ('x' * api._MAX_BODY_BYTES + KEY).encode() + if self.path == "/large": + body = ("x" * api._MAX_BODY_BYTES + KEY).encode() else: - value = self.headers.get('Authorization') or self.headers.get('X-API-Key') or self.headers.get('x-goog-api-key') - self.send_response(200 if self.path != '/unauthorized' and value in (KEY, 'Bearer ' + KEY) else 401) - body = json.dumps({'ok': True, 'echo': value, 'url': self.path, - 'encoded': quote(KEY, safe=''), - 'base64': base64.b64encode(KEY.encode()).decode()}).encode() - self.send_header('Content-Type', 'application/json') - self.send_header('X-Echo', KEY) # Response headers must never reach the model. + value = ( + self.headers.get("Authorization") + or self.headers.get("X-API-Key") + or self.headers.get("x-goog-api-key") + ) + self.send_response( + 200 + if self.path != "/unauthorized" and value in (KEY, "Bearer " + KEY) + else 401 + ) + body = json.dumps( + { + "ok": True, + "echo": value, + "url": self.path, + "encoded": quote(KEY, safe=""), + "base64": base64.b64encode(KEY.encode()).decode(), + } + ).encode() + self.send_header("Content-Type", "application/json") + self.send_header( + "X-Echo", KEY + ) # Response headers must never reach the model. self.end_headers() self.wfile.write(body) return self.send_response(200) - self.send_header('Content-Type', 'text/plain') + self.send_header("Content-Type", "text/plain") self.end_headers() self.wfile.write(body) def do_POST(self): - body = self.rfile.read(int(self.headers.get('Content-Length', 0))).decode() - calls.append({'path': self.path, 'body': body, 'headers': dict(self.headers)}) + body = self.rfile.read(int(self.headers.get("Content-Length", 0))).decode() + calls.append( + {"path": self.path, "body": body, "headers": dict(self.headers)} + ) self.send_response(200) - self.send_header('Content-Type', 'application/json') + self.send_header("Content-Type", "application/json") self.end_headers() - self.wfile.write(json.dumps({'created': json.loads(body)}).encode()) + self.wfile.write(json.dumps({"created": json.loads(body)}).encode()) - server = ThreadingHTTPServer(('127.0.0.1', 0), Handler) + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() try: - yield f'http://127.0.0.1:{server.server_port}', calls + yield f"http://127.0.0.1:{server.server_port}", calls finally: server.shutdown() server.server_close() thread.join(timeout=2) -def saved(vault, site='api.example.test', *, kind='api_key', key=KEY): - return vault.add_entry(site, 'account' if kind == 'login' else '', key, - 'Example key', kind=kind, group='work') +def saved(vault, site="api.example.test", *, kind="api_key", key=KEY): + return vault.add_entry( + site, + "account" if kind == "login" else "", + key, + "Example key", + kind=kind, + group="work", + ) def run(data): @@ -101,135 +128,201 @@ def run(data): def test_list_only_api_metadata_and_filter(vault): key = saved(vault) - saved(vault, kind='login') - saved(vault, site='', kind='secure_note') - result = asyncio.run(api.list_api_keys({'query': 'example.test'})) - assert result == {'status': 'success', 'credentials': [ - {'id': key['id'], 'site': 'api.example.test', 'label': 'Example key', 'group': 'work'}]} + saved(vault, kind="login") + saved(vault, site="", kind="secure_note") + result = asyncio.run(api.list_api_keys({"query": "example.test"})) + assert result == { + "status": "success", + "credentials": [ + { + "id": key["id"], + "site": "api.example.test", + "label": "Example key", + "group": "work", + } + ], + } assert KEY not in json.dumps(result) - assert asyncio.run(api.list_api_keys({'query': 'absent'}))['credentials'] == [] + assert asyncio.run(api.list_api_keys({"query": "absent"}))["credentials"] == [] -@pytest.mark.parametrize('auth,header', [('bearer', 'Authorization'), ('header', 'X-API-Key')]) +@pytest.mark.parametrize( + "auth,header", [("bearer", "Authorization"), ("header", "X-API-Key")] +) def test_real_request_injects_key_and_sanitizes_echo(vault, server, auth, header): url, calls = server entry = saved(vault, url) - result = run({'credential_id': entry['id'], 'url': url + '/echo', 'auth_type': auth, - 'params': {'page': 2}}) - assert result['status'] == 'success' - assert json.loads(result['body'])['ok'] is True - assert '[redacted]' in result['body'] + result = run( + { + "credential_id": entry["id"], + "url": url + "/echo", + "auth_type": auth, + "params": {"page": 2}, + } + ) + assert result["status"] == "success" + assert json.loads(result["body"])["ok"] is True + assert "[redacted]" in result["body"] text = json.dumps(result) - for variant in (KEY, quote(KEY, safe=''), base64.b64encode(KEY.encode()).decode()): + for variant in (KEY, quote(KEY, safe=""), base64.b64encode(KEY.encode()).decode()): assert variant not in text - assert 'response_headers' not in result - assert calls[0]['headers'][header] == ('Bearer ' if auth == 'bearer' else '') + KEY - assert calls[0]['path'] == '/echo?page=2' - assert KEY not in calls[0]['path'] + assert "response_headers" not in result + assert calls[0]["headers"][header] == ("Bearer " if auth == "bearer" else "") + KEY + assert calls[0]["path"] == "/echo?page=2" + assert KEY not in calls[0]["path"] def test_json_write_request(vault, server): url, calls = server - result = run({'credential_id': saved(vault, url)['id'], 'url': url + '/items', - 'method': 'POST', 'json': {'name': 'Widget'}, 'headers': {'Accept': 'application/json'}}) - assert result['status'] == 'success' - assert json.loads(result['body']) == {'created': {'name': 'Widget'}} - assert json.loads(calls[0]['body']) == {'name': 'Widget'} + result = run( + { + "credential_id": saved(vault, url)["id"], + "url": url + "/items", + "method": "POST", + "json": {"name": "Widget"}, + "headers": {"Accept": "application/json"}, + } + ) + assert result["status"] == "success" + assert json.loads(result["body"]) == {"created": {"name": "Widget"}} + assert json.loads(calls[0]["body"]) == {"name": "Widget"} def test_custom_api_key_header(vault, server): url, calls = server - result = run({'credential_id': saved(vault, url)['id'], 'url': url + '/echo', - 'auth_type': 'header', 'header_name': 'x-goog-api-key'}) - assert result['status'] == 'success' - assert calls[0]['headers']['x-goog-api-key'] == KEY + result = run( + { + "credential_id": saved(vault, url)["id"], + "url": url + "/echo", + "auth_type": "header", + "header_name": "x-goog-api-key", + } + ) + assert result["status"] == "success" + assert calls[0]["headers"]["x-goog-api-key"] == KEY assert KEY not in json.dumps(result) def test_http_error_response_is_also_sanitized(vault, server): url, _ = server - result = run({'credential_id': saved(vault, url)['id'], 'url': url + '/unauthorized'}) - assert result['status'] == 'error' - assert result['status_code'] == 401 - assert '[redacted]' in result['body'] + result = run( + {"credential_id": saved(vault, url)["id"], "url": url + "/unauthorized"} + ) + assert result["status"] == "error" + assert result["status_code"] == 401 + assert "[redacted]" in result["body"] assert KEY not in json.dumps(result) def test_redirect_never_follows_or_returns_location(vault, server): url, calls = server - result = run({'credential_id': saved(vault, url)['id'], 'url': url + '/redirect'}) - assert result['error_code'] == 'CREDENTIAL_REDIRECT_REFUSED' - assert [call['path'] for call in calls] == ['/redirect'] - assert 'Location' not in json.dumps(result) + result = run({"credential_id": saved(vault, url)["id"], "url": url + "/redirect"}) + assert result["error_code"] == "CREDENTIAL_REDIRECT_REFUSED" + assert [call["path"] for call in calls] == ["/redirect"] + assert "Location" not in json.dumps(result) def test_not_modified_is_returned_not_refused_as_redirect(vault, server): url, _ = server - result = run({'credential_id': saved(vault, url)['id'], 'url': url + '/not-modified'}) - assert result['status'] == 'success' - assert result['status_code'] == 304 + result = run( + {"credential_id": saved(vault, url)["id"], "url": url + "/not-modified"} + ) + assert result["status"] == "success" + assert result["status_code"] == 304 -@pytest.mark.parametrize('path', ['/text', '/latin1']) +@pytest.mark.parametrize("path", ["/text", "/latin1"]) def test_text_response_uses_declared_charset_else_utf8(vault, server, path): url, _ = server - result = run({'credential_id': saved(vault, url)['id'], 'url': url + path}) - assert result['body'] == TEXT + result = run({"credential_id": saved(vault, url)["id"], "url": url + path}) + assert result["body"] == TEXT def test_oversize_response_returns_no_partial_value(vault, server): url, _ = server - result = run({'credential_id': saved(vault, url)['id'], 'url': url + '/large'}) - assert result['error_code'] == 'CREDENTIAL_RESPONSE_TOO_LARGE' - assert 'body' not in result - - -@pytest.mark.parametrize('target', [ - 'https://evil.example.test/', 'https://sub.api.example.test/', - 'https://www.api.example.test/', 'https://api.example.test:444/', - 'https://api.example.test:80/', 'http://api.example.test/', -]) + result = run({"credential_id": saved(vault, url)["id"], "url": url + "/large"}) + assert result["error_code"] == "CREDENTIAL_RESPONSE_TOO_LARGE" + assert "body" not in result + + +@pytest.mark.parametrize( + "target", + [ + "https://evil.example.test/", + "https://sub.api.example.test/", + "https://www.api.example.test/", + "https://api.example.test:444/", + "https://api.example.test:80/", + "http://api.example.test/", + ], +) def test_wrong_destination_refused_before_network(vault, monkeypatch, target): entry = saved(vault) calls = [] - monkeypatch.setattr(requests.Session, 'send', lambda *_a, **_kw: calls.append(True)) - result = run({'credential_id': entry['id'], 'url': target}) - assert result['status'] == 'error' + monkeypatch.setattr(requests.Session, "send", lambda *_a, **_kw: calls.append(True)) + result = run({"credential_id": entry["id"], "url": target}) + assert result["status"] == "error" assert not calls def test_www_api_host_is_kept_and_matched_exactly(vault): - entry = saved(vault, 'https://www.api.example.test/v1') - assert entry['site'] == 'www.api.example.test' - assert vault.api_key_for_url(entry['id'], 'https://www.api.example.test/v1').reveal() == KEY + entry = saved(vault, "https://www.api.example.test/v1") + assert entry["site"] == "www.api.example.test" + assert ( + vault.api_key_for_url(entry["id"], "https://www.api.example.test/v1").reveal() + == KEY + ) with pytest.raises(VaultError): - vault.api_key_for_url(entry['id'], 'https://api.example.test/v1') + vault.api_key_for_url(entry["id"], "https://api.example.test/v1") -@pytest.mark.parametrize('kind,site', [('login', 'api.example.test'), ('secure_note', ''), ('api_key', '')]) +@pytest.mark.parametrize( + "kind,site", [("login", "api.example.test"), ("secure_note", ""), ("api_key", "")] +) def test_wrong_type_or_missing_website_refused(vault, kind, site): - result = run({'credential_id': saved(vault, site, kind=kind)['id'], 'url': 'https://api.example.test/'}) - assert result['error_code'] == 'CREDENTIAL_UNAVAILABLE' + result = run( + { + "credential_id": saved(vault, site, kind=kind)["id"], + "url": "https://api.example.test/", + } + ) + assert result["error_code"] == "CREDENTIAL_UNAVAILABLE" assert KEY not in json.dumps(result) -@pytest.mark.parametrize('changes', [ - {'url': 'https://user:pass@api.example.test/'}, - {'url': 'https://api.example.test/#fragment'}, {'url': 'https://api.example.test/\n'}, - {'url': 'https://evil.test\\@api.example.test/'}, {'url': 'file:///tmp/data'}, - {'headers': {'Host': 'evil.test'}}, {'headers': {'authorization': 'override'}}, - {'headers': {'X-Thing': 'value\r\nX-Other: bad'}}, {'header_name': 'Host'}, - {'header_name': 'Proxy-Authorization'}, {'header_name': 'Cookie'}, - {'auth_type': 'query'}, {'verify_tls': False}, {'allow_redirects': True}, - {'save_to': 'leak.txt'}, {'timeout': float('nan')}, {'timeout': True}, - {'timeout': 61}, {'json': {}, 'data': 'both'}, {'params': []}, -]) +@pytest.mark.parametrize( + "changes", + [ + {"url": "https://user:pass@api.example.test/"}, + {"url": "https://api.example.test/#fragment"}, + {"url": "https://api.example.test/\n"}, + {"url": "https://evil.test\\@api.example.test/"}, + {"url": "file:///tmp/data"}, + {"headers": {"Host": "evil.test"}}, + {"headers": {"authorization": "override"}}, + {"headers": {"X-Thing": "value\r\nX-Other: bad"}}, + {"header_name": "Host"}, + {"header_name": "Proxy-Authorization"}, + {"header_name": "Cookie"}, + {"auth_type": "query"}, + {"verify_tls": False}, + {"allow_redirects": True}, + {"save_to": "leak.txt"}, + {"timeout": float("nan")}, + {"timeout": True}, + {"timeout": 61}, + {"json": {}, "data": "both"}, + {"params": []}, + ], +) def test_unsafe_request_inputs_never_send(vault, monkeypatch, changes): entry = saved(vault) calls = [] - monkeypatch.setattr(requests.Session, 'send', lambda *_a, **_kw: calls.append(True)) - result = run({'credential_id': entry['id'], 'url': 'https://api.example.test/', **changes}) - assert result['status'] == 'error' + monkeypatch.setattr(requests.Session, "send", lambda *_a, **_kw: calls.append(True)) + result = run( + {"credential_id": entry["id"], "url": "https://api.example.test/", **changes} + ) + assert result["status"] == "error" assert not calls @@ -239,38 +332,50 @@ def test_transport_failures_do_not_echo_key_or_log_tracebacks(vault, monkeypatch def fail(session, prepared, **kwargs): observed.update(kwargs) - observed['trust_env'] = session.trust_env - assert prepared.headers['Authorization'] == 'Bearer ' + KEY - raise requests.ConnectionError('Request failed: ' + KEY) + observed["trust_env"] = session.trust_env + assert prepared.headers["Authorization"] == "Bearer " + KEY + raise requests.ConnectionError("Request failed: " + KEY) - monkeypatch.setattr(requests.Session, 'send', fail) + monkeypatch.setattr(requests.Session, "send", fail) logs = [] - sink = logger.add(lambda message: logs.append(str(message)), diagnose=True, backtrace=True) + sink = logger.add( + lambda message: logs.append(str(message)), diagnose=True, backtrace=True + ) try: - result = run({'credential_id': entry['id'], 'url': 'https://api.example.test/'}) + result = run({"credential_id": entry["id"], "url": "https://api.example.test/"}) finally: logger.remove(sink) - assert result['error_code'] == 'CREDENTIAL_REQUEST_FAILED' - assert KEY not in json.dumps(result) + ''.join(logs) - assert observed['verify'] is True - assert observed['allow_redirects'] is False - assert observed['trust_env'] is False - assert observed['stream'] is True + assert result["error_code"] == "CREDENTIAL_REQUEST_FAILED" + assert KEY not in json.dumps(result) + "".join(logs) + assert observed["verify"] is True + assert observed["allow_redirects"] is False + assert observed["trust_env"] is False + assert observed["stream"] is True def test_redaction_decodes_json_escapes_and_scrubs_nested_keys(): - escaped = ''.join('\\u%04x' % ord(c) for c in KEY) + escaped = "".join("\\u%04x" % ord(c) for c in KEY) payload = '{"' + escaped + '": [{"value":"' + escaped + '"}]}' - assert json.loads(api._redact(payload, Secret(KEY))) == {'[redacted]': [{'value': '[redacted]'}]} + assert json.loads(api._redact(payload, Secret(KEY))) == { + "[redacted]": [{"value": "[redacted]"}] + } def test_vault_missing_or_unreadable_fails_without_network(vault, monkeypatch): entry = saved(vault) vault.key_path.unlink() calls = [] - monkeypatch.setattr(requests.Session, 'send', lambda *_a, **_kw: calls.append(True)) - assert asyncio.run(api.list_api_keys({}))['error_code'] == 'CREDENTIAL_VAULT_UNAVAILABLE' - assert run({'credential_id': entry['id'], 'url': 'https://api.example.test/'})['status'] == 'error' + monkeypatch.setattr(requests.Session, "send", lambda *_a, **_kw: calls.append(True)) + assert ( + asyncio.run(api.list_api_keys({}))["error_code"] + == "CREDENTIAL_VAULT_UNAVAILABLE" + ) + assert ( + run({"credential_id": entry["id"], "url": "https://api.example.test/"})[ + "status" + ] + == "error" + ) assert not calls @@ -278,29 +383,41 @@ def test_actions_load_and_executor_stripped_source_uses_real_backend(vault, serv from agent_core import load_actions_from_directories, registry_instance root = Path(__file__).resolve().parent.parent - load_actions_from_directories(base_dir=str(root), paths_to_scan=['app/data/action']) + load_actions_from_directories(base_dir=str(root), paths_to_scan=["app/data/action"]) url, calls = server entry = saved(vault, url) def execute(name, data): impl = registry_instance.get_action_implementation(name) - assert impl.metadata.action_sets == ['core'] - assert impl.metadata.mode == 'ALL' - assert impl.metadata.execution_mode == 'internal' + assert impl.metadata.action_sets == ["core"] + assert impl.metadata.mode == "ALL" + assert impl.metadata.execution_mode == "internal" # Irreversible actions replay a stored output for identical calls, # which would hand the agent stale API reads. assert not impl.metadata.irreversible - namespace = {'input_data': data, 'json': json, 'asyncio': asyncio} + namespace = {"input_data": data, "json": json, "asyncio": asyncio} before = set(namespace) - exec(registry_instance.find_action_by_name(name)['code'], namespace) - function = next(v for k, v in namespace.items() if k not in before and inspect.isfunction(v)) + exec(registry_instance.find_action_by_name(name)["code"], namespace) + function = next( + v for k, v in namespace.items() if k not in before and inspect.isfunction(v) + ) return asyncio.run(function(data)) - listed = execute('credential_list_api_keys', {}) - assert listed['credentials'][0]['id'] == entry['id'] - result = execute('credential_api_request', {'credential_id': entry['id'], 'url': url + '/echo', '_session_id': 'test-session'}) - assert result['status'] == 'success' + listed = execute("credential_list_api_keys", {}) + assert listed["credentials"][0]["id"] == entry["id"] + result = execute( + "credential_api_request", + { + "credential_id": entry["id"], + "url": url + "/echo", + "_session_id": "test-session", + }, + ) + assert result["status"] == "success" assert KEY not in json.dumps(result) - assert calls[0]['headers']['Authorization'] == 'Bearer ' + KEY - assert execute('credential_api_request', {'simulated_mode': True})['status'] == 'success' + assert calls[0]["headers"]["Authorization"] == "Bearer " + KEY + assert ( + execute("credential_api_request", {"simulated_mode": True})["status"] + == "success" + ) assert len(calls) == 1 diff --git a/tests/test_mini_browser_actions_api.py b/tests/test_mini_browser_actions_api.py index eab72a99..164673b4 100644 --- a/tests/test_mini_browser_actions_api.py +++ b/tests/test_mini_browser_actions_api.py @@ -165,7 +165,9 @@ def test_choices_flags_and_the_remaining_operations(): _invalid("click", {"element_id": 1, "button": "side"}) _invalid("scroll", {"direction": "sideways"}) assert validate("login", {}) == { - "login_name": None, "username": None, "submit": True + "login_name": None, + "username": None, + "submit": True, } assert validate("login", {"username": " bob ", "submit": 0}) == { "login_name": None, @@ -184,11 +186,15 @@ def test_choices_flags_and_the_remaining_operations(): def test_login_name_validation(): assert validate("login", {"login_name": " Work account ", "username": " bob "}) == { - "login_name": "Work account", "username": "bob", "submit": True + "login_name": "Work account", + "username": "bob", + "submit": True, } assert validate("login", {"login_name": "x" * 100})["login_name"] == "x" * 100 assert "login_name is too long" in _invalid("login", {"login_name": "x" * 101}) - assert "login_name must be text" in _invalid("login", {"login_name": {"label": "Work"}}) + assert "login_name must be text" in _invalid( + "login", {"login_name": {"label": "Work"}} + ) def test_tabs_params(): diff --git a/tests/test_mini_browser_agent_integration.py b/tests/test_mini_browser_agent_integration.py index 854afd9d..dca347b9 100644 --- a/tests/test_mini_browser_agent_integration.py +++ b/tests/test_mini_browser_agent_integration.py @@ -232,7 +232,11 @@ def test_original_handlers_answer_simulated_mode(registry): "hover": {"element_id": 1}, "scroll": {"direction": "down", "amount": 600}, "wait": {"seconds": 1}, - "login": {"login_name": "Work account", "username": "me@example.com", "submit": True}, + "login": { + "login_name": "Work account", + "username": "me@example.com", + "submit": True, + }, "screenshot": {"full_page": False}, "tabs": {"action": "list"}, } diff --git a/tests/test_mini_browser_fleet.py b/tests/test_mini_browser_fleet.py index 6b20e4b3..44566351 100644 --- a/tests/test_mini_browser_fleet.py +++ b/tests/test_mini_browser_fleet.py @@ -527,8 +527,10 @@ async def observed(tab): def test_human_input_supersedes_a_pending_restored_document(site, paths, human_action): import_playwright_or_skip() config.profile_dir("chat-a").mkdir(parents=True) - resume.save_pages(config.profile_dir("chat-a"), resume.Pages( - pages=(resume.Page(site.url("/set"), "Set"),), viewed=0)) + resume.save_pages( + config.profile_dir("chat-a"), + resume.Pages(pages=(resume.Page(site.url("/set"), "Set"),), viewed=0), + ) host = make_host() async def exercise(core): diff --git a/tests/test_mini_browser_ops.py b/tests/test_mini_browser_ops.py index cde43b06..1f86c504 100644 --- a/tests/test_mini_browser_ops.py +++ b/tests/test_mini_browser_ops.py @@ -994,7 +994,9 @@ async def scenario(core, tab): @pytest.mark.parametrize("name", ["Work account", " work ACCOUNT "]) -def test_login_selects_saved_name_instead_of_most_recent_account(browser, login_site, name): +def test_login_selects_saved_name_instead_of_most_recent_account( + browser, login_site, name +): vault = _vault() vault.entries[0]["label"] = "Work account" vault.entries[1]["label"] = "Personal account" @@ -1052,7 +1054,9 @@ def test_login_name_duplicate_can_be_disambiguated_by_username(browser, login_si core = FakeCore(vault=vault) async def scenario(core, tab): - return await ops.login(core, tab, login_name="Work account", username=USER.upper()) + return await ops.login( + core, tab, login_name="Work account", username=USER.upper() + ) result = run(browser, login_site.url("/login"), scenario, core=core) assert result["outcome"] == "signed_in" @@ -1067,7 +1071,8 @@ def test_login_name_does_not_allow_a_different_site(browser, login_site): async def scenario(core, tab): return await expect_error( - ops.login(core, tab, login_name="Work account"), "MINI_BROWSER_NO_SAVED_LOGIN" + ops.login(core, tab, login_name="Work account"), + "MINI_BROWSER_NO_SAVED_LOGIN", ) run(browser, login_site.url("/login", host="localhost"), scenario, core=core) diff --git a/tests/test_mini_browser_vault.py b/tests/test_mini_browser_vault.py index 9f6e82b0..70666582 100644 --- a/tests/test_mini_browser_vault.py +++ b/tests/test_mini_browser_vault.py @@ -920,7 +920,9 @@ def test_browser_compatibility_import_shares_singleton_and_locks(tmp_path): assert _get_vault() is get_vault() assert browser_vault.Secret is Secret assert browser_vault.VaultError is VaultError - assert browser_vault.CredentialVault(tmp_path)._lock is CredentialVault(tmp_path)._lock + assert ( + browser_vault.CredentialVault(tmp_path)._lock is CredentialVault(tmp_path)._lock + ) assert list(tmp_path.iterdir()) == [] @@ -953,7 +955,10 @@ def test_existing_credentials_work_without_migration(vault_dir): shared = CredentialVault(vault_dir) assert shared.list_entries()[0]["id"] == "saved-login-id" - assert shared.candidates_for_url("https://example.com/login")[0]["password"] == PASSWORD + assert ( + shared.candidates_for_url("https://example.com/login")[0]["password"] + == PASSWORD + ) assert shared.vault_path.read_bytes() == encrypted assert shared.key_path.read_bytes() == key_file assert not shared.previous_path.exists() @@ -1295,64 +1300,92 @@ def writer(vault: CredentialVault, tag: str) -> None: assert not errors assert len(CredentialVault(vault_dir).list_entries()) == 22 + # Typed credentials share the encrypted vault but never become browser logins. -@pytest.mark.parametrize('kind', ['api_key', 'secure_note']) +@pytest.mark.parametrize("kind", ["api_key", "secure_note"]) def test_typed_credentials_roundtrip_without_becoming_autofill_candidates(vault, kind): - login = vault.add_entry('example.com', 'alice', PASSWORD) - value = 'First private line\nSecond private line' if kind == 'secure_note' else OTHER_PASSWORD - item = vault.add_entry('example.com', '', value, 'Project credential', kind=kind, group='work') - assert item['kind'] == kind and item['group'] == 'work' - assert 'password' not in item + login = vault.add_entry("example.com", "alice", PASSWORD) + value = ( + "First private line\nSecond private line" + if kind == "secure_note" + else OTHER_PASSWORD + ) + item = vault.add_entry( + "example.com", "", value, "Project credential", kind=kind, group="work" + ) + assert item["kind"] == kind and item["group"] == "work" + assert "password" not in item assert value not in json.dumps(vault.list_entries()) assert value.encode() not in vault.vault_path.read_bytes() - assert [e['id'] for e in vault.candidates_for_url('https://example.com')] == [login['id']] + assert [e["id"] for e in vault.candidates_for_url("https://example.com")] == [ + login["id"] + ] reopened = CredentialVault(vault.directory) - assert reopened.reveal_entry(item['id']).reveal() == value + assert reopened.reveal_entry(item["id"]).reveal() == value assert reopened.list_entries() == vault.list_entries() with pytest.raises(VaultError): - reopened.reveal_entry(login['id']) + reopened.reveal_entry(login["id"]) def test_non_login_items_do_not_upsert_each_other(vault): - first = vault.add_entry('', '', PASSWORD, 'First key', kind='api_key') - second = vault.add_entry('', '', OTHER_PASSWORD, 'Second key', kind='api_key') - note = vault.add_entry('', '', 'Private note\nBody', 'First key', kind='secure_note') - assert len({first['id'], second['id'], note['id']}) == 3 - assert vault.reveal_entry(first['id']).reveal() == PASSWORD - assert vault.reveal_entry(second['id']).reveal() == OTHER_PASSWORD + first = vault.add_entry("", "", PASSWORD, "First key", kind="api_key") + second = vault.add_entry("", "", OTHER_PASSWORD, "Second key", kind="api_key") + note = vault.add_entry( + "", "", "Private note\nBody", "First key", kind="secure_note" + ) + assert len({first["id"], second["id"], note["id"]}) == 3 + assert vault.reveal_entry(first["id"]).reveal() == PASSWORD + assert vault.reveal_entry(second["id"]).reveal() == OTHER_PASSWORD def test_group_and_name_updates_retain_non_login_values(vault): - item = vault.add_entry('', '', 'Private\nMultiline\tNote', 'Original', kind='secure_note', group='personal') - changed = vault.update_entry(item['id'], kind='secure_note', group='work', label='Renamed', password='') - assert changed['group'] == 'work' and changed['label'] == 'Renamed' - assert vault.reveal_entry(item['id']).reveal() == 'Private\nMultiline\tNote' + item = vault.add_entry( + "", + "", + "Private\nMultiline\tNote", + "Original", + kind="secure_note", + group="personal", + ) + changed = vault.update_entry( + item["id"], kind="secure_note", group="work", label="Renamed", password="" + ) + assert changed["group"] == "work" and changed["label"] == "Renamed" + assert vault.reveal_entry(item["id"]).reveal() == "Private\nMultiline\tNote" with pytest.raises(VaultError): - vault.update_entry(item['id'], kind='login') + vault.update_entry(item["id"], kind="login") with pytest.raises(VaultError): - vault.update_entry(item['id'], label='') + vault.update_entry(item["id"], label="") with pytest.raises(VaultError): - vault.update_entry(item['id'], group='invalid') + vault.update_entry(item["id"], group="invalid") assert vault.list_entries() == [changed] -@pytest.mark.parametrize('metadata', [ - {'kind': 'unknown'}, {'kind': []}, {'group': 'unknown'}, {'group': []}, -]) +@pytest.mark.parametrize( + "metadata", + [ + {"kind": "unknown"}, + {"kind": []}, + {"group": "unknown"}, + {"group": []}, + ], +) def test_bad_credential_metadata_does_not_modify_vault(vault, metadata): - existing = vault.add_entry('example.com', 'alice', PASSWORD) + existing = vault.add_entry("example.com", "alice", PASSWORD) with pytest.raises(VaultError): - vault.add_entry('example.org', 'bob', OTHER_PASSWORD, **metadata) + vault.add_entry("example.org", "bob", OTHER_PASSWORD, **metadata) assert vault.list_entries() == [existing] def test_note_limits_and_login_controls_remain_distinct(vault): - note = vault.add_entry('', '', 'a' * vault_mod.MAX_NOTE_CHARS, 'Long note', kind='secure_note') + note = vault.add_entry( + "", "", "a" * vault_mod.MAX_NOTE_CHARS, "Long note", kind="secure_note" + ) with pytest.raises(VaultError): - vault.update_entry(note['id'], password='a' * (vault_mod.MAX_NOTE_CHARS + 1)) + vault.update_entry(note["id"], password="a" * (vault_mod.MAX_NOTE_CHARS + 1)) with pytest.raises(VaultError): - vault.add_entry('example.com', 'alice', 'not\na password') + vault.add_entry("example.com", "alice", "not\na password") with pytest.raises(VaultError): - vault.add_entry('', '', 'key\nnot-valid', 'Key', kind='api_key') + vault.add_entry("", "", "key\nnot-valid", "Key", kind="api_key") with pytest.raises(VaultError): - vault.add_entry('', '', PASSWORD, '', kind='api_key') + vault.add_entry("", "", PASSWORD, "", kind="api_key") diff --git a/tests/test_mini_browser_ws.py b/tests/test_mini_browser_ws.py index 014673f8..14a89e16 100644 --- a/tests/test_mini_browser_ws.py +++ b/tests/test_mini_browser_ws.py @@ -2462,51 +2462,77 @@ def test_cut_ids_are_scoped_to_the_viewer_connection(): assert mbws._cut_key(first, "copy-1") != mbws._cut_key(second, "copy-1") assert mbws._cut_key(first, "copy-1") == mbws._cut_key(first, "copy-1") + # On-demand reveal is a separate requester-only response, not resource state. def test_typed_items_and_explicit_reveal_stay_on_requesting_socket(env, tmp_path): from app.credentials import CredentialVault - env.vault = CredentialVault(tmp_path / 'typed-vault') + + env.vault = CredentialVault(tmp_path / "typed-vault") async def go(): ws, channel = connect(env) _, other = connect(env) - await env.mb.handle(ws, 'mini_browser_vault_add', { - 'kind': 'secure_note', 'group': 'work', 'site': '', 'username': '', - 'label': 'Private note', 'password': 'Private first line\nPrivate second line', - }) - await env.mb.handle(ws, 'mini_browser_vault_list', {}) - [reply] = channel.data('mini_browser_vault_list') - [entry] = reply['entries'] - assert entry['kind'] == 'secure_note' and entry['group'] == 'work' - assert 'Private first line' not in all_traffic(channel, other) - await env.mb.handle(ws, 'mini_browser_vault_reveal', {'id': entry['id'], 'token': 'request1'}) - assert channel.data('mini_browser_vault_value') == [{ - 'id': entry['id'], 'token': 'request1', - 'value': 'Private first line\nPrivate second line', - }] + await env.mb.handle( + ws, + "mini_browser_vault_add", + { + "kind": "secure_note", + "group": "work", + "site": "", + "username": "", + "label": "Private note", + "password": "Private first line\nPrivate second line", + }, + ) + await env.mb.handle(ws, "mini_browser_vault_list", {}) + [reply] = channel.data("mini_browser_vault_list") + [entry] = reply["entries"] + assert entry["kind"] == "secure_note" and entry["group"] == "work" + assert "Private first line" not in all_traffic(channel, other) + await env.mb.handle( + ws, "mini_browser_vault_reveal", {"id": entry["id"], "token": "request1"} + ) + assert channel.data("mini_browser_vault_value") == [ + { + "id": entry["id"], + "token": "request1", + "value": "Private first line\nPrivate second line", + } + ] assert other.sent == [] assert env.get_host_calls == 0 # managing credentials never starts Chromium - login = env.vault.add_entry('example.com', 'alice', PASSWORD) - await env.mb.handle(ws, 'mini_browser_vault_reveal', {'id': login['id'], 'token': 'request2'}) - [failure] = [r for r in channel.data('mini_browser_vault_result') if r['op'] == 'reveal'] - assert failure['ok'] is False + login = env.vault.add_entry("example.com", "alice", PASSWORD) + await env.mb.handle( + ws, "mini_browser_vault_reveal", {"id": login["id"], "token": "request2"} + ) + [failure] = [ + r for r in channel.data("mini_browser_vault_result") if r["op"] == "reveal" + ] + assert failure["ok"] is False assert PASSWORD not in all_traffic(channel, other) run(go) -def test_reveal_transport_failures_do_not_log_the_value(env, tmp_path, monkeypatch, captured_logs): +def test_reveal_transport_failures_do_not_log_the_value( + env, tmp_path, monkeypatch, captured_logs +): from app.credentials import CredentialVault - env.vault = CredentialVault(tmp_path / 'typed-vault') - entry = env.vault.add_entry('', '', PASSWORD, 'API key', kind='api_key') + + env.vault = CredentialVault(tmp_path / "typed-vault") + entry = env.vault.add_entry("", "", PASSWORD, "API key", kind="api_key") async def go(): ws, channel = connect(env) + def fail_send(*args): - raise RuntimeError('transport failed with ' + PASSWORD) - monkeypatch.setattr(env.mb._adapter, 'send_only_to', fail_send) - await env.mb.handle(ws, 'mini_browser_vault_reveal', {'id': entry['id'], 'token': 'request1'}) - assert PASSWORD not in ''.join(captured_logs) + raise RuntimeError("transport failed with " + PASSWORD) + + monkeypatch.setattr(env.mb._adapter, "send_only_to", fail_send) + await env.mb.handle( + ws, "mini_browser_vault_reveal", {"id": entry["id"], "token": "request1"} + ) + assert PASSWORD not in "".join(captured_logs) assert channel.sent == [] run(go) diff --git a/tests/test_process_ledger.py b/tests/test_process_ledger.py index dbcd2c7b..fdb34cf0 100644 --- a/tests/test_process_ledger.py +++ b/tests/test_process_ledger.py @@ -13,11 +13,13 @@ import subprocess import sys import time +from unittest.mock import Mock import psutil import pytest from app.process_ledger import ProcessLedger, ROLE_TUNNEL, listening_pids +from app import process_ledger # A child that listens on the port given as argv[1] and sleeps. _LISTENER = ( @@ -66,6 +68,51 @@ def _gone(proc: subprocess.Popen) -> bool: return False +@pytest.mark.parametrize("failure", ["exit", "missing", "timeout"]) +def test_windows_taskkill_failure_uses_process_api(monkeypatch, procs, failure): + ours, foreign = _spawn_sleeper(), _spawn_sleeper() + procs += [ours, foreign] + run = Mock(return_value=subprocess.CompletedProcess([], 1, b"", b"access denied")) + if failure == "missing": + run.side_effect = FileNotFoundError("taskkill") + elif failure == "timeout": + run.side_effect = subprocess.TimeoutExpired("taskkill", 15) + monkeypatch.setattr(process_ledger.subprocess, "run", run) + + process_ledger._kill_windows_tree(ours.pid, grace=1) + + assert _gone(ours) + assert foreign.poll() is None + + +def test_windows_failed_stop_without_psutil_is_reported(monkeypatch): + monkeypatch.setattr(process_ledger, "psutil", None) + monkeypatch.setattr( + process_ledger.subprocess, + "run", + Mock(return_value=subprocess.CompletedProcess([], 1, b"", b"access denied")), + ) + with pytest.raises(RuntimeError, match="Cannot stop owned pid"): + process_ledger._kill_windows_tree(123, grace=0) + + +def test_windows_failed_process_api_stop_is_reported(monkeypatch): + proc = Mock(pid=123) + proc.children.return_value = [] + api = Mock() + api.NoSuchProcess = psutil.NoSuchProcess + api.Process.return_value = proc + api.wait_procs.return_value = ([], [proc]) + monkeypatch.setattr(process_ledger, "psutil", api) + monkeypatch.setattr( + process_ledger.subprocess, + "run", + Mock(return_value=subprocess.CompletedProcess([], 1, b"", b"access denied")), + ) + with pytest.raises(RuntimeError, match="still running: 123"): + process_ledger._kill_windows_tree(123, grace=0) + + def test_listening_pids_matches_exact_port_only(procs): port = _free_port() proc = _spawn_listener(port) diff --git a/tests/test_question_dismissal.py b/tests/test_question_dismissal.py index 8a4bc62a..174ec37d 100644 --- a/tests/test_question_dismissal.py +++ b/tests/test_question_dismissal.py @@ -112,9 +112,7 @@ async def react(trigger): assert len(late_rows) == 1 for row in ([continuation] if continuation else []) + late_rows: stored = store.get(row.trigger_id) - assert (stored["status"], stored["resolution"]) == ( - "DONE", "cancelled" - ) + assert (stored["status"], stored["resolution"]) == ("DONE", "cancelled") assert finalized.await_count == (0 if mode == "idle" else 1) controller._event_bus.emit.assert_not_called() agent._handle_chat_message.assert_not_awaited() @@ -133,9 +131,11 @@ def test_dismissal_still_stops_when_recording_the_system_message_fails(): request_run_stop=AsyncMock(return_value=True), event_stream_manager=SimpleNamespace(log=Mock(side_effect=OSError("disk"))), ) - asyncio.run(_controller(agent).submit_question_answer( - "", "Continue?", "chat-a", dismissed=True - )) + asyncio.run( + _controller(agent).submit_question_answer( + "", "Continue?", "chat-a", dismissed=True + ) + ) agent.request_run_stop.assert_awaited_once_with("chat-a") diff --git a/tests/test_session_clear.py b/tests/test_session_clear.py index f22e6ebf..8e0a760d 100644 --- a/tests/test_session_clear.py +++ b/tests/test_session_clear.py @@ -250,8 +250,10 @@ async def react(trigger): store = TriggerStore(db_path=str(tmp_path / "clear-triggers.db")) service = TriggerService(store, runtime) agent = SimpleNamespace( - busy_sessions=set(), session_runtime=runtime, - _emit_run_state=Mock(), _notify_mini_browser=Mock(), + busy_sessions=set(), + session_runtime=runtime, + _emit_run_state=Mock(), + _notify_mini_browser=Mock(), session_manager=SimpleNamespace(clear_session=Mock(return_value=True)), _persist_llm_session=Mock(), ) @@ -259,9 +261,17 @@ async def react(trigger): runtime.set_stop_finalizer(AsyncMock()) await runtime.start() try: + async def emit(sid, source, future=True): - return await service.emit(TriggerSpec(session_id=sid, source=source, - description="Synthetic clear regression", fire_at=time.time() + 60 if future else None)) + return await service.emit( + TriggerSpec( + session_id=sid, + source=source, + description="Synthetic clear regression", + fire_at=time.time() + 60 if future else None, + ) + ) + if active: await emit("target", TriggerSource.USER_MESSAGE, future=False) await asyncio.wait_for(started.wait(), 2) @@ -314,7 +324,8 @@ def claim(trigger): finalized = AsyncMock() runtime.set_stop_finalizer(finalized) agent = SimpleNamespace( - session_runtime=runtime, _emit_run_state=Mock(), + session_runtime=runtime, + _emit_run_state=Mock(), _notify_mini_browser=Mock(), session_manager=SimpleNamespace(clear_session=Mock(return_value=True)), _persist_llm_session=Mock(), @@ -322,11 +333,15 @@ def claim(trigger): agent.request_run_stop = MethodType(AgentBase.request_run_stop, agent) await runtime.start() try: + async def emit(sid): - return await service.emit(TriggerSpec( - session_id=sid, source=TriggerSource.USER_MESSAGE, - description="Capacity regression", - )) + return await service.emit( + TriggerSpec( + session_id=sid, + source=TriggerSource.USER_MESSAGE, + description="Capacity regression", + ) + ) other = await emit("other") await asyncio.wait_for(running.wait(), 2) @@ -343,7 +358,7 @@ async def emit(sid): for _ in range(100): if store.get(fresh.trigger_id)["status"] == "DONE": break - await asyncio.sleep(.01) + await asyncio.sleep(0.01) assert store.get(fresh.trigger_id)["status"] == "DONE" assert executed == ["other", "target"] finally: @@ -355,7 +370,9 @@ async def emit(sid): @pytest.mark.parametrize("unwind", ["cancel", "return", "raise"]) @pytest.mark.parametrize("cancel_requester", [False, True]) -def test_concurrent_stops_join_settlement_before_resetting_context(tmp_path, monkeypatch, unwind, cancel_requester): +def test_concurrent_stops_join_settlement_before_resetting_context( + tmp_path, monkeypatch, unwind, cancel_requester +): from app.agent_base import AgentBase from app.triggers import TriggerService, TriggerSource, TriggerSpec from app.triggers.runtime import SessionRuntimeManager @@ -398,7 +415,8 @@ async def react(trigger): finalized = AsyncMock() runtime.set_stop_finalizer(finalized) agent = SimpleNamespace( - session_runtime=runtime, _emit_run_state=Mock(), + session_runtime=runtime, + _emit_run_state=Mock(), _notify_mini_browser=Mock(), session_manager=SimpleNamespace(clear_session=Mock(return_value=True)), _persist_llm_session=Mock(), @@ -407,17 +425,20 @@ async def react(trigger): await runtime.start() tasks = [] try: - row = await service.emit(TriggerSpec( - session_id="target", source=TriggerSource.USER_MESSAGE, - description="Finish while process shutdown is pending", - )) + row = await service.emit( + TriggerSpec( + session_id="target", + source=TriggerSource.USER_MESSAGE, + description="Finish while process shutdown is pending", + ) + ) await asyncio.wait_for(running.wait(), 2) tasks.append(asyncio.create_task(AgentBase.clear_session(agent, "target"))) await asyncio.wait_for(kill_started.wait(), 2) release_turn.set() # Cancellation must be requested before waiting for OS shutdown; # otherwise this turn finishes normally and loses the stop signal. - await asyncio.wait_for(cancelled.wait(), .5) + await asyncio.wait_for(cancelled.wait(), 0.5) tasks.append(asyncio.create_task(runtime.request_stop("target"))) await asyncio.sleep(0) assert not any(task.done() for task in tasks) @@ -429,7 +450,9 @@ async def react(trigger): finalized.assert_not_awaited() agent.session_manager.clear_session.assert_not_called() release_kill.set() - results = await asyncio.wait_for(asyncio.gather(*tasks, return_exceptions=True), 2) + results = await asyncio.wait_for( + asyncio.gather(*tasks, return_exceptions=True), 2 + ) assert results[1] is True if cancel_requester: assert isinstance(results[0], asyncio.CancelledError) @@ -479,17 +502,23 @@ async def react(trigger): finalized = AsyncMock() runtime.set_stop_finalizer(finalized) agent = SimpleNamespace( - session_runtime=runtime, _emit_run_state=Mock(), + session_runtime=runtime, + _emit_run_state=Mock(), _notify_mini_browser=Mock(), session_manager=SimpleNamespace(clear_session=Mock(return_value=True)), _persist_llm_session=Mock(), ) agent.request_run_stop = MethodType(AgentBase.request_run_stop, agent) - monkeypatch.setattr("app.triggers.runtime.STOP_SETTLE_TIMEOUT_S", .02) + monkeypatch.setattr("app.triggers.runtime.STOP_SETTLE_TIMEOUT_S", 0.02) await runtime.start() try: - row = await service.emit(TriggerSpec(session_id="target", - source=TriggerSource.USER_MESSAGE, description="Slow tool cleanup")) + row = await service.emit( + TriggerSpec( + session_id="target", + source=TriggerSource.USER_MESSAGE, + description="Slow tool cleanup", + ) + ) await asyncio.wait_for(started.wait(), 2) with pytest.raises(asyncio.TimeoutError): await AgentBase.clear_session(agent, "target") @@ -522,8 +551,16 @@ def test_clear_command_orders_storage_and_preserves_other_chat(tmp_path, monkeyp from app.usage.chat_storage import ChatStorage, StoredChatMessage storage = ChatStorage(db_path=str(tmp_path / "chat.db")) - storage.insert_message(StoredChatMessage(message_id="other", session_id="other", - sender="agent", style="agent", content="Keep me", timestamp=1)) + storage.insert_message( + StoredChatMessage( + message_id="other", + session_id="other", + sender="agent", + style="agent", + content="Keep me", + timestamp=1, + ) + ) insert_started = threading.Event() release_insert = threading.Event() original_insert = storage.insert_message @@ -537,7 +574,9 @@ def insert(message): monkeypatch.setattr("app.usage.get_chat_storage", lambda: storage) actions = Mock() monkeypatch.setattr("app.usage.get_action_storage", lambda: actions) - adapter = SimpleNamespace(_broadcast=AsyncMock(), broadcast_session_cleared=AsyncMock()) + adapter = SimpleNamespace( + _broadcast=AsyncMock(), broadcast_session_cleared=AsyncMock() + ) chat = BrowserChatComponent.__new__(BrowserChatComponent) chat._storage = storage chat._db = ThreadPoolExecutor(max_workers=1) @@ -547,7 +586,8 @@ def insert(message): adapter.chat_component = chat controller = SimpleNamespace( agent=SimpleNamespace(clear_session=AsyncMock(return_value=True)), - active_adapter=adapter, event_bus=EventBus(), + active_adapter=adapter, + event_bus=EventBus(), ) command = ClearCommand(controller) command.emit_message = Mock() @@ -563,22 +603,47 @@ async def delayed_message(label, message, style, session_id=None, **kwargs): await release_other.wait() return await release_ui.wait() - await chat.append_message(ChatMessage(sender="agent", style="agent", - content="Delayed old event", message_id="delayed", timestamp=3, - session_id="target")) + await chat.append_message( + ChatMessage( + sender="agent", + style="agent", + content="Delayed old event", + message_id="delayed", + timestamp=3, + session_id="target", + ) + ) # Use the actual synchronous adapter handler, which schedules the # asynchronous display task returned to the bus for ownership. display_adapter = SimpleNamespace(_display_chat_message=delayed_message) - controller.event_bus.subscribe(UIEventType.SYSTEM_MESSAGE, - lambda event: InterfaceAdapter._handle_system_message(display_adapter, event)) + controller.event_bus.subscribe( + UIEventType.SYSTEM_MESSAGE, + lambda event: InterfaceAdapter._handle_system_message( + display_adapter, event + ), + ) for sid in ("target", "other"): - controller.event_bus.emit(UIEvent(type=UIEventType.SYSTEM_MESSAGE, - task_id=sid, data={"message": "Old event"})) + controller.event_bus.emit( + UIEvent( + type=UIEventType.SYSTEM_MESSAGE, + task_id=sid, + data={"message": "Old event"}, + ) + ) await asyncio.wait_for(other_pending.wait(), 2) - append = asyncio.create_task(chat.append_message(ChatMessage( - sender="agent", style="agent", content="Old work", message_id="old", - timestamp=2, session_id="target"))) + append = asyncio.create_task( + chat.append_message( + ChatMessage( + sender="agent", + style="agent", + content="Old work", + message_id="old", + timestamp=2, + session_id="target", + ) + ) + ) try: assert await asyncio.to_thread(insert_started.wait, 2) clear = asyncio.create_task(command.execute([], session_id="target")) @@ -593,7 +658,8 @@ async def delayed_message(label, message, style, session_id=None, **kwargs): actions.clear_items.assert_called_once_with("target") adapter.broadcast_session_cleared.assert_awaited_once_with("target") command.emit_message.assert_called_once_with( - "Conversation cleared.", "system", session_id="target") + "Conversation cleared.", "system", session_id="target" + ) controller.agent.clear_session.return_value = False actions.reset_mock() adapter.broadcast_session_cleared.reset_mock() @@ -647,20 +713,34 @@ def claim(trigger): service.claim = claim runtime.set_stop_finalizer(finalize) - agent = SimpleNamespace(session_runtime=runtime, _emit_run_state=Mock(), - _notify_mini_browser=Mock(), _persist_llm_session=Mock(), - session_manager=SimpleNamespace(clear_session=reset)) + agent = SimpleNamespace( + session_runtime=runtime, + _emit_run_state=Mock(), + _notify_mini_browser=Mock(), + _persist_llm_session=Mock(), + session_manager=SimpleNamespace(clear_session=reset), + ) agent.request_run_stop = MethodType(AgentBase.request_run_stop, agent) await runtime.start() clear = None try: - await service.emit(TriggerSpec(session_id="target", - source=TriggerSource.RUN_CONTINUATION, description="Parked run", - fire_at=time.time() + 60)) + await service.emit( + TriggerSpec( + session_id="target", + source=TriggerSource.RUN_CONTINUATION, + description="Parked run", + fire_at=time.time() + 60, + ) + ) clear = asyncio.create_task(AgentBase.clear_session(agent, "target")) await asyncio.wait_for(finalizing.wait(), 2) - await service.emit(TriggerSpec(session_id="target", - source=TriggerSource.USER_MESSAGE, description="Fresh request")) + await service.emit( + TriggerSpec( + session_id="target", + source=TriggerSource.USER_MESSAGE, + description="Fresh request", + ) + ) await asyncio.wait_for(claimed.wait(), 2) await asyncio.sleep(0) reset.assert_not_called() diff --git a/tests/test_settings_transactions.py b/tests/test_settings_transactions.py index e60c3c02..6c17101f 100644 --- a/tests/test_settings_transactions.py +++ b/tests/test_settings_transactions.py @@ -385,5 +385,7 @@ async def run(): saved = json.loads(settings_path.read_text())["model"] # BytePlus is the only provider with a video model, so the video save # rolls back to it as well. - assert saved[f"{kind}_gen_provider"] == ("openai" if kind == "image" else "byteplus") + assert saved[f"{kind}_gen_provider"] == ( + "openai" if kind == "image" else "byteplus" + ) assert saved["llm_provider"] == adapter._controller.agent.provider == "openai"