diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e2f87694..09bec8e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,6 +102,17 @@ jobs: - name: Test run: cargo test --all + - name: Test actual GodPrompt 1.0.3 to 1.1.1 hot reload and rollback + env: + CODEXIFY_CANARY_OLD_GODPROMPT: ${{ runner.temp }}/godprompt-old/node_modules/god-prompt-mcp/dist/stdio.js + CODEXIFY_CANARY_NEW_GODPROMPT: ${{ runner.temp }}/godprompt-new/node_modules/god-prompt-mcp/dist/stdio.js + run: | + npm install --prefix "$RUNNER_TEMP/godprompt-old" --no-save --no-package-lock --ignore-scripts --no-audit --no-fund god-prompt-mcp@1.0.3 + npm install --prefix "$RUNNER_TEMP/godprompt-new" --no-save --no-package-lock --ignore-scripts --no-audit --no-fund god-prompt-mcp@1.1.1 + test -f "$CODEXIFY_CANARY_OLD_GODPROMPT" + test -f "$CODEXIFY_CANARY_NEW_GODPROMPT" + cargo test --locked --lib service_local_godprompt_candidate_upgrade_and_failed_rollback -- --nocapture + - name: Test Rust and TypeScript browser companion contract env: CODEXIFY_TEST_BROWSER_COMPANION_BUN: ${{ steps.browser-bun.outputs.bun-path }} diff --git a/.github/workflows/preview-hot-reload-windows.yml b/.github/workflows/preview-hot-reload-windows.yml new file mode 100644 index 00000000..16ca2cfa --- /dev/null +++ b/.github/workflows/preview-hot-reload-windows.yml @@ -0,0 +1,48 @@ +name: Build experimental Codexify hot reload for Windows + +on: + push: + branches: + - feat/per-mcp-hot-reload-20261119 + paths: + - .github/workflows/preview-hot-reload-windows.yml + - src/markdown_chat/mod.rs + - src/agent_automation/install.rs + +permissions: + contents: read + +jobs: + preview-windows: + name: Package pinned Windows feature binary + runs-on: windows-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: Build from exact branch commit + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + cargo build --release --locked --bin codexify + if ($LASTEXITCODE -ne 0) { throw 'Cargo build failed' } + $out = Join-Path $env:RUNNER_TEMP 'codexify-hot-reload-preview' + New-Item -ItemType Directory -Force -Path $out | Out-Null + $binary = Join-Path (Get-Location) 'target\release\codexify.exe' + if (!(Test-Path -LiteralPath $binary -PathType Leaf)) { throw 'Expected executable missing' } + Copy-Item -LiteralPath $binary -Destination (Join-Path $out 'codexify.exe') + & (Join-Path $out 'codexify.exe') --help | Out-Null + if ($LASTEXITCODE -ne 0) { throw 'Preview binary self-probe failed' } + $hash = (Get-FileHash -LiteralPath (Join-Path $out 'codexify.exe') -Algorithm SHA256).Hash.ToLowerInvariant() + Set-Content -LiteralPath (Join-Path $out 'SHA256SUMS.txt') -Value ("$hash codexify.exe") -Encoding ascii + Set-Content -LiteralPath (Join-Path $out 'SOURCE_COMMIT.txt') -Value "${{ github.sha }}" -Encoding ascii + Set-Content -LiteralPath (Join-Path $out 'PREVIEW_NOTICE.txt') -Value 'EXPERIMENTAL binary from feature branch PR #11. Not an official release. Activate only after backing up the existing executable; coordinate with other Codexify workers.' -Encoding ascii + - uses: actions/upload-artifact@v4 + with: + name: codexify-pr11-windows-preview + path: ${{ runner.temp }}/codexify-hot-reload-preview/* + if-no-files-found: error + retention-days: 7 diff --git a/docs/MCP_HOT_RELOAD.md b/docs/MCP_HOT_RELOAD.md new file mode 100644 index 00000000..75b2476d --- /dev/null +++ b/docs/MCP_HOT_RELOAD.md @@ -0,0 +1,123 @@ +# Per-source MCP hot reload (operator guide) + +**Status:** experimental, opt-in through a staged build. This document describes +source behavior, not a deployed feature. Do not replace an active Codexify +service without separate approval and validation. + +## Scope + +`reload_mcp` reloads exactly one existing upstream source. Its `source` +argument must match the **exact configured MCP server key** from the resolved +`mcpServers` configuration (including inherited Codex MCP sources). For example, +if the configured key is `god-prompt-mcp`, call +`{"source":"god-prompt-mcp"}`. Do not substitute a normalized catalog source ID +such as `god_prompt_mcp` from `mcp_list_sources`: it is not necessarily the +configuration key and can return `SOURCE_UNKNOWN` without changing the running +connection. The call cannot supply a command, executable, working directory, +URL, environment variables, or credentials. + +The server rereads its operator configuration (including inherited Codex MCP +sources) **read-only** using its startup CLI selection. A target that was +removed or disabled fails closed. No global service restart, tunnel reconnect, +task replacement, or unrelated source reconnection is part of this operation. + +The replacement uses the existing MCP transport implementation for **stdio** +and **Streamable HTTP**. Unsupported legacy SSE and WebSocket sources remain +unsupported; the operation never silently falls back to a process-wide restart. + +## Lifecycle and safety + +1. The scoped reload lock rejects concurrent attempts for the same source + (`RELOAD_IN_PROGRESS`); unrelated source reloads can progress concurrently. +2. The updated source specification is validated, started, initialized, and + queried with `tools/list`. A bounded MCP ping must also receive a + successful response or a protocol-level MCP error indicating a live peer. +3. The new tool definitions are validated and an immutable registry view is + constructed. The new view replaces the old view in one atomic pointer swap. + Concurrent distinct-source reloads rebuild against the latest view. +4. Each dispatch retains only its selected validated upstream tool and that + source's connection generation. Write-capable calls are **never replayed**; + unrelated direct/gateway sources are not retained by a long-running call. + Cancellation and timeout semantics remain those of the forwarded request. +5. The old generation is released when no selected old tool or catalog + snapshot retains it. The operation reports whether forwarded calls drained + within the source's configured `requestLimits.slowCancellationMs` interval. + The separate `oldGenerationStillReferenced` flag covers retained references + even when no forwarded calls remain. A drain timeout does **not** kill or + replay an in-progress side-effecting call. +6. On failure **before** the swap, the new candidate is dropped and the old + view is never replaced. Errors use bounded diagnostic codes and do not + echo credentials, command lines, headers, environment variables, or raw + tool-response bodies. + +For stdio, the spawned Tokio child has `kill_on_drop` enabled, so aborted +initializations should not leave the candidate running. A platform-specific +Windows child-lifecycle integration and repeated process-tree soak remain +required acceptance gates before production activation. + +## Result and tool discovery + +A successful result includes `source`, `outcome`, +`previousGeneration`, `replacementGeneration`, old/new reported MCP +server versions, readiness, counts of added/removed/changed/unchanged +tool definitions, `oldCallsDrained`, `oldCallsStillRunning`, +`oldGenerationStillReferenced`, `oldGenerationRetiring`, `rollback`, +`diagnosticCode`, and measured +`timingsMs` for initialization/readiness, registry switch, draining and total +reload duration. + +The source's discovery data and forwarded-call routes change together in the +server's tool registry. Existing transport sessions can observe changed +definitions on **another `tools/list`** request. **An already-open ChatGPT +chat is not guaranteed to receive new tool definitions automatically.** +Its connector may need Refresh, reconnection or a new conversation before +new direct tools appear. The result explicitly sets +`clientRediscoveryRequired: true`. + +Catalog-mode source and tool identifiers remain generated by the existing +catalog and may change when a source's tool set changes. Re-query +`mcp_list_sources`, `mcp_search_tools`, and `mcp_get_tool` before calling +a new or modified tool. Gateway clients must refresh their advertised schema +and generated skill after reload. + +## Failure and boundary conditions + +- `SOURCE_UNKNOWN` / `SOURCE_REMOVED` / `SOURCE_DISABLED`: verify the + exact configured MCP key, not a normalized catalog ID. Do not spawn an + arbitrary process or reinterpret the input as a path. +- `RELOAD_IN_PROGRESS`: no duplicate replacement of the same source. +- `CONFIG_INVALID`, `CONFIG_CLI_INVALID`, `INITIALIZE_FAILED`, + `READINESS_FAILED`: old healthy generation is retained. +- `TOOL_NAME_LIMIT`: bound on unique registered direct/gateway tool names; + the attempted registry swap is discarded. +- `CANCELLED`: an aborted preparation does not switch generations. +- If the old generation became unhealthy independently, a successful new + connection is preferred; the operation does not imply that the old peer can + be resurrected. +- Resource links issued before a reload are separate capabilities. An upstream + resource fetched **after** its old connection retires can fail; request a + fresh resource link from the new source. Already-running resource reads + retain their own peer and cancellation semantics, but cross-generation + resource-link durability is not claimed by this implementation. +- Invalid data from upstream tool descriptions must remain untrusted. + Reload permission uses existing Codexify connector authorization and audit + boundaries. Do not expose the control plane to unauthenticated clients. + +## Verification gate + +The following must pass on the **exact candidate commit** before any +deployment: Rust formatting and compilation, strict lint, full Rust suite, +downstream HTTP MCP handshake, selected-source version and tool-diff checks, +two independent unrelated MCP sources, read/write in-flight calls, duplicate +reload and failure injection, cleanup/leak soak, and Windows native-process +termination. Reproduce an isolated old-to-new GodPrompt candidate switch +without touching the running GodPrompt instance; verify new version and +rollback. Measure switchover, drain and end-to-end request latency using a +repeatable workload. Do not report "zero downtime" without observed proof. + +**Not included:** automatic filesystem watching. It remains disabled rather +than introducing unverified debouncing and partially written config races. + +This feature does not merge the separate Windows scheduled-supervisor orphan +cleanup branch. Keep that work independent and verify its eventual integration +before any production cutover. diff --git a/scripts/fixtures/hot-reload-stdio-mcp.mjs b/scripts/fixtures/hot-reload-stdio-mcp.mjs new file mode 100644 index 00000000..48a41325 --- /dev/null +++ b/scripts/fixtures/hot-reload-stdio-mcp.mjs @@ -0,0 +1,49 @@ +// No dependencies or network: isolated MCP stdio child for Rust hot-reload tests. +// Every invocation is a separate operating-system process. +import { createInterface } from 'node:readline'; +import { writeFileSync } from 'node:fs'; + +const version = process.argv[2] ?? 'old'; +const pid = process.pid; +const input = createInterface({ input: process.stdin, crlfDelay: Infinity }); + +function respond(id, result) { + process.stdout.write(JSON.stringify({ jsonrpc: '2.0', id, result }) + '\n'); +} +function tool(name, description) { + return { name, description, inputSchema: { type: 'object', properties: {}, additionalProperties: false } }; +} +function replyToTool(id) { + respond(id, { content: [{ type: 'text', text: version + ':' + pid }], isError: false }); +} + +for await (const line of input) { + let request; + try { request = JSON.parse(line); } catch { continue; } + if (!request || !Object.hasOwn(request, 'id')) continue; + const { id, method } = request; + if (method === 'initialize') { + respond(id, { + protocolVersion: request.params?.protocolVersion ?? '2025-03-26', + capabilities: { tools: {} }, + serverInfo: { name: 'isolated-hot-reload-stdio-fixture', version }, + }); + } else if (method === 'tools/list') { + const tools = [tool('probe', 'Version probe ' + version), tool('slow', 'Slow probe ' + version)]; + if (version === 'new') tools.push(tool('new_tool', 'New tool in replacement generation')); + respond(id, { tools }); + } else if (method === 'tools/call') { + if (request.params?.name === 'slow') { + if (process.env.HOT_RELOAD_SLOW_MARKER_FILE) { + writeFileSync(process.env.HOT_RELOAD_SLOW_MARKER_FILE, String(pid)); + } + setTimeout(() => replyToTool(id), 300); + } else { + replyToTool(id); + } + } else if (method === 'ping') { + respond(id, {}); + } else { + process.stdout.write(JSON.stringify({ jsonrpc: '2.0', id, error: { code: -32601, message: 'Method not found' } }) + '\n'); + } +} diff --git a/src/agent_automation/install.rs b/src/agent_automation/install.rs index 05c9a0a3..87357e53 100644 --- a/src/agent_automation/install.rs +++ b/src/agent_automation/install.rs @@ -326,6 +326,9 @@ impl BrowserCompanionManager { .stdin(Stdio::null()) .stdout(Stdio::from(log)) .stderr(Stdio::from(stderr)); + if let Some(path) = &self.config.provider_send_gate_path { + command.env("CODEXIFY_BROWSER_SEND_GATE_PATH", path); + } configure_detached_process(&mut command); let mut child = command .spawn() diff --git a/src/bridge.rs b/src/bridge.rs index 68bdcef4..8cb16d98 100644 --- a/src/bridge.rs +++ b/src/bridge.rs @@ -35,26 +35,25 @@ use crate::process_env::scrub_untrusted_child_env; use crate::tool::{Tool, ToolBehavior, ToolCallIdentity, ToolRequestContext}; use crate::types::{AppConfig, McpToolExposure, ToolContent, ToolResult}; +mod hot_reload; +#[cfg(test)] +mod hot_reload_tests; mod stderr; +pub(crate) use hot_reload::{Generation, HotReloadRuntime, ReloadMcpTool}; /// How long to wait for an upstream to start up and answer `tools/list` before /// giving up on it. const CONNECT_TIMEOUT: Duration = Duration::from_secs(20); -/// The result of connecting to every configured upstream: the tools to merge -/// into the registry, plus the running services kept alive for the server's -/// lifetime (dropping one closes its transport and any child process). +/// Connected MCP sources are owned by a transactional registry. Its immutable +/// view is shared across transport sessions, while calls pin their generation. pub struct Bridge { - pub tools: Vec>, - /// Shared opaque-capability store for resource links returned by upstream tools. + pub tools: Arc>>, pub resources: Arc, - /// Held only to keep upstream transports alive; never read directly. - pub services: Vec>, - /// One human-readable line per configured server (connected or failed), - /// printed in the startup banner so a bad path or handshake is never silent. pub report: Vec, - pub(crate) upstreams: Arc>>, - pub(crate) monitors: Vec, + pub(crate) runtime: Arc, + pub(crate) upstreams: + Arc>>>, } pub(crate) struct UpstreamMonitor(tokio::task::JoinHandle<()>); @@ -120,10 +119,12 @@ struct BridgedTool { tool_timeout: Option, control: Arc, resources: Arc, + generation: Arc, } impl BridgedTool { async fn run(&self, args: Value, cancellation: Option<&CancellationToken>) -> ToolResult { + let _lease = self.generation.begin_call(); let mut params = CallToolRequestParams::new(self.original_name.clone()); if let Some(obj) = args.as_object() && !obj.is_empty() @@ -518,157 +519,90 @@ fn unique_name(base: String, used: &std::collections::HashSet) -> String /// materialize its effective exposure mode. A server that fails to launch or /// answer is logged and skipped — it never blocks startup. pub async fn connect_upstreams(config: &AppConfig) -> Bridge { - let mut tools: Vec> = Vec::new(); - let mut services: Vec> = Vec::new(); - let mut upstreams = HashMap::new(); - let mut monitors = Vec::new(); - let mut report: Vec = Vec::new(); - let mut catalog_sources = Vec::new(); let resources = Arc::new(BridgedResourceStore::new(config.artifact_egress.clone())); - - // Deterministic order so logs and tool ordering are stable across runs. - let mut names: Vec<&String> = config.mcp_servers.keys().collect(); + let mut sources = HashMap::new(); + let mut report = Vec::new(); + let mut names: Vec<_> = config.mcp_servers.keys().collect(); names.sort(); - for server_name in names { - let spec = &config.mcp_servers[server_name]; + for name in names { + let spec = &config.mcp_servers[name]; if spec.disabled { - report.push(format!("{server_name} -> disabled")); + report.push(format!("{name} -> disabled")); continue; } - let sanitized = sanitize(server_name); - let exposure = spec.exposure(); - - match connect_one(server_name, spec, config).await { - Ok((service, upstream_tools, tool_timeout)) => { - let peer = service.peer().clone(); - let control = Arc::new(crate::upstream_control::UpstreamControl::new( - spec.request_limits.clone(), - )); - upstreams.insert(server_name.clone(), control.clone()); - monitors.push(monitor_upstream(peer.clone(), control.clone())); - let count = upstream_tools.len(); - - match exposure { - McpToolExposure::Gateway => { - let functions: Vec = upstream_tools - .iter() - .map(|t| GatewayFunction { - name: t.name.to_string(), - description: t - .description - .as_ref() - .map(|c| c.to_string()) - .unwrap_or_default(), - input_schema: Value::Object((*t.input_schema).clone()), - }) - .collect(); - - write_gateway_skill(config, server_name, &sanitized, &functions); - - let leaked: &'static str = Box::leak(sanitized.clone().into_boxed_str()); - tools.push(Box::new(GatewayTool { - name: leaked, - server: server_name.clone(), - description: gateway_description(server_name, &functions), - function_names: functions.iter().map(|f| f.name.clone()).collect(), - peer: peer.clone(), - tool_timeout, - control: control.clone(), - resources: resources.clone(), - })); - report.push(format!( - "{server_name} -> gateway ({count} functions via `{sanitized}`)" - )); - tracing::info!( - "bridged MCP server '{server_name}' as gateway: {count} function(s)" - ); - } - McpToolExposure::Direct => { - report.push(format!("{server_name} -> direct ({count} tool(s))")); - let mut used: std::collections::HashSet = - std::collections::HashSet::new(); - for tool in upstream_tools { - let original_name = tool.name.to_string(); - let display = - unique_name(bridged_name(&sanitized, &original_name), &used); - used.insert(display.clone()); - let leaked: &'static str = Box::leak(display.into_boxed_str()); - let title = tool.title.filter(|title| !title.trim().is_empty()); - let description = tool - .description - .map(|description| description.to_string()) - .filter(|description| !description.trim().is_empty()) - .unwrap_or_else(|| { - format!( - "Call the `{original_name}` tool on the `{server_name}` MCP server." - ) - }); - tools.push(Box::new(BridgedTool { - name: leaked, - original_name, - server: server_name.clone(), - title, - description, - input_schema: Value::Object((*tool.input_schema).clone()), - output_schema: tool - .output_schema - .map(|schema| Value::Object((*schema).clone())), - annotations: tool.annotations, - icons: tool.icons, - meta: tool.meta, - peer: peer.clone(), - tool_timeout, - control: control.clone(), - resources: resources.clone(), - })); - } - tracing::info!( - "bridged MCP server '{server_name}' directly: {count} tool(s)" - ); - } - McpToolExposure::Catalog => { - report.push(format!( - "{server_name} -> catalog ({count} private tool(s))" - )); - catalog_sources.push(CatalogSourceInput { - raw_name: server_name.clone(), - provenance: spec.provenance, - transport: transport_label(spec), - peer_info: peer.peer_info(), - tools: upstream_tools, - peer: peer.clone(), - tool_timeout, - control: control.clone(), - }); - tracing::info!( - "indexed MCP server '{server_name}' privately: {count} tool(s)" - ); - } + match connect_one(name, spec, config).await { + Ok((service, tools, timeout)) => { + let count = tools.len(); + let mode = spec.exposure(); + if mode == McpToolExposure::Gateway { + let functions = gateway_functions(&tools); + write_gateway_skill(config, name, &sanitize(name), &functions); } - services.push(service); + let generation = + hot_reload::Generation::new(spec.clone(), service, tools, timeout, 1); + sources.insert(name.clone(), generation); + let visibility = if mode == McpToolExposure::Catalog { + " private" + } else { + "" + }; + report.push(format!( + "{name} -> {} ({count}{visibility} tool(s))", + mode.as_str() + )); + tracing::info!(source = %name, %count, exposure = mode.as_str(), "MCP source connected"); } - Err(e) => { - report.push(format!("{server_name} -> FAILED: {e}")); - tracing::warn!("skipping MCP server '{server_name}': {e}"); + Err(_) => { + // Upstream configuration can contain private header or command + // values; do not echo a failed spawn/handshake error to clients. + // An unsupported transport type is a safe, useful static + // diagnostic and should not be hidden as an initialization + // failure; neither diagnostic includes the configured URL. + let legacy_transport = spec.transport.as_deref().is_some_and(|kind| { + matches!( + kind.to_ascii_lowercase().as_str(), + "sse" | "websocket" | "ws" + ) + }); + if legacy_transport { + report.push(format!("{name} -> not supported (transport)")); + tracing::warn!(source = %name, "MCP source transport is not supported"); + } else { + report.push(format!("{name} -> FAILED (initialization)")); + tracing::warn!(source = %name, "MCP source failed initialization"); + } } } } - let mut catalog_tools = build_catalog_tools(catalog_sources, resources.clone()); - catalog_tools.append(&mut tools); - tools = catalog_tools; - + let runtime = HotReloadRuntime::new(Arc::new(config.clone()), resources.clone(), sources); Bridge { - tools, + tools: runtime.snapshot().tools.clone(), resources, - services, report, - upstreams: Arc::new(upstreams), - monitors, + upstreams: runtime.upstreams.clone(), + runtime, } } +/// Use one conversion for startup registration, registry replacement and +/// generated gateway skills, so schema/description metadata cannot diverge. +fn gateway_functions(tools: &[rmcp::model::Tool]) -> Vec { + tools + .iter() + .map(|tool| GatewayFunction { + name: tool.name.to_string(), + description: tool + .description + .as_ref() + .map(ToString::to_string) + .unwrap_or_default(), + input_schema: Value::Object((*tool.input_schema).clone()), + }) + .collect() +} + /// One function exposed by a gateway-mode server. struct GatewayFunction { name: String, @@ -767,10 +701,12 @@ struct GatewayTool { tool_timeout: Option, control: Arc, resources: Arc, + generation: Arc, } impl GatewayTool { async fn run(&self, args: Value, cancellation: Option<&CancellationToken>) -> ToolResult { + let _lease = self.generation.begin_call(); let Some(function) = args.get("function").and_then(|v| v.as_str()) else { return ToolResult::error("`function` is required (the name of the function to call)"); }; @@ -1112,7 +1048,7 @@ async fn connect_one_with_env( server_name: &str, spec: &crate::types::McpServerSpec, config: &AppConfig, - env_lookup: &dyn Fn(&str) -> Option, + env_lookup: &(dyn Fn(&str) -> Option + Sync), ) -> Result< ( RunningService, @@ -1133,6 +1069,7 @@ async fn connect_one_with_env( let connect = match select_transport(spec)? { UpstreamTransport::Stdio(command_path) => { let mut command = Command::new(command_path); + command.kill_on_drop(true); command.args(&spec.args); for (key, value) in &spec.env { command.env(key, value); @@ -2852,7 +2789,7 @@ mod tests { bridge.tools.is_empty(), "a failing upstream must be skipped, not fatal" ); - assert!(bridge.services.is_empty()); + assert!(bridge.runtime.snapshot().sources.is_empty()); // The failure is surfaced in the report, not swallowed. assert!(bridge.report.iter().any(|l| l.contains("bad -> FAILED"))); } diff --git a/src/bridge/hot_reload.rs b/src/bridge/hot_reload.rs new file mode 100644 index 00000000..a46c0a2f --- /dev/null +++ b/src/bridge/hot_reload.rs @@ -0,0 +1,568 @@ +//! Transactional replacement of one upstream MCP connection. +//! +//! Tool definitions are rebuilt into an immutable registry snapshot. Dispatch +//! clones only its selected tool into the call, preserving that source's old +//! generation without retaining unrelated MCP connections. +use super::*; +use clap::Parser; +use std::collections::{HashMap, HashSet}; +use std::sync::{Arc, Mutex, RwLock}; +use tokio::sync::{Mutex as AsyncMutex, Notify}; + +const MAX_INTERNED_TOOL_NAMES: usize = 4096; +const REPLACEMENT_PING_TIMEOUT: Duration = Duration::from_secs(3); + +pub(crate) struct Generation { + pub(super) spec: crate::types::McpServerSpec, + pub(super) peer: Peer, + pub(super) tools: Vec, + pub(super) timeout: Option, + pub(super) control: Arc, + pub(super) number: u64, + // Keep the transport alive until every snapshot/call using it is dropped. + _service: Mutex>, + _monitor: UpstreamMonitor, + calls: Mutex, + drained: Notify, +} + +pub(crate) struct CallLease<'a>(&'a Generation); + +impl Drop for CallLease<'_> { + fn drop(&mut self) { + let mut calls = self.0.calls.lock().unwrap(); + *calls -= 1; + if *calls == 0 { + self.0.drained.notify_one(); + } + } +} + +impl Generation { + pub(super) fn new( + spec: crate::types::McpServerSpec, + service: RunningService, + tools: Vec, + timeout: Option, + number: u64, + ) -> Arc { + let peer = service.peer().clone(); + let control = Arc::new(crate::upstream_control::UpstreamControl::new( + spec.request_limits.clone(), + )); + let monitor = monitor_upstream(peer.clone(), control.clone()); + Arc::new(Self { + spec, + peer, + tools, + timeout, + control, + number, + _service: Mutex::new(service), + _monitor: monitor, + calls: Mutex::new(0), + drained: Notify::new(), + }) + } + + pub(crate) fn begin_call(&self) -> CallLease<'_> { + *self.calls.lock().unwrap() += 1; + CallLease(self) + } + + async fn wait_for_calls(&self, limit: Duration) -> bool { + tokio::time::timeout(limit, async { + loop { + // Register the waiter before checking the counter. Otherwise + // the last call can finish between the check and registration, + // losing its wakeup and spuriously timing out the drain. + let notified = self.drained.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if *self.calls.lock().unwrap() == 0 { + break; + } + notified.await; + } + }) + .await + .is_ok() + } + + fn version(&self) -> Option { + self.peer.peer_info().and_then(|info| { + info.server_info + .as_ref() + .map(|info| info.version.to_string()) + }) + } +} + +struct NamePool(HashMap); + +impl NamePool { + fn intern(&mut self, name: String) -> Result<&'static str, &'static str> { + if let Some(value) = self.0.get(&name) { + return Ok(*value); + } + if self.0.len() >= MAX_INTERNED_TOOL_NAMES { + return Err("TOOL_NAME_LIMIT"); + } + let leaked: &'static str = Box::leak(name.clone().into_boxed_str()); + self.0.insert(name, leaked); + Ok(leaked) + } +} + +pub(crate) struct BridgeView { + pub(crate) tools: Arc>>, + pub(crate) sources: HashMap>, +} + +pub(crate) struct HotReloadRuntime { + view: RwLock>, + config: Arc, + resources: Arc, + names: Mutex, + locks: HashMap>>, + pub(crate) upstreams: + Arc>>>, +} + +impl HotReloadRuntime { + pub(super) fn new( + config: Arc, + resources: Arc, + sources: HashMap>, + ) -> Arc { + let upstreams = sources + .iter() + .map(|(name, generation)| (name.clone(), generation.control.clone())) + .collect(); + let locks = config + .mcp_servers + .keys() + .map(|name| (name.clone(), Arc::new(AsyncMutex::new(())))) + .collect(); + let runtime = Arc::new(Self { + view: RwLock::new(Arc::new(BridgeView { + tools: Arc::new(Vec::new()), + sources: sources.clone(), + })), + config, + resources, + names: Mutex::new(NamePool(HashMap::new())), + locks, + upstreams: Arc::new(RwLock::new(upstreams)), + }); + match runtime.build_view(sources) { + Ok(view) => *runtime.view.write().unwrap() = Arc::new(view), + Err(code) => tracing::error!(%code, "initial MCP registry could not be built"), + } + runtime + } + + pub(crate) fn snapshot(&self) -> Arc { + self.view.read().unwrap().clone() + } + + fn build_view( + &self, + sources: HashMap>, + ) -> Result { + let mut exposed = Vec::new(); + let mut catalog = Vec::new(); + let mut names = self.names.lock().unwrap(); + let mut sorted: Vec<_> = sources.keys().collect(); + sorted.sort(); + for name in sorted { + let source = &sources[name]; + let sanitized = sanitize(name); + match source.spec.exposure() { + McpToolExposure::Gateway => { + let functions = gateway_functions(&source.tools); + let tool = GatewayTool { + name: names.intern(sanitized)?, + server: name.clone(), + description: gateway_description(name, &functions), + function_names: functions.iter().map(|tool| tool.name.clone()).collect(), + peer: source.peer.clone(), + tool_timeout: source.timeout, + control: source.control.clone(), + resources: self.resources.clone(), + generation: source.clone(), + }; + exposed.push(Box::new(tool) as Box); + } + McpToolExposure::Direct => { + let mut used = HashSet::new(); + for tool in &source.tools { + let raw_name = tool.name.to_string(); + let display = unique_name(bridged_name(&sanitized, &raw_name), &used); + used.insert(display.clone()); + let description = tool + .description + .as_ref() + .map(ToString::to_string) + .filter(|description| !description.trim().is_empty()) + .unwrap_or_else(|| { + format!("Call the '{raw_name}' tool on the '{name}' MCP server.") + }); + exposed.push(Box::new(BridgedTool { + name: names.intern(display)?, + original_name: raw_name, + server: name.clone(), + title: tool.title.clone().filter(|value| !value.trim().is_empty()), + description, + input_schema: Value::Object((*tool.input_schema).clone()), + output_schema: tool + .output_schema + .as_ref() + .map(|schema| Value::Object((**schema).clone())), + annotations: tool.annotations.clone(), + icons: tool.icons.clone(), + meta: tool.meta.clone(), + peer: source.peer.clone(), + tool_timeout: source.timeout, + control: source.control.clone(), + resources: self.resources.clone(), + generation: source.clone(), + }) as Box); + } + } + McpToolExposure::Catalog => { + catalog.push(CatalogSourceInput { + raw_name: name.clone(), + provenance: source.spec.provenance, + transport: transport_label(&source.spec), + peer_info: source.peer.peer_info(), + tools: source.tools.clone(), + peer: source.peer.clone(), + tool_timeout: source.timeout, + control: source.control.clone(), + generation: source.clone(), + }); + } + } + } + let mut tools = build_catalog_tools(catalog, self.resources.clone()); + tools.append(&mut exposed); + let mut seen = HashSet::new(); + let tools = tools + .into_iter() + .filter_map(|tool| { + if !seen.insert(tool.name()) { + tracing::warn!( + name = tool.name(), + "duplicate upstream MCP tool name skipped" + ); + return None; + } + match crate::tool::validate_and_wrap_tool(tool) { + Ok(tool) => Some(Arc::::from(tool)), + Err(error) => { + tracing::warn!(%error, "invalid upstream MCP tool skipped"); + None + } + } + }) + .collect(); + Ok(BridgeView { + sources, + tools: Arc::new(tools), + }) + } + + fn latest_spec(&self, name: &str) -> Result { + // Parse the same CLI arguments as startup; this is read-only and never + // rewrites a config migration. No executable or path comes from the MCP + // request: only the operator's configured source ID is accepted. + let cli = crate::config::Cli::try_parse_from(std::env::args_os()) + .map_err(|_| "CONFIG_CLI_INVALID")?; + let config = + crate::config::load_config_quiet_read_only(cli).map_err(|_| "CONFIG_INVALID")?; + config + .mcp_servers + .get(name) + .cloned() + .ok_or("SOURCE_REMOVED") + } + + pub(crate) async fn reload( + &self, + name: &str, + cancellation: Option<&CancellationToken>, + ) -> Result { + if name.is_empty() || name.len() > 128 || name.chars().any(char::is_control) { + return Err("SOURCE_ID_INVALID"); + } + if !self.locks.contains_key(name) { + return Err("SOURCE_UNKNOWN"); + } + let spec = self.latest_spec(name)?; + self.reload_with_spec(name, spec, cancellation).await + } + + // A test harness can supply an isolated spec without reloading real + // operator configuration. Production accepts source ID only and always + // resolves the current configured spec before reaching this boundary. + pub(super) async fn reload_with_spec( + &self, + name: &str, + spec: crate::types::McpServerSpec, + cancellation: Option<&CancellationToken>, + ) -> Result { + let lock = self.locks.get(name).ok_or("SOURCE_UNKNOWN")?; + let _lease = lock.try_lock().map_err(|_| "RELOAD_IN_PROGRESS")?; + self.reload_with_spec_locked(name, spec, cancellation).await + } + + async fn reload_with_spec_locked( + &self, + name: &str, + spec: crate::types::McpServerSpec, + cancellation: Option<&CancellationToken>, + ) -> Result { + if spec.disabled { + return Err("SOURCE_DISABLED"); + } + if cancellation.is_some_and(CancellationToken::is_cancelled) { + return Err("CANCELLED"); + } + let started = std::time::Instant::now(); + let previous = self.snapshot().sources.get(name).cloned(); + let old_generation = previous.as_ref().map_or(0, |source| source.number); + let future = connect_one(name, &spec, &self.config); + let (service, tools, timeout) = if let Some(token) = cancellation { + tokio::select! { + _ = token.cancelled() => return Err("CANCELLED"), + result = future => result.map_err(|_| "INITIALIZE_FAILED")?, + } + } else { + future.await.map_err(|_| "INITIALIZE_FAILED")? + }; + let ready = matches!( + tokio::time::timeout( + REPLACEMENT_PING_TIMEOUT, + service + .peer() + .send_request(ClientRequest::PingRequest(Default::default())) + ) + .await, + Ok(Ok(_) | Err(ServiceError::McpError(_))) + ); + if !ready { + return Err("READINESS_FAILED"); + } + + let added; + let removed; + let changed; + let unchanged; + { + let before: HashMap<_, _> = previous + .as_ref() + .map(|generation| &generation.tools) + .into_iter() + .flatten() + .map(|tool| { + ( + tool.name.as_ref(), + serde_json::to_value(tool).unwrap_or(Value::Null), + ) + }) + .collect(); + let after: HashMap<_, _> = tools + .iter() + .map(|tool| { + ( + tool.name.as_ref(), + serde_json::to_value(tool).unwrap_or(Value::Null), + ) + }) + .collect(); + added = after + .keys() + .filter(|key| !before.contains_key(**key)) + .count(); + removed = before + .keys() + .filter(|key| !after.contains_key(**key)) + .count(); + changed = after + .iter() + .filter(|(key, value)| before.get(**key).is_some_and(|old| old != *value)) + .count(); + unchanged = after + .iter() + .filter(|(key, value)| before.get(**key).is_some_and(|old| old == *value)) + .count(); + } + let initialization_ms = started.elapsed().as_millis() as u64; + let new = Generation::new(spec, service, tools, timeout, old_generation + 1); + let new_version = new.version(); + // Connection preparation and view construction are outside the write + // lock. Compare-and-rebuild prevents concurrent distinct-source reloads + // from overwriting each other. + if cancellation.is_some_and(CancellationToken::is_cancelled) { + return Err("CANCELLED"); + } + let switching = std::time::Instant::now(); + loop { + let before = self.snapshot(); + let mut sources = before.sources.clone(); + sources.insert(name.to_owned(), new.clone()); + let candidate = Arc::new(self.build_view(sources)?); + let mut current = self.view.write().unwrap(); + if !Arc::ptr_eq(&before, ¤t) { + continue; + } + *current = candidate; + break; + } + self.upstreams + .write() + .unwrap() + .insert(name.to_owned(), new.control.clone()); + let switching_ms = switching.elapsed().as_millis() as u64; + + if new.spec.exposure() == McpToolExposure::Gateway { + let funcs = gateway_functions(&new.tools); + write_gateway_skill(&self.config, name, &sanitize(name), &funcs); + } + + // The old generation stays alive in requests pinned to its tools. + // On a bounded drain timeout its existing calls are NOT cancelled or + // replayed; the generation retires when the last call releases it. + let drain_ms = previous + .as_ref() + .map_or(0, |source| source.spec.request_limits.slow_cancellation_ms); + let draining = std::time::Instant::now(); + let drained = if let Some(old) = &previous { + old.wait_for_calls(Duration::from_millis(drain_ms)).await + } else { + true + }; + let in_flight = previous + .as_ref() + .map_or(0, |old| *old.calls.lock().unwrap()); + // A selected but not yet forwarded call, an old catalog snapshot, + // or an active resource may still pin the retiring generation. + // Do not misreport that its process has already been released. + let still_referenced = previous + .as_ref() + .is_some_and(|old| Arc::strong_count(old) > 1); + Ok(json!({ + "source": name, + "outcome": "reloaded", + "previousGeneration": old_generation, + "replacementGeneration": new.number, + "previousVersion": previous.as_ref().and_then(|old| old.version()), + "replacementVersion": new_version, + "ready": true, + "toolChanges": { + "added": added, "removed": removed, "changed": changed, "unchanged": unchanged + }, + "oldCallsDrained": drained, + "oldCallsStillRunning": in_flight, + "oldGenerationStillReferenced": still_referenced, + "oldGenerationRetiring": !drained || still_referenced, + "rollback": "not_required", + "timingsMs": { + "initializationAndReadiness": initialization_ms, + "registrySwitch": switching_ms, + "drain": draining.elapsed().as_millis() as u64, + "total": started.elapsed().as_millis() as u64 + }, + "clientRediscoveryRequired": true, + "diagnosticCode": "OK" + })) + } +} + +pub(crate) struct ReloadMcpTool { + pub(crate) runtime: Arc, +} + +#[async_trait] +impl Tool for ReloadMcpTool { + fn name(&self) -> &'static str { + "reload_mcp" + } + + fn title(&self) -> String { + "Reload one MCP source".into() + } + + fn description(&self) -> String { + "Transactionally replace one configured upstream MCP source without restarting Codexify or reconnecting any other source. The source is its exact configured MCP server key, not an executable path. Existing calls keep the previous generation. Tool definition changes require the client to rediscover tools/list; in-chat ChatGPT schema updates are not guaranteed.".into() + } + + fn behavior(&self) -> ToolBehavior { + ToolBehavior::new( + false, + true, + false, + false, + "Process replacement is an operator action; no arbitrary executable arguments are accepted.", + ) + } + + fn input_schema(&self) -> Value { + json!({ + "type": "object", + "properties": { + "source": {"type": "string", "minLength": 1, "maxLength": 128, + "description": "Exact stable key of an existing configured MCP server"} + }, + "required": ["source"], + "additionalProperties": false + }) + } + + fn requires_project_root(&self) -> bool { + false + } + + async fn call(&self, args: Value, _config: &AppConfig, _session: &SessionState) -> ToolResult { + self.run(&args, None).await + } + + async fn call_with_context( + &self, + args: Value, + _config: &AppConfig, + _session: &SessionState, + context: &ToolRequestContext, + ) -> ToolResult { + self.run(&args, Some(&context.cancellation)).await + } +} + +impl ReloadMcpTool { + async fn run(&self, args: &Value, cancellation: Option<&CancellationToken>) -> ToolResult { + let Some(source) = args.get("source").and_then(Value::as_str) else { + return ToolResult::error("SOURCE_ID_INVALID"); + }; + match self.runtime.reload(source, cancellation).await { + Ok(report) => { + ToolResult::text("MCP source reloaded; rediscover tools to refresh client schemas.") + .with_structured(report) + } + Err(code) => { + let old = self.runtime.snapshot().sources.get(source).cloned(); + let report = json!({ + "source": source, + "outcome": "failed_without_switch", + "previousGeneration": old.as_ref().map(|generation| generation.number), + "previousVersion": old.as_ref().and_then(|generation| generation.version()), + "rollback": "previous_registry_retained", + "clientRediscoveryRequired": false, + "diagnosticCode": code + }); + ToolResult::error(format!("MCP reload did not switch connections: {code}")) + .with_structured(report) + } + } + } +} diff --git a/src/bridge/hot_reload_tests.rs b/src/bridge/hot_reload_tests.rs new file mode 100644 index 00000000..5dc8b3e3 --- /dev/null +++ b/src/bridge/hot_reload_tests.rs @@ -0,0 +1,926 @@ +//! Isolated, real HTTP MCP fault-injection coverage for per-source reload. +use super::*; +use axum::Router; +use rmcp::{ + ErrorData as McpError, ServerHandler, + model::{ + CallToolResponse, Implementation, InitializeResult, ListToolsResult, + PaginatedRequestParams, ServerCapabilities, ServerInfo, + }, + service::{RequestContext, RoleServer}, + transport::streamable_http_server::{ + StreamableHttpServerConfig, StreamableHttpService, session::local::LocalSessionManager, + }, +}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +#[derive(Clone)] +struct Fixture { + version: &'static str, + new_tool: bool, + list_delay: Duration, + call_delay: Duration, + started: Arc, +} + +impl ServerHandler for Fixture { + fn get_info(&self) -> ServerInfo { + InitializeResult::new(ServerCapabilities::builder().enable_tools().build()) + .with_server_info(Implementation::new("hot-reload-fixture", self.version)) + } + + async fn list_tools( + &self, + _request: Option, + _context: RequestContext, + ) -> Result { + tokio::time::sleep(self.list_delay).await; + let schema = serde_json::json!({ + "type": "object", + "properties": {}, + "additionalProperties": false + }) + .as_object() + .cloned() + .unwrap(); + let mut tools = vec![ + rmcp::model::Tool::new("probe", "Report this fixture's version", schema.clone()), + rmcp::model::Tool::new( + "slow", + "Report this fixture's version after a delay", + schema.clone(), + ), + ]; + if self.new_tool { + tools.push(rmcp::model::Tool::new( + "new_tool", + "Added in version two", + schema, + )); + } + Ok(ListToolsResult::with_all_items(tools)) + } + + async fn call_tool( + &self, + request: CallToolRequestParams, + _context: RequestContext, + ) -> Result { + self.started.fetch_add(1, Ordering::SeqCst); + if request.name.as_ref() == "slow" { + tokio::time::sleep(self.call_delay).await; + } + Ok(CallToolResult::success(vec![ContentBlock::text(self.version)]).into()) + } +} + +async fn fixture( + version: &'static str, + new_tool: bool, + list_delay: Duration, + call_delay: Duration, +) -> (String, Arc, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let address = listener.local_addr().unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + let fixture = Fixture { + version, + new_tool, + list_delay, + call_delay, + started: calls.clone(), + }; + let mut transport = StreamableHttpServerConfig::default(); + transport.json_response = true; + let service = StreamableHttpService::new( + move || Ok(fixture.clone()), + Arc::new(LocalSessionManager::default()), + transport, + ); + let app = Router::new().nest_service("/mcp", service); + let task = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + (format!("http://{address}/mcp"), calls, task) +} + +fn spec(url: &str, exposure: McpToolExposure) -> crate::types::McpServerSpec { + crate::types::McpServerSpec { + url: Some(url.to_string()), + mode: Some(exposure), + ..Default::default() + } +} + +async fn invoke(view: Arc, name: &str) -> ToolResult { + invoke_args(view, name, json!({})).await +} + +async fn invoke_args(view: Arc, name: &str, args: Value) -> ToolResult { + let tool = view + .tools + .iter() + .find(|tool| tool.name() == name) + .unwrap_or_else(|| panic!("tool {name} missing from view")); + tool.call( + args, + &crate::config::default_config(std::env::temp_dir()), + &SessionState::new(), + ) + .await +} + +#[tokio::test] +async fn single_source_reload_updates_tools_but_not_other_mcp() { + let (old_url, _, old_task) = fixture("1", false, Duration::ZERO, Duration::ZERO).await; + let (new_url, _, new_task) = fixture("2", true, Duration::ZERO, Duration::ZERO).await; + let (other_url, _, other_task) = fixture("other", false, Duration::ZERO, Duration::ZERO).await; + let mut config = crate::config::default_config(std::env::temp_dir()); + config + .mcp_servers + .insert("alpha".into(), spec(&old_url, McpToolExposure::Direct)); + config + .mcp_servers + .insert("beta".into(), spec(&other_url, McpToolExposure::Direct)); + let bridge = connect_upstreams(&config).await; + let runtime = bridge.runtime.clone(); + let before = runtime.snapshot(); + assert_eq!( + invoke(before.clone(), "alpha__probe").await.joined_text(), + "1" + ); + assert_eq!( + invoke(before.clone(), "beta__probe").await.joined_text(), + "other" + ); + + let result = runtime + .reload_with_spec("alpha", spec(&new_url, McpToolExposure::Direct), None) + .await + .unwrap(); + assert_eq!(result["outcome"], "reloaded"); + assert_eq!(result["previousGeneration"], 1); + assert_eq!(result["replacementGeneration"], 2); + assert_eq!(result["previousVersion"], "1"); + assert_eq!(result["replacementVersion"], "2"); + assert_eq!(result["toolChanges"]["added"], 1); + assert_eq!(result["oldCallsDrained"], true); + + let after = runtime.snapshot(); + assert!(Arc::ptr_eq(&before.sources["beta"], &after.sources["beta"])); + assert!(!Arc::ptr_eq( + &before.sources["alpha"], + &after.sources["alpha"] + )); + assert_eq!( + invoke(after.clone(), "alpha__probe").await.joined_text(), + "2" + ); + assert_eq!( + invoke(after.clone(), "alpha__new_tool").await.joined_text(), + "2" + ); + assert_eq!(invoke(after, "beta__probe").await.joined_text(), "other"); + assert_eq!(invoke(before, "alpha__probe").await.joined_text(), "1"); + old_task.abort(); + new_task.abort(); + other_task.abort(); +} + +#[tokio::test] +async fn active_calls_keep_their_original_generation_without_replay() { + let (old_url, old_count, old_task) = + fixture("1", false, Duration::ZERO, Duration::from_millis(450)).await; + let (new_url, new_count, new_task) = fixture("2", true, Duration::ZERO, Duration::ZERO).await; + let mut config = crate::config::default_config(std::env::temp_dir()); + config + .mcp_servers + .insert("alpha".into(), spec(&old_url, McpToolExposure::Direct)); + let runtime = connect_upstreams(&config).await.runtime; + let initial = runtime.snapshot(); + let pending = tokio::spawn(invoke(initial, "alpha__slow")); + tokio::time::timeout(Duration::from_secs(2), async { + while old_count.load(Ordering::SeqCst) == 0 { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .unwrap(); + let started = std::time::Instant::now(); + let result = runtime + .reload_with_spec("alpha", spec(&new_url, McpToolExposure::Direct), None) + .await + .unwrap(); + assert_eq!(result["oldCallsDrained"], true); + assert!(started.elapsed() >= Duration::from_millis(100)); + assert_eq!(pending.await.unwrap().joined_text(), "1"); + assert_eq!(old_count.load(Ordering::SeqCst), 1); + assert_eq!( + invoke(runtime.snapshot(), "alpha__probe") + .await + .joined_text(), + "2" + ); + assert_eq!(new_count.load(Ordering::SeqCst), 1); + old_task.abort(); + new_task.abort(); +} + +#[tokio::test] +async fn failed_replacement_and_unknown_ids_cannot_mutate_active_generation() { + let (old_url, _, old_task) = fixture("1", false, Duration::ZERO, Duration::ZERO).await; + let mut config = crate::config::default_config(std::env::temp_dir()); + config + .mcp_servers + .insert("alpha".into(), spec(&old_url, McpToolExposure::Direct)); + let runtime = connect_upstreams(&config).await.runtime; + let old = runtime.snapshot(); + + assert_eq!( + runtime + .reload_with_spec( + "missing", + spec("http://127.0.0.1:9/mcp", McpToolExposure::Direct), + None + ) + .await + .unwrap_err(), + "SOURCE_UNKNOWN" + ); + assert_eq!( + runtime + .reload_with_spec( + "alpha", + spec("http://127.0.0.1:9/mcp", McpToolExposure::Direct), + None + ) + .await + .unwrap_err(), + "INITIALIZE_FAILED" + ); + let after = runtime.snapshot(); + assert!(Arc::ptr_eq(&old, &after)); + assert_eq!(invoke(after, "alpha__probe").await.joined_text(), "1"); + old_task.abort(); +} + +#[tokio::test] +async fn same_source_reload_conflicts_while_other_source_can_reload() { + let (a1, _, first_task) = fixture("1", false, Duration::ZERO, Duration::ZERO).await; + let (a2, _, second_task) = fixture("2", true, Duration::from_millis(300), Duration::ZERO).await; + let (b1, _, third_task) = fixture("other1", false, Duration::ZERO, Duration::ZERO).await; + let (b2, _, fourth_task) = fixture("other2", false, Duration::ZERO, Duration::ZERO).await; + let mut config = crate::config::default_config(std::env::temp_dir()); + config + .mcp_servers + .insert("alpha".into(), spec(&a1, McpToolExposure::Direct)); + config + .mcp_servers + .insert("beta".into(), spec(&b1, McpToolExposure::Direct)); + let runtime = connect_upstreams(&config).await.runtime; + let clone = runtime.clone(); + let initial = tokio::spawn(async move { + clone + .reload_with_spec("alpha", spec(&a2, McpToolExposure::Direct), None) + .await + }); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!( + runtime + .reload_with_spec("alpha", spec(&a1, McpToolExposure::Direct), None) + .await + .unwrap_err(), + "RELOAD_IN_PROGRESS" + ); + let other = runtime + .reload_with_spec("beta", spec(&b2, McpToolExposure::Direct), None) + .await + .unwrap(); + assert_eq!(other["replacementGeneration"], 2); + assert_eq!(initial.await.unwrap().unwrap()["replacementGeneration"], 2); + let view = runtime.snapshot(); + assert_eq!( + invoke(view.clone(), "alpha__probe").await.joined_text(), + "2" + ); + assert_eq!(invoke(view, "beta__probe").await.joined_text(), "other2"); + first_task.abort(); + second_task.abort(); + third_task.abort(); + fourth_task.abort(); +} + +#[tokio::test] +async fn catalog_reload_refreshes_search_schema_and_call_routing() { + let (old_url, _, old_task) = fixture("1", false, Duration::ZERO, Duration::ZERO).await; + let (new_url, _, new_task) = fixture("2", true, Duration::ZERO, Duration::ZERO).await; + let mut config = crate::config::default_config(std::env::temp_dir()); + config + .mcp_servers + .insert("catalog".into(), spec(&old_url, McpToolExposure::Catalog)); + let runtime = connect_upstreams(&config).await.runtime; + let before = runtime.snapshot(); + let source = invoke(before, "mcp_list_sources") + .await + .structured_content + .unwrap(); + assert_eq!(source["sources"][0]["toolCount"], 2); + let result = runtime + .reload_with_spec("catalog", spec(&new_url, McpToolExposure::Catalog), None) + .await + .unwrap(); + assert_eq!(result["toolChanges"]["added"], 1); + let after = runtime.snapshot(); + let list = invoke(after.clone(), "mcp_list_sources") + .await + .structured_content + .unwrap(); + assert_eq!(list["sources"][0]["toolCount"], 3); + let source_id = list["sources"][0]["id"].as_str().unwrap(); + let found = invoke_args( + after.clone(), + "mcp_search_tools", + json!({"query":"new_tool","source":source_id}), + ) + .await + .structured_content + .unwrap(); + assert!(found["total"].as_u64().unwrap_or(0) >= 1); + let tool_id = found["matches"][0]["toolId"].as_str().unwrap(); + let call = invoke_args( + after, + "mcp_call_tool", + json!({"source":source_id,"tool":tool_id,"arguments":{}}), + ) + .await; + assert!(!call.is_error); + assert_eq!(call.joined_text(), "2"); + old_task.abort(); + new_task.abort(); +} + +#[tokio::test] +async fn gateway_reload_reflects_added_functions() { + let (old_url, _, old_task) = fixture("1", false, Duration::ZERO, Duration::ZERO).await; + let (new_url, _, new_task) = fixture("2", true, Duration::ZERO, Duration::ZERO).await; + let mut config = crate::config::default_config(std::env::temp_dir()); + config + .mcp_servers + .insert("gateway".into(), spec(&old_url, McpToolExposure::Gateway)); + let runtime = connect_upstreams(&config).await.runtime; + let before = runtime.snapshot(); + let old = invoke_args( + before, + "gateway", + json!({"function":"probe","arguments":{}}), + ) + .await; + assert_eq!(old.joined_text(), "1"); + runtime + .reload_with_spec("gateway", spec(&new_url, McpToolExposure::Gateway), None) + .await + .unwrap(); + let after = runtime.snapshot(); + let tool = after + .tools + .iter() + .find(|tool| tool.name() == "gateway") + .unwrap(); + assert!( + tool.input_schema()["properties"]["function"]["enum"] + .as_array() + .unwrap() + .contains(&json!("new_tool")) + ); + let new = invoke_args( + after, + "gateway", + json!({"function":"new_tool","arguments":{}}), + ) + .await; + assert_eq!(new.joined_text(), "2"); + old_task.abort(); + new_task.abort(); +} + +#[tokio::test] +async fn reconnect_failed_initial_source_and_cancel_without_switching() { + let (url, _, task) = fixture("recovered", false, Duration::ZERO, Duration::ZERO).await; + let mut config = crate::config::default_config(std::env::temp_dir()); + config.mcp_servers.insert( + "unavailable".into(), + crate::types::McpServerSpec { + url: Some("http://127.0.0.1:9/mcp".into()), + startup_timeout_sec: Some(0.5), + ..Default::default() + }, + ); + let runtime = connect_upstreams(&config).await.runtime; + assert!(!runtime.snapshot().sources.contains_key("unavailable")); + let result = runtime + .reload_with_spec("unavailable", spec(&url, McpToolExposure::Direct), None) + .await + .unwrap(); + assert_eq!(result["previousGeneration"], 0); + assert_eq!(result["replacementGeneration"], 1); + assert_eq!( + invoke(runtime.snapshot(), "unavailable__probe") + .await + .joined_text(), + "recovered" + ); + + let old = runtime.snapshot(); + let cancelled = CancellationToken::new(); + cancelled.cancel(); + assert_eq!( + runtime + .reload_with_spec( + "unavailable", + spec(&url, McpToolExposure::Direct), + Some(&cancelled) + ) + .await + .unwrap_err(), + "CANCELLED" + ); + assert!(Arc::ptr_eq(&old, &runtime.snapshot())); + task.abort(); +} + +#[tokio::test] +async fn repeated_reloads_keep_one_active_generation_per_source() { + let (url, _, task) = fixture("stable", false, Duration::ZERO, Duration::ZERO).await; + let mut config = crate::config::default_config(std::env::temp_dir()); + config + .mcp_servers + .insert("alpha".into(), spec(&url, McpToolExposure::Direct)); + let runtime = connect_upstreams(&config).await.runtime; + for expected in 2..=6 { + let result = runtime + .reload_with_spec("alpha", spec(&url, McpToolExposure::Direct), None) + .await + .unwrap(); + assert_eq!(result["replacementGeneration"], expected); + assert_eq!(result["oldCallsDrained"], true); + let view = runtime.snapshot(); + assert_eq!(view.sources.len(), 1); + assert_eq!(invoke(view, "alpha__probe").await.joined_text(), "stable"); + } + task.abort(); +} + +#[tokio::test] +async fn direct_name_collisions_are_not_ambiguous_after_reload() { + let (url1, _, task1) = fixture("one", false, Duration::ZERO, Duration::ZERO).await; + let (url2, _, task2) = fixture("two", true, Duration::ZERO, Duration::ZERO).await; + let mut config = crate::config::default_config(std::env::temp_dir()); + // sanitize("dup!")==sanitize("dup?"), so both would generate dup___probe. + config + .mcp_servers + .insert("dup!".into(), spec(&url1, McpToolExposure::Direct)); + config + .mcp_servers + .insert("dup?".into(), spec(&url2, McpToolExposure::Direct)); + let runtime = connect_upstreams(&config).await.runtime; + let view = runtime.snapshot(); + let names = view + .tools + .iter() + .map(|tool| tool.name()) + .collect::>(); + let unique = names + .iter() + .copied() + .collect::>(); + assert_eq!(unique.len(), names.len()); + assert_eq!( + names.iter().filter(|name| **name == "dup___probe").count(), + 1 + ); + task1.abort(); + task2.abort(); +} + +#[tokio::test] +async fn old_generation_is_released_after_draining_without_stale_snapshots() { + let (old_url, _, old_task) = fixture("old", false, Duration::ZERO, Duration::ZERO).await; + let (new_url, _, new_task) = fixture("new", true, Duration::ZERO, Duration::ZERO).await; + let mut config = crate::config::default_config(std::env::temp_dir()); + config + .mcp_servers + .insert("alpha".into(), spec(&old_url, McpToolExposure::Direct)); + let bridge = connect_upstreams(&config).await; + let runtime = bridge.runtime.clone(); + let first = runtime.snapshot(); + let weak = Arc::downgrade(&first.sources["alpha"]); + drop(first); + // The service keeps only the runtime after consuming the startup report. + drop(bridge); + assert!(weak.upgrade().is_some()); + + let result = runtime + .reload_with_spec("alpha", spec(&new_url, McpToolExposure::Direct), None) + .await + .unwrap(); + assert_eq!(result["oldCallsDrained"], true); + assert!( + weak.upgrade().is_none(), + "retired service generation leaked" + ); + assert_eq!( + invoke(runtime.snapshot(), "alpha__probe") + .await + .joined_text(), + "new" + ); + old_task.abort(); + new_task.abort(); +} + +#[tokio::test] +async fn unrelated_in_flight_call_does_not_retain_a_retired_source() { + let (a1, _, task_a1) = fixture("alpha-1", false, Duration::ZERO, Duration::ZERO).await; + let (a2, _, task_a2) = fixture("alpha-2", false, Duration::ZERO, Duration::ZERO).await; + let (b, b_calls, task_b) = + fixture("beta", false, Duration::ZERO, Duration::from_millis(400)).await; + let mut config = crate::config::default_config(std::env::temp_dir()); + config + .mcp_servers + .insert("alpha".into(), spec(&a1, McpToolExposure::Direct)); + config + .mcp_servers + .insert("beta".into(), spec(&b, McpToolExposure::Direct)); + let runtime = connect_upstreams(&config).await.runtime; + let initial = runtime.snapshot(); + let alpha = Arc::downgrade(&initial.sources["alpha"]); + // Copy only beta's validated descriptor, just as server dispatch does. + let beta = initial + .tools + .iter() + .find(|tool| tool.name() == "beta__slow") + .unwrap() + .clone(); + drop(initial); + + let pending = tokio::spawn(async move { + beta.call( + json!({}), + &crate::config::default_config(std::env::temp_dir()), + &SessionState::new(), + ) + .await + }); + tokio::time::timeout(Duration::from_secs(2), async { + while b_calls.load(Ordering::SeqCst) == 0 { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .unwrap(); + + let result = runtime + .reload_with_spec("alpha", spec(&a2, McpToolExposure::Direct), None) + .await + .unwrap(); + assert_eq!(result["oldCallsDrained"], true); + assert!( + alpha.upgrade().is_none(), + "a beta request must not retain alpha's retired connection" + ); + assert_eq!(pending.await.unwrap().joined_text(), "beta"); + assert_eq!( + invoke(runtime.snapshot(), "alpha__probe") + .await + .joined_text(), + "alpha-2" + ); + task_a1.abort(); + task_a2.abort(); + task_b.abort(); +} + +fn isolated_stdio_spec(version: &str) -> crate::types::McpServerSpec { + let fixture = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("scripts") + .join("fixtures") + .join("hot-reload-stdio-mcp.mjs"); + crate::types::McpServerSpec { + command: Some("node".into()), + args: vec![fixture.to_string_lossy().into_owned(), version.into()], + mode: Some(McpToolExposure::Direct), + startup_timeout_sec: Some(25.0), + ..Default::default() + } +} + +fn stdio_process_id(result: ToolResult, expected_version: &str) -> u32 { + assert!( + !result.is_error, + "stdio call failed: {}", + result.joined_text() + ); + let response = result.joined_text(); + let (version, pid) = response + .trim() + .split_once(':') + .unwrap_or_else(|| panic!("invalid stdio fixture response: {response}")); + assert_eq!(version, expected_version); + pid.parse() + .expect("stdio fixture process id must be numeric") +} + +async fn stdio_process_still_alive(pid: u32) -> bool { + // Signal 0 checks existence without terminating or signaling a process. + tokio::process::Command::new("node") + .arg("-e") + .arg("try { process.kill(Number(process.argv[1]), 0); } catch { process.exit(1); }") + .arg(pid.to_string()) + .status() + .await + .is_ok_and(|status| status.success()) +} + +#[tokio::test] +async fn service_isolated_real_stdio_process_reload_preserves_unrelated_sources() { + // This fixture actually spawns three independent OS processes through the + // production TokioChildProcess transport; no live Codexify is involved. + let directory = tempfile::tempdir().unwrap(); + let started = directory.path().join("slow_started"); + let mut old_spec = isolated_stdio_spec("old"); + old_spec.env.insert( + "HOT_RELOAD_SLOW_MARKER_FILE".into(), + started.to_string_lossy().into_owned(), + ); + + let mut config = crate::config::default_config(std::env::temp_dir()); + config.mcp_servers.insert("alpha".into(), old_spec); + config + .mcp_servers + .insert("beta".into(), isolated_stdio_spec("beta")); + config + .mcp_servers + .insert("gamma".into(), isolated_stdio_spec("gamma")); + let bridge = connect_upstreams(&config).await; + assert_eq!( + bridge.runtime.snapshot().sources.len(), + 3, + "all isolated stdio fixtures must start: {:?}", + bridge.report + ); + let runtime = bridge.runtime.clone(); + drop(bridge); + + let initial = runtime.snapshot(); + let retired = Arc::downgrade(&initial.sources["alpha"]); + let old_probe = initial + .tools + .iter() + .find(|tool| tool.name() == "alpha__probe") + .unwrap() + .clone(); + let old_slow = initial + .tools + .iter() + .find(|tool| tool.name() == "alpha__slow") + .unwrap() + .clone(); + drop(initial); + let config_for_call = crate::config::default_config(std::env::temp_dir()); + let original_pid = stdio_process_id( + old_probe + .call(json!({}), &config_for_call, &SessionState::new()) + .await, + "old", + ); + let beta_pid = stdio_process_id(invoke(runtime.snapshot(), "beta__probe").await, "beta"); + let gamma_pid = stdio_process_id(invoke(runtime.snapshot(), "gamma__probe").await, "gamma"); + + let pending = tokio::spawn(async move { + old_slow + .call( + json!({}), + &crate::config::default_config(std::env::temp_dir()), + &SessionState::new(), + ) + .await + }); + // The marker is written by the actual old process when it receives the + // slow request, making this test independent of host scheduling. + tokio::time::timeout(Duration::from_secs(4), async { + while !started.exists() { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("old stdio process did not begin the slow call"); + + let result = runtime + .reload_with_spec("alpha", isolated_stdio_spec("new"), None) + .await + .unwrap(); + assert_eq!(result["previousVersion"], "old"); + assert_eq!(result["replacementVersion"], "new"); + assert_eq!(result["previousGeneration"], 1); + assert_eq!(result["replacementGeneration"], 2); + assert_eq!(result["toolChanges"]["added"], 1); + assert_eq!(result["oldCallsDrained"], true); + assert_eq!( + stdio_process_id(pending.await.unwrap(), "old"), + original_pid + ); + assert_eq!( + stdio_process_id( + old_probe + .call(json!({}), &config_for_call, &SessionState::new()) + .await, + "old", + ), + original_pid, + "a pinned retired generation must remain callable until released" + ); + let new_pid = stdio_process_id(invoke(runtime.snapshot(), "alpha__probe").await, "new"); + assert_ne!( + new_pid, original_pid, + "replacement must be a new OS process" + ); + assert_eq!( + stdio_process_id(invoke(runtime.snapshot(), "alpha__new_tool").await, "new"), + new_pid + ); + assert_eq!( + stdio_process_id(invoke(runtime.snapshot(), "beta__probe").await, "beta"), + beta_pid, + "beta must not reconnect" + ); + assert_eq!( + stdio_process_id(invoke(runtime.snapshot(), "gamma__probe").await, "gamma"), + gamma_pid, + "gamma must not reconnect" + ); + + let mut invalid_spec = isolated_stdio_spec("broken"); + invalid_spec.command = Some("codexify-nonexistent-hot-reload-fixture".into()); + assert_eq!( + runtime + .reload_with_spec("alpha", invalid_spec, None) + .await + .err(), + Some("INITIALIZE_FAILED") + ); + assert_eq!( + stdio_process_id(invoke(runtime.snapshot(), "alpha__probe").await, "new"), + new_pid, + "an unsuccessful replacement must preserve the healthy process" + ); + + assert!( + stdio_process_still_alive(original_pid).await, + "retiring process must remain alive while a pinned old handle exists" + ); + drop(old_probe); + tokio::task::yield_now().await; + assert!( + retired.upgrade().is_none(), + "the retired generation must be released after its last handle drops" + ); + tokio::time::timeout(Duration::from_secs(5), async { + while stdio_process_still_alive(original_pid).await { + tokio::time::sleep(Duration::from_millis(25)).await; + } + }) + .await + .expect("retired OS child process remained alive after the last handle dropped"); + + let repeated = runtime + .reload_with_spec("alpha", isolated_stdio_spec("new"), None) + .await + .unwrap(); + assert_eq!(repeated["previousGeneration"], 2); + assert_eq!(repeated["replacementGeneration"], 3); + assert_eq!(repeated["toolChanges"]["unchanged"], 3); + assert_eq!( + stdio_process_id(invoke(runtime.snapshot(), "beta__probe").await, "beta"), + beta_pid + ); + assert_eq!( + stdio_process_id(invoke(runtime.snapshot(), "gamma__probe").await, "gamma"), + gamma_pid + ); +} + +/// Opt-in, isolated acceptance test using two real, read-only GodPrompt stdio +/// installs. Only initialize, tools/list, and ping are sent; no GodPrompt tool +/// calls, external APIs, operator configuration, or running service are touched. +/// +/// CI installs and selects the pinned published 1.0.3 / 1.1.1 npm versions. +/// A developer can instead supply two existing entrypoint paths locally. +#[tokio::test] +async fn service_local_godprompt_candidate_upgrade_and_failed_rollback() { + let old_entry = std::env::var_os("CODEXIFY_CANARY_OLD_GODPROMPT"); + let new_entry = std::env::var_os("CODEXIFY_CANARY_NEW_GODPROMPT"); + let (Some(old_entry), Some(new_entry)) = (old_entry, new_entry) else { + assert!( + std::env::var_os("CODEXIFY_CANARY_OLD_GODPROMPT").is_none() + && std::env::var_os("CODEXIFY_CANARY_NEW_GODPROMPT").is_none(), + "provide both isolated GodPrompt entrypoints, or neither" + ); + return; + }; + let node = std::env::var("CODEXIFY_CANARY_NODE").unwrap_or_else(|_| "node".into()); + for entry in [&old_entry, &new_entry] { + assert!( + std::path::Path::new(entry).is_file(), + "GodPrompt candidate stdio entrypoint does not exist" + ); + } + let make_candidate = |entry: &std::ffi::OsStr| crate::types::McpServerSpec { + command: Some(node.clone()), + args: vec![entry.to_string_lossy().into_owned()], + mode: Some(McpToolExposure::Direct), + startup_timeout_sec: Some(18.0), + ..Default::default() + }; + + let (beta_url, _, beta_server) = fixture("beta", false, Duration::ZERO, Duration::ZERO).await; + let (gamma_url, _, gamma_server) = + fixture("gamma", false, Duration::ZERO, Duration::ZERO).await; + let mut config = crate::config::default_config(std::env::temp_dir()); + config.mcp_servers.insert( + "godprompt_canary".into(), + make_candidate(old_entry.as_os_str()), + ); + config + .mcp_servers + .insert("beta".into(), spec(&beta_url, McpToolExposure::Direct)); + config + .mcp_servers + .insert("gamma".into(), spec(&gamma_url, McpToolExposure::Direct)); + + let bridge = connect_upstreams(&config).await; + assert_eq!( + bridge.runtime.snapshot().sources.len(), + 3, + "{:?}", + bridge.report + ); + let runtime = bridge.runtime.clone(); + drop(bridge); + let before = runtime.snapshot(); + let old = before.sources["godprompt_canary"].clone(); + let beta = before.sources["beta"].clone(); + let gamma = before.sources["gamma"].clone(); + let old_tool_count = old.tools.len(); + assert!(old_tool_count > 0, "old GodPrompt must advertise tools"); + drop(before); + + let result = runtime + .reload_with_spec( + "godprompt_canary", + make_candidate(new_entry.as_os_str()), + None, + ) + .await + .expect("replacement GodPrompt must initialize, discover tools and pass ping"); + assert_eq!(result["previousGeneration"], 1); + assert_eq!(result["replacementGeneration"], 2); + assert_eq!(result["ready"], true); + assert_eq!(result["previousVersion"], "1.0.3"); + assert_eq!(result["replacementVersion"], "1.1.1"); + let updated = runtime.snapshot(); + let new = updated.sources["godprompt_canary"].clone(); + assert!(!Arc::ptr_eq(&old, &new)); + assert!(Arc::ptr_eq(&beta, &updated.sources["beta"])); + assert!(Arc::ptr_eq(&gamma, &updated.sources["gamma"])); + assert!(!new.tools.is_empty(), "new GodPrompt must advertise tools"); + println!( + "isolated GodPrompt {} -> {}: {} -> {} tools; timings {:?}; testing rollback", + result["previousVersion"], + result["replacementVersion"], + old_tool_count, + new.tools.len(), + result["timingsMs"] + ); + drop(updated); + + let mut broken = make_candidate(new_entry.as_os_str()); + broken.command = Some("codexify-nonexistent-godprompt-canary".into()); + assert_eq!( + runtime + .reload_with_spec("godprompt_canary", broken, None) + .await + .err(), + Some("INITIALIZE_FAILED") + ); + let after_rollback = runtime.snapshot(); + assert!(Arc::ptr_eq( + &new, + &after_rollback.sources["godprompt_canary"] + )); + assert!(Arc::ptr_eq(&beta, &after_rollback.sources["beta"])); + assert!(Arc::ptr_eq(&gamma, &after_rollback.sources["gamma"])); + beta_server.abort(); + gamma_server.abort(); +} diff --git a/src/markdown_chat/mod.rs b/src/markdown_chat/mod.rs index 5e11b839..2104b2ba 100644 --- a/src/markdown_chat/mod.rs +++ b/src/markdown_chat/mod.rs @@ -178,6 +178,9 @@ pub struct AgentAutomationConfig { #[serde(skip_serializing_if = "Option::is_none")] pub browser_simplicity: Option, pub recovery_enhancements: bool, + /// Preserve the operator-configured browser Send gate in the v2.4 preview. + #[serde(skip_serializing_if = "Option::is_none")] + pub provider_send_gate_path: Option, } impl Default for AgentAutomationConfig { @@ -204,6 +207,7 @@ impl Default for AgentAutomationConfig { minimal_ui: false, browser_simplicity: None, recovery_enhancements: false, + provider_send_gate_path: None, } } } @@ -227,6 +231,15 @@ impl AgentAutomationConfig { fn validate(&self) -> Result<(), String> { self.retry_timings.validate()?; + if let Some(path) = &self.provider_send_gate_path + && (path.len() > 4096 + || !std::path::Path::new(path).is_absolute() + || path.contains('\0')) + { + return Err( + "agentChat.automation.providerSendGatePath must be an absolute path".into(), + ); + } if !(1..=4).contains(&self.browser_action_lanes) { return Err("agentChat.automation.browserActionLanes must be between 1 and 4".into()); } @@ -1051,6 +1064,27 @@ mod tests { } } + #[test] + fn browser_send_gate_path_round_trips_and_rejects_relative_paths() { + let absolute = if cfg!(windows) { + r"C:\codexify\browser-send-gate.json" + } else { + "/tmp/codexify/browser-send-gate.json" + }; + let config: AgentAutomationConfig = + serde_json::from_value(serde_json::json!({"providerSendGatePath": absolute})).unwrap(); + assert_eq!(config.provider_send_gate_path.as_deref(), Some(absolute)); + assert!(config.validate().is_ok()); + let encoded = serde_json::to_value(&config).unwrap(); + assert_eq!(encoded["providerSendGatePath"], absolute); + for invalid in ["", "relative/send-gate.json"] { + let config: AgentAutomationConfig = + serde_json::from_value(serde_json::json!({"providerSendGatePath": invalid})) + .unwrap(); + assert!(config.validate().is_err()); + } + } + #[test] fn new_chat_project_configuration_is_optional_and_validated() { assert!(AgentAutomationConfig::default().validate().is_ok()); diff --git a/src/mcp_catalog.rs b/src/mcp_catalog.rs index 2144716f..6a607424 100644 --- a/src/mcp_catalog.rs +++ b/src/mcp_catalog.rs @@ -40,6 +40,7 @@ pub(crate) struct CatalogSourceInput { pub peer: Peer, pub tool_timeout: Option, pub control: Arc, + pub generation: Arc, } struct CatalogTool { @@ -59,6 +60,7 @@ struct CatalogSource { peer: Peer, tool_timeout: Option, control: Arc, + generation: Arc, } struct SearchDocument { @@ -128,6 +130,7 @@ impl Catalog { peer: input.peer, tool_timeout: input.tool_timeout, control: input.control, + generation: input.generation, }); } @@ -1032,6 +1035,7 @@ impl McpCallToolTool { Err(error) => return ToolResult::error(error), }; + let _lease = source.generation.begin_call(); let mut params = CallToolRequestParams::new(tool.raw.name.to_string()); match args.get("arguments") { None | Some(Value::Null) => {} diff --git a/src/server.rs b/src/server.rs index 456491a4..770b8647 100644 --- a/src/server.rs +++ b/src/server.rs @@ -146,6 +146,7 @@ type HttpServerTask = JoinHandle>; pub struct CodexHandler { config: Arc, tools: Arc>>, + bridge_runtime: Option>, project_bindings: Arc, connector_schemas: Arc, markdown_chat: Arc, @@ -653,11 +654,20 @@ impl ServerHandler for CodexHandler { }) .await .unwrap_or_else(|_| self.config.as_ref().clone()); - let tools = self - .tools - .iter() - .map(|tool| advertised_tool(tool.as_ref(), &advertising)) - .collect(); + let mut names = std::collections::HashSet::new(); + let mut tools = Vec::new(); + for tool in self.tools.iter() { + names.insert(tool.name()); + tools.push(advertised_tool(tool.as_ref(), &advertising)); + } + if let Some(runtime) = &self.bridge_runtime { + let view = runtime.snapshot(); + for tool in view.tools.iter() { + if names.insert(tool.name()) { + tools.push(advertised_tool(tool.as_ref(), &advertising)); + } + } + } let mut result = ListToolsResult::with_all_items(tools); let caller = crate::connector_schema::caller_key(&context.meta).or_else(|| { request @@ -826,9 +836,21 @@ impl ServerHandler for CodexHandler { .arguments .map(Value::Object) .unwrap_or_else(|| json!({})); - let tool = self.tools.iter().find(|t| t.name() == name); + // Pin only the selected upstream tool for this entire async dispatch. + // Holding the full registry snapshot here would keep *unrelated* retired + // MCP processes alive while another source runs a long request. + let bridge_tool = self.bridge_runtime.as_ref().and_then(|runtime| { + let view = runtime.snapshot(); + view.tools.iter().find(|tool| tool.name() == name).cloned() + }); + let tool = self + .tools + .iter() + .find(|tool| tool.name() == name) + .map(|tool| tool.as_ref()) + .or(bridge_tool.as_deref()); let ticket_wait = name == TicketWait::NAME && tool.is_some(); - let model_call = !tool.is_some_and(|tool| app_only_tool(tool.as_ref())); + let model_call = !tool.is_some_and(app_only_tool); let mut task_conversation = conversation.clone(); let mut task_owner_generation = None; let mut conversation_retired = false; @@ -1024,7 +1046,7 @@ impl ServerHandler for CodexHandler { // Keep `tool` as an Option so that even an unknown-tool call flows through // the audit begin/finish pairing below rather than short-circuiting. let agent_call = self.config.markdown_chat.enabled - && tool.is_some_and(|tool| !app_only_tool(tool.as_ref())) + && tool.is_some_and(|tool| !app_only_tool(tool)) && !ticket_wait && !context.ct.is_cancelled(); let called_at_ms = crate::markdown_chat::now_ms(); @@ -1657,7 +1679,7 @@ impl ServerHandler for CodexHandler { .await; } } - finalize_model_visible_result(tool.map(|tool| tool.as_ref()), &mut result, &self.config); + finalize_model_visible_result(tool, &mut result, &self.config); if !result.is_error && let Some(tool) = tool && let Err(error) = tool.validate_structured_output(result.structured_content.as_ref()) @@ -1790,7 +1812,7 @@ impl ServerHandler for CodexHandler { && name != "automation_verify_pairing" && !pairing_continuation && !ticket_wait - && !tool.is_some_and(|tool| app_only_tool(tool.as_ref())) + && !tool.is_some_and(app_only_tool) && !context.ct.is_cancelled() && self .conversation_auth_error("chat_read", conversation.as_ref()) @@ -2062,7 +2084,9 @@ struct HealthState { tool_count: usize, ready: Arc, multi_tunnel: Option<(Arc, usize)>, - upstreams: Arc>>, + upstreams: + Arc>>>, + runtime: Option>, filesystem: Option>, } @@ -2083,14 +2107,18 @@ async fn health(State(state): State) -> (StatusCode, Json) { || state.ready.load(Ordering::Acquire), |(active, _)| *active > 0, ); + let exposed = state + .runtime + .as_ref() + .map_or(0, |runtime| runtime.snapshot().tools.len()); let mut payload = json!({ "status": if is_ready { "ok" } else { "starting" }, - "tools": state.tool_count, + "tools": state.tool_count + exposed, }); - if !state.upstreams.is_empty() { + let upstreams = state.upstreams.read().unwrap(); + if !upstreams.is_empty() { payload["upstreams"] = Value::Object( - state - .upstreams + upstreams .iter() .map(|(name, state)| (name.clone(), state.snapshot())) .collect(), @@ -2180,35 +2208,24 @@ pub async fn start_http_server(mut config: AppConfig) -> anyhow::Result<()> { let bridge = crate::bridge::connect_upstreams(&config).await; let bridge_report = bridge.report; let bridged_resources = bridge.resources; - let _bridge_services = bridge.services; - let _bridge_monitors = bridge.monitors; let upstreams = bridge.upstreams; + let bridge_runtime = bridge.runtime; + // The test-facing startup tool vector holds references to generation 1. + // Drop it immediately: only the runtime's replaceable snapshot may retain + // active sources, or retired child processes would remain pinned forever. + drop(bridge.tools); let mut all_tools = validate_and_wrap_tools(load_tools_for_config(&config)) .map_err(|error| anyhow::anyhow!("invalid native tool contract: {error}"))?; - let native: std::collections::HashSet<&'static str> = - all_tools.iter().map(|t| t.name()).collect(); - let mut seen = native.clone(); - for tool in bridge.tools { - let name = tool.name(); - if seen.contains(name) { - tracing::warn!("bridged tool '{name}' collides with an existing tool; skipping"); - continue; - } - match validate_and_wrap_tool(tool) { - Ok(tool) => { - seen.insert(name); - all_tools.push(tool); - } - Err(error) => { - tracing::warn!( - "bridged tool '{name}' has an invalid contract and was skipped: {error}" - ); - } - } + let bridged_count = bridge_runtime.snapshot().tools.len(); + if !config.mcp_servers.is_empty() { + all_tools.push( + validate_and_wrap_tool(Box::new(crate::bridge::ReloadMcpTool { + runtime: bridge_runtime.clone(), + })) + .map_err(|error| anyhow::anyhow!("invalid reload MCP tool contract: {error}"))?, + ); } - let bridged_count = all_tools.len() - native.len(); - let tools = Arc::new(all_tools); if let Some(supervisor) = &agent_automation { supervisor @@ -2272,6 +2289,7 @@ pub async fn start_http_server(mut config: AppConfig) -> anyhow::Result<()> { let factory_diff_checkpoints = diff_checkpoints.clone(); let factory_artifact_egress = artifact_egress.clone(); let factory_bridged_resources = bridged_resources.clone(); + let factory_bridge_runtime = bridge_runtime.clone(); let factory_audit = audit.clone(); let factory_tool_logging = tool_logging.clone(); let factory_next_tool_call_id = next_tool_call_id.clone(); @@ -2294,6 +2312,7 @@ pub async fn start_http_server(mut config: AppConfig) -> anyhow::Result<()> { diff_checkpoints: factory_diff_checkpoints.clone(), artifact_egress: factory_artifact_egress.clone(), bridged_resources: factory_bridged_resources.clone(), + bridge_runtime: Some(factory_bridge_runtime.clone()), audit: factory_audit.clone(), tool_logging: factory_tool_logging.clone(), next_tool_call_id: factory_next_tool_call_id.clone(), @@ -2320,6 +2339,7 @@ pub async fn start_http_server(mut config: AppConfig) -> anyhow::Result<()> { .with_state(HealthState { tool_count, upstreams: upstreams.clone(), + runtime: Some(bridge_runtime.clone()), filesystem: Some(config.filesystem_worker.clone()), ready: ready.clone(), multi_tunnel: multi_availability.as_ref().map(|availability| { @@ -2385,8 +2405,8 @@ pub async fn start_http_server(mut config: AppConfig) -> anyhow::Result<()> { ); } println!( - "Tools loaded ({tool_count}): {} native + {bridged_count} upstream-facing MCP tools", - tool_count - bridged_count + "Tools loaded ({}): {tool_count} native + {bridged_count} upstream-facing MCP tools", + tool_count + bridged_count ); if !bridge_report.is_empty() { println!("Upstream MCP servers:"); @@ -3292,6 +3312,7 @@ mod tests { let tools = validate_and_wrap_tools(tools).expect("test tool contracts must be valid"); CodexHandler { config: Arc::new(config), + bridge_runtime: None, tools: Arc::new(tools), connector_schemas: Arc::new(crate::connector_schema::ConnectorSchemaStore::default()), markdown_chat: Arc::new(crate::markdown_chat::MarkdownChatStore::default()), @@ -4039,6 +4060,7 @@ mod tests { let root = tempfile::tempdir().unwrap(); let mut handler = CodexHandler { config: Arc::new(crate::config::default_config(root.path().to_path_buf())), + bridge_runtime: None, tools: Arc::new(crate::registry::load_tools()), connector_schemas: Arc::new(crate::connector_schema::ConnectorSchemaStore::default()), markdown_chat: Arc::new(crate::markdown_chat::MarkdownChatStore::default()), @@ -4158,6 +4180,7 @@ mod tests { let handler = CodexHandler { config: Arc::new(config), + bridge_runtime: None, tools: Arc::new(crate::registry::load_tools()), connector_schemas: Arc::new(crate::connector_schema::ConnectorSchemaStore::default()), markdown_chat: Arc::new(crate::markdown_chat::MarkdownChatStore::default()), @@ -4202,6 +4225,7 @@ mod tests { let authorizations = Arc::new(ConversationAuthorizationStore::new()); let handler = CodexHandler { config: Arc::new(config), + bridge_runtime: None, tools: Arc::new(tools), connector_schemas: Arc::new(crate::connector_schema::ConnectorSchemaStore::default()), markdown_chat: Arc::new(crate::markdown_chat::MarkdownChatStore::default()), @@ -4808,7 +4832,8 @@ mod tests { let ready = Arc::new(AtomicBool::new(false)); let state = HealthState { tool_count: 38, - upstreams: Arc::new(HashMap::new()), + upstreams: Arc::new(std::sync::RwLock::new(HashMap::new())), + runtime: None, filesystem: None, ready: ready.clone(), multi_tunnel: None, @@ -4833,7 +4858,8 @@ mod tests { }); let state = HealthState { tool_count: 38, - upstreams: Arc::new(HashMap::new()), + upstreams: Arc::new(std::sync::RwLock::new(HashMap::new())), + runtime: None, filesystem: None, ready, multi_tunnel: Some((availability.clone(), 2)),