From 69167b51dd2baf624182e8ba136aebc0b48092bf Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:18:56 +0100 Subject: [PATCH 01/62] feat(mcp): introduce isolated transactional upstream reload runtime --- src/bridge/hot_reload.rs | 512 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 512 insertions(+) create mode 100644 src/bridge/hot_reload.rs diff --git a/src/bridge/hot_reload.rs b/src/bridge/hot_reload.rs new file mode 100644 index 00000000..77b14cc0 --- /dev/null +++ b/src/bridge/hot_reload.rs @@ -0,0 +1,512 @@ +//! Transactional replacement of one upstream MCP connection. +//! +//! All model-visible tools are rebuilt as one immutable registry snapshot. A +//! dispatch keeps its snapshot for its entire call, so old generations cannot +//! be disposed while requests are using them. +use super::*; +use clap::Parser; +use std::collections::{HashMap, HashSet}; +use std::sync::{Arc, Mutex, RwLock}; +use tokio::sync::{Mutex as AsyncMutex, Notify}; + +const MAX_INTERNED_TOOL_NAMES: usize = 4096; +const REPLACEMENT_PING_TIMEOUT: Duration = Duration::from_secs(3); + +pub(super) struct Generation { + pub(super) name: String, + pub(super) spec: crate::types::McpServerSpec, + pub(super) peer: Peer, + pub(super) tools: Vec, + pub(super) timeout: Option, + pub(super) control: Arc, + pub(super) number: u64, + // Keep the transport alive until every snapshot/call using it is dropped. + _service: Mutex>, + _monitor: UpstreamMonitor, + calls: Mutex, + drained: Notify, +} + +pub(super) struct CallLease<'a>(&'a Generation); + +impl Drop for CallLease<'_> { + fn drop(&mut self) { + let mut calls = self.0.calls.lock().unwrap(); + *calls -= 1; + if *calls == 0 { + self.0.drained.notify_one(); + } + } +} + +impl Generation { + pub(super) fn new( + name: String, + spec: crate::types::McpServerSpec, + service: RunningService, + tools: Vec, + timeout: Option, + number: u64, + ) -> Arc { + let peer = service.peer().clone(); + let control = Arc::new(crate::upstream_control::UpstreamControl::new( + spec.request_limits.clone(), + )); + let monitor = monitor_upstream(peer.clone(), control.clone()); + Arc::new(Self { + name, + spec, + peer, + tools, + timeout, + control, + number, + _service: Mutex::new(service), + _monitor: monitor, + calls: Mutex::new(0), + drained: Notify::new(), + }) + } + + pub(super) fn begin_call(&self) -> CallLease<'_> { + *self.calls.lock().unwrap() += 1; + CallLease(self) + } + + async fn wait_for_calls(&self, limit: Duration) -> bool { + tokio::time::timeout(limit, async { + loop { + if *self.calls.lock().unwrap() == 0 { + break; + } + self.drained.notified().await; + } + }) + .await + .is_ok() + } + + fn version(&self) -> Option { + self.peer + .peer_info() + .and_then(|info| info.server_info.as_ref().map(|info| info.version.to_string())) + } +} + +struct NamePool(HashMap); + +impl NamePool { + fn intern(&mut self, name: String) -> Result<&'static str, &'static str> { + if let Some(value) = self.0.get(&name) { + return Ok(value); + } + if self.0.len() >= MAX_INTERNED_TOOL_NAMES { + return Err("TOOL_NAME_LIMIT"); + } + let leaked: &'static str = Box::leak(name.clone().into_boxed_str()); + self.0.insert(name, leaked); + Ok(leaked) + } +} + +pub(crate) struct BridgeView { + pub(crate) tools: Arc>>, + sources: HashMap>, +} + +pub(crate) struct HotReloadRuntime { + view: RwLock>, + config: Arc, + resources: Arc, + names: Mutex, + locks: HashMap>>, + pub(crate) upstreams: + Arc>>>, +} + +impl HotReloadRuntime { + pub(super) fn new( + config: Arc, + resources: Arc, + sources: HashMap>, + ) -> Arc { + let upstreams = sources + .iter() + .map(|(name, generation)| (name.clone(), generation.control.clone())) + .collect(); + let locks = config + .mcp_servers + .keys() + .map(|name| (name.clone(), Arc::new(AsyncMutex::new(())))) + .collect(); + let runtime = Arc::new(Self { + view: RwLock::new(Arc::new(BridgeView { + tools: Arc::new(Vec::new()), + sources: sources.clone(), + })), + config, + resources, + names: Mutex::new(NamePool(HashMap::new())), + locks, + upstreams: Arc::new(RwLock::new(upstreams)), + }); + match runtime.build_view(sources) { + Ok(view) => *runtime.view.write().unwrap() = Arc::new(view), + Err(code) => tracing::error!(%code, "initial MCP registry could not be built"), + } + runtime + } + + pub(crate) fn snapshot(&self) -> Arc { + self.view.read().unwrap().clone() + } + + fn build_view( + &self, + sources: HashMap>, + ) -> Result { + let mut exposed = Vec::new(); + let mut catalog = Vec::new(); + let mut names = self.names.lock().unwrap(); + let mut sorted: Vec<_> = sources.keys().collect(); + sorted.sort(); + for name in sorted { + let source = &sources[name]; + let sanitized = sanitize(name); + match source.spec.exposure() { + McpToolExposure::Gateway => { + let functions: Vec = source + .tools + .iter() + .map(|tool| GatewayFunction { + name: tool.name.to_string(), + description: tool + .description + .as_ref() + .map(ToString::to_string) + .unwrap_or_default(), + input_schema: Value::Object((*tool.input_schema).clone()), + }) + .collect(); + let tool = GatewayTool { + name: names.intern(sanitized)?, + server: name.clone(), + description: gateway_description(name, &functions), + function_names: functions.iter().map(|tool| tool.name.clone()).collect(), + peer: source.peer.clone(), + tool_timeout: source.timeout, + control: source.control.clone(), + resources: self.resources.clone(), + generation: source.clone(), + }; + exposed.push(Box::new(tool) as Box); + } + McpToolExposure::Direct => { + let mut used = HashSet::new(); + for tool in &source.tools { + let raw_name = tool.name.to_string(); + let display = + unique_name(bridged_name(&sanitized, &raw_name), &used); + used.insert(display.clone()); + let description = tool + .description + .as_ref() + .map(ToString::to_string) + .filter(|description| !description.trim().is_empty()) + .unwrap_or_else(|| { + format!("Call the '{raw_name}' tool on the '{name}' MCP server.") + }); + exposed.push(Box::new(BridgedTool { + name: names.intern(display)?, + original_name: raw_name, + server: name.clone(), + title: tool.title.clone().filter(|value| !value.trim().is_empty()), + description, + input_schema: Value::Object((*tool.input_schema).clone()), + output_schema: tool + .output_schema + .as_ref() + .map(|schema| Value::Object((**schema).clone())), + annotations: tool.annotations.clone(), + icons: tool.icons.clone(), + meta: tool.meta.clone(), + peer: source.peer.clone(), + tool_timeout: source.timeout, + control: source.control.clone(), + resources: self.resources.clone(), + generation: source.clone(), + }) as Box); + } + } + McpToolExposure::Catalog => { + catalog.push(CatalogSourceInput { + raw_name: name.clone(), + provenance: source.spec.provenance, + transport: transport_label(&source.spec), + peer_info: source.peer.peer_info(), + tools: source.tools.clone(), + peer: source.peer.clone(), + tool_timeout: source.timeout, + control: source.control.clone(), + generation: source.clone(), + }); + } + } + } + let mut tools = build_catalog_tools(catalog, self.resources.clone()); + tools.append(&mut exposed); + let tools = tools + .into_iter() + .filter_map(|tool| match crate::tool::validate_and_wrap_tool(tool) { + Ok(tool) => Some(tool), + Err(error) => { + tracing::warn!(%error, "invalid upstream MCP tool skipped"); + None + } + }) + .collect(); + Ok(BridgeView { + sources, + tools: Arc::new(tools), + }) + } + + fn latest_spec(&self, name: &str) -> Result { + // Parse the same CLI arguments as startup; this is read-only and never + // rewrites a config migration. No executable or path comes from the MCP + // request: only the operator's configured source ID is accepted. + let cli = crate::config::Cli::try_parse_from(std::env::args_os()) + .map_err(|_| "CONFIG_CLI_INVALID")?; + let config = crate::config::load_config_quiet_read_only(cli) + .map_err(|_| "CONFIG_INVALID")?; + config + .mcp_servers + .get(name) + .cloned() + .ok_or("SOURCE_REMOVED") + } + + pub(crate) async fn reload( + &self, + name: &str, + cancellation: Option<&CancellationToken>, + ) -> Result { + if name.is_empty() || name.len() > 128 || name.chars().any(char::is_control) { + return Err("SOURCE_ID_INVALID"); + } + let lock = self.locks.get(name).ok_or("SOURCE_UNKNOWN")?; + let _lease = lock.try_lock().map_err(|_| "RELOAD_IN_PROGRESS")?; + let spec = self.latest_spec(name)?; + self.reload_with_spec_locked(name, spec, cancellation).await + } + + // Injecting a spec is intentionally private to the server process. It also + // allows hermetic fault-injection tests without reading the operator config. + async fn reload_with_spec_locked( + &self, + name: &str, + spec: crate::types::McpServerSpec, + cancellation: Option<&CancellationToken>, + ) -> Result { + if spec.disabled { + return Err("SOURCE_DISABLED"); + } + if cancellation.is_some_and(CancellationToken::is_cancelled) { + return Err("CANCELLED"); + } + let previous = self.snapshot().sources.get(name).cloned(); + let old_generation = previous.as_ref().map_or(0, |source| source.number); + let future = connect_one(name, &spec, &self.config); + let (service, tools, timeout) = if let Some(token) = cancellation { + tokio::select! { + _ = token.cancelled() => return Err("CANCELLED"), + result = future => result.map_err(|_| "INITIALIZE_FAILED")?, + } + } else { + future.await.map_err(|_| "INITIALIZE_FAILED")? + }; + let ready = matches!( + tokio::time::timeout( + REPLACEMENT_PING_TIMEOUT, + service + .peer() + .send_request(ClientRequest::PingRequest(Default::default())) + ) + .await, + Ok(Ok(_) | Err(ServiceError::McpError(_))) + ); + if !ready { + return Err("READINESS_FAILED"); + } + + let added; + let removed; + let changed; + let unchanged; + { + let before: HashMap<_, _> = previous + .as_ref() + .map(|generation| &generation.tools) + .into_iter() + .flatten() + .map(|tool| (tool.name.as_ref(), serde_json::to_value(tool).unwrap_or(Value::Null))) + .collect(); + let after: HashMap<_, _> = tools + .iter() + .map(|tool| (tool.name.as_ref(), serde_json::to_value(tool).unwrap_or(Value::Null))) + .collect(); + added = after.keys().filter(|key| !before.contains_key(**key)).count(); + removed = before.keys().filter(|key| !after.contains_key(**key)).count(); + changed = after + .iter() + .filter(|(key, value)| before.get(**key).is_some_and(|old| old != *value)) + .count(); + unchanged = after + .iter() + .filter(|(key, value)| before.get(**key).is_some_and(|old| old == *value)) + .count(); + } + let new = Generation::new( + name.to_owned(), + spec, + service, + tools, + timeout, + old_generation + 1, + ); + let new_version = new.version(); + // Connection preparation and view construction are outside the write + // lock. Compare-and-rebuild prevents concurrent distinct-source reloads + // from overwriting each other. + loop { + let before = self.snapshot(); + let mut sources = before.sources.clone(); + sources.insert(name.to_owned(), new.clone()); + let candidate = Arc::new(self.build_view(sources)?); + let mut current = self.view.write().unwrap(); + if !Arc::ptr_eq(&before, ¤t) { + continue; + } + *current = candidate; + break; + } + self.upstreams + .write() + .unwrap() + .insert(name.to_owned(), new.control.clone()); + + if new.spec.exposure() == McpToolExposure::Gateway { + let funcs = new + .tools + .iter() + .map(|t| GatewayFunction { + name: t.name.to_string(), + description: t.description.as_ref().map(ToString::to_string).unwrap_or_default(), + input_schema: Value::Object((*t.input_schema).clone()), + }) + .collect::>(); + write_gateway_skill(&self.config, name, &sanitize(name), &funcs); + } + + // The old generation stays alive in any request's registry snapshot. + // On a bounded drain timeout its existing calls are NOT cancelled or + // replayed; the generation retires when the last call releases it. + let drain_ms = previous + .as_ref() + .map_or(0, |source| source.spec.request_limits.slow_cancellation_ms); + let drained = if let Some(old) = &previous { + old.wait_for_calls(Duration::from_millis(drain_ms)).await + } else { + true + }; + Ok(json!({ + "source": name, + "outcome": "reloaded", + "previousGeneration": old_generation, + "replacementGeneration": new.number, + "previousVersion": previous.as_ref().and_then(|old| old.version()), + "replacementVersion": new_version, + "ready": true, + "toolChanges": { + "added": added, "removed": removed, "changed": changed, "unchanged": unchanged + }, + "oldCallsDrained": drained, + "oldGenerationRetiring": !drained, + "rollback": "not_required", + "clientRediscoveryRequired": true, + "diagnosticCode": "OK" + })) + } +} + +pub(crate) struct ReloadMcpTool { + pub(crate) runtime: Arc, +} + +#[async_trait] +impl Tool for ReloadMcpTool { + fn name(&self) -> &'static str { + "reload_mcp" + } + + fn title(&self) -> String { + "Reload one MCP source".into() + } + + fn description(&self) -> String { + "Transactionally replace one configured upstream MCP source without restarting Codexify or reconnecting any other source. The source is its exact configured MCP server key, not an executable path. Existing calls keep the previous generation. Tool definition changes require the client to rediscover tools/list; in-chat ChatGPT schema updates are not guaranteed.".into() + } + + fn behavior(&self) -> ToolBehavior { + ToolBehavior::new( + false, + true, + false, + false, + "Process replacement is an operator action; no arbitrary executable arguments are accepted.", + ) + } + + fn input_schema(&self) -> Value { + json!({ + "type": "object", + "properties": { + "source": {"type": "string", "minLength": 1, "maxLength": 128, + "description": "Exact stable key of an existing configured MCP server"} + }, + "required": ["source"], + "additionalProperties": false + }) + } + + fn requires_project_root(&self) -> bool { + false + } + + async fn call(&self, args: Value, _config: &AppConfig, _session: &SessionState) -> ToolResult { + self.run(&args, None).await + } + + async fn call_with_context( + &self, + args: Value, + _config: &AppConfig, + _session: &SessionState, + context: &ToolRequestContext, + ) -> ToolResult { + self.run(&args, Some(&context.cancellation)).await + } +} + +impl ReloadMcpTool { + async fn run(&self, args: &Value, cancellation: Option<&CancellationToken>) -> ToolResult { + let Some(source) = args.get("source").and_then(Value::as_str) else { + return ToolResult::error("SOURCE_ID_INVALID"); + }; + match self.runtime.reload(source, cancellation).await { + Ok(report) => ToolResult::text("MCP source reloaded; rediscover tools to refresh client schemas.") + .with_structured(report), + Err(code) => ToolResult::error(format!("MCP reload did not switch connections: {code}")), + } + } +} From 7feef36064aadee73edb5a171b6f7ba05940dd51 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:19:25 +0100 Subject: [PATCH 02/62] refactor(mcp): own upstream services by immutable connection generation --- src/bridge.rs | 190 ++++++++++++-------------------------------------- 1 file changed, 46 insertions(+), 144 deletions(-) diff --git a/src/bridge.rs b/src/bridge.rs index 68bdcef4..37a32e78 100644 --- a/src/bridge.rs +++ b/src/bridge.rs @@ -36,25 +36,22 @@ use crate::tool::{Tool, ToolBehavior, ToolCallIdentity, ToolRequestContext}; use crate::types::{AppConfig, McpToolExposure, ToolContent, ToolResult}; mod stderr; +mod hot_reload; +pub(crate) use hot_reload::{HotReloadRuntime, ReloadMcpTool}; /// How long to wait for an upstream to start up and answer `tools/list` before /// giving up on it. const CONNECT_TIMEOUT: Duration = Duration::from_secs(20); -/// The result of connecting to every configured upstream: the tools to merge -/// into the registry, plus the running services kept alive for the server's -/// lifetime (dropping one closes its transport and any child process). +/// Connected MCP sources are owned by a transactional registry. Its immutable +/// view is shared across transport sessions, while calls pin their generation. pub struct Bridge { - pub tools: Vec>, - /// Shared opaque-capability store for resource links returned by upstream tools. + pub tools: Arc>>, pub resources: Arc, - /// Held only to keep upstream transports alive; never read directly. - pub services: Vec>, - /// One human-readable line per configured server (connected or failed), - /// printed in the startup banner so a bad path or handshake is never silent. pub report: Vec, - pub(crate) upstreams: Arc>>, - pub(crate) monitors: Vec, + pub(crate) runtime: Arc, + pub(crate) upstreams: + Arc>>>, } pub(crate) struct UpstreamMonitor(tokio::task::JoinHandle<()>); @@ -120,10 +117,12 @@ struct BridgedTool { tool_timeout: Option, control: Arc, resources: Arc, + generation: Arc, } impl BridgedTool { async fn run(&self, args: Value, cancellation: Option<&CancellationToken>) -> ToolResult { + let _lease = self.generation.begin_call(); let mut params = CallToolRequestParams::new(self.original_name.clone()); if let Some(obj) = args.as_object() && !obj.is_empty() @@ -518,154 +517,54 @@ fn unique_name(base: String, used: &std::collections::HashSet) -> String /// materialize its effective exposure mode. A server that fails to launch or /// answer is logged and skipped — it never blocks startup. pub async fn connect_upstreams(config: &AppConfig) -> Bridge { - let mut tools: Vec> = Vec::new(); - let mut services: Vec> = Vec::new(); - let mut upstreams = HashMap::new(); - let mut monitors = Vec::new(); - let mut report: Vec = Vec::new(); - let mut catalog_sources = Vec::new(); let resources = Arc::new(BridgedResourceStore::new(config.artifact_egress.clone())); - - // Deterministic order so logs and tool ordering are stable across runs. - let mut names: Vec<&String> = config.mcp_servers.keys().collect(); + let mut sources = HashMap::new(); + let mut report = Vec::new(); + let mut names: Vec<_> = config.mcp_servers.keys().collect(); names.sort(); - for server_name in names { - let spec = &config.mcp_servers[server_name]; + for name in names { + let spec = &config.mcp_servers[name]; if spec.disabled { - report.push(format!("{server_name} -> disabled")); + report.push(format!("{name} -> disabled")); continue; } - let sanitized = sanitize(server_name); - let exposure = spec.exposure(); - - match connect_one(server_name, spec, config).await { - Ok((service, upstream_tools, tool_timeout)) => { - let peer = service.peer().clone(); - let control = Arc::new(crate::upstream_control::UpstreamControl::new( - spec.request_limits.clone(), - )); - upstreams.insert(server_name.clone(), control.clone()); - monitors.push(monitor_upstream(peer.clone(), control.clone())); - let count = upstream_tools.len(); - - match exposure { - McpToolExposure::Gateway => { - let functions: Vec = upstream_tools - .iter() - .map(|t| GatewayFunction { - name: t.name.to_string(), - description: t - .description - .as_ref() - .map(|c| c.to_string()) - .unwrap_or_default(), - input_schema: Value::Object((*t.input_schema).clone()), - }) - .collect(); - - write_gateway_skill(config, server_name, &sanitized, &functions); - - let leaked: &'static str = Box::leak(sanitized.clone().into_boxed_str()); - tools.push(Box::new(GatewayTool { - name: leaked, - server: server_name.clone(), - description: gateway_description(server_name, &functions), - function_names: functions.iter().map(|f| f.name.clone()).collect(), - peer: peer.clone(), - tool_timeout, - control: control.clone(), - resources: resources.clone(), - })); - report.push(format!( - "{server_name} -> gateway ({count} functions via `{sanitized}`)" - )); - tracing::info!( - "bridged MCP server '{server_name}' as gateway: {count} function(s)" - ); - } - McpToolExposure::Direct => { - report.push(format!("{server_name} -> direct ({count} tool(s))")); - let mut used: std::collections::HashSet = - std::collections::HashSet::new(); - for tool in upstream_tools { - let original_name = tool.name.to_string(); - let display = - unique_name(bridged_name(&sanitized, &original_name), &used); - used.insert(display.clone()); - let leaked: &'static str = Box::leak(display.into_boxed_str()); - let title = tool.title.filter(|title| !title.trim().is_empty()); - let description = tool - .description - .map(|description| description.to_string()) - .filter(|description| !description.trim().is_empty()) - .unwrap_or_else(|| { - format!( - "Call the `{original_name}` tool on the `{server_name}` MCP server." - ) - }); - tools.push(Box::new(BridgedTool { - name: leaked, - original_name, - server: server_name.clone(), - title, - description, - input_schema: Value::Object((*tool.input_schema).clone()), - output_schema: tool - .output_schema - .map(|schema| Value::Object((*schema).clone())), - annotations: tool.annotations, - icons: tool.icons, - meta: tool.meta, - peer: peer.clone(), - tool_timeout, - control: control.clone(), - resources: resources.clone(), - })); - } - tracing::info!( - "bridged MCP server '{server_name}' directly: {count} tool(s)" - ); - } - McpToolExposure::Catalog => { - report.push(format!( - "{server_name} -> catalog ({count} private tool(s))" - )); - catalog_sources.push(CatalogSourceInput { - raw_name: server_name.clone(), - provenance: spec.provenance, - transport: transport_label(spec), - peer_info: peer.peer_info(), - tools: upstream_tools, - peer: peer.clone(), - tool_timeout, - control: control.clone(), - }); - tracing::info!( - "indexed MCP server '{server_name}' privately: {count} tool(s)" - ); - } + match connect_one(name, spec, config).await { + Ok((service, tools, timeout)) => { + let count = tools.len(); + let mode = spec.exposure(); + if mode == McpToolExposure::Gateway { + let functions = tools.iter().map(|tool| GatewayFunction { + name: tool.name.to_string(), + description: tool.description.as_ref() + .map(ToString::to_string).unwrap_or_default(), + input_schema: Value::Object((*tool.input_schema).clone()), + }).collect::>(); + write_gateway_skill(config, name, &sanitize(name), &functions); } - services.push(service); + let generation = hot_reload::Generation::new( + name.clone(), spec.clone(), service, tools, timeout, 1, + ); + sources.insert(name.clone(), generation); + report.push(format!("{name} -> {} ({count} tool(s))", mode.as_str())); + tracing::info!(source = %name, %count, exposure = mode.as_str(), "MCP source connected"); } - Err(e) => { - report.push(format!("{server_name} -> FAILED: {e}")); - tracing::warn!("skipping MCP server '{server_name}': {e}"); + Err(_) => { + // Upstream configuration can contain private header or command + // values; do not echo a failed spawn/handshake error to clients. + report.push(format!("{name} -> FAILED (initialization)")); + tracing::warn!(source = %name, "MCP source failed initialization"); } } } - let mut catalog_tools = build_catalog_tools(catalog_sources, resources.clone()); - catalog_tools.append(&mut tools); - tools = catalog_tools; - + let runtime = HotReloadRuntime::new(Arc::new(config.clone()), resources.clone(), sources); Bridge { - tools, + tools: runtime.snapshot().tools.clone(), resources, - services, report, - upstreams: Arc::new(upstreams), - monitors, + upstreams: runtime.upstreams.clone(), + runtime, } } @@ -767,10 +666,12 @@ struct GatewayTool { tool_timeout: Option, control: Arc, resources: Arc, + generation: Arc, } impl GatewayTool { async fn run(&self, args: Value, cancellation: Option<&CancellationToken>) -> ToolResult { + let _lease = self.generation.begin_call(); let Some(function) = args.get("function").and_then(|v| v.as_str()) else { return ToolResult::error("`function` is required (the name of the function to call)"); }; @@ -1133,6 +1034,7 @@ async fn connect_one_with_env( let connect = match select_transport(spec)? { UpstreamTransport::Stdio(command_path) => { let mut command = Command::new(command_path); + command.kill_on_drop(true); command.args(&spec.args); for (key, value) in &spec.env { command.env(key, value); From 6529804e3b9387ab029da039742a3ce709467982 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:19:41 +0100 Subject: [PATCH 03/62] fix(mcp): expose generation call lease within internal catalog --- src/bridge/hot_reload.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/bridge/hot_reload.rs b/src/bridge/hot_reload.rs index 77b14cc0..92bda613 100644 --- a/src/bridge/hot_reload.rs +++ b/src/bridge/hot_reload.rs @@ -12,7 +12,7 @@ use tokio::sync::{Mutex as AsyncMutex, Notify}; const MAX_INTERNED_TOOL_NAMES: usize = 4096; const REPLACEMENT_PING_TIMEOUT: Duration = Duration::from_secs(3); -pub(super) struct Generation { +pub(crate) struct Generation { pub(super) name: String, pub(super) spec: crate::types::McpServerSpec, pub(super) peer: Peer, @@ -27,7 +27,7 @@ pub(super) struct Generation { drained: Notify, } -pub(super) struct CallLease<'a>(&'a Generation); +pub(crate) struct CallLease<'a>(&'a Generation); impl Drop for CallLease<'_> { fn drop(&mut self) { @@ -68,7 +68,7 @@ impl Generation { }) } - pub(super) fn begin_call(&self) -> CallLease<'_> { + pub(crate) fn begin_call(&self) -> CallLease<'_> { *self.calls.lock().unwrap() += 1; CallLease(self) } @@ -98,7 +98,7 @@ struct NamePool(HashMap); impl NamePool { fn intern(&mut self, name: String) -> Result<&'static str, &'static str> { if let Some(value) = self.0.get(&name) { - return Ok(value); + return Ok(*value); } if self.0.len() >= MAX_INTERNED_TOOL_NAMES { return Err("TOOL_NAME_LIMIT"); From 529c728329eef7776b2d0de596707d7443cc863e Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:19:44 +0100 Subject: [PATCH 04/62] fix(mcp): share generation type with catalog router --- src/bridge.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bridge.rs b/src/bridge.rs index 37a32e78..546b7f1a 100644 --- a/src/bridge.rs +++ b/src/bridge.rs @@ -37,7 +37,7 @@ use crate::types::{AppConfig, McpToolExposure, ToolContent, ToolResult}; mod stderr; mod hot_reload; -pub(crate) use hot_reload::{HotReloadRuntime, ReloadMcpTool}; +pub(crate) use hot_reload::{Generation, HotReloadRuntime, ReloadMcpTool}; /// How long to wait for an upstream to start up and answer `tools/list` before /// giving up on it. From b82396bf82b4534adfe2133e8c0bd5e2e9d1142f Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:19:46 +0100 Subject: [PATCH 05/62] feat(mcp): pin catalog calls to their source generation --- src/mcp_catalog.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/mcp_catalog.rs b/src/mcp_catalog.rs index 2144716f..6a607424 100644 --- a/src/mcp_catalog.rs +++ b/src/mcp_catalog.rs @@ -40,6 +40,7 @@ pub(crate) struct CatalogSourceInput { pub peer: Peer, pub tool_timeout: Option, pub control: Arc, + pub generation: Arc, } struct CatalogTool { @@ -59,6 +60,7 @@ struct CatalogSource { peer: Peer, tool_timeout: Option, control: Arc, + generation: Arc, } struct SearchDocument { @@ -128,6 +130,7 @@ impl Catalog { peer: input.peer, tool_timeout: input.tool_timeout, control: input.control, + generation: input.generation, }); } @@ -1032,6 +1035,7 @@ impl McpCallToolTool { Err(error) => return ToolResult::error(error), }; + let _lease = source.generation.begin_call(); let mut params = CallToolRequestParams::new(tool.raw.name.to_string()); match args.get("arguments") { None | Some(Value::Null) => {} From 68beb01b7a7d206a8eefbcbefa59374542ec8824 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:20:40 +0100 Subject: [PATCH 06/62] feat(mcp): dispatch across atomic reloadable tool snapshots --- src/server.rs | 92 +++++++++++++++++++++++++++++---------------------- 1 file changed, 52 insertions(+), 40 deletions(-) diff --git a/src/server.rs b/src/server.rs index 456491a4..47564f55 100644 --- a/src/server.rs +++ b/src/server.rs @@ -146,6 +146,7 @@ type HttpServerTask = JoinHandle>; pub struct CodexHandler { config: Arc, tools: Arc>>, + bridge_runtime: Option>, project_bindings: Arc, connector_schemas: Arc, markdown_chat: Arc, @@ -653,11 +654,20 @@ impl ServerHandler for CodexHandler { }) .await .unwrap_or_else(|_| self.config.as_ref().clone()); - let tools = self - .tools - .iter() - .map(|tool| advertised_tool(tool.as_ref(), &advertising)) - .collect(); + let mut names = std::collections::HashSet::new(); + let mut tools = Vec::new(); + for tool in self.tools.iter() { + names.insert(tool.name()); + tools.push(advertised_tool(tool.as_ref(), &advertising)); + } + if let Some(runtime) = &self.bridge_runtime { + let view = runtime.snapshot(); + for tool in view.tools.iter() { + if names.insert(tool.name()) { + tools.push(advertised_tool(tool.as_ref(), &advertising)); + } + } + } let mut result = ListToolsResult::with_all_items(tools); let caller = crate::connector_schema::caller_key(&context.meta).or_else(|| { request @@ -826,7 +836,18 @@ impl ServerHandler for CodexHandler { .arguments .map(Value::Object) .unwrap_or_else(|| json!({})); - let tool = self.tools.iter().find(|t| t.name() == name); + // Pin a complete immutable upstream registry through the entire async + // dispatch. Hot reload cannot close this request's source generation. + let bridge_view = self.bridge_runtime.as_ref().map(|runtime| runtime.snapshot()); + let tool = self + .tools + .iter() + .find(|tool| tool.name() == name) + .or_else(|| { + bridge_view.as_ref().and_then(|view| { + view.tools.iter().find(|tool| tool.name() == name) + }) + }); let ticket_wait = name == TicketWait::NAME && tool.is_some(); let model_call = !tool.is_some_and(|tool| app_only_tool(tool.as_ref())); let mut task_conversation = conversation.clone(); @@ -2062,7 +2083,8 @@ struct HealthState { tool_count: usize, ready: Arc, multi_tunnel: Option<(Arc, usize)>, - upstreams: Arc>>, + upstreams: Arc>>>, + runtime: Option>, filesystem: Option>, } @@ -2083,17 +2105,15 @@ async fn health(State(state): State) -> (StatusCode, Json) { || state.ready.load(Ordering::Acquire), |(active, _)| *active > 0, ); + let exposed = state.runtime.as_ref().map_or(0, |runtime| runtime.snapshot().tools.len()); let mut payload = json!({ "status": if is_ready { "ok" } else { "starting" }, - "tools": state.tool_count, + "tools": state.tool_count + exposed, }); - if !state.upstreams.is_empty() { + let upstreams = state.upstreams.read().unwrap(); + if !upstreams.is_empty() { payload["upstreams"] = Value::Object( - state - .upstreams - .iter() - .map(|(name, state)| (name.clone(), state.snapshot())) - .collect(), + upstreams.iter().map(|(name, state)| (name.clone(), state.snapshot())).collect(), ); } if let Some(filesystem) = &state.filesystem { @@ -2180,35 +2200,18 @@ pub async fn start_http_server(mut config: AppConfig) -> anyhow::Result<()> { let bridge = crate::bridge::connect_upstreams(&config).await; let bridge_report = bridge.report; let bridged_resources = bridge.resources; - let _bridge_services = bridge.services; - let _bridge_monitors = bridge.monitors; let upstreams = bridge.upstreams; + let bridge_runtime = bridge.runtime; let mut all_tools = validate_and_wrap_tools(load_tools_for_config(&config)) .map_err(|error| anyhow::anyhow!("invalid native tool contract: {error}"))?; - let native: std::collections::HashSet<&'static str> = - all_tools.iter().map(|t| t.name()).collect(); - let mut seen = native.clone(); - for tool in bridge.tools { - let name = tool.name(); - if seen.contains(name) { - tracing::warn!("bridged tool '{name}' collides with an existing tool; skipping"); - continue; - } - match validate_and_wrap_tool(tool) { - Ok(tool) => { - seen.insert(name); - all_tools.push(tool); - } - Err(error) => { - tracing::warn!( - "bridged tool '{name}' has an invalid contract and was skipped: {error}" - ); - } - } + let bridged_count = bridge_runtime.snapshot().tools.len(); + if !config.mcp_servers.is_empty() { + all_tools.push(validate_and_wrap_tool(Box::new(crate::bridge::ReloadMcpTool { + runtime: bridge_runtime.clone(), + })) + .map_err(|error| anyhow::anyhow!("invalid reload MCP tool contract: {error}"))?); } - let bridged_count = all_tools.len() - native.len(); - let tools = Arc::new(all_tools); if let Some(supervisor) = &agent_automation { supervisor @@ -2272,6 +2275,7 @@ pub async fn start_http_server(mut config: AppConfig) -> anyhow::Result<()> { let factory_diff_checkpoints = diff_checkpoints.clone(); let factory_artifact_egress = artifact_egress.clone(); let factory_bridged_resources = bridged_resources.clone(); + let factory_bridge_runtime = bridge_runtime.clone(); let factory_audit = audit.clone(); let factory_tool_logging = tool_logging.clone(); let factory_next_tool_call_id = next_tool_call_id.clone(); @@ -2294,6 +2298,7 @@ pub async fn start_http_server(mut config: AppConfig) -> anyhow::Result<()> { diff_checkpoints: factory_diff_checkpoints.clone(), artifact_egress: factory_artifact_egress.clone(), bridged_resources: factory_bridged_resources.clone(), + bridge_runtime: Some(factory_bridge_runtime.clone()), audit: factory_audit.clone(), tool_logging: factory_tool_logging.clone(), next_tool_call_id: factory_next_tool_call_id.clone(), @@ -2320,6 +2325,7 @@ pub async fn start_http_server(mut config: AppConfig) -> anyhow::Result<()> { .with_state(HealthState { tool_count, upstreams: upstreams.clone(), + runtime: Some(bridge_runtime.clone()), filesystem: Some(config.filesystem_worker.clone()), ready: ready.clone(), multi_tunnel: multi_availability.as_ref().map(|availability| { @@ -3292,6 +3298,7 @@ mod tests { let tools = validate_and_wrap_tools(tools).expect("test tool contracts must be valid"); CodexHandler { config: Arc::new(config), + bridge_runtime: None, tools: Arc::new(tools), connector_schemas: Arc::new(crate::connector_schema::ConnectorSchemaStore::default()), markdown_chat: Arc::new(crate::markdown_chat::MarkdownChatStore::default()), @@ -4039,6 +4046,7 @@ mod tests { let root = tempfile::tempdir().unwrap(); let mut handler = CodexHandler { config: Arc::new(crate::config::default_config(root.path().to_path_buf())), + bridge_runtime: None, tools: Arc::new(crate::registry::load_tools()), connector_schemas: Arc::new(crate::connector_schema::ConnectorSchemaStore::default()), markdown_chat: Arc::new(crate::markdown_chat::MarkdownChatStore::default()), @@ -4158,6 +4166,7 @@ mod tests { let handler = CodexHandler { config: Arc::new(config), + bridge_runtime: None, tools: Arc::new(crate::registry::load_tools()), connector_schemas: Arc::new(crate::connector_schema::ConnectorSchemaStore::default()), markdown_chat: Arc::new(crate::markdown_chat::MarkdownChatStore::default()), @@ -4202,6 +4211,7 @@ mod tests { let authorizations = Arc::new(ConversationAuthorizationStore::new()); let handler = CodexHandler { config: Arc::new(config), + bridge_runtime: None, tools: Arc::new(tools), connector_schemas: Arc::new(crate::connector_schema::ConnectorSchemaStore::default()), markdown_chat: Arc::new(crate::markdown_chat::MarkdownChatStore::default()), @@ -4808,7 +4818,8 @@ mod tests { let ready = Arc::new(AtomicBool::new(false)); let state = HealthState { tool_count: 38, - upstreams: Arc::new(HashMap::new()), + upstreams: Arc::new(std::sync::RwLock::new(HashMap::new())), + runtime: None, filesystem: None, ready: ready.clone(), multi_tunnel: None, @@ -4833,7 +4844,8 @@ mod tests { }); let state = HealthState { tool_count: 38, - upstreams: Arc::new(HashMap::new()), + upstreams: Arc::new(std::sync::RwLock::new(HashMap::new())), + runtime: None, filesystem: None, ready, multi_tunnel: Some((availability.clone(), 2)), From 405de74e09223a654153b89e078f2a955c56cdcb Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:21:00 +0100 Subject: [PATCH 07/62] fix(mcp): count reloadable bridge tools correctly in server banner --- src/server.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server.rs b/src/server.rs index 47564f55..4e6533bf 100644 --- a/src/server.rs +++ b/src/server.rs @@ -2391,8 +2391,8 @@ pub async fn start_http_server(mut config: AppConfig) -> anyhow::Result<()> { ); } println!( - "Tools loaded ({tool_count}): {} native + {bridged_count} upstream-facing MCP tools", - tool_count - bridged_count + "Tools loaded ({}): {tool_count} native + {bridged_count} upstream-facing MCP tools", + tool_count + bridged_count ); if !bridge_report.is_empty() { println!("Upstream MCP servers:"); From bb49f57e1b2a43a3cff6e5a74f9c68129f59752e Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:23:21 +0100 Subject: [PATCH 08/62] refactor(mcp): enforce per-source reload leases and remove unused state --- src/bridge/hot_reload.rs | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/bridge/hot_reload.rs b/src/bridge/hot_reload.rs index 92bda613..0f97c213 100644 --- a/src/bridge/hot_reload.rs +++ b/src/bridge/hot_reload.rs @@ -13,7 +13,6 @@ const MAX_INTERNED_TOOL_NAMES: usize = 4096; const REPLACEMENT_PING_TIMEOUT: Duration = Duration::from_secs(3); pub(crate) struct Generation { - pub(super) name: String, pub(super) spec: crate::types::McpServerSpec, pub(super) peer: Peer, pub(super) tools: Vec, @@ -41,7 +40,6 @@ impl Drop for CallLease<'_> { impl Generation { pub(super) fn new( - name: String, spec: crate::types::McpServerSpec, service: RunningService, tools: Vec, @@ -54,7 +52,6 @@ impl Generation { )); let monitor = monitor_upstream(peer.clone(), control.clone()); Arc::new(Self { - name, spec, peer, tools, @@ -294,14 +291,24 @@ impl HotReloadRuntime { if name.is_empty() || name.len() > 128 || name.chars().any(char::is_control) { return Err("SOURCE_ID_INVALID"); } + let spec = self.latest_spec(name)?; + self.reload_with_spec(name, spec, cancellation).await + } + + // A test harness can supply an isolated spec without reloading real + // operator configuration. Production accepts source ID only and always + // resolves the current configured spec before reaching this boundary. + pub(super) async fn reload_with_spec( + &self, + name: &str, + spec: crate::types::McpServerSpec, + cancellation: Option<&CancellationToken>, + ) -> Result { let lock = self.locks.get(name).ok_or("SOURCE_UNKNOWN")?; let _lease = lock.try_lock().map_err(|_| "RELOAD_IN_PROGRESS")?; - let spec = self.latest_spec(name)?; self.reload_with_spec_locked(name, spec, cancellation).await } - // Injecting a spec is intentionally private to the server process. It also - // allows hermetic fault-injection tests without reading the operator config. async fn reload_with_spec_locked( &self, name: &str, @@ -367,7 +374,6 @@ impl HotReloadRuntime { .count(); } let new = Generation::new( - name.to_owned(), spec, service, tools, From 3ce2d61c598d27cd0ebb9d94ec9863e43261b8c3 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:23:23 +0100 Subject: [PATCH 09/62] refactor(mcp): align generation construction with minimal state --- src/bridge.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bridge.rs b/src/bridge.rs index 546b7f1a..d47c06c0 100644 --- a/src/bridge.rs +++ b/src/bridge.rs @@ -543,7 +543,7 @@ pub async fn connect_upstreams(config: &AppConfig) -> Bridge { write_gateway_skill(config, name, &sanitize(name), &functions); } let generation = hot_reload::Generation::new( - name.clone(), spec.clone(), service, tools, timeout, 1, + spec.clone(), service, tools, timeout, 1, ); sources.insert(name.clone(), generation); report.push(format!("{name} -> {} ({count} tool(s))", mode.as_str())); From 43fcde4e233cfd22388973230e06dea5673d84cc Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:24:02 +0100 Subject: [PATCH 10/62] test(mcp): exercise real upstream swapping, pinned calls, rollback and concurrency --- src/bridge/hot_reload_tests.rs | 250 +++++++++++++++++++++++++++++++++ 1 file changed, 250 insertions(+) create mode 100644 src/bridge/hot_reload_tests.rs diff --git a/src/bridge/hot_reload_tests.rs b/src/bridge/hot_reload_tests.rs new file mode 100644 index 00000000..9a80750d --- /dev/null +++ b/src/bridge/hot_reload_tests.rs @@ -0,0 +1,250 @@ +//! Isolated, real HTTP MCP fault-injection coverage for per-source reload. +use super::*; +use axum::Router; +use rmcp::{ + ErrorData as McpError, ServerHandler, + model::{ + CallToolResponse, Implementation, InitializeResult, ListToolsResult, + PaginatedRequestParams, ServerCapabilities, ServerInfo, + }, + service::{RequestContext, RoleServer}, + transport::streamable_http_server::{ + StreamableHttpServerConfig, StreamableHttpService, + session::local::LocalSessionManager, + }, +}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +#[derive(Clone)] +struct Fixture { + version: &'static str, + new_tool: bool, + list_delay: Duration, + call_delay: Duration, + started: Arc, +} + +impl ServerHandler for Fixture { + fn get_info(&self) -> ServerInfo { + InitializeResult::new(ServerCapabilities::builder().enable_tools().build()) + .with_server_info(Implementation::new("hot-reload-fixture", self.version)) + } + + async fn list_tools( + &self, + _request: Option, + _context: RequestContext, + ) -> Result { + tokio::time::sleep(self.list_delay).await; + let schema = serde_json::json!({ + "type": "object", + "properties": {}, + "additionalProperties": false + }).as_object().cloned().unwrap(); + let mut tools = vec![ + rmcp::model::Tool::new("probe", "Report this fixture's version", schema.clone()), + rmcp::model::Tool::new("slow", "Report this fixture's version after a delay", schema.clone()), + ]; + if self.new_tool { + tools.push(rmcp::model::Tool::new( + "new_tool", "Added in version two", schema + )); + } + Ok(ListToolsResult::with_all_items(tools)) + } + + async fn call_tool( + &self, + request: CallToolRequestParams, + _context: RequestContext, + ) -> Result { + self.started.fetch_add(1, Ordering::SeqCst); + if request.name.as_ref() == "slow" { + tokio::time::sleep(self.call_delay).await; + } + Ok(CallToolResult::success(vec![ContentBlock::text(self.version)]).into()) + } +} + +async fn fixture( + version: &'static str, + new_tool: bool, + list_delay: Duration, + call_delay: Duration, +) -> (String, Arc, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)).await.unwrap(); + let address = listener.local_addr().unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + let fixture = Fixture { + version, + new_tool, + list_delay, + call_delay, + started: calls.clone(), + }; + let mut transport = StreamableHttpServerConfig::default(); + transport.json_response = true; + let service = StreamableHttpService::new( + move || Ok(fixture.clone()), + Arc::new(LocalSessionManager::default()), + transport, + ); + let app = Router::new().nest_service("/mcp", service); + let task = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + (format!("http://{address}/mcp"), calls, task) +} + +fn spec(url: &str, exposure: McpToolExposure) -> crate::types::McpServerSpec { + crate::types::McpServerSpec { + url: Some(url.to_string()), + mode: Some(exposure), + ..Default::default() + } +} + +async fn invoke(view: Arc, name: &str) -> ToolResult { + let tool = view.tools.iter().find(|tool| tool.name() == name) + .unwrap_or_else(|| panic!("tool {name} missing from view")); + tool.call( + json!({}), + &crate::config::default_config(std::env::temp_dir()), + &SessionState::new(), + ).await +} + +#[tokio::test] +async fn single_source_reload_updates_tools_but_not_other_mcp() { + let (old_url, _, old_task) = + fixture("1", false, Duration::ZERO, Duration::ZERO).await; + let (new_url, _, new_task) = + fixture("2", true, Duration::ZERO, Duration::ZERO).await; + let (other_url, _, other_task) = + fixture("other", false, Duration::ZERO, Duration::ZERO).await; + let mut config = crate::config::default_config(std::env::temp_dir()); + config.mcp_servers.insert("alpha".into(), spec(&old_url, McpToolExposure::Direct)); + config.mcp_servers.insert("beta".into(), spec(&other_url, McpToolExposure::Direct)); + let bridge = connect_upstreams(&config).await; + let runtime = bridge.runtime.clone(); + let before = runtime.snapshot(); + assert_eq!(invoke(before.clone(), "alpha__probe").await.joined_text(), "1"); + assert_eq!(invoke(before.clone(), "beta__probe").await.joined_text(), "other"); + + let result = runtime.reload_with_spec( + "alpha", spec(&new_url, McpToolExposure::Direct), None + ).await.unwrap(); + assert_eq!(result["outcome"], "reloaded"); + assert_eq!(result["previousGeneration"], 1); + assert_eq!(result["replacementGeneration"], 2); + assert_eq!(result["previousVersion"], "1"); + assert_eq!(result["replacementVersion"], "2"); + assert_eq!(result["toolChanges"]["added"], 1); + assert_eq!(result["oldCallsDrained"], true); + + let after = runtime.snapshot(); + assert!(Arc::ptr_eq(&before.sources["beta"], &after.sources["beta"])); + assert!(!Arc::ptr_eq(&before.sources["alpha"], &after.sources["alpha"])); + assert_eq!(invoke(after.clone(), "alpha__probe").await.joined_text(), "2"); + assert_eq!(invoke(after.clone(), "alpha__new_tool").await.joined_text(), "2"); + assert_eq!(invoke(after, "beta__probe").await.joined_text(), "other"); + assert_eq!(invoke(before, "alpha__probe").await.joined_text(), "1"); + old_task.abort(); + new_task.abort(); + other_task.abort(); +} + +#[tokio::test] +async fn active_calls_keep_their_original_generation_without_replay() { + let (old_url, old_count, old_task) = + fixture("1", false, Duration::ZERO, Duration::from_millis(450)).await; + let (new_url, new_count, new_task) = + fixture("2", true, Duration::ZERO, Duration::ZERO).await; + let mut config = crate::config::default_config(std::env::temp_dir()); + config.mcp_servers.insert("alpha".into(), spec(&old_url, McpToolExposure::Direct)); + let runtime = connect_upstreams(&config).await.runtime; + let initial = runtime.snapshot(); + let pending = tokio::spawn(invoke(initial, "alpha__slow")); + tokio::time::timeout(Duration::from_secs(2), async { + while old_count.load(Ordering::SeqCst) == 0 { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }).await.unwrap(); + let started = std::time::Instant::now(); + let result = runtime.reload_with_spec( + "alpha", spec(&new_url, McpToolExposure::Direct), None + ).await.unwrap(); + assert_eq!(result["oldCallsDrained"], true); + assert!(started.elapsed() >= Duration::from_millis(100)); + assert_eq!(pending.await.unwrap().joined_text(), "1"); + assert_eq!(old_count.load(Ordering::SeqCst), 1); + assert_eq!(invoke(runtime.snapshot(), "alpha__probe").await.joined_text(), "2"); + assert_eq!(new_count.load(Ordering::SeqCst), 1); + old_task.abort(); + new_task.abort(); +} + +#[tokio::test] +async fn failed_replacement_and_unknown_ids_cannot_mutate_active_generation() { + let (old_url, _, old_task) = + fixture("1", false, Duration::ZERO, Duration::ZERO).await; + let mut config = crate::config::default_config(std::env::temp_dir()); + config.mcp_servers.insert("alpha".into(), spec(&old_url, McpToolExposure::Direct)); + let runtime = connect_upstreams(&config).await.runtime; + let old = runtime.snapshot(); + + assert_eq!( + runtime.reload_with_spec( + "missing", spec("http://127.0.0.1:9/mcp", McpToolExposure::Direct), None + ).await.unwrap_err(), + "SOURCE_UNKNOWN" + ); + assert_eq!( + runtime.reload_with_spec( + "alpha", spec("http://127.0.0.1:9/mcp", McpToolExposure::Direct), None + ).await.unwrap_err(), + "INITIALIZE_FAILED" + ); + let after = runtime.snapshot(); + assert!(Arc::ptr_eq(&old, &after)); + assert_eq!(invoke(after, "alpha__probe").await.joined_text(), "1"); + old_task.abort(); +} + +#[tokio::test] +async fn same_source_reload_conflicts_while_other_source_can_reload() { + let (a1, _, first_task) = + fixture("1", false, Duration::ZERO, Duration::ZERO).await; + let (a2, _, second_task) = + fixture("2", true, Duration::from_millis(300), Duration::ZERO).await; + let (b1, _, third_task) = + fixture("other1", false, Duration::ZERO, Duration::ZERO).await; + let (b2, _, fourth_task) = + fixture("other2", false, Duration::ZERO, Duration::ZERO).await; + let mut config = crate::config::default_config(std::env::temp_dir()); + config.mcp_servers.insert("alpha".into(), spec(&a1, McpToolExposure::Direct)); + config.mcp_servers.insert("beta".into(), spec(&b1, McpToolExposure::Direct)); + let runtime = connect_upstreams(&config).await.runtime; + let clone = runtime.clone(); + let initial = tokio::spawn(async move { + clone.reload_with_spec("alpha", spec(&a2, McpToolExposure::Direct), None).await + }); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!( + runtime.reload_with_spec("alpha", spec(&a1, McpToolExposure::Direct), None) + .await.unwrap_err(), + "RELOAD_IN_PROGRESS" + ); + let other = runtime.reload_with_spec( + "beta", spec(&b2, McpToolExposure::Direct), None + ).await.unwrap(); + assert_eq!(other["replacementGeneration"], 2); + assert_eq!(initial.await.unwrap().unwrap()["replacementGeneration"], 2); + let view = runtime.snapshot(); + assert_eq!(invoke(view.clone(), "alpha__probe").await.joined_text(), "2"); + assert_eq!(invoke(view, "beta__probe").await.joined_text(), "other2"); + first_task.abort(); + second_task.abort(); + third_task.abort(); + fourth_task.abort(); +} From 93ee0f861c40504f5e023ecc2661949fecbce5ca Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:24:05 +0100 Subject: [PATCH 11/62] test(mcp): enable isolated reload integration fixtures --- src/bridge.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bridge.rs b/src/bridge.rs index d47c06c0..7145e8a2 100644 --- a/src/bridge.rs +++ b/src/bridge.rs @@ -37,6 +37,8 @@ use crate::types::{AppConfig, McpToolExposure, ToolContent, ToolResult}; mod stderr; mod hot_reload; +#[cfg(test)] +mod hot_reload_tests; pub(crate) use hot_reload::{Generation, HotReloadRuntime, ReloadMcpTool}; /// How long to wait for an upstream to start up and answer `tools/list` before From 755cb026ca4a893f0ad9752afa491f006cf9c4ca Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:24:35 +0100 Subject: [PATCH 12/62] fix(mcp): make reload initialization future Send and update legacy assertions --- src/bridge.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bridge.rs b/src/bridge.rs index 7145e8a2..96de8ff7 100644 --- a/src/bridge.rs +++ b/src/bridge.rs @@ -1015,7 +1015,7 @@ async fn connect_one_with_env( server_name: &str, spec: &crate::types::McpServerSpec, config: &AppConfig, - env_lookup: &dyn Fn(&str) -> Option, + env_lookup: &(dyn Fn(&str) -> Option + Sync), ) -> Result< ( RunningService, @@ -2756,7 +2756,7 @@ mod tests { bridge.tools.is_empty(), "a failing upstream must be skipped, not fatal" ); - assert!(bridge.services.is_empty()); + assert!(bridge.runtime.snapshot().sources.is_empty()); // The failure is surfaced in the report, not swallowed. assert!(bridge.report.iter().any(|l| l.contains("bad -> FAILED"))); } From 1034a6ce42779413486582535d4928bc73cb8892 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:25:11 +0100 Subject: [PATCH 13/62] docs(mcp): document transactional reload behavior, limits and gated dogfooding --- docs/MCP_HOT_RELOAD.md | 106 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 docs/MCP_HOT_RELOAD.md diff --git a/docs/MCP_HOT_RELOAD.md b/docs/MCP_HOT_RELOAD.md new file mode 100644 index 00000000..0c0203bf --- /dev/null +++ b/docs/MCP_HOT_RELOAD.md @@ -0,0 +1,106 @@ +# Per-source MCP hot reload (operator guide) + +**Status:** experimental, opt-in through a staged build. This document describes +source behavior, not a deployed feature. Do not replace an active Codexify +service without separate approval and validation. + +## Scope + +\`reload_mcp\` reloads exactly one existing upstream source, identified by the +key in the resolved \`mcpServers\` configuration. A caller provides +\`{"source":"god_prompt_mcp"}\`; it cannot provide a command, executable, +working directory, URL, environment variables, or credentials. + +The server rereads its operator configuration (including inherited Codex MCP +sources) **read-only** using its startup CLI selection. A target that was +removed or disabled fails closed. No global service restart, tunnel reconnect, +task replacement, or unrelated source reconnection is part of this operation. + +The replacement uses the existing MCP transport implementation for **stdio** +and **Streamable HTTP**. Unsupported legacy SSE and WebSocket sources remain +unsupported; the operation never silently falls back to a process-wide restart. + +## Lifecycle and safety + +1. The scoped reload lock rejects concurrent attempts for the same source + (\`RELOAD_IN_PROGRESS\`); unrelated source reloads can progress concurrently. +2. The updated source specification is validated, started, initialized, and + queried with \`tools/list\`. A bounded MCP ping must also receive a + successful response or a protocol-level MCP error indicating a live peer. +3. The new tool definitions are validated and an immutable registry view is + constructed. The new view replaces the old view in one atomic pointer swap. + Concurrent distinct-source reloads rebuild against the latest view. +4. Calls already using an old registry snapshot stay pinned to that snapshot + and connection generation; write-capable calls are **never replayed**. + Cancellation and timeout semantics remain those of the forwarded request. +5. The old generation is released when no surviving snapshot or call retains + it. The operation reports whether active calls drained within the source's + configured \`requestLimits.slowCancellationMs\` interval. A drain timeout + does **not** kill or replay an in-progress side-effecting call. +6. On failure **before** the swap, the new candidate is dropped and the old + view is never replaced. Errors use bounded diagnostic codes and do not + echo credentials, command lines, headers, environment variables, or raw + tool-response bodies. + +For stdio, the spawned Tokio child has \`kill_on_drop\` enabled, so aborted +initializations should not leave the candidate running. A platform-specific +Windows child-lifecycle integration and repeated process-tree soak remain +required acceptance gates before production activation. + +## Result and tool discovery + +A successful result includes \`source\`, \`outcome\`, +\`previousGeneration\`, \`replacementGeneration\`, old/new reported MCP +server versions, readiness, counts of added/removed/changed/unchanged +tool definitions, \`oldCallsDrained\`, \`oldGenerationRetiring\`, +\`rollback\`, and \`diagnosticCode\`. + +The source's discovery data and forwarded-call routes change together in the +server's tool registry. Existing transport sessions can observe changed +definitions on **another \`tools/list\`** request. **An already-open ChatGPT +chat is not guaranteed to receive new tool definitions automatically.** +Its connector may need Refresh, reconnection or a new conversation before +new direct tools appear. The result explicitly sets +\`clientRediscoveryRequired: true\`. + +Catalog-mode source and tool identifiers remain generated by the existing +catalog and may change when a source's tool set changes. Re-query +\`mcp_list_sources\`, \`mcp_search_tools\`, and \`mcp_get_tool\` before calling +a new or modified tool. Gateway clients must refresh their advertised schema +and generated skill after reload. + +## Failure and boundary conditions + +- \`SOURCE_UNKNOWN\` / \`SOURCE_REMOVED\` / \`SOURCE_DISABLED\`: do not spawn + an arbitrary process or reinterpret the input as a path. +- \`RELOAD_IN_PROGRESS\`: no duplicate replacement of the same source. +- \`CONFIG_INVALID\`, \`CONFIG_CLI_INVALID\`, \`INITIALIZE_FAILED\`, + \`READINESS_FAILED\`: old healthy generation is retained. +- \`TOOL_NAME_LIMIT\`: bound on unique registered direct/gateway tool names; + the attempted registry swap is discarded. +- \`CANCELLED\`: an aborted preparation does not switch generations. +- If the old generation became unhealthy independently, a successful new + connection is preferred; the operation does not imply that the old peer can + be resurrected. +- Invalid data from upstream tool descriptions must remain untrusted. + Reload permission uses existing Codexify connector authorization and audit + boundaries. Do not expose the control plane to unauthenticated clients. + +## Verification gate + +The following must pass on the **exact candidate commit** before any +deployment: Rust formatting and compilation, strict lint, full Rust suite, +downstream HTTP MCP handshake, selected-source version and tool-diff checks, +two independent unrelated MCP sources, read/write in-flight calls, duplicate +reload and failure injection, cleanup/leak soak, and Windows native-process +termination. Reproduce an isolated old-to-new GodPrompt candidate switch +without touching the running GodPrompt instance; verify new version and +rollback. Measure switchover, drain and end-to-end request latency using a +repeatable workload. Do not report "zero downtime" without observed proof. + +**Not included:** automatic filesystem watching. It remains disabled rather +than introducing unverified debouncing and partially written config races. + +This feature does not merge the separate Windows scheduled-supervisor orphan +cleanup branch. Keep that work independent and verify its eventual integration +before any production cutover. From 09dcce8bf5c4d41a87b4e681bb51291f68a728e3 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:26:04 +0100 Subject: [PATCH 14/62] test(mcp): expose immutable source snapshot to integration assertions --- src/bridge/hot_reload.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bridge/hot_reload.rs b/src/bridge/hot_reload.rs index 0f97c213..91bda8b6 100644 --- a/src/bridge/hot_reload.rs +++ b/src/bridge/hot_reload.rs @@ -108,7 +108,7 @@ impl NamePool { pub(crate) struct BridgeView { pub(crate) tools: Arc>>, - sources: HashMap>, + pub(crate) sources: HashMap>, } pub(crate) struct HotReloadRuntime { From 62f12eb4d9670cf4de28c127a91fa0b038713648 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:27:37 +0100 Subject: [PATCH 15/62] fix(mcp): keep direct and catalog tool registry unique across sources --- src/bridge/hot_reload.rs | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/bridge/hot_reload.rs b/src/bridge/hot_reload.rs index 91bda8b6..b30cc508 100644 --- a/src/bridge/hot_reload.rs +++ b/src/bridge/hot_reload.rs @@ -252,13 +252,23 @@ impl HotReloadRuntime { } let mut tools = build_catalog_tools(catalog, self.resources.clone()); tools.append(&mut exposed); + let mut seen = HashSet::new(); let tools = tools .into_iter() - .filter_map(|tool| match crate::tool::validate_and_wrap_tool(tool) { - Ok(tool) => Some(tool), - Err(error) => { - tracing::warn!(%error, "invalid upstream MCP tool skipped"); - None + .filter_map(|tool| { + if !seen.insert(tool.name()) { + tracing::warn!( + name = tool.name(), + "duplicate upstream MCP tool name skipped" + ); + return None; + } + match crate::tool::validate_and_wrap_tool(tool) { + Ok(tool) => Some(tool), + Err(error) => { + tracing::warn!(%error, "invalid upstream MCP tool skipped"); + None + } } }) .collect(); From c3fdffc728caf155d63fa1ba80785fa71ef8bd7c Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:28:30 +0100 Subject: [PATCH 16/62] test(mcp): cover catalog/gateway rediscovery, reconnect, cancellation, repeated reload and collisions --- src/bridge/hot_reload_tests.rs | 145 ++++++++++++++++++++++++++++++++- 1 file changed, 144 insertions(+), 1 deletion(-) diff --git a/src/bridge/hot_reload_tests.rs b/src/bridge/hot_reload_tests.rs index 9a80750d..78b1b59a 100644 --- a/src/bridge/hot_reload_tests.rs +++ b/src/bridge/hot_reload_tests.rs @@ -105,10 +105,18 @@ fn spec(url: &str, exposure: McpToolExposure) -> crate::types::McpServerSpec { } async fn invoke(view: Arc, name: &str) -> ToolResult { + invoke_args(view, name, json!({})).await +} + +async fn invoke_args( + view: Arc, + name: &str, + args: Value, +) -> ToolResult { let tool = view.tools.iter().find(|tool| tool.name() == name) .unwrap_or_else(|| panic!("tool {name} missing from view")); tool.call( - json!({}), + args, &crate::config::default_config(std::env::temp_dir()), &SessionState::new(), ).await @@ -248,3 +256,138 @@ async fn same_source_reload_conflicts_while_other_source_can_reload() { third_task.abort(); fourth_task.abort(); } + +#[tokio::test] +async fn catalog_reload_refreshes_search_schema_and_call_routing() { + let (old_url, _, old_task) = + fixture("1", false, Duration::ZERO, Duration::ZERO).await; + let (new_url, _, new_task) = + fixture("2", true, Duration::ZERO, Duration::ZERO).await; + let mut config = crate::config::default_config(std::env::temp_dir()); + config.mcp_servers.insert("catalog".into(), spec(&old_url, McpToolExposure::Catalog)); + let runtime = connect_upstreams(&config).await.runtime; + let before = runtime.snapshot(); + let source = invoke(before, "mcp_list_sources").await.structured_content.unwrap(); + assert_eq!(source["sources"][0]["toolCount"], 2); + let result = runtime.reload_with_spec( + "catalog", spec(&new_url, McpToolExposure::Catalog), None + ).await.unwrap(); + assert_eq!(result["toolChanges"]["added"], 1); + let after = runtime.snapshot(); + let list = invoke(after.clone(), "mcp_list_sources").await.structured_content.unwrap(); + assert_eq!(list["sources"][0]["toolCount"], 3); + let source_id = list["sources"][0]["id"].as_str().unwrap(); + let found = invoke_args( + after.clone(), "mcp_search_tools", json!({"query":"new_tool","source":source_id}) + ).await.structured_content.unwrap(); + assert!(found["total"].as_u64().unwrap_or(0) >= 1); + let tool_id = found["matches"][0]["toolId"].as_str().unwrap(); + let call = invoke_args( + after, "mcp_call_tool", + json!({"source":source_id,"tool":tool_id,"arguments":{}}) + ).await; + assert!(!call.is_error); + assert_eq!(call.joined_text(), "2"); + old_task.abort(); + new_task.abort(); +} + +#[tokio::test] +async fn gateway_reload_reflects_added_functions() { + let (old_url, _, old_task) = + fixture("1", false, Duration::ZERO, Duration::ZERO).await; + let (new_url, _, new_task) = + fixture("2", true, Duration::ZERO, Duration::ZERO).await; + let mut config = crate::config::default_config(std::env::temp_dir()); + config.mcp_servers.insert("gateway".into(), spec(&old_url, McpToolExposure::Gateway)); + let runtime = connect_upstreams(&config).await.runtime; + let before = runtime.snapshot(); + let old = invoke_args( + before, "gateway", json!({"function":"probe","arguments":{}}) + ).await; + assert_eq!(old.joined_text(), "1"); + runtime.reload_with_spec( + "gateway", spec(&new_url, McpToolExposure::Gateway), None + ).await.unwrap(); + let after = runtime.snapshot(); + let tool = after.tools.iter().find(|tool| tool.name() == "gateway").unwrap(); + assert!( + tool.input_schema()["properties"]["function"]["enum"] + .as_array().unwrap().contains(&json!("new_tool")) + ); + let new = invoke_args( + after, "gateway", json!({"function":"new_tool","arguments":{}}) + ).await; + assert_eq!(new.joined_text(), "2"); + old_task.abort(); + new_task.abort(); +} + +#[tokio::test] +async fn reconnect_failed_initial_source_and_cancel_without_switching() { + let (url, _, task) = fixture("recovered", false, Duration::ZERO, Duration::ZERO).await; + let mut config = crate::config::default_config(std::env::temp_dir()); + config.mcp_servers.insert( + "unavailable".into(), crate::types::McpServerSpec { + url: Some("http://127.0.0.1:9/mcp".into()), + startup_timeout_sec: Some(0.5), + ..Default::default() + } + ); + let runtime = connect_upstreams(&config).await.runtime; + assert!(!runtime.snapshot().sources.contains_key("unavailable")); + let result = runtime.reload_with_spec( + "unavailable", spec(&url, McpToolExposure::Direct), None + ).await.unwrap(); + assert_eq!(result["previousGeneration"], 0); + assert_eq!(result["replacementGeneration"], 1); + assert_eq!(invoke(runtime.snapshot(), "unavailable__probe").await.joined_text(), "recovered"); + + let old = runtime.snapshot(); + let cancelled = CancellationToken::new(); + cancelled.cancel(); + assert_eq!( + runtime.reload_with_spec( + "unavailable", spec(&url, McpToolExposure::Direct), Some(&cancelled) + ).await.unwrap_err(), "CANCELLED" + ); + assert!(Arc::ptr_eq(&old, &runtime.snapshot())); + task.abort(); +} + +#[tokio::test] +async fn repeated_reloads_keep_one_active_generation_per_source() { + let (url, _, task) = fixture("stable", false, Duration::ZERO, Duration::ZERO).await; + let mut config = crate::config::default_config(std::env::temp_dir()); + config.mcp_servers.insert("alpha".into(), spec(&url, McpToolExposure::Direct)); + let runtime = connect_upstreams(&config).await.runtime; + for expected in 2..=6 { + let result = runtime.reload_with_spec( + "alpha", spec(&url, McpToolExposure::Direct), None + ).await.unwrap(); + assert_eq!(result["replacementGeneration"], expected); + assert_eq!(result["oldCallsDrained"], true); + let view = runtime.snapshot(); + assert_eq!(view.sources.len(), 1); + assert_eq!(invoke(view, "alpha__probe").await.joined_text(), "stable"); + } + task.abort(); +} + +#[tokio::test] +async fn direct_name_collisions_are_not_ambiguous_after_reload() { + let (url1, _, task1) = fixture("one", false, Duration::ZERO, Duration::ZERO).await; + let (url2, _, task2) = fixture("two", true, Duration::ZERO, Duration::ZERO).await; + let mut config = crate::config::default_config(std::env::temp_dir()); + // sanitize("dup!")==sanitize("dup?"), so both would generate dup___probe. + config.mcp_servers.insert("dup!".into(), spec(&url1, McpToolExposure::Direct)); + config.mcp_servers.insert("dup?".into(), spec(&url2, McpToolExposure::Direct)); + let runtime = connect_upstreams(&config).await.runtime; + let view = runtime.snapshot(); + let names = view.tools.iter().map(|tool| tool.name()).collect::>(); + let unique = names.iter().copied().collect::>(); + assert_eq!(unique.len(), names.len()); + assert_eq!(names.iter().filter(|name| **name == "dup___probe").count(), 1); + task1.abort(); + task2.abort(); +} From 138abc1536649246a9ec89af29c9160858ca64ce Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:29:30 +0100 Subject: [PATCH 17/62] style(mcp): apply rustfmt fixes confirmed by CI (src/bridge/hot_reload.rs) --- src/bridge/hot_reload.rs | 55 +++++++++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/src/bridge/hot_reload.rs b/src/bridge/hot_reload.rs index b30cc508..01518355 100644 --- a/src/bridge/hot_reload.rs +++ b/src/bridge/hot_reload.rs @@ -84,9 +84,11 @@ impl Generation { } fn version(&self) -> Option { - self.peer - .peer_info() - .and_then(|info| info.server_info.as_ref().map(|info| info.version.to_string())) + self.peer.peer_info().and_then(|info| { + info.server_info + .as_ref() + .map(|info| info.version.to_string()) + }) } } @@ -202,8 +204,7 @@ impl HotReloadRuntime { let mut used = HashSet::new(); for tool in &source.tools { let raw_name = tool.name.to_string(); - let display = - unique_name(bridged_name(&sanitized, &raw_name), &used); + let display = unique_name(bridged_name(&sanitized, &raw_name), &used); used.insert(display.clone()); let description = tool .description @@ -284,8 +285,8 @@ impl HotReloadRuntime { // request: only the operator's configured source ID is accepted. let cli = crate::config::Cli::try_parse_from(std::env::args_os()) .map_err(|_| "CONFIG_CLI_INVALID")?; - let config = crate::config::load_config_quiet_read_only(cli) - .map_err(|_| "CONFIG_INVALID")?; + let config = + crate::config::load_config_quiet_read_only(cli).map_err(|_| "CONFIG_INVALID")?; config .mcp_servers .get(name) @@ -366,14 +367,30 @@ impl HotReloadRuntime { .map(|generation| &generation.tools) .into_iter() .flatten() - .map(|tool| (tool.name.as_ref(), serde_json::to_value(tool).unwrap_or(Value::Null))) + .map(|tool| { + ( + tool.name.as_ref(), + serde_json::to_value(tool).unwrap_or(Value::Null), + ) + }) .collect(); let after: HashMap<_, _> = tools .iter() - .map(|tool| (tool.name.as_ref(), serde_json::to_value(tool).unwrap_or(Value::Null))) + .map(|tool| { + ( + tool.name.as_ref(), + serde_json::to_value(tool).unwrap_or(Value::Null), + ) + }) .collect(); - added = after.keys().filter(|key| !before.contains_key(**key)).count(); - removed = before.keys().filter(|key| !after.contains_key(**key)).count(); + added = after + .keys() + .filter(|key| !before.contains_key(**key)) + .count(); + removed = before + .keys() + .filter(|key| !after.contains_key(**key)) + .count(); changed = after .iter() .filter(|(key, value)| before.get(**key).is_some_and(|old| old != *value)) @@ -417,7 +434,11 @@ impl HotReloadRuntime { .iter() .map(|t| GatewayFunction { name: t.name.to_string(), - description: t.description.as_ref().map(ToString::to_string).unwrap_or_default(), + description: t + .description + .as_ref() + .map(ToString::to_string) + .unwrap_or_default(), input_schema: Value::Object((*t.input_schema).clone()), }) .collect::>(); @@ -520,9 +541,13 @@ impl ReloadMcpTool { return ToolResult::error("SOURCE_ID_INVALID"); }; match self.runtime.reload(source, cancellation).await { - Ok(report) => ToolResult::text("MCP source reloaded; rediscover tools to refresh client schemas.") - .with_structured(report), - Err(code) => ToolResult::error(format!("MCP reload did not switch connections: {code}")), + Ok(report) => { + ToolResult::text("MCP source reloaded; rediscover tools to refresh client schemas.") + .with_structured(report) + } + Err(code) => { + ToolResult::error(format!("MCP reload did not switch connections: {code}")) + } } } } From cc4dcd4156dd76079874b30539d66fe345e92603 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:29:32 +0100 Subject: [PATCH 18/62] style(mcp): apply rustfmt fixes confirmed by CI (src/bridge.rs) --- src/bridge.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/bridge.rs b/src/bridge.rs index 96de8ff7..1d1d111a 100644 --- a/src/bridge.rs +++ b/src/bridge.rs @@ -536,12 +536,18 @@ pub async fn connect_upstreams(config: &AppConfig) -> Bridge { let count = tools.len(); let mode = spec.exposure(); if mode == McpToolExposure::Gateway { - let functions = tools.iter().map(|tool| GatewayFunction { - name: tool.name.to_string(), - description: tool.description.as_ref() - .map(ToString::to_string).unwrap_or_default(), - input_schema: Value::Object((*tool.input_schema).clone()), - }).collect::>(); + let functions = tools + .iter() + .map(|tool| GatewayFunction { + name: tool.name.to_string(), + description: tool + .description + .as_ref() + .map(ToString::to_string) + .unwrap_or_default(), + input_schema: Value::Object((*tool.input_schema).clone()), + }) + .collect::>(); write_gateway_skill(config, name, &sanitize(name), &functions); } let generation = hot_reload::Generation::new( From 8f0ee099e238d4a826bd469808cea5b206a6b679 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:29:35 +0100 Subject: [PATCH 19/62] style(mcp): apply rustfmt fixes confirmed by CI (src/server.rs) --- src/server.rs | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/src/server.rs b/src/server.rs index 4e6533bf..164f696b 100644 --- a/src/server.rs +++ b/src/server.rs @@ -838,15 +838,18 @@ impl ServerHandler for CodexHandler { .unwrap_or_else(|| json!({})); // Pin a complete immutable upstream registry through the entire async // dispatch. Hot reload cannot close this request's source generation. - let bridge_view = self.bridge_runtime.as_ref().map(|runtime| runtime.snapshot()); + let bridge_view = self + .bridge_runtime + .as_ref() + .map(|runtime| runtime.snapshot()); let tool = self .tools .iter() .find(|tool| tool.name() == name) .or_else(|| { - bridge_view.as_ref().and_then(|view| { - view.tools.iter().find(|tool| tool.name() == name) - }) + bridge_view + .as_ref() + .and_then(|view| view.tools.iter().find(|tool| tool.name() == name)) }); let ticket_wait = name == TicketWait::NAME && tool.is_some(); let model_call = !tool.is_some_and(|tool| app_only_tool(tool.as_ref())); @@ -2083,7 +2086,8 @@ struct HealthState { tool_count: usize, ready: Arc, multi_tunnel: Option<(Arc, usize)>, - upstreams: Arc>>>, + upstreams: + Arc>>>, runtime: Option>, filesystem: Option>, } @@ -2105,7 +2109,10 @@ async fn health(State(state): State) -> (StatusCode, Json) { || state.ready.load(Ordering::Acquire), |(active, _)| *active > 0, ); - let exposed = state.runtime.as_ref().map_or(0, |runtime| runtime.snapshot().tools.len()); + let exposed = state + .runtime + .as_ref() + .map_or(0, |runtime| runtime.snapshot().tools.len()); let mut payload = json!({ "status": if is_ready { "ok" } else { "starting" }, "tools": state.tool_count + exposed, @@ -2113,7 +2120,10 @@ async fn health(State(state): State) -> (StatusCode, Json) { let upstreams = state.upstreams.read().unwrap(); if !upstreams.is_empty() { payload["upstreams"] = Value::Object( - upstreams.iter().map(|(name, state)| (name.clone(), state.snapshot())).collect(), + upstreams + .iter() + .map(|(name, state)| (name.clone(), state.snapshot())) + .collect(), ); } if let Some(filesystem) = &state.filesystem { @@ -2207,10 +2217,12 @@ pub async fn start_http_server(mut config: AppConfig) -> anyhow::Result<()> { .map_err(|error| anyhow::anyhow!("invalid native tool contract: {error}"))?; let bridged_count = bridge_runtime.snapshot().tools.len(); if !config.mcp_servers.is_empty() { - all_tools.push(validate_and_wrap_tool(Box::new(crate::bridge::ReloadMcpTool { - runtime: bridge_runtime.clone(), - })) - .map_err(|error| anyhow::anyhow!("invalid reload MCP tool contract: {error}"))?); + all_tools.push( + validate_and_wrap_tool(Box::new(crate::bridge::ReloadMcpTool { + runtime: bridge_runtime.clone(), + })) + .map_err(|error| anyhow::anyhow!("invalid reload MCP tool contract: {error}"))?, + ); } let tools = Arc::new(all_tools); if let Some(supervisor) = &agent_automation { From 9c48bc8db3d6d2a2d1e3318ff84072a5c83960fb Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:30:42 +0100 Subject: [PATCH 20/62] fix(mcp): return structured failure and retained-generation diagnostics --- src/bridge/hot_reload.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/bridge/hot_reload.rs b/src/bridge/hot_reload.rs index 01518355..2cfa4d5c 100644 --- a/src/bridge/hot_reload.rs +++ b/src/bridge/hot_reload.rs @@ -302,6 +302,9 @@ impl HotReloadRuntime { if name.is_empty() || name.len() > 128 || name.chars().any(char::is_control) { return Err("SOURCE_ID_INVALID"); } + if !self.locks.contains_key(name) { + return Err("SOURCE_UNKNOWN"); + } let spec = self.latest_spec(name)?; self.reload_with_spec(name, spec, cancellation).await } @@ -546,7 +549,18 @@ impl ReloadMcpTool { .with_structured(report) } Err(code) => { + let old = self.runtime.snapshot().sources.get(source).cloned(); + let report = json!({ + "source": source, + "outcome": "failed_without_switch", + "previousGeneration": old.as_ref().map(|generation| generation.number), + "previousVersion": old.as_ref().and_then(|generation| generation.version()), + "rollback": "previous_registry_retained", + "clientRediscoveryRequired": false, + "diagnosticCode": code + }); ToolResult::error(format!("MCP reload did not switch connections: {code}")) + .with_structured(report) } } } From cfbc29526a3ceb5864e255b9c4c159d8670d8400 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:30:50 +0100 Subject: [PATCH 21/62] docs(mcp): correct operator guide Markdown code spans --- docs/MCP_HOT_RELOAD.md | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/docs/MCP_HOT_RELOAD.md b/docs/MCP_HOT_RELOAD.md index 0c0203bf..952001d1 100644 --- a/docs/MCP_HOT_RELOAD.md +++ b/docs/MCP_HOT_RELOAD.md @@ -6,9 +6,9 @@ service without separate approval and validation. ## Scope -\`reload_mcp\` reloads exactly one existing upstream source, identified by the -key in the resolved \`mcpServers\` configuration. A caller provides -\`{"source":"god_prompt_mcp"}\`; it cannot provide a command, executable, +`reload_mcp` reloads exactly one existing upstream source, identified by the +key in the resolved `mcpServers` configuration. A caller provides +`{"source":"god_prompt_mcp"}`; it cannot provide a command, executable, working directory, URL, environment variables, or credentials. The server rereads its operator configuration (including inherited Codex MCP @@ -23,9 +23,9 @@ unsupported; the operation never silently falls back to a process-wide restart. ## Lifecycle and safety 1. The scoped reload lock rejects concurrent attempts for the same source - (\`RELOAD_IN_PROGRESS\`); unrelated source reloads can progress concurrently. + (`RELOAD_IN_PROGRESS`); unrelated source reloads can progress concurrently. 2. The updated source specification is validated, started, initialized, and - queried with \`tools/list\`. A bounded MCP ping must also receive a + queried with `tools/list`. A bounded MCP ping must also receive a successful response or a protocol-level MCP error indicating a live peer. 3. The new tool definitions are validated and an immutable registry view is constructed. The new view replaces the old view in one atomic pointer swap. @@ -35,50 +35,50 @@ unsupported; the operation never silently falls back to a process-wide restart. Cancellation and timeout semantics remain those of the forwarded request. 5. The old generation is released when no surviving snapshot or call retains it. The operation reports whether active calls drained within the source's - configured \`requestLimits.slowCancellationMs\` interval. A drain timeout + configured `requestLimits.slowCancellationMs` interval. A drain timeout does **not** kill or replay an in-progress side-effecting call. 6. On failure **before** the swap, the new candidate is dropped and the old view is never replaced. Errors use bounded diagnostic codes and do not echo credentials, command lines, headers, environment variables, or raw tool-response bodies. -For stdio, the spawned Tokio child has \`kill_on_drop\` enabled, so aborted +For stdio, the spawned Tokio child has `kill_on_drop` enabled, so aborted initializations should not leave the candidate running. A platform-specific Windows child-lifecycle integration and repeated process-tree soak remain required acceptance gates before production activation. ## Result and tool discovery -A successful result includes \`source\`, \`outcome\`, -\`previousGeneration\`, \`replacementGeneration\`, old/new reported MCP +A successful result includes `source`, `outcome`, +`previousGeneration`, `replacementGeneration`, old/new reported MCP server versions, readiness, counts of added/removed/changed/unchanged -tool definitions, \`oldCallsDrained\`, \`oldGenerationRetiring\`, -\`rollback\`, and \`diagnosticCode\`. +tool definitions, `oldCallsDrained`, `oldGenerationRetiring`, +`rollback`, and `diagnosticCode`. The source's discovery data and forwarded-call routes change together in the server's tool registry. Existing transport sessions can observe changed -definitions on **another \`tools/list\`** request. **An already-open ChatGPT +definitions on **another `tools/list`** request. **An already-open ChatGPT chat is not guaranteed to receive new tool definitions automatically.** Its connector may need Refresh, reconnection or a new conversation before new direct tools appear. The result explicitly sets -\`clientRediscoveryRequired: true\`. +`clientRediscoveryRequired: true`. Catalog-mode source and tool identifiers remain generated by the existing catalog and may change when a source's tool set changes. Re-query -\`mcp_list_sources\`, \`mcp_search_tools\`, and \`mcp_get_tool\` before calling +`mcp_list_sources`, `mcp_search_tools`, and `mcp_get_tool` before calling a new or modified tool. Gateway clients must refresh their advertised schema and generated skill after reload. ## Failure and boundary conditions -- \`SOURCE_UNKNOWN\` / \`SOURCE_REMOVED\` / \`SOURCE_DISABLED\`: do not spawn +- `SOURCE_UNKNOWN` / `SOURCE_REMOVED` / `SOURCE_DISABLED`: do not spawn an arbitrary process or reinterpret the input as a path. -- \`RELOAD_IN_PROGRESS\`: no duplicate replacement of the same source. -- \`CONFIG_INVALID\`, \`CONFIG_CLI_INVALID\`, \`INITIALIZE_FAILED\`, - \`READINESS_FAILED\`: old healthy generation is retained. -- \`TOOL_NAME_LIMIT\`: bound on unique registered direct/gateway tool names; +- `RELOAD_IN_PROGRESS`: no duplicate replacement of the same source. +- `CONFIG_INVALID`, `CONFIG_CLI_INVALID`, `INITIALIZE_FAILED`, + `READINESS_FAILED`: old healthy generation is retained. +- `TOOL_NAME_LIMIT`: bound on unique registered direct/gateway tool names; the attempted registry swap is discarded. -- \`CANCELLED\`: an aborted preparation does not switch generations. +- `CANCELLED`: an aborted preparation does not switch generations. - If the old generation became unhealthy independently, a successful new connection is preferred; the operation does not imply that the old peer can be resurrected. From 8281331880a8be2ca9d2a1ea195d9060005e44fc Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:32:58 +0100 Subject: [PATCH 22/62] feat(mcp): expose reload phase timings and remaining in-flight calls --- src/bridge/hot_reload.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/bridge/hot_reload.rs b/src/bridge/hot_reload.rs index 2cfa4d5c..30f03399 100644 --- a/src/bridge/hot_reload.rs +++ b/src/bridge/hot_reload.rs @@ -335,6 +335,7 @@ impl HotReloadRuntime { if cancellation.is_some_and(CancellationToken::is_cancelled) { return Err("CANCELLED"); } + let started = std::time::Instant::now(); let previous = self.snapshot().sources.get(name).cloned(); let old_generation = previous.as_ref().map_or(0, |source| source.number); let future = connect_one(name, &spec, &self.config); @@ -403,6 +404,7 @@ impl HotReloadRuntime { .filter(|(key, value)| before.get(**key).is_some_and(|old| old == *value)) .count(); } + let initialization_ms = started.elapsed().as_millis() as u64; let new = Generation::new( spec, service, @@ -414,6 +416,7 @@ impl HotReloadRuntime { // Connection preparation and view construction are outside the write // lock. Compare-and-rebuild prevents concurrent distinct-source reloads // from overwriting each other. + let switching = std::time::Instant::now(); loop { let before = self.snapshot(); let mut sources = before.sources.clone(); @@ -430,6 +433,7 @@ impl HotReloadRuntime { .write() .unwrap() .insert(name.to_owned(), new.control.clone()); + let switching_ms = switching.elapsed().as_millis() as u64; if new.spec.exposure() == McpToolExposure::Gateway { let funcs = new @@ -454,11 +458,15 @@ impl HotReloadRuntime { let drain_ms = previous .as_ref() .map_or(0, |source| source.spec.request_limits.slow_cancellation_ms); + let draining = std::time::Instant::now(); let drained = if let Some(old) = &previous { old.wait_for_calls(Duration::from_millis(drain_ms)).await } else { true }; + let in_flight = previous + .as_ref() + .map_or(0, |old| *old.calls.lock().unwrap()); Ok(json!({ "source": name, "outcome": "reloaded", @@ -471,8 +479,15 @@ impl HotReloadRuntime { "added": added, "removed": removed, "changed": changed, "unchanged": unchanged }, "oldCallsDrained": drained, + "oldCallsStillRunning": in_flight, "oldGenerationRetiring": !drained, "rollback": "not_required", + "timingsMs": { + "initializationAndReadiness": initialization_ms, + "registrySwitch": switching_ms, + "drain": draining.elapsed().as_millis() as u64, + "total": started.elapsed().as_millis() as u64 + }, "clientRediscoveryRequired": true, "diagnosticCode": "OK" })) From 9c5367eaea74c057bd2b12308e13dbe5f857f9fc Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:33:53 +0100 Subject: [PATCH 23/62] refactor(mcp): use one conversion for gateway function descriptors --- src/bridge.rs | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/src/bridge.rs b/src/bridge.rs index 1d1d111a..557dcea7 100644 --- a/src/bridge.rs +++ b/src/bridge.rs @@ -536,18 +536,7 @@ pub async fn connect_upstreams(config: &AppConfig) -> Bridge { let count = tools.len(); let mode = spec.exposure(); if mode == McpToolExposure::Gateway { - let functions = tools - .iter() - .map(|tool| GatewayFunction { - name: tool.name.to_string(), - description: tool - .description - .as_ref() - .map(ToString::to_string) - .unwrap_or_default(), - input_schema: Value::Object((*tool.input_schema).clone()), - }) - .collect::>(); + let functions = gateway_functions(&tools); write_gateway_skill(config, name, &sanitize(name), &functions); } let generation = hot_reload::Generation::new( @@ -576,6 +565,23 @@ pub async fn connect_upstreams(config: &AppConfig) -> Bridge { } } +/// Use one conversion for startup registration, registry replacement and +/// generated gateway skills, so schema/description metadata cannot diverge. +fn gateway_functions(tools: &[rmcp::model::Tool]) -> Vec { + tools + .iter() + .map(|tool| GatewayFunction { + name: tool.name.to_string(), + description: tool + .description + .as_ref() + .map(ToString::to_string) + .unwrap_or_default(), + input_schema: Value::Object((*tool.input_schema).clone()), + }) + .collect() +} + /// One function exposed by a gateway-mode server. struct GatewayFunction { name: String, From 8987a6c25d739c15ec6d0d17a65391de76fd9477 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:33:55 +0100 Subject: [PATCH 24/62] refactor(mcp): reuse gateway metadata conversion on reload --- src/bridge/hot_reload.rs | 28 ++-------------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/src/bridge/hot_reload.rs b/src/bridge/hot_reload.rs index 30f03399..517bef1a 100644 --- a/src/bridge/hot_reload.rs +++ b/src/bridge/hot_reload.rs @@ -174,19 +174,7 @@ impl HotReloadRuntime { let sanitized = sanitize(name); match source.spec.exposure() { McpToolExposure::Gateway => { - let functions: Vec = source - .tools - .iter() - .map(|tool| GatewayFunction { - name: tool.name.to_string(), - description: tool - .description - .as_ref() - .map(ToString::to_string) - .unwrap_or_default(), - input_schema: Value::Object((*tool.input_schema).clone()), - }) - .collect(); + let functions = gateway_functions(&source.tools); let tool = GatewayTool { name: names.intern(sanitized)?, server: name.clone(), @@ -436,19 +424,7 @@ impl HotReloadRuntime { let switching_ms = switching.elapsed().as_millis() as u64; if new.spec.exposure() == McpToolExposure::Gateway { - let funcs = new - .tools - .iter() - .map(|t| GatewayFunction { - name: t.name.to_string(), - description: t - .description - .as_ref() - .map(ToString::to_string) - .unwrap_or_default(), - input_schema: Value::Object((*t.input_schema).clone()), - }) - .collect::>(); + let funcs = gateway_functions(&new.tools); write_gateway_skill(&self.config, name, &sanitize(name), &funcs); } From f10b5040bc6eee164ccd3aac1776b6da2ac100b6 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:35:00 +0100 Subject: [PATCH 25/62] fix(mcp): honor cancellation after readiness but before registry publication --- src/bridge/hot_reload.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/bridge/hot_reload.rs b/src/bridge/hot_reload.rs index 517bef1a..844a4248 100644 --- a/src/bridge/hot_reload.rs +++ b/src/bridge/hot_reload.rs @@ -404,6 +404,9 @@ impl HotReloadRuntime { // Connection preparation and view construction are outside the write // lock. Compare-and-rebuild prevents concurrent distinct-source reloads // from overwriting each other. + if cancellation.is_some_and(CancellationToken::is_cancelled) { + return Err("CANCELLED"); + } let switching = std::time::Instant::now(); loop { let before = self.snapshot(); From f3d74c17a7628131c9c4d736b510c0064dd9246c Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:35:35 +0100 Subject: [PATCH 26/62] fix(mcp): release startup bridge tool snapshot to retire old processes --- src/server.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/server.rs b/src/server.rs index 164f696b..b5b5df4c 100644 --- a/src/server.rs +++ b/src/server.rs @@ -2212,6 +2212,10 @@ pub async fn start_http_server(mut config: AppConfig) -> anyhow::Result<()> { let bridged_resources = bridge.resources; let upstreams = bridge.upstreams; let bridge_runtime = bridge.runtime; + // The test-facing startup tool vector holds references to generation 1. + // Drop it immediately: only the runtime's replaceable snapshot may retain + // active sources, or retired child processes would remain pinned forever. + drop(bridge.tools); let mut all_tools = validate_and_wrap_tools(load_tools_for_config(&config)) .map_err(|error| anyhow::anyhow!("invalid native tool contract: {error}"))?; From 63d325779db0c99798df030a60f0a6e65f9ec4c6 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:35:48 +0100 Subject: [PATCH 27/62] test(mcp): assert drained generations and services are not retained --- src/bridge/hot_reload_tests.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/bridge/hot_reload_tests.rs b/src/bridge/hot_reload_tests.rs index 78b1b59a..65c5e0ef 100644 --- a/src/bridge/hot_reload_tests.rs +++ b/src/bridge/hot_reload_tests.rs @@ -391,3 +391,30 @@ async fn direct_name_collisions_are_not_ambiguous_after_reload() { task1.abort(); task2.abort(); } + +#[tokio::test] +async fn old_generation_is_released_after_draining_without_stale_snapshots() { + let (old_url, _, old_task) = + fixture("old", false, Duration::ZERO, Duration::ZERO).await; + let (new_url, _, new_task) = + fixture("new", true, Duration::ZERO, Duration::ZERO).await; + let mut config = crate::config::default_config(std::env::temp_dir()); + config.mcp_servers.insert("alpha".into(), spec(&old_url, McpToolExposure::Direct)); + let bridge = connect_upstreams(&config).await; + let runtime = bridge.runtime.clone(); + let first = runtime.snapshot(); + let weak = Arc::downgrade(&first.sources["alpha"]); + drop(first); + // The service keeps only the runtime after consuming the startup report. + drop(bridge); + assert!(weak.upgrade().is_some()); + + let result = runtime.reload_with_spec( + "alpha", spec(&new_url, McpToolExposure::Direct), None + ).await.unwrap(); + assert_eq!(result["oldCallsDrained"], true); + assert!(weak.upgrade().is_none(), "retired service generation leaked"); + assert_eq!(invoke(runtime.snapshot(), "alpha__probe").await.joined_text(), "new"); + old_task.abort(); + new_task.abort(); +} From abd069772f16963443585bc53aceb88dc5305737 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:36:35 +0100 Subject: [PATCH 28/62] docs(mcp): record phase timings and cross-generation resource-link caveat --- docs/MCP_HOT_RELOAD.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/MCP_HOT_RELOAD.md b/docs/MCP_HOT_RELOAD.md index 952001d1..913f8fac 100644 --- a/docs/MCP_HOT_RELOAD.md +++ b/docs/MCP_HOT_RELOAD.md @@ -52,8 +52,10 @@ required acceptance gates before production activation. A successful result includes `source`, `outcome`, `previousGeneration`, `replacementGeneration`, old/new reported MCP server versions, readiness, counts of added/removed/changed/unchanged -tool definitions, `oldCallsDrained`, `oldGenerationRetiring`, -`rollback`, and `diagnosticCode`. +tool definitions, `oldCallsDrained`, `oldCallsStillRunning`, +`oldGenerationRetiring`, `rollback`, `diagnosticCode`, and measured +`timingsMs` for initialization/readiness, registry switch, draining and total +reload duration. The source's discovery data and forwarded-call routes change together in the server's tool registry. Existing transport sessions can observe changed @@ -82,6 +84,11 @@ and generated skill after reload. - If the old generation became unhealthy independently, a successful new connection is preferred; the operation does not imply that the old peer can be resurrected. +- Resource links issued before a reload are separate capabilities. An upstream + resource fetched **after** its old connection retires can fail; request a + fresh resource link from the new source. Already-running resource reads + retain their own peer and cancellation semantics, but cross-generation + resource-link durability is not claimed by this implementation. - Invalid data from upstream tool descriptions must remain untrusted. Reload permission uses existing Codexify connector authorization and audit boundaries. Do not expose the control plane to unauthenticated clients. From 49d93e39a889c49d5b439080c9755892ac1297c7 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:42:40 +0100 Subject: [PATCH 29/62] style(mcp): apply CI rustfmt corrections in src/bridge/hot_reload_tests.rs --- src/bridge/hot_reload_tests.rs | 332 ++++++++++++++++++++++----------- 1 file changed, 222 insertions(+), 110 deletions(-) diff --git a/src/bridge/hot_reload_tests.rs b/src/bridge/hot_reload_tests.rs index 65c5e0ef..73bac52f 100644 --- a/src/bridge/hot_reload_tests.rs +++ b/src/bridge/hot_reload_tests.rs @@ -9,8 +9,7 @@ use rmcp::{ }, service::{RequestContext, RoleServer}, transport::streamable_http_server::{ - StreamableHttpServerConfig, StreamableHttpService, - session::local::LocalSessionManager, + StreamableHttpServerConfig, StreamableHttpService, session::local::LocalSessionManager, }, }; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -40,14 +39,23 @@ impl ServerHandler for Fixture { "type": "object", "properties": {}, "additionalProperties": false - }).as_object().cloned().unwrap(); + }) + .as_object() + .cloned() + .unwrap(); let mut tools = vec![ rmcp::model::Tool::new("probe", "Report this fixture's version", schema.clone()), - rmcp::model::Tool::new("slow", "Report this fixture's version after a delay", schema.clone()), + rmcp::model::Tool::new( + "slow", + "Report this fixture's version after a delay", + schema.clone(), + ), ]; if self.new_tool { tools.push(rmcp::model::Tool::new( - "new_tool", "Added in version two", schema + "new_tool", + "Added in version two", + schema, )); } Ok(ListToolsResult::with_all_items(tools)) @@ -72,7 +80,9 @@ async fn fixture( list_delay: Duration, call_delay: Duration, ) -> (String, Arc, tokio::task::JoinHandle<()>) { - let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)).await.unwrap(); + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); let address = listener.local_addr().unwrap(); let calls = Arc::new(AtomicUsize::new(0)); let fixture = Fixture { @@ -108,40 +118,48 @@ async fn invoke(view: Arc, name: &str) -> ToolResult { invoke_args(view, name, json!({})).await } -async fn invoke_args( - view: Arc, - name: &str, - args: Value, -) -> ToolResult { - let tool = view.tools.iter().find(|tool| tool.name() == name) +async fn invoke_args(view: Arc, name: &str, args: Value) -> ToolResult { + let tool = view + .tools + .iter() + .find(|tool| tool.name() == name) .unwrap_or_else(|| panic!("tool {name} missing from view")); tool.call( args, &crate::config::default_config(std::env::temp_dir()), &SessionState::new(), - ).await + ) + .await } #[tokio::test] async fn single_source_reload_updates_tools_but_not_other_mcp() { - let (old_url, _, old_task) = - fixture("1", false, Duration::ZERO, Duration::ZERO).await; - let (new_url, _, new_task) = - fixture("2", true, Duration::ZERO, Duration::ZERO).await; - let (other_url, _, other_task) = - fixture("other", false, Duration::ZERO, Duration::ZERO).await; + let (old_url, _, old_task) = fixture("1", false, Duration::ZERO, Duration::ZERO).await; + let (new_url, _, new_task) = fixture("2", true, Duration::ZERO, Duration::ZERO).await; + let (other_url, _, other_task) = fixture("other", false, Duration::ZERO, Duration::ZERO).await; let mut config = crate::config::default_config(std::env::temp_dir()); - config.mcp_servers.insert("alpha".into(), spec(&old_url, McpToolExposure::Direct)); - config.mcp_servers.insert("beta".into(), spec(&other_url, McpToolExposure::Direct)); + config + .mcp_servers + .insert("alpha".into(), spec(&old_url, McpToolExposure::Direct)); + config + .mcp_servers + .insert("beta".into(), spec(&other_url, McpToolExposure::Direct)); let bridge = connect_upstreams(&config).await; let runtime = bridge.runtime.clone(); let before = runtime.snapshot(); - assert_eq!(invoke(before.clone(), "alpha__probe").await.joined_text(), "1"); - assert_eq!(invoke(before.clone(), "beta__probe").await.joined_text(), "other"); + assert_eq!( + invoke(before.clone(), "alpha__probe").await.joined_text(), + "1" + ); + assert_eq!( + invoke(before.clone(), "beta__probe").await.joined_text(), + "other" + ); - let result = runtime.reload_with_spec( - "alpha", spec(&new_url, McpToolExposure::Direct), None - ).await.unwrap(); + let result = runtime + .reload_with_spec("alpha", spec(&new_url, McpToolExposure::Direct), None) + .await + .unwrap(); assert_eq!(result["outcome"], "reloaded"); assert_eq!(result["previousGeneration"], 1); assert_eq!(result["replacementGeneration"], 2); @@ -152,9 +170,18 @@ async fn single_source_reload_updates_tools_but_not_other_mcp() { let after = runtime.snapshot(); assert!(Arc::ptr_eq(&before.sources["beta"], &after.sources["beta"])); - assert!(!Arc::ptr_eq(&before.sources["alpha"], &after.sources["alpha"])); - assert_eq!(invoke(after.clone(), "alpha__probe").await.joined_text(), "2"); - assert_eq!(invoke(after.clone(), "alpha__new_tool").await.joined_text(), "2"); + assert!(!Arc::ptr_eq( + &before.sources["alpha"], + &after.sources["alpha"] + )); + assert_eq!( + invoke(after.clone(), "alpha__probe").await.joined_text(), + "2" + ); + assert_eq!( + invoke(after.clone(), "alpha__new_tool").await.joined_text(), + "2" + ); assert_eq!(invoke(after, "beta__probe").await.joined_text(), "other"); assert_eq!(invoke(before, "alpha__probe").await.joined_text(), "1"); old_task.abort(); @@ -166,10 +193,11 @@ async fn single_source_reload_updates_tools_but_not_other_mcp() { async fn active_calls_keep_their_original_generation_without_replay() { let (old_url, old_count, old_task) = fixture("1", false, Duration::ZERO, Duration::from_millis(450)).await; - let (new_url, new_count, new_task) = - fixture("2", true, Duration::ZERO, Duration::ZERO).await; + let (new_url, new_count, new_task) = fixture("2", true, Duration::ZERO, Duration::ZERO).await; let mut config = crate::config::default_config(std::env::temp_dir()); - config.mcp_servers.insert("alpha".into(), spec(&old_url, McpToolExposure::Direct)); + config + .mcp_servers + .insert("alpha".into(), spec(&old_url, McpToolExposure::Direct)); let runtime = connect_upstreams(&config).await.runtime; let initial = runtime.snapshot(); let pending = tokio::spawn(invoke(initial, "alpha__slow")); @@ -177,16 +205,24 @@ async fn active_calls_keep_their_original_generation_without_replay() { while old_count.load(Ordering::SeqCst) == 0 { tokio::time::sleep(Duration::from_millis(5)).await; } - }).await.unwrap(); + }) + .await + .unwrap(); let started = std::time::Instant::now(); - let result = runtime.reload_with_spec( - "alpha", spec(&new_url, McpToolExposure::Direct), None - ).await.unwrap(); + let result = runtime + .reload_with_spec("alpha", spec(&new_url, McpToolExposure::Direct), None) + .await + .unwrap(); assert_eq!(result["oldCallsDrained"], true); assert!(started.elapsed() >= Duration::from_millis(100)); assert_eq!(pending.await.unwrap().joined_text(), "1"); assert_eq!(old_count.load(Ordering::SeqCst), 1); - assert_eq!(invoke(runtime.snapshot(), "alpha__probe").await.joined_text(), "2"); + assert_eq!( + invoke(runtime.snapshot(), "alpha__probe") + .await + .joined_text(), + "2" + ); assert_eq!(new_count.load(Ordering::SeqCst), 1); old_task.abort(); new_task.abort(); @@ -194,23 +230,34 @@ async fn active_calls_keep_their_original_generation_without_replay() { #[tokio::test] async fn failed_replacement_and_unknown_ids_cannot_mutate_active_generation() { - let (old_url, _, old_task) = - fixture("1", false, Duration::ZERO, Duration::ZERO).await; + let (old_url, _, old_task) = fixture("1", false, Duration::ZERO, Duration::ZERO).await; let mut config = crate::config::default_config(std::env::temp_dir()); - config.mcp_servers.insert("alpha".into(), spec(&old_url, McpToolExposure::Direct)); + config + .mcp_servers + .insert("alpha".into(), spec(&old_url, McpToolExposure::Direct)); let runtime = connect_upstreams(&config).await.runtime; let old = runtime.snapshot(); assert_eq!( - runtime.reload_with_spec( - "missing", spec("http://127.0.0.1:9/mcp", McpToolExposure::Direct), None - ).await.unwrap_err(), + runtime + .reload_with_spec( + "missing", + spec("http://127.0.0.1:9/mcp", McpToolExposure::Direct), + None + ) + .await + .unwrap_err(), "SOURCE_UNKNOWN" ); assert_eq!( - runtime.reload_with_spec( - "alpha", spec("http://127.0.0.1:9/mcp", McpToolExposure::Direct), None - ).await.unwrap_err(), + runtime + .reload_with_spec( + "alpha", + spec("http://127.0.0.1:9/mcp", McpToolExposure::Direct), + None + ) + .await + .unwrap_err(), "INITIALIZE_FAILED" ); let after = runtime.snapshot(); @@ -221,35 +268,43 @@ async fn failed_replacement_and_unknown_ids_cannot_mutate_active_generation() { #[tokio::test] async fn same_source_reload_conflicts_while_other_source_can_reload() { - let (a1, _, first_task) = - fixture("1", false, Duration::ZERO, Duration::ZERO).await; - let (a2, _, second_task) = - fixture("2", true, Duration::from_millis(300), Duration::ZERO).await; - let (b1, _, third_task) = - fixture("other1", false, Duration::ZERO, Duration::ZERO).await; - let (b2, _, fourth_task) = - fixture("other2", false, Duration::ZERO, Duration::ZERO).await; + let (a1, _, first_task) = fixture("1", false, Duration::ZERO, Duration::ZERO).await; + let (a2, _, second_task) = fixture("2", true, Duration::from_millis(300), Duration::ZERO).await; + let (b1, _, third_task) = fixture("other1", false, Duration::ZERO, Duration::ZERO).await; + let (b2, _, fourth_task) = fixture("other2", false, Duration::ZERO, Duration::ZERO).await; let mut config = crate::config::default_config(std::env::temp_dir()); - config.mcp_servers.insert("alpha".into(), spec(&a1, McpToolExposure::Direct)); - config.mcp_servers.insert("beta".into(), spec(&b1, McpToolExposure::Direct)); + config + .mcp_servers + .insert("alpha".into(), spec(&a1, McpToolExposure::Direct)); + config + .mcp_servers + .insert("beta".into(), spec(&b1, McpToolExposure::Direct)); let runtime = connect_upstreams(&config).await.runtime; let clone = runtime.clone(); let initial = tokio::spawn(async move { - clone.reload_with_spec("alpha", spec(&a2, McpToolExposure::Direct), None).await + clone + .reload_with_spec("alpha", spec(&a2, McpToolExposure::Direct), None) + .await }); tokio::time::sleep(Duration::from_millis(50)).await; assert_eq!( - runtime.reload_with_spec("alpha", spec(&a1, McpToolExposure::Direct), None) - .await.unwrap_err(), + runtime + .reload_with_spec("alpha", spec(&a1, McpToolExposure::Direct), None) + .await + .unwrap_err(), "RELOAD_IN_PROGRESS" ); - let other = runtime.reload_with_spec( - "beta", spec(&b2, McpToolExposure::Direct), None - ).await.unwrap(); + let other = runtime + .reload_with_spec("beta", spec(&b2, McpToolExposure::Direct), None) + .await + .unwrap(); assert_eq!(other["replacementGeneration"], 2); assert_eq!(initial.await.unwrap().unwrap()["replacementGeneration"], 2); let view = runtime.snapshot(); - assert_eq!(invoke(view.clone(), "alpha__probe").await.joined_text(), "2"); + assert_eq!( + invoke(view.clone(), "alpha__probe").await.joined_text(), + "2" + ); assert_eq!(invoke(view, "beta__probe").await.joined_text(), "other2"); first_task.abort(); second_task.abort(); @@ -259,33 +314,47 @@ async fn same_source_reload_conflicts_while_other_source_can_reload() { #[tokio::test] async fn catalog_reload_refreshes_search_schema_and_call_routing() { - let (old_url, _, old_task) = - fixture("1", false, Duration::ZERO, Duration::ZERO).await; - let (new_url, _, new_task) = - fixture("2", true, Duration::ZERO, Duration::ZERO).await; + let (old_url, _, old_task) = fixture("1", false, Duration::ZERO, Duration::ZERO).await; + let (new_url, _, new_task) = fixture("2", true, Duration::ZERO, Duration::ZERO).await; let mut config = crate::config::default_config(std::env::temp_dir()); - config.mcp_servers.insert("catalog".into(), spec(&old_url, McpToolExposure::Catalog)); + config + .mcp_servers + .insert("catalog".into(), spec(&old_url, McpToolExposure::Catalog)); let runtime = connect_upstreams(&config).await.runtime; let before = runtime.snapshot(); - let source = invoke(before, "mcp_list_sources").await.structured_content.unwrap(); + let source = invoke(before, "mcp_list_sources") + .await + .structured_content + .unwrap(); assert_eq!(source["sources"][0]["toolCount"], 2); - let result = runtime.reload_with_spec( - "catalog", spec(&new_url, McpToolExposure::Catalog), None - ).await.unwrap(); + let result = runtime + .reload_with_spec("catalog", spec(&new_url, McpToolExposure::Catalog), None) + .await + .unwrap(); assert_eq!(result["toolChanges"]["added"], 1); let after = runtime.snapshot(); - let list = invoke(after.clone(), "mcp_list_sources").await.structured_content.unwrap(); + let list = invoke(after.clone(), "mcp_list_sources") + .await + .structured_content + .unwrap(); assert_eq!(list["sources"][0]["toolCount"], 3); let source_id = list["sources"][0]["id"].as_str().unwrap(); let found = invoke_args( - after.clone(), "mcp_search_tools", json!({"query":"new_tool","source":source_id}) - ).await.structured_content.unwrap(); + after.clone(), + "mcp_search_tools", + json!({"query":"new_tool","source":source_id}), + ) + .await + .structured_content + .unwrap(); assert!(found["total"].as_u64().unwrap_or(0) >= 1); let tool_id = found["matches"][0]["toolId"].as_str().unwrap(); let call = invoke_args( - after, "mcp_call_tool", - json!({"source":source_id,"tool":tool_id,"arguments":{}}) - ).await; + after, + "mcp_call_tool", + json!({"source":source_id,"tool":tool_id,"arguments":{}}), + ) + .await; assert!(!call.is_error); assert_eq!(call.joined_text(), "2"); old_task.abort(); @@ -294,30 +363,43 @@ async fn catalog_reload_refreshes_search_schema_and_call_routing() { #[tokio::test] async fn gateway_reload_reflects_added_functions() { - let (old_url, _, old_task) = - fixture("1", false, Duration::ZERO, Duration::ZERO).await; - let (new_url, _, new_task) = - fixture("2", true, Duration::ZERO, Duration::ZERO).await; + let (old_url, _, old_task) = fixture("1", false, Duration::ZERO, Duration::ZERO).await; + let (new_url, _, new_task) = fixture("2", true, Duration::ZERO, Duration::ZERO).await; let mut config = crate::config::default_config(std::env::temp_dir()); - config.mcp_servers.insert("gateway".into(), spec(&old_url, McpToolExposure::Gateway)); + config + .mcp_servers + .insert("gateway".into(), spec(&old_url, McpToolExposure::Gateway)); let runtime = connect_upstreams(&config).await.runtime; let before = runtime.snapshot(); let old = invoke_args( - before, "gateway", json!({"function":"probe","arguments":{}}) - ).await; + before, + "gateway", + json!({"function":"probe","arguments":{}}), + ) + .await; assert_eq!(old.joined_text(), "1"); - runtime.reload_with_spec( - "gateway", spec(&new_url, McpToolExposure::Gateway), None - ).await.unwrap(); + runtime + .reload_with_spec("gateway", spec(&new_url, McpToolExposure::Gateway), None) + .await + .unwrap(); let after = runtime.snapshot(); - let tool = after.tools.iter().find(|tool| tool.name() == "gateway").unwrap(); + let tool = after + .tools + .iter() + .find(|tool| tool.name() == "gateway") + .unwrap(); assert!( tool.input_schema()["properties"]["function"]["enum"] - .as_array().unwrap().contains(&json!("new_tool")) + .as_array() + .unwrap() + .contains(&json!("new_tool")) ); let new = invoke_args( - after, "gateway", json!({"function":"new_tool","arguments":{}}) - ).await; + after, + "gateway", + json!({"function":"new_tool","arguments":{}}), + ) + .await; assert_eq!(new.joined_text(), "2"); old_task.abort(); new_task.abort(); @@ -328,28 +410,41 @@ async fn reconnect_failed_initial_source_and_cancel_without_switching() { let (url, _, task) = fixture("recovered", false, Duration::ZERO, Duration::ZERO).await; let mut config = crate::config::default_config(std::env::temp_dir()); config.mcp_servers.insert( - "unavailable".into(), crate::types::McpServerSpec { + "unavailable".into(), + crate::types::McpServerSpec { url: Some("http://127.0.0.1:9/mcp".into()), startup_timeout_sec: Some(0.5), ..Default::default() - } + }, ); let runtime = connect_upstreams(&config).await.runtime; assert!(!runtime.snapshot().sources.contains_key("unavailable")); - let result = runtime.reload_with_spec( - "unavailable", spec(&url, McpToolExposure::Direct), None - ).await.unwrap(); + let result = runtime + .reload_with_spec("unavailable", spec(&url, McpToolExposure::Direct), None) + .await + .unwrap(); assert_eq!(result["previousGeneration"], 0); assert_eq!(result["replacementGeneration"], 1); - assert_eq!(invoke(runtime.snapshot(), "unavailable__probe").await.joined_text(), "recovered"); + assert_eq!( + invoke(runtime.snapshot(), "unavailable__probe") + .await + .joined_text(), + "recovered" + ); let old = runtime.snapshot(); let cancelled = CancellationToken::new(); cancelled.cancel(); assert_eq!( - runtime.reload_with_spec( - "unavailable", spec(&url, McpToolExposure::Direct), Some(&cancelled) - ).await.unwrap_err(), "CANCELLED" + runtime + .reload_with_spec( + "unavailable", + spec(&url, McpToolExposure::Direct), + Some(&cancelled) + ) + .await + .unwrap_err(), + "CANCELLED" ); assert!(Arc::ptr_eq(&old, &runtime.snapshot())); task.abort(); @@ -359,12 +454,15 @@ async fn reconnect_failed_initial_source_and_cancel_without_switching() { async fn repeated_reloads_keep_one_active_generation_per_source() { let (url, _, task) = fixture("stable", false, Duration::ZERO, Duration::ZERO).await; let mut config = crate::config::default_config(std::env::temp_dir()); - config.mcp_servers.insert("alpha".into(), spec(&url, McpToolExposure::Direct)); + config + .mcp_servers + .insert("alpha".into(), spec(&url, McpToolExposure::Direct)); let runtime = connect_upstreams(&config).await.runtime; for expected in 2..=6 { - let result = runtime.reload_with_spec( - "alpha", spec(&url, McpToolExposure::Direct), None - ).await.unwrap(); + let result = runtime + .reload_with_spec("alpha", spec(&url, McpToolExposure::Direct), None) + .await + .unwrap(); assert_eq!(result["replacementGeneration"], expected); assert_eq!(result["oldCallsDrained"], true); let view = runtime.snapshot(); @@ -380,14 +478,28 @@ async fn direct_name_collisions_are_not_ambiguous_after_reload() { let (url2, _, task2) = fixture("two", true, Duration::ZERO, Duration::ZERO).await; let mut config = crate::config::default_config(std::env::temp_dir()); // sanitize("dup!")==sanitize("dup?"), so both would generate dup___probe. - config.mcp_servers.insert("dup!".into(), spec(&url1, McpToolExposure::Direct)); - config.mcp_servers.insert("dup?".into(), spec(&url2, McpToolExposure::Direct)); + config + .mcp_servers + .insert("dup!".into(), spec(&url1, McpToolExposure::Direct)); + config + .mcp_servers + .insert("dup?".into(), spec(&url2, McpToolExposure::Direct)); let runtime = connect_upstreams(&config).await.runtime; let view = runtime.snapshot(); - let names = view.tools.iter().map(|tool| tool.name()).collect::>(); - let unique = names.iter().copied().collect::>(); + let names = view + .tools + .iter() + .map(|tool| tool.name()) + .collect::>(); + let unique = names + .iter() + .copied() + .collect::>(); assert_eq!(unique.len(), names.len()); - assert_eq!(names.iter().filter(|name| **name == "dup___probe").count(), 1); + assert_eq!( + names.iter().filter(|name| **name == "dup___probe").count(), + 1 + ); task1.abort(); task2.abort(); } From d5feede9d571e2f6d37c584e8456b9c2a901c745 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:42:42 +0100 Subject: [PATCH 30/62] style(mcp): apply CI rustfmt corrections in src/bridge.rs --- src/bridge.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bridge.rs b/src/bridge.rs index 557dcea7..2a8cd172 100644 --- a/src/bridge.rs +++ b/src/bridge.rs @@ -35,10 +35,10 @@ use crate::process_env::scrub_untrusted_child_env; use crate::tool::{Tool, ToolBehavior, ToolCallIdentity, ToolRequestContext}; use crate::types::{AppConfig, McpToolExposure, ToolContent, ToolResult}; -mod stderr; mod hot_reload; #[cfg(test)] mod hot_reload_tests; +mod stderr; pub(crate) use hot_reload::{Generation, HotReloadRuntime, ReloadMcpTool}; /// How long to wait for an upstream to start up and answer `tools/list` before From 3acf5cddadee4ba16807795c4b09615a8ac37135 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:46:24 +0100 Subject: [PATCH 31/62] refactor(mcp): share individual dynamic tools independently of registry snapshots --- src/bridge.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bridge.rs b/src/bridge.rs index 2a8cd172..6dae727e 100644 --- a/src/bridge.rs +++ b/src/bridge.rs @@ -48,7 +48,7 @@ const CONNECT_TIMEOUT: Duration = Duration::from_secs(20); /// Connected MCP sources are owned by a transactional registry. Its immutable /// view is shared across transport sessions, while calls pin their generation. pub struct Bridge { - pub tools: Arc>>, + pub tools: Arc>>, pub resources: Arc, pub report: Vec, pub(crate) runtime: Arc, From 998130af92dc3f9599481063d86b660b245b6e61 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:46:26 +0100 Subject: [PATCH 32/62] fix(mcp): store validated MCP tool descriptors as independently pinned Arc --- src/bridge/hot_reload.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bridge/hot_reload.rs b/src/bridge/hot_reload.rs index 844a4248..046ec945 100644 --- a/src/bridge/hot_reload.rs +++ b/src/bridge/hot_reload.rs @@ -109,7 +109,7 @@ impl NamePool { } pub(crate) struct BridgeView { - pub(crate) tools: Arc>>, + pub(crate) tools: Arc>>, pub(crate) sources: HashMap>, } @@ -253,7 +253,7 @@ impl HotReloadRuntime { return None; } match crate::tool::validate_and_wrap_tool(tool) { - Ok(tool) => Some(tool), + Ok(tool) => Some(Arc::::from(tool)), Err(error) => { tracing::warn!(%error, "invalid upstream MCP tool skipped"); None From 8556193b84f58d2c8b4328948bedcc8a16cde972 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:46:28 +0100 Subject: [PATCH 33/62] fix(mcp): dispatch pins only the selected upstream generation to prevent cross-source retention --- src/server.rs | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/server.rs b/src/server.rs index b5b5df4c..c4d30e51 100644 --- a/src/server.rs +++ b/src/server.rs @@ -836,21 +836,19 @@ impl ServerHandler for CodexHandler { .arguments .map(Value::Object) .unwrap_or_else(|| json!({})); - // Pin a complete immutable upstream registry through the entire async - // dispatch. Hot reload cannot close this request's source generation. - let bridge_view = self - .bridge_runtime - .as_ref() - .map(|runtime| runtime.snapshot()); + // Pin only the selected upstream tool for this entire async dispatch. + // Holding the full registry snapshot here would keep *unrelated* retired + // MCP processes alive while another source runs a long request. + let bridge_tool = self.bridge_runtime.as_ref().and_then(|runtime| { + let view = runtime.snapshot(); + view.tools.iter().find(|tool| tool.name() == name).cloned() + }); let tool = self .tools .iter() .find(|tool| tool.name() == name) - .or_else(|| { - bridge_view - .as_ref() - .and_then(|view| view.tools.iter().find(|tool| tool.name() == name)) - }); + .map(|tool| tool.as_ref()) + .or_else(|| bridge_tool.as_deref()); let ticket_wait = name == TicketWait::NAME && tool.is_some(); let model_call = !tool.is_some_and(|tool| app_only_tool(tool.as_ref())); let mut task_conversation = conversation.clone(); From 040e5e28a729e833b71445b66c0cf12d801cb800 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:46:52 +0100 Subject: [PATCH 34/62] fix(mcp): propagate trait-object dispatch references after source-specific pinning --- src/server.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/server.rs b/src/server.rs index c4d30e51..9d94765f 100644 --- a/src/server.rs +++ b/src/server.rs @@ -850,7 +850,7 @@ impl ServerHandler for CodexHandler { .map(|tool| tool.as_ref()) .or_else(|| bridge_tool.as_deref()); let ticket_wait = name == TicketWait::NAME && tool.is_some(); - let model_call = !tool.is_some_and(|tool| app_only_tool(tool.as_ref())); + let model_call = !tool.is_some_and(app_only_tool); let mut task_conversation = conversation.clone(); let mut task_owner_generation = None; let mut conversation_retired = false; @@ -1046,7 +1046,7 @@ impl ServerHandler for CodexHandler { // Keep `tool` as an Option so that even an unknown-tool call flows through // the audit begin/finish pairing below rather than short-circuiting. let agent_call = self.config.markdown_chat.enabled - && tool.is_some_and(|tool| !app_only_tool(tool.as_ref())) + && tool.is_some_and(|tool| !app_only_tool(tool)) && !ticket_wait && !context.ct.is_cancelled(); let called_at_ms = crate::markdown_chat::now_ms(); @@ -1679,7 +1679,7 @@ impl ServerHandler for CodexHandler { .await; } } - finalize_model_visible_result(tool.map(|tool| tool.as_ref()), &mut result, &self.config); + finalize_model_visible_result(tool, &mut result, &self.config); if !result.is_error && let Some(tool) = tool && let Err(error) = tool.validate_structured_output(result.structured_content.as_ref()) From 84a081dfe6b9e3e14ce0f8016827d23c3b2a8ea0 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:47:04 +0100 Subject: [PATCH 35/62] fix(mcp): use trait object in owner-chat cleanup gate --- src/server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.rs b/src/server.rs index 9d94765f..792fc2cb 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1812,7 +1812,7 @@ impl ServerHandler for CodexHandler { && name != "automation_verify_pairing" && !pairing_continuation && !ticket_wait - && !tool.is_some_and(|tool| app_only_tool(tool.as_ref())) + && !tool.is_some_and(app_only_tool) && !context.ct.is_cancelled() && self .conversation_auth_error("chat_read", conversation.as_ref()) From 15e0a992e78723049838e6d04473029dd9c74c47 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:47:17 +0100 Subject: [PATCH 36/62] test(mcp): verify unrelated long-running request cannot pin old source generation --- src/bridge/hot_reload_tests.rs | 62 ++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/src/bridge/hot_reload_tests.rs b/src/bridge/hot_reload_tests.rs index 73bac52f..b950ea32 100644 --- a/src/bridge/hot_reload_tests.rs +++ b/src/bridge/hot_reload_tests.rs @@ -530,3 +530,65 @@ async fn old_generation_is_released_after_draining_without_stale_snapshots() { old_task.abort(); new_task.abort(); } + +#[tokio::test] +async fn unrelated_in_flight_call_does_not_retain_a_retired_source() { + let (a1, _, task_a1) = + fixture("alpha-1", false, Duration::ZERO, Duration::ZERO).await; + let (a2, _, task_a2) = + fixture("alpha-2", false, Duration::ZERO, Duration::ZERO).await; + let (b, b_calls, task_b) = + fixture("beta", false, Duration::ZERO, Duration::from_millis(400)).await; + let mut config = crate::config::default_config(std::env::temp_dir()); + config + .mcp_servers + .insert("alpha".into(), spec(&a1, McpToolExposure::Direct)); + config + .mcp_servers + .insert("beta".into(), spec(&b, McpToolExposure::Direct)); + let runtime = connect_upstreams(&config).await.runtime; + let initial = runtime.snapshot(); + let alpha = Arc::downgrade(&initial.sources["alpha"]); + // Copy only beta's validated descriptor, just as server dispatch does. + let beta = initial + .tools + .iter() + .find(|tool| tool.name() == "beta__slow") + .unwrap() + .clone(); + drop(initial); + + let pending = tokio::spawn(async move { + beta.call( + json!({}), + &crate::config::default_config(std::env::temp_dir()), + &SessionState::new(), + ) + .await + }); + tokio::time::timeout(Duration::from_secs(2), async { + while b_calls.load(Ordering::SeqCst) == 0 { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .unwrap(); + + let result = runtime + .reload_with_spec("alpha", spec(&a2, McpToolExposure::Direct), None) + .await + .unwrap(); + assert_eq!(result["oldCallsDrained"], true); + assert!( + alpha.upgrade().is_none(), + "a beta request must not retain alpha's retired connection" + ); + assert_eq!(pending.await.unwrap().joined_text(), "beta"); + assert_eq!( + invoke(runtime.snapshot(), "alpha__probe").await.joined_text(), + "alpha-2" + ); + task_a1.abort(); + task_a2.abort(); + task_b.abort(); +} From c89ee1007d1f098c4a6f76cd80d1d022986684bf Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:47:27 +0100 Subject: [PATCH 37/62] docs(mcp): clarify source-specific dispatch pinning and retirement --- src/bridge/hot_reload.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/bridge/hot_reload.rs b/src/bridge/hot_reload.rs index 046ec945..6a174ee5 100644 --- a/src/bridge/hot_reload.rs +++ b/src/bridge/hot_reload.rs @@ -1,8 +1,8 @@ //! Transactional replacement of one upstream MCP connection. //! -//! All model-visible tools are rebuilt as one immutable registry snapshot. A -//! dispatch keeps its snapshot for its entire call, so old generations cannot -//! be disposed while requests are using them. +//! Tool definitions are rebuilt into an immutable registry snapshot. Dispatch +//! clones only its selected tool into the call, preserving that source's old +//! generation without retaining unrelated MCP connections. use super::*; use clap::Parser; use std::collections::{HashMap, HashSet}; @@ -431,7 +431,7 @@ impl HotReloadRuntime { write_gateway_skill(&self.config, name, &sanitize(name), &funcs); } - // The old generation stays alive in any request's registry snapshot. + // The old generation stays alive in requests pinned to its tools. // On a bounded drain timeout its existing calls are NOT cancelled or // replayed; the generation retires when the last call releases it. let drain_ms = previous From da24d1ad9c882800ad70a624390b5c49ff519dc6 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:48:05 +0100 Subject: [PATCH 38/62] style(mcp): apply remaining format hints (src/bridge/hot_reload.rs) --- src/bridge/hot_reload.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/bridge/hot_reload.rs b/src/bridge/hot_reload.rs index 6a174ee5..a7b27c1f 100644 --- a/src/bridge/hot_reload.rs +++ b/src/bridge/hot_reload.rs @@ -393,13 +393,7 @@ impl HotReloadRuntime { .count(); } let initialization_ms = started.elapsed().as_millis() as u64; - let new = Generation::new( - spec, - service, - tools, - timeout, - old_generation + 1, - ); + let new = Generation::new(spec, service, tools, timeout, old_generation + 1); let new_version = new.version(); // Connection preparation and view construction are outside the write // lock. Compare-and-rebuild prevents concurrent distinct-source reloads From a86959440a79b4113fa4cf59b021c2765e432033 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:48:07 +0100 Subject: [PATCH 39/62] style(mcp): apply remaining format hints (src/bridge.rs) --- src/bridge.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bridge.rs b/src/bridge.rs index 6dae727e..198904e0 100644 --- a/src/bridge.rs +++ b/src/bridge.rs @@ -39,6 +39,7 @@ mod hot_reload; #[cfg(test)] mod hot_reload_tests; mod stderr; +mod stderr; pub(crate) use hot_reload::{Generation, HotReloadRuntime, ReloadMcpTool}; /// How long to wait for an upstream to start up and answer `tools/list` before From 115576b43fa265e754d67382ea2462e12c567d21 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:48:36 +0100 Subject: [PATCH 40/62] fix(mcp): report retained old source generations accurately after drain --- src/bridge/hot_reload.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/bridge/hot_reload.rs b/src/bridge/hot_reload.rs index a7b27c1f..bbe7c311 100644 --- a/src/bridge/hot_reload.rs +++ b/src/bridge/hot_reload.rs @@ -440,6 +440,12 @@ impl HotReloadRuntime { let in_flight = previous .as_ref() .map_or(0, |old| *old.calls.lock().unwrap()); + // A selected but not yet forwarded call, an old catalog snapshot, + // or an active resource may still pin the retiring generation. + // Do not misreport that its process has already been released. + let still_referenced = previous + .as_ref() + .is_some_and(|old| Arc::strong_count(old) > 1); Ok(json!({ "source": name, "outcome": "reloaded", @@ -453,7 +459,8 @@ impl HotReloadRuntime { }, "oldCallsDrained": drained, "oldCallsStillRunning": in_flight, - "oldGenerationRetiring": !drained, + "oldGenerationStillReferenced": still_referenced, + "oldGenerationRetiring": !drained || still_referenced, "rollback": "not_required", "timingsMs": { "initializationAndReadiness": initialization_ms, From f27c9bf43a1ff217dcb584146e6c4fb313a368de Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:48:53 +0100 Subject: [PATCH 41/62] docs(mcp): describe per-tool dispatch ownership and honest draining report --- docs/MCP_HOT_RELOAD.md | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/MCP_HOT_RELOAD.md b/docs/MCP_HOT_RELOAD.md index 913f8fac..a2feb807 100644 --- a/docs/MCP_HOT_RELOAD.md +++ b/docs/MCP_HOT_RELOAD.md @@ -30,13 +30,16 @@ unsupported; the operation never silently falls back to a process-wide restart. 3. The new tool definitions are validated and an immutable registry view is constructed. The new view replaces the old view in one atomic pointer swap. Concurrent distinct-source reloads rebuild against the latest view. -4. Calls already using an old registry snapshot stay pinned to that snapshot - and connection generation; write-capable calls are **never replayed**. +4. Each dispatch retains only its selected validated upstream tool and that + source's connection generation. Write-capable calls are **never replayed**; + unrelated direct/gateway sources are not retained by a long-running call. Cancellation and timeout semantics remain those of the forwarded request. -5. The old generation is released when no surviving snapshot or call retains - it. The operation reports whether active calls drained within the source's - configured `requestLimits.slowCancellationMs` interval. A drain timeout - does **not** kill or replay an in-progress side-effecting call. +5. The old generation is released when no selected old tool or catalog + snapshot retains it. The operation reports whether forwarded calls drained + within the source's configured `requestLimits.slowCancellationMs` interval. + The separate `oldGenerationStillReferenced` flag covers retained references + even when no forwarded calls remain. A drain timeout does **not** kill or + replay an in-progress side-effecting call. 6. On failure **before** the swap, the new candidate is dropped and the old view is never replaced. Errors use bounded diagnostic codes and do not echo credentials, command lines, headers, environment variables, or raw @@ -53,7 +56,8 @@ A successful result includes `source`, `outcome`, `previousGeneration`, `replacementGeneration`, old/new reported MCP server versions, readiness, counts of added/removed/changed/unchanged tool definitions, `oldCallsDrained`, `oldCallsStillRunning`, -`oldGenerationRetiring`, `rollback`, `diagnosticCode`, and measured +`oldGenerationStillReferenced`, `oldGenerationRetiring`, `rollback`, +`diagnosticCode`, and measured `timingsMs` for initialization/readiness, registry switch, draining and total reload duration. From 402c1bf1bfa31db3a5d765b30fd9d1da97b5da02 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 23:31:48 +0100 Subject: [PATCH 42/62] fix(mcp): remove duplicate stderr module and format bridge generation --- src/bridge.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/bridge.rs b/src/bridge.rs index 198904e0..5583fcac 100644 --- a/src/bridge.rs +++ b/src/bridge.rs @@ -39,7 +39,6 @@ mod hot_reload; #[cfg(test)] mod hot_reload_tests; mod stderr; -mod stderr; pub(crate) use hot_reload::{Generation, HotReloadRuntime, ReloadMcpTool}; /// How long to wait for an upstream to start up and answer `tools/list` before @@ -540,9 +539,8 @@ pub async fn connect_upstreams(config: &AppConfig) -> Bridge { let functions = gateway_functions(&tools); write_gateway_skill(config, name, &sanitize(name), &functions); } - let generation = hot_reload::Generation::new( - spec.clone(), service, tools, timeout, 1, - ); + let generation = + hot_reload::Generation::new(spec.clone(), service, tools, timeout, 1); sources.insert(name.clone(), generation); report.push(format!("{name} -> {} ({count} tool(s))", mode.as_str())); tracing::info!(source = %name, %count, exposure = mode.as_str(), "MCP source connected"); From bfb4ea28e033020f0eece13a73347603950c6c87 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 23:31:51 +0100 Subject: [PATCH 43/62] style(mcp): apply CI rustfmt changes to live hot reload tests --- src/bridge/hot_reload_tests.rs | 39 +++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/src/bridge/hot_reload_tests.rs b/src/bridge/hot_reload_tests.rs index b950ea32..be24d6c0 100644 --- a/src/bridge/hot_reload_tests.rs +++ b/src/bridge/hot_reload_tests.rs @@ -506,12 +506,12 @@ async fn direct_name_collisions_are_not_ambiguous_after_reload() { #[tokio::test] async fn old_generation_is_released_after_draining_without_stale_snapshots() { - let (old_url, _, old_task) = - fixture("old", false, Duration::ZERO, Duration::ZERO).await; - let (new_url, _, new_task) = - fixture("new", true, Duration::ZERO, Duration::ZERO).await; + let (old_url, _, old_task) = fixture("old", false, Duration::ZERO, Duration::ZERO).await; + let (new_url, _, new_task) = fixture("new", true, Duration::ZERO, Duration::ZERO).await; let mut config = crate::config::default_config(std::env::temp_dir()); - config.mcp_servers.insert("alpha".into(), spec(&old_url, McpToolExposure::Direct)); + config + .mcp_servers + .insert("alpha".into(), spec(&old_url, McpToolExposure::Direct)); let bridge = connect_upstreams(&config).await; let runtime = bridge.runtime.clone(); let first = runtime.snapshot(); @@ -521,22 +521,29 @@ async fn old_generation_is_released_after_draining_without_stale_snapshots() { drop(bridge); assert!(weak.upgrade().is_some()); - let result = runtime.reload_with_spec( - "alpha", spec(&new_url, McpToolExposure::Direct), None - ).await.unwrap(); + let result = runtime + .reload_with_spec("alpha", spec(&new_url, McpToolExposure::Direct), None) + .await + .unwrap(); assert_eq!(result["oldCallsDrained"], true); - assert!(weak.upgrade().is_none(), "retired service generation leaked"); - assert_eq!(invoke(runtime.snapshot(), "alpha__probe").await.joined_text(), "new"); + assert!( + weak.upgrade().is_none(), + "retired service generation leaked" + ); + assert_eq!( + invoke(runtime.snapshot(), "alpha__probe") + .await + .joined_text(), + "new" + ); old_task.abort(); new_task.abort(); } #[tokio::test] async fn unrelated_in_flight_call_does_not_retain_a_retired_source() { - let (a1, _, task_a1) = - fixture("alpha-1", false, Duration::ZERO, Duration::ZERO).await; - let (a2, _, task_a2) = - fixture("alpha-2", false, Duration::ZERO, Duration::ZERO).await; + let (a1, _, task_a1) = fixture("alpha-1", false, Duration::ZERO, Duration::ZERO).await; + let (a2, _, task_a2) = fixture("alpha-2", false, Duration::ZERO, Duration::ZERO).await; let (b, b_calls, task_b) = fixture("beta", false, Duration::ZERO, Duration::from_millis(400)).await; let mut config = crate::config::default_config(std::env::temp_dir()); @@ -585,7 +592,9 @@ async fn unrelated_in_flight_call_does_not_retain_a_retired_source() { ); assert_eq!(pending.await.unwrap().joined_text(), "beta"); assert_eq!( - invoke(runtime.snapshot(), "alpha__probe").await.joined_text(), + invoke(runtime.snapshot(), "alpha__probe") + .await + .joined_text(), "alpha-2" ); task_a1.abort(); From a0c72d3582578400e69cafb9053db8c25ddac32c Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Thu, 19 Nov 2026 23:59:46 +0100 Subject: [PATCH 44/62] fix(mcp): satisfy strict Clippy in per-source dispatch fallback --- src/server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.rs b/src/server.rs index 792fc2cb..770b8647 100644 --- a/src/server.rs +++ b/src/server.rs @@ -848,7 +848,7 @@ impl ServerHandler for CodexHandler { .iter() .find(|tool| tool.name() == name) .map(|tool| tool.as_ref()) - .or_else(|| bridge_tool.as_deref()); + .or(bridge_tool.as_deref()); let ticket_wait = name == TicketWait::NAME && tool.is_some(); let model_call = !tool.is_some_and(app_only_tool); let mut task_conversation = conversation.clone(); From cc92b3d413e04c5cd56fc82e72e1874e7e456b02 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Fri, 20 Nov 2026 00:13:26 +0100 Subject: [PATCH 45/62] fix(mcp): preserve safe startup report wording for catalog and unsupported transports --- src/bridge.rs | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/src/bridge.rs b/src/bridge.rs index 5583fcac..47626fc3 100644 --- a/src/bridge.rs +++ b/src/bridge.rs @@ -542,14 +542,33 @@ pub async fn connect_upstreams(config: &AppConfig) -> Bridge { let generation = hot_reload::Generation::new(spec.clone(), service, tools, timeout, 1); sources.insert(name.clone(), generation); - report.push(format!("{name} -> {} ({count} tool(s))", mode.as_str())); + let visibility = if mode == McpToolExposure::Catalog { + " private" + } else { + "" + }; + report.push(format!( + "{name} -> {} ({count}{visibility} tool(s))", + mode.as_str() + )); tracing::info!(source = %name, %count, exposure = mode.as_str(), "MCP source connected"); } Err(_) => { // Upstream configuration can contain private header or command // values; do not echo a failed spawn/handshake error to clients. - report.push(format!("{name} -> FAILED (initialization)")); - tracing::warn!(source = %name, "MCP source failed initialization"); + // An unsupported transport type is a safe, useful static + // diagnostic and should not be hidden as an initialization + // failure; neither diagnostic includes the configured URL. + let legacy_transport = spec.transport.as_deref().is_some_and(|kind| { + matches!(kind.to_ascii_lowercase().as_str(), "sse" | "websocket" | "ws") + }); + if legacy_transport { + report.push(format!("{name} -> not supported (transport)")); + tracing::warn!(source = %name, "MCP source transport is not supported"); + } else { + report.push(format!("{name} -> FAILED (initialization)")); + tracing::warn!(source = %name, "MCP source failed initialization"); + } } } } From 8833c5d3787806135d7826eab25295df3a44f3df Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Fri, 20 Nov 2026 00:14:08 +0100 Subject: [PATCH 46/62] fix(mcp): register drain waiter before reading in-flight count --- src/bridge/hot_reload.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/bridge/hot_reload.rs b/src/bridge/hot_reload.rs index bbe7c311..a46c0a2f 100644 --- a/src/bridge/hot_reload.rs +++ b/src/bridge/hot_reload.rs @@ -73,10 +73,16 @@ impl Generation { async fn wait_for_calls(&self, limit: Duration) -> bool { tokio::time::timeout(limit, async { loop { + // Register the waiter before checking the counter. Otherwise + // the last call can finish between the check and registration, + // losing its wakeup and spuriously timing out the drain. + let notified = self.drained.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); if *self.calls.lock().unwrap() == 0 { break; } - self.drained.notified().await; + notified.await; } }) .await From c85264f8a1e8df9b34c7542df9c5adbbd3398647 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Fri, 20 Nov 2026 00:21:15 +0100 Subject: [PATCH 47/62] style(mcp): apply exact rustfmt fix to unsupported-transport report --- src/bridge.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/bridge.rs b/src/bridge.rs index 47626fc3..8cb16d98 100644 --- a/src/bridge.rs +++ b/src/bridge.rs @@ -560,7 +560,10 @@ pub async fn connect_upstreams(config: &AppConfig) -> Bridge { // diagnostic and should not be hidden as an initialization // failure; neither diagnostic includes the configured URL. let legacy_transport = spec.transport.as_deref().is_some_and(|kind| { - matches!(kind.to_ascii_lowercase().as_str(), "sse" | "websocket" | "ws") + matches!( + kind.to_ascii_lowercase().as_str(), + "sse" | "websocket" | "ws" + ) }); if legacy_transport { report.push(format!("{name} -> not supported (transport)")); From 8a1540e8ba99f14393419f76e171b64567c10283 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Fri, 20 Nov 2026 00:50:43 +0100 Subject: [PATCH 48/62] test(mcp): add isolated real stdio fixture with distinct process identity --- scripts/fixtures/hot-reload-stdio-mcp.mjs | 49 +++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 scripts/fixtures/hot-reload-stdio-mcp.mjs diff --git a/scripts/fixtures/hot-reload-stdio-mcp.mjs b/scripts/fixtures/hot-reload-stdio-mcp.mjs new file mode 100644 index 00000000..48a41325 --- /dev/null +++ b/scripts/fixtures/hot-reload-stdio-mcp.mjs @@ -0,0 +1,49 @@ +// No dependencies or network: isolated MCP stdio child for Rust hot-reload tests. +// Every invocation is a separate operating-system process. +import { createInterface } from 'node:readline'; +import { writeFileSync } from 'node:fs'; + +const version = process.argv[2] ?? 'old'; +const pid = process.pid; +const input = createInterface({ input: process.stdin, crlfDelay: Infinity }); + +function respond(id, result) { + process.stdout.write(JSON.stringify({ jsonrpc: '2.0', id, result }) + '\n'); +} +function tool(name, description) { + return { name, description, inputSchema: { type: 'object', properties: {}, additionalProperties: false } }; +} +function replyToTool(id) { + respond(id, { content: [{ type: 'text', text: version + ':' + pid }], isError: false }); +} + +for await (const line of input) { + let request; + try { request = JSON.parse(line); } catch { continue; } + if (!request || !Object.hasOwn(request, 'id')) continue; + const { id, method } = request; + if (method === 'initialize') { + respond(id, { + protocolVersion: request.params?.protocolVersion ?? '2025-03-26', + capabilities: { tools: {} }, + serverInfo: { name: 'isolated-hot-reload-stdio-fixture', version }, + }); + } else if (method === 'tools/list') { + const tools = [tool('probe', 'Version probe ' + version), tool('slow', 'Slow probe ' + version)]; + if (version === 'new') tools.push(tool('new_tool', 'New tool in replacement generation')); + respond(id, { tools }); + } else if (method === 'tools/call') { + if (request.params?.name === 'slow') { + if (process.env.HOT_RELOAD_SLOW_MARKER_FILE) { + writeFileSync(process.env.HOT_RELOAD_SLOW_MARKER_FILE, String(pid)); + } + setTimeout(() => replyToTool(id), 300); + } else { + replyToTool(id); + } + } else if (method === 'ping') { + respond(id, {}); + } else { + process.stdout.write(JSON.stringify({ jsonrpc: '2.0', id, error: { code: -32601, message: 'Method not found' } }) + '\n'); + } +} From d2c4f64e7d6e49a8333c20b9bd4441e99722ef9d Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Fri, 20 Nov 2026 00:51:09 +0100 Subject: [PATCH 49/62] test(mcp): verify real stdio process replacement, drain and rollback on isolated fixtures --- src/bridge/hot_reload_tests.rs | 173 +++++++++++++++++++++++++++++++++ 1 file changed, 173 insertions(+) diff --git a/src/bridge/hot_reload_tests.rs b/src/bridge/hot_reload_tests.rs index be24d6c0..e2f7e033 100644 --- a/src/bridge/hot_reload_tests.rs +++ b/src/bridge/hot_reload_tests.rs @@ -601,3 +601,176 @@ async fn unrelated_in_flight_call_does_not_retain_a_retired_source() { task_a2.abort(); task_b.abort(); } + + +fn isolated_stdio_spec(version: &str) -> crate::types::McpServerSpec { + let fixture = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("scripts") + .join("fixtures") + .join("hot-reload-stdio-mcp.mjs"); + crate::types::McpServerSpec { + command: Some("node".into()), + args: vec![fixture.to_string_lossy().into_owned(), version.into()], + mode: Some(McpToolExposure::Direct), + startup_timeout_sec: Some(8.0), + ..Default::default() + } +} + +fn stdio_process_id(result: ToolResult, expected_version: &str) -> u32 { + assert!(!result.is_error, "stdio call failed: {}", result.joined_text()); + let response = result.joined_text(); + let (version, pid) = response + .trim() + .split_once(':') + .unwrap_or_else(|| panic!("invalid stdio fixture response: {response}")); + assert_eq!(version, expected_version); + pid.parse().expect("stdio fixture process id must be numeric") +} + +#[tokio::test] +async fn isolated_real_stdio_process_reload_preserves_unrelated_sources() { + // This fixture actually spawns three independent OS processes through the + // production TokioChildProcess transport; no live Codexify is involved. + let directory = tempfile::tempdir().unwrap(); + let started = directory.path().join("slow_started"); + let mut old_spec = isolated_stdio_spec("old"); + old_spec.env.insert( + "HOT_RELOAD_SLOW_MARKER_FILE".into(), + started.to_string_lossy().into_owned(), + ); + + let mut config = crate::config::default_config(std::env::temp_dir()); + config.mcp_servers.insert("alpha".into(), old_spec); + config + .mcp_servers + .insert("beta".into(), isolated_stdio_spec("beta")); + config + .mcp_servers + .insert("gamma".into(), isolated_stdio_spec("gamma")); + let bridge = connect_upstreams(&config).await; + assert_eq!( + bridge.runtime.snapshot().sources.len(), + 3, + "all isolated stdio fixtures must start: {:?}", + bridge.report + ); + let runtime = bridge.runtime.clone(); + drop(bridge); + + let initial = runtime.snapshot(); + let retired = Arc::downgrade(&initial.sources["alpha"]); + let old_probe = initial + .tools + .iter() + .find(|tool| tool.name() == "alpha__probe") + .unwrap() + .clone(); + let old_slow = initial + .tools + .iter() + .find(|tool| tool.name() == "alpha__slow") + .unwrap() + .clone(); + drop(initial); + let config_for_call = crate::config::default_config(std::env::temp_dir()); + let original_pid = stdio_process_id( + old_probe.call(json!({}), &config_for_call, &SessionState::new()).await, + "old", + ); + let beta_pid = stdio_process_id(invoke(runtime.snapshot(), "beta__probe").await, "beta"); + let gamma_pid = stdio_process_id(invoke(runtime.snapshot(), "gamma__probe").await, "gamma"); + + let pending = tokio::spawn(async move { + old_slow + .call( + json!({}), + &crate::config::default_config(std::env::temp_dir()), + &SessionState::new(), + ) + .await + }); + // The marker is written by the actual old process when it receives the + // slow request, making this test independent of host scheduling. + tokio::time::timeout(Duration::from_secs(4), async { + while !started.exists() { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("old stdio process did not begin the slow call"); + + let result = runtime + .reload_with_spec("alpha", isolated_stdio_spec("new"), None) + .await + .unwrap(); + assert_eq!(result["previousVersion"], "old"); + assert_eq!(result["replacementVersion"], "new"); + assert_eq!(result["previousGeneration"], 1); + assert_eq!(result["replacementGeneration"], 2); + assert_eq!(result["toolChanges"]["added"], 1); + assert_eq!(result["oldCallsDrained"], true); + assert_eq!(stdio_process_id(pending.await.unwrap(), "old"), original_pid); + assert_eq!( + stdio_process_id( + old_probe.call(json!({}), &config_for_call, &SessionState::new()).await, + "old", + ), + original_pid, + "a pinned retired generation must remain callable until released" + ); + let new_pid = stdio_process_id(invoke(runtime.snapshot(), "alpha__probe").await, "new"); + assert_ne!(new_pid, original_pid, "replacement must be a new OS process"); + assert_eq!( + stdio_process_id(invoke(runtime.snapshot(), "alpha__new_tool").await, "new"), + new_pid + ); + assert_eq!( + stdio_process_id(invoke(runtime.snapshot(), "beta__probe").await, "beta"), + beta_pid, + "beta must not reconnect" + ); + assert_eq!( + stdio_process_id(invoke(runtime.snapshot(), "gamma__probe").await, "gamma"), + gamma_pid, + "gamma must not reconnect" + ); + + let mut invalid_spec = isolated_stdio_spec("broken"); + invalid_spec.command = Some("codexify-nonexistent-hot-reload-fixture".into()); + assert_eq!( + runtime + .reload_with_spec("alpha", invalid_spec, None) + .await + .err(), + Some("INITIALIZE_FAILED") + ); + assert_eq!( + stdio_process_id(invoke(runtime.snapshot(), "alpha__probe").await, "new"), + new_pid, + "an unsuccessful replacement must preserve the healthy process" + ); + + drop(old_probe); + tokio::task::yield_now().await; + assert!( + retired.upgrade().is_none(), + "the retired generation must be released after its last handle drops" + ); + + let repeated = runtime + .reload_with_spec("alpha", isolated_stdio_spec("new"), None) + .await + .unwrap(); + assert_eq!(repeated["previousGeneration"], 2); + assert_eq!(repeated["replacementGeneration"], 3); + assert_eq!(repeated["toolChanges"]["unchanged"], 3); + assert_eq!( + stdio_process_id(invoke(runtime.snapshot(), "beta__probe").await, "beta"), + beta_pid + ); + assert_eq!( + stdio_process_id(invoke(runtime.snapshot(), "gamma__probe").await, "gamma"), + gamma_pid + ); +} From 133b0e26c18f2685ac2aea5a04589a5ea7c6b41e Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Fri, 20 Nov 2026 00:53:57 +0100 Subject: [PATCH 50/62] test(mcp): run isolated stdio process reload in Windows and macOS service CI --- src/bridge/hot_reload_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bridge/hot_reload_tests.rs b/src/bridge/hot_reload_tests.rs index e2f7e033..b8dfaa88 100644 --- a/src/bridge/hot_reload_tests.rs +++ b/src/bridge/hot_reload_tests.rs @@ -629,7 +629,7 @@ fn stdio_process_id(result: ToolResult, expected_version: &str) -> u32 { } #[tokio::test] -async fn isolated_real_stdio_process_reload_preserves_unrelated_sources() { +async fn service_isolated_real_stdio_process_reload_preserves_unrelated_sources() { // This fixture actually spawns three independent OS processes through the // production TokioChildProcess transport; no live Codexify is involved. let directory = tempfile::tempdir().unwrap(); From 17b0d3105d02fc8ffc149583e66e57918d7108b7 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Fri, 20 Nov 2026 00:59:23 +0100 Subject: [PATCH 51/62] test(mcp): verify retired stdio OS child exits and apply CI rustfmt --- src/bridge/hot_reload_tests.rs | 50 +++++++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/src/bridge/hot_reload_tests.rs b/src/bridge/hot_reload_tests.rs index b8dfaa88..a029da4d 100644 --- a/src/bridge/hot_reload_tests.rs +++ b/src/bridge/hot_reload_tests.rs @@ -602,7 +602,6 @@ async fn unrelated_in_flight_call_does_not_retain_a_retired_source() { task_b.abort(); } - fn isolated_stdio_spec(version: &str) -> crate::types::McpServerSpec { let fixture = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) .join("scripts") @@ -618,14 +617,30 @@ fn isolated_stdio_spec(version: &str) -> crate::types::McpServerSpec { } fn stdio_process_id(result: ToolResult, expected_version: &str) -> u32 { - assert!(!result.is_error, "stdio call failed: {}", result.joined_text()); + assert!( + !result.is_error, + "stdio call failed: {}", + result.joined_text() + ); let response = result.joined_text(); let (version, pid) = response .trim() .split_once(':') .unwrap_or_else(|| panic!("invalid stdio fixture response: {response}")); assert_eq!(version, expected_version); - pid.parse().expect("stdio fixture process id must be numeric") + pid.parse() + .expect("stdio fixture process id must be numeric") +} + +async fn stdio_process_still_alive(pid: u32) -> bool { + // Signal 0 checks existence without terminating or signaling a process. + tokio::process::Command::new("node") + .arg("-e") + .arg("try { process.kill(Number(process.argv[1]), 0); } catch { process.exit(1); }") + .arg(pid.to_string()) + .status() + .await + .is_ok_and(|status| status.success()) } #[tokio::test] @@ -675,7 +690,9 @@ async fn service_isolated_real_stdio_process_reload_preserves_unrelated_sources( drop(initial); let config_for_call = crate::config::default_config(std::env::temp_dir()); let original_pid = stdio_process_id( - old_probe.call(json!({}), &config_for_call, &SessionState::new()).await, + old_probe + .call(json!({}), &config_for_call, &SessionState::new()) + .await, "old", ); let beta_pid = stdio_process_id(invoke(runtime.snapshot(), "beta__probe").await, "beta"); @@ -710,17 +727,25 @@ async fn service_isolated_real_stdio_process_reload_preserves_unrelated_sources( assert_eq!(result["replacementGeneration"], 2); assert_eq!(result["toolChanges"]["added"], 1); assert_eq!(result["oldCallsDrained"], true); - assert_eq!(stdio_process_id(pending.await.unwrap(), "old"), original_pid); + assert_eq!( + stdio_process_id(pending.await.unwrap(), "old"), + original_pid + ); assert_eq!( stdio_process_id( - old_probe.call(json!({}), &config_for_call, &SessionState::new()).await, + old_probe + .call(json!({}), &config_for_call, &SessionState::new()) + .await, "old", ), original_pid, "a pinned retired generation must remain callable until released" ); let new_pid = stdio_process_id(invoke(runtime.snapshot(), "alpha__probe").await, "new"); - assert_ne!(new_pid, original_pid, "replacement must be a new OS process"); + assert_ne!( + new_pid, original_pid, + "replacement must be a new OS process" + ); assert_eq!( stdio_process_id(invoke(runtime.snapshot(), "alpha__new_tool").await, "new"), new_pid @@ -751,12 +776,23 @@ async fn service_isolated_real_stdio_process_reload_preserves_unrelated_sources( "an unsuccessful replacement must preserve the healthy process" ); + assert!( + stdio_process_still_alive(original_pid).await, + "retiring process must remain alive while a pinned old handle exists" + ); drop(old_probe); tokio::task::yield_now().await; assert!( retired.upgrade().is_none(), "the retired generation must be released after its last handle drops" ); + tokio::time::timeout(Duration::from_secs(5), async { + while stdio_process_still_alive(original_pid).await { + tokio::time::sleep(Duration::from_millis(25)).await; + } + }) + .await + .expect("retired OS child process remained alive after the last handle dropped"); let repeated = runtime .reload_with_spec("alpha", isolated_stdio_spec("new"), None) From 4df4f73b81b0d64087f93d447b7ee12b72494abd Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Fri, 20 Nov 2026 04:08:53 +0100 Subject: [PATCH 52/62] test(mcp): cover real GodPrompt 1.0.3 to 1.1.1 reload and rollback --- src/bridge/hot_reload_tests.rs | 115 +++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/src/bridge/hot_reload_tests.rs b/src/bridge/hot_reload_tests.rs index a029da4d..6425e9b5 100644 --- a/src/bridge/hot_reload_tests.rs +++ b/src/bridge/hot_reload_tests.rs @@ -810,3 +810,118 @@ async fn service_isolated_real_stdio_process_reload_preserves_unrelated_sources( gamma_pid ); } + + +/// Opt-in, isolated acceptance test using two real, read-only GodPrompt stdio +/// installs. Only initialize, tools/list, and ping are sent; no GodPrompt tool +/// calls, external APIs, operator configuration, or running service are touched. +/// +/// CI installs and selects the pinned published 1.0.3 / 1.1.1 npm versions. +/// A developer can instead supply two existing entrypoint paths locally. +#[tokio::test] +async fn service_local_godprompt_candidate_upgrade_and_failed_rollback() { + let old_entry = std::env::var_os("CODEXIFY_CANARY_OLD_GODPROMPT"); + let new_entry = std::env::var_os("CODEXIFY_CANARY_NEW_GODPROMPT"); + let (Some(old_entry), Some(new_entry)) = (old_entry, new_entry) else { + assert!( + std::env::var_os("CODEXIFY_CANARY_OLD_GODPROMPT").is_none() + && std::env::var_os("CODEXIFY_CANARY_NEW_GODPROMPT").is_none(), + "provide both isolated GodPrompt entrypoints, or neither" + ); + return; + }; + let node = std::env::var("CODEXIFY_CANARY_NODE").unwrap_or_else(|_| "node".into()); + for entry in [&old_entry, &new_entry] { + assert!( + std::path::Path::new(entry).is_file(), + "GodPrompt candidate stdio entrypoint does not exist" + ); + } + let make_candidate = |entry: &std::ffi::OsStr| crate::types::McpServerSpec { + command: Some(node.clone()), + args: vec![entry.to_string_lossy().into_owned()], + mode: Some(McpToolExposure::Direct), + startup_timeout_sec: Some(18.0), + ..Default::default() + }; + + let (beta_url, _, beta_server) = fixture("beta", false, Duration::ZERO, Duration::ZERO).await; + let (gamma_url, _, gamma_server) = + fixture("gamma", false, Duration::ZERO, Duration::ZERO).await; + let mut config = crate::config::default_config(std::env::temp_dir()); + config.mcp_servers.insert( + "godprompt_canary".into(), + make_candidate(old_entry.as_os_str()), + ); + config + .mcp_servers + .insert("beta".into(), spec(&beta_url, McpToolExposure::Direct)); + config + .mcp_servers + .insert("gamma".into(), spec(&gamma_url, McpToolExposure::Direct)); + + let bridge = connect_upstreams(&config).await; + assert_eq!( + bridge.runtime.snapshot().sources.len(), + 3, + "{:?}", + bridge.report + ); + let runtime = bridge.runtime.clone(); + drop(bridge); + let before = runtime.snapshot(); + let old = before.sources["godprompt_canary"].clone(); + let beta = before.sources["beta"].clone(); + let gamma = before.sources["gamma"].clone(); + let old_tool_count = old.tools.len(); + assert!(old_tool_count > 0, "old GodPrompt must advertise tools"); + drop(before); + + let result = runtime + .reload_with_spec( + "godprompt_canary", + make_candidate(new_entry.as_os_str()), + None, + ) + .await + .expect("replacement GodPrompt must initialize, discover tools and pass ping"); + assert_eq!(result["previousGeneration"], 1); + assert_eq!(result["replacementGeneration"], 2); + assert_eq!(result["ready"], true); + assert_eq!(result["previousVersion"], "1.0.3"); + assert_eq!(result["replacementVersion"], "1.1.1"); + let updated = runtime.snapshot(); + let new = updated.sources["godprompt_canary"].clone(); + assert!(!Arc::ptr_eq(&old, &new)); + assert!(Arc::ptr_eq(&beta, &updated.sources["beta"])); + assert!(Arc::ptr_eq(&gamma, &updated.sources["gamma"])); + assert!(new.tools.len() > 0, "new GodPrompt must advertise tools"); + println!( + "isolated GodPrompt {} -> {}: {} -> {} tools; timings {:?}; testing rollback", + result["previousVersion"], + result["replacementVersion"], + old_tool_count, + new.tools.len(), + result["timingsMs"] + ); + drop(updated); + + let mut broken = make_candidate(new_entry.as_os_str()); + broken.command = Some("codexify-nonexistent-godprompt-canary".into()); + assert_eq!( + runtime + .reload_with_spec("godprompt_canary", broken, None) + .await + .err(), + Some("INITIALIZE_FAILED") + ); + let after_rollback = runtime.snapshot(); + assert!(Arc::ptr_eq( + &new, + &after_rollback.sources["godprompt_canary"] + )); + assert!(Arc::ptr_eq(&beta, &after_rollback.sources["beta"])); + assert!(Arc::ptr_eq(&gamma, &after_rollback.sources["gamma"])); + beta_server.abort(); + gamma_server.abort(); +} From 39c21894922ddff44c28c77b0b971d432d32da37 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Fri, 20 Nov 2026 04:09:00 +0100 Subject: [PATCH 53/62] ci(mcp): run isolated real GodPrompt npm version switch as required gate --- .github/workflows/ci.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e2f87694..09bec8e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -102,6 +102,17 @@ jobs: - name: Test run: cargo test --all + - name: Test actual GodPrompt 1.0.3 to 1.1.1 hot reload and rollback + env: + CODEXIFY_CANARY_OLD_GODPROMPT: ${{ runner.temp }}/godprompt-old/node_modules/god-prompt-mcp/dist/stdio.js + CODEXIFY_CANARY_NEW_GODPROMPT: ${{ runner.temp }}/godprompt-new/node_modules/god-prompt-mcp/dist/stdio.js + run: | + npm install --prefix "$RUNNER_TEMP/godprompt-old" --no-save --no-package-lock --ignore-scripts --no-audit --no-fund god-prompt-mcp@1.0.3 + npm install --prefix "$RUNNER_TEMP/godprompt-new" --no-save --no-package-lock --ignore-scripts --no-audit --no-fund god-prompt-mcp@1.1.1 + test -f "$CODEXIFY_CANARY_OLD_GODPROMPT" + test -f "$CODEXIFY_CANARY_NEW_GODPROMPT" + cargo test --locked --lib service_local_godprompt_candidate_upgrade_and_failed_rollback -- --nocapture + - name: Test Rust and TypeScript browser companion contract env: CODEXIFY_TEST_BROWSER_COMPANION_BUN: ${{ steps.browser-bun.outputs.bun-path }} From eb029f3d8e571d28ce0605304adc357a350723da Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Fri, 20 Nov 2026 04:16:26 +0100 Subject: [PATCH 54/62] style(mcp): apply exact Rustfmt spacing in GodPrompt canary --- src/bridge/hot_reload_tests.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/bridge/hot_reload_tests.rs b/src/bridge/hot_reload_tests.rs index 6425e9b5..5560dc4a 100644 --- a/src/bridge/hot_reload_tests.rs +++ b/src/bridge/hot_reload_tests.rs @@ -811,7 +811,6 @@ async fn service_isolated_real_stdio_process_reload_preserves_unrelated_sources( ); } - /// Opt-in, isolated acceptance test using two real, read-only GodPrompt stdio /// installs. Only initialize, tools/list, and ping are sent; no GodPrompt tool /// calls, external APIs, operator configuration, or running service are touched. From 91cd90b3faf8e7e00c8947f05f2c156bd44b79ea Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Fri, 20 Nov 2026 04:25:23 +0100 Subject: [PATCH 55/62] style(mcp): satisfy strict Clippy in real GodPrompt canary --- src/bridge/hot_reload_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bridge/hot_reload_tests.rs b/src/bridge/hot_reload_tests.rs index 5560dc4a..38b5d4d1 100644 --- a/src/bridge/hot_reload_tests.rs +++ b/src/bridge/hot_reload_tests.rs @@ -894,7 +894,7 @@ async fn service_local_godprompt_candidate_upgrade_and_failed_rollback() { assert!(!Arc::ptr_eq(&old, &new)); assert!(Arc::ptr_eq(&beta, &updated.sources["beta"])); assert!(Arc::ptr_eq(&gamma, &updated.sources["gamma"])); - assert!(new.tools.len() > 0, "new GodPrompt must advertise tools"); + assert!(!new.tools.is_empty(), "new GodPrompt must advertise tools"); println!( "isolated GodPrompt {} -> {}: {} -> {} tools; timings {:?}; testing rollback", result["previousVersion"], From d91e73d974f9131bc569181d2b62d8b92eb10f8e Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Fri, 20 Nov 2026 05:04:58 +0100 Subject: [PATCH 56/62] ci: build isolated Windows hot reload preview artifact without deployment --- .../workflows/preview-hot-reload-windows.yml | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 .github/workflows/preview-hot-reload-windows.yml diff --git a/.github/workflows/preview-hot-reload-windows.yml b/.github/workflows/preview-hot-reload-windows.yml new file mode 100644 index 00000000..3b2b44bf --- /dev/null +++ b/.github/workflows/preview-hot-reload-windows.yml @@ -0,0 +1,46 @@ +name: Build experimental Codexify hot reload for Windows + +on: + push: + branches: + - feat/per-mcp-hot-reload-20261119 + paths: + - .github/workflows/preview-hot-reload-windows.yml + +permissions: + contents: read + +jobs: + preview-windows: + name: Package pinned Windows feature binary + runs-on: windows-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: Build from exact branch commit + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + cargo build --release --locked --bin codexify + if ($LASTEXITCODE -ne 0) { throw 'Cargo build failed' } + $out = Join-Path $env:RUNNER_TEMP 'codexify-hot-reload-preview' + New-Item -ItemType Directory -Force -Path $out | Out-Null + $binary = Join-Path (Get-Location) 'target\release\codexify.exe' + if (!(Test-Path -LiteralPath $binary -PathType Leaf)) { throw 'Expected executable missing' } + Copy-Item -LiteralPath $binary -Destination (Join-Path $out 'codexify.exe') + & (Join-Path $out 'codexify.exe') --help | Out-Null + if ($LASTEXITCODE -ne 0) { throw 'Preview binary self-probe failed' } + $hash = (Get-FileHash -LiteralPath (Join-Path $out 'codexify.exe') -Algorithm SHA256).Hash.ToLowerInvariant() + Set-Content -LiteralPath (Join-Path $out 'SHA256SUMS.txt') -Value ("$hash codexify.exe") -Encoding ascii + Set-Content -LiteralPath (Join-Path $out 'SOURCE_COMMIT.txt') -Value "${{ github.sha }}" -Encoding ascii + Set-Content -LiteralPath (Join-Path $out 'PREVIEW_NOTICE.txt') -Value 'EXPERIMENTAL binary from feature branch PR #11. Not an official release. Activate only after backing up the existing executable; coordinate with other Codexify workers.' -Encoding ascii + - uses: actions/upload-artifact@v4 + with: + name: codexify-pr11-windows-preview + path: ${{ runner.temp }}/codexify-hot-reload-preview/* + if-no-files-found: error + retention-days: 7 From e0670de5f45928ac4ebc8a00bdfdbfe268321caa Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Fri, 20 Nov 2026 05:17:20 +0100 Subject: [PATCH 57/62] test(mcp): allow Windows runner stdio startup under resource contention --- src/bridge/hot_reload_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bridge/hot_reload_tests.rs b/src/bridge/hot_reload_tests.rs index 38b5d4d1..5dc8b3e3 100644 --- a/src/bridge/hot_reload_tests.rs +++ b/src/bridge/hot_reload_tests.rs @@ -611,7 +611,7 @@ fn isolated_stdio_spec(version: &str) -> crate::types::McpServerSpec { command: Some("node".into()), args: vec![fixture.to_string_lossy().into_owned(), version.into()], mode: Some(McpToolExposure::Direct), - startup_timeout_sec: Some(8.0), + startup_timeout_sec: Some(25.0), ..Default::default() } } From ce590c5206717216c1ce9f1251c4dc7e8df5bf29 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Fri, 20 Nov 2026 12:38:33 +0100 Subject: [PATCH 58/62] fix(mcp): parse and validate existing browser Send gate config in preview --- src/markdown_chat/mod.rs | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/markdown_chat/mod.rs b/src/markdown_chat/mod.rs index 5e11b839..76e6b01b 100644 --- a/src/markdown_chat/mod.rs +++ b/src/markdown_chat/mod.rs @@ -178,6 +178,9 @@ pub struct AgentAutomationConfig { #[serde(skip_serializing_if = "Option::is_none")] pub browser_simplicity: Option, pub recovery_enhancements: bool, + /// Preserve the operator-configured browser Send gate in the v2.4 preview. + #[serde(skip_serializing_if = "Option::is_none")] + pub provider_send_gate_path: Option, } impl Default for AgentAutomationConfig { @@ -204,6 +207,7 @@ impl Default for AgentAutomationConfig { minimal_ui: false, browser_simplicity: None, recovery_enhancements: false, + provider_send_gate_path: None, } } } @@ -227,6 +231,13 @@ impl AgentAutomationConfig { fn validate(&self) -> Result<(), String> { self.retry_timings.validate()?; + if let Some(path) = &self.provider_send_gate_path + && (path.len() > 4096 + || !std::path::Path::new(path).is_absolute() + || path.contains('\0')) + { + return Err("agentChat.automation.providerSendGatePath must be an absolute path".into()); + } if !(1..=4).contains(&self.browser_action_lanes) { return Err("agentChat.automation.browserActionLanes must be between 1 and 4".into()); } @@ -1051,6 +1062,27 @@ mod tests { } } + #[test] + fn browser_send_gate_path_round_trips_and_rejects_relative_paths() { + let absolute = if cfg!(windows) { + r"C:\codexify\browser-send-gate.json" + } else { + "/tmp/codexify/browser-send-gate.json" + }; + let config: AgentAutomationConfig = + serde_json::from_value(serde_json::json!({"providerSendGatePath": absolute})).unwrap(); + assert_eq!(config.provider_send_gate_path.as_deref(), Some(absolute)); + assert!(config.validate().is_ok()); + let encoded = serde_json::to_value(&config).unwrap(); + assert_eq!(encoded["providerSendGatePath"], absolute); + for invalid in ["", "relative/send-gate.json"] { + let config: AgentAutomationConfig = + serde_json::from_value(serde_json::json!({"providerSendGatePath": invalid})) + .unwrap(); + assert!(config.validate().is_err()); + } + } + #[test] fn new_chat_project_configuration_is_optional_and_validated() { assert!(AgentAutomationConfig::default().validate().is_ok()); From 29650c5b479349a5e14e4d446c85fcd79fa78023 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Fri, 20 Nov 2026 12:38:47 +0100 Subject: [PATCH 59/62] fix(mcp): preserve existing Send-gate environment for browser companion --- src/agent_automation/install.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/agent_automation/install.rs b/src/agent_automation/install.rs index 05c9a0a3..87357e53 100644 --- a/src/agent_automation/install.rs +++ b/src/agent_automation/install.rs @@ -326,6 +326,9 @@ impl BrowserCompanionManager { .stdin(Stdio::null()) .stdout(Stdio::from(log)) .stderr(Stdio::from(stderr)); + if let Some(path) = &self.config.provider_send_gate_path { + command.env("CODEXIFY_BROWSER_SEND_GATE_PATH", path); + } configure_detached_process(&mut command); let mut child = command .spawn() From 85a6608e85e86c705a482064681c9e97c54a5055 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Fri, 20 Nov 2026 12:39:01 +0100 Subject: [PATCH 60/62] ci(mcp): rebuild Windows preview when browser Send-gate compatibility changes --- .github/workflows/preview-hot-reload-windows.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/preview-hot-reload-windows.yml b/.github/workflows/preview-hot-reload-windows.yml index 3b2b44bf..16ca2cfa 100644 --- a/.github/workflows/preview-hot-reload-windows.yml +++ b/.github/workflows/preview-hot-reload-windows.yml @@ -6,6 +6,8 @@ on: - feat/per-mcp-hot-reload-20261119 paths: - .github/workflows/preview-hot-reload-windows.yml + - src/markdown_chat/mod.rs + - src/agent_automation/install.rs permissions: contents: read From 8c305401150f9641d4db7ffc32e1c950503f6b4b Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Fri, 20 Nov 2026 12:54:54 +0100 Subject: [PATCH 61/62] style(mcp): apply CI Rustfmt for browser Send-gate compatibility --- src/markdown_chat/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/markdown_chat/mod.rs b/src/markdown_chat/mod.rs index 76e6b01b..2104b2ba 100644 --- a/src/markdown_chat/mod.rs +++ b/src/markdown_chat/mod.rs @@ -236,7 +236,9 @@ impl AgentAutomationConfig { || !std::path::Path::new(path).is_absolute() || path.contains('\0')) { - return Err("agentChat.automation.providerSendGatePath must be an absolute path".into()); + return Err( + "agentChat.automation.providerSendGatePath must be an absolute path".into(), + ); } if !(1..=4).contains(&self.browser_action_lanes) { return Err("agentChat.automation.browserActionLanes must be between 1 and 4".into()); From 29d4b37cb152fb84a5573c82979dc6af74a12227 Mon Sep 17 00:00:00 2001 From: Tommy <104433268+AKzar1el@users.noreply.github.com> Date: Sat, 21 Nov 2026 00:53:17 +0100 Subject: [PATCH 62/62] docs(mcp): clarify exact config source keys for hot reload --- docs/MCP_HOT_RELOAD.md | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/docs/MCP_HOT_RELOAD.md b/docs/MCP_HOT_RELOAD.md index a2feb807..75b2476d 100644 --- a/docs/MCP_HOT_RELOAD.md +++ b/docs/MCP_HOT_RELOAD.md @@ -6,10 +6,15 @@ service without separate approval and validation. ## Scope -`reload_mcp` reloads exactly one existing upstream source, identified by the -key in the resolved `mcpServers` configuration. A caller provides -`{"source":"god_prompt_mcp"}`; it cannot provide a command, executable, -working directory, URL, environment variables, or credentials. +`reload_mcp` reloads exactly one existing upstream source. Its `source` +argument must match the **exact configured MCP server key** from the resolved +`mcpServers` configuration (including inherited Codex MCP sources). For example, +if the configured key is `god-prompt-mcp`, call +`{"source":"god-prompt-mcp"}`. Do not substitute a normalized catalog source ID +such as `god_prompt_mcp` from `mcp_list_sources`: it is not necessarily the +configuration key and can return `SOURCE_UNKNOWN` without changing the running +connection. The call cannot supply a command, executable, working directory, +URL, environment variables, or credentials. The server rereads its operator configuration (including inherited Codex MCP sources) **read-only** using its startup CLI selection. A target that was @@ -77,8 +82,9 @@ and generated skill after reload. ## Failure and boundary conditions -- `SOURCE_UNKNOWN` / `SOURCE_REMOVED` / `SOURCE_DISABLED`: do not spawn - an arbitrary process or reinterpret the input as a path. +- `SOURCE_UNKNOWN` / `SOURCE_REMOVED` / `SOURCE_DISABLED`: verify the + exact configured MCP key, not a normalized catalog ID. Do not spawn an + arbitrary process or reinterpret the input as a path. - `RELOAD_IN_PROGRESS`: no duplicate replacement of the same source. - `CONFIG_INVALID`, `CONFIG_CLI_INVALID`, `INITIALIZE_FAILED`, `READINESS_FAILED`: old healthy generation is retained.