From 09d25c79760e59250def7a523d88c0b23c924848 Mon Sep 17 00:00:00 2001 From: fylorn <249551762+fylorn@users.noreply.github.com> Date: Thu, 12 Nov 2026 12:38:03 +0800 Subject: [PATCH 1/5] feat(gateway): negotiate upstream streaming capabilities Retry one explicitly rejected response mode using the alternative and convert the result to the client protocol. Learn only complete successful answers, sharing the existing automatic repair budget. --- crates/tw-gateway/src/lib.rs | 1 + crates/tw-gateway/src/server/pipeline/hop.rs | 131 ++++- .../tw-gateway/src/server/pipeline/relay.rs | 31 + crates/tw-gateway/src/state.rs | 4 + crates/tw-gateway/src/stream_mode.rs | 501 ++++++++++++++++ crates/tw-gateway/tests/compressed.rs | 6 +- crates/tw-gateway/tests/stream_negotiation.rs | 533 ++++++++++++++++++ docs/config.md | 20 + docs/config.zh-CN.md | 6 + 9 files changed, 1228 insertions(+), 5 deletions(-) create mode 100644 crates/tw-gateway/src/stream_mode.rs create mode 100644 crates/tw-gateway/tests/stream_negotiation.rs diff --git a/crates/tw-gateway/src/lib.rs b/crates/tw-gateway/src/lib.rs index 610be338..449fe9ad 100644 --- a/crates/tw-gateway/src/lib.rs +++ b/crates/tw-gateway/src/lib.rs @@ -64,6 +64,7 @@ pub mod slots; pub mod slow; mod splice; pub mod state; +pub mod stream_mode; pub mod throttle; pub mod traffic; pub mod translate; diff --git a/crates/tw-gateway/src/server/pipeline/hop.rs b/crates/tw-gateway/src/server/pipeline/hop.rs index 97e6f8b0..5a1285ee 100644 --- a/crates/tw-gateway/src/server/pipeline/hop.rs +++ b/crates/tw-gateway/src/server/pipeline/hop.rs @@ -58,6 +58,8 @@ pub(super) struct Served<'a> { /// 内容、在开头报了限流以外的错,原样交了出去。成败的账等回程看到第一段内容才记(见 /// `relay`)。没有内容就超时了记失败,和压着开头时一样 pub(super) unconfirmed: bool, + /// Learn an alternative only after a complete, clean response. + pub(super) stream_trial: Option<(crate::stream_mode::Key, bool)>, } /// 这个请求的着落。 @@ -200,6 +202,7 @@ pub(super) async fn try_upstreams<'a>( let mut repair_used = false; let mut history_trial: Option<(crate::history_memory::Key, crate::history_memory::Policy)> = None; + let mut stream_trial: Option<(crate::stream_mode::Key, bool)> = None; // 客户端收的是 SSE 流:压着开头过了时限先交响应头、发保活(见 `super::commit`)。最后一家的 // 开头只在这时为重试压着 let early = super::commit::early(state, req, reading).is_some(); @@ -713,6 +716,48 @@ pub(super) async fn try_upstreams<'a>( }); } + // Keep the client mode on the response session; change only this hop's + // wire mode. A trial and a cached choice use the same conversion path. + let stream_dialect = wire(req, &out).filter(|_| { + reading.generates + && !counting + && !out.chatgpt + && asked.decoded.is_some_and(|d| d.is_ok()) + }); + let stream_key = stream_dialect.and_then(|dialect| { + crate::stream_mode::Key::of(provider, &sent_model, dialect, &out.path, &out.body) + }); + let preferred_stream = stream_key.as_ref().and_then(|key| { + stream_trial + .as_ref() + .filter(|(k, _)| k == key) + .map(|(_, mode)| *mode) + .or_else(|| rt.stream_modes.lookup(key, now_ms())) + }); + if let (Some(dialect), Some(mode)) = (stream_dialect, preferred_stream) + && crate::stream_mode::requested(dialect, &out.path, &out.body) != Some(mode) + && let Some(body) = + crate::stream_mode::rewrite(dialect, &mut out.path, &mut out.query, &out.body, mode) + { + out.body = Bytes::from(body); + out.reshaped = true; + mode_session(&asked, provider, &sent_model, dialect, &mut out); + tracing::info!(provider = %provider.name, model = %sent_model, stream = mode, + "using an alternative upstream response mode"); + } + let stream_sent = + stream_dialect.and_then(|d| crate::stream_mode::requested(d, &out.path, &out.body)); + let stream_trial_applied = stream_key.as_ref().is_some_and(|key| { + stream_trial + .as_ref() + .is_some_and(|(trial, mode)| trial == key && stream_sent == Some(*mode)) + }); + if stream_trial_applied && let Some(dialect) = stream_dialect { + // Relearning may return to the client's original mode. Still keep + // a session so the complete trial response can be validated. + mode_session(&asked, provider, &sent_model, dialect, &mut out); + } + // 出站脱敏的拦截档:换掉**这一跳真正发出去的那一份**(可能转换过 // 格式)。规则是全局的,每一跳换掉的是同一批东西;**接着原文那本账换**, // 同一个值在每一跳、在存下来的那份请求里都是同一个占位符。插件改过的一跳接着 @@ -826,7 +871,7 @@ pub(super) async fn try_upstreams<'a>( .map(|r| (r.body.clone(), ledger.clone())); // 这一跳接下了的话,回答钩子要的:插件改了名的是改过的 let sent_model = effective_set.model.clone().unwrap_or(sent_model); - let (attempt, bridge) = (chain.len(), plugged.bridge); + let attempt = chain.len(); // 这一跳在报文记录里记到哪儿(见 `crate::content`):尝试链上的第几跳,从 1 数 let tap = crate::content::Tap::new( req.seat.get(), @@ -987,6 +1032,37 @@ pub(super) async fn try_upstreams<'a>( if !matches!(verdict, Verdict::Failed(Cause::RateLimited { .. })) { to_throttle(state, &provider.name, sent_at, verdict); } + if let (Some(key), Some(mode)) = (&stream_key, stream_sent) + && crate::stream_mode::refusal(status, &head, mode) + { + // A remembered capability may have changed. Forget even + // when this request has already spent its repair budget. + rt.stream_modes.forget(key); + if !repair_used && tokio::time::Instant::now() < until { + repair_used = true; + stream_trial = Some((key.clone(), !mode)); + tap.said(&head); + state.note_proxy_ok(&provider.proxy); + note_health( + &state.bus, + &state.health, + &provider.name, + state.health.record_success(&provider.name), + ); + chain.push(hop( + &provider.name, + model.clone(), + tw_api::AttemptOutcome::Status, + status, + hop_started, + )); + // Account and record a separate attempt, reusing the + // slot. Other providers still start from the original. + effort_hold = Some((name, slot, probe.take())); + queue.push_front(name); + continue; + } + } if !repair_used && tokio::time::Instant::now() < until && let (Some(key), Some(mut policy)) = (history_key.clone(), history_policy.clone()) && let Some(fields) = crate::history_memory::refusal(status, &head, &body) @@ -1194,19 +1270,37 @@ pub(super) async fn try_upstreams<'a>( provider, model: sent_model, attempt, - bridge, + bridge: plugged.bridge, ledger, session: out.session, refusal, slot, sent_at, unconfirmed: false, + stream_trial: None, }); break; } } } Ok(r) => { + // Some compatible APIs silently ignore `stream`. Successful + // responses in the other mode need conversion, not a resend. + if out.session.is_none() + && stream_key.is_some() + && let (Some(dialect), Some(Ok(decoded))) = (stream_dialect, asked.decoded) + && let Some(ct) = r + .headers() + .get(http::header::CONTENT_TYPE) + .and_then(|v| v.to_str().ok()) + { + let sse = ct.starts_with("text/event-stream"); + let json = ct.starts_with("application/json") + && dialect != tw_dialect::ir::Dialect::Gemini; + if (sse && !decoded.request.stream) || (json && decoded.request.stream) { + mode_session(&asked, provider, &sent_model, dialect, &mut out); + } + } // 回答的开头:**第一段内容到之前**上游在流里报的错照样换下一家(报的是限流就排队 // 等它,和回 429 一样),一直没有内容也换(见 `opening`)。 // @@ -1221,7 +1315,8 @@ pub(super) async fn try_upstreams<'a>( let held = hold.as_ref().is_some_and(|h| { history_key.as_ref().is_some_and(|k| { history_trial.as_ref().is_some_and(|(trial, _)| trial == k) - }) || !last + }) || stream_trial_applied + || !last || (matches!(h, Hold::Stream(..)) && tokio::time::Instant::now() < until && (early || state.throttles.wary(&provider.name))) @@ -1470,13 +1565,14 @@ pub(super) async fn try_upstreams<'a>( provider, model: sent_model, attempt, - bridge, + bridge: plugged.bridge, ledger, session: out.session, refusal: None, slot, sent_at, unconfirmed, + stream_trial: stream_trial.clone().filter(|_| stream_trial_applied), }); break; } @@ -2174,6 +2270,33 @@ fn unsendable_tool( Some(GatewayError::new(crate::error::Source::Request, msg)) } +/// A native-protocol hop normally needs no session. Changing only its response +/// mode still needs the client's shape, tools and original stream preference. +fn mode_session( + asked: &Asked<'_>, + provider: &tw_config::Provider, + model: &str, + dialect: tw_dialect::ir::Dialect, + out: &mut Outbound, +) { + if out.session.is_none() + && let Some(Ok(decoded)) = asked.decoded + { + let mut session = decoded + .encode_with_history( + &tw_dialect::ir::Target { + dialect, + official: tw_dialect::official::is_official_host(&provider.base_url), + default_max_tokens: 0, + }, + &crate::translate::history_target(provider, model), + ) + .session; + session.model = model.into(); + out.session = Some(session); + } +} + /// 把这一跳的请求(客户端那种格式,插件改过的话是改过的,见 [`Asked`])改成要发的 /// 样子:同格式时只做参数改写,跨格式时转换。转换不了就换下一家:同格式的上游可能 /// 还在后面。`id` 是这个请求的号,做了转换、丢了字段要报在它上面。 diff --git a/crates/tw-gateway/src/server/pipeline/relay.rs b/crates/tw-gateway/src/server/pipeline/relay.rs index dd141d0b..7891ddc2 100644 --- a/crates/tw-gateway/src/server/pipeline/relay.rs +++ b/crates/tw-gateway/src/server/pipeline/relay.rs @@ -53,6 +53,7 @@ pub(super) fn respond( slot, sent_at, unconfirmed, + stream_trial, attempt, .. } = served; @@ -139,6 +140,9 @@ pub(super) fn respond( session.as_ref(), ); let upstream_dialect = upstream_dialect(req, session.as_ref()); + let mut mode_completion = (stream_trial.is_some() && plan.is_sse) + .then(|| crate::stream_mode::Completion::new(upstream_dialect)); + let stream_modes = rt.stream_modes.clone(); // 成功的流才有「第一个 token」:整包的一起到,错误不是回答 if generates && plan.is_sse && status.is_success() { ending.streaming(upstream_dialect, &provider.name, &state.pace); @@ -332,6 +336,9 @@ pub(super) fn respond( // **旁路嗅探和留档,不缓冲**:看的是上游原话(带占位符的那一版), // usage 和请求记录不受后面的还原、转换影响。先认错误,再记这一家的成败。 ending.feed(&chunk); + if let Some(proof) = mode_completion.as_mut() { + proof.feed(&chunk); + } // 真内容才重新计时:心跳、开头的例行事件不算 if pulse.feed(&chunk) { said = true; @@ -381,6 +388,10 @@ pub(super) fn respond( } else { (Some(chunks), Some(_slot)) }; + let mode_complete = stream_trial.is_some() && match mode_completion { + Some(proof) => proof.finish(), + None => crate::stream_mode::whole_answer(upstream_dialect, &relay.whole), + }; let (tail, denied) = relay.finish(broke.is_some()).await; // 回答里的命中这时报:一次回答一条规则一条 relay.report_answer(); @@ -431,6 +442,11 @@ pub(super) fn respond( } match broke { None => { + if mode_complete && ending.upstream_fault().is_none() + && let Some((key, mode)) = stream_trial + { + stream_modes.accepted(key, mode, crate::server::now_ms()); + } if !tail.is_empty() { yield Ok::(Bytes::from(tail)); } @@ -1025,6 +1041,21 @@ impl Relay { Vec::new() } else { self.whole.extend_from_slice(&tail); + // Some compatible APIs put an error inside HTTP 200. A + // mode conversion must not turn it into an empty answer. + if status.is_success() + && let Some(message) = + tw_dialect::convert::error_message(s.upstream, &self.whole) + { + let message: String = message.chars().take(500).collect(); + return ( + Vec::new(), + Some(GatewayError::upstream(msg!( + "gw.upstream.response_error", upstream = self.provider.clone(), message = message => + "Upstream `{upstream}` returned an error in the response: {message}" + ))), + ); + } if !status.is_success() { s.error(status.as_u16(), &self.whole) } else if s.stream { diff --git a/crates/tw-gateway/src/state.rs b/crates/tw-gateway/src/state.rs index 54267c62..ddfb47de 100644 --- a/crates/tw-gateway/src/state.rs +++ b/crates/tw-gateway/src/state.rs @@ -40,6 +40,8 @@ pub struct Runtime { pub redact: Arc, /// Verified request-side omissions; replaced on configuration reload. pub history_memory: Arc, + /// Accepted alternatives to explicitly rejected upstream stream modes. + pub stream_modes: Arc, /// 出站脱敏的扫描缓存:对话历史里没变的消息不再扫(见 [`tw_guard::redact::cache`])。 /// **跟着规则一起换**:重载配置换上一份空的,旧规则下扫出来的结果不会留到新规则下 pub scan_cache: Arc, @@ -127,6 +129,7 @@ impl Runtime { allow, redact: Arc::new(redact), history_memory: Arc::new(crate::history_memory::Cache::default()), + stream_modes: Arc::new(crate::stream_mode::Cache::default()), scan_cache: Arc::new(tw_guard::redact::cache::ScanCache::new()), tools: Arc::new(tools), content: Arc::new(content), @@ -146,6 +149,7 @@ impl Runtime { // 规则没变,扫过的照样作数 scan_cache: self.scan_cache.clone(), history_memory: self.history_memory.clone(), + stream_modes: self.stream_modes.clone(), tools: self.tools.clone(), content: self.content.clone(), answer: self.answer.clone(), diff --git a/crates/tw-gateway/src/stream_mode.rs b/crates/tw-gateway/src/stream_mode.rs new file mode 100644 index 00000000..2cb2391c --- /dev/null +++ b/crates/tw-gateway/src/stream_mode.rs @@ -0,0 +1,501 @@ +//! Negotiate an upstream's response mode only after an explicit rejection. +//! The client mode stays on the conversion session; only the outbound request +//! changes. An accepted alternative is remembered locally until reload or expiry. + +use std::collections::HashMap; +use std::sync::Mutex; + +use serde_json::Value; +use tw_dialect::ir::Dialect; + +const TTL_MS: u64 = 24 * 60 * 60 * 1000; +const MAX: usize = 1024; + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct Key { + provider: String, + identity: String, + model: String, + dialect: Dialect, + path: String, + options: String, +} + +impl Key { + pub fn of( + provider: &tw_config::Provider, + model: &str, + dialect: Dialect, + path: &str, + body: &[u8], + ) -> Option { + let mut options: Value = serde_json::from_slice(body).ok()?; + if options.get("stream").is_some_and(|v| !v.is_boolean()) + || options + .get("stream_options") + .is_some_and(|v| !v.is_object() && !v.is_null()) + { + return None; + } + // The response IR represents one text/reasoning/tool answer. Do not + // silently discard richer native response contracts on a mode change. + if options + .get("n") + .and_then(Value::as_u64) + .is_some_and(|n| n > 1) + || options.get("logprobs").and_then(Value::as_bool) == Some(true) + || options.get("top_logprobs").is_some_and(|v| !v.is_null()) + || options.get("audio").is_some_and(|v| !v.is_null()) + || options + .get("modalities") + .and_then(Value::as_array) + .is_some_and(|a| a.iter().any(|v| v.as_str() != Some("text"))) + || options + .pointer("/generationConfig/candidateCount") + .and_then(Value::as_u64) + .is_some_and(|n| n > 1) + || options + .pointer("/generationConfig/responseLogprobs") + .and_then(Value::as_bool) + == Some(true) + || options + .pointer("/generationConfig/responseModalities") + .and_then(Value::as_array) + .is_some_and(|a| a.iter().any(|v| v.as_str() != Some("TEXT"))) + || options + .get("include") + .and_then(Value::as_array) + .is_some_and(|a| { + a.iter() + .any(|v| v.as_str().is_some_and(|s| s.contains("logprobs"))) + }) + { + return None; + } + let fields = options.as_object_mut()?; + // Keep capability-affecting options (tools, thinking, modalities, etc.) + // isolated without retaining prompts or splitting the two stream modes. + for name in [ + "model", + "stream", + "stream_options", + "messages", + "input", + "instructions", + "system", + "contents", + ] { + fields.remove(name); + } + let mut identity = blake3::Hasher::new(); + let mut add = |value: &str| { + identity.update(&(value.len() as u64).to_le_bytes()); + identity.update(value.as_bytes()); + }; + add(&provider.base_url); + add(&provider.credential_identity()); + if let Some(key) = &provider.key { + add(&key.resolve().ok()?); + } + if let Some(oauth) = &provider.oauth { + add(&oauth.refresh); + } + for header in provider.headers.iter() { + add(&header.name.to_ascii_lowercase()); + add(&header.value.resolve().ok()?); + } + if let Some(aws) = &provider.aws { + for secret in [ + &aws.access_key_id, + &aws.secret_access_key, + &aws.session_token, + ] + .into_iter() + .flatten() + { + add(&secret.resolve().ok()?); + } + } + Some(Self { + provider: provider.name.clone(), + identity: identity.finalize().to_hex().to_string(), + model: model.into(), + dialect, + path: match dialect { + Dialect::Gemini => path.replace(":streamGenerateContent", ":generateContent"), + Dialect::Bedrock => path.trim_end_matches("-stream").into(), + _ => path.into(), + }, + options: blake3::hash(options.to_string().as_bytes()) + .to_hex() + .to_string(), + }) + } +} + +#[derive(Default)] +pub struct Cache(Mutex>); + +impl Cache { + pub fn lookup(&self, key: &Key, now: u64) -> Option { + let mut entries = self.0.lock().unwrap(); + entries.retain(|_, (_, at)| now.saturating_sub(*at) < TTL_MS); + entries.get(key).map(|(stream, _)| *stream) + } + + pub fn accepted(&self, key: Key, stream: bool, now: u64) { + let mut entries = self.0.lock().unwrap(); + if entries.len() >= MAX + && !entries.contains_key(&key) + && let Some(oldest) = entries + .iter() + .min_by_key(|(_, (_, at))| *at) + .map(|(k, _)| k.clone()) + { + entries.remove(&oldest); + } + entries.insert(key, (stream, now)); + } + + pub fn forget(&self, key: &Key) { + self.0.lock().unwrap().remove(key); + } +} + +pub fn requested(dialect: Dialect, path: &str, body: &[u8]) -> Option { + match dialect { + Dialect::Gemini => Some(path.ends_with(":streamGenerateContent")), + Dialect::Bedrock => Some(path.ends_with("/converse-stream")), + _ => { + let v: Value = serde_json::from_slice(body).ok()?; + match v.get("stream") { + None => Some(false), + Some(v) => v.as_bool(), + } + } + } +} + +/// Change the wire mode without round-tripping any prompt through the IR. +/// Gemini and Bedrock put the mode in the endpoint, not in the JSON body. +pub fn rewrite( + dialect: Dialect, + path: &mut String, + query: &mut Option, + body: &[u8], + stream: bool, +) -> Option> { + match dialect { + Dialect::Gemini => { + let base = path + .strip_suffix(":streamGenerateContent") + .or_else(|| path.strip_suffix(":generateContent"))?; + *path = format!( + "{base}:{}", + if stream { + "streamGenerateContent" + } else { + "generateContent" + } + ); + let mut args: Vec<_> = query + .as_deref() + .unwrap_or_default() + .split('&') + .filter(|s| !s.is_empty() && s.split('=').next() != Some("alt")) + .map(str::to_owned) + .collect(); + if stream { + args.push("alt=sse".into()); + } + *query = (!args.is_empty()).then(|| args.join("&")); + Some(body.to_vec()) + } + Dialect::Bedrock => { + let base = path + .strip_suffix("/converse-stream") + .or_else(|| path.strip_suffix("/converse"))?; + *path = format!( + "{base}/{}", + if stream { + "converse-stream" + } else { + "converse" + } + ); + Some(body.to_vec()) + } + _ => { + let mut v: Value = serde_json::from_slice(body).ok()?; + let fields = v.as_object_mut()?; + fields.insert("stream".into(), Value::Bool(stream)); + if !stream { + fields.remove("stream_options"); + } else if dialect == Dialect::Chat { + // A whole-answer client expects usage too. Chat only sends the + // final usage chunk when explicitly requested. + let options = fields + .entry("stream_options") + .or_insert_with(|| serde_json::json!({})); + if options.is_null() { + *options = serde_json::json!({}); + } + options + .as_object_mut()? + .insert("include_usage".into(), Value::Bool(true)); + } + serde_json::to_vec(&v).ok() + } + } +} + +/// Only a top-level API error describing the actual rejected mode is evidence. +/// Never infer capabilities from rate limits, answer text, or stream_options. +pub fn refusal(status: u16, body: &[u8], sent_stream: bool) -> bool { + if !matches!(status, 400 | 422 | 501) || body.len() > 64 * 1024 { + return false; + } + let Ok(v) = serde_json::from_slice::(body) else { + return false; + }; + let e = v.get("error").filter(|e| e.is_object()).unwrap_or(&v); + if let Some(param) = e.get("param").and_then(Value::as_str) + && param != "stream" + { + return false; + } + let message = e.get("message").and_then(Value::as_str).unwrap_or_default(); + let text = message.to_ascii_lowercase(); + if text.contains("rate limit") || text.contains("quota") || text.contains("overload") { + return false; + } + let code = e.get("code").and_then(Value::as_str).unwrap_or_default(); + let codes: &[&str] = if sent_stream { + &[ + "streaming_not_supported", + "stream_not_supported", + "unsupported_streaming", + ] + } else { + &[ + "non_streaming_not_supported", + "stream_required", + "streaming_required", + ] + }; + if codes.contains(&code) { + return true; + } + // Intentionally narrow phrases: "streaming" inside "non-streaming" must + // never be mistaken for a refusal in the opposite direction. + let text = text + .replace("non streaming", "non-streaming") + .replace("nonstreaming", "non-streaming"); + let phrases: &[&str] = if sent_stream { + &[ + "streaming is not supported", + "streaming not supported", + "streaming is unsupported", + "does not support streaming", + "doesn't support streaming", + "only supports non-streaming", + "only non-streaming requests are supported", + "only non-streaming responses are supported", + "only non-streaming is supported", + "supports only non-streaming", + "stream must be false", + "stream must be set to false", + "stream=false is required", + ] + } else { + &[ + "non-streaming is not supported", + "non-streaming not supported", + "non-streaming is unsupported", + "does not support non-streaming", + "doesn't support non-streaming", + "only supports streaming", + "only streaming requests are supported", + "only streaming responses are supported", + "only streaming is supported", + "supports only streaming", + "stream must be true", + "stream must be set to true", + "stream=true is required", + ] + }; + phrases.iter().any(|p| { + text.match_indices(p).any(|(at, _)| { + at == 0 + || !text.as_bytes()[at - 1].is_ascii_alphanumeric() + && !matches!(text.as_bytes()[at - 1], b'-' | b'_') + }) + }) +} + +/// Evidence of a complete stream, used only while learning a new mode. +pub struct Completion { + dialect: Dialect, + decoder: tw_dialect::frame::Decoder, + ended: bool, +} + +impl Completion { + pub fn new(dialect: Dialect) -> Self { + Self { + dialect, + decoder: Default::default(), + ended: false, + } + } + + fn frames(&mut self, frames: Vec) { + for f in frames { + self.ended |= tw_dialect::convert::ends_answer(self.dialect, &f); + if self.dialect == Dialect::Gemini { + self.ended |= serde_json::from_str::(&f.data) + .ok() + .is_some_and(|v| { + v.get("candidates") + .and_then(Value::as_array) + .is_some_and(|a| { + a.iter().any(|c| { + c.get("finishReason").and_then(Value::as_str).is_some() + }) + }) + }); + } else if self.dialect == Dialect::Bedrock { + self.ended |= f.event.as_deref() == Some("messageStop"); + } + } + } + + pub fn feed(&mut self, bytes: &[u8]) { + let frames = self.decoder.feed(bytes); + self.frames(frames); + } + + pub fn finish(mut self) -> bool { + let frames = self.decoder.flush(); + self.frames(frames); + self.ended + } +} + +pub fn whole_answer(dialect: Dialect, body: &[u8]) -> bool { + let Ok(v) = serde_json::from_slice::(body) else { + return false; + }; + if v.get("error").is_some_and(|e| !e.is_null()) + || v.get("status") + .and_then(Value::as_str) + .is_some_and(|s| matches!(s, "failed" | "cancelled")) + { + return false; + } + let field = match dialect { + Dialect::Chat => "choices", + Dialect::Anthropic => "content", + Dialect::Responses => "output", + Dialect::Gemini => "candidates", + Dialect::Bedrock => { + return v + .pointer("/output/message/content") + .is_some_and(Value::is_array); + } + }; + v.get(field).is_some_and(Value::is_array) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn only_explicit_refusals_for_the_sent_mode_trigger_a_switch() { + let error = |text| json!({"error":{"message":text}}).to_string(); + for (stream, text) in [ + (false, "This model only supports streaming."), + (false, "Non-streaming is not supported."), + (false, "stream must be set to true"), + (true, "This model does not support streaming."), + (true, "stream must be false"), + (true, "Only non-streaming requests are supported."), + ] { + assert!(refusal(400, error(text).as_bytes(), stream), "{text}"); + assert!(!refusal(400, error(text).as_bytes(), !stream), "{text}"); + for status in [200, 401, 403, 404, 429, 500, 503] { + assert!(!refusal(status, error(text).as_bytes(), stream)); + } + } + for text in [ + "Rate limit exceeded", + "streaming failed", + "invalid stream_options", + "Quota exceeded; only streaming allowed", + ] { + for stream in [false, true] { + assert!(!refusal(400, error(text).as_bytes(), stream)); + } + } + assert!(!refusal( + 400, + br#"{"choices":[{"message":{"content":"only streaming"}}]}"#, + false + )); + assert!(!refusal( + 400, + br#"{"error":{"param":"stream_options","message":"streaming not supported"}}"#, + true + )); + } + + #[test] + fn cache_expires_and_isolates_models_accounts_apis_and_options() { + let p = tw_config::Provider { + name: "up".into(), + base_url: "https://relay.example".into(), + ..Default::default() + }; + let key = Key::of(&p, "m", Dialect::Chat, "/v1/chat/completions", b"{}").unwrap(); + let cache = Cache::default(); + cache.accepted(key.clone(), true, 1); + assert_eq!(cache.lookup(&key, 2), Some(true)); + for (model, dialect, body) in [ + ("other", Dialect::Chat, "{}"), + ("m", Dialect::Responses, "{}"), + ("m", Dialect::Chat, r#"{"tools":[{}]}"#), + ] { + assert_eq!( + cache.lookup( + &Key::of(&p, model, dialect, "/v1/chat/completions", body.as_bytes()).unwrap(), + 2 + ), + None + ); + } + let mut changed = p.clone(); + changed.key = Some("other-account".into()); + assert_ne!( + key, + Key::of(&changed, "m", Dialect::Chat, "/v1/chat/completions", b"{}").unwrap() + ); + assert_eq!(cache.lookup(&key, TTL_MS + 1), None); + for i in 0..MAX + 1 { + cache.accepted( + Key::of( + &p, + &format!("m{i}"), + Dialect::Chat, + "/v1/chat/completions", + b"{}", + ) + .unwrap(), + true, + i as u64, + ); + } + assert_eq!(cache.0.lock().unwrap().len(), MAX); + } +} diff --git a/crates/tw-gateway/tests/compressed.rs b/crates/tw-gateway/tests/compressed.rs index 3e664c82..d78c4203 100644 --- a/crates/tw-gateway/tests/compressed.rs +++ b/crates/tw-gateway/tests/compressed.rs @@ -293,7 +293,11 @@ async fn an_upstream_that_answers_gzip_json_reaches_the_client_decoded() { async fn an_upstream_that_streams_gzip_sse_reaches_the_client_decoded_event_by_event() { let (up, _) = upstream(Reply::GzipStream(STREAM)).await; let gw = gateway(up).await; - let r = post(gw, "/v1/messages", None, claude_request().into_bytes()).await; + // Ask for a stream: a whole-answer client now correctly receives collected + // JSON when an upstream ignores its mode and returns SSE. + let mut request: Value = serde_json::from_str(&claude_request()).unwrap(); + request["stream"] = json!(true); + let r = post(gw, "/v1/messages", None, request.to_string().into_bytes()).await; assert_eq!(r.status(), 200); assert!( r.headers().get("content-encoding").is_none(), diff --git a/crates/tw-gateway/tests/stream_negotiation.rs b/crates/tw-gateway/tests/stream_negotiation.rs new file mode 100644 index 00000000..f8dbbd03 --- /dev/null +++ b/crates/tw-gateway/tests/stream_negotiation.rs @@ -0,0 +1,533 @@ +//! Response-mode negotiation through real HTTP, including the client wire shape. +use axum::{Router, body::Body, http::Uri, response::Response}; +use bytes::Bytes; +use serde_json::{Value, json}; +use std::net::SocketAddr; +use std::sync::{Arc, Mutex}; +use tw_config::{Client, Config, Protocol, Provider}; +use tw_dialect::ir::{Dialect, Target}; + +type Seen = Arc>>; + +fn response(status: u16, ct: &str, body: String) -> Response { + // Deliberately split SSE frames and tool arguments across transport chunks. + let chunks: Vec<_> = body + .as_bytes() + .chunks(7) + .map(|b| Ok::<_, std::io::Error>(Bytes::copy_from_slice(b))) + .collect(); + Response::builder() + .status(status) + .header("content-type", ct) + .header("retry-after", "3600") + .body(Body::from_stream(futures::stream::iter(chunks))) + .unwrap() +} + +async fn upstream( + reply: impl Fn(&str, &Value, usize) -> Response + Send + Sync + 'static, +) -> (SocketAddr, Seen) { + let seen = Seen::default(); + let app = Router::new().fallback({ + let seen = seen.clone(); + let reply = Arc::new(reply); + move |uri: Uri, body: Bytes| { + let (seen, reply) = (seen.clone(), reply.clone()); + async move { + let v: Value = serde_json::from_slice(&body).unwrap(); + let path = uri.to_string(); + let n = { + let mut seen = seen.lock().unwrap(); + seen.push((path.clone(), v.clone())); + seen.len() + }; + reply(&path, &v, n) + } + } + }); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + (addr, seen) +} + +async fn gateway(up: SocketAddr, protocol: Protocol) -> (SocketAddr, tw_gateway::AppState) { + let mut cfg = Config { + version: 1, + clients: vec![Client { + name: "c".into(), + key: "tw-k".into(), + ..Default::default() + }], + providers: vec![Provider { + name: "up".into(), + base_url: format!("http://{up}"), + protocol: Some(protocol), + key: (protocol == Protocol::Bedrock).then(|| "ABSK-test".into()), + ..Default::default() + }], + ..Default::default() + }; + cfg.failover.wait_secs = 1; + let state = tw_gateway::AppState::new(cfg).unwrap(); + let addr = tw_gateway::serve(state.clone(), ([127, 0, 0, 1], 0).into()) + .await + .unwrap(); + (addr, state) +} + +fn refusal(stream: bool) -> Response { + response(400, "application/json", json!({"error":{ + "type":"invalid_request_error", "param":"stream", + "message": if stream { "Streaming is not supported." } else { "This model only supports streaming." } + }}).to_string()) +} + +fn whole() -> Value { + json!({"id":"answer","object":"chat.completion","model":"m","choices":[{"index":0,"message":{ + "role":"assistant","content":"ok","tool_calls":[{"id":"call_1","type":"function","function":{"name":"weather","arguments":"{\"city\":\"Paris\"}"}}] + },"finish_reason":"tool_calls"}],"usage":{"prompt_tokens":11,"completion_tokens":7,"total_tokens":18}}) +} + +fn sse() -> String { + let chunks = [ + json!({"choices":[{"index":0,"delta":{"role":"assistant","content":"o"}}]}), + json!({"choices":[{"index":0,"delta":{"content":"k","tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"weather","arguments":"{\"city\":"}}]}}]}), + json!({"choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"Paris\"}"}}]},"finish_reason":"tool_calls"}]}), + json!({"choices":[],"usage":{"prompt_tokens":11,"completion_tokens":7,"total_tokens":18}}), + ]; + let mut out: String = chunks.iter().map(|v| format!("data: {v}\n\n")).collect(); + out.push_str("data: [DONE]\n\n"); + out +} + +fn request(dialect: Dialect, stream: bool) -> (&'static str, Value) { + match dialect { + Dialect::Chat => { + let mut v = json!({"model":"m","stream":stream,"messages":[{"role":"user","content":"hello"}],"tools":[{"type":"function","function":{"name":"weather","parameters":{"type":"object","properties":{"city":{"type":"string"}}}}}]}); + if stream { + v["stream_options"] = json!({"include_usage":true}); + } + ("/v1/chat/completions", v) + } + Dialect::Anthropic => ( + "/v1/messages", + json!({"model":"m","stream":stream,"max_tokens":4096,"messages":[{"role":"user","content":"hello"}],"tools":[{"name":"weather","input_schema":{"type":"object","properties":{"city":{"type":"string"}}}}]}), + ), + Dialect::Responses => ( + "/v1/responses", + json!({"model":"m","stream":stream,"input":[{"role":"user","content":"hello"}],"tools":[{"type":"function","name":"weather","parameters":{"type":"object","properties":{"city":{"type":"string"}}}}]}), + ), + _ => unreachable!(), + } +} + +async fn post(gw: SocketAddr, path: &str, body: &Value) -> (u16, String, Vec) { + let r = reqwest::Client::new() + .post(format!("http://{gw}{path}")) + .header("authorization", "Bearer tw-k") + .json(body) + .send() + .await + .unwrap(); + let status = r.status().as_u16(); + let ct = r.headers()["content-type"].to_str().unwrap().to_owned(); + (status, ct, r.bytes().await.unwrap().to_vec()) +} + +fn assert_answer(dialect: Dialect, stream: bool, path: &str, req: &Value, ct: &str, body: &[u8]) { + assert!( + ct.starts_with(if stream { + "text/event-stream" + } else { + "application/json" + }), + "{ct}" + ); + // Read the client's actual response back as a common Chat shape. This also + // verifies that tool deltas form valid JSON and that usage survived both ways. + let mut session = tw_dialect::convert::decode(dialect, req, path, None) + .unwrap() + .encode(&Target { + dialect, + official: false, + default_max_tokens: 0, + }) + .session; + session.client = Dialect::Chat; + let normalized = if stream { + let mut collector = session.collector(); + collector.process(body); + collector.finish().unwrap() + } else { + session.response(body).unwrap() + }; + let v: Value = serde_json::from_slice(&normalized).unwrap(); + assert_eq!(v["choices"][0]["message"]["content"], "ok", "{v}"); + let call = &v["choices"][0]["message"]["tool_calls"][0]; + assert_eq!(call["id"], "call_1", "{v}"); + assert_eq!(call["function"]["name"], "weather"); + assert_eq!( + serde_json::from_str::(call["function"]["arguments"].as_str().unwrap()).unwrap(), + json!({"city":"Paris"}) + ); + assert_eq!(v["usage"]["prompt_tokens"], 11, "{v}"); + assert_eq!(v["usage"]["completion_tokens"], 7, "{v}"); +} + +#[tokio::test] +async fn both_directions_keep_the_client_protocol_tools_usage_and_learn_the_mode() { + for dialect in [Dialect::Chat, Dialect::Anthropic, Dialect::Responses] { + for client_stream in [false, true] { + let (up, seen) = upstream(move |_, v, _| { + let stream = v["stream"].as_bool().unwrap_or(false); + if stream == client_stream { + refusal(stream) + } else if stream { + response(200, "text/event-stream", sse()) + } else { + response(200, "application/json", whole().to_string()) + } + }) + .await; + let (gw, state) = gateway(up, Protocol::OpenaiChat).await; + let (path, req) = request(dialect, client_stream); + for count in [2, 3] { + let (status, ct, body) = post(gw, path, &req).await; + assert_eq!(status, 200, "{}", String::from_utf8_lossy(&body)); + assert_answer(dialect, client_stream, path, &req, &ct, &body); + assert_eq!(seen.lock().unwrap().len(), count); + } + let sent = seen.lock().unwrap().clone(); + assert_eq!( + sent[0].1["stream"].as_bool().unwrap_or(false), + client_stream + ); + assert_eq!(sent[1].1["stream"], !client_stream); + assert_eq!(sent[0].1["messages"], sent[1].1["messages"]); + assert_eq!(sent[0].1["tools"], sent[1].1["tools"]); + if client_stream { + assert!(sent[1].1.get("stream_options").is_none()); + } else { + assert_eq!(sent[1].1["stream_options"]["include_usage"], true); + } + // A reload forgets runtime capability evidence. + state.reload((*state.config()).clone()).unwrap(); + assert_eq!(post(gw, path, &req).await.0, 200); + assert_eq!(seen.lock().unwrap().len(), 5); + } + } +} + +#[tokio::test] +async fn successful_opposite_mode_is_converted_without_an_extra_request() { + for client_stream in [false, true] { + let (up, seen) = upstream(move |_, _, _| { + if client_stream { + response(200, "application/json", whole().to_string()) + } else { + response(200, "text/event-stream", sse()) + } + }) + .await; + let (gw, _) = gateway(up, Protocol::OpenaiChat).await; + let (path, req) = request(Dialect::Chat, client_stream); + let (status, ct, body) = post(gw, path, &req).await; + assert_eq!(status, 200); + assert_answer(Dialect::Chat, client_stream, path, &req, &ct, &body); + assert_eq!(seen.lock().unwrap().len(), 1); + } +} + +#[tokio::test] +async fn failed_alternatives_and_incomplete_or_error_streams_are_not_cached() { + for alternative in [0, 1, 2, 3] { + let (up, seen) = upstream(move |_, v, _| { + if v["stream"] != true { + return refusal(false); + } + match alternative { + 0 => refusal(true), + 1 => response( + 200, + "text/event-stream", + "data: {\"choices\":[{\"delta\":{\"content\":\"partial\"}}]}\n\n".into(), + ), + 2 => response( + 200, + "text/event-stream", + format!( + "data: {}\n\ndata: [DONE]\n\n", + json!({"error":{"message":"failed","type":"invalid_request_error"}}) + ), + ), + _ => response( + 200, + "application/json", + "{\"error\":{\"message\":\"failed\"}}".into(), + ), + } + }) + .await; + let (gw, _) = gateway(up, Protocol::OpenaiChat).await; + let (path, req) = request(Dialect::Chat, false); + for count in [2, 4] { + let _ = post(gw, path, &req).await; + assert_eq!( + seen.lock().unwrap().len(), + count, + "alternative {alternative}" + ); + } + } +} + +#[tokio::test] +async fn rate_limits_authentication_and_unrelated_validation_never_switch_modes() { + for (status, message, param) in [ + (429, "This model only supports streaming.", "stream"), + (400, "Rate limit exceeded", "stream"), + (401, "This model only supports streaming.", "stream"), + (403, "This model only supports streaming.", "stream"), + (422, "invalid messages", "messages"), + (400, "Streaming is not supported.", "stream_options"), + ] { + let (up, seen) = upstream(move |_, _, _| { + response( + status, + "application/json", + json!({"error":{"message":message,"param":param}}).to_string(), + ) + }) + .await; + let (gw, _) = gateway(up, Protocol::OpenaiChat).await; + let (path, req) = request(Dialect::Chat, false); + let _ = post(gw, path, &req).await; + assert_eq!(seen.lock().unwrap().len(), 1, "status {status}: {message}"); + assert_eq!(seen.lock().unwrap()[0].1["stream"], false); + } +} + +#[tokio::test] +async fn native_response_features_the_mode_converter_cannot_preserve_are_not_retried() { + let (up, seen) = upstream(|_, _, _| refusal(false)).await; + let (gw, _) = gateway(up, Protocol::OpenaiChat).await; + let (path, original) = request(Dialect::Chat, false); + for (i, extra) in [ + json!({"n":2}), + json!({"logprobs":true}), + json!({"modalities":["text","audio"]}), + ] + .into_iter() + .enumerate() + { + let mut req = original.clone(); + req.as_object_mut() + .unwrap() + .extend(extra.as_object().unwrap().clone()); + assert_eq!(post(gw, path, &req).await.0, 400); + assert_eq!(seen.lock().unwrap().len(), i + 1); + } +} + +#[tokio::test] +async fn a_json_error_inside_http_200_is_kept_as_an_error_during_mode_conversion() { + for client_stream in [false, true] { + let (up, seen) = upstream(move |_, v, _| { + let stream = v["stream"].as_bool().unwrap_or(false); + if stream == client_stream { + refusal(stream) + } else { + response( + 200, + "application/json", + json!({"error":{"message":"generation failed"}}).to_string(), + ) + } + }) + .await; + let (gw, _) = gateway(up, Protocol::OpenaiChat).await; + let (path, req) = request(Dialect::Chat, client_stream); + for count in [2, 4] { + let (_, ct, body) = post(gw, path, &req).await; + assert!(ct.starts_with(if client_stream { + "text/event-stream" + } else { + "application/json" + })); + let text = String::from_utf8_lossy(&body); + assert!(text.contains("generation failed"), "{text}"); + assert!(text.contains("error"), "{text}"); + assert_eq!(seen.lock().unwrap().len(), count); + } + } +} + +#[tokio::test] +async fn a_remembered_mode_can_be_relearned_when_the_upstream_changes() { + use std::sync::atomic::{AtomicBool, Ordering}; + let streams = Arc::new(AtomicBool::new(true)); + let allowed = streams.clone(); + let (up, seen) = upstream(move |_, v, _| { + let stream = v["stream"].as_bool().unwrap_or(false); + if stream != allowed.load(Ordering::SeqCst) { + refusal(stream) + } else if stream { + response(200, "text/event-stream", sse()) + } else { + response(200, "application/json", whole().to_string()) + } + }) + .await; + let (gw, _) = gateway(up, Protocol::OpenaiChat).await; + let (path, req) = request(Dialect::Chat, false); + assert_eq!(post(gw, path, &req).await.0, 200); + assert_eq!(seen.lock().unwrap().len(), 2); + streams.store(false, Ordering::SeqCst); + assert_eq!(post(gw, path, &req).await.0, 200); + assert_eq!(seen.lock().unwrap().len(), 4); + assert_eq!(post(gw, path, &req).await.0, 200); + assert_eq!(seen.lock().unwrap().len(), 5); + let (path, req) = request(Dialect::Chat, true); + assert_eq!(post(gw, path, &req).await.0, 200); + assert_eq!(seen.lock().unwrap().len(), 6); +} + +#[tokio::test] +async fn gemini_negotiation_changes_the_endpoint_and_keeps_the_client_framing() { + for client_stream in [false, true] { + let (up, seen) = upstream(move |path, _, _| { + let stream = path.contains(":streamGenerateContent"); + if stream == client_stream { return refusal(stream); } + let v = json!({"candidates":[{"content":{"parts":[{"text":"ok"}],"role":"model"},"finishReason":"STOP"}],"usageMetadata":{"promptTokenCount":11,"candidatesTokenCount":7,"totalTokenCount":18}}); + if stream { response(200, "text/event-stream", format!("data: {v}\n\n")) } + else { response(200, "application/json", v.to_string()) } + }).await; + let (gw, _) = gateway(up, Protocol::Gemini).await; + let path = if client_stream { + "/v1beta/models/m:streamGenerateContent?alt=sse" + } else { + "/v1beta/models/m:generateContent" + }; + let req = json!({"contents":[{"role":"user","parts":[{"text":"hello"}]}]}); + for count in [2, 3] { + let (status, ct, body) = post(gw, path, &req).await; + assert_eq!(status, 200); + assert!(ct.starts_with(if client_stream { + "text/event-stream" + } else { + "application/json" + })); + assert!(String::from_utf8_lossy(&body).contains("ok")); + assert_eq!(seen.lock().unwrap().len(), count); + } + let sent = seen.lock().unwrap(); + assert_eq!(sent[0].1, sent[1].1); + assert_eq!( + sent[1].0.contains(":streamGenerateContent?alt=sse"), + !client_stream + ); + assert!(!sent[1].0.contains("generateContent?alt=sse")); + } +} + +#[tokio::test] +async fn native_anthropic_and_responses_keep_their_response_shape_in_both_directions() { + for (dialect, protocol) in [ + (Dialect::Anthropic, Protocol::Anthropic), + (Dialect::Responses, Protocol::OpenaiResponses), + ] { + for client_stream in [false, true] { + let (up, seen) = upstream(move |_, v, _| { + let stream = v["stream"].as_bool().unwrap_or(false); + if stream == client_stream { return refusal(stream); } + let (message, events) = if dialect == Dialect::Anthropic { + let message = json!({"id":"msg_1","type":"message","role":"assistant","model":"m","content":[{"type":"text","text":"ok"}],"stop_reason":"end_turn","usage":{"input_tokens":11,"output_tokens":7}}); + let events = vec![ + json!({"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","model":"m","content":[],"usage":{"input_tokens":11,"output_tokens":0}}}), + json!({"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}), + json!({"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"ok"}}), + json!({"type":"content_block_stop","index":0}), + json!({"type":"message_delta","delta":{"stop_reason":"end_turn"},"usage":{"output_tokens":7}}), + json!({"type":"message_stop"}), + ]; + (message, events) + } else { + let item = json!({"id":"msg_1","type":"message","role":"assistant","status":"completed","content":[{"type":"output_text","text":"ok","annotations":[]}]}); + let message = json!({"id":"resp_1","object":"response","model":"m","status":"completed","error":null,"output":[item],"usage":{"input_tokens":11,"output_tokens":7,"total_tokens":18}}); + let events = vec![ + json!({"type":"response.created","response":{"id":"resp_1","model":"m","status":"in_progress"}}), + json!({"type":"response.output_item.added","output_index":0,"item":{"id":"msg_1","type":"message","role":"assistant","content":[]}}), + json!({"type":"response.content_part.added","output_index":0,"content_index":0,"part":{"type":"output_text","text":""}}), + json!({"type":"response.output_text.delta","output_index":0,"content_index":0,"delta":"ok"}), + json!({"type":"response.output_item.done","output_index":0,"item":item}), + json!({"type":"response.completed","response":message}), + ]; + (message, events) + }; + if stream { + response(200, "text/event-stream", events.iter().map(|e| format!("event: {}\ndata: {e}\n\n", e["type"].as_str().unwrap())).collect()) + } else { response(200, "application/json", message.to_string()) } + }).await; + let (gw, _) = gateway(up, protocol).await; + let (path, req) = request(dialect, client_stream); + for count in [2, 3] { + let (status, ct, body) = post(gw, path, &req).await; + assert_eq!(status, 200); + assert!(ct.starts_with(if client_stream { + "text/event-stream" + } else { + "application/json" + })); + assert!(String::from_utf8_lossy(&body).contains("ok")); + assert_eq!(seen.lock().unwrap().len(), count); + } + } + } +} + +#[tokio::test] +async fn bedrock_negotiates_converse_actions_and_collects_real_eventstream_frames() { + use aws_smithy_eventstream::frame::write_message_to; + use aws_smithy_types::event_stream::{Header, HeaderValue, Message}; + for client_stream in [false, true] { + let (up, seen) = upstream(move |path, _, _| { + let stream = path.ends_with("/converse-stream"); + if stream == client_stream { return refusal(stream); } + if !stream { + return response(200, "application/json", json!({"output":{"message":{"role":"assistant","content":[{"text":"ok"}]}},"stopReason":"end_turn","usage":{"inputTokens":11,"outputTokens":7,"totalTokens":18}}).to_string()); + } + let mut bytes = Vec::new(); + for (kind, v) in [ + ("messageStart", json!({"role":"assistant"})), + ("contentBlockDelta", json!({"contentBlockIndex":0,"delta":{"text":"ok"}})), + ("contentBlockStop", json!({"contentBlockIndex":0})), + ("messageStop", json!({"stopReason":"end_turn"})), + ("metadata", json!({"usage":{"inputTokens":11,"outputTokens":7,"totalTokens":18}})), + ] { + let frame = Message::new(v.to_string().into_bytes()) + .add_header(Header::new(":message-type", HeaderValue::String("event".into()))) + .add_header(Header::new(":event-type", HeaderValue::String(kind.into()))) + .add_header(Header::new(":content-type", HeaderValue::String("application/json".into()))); + write_message_to(&frame, &mut bytes).unwrap(); + } + Response::builder().header("content-type", "application/vnd.amazon.eventstream").body(Body::from(bytes)).unwrap() + }).await; + let (gw, _) = gateway(up, Protocol::Bedrock).await; + let (path, req) = request(Dialect::Chat, client_stream); + for count in [2, 3] { + let (status, ct, body) = post(gw, path, &req).await; + assert_eq!(status, 200, "{}", String::from_utf8_lossy(&body)); + assert!(ct.starts_with(if client_stream { + "text/event-stream" + } else { + "application/json" + })); + assert!(String::from_utf8_lossy(&body).contains("ok")); + assert_eq!(seen.lock().unwrap().len(), count); + } + let sent = seen.lock().unwrap(); + assert_eq!(sent[1].0.ends_with("/converse-stream"), !client_stream); + assert_eq!(sent[0].1, sent[1].1); + } +} diff --git a/docs/config.md b/docs/config.md index 4367b4ad..daef62c2 100644 --- a/docs/config.md +++ b/docs/config.md @@ -539,6 +539,26 @@ A ChatGPT account upstream (`protocol: chatgpt`) takes only the credential the desktop app obtains by signing in; it cannot be written by hand. Claude and Google subscription sign-ins are not supported; use an API key. +Generation requests first use the client's streaming or non-streaming mode. +When an upstream explicitly rejects that mode in an HTTP 400, 422 or 501 API +error, the gateway can try the opposite mode once on the same upstream. It +collects a streaming answer into a complete JSON response, or writes a complete +answer as the stream the client requested. The latter still waits for the whole +upstream answer. This shares the request's single automatic repair budget with +history, reasoning-effort and cache-mark repairs, and respects its wait deadline. +Rate limits, authentication errors, timeouts and errors inside a successful +stream do not trigger a mode switch. A successful response already in the other +mode is converted without resending it. + +After a complete successful answer, the accepted alternative is remembered in +memory for 24 hours, isolated by upstream account, model, API and request options +such as tools and reasoning. Reloading the configuration clears this memory; +an explicit rejection invalidates a remembered mode. No upstream setting is +required. ChatGPT account backends keep their existing stream-only behavior. +Native requests for multiple candidates, log probabilities or non-text output +are not switched automatically: the current response converters cannot preserve +those response fields. WebSocket requests are not negotiated. + Some relays and accounts accept only a few requests at a time and refuse the rest. `max_concurrent` keeps the gateway within that number: a request takes a slot on the upstream when it is sent, and gives it back when the answer has diff --git a/docs/config.zh-CN.md b/docs/config.zh-CN.md index 26010e6c..6d65f5ed 100644 --- a/docs/config.zh-CN.md +++ b/docs/config.zh-CN.md @@ -368,6 +368,12 @@ Bedrock 上只为 AWS 列出支持提示缓存的 Claude 模型标注:Claude 3 ChatGPT 账号上游(`protocol: chatgpt`)只接受桌面应用登录得到的凭据,不能手写。不支持 Claude 和 Google 的订阅登录,请使用 API 密钥。 +生成请求默认按客户端要求的流式或非流式模式发送。上游在 HTTP 400、422 或 501 的 API 错误中明确拒绝当前模式时,网关可以在同一家上游改用另一种模式重发一次:流式回答收齐后返回完整 JSON,整包回答则写成客户端要求的流。后一种仍需等待上游生成完整回答。它与历史字段、推理强度和缓存断点的修复共用每个请求最多一次的自动修复预算,并受请求等待期限约束。限流、凭据错误、超时和成功响应流中的错误不触发模式切换。上游已经成功返回另一种模式的响应时,直接转换,不再重发。 + +完整回答成功结束后,可用的替代模式在内存中记 24 小时,按上游账号、模型、API 和工具、推理等请求选项隔离;重载配置会清空,记住的模式被明确拒绝时会失效。无需增加上游配置。ChatGPT 账号后端继续使用既有的仅流式处理。 + +原生协议请求要求多个候选、对数概率或非文本输出时不自动切换,避免当前响应转换器丢失这些字段。WebSocket 请求不参与协商。 + 有的中转站和账号同时只接受几个请求,多出来的直接拒绝。`max_concurrent` 让网关守住这个数:请求发出时占用这家的一个位置,回答完整交给客户端、或者客户端断开时归还。这家满了的时候,为复用提示缓存而留在这家的对话等空位,别的请求直接换下一家。最多等多久由 `failover.wait_secs` 决定。等待不算失败,这家不会因此停用。只计算 token 数的请求不占位置。Responses 的 WebSocket 连接上,每个 `response.create` 从发出起占一个位置,直到它的回答结束,密钥的 `max_concurrent` 也一样;空闲的连接不占位置。 #### `providers[].min_speed` From 0980d693d07a421064ae92b2c69906213974f909 Mon Sep 17 00:00:00 2001 From: fylorn <249551762+fylorn@users.noreply.github.com> Date: Thu, 12 Nov 2026 12:52:56 +0800 Subject: [PATCH 2/5] chore(api): sync response error message code --- crates/tw-api/msg-codes.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/tw-api/msg-codes.txt b/crates/tw-api/msg-codes.txt index 9b0f3863..710e38fa 100644 --- a/crates/tw-api/msg-codes.txt +++ b/crates/tw-api/msg-codes.txt @@ -454,6 +454,7 @@ gw.upstream.idle_timeout_mid_stream gw.upstream.paused gw.upstream.period_used_up gw.upstream.rate_limited +gw.upstream.response_error gw.upstream.sealed_elsewhere gw.upstream.sign_failed gw.upstream.status From 56e72b18e44ed2189303d22f7a160da989b9f29d Mon Sep 17 00:00:00 2001 From: fylorn <249551762+fylorn@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:55:43 +0800 Subject: [PATCH 3/5] fix(gateway): probe unknown response-mode rejections safely Allow one controlled opposite-mode trial after eligible opaque API request rejections, after specific schema repairs. Exclude account/capacity errors, retry hints and explicit non-mode validations. Keep the original error and response context if the probe also fails before content; record its actual outcome without repeating the failed trial. Preserve normal failover for original server errors and handle embedded JSON errors in HTTP-success trial responses. Learn only complete working answers for a fixed 30-minute window, then retry the client mode. Extend HTTP regressions for custom/localized errors, failed probes, attempt records, account states and candidate isolation. --- crates/tw-gateway/src/server/pipeline/hop.rs | 313 ++++++++++++++++-- .../tw-gateway/src/server/pipeline/relay.rs | 19 +- crates/tw-gateway/src/stream_mode.rs | 299 +++++++++++------ crates/tw-gateway/tests/stream_negotiation.rs | 232 ++++++++++++- docs/config.md | 38 ++- docs/config.zh-CN.md | 6 +- 6 files changed, 737 insertions(+), 170 deletions(-) diff --git a/crates/tw-gateway/src/server/pipeline/hop.rs b/crates/tw-gateway/src/server/pipeline/hop.rs index 5a1285ee..e5bb99bf 100644 --- a/crates/tw-gateway/src/server/pipeline/hop.rs +++ b/crates/tw-gateway/src/server/pipeline/hop.rs @@ -58,10 +58,42 @@ pub(super) struct Served<'a> { /// 内容、在开头报了限流以外的错,原样交了出去。成败的账等回程看到第一段内容才记(见 /// `relay`)。没有内容就超时了记失败,和压着开头时一样 pub(super) unconfirmed: bool, + /// This error's headers were already accounted before trying another mode. + pub(super) replayed_error: bool, /// Learn an alternative only after a complete, clean response. pub(super) stream_trial: Option<(crate::stream_mode::Key, bool)>, } +/// Return the original error with its own response context. The probe's real +/// outcome stays in the chain; the active slot follows the selected response. +fn restore_mode_error<'a>( + mut original: Served<'a>, + slot: crate::slots::Slot, + chain: &mut [tw_api::AttemptView], +) -> Served<'a> { + chain[original.attempt].outcome = tw_api::AttemptOutcome::Served; + original.slot = slot; + original.replayed_error = true; + original +} + +/// A failed probe ends repair on this provider. An original server error still +/// permits failover; keep it in case no later provider can serve the request. +fn failed_mode_probe<'a>( + original: Served<'a>, + slot: crate::slots::Slot, + chain: &mut [tw_api::AttemptView], + last: bool, + fallback: &mut Option>, +) -> Option> { + if original.upstream.status().is_server_error() && !last { + *fallback = Some(original); + None + } else { + Some(restore_mode_error(original, slot, chain)) + } +} + /// 这个请求的着落。 pub(super) enum Answer<'a> { /// 一家上游接下了它 @@ -203,6 +235,9 @@ pub(super) async fn try_upstreams<'a>( let mut history_trial: Option<(crate::history_memory::Key, crate::history_memory::Policy)> = None; let mut stream_trial: Option<(crate::stream_mode::Key, bool)> = None; + // The probe must not replace a useful original request error with its own. + let mut stream_original: Option> = None; + let mut stream_failure: Option> = None; // 客户端收的是 SSE 流:压着开头过了时限先交响应头、发保活(见 `super::commit`)。最后一家的 // 开头只在这时为重试压着 let early = super::commit::early(state, req, reading).is_some(); @@ -997,6 +1032,14 @@ pub(super) async fn try_upstreams<'a>( }, &mut chain, )); + if stream_trial_applied && let Some(original) = stream_original.take() { + served = + failed_mode_probe(original, slot, &mut chain, last, &mut stream_failure); + if served.is_some() { + break; + } + continue; + } continue; } }; @@ -1032,36 +1075,41 @@ pub(super) async fn try_upstreams<'a>( if !matches!(verdict, Verdict::Failed(Cause::RateLimited { .. })) { to_throttle(state, &provider.name, sent_at, verdict); } - if let (Some(key), Some(mode)) = (&stream_key, stream_sent) - && crate::stream_mode::refusal(status, &head, mode) - { - // A remembered capability may have changed. Forget even - // when this request has already spent its repair budget. - rt.stream_modes.forget(key); - if !repair_used && tokio::time::Instant::now() < until { - repair_used = true; - stream_trial = Some((key.clone(), !mode)); - tap.said(&head); - state.note_proxy_ok(&provider.proxy); - note_health( - &state.bus, - &state.health, - &provider.name, - state.health.record_success(&provider.name), - ); - chain.push(hop( + if stream_trial_applied && let Some(original) = stream_original.take() { + reached = true; + tap.said(&head); + state.note_quota(id, &provider.name, &headers); + state.note_proxy_ok(&provider.proxy); + state.note_auth(&provider.name, status); + let mut failed = hop( + &provider.name, + model.clone(), + tw_api::AttemptOutcome::Status, + status, + hop_started, + ); + let change = match verdict { + Verdict::Failed(Cause::RateLimited { .. }) => { + let hint = crate::throttle::pause_hint(&headers, &head, now_ms()); + let pause = state.throttles.limited(&provider.name, sent_at, hint); + usage_hold.release(); + failed.pause_ms = Some(pause.as_millis() as u64); + None + } + Verdict::Failed(cause) => state.health.record_cause( &provider.name, - model.clone(), - tw_api::AttemptOutcome::Status, - status, - hop_started, - )); - // Account and record a separate attempt, reusing the - // slot. Other providers still start from the original. - effort_hold = Some((name, slot, probe.take())); - queue.push_front(name); - continue; + known_reset(state, &provider.name, cause), + ), + Verdict::ClientError => state.health.record_success(&provider.name), + }; + note_health(&state.bus, &state.health, &provider.name, change); + chain.push(failed); + served = + failed_mode_probe(original, slot, &mut chain, last, &mut stream_failure); + if served.is_some() { + break; } + continue; } if !repair_used && tokio::time::Instant::now() < until && let (Some(key), Some(mut policy)) = (history_key.clone(), history_policy.clone()) @@ -1141,6 +1189,54 @@ pub(super) async fn try_upstreams<'a>( queue.push_front(name); continue; } + if let (Some(key), Some(mode)) = (&stream_key, stream_sent) + && crate::stream_mode::can_probe(status, &headers, &head, verdict) + { + // A rejected remembered preference is no longer useful. + // Learn the replacement only if the whole answer succeeds. + rt.stream_modes.forget(key); + if !repair_used && tokio::time::Instant::now() < until { + repair_used = true; + stream_trial = Some((key.clone(), !mode)); + state.note_quota(id, &provider.name, &headers); + state.note_auth(&provider.name, status); + tap.said(&head); + state.note_proxy_ok(&provider.proxy); + note_health( + &state.bus, + &state.health, + &provider.name, + state.health.record_success(&provider.name), + ); + chain.push(hop( + &provider.name, + model.clone(), + tw_api::AttemptOutcome::Status, + status, + hop_started, + )); + // Account and record a separate attempt, reusing the + // slot. Other providers still start from the original. + stream_original = Some(Served { + upstream: r, + provider, + model: sent_model, + attempt, + bridge: plugged.bridge, + ledger, + session: out.session, + refusal: None, + slot: crate::slots::Slot::free(), + sent_at, + unconfirmed: false, + replayed_error: false, + stream_trial: None, + }); + effort_hold = Some((name, slot, probe.take())); + queue.push_front(name); + continue; + } + } // **最后一家的 4xx 交给客户端**:没有下一家可换了,上游自己的原话 // (「invalid x-api-key」)比我们转述的一句「回了 401」有用得多。 // 5xx 和 429 照旧报我们的话,带着尝试链 @@ -1277,6 +1373,7 @@ pub(super) async fn try_upstreams<'a>( slot, sent_at, unconfirmed: false, + replayed_error: false, stream_trial: None, }); break; @@ -1341,6 +1438,9 @@ pub(super) async fn try_upstreams<'a>( Hold::Stream(dialect, eventstream) => { super::opening::watch(r, dialect, eventstream, quiet_by).await } + Hold::Body if stream_trial_applied => { + check_mode_body(r, stream_dialect.unwrap(), quiet_by).await + } Hold::Body => super::opening::first_bytes(r, quiet_by).await, } }; @@ -1374,6 +1474,36 @@ pub(super) async fn try_upstreams<'a>( // 到点了还没有内容:这一跳没有结论 Some(super::opening::Opening::Slow { response, usage }) => { state.throttles.released(&provider.name, sent_at); + if stream_trial_applied + && let Some(original) = stream_original.take() + { + drop(response); + last_err = Some(gave_up( + state, + started, + reading, + Abandoned { + provider, + model: model.clone(), + status: Some(status), + usage, + quiet, + hop_started, + }, + &mut chain, + )); + served = failed_mode_probe( + original, + slot, + &mut chain, + last, + &mut stream_failure, + ); + if served.is_some() { + break; + } + continue; + } if last { // 最后一家:和不压开头时一样交给回程,由它按无响应超时收尾 unconfirmed = true; @@ -1414,6 +1544,58 @@ pub(super) async fn try_upstreams<'a>( }) => { let verdict = crate::failure::classify(status, &headers, &body, now_ms()); + if stream_trial_applied + && let Some(original) = stream_original.take() + { + tap.said(&body); + state.note_quota(id, &provider.name, &headers); + let mut failed = hop_failed( + &provider.name, + model.clone(), + msg!("gw.upstream.stream_opening_error", + upstream = provider.name.clone(), kind = kind, message = message => + "Upstream `{upstream}` started the answer and reported an error \ + before any content ({kind}): {message}"), + hop_started, + ); + let change = match verdict { + Verdict::Failed(Cause::RateLimited { .. }) => { + let hint = crate::throttle::pause_hint( + &headers, + &body, + now_ms(), + ); + let pause = state.throttles.limited( + &provider.name, + sent_at, + hint, + ); + usage_hold.release(); + failed.pause_ms = Some(pause.as_millis() as u64); + None + } + Verdict::Failed(cause) => state.health.record_cause( + &provider.name, + known_reset(state, &provider.name, cause), + ), + Verdict::ClientError => { + state.health.record_success(&provider.name) + } + }; + note_health(&state.bus, &state.health, &provider.name, change); + chain.push(failed); + served = failed_mode_probe( + original, + slot, + &mut chain, + last, + &mut stream_failure, + ); + if served.is_some() { + break; + } + continue; + } let limited = matches!(verdict, Verdict::Failed(Cause::RateLimited { .. })); if !limited { @@ -1502,7 +1684,9 @@ pub(super) async fn try_upstreams<'a>( } Some(super::opening::Opening::Broken(err)) => { reached = true; - retries.shaky(name, None); + if !stream_trial_applied { + retries.shaky(name, None); + } state.throttles.released(&provider.name, sent_at); note_health( &state.bus, @@ -1517,6 +1701,21 @@ pub(super) async fn try_upstreams<'a>( hop_started, )); last_err = Some(err); + if stream_trial_applied + && let Some(original) = stream_original.take() + { + served = failed_mode_probe( + original, + slot, + &mut chain, + last, + &mut stream_failure, + ); + if served.is_some() { + break; + } + continue; + } continue; } } @@ -1572,6 +1771,7 @@ pub(super) async fn try_upstreams<'a>( slot, sent_at, unconfirmed, + replayed_error: false, stream_trial: stream_trial.clone().filter(|_| stream_trial_applied), }); break; @@ -1588,7 +1788,9 @@ pub(super) async fn try_upstreams<'a>( SendError::Http(e) if e.is_connect() => { usage_hold.release(); reached = true; - retries.shaky(name, None); + if !stream_trial_applied { + retries.shaky(name, None); + } // 没有结论:探路的不再探,交给熔断(见 `crate::throttle`) state.throttles.released(&provider.name, sent_at); state.check_proxy(&provider.proxy); @@ -1597,7 +1799,9 @@ pub(super) async fn try_upstreams<'a>( SendError::Http(e) => { reached = true; // 响应头之前断了:和连不上一样是暂时的错 - retries.shaky(name, None); + if !stream_trial_applied { + retries.shaky(name, None); + } state.throttles.released(&provider.name, sent_at); // 连不上的可能是代理而不是上游 —— 检一次那个代理,说清是哪一件事 state.check_proxy(&provider.proxy); @@ -1624,11 +1828,30 @@ pub(super) async fn try_upstreams<'a>( hop_started, )); last_err = Some(err); + if stream_trial_applied && let Some(original) = stream_original.take() { + served = + failed_mode_probe(original, slot, &mut chain, last, &mut stream_failure); + if served.is_some() { + break; + } + continue; + } continue; } } } + if halt.is_none() + && served.is_none() + && estimated.is_none() + && let Some(original) = stream_failure + { + served = Some(restore_mode_error( + original, + crate::slots::Slot::free(), + &mut chain, + )); + } stamp_queued(&mut chain, queued.take()); stamp_effort(&mut chain, effort_note.take()); stamp_retries(&mut chain, &retries.rounds); @@ -3111,6 +3334,34 @@ async fn bedrock_refusal( (reqwest::Response::from(resp), text) } +/// Some compatible APIs reject a probe inside an HTTP-200 JSON body. Inspect a +/// bounded prefix before handing the trial to relay; all bytes remain replayable. +async fn check_mode_body( + r: reqwest::Response, + dialect: tw_dialect::ir::Dialect, + deadline: tokio::time::Instant, +) -> super::opening::Opening { + let (head, r) = peek_body(r).await; + if head.len() <= crate::failure::BODY_PEEK + && let Some(message) = tw_dialect::convert::error_message(dialect, &head) + { + let status = std::str::from_utf8(&head) + .ok() + .and_then(|text| super::opening::stream_error(dialect, text)) + .map_or(502, |(status, _)| status); + return super::opening::Opening::Failed { + status, + headers: r.headers().clone(), + body: head, + kind: "error".into(), + message, + transient: false, + response: r, + }; + } + super::opening::first_bytes(r, deadline).await +} + /// 读错误响应的开头(最多 [`crate::failure::BODY_PEEK`]),交回读到的和一个照旧能从头 /// 读起的响应 —— 这一跳可能还是要原样交给客户端的。 async fn peek_body(r: reqwest::Response) -> (Bytes, reqwest::Response) { diff --git a/crates/tw-gateway/src/server/pipeline/relay.rs b/crates/tw-gateway/src/server/pipeline/relay.rs index 7891ddc2..cddc9973 100644 --- a/crates/tw-gateway/src/server/pipeline/relay.rs +++ b/crates/tw-gateway/src/server/pipeline/relay.rs @@ -53,6 +53,7 @@ pub(super) fn respond( slot, sent_at, unconfirmed, + replayed_error, stream_trial, attempt, .. @@ -68,14 +69,16 @@ pub(super) fn respond( status: status.as_u16(), ttfb_ms, }); - // 订阅额度。**零成本** —— 这些头本来就在响应里,读一下 - // 就有了。按量付费的账号没有它们,那时什么都不发。 - state.note_quota(id, &provider.name, upstream.headers()); - // GLM Coding Plan 的额度不在响应头里:有请求的时候隔一阵去问一次 - state.glm_traffic(provider); - // 上游收不收我们的凭据、经过的代理通不通:**都是状态变化,各只报一次** - state.note_auth(&provider.name, status.as_u16()); - state.note_proxy_ok(&provider.proxy); + if !replayed_error { + // 订阅额度。**零成本** —— 这些头本来就在响应里,读一下 + // 就有了。按量付费的账号没有它们,那时什么都不发。 + state.note_quota(id, &provider.name, upstream.headers()); + // GLM Coding Plan 的额度不在响应头里:有请求的时候隔一阵去问一次 + state.glm_traffic(provider); + // 上游收不收我们的凭据、经过的代理通不通:**都是状态变化,各只报一次** + state.note_auth(&provider.name, status.as_u16()); + state.note_proxy_ok(&provider.proxy); + } let mut out_headers = forward::response_headers(upstream.headers()); // **哪一家服务的,写在头上。**错误契约要求上游的错误原样透传、不加 diff --git a/crates/tw-gateway/src/stream_mode.rs b/crates/tw-gateway/src/stream_mode.rs index 2cb2391c..21540221 100644 --- a/crates/tw-gateway/src/stream_mode.rs +++ b/crates/tw-gateway/src/stream_mode.rs @@ -1,4 +1,4 @@ -//! Negotiate an upstream's response mode only after an explicit rejection. +//! Try an alternative response mode after an eligible request rejection. //! The client mode stays on the conversion session; only the outbound request //! changes. An accepted alternative is remembered locally until reload or expiry. @@ -8,7 +8,8 @@ use std::sync::Mutex; use serde_json::Value; use tw_dialect::ir::Dialect; -const TTL_MS: u64 = 24 * 60 * 60 * 1000; +// Fixed expiry: successful cached uses do not postpone retrying the default. +const TTL_MS: u64 = 30 * 60 * 1000; const MAX: usize = 1024; #[derive(Clone, Debug, PartialEq, Eq, Hash)] @@ -249,88 +250,144 @@ pub fn rewrite( } } -/// Only a top-level API error describing the actual rejected mode is evidence. -/// Never infer capabilities from rate limits, answer text, or stream_options. -pub fn refusal(status: u16, body: &[u8], sent_stream: bool) -> bool { - if !matches!(status, 400 | 422 | 501) || body.len() > 64 * 1024 { +/// A bounded experiment, not a claim that the requested mode is unsupported. +/// Known repairs run first. Reject non-mode parameter errors and recognizable +/// account, capacity or timeout errors, even when an upstream mislabels them. +pub fn can_probe( + status: u16, + headers: &http::HeaderMap, + body: &[u8], + verdict: crate::failure::Verdict, +) -> bool { + use crate::failure::{Cause, Verdict}; + if !matches!(status, 400 | 422 | 501) + || body.len() > crate::failure::BODY_PEEK + || crate::cache_marks::refusal(body) + || !matches!(verdict, Verdict::ClientError) + && !(status == 501 && verdict == Verdict::Failed(Cause::Unexplained)) + || headers.contains_key("retry-after") + || headers.contains_key("retry-after-ms") + || headers.get("x-should-retry").is_some_and(|v| v == "true") + { return false; } let Ok(v) = serde_json::from_slice::(body) else { return false; }; - let e = v.get("error").filter(|e| e.is_object()).unwrap_or(&v); - if let Some(param) = e.get("param").and_then(Value::as_str) + let error = match v.get("error") { + Some(e) if e.is_object() || e.is_string() => e, + None if ["message", "detail", "code", "type", "status"] + .iter() + .any(|name| v.get(name).is_some_and(Value::is_string)) => + { + &v + } + _ => return false, + }; + if let Some(param) = error.get("param").and_then(Value::as_str) + && !param.is_empty() && param != "stream" { return false; } - let message = e.get("message").and_then(Value::as_str).unwrap_or_default(); - let text = message.to_ascii_lowercase(); - if text.contains("rate limit") || text.contains("quota") || text.contains("overload") { + if error + .get("message") + .and_then(Value::as_str) + .is_some_and(names_other_parameter) + { return false; } - let code = e.get("code").and_then(Value::as_str).unwrap_or_default(); - let codes: &[&str] = if sent_stream { - &[ - "streaming_not_supported", - "stream_not_supported", - "unsupported_streaming", - ] + // Read error fields only: an echoed prompt or metadata must not affect + // eligibility. No list of positive English mode-refusal phrases is needed. + let text = if let Some(text) = error.as_str() { + text.to_lowercase() } else { - &[ - "non_streaming_not_supported", - "stream_required", - "streaming_required", - ] + ["code", "type", "status", "message", "detail"] + .into_iter() + .filter_map(|name| error.get(name).and_then(Value::as_str)) + .collect::>() + .join("\0") + .to_lowercase() }; - if codes.contains(&code) { - return true; + ![ + "rate limit", + "rate_limit", + "ratelimit", + "too_many_requests", + "quota", + "resource_exhausted", + "overload", + "authentication", + "unauthenticated", + "unauthorized", + "permission", + "forbidden", + "api key", + "api_key", + "credential", + "access denied", + "access_denied", + "billing", + "payment", + "balance", + "credit", + "timeout", + "timed out", + "deadline_exceeded", + "content_policy", + "content_filter", + "safety", + "限流", + "频率", + "额度", + "余额", + "欠费", + "鉴权", + "认证", + "凭据", + "权限", + "超时", + ] + .iter() + .any(|word| text.contains(word)) +} + +/// Some APIs put a field path in the message instead of `param`. Keep known +/// request validation errors intact, including explicit reasoning budgets. +fn names_other_parameter(message: &str) -> bool { + let text = message.trim().to_lowercase(); + if text.contains("stream") { + return false; } - // Intentionally narrow phrases: "streaming" inside "non-streaming" must - // never be mistaken for a refusal in the opposite direction. - let text = text - .replace("non streaming", "non-streaming") - .replace("nonstreaming", "non-streaming"); - let phrases: &[&str] = if sent_stream { - &[ - "streaming is not supported", - "streaming not supported", - "streaming is unsupported", - "does not support streaming", - "doesn't support streaming", - "only supports non-streaming", - "only non-streaming requests are supported", - "only non-streaming responses are supported", - "only non-streaming is supported", - "supports only non-streaming", - "stream must be false", - "stream must be set to false", - "stream=false is required", - ] - } else { - &[ - "non-streaming is not supported", - "non-streaming not supported", - "non-streaming is unsupported", - "does not support non-streaming", - "doesn't support non-streaming", - "only supports streaming", - "only streaming requests are supported", - "only streaming responses are supported", - "only streaming is supported", - "supports only streaming", - "stream must be true", - "stream must be set to true", - "stream=true is required", - ] - }; - phrases.iter().any(|p| { - text.match_indices(p).any(|(at, _)| { - at == 0 - || !text.as_bytes()[at - 1].is_ascii_alphanumeric() - && !matches!(text.as_bytes()[at - 1], b'-' | b'_') - }) - }) + let field = text + .trim_start_matches(['`', '\'', '"']) + .split(|c: char| c.is_whitespace() || matches!(c, ':' | '`' | '\'' | '"')) + .next() + .unwrap_or_default(); + let root = field.split(['.', '[', '/']).next().unwrap_or_default(); + matches!( + root, + "messages" + | "input" + | "contents" + | "system" + | "instructions" + | "tools" + | "max_tokens" + | "max_output_tokens" + | "max_completion_tokens" + | "temperature" + | "top_p" + | "top_k" + | "stop" + | "stop_sequences" + | "thinking" + | "reasoning" + | "reasoning_effort" + | "output_config" + | "generationconfig" + | "inferenceconfig" + ) } /// Evidence of a complete stream, used only while learning a new mode. @@ -413,42 +470,81 @@ mod tests { use serde_json::json; #[test] - fn only_explicit_refusals_for_the_sent_mode_trigger_a_switch() { - let error = |text| json!({"error":{"message":text}}).to_string(); - for (stream, text) in [ - (false, "This model only supports streaming."), - (false, "Non-streaming is not supported."), - (false, "stream must be set to true"), - (true, "This model does not support streaming."), - (true, "stream must be false"), - (true, "Only non-streaming requests are supported."), + fn probes_unknown_rejections_without_requiring_mode_error_phrases() { + let headers = http::HeaderMap::new(); + for body in [ + json!({"error":{"message":"Use the other response framing."}}), + json!({"error":{"message":"请换一种返回方式。"}}), + json!({"error":{"code":"vendor_1027"}}), + json!({"message":"Request rejected", "type":"invalid_request_error"}), + json!({"error":"Request rejected"}), + json!({"error":{"message":"Tools require streaming for this model"}}), ] { - assert!(refusal(400, error(text).as_bytes(), stream), "{text}"); - assert!(!refusal(400, error(text).as_bytes(), !stream), "{text}"); - for status in [200, 401, 403, 404, 429, 500, 503] { - assert!(!refusal(status, error(text).as_bytes(), stream)); + for status in [400, 422, 501] { + let body = body.to_string(); + let verdict = crate::failure::classify(status, &headers, body.as_bytes(), 0); + assert!( + can_probe(status, &headers, body.as_bytes(), verdict), + "{body}" + ); } } - for text in [ - "Rate limit exceeded", - "streaming failed", - "invalid stream_options", - "Quota exceeded; only streaming allowed", + for status in [200, 401, 402, 403, 404, 408, 429, 500, 502, 503] { + assert!(!can_probe( + status, + &headers, + br#"{"error":{}}"#, + crate::failure::classify(status, &headers, b"{}", 0) + )); + } + for body in [ + json!({"error":{"message":"Rate limit exceeded"}}), + json!({"error":{"code":"rate_limit_exceeded", "message":"opaque"}}), + json!({"error":{"type":"authentication_error", "message":"opaque"}}), + json!({"error":{"status":"RESOURCE_EXHAUSTED", "message":"opaque"}}), + json!({"error":{"message":"Quota exceeded; only streaming allowed"}}), + json!({"error":{"message":"invalid api key"}}), + json!({"error":{"message":"Insufficient balance"}}), + json!({"error":{"message":"Model not found"}}), + json!({"error":{"message":"Request timed out"}}), + json!({"error":{"message":"请求频率过高"}}), + json!({"error":{"message":"messages.0.content: field required"}}), + json!({"error":{"message":"max_output_tokens is too large"}}), + json!({"error":{"message":"thinking.budget_tokens: budget too large"}}), + json!({"message":"cachePoint is not supported for this model"}), + json!({"error":{"param":"messages", "message":"invalid messages"}}), + json!({"error":{"param":"stream_options", "message":"not supported"}}), + json!({"choices":[{"message":{"content":"only streaming"}}]}), + json!({"error":null}), ] { - for stream in [false, true] { - assert!(!refusal(400, error(text).as_bytes(), stream)); - } + let body = body.to_string(); + let verdict = crate::failure::classify(400, &headers, body.as_bytes(), 0); + assert!( + !can_probe(400, &headers, body.as_bytes(), verdict), + "{body}" + ); + } + for name in ["retry-after", "retry-after-ms", "x-should-retry"] { + let mut headers = http::HeaderMap::new(); + headers.insert(http::HeaderName::from_static(name), "true".parse().unwrap()); + assert!(!can_probe( + 400, + &headers, + br#"{"error":{}}"#, + crate::failure::Verdict::ClientError + )); + } + for body in [ + b"not JSON".as_slice(), + &[b'x'; crate::failure::BODY_PEEK + 1], + ] { + assert!(!can_probe( + 400, + &headers, + body, + crate::failure::Verdict::ClientError + )); } - assert!(!refusal( - 400, - br#"{"choices":[{"message":{"content":"only streaming"}}]}"#, - false - )); - assert!(!refusal( - 400, - br#"{"error":{"param":"stream_options","message":"streaming not supported"}}"#, - true - )); } #[test] @@ -481,6 +577,7 @@ mod tests { key, Key::of(&changed, "m", Dialect::Chat, "/v1/chat/completions", b"{}").unwrap() ); + assert_eq!(cache.lookup(&key, TTL_MS), Some(true)); assert_eq!(cache.lookup(&key, TTL_MS + 1), None); for i in 0..MAX + 1 { cache.accepted( diff --git a/crates/tw-gateway/tests/stream_negotiation.rs b/crates/tw-gateway/tests/stream_negotiation.rs index f8dbbd03..16a4b88b 100644 --- a/crates/tw-gateway/tests/stream_negotiation.rs +++ b/crates/tw-gateway/tests/stream_negotiation.rs @@ -16,10 +16,13 @@ fn response(status: u16, ct: &str, body: String) -> Response { .chunks(7) .map(|b| Ok::<_, std::io::Error>(Bytes::copy_from_slice(b))) .collect(); - Response::builder() + let mut builder = Response::builder() .status(status) - .header("content-type", ct) - .header("retry-after", "3600") + .header("content-type", ct); + if status == 429 { + builder = builder.header("retry-after", "3600"); + } + builder .body(Body::from_stream(futures::stream::iter(chunks))) .unwrap() } @@ -219,6 +222,204 @@ async fn both_directions_keep_the_client_protocol_tools_usage_and_learn_the_mode } } +#[tokio::test] +async fn opaque_rejections_can_probe_both_directions_and_learn_only_a_working_mode() { + for status in [400, 422, 501] { + for client_stream in [false, true] { + let (up, seen) = upstream(move |_, v, _| { + if v["stream"].as_bool().unwrap_or(false) == client_stream { + response( + status, + "application/json", + json!({"error":{ + "code":"vendor_1027", "message":"请使用另一种返回方式。" + }}) + .to_string(), + ) + } else if client_stream { + response(200, "application/json", whole().to_string()) + } else { + response(200, "text/event-stream", sse()) + } + }) + .await; + let (gw, _) = gateway(up, Protocol::OpenaiChat).await; + let (path, req) = request(Dialect::Chat, client_stream); + for count in [2, 3] { + let (status, ct, body) = post(gw, path, &req).await; + assert_eq!(status, 200, "{}", String::from_utf8_lossy(&body)); + assert_answer(Dialect::Chat, client_stream, path, &req, &ct, &body); + assert_eq!(seen.lock().unwrap().len(), count); + } + } + } +} + +#[tokio::test] +async fn failed_probes_preserve_the_original_response_and_both_attempt_outcomes() { + for probe_status in [400, 422, 501, 503, 401, 403, 429, 200] { + let original = json!({"error":{"code":"vendor_1027", "message":"Original rejection"}}); + let first = original.clone(); + let (up, seen) = upstream(move |_, _, n| { + if n == 1 { + let mut r = response(422, "application/json", first.to_string()); + r.headers_mut() + .insert("x-request-id", "original-id".parse().unwrap()); + r + } else if probe_status == 200 { + response(200, "text/event-stream", format!("data: {}\n\n", json!({"error":{"type":"invalid_request_error","message":"probe failed before content"}}))) + } else { + response( + probe_status, + "application/json", + json!({"error":{"message":"Probe rejection"}}).to_string(), + ) + } + }) + .await; + let (gw, state) = gateway(up, Protocol::OpenaiChat).await; + let mut events = state.bus.subscribe(); + let (path, req) = request(Dialect::Chat, false); + let r = reqwest::Client::new() + .post(format!("http://{gw}{path}")) + .header("authorization", "Bearer tw-k") + .json(&req) + .send() + .await + .unwrap(); + assert_eq!(r.status().as_u16(), 422, "probe {probe_status}"); + assert_eq!(r.headers()["x-request-id"], "original-id"); + assert_eq!(r.json::().await.unwrap(), original); + let sent = seen.lock().unwrap().clone(); + assert_eq!(sent.len(), 2, "probe {probe_status}"); + assert_eq!(sent[0].1["stream"], false); + assert_eq!(sent[1].1["stream"], true); + let mut routed = None; + let mut auth = Vec::new(); + while let Ok(event) = events.try_recv() { + match event { + tw_api::Event::RequestRouted { attempts, .. } => routed = Some(attempts), + tw_api::Event::AuthChanged { state, .. } => auth.push(state), + _ => {} + } + } + let attempts = routed.unwrap(); + assert_eq!(attempts.len(), 2); + assert_eq!(attempts[0].status, Some(422)); + assert_eq!(attempts[0].outcome, tw_api::AttemptOutcome::Served); + if probe_status == 200 { + assert_eq!(attempts[1].outcome, tw_api::AttemptOutcome::Error); + assert!( + attempts[1] + .error + .as_ref() + .unwrap() + .text + .contains("probe failed before content") + ); + } else { + assert_eq!(attempts[1].status, Some(probe_status)); + assert_eq!(attempts[1].outcome, tw_api::AttemptOutcome::Status); + } + if matches!(probe_status, 401 | 403) { + assert_eq!(auth.last(), Some(&tw_api::AuthState::Rejected)); + } + if probe_status == 429 { + assert!(attempts[1].pause_ms.is_some()); + assert!(!attempts[1].counts_for_upstream()); + } + } +} + +#[tokio::test] +async fn a_failed_server_error_probe_still_fails_over_with_the_original_request_mode() { + let (up, first) = upstream(|_, _, n| { + response( + if n == 1 { 501 } else { 503 }, + "application/json", + json!({"error":{"message":"Request rejected"}}).to_string(), + ) + }) + .await; + let (backup, second) = + upstream(|_, _, _| response(200, "application/json", whole().to_string())).await; + let cfg: Config = serde_yaml_ng::from_str(&format!( + "version: 1\ndefault_route: all\nclients:\n - name: c\n key: tw-k\nproviders:\n - name: up\n base_url: http://{up}\n protocol: openai-chat\n - name: next\n base_url: http://{backup}\n protocol: openai-chat\ngroups:\n - name: pool\n type: fallback\n providers: [up, next]\nroutes:\n - name: all\n rules:\n - name: all\n to: pool\n" + )).unwrap(); + let state = tw_gateway::AppState::new(cfg).unwrap(); + let gw = tw_gateway::serve(state.clone(), ([127, 0, 0, 1], 0).into()) + .await + .unwrap(); + let mut events = state.bus.subscribe(); + let (path, req) = request(Dialect::Chat, false); + let (status, ct, body) = post(gw, path, &req).await; + assert_eq!(status, 200, "{}", String::from_utf8_lossy(&body)); + assert_answer(Dialect::Chat, false, path, &req, &ct, &body); + let first = first.lock().unwrap(); + assert_eq!(first.len(), 2); + assert_eq!(first[0].1["stream"], false); + assert_eq!(first[1].1["stream"], true); + let second = second.lock().unwrap(); + assert_eq!(second.len(), 1); + assert_eq!(second[0].1["stream"], false); + let mut routed = None; + while let Ok(event) = events.try_recv() { + if let tw_api::Event::RequestRouted { attempts, .. } = event { + routed = Some(attempts); + } + } + let attempts = routed.unwrap(); + assert_eq!( + attempts.iter().map(|a| a.status).collect::>(), + vec![Some(501), Some(503), Some(200)] + ); +} + +#[tokio::test] +async fn an_unrelated_opaque_rejection_can_cost_one_probe_but_never_teaches_a_preference() { + let original = json!({"error":{"message":"Invalid request"}}); + let error = original.clone(); + let (up, seen) = + upstream(move |_, _, _| response(400, "application/json", error.to_string())).await; + let (gw, _) = gateway(up, Protocol::OpenaiChat).await; + let (path, req) = request(Dialect::Chat, false); + for count in [2, 4] { + let (status, _, body) = post(gw, path, &req).await; + assert_eq!(status, 400); + assert_eq!(serde_json::from_slice::(&body).unwrap(), original); + let seen = seen.lock().unwrap(); + assert_eq!(seen.len(), count); + assert_eq!(seen[count - 2].1["stream"], false); + assert_eq!(seen[count - 1].1["stream"], true); + } +} + +#[tokio::test] +async fn retry_hints_and_mislabelled_account_errors_do_not_probe() { + for error in [ + json!({"type":"authentication_error","message":"opaque"}), + json!({"code":"rate_limit_exceeded","message":"opaque"}), + json!({"status":"RESOURCE_EXHAUSTED","message":"opaque"}), + json!({"message":"Request timed out"}), + json!({"message":"请求频率过高"}), + json!({"message":"Retry later"}), + ] { + let with_hint = error["message"] == "Retry later"; + let (up, seen) = upstream(move |_, _, _| { + let mut r = response(400, "application/json", json!({"error":error}).to_string()); + if with_hint { + r.headers_mut().insert("retry-after", "30".parse().unwrap()); + } + r + }) + .await; + let (gw, _) = gateway(up, Protocol::OpenaiChat).await; + let (path, req) = request(Dialect::Chat, false); + assert_eq!(post(gw, path, &req).await.0, 400); + assert_eq!(seen.lock().unwrap().len(), 1); + } +} + #[tokio::test] async fn successful_opposite_mode_is_converted_without_an_extra_request() { for client_stream in [false, true] { @@ -283,7 +484,7 @@ async fn failed_alternatives_and_incomplete_or_error_streams_are_not_cached() { } #[tokio::test] -async fn rate_limits_authentication_and_unrelated_validation_never_switch_modes() { +async fn rate_limits_authentication_and_explicit_non_mode_validation_never_switch_modes() { for (status, message, param) in [ (429, "This model only supports streaming.", "stream"), (400, "Rate limit exceeded", "stream"), @@ -331,7 +532,7 @@ async fn native_response_features_the_mode_converter_cannot_preserve_are_not_ret } #[tokio::test] -async fn a_json_error_inside_http_200_is_kept_as_an_error_during_mode_conversion() { +async fn a_json_error_inside_http_200_preserves_the_original_error_and_is_not_cached() { for client_stream in [false, true] { let (up, seen) = upstream(move |_, v, _| { let stream = v["stream"].as_bool().unwrap_or(false); @@ -349,15 +550,18 @@ async fn a_json_error_inside_http_200_is_kept_as_an_error_during_mode_conversion let (gw, _) = gateway(up, Protocol::OpenaiChat).await; let (path, req) = request(Dialect::Chat, client_stream); for count in [2, 4] { - let (_, ct, body) = post(gw, path, &req).await; - assert!(ct.starts_with(if client_stream { - "text/event-stream" - } else { - "application/json" - })); - let text = String::from_utf8_lossy(&body); - assert!(text.contains("generation failed"), "{text}"); - assert!(text.contains("error"), "{text}"); + let (status, ct, body) = post(gw, path, &req).await; + assert_eq!(status, 400); + assert!(ct.starts_with("application/json")); + let error: Value = serde_json::from_slice(&body).unwrap(); + assert_eq!( + error["error"]["message"], + if client_stream { + "Streaming is not supported." + } else { + "This model only supports streaming." + } + ); assert_eq!(seen.lock().unwrap().len(), count); } } diff --git a/docs/config.md b/docs/config.md index daef62c2..72452004 100644 --- a/docs/config.md +++ b/docs/config.md @@ -540,21 +540,31 @@ the desktop app obtains by signing in; it cannot be written by hand. Claude and Google subscription sign-ins are not supported; use an API key. Generation requests first use the client's streaming or non-streaming mode. -When an upstream explicitly rejects that mode in an HTTP 400, 422 or 501 API -error, the gateway can try the opposite mode once on the same upstream. It -collects a streaming answer into a complete JSON response, or writes a complete +Known backend requirements take priority: ChatGPT account backends keep their +existing stream-only behavior. For other backends, an HTTP 400, 422 or 501 JSON +API rejection can trigger one trial of the opposite mode on the same upstream. +No particular mode-error code or English message is required. Known history, +reasoning-effort and cache-mark repairs run first and share the request's single +automatic repair budget; a trial also respects its wait deadline. Explicit errors +about another parameter, recognizable authentication, quota, rate-limit, overload +or timeout errors, and responses carrying a retry hint do not trigger a trial. +An unrelated opaque request error can therefore cost one extra rejected attempt. + +The gateway collects a streaming answer into complete JSON, or writes a complete answer as the stream the client requested. The latter still waits for the whole -upstream answer. This shares the request's single automatic repair budget with -history, reasoning-effort and cache-mark repairs, and respects its wait deadline. -Rate limits, authentication errors, timeouts and errors inside a successful -stream do not trigger a mode switch. A successful response already in the other -mode is converted without resending it. - -After a complete successful answer, the accepted alternative is remembered in -memory for 24 hours, isolated by upstream account, model, API and request options -such as tools and reasoning. Reloading the configuration clears this memory; -an explicit rejection invalidates a remembered mode. No upstream setting is -required. ChatGPT account backends keep their existing stream-only behavior. +upstream answer. A successful response already in the other mode is converted +without resending it. If the trial is itself rejected over HTTP, or fails before +its answer starts, the original error is retained; both attempt outcomes remain +in the request record. An original server error can still fail over to another +upstream. Partially received answers are never replayed, and errors inside a +successful stream do not initiate mode negotiation. + +Only a complete successful answer teaches a working preference. It is kept in +memory for 30 minutes from that trial, isolated by upstream account, model, API +and request options such as tools and reasoning. Using the preference does not +extend its expiry: after expiry the next request tries the client's mode again. +Reloading the configuration clears this memory; an eligible rejection invalidates +a remembered preference. No upstream setting is required. Native requests for multiple candidates, log probabilities or non-text output are not switched automatically: the current response converters cannot preserve those response fields. WebSocket requests are not negotiated. diff --git a/docs/config.zh-CN.md b/docs/config.zh-CN.md index 6d65f5ed..87d0e5a4 100644 --- a/docs/config.zh-CN.md +++ b/docs/config.zh-CN.md @@ -368,9 +368,11 @@ Bedrock 上只为 AWS 列出支持提示缓存的 Claude 模型标注:Claude 3 ChatGPT 账号上游(`protocol: chatgpt`)只接受桌面应用登录得到的凭据,不能手写。不支持 Claude 和 Google 的订阅登录,请使用 API 密钥。 -生成请求默认按客户端要求的流式或非流式模式发送。上游在 HTTP 400、422 或 501 的 API 错误中明确拒绝当前模式时,网关可以在同一家上游改用另一种模式重发一次:流式回答收齐后返回完整 JSON,整包回答则写成客户端要求的流。后一种仍需等待上游生成完整回答。它与历史字段、推理强度和缓存断点的修复共用每个请求最多一次的自动修复预算,并受请求等待期限约束。限流、凭据错误、超时和成功响应流中的错误不触发模式切换。上游已经成功返回另一种模式的响应时,直接转换,不再重发。 +生成请求默认按客户端要求的流式或非流式模式发送。已知的后端要求优先:ChatGPT 账号后端继续使用既有的仅流式处理。其他后端在 HTTP 400、422 或 501 的 JSON API 错误中拒绝请求时,网关可以在同一家上游试一次另一种模式,不要求特定错误码或英文文案。历史字段、推理强度和缓存断点的已知修复优先,与模式试探共用每个请求最多一次的自动修复预算,并受请求等待期限约束。明确指向其他参数的错误、可识别的鉴权、额度、限流、过载或超时错误,以及带重试提示的响应不试探。因此,无法识别原因的其他请求错误也可能多一次被拒的尝试。 -完整回答成功结束后,可用的替代模式在内存中记 24 小时,按上游账号、模型、API 和工具、推理等请求选项隔离;重载配置会清空,记住的模式被明确拒绝时会失效。无需增加上游配置。ChatGPT 账号后端继续使用既有的仅流式处理。 +流式回答收齐后返回完整 JSON,整包回答则写成客户端要求的流,后一种仍需等待上游生成完整回答。上游已经成功返回另一种模式的响应时,直接转换,不再重发。试探同样被 HTTP 拒绝、或者回答开始之前失败时,保留原始错误,两次尝试的结果都记录下来;原始服务端错误仍可故障转移到下一家上游。已经收到的部分回答不重放,成功响应流中的错误也不发起模式试探。 + +只有完整成功的回答才留下可用模式偏好,在内存中记 30 分钟,按上游账号、模型、API 和工具、推理等请求选项隔离。使用缓存不会延长过期时间,到期后下个请求重新尝试客户端模式;重载配置会清空,符合试探条件的拒绝会使记住的偏好失效。无需增加上游配置。 原生协议请求要求多个候选、对数概率或非文本输出时不自动切换,避免当前响应转换器丢失这些字段。WebSocket 请求不参与协商。 From b2b5e9c18d1e6225884e00f277f37709fa82fd6e Mon Sep 17 00:00:00 2001 From: fylorn <249551762+fylorn@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:59:12 +0800 Subject: [PATCH 4/5] fix(gateway): retain probe origin when later candidates are throttled --- crates/tw-gateway/src/server/pipeline/hop.rs | 1 + crates/tw-gateway/tests/stream_negotiation.rs | 115 ++++++++++++------ 2 files changed, 78 insertions(+), 38 deletions(-) diff --git a/crates/tw-gateway/src/server/pipeline/hop.rs b/crates/tw-gateway/src/server/pipeline/hop.rs index e5bb99bf..29328ccd 100644 --- a/crates/tw-gateway/src/server/pipeline/hop.rs +++ b/crates/tw-gateway/src/server/pipeline/hop.rs @@ -1196,6 +1196,7 @@ pub(super) async fn try_upstreams<'a>( // Learn the replacement only if the whole answer succeeds. rt.stream_modes.forget(key); if !repair_used && tokio::time::Instant::now() < until { + reached = true; repair_used = true; stream_trial = Some((key.clone(), !mode)); state.note_quota(id, &provider.name, &headers); diff --git a/crates/tw-gateway/tests/stream_negotiation.rs b/crates/tw-gateway/tests/stream_negotiation.rs index 16a4b88b..06862cb1 100644 --- a/crates/tw-gateway/tests/stream_negotiation.rs +++ b/crates/tw-gateway/tests/stream_negotiation.rs @@ -333,46 +333,85 @@ async fn failed_probes_preserve_the_original_response_and_both_attempt_outcomes( #[tokio::test] async fn a_failed_server_error_probe_still_fails_over_with_the_original_request_mode() { - let (up, first) = upstream(|_, _, n| { - response( - if n == 1 { 501 } else { 503 }, - "application/json", - json!({"error":{"message":"Request rejected"}}).to_string(), - ) - }) - .await; - let (backup, second) = - upstream(|_, _, _| response(200, "application/json", whole().to_string())).await; - let cfg: Config = serde_yaml_ng::from_str(&format!( - "version: 1\ndefault_route: all\nclients:\n - name: c\n key: tw-k\nproviders:\n - name: up\n base_url: http://{up}\n protocol: openai-chat\n - name: next\n base_url: http://{backup}\n protocol: openai-chat\ngroups:\n - name: pool\n type: fallback\n providers: [up, next]\nroutes:\n - name: all\n rules:\n - name: all\n to: pool\n" - )).unwrap(); - let state = tw_gateway::AppState::new(cfg).unwrap(); - let gw = tw_gateway::serve(state.clone(), ([127, 0, 0, 1], 0).into()) - .await - .unwrap(); - let mut events = state.bus.subscribe(); - let (path, req) = request(Dialect::Chat, false); - let (status, ct, body) = post(gw, path, &req).await; - assert_eq!(status, 200, "{}", String::from_utf8_lossy(&body)); - assert_answer(Dialect::Chat, false, path, &req, &ct, &body); - let first = first.lock().unwrap(); - assert_eq!(first.len(), 2); - assert_eq!(first[0].1["stream"], false); - assert_eq!(first[1].1["stream"], true); - let second = second.lock().unwrap(); - assert_eq!(second.len(), 1); - assert_eq!(second[0].1["stream"], false); - let mut routed = None; - while let Ok(event) = events.try_recv() { - if let tw_api::Event::RequestRouted { attempts, .. } = event { - routed = Some(attempts); + for (probe_status, backup_status) in [(503, 200), (200, 200), (200, 429)] { + let (up, first) = upstream(move |_, _, n| { + if n == 1 { + response( + 501, + "application/json", + json!({"error":{"message":"Request rejected"}}).to_string(), + ) + } else if probe_status == 200 { + response( + 200, + "text/event-stream", + format!( + "data: {}\n\n", + json!({"error":{"type":"invalid_request_error","message":"probe failed"}}) + ), + ) + } else { + response( + probe_status, + "application/json", + json!({"error":{"message":"Request rejected"}}).to_string(), + ) + } + }) + .await; + let (backup, second) = upstream(move |_, _, _| { + response(backup_status, "application/json", whole().to_string()) + }) + .await; + let mut cfg: Config = serde_yaml_ng::from_str(&format!( + "version: 1\ndefault_route: all\nclients:\n - name: c\n key: tw-k\nproviders:\n - name: up\n base_url: http://{up}\n protocol: openai-chat\n - name: next\n base_url: http://{backup}\n protocol: openai-chat\ngroups:\n - name: pool\n type: fallback\n providers: [up, next]\nroutes:\n - name: all\n rules:\n - name: all\n to: pool\n" + )).unwrap(); + cfg.failover.wait_secs = 1; + let state = tw_gateway::AppState::new(cfg).unwrap(); + let gw = tw_gateway::serve(state.clone(), ([127, 0, 0, 1], 0).into()) + .await + .unwrap(); + let mut events = state.bus.subscribe(); + let (path, req) = request(Dialect::Chat, false); + let (status, ct, body) = post(gw, path, &req).await; + if backup_status == 200 { + assert_eq!(status, 200, "{}", String::from_utf8_lossy(&body)); + assert_answer(Dialect::Chat, false, path, &req, &ct, &body); + } else { + // A limited backup must not hide the original server rejection. + assert_eq!(status, 501, "{}", String::from_utf8_lossy(&body)); + assert_eq!( + serde_json::from_slice::(&body).unwrap()["error"]["message"], + "Request rejected" + ); + } + let first = first.lock().unwrap(); + assert_eq!(first.len(), 2); + assert_eq!(first[0].1["stream"], false); + assert_eq!(first[1].1["stream"], true); + let second = second.lock().unwrap(); + assert_eq!(second.len(), 1); + assert_eq!(second[0].1["stream"], false); + let mut routed = None; + while let Ok(event) = events.try_recv() { + if let tw_api::Event::RequestRouted { attempts, .. } = event { + routed = Some(attempts); + } } + let attempts = routed.unwrap(); + assert_eq!( + attempts.iter().map(|a| a.status).collect::>(), + vec![ + Some(501), + if probe_status == 200 { + None + } else { + Some(probe_status) + }, + Some(backup_status) + ] + ); } - let attempts = routed.unwrap(); - assert_eq!( - attempts.iter().map(|a| a.status).collect::>(), - vec![Some(501), Some(503), Some(200)] - ); } #[tokio::test] From 3c0f798cbfc8610aa8b77331264640697ac8869a Mon Sep 17 00:00:00 2001 From: fylorn <249551762+fylorn@users.noreply.github.com> Date: Thu, 12 Nov 2026 23:40:13 +0800 Subject: [PATCH 5/5] chore(release): prepare Core 0.96.0 --- Cargo.lock | 44 ++++++++++++++++++++--------------------- Cargo.toml | 2 +- release-notes/0.96.0.md | 3 +++ 3 files changed, 26 insertions(+), 23 deletions(-) create mode 100644 release-notes/0.96.0.md diff --git a/Cargo.lock b/Cargo.lock index 1fdff976..97d3db69 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3763,7 +3763,7 @@ dependencies = [ [[package]] name = "tw-api" -version = "0.95.0" +version = "0.96.0" dependencies = [ "serde", "serde_json", @@ -3774,7 +3774,7 @@ dependencies = [ [[package]] name = "tw-bedrock" -version = "0.95.0" +version = "0.96.0" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -3795,7 +3795,7 @@ dependencies = [ [[package]] name = "tw-breaker" -version = "0.95.0" +version = "0.96.0" dependencies = [ "serde", "serde_json", @@ -3803,7 +3803,7 @@ dependencies = [ [[package]] name = "tw-config" -version = "0.95.0" +version = "0.96.0" dependencies = [ "blake3", "libc", @@ -3829,7 +3829,7 @@ dependencies = [ [[package]] name = "tw-control" -version = "0.95.0" +version = "0.96.0" dependencies = [ "axum", "base64 0.22.1", @@ -3875,7 +3875,7 @@ dependencies = [ [[package]] name = "tw-dialect" -version = "0.95.0" +version = "0.96.0" dependencies = [ "memchr", "serde", @@ -3884,7 +3884,7 @@ dependencies = [ [[package]] name = "tw-engine" -version = "0.95.0" +version = "0.96.0" dependencies = [ "chrono", "serde", @@ -3898,7 +3898,7 @@ dependencies = [ [[package]] name = "tw-gateway" -version = "0.95.0" +version = "0.96.0" dependencies = [ "arc-swap", "async-stream", @@ -3950,7 +3950,7 @@ dependencies = [ [[package]] name = "tw-guard" -version = "0.95.0" +version = "0.96.0" dependencies = [ "aho-corasick", "base64 0.22.1", @@ -3971,7 +3971,7 @@ dependencies = [ [[package]] name = "tw-handover" -version = "0.95.0" +version = "0.96.0" dependencies = [ "libc", "serde", @@ -3984,7 +3984,7 @@ dependencies = [ [[package]] name = "tw-link" -version = "0.95.0" +version = "0.96.0" dependencies = [ "serde", "serde_json", @@ -3998,7 +3998,7 @@ dependencies = [ [[package]] name = "tw-observe" -version = "0.95.0" +version = "0.96.0" dependencies = [ "tokio", "tracing", @@ -4007,7 +4007,7 @@ dependencies = [ [[package]] name = "tw-pack" -version = "0.95.0" +version = "0.96.0" dependencies = [ "age", "blake3", @@ -4031,7 +4031,7 @@ dependencies = [ [[package]] name = "tw-plugin" -version = "0.95.0" +version = "0.96.0" dependencies = [ "libc", "rand 0.10.2", @@ -4047,7 +4047,7 @@ dependencies = [ [[package]] name = "tw-pricing" -version = "0.95.0" +version = "0.96.0" dependencies = [ "arc-swap", "flate2", @@ -4060,14 +4060,14 @@ dependencies = [ [[package]] name = "tw-secret" -version = "0.95.0" +version = "0.96.0" dependencies = [ "thiserror 2.0.20", ] [[package]] name = "tw-store" -version = "0.95.0" +version = "0.96.0" dependencies = [ "blake3", "bytes", @@ -4087,7 +4087,7 @@ dependencies = [ [[package]] name = "tw-sync" -version = "0.95.0" +version = "0.96.0" dependencies = [ "age", "axum", @@ -4115,7 +4115,7 @@ dependencies = [ [[package]] name = "tw-types" -version = "0.95.0" +version = "0.96.0" dependencies = [ "serde", "serde_json", @@ -4124,7 +4124,7 @@ dependencies = [ [[package]] name = "tw-watch" -version = "0.95.0" +version = "0.96.0" dependencies = [ "libc", "notify", @@ -4135,7 +4135,7 @@ dependencies = [ [[package]] name = "tw-yaml" -version = "0.95.0" +version = "0.96.0" dependencies = [ "saphyr-parser", "serde_yaml_ng", @@ -4145,7 +4145,7 @@ dependencies = [ [[package]] name = "twcore" -version = "0.95.0" +version = "0.96.0" dependencies = [ "anyhow", "axum", diff --git a/Cargo.toml b/Cargo.toml index 9f0678a3..41b92b8f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,7 +61,7 @@ categories = ["network-programming", "web-programming::http-server"] # 二进制走 CalVer,crate 走 SemVer —— 两者是两回事,卖给不同的人。 # 这里是 crate 的版本。 -version = "0.95.0" +version = "0.96.0" [workspace.dependencies] # ── 内部 crate ─────────────────────────────────────────── diff --git a/release-notes/0.96.0.md b/release-notes/0.96.0.md new file mode 100644 index 00000000..907d475b --- /dev/null +++ b/release-notes/0.96.0.md @@ -0,0 +1,3 @@ +Generation requests automatically adapt to upstream streaming and non-streaming support. Requests follow the client's mode first; an eligible HTTP 400, 422 or 501 API rejection can trigger one bounded trial of the opposite mode. A successful answer is converted back to the client's protocol and response framing, including tool calls and usage. Complete successful trials teach a working preference for 30 minutes, isolated by upstream account, model, API and request options. + +Known request repairs take priority and share one automatic repair budget. Failed trials preserve the original error, partially delivered answers are never replayed, and configuration reloads clear learned preferences. Authentication, quota, rate-limit, overload and timeout errors do not trigger mode trials; a `Rate limit exceeded` message alone does not establish that a response mode is unsupported. Whole-to-stream conversion waits for the complete upstream answer. WebSocket requests and richer native output contracts that cannot be preserved by the response converters are excluded.