From f1d4de6fa1095bef1ed8ab2d1faf8550e7d497ca Mon Sep 17 00:00:00 2001
From: fylorn <249551762+fylorn@users.noreply.github.com>
Date: Sat, 14 Nov 2026 23:39:36 +0800
Subject: [PATCH 1/3] feat(web-tools): execute hosted search and page tools in
Core
Run standalone Claude Code searches without a helper model and retain normal metering for each model round in hosted Anthropic and Responses requests. Share protected collection, caching and configuration with the control-plane tester, including public-IP proxy tunnels and native HTTP/SSE/WebSocket adapters.
---
Cargo.lock | 260 ++++++++
crates/tw-api/msg-codes.txt | 2 +
crates/tw-api/src/ep.rs | 3 +
crates/tw-api/src/lib.rs | 6 +-
crates/tw-api/src/web_tools.rs | 138 +++++
crates/tw-config/src/lib.rs | 5 +
crates/tw-config/src/refs.rs | 22 +-
crates/tw-config/src/validate.rs | 6 +
crates/tw-config/src/web_tools.rs | 86 +++
crates/tw-config/tests/manual/schema.rs | 153 +++++
crates/tw-control/src/lib.rs | 2 +
crates/tw-control/src/web_tools.rs | 52 ++
crates/tw-control/tests/web_tools.rs | 105 ++++
crates/tw-gateway/Cargo.toml | 5 +
crates/tw-gateway/src/l1.rs | 11 +-
crates/tw-gateway/src/lib.rs | 2 +
crates/tw-gateway/src/server.rs | 1 +
crates/tw-gateway/src/server/pipeline.rs | 67 +-
.../src/server/pipeline/admission.rs | 2 +-
crates/tw-gateway/src/server/pipeline/web.rs | 426 +++++++++++++
.../src/server/pipeline/web/socket.rs | 264 ++++++++
.../src/server/pipeline/web/tests.rs | 390 ++++++++++++
crates/tw-gateway/src/server/upgrade.rs | 20 +
crates/tw-gateway/src/state.rs | 8 +
crates/tw-gateway/src/web_tools/collect.rs | 281 +++++++++
crates/tw-gateway/src/web_tools/history.rs | 234 +++++++
crates/tw-gateway/src/web_tools/mod.rs | 585 ++++++++++++++++++
crates/tw-gateway/src/web_tools/protocol.rs | 375 +++++++++++
crates/tw-gateway/src/web_tools/public.rs | 432 +++++++++++++
crates/tw-gateway/src/web_tools/wire.rs | 238 +++++++
crates/tw-gateway/tests/passthrough.rs | 5 +
crates/tw-pack/src/model/items.rs | 12 +
crates/tw-pack/src/model/mod.rs | 2 +
crates/tw-pack/src/model/rename.rs | 3 +
crates/tw-pack/src/model/tests.rs | 23 +
crates/tw-pack/src/model/write.rs | 4 +
crates/tw-pack/src/repair.rs | 1 +
docs/config.md | 31 +
docs/config.zh-CN.md | 31 +
scripts/smoke.sh | 2 +-
40 files changed, 4286 insertions(+), 9 deletions(-)
create mode 100644 crates/tw-api/src/web_tools.rs
create mode 100644 crates/tw-config/src/web_tools.rs
create mode 100644 crates/tw-control/src/web_tools.rs
create mode 100644 crates/tw-control/tests/web_tools.rs
create mode 100644 crates/tw-gateway/src/server/pipeline/web.rs
create mode 100644 crates/tw-gateway/src/server/pipeline/web/socket.rs
create mode 100644 crates/tw-gateway/src/server/pipeline/web/tests.rs
create mode 100644 crates/tw-gateway/src/web_tools/collect.rs
create mode 100644 crates/tw-gateway/src/web_tools/history.rs
create mode 100644 crates/tw-gateway/src/web_tools/mod.rs
create mode 100644 crates/tw-gateway/src/web_tools/protocol.rs
create mode 100644 crates/tw-gateway/src/web_tools/public.rs
create mode 100644 crates/tw-gateway/src/web_tools/wire.rs
diff --git a/Cargo.lock b/Cargo.lock
index 97d3db69..a001c227 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -982,6 +982,29 @@ dependencies = [
"hybrid-array",
]
+[[package]]
+name = "cssparser"
+version = "0.38.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "11119743ad110e8c1bdccd930d7f5c30c99e5fc76a7b63ec9807e84eef0c5f59"
+dependencies = [
+ "cssparser-macros",
+ "dtoa-short",
+ "itoa",
+ "phf",
+ "smallvec",
+]
+
+[[package]]
+name = "cssparser-macros"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d045de693cb712d0b22c6a64be5b953f67b3ce00ab5ad3dd5d8b441886ab8e1a"
+dependencies = [
+ "quote",
+ "syn 3.0.5",
+]
+
[[package]]
name = "ctutils"
version = "0.4.5"
@@ -1029,6 +1052,27 @@ version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c"
+[[package]]
+name = "derive_more"
+version = "2.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134"
+dependencies = [
+ "derive_more-impl",
+]
+
+[[package]]
+name = "derive_more-impl"
+version = "2.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "rustc_version",
+ "syn 2.0.119",
+]
+
[[package]]
name = "digest"
version = "0.10.7"
@@ -1063,12 +1107,33 @@ dependencies = [
"syn 3.0.5",
]
+[[package]]
+name = "dtoa"
+version = "1.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590"
+
+[[package]]
+name = "dtoa-short"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87"
+dependencies = [
+ "dtoa",
+]
+
[[package]]
name = "dunce"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
+[[package]]
+name = "ego-tree"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7bd50d24b1b75aa817bb0339cb9f2e0e664bdfab1e160a69a0fda9145b77fab2"
+
[[package]]
name = "either"
version = "1.18.0"
@@ -1331,6 +1396,15 @@ dependencies = [
"version_check",
]
+[[package]]
+name = "getopts"
+version = "0.2.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df"
+dependencies = [
+ "unicode-width",
+]
+
[[package]]
name = "getrandom"
version = "0.2.17"
@@ -1469,6 +1543,17 @@ dependencies = [
"digest 0.11.3",
]
+[[package]]
+name = "html5ever"
+version = "0.40.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "456a1a377e608e555d22ddab27ac0114bc7a7b4199078108e34c2aeae6c9b130"
+dependencies = [
+ "log",
+ "markup5ever",
+ "memchr",
+]
+
[[package]]
name = "http"
version = "1.5.0"
@@ -2045,6 +2130,17 @@ version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dae608c151f68243f2b000364e1f7b186d9c29845f7d2d85bd31b9ad77ad552b"
+[[package]]
+name = "markup5ever"
+version = "0.40.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ab3dc68ac4a0f5719e560136778c1ee716e296030d75dbd4484e37e39e3a842"
+dependencies = [
+ "log",
+ "tendril",
+ "web_atoms",
+]
+
[[package]]
name = "matchers"
version = "0.2.0"
@@ -2128,6 +2224,12 @@ dependencies = [
"windows-sys 0.61.2",
]
+[[package]]
+name = "new_debug_unreachable"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"
+
[[package]]
name = "nix"
version = "0.31.3"
@@ -2319,6 +2421,59 @@ version = "2.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
+[[package]]
+name = "phf"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "010378780309880b08997fae13be7834dba947d36393bd372f2b1556deb2a2f6"
+dependencies = [
+ "phf_macros",
+ "phf_shared",
+ "serde",
+]
+
+[[package]]
+name = "phf_codegen"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41b585a510fb76fdebead6897982ef2a03a21d8e6cbcca904999742a4afc6ffe"
+dependencies = [
+ "phf_generator",
+ "phf_shared",
+]
+
+[[package]]
+name = "phf_generator"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aeb62e0959d5a1bebc965f4d15d9e2b7cea002b6b0f5ba8cde6cc26738467100"
+dependencies = [
+ "fastrand",
+ "phf_shared",
+]
+
+[[package]]
+name = "phf_macros"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fa8d0ca26d424d27630da600c6624696e7dec8bf7b3b492b383c5dc49e5e085"
+dependencies = [
+ "phf_generator",
+ "phf_shared",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.119",
+]
+
+[[package]]
+name = "phf_shared"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6fd9027e2d9319be6349febd1db4e8d02aa544921200c9b777720ac34a3aa89"
+dependencies = [
+ "siphasher",
+]
+
[[package]]
name = "pin-project"
version = "1.1.13"
@@ -2404,6 +2559,12 @@ dependencies = [
"zerocopy",
]
+[[package]]
+name = "precomputed-hash"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
+
[[package]]
name = "proc-macro-error-attr2"
version = "2.0.0"
@@ -2990,6 +3151,20 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+[[package]]
+name = "scraper"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6844b5f49a5579f12ba4f89d9f7eec6d71889e4aa0bc7181821040ce66f08fff"
+dependencies = [
+ "cssparser",
+ "ego-tree",
+ "getopts",
+ "html5ever",
+ "precomputed-hash",
+ "selectors",
+]
+
[[package]]
name = "scrypt"
version = "0.11.0"
@@ -3033,6 +3208,23 @@ dependencies = [
"libc",
]
+[[package]]
+name = "selectors"
+version = "0.41.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8322aebea94a6cc288ff884e8a49bdbceef60206c400c6c46d22e62fc4fd91a6"
+dependencies = [
+ "bitflags",
+ "cssparser",
+ "derive_more",
+ "log",
+ "new_debug_unreachable",
+ "precomputed-hash",
+ "rustc-hash 2.1.3",
+ "servo_arc",
+ "smallvec",
+]
+
[[package]]
name = "self_cell"
version = "0.10.3"
@@ -3137,6 +3329,15 @@ dependencies = [
"unsafe-libyaml",
]
+[[package]]
+name = "servo_arc"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4eceb519ec34d8290aaf6e46f435102f2bfa63f22b05a144fb16ab266d3b198"
+dependencies = [
+ "stable_deref_trait",
+]
+
[[package]]
name = "sha1"
version = "0.10.7"
@@ -3217,6 +3418,12 @@ version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
+[[package]]
+name = "siphasher"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256"
+
[[package]]
name = "slab"
version = "0.4.12"
@@ -3274,6 +3481,30 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
+[[package]]
+name = "string_cache"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ffa8a5dbe8b3f0bbe29d4c3225daafaeead63afdc1b65fc4c01a1384166038e6"
+dependencies = [
+ "new_debug_unreachable",
+ "parking_lot",
+ "phf_shared",
+ "precomputed-hash",
+]
+
+[[package]]
+name = "string_cache_codegen"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "928dcdf75e47626b3617a976ec205d9f057584c371c1f23b782129268d0e6edc"
+dependencies = [
+ "phf_generator",
+ "phf_shared",
+ "proc-macro2",
+ "quote",
+]
+
[[package]]
name = "strsim"
version = "0.11.1"
@@ -3382,6 +3613,15 @@ dependencies = [
"windows-sys 0.61.2",
]
+[[package]]
+name = "tendril"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fed54709c5b3a53d09bb1c113ea4f5ceafd1e772ddcb0030a82e1d56c087b08"
+dependencies = [
+ "new_debug_unreachable",
+]
+
[[package]]
name = "termcolor"
version = "1.4.1"
@@ -3916,10 +4156,12 @@ dependencies = [
"hyper",
"hyper-util",
"memchr",
+ "quick-xml",
"rand 0.10.2",
"reqwest",
"rustls",
"rustls-platform-verifier",
+ "scraper",
"serde",
"serde_json",
"serde_yaml_ng",
@@ -4234,6 +4476,12 @@ version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
+[[package]]
+name = "unicode-width"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca718471876992059f251e33aafc334346ada08042b7661dd1fe313a9bced557"
+
[[package]]
name = "universal-hash"
version = "0.5.1"
@@ -4666,6 +4914,18 @@ dependencies = [
"wasm-bindgen",
]
+[[package]]
+name = "web_atoms"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7572660c8890448ba236b7376f27e389c6a7e1c70195622faced601f855c0ada"
+dependencies = [
+ "phf",
+ "phf_codegen",
+ "string_cache",
+ "string_cache_codegen",
+]
+
[[package]]
name = "webpki-root-certs"
version = "1.0.10"
diff --git a/crates/tw-api/msg-codes.txt b/crates/tw-api/msg-codes.txt
index b55a524a..08b1a660 100644
--- a/crates/tw-api/msg-codes.txt
+++ b/crates/tw-api/msg-codes.txt
@@ -146,6 +146,7 @@ config.upstream_limit_every_calendar
config.upstream_limit_not_positive
config.upstream_limit_retention
config.upstream_limit_two_measures
+config.web_tools.invalid
config.zero_concurrency
control.account_service_not_json
control.account_service_refused
@@ -474,6 +475,7 @@ gw.wait.mixed
gw.wait.rate_limited
gw.wait.used_up
gw.wait.window_full
+gw.web_tools.failed
gw.ws.alias_unserved
gw.ws.alias_unserved_rewritten
gw.ws.bad_header
diff --git a/crates/tw-api/src/ep.rs b/crates/tw-api/src/ep.rs
index dee382fa..ed1122ac 100644
--- a/crates/tw-api/src/ep.rs
+++ b/crates/tw-api/src/ep.rs
@@ -66,6 +66,9 @@ endpoints! {
ConfigRepairPlan: GET "/config/repair", () => api::ConfigRepair;
/// 照那几处修好写回:取值改回默认值、删掉不认识的字段
RepairConfig: POST "/config/repair", api::ConfigRepairRequest => api::ConfigWritten;
+ WebTools: GET "/web-tools", () => api::WebToolsView;
+ SaveWebTools: PUT "/web-tools", api::WebToolsSave => api::ConfigWritten;
+ TestWebTool: POST "/web-tools/test", api::WebToolRequest => api::WebToolResult;
SaveListen: PUT "/listen", api::ListenSave => api::ConfigWritten;
// ─────────────────────────────────────────────── 用量与记录
diff --git a/crates/tw-api/src/lib.rs b/crates/tw-api/src/lib.rs
index 7fd5971b..4454612b 100644
--- a/crates/tw-api/src/lib.rs
+++ b/crates/tw-api/src/lib.rs
@@ -17,6 +17,8 @@ pub use tw_types::Msg;
mod endpoint;
pub mod ep;
+mod web_tools;
+pub use web_tools::*;
#[cfg(feature = "ts")]
pub mod ts;
pub use endpoint::{Endpoint, ErrorBody, Format, Info, Method, fill};
@@ -1222,7 +1224,9 @@ pub const MSG_CODES: &str = include_str!("../msg-codes.txt");
/// **60 起模型规格支持手动推理档位**:[`ModelSpecSave::reasoning_effort`] 区分保留、
/// 设置和恢复自动。[`ModelRow`] 带上游报告、手动覆盖、实际可用档位和过滤结果。
/// [`SpecSource::Upstream`] 表示上游报告的规格。请求库 SCHEMA 不变。
-pub const CONTROL_API_VERSION: u32 = 60;
+/// **61 起支持 Core 执行联网工具**:新增 `WebTools`、`SaveWebTools` 和 `TestWebTool`。
+/// 搜索、网页读取和页内查找共用当前连接 Core 的配置。请求库 SCHEMA 不变。
+pub const CONTROL_API_VERSION: u32 = 61;
/// `/live` 的速度看最近多久(见 [`LiveView::tokens_per_sec`]),毫秒
pub const LIVE_RATE_WINDOW_MS: u64 = 3_000;
diff --git a/crates/tw-api/src/web_tools.rs b/crates/tw-api/src/web_tools.rs
new file mode 100644
index 00000000..020b7209
--- /dev/null
+++ b/crates/tw-api/src/web_tools.rs
@@ -0,0 +1,138 @@
+//! The gateway's web tools. Configuration belongs to the connected Core.
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
+#[cfg_attr(feature = "ts", derive(ts_rs::TS))]
+#[serde(rename_all = "snake_case")]
+pub enum WebToolsMode {
+ #[default]
+ Upstream,
+ Core,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+#[cfg_attr(feature = "ts", derive(ts_rs::TS))]
+#[serde(rename_all = "snake_case")]
+pub enum SearchEngine {
+ Bing,
+ Duckduckgo,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[cfg_attr(feature = "ts", derive(ts_rs::TS))]
+#[serde(default, deny_unknown_fields)]
+pub struct WebToolsConfig {
+ pub mode: WebToolsMode,
+ /// Empty chooses healthy engines automatically; otherwise tried in this order.
+ pub engines: Vec,
+ /// An existing proxy name, "system", or "direct".
+ pub proxy: String,
+ pub language: String,
+ pub region: String,
+ pub allowed_domains: Vec,
+ pub blocked_domains: Vec,
+ pub max_results: u32,
+ /// Number of search result pages to read when snippets are insufficient. Zero disables it.
+ pub read_pages: u32,
+ pub max_page_chars: u32,
+ pub timeout_secs: u32,
+ pub max_concurrent: u32,
+ pub interval_ms: u32,
+ pub cache_secs: u32,
+ pub max_tool_calls: u32,
+ /// Only before any search result is used; never silently switches paid services on.
+ pub fallback_to_upstream: bool,
+}
+
+impl Default for WebToolsConfig {
+ fn default() -> Self {
+ Self {
+ mode: WebToolsMode::Upstream,
+ engines: Vec::new(),
+ proxy: "system".into(),
+ language: String::new(),
+ region: String::new(),
+ allowed_domains: Vec::new(),
+ blocked_domains: Vec::new(),
+ max_results: 8,
+ read_pages: 2,
+ max_page_chars: 12_000,
+ timeout_secs: 30,
+ max_concurrent: 4,
+ interval_ms: 1000,
+ cache_secs: 600,
+ max_tool_calls: 8,
+ fallback_to_upstream: false,
+ }
+ }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "ts", derive(ts_rs::TS))]
+pub struct WebToolsView {
+ pub config: WebToolsConfig,
+ pub config_version: String,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "ts", derive(ts_rs::TS))]
+pub struct WebToolsSave {
+ pub base_version: String,
+ pub config: WebToolsConfig,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "ts", derive(ts_rs::TS))]
+#[serde(tag = "action", rename_all = "snake_case", deny_unknown_fields)]
+pub enum WebToolRequest {
+ Search {
+ query: String,
+ #[serde(default)]
+ allowed_domains: Vec,
+ #[serde(default)]
+ blocked_domains: Vec,
+ },
+ OpenPage {
+ url: String,
+ #[serde(default)]
+ offset: u32,
+ },
+ FindInPage {
+ url: String,
+ pattern: String,
+ },
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "ts", derive(ts_rs::TS))]
+pub struct WebSource {
+ pub url: String,
+ pub title: String,
+ pub text: String,
+ pub engine: Option,
+ pub published_at: Option,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "ts", derive(ts_rs::TS))]
+pub struct WebToolResult {
+ pub sources: Vec,
+ pub cached: bool,
+ pub elapsed_ms: u32,
+ /// Partial-source failures, distinct from a successful search with zero results.
+ pub warnings: Vec,
+ pub truncated: bool,
+}
+
+/// Compact, non-content diagnostics carried in the recorded response headers.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[cfg_attr(feature = "ts", derive(ts_rs::TS))]
+pub struct WebToolTrace {
+ pub action: String,
+ pub engines: Vec,
+ pub cached: bool,
+ pub elapsed_ms: u32,
+ pub sources: u32,
+ pub warnings: u32,
+}
diff --git a/crates/tw-config/src/lib.rs b/crates/tw-config/src/lib.rs
index 70ec66fb..ee54ad93 100644
--- a/crates/tw-config/src/lib.rs
+++ b/crates/tw-config/src/lib.rs
@@ -33,6 +33,7 @@ pub mod store;
pub mod sync;
mod validate;
pub mod watch;
+mod web_tools;
mod wire;
pub use aliases::{Alias, Aliases};
@@ -96,6 +97,9 @@ pub struct Config {
/// 客户端自己发的辅助请求怎么处理。默认只拦 A 类。
#[serde(default, skip_serializing_if = "is_default")]
pub client_probes: ClientProbes,
+ /// Search and page tools executed by this gateway.
+ #[serde(default, skip_serializing_if = "is_default")]
+ pub web_tools: tw_api::WebToolsConfig,
/// 两项防护:出站脱敏、工具调用审查。**出厂时都停在「观察」** ——
/// 只记录,不改变任何行为。对所有上游一视同仁。
#[serde(default, skip_serializing_if = "is_default")]
@@ -153,6 +157,7 @@ impl Default for Config {
proxies: Vec::new(),
pricing: tw_pricing::PricingConfig::default(),
client_probes: ClientProbes::default(),
+ web_tools: tw_api::WebToolsConfig::default(),
security: Security::default(),
retention: Retention::default(),
failover: Failover::default(),
diff --git a/crates/tw-config/src/refs.rs b/crates/tw-config/src/refs.rs
index f4022833..c2d64cca 100644
--- a/crates/tw-config/src/refs.rs
+++ b/crates/tw-config/src/refs.rs
@@ -59,12 +59,13 @@ pub fn provider_refs(cfg: &Config, name: &str) -> Vec {
out
}
-/// 用着 `name` 这个代理的上游。
+/// 用着 `name` 这个代理的上游或网关功能。
pub fn proxy_users(cfg: &Config, name: &str) -> Vec {
cfg.providers
.iter()
.filter(|p| p.proxy == name)
.map(|p| p.name.clone())
+ .chain((cfg.web_tools.proxy == name).then(|| "web_tools".to_string()))
.collect()
}
@@ -474,6 +475,13 @@ pub fn rename_proxy(text: &str, cfg: &Config, old: &str, new: &str) -> Result crate::policy_msg(e),
+ WebTools(detail) => {
+ msg!("config.web_tools.invalid", detail = detail => "web_tools: {detail}")
+ }
FailoverRange {
field,
value,
@@ -661,6 +666,7 @@ impl ValidationError {
}
pub fn validate(cfg: &Config) -> Result<(), ValidationError> {
+ crate::web_tools::validate(cfg).map_err(ValidationError::WebTools)?;
// 版本检查放在最前面:一个来自更新版本的配置,我们对它的任何
// 其他判断都不作数(「schema 太新」)。
if cfg.version > SCHEMA_VERSION {
diff --git a/crates/tw-config/src/web_tools.rs b/crates/tw-config/src/web_tools.rs
new file mode 100644
index 00000000..9642036d
--- /dev/null
+++ b/crates/tw-config/src/web_tools.rs
@@ -0,0 +1,86 @@
+use crate::Config;
+
+pub(crate) fn validate(cfg: &Config) -> Result<(), String> {
+ let c = &cfg.web_tools;
+ for (name, value, min, max) in [
+ ("max_results", c.max_results, 1, 20),
+ ("read_pages", c.read_pages, 0, 5),
+ ("max_page_chars", c.max_page_chars, 1000, 50_000),
+ ("timeout_secs", c.timeout_secs, 1, 120),
+ ("max_concurrent", c.max_concurrent, 1, 16),
+ ("interval_ms", c.interval_ms, 250, 60_000),
+ ("cache_secs", c.cache_secs, 0, 86_400),
+ ("max_tool_calls", c.max_tool_calls, 1, 32),
+ ] {
+ if !(min..=max).contains(&value) {
+ return Err(format!("{name} must be between {min} and {max}"));
+ }
+ }
+ if c.engines
+ .iter()
+ .collect::>()
+ .len()
+ != c.engines.len()
+ {
+ return Err("engines must not contain duplicates".into());
+ }
+ if c.proxy != crate::SYSTEM
+ && c.proxy != crate::DIRECT
+ && !cfg.proxies.iter().any(|p| p.name == c.proxy)
+ {
+ return Err("proxy must be system, direct, or an existing proxy name".into());
+ }
+ if c.allowed_domains.len() > 100 || c.blocked_domains.len() > 100 {
+ return Err("each domain filter supports at most 100 hostnames".into());
+ }
+ for domain in c.allowed_domains.iter().chain(&c.blocked_domains) {
+ if !valid_domain(domain) {
+ return Err(
+ "domain rules must be hostnames without a scheme, path, port, or wildcard".into(),
+ );
+ }
+ }
+ for value in [&c.language, &c.region] {
+ if value.len() > 35
+ || !value
+ .bytes()
+ .all(|b| b.is_ascii_alphanumeric() || b == b'-')
+ {
+ return Err("language and region must be empty or short language/region codes".into());
+ }
+ }
+ Ok(())
+}
+
+fn valid_domain(s: &str) -> bool {
+ !s.is_empty()
+ && s.len() <= 253
+ && s.split('.').all(|label| {
+ !label.is_empty()
+ && label.len() <= 63
+ && !label.starts_with('-')
+ && !label.ends_with('-')
+ && label
+ .bytes()
+ .all(|b| b.is_ascii_alphanumeric() || b == b'-')
+ })
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn invalid_web_configuration_is_rejected_before_reload() {
+ let mut cfg = Config::default();
+ assert!(validate(&cfg).is_ok());
+ cfg.web_tools.max_concurrent = 0;
+ assert!(validate(&cfg).is_err());
+ cfg.web_tools.max_concurrent = 4;
+ cfg.web_tools.allowed_domains = vec!["https://example.org/private".into()];
+ assert!(validate(&cfg).is_err());
+ cfg.web_tools.allowed_domains = vec!["docs.example.org".into()];
+ cfg.web_tools.proxy = "missing".into();
+ assert!(validate(&cfg).is_err());
+ }
+}
diff --git a/crates/tw-config/tests/manual/schema.rs b/crates/tw-config/tests/manual/schema.rs
index 52777c86..2769663f 100644
--- a/crates/tw-config/tests/manual/schema.rs
+++ b/crates/tw-config/tests/manual/schema.rs
@@ -230,6 +230,15 @@ pub fn sections() -> Vec {
"默认价目表是否定期刷新,以及自定义价目表。",
),
),
+ row(
+ "web_tools",
+ Kind::Obj("web_tools"),
+ Def::Section,
+ t(
+ "Gateway-owned search and public-page tools. Configured under Routing → Auxiliary in Lite.",
+ "网关执行的搜索与公开网页工具,在 Lite 的「路由 → 辅助请求」中配置。",
+ ),
+ ),
row(
"client_probes",
Kind::Obj("client_probes"),
@@ -1283,6 +1292,150 @@ pub fn sections() -> Vec {
),
],
},
+ Section {
+ path: "web_tools",
+ ty: checked!(tw_api::WebToolsConfig, "{}"),
+ rows: vec![
+ row(
+ "mode",
+ Kind::Enum(super::fields::),
+ Def::Is("upstream"),
+ t(
+ "upstream forwards hosted tools; core executes supported web operations.",
+ "upstream 沿用上游;core 执行已适配的联网工具。",
+ ),
+ ),
+ row(
+ "engines",
+ Kind::Strs,
+ Def::Is("[]"),
+ t(
+ "Empty selects available engines automatically. Otherwise list bing and/or duckduckgo in priority order.",
+ "空列表自动选择可用引擎;手动填写 bing、duckduckgo,按顺序尝试。",
+ ),
+ ),
+ row(
+ "proxy",
+ Kind::Str,
+ Def::Is("system"),
+ t(
+ "system, direct, or an existing proxy name.",
+ "system、direct 或已定义的代理名称。",
+ ),
+ ),
+ row(
+ "language",
+ Kind::Str,
+ Def::Is("''"),
+ t(
+ "Language code; empty uses the engine default.",
+ "语言代码,空值使用引擎默认值。",
+ ),
+ ),
+ row(
+ "region",
+ Kind::Str,
+ Def::Is("''"),
+ t(
+ "Region code; empty uses the engine default.",
+ "地区代码,空值使用引擎默认值。",
+ ),
+ ),
+ row(
+ "allowed_domains",
+ Kind::Strs,
+ Def::Is("[]"),
+ t(
+ "Only these domains and their subdomains; also applies to page reads. Empty allows all public domains.",
+ "仅允许这些域名及子域名,也约束网页读取;空列表允许所有公开域名。",
+ ),
+ ),
+ row(
+ "blocked_domains",
+ Kind::Strs,
+ Def::Is("[]"),
+ t(
+ "Excluded domains and their subdomains; exclusions take precedence.",
+ "排除的域名及子域名,排除优先。",
+ ),
+ ),
+ row(
+ "max_results",
+ Kind::Int,
+ Def::Is("8"),
+ t("Search results, 1–20.", "搜索结果数,1–20。"),
+ ),
+ row(
+ "read_pages",
+ Kind::Int,
+ Def::Is("2"),
+ t(
+ "Read up to 0–5 pages when search snippets are insufficient. Explicit page tools remain available at zero.",
+ "摘要不足时读取 0–5 个网页;设为 0 不影响模型明确调用网页读取。",
+ ),
+ ),
+ row(
+ "max_page_chars",
+ Kind::Int,
+ Def::Is("12000"),
+ t(
+ "Returned page characters, 1000–50000.",
+ "每页返回字符上限,1000–50000。",
+ ),
+ ),
+ row(
+ "timeout_secs",
+ Kind::Int,
+ Def::Is("30"),
+ t(
+ "Whole operation timeout including queueing, 1–120 seconds.",
+ "包括排队在内的单次操作超时,1–120 秒。",
+ ),
+ ),
+ row(
+ "max_concurrent",
+ Kind::Int,
+ Def::Is("4"),
+ t("Concurrent web operations, 1–16.", "联网操作并发数,1–16。"),
+ ),
+ row(
+ "interval_ms",
+ Kind::Int,
+ Def::Is("1000"),
+ t(
+ "Minimum interval per search engine, 250–60000 milliseconds.",
+ "同一搜索引擎的最小请求间隔,250–60000 毫秒。",
+ ),
+ ),
+ row(
+ "cache_secs",
+ Kind::Int,
+ Def::Is("600"),
+ t(
+ "Successful result lifetime, 0–86400 seconds. Zero disables caching.",
+ "成功结果缓存时间,0–86400 秒;0 关闭缓存。",
+ ),
+ ),
+ row(
+ "max_tool_calls",
+ Kind::Int,
+ Def::Is("8"),
+ t(
+ "Maximum web operations per model request, 1–32.",
+ "每个模型请求最多执行联网工具次数,1–32。",
+ ),
+ ),
+ row(
+ "fallback_to_upstream",
+ Kind::Bool,
+ Def::Is("false"),
+ t(
+ "Allow standalone searches to fall back to the upstream before returning results. This may incur upstream fees.",
+ "独立搜索返回结果之前失败时允许回退上游,可能产生上游费用。",
+ ),
+ ),
+ ],
+ },
// ── client_probes ─────────────────────────────────────
Section {
path: "client_probes",
diff --git a/crates/tw-control/src/lib.rs b/crates/tw-control/src/lib.rs
index d63d61eb..561121e2 100644
--- a/crates/tw-control/src/lib.rs
+++ b/crates/tw-control/src/lib.rs
@@ -45,6 +45,7 @@ pub mod routes;
pub mod security;
pub mod shutdown;
pub mod sync;
+pub mod web_tools;
pub mod zai;
pub use config::{ApplyError, ConfigManager, resolve_path, spawn_watcher};
pub use shutdown::Shutdown;
@@ -159,6 +160,7 @@ pub fn router(state: ControlState) -> Router {
.merge(pricing::router())
.merge(chatgpt::router())
.merge(zai::router())
+ .merge(web_tools::router())
.merge(sync::router())
.fallback(contract::no_such_endpoint)
.layer(axum::middleware::from_fn(contract::errors_are_messages))
diff --git a/crates/tw-control/src/web_tools.rs b/crates/tw-control/src/web_tools.rs
new file mode 100644
index 00000000..7ac5f3cc
--- /dev/null
+++ b/crates/tw-control/src/web_tools.rs
@@ -0,0 +1,52 @@
+use axum::{Json, Router, extract::State, http::StatusCode};
+use tw_api::ep;
+
+use crate::{ControlState, Fail, apply_fail, contract::RouterExt, fail};
+
+pub fn router() -> Router {
+ Router::new()
+ .at(ep::WebTools, view)
+ .at(ep::SaveWebTools, save)
+ .at(ep::TestWebTool, test)
+}
+
+async fn view(State(s): State) -> Json {
+ Json(tw_api::WebToolsView {
+ config: s.config().web_tools.clone(),
+ config_version: s.cfg.current().map(|c| c.version()).unwrap_or_default(),
+ })
+}
+
+async fn save(
+ State(s): State,
+ Json(req): Json,
+) -> Result, Fail> {
+ let value = serde_yaml_ng::to_value(&req.config).expect("web config serializes");
+ let version = s
+ .cfg
+ .transform(
+ Some(&req.base_version),
+ tw_config::history::Origin::Ui,
+ |text, _| {
+ Ok(tw_config::edit::set(
+ text,
+ &[tw_yaml::Step::key("web_tools")],
+ Some(&value),
+ )?)
+ },
+ )
+ .await
+ .map_err(apply_fail)?;
+ Ok(Json(tw_api::ConfigWritten { version }))
+}
+
+async fn test(
+ State(s): State,
+ Json(req): Json,
+) -> Result, Fail> {
+ let rt = s.gateway.runtime();
+ tw_gateway::web_tools::execute(&rt, "control-plane", &req)
+ .await
+ .map(Json)
+ .map_err(|m| fail(StatusCode::BAD_GATEWAY, m))
+}
diff --git a/crates/tw-control/tests/web_tools.rs b/crates/tw-control/tests/web_tools.rs
new file mode 100644
index 00000000..0b48b640
--- /dev/null
+++ b/crates/tw-control/tests/web_tools.rs
@@ -0,0 +1,105 @@
+use std::sync::Arc;
+
+use axum::body::Body;
+use axum::http::{Request, StatusCode};
+use serde_json::{Value, json};
+use tower::ServiceExt;
+use tw_control::{ConfigManager, ControlState};
+
+struct Bed {
+ dir: tempfile::TempDir,
+ app: axum::Router,
+}
+
+impl Bed {
+ fn file(&self) -> String {
+ std::fs::read_to_string(self.dir.path().join("config.yaml")).unwrap()
+ }
+}
+
+fn bed(yaml: &str) -> Bed {
+ let d = tempfile::tempdir().unwrap();
+ let p = d.path().join("config.yaml");
+ std::fs::write(&p, yaml).unwrap();
+ let cfg = tw_config::try_parse(yaml).unwrap();
+ let gw = tw_gateway::AppState::new(cfg).unwrap();
+ let bus = gw.bus.clone();
+ let state = ControlState {
+ shutdown: Default::default(),
+ remote: Default::default(),
+ handover: Default::default(),
+ cfg: Arc::new(ConfigManager::new(p, gw.clone(), bus)),
+ gateway: gw,
+ store: None,
+ started: std::time::Instant::now(),
+ price_updater: Default::default(),
+ chatgpt: Default::default(),
+ zai: Default::default(),
+ };
+ Bed {
+ app: tw_control::router(state),
+ dir: d,
+ }
+}
+
+async fn call(app: &axum::Router, method: &str, path: &str, body: Value) -> (StatusCode, Value) {
+ let r = app
+ .clone()
+ .oneshot(
+ Request::builder()
+ .method(method)
+ .uri(path)
+ .header("content-type", "application/json")
+ .body(Body::from(body.to_string()))
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ let st = r.status();
+ let b = axum::body::to_bytes(r.into_body(), 1 << 20).await.unwrap();
+ let text = String::from_utf8_lossy(&b).to_string();
+ let v = serde_json::from_str(&text).unwrap_or(Value::String(text));
+ (st, v)
+}
+
+#[tokio::test]
+async fn web_settings_persist_validate_versions_and_use_the_same_public_url_policy() {
+ let bed = bed(&format!(
+ "version: 1\nlisten:\n control:\n key: {}\nclients:\n - name: test\n key: tw-test\n",
+ "11".repeat(32)
+ ));
+ let (status, view) = call(&bed.app, "GET", "/web-tools", Value::Null).await;
+ assert_eq!(status, StatusCode::OK);
+ assert_eq!(view["config"]["mode"], "upstream");
+ let mut config = view["config"].clone();
+ config["mode"] = json!("core");
+ config["engines"] = json!(["bing"]);
+ let request = json!({"base_version":view["config_version"],"config":config});
+ let (status, saved) = call(&bed.app, "PUT", "/web-tools", request.clone()).await;
+ assert_eq!(status, StatusCode::OK, "{saved}");
+ assert_eq!(
+ tw_config::try_parse(&bed.file()).unwrap().web_tools.mode,
+ tw_api::WebToolsMode::Core
+ );
+ assert_eq!(
+ call(&bed.app, "PUT", "/web-tools", request).await.0,
+ StatusCode::CONFLICT
+ );
+ config["max_concurrent"] = json!(0);
+ let invalid = json!({"base_version":saved["version"],"config":config});
+ assert!(
+ !call(&bed.app, "PUT", "/web-tools", invalid)
+ .await
+ .0
+ .is_success()
+ );
+ let (status, error) = call(
+ &bed.app,
+ "POST",
+ "/web-tools/test",
+ json!({"action":"open_page","url":"http://127.0.0.1/"}),
+ )
+ .await;
+ assert_eq!(status, StatusCode::BAD_GATEWAY);
+ assert!(error.to_string().contains("non-public"), "{error}");
+}
diff --git a/crates/tw-gateway/Cargo.toml b/crates/tw-gateway/Cargo.toml
index 9683a125..0785b080 100644
--- a/crates/tw-gateway/Cargo.toml
+++ b/crates/tw-gateway/Cargo.toml
@@ -27,6 +27,8 @@ arc-swap = { workspace = true }
async-stream = { workspace = true }
axum = { workspace = true }
http = "1"
+hyper = { workspace = true }
+hyper-util = { workspace = true, features = ["client-proxy-system", "tokio"] }
tokio-tungstenite = { workspace = true }
bytes = { workspace = true }
chrono = { workspace = true }
@@ -57,6 +59,9 @@ zstd = { workspace = true }
# 压缩过的请求体和上游回答:gzip、deflate 用它解,zstd 用上面那个
flate2 = { workspace = true }
+scraper = "0.29"
+quick-xml = { workspace = true }
+
[dev-dependencies]
# 假的 Bedrock 要回 eventstream 的二进制帧
aws-smithy-eventstream = { workspace = true }
diff --git a/crates/tw-gateway/src/l1.rs b/crates/tw-gateway/src/l1.rs
index 07613cf6..accf88b6 100644
--- a/crates/tw-gateway/src/l1.rs
+++ b/crates/tw-gateway/src/l1.rs
@@ -176,7 +176,7 @@ pub fn proxy_target(cfg: &tw_config::Config, proxy: &str) -> (String, u16) {
.unwrap_or_else(|| ("api.anthropic.com".to_string(), 443))
}
-trait Io: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send {}
+pub(crate) trait Io: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send {}
impl Io for T {}
struct Timer {
@@ -578,7 +578,7 @@ fn split_hostport(s: &str) -> Result<(String, u16), Msg> {
))
}
-async fn socks5_connect(
+pub(crate) async fn socks5_connect(
io: &mut Box,
p: &ProxyHop,
host: &str,
@@ -719,7 +719,7 @@ fn socks5_reply(code: u8) -> Msg {
}
}
-async fn http_connect(
+pub(crate) async fn http_connect(
io: &mut Box,
p: &ProxyHop,
host: &str,
@@ -807,7 +807,10 @@ async fn tls_handshake(stream: Box, host: &str) -> Result<(), Msg> {
tls_handshake_boxed(stream, host).await.map(|_| ())
}
-async fn tls_handshake_boxed(stream: Box, host: &str) -> Result, Msg> {
+pub(crate) async fn tls_handshake_boxed(
+ stream: Box,
+ host: &str,
+) -> Result, Msg> {
let name = rustls::pki_types::ServerName::try_from(host.to_string()).map_err(
|_| msg!("l1.tls.bad_hostname", host = host => "{host} is not a valid TLS host name."),
)?;
diff --git a/crates/tw-gateway/src/lib.rs b/crates/tw-gateway/src/lib.rs
index a66c1853..7f93c3c9 100644
--- a/crates/tw-gateway/src/lib.rs
+++ b/crates/tw-gateway/src/lib.rs
@@ -151,3 +151,5 @@ pub fn probe_id() -> &'static str {
static ID: std::sync::OnceLock = std::sync::OnceLock::new();
ID.get_or_init(|| format!("{:032x}", rand::random::()))
}
+
+pub mod web_tools;
diff --git a/crates/tw-gateway/src/server.rs b/crates/tw-gateway/src/server.rs
index 117006fe..ca3a43f9 100644
--- a/crates/tw-gateway/src/server.rs
+++ b/crates/tw-gateway/src/server.rs
@@ -203,6 +203,7 @@ async fn passthrough(
// 响应在这里包一层
let seat = crate::content::Seat::default();
let req = pipeline::Inbound {
+ web_parent: false,
uri,
query,
headers,
diff --git a/crates/tw-gateway/src/server/pipeline.rs b/crates/tw-gateway/src/server/pipeline.rs
index dd8742e3..378302f8 100644
--- a/crates/tw-gateway/src/server/pipeline.rs
+++ b/crates/tw-gateway/src/server/pipeline.rs
@@ -28,10 +28,12 @@ mod idle;
mod opening;
mod plug;
mod relay;
+mod web;
pub(crate) use commit::KEEPALIVE;
pub(crate) use hop::stream_fault;
pub(crate) use relay::PING;
+pub(super) use web::serve_socket as web_socket;
/// 进管线时就定了的东西:身份识别之后,每一步都只读不改。
pub(super) struct Inbound {
@@ -55,6 +57,8 @@ pub(super) struct Inbound {
pub(super) traffic: std::sync::Arc,
/// 这个请求的报文记录(见 [`crate::content`]):开始时放进去,每一跳从这里拿
pub(super) seat: crate::content::Seat,
+ /// A local orchestration record; nested model rounds own the quota usage.
+ pub(super) web_parent: bool,
}
/// 发出开始事件之后,后面几步都要用的。
@@ -85,16 +89,77 @@ struct Started {
}
pub(super) async fn pipeline(
+ state: AppState,
+ rt: Arc,
+ mut req: Inbound,
+ live: crate::live::Pass,
+ ending: &mut Option,
+) -> Result {
+ let history = memchr::memmem::find(&req.body, b"srvtoolu_tw_").is_some()
+ || memchr::memmem::find(&req.body, b"ws_tw_").is_some();
+ if (rt.config.web_tools.mode == tw_api::WebToolsMode::Core || history)
+ && matches!(
+ req.dialect,
+ tw_dialect::ir::Dialect::Anthropic | tw_dialect::ir::Dialect::Responses
+ )
+ && let Ok(mut body) = serde_json::from_slice::(&req.body)
+ {
+ let plan = if rt.config.web_tools.mode == tw_api::WebToolsMode::Core {
+ crate::web_tools::protocol::recognize(req.dialect, &body)
+ } else {
+ Ok(None)
+ };
+ // Restore history even after tools were removed or local execution was
+ // disabled. Expired excerpts degrade to the client's public source metadata.
+ let restored = history
+ && rt
+ .web_tools
+ .history
+ .restore(
+ &req.client_name,
+ req.dialect,
+ &mut body,
+ matches!(plan, Ok(Some(_))),
+ )
+ .map_err(|m| GatewayError::new(crate::error::Source::Request, m))?;
+ if restored {
+ req.body = Bytes::from(body.to_string());
+ }
+ match plan {
+ Ok(Some(plan)) => {
+ let (reading, _) = read(&req, String::new(), Some(&body), (state.local_time)());
+ let early = commit::early(&state, &req, &reading);
+ let dialect = req.dialect;
+ let abort = req.abort.clone();
+ let run = web::run(state, rt, req, live, plan, body);
+ return match early {
+ Some(early) => commit::hold(run, dialect, early, abort).await,
+ None => run.await,
+ };
+ }
+ Err(message) => {
+ return Err(GatewayError::new(crate::error::Source::Request, message));
+ }
+ _ => {}
+ }
+ return forward_pipeline(state, rt, req, live, ending, Some(body)).await;
+ }
+ forward_pipeline(state, rt, req, live, ending, None).await
+}
+
+async fn forward_pipeline(
state: AppState,
rt: Arc,
req: Inbound,
live: crate::live::Pass,
ending: &mut Option,
+ prepared: Option,
) -> Result {
// 请求体**只解析一次**:本地应答的判定、路由事实、对话的指纹、内容过滤都用这一份(见
// `read`)。解不开是 None —— 照样往下走,同格式直通照样发
let (parsed, probed) = heavy(&req.body, || {
- let parsed = serde_json::from_slice::(&req.body).ok();
+ let parsed =
+ prepared.or_else(|| serde_json::from_slice::(&req.body).ok());
let probed = probe(&state, &rt, &req, parsed.as_ref());
(parsed, probed)
})
diff --git a/crates/tw-gateway/src/server/pipeline/admission.rs b/crates/tw-gateway/src/server/pipeline/admission.rs
index 33541dff..2ccd0d9a 100644
--- a/crates/tw-gateway/src/server/pipeline/admission.rs
+++ b/crates/tw-gateway/src/server/pipeline/admission.rs
@@ -51,7 +51,7 @@ pub(super) async fn admit(
let key = rt.config.clients.iter().find(|c| c.name == req.client_name);
let max_concurrent = key.and_then(|c| c.max_concurrent);
let limits: &[tw_config::KeyLimit] = match key {
- Some(c) if !crate::key_limits::uncounted(req.uri.path()) => &c.limits,
+ Some(c) if !req.web_parent && !crate::key_limits::uncounted(req.uri.path()) => &c.limits,
_ => &[],
};
if let Err(r) = state.key_limits.calendar(&req.client_name, limits) {
diff --git a/crates/tw-gateway/src/server/pipeline/web.rs b/crates/tw-gateway/src/server/pipeline/web.rs
new file mode 100644
index 00000000..748dc126
--- /dev/null
+++ b/crates/tw-gateway/src/server/pipeline/web.rs
@@ -0,0 +1,426 @@
+//! Hosted web operations keep the normal model pipeline for every model round.
+//! The outer operation records zero model usage; each actual model round is metered
+//! by its own normal request record, so tool loops do not hide or double bill tokens.
+
+use std::sync::Arc;
+
+mod socket;
+pub(in crate::server) use socket::serve_socket;
+
+use axum::response::{IntoResponse, Response};
+use bytes::Bytes;
+use serde_json::{Value, json};
+use tw_dialect::ir::Dialect;
+
+use super::{Chosen, Inbound};
+use crate::web_tools::{
+ self,
+ protocol::{self, Call, Plan},
+};
+use crate::{
+ GatewayError,
+ state::{AppState, Runtime},
+};
+
+fn failed(message: tw_types::Msg) -> GatewayError {
+ GatewayError::new(crate::error::Source::Request, message)
+}
+
+pub(super) async fn run(
+ state: AppState,
+ rt: Arc,
+ mut req: Inbound,
+ live: crate::live::Pass,
+ plan: Plan,
+ original: Value,
+) -> Result {
+ let mut ending = None;
+ let mut trace = Vec::new();
+ let result = async {
+ let (mut reading, fp) = super::read(
+ &req,
+ "web_search".into(),
+ Some(&original),
+ (state.local_time)(),
+ );
+ req.web_parent = plan.standalone(&original).is_none();
+ // Local execution may work without any upstream. Explicit routing denials
+ // still apply, as do key limits, screening, redaction and cancellation.
+ if let Ok(tw_engine::Outcome::Deny { reason, .. }) = rt.engine.route(&reading.facts) {
+ return Err(GatewayError::denied(web_tools::error(reason)));
+ }
+ // The local parent has no billable model input.
+ reading.facts.input_tokens = 0;
+ let choice = crate::server::Choice {
+ route: rt.engine.route_of(&req.client_name).into(),
+ ..Default::default()
+ };
+ let decision = tw_engine::Decision {
+ candidates: Vec::new(),
+ matched_rule: String::new(),
+ via_group: None,
+ set: Default::default(),
+ rewritten_by: Vec::new(),
+ pinned: Vec::new(),
+ };
+ let chosen = Chosen {
+ parsed: Some(original.clone()),
+ reading,
+ fp,
+ conv: None,
+ choice,
+ decision,
+ };
+ let tail = super::enter(&state, &rt, &mut req, chosen, live, &mut ending).await?;
+ let id = tail.started.id;
+ ending
+ .as_mut()
+ .expect("entered request")
+ .add_usage(&tw_dialect::ir::Usage::default());
+ state.bus.emit(crate::server::routed_nowhere(
+ id,
+ tail.started.choice.clone(),
+ ));
+ let screened: Value = serde_json::from_slice(&req.body)
+ .map_err(|_| failed(web_tools::error("invalid screened request")))?;
+ let stream = original["stream"] == true;
+ let standalone = plan.standalone(&screened);
+ let body = if let Some(request) = standalone {
+ let result = tokio::select! {
+ _ = req.abort.wait() => return Err(GatewayError::aborted()),
+ result = web_tools::execute_scoped(&rt, &req.client_name, &request, &plan.allowed, &plan.blocked) => result,
+ };
+ match result {
+ Ok(result) => {
+ trace.push(trace_of(&request, &result));
+ let call = Call {
+ native_id: uuid::Uuid::new_v4().simple().to_string(),
+ id: uuid::Uuid::new_v4().simple().to_string(),
+ request,
+ };
+ rt.web_tools
+ .history
+ .remember(&req.client_name, req.dialect, &call, &result);
+ let mut answer = protocol::standalone_response(
+ original["model"].as_str().unwrap_or(""), &call, &result,
+ );
+ if original["safeguards"].as_array().is_some_and(|items| items.iter().any(|v| v["type"] == "dangerous_tool_use")) {
+ // There are no client-executed tool_use blocks to classify.
+ answer["safeguard_results"] = json!([{"type":"dangerous_tool_use","status":{"type":"available","tool_uses":{}}}]);
+ }
+ answer
+ }
+ Err(_) if rt.config.web_tools.fallback_to_upstream => {
+ drop(tail.passes.1);
+ let mut response = forward_original(&state, &rt, &req, &original).await?;
+ response.headers_mut().insert(
+ "x-thinkwatch-web-tools",
+ axum::http::HeaderValue::from_static("upstream"),
+ );
+ response.headers_mut().insert(
+ "x-thinkwatch-web-fallback",
+ axum::http::HeaderValue::from_static("1"),
+ );
+ return Ok(response);
+ }
+ Err(e) => return Err(failed(e)),
+ }
+ } else {
+ // Model rounds acquire the same key's concurrency slot. Release the
+ // outer slot first, including when the key only allows one request.
+ drop(tail.passes.1);
+ match model_loop(&state, &rt, &req, &plan, screened, &mut trace).await? {
+ ModelResult::Answer(body) => body,
+ ModelResult::Failed(response) => return Ok(response),
+ }
+ };
+ let body = if let Some(filter) = crate::guard::AnswerFilter::of(&rt) {
+ let (changed, hits) = tw_guard::content::answer::whole(
+ &filter.answer,
+ filter.acts,
+ req.dialect,
+ body.to_string().as_bytes(),
+ );
+ crate::guard::report_answer(&state.bus, id, "", hits,
+ &super::redaction(&rt, tail.started.ledger.clone()));
+ match changed {
+ Some(bytes) => serde_json::from_slice(&bytes)
+ .map_err(|_| failed(web_tools::error("invalid filtered web response")))?,
+ None => body,
+ }
+ } else {
+ body
+ };
+ let bytes = if stream {
+ web_tools::wire::stream(req.dialect, &body)
+ } else {
+ body.to_string().into_bytes()
+ };
+ let end = ending.as_mut().expect("entered request");
+ end.count(bytes.len());
+ end.responded(200);
+ state.bus.emit(tw_api::Event::RequestHeaders {
+ id,
+ status: 200,
+ ttfb_ms: req.started.elapsed().as_millis() as u64,
+ });
+ let mut response = (
+ [
+ (
+ "content-type",
+ if stream {
+ "text/event-stream"
+ } else {
+ "application/json"
+ },
+ ),
+ ("x-thinkwatch-web-tools", "core"),
+ ],
+ bytes,
+ )
+ .into_response();
+ if let Ok(value) =
+ axum::http::HeaderValue::from_str(&serde_json::to_string(&trace).unwrap_or_default())
+ {
+ response
+ .headers_mut()
+ .insert("x-thinkwatch-web-trace", value);
+ }
+ Ok(response)
+ }
+ .await;
+ if let Some(end) = ending.take() {
+ match &result {
+ Ok(response) => end.finished(response.status().as_u16()),
+ Err(e) => end.failed(e.source.into(), e.detail.clone()),
+ }
+ }
+ result
+}
+
+fn round_request(req: &Inbound, body: &Value) -> Inbound {
+ Inbound {
+ web_parent: false,
+ uri: req.uri.clone(),
+ query: req.query.clone(),
+ headers: req.headers.clone(),
+ body: Bytes::from(body.to_string()),
+ version: req.version,
+ client_name: req.client_name.clone(),
+ api: req.api,
+ dialect: req.dialect,
+ started: std::time::Instant::now(),
+ from: req.from.clone(),
+ abort: req.abort.clone(),
+ traffic: crate::traffic::Traffic::new(),
+ seat: crate::content::Seat::default(),
+ }
+}
+
+async fn forward_original(
+ state: &AppState,
+ rt: &Arc,
+ req: &Inbound,
+ body: &Value,
+) -> Result {
+ let mut ending = None;
+ let result = super::forward_pipeline(
+ state.clone(),
+ rt.clone(),
+ round_request(req, body),
+ state.live.enter(),
+ &mut ending,
+ Some(body.clone()),
+ )
+ .await;
+ if let Some(end) = ending {
+ match &result {
+ Ok(response) => end.finished(response.status().as_u16()),
+ Err(e) => end.failed(e.source.into(), e.detail.clone()),
+ }
+ }
+ result
+}
+
+enum ModelResult {
+ Answer(Value),
+ Failed(Response),
+}
+
+async fn model_loop(
+ state: &AppState,
+ rt: &Arc,
+ req: &Inbound,
+ plan: &Plan,
+ mut body: Value,
+ trace: &mut Vec,
+) -> Result {
+ plan.rewrite(&mut body).map_err(failed)?;
+ let max = plan
+ .max_uses
+ .unwrap_or(rt.config.web_tools.max_tool_calls)
+ .min(rt.config.web_tools.max_tool_calls) as usize;
+ let mut used = 0;
+ let mut accumulated = Vec::new();
+ let mut usage = tw_dialect::ir::Usage::default();
+ let mut sources = Vec::new();
+ loop {
+ let response = forward_original(state, rt, req, &body).await?;
+ if !response.status().is_success() {
+ // Preserve native status, error details and retry headers.
+ return Ok(ModelResult::Failed(response));
+ }
+ let bytes = axum::body::to_bytes(response.into_body(), 16 * 1024 * 1024)
+ .await
+ .map_err(|_| {
+ failed(web_tools::error(
+ "model response was interrupted or exceeded 16 MiB",
+ ))
+ })?;
+ let mut answer: Value = serde_json::from_slice(&bytes).map_err(|_| {
+ failed(web_tools::error(
+ "model did not return a complete JSON response",
+ ))
+ })?;
+ let next_usage = if req.dialect == Dialect::Anthropic {
+ tw_dialect::anthropic::response::usage(&answer["usage"])
+ } else {
+ tw_dialect::responses::response::usage(&answer["usage"])
+ };
+ usage.input = usage.input.saturating_add(next_usage.input);
+ usage.output = usage.output.saturating_add(next_usage.output);
+ usage.cache_read = usage.cache_read.saturating_add(next_usage.cache_read);
+ usage.cache_write = usage.cache_write.saturating_add(next_usage.cache_write);
+ usage.reasoning = usage.reasoning.saturating_add(next_usage.reasoning);
+ usage.cache_1h |= next_usage.cache_1h;
+ let calls = plan.calls(&answer).map_err(failed)?;
+ let key = if req.dialect == Dialect::Anthropic {
+ "content"
+ } else {
+ "output"
+ };
+ let items = answer[key]
+ .as_array()
+ .cloned()
+ .ok_or_else(|| failed(web_tools::error("model response has no content")))?;
+ let external = items.iter().any(|item| {
+ matches!(
+ item["type"].as_str(),
+ Some("tool_use" | "function_call" | "custom_tool_call")
+ ) && ![protocol::SEARCH, protocol::OPEN, protocol::FIND]
+ .contains(&item["name"].as_str().unwrap_or(""))
+ });
+ if used + calls.len() > max {
+ return Err(failed(web_tools::error(
+ "model exceeded the configured web-tool call limit",
+ )));
+ }
+ used += calls.len();
+ let mut results = Vec::new();
+ for call in calls {
+ let result = tokio::select! {
+ _ = req.abort.wait() => return Err(GatewayError::aborted()),
+ r = web_tools::execute_scoped(rt, &req.client_name, &call.request, &plan.allowed, &plan.blocked) => r,
+ }
+ .map_err(failed)?;
+ trace.push(trace_of(&call.request, &result));
+ sources.extend(result.sources.clone());
+ rt.web_tools
+ .history
+ .remember(&req.client_name, req.dialect, &call, &result);
+ results.push((call, result));
+ }
+ // Keep client-executed calls after completed hosted exchanges, so their
+ // results on the next turn immediately follow the matching assistant turn.
+ let (client_calls, items): (Vec<_>, Vec<_>) = items.into_iter().partition(|item| {
+ matches!(
+ item["type"].as_str(),
+ Some("tool_use" | "function_call" | "custom_tool_call")
+ ) && ![protocol::SEARCH, protocol::OPEN, protocol::FIND]
+ .contains(&item["name"].as_str().unwrap_or(""))
+ });
+ for item in items {
+ let id = if req.dialect == Dialect::Anthropic {
+ item["id"].as_str()
+ } else {
+ item["call_id"].as_str()
+ };
+ if let Some((call, result)) = results.iter().find(|(c, _)| Some(c.id.as_str()) == id) {
+ accumulated.extend(protocol::native_blocks(req.dialect, call, result));
+ } else {
+ accumulated.push(item);
+ }
+ }
+ if results.is_empty() || external {
+ accumulated.extend(client_calls);
+ answer[key] = json!(accumulated);
+ answer["usage"] = if req.dialect == Dialect::Anthropic {
+ tw_dialect::anthropic::response::usage_json(&usage)
+ } else {
+ tw_dialect::responses::response::usage_json(&usage)
+ };
+ if req.dialect == Dialect::Responses {
+ citations(&mut answer, &sources);
+ }
+ return Ok(ModelResult::Answer(answer));
+ }
+ plan.append(&mut body, &answer, &results).map_err(failed)?;
+ if body.to_string().len() > 16 * 1024 * 1024 {
+ return Err(failed(web_tools::error(
+ "web-tool conversation exceeded 16 MiB",
+ )));
+ }
+ }
+}
+
+fn citations(answer: &mut Value, sources: &[tw_api::WebSource]) {
+ for item in answer["output"].as_array_mut().into_iter().flatten() {
+ for part in item["content"].as_array_mut().into_iter().flatten() {
+ if part["type"] != "output_text" {
+ continue;
+ }
+ let text = part["text"].as_str().unwrap_or("").to_string();
+ let mut annotations = part["annotations"].as_array().cloned().unwrap_or_default();
+ for source in sources {
+ if let Some(start) = text.find(&source.url) {
+ let start = text[..start].chars().count();
+ let end = start + source.url.chars().count();
+ if !annotations.iter().any(|a| a["url"] == source.url) {
+ annotations.push(json!({"type":"url_citation","url":source.url,"title":source.title,"start_index":start,"end_index":end}));
+ }
+ }
+ }
+ part["annotations"] = json!(annotations);
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests;
+
+fn trace_of(
+ request: &tw_api::WebToolRequest,
+ result: &tw_api::WebToolResult,
+) -> tw_api::WebToolTrace {
+ let mut engines = Vec::new();
+ for source in &result.sources {
+ if let Some(engine) = source.engine
+ && !engines.contains(&engine)
+ {
+ engines.push(engine);
+ }
+ }
+ tw_api::WebToolTrace {
+ action: match request {
+ tw_api::WebToolRequest::Search { .. } => "search",
+ tw_api::WebToolRequest::OpenPage { .. } => "open_page",
+ tw_api::WebToolRequest::FindInPage { .. } => "find_in_page",
+ }
+ .into(),
+ engines,
+ cached: result.cached,
+ elapsed_ms: result.elapsed_ms,
+ sources: result.sources.len() as u32,
+ warnings: result.warnings.len() as u32,
+ }
+}
diff --git a/crates/tw-gateway/src/server/pipeline/web/socket.rs b/crates/tw-gateway/src/server/pipeline/web/socket.rs
new file mode 100644
index 00000000..03447ec7
--- /dev/null
+++ b/crates/tw-gateway/src/server/pipeline/web/socket.rs
@@ -0,0 +1,264 @@
+//! Responses WebSocket requests use the same protected web-tool pipeline as HTTP.
+//! Only this connection's last response can supply implicit conversation history.
+
+use axum::extract::ws::{Message, WebSocket};
+use axum::response::IntoResponse;
+use futures::{SinkExt, StreamExt};
+
+use super::*;
+
+type Writer = futures::stream::SplitSink;
+
+#[allow(clippy::too_many_arguments)]
+pub(in crate::server) async fn serve_socket(
+ socket: WebSocket,
+ state: AppState,
+ rt: Arc,
+ client_name: String,
+ uri: axum::http::Uri,
+ query: Option,
+ headers: axum::http::HeaderMap,
+ from: crate::server::Sender,
+ _live: crate::live::Pass,
+) {
+ let (mut writer, mut reader) = socket.split();
+ let mut previous: Option<(String, Value, Vec)> = None;
+ while let Some(Ok(message)) = reader.next().await {
+ let text = match message {
+ Message::Text(text) => text,
+ Message::Ping(data) => {
+ if writer.send(Message::Pong(data)).await.is_err() {
+ return;
+ }
+ continue;
+ }
+ Message::Close(_) => return,
+ _ => continue,
+ };
+ let mut body: Value = match serde_json::from_str(&text) {
+ Ok(body) => body,
+ Err(_) => {
+ send_error(
+ &mut writer,
+ "invalid_request_error",
+ "Expected a JSON response.create message",
+ )
+ .await;
+ continue;
+ }
+ };
+ if body["type"] != "response.create" {
+ send_error(
+ &mut writer,
+ "invalid_request_error",
+ "Only response.create is accepted while no response is running",
+ )
+ .await;
+ continue;
+ }
+ body.as_object_mut()
+ .expect("response.create object")
+ .remove("type");
+ body["stream"] = json!(true);
+ if let Some((id, request, output)) = &previous
+ && body["previous_response_id"] == *id
+ {
+ let mut input = input_items(&request["input"]);
+ input.extend(output.clone());
+ input.extend(input_items(&body["input"]));
+ body["input"] = json!(input);
+ body.as_object_mut()
+ .expect("object")
+ .remove("previous_response_id");
+ }
+ let bytes = body.to_string();
+ if bytes.len() > 16 * 1024 * 1024 {
+ send_error(
+ &mut writer,
+ "invalid_request_error",
+ "Conversation exceeds 16 MiB; start a new conversation",
+ )
+ .await;
+ continue;
+ }
+ let abort = crate::abort::Switch::default();
+ let seat = crate::content::Seat::default();
+ let mut request_headers = headers.clone();
+ for header in [
+ "connection",
+ "upgrade",
+ "sec-websocket-key",
+ "sec-websocket-version",
+ "sec-websocket-protocol",
+ "sec-websocket-extensions",
+ ] {
+ request_headers.remove(header);
+ }
+ let req = Inbound {
+ web_parent: false,
+ uri: uri.clone(),
+ query: query.clone(),
+ headers: request_headers,
+ body: Bytes::from(bytes),
+ version: axum::http::Version::HTTP_11,
+ client_name: client_name.clone(),
+ api: Some(crate::client_api::ClientApi::OpenaiResponses),
+ dialect: Dialect::Responses,
+ started: std::time::Instant::now(),
+ from: from.clone(),
+ abort: abort.clone(),
+ traffic: crate::traffic::Traffic::new(),
+ seat: seat.clone(),
+ };
+ let mut ending = None;
+ let response = {
+ let run = super::super::pipeline(
+ state.clone(),
+ rt.clone(),
+ req,
+ state.live.enter(),
+ &mut ending,
+ );
+ tokio::pin!(run);
+ loop {
+ tokio::select! {
+ _ = abort.wait() => break Err(GatewayError::aborted()),
+ result = &mut run => break result,
+ incoming = reader.next() => {
+ if !during(incoming, &mut writer, &abort).await { return; }
+ }
+ }
+ }
+ };
+ if let Some(end) = ending.take() {
+ match &response {
+ Ok(r) => end.finished(r.status().as_u16()),
+ Err(e) => end.failed(e.source.into(), e.detail.clone()),
+ }
+ }
+ if abort.thrown() {
+ send_error(&mut writer, "cancelled", "Response was cancelled").await;
+ continue;
+ }
+ let response =
+ response.unwrap_or_else(|e| e.in_dialect(Dialect::Responses).into_response());
+ let response = match seat.get() {
+ Some(capture) => capture.client_answer(axum::http::Version::HTTP_11, response),
+ None => response,
+ };
+ let success = response.status().is_success();
+ let mut data = response.into_body().into_data_stream();
+ let mut frames = tw_dialect::frame::Decoder::default();
+ let mut error_body = Vec::new();
+ loop {
+ tokio::select! {
+ _ = abort.wait() => { send_error(&mut writer, "cancelled", "Response was cancelled").await; break; }
+ incoming = reader.next() => {
+ if !during(incoming, &mut writer, &abort).await { return; }
+ }
+ next = data.next() => {
+ let Some(chunk) = next else { break; };
+ let Ok(chunk) = chunk else { send_error(&mut writer, "server_error", "Response stream was interrupted").await; break; };
+ if !success {
+ if error_body.len() + chunk.len() <= 1024 * 1024 { error_body.extend_from_slice(&chunk); }
+ continue;
+ }
+ for frame in frames.feed(&chunk) {
+ if !send_frame(&mut writer, &mut previous, &body, &frame.data).await { return; }
+ }
+ }
+ }
+ }
+ if abort.thrown() {
+ continue;
+ }
+ if success {
+ for frame in frames.flush() {
+ if !send_frame(&mut writer, &mut previous, &body, &frame.data).await {
+ return;
+ }
+ }
+ } else {
+ let error: Value = serde_json::from_slice(&error_body).unwrap_or_default();
+ let message = error["error"]["message"]
+ .as_str()
+ .unwrap_or("The request failed");
+ send_error(&mut writer, "server_error", message).await;
+ }
+ }
+}
+
+fn input_items(value: &Value) -> Vec {
+ match value {
+ Value::Array(items) => items.clone(),
+ Value::String(text) => vec![json!({"role":"user","content":text})],
+ _ => Vec::new(),
+ }
+}
+
+async fn send_frame(
+ writer: &mut Writer,
+ previous: &mut Option<(String, Value, Vec)>,
+ request: &Value,
+ data: &str,
+) -> bool {
+ let Ok(value) = serde_json::from_str::(data) else {
+ return true;
+ };
+ if matches!(
+ value["type"].as_str(),
+ Some("response.completed" | "response.incomplete")
+ ) && let Some(id) = value["response"]["id"].as_str()
+ {
+ *previous = Some((
+ id.into(),
+ request.clone(),
+ value["response"]["output"]
+ .as_array()
+ .cloned()
+ .unwrap_or_default(),
+ ));
+ }
+ writer
+ .send(Message::Text(data.to_string().into()))
+ .await
+ .is_ok()
+}
+
+async fn during(
+ incoming: Option>,
+ writer: &mut Writer,
+ abort: &crate::abort::Switch,
+) -> bool {
+ match incoming {
+ None | Some(Err(_)) | Some(Ok(Message::Close(_))) => false,
+ Some(Ok(Message::Ping(data))) => writer.send(Message::Pong(data)).await.is_ok(),
+ Some(Ok(Message::Text(text))) => {
+ if serde_json::from_str::(&text)
+ .ok()
+ .is_some_and(|v| v["type"] == "response.cancel")
+ {
+ abort.throw();
+ } else {
+ send_error(
+ writer,
+ "response_in_progress",
+ "Wait for the current response or cancel it before creating another response",
+ )
+ .await;
+ }
+ true
+ }
+ _ => true,
+ }
+}
+
+async fn send_error(writer: &mut Writer, kind: &str, message: &str) {
+ let _ = writer
+ .send(Message::Text(
+ json!({"type":"error","error":{"type":kind,"message":message}})
+ .to_string()
+ .into(),
+ ))
+ .await;
+}
diff --git a/crates/tw-gateway/src/server/pipeline/web/tests.rs b/crates/tw-gateway/src/server/pipeline/web/tests.rs
new file mode 100644
index 00000000..62b2c1fa
--- /dev/null
+++ b/crates/tw-gateway/src/server/pipeline/web/tests.rs
@@ -0,0 +1,390 @@
+use super::*;
+use axum::{Json, Router, routing::post};
+use std::sync::atomic::{AtomicUsize, Ordering};
+use tower::ServiceExt;
+
+fn config() -> tw_config::Config {
+ let mut config = tw_config::Config::default();
+ config.clients.push(tw_config::Client {
+ name: "test".into(),
+ key: "test-key".into(),
+ max_concurrent: Some(1),
+ ..Default::default()
+ });
+ config.web_tools.mode = tw_api::WebToolsMode::Core;
+ config.web_tools.proxy = tw_config::DIRECT.into();
+ config
+}
+
+fn sample() -> tw_api::WebToolResult {
+ tw_api::WebToolResult {
+ sources: vec![tw_api::WebSource {
+ url: "https://sqlite.org/changes.html".into(),
+ title: "SQLite changes".into(),
+ text: "Release 3.50".into(),
+ engine: Some(tw_api::SearchEngine::Bing),
+ published_at: None,
+ }],
+ cached: false,
+ elapsed_ms: 1,
+ warnings: Vec::new(),
+ truncated: false,
+ }
+}
+
+async fn send(state: AppState, path: &str, body: Value, key: &str) -> Response {
+ let request = axum::http::Request::builder()
+ .method("POST")
+ .uri(path)
+ .header("content-type", "application/json")
+ .header("x-api-key", key)
+ .extension(axum::extract::ConnectInfo(
+ "127.0.0.1:12345".parse::().unwrap(),
+ ))
+ .body(axum::body::Body::from(body.to_string()))
+ .unwrap();
+ tokio::time::timeout(
+ std::time::Duration::from_secs(5),
+ crate::router(state).oneshot(request),
+ )
+ .await
+ .unwrap()
+ .unwrap()
+}
+
+async fn seed(state: &AppState) {
+ state
+ .runtime()
+ .web_tools
+ .seed(
+ "test",
+ &tw_api::WebToolRequest::Search {
+ query: "SQLite".into(),
+ allowed_domains: Vec::new(),
+ blocked_domains: Vec::new(),
+ },
+ sample(),
+ )
+ .await;
+}
+
+#[tokio::test]
+async fn standalone_search_needs_no_upstream_and_preserves_auth_and_native_sse() {
+ let state = AppState::new(config()).unwrap();
+ seed(&state).await;
+ for stream in [false, true] {
+ let body = json!({"model":"any-model","stream":stream,"max_tokens":1024,"messages":[{"role":"user","content":"Perform a web search for the query: SQLite"}],"tools":[{"type":"web_search_20250305","name":"web_search"}],"tool_choice":{"type":"tool","name":"web_search"}});
+ assert_eq!(
+ send(state.clone(), "/v1/messages", body.clone(), "wrong")
+ .await
+ .status(),
+ 401
+ );
+ let response = send(state.clone(), "/v1/messages", body, "test-key").await;
+ assert_eq!(response.status(), 200);
+ let bytes = axum::body::to_bytes(response.into_body(), 100000)
+ .await
+ .unwrap();
+ let text = String::from_utf8(bytes.to_vec()).unwrap();
+ assert!(text.contains("web_search_tool_result"));
+ assert!(text.contains("https://sqlite.org/changes.html"));
+ if stream {
+ assert!(text.contains("event: message_stop"));
+ } else {
+ assert_eq!(
+ serde_json::from_str::(&text).unwrap()["usage"]["output_tokens"],
+ 0
+ );
+ }
+ }
+}
+
+#[tokio::test]
+async fn responses_loop_searches_then_resumes_model_without_deadlocking_a_one_slot_key() {
+ let calls = Arc::new(AtomicUsize::new(0));
+ let n = calls.clone();
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let addr = listener.local_addr().unwrap();
+ let server = tokio::spawn(async move {
+ axum::serve(listener, Router::new().route("/v1/responses", post(move |Json(body): Json| {
+ let n = n.clone(); async move {
+ let i = n.fetch_add(1, Ordering::SeqCst);
+ assert!(!body["tools"].as_array().unwrap().iter().any(|t| t["type"] == "web_search"));
+ if i == 0 {
+ Json(json!({"id":"resp_1","object":"response","status":"completed","model":"model","output":[{"type":"function_call","id":"fc_1","call_id":"call_1","name":protocol::SEARCH,"arguments":"{\"query\":\"SQLite\"}"}],"usage":{"input_tokens":10,"output_tokens":3}}))
+ } else {
+ assert!(body["input"].as_array().unwrap().iter().any(|v| v["type"] == "function_call_output"));
+ Json(json!({"id":"resp_2","object":"response","status":"completed","model":"model","output":[{"type":"message","id":"msg_2","role":"assistant","status":"completed","content":[{"type":"output_text","text":"See https://sqlite.org/changes.html","annotations":[]}]}],"usage":{"input_tokens":20,"output_tokens":6}}))
+ }
+ }
+ }))).await.unwrap();
+ });
+ let mut config = config();
+ config.providers.push(tw_config::Provider {
+ name: "fake".into(),
+ base_url: format!("http://{addr}"),
+ protocol: Some(tw_config::Protocol::OpenaiResponses),
+ proxy: tw_config::DIRECT.into(),
+ ..Default::default()
+ });
+ let state = AppState::new(config).unwrap();
+ seed(&state).await;
+ let body = json!({"model":"model","input":"Find SQLite","tools":[{"type":"web_search"}],"stream":false});
+ let response = send(state.clone(), "/v1/responses", body, "test-key").await;
+ let status = response.status();
+ let bytes = axum::body::to_bytes(response.into_body(), 100000)
+ .await
+ .unwrap();
+ let reply: Value = serde_json::from_slice(&bytes).unwrap();
+ assert_eq!(status, 200, "{reply}");
+ assert_eq!(calls.load(Ordering::SeqCst), 2);
+ assert_eq!(reply["output"][0]["type"], "web_search_call");
+ assert_eq!(reply["usage"]["input_tokens"], 30);
+ assert_eq!(
+ reply["output"][1]["content"][0]["annotations"][0]["url"],
+ "https://sqlite.org/changes.html"
+ );
+ let mut next = json!({"input":reply["output"]});
+ state
+ .runtime()
+ .web_tools
+ .history
+ .restore("test", Dialect::Responses, &mut next, true)
+ .unwrap();
+ assert_eq!(next["input"][0]["type"], "function_call");
+ assert_eq!(next["input"][1]["type"], "function_call_output");
+ server.abort();
+}
+
+async fn mock_upstream(
+ config: &mut tw_config::Config,
+ api: tw_config::Protocol,
+ path: &'static str,
+ handler: impl Fn(Value, usize) -> Response + Send + Sync + 'static,
+) -> (tokio::task::JoinHandle<()>, Arc) {
+ let calls = Arc::new(AtomicUsize::new(0));
+ let count = calls.clone();
+ let handler = Arc::new(handler);
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let addr = listener.local_addr().unwrap();
+ config.providers.push(tw_config::Provider {
+ name: "mock".into(),
+ base_url: format!("http://{addr}"),
+ protocol: Some(api),
+ proxy: tw_config::DIRECT.into(),
+ ..Default::default()
+ });
+ let server = tokio::spawn(async move {
+ axum::serve(
+ listener,
+ Router::new().route(
+ path,
+ post(move |Json(body): Json| {
+ let handler = handler.clone();
+ let count = count.clone();
+ async move { handler(body, count.fetch_add(1, Ordering::SeqCst)) }
+ }),
+ ),
+ )
+ .await
+ .unwrap();
+ });
+ (server, calls)
+}
+
+async fn json_body(response: Response) -> Value {
+ let bytes = axum::body::to_bytes(response.into_body(), 1024 * 1024)
+ .await
+ .unwrap();
+ serde_json::from_slice(&bytes).unwrap()
+}
+
+#[tokio::test]
+async fn anthropic_mixed_client_tool_history_and_cached_usage_survive_the_next_turn() {
+ let mut cfg = config();
+ let (server, calls) = mock_upstream(&mut cfg, tw_config::Protocol::Anthropic, "/v1/messages", |body, i| {
+ if i == 0 {
+ Json(json!({"id":"msg_1","type":"message","role":"assistant","model":"model","content":[
+ {"type":"tool_use","id":"client_1","name":"Read","input":{"path":"README.md"}},
+ {"type":"tool_use","id":"search_1","name":protocol::SEARCH,"input":{"query":"SQLite"}}
+ ],"stop_reason":"tool_use","usage":{"input_tokens":10,"output_tokens":5,"cache_read_input_tokens":30}})).into_response()
+ } else {
+ let messages = body["messages"].as_array().unwrap();
+ assert_eq!(messages[1]["content"][0]["name"], protocol::SEARCH);
+ assert_eq!(messages[2]["content"][0]["tool_use_id"], "search_1");
+ assert_eq!(messages[3]["content"][0]["id"], "client_1");
+ assert_eq!(messages[4]["content"][0]["tool_use_id"], "client_1");
+ assert!(!body.to_string().contains("encrypted_content"));
+ Json(json!({"id":"msg_2","type":"message","role":"assistant","model":"model","content":[{"type":"text","text":"Done"}],"stop_reason":"end_turn","usage":{"input_tokens":4,"output_tokens":2}})).into_response()
+ }
+ }).await;
+ let state = AppState::new(cfg).unwrap();
+ seed(&state).await;
+ let tools = json!([{"type":"web_search_20250305","name":"web_search"},{"name":"Read","input_schema":{"type":"object","properties":{"path":{"type":"string"}}}}]);
+ let request = json!({"model":"model","max_tokens":1024,"tools":tools,"messages":[{"role":"user","content":"Search SQLite and read README"}]});
+ let response = send(state.clone(), "/v1/messages", request.clone(), "test-key").await;
+ assert_eq!(response.status(), 200);
+ let response = json_body(response).await;
+ assert_eq!(
+ calls.load(Ordering::SeqCst),
+ 1,
+ "client tools must return to the client immediately"
+ );
+ assert_eq!(response["usage"]["cache_read_input_tokens"], 30);
+ assert_eq!(response["content"][0]["type"], "server_tool_use");
+ assert_eq!(response["content"][2]["id"], "client_1");
+ let mut next = request;
+ next["messages"].as_array_mut().unwrap().extend([
+ json!({"role":"assistant","content":response["content"]}),
+ json!({"role":"user","content":[{"type":"tool_result","tool_use_id":"client_1","content":"README content"}]}),
+ ]);
+ let response = send(state, "/v1/messages", next, "test-key").await;
+ assert_eq!(response.status(), 200, "{}", json_body(response).await);
+ server.abort();
+}
+
+#[tokio::test]
+async fn responses_open_find_and_search_preserve_cached_token_totals() {
+ let mut cfg = config();
+ let (server, _) = mock_upstream(&mut cfg, tw_config::Protocol::OpenaiResponses, "/v1/responses", |body, i| {
+ let output = if i < 3 {
+ let (name, arguments) = match i {
+ 0 => (protocol::SEARCH, json!({"query":"SQLite"})),
+ 1 => (protocol::OPEN, json!({"url":"https://sqlite.org/changes.html"})),
+ _ => (protocol::FIND, json!({"url":"https://sqlite.org/changes.html","pattern":"Release"})),
+ };
+ json!([{"type":"function_call","id":format!("fc_{i}"),"call_id":format!("call_{i}"),"name":name,"arguments":arguments.to_string()}])
+ } else {
+ assert_eq!(body["input"].as_array().unwrap().iter().filter(|v| v["type"] == "function_call_output").count(), 3);
+ json!([{"type":"message","id":"msg_final","role":"assistant","status":"completed","content":[{"type":"output_text","text":"Source https://sqlite.org/changes.html","annotations":[]}]}])
+ };
+ Json(json!({"id":format!("resp_{i}"),"object":"response","status":"completed","model":"model","output":output,"usage":{"input_tokens":20,"input_tokens_details":{"cached_tokens":10},"output_tokens":5}})).into_response()
+ }).await;
+ let state = AppState::new(cfg).unwrap();
+ seed(&state).await;
+ for request in [
+ tw_api::WebToolRequest::OpenPage {
+ url: "https://sqlite.org/changes.html".into(),
+ offset: 0,
+ },
+ tw_api::WebToolRequest::FindInPage {
+ url: "https://sqlite.org/changes.html".into(),
+ pattern: "Release".into(),
+ },
+ ] {
+ state
+ .runtime()
+ .web_tools
+ .seed("test", &request, sample())
+ .await;
+ }
+ let response = send(
+ state,
+ "/v1/responses",
+ json!({"model":"model","input":"Read the release","tools":[{"type":"web_search"}]}),
+ "test-key",
+ )
+ .await;
+ assert_eq!(response.status(), 200);
+ let body = json_body(response).await;
+ assert_eq!(body["usage"]["input_tokens"], 80);
+ assert_eq!(body["usage"]["input_tokens_details"]["cached_tokens"], 40);
+ assert_eq!(body["usage"]["total_tokens"], 100);
+ let actions: Vec<_> = body["output"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .filter_map(|v| v["action"]["type"].as_str())
+ .collect();
+ assert_eq!(actions, ["search", "open_page", "find_in_page"]);
+ server.abort();
+}
+
+#[tokio::test]
+async fn upstream_failure_status_and_retry_headers_are_preserved() {
+ let mut cfg = config();
+ let (server, _) = mock_upstream(&mut cfg, tw_config::Protocol::OpenaiResponses, "/v1/responses", |_, _| {
+ (axum::http::StatusCode::UNPROCESSABLE_ENTITY, [("retry-after", "17")], Json(json!({"error":{"type":"invalid_request_error","message":"Model option is unsupported"}}))).into_response()
+ }).await;
+ let state = AppState::new(cfg).unwrap();
+ let response = send(
+ state,
+ "/v1/responses",
+ json!({"model":"model","input":"Search","tools":[{"type":"web_search"}]}),
+ "test-key",
+ )
+ .await;
+ assert_eq!(response.status(), 422);
+ assert_eq!(response.headers()["retry-after"], "17");
+ assert_eq!(
+ json_body(response).await["error"]["message"],
+ "Model option is unsupported"
+ );
+ server.abort();
+}
+
+#[tokio::test]
+async fn websocket_search_and_previous_response_use_the_http_pipeline() {
+ use futures::{SinkExt, StreamExt};
+ use tokio_tungstenite::tungstenite::{Message, client::IntoClientRequest};
+ let mut cfg = config();
+ let (server, _) = mock_upstream(&mut cfg, tw_config::Protocol::OpenaiResponses, "/v1/responses", |body, i| {
+ let output = if i == 0 {
+ json!([{"type":"function_call","id":"fc_ws","call_id":"c_ws","name":protocol::SEARCH,"arguments":"{\"query\":\"SQLite\"}"}])
+ } else {
+ if i == 2 {
+ assert!(body.get("previous_response_id").is_none());
+ assert!(body["input"].as_array().unwrap().iter().any(|v| v["type"] == "function_call_output"));
+ }
+ json!([{"type":"message","id":format!("msg_{i}"),"role":"assistant","status":"completed","content":[{"type":"output_text","text":"SQLite source","annotations":[]}]}])
+ };
+ Json(json!({"id":format!("resp_{i}"),"object":"response","status":"completed","model":"model","output":output,"usage":{"input_tokens":2,"output_tokens":2}})).into_response()
+ }).await;
+ let state = AppState::new(cfg).unwrap();
+ seed(&state).await;
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let addr = listener.local_addr().unwrap();
+ let gateway = tokio::spawn(async move {
+ axum::serve(
+ listener,
+ crate::router(state).into_make_service_with_connect_info::(),
+ )
+ .await
+ .unwrap();
+ });
+ let mut request = format!("ws://{addr}/v1/responses")
+ .into_client_request()
+ .unwrap();
+ request
+ .headers_mut()
+ .insert("authorization", "Bearer test-key".parse().unwrap());
+ let (mut ws, _) = tokio_tungstenite::connect_async(request).await.unwrap();
+ for i in 0..2 {
+ let mut body = json!({"type":"response.create","model":"model","input":"Search SQLite","tools":[{"type":"web_search"}]});
+ if i == 1 {
+ body["previous_response_id"] = json!("resp_1");
+ }
+ ws.send(Message::Text(body.to_string().into()))
+ .await
+ .unwrap();
+ let done = tokio::time::timeout(std::time::Duration::from_secs(5), async {
+ while let Some(Ok(Message::Text(text))) = ws.next().await {
+ let value: Value = serde_json::from_str(&text).unwrap();
+ assert_ne!(value["type"], "error", "{value}");
+ if value["type"] == "response.completed" {
+ return value;
+ }
+ }
+ panic!("WebSocket closed without a completed response");
+ })
+ .await
+ .unwrap();
+ assert_eq!(done["response"]["status"], "completed");
+ if i == 0 {
+ assert_eq!(done["response"]["output"][0]["type"], "web_search_call");
+ }
+ }
+ ws.close(None).await.unwrap();
+ gateway.abort();
+ server.abort();
+}
diff --git a/crates/tw-gateway/src/server/upgrade.rs b/crates/tw-gateway/src/server/upgrade.rs
index 1783b03a..1feb83e5 100644
--- a/crates/tw-gateway/src/server/upgrade.rs
+++ b/crates/tw-gateway/src/server/upgrade.rs
@@ -41,6 +41,26 @@ pub(super) async fn ws_upgrade(
//
// **Realtime 的连接例外**:它的模型写在查询串里(`?model=gpt-realtime`),一条连接只用
// 这一个。和 HTTP 那条路按请求体里的模型一样,路由、密钥的模型范围、别名都按它来
+ if rt.config.web_tools.mode == tw_api::WebToolsMode::Core
+ && crate::client_api::ClientApi::of_path(uri.path())
+ == Some(crate::client_api::ClientApi::OpenaiResponses)
+ {
+ return Ok(ws
+ .max_message_size(16 * 1024 * 1024)
+ .on_upgrade(move |socket| {
+ super::pipeline::web_socket(
+ socket,
+ state,
+ rt,
+ client_name,
+ uri,
+ query,
+ headers,
+ from,
+ live,
+ )
+ }));
+ }
let realtime = crate::ws::realtime(uri.path());
let facts = tw_engine::RequestFacts {
client: client_name.clone(),
diff --git a/crates/tw-gateway/src/state.rs b/crates/tw-gateway/src/state.rs
index ddfb47de..f3f0ba5b 100644
--- a/crates/tw-gateway/src/state.rs
+++ b/crates/tw-gateway/src/state.rs
@@ -23,6 +23,7 @@ mod upstream;
/// 了,正在排队的请求会失去它们的位置;事件流丢了,界面上的实时列表会
/// 断一次。这些都不该因为改了一条路由规则而发生。
pub struct Runtime {
+ pub web_tools: Arc,
pub config: Arc,
/// 路由引擎。**和配置一起建,一起换** —— 分开持有会让「规则改了但
/// 引擎还是旧的」变成可能,而那种不一致完全静默。
@@ -122,7 +123,13 @@ impl Runtime {
let content = sec.content.rules().map_err(bad)?;
let answer = tw_guard::content::answer::Answer::build(&content);
let plugins = Arc::new(plugins.build(&config));
+ let mut web_tools = crate::web_tools::WebTools::new(&config)?;
+ if let Some(previous) = previous {
+ web_tools.history = previous.web_tools.history.clone();
+ }
+ let web_tools = Arc::new(web_tools);
Ok(Self {
+ web_tools,
engine: Arc::new(config.engine()),
config: Arc::new(config),
clients,
@@ -141,6 +148,7 @@ impl Runtime {
/// 同一份配置、换一份插件。插件文件变了时走这条:配置没变,别的都不用重建
fn with_plugins(&self, plugins: crate::plugin::PluginSet) -> Self {
Self {
+ web_tools: self.web_tools.clone(),
config: self.config.clone(),
engine: self.engine.clone(),
clients: self.clients.clone(),
diff --git a/crates/tw-gateway/src/web_tools/collect.rs b/crates/tw-gateway/src/web_tools/collect.rs
new file mode 100644
index 00000000..561a8593
--- /dev/null
+++ b/crates/tw-gateway/src/web_tools/collect.rs
@@ -0,0 +1,281 @@
+use scraper::{Html, Selector};
+use tw_api::{SearchEngine, WebSource};
+use tw_types::Msg;
+
+use super::error;
+
+fn selector(s: &str) -> Selector {
+ Selector::parse(s).expect("static CSS selector")
+}
+fn clean(s: &str) -> String {
+ s.split_whitespace().collect::>().join(" ")
+}
+
+pub(super) fn search(engine: SearchEngine, body: &str) -> Result, Msg> {
+ if body.contains("anomaly.js") || body.contains("challenge-form") || body.contains("b_captcha")
+ {
+ return Err(error("search engine requires a verification challenge"));
+ }
+ match engine {
+ SearchEngine::Bing => rss(body),
+ SearchEngine::Duckduckgo => duckduckgo(body),
+ }
+}
+
+fn source(engine: SearchEngine, url: &str, title: &str, text: &str) -> Option {
+ let mut url = super::public::parse(url).ok()?;
+ url.set_fragment(None);
+ let params: Vec<_> = url
+ .query_pairs()
+ .filter(|(k, _)| !k.starts_with("utm_") && !matches!(k.as_ref(), "gclid" | "fbclid"))
+ .map(|(k, v)| (k.into_owned(), v.into_owned()))
+ .collect();
+ if params.is_empty() {
+ url.set_query(None);
+ } else {
+ url.query_pairs_mut().clear().extend_pairs(params);
+ }
+ Some(WebSource {
+ url: url.to_string(),
+ title: clean(title).chars().take(500).collect(),
+ text: clean(text).chars().take(2000).collect(),
+ engine: Some(engine),
+ published_at: None,
+ })
+}
+
+fn rss(body: &str) -> Result, Msg> {
+ use quick_xml::events::Event;
+ let mut reader = quick_xml::Reader::from_str(body);
+ reader.config_mut().trim_text(false);
+ let mut items = Vec::new();
+ let mut in_item = false;
+ let mut rss = false;
+ let (mut tag, mut title, mut url, mut text, mut date) = (
+ Vec::new(),
+ String::new(),
+ String::new(),
+ String::new(),
+ String::new(),
+ );
+ loop {
+ match reader.read_event() {
+ Ok(Event::Start(e)) => {
+ tag = e.name().as_ref().as_bytes().to_vec();
+ if tag == b"rss" {
+ rss = true;
+ }
+ if tag == b"item" {
+ in_item = true;
+ title.clear();
+ url.clear();
+ text.clear();
+ date.clear();
+ }
+ }
+ Ok(event @ (Event::Text(_) | Event::CData(_) | Event::GeneralRef(_))) if in_item => {
+ let value = match &event {
+ Event::Text(e) => e.as_ref().to_string(),
+ Event::CData(e) => e.as_ref().to_string(),
+ Event::GeneralRef(e) => {
+ quick_xml::escape::unescape(&format!("&{};", e.as_ref()))
+ .map_err(|_| error("invalid search XML entity"))?
+ .into_owned()
+ }
+ _ => unreachable!(),
+ };
+ match tag.as_slice() {
+ b"title" => title.push_str(&value),
+ b"link" => url.push_str(&value),
+ b"description" => text.push_str(&value),
+ b"pubDate" => date.push_str(&value),
+ _ => {}
+ }
+ }
+ Ok(Event::End(e)) if e.name().as_ref() == "item" => {
+ if let Some(mut s) = source(
+ SearchEngine::Bing,
+ &url,
+ &title,
+ &clean(
+ &Html::parse_fragment(&text)
+ .root_element()
+ .text()
+ .collect::>()
+ .join(" "),
+ ),
+ ) {
+ if !date.is_empty() {
+ s.published_at = Some(date.clone());
+ }
+ items.push(s);
+ }
+ in_item = false;
+ }
+ Ok(Event::Eof) => break,
+ Err(_) => return Err(error("invalid search XML")),
+ _ => {}
+ }
+ }
+ if !rss {
+ return Err(error("search engine did not return RSS results"));
+ }
+ Ok(items)
+}
+
+fn duckduckgo(body: &str) -> Result, Msg> {
+ let doc = Html::parse_document(body);
+ let mut result = Vec::new();
+ for row in doc.select(&selector(".result")) {
+ let Some(link) = row.select(&selector(".result__a")).next() else {
+ continue;
+ };
+ let Some(href) = link.value().attr("href") else {
+ continue;
+ };
+ let absolute = if href.starts_with("//") {
+ format!("https:{href}")
+ } else {
+ href.to_string()
+ };
+ let Ok(url) = reqwest::Url::parse(&absolute) else {
+ continue;
+ };
+ let destination = url
+ .query_pairs()
+ .find(|(k, _)| k == "uddg")
+ .map(|(_, v)| v.into_owned())
+ .unwrap_or(absolute);
+ let title = link.text().collect::>().join(" ");
+ let text = row
+ .select(&selector(".result__snippet"))
+ .next()
+ .map(|n| n.text().collect::>().join(" "))
+ .unwrap_or_default();
+ if let Some(s) = source(SearchEngine::Duckduckgo, &destination, &title, &text) {
+ result.push(s);
+ }
+ }
+ if result.is_empty() && !body.contains("no-results") {
+ return Err(error("search page layout was not recognized"));
+ }
+ Ok(result)
+}
+
+pub(super) fn page(body: &str) -> (String, String) {
+ let mut doc = Html::parse_document(body);
+ let remove: Vec<_> = doc
+ .select(&selector(
+ "script,style,noscript,nav,footer,header,aside,form,svg,template",
+ ))
+ .map(|node| node.id())
+ .collect();
+ for id in remove {
+ if let Some(mut node) = doc.tree.get_mut(id) {
+ node.detach();
+ }
+ }
+ let title = doc
+ .select(&selector("title"))
+ .next()
+ .map(|n| clean(&n.text().collect::>().join(" ")))
+ .unwrap_or_default();
+ let root = doc
+ .select(&selector("article,main,[role=main]"))
+ .next()
+ .unwrap_or_else(|| doc.root_element());
+ let text = root
+ .text()
+ .map(clean)
+ .filter(|s| !s.is_empty())
+ .collect::>()
+ .join("\n");
+ (title.chars().take(500).collect(), text)
+}
+
+pub(super) fn excerpt(text: &str, query: &str, limit: usize) -> String {
+ let words: Vec<_> = query
+ .split_whitespace()
+ .filter(|s| s.chars().count() > 2)
+ .map(str::to_lowercase)
+ .collect();
+ let mut lines: Vec<_> = text.lines().enumerate().collect();
+ lines.sort_by_key(|(i, line)| {
+ let l = line.to_lowercase();
+ (
+ std::cmp::Reverse(words.iter().filter(|w| l.contains(w.as_str())).count()),
+ *i,
+ )
+ });
+ lines
+ .iter()
+ .take(12)
+ .map(|(_, s)| *s)
+ .collect::>()
+ .join("\n")
+ .chars()
+ .take(limit)
+ .collect()
+}
+
+pub(super) fn find(text: &str, pattern: &str, limit: usize) -> String {
+ let lines: Vec<_> = text.lines().collect();
+ let pattern = pattern.to_lowercase();
+ let mut selected = std::collections::BTreeSet::new();
+ for (i, line) in lines.iter().enumerate() {
+ if line.to_lowercase().contains(&pattern) {
+ selected.extend(i.saturating_sub(1)..=(i + 1).min(lines.len().saturating_sub(1)));
+ }
+ }
+ if selected.is_empty() {
+ return "No matching text was found.".into();
+ }
+ selected
+ .into_iter()
+ .map(|i| format!("{}: {}", i + 1, lines[i]))
+ .collect::>()
+ .join("\n")
+ .chars()
+ .take(limit)
+ .collect()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ #[test]
+ fn parses_search_sources_and_rejects_challenges() {
+ let r = search(SearchEngine::Bing, "- Releasehttps://example.org/?utm_source=xVersion 3
").unwrap();
+ assert_eq!(r[0].url, "https://example.org/");
+ let html = r#""#;
+ assert_eq!(
+ search(SearchEngine::Duckduckgo, html).unwrap()[0].text,
+ "Version 3"
+ );
+ assert!(search(SearchEngine::Duckduckgo, "
]]>"#).unwrap();
+ assert_eq!(result[0].title, "Rust & Web 中文");
+ assert_eq!(result[0].url, "https://example.org/?a=1&b=2");
+ assert_eq!(result[0].text, "A & B");
+ }
+}
diff --git a/crates/tw-gateway/src/web_tools/history.rs b/crates/tw-gateway/src/web_tools/history.rs
new file mode 100644
index 00000000..5b8d2827
--- /dev/null
+++ b/crates/tw-gateway/src/web_tools/history.rs
@@ -0,0 +1,234 @@
+//! Replay our own hosted-tool items as ordinary tool exchanges on the next turn.
+//! Never forward fabricated provider-encrypted search content to an upstream.
+
+use std::collections::HashMap;
+use std::sync::Mutex;
+use std::time::{Duration, Instant};
+
+use serde_json::{Value, json};
+use tw_dialect::ir::Dialect;
+use tw_types::Msg;
+
+use super::{
+ error,
+ protocol::{Call, FIND, OPEN, SEARCH},
+};
+
+#[derive(Default)]
+pub(crate) struct History(Mutex>);
+
+struct Entry {
+ at: Instant,
+ call: Value,
+ result: Value,
+}
+
+impl History {
+ pub fn remember(
+ &self,
+ identity: &str,
+ dialect: Dialect,
+ call: &Call,
+ result: &tw_api::WebToolResult,
+ ) {
+ let (name, arguments) = match &call.request {
+ tw_api::WebToolRequest::Search { query, .. } => (SEARCH, json!({"query":query})),
+ tw_api::WebToolRequest::OpenPage { url, offset } => {
+ (OPEN, json!({"url":url,"offset":offset}))
+ }
+ tw_api::WebToolRequest::FindInPage { url, pattern } => {
+ (FIND, json!({"url":url,"pattern":pattern}))
+ }
+ };
+ let text = serde_json::to_string(result).unwrap_or_default();
+ let (id, item, output) = if dialect == Dialect::Anthropic {
+ (
+ format!("srvtoolu_tw_{}", call.native_id),
+ json!({"type":"tool_use","id":call.id,"name":name,"input":arguments}),
+ json!({"type":"tool_result","tool_use_id":call.id,"content":text}),
+ )
+ } else {
+ (
+ format!("ws_tw_{}", call.native_id),
+ json!({"type":"function_call","call_id":call.id,"name":name,"arguments":arguments.to_string()}),
+ json!({"type":"function_call_output","call_id":call.id,"output":text}),
+ )
+ };
+ let mut entries = self.0.lock().unwrap_or_else(|e| e.into_inner());
+ entries.retain(|_, e| e.at.elapsed() < Duration::from_secs(1800));
+ if entries.len() >= 256
+ && let Some(oldest) = entries
+ .iter()
+ .min_by_key(|(_, e)| e.at)
+ .map(|(k, _)| k.clone())
+ {
+ entries.remove(&oldest);
+ }
+ entries.insert(
+ (identity.into(), id),
+ Entry {
+ at: Instant::now(),
+ call: item,
+ result: output,
+ },
+ );
+ }
+
+ fn get(&self, identity: &str, id: &str) -> Result<(Value, Value), Msg> {
+ let entries = self.0.lock().unwrap_or_else(|e| e.into_inner());
+ let e = entries.get(&(identity.into(), id.into())).filter(|e| e.at.elapsed() < Duration::from_secs(1800))
+ .ok_or_else(|| error("Core's web-tool history has expired; start a new conversation or resend the source text"))?;
+ Ok((e.call.clone(), e.result.clone()))
+ }
+
+ pub fn restore(
+ &self,
+ identity: &str,
+ dialect: Dialect,
+ body: &mut Value,
+ tools: bool,
+ ) -> Result {
+ let mut changed = false;
+ if dialect == Dialect::Responses {
+ if let Some(items) = body["input"].as_array_mut() {
+ let mut restored = Vec::new();
+ for item in std::mem::take(items) {
+ if item["type"] == "web_search_call"
+ && item["id"].as_str().is_some_and(|s| s.starts_with("ws_tw_"))
+ {
+ changed = true;
+ let saved = self.get(identity, item["id"].as_str().unwrap_or("")).ok();
+ if let Some((call, output)) = saved.as_ref().filter(|_| tools) {
+ restored.extend([call.clone(), output.clone()]);
+ } else {
+ restored.push(json!({"role":"assistant","content":[{"type":"output_text","text":recalled(saved.as_ref().map(|(_, out)| out), &item)}]}));
+ }
+ } else {
+ restored.push(item);
+ }
+ }
+ *items = restored;
+ }
+ } else if dialect == Dialect::Anthropic
+ && let Some(messages) = body["messages"].as_array_mut()
+ {
+ let mut restored = Vec::new();
+ for message in std::mem::take(messages) {
+ let Some(blocks) = message["content"]
+ .as_array()
+ .filter(|_| message["role"] == "assistant")
+ else {
+ restored.push(message);
+ continue;
+ };
+ let mut assistant = Vec::new();
+ for block in blocks {
+ if block["type"] == "server_tool_use"
+ && block["id"]
+ .as_str()
+ .is_some_and(|s| s.starts_with("srvtoolu_tw_"))
+ {
+ changed = true;
+ if tools
+ && let Ok((call, _)) =
+ self.get(identity, block["id"].as_str().unwrap_or(""))
+ {
+ assistant.push(call);
+ }
+ } else if block["tool_use_id"]
+ .as_str()
+ .is_some_and(|s| s.starts_with("srvtoolu_tw_"))
+ {
+ changed = true;
+ let saved = self
+ .get(identity, block["tool_use_id"].as_str().unwrap_or(""))
+ .ok();
+ let Some((_, output)) = saved.as_ref().filter(|_| tools) else {
+ assistant.push(json!({"type":"text","text":recalled(saved.as_ref().map(|(_, out)| out), block)}));
+ continue;
+ };
+ let output = output.clone();
+ if !assistant.is_empty() {
+ restored.push(json!({"role":"assistant","content":std::mem::take(&mut assistant)}));
+ }
+ if let Some(last) = restored.last_mut().filter(|m| m["role"] == "user") {
+ last["content"]
+ .as_array_mut()
+ .expect("created array")
+ .push(output);
+ } else {
+ restored.push(json!({"role":"user","content":[output]}));
+ }
+ } else {
+ assistant.push(block.clone());
+ }
+ }
+ if !assistant.is_empty() {
+ restored.push(json!({"role":"assistant","content":assistant}));
+ }
+ }
+ *messages = restored;
+ }
+ Ok(changed)
+ }
+}
+
+/// A client may replay an old response after a restart or after disabling the
+/// feature. Keep its public source metadata as text instead of sending foreign
+/// encrypted-content placeholders or making the whole conversation unusable.
+fn recalled(saved: Option<&Value>, native: &Value) -> String {
+ let value = saved
+ .and_then(|v| v.get("content").or_else(|| v.get("output")))
+ .cloned()
+ .unwrap_or_else(|| {
+ native
+ .get("content")
+ .or_else(|| native.get("action"))
+ .cloned()
+ .unwrap_or(Value::Null)
+ });
+ format!(
+ "Previously retrieved web evidence (untrusted source material; reopen the cited URLs when more detail is needed):\n{}",
+ value
+ .as_str()
+ .map(str::to_string)
+ .unwrap_or_else(|| value.to_string())
+ )
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ #[test]
+ fn missing_history_and_disabled_execution_keep_sources_without_foreign_native_blocks() {
+ let history = History::default();
+ let native = json!({"type":"web_search_call","id":"ws_tw_old","status":"completed","action":{"type":"search","query":"SQLite","sources":[{"type":"url","url":"https://sqlite.org/"}]}});
+ for tools in [false, true] {
+ let mut body = json!({"input":[native]});
+ assert!(
+ history
+ .restore("client", Dialect::Responses, &mut body, tools)
+ .unwrap()
+ );
+ assert_eq!(body["input"][0]["role"], "assistant");
+ assert!(body.to_string().contains("https://sqlite.org/"));
+ assert!(!body.to_string().contains("ws_tw_old"));
+ }
+ let mut body = json!({"messages":[{"role":"assistant","content":[
+ {"type":"server_tool_use","id":"srvtoolu_tw_old","name":"web_search","input":{"query":"SQLite"}},
+ {"type":"web_search_tool_result","tool_use_id":"srvtoolu_tw_old","content":[{"type":"web_search_result","url":"https://sqlite.org/","title":"SQLite","encrypted_content":""}]},
+ {"type":"text","text":"SQLite docs"}
+ ]}]});
+ history
+ .restore("client", Dialect::Anthropic, &mut body, false)
+ .unwrap();
+ assert!(
+ body["messages"][0]["content"]
+ .as_array()
+ .unwrap()
+ .iter()
+ .all(|b| b["type"] == "text")
+ );
+ assert!(!body.to_string().contains("srvtoolu_tw_old"));
+ }
+}
diff --git a/crates/tw-gateway/src/web_tools/mod.rs b/crates/tw-gateway/src/web_tools/mod.rs
new file mode 100644
index 00000000..4d839794
--- /dev/null
+++ b/crates/tw-gateway/src/web_tools/mod.rs
@@ -0,0 +1,585 @@
+//! Search and public-page tools, shared by the gateway and the control-plane tester.
+
+mod collect;
+mod history;
+pub(crate) mod protocol;
+mod public;
+pub(crate) mod wire;
+
+use std::collections::HashMap;
+use std::sync::{Arc, Mutex};
+use std::time::{Duration, Instant};
+
+use tokio::sync::{Mutex as AsyncMutex, Semaphore};
+use tw_api::{SearchEngine, WebToolRequest, WebToolResult, WebToolsConfig};
+use tw_types::{Msg, msg};
+
+type PageCached = Arc>>;
+
+type Cached = Arc>>;
+
+pub struct WebTools {
+ pub(crate) history: Arc,
+ config: WebToolsConfig,
+ client: public::ClientSettings,
+ slots: Semaphore,
+ cache: Mutex>,
+ pages: Mutex>,
+ engines: AsyncMutex>,
+}
+
+#[derive(Default)]
+struct EngineState {
+ next: Option,
+ failed_until: Option,
+}
+
+pub(crate) fn error(detail: impl Into) -> Msg {
+ msg!("gw.web_tools.failed", detail = detail.into() => "Web tools: {detail}")
+}
+
+/// Both API interception and the control tester use the same outbound protections.
+pub async fn execute(
+ rt: &crate::state::Runtime,
+ identity: &str,
+ request: &WebToolRequest,
+) -> Result {
+ execute_scoped(rt, identity, request, &[], &[]).await
+}
+
+pub(crate) async fn execute_scoped(
+ rt: &crate::state::Runtime,
+ identity: &str,
+ request: &WebToolRequest,
+ allowed: &[String],
+ blocked: &[String],
+) -> Result {
+ let text = serde_json::to_string(request).map_err(|_| error("invalid tool request"))?;
+ let body = serde_json::json!({"messages":[{"role":"user","content":text}]});
+ let screened = crate::guard::screen_value(
+ &crate::guard::Screen::of(rt),
+ tw_dialect::ir::Dialect::Anthropic,
+ &body,
+ );
+ if screened.refusal().is_some() {
+ return Err(error("request was blocked by the configured content rules"));
+ }
+ let bytes = screened
+ .body
+ .unwrap_or_else(|| bytes::Bytes::from(body.to_string()));
+ let shape = Some(crate::guard::Shape::Generate(
+ tw_dialect::ir::Dialect::Anthropic,
+ ));
+ let ledger = crate::guard::ledger_for(&bytes);
+ let (bytes, _) = crate::guard::replace(
+ rt.config.security.redact.mode,
+ &rt.redact,
+ bytes,
+ &ledger,
+ shape,
+ );
+ let body: serde_json::Value =
+ serde_json::from_slice(&bytes).map_err(|_| error("protected tool request is invalid"))?;
+ let request = serde_json::from_str(
+ body["messages"][0]["content"]
+ .as_str()
+ .ok_or_else(|| error("protected tool input is missing"))?,
+ )
+ .map_err(|_| error("protected tool input is invalid"))?;
+ rt.web_tools
+ .execute_scoped(identity, &request, allowed, blocked)
+ .await
+}
+
+impl WebTools {
+ #[cfg(test)]
+ pub(crate) async fn seed(
+ &self,
+ identity: &str,
+ request: &WebToolRequest,
+ result: WebToolResult,
+ ) {
+ let key = format!("{identity}\n{}", serde_json::to_string(request).unwrap());
+ self.cache.lock().unwrap().insert(
+ key,
+ Arc::new(AsyncMutex::new(Some((Instant::now(), result)))),
+ );
+ }
+ pub fn new(config: &tw_config::Config) -> Result {
+ Ok(Self {
+ history: Arc::new(history::History::default()),
+ config: config.web_tools.clone(),
+ client: public::ClientSettings::new(config)?,
+ slots: Semaphore::new(config.web_tools.max_concurrent as usize),
+ cache: Mutex::new(HashMap::new()),
+ pages: Mutex::new(HashMap::new()),
+ engines: AsyncMutex::new(HashMap::new()),
+ })
+ }
+
+ /// Caller identity is part of the cache key. Only successful results are cached.
+ pub async fn execute(
+ &self,
+ identity: &str,
+ request: &WebToolRequest,
+ ) -> Result {
+ self.execute_scoped(identity, request, &[], &[]).await
+ }
+
+ async fn execute_scoped(
+ &self,
+ identity: &str,
+ request: &WebToolRequest,
+ allowed: &[String],
+ blocked: &[String],
+ ) -> Result {
+ let started = Instant::now();
+ let run = async {
+ let key = format!(
+ "{identity}\n{}",
+ serde_json::to_string(request).map_err(|_| error("invalid request"))?
+ );
+ let key = if allowed.is_empty() && blocked.is_empty() {
+ key
+ } else {
+ format!("{key}\n{}", serde_json::json!([allowed, blocked]))
+ };
+ let entry = {
+ let mut cache = self.cache.lock().unwrap_or_else(|e| e.into_inner());
+ if cache.len() >= 256 {
+ cache.retain(|_, e| Arc::strong_count(e) > 1);
+ }
+ if cache.len() >= 256 && !cache.contains_key(&key) {
+ return Err(error("too many pending web operations"));
+ }
+ cache
+ .entry(key)
+ .or_insert_with(|| Arc::new(AsyncMutex::new(None)))
+ .clone()
+ };
+ // Holding this lock coalesces concurrent requests without leaving a background
+ // operation behind when the last caller is cancelled.
+ let mut cached = entry.lock().await;
+ if let Some((at, result)) = &*cached
+ && at.elapsed() < Duration::from_secs(u64::from(self.config.cache_secs))
+ {
+ let mut result = result.clone();
+ result.cached = true;
+ return Ok(result);
+ }
+ let _slot = self
+ .slots
+ .acquire()
+ .await
+ .map_err(|_| error("executor stopped"))?;
+ let deadline = started + Duration::from_secs(u64::from(self.config.timeout_secs));
+ let result = self
+ .run(identity, request, allowed, blocked, deadline)
+ .await?;
+ if self.config.cache_secs > 0 {
+ *cached = Some((Instant::now(), result.clone()));
+ }
+ Ok(result)
+ };
+ let mut result = tokio::time::timeout(
+ Duration::from_secs(u64::from(self.config.timeout_secs)),
+ run,
+ )
+ .await
+ .map_err(|_| error("operation timed out"))??;
+ result.elapsed_ms = started.elapsed().as_millis().min(u128::from(u32::MAX)) as u32;
+ Ok(result)
+ }
+
+ async fn run(
+ &self,
+ identity: &str,
+ request: &WebToolRequest,
+ scope_allowed: &[String],
+ scope_blocked: &[String],
+ deadline: Instant,
+ ) -> Result {
+ let mut result = WebToolResult {
+ sources: Vec::new(),
+ cached: false,
+ elapsed_ms: 0,
+ warnings: Vec::new(),
+ truncated: false,
+ };
+ match request {
+ WebToolRequest::Search {
+ query,
+ allowed_domains,
+ blocked_domains,
+ } => {
+ if query.trim().is_empty() || query.len() > 4096 {
+ return Err(error("query must contain between 1 and 4096 bytes"));
+ }
+ public::validate_domains(allowed_domains)?;
+ public::validate_domains(blocked_domains)?;
+ let engines = if self.config.engines.is_empty() {
+ vec![SearchEngine::Bing, SearchEngine::Duckduckgo]
+ } else {
+ self.config.engines.clone()
+ };
+ let mut succeeded = false;
+ let engine_count = engines.len();
+ for (index, engine) in engines.into_iter().enumerate() {
+ // Leave time for the next engine. A stalled engine must not
+ // consume the entire operation's timeout and prevent failover.
+ let remaining = deadline
+ .saturating_duration_since(Instant::now())
+ .saturating_sub(Duration::from_millis(20));
+ let budget = remaining
+ / (engine_count - index + usize::from(self.config.read_pages > 0)) as u32;
+ let attempt = tokio::time::timeout(budget, async {
+ if !self.reserve(engine).await {
+ return Ok(None);
+ }
+ self.search(engine, query, allowed_domains, blocked_domains)
+ .await
+ .map(Some)
+ })
+ .await
+ .unwrap_or_else(|_| Err(error("search engine timed out")));
+ match attempt {
+ Ok(None) => continue,
+ Ok(Some(sources)) => {
+ succeeded = true;
+ for source in sources {
+ if !result.sources.iter().any(|s| s.url == source.url) {
+ result.sources.push(source);
+ }
+ }
+ if result.sources.len() >= self.config.max_results as usize {
+ break;
+ }
+ }
+ Err(e) => {
+ self.engines
+ .lock()
+ .await
+ .entry(engine)
+ .or_default()
+ .failed_until = Some(Instant::now() + Duration::from_secs(60));
+ result.warnings.push(e);
+ }
+ }
+ }
+ if !succeeded {
+ return Err(error(
+ "all configured search engines failed or are cooling down",
+ ));
+ }
+ result.sources.truncate(self.config.max_results as usize);
+ let mut read = 0;
+ for source in &mut result.sources {
+ if read >= self.config.read_pages || source.text.chars().count() >= 300 {
+ continue;
+ }
+ read += 1;
+ let budget = deadline
+ .saturating_duration_since(Instant::now())
+ .saturating_sub(Duration::from_millis(20))
+ / (self.config.read_pages - read + 1);
+ if budget < Duration::from_millis(100) {
+ break;
+ }
+ match tokio::time::timeout(
+ budget.min(Duration::from_secs(5)),
+ self.page(identity, &source.url, allowed_domains, blocked_domains),
+ )
+ .await
+ .unwrap_or_else(|_| Err(error("optional page reading timed out")))
+ {
+ Ok((_, text)) => source.text = collect::excerpt(&text, query, 3000),
+ Err(e) => result.warnings.push(e),
+ }
+ }
+ }
+ WebToolRequest::OpenPage { url, offset } => {
+ let (title, text) = self
+ .page(identity, url, scope_allowed, scope_blocked)
+ .await?;
+ let chars = text.chars().count();
+ let limit = self.config.max_page_chars as usize;
+ result.truncated = chars > *offset as usize + limit;
+ result.sources.push(tw_api::WebSource {
+ url: url.clone(),
+ title,
+ text: text.chars().skip(*offset as usize).take(limit).collect(),
+ engine: None,
+ published_at: None,
+ });
+ }
+ WebToolRequest::FindInPage { url, pattern } => {
+ if pattern.trim().is_empty() || pattern.len() > 512 {
+ return Err(error("find pattern must contain between 1 and 512 bytes"));
+ }
+ let (title, text) = self
+ .page(identity, url, scope_allowed, scope_blocked)
+ .await?;
+ let found = collect::find(&text, pattern, self.config.max_page_chars as usize);
+ result.sources.push(tw_api::WebSource {
+ url: url.clone(),
+ title,
+ text: found,
+ engine: None,
+ published_at: None,
+ });
+ }
+ }
+ Ok(result)
+ }
+
+ async fn reserve(&self, engine: SearchEngine) -> bool {
+ let delay = {
+ let mut engines = self.engines.lock().await;
+ let state = engines.entry(engine).or_default();
+ let now = Instant::now();
+ if state.failed_until.is_some_and(|until| until > now) {
+ return false;
+ }
+ let at = state.next.unwrap_or(now).max(now);
+ state.next = Some(at + Duration::from_millis(u64::from(self.config.interval_ms)));
+ at.saturating_duration_since(now)
+ };
+ tokio::time::sleep(delay).await;
+ true
+ }
+
+ async fn search(
+ &self,
+ engine: SearchEngine,
+ query: &str,
+ allowed: &[String],
+ blocked: &[String],
+ ) -> Result, Msg> {
+ let mut q = query.to_string();
+ if !allowed.is_empty() {
+ q.push_str(&format!(
+ " ({})",
+ allowed
+ .iter()
+ .map(|d| format!("site:{d}"))
+ .collect::>()
+ .join(" OR ")
+ ));
+ } else if !self.config.allowed_domains.is_empty() {
+ q.push_str(&format!(
+ " ({})",
+ self.config
+ .allowed_domains
+ .iter()
+ .map(|d| format!("site:{d}"))
+ .collect::>()
+ .join(" OR ")
+ ));
+ }
+ let mut url = reqwest::Url::parse(match engine {
+ SearchEngine::Bing => "https://www.bing.com/search",
+ SearchEngine::Duckduckgo => "https://html.duckduckgo.com/html/",
+ })
+ .expect("built-in URL");
+ url.query_pairs_mut().append_pair("q", &q);
+ match engine {
+ SearchEngine::Bing => {
+ url.query_pairs_mut().append_pair("format", "rss");
+ if !self.config.language.is_empty() {
+ url.query_pairs_mut()
+ .append_pair("setlang", &self.config.language);
+ }
+ if !self.config.region.is_empty() {
+ url.query_pairs_mut().append_pair("cc", &self.config.region);
+ }
+ }
+ SearchEngine::Duckduckgo => {
+ if !self.config.region.is_empty() && !self.config.language.is_empty() {
+ url.query_pairs_mut().append_pair(
+ "kl",
+ &format!(
+ "{}-{}",
+ self.config.region.to_lowercase(),
+ self.config.language.to_lowercase()
+ ),
+ );
+ }
+ }
+ }
+ let (_, body) = self.fetch(url, None).await?;
+ let mut sources = collect::search(engine, &body)?;
+ sources.retain(|s| {
+ reqwest::Url::parse(&s.url).ok().is_some_and(|url| {
+ self.allowed(&url) && public::domains_allow(&url, allowed, blocked)
+ })
+ });
+ Ok(sources)
+ }
+
+ fn allowed(&self, url: &reqwest::Url) -> bool {
+ public::domains_allow(
+ url,
+ &self.config.allowed_domains,
+ &self.config.blocked_domains,
+ )
+ }
+
+ async fn page(
+ &self,
+ identity: &str,
+ url: &str,
+ allowed: &[String],
+ blocked: &[String],
+ ) -> Result<(String, String), Msg> {
+ let key = format!(
+ "{identity}\n{url}\n{}",
+ serde_json::json!([allowed, blocked])
+ );
+ let entry = {
+ let mut pages = self.pages.lock().unwrap_or_else(|e| e.into_inner());
+ if pages.len() >= 32 {
+ pages.retain(|_, p| Arc::strong_count(p) > 1);
+ }
+ if pages.len() >= 32 && !pages.contains_key(&key) {
+ return Err(error("too many pending page reads"));
+ }
+ pages
+ .entry(key)
+ .or_insert_with(|| Arc::new(AsyncMutex::new(None)))
+ .clone()
+ };
+ let mut cache = entry.lock().await;
+ if let Some((at, page)) = &*cache
+ && at.elapsed() < Duration::from_secs(u64::from(self.config.cache_secs))
+ {
+ return Ok(page.clone());
+ }
+ let page = self.fetch_page(url, allowed, blocked).await?;
+ if self.config.cache_secs > 0 {
+ *cache = Some((Instant::now(), page.clone()));
+ }
+ Ok(page)
+ }
+
+ async fn fetch_page(
+ &self,
+ url: &str,
+ allowed: &[String],
+ blocked: &[String],
+ ) -> Result<(String, String), Msg> {
+ let url = public::parse(url)?;
+ if !self.allowed(&url) {
+ return Err(error("page is excluded by the configured domain rules"));
+ }
+ let (kind, body) = self.fetch(url, Some((allowed, blocked))).await?;
+ if kind.contains("html") {
+ Ok(collect::page(&body))
+ } else if kind.starts_with("text/") {
+ Ok((String::new(), body))
+ } else {
+ Err(error("page must be HTML or plain text"))
+ }
+ }
+
+ async fn fetch(
+ &self,
+ mut url: reqwest::Url,
+ filters: Option<(&[String], &[String])>,
+ ) -> Result<(String, String), Msg> {
+ use futures::StreamExt;
+ for _ in 0..6 {
+ let addresses = public::check(&url).await?;
+ if let Some((allowed, blocked)) = filters
+ && (!self.allowed(&url) || !public::domains_allow(&url, allowed, blocked))
+ {
+ return Err(error("redirect is excluded by the configured domain rules"));
+ }
+ let response = self.client.get(&url, &addresses).await?;
+ if response.status().is_redirection() {
+ let to = response
+ .headers()
+ .get("location")
+ .and_then(|h| h.to_str().ok())
+ .ok_or_else(|| error("redirect has no location"))?;
+ let next = url.join(to).map_err(|_| error("invalid redirect"))?;
+ if url.scheme() == "https" && next.scheme() != "https" {
+ return Err(error("refused an insecure redirect"));
+ }
+ url = next;
+ continue;
+ }
+ if !response.status().is_success() {
+ return Err(error(format!(
+ "source returned HTTP {}",
+ response.status().as_u16()
+ )));
+ }
+ let kind = response
+ .headers()
+ .get("content-type")
+ .and_then(|v| v.to_str().ok())
+ .unwrap_or("application/octet-stream")
+ .to_lowercase();
+ let mut bytes = Vec::new();
+ let mut stream = response.bytes_stream();
+ while let Some(chunk) = stream.next().await {
+ let chunk = chunk.map_err(|e| error(e.without_url().to_string()))?;
+ if bytes.len() + chunk.len() > 2 * 1024 * 1024 {
+ return Err(error("source exceeded the 2 MiB download limit"));
+ }
+ bytes.extend_from_slice(&chunk);
+ }
+ return String::from_utf8(bytes)
+ .map(|body| (kind, body))
+ .map_err(|_| error("source is not UTF-8 text"));
+ }
+ Err(error("too many redirects"))
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ #[tokio::test]
+ #[ignore = "uses public search engines and public webpages"]
+ async fn live_search_read_and_find() {
+ let mut config = tw_config::Config::default();
+ config.web_tools.engines = vec![SearchEngine::Bing];
+ config.web_tools.read_pages = 0;
+ let tools = WebTools::new(&config).unwrap();
+ let search = tools
+ .execute(
+ "live-test",
+ &WebToolRequest::Search {
+ query: "SQLite release notes".into(),
+ allowed_domains: vec![],
+ blocked_domains: vec![],
+ },
+ )
+ .await
+ .unwrap();
+ assert!(!search.sources.is_empty());
+ let page = tools
+ .execute(
+ "live-test",
+ &WebToolRequest::OpenPage {
+ url: "https://www.sqlite.org/changes.html".into(),
+ offset: 0,
+ },
+ )
+ .await
+ .unwrap();
+ assert!(page.sources[0].text.to_lowercase().contains("sqlite"));
+ let found = tools
+ .execute(
+ "live-test",
+ &WebToolRequest::FindInPage {
+ url: "https://www.sqlite.org/changes.html".into(),
+ pattern: "release".into(),
+ },
+ )
+ .await
+ .unwrap();
+ assert!(found.sources[0].text.to_lowercase().contains("release"));
+ }
+}
diff --git a/crates/tw-gateway/src/web_tools/protocol.rs b/crates/tw-gateway/src/web_tools/protocol.rs
new file mode 100644
index 00000000..e541fa93
--- /dev/null
+++ b/crates/tw-gateway/src/web_tools/protocol.rs
@@ -0,0 +1,375 @@
+//! Hosted-tool adapters. Ordinary client tools are never executed by Core.
+
+use serde_json::{Value, json};
+use tw_api::{WebToolRequest, WebToolResult};
+use tw_dialect::ir::Dialect;
+use tw_types::Msg;
+
+use super::error;
+
+pub(crate) const SEARCH: &str = "thinkwatch_web_search";
+pub(crate) const OPEN: &str = "thinkwatch_open_page";
+pub(crate) const FIND: &str = "thinkwatch_find_in_page";
+
+#[derive(Clone, Debug)]
+pub(crate) struct Plan {
+ pub dialect: Dialect,
+ pub search: bool,
+ pub fetch: bool,
+ pub allowed: Vec,
+ pub blocked: Vec,
+ pub max_uses: Option,
+}
+
+pub(crate) fn recognize(dialect: Dialect, body: &Value) -> Result