diff --git a/.changeset/throttling-403-classification.md b/.changeset/throttling-403-classification.md new file mode 100644 index 0000000000..a6fa21a3bb --- /dev/null +++ b/.changeset/throttling-403-classification.md @@ -0,0 +1,5 @@ +--- +"@executor-js/plugin-openapi": patch +--- + +Fail throttling 403 responses as `upstream_http_error` instead of `connection_rejected`. A 403 that names a rate limit (Google's `rateLimitExceeded` / `RESOURCE_EXHAUSTED`, GitHub's secondary rate limit with `Retry-After`, or `x-ratelimit-remaining: 0`) is a quota signal, not a bad credential: the previous classification told the agent to re-authenticate, which cannot clear a quota, and replaced the upstream reason with a re-auth prompt. Detection is structural (throttling headers or an exact `error`/`reason`/`status`/`code` value), so permission 403s keep their existing `connection_rejected` guidance. diff --git a/packages/plugins/openapi/src/sdk/backing.ts b/packages/plugins/openapi/src/sdk/backing.ts index 82cd6dcdba..2f49c11f7e 100644 --- a/packages/plugins/openapi/src/sdk/backing.ts +++ b/packages/plugins/openapi/src/sdk/backing.ts @@ -646,6 +646,53 @@ const recordUpstreamUnreachable = (integration: string, error: OpenApiInvocation ); }; +// A 403 that means "you are going too fast" is a throttling signal, not a +// credential rejection. Google reports per-user quotas as HTTP 403 with +// `reason: "rateLimitExceeded"` or `status: "RESOURCE_EXHAUSTED"`; GitHub +// sends a 403 "secondary rate limit" with Retry-After. Classifying those as +// connection_rejected tells the agent to re-authenticate, which can never +// clear a quota, and drops the upstream reason. Detection is structural: +// throttling headers, or an exact error/reason/status/code field value. Prose +// alone never classifies, so a permission 403 keeps its re-auth guidance. +const RATE_LIMIT_VALUES = new Set([ + "ratelimitexceeded", + "userratelimitexceeded", + "rate_limit_exceeded", + "resource_exhausted", + "too_many_requests", +]); + +const hasRateLimitField = (body: unknown, depth: number): boolean => { + if (depth > 6) return false; + if (Array.isArray(body)) return body.some((item) => hasRateLimitField(item, depth + 1)); + if (typeof body !== "object" || body === null) return false; + for (const [key, value] of Object.entries(body)) { + if ( + (key === "error" || key === "reason" || key === "status" || key === "code") && + typeof value === "string" && + RATE_LIMIT_VALUES.has(value.toLowerCase()) + ) { + return true; + } + if (typeof value === "object" && value !== null && hasRateLimitField(value, depth + 1)) { + return true; + } + } + return false; +}; + +const detectRateLimited = (result: { + readonly error: unknown; + readonly headers?: Record; +}): boolean => { + for (const [name, value] of Object.entries(result.headers ?? {})) { + const header = name.toLowerCase(); + if (header === "retry-after" && value.trim().length > 0) return true; + if (header === "x-ratelimit-remaining" && value.trim() === "0") return true; + } + return hasRateLimitField(result.error, 0); +}; + export const invokeOpenApiBackedTool = (input: { readonly ctx: PluginCtx; readonly toolRow: { readonly integration: string; readonly name: string }; @@ -774,7 +821,10 @@ export const invokeOpenApiBackedTool = (input: { const result = invocation.result; const ok = result.status >= 200 && result.status < 300; if (!ok) { - if (result.status === 401 || result.status === 403) { + // Throttling 403s keep the upstream message and stay on the ordinary + // upstream_http_error path; re-auth cannot clear a quota. + const throttled = result.status === 403 && detectRateLimited(result); + if (!throttled && (result.status === 401 || result.status === 403)) { // A 403 naming a scope shortfall (RFC 6750 insufficient_scope, // Google's ACCESS_TOKEN_SCOPE_INSUFFICIENT) cannot be fixed by // re-running the same grant, so it gets its own code and recovery diff --git a/packages/plugins/openapi/src/sdk/upstream-failures.test.ts b/packages/plugins/openapi/src/sdk/upstream-failures.test.ts index 1f52b30c50..d4d1e45880 100644 --- a/packages/plugins/openapi/src/sdk/upstream-failures.test.ts +++ b/packages/plugins/openapi/src/sdk/upstream-failures.test.ts @@ -402,6 +402,69 @@ describe("OpenAPI upstream failure modes", () => { }), ); + // A throttling 403 is not a credential failure. Re-authenticating cannot + // clear a quota, so it must keep the upstream reason instead of telling the + // agent to reconnect (the #2420 Gmail burst case). + it.effect("rate-limit 403 (Google RESOURCE_EXHAUSTED) is upstream_http_error", () => + Effect.gen(function* () { + const server = yield* startScriptedServer(() => ({ + status: 403, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + error: { + code: 403, + message: "Rate Limit Exceeded", + status: "RESOURCE_EXHAUSTED", + errors: [ + { + message: "Rate Limit Exceeded", + domain: "usageLimits", + reason: "rateLimitExceeded", + }, + ], + }, + }), + })); + const { executor, address } = yield* buildExecutorForOpenApiServer(server); + + const result = yield* executor.execute(address, {}); + + expect(result).toMatchObject({ + ok: false, + error: { + code: "upstream_http_error", + status: 403, + message: expect.stringContaining("Rate Limit Exceeded"), + }, + }); + }), + ); + + it.effect("throttling 403 with Retry-After is upstream_http_error", () => + Effect.gen(function* () { + const server = yield* startScriptedServer(() => ({ + status: 403, + headers: { "content-type": "application/json", "retry-after": "60" }, + body: JSON.stringify({ + message: "You have exceeded a secondary rate limit. Please wait before retrying.", + documentation_url: "https://docs.github.com/rest/using-the-rest-api/rate-limits", + }), + })); + const { executor, address } = yield* buildExecutorForOpenApiServer(server); + + const result = yield* executor.execute(address, {}); + + expect(result).toMatchObject({ + ok: false, + error: { + code: "upstream_http_error", + status: 403, + message: expect.stringContaining("secondary rate limit"), + }, + }); + }), + ); + it.effect("upstream returns malformed JSON despite Content-Type: application/json", () => Effect.gen(function* () { const server = yield* startScriptedServer(() => ({