Classify throttling 403 responses as upstream_http_error

.changeset/throttling-403-classification.md

@@ -0,0 +1,5 @@
+---
+"@executor-js/plugin-openapi": patch
+---
+
+Fail throttling 403 responses as `upstream_http_error` instead of `connection_rejected`. A 403 that names a rate limit (Google's `rateLimitExceeded` / `RESOURCE_EXHAUSTED`, GitHub's secondary rate limit with `Retry-After`, or `x-ratelimit-remaining: 0`) is a quota signal, not a bad credential: the previous classification told the agent to re-authenticate, which cannot clear a quota, and replaced the upstream reason with a re-auth prompt. Detection is structural (throttling headers or an exact `error`/`reason`/`status`/`code` value), so permission 403s keep their existing `connection_rejected` guidance.

packages/plugins/openapi/src/sdk/backing.ts

@@ -646,6 +646,53 @@ const recordUpstreamUnreachable = (integration: string, error: OpenApiInvocation
   );
 };
 
+// A 403 that means "you are going too fast" is a throttling signal, not a
+// credential rejection. Google reports per-user quotas as HTTP 403 with
+// `reason: "rateLimitExceeded"` or `status: "RESOURCE_EXHAUSTED"`; GitHub
+// sends a 403 "secondary rate limit" with Retry-After. Classifying those as
+// connection_rejected tells the agent to re-authenticate, which can never
+// clear a quota, and drops the upstream reason. Detection is structural:
+// throttling headers, or an exact error/reason/status/code field value. Prose
+// alone never classifies, so a permission 403 keeps its re-auth guidance.
+const RATE_LIMIT_VALUES = new Set([
+  "ratelimitexceeded",
+  "userratelimitexceeded",
+  "rate_limit_exceeded",
+  "resource_exhausted",
+  "too_many_requests",
+]);
+
+const hasRateLimitField = (body: unknown, depth: number): boolean => {
+  if (depth > 6) return false;
+  if (Array.isArray(body)) return body.some((item) => hasRateLimitField(item, depth + 1));
+  if (typeof body !== "object" || body === null) return false;
+  for (const [key, value] of Object.entries(body)) {
+    if (
+      (key === "error" || key === "reason" || key === "status" || key === "code") &&
+      typeof value === "string" &&
+      RATE_LIMIT_VALUES.has(value.toLowerCase())
+    ) {
+      return true;
+    }
+    if (typeof value === "object" && value !== null && hasRateLimitField(value, depth + 1)) {
+      return true;
+    }
+  }
+  return false;
+};
+
+const detectRateLimited = (result: {
+  readonly error: unknown;
+  readonly headers?: Record<string, string>;
+}): boolean => {
+  for (const [name, value] of Object.entries(result.headers ?? {})) {
+    const header = name.toLowerCase();
+    if (header === "retry-after" && value.trim().length > 0) return true;
+    if (header === "x-ratelimit-remaining" && value.trim() === "0") return true;
+  }
+  return hasRateLimitField(result.error, 0);
+};
+
 export const invokeOpenApiBackedTool = (input: {
   readonly ctx: PluginCtx<OpenapiStore>;
   readonly toolRow: { readonly integration: string; readonly name: string };
@@ -774,7 +821,10 @@ export const invokeOpenApiBackedTool = (input: {
     const result = invocation.result;
     const ok = result.status >= 200 && result.status < 300;
     if (!ok) {
-      if (result.status === 401 || result.status === 403) {
+      // Throttling 403s keep the upstream message and stay on the ordinary
+      // upstream_http_error path; re-auth cannot clear a quota.
+      const throttled = result.status === 403 && detectRateLimited(result);
+      if (!throttled && (result.status === 401 || result.status === 403)) {
         // A 403 naming a scope shortfall (RFC 6750 insufficient_scope,
         // Google's ACCESS_TOKEN_SCOPE_INSUFFICIENT) cannot be fixed by
         // re-running the same grant, so it gets its own code and recovery

packages/plugins/openapi/src/sdk/upstream-failures.test.ts

@@ -402,6 +402,69 @@ describe("OpenAPI upstream failure modes", () => {
     }),
   );
 
+  // A throttling 403 is not a credential failure. Re-authenticating cannot
+  // clear a quota, so it must keep the upstream reason instead of telling the
+  // agent to reconnect (the #2420 Gmail burst case).
+  it.effect("rate-limit 403 (Google RESOURCE_EXHAUSTED) is upstream_http_error", () =>
+    Effect.gen(function* () {
+      const server = yield* startScriptedServer(() => ({
+        status: 403,
+        headers: { "content-type": "application/json" },
+        body: JSON.stringify({
+          error: {
+            code: 403,
+            message: "Rate Limit Exceeded",
+            status: "RESOURCE_EXHAUSTED",
+            errors: [
+              {
+                message: "Rate Limit Exceeded",
+                domain: "usageLimits",
+                reason: "rateLimitExceeded",
+              },
+            ],
+          },
+        }),
+      }));
+      const { executor, address } = yield* buildExecutorForOpenApiServer(server);
+
+      const result = yield* executor.execute(address, {});
+
+      expect(result).toMatchObject({
+        ok: false,
+        error: {
+          code: "upstream_http_error",
+          status: 403,
+          message: expect.stringContaining("Rate Limit Exceeded"),
+        },
+      });
+    }),
+  );
+
+  it.effect("throttling 403 with Retry-After is upstream_http_error", () =>
+    Effect.gen(function* () {
+      const server = yield* startScriptedServer(() => ({
+        status: 403,
+        headers: { "content-type": "application/json", "retry-after": "60" },
+        body: JSON.stringify({
+          message: "You have exceeded a secondary rate limit. Please wait before retrying.",
+          documentation_url: "https://docs.github.com/rest/using-the-rest-api/rate-limits",
+        }),
+      }));
+      const { executor, address } = yield* buildExecutorForOpenApiServer(server);
+
+      const result = yield* executor.execute(address, {});
+
+      expect(result).toMatchObject({
+        ok: false,
+        error: {
+          code: "upstream_http_error",
+          status: 403,
+          message: expect.stringContaining("secondary rate limit"),
+        },
+      });
+    }),
+  );
+
   it.effect("upstream returns malformed JSON despite Content-Type: application/json", () =>
     Effect.gen(function* () {
       const server = yield* startScriptedServer(() => ({