Deliver SDK spec and replay manifests through Tailscale exits

.github/workflows/replay-spec-changes.yml

@@ -29,18 +29,18 @@ concurrency:
   group: spec-changes
   cancel-in-progress: false
 
-env:
-  SDK_BOT_URL: https://sdk-automation-bot.workos.tools
-
 jobs:
   replay:
     runs-on: ubuntu-latest
+    outputs:
+      deliver: ${{ steps.prepare.outputs.deliver }}
     permissions:
       contents: read
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           fetch-depth: 0
+          persist-credentials: false
 
       - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
         with:
@@ -82,15 +82,83 @@ jobs:
             exit 1
           fi
 
-      - name: Rebuild and idempotently push manifests
+      - name: Prepare confirmed replay delivery
+        id: prepare
         if: inputs.mode == 'push'
         env:
           TARGET_SHAS: ${{ inputs.target_shas }}
-          SPEC_CHANGES_SECRET: ${{ secrets.SPEC_CHANGES_SECRET }}
         run: |
+          set -euo pipefail
           node scripts/replay-spec-changes.mjs \
             --shas "$TARGET_SHAS" \
             --repository "$GITHUB_REPOSITORY" \
-            --bot-url "$SDK_BOT_URL" \
-            --push \
+            --prepare-delivery "$RUNNER_TEMP/replay-delivery" \
             --confirm REPLAY_SPEC_CHANGES
+          if [ "$(jq 'length' "$RUNNER_TEMP/replay-delivery/index.json")" -gt 0 ]; then
+            echo "deliver=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Hand replay data to isolated delivery job
+        if: steps.prepare.outputs.deliver == 'true'
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: replay-spec-changes-manifests
+          path: ${{ runner.temp }}/replay-delivery/*.json
+          if-no-files-found: error
+          retention-days: 1
+
+  deliver:
+    needs: replay
+    if: >-
+      needs.replay.outputs.deliver == 'true' && inputs.mode == 'push' &&
+      github.repository == 'workos/openapi-spec' &&
+      github.ref == 'refs/heads/main' &&
+      github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      contents: read
+      id-token: write
+    env:
+      REPLAY_MODE: ${{ inputs.mode }}
+      REPLAY_CONFIRMATION: REPLAY_SPEC_CHANGES
+      TAILSCALE_EXIT_NODE: ${{ vars.SDK_SPEC_CHANGES_EXIT_NODE }}
+      TAILSCALE_CLIENT_ID: ${{ vars.SDK_SPEC_REPLAY_CLIENT_ID }}
+      TAILSCALE_AUDIENCE: ${{ vars.SDK_SPEC_REPLAY_AUDIENCE }}
+    steps:
+      # The build job has no OIDC or signing secret. Delivery uses only this
+      # checked-in, dependency-free script; never execute artifact contents.
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+          sparse-checkout: scripts/push-spec-changes.mjs
+          sparse-checkout-cone-mode: false
+
+      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: '24'
+
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: replay-spec-changes-manifests
+          path: ${{ runner.temp }}/spec-changes
+
+      - name: Require exact workflow and public identity configuration
+        run: node scripts/push-spec-changes.mjs --check-config
+
+      - name: Join Tailscale as an ephemeral SDK delivery runner
+        uses: tailscale/github-action@064b10558093f38325e838ddea35486bdc6b1d5c # v4
+        with:
+          oauth-client-id: ${{ env.TAILSCALE_CLIENT_ID }}
+          audience: ${{ env.TAILSCALE_AUDIENCE }}
+          tags: tag:ci-sdk-spec-changes
+          version: '1.104.4'
+          use-cache: 'false'
+          hostname: sdk-replay-${{ github.run_id }}-${{ github.run_attempt }}
+          tailscaled-args: --tun=userspace-networking --socks5-server=127.0.0.1:1055
+          args: --exit-node=${{ env.TAILSCALE_EXIT_NODE }} --exit-node-allow-lan-access=false --accept-routes=false --accept-dns=false
+
+      - name: Verify exit NAT and deliver exact HMAC-signed bytes
+        env:
+          SPEC_CHANGES_SECRET: ${{ secrets.SPEC_CHANGES_SECRET }}
+        run: node scripts/push-spec-changes.mjs --replay-dir "$RUNNER_TEMP/spec-changes"

.github/workflows/spec-changes.yml

@@ -25,18 +25,18 @@ concurrency:
   group: spec-changes
   cancel-in-progress: false
 
-env:
-  SDK_BOT_URL: https://sdk-automation-bot.workos.tools
-
 jobs:
   record:
     runs-on: ubuntu-latest
+    outputs:
+      deliver: ${{ steps.manifest.outputs.deliver }}
     permissions:
       contents: read
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           fetch-depth: 0
+          persist-credentials: false
 
       - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
         with:
@@ -104,10 +104,10 @@ jobs:
           npx oagen parse --spec /tmp/previous-open-api-spec.yaml > /tmp/previous-ir.json
           npx oagen parse --spec spec/open-api-spec.yaml > /tmp/current-ir.json
 
-      - name: Push changed-services manifest to the SDK bot
+      - name: Build changed-services manifest
+        id: manifest
         if: steps.parent.outputs.available == 'true'
         env:
-          SPEC_CHANGES_SECRET: ${{ secrets.SPEC_CHANGES_SECRET }}
           HEAD_COMMIT_MSG: ${{ github.event.head_commit.message }}
           REPO: ${{ github.repository }}
         run: |
@@ -130,30 +130,72 @@ jobs:
             ARGS+=(--pr-number "$PR_NUM" --pr-url "https://github.com/$REPO/pull/$PR_NUM")
           fi
 
-          # build-spec-changes.mjs writes the manifest to stdout when --output is
-          # omitted. Capture it, then HMAC-sign the exact bytes we POST.
-          MANIFEST=$(node scripts/build-spec-changes.mjs "${ARGS[@]}")
-          echo "Manifest:"
-          echo "$MANIFEST" | jq .
-
-          if [ "$(echo "$MANIFEST" | jq '.changedServices | length')" -eq 0 ]; then
+          node scripts/build-spec-changes.mjs "${ARGS[@]}" --output /tmp/spec-changes/manifest.json
+          if [ "$(jq '.changedServices | length' /tmp/spec-changes/manifest.json)" -gt 0 ]; then
+            echo "deliver=true" >> "$GITHUB_OUTPUT"
+          else
             echo "No service changes; nothing to push."
-            exit 0
           fi
 
-          SIG="sha256=$(printf '%s' "$MANIFEST" | openssl dgst -sha256 -hmac "$SPEC_CHANGES_SECRET" | sed 's/^.*= //')"
-          for attempt in 1 2 3; do
-            if curl -sS -X POST "$SDK_BOT_URL/internal/spec-changes" \
-                 -H "Content-Type: application/json" \
-                 -H "X-Spec-Changes-Signature: $SIG" \
-                 --data-raw "$MANIFEST" \
-                 --fail-with-body; then
-              echo ""
-              echo "Pushed manifest for ${{ github.sha }}."
-              exit 0
-            fi
-            echo "push attempt ${attempt} failed; retrying in 5s..."
-            sleep 5
-          done
-          echo "::error::could not push spec-changes manifest after 3 attempts"
-          exit 1
+      - name: Hand manifest to isolated delivery job
+        if: steps.manifest.outputs.deliver == 'true'
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: spec-changes-manifest
+          path: /tmp/spec-changes/manifest.json
+          if-no-files-found: error
+          retention-days: 1
+
+  deliver:
+    needs: record
+    if: >-
+      needs.record.outputs.deliver == 'true' &&
+      github.repository == 'workos/openapi-spec' &&
+      github.ref == 'refs/heads/main' &&
+      (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+      id-token: write
+    env:
+      TAILSCALE_EXIT_NODE: ${{ vars.SDK_SPEC_CHANGES_EXIT_NODE }}
+      TAILSCALE_CLIENT_ID: ${{ vars[github.event_name == 'push' && 'SDK_SPEC_CHANGES_PUSH_CLIENT_ID' || 'SDK_SPEC_CHANGES_DISPATCH_CLIENT_ID'] }}
+      TAILSCALE_AUDIENCE: ${{ vars[github.event_name == 'push' && 'SDK_SPEC_CHANGES_PUSH_AUDIENCE' || 'SDK_SPEC_CHANGES_DISPATCH_AUDIENCE'] }}
+    steps:
+      # The build job has no OIDC or signing secret. Delivery uses only this
+      # checked-in, dependency-free script; never execute artifact contents.
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+          sparse-checkout: scripts/push-spec-changes.mjs
+          sparse-checkout-cone-mode: false
+
+      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: '24'
+
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: spec-changes-manifest
+          path: ${{ runner.temp }}/spec-changes
+
+      - name: Require exact workflow and public identity configuration
+        run: node scripts/push-spec-changes.mjs --check-config
+
+      - name: Join Tailscale as an ephemeral SDK delivery runner
+        uses: tailscale/github-action@064b10558093f38325e838ddea35486bdc6b1d5c # v4
+        with:
+          oauth-client-id: ${{ env.TAILSCALE_CLIENT_ID }}
+          audience: ${{ env.TAILSCALE_AUDIENCE }}
+          tags: tag:ci-sdk-spec-changes
+          version: '1.104.4'
+          use-cache: 'false'
+          hostname: sdk-spec-${{ github.run_id }}-${{ github.run_attempt }}
+          tailscaled-args: --tun=userspace-networking --socks5-server=127.0.0.1:1055
+          args: --exit-node=${{ env.TAILSCALE_EXIT_NODE }} --exit-node-allow-lan-access=false --accept-routes=false --accept-dns=false
+
+      - name: Verify exit NAT and deliver exact HMAC-signed bytes
+        env:
+          SPEC_CHANGES_SECRET: ${{ secrets.SPEC_CHANGES_SECRET }}
+        run: node scripts/push-spec-changes.mjs "$RUNNER_TEMP/spec-changes/manifest.json"

.github/workflows/validate-sdks.yml

@@ -28,6 +28,9 @@ jobs:
       - name: Test sync conflict resolution
         run: node --test scripts/__tests__/resolve-sync-conflicts.spec.mjs
 
+      - name: Test signed Tailscale delivery
+        run: node --test scripts/__tests__/push-spec-changes.spec.mjs scripts/__tests__/replay-spec-changes.spec.mjs
+
       - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
         id: filter
         with:

docs/spec-change-replay.md

@@ -34,13 +34,27 @@ node scripts/replay-spec-changes.mjs \
 
 After reviewing a dry run, launch the workflow again with the same explicit
 SHAs and set **mode** to **push**. Push mode is accepted only from the default
-branch. The workflow signs the exact manifest bytes and POSTs them to the
-existing SDK bot endpoint.
+branch (`main` for the configured trust). The build job prepares exact manifest
+bytes plus an ordered index as a same-run artifact, without OIDC permission or
+the signing secret. A separate delivery job uses the replay-only OIDC identity
+and the shared fail-closed Tailscale transport, signs those exact bytes and POSTs
+them to the existing SDK bot endpoint.
 
 The SDK bot upserts on the target SHA, so repeating the same replay is
 idempotent. A target that produces no changed services is shown but not POSTed.
 Every target is validated and rebuilt before the first POST, and the workflow
 serializes with the normal spec-change producer to avoid delivery races.
 
 Do not put the signing secret in the SHA input or logs. The workflow exposes
-`SPEC_CHANGES_SECRET` only to the confirmed push step.
+`SPEC_CHANGES_SECRET` only to the isolated delivery step. Configure the dedicated
+`SDK_SPEC_REPLAY_CLIENT_ID` and `SDK_SPEC_REPLAY_AUDIENCE` public variables, plus
+the shared reviewed exit, after the infra trust/policy rollout described in
+[delivery setup](spec-changes-delivery.md).
+
+The script's `--prepare-delivery <new-directory> --confirm REPLAY_SPEC_CHANGES`
+mode writes delivery data without posting or reading a signing secret. The
+legacy `--push --confirm REPLAY_SPEC_CHANGES` option has no direct `fetch` path:
+it invokes the shared transport and requires the configured, confirmed main
+replay workflow context and Tailscale runner. Use the Actions workflow to push;
+local dry-run needs no such identity. No production replay was run while
+preparing this change.

docs/spec-changes-delivery.md

@@ -0,0 +1,76 @@
+# Spec-change delivery through Tailscale
+
+`spec-changes.yml` builds a manifest without a signing secret or OIDC permission.
+Confirmed replay push mode similarly prepares an ordered manifest artifact.
+A separate delivery job downloads that same-run data artifact, obtains GitHub
+OIDC identity, joins Tailscale ephemerally as `tag:ci-sdk-spec-changes`, and signs
+the exact manifest file bytes with the existing `SPEC_CHANGES_SECRET`.
+
+The transport is a userspace SOCKS5 listener on `127.0.0.1:1055` through an
+explicit internal exit, then TLS to `sdk-automation-bot.workos.tools`. The
+Cloudflare Worker remains a public endpoint. It needs no deployment change:
+the existing WAF permits the internal exit NAT addresses and the Worker still
+authenticates `X-Spec-Changes-Signature` with HMAC. This change does not open a
+new public exception or rely on held infra PR #2126.
+
+Delivery refuses missing identity configuration, unsupported workflow/ref/event,
+non-userspace Tailscale, a missing/offline/wrong/untagged exit, an unapproved NAT,
+proxy failures and redirects. All retries use the same fixed host/path through
+the proxy. Before any HTTP request, delivery pins the host's resolved IPv4 with
+curl `--resolve` and installs a runner-local iptables OUTPUT reject for direct
+TCP/443 to that exact IP. This also blocks tailscaled's host-network fallback
+if its exit route disappears. Missing sudo/iptables permission stops delivery;
+the rule disappears with the hosted VM. curlrc and `NO_PROXY` cannot select a
+direct route. Local DNS plus
+`--ipv4`/`socks5://` sends a numeric IPv4 to SOCKS5 without exit DNS permission.
+Only the delivery script gets the signing secret; no npm install runs there.
+The signature travels to curl over stdin, not its command line, and response
+bodies and diagnostics are kept out of logs.
+
+Apply the reviewed `workos/infra` trust/policy PR **before** merging this one.
+Three workflow/event-specific identities require exact repository/owner numeric IDs,
+main ref, workflow path and GitHub-hosted runner. Each has `auth_keys` only.
+Their grant permits general IPv4 Internet TCP/443 through tagged internal exits;
+it is not a hostname/path ACL. Private/tailnet/link-local/loopback destinations,
+IPv6 and other ports are denied. Repository main workflow review remains a
+trust boundary; trusted main code can change this workflow.
+
+Set these nonsecret Actions repository variables from the infra output
+`sdk_spec_changes_wif` and a reviewed exit:
+
+| Variable | Source |
+| --- | --- |
+| `SDK_SPEC_CHANGES_PUSH_CLIENT_ID` | output `["push"].client_id` |
+| `SDK_SPEC_CHANGES_PUSH_AUDIENCE` | output `["push"].audience` |
+| `SDK_SPEC_CHANGES_DISPATCH_CLIENT_ID` | output `["workflow_dispatch"].client_id` |
+| `SDK_SPEC_CHANGES_DISPATCH_AUDIENCE` | output `["workflow_dispatch"].audience` |
+| `SDK_SPEC_REPLAY_CLIENT_ID` | output `["replay"].client_id` |
+| `SDK_SPEC_REPLAY_AUDIENCE` | output `["replay"].audience` |
+| `SDK_SPEC_CHANGES_EXIT_NODE` | approved, online internal exit's Tailscale IPv4 address |
+
+Keep the existing `SPEC_CHANGES_SECRET` synchronized with the bot. No new static
+Tailscale secret is needed. The action's pinned version uses memory-only state,
+requests an ephemeral node and logs out after the job. If the configured exit
+is replaced, verify its tag/approval/NAT and update the variable; there is no
+automatic selection of an arbitrary or external exit.
+
+The complete human bootstrap, apply, rollback and verification procedure is in
+[the infra runbook](https://github.com/workos/infra/blob/codex/github-actions-sdk-tailscale/docs/runbooks/tailscale-control/github-actions-sdk.md).
+After landing, manually dispatch on main (workflow-only changes do not match the
+spec path filter), then verify the next real spec merge independently. Confirm
+the bot's manifest row and ephemeral node removal. Preserve application HMAC
+verification; never fall back to direct public delivery on failure.
+
+`node --test scripts/__tests__/push-spec-changes.spec.mjs scripts/__tests__/replay-spec-changes.spec.mjs` checks byte-exact
+signing, identity/config/status denial cases, proxy-only commands, NAT checks,
+redirects, retries and real curl failure with a closed loopback proxy. SDK
+validation CI runs it even when there is no spec change. Live OIDC, tailnet
+policy enforcement, actual exit/NAT/Worker access, Linux firewall enforcement
+and cleanup need the human
+rollout checks. The historical replay workflow has its own exact manual-dispatch
+trust and uses the same isolated transport. It preserves dry-run, main/default-branch and
+explicit replay confirmation, validates every target before posting, and sends
+changed manifests sequentially in the supplied order. See [replay instructions](spec-change-replay.md).
+
+The two Socket upstream argument-flow warnings have been traced without
+suppression; see the [pinned action review](tailscale-action-review.md).