diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c7de7220..97d0a6e7f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,9 +146,7 @@ jobs: run: pnpm --filter vite-task-tools exec playwright install-deps chromium - name: Run ignored tests - run: | - pnpm --filter vite-task-tools test - $TEST_CARGO_CMD test --target $TEST_BUILD_TARGET -- --ignored + run: $TEST_CARGO_CMD test --target $TEST_BUILD_TARGET -- --ignored # Running x64 binaries under Rosetta makes this the slowest job, so pull # requests run it only with the `ci: full` label. @@ -300,9 +298,7 @@ jobs: - name: Run ignored tests if: matrix.mode == 'ignored' - run: | - pnpm --filter vite-task-tools test - cargo-nextest nextest run --archive-file windows-tests.tar.zst --workspace-remap . --run-ignored ignored-only + run: cargo-nextest nextest run --archive-file windows-tests.tar.zst --workspace-remap . --run-ignored ignored-only # Building without a Rust cache inside the Alpine container makes this one of # the slowest jobs, so pull requests run it only with the `ci: full` label. @@ -360,9 +356,7 @@ jobs: - uses: oxc-project/setup-node@f46a72f95efdc55273fcd042d61c84e723b2892c # v1.4.1 - name: Run ignored tests - run: | - pnpm --filter vite-task-tools test - cargo test -- --ignored + run: cargo test -- --ignored fmt: name: Format and Check Deps diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots.toml b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots.toml index 0647a836e..10fed2f25 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots.toml +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots.toml @@ -1,16 +1,23 @@ # Cases that use the remote cache backend are ignored because it runs on # Node.js. Windows is skipped because the PTY launcher cannot execute pnpm -# command shims. Whether an upload to the backend is still running when the -# tasks finish depends on timing, so steps that upload hide the message about -# pending uploads. The cases that test it stall the uploads instead, with +# command shims. Each of these cases starts its own backend first, runs +# commands against it with `remote-cache-server run`, and stops it last. +# Whether an upload to the backend is still running when the tasks finish +# depends on timing, so steps that upload hide the message about pending +# uploads. The cases that test it stall the uploads instead, with # stalled-remote-cache in front of the backend. [[e2e]] name = "read_write" cfg = "not(windows)" ignore = true steps = [ + [ + "remote-cache-server", + "start", + ], { argv = [ "remote-cache-server", + "run", "vt", "run", "build", @@ -26,6 +33,7 @@ steps = [ ], comment = "The fetch finds no entry. The new execution is uploaded with one store request, and vt run waits for it after the task finishes." }, { argv = [ "remote-cache-server", + "run", "vt", "run", "build", @@ -35,6 +43,10 @@ steps = [ "read-write", ], ], comment = "A local hit makes no requests." }, + [ + "remote-cache-server", + "stop", + ], ] [[e2e]] @@ -42,8 +54,13 @@ name = "read" cfg = "not(windows)" ignore = true steps = [ + [ + "remote-cache-server", + "start", + ], { argv = [ "remote-cache-server", + "run", "vt", "run", "build", @@ -65,10 +82,15 @@ steps = [ ], { argv = [ "remote-cache-server", + "run", "vt", "run", "build", ], comment = "An endpoint without a mode selects read. The task reruns without uploading." }, + [ + "remote-cache-server", + "stop", + ], ] [[e2e]] @@ -76,8 +98,13 @@ name = "restore" cfg = "not(windows)" ignore = true steps = [ + [ + "remote-cache-server", + "start", + ], { argv = [ "remote-cache-server", + "run", "vt", "run", "build", @@ -104,6 +131,7 @@ steps = [ ], { argv = [ "remote-cache-server", + "run", "vt", "run", "build", @@ -120,6 +148,7 @@ steps = [ ], comment = "The outputs are restored." }, { argv = [ "remote-cache-server", + "run", "vt", "run", "build", @@ -137,10 +166,15 @@ steps = [ ], { argv = [ "remote-cache-server", + "run", "vt", "run", "build", ], comment = "The exact entry fails validation, so the archive isn't downloaded." }, + [ + "remote-cache-server", + "stop", + ], ] [[e2e]] @@ -148,8 +182,13 @@ name = "local_and_remote_hits" cfg = "not(windows)" ignore = true steps = [ + [ + "remote-cache-server", + "start", + ], { argv = [ "remote-cache-server", + "run", "vt", "run", "build", @@ -175,6 +214,7 @@ steps = [ ], comment = "Without an endpoint, check is cached only locally." }, { argv = [ "remote-cache-server", + "run", "vt", "run", "all", @@ -184,6 +224,10 @@ steps = [ "run", "--last-details", ], comment = "The details show which hit came from the remote cache." }, + [ + "remote-cache-server", + "stop", + ], ] [[e2e]] @@ -191,8 +235,13 @@ name = "fallback" cfg = "not(windows)" ignore = true steps = [ + [ + "remote-cache-server", + "start", + ], { argv = [ "remote-cache-server", + "run", "vt", "run", "build", @@ -220,10 +269,15 @@ steps = [ ], { argv = [ "remote-cache-server", + "run", "vt", "run", "build", ], comment = "The entry stored for this task has a different key. The miss reason compares it with the current key." }, + [ + "remote-cache-server", + "stop", + ], ] [[e2e]] @@ -231,8 +285,13 @@ name = "corrupt_archive" cfg = "not(windows)" ignore = true steps = [ + [ + "remote-cache-server", + "start", + ], { argv = [ "remote-cache-server", + "run", "vt", "run", "build", @@ -259,6 +318,7 @@ steps = [ ], { argv = [ "remote-cache-server", + "run", "vt", "run", "build", @@ -276,6 +336,10 @@ steps = [ ".tmp", "--recursive", ], comment = "The corrupt download was removed." }, + [ + "remote-cache-server", + "stop", + ], ] [[e2e]] @@ -283,8 +347,13 @@ name = "restore_failure" cfg = "not(windows)" ignore = true steps = [ + [ + "remote-cache-server", + "start", + ], { argv = [ "remote-cache-server", + "run", "vt", "run", "build", @@ -317,6 +386,7 @@ steps = [ ], comment = "A file where the output directory goes makes restoring fail." }, { argv = [ "remote-cache-server", + "run", "vt", "run", "build", @@ -341,10 +411,15 @@ steps = [ ], { argv = [ "remote-cache-server", + "run", "vt", "run", "build", ], comment = "Nothing was cached locally, so the remote entry is fetched and restored again." }, + [ + "remote-cache-server", + "stop", + ], ] [[e2e]] @@ -352,8 +427,13 @@ name = "pending_uploads" cfg = "not(windows)" ignore = true steps = [ + [ + "remote-cache-server", + "start", + ], { argv = [ "remote-cache-server", + "run", "vtt", "stalled-remote-cache", "--stall", @@ -375,6 +455,10 @@ steps = [ "run", "--last-details", ], comment = "Both uploads were cancelled." }, + [ + "remote-cache-server", + "stop", + ], ] [[e2e]] @@ -565,8 +649,13 @@ name = "ctrl_c_during_upload" cfg = "not(windows)" ignore = true steps = [ + [ + "remote-cache-server", + "start", + ], { argv = [ "remote-cache-server", + "run", "vtt", "stalled-remote-cache", "--stall", @@ -593,6 +682,10 @@ steps = [ "run", "build", ], comment = "The entry is still in the local cache." }, + [ + "remote-cache-server", + "stop", + ], ] [[e2e]] @@ -600,8 +693,13 @@ name = "fast_fail_during_upload" cfg = "not(windows)" ignore = true steps = [ + [ + "remote-cache-server", + "start", + ], { argv = [ "remote-cache-server", + "run", "vtt", "stalled-remote-cache", "--stall", @@ -618,6 +716,10 @@ steps = [ { "expect-milestone" = "uploads-pending" }, { "write-key" = "ctrl-c" }, ], comment = "The proxy never forwards the upload. fail-after-build exits after build finishes, which doesn't cancel build's upload, so vt run waits for it until Ctrl-C." }, + [ + "remote-cache-server", + "stop", + ], ] [[e2e]] @@ -625,8 +727,13 @@ name = "hide_pending_uploads" cfg = "not(windows)" ignore = true steps = [ + [ + "remote-cache-server", + "start", + ], { argv = [ "remote-cache-server", + "run", "vtt", "stalled-remote-cache", "--stall", @@ -647,4 +754,8 @@ steps = [ { "expect-milestone" = "uploads-pending" }, { "write-key" = "ctrl-c" }, ], comment = "The proxy never forwards the upload. vt run waits for it until Ctrl-C, without the message about pending uploads." }, + [ + "remote-cache-server", + "stop", + ], ] diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/corrupt_archive.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/corrupt_archive.md index e6f510756..610ff5807 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/corrupt_archive.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/corrupt_archive.md @@ -1,6 +1,11 @@ # corrupt_archive -## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server vt run build` +## `remote-cache-server start` + +``` +``` + +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run vt run build` ``` $ vtt write-file dist/output.txt built @@ -21,7 +26,7 @@ Overwrite the stored archive. ``` ``` -## `remote-cache-server vt run build` +## `remote-cache-server run vt run build` The downloaded archive doesn't decode, so the task reruns. @@ -59,3 +64,8 @@ The corrupt download was removed. ``` ``` + +## `remote-cache-server stop` + +``` +``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/ctrl_c_during_upload.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/ctrl_c_during_upload.md index 71aa1b51f..7b58cfb0a 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/ctrl_c_during_upload.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/ctrl_c_during_upload.md @@ -1,6 +1,11 @@ # ctrl_c_during_upload -## `VP_REMOTE_CACHE=read-write remote-cache-server vtt stalled-remote-cache --stall /store vt run build` +## `remote-cache-server start` + +``` +``` + +## `VP_REMOTE_CACHE=read-write remote-cache-server run vtt stalled-remote-cache --stall /store vt run build` The proxy forwards the fetch to the backend, which has no entry, but never forwards the upload. Ctrl-C cancels it while vt run waits. @@ -54,3 +59,8 @@ $ vtt write-file dist/output.txt built ◉ cache hit, replaying --- vt run: cache hit. ``` + +## `remote-cache-server stop` + +``` +``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/fallback.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/fallback.md index d2f8cffcd..46d51b4d5 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/fallback.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/fallback.md @@ -1,6 +1,11 @@ # fallback -## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server vt run build` +## `remote-cache-server start` + +``` +``` + +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run vt run build` ``` $ vtt write-file dist/output.txt built @@ -19,7 +24,7 @@ $ vtt write-file dist/output.txt built ``` ``` -## `remote-cache-server vt run build` +## `remote-cache-server run vt run build` The entry stored for this task has a different key. The miss reason compares it with the current key. @@ -28,3 +33,8 @@ $ vtt write-file dist/output.txt rebuilt ○ cache miss: args changed, executing [remote-cache] POST /fetch 200 fallback ``` + +## `remote-cache-server stop` + +``` +``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/fast_fail_during_upload.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/fast_fail_during_upload.md index bae30e8f5..a9ae14f86 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/fast_fail_during_upload.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/fast_fail_during_upload.md @@ -1,6 +1,11 @@ # fast_fail_during_upload -## `VP_REMOTE_CACHE=read-write remote-cache-server vtt stalled-remote-cache --stall /store vt run fail-after-build` +## `remote-cache-server start` + +``` +``` + +## `VP_REMOTE_CACHE=read-write remote-cache-server run vtt stalled-remote-cache --stall /store vt run fail-after-build` The proxy never forwards the upload. fail-after-build exits after build finishes, which doesn't cancel build's upload, so vt run waits for it until Ctrl-C. @@ -28,3 +33,8 @@ Waiting for 1 remote cache upload to finish (Ctrl-C to cancel)... vt run: 0/2 cache hit (0%), 1 failed. remote-cache#build not uploaded to the remote cache: interrupted. (Run `vt run --last-details` for full details) [remote-cache] POST /fetch 404 ``` + +## `remote-cache-server stop` + +``` +``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/hide_pending_uploads.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/hide_pending_uploads.md index e9b913dd3..77394fc6c 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/hide_pending_uploads.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/hide_pending_uploads.md @@ -1,6 +1,11 @@ # hide_pending_uploads -## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server vtt stalled-remote-cache --stall /store vt run build` +## `remote-cache-server start` + +``` +``` + +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run vtt stalled-remote-cache --stall /store vt run build` The proxy never forwards the upload. vt run waits for it until Ctrl-C, without the message about pending uploads. @@ -19,3 +24,8 @@ $ vtt write-file dist/output.txt built vt run: remote-cache#build not uploaded to the remote cache: interrupted. (Run `vt run --last-details` for full details) [remote-cache] POST /fetch 404 ``` + +## `remote-cache-server stop` + +``` +``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/local_and_remote_hits.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/local_and_remote_hits.md index ce193c69c..a514d151c 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/local_and_remote_hits.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/local_and_remote_hits.md @@ -1,6 +1,11 @@ # local_and_remote_hits -## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server vt run build` +## `remote-cache-server start` + +``` +``` + +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run vt run build` ``` $ vtt write-file dist/output.txt built @@ -23,7 +28,7 @@ $ vtt print checked checked ``` -## `remote-cache-server vt run all` +## `remote-cache-server run vt run all` build is a remote hit, and check is a local hit. The summary counts the remote hit. @@ -61,3 +66,8 @@ Task Details: → Cache hit - output replayed - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ ``` + +## `remote-cache-server stop` + +``` +``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/pending_uploads.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/pending_uploads.md index 491414e5b..fe68b6a0c 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/pending_uploads.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/pending_uploads.md @@ -1,6 +1,11 @@ # pending_uploads -## `VP_REMOTE_CACHE=read-write remote-cache-server vtt stalled-remote-cache --stall /store vt run all` +## `remote-cache-server start` + +``` +``` + +## `VP_REMOTE_CACHE=read-write remote-cache-server run vtt stalled-remote-cache --stall /store vt run all` The proxy never forwards the uploads. check doesn't wait for build's upload, so both are still running when check finishes, and vt run waits for them until Ctrl-C. @@ -54,3 +59,8 @@ Task Details: ⚠ Not uploaded to the remote cache: interrupted ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ ``` + +## `remote-cache-server stop` + +``` +``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/read.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/read.md index f18b8d8c2..dd843e4e2 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/read.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/read.md @@ -1,6 +1,11 @@ # read -## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server vt run build` +## `remote-cache-server start` + +``` +``` + +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run vt run build` ``` $ vtt write-file dist/output.txt built @@ -14,7 +19,7 @@ $ vtt write-file dist/output.txt built ``` ``` -## `remote-cache-server vt run build` +## `remote-cache-server run vt run build` An endpoint without a mode selects read. The task reruns without uploading. @@ -23,3 +28,8 @@ $ vtt write-file dist/output.txt built ○ cache miss: 'src/a.txt' modified, exe [remote-cache] POST /fetch 200 exact ``` + +## `remote-cache-server stop` + +``` +``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/read_write.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/read_write.md index 677abcc78..1f8cf0286 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/read_write.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/read_write.md @@ -1,6 +1,11 @@ # read_write -## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server vt run build` +## `remote-cache-server start` + +``` +``` + +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run vt run build` The fetch finds no entry. The new execution is uploaded with one store request, and vt run waits for it after the task finishes. @@ -11,7 +16,7 @@ $ vtt write-file dist/output.txt built [remote-cache] POST /store 200 ``` -## `VP_REMOTE_CACHE=read-write remote-cache-server vt run build` +## `VP_REMOTE_CACHE=read-write remote-cache-server run vt run build` A local hit makes no requests. @@ -21,3 +26,8 @@ $ vtt write-file dist/output.txt built ◉ cache hit, replaying --- vt run: cache hit. ``` + +## `remote-cache-server stop` + +``` +``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/restore.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/restore.md index 763d33941..04a6e1983 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/restore.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/restore.md @@ -1,6 +1,11 @@ # restore -## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server vt run build` +## `remote-cache-server start` + +``` +``` + +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run vt run build` ``` $ vtt write-file dist/output.txt built @@ -19,7 +24,7 @@ $ vtt write-file dist/output.txt built ``` ``` -## `VP_REMOTE_CACHE=read-write remote-cache-server vt run build` +## `VP_REMOTE_CACHE=read-write remote-cache-server run vt run build` A remote hit downloads the output archive. Hits never upload. @@ -40,7 +45,7 @@ The outputs are restored. built ``` -## `remote-cache-server vt run build` +## `remote-cache-server run vt run build` The remote hit was recorded locally, so this is a local hit with no requests. @@ -61,7 +66,7 @@ vt run: cache hit. ``` ``` -## `remote-cache-server vt run build` +## `remote-cache-server run vt run build` The exact entry fails validation, so the archive isn't downloaded. @@ -70,3 +75,8 @@ $ vtt write-file dist/output.txt built ○ cache miss: 'src/a.txt' modified, exe [remote-cache] POST /fetch 200 exact ``` + +## `remote-cache-server stop` + +``` +``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/restore_failure.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/restore_failure.md index 63f5937cd..b15d0e160 100644 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/restore_failure.md +++ b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache/snapshots/restore_failure.md @@ -1,6 +1,11 @@ # restore_failure -## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server vt run build` +## `remote-cache-server start` + +``` +``` + +## `VP_REMOTE_CACHE=read-write VP_RUN_INTERNAL_HIDE_PENDING_UPLOADS=1 remote-cache-server run vt run build` ``` $ vtt write-file dist/output.txt built @@ -26,7 +31,7 @@ A file where the output directory goes makes restoring fail. ``` ``` -## `remote-cache-server vt run build` +## `remote-cache-server run vt run build` The remote hit can't be restored, so the task fails. @@ -79,7 +84,7 @@ The downloaded archive was removed, and the entry wasn't cached locally. ``` ``` -## `remote-cache-server vt run build` +## `remote-cache-server run vt run build` Nothing was cached locally, so the remote entry is fetched and restored again. @@ -91,3 +96,8 @@ vt run: remote cache hit. [remote-cache] POST /fetch 200 exact [remote-cache] GET /blob/1 200 ``` + +## `remote-cache-server stop` + +``` +``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/package.json b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/package.json deleted file mode 100644 index 0967ef424..000000000 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/package.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots.toml b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots.toml deleted file mode 100644 index 7fbf89fcc..000000000 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots.toml +++ /dev/null @@ -1,479 +0,0 @@ -# Windows is skipped because the PTY launcher cannot execute pnpm command shims. -[[e2e]] -name = "entries_and_associations" -cfg = "not(windows)" -ignore = true -steps = [ - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": 'A', \"secondary_key\": 'S'}", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'VA'}", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": 'A', \"secondary_key\": 'U', \"value\": 'VA'}", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": 'B', \"secondary_key\": 'S', \"value\": 'VB'}", - ], - { argv = [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": 'A', \"secondary_key\": 'S'}", - ], comment = "The exact entry survives reassignment of S to B." }, - { argv = [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": 'C', \"secondary_key\": 'S'}", - ], comment = "The previous fetch did not change S, so fallback returns B's entry." }, - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": 'A', \"secondary_key\": 'T', \"value\": 'VA2'}", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": 'A', \"secondary_key\": 'missing'}", - ], - { argv = [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": 'C', \"secondary_key\": 'U'}", - ], comment = "An older association to A still resolves after A is replaced." }, - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": 'C', \"secondary_key\": 'T'}", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": 'C', \"secondary_key\": 'S'}", - ], -] - -[[e2e]] -name = "blob_replacement" -cfg = "not(windows)" -ignore = true -steps = [ - [ - "remote-cache-server", - "cbor-http", - "GET", - "/blob/missing", - ], - [ - "vtt", - "write-file", - "archive.txt", - "first archive", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'first'}", - "--form-file", - "blob=archive.txt", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": 'A', \"secondary_key\": 'S'}", - ], - [ - "remote-cache-server", - "cbor-http", - "GET", - "/blob/1", - ], - { argv = [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-data", - "blob='second archive'", - "--form-cbor", - "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'second'}", - ], comment = "Accept blob before metadata, with neither part supplying a filename." }, - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": 'A', \"secondary_key\": 'S'}", - ], - [ - "remote-cache-server", - "cbor-http", - "GET", - "/blob/2", - ], - { argv = [ - "remote-cache-server", - "cbor-http", - "GET", - "/blob/1", - ], comment = "Previously returned IDs retain their original bytes." }, - { argv = [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'no archive'}", - ], comment = "Omitting blob clears the association." }, - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": 'A', \"secondary_key\": 'S'}", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'empty archive'}", - "--form-data", - "blob=''", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": 'A', \"secondary_key\": 'S'}", - ], - { argv = [ - "remote-cache-server", - "cbor-http", - "GET", - "/blob/3", - ], comment = "An empty blob has an ID and downloads as an empty byte string." }, -] - -[[e2e]] -name = "opaque_binary_data" -cfg = "not(windows)" -ignore = true -steps = [ - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": b64'AP+A', \"secondary_key\": '', \"value\": b64'AP+A'}", - "--form-data", - "blob=b64'AP+A'", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": b64'AP+A', \"secondary_key\": ''}", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": '', \"secondary_key\": ''}", - ], - [ - "remote-cache-server", - "cbor-http", - "GET", - "/blob/1", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": '', \"secondary_key\": '', \"value\": ''}", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": '', \"secondary_key\": ''}", - ], -] - -[[e2e]] -name = "invalid_requests" -cfg = "not(windows)" -ignore = true -steps = [ - [ - "remote-cache-server", - "cbor-http", - "GET", - "/missing", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--data", - "'bad'", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--data", - "'bad'", - "--content-type", - "application/cbor", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": \"text\", \"secondary_key\": ''}", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": ''}", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "[]", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": '', \"key\": 'duplicate', \"secondary_key\": ''}", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'original'}", - "--form-data", - "blob='original archive'", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--cbor", - "{}", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-data", - "blob='missing metadata'", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": 'A', \"secondary_key\": 'T', \"value\": \"wrong type\"}", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": 'A', \"secondary_key\": 'T', \"value\": 'replacement'}", - "--form-cbor", - "metadata={}", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": 'A', \"secondary_key\": 'T', \"value\": 'replacement'}", - "--form-data", - "blob='one'", - "--form-data", - "blob='two'", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={}", - "--form-data", - "unexpected='part'", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-data", - "metadata='wrong content type'", - ], - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--data", - "'--boundary\\r\\nContent-Disposition: form-data; name=\"metadata\"\\r\\nContent-Type: application/cbor\\r\\n\\r\\n'", - "--content-type", - "multipart/form-data; boundary=boundary", - ], - { argv = [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": 'A', \"secondary_key\": 'S'}", - ], comment = "Failed stores did not replace the value or blob." }, - [ - "remote-cache-server", - "cbor-http", - "GET", - "/blob/1", - ], - { argv = [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": 'B', \"secondary_key\": 'T'}", - ], comment = "Failed stores did not publish a secondary association." }, -] - -[[e2e]] -name = "state_across_invocations" -cfg = "not(windows)" -ignore = true -steps = [ - [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'VA'}", - "--form-data", - "blob='first archive'", - ], - { argv = [ - "remote-cache-server", - "cbor-http", - "POST", - "/fetch", - "--cbor", - "{\"key\": 'C', \"secondary_key\": 'S'}", - ], comment = "A later invocation reads the stored entry and association." }, - { argv = [ - "remote-cache-server", - "cbor-http", - "POST", - "/store", - "--form-cbor", - "metadata={\"key\": 'B', \"secondary_key\": 'T', \"value\": 'VB'}", - "--form-data", - "blob='second archive'", - ], comment = "Blob numbering continues across invocations." }, - { argv = [ - "vtt", - "write-file", - "remote-cache/blobs/1", - "replaced archive", - ], comment = "Each blob is a file named by its ID." }, - [ - "remote-cache-server", - "cbor-http", - "GET", - "/blob/1", - ], -] diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/blob_replacement.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/blob_replacement.md deleted file mode 100644 index ecf1cae0f..000000000 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/blob_replacement.md +++ /dev/null @@ -1,105 +0,0 @@ -# blob_replacement - -## `remote-cache-server cbor-http GET /blob/missing` - -``` -{"status": 404, "content_type": "text/plain; charset=utf-8", "body": "Blob not found"} -[remote-cache] GET /blob/missing 404 -``` - -## `vtt write-file archive.txt 'first archive'` - -``` -``` - -## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''first'\''}' --form-file blob=archive.txt` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"blob_id": "1"}} -[remote-cache] POST /store 200 -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'first', "blob_id": "1"}} -[remote-cache] POST /fetch 200 exact -``` - -## `remote-cache-server cbor-http GET /blob/1` - -``` -{"status": 200, "content_type": "application/octet-stream", "body": 'first archive'} -[remote-cache] GET /blob/1 200 -``` - -## `remote-cache-server cbor-http POST /store --form-data 'blob='\''second archive'\''' --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''second'\''}'` - -Accept blob before metadata, with neither part supplying a filename. - -``` -{"status": 200, "content_type": "application/cbor", "body": {"blob_id": "2"}} -[remote-cache] POST /store 200 -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'second', "blob_id": "2"}} -[remote-cache] POST /fetch 200 exact -``` - -## `remote-cache-server cbor-http GET /blob/2` - -``` -{"status": 200, "content_type": "application/octet-stream", "body": 'second archive'} -[remote-cache] GET /blob/2 200 -``` - -## `remote-cache-server cbor-http GET /blob/1` - -Previously returned IDs retain their original bytes. - -``` -{"status": 200, "content_type": "application/octet-stream", "body": 'first archive'} -[remote-cache] GET /blob/1 200 -``` - -## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''no archive'\''}'` - -Omitting blob clears the association. - -``` -{"status": 200, "content_type": "application/cbor", "body": {"blob_id": null}} -[remote-cache] POST /store 200 -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'no archive', "blob_id": null}} -[remote-cache] POST /fetch 200 exact -``` - -## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''empty archive'\''}' --form-data blob=''` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"blob_id": "3"}} -[remote-cache] POST /store 200 -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'empty archive', "blob_id": "3"}} -[remote-cache] POST /fetch 200 exact -``` - -## `remote-cache-server cbor-http GET /blob/3` - -An empty blob has an ID and downloads as an empty byte string. - -``` -{"status": 200, "content_type": "application/octet-stream", "body": ''} -[remote-cache] GET /blob/3 200 -``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/entries_and_associations.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/entries_and_associations.md deleted file mode 100644 index 9b5ad0152..000000000 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/entries_and_associations.md +++ /dev/null @@ -1,84 +0,0 @@ -# entries_and_associations - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` - -``` -{"status": 404, "content_type": "text/plain; charset=utf-8", "body": "Not found"} -[remote-cache] POST /fetch 404 -``` - -## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''VA'\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"blob_id": null}} -[remote-cache] POST /store 200 -``` - -## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''U'\'', "value": '\''VA'\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"blob_id": null}} -[remote-cache] POST /store 200 -``` - -## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''B'\'', "secondary_key": '\''S'\'', "value": '\''VB'\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"blob_id": null}} -[remote-cache] POST /store 200 -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` - -The exact entry survives reassignment of S to B. - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'VA', "blob_id": null}} -[remote-cache] POST /fetch 200 exact -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''C'\'', "secondary_key": '\''S'\''}'` - -The previous fetch did not change S, so fallback returns B's entry. - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "fallback", "key": 'B', "value": 'VB', "blob_id": null}} -[remote-cache] POST /fetch 200 fallback -``` - -## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''T'\'', "value": '\''VA2'\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"blob_id": null}} -[remote-cache] POST /store 200 -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''missing'\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'VA2', "blob_id": null}} -[remote-cache] POST /fetch 200 exact -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''C'\'', "secondary_key": '\''U'\''}'` - -An older association to A still resolves after A is replaced. - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "fallback", "key": 'A', "value": 'VA2', "blob_id": null}} -[remote-cache] POST /fetch 200 fallback -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''C'\'', "secondary_key": '\''T'\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "fallback", "key": 'A', "value": 'VA2', "blob_id": null}} -[remote-cache] POST /fetch 200 fallback -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''C'\'', "secondary_key": '\''S'\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "fallback", "key": 'B', "value": 'VB', "blob_id": null}} -[remote-cache] POST /fetch 200 fallback -``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/invalid_requests.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/invalid_requests.md deleted file mode 100644 index 9d468db74..000000000 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/invalid_requests.md +++ /dev/null @@ -1,138 +0,0 @@ -# invalid_requests - -## `remote-cache-server cbor-http GET /missing` - -``` -{"status": 404, "content_type": "text/plain; charset=utf-8", "body": "Route not found"} -[remote-cache] GET /missing 404 -``` - -## `remote-cache-server cbor-http POST /fetch --data 'bad'` - -``` -{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Expected application/cbor"} -[remote-cache] POST /fetch 400 -``` - -## `remote-cache-server cbor-http POST /fetch --data 'bad' --content-type application/cbor` - -``` -{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Expected byte strings: key, secondary_key"} -[remote-cache] POST /fetch 400 -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": "text", "secondary_key": '\'''\''}'` - -``` -{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Expected byte strings: key, secondary_key"} -[remote-cache] POST /fetch 400 -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\'''\''}'` - -``` -{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Expected byte strings: key, secondary_key"} -[remote-cache] POST /fetch 400 -``` - -## `remote-cache-server cbor-http POST /fetch --cbor []` - -``` -{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Expected byte strings: key, secondary_key"} -[remote-cache] POST /fetch 400 -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\'''\'', "key": '\''duplicate'\'', "secondary_key": '\'''\''}'` - -``` -{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Invalid CBOR"} -[remote-cache] POST /fetch 400 -``` - -## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''original'\''}' --form-data 'blob='\''original archive'\'''` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"blob_id": "1"}} -[remote-cache] POST /store 200 -``` - -## `remote-cache-server cbor-http POST /store --cbor {}` - -``` -{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Expected multipart/form-data"} -[remote-cache] POST /store 400 -``` - -## `remote-cache-server cbor-http POST /store --form-data 'blob='\''missing metadata'\'''` - -``` -{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Missing metadata"} -[remote-cache] POST /store 400 -``` - -## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''T'\'', "value": "wrong type"}'` - -``` -{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Expected byte strings: key, secondary_key, value"} -[remote-cache] POST /store 400 -``` - -## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''T'\'', "value": '\''replacement'\''}' --form-cbor metadata={}` - -``` -{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Invalid multipart body"} -[remote-cache] POST /store 400 -``` - -## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''T'\'', "value": '\''replacement'\''}' --form-data blob='one' --form-data blob='two'` - -``` -{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Invalid multipart body"} -[remote-cache] POST /store 400 -``` - -## `remote-cache-server cbor-http POST /store --form-cbor metadata={} --form-data unexpected='part'` - -``` -{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Invalid multipart body"} -[remote-cache] POST /store 400 -``` - -## `remote-cache-server cbor-http POST /store --form-data 'metadata='\''wrong content type'\'''` - -``` -{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Invalid multipart body"} -[remote-cache] POST /store 400 -``` - -## `remote-cache-server cbor-http POST /store --data ''\''--boundary\r\nContent-Disposition: form-data; name="metadata"\r\nContent-Type: application/cbor\r\n\r\n'\''' --content-type 'multipart/form-data; boundary=boundary'` - -``` -{"status": 400, "content_type": "text/plain; charset=utf-8", "body": "Invalid multipart body"} -[remote-cache] POST /store 400 -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''A'\'', "secondary_key": '\''S'\''}'` - -Failed stores did not replace the value or blob. - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'original', "blob_id": "1"}} -[remote-cache] POST /fetch 200 exact -``` - -## `remote-cache-server cbor-http GET /blob/1` - -``` -{"status": 200, "content_type": "application/octet-stream", "body": 'original archive'} -[remote-cache] GET /blob/1 200 -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''B'\'', "secondary_key": '\''T'\''}'` - -Failed stores did not publish a secondary association. - -``` -{"status": 404, "content_type": "text/plain; charset=utf-8", "body": "Not found"} -[remote-cache] POST /fetch 404 -``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/opaque_binary_data.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/opaque_binary_data.md deleted file mode 100644 index b9bb2cd5e..000000000 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/opaque_binary_data.md +++ /dev/null @@ -1,43 +0,0 @@ -# opaque_binary_data - -## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": b64'\''AP+A'\'', "secondary_key": '\'''\'', "value": b64'\''AP+A'\''}' --form-data blob=b64'AP+A'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"blob_id": "1"}} -[remote-cache] POST /store 200 -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": b64'\''AP+A'\'', "secondary_key": '\'''\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": b64'AP+A', "blob_id": "1"}} -[remote-cache] POST /fetch 200 exact -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\'''\'', "secondary_key": '\'''\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "fallback", "key": b64'AP+A', "value": b64'AP+A', "blob_id": "1"}} -[remote-cache] POST /fetch 200 fallback -``` - -## `remote-cache-server cbor-http GET /blob/1` - -``` -{"status": 200, "content_type": "application/octet-stream", "body": b64'AP+A'} -[remote-cache] GET /blob/1 200 -``` - -## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\'''\'', "secondary_key": '\'''\'', "value": '\'''\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"blob_id": null}} -[remote-cache] POST /store 200 -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\'''\'', "secondary_key": '\'''\''}'` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": '', "blob_id": null}} -[remote-cache] POST /fetch 200 exact -``` diff --git a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/state_across_invocations.md b/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/state_across_invocations.md deleted file mode 100644 index 4eb0f38fd..000000000 --- a/crates/vt_bin/tests/e2e_snapshots/fixtures/remote_cache_backend/snapshots/state_across_invocations.md +++ /dev/null @@ -1,40 +0,0 @@ -# state_across_invocations - -## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''A'\'', "secondary_key": '\''S'\'', "value": '\''VA'\''}' --form-data 'blob='\''first archive'\'''` - -``` -{"status": 200, "content_type": "application/cbor", "body": {"blob_id": "1"}} -[remote-cache] POST /store 200 -``` - -## `remote-cache-server cbor-http POST /fetch --cbor '{"key": '\''C'\'', "secondary_key": '\''S'\''}'` - -A later invocation reads the stored entry and association. - -``` -{"status": 200, "content_type": "application/cbor", "body": {"kind": "fallback", "key": 'A', "value": 'VA', "blob_id": "1"}} -[remote-cache] POST /fetch 200 fallback -``` - -## `remote-cache-server cbor-http POST /store --form-cbor 'metadata={"key": '\''B'\'', "secondary_key": '\''T'\'', "value": '\''VB'\''}' --form-data 'blob='\''second archive'\'''` - -Blob numbering continues across invocations. - -``` -{"status": 200, "content_type": "application/cbor", "body": {"blob_id": "2"}} -[remote-cache] POST /store 200 -``` - -## `vtt write-file remote-cache/blobs/1 'replaced archive'` - -Each blob is a file named by its ID. - -``` -``` - -## `remote-cache-server cbor-http GET /blob/1` - -``` -{"status": 200, "content_type": "application/octet-stream", "body": 'replaced archive'} -[remote-cache] GET /blob/1 200 -``` diff --git a/packages/tools/README.md b/packages/tools/README.md index 0e466a0b5..34b70664b 100644 --- a/packages/tools/README.md +++ b/packages/tools/README.md @@ -2,16 +2,23 @@ This package provides Node.js dependencies and commands for the test suites. ## Remote cache backend -Run `pnpm install` at the repository root to install `remote-cache-server` and `cbor-http` into this package's `node_modules/.bin`. The workspace self-dependency makes pnpm link the package's own commands there. The E2E harness already includes that directory in PATH. Both commands run TypeScript directly using the Node version in `.node-version`. +Run `pnpm install` at the repository root to install `remote-cache-server` into this package's `node_modules/.bin`. The workspace self-dependency makes pnpm link the package's own commands there. The E2E harness already includes that directory in PATH. The command runs TypeScript directly using the Node version in `.node-version`. ```sh -remote-cache-server cbor-http POST /store --form-cbor "metadata={\"key\": 'A', \"secondary_key\": 'S', \"value\": 'record'}" -remote-cache-server cbor-http POST /fetch --cbor "{\"key\": 'A', \"secondary_key\": 'S'}" +remote-cache-server start +VP_REMOTE_CACHE=read-write remote-cache-server run vt run build +remote-cache-server stop ``` -`remote-cache-server COMMAND [ARGS...]` starts the backend on a free loopback port and runs the command with `VP_REMOTE_CACHE_URL` set to the endpoint, `http://127.0.0.1:/projects/test`. The fixed base path gives every endpoint a namespace path. The wrapper takes no options and passes all arguments to the command unchanged. The command inherits stdio and handles Ctrl-C, which the wrapper ignores. When it exits, the server stops and the wrapper exits with the command's exit code. +An E2E case starts its own backend in its first step and stops it in its last, so the backend keeps its state for the whole case. Each subcommand works in the current directory, the case's directory: -After the command exits, the wrapper prints one line to stderr for each request it served, in the order of the responses. Each line has the method, the path below the base path, and the status. Successful fetch responses add their kind: +- `remote-cache-server start` starts the backend in the background on free loopback ports and returns once it's ready. The backend runs in its own session and doesn't use the terminal, so the step can finish and Ctrl-C in later steps doesn't reach it. It writes its endpoint, `http://127.0.0.1:/projects/test`, to `remote-cache/server.json`, and its output to `remote-cache/server.log`. +- `remote-cache-server run COMMAND [ARGS...]` runs the command with `VP_REMOTE_CACHE_URL` set to the endpoint. The fixed base path gives the endpoint a namespace path. The command inherits stdio and handles Ctrl-C, which `run` ignores. `run` exits with the command's exit code. +- `remote-cache-server stop` stops the backend. It fails if the backend answered a request with a 5xx status or couldn't be reached. + +A backend whose case directory disappears stops by itself, as does one that has had no requests for ten minutes, e.g. because its case timed out before the stop step. + +The endpoint is a tap in front of the backend. It forwards every request and response unchanged and records a line for each response, in the order of the responses. Each line has the method, the path below the base path, and the status. Successful fetch responses add their kind. After the command exits, `run` prints the lines for the requests it caused to stderr, and `stop` prints any that are left: ```text [remote-cache] POST /fetch 404 @@ -20,31 +27,10 @@ After the command exits, the wrapper prints one line to stderr for each request [remote-cache] GET /blob/1 200 ``` -State persists in `remote-cache/` in the current directory, so consecutive commands share it. Each E2E case has its own directory and state. `state.json` holds the entries, associations, and next blob ID, with keys and values hex-encoded. Each blob is a file in `remote-cache/blobs/` named by its blob ID. Blob IDs are sequential strings and continue across invocations, keeping snapshots deterministic. +The backend keeps its state in `remote-cache/`. `state.json` holds the entries, associations, and next blob ID, with keys and values hex-encoded. Each blob is a file in `remote-cache/blobs/` named by its blob ID. Blob IDs are sequential strings, keeping snapshots deterministic. The backend implements `POST /fetch`, `POST /store`, and `GET /blob/{blob_id}` from the [remote cache server API](https://github.com/voidzero-dev/vite-task/pull/713). A fetch that matches neither key gets a `404` with the plain-text body `Not found`. Keys, values, and blobs are opaque bytes without length limits. There is no authentication. -## CBOR HTTP client - -`cbor-http METHOD PATH` appends the path to `VP_REMOTE_CACHE_URL`, so run it through `remote-cache-server`. Request bodies use [CBOR extended diagnostic notation (EDN)](https://www.rfc-editor.org/rfc/rfc8610.html#appendix-G): `"text"` is text, `'bytes'` is a UTF-8 byte string, and `b64'AP+A'` contains arbitrary binary bytes. - -| Option | Body | -| ----------------------- | --------------------------------------------------------- | -| `--cbor EDN` | Encode EDN as CBOR with `Content-Type: application/cbor`. | -| `--data EDN` | Send an EDN byte string as raw bytes. | -| `--form-cbor NAME=EDN` | Add a CBOR multipart part. | -| `--form-data NAME=EDN` | Add an EDN byte string as a raw multipart part. | -| `--form-file NAME=FILE` | Add file contents as a raw multipart part. | -| `--content-type TYPE` | Override the request content type. | - -Multipart options can repeat and retain their order. Raw bodies and parts default to `application/octet-stream`. Parts have no filename, including binary metadata. - -Every response prints one EDN map containing `status`, `content_type`, and `body`. CBOR is decoded, text becomes a text string, and other bodies become byte strings. Byte strings use readable UTF-8 when possible and base64 EDN otherwise. HTTP error responses exit successfully so snapshots can assert their status and body. Argument errors, transport failures, and invalid CBOR responses exit unsuccessfully. - -```text -{"status": 200, "content_type": "application/cbor", "body": {"kind": "exact", "value": 'record', "blob_id": null}} -``` - -Run `pnpm --filter vite-task-tools check` for type checking and `pnpm --filter vite-task-tools test` for the EDN-formatting unit test. Run `cargo test -p vt_bin --test e2e_snapshots -- remote_cache_backend --ignored` for the backend snapshots. +Run `pnpm --filter vite-task-tools check` for type checking. Run `cargo test -p vt_bin --test e2e_snapshots -- remote_cache --ignored` for the snapshots that use the backend. -The backend snapshots are skipped on Windows because the PTY launcher cannot execute pnpm command shims. +Those snapshots are skipped on Windows because the PTY launcher cannot execute pnpm command shims. diff --git a/packages/tools/package.json b/packages/tools/package.json index ebd6e8fe4..f0074e0b9 100644 --- a/packages/tools/package.json +++ b/packages/tools/package.json @@ -2,13 +2,11 @@ "name": "vite-task-tools", "private": true, "bin": { - "cbor-http": "./src/cbor-http.ts", "remote-cache-server": "./src/remote-cache/cli.ts" }, "type": "module", "scripts": { - "check": "tsc --noEmit -p tsconfig.json", - "test": "node --test src/*.test.ts" + "check": "tsc --noEmit -p tsconfig.json" }, "dependencies": { "@anthropic-ai/sandbox-runtime": "catalog:", @@ -18,7 +16,6 @@ "@vitest/browser-playwright": "catalog:", "@voidzero-dev/vite-task-client": "workspace:*", "bun": "catalog:", - "cbor-edn": "catalog:", "cbor2": "catalog:", "cross-env": "catalog:", "deno": "catalog:", diff --git a/packages/tools/src/cbor-http.ts b/packages/tools/src/cbor-http.ts deleted file mode 100755 index 30535fd77..000000000 --- a/packages/tools/src/cbor-http.ts +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env node -import { parseEDN } from 'cbor-edn'; -import { decode } from 'cbor2/decoder'; -import { encode } from 'cbor2/encoder'; -import { randomUUID } from 'node:crypto'; -import { readFile } from 'node:fs/promises'; -import { parseArgs } from 'node:util'; -import { toEDN } from './edn.ts'; - -function bytes(edn: string): Uint8Array { - const value: unknown = decode(parseEDN(edn, {})); - if (!(value instanceof Uint8Array)) throw new Error('Expected an EDN byte string'); - return value; -} - -async function main(): Promise { - const { values, positionals, tokens } = parseArgs({ - allowPositionals: true, - tokens: true, - options: { - cbor: { type: 'string' }, - data: { type: 'string' }, - 'content-type': { type: 'string' }, - 'form-cbor': { type: 'string', multiple: true }, - 'form-data': { type: 'string', multiple: true }, - 'form-file': { type: 'string', multiple: true }, - }, - }); - const [method, path] = positionals; - if (positionals.length !== 2 || !method || !path) { - throw new Error( - 'Usage: cbor-http METHOD PATH [--cbor EDN | --data EDN | --form-cbor NAME=EDN | --form-data NAME=EDN | --form-file NAME=FILE]', - ); - } - let body: Buffer | undefined; - let contentType: string | undefined; - const formTokens = tokens.filter( - (token) => token.kind === 'option' && token.name.startsWith('form-'), - ); - if ( - [values.cbor !== undefined, values.data !== undefined, formTokens.length > 0].filter(Boolean) - .length > 1 - ) { - throw new Error('Choose one request body format'); - } - if (values.cbor !== undefined) { - body = Buffer.from(parseEDN(values.cbor, {})); - contentType = 'application/cbor'; - } else if (values.data !== undefined) { - body = Buffer.from(bytes(values.data)); - contentType = 'application/octet-stream'; - } else if (formTokens.length) { - const boundary = randomUUID(); - const chunks: Buffer[] = []; - for (const token of formTokens) { - if (token.kind !== 'option' || token.value === undefined) continue; - const equal = token.value.indexOf('='); - const name = token.value.slice(0, equal); - const value = token.value.slice(equal + 1); - if (equal < 1 || /["\r\n\\]/.test(name)) throw new Error('Expected multipart NAME=VALUE'); - const isCbor = token.name === 'form-cbor'; - const data = isCbor - ? parseEDN(value, {}) - : token.name === 'form-file' - ? await readFile(value) - : bytes(value); - // Do not add filenames: binary metadata is valid without one. - chunks.push( - Buffer.from( - `--${boundary}\r\nContent-Disposition: form-data; name="${name}"\r\nContent-Type: ${isCbor ? 'application/cbor' : 'application/octet-stream'}\r\n\r\n`, - ), - ); - chunks.push(Buffer.from(data), Buffer.from('\r\n')); - } - chunks.push(Buffer.from(`--${boundary}--\r\n`)); - body = Buffer.concat(chunks); - contentType = `multipart/form-data; boundary=${boundary}`; - } - contentType = values['content-type'] ?? contentType; - const response = await fetch(`${process.env['VP_REMOTE_CACHE_URL']}${path}`, { - method, - ...(body === undefined ? {} : { body }), - ...(contentType === undefined ? {} : { headers: { 'content-type': contentType } }), - }); - const responseType = response.headers.get('content-type') ?? ''; - const raw = new Uint8Array(await response.arrayBuffer()); - const type = responseType.split(';')[0]?.trim(); - let printedBody: string; - if (type === 'application/cbor') { - decode(raw); // diagnose alone does not reject every incomplete CBOR item. - printedBody = toEDN(raw); - } else { - printedBody = toEDN(encode(type?.startsWith('text/') ? new TextDecoder().decode(raw) : raw)); - } - console.log( - `{"status": ${response.status}, "content_type": ${JSON.stringify(responseType)}, "body": ${printedBody}}`, - ); -} - -try { - await main(); -} catch (error) { - console.error(error instanceof Error ? error.message : String(error)); - process.exitCode = 1; -} diff --git a/packages/tools/src/edn.test.ts b/packages/tools/src/edn.test.ts deleted file mode 100644 index 50f5d5721..000000000 --- a/packages/tools/src/edn.test.ts +++ /dev/null @@ -1,17 +0,0 @@ -import assert from 'node:assert/strict'; -import test from 'node:test'; -import { parseEDN } from 'cbor-edn'; -import { decode } from 'cbor2/decoder'; -import { encode } from 'cbor2/encoder'; -import { toEDN } from './edn.ts'; - -await test('EDN formatting preserves escapes, a UTF-8 BOM, and text resembling byte strings', () => { - for (const value of [ - new TextEncoder().encode("quotes: '\"\\\n\t\0 and h'ff'"), - new Uint8Array([239, 187, 191, 65]), - { text: "h'ff'", bytes: new TextEncoder().encode('hello') }, - ]) { - const edn = toEDN(encode(value)); - assert.deepEqual(decode(parseEDN(edn, {})), value); - } -}); diff --git a/packages/tools/src/edn.ts b/packages/tools/src/edn.ts deleted file mode 100644 index 44919d221..000000000 --- a/packages/tools/src/edn.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { diagnose } from 'cbor2/diagnostic'; - -/** Keep byte strings readable without mistaking text containing h'...' for bytes. */ -export function toEDN(bytes: Uint8Array): string { - return diagnose(bytes).replace( - /"(?:[^"\\]|\\.)*"|h'([0-9a-f]*)'/gi, - (token, hex: string | undefined) => { - if (hex === undefined) return token; - const binary = Buffer.from(hex, 'hex'); - try { - const text = new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode(binary); - const escaped = JSON.stringify(text) - .slice(1, -1) - .replace(/\\.|'/g, (part) => { - if (part === '\\"') return '"'; - return part === "'" ? "\\'" : part; - }); - return `'${escaped}'`; - } catch { - return `b64'${binary.toString('base64')}'`; - } - }, - ); -} diff --git a/packages/tools/src/remote-cache/server.ts b/packages/tools/src/remote-cache/backend.ts similarity index 86% rename from packages/tools/src/remote-cache/server.ts rename to packages/tools/src/remote-cache/backend.ts index 76a14e33a..3c4c24449 100644 --- a/packages/tools/src/remote-cache/server.ts +++ b/packages/tools/src/remote-cache/backend.ts @@ -1,8 +1,10 @@ import { Busboy } from '@fastify/busboy'; import { decode } from 'cbor2/decoder'; import { encode } from 'cbor2/encoder'; +import { once } from 'node:events'; import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { createServer, type IncomingMessage, type ServerResponse } from 'node:http'; +import type { AddressInfo } from 'node:net'; import { join } from 'node:path'; interface Entry { @@ -94,23 +96,26 @@ function cbor(response: ServerResponse, value: unknown): void { response.end(encode(value)); } +/** A running backend. */ +export interface Backend { + /** Where the backend listens, e.g. `http://127.0.0.1:1234`, without a path. */ + origin: string; + close(): Promise; +} + /** - * A test backend that keeps its state in `directory`: entries and associations - * in `state.json`, and each blob in `blobs/` under its ID. Keys, values, and - * blobs remain opaque bytes. A fetch that matches neither key gets a plain-text - * 404. After each response, `logRequest` receives a line with the method, the - * route below `basePath`, the status, and for successful fetch responses, the - * kind. + * Start a test backend on a free loopback port that keeps its state in + * `directory`: entries and associations in `state.json`, and each blob in + * `blobs/` under its ID. Keys, values, and blobs remain opaque bytes. A fetch + * that matches neither key gets a plain-text 404. */ -export function createCacheServer({ +export async function startBackend({ basePath, directory, - logRequest, }: { basePath: string; directory: string; - logRequest: (line: string) => void; -}) { +}): Promise { const stateFile = join(directory, 'state.json'); const blobDirectory = join(directory, 'blobs'); const state: State = existsSync(stateFile) @@ -120,18 +125,17 @@ export function createCacheServer({ const associations = new Map(Object.entries(state.associations)); let nextBlobId = state.next_blob_id; - /** Respond to `request`, returning the kind of a successful fetch response. */ async function handle( request: IncomingMessage, response: ServerResponse, path: string, - ): Promise { + ): Promise { if (request.method === 'GET' && path.startsWith(`${basePath}/blob/`)) { const file = join(blobDirectory, path.slice(`${basePath}/blob/`.length)); if (!existsSync(file)) throw new RequestError(404, 'Blob not found'); response.writeHead(200, { 'content-type': 'application/octet-stream' }); response.end(readFileSync(file)); - return undefined; + return; } if (request.method !== 'POST' || ![`${basePath}/fetch`, `${basePath}/store`].includes(path)) { throw new RequestError(404, 'Route not found'); @@ -149,7 +153,7 @@ export function createCacheServer({ const fallback = associatedKey === undefined ? undefined : entries.get(associatedKey); if (exact) { cbor(response, { kind: 'exact', value: fromHex(exact.value), blob_id: exact.blob_id }); - return 'exact'; + return; } if (fallback) { cbor(response, { @@ -158,7 +162,7 @@ export function createCacheServer({ value: fromHex(fallback.value), blob_id: fallback.blob_id, }); - return 'fallback'; + return; } throw new RequestError(404, 'Not found'); } @@ -189,16 +193,11 @@ export function createCacheServer({ }; writeFileSync(stateFile, `${JSON.stringify(saved, null, 2)}\n`); cbor(response, { blob_id: blobId }); - return undefined; } - return createServer((request, response) => { + const server = createServer((request, response) => { const path = new URL(request.url ?? '/', 'http://localhost').pathname; - const log = (kind?: string) => { - const parts = [request.method, path.slice(basePath.length), response.statusCode, kind]; - logRequest(parts.filter((part) => part !== undefined).join(' ')); - }; - void handle(request, response, path).then(log, (error: unknown) => { + handle(request, response, path).catch((error: unknown) => { const known = error instanceof RequestError; if (!known) console.error(error); response.writeHead(known ? error.status : 500, { @@ -206,7 +205,17 @@ export function createCacheServer({ }); response.end(known ? error.message : 'Internal server error'); request.resume(); - log(); }); }); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + const { port } = server.address() as AddressInfo; + return { + origin: `http://127.0.0.1:${port}`, + close: () => + new Promise((resolve) => { + server.close(() => resolve()); + server.closeAllConnections(); + }), + }; } diff --git a/packages/tools/src/remote-cache/cli.ts b/packages/tools/src/remote-cache/cli.ts index 82b00339a..8df4a9a18 100755 --- a/packages/tools/src/remote-cache/cli.ts +++ b/packages/tools/src/remote-cache/cli.ts @@ -1,28 +1,96 @@ #!/usr/bin/env node import { spawn } from 'node:child_process'; import { once } from 'node:events'; -import type { AddressInfo } from 'node:net'; -import { createCacheServer } from './server.ts'; - -const basePath = '/projects/test'; - -const [command, ...args] = process.argv.slice(2); -const requests: string[] = []; -const server = createCacheServer({ - basePath, - directory: 'remote-cache', - logRequest: (line) => requests.push(line), -}); -server.listen(0, '127.0.0.1'); -await once(server, 'listening'); -const { port } = server.address() as AddressInfo; -// Ctrl-C is left to the command. -process.on('SIGINT', () => {}); -const child = spawn(command!, args, { - stdio: 'inherit', - env: { ...process.env, VP_REMOTE_CACHE_URL: `http://127.0.0.1:${port}${basePath}` }, -}); -const [code] = (await once(child, 'exit')) as [number | null]; -server.close(); -for (const line of requests) console.error(`[remote-cache] ${line}`); -process.exitCode = code; +import { closeSync, existsSync, mkdirSync, openSync, readFileSync } from 'node:fs'; +import { request } from 'node:http'; +import { tmpdir } from 'node:os'; +import { resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { logFile, serverFile, stateDirectory, type ServerInfo } from './state.ts'; + +const usage = 'Usage: remote-cache-server start | run COMMAND [ARGS...] | stop'; +const directory = resolve(stateDirectory); + +function fail(message: string): never { + console.error(`remote-cache-server: ${message}`); + process.exit(1); +} + +function readInfo(): ServerInfo { + if (!existsSync(serverFile(directory))) { + fail('no backend is running here. Start one with `remote-cache-server start`.'); + } + return JSON.parse(readFileSync(serverFile(directory), 'utf8')) as ServerInfo; +} + +/** POST to the backend's control server and return the JSON response. */ +function control(info: ServerInfo, path: string): Promise { + return new Promise((resolve) => { + const unreachable = (error: Error) => + fail(`the backend is unreachable (${error.message}). See ${logFile(stateDirectory)}.`); + const call = request(`${info.control}${path}`, { method: 'POST', agent: false }, (response) => { + const chunks: Buffer[] = []; + response.on('data', (chunk: Buffer) => chunks.push(chunk)); + response.on('end', () => resolve(JSON.parse(Buffer.concat(chunks).toString()) as T)); + response.on('error', unreachable); + }); + call.on('error', unreachable); + call.end(); + }); +} + +function printRequests(lines: string[]): void { + for (const line of lines) console.error(`[remote-cache] ${line}`); +} + +async function start(): Promise { + if (existsSync(serverFile(directory))) fail('a backend is already running here.'); + mkdirSync(directory, { recursive: true }); + const log = openSync(logFile(directory), 'a'); + // The backend outlives this step, so it gets its own session and none of the + // terminal's file descriptors. The terminal then closes when this step + // exits, and Ctrl-C in later steps doesn't reach the backend. + const daemon = spawn( + process.execPath, + [...process.execArgv, fileURLToPath(new URL('daemon.ts', import.meta.url)), directory], + { cwd: tmpdir(), detached: true, stdio: ['ignore', log, log, 'ipc'] }, + ); + closeSync(log); + const ready = await Promise.race([ + once(daemon, 'message').then(() => true), + once(daemon, 'exit').then(() => false), + ]); + if (!ready) fail(`the backend failed to start:\n${readFileSync(logFile(directory), 'utf8')}`); + daemon.disconnect(); + daemon.unref(); +} + +async function run([command, ...args]: string[]): Promise { + if (command === undefined) fail(usage); + const info = readInfo(); + // Ctrl-C is left to the command. + process.on('SIGINT', () => {}); + const child = spawn(command, args, { + stdio: 'inherit', + env: { ...process.env, VP_REMOTE_CACHE_URL: info.url }, + }); + const [code] = (await once(child, 'exit')) as [number | null]; + printRequests(await control(info, '/take')); + process.exitCode = code; +} + +async function stop(): Promise { + const { requests, anomalies } = await control<{ requests: string[]; anomalies: string[] }>( + readInfo(), + '/stop', + ); + printRequests(requests); + for (const anomaly of anomalies) console.error(`remote-cache-server: ${anomaly}`); + if (anomalies.length > 0) process.exitCode = 1; +} + +const [subcommand, ...args] = process.argv.slice(2); +if (subcommand === 'start') await start(); +else if (subcommand === 'run') await run(args); +else if (subcommand === 'stop') await stop(); +else fail(usage); diff --git a/packages/tools/src/remote-cache/daemon.ts b/packages/tools/src/remote-cache/daemon.ts new file mode 100644 index 000000000..594b8b5af --- /dev/null +++ b/packages/tools/src/remote-cache/daemon.ts @@ -0,0 +1,136 @@ +import { decode } from 'cbor2/decoder'; +import { once } from 'node:events'; +import { existsSync, rmSync, writeFileSync } from 'node:fs'; +import { createServer, request as forward, type IncomingHttpHeaders, type Server } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { startBackend } from './backend.ts'; +import { basePath, serverFile, type ServerInfo } from './state.ts'; + +// The process that `remote-cache-server start` leaves running for one e2e case. +// It serves the backend at the endpoint through a tap that forwards requests +// and responses unchanged and records a line for each response. A control +// server hands those lines to `remote-cache-server run` and `stop`. + +const directory = process.argv[2]!; +/** Stop after this long without requests, e.g. when a case timed out before its stop step. */ +const idleTimeout = 10 * 60 * 1000; +/** Headers about a single connection. Node sets them for each hop itself. */ +const hopByHop = new Set([ + 'connection', + 'keep-alive', + 'proxy-connection', + 'transfer-encoding', + 'upgrade', +]); + +const backend = await startBackend({ basePath, directory }); +/** Request lines that no `run` or `stop` has taken yet. */ +const requests: string[] = []; +/** Problems on the backend's side, which make `stop` fail. */ +const anomalies: string[] = []; +let lastUse = Date.now(); +let stopping = false; + +function endToEnd(headers: IncomingHttpHeaders): IncomingHttpHeaders { + return Object.fromEntries(Object.entries(headers).filter(([name]) => !hopByHop.has(name))); +} + +/** The kind of a successful fetch response. */ +function fetchKind(body: Buffer): string | undefined { + try { + const value: unknown = decode(body); + if (typeof value === 'object' && value !== null && 'kind' in value) { + return typeof value.kind === 'string' ? value.kind : undefined; + } + } catch { + // Not CBOR. vp reports the malformed response itself. + } + return undefined; +} + +const tap = createServer((request, response) => { + lastUse = Date.now(); + const path = new URL(request.url ?? '/', 'http://localhost').pathname; + const route = path.startsWith(basePath) ? path.slice(basePath.length) : path; + const label = `${request.method} ${route}`; + let clientGone = false; + const upstream = forward( + `${backend.origin}${request.url}`, + { method: request.method!, headers: endToEnd(request.headers) }, + (reply) => { + const status = reply.statusCode!; + if (status >= 500) anomalies.push(`${label} got ${status} from the backend`); + response.writeHead(status, endToEnd(reply.headers)); + const chunks: Buffer[] = []; + if (route === '/fetch' && status === 200) { + reply.on('data', (chunk: Buffer) => chunks.push(chunk)); + } + reply.pipe(response); + response.on('finish', () => { + const kind = chunks.length > 0 ? fetchKind(Buffer.concat(chunks)) : undefined; + requests.push([label, status, kind].filter((part) => part !== undefined).join(' ')); + }); + }, + ); + upstream.on('error', (error) => { + if (clientGone) return; + anomalies.push(`${label} failed: ${error.message}`); + if (response.headersSent) response.destroy(); + else response.writeHead(502).end(); + }); + // The client can disconnect before the response ends, e.g. after Ctrl-C. + response.on('close', () => { + if (response.writableFinished) return; + clientGone = true; + upstream.destroy(); + }); + request.pipe(upstream); +}); + +const control = createServer((request, response) => { + lastUse = Date.now(); + request.resume(); + const reply = (value: unknown) => { + response.writeHead(200, { 'content-type': 'application/json' }); + response.end(JSON.stringify(value)); + }; + if (request.method === 'POST' && request.url === '/take') { + reply(requests.splice(0)); + } else if (request.method === 'POST' && request.url === '/stop') { + response.on('finish', () => void stop()); + reply({ requests: requests.splice(0), anomalies }); + } else { + response.writeHead(404).end(); + } +}); + +async function listen(server: Server): Promise { + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + return (server.address() as AddressInfo).port; +} + +async function stop(): Promise { + if (stopping) return; + stopping = true; + rmSync(serverFile(directory), { force: true }); + for (const server of [tap, control]) { + server.close(); + server.closeAllConnections(); + } + await backend.close(); + process.exit(0); +} + +const info: ServerInfo = { + url: `http://127.0.0.1:${await listen(tap)}${basePath}`, + control: `http://127.0.0.1:${await listen(control)}`, +}; +writeFileSync(serverFile(directory), `${JSON.stringify(info, null, 2)}\n`); +process.on('SIGTERM', () => void stop()); +setInterval(() => { + // The e2e harness removes case directories when the test run ends. + if (!existsSync(directory) || Date.now() - lastUse > idleTimeout) void stop(); +}, 1000); +process.send?.('ready'); +process.disconnect?.(); diff --git a/packages/tools/src/remote-cache/state.ts b/packages/tools/src/remote-cache/state.ts new file mode 100644 index 000000000..1312aeda9 --- /dev/null +++ b/packages/tools/src/remote-cache/state.ts @@ -0,0 +1,23 @@ +import { join } from 'node:path'; + +/** The directory, relative to the current directory, that holds a case's backend. */ +export const stateDirectory = 'remote-cache'; + +/** The path below the backend's origin that every endpoint has. */ +export const basePath = '/projects/test'; + +/** How a running backend is reached, saved in `server.json`. */ +export interface ServerInfo { + /** The endpoint commands use, e.g. `http://127.0.0.1:1234/projects/test`. */ + url: string; + /** The origin of the control server, e.g. `http://127.0.0.1:1235`. */ + control: string; +} + +export function serverFile(directory: string): string { + return join(directory, 'server.json'); +} + +export function logFile(directory: string): string { + return join(directory, 'server.log'); +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b69c0ba72..f937ac624 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -30,9 +30,6 @@ catalogs: bun: specifier: 1.4.2 version: 1.4.2 - cbor-edn: - specifier: 0.2.2 - version: 0.2.2 cbor2: specifier: 1.12.0 version: 1.12.0 @@ -109,9 +106,6 @@ importers: bun: specifier: 'catalog:' version: 1.4.2 - cbor-edn: - specifier: 'catalog:' - version: 0.2.2 cbor2: specifier: 'catalog:' version: 1.12.0 @@ -1395,15 +1389,6 @@ packages: os: [darwin, linux, android, freebsd, win32] hasBin: true - cbor-edn@0.2.2: - resolution: {integrity: sha512-/zu3hPQmrkHtHoW1Bp73pYRlIg1R34rNiP4L87FMI2EbBf7l0cO3B2W/rBx3W8mvqDh3Qc4cJy3DcAnVaamQPQ==} - engines: {node: '>=18.7'} - hasBin: true - - cbor2@1.11.0: - resolution: {integrity: sha512-teHnpDGcDzwouFJgCJLpDY2zMaEOYvYDryd6sln67wPrCuObmwriG1NAA1w7rUIf/l++bNdts4U7WeVdYTGwLg==} - engines: {node: '>=18.7'} - cbor2@1.12.0: resolution: {integrity: sha512-3Cco8XQhi27DogSp9Ri6LYNZLi/TBY/JVnDe+mj06NkBjW/ZYOtekaEU4wZ4xcRMNrFkDv8KNtOAqHyDfz3lYg==} engines: {node: '>=18.7'} @@ -2741,12 +2726,6 @@ snapshots: '@oven/bun-windows-aarch64': 1.4.2 '@oven/bun-windows-x64': 1.4.2 - cbor-edn@0.2.2: - dependencies: - cbor2: 1.11.0 - - cbor2@1.11.0: {} - cbor2@1.12.0: {} chai@6.2.2: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 8f315a781..ba7fb6d5e 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -21,7 +21,6 @@ catalog: '@playwright/browser-chromium': 1.63.0 '@vitest/browser-playwright': 5.0.3 bun: 1.4.2 - cbor-edn: 0.2.2 cbor2: 1.12.0 cross-env: ^10.1.0 deno: 2.9.6