diff --git a/.github/scripts/print-crash-reports.py b/.github/scripts/print-crash-reports.py index 62c3f55f..ed054442 100644 --- a/.github/scripts/print-crash-reports.py +++ b/.github/scripts/print-crash-reports.py @@ -6,7 +6,9 @@ - macOS writes a crash report (.ips) for every crashed process, naming the library and function that faulted; this prints the newest few for the given - process names. + process names: the faulting thread, then every other thread's top frames + (task #72: the teardown race's other half is the main thread, in + `freeScript` or `exit`). The workflows also upload the raw reports. - Linux keeps nothing unless the job asked for core dumps (`.github/scripts/enable-core-dumps.sh`: `kernel.core_pattern` piped to a handler that writes into `$VAPOURBOX_CORE_DIR`, default /tmp/cores). This @@ -15,9 +17,21 @@ frame can still be resolved later (task #66). - Windows: nothing to read; says so. +Each crash is then classified: the first frame outside the system libraries +names the image that faulted, and one signature is recognised by name, +vspipe R78's teardown race (task #68): that frame is vspipe's own +`frameDoneCallback` or `outputFrame`. On macOS it arrives as SIGILL +(`pthread_cond_signal` on the freed condition variable traps in +`_os_unfair_lock_unowned_abort`), on Linux as SIGSEGV (task #72). The line +also says whether the installed bundle's vspipe carries the fix (its +`--version` marker, read from the binary). + Never fails the step: it is diagnostic. -Usage: print-crash-reports.py [process-name ...] (default: vspipe-bin python3) +Usage: + print-crash-reports.py [process-name ...] (default: vspipe-bin python3 python3.12) + print-crash-reports.py --classify FILE ... one line per file: FILEverdict + (.ips, or gdb backtrace text) """ import glob import json @@ -27,42 +41,171 @@ import subprocess import sys -NAMES = sys.argv[1:] or ["vspipe-bin", "python3", "python3.12"] +DEFAULT_NAMES = ["vspipe-bin", "python3", "python3.12"] NEWEST = 3 +TEARDOWN = "vspipe-teardown" +# The marker task #68's patch adds to `vspipe --version`; it is a literal in +# the binary, so it can be found without running vspipe (which needs the +# bundle's environment). +FIX_MARKER = b"VapourBox patch: vspipe frame-done count" +VSPIPE_IMAGES = {"vspipe", "vspipe-bin", "vspipe-bin.real", "vspipe.exe"} +VSPIPE_CALLBACKS = ("frameDoneCallback", "outputFrame") +TEARDOWN_MAIN = ("freeScript", "vpy4_freeScript", "exit", "__cxa_finalize") +_SYSTEM = re.compile( + r"^(libsystem_\w+\.dylib|libc\+\+(abi)?\.1\.dylib|libdyld\.dylib|dyld" + r"|libc\.so\.\d+|libpthread\.so\.\d+|libstdc\+\+\.so\.\d+|libgcc_s\.so\.\d+" + r"|ld-linux[-\w]*\.so\.\d+|\?\?)$" +) + + +def is_system(image): + return bool(_SYSTEM.match(image or "??")) + + +def classify(frames, other_threads=()): + """frames: the faulting thread, top first, as (image basename, symbol). + other_threads: the other threads' frames, the same way. Returns + (verdict, detail): verdict is TEARDOWN for task #68's race, else + "image:" for the first frame outside the system libraries, else + "unknown".""" + first = next(((img, sym) for img, sym in frames if not is_system(img)), None) + if first is None: + return "unknown", "every frame is in a system library" + img, sym = first + if img in VSPIPE_IMAGES and any(cb in sym for cb in VSPIPE_CALLBACKS): + main = next((s for t in other_threads for _, s in t[:12] + if any(m in s for m in TEARDOWN_MAIN)), None) + detail = f"{img} {sym.split('(')[0]}" + if main: + detail += f"; another thread is in {main.split('(')[0]}" + return TEARDOWN, detail + return f"image:{img}", sym + + +def vspipe_fix_state(): + """'yes' / 'no' for the installed bundle's vspipe, or None if not found.""" + deps = os.environ.get("VAPOURBOX_DEPS_DIR", "") + if not deps: + return None + for pattern in ("vapoursynth/vspipe-bin.real", "vapoursynth/vspipe-bin", + "vapoursynth/Lib/site-packages/vapoursynth/vspipe.exe"): + path = os.path.join(deps, pattern) + if os.path.isfile(path): + with open(path, "rb") as f: + return "yes" if FIX_MARKER in f.read() else "no" + return None + + +def verdict_line(verdict, detail): + if verdict == TEARDOWN: + fix = vspipe_fix_state() + has = {"yes": "the installed vspipe carries the fix, so this is NOT that race being unfixed: look again", + "no": "the installed vspipe does not carry the fix (deps before 1.18.0)", + None: "could not check the installed vspipe for the fix"}[fix] + return (f" => vspipe R78's teardown race (task #68): {detail}. " + f"Not a plugin or an instruction-set fault; {has}.") + if verdict.startswith("image:"): + return f" => faulted in {verdict[6:]}: {detail}" + return f" => {detail}" + + +# --- macOS ----------------------------------------------------------------- + +def ips_threads(path): + """(exception summary, faulting thread frames, other threads' frames).""" + with open(path, encoding="utf-8", errors="replace") as f: + f.readline() # one-line JSON header + body = json.loads(f.read()) + exc = body.get("exception", {}) + summary = f"{exc.get('type')} {exc.get('signal')} {exc.get('subtype', '')}".rstrip() + images = body.get("usedImages", []) + + def frames_of(thread): + out = [] + for frame in thread.get("frames", []): + idx = frame.get("imageIndex", -1) + image = images[idx] if 0 <= idx < len(images) else {} + lib = os.path.basename(image.get("path", image.get("name", "??")) or "??") + sym = frame.get("symbol", f"+{frame.get('imageOffset', 0):#x}") + out.append((lib, sym)) + return out + + threads = body.get("threads", []) + faulting = body.get("faultingThread", 0) + fault = frames_of(threads[faulting]) if faulting < len(threads) else [] + others = [(i, frames_of(t)) for i, t in enumerate(threads) if i != faulting] + return summary, fault, others + + +def print_ips(path): + print(f"=== {os.path.basename(path)}") + try: + summary, fault, others = ips_threads(path) + except (OSError, ValueError) as e: + print(f" (unparseable: {e})") + return + print(f" {summary}") + print(" --- faulting thread") + for i, (lib, sym) in enumerate(fault[:30]): + print(f" {i:2} {lib:32} {sym}") + for n, frames in others: + print(f" --- thread {n}") + for i, (lib, sym) in enumerate(frames[:8]): + print(f" {i:2} {lib:32} {sym}") + print(verdict_line(*classify(fault, [f for _, f in others]))) -def macos(): + +def macos(names): dirs = [os.path.expanduser("~/Library/Logs/DiagnosticReports"), "/Library/Logs/DiagnosticReports"] reports = sorted( (p for d in dirs for p in glob.glob(os.path.join(d, "*.ips")) - if any(os.path.basename(p).startswith(n) for n in NAMES)), + if any(os.path.basename(p).startswith(n) for n in names)), key=os.path.getmtime, reverse=True, )[:NEWEST] if not reports: - print(f"No crash reports for {', '.join(NAMES)}.") + print(f"No crash reports for {', '.join(names)}.") for path in reports: - print(f"=== {os.path.basename(path)}") - with open(path, encoding="utf-8", errors="replace") as f: - f.readline() # one-line JSON header - try: - body = json.loads(f.read()) - except json.JSONDecodeError as e: - print(f" (unparseable: {e})") - continue - exc = body.get("exception", {}) - print(f" {exc.get('type')} {exc.get('signal')} {exc.get('subtype', '')}".rstrip()) - images = body.get("usedImages", []) - threads = body.get("threads", []) - faulting = body.get("faultingThread", 0) - if faulting >= len(threads): - continue - for i, frame in enumerate(threads[faulting].get("frames", [])[:30]): - image = images[frame.get("imageIndex", -1)] if 0 <= frame.get("imageIndex", -1) < len(images) else {} - lib = os.path.basename(image.get("path", image.get("name", "?"))) - sym = frame.get("symbol", f"+{frame.get('imageOffset', 0):#x}") - print(f" {i:2} {lib:32} {sym}") + print_ips(path) + + +# --- Linux ----------------------------------------------------------------- + +_GDB_FRAME = re.compile(r"^\s*#\d+\s+(?:0x[0-9a-fA-F]+ in )?(.*?)(?: from (\S+))?\s*$") + + +def gdb_threads(text, exe="vspipe-bin"): + """(faulting thread frames, other threads' frames) from gdb's output. + The faulting thread is the block after "--- faulting thread" if this + script printed it, else the first backtrace in the text.""" + def parse(block): + out = [] + for line in block: + m = _GDB_FRAME.match(line) + if m: + sym, lib = m.group(1), m.group(2) + out.append((os.path.basename(lib) if lib else exe, sym)) + return out + + lines = text.splitlines() + blocks, cur, start = [], [], 0 # (index of the block's first line, its lines) + for i, line in enumerate(lines): + if _GDB_FRAME.match(line) and line.strip().startswith("#"): + if not cur: + start = i + cur.append(line) + elif cur: + blocks.append((start, cur)) + cur = [] + if cur: + blocks.append((start, cur)) + marker = next((i for i, l in enumerate(lines) if "--- faulting thread" in l), -1) + chosen = next((b for s, b in blocks if s > marker), None) + fault = parse(chosen) if chosen else [] + others = [parse(b) for _, b in blocks if b is not chosen] + return fault, others def core_executable(core): @@ -124,12 +267,34 @@ def linux(): print(f" executable: {exe}") for line in lines: print(f" {line}") + fault, others = gdb_threads(text, os.path.basename(exe)) + print(verdict_line(*classify(fault, others))) + +# --- command line ---------------------------------------------------------- -def main(): +def classify_file(path): + if path.endswith(".ips"): + _, fault, others = ips_threads(path) + return classify(fault, [f for _, f in others]) + with open(path, encoding="utf-8", errors="replace") as f: + fault, others = gdb_threads(f.read()) + return classify(fault, others) + + +def main(argv): + if argv and argv[0] == "--classify": + for path in argv[1:]: + try: + verdict, detail = classify_file(path) + except (OSError, ValueError) as e: + verdict, detail = "unreadable", str(e) + print(f"{path}\t{verdict}\t{detail}") + return + names = argv or DEFAULT_NAMES try: if sys.platform == "darwin": - macos() + macos(names) elif sys.platform.startswith("linux"): linux() else: @@ -139,4 +304,4 @@ def main(): if __name__ == "__main__": - main() + main(sys.argv[1:]) diff --git a/.github/scripts/replay-vspipe.sh b/.github/scripts/replay-vspipe.sh index 711c6f0b..9cd4d808 100755 --- a/.github/scripts/replay-vspipe.sh +++ b/.github/scripts/replay-vspipe.sh @@ -33,7 +33,8 @@ if [ -n "\$script" ] && grep -Eq '$MATCH' "\$script" 2>/dev/null; then d=\$(mktemp -d "$CAP/run.XXXXXX") cp "\$script" "\$d/script.vpy" printf '%s\0' "\$@" > "\$d/args" - env -0 > "\$d/env" + # BSD env (macOS 15) may lack -0; perl is on every runner. + env -0 > "\$d/env" 2>/dev/null || perl -e 'print "\$_=\$ENV{\$_}\0" for keys %ENV' > "\$d/env" cat > "\$d/stdin.raw" "\$REAL" "\$@" < "\$d/stdin.raw" rc=\$? @@ -73,7 +74,7 @@ run) case "$a" in *.vpy) args+=("$script") ;; *) args+=("$a") ;; esac done < "$r/args" ( - mode="$RV_MODE" vspipe="$RV_VSPIPE" + mode="$RV_MODE" vspipe="$RV_VSPIPE" RV_TIMEOUT="$RV_TIMEOUT" while IFS= read -r -d '' kv; do export "$kv" 2>/dev/null; done < "$r/env" pre=() if [ "$mode" = valgrind ]; then @@ -81,7 +82,9 @@ run) export PYTHONMALLOC=malloc pre=(valgrind --error-exitcode=99 --track-origins=yes --num-callers=40 --error-limit=no) fi - timeout 3600 "${pre[@]}" "$vspipe" "${args[@]}" < "$r/stdin.raw" > /dev/null 2> "$err" + # GNU timeout where there is one (macOS 15 has none built in). + tmo=(); [ -n "$RV_TIMEOUT" ] && tmo=("$RV_TIMEOUT" 3600) + "${tmo[@]}" "${pre[@]}" "$vspipe" "${args[@]}" < "$r/stdin.raw" > /dev/null 2> "$err" ) rc=$? printf '%s\t%s\t%s\n' "$i" "$(basename "$r")" "$rc" >> "$RV_OUT/results.tsv" @@ -90,6 +93,7 @@ run) } export -f one export RV_MODE="$MODE" RV_VSPIPE="$VSPIPE" RV_OUT="$OUT" + RV_TIMEOUT="$(command -v timeout || command -v gtimeout || true)"; export RV_TIMEOUT : > "$OUT/results.tsv" start=$(date +%s) for i in $(seq 1 "$REPEAT"); do for r in "${runs[@]}"; do printf '%s\0%s\0' "$i" "$r"; done; done \ diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 719f9287..f445961c 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -130,6 +130,26 @@ jobs: if: failure() run: python3 .github/scripts/print-crash-reports.py + - name: Keep the raw crash reports (on failure) + # The .ips files themselves, for symbolication or a second look: the + # step above prints only their top frames (task #72). + if: failure() + run: | + mkdir -p crash-reports + for f in ~/Library/Logs/DiagnosticReports/{vspipe-bin,python3,vapourbox-worker}*.ips; do + [ -f "$f" ] && cp "$f" crash-reports/ + done + ls crash-reports + + - name: Upload the raw crash reports (on failure) + if: failure() + uses: actions/upload-artifact@v5 + with: + name: crash-reports-macos-${{ matrix.arch }} + path: crash-reports/*.ips + if-no-files-found: ignore + retention-days: 14 + windows: runs-on: windows-latest env: diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 3056351e..77f2575a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -157,6 +157,26 @@ jobs: if: failure() run: python3 .github/scripts/print-crash-reports.py + - name: Keep the raw crash reports (on failure) + # The .ips files themselves, for symbolication or a second look: the + # step above prints only their top frames (task #72). + if: failure() + run: | + mkdir -p crash-reports + for f in ~/Library/Logs/DiagnosticReports/{vspipe-bin,python3,vapourbox-worker}*.ips; do + [ -f "$f" ] && cp "$f" crash-reports/ + done + ls crash-reports + + - name: Upload the raw crash reports (on failure) + if: failure() + uses: actions/upload-artifact@v5 + with: + name: crash-reports-${{ matrix.asset }}-${{ matrix.suite }} + path: crash-reports/*.ips + if-no-files-found: ignore + retention-days: 14 + windows: if: ${{ github.event_name == 'schedule' || inputs.platform == 'all' || inputs.platform == 'windows' }} # Both CPU-tier bundles (issue #92); see the macOS job for what the v2 run proves. diff --git a/CLAUDE.md b/CLAUDE.md index 8e5ef244..7d38cb95 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -3615,6 +3615,14 @@ Rules: in vspipe's own `frameDoneCallback` / `outputFrame` and main in `freeScript` is vspipe R78's teardown race** (task #68, fixed in R80), not the script: any one-frame preview can hit it, roughly 1 in 1,000. + **On macOS the same race is a SIGILL** (task #72): `pthread_cond_signal` + from vspipe's `frameDoneCallback` on the condition variable main has + freed traps in `_os_unfair_lock_unowned_abort`. It is not an instruction + the CPU lacks. The crash-report step prints every thread, names this + signature and says whether the bundle's vspipe has the fix + (`print-crash-reports.py --classify FILE…` does it for any `.ips` or + gdb text); the macOS jobs upload the raw `.ips` as + `crash-reports-*` artifacts. **A memcheck "uninitialised" report at more than 8 bits and an odd plane width is expected and harmless on its own** (task #70): VapourSynth's AVX2 kernels read stride padding into lanes they discard, diff --git a/Tests/TestResources/crash_reports/README.md b/Tests/TestResources/crash_reports/README.md new file mode 100644 index 00000000..38b0a79c --- /dev/null +++ b/Tests/TestResources/crash_reports/README.md @@ -0,0 +1,19 @@ +# Crash-report fixtures (task #72) + +Inputs for `app/test/crash_report_classifier_test.dart`, which runs +`.github/scripts/print-crash-reports.py --classify` over them. + +- `macos-x64-vspipe-teardown-sigill.ips`: the faulting thread of + `vspipe-bin-2026-11-18-023804.ips` from the push gate's macOS x64 job, + run 44088351262 attempt 2 (deps 1.17.0-rc3), rebuilt as an `.ips` from the + frames CI printed. CI printed only the faulting thread, so it has only that + thread; the image paths are placeholders. vspipe R78's teardown race (task + #68) as macOS reports it: SIGILL in `_os_unfair_lock_unowned_abort`. +- `macos-arm64-moltenvk-sigsegv.ips`: run 44064242986's macOS arm64 + `vspipe-bin-2026-11-17-103557.ips` (RIFE creating its Vulkan device + through MoltenVK), rebuilt the same way, with some frames left out. A + crash that is not the race. +- `linux-vspipe-teardown-sigsegv.gdb.txt`: the gdb output CI printed for + `core.vspipe-bin.52015.1795010338`, run 44318045317's Linux job, from + "faulting thread" to the end of "every thread (top frames)". The same race + on Linux: SIGSEGV in `outputFrame`, main in `vpy4_freeScript`. diff --git a/Tests/TestResources/crash_reports/linux-vspipe-teardown-sigsegv.gdb.txt b/Tests/TestResources/crash_reports/linux-vspipe-teardown-sigsegv.gdb.txt new file mode 100644 index 00000000..f771ec2a --- /dev/null +++ b/Tests/TestResources/crash_reports/linux-vspipe-teardown-sigsegv.gdb.txt @@ -0,0 +1,81 @@ +[New LWP 52133] +[New LWP 52136] +[New LWP 52135] +[New LWP 52134] +[New LWP 52015] +[Thread debugging using libthread_db enabled] +Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". +Core was generated by `/home/runner/.local/share/VapourBox/deps/linux-x64/vapoursynth/vspipe-bin -c y4'. +Program terminated with signal SIGSEGV, Segmentation fault. +#0 0x000055c366d8e04b in outputFrame(VSFrame const*, VSPipeOutputData*) () +[Current thread is 1 (Thread 0x7fac173fe6c0 (LWP 52133))] + +--- faulting thread +#0 0x000055c366d8e04b in outputFrame(VSFrame const*, VSPipeOutputData*) () +#1 0x000055c366d8eaad in frameDoneCallback(void*, VSFrame const*, int, VSNode*, char const*) [clone .lto_priv.0] () +#2 0x00007fac1e92ef8b in VSThreadPool::returnFrame(VSFrameContext*, vs_intrusive_ptr const&, std::unique_lock&) () from /home/runner/.local/share/VapourBox/deps/linux-x64/vapoursynth/libvapoursynth.so.4 +#3 0x00007fac1e931b25 in VSThreadPool::runTasks(bool&) () from /home/runner/.local/share/VapourBox/deps/linux-x64/vapoursynth/libvapoursynth.so.4 +#4 0x00007fac20eecdb4 in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6 +#5 0x00007fac20a9cb84 in ?? () from /lib/x86_64-linux-gnu/libc.so.6 +#6 0x00007fac20b29ecc in ?? () from /lib/x86_64-linux-gnu/libc.so.6 + +--- registers +rip 0x55c366d8e04b 0x55c366d8e04b +rsp 0x7fac173fd560 0x7fac173fd560 +rbp 0x7fac100bbdb0 0x7fac100bbdb0 +rdi 0x7fac100bbdb0 140376980307376 +rsi 0x1 1 +rdx 0x1 1 +rcx 0x7fac20a99098 140377259085976 +=> 0x55c366d8e04b <_ZL11outputFramePK7VSFrameP16VSPipeOutputData+587>: + 0x55c366d8e051 <_ZL11outputFramePK7VSFrameP16VSPipeOutputData+593>: + 0x55c366d8e054 <_ZL11outputFramePK7VSFrameP16VSPipeOutputData+596>: + 0x55c366d8e057 <_ZL11outputFramePK7VSFrameP16VSPipeOutputData+599>: + 0x55c366d8e05a <_ZL11outputFramePK7VSFrameP16VSPipeOutputData+602>: + 0x55c366d8e05d <_ZL11outputFramePK7VSFrameP16VSPipeOutputData+605>: + +--- every thread (top frames) + +Thread 5 (Thread 0x7fac21152740 (LWP 52015)): +#0 0x00007fac1f44432b in visit_reachable () from /home/runner/work/VapourBox/VapourBox/deps/linux-x64/python/lib/libpython3.12.so.1.0 +#1 0x00007fac1f39560e in dict_traverse () from /home/runner/work/VapourBox/VapourBox/deps/linux-x64/python/lib/libpython3.12.so.1.0 +#2 0x00007fac1f444277 in deduce_unreachable () from /home/runner/work/VapourBox/VapourBox/deps/linux-x64/python/lib/libpython3.12.so.1.0 +#3 0x00007fac1f4427f5 in gc_collect_main.llvm () from /home/runner/work/VapourBox/VapourBox/deps/linux-x64/python/lib/libpython3.12.so.1.0 +#4 0x00007fac1f574275 in gc_collect () from /home/runner/work/VapourBox/VapourBox/deps/linux-x64/python/lib/libpython3.12.so.1.0 +#5 0x00007fac1f398876 in cfunction_vectorcall_FASTCALL_KEYWORDS.llvm.16519472471476773533 () from /home/runner/work/VapourBox/VapourBox/deps/linux-x64/python/lib/libpython3.12.so.1.0 +#6 0x00007fac1f36b486 in PyObject_Vectorcall () from /home/runner/work/VapourBox/VapourBox/deps/linux-x64/python/lib/libpython3.12.so.1.0 +#7 0x00007fac1eaaced2 in vpy4_freeScript(VSScript*) () from /home/runner/.local/share/VapourBox/deps/linux-x64/vapoursynth/vapoursynth.abi3.so + +Thread 4 (Thread 0x7fac163fd6c0 (LWP 52134)): +#0 0x00007fac20a98e51 in ?? () from /lib/x86_64-linux-gnu/libc.so.6 +#1 0x00007fac20a9b8cd in pthread_cond_wait () from /lib/x86_64-linux-gnu/libc.so.6 +#2 0x00007fac1e930975 in VSThreadPool::runTasks(bool&) () from /home/runner/.local/share/VapourBox/deps/linux-x64/vapoursynth/libvapoursynth.so.4 +#3 0x00007fac20eecdb4 in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6 +#4 0x00007fac20a9cb84 in ?? () from /lib/x86_64-linux-gnu/libc.so.6 +#5 0x00007fac20b29ecc in ?? () from /lib/x86_64-linux-gnu/libc.so.6 + +Thread 3 (Thread 0x7fac153fc6c0 (LWP 52135)): +#0 0x00007fac20a98e51 in ?? () from /lib/x86_64-linux-gnu/libc.so.6 +#1 0x00007fac20a9b8cd in pthread_cond_wait () from /lib/x86_64-linux-gnu/libc.so.6 +#2 0x00007fac1e930975 in VSThreadPool::runTasks(bool&) () from /home/runner/.local/share/VapourBox/deps/linux-x64/vapoursynth/libvapoursynth.so.4 +#3 0x00007fac20eecdb4 in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6 +#4 0x00007fac20a9cb84 in ?? () from /lib/x86_64-linux-gnu/libc.so.6 +#5 0x00007fac20b29ecc in ?? () from /lib/x86_64-linux-gnu/libc.so.6 + +Thread 2 (Thread 0x7fac0ffff6c0 (LWP 52136)): +#0 0x00007fac20a98e51 in ?? () from /lib/x86_64-linux-gnu/libc.so.6 +#1 0x00007fac20a9b8cd in pthread_cond_wait () from /lib/x86_64-linux-gnu/libc.so.6 +#2 0x00007fac1e930975 in VSThreadPool::runTasks(bool&) () from /home/runner/.local/share/VapourBox/deps/linux-x64/vapoursynth/libvapoursynth.so.4 +#3 0x00007fac20eecdb4 in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6 +#4 0x00007fac20a9cb84 in ?? () from /lib/x86_64-linux-gnu/libc.so.6 +#5 0x00007fac20b29ecc in ?? () from /lib/x86_64-linux-gnu/libc.so.6 + +Thread 1 (Thread 0x7fac173fe6c0 (LWP 52133)): +#0 0x000055c366d8e04b in outputFrame(VSFrame const*, VSPipeOutputData*) () +#1 0x000055c366d8eaad in frameDoneCallback(void*, VSFrame const*, int, VSNode*, char const*) [clone .lto_priv.0] () +#2 0x00007fac1e92ef8b in VSThreadPool::returnFrame(VSFrameContext*, vs_intrusive_ptr const&, std::unique_lock&) () from /home/runner/.local/share/VapourBox/deps/linux-x64/vapoursynth/libvapoursynth.so.4 +#3 0x00007fac1e931b25 in VSThreadPool::runTasks(bool&) () from /home/runner/.local/share/VapourBox/deps/linux-x64/vapoursynth/libvapoursynth.so.4 +#4 0x00007fac20eecdb4 in ?? () from /lib/x86_64-linux-gnu/libstdc++.so.6 +#5 0x00007fac20a9cb84 in ?? () from /lib/x86_64-linux-gnu/libc.so.6 +#6 0x00007fac20b29ecc in ?? () from /lib/x86_64-linux-gnu/libc.so.6 + diff --git a/Tests/TestResources/crash_reports/macos-arm64-moltenvk-sigsegv.ips b/Tests/TestResources/crash_reports/macos-arm64-moltenvk-sigsegv.ips new file mode 100644 index 00000000..3c7c147f --- /dev/null +++ b/Tests/TestResources/crash_reports/macos-arm64-moltenvk-sigsegv.ips @@ -0,0 +1,73 @@ +{"app_name": "vspipe-bin", "bug_type": "309"} +{ + "exception": { + "type": "EXC_BAD_ACCESS", + "signal": "SIGSEGV", + "subtype": "KERN_INVALID_ADDRESS at 0x000000000000000c" + }, + "faultingThread": 0, + "usedImages": [ + { + "name": "AppleParavirtGPUMetalIOGPUFamily", + "path": "/x/AppleParavirtGPUMetalIOGPUFamily" + }, + { + "name": "libMoltenVK.dylib", + "path": "/x/libMoltenVK.dylib" + }, + { + "name": "librife.dylib", + "path": "/x/librife.dylib" + }, + { + "name": "libvapoursynth.4.dylib", + "path": "/x/libvapoursynth.4.dylib" + }, + { + "name": "vspipe-bin", + "path": "/x/vspipe-bin" + }, + { + "name": "dyld", + "path": "/x/dyld" + } + ], + "threads": [ + { + "frames": [ + { + "imageIndex": 0, + "imageOffset": 27772 + }, + { + "imageIndex": 1, + "symbol": "MVKImagePlane::updateMTLTextureContent(MVKImageSubresource&, unsigned long long, unsigned long long)" + }, + { + "imageIndex": 1, + "symbol": "vkBindImageMemory" + }, + { + "imageIndex": 2, + "symbol": "ncnn::VkBlobAllocator::fastMalloc(int, int, int, unsigned long, int)" + }, + { + "imageIndex": 2, + "symbol": "rifeCreate(VSMap const*, VSMap*, void*, VSCore*, VSAPI const*)" + }, + { + "imageIndex": 3, + "symbol": "invoke(VSPlugin*, char const*, VSMap const*)" + }, + { + "imageIndex": 4, + "symbol": "main" + }, + { + "imageIndex": 5, + "symbol": "start" + } + ] + } + ] +} diff --git a/Tests/TestResources/crash_reports/macos-x64-vspipe-teardown-sigill.ips b/Tests/TestResources/crash_reports/macos-x64-vspipe-teardown-sigill.ips new file mode 100644 index 00000000..f28a5c6c --- /dev/null +++ b/Tests/TestResources/crash_reports/macos-x64-vspipe-teardown-sigill.ips @@ -0,0 +1,76 @@ +{"app_name": "vspipe-bin", "bug_type": "309"} +{ + "exception": { + "type": "EXC_BAD_INSTRUCTION", + "signal": "SIGILL" + }, + "faultingThread": 0, + "usedImages": [ + { + "name": "libsystem_platform.dylib", + "path": "/x/libsystem_platform.dylib" + }, + { + "name": "libsystem_pthread.dylib", + "path": "/x/libsystem_pthread.dylib" + }, + { + "name": "libc++.1.dylib", + "path": "/x/libc++.1.dylib" + }, + { + "name": "vspipe-bin", + "path": "/x/vspipe-bin" + }, + { + "name": "libvapoursynth.4.dylib", + "path": "/x/libvapoursynth.4.dylib" + } + ], + "threads": [ + { + "frames": [ + { + "imageIndex": 0, + "symbol": "_os_unfair_lock_unowned_abort" + }, + { + "imageIndex": 0, + "symbol": "_os_unfair_lock_unlock_slow" + }, + { + "imageIndex": 1, + "symbol": "_pthread_cond_check_init_slow" + }, + { + "imageIndex": 1, + "symbol": "pthread_cond_signal" + }, + { + "imageIndex": 2, + "symbol": "std::__1::condition_variable::notify_one()" + }, + { + "imageIndex": 3, + "symbol": "frameDoneCallback(void*, VSFrame const*, int, VSNode*, char const*)" + }, + { + "imageIndex": 4, + "symbol": "VSThreadPool::returnFrame(VSFrameContext*, vs_intrusive_ptr const&, std::__1::unique_lock&)" + }, + { + "imageIndex": 4, + "symbol": "VSThreadPool::runTasks(bool&)" + }, + { + "imageIndex": 1, + "symbol": "_pthread_start" + }, + { + "imageIndex": 1, + "symbol": "thread_start" + } + ] + } + ] +} diff --git a/app/test/crash_report_classifier_test.dart b/app/test/crash_report_classifier_test.dart new file mode 100644 index 00000000..62d26ab2 --- /dev/null +++ b/app/test/crash_report_classifier_test.dart @@ -0,0 +1,65 @@ +// The CI crash-report step names vspipe R78's teardown race (task #68) for +// what it is, on both platforms, and names the faulting image of anything +// else (task #72). +// +// The macOS x64 push gate failed a random format-sweep row in 4 of its 20 +// runs on deps 1.17.0-rc3 with "vspipe exited with signal 4 (SIGILL)", which +// reads like a plugin using an instruction the runner lacks. It was the teardown race: vspipe's +// frameDoneCallback signalling a condition variable main had already freed, +// which macOS traps in _os_unfair_lock_unowned_abort (SIGILL); on Linux the +// same race is a SIGSEGV in outputFrame. The fixtures are those crashes as +// CI printed them (Tests/TestResources/crash_reports/README.md). +import 'dart:io'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; + +import 'support/worker_harness.dart'; + +void main() { + final script = p.join(WorkerHarness.repoRoot, '.github', 'scripts', 'print-crash-reports.py'); + final fixtures = p.join(WorkerHarness.repoRoot, 'Tests', 'TestResources', 'crash_reports'); + + // The system Python, as the workflows' crash-report step runs it; the script + // needs nothing but the standard library. + final python = Platform.isWindows ? 'python' : 'python3'; + + Future>> classify(List names) async { + final r = await Process.run( + python, + ['-I', script, '--classify', ...names.map((n) => p.join(fixtures, n))], + ); + expect(r.exitCode, 0, reason: '${r.stdout}\n${r.stderr}'); + return { + for (final line in (r.stdout as String).replaceAll('\r\n', '\n').trim().split('\n')) + p.basename(line.split('\t').first): line.split('\t').sublist(1), + }; + } + + test('the macOS SIGILL and the Linux SIGSEGV are both the teardown race', () async { + final out = await classify([ + 'macos-x64-vspipe-teardown-sigill.ips', + 'linux-vspipe-teardown-sigsegv.gdb.txt', + ]); + expect(out['macos-x64-vspipe-teardown-sigill.ips']![0], 'vspipe-teardown'); + expect(out['macos-x64-vspipe-teardown-sigill.ips']![1], contains('frameDoneCallback')); + expect(out['linux-vspipe-teardown-sigsegv.gdb.txt']![0], 'vspipe-teardown'); + expect(out['linux-vspipe-teardown-sigsegv.gdb.txt']![1], contains('outputFrame')); + expect(out['linux-vspipe-teardown-sigsegv.gdb.txt']![1], contains('vpy4_freeScript'), + reason: "the race's other half: main is freeing the script"); + }); + + test('a crash inside a plugin or driver is named by its image, not as the race', () async { + final out = await classify(['macos-arm64-moltenvk-sigsegv.ips']); + final verdict = out['macos-arm64-moltenvk-sigsegv.ips']![0]; + expect(verdict, isNot('vspipe-teardown')); + expect(verdict, 'image:AppleParavirtGPUMetalIOGPUFamily'); + }); + + test('an unreadable file is reported, never a failed step', () async { + final missing = p.join(fixtures, 'does-not-exist.ips'); + final r = await Process.run(python, ['-I', script, '--classify', missing]); + expect(r.exitCode, 0); + expect(r.stdout as String, contains('unreadable')); + }); +} diff --git a/docs/ENGINEERING_NOTES.md b/docs/ENGINEERING_NOTES.md index e8515fef..01c04401 100644 --- a/docs/ENGINEERING_NOTES.md +++ b/docs/ENGINEERING_NOTES.md @@ -5392,6 +5392,31 @@ consumes a significant amount of CPU" (Task Manager: vspipe.exe 50%). in-process plugin crashes take the encoder down, preview needs the same change. +### The macOS x64 sweep SIGILL was vspipe's teardown race (task #72, 2026-11-19) + +Report: [docs/reports/task-72-macos-x64-sigill](reports/task-72-macos-x64-sigill/index.html). + +- **A SIGILL is not always an instruction the CPU lacks.** On deps 1.17.0-rc3 + the macOS x64 push gate lost a random format-sweep row in 4 of 20 runs + (5 rows) to `vspipe exited with signal 4 (SIGILL)`. Every crash report has + vspipe's own `frameDoneCallback` → `notify_one` → `pthread_cond_signal` → + `_os_unfair_lock_unowned_abort`: macOS's deliberate `ud2` when a lock's + owner field is garbage, here because main had freed the condition variable + (task #68's race). No plugin is on the stack, so neither CPU dispatch nor a + Zig safety trap. Linux shows the same race as SIGSEGV in `outputFrame`; + on macOS it is SIGSEGV too when the callback is still writing the frame. +- **Measured fixed on x86.** With only `vspipe-bin` swapped in a 1.17.0-rc3 + bundle: R78 71 of 32,160 one-frame runs crashed, deps 1.18.0-rc1's 0 + (Linux x64 66 vs 0, macOS x64 2 vs 0, Windows x64 3 vs 0). Since rc1, 8 + macOS x64 push-gate jobs (4,704 rows) had no SIGILL (2.0 expected). +- **Replaying captured sweep runs did not reproduce it** (0 races in 6,340 + replays a side on macOS x64): a run replayed alone is too quiet. Use the + loops of task #68's `compare_vspipe.sh` under CPU load instead. +- **No retry or quarantine**: the sweep's native-crash class still finds real + plugin bugs (tasks #69, #70). `print-crash-reports.py` now names the + signature and whether the bundle has the fix, and the macOS jobs upload the + raw `.ips`. + ### vspipe's teardown race: upstream's fix backported to R78, not R80 (task #68, 2026-11-19) Report: [docs/reports/task-68-vspipe-teardown-backport](reports/task-68-vspipe-teardown-backport/index.html). diff --git a/docs/reports/README.md b/docs/reports/README.md index a2bd7ee8..5592acc6 100644 --- a/docs/reports/README.md +++ b/docs/reports/README.md @@ -128,5 +128,6 @@ to it. | 2026-11-17 | [Task #60: Bifrost v3.0 → v4.0](task-60-bifrost-v4/index.html) | Built: every platform takes Bifrost 4.0 from its PyPI wheel. v3.0's chroma past the last whole 4x4 block was uninitialised (2–3 results in 3 runs at width 98/718, 720x486); v4.0 gives one, identical to v3.0 inside whole blocks. Runs natively at 8–16 bits, luma bit-exact, 0.1% of chroma decisions differing from 8-bit at the threshold; the 8-bit copy is gone. 4:2:0 at 720x486 with `interlaced=True` is still refused (task #57) | | 2026-11-17 | [Task #58: what the output conformance matrix found, fixed](task-58-output-conformance-findings/index.html) | Fixed all five: `pipe_source` keeps frames until served and raises on a dropped one (6/320 wrong runs under load → 0); primed AVI trims end with `-t`, not `-shortest` (49 → 50 pictures, Convert trimmed to the end too); MP1/MP2 rendered with the float decoder (exact from any seek); the reset join decided by the picture (`-copyts` + `setts`); the delayed AVI's late click was the oracle's (the source's own 23 ms). `knownFailures` empty | | 2026-11-18 | [Task #65: the IVTC preview's vspipe crash, and IVTC parity](task-65-ivtc-preview-teardown/index.html) | Fixed: vspipe R78 counts a frame done before writing it, so a one-frame preview whose frame is ready at once can be torn down under its callback (SIGSEGV; fixed upstream in R80). Task #46's evaluation-time `get_frame()` made the IVTC preview's frame ready at once: 2–3% of x86 IVTC previews under load crashed, 0/3960 with the read moved into the graph. Any preview can still hit it rarely (2/1200 Bwdif previews on macOS x64) until the bundle has R80+. The IVTC parity case needed a telecined fixture | -| 2026-11-19 | [Task #68: vspipe's teardown race, backported to R78](task-68-vspipe-teardown-backport/index.html) | Built: R80+ is blocked (it dropped API 3; 12 bundled plugins are API 3 only, most with no API 4 release), so upstream's vspipe fix (4ff3f8a0) is applied to R78 on every platform, Windows by rebuilding `vspipe.exe` with Zig. With the race window widened, unpatched R78 crashed 400/400 runs and the patch 0/400; the x86 A/B on the built bundles is still to run | +| 2026-11-19 | [Task #68: vspipe's teardown race, backported to R78](task-68-vspipe-teardown-backport/index.html) | Built: R80+ is blocked (it dropped API 3; 12 bundled plugins are API 3 only, most with no API 4 release), so upstream's vspipe fix (4ff3f8a0) is applied to R78 on every platform, Windows by rebuilding `vspipe.exe` with Zig. With the race window widened, unpatched R78 crashed 400/400 runs and the patch 0/400; the x86 A/B on the built bundles is still to run (2026-11-19, measured in task #72: R78 71/32,160 crashes, patched 0/32,160) | | 2026-11-19 | [Task #73: matrices FFmpeg 9's scale can't name](task-73-swscale-matrix-tags/index.html) | Fixed: the preview's `in_color_matrix` goes through `ColorMetadata::swscale_matrix`. `scale` has no conversion (by name or number) for ycgco, bt2020c, smpte2085, chroma-derived-* or ictcp, so each SDR preview failed; now mapped (bt2020c/ictcp → bt2020, chroma-derived by primaries) or left to auto (ycgco, smpte2085). A frame tagged with one of them can't be converted at all (before frame, thumbnails, detectors): task #76 | +| 2026-11-19 | [Task #72: the macOS x64 format-sweep SIGILL was vspipe's teardown race](task-72-macos-x64-sigill/index.html) | Diagnosed, fixed by deps 1.18.0-rc1: every macOS x64 sweep SIGILL on rc3 (5 rows, 4 runs) has vspipe's `frameDoneCallback` signalling a freed condition variable, which macOS traps (`_os_unfair_lock_unowned_abort`, `ud2`); Linux shows the same race as SIGSEGV. No plugin on the stack. x86 A/B with only `vspipe-bin` swapped: R78 71/32,160 crashes, patched 0/32,160 (Linux 66 vs 0, macOS 2 vs 0, Windows 3 vs 0). The CI crash-report step now names the race; no retry added | diff --git a/docs/reports/task-68-vspipe-teardown-backport/index.html b/docs/reports/task-68-vspipe-teardown-backport/index.html index e7b51788..efeab58c 100644 --- a/docs/reports/task-68-vspipe-teardown-backport/index.html +++ b/docs/reports/task-68-vspipe-teardown-backport/index.html @@ -23,7 +23,6 @@ -

Task #68: vspipe's teardown race, backported to R78

2026-11-19 · macOS arm64 (M1 Pro), deps 1.17.0-rc3 · scripts in scripts/, raw counts in data/

@@ -102,6 +101,11 @@

4. Not yet measured: the x86 A/B on the built bundles

pre-#65 template, which crashed 2–3%) on Linux x64, macOS x64 and Windows x64. That workflow (.github/workflows/repro-vspipe-teardown.yml, temporary) had not run when this report was written; its counts belong here when it has.

+

2026-11-19, measured in task #72 (run 44500126463, +its report, section 3): with only vspipe-bin swapped in a +1.17.0-rc3 bundle, R78 crashed 71 of 32,160 one-frame runs and the 1.18.0-rc1 vspipe 0 (Linux x64 66 vs 0, macOS x64 2 vs 0, +Windows x64 3 vs 0), with this report's compare_vspipe.sh under CPU load. The workflow is kept in task #72's +scripts/.

diff --git a/docs/reports/task-72-macos-x64-sigill/data/ab-loops-linux-x64.txt b/docs/reports/task-72-macos-x64-sigill/data/ab-loops-linux-x64.txt new file mode 100644 index 00000000..9fdd073f --- /dev/null +++ b/docs/reports/task-72-macos-x64-sigill/data/ab-loops-linux-x64.txt @@ -0,0 +1,6 @@ +unpatched trigger (4 loops, CPU load): runs=4000 failures=0 +unpatched ivtc (4 loops, CPU load): runs=3360 failures=66 +unpatched bwdif (4 loops, CPU load): runs=3360 failures=0 +patched trigger (4 loops, CPU load): runs=4000 failures=0 +patched ivtc (4 loops, CPU load): runs=3360 failures=0 +patched bwdif (4 loops, CPU load): runs=3360 failures=0 diff --git a/docs/reports/task-72-macos-x64-sigill/data/ab-loops-macos-x64-crash-reports.tsv b/docs/reports/task-72-macos-x64-sigill/data/ab-loops-macos-x64-crash-reports.tsv new file mode 100644 index 00000000..0d5198a0 --- /dev/null +++ b/docs/reports/task-72-macos-x64-sigill/data/ab-loops-macos-x64-crash-reports.tsv @@ -0,0 +1,2 @@ +crash-reports/vspipe-bin-2026-11-19-095910.ips vspipe-teardown vspipe-bin frameDoneCallback +crash-reports/vspipe-bin-2026-11-19-101530.ips vspipe-teardown vspipe-bin outputFrame diff --git a/docs/reports/task-72-macos-x64-sigill/data/ab-loops-macos-x64.txt b/docs/reports/task-72-macos-x64-sigill/data/ab-loops-macos-x64.txt new file mode 100644 index 00000000..56823ca2 --- /dev/null +++ b/docs/reports/task-72-macos-x64-sigill/data/ab-loops-macos-x64.txt @@ -0,0 +1,6 @@ +unpatched trigger (4 loops, CPU load): runs=4000 failures=1 +unpatched ivtc (4 loops, CPU load): runs=3360 failures=1 +unpatched bwdif (4 loops, CPU load): runs=3360 failures=0 +patched trigger (4 loops, CPU load): runs=4000 failures=0 +patched ivtc (4 loops, CPU load): runs=3360 failures=0 +patched bwdif (4 loops, CPU load): runs=3360 failures=0 diff --git a/docs/reports/task-72-macos-x64-sigill/data/ab-loops-windows-x64.txt b/docs/reports/task-72-macos-x64-sigill/data/ab-loops-windows-x64.txt new file mode 100644 index 00000000..38949a9c --- /dev/null +++ b/docs/reports/task-72-macos-x64-sigill/data/ab-loops-windows-x64.txt @@ -0,0 +1,6 @@ +unpatched trigger (4 loops, CPU load): runs=4000 failures=1 +unpatched ivtc (4 loops, CPU load): runs=3360 failures=2 +unpatched bwdif (4 loops, CPU load): runs=3360 failures=0 +patched trigger (4 loops, CPU load): runs=4000 failures=0 +patched ivtc (4 loops, CPU load): runs=3360 failures=0 +patched bwdif (4 loops, CPU load): runs=3360 failures=0 diff --git a/docs/reports/task-72-macos-x64-sigill/data/ab-sweep-linux-x64.txt b/docs/reports/task-72-macos-x64-sigill/data/ab-sweep-linux-x64.txt new file mode 100644 index 00000000..478cf190 --- /dev/null +++ b/docs/reports/task-72-macos-x64-sigill/data/ab-sweep-linux-x64.txt @@ -0,0 +1,2 @@ +sweep unpatched (fix marker: no): runs=6300 failures=60 (60 exit 1; ); cores: 0 classified (0 the race) +sweep patched (fix marker: yes): runs=6300 failures=60 (60 exit 1; ); cores: 0 classified (0 the race) diff --git a/docs/reports/task-72-macos-x64-sigill/data/ab-sweep-macos-x64.txt b/docs/reports/task-72-macos-x64-sigill/data/ab-sweep-macos-x64.txt new file mode 100644 index 00000000..5eb78a20 --- /dev/null +++ b/docs/reports/task-72-macos-x64-sigill/data/ab-sweep-macos-x64.txt @@ -0,0 +1,2 @@ +sweep unpatched (fix marker: no): runs=6340 failures=100 (20 exit 139; 80 exit 1; ); crash reports: 20 image:OpenCL, +sweep patched (fix marker: yes): runs=6340 failures=100 (20 exit 139; 80 exit 1; ); crash reports: 6 image:OpenCL, diff --git a/docs/reports/task-72-macos-x64-sigill/data/ci-failures.tsv b/docs/reports/task-72-macos-x64-sigill/data/ci-failures.tsv new file mode 100644 index 00000000..d050757b --- /dev/null +++ b/docs/reports/task-72-macos-x64-sigill/data/ci-failures.tsv @@ -0,0 +1,20 @@ +run job deps failed tests native-crash rows crash reports +44062306190 linux deps-v1.17.0-rc2 integration_format_sweep_test.dart: format sweep (push gate) deinterlace/qtgmc[preset=Draft] 98x64/yuv444p10le/tff signal 11 (SIGSEGV) +44064242986 linux deps-v1.17.0-rc2 integration_format_sweep_test.dart: format sweep (push gate) chroma_fixes/applyBifrost=true | plugin_format_contracts_test.dart: plugin_formats.CONTRACTS states what the table measured | plugin_format_contracts_test.dart: the bundle still refuses exactly what the table says (quick probe) | vapoursynth_integration_test.dart: VapourSynth Plugin Loading the bundled Bifrost is v4.0 or later: it runs at 10-bit +44064242986 windows deps-v1.17.0-rc2 integration_format_sweep_test.dart: format sweep (push gate) chroma_fixes/applyBifrost=true | plugin_format_contracts_test.dart: plugin_formats.CONTRACTS states what the table measured | plugin_format_contracts_test.dart: the bundle still refuses exactly what the table says (quick probe) | vapoursynth_integration_test.dart: VapourSynth Plugin Loading the bundled Bifrost is v4.0 or later: it runs at 10-bit +44064242986 macos (x64, macos-15-intel) deps-v1.17.0-rc2 integration_format_sweep_test.dart: format sweep (push gate) chroma_fixes/applyBifrost=true | plugin_format_contracts_test.dart: plugin_formats.CONTRACTS states what the table measured | plugin_format_contracts_test.dart: the bundle still refuses exactly what the table says (quick probe) | vapoursynth_integration_test.dart: VapourSynth Plugin Loading the bundled Bifrost is v4.0 or later: it runs at 10-bit +44064242986 macos (arm64, macos-15) deps-v1.17.0-rc2 integration_format_sweep_test.dart: format sweep (push gate) chroma_fixes/applyBifrost=true | plugin_format_contracts_test.dart: plugin_formats.CONTRACTS states what the table measured | plugin_format_contracts_test.dart: the bundle still refuses exactly what the table says (quick probe) | vapoursynth_integration_test.dart: VapourSynth Plugin Loading the bundled Bifrost is v4.0 or later: it runs at 10-bit vspipe-bin-2026-11-17-103557.ips: image:AppleParavirtGPUMetalIOGPUFamily (+0x6c7c) +44078793187 linux deps-v1.17.0-rc3 integration_format_sweep_test.dart: format sweep (push gate) crop_resize/padToAspect=true+resizeEnabled=true 98x64/yuv420p10le/tff signal 11 (SIGSEGV) +44087946739 linux deps-v1.17.0-rc3 integration_format_sweep_test.dart: format sweep (push gate) chroma_fixes/applyDedot=true 98x64/yuv420p12le/prog signal 11 (SIGSEGV) +44087946739 macos (x64, macos-15-intel) deps-v1.17.0-rc3 integration_format_sweep_test.dart: format sweep (push gate) deinterlace/qtgmc[preset=Fast] 96x64/yuvj420p/tff signal 4 (SIGILL) vspipe-bin-2026-11-17-131146.ips: vspipe-teardown (vspipe-bin frameDoneCallback) +44088351262 macos (x64, macos-15-intel) deps-v1.17.0-rc3 integration_format_sweep_test.dart: format sweep (push gate) crop_resize/padToAspect=true+resizeEnabled=true | integration_format_sweep_test.dart: format sweep (push gate) edge_repair 96x64/yuv411p/prog signal 4 (SIGILL) | 98x64/yuv422p12le/prog signal 4 (SIGILL) vspipe-bin-2026-11-18-023804.ips: vspipe-teardown (vspipe-bin frameDoneCallback) | vspipe-bin-2026-11-18-023234.ips: vspipe-teardown (vspipe-bin frameDoneCallback) +44222529261 linux deps-v1.17.0-rc3 integration_format_sweep_test.dart: format sweep (push gate) crop_resize/padEnabled=true 96x64/yuv444p12le/tff signal 11 (SIGSEGV) +44223067662 linux deps-v1.17.0-rc3 integration_format_sweep_test.dart: format sweep (push gate) crop_resize/padEnabled=true 96x66/yuv420p10le/prog signal 11 (SIGSEGV) +44223147893 linux deps-v1.17.0-rc3 integration_format_sweep_test.dart: format sweep (push gate) crop_resize/padToAspect=true+resizeEnabled=true 98x64/yuv422p12le/prog signal 11 (SIGSEGV) +44227421728 linux deps-v1.17.0-rc3 integration_format_sweep_test.dart: format sweep (push gate) chroma_fixes/applyAutoChroma=true | integration_format_sweep_test.dart: format sweep (push gate) crop_resize/padEnabled=true 96x64/yuv411p/prog signal 11 (SIGSEGV) | 96x64/yuv444p12le/tff signal 11 (SIGSEGV) +44234468352 macos (x64, macos-15-intel) deps-v1.17.0-rc3 integration_format_sweep_test.dart: format sweep (push gate) noise_reduction/ctmf | plugin_format_contracts_test.dart: the bundle still refuses exactly what the table says (quick probe) 96x66/yuv420p/tff signal 4 (SIGILL) vspipe-bin-2026-11-18-042838.ips: vspipe-teardown (vspipe-bin frameDoneCallback) +44260782676 macos (x64, macos-15-intel) deps-v1.17.0-rc3 integration_format_sweep_test.dart: format sweep (push gate) color_correction 98x64/yuv422p/tff signal 4 (SIGILL) vspipe-bin-2026-11-18-075929.ips: vspipe-teardown (vspipe-bin frameDoneCallback) +44291218371 linux deps-v1.17.0-rc3 integration_format_sweep_test.dart: format sweep (push gate) color_correction 98x64/yuv422p/tff signal 11 (SIGSEGV) +44291218371 macos (x64, macos-15-intel) deps-v1.17.0-rc3 plugin_format_contracts_test.dart: the bundle still refuses exactly what the table says (quick probe) +44306273218 macos (x64, macos-15-intel) deps-v1.17.0-rc3 plugin_format_contracts_test.dart: the bundle still refuses exactly what the table says (quick probe) +44318045317 linux deps-v1.17.0-rc3 integration_format_sweep_test.dart: format sweep (push gate) chroma_fixes/applyAutoChroma=true 98x64/yuv422p10le/tff signal 11 (SIGSEGV) core.vspipe-bin.52015.1795010338: vspipe-teardown (vspipe-bin outputFrame; another thread is in vpy4_freeScript) diff --git a/docs/reports/task-72-macos-x64-sigill/index.html b/docs/reports/task-72-macos-x64-sigill/index.html new file mode 100644 index 00000000..84a18040 --- /dev/null +++ b/docs/reports/task-72-macos-x64-sigill/index.html @@ -0,0 +1,186 @@ + + + + + +macOS x64 sweep SIGILL + + + +
+

Task #72: the macOS x64 format-sweep SIGILL was vspipe's teardown race

+

2026-11-19 · GitHub-hosted x86 runners (macos-15-intel, ubuntu-24.04, windows-latest) · deps 1.17.0-rc3 against 1.18.0-rc1 · +scripts in scripts/, raw counts in data/

+ +

Outcome

+
    +
  • Every macOS x64 SIGILL in the push gate was task #68's race. On deps 1.17.0-rc3, 4 push-gate runs +failed a random format-sweep row with vspipe exited with signal 4 (SIGILL) (5 rows). Every one left a crash +report, and every report has the same faulting thread: vspipe's frameDoneCallback → +condition_variable::notify_one → pthread_cond_signal → +_os_unfair_lock_unowned_abort (section 1).
  • +
  • It is fixed by deps 1.18.0-rc1, which carries upstream's vspipe fix (task #68). On the same x86 +runners, with only vspipe-bin swapped, R78's own vspipe crashed 71 of 32,160 one-frame +runs and the patched one 0 of 32,160 (section 3).
  • +
  • No retry or quarantine was added. The sweep's native-crash class still finds real bugs (tasks #69, #70). +Instead, the CI crash-report step now names this signature and says whether the bundle has the fix (section 4).
  • +
+ +

1. What the crash reports say

+

scripts/classify_ci_failures.py reads every failed push-gate job's log (gh run view --job --log), +lists its failed tests and native-crash rows, and classifies each crash report the log printed with +print-crash-reports.py's classify(). Output: data/ci-failures.tsv (19 failed jobs, +deps 1.17.0-rc2 and rc3).

+ + + + + +
PlatformSweep rows that died nativelySignalCrash reportsVerdict
macOS x645 (4 runs)SIGILL5 .ips5 × vspipe-teardown (frameDoneCallback)
Linux x6410 (9 runs)SIGSEGV1 gdb core (the rest predate core capture)vspipe-teardown (outputFrame; main in vpy4_freeScript)
macOS arm6401 (a Bifrost-era test)image:AppleParavirtGPUMetalIOGPUFamily, not the race
+

The rows were all over the sweep (QTGMC Fast, Crop & Resize, Edge Repair, CTMF, Colour Correction, at different sizes and +formats), which is what a race in vspipe's exit looks like and what a plugin bug does not.

+ +

The three suspected causes

+ + + + + +
Hypothesis (task description)VerdictWhy
A CPU-dispatched instruction the runner lacksruled outNo plugin image is on the +faulting thread; the faulting instruction is ud2 in libsystem_platform, put there deliberately.
A Zig ud2 (vszip / zsmooth safety trap)ruled outSame: the trap is in +_os_unfair_lock_unowned_abort, called from vspipe, with neither plugin on the stack.
vspipe R78's teardown race (task #68)the causeEvery report: vspipe's callback signalling a +condition variable that main has already freed.
+ +

Why the same race is SIGILL on macOS and SIGSEGV on Linux

+

R78's frameDoneCallback increments completedFrames before it writes the frame and notifies. Main can +see "every frame done" in that gap and free VSPipeOutputData, its mutex and condition variable included. What the +callback then does with freed memory decides the signal:

+
    +
  • macOS: pthread_cond_signal takes the condition variable's internal os_unfair_lock. +In freed (or reused) memory its owner field no longer matches, and libplatform aborts on purpose with ud2: +EXC_BAD_INSTRUCTION, SIGILL. When the callback is still in outputFrame writing to the freed buffer, +it is a SIGSEGV instead; both A/B crashes on macOS x64 (section 3) were that form.
  • +
  • Linux: glibc's condition variable has no owner check, so the fault is the write itself: SIGSEGV in +outputFrame, with main in vpy4_freeScript.
  • +
  • Windows: an access violation (0xC0000005).
  • +
+

The fixture Tests/TestResources/crash_reports/macos-x64-vspipe-teardown-sigill.ips is one of these +reports as CI printed it (paths reduced to the runner's own).

+ +

2. Push-gate tally since deps 1.18.0-rc1

+

The macOS x64 push gate runs 588 sweep rows a job. On rc3, 5 rows died in 20 jobs (11,760 rows): about 1 in 2,400.

+ + + + + + + + + + +
RunBranchmacOS x64
44411658811task #68passed
44481983232mainpassed
44484258865task #67passed
44497719057mainfailed: quick-probe 15 min timeout (task #75), no sweep row failed
44500281558task #70passed
44510129664task #73passed
44510992864task #70passed
44515328673task #75failed: one 30 s test timeout (task #75), no sweep row failed
+

8 jobs, 4,704 rows, no SIGILL. At the rc3 rate 2.0 were expected; the chance of none by luck alone is about 0.14.

+

Limit: this is consistent with the fix but would not establish it on its own. The A/B in section 3 is the evidence. +Corrections: the plan for this task gave P(0) ≈ 0.08 for the first three green runs; at 588 rows a job that is +1,764 rows and P(0) ≈ 0.47. The plan and the review also counted 5 runs and 6 SIGILL rows on rc3; data/ci-failures.tsv +has 4 runs and 5 rows (with 5 reports). The eight runs above bring P(0) to 0.14.

+ +

3. The x86 A/B

+

scripts/repro-vspipe-teardown.yml (the temporary workflow, copied here; run 44500126463). Every job installs the +deps 1.17.0-rc3 bundle and runs it twice, once with its own vspipe-bin (R78) and once with the +vspipe-bin from 1.18.0-rc1 (R78 + the patch, checked by its --version marker). Nothing else differs.

+ +

Loops

+

Task #68's compare_vspipe.sh, 4 loops at once beside CPU hogs: vspipe on trigger.vpy (1,000 runs a +loop), and the worker's IVTC preview with the pre-#65 template and its Bwdif preview (28 passes over the 30-frame fixture a loop). +data/ab-loops-*.txt.

+ + + + + + + + + + + + +
PlatformScriptRuns per sideR78R78 + patch
Linux x64trigger4,00000
IVTC3,360660
Bwdif3,36000
macOS x64trigger4,00010
IVTC3,36010
Bwdif3,36000
Windows x64trigger4,00010
IVTC3,36020
Bwdif3,36000
Total32,160710
+
    +
  • Both macOS crashes left a report; both classify as vspipe-teardown, one in frameDoneCallback, one in +outputFrame, both SIGSEGV (data/ab-loops-macos-x64-crash-reports.tsv). The Windows ones were access +violations (0xC0000005); Windows keeps no crash report on the runner.
  • +
  • If the two sides crashed at the same rate, 71 crashes all falling on R78 would have probability 2−71. Without Linux +(macOS and Windows, 5 against 0) it is 2−5 ≈ 0.03; macOS alone (2 against 0) is 0.25, not significant by itself.
  • +
  • The patched rate is below 3 in 32,160 (about 1 in 10,700) at 95% confidence; on macOS x64 alone, below 3 in 10,720.
  • +
+

Limits: the loops are task #65's workloads, not the sweep's. They hit the race much more often on Linux (IVTC: 2%) +than on macOS (1 in 5,400 over all three scripts), so on macOS they do not reproduce the push gate's rate of about 1 in 2,400, and +neither macOS crash was the SIGILL form. What they show on macOS is that the race fires there, under the same signature, and that +the patched side did not crash in 10,720 runs.

+ +

Format-sweep replays: inconclusive

+

Each sweep job ran the push gate's format sweep once with a capturing vspipe-bin +(.github/scripts/replay-vspipe.sh), then replayed every captured run 10 times with each vspipe +(scripts/ab_sweep_replay.sh; data/ab-sweep-*.txt).

+ + + + +
PlatformCaptured runsRuns per sideR78R78 + patch
macOS x646346,340100100
Linux x646306,3006060
+

The failures are the same few captured runs failing all 10 times on both sides: KNLMeansCL refusing the runner's OpenCL +device ("the opencl device does not support this video format", exit 1) and, on macOS, an OpenCL kernel build crashing +(exit 139, crash reports in image:OpenCL). In the test these rows run with the worker's OpenCL probe, which a +bare replay skips. No replay hit the race on either side, so this arm says nothing about the fix. Replayed +alone, without the sweep's other jobs and the test runner around them, a run is apparently too quiet to open the window.

+ +

4. What changed

+
    +
  • .github/scripts/print-crash-reports.py: prints every thread of a macOS .ips (it printed the +faulting thread only), classifies a report (vspipe-teardown, image:<name> or +unknown), and says whether the bundle's vspipe carries the fix (its --version marker read from the +binary). --classify FILE… runs it on any .ips or gdb text and never fails.
  • +
  • ci-test.yml and nightly.yml: the macOS jobs upload the raw .ips files as +crash-reports-* artifacts on failure (14 days).
  • +
  • app/test/crash_report_classifier_test.dart (push gate): the macOS SIGILL and the Linux SIGSEGV fixtures both +classify as the race; a plugin/driver crash is named by its image.
  • +
  • replay-vspipe.sh: runs on macOS (no env -0 or timeout there).
  • +
  • CLAUDE.md Debugging Tip 11: the macOS signature.
  • +
+ +

5. Not tested

+
    +
  • The SIGILL form itself under the A/B: the loops on macOS x64 produced only the SIGSEGV form. The SIGILL form's evidence is the +five CI reports' stacks, which are the same callback.
  • +
  • A larger macOS-only A/B. 10,720 runs a side gave 2 against 0; telling the two apart on macOS alone would take roughly +five times as many runs.
  • +
+ +

Files

+
    +
  • scripts/classify_ci_failures.py → data/ci-failures.tsv
  • +
  • scripts/repro-vspipe-teardown.yml (workflow, removed from .github/workflows/) with +scripts/ab_sweep_replay.sh and task #68's compare_vspipe.sh → data/ab-*.txt, +data/ab-loops-macos-x64-crash-reports.tsv
  • +
+
+ + diff --git a/docs/reports/task-72-macos-x64-sigill/scripts/ab_sweep_replay.sh b/docs/reports/task-72-macos-x64-sigill/scripts/ab_sweep_replay.sh new file mode 100755 index 00000000..d6a36bc8 --- /dev/null +++ b/docs/reports/task-72-macos-x64-sigill/scripts/ab_sweep_replay.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Task #72: replay captured push-gate format-sweep vspipe runs with R78's own +# vspipe and with the patched one (task #68), in the same deps bundle. +# Run from the repository root: +# +# ab_sweep_replay.sh +# +# deps//vapoursynth of an UNPATCHED bundle +# (1.17.0-rc3), after `.github/scripts/replay-vspipe.sh +# install` captured the format sweep's runs into +# (it moved the bundle's own vspipe-bin to +# vspipe-bin.real) +# vspipe-bin from a 1.18.0 bundle +# +# Only vspipe-bin.real changes between the two sides. Each side replays every +# captured run times, at once (replay-vspipe.sh run, +# plain mode), and the exit codes are tallied: 132 = SIGILL, 134 = SIGABRT, +# 139 = SIGSEGV. On macOS each new crash report is classified with +# print-crash-reports.py --classify; on Linux, print-crash-reports.py runs +# gdb over the newest cores (enable-core-dumps.sh) after each side. +# Writes /summary.txt. +set -u +VSDIR="$1"; PATCHED="$2"; CAP="$3"; REPEAT="$4"; PAR="$5"; OUT="$6" +REPO="$(pwd)" +mkdir -p "$OUT"; OUT="$(cd "$OUT" && pwd)" +REAL="$VSDIR/vspipe-bin.real" +[ -f "$REAL" ] || { echo "no $REAL: run replay-vspipe.sh install first" >&2; exit 2; } +cp "$REAL" "$OUT/vspipe-unpatched.bin" +restore() { cp "$OUT/vspipe-unpatched.bin" "$REAL"; } +trap restore EXIT +MARK='VapourBox patch: vspipe frame-done count' +REPORTS="$HOME/Library/Logs/DiagnosticReports" + +for side in unpatched patched; do + if [ "$side" = patched ]; then cp "$PATCHED" "$REAL"; else cp "$OUT/vspipe-unpatched.bin" "$REAL"; fi + chmod +x "$REAL" + [ "$(uname -s)" = Darwin ] && codesign -s - -f "$REAL" >/dev/null 2>&1 + if grep -aq "$MARK" "$REAL"; then fix=yes; else fix=no; fi + touch "$OUT/$side.stamp"; sleep 1 + bash .github/scripts/replay-vspipe.sh run "$VSDIR" "$CAP" "$REPEAT" "$PAR" plain "$OUT/$side" > "$OUT/$side.log" 2>&1 + cat "$OUT/$side.log" + total=$(wc -l < "$OUT/$side/results.tsv" | tr -d ' ') + codes=$(awk -F'\t' '$3 != 0 {c[$3]++} END {for (k in c) printf "%s exit %s; ", c[k], k}' "$OUT/$side/results.tsv") + failed=$(awk -F'\t' '$3 != 0' "$OUT/$side/results.tsv" | wc -l | tr -d ' ') + line="sweep $side (fix marker: $fix): runs=$total failures=$failed ${codes:+($codes)}" + if [ -d "$REPORTS" ]; then + find "$REPORTS" -name 'vspipe-bin*.ips' -newer "$OUT/$side.stamp" -exec cp {} "$OUT/$side/" \; + shopt -s nullglob; ips=( "$OUT/$side"/*.ips ); shopt -u nullglob + if [ ${#ips[@]} -gt 0 ]; then + python3 -I .github/scripts/print-crash-reports.py --classify "${ips[@]}" > "$OUT/$side/classified.tsv" + line="$line; crash reports: $(cut -f2 "$OUT/$side/classified.tsv" | sort | uniq -c | awk '{printf "%s %s, ", $1, $2}')" + fi + elif [ "$(uname -s)" = Linux ]; then + python3 .github/scripts/print-crash-reports.py > "$OUT/$side/cores.txt" 2>&1 + line="$line; cores: $(grep -c '^ => ' "$OUT/$side/cores.txt") classified ($(grep -o 'teardown race' "$OUT/$side/cores.txt" | wc -l | tr -d ' ') the race)" + sudo rm -f "${VAPOURBOX_CORE_DIR:-/tmp/cores}"/core* 2>/dev/null # root's, in a sticky dir + fi + echo "$line" | tee -a "$OUT/summary.txt" +done +cat "$OUT/summary.txt" diff --git a/docs/reports/task-72-macos-x64-sigill/scripts/classify_ci_failures.py b/docs/reports/task-72-macos-x64-sigill/scripts/classify_ci_failures.py new file mode 100644 index 00000000..7d9df095 --- /dev/null +++ b/docs/reports/task-72-macos-x64-sigill/scripts/classify_ci_failures.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Task #72: classify the push gate's failed jobs from their CI logs. + +For each run ID, every failed job's log is fetched with `gh run view --job +--log` and summarised: the deps bundle it installed, the tests that failed, +the format-sweep rows that died natively, and each crash report the +on-failure step printed, classified with print-crash-reports.py's own +`classify` (task #68's teardown race, or the first image outside the system +libraries). + +Run from the repository root (needs `gh`, authenticated, and read access to +the repository): + + python3 docs/reports/task-72-macos-x64-sigill/scripts/classify_ci_failures.py RUN_ID ... > out.tsv + +Prints one TSV row per failed job: +run, job, deps, failing tests, native-crash rows, crash-report verdicts. +""" +import importlib.util +import os +import re +import subprocess +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +SCRIPT = os.path.join(HERE, "..", "..", "..", "..", ".github", "scripts", "print-crash-reports.py") +spec = importlib.util.spec_from_file_location("pcr", SCRIPT) +pcr = importlib.util.module_from_spec(spec) +spec.loader.exec_module(pcr) + +TS = re.compile(r"^\S*\d{4}-\d\d-\d\dT[\d:.]+Z ?") + + +def gh(*args): + return subprocess.run(["gh", *args], capture_output=True, text=True, check=True).stdout + + +def log_lines(job_id): + out = [] + for raw in gh("run", "view", "--job", str(job_id), "--log").splitlines(): + parts = raw.split("\t", 2) + out.append(TS.sub("", parts[-1]).lstrip("")) + return out + + +def printed_reports(lines): + """Crash reports as print-crash-reports.py printed them: macOS frame lines + (' N lib symbol') after '=== .ips', gdb text after '=== core.'.""" + reports, cur, kind = [], None, None + for line in lines: + m = re.match(r"^=== (\S+)", line) + if m: + if cur is not None: + reports.append((name, kind, cur)) + name, kind, cur = m.group(1), ("ips" if m.group(1).endswith(".ips") else "core"), [] + continue + if cur is None: + continue + if line.startswith(("Post job cleanup", "##[")): + reports.append((name, kind, cur)) + cur = None + continue + cur.append(line) + if cur is not None: + reports.append((name, kind, cur)) + verdicts = [] + for name, kind, body in reports: + if kind == "ips": + frames = [] + for l in body: + m = re.match(r"^\s+\d+ (\S+)\s+(.*)$", l) + if m and not l.strip().startswith(("---", "=>")): + frames.append((m.group(1), m.group(2))) + elif l.strip().startswith("--- thread"): + break # the faulting thread is printed first + verdict, detail = pcr.classify(frames) + else: + fault, others = pcr.gdb_threads("\n".join(l[2:] if l.startswith(" ") else l for l in body)) + verdict, detail = pcr.classify(fault, others) + verdicts.append(f"{name}: {verdict} ({detail})") + return verdicts + + +def main(run_ids): + print("run\tjob\tdeps\tfailed tests\tnative-crash rows\tcrash reports") + for run in run_ids: + jobs = gh("api", f"repos/{{owner}}/{{repo}}/actions/runs/{run}/jobs", + "--jq", '.jobs[]|select(.conclusion=="failure")|"\\(.id)\\t\\(.name)"') + for row in jobs.splitlines(): + job_id, name = row.split("\t", 1) + lines = log_lines(job_id) + deps = sorted(set(re.findall(r"deps-v\d+\.\d+\.\d+(?:-rc\d+)?", "\n".join(lines)))) + failed = sorted({re.sub(r".*app/test/", "", l).replace(" (failed)", "") + for l in lines if l.startswith("##[group]❌")}) + crashes = [l.strip() for l in lines if "native crash:" in l] + print("\t".join([run, name, ",".join(deps), " | ".join(failed), + " | ".join(c.split(": native crash")[0] + " " + + (re.search(r"signal \d+ \(\w+\)", c) or [""])[0] for c in crashes), + " | ".join(printed_reports(lines))])) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/docs/reports/task-72-macos-x64-sigill/scripts/repro-vspipe-teardown.yml b/docs/reports/task-72-macos-x64-sigill/scripts/repro-vspipe-teardown.yml new file mode 100644 index 00000000..b8448873 --- /dev/null +++ b/docs/reports/task-72-macos-x64-sigill/scripts/repro-vspipe-teardown.yml @@ -0,0 +1,183 @@ +name: Repro vspipe teardown A/B (task 72, temporary) + +# Task #72: R78's own vspipe against the one with task #68's backported fix, +# in the same deps bundle (1.17.0-rc3), on x86 runners. It ran from +# .github/workflows/ on the task's branch (run 44500126463) and is kept here +# with the report; copy it back into .github/workflows/ (and point the push +# trigger at your branch) to run it again. +# +# loops task #68's compare_vspipe.sh: vspipe on its trigger script, and the +# worker's IVTC (pre-#65 template) and Bwdif previews, 4 loops at +# once under CPU load +# sweep the push gate's format sweep captured once (replay-vspipe.sh), then +# every captured vspipe run replayed `sweep_repeat` times per side + +on: + push: + branches: [stuart/72-macos-x64-ci-format-sweep-previews-die-w] + paths: + - .github/workflows/repro-vspipe-teardown.yml + - docs/reports/task-72-macos-x64-sigill/scripts/** + workflow_dispatch: + inputs: + passes: + description: 'Preview passes over the 30-frame fixture, per loop (4 loops)' + default: '28' + runs: + description: 'Trigger-script runs per loop (4 loops)' + default: '1000' + sweep_repeat: + description: 'Replays of every captured format-sweep run, per side' + default: '10' + +permissions: + contents: read + actions: read + +env: + UNPATCHED_TAG: deps-v1.17.0-rc3 + PATCHED_TAG: deps-v1.18.0-rc1 + PASSES: ${{ inputs.passes || '28' }} + RUNS: ${{ inputs.runs || '1000' }} + SWEEP_REPEAT: ${{ inputs.sweep_repeat || '10' }} + +jobs: + loops: + strategy: + fail-fast: false + matrix: + include: + - {platform: macos-x64, runner: macos-15-intel} + - {platform: linux-x64, runner: ubuntu-24.04} + - {platform: windows-x64, runner: windows-latest} + name: loops (${{ matrix.platform }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 340 + defaults: + run: + shell: bash + env: + VAPOURBOX_DEPS_DIR: ${{ github.workspace }}/deps/${{ matrix.platform }} + steps: + - uses: actions/checkout@v5 + with: {fetch-depth: 0} + - name: Deps (R78 bundle, and the patched vspipe from 1.18.0-rc1) + env: {GH_TOKEN: '${{ github.token }}'} + run: | + P=${{ matrix.platform }} + bash .github/scripts/install-deps-bundle.sh "$P" "$UNPATCHED_TAG" "${UNPATCHED_TAG#deps-v}" "deps/$P" + bash .github/scripts/install-deps-bundle.sh "$P" "$PATCHED_TAG" "${PATCHED_TAG#deps-v}" "patched/$P" + if [ "$P" = windows-x64 ]; then + cp "patched/$P/vapoursynth/Lib/site-packages/vapoursynth/vspipe.exe" patched/vspipe + else + cp "patched/$P/vapoursynth/vspipe-bin" patched/vspipe + fi + rm -rf "patched/$P" + grep -ac 'VapourBox patch: vspipe frame-done count' patched/vspipe + if [ "$P" = macos-x64 ]; then xattr -cr deps 2>/dev/null || true; brew install stress-ng || true; fi + if [ "$P" = linux-x64 ]; then + sudo apt-get update && sudo apt-get install -y stress-ng gdb file + mkdir -p "$HOME/.local/share/VapourBox" + ln -sfn "$GITHUB_WORKSPACE/deps" "$HOME/.local/share/VapourBox/deps" + bash .github/scripts/enable-core-dumps.sh "$GITHUB_WORKSPACE/repro-cores" + fi + - uses: dtolnay/rust-toolchain@stable + - name: Build worker + run: cd worker && cargo build + - name: Loops + run: | + export CORE_DIR="$GITHUB_WORKSPACE/repro-cores" + bash docs/reports/task-68-vspipe-teardown-backport/scripts/compare_vspipe.sh \ + "deps/${{ matrix.platform }}" patched/vspipe 'f3706092^1' repro + echo "## loops: ${{ matrix.platform }}" >> "$GITHUB_STEP_SUMMARY" + sed 's/$/ /' repro/summary.txt >> "$GITHUB_STEP_SUMMARY" + - name: Classify the crashes + if: always() + run: | + mkdir -p repro/crash-reports + if [ -d ~/Library/Logs/DiagnosticReports ]; then + find ~/Library/Logs/DiagnosticReports -name 'vspipe-bin*.ips' -exec cp {} repro/crash-reports/ \; + shopt -s nullglob; f=(repro/crash-reports/*.ips) + if [ ${#f[@]} -gt 0 ]; then + python3 -I .github/scripts/print-crash-reports.py --classify "${f[@]}" | tee repro/crash-reports/classified.tsv + echo "crash reports: $(cut -f2 repro/crash-reports/classified.tsv | sort | uniq -c | tr '\n' ' ')" >> "$GITHUB_STEP_SUMMARY" + fi + fi + if [ -d repro-cores ]; then + VAPOURBOX_CORE_DIR=repro-cores python3 .github/scripts/print-crash-reports.py | tee repro/crash-reports/cores.txt + fi + - uses: actions/upload-artifact@v5 + if: always() + with: + name: loops-${{ matrix.platform }} + path: | + repro/summary.txt + repro/**/*.log + repro/**/*.err + repro/crash-reports/ + if-no-files-found: ignore + + sweep: + strategy: + fail-fast: false + matrix: + include: + - {platform: macos-x64, runner: macos-15-intel} + - {platform: linux-x64, runner: ubuntu-24.04} + name: sweep (${{ matrix.platform }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 340 + env: + VAPOURBOX_DEPS_DIR: ${{ github.workspace }}/deps/${{ matrix.platform }} + CAPTURE: ${{ github.workspace }}/repro/capture + steps: + - uses: actions/checkout@v5 + - name: Deps (R78 bundle, and the patched vspipe from 1.18.0-rc1) + env: {GH_TOKEN: '${{ github.token }}'} + run: | + P=${{ matrix.platform }} + bash .github/scripts/install-deps-bundle.sh "$P" "$UNPATCHED_TAG" "${UNPATCHED_TAG#deps-v}" "deps/$P" + bash .github/scripts/install-deps-bundle.sh "$P" "$PATCHED_TAG" "${PATCHED_TAG#deps-v}" "patched/$P" + cp "patched/$P/vapoursynth/vspipe-bin" patched/vspipe; rm -rf "patched/$P" + grep -ac 'VapourBox patch: vspipe frame-done count' patched/vspipe + if [ "$P" = macos-x64 ]; then xattr -cr deps 2>/dev/null || true; fi + if [ "$P" = linux-x64 ]; then + mkdir -p "$HOME/.local/share/VapourBox" + ln -sfn "$GITHUB_WORKSPACE/deps" "$HOME/.local/share/VapourBox/deps" + bash .github/scripts/enable-core-dumps.sh + fi + - uses: subosito/flutter-action@v2 + with: {channel: stable, cache: true} + - uses: dtolnay/rust-toolchain@stable + - name: Build worker and generated code + run: | + (cd worker && cargo build) + cd app && flutter pub get && dart run build_runner build --delete-conflicting-outputs + - name: Capture the push gate's format sweep + env: + VAPOURBOX_WORKER: ${{ github.workspace }}/worker/target/debug/vapourbox-worker + run: | + bash .github/scripts/replay-vspipe.sh install "deps/${{ matrix.platform }}/vapoursynth" '.' "$CAPTURE" + (cd app && flutter test test/integration_format_sweep_test.dart --exclude-tags heavy,gpu) \ + || echo "::warning::the sweep failed while capturing (R78's vspipe: expected now and then)" + echo "captured $(ls -d "$CAPTURE"/run.* | wc -l) vspipe runs; $(du -sh "$CAPTURE" | cut -f1)" + - name: Replay, R78 then patched + run: | + P=$(getconf _NPROCESSORS_ONLN) + bash docs/reports/task-72-macos-x64-sigill/scripts/ab_sweep_replay.sh \ + "deps/${{ matrix.platform }}/vapoursynth" patched/vspipe "$CAPTURE" "$SWEEP_REPEAT" "$P" repro/ab + echo "## sweep: ${{ matrix.platform }} ($(ls -d "$CAPTURE"/run.* | wc -l) captured runs)" >> "$GITHUB_STEP_SUMMARY" + sed 's/$/ /' repro/ab/summary.txt >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@v5 + if: always() + with: + name: sweep-${{ matrix.platform }} + path: | + repro/ab/summary.txt + repro/ab/*.log + repro/ab/*/err.* + repro/ab/*/results.tsv + repro/ab/*/classified.tsv + repro/ab/*/cores.txt + repro/ab/*/*.ips + if-no-files-found: ignore