test: fix what the run of #3282 showed, and use the 64-core runners

.github/workflows/ci.yml

@@ -43,59 +43,59 @@ jobs:
     env:
       # The dot reporter of the test steps (scripts/pytest_dot_reporter.py).
       PYTHONPATH: scripts
+      # 16 workers on the 64-core runners, as upstream runs its tests there; two on
+      # the 3-core macOS runner, upstream's default of half the cores.
+      WORKERS: ${{ startsWith(matrix.os, 'macos') && '2' || '16' }}
     strategy:
       fail-fast: false
       matrix:
-        # ubuntu-22.04, not ubuntu-latest: the 24.04 image has no WebGL for headless
-        # Firefox (CA-008, CA-009).
-        os: [ubuntu-22.04, windows-latest, macos-latest]
-        python-version: ['3.14']
+        # Linux and Windows run on the 64-core runners of the organization, as
+        # upstream's tests_primary.yml; macOS on the hosted runner.
+        # 3.12 is our most used Python version (base)
+        os: [playwright-x64-ubuntu24-64-core, playwright-x64-server2025-64-core, macos-15-xlarge]
+        python-version: ['3.12']
         browser: [chromium, firefox, webkit]
-        exclude:
-        # WebKit on standard macOS-latest (currently macos-15-arm64) is unstable;
-        # upstream pins paid macos-15-xlarge for cross-browser webkit too.
-        - os: macos-latest
-          browser: webkit
         include:
-        - os: windows-latest
+        - os: playwright-x64-server2025-64-core
           python-version: '3.10'
           browser: chromium
-        - os: macos-latest
+        - os: macos-15-xlarge
           python-version: '3.10'
           browser: chromium
-        - os: ubuntu-22.04
+        - os: playwright-x64-ubuntu24-64-core
           python-version: '3.10'
           browser: chromium
-        - os: macos-15-xlarge
-          python-version: '3.14'
-          browser: webkit
-        - os: windows-latest
+        - os: playwright-x64-server2025-64-core
           python-version: '3.11'
           browser: chromium
-        - os: macos-latest
+        - os: macos-15-xlarge
           python-version: '3.11'
           browser: chromium
-        - os: ubuntu-22.04
+        - os: playwright-x64-ubuntu24-64-core
           python-version: '3.11'
           browser: chromium
-        - os: windows-latest
-          python-version: '3.12'
-          browser: chromium
-        - os: macos-latest
-          python-version: '3.12'
-          browser: chromium
-        - os: ubuntu-22.04
-          python-version: '3.12'
-          browser: chromium
-        - os: windows-latest
+        - os: playwright-x64-server2025-64-core
           python-version: '3.13'
           browser: chromium
-        - os: macos-latest
+        - os: macos-15-xlarge
           python-version: '3.13'
           browser: chromium
-        - os: ubuntu-22.04
+        - os: playwright-x64-ubuntu24-64-core
           python-version: '3.13'
           browser: chromium
+        - os: playwright-x64-server2025-64-core
+          python-version: '3.14'
+          browser: chromium
+        - os: macos-15-xlarge
+          python-version: '3.14'
+          browser: chromium
+        - os: playwright-x64-ubuntu24-64-core
+          python-version: '3.14'
+          browser: chromium
+        # Intel macOS, once: *-large is Intel, *-xlarge is arm64, as upstream notes.
+        - os: macos-15-large
+          python-version: '3.12'
+          browser: chromium
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -119,34 +119,34 @@ jobs:
     - name: Test Wheel Installation
       run: pytest tests/test_installation.py --browser=${{ matrix.browser }}
     - name: Test Sync API
-      if: matrix.os != 'ubuntu-22.04'
-      run: pytest tests/sync --browser=${{ matrix.browser }} -n auto -p pytest_dot_reporter -rfE --color=yes
+      if: runner.os != 'Linux'
+      run: pytest tests/sync --browser=${{ matrix.browser }} -n ${{ env.WORKERS }} -p pytest_dot_reporter -rfE --color=yes
     - name: Test Sync API
-      if: matrix.os == 'ubuntu-22.04'
-      run: xvfb-run pytest tests/sync --browser=${{ matrix.browser }} -n auto -p pytest_dot_reporter -rfE --color=yes
+      if: runner.os == 'Linux'
+      run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" pytest tests/sync --browser=${{ matrix.browser }} -n ${{ env.WORKERS }} -p pytest_dot_reporter -rfE --color=yes
     - name: Test Async API
       # Also after a failed sync step: the two suites report on their own.
-      if: ${{ !cancelled() && steps.install.outcome == 'success' && matrix.os != 'ubuntu-22.04' }}
-      run: pytest tests/async --browser=${{ matrix.browser }} -n auto -p pytest_dot_reporter -rfE --color=yes
+      if: ${{ !cancelled() && steps.install.outcome == 'success' && runner.os != 'Linux' }}
+      run: pytest tests/async --browser=${{ matrix.browser }} -n ${{ env.WORKERS }} -p pytest_dot_reporter -rfE --color=yes
     - name: Test Async API
-      if: ${{ !cancelled() && steps.install.outcome == 'success' && matrix.os == 'ubuntu-22.04' }}
-      run: xvfb-run pytest tests/async --browser=${{ matrix.browser }} -n auto -p pytest_dot_reporter -rfE --color=yes
+      if: ${{ !cancelled() && steps.install.outcome == 'success' && runner.os == 'Linux' }}
+      run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" pytest tests/async --browser=${{ matrix.browser }} -n ${{ env.WORKERS }} -p pytest_dot_reporter -rfE --color=yes
 
   test-stable:
     name: Stable
-    timeout-minutes: 90
+    # The branded browsers on the 3-core macOS runner take longer than the bundled ones.
+    timeout-minutes: 120
     env:
       # The dot reporter of the test steps (scripts/pytest_dot_reporter.py).
       PYTHONPATH: scripts
+      WORKERS: ${{ startsWith(matrix.os, 'macos') && '2' || '16' }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04, windows-latest, macos-latest]
+        os: [playwright-x64-ubuntu24-64-core, playwright-x64-server2025-64-core, macos-15-xlarge]
         browser-channel: [chrome]
         include:
-        - os: windows-latest
-          browser-channel: msedge
-        - os: macos-latest
+        - os: playwright-x64-server2025-64-core
           browser-channel: msedge
     runs-on: ${{ matrix.os }}
     steps:
@@ -167,18 +167,18 @@ jobs:
     - name: Common Tests
       run: pytest tests/common --browser=chromium --browser-channel=${{ matrix.browser-channel }}
     - name: Test Sync API
-      if: matrix.os != 'ubuntu-22.04'
-      run: pytest tests/sync --browser=chromium --browser-channel=${{ matrix.browser-channel }} -n auto -p pytest_dot_reporter -rfE --color=yes
+      if: runner.os != 'Linux'
+      run: pytest tests/sync --browser=chromium --browser-channel=${{ matrix.browser-channel }} -n ${{ env.WORKERS }} -p pytest_dot_reporter -rfE --color=yes
     - name: Test Sync API
-      if: matrix.os == 'ubuntu-22.04'
-      run: xvfb-run pytest tests/sync --browser=chromium --browser-channel=${{ matrix.browser-channel }} -n auto -p pytest_dot_reporter -rfE --color=yes
+      if: runner.os == 'Linux'
+      run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" pytest tests/sync --browser=chromium --browser-channel=${{ matrix.browser-channel }} -n ${{ env.WORKERS }} -p pytest_dot_reporter -rfE --color=yes
     - name: Test Async API
       # Also after a failed sync step: the two suites report on their own.
-      if: ${{ !cancelled() && steps.install.outcome == 'success' && matrix.os != 'ubuntu-22.04' }}
-      run: pytest tests/async --browser=chromium --browser-channel=${{ matrix.browser-channel }} -n auto -p pytest_dot_reporter -rfE --color=yes
+      if: ${{ !cancelled() && steps.install.outcome == 'success' && runner.os != 'Linux' }}
+      run: pytest tests/async --browser=chromium --browser-channel=${{ matrix.browser-channel }} -n ${{ env.WORKERS }} -p pytest_dot_reporter -rfE --color=yes
     - name: Test Async API
-      if: ${{ !cancelled() && steps.install.outcome == 'success' && matrix.os == 'ubuntu-22.04' }}
-      run: xvfb-run pytest tests/async --browser=chromium --browser-channel=${{ matrix.browser-channel }} -n auto -p pytest_dot_reporter -rfE --color=yes
+      if: ${{ !cancelled() && steps.install.outcome == 'success' && runner.os == 'Linux' }}
+      run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" pytest tests/async --browser=chromium --browser-channel=${{ matrix.browser-channel }} -n ${{ env.WORKERS }} -p pytest_dot_reporter -rfE --color=yes
 
   test_examples:
     name: Examples

local-requirements.txt

@@ -4,6 +4,7 @@ build==1.6.1
 mypy==2.4.0
 objgraph==3.6.2
 Pillow==12.3.0
+pixelmatch==0.3.0
 pre-commit==3.5.0
 pyOpenSSL==26.0.0
 pytest==9.0.3

tests/async/library/test_browsercontext_locale.py

@@ -130,6 +130,7 @@ async def test_LC_007_should_work_for_multiple_pages_sharing_same_process(
     await context.close()
 
 
+@pytest.mark.slow
 async def test_LC_008_should_be_isolated_between_contexts(
     browser: Browser, server: Server
 ) -> None:

tests/async/library/test_browsercontext_pages.py

@@ -166,6 +166,7 @@ async def test_BC_058_should_return_bounding_box_with_page_scale(
     await context.close()
 
 
+@pytest.mark.slow
 async def test_BC_059_should_not_leak_listeners_during_navigation_of_20_pages(
     context_factory: Callable[..., Awaitable[BrowserContext]], server: Server
 ) -> None:

tests/async/library/test_defaultbrowsercontext_2.py

@@ -210,6 +210,7 @@ async def test_DC_024_should_accept_relative_user_data_dir(
     await context.close()
 
 
+@pytest.mark.slow
 async def test_DC_025_should_restore_state_from_user_data_dir(
     browser_type: BrowserType,
     server: Server,

tests/async/library/test_har.py

@@ -1326,6 +1326,7 @@ async def test_HA_057_should_close_the_context_when_saving_the_har_fails(
         await browser.close()
 
 
+@pytest.mark.slow
 async def test_HA_058_should_support_har_larger_than_512_mb(
     context_factory: Callable[..., Awaitable[BrowserContext]],
     server: Server,

tests/async/library/test_headful.py

@@ -121,6 +121,7 @@ async def test_HE_001_should_have_default_url_when_launching_browser_smoke(
     assert urls == ["about:blank"]
 
 
+@pytest.mark.slow
 async def test_HE_002_should_close_browser_with_beforeunload_page(
     launch_persistent: LaunchPersistent, server: Server
 ) -> None:

tests/async/library/test_tracing.py

@@ -1145,23 +1145,30 @@ async def test_TR_039_should_not_flush_console_events(
     page = await context.new_page()
     test_id = "TR-039"
     await context.tracing.start(name=test_id)
+    promise: "asyncio.Future[None]" = page._loop.create_future()
     counter = 0
 
-    def predicate(_: Any) -> bool:
+    def on_console(_: Any) -> None:
         nonlocal counter
         counter += 1
-        return counter == 100
+        if counter == 100:
+            promise.set_result(None)
 
-    async with page.expect_event("console", predicate=predicate):
-        await page.evaluate(
-            """() => {
+    # Python: a listener and a future, as upstream: expect_event() would be an
+    # action of its own in the trace, which flushes the console events.
+    page.on("console", on_console)
+
+    await page.evaluate(
+        """() => {
     window.builtins.setTimeout(() => {
       for (let i = 0; i < 100; ++i)
         console.log('hello ' + i);
     }, 10);
     return 31415926;
   }"""
-        )
+    )
+
+    await promise
 
     def read_trace() -> str:
         trace_name = next(

tests/async/library/test_video.py

@@ -819,6 +819,7 @@ async def test_VI_030_screencast_should_work_with_video_trace(
     assert is_almost_red(pixel) is True
 
 
+@pytest.mark.slow
 async def test_VI_031_should_save_as_video(browser: Browser, tmp_path: Path) -> None:
     videos_path = tmp_path
     size: ViewportSize = {"width": 320, "height": 240}

tests/async/page/test_page_screenshot.py

@@ -654,6 +654,7 @@ async def test_SS_038_page_screenshot_should_work_with_array_deleted(
 
 
 @pytest.mark.usefixtures("skip_firefox_headed")
+@pytest.mark.slow
 async def test_SS_039_page_screenshot_should_take_full_page_screenshots_during_navigation(
     page: Page, server: Server
 ) -> None:

tests/conftest.py

@@ -23,7 +23,17 @@
 import threading
 import traceback
 from pathlib import Path
-from typing import Any, Callable, Dict, Generator, List, Optional, cast
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Generator,
+    Iterable,
+    List,
+    Optional,
+    Tuple,
+    cast,
+)
 
 import greenlet
 import pytest
@@ -103,10 +113,37 @@ def pytest_timeout_cancel_timer(item: pytest.Item) -> Optional[bool]:
     return True
 
 
+_REPO = _dirname.parent
+
+# The waits that every run has, which say nothing about a hang: the reader of the
+# driver's pipe, and the main task of the sync API's event loop.
+_IDLE_COROUTINES = (
+    "PipeTransport.run",
+    "PlaywrightContextManager.__enter__.<locals>.main",
+)
+
+
+def _own_frames(frames: Iterable[Tuple[Any, int]]) -> List[str]:
+    """The frames of this repository, formatted: the test and the client. The
+    frames of pytest, asyncio and the other packages around them are left out."""
+    summary = traceback.StackSummary.extract(frames)
+    own = [
+        f
+        for f in summary
+        if f.filename.startswith(str(_REPO))
+        and f"{os.sep}env{os.sep}" not in f.filename
+        and not f.filename.endswith(f"tests{os.sep}server.py")
+    ]
+    return summary.__class__.from_list(own).format()
+
+
 def _dump_stacks_and_exit(item: pytest.Item, settings: pytest_timeout.Settings) -> None:
-    """Writes the stacks of the process to its standard error and ends the
-    process, as the thread method of pytest-timeout does; xdist replaces the
-    worker."""
+    """Writes to the standard error where the process waits, and ends the process,
+    as the thread method of pytest-timeout does; xdist replaces the worker.
+
+    Each thread, each greenlet that waits for the event loop under the sync API,
+    and each pending task of the event loop gets its frames in this repository.
+    One without any, such as the idle event loop, is left out."""
     try:
         capman = cast(
             Optional[CaptureManager],
@@ -122,6 +159,12 @@ def _dump_stacks_and_exit(item: pytest.Item, settings: pytest_timeout.Settings)
         def title(text: str, sep: str = "~") -> None:
             stderr.write(f"\n{sep * 10} {text} {sep * 10}\n")
 
+        def write(name: str, frames: Iterable[Tuple[Any, int]]) -> None:
+            lines = _own_frames(frames)
+            if lines:
+                title(name)
+                stderr.write("".join(lines))
+
         title(f"Timeout ({settings.timeout}s): {item.nodeid}", "+")
         if captured_stdout:
             title("Captured stdout")
@@ -137,15 +180,22 @@ def title(text: str, sep: str = "~") -> None:
                 (t.name for t in threading.enumerate() if t.ident == thread_ident),
                 "<unknown>",
             )
-            title(f"Stack of {thread_name} ({thread_ident})")
-            stderr.write("".join(traceback.format_stack(frame)))
+            # walk_stack() yields the innermost frame first.
+            write(f"Thread {thread_name}", reversed(list(traceback.walk_stack(frame))))
         for obj in gc.get_objects():
             if isinstance(obj, greenlet.greenlet) and obj.gr_frame is not None:
-                title(f"Stack of {obj!r}, switched away")
-                stderr.write("".join(traceback.format_stack(obj.gr_frame)))
+                write(
+                    f"{type(obj).__name__}, waiting for the event loop",
+                    reversed(list(traceback.walk_stack(obj.gr_frame))),
+                )
             elif isinstance(obj, asyncio.Task) and not obj.done():
-                title(f"Stack of {obj!r}")
-                obj.print_stack(file=stderr)
+                coroutine = obj.get_coro()
+                name = getattr(coroutine, "__qualname__", repr(coroutine))
+                if name in _IDLE_COROUTINES:
+                    continue
+                # Python: the frames of the task, from the coroutine out to the
+                # awaited one, as Task.print_stack() prints them.
+                write(f"Task {name}", ((f, f.f_lineno) for f in obj.get_stack()))
         title("Timeout", "+")
     except Exception:
         traceback.print_exc()