From 58781b730107238df56a49ab8c8e71c5a10a6cf6 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Thu, 8 Oct 2026 09:01:27 -0500 Subject: [PATCH 1/3] Support Python 3.15, drop 3.10, upgrade tools --- .github/workflows/publish.yml | 15 +++++++++++---- .github/workflows/test.yml | 26 +++++++++++++++++--------- .pre-commit-config.yaml | 8 ++++---- .readthedocs.yml | 6 ++---- README.rst | 2 +- docs/_ext/__init__.py | 1 + docs/requirements.in | 2 +- docs/requirements.txt | 26 ++++++++++++++++---------- pyproject.toml | 16 +++++++++++----- tox.ini | 27 ++++++++++++++------------- zyte_api/__main__.py | 2 +- zyte_api/_async.py | 4 +--- zyte_api/_sync.py | 25 ++++++++++++++----------- zyte_api/stats.py | 5 +---- 14 files changed, 95 insertions(+), 70 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a20b249..1200ece 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,4 +1,8 @@ name: Publish + +permissions: + contents: read + on: push: tags: @@ -10,14 +14,17 @@ jobs: name: pypi url: https://pypi.org/p/zyte-api permissions: + contents: read id-token: write steps: - - uses: actions/checkout@v5 - - uses: actions/setup-python@v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - python-version: 3.13 + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: 3.14 - run: | python -m pip install --upgrade build python -m build - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e3d8bac..a374b2c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,6 +3,9 @@ name: tox +permissions: + contents: read + on: push: branches: [ main ] @@ -17,24 +20,27 @@ jobs: fail-fast: false matrix: include: - - python-version: "3.10" + - python-version: "3.11" tox: min - - python-version: "3.10" + - python-version: "3.11" tox: min-x402 - - python-version: "3.10" - python-version: "3.11" - python-version: "3.12" - python-version: "3.13" - python-version: "3.14" - python-version: "3.14" tox: x402 + - python-version: "3.15" steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} + allow-prereleases: true - name: Install dependencies run: | python -m pip install --upgrade pip @@ -44,7 +50,7 @@ jobs: tox -e ${{ matrix.tox || 'py' }} - name: coverage if: ${{ success() }} - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1 with: token: ${{ secrets.CODECOV_TOKEN }} @@ -53,13 +59,15 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.13"] # In sync with pyproject.toml + python-version: ["3.14"] # In sync with pyproject.toml tox-job: ["mypy", "docs"] steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v6 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9a141d1..6d75f00 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.13.3 + rev: v0.16.10 hooks: - id: ruff-check args: [ --fix ] @@ -10,17 +10,17 @@ repos: hooks: - id: blacken-docs additional_dependencies: - - black==25.9.0 + - black==26.10.0 - repo: https://github.com/pre-commit/pre-commit-hooks rev: v6.0.0 hooks: - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/sphinx-contrib/sphinx-lint - rev: v1.0.0 + rev: v1.0.2 hooks: - id: sphinx-lint - repo: https://github.com/scrapy/sphinx-scrapy - rev: 0.8.4 + rev: 0.13.0 hooks: - id: sphinx-scrapy diff --git a/.readthedocs.yml b/.readthedocs.yml index 6d1aeb5..fa9aa07 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -2,9 +2,7 @@ version: 2 build: os: ubuntu-24.04 tools: - python: "3.13" + python: "3.14" commands: - pip install tox - - tox -e docs - - mkdir -p $READTHEDOCS_OUTPUT/html - - cp -a docs/_build/all/. $READTHEDOCS_OUTPUT/html/ + - tox -e docs -- $READTHEDOCS_OUTPUT/html diff --git a/README.rst b/README.rst index 7984fe4..7dbcdcb 100644 --- a/README.rst +++ b/README.rst @@ -43,7 +43,7 @@ Or, to use x402_: pip install zyte-api[x402] -.. note:: Python 3.10+ is required. +.. note:: Python 3.11+ is required. .. install-end diff --git a/docs/_ext/__init__.py b/docs/_ext/__init__.py index ee080d2..39d8314 100644 --- a/docs/_ext/__init__.py +++ b/docs/_ext/__init__.py @@ -41,3 +41,4 @@ def http_api_reference_role( def setup(app): # https://github.com/scrapy-plugins/scrapy-zyte-api/blob/2bfb2bef2e43293a62f47781914331bc4fa08f06/docs/_ext/__init__.py#L42 app.add_role("http", http_api_reference_role) + return {"parallel_read_safe": True, "parallel_write_safe": True} diff --git a/docs/requirements.in b/docs/requirements.in index ddfe39e..e1908dd 100644 --- a/docs/requirements.in +++ b/docs/requirements.in @@ -2,5 +2,5 @@ aiohttp sphinx sphinx-argparse sphinx-rtd-theme -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.4 +sphinx-scrapy==0.13.0 tenacity diff --git a/docs/requirements.txt b/docs/requirements.txt index 1ece4a1..85088c2 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile -p 3.13 docs/requirements.in -o docs/requirements.txt +# uv pip compile -p 3.14 requirements.in -o requirements.txt aiohappyeyeballs==2.6.1 # via aiohttp aiohttp==3.14.3 @@ -20,8 +20,9 @@ docutils==0.22.4 # via # sphinx # sphinx-argparse - # sphinx-markdown-builder + # sphinx-llm-friendly # sphinx-rtd-theme + # sphinx-scrapy frozenlist==1.8.0 # via # aiohttp @@ -51,7 +52,9 @@ propcache==0.4.1 pygments==2.20.0 # via sphinx requests==2.33.1 - # via sphinx + # via + # sphinx + # sphinxcontrib-youtube roman-numerals==4.1.0 # via sphinx snowballstemmer==3.0.1 @@ -61,25 +64,26 @@ sphinx==9.1.0 # -r requirements.in # sphinx-argparse # sphinx-copybutton + # sphinx-design # sphinx-last-updated-by-git - # sphinx-llms-txt - # sphinx-markdown-builder + # sphinx-llm-friendly # sphinx-rtd-theme # sphinx-scrapy # sphinxcontrib-jquery + # sphinxcontrib-youtube sphinx-argparse==0.5.2 # via -r requirements.in sphinx-copybutton==0.5.2 # via sphinx-scrapy +sphinx-design==0.7.0 + # via sphinx-scrapy sphinx-last-updated-by-git==0.3.8 # via sphinx-sitemap -sphinx-llms-txt @ git+https://github.com/zytedata/sphinx-llms-txt.git@5e8866cb0cc249aa2017ad9050b3b83a7ca16f69 - # via sphinx-scrapy -sphinx-markdown-builder @ git+https://github.com/zytedata/sphinx-markdown-builder.git@cfe4c0bfd7b4542f7e6b65a58cdf9ec765829940 +sphinx-llm-friendly==0.7.0 # via sphinx-scrapy sphinx-rtd-theme==3.1.0 # via -r requirements.in -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.4 +sphinx-scrapy==0.13.0 # via -r requirements.in sphinx-sitemap==2.9.0 # via sphinx-scrapy @@ -97,8 +101,10 @@ sphinxcontrib-qthelp==2.0.0 # via sphinx sphinxcontrib-serializinghtml==2.0.0 # via sphinx +sphinxcontrib-youtube==1.5.0 + # via sphinx-scrapy tabulate==0.10.0 - # via sphinx-markdown-builder + # via sphinx-llm-friendly tenacity==9.1.4 # via -r requirements.in urllib3==2.8.0 diff --git a/pyproject.toml b/pyproject.toml index d57cd2f..f00bcbf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,9 +11,9 @@ authors = [ { name = "Zyte Group Ltd", email = "opensource@zyte.com" }, ] license = { file = "LICENSE" } -requires-python = ">=3.10" +requires-python = ">=3.11" dependencies = [ - "aiohttp>=3.8.0", + "aiohttp>=3.8.2", "attrs>=20.1.0", "brotli>=0.5.2", "python-dotenv>=1.0.0", @@ -30,11 +30,11 @@ classifiers = [ "Natural Language :: English", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: 3.15", ] [project.optional-dependencies] @@ -175,6 +175,8 @@ extend-select = [ "YTT", ] ignore = [ + # Do not catch blind exception + "BLE001", # Trailing comma missing "COM812", # Missing docstring in public module @@ -215,6 +217,8 @@ ignore = [ "PLR0913", # Too many statements "PLR0915", + # Too many positional arguments + "PLR0917", # Magic value used in comparison "PLR2004", # String contains ambiguous {}. @@ -227,6 +231,8 @@ ignore = [ "RUF012", # Use of `assert` detected "S101", + # Prefer `TypeError` exception for invalid type + "TRY004", ] [tool.ruff.lint.flake8-pytest-style] @@ -245,7 +251,7 @@ split-on-trailing-comma = false "zyte_api/__init__.py" = ["F401"] "zyte_api/aio/errors.py" = ["F401"] "zyte_api/aio/retry.py" = ["F401"] -"tests/*" = ["S"] +"tests/*" = ["ASYNC240", "S"] "docs/**" = ["B006"] # Skip PEP 604 suggestions for files with attr classes "zyte_api/errors.py" = ["UP007", "UP045"] @@ -255,4 +261,4 @@ split-on-trailing-comma = false convention = "pep257" [tool.sphinx-scrapy] -python-version = "3.13" # In sync with test.yml +python-version = "3.14" # In sync with test.yml diff --git a/tox.ini b/tox.ini index e907f01..bda6de6 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,8 @@ [tox] +min_version = 4.25.0 requires = - sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.4 -envlist = pre-commit,mypy,min,min-x402,py310,py311,py312,py313,py314,x402,docs,twine + sphinx-scrapy[tox]==0.13.0 +envlist = pre-commit,mypy,min,min-x402,py3{11-15},x402,docs,twine [testenv] deps = @@ -24,7 +25,7 @@ extras = x402 [min] deps = {[testenv]deps} - aiohttp==3.8.0 + aiohttp==3.8.2 attrs==20.1.0 brotli==0.5.2 runstats==0.0.1 @@ -33,13 +34,13 @@ deps = w3lib==2.1.1 [testenv:min] -basepython = python3.10 +basepython = python3.11 deps = {[min]deps} typing-extensions==4.5.0 [testenv:min-x402] -basepython = python3.10 +basepython = python3.11 deps = {[min]deps} eth_account==0.13.7 @@ -49,12 +50,12 @@ deps = [testenv:mypy] deps = - mypy==1.18.2 - eth-account==0.13.7 - pytest==8.3.3 - Twisted==25.5.0 - types-tqdm==4.67.0.20250809 - x402==0.2.1 + mypy==2.4.0 + eth-account==0.14.0 + pytest==9.1.1 + Twisted==26.4.0 + types-tqdm==4.70.0.20260906 + x402==1.0.0 commands = mypy \ zyte_api \ @@ -66,8 +67,8 @@ commands = pre-commit run --all-files --show-diff-on-failure [testenv:twine] deps = - twine==6.2.0 - build==1.3.0 + twine==7.0.0 + build==1.6.1 commands = python -m build --sdist twine check dist/* diff --git a/zyte_api/__main__.py b/zyte_api/__main__.py index 337cf0e..2ce06df 100644 --- a/zyte_api/__main__.py +++ b/zyte_api/__main__.py @@ -124,7 +124,7 @@ def read_input( intype = _guess_intype(input_fp.name, lines) if intype == "txt": urls = [u.strip() for u in lines if u.strip()] - base = params if params else {"browserHtml": True} + base = params or {"browserHtml": True} records = [{"url": url, **base} for url in urls] else: records = [ diff --git a/zyte_api/_async.py b/zyte_api/_async.py index a1ff1ff..e68f364 100644 --- a/zyte_api/_async.py +++ b/zyte_api/_async.py @@ -25,12 +25,10 @@ if TYPE_CHECKING: from collections.abc import Awaitable, Callable, Iterator from contextlib import AbstractAsyncContextManager + from typing import Self from eth_account.signers.local import LocalAccount - # typing.Self requires Python 3.11 - from typing_extensions import Self - _ResponseFuture = Awaitable[dict[str, Any]] diff --git a/zyte_api/_sync.py b/zyte_api/_sync.py index 4b3fbe5..cbc3e4f 100644 --- a/zyte_api/_sync.py +++ b/zyte_api/_sync.py @@ -7,14 +7,12 @@ from ._async import AsyncZyteAPI if TYPE_CHECKING: - from collections.abc import Generator + from collections.abc import Awaitable, Generator, Iterator + from typing import Self from aiohttp import ClientSession from tenacity import AsyncRetrying - # typing.Self requires Python 3.11 - from typing_extensions import Self - def _get_loop() -> AbstractEventLoop: try: @@ -202,14 +200,19 @@ def iter( The remaining parameters work the same as in :meth:`get`. """ + + # asyncio.as_completed() requires a running event loop. + async def _iter() -> Iterator[Awaitable[dict[str, Any]]]: + return self._async_client.iter( + queries=queries, + endpoint=endpoint, + session=session, + handle_retries=handle_retries, + retrying=retrying, + ) + loop = _get_loop() - for future in self._async_client.iter( - queries=queries, - endpoint=endpoint, - session=session, - handle_retries=handle_retries, - retrying=retrying, - ): + for future in loop.run_until_complete(_iter()): try: yield loop.run_until_complete(future) except Exception as exception: diff --git a/zyte_api/stats.py b/zyte_api/stats.py index f7f95e4..44f784e 100644 --- a/zyte_api/stats.py +++ b/zyte_api/stats.py @@ -12,10 +12,7 @@ if TYPE_CHECKING: from collections.abc import Callable - from typing import ParamSpec - - # typing.Self requires Python 3.11 - from typing_extensions import Self + from typing import ParamSpec, Self _P = ParamSpec("_P") From 048ee4be9273fd6a88210476f0c69cb7eefa5720 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Thu, 8 Oct 2026 10:32:42 -0500 Subject: [PATCH 2/3] Link to the upstream issue the workaround exists for --- zyte_api/_sync.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zyte_api/_sync.py b/zyte_api/_sync.py index cbc3e4f..815b489 100644 --- a/zyte_api/_sync.py +++ b/zyte_api/_sync.py @@ -201,7 +201,8 @@ def iter( The remaining parameters work the same as in :meth:`get`. """ - # asyncio.as_completed() requires a running event loop. + # asyncio.as_completed() requires a running event loop on Python + # 3.14.8+: https://github.com/python/cpython/issues/157856 async def _iter() -> Iterator[Awaitable[dict[str, Any]]]: return self._async_client.iter( queries=queries, From ff900d1f4480cb89f4c0cb233589bfb09ea3ee57 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Thu, 8 Oct 2026 14:56:52 -0500 Subject: [PATCH 3/3] Use typing.Self and PEP 604 unions now that 3.11+ is required --- pyproject.toml | 3 --- zyte_api/_async.py | 3 +-- zyte_api/_sync.py | 3 +-- zyte_api/errors.py | 7 +++---- zyte_api/stats.py | 20 ++++++++++---------- 5 files changed, 15 insertions(+), 21 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f00bcbf..40dcf49 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -253,9 +253,6 @@ split-on-trailing-comma = false "zyte_api/aio/retry.py" = ["F401"] "tests/*" = ["ASYNC240", "S"] "docs/**" = ["B006"] -# Skip PEP 604 suggestions for files with attr classes -"zyte_api/errors.py" = ["UP007", "UP045"] -"zyte_api/stats.py" = ["UP007", "UP045"] [tool.ruff.lint.pydocstyle] convention = "pep257" diff --git a/zyte_api/_async.py b/zyte_api/_async.py index e68f364..4335d85 100644 --- a/zyte_api/_async.py +++ b/zyte_api/_async.py @@ -4,7 +4,7 @@ import time from functools import partial from os import environ -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any, Self, cast from warnings import warn import aiohttp @@ -25,7 +25,6 @@ if TYPE_CHECKING: from collections.abc import Awaitable, Callable, Iterator from contextlib import AbstractAsyncContextManager - from typing import Self from eth_account.signers.local import LocalAccount diff --git a/zyte_api/_sync.py b/zyte_api/_sync.py index 815b489..29bc4a7 100644 --- a/zyte_api/_sync.py +++ b/zyte_api/_sync.py @@ -2,13 +2,12 @@ import asyncio from asyncio import AbstractEventLoop -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Self from ._async import AsyncZyteAPI if TYPE_CHECKING: from collections.abc import Awaitable, Generator, Iterator - from typing import Self from aiohttp import ClientSession from tenacity import AsyncRetrying diff --git a/zyte_api/errors.py b/zyte_api/errors.py index ff7d19d..b540357 100644 --- a/zyte_api/errors.py +++ b/zyte_api/errors.py @@ -1,7 +1,6 @@ from __future__ import annotations import json -from typing import Optional import attr @@ -20,12 +19,12 @@ class ParsedError: #: JSON-decoded response body. #: #: If ``None``, :data:`parse_error` indicates the reason. - data: Optional[dict] + data: dict | None #: If :data:`data` is ``None``, this indicates whether the reason is that #: :data:`response_body` is not valid JSON (``"bad_json"``) or that it is #: not a JSON object (``"bad_format"``). - parse_error: Optional[str] + parse_error: str | None @classmethod def from_body(cls, response_body: bytes) -> ParsedError: @@ -46,7 +45,7 @@ def from_body(cls, response_body: bytes) -> ParsedError: return cls(response_body=response_body, data=data, parse_error=parse_error) @property - def type(self) -> Optional[str]: + def type(self) -> str | None: """ID of the error type, e.g. ``"/limits/over-user-limit"`` or ``"/download/temporary-error"``.""" data = self.data or {} diff --git a/zyte_api/stats.py b/zyte_api/stats.py index 44f784e..6d6b87d 100644 --- a/zyte_api/stats.py +++ b/zyte_api/stats.py @@ -3,7 +3,7 @@ import functools import time from collections import Counter -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Self import attr from runstats import Statistics @@ -12,7 +12,7 @@ if TYPE_CHECKING: from collections.abc import Callable - from typing import ParamSpec, Self + from typing import ParamSpec _P = ParamSpec("_P") @@ -95,29 +95,29 @@ class ResponseStats: _start: float = attr.ib(repr=False) # Wait time, before this request is sent. Can be large in case of retries. - time_delayed: Optional[float] = attr.ib(default=None) + time_delayed: float | None = attr.ib(default=None) # Time between sending a request and having a connection established - time_connect: Optional[float] = attr.ib(default=None) + time_connect: float | None = attr.ib(default=None) # Time to read & decode the response - time_read: Optional[float] = attr.ib(default=None) + time_read: float | None = attr.ib(default=None) # time to get an exception (usually, a network error) - time_exception: Optional[float] = attr.ib(default=None) + time_exception: float | None = attr.ib(default=None) # Total time to process the response, excluding the wait time caused # by retries. - time_total: Optional[float] = attr.ib(default=None) + time_total: float | None = attr.ib(default=None) # HTTP status code - status: Optional[int] = attr.ib(default=None) + status: int | None = attr.ib(default=None) # error (parsed), in case of error response - error: Optional[ParsedError] = attr.ib(default=None) + error: ParsedError | None = attr.ib(default=None) # exception raised - exception: Optional[Exception] = attr.ib(default=None) + exception: Exception | None = attr.ib(default=None) @classmethod def create(cls, start_global: float) -> Self: