diff --git a/CLAUDE.md b/CLAUDE.md index 547cbf4e8..3c54ecf76 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -66,6 +66,27 @@ a domain path when the name it needs is not bound there. - Add specific `type: ignore` comments (e.g., `# type: ignore[no-any-return]`) rather than blanket ignores - Test type checking in CI across multiple Python versions (3.10+) +## Import-time model patches must defer their rebuild (#1694) + +`_ergonomic`, `_forward_compat`, `canonical_creative` and `aliases` edit generated +models' `model_fields` at import. Never follow such an edit with +`model_rebuild(force=True)`: pydantic does not share nested `defer_build` +schemas between parents, so each forced rebuild of a request/response root +walks its whole subtree into pydantic-core — two dozen of them cost ~400 MiB +and 10 s at import on 9.3.0. Call `adcp.types.base.defer_model_rebuild(model)` +instead; the patched fields take effect on the model's first use. + +A generated `class X(RootModel[A | B])` builds at the `RootModel[...]` +expression, before any config `X` declares. Generated composing roots +therefore parametrize `adcp.types.base.AdCPRootModel` (deferred); the +post-generate fixer does the rewrite. Every deferred SDK model also carries +`_DeferredValSer` placeholders (installed by `__pydantic_init_subclass__`) that +build on *instance* attribute access, which is how pydantic-core reaches a +serializer when it dumps by inference (`Any` field, `serialize_as_any`); a +subclass that overrides `__pydantic_init_subclass__` must call `super()`. +`tests/test_import_memory.py` imports +each entry module in a subprocess and fails when peak RSS exceeds its budget. + ## ctx_metadata: write-only credentials prohibited `RequestContext.metadata` is for non-secret request hints, not credentials. The diff --git a/scripts/generate_ergonomic_coercion.py b/scripts/generate_ergonomic_coercion.py index 7701a78e0..06c1fe42b 100644 --- a/scripts/generate_ergonomic_coercion.py +++ b/scripts/generate_ergonomic_coercion.py @@ -369,15 +369,18 @@ def _find_media_buy_success_variant(module: Any) -> type[_PydBaseModel] | None: '"""Apply type coercion to generated types for better ergonomics.', "", "This module patches the generated types to accept more flexible input types", - "while maintaining type safety. It uses Pydantic's model_rebuild() to add", - "BeforeValidator annotations to fields.", + "while maintaining type safety. It edits the generated field metadata and then", + "invalidates each patched model with ``defer_model_rebuild`` so pydantic", + "re-derives its core schema, with the coercion validators included, on first use.", "", "Why import-time patching?", " We apply coercion at module load time rather than lazily because:", " 1. Pydantic validation runs during __init__, before any lazy access", - " 2. model_rebuild() is the standard Pydantic pattern for post-hoc changes", - " 3. The cost is minimal (~10-20ms for all types, once at import)", - " 4. After import, there is zero runtime overhead", + " 2. Editing field metadata and rebuilding is the standard Pydantic pattern", + " for post-hoc changes", + " 3. The patch itself is cheap; the rebuild it needs is deferred to the", + " model's first use rather than paid at import for every model (#1694)", + " 4. After the first use of a model, there is zero runtime overhead", " 5. This approach maintains full type checker compatibility", "", "Coercion rules applied:", @@ -401,6 +404,7 @@ def _find_media_buy_success_variant(module: Any) -> type[_PydBaseModel] | None: "", "from pydantic import BeforeValidator, WrapValidator", "", + "from adcp.types.base import defer_model_rebuild", "from adcp.types.coercion import (", " coerce_subclass_list,", " coerce_to_enum,", @@ -546,14 +550,14 @@ def _find_media_buy_success_variant(module: Any) -> type[_PydBaseModel] | None: for c in coercions: if c["type"] == "enum": target = get_symbol_name(c["target_class"]) - field_comments.append(f'{c["field"]}: {target} | str | None') + field_comments.append(f"{c['field']}: {target} | str | None") elif c["type"] in {"enum_list", "unique_enum_list"}: target = get_symbol_name(c["target_class"]) - field_comments.append(f'{c["field"]}: list[{target} | str] | None') + field_comments.append(f"{c['field']}: list[{target} | str] | None") elif c["type"] == "context": - field_comments.append(f'{c["field"]}: ContextObject | dict | None') + field_comments.append(f"{c['field']}: ContextObject | dict | None") elif c["type"] == "ext": - field_comments.append(f'{c["field"]}: ExtensionObject | dict | None') + field_comments.append(f"{c['field']}: ExtensionObject | dict | None") elif c["type"] == "subclass_list": from collections.abc import Sequence as AbcSequence @@ -563,7 +567,7 @@ def _find_media_buy_success_variant(module: Any) -> type[_PydBaseModel] | None: is_seq = get_origin(base_ann if base_ann is not None else ann) is AbcSequence container = "Sequence" if is_seq else "list" field_comments.append( - f'{c["field"]}: {container}[{get_symbol_name(c["target_class"])}] ' + f"{c['field']}: {container}[{get_symbol_name(c['target_class'])}] " "(accepts subclass instances)" ) @@ -648,7 +652,7 @@ def _find_media_buy_success_variant(module: Any) -> type[_PydBaseModel] | None: lines.append(" ],") lines.append(" )") - lines.append(f" {type_name}.model_rebuild(force=True)") + lines.append(f" defer_model_rebuild({type_name})") lines.append("") # PackageUpdate is now a single class (flattened from validation-only oneOf) diff --git a/src/adcp/types/_ergonomic.py b/src/adcp/types/_ergonomic.py index 4ff4c5c11..c35ab2568 100644 --- a/src/adcp/types/_ergonomic.py +++ b/src/adcp/types/_ergonomic.py @@ -6,15 +6,18 @@ """Apply type coercion to generated types for better ergonomics. This module patches the generated types to accept more flexible input types -while maintaining type safety. It uses Pydantic's model_rebuild() to add -BeforeValidator annotations to fields. +while maintaining type safety. It edits the generated field metadata and then +invalidates each patched model with ``defer_model_rebuild`` so pydantic +re-derives its core schema, with the coercion validators included, on first use. Why import-time patching? We apply coercion at module load time rather than lazily because: 1. Pydantic validation runs during __init__, before any lazy access - 2. model_rebuild() is the standard Pydantic pattern for post-hoc changes - 3. The cost is minimal (~10-20ms for all types, once at import) - 4. After import, there is zero runtime overhead + 2. Editing field metadata and rebuilding is the standard Pydantic pattern + for post-hoc changes + 3. The patch itself is cheap; the rebuild it needs is deferred to the + model's first use rather than paid at import for every model (#1694) + 4. After the first use of a model, there is zero runtime overhead 5. This approach maintains full type checker compatibility Coercion rules applied: @@ -38,6 +41,7 @@ from pydantic import BeforeValidator, WrapValidator +from adcp.types.base import defer_model_rebuild from adcp.types.coercion import ( coerce_subclass_list, coerce_to_enum, @@ -159,7 +163,7 @@ def _apply_coercion() -> None: "ext", Annotated[ExtensionObject | None, BeforeValidator(coerce_to_model(ExtensionObject))], ) - ListCreativeFormatsRequest.model_rebuild(force=True) + defer_model_rebuild(ListCreativeFormatsRequest) # Apply coercion to ListCreativeFormatsRequestCreativeAgent # - type: Type | str | None @@ -207,7 +211,7 @@ def _apply_coercion() -> None: "ext", Annotated[ExtensionObject | None, BeforeValidator(coerce_to_model(ExtensionObject))], ) - ListCreativeFormatsRequestCreativeAgent.model_rebuild(force=True) + defer_model_rebuild(ListCreativeFormatsRequestCreativeAgent) # Apply coercion to ListCreativesRequest # - assignment_projection: AssignmentProjection | str | None @@ -239,7 +243,7 @@ def _apply_coercion() -> None: "ext", Annotated[ExtensionObject | None, BeforeValidator(coerce_to_model(ExtensionObject))], ) - ListCreativesRequest.model_rebuild(force=True) + defer_model_rebuild(ListCreativesRequest) # Apply coercion to Sort # - field: CreativeSortField | str | None @@ -254,7 +258,7 @@ def _apply_coercion() -> None: "direction", Annotated[SortDirection | None, BeforeValidator(coerce_to_enum(SortDirection))], ) - Sort.model_rebuild(force=True) + defer_model_rebuild(Sort) # Apply coercion to GetProductsRequest # - buying_mode: BuyingMode | str | None @@ -284,7 +288,7 @@ def _apply_coercion() -> None: "ext", Annotated[ExtensionObject | None, BeforeValidator(coerce_to_model(ExtensionObject))], ) - GetProductsRequest.model_rebuild(force=True) + defer_model_rebuild(GetProductsRequest) # Apply coercion to PackageRequest # - pacing: Pacing | str | None @@ -323,7 +327,7 @@ def _apply_coercion() -> None: "ext", Annotated[ExtensionObject | None, BeforeValidator(coerce_to_model(ExtensionObject))], ) - PackageRequest.model_rebuild(force=True) + defer_model_rebuild(PackageRequest) # Apply coercion to CreateMediaBuyRequest # - packages: Sequence[PackageRequest] (accepts subclass instances) @@ -367,7 +371,7 @@ def _apply_coercion() -> None: "ext", Annotated[ExtensionObject | None, BeforeValidator(coerce_to_model(ExtensionObject))], ) - CreateMediaBuyRequest.model_rebuild(force=True) + defer_model_rebuild(CreateMediaBuyRequest) # Apply coercion to PackageUpdate # - pacing: Pacing | str | None @@ -406,7 +410,7 @@ def _apply_coercion() -> None: "ext", Annotated[ExtensionObject | None, BeforeValidator(coerce_to_model(ExtensionObject))], ) - PackageUpdate.model_rebuild(force=True) + defer_model_rebuild(PackageUpdate) # Apply coercion to GetProductsResponse # - context: ContextObject | dict | None @@ -451,7 +455,7 @@ def _apply_coercion() -> None: "ext", Annotated[ExtensionObject | None, BeforeValidator(coerce_to_model(ExtensionObject))], ) - GetProductsResponse.model_rebuild(force=True) + defer_model_rebuild(GetProductsResponse) # Apply coercion to ListCreativesResponse # - context: ContextObject | dict | None @@ -490,7 +494,7 @@ def _apply_coercion() -> None: "ext", Annotated[ExtensionObject | None, BeforeValidator(coerce_to_model(ExtensionObject))], ) - ListCreativesResponse.model_rebuild(force=True) + defer_model_rebuild(ListCreativesResponse) # Apply coercion to ListCreativeFormatsResponse # - context: ContextObject | dict | None @@ -544,7 +548,7 @@ def _apply_coercion() -> None: "ext", Annotated[ExtensionObject | None, BeforeValidator(coerce_to_model(ExtensionObject))], ) - ListCreativeFormatsResponse.model_rebuild(force=True) + defer_model_rebuild(ListCreativeFormatsResponse) # Apply coercion to CreateMediaBuyResponse1 # - context: ContextObject | dict | None @@ -589,7 +593,7 @@ def _apply_coercion() -> None: "ext", Annotated[ExtensionObject | None, BeforeValidator(coerce_to_model(ExtensionObject))], ) - CreateMediaBuyResponse1.model_rebuild(force=True) + defer_model_rebuild(CreateMediaBuyResponse1) # Apply coercion to UpdateMediaBuyResponse1 # - context: ContextObject | dict | None @@ -634,7 +638,7 @@ def _apply_coercion() -> None: "ext", Annotated[ExtensionObject | None, BeforeValidator(coerce_to_model(ExtensionObject))], ) - UpdateMediaBuyResponse1.model_rebuild(force=True) + defer_model_rebuild(UpdateMediaBuyResponse1) # Apply coercion to GetMediaBuyDeliveryResponse # - context: ContextObject | dict | None @@ -679,7 +683,7 @@ def _apply_coercion() -> None: "ext", Annotated[ExtensionObject | None, BeforeValidator(coerce_to_model(ExtensionObject))], ) - GetMediaBuyDeliveryResponse.model_rebuild(force=True) + defer_model_rebuild(GetMediaBuyDeliveryResponse) def _patch_unique_enum_list( diff --git a/src/adcp/types/_forward_compat.py b/src/adcp/types/_forward_compat.py index a31b2c90b..2ffb5cc67 100644 --- a/src/adcp/types/_forward_compat.py +++ b/src/adcp/types/_forward_compat.py @@ -59,7 +59,7 @@ from pydantic_core import CoreSchema, InitErrorDetails, PydanticCustomError, core_schema from pydantic_core.core_schema import ErrorType -from adcp.types.base import AdCPBaseModel +from adcp.types.base import AdCPBaseModel, defer_model_rebuild from adcp.types.canonical_creative import CreateMediaBuyRequest as _PublicCreateMediaBuyRequest from adcp.types.canonical_creative import GetProductsRequest as _PublicGetProductsRequest from adcp.types.canonical_creative import PackageRequest as PublicPackageRequest @@ -244,8 +244,10 @@ def _patch_model_field(model: type[BaseModel], field_name: str, new_annotation: """Replace a Pydantic model field's annotation in-place. Sets both model_fields (the FieldInfo dict Pydantic uses for schema - generation) and __annotations__ (for introspection), then forces a - schema rebuild. Preserves all constraints and metadata from the original field. + generation) and __annotations__ (for introspection). The caller then + invalidates the model with ``defer_model_rebuild`` so its next use builds + from the patched field. Preserves all constraints and metadata from the + original field. """ old_fi = model.model_fields.get(field_name) if old_fi is None: @@ -392,7 +394,7 @@ def _patch_targeting_overlay(model: type[BaseModel]) -> None: ), ], ) - model.model_rebuild(force=True) + defer_model_rebuild(model) def _patch_package_forecast_targeting() -> None: @@ -410,9 +412,12 @@ def _patch_package_forecast_targeting() -> None: forecast_model = forecast_module.PackageForecastInput _patch_targeting_overlay(forecast_model) request_module = import_module("adcp.types.domains.media_buy.get_products_request") - # _ergonomic eagerly builds get_products before this patch. Rebuild its - # product-scoped union arm first, then both public request parents. Names - # of generated anonymous union arms change during code generation. + # Invalidate every parent that nests the forecast input -- its + # product-scoped union arm, then both public request parents -- so none + # keeps a schema built before this patch (#1694 defers the builds to first + # use, so at import nothing is built yet; invalidating is what keeps that + # true whatever ran first). Names of generated anonymous union arms change + # during code generation. for model in vars(request_module).values(): if ( isinstance(model, type) @@ -423,9 +428,9 @@ def _patch_package_forecast_targeting() -> None: for field in model.model_fields.values() ) ): - model.model_rebuild(force=True) - request_module.GetProductsRequest.model_rebuild(force=True) - _PublicGetProductsRequest.model_rebuild(force=True) + defer_model_rebuild(model) + defer_model_rebuild(request_module.GetProductsRequest) + defer_model_rebuild(_PublicGetProductsRequest) def _apply_forward_compat() -> None: @@ -443,9 +448,9 @@ def _apply_forward_compat() -> None: _patch_targeting_overlay(model) _patch_package_forecast_targeting() - # _ergonomic eagerly builds this legacy parent before the targeting patch. - # Refresh its cached nested validator as well as the package model itself. - _GeneratedCreateMediaBuyRequest.model_rebuild(force=True) + # _ergonomic touched this legacy parent before the targeting patch. + # Invalidate it too, so its first build nests the patched package input. + defer_model_rebuild(_GeneratedCreateMediaBuyRequest) # Response manifests retain unknown future kinds because the field is # ``str`` everywhere. These patches thread the readback NODES through the @@ -457,30 +462,30 @@ def _apply_forward_compat() -> None: # ``format_kind`` needs no patch: it is generated as ``str``, so a response # retains an unknown kind without one. _patch_model_field(DeliveryCreative, "variants", GenericAlias(list, _DeliveryVariant)) - DeliveryCreative.model_rebuild(force=True) - GetCreativeDeliveryResponse.model_rebuild(force=True) + defer_model_rebuild(DeliveryCreative) + defer_model_rebuild(GetCreativeDeliveryResponse) _patch_model_field(PreviewCreativeResponse3, "manifest", _ReadbackCreativeManifest | None) - PreviewCreativeResponse3.model_rebuild(force=True) + defer_model_rebuild(PreviewCreativeResponse3) _patch_model_field(BuildCreativeResponse1, "creative_manifest", _ReadbackCreativeManifest) - BuildCreativeResponse1.model_rebuild(force=True) + defer_model_rebuild(BuildCreativeResponse1) _patch_model_field( BuildCreativeResponse3, "creative_manifests", GenericAlias(list, _ReadbackCreativeManifest) ) - BuildCreativeResponse3.model_rebuild(force=True) + defer_model_rebuild(BuildCreativeResponse3) _patch_model_field( BuildCreativeResponse4, "creatives", GenericAlias(list, _BuildReadbackCreative) ) - BuildCreativeResponse4.model_rebuild(force=True) + defer_model_rebuild(BuildCreativeResponse4) for response in (ContextMatchResponseRouterPublisher, ContextMatchResponseProviderRouter): _patch_model_field(response, "offers", GenericAlias(list, _ReadbackOffer)) - response.model_rebuild(force=True) + defer_model_rebuild(response) - # This eager wrapper captured build/preview validators before the patches. - # Rebuilding it re-resolves AdcpAsyncResponseData, a union type alias, so - # completed task callbacks retain typed manifests. - McpWebhookPayload.model_rebuild(force=True) + # This wrapper nests the build/preview responses patched above. + # Invalidating it re-resolves AdcpAsyncResponseData, a union type alias, + # on first use, so completed task callbacks retain typed manifests. + defer_model_rebuild(McpWebhookPayload) # The 3.1 canonical-creatives capability was published after the bundled # generated model. Preserve it across code generation until the schema @@ -507,7 +512,7 @@ def _apply_forward_compat() -> None: ), ) features_model.__annotations__["canonical_creatives"] = canonical_creatives_annotation - features_model.model_rebuild(force=True) + defer_model_rebuild(features_model) # Canonical schemas referenced through the bundled capabilities schema are # generated a second time as identity-distinct Pydantic classes. Accept the @@ -519,7 +524,7 @@ def _apply_forward_compat() -> None: BundledAcceptancePolicyDiscovery, AcceptancePolicyDiscovery | None, ) - BundledCapabilitiesMediaBuy.model_rebuild(force=True) + defer_model_rebuild(BundledCapabilitiesMediaBuy) _patch_equivalent_model_field( BundledCapabilitiesPortfolio, @@ -533,7 +538,7 @@ def _apply_forward_compat() -> None: BundledPrimaryCountry, list[PrimaryCountry] | None, ) - BundledCapabilitiesPortfolio.model_rebuild(force=True) + defer_model_rebuild(BundledCapabilitiesPortfolio) _apply_forward_compat() diff --git a/src/adcp/types/aliases.py b/src/adcp/types/aliases.py index e9b0ca238..740a142e6 100644 --- a/src/adcp/types/aliases.py +++ b/src/adcp/types/aliases.py @@ -39,6 +39,7 @@ from pydantic import BeforeValidator, Discriminator from adcp.types import _generated as _g +from adcp.types.base import defer_model_rebuild from adcp.types._generated import ( # Account reference variants AccountReference1, @@ -2686,8 +2687,9 @@ def _attach_selector_xor_validator(cls: type) -> None: generated selector classes can't carry the validator without modifying generated code (forbidden — overwritten on next regen). This walks the same `_internal._decorators` machinery the decorator - syntax uses, then forces a `model_rebuild` so Pydantic re-derives - its core schema with the new validator included. + syntax uses, then invalidates the built schema (`defer_model_rebuild`) + so Pydantic re-derives its core schema with the new validator included + on first use. """ cls._selector_xor_validate = _selector_xor_validate # type: ignore[attr-defined] info = _ModelValidatorDecoratorInfo(mode="after") @@ -2700,7 +2702,7 @@ def _attach_selector_xor_validator(cls: type) -> None: cls.__pydantic_decorators__.model_validators[ # type: ignore[attr-defined] "_selector_xor_validate" ] = decorator - cls.model_rebuild(force=True) # type: ignore[attr-defined] + defer_model_rebuild(cls) _attach_selector_xor_validator(_Selector1) diff --git a/src/adcp/types/base.py b/src/adcp/types/base.py index 64d859ff6..f3a17e615 100644 --- a/src/adcp/types/base.py +++ b/src/adcp/types/base.py @@ -46,6 +46,7 @@ WithJsonSchema, model_serializer, ) +from pydantic._internal._mock_val_ser import MockValSer, set_model_mocks from pydantic.fields import FieldInfo from pydantic_core import PydanticSerializationError, PydanticUndefined, core_schema @@ -155,6 +156,86 @@ def _build_deferred_serializers(value: Any, seen: set[int]) -> None: _build_deferred_serializers(item, seen) +class _DeferredValSer(MockValSer[Any]): + """pydantic's rebuild-on-access placeholder, extended to instance attribute access. + + ``MockValSer`` rebuilds the model when one of its *methods* is accessed, + which covers ``Model(...)`` and ``instance.model_dump()``. pydantic-core + reaches a serializer another way: when it serializes a model instance it + meets by inference — an ``Any``-typed field, ``serialize_as_any``, a + plain-pydantic parent — it reads ``instance.__pydantic_serializer__`` and + downcasts, which a placeholder fails with ``TypeError: 'MockValSer' object + is not an instance of 'SchemaSerializer'`` (#1310, #1339). A placeholder + that is also a descriptor builds the model on that instance-level read and + hands back the real object. Class-level reads still return the placeholder, + which is what pydantic's own ``isinstance(..., MockValSer)`` checks expect. + """ + + __slots__ = ("_attr",) + + def __init__(self, mock: MockValSer[Any], attr: str) -> None: + super().__init__( + mock._error_message, + code=mock._code, + val_or_ser="validator" if attr == "__pydantic_validator__" else "serializer", + attempt_rebuild=mock._attempt_rebuild, + ) + self._attr = attr + + def __get__(self, instance: object, owner: type[BaseModel] | None = None) -> Any: + if instance is None or owner is None: + return self + # An empty parent namespace: the model's own module and + # ``__pydantic_parent_namespace__`` resolve everything a generated or + # adopter class declares at module level. A class that still needs the + # caller's locals keeps the placeholder, and pydantic's frame-walking + # rebuild runs on the next method access as before. + if owner.model_rebuild(raise_errors=False, _parent_namespace_depth=0) is False: + return self + return getattr(owner, self._attr) + + +def _install_deferred_valsers(cls: type[BaseModel]) -> None: + """Swap pydantic's placeholders on ``cls`` for ones that build on instance access.""" + for attr in ("__pydantic_validator__", "__pydantic_serializer__"): + mock = cls.__dict__.get(attr) + if isinstance(mock, MockValSer) and not isinstance(mock, _DeferredValSer): + setattr(cls, attr, _DeferredValSer(mock, attr)) + + +def defer_model_rebuild(model: type[BaseModel]) -> None: + """Discard ``model``'s built core schema so pydantic rebuilds it on first use. + + The import-time patchers (``_ergonomic``, ``_forward_compat``, + ``canonical_creative``) edit ``model_fields`` in place and then need the + core schema re-derived. ``model_rebuild(force=True)`` does that + immediately: it walks the model's whole nested subtree into a fresh + pydantic-core validator and serializer, and pydantic does not share the + schemas of nested ``defer_build`` models between parents, so each forced + rebuild of a request or response root costs tens of MiB and hundreds of + milliseconds — at import, for models the process may never use (#1694). + + This helper instead puts the model back into the state ``defer_build=True`` + leaves a freshly declared class in: the built artefacts are replaced by + pydantic's rebuild-on-access placeholders, and the next validation, + serialization or schema request builds the schema from the patched fields. + The outcome is identical to a forced rebuild; only the moment differs. + :func:`adcp.testing.build_all_models` still realizes every model eagerly + for callers (freezegun) that need the build to happen before patching + ``datetime``. + + The placeholders are pydantic's own (``pydantic._internal._mock_val_ser``); + the pinned ``pydantic<2.14`` range is what makes relying on them safe, and + ``tests/test_lazy_rebuild.py`` grades the contract. + """ + model.__pydantic_complete__ = False + for attr in ("__pydantic_core_schema__", "__pydantic_validator__", "__pydantic_serializer__"): + if attr in model.__dict__ and not isinstance(getattr(model, attr), MockValSer): + delattr(model, attr) + set_model_mocks(model) + _install_deferred_valsers(model) + + def _pluralize(count: int, singular: str, plural: str | None = None) -> str: """Return singular or plural form based on count.""" if count == 1: @@ -761,6 +842,14 @@ class AdCPBaseModel(BaseModel): validate_by_name=True, ) + @classmethod + def __pydantic_init_subclass__(cls, **kwargs: Any) -> None: + super().__pydantic_init_subclass__(**kwargs) + # The deferred build's placeholders, made to build on instance access + # as well (see ``_DeferredValSer``). A subclass that overrides this + # hook must call ``super()``. + _install_deferred_valsers(cls) + # Pydantic derives serialization JSON Schema from the model fields only when # this serializer's return is unannotated. Even ``-> Any`` erases the shape. @model_serializer(mode="wrap") @@ -1005,8 +1094,23 @@ class AdCPRootModel(RootModel[_RootT], Generic[_RootT]): wrapper's dump differed from the arm's; this base gives it :class:`AdCPBaseModel`'s defaults (``exclude_none``, ``serialize_as_any``) so a root dumps the document its arm would (#1613). + + It also parametrizes under the SDK's deferred-build policy. ``class + X(RootModel[A | B])`` builds the union's validator at the ``RootModel[...]`` + expression, before ``X`` or any config it declares exists; a composing root + such as ``adagents.AdcpAgentsAuthorization`` walked most of its domain into + pydantic-core at import that way (#1694). Parametrizing this class inherits + ``defer_build=True`` into ``AdCPRootModel[A | B]`` and into ``X``, so the + build waits for first use like every other generated model's. """ + model_config = ConfigDict(defer_build=True) + + @classmethod + def __pydantic_init_subclass__(cls, **kwargs: Any) -> None: + super().__pydantic_init_subclass__(**kwargs) + _install_deferred_valsers(cls) + def model_dump(self, **kwargs: Any) -> Any: if "exclude_none" not in kwargs: kwargs["exclude_none"] = True diff --git a/src/adcp/types/canonical_creative.py b/src/adcp/types/canonical_creative.py index 27edc8c02..1ed6e433c 100644 --- a/src/adcp/types/canonical_creative.py +++ b/src/adcp/types/canonical_creative.py @@ -104,7 +104,13 @@ check_declaration_rules, ) from adcp.types._scalar import _ScalarRoot -from adcp.types.base import _REFUSE_JSON_NULL, AdCPBaseModel, NoExplicitNull, inherit_field +from adcp.types.base import ( + _REFUSE_JSON_NULL, + AdCPBaseModel, + NoExplicitNull, + defer_model_rebuild, + inherit_field, +) from adcp.types.domains.core.canonical_format_kind import CanonicalFormatKind from adcp.types.domains.core.creative_asset import CreativeAsset as _CanonicalCreativeWire from adcp.types.domains.core.creative_filters import CreativeFilters as _LegacyCreativeFilters @@ -721,13 +727,14 @@ def __pydantic_init_subclass__(cls, **kwargs: Any) -> None: are all unaffected. Measured scope: one such validator, on one model. """ + super().__pydantic_init_subclass__(**kwargs) removed = [name for name in cls.model_fields if is_legacy_creative_identity_key(name)] for name in removed: del cls.model_fields[name] cls.__annotations__.pop(name, None) setattr(cls, name, None) if removed: - cls.model_rebuild(force=True) + defer_model_rebuild(cls) @model_validator(mode="before") @classmethod diff --git a/tests/test_canonical_removed_fields.py b/tests/test_canonical_removed_fields.py index d0ad661b8..30c022a5b 100644 --- a/tests/test_canonical_removed_fields.py +++ b/tests/test_canonical_removed_fields.py @@ -6,7 +6,7 @@ ``CanonicalBoundaryModel.__pydantic_init_subclass__`` removes those inherited declarations, and it does so through pydantic internals: it deletes from ``cls.model_fields``, pops ``__annotations__``, binds the name to ``None`` and -forces ``model_rebuild``. None of that is a supported pydantic API. It works on +defers the rebuild to first use. None of that is a supported pydantic API. It works on 2.13; a minor release could change any of it without saying so. This module is the alarm for that. For every canonical model that inherits a diff --git a/tests/test_import_memory.py b/tests/test_import_memory.py new file mode 100644 index 000000000..f55dda5b9 --- /dev/null +++ b/tests/test_import_memory.py @@ -0,0 +1,78 @@ +"""Peak RSS of importing the SDK's entry modules stays under a budget (#1694). + +Importing ``adcp.client`` or ``adcp.server`` realizes the generated type +surface through ``adcp.types._eager``. On 9.3.0 that cost about 520 MiB of +resident memory — 3.4x the 6.6 figure — because the import-time patchers +forced ``model_rebuild`` on two dozen request and response roots and every +generated ``RootModel`` built its validator at class creation. A seller that +ran in a 512 MiB container was OOM-killed at startup. + +Each module is imported in a fresh ``python -I`` subprocess, as an adopter's +process would, and its own peak RSS is read back. The budget sits well above the +measured figure (about 115 MiB for ``adcp.types._eager`` and 185 MiB for +``adcp.server`` on Linux / CPython 3.12) so platform and interpreter variance +do not trip it, and well below the 512 MiB the issue was about, so a change +that reintroduces an import-time build of a large model subtree fails here +rather than in an adopter's container. +""" + +from __future__ import annotations + +import subprocess +import sys + +import pytest + +pytest.importorskip("resource", reason="peak RSS needs a POSIX platform") + +#: Peak-RSS budget for a bare ``import ``. +IMPORT_RSS_BUDGET_MIB = 320 + +ENTRY_MODULES = [ + "adcp.types._eager", + "adcp.client", + "adcp.server", + "adcp.server.test_controller", + "adcp.decisioning", + "adcp.compat.versions", +] + +_PROBE = """ +import sys +import {module} +if sys.platform == "linux": + # The address space's own high-water mark. ru_maxrss would not do: Linux + # carries the forking parent's peak across exec, so a probe spawned from a + # large pytest process would report pytest's peak rather than its own. + with open("/proc/self/status") as status: + line = next(line for line in status if line.startswith("VmHWM:")) + print(int(line.split()[1]) / 1024) +else: + import resource + rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + # ru_maxrss is KiB on most POSIX platforms and bytes on macOS. + print(rss / (1024 * 1024) if sys.platform == "darwin" else rss / 1024) +""" + + +def _import_peak_rss_mib(module: str) -> float: + result = subprocess.run( + [sys.executable, "-I", "-c", _PROBE.format(module=module)], + capture_output=True, + text=True, + check=True, + timeout=600, + ) + return float(result.stdout.strip().splitlines()[-1]) + + +@pytest.mark.slow +@pytest.mark.parametrize("module", ENTRY_MODULES) +def test_import_peak_rss_within_budget(module: str) -> None: + peak = _import_peak_rss_mib(module) + assert peak < IMPORT_RSS_BUDGET_MIB, ( + f"import {module} peaked at {peak:.0f} MiB RSS, over the {IMPORT_RSS_BUDGET_MIB} MiB " + "budget. Something now builds a large pydantic model subtree at import time: look " + "for a model_rebuild(force=True) that should be defer_model_rebuild(), or a " + "RootModel[...] that should parametrize adcp.types.base.AdCPRootModel (#1694)." + ) diff --git a/tests/test_lazy_rebuild.py b/tests/test_lazy_rebuild.py new file mode 100644 index 000000000..8b2da2e36 --- /dev/null +++ b/tests/test_lazy_rebuild.py @@ -0,0 +1,267 @@ +"""The import-time patchers defer their rebuilds to first use (#1694). + +``_ergonomic``, ``_forward_compat``, ``canonical_creative`` and ``aliases`` +edit generated models' ``model_fields`` at import and then need the core +schema re-derived. They used to call ``model_rebuild(force=True)`` on the +spot, which walked each request or response root's whole subtree into +pydantic-core at import — the bulk of the 520 MiB ``adcp.client`` cost. +:func:`adcp.types.base.defer_model_rebuild` puts the model back into the +``defer_build`` state instead, so the patched fields take effect on first use. + +This module grades that contract on pydantic's private placeholders, which +the helper depends on: that the patched models are NOT built by importing +the surface, that they DO build on first use with the patches applied, and +that a generated ``RootModel`` parametrizes :class:`AdCPRootModel` without +building at class creation. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +from typing import Literal + +import pytest +from pydantic import BaseModel, ConfigDict, Field, RootModel, ValidationError + +from adcp.types.base import AdCPBaseModel, AdCPRootModel, defer_model_rebuild + +# Models each patcher force-rebuilt at import before #1694. Read from a +# subprocess so the assertion is about the import itself, not about whatever +# an earlier test in this process already validated. +_PATCHED_MODELS = { + "adcp.types._forward_compat": [ + "CreateMediaBuyRequest", + "UpdateMediaBuyRequest", + "ReportingWebhook", + "McpWebhookPayload", + "GetCreativeDeliveryResponse", + ], + "adcp.types._ergonomic": [ + "GetProductsRequest", + "ListCreativesResponse", + "GetMediaBuyDeliveryResponse", + ], + "adcp.types.canonical_creative": ["CreativeManifest", "Format", "Product"], + "adcp.types.domains.adagents": ["AdcpAgentsAuthorization"], + "adcp.types.domains.core.format": ["CanonicalParameters"], +} + +_PROBE = """ +import adcp.types._eager +import importlib, json +out = {} +for module, names in SPEC.items(): + mod = importlib.import_module(module) + for name in names: + out[f"{module}.{name}"] = getattr(mod, name).__pydantic_complete__ +print(json.dumps(out)) +""" + + +def test_importing_the_surface_builds_no_patched_model() -> None: + result = subprocess.run( + [sys.executable, "-I", "-c", _PROBE.replace("SPEC", repr(_PATCHED_MODELS))], + capture_output=True, + text=True, + check=True, + timeout=600, + ) + built = {name for name, complete in json.loads(result.stdout).items() if complete} + assert not built, f"importing adcp.types._eager built {sorted(built)} at import time" + + +def test_forward_compat_patches_apply_on_first_use() -> None: + from adcp.types import CreateMediaBuyRequest, ReportingWebhook + + # The reporting_webhook extension (#1547) is a patched-in field. + registration = { + "url": "https://buyer.example/hook", + "authentication": { + "schemes": ["Bearer"], + "credentials": "buyer-reporting-token-1234567890", + }, + "reporting_frequency": "daily", + } + webhook = ReportingWebhook.model_validate({**registration, "operation_id": "op-1"}) + assert webhook.operation_id == "op-1" + with pytest.raises(ValidationError): + ReportingWebhook.model_validate({**registration, "operation_id": ""}) + # And the request parent sees it through its own lazily built schema. + schema = CreateMediaBuyRequest.model_json_schema() + assert "operation_id" in schema["$defs"]["ReportingWebhook"]["properties"] + + +def test_ergonomic_patches_apply_on_first_use() -> None: + from adcp.types.domains.media_buy.list_creative_formats_request import ( + ListCreativeFormatsRequest, + ) + + # The order-preserving unique-items validator is appended to field metadata + # at import and only takes effect once the schema is built. + request = ListCreativeFormatsRequest.model_validate( + {"disclosure_positions": ["overlay", "overlay"]} + ) + assert [p.value for p in request.disclosure_positions or []] == ["overlay"] + + +def test_defer_model_rebuild_rebuilds_from_patched_fields() -> None: + class Node(AdCPBaseModel): + model_config = ConfigDict(defer_build=True) + value: int = 0 + + assert Node.model_validate({"value": 1}).value == 1 + assert Node.__pydantic_complete__ + + # Widen the field the way the patchers do, then defer instead of forcing. + field = Node.model_fields["value"] + field.annotation = int | str + Node.__annotations__["value"] = int | str + defer_model_rebuild(Node) + assert not Node.__pydantic_complete__ + + assert Node.model_validate({"value": "x"}).value == "x" + assert Node.__pydantic_complete__ + assert Node(value=2).model_dump() == {"value": 2} + assert Node.model_json_schema()["properties"]["value"]["anyOf"] == [ + {"type": "integer"}, + {"type": "string"}, + ] + + +def test_defer_model_rebuild_on_a_never_built_model_is_harmless() -> None: + class Fresh(AdCPBaseModel): + model_config = ConfigDict(defer_build=True) + value: int = 0 + + assert not Fresh.__pydantic_complete__ + defer_model_rebuild(Fresh) + assert not Fresh.__pydantic_complete__ + assert Fresh.model_validate({"value": 3}).value == 3 + + +def test_adcp_root_model_parametrizes_deferred() -> None: + class Left(BaseModel): + kind: Literal["left"] = "left" + x: int + + class Right(BaseModel): + kind: Literal["right"] = "right" + y: str + + parametrized = AdCPRootModel[Left | Right] + assert not parametrized.__pydantic_complete__ + # pydantic's own RootModel builds at the subscript, which is the cost + # this base exists to avoid. + assert RootModel[Left | Right].__pydantic_complete__ + + class Either(AdCPRootModel[Left | Right]): + root: Left | Right = Field(discriminator="kind") + + assert not Either.__pydantic_complete__ + assert issubclass(Either, RootModel) + assert Either.model_validate({"kind": "right", "y": "q"}).root == Right(y="q") + assert Either.__pydantic_complete__ + assert Either(Left(x=1)).model_dump() == {"kind": "left", "x": 1} + assert set(Either.model_json_schema()["oneOf"][0]) == {"$ref"} + + +def test_generated_root_models_parametrize_adcp_root_model() -> None: + from adcp.types.domains.adagents import AdcpAgentsAuthorization + from adcp.types.domains.core.format import CanonicalParameters + + for root in (AdcpAgentsAuthorization, CanonicalParameters): + assert issubclass(root, AdCPRootModel), root + assert root.model_config.get("defer_build") is True + + +def test_a_canonical_subclass_may_forward_reference_a_later_model() -> None: + """The removal hook's deferred rebuild tolerates an unresolved forward reference. + + A subclass of a canonical model re-inherits the removed legacy identity, so + ``__pydantic_init_subclass__`` removes it again and used to force a rebuild + there -- which raised when a field named a model defined further down the + module, an ordinary pydantic pattern that plain ``model_rebuild()`` defers + (#1690). Deferring the rebuild leaves the class incomplete until the + caller's own ``model_rebuild()``, with the removal intact. + """ + from adcp.types import PackageRequest + from adcp.types.canonical_creative import is_legacy_creative_identity_key + + class WithForwardRef(PackageRequest): + later: list[Later] | None = None + + class Later(BaseModel): + name: str + + assert not WithForwardRef.__pydantic_complete__ + WithForwardRef.model_rebuild() + + assert WithForwardRef.__pydantic_complete__ + assert "later" in WithForwardRef.model_fields + assert not any(is_legacy_creative_identity_key(name) for name in WithForwardRef.model_fields) + payload = { + "buyer_ref": "p", + "product_id": "prod", + "pricing_option_id": "po", + "later": [{"name": "n"}], + } + assert WithForwardRef.model_validate(payload).later + + +_DUMP_PROBE = """ +from typing import Any +from pydantic import BaseModel +from adcp.types import CreateMediaBuyRequest, Placement +from adcp.types.domains.adagents import AdcpAgentsAuthorization +from adcp.types.domains.core.targeting import TargetingOverlay + + +class Parent(BaseModel): + item: Any + + +# A canonical model, unvalidated: the #1694 CI failure. +assert Parent(item=Placement.model_construct(name="n")).model_dump(mode="json") == { + "item": {"name": "n"} +} +# A config-deferred generated model, unvalidated: a TypeError on 9.3.0. +assert Parent(item=TargetingOverlay.model_construct()).model_dump(mode="json") == {"item": {}} +# A nested instance built through its parent's schema; its own class never built. +request = CreateMediaBuyRequest.model_validate( + { + "buyer_ref": "b", + "brand": {"domain": "brand.example"}, + "packages": [{"buyer_ref": "p", "product_id": "prod", "pricing_option_id": "po"}], + "start_time": "asap", + "end_time": "2030-01-01T00:00:00Z", + "idempotency_key": "k" * 20, + "account": {"account_id": "a"}, + } +) +package = request.packages[0] +assert not type(package).__pydantic_complete__ +assert Parent(item=package).model_dump(mode="json")["item"]["buyer_ref"] == "p" +assert type(package).__pydantic_complete__ +# A deferred generated root, unvalidated. +root = AdcpAgentsAuthorization.model_construct(root=None) +assert Parent(item=root).model_dump(mode="json") == {"item": None} +print("ok") +""" + + +def test_a_deferred_instance_dumps_through_a_plain_pydantic_parent() -> None: + """pydantic-core reads ``instance.__pydantic_serializer__`` when it serializes by + inference (an ``Any`` field, ``serialize_as_any``), and a placeholder there fails + with ``TypeError: 'MockValSer' object is not an instance of 'SchemaSerializer'``. + The SDK's placeholders build the model on that read instead. In a subprocess so + nothing in this process has built the models first.""" + result = subprocess.run( + [sys.executable, "-I", "-c", _DUMP_PROBE], + capture_output=True, + text=True, + timeout=600, + ) + assert result.returncode == 0, result.stderr[-2000:] + assert result.stdout.strip() == "ok"