fix: conformance on caldav's test servers

docs/caldav-servers.md

@@ -25,3 +25,14 @@ With caldav 3.7, whose `caldav.testing` has no search-cache helper,
 nothing waits for an index that lags behind writes: three stories fail on
 Bedework 3 (an event or task just written is not found yet), and
 purelymail fails most stories that search.
+
+## The conformance tests: a known failure, 2026-11-03
+
+SOGo adds `RSVP=TRUE` and `PARTSTAT` to an ATTENDEE that had none, and Ox
+adds `EMAIL=`.  caldav records this
+(`save-load.icalendar.attendee-params`), but no capability declaration
+can say that a server *adds* parameters: LOSSY only describes a server
+dropping or rewriting what it was sent.  The round trip and `verify`
+therefore report the attendees on these servers as changed, and the
+tests fail.  Whether verification should tolerate added parameters, or
+`attendees` should count as LOSSY there, is undecided.

news/+caldav-hints.bugfix.rst

@@ -0,0 +1 @@
+The CalDAV backend now declares what caldav's hints already record: a task list on Ox stores no attendees, recurrences or X- properties (so no planned times), Ox and Zimbra drop ``ESTIMATED-DURATION``, and Zimbra renames every object to ``<uid>.ics``, so ``identity.client-native-id`` is unsupported there.  Where the server keeps ``RELATED-TO`` but refuses an event holding a ``PARTICIPANT`` (CCS, GMX), ``time-log`` is ``lossy`` with ``users`` lost, and entries are written without their users instead of being refused.

news/+caldav-values.bugfix.rst

@@ -0,0 +1 @@
+With a caldav whose hints record them, the CalDAV backend declares the ``STATUS`` and ``PRIORITY`` values a server's task lists keep (Ox: priorities 1, 5 and 9, and no ``PENDING`` or ``FAILED``; Zimbra: no ``CANCELLED``), ``write.wire`` unsupported where X- parameters are dropped, and ``identity.client-native-id`` unsupported in Ox task lists.  On Zimbra, whose calendar-multiget still returns deleted objects, an object is read with a GET.  ``PRIORITY:0`` is left out rather than stored as another priority.  The conformance suite's ``test_values_map_as_declared`` no longer adds one item twice, which failed on a backend keeping the client's UID.

news/+delete-if-match.bugfix.rst

@@ -0,0 +1 @@
+On SOGo, Zimbra, Bedework and Posteo, which delete on a stale ``If-Match``, ``write.conditional`` is now declared ``emulated``: with a caldav that reads the ETag before such a DELETE, ``delete()`` of a changed item raises ``ConflictError`` instead of deleting it, with a short window between the read and the delete.  With an older caldav, ``write.conditional`` is ``unsupported`` on those servers, though ``save()`` stays conditional.

src/calendaring/_async/caldav.py

@@ -21,7 +21,12 @@
 import icalendar
 from icalendar_searcher import Searcher
 
-from .._caldav_features import backend_capabilities, collection_capabilities, stored_components
+from .._caldav_features import (
+    backend_capabilities,
+    collection_capabilities,
+    multiget_serves_deleted,
+    stored_components,
+)
 from .._caldav_io import Calendar
 from ..capabilities import Feature
 from ..errors import (
@@ -115,6 +120,8 @@ def _kinds(self, searcher: Searcher | None) -> frozenset[Component]:
         return self.components & wanted if wanted else self.components
 
     async def _fetch(self, native_id: str) -> StoredObject:
+        if multiget_serves_deleted(self._backend._dav.client.features):  # Zimbra
+            return self._stored(await self._backend._dav.get(self.calendar, self._href(native_id)))
         for obj in await self._backend._dav.multiget(self.calendar, [self._href(native_id)]):
             if obj.data is not None:  # caldav returns a missing href as an object without data
                 return self._stored(obj)

src/calendaring/_async/collection.py

@@ -283,15 +283,21 @@ async def _time_entries(
     async def _time_log(self, item: Item) -> list[TimeEntry]:
         return [entry for _, entry in await self._log_target()._time_entries(None, None, parent=item.uid)]
 
+    def _storable_entry(self, entry: TimeEntry) -> TimeEntry:
+        """``entry`` less the users the log collection declares lost: a server refusing them would refuse it all."""
+        lost = self._log_target().capabilities.details(Feature.TIME_LOG).get("lost", ())
+        return replace(entry, users=()) if "users" in lost else entry
+
     async def _add_time(self, item: Item, entry: TimeEntry) -> TimeEntry:
+        entry = self._storable_entry(entry)
         stored = await self._log_target().add(Event(_wrap_component(entry_to_component(entry, parent=item))))
         return entry_from_component(stored.component)[1]
 
     async def _amend_time(self, item: Item, old: TimeEntry, new: TimeEntry) -> TimeEntry:
         assert old.id is not None
         stored = await self._log_target().get(old.id)
         component = stored.component  # edited in place: what other clients added stays
-        apply_entry(component, new)
+        apply_entry(component, self._storable_entry(new))
         sequence = int(component.pop("SEQUENCE", 0))
         for name, value in (("SEQUENCE", sequence + 1), ("LAST-MODIFIED", datetime.now(UTC))):
             component.pop(name, None)

src/calendaring/_async/conformance.py

@@ -825,7 +825,9 @@ async def test_values_map_as_declared(self, collection: AsyncCollection, feature
         def policy(losses: Sequence[Loss], item: Item) -> None:
             seen.append(losses)
 
-        await collection.add(item, loss=policy)
+        again = Task.new(summary=f"{name} outside, again")  # the first is stored now, under its UID
+        again.component[name] = outside[0]
+        await collection.add(again, loss=policy)
         assert [[loss.property for loss in losses] for losses in seen] == [[name]]
         assert seen[0][0].sent == outside[0]
 

src/calendaring/_async/dav.py

@@ -82,6 +82,14 @@ async def multiget(self, calendar: Calendar, urls: list[str]) -> list[CalendarOb
             found: Any = calendar.multiget([URL.objectify(u) for u in urls])
             return list(await _awaited(cast(Any, found)))
 
+    async def get(self, calendar: Calendar, url: str) -> CalendarObjectResource:
+        """The object at ``url``, read with a GET: ``NotFoundError`` if it is gone."""
+        obj = _resource(self.client, calendar, url)
+        with translated():
+            # No multiget fallback: the reason to GET is a multiget serving deleted objects.
+            await _awaited(obj.load(multiget_fallback=False))
+        return obj
+
     async def etags(self, url: str) -> dict[str, str]:
         with translated():
             # Positional: the sync client's second parameter is props=, which also takes an XML body.

src/calendaring/_caldav_features.py

@@ -15,6 +15,7 @@
 
 from typing import Any
 
+from ._caldav_io import CalendarObjectResource
 from .capabilities import (
     BACKEND_FEATURES,
     EDIT_ICALENDAR_PENDING,
@@ -53,6 +54,30 @@
     Feature.TASK_UNCOMPLETE: "save-load.todo.uncomplete",
 }
 
+# Feature -> the caldav feature probed on a task.  A level is the collection's, not a component's, so
+# a calendar holding events too keeps these: lowering them would cost its events what they can keep.
+TASK_FROM_SERVER = {
+    Feature.ATTENDEES: "save-load.todo.attendee",
+    Feature.RECURRENCE: "save-load.todo.recurrences",
+    Feature.PROPERTIES_PASSTHROUGH: "save-load.icalendar.x-property",  # Ox drops X- properties from tasks only
+}
+
+# The same for what only a task has: lowered wherever tasks are held.
+TASK_ONLY_FROM_SERVER = {
+    Feature.TASK_PLANNED: "save-load.icalendar.x-property",  # X-PYCAL-PLANNED-*
+    Feature.TASK_ESTIMATE: "save-load.icalendar.unknown-property",  # probed with ESTIMATED-DURATION
+}
+
+# Feature -> the caldav features listing the values a task keeps (``values``) and stores as another
+# (``map``), each with the values it probes: what is kept where the server keeps them all.
+VALUES_FROM_SERVER = {
+    Feature.TASK_STATUS: (
+        ("save-load.todo.status", ("NEEDS-ACTION", "IN-PROCESS", "COMPLETED", "CANCELLED")),
+        ("save-load.todo.status.rfc10051", ("PENDING", "FAILED")),  # Ox drops them, Zimbra maps them
+    ),
+    Feature.TASK_PRIORITY: (("save-load.todo.priority", tuple(range(10))),),
+}
+
 COMPONENT_FROM_SERVER = {
     Component.EVENT: (Feature.COMPONENT_EVENT, "save-load.event"),
     Component.TASK: (Feature.COMPONENT_TASK, "save-load.todo"),
@@ -79,7 +104,7 @@ def _hinted(features: Any) -> bool:
 
 
 def _level(features: Any, name: str) -> Support:
-    if not _hinted(features):
+    if not _hinted(features) or name not in getattr(features, "FEATURES", {name}):  # a hint newer than this caldav
         return X
     return LEVELS.get(str(features.is_supported(name, str)), X)
 
@@ -183,6 +208,40 @@ def stored_components(features: Any, listed: frozenset[Component]) -> frozenset[
     return frozenset(c for c in listed if _level(features, COMPONENT_FROM_SERVER[c][1]) is not U)
 
 
+def _values(features: Any, feature: Feature) -> dict[str, Any] | None:
+    """LOSSY with the values the server keeps, where one of the hints says it does not keep them all.
+    A refused value is left out, as is a dropped one: the loss planner drops it before sending.
+    PRIORITY:0 is not sent as the server's stand-in either (Ox: 5), since no PRIORITY means the same as 0."""
+    hints = VALUES_FROM_SERVER[feature]
+    if not any(_level(features, name) is U for name, _ in hints):
+        return None
+    number = int if feature is Feature.TASK_PRIORITY else str  # a hint from YAML or JSON may carry "5"
+    values: list[Any] = []
+    mapping: dict[Any, Any] = {}
+    notes = []
+    seen: list[dict[str, Any]] = []
+    for name, probed in hints:
+        level = _level(features, name)
+        if level is F:
+            values.extend(probed)
+        if level is not U:
+            continue  # unknown: nothing is known to be kept
+        hint = features.is_supported(name, dict)
+        if hint in seen:  # caldav answers a child with its parent's node where only the parent is set
+            continue
+        seen.append(hint)
+        values.extend(number(v) for v in hint.get("values", []))
+        mapping.update({number(k): number(v) for k, v in hint.get("map", {}).items() if number(k) != 0})
+        if hint.get("behaviour"):
+            notes.append(hint["behaviour"])
+    return {"level": L, "values": values, "map": mapping, "note": "; ".join(notes) or None}
+
+
+def multiget_serves_deleted(features: Any) -> bool:
+    """Zimbra answers a GET of a deleted object with 404, and a calendar-multiget of it with the old data."""
+    return _level(features, "multiget.deleted") is U
+
+
 def backend_capabilities(features: Any) -> Capabilities:
     return Capabilities({f: v for f, v in _table(features).items() if f in BACKEND_FEATURES})
 
@@ -196,7 +255,47 @@ def collection_capabilities(features: Any, components: frozenset[Component]) ->
     for kind, (feature, name) in COMPONENT_FROM_SERVER.items():
         if kind in components:
             table[feature] = _level(features, name)
+    if _hinted(features) and str(features.is_supported("delete.if-match", str)) == "broken":
+        if hasattr(CalendarObjectResource, "_ignores_delete_if_match"):  # caldav 3.8
+            table[Feature.WRITE_CONDITIONAL] = {
+                "level": E,
+                "note": "the server deletes on a stale If-Match; caldav reads the ETag first, a window before the delete",
+            }
+        else:
+            table[Feature.WRITE_CONDITIONAL] = {
+                "level": U,
+                "note": "the server deletes on a stale If-Match: delete() is not conditional before caldav 3.8; save() is",
+            }
     if _level(features, "save-load.icalendar.related-to") is U:
         table[Feature.TIME_LOG] = {"level": U, "note": "the server strips RELATED-TO, which ties an entry to its item"}
+    elif _level(features, "save-load.icalendar.participant") is U:
+        table[Feature.TIME_LOG] = {
+            "level": L,
+            "lost": ["users"],
+            "note": "the server refuses an event with a PARTICIPANT, which names who did the work",
+        }
+    if Component.TASK in components:
+        lowered = {**TASK_ONLY_FROM_SERVER, **(TASK_FROM_SERVER if Component.EVENT not in components else {})}
+        for feature, name in lowered.items():
+            if _level(features, name) is U:
+                table[feature] = U
+        for feature in VALUES_FROM_SERVER:  # Ox keeps priorities 1, 5 and 9; Zimbra refuses CANCELLED
+            if (declared := _values(features, feature)) is not None:
+                table[feature] = declared
+    stable_url = features.is_supported("save-load.stable-url", dict) if _hinted(features) else {}
+    if stable_url.get("forces_uid_ics"):
+        table[Feature.IDENTITY_CLIENT_NATIVE_ID] = {"level": U, "note": "the server renames every object <uid>.ics"}
+    elif renamed := sorted(c.value for c in components if c.value in stable_url.get("uid_ics_components", ())):
+        table[Feature.IDENTITY_CLIENT_NATIVE_ID] = {
+            "level": U,
+            "note": f"the server renames a {', '.join(renamed)} <uid>.ics",
+        }
+    x_property = features.is_supported("save-load.icalendar.x-property", dict) if _hinted(features) else {}
+    if _level(features, "save-load.icalendar.x-property") is U and not (x_property.get("behaviour") or "").endswith(
+        "in a task"
+    ):  # dropped from events too
+        table[Feature.PROPERTIES_PASSTHROUGH] = U
+    if _level(features, "save-load.icalendar.x-parameter") is U:
+        table[Feature.WRITE_WIRE] = {"level": U, "note": "the server drops X- parameters: bytes are not stored as sent"}
     held_only(table, components)
     return Capabilities({f: Capability.coerce(v) for f, v in table.items()})

tests/_async/test_caldav_backend.py

@@ -27,6 +27,7 @@
 pytest.importorskip("caldav")
 
 from caldav.collection import Calendar  # noqa: E402
+from caldav.compatibility_hints import FeatureSet  # noqa: E402
 from caldav.lib.url import URL  # noqa: E402
 
 from calendaring._async.caldav import _name_of  # noqa: E402
@@ -346,3 +347,39 @@ async def test_get_of_an_object_that_does_not_parse_says_so(
     serve_broken(collection, monkeypatch)
     with pytest.raises(CalendaringError):
         await collection.get("from-phone")
+
+
+@pytest.mark.skipif("multiget.deleted" not in FeatureSet.FEATURES, reason="caldav has no multiget.deleted hint yet")
+async def test_a_deleted_object_is_not_read_back_where_multiget_still_serves_it(
+    xandikos_server: Any, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """Zimbra answers a GET of a deleted object with 404, and a multiget of it with the old data."""
+    features = {**xandikos_server.config["features"], "multiget.deleted": {"support": "broken"}}
+    backend = await AsyncBackend.open(
+        xandikos_server.url,
+        kind="caldav",
+        username=xandikos_server.username,
+        password=xandikos_server.password,
+        features=features,
+    )
+    async with backend:
+        collection = await backend.create_collection("multiget-ghosts", components=(Component.TASK,))
+        try:
+            native: Any = collection  # the CalDAV collection, for its calendar and its DAV client
+            live = await collection.add_task(summary="kept")
+            assert live.native_id is not None
+            read = await collection.get_by_native_id(live.native_id)
+            assert (read.uid, read.etag) == (live.uid, live.etag), "read with a GET, ETag and all"
+            stored = await collection.add_task(summary="deleted")
+            assert stored.native_id is not None
+            ghost = await native._backend._dav.multiget(native.calendar, [native._href(stored.native_id)])
+            await stored.delete()
+
+            async def haunted(calendar: Any, urls: list[str]) -> list[Any]:
+                return list(ghost)
+
+            monkeypatch.setattr(native._backend._dav, "multiget", haunted)
+            with pytest.raises(NotFoundError):
+                await collection.get_by_native_id(stored.native_id)
+        finally:
+            await collection.delete_collection()