diff --git a/docs/caldav-servers.md b/docs/caldav-servers.md index dbbb4e8..367984e 100644 --- a/docs/caldav-servers.md +++ b/docs/caldav-servers.md @@ -25,3 +25,14 @@ With caldav 3.7, whose `caldav.testing` has no search-cache helper, nothing waits for an index that lags behind writes: three stories fail on Bedework 3 (an event or task just written is not found yet), and purelymail fails most stories that search. + +## The conformance tests: a known failure, 2026-11-03 + +SOGo adds `RSVP=TRUE` and `PARTSTAT` to an ATTENDEE that had none, and Ox +adds `EMAIL=`. caldav records this +(`save-load.icalendar.attendee-params`), but no capability declaration +can say that a server *adds* parameters: LOSSY only describes a server +dropping or rewriting what it was sent. The round trip and `verify` +therefore report the attendees on these servers as changed, and the +tests fail. Whether verification should tolerate added parameters, or +`attendees` should count as LOSSY there, is undecided. diff --git a/news/+caldav-hints.bugfix.rst b/news/+caldav-hints.bugfix.rst new file mode 100644 index 0000000..68e29ad --- /dev/null +++ b/news/+caldav-hints.bugfix.rst @@ -0,0 +1 @@ +The CalDAV backend now declares what caldav's hints already record: a task list on Ox stores no attendees, recurrences or X- properties (so no planned times), Ox and Zimbra drop ``ESTIMATED-DURATION``, and Zimbra renames every object to ``.ics``, so ``identity.client-native-id`` is unsupported there. Where the server keeps ``RELATED-TO`` but refuses an event holding a ``PARTICIPANT`` (CCS, GMX), ``time-log`` is ``lossy`` with ``users`` lost, and entries are written without their users instead of being refused. diff --git a/news/+caldav-values.bugfix.rst b/news/+caldav-values.bugfix.rst new file mode 100644 index 0000000..0b6a5d4 --- /dev/null +++ b/news/+caldav-values.bugfix.rst @@ -0,0 +1 @@ +With a caldav whose hints record them, the CalDAV backend declares the ``STATUS`` and ``PRIORITY`` values a server's task lists keep (Ox: priorities 1, 5 and 9, and no ``PENDING`` or ``FAILED``; Zimbra: no ``CANCELLED``), ``write.wire`` unsupported where X- parameters are dropped, and ``identity.client-native-id`` unsupported in Ox task lists. On Zimbra, whose calendar-multiget still returns deleted objects, an object is read with a GET. ``PRIORITY:0`` is left out rather than stored as another priority. The conformance suite's ``test_values_map_as_declared`` no longer adds one item twice, which failed on a backend keeping the client's UID. diff --git a/news/+delete-if-match.bugfix.rst b/news/+delete-if-match.bugfix.rst new file mode 100644 index 0000000..f03b587 --- /dev/null +++ b/news/+delete-if-match.bugfix.rst @@ -0,0 +1 @@ +On SOGo, Zimbra, Bedework and Posteo, which delete on a stale ``If-Match``, ``write.conditional`` is now declared ``emulated``: with a caldav that reads the ETag before such a DELETE, ``delete()`` of a changed item raises ``ConflictError`` instead of deleting it, with a short window between the read and the delete. With an older caldav, ``write.conditional`` is ``unsupported`` on those servers, though ``save()`` stays conditional. diff --git a/src/calendaring/_async/caldav.py b/src/calendaring/_async/caldav.py index 44eab8c..950b6b3 100644 --- a/src/calendaring/_async/caldav.py +++ b/src/calendaring/_async/caldav.py @@ -21,7 +21,12 @@ import icalendar from icalendar_searcher import Searcher -from .._caldav_features import backend_capabilities, collection_capabilities, stored_components +from .._caldav_features import ( + backend_capabilities, + collection_capabilities, + multiget_serves_deleted, + stored_components, +) from .._caldav_io import Calendar from ..capabilities import Feature from ..errors import ( @@ -115,6 +120,8 @@ def _kinds(self, searcher: Searcher | None) -> frozenset[Component]: return self.components & wanted if wanted else self.components async def _fetch(self, native_id: str) -> StoredObject: + if multiget_serves_deleted(self._backend._dav.client.features): # Zimbra + return self._stored(await self._backend._dav.get(self.calendar, self._href(native_id))) for obj in await self._backend._dav.multiget(self.calendar, [self._href(native_id)]): if obj.data is not None: # caldav returns a missing href as an object without data return self._stored(obj) diff --git a/src/calendaring/_async/collection.py b/src/calendaring/_async/collection.py index 8431c19..26f9c48 100644 --- a/src/calendaring/_async/collection.py +++ b/src/calendaring/_async/collection.py @@ -283,7 +283,13 @@ async def _time_entries( async def _time_log(self, item: Item) -> list[TimeEntry]: return [entry for _, entry in await self._log_target()._time_entries(None, None, parent=item.uid)] + def _storable_entry(self, entry: TimeEntry) -> TimeEntry: + """``entry`` less the users the log collection declares lost: a server refusing them would refuse it all.""" + lost = self._log_target().capabilities.details(Feature.TIME_LOG).get("lost", ()) + return replace(entry, users=()) if "users" in lost else entry + async def _add_time(self, item: Item, entry: TimeEntry) -> TimeEntry: + entry = self._storable_entry(entry) stored = await self._log_target().add(Event(_wrap_component(entry_to_component(entry, parent=item)))) return entry_from_component(stored.component)[1] @@ -291,7 +297,7 @@ async def _amend_time(self, item: Item, old: TimeEntry, new: TimeEntry) -> TimeE assert old.id is not None stored = await self._log_target().get(old.id) component = stored.component # edited in place: what other clients added stays - apply_entry(component, new) + apply_entry(component, self._storable_entry(new)) sequence = int(component.pop("SEQUENCE", 0)) for name, value in (("SEQUENCE", sequence + 1), ("LAST-MODIFIED", datetime.now(UTC))): component.pop(name, None) diff --git a/src/calendaring/_async/conformance.py b/src/calendaring/_async/conformance.py index ab9967c..f6ff83a 100644 --- a/src/calendaring/_async/conformance.py +++ b/src/calendaring/_async/conformance.py @@ -825,7 +825,9 @@ async def test_values_map_as_declared(self, collection: AsyncCollection, feature def policy(losses: Sequence[Loss], item: Item) -> None: seen.append(losses) - await collection.add(item, loss=policy) + again = Task.new(summary=f"{name} outside, again") # the first is stored now, under its UID + again.component[name] = outside[0] + await collection.add(again, loss=policy) assert [[loss.property for loss in losses] for losses in seen] == [[name]] assert seen[0][0].sent == outside[0] diff --git a/src/calendaring/_async/dav.py b/src/calendaring/_async/dav.py index a7e6794..3bf85c3 100644 --- a/src/calendaring/_async/dav.py +++ b/src/calendaring/_async/dav.py @@ -82,6 +82,14 @@ async def multiget(self, calendar: Calendar, urls: list[str]) -> list[CalendarOb found: Any = calendar.multiget([URL.objectify(u) for u in urls]) return list(await _awaited(cast(Any, found))) + async def get(self, calendar: Calendar, url: str) -> CalendarObjectResource: + """The object at ``url``, read with a GET: ``NotFoundError`` if it is gone.""" + obj = _resource(self.client, calendar, url) + with translated(): + # No multiget fallback: the reason to GET is a multiget serving deleted objects. + await _awaited(obj.load(multiget_fallback=False)) + return obj + async def etags(self, url: str) -> dict[str, str]: with translated(): # Positional: the sync client's second parameter is props=, which also takes an XML body. diff --git a/src/calendaring/_caldav_features.py b/src/calendaring/_caldav_features.py index eedd38a..df9c0ee 100644 --- a/src/calendaring/_caldav_features.py +++ b/src/calendaring/_caldav_features.py @@ -15,6 +15,7 @@ from typing import Any +from ._caldav_io import CalendarObjectResource from .capabilities import ( BACKEND_FEATURES, EDIT_ICALENDAR_PENDING, @@ -53,6 +54,30 @@ Feature.TASK_UNCOMPLETE: "save-load.todo.uncomplete", } +# Feature -> the caldav feature probed on a task. A level is the collection's, not a component's, so +# a calendar holding events too keeps these: lowering them would cost its events what they can keep. +TASK_FROM_SERVER = { + Feature.ATTENDEES: "save-load.todo.attendee", + Feature.RECURRENCE: "save-load.todo.recurrences", + Feature.PROPERTIES_PASSTHROUGH: "save-load.icalendar.x-property", # Ox drops X- properties from tasks only +} + +# The same for what only a task has: lowered wherever tasks are held. +TASK_ONLY_FROM_SERVER = { + Feature.TASK_PLANNED: "save-load.icalendar.x-property", # X-PYCAL-PLANNED-* + Feature.TASK_ESTIMATE: "save-load.icalendar.unknown-property", # probed with ESTIMATED-DURATION +} + +# Feature -> the caldav features listing the values a task keeps (``values``) and stores as another +# (``map``), each with the values it probes: what is kept where the server keeps them all. +VALUES_FROM_SERVER = { + Feature.TASK_STATUS: ( + ("save-load.todo.status", ("NEEDS-ACTION", "IN-PROCESS", "COMPLETED", "CANCELLED")), + ("save-load.todo.status.rfc10051", ("PENDING", "FAILED")), # Ox drops them, Zimbra maps them + ), + Feature.TASK_PRIORITY: (("save-load.todo.priority", tuple(range(10))),), +} + COMPONENT_FROM_SERVER = { Component.EVENT: (Feature.COMPONENT_EVENT, "save-load.event"), Component.TASK: (Feature.COMPONENT_TASK, "save-load.todo"), @@ -79,7 +104,7 @@ def _hinted(features: Any) -> bool: def _level(features: Any, name: str) -> Support: - if not _hinted(features): + if not _hinted(features) or name not in getattr(features, "FEATURES", {name}): # a hint newer than this caldav return X return LEVELS.get(str(features.is_supported(name, str)), X) @@ -183,6 +208,40 @@ def stored_components(features: Any, listed: frozenset[Component]) -> frozenset[ return frozenset(c for c in listed if _level(features, COMPONENT_FROM_SERVER[c][1]) is not U) +def _values(features: Any, feature: Feature) -> dict[str, Any] | None: + """LOSSY with the values the server keeps, where one of the hints says it does not keep them all. + A refused value is left out, as is a dropped one: the loss planner drops it before sending. + PRIORITY:0 is not sent as the server's stand-in either (Ox: 5), since no PRIORITY means the same as 0.""" + hints = VALUES_FROM_SERVER[feature] + if not any(_level(features, name) is U for name, _ in hints): + return None + number = int if feature is Feature.TASK_PRIORITY else str # a hint from YAML or JSON may carry "5" + values: list[Any] = [] + mapping: dict[Any, Any] = {} + notes = [] + seen: list[dict[str, Any]] = [] + for name, probed in hints: + level = _level(features, name) + if level is F: + values.extend(probed) + if level is not U: + continue # unknown: nothing is known to be kept + hint = features.is_supported(name, dict) + if hint in seen: # caldav answers a child with its parent's node where only the parent is set + continue + seen.append(hint) + values.extend(number(v) for v in hint.get("values", [])) + mapping.update({number(k): number(v) for k, v in hint.get("map", {}).items() if number(k) != 0}) + if hint.get("behaviour"): + notes.append(hint["behaviour"]) + return {"level": L, "values": values, "map": mapping, "note": "; ".join(notes) or None} + + +def multiget_serves_deleted(features: Any) -> bool: + """Zimbra answers a GET of a deleted object with 404, and a calendar-multiget of it with the old data.""" + return _level(features, "multiget.deleted") is U + + def backend_capabilities(features: Any) -> Capabilities: return Capabilities({f: v for f, v in _table(features).items() if f in BACKEND_FEATURES}) @@ -196,7 +255,47 @@ def collection_capabilities(features: Any, components: frozenset[Component]) -> for kind, (feature, name) in COMPONENT_FROM_SERVER.items(): if kind in components: table[feature] = _level(features, name) + if _hinted(features) and str(features.is_supported("delete.if-match", str)) == "broken": + if hasattr(CalendarObjectResource, "_ignores_delete_if_match"): # caldav 3.8 + table[Feature.WRITE_CONDITIONAL] = { + "level": E, + "note": "the server deletes on a stale If-Match; caldav reads the ETag first, a window before the delete", + } + else: + table[Feature.WRITE_CONDITIONAL] = { + "level": U, + "note": "the server deletes on a stale If-Match: delete() is not conditional before caldav 3.8; save() is", + } if _level(features, "save-load.icalendar.related-to") is U: table[Feature.TIME_LOG] = {"level": U, "note": "the server strips RELATED-TO, which ties an entry to its item"} + elif _level(features, "save-load.icalendar.participant") is U: + table[Feature.TIME_LOG] = { + "level": L, + "lost": ["users"], + "note": "the server refuses an event with a PARTICIPANT, which names who did the work", + } + if Component.TASK in components: + lowered = {**TASK_ONLY_FROM_SERVER, **(TASK_FROM_SERVER if Component.EVENT not in components else {})} + for feature, name in lowered.items(): + if _level(features, name) is U: + table[feature] = U + for feature in VALUES_FROM_SERVER: # Ox keeps priorities 1, 5 and 9; Zimbra refuses CANCELLED + if (declared := _values(features, feature)) is not None: + table[feature] = declared + stable_url = features.is_supported("save-load.stable-url", dict) if _hinted(features) else {} + if stable_url.get("forces_uid_ics"): + table[Feature.IDENTITY_CLIENT_NATIVE_ID] = {"level": U, "note": "the server renames every object .ics"} + elif renamed := sorted(c.value for c in components if c.value in stable_url.get("uid_ics_components", ())): + table[Feature.IDENTITY_CLIENT_NATIVE_ID] = { + "level": U, + "note": f"the server renames a {', '.join(renamed)} .ics", + } + x_property = features.is_supported("save-load.icalendar.x-property", dict) if _hinted(features) else {} + if _level(features, "save-load.icalendar.x-property") is U and not (x_property.get("behaviour") or "").endswith( + "in a task" + ): # dropped from events too + table[Feature.PROPERTIES_PASSTHROUGH] = U + if _level(features, "save-load.icalendar.x-parameter") is U: + table[Feature.WRITE_WIRE] = {"level": U, "note": "the server drops X- parameters: bytes are not stored as sent"} held_only(table, components) return Capabilities({f: Capability.coerce(v) for f, v in table.items()}) diff --git a/src/calendaring/_sync/caldav.py b/src/calendaring/_sync/caldav.py index ad65a45..19e609e 100644 --- a/src/calendaring/_sync/caldav.py +++ b/src/calendaring/_sync/caldav.py @@ -21,7 +21,12 @@ import icalendar from icalendar_searcher import Searcher -from .._caldav_features import backend_capabilities, collection_capabilities, stored_components +from .._caldav_features import ( + backend_capabilities, + collection_capabilities, + multiget_serves_deleted, + stored_components, +) from .._caldav_io import Calendar from ..capabilities import Feature from ..errors import ( @@ -115,6 +120,8 @@ def _kinds(self, searcher: Searcher | None) -> frozenset[Component]: return self.components & wanted if wanted else self.components def _fetch(self, native_id: str) -> StoredObject: + if multiget_serves_deleted(self._backend._dav.client.features): # Zimbra + return self._stored(self._backend._dav.get(self.calendar, self._href(native_id))) for obj in self._backend._dav.multiget(self.calendar, [self._href(native_id)]): if obj.data is not None: # caldav returns a missing href as an object without data return self._stored(obj) diff --git a/src/calendaring/_sync/collection.py b/src/calendaring/_sync/collection.py index fad3d33..4392044 100644 --- a/src/calendaring/_sync/collection.py +++ b/src/calendaring/_sync/collection.py @@ -283,7 +283,13 @@ def _time_entries( def _time_log(self, item: Item) -> list[TimeEntry]: return [entry for _, entry in self._log_target()._time_entries(None, None, parent=item.uid)] + def _storable_entry(self, entry: TimeEntry) -> TimeEntry: + """``entry`` less the users the log collection declares lost: a server refusing them would refuse it all.""" + lost = self._log_target().capabilities.details(Feature.TIME_LOG).get("lost", ()) + return replace(entry, users=()) if "users" in lost else entry + def _add_time(self, item: Item, entry: TimeEntry) -> TimeEntry: + entry = self._storable_entry(entry) stored = self._log_target().add(Event(_wrap_component(entry_to_component(entry, parent=item)))) return entry_from_component(stored.component)[1] @@ -291,7 +297,7 @@ def _amend_time(self, item: Item, old: TimeEntry, new: TimeEntry) -> TimeEntry: assert old.id is not None stored = self._log_target().get(old.id) component = stored.component # edited in place: what other clients added stays - apply_entry(component, new) + apply_entry(component, self._storable_entry(new)) sequence = int(component.pop("SEQUENCE", 0)) for name, value in (("SEQUENCE", sequence + 1), ("LAST-MODIFIED", datetime.now(UTC))): component.pop(name, None) diff --git a/src/calendaring/_sync/conformance.py b/src/calendaring/_sync/conformance.py index 32d9b1e..021c24c 100644 --- a/src/calendaring/_sync/conformance.py +++ b/src/calendaring/_sync/conformance.py @@ -825,7 +825,9 @@ def test_values_map_as_declared(self, collection: Collection, feature: Feature) def policy(losses: Sequence[Loss], item: Item) -> None: seen.append(losses) - collection.add(item, loss=policy) + again = Task.new(summary=f"{name} outside, again") # the first is stored now, under its UID + again.component[name] = outside[0] + collection.add(again, loss=policy) assert [[loss.property for loss in losses] for losses in seen] == [[name]] assert seen[0][0].sent == outside[0] diff --git a/src/calendaring/_sync/dav.py b/src/calendaring/_sync/dav.py index a83d29e..e01f137 100644 --- a/src/calendaring/_sync/dav.py +++ b/src/calendaring/_sync/dav.py @@ -82,6 +82,14 @@ def multiget(self, calendar: Calendar, urls: list[str]) -> list[CalendarObjectRe found: Any = calendar.multiget([URL.objectify(u) for u in urls]) return list(_returned(cast(Any, found))) + def get(self, calendar: Calendar, url: str) -> CalendarObjectResource: + """The object at ``url``, read with a GET: ``NotFoundError`` if it is gone.""" + obj = _resource(self.client, calendar, url) + with translated(): + # No multiget fallback: the reason to GET is a multiget serving deleted objects. + _returned(obj.load(multiget_fallback=False)) + return obj + def etags(self, url: str) -> dict[str, str]: with translated(): # Positional: the sync client's second parameter is props=, which also takes an XML body. diff --git a/tests/_async/test_caldav_backend.py b/tests/_async/test_caldav_backend.py index 94943e4..7edf749 100644 --- a/tests/_async/test_caldav_backend.py +++ b/tests/_async/test_caldav_backend.py @@ -27,6 +27,7 @@ pytest.importorskip("caldav") from caldav.collection import Calendar # noqa: E402 +from caldav.compatibility_hints import FeatureSet # noqa: E402 from caldav.lib.url import URL # noqa: E402 from calendaring._async.caldav import _name_of # noqa: E402 @@ -346,3 +347,39 @@ async def test_get_of_an_object_that_does_not_parse_says_so( serve_broken(collection, monkeypatch) with pytest.raises(CalendaringError): await collection.get("from-phone") + + +@pytest.mark.skipif("multiget.deleted" not in FeatureSet.FEATURES, reason="caldav has no multiget.deleted hint yet") +async def test_a_deleted_object_is_not_read_back_where_multiget_still_serves_it( + xandikos_server: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + """Zimbra answers a GET of a deleted object with 404, and a multiget of it with the old data.""" + features = {**xandikos_server.config["features"], "multiget.deleted": {"support": "broken"}} + backend = await AsyncBackend.open( + xandikos_server.url, + kind="caldav", + username=xandikos_server.username, + password=xandikos_server.password, + features=features, + ) + async with backend: + collection = await backend.create_collection("multiget-ghosts", components=(Component.TASK,)) + try: + native: Any = collection # the CalDAV collection, for its calendar and its DAV client + live = await collection.add_task(summary="kept") + assert live.native_id is not None + read = await collection.get_by_native_id(live.native_id) + assert (read.uid, read.etag) == (live.uid, live.etag), "read with a GET, ETag and all" + stored = await collection.add_task(summary="deleted") + assert stored.native_id is not None + ghost = await native._backend._dav.multiget(native.calendar, [native._href(stored.native_id)]) + await stored.delete() + + async def haunted(calendar: Any, urls: list[str]) -> list[Any]: + return list(ghost) + + monkeypatch.setattr(native._backend._dav, "multiget", haunted) + with pytest.raises(NotFoundError): + await collection.get_by_native_id(stored.native_id) + finally: + await collection.delete_collection() diff --git a/tests/_async/test_caldav_conformance.py b/tests/_async/test_caldav_conformance.py index 2a57eaf..6a9fc6b 100644 --- a/tests/_async/test_caldav_conformance.py +++ b/tests/_async/test_caldav_conformance.py @@ -6,6 +6,7 @@ from typing import Any import pytest +from conftest import wait_out_search_cache from calendaring import AsyncBackend from calendaring.testing import AsyncConformance @@ -15,12 +16,16 @@ class CalDAVConformance(AsyncConformance): # A CalDAV server is probed, not known in advance (§5.1). allow_unknown = True server: Any + monkeypatch: pytest.MonkeyPatch async def open_backend(self) -> AsyncBackend: - return await AsyncBackend.open(self.server.url, kind="caldav", **self.server.options) + backend = await AsyncBackend.open(self.server.url, kind="caldav", **self.server.options) + wait_out_search_cache(backend, self.monkeypatch) + return backend class TestCalDAVServers(CalDAVConformance): @pytest.fixture(autouse=True) - def _server(self, caldav_server: Any) -> None: + def _server(self, caldav_server: Any, monkeypatch: pytest.MonkeyPatch) -> None: self.server = caldav_server + self.monkeypatch = monkeypatch diff --git a/tests/_async/test_generic.py b/tests/_async/test_generic.py index ab7a8ff..94948c7 100644 --- a/tests/_async/test_generic.py +++ b/tests/_async/test_generic.py @@ -611,6 +611,22 @@ async def test_a_day_lines_up_per_person_and_may_run_past_midnight() -> None: assert late.end.date() > day, "a long day keeps its sum, past midnight" +async def test_users_declared_lost_are_not_sent() -> None: + """Where the server refuses an event with a PARTICIPANT (CCS, GMX), an entry is stored without it.""" + lossy = {"level": Support.LOSSY, "lost": ["users"]} + profile = Profile("no-participants", {**FULL.capabilities, Feature.TIME_LOG: lossy}, FULL.components) + collection = await AsyncMemoryBackend(profile).collection("default") + task = await collection.add_task(summary="x") + alice = ("mailto:alice@example.com",) + entry = await task.log_time(start=at(3, 9), end=at(3, 10), users=alice) + assert entry.users == () + assert entry.id is not None + assert "PARTICIPANT" not in (await collection.get(entry.id)).to_ical().decode() + amended = await task.amend_time(entry, users=alice, comment="later") + assert amended.users == () + assert "PARTICIPANT" not in (await collection.get(entry.id)).to_ical().decode() + + async def test_time_entries_are_not_subtasks() -> None: collection = await AsyncMemoryBackend().collection("default") task = await collection.add_task(summary="parent") diff --git a/tests/_async/test_memory_conformance.py b/tests/_async/test_memory_conformance.py index 69bf720..04458af 100644 --- a/tests/_async/test_memory_conformance.py +++ b/tests/_async/test_memory_conformance.py @@ -121,6 +121,19 @@ async def unseed(self, collection: AsyncCollection, item: AsyncItem) -> None: del collection.store.objects[item.native_id] +# Values mapped as the tracker maps them, but with the UID the client gave (Ox's three priorities). +MAPPED_KEEPING_UIDS = Profile( + "mapped-keeping-uids", + {**FULL.capabilities, Feature.TASK_PRIORITY: TRACKER.capabilities[Feature.TASK_PRIORITY]}, + FULL.components, +) + + +class TestMappedKeepingUidsProfile(AsyncConformance): + async def open_backend(self) -> AsyncBackend: + return AsyncMemoryBackend(profile=MAPPED_KEEPING_UIDS) + + # Three more profiles, for the branches the four above leave out. EVENTS_ONLY = Profile( "events-only", diff --git a/tests/_sync/test_caldav_backend.py b/tests/_sync/test_caldav_backend.py index 76553bb..eefa38f 100644 --- a/tests/_sync/test_caldav_backend.py +++ b/tests/_sync/test_caldav_backend.py @@ -27,6 +27,7 @@ pytest.importorskip("caldav") from caldav.collection import Calendar # noqa: E402 +from caldav.compatibility_hints import FeatureSet # noqa: E402 from caldav.lib.url import URL # noqa: E402 from calendaring._sync.caldav import _name_of # noqa: E402 @@ -346,3 +347,39 @@ def test_get_of_an_object_that_does_not_parse_says_so( serve_broken(collection, monkeypatch) with pytest.raises(CalendaringError): collection.get("from-phone") + + +@pytest.mark.skipif("multiget.deleted" not in FeatureSet.FEATURES, reason="caldav has no multiget.deleted hint yet") +def test_a_deleted_object_is_not_read_back_where_multiget_still_serves_it( + xandikos_server: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + """Zimbra answers a GET of a deleted object with 404, and a multiget of it with the old data.""" + features = {**xandikos_server.config["features"], "multiget.deleted": {"support": "broken"}} + backend = Backend.open( + xandikos_server.url, + kind="caldav", + username=xandikos_server.username, + password=xandikos_server.password, + features=features, + ) + with backend: + collection = backend.create_collection("multiget-ghosts", components=(Component.TASK,)) + try: + native: Any = collection # the CalDAV collection, for its calendar and its DAV client + live = collection.add_task(summary="kept") + assert live.native_id is not None + read = collection.get_by_native_id(live.native_id) + assert (read.uid, read.etag) == (live.uid, live.etag), "read with a GET, ETag and all" + stored = collection.add_task(summary="deleted") + assert stored.native_id is not None + ghost = native._backend._dav.multiget(native.calendar, [native._href(stored.native_id)]) + stored.delete() + + def haunted(calendar: Any, urls: list[str]) -> list[Any]: + return list(ghost) + + monkeypatch.setattr(native._backend._dav, "multiget", haunted) + with pytest.raises(NotFoundError): + collection.get_by_native_id(stored.native_id) + finally: + collection.delete_collection() diff --git a/tests/_sync/test_caldav_conformance.py b/tests/_sync/test_caldav_conformance.py index 40d8b52..0964345 100644 --- a/tests/_sync/test_caldav_conformance.py +++ b/tests/_sync/test_caldav_conformance.py @@ -6,6 +6,7 @@ from typing import Any import pytest +from conftest import wait_out_search_cache from calendaring import Backend from calendaring.testing import SyncConformance @@ -15,12 +16,16 @@ class CalDAVConformance(SyncConformance): # A CalDAV server is probed, not known in advance (§5.1). allow_unknown = True server: Any + monkeypatch: pytest.MonkeyPatch def open_backend(self) -> Backend: - return Backend.open(self.server.url, kind="caldav", **self.server.options) + backend = Backend.open(self.server.url, kind="caldav", **self.server.options) + wait_out_search_cache(backend, self.monkeypatch) + return backend class TestCalDAVServers(CalDAVConformance): @pytest.fixture(autouse=True) - def _server(self, caldav_server: Any) -> None: + def _server(self, caldav_server: Any, monkeypatch: pytest.MonkeyPatch) -> None: self.server = caldav_server + self.monkeypatch = monkeypatch diff --git a/tests/_sync/test_generic.py b/tests/_sync/test_generic.py index 714dc72..0db6252 100644 --- a/tests/_sync/test_generic.py +++ b/tests/_sync/test_generic.py @@ -611,6 +611,22 @@ def test_a_day_lines_up_per_person_and_may_run_past_midnight() -> None: assert late.end.date() > day, "a long day keeps its sum, past midnight" +def test_users_declared_lost_are_not_sent() -> None: + """Where the server refuses an event with a PARTICIPANT (CCS, GMX), an entry is stored without it.""" + lossy = {"level": Support.LOSSY, "lost": ["users"]} + profile = Profile("no-participants", {**FULL.capabilities, Feature.TIME_LOG: lossy}, FULL.components) + collection = MemoryBackend(profile).collection("default") + task = collection.add_task(summary="x") + alice = ("mailto:alice@example.com",) + entry = task.log_time(start=at(3, 9), end=at(3, 10), users=alice) + assert entry.users == () + assert entry.id is not None + assert "PARTICIPANT" not in (collection.get(entry.id)).to_ical().decode() + amended = task.amend_time(entry, users=alice, comment="later") + assert amended.users == () + assert "PARTICIPANT" not in (collection.get(entry.id)).to_ical().decode() + + def test_time_entries_are_not_subtasks() -> None: collection = MemoryBackend().collection("default") task = collection.add_task(summary="parent") diff --git a/tests/_sync/test_memory_conformance.py b/tests/_sync/test_memory_conformance.py index 06d259e..e47c54a 100644 --- a/tests/_sync/test_memory_conformance.py +++ b/tests/_sync/test_memory_conformance.py @@ -121,6 +121,19 @@ def unseed(self, collection: Collection, item: SyncItem) -> None: del collection.store.objects[item.native_id] +# Values mapped as the tracker maps them, but with the UID the client gave (Ox's three priorities). +MAPPED_KEEPING_UIDS = Profile( + "mapped-keeping-uids", + {**FULL.capabilities, Feature.TASK_PRIORITY: TRACKER.capabilities[Feature.TASK_PRIORITY]}, + FULL.components, +) + + +class TestMappedKeepingUidsProfile(SyncConformance): + def open_backend(self) -> Backend: + return MemoryBackend(profile=MAPPED_KEEPING_UIDS) + + # Three more profiles, for the branches the four above leave out. EVENTS_ONLY = Profile( "events-only", diff --git a/tests/test_caldav_features.py b/tests/test_caldav_features.py index 9b71042..3e33a2b 100644 --- a/tests/test_caldav_features.py +++ b/tests/test_caldav_features.py @@ -25,6 +25,11 @@ def features(table: dict[str, Any] | None = None) -> Any: return FeatureSet(table) # type: ignore[no-untyped-call] # caldav leaves it unannotated +def needs_hint(name: str) -> pytest.MarkDecorator: + """For a hint newer than the caldav installed (due in caldav 3.8).""" + return pytest.mark.skipif(name not in FeatureSet.FEATURES, reason=f"caldav has no {name} hint yet") + + def hinted(**levels: str) -> Any: """A server caldav has hints for: ``sync_token="unsupported"`` sets ``sync-token``.""" return features({name.replace("_", "-"): {"support": level} for name, level in levels.items()}) @@ -111,6 +116,181 @@ def test_no_uncomplete_where_the_server_keeps_a_task_completed() -> None: assert collection_capabilities(server, ALL).level(Feature.TASK_UNCOMPLETE) is Support.UNSUPPORTED +def test_a_conditional_delete_is_emulated_where_the_server_ignores_if_match(monkeypatch: pytest.MonkeyPatch) -> None: + """SOGo, Zimbra, Bedework and Posteo delete on a stale If-Match; a caldav that reads the ETag first emulates it.""" + from caldav.calendarobjectresource import CalendarObjectResource + + server = features({"save-load": {"support": "full"}, "delete.if-match": {"support": "broken"}}) + monkeypatch.setattr(CalendarObjectResource, "_ignores_delete_if_match", lambda *a: True, raising=False) + assert collection_capabilities(server, ALL).level(Feature.WRITE_CONDITIONAL) is Support.EMULATED + monkeypatch.delattr(CalendarObjectResource, "_ignores_delete_if_match") + older = collection_capabilities(server, ALL) # a caldav that deletes whatever the ETag + assert older.level(Feature.WRITE_CONDITIONAL) is Support.UNSUPPORTED + + +def test_time_entries_lose_their_users_where_the_server_refuses_participants() -> None: + """CCS, Bedework 3, robur, GMX, Ox and Zimbra refuse an event holding a PARTICIPANT.""" + server = hinted(save_load="full", **{"save-load.icalendar.participant": "ungraceful"}) + table = collection_capabilities(server, ALL) + assert table.level(Feature.TIME_LOG) is Support.LOSSY + assert table.details(Feature.TIME_LOG)["lost"] == ["users"] + tasks_only = collection_capabilities(server, frozenset({Component.TASK})) + assert tasks_only.level(Feature.TIME_LOG) is Support.UNSUPPORTED # nowhere to log time stays the reason + + +@pytest.mark.parametrize( + ("hint", "feature"), + [ + ("save-load.todo.attendee", Feature.ATTENDEES), # Ox: 500 + ("save-load.todo.recurrences", Feature.RECURRENCE), # Ox: 400 + ("save-load.icalendar.x-property", Feature.PROPERTIES_PASSTHROUGH), # Ox drops it from a task + ("save-load.icalendar.x-property", Feature.TASK_PLANNED), # X-PYCAL-PLANNED-* + ("save-load.icalendar.unknown-property", Feature.TASK_ESTIMATE), # Ox, Zimbra: ESTIMATED-DURATION + ], +) +def test_what_a_task_list_cannot_store(hint: str, feature: Feature) -> None: + """These hints are probed on a task: a calendar of events keeps the feature, and so does one holding + events too, whose events would otherwise lose it (levels are per collection, not per component).""" + server = features( + {"save-load": {"support": "full"}, hint: {"support": "ungraceful", "behaviour": "dropped, in a task"}} + ) + assert collection_capabilities(server, frozenset({Component.TASK})).level(feature) is Support.UNSUPPORTED + if feature in (Feature.TASK_PLANNED, Feature.TASK_ESTIMATE): # nothing of an event's to lose + assert collection_capabilities(server, ALL).level(feature) is Support.UNSUPPORTED + else: + assert collection_capabilities(server, ALL).level(feature) is Support.FULL + assert collection_capabilities(server, frozenset({Component.EVENT})).level(feature) is Support.FULL + + +def test_x_properties_dropped_from_events_too_lower_every_collection() -> None: + """The x-property hint names where it was dropped; not only from a task means from events too.""" + server = features( + { + "save-load": {"support": "full"}, + "save-load.icalendar.x-property": { + "support": "unsupported", + "behaviour": "X-CALDAV-SERVER-TESTER is dropped, in an event", + }, + } + ) + events = collection_capabilities(server, frozenset({Component.EVENT})) + assert events.level(Feature.PROPERTIES_PASSTHROUGH) is Support.UNSUPPORTED + + +def test_an_x_property_hint_with_an_empty_behaviour_is_read() -> None: + """A config file may write ``behaviour:`` with no value.""" + server = features( + { + "save-load": {"support": "full"}, + "save-load.icalendar.x-property": {"support": "unsupported", "behaviour": None}, + } + ) + events = collection_capabilities(server, frozenset({Component.EVENT})) + assert events.level(Feature.PROPERTIES_PASSTHROUGH) is Support.UNSUPPORTED + + +@needs_hint("save-load.todo.status") +def test_statuses_a_task_list_does_not_keep() -> None: + """Zimbra stores PENDING and FAILED as NEEDS-ACTION and refuses CANCELLED.""" + from caldav import compatibility_hints + + server = features({"save-load": {"support": "full"}, **compatibility_hints.zimbra_8_8}) + table = collection_capabilities(server, frozenset({Component.TASK})) + assert table.level(Feature.TASK_STATUS) is Support.LOSSY + assert table.details(Feature.TASK_STATUS)["values"] == ["NEEDS-ACTION", "IN-PROCESS", "COMPLETED"] + assert table.details(Feature.TASK_STATUS)["map"] == {"PENDING": "NEEDS-ACTION", "FAILED": "NEEDS-ACTION"} + assert collection_capabilities(server, frozenset({Component.EVENT})).level(Feature.TASK_STATUS) is not Support.LOSSY + + +@needs_hint("save-load.todo.status.rfc10051") +def test_rfc_10051_statuses_lost_where_the_rfc_5545_ones_are_kept() -> None: + """Ox keeps RFC 5545's four statuses and drops PENDING and FAILED.""" + from caldav import compatibility_hints + + server = features({"save-load": {"support": "full"}, **compatibility_hints.ox}) + table = collection_capabilities(server, frozenset({Component.TASK})) + details = table.details(Feature.TASK_STATUS) + assert details["values"] == ["NEEDS-ACTION", "IN-PROCESS", "COMPLETED", "CANCELLED"] + assert details["map"] == {} + assert "does not advertise RFC 10051" in (table[Feature.TASK_STATUS].note or "") + + +@needs_hint("save-load.todo.status.rfc10051") +def test_a_parent_status_hint_alone_is_not_counted_twice() -> None: + """caldav answers the child with the parent's node where only the parent is set.""" + hint = {"support": "ungraceful", "values": ["NEEDS-ACTION", "IN-PROCESS", "COMPLETED"], "refused": ["CANCELLED"]} + server = features({"save-load": {"support": "full"}, "save-load.todo.status": hint}) + details = collection_capabilities(server, frozenset({Component.TASK})).details(Feature.TASK_STATUS) + assert details["values"] == ["NEEDS-ACTION", "IN-PROCESS", "COMPLETED"] + + +@needs_hint("save-load.todo.status.rfc10051") +def test_statuses_not_known_to_be_kept_are_not_declared_kept() -> None: + """A hint the tester could not settle says nothing kept: PENDING and FAILED are left out.""" + server = features( + { + "save-load": {"support": "full"}, + "save-load.todo.status": {"support": "unsupported", "values": ["NEEDS-ACTION"]}, + "save-load.todo.status.rfc10051": {"support": "unknown"}, + } + ) + details = collection_capabilities(server, frozenset({Component.TASK})).details(Feature.TASK_STATUS) + assert details["values"] == ["NEEDS-ACTION"] + + +@needs_hint("save-load.todo.priority") +def test_priority_zero_is_left_out_not_sent_as_the_servers_stand_in() -> None: + """Ox keeps 1, 5 and 9 and stores 0 as 5; no PRIORITY means the same as 0.""" + from caldav import compatibility_hints + + server = features({"save-load": {"support": "full"}, **compatibility_hints.ox}) + details = collection_capabilities(server, frozenset({Component.TASK})).details(Feature.TASK_PRIORITY) + assert details["values"] == [1, 5, 9] + assert 0 not in details["map"] + assert details["map"][7] == 9 + + +@needs_hint("save-load.todo.priority") +def test_priorities_from_a_config_file_are_numbers() -> None: + """A hint read from YAML or JSON may carry the priorities as strings.""" + hint = {"support": "unsupported", "values": ["1", "5", "9"], "map": {"0": "5", "2": "1"}} + server = features({"save-load": {"support": "full"}, "save-load.todo.priority": hint}) + details = collection_capabilities(server, frozenset({Component.TASK})).details(Feature.TASK_PRIORITY) + assert details["values"] == [1, 5, 9] + assert details["map"] == {2: 1} + + +@needs_hint("save-load.icalendar.x-parameter") +def test_no_wire_writes_where_the_server_drops_x_parameters() -> None: + """Ox and Zimbra drop SUMMARY;X-...: the bytes are not stored as sent.""" + server = hinted(save_load="full", **{"save-load.icalendar.x-parameter": "unsupported"}) + assert collection_capabilities(server, ALL).level(Feature.WRITE_WIRE) is Support.UNSUPPORTED + + +def test_no_native_id_in_a_task_list_where_the_server_renames_tasks_to_their_uid() -> None: + """Ox keeps an event's name and names a task .ics.""" + server = features( + { + "save-load": {"support": "full"}, + "save-load.stable-url": {"support": "unsupported", "uid_ics_components": ["VTODO"]}, + } + ) + tasks = collection_capabilities(server, frozenset({Component.TASK})) + assert tasks.level(Feature.IDENTITY_CLIENT_NATIVE_ID) is Support.UNSUPPORTED + events = collection_capabilities(server, frozenset({Component.EVENT})) + assert events.level(Feature.IDENTITY_CLIENT_NATIVE_ID) is Support.FULL + + +def test_no_native_id_where_the_server_renames_every_object_to_its_uid() -> None: + """Zimbra redirects a PUT under another name to .ics.""" + renaming = features( + {"save-load": {"support": "full"}, "save-load.stable-url": {"support": "unsupported", "forces_uid_ics": True}} + ) + assert collection_capabilities(renaming, ALL).level(Feature.IDENTITY_CLIENT_NATIVE_ID) is Support.UNSUPPORTED + unstable = hinted(save_load="full", **{"save-load.stable-url": "unsupported"}) # Ox: other reasons + assert collection_capabilities(unstable, ALL).level(Feature.IDENTITY_CLIENT_NATIVE_ID) is Support.FULL + + def test_one_kind_per_calendar_where_the_server_refuses_several() -> None: """CCS refuses a MKCALENDAR naming VEVENT and VTODO; Bedework 3, Zimbra and Ox misreport one.""" multiple = "create-calendar.with-supported-component-types.multiple"