fix(registry): continue the tail from the last stored release

src/Module/Registry/Internal/StoredVersionRegistry.php

@@ -25,7 +25,8 @@
  *    attached after the check are updated too.
  * 2. **Serve.** The stored releases are yielded segment by segment without any request.
  * 3. **Extend.** When the consumer runs past the stored releases and the listing is not known
- *    to be complete, older pages are fetched one by one and appended to the record.
+ *    to be complete, older pages are fetched one by one and appended to the record, from the
+ *    last stored release on.
  *
  * A check that fails while releases are stored falls back to the stored ones: an outage or a rate
  * limit should not break what worked a minute ago. A rate limit is reported once per run in plain
@@ -60,8 +61,10 @@ public function releases(RepositoryId $id, ReleaseSource $source): \Generator
     {
         $record = $this->storage->load($id) ?? RepositoryRecord::empty($id);
 
+        // Only a check can tell that the stored releases are still where they were listed
+        $seen = false;
         if ($this->refresh || $record->isStale(($this->clock)(), $this->ttl)) {
-            $record = $this->check($record, $source);
+            [$record, $seen] = $this->check($record, $source);
         } else {
             $this->logger->debug('Releases of `%s` are served from the version registry.', (string) $id);
         }
@@ -75,7 +78,7 @@ public function releases(RepositoryId $id, ReleaseSource $source): \Generator
         }
 
         // Older releases are loaded only when the consumer actually needs them
-        yield from $this->extend($record, $source);
+        yield from $this->extend($record, $source, $seen);
     }
 
     public function attach(RepositoryId $id, string $software): void
@@ -99,6 +102,21 @@ public function forget(RepositoryId $id, string $tag): void
         $this->persist($record->withoutRelease($tag)->withoutCheck());
     }
 
+    /**
+     * @param list<ReleaseRecord> $page
+     * @return int<0, max>|null Index of the last stored release of the page, if any.
+     */
+    private static function lastKnown(RepositoryRecord $record, array $page): ?int
+    {
+        for ($index = \count($page) - 1; $index >= 0; --$index) {
+            if ($record->has($page[$index]->tag)) {
+                return $index;
+            }
+        }
+
+        return null;
+    }
+
     /**
      * @param list<ReleaseRecord> $page
      */
@@ -116,9 +134,12 @@ private static function hasKnown(RepositoryRecord $record, array $page): bool
     /**
      * Fetches the releases published since the last check and stores the result.
      *
+     * Also tells whether the check has seen every stored release where it is listed now.
+     *
+     * @return array{RepositoryRecord, bool}
      * @throws RepositoryException When the source fails and nothing is stored to fall back on.
      */
-    private function check(RepositoryRecord $record, ReleaseSource $source): RepositoryRecord
+    private function check(RepositoryRecord $record, ReleaseSource $source): array
     {
         try {
             $fetched = [];
@@ -136,7 +157,7 @@ private function check(RepositoryRecord $record, ReleaseSource $source): Reposit
                         $record->count(),
                     );
 
-                    return $record;
+                    return [$record, false];
                 }
 
                 $fetched = [...$fetched, ...$page->releases];
@@ -151,42 +172,85 @@ private function check(RepositoryRecord $record, ReleaseSource $source): Reposit
                 }
             }
 
-            return $this->persist(
+            // A listing without a single page has no releases at all
+            $fetched === [] and $complete = true;
+
+            $record = $this->persist(
                 $record
                     ->withHead($fetched)
                     ->withComplete($complete)
                     ->withCheckedAt(($this->clock)()),
             );
+
+            $tags = \array_map(static fn(ReleaseRecord $release): string => $release->tag, $fetched);
+
+            return [$record, $record->count() <= \count(\array_unique($tags))];
         } catch (RepositoryException $e) {
             $record->count() > 0 or throw $e;
 
             $this->report($e, $record);
 
-            return $record;
+            return [$record, false];
         }
     }
 
     /**
      * Loads the releases older than the stored ones page by page, persisting every page.
      *
+     * The stored count only hints at the listing position of the last stored release: a stored
+     * release deleted upstream moves it up, a release the record misses above it moves it down.
+     * So the loading starts with the page holding that position and goes on after the last
+     * stored release found there. A page holding none sends the search up the listing; when it
+     * reaches the top without one, the search goes down from the first page instead. The page
+     * fetched first is the one needed anyway when the stored list ends inside a page; a list
+     * ending on a page boundary costs one request more. A list the check has just seen whole is
+     * where it was fetched, so its tail goes on right after it.
+     *
+     * A stored release deleted upstream stays stored until its download fails and it is
+     * forgotten. A release the listing gained above the last stored one, such as an unreadable
+     * one turned readable, is not loaded.
+     *
+     * @param bool $seen Whether the check has just seen every stored release where it is.
      * @return \Generator<int, list<ReleaseRecord>, mixed, void>
      * @throws RepositoryException
      */
-    private function extend(RepositoryRecord $record, ReleaseSource $source): \Generator
+    private function extend(RepositoryRecord $record, ReleaseSource $source, bool $seen): \Generator
     {
-        foreach ($source->pages($record->count()) as $page) {
-            $new = \array_values(\array_filter(
-                $page->releases,
-                static fn(ReleaseRecord $release): bool => !$record->has($release->tag),
-            ));
+        $found = $seen || $record->count() === 0;
+        $first = $pages = $source->pages($found ? $record->count() : \max(0, $record->count() - 1));
+        // The search goes up while the pages it meets start ever higher, then down once
+        $up = true;
+        $above = \PHP_INT_MAX;
+
+        while (($page = $pages->current()) !== null) {
+            $at = self::lastKnown($record, $page->releases);
+            if (!$found && $at === null) {
+                if ($up && $page->offset > 0 && $page->offset < $above) {
+                    $above = $page->offset;
+                    $pages = $source->pages($page->offset - 1);
+                    continue;
+                }
+
+                // Nothing stored up to the top: releases the record misses pushed the tail down
+                $up and $pages = $first;
+                $up = false;
+                $pages->next();
+                continue;
+            }
+
+            $found = true;
+            $new = $at === null ? $page->releases : \array_slice($page->releases, $at + 1);
 
             $record = $this->persist($record->withTail($new)->withComplete($page->last));
 
             $visible = ReleaseRecord::visible($new);
             $visible === [] or yield $visible;
+
+            $pages->next();
         }
 
-        $record->complete or $this->persist($record->withComplete(true));
+        // A listing without any stored release is left for the next check to replace
+        $found and !$record->complete and $this->persist($record->withComplete(true));
     }
 
     /**

src/Module/Registry/Record/ReleasePage.php

@@ -19,10 +19,12 @@ final class ReleasePage
      * @param bool $last Whether the listing has no page after this one.
      * @param int<0, max> $skipped Number of releases of the page the source could not read. They
      *        are absent from `$releases` although they exist upstream.
+     * @param int<0, max> $offset Listing position the page starts at, unreadable releases included.
      */
     public function __construct(
         public readonly array $releases,
         public readonly bool $last,
         public readonly int $skipped = 0,
+        public readonly int $offset = 0,
     ) {}
 }

src/Module/Registry/Record/RepositoryRecord.php

@@ -11,8 +11,9 @@
  *
  * Releases are kept newest first, exactly in the order the source lists them, and they form
  * a contiguous prefix of that listing: the registry only ever extends the list at the head (new
- * releases found on a check) or at the tail (older releases loaded on demand). `complete` tells
- * whether the tail has reached the end of the listing.
+ * releases found on a check) or at the tail (older releases loaded on demand). A release deleted
+ * upstream below the span of the last check stays until its download fails and it is forgotten.
+ * `complete` tells whether the tail has reached the end of the listing.
  *
  * The list is split into segments of at most `SEGMENT_SIZE` releases, so a repository with
  * thousands of releases costs one small index read plus the segments the run actually iterates.
@@ -204,9 +205,8 @@ public function isStale(int $now, int $ttl): bool
      * The fetched releases are the newest ones, and sources only ever add releases at the top,
      * so from the first stored release they contain onwards both lists walk the same listing
      * positions. A stored release missing from the fetched span at its position was deleted
-     * upstream and is dropped: keeping it would inflate `count()`, which the registry uses as
-     * the offset for loading the tail. Stored releases beyond the span are kept, and when the
-     * fetched releases contain no stored one they are the whole listing.
+     * upstream and is dropped. Stored releases beyond the span are kept, and when the fetched
+     * releases contain no stored one they are the whole listing.
      *
      * @param list<ReleaseRecord> $fetched Newest first.
      */

src/Module/Registry/ReleaseSource.php

@@ -18,12 +18,14 @@
 interface ReleaseSource
 {
     /**
-     * Lists releases newest first, skipping the given number of them.
+     * Lists releases newest first in whole pages, from the page holding the given position on.
      *
+     * Positions count every release of the listing, unreadable ones included. The first page may
+     * start before the position; when the listing ends before it, its last page comes first.
      * A page is requested only when the generator advances to it. The generator ends when the
      * listing ends.
      *
-     * @param int<0, max> $offset Number of newest releases to skip.
+     * @param int<0, max> $offset Listing position of the first release wanted.
      * @return \Generator<int, ReleasePage, mixed, void> Pages of releases.
      * @throws RepositoryException When a page cannot be loaded.
      */

src/Module/Repository/Internal/GitHub/GitHubRepository.php

@@ -10,6 +10,7 @@
 use Internal\DLoad\Module\Repository\Collection\ReleasesCollection;
 use Internal\DLoad\Module\Repository\Exception\RateLimitException;
 use Internal\DLoad\Module\Repository\Internal\GitHub\Api\RepositoryApi;
+use Internal\DLoad\Module\Repository\Internal\PagedReleaseSource;
 use Internal\DLoad\Module\Repository\Internal\Paginator;
 use Internal\DLoad\Module\Repository\Repository;
 use Internal\DLoad\Service\Logger;
@@ -71,7 +72,7 @@ public function getReleases(): ReleasesCollection
 
             $pages = $this->registry->releases(
                 $this->getId(),
-                new GitHubReleaseSource($this->api),
+                new PagedReleaseSource($this->api->releasePages(...), RepositoryApi::RELEASES_PER_PAGE),
             );
             $anyPageLoaded = false;
 

src/Module/Repository/Internal/GitLab/GitLabRepository.php

@@ -10,6 +10,7 @@
 use Internal\DLoad\Module\Repository\Collection\ReleasesCollection;
 use Internal\DLoad\Module\Repository\Exception\RateLimitException;
 use Internal\DLoad\Module\Repository\Internal\GitLab\Api\RepositoryApi;
+use Internal\DLoad\Module\Repository\Internal\PagedReleaseSource;
 use Internal\DLoad\Module\Repository\Internal\Paginator;
 use Internal\DLoad\Module\Repository\Repository;
 use Internal\DLoad\Service\Logger;
@@ -69,7 +70,7 @@ public function getReleases(): ReleasesCollection
 
             $pages = $this->registry->releases(
                 $this->getId(),
-                new GitLabReleaseSource($this->api),
+                new PagedReleaseSource($this->api->releasePages(...), RepositoryApi::RELEASES_PER_PAGE),
             );
             $anyPageLoaded = false;