diff --git a/src/Module/Registry/Internal/StoredVersionRegistry.php b/src/Module/Registry/Internal/StoredVersionRegistry.php index 578b75b9..a0da4b89 100644 --- a/src/Module/Registry/Internal/StoredVersionRegistry.php +++ b/src/Module/Registry/Internal/StoredVersionRegistry.php @@ -25,7 +25,8 @@ * attached after the check are updated too. * 2. **Serve.** The stored releases are yielded segment by segment without any request. * 3. **Extend.** When the consumer runs past the stored releases and the listing is not known - * to be complete, older pages are fetched one by one and appended to the record. + * to be complete, older pages are fetched one by one and appended to the record, from the + * last stored release on. * * A check that fails while releases are stored falls back to the stored ones: an outage or a rate * limit should not break what worked a minute ago. A rate limit is reported once per run in plain @@ -60,8 +61,10 @@ public function releases(RepositoryId $id, ReleaseSource $source): \Generator { $record = $this->storage->load($id) ?? RepositoryRecord::empty($id); + // Only a check can tell that the stored releases are still where they were listed + $seen = false; if ($this->refresh || $record->isStale(($this->clock)(), $this->ttl)) { - $record = $this->check($record, $source); + [$record, $seen] = $this->check($record, $source); } else { $this->logger->debug('Releases of `%s` are served from the version registry.', (string) $id); } @@ -75,7 +78,7 @@ public function releases(RepositoryId $id, ReleaseSource $source): \Generator } // Older releases are loaded only when the consumer actually needs them - yield from $this->extend($record, $source); + yield from $this->extend($record, $source, $seen); } public function attach(RepositoryId $id, string $software): void @@ -99,6 +102,21 @@ public function forget(RepositoryId $id, string $tag): void $this->persist($record->withoutRelease($tag)->withoutCheck()); } + /** + * @param list $page + * @return int<0, max>|null Index of the last stored release of the page, if any. + */ + private static function lastKnown(RepositoryRecord $record, array $page): ?int + { + for ($index = \count($page) - 1; $index >= 0; --$index) { + if ($record->has($page[$index]->tag)) { + return $index; + } + } + + return null; + } + /** * @param list $page */ @@ -116,9 +134,12 @@ private static function hasKnown(RepositoryRecord $record, array $page): bool /** * Fetches the releases published since the last check and stores the result. * + * Also tells whether the check has seen every stored release where it is listed now. + * + * @return array{RepositoryRecord, bool} * @throws RepositoryException When the source fails and nothing is stored to fall back on. */ - private function check(RepositoryRecord $record, ReleaseSource $source): RepositoryRecord + private function check(RepositoryRecord $record, ReleaseSource $source): array { try { $fetched = []; @@ -136,7 +157,7 @@ private function check(RepositoryRecord $record, ReleaseSource $source): Reposit $record->count(), ); - return $record; + return [$record, false]; } $fetched = [...$fetched, ...$page->releases]; @@ -151,42 +172,85 @@ private function check(RepositoryRecord $record, ReleaseSource $source): Reposit } } - return $this->persist( + // A listing without a single page has no releases at all + $fetched === [] and $complete = true; + + $record = $this->persist( $record ->withHead($fetched) ->withComplete($complete) ->withCheckedAt(($this->clock)()), ); + + $tags = \array_map(static fn(ReleaseRecord $release): string => $release->tag, $fetched); + + return [$record, $record->count() <= \count(\array_unique($tags))]; } catch (RepositoryException $e) { $record->count() > 0 or throw $e; $this->report($e, $record); - return $record; + return [$record, false]; } } /** * Loads the releases older than the stored ones page by page, persisting every page. * + * The stored count only hints at the listing position of the last stored release: a stored + * release deleted upstream moves it up, a release the record misses above it moves it down. + * So the loading starts with the page holding that position and goes on after the last + * stored release found there. A page holding none sends the search up the listing; when it + * reaches the top without one, the search goes down from the first page instead. The page + * fetched first is the one needed anyway when the stored list ends inside a page; a list + * ending on a page boundary costs one request more. A list the check has just seen whole is + * where it was fetched, so its tail goes on right after it. + * + * A stored release deleted upstream stays stored until its download fails and it is + * forgotten. A release the listing gained above the last stored one, such as an unreadable + * one turned readable, is not loaded. + * + * @param bool $seen Whether the check has just seen every stored release where it is. * @return \Generator, mixed, void> * @throws RepositoryException */ - private function extend(RepositoryRecord $record, ReleaseSource $source): \Generator + private function extend(RepositoryRecord $record, ReleaseSource $source, bool $seen): \Generator { - foreach ($source->pages($record->count()) as $page) { - $new = \array_values(\array_filter( - $page->releases, - static fn(ReleaseRecord $release): bool => !$record->has($release->tag), - )); + $found = $seen || $record->count() === 0; + $first = $pages = $source->pages($found ? $record->count() : \max(0, $record->count() - 1)); + // The search goes up while the pages it meets start ever higher, then down once + $up = true; + $above = \PHP_INT_MAX; + + while (($page = $pages->current()) !== null) { + $at = self::lastKnown($record, $page->releases); + if (!$found && $at === null) { + if ($up && $page->offset > 0 && $page->offset < $above) { + $above = $page->offset; + $pages = $source->pages($page->offset - 1); + continue; + } + + // Nothing stored up to the top: releases the record misses pushed the tail down + $up and $pages = $first; + $up = false; + $pages->next(); + continue; + } + + $found = true; + $new = $at === null ? $page->releases : \array_slice($page->releases, $at + 1); $record = $this->persist($record->withTail($new)->withComplete($page->last)); $visible = ReleaseRecord::visible($new); $visible === [] or yield $visible; + + $pages->next(); } - $record->complete or $this->persist($record->withComplete(true)); + // A listing without any stored release is left for the next check to replace + $found and !$record->complete and $this->persist($record->withComplete(true)); } /** diff --git a/src/Module/Registry/Record/ReleasePage.php b/src/Module/Registry/Record/ReleasePage.php index 4a19216a..3e224f30 100644 --- a/src/Module/Registry/Record/ReleasePage.php +++ b/src/Module/Registry/Record/ReleasePage.php @@ -19,10 +19,12 @@ final class ReleasePage * @param bool $last Whether the listing has no page after this one. * @param int<0, max> $skipped Number of releases of the page the source could not read. They * are absent from `$releases` although they exist upstream. + * @param int<0, max> $offset Listing position the page starts at, unreadable releases included. */ public function __construct( public readonly array $releases, public readonly bool $last, public readonly int $skipped = 0, + public readonly int $offset = 0, ) {} } diff --git a/src/Module/Registry/Record/RepositoryRecord.php b/src/Module/Registry/Record/RepositoryRecord.php index 089e91a6..0e709677 100644 --- a/src/Module/Registry/Record/RepositoryRecord.php +++ b/src/Module/Registry/Record/RepositoryRecord.php @@ -11,8 +11,9 @@ * * Releases are kept newest first, exactly in the order the source lists them, and they form * a contiguous prefix of that listing: the registry only ever extends the list at the head (new - * releases found on a check) or at the tail (older releases loaded on demand). `complete` tells - * whether the tail has reached the end of the listing. + * releases found on a check) or at the tail (older releases loaded on demand). A release deleted + * upstream below the span of the last check stays until its download fails and it is forgotten. + * `complete` tells whether the tail has reached the end of the listing. * * The list is split into segments of at most `SEGMENT_SIZE` releases, so a repository with * thousands of releases costs one small index read plus the segments the run actually iterates. @@ -204,9 +205,8 @@ public function isStale(int $now, int $ttl): bool * The fetched releases are the newest ones, and sources only ever add releases at the top, * so from the first stored release they contain onwards both lists walk the same listing * positions. A stored release missing from the fetched span at its position was deleted - * upstream and is dropped: keeping it would inflate `count()`, which the registry uses as - * the offset for loading the tail. Stored releases beyond the span are kept, and when the - * fetched releases contain no stored one they are the whole listing. + * upstream and is dropped. Stored releases beyond the span are kept, and when the fetched + * releases contain no stored one they are the whole listing. * * @param list $fetched Newest first. */ diff --git a/src/Module/Registry/ReleaseSource.php b/src/Module/Registry/ReleaseSource.php index 1ae4e4c8..b28bd057 100644 --- a/src/Module/Registry/ReleaseSource.php +++ b/src/Module/Registry/ReleaseSource.php @@ -18,12 +18,14 @@ interface ReleaseSource { /** - * Lists releases newest first, skipping the given number of them. + * Lists releases newest first in whole pages, from the page holding the given position on. * + * Positions count every release of the listing, unreadable ones included. The first page may + * start before the position; when the listing ends before it, its last page comes first. * A page is requested only when the generator advances to it. The generator ends when the * listing ends. * - * @param int<0, max> $offset Number of newest releases to skip. + * @param int<0, max> $offset Listing position of the first release wanted. * @return \Generator Pages of releases. * @throws RepositoryException When a page cannot be loaded. */ diff --git a/src/Module/Repository/Internal/GitHub/GitHubReleaseSource.php b/src/Module/Repository/Internal/GitHub/GitHubReleaseSource.php deleted file mode 100644 index 6d334c39..00000000 --- a/src/Module/Repository/Internal/GitHub/GitHubReleaseSource.php +++ /dev/null @@ -1,36 +0,0 @@ - $first */ - $first = \intdiv($offset, RepositoryApi::RELEASES_PER_PAGE) + 1; - - foreach ($this->api->releasePages($first) as $page) { - yield $skip === 0 ? $page : new ReleasePage(\array_slice($page->releases, $skip), $page->last, $page->skipped); - $skip = 0; - } - } -} diff --git a/src/Module/Repository/Internal/GitHub/GitHubRepository.php b/src/Module/Repository/Internal/GitHub/GitHubRepository.php index bd78ceb5..0d689b8f 100644 --- a/src/Module/Repository/Internal/GitHub/GitHubRepository.php +++ b/src/Module/Repository/Internal/GitHub/GitHubRepository.php @@ -10,6 +10,7 @@ use Internal\DLoad\Module\Repository\Collection\ReleasesCollection; use Internal\DLoad\Module\Repository\Exception\RateLimitException; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\RepositoryApi; +use Internal\DLoad\Module\Repository\Internal\PagedReleaseSource; use Internal\DLoad\Module\Repository\Internal\Paginator; use Internal\DLoad\Module\Repository\Repository; use Internal\DLoad\Service\Logger; @@ -71,7 +72,7 @@ public function getReleases(): ReleasesCollection $pages = $this->registry->releases( $this->getId(), - new GitHubReleaseSource($this->api), + new PagedReleaseSource($this->api->releasePages(...), RepositoryApi::RELEASES_PER_PAGE), ); $anyPageLoaded = false; diff --git a/src/Module/Repository/Internal/GitLab/GitLabReleaseSource.php b/src/Module/Repository/Internal/GitLab/GitLabReleaseSource.php deleted file mode 100644 index cb0e22d2..00000000 --- a/src/Module/Repository/Internal/GitLab/GitLabReleaseSource.php +++ /dev/null @@ -1,36 +0,0 @@ - $first */ - $first = \intdiv($offset, RepositoryApi::RELEASES_PER_PAGE) + 1; - - foreach ($this->api->releasePages($first) as $page) { - yield $skip === 0 ? $page : new ReleasePage(\array_slice($page->releases, $skip), $page->last, $page->skipped); - $skip = 0; - } - } -} diff --git a/src/Module/Repository/Internal/GitLab/GitLabRepository.php b/src/Module/Repository/Internal/GitLab/GitLabRepository.php index 27b8cd66..753af6ff 100644 --- a/src/Module/Repository/Internal/GitLab/GitLabRepository.php +++ b/src/Module/Repository/Internal/GitLab/GitLabRepository.php @@ -10,6 +10,7 @@ use Internal\DLoad\Module\Repository\Collection\ReleasesCollection; use Internal\DLoad\Module\Repository\Exception\RateLimitException; use Internal\DLoad\Module\Repository\Internal\GitLab\Api\RepositoryApi; +use Internal\DLoad\Module\Repository\Internal\PagedReleaseSource; use Internal\DLoad\Module\Repository\Internal\Paginator; use Internal\DLoad\Module\Repository\Repository; use Internal\DLoad\Service\Logger; @@ -69,7 +70,7 @@ public function getReleases(): ReleasesCollection $pages = $this->registry->releases( $this->getId(), - new GitLabReleaseSource($this->api), + new PagedReleaseSource($this->api->releasePages(...), RepositoryApi::RELEASES_PER_PAGE), ); $anyPageLoaded = false; diff --git a/src/Module/Repository/Internal/PagedReleaseSource.php b/src/Module/Repository/Internal/PagedReleaseSource.php new file mode 100644 index 00000000..1f2045e2 --- /dev/null +++ b/src/Module/Repository/Internal/PagedReleaseSource.php @@ -0,0 +1,98 @@ + */ + private int $perPage; + + /** + * @param \Closure(int<1, max>): \Generator $pages Lists the + * pages from the given one on, requesting each one when reached. + * @param int<1, max> $perPage Number of releases the API is asked for per page. + */ + public function __construct( + private readonly \Closure $pages, + int $perPage, + ) { + $this->perPage = $perPage; + } + + /** + * @throws RepositoryException + */ + public function pages(int $offset = 0): \Generator + { + $number = $this->pageOf($offset); + $pages = ($this->pages)($number); + $page = $pages->current(); + + if ($number === 1 && $page !== null && !$page->last && $this->learn($page) && $this->pageOf($offset) > 1) { + $number = $this->pageOf($offset); + $pages = ($this->pages)($number); + $page = $pages->current(); + } + + // The listing ends before the position: it is listed from its last page + while ($page === null && $number > 1) { + $pages = ($this->pages)(--$number); + $page = $pages->current(); + } + + // The generator is walked by hand: one that ended before its first page is closed, and + // `foreach` cannot start it over + $size = $this->perPage; + while ($page !== null) { + yield new ReleasePage($page->releases, $page->last, $page->skipped, ($number - 1) * $size); + + ++$number; + $pages->next(); + $page = $pages->current(); + } + } + + /** + * Takes the length of a short first page followed by another one as the page size; tells + * whether it changed. + */ + private function learn(ReleasePage $page): bool + { + $size = \count($page->releases) + $page->skipped; + if ($size < 1 || $size * 2 > $this->perPage) { + return false; + } + + $this->perPage = $size; + + return true; + } + + /** + * @param int<0, max> $offset + * @return int<1, max> + */ + private function pageOf(int $offset): int + { + /** @var int<1, max> */ + return \intdiv($offset, $this->perPage) + 1; + } +} diff --git a/tests/Unit/Module/Registry/StoredVersionRegistryTest.php b/tests/Unit/Module/Registry/StoredVersionRegistryTest.php index 45cbf5ad..5be88cc6 100644 --- a/tests/Unit/Module/Registry/StoredVersionRegistryTest.php +++ b/tests/Unit/Module/Registry/StoredVersionRegistryTest.php @@ -141,6 +141,299 @@ public function releaseDeletedUpstreamDoesNotShiftTheTail(): void Assert::same($source->served, [0, 2, 4]); } + #[Test] + public function tailEndingInsideAPageCostsNoExtraRequest(): void + { + $source = ArrayReleaseSource::ofTags(['v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + $this->registry()->releases($this->id, $source)->current(); + $source->served = []; + + // The check leaves three stored, the last one inside page 2 + $source->publish('v7'); + $this->now += 601; + $again = self::flatten($this->registry()->releases($this->id, $source)); + + Assert::same($again, ['v7', 'v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + Assert::count($source->served, 4); + } + + #[Test] + public function tailEndingOnAPageBoundaryCostsOneRequestMore(): void + { + $source = ArrayReleaseSource::ofTags(['v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + self::take($this->registry()->releases($this->id, $source), 4); + $source->served = []; + + $this->now += 601; + $again = self::flatten($this->registry()->releases($this->id, $source)); + + // The check, the page ending with the last stored release, and the page the tail needs + Assert::same($again, ['v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + Assert::count($source->served, 3); + } + + #[Test] + public function tailSeenWholeByTheCheckGoesOnWithoutExtraRequest(): void + { + $source = ArrayReleaseSource::ofTags(['v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + $this->registry()->releases($this->id, $source)->current(); + $source->served = []; + + $this->now += 601; + $again = self::flatten($this->registry()->releases($this->id, $source)); + + Assert::same($again, ['v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + Assert::count($source->served, 3); + } + + #[Test] + public function releaseDeletedBelowTheCheckedSpanDoesNotShiftTheTail(): void + { + $source = ArrayReleaseSource::ofTags(['v8', 'v7', 'v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + self::take($this->registry()->releases($this->id, $source), 6); + $source->served = []; + + // `v5` is deleted upstream, outside the page the check fetches + $source->delete('v5'); + $this->now += 601; + $again = self::flatten($this->registry()->releases($this->id, $source)); + + // `v2` moved up into the stored part of the listing and is still loaded; the stored + // `v5` stays until its download fails + Assert::same($again, ['v8', 'v7', 'v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + Assert::same($this->stored(), ['v8', 'v7', 'v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + Assert::true($this->storage->load($this->id)->complete); + Assert::count($source->served, 3); + } + + #[Test] + public function deletedLastStoredReleaseDoesNotShiftTheTail(): void + { + $source = ArrayReleaseSource::ofTags(['v8', 'v7', 'v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + self::take($this->registry()->releases($this->id, $source), 6); + + $source->delete('v3'); + $this->now += 601; + $again = self::flatten($this->registry()->releases($this->id, $source)); + + Assert::same($again, ['v8', 'v7', 'v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + Assert::same($this->stored(), $again); + } + + #[Test] + public function deletionsMovingTheTailOffItsPageAreFollowedUp(): void + { + $source = ArrayReleaseSource::ofTags(['v12', 'v11', 'v10', 'v9', 'v8', 'v7', 'v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + self::take($this->registry()->releases($this->id, $source), 8); + + // The last stored `v5` moves up three positions, off the page its stored count points at + $source->delete('v8'); + $source->delete('v7'); + $source->delete('v6'); + $this->now += 601; + $again = self::flatten($this->registry()->releases($this->id, $source)); + + Assert::same($again, ['v12', 'v11', 'v10', 'v9', 'v8', 'v7', 'v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + Assert::same($this->stored(), $again); + } + + #[Test] + public function deepDeletionInALongTailCostsNoReload(): void + { + $tags = \array_map(static fn(int $i): string => "v$i", \range(2500, 1)); + $source = ArrayReleaseSource::ofTags($tags, perPage: 100); + self::take($this->registry()->releases($this->id, $source), 2000); + $source->served = []; + + $source->delete('v1000'); + $this->now += 601; + self::take($this->registry()->releases($this->id, $source), 2050); + + // The check, the page holding the last stored release and the next one + Assert::count($source->served, 3); + Assert::same($this->stored(), \array_slice($tags, 0, 2101)); + } + + #[Test] + public function releasesPublishedWithinTheTtlAreNeitherRepeatedNorAppended(): void + { + $source = ArrayReleaseSource::ofTags(['v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + $this->registry()->releases($this->id, $source)->current(); + + // More releases than a page holds: the page at the stored count holds no stored one + $source->publish('v9', 'v8', 'v7'); + $this->now += 100; + $again = self::flatten($this->registry()->releases($this->id, $source)); + + Assert::same($again, ['v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + Assert::same($this->stored(), $again); + + // The next check puts them on top + $this->now += 601; + Assert::same(self::flatten($this->registry()->releases($this->id, $source)), ['v9', 'v8', 'v7', 'v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + } + + #[Test] + public function releaseDeletedWithinTheTtlDoesNotShiftTheTail(): void + { + $source = ArrayReleaseSource::ofTags(['v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + self::take($this->registry()->releases($this->id, $source), 4); + + $source->delete('v4'); + $this->now += 100; + $again = self::flatten($this->registry()->releases($this->id, $source)); + + Assert::same($again, ['v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + Assert::same($this->stored(), $again); + } + + #[Test] + public function failedCheckLeavesReleasesPublishedAboveTheTailToTheNextOne(): void + { + $source = ArrayReleaseSource::ofTags(['v8', 'v7', 'v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + $this->registry()->releases($this->id, $source)->current(); + + $source->publish('n3', 'n2', 'n1'); + $source->fail(); + $this->now += 601; + $pages = $this->registry()->releases($this->id, $source); + Assert::same(self::tagsOf($pages->current()), ['v8', 'v7']); + + // The source is back for the tail + $source->failure = null; + Assert::same(self::flatten($pages), ['v8', 'v7', 'v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + Assert::same($this->stored(), ['v8', 'v7', 'v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + + $this->now += 601; + self::flatten($this->registry()->releases($this->id, $source)); + Assert::same($this->stored(), ['n3', 'n2', 'n1', 'v8', 'v7', 'v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + } + + #[Test] + public function unreadableReleaseAboveTheTailDoesNotShiftIt(): void + { + $source = ArrayReleaseSource::ofTags(['v10', 'v9', 'v8', 'v7', 'v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + $source->unreadable = ['v9']; + self::take($this->registry()->releases($this->id, $source), 5); + Assert::same($this->stored(), ['v10', 'v8', 'v7', 'v6', 'v5']); + + // The check bails out on `v9`; the stored `v5` lies one position further than its count + $this->now += 601; + $again = self::flatten($this->registry()->releases($this->id, $source)); + + Assert::same($again, ['v10', 'v8', 'v7', 'v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + Assert::same($this->stored(), $again); + } + + #[Test] + public function unreadableReleaseInTheStoredTailDoesNotShiftIt(): void + { + $source = ArrayReleaseSource::ofTags(['v10', 'v9', 'v8', 'v7', 'v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + $source->unreadable = ['v6']; + self::take($this->registry()->releases($this->id, $source), 7); + $source->served = []; + + $this->now += 601; + $again = self::flatten($this->registry()->releases($this->id, $source)); + + Assert::same($again, ['v10', 'v9', 'v8', 'v7', 'v5', 'v4', 'v3', 'v2', 'v1']); + Assert::count($source->served, 3); + + // Every later check costs the first page only + $source->served = []; + $this->now += 601; + self::flatten($this->registry()->releases($this->id, $source)); + Assert::same($source->served, [0]); + } + + #[Test] + public function releaseTurnedReadableAboveTheTailIsNotLoaded(): void + { + $source = ArrayReleaseSource::ofTags(['v10', 'v9', 'v8', 'v7', 'v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + $source->unreadable = ['v6']; + self::take($this->registry()->releases($this->id, $source), 7); + + // `v6` lies above the last stored release, where only a check could see it + $source->unreadable = []; + $this->now += 601; + $again = self::flatten($this->registry()->releases($this->id, $source)); + + Assert::same($again, ['v10', 'v9', 'v8', 'v7', 'v5', 'v4', 'v3', 'v2', 'v1']); + } + + #[Test] + public function listingShorterThanTheStoredCountIsContinuedFromItsLastPage(): void + { + $source = ArrayReleaseSource::ofTags(['v8', 'v7', 'v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + self::take($this->registry()->releases($this->id, $source), 6); + + $source->delete('v5'); + $source->delete('v4'); + $source->delete('v3'); + $source->delete('v2'); + $this->now += 601; + $again = self::flatten($this->registry()->releases($this->id, $source)); + + Assert::same($again, ['v8', 'v7', 'v6', 'v5', 'v4', 'v3', 'v1']); + Assert::same($this->stored(), $again); + Assert::true($this->storage->load($this->id)->complete); + } + + #[Test] + public function listingEmptiedUpstreamLeavesNothingStored(): void + { + $source = ArrayReleaseSource::ofTags(['v4', 'v3', 'v2', 'v1']); + $this->registry()->releases($this->id, $source)->current(); + + foreach (['v4', 'v3', 'v2', 'v1'] as $tag) { + $source->delete($tag); + } + $this->now += 601; + + Assert::same(self::flatten($this->registry()->releases($this->id, $source)), []); + Assert::same($this->stored(), []); + Assert::true($this->storage->load($this->id)->complete); + } + + #[Test] + public function tailIsLoadedOnlyAsFarAsTheConsumerGoes(): void + { + $source = ArrayReleaseSource::ofTags(['v8', 'v7', 'v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + self::take($this->registry()->releases($this->id, $source), 4); + $source->served = []; + + $this->now += 601; + self::take($this->registry()->releases($this->id, $source), 5); + + Assert::count($source->served, 3); + Assert::same($this->stored(), ['v8', 'v7', 'v6', 'v5', 'v4', 'v3']); + Assert::false($this->storage->load($this->id)->complete); + } + + #[Test] + public function failureWhileLoadingTheTailKeepsWhatWasLoaded(): void + { + $source = ArrayReleaseSource::ofTags(['v8', 'v7', 'v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + self::take($this->registry()->releases($this->id, $source), 4); + $source->served = []; + + // The check and two pages of the tail succeed, the next one fails + $source->failAfter = 3; + $this->now += 601; + try { + self::flatten($this->registry()->releases($this->id, $source)); + Assert::fail('The failure of the tail request must reach the caller.'); + } catch (ApiException) { + } + + Assert::same($this->stored(), ['v8', 'v7', 'v6', 'v5', 'v4', 'v3']); + Assert::false($this->storage->load($this->id)->complete); + + $source->failAfter = \PHP_INT_MAX; + $this->now += 601; + Assert::same(self::flatten($this->registry()->releases($this->id, $source)), ['v8', 'v7', 'v6', 'v5', 'v4', 'v3', 'v2', 'v1']); + } + #[Test] public function unreadableReleaseIsNotTakenForADeletedOne(): void { @@ -362,6 +655,22 @@ private static function flatten(iterable $pages): array return $tags; } + /** + * Consumes pages until the given number of releases has been served. + * + * @param \Generator, mixed, void> $pages + */ + private static function take(\Generator $pages, int $count): void + { + $seen = 0; + foreach ($pages as $page) { + $seen += \count($page); + if ($seen >= $count) { + break; + } + } + } + /** * @param list $releases * @return list @@ -371,6 +680,14 @@ private static function tagsOf(array $releases): array return \array_map(static fn(ReleaseRecord $release): string => $release->tag, $releases); } + /** + * @return list + */ + private function stored(): array + { + return self::tagsOf($this->storage->load($this->id)?->releases() ?? []); + } + private function registry(bool $refresh = false, ?Logger $logger = null): StoredVersionRegistry { return new StoredVersionRegistry($this->storage, 600, $logger ?? new Logger(), $refresh, fn(): int => $this->now); diff --git a/tests/Unit/Module/Registry/Stub/ArrayReleaseSource.php b/tests/Unit/Module/Registry/Stub/ArrayReleaseSource.php index d1e1921b..325b97ba 100644 --- a/tests/Unit/Module/Registry/Stub/ArrayReleaseSource.php +++ b/tests/Unit/Module/Registry/Stub/ArrayReleaseSource.php @@ -27,6 +27,9 @@ final class ArrayReleaseSource implements ReleaseSource /** When set, every page request fails with this exception. */ public ?\Throwable $failure = null; + /** Number of pages served before every further request fails. */ + public int $failAfter = \PHP_INT_MAX; + /** * Tags of releases the source cannot read: they are left out of the pages and counted as skipped. * @@ -81,24 +84,33 @@ public function fail(?\Throwable $failure = null): void public function pages(int $offset = 0): \Generator { - // Align with a page boundary and skip within the page, like a real paged API + // Whole pages from the one holding the offset, like a real paged API $page = \intdiv($offset, $this->perPage); - $skip = $offset % $this->perPage; do { $this->failure === null or throw $this->failure; + \count($this->served) < $this->failAfter or throw new ApiException('API is unavailable.', 'stub/stub'); $this->served[] = $page * $this->perPage; $items = \array_slice($this->releases, $page * $this->perPage, $this->perPage); + + // A page past the end of the listing is no page at all, so the listing is served + // from its last page + if ($items === [] && $page > 0) { + --$page; + $last = false; + continue; + } + $last = ($page + 1) * $this->perPage >= \count($this->releases); $readable = \array_values(\array_filter( $items, fn(ReleaseRecord $release): bool => !\in_array($release->tag, $this->unreadable, true), )); - yield new ReleasePage(\array_slice($readable, $skip), $last, \count($items) - \count($readable)); + // Unreadable releases hold their positions on the page, so the offset counts them + yield new ReleasePage($readable, $last, \count($items) - \count($readable), $page * $this->perPage); - $skip = 0; ++$page; } while (!$last); } diff --git a/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php b/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php index e678d35e..e1ef9586 100644 --- a/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php +++ b/tests/Unit/Module/Repository/Internal/GitHub/GitHubRepositoryTest.php @@ -14,8 +14,8 @@ use Internal\DLoad\Module\Repository\Exception\RateLimitException; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\Client; use Internal\DLoad\Module\Repository\Internal\GitHub\Api\RepositoryApi; -use Internal\DLoad\Module\Repository\Internal\GitHub\GitHubReleaseSource; use Internal\DLoad\Module\Repository\Internal\GitHub\GitHubRepository; +use Internal\DLoad\Module\Repository\Internal\PagedReleaseSource; use Internal\DLoad\Module\Repository\ReleaseInterface; use Internal\DLoad\Service\Logger; use Internal\DLoad\Tests\Unit\Module\Registry\Stub\InMemoryRegistryStorage; @@ -27,7 +27,7 @@ use Testo\Test; #[Covers(GitHubRepository::class)] -#[Covers(GitHubReleaseSource::class)] +#[Covers(PagedReleaseSource::class)] final class GitHubRepositoryTest { #[Test] @@ -101,6 +101,81 @@ public function secondRunIsServedFromTheRegistryWithoutRequests(): void Assert::same(\count($secondRun), 200); } + #[Test] + public function firstPageShortOfAFewReleasesDoesNotChangeThePageSize(): void + { + // Page 1 comes back one release short, as a server filtering after paging answers + $storage = new InMemoryRegistryStorage(); + $firstClient = new PagedClientStub(pages: 4, withheld: ['v1.0.3']); + $seen = 0; + foreach (self::createRepository($firstClient, self::registry($storage))->getReleases() as $release) { + unset($release); + if (++$seen === 150) { + break; + } + } + + // Every page keeps its place: the tail continues on page 2, after the last stored release + $secondClient = new PagedClientStub(pages: 4, withheld: ['v1.0.3']); + $all = self::names(self::createRepository($secondClient, self::registry($storage, refresh: true))); + + Assert::same($secondClient->requestedPages(), [1, 2, 3, 4]); + Assert::same(\count($all), 399); + Assert::same(\count(\array_unique($all)), 399); + } + + #[Test] + public function unreadableReleaseAfterTheLastStoredOneOnItsPageDoesNotReloadTheTail(): void + { + // Three pages are stored; the last release of page 3 cannot be read, so the page the tail + // continues from starts one release before the last stored one + $storage = new InMemoryRegistryStorage(); + $firstClient = new PagedClientStub(pages: 4, unreadable: ['v1.0.300']); + $seen = 0; + foreach (self::createRepository($firstClient, self::registry($storage))->getReleases() as $release) { + unset($release); + if (++$seen === 250) { + break; + } + } + Assert::same($storage->load(new RepositoryId(GitHubRepository::TYPE, 'owner/repo'))?->count(), 299); + + $secondClient = new PagedClientStub(pages: 4, unreadable: ['v1.0.300']); + $all = self::names(self::createRepository($secondClient, self::registry($storage, refresh: true))); + + Assert::same($secondClient->requestedPages(), [1, 3, 4]); + Assert::same(\count($all), 399); + Assert::same(\count(\array_unique($all)), 399); + + $thirdClient = new PagedClientStub(pages: 4, unreadable: ['v1.0.300']); + self::names(self::createRepository($thirdClient, self::registry($storage, refresh: true))); + Assert::same($thirdClient->requestedPages(), [1]); + } + + #[Test] + public function unreadableReleaseAboveTheTailDoesNotMakeEveryRunReloadIt(): void + { + // One release of page 1 cannot be read from the start; the first run stores two pages + $storage = new InMemoryRegistryStorage(); + $firstClient = new PagedClientStub(pages: 4, broken: 1); + $seen = 0; + foreach (self::createRepository($firstClient, self::registry($storage))->getReleases() as $release) { + unset($release); + if (++$seen === 150) { + break; + } + } + + // A check that bails out on the unreadable release leaves the record as it is and does + // not go looking for a shift it could not tell from the missing release + $secondClient = new PagedClientStub(pages: 4, broken: 1); + $all = self::names(self::createRepository($secondClient, self::registry($storage, refresh: true))); + + Assert::same($secondClient->requestedPages(), [1, 2, 3, 4]); + Assert::same(\count($all), 399); + Assert::same(\count(\array_unique($all)), 399); + } + #[Test] public function olderReleasesAreLoadedFromTheApiWhenTheRegistryRunsOut(): void { @@ -113,12 +188,13 @@ public function olderReleasesAreLoadedFromTheApiWhenTheRegistryRunsOut(): void break; } - // The second run needs everything: the stored page costs nothing, the rest is fetched + // The second run needs everything: the stored page is served without a request, and + // fetched once more only to find where the tail continues $secondClient = new PagedClientStub(pages: 3); $all = self::names(self::createRepository($secondClient, self::registry($storage))); Assert::same($firstClient->requestedPages(), [1]); - Assert::same($secondClient->requestedPages(), [2, 3]); + Assert::same($secondClient->requestedPages(), [1, 2, 3]); Assert::same(\count($all), 300); } @@ -145,7 +221,7 @@ public function draftReleasesAreNeitherServedNorStored(): void #[Test] public function unreadableReleasesAreCountedOnThePage(): void { - $page = (new GitHubReleaseSource(self::api(new PagedClientStub(pages: 1, broken: 2))))->pages()->current(); + $page = self::source(new PagedClientStub(pages: 1, broken: 2))->pages()->current(); Assert::same(\count($page->releases), 98); Assert::same($page->skipped, 2); @@ -181,6 +257,43 @@ public function tailIsLoadedFromInsideAPageWhenTheStoredCountIsNotPageAligned(): Assert::same($all[50], 'v1.0.51'); } + #[Test] + public function repositoryWithoutReleasesListsNothing(): void + { + $client = new PagedClientStub(pages: 0); + + Assert::same(self::names(self::createRepository($client, self::registry(new InMemoryRegistryStorage()))), []); + Assert::same($client->requestedPages(), [1]); + Assert::same(\iterator_to_array(self::source($client)->pages(150), false), []); + } + + #[Test] + public function serverCappingThePageSizeIsAskedForEveryPageOnce(): void + { + // Six releases, two per page whatever `per_page` says; the first run needs three of them + $storage = new InMemoryRegistryStorage(); + $firstClient = new PagedClientStub(pages: 3, releasesPerPage: 2, perPageLimit: 2); + $seen = 0; + foreach (self::createRepository($firstClient, self::registry($storage))->getReleases() as $release) { + unset($release); + if (++$seen === 3) { + break; + } + } + + // The first page tells the page size, so the tail continues on page 2, not on page 1 again + Assert::same($firstClient->requestedPages(), [1, 2]); + + // A later run learns the page size from the first page anew and jumps to the page + // ending with the last stored release + $secondClient = new PagedClientStub(pages: 3, releasesPerPage: 2, perPageLimit: 2); + $all = self::names(self::createRepository($secondClient, self::registry($storage))); + + Assert::same($secondClient->requestedPages(), [1, 2, 3]); + Assert::same(\count($all), 6); + Assert::same(\count(\array_unique($all)), 6); + } + #[Test] public function theSameCollectionIsReturnedOnEveryCall(): void { @@ -306,9 +419,14 @@ private static function api(PagedClientStub $client): RepositoryApi ); } - private static function registry(InMemoryRegistryStorage $storage): StoredVersionRegistry + private static function source(PagedClientStub $client): PagedReleaseSource + { + return new PagedReleaseSource(self::api($client)->releasePages(...), RepositoryApi::RELEASES_PER_PAGE); + } + + private static function registry(InMemoryRegistryStorage $storage, bool $refresh = false): StoredVersionRegistry { - return new StoredVersionRegistry($storage, 600, new Logger()); + return new StoredVersionRegistry($storage, 600, new Logger(), $refresh); } /** diff --git a/tests/Unit/Module/Repository/Internal/GitHub/Stub/PagedClientStub.php b/tests/Unit/Module/Repository/Internal/GitHub/Stub/PagedClientStub.php index d1070a12..9b8dad52 100644 --- a/tests/Unit/Module/Repository/Internal/GitHub/Stub/PagedClientStub.php +++ b/tests/Unit/Module/Repository/Internal/GitHub/Stub/PagedClientStub.php @@ -26,16 +26,22 @@ final class PagedClientStub implements ClientInterface public array $requests = []; /** - * @param int<1, max> $pages Number of pages the list is split into when 100 releases are requested per page. + * @param int<0, max> $pages Number of pages the list is split into when 100 releases are requested per page. * @param int<1, max> $releasesPerPage Number of releases on every such page. * @param int<0, max> $drafts Draft releases listed on top, as the API shows them to the token holder. * @param int<0, max> $broken Number of the newest published releases that cannot be decoded. + * @param int<1, max>|null $perPageLimit Greatest number of releases served per page whatever `per_page` asks for. + * @param list $withheld Tags left out of a page after it was cut, as a server filtering after paging does. + * @param list $unreadable Tags of releases that cannot be decoded, wherever they are listed. */ public function __construct( private readonly int $pages = 1, private readonly int $releasesPerPage = 100, private readonly int $drafts = 0, private readonly int $broken = 0, + private readonly ?int $perPageLimit = null, + private readonly array $withheld = [], + private readonly array $unreadable = [], ) {} public function sendRequest(RequestInterface $request): ResponseInterface @@ -44,7 +50,7 @@ public function sendRequest(RequestInterface $request): ResponseInterface $this->requests[] = $query; $page = self::pageOf($query); - $perPage = self::perPageOf($query); + $perPage = \min(self::perPageOf($query), $this->perPageLimit ?? \PHP_INT_MAX); $all = $this->allReleases(); // Serve the slice the real API would serve for the requested page size @@ -57,6 +63,11 @@ public function sendRequest(RequestInterface $request): ResponseInterface ? ['link' => [\sprintf('; rel="next"', $page + 1)]] : []; + $releases = \array_values(\array_filter( + $releases, + fn(array $release): bool => !\in_array($release['name'], $this->withheld, true), + )); + return new ResponseStub(200, $headers, \json_encode($releases)); } @@ -97,7 +108,7 @@ private function allReleases(): array $releases[] = [ 'name' => $tag, // A number where a string belongs fails the strict constructor of the response object - 'tag_name' => $i >= 1 && $i <= $this->broken ? $i : $tag, + 'tag_name' => ($i >= 1 && $i <= $this->broken) || \in_array($tag, $this->unreadable, true) ? $i : $tag, 'published_at' => $i < 1 ? null : '2024-01-01T00:00:00Z', 'assets' => [[ 'name' => 'rr-linux-amd64.tar.gz', diff --git a/tests/Unit/Module/Repository/Internal/GitLab/GitLabRepositoryTest.php b/tests/Unit/Module/Repository/Internal/GitLab/GitLabRepositoryTest.php index 396b9a29..c96089d5 100644 --- a/tests/Unit/Module/Repository/Internal/GitLab/GitLabRepositoryTest.php +++ b/tests/Unit/Module/Repository/Internal/GitLab/GitLabRepositoryTest.php @@ -16,8 +16,8 @@ use Internal\DLoad\Module\Repository\Internal\GitLab\Api\Client; use Internal\DLoad\Module\Repository\Internal\GitLab\Api\RepositoryApi; use Internal\DLoad\Module\Repository\Internal\GitLab\GitLabAsset; -use Internal\DLoad\Module\Repository\Internal\GitLab\GitLabReleaseSource; use Internal\DLoad\Module\Repository\Internal\GitLab\GitLabRepository; +use Internal\DLoad\Module\Repository\Internal\PagedReleaseSource; use Internal\DLoad\Module\Repository\ReleaseInterface; use Internal\DLoad\Service\Logger; use Internal\DLoad\Tests\Unit\Module\Registry\Stub\InMemoryRegistryStorage; @@ -29,7 +29,7 @@ use Testo\Test; #[Covers(GitLabRepository::class)] -#[Covers(GitLabReleaseSource::class)] +#[Covers(PagedReleaseSource::class)] #[Covers(GitLabAsset::class)] final class GitLabRepositoryTest { @@ -116,10 +116,40 @@ public function olderReleasesAreLoadedFromTheApiWhenTheRegistryRunsOut(): void $all = self::names(self::createRepository($secondClient, self::registry($storage))); Assert::same($firstClient->requestedPages(), [1]); - Assert::same($secondClient->requestedPages(), [2, 3]); + Assert::same($secondClient->requestedPages(), [1, 2, 3]); Assert::same(\count($all), 300); } + #[Test] + public function projectWithoutReleasesListsNothing(): void + { + $client = new PagedClientStub(pages: 0); + + Assert::same(self::names(self::createRepository($client, self::registry(new InMemoryRegistryStorage()))), []); + Assert::same($client->requestedPages(), [1]); + Assert::same(\iterator_to_array(self::source($client)->pages(150), false), []); + } + + #[Test] + public function serverCappingThePageSizeIsAskedForEveryPageOnce(): void + { + // Four releases, two per page whatever `per_page` says + $client = new PagedClientStub(pages: 2, releasesPerPage: 2, perPageLimit: 2); + $all = self::names(self::createRepository($client, self::registry(new InMemoryRegistryStorage()))); + + // The first page tells the page size, so the tail continues on page 2, not on page 1 again + Assert::same($client->requestedPages(), [1, 2]); + Assert::same(\count($all), 4); + + // A source that does not know the page size yet learns it from page 1 and jumps to the right one + $client = new PagedClientStub(pages: 3, releasesPerPage: 2, perPageLimit: 2); + $pages = \iterator_to_array(self::source($client)->pages(4), false); + + Assert::same($client->requestedPages(), [1, 3]); + Assert::count($pages, 1); + Assert::same($pages[0]->releases[0]->tag, 'v1.0.5'); + } + #[Test] public function tailIsLoadedFromInsideAPageWhenTheStoredCountIsNotPageAligned(): void { @@ -283,6 +313,15 @@ private static function registry(InMemoryRegistryStorage $storage): StoredVersio return new StoredVersionRegistry($storage, 600, new Logger()); } + private static function source(PagedClientStub $client): PagedReleaseSource + { + $httpFactory = new NyholmFactoryImpl(new Logger()); + + $api = new RepositoryApi(new Client($httpFactory, $client), $httpFactory, 'group/project'); + + return new PagedReleaseSource($api->releasePages(...), RepositoryApi::RELEASES_PER_PAGE); + } + /** * @return list */ diff --git a/tests/Unit/Module/Repository/Internal/GitLab/Stub/PagedClientStub.php b/tests/Unit/Module/Repository/Internal/GitLab/Stub/PagedClientStub.php index ca644008..31ff23bc 100644 --- a/tests/Unit/Module/Repository/Internal/GitLab/Stub/PagedClientStub.php +++ b/tests/Unit/Module/Repository/Internal/GitLab/Stub/PagedClientStub.php @@ -33,12 +33,14 @@ final class PagedClientStub implements ClientInterface public array $uris = []; /** - * @param int<1, max> $pages Number of pages the list is split into when 100 releases are requested per page. + * @param int<0, max> $pages Number of pages the list is split into when 100 releases are requested per page. * @param int<1, max> $releasesPerPage Number of releases on every such page. + * @param int<1, max>|null $perPageLimit Greatest number of releases served per page whatever `per_page` asks for. */ public function __construct( private readonly int $pages = 1, private readonly int $releasesPerPage = 100, + private readonly ?int $perPageLimit = null, ) {} public function sendRequest(RequestInterface $request): ResponseInterface @@ -48,7 +50,7 @@ public function sendRequest(RequestInterface $request): ResponseInterface $this->uris[] = (string) $request->getUri(); $page = self::pageOf($query); - $perPage = self::perPageOf($query); + $perPage = \min(self::perPageOf($query), $this->perPageLimit ?? \PHP_INT_MAX); $all = $this->allReleases(); // Serve the slice the real API would serve for the requested page size diff --git a/tests/Unit/Module/Repository/Internal/PagedReleaseSourceTest.php b/tests/Unit/Module/Repository/Internal/PagedReleaseSourceTest.php new file mode 100644 index 00000000..e258feda --- /dev/null +++ b/tests/Unit/Module/Repository/Internal/PagedReleaseSourceTest.php @@ -0,0 +1,201 @@ + + */ + private array $requested = []; + + #[Test] + public function pagesStartWithTheOneHoldingTheOffset(): void + { + $source = $this->source(total: 250, perPage: 100); + + $pages = \iterator_to_array($source->pages(150), false); + + Assert::same($this->requested, [2, 3]); + Assert::same(self::tagsOf($pages[0]), self::tags(100, 200)); + Assert::same($pages[0]->offset, 100); + Assert::same(self::tagsOf($pages[1]), self::tags(200, 250)); + Assert::same($pages[1]->offset, 200); + Assert::true($pages[1]->last); + } + + #[Test] + public function pagesAreRequestedOnlyWhenReached(): void + { + $source = $this->source(total: 250, perPage: 100); + + $pages = $source->pages(); + Assert::same($this->requested, []); + + $pages->current(); + Assert::same($this->requested, [1]); + + $pages->next(); + Assert::same($this->requested, [1, 2]); + } + + #[Test] + public function listingEndingBeforeTheOffsetStartsWithItsLastPage(): void + { + $source = $this->source(total: 250, perPage: 100); + + $pages = \iterator_to_array($source->pages(450), false); + + Assert::same($this->requested, [5, 4, 3]); + Assert::count($pages, 1); + Assert::same($pages[0]->offset, 200); + Assert::true($pages[0]->last); + } + + #[Test] + public function emptyListingYieldsNoPage(): void + { + $source = $this->source(total: 0, perPage: 100); + + Assert::same(\iterator_to_array($source->pages(), false), []); + Assert::same(\iterator_to_array($source->pages(150), false), []); + Assert::same($this->requested, [1, 2, 1]); + } + + #[Test] + public function smallFirstPageTellsThePageSize(): void + { + // The server puts two releases on a page whatever is asked for + $source = $this->source(total: 10, perPage: 100, served: 2); + + $pages = \iterator_to_array($source->pages(5), false); + + // Page 1 was fetched on the assumption of a hundred per page; it showed the real size + Assert::same($this->requested, [1, 3, 4, 5]); + Assert::same(self::tagsOf($pages[0]), self::tags(4, 6)); + Assert::same($pages[0]->offset, 4); + + // The size learned serves the next listing of the same source + $this->requested = []; + \iterator_to_array($source->pages(8), false); + Assert::same($this->requested, [5]); + } + + #[Test] + public function firstPageShortOfAFewReleasesKeepsThePageSize(): void + { + // A release filtered out after paging, not a smaller page + $source = $this->source(total: 300, perPage: 100, missing: ['v3']); + + $source->pages()->current(); + $pages = \iterator_to_array($source->pages(200), false); + + Assert::same($this->requested, [1, 3]); + Assert::same($pages[0]->offset, 200); + } + + #[Test] + public function shortPageBelowTheFirstOneIsNotTakenForThePageSize(): void + { + $source = $this->source(total: 300, perPage: 100, missing: ['v150']); + + $pages = \iterator_to_array($source->pages(200), false); + + Assert::same($this->requested, [3]); + Assert::same(self::tagsOf($pages[0]), self::tags(200, 300)); + } + + #[Test] + public function unreadableReleasesHoldTheirPositionsOnThePage(): void + { + $source = $this->source(total: 300, perPage: 100, unreadable: ['v102']); + + $page = $source->pages(105)->current(); + + Assert::same($page->offset, 100); + Assert::same($page->skipped, 1); + Assert::count($page->releases, 99); + } + + #[BeforeTest] + protected function prepare(): void + { + $this->requested = []; + } + + /** + * @return list + */ + private static function tags(int $from, int $to): array + { + return $to > $from ? \array_map(static fn(int $i): string => "v$i", \range($from, $to - 1)) : []; + } + + /** + * @return list + */ + private static function tagsOf(ReleasePage $page): array + { + return \array_map(static fn(ReleaseRecord $release): string => $release->tag, $page->releases); + } + + /** + * Source over a listing of `v0`..`v{total-1}`. + * + * @param int<0, max> $total + * @param int<1, max> $perPage Releases asked for per page. + * @param int<1, max>|null $served Releases the server puts on a page instead. + * @param list $missing Tags filtered out after paging. + * @param list $unreadable Tags the API cannot read. + */ + private function source(int $total, int $perPage, ?int $served = null, array $missing = [], array $unreadable = []): PagedReleaseSource + { + $size = $served ?? $perPage; + $pages = function (int $page) use ($total, $size, $missing, $unreadable): \Generator { + while (true) { + // Like a real API, a page past the end is requested and comes back empty + $this->requested[] = $page; + $slice = \array_slice(self::tags(0, $total), ($page - 1) * $size, $size); + if ($slice === []) { + return; + } + + $releases = []; + $skipped = 0; + foreach ($slice as $tag) { + if (\in_array($tag, $missing, true)) { + continue; + } + if (\in_array($tag, $unreadable, true)) { + ++$skipped; + continue; + } + $releases[] = new ReleaseRecord($tag, $tag); + } + + $last = $page * $size >= $total; + yield new ReleasePage($releases, $last, $skipped); + + if ($last) { + return; + } + ++$page; + } + }; + + return new PagedReleaseSource($pages, $perPage); + } +}