Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 69 additions & 30 deletions internal/distribute/history.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,11 @@ import (
// default branch's history is fetched: its date less a day.
const deepenMargin = 24 * time.Hour

// cutDeepens are the depths, in commits, that ancestor fetches below the
// shallow boundaries one after the other while a merge of an older branch
// keeps the default branch's history cut short.
var cutDeepens = []int{decide.MaxHistory, 25 * decide.MaxHistory}

// branchReads is what step 6 learned about a target's sync branches.
type branchReads struct {
// branches are the classified branches by name (the sync branch and the
Expand Down Expand Up @@ -62,10 +67,11 @@ type branchRead struct {
// the snapshot fetched) and its first-parent chain read at once, since a
// later fetch may cut the shallow history; then the ancestry of Hc's first
// parent and of the second parents of the merges after Hc is proven
// against B (DeepenSince, then IsAncestor; what cannot be proven is
// TriUnknown), the branches are classified, and with adopt the one-off
// migration rule is applied to foreign ones. moving is the branch a push
// would move, whose workflows difference is read.
// against B (DeepenSince, deeper where a merge of an older branch cut the
// history short, then IsAncestor; what cannot be proven is TriUnknown), the
// branches are classified, and with adopt the one-off migration rule is
// applied to foreign ones. moving is the branch a push would move, whose
// workflows difference is read.
//
// An error is one of the target: a fetch or a read of the branch failed, or
// the fetch of the default branch's history for the proofs failed in
Expand Down Expand Up @@ -209,15 +215,17 @@ func (r *run) prove(ctx context.Context, w *Work, hs []*branchRead, adopt bool)
for _, h := range legacy {
dates = append(dates, h.hist.Commits[len(h.hist.Commits)-1].SHA)
}
deepened, err := r.deepen(ctx, w, dates)
since, err := r.deepen(ctx, w, dates)
if err != nil {
return err
}
for _, sha := range slices.Sorted(maps.Keys(need)) {
proven[sha] = r.ancestor(ctx, w, sha, deepened)
if proven[sha], err = r.ancestor(ctx, w, sha, since); err != nil {
return err
}
}
for _, h := range legacy {
h.legacy = deepened && r.legacyRewritable(ctx, w, h.hist.Head)
h.legacy = !since.IsZero() && r.legacyRewritable(ctx, w, h.hist.Head)
}
}
for _, h := range hs {
Expand Down Expand Up @@ -246,38 +254,38 @@ func triOf(sha, b string, proven map[string]decide.Tri) decide.Tri {
}

// deepen fetches the default branch's history back to a day before the
// oldest of the commits shas (DeepenSince) and reports whether it did. A
// fetch that failed in transport (transportFailure), after three attempts,
// is an error; one that failed otherwise, and an unknown date, prove
// nothing (false, with a warning).
func (r *run) deepen(ctx context.Context, w *Work, shas []string) (bool, error) {
// oldest of the commits shas (DeepenSince) and returns that date. A fetch
// that failed in transport (transportFailure), after three attempts, is an
// error; one that failed otherwise, and an unknown date, prove nothing (the
// zero time, with a warning).
func (r *run) deepen(ctx context.Context, w *Work, shas []string) (time.Time, error) {
var oldest time.Time
for _, sha := range shas {
c, err := w.Repo.Commit(ctx, sha)
if err != nil && ctx.Err() != nil {
return false, err
return time.Time{}, err
}
if err != nil || c.Time.IsZero() {
r.proofWarning(w, fmt.Sprintf("the date of %s is unknown (%v)", short(sha), err))
return false, nil
return time.Time{}, nil
}
if oldest.IsZero() || c.Time.Before(oldest) {
oldest = c.Time
}
}
if oldest.IsZero() {
return false, nil
return time.Time{}, nil
}
since := oldest.Add(-deepenMargin)
err := r.gitRetry(ctx, w.t.prov, func() error { return gitFailure(w.Repo.DeepenSince(ctx, w.DefaultBranch, since)) })
switch {
case err == nil:
return true, nil
return since, nil
case transportFailure(ctx, err):
return false, fmt.Errorf("fetch it back to %s: %w", since.UTC().Format(time.DateOnly), err)
return time.Time{}, fmt.Errorf("fetch it back to %s: %w", since.UTC().Format(time.DateOnly), err)
}
r.proofWarning(w, fmt.Sprintf("the history of %s could not be fetched back to %s: %v", w.DefaultBranch, since.UTC().Format(time.DateOnly), err))
return false, nil
return time.Time{}, nil
}

// transportFailure reports whether err, a failed fetch, says nothing about
Expand All @@ -289,20 +297,51 @@ func transportFailure(ctx context.Context, err error) bool {
platform.ClassOf(err) != platform.ClassUnknown
}

// ancestor proves whether sha is an ancestor of B, after deepened history.
func (r *run) ancestor(ctx context.Context, w *Work, sha string, deepened bool) decide.Tri {
if !deepened {
return decide.TriUnknown
// ancestor proves whether sha is an ancestor of B on the history deepen
// fetched back to since (the zero time: none, nothing is proven). git's
// "no" is a proof only when no shallow boundary in B's history is as new as
// since (gitx.TargetRepo.ShallowNotBefore): a merge of a branch older than
// since cuts the default branch's own history short. Until then the
// history is fetched deeper below the boundaries, by each of cutDeepens in
// turn, and asked again; what is still cut short after the last is
// unknown. An error is a transport failure of such a fetch, after three
// attempts.
func (r *run) ancestor(ctx context.Context, w *Work, sha string, since time.Time) (decide.Tri, error) {
if since.IsZero() {
return decide.TriUnknown, nil
}
ok, err := w.Repo.IsAncestor(ctx, sha, w.B)
switch {
case err != nil:
r.proofWarning(w, fmt.Sprintf("whether %s is on %s is unknown: %v", short(sha), w.DefaultBranch, err))
return decide.TriUnknown
case ok:
return decide.TriYes
unknown := func(why any) (decide.Tri, error) {
r.proofWarning(w, fmt.Sprintf("whether %s is on %s is unknown: %v", short(sha), w.DefaultBranch, why))
return decide.TriUnknown, nil
}
for i := 0; ; i++ {
ok, err := w.Repo.IsAncestor(ctx, sha, w.B)
switch {
case err != nil:
return unknown(err)
case ok:
return decide.TriYes, nil
}
cut, err := w.Repo.ShallowNotBefore(ctx, w.B, since)
switch {
case err != nil:
return unknown(err)
case len(cut) == 0:
return decide.TriNo, nil
case i == len(cutDeepens):
return unknown("the history is still cut short at " + short(cut[0]))
}
by := cutDeepens[i]
err = r.gitRetry(ctx, w.t.prov, func() error { return gitFailure(w.Repo.Deepen(ctx, w.DefaultBranch, by)) })
switch {
case err == nil:
case transportFailure(ctx, err):
return decide.TriUnknown, fmt.Errorf("fetch it %d commits deeper: %w", by, err)
default:
r.proofWarning(w, fmt.Sprintf("the history of %s could not be fetched %d commits deeper: %v", w.DefaultBranch, by, err))
return decide.TriUnknown, nil
}
}
return decide.TriNo
}

// ancestryAnswer is one answer of Write.HubIsAncestor about a hub commit.
Expand Down
28 changes: 28 additions & 0 deletions internal/distribute/run_branch_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -111,3 +111,31 @@ func TestRunBranchSafety(t *testing.T) {
}
}
}

// A merge into the default branch of a branch older than the history the
// proof fetches (a day before Hc's parent) cuts that history short: git
// makes the merge a shallow boundary with both of its parents, and Hc's
// parent below it is not reached. It once paused the pull request as
// edited ("sits on commits that are not on the default branch").
func TestRunBranchOldMerge(t *testing.T) {
t.Parallel()
w := newGitWorld(t)
api := w.optedIn("acme/api", nil)
old := map[string][]byte{"src/old.txt": []byte("work begun long ago\n")}
if _, err := w.p.PushFiles(api.ID, "feature", old, w.person, fake.Epoch.Add(-72*time.Hour)); err != nil {
t.Fatalf("PushFiles: %v", err)
}
w.syncCommit(api, branch, "", baseFiles...)
n := w.openOwn(api)
w.written(api, n)
feature := w.pr(api, platform.PR{Head: "feature", Base: "main", Author: w.person, Title: "old work"})
if _, err := w.p.MergePR(api.ID, feature, fake.MergeCommit, w.person, time.Time{}); err != nil {
t.Fatalf("MergePR: %v", err)
}
w.ok()
rep := w.run(w.deps(ModeDistribute), ModeDistribute)
tg := want(t, rep, "gh:acme/api", report.OutcomeUnchanged, "", n)
if writes := w.p.Writes(); len(writes) > 0 {
t.Errorf("wrote %q (warnings %q)", writes, tg.Warnings)
}
}
11 changes: 6 additions & 5 deletions internal/gitx/delivery.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@ package gitx
// fetches, commits built without a work tree, and pushes with a lease.
//
// The implementation is split by concern: target.go (isolation, InitTarget,
// running commands), fetch.go (RemoteRefs, FetchBranch, DeepenSince,
// FetchBlobs, ReadBlob), log.go (FirstParentLog, Commit, DiffTree,
// IsCleanMerge, IsAncestor, Tree), attrs.go (Attrs, Renormalized),
// running commands), fetch.go (RemoteRefs, FetchBranch, DeepenSince, Deepen,
// ShallowNotBefore, FetchBlobs, ReadBlob), log.go (FirstParentLog, Commit,
// DiffTree, IsCleanMerge, IsAncestor, Tree), attrs.go (Attrs, Renormalized),
// build.go (BuildCommit), push.go (Push, ParsePush) and failure.go
// (ClassifyFailure, PushResult.Transient).
//
Expand All @@ -27,8 +27,9 @@ package gitx
//
// The proofs of ancestry (a merge's second parent, Hc's first parent) run
// after DeepenSince back to a day before the older of the commits to prove,
// and anything but a yes counts as no proof: decide.ClassifyBranch calls
// the branch edited then.
// and after Deepen below the boundaries ShallowNotBefore names, where a
// merge of an older branch cut the history short. Anything but a yes
// counts as no proof: decide.ClassifyBranch calls the branch edited then.

import (
"context"
Expand Down
85 changes: 85 additions & 0 deletions internal/gitx/fetch.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ import (
"context"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"slices"
"strconv"
"strings"
Expand Down Expand Up @@ -132,6 +135,88 @@ func (t *TargetRepo) DeepenSince(ctx context.Context, branch string, since time.
return err
}

// Deepen fetches by more commits of branch's history below the current
// shallow boundaries (--deepen), without blobs, and updates no ref, as
// DeepenSince does. It reaches the history DeepenSince leaves out: git
// makes a commit with a parent older than since a boundary with all of its
// parents, so a merge of an older branch cuts the default branch's own
// history short (ShallowNotBefore finds such boundaries). by must be
// positive.
func (t *TargetRepo) Deepen(ctx context.Context, branch string, by int) error {
if err := checkBranch(branch); err != nil {
return fmt.Errorf("git fetch: %w", err)
}
if by < 1 {
return fmt.Errorf("git fetch: deepen %d is not positive", by)
}
unlock, err := t.lockFetch()
if err != nil {
return err
}
defer unlock()
_, err = t.run(ctx, cmdOpts{network: true}, "fetch", "--quiet",
"--filter=blob:none", "--deepen="+strconv.Itoa(by),
"--no-tags", "--no-write-fetch-head", "--recurse-submodules=no",
remoteName, "refs/heads/"+branch)
if isMissingRemoteRef(err) {
return fmt.Errorf("git fetch: branch %s no longer exists: %w", branch, ErrNotFound)
}
return err
}

// ShallowNotBefore returns the shallow boundaries in rev's history whose
// committer date is not before since, sorted: after DeepenSince back to
// since, a commit that is a boundary for its own age is older than since,
// so these were cut for a parent's age, and the history beyond them is
// missing although it may be newer than since. A boundary whose committer
// date cannot be read is returned too. Boundaries outside rev's history
// (the bottom of another branch fetched to a depth) are left out. A
// repository that is not shallow has none.
func (t *TargetRepo) ShallowNotBefore(ctx context.Context, rev string, since time.Time) ([]string, error) {
if err := checkObjectName(rev); err != nil {
return nil, err
}
data, err := os.ReadFile(filepath.Join(t.Dir, "shallow"))
switch {
case errors.Is(err, fs.ErrNotExist):
return nil, nil
case err != nil:
return nil, fmt.Errorf("git target: %w", err)
}
var ids []string
for line := range strings.SplitSeq(string(data), "\n") {
if line == "" {
continue
}
if !isOID(line) {
return nil, fmt.Errorf("git target: the shallow file holds %q", abbrev(line))
}
ids = append(ids, line)
}
if len(ids) == 0 {
return nil, nil
}
commits, err := t.commits(ctx, ids)
if err != nil {
return nil, err
}
var out []string
for _, c := range commits {
if !c.Time.IsZero() && c.Time.Before(since) {
continue
}
in, err := t.IsAncestor(ctx, c.SHA, rev)
if err != nil {
return nil, err
}
if in {
out = append(out, c.SHA)
}
}
slices.Sort(out)
return out, nil
}

// FetchBlobs fetches blobs by id the way git's promisor fetch does:
// "-c fetch.negotiationAlgorithm=noop fetch origin --no-tags
// --no-write-fetch-head --recurse-submodules=no --filter=blob:none --stdin"
Expand Down
65 changes: 65 additions & 0 deletions internal/gitx/fetch_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,71 @@ func TestDeepenSince(t *testing.T) {
}
}

// A merge of a branch older than since is a shallow boundary with both of
// its parents after DeepenSince, so the default branch's own commits below
// it are missing; ShallowNotBefore names that boundary, not the bottom of
// another branch fetched to a depth, and Deepen fetches what is missing.
func TestDeepenOldMerge(t *testing.T) {
t.Parallel()
s := newServed(t, t.TempDir(), "repo.git")
day := int64(86400)
base := int64(1767225600)
at := func(d int64) string { return fmt.Sprintf("%d +0000", base+d*day) }
root := s.commitAt("main", nil, []file{{path: "a.txt", content: "v1\n"}}, "root\n", at(1))
old := s.commitAt("feature", []string{root}, []file{{path: "b.txt", content: "old\n"}}, "old work\n", at(2))
mine := s.commitAt("main", []string{root}, []file{{path: "a.txt", content: "v2\n"}}, "mine\n", at(10))
s.commitAt("sync", []string{mine}, []file{{path: "c.txt", content: "ours\n"}}, "ours\n", at(10))
other := s.commitAt("other", []string{root}, []file{{path: "d.txt", content: "o1\n"}}, "o1\n", at(10))
s.commitAt("other", []string{other}, []file{{path: "d.txt", content: "o2\n"}}, "o2\n", at(10))
merge := s.commitAt("main", []string{mine, old}, []file{{path: "b.txt", content: "old\n"}}, "merge\n", at(11))
tr := newTarget(t, s.dir, Auth{})
ctx := t.Context()
for _, f := range []struct {
branch string
depth int
}{{"main", 1}, {"sync", 2}, {"other", 1}} {
if _, _, err := tr.FetchBranch(ctx, f.branch, f.depth); err != nil {
t.Fatalf("FetchBranch(%s): %v", f.branch, err)
}
}
since := time.Unix(base+9*day, 0)
if err := tr.DeepenSince(ctx, "main", since); err != nil {
t.Fatal(err)
}
if ok, err := tr.IsAncestor(ctx, mine, merge); err != nil || ok {
t.Fatalf("IsAncestor(mine, merge) after DeepenSince = %v, %v; want false: git cuts the merge with both parents", ok, err)
}
if cut, err := tr.ShallowNotBefore(ctx, merge, since); err != nil || !slices.Equal(cut, []string{merge}) {
t.Fatalf("ShallowNotBefore(merge) = %v, %v; want the merge %s", cut, err, merge)
}
if err := tr.Deepen(ctx, "main", 1); err != nil {
t.Fatal(err)
}
if ok, err := tr.IsAncestor(ctx, mine, merge); err != nil || !ok {
t.Errorf("IsAncestor(mine, merge) after Deepen = %v, %v", ok, err)
}
if err := tr.Deepen(ctx, "gone", 1); !errors.Is(err, ErrNotFound) {
t.Errorf("Deepen(gone) = %v, want ErrNotFound", err)
}
if err := tr.Deepen(ctx, "main", 0); err == nil {
t.Error("Deepen(0) succeeded")
}
}

// A repository that is not shallow has no boundary to name.
func TestShallowNotBeforeFull(t *testing.T) {
t.Parallel()
s := newServed(t, t.TempDir(), "repo.git")
ids := s.chain("main", "", 2)
tr := newTarget(t, s.dir, Auth{})
if _, _, err := tr.FetchBranch(t.Context(), "main", 5); err != nil {
t.Fatal(err)
}
if cut, err := tr.ShallowNotBefore(t.Context(), ids[1], time.Unix(0, 0)); err != nil || cut != nil {
t.Errorf("ShallowNotBefore(full history) = %v, %v", cut, err)
}
}

func TestFetchBlobs(t *testing.T) {
t.Parallel()
root := t.TempDir()
Expand Down
Loading