From 5d652a024d29ea228a55fc00a7f0980d586a82cc Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 01:44:29 +0800 Subject: [PATCH 01/36] archives: node leases gate Take and the departed-owner sweep Add sandboxd_nodes(cell, node, lease_until) and Registry.Renew, one upsert per node on the database clock. Take moves another node's row only once its owner holds no live lease, in the same statement as the epoch compare-and-set, and Sweepable lists a foreign expired row only for an owner without a lease. Open adds the nodes table to a database that already has the archives table. --- sandboxd/archives/archives.go | 10 +- .../archives/archivestest/archivestest.go | 128 +++++++++++++++++- sandboxd/archives/pg/nodes.sql | 6 + sandboxd/archives/pg/pg.go | 38 +++++- sandboxd/archives/pg/pg_test.go | 28 ++++ sandboxd/metastore/metastore.go | 5 +- 6 files changed, 196 insertions(+), 19 deletions(-) create mode 100644 sandboxd/archives/pg/nodes.sql diff --git a/sandboxd/archives/archives.go b/sandboxd/archives/archives.go index 1738fd78..2f8c3408 100644 --- a/sandboxd/archives/archives.go +++ b/sandboxd/archives/archives.go @@ -1,4 +1,4 @@ -// Package archives keeps one cluster-wide row per archived claim, so another node can adopt the claim once its owner is gone. +// Package archives keeps one cluster-wide row per archived or replicated claim, so another node can adopt the claim once its owner's lease expires. package archives import ( @@ -11,7 +11,7 @@ import ( var ( // ErrUnavailable answers a call the registry cannot serve now; a retry may succeed. ErrUnavailable = errors.New("archive registry down") - // ErrLost answers a compare-and-set whose row is gone, at another epoch, consumed, or held by another owner. + // ErrLost answers a compare-and-set whose row is gone, at another epoch, consumed, or held by an owner whose lease is live. ErrLost = errors.New("archive row moved on") ) @@ -33,7 +33,7 @@ type Record struct { type Registry interface { // Put writes rec for rec.Owner as a live row at a new epoch and returns it; ErrLost when another owner holds the row. Put(ctx context.Context, rec Record) (int64, error) - // Take moves the live row at epoch expect to owner at a new epoch and returns it. + // Take moves the live row at epoch expect to owner at a new epoch and returns it; another node's row moves only once its owner's lease expired. Take(ctx context.Context, id string, expect int64, owner string) (int64, error) // Consume turns the live row at epoch expect into a tombstone at a new epoch and returns it. Consume(ctx context.Context, id string, expect int64) (int64, error) @@ -41,8 +41,10 @@ type Registry interface { Delete(ctx context.Context, id string, epoch int64) error // Get returns the row of id; false when there is none. Get(ctx context.Context, id string) (Record, bool, error) - // Sweepable returns the rows past their deadline, consumed, or held by owner, without their claims. + // Sweepable returns the rows held by owner, consumed, or past their deadline with an owner whose lease expired, without their claims. Sweepable(ctx context.Context, now time.Time, owner string) ([]Record, error) // Rows returns the rows of ids by id, without their claims. Rows(ctx context.Context, ids []string) (map[string]Record, error) + // Renew extends node's lease to ttl past the registry's clock. + Renew(ctx context.Context, node string, ttl time.Duration) error } diff --git a/sandboxd/archives/archivestest/archivestest.go b/sandboxd/archives/archivestest/archivestest.go index b622cb78..620c43f1 100644 --- a/sandboxd/archives/archivestest/archivestest.go +++ b/sandboxd/archives/archivestest/archivestest.go @@ -12,7 +12,12 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/archives" ) -const racers = 8 +const ( + racers = 8 + + shortLease = 300 * time.Millisecond + leaseWait = 10 * time.Second +) // Opener returns an empty registry; each call is a fresh table. type Opener func(t *testing.T) archives.Registry @@ -23,6 +28,7 @@ var _ archives.Registry = (*Mem)(nil) type Mem struct { mu sync.Mutex rows map[string]archives.Record + leases map[string]time.Time seq int64 down bool deletes int @@ -30,7 +36,7 @@ type Mem struct { // NewMem returns an empty in-memory registry. func NewMem() *Mem { - return &Mem{rows: map[string]archives.Record{}} + return &Mem{rows: map[string]archives.Record{}, leases: map[string]time.Time{}} } func (m *Mem) Put(_ context.Context, rec archives.Record) (int64, error) { @@ -49,11 +55,20 @@ func (m *Mem) Put(_ context.Context, rec archives.Record) (int64, error) { } func (m *Mem) Take(_ context.Context, id string, expect int64, owner string) (int64, error) { - return m.advance(id, expect, func(rec *archives.Record) { rec.Owner = owner }) + return m.advance(id, expect, func(rec *archives.Record) bool { + if rec.Owner != owner && m.leased(rec.Owner) { + return false + } + rec.Owner = owner + return true + }) } func (m *Mem) Consume(_ context.Context, id string, expect int64) (int64, error) { - return m.advance(id, expect, func(rec *archives.Record) { rec.Consumed = true }) + return m.advance(id, expect, func(rec *archives.Record) bool { + rec.Consumed = true + return true + }) } func (m *Mem) Delete(_ context.Context, id string, epoch int64) error { @@ -88,7 +103,7 @@ func (m *Mem) Sweepable(_ context.Context, now time.Time, owner string) ([]archi } var out []archives.Record for _, rec := range m.rows { - if rec.Consumed || rec.Owner == owner || !rec.Deadline.IsZero() && rec.Deadline.Before(now) { + if rec.Consumed || rec.Owner == owner || !rec.Deadline.IsZero() && rec.Deadline.Before(now) && !m.leased(rec.Owner) { rec.Claim = nil out = append(out, rec) } @@ -112,6 +127,23 @@ func (m *Mem) Rows(_ context.Context, ids []string) (map[string]archives.Record, return rows, nil } +func (m *Mem) Renew(_ context.Context, node string, ttl time.Duration) error { + m.mu.Lock() + defer m.mu.Unlock() + if m.down { + return archives.ErrUnavailable + } + m.leases[node] = time.Now().Add(ttl) + return nil +} + +// Expire ends node's lease now. +func (m *Mem) Expire(node string) { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.leases, node) +} + // SetDown makes every call answer ErrUnavailable while down is set. func (m *Mem) SetDown(down bool) { m.mu.Lock() @@ -126,7 +158,11 @@ func (m *Mem) Deletes() int { return m.deletes } -func (m *Mem) advance(id string, expect int64, change func(*archives.Record)) (int64, error) { +func (m *Mem) leased(node string) bool { + return time.Now().Before(m.leases[node]) +} + +func (m *Mem) advance(id string, expect int64, change func(*archives.Record) bool) (int64, error) { m.mu.Lock() defer m.mu.Unlock() if m.down { @@ -136,7 +172,9 @@ func (m *Mem) advance(id string, expect int64, change func(*archives.Record)) (i if !ok || rec.Epoch != expect || rec.Consumed { return 0, archives.ErrLost } - change(&rec) + if !change(&rec) { + return 0, archives.ErrLost + } m.seq++ rec.Epoch = m.seq m.rows[id] = rec @@ -154,6 +192,9 @@ func Run(t *testing.T, open Opener) { t.Run("epochs are never reused", func(t *testing.T) { testEpochsNeverRepeat(t, open(t)) }) t.Run("sweepable lists expired, consumed and own rows", func(t *testing.T) { testSweepable(t, open(t)) }) t.Run("rows lists the named ids", func(t *testing.T) { testRows(t, open(t)) }) + t.Run("take waits out a live lease", func(t *testing.T) { testTakeLease(t, open(t)) }) + t.Run("sweepable spares a live owner's expired row", func(t *testing.T) { testSweepableLease(t, open(t)) }) + t.Run("renew moves a lease forward", func(t *testing.T) { testRenewExtends(t, open(t)) }) } func testPutGet(t *testing.T, r archives.Registry) { @@ -353,6 +394,79 @@ func testRows(t *testing.T, r archives.Registry) { } } +func testTakeLease(t *testing.T, r archives.Registry) { + ctx := t.Context() + first := mustPut(t, r, record("sb_1", "a", time.Time{})) + if err := r.Renew(ctx, "a", shortLease); err != nil { + t.Fatalf("renew: %v", err) + } + if _, err := r.Take(ctx, "sb_1", first, "b"); !errors.Is(err, archives.ErrLost) { + t.Fatalf("take while the owner's lease is live: %v, want ErrLost", err) + } + if got, _, _ := r.Get(ctx, "sb_1"); got.Owner != "a" || got.Epoch != first { + t.Fatalf("row after a refused take: %s@%d, want a@%d", got.Owner, got.Epoch, first) + } + own, err := r.Take(ctx, "sb_1", first, "a") + if err != nil || own <= first { + t.Fatalf("the owner's own take with a live lease: %d %v, want a new epoch", own, err) + } + eventually(t, "the owner's lease to expire", func() bool { + _, err = r.Take(ctx, "sb_1", own, "b") + return err == nil + }) + if got, _, _ := r.Get(ctx, "sb_1"); got.Owner != "b" { + t.Errorf("row owner after the expired-lease take: %s, want b", got.Owner) + } + never := mustPut(t, r, record("sb_2", "c", time.Time{})) + if _, err = r.Take(ctx, "sb_2", never, "b"); err != nil { + t.Errorf("take from a node that never held a lease: %v, want its row adoptable", err) + } +} + +func testSweepableLease(t *testing.T, r archives.Registry) { + ctx := t.Context() + past := time.Now().Add(-time.Minute) + mustPut(t, r, record("sb_live", "a", past)) + mustPut(t, r, record("sb_dead", "z", past)) + if err := r.Renew(ctx, "a", shortLease); err != nil { + t.Fatalf("renew: %v", err) + } + got, err := r.Sweepable(ctx, time.Now(), "me") + if err != nil || len(got) != 1 || got[0].ID != "sb_dead" { + t.Fatalf("sweepable while a's lease is live: %+v %v, want sb_dead only", got, err) + } + eventually(t, "a's lease to expire", func() bool { + got, err = r.Sweepable(ctx, time.Now(), "me") + return err == nil && len(got) == 2 + }) +} + +func testRenewExtends(t *testing.T, r archives.Registry) { + ctx := t.Context() + first := mustPut(t, r, record("sb_1", "a", time.Time{})) + if err := r.Renew(ctx, "a", shortLease); err != nil { + t.Fatalf("renew: %v", err) + } + if err := r.Renew(ctx, "a", time.Hour); err != nil { + t.Fatalf("renew again: %v", err) + } + time.Sleep(2 * shortLease) + if _, err := r.Take(ctx, "sb_1", first, "b"); !errors.Is(err, archives.ErrLost) { + t.Errorf("take after the first ttl but inside the renewed one: %v, want ErrLost", err) + } +} + +func eventually(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(leaseWait) + for !cond() { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %s", what) + } + time.Sleep(shortLease / 10) + } +} + func mustPut(t *testing.T, r archives.Registry, rec archives.Record) int64 { t.Helper() epoch, err := r.Put(t.Context(), rec) diff --git a/sandboxd/archives/pg/nodes.sql b/sandboxd/archives/pg/nodes.sql new file mode 100644 index 00000000..1f1da61c --- /dev/null +++ b/sandboxd/archives/pg/nodes.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS sandboxd_nodes ( + cell text NOT NULL, + node text NOT NULL, + lease_until timestamptz NOT NULL, + PRIMARY KEY (cell, node) +); diff --git a/sandboxd/archives/pg/pg.go b/sandboxd/archives/pg/pg.go index 01d8fd1f..173cbb15 100644 --- a/sandboxd/archives/pg/pg.go +++ b/sandboxd/archives/pg/pg.go @@ -15,10 +15,20 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/metastore" ) -const table = "sandboxd_archives" +const ( + table = "sandboxd_archives" + nodesTable = "sandboxd_nodes" -//go:embed schema.sql -var schema string + leaseExpired = `NOT EXISTS (SELECT 1 FROM sandboxd_nodes n WHERE n.cell = sandboxd_archives.cell AND n.node = sandboxd_archives.owner AND n.lease_until > now())` +) + +var ( + //go:embed schema.sql + schema string + + //go:embed nodes.sql + nodesSchema string +) var _ archives.Registry = (*Registry)(nil) @@ -28,12 +38,16 @@ type Registry struct { cell string } -// Open connects to cell's rows and creates the table when it is missing. +// Open connects to cell's rows and creates the tables when they are missing. func Open(ctx context.Context, dsn, cell string) (*Registry, error) { pool, _, err := metastore.Open(ctx, dsn, table, schema) if err != nil { return nil, err } + if err = metastore.EnsureTable(ctx, pool, nodesTable, nodesSchema); err != nil { + pool.Close() + return nil, err + } return &Registry{pool: pool, cell: cell}, nil } @@ -60,7 +74,7 @@ func (r *Registry) Put(ctx context.Context, rec archives.Record) (int64, error) func (r *Registry) Take(ctx context.Context, id string, expect int64, owner string) (int64, error) { return r.advance(ctx, `UPDATE sandboxd_archives SET owner = $4, epoch = nextval('sandboxd_archives_epoch'), updated_at = now() - WHERE cell = $1 AND id = $2 AND epoch = $3 AND NOT consumed + WHERE cell = $1 AND id = $2 AND epoch = $3 AND NOT consumed AND (owner = $4 OR `+leaseExpired+`) RETURNING epoch`, r.cell, id, expect, owner) } @@ -99,7 +113,7 @@ func (r *Registry) Get(ctx context.Context, id string) (archives.Record, bool, e func (r *Registry) Sweepable(ctx context.Context, now time.Time, owner string) ([]archives.Record, error) { return r.collect(ctx, `SELECT id, tenant, token_sha256, owner, epoch, consumed, checkpoint, deadline, NULL::jsonb - FROM sandboxd_archives WHERE cell = $2 AND (consumed OR deadline < $1 OR owner = $3)`, now, r.cell, owner) + FROM sandboxd_archives WHERE cell = $2 AND (consumed OR owner = $3 OR deadline < $1 AND `+leaseExpired+`)`, now, r.cell, owner) } func (r *Registry) Rows(ctx context.Context, ids []string) (map[string]archives.Record, error) { @@ -115,6 +129,18 @@ func (r *Registry) Rows(ctx context.Context, ids []string) (map[string]archives. return rows, nil } +func (r *Registry) Renew(ctx context.Context, node string, ttl time.Duration) error { + qctx, cancel := context.WithTimeout(ctx, metastore.QueryTimeout) + defer cancel() + _, err := r.pool.Exec(qctx, `INSERT INTO sandboxd_nodes (cell, node, lease_until) + VALUES ($1, $2, now() + $3::float8 * interval '1 second') + ON CONFLICT (cell, node) DO UPDATE SET lease_until = EXCLUDED.lease_until`, r.cell, node, ttl.Seconds()) + if err != nil { + return unavailable(err) + } + return nil +} + // Close releases the connection pool. func (r *Registry) Close() { r.pool.Close() diff --git a/sandboxd/archives/pg/pg_test.go b/sandboxd/archives/pg/pg_test.go index d3766331..d07a9de9 100644 --- a/sandboxd/archives/pg/pg_test.go +++ b/sandboxd/archives/pg/pg_test.go @@ -95,6 +95,34 @@ func TestADownDatabaseIsUnavailable(t *testing.T) { } } +func TestOpenAddsTheNodesTableToAnArchivesOnlyDatabase(t *testing.T) { + dsn := metastoretest.PGSchema(t) + first := open(t, dsn, "c1") + if _, err := first.pool.Exec(t.Context(), `DROP TABLE sandboxd_nodes`); err != nil { + t.Fatalf("drop nodes: %v", err) + } + second := open(t, dsn, "c1") + if err := second.Renew(t.Context(), "a", time.Minute); err != nil { + t.Errorf("renew on a database that predates the nodes table: %v", err) + } +} + +func TestLeasesAreScopedToTheirCell(t *testing.T) { + dsn := metastoretest.PGSchema(t) + c1, c2 := open(t, dsn, "c1"), open(t, dsn, "c2") + ctx := t.Context() + if err := c1.Renew(ctx, "a", time.Hour); err != nil { + t.Fatalf("renew in c1: %v", err) + } + epoch, err := c2.Put(ctx, archives.Record{ID: "sb_1", TokenSHA256: "ab", Owner: "a", Checkpoint: "ck_1", Claim: []byte(`{}`)}) + if err != nil { + t.Fatalf("put in c2: %v", err) + } + if _, err = c2.Take(ctx, "sb_1", epoch, "b"); err != nil { + t.Errorf("c2 take of a row whose owner holds a lease only in c1: %v, want it taken", err) + } +} + func open(t *testing.T, dsn, cell string) *Registry { t.Helper() r, err := Open(t.Context(), dsn, cell) diff --git a/sandboxd/metastore/metastore.go b/sandboxd/metastore/metastore.go index 6697a883..fee99631 100644 --- a/sandboxd/metastore/metastore.go +++ b/sandboxd/metastore/metastore.go @@ -38,7 +38,7 @@ func Open(ctx context.Context, dsn, table, schema string) (*pgxpool.Pool, *pgx.C if err != nil { return nil, nil, fmt.Errorf("open meta store: %w", err) } - if err = ensureTable(ctx, pool, table, schema); err != nil { + if err = EnsureTable(ctx, pool, table, schema); err != nil { pool.Close() return nil, nil, err } @@ -86,7 +86,8 @@ func Listen(ctx context.Context, cfg *pgx.ConnConfig, channel string, synced fun } } -func ensureTable(ctx context.Context, pool *pgxpool.Pool, table, schema string) error { +// EnsureTable runs schema only when table is missing. +func EnsureTable(ctx context.Context, pool *pgxpool.Pool, table, schema string) error { qctx, cancel := context.WithTimeout(ctx, QueryTimeout) defer cancel() var exists bool From 0d199367aa3c01947d89dfc34be12d35f42811e4 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 01:54:15 +0800 Subject: [PATCH 02/36] config: replicate_seconds pool setting Config load rejects it unless archive_failover is on and the pool is on the none lane. It is config-owned like storage: PUT /v1/pools refuses it, the pool seed hash ignores it, and a reload reports it as changed. --- sandboxd/config/config.go | 18 +++++++++++++++++- sandboxd/config/config_test.go | 23 +++++++++++++++++++++++ sandboxd/config/reload.go | 1 + sandboxd/config/reload_test.go | 3 ++- sandboxd/pool/pool_test.go | 1 + sandboxd/pool/poolstore.go | 3 ++- sandboxd/pool/poolstore_test.go | 1 + sandboxd/pool/setpools.go | 3 +++ 8 files changed, 50 insertions(+), 3 deletions(-) diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index fdbad5b8..684a61bb 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -51,6 +51,8 @@ type PoolSpec struct { CaptureTrim bool `json:"capture_trim,omitzero"` // Storage sizes the golden's copy-on-write disk in cocoon's --storage spelling; every clone inherits it, and empty keeps cocoon's default. Storage string `json:"storage,omitempty"` + // ReplicateSeconds snapshots each running claim of the pool to the archive registry at this interval; zero turns replication off. + ReplicateSeconds int `json:"replicate_seconds,omitzero"` IdleHibernateSeconds int `json:"idle_hibernate_seconds,omitzero"` ArchiveAfterSeconds int `json:"archive_after_seconds,omitzero"` @@ -364,7 +366,7 @@ func (c *Config) validate() error { if c.CheckpointPeerHeal && c.APIToken == "" { return fmt.Errorf("checkpoint_peer_heal requires api_token: without it resolveScope leaves the raw checkpoint blob GET reachable with no credential") } - for _, check := range []func() error{c.validateNodeEgress, c.validateDiskReserve, c.validateMetaStore, c.validateVolumes, c.validateMesh} { + for _, check := range []func() error{c.validateNodeEgress, c.validateDiskReserve, c.validateMetaStore, c.validateVolumes, c.validateMesh, c.validateReplication} { if err := check(); err != nil { return err } @@ -376,6 +378,20 @@ func (c *Config) validate() error { return c.validateEgress(secrets) } +func (c *Config) validateReplication() error { + for _, p := range c.Pools { + switch { + case p.ReplicateSeconds < 0: + return fmt.Errorf("pool %q: replicate_seconds must not be negative", p.Template) + case p.ReplicateSeconds > 0 && !c.ArchiveFailover: + return fmt.Errorf("pool %q: replicate_seconds requires archive_failover", p.Template) + case p.ReplicateSeconds > 0 && p.Net != types.NetNone: + return fmt.Errorf("pool %q: replicate_seconds needs net %q: only the none lane captures", p.Template, types.NetNone) + } + } + return nil +} + func (c *Config) validateVolumes() error { names := make(map[string]struct{}, len(c.Volumes)) paths := make(map[string]string, len(c.Volumes)) diff --git a/sandboxd/config/config_test.go b/sandboxd/config/config_test.go index dbc93806..16b385a7 100644 --- a/sandboxd/config/config_test.go +++ b/sandboxd/config/config_test.go @@ -227,6 +227,29 @@ func TestLoadArchiveFailover(t *testing.T) { } } +func TestLoadReplicateSeconds(t *testing.T) { + t.Setenv("SANDBOX_TEST_DSN", "postgres://sandboxd@db/sandboxd") + failover := `"archive_failover":true,"meta_store":{"kind":"pg","dsn_env":"SANDBOX_TEST_DSN","cell":"c1"},"checkpoint_store":{"kind":"s3","s3":{"bucket":"b"}},"mesh":{"bind":"node1:7946"},` + body := func(on, pool string) string { + return `{"api_token":"r","advertise_addr":"node1:7777",` + on + `"pools":[` + pool + `]}` + } + cfg, err := Load(writeConfig(t, body(failover, `{"template":"rt:24.04","replicate_seconds":60}`))) + if err != nil || cfg.Pools[0].ReplicateSeconds != 60 { + t.Fatalf("Load: %+v %v, want replicate_seconds 60", cfg, err) + } + for name, tt := range map[string]struct{ on, pool, want string }{ + "no failover": {"", `{"template":"rt:24.04","replicate_seconds":60}`, "requires archive_failover"}, + "negative": {failover, `{"template":"rt:24.04","replicate_seconds":-1}`, "must not be negative"}, + "egress lane": {failover + `"bridges":["br0"],`, `{"template":"rt:24.04","net":"egress","replicate_seconds":60}`, "needs net"}, + "off is valid": {"", `{"template":"rt:24.04","replicate_seconds":0}`, ""}, + } { + _, err := Load(writeConfig(t, body(tt.on, tt.pool))) + if tt.want == "" && err != nil || tt.want != "" && (err == nil || !strings.Contains(err.Error(), tt.want)) { + t.Errorf("%s: %v, want %q", name, err, tt.want) + } + } +} + func TestLoadEgressUpstream(t *testing.T) { t.Setenv("SANDBOX_TEST_RES", "http://user:pw@res.example:3128") t.Setenv("SANDBOX_TEST_DC", "socks5://10.1.0.5:1080") diff --git a/sandboxd/config/reload.go b/sandboxd/config/reload.go index 2896085f..cf147dbc 100644 --- a/sandboxd/config/reload.go +++ b/sandboxd/config/reload.go @@ -62,6 +62,7 @@ func diffPools(cur, next []PoolSpec) (changed, ignored []string) { {"warmup", slices.Equal(x.Warmup, y.Warmup)}, {"capture_trim", x.CaptureTrim == y.CaptureTrim}, {"storage", x.Storage == y.Storage}, + {"replicate_seconds", x.ReplicateSeconds == y.ReplicateSeconds}, {"egress_upstream_env", x.EgressUpstreamEnv == y.EgressUpstreamEnv}, } { if !f.same { diff --git a/sandboxd/config/reload_test.go b/sandboxd/config/reload_test.go index f9cfe930..550a9d83 100644 --- a/sandboxd/config/reload_test.go +++ b/sandboxd/config/reload_test.go @@ -19,7 +19,7 @@ func TestReloadDiffSortsReloadableOwnedAndRestartFields(t *testing.T) { next.EgressInternalAllow = []string{"10.8.0.0/16"} next.Pools = []PoolSpec{ {PoolKey: key, Warm: 5, Egress: &egress.Policy{Allow: []egress.Rule{{Host: "pypi.org"}}}}, - {PoolKey: v13, Warmup: []string{"node", "-e", "0"}, Storage: "40G"}, + {PoolKey: v13, Warmup: []string{"node", "-e", "0"}, Storage: "40G", ReplicateSeconds: 60}, } next.EgressClasses = []EgressClass{{Name: "desk", Egress: &egress.Policy{}}} next.EgressConnRate = &EgressConnRate{PerSecond: 30, Burst: 600} @@ -36,6 +36,7 @@ func TestReloadDiffSortsReloadableOwnedAndRestartFields(t *testing.T) { "pools[desktop:v12 none small].egress", "pools[desktop:v13 none small].warmup", "pools[desktop:v13 none small].storage", + "pools[desktop:v13 none small].replicate_seconds", } if !slices.Equal(changed, wantChanged) { t.Errorf("changed %v, want %v", changed, wantChanged) diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index 29155c93..871b42a0 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -622,6 +622,7 @@ func TestSetPoolsRejectsConfigOwnedFields(t *testing.T) { {"warmup", config.PoolSpec{PoolKey: testKey, Warm: 1, Warmup: []string{"true"}}}, {"capture_trim", config.PoolSpec{PoolKey: testKey, Warm: 1, CaptureTrim: true}}, {"storage", config.PoolSpec{PoolKey: testKey, Warm: 1, Storage: "40G"}}, + {"replicate_seconds", config.PoolSpec{PoolKey: testKey, Warm: 1, ReplicateSeconds: 60}}, } { err := m.SetPools(t.Context(), []config.PoolSpec{tt.spec}) if !errors.Is(err, ErrBadKey) || !strings.Contains(err.Error(), tt.field+" is set in the config file") { diff --git a/sandboxd/pool/poolstore.go b/sandboxd/pool/poolstore.go index 3a1488aa..f677a689 100644 --- a/sandboxd/pool/poolstore.go +++ b/sandboxd/pool/poolstore.go @@ -117,7 +117,7 @@ func (m *Manager) adoptPools(ctx context.Context) error { return nil } -// poolSeedHash digests a pool set's warm-target shape, order-independent, without the config-owned egress, warmup, capture_trim, storage and egress_upstream_env. +// poolSeedHash digests a pool set's warm-target shape, order-independent, without the config-owned egress, warmup, capture_trim, storage, replicate_seconds and egress_upstream_env. func poolSeedHash(specs []config.PoolSpec) string { shaped := slices.Clone(specs) for i := range shaped { @@ -125,6 +125,7 @@ func poolSeedHash(specs []config.PoolSpec) string { shaped[i].Warmup = nil shaped[i].CaptureTrim = false shaped[i].Storage = "" + shaped[i].ReplicateSeconds = 0 shaped[i].EgressUpstreamEnv = "" } slices.SortFunc(shaped, func(a, b config.PoolSpec) int { return a.Compare(b.PoolKey) }) diff --git a/sandboxd/pool/poolstore_test.go b/sandboxd/pool/poolstore_test.go index 51afaab7..8f89c598 100644 --- a/sandboxd/pool/poolstore_test.go +++ b/sandboxd/pool/poolstore_test.go @@ -87,6 +87,7 @@ func TestPoolSeedHashIgnoresConfigOwnedFields(t *testing.T) { "warmup": func(s *config.PoolSpec) { s.Warmup = []string{"true"} }, "capture_trim": func(s *config.PoolSpec) { s.CaptureTrim = true }, "storage": func(s *config.PoolSpec) { s.Storage = "20G" }, + "replicate_seconds": func(s *config.PoolSpec) { s.ReplicateSeconds = 60 }, "egress_upstream_env": func(s *config.PoolSpec) { s.EgressUpstreamEnv = "POOL_UPSTREAM" }, } { spec := base diff --git a/sandboxd/pool/setpools.go b/sandboxd/pool/setpools.go index cfe5178c..f994cdd6 100644 --- a/sandboxd/pool/setpools.go +++ b/sandboxd/pool/setpools.go @@ -53,6 +53,9 @@ func (m *Manager) desiredPools(specs []config.PoolSpec) (map[types.PoolKey]confi if spec.Storage != "" { return nil, fmt.Errorf("%w: pool %q: storage is set in the config file, not via the API", ErrBadKey, spec.Template) } + if spec.ReplicateSeconds != 0 { + return nil, fmt.Errorf("%w: pool %q: replicate_seconds is set in the config file, not via the API", ErrBadKey, spec.Template) + } if spec.EgressUpstreamEnv != "" { return nil, fmt.Errorf("%w: pool %q: egress_upstream_env is set in the config file, not via the API", ErrBadKey, spec.Template) } From bb52fe6abb7051ecd2a3230a4db6d95ee69916c3 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 02:12:23 +0800 Subject: [PATCH 03/36] pool: replicate running claims behind a node lease A pool with replicate_seconds snapshots each running claim through the existing capture path (pause, memory and disk, resume, publish) as a hidden replica record, puts the claim's registry row on it, then deletes the previous replica. A failed round logs a warning and retries next interval. Rounds run on the refill/maintenance budget and start none while the node is fenced. Adoption now rests on the node lease alone. Take refuses another node's row while its owner holds a live lease, in the same statement as the epoch compare-and-set, so the memberlist ownerAlive gate, WithArchiveOwners and mesh.Alive are gone. The adopter restores the claim from the replica through the archive wake, memory included. Each node renews its lease every 5 s for 15 s. A lease outlives a missed renewal, and the fence trips 3 s before it ends (12 s after the last renewal's start), so one lost renewal never fences while two do before any adopter may take the row. A fenced node answers 503 "lease lost; retry" on the relay, exec, ports and preview of its replicated claims and destroys nothing; a node starts fenced. The first successful renewal reads the rows of its replicated claims: rows still its own are served again, rows adopted elsewhere or consumed by a finished adopter drop the claim and remove its VM. Lifecycle: release consumes the row, then deletes replica and tombstone; hibernate leaves the row on the last replica; archive puts over the row and deletes the replica (its rollback now restores epoch 0, since the abandon deletes the row); a lease expiry drops row and replica before the reap destroys the claim; a restart keeps its own live row, takes the checkpoint a crashed round left on it, and drops a claim whose row another node holds or a finished adopter consumed; turning replicate_seconds off retires the rows. dropMoved is folded into dropAdopted, which now removes the local VM. Hot path: the claim path and data plane gain one atomic load. Per node, one lease upsert every 5 s and one in-memory scan of the claims every 5 s; per replicated claim per interval, one capture pause, upload, row read, row write and delete. --- e2e/wire_parity_test.go | 2 +- sandboxd/main.go | 1 - sandboxd/mesh/mesh.go | 8 - sandboxd/mesh/mesh_test.go | 15 - sandboxd/pool/archive.go | 9 +- sandboxd/pool/checkpoint.go | 3 + sandboxd/pool/claim.go | 8 +- sandboxd/pool/claims.go | 3 +- sandboxd/pool/failover.go | 82 ++--- sandboxd/pool/failover_test.go | 111 ++++--- sandboxd/pool/hibernate.go | 3 + sandboxd/pool/lease.go | 85 +++++ sandboxd/pool/lease_test.go | 168 ++++++++++ sandboxd/pool/pool.go | 17 +- sandboxd/pool/pool_test.go | 19 +- sandboxd/pool/reconcile.go | 2 +- sandboxd/pool/replica.go | 170 ++++++++++ sandboxd/pool/replica_test.go | 546 ++++++++++++++++++++++++++++++++ sandboxd/pool/view.go | 27 +- sandboxd/server/port_test.go | 1 + sandboxd/server/preview.go | 6 + sandboxd/server/preview_test.go | 18 ++ sandboxd/server/server.go | 1 + sandboxd/types/types.go | 2 + 24 files changed, 1171 insertions(+), 136 deletions(-) create mode 100644 sandboxd/pool/lease.go create mode 100644 sandboxd/pool/lease_test.go create mode 100644 sandboxd/pool/replica.go create mode 100644 sandboxd/pool/replica_test.go diff --git a/e2e/wire_parity_test.go b/e2e/wire_parity_test.go index 84312e84..bf1a6b10 100644 --- a/e2e/wire_parity_test.go +++ b/e2e/wire_parity_test.go @@ -59,7 +59,7 @@ var wirePairs = []struct { {"SandboxSummary", false, pool.SandboxSummary{}, []string{"token"}}, {"sandboxListResponse", false, server.SandboxListResponse{}, nil}, {"poolUpdate", true, server.PoolUpdateRequest{}, nil}, - {"PoolSpec", true, config.PoolSpec{}, []string{"egress", "warmup", "egress_upstream_env", "capture_trim", "storage", "net", "size"}}, + {"PoolSpec", true, config.PoolSpec{}, []string{"egress", "warmup", "egress_upstream_env", "capture_trim", "storage", "replicate_seconds", "net", "size"}}, {"tenantsUpdate", true, server.TenantsRequest{}, nil}, {"tenantUpdate", true, server.TenantRequest{}, nil}, {"TenantSpec", true, reflect.Zero(reflect.TypeOf(server.TenantsRequest{}.Tenants).Elem()).Interface(), []string{"token"}}, diff --git a/sandboxd/main.go b/sandboxd/main.go index dc410e9c..1c6298a9 100644 --- a/sandboxd/main.go +++ b/sandboxd/main.go @@ -130,7 +130,6 @@ func main() { broadcaster := &peer.Broadcaster{Peers: msh.PeerAddrs, Token: cfg.APIToken} mgr.WithPeerDelete(broadcaster.Delete) go gossipNodeState(ctx, msh, mgr) - mgr.WithArchiveOwners(msh.Alive) logger.Infof(ctx, "mesh %s joined (%d seeds)", cfg.Mesh.ID(), len(cfg.Mesh.Join)) } diff --git a/sandboxd/mesh/mesh.go b/sandboxd/mesh/mesh.go index 9e05e7ce..73957334 100644 --- a/sandboxd/mesh/mesh.go +++ b/sandboxd/mesh/mesh.go @@ -191,14 +191,6 @@ func (m *Mesh) Members() []NodeState { return slices.Collect(maps.Values(m.view)) } -// Alive reports whether SWIM counts nodeID as a live member. -func (m *Mesh) Alive(nodeID string) bool { - m.mu.Lock() - defer m.mu.Unlock() - _, ok := m.live[nodeID] - return ok -} - // PeerAddrs returns the data-plane addresses of the other nodes. func (m *Mesh) PeerAddrs() []string { m.mu.Lock() diff --git a/sandboxd/mesh/mesh_test.go b/sandboxd/mesh/mesh_test.go index f1579a4d..a82548a8 100644 --- a/sandboxd/mesh/mesh_test.go +++ b/sandboxd/mesh/mesh_test.go @@ -119,21 +119,6 @@ func TestForgottenNodeStaysGoneUntilItRejoins(t *testing.T) { } } -func TestAliveFollowsMembership(t *testing.T) { - m := newTestMesh(t, "a") - if m.Alive("b") { - t.Fatal("a node never admitted is alive") - } - m.admit("b") - if !m.Alive("b") { - t.Error("an admitted node is not alive") - } - m.forget("b") - if m.Alive("b") { - t.Error("a forgotten node is still alive") - } -} - func TestCandidatesPowerOfTwo(t *testing.T) { m := newTestMesh(t, "a") mergeStates(t, m, []NodeState{ diff --git a/sandboxd/pool/archive.go b/sandboxd/pool/archive.go index c0ad60a5..9d092bf1 100644 --- a/sandboxd/pool/archive.go +++ b/sandboxd/pool/archive.go @@ -144,8 +144,8 @@ func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error { return errWokeMeanwhile } } - snap, vmName, sock, prevDeadline, prevEpoch, prevPending := sb.HibernateSnap, sb.VMName, sb.VsockSocket, sb.Deadline, sb.ArchiveEpoch, sb.ArchivePending - sb.ArchiveCk, sb.ArchiveEpoch, sb.ArchivePending = ck.ID, epoch, "" + snap, vmName, sock, prevDeadline, prevPending, replica := sb.HibernateSnap, sb.VMName, sb.VsockSocket, sb.Deadline, sb.ArchivePending, sb.ReplicaCk + sb.ArchiveCk, sb.ArchiveEpoch, sb.ArchivePending, sb.ReplicaCk = ck.ID, epoch, "", "" sb.HibernateSnap, sb.VsockSocket, sb.VMName = "", "", "" sb.Deadline = m.retentionDeadline(sb.Key) js := m.store.set(sb) @@ -156,7 +156,7 @@ func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error { var rb claimSnapshot // a Release that landed meanwhile already deleted the claim; re-setting it would resurrect it if m.claimed[sb.ID] == sb { - sb.ArchiveCk, sb.ArchiveEpoch, sb.ArchivePending = "", prevEpoch, prevPending + sb.ArchiveCk, sb.ArchiveEpoch, sb.ArchivePending, sb.ReplicaCk = "", 0, prevPending, replica sb.HibernateSnap, sb.VsockSocket, sb.VMName = snap, sock, vmName sb.Deadline = prevDeadline rb = m.store.set(sb) @@ -173,6 +173,9 @@ func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error { // the store ck is authoritative from here m.destroy(ctx, vmName) m.dropSnap(ctx, snap) + if replica != "" { + m.deleteOrphanArchiveCk(ctx, replica) + } m.counters.archives.Add(1) m.recordUsage(ctx, usageEvent{Event: "archive", ID: sb.ID, Reference: ck.ID, Tenant: sb.Tenant}) return nil diff --git a/sandboxd/pool/checkpoint.go b/sandboxd/pool/checkpoint.go index 9234f71e..a53f3cd6 100644 --- a/sandboxd/pool/checkpoint.go +++ b/sandboxd/pool/checkpoint.go @@ -320,6 +320,9 @@ func (m *Manager) pinnedArchiveCks() map[string]struct{} { if sb.ArchiveCk != "" { pinned[sb.ArchiveCk] = struct{}{} } + if sb.ReplicaCk != "" { + pinned[sb.ReplicaCk] = struct{}{} + } } return pinned } diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index 4a4588e2..11dc0add 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -226,6 +226,9 @@ func (m *Manager) dialPort(ctx context.Context, id string, cred Cred, port uint1 if !ok { return nil, ErrUnknownSandbox } + if m.leaseFenced(sb) { + return nil, ErrLeaseLost + } sb.Touch() m.recordAudit(ctx, id, auditFrame{Op: op, Port: port}) sb.Hold() @@ -247,6 +250,9 @@ func (m *Manager) dialPassive(ctx context.Context, id string, port uint16) (net. if !ok { return nil, ErrUnknownSandbox } + if m.leaseFenced(sb) { + return nil, ErrLeaseLost + } if !sb.Transition.TryLock() { return nil, ErrPaused } @@ -530,7 +536,7 @@ func (m *Manager) reapOnce(ctx context.Context) { continue } switch { - case m.archiveRowed(sb): + case m.expiryDropsRow(sb): if _, busy := m.archiving[id]; !busy { m.archiving[id] = struct{}{} unrows = append(unrows, sb) diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index cad3dbdf..f97a1767 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -57,6 +57,7 @@ type claimDTO struct { ArchiveCk string `json:"archive_ck,omitempty"` ArchiveEpoch int64 `json:"archive_epoch,omitzero"` ArchivePending string `json:"archive_pending,omitempty"` + ReplicaCk string `json:"replica_ck,omitempty"` FromCheckpoint string `json:"from_checkpoint,omitempty"` Restarts int `json:"restarts,omitzero"` RestartedAt time.Time `json:"restarted_at,omitzero"` @@ -264,7 +265,7 @@ func dtoOf(sb *types.Sandbox) claimDTO { Tenant: sb.Tenant, EgressClass: sb.EgressClass, ClaimRef: sb.ClaimRef, Metadata: sb.Metadata, OnExpire: sb.OnExpire, Volumes: slices.Clone(sb.Volumes), VolumesWant: volumesWantDTO(sb), Env: sb.Env, VsockSocket: sb.VsockSocket, TAP: sb.TAP, HibernateSnap: sb.HibernateSnap, PendingSnap: sb.PendingSnap, - ArchiveCk: sb.ArchiveCk, ArchiveEpoch: sb.ArchiveEpoch, ArchivePending: sb.ArchivePending, FromCheckpoint: sb.FromCheckpoint, + ArchiveCk: sb.ArchiveCk, ArchiveEpoch: sb.ArchiveEpoch, ArchivePending: sb.ArchivePending, ReplicaCk: sb.ReplicaCk, FromCheckpoint: sb.FromCheckpoint, Restarts: sb.Restarts, RestartedAt: sb.RestartedAt, Failed: sb.Failed, } } diff --git a/sandboxd/pool/failover.go b/sandboxd/pool/failover.go index 1644a620..98f918c9 100644 --- a/sandboxd/pool/failover.go +++ b/sandboxd/pool/failover.go @@ -1,6 +1,7 @@ package pool import ( + "cmp" "context" "crypto/subtle" "encoding/json/v2" @@ -18,15 +19,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// OwnerAlive reports whether the mesh counts node as a live member. -type OwnerAlive func(node string) bool - -// WithArchiveOwners wires the mesh view that archive adoption and the departed-owner sweep consult. -func (m *Manager) WithArchiveOwners(alive OwnerAlive) { - m.ownerAlive.Store(&alive) -} - -// AdoptArchived takes over an archived claim whose owner left the mesh and reports whether this node owns it now. +// AdoptArchived takes over a claim whose owner's lease expired and reports whether this node owns it now. func (m *Manager) AdoptArchived(ctx context.Context, id, token string) (bool, error) { if m.archives == nil { return false, nil @@ -48,8 +41,7 @@ func (m *Manager) adoptArchived(ctx context.Context, id, token string) (bool, er return false, err } sum := tenants.TokenSHA256(token) - if rec.Consumed || subtle.ConstantTimeCompare([]byte(sum), []byte(rec.TokenSHA256)) != 1 || (!rec.Deadline.IsZero() && time.Now().After(rec.Deadline)) || - (rec.Owner != m.nodeID && !m.ownerGone(rec.Owner)) { + if rec.Consumed || subtle.ConstantTimeCompare([]byte(sum), []byte(rec.TokenSHA256)) != 1 || (!rec.Deadline.IsZero() && time.Now().After(rec.Deadline)) { return false, nil } sb := &types.Sandbox{} @@ -110,7 +102,7 @@ func (m *Manager) putArchive(ctx context.Context, sb *types.Sandbox, ck string) deadline = m.retentionDeadline(sb.Key) } m.mu.Unlock() - dto.Token, dto.VMName, dto.VsockSocket, dto.HibernateSnap, dto.PendingSnap, dto.ArchivePending = "", "", "", "", "", "" + dto.Token, dto.VMName, dto.VsockSocket, dto.HibernateSnap, dto.PendingSnap, dto.ArchivePending, dto.ReplicaCk = "", "", "", "", "", "", "" dto.ArchiveCk, dto.Deadline = ck, deadline claim, err := json.Marshal(&dto) if err != nil { @@ -244,7 +236,7 @@ func (m *Manager) reputArchive(ctx context.Context, sb *types.Sandbox, epoch int if epoch == 0 { return } - epoch, err := m.putArchive(ctx, sb, sb.ArchiveCk) + epoch, err := m.putArchive(ctx, sb, cmp.Or(sb.ArchiveCk, sb.ReplicaCk)) if err != nil { log.WithFunc("pool.reputArchive").Warnf(ctx, "re-put archive row %s: %v", sb.ID, err) return @@ -258,6 +250,7 @@ func (m *Manager) releaseRowed(ctx context.Context, id string, sb *types.Sandbox if err := m.resolveIntent(context.WithoutCancel(ctx), sb); err != nil { return err } + replica := m.replicaOf(sb) consumed, err := m.consumeArchive(ctx, sb) if err != nil { return err @@ -265,12 +258,16 @@ func (m *Manager) releaseRowed(ctx context.Context, id string, sb *types.Sandbox if err = m.releaseResolved(ctx, id, sb); err != nil { return err } - m.dropTombstone(context.WithoutCancel(ctx), id, consumed) + ctx = context.WithoutCancel(ctx) + m.dropTombstone(ctx, id, consumed) + if replica != "" { + m.deleteOrphanArchiveCk(ctx, replica) + } return nil } func (m *Manager) consumeArchive(ctx context.Context, sb *types.Sandbox) (int64, error) { - if !m.archiveRowed(sb) { + if !m.archiveRowed(sb) && !m.replicaRowed(sb) { return 0, nil } ctx = context.WithoutCancel(ctx) @@ -301,24 +298,39 @@ func (m *Manager) settleConsume(ctx context.Context, sb *types.Sandbox) (int64, return consumed, err } -// unrowExpired drops an expired archive's row, so the next reap purges it as a row-less archive. +// unrowExpired drops an expired claim's row, so the next reap purges or destroys it as a row-less claim. func (m *Manager) unrowExpired(ctx context.Context, sb *types.Sandbox) { defer m.untrack(m.archiving, sb.ID) sb.Transition.Lock() defer sb.Transition.Unlock() - if _, err := m.consumeArchive(ctx, sb); err != nil && !errors.Is(err, ErrUnknownSandbox) { + var err error + if m.replicaOf(sb) != "" { + err = m.dropReplica(ctx, sb) + } else { + _, err = m.consumeArchive(ctx, sb) + } + if err != nil && !errors.Is(err, ErrUnknownSandbox) { log.WithFunc("pool.unrowExpired").Warnf(ctx, "purge %s: %v", sb.ID, err) } } -// dropAdopted forgets an archived claim another node owns now; its checkpoint is the new owner's. +// dropAdopted forgets a claim another node owns now and removes its local VM; an archive checkpoint is the new owner's. func (m *Manager) dropAdopted(ctx context.Context, sb *types.Sandbox) { logger := log.WithFunc("pool.dropAdopted") - if err := m.clearArchiveCk(sb.ArchiveCk); err != nil { - logger.Warnf(ctx, "clear archive ck %s: %v", sb.ArchiveCk, err) + m.mu.Lock() + ck, vm, snap := sb.ArchiveCk, sb.VMName, sb.HibernateSnap + m.mu.Unlock() + if ck != "" { + if err := m.clearArchiveCk(ck); err != nil { + logger.Warnf(ctx, "clear archive ck %s: %v", ck, err) + } } if m.forgetClaim(ctx, sb) { - logger.Infof(ctx, "archived sandbox %s moved to another node", sb.ID) + if vm != "" { + m.destroy(ctx, vm) + } + m.dropSnap(ctx, snap) + logger.Infof(ctx, "sandbox %s moved to another node", sb.ID) } } @@ -353,7 +365,7 @@ func (m *Manager) sweepOrphanArchives(ctx context.Context) { switch { case rec.Owner == m.nodeID: m.reconcileOwnRow(ctx, rec, now) - case m.ownerGone(rec.Owner) && !rec.Consumed: + case !rec.Consumed: m.purgeArchiveRow(ctx, rec) } } @@ -372,7 +384,7 @@ func (m *Manager) reconcileOwnRow(ctx context.Context, rec archives.Record, now sb.Transition.Lock() defer sb.Transition.Unlock() m.mu.Lock() - held, refs, epoch := m.claimed[rec.ID] == sb, sb.ArchiveCk == rec.Checkpoint, sb.ArchiveEpoch + held, refs, epoch := m.claimed[rec.ID] == sb, sb.ArchiveCk == rec.Checkpoint || sb.ReplicaCk == rec.Checkpoint, sb.ArchiveEpoch m.mu.Unlock() switch { case !held: @@ -440,6 +452,13 @@ func (m *Manager) fenceArchived(ctx context.Context, claims map[string]*types.Sa } case ok && rec.Owner != m.nodeID: drop = true + case sb.ReplicaCk != "" && sb.ArchivePending == "": + switch { + case !ok && m.replicaConfirmed(sb): + drop = true + case ok && !rec.Consumed: + sb.ReplicaCk, sb.ArchiveEpoch = rec.Checkpoint, rec.Epoch + } case sb.ArchivePending != "" || ok && !rec.Consumed: moved, err := m.settleIntent(ctx, id, sb.ArchivePending, rec, ok) switch { @@ -513,7 +532,7 @@ func (m *Manager) resolveIntent(ctx context.Context, sb *types.Sandbox) error { case err != nil: return err case moved: - m.dropMoved(ctx, sb) + m.dropAdopted(ctx, sb) return ErrUnknownSandbox } if m.clearArchiveIntent(ctx, sb, pending) { @@ -540,21 +559,10 @@ func (m *Manager) settleIntents(ctx context.Context) { } } -func (m *Manager) dropMoved(ctx context.Context, sb *types.Sandbox) { - m.mu.Lock() - vm, snap := sb.VMName, sb.HibernateSnap - m.mu.Unlock() - if m.forgetClaim(ctx, sb) { - m.destroy(ctx, vm) - m.dropSnap(ctx, snap) - } -} - func (m *Manager) archiveRowed(sb *types.Sandbox) bool { return m.archives != nil && sb.ArchiveCk != "" && sb.ArchiveEpoch != 0 } -func (m *Manager) ownerGone(node string) bool { - alive := m.ownerAlive.Load() - return alive != nil && !(*alive)(node) +func (m *Manager) expiryDropsRow(sb *types.Sandbox) bool { + return m.archiveRowed(sb) || sb.ReplicaCk != "" && !m.archivesAtDeadline(sb) } diff --git a/sandboxd/pool/failover_test.go b/sandboxd/pool/failover_test.go index 7c263a8e..2b612e96 100644 --- a/sandboxd/pool/failover_test.go +++ b/sandboxd/pool/failover_test.go @@ -117,7 +117,7 @@ func TestFailoverAdoptionMovesTheClaimAndTheOldOwnerLetsGo(t *testing.T) { sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) ck := sb.ArchiveCk - c.down["a"] = true + c.expire("a") if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { t.Fatalf("adopt: %v %v, want adopted", adopted, err) @@ -125,7 +125,7 @@ func TestFailoverAdoptionMovesTheClaimAndTheOldOwnerLetsGo(t *testing.T) { if rec, _, _ := c.reg.Get(t.Context(), sb.ID); rec.Owner != "b" || rec.Epoch != 2 { t.Errorf("row %s@%d, want b@2", rec.Owner, rec.Epoch) } - c.down["a"] = false + c.renew(t, "a") if _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrUnknownSandbox) { t.Errorf("old owner's wake: %v, want ErrUnknownSandbox", err) } @@ -162,7 +162,7 @@ func TestFailoverADrainingNodeDoesNotAdopt(t *testing.T) { a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) - c.down["a"] = true + c.expire("a") b.Drain(t.Context()) if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || adopted { @@ -184,7 +184,7 @@ func TestFailoverAdoptionNeedsTheToken(t *testing.T) { a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) - c.down["a"] = true + c.expire("a") if adopted, err := b.AdoptArchived(t.Context(), sb.ID, "wrong"); err != nil || adopted { t.Errorf("adopt with a wrong token: %v %v, want refused", adopted, err) @@ -203,7 +203,7 @@ func TestFailoverConcurrentAdoptionsLeaveOneOwner(t *testing.T) { b, d := c.node(t, "b", t.TempDir(), keepAnHour), c.node(t, "d", t.TempDir(), keepAnHour) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) - c.down["a"] = true + c.expire("a") adopters := []*Manager{b, d} won := make([]bool, len(adopters)) @@ -239,7 +239,7 @@ func TestFailoverRestartDropsAClaimAdoptedElsewhere(t *testing.T) { a, b := c.node(t, "a", dirA, keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) - c.down["a"] = true + c.expire("a") if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { t.Fatalf("adopt: %v %v", adopted, err) } @@ -250,7 +250,7 @@ func TestFailoverRestartDropsAClaimAdoptedElsewhere(t *testing.T) { mustArchive(t, b, moved) adopterCk := moved.ArchiveCk - c.down["a"] = false + c.renew(t, "a") restarted := c.node(t, "a", dirA, keepAnHour) if err := restarted.Reconcile(t.Context()); err != nil { t.Fatalf("reconcile: %v", err) @@ -375,7 +375,7 @@ func TestFailoverReleaseAfterAdoptionKeepsTheCheckpoint(t *testing.T) { a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) - c.down["a"] = true + c.expire("a") if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { t.Fatalf("adopt: %v %v", adopted, err) } @@ -431,7 +431,7 @@ func TestFailoverSweepPurgesADepartedOwnersArchiveOnce(t *testing.T) { if _, ok, _ := c.reg.Get(t.Context(), sb.ID); !ok { t.Fatal("a sweep purged the archive of a live owner") } - c.down["a"] = true + c.expire("a") var wg sync.WaitGroup for _, m := range []*Manager{b, d} { wg.Go(func() { m.sweepOrphanArchives(t.Context()) }) @@ -453,7 +453,7 @@ func TestFailoverAdoptionMidWakeFencesTheOwnerOut(t *testing.T) { mustArchive(t, a, sb) ck := sb.ArchiveCk a.archives = &hookRegistry{Registry: c.reg, beforeConsume: func() { - c.down["a"] = true + c.expire("a") if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { t.Errorf("adopt mid-wake: %v %v, want adopted", adopted, err) } @@ -643,14 +643,14 @@ func TestFailoverAnArchiveWokenElsewhereIsDropped(t *testing.T) { if err := a.FlushClaims(); err != nil { t.Fatalf("flush: %v", err) } - c.down["a"] = true + c.expire("a") if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { t.Fatalf("adopt: %v %v", adopted, err) } if _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("adopter's wake: %v", err) } - c.down["a"] = false + c.renew(t, "a") restarted := c.restart(t, a) if err := restarted.Reconcile(t.Context()); err != nil { @@ -676,11 +676,11 @@ func TestFailoverALeftoverDeleteMarkerSparesTheAdoptersCheckpoint(t *testing.T) if err := a.markArchiveCk(ck); err != nil { t.Fatalf("mark: %v", err) } - c.down["a"] = true + c.expire("a") if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { t.Fatalf("adopt: %v %v", adopted, err) } - c.down["a"] = false + c.renew(t, "a") if _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrUnknownSandbox) { t.Fatalf("old owner's wake: %v, want ErrUnknownSandbox", err) } @@ -702,11 +702,11 @@ func TestFailoverRestartClearsTheMarkerOfAnAdoptedClaim(t *testing.T) { if err := a.FlushClaims(); err != nil { t.Fatalf("flush: %v", err) } - c.down["a"] = true + c.expire("a") if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { t.Fatalf("adopt: %v %v", adopted, err) } - c.down["a"] = false + c.renew(t, "a") restarted := c.restart(t, a) if err := restarted.Reconcile(t.Context()); err != nil { @@ -730,7 +730,7 @@ func TestFailoverARepeatedAdoptionOnOneNodeKeepsOneClaim(t *testing.T) { t.Fatalf("claim: %v", err) } mustArchive(t, a, sb) - c.down["a"] = true + c.expire("a") var wg sync.WaitGroup for range 2 { @@ -769,7 +769,7 @@ func TestFailoverRestartInsideAnAdoptersWakeDropsTheClaim(t *testing.T) { t.Fatalf("flush: %v", err) } ck := sb.ArchiveCk - c.down["a"] = true + c.expire("a") if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { t.Fatalf("adopt: %v %v", adopted, err) } @@ -783,7 +783,7 @@ func TestFailoverRestartInsideAnAdoptersWakeDropsTheClaim(t *testing.T) { if rec, ok, _ := c.reg.Get(t.Context(), sb.ID); !ok || !rec.Consumed || rec.Owner != "b" { t.Errorf("row %+v present %v, want b's tombstone kept while the checkpoint remains", rec, ok) } - c.down["a"] = false + c.renew(t, "a") restarted := c.restart(t, a) if err := restarted.Reconcile(t.Context()); err != nil { @@ -807,7 +807,7 @@ func TestFailoverRestartAfterAnAdopterReleasedDropsTheClaim(t *testing.T) { if err := a.FlushClaims(); err != nil { t.Fatalf("flush: %v", err) } - c.down["a"] = true + c.expire("a") if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { t.Fatalf("adopt: %v %v", adopted, err) } @@ -818,7 +818,7 @@ func TestFailoverRestartAfterAnAdopterReleasedDropsTheClaim(t *testing.T) { if !ckExists(t, b, sb.ArchiveCk) { t.Fatal("setup: the adopter's checkpoint delete went through") } - c.down["a"] = false + c.renew(t, "a") restarted := c.restart(t, a) if err := restarted.Reconcile(t.Context()); err != nil { @@ -883,7 +883,7 @@ func TestFailoverRestartDropsAClaimAdoptedWhileItsInterruptedArchiveSettles(t *t restarted := c.restart(t, a) restarted.archives = &hookRegistry{Registry: c.reg, beforeDelete: func() { - c.down["a"] = true + c.expire("a") if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { t.Errorf("adopt during the restart: %v %v, want adopted", adopted, err) } @@ -966,7 +966,7 @@ func TestFailoverSweepLeavesADepartedOwnersTombstoneUntilItReturns(t *testing.T) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) ck := sb.ArchiveCk - c.down["a"] = true + c.expire("a") if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { t.Fatalf("adopt: %v %v", adopted, err) } @@ -976,13 +976,13 @@ func TestFailoverSweepLeavesADepartedOwnersTombstoneUntilItReturns(t *testing.T) t.Fatalf("adopter's wake: %v", err) } before, _, _ := c.reg.Get(t.Context(), sb.ID) - c.down["b"] = true + c.expire("b") d.sweepOrphanArchives(t.Context()) if rec, ok, _ := c.reg.Get(t.Context(), sb.ID); !ok || rec.Epoch != before.Epoch || !ckExists(t, d, ck) { t.Fatalf("row %+v present %v after a peer's sweep, want the departed owner's tombstone and checkpoint untouched", rec, ok) } - c.down["b"] = false + c.renew(t, "b") b.ckpts = healthy b.sweepOrphanArchives(t.Context()) if _, ok, _ := c.reg.Get(t.Context(), sb.ID); ok || ckExists(t, b, ck) { @@ -1057,7 +1057,7 @@ func TestFailoverATombstoneIsNotAdopted(t *testing.T) { if _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake: %v", err) } - c.down["a"] = true + c.expire("a") counted := &countingRegistry{Registry: c.reg} d.archives = counted @@ -1079,7 +1079,7 @@ func TestFailoverADropClearsTheMarkerBeforeTheCheckpointIsUnpinned(t *testing.T) if err := a.markArchiveCk(ck); err != nil { t.Fatalf("mark: %v", err) } - c.down["a"] = true + c.expire("a") if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { t.Fatalf("adopt: %v %v", adopted, err) } @@ -1149,7 +1149,7 @@ func TestFailoverARepeatedAdoptionOfAReleasedClaimIsRefused(t *testing.T) { a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) - c.down["a"] = true + c.expire("a") if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { t.Fatalf("adopt: %v %v", adopted, err) } @@ -1224,7 +1224,7 @@ func TestFailoverSweepRearmsATombstoneItsArchivedClaimHolds(t *testing.T) { if rec.Consumed || rec.Owner != "a" || rec.Epoch != epoch { t.Fatalf("row %+v, local epoch %d, want the tombstone re-armed at the claim's epoch", rec, epoch) } - c.down["a"] = true + c.expire("a") if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { t.Errorf("adopt of the re-armed archive: %v %v, want adopted", adopted, err) } @@ -1242,7 +1242,7 @@ func TestFailoverSweepLeavesADepartedOwnersWakeTombstone(t *testing.T) { if _, err = c.reg.Consume(t.Context(), sb.ID, epoch); err != nil { t.Fatalf("consume: %v", err) } - c.down["a"] = true + c.expire("a") b.sweepOrphanArchives(t.Context()) if rec, ok, _ := c.reg.Get(t.Context(), sb.ID); !ok || !rec.Consumed || !ckExists(t, b, sb.ArchiveCk) { @@ -1266,9 +1266,9 @@ func TestFailoverRestartAfterAConsumeWhileAPeerSweptKeepsTheClaim(t *testing.T) if err = a.FlushClaims(); err != nil { t.Fatalf("flush: %v", err) } - c.down["a"] = true + c.expire("a") b.sweepOrphanArchives(t.Context()) - c.down["a"] = false + c.renew(t, "a") restarted := c.restart(t, a) if err = restarted.Reconcile(t.Context()); err != nil { @@ -1329,7 +1329,7 @@ func TestFailoverAWakeKeepsItsFenceAgainstThisNodesOwnAdoption(t *testing.T) { sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) ck := sb.ArchiveCk - c.down["a"] = true + c.expire("a") if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { t.Fatalf("adopt: %v %v", adopted, err) } @@ -1377,7 +1377,7 @@ func TestFailoverConcurrentAdoptionsOnOneNodeAllAnswerForIt(t *testing.T) { t.Fatalf("claim: %v", err) } mustArchive(t, a, sb) - c.down["a"] = true + c.expire("a") const callers = 10 start := make(chan struct{}) @@ -1410,7 +1410,7 @@ func TestFailoverACoalescedAdoptionStillNeedsTheToken(t *testing.T) { a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) - c.down["a"] = true + c.expire("a") wrong := make(chan bool, 1) b.archives = &hookRegistry{Registry: c.reg, beforeTake: func() { go func() { @@ -1470,7 +1470,7 @@ func TestFailoverRestartDropsAClaimWhoseInterruptedArchiveWasConsumedElsewhere(t vm := sb.VMName a.mu.Unlock() ck := interruptArchive(t, a, sb, true) - c.down["a"] = true + c.expire("a") if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { t.Fatalf("adopt: %v %v", adopted, err) } @@ -1480,7 +1480,7 @@ func TestFailoverRestartDropsAClaimWhoseInterruptedArchiveWasConsumedElsewhere(t if _, ok, _ := c.reg.Get(t.Context(), sb.ID); ok || ckExists(t, b, ck) { t.Fatal("setup: the adopter's wake left the row or the checkpoint") } - c.down["a"] = false + c.renew(t, "a") restarted := c.restart(t, a) if err := restarted.Reconcile(t.Context()); err != nil { @@ -1593,9 +1593,9 @@ func TestFailoverAnOwnerReturningWithItsWakeCommittedPurgesItsTombstone(t *testi if err := a.FlushClaims(); err != nil { t.Fatalf("flush: %v", err) } - c.down["a"] = true + c.expire("a") b.sweepOrphanArchives(t.Context()) - c.down["a"] = false + c.renew(t, "a") restarted := c.restart(t, a) if err := restarted.Reconcile(t.Context()); err != nil { @@ -1621,7 +1621,7 @@ func TestFailoverRestartDropsAClaimConsumedWhileItSettlesItsInterruptedArchive(t restarted := c.restart(t, a) restarted.archives = &hookRegistry{Registry: c.reg, beforeDelete: func() { - c.down["a"] = true + c.expire("a") if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { t.Errorf("adopt during the restart: %v %v, want adopted", adopted, err) } @@ -1802,14 +1802,14 @@ func TestFailoverAReleaseFindsItsUnsettledArchiveTakenElsewhere(t *testing.T) { if err := a.archive(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { t.Fatalf("archive whose put reply was lost: %v, want ErrArchiveStoreDown", err) } - c.down["a"] = true + c.expire("a") if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { t.Fatalf("adopt: %v %v", adopted, err) } if _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("adopter's wake: %v", err) } - c.down["a"] = false + c.renew(t, "a") if err := a.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); !errors.Is(err, ErrUnknownSandbox) { t.Fatalf("old owner's release: %v, want ErrUnknownSandbox", err) @@ -1823,13 +1823,24 @@ func TestFailoverAReleaseFindsItsUnsettledArchiveTakenElsewhere(t *testing.T) { } type failoverCluster struct { - reg *archivestest.Mem - ckDir string - down map[string]bool + reg *archivestest.Mem + ckDir string + replicate int } func newFailoverCluster(t *testing.T) *failoverCluster { - return &failoverCluster{reg: archivestest.NewMem(), ckDir: t.TempDir(), down: map[string]bool{}} + return &failoverCluster{reg: archivestest.NewMem(), ckDir: t.TempDir()} +} + +func (c *failoverCluster) expire(node string) { + c.reg.Expire(node) +} + +func (c *failoverCluster) renew(t *testing.T, node string) { + t.Helper() + if err := c.reg.Renew(t.Context(), node, time.Hour); err != nil { + t.Fatalf("renew %s: %v", node, err) + } } func (c *failoverCluster) node(t *testing.T, name, dataDir string, deleteAfter int) *Manager { @@ -1844,8 +1855,10 @@ func (c *failoverCluster) restart(t *testing.T, old *Manager) *Manager { func (c *failoverCluster) nodeOn(t *testing.T, eng *fakeEngine, name, dataDir string, deleteAfter int) *Manager { t.Helper() + spec := archivePool(deleteAfter) + spec.ReplicateSeconds = c.replicate cfg := &config.Config{ - APIToken: "root", DataDir: dataDir, CheckpointDir: c.ckDir, Pools: []config.PoolSpec{archivePool(deleteAfter)}, + APIToken: "root", DataDir: dataDir, CheckpointDir: c.ckDir, Pools: []config.PoolSpec{spec}, Mesh: &config.MeshConfig{NodeID: name}, } b := testBackends(t, cfg) @@ -1854,7 +1867,7 @@ func (c *failoverCluster) nodeOn(t *testing.T, eng *fakeEngine, name, dataDir st if err != nil { t.Fatalf("manager %s: %v", name, err) } - m.WithArchiveOwners(func(node string) bool { return !c.down[node] }) + _ = c.reg.Renew(t.Context(), name, time.Hour) return m } @@ -1863,7 +1876,7 @@ func assertReleasedForGood(t *testing.T, c *failoverCluster, sb *types.Sandbox, if rec, ok, _ := c.reg.Get(t.Context(), sb.ID); ok && !rec.Consumed { t.Errorf("a live row %+v outlived the release", rec) } - c.down["a"] = true + c.expire("a") for _, m := range nodes { if adopted, err := m.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || adopted { t.Errorf("node %s adopts the released claim: %v %v, want a 404", m.nodeID, adopted, err) diff --git a/sandboxd/pool/hibernate.go b/sandboxd/pool/hibernate.go index b5f32eba..35108fb8 100644 --- a/sandboxd/pool/hibernate.go +++ b/sandboxd/pool/hibernate.go @@ -30,6 +30,9 @@ func (m *Manager) WakeAgentSocket(ctx context.Context, id, token string) (string if !ok { return "", nil, ErrUnknownSandbox } + if m.leaseFenced(sb) { + return "", nil, ErrLeaseLost + } sb.Touch() sb.Hold() sock, err := m.wakeResolved(ctx, sb) diff --git a/sandboxd/pool/lease.go b/sandboxd/pool/lease.go new file mode 100644 index 00000000..b63498ab --- /dev/null +++ b/sandboxd/pool/lease.go @@ -0,0 +1,85 @@ +package pool + +import ( + "context" + "maps" + "slices" + "time" + + "github.com/projecteru2/core/log" + + "github.com/cocoonstack/sandbox/sandboxd/archives" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +type leaseTiming struct { + ttl, every, margin time.Duration +} + +// keepLease renews this node's lease until ctx ends and fences its replicated claims when a renewal is overdue. +func (m *Manager) keepLease(ctx context.Context) { + fence := time.AfterFunc(m.lease.ttl-m.lease.margin, func() { m.fence(ctx) }) + defer fence.Stop() + tick := time.NewTicker(m.lease.every) + defer tick.Stop() + for { + m.renewLease(ctx, fence) + select { + case <-ctx.Done(): + return + case <-tick.C: + } + } +} + +func (m *Manager) renewLease(ctx context.Context, fence *time.Timer) { + start := time.Now() + if err := m.archives.Renew(ctx, m.nodeID, m.lease.ttl); err != nil { + log.WithFunc("pool.renewLease").Warnf(ctx, "renew node lease: %v", err) + return + } + fence.Reset(m.lease.ttl - m.lease.margin - time.Since(start)) + if m.fenced.Load() { + m.unfence(ctx) + } +} + +func (m *Manager) fence(ctx context.Context) { + if !m.fenced.Swap(true) { + log.WithFunc("pool.fence").Warnf(ctx, "node lease not renewed for %s; replicated claims answer 503 until it is", m.lease.ttl-m.lease.margin) + } +} + +func (m *Manager) unfence(ctx context.Context) { + logger := log.WithFunc("pool.unfence") + m.mu.Lock() + held := map[string]*types.Sandbox{} + for id, sb := range m.claimed { + if sb.ReplicaCk != "" { + held[id] = sb + } + } + m.mu.Unlock() + var rows map[string]archives.Record + if len(held) > 0 { + var err error + if rows, err = m.archives.Rows(ctx, slices.Collect(maps.Keys(held))); err != nil { + logger.Warnf(ctx, "replicated claims stay fenced: %v", err) + return + } + } + for id, sb := range held { + rec, ok := rows[id] + if ok && rec.Owner != m.nodeID || !ok && m.replicaConfirmed(sb) { + sb.Transition.Lock() + m.dropAdopted(ctx, sb) + sb.Transition.Unlock() + } + } + m.fenced.Store(false) + logger.Info(ctx, "node lease renewed; replicated claims are served again") +} + +func (m *Manager) leaseFenced(sb *types.Sandbox) bool { + return m.fenced.Load() && m.replicaOf(sb) != "" +} diff --git a/sandboxd/pool/lease_test.go b/sandboxd/pool/lease_test.go new file mode 100644 index 00000000..389f831b --- /dev/null +++ b/sandboxd/pool/lease_test.go @@ -0,0 +1,168 @@ +package pool + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/archives" +) + +func TestLeaseRenewLoopKeepsTheNodeLeasedUntilItStops(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + c := newReplicaCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + replicateNow(t, a, sb) + row, _, _ := c.reg.Get(t.Context(), sb.ID) + c.expire("a") + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + go a.keepLease(ctx) + time.Sleep(10 * leaseTTL) + if _, err := c.reg.Take(ctx, sb.ID, row.Epoch, "b"); !errors.Is(err, archives.ErrLost) { + t.Fatalf("take while the loop renews: %v, want ErrLost", err) + } + cancel() + synctest.Wait() + time.Sleep(leaseTTL + time.Second) + if _, err := c.reg.Take(t.Context(), sb.ID, row.Epoch, "b"); err != nil { + t.Errorf("take after the loop stopped and the lease ran out: %v, want it taken", err) + } + }) +} + +func TestLeaseOverdueRenewalFencesReplicatedClaimsOnly(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + c := newReplicaCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + cut := &cutRegistry{Registry: c.reg} + a.archives = cut + replicated, plain := mustClaim(t, a, testKey), mustClaim(t, a, testKey) + replicateNow(t, a, replicated) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + go a.keepLease(ctx) + synctest.Wait() + time.Sleep(4*leaseRenew + 2*time.Second) + synctest.Wait() + assertServed(t, a, replicated.ID, replicated.Token) + + cut.cut.Store(true) + time.Sleep(leaseTTL - leaseMargin - 3*time.Second) + synctest.Wait() + assertServed(t, a, replicated.ID, replicated.Token) + time.Sleep(2 * time.Second) + synctest.Wait() + if _, _, err := a.WakeAgentSocket(ctx, replicated.ID, replicated.Token); !errors.Is(err, ErrLeaseLost) { + t.Errorf("relay or exec of a replicated claim past the fence: %v, want ErrLeaseLost", err) + } + if _, err := a.DialPort(ctx, replicated.ID, Cred{Token: replicated.Token}, 80); !errors.Is(err, ErrLeaseLost) { + t.Errorf("port dial past the fence: %v, want ErrLeaseLost", err) + } + if _, err := a.DialPort(ctx, replicated.ID, Cred{Operator: true}, 80); !errors.Is(err, ErrLeaseLost) { + t.Errorf("preview dial past the fence: %v, want ErrLeaseLost", err) + } + assertServed(t, a, plain.ID, plain.Token) + if _, held := a.byID(replicated.ID); !held || a.eng.(*fakeEngine).removed(lockedVMName(replicated)) { + t.Error("the fence dropped the claim or removed its VM") + } + + cut.cut.Store(false) + time.Sleep(leaseRenew) + synctest.Wait() + assertServed(t, a, replicated.ID, replicated.Token) + }) +} + +func TestLeaseAdoptionWaitsOutTheFenceAndTheOldOwnerDropsItsCopy(t *testing.T) { + for _, tt := range []struct { + name string + wakeBefore bool + }{ + {"recovers before the adopter wakes", false}, + {"recovers after the adopter woke", true}, + } { + t.Run(tt.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + c := newReplicaCluster(t) + a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + cut := &cutRegistry{Registry: c.reg} + a.archives = cut + sb := mustClaim(t, a, testKey) + vm := lockedVMName(sb) + replicateNow(t, a, sb) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + go a.keepLease(ctx) + synctest.Wait() + + cut.cut.Store(true) + time.Sleep(leaseTTL - leaseMargin + time.Second) + synctest.Wait() + if _, _, err := a.WakeAgentSocket(ctx, sb.ID, sb.Token); !errors.Is(err, ErrLeaseLost) { + t.Fatalf("the cut-off owner past its fence: %v, want ErrLeaseLost", err) + } + if adopted, err := b.AdoptArchived(ctx, sb.ID, sb.Token); err != nil || adopted { + t.Fatalf("adoption while the fenced owner's lease is still live: %v %v, want refused", adopted, err) + } + + time.Sleep(leaseMargin) + if adopted, err := b.AdoptArchived(ctx, sb.ID, sb.Token); err != nil || !adopted { + t.Fatalf("adoption after the lease expired: %v %v, want adopted", adopted, err) + } + if tt.wakeBefore { + if _, _, err := b.WakeAgentSocket(ctx, sb.ID, sb.Token); err != nil { + t.Fatalf("adopter's wake: %v", err) + } + } + + cut.cut.Store(false) + time.Sleep(leaseRenew) + synctest.Wait() + if _, held := a.byID(sb.ID); held || !a.eng.(*fakeEngine).removed(vm) { + t.Errorf("old owner holds the claim %v, VM removed %v, want its copy dropped and its VM removed", held, a.eng.(*fakeEngine).removed(vm)) + } + if a.fenced.Load() { + t.Error("the old owner stays fenced after its lease renewed") + } + if _, _, err := b.WakeAgentSocket(ctx, sb.ID, sb.Token); err != nil { + t.Errorf("adopter's wake after the old owner dropped its copy: %v", err) + } + }) + }) + } +} + +type cutRegistry struct { + archives.Registry + cut atomic.Bool +} + +func (r *cutRegistry) Renew(ctx context.Context, node string, ttl time.Duration) error { + if r.cut.Load() { + return archives.ErrUnavailable + } + return r.Registry.Renew(ctx, node, ttl) +} + +func (r *cutRegistry) Rows(ctx context.Context, ids []string) (map[string]archives.Record, error) { + if r.cut.Load() { + return nil, archives.ErrUnavailable + } + return r.Registry.Rows(ctx, ids) +} + +func assertServed(t *testing.T, m *Manager, id, token string) { + t.Helper() + _, done, err := m.WakeAgentSocket(t.Context(), id, token) + if err != nil { + t.Fatalf("serving %s: %v", id, err) + } + done() +} diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index ca47f286..7868809c 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -52,6 +52,10 @@ const ( vsockPollInterval = 100 * time.Millisecond defaultTTL = 5 * time.Minute maxTTL = 24 * time.Hour + // One missed renewal never fences: the next one lands before the fence at ttl-margin. + leaseTTL = 15 * time.Second + leaseRenew = 5 * time.Second + leaseMargin = 3 * time.Second // One failed boot is ordinary; only an unbroken run predicts the next failure refillFailStreak = 8 refillBackoffBase = 250 * time.Millisecond @@ -101,6 +105,7 @@ var ( // Replaying a journal takes a writable mount, so readers stay out. ErrVolumeNeedsRecovery = errors.New("volume needs recovery by a writable claim") ErrArchived = errors.New("sandbox is archived; an exec or file call wakes it first") + ErrLeaseLost = errors.New("node lease lost; retry") ErrPaused = errors.New("sandbox is paused; an operator call does not wake it") ErrFailed = errors.New("sandbox failed: its VMM is down; release it, or wake it to cold-boot its disk") ErrNoInstanceMetadata = engine.ErrNoInstanceMetadata @@ -338,9 +343,13 @@ type Manager struct { // archives is the cluster archive registry, nil unless archive_failover is set. archives archives.Registry nodeID string - ownerAlive atomic.Pointer[OwnerAlive] orphanSweep atomic.Bool adoptFlights singleflight.Group + // fenced gates replicated claims while the lease is overdue; replicas belongs to the pass holding replicating. + lease leaseTiming + fenced atomic.Bool + replicating atomic.Bool + replicas map[string]replicaState view atomic.Pointer[configView] // cfg is the config the view was built from; reloadMu orders reloads, the only writers of cfg and view. @@ -425,6 +434,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg poolStore: newPoolStore(cfg.DataDir), poolShared: b.CellPools, archives: b.Archives, + lease: leaseTiming{ttl: leaseTTL, every: leaseRenew, margin: leaseMargin}, tenants: b.Tenants, ckpts: b.Checkpoints, ckptsShared: b.SharedCheckpoints, @@ -503,6 +513,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg m.configSeedHash = poolSeedHash(cfg.Pools) if m.archives != nil { m.nodeID = cfg.Mesh.ID() + m.fenced.Store(true) } if err := m.adoptPools(ctx); err != nil { return nil, err @@ -540,6 +551,9 @@ func (m *Manager) Run(ctx context.Context) { if m.poolShared != nil { go m.followCellPools(ctx) } + if m.archives != nil { + go m.keepLease(ctx) + } m.livenessOnce(ctx) m.checkDisk(ctx) m.refillOnce(ctx) @@ -560,6 +574,7 @@ func (m *Manager) Run(ctx context.Context) { m.reapOnce(ctx) m.idleOnce(ctx) m.archiveOnce(ctx) + m.replicateOnce(ctx) m.shrinkOnce(ctx) go m.retryArchiveDeletes(ctx) case <-liveness.C: diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index 871b42a0..81b2269e 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -1247,14 +1247,14 @@ type fakeEngine struct { vsockLateN int sockRoot string - cloneErr, runColdErr, probeErr, hibernateErr, restoreErr, snapListErr, startErr error - removeErrFor string - cloneFailNth int - hibernateErrCompletes bool - tap string - listCount int - inspects int - vmEvents chan fakeVMEvent + cloneErr, runColdErr, probeErr, hibernateErr, restoreErr, snapListErr, snapSaveErr, startErr error + removeErrFor string + cloneFailNth int + hibernateErrCompletes bool + tap string + listCount int + inspects int + vmEvents chan fakeVMEvent attachRendezvous *sync.WaitGroup @@ -1374,6 +1374,9 @@ func (f *fakeEngine) ReconcileStaleCreate(ctx context.Context, name string) (eng func (f *fakeEngine) SnapshotSave(_ context.Context, _, snapName string) error { f.mu.Lock() defer f.mu.Unlock() + if f.snapSaveErr != nil { + return f.snapSaveErr + } f.snapSaves = append(f.snapSaves, snapName) return nil } diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 6ed8db81..11cbc259 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -293,7 +293,7 @@ func (m *Manager) reclaimOrphanArchiveCks(ctx context.Context, claims map[string continue } orig, mine := claims[ckpt.SandboxID] - if !mine || orig.ArchiveCk == ckpt.ID || orig.ArchivePending == ckpt.ID || rows[ckpt.SandboxID].Checkpoint == ckpt.ID { + if !mine || orig.ArchiveCk == ckpt.ID || orig.ReplicaCk == ckpt.ID || orig.ArchivePending == ckpt.ID || rows[ckpt.SandboxID].Checkpoint == ckpt.ID { continue } if err := m.deleteArchiveCk(ctx, ckpt.ID); err != nil { diff --git a/sandboxd/pool/replica.go b/sandboxd/pool/replica.go new file mode 100644 index 00000000..ab7a65db --- /dev/null +++ b/sandboxd/pool/replica.go @@ -0,0 +1,170 @@ +package pool + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/projecteru2/core/log" + + "github.com/cocoonstack/sandbox/sandboxd/archives" + "github.com/cocoonstack/sandbox/sandboxd/store" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +type replicaState struct { + tried time.Time + skipLogged bool +} + +func (m *Manager) replicateOnce(ctx context.Context) { + if m.archives == nil || m.fenced.Load() || !m.replicating.CompareAndSwap(false, true) { + return + } + now, v := time.Now(), m.view.Load() + var due, retire []*types.Sandbox + var withVolumes []string + next := make(map[string]replicaState, len(m.replicas)) + m.mu.Lock() + for id, sb := range m.claimed { + every, st := v.poolReplicate[sb.Key], m.replicas[id] + switch { + case every <= 0: + if sb.ReplicaCk != "" { + retire = append(retire, sb) + } + case !m.replicable(sb): + case m.hasVolumeHolds(sb): + if !st.skipLogged { + withVolumes = append(withVolumes, id) + } + next[id] = replicaState{skipLogged: true} + case st.tried.IsZero(): + next[id] = replicaState{tried: now} + case now.Sub(st.tried) < every: + next[id] = st + default: + next[id] = replicaState{tried: now} + due = append(due, sb) + } + } + m.mu.Unlock() + m.replicas = next + logger := log.WithFunc("pool.replicateOnce") + for _, id := range withVolumes { + logger.Infof(ctx, "sandbox %s holds volumes and is not replicated", id) + } + if len(due)+len(retire) == 0 { + m.replicating.Store(false) + return + } + go func() { + defer m.replicating.Store(false) + m.runBounded(ctx, len(due)+len(retire), func(ctx context.Context, i int) { + if i >= len(due) { + m.retireReplica(ctx, retire[i-len(due)]) + return + } + sb := due[i] + if err := m.replicate(ctx, sb); err != nil && !benignSweepErr(err) { + logger.Warnf(ctx, "replicate %s, retrying next interval: %v", sb.ID, err) + } + }).Wait() + }() +} + +// replicable reports whether sb is a running claim; callers hold m.mu. +func (m *Manager) replicable(sb *types.Sandbox) bool { + _, archiving := m.archiving[sb.ID] + return !archiving && sb.HibernateSnap == "" && sb.PendingSnap == "" && sb.ArchiveCk == "" && sb.ArchivePending == "" && sb.Failed == "" && !m.tenantGone(sb.Tenant) +} + +func (m *Manager) replicate(ctx context.Context, sb *types.Sandbox) error { + if err := m.diskErr(); err != nil { + return fmt.Errorf("replicate %s: %w", sb.ID, err) + } + if err := m.checkArchiveRow(ctx, sb.ID); err != nil { + if errors.Is(err, archives.ErrLost) { + sb.Transition.Lock() + m.dropAdopted(ctx, sb) + sb.Transition.Unlock() + } + return fmt.Errorf("replicate %s: registry: %w", sb.ID, err) + } + ckID := store.CheckpointID(randHex(8)) + m.mu.Lock() + m.pendingCks[ckID] = struct{}{} + m.mu.Unlock() + defer m.untrack(m.pendingCks, ckID) + if _, _, err := m.publishCheckpoint(ctx, sb, ckID, "replica", sb.Tenant, true); err != nil { + return err + } + sb.Transition.Lock() + defer sb.Transition.Unlock() + m.mu.Lock() + ok := m.claimed[sb.ID] == sb && sb.ArchiveCk == "" && sb.ArchivePending == "" + m.mu.Unlock() + if !ok { + m.deleteOrphanArchiveCk(ctx, ckID) + return errWokeMeanwhile + } + epoch, err := m.putArchive(ctx, sb, ckID) + if err != nil { + m.deleteOrphanArchiveCk(ctx, ckID) + if errors.Is(err, archives.ErrLost) { + m.dropAdopted(ctx, sb) + } + return fmt.Errorf("replicate %s: registry: %w", sb.ID, err) + } + var prev string + live, _ := m.commitIfLive(ctx, sb, func() { prev, sb.ReplicaCk, sb.ArchiveEpoch = sb.ReplicaCk, ckID, epoch }) + if !live { + m.abandonArchive(ctx, sb, epoch, ckID) + return errWokeMeanwhile + } + if prev != "" { + m.deleteOrphanArchiveCk(ctx, prev) + } + return nil +} + +func (m *Manager) retireReplica(ctx context.Context, sb *types.Sandbox) { + sb.Transition.Lock() + defer sb.Transition.Unlock() + if err := m.dropReplica(ctx, sb); err != nil && !benignSweepErr(err) { + log.WithFunc("pool.retireReplica").Warnf(ctx, "stop replicating %s: %v", sb.ID, err) + } +} + +// dropReplica ends sb's replication; the caller holds sb.Transition. +func (m *Manager) dropReplica(ctx context.Context, sb *types.Sandbox) error { + consumed, err := m.consumeArchive(ctx, sb) + if err != nil { + return err + } + var ck string + live, _ := m.commitIfLive(ctx, sb, func() { ck, sb.ReplicaCk = sb.ReplicaCk, "" }) + if !live { + return ErrUnknownSandbox + } + m.dropTombstone(ctx, sb.ID, consumed) + m.deleteOrphanArchiveCk(ctx, ck) + return nil +} + +func (m *Manager) replicaRowed(sb *types.Sandbox) bool { + return m.archives != nil && sb.ReplicaCk != "" && sb.ArchiveEpoch != 0 +} + +func (m *Manager) replicaOf(sb *types.Sandbox) string { + m.mu.Lock() + defer m.mu.Unlock() + return sb.ReplicaCk +} + +func (m *Manager) replicaConfirmed(sb *types.Sandbox) bool { + m.mu.Lock() + defer m.mu.Unlock() + return sb.ArchiveEpoch != 0 +} diff --git a/sandboxd/pool/replica_test.go b/sandboxd/pool/replica_test.go new file mode 100644 index 00000000..bf917d88 --- /dev/null +++ b/sandboxd/pool/replica_test.go @@ -0,0 +1,546 @@ +package pool + +import ( + "encoding/json/v2" + "errors" + "testing" + "testing/synctest" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/archives" + "github.com/cocoonstack/sandbox/sandboxd/store" + "github.com/cocoonstack/sandbox/sandboxd/tenants" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +const replicateEvery = 60 + +func TestReplicaRoundPublishesMovesTheRowAndDropsThePrevious(t *testing.T) { + c := newReplicaCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + vm := lockedVMName(sb) + + replicateNow(t, a, sb) + first := a.replicaOf(sb) + rec, ok, err := c.reg.Get(t.Context(), sb.ID) + if err != nil || !ok { + t.Fatalf("row: %v %v, want one", ok, err) + } + if first == "" || rec.Owner != "a" || rec.Checkpoint != first || rec.Consumed || rec.TokenSHA256 != tenants.TokenSHA256(sb.Token) || rec.Deadline.IsZero() { + t.Fatalf("row %+v with replica %q, want a live row of a on the replica with the token hash and a deadline", rec, first) + } + if stored, loadErr := a.store.load(); loadErr != nil || stored[sb.ID].ReplicaCk != first || stored[sb.ID].ArchiveEpoch != rec.Epoch { + t.Errorf("claims.json replica %q epoch %d (%v), want %q at %d", stored[sb.ID].ReplicaCk, stored[sb.ID].ArchiveEpoch, loadErr, first, rec.Epoch) + } + meta, err := a.loadCheckpoint(t.Context(), first) + if err != nil || !meta.Archive { + t.Errorf("replica record %+v %v, want an archive record: not branchable and not listed", meta, err) + } + if listed, listErr := a.Checkpoints(t.Context(), ""); listErr != nil || len(listed) != 0 { + t.Errorf("listing shows %d checkpoints %v, want the replica hidden", len(listed), listErr) + } + if delErr := a.DeleteCheckpoint(t.Context(), first, "", DeleteFleet); !errors.Is(delErr, ErrUnknownCheckpoint) { + t.Errorf("delete of the replica: %v, want ErrUnknownCheckpoint", delErr) + } + + replicateNow(t, a, sb) + second := a.replicaOf(sb) + next, _, _ := c.reg.Get(t.Context(), sb.ID) + if second == "" || second == first || next.Checkpoint != second || next.Epoch <= rec.Epoch { + t.Errorf("second round replica %q row %s@%d, want a new replica at a new epoch above %d", second, next.Checkpoint, next.Epoch, rec.Epoch) + } + if ckExists(t, a, first) || !ckExists(t, a, second) { + t.Errorf("previous replica kept %v, new replica kept %v, want only the new one", ckExists(t, a, first), ckExists(t, a, second)) + } + claim := &types.Sandbox{} + if err = json.Unmarshal(next.Claim, claim); err != nil || claim.ID != sb.ID || claim.Token != "" || claim.VMName != "" || claim.ReplicaCk != "" { + t.Errorf("row claim %+v %v, want the claim without token, VM or replica", claim, err) + } + if a.eng.(*fakeEngine).removed(vm) { + t.Error("a replication round removed the running VM") + } +} + +func TestReplicaRefusedPutDropsTheCopyAndItsExport(t *testing.T) { + c := newReplicaCluster(t) + a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + vm := lockedVMName(sb) + replicateNow(t, a, sb) + a.archives = &hookRegistry{Registry: c.reg, beforePut: func() { + c.expire("a") + if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { + t.Errorf("adopt: %v %v", adopted, err) + } + }} + + if err := a.replicate(t.Context(), sb); !errors.Is(err, archives.ErrLost) { + t.Fatalf("round whose row another node took: %v, want ErrLost", err) + } + if _, held := a.byID(sb.ID); held || !a.eng.(*fakeEngine).removed(vm) { + t.Errorf("old owner holds the claim %v, VM removed %v, want its copy dropped", held, a.eng.(*fakeEngine).removed(vm)) + } + if metas, err := a.ckpts.Metas(t.Context()); err != nil || len(metas) != 1 { + t.Errorf("store holds %d checkpoints %v, want only the adopter's replica", len(metas), err) + } +} + +func TestReplicaFailedCaptureKeepsTheOldReplicaAndRow(t *testing.T) { + c := newReplicaCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + replicateNow(t, a, sb) + first := a.replicaOf(sb) + before, _, _ := c.reg.Get(t.Context(), sb.ID) + + a.eng.(*fakeEngine).snapSaveErr = errors.New("pause failed") + if err := a.replicate(t.Context(), sb); err == nil { + t.Fatal("a round whose capture failed reported success") + } + after, _, _ := c.reg.Get(t.Context(), sb.ID) + if after.Checkpoint != before.Checkpoint || after.Epoch != before.Epoch { + t.Errorf("row %s@%d after a failed round, want %s@%d", after.Checkpoint, after.Epoch, before.Checkpoint, before.Epoch) + } + if a.replicaOf(sb) != first || !ckExists(t, a, first) { + t.Errorf("replica %q kept %v, want %q kept", a.replicaOf(sb), ckExists(t, a, first), first) + } + if metas, err := a.ckpts.Metas(t.Context()); err != nil || len(metas) != 1 { + t.Errorf("store holds %d checkpoints %v, want only the old replica", len(metas), err) + } +} + +func TestReplicaRegistryOutagePausesRoundsWithoutACapture(t *testing.T) { + c := newReplicaCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + eng := a.eng.(*fakeEngine) + + c.reg.SetDown(true) + if err := a.replicate(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { + t.Fatalf("round with the registry down: %v, want ErrArchiveStoreDown", err) + } + if len(eng.snapSaves) != 0 || a.replicaOf(sb) != "" { + t.Errorf("%d captures and replica %q during the outage, want none", len(eng.snapSaves), a.replicaOf(sb)) + } + c.reg.SetDown(false) + replicateNow(t, a, sb) + if rec, ok, _ := c.reg.Get(t.Context(), sb.ID); !ok || rec.Checkpoint != a.replicaOf(sb) { + t.Errorf("row %+v present %v after recovery, want it on the new replica", rec, ok) + } +} + +func TestReplicaPassSnapshotsOnlyDueRunningClaims(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + c := newReplicaCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + a.fenced.Store(false) + running := mustClaim(t, a, testKey) + paused := mustClaim(t, a, testKey) + withVolume := mustClaim(t, a, testKey) + if err := a.Hibernate(t.Context(), paused.ID, Cred{Token: paused.Token}); err != nil { + t.Fatalf("hibernate: %v", err) + } + a.mu.Lock() + withVolume.Volumes = []types.Volume{{Name: "data"}} + a.mu.Unlock() + eng := a.eng.(*fakeEngine) + base := len(eng.snapSaves) + + a.replicateOnce(t.Context()) + synctest.Wait() + time.Sleep((replicateEvery - 1) * time.Second) + a.replicateOnce(t.Context()) + synctest.Wait() + if got := len(eng.snapSaves) - base; got != 0 || a.replicaOf(running) != "" { + t.Fatalf("%d captures before the interval, want none", got) + } + + time.Sleep(time.Second) + a.replicateOnce(t.Context()) + synctest.Wait() + if got := len(eng.snapSaves) - base; got != 1 || a.replicaOf(running) == "" { + t.Errorf("%d captures at the interval, replica %q, want exactly the running claim's", got, a.replicaOf(running)) + } + if a.replicaOf(paused) != "" || a.replicaOf(withVolume) != "" { + t.Error("a hibernated claim or one with volumes was replicated") + } + if !a.replicas[withVolume.ID].skipLogged { + t.Error("a claim with volumes was not skipped and logged") + } + if _, ok, _ := c.reg.Get(t.Context(), running.ID); !ok { + t.Error("the due claim has no row") + } + }) +} + +func TestReplicaPassSkipsWhileFenced(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + c := newReplicaCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + a.replicateOnce(t.Context()) + time.Sleep(replicateEvery * time.Second) + a.replicateOnce(t.Context()) + synctest.Wait() + if a.replicaOf(sb) != "" || len(a.eng.(*fakeEngine).snapSaves) != 0 { + t.Error("a fenced node replicated") + } + }) +} + +func TestReplicaPassRetiresAReplicaWhenThePoolTurnsItOff(t *testing.T) { + c := newReplicaCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + a.fenced.Store(false) + sb := mustClaim(t, a, testKey) + replicateNow(t, a, sb) + ck := a.replicaOf(sb) + + view := *a.view.Load() + view.poolReplicate = nil + a.view.Store(&view) + a.replicateOnce(t.Context()) + waitFor(t, func() bool { return !a.replicating.Load() }) + if _, ok, _ := c.reg.Get(t.Context(), sb.ID); ok || a.replicaOf(sb) != "" || ckExists(t, a, ck) { + t.Errorf("row present %v, replica %q, ck kept %v, want all gone", ok, a.replicaOf(sb), ckExists(t, a, ck)) + } + if _, held := a.byID(sb.ID); !held { + t.Error("turning replication off dropped the claim") + } +} + +func TestReplicaReleaseDropsTheRowTheReplicaAndTheVM(t *testing.T) { + c := newReplicaCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + vm := lockedVMName(sb) + replicateNow(t, a, sb) + ck := a.replicaOf(sb) + + c.reg.SetDown(true) + if err := a.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); !errors.Is(err, ErrArchiveStoreDown) { + t.Fatalf("release with the registry down: %v, want ErrArchiveStoreDown", err) + } + if _, held := a.byID(sb.ID); !held || !ckExists(t, a, ck) { + t.Fatal("a failed release dropped the claim or its replica") + } + c.reg.SetDown(false) + if err := a.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatalf("release: %v", err) + } + if _, ok, _ := c.reg.Get(t.Context(), sb.ID); ok || ckExists(t, a, ck) || !a.eng.(*fakeEngine).removed(vm) { + t.Errorf("row present %v, replica kept %v, VM removed %v, want the row and replica gone and the VM removed", ok, ckExists(t, a, ck), a.eng.(*fakeEngine).removed(vm)) + } + assertReleasedForGood(t, c, sb) +} + +func TestReplicaHibernateKeepsTheRowOnTheLastReplica(t *testing.T) { + c := newReplicaCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + replicateNow(t, a, sb) + before, _, _ := c.reg.Get(t.Context(), sb.ID) + + if err := a.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatalf("hibernate: %v", err) + } + after, ok, _ := c.reg.Get(t.Context(), sb.ID) + if !ok || after.Epoch != before.Epoch || after.Checkpoint != before.Checkpoint || !ckExists(t, a, before.Checkpoint) { + t.Errorf("row %+v present %v after the hibernate, want it untouched on %s", after, ok, before.Checkpoint) + } +} + +func TestReplicaArchiveReplacesTheRowAndDropsTheReplica(t *testing.T) { + c := newReplicaCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + replicateNow(t, a, sb) + replica := a.replicaOf(sb) + before, _, _ := c.reg.Get(t.Context(), sb.ID) + + mustArchive(t, a, sb) + rec, ok, _ := c.reg.Get(t.Context(), sb.ID) + a.mu.Lock() + archiveCk, left, epoch := sb.ArchiveCk, sb.ReplicaCk, sb.ArchiveEpoch + a.mu.Unlock() + if !ok || rec.Checkpoint != archiveCk || rec.Epoch <= before.Epoch || epoch != rec.Epoch || left != "" { + t.Errorf("row %s@%d, claim ck %q epoch %d replica %q, want the archive's row at a new epoch and no replica", rec.Checkpoint, rec.Epoch, archiveCk, epoch, left) + } + if ckExists(t, a, replica) || !ckExists(t, a, archiveCk) { + t.Errorf("replica kept %v, archive ck kept %v, want only the archive's", ckExists(t, a, replica), ckExists(t, a, archiveCk)) + } +} + +func TestReplicaExpiryDropsTheRowThenTheClaim(t *testing.T) { + c := newReplicaCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + vm := lockedVMName(sb) + replicateNow(t, a, sb) + ck := a.replicaOf(sb) + a.mu.Lock() + sb.Deadline = time.Now().Add(-time.Second) + a.mu.Unlock() + + a.reapOnce(t.Context()) + waitFor(t, func() bool { return a.replicaOf(sb) == "" && !ckExists(t, a, ck) }) + if _, ok, _ := c.reg.Get(t.Context(), sb.ID); ok { + t.Error("the row outlived the first reap") + } + if _, held := a.byID(sb.ID); !held { + t.Fatal("the first reap destroyed the claim before its row was dropped") + } + a.reapOnce(t.Context()) + waitFor(t, func() bool { return a.eng.(*fakeEngine).removed(vm) }) + if _, held := a.byID(sb.ID); held { + t.Error("the second reap kept the claim") + } +} + +func TestReplicaRestartKeepsItsRowAndStaysFencedUntilALeaseRenews(t *testing.T) { + c := newReplicaCluster(t) + dirA := t.TempDir() + a := c.node(t, "a", dirA, keepAnHour) + sb := mustClaim(t, a, testKey) + replicateNow(t, a, sb) + ck := a.replicaOf(sb) + if err := a.FlushClaims(); err != nil { + t.Fatalf("flush: %v", err) + } + before, _, _ := c.reg.Get(t.Context(), sb.ID) + + restarted := c.restart(t, a) + if err := restarted.Reconcile(t.Context()); err != nil { + t.Fatalf("reconcile: %v", err) + } + held, ok := restarted.byID(sb.ID) + if !ok { + t.Fatal("the restart dropped a replicated claim whose row is its own") + } + rec, rowOK, _ := c.reg.Get(t.Context(), sb.ID) + if !rowOK || rec.Epoch != before.Epoch || restarted.replicaOf(held) != ck || !ckExists(t, restarted, ck) { + t.Errorf("row %+v present %v, replica %q, ck kept %v, want the row and replica kept", rec, rowOK, restarted.replicaOf(held), ckExists(t, restarted, ck)) + } + if _, _, err := restarted.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrLeaseLost) { + t.Errorf("serving before the first renewal: %v, want ErrLeaseLost", err) + } +} + +func TestReplicaRestartAdoptsTheRowsCheckpointOfAnInterruptedRound(t *testing.T) { + c := newReplicaCluster(t) + dirA := t.TempDir() + a := c.node(t, "a", dirA, keepAnHour) + sb := mustClaim(t, a, testKey) + replicateNow(t, a, sb) + first := a.replicaOf(sb) + if err := a.FlushClaims(); err != nil { + t.Fatalf("flush: %v", err) + } + second := interruptReplica(t, a, sb) + + restarted := c.restart(t, a) + if err := restarted.Reconcile(t.Context()); err != nil { + t.Fatalf("reconcile: %v", err) + } + held, ok := restarted.byID(sb.ID) + if !ok { + t.Fatal("the restart dropped the claim") + } + if restarted.replicaOf(held) != second || !ckExists(t, restarted, second) || ckExists(t, restarted, first) { + t.Errorf("replica %q, new kept %v, old kept %v, want the row's checkpoint %s adopted and the old one reclaimed", restarted.replicaOf(held), ckExists(t, restarted, second), ckExists(t, restarted, first), second) + } + if rec, _, _ := c.reg.Get(t.Context(), sb.ID); rec.Checkpoint != second || rec.Owner != "a" { + t.Errorf("row %s on %s, want a's on %s", rec.Owner, rec.Checkpoint, second) + } +} + +func TestReplicaRestartDropsAClaimAdoptedElsewhere(t *testing.T) { + c := newReplicaCluster(t) + dirA := t.TempDir() + a, b := c.node(t, "a", dirA, keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + replicateNow(t, a, sb) + ck := a.replicaOf(sb) + if err := a.FlushClaims(); err != nil { + t.Fatalf("flush: %v", err) + } + c.expire("a") + if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { + t.Fatalf("adopt: %v %v", adopted, err) + } + + c.renew(t, "a") + restarted := c.restart(t, a) + if err := restarted.Reconcile(t.Context()); err != nil { + t.Fatalf("reconcile: %v", err) + } + if _, held := restarted.byID(sb.ID); held { + t.Error("the restarted node kept a claim another node adopted") + } + if !ckExists(t, b, ck) { + t.Error("the restarted node deleted the adopter's replica") + } +} + +func TestReplicaRestartDropsAClaimTheAdopterWokeAndFinished(t *testing.T) { + c := newReplicaCluster(t) + dirA := t.TempDir() + a, b := c.node(t, "a", dirA, keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + replicateNow(t, a, sb) + if err := a.FlushClaims(); err != nil { + t.Fatalf("flush: %v", err) + } + c.expire("a") + if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { + t.Fatalf("adopt: %v %v", adopted, err) + } + if _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + t.Fatalf("adopter's wake: %v", err) + } + + c.renew(t, "a") + restarted := c.restart(t, a) + if err := restarted.Reconcile(t.Context()); err != nil { + t.Fatalf("reconcile: %v", err) + } + if _, held := restarted.byID(sb.ID); held { + t.Error("the restarted node kept a claim whose adopter woke it and consumed the row") + } + if _, held := b.byID(sb.ID); !held { + t.Error("the adopter lost the claim it woke") + } +} + +func TestReplicaRestartKeepsAClaimWhoseUnconfirmedRowIsGone(t *testing.T) { + c := newReplicaCluster(t) + dirA := t.TempDir() + a := c.node(t, "a", dirA, keepAnHour) + sb := mustClaim(t, a, testKey) + replicateNow(t, a, sb) + ck := a.replicaOf(sb) + rec, _, _ := c.reg.Get(t.Context(), sb.ID) + if err := c.reg.Delete(t.Context(), sb.ID, rec.Epoch); err != nil { + t.Fatalf("delete row: %v", err) + } + a.mu.Lock() + sb.ArchiveEpoch = 0 + js := a.store.set(sb) + a.mu.Unlock() + if err := a.store.commit(js); err != nil { + t.Fatalf("commit: %v", err) + } + + restarted := c.restart(t, a) + if err := restarted.Reconcile(t.Context()); err != nil { + t.Fatalf("reconcile: %v", err) + } + held, ok := restarted.byID(sb.ID) + if !ok || restarted.replicaOf(held) != ck || !ckExists(t, restarted, ck) { + t.Errorf("claim held %v, replica %q, ck kept %v, want the claim and its replica kept", ok, restarted.replicaOf(held), ckExists(t, restarted, ck)) + } +} + +func TestReplicaSweepKeepsTheRowOfAReplicatedClaim(t *testing.T) { + c := newReplicaCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + replicateNow(t, a, sb) + ck := a.replicaOf(sb) + before, _, _ := c.reg.Get(t.Context(), sb.ID) + + a.sweepOrphanArchives(t.Context()) + if rec, ok, _ := c.reg.Get(t.Context(), sb.ID); !ok || rec.Epoch != before.Epoch || !ckExists(t, a, ck) { + t.Errorf("row %+v present %v, replica kept %v, want the sweep to leave a live replicated claim alone", rec, ok, ckExists(t, a, ck)) + } +} + +func TestReplicaSweepRearmsATombstoneItsClaimStillNames(t *testing.T) { + c := newReplicaCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + replicateNow(t, a, sb) + ck := a.replicaOf(sb) + row, _, _ := c.reg.Get(t.Context(), sb.ID) + if _, err := c.reg.Consume(t.Context(), sb.ID, row.Epoch); err != nil { + t.Fatalf("consume: %v", err) + } + + a.sweepOrphanArchives(t.Context()) + rec, ok, _ := c.reg.Get(t.Context(), sb.ID) + if !ok || rec.Consumed || rec.Checkpoint != ck || rec.Owner != "a" || !ckExists(t, a, ck) { + t.Errorf("row %+v present %v, replica kept %v, want a live row of a re-armed on %s", rec, ok, ckExists(t, a, ck), ck) + } +} + +func TestReplicaALeftoverDeleteMarkerSparesTheLiveReplica(t *testing.T) { + c := newReplicaCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + replicateNow(t, a, sb) + ck := a.replicaOf(sb) + if err := a.markArchiveCk(ck); err != nil { + t.Fatalf("mark: %v", err) + } + + a.retryArchiveDeletes(t.Context()) + if !ckExists(t, a, ck) { + t.Error("the delete retry removed a replica its claim still names") + } +} + +func TestReplicaAdoptionWakesTheClaimFromTheLastReplica(t *testing.T) { + c := newReplicaCluster(t) + a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + replicateNow(t, a, sb) + ck := a.replicaOf(sb) + + if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || adopted { + t.Fatalf("adopt while a's lease is live: %v %v, want refused", adopted, err) + } + c.expire("a") + if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { + t.Fatalf("adopt after the lease expired: %v %v, want adopted", adopted, err) + } + moved, _ := b.byID(sb.ID) + if moved.ArchiveCk != ck { + t.Fatalf("adopted claim names %q, want the replica %q", moved.ArchiveCk, ck) + } + if _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + t.Fatalf("adopter's wake: %v", err) + } + if got := b.eng.(*fakeEngine).cloneCount(); got != 1 { + t.Errorf("the wake cloned %d VMs, want the one restored from the replica", got) + } + if _, ok, _ := c.reg.Get(t.Context(), sb.ID); ok || ckExists(t, b, ck) { + t.Error("the adopter's wake left the row or the replica") + } +} + +func newReplicaCluster(t *testing.T) *failoverCluster { + t.Helper() + c := newFailoverCluster(t) + c.replicate = replicateEvery + return c +} + +func replicateNow(t *testing.T, m *Manager, sb *types.Sandbox) { + t.Helper() + if err := m.replicate(t.Context(), sb); err != nil { + t.Fatalf("replicate %s: %v", sb.ID, err) + } +} + +func interruptReplica(t *testing.T, m *Manager, sb *types.Sandbox) string { + t.Helper() + ck, _, err := m.publishCheckpoint(t.Context(), sb, store.CheckpointID(randHex(8)), "replica", sb.Tenant, true) + if err != nil { + t.Fatalf("publish: %v", err) + } + if _, err = m.putArchive(t.Context(), sb, ck.ID); err != nil { + t.Fatalf("put: %v", err) + } + return ck.ID +} diff --git a/sandboxd/pool/view.go b/sandboxd/pool/view.go index c7113edb..1b904bef 100644 --- a/sandboxd/pool/view.go +++ b/sandboxd/pool/view.go @@ -1,6 +1,8 @@ package pool import ( + "time" + "github.com/cocoonstack/sandbox/sandboxd/config" "github.com/cocoonstack/sandbox/sandboxd/egress" "github.com/cocoonstack/sandbox/sandboxd/outbound" @@ -9,20 +11,22 @@ import ( // configView is the config-owned state a reload replaces whole; readers load it once and never see a half-applied reload. type configView struct { - out *outbound.View - poolWarmups map[types.PoolKey][]string - poolTrims map[types.PoolKey]bool - poolStorage map[types.PoolKey]string - diskReserve int64 - diskDirs []string + out *outbound.View + poolWarmups map[types.PoolKey][]string + poolTrims map[types.PoolKey]bool + poolStorage map[types.PoolKey]string + poolReplicate map[types.PoolKey]time.Duration + diskReserve int64 + diskDirs []string } func newConfigView(cfg *config.Config, secrets *egress.SecretStore) *configView { v := &configView{ - out: outbound.NewView(cfg, secrets), - poolWarmups: map[types.PoolKey][]string{}, - poolTrims: map[types.PoolKey]bool{}, - poolStorage: map[types.PoolKey]string{}, + out: outbound.NewView(cfg, secrets), + poolWarmups: map[types.PoolKey][]string{}, + poolTrims: map[types.PoolKey]bool{}, + poolStorage: map[types.PoolKey]string{}, + poolReplicate: map[types.PoolKey]time.Duration{}, } if r := cfg.DiskReserve; r != nil { v.diskReserve, _ = config.StorageBytes(r.Free) @@ -42,6 +46,9 @@ func newConfigView(cfg *config.Config, secrets *egress.SecretStore) *configView if spec.Storage != "" { v.poolStorage[key] = spec.Storage } + if spec.ReplicateSeconds > 0 { + v.poolReplicate[key] = time.Duration(spec.ReplicateSeconds) * time.Second + } } return v } diff --git a/sandboxd/server/port_test.go b/sandboxd/server/port_test.go index 44ccfd8f..89a38f77 100644 --- a/sandboxd/server/port_test.go +++ b/sandboxd/server/port_test.go @@ -104,6 +104,7 @@ func TestPortRelayMapsDialErrors(t *testing.T) { want int }{ {"unknown sandbox or wrong token", pool.ErrUnknownSandbox, http.StatusNotFound}, + {"node lease lost", pool.ErrLeaseLost, http.StatusServiceUnavailable}, {"no guest listener", errors.New("port_forward 49983: not_found"), http.StatusBadGateway}, } for _, tt := range tests { diff --git a/sandboxd/server/preview.go b/sandboxd/server/preview.go index 5529f08c..853fd8af 100644 --- a/sandboxd/server/preview.go +++ b/sandboxd/server/preview.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "encoding/base64" "encoding/json/v2" + "errors" "fmt" "net" "net/http" @@ -131,6 +132,11 @@ func (p *PreviewServer) proxyLocal(w http.ResponseWriter, r *http.Request, claim }, Transport: p.transport, ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) { + if errors.Is(err, pool.ErrLeaseLost) { + w.Header().Set("Retry-After", retryAfterSeconds) + http.Error(w, "lease lost; retry", http.StatusServiceUnavailable) + return + } log.WithFunc("server.proxyLocal").Errorf(r.Context(), err, "proxy %s:%d", claims.ID, claims.Port) http.Error(w, "preview target unreachable", http.StatusBadGateway) }, diff --git a/sandboxd/server/preview_test.go b/sandboxd/server/preview_test.go index 9d5b4ba6..2dfedcb6 100644 --- a/sandboxd/server/preview_test.go +++ b/sandboxd/server/preview_test.go @@ -96,6 +96,24 @@ func TestPreviewRevokedWhenDialFails(t *testing.T) { } } +func TestPreviewAnswers503WhileTheLeaseIsLost(t *testing.T) { + ps := NewPreviewServer("secret", "node:9000", "node:7777", &fakePreviewMgr{ + dial: func(string, uint16) (net.Conn, error) { return nil, pool.ErrLeaseLost }, + }) + ts := httptest.NewServer(ps.Handler()) + t.Cleanup(ts.Close) + token := mintToken(ps, "sb_1", 3000, time.Hour) + + resp, err := http.Get(ts.URL + "/p/" + token + "/") + if err != nil { + t.Fatalf("get: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusServiceUnavailable || resp.Header.Get("Retry-After") == "" { + t.Errorf("status %d retry-after %q, want 503 with a retry hint", resp.StatusCode, resp.Header.Get("Retry-After")) + } +} + func TestPreviewRechecksClaimForEveryRequest(t *testing.T) { guestAddr := newGuestServer(t, func(*http.Request) string { return "guest" }) diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index f49b2e12..be683757 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -57,6 +57,7 @@ var poolErrHTTP = []struct { {pool.ErrTenantStoreDown, http.StatusServiceUnavailable, "tenant store down; retry"}, {pool.ErrPoolStoreDown, http.StatusServiceUnavailable, "pool set store down; retry"}, {pool.ErrArchiveStoreDown, http.StatusServiceUnavailable, "archive registry down; retry"}, + {pool.ErrLeaseLost, http.StatusServiceUnavailable, "lease lost; retry"}, {pool.ErrNoEgress, http.StatusConflict, ""}, {pool.ErrNoEgressHibernate, http.StatusConflict, ""}, {pool.ErrNoEgressFork, http.StatusConflict, ""}, diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index c5de02d2..8703bc83 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -576,6 +576,8 @@ type Sandbox struct { // ArchiveEpoch is the archive registry epoch this node last held for the claim; zero means no row. ArchiveEpoch int64 `json:"archive_epoch,omitzero"` ArchivePending string `json:"archive_pending,omitempty"` + // ReplicaCk names the checkpoint the claim's registry row points at while the claim runs; empty means unreplicated. + ReplicaCk string `json:"replica_ck,omitempty"` // FromCheckpoint names the checkpoint this sandbox branched from; empty otherwise. FromCheckpoint string `json:"from_checkpoint,omitempty"` From c5fb81bcb7d4757047510ff54a72aa31496bf879 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 02:12:29 +0800 Subject: [PATCH 04/36] docs: node lease and running-claim replication Extend the failover section of cluster.md with the node lease, the replication round, restore, lifecycle and self-fencing; add replicate_seconds and the lease to deploy.md; document the lease rule on /owner and the 503 lease-lost answer in sandboxd-api.md. --- docs/cluster.md | 107 +++++++++++++++++++++++++++++++++++-------- docs/deploy.md | 5 +- docs/sandboxd-api.md | 22 +++++---- 3 files changed, 106 insertions(+), 28 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index 08281981..d5a88bf9 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -153,8 +153,8 @@ If a client kept a sandbox's `id` and `token` but lost the owner address entry node, then queries every peer concurrently (`GET /v1/sandboxes/{id}/owner` — the token is both authorization and ownership proof) and returns a handle bound to whichever node confirms -ownership first. With `archive_failover`, a node may adopt an archived claim -whose owner left the mesh and answer for it (see +ownership first. With `archive_failover`, a node may adopt a claim whose +owner's node lease expired and answer for it (see [archived claims across node loss](#archived-claims-across-node-loss)). ```go @@ -167,21 +167,24 @@ sb, err := client.Lookup(ctx, savedID, savedToken) ## Archived claims across node loss A claim lives on one node, so a lost node takes its running and hibernated -claims with it. An archived claim keeps only a pointer on the node; with +claims with it, unless the claim is archived or [replicated](#running-claims-across-node-loss). +An archived claim keeps only a pointer on the node; with [`archive_failover`](deploy.md#configuration) on every node (it needs `meta_store` with a `cell`, an `s3` `checkpoint_store` and a `mesh`), the archive also writes one row to the shared PostgreSQL, scoped by the cell: id, tenant, token SHA-256, owner node, epoch, a consumed flag, checkpoint id, -retention deadline and the claim record without its token. Only archives write -it, so the claim path is unchanged. +retention deadline and the claim record without its token. Only archives and +replication rounds write it, so the claim path is unchanged. - **Adoption.** `Lookup` asks the entry node, then every peer. A node that does - not hold the claim reads the row; when the token hash matches and the owner - has left the mesh (or is this node, with its record gone), it takes the row - with a compare-and-set on the epoch, records the claim as archived, and + not hold the claim reads the row; when the token hash matches, it takes the + row with one compare-and-set on the epoch. The same statement requires the + owner's [node lease](#node-lease) to be expired, unless the owner is this + node (with its record gone). The node records the claim as archived and answers as the owner. Two nodes adopting at once leave one owner; the other - answers 404. A tombstone is never adopted. The next exec or file call wakes the claim from the store as - usual. An owner still in the mesh answers for itself, so no other node adopts. + answers 404. A tombstone is never adopted. The next exec or file call wakes + the claim from the store as usual. An owner with a live lease answers for + itself, and a node that tries to take its row is refused with a 404. A draining node never adopts. Adoption checks no tenant `max_claims` or node `max_claims`: the claim already exists, and a refusal could strand it. - **Fencing.** Every write is a compare-and-set on the row's epoch, and every @@ -209,9 +212,9 @@ it, so the claim path is unchanged. the next archive and the hourly sweep settle it by the same rule before they act on the claim. - **Departed owners and tombstones.** Every hour, on the store retention sweep, - each node reads the rows past their retention deadline, the consumed rows, - and its own rows. An expired row whose owner has left the mesh is purged - checkpoint first, then the row. No node touches another node's tombstone: + each node reads the rows past their retention deadline whose owner's lease + expired, the consumed rows, and its own rows. An expired row of such an owner + is purged checkpoint first, then the row. No node touches another node's tombstone: the owner settles it when it returns, through its sweep or its restart. The epoch compare-and-set lets exactly one node win each, and `archive_delete` is recorded only for a row that was still archived. @@ -227,11 +230,79 @@ it, so the claim path is unchanged. `archive_delete` is in the adopter's or the purging node's. Fold the journals of all nodes by sandbox id. -Mesh liveness decides adoption, so a partitioned owner can lose an archived -claim it still lists; the epoch fences it out. A claim that has entered its -wake counts as running: running and hibernated claims are lost with their -node, and an owner that never returns leaves behind only the tombstones and -checkpoints of the claims it was waking or releasing when it died. +The node lease decides adoption, so a partitioned owner can lose an archived +claim it still lists once its lease runs out; the epoch fences it out. A claim +that has entered its wake counts as running. Running and hibernated claims of a +pool without `replicate_seconds` are lost with their node, and an owner that +never returns leaves behind only the tombstones and checkpoints of the claims it +was waking or releasing when it died. + +### Node lease + +With `archive_failover` every node renews one row of `sandboxd_nodes (cell, +node, lease_until)` every 5 seconds, with a single upsert on the database clock +that covers the whole node: `lease_until = now() + 15 s`. The lease outlives a +missed renewal. The table is created at start when it is missing, and a node +with no row counts as expired. Adoption and the departed-owner sweep consult the +lease, never mesh membership, so a node the mesh lost but the database still +hears keeps its claims, and a node the mesh still lists but the database no +longer hears loses them. The cost is one statement per node every 5 seconds; no +claim path or data plane reads it. + +### Running claims across node loss + +A pool entry with [`replicate_seconds`](deploy.md#configuration) (a `none`-lane +pool on a node with `archive_failover`) snapshots its running claims to the +checkpoint store and keeps each claim's registry row on the latest snapshot: + +- **Round.** Every interval, each running claim of the pool is checkpointed: + cocoon pauses the guest, captures memory and the writable disk, and resumes + it, so the capture is consistent for any disk format and needs no guest + cooperation. The record is published as a replica: hidden like an archive's + wake image, so it is not listed, branched or deleted through the API. Then + the claim's row is put with this node as owner and the replica as checkpoint, + and the previous replica is deleted. A round that fails logs a warning and + retries at the next interval; the old replica and row stay as they were, and + the claim keeps running. A claim gets its first replica one interval after + it is claimed (or after a restart). A hibernated, archived or failed claim, + one whose tenant was removed, and one with volumes (capture refuses them; + logged once per claim) is skipped. Rounds share the maintenance budget with + refill and never take the share refill keeps. A node whose lease is overdue + starts none, and a round whose registry is down fails before it captures. +- **Restore.** A lost node's replica is adopted like an archive: the first + `Lookup` that reaches another node after the lease expires takes the row, + records the claim as archived on the replica, and the next call wakes it + through the archive wake, which restores the guest with its memory and + processes. The claim resumes from its last snapshot: everything after it, and + the connections open at the time, are lost, so the data loss is at most the + interval. +- **Lifecycle.** A release consumes the row, then deletes the replica and the + tombstone. A hibernate leaves the row on the last replica. An archive puts its + own checkpoint over the row and deletes the replica. A lease expiry + with `on_expire` release drops the row and the replica first, and the next reap + destroys the claim; with `on_expire` archive, the archive puts its checkpoint + over the row. Turning `replicate_seconds` + off, by reload or restart, drops the rows and replicas of the pool's claims on + the next pass. A restart keeps the node's own live row for a running claim + and takes the checkpoint a crashed round left on it; a claim whose row another + node holds, or whose row a finished adopter consumed, is dropped with its VM. + The hourly sweep is unchanged apart from the lease rule. +- **Self-fencing.** A node whose renewal is overdue by 12 seconds (the lease + length minus a 3 second margin; one missed renewal never gets there) stops + serving every replicated claim it holds: relay, exec, ports and preview answer `503 lease lost; retry`. The + guest has no NIC on the `none` lane, so it is unreachable except through the + node, and the node destroys nothing. Claims that are not replicated stay + served. When a renewal succeeds again, the node reads the rows of its + replicated claims: those it still owns are served again, and those another + node took, or took and finished, are dropped with their VMs. A node that + starts fenced and lifts the fence with its first renewal. The margin makes + the fence close 3 seconds before another node may adopt, so no moment has two + serving copies, partitions included. Connections already open when the fence + closes are not cut, and end when the client closes them. + +A replica costs, per claim per interval, one capture pause, one upload, one +registry read, one write and one delete; the node's own cost is one scan of its +claims every 5 seconds. The claim path and the data plane gain only a flag read. ## Templates on a cluster diff --git a/docs/deploy.md b/docs/deploy.md index 7191780f..9b970a68 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -143,6 +143,7 @@ gets the intersection of the two. | `capture_trim` (pool entry) | `false` | Trim the guest's copy-on-write disk before a promote or checkpoint of this pool's sandboxes (and of the clones of a template promoted from it), so the record carries only live data: blocks a build deleted — a `COPY` step's archive, a `RUN` step's caches — are no longer captured. sandboxd mounts the disk init resolved from `cocoon.cow` a second time at `/run/sandboxd-trim` through silkd as root, runs `fstrim` and unmounts, under the capture's transition lock (about 15 ms on a warm clone). A failed trim is logged and the capture proceeds, larger. A hibernated or archived sandbox is captured untrimmed. Config-owned like `egress`: `PUT /v1/pools` rejects it | | `egress_classes` | unset | named tenant egress layers: `[{"name": "desk", "egress": {…}, "egress_upstream_env": "DESK_UPSTREAM"}]`. A tenant names one with `egress_class` through the [tenant API](sandboxd-api.md#tenants-v1tenants) and its claims take that policy intersected with the pool's. `egress` is required; `intercept` stays a pool-rule flag. Reloadable: a class's policy reaches live claims on their next request. A class removed while tenants still name it leaves them with no egress and is logged at load and at reload | | `egress_upstream_env` (pool or egress class entry) | unset | names a node environment variable holding the default upstream proxy URL for the pool's or class's claims (class before pool, a claim's own entry before both); needs `egress_upstream`, and the URL must be on its `allow`. Read and checked at startup. Config-owned like `egress`: `PUT /v1/pools` rejects it | +| `replicate_seconds` (pool entry) | 0 (off) | snapshot every running claim of this pool to the checkpoint store this often and point the claim's registry row at the snapshot, so another node restores the claim, memory and processes included, when this node is lost (see [running claims across node loss](cluster.md#running-claims-across-node-loss)). Each round pauses the guest for the length of a memory and disk capture, so the interval is also the data a loss costs. Requires `archive_failover` and a `none`-lane pool, both checked at config load; a claim with volumes is skipped. Config-owned like `egress`: `PUT /v1/pools` rejects it, and a reload applies it (a pool set back to 0 drops its claims' rows and replicas) | | `storage` (pool entry) | unset (cocoon's default, 10G) | size of the copy-on-write disk the pool's golden is cold-booted with, in cocoon's own `--storage` spelling (`"40G"`, `"40GiB"`, `"40Gi"`: Docker/Kubernetes units, all binary) and passed to it verbatim; every clone, fork, checkpoint branch and promoted template of the pool inherits it. At least cocoon's default of 10G. Equal sizes spelled differently build the same golden; a changed size rebuilds the golden on the next build, and claims already out keep their disk. Config-owned like `egress`: `PUT /v1/pools` rejects it | | `max_claims` | 0 (unlimited) | node-wide cap on live claims; claim/fork/branch requests beyond it answer 429 with the pool state unharmed (on a cluster a non-volume claim tries a warm-peer redirect first; a volume claim answers the 429 with no redirect) | | `audit_log` | false | append every relayed frame that opens an RPC — its op + addressing fields, never payloads — to `/audit.jsonl`; continuation `data`, `data_end`, `stdin`, and `stdin_close` frames pass through unrecorded. The file is size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`), plus `method` (`GET`, `CONNECT`, `SOCKS5`, …), `decision` (`allow`, `deny`, `rate_limited`), `secret` (the ref name, never its value), `upstream` (the [upstream proxy](egress.md#upstream-proxies)'s `host:port`, when one carried it), `tenant`, and on an allowed connection `src_addr` and `dst_addr` (the node-side and destination `ip:port` of the upstream TCP connection, the tunnel target through an upstream proxy) on `egress` records, which a relayed frame cannot set; preview accesses record as op `preview`, one per request; a tenant change records as op `tenants` with the `added`, `changed` and `removed` names, never a token. An opening frame whose first line exceeds 4 KiB records as op `oversized` with no addressing fields | @@ -150,7 +151,7 @@ gets the intersection of the two. | `idle_hibernate_seconds` | 0 (off) | node-wide idle policy for unpooled claims (template/checkpoint claims): a none-lane claim is hibernated once it has had no open data-plane connection (relay, buffered exec, preview) and no egress request in flight for this long; the clock restarts when the last connection closes, so a long command is never cut short. The next call that reaches the guest wakes it transparently. Per-pool `idle_hibernate_seconds` does the same for that pool's claims; pooled keys ignore the node-wide value, and egress pools reject it because they cannot resume safely. Opt in deliberately: a wake costs latency and the snapshot, so callers with their own idle logic must not pay twice | | `archive_after_seconds` | 0 (off) | tier below hibernation: a hibernated claim idle this long is checkpointed to the store and its local VM dropped, freeing the node entirely; the next call that reaches the guest restores it transparently (a checkpoint restore's latency) with a fresh lease of the length the claim was granted (the server default of 5m when it asked for none); a lease moves only when a client calls [`/renew`](sandboxd-api.md#post-v1sandboxesidrenew). Keep `idle_hibernate_seconds` under the leases your claims use, or a woken sandbox that idles again is destroyed by the reap before it can hibernate, its archive already consumed by the wake; one still in use when its lease ends is destroyed the same way, unless it claimed `on_expire: archive`, which archives at the lease end whatever this setting. Requires `idle_hibernate_seconds > 0` and must exceed it. Node-wide for unpooled keys; per-pool overrides for that pool | | `archive_delete_after_seconds` | 0 (keep) | purge an archived claim's store checkpoint this long after it was archived; the claim is then gone for good. On S3, chunks and manifests remain until [offline reclamation](#reclaiming-s3-storage). On archive this retention window replaces the live claim deadline; 0 clears the deadline so the archive is kept forever. Same node-wide/per-pool split | -| `archive_failover` | false | let another node adopt this node's archived claims when it leaves the mesh: each archive also writes a row (token SHA-256, owner, epoch, checkpoint id, retention deadline) to the shared PostgreSQL, and a `Lookup` that reaches any node takes the claim over there (see [archived claims across node loss](cluster.md#archived-claims-across-node-loss)). Requires `meta_store` with a `cell` (rows are scoped by it, so meshes sharing one database never adopt from each other), a `checkpoint_store` of kind `s3` and a `mesh`, all checked at config load. With the meta store down, archives, wakes and releases of archived claims fail and retry; running claims never touch it. Restart only | +| `archive_failover` | false | let another node adopt this node's archived claims once its node lease expires, and the running claims of pools with `replicate_seconds`: each archive also writes a row (token SHA-256, owner, epoch, checkpoint id, retention deadline) to the shared PostgreSQL, and a `Lookup` that reaches any node takes the claim over there (see [archived claims across node loss](cluster.md#archived-claims-across-node-loss)). Requires `meta_store` with a `cell` (rows are scoped by it, so meshes sharing one database never adopt from each other), a `checkpoint_store` of kind `s3` and a `mesh`, all checked at config load. The node also renews a lease row every 5 s (see [node lease](cluster.md#node-lease)). With the meta store down, archives, wakes and releases of archived claims fail and retry, and replicated claims stop being served once the lease is overdue (see [running claims](cluster.md#running-claims-across-node-loss)); other running claims never touch it. Restart only | | `mesh` | unset | join a cluster ([Clusters](cluster.md)); unset = single node | | `pools[]` | — | warm pools, keyed by `(template, net, size)`. `warm` defaults to 4 only when both `warm` and `warm_max` are omitted; an explicit `warm: 0` is kept, so `warm: 0` under a `warm_max` starts the pool empty and grows it on demand, and `warm: 0` alone registers the key with no warm VMs (`PUT /v1/pools` never defaults `warm`); `net` is `none` or `egress`; `size` is a tier, below. Retune online without a restart via [`PUT /v1/pools`](sandboxd-api.md#put-v1pools) — omitted pools drain, and their goldens are deleted once they are idle (`warm: 0` keeps the golden). This is the **first-boot seed**: once a node takes a `PUT /v1/pools`, the applied set persists to `/pools.json` and overrides this section on every later boot (a startup log notes it); delete `pools.json` to return to config-owned pools. With a `meta_store` `cell`, the cell's set overrides it instead (see [shared pool set](#shared-pool-set)). Egress stays config-owned either way. A template re-pulled or re-imported under the same ref is picked up within a minute: the pool rebuilds its golden and replaces its warm VMs. See [state ownership](cluster.md#state-ownership) | @@ -710,7 +711,7 @@ validates it with the same code as boot, and applies the reloadable settings all at once or not at all. Every field falls into one of three classes: - **Reloaded**: - - per pool: `egress`, `warmup`, `capture_trim`, `storage`, `egress_upstream_env`, for existing keys and for keys new in the file; + - per pool: `egress`, `warmup`, `capture_trim`, `storage`, `replicate_seconds`, `egress_upstream_env`, for existing keys and for keys new in the file; - `egress_classes`: each class's `egress` and `egress_upstream_env`; - node-wide: `egress_internal_allow`, `secrets`, `egress_upstream`, `egress_usage_bytes`, `egress_conn_rate` and `disk_reserve`. - **Left to their API**: a pool's targets (`warm`, `warm_max`, idle and archive diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index beb32aaf..31162d48 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -578,7 +578,7 @@ is deleted; a pool kept with `warm: 0` keeps its golden. Answers the fresh `GET /v1/info` payload. 400 bad key, negative warm/idle, `warm_max` below `warm`, `idle_hibernate_seconds` on an egress pool, a negative archive duration or an `archive_after_seconds` not above the pool's -`idle_hibernate_seconds`, duplicate pool, or a config-owned `egress`/`warmup`/`capture_trim`/`storage`/`egress_upstream_env` +`idle_hibernate_seconds`, duplicate pool, or a config-owned `egress`/`warmup`/`capture_trim`/`storage`/`replicate_seconds`/`egress_upstream_env` field; 401 bad api token; 409 egress pool on a node without an egress attachment; 503 the `meta_store` cannot take a cell's write. @@ -695,7 +695,8 @@ URL; releasing the sandbox ends it early either way. has no `preview_listen`. The signed token embeds the sandbox id, port, and owner `advertise_addr`, so any node's preview listener can serve it (forwarding to the owner's main listener) and a released sandbox's URL simply stops -resolving — no revocation list. See [deploy](deploy.md#preview-urls). +resolving — no revocation list. A request for a replicated claim answers 503 +`lease lost; retry` while its node's lease is overdue. See [deploy](deploy.md#preview-urls). ## POST /v1/sandboxes/{id}/checkpoint @@ -957,7 +958,10 @@ before the 101, reach silkd in order; the Go and Python SDKs send their first frames that way over plain HTTP, so a proxy in front of a plain-HTTP endpoint must forward them too (Go's `httputil.ReverseProxy` drops them; Caddy forwards them). 426 without the upgrade header, 404 unknown sandbox, 409 a failed -sandbox, 502 guest unreachable. +sandbox, 502 guest unreachable, 503 `lease lost; retry` for a replicated claim +while its node's lease is overdue (see +[running claims](cluster.md#running-claims-across-node-loss)). A relay already +open when the lease turns overdue is not cut. ## GET /v1/sandboxes/{id}/ports/{port} @@ -987,7 +991,8 @@ fork in flight, answers `409` at once; an open passive relay still holds the san 400 a port outside 1-65535; 401 missing bearer token; 404 unknown sandbox or wrong token; 409 a failed sandbox or a passive relay to a paused one; 426 without the upgrade header; 502 when the guest cannot be reached on that port — no listener, or a -hibernated sandbox whose restore failed. The node log carries which. +hibernated sandbox whose restore failed. The node log carries which; 503 +`lease lost; retry` for a replicated claim while its node's lease is overdue. ## POST /v1/sandboxes/{id}/exec @@ -1012,16 +1017,17 @@ does not exist are the caller's, not 502s; 401 missing bearer token; 404 unknown 413 when stdout+stderr exceed 8 MiB; 503 with `Retry-After: 1` when the node's buffered-exec memory is exhausted (64 MiB of output across every buffered exec in flight; the command is killed); 502 guest unreachable or any other -silkd error; 409 a failed sandbox; 504 when the command outlives `timeout_seconds`. A client that hangs up first gets no reply and the command is killed. A hibernated sandbox wakes transparently like on the relay. +silkd error; 409 a failed sandbox; 503 `lease lost; retry` for a replicated claim +while its node's lease is overdue; 504 when the command outlives `timeout_seconds`. A client that hangs up first gets no reply and the command is killed. A hibernated sandbox wakes transparently like on the relay. ## GET /v1/sandboxes/{id}/owner Auth: the sandbox's own token. Answers `{"owner_addr": "host:port"}` when this node owns the sandbox, 404 otherwise. With [`archive_failover`](cluster.md#archived-claims-across-node-loss), a node that -does not hold the sandbox adopts it when its archive row names an owner that -left the mesh, and answers 200; 503 with `Retry-After` when the meta store is -down. With `client_advertise` configured, +does not hold the sandbox adopts it when its row names an owner whose node +lease expired, and answers 200; 503 with `Retry-After` when the meta store is +down. A row whose owner's lease is live is never adopted: 404. With `client_advertise` configured, `owner_addr` is that node's full HTTP(S) origin instead; the same contract applies to claim and fork responses. Used by the SDK's `Lookup` scatter. From 988ea5355301c6c3b58fcec22149550251223f3c Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 02:35:47 +0800 Subject: [PATCH 05/36] pool: close open streams of replicated claims when the fence trips A parked keep-alive relay runs later RPCs back to back, and port and preview streams run for as long as the client keeps them, so none of them passes the claim-resolution gate again. WakeAgentSocket now returns a fence channel that the relay watches, DialPort and PreviewDial wrap their conns with it, and the channel closes when the fence trips; unfence installs a fresh one. An unreplicated claim gets none. Replicated state is an atomic on the sandbox, so resolution takes no mutex. --- docs/cluster.md | 17 ++++- docs/sandboxd-api.md | 4 +- sandboxd/pool/archive.go | 2 + sandboxd/pool/archive_test.go | 36 ++++----- sandboxd/pool/checkpoint_test.go | 2 +- sandboxd/pool/claim.go | 16 ++-- sandboxd/pool/expiry_test.go | 4 +- sandboxd/pool/failover.go | 1 + sandboxd/pool/failover_test.go | 72 +++++++++--------- sandboxd/pool/fork_test.go | 4 +- sandboxd/pool/hibernate.go | 15 ++-- sandboxd/pool/hibernate_test.go | 34 ++++----- sandboxd/pool/idle_test.go | 4 +- sandboxd/pool/lease.go | 31 +++++++- sandboxd/pool/lease_test.go | 101 ++++++++++++++++++++++++-- sandboxd/pool/liveness_test.go | 4 +- sandboxd/pool/pool.go | 4 + sandboxd/pool/promote_test.go | 2 +- sandboxd/pool/reconcile.go | 1 + sandboxd/pool/reconcile_test.go | 2 +- sandboxd/pool/replica.go | 10 ++- sandboxd/pool/replica_test.go | 6 +- sandboxd/pool/telemetry_test.go | 2 +- sandboxd/pool/tenants_test.go | 2 +- sandboxd/pool/volume_capture_test.go | 2 +- sandboxd/pool/volume_mutation_test.go | 2 +- sandboxd/server/exec.go | 2 +- sandboxd/server/exec_test.go | 4 +- sandboxd/server/relay.go | 7 +- sandboxd/server/relay_test.go | 31 ++++++++ sandboxd/server/server.go | 2 +- sandboxd/server/server_test.go | 5 +- sandboxd/types/types.go | 8 ++ 33 files changed, 314 insertions(+), 125 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index d5a88bf9..78f4bdfc 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -297,12 +297,23 @@ checkpoint store and keeps each claim's registry row on the latest snapshot: node took, or took and finished, are dropped with their VMs. A node that starts fenced and lifts the fence with its first renewal. The margin makes the fence close 3 seconds before another node may adopt, so no moment has two - serving copies, partitions included. Connections already open when the fence - closes are not cut, and end when the client closes them. + serving copies, partitions included. When the fence closes, every open + stream of a replicated claim on the node is closed too: the agent relay + (including parked keep-alive relays, whose later RPCs never pass the + resolution gate), the exec call, port relays and preview requests. A client + retries and meets the 503 until the lease is back or the claim moved. + Streams of claims that are not replicated stay open. +- **Availability.** The design chooses consistency over availability for + replicated claims. A registry outage longer than the fence threshold (12 + seconds) stops service of replicated claims on every node until PostgreSQL + returns, because no node can prove it still owns them. Claims that are not + replicated, and archived claims, are unaffected. A replica costs, per claim per interval, one capture pause, one upload, one registry read, one write and one delete; the node's own cost is one scan of its -claims every 5 seconds. The claim path and the data plane gain only a flag read. +claims every 5 seconds. Resolving a claim for a stream gains one atomic load (two for a +replicated claim); a replicated stream also holds one watcher goroutine. +Nothing runs per frame. ## Templates on a cluster diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 31162d48..2d8ff594 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -960,8 +960,8 @@ must forward them too (Go's `httputil.ReverseProxy` drops them; Caddy forwards them). 426 without the upgrade header, 404 unknown sandbox, 409 a failed sandbox, 502 guest unreachable, 503 `lease lost; retry` for a replicated claim while its node's lease is overdue (see -[running claims](cluster.md#running-claims-across-node-loss)). A relay already -open when the lease turns overdue is not cut. +[running claims](cluster.md#running-claims-across-node-loss)). A relay of a +replicated claim already open when the lease turns overdue is closed. ## GET /v1/sandboxes/{id}/ports/{port} diff --git a/sandboxd/pool/archive.go b/sandboxd/pool/archive.go index 9d092bf1..f2798d87 100644 --- a/sandboxd/pool/archive.go +++ b/sandboxd/pool/archive.go @@ -146,6 +146,7 @@ func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error { } snap, vmName, sock, prevDeadline, prevPending, replica := sb.HibernateSnap, sb.VMName, sb.VsockSocket, sb.Deadline, sb.ArchivePending, sb.ReplicaCk sb.ArchiveCk, sb.ArchiveEpoch, sb.ArchivePending, sb.ReplicaCk = ck.ID, epoch, "", "" + sb.SyncReplicated() sb.HibernateSnap, sb.VsockSocket, sb.VMName = "", "", "" sb.Deadline = m.retentionDeadline(sb.Key) js := m.store.set(sb) @@ -157,6 +158,7 @@ func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error { // a Release that landed meanwhile already deleted the claim; re-setting it would resurrect it if m.claimed[sb.ID] == sb { sb.ArchiveCk, sb.ArchiveEpoch, sb.ArchivePending, sb.ReplicaCk = "", 0, prevPending, replica + sb.SyncReplicated() sb.HibernateSnap, sb.VsockSocket, sb.VMName = snap, sock, vmName sb.Deadline = prevDeadline rb = m.store.set(sb) diff --git a/sandboxd/pool/archive_test.go b/sandboxd/pool/archive_test.go index 9bacdd2a..c9ffeee9 100644 --- a/sandboxd/pool/archive_test.go +++ b/sandboxd/pool/archive_test.go @@ -46,7 +46,7 @@ func TestArchivePublishWindowPinsCheckpoint(t *testing.T) { if err := <-archived; err != nil { t.Fatalf("archive: %v", err) } - if _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake archived: %v", err) } } @@ -82,7 +82,7 @@ func TestArchiveWakeRehibernateWindowAborts(t *testing.T) { go func() { archived <- m.archive(t.Context(), sb) }() ck := <-stall.published - if _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake mid-publish: %v", err) } if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { @@ -110,7 +110,7 @@ func TestArchiveWakeRehibernateWindowAborts(t *testing.T) { if dropped { t.Error("abort dropped the newer hibernate snapshot") } - if _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake after aborted archive: %v", err) } } @@ -144,7 +144,7 @@ func TestArchiveCkHiddenAcrossNodes(t *testing.T) { if !ckExists(t, mA, ck) { t.Fatal("peer TTL sweep deleted the archive wake image") } - if _, _, err := mA.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := mA.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("owner wake after peer sweep: %v", err) } } @@ -199,7 +199,7 @@ func TestClaimCheckpointRefusesArchive(t *testing.T) { if _, err := m.ClaimCheckpoint(t.Context(), sb.ArchiveCk, ClaimOptions{}); !errors.Is(err, ErrUnknownCheckpoint) { t.Errorf("branch from archive ck: %v, want ErrUnknownCheckpoint", err) } - if _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("owner wake: %v", err) } } @@ -234,7 +234,7 @@ func TestArchiveLifecycleRoundTrip(t *testing.T) { } waitFor(t, func() bool { return eng.removed(origVM) }) - sock, _, err := m.WakeAgentSocket(t.Context(), id, token) + sock, _, _, err := m.WakeAgentSocket(t.Context(), id, token) if err != nil { t.Fatalf("wake archived: %v", err) } @@ -308,7 +308,7 @@ func TestArchiveTTLSweepSparesLiveCheckpoint(t *testing.T) { if !ckExists(t, m, ck) { t.Fatal("TTL sweep deleted the checkpoint backing a live archived sandbox") } - if _, _, err := m.WakeAgentSocket(t.Context(), id, token); err != nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), id, token); err != nil { t.Fatalf("wake after sweep: %v", err) } } @@ -332,7 +332,7 @@ func TestArchiveKeepsForeverWhenDeleteZero(t *testing.T) { if archivedCount(m) != 1 || !ckExists(t, m, ck) { t.Fatal("reap purged a keep-forever archived sandbox") } - if _, _, err := m.WakeAgentSocket(t.Context(), id, token); err != nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), id, token); err != nil { t.Fatalf("wake keep-forever archive: %v", err) } } @@ -379,7 +379,7 @@ func TestArchiveWakeRegrantsTheClaimsLease(t *testing.T) { t.Fatalf("claim: %v", err) } mustArchive(t, m, sb) - if _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake: %v", err) } m.mu.Lock() @@ -479,7 +479,7 @@ func TestArchiveWakeDeleteFailureRetries(t *testing.T) { failed := &failingDeleteStore{Store: base, attempts: make(chan string, 1)} m.ckpts = failed - if _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake: %v", err) } select { @@ -513,7 +513,7 @@ func TestArchiveWakeRequiresDeleteMarker(t *testing.T) { } removesBefore := len(eng.removedNames()) - if _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err == nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err == nil { t.Fatal("wake succeeded with an unwritable marker dir") } m.mu.Lock() @@ -533,7 +533,7 @@ func TestArchiveWakeRequiresDeleteMarker(t *testing.T) { if err := os.Remove(blocker); err != nil { t.Fatalf("unblock marker dir: %v", err) } - if _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake after heal: %v", err) } if ckExists(t, m, ck) || archiveCkMarked(m, ck) { @@ -598,7 +598,7 @@ func TestArchiveRemovalCommitPinsCheckpoint(t *testing.T) { t.Error("retry deleted the checkpoint restored by rollback") } healStore(t, m) - if _, _, err := m.WakeAgentSocket(t.Context(), id, token); err != nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), id, token); err != nil { t.Fatalf("wake after rollback: %v", err) } }) @@ -664,7 +664,7 @@ func TestArchiveDeleteRetryRechecksWakeRollback(t *testing.T) { }) woke := make(chan error, 1) go func() { - _, _, err := m.WakeAgentSocket(t.Context(), id, token) + _, _, _, err := m.WakeAgentSocket(t.Context(), id, token) woke <- err }() waitFor(t, func() bool { @@ -699,7 +699,7 @@ func TestArchiveDeleteRetryRechecksWakeRollback(t *testing.T) { t.Fatal("retry deleted the archive restored by wake rollback") } healStore(t, m) - if _, _, err := m.WakeAgentSocket(t.Context(), id, token); err != nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), id, token); err != nil { t.Fatalf("wake after rollback: %v", err) } if ckExists(t, m, ck) || archiveCkMarked(m, ck) { @@ -743,7 +743,7 @@ func TestDeleteCheckpointCannotBrickArchive(t *testing.T) { if !ckExists(t, m, ck) { t.Fatal("DeleteCheckpoint removed the archive image backing a live sandbox") } - if _, _, err := m.WakeAgentSocket(t.Context(), id, token); err != nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), id, token); err != nil { t.Fatalf("wake after delete attempt: %v", err) } } @@ -811,7 +811,7 @@ func TestWakeArchivedRollsBackOnPersistFailure(t *testing.T) { ck := sb.ArchiveCk breakStore(t, m) - if _, _, err := m.WakeAgentSocket(t.Context(), id, token); err == nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), id, token); err == nil { t.Fatal("wake succeeded despite a persist failure") } m.mu.Lock() @@ -889,7 +889,7 @@ func TestArchiveWakeEvictsRecLock(t *testing.T) { if err := m.archive(t.Context(), sb); err != nil { t.Fatalf("archive: %v", err) } - if _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake archived: %v", err) } if got := lockCount(m); got != base { diff --git a/sandboxd/pool/checkpoint_test.go b/sandboxd/pool/checkpoint_test.go index bb38cd61..447db314 100644 --- a/sandboxd/pool/checkpoint_test.go +++ b/sandboxd/pool/checkpoint_test.go @@ -75,7 +75,7 @@ func TestCheckpointExportLetsADialReachTheRunningSource(t *testing.T) { }) dialed := make(chan error, 1) go func() { - _, release, err := m.WakeAgentSocket(t.Context(), src.ID, src.Token) + _, _, release, err := m.WakeAgentSocket(t.Context(), src.ID, src.Token) if err == nil { release() } diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index 11dc0add..614f0e8d 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -226,8 +226,9 @@ func (m *Manager) dialPort(ctx context.Context, id string, cred Cred, port uint1 if !ok { return nil, ErrUnknownSandbox } - if m.leaseFenced(sb) { - return nil, ErrLeaseLost + fenced, err := m.fenceWatch(sb) + if err != nil { + return nil, err } sb.Touch() m.recordAudit(ctx, id, auditFrame{Op: op, Port: port}) @@ -242,7 +243,8 @@ func (m *Manager) dialPort(ctx context.Context, id string, cred Cred, port uint1 sb.Unhold() return nil, err } - return &heldConn{Conn: conn, release: sync.OnceFunc(sb.Unhold)}, nil + stop := CloseOnFence(fenced, conn) + return &heldConn{Conn: conn, release: sync.OnceFunc(func() { stop(); sb.Unhold() })}, nil } func (m *Manager) dialPassive(ctx context.Context, id string, port uint16) (net.Conn, error) { @@ -250,8 +252,9 @@ func (m *Manager) dialPassive(ctx context.Context, id string, port uint16) (net. if !ok { return nil, ErrUnknownSandbox } - if m.leaseFenced(sb) { - return nil, ErrLeaseLost + fenced, err := m.fenceWatch(sb) + if err != nil { + return nil, err } if !sb.Transition.TryLock() { return nil, ErrPaused @@ -274,7 +277,8 @@ func (m *Manager) dialPassive(ctx context.Context, id string, port uint16) (net. sb.UnholdIdle() return nil, err } - return &heldConn{Conn: conn, release: sync.OnceFunc(sb.UnholdIdle)}, nil + stop := CloseOnFence(fenced, conn) + return &heldConn{Conn: conn, release: sync.OnceFunc(func() { stop(); sb.UnholdIdle() })}, nil } // releaseResolved re-checks under m.mu that sb is still the live claim: no double teardown. diff --git a/sandboxd/pool/expiry_test.go b/sandboxd/pool/expiry_test.go index 7e2e0ec9..a0bbee2f 100644 --- a/sandboxd/pool/expiry_test.go +++ b/sandboxd/pool/expiry_test.go @@ -29,7 +29,7 @@ func TestExpiryArchiveHibernatesAndArchivesARunningClaim(t *testing.T) { if ck == "" || !deadline.IsZero() || !live || action != types.ExpireArchive { t.Fatalf("round %d: archive %q deadline %v live %t action %q, want an archive kept forever with the action", round, ck, deadline, live, action) } - if _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("round %d: wake: %v", round, err) } m.mu.Lock() @@ -175,7 +175,7 @@ func TestExpiryArchiveWakeBeforeTheExportGrantsAFreshLease(t *testing.T) { if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("hibernate: %v", err) } - if _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake: %v", err) } m.mu.Lock() diff --git a/sandboxd/pool/failover.go b/sandboxd/pool/failover.go index 98f918c9..d36d89eb 100644 --- a/sandboxd/pool/failover.go +++ b/sandboxd/pool/failover.go @@ -458,6 +458,7 @@ func (m *Manager) fenceArchived(ctx context.Context, claims map[string]*types.Sa drop = true case ok && !rec.Consumed: sb.ReplicaCk, sb.ArchiveEpoch = rec.Checkpoint, rec.Epoch + sb.SyncReplicated() } case sb.ArchivePending != "" || ok && !rec.Consumed: moved, err := m.settleIntent(ctx, id, sb.ArchivePending, rec, ok) diff --git a/sandboxd/pool/failover_test.go b/sandboxd/pool/failover_test.go index 2b612e96..185492b3 100644 --- a/sandboxd/pool/failover_test.go +++ b/sandboxd/pool/failover_test.go @@ -79,7 +79,7 @@ func TestFailoverOwnerWakeTakesThenDeletesTheRow(t *testing.T) { mustArchive(t, a, sb) ck := sb.ArchiveCk - if _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake: %v", err) } if _, ok, _ := c.reg.Get(t.Context(), sb.ID); ok || c.reg.Deletes() != 1 { @@ -100,7 +100,7 @@ func TestFailoverOwnerWakeWithTheRegistryDownFailsClosed(t *testing.T) { sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) c.reg.SetDown(true) - if _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrArchiveStoreDown) { + if _, _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrArchiveStoreDown) { t.Fatalf("wake: %v, want ErrArchiveStoreDown", err) } a.mu.Lock() @@ -126,7 +126,7 @@ func TestFailoverAdoptionMovesTheClaimAndTheOldOwnerLetsGo(t *testing.T) { t.Errorf("row %s@%d, want b@2", rec.Owner, rec.Epoch) } c.renew(t, "a") - if _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrUnknownSandbox) { + if _, _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrUnknownSandbox) { t.Errorf("old owner's wake: %v, want ErrUnknownSandbox", err) } if _, held := a.byID(sb.ID); held { @@ -135,7 +135,7 @@ func TestFailoverAdoptionMovesTheClaimAndTheOldOwnerLetsGo(t *testing.T) { if !ckExists(t, b, ck) { t.Fatal("the old owner deleted the adopter's checkpoint") } - if _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("adopter's wake: %v", err) } if _, ok, _ := c.reg.Get(t.Context(), sb.ID); ok || ckExists(t, b, ck) { @@ -243,7 +243,7 @@ func TestFailoverRestartDropsAClaimAdoptedElsewhere(t *testing.T) { if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { t.Fatalf("adopt: %v %v", adopted, err) } - if _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("adopter's wake: %v", err) } moved, _ := b.byID(sb.ID) @@ -261,7 +261,7 @@ func TestFailoverRestartDropsAClaimAdoptedElsewhere(t *testing.T) { if !ckExists(t, b, adopterCk) { t.Fatal("the restarted node deleted the adopter's checkpoint") } - if _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Errorf("adopter's wake after the restart: %v", err) } } @@ -459,7 +459,7 @@ func TestFailoverAdoptionMidWakeFencesTheOwnerOut(t *testing.T) { } }} - if _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrUnknownSandbox) { + if _, _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrUnknownSandbox) { t.Fatalf("owner's wake after a mid-wake adoption: %v, want ErrUnknownSandbox", err) } eng := a.eng.(*fakeEngine) @@ -485,7 +485,7 @@ func TestFailoverWakeRollbackRecoversOnTheNextWake(t *testing.T) { sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) breakStore(t, a) - if _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err == nil { + if _, _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err == nil { t.Fatal("wake with claims.json unwritable succeeded") } a.mu.Lock() @@ -495,7 +495,7 @@ func TestFailoverWakeRollbackRecoversOnTheNextWake(t *testing.T) { t.Fatalf("row %+v present %v, want this node's row for the rolled-back archive", rec, ok) } healStore(t, a) - if _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake after the store heals: %v", err) } } @@ -590,13 +590,13 @@ func TestFailoverWakeAfterALostConsumeReplyKeepsTheClaim(t *testing.T) { ck := sb.ArchiveCk a.archives = &hookRegistry{Registry: c.reg, loseConsumeReply: true} - if _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrArchiveStoreDown) { + if _, _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrArchiveStoreDown) { t.Fatalf("wake whose consume reply was lost: %v, want ErrArchiveStoreDown", err) } if rec, _, _ := c.reg.Get(t.Context(), sb.ID); !rec.Consumed { t.Fatal("setup: the consume did not land") } - if _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("retried wake: %v, want the claim re-armed from its own tombstone", err) } if _, held := a.byID(sb.ID); !held || ckExists(t, a, ck) { @@ -629,7 +629,7 @@ func TestFailoverRestartAfterAConsumeWithoutCommitKeepsTheClaim(t *testing.T) { if rec, ok, _ := c.reg.Get(t.Context(), sb.ID); !ok || rec.Owner != "a" || rec.Consumed || held == nil || held.ArchiveEpoch != rec.Epoch { t.Fatalf("row %+v present %v, held %v, want the tombstone re-armed at the claim's epoch", rec, ok, held != nil) } - if _, _, err = restarted.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err = restarted.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Errorf("wake after the restart: %v", err) } } @@ -647,7 +647,7 @@ func TestFailoverAnArchiveWokenElsewhereIsDropped(t *testing.T) { if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { t.Fatalf("adopt: %v %v", adopted, err) } - if _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("adopter's wake: %v", err) } c.renew(t, "a") @@ -659,7 +659,7 @@ func TestFailoverAnArchiveWokenElsewhereIsDropped(t *testing.T) { if _, held := restarted.byID(sb.ID); held { t.Error("the restart kept a claim whose checkpoint an adopter consumed") } - if _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrUnknownSandbox) { + if _, _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrUnknownSandbox) { t.Errorf("old owner's wake: %v, want ErrUnknownSandbox", err) } if _, ok, _ := c.reg.Get(t.Context(), sb.ID); ok { @@ -681,7 +681,7 @@ func TestFailoverALeftoverDeleteMarkerSparesTheAdoptersCheckpoint(t *testing.T) t.Fatalf("adopt: %v %v", adopted, err) } c.renew(t, "a") - if _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrUnknownSandbox) { + if _, _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrUnknownSandbox) { t.Fatalf("old owner's wake: %v, want ErrUnknownSandbox", err) } a.retryArchiveDeletes(t.Context()) @@ -755,7 +755,7 @@ func TestFailoverARepeatedAdoptionOnOneNodeKeepsOneClaim(t *testing.T) { if rec.Owner != "b" || epoch != rec.Epoch { t.Errorf("row %s@%d, local epoch %d, want the claim at the row's epoch", rec.Owner, rec.Epoch, epoch) } - if _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Errorf("wake after repeated adoptions: %v", err) } } @@ -774,7 +774,7 @@ func TestFailoverRestartInsideAnAdoptersWakeDropsTheClaim(t *testing.T) { t.Fatalf("adopt: %v %v", adopted, err) } b.ckpts = &failingDeleteStore{Store: b.ckpts, attempts: make(chan string, 8)} - if _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("adopter's wake: %v", err) } if !ckExists(t, b, ck) { @@ -939,7 +939,7 @@ func TestFailoverRestartLeavesItsOwnTombstoneToTheSweep(t *testing.T) { sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) a.ckpts = &failingDeleteStore{Store: a.ckpts, attempts: make(chan string, 8)} - if _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake: %v", err) } if err := a.FlushClaims(); err != nil { @@ -972,7 +972,7 @@ func TestFailoverSweepLeavesADepartedOwnersTombstoneUntilItReturns(t *testing.T) } healthy := b.ckpts b.ckpts = &failingDeleteStore{Store: healthy, attempts: make(chan string, 8)} - if _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("adopter's wake: %v", err) } before, _, _ := c.reg.Get(t.Context(), sb.ID) @@ -998,7 +998,7 @@ func TestFailoverSweepPurgesItsOwnTombstoneNoClaimHolds(t *testing.T) { ck := sb.ArchiveCk healthy := a.ckpts a.ckpts = &failingDeleteStore{Store: healthy, attempts: make(chan string, 8)} - if _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake: %v", err) } a.ckpts = healthy @@ -1035,7 +1035,7 @@ func TestFailoverSweepSparesATombstoneItsClaimStillHolds(t *testing.T) { sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) a.archives = &hookRegistry{Registry: c.reg, loseConsumeReply: true} - if _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrArchiveStoreDown) { + if _, _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrArchiveStoreDown) { t.Fatalf("wake whose consume reply was lost: %v, want ErrArchiveStoreDown", err) } @@ -1043,7 +1043,7 @@ func TestFailoverSweepSparesATombstoneItsClaimStillHolds(t *testing.T) { if _, ok, _ := c.reg.Get(t.Context(), sb.ID); !ok || !ckExists(t, a, sb.ArchiveCk) { t.Fatal("the sweep purged a tombstone whose archived claim still holds the checkpoint") } - if _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Errorf("retried wake after the sweep: %v", err) } } @@ -1054,7 +1054,7 @@ func TestFailoverATombstoneIsNotAdopted(t *testing.T) { sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) a.ckpts = &failingDeleteStore{Store: a.ckpts, attempts: make(chan string, 8)} - if _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake: %v", err) } c.expire("a") @@ -1095,7 +1095,7 @@ func TestFailoverADropClearsTheMarkerBeforeTheCheckpointIsUnpinned(t *testing.T) } } }) - _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token) + _, _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token) close(done) wg.Wait() if !errors.Is(err, ErrUnknownSandbox) { @@ -1113,7 +1113,7 @@ func TestFailoverReleaseAfterALostTakeReplyLeavesNoLiveRow(t *testing.T) { sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) a.archives = &hookRegistry{Registry: c.reg, loseTakeReply: true} - if _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrArchiveStoreDown) { + if _, _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrArchiveStoreDown) { t.Fatalf("wake whose take reply was lost: %v, want ErrArchiveStoreDown", err) } if err := a.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { @@ -1128,11 +1128,11 @@ func TestFailoverReleaseAfterALostRearmReplyLeavesNoLiveRow(t *testing.T) { sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) a.archives = &hookRegistry{Registry: c.reg, loseConsumeReply: true} - if _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrArchiveStoreDown) { + if _, _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrArchiveStoreDown) { t.Fatalf("wake whose consume reply was lost: %v, want ErrArchiveStoreDown", err) } a.archives = &hookRegistry{Registry: c.reg, losePutReply: true} - if _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrArchiveStoreDown) { + if _, _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrArchiveStoreDown) { t.Fatalf("wake whose re-arm reply was lost: %v, want ErrArchiveStoreDown", err) } if rec, _, _ := c.reg.Get(t.Context(), sb.ID); rec.Consumed { @@ -1212,7 +1212,7 @@ func TestFailoverSweepRearmsATombstoneItsArchivedClaimHolds(t *testing.T) { sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) a.archives = &hookRegistry{Registry: c.reg, loseConsumeReply: true} - if _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrArchiveStoreDown) { + if _, _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrArchiveStoreDown) { t.Fatalf("wake whose consume reply was lost: %v, want ErrArchiveStoreDown", err) } @@ -1277,7 +1277,7 @@ func TestFailoverRestartAfterAConsumeWhileAPeerSweptKeepsTheClaim(t *testing.T) if _, held := restarted.byID(sb.ID); !held { t.Fatal("the restart dropped a claim whose only copy a peer swept") } - if _, _, err = restarted.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err = restarted.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Errorf("wake after the restart: %v", err) } } @@ -1346,7 +1346,7 @@ func TestFailoverAWakeKeepsItsFenceAgainstThisNodesOwnAdoption(t *testing.T) { waitFor(t, func() bool { rec, _, _ := c.reg.Get(t.Context(), sb.ID); return rec.Epoch != taken.Epoch }) }} - if _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake raced by this node's own adoption: %v, want it to commit", err) } if !<-second { @@ -1474,7 +1474,7 @@ func TestFailoverRestartDropsAClaimWhoseInterruptedArchiveWasConsumedElsewhere(t if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { t.Fatalf("adopt: %v %v", adopted, err) } - if _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("adopter's wake: %v", err) } if _, ok, _ := c.reg.Get(t.Context(), sb.ID); ok || ckExists(t, b, ck) { @@ -1587,7 +1587,7 @@ func TestFailoverAnOwnerReturningWithItsWakeCommittedPurgesItsTombstone(t *testi mustArchive(t, a, sb) ck := sb.ArchiveCk a.ckpts = &failingDeleteStore{Store: a.ckpts, attempts: make(chan string, 8)} - if _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake: %v", err) } if err := a.FlushClaims(); err != nil { @@ -1625,7 +1625,7 @@ func TestFailoverRestartDropsAClaimConsumedWhileItSettlesItsInterruptedArchive(t if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { t.Errorf("adopt during the restart: %v %v, want adopted", adopted, err) } - if _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Errorf("adopter's wake during the restart: %v", err) } }} @@ -1751,7 +1751,7 @@ func TestFailoverAWakeSettlesALostArchivePutReplyFirst(t *testing.T) { t.Fatalf("archive whose put reply was lost: %v, want ErrArchiveStoreDown", err) } a.archives = c.reg - if _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake: %v", err) } a.mu.Lock() @@ -1780,7 +1780,7 @@ func TestFailoverAWakeWithoutARegistryKeepsItsPendingIntent(t *testing.T) { t.Fatalf("archive whose put reply was lost: %v, want ErrArchiveStoreDown", err) } a.archives = nil - if _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake with failover off: %v", err) } if hibernated(a) != 0 { @@ -1806,7 +1806,7 @@ func TestFailoverAReleaseFindsItsUnsettledArchiveTakenElsewhere(t *testing.T) { if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { t.Fatalf("adopt: %v %v", adopted, err) } - if _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("adopter's wake: %v", err) } c.renew(t, "a") diff --git a/sandboxd/pool/fork_test.go b/sandboxd/pool/fork_test.go index a74d69de..fe79a870 100644 --- a/sandboxd/pool/fork_test.go +++ b/sandboxd/pool/fork_test.go @@ -130,7 +130,7 @@ func TestForkFromHibernatedUsesWakeImage(t *testing.T) { t.Error("hibernate snapshot dropped by fork — the parent could never wake") } - if _, _, err := m.WakeAgentSocket(t.Context(), parent.ID, parent.Token); err != nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), parent.ID, parent.Token); err != nil { t.Fatalf("wake after fork: %v", err) } } @@ -218,7 +218,7 @@ func TestReconcileSweepsOrphanSnapshots(t *testing.T) { } } - if _, _, err := m2.WakeAgentSocket(t.Context(), parent.ID, parent.Token); err != nil { + if _, _, _, err := m2.WakeAgentSocket(t.Context(), parent.ID, parent.Token); err != nil { t.Fatalf("wake after reconcile: %v", err) } } diff --git a/sandboxd/pool/hibernate.go b/sandboxd/pool/hibernate.go index 35108fb8..478b6aa2 100644 --- a/sandboxd/pool/hibernate.go +++ b/sandboxd/pool/hibernate.go @@ -24,23 +24,24 @@ func (m *Manager) Hibernate(ctx context.Context, id string, cred Cred) error { return m.hibernateLocked(ctx, sb) } -// WakeAgentSocket resolves a data-plane connection's guest socket; the returned func ends the hold it takes. -func (m *Manager) WakeAgentSocket(ctx context.Context, id, token string) (string, func(), error) { +// WakeAgentSocket resolves a data-plane connection's guest socket; the returned func ends the hold it takes, and the channel closes when the stream must end. +func (m *Manager) WakeAgentSocket(ctx context.Context, id, token string) (string, <-chan struct{}, func(), error) { sb, ok := m.claim(id, token) if !ok { - return "", nil, ErrUnknownSandbox + return "", nil, nil, ErrUnknownSandbox } - if m.leaseFenced(sb) { - return "", nil, ErrLeaseLost + fenced, err := m.fenceWatch(sb) + if err != nil { + return "", nil, nil, err } sb.Touch() sb.Hold() sock, err := m.wakeResolved(ctx, sb) if err != nil { sb.Unhold() - return "", nil, err + return "", nil, nil, err } - return sock, sb.Unhold, nil + return sock, fenced, sb.Unhold, nil } // hibernateLocked is Hibernate's body; the caller holds sb.Transition. diff --git a/sandboxd/pool/hibernate_test.go b/sandboxd/pool/hibernate_test.go index 72069820..1d355272 100644 --- a/sandboxd/pool/hibernate_test.go +++ b/sandboxd/pool/hibernate_test.go @@ -53,13 +53,13 @@ func TestWakePersistFailureSurfaces(t *testing.T) { broken := filepath.Join(t.TempDir(), "gone", "claims.json") m.store.path = broken - if _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err == nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err == nil { t.Fatal("wake reported success despite a persist failure") } if eng.snapRemoved(snap) { t.Error("wake dropped the snapshot the lagging journal references") } - if sock, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil || sock == "" { + if sock, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil || sock == "" { t.Fatalf("fast-path wake of the awake VM: %q, %v", sock, err) } if eng.snapRemoved(snap) { @@ -70,7 +70,7 @@ func TestWakePersistFailureSurfaces(t *testing.T) { } waitFor(t, func() bool { - if _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("fast-path wake: %v", err) } return eng.snapRemoved(snap) @@ -152,7 +152,7 @@ func TestWakeRestoreErrorStopsAndRetries(t *testing.T) { snap := eng.hibernates[0] eng.restoreErr = errors.New("restore timeout") - if _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err == nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err == nil { t.Fatal("wake reported success despite a restore error") } if !slices.Contains(eng.stops, sb.VMName) || !eng.stopped[sb.VMName] { @@ -168,7 +168,7 @@ func TestWakeRestoreErrorStopsAndRetries(t *testing.T) { t.Fatalf("journal %+v, want HibernateSnap=%q kept for the retry", got[sb.ID], snap) } eng.restoreErr = nil - sock, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) + sock, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) if err != nil || sock == "" { t.Fatalf("wake retry after restore recovered: %q, %v", sock, err) } @@ -208,7 +208,7 @@ func TestRetryResolvesDanglingIntent(t *testing.T) { if n := m.Counters().Hibernates; n != 1 { t.Errorf("hibernates billed=%d, want 1 (the adopted transition is recorded)", n) } - if _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake after resolve: %v", err) } if len(eng.restores) != 1 { @@ -229,7 +229,7 @@ func TestWakeResolvesDanglingIntent(t *testing.T) { eng.hibernateErr = nil eng.snapListErr = nil - sock, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) + sock, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) if err != nil { t.Fatalf("wake: %v", err) } @@ -263,7 +263,7 @@ func TestResolvePendingSnapReleaseRaceDropsOrphan(t *testing.T) { callsBefore := eng.snapListCalls() wakeErr := make(chan error, 1) go func() { - _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) + _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) wakeErr <- err }() waitFor(t, func() bool { return eng.snapListCalls() > callsBefore }) @@ -321,7 +321,7 @@ func TestWakeProbeFailureStopsAndRetries(t *testing.T) { snap := eng.hibernates[0] eng.probeErr = errors.New("probe timeout") - if _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err == nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err == nil { t.Fatal("wake reported success despite a probe timeout") } if !slices.Contains(eng.stops, sb.VMName) || !eng.stopped[sb.VMName] { @@ -337,7 +337,7 @@ func TestWakeProbeFailureStopsAndRetries(t *testing.T) { t.Fatalf("journal %+v, want HibernateSnap=%q kept for the retry", got[sb.ID], snap) } eng.probeErr = nil - sock, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) + sock, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) if err != nil || sock == "" { t.Fatalf("wake retry after the probe recovered: %q, %v", sock, err) } @@ -356,7 +356,7 @@ func TestFlushClaimsClosesShutdownWindow(t *testing.T) { broken := filepath.Join(t.TempDir(), "gone", "claims.json") m.store.path = broken - if _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err == nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err == nil { t.Fatal("wake reported success despite a persist failure") } if err := os.MkdirAll(filepath.Dir(broken), 0o750); err != nil { @@ -397,7 +397,7 @@ func TestHibernateWakeCycle(t *testing.T) { t.Errorf("snapshot name %q, want %s prefix", hibSnap, hibernatePrefix) } - sock, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) + sock, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) if err != nil { t.Fatalf("WakeAgentSocket: %v", err) } @@ -412,7 +412,7 @@ func TestHibernateWakeCycle(t *testing.T) { t.Errorf("hibernated count %d after wake, want 0", g.Hibernated) } - if _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("fast-path WakeAgentSocket: %v", err) } if len(eng.restores) != 1 { @@ -431,7 +431,7 @@ func TestWakeDropsSnapshotOffReturnPath(t *testing.T) { } eng.snapRemoveStall = make(chan struct{}) - sock, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) + sock, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) if err != nil { t.Fatalf("wake: %v", err) } @@ -455,7 +455,7 @@ func TestWakeFailureKeepsHibernated(t *testing.T) { } eng.restoreErr = errors.New("restore boom") - if _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err == nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err == nil { t.Fatal("WakeAgentSocket succeeded despite restore failure") } if _, g := m.Info(); g.Hibernated != 1 { @@ -542,7 +542,7 @@ func TestReconcileAdoptsJournaledIntent(t *testing.T) { if eng.snapRemoved(hibernatePrefix + "lagged-1-abc123") { t.Error("reconcile swept the wake image it adopted") } - if _, _, err := m.WakeAgentSocket(t.Context(), "sb_lag", "tok"); err != nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), "sb_lag", "tok"); err != nil { t.Fatalf("wake adopted claim: %v", err) } } @@ -627,7 +627,7 @@ func TestReconcileAdoptsHibernated(t *testing.T) { if _, g := m.Info(); g.Hibernated != 1 { t.Errorf("hibernated count %d after reconcile, want 1", g.Hibernated) } - if _, _, err := m.WakeAgentSocket(t.Context(), "sb_hib", "tok"); err != nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), "sb_hib", "tok"); err != nil { t.Fatalf("wake after reconcile: %v", err) } if len(eng.restores) != 1 { diff --git a/sandboxd/pool/idle_test.go b/sandboxd/pool/idle_test.go index 69365d4b..242704f8 100644 --- a/sandboxd/pool/idle_test.go +++ b/sandboxd/pool/idle_test.go @@ -74,7 +74,7 @@ func TestActivityStampsBlockIdleSweep(t *testing.T) { sb := mustClaim(t, m, testKey) backdate(m, sb, 2*time.Second) - _, done, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) + _, _, done, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) if err != nil { t.Fatalf("WakeAgentSocket: %v", err) } @@ -91,7 +91,7 @@ func TestHeldConnectionBlocksIdleSweep(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, IdleHibernateSeconds: 1}) sb := mustClaim(t, m, testKey) - _, done, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) + _, _, done, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) if err != nil { t.Fatalf("WakeAgentSocket: %v", err) } diff --git a/sandboxd/pool/lease.go b/sandboxd/pool/lease.go index b63498ab..f8c0ad38 100644 --- a/sandboxd/pool/lease.go +++ b/sandboxd/pool/lease.go @@ -2,8 +2,10 @@ package pool import ( "context" + "io" "maps" "slices" + "sync" "time" "github.com/projecteru2/core/log" @@ -46,6 +48,7 @@ func (m *Manager) renewLease(ctx context.Context, fence *time.Timer) { func (m *Manager) fence(ctx context.Context) { if !m.fenced.Swap(true) { + close(*m.fenceCh.Load()) log.WithFunc("pool.fence").Warnf(ctx, "node lease not renewed for %s; replicated claims answer 503 until it is", m.lease.ttl-m.lease.margin) } } @@ -76,10 +79,34 @@ func (m *Manager) unfence(ctx context.Context) { sb.Transition.Unlock() } } + m.fenceCh.Store(new(make(chan struct{}))) m.fenced.Store(false) logger.Info(ctx, "node lease renewed; replicated claims are served again") } -func (m *Manager) leaseFenced(sb *types.Sandbox) bool { - return m.fenced.Load() && m.replicaOf(sb) != "" +// fenceWatch gates sb at resolution and returns the channel that closes when its streams must end; nil for an unreplicated claim. +func (m *Manager) fenceWatch(sb *types.Sandbox) (<-chan struct{}, error) { + if !sb.Replicated() { + return nil, nil + } + if m.fenced.Load() { + return nil, ErrLeaseLost + } + return *m.fenceCh.Load(), nil +} + +// CloseOnFence closes c when fenced closes; stop ends the watch. +func CloseOnFence(fenced <-chan struct{}, c io.Closer) (stop func()) { + if fenced == nil { + return func() {} + } + stopped := make(chan struct{}) + go func() { + select { + case <-fenced: + _ = c.Close() + case <-stopped: + } + }() + return sync.OnceFunc(func() { close(stopped) }) } diff --git a/sandboxd/pool/lease_test.go b/sandboxd/pool/lease_test.go index 389f831b..3143d2d7 100644 --- a/sandboxd/pool/lease_test.go +++ b/sandboxd/pool/lease_test.go @@ -3,6 +3,8 @@ package pool import ( "context" "errors" + "net" + "os" "sync/atomic" "testing" "testing/synctest" @@ -59,7 +61,7 @@ func TestLeaseOverdueRenewalFencesReplicatedClaimsOnly(t *testing.T) { assertServed(t, a, replicated.ID, replicated.Token) time.Sleep(2 * time.Second) synctest.Wait() - if _, _, err := a.WakeAgentSocket(ctx, replicated.ID, replicated.Token); !errors.Is(err, ErrLeaseLost) { + if _, _, _, err := a.WakeAgentSocket(ctx, replicated.ID, replicated.Token); !errors.Is(err, ErrLeaseLost) { t.Errorf("relay or exec of a replicated claim past the fence: %v, want ErrLeaseLost", err) } if _, err := a.DialPort(ctx, replicated.ID, Cred{Token: replicated.Token}, 80); !errors.Is(err, ErrLeaseLost) { @@ -80,6 +82,95 @@ func TestLeaseOverdueRenewalFencesReplicatedClaimsOnly(t *testing.T) { }) } +func TestLeaseFenceClosesOpenStreamsOfReplicatedClaimsOnly(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + c := newReplicaCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + cut := &cutRegistry{Registry: c.reg} + a.archives = cut + replicated, plain := mustClaim(t, a, testKey), mustClaim(t, a, testKey) + replicateNow(t, a, replicated) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + go a.keepLease(ctx) + synctest.Wait() + + replicatedCred, plainCred := Cred{Token: replicated.Token}, Cred{Token: plain.Token} + a.eng.(*fakeEngine).guestDial = func() (net.Conn, error) { + host, _ := net.Pipe() + return host, nil + } + streams := map[string]net.Conn{} + defer func() { + for _, conn := range streams { + _ = conn.Close() + } + }() + for name, dial := range map[string]func() (net.Conn, error){ + "port": func() (net.Conn, error) { return a.DialPort(ctx, replicated.ID, replicatedCred, 80) }, + "passive": func() (net.Conn, error) { return a.DialPort(ctx, replicated.ID, Cred{Operator: true}, 80) }, + "preview": func() (net.Conn, error) { return a.PreviewDial(ctx, replicated.ID, 80) }, + "plain": func() (net.Conn, error) { return a.DialPort(ctx, plain.ID, plainCred, 80) }, + } { + conn, err := dial() + if err != nil { + t.Fatalf("open %s: %v", name, err) + } + streams[name] = conn + } + _, relayFenced, relayDone, err := a.WakeAgentSocket(ctx, replicated.ID, replicated.Token) + if err != nil { + t.Fatalf("relay of the replicated claim: %v", err) + } + defer relayDone() + _, plainFenced, plainDone, err := a.WakeAgentSocket(ctx, plain.ID, plain.Token) + if err != nil || plainFenced != nil { + t.Fatalf("relay of the plain claim: fence channel %v, %v, want none", plainFenced, err) + } + defer plainDone() + + cut.cut.Store(true) + time.Sleep(leaseTTL - leaseMargin + time.Second) + synctest.Wait() + select { + case <-relayFenced: + default: + t.Error("the replicated claim's relay was not told to end") + } + for name, conn := range streams { + _ = conn.SetReadDeadline(time.Now().Add(time.Second)) + _, readErr := conn.Read(make([]byte, 1)) + closed := !errors.Is(readErr, os.ErrDeadlineExceeded) + if closed != (name != "plain") { + t.Errorf("%s stream closed=%v (%v) after the fence, want closed=%v", name, closed, readErr, name != "plain") + } + } + + cut.cut.Store(false) + time.Sleep(leaseRenew) + synctest.Wait() + conn, err := a.DialPort(ctx, replicated.ID, replicatedCred, 80) + if err != nil { + t.Fatalf("a new stream after the fence lifted: %v", err) + } + defer conn.Close() + _ = conn.SetReadDeadline(time.Now().Add(time.Second)) + if _, readErr := conn.Read(make([]byte, 1)); !errors.Is(readErr, os.ErrDeadlineExceeded) { + t.Errorf("the new stream after unfence: %v, want it open", readErr) + } + _, fresh, done, err := a.WakeAgentSocket(ctx, replicated.ID, replicated.Token) + if err != nil { + t.Fatalf("relay after the fence lifted: %v", err) + } + defer done() + select { + case <-fresh: + t.Error("the fence channel after unfence is already closed") + default: + } + }) +} + func TestLeaseAdoptionWaitsOutTheFenceAndTheOldOwnerDropsItsCopy(t *testing.T) { for _, tt := range []struct { name string @@ -105,7 +196,7 @@ func TestLeaseAdoptionWaitsOutTheFenceAndTheOldOwnerDropsItsCopy(t *testing.T) { cut.cut.Store(true) time.Sleep(leaseTTL - leaseMargin + time.Second) synctest.Wait() - if _, _, err := a.WakeAgentSocket(ctx, sb.ID, sb.Token); !errors.Is(err, ErrLeaseLost) { + if _, _, _, err := a.WakeAgentSocket(ctx, sb.ID, sb.Token); !errors.Is(err, ErrLeaseLost) { t.Fatalf("the cut-off owner past its fence: %v, want ErrLeaseLost", err) } if adopted, err := b.AdoptArchived(ctx, sb.ID, sb.Token); err != nil || adopted { @@ -117,7 +208,7 @@ func TestLeaseAdoptionWaitsOutTheFenceAndTheOldOwnerDropsItsCopy(t *testing.T) { t.Fatalf("adoption after the lease expired: %v %v, want adopted", adopted, err) } if tt.wakeBefore { - if _, _, err := b.WakeAgentSocket(ctx, sb.ID, sb.Token); err != nil { + if _, _, _, err := b.WakeAgentSocket(ctx, sb.ID, sb.Token); err != nil { t.Fatalf("adopter's wake: %v", err) } } @@ -131,7 +222,7 @@ func TestLeaseAdoptionWaitsOutTheFenceAndTheOldOwnerDropsItsCopy(t *testing.T) { if a.fenced.Load() { t.Error("the old owner stays fenced after its lease renewed") } - if _, _, err := b.WakeAgentSocket(ctx, sb.ID, sb.Token); err != nil { + if _, _, _, err := b.WakeAgentSocket(ctx, sb.ID, sb.Token); err != nil { t.Errorf("adopter's wake after the old owner dropped its copy: %v", err) } }) @@ -160,7 +251,7 @@ func (r *cutRegistry) Rows(ctx context.Context, ids []string) (map[string]archiv func assertServed(t *testing.T, m *Manager, id, token string) { t.Helper() - _, done, err := m.WakeAgentSocket(t.Context(), id, token) + _, _, done, err := m.WakeAgentSocket(t.Context(), id, token) if err != nil { t.Fatalf("serving %s: %v", id, err) } diff --git a/sandboxd/pool/liveness_test.go b/sandboxd/pool/liveness_test.go index 193f8397..2864c615 100644 --- a/sandboxd/pool/liveness_test.go +++ b/sandboxd/pool/liveness_test.go @@ -33,7 +33,7 @@ func TestLivenessRestartsExitedVMM(t *testing.T) { if !slices.Equal(eng.starts, []string{sb.VMName}) { t.Errorf("starts %v, want %s cold-booted once", eng.starts, sb.VMName) } - if _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Errorf("data plane after restart: %v", err) } if got, want := usageEventsOf(t, m, sb.ID), "claim,vmm_exit,vmm_restart"; got != want { @@ -63,7 +63,7 @@ func TestLivenessNonePolicyFailsAndWakeRestarts(t *testing.T) { if _, g := m.Info(); g.Failed != 1 { t.Errorf("failed gauge %d, want 1", g.Failed) } - if _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrFailed) { + if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrFailed) { t.Errorf("relay on a failed claim: %v, want ErrFailed", err) } if _, err := m.DialPort(t.Context(), sb.ID, Cred{Operator: true}, 80); !errors.Is(err, ErrFailed) { diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 7868809c..af9b71f7 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -348,6 +348,7 @@ type Manager struct { // fenced gates replicated claims while the lease is overdue; replicas belongs to the pass holding replicating. lease leaseTiming fenced atomic.Bool + fenceCh atomic.Pointer[chan struct{}] replicating atomic.Bool replicas map[string]replicaState @@ -513,6 +514,9 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg m.configSeedHash = poolSeedHash(cfg.Pools) if m.archives != nil { m.nodeID = cfg.Mesh.ID() + closed := make(chan struct{}) + close(closed) + m.fenceCh.Store(&closed) m.fenced.Store(true) } if err := m.adoptPools(ctx); err != nil { diff --git a/sandboxd/pool/promote_test.go b/sandboxd/pool/promote_test.go index c33f895a..ba6c5971 100644 --- a/sandboxd/pool/promote_test.go +++ b/sandboxd/pool/promote_test.go @@ -114,7 +114,7 @@ func TestPromoteHibernatedUsesWakeImage(t *testing.T) { if slices.Contains(eng.snapRemoves, eng.hibernates[0]) { t.Error("hibernate snapshot dropped by promote — the parent could never wake") } - if _, _, err := m.WakeAgentSocket(t.Context(), parent.ID, parent.Token); err != nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), parent.ID, parent.Token); err != nil { t.Fatalf("wake after promote: %v", err) } } diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 11cbc259..6e6bfcf9 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -42,6 +42,7 @@ func (m *Manager) Reconcile(ctx context.Context) error { m.mu.Lock() for _, sb := range claims { sb.SetEnv(sb.Env) + sb.SyncReplicated() rec, ok := live[sb.VMName] switch { case sb.ArchiveCk != "": diff --git a/sandboxd/pool/reconcile_test.go b/sandboxd/pool/reconcile_test.go index 88cfe182..907c2ba4 100644 --- a/sandboxd/pool/reconcile_test.go +++ b/sandboxd/pool/reconcile_test.go @@ -32,7 +32,7 @@ func TestReconcileJournalsAnAdoptedHibernate(t *testing.T) { if err := restarted.Reconcile(t.Context()); err != nil { t.Fatalf("reconcile: %v", err) } - if _, _, err := restarted.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := restarted.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake: %v", err) } if got, want := usageEventsOf(t, restarted, sb.ID), "claim,hibernate,wake"; got != want { diff --git a/sandboxd/pool/replica.go b/sandboxd/pool/replica.go index ab7a65db..d21c3b53 100644 --- a/sandboxd/pool/replica.go +++ b/sandboxd/pool/replica.go @@ -118,7 +118,10 @@ func (m *Manager) replicate(ctx context.Context, sb *types.Sandbox) error { return fmt.Errorf("replicate %s: registry: %w", sb.ID, err) } var prev string - live, _ := m.commitIfLive(ctx, sb, func() { prev, sb.ReplicaCk, sb.ArchiveEpoch = sb.ReplicaCk, ckID, epoch }) + live, _ := m.commitIfLive(ctx, sb, func() { + prev, sb.ReplicaCk, sb.ArchiveEpoch = sb.ReplicaCk, ckID, epoch + sb.SyncReplicated() + }) if !live { m.abandonArchive(ctx, sb, epoch, ckID) return errWokeMeanwhile @@ -144,7 +147,10 @@ func (m *Manager) dropReplica(ctx context.Context, sb *types.Sandbox) error { return err } var ck string - live, _ := m.commitIfLive(ctx, sb, func() { ck, sb.ReplicaCk = sb.ReplicaCk, "" }) + live, _ := m.commitIfLive(ctx, sb, func() { + ck, sb.ReplicaCk = sb.ReplicaCk, "" + sb.SyncReplicated() + }) if !live { return ErrUnknownSandbox } diff --git a/sandboxd/pool/replica_test.go b/sandboxd/pool/replica_test.go index bf917d88..e05f0212 100644 --- a/sandboxd/pool/replica_test.go +++ b/sandboxd/pool/replica_test.go @@ -322,7 +322,7 @@ func TestReplicaRestartKeepsItsRowAndStaysFencedUntilALeaseRenews(t *testing.T) if !rowOK || rec.Epoch != before.Epoch || restarted.replicaOf(held) != ck || !ckExists(t, restarted, ck) { t.Errorf("row %+v present %v, replica %q, ck kept %v, want the row and replica kept", rec, rowOK, restarted.replicaOf(held), ckExists(t, restarted, ck)) } - if _, _, err := restarted.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrLeaseLost) { + if _, _, _, err := restarted.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrLeaseLost) { t.Errorf("serving before the first renewal: %v, want ErrLeaseLost", err) } } @@ -396,7 +396,7 @@ func TestReplicaRestartDropsAClaimTheAdopterWokeAndFinished(t *testing.T) { if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { t.Fatalf("adopt: %v %v", adopted, err) } - if _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("adopter's wake: %v", err) } @@ -508,7 +508,7 @@ func TestReplicaAdoptionWakesTheClaimFromTheLastReplica(t *testing.T) { if moved.ArchiveCk != ck { t.Fatalf("adopted claim names %q, want the replica %q", moved.ArchiveCk, ck) } - if _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("adopter's wake: %v", err) } if got := b.eng.(*fakeEngine).cloneCount(); got != 1 { diff --git a/sandboxd/pool/telemetry_test.go b/sandboxd/pool/telemetry_test.go index 3dbf0ecd..e2715401 100644 --- a/sandboxd/pool/telemetry_test.go +++ b/sandboxd/pool/telemetry_test.go @@ -25,7 +25,7 @@ func TestUsageJournalRecordsLifecycle(t *testing.T) { if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("Hibernate: %v", err) } - if _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("Wake: %v", err) } if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { diff --git a/sandboxd/pool/tenants_test.go b/sandboxd/pool/tenants_test.go index 6970e1ea..36b315ec 100644 --- a/sandboxd/pool/tenants_test.go +++ b/sandboxd/pool/tenants_test.go @@ -150,7 +150,7 @@ func TestRemovedTenantsClaimIsNeitherArchivedNorWokenNorForked(t *testing.T) { if marked || ck != "" { t.Errorf("the idle sweep archived a removed tenant's claim: marked %t archive %q", marked, ck) } - if _, _, err = m.WakeAgentSocket(t.Context(), archived.ID, archived.Token); !errors.Is(err, ErrTenantRemoved) { + if _, _, _, err = m.WakeAgentSocket(t.Context(), archived.ID, archived.Token); !errors.Is(err, ErrTenantRemoved) { t.Errorf("wake from the archive: %v, want ErrTenantRemoved", err) } if _, err = m.Fork(t.Context(), running.ID, Cred{Token: running.Token}, 1, time.Hour, "", ""); !errors.Is(err, ErrTenantRemoved) { diff --git a/sandboxd/pool/volume_capture_test.go b/sandboxd/pool/volume_capture_test.go index 52bcd1fc..27c1b1dc 100644 --- a/sandboxd/pool/volume_capture_test.go +++ b/sandboxd/pool/volume_capture_test.go @@ -53,7 +53,7 @@ func TestVolumeClaimRunningWakeUnchanged(t *testing.T) { sb := mustClaim(t, m, testKey) sb.Volumes = []types.Volume{{Name: "dataset", Mount: "/datasets"}} - sock, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) + sock, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) if err != nil { t.Fatalf("WakeAgentSocket: %v", err) } diff --git a/sandboxd/pool/volume_mutation_test.go b/sandboxd/pool/volume_mutation_test.go index d22f9a60..ce30fd1a 100644 --- a/sandboxd/pool/volume_mutation_test.go +++ b/sandboxd/pool/volume_mutation_test.go @@ -256,7 +256,7 @@ func TestLiveVolumeDoesNotBlockLifecycle(t *testing.T) { _, err = m.GetVolumes(sb.ID, sb.Tenant) case "relay": var unhold func() - _, unhold, err = m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) + _, _, unhold, err = m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) if unhold != nil { unhold() } diff --git a/sandboxd/server/exec.go b/sandboxd/server/exec.go index f9ce6007..1f432613 100644 --- a/sandboxd/server/exec.go +++ b/sandboxd/server/exec.go @@ -125,7 +125,7 @@ func (s *Server) handleExec(w http.ResponseWriter, r *http.Request) { func (s *Server) killExec(ctx context.Context, id, token string, pid uint32) error { ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), execKillWait) defer cancel() - sock, done, err := s.mgr.WakeAgentSocket(ctx, id, token) + sock, _, done, err := s.mgr.WakeAgentSocket(ctx, id, token) if err != nil { return err } diff --git a/sandboxd/server/exec_test.go b/sandboxd/server/exec_test.go index 274556cb..922aa720 100644 --- a/sandboxd/server/exec_test.go +++ b/sandboxd/server/exec_test.go @@ -300,9 +300,9 @@ type execManager struct { wake func(context.Context, string, string) (string, error) } -func (m *execManager) WakeAgentSocket(ctx context.Context, id, token string) (string, func(), error) { +func (m *execManager) WakeAgentSocket(ctx context.Context, id, token string) (string, <-chan struct{}, func(), error) { sock, err := m.wake(ctx, id, token) - return sock, func() {}, err + return sock, nil, func() {}, err } func postExec(t *testing.T, ts *httptest.Server, body string) (int, []byte) { diff --git a/sandboxd/server/relay.go b/sandboxd/server/relay.go index 05916f1a..351e7507 100644 --- a/sandboxd/server/relay.go +++ b/sandboxd/server/relay.go @@ -69,8 +69,8 @@ func (s *Server) handleAgent(w http.ResponseWriter, r *http.Request) { } // wakeGuest dials the sandbox's silkd, waking a hibernated VM first; on failure it has already answered the request. -func (s *Server) wakeGuest(ctx context.Context, w http.ResponseWriter, id, token string) (guest net.Conn, done func(), ok bool) { - sock, done, err := s.mgr.WakeAgentSocket(ctx, id, token) +func (s *Server) wakeGuest(ctx context.Context, w http.ResponseWriter, id, token string) (guest net.Conn, release func(), ok bool) { + sock, fenced, done, err := s.mgr.WakeAgentSocket(ctx, id, token) switch { case writePoolErr(w, err): return nil, nil, false @@ -86,7 +86,8 @@ func (s *Server) wakeGuest(ctx context.Context, w http.ResponseWriter, id, token writeErr(w, http.StatusBadGateway, "guest agent unreachable") return nil, nil, false } - return guest, done, true + stop := pool.CloseOnFence(fenced, guest) + return guest, func() { stop(); done() }, true } func (s *Server) relay(ctx context.Context, id string, client net.Conn, clientBuf *bufio.Reader, guest net.Conn) { diff --git a/sandboxd/server/relay_test.go b/sandboxd/server/relay_test.go index 43e02ade..d6e5cf87 100644 --- a/sandboxd/server/relay_test.go +++ b/sandboxd/server/relay_test.go @@ -9,6 +9,7 @@ import ( "net" "net/http" "net/http/httptest" + "os" "slices" "strings" "testing" @@ -138,6 +139,36 @@ func TestRelayClientDisconnectClosesGuest(t *testing.T) { } } +func TestRelayEndsWhenItsClaimIsFenced(t *testing.T) { + relayWithFence := func(fenced <-chan struct{}) (net.Conn, *bufio.Reader) { + dialer := &fakeDialer{dial: func(context.Context, string) (net.Conn, error) { + relayEnd, guestEnd := net.Pipe() + go func() { _, _ = io.Copy(io.Discard, guestEnd) }() + return relayEnd, nil + }} + srv := New("", "node:7777", &fakeManager{fenced: fenced}, dialer, nil, nil, nil, nil) + ts := httptest.NewServer(srv.Handler()) + t.Cleanup(func() { + srv.CloseRelays() + ts.Close() + }) + return upgradeConn(t, ts) + } + fenced := make(chan struct{}) + conn, r := relayWithFence(fenced) + plainConn, plainR := relayWithFence(nil) + close(fenced) + + _ = conn.SetReadDeadline(time.Now().Add(3 * time.Second)) + if _, err := r.ReadByte(); !errors.Is(err, io.EOF) { + t.Errorf("fenced relay read: %v, want EOF", err) + } + _ = plainConn.SetReadDeadline(time.Now().Add(200 * time.Millisecond)) + if _, err := plainR.ReadByte(); !errors.Is(err, os.ErrDeadlineExceeded) { + t.Errorf("unreplicated relay read: %v, want it still open", err) + } +} + func TestAuditTeeRecordsRequestLines(t *testing.T) { big := line(wire.Exec{Argv: []string{strings.Repeat("x", pool.AuditLineCap)}}) chunk := line(&wire.Data{Data: make([]byte, 3*pool.AuditLineCap)}) diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index be683757..473cd4aa 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -122,7 +122,7 @@ type Manager interface { AgentSocket(id, token string) (string, error) AdoptArchived(ctx context.Context, id, token string) (bool, error) DialPort(ctx context.Context, id string, cred pool.Cred, port uint16) (net.Conn, error) - WakeAgentSocket(ctx context.Context, id, token string) (string, func(), error) + WakeAgentSocket(ctx context.Context, id, token string) (string, <-chan struct{}, func(), error) SetPools(ctx context.Context, pools []config.PoolSpec) error TenantByToken(ctx context.Context, token string) (*tenants.TenantRecord, error) Tenants(ctx context.Context, after string, limit int) (pool.TenantPage, error) diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index 371c5d9d..1cc29b51 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -2484,6 +2484,7 @@ type fakeManager struct { socket func(id, token string) (string, error) adopt func(id, token string) (bool, error) dialPort func(id, token string, port uint16) (net.Conn, error) + fenced <-chan struct{} hibernate sandboxVerbFunc wake sandboxVerbFunc renew func(id, token string, ttl time.Duration) (time.Time, error) @@ -2597,9 +2598,9 @@ func (f *fakeManager) AdoptArchived(_ context.Context, id, token string) (bool, return f.adopt(id, token) } -func (f *fakeManager) WakeAgentSocket(_ context.Context, id, token string) (string, func(), error) { +func (f *fakeManager) WakeAgentSocket(_ context.Context, id, token string) (string, <-chan struct{}, func(), error) { sock, err := f.AgentSocket(id, token) - return sock, func() {}, err + return sock, f.fenced, func() {}, err } func (f *fakeManager) Renew(_ context.Context, id string, cred pool.Cred, ttl time.Duration, onExpire types.ExpireAction) (time.Time, error) { diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index 8703bc83..520933bd 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -599,6 +599,8 @@ type Sandbox struct { // lastActivity is unix-nanos of the last data-plane connection, stamped lock-free. lastActivity atomic.Int64 + // replicated mirrors ReplicaCk != "", so the data plane reads it without the manager mutex. + replicated atomic.Bool // open counts the data-plane connections held right now. open atomic.Int32 // live is the env as of the last SetEnv, read by the egress proxy without the manager mutex. @@ -617,6 +619,12 @@ func (s *Sandbox) TouchAt(t time.Time) { s.lastActivity.Store(t.UnixNano()) } // LastSeen returns the last data-plane activity time. func (s *Sandbox) LastSeen() time.Time { return time.Unix(0, s.lastActivity.Load()) } +// SyncReplicated republishes the replica state to Replicated; callers hold the manager mutex. +func (s *Sandbox) SyncReplicated() { s.replicated.Store(s.ReplicaCk != "") } + +// Replicated reports whether the claim's registry row names a replica. +func (s *Sandbox) Replicated() bool { return s.replicated.Load() } + // Hold registers an open data-plane connection; the idle sweep leaves a held sandbox alone. func (s *Sandbox) Hold() { s.open.Add(1) } From dd5b86ca486656cc4425ed44eca50ac999661f85 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 03:33:50 +0800 Subject: [PATCH 06/36] pool: journal a replication round's intent before its registry write A round whose Put reply is lost left the claim with no record of the checkpoint the row now names, and deleted that checkpoint. If the partition went on, another node adopted a row whose checkpoint was gone and the returning owner then dropped its live VM: irreversible loss from one fault, and on the first round the claim was not even fenced. A round now persists ReplicaPending before the Put, deletes the checkpoint only on ErrLost, and leaves an unknown outcome to settle: the next round, release, archive, wake, the hourly sweep and a restart read the row. A row on the pending checkpoint promotes it; another owner's row drops the claim; an absent row means the adopter finished when the checkpoint is gone, and a never-committed Put when it is still there. A claim with a pending intent counts as replicated for the fence. --- sandboxd/pool/claims.go | 3 +- sandboxd/pool/failover.go | 28 ++++++--- sandboxd/pool/failover_test.go | 4 ++ sandboxd/pool/lease.go | 2 +- sandboxd/pool/replica.go | 111 +++++++++++++++++++++++++++++++-- sandboxd/pool/replica_test.go | 100 +++++++++++++++++++++++++++++ sandboxd/types/types.go | 8 ++- 7 files changed, 237 insertions(+), 19 deletions(-) diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index f97a1767..f7712913 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -58,6 +58,7 @@ type claimDTO struct { ArchiveEpoch int64 `json:"archive_epoch,omitzero"` ArchivePending string `json:"archive_pending,omitempty"` ReplicaCk string `json:"replica_ck,omitempty"` + ReplicaPending string `json:"replica_pending,omitempty"` FromCheckpoint string `json:"from_checkpoint,omitempty"` Restarts int `json:"restarts,omitzero"` RestartedAt time.Time `json:"restarted_at,omitzero"` @@ -265,7 +266,7 @@ func dtoOf(sb *types.Sandbox) claimDTO { Tenant: sb.Tenant, EgressClass: sb.EgressClass, ClaimRef: sb.ClaimRef, Metadata: sb.Metadata, OnExpire: sb.OnExpire, Volumes: slices.Clone(sb.Volumes), VolumesWant: volumesWantDTO(sb), Env: sb.Env, VsockSocket: sb.VsockSocket, TAP: sb.TAP, HibernateSnap: sb.HibernateSnap, PendingSnap: sb.PendingSnap, - ArchiveCk: sb.ArchiveCk, ArchiveEpoch: sb.ArchiveEpoch, ArchivePending: sb.ArchivePending, ReplicaCk: sb.ReplicaCk, FromCheckpoint: sb.FromCheckpoint, + ArchiveCk: sb.ArchiveCk, ArchiveEpoch: sb.ArchiveEpoch, ArchivePending: sb.ArchivePending, ReplicaCk: sb.ReplicaCk, ReplicaPending: sb.ReplicaPending, FromCheckpoint: sb.FromCheckpoint, Restarts: sb.Restarts, RestartedAt: sb.RestartedAt, Failed: sb.Failed, } } diff --git a/sandboxd/pool/failover.go b/sandboxd/pool/failover.go index d36d89eb..6aeba6c2 100644 --- a/sandboxd/pool/failover.go +++ b/sandboxd/pool/failover.go @@ -102,7 +102,7 @@ func (m *Manager) putArchive(ctx context.Context, sb *types.Sandbox, ck string) deadline = m.retentionDeadline(sb.Key) } m.mu.Unlock() - dto.Token, dto.VMName, dto.VsockSocket, dto.HibernateSnap, dto.PendingSnap, dto.ArchivePending, dto.ReplicaCk = "", "", "", "", "", "", "" + dto.Token, dto.VMName, dto.VsockSocket, dto.HibernateSnap, dto.PendingSnap, dto.ArchivePending, dto.ReplicaCk, dto.ReplicaPending = "", "", "", "", "", "", "", "" dto.ArchiveCk, dto.Deadline = ck, deadline claim, err := json.Marshal(&dto) if err != nil { @@ -304,7 +304,7 @@ func (m *Manager) unrowExpired(ctx context.Context, sb *types.Sandbox) { sb.Transition.Lock() defer sb.Transition.Unlock() var err error - if m.replicaOf(sb) != "" { + if sb.Replicated() { err = m.dropReplica(ctx, sb) } else { _, err = m.consumeArchive(ctx, sb) @@ -452,12 +452,21 @@ func (m *Manager) fenceArchived(ctx context.Context, claims map[string]*types.Sa } case ok && rec.Owner != m.nodeID: drop = true - case sb.ReplicaCk != "" && sb.ArchivePending == "": + case (sb.ReplicaCk != "" || sb.ReplicaPending != "") && sb.ArchivePending == "": + moved, err := false, error(nil) + if !ok { + moved, err = m.replicaGone(ctx, sb) + } switch { - case !ok && m.replicaConfirmed(sb): + case err != nil: + logger.Warnf(ctx, "replica intent of %s left for the next settle: %v", id, err) + case moved: drop = true - case ok && !rec.Consumed: - sb.ReplicaCk, sb.ArchiveEpoch = rec.Checkpoint, rec.Epoch + default: + if ok && !rec.Consumed { + sb.ReplicaCk, sb.ArchiveEpoch = rec.Checkpoint, rec.Epoch + } + sb.ReplicaPending = "" sb.SyncReplicated() } case sb.ArchivePending != "" || ok && !rec.Consumed: @@ -518,6 +527,9 @@ func (m *Manager) resolveIntent(ctx context.Context, sb *types.Sandbox) error { if m.archives == nil { return nil } + if err := m.settleReplicaIntent(ctx, sb); err != nil { + return err + } m.mu.Lock() pending := sb.ArchivePending m.mu.Unlock() @@ -546,7 +558,7 @@ func (m *Manager) settleIntents(ctx context.Context) { m.mu.Lock() var pending []*types.Sandbox for id, sb := range m.claimed { - if _, busy := m.archiving[id]; sb.ArchivePending != "" && !busy { + if _, busy := m.archiving[id]; (sb.ArchivePending != "" || sb.ReplicaPending != "") && !busy { pending = append(pending, sb) } } @@ -565,5 +577,5 @@ func (m *Manager) archiveRowed(sb *types.Sandbox) bool { } func (m *Manager) expiryDropsRow(sb *types.Sandbox) bool { - return m.archiveRowed(sb) || sb.ReplicaCk != "" && !m.archivesAtDeadline(sb) + return m.archiveRowed(sb) || sb.Replicated() && !m.archivesAtDeadline(sb) } diff --git a/sandboxd/pool/failover_test.go b/sandboxd/pool/failover_test.go index 185492b3..8967545f 100644 --- a/sandboxd/pool/failover_test.go +++ b/sandboxd/pool/failover_test.go @@ -1941,9 +1941,13 @@ type hookRegistry struct { loseConsumeReply bool loseTakeReply bool losePutReply bool + failPut bool } func (r *hookRegistry) Put(ctx context.Context, rec archives.Record) (int64, error) { + if r.failPut { + return 0, archives.ErrUnavailable + } if hook := r.beforePut; hook != nil { r.beforePut = nil hook() diff --git a/sandboxd/pool/lease.go b/sandboxd/pool/lease.go index f8c0ad38..13b6d8ec 100644 --- a/sandboxd/pool/lease.go +++ b/sandboxd/pool/lease.go @@ -58,7 +58,7 @@ func (m *Manager) unfence(ctx context.Context) { m.mu.Lock() held := map[string]*types.Sandbox{} for id, sb := range m.claimed { - if sb.ReplicaCk != "" { + if sb.Replicated() { held[id] = sb } } diff --git a/sandboxd/pool/replica.go b/sandboxd/pool/replica.go index d21c3b53..0f363ec6 100644 --- a/sandboxd/pool/replica.go +++ b/sandboxd/pool/replica.go @@ -31,7 +31,7 @@ func (m *Manager) replicateOnce(ctx context.Context) { every, st := v.poolReplicate[sb.Key], m.replicas[id] switch { case every <= 0: - if sb.ReplicaCk != "" { + if sb.Replicated() { retire = append(retire, sb) } case !m.replicable(sb): @@ -84,6 +84,12 @@ func (m *Manager) replicate(ctx context.Context, sb *types.Sandbox) error { if err := m.diskErr(); err != nil { return fmt.Errorf("replicate %s: %w", sb.ID, err) } + sb.Transition.Lock() + err := m.settleReplicaIntent(ctx, sb) + sb.Transition.Unlock() + if err != nil { + return fmt.Errorf("replicate %s: settle intent: %w", sb.ID, err) + } if err := m.checkArchiveRow(ctx, sb.ID); err != nil { if errors.Is(err, archives.ErrLost) { sb.Transition.Lock() @@ -104,22 +110,35 @@ func (m *Manager) replicate(ctx context.Context, sb *types.Sandbox) error { defer sb.Transition.Unlock() m.mu.Lock() ok := m.claimed[sb.ID] == sb && sb.ArchiveCk == "" && sb.ArchivePending == "" + var intent claimSnapshot + if ok { + sb.ReplicaPending = ckID + sb.SyncReplicated() + intent = m.store.set(sb) + } m.mu.Unlock() if !ok { m.deleteOrphanArchiveCk(ctx, ckID) return errWokeMeanwhile } + if err = m.store.commit(intent); err != nil { + m.clearReplicaIntent(ctx, sb, ckID) + m.deleteOrphanArchiveCk(ctx, ckID) + return fmt.Errorf("replicate %s: persist intent: %w", sb.ID, err) + } epoch, err := m.putArchive(ctx, sb, ckID) - if err != nil { + switch { + case errors.Is(err, archives.ErrLost): + m.clearReplicaIntent(ctx, sb, ckID) m.deleteOrphanArchiveCk(ctx, ckID) - if errors.Is(err, archives.ErrLost) { - m.dropAdopted(ctx, sb) - } + m.dropAdopted(ctx, sb) + return fmt.Errorf("replicate %s: registry: %w", sb.ID, err) + case err != nil: return fmt.Errorf("replicate %s: registry: %w", sb.ID, err) } var prev string live, _ := m.commitIfLive(ctx, sb, func() { - prev, sb.ReplicaCk, sb.ArchiveEpoch = sb.ReplicaCk, ckID, epoch + prev, sb.ReplicaCk, sb.ReplicaPending, sb.ArchiveEpoch = sb.ReplicaCk, ckID, "", epoch sb.SyncReplicated() }) if !live { @@ -132,6 +151,80 @@ func (m *Manager) replicate(ctx context.Context, sb *types.Sandbox) error { return nil } +// settleReplicaIntent resolves a round whose registry write may or may not have landed; the caller holds sb.Transition. +func (m *Manager) settleReplicaIntent(ctx context.Context, sb *types.Sandbox) error { + m.mu.Lock() + pending := sb.ReplicaPending + m.mu.Unlock() + if pending == "" || m.archives == nil { + return nil + } + rec, ok, err := m.archives.Get(ctx, sb.ID) + if err != nil { + return err + } + gone := false + if !ok { + if gone, err = m.replicaGone(ctx, sb); err != nil { + return err + } + } + switch { + case gone: + m.dropAdopted(ctx, sb) + return ErrUnknownSandbox + case ok && rec.Owner != m.nodeID: + if rec.Checkpoint != pending { + m.deleteOrphanArchiveCk(ctx, pending) + } + m.dropAdopted(ctx, sb) + return ErrUnknownSandbox + case ok && !rec.Consumed && rec.Checkpoint == pending: + var prev string + live, _ := m.commitIfLive(ctx, sb, func() { + prev, sb.ReplicaCk, sb.ReplicaPending, sb.ArchiveEpoch = sb.ReplicaCk, pending, "", rec.Epoch + sb.SyncReplicated() + }) + if !live { + return ErrUnknownSandbox + } + if prev != "" { + m.deleteOrphanArchiveCk(ctx, prev) + } + return nil + } + m.clearReplicaIntent(ctx, sb, pending) + m.deleteOrphanArchiveCk(ctx, pending) + return nil +} + +// replicaGone reports whether the absence of sb's row means another node adopted the claim and finished with it. +func (m *Manager) replicaGone(ctx context.Context, sb *types.Sandbox) (bool, error) { + m.mu.Lock() + confirmed, pending := sb.ArchiveEpoch != 0, sb.ReplicaPending + m.mu.Unlock() + if confirmed { + return true, nil + } + if pending == "" { + return false, nil + } + _, err := m.ckpts.ReadMeta(ctx, pending) + if errors.Is(err, store.ErrNotFound) { + return true, nil + } + return false, err +} + +func (m *Manager) clearReplicaIntent(ctx context.Context, sb *types.Sandbox, ck string) { + _, _ = m.commitIfLive(ctx, sb, func() { + if sb.ReplicaPending == ck { + sb.ReplicaPending = "" + sb.SyncReplicated() + } + }) +} + func (m *Manager) retireReplica(ctx context.Context, sb *types.Sandbox) { sb.Transition.Lock() defer sb.Transition.Unlock() @@ -142,6 +235,12 @@ func (m *Manager) retireReplica(ctx context.Context, sb *types.Sandbox) { // dropReplica ends sb's replication; the caller holds sb.Transition. func (m *Manager) dropReplica(ctx context.Context, sb *types.Sandbox) error { + if err := m.settleReplicaIntent(ctx, sb); err != nil { + return err + } + if m.replicaOf(sb) == "" { + return nil + } consumed, err := m.consumeArchive(ctx, sb) if err != nil { return err diff --git a/sandboxd/pool/replica_test.go b/sandboxd/pool/replica_test.go index e05f0212..9357b436 100644 --- a/sandboxd/pool/replica_test.go +++ b/sandboxd/pool/replica_test.go @@ -86,6 +86,106 @@ func TestReplicaRefusedPutDropsTheCopyAndItsExport(t *testing.T) { } } +func TestReplicaLostPutReplyKeepsTheCheckpointTheRowNames(t *testing.T) { + c := newReplicaCluster(t) + a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + vm := lockedVMName(sb) + a.archives = &hookRegistry{Registry: c.reg, losePutReply: true} + + if err := a.replicate(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { + t.Fatalf("round whose put reply was lost: %v, want ErrArchiveStoreDown", err) + } + row, ok, _ := c.reg.Get(t.Context(), sb.ID) + a.mu.Lock() + pending := sb.ReplicaPending + a.mu.Unlock() + if !ok || pending == "" || row.Checkpoint != pending || !ckExists(t, a, pending) || !sb.Replicated() { + t.Fatalf("row %+v present %v, pending %q kept %v replicated %v, want the row's checkpoint kept and the claim fenced", + row, ok, pending, ckExists(t, a, pending), sb.Replicated()) + } + + c.expire("a") + if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { + t.Fatalf("adopt: %v %v", adopted, err) + } + if _, _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + t.Fatalf("the adopter's wake from the checkpoint: %v", err) + } + if got := b.eng.(*fakeEngine).cloneCount(); got != 1 { + t.Errorf("the wake cloned %d VMs, want the one restored from the checkpoint", got) + } + + c.renew(t, "a") + sb.Transition.Lock() + err := a.resolveIntent(t.Context(), sb) + sb.Transition.Unlock() + if !errors.Is(err, ErrUnknownSandbox) { + t.Fatalf("the returning owner's settle: %v, want ErrUnknownSandbox", err) + } + if _, held := a.byID(sb.ID); held || !a.eng.(*fakeEngine).removed(vm) { + t.Errorf("old owner holds the claim %v, VM removed %v, want its copy dropped", held, a.eng.(*fakeEngine).removed(vm)) + } +} + +func TestReplicaUnavailablePutThatNeverLandedIsSettledAtTheNextRound(t *testing.T) { + c := newReplicaCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + replicateNow(t, a, sb) + old := a.replicaOf(sb) + row, _, _ := c.reg.Get(t.Context(), sb.ID) + a.archives = &hookRegistry{Registry: c.reg, failPut: true} + + if err := a.replicate(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { + t.Fatalf("round whose put failed: %v, want ErrArchiveStoreDown", err) + } + a.mu.Lock() + pending := sb.ReplicaPending + a.mu.Unlock() + if pending == "" || !ckExists(t, a, pending) || !ckExists(t, a, old) { + t.Fatalf("pending %q kept %v, old kept %v, want both checkpoints kept until settled", pending, ckExists(t, a, pending), ckExists(t, a, old)) + } + + a.archives = c.reg + replicateNow(t, a, sb) + if ckExists(t, a, pending) || ckExists(t, a, old) { + t.Errorf("failed round's checkpoint kept %v, old replica kept %v, want both gone", ckExists(t, a, pending), ckExists(t, a, old)) + } + if rec, _, _ := c.reg.Get(t.Context(), sb.ID); rec.Checkpoint != a.replicaOf(sb) || rec.Epoch <= row.Epoch { + t.Errorf("row %s@%d, want it on the new replica %s", rec.Checkpoint, rec.Epoch, a.replicaOf(sb)) + } +} + +func TestReplicaRestartSettlesALostPutReply(t *testing.T) { + c := newReplicaCluster(t) + dirA := t.TempDir() + a := c.node(t, "a", dirA, keepAnHour) + sb := mustClaim(t, a, testKey) + a.archives = &hookRegistry{Registry: c.reg, losePutReply: true} + if err := a.replicate(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { + t.Fatalf("round whose put reply was lost: %v", err) + } + a.mu.Lock() + pending := sb.ReplicaPending + a.mu.Unlock() + if err := a.FlushClaims(); err != nil { + t.Fatalf("flush: %v", err) + } + + restarted := c.restart(t, a) + if err := restarted.Reconcile(t.Context()); err != nil { + t.Fatalf("reconcile: %v", err) + } + held, ok := restarted.byID(sb.ID) + if !ok || restarted.replicaOf(held) != pending || !ckExists(t, restarted, pending) { + t.Fatalf("claim held %v, replica %q, ck kept %v, want the row's checkpoint %s adopted", ok, restarted.replicaOf(held), ckExists(t, restarted, pending), pending) + } + if held.ReplicaPending != "" { + t.Errorf("intent %q left after the restart settled it", held.ReplicaPending) + } +} + func TestReplicaFailedCaptureKeepsTheOldReplicaAndRow(t *testing.T) { c := newReplicaCluster(t) a := c.node(t, "a", t.TempDir(), keepAnHour) diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index 520933bd..c748a80d 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -578,6 +578,8 @@ type Sandbox struct { ArchivePending string `json:"archive_pending,omitempty"` // ReplicaCk names the checkpoint the claim's registry row points at while the claim runs; empty means unreplicated. ReplicaCk string `json:"replica_ck,omitempty"` + // ReplicaPending is the checkpoint a replication round is about to register; the row may already name it. + ReplicaPending string `json:"replica_pending,omitempty"` // FromCheckpoint names the checkpoint this sandbox branched from; empty otherwise. FromCheckpoint string `json:"from_checkpoint,omitempty"` @@ -599,7 +601,7 @@ type Sandbox struct { // lastActivity is unix-nanos of the last data-plane connection, stamped lock-free. lastActivity atomic.Int64 - // replicated mirrors ReplicaCk != "", so the data plane reads it without the manager mutex. + // replicated mirrors ReplicaCk or ReplicaPending being set, so the data plane reads it without the manager mutex. replicated atomic.Bool // open counts the data-plane connections held right now. open atomic.Int32 @@ -620,9 +622,9 @@ func (s *Sandbox) TouchAt(t time.Time) { s.lastActivity.Store(t.UnixNano()) } func (s *Sandbox) LastSeen() time.Time { return time.Unix(0, s.lastActivity.Load()) } // SyncReplicated republishes the replica state to Replicated; callers hold the manager mutex. -func (s *Sandbox) SyncReplicated() { s.replicated.Store(s.ReplicaCk != "") } +func (s *Sandbox) SyncReplicated() { s.replicated.Store(s.ReplicaCk != "" || s.ReplicaPending != "") } -// Replicated reports whether the claim's registry row names a replica. +// Replicated reports whether the claim's registry row may name a replica. func (s *Sandbox) Replicated() bool { return s.replicated.Load() } // Hold registers an open data-plane connection; the idle sweep leaves a held sandbox alone. From 32dff5f23efce8bae4f4823ae237c698691d432c Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 03:35:05 +0800 Subject: [PATCH 07/36] pool: write a running claim's registry row without a retention deadline A replica row carried the retention deadline of its Put. A hibernated claim, which no round refreshes, saw that deadline pass while its owner was partitioned or down; another node's departed-owner sweep then purged the checkpoint and the row, and the returning owner dropped the claim. Expired rows were also refused for adoption. Replica rows now have no deadline. Retention starts where the claim leaves the node's care: the archive Put sets it as before, and adoption sets the adopter pool's window when the row has none. The rows of a node that never returns stay until someone adopts them. --- sandboxd/pool/failover.go | 30 ++++++++++++++++++++++----- sandboxd/pool/replica.go | 2 +- sandboxd/pool/replica_test.go | 38 ++++++++++++++++++++++++++++++----- 3 files changed, 59 insertions(+), 11 deletions(-) diff --git a/sandboxd/pool/failover.go b/sandboxd/pool/failover.go index 6aeba6c2..b2c9cca3 100644 --- a/sandboxd/pool/failover.go +++ b/sandboxd/pool/failover.go @@ -1,7 +1,6 @@ package pool import ( - "cmp" "context" "crypto/subtle" "encoding/json/v2" @@ -58,6 +57,9 @@ func (m *Manager) adoptArchived(ctx context.Context, id, token string) (bool, er sb.Token, sb.ArchiveCk, sb.ArchiveEpoch, sb.Deadline = token, rec.Checkpoint, epoch, rec.Deadline sb.SetEnv(sb.Env) m.mu.Lock() + if sb.Deadline.IsZero() { + sb.Deadline = m.retentionDeadline(sb.Key) + } if cur, dup := m.claimed[id]; dup { m.mu.Unlock() return m.adoptHeld(ctx, cur, epoch), nil @@ -93,12 +95,24 @@ func (m *Manager) adoptHeld(ctx context.Context, cur *types.Sandbox, epoch int64 } func (m *Manager) putArchive(ctx context.Context, sb *types.Sandbox, ck string) (int64, error) { + return m.putRow(ctx, sb, ck, false) +} + +// putReplica writes a live claim's row with no deadline: retention starts when the claim is archived or adopted. +func (m *Manager) putReplica(ctx context.Context, sb *types.Sandbox, ck string) (int64, error) { + return m.putRow(ctx, sb, ck, true) +} + +func (m *Manager) putRow(ctx context.Context, sb *types.Sandbox, ck string, replica bool) (int64, error) { if m.archives == nil { return 0, nil } m.mu.Lock() dto, deadline := dtoOf(sb), sb.Deadline - if sb.ArchiveCk == "" { + switch { + case replica: + deadline = time.Time{} + case sb.ArchiveCk == "": deadline = m.retentionDeadline(sb.Key) } m.mu.Unlock() @@ -232,11 +246,17 @@ func (m *Manager) dropTombstone(ctx context.Context, id string, epoch int64) { } } -func (m *Manager) reputArchive(ctx context.Context, sb *types.Sandbox, epoch int64) { - if epoch == 0 { +func (m *Manager) reputArchive(ctx context.Context, sb *types.Sandbox, held int64) { + if held == 0 { return } - epoch, err := m.putArchive(ctx, sb, cmp.Or(sb.ArchiveCk, sb.ReplicaCk)) + var epoch int64 + var err error + if sb.ArchiveCk != "" { + epoch, err = m.putArchive(ctx, sb, sb.ArchiveCk) + } else { + epoch, err = m.putReplica(ctx, sb, sb.ReplicaCk) + } if err != nil { log.WithFunc("pool.reputArchive").Warnf(ctx, "re-put archive row %s: %v", sb.ID, err) return diff --git a/sandboxd/pool/replica.go b/sandboxd/pool/replica.go index 0f363ec6..9fa637a3 100644 --- a/sandboxd/pool/replica.go +++ b/sandboxd/pool/replica.go @@ -126,7 +126,7 @@ func (m *Manager) replicate(ctx context.Context, sb *types.Sandbox) error { m.deleteOrphanArchiveCk(ctx, ckID) return fmt.Errorf("replicate %s: persist intent: %w", sb.ID, err) } - epoch, err := m.putArchive(ctx, sb, ckID) + epoch, err := m.putReplica(ctx, sb, ckID) switch { case errors.Is(err, archives.ErrLost): m.clearReplicaIntent(ctx, sb, ckID) diff --git a/sandboxd/pool/replica_test.go b/sandboxd/pool/replica_test.go index 9357b436..3a9f398c 100644 --- a/sandboxd/pool/replica_test.go +++ b/sandboxd/pool/replica_test.go @@ -27,8 +27,8 @@ func TestReplicaRoundPublishesMovesTheRowAndDropsThePrevious(t *testing.T) { if err != nil || !ok { t.Fatalf("row: %v %v, want one", ok, err) } - if first == "" || rec.Owner != "a" || rec.Checkpoint != first || rec.Consumed || rec.TokenSHA256 != tenants.TokenSHA256(sb.Token) || rec.Deadline.IsZero() { - t.Fatalf("row %+v with replica %q, want a live row of a on the replica with the token hash and a deadline", rec, first) + if first == "" || rec.Owner != "a" || rec.Checkpoint != first || rec.Consumed || rec.TokenSHA256 != tenants.TokenSHA256(sb.Token) || !rec.Deadline.IsZero() { + t.Fatalf("row %+v with replica %q, want a live row of a on the replica with the token hash and no deadline", rec, first) } if stored, loadErr := a.store.load(); loadErr != nil || stored[sb.ID].ReplicaCk != first || stored[sb.ID].ArchiveEpoch != rec.Epoch { t.Errorf("claims.json replica %q epoch %d (%v), want %q at %d", stored[sb.ID].ReplicaCk, stored[sb.ID].ArchiveEpoch, loadErr, first, rec.Epoch) @@ -186,6 +186,34 @@ func TestReplicaRestartSettlesALostPutReply(t *testing.T) { } } +func TestReplicaRowOfAHibernatedClaimHasNoRetentionDeadline(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + c := newReplicaCluster(t) + a, b := c.node(t, "a", t.TempDir(), 1), c.node(t, "b", t.TempDir(), 1) + sb := mustClaim(t, a, testKey) + replicateNow(t, a, sb) + ck := a.replicaOf(sb) + if err := a.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatalf("hibernate: %v", err) + } + time.Sleep(2 * time.Second) + c.expire("a") + + b.sweepOrphanArchives(t.Context()) + rec, ok, _ := c.reg.Get(t.Context(), sb.ID) + if !ok || !rec.Deadline.IsZero() || !ckExists(t, b, ck) { + t.Fatalf("row %+v present %v, replica kept %v after a sweep past the retention window, want the row kept with no deadline", rec, ok, ckExists(t, b, ck)) + } + if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { + t.Fatalf("adopt: %v %v, want adopted", adopted, err) + } + moved, _ := b.byID(sb.ID) + if moved.Deadline.IsZero() { + t.Error("the adopted claim has no retention deadline, want the pool's window from the adoption") + } + }) +} + func TestReplicaFailedCaptureKeepsTheOldReplicaAndRow(t *testing.T) { c := newReplicaCluster(t) a := c.node(t, "a", t.TempDir(), keepAnHour) @@ -569,8 +597,8 @@ func TestReplicaSweepRearmsATombstoneItsClaimStillNames(t *testing.T) { a.sweepOrphanArchives(t.Context()) rec, ok, _ := c.reg.Get(t.Context(), sb.ID) - if !ok || rec.Consumed || rec.Checkpoint != ck || rec.Owner != "a" || !ckExists(t, a, ck) { - t.Errorf("row %+v present %v, replica kept %v, want a live row of a re-armed on %s", rec, ok, ckExists(t, a, ck), ck) + if !ok || rec.Consumed || rec.Checkpoint != ck || rec.Owner != "a" || !rec.Deadline.IsZero() || !ckExists(t, a, ck) { + t.Errorf("row %+v present %v, replica kept %v, want a live row of a with no deadline re-armed on %s", rec, ok, ckExists(t, a, ck), ck) } } @@ -639,7 +667,7 @@ func interruptReplica(t *testing.T, m *Manager, sb *types.Sandbox) string { if err != nil { t.Fatalf("publish: %v", err) } - if _, err = m.putArchive(t.Context(), sb, ck.ID); err != nil { + if _, err = m.putReplica(t.Context(), sb, ck.ID); err != nil { t.Fatalf("put: %v", err) } return ck.ID From 6c4d7909c27c182d870aea4560844a1994fe5195 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 03:37:27 +0800 Subject: [PATCH 08/36] pool: confirm each claim's row by compare-and-set before lifting the fence The renew loop ran unfence inline, so long Transition waits or VM removals there stopped renewals; the lease lapsed, an adopter took the row, and the unconditional Store(false) then let both nodes serve. Unfence now runs beside the renew loop, which keeps renewing. For each replicated claim it takes the row for this node in one compare-and-set, which the row lock serializes against adopters, retrying once on a stale epoch; a lost row forgets the claim at once and removes its VM in the background. The fence lifts only when all of that finished inside ttl-margin of the renewal that started it; otherwise the node stays fenced and the next renewal tries again. --- sandboxd/pool/failover.go | 12 +++- sandboxd/pool/lease.go | 69 +++++++++++++++++++--- sandboxd/pool/lease_test.go | 112 +++++++++++++++++++++++++++++++++++- sandboxd/pool/pool.go | 1 + sandboxd/pool/replica.go | 6 +- 5 files changed, 188 insertions(+), 12 deletions(-) diff --git a/sandboxd/pool/failover.go b/sandboxd/pool/failover.go index b2c9cca3..97ebda69 100644 --- a/sandboxd/pool/failover.go +++ b/sandboxd/pool/failover.go @@ -336,6 +336,13 @@ func (m *Manager) unrowExpired(ctx context.Context, sb *types.Sandbox) { // dropAdopted forgets a claim another node owns now and removes its local VM; an archive checkpoint is the new owner's. func (m *Manager) dropAdopted(ctx context.Context, sb *types.Sandbox) { + if teardown := m.forgetAdopted(ctx, sb); teardown != nil { + teardown() + } +} + +// forgetAdopted forgets the claim at once and returns the removal of its VM; nil when the claim was already gone. +func (m *Manager) forgetAdopted(ctx context.Context, sb *types.Sandbox) func() { logger := log.WithFunc("pool.dropAdopted") m.mu.Lock() ck, vm, snap := sb.ArchiveCk, sb.VMName, sb.HibernateSnap @@ -345,7 +352,10 @@ func (m *Manager) dropAdopted(ctx context.Context, sb *types.Sandbox) { logger.Warnf(ctx, "clear archive ck %s: %v", ck, err) } } - if m.forgetClaim(ctx, sb) { + if !m.forgetClaim(ctx, sb) { + return nil + } + return func() { if vm != "" { m.destroy(ctx, vm) } diff --git a/sandboxd/pool/lease.go b/sandboxd/pool/lease.go index 13b6d8ec..d3d74d46 100644 --- a/sandboxd/pool/lease.go +++ b/sandboxd/pool/lease.go @@ -2,6 +2,7 @@ package pool import ( "context" + "errors" "io" "maps" "slices" @@ -41,8 +42,11 @@ func (m *Manager) renewLease(ctx context.Context, fence *time.Timer) { return } fence.Reset(m.lease.ttl - m.lease.margin - time.Since(start)) - if m.fenced.Load() { - m.unfence(ctx) + if m.fenced.Load() && m.unfencing.CompareAndSwap(false, true) { + go func() { + defer m.unfencing.Store(false) + m.unfence(ctx, start) + }() } } @@ -53,7 +57,8 @@ func (m *Manager) fence(ctx context.Context) { } } -func (m *Manager) unfence(ctx context.Context) { +// unfence confirms each replicated claim against its row, then lifts the fence if that finished inside the window the renewal at start still covers. +func (m *Manager) unfence(ctx context.Context, start time.Time) { logger := log.WithFunc("pool.unfence") m.mu.Lock() held := map[string]*types.Sandbox{} @@ -71,19 +76,67 @@ func (m *Manager) unfence(ctx context.Context) { return } } + var teardowns []func() for id, sb := range held { - rec, ok := rows[id] - if ok && rec.Owner != m.nodeID || !ok && m.replicaConfirmed(sb) { - sb.Transition.Lock() - m.dropAdopted(ctx, sb) - sb.Transition.Unlock() + keep, err := m.confirmOwnership(ctx, sb, rows[id]) + if err != nil { + logger.Warnf(ctx, "replicated claims stay fenced: %v", err) + return + } + if keep { + continue } + if td := m.forgetAdopted(ctx, sb); td != nil { + teardowns = append(teardowns, func() { + sb.Transition.Lock() + defer sb.Transition.Unlock() + td() + }) + } + } + for _, td := range teardowns { + go td() + } + if time.Since(start) > m.lease.ttl-m.lease.margin { + logger.Warn(ctx, "replicated claims stay fenced: confirming ownership outlasted the lease window") + return } m.fenceCh.Store(new(make(chan struct{}))) m.fenced.Store(false) logger.Info(ctx, "node lease renewed; replicated claims are served again") } +// confirmOwnership takes sb's row for this node in one compare-and-set; false means another node holds the claim. +func (m *Manager) confirmOwnership(ctx context.Context, sb *types.Sandbox, rec archives.Record) (bool, error) { + for range 2 { + switch { + case rec.ID == "": + gone, err := m.replicaGone(ctx, sb) + return !gone, err + case rec.Owner != m.nodeID: + return false, nil + case rec.Consumed: + return true, nil + } + epoch, err := m.archives.Take(ctx, sb.ID, rec.Epoch, m.nodeID) + if err == nil { + _, _ = m.commitIfLive(ctx, sb, func() { sb.ArchiveEpoch = max(sb.ArchiveEpoch, epoch) }) + return true, nil + } + if !errors.Is(err, archives.ErrLost) { + return false, err + } + var found bool + if rec, found, err = m.archives.Get(ctx, sb.ID); err != nil { + return false, err + } + if !found { + rec = archives.Record{} + } + } + return false, nil +} + // fenceWatch gates sb at resolution and returns the channel that closes when its streams must end; nil for an unreplicated claim. func (m *Manager) fenceWatch(sb *types.Sandbox) (<-chan struct{}, error) { if !sb.Replicated() { diff --git a/sandboxd/pool/lease_test.go b/sandboxd/pool/lease_test.go index 3143d2d7..8b5eb96a 100644 --- a/sandboxd/pool/lease_test.go +++ b/sandboxd/pool/lease_test.go @@ -19,13 +19,13 @@ func TestLeaseRenewLoopKeepsTheNodeLeasedUntilItStops(t *testing.T) { a := c.node(t, "a", t.TempDir(), keepAnHour) sb := mustClaim(t, a, testKey) replicateNow(t, a, sb) - row, _, _ := c.reg.Get(t.Context(), sb.ID) c.expire("a") ctx, cancel := context.WithCancel(t.Context()) defer cancel() go a.keepLease(ctx) time.Sleep(10 * leaseTTL) + row, _, _ := c.reg.Get(ctx, sb.ID) if _, err := c.reg.Take(ctx, sb.ID, row.Epoch, "b"); !errors.Is(err, archives.ErrLost) { t.Fatalf("take while the loop renews: %v, want ErrLost", err) } @@ -171,6 +171,103 @@ func TestLeaseFenceClosesOpenStreamsOfReplicatedClaimsOnly(t *testing.T) { }) } +func TestLeaseSlowUnfenceStaysFencedAndKeepsRenewing(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + c := newReplicaCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + cut := &cutRegistry{Registry: c.reg} + a.archives = cut + sb := mustClaim(t, a, testKey) + replicateNow(t, a, sb) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + go a.keepLease(ctx) + synctest.Wait() + + cut.cut.Store(true) + time.Sleep(leaseTTL) + synctest.Wait() + cut.cut.Store(false) + cut.rowsDelay.Store(int64(leaseTTL)) + time.Sleep(leaseRenew) + time.Sleep(leaseTTL + time.Second) + synctest.Wait() + if _, _, _, err := a.WakeAgentSocket(ctx, sb.ID, sb.Token); !errors.Is(err, ErrLeaseLost) { + t.Fatalf("serving after a confirmation that outlasted the lease window: %v, want ErrLeaseLost", err) + } + row, _, _ := c.reg.Get(ctx, sb.ID) + if _, err := c.reg.Take(ctx, sb.ID, row.Epoch, "b"); !errors.Is(err, archives.ErrLost) { + t.Errorf("take while the slow unfence is renewing: %v, want ErrLost: renewals must go on", err) + } + + cut.rowsDelay.Store(0) + time.Sleep(2 * leaseRenew) + synctest.Wait() + assertServed(t, a, sb.ID, sb.Token) + }) +} + +func TestLeaseAdoptionDuringUnfenceDropsTheClaimWithoutServingIt(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + c := newReplicaCluster(t) + a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + cut := &cutRegistry{Registry: c.reg} + a.archives = cut + sb := mustClaim(t, a, testKey) + vm := lockedVMName(sb) + replicateNow(t, a, sb) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + go a.keepLease(ctx) + synctest.Wait() + + cut.cut.Store(true) + time.Sleep(leaseTTL) + synctest.Wait() + fencedAtTake := false + cut.beforeTake = func() { + fencedAtTake = a.fenced.Load() + c.expire("a") + if adopted, err := b.AdoptArchived(ctx, sb.ID, sb.Token); err != nil || !adopted { + t.Errorf("adopt during the owner's unfence: %v %v", adopted, err) + } + } + cut.cut.Store(false) + time.Sleep(leaseRenew) + synctest.Wait() + + if !fencedAtTake { + t.Error("the owner served again before it confirmed the row") + } + if _, _, _, err := a.WakeAgentSocket(ctx, sb.ID, sb.Token); !errors.Is(err, ErrUnknownSandbox) { + t.Errorf("the owner after losing the row: %v, want ErrUnknownSandbox", err) + } + if !a.eng.(*fakeEngine).removed(vm) { + t.Error("the owner kept its VM of the adopted claim") + } + if _, _, _, err := b.WakeAgentSocket(ctx, sb.ID, sb.Token); err != nil { + t.Errorf("the adopter's wake: %v", err) + } + }) +} + +func TestLeaseConfirmRetriesOnceOnAStaleEpoch(t *testing.T) { + c := newReplicaCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + replicateNow(t, a, sb) + stale, _, _ := c.reg.Get(t.Context(), sb.ID) + replicateNow(t, a, sb) + + if keep, err := a.confirmOwnership(t.Context(), sb, stale); err != nil || !keep { + t.Fatalf("confirm on a row read before a newer round: %v %v, want kept", keep, err) + } + current, _, _ := c.reg.Get(t.Context(), sb.ID) + if a.epochOf(sb) != current.Epoch { + t.Errorf("claim epoch %d, want the row's %d", a.epochOf(sb), current.Epoch) + } +} + func TestLeaseAdoptionWaitsOutTheFenceAndTheOldOwnerDropsItsCopy(t *testing.T) { for _, tt := range []struct { name string @@ -232,7 +329,17 @@ func TestLeaseAdoptionWaitsOutTheFenceAndTheOldOwnerDropsItsCopy(t *testing.T) { type cutRegistry struct { archives.Registry - cut atomic.Bool + cut atomic.Bool + rowsDelay atomic.Int64 + beforeTake func() +} + +func (r *cutRegistry) Take(ctx context.Context, id string, expect int64, owner string) (int64, error) { + if hook := r.beforeTake; hook != nil { + r.beforeTake = nil + hook() + } + return r.Registry.Take(ctx, id, expect, owner) } func (r *cutRegistry) Renew(ctx context.Context, node string, ttl time.Duration) error { @@ -246,6 +353,7 @@ func (r *cutRegistry) Rows(ctx context.Context, ids []string) (map[string]archiv if r.cut.Load() { return nil, archives.ErrUnavailable } + time.Sleep(time.Duration(r.rowsDelay.Load())) return r.Registry.Rows(ctx, ids) } diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index af9b71f7..396d310e 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -348,6 +348,7 @@ type Manager struct { // fenced gates replicated claims while the lease is overdue; replicas belongs to the pass holding replicating. lease leaseTiming fenced atomic.Bool + unfencing atomic.Bool fenceCh atomic.Pointer[chan struct{}] replicating atomic.Bool replicas map[string]replicaState diff --git a/sandboxd/pool/replica.go b/sandboxd/pool/replica.go index 9fa637a3..9863b4d8 100644 --- a/sandboxd/pool/replica.go +++ b/sandboxd/pool/replica.go @@ -269,7 +269,11 @@ func (m *Manager) replicaOf(sb *types.Sandbox) string { } func (m *Manager) replicaConfirmed(sb *types.Sandbox) bool { + return m.epochOf(sb) != 0 +} + +func (m *Manager) epochOf(sb *types.Sandbox) int64 { m.mu.Lock() defer m.mu.Unlock() - return sb.ArchiveEpoch != 0 + return sb.ArchiveEpoch } From 0b3d433630da91cbc0bfc36ba49c7a818165804c Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 03:38:43 +0800 Subject: [PATCH 09/36] server: a fenced node answers 404 on /owner for its replicated claims AgentSocket had no fence check, so /owner kept answering 200 for a claim every other call on the node answered with 503, and the Lookup scatter kept binding to it. The node now answers 404 there and the scatter moves on; once the lease has expired another node adopts the claim. --- sandboxd/pool/claim.go | 3 +++ sandboxd/pool/lease_test.go | 18 ++++++++++++++++++ sandboxd/server/server.go | 7 ++++++- sandboxd/server/server_test.go | 18 ++++++++++++++++++ 4 files changed, 45 insertions(+), 1 deletion(-) diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index 614f0e8d..1b805ff1 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -216,6 +216,9 @@ func (m *Manager) AgentSocket(id, token string) (string, error) { if !ok { return "", ErrUnknownSandbox } + if m.fenced.Load() && sb.Replicated() { + return "", ErrLeaseLost + } // no activity stamp: a control-plane poll must not keep an idle sandbox awake return sb.VsockSocket, nil } diff --git a/sandboxd/pool/lease_test.go b/sandboxd/pool/lease_test.go index 8b5eb96a..05e8091f 100644 --- a/sandboxd/pool/lease_test.go +++ b/sandboxd/pool/lease_test.go @@ -251,6 +251,24 @@ func TestLeaseAdoptionDuringUnfenceDropsTheClaimWithoutServingIt(t *testing.T) { }) } +func TestLeaseFencedNodeDoesNotAnswerAsOwnerOfReplicatedClaims(t *testing.T) { + c := newReplicaCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + replicated, plain := mustClaim(t, a, testKey), mustClaim(t, a, testKey) + replicateNow(t, a, replicated) + + if _, err := a.AgentSocket(replicated.ID, replicated.Token); !errors.Is(err, ErrLeaseLost) { + t.Errorf("owner lookup of a replicated claim while fenced: %v, want ErrLeaseLost", err) + } + if _, err := a.AgentSocket(plain.ID, plain.Token); err != nil { + t.Errorf("owner lookup of an unreplicated claim while fenced: %v", err) + } + a.fenced.Store(false) + if _, err := a.AgentSocket(replicated.ID, replicated.Token); err != nil { + t.Errorf("owner lookup once the fence lifted: %v", err) + } +} + func TestLeaseConfirmRetriesOnceOnAStaleEpoch(t *testing.T) { c := newReplicaCluster(t) a := c.node(t, "a", t.TempDir(), keepAnHour) diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 473cd4aa..3521dedd 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -701,7 +701,12 @@ func (s *Server) handleOwner(w http.ResponseWriter, r *http.Request) { return } id := r.PathValue("id") - if _, err := s.mgr.AgentSocket(id, token); err != nil { + _, err := s.mgr.AgentSocket(id, token) + if errors.Is(err, pool.ErrLeaseLost) { + writeErr(w, http.StatusNotFound, "not owned here") + return + } + if err != nil { adopted, adoptErr := s.mgr.AdoptArchived(r.Context(), id, token) switch { case adoptErr != nil: diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index 1cc29b51..4fb76cb8 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -1865,6 +1865,24 @@ func TestOwnerEndpoint(t *testing.T) { } } +func TestOwnerEndpointAnswers404ForAClaimItsFencedNodeCannotServe(t *testing.T) { + adopted := false + mgr := &fakeManager{ + socket: func(string, string) (string, error) { return "", pool.ErrLeaseLost }, + adopt: func(string, string) (bool, error) { + adopted = true + return true, nil + }, + } + ts := newMeshTestServer(t, "", mgr, nil, nil) + + resp := doReq(t, http.MethodGet, ts.URL+"/v1/sandboxes/sb_1/owner", "good", "") + defer resp.Body.Close() + if resp.StatusCode != http.StatusNotFound || adopted { + t.Errorf("fenced owner: status %d adopt tried %v, want 404 without an adoption", resp.StatusCode, adopted) + } +} + func TestOwnerEndpointAdoptsAnArchivedClaimItDoesNotHold(t *testing.T) { var adoptErr error mgr := &fakeManager{ From 6144cdb6aef1c376d909c3306e5e81b86a78e66a Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 03:38:44 +0800 Subject: [PATCH 10/36] pool: replica rounds follow the policy pool and retire stale rows The pool's replicate setting is read by the claim's policy key, as capture_trim is, so a claim cloned from a template promoted out of a replicating pool follows that pool. A replicated claim that gains volumes retires its replica instead of leaving an adoptable stale row. A retire that raced an archive finds no replica left and leaves the archive row alone (the guard is in dropReplica; this adds the tests). --- sandboxd/pool/replica.go | 5 ++- sandboxd/pool/replica_test.go | 58 +++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/sandboxd/pool/replica.go b/sandboxd/pool/replica.go index 9863b4d8..ee004201 100644 --- a/sandboxd/pool/replica.go +++ b/sandboxd/pool/replica.go @@ -28,7 +28,7 @@ func (m *Manager) replicateOnce(ctx context.Context) { next := make(map[string]replicaState, len(m.replicas)) m.mu.Lock() for id, sb := range m.claimed { - every, st := v.poolReplicate[sb.Key], m.replicas[id] + every, st := v.poolReplicate[sb.PolicyKey()], m.replicas[id] switch { case every <= 0: if sb.Replicated() { @@ -36,6 +36,9 @@ func (m *Manager) replicateOnce(ctx context.Context) { } case !m.replicable(sb): case m.hasVolumeHolds(sb): + if sb.Replicated() { + retire = append(retire, sb) + } if !st.skipLogged { withVolumes = append(withVolumes, id) } diff --git a/sandboxd/pool/replica_test.go b/sandboxd/pool/replica_test.go index 3a9f398c..d2eb3222 100644 --- a/sandboxd/pool/replica_test.go +++ b/sandboxd/pool/replica_test.go @@ -214,6 +214,64 @@ func TestReplicaRowOfAHibernatedClaimHasNoRetentionDeadline(t *testing.T) { }) } +func TestReplicaRetireAfterAnArchiveLeavesTheArchiveRowAlone(t *testing.T) { + c := newReplicaCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + replicateNow(t, a, sb) + mustArchive(t, a, sb) + row, _, _ := c.reg.Get(t.Context(), sb.ID) + ck := sb.ArchiveCk + + a.retireReplica(t.Context(), sb) + rec, ok, _ := c.reg.Get(t.Context(), sb.ID) + if !ok || rec.Consumed || rec.Epoch != row.Epoch || rec.Checkpoint != ck || !ckExists(t, a, ck) { + t.Errorf("row %+v present %v, archive ck kept %v, want the archive's row and checkpoint untouched by a stale retire", rec, ok, ckExists(t, a, ck)) + } +} + +func TestReplicaClaimThatGainsVolumesStopsBeingReplicated(t *testing.T) { + c := newReplicaCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + a.fenced.Store(false) + sb := mustClaim(t, a, testKey) + replicateNow(t, a, sb) + ck := a.replicaOf(sb) + a.mu.Lock() + sb.Volumes = []types.Volume{{Name: "data"}} + a.mu.Unlock() + + a.replicateOnce(t.Context()) + waitFor(t, func() bool { return !a.replicating.Load() }) + if _, ok, _ := c.reg.Get(t.Context(), sb.ID); ok || a.replicaOf(sb) != "" || ckExists(t, a, ck) || sb.Replicated() { + t.Errorf("row present %v, replica %q, ck kept %v, replicated %v, want the stale row and replica retired", ok, a.replicaOf(sb), ckExists(t, a, ck), sb.Replicated()) + } + if _, held := a.byID(sb.ID); !held { + t.Error("retiring the replica dropped the claim") + } +} + +func TestReplicaFollowsThePolicyPoolOfAPromotedTemplate(t *testing.T) { + c := newReplicaCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + a.fenced.Store(false) + sb := mustClaim(t, a, testKey) + a.mu.Lock() + sb.Key, sb.PolicySource = types.PoolKey{Template: "promoted:v1", Net: types.NetNone, Size: testKey.Size}, testKey + a.mu.Unlock() + + a.replicateOnce(t.Context()) + waitFor(t, func() bool { return !a.replicating.Load() }) + a.mu.Lock() + a.replicas[sb.ID] = replicaState{tried: time.Now().Add(-time.Hour)} + a.mu.Unlock() + a.replicateOnce(t.Context()) + waitFor(t, func() bool { return !a.replicating.Load() }) + if a.replicaOf(sb) == "" { + t.Error("a claim of a template promoted out of a replicating pool was not replicated") + } +} + func TestReplicaFailedCaptureKeepsTheOldReplicaAndRow(t *testing.T) { c := newReplicaCluster(t) a := c.node(t, "a", t.TempDir(), keepAnHour) From 1e506f0887d7c3661fce663d29d0ad91022cba96 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 03:39:28 +0800 Subject: [PATCH 11/36] docs: replica intent, row deadline, unfence confirmation and the owner answer State the fence guarantee with its precise conditions, the retention rule of replica rows, the unfence confirmation, and the /owner answer of a fenced node; align the lease, replicate_seconds and archive_failover comments with what the code does. --- docs/cluster.md | 63 ++++++++++++++++++++++++++++----------- docs/deploy.md | 2 +- docs/sandboxd-api.md | 4 ++- sandboxd/config/config.go | 4 +-- sandboxd/pool/pool.go | 2 +- sandboxd/pool/replica.go | 2 +- 6 files changed, 54 insertions(+), 23 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index 78f4bdfc..087df9e6 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -241,8 +241,9 @@ was waking or releasing when it died. With `archive_failover` every node renews one row of `sandboxd_nodes (cell, node, lease_until)` every 5 seconds, with a single upsert on the database clock -that covers the whole node: `lease_until = now() + 15 s`. The lease outlives a -missed renewal. The table is created at start when it is missing, and a node +that covers the whole node: `lease_until = now() + 15 s`. A renewal that fails +once does not lose the lease or trip the fence, as long as the next one +answers within 2 seconds of its start. The table is created at start when it is missing, and a node with no row counts as expired. Adoption and the departed-owner sweep consult the lease, never mesh membership, so a node the mesh lost but the database still hears keeps its claims, and a node the mesh still lists but the database no @@ -263,22 +264,34 @@ checkpoint store and keeps each claim's registry row on the latest snapshot: the claim's row is put with this node as owner and the replica as checkpoint, and the previous replica is deleted. A round that fails logs a warning and retries at the next interval; the old replica and row stay as they were, and - the claim keeps running. A claim gets its first replica one interval after + the claim keeps running. The round journals the checkpoint it is about to + register before it writes the row, and deletes it only when the write is + refused. A write whose outcome is unknown leaves the checkpoint in place, and + the next round, a release, an archive, a wake, the hourly sweep and a restart + settle it by reading the row: a row on that checkpoint promotes it, another + owner's row drops the claim, and an absent row means the adopter finished + when the checkpoint is gone and a write that never landed when it is still + there. A claim with such an intent counts as replicated for the fence. A claim gets its first replica one interval after it is claimed (or after a restart). A hibernated, archived or failed claim, one whose tenant was removed, and one with volumes (capture refuses them; - logged once per claim) is skipped. Rounds share the maintenance budget with + logged once per claim) is skipped; a replicated claim that gains volumes + drops its row and replica. The claim's policy pool, not its own template key, + picks the interval, so a claim cloned from a template promoted out of a + replicating pool follows that pool. Rounds share the maintenance budget with refill and never take the share refill keeps. A node whose lease is overdue starts none, and a round whose registry is down fails before it captures. - **Restore.** A lost node's replica is adopted like an archive: the first `Lookup` that reaches another node after the lease expires takes the row, - records the claim as archived on the replica, and the next call wakes it + records the claim as archived on the replica with the adopter pool's + retention window, and the next call wakes it through the archive wake, which restores the guest with its memory and processes. The claim resumes from its last snapshot: everything after it, and the connections open at the time, are lost, so the data loss is at most the interval. - **Lifecycle.** A release consumes the row, then deletes the replica and the tombstone. A hibernate leaves the row on the last replica. An archive puts its - own checkpoint over the row and deletes the replica. A lease expiry + own checkpoint over the row, with its retention deadline, and deletes the + replica. A lease expiry with `on_expire` release drops the row and the replica first, and the next reap destroys the claim; with `on_expire` archive, the archive puts its checkpoint over the row. Turning `replicate_seconds` @@ -286,18 +299,34 @@ checkpoint store and keeps each claim's registry row on the latest snapshot: the next pass. A restart keeps the node's own live row for a running claim and takes the checkpoint a crashed round left on it; a claim whose row another node holds, or whose row a finished adopter consumed, is dropped with its VM. - The hourly sweep is unchanged apart from the lease rule. + A replica row has no retention deadline: the hourly sweep never purges it, + because no round refreshes it while the claim is hibernated, and retention + starts when the claim is archived or adopted. The rows of a node that never + returns therefore stay until a client adopts them, a bounded residue of one + row and one checkpoint per claim. Apart from that and the lease rule the + sweep is unchanged. - **Self-fencing.** A node whose renewal is overdue by 12 seconds (the lease - length minus a 3 second margin; one missed renewal never gets there) stops - serving every replicated claim it holds: relay, exec, ports and preview answer `503 lease lost; retry`. The - guest has no NIC on the `none` lane, so it is unreachable except through the - node, and the node destroys nothing. Claims that are not replicated stay - served. When a renewal succeeds again, the node reads the rows of its - replicated claims: those it still owns are served again, and those another - node took, or took and finished, are dropped with their VMs. A node that - starts fenced and lifts the fence with its first renewal. The margin makes - the fence close 3 seconds before another node may adopt, so no moment has two - serving copies, partitions included. When the fence closes, every open + length minus a 3 second margin, counted from the start of the last renewal + that answered) stops serving every replicated claim it holds: relay, exec, + ports, preview and `/owner` answer `503 lease lost; retry` (`/owner` answers + 404, so a `Lookup` moves on to the other nodes). The guest has no NIC on the + `none` lane, so it is unreachable except through the node, and the node + destroys nothing. Claims that are not replicated stay served. A node starts + fenced and lifts the fence after its first renewal that is followed by a + confirmation. The confirmation runs beside the renew loop, which keeps + renewing: for each replicated claim the node takes the row back for itself in + one compare-and-set, which the row lock serializes against adopters, and + retries once on a stale epoch. A claim whose row another node took, or took + and finished, is forgotten at once and its VM removed in the background. The + fence lifts only if the whole confirmation finished within 12 seconds of the + start of the renewal that began it; a slower one leaves the node fenced for + the next renewal to repeat. The margin makes the fence close 3 seconds before + another node may adopt, so no moment has two serving copies, partitions + included. This holds while the node's clock and the database's run at the same + rate to within that margin over 12 seconds, and ignores skew between them, + because both sides count from a renewal's own start. The guest of a claim the + fence closed keeps running without a NIC until the node returns and removes + it, and its state is discarded. When the fence closes, every open stream of a replicated claim on the node is closed too: the agent relay (including parked keep-alive relays, whose later RPCs never pass the resolution gate), the exec call, port relays and preview requests. A client diff --git a/docs/deploy.md b/docs/deploy.md index 9b970a68..8026ef7c 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -143,7 +143,7 @@ gets the intersection of the two. | `capture_trim` (pool entry) | `false` | Trim the guest's copy-on-write disk before a promote or checkpoint of this pool's sandboxes (and of the clones of a template promoted from it), so the record carries only live data: blocks a build deleted — a `COPY` step's archive, a `RUN` step's caches — are no longer captured. sandboxd mounts the disk init resolved from `cocoon.cow` a second time at `/run/sandboxd-trim` through silkd as root, runs `fstrim` and unmounts, under the capture's transition lock (about 15 ms on a warm clone). A failed trim is logged and the capture proceeds, larger. A hibernated or archived sandbox is captured untrimmed. Config-owned like `egress`: `PUT /v1/pools` rejects it | | `egress_classes` | unset | named tenant egress layers: `[{"name": "desk", "egress": {…}, "egress_upstream_env": "DESK_UPSTREAM"}]`. A tenant names one with `egress_class` through the [tenant API](sandboxd-api.md#tenants-v1tenants) and its claims take that policy intersected with the pool's. `egress` is required; `intercept` stays a pool-rule flag. Reloadable: a class's policy reaches live claims on their next request. A class removed while tenants still name it leaves them with no egress and is logged at load and at reload | | `egress_upstream_env` (pool or egress class entry) | unset | names a node environment variable holding the default upstream proxy URL for the pool's or class's claims (class before pool, a claim's own entry before both); needs `egress_upstream`, and the URL must be on its `allow`. Read and checked at startup. Config-owned like `egress`: `PUT /v1/pools` rejects it | -| `replicate_seconds` (pool entry) | 0 (off) | snapshot every running claim of this pool to the checkpoint store this often and point the claim's registry row at the snapshot, so another node restores the claim, memory and processes included, when this node is lost (see [running claims across node loss](cluster.md#running-claims-across-node-loss)). Each round pauses the guest for the length of a memory and disk capture, so the interval is also the data a loss costs. Requires `archive_failover` and a `none`-lane pool, both checked at config load; a claim with volumes is skipped. Config-owned like `egress`: `PUT /v1/pools` rejects it, and a reload applies it (a pool set back to 0 drops its claims' rows and replicas) | +| `replicate_seconds` (pool entry) | 0 (off) | snapshot every running claim of this pool to the checkpoint store this often and point the claim's registry row at the snapshot, so another node restores the claim, memory and processes included, when this node is lost (see [running claims across node loss](cluster.md#running-claims-across-node-loss)). Each round pauses the guest for the length of a memory and disk capture, so the interval is also the data a loss costs. The claim's row carries no retention deadline while it runs. Requires `archive_failover` and a `none`-lane pool, both checked at config load; a claim with volumes is skipped. Config-owned like `egress`: `PUT /v1/pools` rejects it, and a reload applies it (a pool set back to 0 drops its claims' rows and replicas) | | `storage` (pool entry) | unset (cocoon's default, 10G) | size of the copy-on-write disk the pool's golden is cold-booted with, in cocoon's own `--storage` spelling (`"40G"`, `"40GiB"`, `"40Gi"`: Docker/Kubernetes units, all binary) and passed to it verbatim; every clone, fork, checkpoint branch and promoted template of the pool inherits it. At least cocoon's default of 10G. Equal sizes spelled differently build the same golden; a changed size rebuilds the golden on the next build, and claims already out keep their disk. Config-owned like `egress`: `PUT /v1/pools` rejects it | | `max_claims` | 0 (unlimited) | node-wide cap on live claims; claim/fork/branch requests beyond it answer 429 with the pool state unharmed (on a cluster a non-volume claim tries a warm-peer redirect first; a volume claim answers the 429 with no redirect) | | `audit_log` | false | append every relayed frame that opens an RPC — its op + addressing fields, never payloads — to `/audit.jsonl`; continuation `data`, `data_end`, `stdin`, and `stdin_close` frames pass through unrecorded. The file is size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`), plus `method` (`GET`, `CONNECT`, `SOCKS5`, …), `decision` (`allow`, `deny`, `rate_limited`), `secret` (the ref name, never its value), `upstream` (the [upstream proxy](egress.md#upstream-proxies)'s `host:port`, when one carried it), `tenant`, and on an allowed connection `src_addr` and `dst_addr` (the node-side and destination `ip:port` of the upstream TCP connection, the tunnel target through an upstream proxy) on `egress` records, which a relayed frame cannot set; preview accesses record as op `preview`, one per request; a tenant change records as op `tenants` with the `added`, `changed` and `removed` names, never a token. An opening frame whose first line exceeds 4 KiB records as op `oversized` with no addressing fields | diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 2d8ff594..1334ca80 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -1027,7 +1027,9 @@ this node owns the sandbox, 404 otherwise. With [`archive_failover`](cluster.md#archived-claims-across-node-loss), a node that does not hold the sandbox adopts it when its row names an owner whose node lease expired, and answers 200; 503 with `Retry-After` when the meta store is -down. A row whose owner's lease is live is never adopted: 404. With `client_advertise` configured, +down. A row whose owner's lease is live is never adopted: 404. A node whose +lease is overdue answers 404 for the replicated claims it holds, so the +`Lookup` scatter moves on. With `client_advertise` configured, `owner_addr` is that node's full HTTP(S) origin instead; the same contract applies to claim and fork responses. Used by the SDK's `Lookup` scatter. diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index 684a61bb..bcc4d57e 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -51,7 +51,7 @@ type PoolSpec struct { CaptureTrim bool `json:"capture_trim,omitzero"` // Storage sizes the golden's copy-on-write disk in cocoon's --storage spelling; every clone inherits it, and empty keeps cocoon's default. Storage string `json:"storage,omitempty"` - // ReplicateSeconds snapshots each running claim of the pool to the archive registry at this interval; zero turns replication off. + // ReplicateSeconds snapshots each running claim of the pool to the checkpoint store at this interval; zero turns replication off. ReplicateSeconds int `json:"replicate_seconds,omitzero"` IdleHibernateSeconds int `json:"idle_hibernate_seconds,omitzero"` @@ -242,7 +242,7 @@ type Config struct { IdleHibernateSeconds int `json:"idle_hibernate_seconds,omitzero"` ArchiveAfterSeconds int `json:"archive_after_seconds,omitzero"` ArchiveDeleteAfterSeconds int `json:"archive_delete_after_seconds,omitzero"` - // ArchiveFailover lets another node adopt this node's archived claims once it leaves the mesh. + // ArchiveFailover lets another node adopt this node's archived and replicated claims once its node lease expires. ArchiveFailover bool `json:"archive_failover,omitzero"` PreviewListen string `json:"preview_listen,omitempty"` diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 396d310e..b6686b5d 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -52,7 +52,7 @@ const ( vsockPollInterval = 100 * time.Millisecond defaultTTL = 5 * time.Minute maxTTL = 24 * time.Hour - // One missed renewal never fences: the next one lands before the fence at ttl-margin. + // A renewal missed once fences only if the next, 2*every after the last success, takes over ttl-margin-2*every to answer. leaseTTL = 15 * time.Second leaseRenew = 5 * time.Second leaseMargin = 3 * time.Second diff --git a/sandboxd/pool/replica.go b/sandboxd/pool/replica.go index ee004201..28c0d071 100644 --- a/sandboxd/pool/replica.go +++ b/sandboxd/pool/replica.go @@ -77,7 +77,7 @@ func (m *Manager) replicateOnce(ctx context.Context) { }() } -// replicable reports whether sb is a running claim; callers hold m.mu. +// replicable excludes a claim that is paused, archived or archiving, failed, or whose tenant was removed; callers hold m.mu. func (m *Manager) replicable(sb *types.Sandbox) bool { _, archiving := m.archiving[sb.ID] return !archiving && sb.HibernateSnap == "" && sb.PendingSnap == "" && sb.ArchiveCk == "" && sb.ArchivePending == "" && sb.Failed == "" && !m.tenantGone(sb.Tenant) From 536d5fc5fc8c565bd503839d63c60240925e0e86 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 03:41:11 +0800 Subject: [PATCH 12/36] pool: drop an unused helper and a shadowed err in replicate --- sandboxd/pool/lease_test.go | 7 +++++++ sandboxd/pool/replica.go | 18 ++++-------------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/sandboxd/pool/lease_test.go b/sandboxd/pool/lease_test.go index 05e8091f..7f90c31e 100644 --- a/sandboxd/pool/lease_test.go +++ b/sandboxd/pool/lease_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/cocoonstack/sandbox/sandboxd/archives" + "github.com/cocoonstack/sandbox/sandboxd/types" ) func TestLeaseRenewLoopKeepsTheNodeLeasedUntilItStops(t *testing.T) { @@ -375,6 +376,12 @@ func (r *cutRegistry) Rows(ctx context.Context, ids []string) (map[string]archiv return r.Registry.Rows(ctx, ids) } +func (m *Manager) epochOf(sb *types.Sandbox) int64 { + m.mu.Lock() + defer m.mu.Unlock() + return sb.ArchiveEpoch +} + func assertServed(t *testing.T, m *Manager, id, token string) { t.Helper() _, _, done, err := m.WakeAgentSocket(t.Context(), id, token) diff --git a/sandboxd/pool/replica.go b/sandboxd/pool/replica.go index 28c0d071..41290544 100644 --- a/sandboxd/pool/replica.go +++ b/sandboxd/pool/replica.go @@ -88,10 +88,10 @@ func (m *Manager) replicate(ctx context.Context, sb *types.Sandbox) error { return fmt.Errorf("replicate %s: %w", sb.ID, err) } sb.Transition.Lock() - err := m.settleReplicaIntent(ctx, sb) + settleErr := m.settleReplicaIntent(ctx, sb) sb.Transition.Unlock() - if err != nil { - return fmt.Errorf("replicate %s: settle intent: %w", sb.ID, err) + if settleErr != nil { + return fmt.Errorf("replicate %s: settle intent: %w", sb.ID, settleErr) } if err := m.checkArchiveRow(ctx, sb.ID); err != nil { if errors.Is(err, archives.ErrLost) { @@ -124,7 +124,7 @@ func (m *Manager) replicate(ctx context.Context, sb *types.Sandbox) error { m.deleteOrphanArchiveCk(ctx, ckID) return errWokeMeanwhile } - if err = m.store.commit(intent); err != nil { + if err := m.store.commit(intent); err != nil { m.clearReplicaIntent(ctx, sb, ckID) m.deleteOrphanArchiveCk(ctx, ckID) return fmt.Errorf("replicate %s: persist intent: %w", sb.ID, err) @@ -270,13 +270,3 @@ func (m *Manager) replicaOf(sb *types.Sandbox) string { defer m.mu.Unlock() return sb.ReplicaCk } - -func (m *Manager) replicaConfirmed(sb *types.Sandbox) bool { - return m.epochOf(sb) != 0 -} - -func (m *Manager) epochOf(sb *types.Sandbox) int64 { - m.mu.Lock() - defer m.mu.Unlock() - return sb.ArchiveEpoch -} From 1db9b5138e2ed91f832b1c136531e7943169cf8c Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 04:03:54 +0800 Subject: [PATCH 13/36] pool: write a replica row only over the epoch this node holds A round that outlived an adoption could Put after the adopter had woken the claim and deleted its row; the upsert inserted a fresh row owned by the old node, and its next renewal confirmed that row and lifted the fence while the adopter served. Registry.Replace is a compare-and-set on the row at the held epoch and owner (with epoch 0, the first replica, it is Put), and a replicated claim's archive writes through it too, since the same hole applied to a row the claim already had. A refused write reads the row: a row this node still owns only had a stale epoch and is refreshed; an absent or foreign row means the claim moved and its copy and VM are dropped. --- sandboxd/archives/archives.go | 2 + .../archives/archivestest/archivestest.go | 64 ++++++++++++++++ sandboxd/archives/pg/pg.go | 10 +++ sandboxd/pool/archive.go | 12 ++- sandboxd/pool/failover.go | 40 ++++++++-- sandboxd/pool/failover_test.go | 16 ++++ sandboxd/pool/replica.go | 6 +- sandboxd/pool/replica_test.go | 76 +++++++++++++++++++ 8 files changed, 217 insertions(+), 9 deletions(-) diff --git a/sandboxd/archives/archives.go b/sandboxd/archives/archives.go index 2f8c3408..23add9bc 100644 --- a/sandboxd/archives/archives.go +++ b/sandboxd/archives/archives.go @@ -33,6 +33,8 @@ type Record struct { type Registry interface { // Put writes rec for rec.Owner as a live row at a new epoch and returns it; ErrLost when another owner holds the row. Put(ctx context.Context, rec Record) (int64, error) + // Replace writes rec over the live row of rec.Owner at epoch expect, at a new epoch; with expect 0 it is Put. + Replace(ctx context.Context, rec Record, expect int64) (int64, error) // Take moves the live row at epoch expect to owner at a new epoch and returns it; another node's row moves only once its owner's lease expired. Take(ctx context.Context, id string, expect int64, owner string) (int64, error) // Consume turns the live row at epoch expect into a tombstone at a new epoch and returns it. diff --git a/sandboxd/archives/archivestest/archivestest.go b/sandboxd/archives/archivestest/archivestest.go index 620c43f1..42b33463 100644 --- a/sandboxd/archives/archivestest/archivestest.go +++ b/sandboxd/archives/archivestest/archivestest.go @@ -54,6 +54,25 @@ func (m *Mem) Put(_ context.Context, rec archives.Record) (int64, error) { return rec.Epoch, nil } +func (m *Mem) Replace(ctx context.Context, rec archives.Record, expect int64) (int64, error) { + if expect == 0 { + return m.Put(ctx, rec) + } + m.mu.Lock() + defer m.mu.Unlock() + if m.down { + return 0, archives.ErrUnavailable + } + old, ok := m.rows[rec.ID] + if !ok || old.Owner != rec.Owner || old.Epoch != expect || old.Consumed { + return 0, archives.ErrLost + } + m.seq++ + rec.Epoch, rec.Consumed = m.seq, false + m.rows[rec.ID] = rec + return rec.Epoch, nil +} + func (m *Mem) Take(_ context.Context, id string, expect int64, owner string) (int64, error) { return m.advance(id, expect, func(rec *archives.Record) bool { if rec.Owner != owner && m.leased(rec.Owner) { @@ -192,6 +211,7 @@ func Run(t *testing.T, open Opener) { t.Run("epochs are never reused", func(t *testing.T) { testEpochsNeverRepeat(t, open(t)) }) t.Run("sweepable lists expired, consumed and own rows", func(t *testing.T) { testSweepable(t, open(t)) }) t.Run("rows lists the named ids", func(t *testing.T) { testRows(t, open(t)) }) + t.Run("replace is a compare-and-set", func(t *testing.T) { testReplace(t, open(t)) }) t.Run("take waits out a live lease", func(t *testing.T) { testTakeLease(t, open(t)) }) t.Run("sweepable spares a live owner's expired row", func(t *testing.T) { testSweepableLease(t, open(t)) }) t.Run("renew moves a lease forward", func(t *testing.T) { testRenewExtends(t, open(t)) }) @@ -394,6 +414,50 @@ func testRows(t *testing.T, r archives.Registry) { } } +func testReplace(t *testing.T, r archives.Registry) { + ctx := t.Context() + absent := record("sb_none", "a", time.Time{}) + if _, err := r.Replace(ctx, absent, 7); !errors.Is(err, archives.ErrLost) { + t.Errorf("replace of an absent row at a held epoch: %v, want ErrLost", err) + } + if _, ok, _ := r.Get(ctx, "sb_none"); ok { + t.Error("a refused replace inserted a row") + } + first, err := r.Replace(ctx, record("sb_1", "a", time.Time{}), 0) + if err != nil || first <= 0 { + t.Fatalf("replace at epoch 0: %d %v, want an insert", first, err) + } + next := record("sb_1", "a", time.Time{}) + next.Checkpoint = "ck_next" + first, err = r.Replace(ctx, next, first) + if err != nil { + t.Fatalf("first replace: %v", err) + } + if _, err = r.Replace(ctx, next, first-1); !errors.Is(err, archives.ErrLost) { + t.Errorf("replace at a stale epoch: %v, want ErrLost", err) + } + if _, err = r.Replace(ctx, record("sb_1", "b", time.Time{}), first); !errors.Is(err, archives.ErrLost) { + t.Errorf("replace by another owner: %v, want ErrLost", err) + } + epoch, err := r.Replace(ctx, next, first) + if err != nil || epoch <= first { + t.Fatalf("replace at the held epoch: %d %v, want a new epoch above %d", epoch, err, first) + } + if got, _, _ := r.Get(ctx, "sb_1"); got.Checkpoint != "ck_next" || got.Epoch != epoch { + t.Errorf("row after the replace: %+v, want ck_next at %d", got, epoch) + } + if _, err = r.Replace(ctx, next, first); !errors.Is(err, archives.ErrLost) { + t.Errorf("replace at the superseded epoch: %v, want ErrLost", err) + } + spent, err := r.Consume(ctx, "sb_1", epoch) + if err != nil { + t.Fatalf("consume: %v", err) + } + if _, err = r.Replace(ctx, next, spent); !errors.Is(err, archives.ErrLost) { + t.Errorf("replace of a tombstone: %v, want ErrLost", err) + } +} + func testTakeLease(t *testing.T, r archives.Registry) { ctx := t.Context() first := mustPut(t, r, record("sb_1", "a", time.Time{})) diff --git a/sandboxd/archives/pg/pg.go b/sandboxd/archives/pg/pg.go index 173cbb15..eab199e2 100644 --- a/sandboxd/archives/pg/pg.go +++ b/sandboxd/archives/pg/pg.go @@ -72,6 +72,16 @@ func (r *Registry) Put(ctx context.Context, rec archives.Record) (int64, error) return epoch, nil } +func (r *Registry) Replace(ctx context.Context, rec archives.Record, expect int64) (int64, error) { + if expect == 0 { + return r.Put(ctx, rec) + } + return r.advance(ctx, `UPDATE sandboxd_archives SET tenant = $5, token_sha256 = $6, epoch = nextval('sandboxd_archives_epoch'), + checkpoint = $7, deadline = $8, claim = $9, updated_at = now() + WHERE cell = $1 AND id = $2 AND epoch = $3 AND owner = $4 AND NOT consumed + RETURNING epoch`, r.cell, rec.ID, expect, rec.Owner, rec.Tenant, rec.TokenSHA256, rec.Checkpoint, nullTime(rec.Deadline), []byte(rec.Claim)) +} + func (r *Registry) Take(ctx context.Context, id string, expect int64, owner string) (int64, error) { return r.advance(ctx, `UPDATE sandboxd_archives SET owner = $4, epoch = nextval('sandboxd_archives_epoch'), updated_at = now() WHERE cell = $1 AND id = $2 AND epoch = $3 AND NOT consumed AND (owner = $4 OR `+leaseExpired+`) diff --git a/sandboxd/pool/archive.go b/sandboxd/pool/archive.go index f2798d87..bf3167bf 100644 --- a/sandboxd/pool/archive.go +++ b/sandboxd/pool/archive.go @@ -12,6 +12,7 @@ import ( "github.com/projecteru2/core/log" + "github.com/cocoonstack/sandbox/sandboxd/archives" "github.com/cocoonstack/sandbox/sandboxd/store" "github.com/cocoonstack/sandbox/sandboxd/types" ) @@ -131,7 +132,16 @@ func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error { m.deleteOrphanArchiveCk(ctx, ck.ID) return fmt.Errorf("archive %s: persist intent: %w", sb.ID, err) } - if epoch, err = m.putArchive(ctx, sb, ck.ID); err != nil { + replicated := m.replicaOf(sb) != "" + if replicated { + epoch, err = m.replaceArchive(ctx, sb, ck.ID) + } else { + epoch, err = m.putArchive(ctx, sb, ck.ID) + } + if err != nil { + if replicated && errors.Is(err, archives.ErrLost) { + _ = m.refusedReplace(ctx, sb) + } sb.Transition.Unlock() m.abandonRefusedPut(ctx, sb, ck.ID, err) return fmt.Errorf("archive %s: registry: %w", sb.ID, err) diff --git a/sandboxd/pool/failover.go b/sandboxd/pool/failover.go index 97ebda69..847d817c 100644 --- a/sandboxd/pool/failover.go +++ b/sandboxd/pool/failover.go @@ -95,20 +95,30 @@ func (m *Manager) adoptHeld(ctx context.Context, cur *types.Sandbox, epoch int64 } func (m *Manager) putArchive(ctx context.Context, sb *types.Sandbox, ck string) (int64, error) { - return m.putRow(ctx, sb, ck, false) + return m.putRow(ctx, sb, ck, false, false) +} + +// replaceArchive writes an archive row over the replica row this node holds, so a row that left meanwhile is not recreated. +func (m *Manager) replaceArchive(ctx context.Context, sb *types.Sandbox, ck string) (int64, error) { + return m.putRow(ctx, sb, ck, false, true) } // putReplica writes a live claim's row with no deadline: retention starts when the claim is archived or adopted. func (m *Manager) putReplica(ctx context.Context, sb *types.Sandbox, ck string) (int64, error) { - return m.putRow(ctx, sb, ck, true) + return m.putRow(ctx, sb, ck, true, false) +} + +// replaceReplica moves the row from the epoch this node holds; a row another node took or deleted refuses the write. +func (m *Manager) replaceReplica(ctx context.Context, sb *types.Sandbox, ck string) (int64, error) { + return m.putRow(ctx, sb, ck, true, true) } -func (m *Manager) putRow(ctx context.Context, sb *types.Sandbox, ck string, replica bool) (int64, error) { +func (m *Manager) putRow(ctx context.Context, sb *types.Sandbox, ck string, replica, replace bool) (int64, error) { if m.archives == nil { return 0, nil } m.mu.Lock() - dto, deadline := dtoOf(sb), sb.Deadline + dto, deadline, held := dtoOf(sb), sb.Deadline, sb.ArchiveEpoch switch { case replica: deadline = time.Time{} @@ -122,10 +132,28 @@ func (m *Manager) putRow(ctx context.Context, sb *types.Sandbox, ck string, repl if err != nil { return 0, fmt.Errorf("encode claim: %w", err) } - return m.archives.Put(ctx, archives.Record{ + rec := archives.Record{ ID: sb.ID, Tenant: sb.Tenant, TokenSHA256: tenants.TokenSHA256(sb.Token), Owner: m.nodeID, Checkpoint: ck, Deadline: deadline, Claim: claim, - }) + } + if replace { + return m.archives.Replace(ctx, rec, held) + } + return m.archives.Put(ctx, rec) +} + +// refusedReplace settles a refused replace: a row this node still owns only had a stale epoch, anything else means the claim moved. +func (m *Manager) refusedReplace(ctx context.Context, sb *types.Sandbox) error { + rec, ok, err := m.archives.Get(ctx, sb.ID) + switch { + case err != nil: + return err + case ok && rec.Owner == m.nodeID: + _, _ = m.commitIfLive(ctx, sb, func() { sb.ArchiveEpoch = max(sb.ArchiveEpoch, rec.Epoch) }) + return nil + } + m.dropAdopted(ctx, sb) + return ErrUnknownSandbox } func (m *Manager) checkArchiveRow(ctx context.Context, id string) error { diff --git a/sandboxd/pool/failover_test.go b/sandboxd/pool/failover_test.go index 8967545f..1ce8fc87 100644 --- a/sandboxd/pool/failover_test.go +++ b/sandboxd/pool/failover_test.go @@ -1960,6 +1960,22 @@ func (r *hookRegistry) Put(ctx context.Context, rec archives.Record) (int64, err return epoch, err } +func (r *hookRegistry) Replace(ctx context.Context, rec archives.Record, expect int64) (int64, error) { + if r.failPut { + return 0, archives.ErrUnavailable + } + if hook := r.beforePut; hook != nil { + r.beforePut = nil + hook() + } + epoch, err := r.Registry.Replace(ctx, rec, expect) + if r.losePutReply && err == nil { + r.losePutReply = false + return 0, archives.ErrUnavailable + } + return epoch, err +} + func (r *hookRegistry) Take(ctx context.Context, id string, expect int64, owner string) (int64, error) { if hook := r.beforeTake; hook != nil { r.beforeTake = nil diff --git a/sandboxd/pool/replica.go b/sandboxd/pool/replica.go index 41290544..02460fef 100644 --- a/sandboxd/pool/replica.go +++ b/sandboxd/pool/replica.go @@ -129,12 +129,14 @@ func (m *Manager) replicate(ctx context.Context, sb *types.Sandbox) error { m.deleteOrphanArchiveCk(ctx, ckID) return fmt.Errorf("replicate %s: persist intent: %w", sb.ID, err) } - epoch, err := m.putReplica(ctx, sb, ckID) + epoch, err := m.replaceReplica(ctx, sb, ckID) switch { case errors.Is(err, archives.ErrLost): m.clearReplicaIntent(ctx, sb, ckID) m.deleteOrphanArchiveCk(ctx, ckID) - m.dropAdopted(ctx, sb) + if settleErr := m.refusedReplace(ctx, sb); settleErr != nil && !errors.Is(settleErr, ErrUnknownSandbox) { + return fmt.Errorf("replicate %s: settle refused write: %w", sb.ID, settleErr) + } return fmt.Errorf("replicate %s: registry: %w", sb.ID, err) case err != nil: return fmt.Errorf("replicate %s: registry: %w", sb.ID, err) diff --git a/sandboxd/pool/replica_test.go b/sandboxd/pool/replica_test.go index d2eb3222..02736d22 100644 --- a/sandboxd/pool/replica_test.go +++ b/sandboxd/pool/replica_test.go @@ -272,6 +272,82 @@ func TestReplicaFollowsThePolicyPoolOfAPromotedTemplate(t *testing.T) { } } +func TestReplicaInFlightRoundCannotRecreateARowTheAdopterFinished(t *testing.T) { + c := newReplicaCluster(t) + a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + vm := lockedVMName(sb) + replicateNow(t, a, sb) + a.archives = &hookRegistry{Registry: c.reg, beforePut: func() { + c.expire("a") + if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { + t.Errorf("adopt: %v %v", adopted, err) + } + if _, _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + t.Errorf("the adopter's wake: %v", err) + } + }} + + if err := a.replicate(t.Context(), sb); !errors.Is(err, archives.ErrLost) { + t.Fatalf("round that outlived the adoption: %v, want ErrLost", err) + } + if rec, ok, _ := c.reg.Get(t.Context(), sb.ID); ok { + t.Errorf("row %+v recreated by the old owner's round", rec) + } + if _, held := a.byID(sb.ID); held || !a.eng.(*fakeEngine).removed(vm) { + t.Errorf("old owner holds the claim %v, VM removed %v, want its copy dropped", held, a.eng.(*fakeEngine).removed(vm)) + } + if _, held := b.byID(sb.ID); !held { + t.Error("the adopter lost the claim it woke") + } +} + +func TestReplicaRoundWithAStaleEpochOfItsOwnRowKeepsTheClaim(t *testing.T) { + c := newReplicaCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + replicateNow(t, a, sb) + row, _, _ := c.reg.Get(t.Context(), sb.ID) + if _, err := c.reg.Take(t.Context(), sb.ID, row.Epoch, "a"); err != nil { + t.Fatalf("take: %v", err) + } + + if err := a.replicate(t.Context(), sb); !errors.Is(err, archives.ErrLost) { + t.Fatalf("round at a stale epoch: %v, want ErrLost", err) + } + if _, held := a.byID(sb.ID); !held { + t.Fatal("a stale epoch of the node's own row dropped the claim") + } + replicateNow(t, a, sb) +} + +func TestReplicaArchiveCannotRecreateARowTheAdopterFinished(t *testing.T) { + c := newReplicaCluster(t) + a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + replicateNow(t, a, sb) + if err := a.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatalf("hibernate: %v", err) + } + c.expire("a") + if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { + t.Fatalf("adopt: %v %v", adopted, err) + } + if _, _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + t.Fatalf("the adopter's wake: %v", err) + } + + if err := a.archive(t.Context(), sb); err == nil { + t.Fatal("an archive over a row the adopter finished succeeded") + } + if rec, ok, _ := c.reg.Get(t.Context(), sb.ID); ok { + t.Errorf("row %+v recreated by the old owner's archive", rec) + } + if _, held := a.byID(sb.ID); held { + t.Error("the old owner kept a claim the adopter runs") + } +} + func TestReplicaFailedCaptureKeepsTheOldReplicaAndRow(t *testing.T) { c := newReplicaCluster(t) a := c.node(t, "a", t.TempDir(), keepAnHour) From 37dfa7920044fa89568319adaf3fb2d6210a8380 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 04:07:53 +0800 Subject: [PATCH 14/36] pool: decide replicated from the pool policy and fold the fence into one atomic A stream resolved before a claim's first replica got no fence channel and survived the fence, because replicated was derived from ReplicaCk. A claim is now replicated when its policy pool replicates it (or already holds a replica from a journal), decided once at add time into a plain field, so an unreplicated claim reads nothing on the data plane and a replicated one reads a single atomic pointer: the fence state, which carries both the flag and the channel and is replaced on trip and on lift. replicate_seconds becomes restart-only, so the decision cannot go stale: a reload that changes it is refused with the pool named. A node that starts fenced lifts the fence only after its first confirmation, so the adopter's wake of an adopted claim of a replicating pool waits for that too. An unreplicated relay, port or preview connection no longer allocates a closure for the fence. CloseOnFence moves above the unexported functions. --- sandboxd/config/reload.go | 12 +++--- sandboxd/config/reload_test.go | 9 ++++- sandboxd/pool/archive.go | 2 - sandboxd/pool/claim.go | 22 ++++++++--- sandboxd/pool/failover.go | 7 ++-- sandboxd/pool/failover_test.go | 6 ++- sandboxd/pool/lease.go | 70 +++++++++++++++++++++------------- sandboxd/pool/lease_test.go | 26 ++++++++++--- sandboxd/pool/pool.go | 27 +++++++------ sandboxd/pool/reconcile.go | 1 - sandboxd/pool/replica.go | 17 +++++---- sandboxd/pool/replica_test.go | 22 +++++++---- sandboxd/server/relay.go | 3 ++ sandboxd/types/types.go | 19 ++++++--- 14 files changed, 157 insertions(+), 86 deletions(-) diff --git a/sandboxd/config/reload.go b/sandboxd/config/reload.go index cf147dbc..7beb4132 100644 --- a/sandboxd/config/reload.go +++ b/sandboxd/config/reload.go @@ -32,8 +32,8 @@ func (c *Config) ReloadDiff(next *Config) (changed, ignored []string, err error) name, _, _ := strings.Cut(cur.Type().Field(i).Tag.Get("json"), ",") switch { case name == "pools": - ch, ig := diffPools(c.Pools, next.Pools) - changed, ignored = append(changed, ch...), append(ignored, ig...) + ch, ig, rs := diffPools(c.Pools, next.Pools) + changed, ignored, restart = append(changed, ch...), append(ignored, ig...), append(restart, rs...) case reflect.DeepEqual(cur.Field(i).Interface(), nxt.Field(i).Interface()): case reloadableFields[name]: changed = append(changed, name) @@ -47,7 +47,7 @@ func (c *Config) ReloadDiff(next *Config) (changed, ignored []string, err error) return changed, ignored, nil } -func diffPools(cur, next []PoolSpec) (changed, ignored []string) { +func diffPools(cur, next []PoolSpec) (changed, ignored, restart []string) { a, b := poolsByKey(cur), poolsByKey(next) all := poolsByKey(slices.Concat(cur, next)) for _, k := range slices.SortedFunc(maps.Keys(all), func(x, y types.PoolKey) int { return strings.Compare(poolLabel(x), poolLabel(y)) }) { @@ -62,7 +62,6 @@ func diffPools(cur, next []PoolSpec) (changed, ignored []string) { {"warmup", slices.Equal(x.Warmup, y.Warmup)}, {"capture_trim", x.CaptureTrim == y.CaptureTrim}, {"storage", x.Storage == y.Storage}, - {"replicate_seconds", x.ReplicateSeconds == y.ReplicateSeconds}, {"egress_upstream_env", x.EgressUpstreamEnv == y.EgressUpstreamEnv}, } { if !f.same { @@ -72,8 +71,11 @@ func diffPools(cur, next []PoolSpec) (changed, ignored []string) { if inA != inB || targetsOf(x) != targetsOf(y) { ignored = append(ignored, label+" targets (PUT /v1/pools owns them)") } + if x.ReplicateSeconds != y.ReplicateSeconds { + restart = append(restart, label+".replicate_seconds") + } } - return changed, ignored + return changed, ignored, restart } func targetsOf(s PoolSpec) poolTargets { diff --git a/sandboxd/config/reload_test.go b/sandboxd/config/reload_test.go index 550a9d83..e7103a80 100644 --- a/sandboxd/config/reload_test.go +++ b/sandboxd/config/reload_test.go @@ -19,7 +19,7 @@ func TestReloadDiffSortsReloadableOwnedAndRestartFields(t *testing.T) { next.EgressInternalAllow = []string{"10.8.0.0/16"} next.Pools = []PoolSpec{ {PoolKey: key, Warm: 5, Egress: &egress.Policy{Allow: []egress.Rule{{Host: "pypi.org"}}}}, - {PoolKey: v13, Warmup: []string{"node", "-e", "0"}, Storage: "40G", ReplicateSeconds: 60}, + {PoolKey: v13, Warmup: []string{"node", "-e", "0"}, Storage: "40G"}, } next.EgressClasses = []EgressClass{{Name: "desk", Egress: &egress.Policy{}}} next.EgressConnRate = &EgressConnRate{PerSecond: 30, Burst: 600} @@ -36,7 +36,6 @@ func TestReloadDiffSortsReloadableOwnedAndRestartFields(t *testing.T) { "pools[desktop:v12 none small].egress", "pools[desktop:v13 none small].warmup", "pools[desktop:v13 none small].storage", - "pools[desktop:v13 none small].replicate_seconds", } if !slices.Equal(changed, wantChanged) { t.Errorf("changed %v, want %v", changed, wantChanged) @@ -49,6 +48,12 @@ func TestReloadDiffSortsReloadableOwnedAndRestartFields(t *testing.T) { t.Errorf("ignored %v, want %v", ignored, wantIgnored) } + replicating := *cur + replicating.Pools = []PoolSpec{{PoolKey: key, Warm: 2, ReplicateSeconds: 60}} + if _, _, err := cur.ReloadDiff(&replicating); err == nil || err.Error() != "pools[desktop:v12 none small].replicate_seconds change only at a restart" { + t.Errorf("ReloadDiff over replicate_seconds: %v, want a restart refusal", err) + } + restart := *cur restart.Listen, restart.Bridges, restart.EgressUsageBytes = "127.0.0.1:8888", []string{"br1"}, true if _, _, err := cur.ReloadDiff(&restart); err == nil || err.Error() != "listen, bridges change only at a restart" { diff --git a/sandboxd/pool/archive.go b/sandboxd/pool/archive.go index bf3167bf..2706d719 100644 --- a/sandboxd/pool/archive.go +++ b/sandboxd/pool/archive.go @@ -156,7 +156,6 @@ func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error { } snap, vmName, sock, prevDeadline, prevPending, replica := sb.HibernateSnap, sb.VMName, sb.VsockSocket, sb.Deadline, sb.ArchivePending, sb.ReplicaCk sb.ArchiveCk, sb.ArchiveEpoch, sb.ArchivePending, sb.ReplicaCk = ck.ID, epoch, "", "" - sb.SyncReplicated() sb.HibernateSnap, sb.VsockSocket, sb.VMName = "", "", "" sb.Deadline = m.retentionDeadline(sb.Key) js := m.store.set(sb) @@ -168,7 +167,6 @@ func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error { // a Release that landed meanwhile already deleted the claim; re-setting it would resurrect it if m.claimed[sb.ID] == sb { sb.ArchiveCk, sb.ArchiveEpoch, sb.ArchivePending, sb.ReplicaCk = "", 0, prevPending, replica - sb.SyncReplicated() sb.HibernateSnap, sb.VsockSocket, sb.VMName = snap, sock, vmName sb.Deadline = prevDeadline rb = m.store.set(sb) diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index 1b805ff1..259a40ef 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -216,7 +216,7 @@ func (m *Manager) AgentSocket(id, token string) (string, error) { if !ok { return "", ErrUnknownSandbox } - if m.fenced.Load() && sb.Replicated() { + if sb.Replicated() && m.fence.Load().fenced { return "", ErrLeaseLost } // no activity stamp: a control-plane poll must not keep an idle sandbox awake @@ -246,8 +246,12 @@ func (m *Manager) dialPort(ctx context.Context, id string, cred Cred, port uint1 sb.Unhold() return nil, err } - stop := CloseOnFence(fenced, conn) - return &heldConn{Conn: conn, release: sync.OnceFunc(func() { stop(); sb.Unhold() })}, nil + release := sb.Unhold + if fenced != nil { + stop := CloseOnFence(fenced, conn) + release = func() { stop(); sb.Unhold() } + } + return &heldConn{Conn: conn, release: sync.OnceFunc(release)}, nil } func (m *Manager) dialPassive(ctx context.Context, id string, port uint16) (net.Conn, error) { @@ -280,8 +284,12 @@ func (m *Manager) dialPassive(ctx context.Context, id string, port uint16) (net. sb.UnholdIdle() return nil, err } - stop := CloseOnFence(fenced, conn) - return &heldConn{Conn: conn, release: sync.OnceFunc(func() { stop(); sb.UnholdIdle() })}, nil + release := sb.UnholdIdle + if fenced != nil { + stop := CloseOnFence(fenced, conn) + release = func() { stop(); sb.UnholdIdle() } + } + return &heldConn{Conn: conn, release: sync.OnceFunc(release)}, nil } // releaseResolved re-checks under m.mu that sb is still the live claim: no double teardown. @@ -424,6 +432,10 @@ func (m *Manager) tenantDelta(tenant string, delta int) { } func (m *Manager) addClaimLocked(sb *types.Sandbox) { + if m.archives != nil { + _, replicating := m.replicaPools[sb.PolicyKey()] + sb.SetReplicated(replicating || sb.HasReplica()) + } m.claimed[sb.ID] = sb m.tenantDelta(sb.Tenant, 1) } diff --git a/sandboxd/pool/failover.go b/sandboxd/pool/failover.go index 847d817c..c91f1589 100644 --- a/sandboxd/pool/failover.go +++ b/sandboxd/pool/failover.go @@ -352,7 +352,7 @@ func (m *Manager) unrowExpired(ctx context.Context, sb *types.Sandbox) { sb.Transition.Lock() defer sb.Transition.Unlock() var err error - if sb.Replicated() { + if m.replicaHeld(sb) { err = m.dropReplica(ctx, sb) } else { _, err = m.consumeArchive(ctx, sb) @@ -510,7 +510,7 @@ func (m *Manager) fenceArchived(ctx context.Context, claims map[string]*types.Sa } case ok && rec.Owner != m.nodeID: drop = true - case (sb.ReplicaCk != "" || sb.ReplicaPending != "") && sb.ArchivePending == "": + case sb.HasReplica() && sb.ArchivePending == "": moved, err := false, error(nil) if !ok { moved, err = m.replicaGone(ctx, sb) @@ -525,7 +525,6 @@ func (m *Manager) fenceArchived(ctx context.Context, claims map[string]*types.Sa sb.ReplicaCk, sb.ArchiveEpoch = rec.Checkpoint, rec.Epoch } sb.ReplicaPending = "" - sb.SyncReplicated() } case sb.ArchivePending != "" || ok && !rec.Consumed: moved, err := m.settleIntent(ctx, id, sb.ArchivePending, rec, ok) @@ -635,5 +634,5 @@ func (m *Manager) archiveRowed(sb *types.Sandbox) bool { } func (m *Manager) expiryDropsRow(sb *types.Sandbox) bool { - return m.archiveRowed(sb) || sb.Replicated() && !m.archivesAtDeadline(sb) + return m.archiveRowed(sb) || sb.HasReplica() && !m.archivesAtDeadline(sb) } diff --git a/sandboxd/pool/failover_test.go b/sandboxd/pool/failover_test.go index 1ce8fc87..71ea25e9 100644 --- a/sandboxd/pool/failover_test.go +++ b/sandboxd/pool/failover_test.go @@ -1845,7 +1845,9 @@ func (c *failoverCluster) renew(t *testing.T, node string) { func (c *failoverCluster) node(t *testing.T, name, dataDir string, deleteAfter int) *Manager { t.Helper() - return c.nodeOn(t, newFakeEngine(), name, dataDir, deleteAfter) + m := c.nodeOn(t, newFakeEngine(), name, dataDir, deleteAfter) + m.liftFence() + return m } func (c *failoverCluster) restart(t *testing.T, old *Manager) *Manager { @@ -1858,7 +1860,7 @@ func (c *failoverCluster) nodeOn(t *testing.T, eng *fakeEngine, name, dataDir st spec := archivePool(deleteAfter) spec.ReplicateSeconds = c.replicate cfg := &config.Config{ - APIToken: "root", DataDir: dataDir, CheckpointDir: c.ckDir, Pools: []config.PoolSpec{spec}, + APIToken: "root", DataDir: dataDir, CheckpointDir: c.ckDir, Pools: []config.PoolSpec{spec, {PoolKey: plainKey, Warm: 1}}, Mesh: &config.MeshConfig{NodeID: name}, } b := testBackends(t, cfg) diff --git a/sandboxd/pool/lease.go b/sandboxd/pool/lease.go index d3d74d46..a9b2d2da 100644 --- a/sandboxd/pool/lease.go +++ b/sandboxd/pool/lease.go @@ -19,9 +19,31 @@ type leaseTiming struct { ttl, every, margin time.Duration } +// fenceState is the whole fence: while fenced no replicated claim is served, and ch closes when it trips. +type fenceState struct { + ch chan struct{} + fenced bool +} + +// CloseOnFence closes c when fenced closes; stop ends the watch. +func CloseOnFence(fenced <-chan struct{}, c io.Closer) (stop func()) { + if fenced == nil { + return func() {} + } + stopped := make(chan struct{}) + go func() { + select { + case <-fenced: + _ = c.Close() + case <-stopped: + } + }() + return sync.OnceFunc(func() { close(stopped) }) +} + // keepLease renews this node's lease until ctx ends and fences its replicated claims when a renewal is overdue. func (m *Manager) keepLease(ctx context.Context) { - fence := time.AfterFunc(m.lease.ttl-m.lease.margin, func() { m.fence(ctx) }) + fence := time.AfterFunc(m.lease.ttl-m.lease.margin, func() { m.trip(ctx) }) defer fence.Stop() tick := time.NewTicker(m.lease.every) defer tick.Stop() @@ -42,7 +64,7 @@ func (m *Manager) renewLease(ctx context.Context, fence *time.Timer) { return } fence.Reset(m.lease.ttl - m.lease.margin - time.Since(start)) - if m.fenced.Load() && m.unfencing.CompareAndSwap(false, true) { + if m.isFenced() && m.unfencing.CompareAndSwap(false, true) { go func() { defer m.unfencing.Store(false) m.unfence(ctx, start) @@ -50,10 +72,22 @@ func (m *Manager) renewLease(ctx context.Context, fence *time.Timer) { } } -func (m *Manager) fence(ctx context.Context) { - if !m.fenced.Swap(true) { - close(*m.fenceCh.Load()) +func (m *Manager) isFenced() bool { + return m.fence.Load().fenced +} + +func (m *Manager) trip(ctx context.Context) { + for { + old := m.fence.Load() + if old.fenced { + return + } + if !m.fence.CompareAndSwap(old, &fenceState{ch: old.ch, fenced: true}) { + continue + } + close(old.ch) log.WithFunc("pool.fence").Warnf(ctx, "node lease not renewed for %s; replicated claims answer 503 until it is", m.lease.ttl-m.lease.margin) + return } } @@ -63,7 +97,7 @@ func (m *Manager) unfence(ctx context.Context, start time.Time) { m.mu.Lock() held := map[string]*types.Sandbox{} for id, sb := range m.claimed { - if sb.Replicated() { + if sb.HasReplica() { held[id] = sb } } @@ -101,8 +135,7 @@ func (m *Manager) unfence(ctx context.Context, start time.Time) { logger.Warn(ctx, "replicated claims stay fenced: confirming ownership outlasted the lease window") return } - m.fenceCh.Store(new(make(chan struct{}))) - m.fenced.Store(false) + m.fence.Store(&fenceState{ch: make(chan struct{})}) logger.Info(ctx, "node lease renewed; replicated claims are served again") } @@ -142,24 +175,9 @@ func (m *Manager) fenceWatch(sb *types.Sandbox) (<-chan struct{}, error) { if !sb.Replicated() { return nil, nil } - if m.fenced.Load() { + st := m.fence.Load() + if st.fenced { return nil, ErrLeaseLost } - return *m.fenceCh.Load(), nil -} - -// CloseOnFence closes c when fenced closes; stop ends the watch. -func CloseOnFence(fenced <-chan struct{}, c io.Closer) (stop func()) { - if fenced == nil { - return func() {} - } - stopped := make(chan struct{}) - go func() { - select { - case <-fenced: - _ = c.Close() - case <-stopped: - } - }() - return sync.OnceFunc(func() { close(stopped) }) + return st.ch, nil } diff --git a/sandboxd/pool/lease_test.go b/sandboxd/pool/lease_test.go index 7f90c31e..531ebf30 100644 --- a/sandboxd/pool/lease_test.go +++ b/sandboxd/pool/lease_test.go @@ -14,6 +14,8 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) +var plainKey = types.PoolKey{Template: "plain:24.04", Net: types.NetNone, Size: testKey.Size} + func TestLeaseRenewLoopKeepsTheNodeLeasedUntilItStops(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := newReplicaCluster(t) @@ -45,7 +47,7 @@ func TestLeaseOverdueRenewalFencesReplicatedClaimsOnly(t *testing.T) { a := c.node(t, "a", t.TempDir(), keepAnHour) cut := &cutRegistry{Registry: c.reg} a.archives = cut - replicated, plain := mustClaim(t, a, testKey), mustClaim(t, a, testKey) + replicated, plain := mustClaim(t, a, testKey), mustClaim(t, a, plainKey) replicateNow(t, a, replicated) ctx, cancel := context.WithCancel(t.Context()) defer cancel() @@ -89,7 +91,7 @@ func TestLeaseFenceClosesOpenStreamsOfReplicatedClaimsOnly(t *testing.T) { a := c.node(t, "a", t.TempDir(), keepAnHour) cut := &cutRegistry{Registry: c.reg} a.archives = cut - replicated, plain := mustClaim(t, a, testKey), mustClaim(t, a, testKey) + replicated, plain, early := mustClaim(t, a, testKey), mustClaim(t, a, plainKey), mustClaim(t, a, testKey) replicateNow(t, a, replicated) ctx, cancel := context.WithCancel(t.Context()) defer cancel() @@ -112,6 +114,7 @@ func TestLeaseFenceClosesOpenStreamsOfReplicatedClaimsOnly(t *testing.T) { "passive": func() (net.Conn, error) { return a.DialPort(ctx, replicated.ID, Cred{Operator: true}, 80) }, "preview": func() (net.Conn, error) { return a.PreviewDial(ctx, replicated.ID, 80) }, "plain": func() (net.Conn, error) { return a.DialPort(ctx, plain.ID, plainCred, 80) }, + "early": func() (net.Conn, error) { return a.DialPort(ctx, early.ID, Cred{Token: early.Token}, 80) }, } { conn, err := dial() if err != nil { @@ -227,7 +230,7 @@ func TestLeaseAdoptionDuringUnfenceDropsTheClaimWithoutServingIt(t *testing.T) { synctest.Wait() fencedAtTake := false cut.beforeTake = func() { - fencedAtTake = a.fenced.Load() + fencedAtTake = a.isFenced() c.expire("a") if adopted, err := b.AdoptArchived(ctx, sb.ID, sb.Token); err != nil || !adopted { t.Errorf("adopt during the owner's unfence: %v %v", adopted, err) @@ -255,8 +258,9 @@ func TestLeaseAdoptionDuringUnfenceDropsTheClaimWithoutServingIt(t *testing.T) { func TestLeaseFencedNodeDoesNotAnswerAsOwnerOfReplicatedClaims(t *testing.T) { c := newReplicaCluster(t) a := c.node(t, "a", t.TempDir(), keepAnHour) - replicated, plain := mustClaim(t, a, testKey), mustClaim(t, a, testKey) + replicated, plain := mustClaim(t, a, testKey), mustClaim(t, a, plainKey) replicateNow(t, a, replicated) + a.tripFence() if _, err := a.AgentSocket(replicated.ID, replicated.Token); !errors.Is(err, ErrLeaseLost) { t.Errorf("owner lookup of a replicated claim while fenced: %v, want ErrLeaseLost", err) @@ -264,7 +268,7 @@ func TestLeaseFencedNodeDoesNotAnswerAsOwnerOfReplicatedClaims(t *testing.T) { if _, err := a.AgentSocket(plain.ID, plain.Token); err != nil { t.Errorf("owner lookup of an unreplicated claim while fenced: %v", err) } - a.fenced.Store(false) + a.liftFence() if _, err := a.AgentSocket(replicated.ID, replicated.Token); err != nil { t.Errorf("owner lookup once the fence lifted: %v", err) } @@ -335,7 +339,7 @@ func TestLeaseAdoptionWaitsOutTheFenceAndTheOldOwnerDropsItsCopy(t *testing.T) { if _, held := a.byID(sb.ID); held || !a.eng.(*fakeEngine).removed(vm) { t.Errorf("old owner holds the claim %v, VM removed %v, want its copy dropped and its VM removed", held, a.eng.(*fakeEngine).removed(vm)) } - if a.fenced.Load() { + if a.isFenced() { t.Error("the old owner stays fenced after its lease renewed") } if _, _, _, err := b.WakeAgentSocket(ctx, sb.ID, sb.Token); err != nil { @@ -382,6 +386,16 @@ func (m *Manager) epochOf(sb *types.Sandbox) int64 { return sb.ArchiveEpoch } +func (m *Manager) tripFence() { + ch := make(chan struct{}) + close(ch) + m.fence.Store(&fenceState{ch: ch, fenced: true}) +} + +func (m *Manager) liftFence() { + m.fence.Store(&fenceState{ch: make(chan struct{})}) +} + func assertServed(t *testing.T, m *Manager, id, token string) { t.Helper() _, _, done, err := m.WakeAgentSocket(t.Context(), id, token) diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index b6686b5d..5c64479f 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -345,13 +345,13 @@ type Manager struct { nodeID string orphanSweep atomic.Bool adoptFlights singleflight.Group - // fenced gates replicated claims while the lease is overdue; replicas belongs to the pass holding replicating. - lease leaseTiming - fenced atomic.Bool - unfencing atomic.Bool - fenceCh atomic.Pointer[chan struct{}] - replicating atomic.Bool - replicas map[string]replicaState + // fence gates replicated claims while the lease is overdue; replicaPools is fixed at start; replicas belongs to the pass holding replicating. + lease leaseTiming + fence atomic.Pointer[fenceState] + unfencing atomic.Bool + replicaPools map[types.PoolKey]struct{} + replicating atomic.Bool + replicas map[string]replicaState view atomic.Pointer[configView] // cfg is the config the view was built from; reloadMu orders reloads, the only writers of cfg and view. @@ -513,13 +513,18 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg } m.out = outbound.New(m.outboundOptions(ca)) m.configSeedHash = poolSeedHash(cfg.Pools) + start := &fenceState{ch: make(chan struct{}), fenced: m.archives != nil} if m.archives != nil { m.nodeID = cfg.Mesh.ID() - closed := make(chan struct{}) - close(closed) - m.fenceCh.Store(&closed) - m.fenced.Store(true) + close(start.ch) + m.replicaPools = map[types.PoolKey]struct{}{} + for _, spec := range cfg.Pools { + if spec.ReplicateSeconds > 0 { + m.replicaPools[spec.PoolKey] = struct{}{} + } + } } + m.fence.Store(start) if err := m.adoptPools(ctx); err != nil { return nil, err } diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 6e6bfcf9..11cbc259 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -42,7 +42,6 @@ func (m *Manager) Reconcile(ctx context.Context) error { m.mu.Lock() for _, sb := range claims { sb.SetEnv(sb.Env) - sb.SyncReplicated() rec, ok := live[sb.VMName] switch { case sb.ArchiveCk != "": diff --git a/sandboxd/pool/replica.go b/sandboxd/pool/replica.go index 02460fef..4706ddd4 100644 --- a/sandboxd/pool/replica.go +++ b/sandboxd/pool/replica.go @@ -19,7 +19,7 @@ type replicaState struct { } func (m *Manager) replicateOnce(ctx context.Context) { - if m.archives == nil || m.fenced.Load() || !m.replicating.CompareAndSwap(false, true) { + if m.archives == nil || m.isFenced() || !m.replicating.CompareAndSwap(false, true) { return } now, v := time.Now(), m.view.Load() @@ -31,12 +31,12 @@ func (m *Manager) replicateOnce(ctx context.Context) { every, st := v.poolReplicate[sb.PolicyKey()], m.replicas[id] switch { case every <= 0: - if sb.Replicated() { + if sb.HasReplica() { retire = append(retire, sb) } case !m.replicable(sb): case m.hasVolumeHolds(sb): - if sb.Replicated() { + if sb.HasReplica() { retire = append(retire, sb) } if !st.skipLogged { @@ -116,7 +116,6 @@ func (m *Manager) replicate(ctx context.Context, sb *types.Sandbox) error { var intent claimSnapshot if ok { sb.ReplicaPending = ckID - sb.SyncReplicated() intent = m.store.set(sb) } m.mu.Unlock() @@ -144,7 +143,6 @@ func (m *Manager) replicate(ctx context.Context, sb *types.Sandbox) error { var prev string live, _ := m.commitIfLive(ctx, sb, func() { prev, sb.ReplicaCk, sb.ReplicaPending, sb.ArchiveEpoch = sb.ReplicaCk, ckID, "", epoch - sb.SyncReplicated() }) if !live { m.abandonArchive(ctx, sb, epoch, ckID) @@ -188,7 +186,6 @@ func (m *Manager) settleReplicaIntent(ctx context.Context, sb *types.Sandbox) er var prev string live, _ := m.commitIfLive(ctx, sb, func() { prev, sb.ReplicaCk, sb.ReplicaPending, sb.ArchiveEpoch = sb.ReplicaCk, pending, "", rec.Epoch - sb.SyncReplicated() }) if !live { return ErrUnknownSandbox @@ -225,7 +222,6 @@ func (m *Manager) clearReplicaIntent(ctx context.Context, sb *types.Sandbox, ck _, _ = m.commitIfLive(ctx, sb, func() { if sb.ReplicaPending == ck { sb.ReplicaPending = "" - sb.SyncReplicated() } }) } @@ -253,7 +249,6 @@ func (m *Manager) dropReplica(ctx context.Context, sb *types.Sandbox) error { var ck string live, _ := m.commitIfLive(ctx, sb, func() { ck, sb.ReplicaCk = sb.ReplicaCk, "" - sb.SyncReplicated() }) if !live { return ErrUnknownSandbox @@ -267,6 +262,12 @@ func (m *Manager) replicaRowed(sb *types.Sandbox) bool { return m.archives != nil && sb.ReplicaCk != "" && sb.ArchiveEpoch != 0 } +func (m *Manager) replicaHeld(sb *types.Sandbox) bool { + m.mu.Lock() + defer m.mu.Unlock() + return sb.HasReplica() +} + func (m *Manager) replicaOf(sb *types.Sandbox) string { m.mu.Lock() defer m.mu.Unlock() diff --git a/sandboxd/pool/replica_test.go b/sandboxd/pool/replica_test.go index 02736d22..0c489b95 100644 --- a/sandboxd/pool/replica_test.go +++ b/sandboxd/pool/replica_test.go @@ -100,9 +100,9 @@ func TestReplicaLostPutReplyKeepsTheCheckpointTheRowNames(t *testing.T) { a.mu.Lock() pending := sb.ReplicaPending a.mu.Unlock() - if !ok || pending == "" || row.Checkpoint != pending || !ckExists(t, a, pending) || !sb.Replicated() { + if !ok || pending == "" || row.Checkpoint != pending || !ckExists(t, a, pending) || !sb.HasReplica() { t.Fatalf("row %+v present %v, pending %q kept %v replicated %v, want the row's checkpoint kept and the claim fenced", - row, ok, pending, ckExists(t, a, pending), sb.Replicated()) + row, ok, pending, ckExists(t, a, pending), sb.HasReplica()) } c.expire("a") @@ -233,7 +233,7 @@ func TestReplicaRetireAfterAnArchiveLeavesTheArchiveRowAlone(t *testing.T) { func TestReplicaClaimThatGainsVolumesStopsBeingReplicated(t *testing.T) { c := newReplicaCluster(t) a := c.node(t, "a", t.TempDir(), keepAnHour) - a.fenced.Store(false) + a.liftFence() sb := mustClaim(t, a, testKey) replicateNow(t, a, sb) ck := a.replicaOf(sb) @@ -243,8 +243,8 @@ func TestReplicaClaimThatGainsVolumesStopsBeingReplicated(t *testing.T) { a.replicateOnce(t.Context()) waitFor(t, func() bool { return !a.replicating.Load() }) - if _, ok, _ := c.reg.Get(t.Context(), sb.ID); ok || a.replicaOf(sb) != "" || ckExists(t, a, ck) || sb.Replicated() { - t.Errorf("row present %v, replica %q, ck kept %v, replicated %v, want the stale row and replica retired", ok, a.replicaOf(sb), ckExists(t, a, ck), sb.Replicated()) + if _, ok, _ := c.reg.Get(t.Context(), sb.ID); ok || a.replicaOf(sb) != "" || ckExists(t, a, ck) || sb.HasReplica() { + t.Errorf("row present %v, replica %q, ck kept %v, has replica %v, want the stale row and replica retired", ok, a.replicaOf(sb), ckExists(t, a, ck), sb.HasReplica()) } if _, held := a.byID(sb.ID); !held { t.Error("retiring the replica dropped the claim") @@ -254,11 +254,16 @@ func TestReplicaClaimThatGainsVolumesStopsBeingReplicated(t *testing.T) { func TestReplicaFollowsThePolicyPoolOfAPromotedTemplate(t *testing.T) { c := newReplicaCluster(t) a := c.node(t, "a", t.TempDir(), keepAnHour) - a.fenced.Store(false) + a.liftFence() sb := mustClaim(t, a, testKey) a.mu.Lock() + a.dropClaimLocked(sb) sb.Key, sb.PolicySource = types.PoolKey{Template: "promoted:v1", Net: types.NetNone, Size: testKey.Size}, testKey + a.addClaimLocked(sb) a.mu.Unlock() + if !sb.Replicated() { + t.Error("a claim of a template promoted out of a replicating pool is not marked replicated") + } a.replicateOnce(t.Context()) waitFor(t, func() bool { return !a.replicating.Load() }) @@ -396,7 +401,7 @@ func TestReplicaPassSnapshotsOnlyDueRunningClaims(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := newReplicaCluster(t) a := c.node(t, "a", t.TempDir(), keepAnHour) - a.fenced.Store(false) + a.liftFence() running := mustClaim(t, a, testKey) paused := mustClaim(t, a, testKey) withVolume := mustClaim(t, a, testKey) @@ -440,6 +445,7 @@ func TestReplicaPassSkipsWhileFenced(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := newReplicaCluster(t) a := c.node(t, "a", t.TempDir(), keepAnHour) + a.tripFence() sb := mustClaim(t, a, testKey) a.replicateOnce(t.Context()) time.Sleep(replicateEvery * time.Second) @@ -454,7 +460,7 @@ func TestReplicaPassSkipsWhileFenced(t *testing.T) { func TestReplicaPassRetiresAReplicaWhenThePoolTurnsItOff(t *testing.T) { c := newReplicaCluster(t) a := c.node(t, "a", t.TempDir(), keepAnHour) - a.fenced.Store(false) + a.liftFence() sb := mustClaim(t, a, testKey) replicateNow(t, a, sb) ck := a.replicaOf(sb) diff --git a/sandboxd/server/relay.go b/sandboxd/server/relay.go index 351e7507..5fe0a530 100644 --- a/sandboxd/server/relay.go +++ b/sandboxd/server/relay.go @@ -86,6 +86,9 @@ func (s *Server) wakeGuest(ctx context.Context, w http.ResponseWriter, id, token writeErr(w, http.StatusBadGateway, "guest agent unreachable") return nil, nil, false } + if fenced == nil { + return guest, done, true + } stop := pool.CloseOnFence(fenced, guest) return guest, func() { stop(); done() }, true } diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index c748a80d..a90771c7 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -601,8 +601,8 @@ type Sandbox struct { // lastActivity is unix-nanos of the last data-plane connection, stamped lock-free. lastActivity atomic.Int64 - // replicated mirrors ReplicaCk or ReplicaPending being set, so the data plane reads it without the manager mutex. - replicated atomic.Bool + // replicated marks a claim of a replicating pool; it is set before the claim is published and read without a lock. + replicated bool // open counts the data-plane connections held right now. open atomic.Int32 // live is the env as of the last SetEnv, read by the egress proxy without the manager mutex. @@ -621,11 +621,18 @@ func (s *Sandbox) TouchAt(t time.Time) { s.lastActivity.Store(t.UnixNano()) } // LastSeen returns the last data-plane activity time. func (s *Sandbox) LastSeen() time.Time { return time.Unix(0, s.lastActivity.Load()) } -// SyncReplicated republishes the replica state to Replicated; callers hold the manager mutex. -func (s *Sandbox) SyncReplicated() { s.replicated.Store(s.ReplicaCk != "" || s.ReplicaPending != "") } +// SetReplicated marks the claim as one of a replicating pool; it writes only a change, so a re-add of the same claim never races a reader. +func (s *Sandbox) SetReplicated(v bool) { + if s.replicated != v { + s.replicated = v + } +} + +// Replicated reports whether the claim's pool replicates it. +func (s *Sandbox) Replicated() bool { return s.replicated } -// Replicated reports whether the claim's registry row may name a replica. -func (s *Sandbox) Replicated() bool { return s.replicated.Load() } +// HasReplica reports whether the claim's registry row may name a replica. +func (s *Sandbox) HasReplica() bool { return s.ReplicaCk != "" || s.ReplicaPending != "" } // Hold registers an open data-plane connection; the idle sweep leaves a held sandbox alone. func (s *Sandbox) Hold() { s.open.Add(1) } From 0c680e29a81c011ff07009e56ad3544296c1297a Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 04:13:12 +0800 Subject: [PATCH 15/36] pool: lift the fence by compare-and-set and re-check the window after it A fence trip that fired between the window check and Store(false) found the node already fenced and did nothing, and the lift then ran past the lease with no timer left to trip it. The lift now replaces the fenced state this renewal saw, only if it is still current, and trips again when the window closed before it landed; the earlier window check goes, since the lift carries it. --- sandboxd/pool/lease.go | 15 ++++++++++++--- sandboxd/pool/lease_test.go | 31 +++++++++++++++++++++++++++++++ sandboxd/pool/replica_test.go | 3 +++ 3 files changed, 46 insertions(+), 3 deletions(-) diff --git a/sandboxd/pool/lease.go b/sandboxd/pool/lease.go index a9b2d2da..33ce9e3f 100644 --- a/sandboxd/pool/lease.go +++ b/sandboxd/pool/lease.go @@ -94,6 +94,7 @@ func (m *Manager) trip(ctx context.Context) { // unfence confirms each replicated claim against its row, then lifts the fence if that finished inside the window the renewal at start still covers. func (m *Manager) unfence(ctx context.Context, start time.Time) { logger := log.WithFunc("pool.unfence") + fenced := m.fence.Load() m.mu.Lock() held := map[string]*types.Sandbox{} for id, sb := range m.claimed { @@ -131,12 +132,20 @@ func (m *Manager) unfence(ctx context.Context, start time.Time) { for _, td := range teardowns { go td() } + m.lift(ctx, fenced, start) +} + +// lift replaces the fenced state this renewal saw, and trips again when the window closed before the lift landed. +func (m *Manager) lift(ctx context.Context, fenced *fenceState, start time.Time) { + if !m.fence.CompareAndSwap(fenced, &fenceState{ch: make(chan struct{})}) { + return + } if time.Since(start) > m.lease.ttl-m.lease.margin { - logger.Warn(ctx, "replicated claims stay fenced: confirming ownership outlasted the lease window") + log.WithFunc("pool.unfence").Warn(ctx, "replicated claims stay fenced: confirming ownership outlasted the lease window") + m.trip(ctx) return } - m.fence.Store(&fenceState{ch: make(chan struct{})}) - logger.Info(ctx, "node lease renewed; replicated claims are served again") + log.WithFunc("pool.unfence").Info(ctx, "node lease renewed; replicated claims are served again") } // confirmOwnership takes sb's row for this node in one compare-and-set; false means another node holds the claim. diff --git a/sandboxd/pool/lease_test.go b/sandboxd/pool/lease_test.go index 531ebf30..b9307c7a 100644 --- a/sandboxd/pool/lease_test.go +++ b/sandboxd/pool/lease_test.go @@ -274,6 +274,37 @@ func TestLeaseFencedNodeDoesNotAnswerAsOwnerOfReplicatedClaims(t *testing.T) { } } +func TestLeaseLiftAfterTheWindowClosedFencesAgain(t *testing.T) { + c := newReplicaCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + a.tripFence() + late := time.Now().Add(-(leaseTTL - leaseMargin + time.Second)) + + a.lift(t.Context(), a.fence.Load(), late) + if !a.isFenced() { + t.Error("a lift that landed after the lease window left the node serving") + } + + a.tripFence() + a.lift(t.Context(), a.fence.Load(), time.Now()) + if a.isFenced() { + t.Error("a lift inside the window left the node fenced") + } +} + +func TestLeaseLiftOfAReplacedStateChangesNothing(t *testing.T) { + c := newReplicaCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + a.tripFence() + seen := a.fence.Load() + a.tripFence() + + a.lift(t.Context(), seen, time.Now()) + if !a.isFenced() { + t.Error("a lift of a state that was replaced unfenced the node") + } +} + func TestLeaseConfirmRetriesOnceOnAStaleEpoch(t *testing.T) { c := newReplicaCluster(t) a := c.node(t, "a", t.TempDir(), keepAnHour) diff --git a/sandboxd/pool/replica_test.go b/sandboxd/pool/replica_test.go index 0c489b95..4914c469 100644 --- a/sandboxd/pool/replica_test.go +++ b/sandboxd/pool/replica_test.go @@ -305,6 +305,9 @@ func TestReplicaInFlightRoundCannotRecreateARowTheAdopterFinished(t *testing.T) if _, held := b.byID(sb.ID); !held { t.Error("the adopter lost the claim it woke") } + if metas, err := a.ckpts.Metas(t.Context()); err != nil || len(metas) != 0 { + t.Errorf("store holds %d checkpoints %v, want the lost round's export deleted", len(metas), err) + } } func TestReplicaRoundWithAStaleEpochOfItsOwnRowKeepsTheClaim(t *testing.T) { From 408d73e032f18555694d1a9ca8d0566f4112549a Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 04:13:52 +0800 Subject: [PATCH 16/36] pool: test that a leftover replica_ck with failover off still serves streams The fence state is initialised in every manager and a claim is replicated by pool policy, which needs archive_failover, so a journal that still holds replica_ck after failover was turned off no longer reaches a nil fence. --- sandboxd/pool/replica_test.go | 38 +++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/sandboxd/pool/replica_test.go b/sandboxd/pool/replica_test.go index 4914c469..799a06f5 100644 --- a/sandboxd/pool/replica_test.go +++ b/sandboxd/pool/replica_test.go @@ -3,11 +3,13 @@ package pool import ( "encoding/json/v2" "errors" + "net" "testing" "testing/synctest" "time" "github.com/cocoonstack/sandbox/sandboxd/archives" + "github.com/cocoonstack/sandbox/sandboxd/config" "github.com/cocoonstack/sandbox/sandboxd/store" "github.com/cocoonstack/sandbox/sandboxd/tenants" "github.com/cocoonstack/sandbox/sandboxd/types" @@ -356,6 +358,42 @@ func TestReplicaArchiveCannotRecreateARowTheAdopterFinished(t *testing.T) { } } +func TestReplicaLeftoverJournalWithFailoverOffStillServesStreams(t *testing.T) { + eng := newFakeEngine() + dir := t.TempDir() + m := newTestManagerAt(t, eng, dir, config.PoolSpec{PoolKey: testKey, Warm: 1}) + sb := mustClaim(t, m, testKey) + m.mu.Lock() + sb.ReplicaCk = "ck_leftover" + js := m.store.set(sb) + m.mu.Unlock() + if err := m.store.commit(js); err != nil { + t.Fatalf("commit: %v", err) + } + + restarted := newTestManagerAt(t, eng, dir, config.PoolSpec{PoolKey: testKey, Warm: 1}) + if err := restarted.Reconcile(t.Context()); err != nil { + t.Fatalf("reconcile: %v", err) + } + held, ok := restarted.byID(sb.ID) + if !ok || held.ReplicaCk != "ck_leftover" || held.Replicated() { + t.Fatalf("claim held %v replica %q replicated %v, want the leftover kept and not replicated", ok, held.ReplicaCk, held.Replicated()) + } + _, fenced, done, err := restarted.WakeAgentSocket(t.Context(), sb.ID, sb.Token) + if err != nil || fenced != nil { + t.Fatalf("stream to a claim with a leftover replica and failover off: %v, fence %v, want served with no fence", err, fenced) + } + done() + guest, host := net.Pipe() + t.Cleanup(func() { _ = host.Close() }) + eng.guestDial = func() (net.Conn, error) { return guest, nil } + conn, err := restarted.DialPort(t.Context(), sb.ID, Cred{Token: sb.Token}, 80) + if err != nil { + t.Fatalf("port dial: %v", err) + } + _ = conn.Close() +} + func TestReplicaFailedCaptureKeepsTheOldReplicaAndRow(t *testing.T) { c := newReplicaCluster(t) a := c.node(t, "a", t.TempDir(), keepAnHour) From 2750959f5c0d4c6b9a4ec92066dcbf867efa7fc6 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 04:14:55 +0800 Subject: [PATCH 17/36] pool: the sweep, the pin set and orphan reclaim know a round's pending checkpoint An hourly sweep that reached reconcileOwnRow while a round's Put outcome was unsettled saw a row on a checkpoint the claim did not name and deleted both; the settle then found the row absent with a held epoch, took the claim as moved and destroyed a running claim no other node held. The pending checkpoint now counts as referenced by the sweep, is pinned, and is skipped by orphan reclaim at restart. --- sandboxd/pool/checkpoint.go | 3 ++ sandboxd/pool/failover.go | 2 +- sandboxd/pool/reconcile.go | 2 +- sandboxd/pool/replica_test.go | 61 +++++++++++++++++++++++++++++++++++ 4 files changed, 66 insertions(+), 2 deletions(-) diff --git a/sandboxd/pool/checkpoint.go b/sandboxd/pool/checkpoint.go index a53f3cd6..cc220d25 100644 --- a/sandboxd/pool/checkpoint.go +++ b/sandboxd/pool/checkpoint.go @@ -323,6 +323,9 @@ func (m *Manager) pinnedArchiveCks() map[string]struct{} { if sb.ReplicaCk != "" { pinned[sb.ReplicaCk] = struct{}{} } + if sb.ReplicaPending != "" { + pinned[sb.ReplicaPending] = struct{}{} + } } return pinned } diff --git a/sandboxd/pool/failover.go b/sandboxd/pool/failover.go index c91f1589..1fd0917c 100644 --- a/sandboxd/pool/failover.go +++ b/sandboxd/pool/failover.go @@ -442,7 +442,7 @@ func (m *Manager) reconcileOwnRow(ctx context.Context, rec archives.Record, now sb.Transition.Lock() defer sb.Transition.Unlock() m.mu.Lock() - held, refs, epoch := m.claimed[rec.ID] == sb, sb.ArchiveCk == rec.Checkpoint || sb.ReplicaCk == rec.Checkpoint, sb.ArchiveEpoch + held, refs, epoch := m.claimed[rec.ID] == sb, sb.ArchiveCk == rec.Checkpoint || sb.ReplicaCk == rec.Checkpoint || sb.ReplicaPending == rec.Checkpoint, sb.ArchiveEpoch m.mu.Unlock() switch { case !held: diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index 11cbc259..afbc1485 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -293,7 +293,7 @@ func (m *Manager) reclaimOrphanArchiveCks(ctx context.Context, claims map[string continue } orig, mine := claims[ckpt.SandboxID] - if !mine || orig.ArchiveCk == ckpt.ID || orig.ReplicaCk == ckpt.ID || orig.ArchivePending == ckpt.ID || rows[ckpt.SandboxID].Checkpoint == ckpt.ID { + if !mine || orig.ArchiveCk == ckpt.ID || orig.ReplicaCk == ckpt.ID || orig.ReplicaPending == ckpt.ID || orig.ArchivePending == ckpt.ID || rows[ckpt.SandboxID].Checkpoint == ckpt.ID { continue } if err := m.deleteArchiveCk(ctx, ckpt.ID); err != nil { diff --git a/sandboxd/pool/replica_test.go b/sandboxd/pool/replica_test.go index 799a06f5..b08e4da8 100644 --- a/sandboxd/pool/replica_test.go +++ b/sandboxd/pool/replica_test.go @@ -783,6 +783,67 @@ func TestReplicaSweepRearmsATombstoneItsClaimStillNames(t *testing.T) { } } +func TestReplicaSweepSparesTheRowAndCheckpointOfAnUnsettledRound(t *testing.T) { + c := newReplicaCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + a.archives = &hookRegistry{Registry: c.reg, losePutReply: true} + if err := a.replicate(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { + t.Fatalf("round whose put reply was lost: %v", err) + } + a.mu.Lock() + pending := sb.ReplicaPending + a.mu.Unlock() + rec, ok, _ := c.reg.Get(t.Context(), sb.ID) + if !ok || rec.Checkpoint != pending { + t.Fatalf("row %+v present %v, want it on the pending checkpoint %s", rec, ok, pending) + } + + a.reconcileOwnRow(t.Context(), rec, time.Now()) + if _, held := a.byID(sb.ID); !held { + t.Fatal("the sweep dropped the claim") + } + if again, ok, _ := c.reg.Get(t.Context(), sb.ID); !ok || again.Epoch != rec.Epoch || !ckExists(t, a, pending) { + t.Errorf("row %+v present %v, checkpoint kept %v, want both intact until the intent settles", again, ok, ckExists(t, a, pending)) + } + if _, pinned := a.pinnedArchiveCks()[pending]; !pinned { + t.Error("the unsettled round's checkpoint is not pinned") + } + sb.Transition.Lock() + err := a.resolveIntent(t.Context(), sb) + sb.Transition.Unlock() + if err != nil || a.replicaOf(sb) != pending { + t.Errorf("settle: %v, replica %q, want the row's checkpoint %s adopted", err, a.replicaOf(sb), pending) + } +} + +func TestReplicaRestartKeepsTheCheckpointOfARoundItCannotSettleYet(t *testing.T) { + c := newReplicaCluster(t) + dirA := t.TempDir() + a := c.node(t, "a", dirA, keepAnHour) + sb := mustClaim(t, a, testKey) + a.archives = &hookRegistry{Registry: c.reg, failPut: true} + if err := a.replicate(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { + t.Fatalf("round whose put failed: %v", err) + } + a.mu.Lock() + pending := sb.ReplicaPending + a.mu.Unlock() + if err := a.FlushClaims(); err != nil { + t.Fatalf("flush: %v", err) + } + + restarted := c.restart(t, a) + restarted.ckpts = &unreadableStore{Store: restarted.ckpts} + if err := restarted.Reconcile(t.Context()); err != nil { + t.Fatalf("reconcile: %v", err) + } + held, ok := restarted.byID(sb.ID) + if !ok || held.ReplicaPending != pending || !ckExists(t, restarted, pending) { + t.Errorf("claim held %v, intent %q, ck kept %v, want the intent and its checkpoint kept for the next settle", ok, held.ReplicaPending, ckExists(t, restarted, pending)) + } +} + func TestReplicaALeftoverDeleteMarkerSparesTheLiveReplica(t *testing.T) { c := newReplicaCluster(t) a := c.node(t, "a", t.TempDir(), keepAnHour) From d7423aaa254649031e074aec8a9f41c9b162072a Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 04:15:44 +0800 Subject: [PATCH 18/36] pool: a confirmation that keeps losing on this node's own row stays fenced After two refusals on a row whose owner was still this node, confirmOwnership fell through to false and unfence destroyed a claim it owns. It now returns an error, so the node stays fenced and the next renewal retries; only an absent row or another node's row drops the claim. --- sandboxd/pool/lease.go | 8 ++++++-- sandboxd/pool/lease_test.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/sandboxd/pool/lease.go b/sandboxd/pool/lease.go index 33ce9e3f..8a74fd08 100644 --- a/sandboxd/pool/lease.go +++ b/sandboxd/pool/lease.go @@ -3,6 +3,7 @@ package pool import ( "context" "errors" + "fmt" "io" "maps" "slices" @@ -150,7 +151,7 @@ func (m *Manager) lift(ctx context.Context, fenced *fenceState, start time.Time) // confirmOwnership takes sb's row for this node in one compare-and-set; false means another node holds the claim. func (m *Manager) confirmOwnership(ctx context.Context, sb *types.Sandbox, rec archives.Record) (bool, error) { - for range 2 { + for attempt := range 3 { switch { case rec.ID == "": gone, err := m.replicaGone(ctx, sb) @@ -160,6 +161,9 @@ func (m *Manager) confirmOwnership(ctx context.Context, sb *types.Sandbox, rec a case rec.Consumed: return true, nil } + if attempt == 2 { + break + } epoch, err := m.archives.Take(ctx, sb.ID, rec.Epoch, m.nodeID) if err == nil { _, _ = m.commitIfLive(ctx, sb, func() { sb.ArchiveEpoch = max(sb.ArchiveEpoch, epoch) }) @@ -176,7 +180,7 @@ func (m *Manager) confirmOwnership(ctx context.Context, sb *types.Sandbox, rec a rec = archives.Record{} } } - return false, nil + return false, fmt.Errorf("confirm %s: its row kept moving under this node", sb.ID) } // fenceWatch gates sb at resolution and returns the channel that closes when its streams must end; nil for an unreplicated claim. diff --git a/sandboxd/pool/lease_test.go b/sandboxd/pool/lease_test.go index b9307c7a..c51c0255 100644 --- a/sandboxd/pool/lease_test.go +++ b/sandboxd/pool/lease_test.go @@ -305,6 +305,28 @@ func TestLeaseLiftOfAReplacedStateChangesNothing(t *testing.T) { } } +func TestLeaseConfirmThatKeepsLosingOnItsOwnRowStaysFencedAndKeepsTheClaim(t *testing.T) { + c := newReplicaCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + vm := lockedVMName(sb) + replicateNow(t, a, sb) + a.archives = &lostTakeRegistry{Registry: c.reg} + row, _, _ := c.reg.Get(t.Context(), sb.ID) + + if keep, err := a.confirmOwnership(t.Context(), sb, row); err == nil || keep { + t.Fatalf("confirm on a row that keeps refusing: %v %v, want an error", keep, err) + } + a.tripFence() + a.unfence(t.Context(), time.Now()) + if !a.isFenced() { + t.Error("the node lifted its fence without confirming a claim it owns") + } + if _, held := a.byID(sb.ID); !held || a.eng.(*fakeEngine).removed(vm) { + t.Errorf("claim held %v, VM removed %v, want the claim and VM kept: its row is still this node's", held, a.eng.(*fakeEngine).removed(vm)) + } +} + func TestLeaseConfirmRetriesOnceOnAStaleEpoch(t *testing.T) { c := newReplicaCluster(t) a := c.node(t, "a", t.TempDir(), keepAnHour) @@ -381,6 +403,14 @@ func TestLeaseAdoptionWaitsOutTheFenceAndTheOldOwnerDropsItsCopy(t *testing.T) { } } +type lostTakeRegistry struct { + archives.Registry +} + +func (r *lostTakeRegistry) Take(context.Context, string, int64, string) (int64, error) { + return 0, archives.ErrLost +} + type cutRegistry struct { archives.Registry cut atomic.Bool From 81fb40e86181ba50ebfdb575ac26981fd7f3b926 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 04:16:36 +0800 Subject: [PATCH 19/36] review: tag forgetAdopted by its own name, declare moved with var, and align the docs forgetAdopted logged under pool.dropAdopted and a var pair used a tuple declaration. The docs now state which registry writes are compare-and-sets and which stay owner-guarded upserts, the loss bound as the time since the last successful round, the 404 of /owner on a fenced node, the fence as decided by pool policy with its real hot-path cost, replicate_seconds as restart-only, and that an open relay of a replicating pool's claim closes whether or not it has a replica yet. --- docs/cluster.md | 60 +++++++++++++++++++++++++-------------- docs/deploy.md | 4 +-- docs/sandboxd-api.md | 3 +- sandboxd/pool/failover.go | 5 ++-- 4 files changed, 46 insertions(+), 26 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index 087df9e6..08ce48e4 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -187,8 +187,15 @@ replication rounds write it, so the claim path is unchanged. itself, and a node that tries to take its row is refused with a 404. A draining node never adopts. Adoption checks no tenant `max_claims` or node `max_claims`: the claim already exists, and a refusal could strand it. -- **Fencing.** Every write is a compare-and-set on the row's epoch, and every - epoch comes from one database sequence, so none is ever reused. A wake takes +- **Fencing.** Take, consume and delete are compare-and-sets on the row's + epoch, and so is the write of a replica round and of the archive of a claim + that holds a replica (`Replace`, which also requires this node as owner and a + live row): an absent row at a held epoch refuses it, so a round that outlived + an adoption cannot recreate the row. The writes that remain owner-guarded + upserts are the first write of a row, when this node holds no epoch (the first + replica, the archive of a claim with no row), and the re-arm of this node's + own tombstone. Every epoch comes from one database sequence, so none is ever + reused. A wake takes the row before it restores and consumes it before it commits: the row turns into a tombstone, deleted only once the checkpoint is gone. A release or a retention purge consumes the row before the checkpoint goes; with failover @@ -271,8 +278,10 @@ checkpoint store and keeps each claim's registry row on the latest snapshot: settle it by reading the row: a row on that checkpoint promotes it, another owner's row drops the claim, and an absent row means the adopter finished when the checkpoint is gone and a write that never landed when it is still - there. A claim with such an intent counts as replicated for the fence. A claim gets its first replica one interval after - it is claimed (or after a restart). A hibernated, archived or failed claim, + there. A refused write reads the row: one this node still owns only had a + stale epoch and is refreshed, and an absent or foreign row drops the claim and + its VM. A claim gets its first replica one interval after it is claimed (or + after a restart). A hibernated, archived or failed claim, one whose tenant was removed, and one with volumes (capture refuses them; logged once per claim) is skipped; a replicated claim that gains volumes drops its row and replica. The claim's policy pool, not its own template key, @@ -287,7 +296,8 @@ checkpoint store and keeps each claim's registry row on the latest snapshot: through the archive wake, which restores the guest with its memory and processes. The claim resumes from its last snapshot: everything after it, and the connections open at the time, are lost, so the data loss is at most the - interval. + time since the last successful round: one interval while rounds succeed, more + while they fail and the older replica stands. - **Lifecycle.** A release consumes the row, then deletes the replica and the tombstone. A hibernate leaves the row on the last replica. An archive puts its own checkpoint over the row, with its retention deadline, and deletes the @@ -308,26 +318,32 @@ checkpoint store and keeps each claim's registry row on the latest snapshot: - **Self-fencing.** A node whose renewal is overdue by 12 seconds (the lease length minus a 3 second margin, counted from the start of the last renewal that answered) stops serving every replicated claim it holds: relay, exec, - ports, preview and `/owner` answer `503 lease lost; retry` (`/owner` answers - 404, so a `Lookup` moves on to the other nodes). The guest has no NIC on the - `none` lane, so it is unreachable except through the node, and the node - destroys nothing. Claims that are not replicated stay served. A node starts - fenced and lifts the fence after its first renewal that is followed by a - confirmation. The confirmation runs beside the renew loop, which keeps + ports and preview answer `503 lease lost; retry`, and `/owner` answers 404 so + a `Lookup` moves on to the other nodes. A claim is replicated when its policy + pool has `replicate_seconds`, from the moment it is claimed, adopted or + loaded, whether or not a round has run, so the fence covers it before its + first replica; the setting is restart-only so that decision cannot go stale. + The guest has no NIC on the `none` lane, so it is unreachable except through + the node, and the node destroys nothing. Claims of pools that do not + replicate stay served. A node starts fenced and lifts the fence after its + first renewal that is followed by a confirmation. The confirmation runs beside the renew loop, which keeps renewing: for each replicated claim the node takes the row back for itself in one compare-and-set, which the row lock serializes against adopters, and retries once on a stale epoch. A claim whose row another node took, or took - and finished, is forgotten at once and its VM removed in the background. The - fence lifts only if the whole confirmation finished within 12 seconds of the - start of the renewal that began it; a slower one leaves the node fenced for - the next renewal to repeat. The margin makes the fence close 3 seconds before + and finished, is forgotten at once and its VM removed in the background; a + claim whose row is still this node's but keeps refusing the write leaves the + node fenced and is retried at the next renewal. The lift replaces the fenced + state this renewal saw, only if it is still current, and trips again at once + if more than 12 seconds passed since the start of that renewal; a slower + confirmation therefore leaves the node fenced for the next renewal to repeat. The margin makes the fence close 3 seconds before another node may adopt, so no moment has two serving copies, partitions included. This holds while the node's clock and the database's run at the same rate to within that margin over 12 seconds, and ignores skew between them, because both sides count from a renewal's own start. The guest of a claim the fence closed keeps running without a NIC until the node returns and removes it, and its state is discarded. When the fence closes, every open - stream of a replicated claim on the node is closed too: the agent relay + stream of a replicated claim on the node is closed too, one opened before the + claim's first replica included: the agent relay (including parked keep-alive relays, whose later RPCs never pass the resolution gate), the exec call, port relays and preview requests. A client retries and meets the 503 until the lease is back or the claim moved. @@ -335,14 +351,16 @@ checkpoint store and keeps each claim's registry row on the latest snapshot: - **Availability.** The design chooses consistency over availability for replicated claims. A registry outage longer than the fence threshold (12 seconds) stops service of replicated claims on every node until PostgreSQL - returns, because no node can prove it still owns them. Claims that are not - replicated, and archived claims, are unaffected. + returns, because no node can prove it still owns them. Claims of pools that + do not replicate are unaffected. An archived claim of a replicating pool is + gated like a running one while the node is fenced, so its wake waits too. A replica costs, per claim per interval, one capture pause, one upload, one registry read, one write and one delete; the node's own cost is one scan of its -claims every 5 seconds. Resolving a claim for a stream gains one atomic load (two for a -replicated claim); a replicated stream also holds one watcher goroutine. -Nothing runs per frame. +claims every 5 seconds. Resolving a claim for a stream of a pool that does not +replicate reads one plain field and no atomic; a replicated claim adds one +atomic load of the fence state and its stream one watcher goroutine. Adding a +claim to a failover node gains one map lookup. Nothing runs per frame. ## Templates on a cluster diff --git a/docs/deploy.md b/docs/deploy.md index 8026ef7c..7e4c3044 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -143,7 +143,7 @@ gets the intersection of the two. | `capture_trim` (pool entry) | `false` | Trim the guest's copy-on-write disk before a promote or checkpoint of this pool's sandboxes (and of the clones of a template promoted from it), so the record carries only live data: blocks a build deleted — a `COPY` step's archive, a `RUN` step's caches — are no longer captured. sandboxd mounts the disk init resolved from `cocoon.cow` a second time at `/run/sandboxd-trim` through silkd as root, runs `fstrim` and unmounts, under the capture's transition lock (about 15 ms on a warm clone). A failed trim is logged and the capture proceeds, larger. A hibernated or archived sandbox is captured untrimmed. Config-owned like `egress`: `PUT /v1/pools` rejects it | | `egress_classes` | unset | named tenant egress layers: `[{"name": "desk", "egress": {…}, "egress_upstream_env": "DESK_UPSTREAM"}]`. A tenant names one with `egress_class` through the [tenant API](sandboxd-api.md#tenants-v1tenants) and its claims take that policy intersected with the pool's. `egress` is required; `intercept` stays a pool-rule flag. Reloadable: a class's policy reaches live claims on their next request. A class removed while tenants still name it leaves them with no egress and is logged at load and at reload | | `egress_upstream_env` (pool or egress class entry) | unset | names a node environment variable holding the default upstream proxy URL for the pool's or class's claims (class before pool, a claim's own entry before both); needs `egress_upstream`, and the URL must be on its `allow`. Read and checked at startup. Config-owned like `egress`: `PUT /v1/pools` rejects it | -| `replicate_seconds` (pool entry) | 0 (off) | snapshot every running claim of this pool to the checkpoint store this often and point the claim's registry row at the snapshot, so another node restores the claim, memory and processes included, when this node is lost (see [running claims across node loss](cluster.md#running-claims-across-node-loss)). Each round pauses the guest for the length of a memory and disk capture, so the interval is also the data a loss costs. The claim's row carries no retention deadline while it runs. Requires `archive_failover` and a `none`-lane pool, both checked at config load; a claim with volumes is skipped. Config-owned like `egress`: `PUT /v1/pools` rejects it, and a reload applies it (a pool set back to 0 drops its claims' rows and replicas) | +| `replicate_seconds` (pool entry) | 0 (off) | snapshot every running claim of this pool to the checkpoint store this often and point the claim's registry row at the snapshot, so another node restores the claim, memory and processes included, when this node is lost (see [running claims across node loss](cluster.md#running-claims-across-node-loss)). Each round pauses the guest for the length of a memory and disk capture, so the interval is also the data a loss costs. The claim's row carries no retention deadline while it runs. Requires `archive_failover` and a `none`-lane pool, both checked at config load; a claim with volumes is skipped. Config-owned like `egress`: `PUT /v1/pools` rejects it. Restart only: a claim is replicated by its pool's setting from the moment it is claimed, so a reload that changes it is refused (a pool restarted with 0 drops its claims' rows and replicas) | | `storage` (pool entry) | unset (cocoon's default, 10G) | size of the copy-on-write disk the pool's golden is cold-booted with, in cocoon's own `--storage` spelling (`"40G"`, `"40GiB"`, `"40Gi"`: Docker/Kubernetes units, all binary) and passed to it verbatim; every clone, fork, checkpoint branch and promoted template of the pool inherits it. At least cocoon's default of 10G. Equal sizes spelled differently build the same golden; a changed size rebuilds the golden on the next build, and claims already out keep their disk. Config-owned like `egress`: `PUT /v1/pools` rejects it | | `max_claims` | 0 (unlimited) | node-wide cap on live claims; claim/fork/branch requests beyond it answer 429 with the pool state unharmed (on a cluster a non-volume claim tries a warm-peer redirect first; a volume claim answers the 429 with no redirect) | | `audit_log` | false | append every relayed frame that opens an RPC — its op + addressing fields, never payloads — to `/audit.jsonl`; continuation `data`, `data_end`, `stdin`, and `stdin_close` frames pass through unrecorded. The file is size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`), plus `method` (`GET`, `CONNECT`, `SOCKS5`, …), `decision` (`allow`, `deny`, `rate_limited`), `secret` (the ref name, never its value), `upstream` (the [upstream proxy](egress.md#upstream-proxies)'s `host:port`, when one carried it), `tenant`, and on an allowed connection `src_addr` and `dst_addr` (the node-side and destination `ip:port` of the upstream TCP connection, the tunnel target through an upstream proxy) on `egress` records, which a relayed frame cannot set; preview accesses record as op `preview`, one per request; a tenant change records as op `tenants` with the `added`, `changed` and `removed` names, never a token. An opening frame whose first line exceeds 4 KiB records as op `oversized` with no addressing fields | @@ -711,7 +711,7 @@ validates it with the same code as boot, and applies the reloadable settings all at once or not at all. Every field falls into one of three classes: - **Reloaded**: - - per pool: `egress`, `warmup`, `capture_trim`, `storage`, `replicate_seconds`, `egress_upstream_env`, for existing keys and for keys new in the file; + - per pool: `egress`, `warmup`, `capture_trim`, `storage`, `egress_upstream_env`, for existing keys and for keys new in the file; - `egress_classes`: each class's `egress` and `egress_upstream_env`; - node-wide: `egress_internal_allow`, `secrets`, `egress_upstream`, `egress_usage_bytes`, `egress_conn_rate` and `disk_reserve`. - **Left to their API**: a pool's targets (`warm`, `warm_max`, idle and archive diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 1334ca80..35a1bdaa 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -961,7 +961,8 @@ them). 426 without the upgrade header, 404 unknown sandbox, 409 a failed sandbox, 502 guest unreachable, 503 `lease lost; retry` for a replicated claim while its node's lease is overdue (see [running claims](cluster.md#running-claims-across-node-loss)). A relay of a -replicated claim already open when the lease turns overdue is closed. +claim of a replicating pool that is already open when the lease turns overdue +is closed, whether or not it has a replica yet. ## GET /v1/sandboxes/{id}/ports/{port} diff --git a/sandboxd/pool/failover.go b/sandboxd/pool/failover.go index 1fd0917c..1bad43b4 100644 --- a/sandboxd/pool/failover.go +++ b/sandboxd/pool/failover.go @@ -371,7 +371,7 @@ func (m *Manager) dropAdopted(ctx context.Context, sb *types.Sandbox) { // forgetAdopted forgets the claim at once and returns the removal of its VM; nil when the claim was already gone. func (m *Manager) forgetAdopted(ctx context.Context, sb *types.Sandbox) func() { - logger := log.WithFunc("pool.dropAdopted") + logger := log.WithFunc("pool.forgetAdopted") m.mu.Lock() ck, vm, snap := sb.ArchiveCk, sb.VMName, sb.HibernateSnap m.mu.Unlock() @@ -511,7 +511,8 @@ func (m *Manager) fenceArchived(ctx context.Context, claims map[string]*types.Sa case ok && rec.Owner != m.nodeID: drop = true case sb.HasReplica() && sb.ArchivePending == "": - moved, err := false, error(nil) + var moved bool + var err error if !ok { moved, err = m.replicaGone(ctx, sb) } From 1264a46f5231642c438b326f3e0f67fa65fd7137 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 04:22:43 +0800 Subject: [PATCH 20/36] review: move the archive row write out of archive to keep it flat --- sandboxd/pool/archive.go | 12 +----------- sandboxd/pool/failover.go | 12 ++++++++++++ 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/sandboxd/pool/archive.go b/sandboxd/pool/archive.go index 2706d719..d0d03278 100644 --- a/sandboxd/pool/archive.go +++ b/sandboxd/pool/archive.go @@ -12,7 +12,6 @@ import ( "github.com/projecteru2/core/log" - "github.com/cocoonstack/sandbox/sandboxd/archives" "github.com/cocoonstack/sandbox/sandboxd/store" "github.com/cocoonstack/sandbox/sandboxd/types" ) @@ -132,16 +131,7 @@ func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error { m.deleteOrphanArchiveCk(ctx, ck.ID) return fmt.Errorf("archive %s: persist intent: %w", sb.ID, err) } - replicated := m.replicaOf(sb) != "" - if replicated { - epoch, err = m.replaceArchive(ctx, sb, ck.ID) - } else { - epoch, err = m.putArchive(ctx, sb, ck.ID) - } - if err != nil { - if replicated && errors.Is(err, archives.ErrLost) { - _ = m.refusedReplace(ctx, sb) - } + if epoch, err = m.writeArchiveRow(ctx, sb, ck.ID); err != nil { sb.Transition.Unlock() m.abandonRefusedPut(ctx, sb, ck.ID, err) return fmt.Errorf("archive %s: registry: %w", sb.ID, err) diff --git a/sandboxd/pool/failover.go b/sandboxd/pool/failover.go index 1bad43b4..57974d8c 100644 --- a/sandboxd/pool/failover.go +++ b/sandboxd/pool/failover.go @@ -98,6 +98,18 @@ func (m *Manager) putArchive(ctx context.Context, sb *types.Sandbox, ck string) return m.putRow(ctx, sb, ck, false, false) } +// writeArchiveRow upserts the archive row, or replaces the replica row the claim holds; the caller holds sb.Transition. +func (m *Manager) writeArchiveRow(ctx context.Context, sb *types.Sandbox, ck string) (int64, error) { + if m.replicaOf(sb) == "" { + return m.putArchive(ctx, sb, ck) + } + epoch, err := m.replaceArchive(ctx, sb, ck) + if errors.Is(err, archives.ErrLost) { + _ = m.refusedReplace(ctx, sb) + } + return epoch, err +} + // replaceArchive writes an archive row over the replica row this node holds, so a row that left meanwhile is not recreated. func (m *Manager) replaceArchive(ctx context.Context, sb *types.Sandbox, ck string) (int64, error) { return m.putRow(ctx, sb, ck, false, true) From f80bdf798ddc3f1d97017c29a6dbb225b187deba Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 04:31:00 +0800 Subject: [PATCH 21/36] server: log a client cancel of the silkd dial below error A client that cancels while its request wakes a sandbox made wakeGuest log the failed dial at error level although the wake completes. A dial that fails with context.Canceled while the request context is canceled is the client leaving and logs at debug; every other dial failure stays an error. --- sandboxd/server/relay.go | 7 +++++- sandboxd/server/relay_test.go | 40 +++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/sandboxd/server/relay.go b/sandboxd/server/relay.go index 5fe0a530..865feabf 100644 --- a/sandboxd/server/relay.go +++ b/sandboxd/server/relay.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "context" + "errors" "io" "net" "net/http" @@ -82,7 +83,11 @@ func (s *Server) wakeGuest(ctx context.Context, w http.ResponseWriter, id, token guest, err = s.dialer.DialSilkd(ctx, sock) if err != nil { done() - log.WithFunc("server.wakeGuest").Errorf(ctx, err, "dial silkd for %s", id) + if errors.Is(err, context.Canceled) && errors.Is(ctx.Err(), context.Canceled) { + log.WithFunc("server.wakeGuest").Debugf(ctx, "client canceled the dial of silkd for %s: %v", id, err) + } else { + log.WithFunc("server.wakeGuest").Errorf(ctx, err, "dial silkd for %s", id) + } writeErr(w, http.StatusBadGateway, "guest agent unreachable") return nil, nil, false } diff --git a/sandboxd/server/relay_test.go b/sandboxd/server/relay_test.go index d6e5cf87..0c0b5aef 100644 --- a/sandboxd/server/relay_test.go +++ b/sandboxd/server/relay_test.go @@ -16,6 +16,9 @@ import ( "testing/iotest" "time" + "github.com/projecteru2/core/log" + "github.com/rs/zerolog" + "github.com/cocoonstack/sandbox/protocol/wire" "github.com/cocoonstack/sandbox/sandboxd/pool" ) @@ -169,6 +172,43 @@ func TestRelayEndsWhenItsClaimIsFenced(t *testing.T) { } } +func TestWakeGuestLogsAClientCancelBelowError(t *testing.T) { + logs := &bytes.Buffer{} + global := log.GetGlobalLogger() + saved := *global + *global = zerolog.New(logs).Level(zerolog.InfoLevel) + t.Cleanup(func() { *global = saved }) + dialer := &fakeDialer{dial: func(ctx context.Context, _ string) (net.Conn, error) { return nil, ctx.Err() }} + srv := New("", "node:7777", &fakeManager{}, dialer, nil, nil, nil, nil) + + canceled, cancel := context.WithCancel(t.Context()) + cancel() + w := httptest.NewRecorder() + if _, _, ok := srv.wakeGuest(canceled, w, "sb_1", "tok"); ok || w.Code != http.StatusBadGateway { + t.Fatalf("canceled dial: ok %v status %d, want a refused 502", ok, w.Code) + } + if logs.Len() != 0 { + t.Errorf("a client cancel logged %q, want nothing at info level", logs.String()) + } + + dialer.dial = func(context.Context, string) (net.Conn, error) { return nil, errors.New("connection refused") } + if _, _, ok := srv.wakeGuest(t.Context(), httptest.NewRecorder(), "sb_1", "tok"); ok { + t.Fatal("a failing dial succeeded") + } + if !strings.Contains(logs.String(), `"level":"error"`) { + t.Errorf("a real dial failure logged %q, want an error record", logs.String()) + } + + logs.Reset() + dialer.dial = func(context.Context, string) (net.Conn, error) { return nil, context.Canceled } + if _, _, ok := srv.wakeGuest(t.Context(), httptest.NewRecorder(), "sb_1", "tok"); ok { + t.Fatal("a failing dial succeeded") + } + if !strings.Contains(logs.String(), `"level":"error"`) { + t.Errorf("a cancel that did not come from the client logged %q, want an error record", logs.String()) + } +} + func TestAuditTeeRecordsRequestLines(t *testing.T) { big := line(wire.Exec{Argv: []string{strings.Repeat("x", pool.AuditLineCap)}}) chunk := line(&wire.Data{Data: make([]byte, 3*pool.AuditLineCap)}) From 8a89834e891525b5347938140d373388aaa63cab Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 04:31:21 +0800 Subject: [PATCH 22/36] docs: replicate_seconds is turned off by a restart, and what a replica left at 0 costs A reload that changes replicate_seconds is refused, so the failover docs no longer say a reload turns it off. A claim that holds a replica when the node restarts with replication off stays replicated for the fence until the next restart: during a registry outage it answers 503, an availability cost only. --- docs/cluster.md | 9 ++++++--- docs/deploy.md | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index 08ce48e4..b159ef39 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -304,9 +304,12 @@ checkpoint store and keeps each claim's registry row on the latest snapshot: replica. A lease expiry with `on_expire` release drops the row and the replica first, and the next reap destroys the claim; with `on_expire` archive, the archive puts its checkpoint - over the row. Turning `replicate_seconds` - off, by reload or restart, drops the rows and replicas of the pool's claims on - the next pass. A restart keeps the node's own live row for a running claim + over the row. Turning `replicate_seconds` off takes a restart, since a reload + that changes it is refused; the restarted node drops the rows and replicas of + the pool's claims on its first pass. A claim that holds a replica when the + node restarts with replication off is still treated as replicated by the fence + until the next restart, so during a registry outage it answers 503 like a + replicated claim; that costs availability only, never a second copy or data. A restart keeps the node's own live row for a running claim and takes the checkpoint a crashed round left on it; a claim whose row another node holds, or whose row a finished adopter consumed, is dropped with its VM. A replica row has no retention deadline: the hourly sweep never purges it, diff --git a/docs/deploy.md b/docs/deploy.md index 7e4c3044..8272d4ae 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -143,7 +143,7 @@ gets the intersection of the two. | `capture_trim` (pool entry) | `false` | Trim the guest's copy-on-write disk before a promote or checkpoint of this pool's sandboxes (and of the clones of a template promoted from it), so the record carries only live data: blocks a build deleted — a `COPY` step's archive, a `RUN` step's caches — are no longer captured. sandboxd mounts the disk init resolved from `cocoon.cow` a second time at `/run/sandboxd-trim` through silkd as root, runs `fstrim` and unmounts, under the capture's transition lock (about 15 ms on a warm clone). A failed trim is logged and the capture proceeds, larger. A hibernated or archived sandbox is captured untrimmed. Config-owned like `egress`: `PUT /v1/pools` rejects it | | `egress_classes` | unset | named tenant egress layers: `[{"name": "desk", "egress": {…}, "egress_upstream_env": "DESK_UPSTREAM"}]`. A tenant names one with `egress_class` through the [tenant API](sandboxd-api.md#tenants-v1tenants) and its claims take that policy intersected with the pool's. `egress` is required; `intercept` stays a pool-rule flag. Reloadable: a class's policy reaches live claims on their next request. A class removed while tenants still name it leaves them with no egress and is logged at load and at reload | | `egress_upstream_env` (pool or egress class entry) | unset | names a node environment variable holding the default upstream proxy URL for the pool's or class's claims (class before pool, a claim's own entry before both); needs `egress_upstream`, and the URL must be on its `allow`. Read and checked at startup. Config-owned like `egress`: `PUT /v1/pools` rejects it | -| `replicate_seconds` (pool entry) | 0 (off) | snapshot every running claim of this pool to the checkpoint store this often and point the claim's registry row at the snapshot, so another node restores the claim, memory and processes included, when this node is lost (see [running claims across node loss](cluster.md#running-claims-across-node-loss)). Each round pauses the guest for the length of a memory and disk capture, so the interval is also the data a loss costs. The claim's row carries no retention deadline while it runs. Requires `archive_failover` and a `none`-lane pool, both checked at config load; a claim with volumes is skipped. Config-owned like `egress`: `PUT /v1/pools` rejects it. Restart only: a claim is replicated by its pool's setting from the moment it is claimed, so a reload that changes it is refused (a pool restarted with 0 drops its claims' rows and replicas) | +| `replicate_seconds` (pool entry) | 0 (off) | snapshot every running claim of this pool to the checkpoint store this often and point the claim's registry row at the snapshot, so another node restores the claim, memory and processes included, when this node is lost (see [running claims across node loss](cluster.md#running-claims-across-node-loss)). Each round pauses the guest for the length of a memory and disk capture, so the interval is also the data a loss costs. The claim's row carries no retention deadline while it runs. Requires `archive_failover` and a `none`-lane pool, both checked at config load; a claim with volumes is skipped. Config-owned like `egress`: `PUT /v1/pools` rejects it. Restart only: a claim is replicated by its pool's setting from the moment it is claimed, so a reload that changes it is refused (a pool restarted with 0 drops its claims' rows and replicas on the first pass) | | `storage` (pool entry) | unset (cocoon's default, 10G) | size of the copy-on-write disk the pool's golden is cold-booted with, in cocoon's own `--storage` spelling (`"40G"`, `"40GiB"`, `"40Gi"`: Docker/Kubernetes units, all binary) and passed to it verbatim; every clone, fork, checkpoint branch and promoted template of the pool inherits it. At least cocoon's default of 10G. Equal sizes spelled differently build the same golden; a changed size rebuilds the golden on the next build, and claims already out keep their disk. Config-owned like `egress`: `PUT /v1/pools` rejects it | | `max_claims` | 0 (unlimited) | node-wide cap on live claims; claim/fork/branch requests beyond it answer 429 with the pool state unharmed (on a cluster a non-volume claim tries a warm-peer redirect first; a volume claim answers the 429 with no redirect) | | `audit_log` | false | append every relayed frame that opens an RPC — its op + addressing fields, never payloads — to `/audit.jsonl`; continuation `data`, `data_end`, `stdin`, and `stdin_close` frames pass through unrecorded. The file is size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`), plus `method` (`GET`, `CONNECT`, `SOCKS5`, …), `decision` (`allow`, `deny`, `rate_limited`), `secret` (the ref name, never its value), `upstream` (the [upstream proxy](egress.md#upstream-proxies)'s `host:port`, when one carried it), `tenant`, and on an allowed connection `src_addr` and `dst_addr` (the node-side and destination `ip:port` of the upstream TCP connection, the tunnel target through an upstream proxy) on `egress` records, which a relayed frame cannot set; preview accesses record as op `preview`, one per request; a tenant change records as op `tenants` with the `added`, `changed` and `removed` names, never a token. An opening frame whose first line exceeds 4 KiB records as op `oversized` with no addressing fields | From 91b37c3b841ad79cfed61b6c1d02ab1a8a724e47 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 04:31:24 +0800 Subject: [PATCH 23/36] docs: the loss bound is everything after the last committed replica At a 30 s interval hardware lost 33.1 s: a round starts up to an interval plus the 5 s tick after the previous one and commits only when its capture, upload and registry write finish. State that bound instead of the interval alone. --- docs/cluster.md | 11 +++++++---- docs/deploy.md | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index b159ef39..04f6c33c 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -294,10 +294,13 @@ checkpoint store and keeps each claim's registry row on the latest snapshot: records the claim as archived on the replica with the adopter pool's retention window, and the next call wakes it through the archive wake, which restores the guest with its memory and - processes. The claim resumes from its last snapshot: everything after it, and - the connections open at the time, are lost, so the data loss is at most the - time since the last successful round: one interval while rounds succeed, more - while they fail and the older replica stands. + processes. The claim resumes from its last committed replica: everything + after it, and the connections open at the time, are lost. A round starts up + to one interval plus the 5-second tick after the previous one and commits only + when its capture, upload and registry write finish, so what a loss can cost is + up to about the interval, plus the tick, plus the duration of one round, which + grows with guest memory; while rounds fail it is the time since the last one + that committed. - **Lifecycle.** A release consumes the row, then deletes the replica and the tombstone. A hibernate leaves the row on the last replica. An archive puts its own checkpoint over the row, with its retention deadline, and deletes the diff --git a/docs/deploy.md b/docs/deploy.md index 8272d4ae..9d880c54 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -143,7 +143,7 @@ gets the intersection of the two. | `capture_trim` (pool entry) | `false` | Trim the guest's copy-on-write disk before a promote or checkpoint of this pool's sandboxes (and of the clones of a template promoted from it), so the record carries only live data: blocks a build deleted — a `COPY` step's archive, a `RUN` step's caches — are no longer captured. sandboxd mounts the disk init resolved from `cocoon.cow` a second time at `/run/sandboxd-trim` through silkd as root, runs `fstrim` and unmounts, under the capture's transition lock (about 15 ms on a warm clone). A failed trim is logged and the capture proceeds, larger. A hibernated or archived sandbox is captured untrimmed. Config-owned like `egress`: `PUT /v1/pools` rejects it | | `egress_classes` | unset | named tenant egress layers: `[{"name": "desk", "egress": {…}, "egress_upstream_env": "DESK_UPSTREAM"}]`. A tenant names one with `egress_class` through the [tenant API](sandboxd-api.md#tenants-v1tenants) and its claims take that policy intersected with the pool's. `egress` is required; `intercept` stays a pool-rule flag. Reloadable: a class's policy reaches live claims on their next request. A class removed while tenants still name it leaves them with no egress and is logged at load and at reload | | `egress_upstream_env` (pool or egress class entry) | unset | names a node environment variable holding the default upstream proxy URL for the pool's or class's claims (class before pool, a claim's own entry before both); needs `egress_upstream`, and the URL must be on its `allow`. Read and checked at startup. Config-owned like `egress`: `PUT /v1/pools` rejects it | -| `replicate_seconds` (pool entry) | 0 (off) | snapshot every running claim of this pool to the checkpoint store this often and point the claim's registry row at the snapshot, so another node restores the claim, memory and processes included, when this node is lost (see [running claims across node loss](cluster.md#running-claims-across-node-loss)). Each round pauses the guest for the length of a memory and disk capture, so the interval is also the data a loss costs. The claim's row carries no retention deadline while it runs. Requires `archive_failover` and a `none`-lane pool, both checked at config load; a claim with volumes is skipped. Config-owned like `egress`: `PUT /v1/pools` rejects it. Restart only: a claim is replicated by its pool's setting from the moment it is claimed, so a reload that changes it is refused (a pool restarted with 0 drops its claims' rows and replicas on the first pass) | +| `replicate_seconds` (pool entry) | 0 (off) | snapshot every running claim of this pool to the checkpoint store this often and point the claim's registry row at the snapshot, so another node restores the claim, memory and processes included, when this node is lost (see [running claims across node loss](cluster.md#running-claims-across-node-loss)). Each round pauses the guest for the length of a memory and disk capture. A restore resumes from the last committed replica and loses everything after it: up to about the interval, plus the 5-second tick, plus the duration of one round (capture, upload and registry write, which grows with guest memory). The claim's row carries no retention deadline while it runs. Requires `archive_failover` and a `none`-lane pool, both checked at config load; a claim with volumes is skipped. Config-owned like `egress`: `PUT /v1/pools` rejects it. Restart only: a claim is replicated by its pool's setting from the moment it is claimed, so a reload that changes it is refused (a pool restarted with 0 drops its claims' rows and replicas on the first pass) | | `storage` (pool entry) | unset (cocoon's default, 10G) | size of the copy-on-write disk the pool's golden is cold-booted with, in cocoon's own `--storage` spelling (`"40G"`, `"40GiB"`, `"40Gi"`: Docker/Kubernetes units, all binary) and passed to it verbatim; every clone, fork, checkpoint branch and promoted template of the pool inherits it. At least cocoon's default of 10G. Equal sizes spelled differently build the same golden; a changed size rebuilds the golden on the next build, and claims already out keep their disk. Config-owned like `egress`: `PUT /v1/pools` rejects it | | `max_claims` | 0 (unlimited) | node-wide cap on live claims; claim/fork/branch requests beyond it answer 429 with the pool state unharmed (on a cluster a non-volume claim tries a warm-peer redirect first; a volume claim answers the 429 with no redirect) | | `audit_log` | false | append every relayed frame that opens an RPC — its op + addressing fields, never payloads — to `/audit.jsonl`; continuation `data`, `data_end`, `stdin`, and `stdin_close` frames pass through unrecorded. The file is size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`), plus `method` (`GET`, `CONNECT`, `SOCKS5`, …), `decision` (`allow`, `deny`, `rate_limited`), `secret` (the ref name, never its value), `upstream` (the [upstream proxy](egress.md#upstream-proxies)'s `host:port`, when one carried it), `tenant`, and on an allowed connection `src_addr` and `dst_addr` (the node-side and destination `ip:port` of the upstream TCP connection, the tunnel target through an upstream proxy) on `egress` records, which a relayed frame cannot set; preview accesses record as op `preview`, one per request; a tenant change records as op `tenants` with the `added`, `changed` and `removed` names, never a token. An opening frame whose first line exceeds 4 KiB records as op `oversized` with no addressing fields | From 053d25b59a38e225a9afcf625d235c675471aa71 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 04:34:16 +0800 Subject: [PATCH 24/36] docs: rewrap the replication lifecycle paragraph --- docs/cluster.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index 04f6c33c..d669981d 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -304,16 +304,16 @@ checkpoint store and keeps each claim's registry row on the latest snapshot: - **Lifecycle.** A release consumes the row, then deletes the replica and the tombstone. A hibernate leaves the row on the last replica. An archive puts its own checkpoint over the row, with its retention deadline, and deletes the - replica. A lease expiry - with `on_expire` release drops the row and the replica first, and the next reap + replica. A lease expiry with `on_expire` release drops the row and the replica + first, and the next reap destroys the claim; with `on_expire` archive, the archive puts its checkpoint over the row. Turning `replicate_seconds` off takes a restart, since a reload that changes it is refused; the restarted node drops the rows and replicas of the pool's claims on its first pass. A claim that holds a replica when the node restarts with replication off is still treated as replicated by the fence until the next restart, so during a registry outage it answers 503 like a - replicated claim; that costs availability only, never a second copy or data. A restart keeps the node's own live row for a running claim - and takes the checkpoint a crashed round left on it; a claim whose row another + replicated claim; that costs availability only, never a second copy or data. + A restart keeps the node's own live row for a running claim and takes the checkpoint a crashed round left on it; a claim whose row another node holds, or whose row a finished adopter consumed, is dropped with its VM. A replica row has no retention deadline: the hourly sweep never purges it, because no round refreshes it while the claim is hibernated, and retention From 8181a2089e4e5892130eed0478d5bd144183132b Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 04:34:30 +0800 Subject: [PATCH 25/36] docs: rewrap the replication lifecycle bullet --- docs/cluster.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index d669981d..2ae915ce 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -305,22 +305,22 @@ checkpoint store and keeps each claim's registry row on the latest snapshot: tombstone. A hibernate leaves the row on the last replica. An archive puts its own checkpoint over the row, with its retention deadline, and deletes the replica. A lease expiry with `on_expire` release drops the row and the replica - first, and the next reap - destroys the claim; with `on_expire` archive, the archive puts its checkpoint - over the row. Turning `replicate_seconds` off takes a restart, since a reload - that changes it is refused; the restarted node drops the rows and replicas of - the pool's claims on its first pass. A claim that holds a replica when the - node restarts with replication off is still treated as replicated by the fence - until the next restart, so during a registry outage it answers 503 like a - replicated claim; that costs availability only, never a second copy or data. - A restart keeps the node's own live row for a running claim and takes the checkpoint a crashed round left on it; a claim whose row another - node holds, or whose row a finished adopter consumed, is dropped with its VM. - A replica row has no retention deadline: the hourly sweep never purges it, - because no round refreshes it while the claim is hibernated, and retention - starts when the claim is archived or adopted. The rows of a node that never - returns therefore stay until a client adopts them, a bounded residue of one - row and one checkpoint per claim. Apart from that and the lease rule the - sweep is unchanged. + first, and the next reap destroys the claim; with `on_expire` archive, the + archive puts its checkpoint over the row. Turning `replicate_seconds` off + takes a restart, since a reload that changes it is refused; the restarted node + drops the rows and replicas of the pool's claims on its first pass. A claim + that holds a replica when the node restarts with replication off is still + treated as replicated by the fence until the next restart, so during a + registry outage it answers 503 like a replicated claim; that costs + availability only, never a second copy or data. A restart keeps the node's own + live row for a running claim and takes the checkpoint a crashed round left on + it; a claim whose row another node holds, or whose row a finished adopter + consumed, is dropped with its VM. A replica row has no retention deadline: the + hourly sweep never purges it, because no round refreshes it while the claim is + hibernated, and retention starts when the claim is archived or adopted. The + rows of a node that never returns therefore stay until a client adopts them, a + bounded residue of one row and one checkpoint per claim. Apart from that and + the lease rule the sweep is unchanged. - **Self-fencing.** A node whose renewal is overdue by 12 seconds (the lease length minus a 3 second margin, counted from the start of the last renewal that answered) stops serving every replicated claim it holds: relay, exec, From 00e27b77b830675f816555110c27c4c7ca7afb20 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 04:36:55 +0800 Subject: [PATCH 26/36] pool: a woken claim holds no archive epoch wakeArchived consumes the row at the epoch it took, but commitWake left that epoch on the claim. The first replica round of the woken running claim then wrote through Replace at the consumed epoch, found no live row, and the refused-write settle destroyed the claim: every adopted-and-woken claim, and every archive-then-wake, in a replicating pool. commitWake now clears the epoch with the checkpoint, as consumeArchive does, and its rollback restores both. With no epoch held the round upserts, which also revives this node's own tombstone when its delete failed. --- sandboxd/pool/archive.go | 6 ++--- sandboxd/pool/failover_test.go | 3 +++ sandboxd/pool/replica_test.go | 46 ++++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 3 deletions(-) diff --git a/sandboxd/pool/archive.go b/sandboxd/pool/archive.go index d0d03278..03062c69 100644 --- a/sandboxd/pool/archive.go +++ b/sandboxd/pool/archive.go @@ -258,8 +258,8 @@ func (m *Manager) commitWake(ctx context.Context, sb *types.Sandbox, vmName, soc m.mu.Unlock() return false, false } - ck, deadline := sb.ArchiveCk, sb.Deadline - sb.VMName, sb.VsockSocket, sb.ArchiveCk = vmName, sock, "" + ck, deadline, epoch := sb.ArchiveCk, sb.Deadline, sb.ArchiveEpoch + sb.VMName, sb.VsockSocket, sb.ArchiveCk, sb.ArchiveEpoch = vmName, sock, "", 0 sb.Deadline = time.Now().Add(clampTTL(types.Seconds(sb.LeaseSeconds))) // a woken sandbox is a fresh lease js := m.store.set(sb) m.mu.Unlock() @@ -267,7 +267,7 @@ func (m *Manager) commitWake(ctx context.Context, sb *types.Sandbox, vmName, soc m.mu.Lock() var rb claimSnapshot if m.claimed[sb.ID] == sb { - sb.VMName, sb.VsockSocket, sb.ArchiveCk, sb.Deadline = "", "", ck, deadline + sb.VMName, sb.VsockSocket, sb.ArchiveCk, sb.ArchiveEpoch, sb.Deadline = "", "", ck, epoch, deadline rb = m.store.set(sb) } m.mu.Unlock() diff --git a/sandboxd/pool/failover_test.go b/sandboxd/pool/failover_test.go index 71ea25e9..69c9f6fe 100644 --- a/sandboxd/pool/failover_test.go +++ b/sandboxd/pool/failover_test.go @@ -494,6 +494,9 @@ func TestFailoverWakeRollbackRecoversOnTheNextWake(t *testing.T) { if rec, ok, _ := c.reg.Get(t.Context(), sb.ID); !ok || rec.Owner != "a" || rec.Checkpoint != ck || ck == "" { t.Fatalf("row %+v present %v, want this node's row for the rolled-back archive", rec, ok) } + if a.epochOf(sb) == 0 { + t.Error("the rolled-back wake dropped the claim's archive epoch") + } healStore(t, a) if _, _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake after the store heals: %v", err) diff --git a/sandboxd/pool/replica_test.go b/sandboxd/pool/replica_test.go index b08e4da8..0de3be9e 100644 --- a/sandboxd/pool/replica_test.go +++ b/sandboxd/pool/replica_test.go @@ -394,6 +394,52 @@ func TestReplicaLeftoverJournalWithFailoverOffStillServesStreams(t *testing.T) { _ = conn.Close() } +func TestReplicaRoundAfterAWakeKeepsTheClaimAndWritesAFreshRow(t *testing.T) { + for _, tt := range []struct { + name string + adopt bool + leaveTombstone bool + }{ + {"owner wakes its archive", false, false}, + {"adopter wakes the adopted archive", true, false}, + {"owner wakes and its tombstone delete fails", false, true}, + } { + t.Run(tt.name, func(t *testing.T) { + c := newReplicaCluster(t) + a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + mustArchive(t, a, sb) + waker, holder := a, sb + if tt.adopt { + c.expire("a") + if adopted, err := b.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { + t.Fatalf("adopt: %v %v", adopted, err) + } + waker = b + holder, _ = b.byID(sb.ID) + } + if tt.leaveTombstone { + waker.ckpts = &failingDeleteStore{Store: waker.ckpts, attempts: make(chan string, 8)} + } + if _, _, _, err := waker.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { + t.Fatalf("wake: %v", err) + } + if epoch := waker.epochOf(holder); epoch != 0 { + t.Errorf("woken claim holds epoch %d of the consumed row, want 0", epoch) + } + + replicateNow(t, waker, holder) + if _, held := waker.byID(sb.ID); !held { + t.Fatal("the round after the wake dropped the running claim") + } + rec, ok, _ := c.reg.Get(t.Context(), sb.ID) + if !ok || rec.Consumed || rec.Owner != waker.nodeID || rec.Checkpoint != waker.replicaOf(holder) { + t.Errorf("row %+v present %v, want a live row of %s on the new replica %q", rec, ok, waker.nodeID, waker.replicaOf(holder)) + } + }) + } +} + func TestReplicaFailedCaptureKeepsTheOldReplicaAndRow(t *testing.T) { c := newReplicaCluster(t) a := c.node(t, "a", t.TempDir(), keepAnHour) From 0d9f90ec40e78c88727aee524dd8fe2d4b96802d Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 04:39:58 +0800 Subject: [PATCH 27/36] docs: the loss bound includes a pass that outlasts the interval A claim is due when its last round started an interval ago, and a new pass cannot start while the previous one runs, so the gap between two round starts is the longer of the interval and one pass. State the bound with that term. --- docs/cluster.md | 13 +++++++------ docs/deploy.md | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index 2ae915ce..635843ce 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -295,12 +295,13 @@ checkpoint store and keeps each claim's registry row on the latest snapshot: retention window, and the next call wakes it through the archive wake, which restores the guest with its memory and processes. The claim resumes from its last committed replica: everything - after it, and the connections open at the time, are lost. A round starts up - to one interval plus the 5-second tick after the previous one and commits only - when its capture, upload and registry write finish, so what a loss can cost is - up to about the interval, plus the tick, plus the duration of one round, which - grows with guest memory; while rounds fail it is the time since the last one - that committed. + after it, and the connections open at the time, are lost: up to the longer of + the interval and one replication pass, plus the 5-second tick, plus one + round's capture, upload and registry write. A pass covers all the node's due + claims of the replicating pools with bounded concurrency and a new pass cannot + start before the previous one ends, so it grows with claim count and guest + memory; for example, an interval of 30 s with a 90 s pass can lose about 185 s. + While rounds fail it is the time since the last one that committed. - **Lifecycle.** A release consumes the row, then deletes the replica and the tombstone. A hibernate leaves the row on the last replica. An archive puts its own checkpoint over the row, with its retention deadline, and deletes the diff --git a/docs/deploy.md b/docs/deploy.md index 9d880c54..f5a18cd3 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -143,7 +143,7 @@ gets the intersection of the two. | `capture_trim` (pool entry) | `false` | Trim the guest's copy-on-write disk before a promote or checkpoint of this pool's sandboxes (and of the clones of a template promoted from it), so the record carries only live data: blocks a build deleted — a `COPY` step's archive, a `RUN` step's caches — are no longer captured. sandboxd mounts the disk init resolved from `cocoon.cow` a second time at `/run/sandboxd-trim` through silkd as root, runs `fstrim` and unmounts, under the capture's transition lock (about 15 ms on a warm clone). A failed trim is logged and the capture proceeds, larger. A hibernated or archived sandbox is captured untrimmed. Config-owned like `egress`: `PUT /v1/pools` rejects it | | `egress_classes` | unset | named tenant egress layers: `[{"name": "desk", "egress": {…}, "egress_upstream_env": "DESK_UPSTREAM"}]`. A tenant names one with `egress_class` through the [tenant API](sandboxd-api.md#tenants-v1tenants) and its claims take that policy intersected with the pool's. `egress` is required; `intercept` stays a pool-rule flag. Reloadable: a class's policy reaches live claims on their next request. A class removed while tenants still name it leaves them with no egress and is logged at load and at reload | | `egress_upstream_env` (pool or egress class entry) | unset | names a node environment variable holding the default upstream proxy URL for the pool's or class's claims (class before pool, a claim's own entry before both); needs `egress_upstream`, and the URL must be on its `allow`. Read and checked at startup. Config-owned like `egress`: `PUT /v1/pools` rejects it | -| `replicate_seconds` (pool entry) | 0 (off) | snapshot every running claim of this pool to the checkpoint store this often and point the claim's registry row at the snapshot, so another node restores the claim, memory and processes included, when this node is lost (see [running claims across node loss](cluster.md#running-claims-across-node-loss)). Each round pauses the guest for the length of a memory and disk capture. A restore resumes from the last committed replica and loses everything after it: up to about the interval, plus the 5-second tick, plus the duration of one round (capture, upload and registry write, which grows with guest memory). The claim's row carries no retention deadline while it runs. Requires `archive_failover` and a `none`-lane pool, both checked at config load; a claim with volumes is skipped. Config-owned like `egress`: `PUT /v1/pools` rejects it. Restart only: a claim is replicated by its pool's setting from the moment it is claimed, so a reload that changes it is refused (a pool restarted with 0 drops its claims' rows and replicas on the first pass) | +| `replicate_seconds` (pool entry) | 0 (off) | snapshot every running claim of this pool to the checkpoint store this often and point the claim's registry row at the snapshot, so another node restores the claim, memory and processes included, when this node is lost (see [running claims across node loss](cluster.md#running-claims-across-node-loss)). Each round pauses the guest for the length of a memory and disk capture. A restore resumes from the last committed replica and loses everything after it: up to the longer of the interval and one replication pass, plus the 5-second tick, plus one round's capture, upload and registry write. A pass covers all the node's due claims of the replicating pools with bounded concurrency, so it grows with claim count and guest memory. The claim's row carries no retention deadline while it runs. Requires `archive_failover` and a `none`-lane pool, both checked at config load; a claim with volumes is skipped. Config-owned like `egress`: `PUT /v1/pools` rejects it. Restart only: a claim is replicated by its pool's setting from the moment it is claimed, so a reload that changes it is refused (a pool restarted with 0 drops its claims' rows and replicas on the first pass) | | `storage` (pool entry) | unset (cocoon's default, 10G) | size of the copy-on-write disk the pool's golden is cold-booted with, in cocoon's own `--storage` spelling (`"40G"`, `"40GiB"`, `"40Gi"`: Docker/Kubernetes units, all binary) and passed to it verbatim; every clone, fork, checkpoint branch and promoted template of the pool inherits it. At least cocoon's default of 10G. Equal sizes spelled differently build the same golden; a changed size rebuilds the golden on the next build, and claims already out keep their disk. Config-owned like `egress`: `PUT /v1/pools` rejects it | | `max_claims` | 0 (unlimited) | node-wide cap on live claims; claim/fork/branch requests beyond it answer 429 with the pool state unharmed (on a cluster a non-volume claim tries a warm-peer redirect first; a volume claim answers the 429 with no redirect) | | `audit_log` | false | append every relayed frame that opens an RPC — its op + addressing fields, never payloads — to `/audit.jsonl`; continuation `data`, `data_end`, `stdin`, and `stdin_close` frames pass through unrecorded. The file is size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`), plus `method` (`GET`, `CONNECT`, `SOCKS5`, …), `decision` (`allow`, `deny`, `rate_limited`), `secret` (the ref name, never its value), `upstream` (the [upstream proxy](egress.md#upstream-proxies)'s `host:port`, when one carried it), `tenant`, and on an allowed connection `src_addr` and `dst_addr` (the node-side and destination `ip:port` of the upstream TCP connection, the tunnel target through an upstream proxy) on `egress` records, which a relayed frame cannot set; preview accesses record as op `preview`, one per request; a tenant change records as op `tenants` with the `added`, `changed` and `removed` names, never a token. An opening frame whose first line exceeds 4 KiB records as op `oversized` with no addressing fields | From c56b363b8faccdbc31416325cb7ff323b00cf800 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 04:42:49 +0800 Subject: [PATCH 28/36] docs: the loss bound counts a whole second pass --- docs/cluster.md | 12 +++++++----- docs/deploy.md | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index 635843ce..f175297e 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -296,11 +296,13 @@ checkpoint store and keeps each claim's registry row on the latest snapshot: through the archive wake, which restores the guest with its memory and processes. The claim resumes from its last committed replica: everything after it, and the connections open at the time, are lost: up to the longer of - the interval and one replication pass, plus the 5-second tick, plus one - round's capture, upload and registry write. A pass covers all the node's due - claims of the replicating pools with bounded concurrency and a new pass cannot - start before the previous one ends, so it grows with claim count and guest - memory; for example, an interval of 30 s with a 90 s pass can lose about 185 s. + the interval and one replication pass, plus the 5-second tick, plus one more + pass, since a claim can run first in one pass and last in the next. A pass + covers all the node's due claims of the replicating pools with bounded + concurrency, each round a capture, upload and registry write, and a new pass + cannot start before the previous one ends, so a pass grows with claim count + and guest memory; for example, an interval of 30 s with a 90 s pass can lose + about 185 s. While rounds fail it is the time since the last one that committed. - **Lifecycle.** A release consumes the row, then deletes the replica and the tombstone. A hibernate leaves the row on the last replica. An archive puts its diff --git a/docs/deploy.md b/docs/deploy.md index f5a18cd3..f1e1f5a9 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -143,7 +143,7 @@ gets the intersection of the two. | `capture_trim` (pool entry) | `false` | Trim the guest's copy-on-write disk before a promote or checkpoint of this pool's sandboxes (and of the clones of a template promoted from it), so the record carries only live data: blocks a build deleted — a `COPY` step's archive, a `RUN` step's caches — are no longer captured. sandboxd mounts the disk init resolved from `cocoon.cow` a second time at `/run/sandboxd-trim` through silkd as root, runs `fstrim` and unmounts, under the capture's transition lock (about 15 ms on a warm clone). A failed trim is logged and the capture proceeds, larger. A hibernated or archived sandbox is captured untrimmed. Config-owned like `egress`: `PUT /v1/pools` rejects it | | `egress_classes` | unset | named tenant egress layers: `[{"name": "desk", "egress": {…}, "egress_upstream_env": "DESK_UPSTREAM"}]`. A tenant names one with `egress_class` through the [tenant API](sandboxd-api.md#tenants-v1tenants) and its claims take that policy intersected with the pool's. `egress` is required; `intercept` stays a pool-rule flag. Reloadable: a class's policy reaches live claims on their next request. A class removed while tenants still name it leaves them with no egress and is logged at load and at reload | | `egress_upstream_env` (pool or egress class entry) | unset | names a node environment variable holding the default upstream proxy URL for the pool's or class's claims (class before pool, a claim's own entry before both); needs `egress_upstream`, and the URL must be on its `allow`. Read and checked at startup. Config-owned like `egress`: `PUT /v1/pools` rejects it | -| `replicate_seconds` (pool entry) | 0 (off) | snapshot every running claim of this pool to the checkpoint store this often and point the claim's registry row at the snapshot, so another node restores the claim, memory and processes included, when this node is lost (see [running claims across node loss](cluster.md#running-claims-across-node-loss)). Each round pauses the guest for the length of a memory and disk capture. A restore resumes from the last committed replica and loses everything after it: up to the longer of the interval and one replication pass, plus the 5-second tick, plus one round's capture, upload and registry write. A pass covers all the node's due claims of the replicating pools with bounded concurrency, so it grows with claim count and guest memory. The claim's row carries no retention deadline while it runs. Requires `archive_failover` and a `none`-lane pool, both checked at config load; a claim with volumes is skipped. Config-owned like `egress`: `PUT /v1/pools` rejects it. Restart only: a claim is replicated by its pool's setting from the moment it is claimed, so a reload that changes it is refused (a pool restarted with 0 drops its claims' rows and replicas on the first pass) | +| `replicate_seconds` (pool entry) | 0 (off) | snapshot every running claim of this pool to the checkpoint store this often and point the claim's registry row at the snapshot, so another node restores the claim, memory and processes included, when this node is lost (see [running claims across node loss](cluster.md#running-claims-across-node-loss)). Each round pauses the guest for the length of a memory and disk capture. A restore resumes from the last committed replica and loses everything after it: up to the longer of the interval and one replication pass, plus the 5-second tick, plus one more pass, since a claim can run first in one pass and last in the next. A pass covers all the node's due claims of the replicating pools with bounded concurrency, so it grows with claim count and guest memory. The claim's row carries no retention deadline while it runs. Requires `archive_failover` and a `none`-lane pool, both checked at config load; a claim with volumes is skipped. Config-owned like `egress`: `PUT /v1/pools` rejects it. Restart only: a claim is replicated by its pool's setting from the moment it is claimed, so a reload that changes it is refused (a pool restarted with 0 drops its claims' rows and replicas on the first pass) | | `storage` (pool entry) | unset (cocoon's default, 10G) | size of the copy-on-write disk the pool's golden is cold-booted with, in cocoon's own `--storage` spelling (`"40G"`, `"40GiB"`, `"40Gi"`: Docker/Kubernetes units, all binary) and passed to it verbatim; every clone, fork, checkpoint branch and promoted template of the pool inherits it. At least cocoon's default of 10G. Equal sizes spelled differently build the same golden; a changed size rebuilds the golden on the next build, and claims already out keep their disk. Config-owned like `egress`: `PUT /v1/pools` rejects it | | `max_claims` | 0 (unlimited) | node-wide cap on live claims; claim/fork/branch requests beyond it answer 429 with the pool state unharmed (on a cluster a non-volume claim tries a warm-peer redirect first; a volume claim answers the 429 with no redirect) | | `audit_log` | false | append every relayed frame that opens an RPC — its op + addressing fields, never payloads — to `/audit.jsonl`; continuation `data`, `data_end`, `stdin`, and `stdin_close` frames pass through unrecorded. The file is size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`), plus `method` (`GET`, `CONNECT`, `SOCKS5`, …), `decision` (`allow`, `deny`, `rate_limited`), `secret` (the ref name, never its value), `upstream` (the [upstream proxy](egress.md#upstream-proxies)'s `host:port`, when one carried it), `tenant`, and on an allowed connection `src_addr` and `dst_addr` (the node-side and destination `ip:port` of the upstream TCP connection, the tunnel target through an upstream proxy) on `egress` records, which a relayed frame cannot set; preview accesses record as op `preview`, one per request; a tenant change records as op `tenants` with the `added`, `changed` and `removed` names, never a token. An opening frame whose first line exceeds 4 KiB records as op `oversized` with no addressing fields | From a238d4bc9961811c84c899555a2e0794a0f819ce Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 14:32:07 +0800 Subject: [PATCH 29/36] pool: orphan archive reclaim re-reads the rows before it deletes Reconcile runs before the lease loop, so a node adopting during it can move a claim's row and commit its own replica after the start-of-reconcile read. The reclaim now reads the candidates' rows right before deleting and skips one whose row names the checkpoint or belongs to another node; a failed read skips the reclaim until the next restart. --- sandboxd/pool/failover.go | 11 +++++------ sandboxd/pool/failover_test.go | 35 ++++++++++++++++++++++++++++++++++ sandboxd/pool/reconcile.go | 24 +++++++++++++++++++---- 3 files changed, 60 insertions(+), 10 deletions(-) diff --git a/sandboxd/pool/failover.go b/sandboxd/pool/failover.go index 57974d8c..4bb0b450 100644 --- a/sandboxd/pool/failover.go +++ b/sandboxd/pool/failover.go @@ -494,16 +494,16 @@ func (m *Manager) purgeArchiveRow(ctx context.Context, rec archives.Record) { logger.Infof(ctx, "purged archive %s of %s", rec.ID, rec.Owner) } -// fenceArchived settles loaded claims against their rows and returns the rows; false when the registry did not answer. -func (m *Manager) fenceArchived(ctx context.Context, claims map[string]*types.Sandbox) (map[string]archives.Record, bool) { +// fenceArchived settles loaded claims against their rows; false when the registry did not answer. +func (m *Manager) fenceArchived(ctx context.Context, claims map[string]*types.Sandbox) bool { if m.archives == nil { - return nil, true + return true } logger := log.WithFunc("pool.fenceArchived") rows, err := m.archives.Rows(ctx, slices.Collect(maps.Keys(claims))) if err != nil { logger.Warnf(ctx, "archive fencing and orphan reclaim skipped: %v", err) - return nil, false + return false } for id, sb := range claims { rec, ok := rows[id] @@ -548,7 +548,6 @@ func (m *Manager) fenceArchived(ctx context.Context, claims map[string]*types.Sa drop = true default: sb.ArchivePending = "" - delete(rows, id) } } if drop { @@ -561,7 +560,7 @@ func (m *Manager) fenceArchived(ctx context.Context, claims map[string]*types.Sa logger.Infof(ctx, "dropped sandbox %s: another node holds it", id) } } - return rows, true + return true } func (m *Manager) settleIntent(ctx context.Context, id, pending string, rec archives.Record, ok bool) (bool, error) { diff --git a/sandboxd/pool/failover_test.go b/sandboxd/pool/failover_test.go index 69c9f6fe..32f45f29 100644 --- a/sandboxd/pool/failover_test.go +++ b/sandboxd/pool/failover_test.go @@ -289,6 +289,31 @@ func TestFailoverRestartReclaimsAnInterruptedArchive(t *testing.T) { } } +func TestFailoverRestartReclaimSparesACheckpointAdoptedMidReconcile(t *testing.T) { + c := newFailoverCluster(t) + dirA := t.TempDir() + claims := map[string]*types.Sandbox{ + "sb_x": {ID: "sb_x", VMName: "sbx-gone-1", Key: testKey, Token: "tok", HibernateSnap: "sbx-hib-x"}, + } + if err := newClaimStore(dirA, false).save(claims); err != nil { + t.Fatalf("setup: %v", err) + } + a := c.node(t, "a", dirA, keepAnHour) + const ck = "ck_00000000000000cc" + seedArchiveCk(t, a, ck, "sb_x") + a.archives = &hookRegistry{Registry: c.reg, afterRows: func() { + if _, err := c.reg.Put(t.Context(), archives.Record{ID: "sb_x", Owner: "b", Epoch: 1, Checkpoint: ck, Claim: []byte(`{}`)}); err != nil { + t.Errorf("adopter's put: %v", err) + } + }} + if err := a.Reconcile(t.Context()); err != nil { + t.Fatalf("reconcile: %v", err) + } + if !ckExists(t, a, ck) { + t.Error("the restarted node deleted a checkpoint the adopter committed during its reconcile") + } +} + func TestFailoverRestartDropsAClaimWhoseInterruptedArchiveWasAdopted(t *testing.T) { c := newFailoverCluster(t) a := c.node(t, "a", t.TempDir(), keepAnHour) @@ -1943,6 +1968,7 @@ type hookRegistry struct { beforeTake func() beforeConsume func() beforeDelete func() + afterRows func() loseConsumeReply bool loseTakeReply bool losePutReply bool @@ -2002,6 +2028,15 @@ func (r *hookRegistry) Delete(ctx context.Context, id string, epoch int64) error return r.Registry.Delete(ctx, id, epoch) } +func (r *hookRegistry) Rows(ctx context.Context, ids []string) (map[string]archives.Record, error) { + rows, err := r.Registry.Rows(ctx, ids) + if hook := r.afterRows; hook != nil { + r.afterRows = nil + hook() + } + return rows, err +} + func (r *hookRegistry) Consume(ctx context.Context, id string, expect int64) (int64, error) { if hook := r.beforeConsume; hook != nil { r.beforeConsume = nil diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index afbc1485..f3443bae 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -24,7 +24,7 @@ func (m *Manager) Reconcile(ctx context.Context) error { if err != nil { return err } - rows, fenced := m.fenceArchived(ctx, claims) + fenced := m.fenceArchived(ctx, claims) vms, err := m.eng.List(ctx) if err != nil { return fmt.Errorf("list vms: %w", err) @@ -102,7 +102,7 @@ func (m *Manager) Reconcile(ctx context.Context) error { _ = utils.RemoveDirEntries(m.dataDir, func(name string) bool { return strings.HasPrefix(name, "fork-") }) if fenced { - m.reclaimOrphanArchiveCks(ctx, claims, rows) + m.reclaimOrphanArchiveCks(ctx, claims) } m.retryArchiveDeletes(ctx) removed := m.sweepStaleVMs(ctx, live, owned) @@ -280,20 +280,36 @@ func (m *Manager) readoptEgressTap(sb *types.Sandbox, live map[string]types.VMRe } // reclaimOrphanArchiveCks reaps archives that crashed between publish and journal commit. -func (m *Manager) reclaimOrphanArchiveCks(ctx context.Context, claims map[string]*types.Sandbox, rows map[string]archives.Record) { +func (m *Manager) reclaimOrphanArchiveCks(ctx context.Context, claims map[string]*types.Sandbox) { logger := log.WithFunc("pool.reclaimOrphanArchiveCks") metas, err := m.ckpts.Metas(ctx) if err != nil { logger.Warnf(ctx, "list archive cks skipped: %v", err) return } + var orphans []types.Checkpoint + var ids []string for _, rec := range metas { var ckpt types.Checkpoint if json.Unmarshal(rec.Meta, &ckpt) != nil || !ckpt.Archive { continue } orig, mine := claims[ckpt.SandboxID] - if !mine || orig.ArchiveCk == ckpt.ID || orig.ReplicaCk == ckpt.ID || orig.ReplicaPending == ckpt.ID || orig.ArchivePending == ckpt.ID || rows[ckpt.SandboxID].Checkpoint == ckpt.ID { + if !mine || orig.ArchiveCk == ckpt.ID || orig.ReplicaCk == ckpt.ID || orig.ReplicaPending == ckpt.ID || orig.ArchivePending == ckpt.ID { + continue + } + orphans = append(orphans, ckpt) + ids = append(ids, ckpt.SandboxID) + } + var rows map[string]archives.Record + if m.archives != nil && len(orphans) > 0 { + if rows, err = m.archives.Rows(ctx, ids); err != nil { + logger.Warnf(ctx, "reclaim orphan archive cks skipped: %v", err) + return + } + } + for _, ckpt := range orphans { + if row, ok := rows[ckpt.SandboxID]; ok && (row.Owner != m.nodeID || row.Checkpoint == ckpt.ID) { continue } if err := m.deleteArchiveCk(ctx, ckpt.ID); err != nil { From baaebe30792e7357364213b6ff3d8114e847d831 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 14:32:07 +0800 Subject: [PATCH 30/36] docs: replica storage budget, full first upload, dead-owner residue A round uploads only changed chunks and soft-deletes the previous replica, so its unique chunks live through the store grace plus a sweep. The first round after a wake or adoption is a full upload. A dead owner can also leave a committed pending or previous replica, whose live metas pin their chunks. --- docs/cluster.md | 32 ++++++++++++++++++++++---------- docs/deploy.md | 2 +- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index f175297e..0611d22e 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -242,7 +242,8 @@ claim it still lists once its lease runs out; the epoch fences it out. A claim that has entered its wake counts as running. Running and hibernated claims of a pool without `replicate_seconds` are lost with their node, and an owner that never returns leaves behind only the tombstones and checkpoints of the claims it -was waking or releasing when it died. +was waking or releasing when it died, plus what a replicated claim leaves (see +[running claims](#running-claims-across-node-loss)). ### Node lease @@ -302,7 +303,8 @@ checkpoint store and keeps each claim's registry row on the latest snapshot: concurrency, each round a capture, upload and registry write, and a new pass cannot start before the previous one ends, so a pass grows with claim count and guest memory; for example, an interval of 30 s with a 90 s pass can lose - about 185 s. + about 185 s. The first round of a claim after a wake or adoption uploads the + whole claim, so that pass is the long pass in this bound. While rounds fail it is the time since the last one that committed. - **Lifecycle.** A release consumes the row, then deletes the replica and the tombstone. A hibernate leaves the row on the last replica. An archive puts its @@ -322,8 +324,11 @@ checkpoint store and keeps each claim's registry row on the latest snapshot: hourly sweep never purges it, because no round refreshes it while the claim is hibernated, and retention starts when the claim is archived or adopted. The rows of a node that never returns therefore stay until a client adopts them, a - bounded residue of one row and one checkpoint per claim. Apart from that and - the lease rule the sweep is unchanged. + bounded residue of one row and up to two checkpoints per claim: the row's, and + a committed pending replica or a previous replica the node died before + deleting. Their metas stay live, so the store sweep never reclaims them and + they pin their chunks. Apart from that and the lease rule the sweep is + unchanged. - **Self-fencing.** A node whose renewal is overdue by 12 seconds (the lease length minus a 3 second margin, counted from the start of the last renewal that answered) stops serving every replicated claim it holds: relay, exec, @@ -364,12 +369,19 @@ checkpoint store and keeps each claim's registry row on the latest snapshot: do not replicate are unaffected. An archived claim of a replicating pool is gated like a running one while the node is fenced, so its wake waits too. -A replica costs, per claim per interval, one capture pause, one upload, one -registry read, one write and one delete; the node's own cost is one scan of its -claims every 5 seconds. Resolving a claim for a stream of a pool that does not -replicate reads one plain field and no atomic; a replicated claim adds one -atomic load of the fence state and its stream one watcher goroutine. Adding a -claim to a failover node gains one map lookup. Nothing runs per frame. +A replica costs, per claim per interval, one capture pause, an upload of the +changed chunks plus a manifest and a meta, one registry read, one write and one +delete of the previous replica; the node's own cost is one scan of its claims +every 5 seconds. The delete is soft: the previous replica's manifest stays for +the store's one-hour grace plus up to one hourly sweep, so the chunks only it +names stay that long. Budget storage per replicated claim as the live replica +plus roughly the per-round churn times (grace + sweep period) / interval. The +first round after a wake or adoption uploads the whole claim again, because the +wake deletes the record it restored from; later rounds dedup. Resolving a claim +for a stream of a pool that does not replicate reads one plain field and no +atomic; a replicated claim adds one atomic load of the fence state and its +stream one watcher goroutine. Adding a claim to a failover node gains one map +lookup. Nothing runs per frame. ## Templates on a cluster diff --git a/docs/deploy.md b/docs/deploy.md index f1e1f5a9..5d835c82 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -143,7 +143,7 @@ gets the intersection of the two. | `capture_trim` (pool entry) | `false` | Trim the guest's copy-on-write disk before a promote or checkpoint of this pool's sandboxes (and of the clones of a template promoted from it), so the record carries only live data: blocks a build deleted — a `COPY` step's archive, a `RUN` step's caches — are no longer captured. sandboxd mounts the disk init resolved from `cocoon.cow` a second time at `/run/sandboxd-trim` through silkd as root, runs `fstrim` and unmounts, under the capture's transition lock (about 15 ms on a warm clone). A failed trim is logged and the capture proceeds, larger. A hibernated or archived sandbox is captured untrimmed. Config-owned like `egress`: `PUT /v1/pools` rejects it | | `egress_classes` | unset | named tenant egress layers: `[{"name": "desk", "egress": {…}, "egress_upstream_env": "DESK_UPSTREAM"}]`. A tenant names one with `egress_class` through the [tenant API](sandboxd-api.md#tenants-v1tenants) and its claims take that policy intersected with the pool's. `egress` is required; `intercept` stays a pool-rule flag. Reloadable: a class's policy reaches live claims on their next request. A class removed while tenants still name it leaves them with no egress and is logged at load and at reload | | `egress_upstream_env` (pool or egress class entry) | unset | names a node environment variable holding the default upstream proxy URL for the pool's or class's claims (class before pool, a claim's own entry before both); needs `egress_upstream`, and the URL must be on its `allow`. Read and checked at startup. Config-owned like `egress`: `PUT /v1/pools` rejects it | -| `replicate_seconds` (pool entry) | 0 (off) | snapshot every running claim of this pool to the checkpoint store this often and point the claim's registry row at the snapshot, so another node restores the claim, memory and processes included, when this node is lost (see [running claims across node loss](cluster.md#running-claims-across-node-loss)). Each round pauses the guest for the length of a memory and disk capture. A restore resumes from the last committed replica and loses everything after it: up to the longer of the interval and one replication pass, plus the 5-second tick, plus one more pass, since a claim can run first in one pass and last in the next. A pass covers all the node's due claims of the replicating pools with bounded concurrency, so it grows with claim count and guest memory. The claim's row carries no retention deadline while it runs. Requires `archive_failover` and a `none`-lane pool, both checked at config load; a claim with volumes is skipped. Config-owned like `egress`: `PUT /v1/pools` rejects it. Restart only: a claim is replicated by its pool's setting from the moment it is claimed, so a reload that changes it is refused (a pool restarted with 0 drops its claims' rows and replicas on the first pass) | +| `replicate_seconds` (pool entry) | 0 (off) | snapshot every running claim of this pool to the checkpoint store this often and point the claim's registry row at the snapshot, so another node restores the claim, memory and processes included, when this node is lost (see [running claims across node loss](cluster.md#running-claims-across-node-loss)). Each round pauses the guest for the length of a memory and disk capture. A restore resumes from the last committed replica and loses everything after it: up to the longer of the interval and one replication pass, plus the 5-second tick, plus one more pass, since a claim can run first in one pass and last in the next. A pass covers all the node's due claims of the replicating pools with bounded concurrency, so it grows with claim count and guest memory; the first round of a claim after a wake or adoption uploads the whole claim, so that pass is the long pass. The claim's row carries no retention deadline while it runs. Requires `archive_failover` and a `none`-lane pool, both checked at config load; a claim with volumes is skipped. Config-owned like `egress`: `PUT /v1/pools` rejects it. Restart only: a claim is replicated by its pool's setting from the moment it is claimed, so a reload that changes it is refused (a pool restarted with 0 drops its claims' rows and replicas on the first pass) | | `storage` (pool entry) | unset (cocoon's default, 10G) | size of the copy-on-write disk the pool's golden is cold-booted with, in cocoon's own `--storage` spelling (`"40G"`, `"40GiB"`, `"40Gi"`: Docker/Kubernetes units, all binary) and passed to it verbatim; every clone, fork, checkpoint branch and promoted template of the pool inherits it. At least cocoon's default of 10G. Equal sizes spelled differently build the same golden; a changed size rebuilds the golden on the next build, and claims already out keep their disk. Config-owned like `egress`: `PUT /v1/pools` rejects it | | `max_claims` | 0 (unlimited) | node-wide cap on live claims; claim/fork/branch requests beyond it answer 429 with the pool state unharmed (on a cluster a non-volume claim tries a warm-peer redirect first; a volume claim answers the 429 with no redirect) | | `audit_log` | false | append every relayed frame that opens an RPC — its op + addressing fields, never payloads — to `/audit.jsonl`; continuation `data`, `data_end`, `stdin`, and `stdin_close` frames pass through unrecorded. The file is size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`), plus `method` (`GET`, `CONNECT`, `SOCKS5`, …), `decision` (`allow`, `deny`, `rate_limited`), `secret` (the ref name, never its value), `upstream` (the [upstream proxy](egress.md#upstream-proxies)'s `host:port`, when one carried it), `tenant`, and on an allowed connection `src_addr` and `dst_addr` (the node-side and destination `ip:port` of the upstream TCP connection, the tunnel target through an upstream proxy) on `egress` records, which a relayed frame cannot set; preview accesses record as op `preview`, one per request; a tenant change records as op `tenants` with the `added`, `changed` and `removed` names, never a token. An opening frame whose first line exceeds 4 KiB records as op `oversized` with no addressing fields | From 1504fe798c59e625c20bc93b3a4d9a3bc4a44efa Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 14:32:41 +0800 Subject: [PATCH 31/36] docs: replica chunks wait for the operator's reclaim sandboxd no longer sweeps the S3 store, so a superseded replica's unique chunks and a dead owner's replica records stay until the operator reclaims them. --- docs/cluster.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index 0611d22e..7d31f677 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -326,9 +326,8 @@ checkpoint store and keeps each claim's registry row on the latest snapshot: rows of a node that never returns therefore stay until a client adopts them, a bounded residue of one row and up to two checkpoints per claim: the row's, and a committed pending replica or a previous replica the node died before - deleting. Their metas stay live, so the store sweep never reclaims them and - they pin their chunks. Apart from that and the lease rule the sweep is - unchanged. + deleting. Their records and chunks stay in the store until the operator + removes them. Apart from that and the lease rule the sweep is unchanged. - **Self-fencing.** A node whose renewal is overdue by 12 seconds (the lease length minus a 3 second margin, counted from the start of the last renewal that answered) stops serving every replicated claim it holds: relay, exec, @@ -372,14 +371,15 @@ checkpoint store and keeps each claim's registry row on the latest snapshot: A replica costs, per claim per interval, one capture pause, an upload of the changed chunks plus a manifest and a meta, one registry read, one write and one delete of the previous replica; the node's own cost is one scan of its claims -every 5 seconds. The delete is soft: the previous replica's manifest stays for -the store's one-hour grace plus up to one hourly sweep, so the chunks only it -names stay that long. Budget storage per replicated claim as the live replica -plus roughly the per-round churn times (grace + sweep period) / interval. The -first round after a wake or adoption uploads the whole claim again, because the -wake deletes the record it restored from; later rounds dedup. Resolving a claim -for a stream of a pool that does not replicate reads one plain field and no -atomic; a replicated claim adds one atomic load of the fence state and its +every 5 seconds. The delete is soft: the chunks only the previous replica names +stay in the store until the operator's +[reclaim](deploy.md#reclaiming-s3-storage) removes them. With replication this +is the fastest-growing part of the store: roughly the per-round churn times the +rounds since the last reclaim, per replicated claim, on top of the live replica. +The first round after a wake or adoption uploads the whole claim again, because +the wake deletes the record it restored from; later rounds dedup. Resolving a +claim for a stream of a pool that does not replicate reads one plain field and +no atomic; a replicated claim adds one atomic load of the fence state and its stream one watcher goroutine. Adding a claim to a failover node gains one map lookup. Nothing runs per frame. From caeb8870cc8d1602712ba3246be411ab57e99870 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 14:32:56 +0800 Subject: [PATCH 32/36] docs: dead-owner replica residue is operator-deletable --- docs/cluster.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index 7d31f677..fb2c830a 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -326,8 +326,10 @@ checkpoint store and keeps each claim's registry row on the latest snapshot: rows of a node that never returns therefore stay until a client adopts them, a bounded residue of one row and up to two checkpoints per claim: the row's, and a committed pending replica or a previous replica the node died before - deleting. Their records and chunks stay in the store until the operator - removes them. Apart from that and the lease rule the sweep is unchanged. + deleting. Their metas stay live, so a reclaim by the documented rule keeps + them. They are archive checkpoints (`"archive": true` in their meta) that no + registry row names, which an operator can delete. Apart from that and the + lease rule the sweep is unchanged. - **Self-fencing.** A node whose renewal is overdue by 12 seconds (the lease length minus a 3 second margin, counted from the start of the last renewal that answered) stops serving every replicated claim it holds: relay, exec, From 2bb38e3352fd41900467c31b11322094f946bbc2 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 14:46:07 +0800 Subject: [PATCH 33/36] pool: reclaim only the archive checkpoints this node published The checkpoint meta records the publishing node for archive and replica captures. The restart reclaim skips a checkpoint another node published, which closes the window where an adopter has published but not yet registered its checkpoint when the re-read finds no row. A meta without a node, published by an older build, is skipped too: crash residue from an older build leaks rather than risking a delete. --- docs/cluster.md | 19 ++++++++++--------- e2e/wire_parity_test.go | 2 +- sandboxd/pool/checkpoint.go | 3 +++ sandboxd/pool/failover_test.go | 31 ++++++++++++++++++++++++++++++- sandboxd/pool/reconcile.go | 2 +- sandboxd/types/types.go | 1 + 6 files changed, 46 insertions(+), 12 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index fb2c830a..318a22ab 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -206,15 +206,16 @@ replication rounds write it, so the claim path is unchanged. record, and none of them deletes the checkpoint, which belongs to the new owner; the drop clears the old owner's delete marker first. Only a tombstone of its own means this node's commit never happened: the wake or the restart - re-arms it and keeps the claim. Orphan reclaim at restart never deletes a - checkpoint a row references. A restart also settles an archive a crash - interrupted: a live row it left for a claim it still holds is deleted with - its checkpoint, and a claim whose row another node holds is dropped, its VM - and snapshot swept as stale. An archive journals its intent (the checkpoint - it is about to register) before it writes the row. A restart that finds the - intent and no row keeps the claim while that checkpoint is still in the - store, since the row never committed, and drops it once the checkpoint is - gone, since only a node that consumed the row deletes it. + re-arms it and keeps the claim. Orphan reclaim at restart deletes only archive + checkpoints this node published, and never one a row references. A restart + also settles an archive a crash interrupted: a live row it left for a claim it + still holds is deleted with its checkpoint, and a claim whose row another node + holds is dropped, its VM and snapshot swept as stale. An archive journals its + intent (the checkpoint it is about to register) before it writes the row. A + restart that finds the intent and no row keeps the claim while that + checkpoint is still in the store, since the row never committed, and drops it + once the checkpoint is gone, since only a node that consumed the row deletes + it. A put whose outcome is unknown leaves the intent in place, and a release, the next archive and the hourly sweep settle it by the same rule before they act on the claim. diff --git a/e2e/wire_parity_test.go b/e2e/wire_parity_test.go index bf1a6b10..6e507ad1 100644 --- a/e2e/wire_parity_test.go +++ b/e2e/wire_parity_test.go @@ -41,7 +41,7 @@ var wirePairs = []struct { {"checkpointResponse", false, server.CheckpointResponse{}, nil}, {"checkpointClaimRequest", true, server.CheckpointClaimRequest{}, nil}, {"checkpointListResponse", false, server.CheckpointListResponse{}, nil}, - {"checkpointRecord", false, types.Checkpoint{}, []string{"key", "tenant", "policy_source", "no_egress", "guest_env", "archive"}}, + {"checkpointRecord", false, types.Checkpoint{}, []string{"node", "key", "tenant", "policy_source", "no_egress", "guest_env", "archive"}}, {"volumeListResponse", false, server.VolumeListResponse{}, nil}, {"VolumeInfo", false, types.VolumeInfo{}, nil}, {"VolumeState", false, types.VolumeState{}, nil}, diff --git a/sandboxd/pool/checkpoint.go b/sandboxd/pool/checkpoint.go index cc220d25..e02b70b2 100644 --- a/sandboxd/pool/checkpoint.go +++ b/sandboxd/pool/checkpoint.go @@ -172,6 +172,9 @@ func (m *Manager) publishCheckpoint(ctx context.Context, sb *types.Sandbox, ckID NoEgress: sb.NoEgress, Archive: archive, } + if archive { + ckpt.Node = m.nodeID + } staging, err := m.ckpts.Stage(ckpt.ID) if err != nil { return types.Checkpoint{}, "", fmt.Errorf("stage checkpoint: %w", err) diff --git a/sandboxd/pool/failover_test.go b/sandboxd/pool/failover_test.go index 32f45f29..0adb7413 100644 --- a/sandboxd/pool/failover_test.go +++ b/sandboxd/pool/failover_test.go @@ -314,6 +314,30 @@ func TestFailoverRestartReclaimSparesACheckpointAdoptedMidReconcile(t *testing.T } } +func TestFailoverRestartReclaimsOnlyTheCheckpointsItPublished(t *testing.T) { + c := newFailoverCluster(t) + dirA := t.TempDir() + claims := map[string]*types.Sandbox{ + "sb_x": {ID: "sb_x", VMName: "sbx-gone-1", Key: testKey, Token: "tok", HibernateSnap: "sbx-hib-x"}, + } + if err := newClaimStore(dirA, false).save(claims); err != nil { + t.Fatalf("setup: %v", err) + } + a := c.node(t, "a", dirA, keepAnHour) + const own, foreign = "ck_00000000000000e1", "ck_00000000000000e2" + seedArchiveCkBy(t, a, "a", own, "sb_x") + seedArchiveCkBy(t, a, "b", foreign, "sb_x") + if err := a.Reconcile(t.Context()); err != nil { + t.Fatalf("reconcile: %v", err) + } + if ckExists(t, a, own) { + t.Error("reconcile kept its own orphan archive checkpoint") + } + if !ckExists(t, a, foreign) { + t.Error("reconcile deleted a checkpoint another node published") + } +} + func TestFailoverRestartDropsAClaimWhoseInterruptedArchiveWasAdopted(t *testing.T) { c := newFailoverCluster(t) a := c.node(t, "a", t.TempDir(), keepAnHour) @@ -1945,6 +1969,11 @@ func interruptArchive(t *testing.T, m *Manager, sb *types.Sandbox, putCommitted } func seedArchiveCk(t *testing.T, m *Manager, id, sandboxID string) { + t.Helper() + seedArchiveCkBy(t, m, m.nodeID, id, sandboxID) +} + +func seedArchiveCkBy(t *testing.T, m *Manager, node, id, sandboxID string) { t.Helper() staging, err := m.ckpts.Stage(id) if err != nil { @@ -1953,7 +1982,7 @@ func seedArchiveCk(t *testing.T, m *Manager, id, sandboxID string) { if err := os.MkdirAll(filepath.Join(staging, store.ExportDir), 0o750); err != nil { t.Fatalf("mkdir: %v", err) } - meta := `{"id":"` + id + `","sandbox_id":"` + sandboxID + `","archive":true}` + meta := `{"id":"` + id + `","sandbox_id":"` + sandboxID + `","node":"` + node + `","archive":true}` if err := os.WriteFile(filepath.Join(staging, store.MetaFile), []byte(meta), 0o600); err != nil { t.Fatalf("meta: %v", err) } diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index f3443bae..0905a93c 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -295,7 +295,7 @@ func (m *Manager) reclaimOrphanArchiveCks(ctx context.Context, claims map[string continue } orig, mine := claims[ckpt.SandboxID] - if !mine || orig.ArchiveCk == ckpt.ID || orig.ReplicaCk == ckpt.ID || orig.ReplicaPending == ckpt.ID || orig.ArchivePending == ckpt.ID { + if !mine || ckpt.Node != m.nodeID || orig.ArchiveCk == ckpt.ID || orig.ReplicaCk == ckpt.ID || orig.ReplicaPending == ckpt.ID || orig.ArchivePending == ckpt.ID { continue } orphans = append(orphans, ckpt) diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index a90771c7..7d7b8066 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -693,6 +693,7 @@ type Checkpoint struct { ID string `json:"id"` Name string `json:"name,omitempty"` SandboxID string `json:"sandbox_id"` + Node string `json:"node,omitempty"` Key PoolKey `json:"key"` Tenant string `json:"tenant,omitempty"` CreatedAt time.Time `json:"created_at"` From ef3b211bda90ed08d7426a95c8f2a3426982016b Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 15:02:06 +0800 Subject: [PATCH 34/36] test: wait out an in-flight slow unfence before asserting the lift A second slow confirmation can start before the delay is removed and holds the fence for up to one lease TTL; the old wait of two renew periods made the test flaky (1 in 1800 runs under load, synctest deadlock on exit). --- sandboxd/pool/lease_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sandboxd/pool/lease_test.go b/sandboxd/pool/lease_test.go index c51c0255..56fc23b7 100644 --- a/sandboxd/pool/lease_test.go +++ b/sandboxd/pool/lease_test.go @@ -205,7 +205,7 @@ func TestLeaseSlowUnfenceStaysFencedAndKeepsRenewing(t *testing.T) { } cut.rowsDelay.Store(0) - time.Sleep(2 * leaseRenew) + time.Sleep(leaseTTL + 2*leaseRenew) synctest.Wait() assertServed(t, a, sb.ID, sb.Token) }) From d88218740416da6a218ebe4f7214e44d3d6047ab Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 15:48:51 +0800 Subject: [PATCH 35/36] docs: replica residue waits for offline reclamation --- docs/cluster.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index 318a22ab..1f1bafb9 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -327,8 +327,8 @@ checkpoint store and keeps each claim's registry row on the latest snapshot: rows of a node that never returns therefore stay until a client adopts them, a bounded residue of one row and up to two checkpoints per claim: the row's, and a committed pending replica or a previous replica the node died before - deleting. Their metas stay live, so a reclaim by the documented rule keeps - them. They are archive checkpoints (`"archive": true` in their meta) that no + deleting. Their metas stay live, so offline reclamation keeps them. + They are archive checkpoints (`"archive": true` in their meta) that no registry row names, which an operator can delete. Apart from that and the lease rule the sweep is unchanged. - **Self-fencing.** A node whose renewal is overdue by 12 seconds (the lease @@ -376,9 +376,9 @@ changed chunks plus a manifest and a meta, one registry read, one write and one delete of the previous replica; the node's own cost is one scan of its claims every 5 seconds. The delete is soft: the chunks only the previous replica names stay in the store until the operator's -[reclaim](deploy.md#reclaiming-s3-storage) removes them. With replication this +[offline reclamation](deploy.md#reclaiming-s3-storage) removes them. With replication this is the fastest-growing part of the store: roughly the per-round churn times the -rounds since the last reclaim, per replicated claim, on top of the live replica. +rounds since the last reclamation, per replicated claim, on top of the live replica. The first round after a wake or adoption uploads the whole claim again, because the wake deletes the record it restored from; later rounds dedup. Resolving a claim for a stream of a pool that does not replicate reads one plain field and From 933424b0fb67200ca8c98eeaa6b3641a48a60e47 Mon Sep 17 00:00:00 2001 From: CMGS Date: Sun, 18 Oct 2026 16:35:35 +0800 Subject: [PATCH 36/36] pool: unfence tears down a forgotten claim at once The teardowns were queued until every replicated claim was confirmed, so a registry error on a later claim returned early and dropped the queue; the forgotten claim's VM, already off the claim table and journal, then ran until the next restart. --- sandboxd/pool/lease.go | 8 ++---- sandboxd/pool/lease_test.go | 53 +++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/sandboxd/pool/lease.go b/sandboxd/pool/lease.go index 8a74fd08..4a136af3 100644 --- a/sandboxd/pool/lease.go +++ b/sandboxd/pool/lease.go @@ -112,7 +112,6 @@ func (m *Manager) unfence(ctx context.Context, start time.Time) { return } } - var teardowns []func() for id, sb := range held { keep, err := m.confirmOwnership(ctx, sb, rows[id]) if err != nil { @@ -123,16 +122,13 @@ func (m *Manager) unfence(ctx context.Context, start time.Time) { continue } if td := m.forgetAdopted(ctx, sb); td != nil { - teardowns = append(teardowns, func() { + go func() { sb.Transition.Lock() defer sb.Transition.Unlock() td() - }) + }() } } - for _, td := range teardowns { - go td() - } m.lift(ctx, fenced, start) } diff --git a/sandboxd/pool/lease_test.go b/sandboxd/pool/lease_test.go index 56fc23b7..36ed9199 100644 --- a/sandboxd/pool/lease_test.go +++ b/sandboxd/pool/lease_test.go @@ -211,6 +211,53 @@ func TestLeaseSlowUnfenceStaysFencedAndKeepsRenewing(t *testing.T) { }) } +func TestLeaseFailedConfirmKeepsTheTeardownOfAForgottenClaim(t *testing.T) { + for range 16 { + exercised := false + synctest.Test(t, func(t *testing.T) { + c := newReplicaCluster(t) + a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + moved, kept := mustClaim(t, a, testKey), mustClaim(t, a, testKey) + vm := lockedVMName(moved) + cut := &cutRegistry{Registry: c.reg, takeErr: func(id string) error { + a.mu.Lock() + _, known := a.claimed[moved.ID] + a.mu.Unlock() + if id != kept.ID || known { + return nil + } + exercised = true + return archives.ErrUnavailable + }} + a.archives = cut + replicateNow(t, a, moved) + replicateNow(t, a, kept) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + go a.keepLease(ctx) + synctest.Wait() + + cut.cut.Store(true) + time.Sleep(leaseTTL) + synctest.Wait() + c.expire("a") + if adopted, err := b.AdoptArchived(ctx, moved.ID, moved.Token); err != nil || !adopted { + t.Fatalf("adopt while the owner is fenced: %v %v", adopted, err) + } + cut.cut.Store(false) + time.Sleep(leaseRenew) + synctest.Wait() + if exercised && !a.eng.(*fakeEngine).removed(vm) { + t.Error("a failed confirmation dropped the teardown of a claim the owner had already forgotten") + } + }) + if exercised { + return + } + } + t.Fatal("the failing confirmation never followed the forgotten claim") +} + func TestLeaseAdoptionDuringUnfenceDropsTheClaimWithoutServingIt(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := newReplicaCluster(t) @@ -416,6 +463,7 @@ type cutRegistry struct { cut atomic.Bool rowsDelay atomic.Int64 beforeTake func() + takeErr func(id string) error } func (r *cutRegistry) Take(ctx context.Context, id string, expect int64, owner string) (int64, error) { @@ -423,6 +471,11 @@ func (r *cutRegistry) Take(ctx context.Context, id string, expect int64, owner s r.beforeTake = nil hook() } + if r.takeErr != nil { + if err := r.takeErr(id); err != nil { + return 0, err + } + } return r.Registry.Take(ctx, id, expect, owner) }