From a401edbe99f3d8978c8a16c8b360574c481b7c82 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 15:18:05 -0500 Subject: [PATCH 1/3] fix(lock): release a crashed node's locks once it is proven dead Lock records of a crashed node stayed held until the 15s staleness threshold. Replay after a crash therefore waited on locks the dead node held: TestJournalCrashDuringWrite failed 5/5 on ext4 images because mount-time replay waited 14.5s for the root directory lock, past the 10s mount timeout. lock.Manager.ReclaimOwner releases every record a given node holds. internal/node calls it, through Registry.SetOwnerDeadHandler, only for a node proven dead: its mount process is gone in this PID namespace on this host, or its heartbeat has not changed for the heartbeat timeout plus 5s by this node's monotonic clock. A slow node keeps its locks. Mount releases a dead previous incarnation's locks before replaying its journal, and failure detection does so after winning a slot's recovery lease. Failure detection also skips a slot whose heartbeat it saw advance within the heartbeat timeout, whatever the wall-clock time in it says, and a node whose own slot is marked Dead stops instead of journaling on. Co-Authored-By: Claude Opus 5.5 --- cmd/mount/main.go | 7 +- internal/lock/reclaim.go | 62 ++++++++++ internal/lock/reclaim_test.go | 66 +++++++++++ internal/node/liveness.go | 45 +++++++ internal/node/registry.go | 50 +++++++- internal/node/registry_test.go | 120 +++++++++++++++++++ tests/integration/lock_reclaim_test.go | 157 +++++++++++++++++++++++++ tests/test-manifest.yaml | 25 ++++ 8 files changed, 527 insertions(+), 5 deletions(-) create mode 100644 internal/lock/reclaim.go create mode 100644 internal/lock/reclaim_test.go create mode 100644 tests/integration/lock_reclaim_test.go diff --git a/cmd/mount/main.go b/cmd/mount/main.go index e77fd5d9..800c7263 100644 --- a/cmd/mount/main.go +++ b/cmd/mount/main.go @@ -123,7 +123,7 @@ func main() { log.Fatalf("Failed to register node: %v", err) } nodeReg.SetSlotLostHandler(func(slot int, owner [16]byte) { - log.Fatalf("Node registry slot %d now belongs to node %x; stopping so two nodes never share a journal", slot, owner[:8]) + log.Fatalf("Node registry slot %d was taken over or marked dead (owner %x); stopping so two nodes never share a journal", slot, owner[:8]) }) if err := nodeReg.StartHeartbeat(); err != nil { log.Fatalf("Failed to start node heartbeat: %v", err) @@ -134,6 +134,11 @@ func main() { // Create lock manager for distributed locking lockMgr := lock.NewManager(dev, lockOffset, lockSize, nodeUUID, slotIdx) + nodeReg.SetOwnerDeadHandler(func(owner [16]byte) { + if _, err := lockMgr.ReclaimOwner(owner); err != nil { + log.Printf("Warning: Failed to release the locks of dead node %x: %v", owner[:8], err) + } + }) // Create managers with distributed locking enabled // NOTE: Managers must be created BEFORE journal recovery so we can replay transactions diff --git a/internal/lock/reclaim.go b/internal/lock/reclaim.go new file mode 100644 index 00000000..0d1cb9bf --- /dev/null +++ b/internal/lock/reclaim.go @@ -0,0 +1,62 @@ +package lock + +import "log" + +// ReclaimOwner releases the lock records held by owner and returns how many it +// changed. Call it only once owner is proven dead (its process gone, or its +// lease expired by this node's monotonic clock plus a margin): a slow node +// that still runs would lose locks it is using. +// +// Exclusive records of owner are freed. A shared record names only its latest +// reader, so when that reader is owner its one count is dropped and the record +// is freed if it was the last; other readers' counts stay until they release +// or the record goes stale. +func (m *Manager) ReclaimOwner(owner [16]byte) (int, error) { + reclaimed := 0 + for recordID := uint64(0); recordID < m.size/RecordSize; recordID++ { + changed, err := m.reclaimRecord(recordID, owner) + if err != nil { + return reclaimed, err + } + if changed { + reclaimed++ + } + } + if reclaimed > 0 { + log.Printf("lock: reclaimed %d lock records of dead node %x", reclaimed, owner[:8]) + } + return reclaimed, nil +} + +func (m *Manager) reclaimRecord(recordID uint64, owner [16]byte) (bool, error) { + gate := m.recordGate(recordID) + gate.Lock() + defer gate.Unlock() + + for { + record, err := m.readRecord(recordID) + if err != nil { + return false, err + } + if record.Mode == LockModeNone || record.OwnerUUID != owner { + return false, nil + } + + next := &Record{Sequence: record.Sequence + 1} + if record.Mode == LockModeShared && record.SharedCount > 1 { + kept := *record + kept.OwnerUUID = [16]byte{} + kept.SharedCount-- + kept.Sequence++ + next = &kept + } + + ok, err := m.casRecord(recordID, record, next) + if err != nil { + return false, err + } + if ok { + return true, nil + } + } +} diff --git a/internal/lock/reclaim_test.go b/internal/lock/reclaim_test.go new file mode 100644 index 00000000..c52519b2 --- /dev/null +++ b/internal/lock/reclaim_test.go @@ -0,0 +1,66 @@ +package lock + +import ( + "context" + "testing" + "time" +) + +func TestReclaimOwnerFreesOnlyTheDeadNodesHolds(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + + dead := NewManager(dev, 0, 1<<20, [16]byte{1}, 1) + alive := NewManager(dev, 0, 1<<20, [16]byte{2}, 2) + survivor := NewManager(dev, 0, 1<<20, [16]byte{3}, 3) + ctx := context.Background() + records := map[uint64]bool{} + for _, res := range []uint64{2, 7, 9} { + records[survivor.hashToRecord(LockTypeInode, res)] = true + } + if len(records) != 3 { + t.Fatal("test resources share a lock record; pick others") + } + + if err := dead.Acquire(ctx, LockTypeInode, 2, LockModeExclusive); err != nil { + t.Fatal(err) + } + if err := alive.Acquire(ctx, LockTypeInode, 7, LockModeExclusive); err != nil { + t.Fatal(err) + } + if err := alive.Acquire(ctx, LockTypeInode, 9, LockModeShared); err != nil { + t.Fatal(err) + } + if err := dead.Acquire(ctx, LockTypeInode, 9, LockModeShared); err != nil { + t.Fatal(err) + } + + n, err := survivor.ReclaimOwner([16]byte{1}) + if err != nil { + t.Fatalf("ReclaimOwner: %v", err) + } + if n != 2 { + t.Errorf("reclaimed %d records, want the dead node's exclusive record and its shared count", n) + } + + short, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + if err := survivor.Acquire(short, LockTypeInode, 2, LockModeExclusive); err != nil { + t.Errorf("dead node's exclusive lock still held after ReclaimOwner: %v", err) + } + + shared, err := survivor.readRecord(survivor.hashToRecord(LockTypeInode, 9)) + if err != nil { + t.Fatal(err) + } + if shared.Mode != LockModeShared || shared.SharedCount != 1 { + t.Errorf("shared record is mode %d with %d readers, want the live reader kept", shared.Mode, shared.SharedCount) + } + held, err := survivor.readRecord(survivor.hashToRecord(LockTypeInode, 7)) + if err != nil { + t.Fatal(err) + } + if held.Mode != LockModeExclusive || held.OwnerUUID != ([16]byte{2}) { + t.Errorf("live node's exclusive record changed: mode %d owner %x", held.Mode, held.OwnerUUID) + } +} diff --git a/internal/node/liveness.go b/internal/node/liveness.go index 0dd14199..7dddcf9f 100644 --- a/internal/node/liveness.go +++ b/internal/node/liveness.go @@ -11,6 +11,19 @@ import ( // recoveryLease is how long a peer may hold a dead slot's recovery before others take it over. const recoveryLease = 60 * time.Second +// deathMargin is added to HeartbeatTimeout before this node's own clock counts +// an unchanged heartbeat as proof that its node is dead rather than slow. +const deathMargin = HeartbeatTimeout / 2 + +// sighting is what failure detection last saw in a slot, timed by this node's monotonic clock. +type sighting struct { + uuid [16]byte + seq uint64 + beat int64 + since time.Time + changedAt time.Time +} + var processStart = time.Now().UnixNano() func readBootID() (id [16]byte) { @@ -64,3 +77,35 @@ func (r *Registry) failed(slot *Slot, timeout time.Duration) bool { } return false } + +// observe records a slot's heartbeat as failure detection sees it. Called with r.mu held. +func (r *Registry) observe(slotIdx int, slot *Slot) { + last, seen := r.sightings[slotIdx] + if seen && last.uuid == slot.UUID && last.seq == slot.HeartbeatSeq && last.beat == slot.HeartbeatTime { + return + } + now := time.Now() + next := sighting{uuid: slot.UUID, seq: slot.HeartbeatSeq, beat: slot.HeartbeatTime, since: now} + if seen && last.uuid == slot.UUID { + next.changedAt = now + } + r.sightings[slotIdx] = next +} + +// recentlyAlive reports whether this node saw the slot's heartbeat advance within +// HeartbeatTimeout, whatever the heartbeat's wall-clock time says. Called with r.mu held. +func (r *Registry) recentlyAlive(slotIdx int) bool { + last, seen := r.sightings[slotIdx] + return seen && !last.changedAt.IsZero() && time.Since(last.changedAt) < HeartbeatTimeout +} + +// provenDead reports whether a slot's node is dead, not merely slow: its mount +// process is gone, or this node has seen the same heartbeat for longer than +// HeartbeatTimeout plus deathMargin. Called with r.mu held. +func (r *Registry) provenDead(slotIdx int, slot *Slot) bool { + if r.deadLocalOwner(slot) { + return true + } + last, seen := r.sightings[slotIdx] + return seen && last.uuid == slot.UUID && time.Since(last.since) >= HeartbeatTimeout+deathMargin +} diff --git a/internal/node/registry.go b/internal/node/registry.go index 78ffe056..c7f63e95 100644 --- a/internal/node/registry.go +++ b/internal/node/registry.go @@ -94,6 +94,10 @@ type Registry struct { pidNS uint64 onSlotLost func(slot int, owner [16]byte) + onOwnerDead func(owner [16]byte) + deadOwners [][16]byte + sightings map[int]sighting + // Cross-node recovery recoveryHandler RecoveryHandler failureCheckInterval time.Duration // How often to scan for failed nodes @@ -142,6 +146,7 @@ func NewRegistry(dev *storage.Device, registryOffset uint64) *Registry { stopCh: make(chan struct{}), failureCheckInterval: FailureCheckInterval, recoveryInProgress: make(map[int]bool), + sightings: make(map[int]sighting), bootID: readBootID(), pidNS: readPIDNamespace(), } @@ -168,6 +173,34 @@ func (r *Registry) SetSlotLostHandler(fn func(slot int, owner [16]byte)) { r.onSlotLost = fn } +// SetOwnerDeadHandler sets what to call with the UUID of a node proven dead +// whose slot this node took over, for example to release that node's locks. +// It is called right away for a dead previous incarnation that Register took +// the slot of. +func (r *Registry) SetOwnerDeadHandler(fn func(owner [16]byte)) { + r.mu.Lock() + r.onOwnerDead = fn + pending := r.deadOwners + r.deadOwners = nil + r.mu.Unlock() + + for _, owner := range pending { + fn(owner) + } +} + +func (r *Registry) ownerDead(owner [16]byte) { + r.mu.Lock() + fn := r.onOwnerDead + if fn == nil { + r.deadOwners = append(r.deadOwners, owner) + } + r.mu.Unlock() + if fn != nil { + fn(owner) + } +} + // SetJournalArea makes Register record a journal region in the claimed slot: // slot i owns slotSize bytes at offset + i*slotSize. func (r *Registry) SetJournalArea(offset, slotSize uint64) { @@ -250,6 +283,9 @@ func (r *Registry) claimFirst(slots []int, claimable func(*Slot) bool) (int, err return -1, err } r.localSlot = i + if r.deadLocalOwner(slot) { + r.deadOwners = append(r.deadOwners, slot.UUID) + } return i, nil } return -1, nil @@ -378,8 +414,9 @@ func (r *Registry) heartbeatLoop(slotIdx int) { case <-ticker.C: r.mu.Lock() slot, err := r.readSlot(slotIdx) - takenOver := err == nil && slot.UUID != r.localUUID && !lost - if err == nil && slot.UUID == r.localUUID { + ours := err == nil && slot.UUID == r.localUUID && slot.State != NodeStateDead + takenOver := err == nil && !ours && !lost + if ours { slot.HeartbeatSeq = seq slot.HeartbeatTime = time.Now().Unix() r.writeSlot(slotIdx, slot) @@ -672,12 +709,14 @@ func (r *Registry) detectAndRecoverFailedNodes() { r.mu.Unlock() continue } + r.observe(i, slot) - // Stale heartbeats need 2x timeout for consensus - if !r.failed(slot, 2*HeartbeatTimeout) { + // Stale heartbeats need 2x timeout for consensus, and none this node saw advance lately. + if !r.failed(slot, 2*HeartbeatTimeout) || (r.recentlyAlive(i) && !r.deadLocalOwner(slot)) { r.mu.Unlock() continue } + dead := r.provenDead(i, slot) // Skip if already recovering if r.recoveryInProgress[i] { @@ -695,6 +734,9 @@ func (r *Registry) detectAndRecoverFailedNodes() { r.mu.Unlock() continue // Another node beat us to it } + if dead { + r.ownerDead(info.UUID) + } if err := handler(info); err != nil { log.Printf("[Recovery] Replaying slot %d (node %x) failed: %v; any node retries after %v", diff --git a/internal/node/registry_test.go b/internal/node/registry_test.go index 96e933a7..00809668 100644 --- a/internal/node/registry_test.go +++ b/internal/node/registry_test.go @@ -1036,3 +1036,123 @@ func TestRegisterTakesOwnSlotThatGoesStaleWhileWaiting(t *testing.T) { t.Errorf("Register took slot %d, want slot 2, which went stale after the first scan", slot) } } + +func TestDetectionReleasesLocksOnlyOfProvenDeadNodes(t *testing.T) { + dev, cleanup := createTestDevice(t, 10<<20) + defer cleanup() + + reg := NewRegistry(dev, 0) + if reg.bootID == ([16]byte{}) || reg.pidNS == 0 { + t.Skip("no kernel boot ID or PID namespace") + } + if _, err := reg.Register(context.Background(), "node"); err != nil { + t.Fatal(err) + } + writeLocalSlot(t, reg, 2, exitedPID(t), 1) + slow := writeLocalSlotAt(t, reg, 3, os.Getppid(), time.Now().Add(-3*HeartbeatTimeout)) + + var replayed []int + var released [][16]byte + reg.SetRecoveryHandler(func(info *FailedNodeInfo) error { + replayed = append(replayed, info.SlotIndex) + return nil + }) + reg.SetOwnerDeadHandler(func(owner [16]byte) { released = append(released, owner) }) + reg.detectAndRecoverFailedNodes() + + if len(released) != 1 || released[0] != ([16]byte{3}) { + t.Errorf("released locks of %x, want only the exited node 03", released) + } + for _, owner := range released { + if owner == slow.UUID { + t.Error("released the locks of a node whose process is alive") + } + } + t.Logf("replayed slots %v", replayed) +} + +func TestRegisterReportsItsDeadPreviousIncarnation(t *testing.T) { + dev, cleanup := createTestDevice(t, 10<<20) + defer cleanup() + + reg := NewRegistry(dev, 0) + if reg.bootID == ([16]byte{}) || reg.pidNS == 0 { + t.Skip("no kernel boot ID or PID namespace") + } + writeLocalSlot(t, reg, 0, exitedPID(t), 1) + if slot, err := reg.Register(context.Background(), "node"); err != nil || slot != 0 { + t.Fatalf("Register = %d, %v; want slot 0 of the exited mount", slot, err) + } + + var released [][16]byte + reg.SetOwnerDeadHandler(func(owner [16]byte) { released = append(released, owner) }) + if len(released) != 1 || released[0] != ([16]byte{1}) { + t.Errorf("released locks of %x, want the previous incarnation 01", released) + } +} + +func TestHeartbeatStopsWhenSlotIsMarkedDead(t *testing.T) { + dev, cleanup := createTestDevice(t, 10<<20) + defer cleanup() + + reg := NewRegistry(dev, 0) + lost := make(chan struct{}, 1) + reg.SetSlotLostHandler(func(int, [16]byte) { lost <- struct{}{} }) + slot, err := reg.Register(context.Background(), "node") + if err != nil { + t.Fatal(err) + } + if err := reg.StartHeartbeat(); err != nil { + t.Fatal(err) + } + defer reg.Stop() + + own, err := reg.readSlot(slot) + if err != nil { + t.Fatal(err) + } + own.State = NodeStateDead + if err := reg.writeSlot(slot, own); err != nil { + t.Fatal(err) + } + select { + case <-lost: + case <-time.After(3 * HeartbeatInterval): + t.Error("heartbeat kept running in a slot another node marked dead") + } +} + +func TestOwnClockDecidesDeathNotWallClock(t *testing.T) { + reg := &Registry{sightings: make(map[int]sighting)} + slot := &Slot{UUID: [16]byte{4}, State: NodeStateActive, HeartbeatSeq: 1, + HeartbeatTime: time.Now().Add(-time.Hour).Unix()} + + reg.observe(1, slot) + if reg.provenDead(1, slot) { + t.Error("a heartbeat first seen just now counted as dead because its wall-clock time is old") + } + + reg.sightings[1] = sighting{uuid: slot.UUID, seq: 1, beat: slot.HeartbeatTime, + since: time.Now().Add(-(HeartbeatTimeout + deathMargin + time.Second))} + if !reg.provenDead(1, slot) { + t.Error("a heartbeat unchanged past timeout plus margin by this node's clock did not count as dead") + } + + slot.HeartbeatSeq++ + reg.observe(1, slot) + if reg.provenDead(1, slot) || !reg.recentlyAlive(1) { + t.Error("a heartbeat that advanced still counted as dead") + } +} + +func writeLocalSlotAt(t *testing.T, reg *Registry, idx int, pid int, heartbeat time.Time) *Slot { + t.Helper() + slot := &Slot{Magic: SlotMagic, State: NodeStateActive, UUID: [16]byte{byte(idx + 1)}, + HeartbeatTime: heartbeat.Unix(), BootID: reg.bootID, PID: uint32(pid), ProcStart: 1, + PIDNamespace: reg.pidNS} + copy(slot.Hostname[:], "node") + if err := reg.writeSlot(idx, slot); err != nil { + t.Fatal(err) + } + return slot +} diff --git a/tests/integration/lock_reclaim_test.go b/tests/integration/lock_reclaim_test.go new file mode 100644 index 00000000..4cf5a3c5 --- /dev/null +++ b/tests/integration/lock_reclaim_test.go @@ -0,0 +1,157 @@ +package integration + +import ( + "context" + "fmt" + "regexp" + "syscall" + "testing" + "time" + + "github.com/supporttools/GoClusterFS/internal/inode" + "github.com/supporttools/GoClusterFS/internal/lock" + "github.com/supporttools/GoClusterFS/internal/node" + "github.com/supporttools/GoClusterFS/internal/storage" + "github.com/supporttools/GoClusterFS/internal/superblock" +) + +// TestSurvivorReleasesCrashedNodeLocks crashes node A while it holds the root +// directory lock. Node B must prove A dead and take the lock well before the +// 15s staleness threshold, within a 10s mount timeout. +func TestSurvivorReleasesCrashedNodeLocks(t *testing.T) { + cluster, uuidA, logB := startPairForLockTest(t) + + if err := cluster.KillNode(0); err != nil { + t.Fatalf("kill node A: %v", err) + } + release := holdRootLockAs(t, cluster.ImagePath, uuidA) + defer release() + + start := time.Now() + for { + err := cluster.Node(1).WriteFile("after-crash", []byte("after-crash")) + if err == nil { + break + } + if time.Since(start) > 10*time.Second { + t.Fatalf("node B could not create a file within 10s of node A's crash: %v", err) + } + time.Sleep(200 * time.Millisecond) + } + if took := time.Since(start); took > 10*time.Second { + t.Errorf("node B needed %v to create a file after node A's crash, want under 10s", took) + } else { + t.Logf("node B created a file %v after node A crashed holding the root directory lock", took) + } + + reclaimed := regexp.MustCompile(`reclaimed \d+ lock records of dead node ` + fmt.Sprintf("%x", uuidA[:8])) + if !reclaimed.MatchString(logB.String()) { + t.Error("node B did not log releasing node A's locks") + } +} + +// TestSlowNodeKeepsLocks pauses node A (alive, but its heartbeat stops) while +// it holds the root directory lock. Node B must not take the lock while A's +// process exists and A's heartbeat has not been silent for the heartbeat +// timeout plus margin by B's own clock. +func TestSlowNodeKeepsLocks(t *testing.T) { + cluster, uuidA, logB := startPairForLockTest(t) + pid := cluster.Node(0).PID() + + if err := syscall.Kill(pid, syscall.SIGSTOP); err != nil { + t.Fatalf("pause node A: %v", err) + } + defer syscall.Kill(pid, syscall.SIGCONT) + release := holdRootLockAs(t, cluster.ImagePath, uuidA) + + attempt := func(result chan<- error) { + result <- cluster.Node(1).WriteFile("while-a-paused", []byte("x")) + } + result := make(chan error, 1) + go attempt(result) + paused := time.After(12 * time.Second) + for waiting := true; waiting; { + select { + case err := <-result: + if err == nil { + t.Fatal("node B took the root directory lock of a node that is paused but alive") + } + time.Sleep(500 * time.Millisecond) + go attempt(result) + case <-paused: + waiting = false + } + } + if regexp.MustCompile(`lock records of dead node ` + fmt.Sprintf("%x", uuidA[:8])).MatchString(logB.String()) { + t.Error("node B released the locks of a paused, live node") + } + + syscall.Kill(pid, syscall.SIGCONT) + release() + if err := cluster.Node(1).WriteFile("after-resume", []byte("x")); err != nil { + t.Errorf("node B could not create a file once node A resumed and released the lock: %v", err) + } +} + +func startPairForLockTest(t *testing.T) (*MultiNodeCluster, [16]byte, *syncBuffer) { + t.Helper() + cluster := NewMultiNodeCluster(t, 2, 50) + logA, logB := &syncBuffer{}, &syncBuffer{} + cluster.Node(0).Log, cluster.Node(1).Log = logA, logB + if err := cluster.FormatAndStartFirst(false); err != nil { + t.Fatalf("format and start node A: %v", err) + } + if err := cluster.StartNode(1, false); err != nil { + t.Fatalf("start node B: %v", err) + } + return cluster, nodeUUID(t, cluster.ImagePath, mustMatch(t, registeredRe, logA, "node A registration")), logB +} + +func nodeUUID(t *testing.T, imagePath, prefix string) [16]byte { + t.Helper() + dev, err := storage.Open(imagePath) + if err != nil { + t.Fatal(err) + } + defer dev.Close() + sb, err := superblock.Read(dev) + if err != nil { + t.Fatal(err) + } + slots, err := node.NewRegistry(dev, sb.NodeRegistryOffset).GetActiveNodes() + if err != nil { + t.Fatal(err) + } + for _, slot := range slots { + if fmt.Sprintf("%x", slot.UUID[:8]) == prefix { + return slot.UUID + } + } + t.Fatalf("no active registry slot for node %s", prefix) + return [16]byte{} +} + +// holdRootLockAs writes an exclusive root directory lock record owned by owner, as if owner held it. +func holdRootLockAs(t *testing.T, imagePath string, owner [16]byte) func() { + t.Helper() + dev, err := storage.Open(imagePath) + if err != nil { + t.Fatal(err) + } + sb, err := superblock.Read(dev) + if err != nil { + dev.Close() + t.Fatal(err) + } + locks := lock.NewManager(dev, sb.LockBitmapOffset, sb.LockBitmapSize, owner, 0) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + if err := locks.Acquire(ctx, lock.LockTypeInode, uint64(inode.InodeRoot), lock.LockModeExclusive); err != nil { + dev.Close() + t.Fatalf("hold root directory lock as %x: %v", owner[:8], err) + } + return func() { + locks.Release(lock.LockTypeInode, uint64(inode.InodeRoot)) + dev.Close() + } +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index eb9c88af..b406ef7c 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -447,6 +447,31 @@ scenarios: docker: run-tests.sh::test_restart_keeps_peer_journal vm: run-vm-tests.sh::test_restart_keeps_peer_journal + - id: survivor-releases-crashed-node-locks + name: Survivor releases a crashed node's locks + description: Node A crashes holding the root directory lock; node B proves A dead and creates a file within 10s instead of waiting out the 15s lock staleness threshold + priority: high + environments: + go: lock_reclaim_test.go::TestSurvivorReleasesCrashedNodeLocks + docker: TODO # writes a lock record as the crashed node from the test process; Go-only harness + vm: TODO # same as docker + + - id: slow-node-keeps-locks + name: Paused but alive node keeps its locks + description: Node A is paused (SIGSTOP) holding the root directory lock; node B must not release A's locks while A's process exists + priority: high + environments: + go: lock_reclaim_test.go::TestSlowNodeKeepsLocks + docker: TODO # needs SIGSTOP on a mount process and a lock record written as that node; Go-only harness + vm: TODO # same as docker + + - id: lock-reclaim-owner + name: ReclaimOwner frees only a dead node's holds + description: Exclusive records of the dead node are freed, its shared count is dropped, and other nodes' records stay + priority: go_only + environments: + go: internal/lock/reclaim_test.go::TestReclaimOwnerFreesOnlyTheDeadNodesHolds + - id: registry-slot-journal-region name: Registry records each slot's journal region description: Register stores a distinct journal region in each slot From 645fc995eed70ed894d39613a4575483b5b96423 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 15:18:20 -0500 Subject: [PATCH 2/3] docs(operations): say when a crashed node's locks are released Co-Authored-By: Claude Opus 5.5 --- docs/OPERATIONS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index de88a87b..3e961c7d 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -174,6 +174,7 @@ rm /mnt/fs/.snapshots/snapshot_name - **Clocks:** Nodes judge each other's heartbeats by wall clock. A node more than about 10 seconds ahead treats live peers as failed, replays their journals and takes their registry slots. Run NTP on every node. - **Restarts:** A restarted mount takes back the registry slot of a mount that exited on the same host (same hostname and kernel boot ID) and replays its journal before serving. In the same PID namespace it checks the old process ID; across PID namespaces it waits for the old slot's heartbeat to go stale. +- **Crashed node's locks:** A node's lock records are released as soon as it is proven dead: its mount process is gone (same host and PID namespace), or its heartbeat has not changed for 15 seconds by the observing node's own clock. Otherwise they are freed once they are 15 seconds old. A paused node keeps its locks. - **Node count:** The device has 64 registry slots, each with its own 4 MB journal. A crashed node's slot frees once another node replays its journal, or when the same host mounts again. ### Known Gaps in Slot Ownership From cba5167b066076fb05799c3b42b0b4e865029918 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 15:41:26 -0500 Subject: [PATCH 3/3] fix(lock): take over a dead node's exclusive locks for its replay Review of the first version found that freeing a dead node's locks was unsafe for shared records and too early for exclusive ones: - A shared record's OwnerUUID names only the latest reader, who may have released already, so dropping its count could take a live reader's. Shared records are now left to the staleness rule. - Freeing exclusive records before the replay let peers lock the dead node's directories and inodes mid-replay. AdoptOwner now makes the replaying node their holder, re-reads them after 50ms to notice a racing takeover, and ReleaseAdopted frees them after the replay. The replaying node's own acquires reuse them. A node whose heartbeat writes fail for 5s now stops, since peers would soon prove it dead and take its locks. OPERATIONS.md now says a node paused past about 20s counts as dead. TestSlowNodeKeepsLocks pauses 17s, between the 15s own-clock check and the 20s detection gate. Co-Authored-By: Claude Opus 5.5 --- cmd/mount/main.go | 27 ++++-- docs/OPERATIONS.md | 2 +- internal/lock/reclaim.go | 115 ++++++++++++++++++------- internal/lock/reclaim_test.go | 96 ++++++++++++++------- internal/node/registry.go | 73 +++++++++------- internal/node/registry_test.go | 52 +++++++---- tests/integration/lock_reclaim_test.go | 19 ++-- tests/test-manifest.yaml | 12 +-- 8 files changed, 269 insertions(+), 127 deletions(-) diff --git a/cmd/mount/main.go b/cmd/mount/main.go index 800c7263..e137795e 100644 --- a/cmd/mount/main.go +++ b/cmd/mount/main.go @@ -123,7 +123,7 @@ func main() { log.Fatalf("Failed to register node: %v", err) } nodeReg.SetSlotLostHandler(func(slot int, owner [16]byte) { - log.Fatalf("Node registry slot %d was taken over or marked dead (owner %x); stopping so two nodes never share a journal", slot, owner[:8]) + log.Fatalf("Node registry slot %d is no longer this node's (owner %x); stopping so two nodes never share a journal", slot, owner[:8]) }) if err := nodeReg.StartHeartbeat(); err != nil { log.Fatalf("Failed to start node heartbeat: %v", err) @@ -134,11 +134,11 @@ func main() { // Create lock manager for distributed locking lockMgr := lock.NewManager(dev, lockOffset, lockSize, nodeUUID, slotIdx) - nodeReg.SetOwnerDeadHandler(func(owner [16]byte) { - if _, err := lockMgr.ReclaimOwner(owner); err != nil { - log.Printf("Warning: Failed to release the locks of dead node %x: %v", owner[:8], err) - } - }) + nodeReg.SetOwnerDeadHandler(func(owner [16]byte) func() { return adoptLocks(lockMgr, owner) }) + releasePrevious := func() {} + if owner, ok := nodeReg.DeadPreviousOwner(); ok { + releasePrevious = adoptLocks(lockMgr, owner) + } // Create managers with distributed locking enabled // NOTE: Managers must be created BEFORE journal recovery so we can replay transactions @@ -174,6 +174,7 @@ func main() { } else if *skipRecovery { log.Println("Journal: WARNING - skipping recovery (emergency mode)") } + releasePrevious() // Create journal for write-ahead logging (crash consistency) var j *journal.Journal @@ -417,3 +418,17 @@ func main() { log.Println("Filesystem unmounted.") } + +// adoptLocks takes over a dead node's exclusive locks before its journal is +// replayed and returns the function that frees them after the replay. +func adoptLocks(locks *lock.Manager, owner [16]byte) func() { + adopted, err := locks.AdoptOwner(owner) + if err != nil { + log.Printf("Warning: Failed to take over the locks of dead node %x: %v", owner[:8], err) + } + return func() { + if err := locks.ReleaseAdopted(adopted); err != nil { + log.Printf("Warning: Failed to release the locks taken over from dead node %x: %v", owner[:8], err) + } + } +} diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 3e961c7d..4dabcc73 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -174,7 +174,7 @@ rm /mnt/fs/.snapshots/snapshot_name - **Clocks:** Nodes judge each other's heartbeats by wall clock. A node more than about 10 seconds ahead treats live peers as failed, replays their journals and takes their registry slots. Run NTP on every node. - **Restarts:** A restarted mount takes back the registry slot of a mount that exited on the same host (same hostname and kernel boot ID) and replays its journal before serving. In the same PID namespace it checks the old process ID; across PID namespaces it waits for the old slot's heartbeat to go stale. -- **Crashed node's locks:** A node's lock records are released as soon as it is proven dead: its mount process is gone (same host and PID namespace), or its heartbeat has not changed for 15 seconds by the observing node's own clock. Otherwise they are freed once they are 15 seconds old. A paused node keeps its locks. +- **Crashed node's locks:** Once a node is proven dead, the node replaying its journal takes over its exclusive locks and frees them after the replay, so no other node uses them mid-replay. Proven dead means its mount process is gone (same host and PID namespace), or its heartbeat is more than 20 seconds old and has not changed for 15 seconds by the observing node's own clock. A node paused for more than about 20 seconds therefore counts as dead; it exits on its next heartbeat. A dead node's shared locks, and the locks of a node not proven dead, are freed once they are 15 seconds old. - **Node count:** The device has 64 registry slots, each with its own 4 MB journal. A crashed node's slot frees once another node replays its journal, or when the same host mounts again. ### Known Gaps in Slot Ownership diff --git a/internal/lock/reclaim.go b/internal/lock/reclaim.go index 0d1cb9bf..ac4d14ca 100644 --- a/internal/lock/reclaim.go +++ b/internal/lock/reclaim.go @@ -1,34 +1,49 @@ package lock -import "log" +import ( + "log" + "time" +) -// ReclaimOwner releases the lock records held by owner and returns how many it -// changed. Call it only once owner is proven dead (its process gone, or its -// lease expired by this node's monotonic clock plus a margin): a slow node -// that still runs would lose locks it is using. +// adoptSettle is how long AdoptOwner waits before re-reading the records it +// took, so a peer's racing stale takeover of one of them is noticed. +const adoptSettle = 50 * time.Millisecond + +// AdoptOwner makes this node the holder of every exclusive lock record owner +// holds and returns their IDs. Call it only once owner is proven dead, before +// replaying owner's journal, and pass the result to ReleaseAdopted after the +// replay: until then peers cannot take the locks of the dead node's unfinished +// operations, while this node's own acquires reuse them. // -// Exclusive records of owner are freed. A shared record names only its latest -// reader, so when that reader is owner its one count is dropped and the record -// is freed if it was the last; other readers' counts stay until they release -// or the record goes stale. -func (m *Manager) ReclaimOwner(owner [16]byte) (int, error) { - reclaimed := 0 +// Shared records are left to the staleness rule: OwnerUUID names only the +// latest reader, who may have released already while others still read. +func (m *Manager) AdoptOwner(owner [16]byte) ([]uint64, error) { + var taken []uint64 for recordID := uint64(0); recordID < m.size/RecordSize; recordID++ { - changed, err := m.reclaimRecord(recordID, owner) + ok, err := m.adoptRecord(recordID, owner) if err != nil { - return reclaimed, err + return taken, err } - if changed { - reclaimed++ + if ok { + taken = append(taken, recordID) } } - if reclaimed > 0 { - log.Printf("lock: reclaimed %d lock records of dead node %x", reclaimed, owner[:8]) + if len(taken) == 0 { + return nil, nil + } + + time.Sleep(adoptSettle) + var adopted []uint64 + for _, recordID := range taken { + if m.holdsExclusive(recordID) { + adopted = append(adopted, recordID) + } } - return reclaimed, nil + log.Printf("lock: took over %d lock records of dead node %x", len(adopted), owner[:8]) + return adopted, nil } -func (m *Manager) reclaimRecord(recordID uint64, owner [16]byte) (bool, error) { +func (m *Manager) adoptRecord(recordID uint64, owner [16]byte) (bool, error) { gate := m.recordGate(recordID) gate.Lock() defer gate.Unlock() @@ -38,20 +53,15 @@ func (m *Manager) reclaimRecord(recordID uint64, owner [16]byte) (bool, error) { if err != nil { return false, err } - if record.Mode == LockModeNone || record.OwnerUUID != owner { + if record.Mode != LockModeExclusive || record.OwnerUUID != owner { return false, nil } - - next := &Record{Sequence: record.Sequence + 1} - if record.Mode == LockModeShared && record.SharedCount > 1 { - kept := *record - kept.OwnerUUID = [16]byte{} - kept.SharedCount-- - kept.Sequence++ - next = &kept - } - - ok, err := m.casRecord(recordID, record, next) + adopted := *record + adopted.OwnerUUID = m.nodeUUID + adopted.OwnerSlot = uint32(m.nodeSlot) + adopted.AcquireTime = time.Now().Unix() + adopted.Sequence++ + ok, err := m.casRecord(recordID, record, &adopted) if err != nil { return false, err } @@ -60,3 +70,46 @@ func (m *Manager) reclaimRecord(recordID uint64, owner [16]byte) (bool, error) { } } } + +// ReleaseAdopted frees the records AdoptOwner returned that this node still +// holds without using them itself; a record in use is freed by its own release. +func (m *Manager) ReleaseAdopted(records []uint64) error { + for _, recordID := range records { + if err := m.releaseAdopted(recordID); err != nil { + return err + } + } + return nil +} + +func (m *Manager) releaseAdopted(recordID uint64) error { + gate := m.recordGate(recordID) + gate.Lock() + defer gate.Unlock() + + m.mu.Lock() + _, inUse := m.records[recordID] + m.mu.Unlock() + if inUse { + return nil + } + + for { + record, err := m.readRecord(recordID) + if err != nil { + return err + } + if record.Mode != LockModeExclusive || record.OwnerUUID != m.nodeUUID { + return nil + } + ok, err := m.casRecord(recordID, record, &Record{Sequence: record.Sequence + 1}) + if err != nil || ok { + return err + } + } +} + +func (m *Manager) holdsExclusive(recordID uint64) bool { + record, err := m.readRecord(recordID) + return err == nil && record.Mode == LockModeExclusive && record.OwnerUUID == m.nodeUUID +} diff --git a/internal/lock/reclaim_test.go b/internal/lock/reclaim_test.go index c52519b2..67230b2e 100644 --- a/internal/lock/reclaim_test.go +++ b/internal/lock/reclaim_test.go @@ -6,61 +6,95 @@ import ( "time" ) -func TestReclaimOwnerFreesOnlyTheDeadNodesHolds(t *testing.T) { +func distinctRecords(t *testing.T, m *Manager, resources ...uint64) { + t.Helper() + records := map[uint64]bool{} + for _, res := range resources { + records[m.hashToRecord(LockTypeInode, res)] = true + } + if len(records) != len(resources) { + t.Fatal("test resources share a lock record; pick others") + } +} + +func TestAdoptOwnerHoldsDeadNodesLocksUntilReleased(t *testing.T) { dev, cleanup := createTestDevice(t, 1<<20) defer cleanup() dead := NewManager(dev, 0, 1<<20, [16]byte{1}, 1) alive := NewManager(dev, 0, 1<<20, [16]byte{2}, 2) - survivor := NewManager(dev, 0, 1<<20, [16]byte{3}, 3) + replayer := NewManager(dev, 0, 1<<20, [16]byte{3}, 3) + peer := NewManager(dev, 0, 1<<20, [16]byte{4}, 4) + distinctRecords(t, replayer, 2, 5, 7) ctx := context.Background() - records := map[uint64]bool{} - for _, res := range []uint64{2, 7, 9} { - records[survivor.hashToRecord(LockTypeInode, res)] = true - } - if len(records) != 3 { - t.Fatal("test resources share a lock record; pick others") - } - if err := dead.Acquire(ctx, LockTypeInode, 2, LockModeExclusive); err != nil { - t.Fatal(err) + for _, res := range []uint64{2, 5} { + if err := dead.Acquire(ctx, LockTypeInode, res, LockModeExclusive); err != nil { + t.Fatal(err) + } } if err := alive.Acquire(ctx, LockTypeInode, 7, LockModeExclusive); err != nil { t.Fatal(err) } - if err := alive.Acquire(ctx, LockTypeInode, 9, LockModeShared); err != nil { - t.Fatal(err) + + adopted, err := replayer.AdoptOwner([16]byte{1}) + if err != nil { + t.Fatalf("AdoptOwner: %v", err) + } + if len(adopted) != 2 { + t.Fatalf("adopted %d records, want the dead node's 2 exclusive records", len(adopted)) + } + if err := acquireWithin(peer, time.Second, LockTypeInode, 2, LockModeExclusive); err == nil { + t.Error("a peer took a dead node's lock while its journal was being replayed") + } + if err := acquireWithin(replayer, time.Second, LockTypeInode, 2, LockModeExclusive); err != nil { + t.Errorf("the replaying node could not use a lock it adopted: %v", err) } - if err := dead.Acquire(ctx, LockTypeInode, 9, LockModeShared); err != nil { + if err := replayer.Release(LockTypeInode, 2); err != nil { t.Fatal(err) } - n, err := survivor.ReclaimOwner([16]byte{1}) + if err := replayer.ReleaseAdopted(adopted); err != nil { + t.Fatalf("ReleaseAdopted: %v", err) + } + for _, res := range []uint64{2, 5} { + if err := acquireWithin(peer, 2*time.Second, LockTypeInode, res, LockModeExclusive); err != nil { + t.Errorf("lock %d still held after the replay released it: %v", res, err) + } + } + held, err := replayer.readRecord(replayer.hashToRecord(LockTypeInode, 7)) if err != nil { - t.Fatalf("ReclaimOwner: %v", err) + t.Fatal(err) } - if n != 2 { - t.Errorf("reclaimed %d records, want the dead node's exclusive record and its shared count", n) + if held.Mode != LockModeExclusive || held.OwnerUUID != ([16]byte{2}) { + t.Errorf("live node's lock changed: mode %d owner %x", held.Mode, held.OwnerUUID) } +} - short, cancel := context.WithTimeout(ctx, 2*time.Second) - defer cancel() - if err := survivor.Acquire(short, LockTypeInode, 2, LockModeExclusive); err != nil { - t.Errorf("dead node's exclusive lock still held after ReclaimOwner: %v", err) - } +func TestAdoptOwnerLeavesSharedRecordOfReleasedReader(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() - shared, err := survivor.readRecord(survivor.hashToRecord(LockTypeInode, 9)) - if err != nil { + reader := NewManager(dev, 0, 1<<20, [16]byte{2}, 2) + crashed := NewManager(dev, 0, 1<<20, [16]byte{1}, 1) + replayer := NewManager(dev, 0, 1<<20, [16]byte{3}, 3) + writer := NewManager(dev, 0, 1<<20, [16]byte{4}, 4) + ctx := context.Background() + + if err := reader.Acquire(ctx, LockTypeInode, 9, LockModeShared); err != nil { t.Fatal(err) } - if shared.Mode != LockModeShared || shared.SharedCount != 1 { - t.Errorf("shared record is mode %d with %d readers, want the live reader kept", shared.Mode, shared.SharedCount) + if err := crashed.Acquire(ctx, LockTypeInode, 9, LockModeShared); err != nil { + t.Fatal(err) } - held, err := survivor.readRecord(survivor.hashToRecord(LockTypeInode, 7)) - if err != nil { + if err := crashed.Release(LockTypeInode, 9); err != nil { t.Fatal(err) } - if held.Mode != LockModeExclusive || held.OwnerUUID != ([16]byte{2}) { - t.Errorf("live node's exclusive record changed: mode %d owner %x", held.Mode, held.OwnerUUID) + + if adopted, err := replayer.AdoptOwner([16]byte{1}); err != nil || len(adopted) != 0 { + t.Fatalf("AdoptOwner = %v, %v; want nothing taken from a reader that already released", adopted, err) + } + if err := acquireWithin(writer, time.Second, LockTypeInode, 9, LockModeExclusive); err == nil { + t.Error("a writer took a record another node is still reading") } } diff --git a/internal/node/registry.go b/internal/node/registry.go index c7f63e95..04d64f19 100644 --- a/internal/node/registry.go +++ b/internal/node/registry.go @@ -94,9 +94,9 @@ type Registry struct { pidNS uint64 onSlotLost func(slot int, owner [16]byte) - onOwnerDead func(owner [16]byte) - deadOwners [][16]byte - sightings map[int]sighting + onOwnerDead func(owner [16]byte) (release func()) + deadPrevious *[16]byte + sightings map[int]sighting // Cross-node recovery recoveryHandler RecoveryHandler @@ -173,32 +173,34 @@ func (r *Registry) SetSlotLostHandler(fn func(slot int, owner [16]byte)) { r.onSlotLost = fn } -// SetOwnerDeadHandler sets what to call with the UUID of a node proven dead -// whose slot this node took over, for example to release that node's locks. -// It is called right away for a dead previous incarnation that Register took -// the slot of. -func (r *Registry) SetOwnerDeadHandler(fn func(owner [16]byte)) { +// SetOwnerDeadHandler sets what failure detection calls, before replaying a +// proven-dead node's journal, with that node's UUID; for example to take over +// its locks. The returned function runs after the replay. +func (r *Registry) SetOwnerDeadHandler(fn func(owner [16]byte) (release func())) { r.mu.Lock() + defer r.mu.Unlock() r.onOwnerDead = fn - pending := r.deadOwners - r.deadOwners = nil - r.mu.Unlock() +} - for _, owner := range pending { - fn(owner) +// DeadPreviousOwner returns the UUID of the exited mount on this host whose +// slot Register took, if any. Its journal is this node's to replay at mount. +func (r *Registry) DeadPreviousOwner() ([16]byte, bool) { + r.mu.RLock() + defer r.mu.RUnlock() + if r.deadPrevious == nil { + return [16]byte{}, false } + return *r.deadPrevious, true } -func (r *Registry) ownerDead(owner [16]byte) { +func (r *Registry) ownerDead(owner [16]byte) func() { r.mu.Lock() fn := r.onOwnerDead - if fn == nil { - r.deadOwners = append(r.deadOwners, owner) - } r.mu.Unlock() - if fn != nil { - fn(owner) + if fn == nil { + return func() {} } + return fn(owner) } // SetJournalArea makes Register record a journal region in the claimed slot: @@ -284,7 +286,7 @@ func (r *Registry) claimFirst(slots []int, claimable func(*Slot) bool) (int, err } r.localSlot = i if r.deadLocalOwner(slot) { - r.deadOwners = append(r.deadOwners, slot.UUID) + r.deadPrevious = &slot.UUID } return i, nil } @@ -406,6 +408,7 @@ func (r *Registry) heartbeatLoop(slotIdx int) { seq := uint64(1) lost := false + lastBeat := time.Now() for { select { @@ -415,23 +418,33 @@ func (r *Registry) heartbeatLoop(slotIdx int) { r.mu.Lock() slot, err := r.readSlot(slotIdx) ours := err == nil && slot.UUID == r.localUUID && slot.State != NodeStateDead - takenOver := err == nil && !ours && !lost + takenOver := err == nil && !ours if ours { slot.HeartbeatSeq = seq slot.HeartbeatTime = time.Now().Unix() - r.writeSlot(slotIdx, slot) + if r.writeSlot(slotIdx, slot) == nil { + lastBeat = time.Now() + } seq++ } onSlotLost := r.onSlotLost r.mu.Unlock() - if takenOver { + // Peers prove a silent node dead and take its locks; a node that cannot heartbeat must stop. + failing := time.Since(lastBeat) > HeartbeatTimeout/2 + if (takenOver || failing) && !lost { lost = true + owner := r.localUUID + if takenOver { + owner = slot.UUID + } else { + log.Printf("ERROR: heartbeats to node registry slot %d have failed for %v", slotIdx, time.Since(lastBeat)) + } if onSlotLost != nil { - onSlotLost(slotIdx, slot.UUID) + onSlotLost(slotIdx, owner) } else { - log.Printf("ERROR: node registry slot %d was taken over by node %x; "+ - "this node no longer owns its journal region", slotIdx, slot.UUID[:8]) + log.Printf("ERROR: node registry slot %d is no longer this node's (owner %x); "+ + "this node no longer owns its journal region", slotIdx, owner[:8]) } } } @@ -734,11 +747,13 @@ func (r *Registry) detectAndRecoverFailedNodes() { r.mu.Unlock() continue // Another node beat us to it } + release := func() {} if dead { - r.ownerDead(info.UUID) + release = r.ownerDead(info.UUID) } - - if err := handler(info); err != nil { + err = handler(info) + release() + if err != nil { log.Printf("[Recovery] Replaying slot %d (node %x) failed: %v; any node retries after %v", i, info.UUID[:8], err, recoveryLease) r.mu.Lock() diff --git a/internal/node/registry_test.go b/internal/node/registry_test.go index 00809668..e0f4f4e4 100644 --- a/internal/node/registry_test.go +++ b/internal/node/registry_test.go @@ -3,6 +3,7 @@ package node import ( "context" "errors" + "fmt" "os" "os/exec" "path/filepath" @@ -1051,24 +1052,24 @@ func TestDetectionReleasesLocksOnlyOfProvenDeadNodes(t *testing.T) { writeLocalSlot(t, reg, 2, exitedPID(t), 1) slow := writeLocalSlotAt(t, reg, 3, os.Getppid(), time.Now().Add(-3*HeartbeatTimeout)) - var replayed []int - var released [][16]byte + var events []string reg.SetRecoveryHandler(func(info *FailedNodeInfo) error { - replayed = append(replayed, info.SlotIndex) + events = append(events, fmt.Sprintf("replay %d", info.SlotIndex)) return nil }) - reg.SetOwnerDeadHandler(func(owner [16]byte) { released = append(released, owner) }) + reg.SetOwnerDeadHandler(func(owner [16]byte) func() { + events = append(events, fmt.Sprintf("adopt %x", owner[:1])) + return func() { events = append(events, fmt.Sprintf("release %x", owner[:1])) } + }) reg.detectAndRecoverFailedNodes() - if len(released) != 1 || released[0] != ([16]byte{3}) { - t.Errorf("released locks of %x, want only the exited node 03", released) + want := []string{"adopt 03", "replay 2", "release 03", "replay 3"} + if fmt.Sprint(events) != fmt.Sprint(want) { + t.Errorf("events %v, want %v: only the exited node's locks are taken, and held through its replay", events, want) } - for _, owner := range released { - if owner == slow.UUID { - t.Error("released the locks of a node whose process is alive") - } + if slow.UUID != ([16]byte{4}) { + t.Fatalf("slow node UUID %x", slow.UUID) } - t.Logf("replayed slots %v", replayed) } func TestRegisterReportsItsDeadPreviousIncarnation(t *testing.T) { @@ -1084,10 +1085,31 @@ func TestRegisterReportsItsDeadPreviousIncarnation(t *testing.T) { t.Fatalf("Register = %d, %v; want slot 0 of the exited mount", slot, err) } - var released [][16]byte - reg.SetOwnerDeadHandler(func(owner [16]byte) { released = append(released, owner) }) - if len(released) != 1 || released[0] != ([16]byte{1}) { - t.Errorf("released locks of %x, want the previous incarnation 01", released) + if owner, ok := reg.DeadPreviousOwner(); !ok || owner != ([16]byte{1}) { + t.Errorf("DeadPreviousOwner = %x, %v; want the previous incarnation 01", owner, ok) + } +} + +func TestHeartbeatStopsWhenWritesFail(t *testing.T) { + dev, cleanup := createTestDevice(t, 10<<20) + defer cleanup() + + reg := NewRegistry(dev, 0) + lost := make(chan struct{}, 1) + reg.SetSlotLostHandler(func(int, [16]byte) { lost <- struct{}{} }) + if _, err := reg.Register(context.Background(), "node"); err != nil { + t.Fatal(err) + } + if err := reg.StartHeartbeat(); err != nil { + t.Fatal(err) + } + defer reg.Stop() + + dev.Close() + select { + case <-lost: + case <-time.After(HeartbeatTimeout): + t.Error("node kept running while its heartbeats failed; peers would prove it dead and take its locks") } } diff --git a/tests/integration/lock_reclaim_test.go b/tests/integration/lock_reclaim_test.go index 4cf5a3c5..d5c5c2e4 100644 --- a/tests/integration/lock_reclaim_test.go +++ b/tests/integration/lock_reclaim_test.go @@ -44,16 +44,17 @@ func TestSurvivorReleasesCrashedNodeLocks(t *testing.T) { t.Logf("node B created a file %v after node A crashed holding the root directory lock", took) } - reclaimed := regexp.MustCompile(`reclaimed \d+ lock records of dead node ` + fmt.Sprintf("%x", uuidA[:8])) - if !reclaimed.MatchString(logB.String()) { - t.Error("node B did not log releasing node A's locks") + tookOver := regexp.MustCompile(`took over \d+ lock records of dead node ` + fmt.Sprintf("%x", uuidA[:8])) + if !tookOver.MatchString(logB.String()) { + t.Error("node B did not log taking over node A's locks") } } -// TestSlowNodeKeepsLocks pauses node A (alive, but its heartbeat stops) while -// it holds the root directory lock. Node B must not take the lock while A's -// process exists and A's heartbeat has not been silent for the heartbeat -// timeout plus margin by B's own clock. +// TestSlowNodeKeepsLocks pauses node A for 17s while it holds the root +// directory lock: longer than the heartbeat timeout and B's own-clock check +// (15s), shorter than the 20s after which failure detection treats a silent +// node as dead. B must not take the lock before it goes stale, and must never +// take over A's locks as a dead node's. func TestSlowNodeKeepsLocks(t *testing.T) { cluster, uuidA, logB := startPairForLockTest(t) pid := cluster.Node(0).PID() @@ -70,6 +71,7 @@ func TestSlowNodeKeepsLocks(t *testing.T) { result := make(chan error, 1) go attempt(result) paused := time.After(12 * time.Second) + resume := time.After(17 * time.Second) for waiting := true; waiting; { select { case err := <-result: @@ -82,8 +84,9 @@ func TestSlowNodeKeepsLocks(t *testing.T) { waiting = false } } + <-resume if regexp.MustCompile(`lock records of dead node ` + fmt.Sprintf("%x", uuidA[:8])).MatchString(logB.String()) { - t.Error("node B released the locks of a paused, live node") + t.Error("node B took over the locks of a paused, live node") } syscall.Kill(pid, syscall.SIGCONT) diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index b406ef7c..2d7404a6 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -449,7 +449,7 @@ scenarios: - id: survivor-releases-crashed-node-locks name: Survivor releases a crashed node's locks - description: Node A crashes holding the root directory lock; node B proves A dead and creates a file within 10s instead of waiting out the 15s lock staleness threshold + description: Node A crashes holding the root directory lock; node B proves A dead, takes over its locks for the replay and creates a file within 10s instead of waiting out the 15s lock staleness threshold priority: high environments: go: lock_reclaim_test.go::TestSurvivorReleasesCrashedNodeLocks @@ -458,19 +458,19 @@ scenarios: - id: slow-node-keeps-locks name: Paused but alive node keeps its locks - description: Node A is paused (SIGSTOP) holding the root directory lock; node B must not release A's locks while A's process exists + description: Node A is paused (SIGSTOP) for 17s holding the root directory lock; node B must not take the lock before it is stale, nor take over A's locks as a dead node's priority: high environments: go: lock_reclaim_test.go::TestSlowNodeKeepsLocks docker: TODO # needs SIGSTOP on a mount process and a lock record written as that node; Go-only harness vm: TODO # same as docker - - id: lock-reclaim-owner - name: ReclaimOwner frees only a dead node's holds - description: Exclusive records of the dead node are freed, its shared count is dropped, and other nodes' records stay + - id: lock-adopt-owner + name: Replaying node takes over a dead node's exclusive locks + description: AdoptOwner holds the dead node's exclusive records through the replay and ReleaseAdopted frees them; shared records and other nodes' records stay priority: go_only environments: - go: internal/lock/reclaim_test.go::TestReclaimOwnerFreesOnlyTheDeadNodesHolds + go: internal/lock/reclaim_test.go::TestAdoptOwnerHoldsDeadNodesLocksUntilReleased - id: registry-slot-journal-region name: Registry records each slot's journal region