diff --git a/cmd/mount/main.go b/cmd/mount/main.go index e77fd5d9..e137795e 100644 --- a/cmd/mount/main.go +++ b/cmd/mount/main.go @@ -123,7 +123,7 @@ func main() { log.Fatalf("Failed to register node: %v", err) } nodeReg.SetSlotLostHandler(func(slot int, owner [16]byte) { - log.Fatalf("Node registry slot %d now belongs to node %x; stopping so two nodes never share a journal", slot, owner[:8]) + log.Fatalf("Node registry slot %d is no longer this node's (owner %x); stopping so two nodes never share a journal", slot, owner[:8]) }) if err := nodeReg.StartHeartbeat(); err != nil { log.Fatalf("Failed to start node heartbeat: %v", err) @@ -134,6 +134,11 @@ func main() { // Create lock manager for distributed locking lockMgr := lock.NewManager(dev, lockOffset, lockSize, nodeUUID, slotIdx) + nodeReg.SetOwnerDeadHandler(func(owner [16]byte) func() { return adoptLocks(lockMgr, owner) }) + releasePrevious := func() {} + if owner, ok := nodeReg.DeadPreviousOwner(); ok { + releasePrevious = adoptLocks(lockMgr, owner) + } // Create managers with distributed locking enabled // NOTE: Managers must be created BEFORE journal recovery so we can replay transactions @@ -169,6 +174,7 @@ func main() { } else if *skipRecovery { log.Println("Journal: WARNING - skipping recovery (emergency mode)") } + releasePrevious() // Create journal for write-ahead logging (crash consistency) var j *journal.Journal @@ -412,3 +418,17 @@ func main() { log.Println("Filesystem unmounted.") } + +// adoptLocks takes over a dead node's exclusive locks before its journal is +// replayed and returns the function that frees them after the replay. +func adoptLocks(locks *lock.Manager, owner [16]byte) func() { + adopted, err := locks.AdoptOwner(owner) + if err != nil { + log.Printf("Warning: Failed to take over the locks of dead node %x: %v", owner[:8], err) + } + return func() { + if err := locks.ReleaseAdopted(adopted); err != nil { + log.Printf("Warning: Failed to release the locks taken over from dead node %x: %v", owner[:8], err) + } + } +} diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index de88a87b..4dabcc73 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -174,6 +174,7 @@ rm /mnt/fs/.snapshots/snapshot_name - **Clocks:** Nodes judge each other's heartbeats by wall clock. A node more than about 10 seconds ahead treats live peers as failed, replays their journals and takes their registry slots. Run NTP on every node. - **Restarts:** A restarted mount takes back the registry slot of a mount that exited on the same host (same hostname and kernel boot ID) and replays its journal before serving. In the same PID namespace it checks the old process ID; across PID namespaces it waits for the old slot's heartbeat to go stale. +- **Crashed node's locks:** Once a node is proven dead, the node replaying its journal takes over its exclusive locks and frees them after the replay, so no other node uses them mid-replay. Proven dead means its mount process is gone (same host and PID namespace), or its heartbeat is more than 20 seconds old and has not changed for 15 seconds by the observing node's own clock. A node paused for more than about 20 seconds therefore counts as dead; it exits on its next heartbeat. A dead node's shared locks, and the locks of a node not proven dead, are freed once they are 15 seconds old. - **Node count:** The device has 64 registry slots, each with its own 4 MB journal. A crashed node's slot frees once another node replays its journal, or when the same host mounts again. ### Known Gaps in Slot Ownership diff --git a/internal/lock/reclaim.go b/internal/lock/reclaim.go new file mode 100644 index 00000000..ac4d14ca --- /dev/null +++ b/internal/lock/reclaim.go @@ -0,0 +1,115 @@ +package lock + +import ( + "log" + "time" +) + +// adoptSettle is how long AdoptOwner waits before re-reading the records it +// took, so a peer's racing stale takeover of one of them is noticed. +const adoptSettle = 50 * time.Millisecond + +// AdoptOwner makes this node the holder of every exclusive lock record owner +// holds and returns their IDs. Call it only once owner is proven dead, before +// replaying owner's journal, and pass the result to ReleaseAdopted after the +// replay: until then peers cannot take the locks of the dead node's unfinished +// operations, while this node's own acquires reuse them. +// +// Shared records are left to the staleness rule: OwnerUUID names only the +// latest reader, who may have released already while others still read. +func (m *Manager) AdoptOwner(owner [16]byte) ([]uint64, error) { + var taken []uint64 + for recordID := uint64(0); recordID < m.size/RecordSize; recordID++ { + ok, err := m.adoptRecord(recordID, owner) + if err != nil { + return taken, err + } + if ok { + taken = append(taken, recordID) + } + } + if len(taken) == 0 { + return nil, nil + } + + time.Sleep(adoptSettle) + var adopted []uint64 + for _, recordID := range taken { + if m.holdsExclusive(recordID) { + adopted = append(adopted, recordID) + } + } + log.Printf("lock: took over %d lock records of dead node %x", len(adopted), owner[:8]) + return adopted, nil +} + +func (m *Manager) adoptRecord(recordID uint64, owner [16]byte) (bool, error) { + gate := m.recordGate(recordID) + gate.Lock() + defer gate.Unlock() + + for { + record, err := m.readRecord(recordID) + if err != nil { + return false, err + } + if record.Mode != LockModeExclusive || record.OwnerUUID != owner { + return false, nil + } + adopted := *record + adopted.OwnerUUID = m.nodeUUID + adopted.OwnerSlot = uint32(m.nodeSlot) + adopted.AcquireTime = time.Now().Unix() + adopted.Sequence++ + ok, err := m.casRecord(recordID, record, &adopted) + if err != nil { + return false, err + } + if ok { + return true, nil + } + } +} + +// ReleaseAdopted frees the records AdoptOwner returned that this node still +// holds without using them itself; a record in use is freed by its own release. +func (m *Manager) ReleaseAdopted(records []uint64) error { + for _, recordID := range records { + if err := m.releaseAdopted(recordID); err != nil { + return err + } + } + return nil +} + +func (m *Manager) releaseAdopted(recordID uint64) error { + gate := m.recordGate(recordID) + gate.Lock() + defer gate.Unlock() + + m.mu.Lock() + _, inUse := m.records[recordID] + m.mu.Unlock() + if inUse { + return nil + } + + for { + record, err := m.readRecord(recordID) + if err != nil { + return err + } + if record.Mode != LockModeExclusive || record.OwnerUUID != m.nodeUUID { + return nil + } + ok, err := m.casRecord(recordID, record, &Record{Sequence: record.Sequence + 1}) + if err != nil || ok { + return err + } + } +} + +func (m *Manager) holdsExclusive(recordID uint64) bool { + record, err := m.readRecord(recordID) + return err == nil && record.Mode == LockModeExclusive && record.OwnerUUID == m.nodeUUID +} diff --git a/internal/lock/reclaim_test.go b/internal/lock/reclaim_test.go new file mode 100644 index 00000000..67230b2e --- /dev/null +++ b/internal/lock/reclaim_test.go @@ -0,0 +1,100 @@ +package lock + +import ( + "context" + "testing" + "time" +) + +func distinctRecords(t *testing.T, m *Manager, resources ...uint64) { + t.Helper() + records := map[uint64]bool{} + for _, res := range resources { + records[m.hashToRecord(LockTypeInode, res)] = true + } + if len(records) != len(resources) { + t.Fatal("test resources share a lock record; pick others") + } +} + +func TestAdoptOwnerHoldsDeadNodesLocksUntilReleased(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + + dead := NewManager(dev, 0, 1<<20, [16]byte{1}, 1) + alive := NewManager(dev, 0, 1<<20, [16]byte{2}, 2) + replayer := NewManager(dev, 0, 1<<20, [16]byte{3}, 3) + peer := NewManager(dev, 0, 1<<20, [16]byte{4}, 4) + distinctRecords(t, replayer, 2, 5, 7) + ctx := context.Background() + + for _, res := range []uint64{2, 5} { + if err := dead.Acquire(ctx, LockTypeInode, res, LockModeExclusive); err != nil { + t.Fatal(err) + } + } + if err := alive.Acquire(ctx, LockTypeInode, 7, LockModeExclusive); err != nil { + t.Fatal(err) + } + + adopted, err := replayer.AdoptOwner([16]byte{1}) + if err != nil { + t.Fatalf("AdoptOwner: %v", err) + } + if len(adopted) != 2 { + t.Fatalf("adopted %d records, want the dead node's 2 exclusive records", len(adopted)) + } + if err := acquireWithin(peer, time.Second, LockTypeInode, 2, LockModeExclusive); err == nil { + t.Error("a peer took a dead node's lock while its journal was being replayed") + } + if err := acquireWithin(replayer, time.Second, LockTypeInode, 2, LockModeExclusive); err != nil { + t.Errorf("the replaying node could not use a lock it adopted: %v", err) + } + if err := replayer.Release(LockTypeInode, 2); err != nil { + t.Fatal(err) + } + + if err := replayer.ReleaseAdopted(adopted); err != nil { + t.Fatalf("ReleaseAdopted: %v", err) + } + for _, res := range []uint64{2, 5} { + if err := acquireWithin(peer, 2*time.Second, LockTypeInode, res, LockModeExclusive); err != nil { + t.Errorf("lock %d still held after the replay released it: %v", res, err) + } + } + held, err := replayer.readRecord(replayer.hashToRecord(LockTypeInode, 7)) + if err != nil { + t.Fatal(err) + } + if held.Mode != LockModeExclusive || held.OwnerUUID != ([16]byte{2}) { + t.Errorf("live node's lock changed: mode %d owner %x", held.Mode, held.OwnerUUID) + } +} + +func TestAdoptOwnerLeavesSharedRecordOfReleasedReader(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + + reader := NewManager(dev, 0, 1<<20, [16]byte{2}, 2) + crashed := NewManager(dev, 0, 1<<20, [16]byte{1}, 1) + replayer := NewManager(dev, 0, 1<<20, [16]byte{3}, 3) + writer := NewManager(dev, 0, 1<<20, [16]byte{4}, 4) + ctx := context.Background() + + if err := reader.Acquire(ctx, LockTypeInode, 9, LockModeShared); err != nil { + t.Fatal(err) + } + if err := crashed.Acquire(ctx, LockTypeInode, 9, LockModeShared); err != nil { + t.Fatal(err) + } + if err := crashed.Release(LockTypeInode, 9); err != nil { + t.Fatal(err) + } + + if adopted, err := replayer.AdoptOwner([16]byte{1}); err != nil || len(adopted) != 0 { + t.Fatalf("AdoptOwner = %v, %v; want nothing taken from a reader that already released", adopted, err) + } + if err := acquireWithin(writer, time.Second, LockTypeInode, 9, LockModeExclusive); err == nil { + t.Error("a writer took a record another node is still reading") + } +} diff --git a/internal/node/liveness.go b/internal/node/liveness.go index 0dd14199..7dddcf9f 100644 --- a/internal/node/liveness.go +++ b/internal/node/liveness.go @@ -11,6 +11,19 @@ import ( // recoveryLease is how long a peer may hold a dead slot's recovery before others take it over. const recoveryLease = 60 * time.Second +// deathMargin is added to HeartbeatTimeout before this node's own clock counts +// an unchanged heartbeat as proof that its node is dead rather than slow. +const deathMargin = HeartbeatTimeout / 2 + +// sighting is what failure detection last saw in a slot, timed by this node's monotonic clock. +type sighting struct { + uuid [16]byte + seq uint64 + beat int64 + since time.Time + changedAt time.Time +} + var processStart = time.Now().UnixNano() func readBootID() (id [16]byte) { @@ -64,3 +77,35 @@ func (r *Registry) failed(slot *Slot, timeout time.Duration) bool { } return false } + +// observe records a slot's heartbeat as failure detection sees it. Called with r.mu held. +func (r *Registry) observe(slotIdx int, slot *Slot) { + last, seen := r.sightings[slotIdx] + if seen && last.uuid == slot.UUID && last.seq == slot.HeartbeatSeq && last.beat == slot.HeartbeatTime { + return + } + now := time.Now() + next := sighting{uuid: slot.UUID, seq: slot.HeartbeatSeq, beat: slot.HeartbeatTime, since: now} + if seen && last.uuid == slot.UUID { + next.changedAt = now + } + r.sightings[slotIdx] = next +} + +// recentlyAlive reports whether this node saw the slot's heartbeat advance within +// HeartbeatTimeout, whatever the heartbeat's wall-clock time says. Called with r.mu held. +func (r *Registry) recentlyAlive(slotIdx int) bool { + last, seen := r.sightings[slotIdx] + return seen && !last.changedAt.IsZero() && time.Since(last.changedAt) < HeartbeatTimeout +} + +// provenDead reports whether a slot's node is dead, not merely slow: its mount +// process is gone, or this node has seen the same heartbeat for longer than +// HeartbeatTimeout plus deathMargin. Called with r.mu held. +func (r *Registry) provenDead(slotIdx int, slot *Slot) bool { + if r.deadLocalOwner(slot) { + return true + } + last, seen := r.sightings[slotIdx] + return seen && last.uuid == slot.UUID && time.Since(last.since) >= HeartbeatTimeout+deathMargin +} diff --git a/internal/node/registry.go b/internal/node/registry.go index 78ffe056..04d64f19 100644 --- a/internal/node/registry.go +++ b/internal/node/registry.go @@ -94,6 +94,10 @@ type Registry struct { pidNS uint64 onSlotLost func(slot int, owner [16]byte) + onOwnerDead func(owner [16]byte) (release func()) + deadPrevious *[16]byte + sightings map[int]sighting + // Cross-node recovery recoveryHandler RecoveryHandler failureCheckInterval time.Duration // How often to scan for failed nodes @@ -142,6 +146,7 @@ func NewRegistry(dev *storage.Device, registryOffset uint64) *Registry { stopCh: make(chan struct{}), failureCheckInterval: FailureCheckInterval, recoveryInProgress: make(map[int]bool), + sightings: make(map[int]sighting), bootID: readBootID(), pidNS: readPIDNamespace(), } @@ -168,6 +173,36 @@ func (r *Registry) SetSlotLostHandler(fn func(slot int, owner [16]byte)) { r.onSlotLost = fn } +// SetOwnerDeadHandler sets what failure detection calls, before replaying a +// proven-dead node's journal, with that node's UUID; for example to take over +// its locks. The returned function runs after the replay. +func (r *Registry) SetOwnerDeadHandler(fn func(owner [16]byte) (release func())) { + r.mu.Lock() + defer r.mu.Unlock() + r.onOwnerDead = fn +} + +// DeadPreviousOwner returns the UUID of the exited mount on this host whose +// slot Register took, if any. Its journal is this node's to replay at mount. +func (r *Registry) DeadPreviousOwner() ([16]byte, bool) { + r.mu.RLock() + defer r.mu.RUnlock() + if r.deadPrevious == nil { + return [16]byte{}, false + } + return *r.deadPrevious, true +} + +func (r *Registry) ownerDead(owner [16]byte) func() { + r.mu.Lock() + fn := r.onOwnerDead + r.mu.Unlock() + if fn == nil { + return func() {} + } + return fn(owner) +} + // SetJournalArea makes Register record a journal region in the claimed slot: // slot i owns slotSize bytes at offset + i*slotSize. func (r *Registry) SetJournalArea(offset, slotSize uint64) { @@ -250,6 +285,9 @@ func (r *Registry) claimFirst(slots []int, claimable func(*Slot) bool) (int, err return -1, err } r.localSlot = i + if r.deadLocalOwner(slot) { + r.deadPrevious = &slot.UUID + } return i, nil } return -1, nil @@ -370,6 +408,7 @@ func (r *Registry) heartbeatLoop(slotIdx int) { seq := uint64(1) lost := false + lastBeat := time.Now() for { select { @@ -378,23 +417,34 @@ func (r *Registry) heartbeatLoop(slotIdx int) { case <-ticker.C: r.mu.Lock() slot, err := r.readSlot(slotIdx) - takenOver := err == nil && slot.UUID != r.localUUID && !lost - if err == nil && slot.UUID == r.localUUID { + ours := err == nil && slot.UUID == r.localUUID && slot.State != NodeStateDead + takenOver := err == nil && !ours + if ours { slot.HeartbeatSeq = seq slot.HeartbeatTime = time.Now().Unix() - r.writeSlot(slotIdx, slot) + if r.writeSlot(slotIdx, slot) == nil { + lastBeat = time.Now() + } seq++ } onSlotLost := r.onSlotLost r.mu.Unlock() - if takenOver { + // Peers prove a silent node dead and take its locks; a node that cannot heartbeat must stop. + failing := time.Since(lastBeat) > HeartbeatTimeout/2 + if (takenOver || failing) && !lost { lost = true + owner := r.localUUID + if takenOver { + owner = slot.UUID + } else { + log.Printf("ERROR: heartbeats to node registry slot %d have failed for %v", slotIdx, time.Since(lastBeat)) + } if onSlotLost != nil { - onSlotLost(slotIdx, slot.UUID) + onSlotLost(slotIdx, owner) } else { - log.Printf("ERROR: node registry slot %d was taken over by node %x; "+ - "this node no longer owns its journal region", slotIdx, slot.UUID[:8]) + log.Printf("ERROR: node registry slot %d is no longer this node's (owner %x); "+ + "this node no longer owns its journal region", slotIdx, owner[:8]) } } } @@ -672,12 +722,14 @@ func (r *Registry) detectAndRecoverFailedNodes() { r.mu.Unlock() continue } + r.observe(i, slot) - // Stale heartbeats need 2x timeout for consensus - if !r.failed(slot, 2*HeartbeatTimeout) { + // Stale heartbeats need 2x timeout for consensus, and none this node saw advance lately. + if !r.failed(slot, 2*HeartbeatTimeout) || (r.recentlyAlive(i) && !r.deadLocalOwner(slot)) { r.mu.Unlock() continue } + dead := r.provenDead(i, slot) // Skip if already recovering if r.recoveryInProgress[i] { @@ -695,8 +747,13 @@ func (r *Registry) detectAndRecoverFailedNodes() { r.mu.Unlock() continue // Another node beat us to it } - - if err := handler(info); err != nil { + release := func() {} + if dead { + release = r.ownerDead(info.UUID) + } + err = handler(info) + release() + if err != nil { log.Printf("[Recovery] Replaying slot %d (node %x) failed: %v; any node retries after %v", i, info.UUID[:8], err, recoveryLease) r.mu.Lock() diff --git a/internal/node/registry_test.go b/internal/node/registry_test.go index 96e933a7..e0f4f4e4 100644 --- a/internal/node/registry_test.go +++ b/internal/node/registry_test.go @@ -3,6 +3,7 @@ package node import ( "context" "errors" + "fmt" "os" "os/exec" "path/filepath" @@ -1036,3 +1037,144 @@ func TestRegisterTakesOwnSlotThatGoesStaleWhileWaiting(t *testing.T) { t.Errorf("Register took slot %d, want slot 2, which went stale after the first scan", slot) } } + +func TestDetectionReleasesLocksOnlyOfProvenDeadNodes(t *testing.T) { + dev, cleanup := createTestDevice(t, 10<<20) + defer cleanup() + + reg := NewRegistry(dev, 0) + if reg.bootID == ([16]byte{}) || reg.pidNS == 0 { + t.Skip("no kernel boot ID or PID namespace") + } + if _, err := reg.Register(context.Background(), "node"); err != nil { + t.Fatal(err) + } + writeLocalSlot(t, reg, 2, exitedPID(t), 1) + slow := writeLocalSlotAt(t, reg, 3, os.Getppid(), time.Now().Add(-3*HeartbeatTimeout)) + + var events []string + reg.SetRecoveryHandler(func(info *FailedNodeInfo) error { + events = append(events, fmt.Sprintf("replay %d", info.SlotIndex)) + return nil + }) + reg.SetOwnerDeadHandler(func(owner [16]byte) func() { + events = append(events, fmt.Sprintf("adopt %x", owner[:1])) + return func() { events = append(events, fmt.Sprintf("release %x", owner[:1])) } + }) + reg.detectAndRecoverFailedNodes() + + want := []string{"adopt 03", "replay 2", "release 03", "replay 3"} + if fmt.Sprint(events) != fmt.Sprint(want) { + t.Errorf("events %v, want %v: only the exited node's locks are taken, and held through its replay", events, want) + } + if slow.UUID != ([16]byte{4}) { + t.Fatalf("slow node UUID %x", slow.UUID) + } +} + +func TestRegisterReportsItsDeadPreviousIncarnation(t *testing.T) { + dev, cleanup := createTestDevice(t, 10<<20) + defer cleanup() + + reg := NewRegistry(dev, 0) + if reg.bootID == ([16]byte{}) || reg.pidNS == 0 { + t.Skip("no kernel boot ID or PID namespace") + } + writeLocalSlot(t, reg, 0, exitedPID(t), 1) + if slot, err := reg.Register(context.Background(), "node"); err != nil || slot != 0 { + t.Fatalf("Register = %d, %v; want slot 0 of the exited mount", slot, err) + } + + if owner, ok := reg.DeadPreviousOwner(); !ok || owner != ([16]byte{1}) { + t.Errorf("DeadPreviousOwner = %x, %v; want the previous incarnation 01", owner, ok) + } +} + +func TestHeartbeatStopsWhenWritesFail(t *testing.T) { + dev, cleanup := createTestDevice(t, 10<<20) + defer cleanup() + + reg := NewRegistry(dev, 0) + lost := make(chan struct{}, 1) + reg.SetSlotLostHandler(func(int, [16]byte) { lost <- struct{}{} }) + if _, err := reg.Register(context.Background(), "node"); err != nil { + t.Fatal(err) + } + if err := reg.StartHeartbeat(); err != nil { + t.Fatal(err) + } + defer reg.Stop() + + dev.Close() + select { + case <-lost: + case <-time.After(HeartbeatTimeout): + t.Error("node kept running while its heartbeats failed; peers would prove it dead and take its locks") + } +} + +func TestHeartbeatStopsWhenSlotIsMarkedDead(t *testing.T) { + dev, cleanup := createTestDevice(t, 10<<20) + defer cleanup() + + reg := NewRegistry(dev, 0) + lost := make(chan struct{}, 1) + reg.SetSlotLostHandler(func(int, [16]byte) { lost <- struct{}{} }) + slot, err := reg.Register(context.Background(), "node") + if err != nil { + t.Fatal(err) + } + if err := reg.StartHeartbeat(); err != nil { + t.Fatal(err) + } + defer reg.Stop() + + own, err := reg.readSlot(slot) + if err != nil { + t.Fatal(err) + } + own.State = NodeStateDead + if err := reg.writeSlot(slot, own); err != nil { + t.Fatal(err) + } + select { + case <-lost: + case <-time.After(3 * HeartbeatInterval): + t.Error("heartbeat kept running in a slot another node marked dead") + } +} + +func TestOwnClockDecidesDeathNotWallClock(t *testing.T) { + reg := &Registry{sightings: make(map[int]sighting)} + slot := &Slot{UUID: [16]byte{4}, State: NodeStateActive, HeartbeatSeq: 1, + HeartbeatTime: time.Now().Add(-time.Hour).Unix()} + + reg.observe(1, slot) + if reg.provenDead(1, slot) { + t.Error("a heartbeat first seen just now counted as dead because its wall-clock time is old") + } + + reg.sightings[1] = sighting{uuid: slot.UUID, seq: 1, beat: slot.HeartbeatTime, + since: time.Now().Add(-(HeartbeatTimeout + deathMargin + time.Second))} + if !reg.provenDead(1, slot) { + t.Error("a heartbeat unchanged past timeout plus margin by this node's clock did not count as dead") + } + + slot.HeartbeatSeq++ + reg.observe(1, slot) + if reg.provenDead(1, slot) || !reg.recentlyAlive(1) { + t.Error("a heartbeat that advanced still counted as dead") + } +} + +func writeLocalSlotAt(t *testing.T, reg *Registry, idx int, pid int, heartbeat time.Time) *Slot { + t.Helper() + slot := &Slot{Magic: SlotMagic, State: NodeStateActive, UUID: [16]byte{byte(idx + 1)}, + HeartbeatTime: heartbeat.Unix(), BootID: reg.bootID, PID: uint32(pid), ProcStart: 1, + PIDNamespace: reg.pidNS} + copy(slot.Hostname[:], "node") + if err := reg.writeSlot(idx, slot); err != nil { + t.Fatal(err) + } + return slot +} diff --git a/tests/integration/lock_reclaim_test.go b/tests/integration/lock_reclaim_test.go new file mode 100644 index 00000000..d5c5c2e4 --- /dev/null +++ b/tests/integration/lock_reclaim_test.go @@ -0,0 +1,160 @@ +package integration + +import ( + "context" + "fmt" + "regexp" + "syscall" + "testing" + "time" + + "github.com/supporttools/GoClusterFS/internal/inode" + "github.com/supporttools/GoClusterFS/internal/lock" + "github.com/supporttools/GoClusterFS/internal/node" + "github.com/supporttools/GoClusterFS/internal/storage" + "github.com/supporttools/GoClusterFS/internal/superblock" +) + +// TestSurvivorReleasesCrashedNodeLocks crashes node A while it holds the root +// directory lock. Node B must prove A dead and take the lock well before the +// 15s staleness threshold, within a 10s mount timeout. +func TestSurvivorReleasesCrashedNodeLocks(t *testing.T) { + cluster, uuidA, logB := startPairForLockTest(t) + + if err := cluster.KillNode(0); err != nil { + t.Fatalf("kill node A: %v", err) + } + release := holdRootLockAs(t, cluster.ImagePath, uuidA) + defer release() + + start := time.Now() + for { + err := cluster.Node(1).WriteFile("after-crash", []byte("after-crash")) + if err == nil { + break + } + if time.Since(start) > 10*time.Second { + t.Fatalf("node B could not create a file within 10s of node A's crash: %v", err) + } + time.Sleep(200 * time.Millisecond) + } + if took := time.Since(start); took > 10*time.Second { + t.Errorf("node B needed %v to create a file after node A's crash, want under 10s", took) + } else { + t.Logf("node B created a file %v after node A crashed holding the root directory lock", took) + } + + tookOver := regexp.MustCompile(`took over \d+ lock records of dead node ` + fmt.Sprintf("%x", uuidA[:8])) + if !tookOver.MatchString(logB.String()) { + t.Error("node B did not log taking over node A's locks") + } +} + +// TestSlowNodeKeepsLocks pauses node A for 17s while it holds the root +// directory lock: longer than the heartbeat timeout and B's own-clock check +// (15s), shorter than the 20s after which failure detection treats a silent +// node as dead. B must not take the lock before it goes stale, and must never +// take over A's locks as a dead node's. +func TestSlowNodeKeepsLocks(t *testing.T) { + cluster, uuidA, logB := startPairForLockTest(t) + pid := cluster.Node(0).PID() + + if err := syscall.Kill(pid, syscall.SIGSTOP); err != nil { + t.Fatalf("pause node A: %v", err) + } + defer syscall.Kill(pid, syscall.SIGCONT) + release := holdRootLockAs(t, cluster.ImagePath, uuidA) + + attempt := func(result chan<- error) { + result <- cluster.Node(1).WriteFile("while-a-paused", []byte("x")) + } + result := make(chan error, 1) + go attempt(result) + paused := time.After(12 * time.Second) + resume := time.After(17 * time.Second) + for waiting := true; waiting; { + select { + case err := <-result: + if err == nil { + t.Fatal("node B took the root directory lock of a node that is paused but alive") + } + time.Sleep(500 * time.Millisecond) + go attempt(result) + case <-paused: + waiting = false + } + } + <-resume + if regexp.MustCompile(`lock records of dead node ` + fmt.Sprintf("%x", uuidA[:8])).MatchString(logB.String()) { + t.Error("node B took over the locks of a paused, live node") + } + + syscall.Kill(pid, syscall.SIGCONT) + release() + if err := cluster.Node(1).WriteFile("after-resume", []byte("x")); err != nil { + t.Errorf("node B could not create a file once node A resumed and released the lock: %v", err) + } +} + +func startPairForLockTest(t *testing.T) (*MultiNodeCluster, [16]byte, *syncBuffer) { + t.Helper() + cluster := NewMultiNodeCluster(t, 2, 50) + logA, logB := &syncBuffer{}, &syncBuffer{} + cluster.Node(0).Log, cluster.Node(1).Log = logA, logB + if err := cluster.FormatAndStartFirst(false); err != nil { + t.Fatalf("format and start node A: %v", err) + } + if err := cluster.StartNode(1, false); err != nil { + t.Fatalf("start node B: %v", err) + } + return cluster, nodeUUID(t, cluster.ImagePath, mustMatch(t, registeredRe, logA, "node A registration")), logB +} + +func nodeUUID(t *testing.T, imagePath, prefix string) [16]byte { + t.Helper() + dev, err := storage.Open(imagePath) + if err != nil { + t.Fatal(err) + } + defer dev.Close() + sb, err := superblock.Read(dev) + if err != nil { + t.Fatal(err) + } + slots, err := node.NewRegistry(dev, sb.NodeRegistryOffset).GetActiveNodes() + if err != nil { + t.Fatal(err) + } + for _, slot := range slots { + if fmt.Sprintf("%x", slot.UUID[:8]) == prefix { + return slot.UUID + } + } + t.Fatalf("no active registry slot for node %s", prefix) + return [16]byte{} +} + +// holdRootLockAs writes an exclusive root directory lock record owned by owner, as if owner held it. +func holdRootLockAs(t *testing.T, imagePath string, owner [16]byte) func() { + t.Helper() + dev, err := storage.Open(imagePath) + if err != nil { + t.Fatal(err) + } + sb, err := superblock.Read(dev) + if err != nil { + dev.Close() + t.Fatal(err) + } + locks := lock.NewManager(dev, sb.LockBitmapOffset, sb.LockBitmapSize, owner, 0) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + if err := locks.Acquire(ctx, lock.LockTypeInode, uint64(inode.InodeRoot), lock.LockModeExclusive); err != nil { + dev.Close() + t.Fatalf("hold root directory lock as %x: %v", owner[:8], err) + } + return func() { + locks.Release(lock.LockTypeInode, uint64(inode.InodeRoot)) + dev.Close() + } +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index eb9c88af..2d7404a6 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -447,6 +447,31 @@ scenarios: docker: run-tests.sh::test_restart_keeps_peer_journal vm: run-vm-tests.sh::test_restart_keeps_peer_journal + - id: survivor-releases-crashed-node-locks + name: Survivor releases a crashed node's locks + description: Node A crashes holding the root directory lock; node B proves A dead, takes over its locks for the replay and creates a file within 10s instead of waiting out the 15s lock staleness threshold + priority: high + environments: + go: lock_reclaim_test.go::TestSurvivorReleasesCrashedNodeLocks + docker: TODO # writes a lock record as the crashed node from the test process; Go-only harness + vm: TODO # same as docker + + - id: slow-node-keeps-locks + name: Paused but alive node keeps its locks + description: Node A is paused (SIGSTOP) for 17s holding the root directory lock; node B must not take the lock before it is stale, nor take over A's locks as a dead node's + priority: high + environments: + go: lock_reclaim_test.go::TestSlowNodeKeepsLocks + docker: TODO # needs SIGSTOP on a mount process and a lock record written as that node; Go-only harness + vm: TODO # same as docker + + - id: lock-adopt-owner + name: Replaying node takes over a dead node's exclusive locks + description: AdoptOwner holds the dead node's exclusive records through the replay and ReleaseAdopted frees them; shared records and other nodes' records stay + priority: go_only + environments: + go: internal/lock/reclaim_test.go::TestAdoptOwnerHoldsDeadNodesLocksUntilReleased + - id: registry-slot-journal-region name: Registry records each slot's journal region description: Register stores a distinct journal region in each slot