From b50381a1d4961d47eddcb3985ad3c5bd384254e5 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Fri, 13 Nov 2026 12:06:16 -0600 Subject: [PATCH 1/6] fix(lock): take over a lock only after its holder is released Other nodes ignored a claim once it was 15s old, so a paused or stalled holder lost its locks while it could still write, and a waiter got EIO after 30s even while the holder was failing over. A claim in another node's slot now counts until the registry slot with the same number reads empty. A wait for a failing holder has no time limit: it ends when the holder's replay does, and fails only once that replay's lease lapses. A goroutine that already holds a lock keeps the 30s limit, because the replay may need that lock; the replay's own waits have no limit. Failure detection replays only nodes proven dead and takes over their locks first. A failed replay keeps them held until one succeeds. A mount that takes a failed node's slot keeps that node's claims through its replay, from any host, and retries a transaction it cannot apply yet. Two replays sharing a record take turns. A node warns about each lock it has held for over a minute, since a hold that is never released now lasts until unmount. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 + cmd/mount/main.go | 51 +++- docs/OPERATIONS.md | 24 +- docs/TROUBLESHOOTING.md | 2 +- internal/goclusterfs/fs.go | 1 + internal/journal/cross_node_recovery.go | 5 +- internal/journal/recovery.go | 5 +- internal/journal/recovery_test.go | 37 +++ internal/lock/holds.go | 79 ++++++ internal/lock/holds_test.go | 37 +++ internal/lock/lock.go | 81 ++++-- internal/lock/owner.go | 86 +++++++ internal/lock/owner_test.go | 318 ++++++++++++++++++++++++ internal/lock/reclaim.go | 12 + internal/lock/reclaim_test.go | 19 ++ internal/lock/slots.go | 66 +++-- internal/node/liveness.go | 56 ++++- internal/node/registry.go | 48 ++-- internal/node/registry_test.go | 115 ++++++++- tests/integration/lock_reclaim_test.go | 48 ++++ tests/test-manifest.yaml | 123 ++++++++- 21 files changed, 1119 insertions(+), 99 deletions(-) create mode 100644 internal/lock/holds.go create mode 100644 internal/lock/holds_test.go create mode 100644 internal/lock/owner.go create mode 100644 internal/lock/owner_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index f724d574..ef12009b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,3 +85,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - The README quick start now uses the real binary names and flags. - Create, mkdir, clone and snapshot failed with "No space left on device" when the lock wait for the new inode gave up or its slot could not be read. They now fail with `EIO`. `ENOSPC` means the inode table is full. - Directory entries went only into the first 4 KB of each 1 MB directory block, so a block held about 250 short names or 15 long ones, and a busy directory took a new block every few creates. Entries now fill the whole block. A lookup reads each block in one I/O, and an add or remove writes only the 4 KB it changed. +- A node stalled for 15 seconds while it held a lock, for example a paused VM, lost the lock to other nodes, which could then write the same metadata. Other nodes now wait for a failed node's locks until it is proven dead and its journal is replayed, with no time limit, and give up only if the replay fails or is abandoned, or after 30 seconds if the waiting operation already holds another lock the replay may need. Failure detection now replays only nodes proven dead. +- A mount that took the registry slot of a failed node on another host did not keep that node's locks while it replayed the node's journal. +- Two nodes replaying dead nodes whose locks shared a record waited on each other until the lock timeout. +- A failed journal replay of a dead node still freed that node's locks, so other nodes could use half-replayed metadata until the retry a minute later. The locks now stay held until a replay succeeds. +- A mount whose own journal replay could not apply a transaction, for example while another node held a lock it needed, exited at once. It now retries with backoff; a journal it cannot read still fails the mount. diff --git a/cmd/mount/main.go b/cmd/mount/main.go index bc1f977b..5d2e5f0f 100644 --- a/cmd/mount/main.go +++ b/cmd/mount/main.go @@ -240,9 +240,14 @@ func main() { // Create lock manager for distributed locking lockMgr := lock.NewManager(dev, lockOffset, lockSize, nodeUUID, slotIdx) - nodeReg.SetOwnerDeadHandler(func(owner [16]byte, slot int) func() { return adoptLocks(lockMgr, owner, slot) }) - releasePrevious := func() {} - if owner, ok := nodeReg.DeadPreviousOwner(); ok { + lockMgr.SetOwnerView(func(slot int, fresh bool) ([16]byte, lock.OwnerState) { + occupant, occupancy := nodeReg.Occupant(slot, fresh) + return occupant, ownerStates[occupancy] + }) + lockMgr.Start() + nodeReg.SetOwnerDeadHandler(func(owner [16]byte, slot int) func(bool) { return adoptLocks(lockMgr, owner, slot) }) + releasePrevious := func(bool) {} + if owner, ok := nodeReg.PreviousOwner(); ok { releasePrevious = adoptLocks(lockMgr, owner, slotIdx) } @@ -281,7 +286,7 @@ func main() { recoveryHandler := journal.CreateTxnRecoveryHandler(recoveryCtx) // Replay a dirty journal left in this slot's region by its previous owner - result, err := journal.ScanAndRecover(dev, journalOffset, journalSize, recoveryHandler) + result, err := recoverOwnJournal(dev, journalOffset, journalSize, recoveryHandler, slotIdx) if err != nil { log.Fatalf("Journal recovery for slot %d failed: %v (to discard this journal, mount with -skip-recovery=%d)", slotIdx, err, slotIdx) } @@ -292,7 +297,7 @@ func main() { } else if skip { log.Printf("Journal: WARNING - discarding the journal of slot %d (-skip-recovery)", slotIdx) } - releasePrevious() + releasePrevious(true) // Create journal for write-ahead logging (crash consistency) var j *journal.Journal @@ -585,15 +590,45 @@ func statsSources(cacheMgr *cache.Manager, lockMgr *lock.Manager, j *journal.Jou return src } +var ownerStates = map[node.Occupancy]lock.OwnerState{ + node.OccupiedLive: lock.OwnerAlive, + node.OccupiedFailing: lock.OwnerFailing, + node.OccupiedLapsed: lock.OwnerLost, + node.Unoccupied: lock.OwnerReleased, +} + +// recoverOwnJournal replays the journal this node's slot holds from its +// previous owner. A transaction that cannot be applied yet, for example while a +// lock it needs is held, is retried with backoff; a journal that cannot be read +// fails at once. +func recoverOwnJournal(dev *storage.Device, offset, size uint64, handler journal.RecoveryHandler, slot int) (*journal.ScanRecoveryResult, error) { + delay := time.Second + for { + result, err := journal.ScanAndRecover(dev, offset, size, handler) + if !errors.Is(err, journal.ErrApplyFailed) { + return result, err + } + log.Printf("[WARN] Journal recovery for slot %d could not apply a transaction: %v; retrying in %v", slot, err, delay) + time.Sleep(delay) + delay = min(2*delay, 30*time.Second) + } +} + // adoptLocks takes over a dead node's exclusive locks and clears its read holds // before its journal is replayed, and returns the function that frees the -// exclusive locks after the replay. -func adoptLocks(locks *lock.Manager, owner [16]byte, slot int) func() { +// exclusive locks after a replay that succeeded. After a failed replay the locks +// stay held for the next attempt. +func adoptLocks(locks *lock.Manager, owner [16]byte, slot int) func(replayed bool) { adopted, err := locks.AdoptOwner(owner, slot) if err != nil { log.Printf("Warning: Failed to take over the locks of dead node %x: %v", owner[:8], err) } - return func() { + return func(replayed bool) { + if !replayed { + locks.ForgetAdopted(adopted) + log.Printf("[WARN] The journal replay of dead node %x failed; its %d lock records stay held until a replay succeeds", owner[:8], len(adopted)) + return + } if err := locks.ReleaseAdopted(adopted); err != nil { log.Printf("Warning: Failed to release the locks taken over from dead node %x: %v", owner[:8], err) } diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index f99d35f4..d14b4888 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -156,7 +156,7 @@ Snapshots and clones share blocks with the file and copy a block on its first wr - **Clocks:** Nodes judge each other's heartbeats by wall clock. A node that mounts with its clock more than about 10 seconds ahead treats live peers as failed, and can take a live peer's registry slot and replay that peer's journal. Run NTP on every node. - **Restarts:** A restarted mount takes back the registry slot of a mount that exited on the same host (same hostname and kernel boot ID) and replays its journal before serving. In the same PID namespace it checks the old process ID; across PID namespaces it waits for the old slot's heartbeat to go stale. -- **Crashed node's locks:** Once a node is proven dead, the node replaying its journal takes over its exclusive locks and frees them after the replay, so no other node uses them mid-replay. Proven dead means its mount process is gone (same host and PID namespace), or its heartbeat is more than 20 seconds old and has not changed for 15 seconds by the observing node's own clock. A node paused for more than about 20 seconds therefore counts as dead; it exits on its next heartbeat. The dead node's read locks are freed at once. The locks of a node not proven dead are freed once they are 15 seconds old. +- **Crashed node's locks:** Once a node is proven dead, the node replaying its journal takes over its exclusive locks and frees them after the replay, so no other node uses them mid-replay. Proven dead means its mount process is gone (same host and PID namespace), or its heartbeat is more than 20 seconds old and has not changed for 15 seconds by the observing node's own clock. A node paused for more than about 20 seconds therefore counts as dead; it exits on its next heartbeat. The dead node's read locks are freed at once. Until a failed node is replayed, other nodes wait for its locks however long that takes. Failover takes about 3 to 6 seconds when a surviving mount runs on the same host and in the same PID namespace, and up to about 35 seconds otherwise. A mount that takes a failed node's registry slot keeps that node's locks until it has replayed the slot's journal. - **Node count:** The device has 64 registry slots, each with its own 4 MB journal. A crashed node's slot frees once another node replays its journal, or when the same host mounts again. ### Known Gaps in Slot Ownership @@ -234,11 +234,11 @@ Nodes coordinate metadata writes (block bitmap, inodes, directories, refcounts, **How a node takes a lock.** Each of the 256 lock records has one 4 KB claim slot per registry slot, and a node writes only its own slot. To take a lock, a node writes its claim into its slot, then reads all 64 slots of the record in one read. It holds the lock if no other slot conflicts: for a write lock, any other claim; for a read lock, another node's write claim. Of two nodes that claim at the same time, the one whose write finished second reads the record after the other's write finished, so it sees that claim and does not take the lock. This holds however slow the writes are. It relies on the shared device showing a finished write to every later read, from any node. -**When claims collide.** The node in the lower registry slot keeps its claim and waits for the other claims to go. A node that sees a claim in a lower slot clears its own and waits, reading the record without writing until that claim is gone. Waits only point from lower to higher slots, so two nodes never wait on each other. A claim kept for a journal replay counts as lower than every slot, so the replaying node is never stuck behind a waiter. A slot that fails its checksum, such as one torn by a crash in the middle of a write, counts as a claim. +**When claims collide.** The node in the lower registry slot keeps its claim and waits for the other claims to go. A node that sees a claim in a lower slot clears its own and waits, reading the record without writing until that claim is gone. Waits only point from lower to higher slots, so two nodes never wait on each other. A claim kept for a journal replay counts as lower than every slot, so the replaying node is never stuck behind a waiter. When two nodes replay dead nodes whose claims share a record, the replay of the lower dead slot goes first. A slot that fails its checksum, such as one torn by a crash in the middle of a write, counts as a claim. -**How a lock goes stale.** A claim stores the wall-clock time its node wrote it. Other nodes ignore a claim older than 15 seconds (`StaleThreshold`), judged by their own clocks. This is how locks held by a crashed node get freed. A node writes its claim again when another operation on the same node joins a hold more than 5 seconds old, and before it reuses a kept lock that old; a single long operation is not renewed. A torn slot has no readable time. Each node treats it as a claim until it has read it torn for 15 seconds, or until its node is proven dead and the node that replays its journal clears it. +**When another node's claim stops counting.** A claim counts until the registry slot with the same number is empty: its node unmounted cleanly, or another node replayed its journal and freed the slot. A node checks this by reading the registry slot from the device after it reads the record, so it cannot miss a node that registered in the meantime. A torn slot follows the same rule. How old a claim is does not matter, so a lock that a node never releases, for example after a bug, stays held until that node unmounts. The node logs a warning for each lock it has held for more than 60 seconds. A replay that fails or gets stuck keeps the dead node's locks held until a later replay succeeds; waits for them fail with `EIO` once its recovery lease lapses, about a minute after the replay last made progress. A mount that cannot yet apply a transaction from its own slot's journal, for example because another node holds a lock it needs, retries every 1 to 30 seconds and logs each attempt. A journal it cannot read still fails the mount. -**The hazard: a stalled node keeps writing.** If a node stops for more than 15 seconds while it holds a lock, another node takes the lock. When the stalled node resumes, it finishes its writes. No write path checks that the lock is still held, and there is no fencing token. The stalled node is not told it lost the lock; it holds it in memory until it releases it. Two nodes then write the same metadata, which can double-allocate blocks or lose directory entries. +**The hazard: a stalled node keeps writing.** If a node stops for about 20 seconds while it holds a lock, other nodes prove it dead, replay its journal and take its locks. When the stalled node resumes, it exits on its next heartbeat, up to 1 second later, but it can finish the writes it had started. No write path checks that the lock is still held, and there is no fencing token. Two nodes then write the same metadata, which can double-allocate blocks or lose directory entries. Common causes of a stall: @@ -247,26 +247,24 @@ Common causes of a stall: - Heavy swapping or a long GC pause - Storage I/O that queues instead of failing, for example a multipath or iSCSI path failover -Lock slot reads and writes do not take the device's per-region I/O lock, so a node's own data and journal writes to the same region cannot delay them. The lock manager serializes each record's I/O itself. A release whose write fails leaves the claim to go stale. +Lock slot reads and writes do not take the device's per-region I/O lock, so a node's own data and journal writes to the same region cannot delay them. The lock manager serializes each record's I/O itself. A release whose write fails leaves the claim in place until the node unmounts or is replayed. -**Nodes keep released locks briefly.** After its last user releases a lock, a node keeps it for 10 ms (`CacheGrace`), so back-to-back work on the same node does not write and read the record again. A node reuses a lock it claimed for at most 250 ms (`HoldTurn`). After that, new work on that node waits up to another 250 ms for the current holders to finish. The node then releases the lock and waits 50 ms (`YieldGap`) before taking it again, so another node gets a turn. Another node that wants the lock waits up to about 35 ms while the holder is idle, and up to about 300 ms while the holder keeps using it. A clean unmount releases kept locks at once. A crashed node's last locks, kept or held, stay held until the other nodes detect it is dead: 3 to 6 seconds in the crash tests, and never longer than the 15-second stale limit. A lock wait gives up after 30 seconds (`DefaultTimeout`). +**Nodes keep released locks briefly.** After its last user releases a lock, a node keeps it for 10 ms (`CacheGrace`), so back-to-back work on the same node does not write and read the record again. A node reuses a lock it claimed for at most 250 ms (`HoldTurn`). After that, new work on that node waits up to another 250 ms for the current holders to finish. The node then releases the lock and waits 50 ms (`YieldGap`) before taking it again, so another node gets a turn. Another node that wants the lock waits up to about 35 ms while the holder is idle, and up to about 300 ms while the holder keeps using it. A clean unmount releases kept locks at once. A crashed node's last locks, kept or held, stay held until another node proves it dead and replays its journal (see [Node Requirements](#node-requirements)). A lock wait gives up after 30 seconds (`DefaultTimeout`) while the holder is alive. A wait for a failed holder has no time limit: it logs a warning that names the node, and it ends when the replay does. It fails with `EIO` only if the replay fails or is abandoned, which shows as a lapsed recovery lease, with one exception. An operation that already holds a lock and then waits for a failed node's lock keeps the 30-second limit and fails with `EIO`, because the replay may need the lock it holds. For example, a node holds a directory's lock and waits for a block lock the dead node held, while the dead node's journal changes that directory. The replay's own waits have no limit, so it finishes once the operation gives up. The warning for this case names the record and the dead node. **Writes to one region from two nodes do not mix within a block.** A node takes a block's lock for every write to an existing block, so two writes to the same block land one after the other. A write that spans several blocks takes each block's lock in turn, so if two nodes write overlapping multi-block ranges of a file at the same time, the result can mix both writes, one block at a time. Applications that write the same region from several nodes must coordinate those writes themselves. -**Read locks are not renewed.** A node whose overlapping lookups keep a directory read-locked for more than 15 seconds can lose that lock to another node's writer. Its readers may then see a half-written directory. Two writers still cannot overlap. - -**Locks of one type share records.** Each lock type has its own records of the 256: 128 for inodes, 97 for block extents, 16 for inode table sectors, 8 for SFB containers, and one for each other type. Two files' inode locks can map to one record, and a node holding one of them blocks other nodes from both. A lock never waits on a record another type uses. A node takes locks in type order and, within a type, in record order, so two nodes cannot each wait for a record the other holds. The exception is a replay of a failed node's journal: the replaying node keeps the failed node's records until the replay ends, so a node that holds a lock the replay needs and waits on one of those records waits until one side gives up after 30 seconds. +**Locks of one type share records.** Each lock type has its own records of the 256: 128 for inodes, 97 for block extents, 16 for inode table sectors, 8 for SFB containers, and one for each other type. Two files' inode locks can map to one record, and a node holding one of them blocks other nodes from both. A lock never waits on a record another type uses. A node takes locks in type order and, within a type, in record order, so two nodes cannot each wait for a record the other holds. The exception is a replay of a failed node's journal: the replaying node keeps the failed node's records until the replay ends, so a node that holds a lock the replay needs and waits on one of those records gives up after 30 seconds with `EIO`. The replay's waits have no limit, so it then finishes. **Safe operating limits:** | Limit | Why | |-------|-----| -| Keep node clocks within 1 second (chrony or NTP) | Clock skew S shortens the stale threshold to 15s − S | -| No node may stall for 10 seconds or more while mounted | Leaves margin below the 15s threshold | +| Keep node clocks within 1 second (chrony or NTP) | Nodes judge each other's heartbeats by wall clock | +| No node may stall for 15 seconds or more while mounted | Peers prove a node dead after about 20 seconds of silence and take its locks | | Storage I/O must fail, not queue, within 10 seconds | A queued write that lands after takeover corrupts metadata. Do not use multipath `queue_if_no_path`; lower iSCSI `node.session.timeo.replacement_timeout` (default 120s) | | Unmount before you pause, migrate or snapshot a node VM | Unmounting releases its locks | -If a node did stall for more than 15 seconds while mounted (hung-task warnings in `dmesg`, a VM pause, a path failover), stop its mount process before it does more writes, then check the data from a healthy node. +If a node did stall for 15 seconds or more while mounted (hung-task warnings in `dmesg`, a VM pause, a path failover), stop its mount process before it does more writes, then check the data from a healthy node. What is safe today: a node that crashes or is killed and stays down, and a clean unmount. Other nodes take over a crashed node's locks as described under [Node Requirements](#node-requirements). @@ -275,7 +273,7 @@ What is safe today: a node that crashes or is killed and stays down, and a clean When a node fails unexpectedly: 1. **Other nodes keep running.** Operations that need one of the failed node's locks wait until the lock is taken over. -2. **Locks and journal.** Once the failed node is proven dead, one surviving node takes over its exclusive locks, replays its journal, then frees the locks. Other locks are taken over once they are 15 seconds old. See [Node Requirements](#node-requirements). +2. **Locks and journal.** Once the failed node is proven dead, one surviving node takes over its exclusive locks, replays its journal, then frees the locks. Until then, operations that need its locks wait, however long that takes. See [Node Requirements](#node-requirements). 3. **No action is needed** on the surviving nodes. **When the failed node comes back**, mount it again with the same options. If it gets its old registry slot back (same hostname and boot ID), it replays its own journal before serving. Don't run fsck while other nodes have the device mounted: `-repair` refuses, and a check-only run reads metadata that is changing. diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 74b81eca..881397eb 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -151,7 +151,7 @@ grep "lock.Acquire" /var/log/goclusterfs.log | tail -20 ps aux | grep goclusterfs-mount ``` -A crashed node's locks free up on their own: other nodes take them over once they are 15 seconds old, or once the node is proven dead. A stalled node is dangerous; see [Distributed Lock Limits](OPERATIONS.md#distributed-lock-limits). +A crashed node's locks free up on their own once another node proves it dead and replays its journal: in about 3 to 6 seconds when a surviving mount shares its host and PID namespace, and up to about 35 seconds otherwise. Waits for them log `which holds it and is failing or being recovered`. A stalled node is dangerous; see [Distributed Lock Limits](OPERATIONS.md#distributed-lock-limits). ### "Journal recovery for slot N failed" diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index 8993c336..c7461631 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -315,6 +315,7 @@ func (f *FS) Close() error { } // Other nodes would otherwise wait for these holds to go stale. if f.lockMgr != nil { + f.lockMgr.Stop() f.lockMgr.ReleaseCached() f.lockMgr.LogStats() } diff --git a/internal/journal/cross_node_recovery.go b/internal/journal/cross_node_recovery.go index a18b5ebd..541fdc0c 100644 --- a/internal/journal/cross_node_recovery.go +++ b/internal/journal/cross_node_recovery.go @@ -16,6 +16,7 @@ import ( "github.com/supporttools/GoClusterFS/internal/dir" "github.com/supporttools/GoClusterFS/internal/extent" "github.com/supporttools/GoClusterFS/internal/inode" + "github.com/supporttools/GoClusterFS/internal/lock" "github.com/supporttools/GoClusterFS/internal/node" "github.com/supporttools/GoClusterFS/internal/storage" ) @@ -519,7 +520,7 @@ func updateExtents(ctx context.Context, rctx *RecoveryContext, r *txnReplay, ino // This is the function that should be called to wire up recovery in fs.go. func CreateRecoveryHandler(rctx *RecoveryContext) node.RecoveryHandler { return func(info *node.FailedNodeInfo) error { - ctx := context.Background() + ctx := lock.ForReplay(context.Background()) result, err := ReplayJournal(ctx, rctx, info) if err != nil { return err @@ -538,7 +539,7 @@ func CreateRecoveryHandler(rctx *RecoveryContext) node.RecoveryHandler { // This is used for self-recovery during mount when we need to replay // transactions directly without the node.FailedNodeInfo wrapper. func CreateTxnRecoveryHandler(rctx *RecoveryContext) RecoveryHandler { - ctx := context.Background() + ctx := lock.ForReplay(context.Background()) return func(txn *RecoveredTxn) error { result, err := applyTransaction(ctx, rctx, txn) if err != nil { diff --git a/internal/journal/recovery.go b/internal/journal/recovery.go index 91b1d56e..f5fa3497 100644 --- a/internal/journal/recovery.go +++ b/internal/journal/recovery.go @@ -12,6 +12,9 @@ var ( ErrRecoveryFailed = errors.New("journal recovery failed") ErrCorruptTxn = errors.New("corrupt transaction found") ErrUnexpectedEnd = errors.New("unexpected end of journal data") + // ErrApplyFailed wraps a failure to apply a readable transaction, which a + // later attempt can retry, as opposed to a journal that cannot be read. + ErrApplyFailed = errors.New("applying a transaction failed") ) // RecoveredTxn represents a transaction found during recovery. @@ -299,7 +302,7 @@ func scanJournal(region *Region, handler RecoveryHandler, stats *RecoveryStats, if handler != nil { if err := handler(recoveredTxn); err != nil { stats.Errors++ - return fmt.Errorf("handler failed for txn %d: %w", txnHeader.Seq, err) + return fmt.Errorf("%w: txn %d: %w", ErrApplyFailed, txnHeader.Seq, err) } } diff --git a/internal/journal/recovery_test.go b/internal/journal/recovery_test.go index fe19d05e..23ba647d 100644 --- a/internal/journal/recovery_test.go +++ b/internal/journal/recovery_test.go @@ -487,3 +487,40 @@ func damageJournalHeader(t *testing.T, dev *storage.Device, offset int64, damage } return bytes.Clone(header) } + +// A transaction that cannot be applied yet fails with ErrApplyFailed, which a +// mount retries; the retry resumes the journal and finishes it. +func TestScanAndRecoverRetriesFailedApply(t *testing.T) { + dev, cleanup := createTestDevice(t, 10*1024*1024) + defer cleanup() + j, err := Init(dev, 0, DefaultJournalSlotSize, [16]byte{1, 2, 3, 4}, 1, DefaultConfig()) + if err != nil { + t.Fatal(err) + } + txn := j.Begin() + if err := txn.LogInodeCreate(&InodeCreatePayload{InodeNum: 100}); err != nil { + t.Fatal(err) + } + if _, err := j.Commit(txn); err != nil { + t.Fatal(err) + } + + calls := 0 + handler := func(*RecoveredTxn) error { + calls++ + if calls == 1 { + return errors.New("lock held by a live node") + } + return nil + } + if _, err := ScanAndRecover(dev, 0, DefaultJournalSlotSize, handler); !errors.Is(err, ErrApplyFailed) { + t.Fatalf("first attempt: %v, want ErrApplyFailed", err) + } + result, err := ScanAndRecover(dev, 0, DefaultJournalSlotSize, handler) + if err != nil || result == nil || !result.WasNeeded || calls != 2 { + t.Fatalf("retry = %+v, %v after %d handler calls; want the transaction replayed", result, err, calls) + } + if result, err := ScanAndRecover(dev, 0, DefaultJournalSlotSize, handler); err != nil || result != nil { + t.Errorf("after the retry the journal still needs recovery: %+v, %v", result, err) + } +} diff --git a/internal/lock/holds.go b/internal/lock/holds.go new file mode 100644 index 00000000..3518f70f --- /dev/null +++ b/internal/lock/holds.go @@ -0,0 +1,79 @@ +package lock + +import ( + "fmt" + "log" + "maps" + "slices" + "strings" + "time" +) + +// longHold is how long a node holds a record before it warns that the hold may +// have leaked. Under an owner view a claim never goes stale, so a hold that is +// never released lasts until unmount. +var longHold = time.Minute + +// Start runs a watcher that logs a warning for each record this node has held +// for more than longHold. Stop ends it. +func (m *Manager) Start() { + m.mu.Lock() + defer m.mu.Unlock() + if m.stopWatch != nil { + return + } + m.stopWatch, m.watchDone = make(chan struct{}), make(chan struct{}) + go m.watchHolds(m.stopWatch, m.watchDone) +} + +// Stop ends the watcher and waits for it to finish. +func (m *Manager) Stop() { + m.mu.Lock() + stop, done := m.stopWatch, m.watchDone + m.stopWatch, m.watchDone = nil, nil + m.mu.Unlock() + if stop != nil { + close(stop) + <-done + } +} + +func (m *Manager) watchHolds(stop <-chan struct{}, done chan<- struct{}) { + defer close(done) + tick := time.NewTicker(longHold / 6) + defer tick.Stop() + for { + select { + case <-stop: + return + case <-tick.C: + m.warnLongHolds() + } + } +} + +// warnLongHolds logs each record held for more than longHold once, with the +// keys and goroutines that hold it. +func (m *Manager) warnLongHolds() { + m.mu.Lock() + var warnings []string + for recordID, rec := range m.records { + if rec.keys == 0 || rec.warnedLong || age(rec.since) <= longHold { + continue + } + rec.warnedLong = true + var holders []string + for key, held := range m.heldLocks { + if held.RecordID == recordID { + holders = append(holders, fmt.Sprintf("type=%d res=%d mode=%d goroutines %v", key.Type, key.ResourceID, held.Mode, slices.Sorted(maps.Keys(held.goroutines)))) + } + } + slices.Sort(holders) + warnings = append(warnings, fmt.Sprintf("[WARN] lock: this node has held record %d for %v (%s); other nodes wait for it until it is released", + recordID, age(rec.since).Round(time.Second), strings.Join(holders, "; "))) + } + m.mu.Unlock() + for _, w := range warnings { + log.Print(w) + } +} diff --git a/internal/lock/holds_test.go b/internal/lock/holds_test.go new file mode 100644 index 00000000..0f5aa7b9 --- /dev/null +++ b/internal/lock/holds_test.go @@ -0,0 +1,37 @@ +package lock + +import ( + "bytes" + "fmt" + "log" + "os" + "strings" + "testing" + "time" +) + +func TestLongHoldIsLoggedOnce(t *testing.T) { + old := longHold + t.Cleanup(func() { longHold = old }) + longHold = 120 * time.Millisecond + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + holder := newTestNode(dev, 0, 1<<20, 1) + + var logs bytes.Buffer + log.SetOutput(&logs) + t.Cleanup(func() { log.SetOutput(os.Stderr) }) + holder.Start() + requireAcquired(t, holder, LockTypeInode, 2, LockModeExclusive) + time.Sleep(5 * longHold) + holder.Stop() + log.SetOutput(os.Stderr) + + warning := fmt.Sprintf("held record %d for", holder.hashToRecord(LockTypeInode, 2)) + if got := strings.Count(logs.String(), warning); got != 1 { + t.Fatalf("%d warnings about the long hold, want 1:\n%s", got, logs.String()) + } + if !strings.Contains(logs.String(), "type=8 res=2 mode=2") { + t.Errorf("warning does not name the held key:\n%s", logs.String()) + } +} diff --git a/internal/lock/lock.go b/internal/lock/lock.go index 49b2a436..ebd80742 100644 --- a/internal/lock/lock.go +++ b/internal/lock/lock.go @@ -144,6 +144,8 @@ type Manager struct { claims map[uint64]*pendingClaim // adopted maps each record AdoptOwner kept for a dead node to that node's slot. adopted map[uint64]adoptedSlot + // ownerView, when set, says which node holds each registry slot. + ownerView OwnerView // tornSince is when this node first read each slot that fails its checksum // with the bytes it has now. tornMu sync.Mutex @@ -151,6 +153,8 @@ type Manager struct { tornCount atomic.Int32 // slotWrite, when a test sets it, replaces the device write of a slot. slotWrite func(buf []byte, offset int64) (int, error) + // stopWatch and watchDone run the long-hold watcher between Start and Stop. + stopWatch, watchDone chan struct{} // Callback for cache invalidation onAcquire OnAcquireFunc @@ -193,6 +197,8 @@ type heldRecord struct { renewed time.Time // when this node last wrote its slot // cacheGen identifies the current cached period; keys == 0 means cached. cacheGen uint64 + // warnedLong is set once this hold was logged as held too long. + warnedLong bool } // pendingClaim counts the local Acquire calls waiting for a record. When @@ -368,8 +374,11 @@ func (m *Manager) LogStats() { log.Printf("lock: claim times (us upper bound:count):%s max=%dus", times.String(), time.Duration(m.maxClaimTime.Load()).Microseconds()) } -// Acquire attempts to acquire a lock with the given parameters. -// Blocks until the lock is acquired, the context ends, or DefaultTimeout passes. +// Acquire attempts to acquire a lock with the given parameters. It blocks until +// the lock is acquired, the context ends, a live holder keeps it for +// DefaultTimeout, or the journal replay of a failed holder fails. A goroutine +// that already holds a lock also gives up after DefaultTimeout of waiting for a +// failed holder, since that holder's replay may need the lock it holds. // // Same-node reference counting: When multiple goroutines from the same node // need the same lock, the first goroutine acquires the distributed lock and @@ -379,6 +388,10 @@ func (m *Manager) LogStats() { // // Resources that hash to the same record share it: a node holding the record // for one resource excludes other nodes from every resource in that record. +// +// A goroutine must not hold a lock while it waits for another goroutine that +// needs the same record. Once the hold's turn is over, that goroutine cannot +// join it, and both wait until the timeout. func (m *Manager) Acquire(ctx context.Context, lockType LockType, resourceID uint64, mode LockMode) error { if int(lockType) >= numLockTypes { return fmt.Errorf("lock: unknown lock type %d", lockType) @@ -397,19 +410,25 @@ func (m *Manager) Acquire(ctx context.Context, lockType LockType, resourceID uin m.mu.Unlock() return err } + // A goroutine holding other locks may hold one a failed node's replay needs. + nested := m.goroutineHoldsAnyLocked(gid) m.addClaimWaiterLocked(recordID) m.mu.Unlock() defer m.dropClaimWaiter(recordID) + replay := isReplay(ctx) retryCount := 0 startTime := time.Now() + var liveWait time.Duration + var lastFailingLog time.Time // Retry loop with exponential backoff delay := RetryBaseDelay for { // Always attempt acquire first, then check context after failure // This ensures we try at least once even if context is nearly expired - held, added, err := m.attemptAcquire(key, recordID, mode, gid, time.Since(startTime)) + tryStart := time.Now() + held, added, by, err := m.attemptAcquire(key, recordID, mode, gid, time.Since(startTime)) if err != nil { log.Printf("[DEBUG] lock.Acquire: type=%d res=%d - attempt error: %v", lockType, resourceID, err) return err @@ -437,17 +456,39 @@ func (m *Manager) Acquire(ctx context.Context, lockType LockType, resourceID uin defer m.waiting.Add(-1) } - // Check context cancellation, but allow at least 3 retries. A caller whose - // context never ends still gives up after acquireTimeout. - if retryCount >= 3 && (ctx.Err() != nil || time.Since(startTime) > acquireTimeout) { - log.Printf("[WARN] lock: gave up waiting for type=%d res=%d mode=%d after %v (%d tries); record %d %s", - lockType, resourceID, mode, time.Since(startTime).Round(time.Millisecond), retryCount, recordID, m.describeHolder(recordID)) + switch by.state { + case OwnerLost: + log.Printf("[WARN] lock: gave up waiting for type=%d res=%d mode=%d after %v: the journal replay of node %x (slot %d), which holds it, failed or was abandoned", + lockType, resourceID, mode, time.Since(startTime).Round(time.Millisecond), by.occupant[:8], by.slot) + return ErrLockTimeout + case OwnerFailing: + if time.Since(lastFailingLog) >= acquireTimeout { + lastFailingLog = time.Now() + log.Printf("[WARN] lock: type=%d res=%d mode=%d has waited %v for node %x (slot %d), which holds it and is failing or being recovered; waiting until its journal replay ends", + lockType, resourceID, mode, time.Since(startTime).Round(time.Millisecond), by.occupant[:8], by.slot) + } + } + + // Check context cancellation, but allow at least 3 retries. A wait gives + // up after acquireTimeout of waiting for live holders, or for failing ones + // while this goroutine holds other locks; a replay's wait never does. + if retryCount >= 3 && (ctx.Err() != nil || !replay && liveWait > acquireTimeout) { + if by.state == OwnerFailing { + log.Printf("[WARN] lock: gave up waiting for type=%d res=%d mode=%d after %v: this goroutine holds other locks, and record %d is held by node %x (slot %d), which is failing or being recovered and whose replay may need one of them", + lockType, resourceID, mode, time.Since(startTime).Round(time.Millisecond), recordID, by.occupant[:8], by.slot) + } else { + log.Printf("[WARN] lock: gave up waiting for type=%d res=%d mode=%d after %v (%d tries); record %d %s", + lockType, resourceID, mode, time.Since(startTime).Round(time.Millisecond), retryCount, recordID, m.describeHolder(recordID)) + } return ErrLockTimeout } // Add jitter to prevent thundering herd jitter := time.Duration(rand.Int63n(int64(delay / 2))) time.Sleep(delay + jitter) + if by.state != OwnerFailing || nested { + liveWait += time.Since(tryStart) + } // Exponential backoff delay *= 2 @@ -466,7 +507,7 @@ func (m *Manager) Acquire(ctx context.Context, lockType LockType, resourceID uin // callers must not hold a local mutex across Acquire that those holders may // need. A request from a goroutine that holds other locks stops waiting for the // drain after another turn, so two goroutines cannot wait on each other's records. -func (m *Manager) attemptAcquire(key lockKey, recordID uint64, mode LockMode, gid uint64, waited time.Duration) (held, added bool, err error) { +func (m *Manager) attemptAcquire(key lockKey, recordID uint64, mode LockMode, gid uint64, waited time.Duration) (held, added bool, by blocker, err error) { gate := m.recordGate(recordID) gate.Lock() defer gate.Unlock() @@ -482,26 +523,26 @@ func (m *Manager) attemptAcquire(key lockKey, recordID uint64, mode LockMode, gi selfDeadlock := m.goroutineHoldsRecordLocked(recordID, gid) m.mu.Unlock() if selfDeadlock { - return false, false, fmt.Errorf("%w: cannot upgrade a shared lock to exclusive", ErrDeadlockRisk) + return false, false, blocker{}, fmt.Errorf("%w: cannot upgrade a shared lock to exclusive", ErrDeadlockRisk) } - return false, false, nil + return false, false, blocker{}, nil } if age(rec.since) > holdTurn && !m.goroutineHoldsRecordLocked(recordID, gid) && (waited < holdTurn || !m.goroutineHoldsAnyLocked(gid)) { if rec.keys > 0 { m.mu.Unlock() - return false, false, nil + return false, false, blocker{}, nil } delete(m.records, recordID) m.yieldUntil[recordID] = time.Now().Add(yieldGap) m.mu.Unlock() - return false, false, m.releaseRecord(recordID) + return false, false, blocker{}, m.releaseRecord(recordID) } renew := age(rec.renewed) > renewAfter m.mu.Unlock() if renew { if err := m.renewRecord(recordID, key, rec); err != nil { - return false, false, err + return false, false, blocker{}, err } } m.mu.Lock() @@ -510,7 +551,7 @@ func (m *Manager) attemptAcquire(key lockKey, recordID uint64, mode LockMode, gi held.goroutines[gid]++ m.mu.Unlock() m.countAcquire(key.Type, func(c *acquireCounts) *atomic.Uint64 { return &c.held }) - return true, false, nil + return true, false, blocker{}, nil } if rec.keys == 0 { m.countAcquire(key.Type, func(c *acquireCounts) *atomic.Uint64 { return &c.cached }) @@ -520,20 +561,20 @@ func (m *Manager) attemptAcquire(key lockKey, recordID uint64, mode LockMode, gi rec.keys++ m.addHeldKeyLocked(key, recordID, mode, gid) m.mu.Unlock() - return true, true, nil + return true, true, blocker{}, nil } if until, ok := m.yieldUntil[recordID]; ok { if time.Now().Before(until) { m.mu.Unlock() - return false, false, nil + return false, false, blocker{}, nil } delete(m.yieldUntil, recordID) } m.mu.Unlock() - written, won, err := m.claim(recordID, key, mode) + written, won, by, err := m.claim(recordID, key, mode) if err != nil || !won { - return false, false, err + return false, false, by, err } m.countAcquire(key.Type, func(c *acquireCounts) *atomic.Uint64 { return &c.claimed }) @@ -541,7 +582,7 @@ func (m *Manager) attemptAcquire(key lockKey, recordID uint64, mode LockMode, gi m.records[recordID] = &heldRecord{mode: mode, keys: 1, since: time.Now(), renewed: written} m.addHeldKeyLocked(key, recordID, mode, gid) m.mu.Unlock() - return true, true, nil + return true, true, blocker{}, nil } func (m *Manager) reuseHeldKeyLocked(key lockKey, mode LockMode, gid uint64) bool { diff --git a/internal/lock/owner.go b/internal/lock/owner.go new file mode 100644 index 00000000..32c041fe --- /dev/null +++ b/internal/lock/owner.go @@ -0,0 +1,86 @@ +package lock + +import "context" + +// OwnerState is what the node registry says about the node in a registry slot. +type OwnerState uint8 + +const ( + // OwnerAlive is a registered node with a current heartbeat. + OwnerAlive OwnerState = iota + // OwnerFailing is a node whose heartbeat stopped, or whose journal is being replayed. + OwnerFailing + // OwnerLost is a node whose journal replay failed or was abandoned. + OwnerLost + // OwnerReleased is an empty registry slot: its node left, or its replay finished. + OwnerReleased +) + +// OwnerView reports which node holds registry slot slot and its state. With +// fresh set it reads the registry from the device; otherwise it may answer from +// a recent read. +type OwnerView func(slot int, fresh bool) (occupant [16]byte, state OwnerState) + +// SetOwnerView makes a claim in another node's slot free only once that +// registry slot is empty, and lets a wait for a failing node's claim run +// without a time limit until the node's replay ends. Without a view, claims free +// once they are older than StaleThreshold. +func (m *Manager) SetOwnerView(view OwnerView) { + m.mu.Lock() + defer m.mu.Unlock() + m.ownerView = view +} + +// blocker is what kept an attempt from winning: the state of the node behind the +// claim in its way, and that node. The zero value is a live holder. +type blocker struct { + state OwnerState + occupant [16]byte + slot int +} + +// worse returns the blocker that limits a wait more: a lost owner ends it, a +// live one counts toward the timeout, and a failing one does not. +func worse(a, b blocker) blocker { + rank := func(s OwnerState) int { + switch s { + case OwnerLost: + return 2 + case OwnerAlive: + return 1 + } + return 0 + } + if rank(b.state) > rank(a.state) { + return b + } + return a +} + +// slotState judges a claim in slot, or a torn one (s nil), by the node now in +// the registry slot of the same index, whoever wrote the claim. The claim is +// free only once that registry slot reads empty from the device after the row. +// An adopted claim of a live node means a replay is running. +func (m *Manager) slotState(view OwnerView, slot int, s *claimSlot) blocker { + occupant, state := view(slot, false) + if state == OwnerReleased || state == OwnerLost { + occupant, state = view(slot, true) + } + if state == OwnerAlive && s != nil && s.adopted { + state = OwnerFailing + } + return blocker{state: state, occupant: occupant, slot: slot} +} + +type replayKey struct{} + +// ForReplay marks ctx as a journal replay's. Its lock waits have no time limit, +// so a peer that holds a lock the replay needs gives up first. +func ForReplay(ctx context.Context) context.Context { + return context.WithValue(ctx, replayKey{}, true) +} + +func isReplay(ctx context.Context) bool { + replay, _ := ctx.Value(replayKey{}).(bool) + return replay +} diff --git a/internal/lock/owner_test.go b/internal/lock/owner_test.go new file mode 100644 index 00000000..6945a0a7 --- /dev/null +++ b/internal/lock/owner_test.go @@ -0,0 +1,318 @@ +package lock + +import ( + "bytes" + "context" + "errors" + "fmt" + "log" + "os" + "strings" + "sync" + "testing" + "time" +) + +// fakeRegistry answers an OwnerView. An unset slot is empty. A slot in cached +// answers non-fresh reads differently, as a registry read before a change does. +type fakeRegistry struct { + mu sync.Mutex + slots map[int]OwnerState + cached map[int]OwnerState +} + +func newFakeRegistry() *fakeRegistry { + return &fakeRegistry{slots: map[int]OwnerState{}, cached: map[int]OwnerState{}} +} + +func (f *fakeRegistry) set(slot int, state OwnerState) { + f.mu.Lock() + defer f.mu.Unlock() + f.slots[slot] = state +} + +func (f *fakeRegistry) view(slot int, fresh bool) ([16]byte, OwnerState) { + f.mu.Lock() + defer f.mu.Unlock() + if state, ok := f.cached[slot]; ok && !fresh { + return [16]byte{byte(slot)}, state + } + if state, ok := f.slots[slot]; ok { + return [16]byte{byte(slot)}, state + } + return [16]byte{}, OwnerReleased +} + +func shortenAcquireTimeout(t *testing.T, d time.Duration) { + t.Helper() + old := acquireTimeout + t.Cleanup(func() { acquireTimeout = old }) + acquireTimeout = d +} + +// acquireInBackground starts an Acquire with no deadline and returns its result. +func acquireInBackground(m *Manager, lt LockType, id uint64) <-chan error { + result := make(chan error, 1) + go func() { result <- m.Acquire(context.Background(), lt, id, LockModeExclusive) }() + return result +} + +func requireWaiting(t *testing.T, result <-chan error, d time.Duration) { + t.Helper() + select { + case err := <-result: + t.Fatalf("Acquire returned %v while the holder's replay had not ended", err) + case <-time.After(d): + } +} + +func requireResult(t *testing.T, result <-chan error, want error) { + t.Helper() + select { + case err := <-result: + if !errors.Is(err, want) { + t.Fatalf("Acquire = %v, want %v", err, want) + } + case <-time.After(5 * time.Second): + t.Fatal("Acquire still waiting after 5s") + } +} + +func TestFailingHoldersOldClaimBlocksUntilRegistrySlotEmpties(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + holder := newTestNode(dev, 0, 1<<20, 1) + waiter := newTestNode(dev, 0, 1<<20, 2) + reg := newFakeRegistry() + reg.set(1, OwnerFailing) + waiter.SetOwnerView(reg.view) + + requireAcquired(t, holder, LockTypeInode, 2, LockModeExclusive) + ageSlot(t, holder, LockTypeInode, 2, pastStale) + requireBlocked(t, waiter, LockTypeInode, 2, LockModeExclusive) + + reg.set(1, OwnerReleased) + requireAcquired(t, waiter, LockTypeInode, 2, LockModeExclusive) +} + +func TestWaitForFailingHolderOutlastsTimeout(t *testing.T) { + shortenAcquireTimeout(t, 300*time.Millisecond) + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + holder := newTestNode(dev, 0, 1<<20, 1) + waiter := newTestNode(dev, 0, 1<<20, 2) + reg := newFakeRegistry() + reg.set(1, OwnerFailing) + waiter.SetOwnerView(reg.view) + + requireAcquired(t, holder, LockTypeInode, 2, LockModeExclusive) + result := acquireInBackground(waiter, LockTypeInode, 2) + requireWaiting(t, result, 3*acquireTimeout) + + reg.set(1, OwnerReleased) + requireResult(t, result, nil) +} + +func TestWaitForLiveHolderStillTimesOut(t *testing.T) { + shortenAcquireTimeout(t, 300*time.Millisecond) + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + holder := newTestNode(dev, 0, 1<<20, 1) + waiter := newTestNode(dev, 0, 1<<20, 2) + reg := newFakeRegistry() + reg.set(1, OwnerAlive) + waiter.SetOwnerView(reg.view) + + requireAcquired(t, holder, LockTypeInode, 2, LockModeExclusive) + requireResult(t, acquireInBackground(waiter, LockTypeInode, 2), ErrLockTimeout) +} + +func TestWaitEndsWhenHoldersReplayLapses(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + holder := newTestNode(dev, 0, 1<<20, 1) + waiter := newTestNode(dev, 0, 1<<20, 2) + reg := newFakeRegistry() + reg.set(1, OwnerFailing) + waiter.SetOwnerView(reg.view) + + requireAcquired(t, holder, LockTypeInode, 2, LockModeExclusive) + result := acquireInBackground(waiter, LockTypeInode, 2) + requireWaiting(t, result, 200*time.Millisecond) + + reg.set(1, OwnerLost) + requireResult(t, result, ErrLockTimeout) +} + +// A registry read from before a node registered can show its slot empty; only +// a read from the device after the row may free the claim. +func TestClaimFreesOnlyWhenFreshRegistryReadIsEmpty(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + holder := newTestNode(dev, 0, 1<<20, 1) + waiter := newTestNode(dev, 0, 1<<20, 2) + reg := newFakeRegistry() + reg.set(1, OwnerAlive) + reg.cached[1] = OwnerReleased + waiter.SetOwnerView(reg.view) + + requireAcquired(t, holder, LockTypeInode, 2, LockModeExclusive) + requireBlocked(t, waiter, LockTypeInode, 2, LockModeExclusive) +} + +func TestTornSlotFreesWhenItsRegistrySlotEmpties(t *testing.T) { + dev, cleanup := createTestDevice(t, 1<<20) + defer cleanup() + waiter := newTestNode(dev, 0, 1<<20, 2) + reg := newFakeRegistry() + reg.set(1, OwnerFailing) + waiter.SetOwnerView(reg.view) + + tearSlot(t, waiter, LockTypeInode, 2, 1) + requireBlocked(t, waiter, LockTypeInode, 2, LockModeExclusive) + + reg.set(1, OwnerReleased) + requireAcquired(t, waiter, LockTypeInode, 2, LockModeExclusive) +} + +// A node that took a dead node's registry slot keeps its predecessor's claims +// while it replays that journal at mount. Peers wait for the replay, however +// long, though the node itself is alive. +func TestPeersWaitOutLiveNodesReplayOfItsPredecessor(t *testing.T) { + shortenAcquireTimeout(t, 300*time.Millisecond) + dev, cleanup := createTestDevice(t, reclaimRegion) + defer cleanup() + predecessor := NewManager(dev, 0, reclaimRegion, [16]byte{1}, 1) + successor := NewManager(dev, 0, reclaimRegion, [16]byte{5}, 1) + waiter := NewManager(dev, 0, reclaimRegion, [16]byte{2}, 2) + reg := newFakeRegistry() + reg.set(1, OwnerAlive) + waiter.SetOwnerView(reg.view) + + requireAcquired(t, predecessor, LockTypeInode, 2, LockModeExclusive) + adopted, err := successor.AdoptOwner([16]byte{1}, 1) + if err != nil || len(adopted) != 1 { + t.Fatalf("AdoptOwner = %v, %v; want the predecessor's record", adopted, err) + } + result := acquireInBackground(waiter, LockTypeInode, 2) + requireWaiting(t, result, 3*acquireTimeout) + + if err := successor.ReleaseAdopted(adopted); err != nil { + t.Fatal(err) + } + requireResult(t, result, nil) +} + +// Two nodes each replaying a dead node whose claims share a record: the replay +// of the lower dead slot goes first instead of both waiting on each other. +func TestReplaysSharingRecordTakeTurns(t *testing.T) { + dev, cleanup := createTestDevice(t, reclaimRegion) + defer cleanup() + first := NewManager(dev, 0, reclaimRegion, [16]byte{3}, 3) + second := NewManager(dev, 0, reclaimRegion, [16]byte{4}, 4) + reg := newFakeRegistry() + reg.set(1, OwnerFailing) + reg.set(2, OwnerFailing) + reg.set(3, OwnerAlive) + reg.set(4, OwnerAlive) + first.SetOwnerView(reg.view) + second.SetOwnerView(reg.view) + + writeClaim(t, first, LockTypeInode, 2, 1, LockModeExclusive) + writeClaim(t, first, LockTypeInode, 2, 2, LockModeExclusive) + adoptedFirst, err := first.AdoptOwner([16]byte{1}, 1) + if err != nil || len(adoptedFirst) != 1 { + t.Fatalf("first AdoptOwner = %v, %v", adoptedFirst, err) + } + adoptedSecond, err := second.AdoptOwner([16]byte{2}, 2) + if err != nil || len(adoptedSecond) != 1 { + t.Fatalf("second AdoptOwner = %v, %v", adoptedSecond, err) + } + + requireBlocked(t, second, LockTypeInode, 2, LockModeExclusive) + requireAcquired(t, first, LockTypeInode, 2, LockModeExclusive) + if err := first.Release(LockTypeInode, 2); err != nil { + t.Fatal(err) + } + first.ReleaseCached() + if err := first.ReleaseAdopted(adoptedFirst); err != nil { + t.Fatal(err) + } + reg.set(1, OwnerReleased) + requireAcquired(t, second, LockTypeInode, 2, LockModeExclusive) +} + +// The dead node held extent row X, and its journal also changes inode Y. A peer +// holds Y and asks for X, a legal order, while the replay asks for Y. The peer +// gives up after the timeout and lets Y go, and the replay, which waits without +// a limit, takes Y and can finish. +func TestNestedWaitOnReplayingNodeYieldsToReplay(t *testing.T) { + shortenAcquireTimeout(t, 300*time.Millisecond) + dev, cleanup := createTestDevice(t, reclaimRegion) + defer cleanup() + dead := NewManager(dev, 0, reclaimRegion, [16]byte{1}, 1) + replayer := NewManager(dev, 0, reclaimRegion, [16]byte{3}, 3) + peer := NewManager(dev, 0, reclaimRegion, [16]byte{4}, 4) + reg := newFakeRegistry() + reg.set(1, OwnerFailing) + reg.set(3, OwnerAlive) + reg.set(4, OwnerAlive) + replayer.SetOwnerView(reg.view) + peer.SetOwnerView(reg.view) + const inodeY = 2 + extentX := uint64(1) + for peer.hashToRecord(LockTypeExtent, extentX) == peer.hashToRecord(LockTypeInode, inodeY) { + extentX++ + } + + requireAcquired(t, dead, LockTypeExtent, extentX, LockModeExclusive) + adopted, err := replayer.AdoptOwner([16]byte{1}, 1) + if err != nil || len(adopted) != 1 { + t.Fatalf("AdoptOwner = %v, %v", adopted, err) + } + + var logs bytes.Buffer + log.SetOutput(&logs) + t.Cleanup(func() { log.SetOutput(os.Stderr) }) + holdsY, askX := make(chan struct{}), make(chan struct{}) + peerResult := make(chan error, 1) + go func() { + if err := peer.Acquire(context.Background(), LockTypeInode, inodeY, LockModeExclusive); err != nil { + peerResult <- err + return + } + close(holdsY) + <-askX + err := peer.Acquire(context.Background(), LockTypeExtent, extentX, LockModeExclusive) + peer.Release(LockTypeInode, inodeY) + peer.ReleaseCached() + peerResult <- err + }() + <-holdsY + replayStart := time.Now() + replayResult := make(chan error, 1) + go func() { + replayResult <- replayer.Acquire(ForReplay(context.Background()), LockTypeInode, inodeY, LockModeExclusive) + }() + time.Sleep(acquireTimeout / 2) + close(askX) + + requireResult(t, peerResult, ErrLockTimeout) + requireResult(t, replayResult, nil) + if waited := time.Since(replayStart); waited <= acquireTimeout { + t.Errorf("replay waited %v, want longer than the %v timeout", waited, acquireTimeout) + } + log.SetOutput(os.Stderr) + if want := fmt.Sprintf("held by node %x (slot 1)", [8]byte{1}); !strings.Contains(logs.String(), want) { + t.Errorf("the peer's timeout does not name the dead node:\n%s", logs.String()) + } + + if err := replayer.Release(LockTypeInode, inodeY); err != nil { + t.Fatal(err) + } + if err := replayer.ReleaseAdopted(adopted); err != nil { + t.Fatal(err) + } + reg.set(1, OwnerReleased) + requireAcquired(t, peer, LockTypeExtent, extentX, LockModeExclusive) +} diff --git a/internal/lock/reclaim.go b/internal/lock/reclaim.go index 3a4d75da..918e9725 100644 --- a/internal/lock/reclaim.go +++ b/internal/lock/reclaim.go @@ -90,6 +90,18 @@ func (m *Manager) ReleaseAdopted(records []uint64) error { return nil } +// ForgetAdopted drops this node's own record of the adoptions in records and +// leaves their claims on the device, after a replay that failed. The dead +// node's locks then stay held until another replay adopts and releases them, +// and this node no longer passes over them. +func (m *Manager) ForgetAdopted(records []uint64) { + m.mu.Lock() + defer m.mu.Unlock() + for _, recordID := range records { + delete(m.adopted, recordID) + } +} + func (m *Manager) releaseAdopted(recordID uint64) error { gate := m.recordGate(recordID) gate.Lock() diff --git a/internal/lock/reclaim_test.go b/internal/lock/reclaim_test.go index 922a6236..da6750ce 100644 --- a/internal/lock/reclaim_test.go +++ b/internal/lock/reclaim_test.go @@ -278,3 +278,22 @@ func TestAdoptOwnerClearsDeadReader(t *testing.T) { } requireAcquired(t, writer, LockTypeInode, 9, LockModeExclusive) } + +// After a failed replay the dead node's claims stay on the device, so peers and +// the node that failed both wait for a later replay. +func TestForgetAdoptedKeepsDeadNodesClaims(t *testing.T) { + dev, cleanup := createTestDevice(t, reclaimRegion) + defer cleanup() + dead := NewManager(dev, 0, reclaimRegion, [16]byte{1}, 1) + replayer := NewManager(dev, 0, reclaimRegion, [16]byte{3}, 3) + peer := NewManager(dev, 0, reclaimRegion, [16]byte{4}, 4) + requireAcquired(t, dead, LockTypeInode, 2, LockModeExclusive) + adopted, err := replayer.AdoptOwner([16]byte{1}, 1) + if err != nil || len(adopted) != 1 { + t.Fatalf("AdoptOwner = %v, %v", adopted, err) + } + + replayer.ForgetAdopted(adopted) + requireBlocked(t, peer, LockTypeInode, 2, LockModeExclusive) + requireBlocked(t, replayer, LockTypeInode, 2, LockModeExclusive) +} diff --git a/internal/lock/slots.go b/internal/lock/slots.go index 4af131d3..525fa206 100644 --- a/internal/lock/slots.go +++ b/internal/lock/slots.go @@ -141,11 +141,20 @@ func (m *Manager) readRow(recordID uint64) ([]byte, error) { } // rowConflicts reports whether a slot other than this node's and ignore -// conflicts with a claim in mode, and whether one of those is in a lower slot -// or adopted, which this node must defer to. A write claim conflicts with every -// live slot, a read claim with live write slots. Slots older than StaleThreshold -// do not conflict, and torn slots conflict until they stay torn for tornExpiry. -func (m *Manager) rowConflicts(recordID uint64, row []byte, mode LockMode, ignore int) (conflict, lower bool) { +// conflicts with a claim in mode, whether one of those is in a lower slot or +// adopted, which this node must defer to, and what stands in the way. A write +// claim conflicts with every live slot, a read claim with live write slots, and +// a torn slot counts as a write claim. With an owner view a slot stays live +// until its registry slot is empty. Without one, it stays live until it is older +// than StaleThreshold, or for a torn slot, until it stays torn for tornExpiry. +// +// ignore is the slot of a dead node whose claim this node adopted for a replay. +// Another replay's adopted claim then counts as lower only in a lower slot and +// not at all in a higher one, so two replays sharing a record take turns. +func (m *Manager) rowConflicts(recordID uint64, row []byte, mode LockMode, ignore int) (conflict, lower bool, by blocker) { + m.mu.Lock() + view := m.ownerView + m.mu.Unlock() for slot := range SlotsPerRecord { if slot == m.nodeSlot || slot == ignore { continue @@ -155,18 +164,31 @@ func (m *Manager) rowConflicts(recordID uint64, row []byte, mode LockMode, ignor if err != nil { torn = rowSlot(row, slot) } - if m.tornTooLong(slotRef{recordID, slot}, torn) { + tornTooLong := m.tornTooLong(slotRef{recordID, slot}, torn) + if err == nil && (s == nil || (mode != LockModeExclusive && s.mode != LockModeExclusive)) { continue } - if err == nil && (s == nil || s.stale() || (mode != LockModeExclusive && s.mode != LockModeExclusive)) { + adopted := s != nil && s.adopted + if adopted && ignore >= 0 && slot > ignore { continue } - conflict = true - if slot < m.nodeSlot || (s != nil && s.adopted) { - return true, true + b := blocker{slot: slot} + if view == nil { + if tornTooLong || (s != nil && s.stale()) { + continue + } + } else if b = m.slotState(view, slot, s); b.state == OwnerReleased { + continue + } + if conflict { + b = worse(by, b) + } + conflict, by = true, b + if slot < m.nodeSlot || adopted { + lower = true } } - return conflict, false + return conflict, lower, by } // tornTooLong reports whether ref has read torn, with the same bytes, for @@ -260,7 +282,7 @@ func (m *Manager) dropClaimWaiter(recordID uint64) { // claim tries to win recordID in mode through this node's slot, as described // at the top of this file, and returns when the slot it relies on was written. // Callers hold the record's gate and have a waiter registered for it. -func (m *Manager) claim(recordID uint64, key lockKey, mode LockMode) (written time.Time, won bool, err error) { +func (m *Manager) claim(recordID uint64, key lockKey, mode LockMode) (written time.Time, won bool, by blocker, err error) { m.mu.Lock() c := m.claims[recordID] rewrite := c.written.IsZero() || c.mode != mode || age(c.written) > renewAfter @@ -274,11 +296,11 @@ func (m *Manager) claim(recordID uint64, key lockKey, mode LockMode) (written ti if behindLower { row, err := m.readRow(recordID) if err != nil { - return time.Time{}, false, err + return time.Time{}, false, blocker{}, err } - if _, lower := m.rowConflicts(recordID, row, mode, ignore); lower { + if _, lower, by := m.rowConflicts(recordID, row, mode, ignore); lower { m.deferred.Add(1) - return time.Time{}, false, nil + return time.Time{}, false, by, nil } m.mu.Lock() c.behindLower = false @@ -292,37 +314,37 @@ func (m *Manager) claim(recordID uint64, key lockKey, mode LockMode) (written ti c.written, c.mode = start, mode m.mu.Unlock() if err != nil { - return time.Time{}, false, err + return time.Time{}, false, blocker{}, err } } row, err := m.readRow(recordID) if err != nil { - return time.Time{}, false, err + return time.Time{}, false, blocker{}, err } if rewrite { m.recordClaimTime(time.Since(start)) } - conflict, lower := m.rowConflicts(recordID, row, mode, ignore) + conflict, lower, by := m.rowConflicts(recordID, row, mode, ignore) switch { case !conflict: m.mu.Lock() written = c.written c.written = time.Time{} m.mu.Unlock() - return written, true, nil + return written, true, blocker{}, nil case lower: m.conflicts.Add(1) if err := m.releaseRecord(recordID); err != nil { - return time.Time{}, false, err + return time.Time{}, false, by, err } m.mu.Lock() c.written, c.behindLower = time.Time{}, true m.mu.Unlock() - return time.Time{}, false, nil + return time.Time{}, false, by, nil default: m.waits.Add(1) - return time.Time{}, false, nil + return time.Time{}, false, by, nil } } diff --git a/internal/node/liveness.go b/internal/node/liveness.go index dccd8e29..cd2a8b2c 100644 --- a/internal/node/liveness.go +++ b/internal/node/liveness.go @@ -117,12 +117,62 @@ func (r *Registry) recentlyAlive(slotIdx int) bool { } // provenDead reports whether a slot's node is dead, not merely slow: its mount -// process is gone, or this node has seen the same heartbeat for longer than -// HeartbeatTimeout plus deathMargin. Called with r.mu held. +// process is gone, this node has seen the same heartbeat for longer than +// HeartbeatTimeout plus deathMargin, or the slot is marked dead, which a node +// does only after proving it or fsck after releasing it. Called with r.mu held. func (r *Registry) provenDead(slotIdx int, slot *Slot) bool { - if r.deadLocalOwner(slot) { + if slot.State == NodeStateDead || r.deadLocalOwner(slot) { return true } last, seen := r.sightings[slotIdx] return seen && last.uuid == slot.UUID && time.Since(last.since) >= HeartbeatTimeout+deathMargin } + +// Occupancy is how the node in a registry slot stands. +type Occupancy uint8 + +const ( + // OccupiedLive is a node with a current heartbeat. + OccupiedLive Occupancy = iota + // OccupiedFailing is a node whose heartbeat stopped, or whose journal is being replayed. + OccupiedFailing + // OccupiedLapsed is a dead node whose replay let its recovery lease lapse: the replay failed or was abandoned. + OccupiedLapsed + // Unoccupied is an empty slot. + Unoccupied +) + +// Occupant returns the node in registry slot slotIdx and how it stands. With +// fresh set, or before failure detection has read the slot, it reads the slot +// from the device; otherwise it uses failure detection's last read. A slot it +// cannot read counts as live. +func (r *Registry) Occupant(slotIdx int, fresh bool) ([16]byte, Occupancy) { + r.seenMu.Lock() + slot, ok := r.seen[slotIdx] + r.seenMu.Unlock() + if fresh || !ok { + read, err := r.readSlot(slotIdx) + if err != nil { + return [16]byte{}, OccupiedLive + } + r.remember(slotIdx, read) + slot = read + } + r.mu.RLock() + defer r.mu.RUnlock() + switch { + case slot.State == NodeStateEmpty: + return slot.UUID, Unoccupied + case slot.State == NodeStateDead && r.failed(slot, HeartbeatTimeout): + return slot.UUID, OccupiedLapsed + case slot.State == NodeStateDead || r.failed(slot, HeartbeatTimeout): + return slot.UUID, OccupiedFailing + } + return slot.UUID, OccupiedLive +} + +func (r *Registry) remember(slotIdx int, slot *Slot) { + r.seenMu.Lock() + defer r.seenMu.Unlock() + r.seen[slotIdx] = slot +} diff --git a/internal/node/registry.go b/internal/node/registry.go index 896ce63c..b1fced75 100644 --- a/internal/node/registry.go +++ b/internal/node/registry.go @@ -108,9 +108,13 @@ type Registry struct { pidNS uint64 onSlotLost func(slot int, owner [16]byte) - onOwnerDead func(owner [16]byte, slot int) (release func()) - deadPrevious *[16]byte - sightings map[int]sighting + onOwnerDead func(owner [16]byte, slot int) (done func(replayed bool)) + previousOwner *[16]byte + sightings map[int]sighting + + // seen holds the last read of each slot, for Occupant. + seenMu sync.Mutex + seen map[int]*Slot // Cross-node recovery recoveryHandler RecoveryHandler @@ -161,6 +165,7 @@ func NewRegistry(dev *storage.Device, registryOffset uint64) *Registry { failureCheckInterval: FailureCheckInterval, recoveryInProgress: make(map[int]bool), sightings: make(map[int]sighting), + seen: make(map[int]*Slot), bootID: readBootID(), pidNS: readPIDNamespace(), } @@ -189,30 +194,33 @@ func (r *Registry) SetSlotLostHandler(fn func(slot int, owner [16]byte)) { // SetOwnerDeadHandler sets what failure detection calls, before replaying a // proven-dead node's journal, with that node's UUID and registry slot; for -// example to take over its locks. The returned function runs after the replay. -func (r *Registry) SetOwnerDeadHandler(fn func(owner [16]byte, slot int) (release func())) { +// example to take over its locks. The returned function runs after the replay +// and is told whether it succeeded: after a failed replay, the dead node's locks +// must stay held until a later replay succeeds. +func (r *Registry) SetOwnerDeadHandler(fn func(owner [16]byte, slot int) (done func(replayed bool))) { r.mu.Lock() defer r.mu.Unlock() r.onOwnerDead = fn } -// DeadPreviousOwner returns the UUID of the exited mount on this host whose -// slot Register took, if any. Its journal is this node's to replay at mount. -func (r *Registry) DeadPreviousOwner() ([16]byte, bool) { +// PreviousOwner returns the UUID of the node whose slot Register took, if the +// slot was not empty. Its journal is this node's to replay at mount, and its +// lock claims are this node's to keep until then. +func (r *Registry) PreviousOwner() ([16]byte, bool) { r.mu.RLock() defer r.mu.RUnlock() - if r.deadPrevious == nil { + if r.previousOwner == nil { return [16]byte{}, false } - return *r.deadPrevious, true + return *r.previousOwner, true } -func (r *Registry) ownerDead(owner [16]byte, slot int) func() { +func (r *Registry) ownerDead(owner [16]byte, slot int) func(replayed bool) { r.mu.Lock() fn := r.onOwnerDead r.mu.Unlock() if fn == nil { - return func() {} + return func(bool) {} } return fn(owner, slot) } @@ -299,8 +307,8 @@ func (r *Registry) claimFirst(slots []int, claimable func(*Slot) bool) (int, err return -1, err } r.localSlot = i - if r.deadLocalOwner(slot) { - r.deadPrevious = &slot.UUID + if slot.State != NodeStateEmpty { + r.previousOwner = &slot.UUID } return i, nil } @@ -759,14 +767,15 @@ func (r *Registry) detectAndRecoverFailedNodes() { r.mu.Unlock() continue } + r.remember(i, slot) r.observe(i, slot) // Stale heartbeats need 2x timeout for consensus, and none this node saw advance lately. - if !r.failed(slot, 2*HeartbeatTimeout) || (r.recentlyAlive(i) && !r.deadLocalOwner(slot)) { + // Peers wait for a failed node's locks until its replay ends, so only a node proven dead, not one paused, is replayed. + if !r.failed(slot, 2*HeartbeatTimeout) || (r.recentlyAlive(i) && !r.deadLocalOwner(slot)) || !r.provenDead(i, slot) { r.mu.Unlock() continue } - dead := r.provenDead(i, slot) // Skip if already recovering if r.recoveryInProgress[i] { @@ -784,12 +793,9 @@ func (r *Registry) detectAndRecoverFailedNodes() { r.mu.Unlock() continue // Another node beat us to it } - release := func() {} - if dead { - release = r.ownerDead(info.UUID, i) - } + done := r.ownerDead(info.UUID, i) err = handler(info) - release() + done(err == nil) if err != nil { log.Printf("[Recovery] Replaying slot %d (node %x) failed: %v; any node retries after %v", i, info.UUID[:8], err, recoveryLease) diff --git a/internal/node/registry_test.go b/internal/node/registry_test.go index de6451be..27ffb9a0 100644 --- a/internal/node/registry_test.go +++ b/internal/node/registry_test.go @@ -628,6 +628,10 @@ func TestFailureDetectionLoop(t *testing.T) { if err != nil { t.Fatalf("writeSlot failed: %v", err) } + reg2.mu.Lock() + reg2.sightings[failingSlot] = sighting{uuid: staleSlot.UUID, seq: staleSlot.HeartbeatSeq, beat: staleSlot.HeartbeatTime, + since: time.Now().Add(-HeartbeatTimeout - deathMargin)} + reg2.mu.Unlock() // Set up recovery handler recovered := make(chan *FailedNodeInfo, 1) @@ -1053,7 +1057,7 @@ func TestRegisterTakesOwnSlotThatGoesStaleWhileWaiting(t *testing.T) { } } -func TestDetectionReleasesLocksOnlyOfProvenDeadNodes(t *testing.T) { +func TestDetectionReplaysOnlyProvenDeadNodes(t *testing.T) { dev, cleanup := createTestDevice(t, 10<<20) defer cleanup() @@ -1072,15 +1076,15 @@ func TestDetectionReleasesLocksOnlyOfProvenDeadNodes(t *testing.T) { events = append(events, fmt.Sprintf("replay %d", info.SlotIndex)) return nil }) - reg.SetOwnerDeadHandler(func(owner [16]byte, _ int) func() { + reg.SetOwnerDeadHandler(func(owner [16]byte, _ int) func(bool) { events = append(events, fmt.Sprintf("adopt %x", owner[:1])) - return func() { events = append(events, fmt.Sprintf("release %x", owner[:1])) } + return func(bool) { events = append(events, fmt.Sprintf("release %x", owner[:1])) } }) reg.detectAndRecoverFailedNodes() - want := []string{"adopt 03", "replay 2", "release 03", "replay 3"} + want := []string{"adopt 03", "replay 2", "release 03"} if fmt.Sprint(events) != fmt.Sprint(want) { - t.Errorf("events %v, want %v: only the exited node's locks are taken, and held through its replay", events, want) + t.Errorf("events %v, want %v: only the exited node is replayed, with its locks held through the replay", events, want) } if slow.UUID != ([16]byte{4}) { t.Fatalf("slow node UUID %x", slow.UUID) @@ -1100,8 +1104,8 @@ func TestRegisterReportsItsDeadPreviousIncarnation(t *testing.T) { t.Fatalf("Register = %d, %v; want slot 0 of the exited mount", slot, err) } - if owner, ok := reg.DeadPreviousOwner(); !ok || owner != ([16]byte{1}) { - t.Errorf("DeadPreviousOwner = %x, %v; want the previous incarnation 01", owner, ok) + if owner, ok := reg.PreviousOwner(); !ok || owner != ([16]byte{1}) { + t.Errorf("PreviousOwner = %x, %v; want the previous incarnation 01", owner, ok) } } @@ -1222,3 +1226,100 @@ func TestReleaseSlotGivesUpOnSlowReset(t *testing.T) { t.Errorf("slot after the abandoned release: %+v, %v; want dead under this registry's lease", slot, err) } } + +func TestOccupantReportsHowSlotStands(t *testing.T) { + dev, cleanup := createTestDevice(t, 10<<20) + defer cleanup() + reg := NewRegistry(dev, 0) + now := time.Now() + slots := []*Slot{ + {Magic: SlotMagic, State: NodeStateActive, UUID: [16]byte{1}, HeartbeatTime: now.Unix()}, + {Magic: SlotMagic, State: NodeStateActive, UUID: [16]byte{2}, HeartbeatTime: now.Add(-3 * HeartbeatTimeout).Unix()}, + {Magic: SlotMagic, State: NodeStateDead, UUID: [16]byte{3}, RecoveryStart: now.Unix()}, + {Magic: SlotMagic, State: NodeStateDead, UUID: [16]byte{4}, RecoveryStart: now.Add(-2 * recoveryLease).Unix()}, + {Magic: SlotMagic, State: NodeStateEmpty}, + } + want := []Occupancy{OccupiedLive, OccupiedFailing, OccupiedFailing, OccupiedLapsed, Unoccupied} + for i, slot := range slots { + if err := reg.writeSlot(i, slot); err != nil { + t.Fatal(err) + } + if uuid, got := reg.Occupant(i, true); got != want[i] || uuid != slot.UUID { + t.Errorf("slot %d: Occupant = %x, %d; want %x, %d", i, uuid[:1], got, slot.UUID[:1], want[i]) + } + } +} + +func TestOccupantAnswersFromLastReadUnlessFresh(t *testing.T) { + dev, cleanup := createTestDevice(t, 10<<20) + defer cleanup() + reg := NewRegistry(dev, 0) + if _, got := reg.Occupant(0, false); got != Unoccupied { + t.Fatalf("empty slot: Occupant = %d", got) + } + if err := reg.writeSlot(0, &Slot{Magic: SlotMagic, State: NodeStateActive, UUID: [16]byte{1}, HeartbeatTime: time.Now().Unix()}); err != nil { + t.Fatal(err) + } + if _, got := reg.Occupant(0, false); got != Unoccupied { + t.Errorf("cached read: Occupant = %d, want the earlier read's %d", got, Unoccupied) + } + if _, got := reg.Occupant(0, true); got != OccupiedLive { + t.Errorf("fresh read: Occupant = %d, want %d", got, OccupiedLive) + } +} + +func TestRegisterReportsPreviousOwnerOfAnyTakenSlot(t *testing.T) { + dev, cleanup := createTestDevice(t, 10<<20) + defer cleanup() + reg := NewRegistry(dev, 0) + gone := &Slot{Magic: SlotMagic, State: NodeStateActive, UUID: [16]byte{7}, HeartbeatTime: time.Now().Add(-3 * HeartbeatTimeout).Unix()} + copy(gone.Hostname[:], "elsewhere") + if err := reg.writeSlot(0, gone); err != nil { + t.Fatal(err) + } + if slot, err := reg.Register(context.Background(), "node"); err != nil || slot != 0 { + t.Fatalf("Register = %d, %v; want the failed slot 0", slot, err) + } + if owner, ok := reg.PreviousOwner(); !ok || owner != gone.UUID { + t.Errorf("PreviousOwner = %x, %v; want the failed node %x of another host", owner, ok, gone.UUID) + } + + fresh := NewRegistry(dev, 0) + if _, err := fresh.Register(context.Background(), "node2"); err != nil { + t.Fatal(err) + } + if owner, ok := fresh.PreviousOwner(); ok { + t.Errorf("PreviousOwner = %x after taking an empty slot, want none", owner) + } +} + +func TestFailedReplayKeepsDeadNodesLocks(t *testing.T) { + dev, cleanup := createTestDevice(t, 10<<20) + defer cleanup() + reg := NewRegistry(dev, 0) + if reg.bootID == ([16]byte{}) || reg.pidNS == 0 { + t.Skip("no kernel boot ID or PID namespace") + } + if _, err := reg.Register(context.Background(), "node"); err != nil { + t.Fatal(err) + } + writeLocalSlot(t, reg, 2, exitedPID(t), 1) + + var replayed []bool + reg.SetRecoveryHandler(func(*FailedNodeInfo) error { return errors.New("journal replay failed") }) + reg.SetOwnerDeadHandler(func([16]byte, int) func(bool) { + return func(ok bool) { replayed = append(replayed, ok) } + }) + reg.detectAndRecoverFailedNodes() + + if fmt.Sprint(replayed) != "[false]" { + t.Fatalf("lock handler told %v, want one failed replay", replayed) + } + slot, err := reg.readSlot(2) + if err != nil { + t.Fatal(err) + } + if slot.State != NodeStateDead || slot.UUID != ([16]byte{3}) { + t.Errorf("after a failed replay slot 2 is %s with node %x; want it still dead with the failed node", slot.State, slot.UUID[:1]) + } +} diff --git a/tests/integration/lock_reclaim_test.go b/tests/integration/lock_reclaim_test.go index 8229d82a..9250b9ac 100644 --- a/tests/integration/lock_reclaim_test.go +++ b/tests/integration/lock_reclaim_test.go @@ -99,6 +99,54 @@ func TestSlowNodeKeepsLocks(t *testing.T) { } } +// TestWaiterOutlastsLockTimeoutThroughFailover has node A hold the root +// directory lock while alive for 13s and then stop. Node B's create waits on +// that lock: for a live holder until A's heartbeat is 10s old, then for a +// failing one until A is proven dead and replayed. The wait runs past the 30s +// lock timeout, and B's create must still succeed. +func TestWaiterOutlastsLockTimeoutThroughFailover(t *testing.T) { + cluster, nodeA, logB := startPairForLockTest(t) + pid := cluster.Node(0).PID() + release := holdRootLockAs(t, cluster.ImagePath, nodeA) + defer release() + + start := time.Now() + result := make(chan error, 1) + go func() { result <- cluster.Node(1).WriteFile("through-failover", []byte("x")) }() + time.Sleep(13 * time.Second) + select { + case err := <-result: + t.Fatalf("node B's create returned (%v) while node A was alive and held the lock", err) + default: + } + if err := syscall.Kill(pid, syscall.SIGSTOP); err != nil { + t.Fatalf("stop node A: %v", err) + } + defer cluster.KillNode(0) + + select { + case err := <-result: + took := time.Since(start) + if err != nil { + t.Fatalf("node B's create failed after %v while node A failed over: %v", took, err) + } + if took < 30*time.Second { + t.Errorf("node B's create took %v; the test needs a wait longer than the 30s lock timeout", took) + } + t.Logf("node B's create succeeded after waiting %v", took) + case <-time.After(90 * time.Second): + t.Fatal("node B's create was still waiting 103s after it started") + } + + waited := fmt.Sprintf(`node %x \(slot %d\), which holds it and is failing or being recovered`, nodeA.uuid[:8], nodeA.slot) + if !regexp.MustCompile(waited).MatchString(logB.String()) { + t.Error("node B did not log that it waited for node A's recovery") + } + if !regexp.MustCompile(`took over \d+ lock records of dead node ` + fmt.Sprintf("%x", nodeA.uuid[:8])).MatchString(logB.String()) { + t.Error("node B did not take over node A's locks") + } +} + type registeredNode struct { uuid [16]byte slot int diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index bcfec3f6..c372cbc4 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -611,13 +611,22 @@ scenarios: - id: slow-node-keeps-locks name: Paused but alive node keeps its locks - description: Node A is paused (SIGSTOP) for 17s holding the root directory lock; node B must not take the lock before it is stale, nor take over A's locks as a dead node's + description: Node A is paused (SIGSTOP) for 17s holding the root directory lock; node B must not take the lock while A is paused, nor take over A's locks as a dead node's priority: high environments: go: lock_reclaim_test.go::TestSlowNodeKeepsLocks docker: TODO # needs SIGSTOP on a mount process and a lock record written as that node; Go-only harness vm: TODO # same as docker + - id: waiter-outlasts-lock-timeout-through-failover + name: Lock wait outlasts the timeout through a failover + description: Node A holds the root directory lock alive for 13s, then stops; node B's create waits past the 30s lock timeout until A is proven dead and replayed, and succeeds without EIO + priority: high + environments: + go: lock_reclaim_test.go::TestWaiterOutlastsLockTimeoutThroughFailover + docker: TODO # needs a lock record written as the stopped node at a known moment; Go-only harness + vm: TODO # same as docker + - id: lock-adopt-owner name: Replaying node takes over a dead node's exclusive locks description: AdoptOwner holds the dead node's exclusive records through the replay and ReleaseAdopted frees them; shared records and other nodes' records stay @@ -2802,6 +2811,118 @@ scenarios: environments: go: internal/goclusterfs/lock_commit_test.go::TestWriteWhenInodeAndBlockShareLockRecord + - id: lock-failing-holder-blocks-until-released + name: Failing holder's claim blocks until its registry slot empties + description: A claim older than the stale threshold keeps blocking while its node is failing, and frees once the node's registry slot is empty + priority: go_only + environments: + go: internal/lock/owner_test.go::TestFailingHoldersOldClaimBlocksUntilRegistrySlotEmpties + + - id: lock-wait-for-failing-holder-has-no-timeout + name: Wait for a failing holder outlasts the lock timeout + description: A wait for a failing holder's claim does not count toward the lock timeout and succeeds once the replay ends + priority: go_only + environments: + go: internal/lock/owner_test.go::TestWaitForFailingHolderOutlastsTimeout + + - id: lock-wait-for-live-holder-times-out + name: Wait for a live holder still times out + description: A wait for a live, heartbeating holder still gives up after the lock timeout + priority: go_only + environments: + go: internal/lock/owner_test.go::TestWaitForLiveHolderStillTimesOut + + - id: lock-wait-ends-when-replay-lapses + name: Lock wait ends when the holder's replay lapses + description: A wait for a failing holder gives up once the replay of its journal fails or is abandoned + priority: go_only + environments: + go: internal/lock/owner_test.go::TestWaitEndsWhenHoldersReplayLapses + + - id: lock-claim-frees-on-fresh-empty-registry-read + name: Claim frees only on a fresh empty registry read + description: A cached registry read showing the slot empty does not free a claim; only a read from the device after the row does + priority: go_only + environments: + go: internal/lock/owner_test.go::TestClaimFreesOnlyWhenFreshRegistryReadIsEmpty + + - id: lock-torn-slot-frees-with-registry-slot + name: Torn lock slot frees when its registry slot empties + description: A torn claim slot blocks while the node in its registry slot is failing and frees once that slot is empty + priority: go_only + environments: + go: internal/lock/owner_test.go::TestTornSlotFreesWhenItsRegistrySlotEmpties + + - id: lock-peers-wait-out-successor-replay + name: Peers wait out a successor's replay of its predecessor + description: A live node that took a dead node's registry slot keeps its predecessor's claims while it replays that journal, and peers wait without a timeout + priority: go_only + environments: + go: internal/lock/owner_test.go::TestPeersWaitOutLiveNodesReplayOfItsPredecessor + + - id: lock-replays-sharing-record-take-turns + name: Replays sharing a lock record take turns + description: Two nodes replaying dead nodes whose claims share a record do not wait on each other; the replay of the lower dead slot goes first + priority: go_only + environments: + go: internal/lock/owner_test.go::TestReplaysSharingRecordTakeTurns + + - id: lock-nested-wait-yields-to-replay + name: Nested wait on a replaying node yields to the replay + description: A peer that holds inode Y and waits for a dead node's adopted extent row gives up after the timeout, and the replay, which waits without a limit, takes Y and finishes + priority: go_only + environments: + go: internal/lock/owner_test.go::TestNestedWaitOnReplayingNodeYieldsToReplay + + - id: lock-long-hold-logged-once + name: Long lock hold is logged once + description: A node that holds a lock longer than a minute logs one warning naming the record and the key that holds it, since a claim never goes stale and a leaked hold lasts until unmount + priority: go_only + environments: + go: internal/lock/holds_test.go::TestLongHoldIsLoggedOnce + + - id: lock-forget-adopted-keeps-claims + name: A failed replay keeps the dead node's lock claims + description: After a failed replay the replaying node forgets its adoptions but leaves the claims on the device, so peers and the failed replayer wait for a later replay + priority: go_only + environments: + go: internal/lock/reclaim_test.go::TestForgetAdoptedKeepsDeadNodesClaims + + - id: registry-failed-replay-keeps-locks + name: Failure detection keeps a dead node's locks after a failed replay + description: When the replay of a dead node's journal fails, the lock handler is told so and the registry slot stays dead with the failed node + priority: go_only + environments: + go: internal/node/registry_test.go::TestFailedReplayKeepsDeadNodesLocks + + - id: journal-retries-failed-apply + name: Journal recovery marks a failed apply as retryable + description: A transaction that cannot be applied fails ScanAndRecover with ErrApplyFailed, and the next attempt resumes the journal and finishes it + priority: go_only + environments: + go: internal/journal/recovery_test.go::TestScanAndRecoverRetriesFailedApply + + - id: registry-occupant-states + name: Registry reports how each slot's node stands + description: Occupant reports a live node, a failing or dead one under replay, a dead one whose replay lapsed, and an empty slot, and answers from the last read unless asked for a fresh one + priority: go_only + environments: + go: internal/node/registry_test.go::TestOccupantReportsHowSlotStands + + - id: registry-occupant-cache + name: Registry answers occupancy from its last read unless fresh + description: Occupant answers from failure detection's last read of a slot, and reads the device when asked for a fresh answer + priority: go_only + environments: + go: internal/node/registry_test.go::TestOccupantAnswersFromLastReadUnlessFresh + + - id: registry-previous-owner-of-taken-slot + name: Register reports the previous owner of any taken slot + description: Register reports the node whose failed slot it took, from any host, so mount keeps that node's lock claims through the replay + priority: go_only + environments: + go: internal/node/registry_test.go::TestRegisterReportsPreviousOwnerOfAnyTakenSlot + - id: lock-leaked-shared-count-goes-stale name: Leaked read claim goes stale description: New readers do not refresh another node's read claim, so a read claim leaked by a stopped node stops blocking writers once it is 15 seconds old From bcc94eea9d2af004fd51d967a86ae5fdb45c9c1e Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Fri, 13 Nov 2026 12:11:54 -0600 Subject: [PATCH 2/6] fix(mount): retry the own-journal replay only while another node holds a lock The mount retried any failure to apply its own journal, so a record that does not decode or an I/O error retried forever instead of failing with the -skip-recovery hint. It now retries only a replay that gave up waiting for another node's lock, and a lock timeout names the holder, so each retry logs who holds the lock. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- cmd/mount/main.go | 23 ++++--- cmd/mount/recovery_test.go | 110 ++++++++++++++++++++++++++++++ docs/OPERATIONS.md | 2 +- internal/journal/recovery.go | 5 +- internal/journal/recovery_test.go | 37 ---------- internal/lock/lock.go | 35 +++++----- tests/test-manifest.yaml | 15 ++-- 8 files changed, 158 insertions(+), 71 deletions(-) create mode 100644 cmd/mount/recovery_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index ef12009b..9d525cd7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -89,4 +89,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - A mount that took the registry slot of a failed node on another host did not keep that node's locks while it replayed the node's journal. - Two nodes replaying dead nodes whose locks shared a record waited on each other until the lock timeout. - A failed journal replay of a dead node still freed that node's locks, so other nodes could use half-replayed metadata until the retry a minute later. The locks now stay held until a replay succeeds. -- A mount whose own journal replay could not apply a transaction, for example while another node held a lock it needed, exited at once. It now retries with backoff; a journal it cannot read still fails the mount. +- A mount whose own journal replay gave up waiting for another node's lock exited at once. It now retries with backoff and logs the holder; any other replay failure still fails the mount. diff --git a/cmd/mount/main.go b/cmd/mount/main.go index 5d2e5f0f..52147e28 100644 --- a/cmd/mount/main.go +++ b/cmd/mount/main.go @@ -288,7 +288,7 @@ func main() { // Replay a dirty journal left in this slot's region by its previous owner result, err := recoverOwnJournal(dev, journalOffset, journalSize, recoveryHandler, slotIdx) if err != nil { - log.Fatalf("Journal recovery for slot %d failed: %v (to discard this journal, mount with -skip-recovery=%d)", slotIdx, err, slotIdx) + log.Fatal(err) } if result != nil && result.WasNeeded { log.Printf("Journal recovery: replayed %d committed transactions (%d bytes scanned) from crashed node %x", @@ -597,18 +597,25 @@ var ownerStates = map[node.Occupancy]lock.OwnerState{ node.Unoccupied: lock.OwnerReleased, } +// ownReplayRetry is the first wait before a mount retries its own journal +// replay; tests shorten it. +var ownReplayRetry = time.Second + // recoverOwnJournal replays the journal this node's slot holds from its -// previous owner. A transaction that cannot be applied yet, for example while a -// lock it needs is held, is retried with backoff; a journal that cannot be read -// fails at once. +// previous owner. A replay that gave up waiting for another node's lock is +// retried with backoff. Any other failure, such as a record that does not +// decode or an I/O error, cannot get better by waiting, so it stops the mount. func recoverOwnJournal(dev *storage.Device, offset, size uint64, handler journal.RecoveryHandler, slot int) (*journal.ScanRecoveryResult, error) { - delay := time.Second + delay := ownReplayRetry for { result, err := journal.ScanAndRecover(dev, offset, size, handler) - if !errors.Is(err, journal.ErrApplyFailed) { - return result, err + if err == nil { + return result, nil + } + if !errors.Is(err, lock.ErrLockTimeout) { + return nil, fmt.Errorf("journal recovery for slot %d failed: %w (to discard this journal, mount with -skip-recovery=%d)", slot, err, slot) } - log.Printf("[WARN] Journal recovery for slot %d could not apply a transaction: %v; retrying in %v", slot, err, delay) + log.Printf("[WARN] Journal recovery for slot %d is waiting for another node's lock: %v; retrying in %v", slot, err, delay) time.Sleep(delay) delay = min(2*delay, 30*time.Second) } diff --git a/cmd/mount/recovery_test.go b/cmd/mount/recovery_test.go new file mode 100644 index 00000000..6a360386 --- /dev/null +++ b/cmd/mount/recovery_test.go @@ -0,0 +1,110 @@ +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/supporttools/GoClusterFS/internal/journal" + "github.com/supporttools/GoClusterFS/internal/lock" + "github.com/supporttools/GoClusterFS/internal/storage" +) + +// dirtyJournal returns a device holding a journal with one committed, +// unreplayed transaction, as a crashed node leaves its slot. +func dirtyJournal(t *testing.T) *storage.Device { + t.Helper() + dir := os.Getenv("GOCLUSTERFS_TEST_DIR") + if dir == "" { + dir = t.TempDir() + } + path := filepath.Join(dir, fmt.Sprintf("own-journal-%d.img", time.Now().UnixNano())) + if err := os.WriteFile(path, nil, 0o600); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.Remove(path) }) + if err := os.Truncate(path, int64(journal.DefaultJournalSlotSize)); err != nil { + t.Fatal(err) + } + dev, err := storage.Open(path) + if err != nil { + t.Skipf("O_DIRECT not supported: %v", err) + } + t.Cleanup(func() { dev.Close() }) + j, err := journal.Init(dev, 0, journal.DefaultJournalSlotSize, [16]byte{1}, 1, journal.DefaultConfig()) + if err != nil { + t.Fatal(err) + } + txn := j.Begin() + if err := txn.LogInodeCreate(&journal.InodeCreatePayload{InodeNum: 42}); err != nil { + t.Fatal(err) + } + if _, err := j.Commit(txn); err != nil { + t.Fatal(err) + } + return dev +} + +// recoverWithin runs recoverOwnJournal and fails the test if it is still +// running after d. +func recoverWithin(t *testing.T, d time.Duration, dev *storage.Device, handler journal.RecoveryHandler) (*journal.ScanRecoveryResult, error) { + t.Helper() + type outcome struct { + result *journal.ScanRecoveryResult + err error + } + done := make(chan outcome, 1) + go func() { + result, err := recoverOwnJournal(dev, 0, journal.DefaultJournalSlotSize, handler, 5) + done <- outcome{result, err} + }() + select { + case o := <-done: + return o.result, o.err + case <-time.After(d): + t.Fatalf("recoverOwnJournal still running after %v", d) + return nil, nil + } +} + +func TestOwnReplayRetriesWhileAnotherNodeHoldsALock(t *testing.T) { + defer func(d time.Duration) { ownReplayRetry = d }(ownReplayRetry) + ownReplayRetry = 10 * time.Millisecond + dev := dirtyJournal(t) + + calls := 0 + handler := func(*journal.RecoveredTxn) error { + calls++ + if calls == 1 { + return fmt.Errorf("failed to acquire inode lock: %w", lock.ErrLockTimeout) + } + return nil + } + result, err := recoverWithin(t, 5*time.Second, dev, handler) + if err != nil || result == nil || !result.WasNeeded || calls != 2 { + t.Fatalf("recoverOwnJournal = %+v, %v after %d tries; want the transaction replayed on the second", result, err, calls) + } +} + +func TestOwnReplayStopsAtARecordThatDoesNotDecode(t *testing.T) { + defer func(d time.Duration) { ownReplayRetry = d }(ownReplayRetry) + ownReplayRetry = time.Minute + dev := dirtyJournal(t) + + calls := 0 + handler := func(*journal.RecoveredTxn) error { + calls++ + return errors.New("failed to apply entry op=InodeCreate: inode create payload too short") + } + _, err := recoverWithin(t, 5*time.Second, dev, handler) + if err == nil || calls != 1 { + t.Fatalf("recoverOwnJournal = %v after %d tries; want one failed try", err, calls) + } + if !strings.Contains(err.Error(), "-skip-recovery=5") { + t.Errorf("error %q does not tell the operator how to discard the journal", err) + } +} diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index d14b4888..176334a1 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -236,7 +236,7 @@ Nodes coordinate metadata writes (block bitmap, inodes, directories, refcounts, **When claims collide.** The node in the lower registry slot keeps its claim and waits for the other claims to go. A node that sees a claim in a lower slot clears its own and waits, reading the record without writing until that claim is gone. Waits only point from lower to higher slots, so two nodes never wait on each other. A claim kept for a journal replay counts as lower than every slot, so the replaying node is never stuck behind a waiter. When two nodes replay dead nodes whose claims share a record, the replay of the lower dead slot goes first. A slot that fails its checksum, such as one torn by a crash in the middle of a write, counts as a claim. -**When another node's claim stops counting.** A claim counts until the registry slot with the same number is empty: its node unmounted cleanly, or another node replayed its journal and freed the slot. A node checks this by reading the registry slot from the device after it reads the record, so it cannot miss a node that registered in the meantime. A torn slot follows the same rule. How old a claim is does not matter, so a lock that a node never releases, for example after a bug, stays held until that node unmounts. The node logs a warning for each lock it has held for more than 60 seconds. A replay that fails or gets stuck keeps the dead node's locks held until a later replay succeeds; waits for them fail with `EIO` once its recovery lease lapses, about a minute after the replay last made progress. A mount that cannot yet apply a transaction from its own slot's journal, for example because another node holds a lock it needs, retries every 1 to 30 seconds and logs each attempt. A journal it cannot read still fails the mount. +**When another node's claim stops counting.** A claim counts until the registry slot with the same number is empty: its node unmounted cleanly, or another node replayed its journal and freed the slot. A node checks this by reading the registry slot from the device after it reads the record, so it cannot miss a node that registered in the meantime. A torn slot follows the same rule. How old a claim is does not matter, so a lock that a node never releases, for example after a bug, stays held until that node unmounts. The node logs a warning for each lock it has held for more than 60 seconds. A replay that fails or gets stuck keeps the dead node's locks held until a later replay succeeds; waits for them fail with `EIO` once its recovery lease lapses, about a minute after the replay last made progress. A mount whose own journal replay gives up waiting for another node's lock retries every 1 to 30 seconds and logs who holds the lock each time. Any other replay failure, such as a record that does not decode or an I/O error, fails the mount at once with the `-skip-recovery` hint. **The hazard: a stalled node keeps writing.** If a node stops for about 20 seconds while it holds a lock, other nodes prove it dead, replay its journal and take its locks. When the stalled node resumes, it exits on its next heartbeat, up to 1 second later, but it can finish the writes it had started. No write path checks that the lock is still held, and there is no fencing token. Two nodes then write the same metadata, which can double-allocate blocks or lose directory entries. diff --git a/internal/journal/recovery.go b/internal/journal/recovery.go index f5fa3497..91b1d56e 100644 --- a/internal/journal/recovery.go +++ b/internal/journal/recovery.go @@ -12,9 +12,6 @@ var ( ErrRecoveryFailed = errors.New("journal recovery failed") ErrCorruptTxn = errors.New("corrupt transaction found") ErrUnexpectedEnd = errors.New("unexpected end of journal data") - // ErrApplyFailed wraps a failure to apply a readable transaction, which a - // later attempt can retry, as opposed to a journal that cannot be read. - ErrApplyFailed = errors.New("applying a transaction failed") ) // RecoveredTxn represents a transaction found during recovery. @@ -302,7 +299,7 @@ func scanJournal(region *Region, handler RecoveryHandler, stats *RecoveryStats, if handler != nil { if err := handler(recoveredTxn); err != nil { stats.Errors++ - return fmt.Errorf("%w: txn %d: %w", ErrApplyFailed, txnHeader.Seq, err) + return fmt.Errorf("handler failed for txn %d: %w", txnHeader.Seq, err) } } diff --git a/internal/journal/recovery_test.go b/internal/journal/recovery_test.go index 23ba647d..fe19d05e 100644 --- a/internal/journal/recovery_test.go +++ b/internal/journal/recovery_test.go @@ -487,40 +487,3 @@ func damageJournalHeader(t *testing.T, dev *storage.Device, offset int64, damage } return bytes.Clone(header) } - -// A transaction that cannot be applied yet fails with ErrApplyFailed, which a -// mount retries; the retry resumes the journal and finishes it. -func TestScanAndRecoverRetriesFailedApply(t *testing.T) { - dev, cleanup := createTestDevice(t, 10*1024*1024) - defer cleanup() - j, err := Init(dev, 0, DefaultJournalSlotSize, [16]byte{1, 2, 3, 4}, 1, DefaultConfig()) - if err != nil { - t.Fatal(err) - } - txn := j.Begin() - if err := txn.LogInodeCreate(&InodeCreatePayload{InodeNum: 100}); err != nil { - t.Fatal(err) - } - if _, err := j.Commit(txn); err != nil { - t.Fatal(err) - } - - calls := 0 - handler := func(*RecoveredTxn) error { - calls++ - if calls == 1 { - return errors.New("lock held by a live node") - } - return nil - } - if _, err := ScanAndRecover(dev, 0, DefaultJournalSlotSize, handler); !errors.Is(err, ErrApplyFailed) { - t.Fatalf("first attempt: %v, want ErrApplyFailed", err) - } - result, err := ScanAndRecover(dev, 0, DefaultJournalSlotSize, handler) - if err != nil || result == nil || !result.WasNeeded || calls != 2 { - t.Fatalf("retry = %+v, %v after %d handler calls; want the transaction replayed", result, err, calls) - } - if result, err := ScanAndRecover(dev, 0, DefaultJournalSlotSize, handler); err != nil || result != nil { - t.Errorf("after the retry the journal still needs recovery: %+v, %v", result, err) - } -} diff --git a/internal/lock/lock.go b/internal/lock/lock.go index ebd80742..74416521 100644 --- a/internal/lock/lock.go +++ b/internal/lock/lock.go @@ -456,17 +456,14 @@ func (m *Manager) Acquire(ctx context.Context, lockType LockType, resourceID uin defer m.waiting.Add(-1) } - switch by.state { - case OwnerLost: - log.Printf("[WARN] lock: gave up waiting for type=%d res=%d mode=%d after %v: the journal replay of node %x (slot %d), which holds it, failed or was abandoned", + if by.state == OwnerLost { + return m.giveUp(lockType, resourceID, mode, startTime, fmt.Sprintf( + "the journal replay of node %x (slot %d), which holds record %d, failed or was abandoned", by.occupant[:8], by.slot, recordID)) + } + if by.state == OwnerFailing && time.Since(lastFailingLog) >= acquireTimeout { + lastFailingLog = time.Now() + log.Printf("[WARN] lock: type=%d res=%d mode=%d has waited %v for node %x (slot %d), which holds it and is failing or being recovered; waiting until its journal replay ends", lockType, resourceID, mode, time.Since(startTime).Round(time.Millisecond), by.occupant[:8], by.slot) - return ErrLockTimeout - case OwnerFailing: - if time.Since(lastFailingLog) >= acquireTimeout { - lastFailingLog = time.Now() - log.Printf("[WARN] lock: type=%d res=%d mode=%d has waited %v for node %x (slot %d), which holds it and is failing or being recovered; waiting until its journal replay ends", - lockType, resourceID, mode, time.Since(startTime).Round(time.Millisecond), by.occupant[:8], by.slot) - } } // Check context cancellation, but allow at least 3 retries. A wait gives @@ -474,13 +471,11 @@ func (m *Manager) Acquire(ctx context.Context, lockType LockType, resourceID uin // while this goroutine holds other locks; a replay's wait never does. if retryCount >= 3 && (ctx.Err() != nil || !replay && liveWait > acquireTimeout) { if by.state == OwnerFailing { - log.Printf("[WARN] lock: gave up waiting for type=%d res=%d mode=%d after %v: this goroutine holds other locks, and record %d is held by node %x (slot %d), which is failing or being recovered and whose replay may need one of them", - lockType, resourceID, mode, time.Since(startTime).Round(time.Millisecond), recordID, by.occupant[:8], by.slot) - } else { - log.Printf("[WARN] lock: gave up waiting for type=%d res=%d mode=%d after %v (%d tries); record %d %s", - lockType, resourceID, mode, time.Since(startTime).Round(time.Millisecond), retryCount, recordID, m.describeHolder(recordID)) + return m.giveUp(lockType, resourceID, mode, startTime, fmt.Sprintf( + "this goroutine holds other locks, and record %d is held by node %x (slot %d), which is failing or being recovered and whose replay may need one of them", + recordID, by.occupant[:8], by.slot)) } - return ErrLockTimeout + return m.giveUp(lockType, resourceID, mode, startTime, fmt.Sprintf("record %d %s", recordID, m.describeHolder(recordID))) } // Add jitter to prevent thundering herd @@ -585,6 +580,14 @@ func (m *Manager) attemptAcquire(key lockKey, recordID uint64, mode LockMode, gi return true, true, blocker{}, nil } +// giveUp logs why an Acquire stopped waiting and returns ErrLockTimeout with +// that reason, so callers can tell who holds the lock. +func (m *Manager) giveUp(lockType LockType, resourceID uint64, mode LockMode, start time.Time, reason string) error { + log.Printf("[WARN] lock: gave up waiting for type=%d res=%d mode=%d after %v: %s", + lockType, resourceID, mode, time.Since(start).Round(time.Millisecond), reason) + return fmt.Errorf("%w: %s", ErrLockTimeout, reason) +} + func (m *Manager) reuseHeldKeyLocked(key lockKey, mode LockMode, gid uint64) bool { held, ok := m.heldLocks[key] if !ok { diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index c372cbc4..27699aa9 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -2895,12 +2895,19 @@ scenarios: environments: go: internal/node/registry_test.go::TestFailedReplayKeepsDeadNodesLocks - - id: journal-retries-failed-apply - name: Journal recovery marks a failed apply as retryable - description: A transaction that cannot be applied fails ScanAndRecover with ErrApplyFailed, and the next attempt resumes the journal and finishes it + - id: mount-own-replay-retries-lock-wait + name: Mount retries its own replay while another node holds a lock + description: A mount whose own journal replay gives up waiting for another node's lock retries with backoff and replays the transaction once the lock frees priority: go_only environments: - go: internal/journal/recovery_test.go::TestScanAndRecoverRetriesFailedApply + go: cmd/mount/recovery_test.go::TestOwnReplayRetriesWhileAnotherNodeHoldsALock + + - id: mount-own-replay-stops-at-bad-record + name: Mount stops at a journal record that does not decode + description: Any own-journal replay failure other than a lock wait fails the mount at once, with the -skip-recovery hint + priority: go_only + environments: + go: cmd/mount/recovery_test.go::TestOwnReplayStopsAtARecordThatDoesNotDecode - id: registry-occupant-states name: Registry reports how each slot's node stands From 28efadf28392df61b51707979ed645d19785e71d Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Fri, 13 Nov 2026 11:39:06 -0600 Subject: [PATCH 3/6] fix(node): refuse device writes once this node no longer owns its registry slot A node that other nodes proved dead while it was stalled kept writing when it resumed, until its next heartbeat noticed. Every device write and discard outside the registry now checks that this node wrote a heartbeat in the last 5s, or writes one while its slot is still its own. Otherwise it refuses the write and stops the node. A write that passed the check before the stall can still land; only storage fencing closes that. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + README.md | 1 + cmd/mount/main.go | 1 + docs/OPERATIONS.md | 9 ++- internal/node/registry.go | 133 +++++++++++++++++++++++--------- internal/node/registry_test.go | 92 ++++++++++++++++++++++ internal/storage/device.go | 27 ++++++- internal/storage/device_test.go | 41 ++++++++++ internal/trim/trim.go | 3 + internal/trim/trim_test.go | 20 +++++ tests/test-manifest.yaml | 35 +++++++++ 11 files changed, 323 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d525cd7..84e7c297 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,6 +85,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - The README quick start now uses the real binary names and flags. - Create, mkdir, clone and snapshot failed with "No space left on device" when the lock wait for the new inode gave up or its slot could not be read. They now fail with `EIO`. `ENOSPC` means the inode table is full. - Directory entries went only into the first 4 KB of each 1 MB directory block, so a block held about 250 short names or 15 long ones, and a busy directory took a new block every few creates. Entries now fill the whole block. A lookup reads each block in one I/O, and an add or remove writes only the 4 KB it changed. +- A node that other nodes declared dead while it was stalled, for example a paused VM, kept writing when it resumed. A node now checks before every device write that it still owns its registry slot, and refuses the write and exits if it does not. A write it started before the stall can still land. - A node stalled for 15 seconds while it held a lock, for example a paused VM, lost the lock to other nodes, which could then write the same metadata. Other nodes now wait for a failed node's locks until it is proven dead and its journal is replayed, with no time limit, and give up only if the replay fails or is abandoned, or after 30 seconds if the waiting operation already holds another lock the replay may need. Failure detection now replays only nodes proven dead. - A mount that took the registry slot of a failed node on another host did not keep that node's locks while it replayed the node's journal. - Two nodes replaying dead nodes whose locks shared a record waited on each other until the lock timeout. diff --git a/README.md b/README.md index 65465262..3132316d 100644 --- a/README.md +++ b/README.md @@ -180,6 +180,7 @@ For clustered deployments: 3. **Mount Command**: Use identical mount options across all nodes 4. **Limits**: At most 64 nodes mount a device at once, and node clocks must agree within a few seconds (run NTP) 5. **Format version**: This release uses on-disk format version 3, which gives each node its own journal and sizes the block bitmap, refcount table and inode table from the device. It refuses devices formatted by earlier builds; copy their data off and reformat +6. **Stalled nodes**: Other nodes declare a node dead after about 20 seconds of silence, for example a paused VM, and replay its journal. The stalled node checks its registry slot before each write, so when it resumes it refuses new writes and exits. A write it started before the stall can still land. Only storage fencing closes that gap, such as SCSI-3 persistent reservations, which GoClusterFS does not use. See [Distributed Lock Limits](docs/OPERATIONS.md#distributed-lock-limits) Format the device once, from one node, with `-format`. Then mount it on every node without `-format`: diff --git a/cmd/mount/main.go b/cmd/mount/main.go index 52147e28..0632c0ac 100644 --- a/cmd/mount/main.go +++ b/cmd/mount/main.go @@ -234,6 +234,7 @@ func main() { if err := nodeReg.StartHeartbeat(); err != nil { log.Fatalf("Failed to start node heartbeat: %v", err) } + dev.SetWriteGate(nodeReg.WriteGate) nodeUUID := nodeReg.LocalUUID() log.Printf("Registered as node %d (UUID: %x)", slotIdx, nodeUUID[:8]) journalOffset, journalSize := nodeReg.LocalJournal() diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 176334a1..a5185b98 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -162,7 +162,7 @@ Snapshots and clones share blocks with the file and copy a block on its first wr ### Known Gaps in Slot Ownership - Slot claims use a read-compare-write on the device, then re-read the slot after 100 ms. Two nodes can still both keep a slot if one node's write lands more than 100 ms after its read, for example during a storage stall. -- A node paused for more than 10 seconds (VM freeze, `SIGSTOP`, a stalled path) looks failed to a new mount, which can take its slot and replay and reset its journal. Running nodes wait until it is proven dead (about 20 seconds, see above). The paused node exits on its next heartbeat after it resumes, up to 1 second later, and can write to the journal region in that second. +- A node paused for more than 10 seconds (VM freeze, `SIGSTOP`, a stalled path) looks failed to a new mount, which can take its slot and replay and reset its journal. Running nodes wait until it is proven dead (about 20 seconds, see above). When the paused node resumes, it refuses further writes and exits, but writes it started before the pause can still land. - A restarted container comes back in a new PID namespace, so its mount cannot prove by PID that the previous incarnation exited. It waits up to about 12 seconds for the old slot's heartbeat to go stale, then replays that journal before serving. ### Adding a New Node @@ -238,7 +238,12 @@ Nodes coordinate metadata writes (block bitmap, inodes, directories, refcounts, **When another node's claim stops counting.** A claim counts until the registry slot with the same number is empty: its node unmounted cleanly, or another node replayed its journal and freed the slot. A node checks this by reading the registry slot from the device after it reads the record, so it cannot miss a node that registered in the meantime. A torn slot follows the same rule. How old a claim is does not matter, so a lock that a node never releases, for example after a bug, stays held until that node unmounts. The node logs a warning for each lock it has held for more than 60 seconds. A replay that fails or gets stuck keeps the dead node's locks held until a later replay succeeds; waits for them fail with `EIO` once its recovery lease lapses, about a minute after the replay last made progress. A mount whose own journal replay gives up waiting for another node's lock retries every 1 to 30 seconds and logs who holds the lock each time. Any other replay failure, such as a record that does not decode or an I/O error, fails the mount at once with the `-skip-recovery` hint. -**The hazard: a stalled node keeps writing.** If a node stops for about 20 seconds while it holds a lock, other nodes prove it dead, replay its journal and take its locks. When the stalled node resumes, it exits on its next heartbeat, up to 1 second later, but it can finish the writes it had started. No write path checks that the lock is still held, and there is no fencing token. Two nodes then write the same metadata, which can double-allocate blocks or lose directory entries. +**The hazard: a stalled node's last writes.** If a node stops for about 20 seconds while it holds a lock, other nodes prove it dead, replay its journal and take its locks. Before every device write, a node checks that it wrote its heartbeat less than 5 seconds ago. If not, it writes a heartbeat first, and if its registry slot is no longer its own, it refuses the write and exits. Some writes still land after the takeover: + +- a write that passed the check just before the stall, and I/O the kernel or the storage queued during it; +- writes in the second after the takeover, if the stalled node resumed while the other node was marking its slot dead. + +Two nodes can then write the same metadata, which can double-allocate blocks or lose directory entries. Only fencing at the storage closes this gap, for example SCSI-3 persistent reservations (or NVMe reservations), with which surviving nodes revoke a dead node's access to the device. GoClusterFS does not use them. Common causes of a stall: diff --git a/internal/node/registry.go b/internal/node/registry.go index b1fced75..305490b5 100644 --- a/internal/node/registry.go +++ b/internal/node/registry.go @@ -14,8 +14,10 @@ import ( "errors" "fmt" "log" + "math" "os" "sync" + "sync/atomic" "time" "github.com/supporttools/GoClusterFS/internal/storage" @@ -116,6 +118,13 @@ type Registry struct { seenMu sync.Mutex seen map[int]*Slot + // lastBeat is when this node last wrote its own slot; beatSeq is its last heartbeat number. + lastBeat atomic.Pointer[time.Time] + beatSeq uint64 + // lost is set once this node's slot was taken over or its heartbeats failed. + lost atomic.Bool + ownMu sync.Mutex + // Cross-node recovery recoveryHandler RecoveryHandler failureCheckInterval time.Duration // How often to scan for failed nodes @@ -130,6 +139,7 @@ var ( ErrInvalidSlot = errors.New("invalid slot data") ErrRecoveryInProgress = errors.New("recovery already in progress for this node") ErrLeaseLost = errors.New("another node took over this slot's recovery") + ErrSlotLost = errors.New("this node no longer owns its registry slot") ) // FailedNodeInfo contains information about a failed node for recovery. @@ -307,6 +317,9 @@ func (r *Registry) claimFirst(slots []int, claimable func(*Slot) bool) (int, err return -1, err } r.localSlot = i + r.beatSeq = 1 + now := time.Now() + r.lastBeat.Store(&now) if slot.State != NodeStateEmpty { r.previousOwner = &slot.UUID } @@ -428,49 +441,99 @@ func (r *Registry) heartbeatLoop(slotIdx int) { ticker := time.NewTicker(HeartbeatInterval) defer ticker.Stop() - seq := uint64(1) - lost := false - lastBeat := time.Now() - for { select { case <-r.stopCh: return case <-ticker.C: - r.mu.Lock() - slot, err := r.readSlot(slotIdx) - ours := err == nil && slot.UUID == r.localUUID && slot.State != NodeStateDead - takenOver := err == nil && !ours - if ours { - slot.HeartbeatSeq = seq - slot.HeartbeatTime = time.Now().Unix() - if r.writeSlot(slotIdx, slot) == nil { - lastBeat = time.Now() - } - seq++ - } - onSlotLost := r.onSlotLost - r.mu.Unlock() + _ = r.beat(slotIdx) + } + } +} - // Peers prove a silent node dead and take its locks; a node that cannot heartbeat must stop. - failing := time.Since(lastBeat) > HeartbeatTimeout/2 - if (takenOver || failing) && !lost { - lost = true - owner := r.localUUID - if takenOver { - owner = slot.UUID - } else { - log.Printf("ERROR: heartbeats to node registry slot %d have failed for %v", slotIdx, time.Since(lastBeat)) - } - if onSlotLost != nil { - onSlotLost(slotIdx, owner) - } else { - log.Printf("ERROR: node registry slot %d is no longer this node's (owner %x); "+ - "this node no longer owns its journal region", slotIdx, owner[:8]) - } - } +// beat writes a heartbeat while slotIdx is still this node's. Peers prove a +// silent node dead and take its locks, so a node whose slot was taken over, or +// whose heartbeats have failed for HeartbeatTimeout/2, must stop: beat tells the +// slot-lost handler once and returns ErrSlotLost. +func (r *Registry) beat(slotIdx int) error { + r.mu.Lock() + slot, err := r.readSlot(slotIdx) + ours := err == nil && slot.UUID == r.localUUID && slot.State != NodeStateDead + takenOver := err == nil && !ours + if ours { + r.beatSeq++ + slot.HeartbeatSeq = r.beatSeq + slot.HeartbeatTime = time.Now().Unix() + if r.writeSlot(slotIdx, slot) == nil { + now := time.Now() + r.lastBeat.Store(&now) + } + } + onSlotLost := r.onSlotLost + r.mu.Unlock() + + sinceBeat := r.sinceLastBeat() + if !takenOver && sinceBeat <= HeartbeatTimeout/2 { + return nil + } + if r.lost.CompareAndSwap(false, true) { + owner := r.localUUID + if takenOver { + owner = slot.UUID + } else { + log.Printf("ERROR: heartbeats to node registry slot %d have failed for %v", slotIdx, sinceBeat.Round(time.Millisecond)) } + if onSlotLost != nil { + onSlotLost(slotIdx, owner) + } else { + log.Printf("ERROR: node registry slot %d is no longer this node's (owner %x); "+ + "this node no longer owns its journal region", slotIdx, owner[:8]) + } + } + return ErrSlotLost +} + +// CheckOwnership returns nil while this node still owns its registry slot and +// ErrSlotLost once it may not, so a node that stalled past its failover stops +// writing. A heartbeat written within HeartbeatTimeout/2 proves ownership +// without I/O, because peers declare a node dead only after 2*HeartbeatTimeout +// of silence. Otherwise it writes a heartbeat first, as the heartbeat loop would. +func (r *Registry) CheckOwnership() error { + if r.lost.Load() { + return ErrSlotLost + } + if r.sinceLastBeat() < HeartbeatTimeout/2 { + return nil + } + r.ownMu.Lock() + defer r.ownMu.Unlock() + if r.sinceLastBeat() < HeartbeatTimeout/2 { + return nil + } + r.mu.RLock() + slotIdx := r.localSlot + r.mu.RUnlock() + if slotIdx < 0 { + return ErrNotRegistered + } + return r.beat(slotIdx) +} + +// WriteGate is a device write gate: it passes writes to the registry, which +// the heartbeat itself must make, and checks ownership before any other write. +func (r *Registry) WriteGate(offset int64) error { + if offset >= int64(r.offset) && offset < int64(r.offset)+MaxNodes*SlotSize { + return nil + } + return r.CheckOwnership() +} + +func (r *Registry) sinceLastBeat() time.Duration { + t := r.lastBeat.Load() + if t == nil { + return math.MaxInt64 } + return max(time.Since(*t), time.Now().Round(0).Sub(t.Round(0))) } // claimSlot replaces the slot bytes Register judged claimable with this node's diff --git a/internal/node/registry_test.go b/internal/node/registry_test.go index 27ffb9a0..ff5c4e5e 100644 --- a/internal/node/registry_test.go +++ b/internal/node/registry_test.go @@ -1,6 +1,7 @@ package node import ( + "bytes" "context" "errors" "fmt" @@ -1323,3 +1324,94 @@ func TestFailedReplayKeepsDeadNodesLocks(t *testing.T) { t.Errorf("after a failed replay slot 2 is %s with node %x; want it still dead with the failed node", slot.State, slot.UUID[:1]) } } + +func TestCheckOwnershipWritesHeartbeatAfterPause(t *testing.T) { + dev, cleanup := createTestDevice(t, 10<<20) + defer cleanup() + reg := NewRegistry(dev, 0) + slot, err := reg.Register(context.Background(), "node") + if err != nil { + t.Fatal(err) + } + if err := reg.CheckOwnership(); err != nil { + t.Fatalf("right after Register: %v", err) + } + + paused := time.Now().Add(-HeartbeatTimeout) + reg.lastBeat.Store(&paused) + before, err := reg.readSlot(slot) + if err != nil { + t.Fatal(err) + } + if err := reg.CheckOwnership(); err != nil { + t.Fatalf("slot still ours after a pause: %v", err) + } + after, err := reg.readSlot(slot) + if err != nil { + t.Fatal(err) + } + if after.HeartbeatSeq <= before.HeartbeatSeq { + t.Errorf("heartbeat %d after the check, want it past %d", after.HeartbeatSeq, before.HeartbeatSeq) + } +} + +func TestCheckOwnershipFailsOnceSlotIsTakenOver(t *testing.T) { + dev, cleanup := createTestDevice(t, 10<<20) + defer cleanup() + reg := NewRegistry(dev, 0) + slot, err := reg.Register(context.Background(), "node") + if err != nil { + t.Fatal(err) + } + var lost []string + reg.SetSlotLostHandler(func(slot int, owner [16]byte) { lost = append(lost, fmt.Sprintf("%d %x", slot, owner[:1])) }) + + if err := reg.writeSlot(slot, &Slot{Magic: SlotMagic, State: NodeStateDead, UUID: reg.LocalUUID(), RecoveryOwner: [16]byte{9}, RecoveryStart: time.Now().Unix()}); err != nil { + t.Fatal(err) + } + if err := reg.CheckOwnership(); err != nil { + t.Fatalf("with a recent heartbeat the check needs no read: %v", err) + } + paused := time.Now().Add(-HeartbeatTimeout) + reg.lastBeat.Store(&paused) + for range 2 { + if err := reg.CheckOwnership(); !errors.Is(err, ErrSlotLost) { + t.Fatalf("after peers marked the slot dead: %v, want ErrSlotLost", err) + } + } + uuid := reg.LocalUUID() + if want := []string{fmt.Sprintf("%d %x", slot, uuid[:1])}; fmt.Sprint(lost) != fmt.Sprint(want) { + t.Errorf("slot-lost handler calls %v, want %v", lost, want) + } +} + +func TestWriteGateRefusesWritesOnceSlotIsLost(t *testing.T) { + dev, cleanup := createTestDevice(t, 10<<20) + defer cleanup() + reg := NewRegistry(dev, 0) + slot, err := reg.Register(context.Background(), "node") + if err != nil { + t.Fatal(err) + } + dev.SetWriteGate(reg.WriteGate) + data := storage.NewAlignedBuffer(storage.SectorSize) + copy(data.Bytes(), bytes.Repeat([]byte{'x'}, storage.SectorSize)) + if _, err := dev.WriteAt(data.Bytes(), 1<<20); err != nil { + t.Fatalf("write while the slot is ours: %v", err) + } + + if err := reg.writeSlot(slot, &Slot{Magic: SlotMagic, State: NodeStateDead, UUID: reg.LocalUUID(), RecoveryOwner: [16]byte{9}, RecoveryStart: time.Now().Unix()}); err != nil { + t.Fatalf("a registry write must pass the gate: %v", err) + } + paused := time.Now().Add(-HeartbeatTimeout) + reg.lastBeat.Store(&paused) + if _, err := dev.WriteAt(data.Bytes(), 2<<20); !errors.Is(err, ErrSlotLost) { + t.Fatalf("write after peers marked the slot dead: %v, want ErrSlotLost", err) + } + if _, err := dev.ReadAt(data.Bytes(), 2<<20); err != nil { + t.Fatal(err) + } + if !bytes.Equal(data.Bytes(), make([]byte, storage.SectorSize)) { + t.Error("the refused write reached the device") + } +} diff --git a/internal/storage/device.go b/internal/storage/device.go index 8f1367e2..b1b4e6c8 100644 --- a/internal/storage/device.go +++ b/internal/storage/device.go @@ -11,6 +11,7 @@ import ( "io" "os" "sync" + "sync/atomic" "syscall" "unsafe" ) @@ -44,9 +45,11 @@ const ( type Device struct { ioShards [NumIOShards]sync.Mutex // Sharded locks for parallel I/O closeMu sync.RWMutex // Protects close; held shared during I/O - file *os.File - path string - size int64 + // writeGate, when set, runs before every write and can refuse it. + writeGate atomic.Pointer[func(offset int64) error] + file *os.File + path string + size int64 } // shardIndex returns the shard index for the given offset. @@ -220,6 +223,9 @@ func (d *Device) writeAt(buf []byte, offset int64, sharded bool) (int, error) { if len(buf) == 0 { return 0, nil } + if err := d.CheckWrite(offset); err != nil { + return 0, err + } // Acquire shared close lock to prevent device close during I/O d.closeMu.RLock() @@ -233,6 +239,21 @@ func (d *Device) writeAt(buf []byte, offset int64, sharded bool) (int, error) { return d.file.WriteAt(buf, offset) } +// SetWriteGate makes every write call gate with its offset first, and fail +// with gate's error instead of writing. +func (d *Device) SetWriteGate(gate func(offset int64) error) { + d.writeGate.Store(&gate) +} + +// CheckWrite runs the write gate for a write at offset. Callers that change the +// device without WriteAt, such as a discard, call it first. +func (d *Device) CheckWrite(offset int64) error { + if gate := d.writeGate.Load(); gate != nil { + return (*gate)(offset) + } + return nil +} + // Sync ensures all written data is persisted to the device. // This is a global operation that does not block concurrent I/O. func (d *Device) Sync() error { diff --git a/internal/storage/device_test.go b/internal/storage/device_test.go index b44eabc7..c7e2765b 100644 --- a/internal/storage/device_test.go +++ b/internal/storage/device_test.go @@ -1,6 +1,8 @@ package storage import ( + "bytes" + "errors" "os" "path/filepath" "sync" @@ -369,3 +371,42 @@ func TestMultiShardIO(t *testing.T) { } } } + +func TestWriteGateRefusesWrites(t *testing.T) { + tmpDir := os.Getenv("GOCLUSTERFS_TEST_DIR") + if tmpDir == "" { + tmpDir = t.TempDir() + } + path := filepath.Join(tmpDir, "test_gate.bin") + if err := os.WriteFile(path, make([]byte, 2*SectorSize), 0o644); err != nil { + t.Fatal(err) + } + defer func() { _ = os.Remove(path) }() + dev, err := Open(path) + if err != nil { + t.Skipf("O_DIRECT not supported: %v", err) + } + defer dev.Close() + + refused := errors.New("refused") + dev.SetWriteGate(func(offset int64) error { + if offset >= SectorSize { + return refused + } + return nil + }) + buf := NewAlignedBuffer(SectorSize) + copy(buf.Bytes(), bytes.Repeat([]byte{'x'}, SectorSize)) + if _, err := dev.WriteAt(buf.Bytes(), 0); err != nil { + t.Fatalf("write the gate allows: %v", err) + } + if _, err := dev.WriteAtUnsharded(buf.Bytes(), SectorSize); !errors.Is(err, refused) { + t.Fatalf("write the gate refuses = %v, want %v", err, refused) + } + if _, err := dev.ReadAt(buf.Bytes(), SectorSize); err != nil { + t.Fatal(err) + } + if !bytes.Equal(buf.Bytes(), make([]byte, SectorSize)) { + t.Error("a refused write reached the device") + } +} diff --git a/internal/trim/trim.go b/internal/trim/trim.go index 7d3cfcb7..f35bf8ef 100644 --- a/internal/trim/trim.go +++ b/internal/trim/trim.go @@ -367,6 +367,9 @@ func (m *Manager) issueDiscard(r blockRange) error { // startBlock is relative to data region, add DataOffset byteOffset := m.config.DataOffset + r.startBlock*BlockSize byteLength := r.count * BlockSize + if err := m.dev.CheckWrite(int64(byteOffset)); err != nil { + return err + } if m.config.Discard != nil { return m.config.Discard(byteOffset, byteLength) } diff --git a/internal/trim/trim_test.go b/internal/trim/trim_test.go index d1f9794b..e029f22e 100644 --- a/internal/trim/trim_test.go +++ b/internal/trim/trim_test.go @@ -445,3 +445,23 @@ func TestRefusedFlushCountsAsError(t *testing.T) { t.Errorf("Stats = %+v, want 1 error and nothing discarded", got) } } + +func TestDiscardRefusedByWriteGate(t *testing.T) { + dev, cleanup := createTestDevice(t) + defer cleanup() + dev.SetWriteGate(func(int64) error { return errors.New("node lost its registry slot") }) + + discarded := false + m := NewManager(dev, Config{BatchSize: 1000, MaxAge: time.Hour, Enabled: true, Discard: func(offset, length uint64) error { + discarded = true + return nil + }}) + m.Start() + m.Queue(5) + if err := m.Stop(); err != nil { + t.Fatal(err) + } + if discarded { + t.Error("a discard ran though the write gate refused it") + } +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 27699aa9..65bd53f2 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -2930,6 +2930,41 @@ scenarios: environments: go: internal/node/registry_test.go::TestRegisterReportsPreviousOwnerOfAnyTakenSlot + - id: storage-write-gate-refuses-writes + name: Device write gate refuses writes + description: A write the gate refuses fails with the gate's error and does not reach the device + priority: go_only + environments: + go: internal/storage/device_test.go::TestWriteGateRefusesWrites + + - id: trim-discard-refused-by-write-gate + name: Discard refused by the write gate + description: A queued discard does not run when the device's write gate refuses it + priority: go_only + environments: + go: internal/trim/trim_test.go::TestDiscardRefusedByWriteGate + + - id: registry-check-ownership-after-pause + name: Ownership check after a pause writes a heartbeat + description: When the last heartbeat is old, CheckOwnership writes a heartbeat and passes while the slot is still the node's + priority: go_only + environments: + go: internal/node/registry_test.go::TestCheckOwnershipWritesHeartbeatAfterPause + + - id: registry-check-ownership-after-takeover + name: Ownership check fails once the slot is taken over + description: Once peers marked the slot dead, CheckOwnership returns ErrSlotLost and calls the slot-lost handler once + priority: go_only + environments: + go: internal/node/registry_test.go::TestCheckOwnershipFailsOnceSlotIsTakenOver + + - id: registry-write-gate-after-takeover + name: Write gate refuses data writes once the slot is lost + description: With the registry's write gate on the device, a node whose slot was marked dead while it was paused cannot write outside the registry, and the write does not reach the device + priority: go_only + environments: + go: internal/node/registry_test.go::TestWriteGateRefusesWritesOnceSlotIsLost + - id: lock-leaked-shared-count-goes-stale name: Leaked read claim goes stale description: New readers do not refresh another node's read claim, so a read claim leaked by a stopped node stops blocking writers once it is 15 seconds old From c98681a40cabbae28cc573ae99729f61fb7c0b38 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Fri, 13 Nov 2026 13:38:22 -0600 Subject: [PATCH 4/6] ci: run checks against main Co-Authored-By: Claude Opus 5.5 From 835e0b22cd2c81e0c7e2b9a4058833e1522803e8 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Fri, 13 Nov 2026 13:49:51 -0600 Subject: [PATCH 5/6] test(storage): check the device close error in the write gate test Co-Authored-By: Claude Opus 5.5 --- internal/storage/device_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/storage/device_test.go b/internal/storage/device_test.go index c7e2765b..8bf5b57b 100644 --- a/internal/storage/device_test.go +++ b/internal/storage/device_test.go @@ -386,7 +386,7 @@ func TestWriteGateRefusesWrites(t *testing.T) { if err != nil { t.Skipf("O_DIRECT not supported: %v", err) } - defer dev.Close() + defer func() { _ = dev.Close() }() refused := errors.New("refused") dev.SetWriteGate(func(offset int64) error { From cede96967f95fc0fc4e27f271a5c45722bea4b23 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Fri, 13 Nov 2026 14:02:30 -0600 Subject: [PATCH 6/6] fix(node): log when the write gate checks the slot after a missed heartbeat A write that finds the last heartbeat over 5s old writes a heartbeat first. It now logs that, so a node whose heartbeats fall behind under load shows up before it ever loses its slot. Co-Authored-By: Claude Opus 5.5 --- internal/node/registry.go | 2 ++ internal/node/registry_test.go | 11 ++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/internal/node/registry.go b/internal/node/registry.go index 305490b5..4a56ce06 100644 --- a/internal/node/registry.go +++ b/internal/node/registry.go @@ -516,6 +516,8 @@ func (r *Registry) CheckOwnership() error { if slotIdx < 0 { return ErrNotRegistered } + log.Printf("[WARN] node: the last heartbeat to registry slot %d was %v ago; checking the slot before a write", + slotIdx, r.sinceLastBeat().Round(time.Millisecond)) return r.beat(slotIdx) } diff --git a/internal/node/registry_test.go b/internal/node/registry_test.go index 24f34ba3..08879381 100644 --- a/internal/node/registry_test.go +++ b/internal/node/registry_test.go @@ -5,9 +5,11 @@ import ( "context" "errors" "fmt" + "log" "os" "os/exec" "path/filepath" + "strings" "testing" "time" @@ -1343,9 +1345,16 @@ func TestCheckOwnershipWritesHeartbeatAfterPause(t *testing.T) { if err != nil { t.Fatal(err) } - if err := reg.CheckOwnership(); err != nil { + var logs bytes.Buffer + log.SetOutput(&logs) + err = reg.CheckOwnership() + log.SetOutput(os.Stderr) + if err != nil { t.Fatalf("slot still ours after a pause: %v", err) } + if !strings.Contains(logs.String(), "checking the slot before a write") { + t.Errorf("the check after a pause logged nothing:\n%s", logs.String()) + } after, err := reg.readSlot(slot) if err != nil { t.Fatal(err)